From 1de2d7adeacfd1b065d9df091997d44c43120ade Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Sat, 24 Oct 2026 13:46:57 +0200 Subject: [PATCH 001/271] fix(ai): treat Kimi K3.1 Pro models as Kimi K3 family Kimi Coding, Moonshot, and OpenCode Go generated K3.1 Pro and K3.1 Pro Highspeed with Kimi K2 settings because K3 detection matched exact IDs. Detect the K3 family by ID pattern, and Kimi Coding K3-style models by their effort options. Add implied subscription costs and clamp gateway output limits like K3. --- packages/ai/CHANGELOG.md | 1 + packages/ai/scripts/generate-models.ts | 48 ++++++++++++++++---------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 9ea77c63b..920d227e7 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -22,6 +22,7 @@ - Fixed streamed tool-call arguments being re-parsed in full on every delta, which made CPU time and memory grow quadratically with the size of a tool call (a 90 KB `write` on Anthropic peaked at about 140 MB of heap; a 500 KB one took minutes). Every provider and the pi-messages client now read each delta once, and keep the raw argument text and their other stream state (`partialJson`, `partialArgs`, `index`, `customInput`, `streamIndex`, Bedrock's encrypted reasoning bytes) off the content blocks, so it no longer reaches partial messages or transcripts of failed streams ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed grammar (custom) tool input streaming copying the whole input so far on every delta ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed bounded error texts (telemetry span messages, truncated provider error bodies, pi-messages diagnostics, Radius gateway errors) keeping the whole text they were cut from alive ([#9265](https://github.com/earendil-works/pi/issues/9265)) +- Fixed Kimi K3.1 Pro and K3.1 Pro Highspeed on Kimi Coding, Moonshot, and OpenCode Go using Kimi K2 request settings. They now send `reasoning_effort` (low/high/max), return earlier reasoning, accept tools added mid-conversation, and report Moonshot rates on Kimi Coding. Kimi Coding `k3-256k` now reports Kimi K3 rates. ## [1.4.0] - 2026-10-21 diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index f5efb94bf..ee01fbe1f 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -349,7 +349,11 @@ const KIMI_K3_OPENAI_COMPAT: OpenAICompletionsCompat = { thinkingFormat: "openai", supportsReasoningEffort: true, }; -const OPENROUTER_KIMI_K3_MODEL_IDS = new Set(["moonshotai/kimi-k3", "~moonshotai/kimi-latest"]); +// Kimi K3 and its point releases (kimi-k3.1-pro, kimi-k3.1-pro-highspeed) share K3's request +// format. https://api.moonshot.ai/v1/models reports effort levels and dynamic tool loading for all. +function isKimiK3FamilyId(modelId: string): boolean { + return /^kimi-k3(?:\.\d+)?(?:-|$)/.test(modelId); +} const ANT_LING_RING_THINKING_LEVEL_MAP = { off: null, @@ -928,9 +932,9 @@ function applyOpenAIToolSearchMetadata(model: Model): void { function applyOpenAICompletionsTranscriptMetadata(model: Model): void { if (model.api !== "openai-completions") return; const isKimiK3 = - (model.provider.startsWith("moonshot") && model.id === "kimi-k3") || + (model.provider.startsWith("moonshot") && isKimiK3FamilyId(model.id)) || (model.provider === "fireworks" && model.id.includes("kimi-k3")) || - ((model.provider === "opencode" || model.provider === "opencode-go") && model.id === "kimi-k3"); + ((model.provider === "opencode" || model.provider === "opencode-go") && isKimiK3FamilyId(model.id)); const isMoonshotKimiK2 = model.provider.startsWith("moonshot") && (model.id === "kimi-k2.6" || model.id === "kimi-k2.7-code" || model.id === "kimi-k2.7-code-highspeed"); @@ -1763,13 +1767,12 @@ function processFireworksModels(provider: ModelsDevProvider | undefined): Model< // equivalent Moonshot API rates to estimate the value of subscription usage. const KIMI_CODING_IMPLIED_COSTS: Record["cost"]> = { k3: KIMI_K3_COST, + "k3-256k": KIMI_K3_COST, + "k3.1-pro": { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 0 }, + "k3.1-pro-highspeed": { input: 4, output: 20, cacheRead: 0.2, cacheWrite: 0 }, "kimi-for-coding": { input: 0.95, output: 4, cacheRead: 0.19, cacheWrite: 0 }, "kimi-for-coding-highspeed": { input: 1.9, output: 8, cacheRead: 0.38, cacheWrite: 0 }, }; -// Models whose https://api.kimi.com/coding/v1/models entry reports effort levels and -// dynamic tool loading (tools declared in later system messages). Highspeed reports -// neither and rejects tool-bearing system messages with a 400. -const KIMI_CODING_K3_STYLE_MODEL_IDS = new Set(["k3", "k3-256k", "kimi-for-coding"]); // Kimi Coding uses Chat Completions, the same protocol as the official Kimi Code client. function createKimiCodingModels(kimiModels: Record): Model<"openai-completions">[] { @@ -1777,7 +1780,11 @@ function createKimiCodingModels(kimiModels: Record): Mod for (const [modelId, m] of Object.entries(kimiModels)) { if (m.tool_call !== true) continue; - const isK3Style = KIMI_CODING_K3_STYLE_MODEL_IDS.has(modelId); + // K3-style models report effort levels and dynamic tool loading (tools declared in + // later system messages) in https://api.kimi.com/coding/v1/models; models.dev mirrors + // the effort levels. kimi-for-coding-highspeed reports neither and rejects + // tool-bearing system messages with a 400. + const isK3Style = m.reasoning_options?.some((option) => option.type === "effort") === true; const impliedCost = KIMI_CODING_IMPLIED_COSTS[modelId]; // K3-style models send reasoning_effort, so expose the effort values models.dev lists. const effortLevels = isK3Style && m.reasoning_options ? getEffortThinkingLevelMap(m.reasoning_options) : undefined; @@ -2440,7 +2447,7 @@ async function loadModelsDevData(): Promise[]> { for (const [modelId, m] of Object.entries(moonshotModels[key])) { if (m.tool_call !== true) continue; - const isKimiK3 = modelId === "kimi-k3"; + const isKimiK3 = isKimiK3FamilyId(modelId); const compat = { ...(isKimiK3 ? KIMI_K3_OPENAI_COMPAT : KIMI_OPENAI_COMPAT) }; models.push({ id: modelId, @@ -2451,14 +2458,15 @@ async function loadModelsDevData(): Promise[]> { reasoning: isKimiK3 || m.reasoning === true, input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], // Moonshot does not bill cache writes; models.dev lists the input rate as cache_write for Kimi K3. - cost: isKimiK3 - ? { ...KIMI_K3_COST } - : { - input: m.cost?.input || 0, - output: m.cost?.output || 0, - cacheRead: m.cost?.cache_read || 0, - cacheWrite: m.cost?.cache_write || 0, - }, + cost: + modelId === "kimi-k3" + ? { ...KIMI_K3_COST } + : { + input: m.cost?.input || 0, + output: m.cost?.output || 0, + cacheRead: m.cost?.cache_read || 0, + cacheWrite: isKimiK3 ? 0 : m.cost?.cache_write || 0, + }, contextWindow: m.limit?.context || 4096, maxTokens: m.limit?.output || 4096, compat, @@ -2906,8 +2914,10 @@ async function generateModels() { } // Keep Kimi K3's canonical output limit when gateway metadata is missing or incorrect. if ( - (candidate.provider === "openrouter" && OPENROUTER_KIMI_K3_MODEL_IDS.has(candidate.id)) || - (candidate.provider === "vercel-ai-gateway" && candidate.id === "moonshotai/kimi-k3") + (candidate.provider === "openrouter" && candidate.id === "~moonshotai/kimi-latest") || + ((candidate.provider === "openrouter" || candidate.provider === "vercel-ai-gateway") && + candidate.id.startsWith("moonshotai/") && + isKimiK3FamilyId(candidate.id.slice("moonshotai/".length))) ) { candidate.maxTokens = KIMI_K3_MAX_TOKENS; } From f321e5bdd26b3624bfb2ab182a334d0a5edb698a Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 24 Oct 2026 12:17:37 +0000 Subject: [PATCH 002/271] docs(chord): delta README, growing strings through drafts and appendText --- packages/chord/src/delta/README.md | 38 ++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/packages/chord/src/delta/README.md b/packages/chord/src/delta/README.md index ecc6cefd6..0db4a8103 100644 --- a/packages/chord/src/delta/README.md +++ b/packages/chord/src/delta/README.md @@ -210,9 +210,47 @@ and restores `Op` tuples. Use one encoder/decoder pair per ordered state stream. Path IDs span batches and an `r` resets both dictionaries. After a decode or apply error, discard the decoder and replica and recover from a later `r`. +## Growing strings: use `appendText` + +The footgun: appending to a string through a draft costs the string's whole length on every change, not the length of +what you append. + +```ts +const change = tracker.beginChange(); +change.state.message.text += chunk; // looks like O(chunk) +const prepared = change.prepare(); // emits ["a", ["message", "text"], chunk], but costs O(text.length) +``` + +The draft's proxy only sees the finished new string, never the chunk, so `prepare()` recovers the append by checking +that the new string starts with the old one. That check reads the whole string, and V8 first copies a string built by +`+=` (a rope) into one flat piece. There is no fast path: measured on Node 24 and 26, 20,000 changes appending 40 +characters each, up to 800 KB, take about 2.4 s with the check and about 1 ms without it. A stream of N chunks into one +string therefore costs O(N x length) in total: quadratic. + +The fix: hand Delta the chunk. + +```ts +import { appendText } from "@earendil-works/chord/delta"; + +appendText(change.state.message, "text", chunk); // records ["a", path, chunk]: O(chunk), no compare, no copy +``` + +- `appendText(parent, key, text)` takes the parent draft and a key whose type is exactly `string` (not optional, not a + string literal type), so the compiler refuses `appendText(block, "type", …)`. +- Several appends to one key in one change become one op. A plain assignment or deletion of the same key in that change + falls back to the compare, as does an append after a plain write. +- The appended text is copied into a string of its own, so neither the op nor the new value keeps a larger string + alive. +- Use it wherever a string grows by appends across many changes (streamed answers, tool output, logs). A one-off + assignment is fine with `=`. +- Derives cannot use it: `diffRevisions()` compares two revisions it did not write, so a derive producing a growing + string pays O(length) per delivery. + ## Limits and footguns - No freezing. Mutating any immutable value corrupts state silently. +- Appending to a string through a draft (`draft.text += chunk`) costs O(string length) per change; use `appendText` + (section "Growing strings"). - In-process sharing: local handles, cell subscribers, and `replicatedState` subscribers receive the provider's containers. A consumer mutation corrupts authority and makes remote replicas diverge. - A change inside a large flat array copies that array's pointer storage for the new revision. - Placement validation costs time and memory on bulk inserts. In a benchmark unshifting 100k three-field objects, it added about 19 ms and 20 MiB of transient heap. From d0aa218cb16c333c008f6aa3dcd670ac8b68383c Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 24 Oct 2026 16:56:53 +0000 Subject: [PATCH 003/271] perf(chord): cheaper container copies in Delta's tracker and applyImmutable Tracker revisions, applyImmutable() and applyImmutableBatches() copy each container on a changed path with spread (Object.assign() onto a null-prototype object) instead of one Object.defineProperty() per key: same own data properties, prototype and own __proto__ member, about 25 times faster per container. The copy-count test counts key listings through a proxy instead of spying on Object.keys. Adds a tracker GC regression test: 10,000 adopted changes retain only the document, not a context registration per change. Follow-up to #9265. --- packages/chord/CHANGELOG.md | 4 ++ .../src/delta/apply-immutable-trusted.ts | 9 ++-- packages/chord/src/delta/index.ts | 16 ++----- .../chord/test/delta-apply-immutable.test.ts | 43 ++++++++++++++----- .../test/delta-tracker/retention.test.ts | 1 + .../test/delta-tracker/retention.worker.ts | 29 +++++++++++++ 6 files changed, 72 insertions(+), 30 deletions(-) diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index bbd4759bc..e8b74b5e9 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -17,6 +17,10 @@ - `ownedSlice(text, start, end?)` in `@earendil-works/chord/delta`: a slice of a string that does not keep the whole string alive ([#9265](https://github.com/earendil-works/pi/issues/9265)) - `appendText(draft, key, text)` in `@earendil-works/chord/delta`: append to a string in a draft at O(text) cost, recorded as an append op without comparing the old and new strings (a plain `draft[key] += text` makes `prepare()` read the whole string, O(length) per change); `StringKeys` names the keys it accepts ([#9265](https://github.com/earendil-works/pi/issues/9265)) +### Changed + +- `applyImmutable()`, `applyImmutableBatches()` and tracker revisions copy each container on a changed path with spread (or `Object.assign()` onto a null-prototype object) instead of one `Object.defineProperty()` per key, about 25 times faster per container ([#9265](https://github.com/earendil-works/pi/issues/9265)) + ### Fixed - Fixed append ops from the delta tracker and `diffRevisions()` keeping the whole string they were cut from alive (V8 represents a slice as a view of its source), so a delta chain, a watch backlog or a replica built from appends to a growing text held one copy of the text per commit ([#9265](https://github.com/earendil-works/pi/issues/9265)) diff --git a/packages/chord/src/delta/apply-immutable-trusted.ts b/packages/chord/src/delta/apply-immutable-trusted.ts index 7944bfae1..cde0300e7 100644 --- a/packages/chord/src/delta/apply-immutable-trusted.ts +++ b/packages/chord/src/delta/apply-immutable-trusted.ts @@ -86,12 +86,9 @@ function copyPath(root: JsonValue, path: Path, length: number, owned: WeakSet; - for (const key of Object.keys(value)) defineData(result, key, value[key]!); - return result; + // Spread and `Object.assign()` onto a null-prototype object define own data properties, `__proto__` included, as + // `defineData()` per key does, at a fraction of its cost. + return Object.getPrototypeOf(value) === null ? Object.assign(Object.create(null), value) : { ...value }; } function read(target: JsonContainer, key: Seg): JsonValue { diff --git a/packages/chord/src/delta/index.ts b/packages/chord/src/delta/index.ts index f2ec7ec53..1af25be29 100644 --- a/packages/chord/src/delta/index.ts +++ b/packages/chord/src/delta/index.ts @@ -490,19 +490,9 @@ function copyContainers(root: JsonValue, path: Path, owned: WeakSet): Js const copy = (value: JsonValue): JsonValue[] | Record => { if (Array.isArray(value)) return value.slice(); if (!isObj(value)) throw new PathError(path); - const result = Object.create(Object.getPrototypeOf(value) === null ? null : Object.prototype) as Record< - string, - JsonValue - >; - for (const key of Object.keys(value)) { - Object.defineProperty(result, key, { - value: (value as Record)[key], - writable: true, - enumerable: true, - configurable: true, - }); - } - return result; + // Spread and `Object.assign()` onto a null-prototype object define own data properties, `__proto__` included, + // as `Object.defineProperty()` per key does, at a fraction of its cost. + return Object.getPrototypeOf(value) === null ? Object.assign(Object.create(null), value) : { ...value }; }; if (!isObj(root)) throw new PathError(path); let copiedRoot: JsonValue[] | Record; diff --git a/packages/chord/test/delta-apply-immutable.test.ts b/packages/chord/test/delta-apply-immutable.test.ts index ff6f1de0d..d00ef0e10 100644 --- a/packages/chord/test/delta-apply-immutable.test.ts +++ b/packages/chord/test/delta-apply-immutable.test.ts @@ -246,6 +246,30 @@ describe("checked immutable operation application", () => { }); }); +describe("immutable container copies", () => { + it("keep an own __proto__ member as data and the prototype of the copied object", () => { + const plain = JSON.parse('{"__proto__":{"polluted":true},"n":1}') as Record; + const bare = Object.assign(Object.create(null) as Record, { n: 1 }); + for (const base of [plain, bare]) { + const checked = applyImmutable(base, [["s", ["n"], 2]]); + // Tracker revisions copy through the trusted path. + const tracker = track(base); + const change = tracker.beginChange(); + change.state.n = 2; + tracker.adopt(change.prepare()); + for (const copy of [checked, tracker.value]) { + expect(copy).not.toBe(base); + expect(Object.getPrototypeOf(copy)).toBe(Object.getPrototypeOf(base)); + expect(Object.keys(copy)).toEqual(Object.keys(base)); + expect(copy.n).toBe(2); + if (base === plain) + expect(Object.getOwnPropertyDescriptor(copy, "__proto__")?.value).toEqual({ polluted: true }); + } + } + expect(({} as { polluted?: boolean }).polluted).toBeUndefined(); + }); +}); + describe("immutable application complexity", () => { it("appends to an array not yet copied in one copy, sharing everything else", () => { const entries = Array.from({ length: 1_000 }, (_, index) => ({ index })); @@ -291,18 +315,15 @@ describe("immutable application complexity", () => { const base: Record = {}; for (let index = 0; index < width; index++) base[`field${index}`] = index; const operations: Op[] = Array.from({ length: 1_000 }, (_, index): Op => ["s", [`field${index}`], -index]); - const originalKeys = Object.keys; + // A copy lists the object's keys; the proxy counts the listings. let shallowCopies = 0; - Object.keys = ((value: object): string[] => { - shallowCopies++; - return originalKeys(value); - }) as typeof Object.keys; - let result: Record; - try { - result = applyImmutable(base, operations); - } finally { - Object.keys = originalKeys; - } + const counted = new Proxy(base, { + ownKeys(target) { + shallowCopies++; + return Reflect.ownKeys(target); + }, + }); + const result = applyImmutable(counted, operations); expect(result.field999).toBe(-999); expect(base.field999).toBe(999); diff --git a/packages/chord/test/delta-tracker/retention.test.ts b/packages/chord/test/delta-tracker/retention.test.ts index 29543857d..fe39b0dd0 100644 --- a/packages/chord/test/delta-tracker/retention.test.ts +++ b/packages/chord/test/delta-tracker/retention.test.ts @@ -18,6 +18,7 @@ describe("delta tracker retention across GC jobs", () => { "same-job-fast-cleanup", "same-job-folded-ops-cleanup", "lifecycle-churn", + "adopted-changes-flat", ])( "validates %s retention semantics", (scenario) => { diff --git a/packages/chord/test/delta-tracker/retention.worker.ts b/packages/chord/test/delta-tracker/retention.worker.ts index a0a80d8d4..23e9efa85 100644 --- a/packages/chord/test/delta-tracker/retention.worker.ts +++ b/packages/chord/test/delta-tracker/retention.worker.ts @@ -260,6 +260,35 @@ if (scenario === "settled-lifecycle") { const retainedMiB = (process.memoryUsage().heapUsed - readyHeap) / (1024 * 1024); assert.ok(retainedMiB < 8, `same-job folded operations retained ${retainedMiB.toFixed(2)} MiB`); assert.equal(tracker.value.field0, 100); +} else if (scenario === "adopted-changes-flat") { + // Each change registers its context with the tracker through a WeakRef; adoption must drop every registration, so + // a long run of small changes keeps only the document, not a record per change. + const gc = global.gc; + assert.ok(gc, "worker requires --expose-gc"); + const tracker = track({ entries: [{ id: 0, text: "" }] }); + const commit = (): void => { + const change = tracker.beginChange(); + change.state.entries[0]!.text += " tok"; + tracker.adopt(change.prepare()); + }; + const settledHeap = async (): Promise => { + for (let round = 0; round < 5; round++) { + await setImmediate(); + gc(); + } + return process.memoryUsage().heapUsed; + }; + for (let index = 0; index < 1_000; index++) commit(); + const readyHeap = await settledHeap(); + for (let index = 0; index < 10_000; index++) { + commit(); + // Let WeakRef targets go, as between the jobs of real commits. + if (index % 100 === 0) await setImmediate(); + } + // The text itself grows 4 bytes per change. + const perChange = ((await settledHeap()) - readyHeap) / 10_000 - 4; + assert.ok(perChange < 48, `each adopted change retained ${perChange.toFixed(1)} bytes`); + assert.equal(tracker.value.entries[0]!.text.length, 4 * 11_000); } else if (scenario === "lifecycle-churn") { const tracker = track({ value: 0 }); for (let index = 0; index < 100_000; index++) { From 6c2fd73b0c545a4034672cb60e34d53e035dd751 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 24 Oct 2026 17:49:30 +0000 Subject: [PATCH 004/271] feat(durable): deferred storage of throttled commits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A commit may be marked deferrable ({ persistence: { kind: "deferred", withinMs? } }) on Session, Conversation, TaskRuntime and a tool's api.commit. When every write changes an existing conversation document with history "latest" and fork "initial", it is adopted in memory and shown to watches at once, and stored with the next commit that reads or writes the document, before the document leaves memory, at close, or within withinMs (default settings.progress.persistWithinMs, 1000 ms). The generation partial throttle and the tool output throttle use it. subscribeCommits() gets these commits only once stored, with exactly what was stored; internal observers get memory-only publications. Spec §1/§13 changed, new §4.1. 32 streams over SQLite: storage transactions 562/s -> 99/s, disk 14.3 -> 4.9 MB/s, CPU 29% -> 22% of one core. A crash loses up to about 1 s of streamed content. --- packages/durable/CHANGELOG.md | 2 + packages/durable/README.md | 18 +- packages/durable/docs/spec.md | 172 ++++- packages/durable/src/harness/agent.ts | 4 +- packages/durable/src/harness/events.ts | 4 +- packages/durable/src/harness/generation.ts | 32 +- packages/durable/src/harness/harness.ts | 16 +- packages/durable/src/harness/json.ts | 12 +- packages/durable/src/harness/live.ts | 18 +- .../durable/src/harness/partial-changes.ts | 52 +- packages/durable/src/harness/scheduler.ts | 13 +- packages/durable/src/harness/submissions.ts | 16 +- packages/durable/src/harness/task-graph.ts | 17 +- packages/durable/src/harness/tool.ts | 57 +- packages/durable/src/harness/types.ts | 19 +- packages/durable/src/harness/view.ts | 10 +- packages/durable/src/index.ts | 2 + packages/durable/src/session/session.ts | 295 +++++++- packages/durable/src/session/transaction.ts | 192 ++++- packages/durable/src/types.ts | 35 +- .../test/harness-deferred-progress.test.ts | 444 ++++++++++++ packages/durable/test/harness-events.test.ts | 10 +- .../durable/test/harness-generation.test.ts | 11 +- .../durable/test/harness-live-deltas.test.ts | 6 +- .../test/harness-partial-changes.test.ts | 12 +- .../durable/test/harness-registry.test.ts | 1 + .../durable/test/harness-structured.test.ts | 3 + .../test/harness-tools-recovery.test.ts | 2 +- .../test/session-deferred-commits.test.ts | 653 ++++++++++++++++++ packages/durable/test/session-support.ts | 22 +- 30 files changed, 1944 insertions(+), 206 deletions(-) create mode 100644 packages/durable/test/harness-deferred-progress.test.ts create mode 100644 packages/durable/test/session-deferred-commits.test.ts diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 41abda519..eff3b87a9 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -9,9 +9,11 @@ - Watches use Chord's `WatchHandle` and `WatchEnd` (Durable re-exports both). End reasons are `stopped`, `cancelled`, `retired`, `ended` (was `session_closed`), `listener_error`, and `failed` (was `session_failed`). Retirement is an explicit event of the watched document incarnation (last frame `[["r", null]]`, then `retired`), never inferred from a `null` value. An ended watch drops its listener and every reference into its Session, so a stopped watch that is still referenced keeps only its last value. - Document, conversation view, and task graph watches deliver the commits made while their listener runs (or before `start()`) in one callback: the newest value, the commits' operations folded (appends to one string join, sets of one field keep the last) and the newest commit's Context. Past 10,000 pending operations the callback gets one root replacement `[["r", newest]]`. Before, each commit was its own callback and more than 100 pending commits became a root replacement. `watchEvents()` still delivers every event batch in order, with one `snapshot` past 100 pending batches. - Removed the Chord state getters `Session.documentState()`, `Conversation.viewState()`, and `Harness.taskGraph()`, and the `DocumentState` type; use `watchDoc()`, `Conversation.watch()`, and `Harness.watchTaskGraph()`. Durable no longer uses Chord's `replicatedState()`. +- Partial answers and running tool output are stored later than they are shown: their throttled commits are deferred (`CommitPersistence`, spec §4.1), so watches, views, and agent events see each at once, while Storage gets them together with the next commit that reads `pi.live` (the end of the answer, a tool settling, a run ending) and at the latest after `settings.progress.persistWithinMs` (default 1,000 ms). With 32 streams over SQLite this cuts storage transactions about 5.7x and disk writes about 2.9x for turns with parallel tools, 27x and 9x for long answers, and CPU by about a quarter. A crash now loses up to about a second of streamed content instead of 100 ms; a stream that crashes before its first store leaves no aborted entry, and that attempt's usage is not recorded. `subscribeCommits()` gets these commits only once they are stored, with exactly what was stored (held operations first), and gets a publication for a deadline's store too; it no longer sees every partial. Migration: a `subscribeCommits()` listener that showed live progress should use `watchDoc()`, `Conversation.watch()`, or `watchEvents()`; one that mirrors Storage keeps working; set `settings.progress.persistWithinMs: 0` to store progress right after each commit, about as before (one storage transaction per partial). ### Added +- Deferred commits: `Session.commit()`, `Conversation.commit()`, `TaskRuntime.commit()`, and a tool's `api.commit()` take `{ persistence: { kind: "deferred", withinMs? } }`. A commit that changes only existing conversation documents with `history: "latest"` and `fork: "initial"` is then adopted in memory and shown to watches at once, and stored with the next commit that reads or writes the document, before the document leaves memory, at close, or within `withinMs` (default `settings.progress.persistWithinMs`, or `createSession(storage, { persistWithinMs })`); any other write makes it immediate. A crash loses what is held. See spec §4.1. - `HarnessOptions.readOnly` opens a Harness as a viewer of a session another process drives: open leaves running tasks as committed instead of recovering them, every commit with writes rejects with `ReadOnlySession` before it reaches Storage, and `resume()` and every call that would run tasks reject with it. ### Changed diff --git a/packages/durable/README.md b/packages/durable/README.md index 257faa8d1..452b7c740 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -413,7 +413,7 @@ const harness = await Harness.open(storage, { stream: { timeoutMs: 120_000 }, retry: { maxRetries: 3 }, compaction: { reserveTokens: 16384 }, - progress: { partialIntervalMs: 100, outputIntervalMs: 100 }, + progress: { partialIntervalMs: 100, outputIntervalMs: 100, persistWithinMs: 1_000 }, toolExecution: "parallel", contextRetentionMs: 600_000, // idle conversations keep their context in memory this long (checked lazily where background timers are declined, as in Workers) documentRetentionMs: 600_000, // loaded documents nothing observes stay in memory this long after their last use; observed ones always @@ -492,7 +492,7 @@ Watches are Chord's `WatchHandle` (exported by `@earendil-works/chord`; Durable Commits that land while a callback runs are delivered together in the next callback: the newest view and the operations of all of them, folded (appends to one string join, sets of one field keep the last). Past 10,000 pending operations the callback gets one root replacement `[["r", view]]` holding the whole newest view instead. A client that joins late or reconnects starts from the current view; nothing is replayed. -Partial answers and tool output are committed at most every 100 ms by default, so a crash loses at most that window. `settings.progress` changes the intervals; a host whose storage is remote can commit less often, for example `{ partialIntervalMs: 500, outputIntervalMs: 500 }`. +Partial answers and tool output are committed at most every 100 ms by default, and watches see each commit at once. Storage gets them later and together, as deferred commits (see [Your Own State](#your-own-state)): with the next commit that reads `pi.live`, such as the end of the answer or of a tool, and at the latest after `settings.progress.persistWithinMs` (default 1,000 ms). A crash loses at most about that window of an answer or of tool output; a stream that crashes in its first second may leave no aborted entry, and that attempt's usage is not recorded. `settings.progress` changes the intervals; a host whose storage is remote can commit less often, for example `{ partialIntervalMs: 500, outputIntervalMs: 500 }`. ## Busy Conversations @@ -739,6 +739,20 @@ await root.commit(async (tx) => { console.log(await harness.snapshot(Todos, root.id, context)); ``` +A commit whose changes are cheap to lose, such as your own streamed progress, can be shown at once and stored later: + +```typescript +await root.commit( + async (tx) => { + (await tx.doc(Todos, root.id)).items.push("draft"); // a document like Todos: latest, fork "initial" + }, + context, + { persistence: { kind: "deferred", withinMs: 1_000 } }, // without withinMs: settings.progress.persistWithinMs +); +``` + +Watches and the view see it at once. Storage gets it with the next commit that reads or writes the document, before the document leaves memory, at close, and at the latest about `withinMs` later; `subscribeCommits()` sees it then, with exactly what was stored. A crash loses what was not stored yet, so derive nothing durable from such a value outside a commit that reads it. Only a commit that changes nothing but existing documents with `history: "latest"` and `fork: "initial"` is deferred; anything else in it, such as an entry, a new document, or a task state, stores it at once. + `harness.watchDoc()` watches one document like the view above. `HarnessOptions.conversationCreated(tx, conversation)` runs in every commit that creates or forks a conversation, including a tool's raw `tx.createConversation()`, so every conversation gets your documents; `init` in `createConversation()`, `fork()`, and `root()` writes per-call data in the same commit. An extension's tools, sections, and hooks read their own documents through `api` or `input.read`, and treat an absent one as its default ([`10-extension-state.ts`](test/examples/10-extension-state.ts)). ## Usage and Cost diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index d63ba63e5..1b95481b1 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -41,7 +41,8 @@ type once the PR lands. The core rule is: > A Session atomically commits immutable entries, full task records, and -> Chord-tracked documents. Only committed state is observable. +> Chord-tracked documents. Only committed state is observable. A commit marked +> deferred may be observed before it is stored (section 4.1). ## 1. Terms and invariants @@ -63,8 +64,16 @@ The core rule is: Required invariants: 1. One Session commit is atomic across all record and document writes. -2. A document update is published only after its storage commit succeeds. -3. All visible progress is durable. There is no volatile publication path. +2. A document update is published only after its storage commit succeeds, + except a deferred commit's (section 4.1): it is published to watches and the + conversation view when memory adopts it, and to `subscribeCommits()` once it + is stored. +3. All visible progress is durable, except what a deferred commit holds: what + watches see can be ahead of Storage by up to that commit's window + (`withinMs`), and a crash loses it. A commit is deferred only when its caller + marks it so, and only when it changes nothing but existing latest, + initial-fork conversation documents (section 4.1). Every stored document is + a state memory had. 4. External effects do not run inside the Session mutation transaction. 5. Entries and IDs are immutable and never reused after a committed write. 6. Document drafts are fully revoked when their transaction callback settles: @@ -649,6 +658,8 @@ interface Conversation { commit( change: (tx: Tx) => T | Promise, context: Context, + /** As for `Session.commit()` (section 4.1). */ + options?: CommitOptions, ): Promise; /** With `at`, the context as of that visible entry, as `fork(at)` would start with. */ context(context: Context, options?: { readonly at?: EntryId }): Promise; @@ -1286,8 +1297,14 @@ There is no mutable `session.document()` API. ```ts type SessionEnd = { readonly reason: "closed" } | { readonly reason: "failed"; readonly error: unknown }; +type CommitPersistence = + | { readonly kind: "immediate" } + | { readonly kind: "deferred"; readonly withinMs?: number }; +type CommitOptions = { readonly persistence?: CommitPersistence }; + interface Session extends DocumentObserver { - commit(change: (tx: Tx) => T | Promise, context: Context): Promise; + /** `options.persistence` may defer storing the commit (section 4.1). */ + commit(change: (tx: Tx) => T | Promise, context: Context, options?: CommitOptions): Promise; close(context: Context): Promise; /** Settles once closed, Storage included: `{ reason: "closed" }`, or `{ reason: "failed", error }` (section 10.1). */ readonly closed: Promise; @@ -1482,6 +1499,12 @@ callback succeeds with no pending acquisition prepare every open change -> immutable next revision + self-contained Chord Op[] Session evaluates each required/ordinary document write exactly once Storage.commit persists the atomic batch while the Session line remains held +commit deferred and holdable (section 4.1) + adopt every prepared change by pointer swap; queue its ops per document, unstored + watches and view mounts capture it synchronously; subscribeCommits() does not + release the line; invoke listeners later +otherwise, right before Storage admission + take the held ops of every document the commit acquired and put them in front of its own storage succeeds adopt every prepared change by pointer swap and enqueue immutable revision/ops publication commit listeners, including conversation view mounts (section 9.3), capture it synchronously @@ -1552,8 +1575,11 @@ The Session evaluates this predicate exactly once after tracker preparation. base, excluding the change being evaluated. Storage reports it when it materializes the current value; the Session advances it after each adopted write and resets it after every base, so evaluation performs no Storage read. -Creation and version transitions require bases and do not call it. The Session -then gives Storage only the selected representation: +Creation and version transitions require bases and do not call it. A deferred +commit's predicate also runs once, when it is held, against the stored count; a +base it selects is stored with the held changes, as a store whose held +operations overflowed is (section 4.1). The Session then gives Storage only the +selected representation: ```text required or predicate true -> base with complete value @@ -1698,14 +1724,14 @@ line through callback execution, preparation, storage settlement, committed baseline adoption, and publication enqueue. Adoption runs inside `Storage.commit()`: the Session passes an `adopt(seq)` callback, which the storage calls synchronously once the batch is durable and before its next -commit (section 10), so memory always shows a complete prefix of the log. The -callback keeps its own error: a failed adoption fails the Session once +commit (section 10), so memory always shows a complete prefix of the log, plus +what deferred commits hold (section 4.1). The callback keeps its own error: a failed adoption fails the Session once `Storage.commit()` returns. A commit that rejects after its adoption fails the Session too, and its adopted changes stay adopted. Commit observers run on the line; watch listeners run later. External model, process, tool, network, and human effects run outside it. -`subscribeCommits()` observes complete immutable publications synchronously on -the line after adoption. `subscribeClose()` observes close synchronously when it +`subscribeCommits()` observes stored commits synchronously on the line after +adoption, each with exactly what that storage transaction stored (section 4.1). `subscribeClose()` observes close synchronously when it begins, after admission is sealed, also when a failure closes the Session; the Harness ends waits, watches, and streams there, and signals task invocations after every close listener has run, so what a task holds ends with the Session's @@ -1747,6 +1773,84 @@ Storage ownership: - Every storage read returns a detached JSON value. - Immutable strings may be shared; mutable arrays and objects may not. +### 4.1 Deferred commits + +A commit whose writes are only cheap to lose, such as a streamed answer or +running tool output, may be stored later than it is shown: + +```ts +await runtime.commit( + async (tx) => { (await tx.doc(LiveDoc, conversationId)).generation!.message = partial; }, + context, + { persistence: { kind: "deferred", withinMs: 1_000 } }, +); +``` + +`Session.commit()`, `Conversation.commit()`, `TaskRuntime.commit()`, and a +tool's `api.commit()` take the option; without it, or +with `{ kind: "immediate" }`, a commit is stored before it resolves. `withinMs` +is a number of milliseconds, `0` or more (`Infinity` waits for one of the other +triggers below); anything else rejects with a `TypeError`. Without it, a +Harness uses `settings.progress.persistWithinMs` (default 1,000) and a plain +Session its `persistWithinMs` option (default 1,000). + +The mark is a permission. A deferred commit is **held** only when, after its +callback, settlement, and the read-only check, every write it makes changes an +existing conversation document with `history: "latest"` and `fork: "initial"` +at its stored version. Anything else, such as an entry, a task state, a +submission, another document, a document's creation, migration, or retirement, +makes it immediate, a normal commit. A held commit is adopted in memory by +pointer swap, resolves, and is published to the Session's own observers: document +watches, the conversation view and agent events (section 9). Its stored +version and delta count stay; its operations wait, folded per incarnation +(Chord's `PendingOps`), and the incarnation is pinned in the cache, so idle +eviction skips it at any retention. + +What is held is stored by the first of: + +1. a normal commit that acquires the document (`tx.doc()`), whether it changes + it or only reads it, and stores anything: the held operations go in front of + its own in the one content write of that incarnation, in its storage + transaction. A stored delta of a held document therefore always begins with + its held operations; a base the commit writes covers them; +2. the document's deadline, the earliest `withinMs` deadline of its held + commits on the Session clock, kept by one foreground timer (section 5.6); + when it fires, every held document is stored in one storage transaction: + a deadline is the latest time, and the others cost little more there; +3. `close()`, after the writes at close and before the trackers are dropped; + a failure of that store fails the Session, and the backend still closes; +4. any drop of a tracker that holds them: `unloadDocuments()`, and a token of + another definition version, which reloads the document from Storage; + retirement is case 1. + +Held operations leave their queue only at the last step before +`Storage.commit()`, after every fallible step: a callback, settlement, or +checkpoint failure of a normal commit keeps them. A store fails like any +storage write: it fails the Session (section 10.1). Past Chord's +`MAX_PENDING_OPS` (10,000) held operations, or when a held commit's +`checkpointWhen` selected a base, the store writes a base of the current value. +A failed Session stores nothing; after a reopen nothing is held. + +Publication: the Session's own observers see each held commit once, when memory +adopts it, and of a normal commit that stores held changes only its own +operations. `subscribeCommits()` sees every storage transaction with exactly +what it stored: nothing for a held commit, and the stored operations, held +first, for a commit or a deadline that stores them. A consumer that applies +`subscribeCommits()` publications to stored state, such as a replica, stays +equal to Storage. Its `seq` values have gaps where nothing was published. + +Consistency: every stored document is a state it had in memory, and a normal +commit that reads a held document through `tx` stores it with whatever it +derives. What a crash loses is a held commit's changes: up to `withinMs` plus the +line and I/O delay. A value read outside a commit (`snapshot()`, a watch) may be +ahead of Storage; nothing durable may be derived from it without reading it in +a commit. Other readers of Storage, such as another process, a replica, or an +export, see held changes once they are stored. + +The Harness defers exactly two kinds of commits: generation partials (section +8.3) and running tool progress (section 7.3). Every commit that consumes or +clears them reads `pi.live` and so stores them first (section 8.2). + ## 5. Tasks ### 5.1 Definition @@ -1853,6 +1957,8 @@ interface TaskRuntime extends DocumentObserver, Docum current: RunningTask, ) => NextTaskState | undefined | Promise | undefined>, context: Context, + /** May defer storing a commit that changes only documents (section 4.1); one that returns a state is stored. */ + options?: CommitOptions, ): Promise; memo(name: string, context: Context): Promise; @@ -1937,7 +2043,8 @@ Harness is closing, the task is terminal, or a run invocation's task carries an abort mark. Its `tx.createTask()` defaults to the task's conversation, and every entry it appends records the task as `byTaskId`. When the callback returns a state, the runtime replaces the task's state in the same -commit, so the checkpoint or outcome is atomic with the callback's entries, +commit (stored at once even when `options` defers it, section 4.1), so the +checkpoint or outcome is atomic with the callback's entries, documents, and child tasks and is type-checked against the task's checkpoint and result types. Returning nothing leaves the state unchanged. A terminal or completing state drops the memos. A `waiting` or `sleeping` state ends the @@ -2448,7 +2555,8 @@ commits and task changes. The default declines them in Cloudflare Workers, since a pending timer keeps a Durable Object from hibernating or being evicted even when `nodejs_compat` gives it an `unref()`, and wherever timers cannot be unreferenced. Foreground timers pace or resume work: throttled -progress commits through `TaskRuntime.setTimer()`, and, without `onWake`, the +progress commits through `TaskRuntime.setTimer()`, the deadline of deferred +commits (section 4.1), and, without `onWake`, the timer for the earliest sleep while scheduling runs. That one keeps a Node.js process alive, since a sleeping task is work left to do. A timer should not fire early and may fire late; every caller checks its clock again when it fires. The @@ -3087,7 +3195,8 @@ interface ToolExecutionApi extends Docum retainedOutput(): { readonly text: string; readonly truncated: boolean }; diagnostic(diagnostic: ToolDiagnostic): void; details(value: TDetails, context: Context): Promise; - commit(change: (tx: Tx) => T | Promise, context: Context): Promise; + /** A commit of the call's task; `options` may defer storing it (section 4.1). */ + commit(change: (tx: Tx) => T | Promise, context: Context, options?: CommitOptions): Promise; memo(name: string, context: Context): Promise; memo(name: string, candidate: T, context: Context): Promise; createTask( @@ -3424,7 +3533,8 @@ canonical path before opening it does not prevent a concurrent rename or symlink swap, so callers that restrict paths do so for hygiene, not security. The `details()` promise resolves after the corresponding or coalesced document -commit. During normal settlement the tool task stops its throttle and awaits the +commit, which memory adopts at once and Storage gets with the next commit that +reads `pi.live` or within `settings.progress.persistWithinMs` (section 4.1). During normal settlement the tool task stops its throttle and awaits the commit in flight; the terminal commit, which appends the result entry, is the final flush and settles any `details()` promise still pending. Abort and close obey invocation and Session admission gates: uncommitted buffered updates may be @@ -3964,8 +4074,12 @@ continuation calls `models.fetchDeferred()` and `models.cancelDeferred()` with that same model and signal. Missing models and synchronous/streamed pi-ai errors are classified into the durable generation outcomes below. -Generation and tool progress are throttled durable document commits. A crash may -lose only the uncommitted throttle window. Recovery converts committed partials +Generation and tool progress are throttled document commits, deferred +(section 4.1): watches see each at once, and Storage gets them with the next +commit that reads `pi.live` or within `settings.progress.persistWithinMs` +(default 1,000 ms). A crash may lose the throttle window plus what was not +stored yet: a stream that crashes before its first store leaves no aborted entry, +and that attempt's usage is not recorded. Recovery converts stored partials to normal interrupted/aborted transcript entries, clears presentation state, and then retries or terminates according to the task phase. Retry deadlines, attempts, compaction, and tool progress are current document state for late @@ -4108,7 +4222,8 @@ each commit writes at most one window, and the throttle bounds the rate. Generation creates `tools` with one `pending` slot per call in the commit that appends the tool-calling answer; a call its request did not offer starts `done` with its result entry. A tool task sets its slot `running` in its intent commit, -publishes throttled output and details into it, and in its terminal commit sets +publishes throttled output and details into it with deferred commits (section +4.1), and in its terminal commit sets `done` and `entry` and removes `output`, `droppedBytes`, `droppedLines`, `details`, and `diagnostics`, which the result entry now carries (section 7.3). The generation's `tools` phase removes `tools`. Slot updates apply only while a slot with the task's @@ -4137,6 +4252,17 @@ and resolves the transaction's latest candidate submission record, falling back to committed state, during assembly, like task-document validation (section 3.3), so it is not a caller table read and works after the commit's first table write. +The streamed fields, `generation.message` and a running slot's `output`, +`droppedBytes`, `droppedLines`, `details`, and `diagnostics`, are written by +deferred commits (section 4.1). Every commit that derives anything from them or +clears them acquires `pi.live`: converting a partial into an aborted entry, +settling a tool from its slot, handing a generation over to its tool round, +ending a run, and clearing a safe rerun's progress. It therefore stores them +first, with its own changes, and after a crash Storage holds a `pi.live` its +memory had: control fields as of the last normal commit, streamed fields as of +the last store. Recovery's own conversions and clears are normal commits, so a +second crash never converts the same partial twice. + ### 8.3 Generation ```ts @@ -4233,7 +4359,9 @@ The run's inputs live in `pi.live.run`, not in the task input. model context through `cutoff` with the invocation signal, the thinking level as `reasoning` (omitted for `off`), the conversation's persisted provider `sessionId`, and the pinned `streamOptions`, committing throttled partials at - most every `settings.progress.partialIntervalMs` (default 100 ms). + most every `settings.progress.partialIntervalMs` (default 100 ms), deferred + (section 4.1), so Storage gets them at the next commit that reads `pi.live` + or within `settings.progress.persistWithinMs`. Before streaming, the `beforeRequest` chain may replace the messages for this request only. Recovery resends the same committed messages with the same pinned model, thinking level, and stream @@ -4766,8 +4894,8 @@ The Harness keeps at most one mount per conversation. The first `watch()` or event attachment builds its revision on the Session line from the committed active transcript and documents; the mount is dropped when its last observer detaches. A mount derives each next revision from the Session's -`subscribeCommits()` publications, like `watchDoc()`: every publication is -already durable, so the view shows only committed state. One publication that +commits as memory adopts them, like `watchDoc()`: it shows committed state, +including deferred commits before they are stored (section 4.1). One publication that touches the conversation yields one Chord batch: ```text @@ -5587,7 +5715,9 @@ These are contracts, not invitations to add defensive machinery: Durable has no: - whole-Session DOM; -- visible-undurable publication; +- visible-undurable publication beyond deferred commits (section 4.1): an + unmarked commit, and every commit that writes anything but existing latest, + initial-fork conversation documents, is stored before anything observes it; - Session-kernel semantic event journal or independently maintained event state; - session-scoped rewindable documents; - automatic checkpoint heuristic; diff --git a/packages/durable/src/harness/agent.ts b/packages/durable/src/harness/agent.ts index 64a15bb6a..7706d945b 100644 --- a/packages/durable/src/harness/agent.ts +++ b/packages/durable/src/harness/agent.ts @@ -1,6 +1,6 @@ import { copyJson, type Draft } from "@earendil-works/chord"; import { defineDoc } from "../documents.ts"; -import { documentRetention } from "../session/session.ts"; +import { documentRetention, persistWithin } from "../session/session.ts"; import type { ConversationId, ConversationRecord, Tx } from "../types.ts"; import type { Agent, @@ -34,6 +34,7 @@ export const DEFAULT_COMPACTION_POLICY: CompactionPolicy = { export const DEFAULT_PROGRESS_POLICY: ProgressPolicy = { partialIntervalMs: 100, outputIntervalMs: 100, + persistWithinMs: 1_000, }; /** The reserved section key of the agent's `instructions`. */ @@ -73,6 +74,7 @@ export function resolveSettings(settings: HarnessSettings | undefined): Settings progress: { partialIntervalMs: settings?.progress?.partialIntervalMs ?? DEFAULT_PROGRESS_POLICY.partialIntervalMs, outputIntervalMs: settings?.progress?.outputIntervalMs ?? DEFAULT_PROGRESS_POLICY.outputIntervalMs, + persistWithinMs: persistWithin(settings?.progress?.persistWithinMs), }, toolExecution: settings?.toolExecution ?? "parallel", steeringMode: settings?.steeringMode ?? "one-at-a-time", diff --git a/packages/durable/src/harness/events.ts b/packages/durable/src/harness/events.ts index 5d5d638ce..ae3e4de9d 100644 --- a/packages/durable/src/harness/events.ts +++ b/packages/durable/src/harness/events.ts @@ -1,9 +1,9 @@ import { type Context, type JsonValue, NativeWatch, type WatchEnd } from "@earendil-works/chord"; import type { Op, Path } from "@earendil-works/chord/delta"; import type { AssistantMessage, Message, Usage } from "@earendil-works/pi-ai"; +import type { ObservedCommit } from "../session/session.ts"; import type { CommitChange, - CommitPublication, ConversationId, EntryRecord, JsonObject, @@ -228,7 +228,7 @@ function translate( before: ConversationView, after: ConversationView, viewOps: readonly Op[], - publication: CommitPublication, + publication: ObservedCommit, held: Set, ): AgentEvent[] { const entries: EntryRecord[] = []; diff --git a/packages/durable/src/harness/generation.ts b/packages/durable/src/harness/generation.ts index d617713bd..087ec9a97 100644 --- a/packages/durable/src/harness/generation.ts +++ b/packages/durable/src/harness/generation.ts @@ -1,4 +1,4 @@ -import { type Context, copyJson, type Draft, type JsonValue } from "@earendil-works/chord"; +import { type Context, copyJson, type Draft } from "@earendil-works/chord"; import type { Api, AssistantMessage, @@ -31,8 +31,8 @@ import type { import { addTools, sectionExtension } from "./agent.ts"; import { compactionPolicy, createCompaction, estimateContext, selectCut } from "./compaction.ts"; import { applyBoundary, type Boundary, prepareBoundary, withdrawQueuedInputs } from "./inbox.ts"; -import { endRun, LiveDoc, type LiveState, type ToolSlot } from "./live.ts"; -import { PartialChanges } from "./partial-changes.ts"; +import { endRun, LiveDoc, type LiveState, PROGRESS_COMMIT, type ToolSlot } from "./live.ts"; +import { PartialChanges, type PartialMessage } from "./partial-changes.ts"; import { planSystemEntries, renderSections, replaySections } from "./prompt.ts"; import { ensureProviderSessionId } from "./provider.ts"; import { appendToolResult, harnessError, ToolTask, type ToolTaskResult } from "./tool.ts"; @@ -440,9 +440,10 @@ export async function convertPartial(tx: Tx, live: Draft, conversatio /** * Stream one request and return the terminal message. Partials commit as trailing writes at most every - * `progress.partialIntervalMs` (default 100 ms) with one commit in flight; `finally` stops the throttle and awaits that - * commit, so no stale partial lands after the outcome. A commit appends what text and thinking grew by and applies - * the argument changes since the last one (`PartialChanges`), so its cost follows the new text, not the strings' length. + * `progress.partialIntervalMs` (default 100 ms) with one commit in flight, deferred (`PROGRESS_COMMIT`); `finally` stops + * the throttle and awaits that commit, so no stale partial lands after the outcome. A commit appends what text and + * thinking grew by and applies the argument changes since the last one (`PartialChanges`), so its cost follows the new + * text, not the strings' length. */ async function streamResponse( runtime: Runtime, @@ -465,15 +466,20 @@ async function streamResponse( if (partial === undefined || stopped) return; inFlight = (async () => { // Copy synchronously: the provider keeps mutating its partial. The changes since the last commit belong to it. - const message = copyJson(partial, { omitUndefinedProperties: true }) as { content: JsonValue[] }; + // `copyJson()` returns untyped JSON; a copy of a message is that message's JSON representation. + const message = copyJson(partial, { omitUndefinedProperties: true }) as PartialMessage; const taken = changes.take(message); try { - await runtime.commit(async (tx) => { - const live = await tx.doc(LiveDoc, runtime.conversationId); - live.generation ??= { attempt }; - changes.commit(live.generation as Draft>, message, taken); - return undefined; - }, context); + await runtime.commit( + async (tx) => { + const live = await tx.doc(LiveDoc, runtime.conversationId); + live.generation ??= { attempt }; + changes.commit(live.generation, message, taken); + return undefined; + }, + context, + PROGRESS_COMMIT, + ); } catch (error) { changes.restore(taken); throw error; diff --git a/packages/durable/src/harness/harness.ts b/packages/durable/src/harness/harness.ts index d22f368cc..7ce8de9c8 100644 --- a/packages/durable/src/harness/harness.ts +++ b/packages/durable/src/harness/harness.ts @@ -20,6 +20,7 @@ import { contained, SessionImpl } from "../session/session.ts"; import type { Transaction } from "../session/transaction.ts"; import { hostTimers } from "../timers.ts"; import type { + CommitOptions, ConversationId, ConversationOwnership, ConversationRecord, @@ -138,8 +139,8 @@ class ConversationImpl implements Conversation { await this.#host.submissions.submit(this.id, { type: "write", entry }, context); } - commit(change: (tx: Tx) => T | Promise, context: Context): Promise { - return this.#host.harness.commitWith(change, context, { conversationId: this.id }); + commit(change: (tx: Tx) => T | Promise, context: Context, options?: CommitOptions): Promise { + return this.#host.harness.commitWith(change, context, { conversationId: this.id }, options); } context(context: Context, options?: { readonly at?: EntryId }): Promise { @@ -213,8 +214,15 @@ class HarnessImpl extends SessionImpl implements const report = safeReport(options.onReport, options.telemetry ?? NOOP_TELEMETRY_CONTEXT); const now = safeNow(options.now, report); const hostSettings = currentSettings(options.settings); - // Read at each use, as all settings are, but only this field: it is read on every commit. - super(rawStorage, now, () => hostSettings()?.documentRetentionMs, options.setTimer, options.readOnly === true); + // Read at each use, as all settings are, but only these fields: they are read on every commit. + super( + rawStorage, + now, + () => hostSettings()?.documentRetentionMs, + options.setTimer, + options.readOnly === true, + () => hostSettings()?.progress?.persistWithinMs, + ); // Every component reads and writes through the Session's failure guard: `rawStorage` serves only `super()`. const storage = this.storage; this.#options = options; diff --git a/packages/durable/src/harness/json.ts b/packages/durable/src/harness/json.ts index c96119fe8..0922fe45c 100644 --- a/packages/durable/src/harness/json.ts +++ b/packages/durable/src/harness/json.ts @@ -1,8 +1,6 @@ import type { JsonValue } from "@earendil-works/chord"; import { appendText, type Draft } from "@earendil-works/chord/delta"; -type JsonContainer = Record | JsonValue[]; - /** How `assignJson()` writes one value instead of assigning it leaf by leaf. */ export type AssignPlan = /** The string grew by `text` since it was last assigned: append it, when the lengths agree. */ @@ -19,10 +17,16 @@ export type AssignPlan = * string member grew by, and the committed string plus that text has the new string's length, the text is appended * with `appendText()` instead. */ -export function assignJson(target: JsonContainer, key: string | number, value: JsonValue, plan?: AssignPlan): void { +export function assignJson( + target: T, + key: K, + value: T[K] & JsonValue, + plan?: AssignPlan, +): void { if (plan?.kind === "keep") return; + // Every target is a document container: JSON members or items, whatever its declared type names them. const slots = target as Record; - const current = slots[key]; + const current: JsonValue | undefined = slots[key]; const fields = plan?.kind === "fields" ? plan.fields : undefined; if (isRecord(current) && isRecord(value)) { for (const name of Object.keys(current)) if (!Object.hasOwn(value, name)) delete current[name]; diff --git a/packages/durable/src/harness/live.ts b/packages/durable/src/harness/live.ts index 4d5397f71..55759479f 100644 --- a/packages/durable/src/harness/live.ts +++ b/packages/durable/src/harness/live.ts @@ -2,7 +2,16 @@ import type { Draft, JsonRepresentation, JsonValue } from "@earendil-works/chord import type { AssistantMessage, Usage } from "@earendil-works/pi-ai"; import { defineDoc } from "../documents.ts"; import type { Transaction } from "../session/transaction.ts"; -import type { EntryId, JsonObject, SubmissionId, SubmissionSettlement, TaskId, TaskRecord, Tx } from "../types.ts"; +import type { + CommitOptions, + EntryId, + JsonObject, + SubmissionId, + SubmissionSettlement, + TaskId, + TaskRecord, + Tx, +} from "../types.ts"; import { convertPartial } from "./generation.ts"; import type { SchedulerOutcome } from "./scheduler.ts"; import type { CompactionReason, CompactionResult, ToolDiagnostic } from "./types.ts"; @@ -126,6 +135,13 @@ export const LiveDoc = defineDoc({ !(value.nestedTools ?? []).some((slot) => slot.status === "running"), }); +/** + * How the progress throttles commit a partial answer and running tool output into `pi.live`: deferred, so storage gets + * them together, at the next commit that reads or writes `pi.live` or after `settings.progress.persistWithinMs`. Every + * commit that consumes or clears streamed fields reads `pi.live`, so it stores them with its own changes (spec §8.2). + */ +export const PROGRESS_COMMIT: CommitOptions = { persistence: { kind: "deferred" } }; + /** Built-in task kinds that can own `pi.live.run`. */ const RUN_TASK_KINDS: ReadonlySet = new Set(["pi.generation"]); const TOOL_TASK_KIND = "pi.tool"; diff --git a/packages/durable/src/harness/partial-changes.ts b/packages/durable/src/harness/partial-changes.ts index 355a9c93c..bf1a8ab04 100644 --- a/packages/durable/src/harness/partial-changes.ts +++ b/packages/durable/src/harness/partial-changes.ts @@ -1,18 +1,25 @@ // What a streamed answer changed in its partial message between two partial commits, so a commit costs the new text. -import type { JsonValue } from "@earendil-works/chord"; +import type { JsonRepresentation, JsonValue } from "@earendil-works/chord"; import { appendText, type Draft } from "@earendil-works/chord/delta"; -import type { AssistantMessageEvent } from "@earendil-works/pi-ai"; +import type { AssistantMessage, AssistantMessageEvent, ToolCall } from "@earendil-works/pi-ai"; import { type JsonChange, StreamingJsonParser } from "@earendil-works/pi-ai/utils/streaming-json"; import { type AssignPlan, assignJson } from "./json.ts"; +import type { LiveState } from "./live.ts"; type JsonContainer = Record | JsonValue[]; +/** A partial message as `pi.live.generation.message` holds it. */ +export type PartialMessage = JsonRepresentation; + +/** Committed arguments of a streamed tool call. */ +type Arguments = JsonRepresentation["arguments"]; + /** What one partial commit writes besides the partial message itself; from `PartialChanges.take()`. */ export type PartialCommit = { /** The plan `assignJson()` writes the message with: text appends, and the arguments it leaves to `changes`. */ readonly plan: AssignPlan; /** Each tracked tool call by content index: its argument changes since the last commit, in order, and its snapshot. */ - readonly toolCalls: ReadonlyMap; + readonly toolCalls: ReadonlyMap; }; /** What a tracked tool call changed since the last commit, and whether any commit took its changes yet. */ @@ -80,16 +87,18 @@ export class PartialChanges { * Take what changed since the last call, for a commit of `message`, a copy of the partial. Clears the arguments of * tracked tool calls in `message`: the commit writes them from the changes. Call `restore()` if the commit fails. */ - take(message: { readonly content: readonly JsonValue[] }): PartialCommit { + take(message: PartialMessage): PartialCommit { const blocks = new Map(); for (const [index, { field, text }] of this.#text) { blocks.set(index, { kind: "fields", fields: new Map([[field, { kind: "append", text }]]) }); } this.#text = new Map(); - const toolCalls = new Map(); + const toolCalls = new Map(); for (const [index, call] of this.#toolCalls) { - const block = message.content[index] as Record; - toolCalls.set(index, { changes: call.changes, snapshot: block.arguments! }); + const block = message.content[index]; + // A tracked call is the tool call its `toolcall_start` named; the narrowing only tells TypeScript. + if (block?.type !== "toolCall") continue; + toolCalls.set(index, { changes: call.changes, snapshot: block.arguments }); block.arguments = {}; blocks.set(index, { kind: "fields", fields: new Map([["arguments", { kind: "keep" }]]) }); call.changes = []; @@ -106,25 +115,32 @@ export class PartialChanges { * its first piece; when the committed message lost a call whose changes were already committed (another writer reset * it), the call falls back to its snapshot from now on. */ - commit(generation: Draft>, message: JsonValue, commit: PartialCommit): void { - const committedBlocks = (generation.message as { content?: unknown[] } | undefined)?.content?.length ?? 0; + commit( + generation: Draft>, + message: PartialMessage, + commit: PartialCommit, + ): void { + const committedBlocks = generation.message?.content.length ?? 0; const apply: [number, readonly JsonChange[]][] = []; for (const [index, { changes, snapshot }] of commit.toolCalls) { const call = this.#toolCalls.get(index); if (call === undefined) continue; - if (index >= committedBlocks && call.committed) { + const block = message.content[index]; + if (index >= committedBlocks && call.committed && block?.type === "toolCall") { // The earlier changes are not in the committed message: commit the snapshot and stop tracking the call. this.#toolCalls.delete(index); - (message as { content: Record[] }).content[index]!.arguments = snapshot; + block.arguments = snapshot; continue; } call.committed = true; apply.push([index, changes]); } assignJson(generation, "message", message, commit.plan); - const content = (generation.message as unknown as { content: Draft>[] }).content; + const content = generation.message?.content ?? []; for (const [index, changes] of apply) { - for (const change of changes) applyChange(content[index]!, change); + const block = content[index]; + if (block?.type !== "toolCall") continue; + for (const change of changes) applyChange(block, change); } } @@ -138,13 +154,15 @@ export class PartialChanges { } /** Apply one argument change to the committed arguments of `block`. */ -function applyChange(block: Draft>, change: JsonChange): void { +function applyChange(block: Draft>, change: JsonChange): void { const key = change.path.at(-1); if (key === undefined) { - // Only an object root is tracked, and only `set` starts it. - if (change.kind === "set") block.arguments = value(change); + // Only an object root is tracked (`observe()` drops any other), and only `set` starts it. + const root = change.kind === "set" ? value(change) : undefined; + if (isRecord(root)) block.arguments = root; return; } + // The path follows the parsed argument text, not a type: each segment names a container `observe()` saw set. let parent = block.arguments as Draft; for (const segment of change.path.slice(0, -1)) { parent = (parent as Record>)[segment]!; @@ -161,6 +179,6 @@ function value(change: Extract): JsonValue { return change.value as JsonValue; } -function isRecord(value: JsonValue | undefined): boolean { +function isRecord(value: JsonValue | undefined): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } diff --git a/packages/durable/src/harness/scheduler.ts b/packages/durable/src/harness/scheduler.ts index 8fde30b63..9a7fcf2e2 100644 --- a/packages/durable/src/harness/scheduler.ts +++ b/packages/durable/src/harness/scheduler.ts @@ -11,11 +11,11 @@ import { } from "@earendil-works/pi-telemetry"; import type { ExecutionEnv } from "../env/index.ts"; import { errorMessage, ReadOnlySession } from "../errors.ts"; -import type { SessionImpl } from "../session/session.ts"; +import type { ObservedCommit, SessionImpl } from "../session/session.ts"; import type { Transaction } from "../session/transaction.ts"; import type { Timers } from "../timers.ts"; import type { - CommitPublication, + CommitOptions, ConversationId, DocumentWatch, EntryId, @@ -635,7 +635,9 @@ export class TaskScheduler { // ─── Scheduling ──────────────────────────────────────────────────────── - #observe(publication: CommitPublication): void { + #observe(publication: ObservedCommit): void { + // A commit held in memory changes documents only. + if (publication.kind === "memory") return; const updated: AnyTaskRecord[] = []; const failed: TaskId[] = []; let changed = false; @@ -2145,7 +2147,7 @@ export class TaskScheduler { get registry() { return phase.snapshot(); }, - commit: async (change, context) => { + commit: async (change, context, options) => { let checkpoint: string | undefined; await this.#gated( invocation, @@ -2157,6 +2159,7 @@ export class TaskScheduler { } }, context, + options, ); if (checkpoint !== undefined) { invocation.context.telemetry.addEvent("pi.durable.checkpoint", { @@ -2343,6 +2346,7 @@ export class TaskScheduler { invocation: Invocation, change: (tx: Transaction, current: ErasedRunningTask) => T | Promise, context: Context, + options?: CommitOptions, ): Promise { if (invocation.ended) return Promise.reject(endedError(invocation)); return this.#session.commitWith( @@ -2360,6 +2364,7 @@ export class TaskScheduler { }, context, { conversationId: invocation.conversationId, taskId: invocation.taskId }, + options, ); } diff --git a/packages/durable/src/harness/submissions.ts b/packages/durable/src/harness/submissions.ts index 0cf838a93..8f44e0793 100644 --- a/packages/durable/src/harness/submissions.ts +++ b/packages/durable/src/harness/submissions.ts @@ -7,16 +7,8 @@ import { type TelemetrySpan, } from "@earendil-works/pi-telemetry"; import { ConversationBusy } from "../errors.ts"; -import type { SessionImpl } from "../session/session.ts"; -import type { - CommitPublication, - ConversationId, - JsonObject, - Storage, - SubmissionId, - SubmissionRecord, - Tx, -} from "../types.ts"; +import type { ObservedCommit, SessionImpl } from "../session/session.ts"; +import type { ConversationId, JsonObject, Storage, SubmissionId, SubmissionRecord, Tx } from "../types.ts"; import { startRun } from "./generation.ts"; import { appendInput, @@ -142,7 +134,9 @@ export class Submissions { }, context); } - #observe(publication: CommitPublication): void { + #observe(publication: ObservedCommit): void { + // A commit held in memory changes documents only. + if (publication.kind === "memory") return; for (const change of publication.changes) { if (change.type !== "submission" || !isSettled(change.value)) continue; this.#waiters.resolve(change.value.id, change.value); diff --git a/packages/durable/src/harness/task-graph.ts b/packages/durable/src/harness/task-graph.ts index ec4e55a23..55ee659ed 100644 --- a/packages/durable/src/harness/task-graph.ts +++ b/packages/durable/src/harness/task-graph.ts @@ -1,16 +1,8 @@ import { type Context, type JsonValue, Watch, type WatchHandle } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT, withoutAbortSignal } from "@earendil-works/chord/context"; import { applyImmutable, type Op } from "@earendil-works/chord/delta"; -import { contained, type SessionImpl } from "../session/session.ts"; -import type { - CommitPublication, - ConversationId, - JoinPolicy, - Storage, - TaskId, - TaskOutcome, - TaskRecord, -} from "../types.ts"; +import { contained, type ObservedCommit, type SessionImpl } from "../session/session.ts"; +import type { ConversationId, JoinPolicy, Storage, TaskId, TaskOutcome, TaskRecord } from "../types.ts"; import { type BlockedReason, type BlockedSubject, blockedSubject } from "./scheduler.ts"; import { closedError, scanAll } from "./util.ts"; @@ -88,7 +80,8 @@ export class TaskGraphView { this.#storage = storage; this.#blocking = blocking; session.observeCommits((publication, context) => { - if (this.#mount === undefined) return; + // A commit held in memory changes documents only, which the graph does not show. + if (this.#mount === undefined || publication.kind === "memory") return; advance(this.#mount, publication, context, blocking.blocked, (error) => session.report(error)); }); blocking.subscribeBlocked(() => this.#refreshBlocked()); @@ -190,7 +183,7 @@ export class TaskGraphView { */ function advance( mount: Mount, - publication: CommitPublication, + publication: ObservedCommit, context: Context, blocked: TaskGraphBlocking["blocked"], report: (error: unknown) => void, diff --git a/packages/durable/src/harness/tool.ts b/packages/durable/src/harness/tool.ts index 215e8f063..483d3b4fe 100644 --- a/packages/durable/src/harness/tool.ts +++ b/packages/durable/src/harness/tool.ts @@ -38,6 +38,7 @@ import { LiveDoc, type NestedToolSlot, type NestedToolSummary, + PROGRESS_COMMIT, removeNestedSlots, type ToolSlot, toolSlot, @@ -358,12 +359,16 @@ async function run( committed.catch(() => {}); return awaitWithContext(committed, detailsContext); }, - commit: async (change, commitContext) => { + commit: async (change, commitContext, options) => { let result: Awaited> | undefined; - await runtime.commit(async (tx) => { - result = await change(tx); - return undefined; - }, commitContext); + await runtime.commit( + async (tx) => { + result = await change(tx); + return undefined; + }, + commitContext, + options, + ); return result as Awaited>; }, memo: runtime.memo, @@ -478,7 +483,7 @@ function progressClock(runtime: Runtime): ProgressClock { /** * Throttled commits of what the tool reported into its slot, in `pi.live.tools` or `pi.live.nestedTools`, each writing - * only what changed since the last one; none when `enabled` is false. + * only what changed since the last one, deferred (`PROGRESS_COMMIT`); none when `enabled` is false. */ function publishProgress(runtime: Runtime, reported: Reported, enabled: boolean, context: Context): Progress { let written = { text: "", details: undefined as JsonValue | undefined, diagnostics: 0 }; @@ -501,24 +506,28 @@ function publishProgress(runtime: Runtime, reported: Reported, enabled: boolean, } if (detailsChanged) bytes += utf8ByteLength(JSON.stringify(current.details ?? null)); if (added.length > 0) bytes += utf8ByteLength(JSON.stringify(added)); - await runtime.commit(async (tx) => { - const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId); - if (slot === undefined) return undefined; - // REMINDER: assign `output` as one string field. Chord then diffs it into an append, or a trim plus an - // append for a sliding tail; replacing the slot object would record the whole window on every commit. - if ((slot.output ?? "") !== snapshot.text) slot.output = snapshot.text; - if (snapshot.droppedBytes > 0) slot.droppedBytes = snapshot.droppedBytes; - if (snapshot.droppedLines > 0) slot.droppedLines = snapshot.droppedLines; - // Diff details leaf by leaf and append new diagnostics, so each commit writes only what changed. - if (detailsChanged && current.details !== undefined) { - assignJson(slot as unknown as Record, "details", current.details); - } - if (added.length > 0) { - if (slot.diagnostics === undefined) slot.diagnostics = []; - for (const diagnostic of added) slot.diagnostics.push(diagnostic); - } - return undefined; - }, context); + await runtime.commit( + async (tx) => { + const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId); + if (slot === undefined) return undefined; + // REMINDER: assign `output` as one string field. Chord then diffs it into an append, or a trim plus an + // append for a sliding tail; replacing the slot object would record the whole window on every commit. + if ((slot.output ?? "") !== snapshot.text) slot.output = snapshot.text; + if (snapshot.droppedBytes > 0) slot.droppedBytes = snapshot.droppedBytes; + if (snapshot.droppedLines > 0) slot.droppedLines = snapshot.droppedLines; + // Diff details leaf by leaf and append new diagnostics, so each commit writes only what changed. + if (detailsChanged && current.details !== undefined) { + assignJson(slot, "details", current.details); + } + if (added.length > 0) { + if (slot.diagnostics === undefined) slot.diagnostics = []; + for (const diagnostic of added) slot.diagnostics.push(diagnostic); + } + return undefined; + }, + context, + PROGRESS_COMMIT, + ); written = current; return bytes; }, diff --git a/packages/durable/src/harness/types.ts b/packages/durable/src/harness/types.ts index 66d8cb75f..fbe636dd7 100644 --- a/packages/durable/src/harness/types.ts +++ b/packages/durable/src/harness/types.ts @@ -18,6 +18,7 @@ import type { import type { TelemetryContext } from "@earendil-works/pi-telemetry"; import type { ExecutionEnv, ShellOutputSkip, ShellOutputWindow } from "../env/index.ts"; import type { + CommitOptions, ConversationId, ConversationOwnership, ConversationRecord, @@ -240,7 +241,8 @@ export interface ToolExecutionApi extend diagnostic(diagnostic: ToolDiagnostic): void; /** Replace running details; the last value becomes the result details when the result omits `details`. */ details(value: TDetails, context: Context): Promise; - commit(change: (tx: Tx) => T | Promise, context: Context): Promise; + /** A commit of this call's task; `options` may defer storing it (`CommitPersistence`). */ + commit(change: (tx: Tx) => T | Promise, context: Context, options?: CommitOptions): Promise; memo(name: string, context: Context): Promise; memo(name: string, candidate: T, context: Context): Promise; /** @@ -521,14 +523,21 @@ export type CompactionSettings = Partial & { }; /** - * How often running progress is committed. Each progress commit is a storage write; a host whose storage is remote can - * commit less often, so live answers and tool output appear in larger steps. + * How often running progress is committed, and how long it may wait in memory before it is stored. Progress commits are + * deferred (`CommitPersistence`): watches see each at once, and Storage gets them together, at the run's next phase + * boundary or after `persistWithinMs`. A host can commit less often, so live answers and tool output appear in larger steps. */ export type ProgressPolicy = { /** Minimum pause between commits of the answer being generated. */ partialIntervalMs: number; /** Minimum pause between commits of running tool output; large commits also pause in proportion to their size. */ outputIntervalMs: number; + /** + * How long these commits, and other deferred commits without their own `withinMs`, may stay in memory before they are + * stored (`CommitPersistence`); a crash loses at most about this much of an answer or of tool output. Default 1,000; + * `Infinity` stores them only at the next commit that reads or writes their document and at close. + */ + persistWithinMs: number; }; /** Why a compaction runs: `compact()`, a threshold in generation preparation, or a context overflow. */ @@ -753,8 +762,8 @@ export interface Conversation { */ compact(instructions: string | undefined, context: Context): Promise>; - /** Session commit whose `tx.createTask()` defaults to this conversation. */ - commit(change: (tx: Tx) => T | Promise, context: Context): Promise; + /** Session commit whose `tx.createTask()` defaults to this conversation; `options` as for `Session.commit()`. */ + commit(change: (tx: Tx) => T | Promise, context: Context, options?: CommitOptions): Promise; /** * Committed raw active transcript and model context. With `at`, the context as of that visible entry: the same view * `fork(at)` would start with, without creating a conversation. diff --git a/packages/durable/src/harness/view.ts b/packages/durable/src/harness/view.ts index d47d2e913..7f6f1a408 100644 --- a/packages/durable/src/harness/view.ts +++ b/packages/durable/src/harness/view.ts @@ -1,9 +1,8 @@ import { type Context, type JsonValue, Watch, type WatchHandle } from "@earendil-works/chord"; import { withoutAbortSignal } from "@earendil-works/chord/context"; import { applyImmutable, type NonEmptyPath, type Op, type Path } from "@earendil-works/chord/delta"; -import { contained, type SessionImpl } from "../session/session.ts"; +import { contained, type ObservedCommit, type SessionImpl } from "../session/session.ts"; import type { - CommitPublication, ConversationDocToken, ConversationId, ConversationRecord, @@ -37,7 +36,7 @@ export type ViewObserver = { before: ConversationView, after: ConversationView, ops: readonly Op[], - publication: CommitPublication, + publication: ObservedCommit, context: Context, ): void; /** The Session closed, or failed with `failure`. */ @@ -72,7 +71,8 @@ export function conversationViews(harness: object): ConversationViews { /** * The Harness's conversation view mounts: at most one per conversation, built on the Session line by its first - * observer and dropped with its last. Each mount advances from the Session's commit publications, which are durable. + * observer and dropped with its last. Each mount advances from the Session's commits as memory adopts them, deferred ones + * (`CommitPersistence`) before they are stored. */ export class ConversationViews { readonly #session: SessionImpl; @@ -172,7 +172,7 @@ export class ConversationViews { function advance( id: ConversationId, mount: Mount, - publication: CommitPublication, + publication: ObservedCommit, context: Context, report: (error: unknown) => void, ): void { diff --git a/packages/durable/src/index.ts b/packages/durable/src/index.ts index 60dad5337..b26a50316 100644 --- a/packages/durable/src/index.ts +++ b/packages/durable/src/index.ts @@ -130,6 +130,8 @@ export { defineTask } from "./tasks.ts"; export type { CheckpointInfo, CommitChange, + CommitOptions, + CommitPersistence, CommitPublication, CommonDocDefinition, ContextEdit, diff --git a/packages/durable/src/session/session.ts b/packages/durable/src/session/session.ts index 4a6563b70..d66045b19 100644 --- a/packages/durable/src/session/session.ts +++ b/packages/durable/src/session/session.ts @@ -1,6 +1,6 @@ import { type Context, type JsonValue, Watch } from "@earendil-works/chord"; import { awaitWithContext, BACKGROUND_CONTEXT, withoutAbortSignal } from "@earendil-works/chord/context"; -import { type Op, track } from "@earendil-works/chord/delta"; +import { type Op, PendingOps, track } from "@earendil-works/chord/delta"; import { type AnyDocToken, addressId, @@ -14,6 +14,8 @@ import { idFromNumber } from "../ids.ts"; import { hostTimers, type Timers } from "../timers.ts"; import type { CommitChange, + CommitOptions, + CommitPersistence, CommitPublication, ConversationDocFamilyToken, ConversationDocToken, @@ -56,11 +58,42 @@ import type { TaskRecord, Tx, } from "../types.ts"; -import { type LoadedDocument, Transaction, type TransactionHost, type TransactionScope } from "./transaction.ts"; +import { + type AdoptedDocuments, + type HeldDocument, + type LoadedDocument, + storeHeld, + Transaction, + type TransactionHost, + type TransactionScope, +} from "./transaction.ts"; /** What a document watch observes: the committed value of its incarnation, `null` once it retired. */ type ObservedDocumentValue = Readonly | null; +/** + * What the Session's own components observe of one commit (`observeCommits()`): a stored commit with its own changes, or + * a deferred one adopted in memory only (`CommitPersistence`), which changes documents only. Memory is never behind: + * a later store of held changes is not observed again. + */ +export type ObservedCommit = + | { readonly kind: "stored"; readonly seq: Seq; readonly changes: readonly CommitChange[] } + | { readonly kind: "memory"; readonly changes: readonly DocumentCommitChange[] }; + +/** A held document (`HeldDocument`) as the Session keeps it: pinned in the cache until stored. */ +type Hold = HeldDocument & { + /** When the earliest of its held commits must be stored, on the Session clock. */ + deadline: number; + readonly unpin: () => void; +}; + +/** The armed store of held documents: a foreground timer for the earliest deadline, `fired` once its job is queued. */ +type HoldTimer = { readonly at: number; disarm: () => void; fired: boolean }; + +const NO_HOLDS: readonly Hold[] = []; + +const NO_OPERATIONS: readonly Op[] = []; + /** The default source of a report: a listener, which never fails what ran it. */ const LISTENER: ReportInfo = { type: "listener_error" }; @@ -71,14 +104,23 @@ const LISTENER: ReportInfo = { type: "listener_error" }; */ export function createSession( storage: Storage, - options?: { readonly now?: () => number; readonly documentRetentionMs?: number; readonly setTimer?: SetTimer }, + options?: { + readonly now?: () => number; + readonly documentRetentionMs?: number; + readonly setTimer?: SetTimer; + /** Default `withinMs` of a deferred commit (`CommitPersistence`); default 1,000. */ + readonly persistWithinMs?: number; + }, ): Session { const retention = options?.documentRetentionMs; + const within = options?.persistWithinMs; return new SessionImpl( storage, options?.now, retention === undefined ? undefined : () => retention, options?.setTimer, + false, + () => within, ); } @@ -98,6 +140,17 @@ export function documentRetention(ms: number | undefined): number { return typeof ms === "number" && ms >= 0 ? ms : defaultRetention; } +/** Default `withinMs` of a deferred commit, `settings.progress.persistWithinMs`. */ +const DEFAULT_PERSIST_WITHIN_MS = 1_000; + +/** + * A deferred commit's default window as given, when it is a number of milliseconds, `0` or more (`Infinity` waits for a + * reading or writing commit or close); otherwise the default. + */ +export function persistWithin(ms: number | undefined): number { + return typeof ms === "number" && ms >= 0 ? ms : DEFAULT_PERSIST_WITHIN_MS; +} + /** * Session kernel: one mutation line, the loaded document tracker cache, and committed publication. * @@ -105,8 +158,11 @@ export function documentRetention(ms: number | undefined): number { * written for the retention period, and the next access loads it from Storage. Eviction runs as a job on the mutation * line, so no transaction ever holds a tracker that leaves. * - * Only committed state is observable. Every commit callback, preparation, Storage settlement, adoption, and - * publication enqueue runs while the line is held; listeners run later. + * Only committed state is observable. A deferred commit (`CommitPersistence`) is committed in memory first: it is + * adopted and shown to the Session's own observers, and its changes wait, folded per document and pinned in the cache, for + * the first commit that reads or writes the document, the document's deadline, a drop of its tracker, or close, which + * store them. Every commit callback, preparation, Storage settlement, adoption, and publication enqueue runs while the + * line is held; listeners run later. * * Storage failures are final. Every Storage call goes through one guard: the first error a Storage method throws, other * than for a cancelled caller, fails the Session, as does a commit it cannot adopt. That call gets the error; every later @@ -123,6 +179,10 @@ export class SessionImpl implements Session { readonly #attachedDocuments = new Map(); readonly #clock: () => number; readonly #documentRetentionMs: () => number; + readonly #persistWithinMs: () => number; + /** Documents with held changes, by incarnation (record ID); each is pinned in the cache until they are stored. */ + readonly #holds = new Map(); + #holdTimer: HoldTimer | undefined; /** Refuse every commit with writes (`HarnessOptions.readOnly`). */ readonly #readOnly: boolean; readonly #timers: Timers; @@ -133,7 +193,7 @@ export class SessionImpl implements Session { #documentTimersUnavailable = false; readonly #commitListeners = new Set<(publication: CommitPublication, context: Context) => void>(); /** The Session's own listeners, such as its scheduler's: they keep memory in step with storage, so a throw fails it. */ - readonly #internalListeners = new Set<(publication: CommitPublication, context: Context) => void>(); + readonly #internalListeners = new Set<(publication: ObservedCommit, context: Context) => void>(); readonly #closeListeners = new Set<() => void>(); readonly #host: TransactionHost; #tail: Promise = Promise.resolve(); @@ -163,6 +223,7 @@ export class SessionImpl implements Session { documentRetentionMs: () => number | undefined = () => undefined, setTimer?: SetTimer, readOnly = false, + persistWithinMs: () => number | undefined = () => undefined, ) { // Waits race it; none has to. this.#failed.promise.catch(() => {}); @@ -185,6 +246,13 @@ export class SessionImpl implements Session { return documentRetention(undefined); } }; + this.#persistWithinMs = () => { + try { + return persistWithin(persistWithinMs()); + } catch { + return persistWithin(undefined); + } + }; this.#host = { storage: this.#storage, now, @@ -210,8 +278,8 @@ export class SessionImpl implements Session { return this.#storage; } - commit(change: (tx: Tx) => T | Promise, context: Context): Promise { - return this.commitWith(change, context); + commit(change: (tx: Tx) => T | Promise, context: Context, options?: CommitOptions): Promise { + return this.commitWith(change, context, undefined, options); } /** @@ -234,13 +302,32 @@ export class SessionImpl implements Session { * bootstrap and task replacement. `scope` sets the default `tx.createTask()` conversation and the task attributed to * appended entries. */ - commitWith(change: (tx: Transaction) => T | Promise, context: Context, scope?: TransactionScope): Promise { + commitWith( + change: (tx: Transaction) => T | Promise, + context: Context, + scope?: TransactionScope, + options?: CommitOptions, + ): Promise { + let withinMs: number | undefined; try { this.assertUsable(); + withinMs = this.#deferral(options?.persistence); } catch (error) { return Promise.reject(error); } - return this.#enqueue(() => this.#runCommit(change, context, scope)); + return this.#enqueue(() => this.#runCommit(change, context, scope, withinMs)); + } + + /** How long a commit may stay in memory before it is stored; `undefined` when it is immediate. */ + #deferral(persistence: CommitPersistence | undefined): number | undefined { + if (persistence?.kind !== "deferred") return undefined; + const ms = persistence.withinMs; + if (ms === undefined) return this.#persistWithinMs(); + // Public options: checked here, as the types cannot. + if (typeof ms !== "number" || !(ms >= 0)) { + throw new TypeError("CommitPersistence.withinMs must be a number of milliseconds, 0 or more"); + } + return ms; } /** Internal: run a read-only job on the mutation line so multi-read derivations observe one committed state. */ @@ -437,11 +524,21 @@ export class SessionImpl implements Session { .then(() => this.beforeClose()) .then(() => this.#enqueue(async () => { + // Last, after the writes at close (`writeAtClose()`), while the trackers are still here. A failed Session + // writes nothing; a failing store fails it, and the backend still closes. + let unstored: { readonly error: unknown } | undefined; + // Only with something to store: otherwise the backend's close begins in this very job. + if (this.#failure === undefined && this.#holds.size > 0) { + await this.#storeHolds([...this.#holds.values()], cleanup).catch((error: unknown) => { + unstored = { error }; + }); + } this.#commitListeners.clear(); this.#internalListeners.clear(); this.#documents.clear(); this.#idleDocuments.clear(); this.#attachedDocuments.clear(); + this.#holds.clear(); try { await this.#storage.close(cleanup); } catch (error) { @@ -452,6 +549,7 @@ export class SessionImpl implements Session { } throw error; } + if (unstored !== undefined) throw unstored.error; }), ) .finally(() => { @@ -469,6 +567,8 @@ export class SessionImpl implements Session { #notifyClose(): void { this.#documentSweep?.disarm?.(); this.#documentSweep = undefined; + this.#holdTimer?.disarm(); + this.#holdTimer = undefined; const listeners = [...this.#closeListeners]; this.#closeListeners.clear(); for (const listener of listeners) contained(listener, (error) => this.report(error)); @@ -506,7 +606,10 @@ export class SessionImpl implements Session { return Promise.resolve(); } - /** Register a synchronous post-adoption listener. It must not block or call Session operations; a throw is reported. */ + /** + * Register a synchronous listener for every stored commit, with exactly what it stored (`CommitPublication`). It must + * not block or call Session operations; a throw is reported. + */ subscribeCommits(listener: (publication: CommitPublication, context: Context) => void): () => void { this.assertUsable(); this.#commitListeners.add(listener); @@ -514,10 +617,11 @@ export class SessionImpl implements Session { } /** - * Internal: `subscribeCommits()` for the Session's own components, whose state follows each commit. A throw leaves - * that state behind storage, so it fails the Session instead of being reported. They run before host listeners. + * Internal: observe every commit as memory adopts it (`ObservedCommit`), for the Session's own components, whose state + * follows memory. A throw leaves that state behind, so it fails the Session instead of being reported. They run + * before host listeners. */ - observeCommits(listener: (publication: CommitPublication, context: Context) => void): () => void { + observeCommits(listener: (publication: ObservedCommit, context: Context) => void): () => void { this.assertUsable(); this.#internalListeners.add(listener); return () => this.#internalListeners.delete(listener); @@ -533,29 +637,32 @@ export class SessionImpl implements Session { return () => this.#closeListeners.delete(listener); } - /** Drop every loaded tracker on the mutation line; later access cold-loads from Storage. */ + /** Drop every loaded tracker on the mutation line, held changes stored first; later access cold-loads from Storage. */ unloadDocuments(): Promise { return this.#enqueue(async () => { + if (this.#failure === undefined) await this.#storeHolds([...this.#holds.values()], BACKGROUND_CONTEXT); this.#documents.clear(); this.#idleDocuments.clear(); }); } /** - * Internal, for tests: how many documents the cache holds, how many are idle, how many incarnations are pinned, and how - * many sweep jobs were queued on the line so far. + * Internal, for tests: how many documents the cache holds, how many are idle, how many incarnations are pinned, how + * many sweep jobs were queued on the line so far, and how many documents have held changes. */ get documentCacheSizes(): { readonly cached: number; readonly idle: number; readonly attached: number; readonly sweeps: number; + readonly held: number; } { return { cached: this.#documents.size, idle: this.#idleDocuments.size, attached: this.#attachedDocuments.size, sweeps: this.#sweepJobs, + held: this.#holds.size, }; } @@ -653,7 +760,8 @@ export class SessionImpl implements Session { async #runCommit( change: (tx: Transaction) => T | Promise, context: Context, - scope?: TransactionScope, + scope: TransactionScope | undefined, + withinMs: number | undefined, ): Promise { this.#assertHealthy(); context.abortSignal?.throwIfAborted(); @@ -687,9 +795,19 @@ export class SessionImpl implements Session { this.#sweepDocuments(); throw new ReadOnlySession("a commit with writes"); } + if (withinMs !== undefined) { + const held = tx.holdable(); + if (held !== undefined) { + this.#hold(tx, held, withinMs, context); + this.#sweepDocuments(); + return result; + } + } + // The last step before Storage: held changes leave their queue only once nothing but Storage can fail. + const taken = this.#holds.size === 0 ? NO_HOLDS : tx.takeHeld((id) => this.#holds.get(id)); // Storage adopts the batch synchronously once it is durable, before its next commit; a throw here would reach // Storage, so the callback keeps its own error. - let adoption: { readonly documents: DocumentCommitChange[] } | { readonly error: unknown } | undefined; + let adoption: { readonly documents: AdoptedDocuments } | { readonly error: unknown } | undefined; const adopt = (committed: Seq): void => { try { adoption = { documents: tx.adopt(committed) }; @@ -717,18 +835,126 @@ export class SessionImpl implements Session { throw adoption.error; } const documents = adoption.documents; - for (const document of documents) this.#touchDocument(addressId(document.record)); + if (taken.length > 0) this.#release(taken); + for (const document of documents.stored) this.#touchDocument(addressId(document.record)); this.#publish(seq, writes, documents, context); this.#sweepDocuments(); return result; } - #publish( - seq: Seq, - writes: readonly StorageWrite[], - documents: readonly DocumentCommitChange[], + /** + * Hold a deferred commit (`Transaction.holdable()`): adopt it in memory, queue its changes per document, pinned, by + * the earliest deadline of the document's held commits, and show it to the Session's own observers. + */ + #hold( + tx: Transaction, + held: NonNullable>, + withinMs: number, context: Context, ): void { + const documents = tx.adoptHeld(); + const deadline = this.#clock() + withinMs; + for (const { loaded, ops, base } of held) { + const id = loaded.record.id; + let hold = this.#holds.get(id); + if (hold === undefined) { + const unpin = this.#pinDocument(loaded.addressId, id); + hold = { loaded, pending: new PendingOps(), base: false, deadline, unpin }; + this.#holds.set(id, hold); + } + hold.pending.push(ops); + if (base) hold.base = true; + if (deadline < hold.deadline) hold.deadline = deadline; + } + this.#armHoldTimer(); + const publication: ObservedCommit = { kind: "memory", changes: documents }; + for (const listener of [...this.#internalListeners]) { + contained( + () => listener(publication, context), + (error) => this.fail(error, context), + ); + } + } + + /** + * Store the held changes of `holds` in one storage transaction, on the line. Memory already has them, so only + * `subscribeCommits()` listeners see it. A failure fails the Session, as every Storage failure does. + */ + async #storeHolds(holds: readonly Hold[], context: Context): Promise { + if (holds.length === 0) return; + const writes: Extract[] = []; + const changes: DocumentCommitChange[] = []; + for (const hold of holds) { + const { record, tracker, valueVersion: version } = hold.loaded; + const value = tracker.value; + const stored = storeHeld(hold, NO_OPERATIONS, value, version); + writes.push({ type: "document.change", id: record.id, content: stored.content }); + // Held documents are conversation documents. + const conversationId = record.scope.kind === "conversation" ? record.scope.conversationId : undefined; + changes.push({ type: "document", record, conversationId, version, value, ops: stored.ops }); + } + let adopted = false; + const adopt = (): void => { + adopted = true; + for (const [index, hold] of holds.entries()) { + if (writes[index]!.content.kind === "base") hold.loaded.deltasSinceBase = 0; + else hold.loaded.deltasSinceBase++; + } + }; + const seq = await this.#storage.commit(writes, adopt, withoutAbortSignal(context)); + if (!adopted) { + const error = new Error("Storage committed without adopting the batch"); + this.fail(error); + throw error; + } + this.#release(holds); + const publication: CommitPublication = { seq, changes }; + for (const listener of [...this.#commitListeners]) { + contained( + () => listener(publication, context), + (error) => this.report(error, LISTENER, context), + ); + } + } + + /** After their store: forget `holds`, unpin their documents, and move the timer to the next deadline. */ + #release(holds: readonly Hold[]): void { + for (const hold of holds) { + this.#holds.delete(hold.loaded.record.id); + hold.unpin(); + } + this.#armHoldTimer(); + } + + /** + * Keep a foreground timer (Workers refuse background ones) for the earliest deadline of the held documents. When it + * fires, every held document is stored, in one storage transaction: a deadline is the latest time, and the others + * cost little more in the same transaction. Once fired, it stays until its job ran. + */ + #armHoldTimer(): void { + if (this.#closing !== undefined) return; + let at = Infinity; + for (const hold of this.#holds.values()) if (hold.deadline < at) at = hold.deadline; + const armed = this.#holdTimer; + if (armed !== undefined && (armed.fired || armed.at === at)) return; + armed?.disarm(); + this.#holdTimer = undefined; + if (at === Infinity) return; + const timer: HoldTimer = { at, disarm: () => {}, fired: false }; + const run = (): void => { + timer.fired = true; + this.#enqueue(async () => { + if (this.#holdTimer === timer) this.#holdTimer = undefined; + // Close stores what is held itself. + if (this.#closing !== undefined || this.#failure !== undefined) return; + await this.#storeHolds([...this.#holds.values()], BACKGROUND_CONTEXT); + }).catch(() => {}); + }; + this.#holdTimer = timer; + timer.disarm = this.#timers.foreground(run, Math.max(0, at - this.#clock())); + } + + #publish(seq: Seq, writes: readonly StorageWrite[], documents: AdoptedDocuments, context: Context): void { const changes: CommitChange[] | undefined = this.#commitListeners.size > 0 || this.#internalListeners.size > 0 ? [] : undefined; for (const write of writes) { @@ -756,18 +982,24 @@ export class SessionImpl implements Session { } } if (changes === undefined) return; - for (const document of documents) changes.push(document); - const publication: CommitPublication = { seq, changes }; + const tables = changes.length; + for (const document of documents.observed) changes.push(document); + const observed: ObservedCommit = { kind: "stored", seq, changes }; + // One publication for both, unless the commit also stored held changes, which only durable listeners get. + const stored: CommitPublication = + documents.stored === documents.observed + ? observed + : { seq, changes: [...changes.slice(0, tables), ...documents.stored] }; // The commit is durable: a failing listener neither fails the commit nor skips the others. for (const listener of [...this.#internalListeners]) { contained( - () => listener(publication, context), + () => listener(observed, context), (error) => this.fail(error, context), ); } for (const listener of [...this.#commitListeners]) { contained( - () => listener(publication, context), + () => listener(stored, context), (error) => this.report(error, LISTENER, context), ); } @@ -822,7 +1054,12 @@ export class SessionImpl implements Session { this.#touchDocument(addressId); return cached; } - if (cached !== undefined) this.#dropDocument(addressId); + if (cached !== undefined) { + // The reload must find what memory holds: held changes are stored before their tracker goes. + const hold = this.#holds.get(cached.record.id); + if (hold !== undefined) await this.#storeHolds([hold], context); + this.#dropDocument(addressId); + } const record = await this.#storage.findDocument(address, "current", context); if (record === undefined) return undefined; const stored = await this.#storage.document(record.id, "current", context); diff --git a/packages/durable/src/session/transaction.ts b/packages/durable/src/session/transaction.ts index e7b783408..7a62c440b 100644 --- a/packages/durable/src/session/transaction.ts +++ b/packages/durable/src/session/transaction.ts @@ -1,6 +1,6 @@ import { type Context, copyJson, type Draft, type JsonValue } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { type Change, type Op, type Prepared, type Tracker, track } from "@earendil-works/chord/delta"; +import { type Change, type Op, type PendingOps, type Prepared, type Tracker, track } from "@earendil-works/chord/delta"; import type { TelemetryCarrier } from "@earendil-works/pi-telemetry"; import { type AnyDocDefinition, @@ -24,6 +24,7 @@ import type { Cursor, DocumentAddress, DocumentCommitChange, + DocumentContent, DocumentCopySource, DocumentCreate, DocumentId, @@ -97,6 +98,25 @@ export type LoadedDocument = { readonly tracker: Tracker; }; +/** + * The changes of a cached incarnation that deferred commits adopted in memory and the Session has not stored yet + * (`CommitPersistence`), folded. The Session owns it; a commit that stores them takes them. + */ +export type HeldDocument = { + readonly loaded: LoadedDocument; + readonly pending: PendingOps; + /** Set when a held commit selected a base (`checkpointWhen`): their store writes the whole value. */ + base: boolean; +}; + +/** What adoption publishes: to the Session's own observers, and the stored changes to `subscribeCommits()`. */ +export type AdoptedDocuments = { + /** This commit's own changes, as memory adopted them. */ + readonly observed: readonly DocumentCommitChange[]; + /** What Storage got: `observed`, unless the commit also stored held changes, which precede its own. */ + readonly stored: readonly DocumentCommitChange[]; +}; + /** Session services used by a transaction while it holds the mutation line. */ export interface TransactionHost { readonly storage: Storage; @@ -188,6 +208,11 @@ type DocumentPlan = { }; /** Resolved before Storage admission so adoption performs no reads. */ conversationId?: ConversationId; + /** + * Set when this commit also stores held changes of the incarnation (`takeHeld()`): the operations its stored change + * publishes, held first, and whether it has a change of its own to show the Session's observers. + */ + held?: { readonly ops: readonly Op[]; readonly observed: boolean }; }; /** @@ -724,9 +749,102 @@ export class Transaction implements Tx { this.#abortChanges(); } - /** Adopt every prepared change by pointer swap after Storage success and describe the publication. */ - adopt(seq: Seq): DocumentCommitChange[] { + /** + * Internal: what a deferred commit holds in memory instead of storing (`CommitPersistence`), each changed incarnation + * with its operations and whether they selected a base; `undefined` when the commit must be stored. It may be held + * only when every write changes an existing `latest`, `initial`-fork conversation document at its stored version. + */ + holdable(): { readonly loaded: LoadedDocument; readonly ops: readonly Op[]; readonly base: boolean }[] | undefined { + if (this.#writes.some((write) => write.type !== "document.change")) return undefined; + const held = []; + for (const plan of this.#plans) { + const { content, change } = plan; + if (content === undefined && !plan.retire) continue; + if (plan.retire || change?.loaded === undefined || content?.type !== "document.change") return undefined; + const { loaded, prepared, version } = change; + const record = loaded.record; + if (record.scope.kind !== "conversation" || record.history !== "latest" || record.fork !== "initial") { + return undefined; + } + // A migration stores a base at once. + if (loaded.storedVersion !== version) return undefined; + held.push({ loaded, ops: prepared.ops, base: content.content.kind === "base" }); + } + return held; + } + + /** + * Internal: adopt a held commit (`holdable()`) in memory only. Trackers advance by pointer swap; stored versions and + * delta counts stay, as nothing was stored. Returns its publication for the Session's own observers. + */ + adoptHeld(): DocumentCommitChange[] { const publications: DocumentCommitChange[] = []; + for (const plan of this.#plans) { + // Every plan of a held commit changes or reads a loaded incarnation (`holdable()`). + const { tracker, prepared, version, loaded } = plan.change!; + if (prepared.ops.length === 0) { + prepared.abort(); + continue; + } + tracker.adopt(prepared); + const record = loaded!.record; + const conversationId = plan.conversationId; + publications.push({ + type: "document", + record, + conversationId, + version, + value: prepared.value, + ops: prepared.ops, + }); + } + return publications; + } + + /** + * Internal: store the held changes of every incarnation this commit reads or writes in front of its own, so a stored + * document never misses an operation its later ones build on. Called right before Storage admission, after every + * fallible step, because it empties the holds: a failure earlier keeps them. Returns the holds it took. + */ + takeHeld(held: (id: DocumentId) => H | undefined): H[] { + const taken: H[] = []; + for (const plan of this.#plans) { + // A new incarnation holds nothing yet. + if (!("createdAt" in plan.record)) continue; + const hold = held(plan.record.id); + if (hold === undefined) continue; + taken.push(hold); + const own = plan.content?.type === "document.change" ? plan.content : undefined; + const change = plan.change; + const value = change?.prepared.value ?? hold.loaded.tracker.value; + const stored = storeHeld(hold, change?.prepared.ops ?? EMPTY_OPERATIONS, value, hold.loaded.valueVersion); + // A base this commit selected or must write covers what was held. + const content = own?.content.kind === "base" ? own.content : stored.content; + const write = { type: "document.change", id: plan.record.id, content } as const; + if (own !== undefined) this.#writes[this.#writes.indexOf(own)] = write; + else this.#insertBeforeRetirement(write); + if (!plan.retire) plan.held = { ops: stored.ops, observed: own !== undefined }; + plan.content = write; + // Held documents are conversation documents; one this commit only read has no owner resolved yet. + if (plan.record.scope.kind === "conversation") plan.conversationId = plan.record.scope.conversationId; + } + return taken; + } + + /** Stage a content write for an incarnation, before its retirement when it retires. */ + #insertBeforeRetirement(write: Extract): void { + const at = this.#writes.findIndex( + (candidate) => candidate.type === "document.retire" && candidate.id === write.id, + ); + if (at < 0) this.#writes.push(write); + else this.#writes.splice(at, 0, write); + } + + /** Adopt every prepared change by pointer swap after Storage success and describe the publication. */ + adopt(seq: Seq): AdoptedDocuments { + const publications: DocumentCommitChange[] = []; + // Only when the commit stored held changes: then what Storage got differs from what memory adopted. + let stored: DocumentCommitChange[] | undefined; for (const plan of this.#plans) { const committed = "createdAt" in plan.record; let record: DocumentRecord = committed ? (plan.record as DocumentRecord) : { ...plan.record, createdAt: seq }; @@ -754,31 +872,21 @@ export class Transaction implements Tx { }); } } - const conversationId = plan.conversationId; - if (plan.retire) { - if (committed) this.#host.evict(plan.addressId, record.id); - const retired = { version: undefined, value: null, ops: EMPTY_OPERATIONS }; - publications.push({ type: "document", record, conversationId, ...retired }); - } else if (plan.content?.type === "document.copy") { - publications.push({ - type: "document.copy", - record, - conversationId: conversationId!, - source: plan.content.source, - }); - } else if (change !== undefined && publishes(plan)) { - const ops = change.loaded === undefined ? EMPTY_OPERATIONS : change.prepared.ops; - publications.push({ - type: "document", - record, - conversationId, - version: change.version, - value: change.prepared.value, - ops, - }); + if (plan.retire && committed) this.#host.evict(plan.addressId, record.id); + const publication = describe(plan, record); + if (publication === undefined) continue; + const held = plan.held; + if (held === undefined) { + publications.push(publication); + stored?.push(publication); + continue; } + stored ??= [...publications]; + if (held.observed) publications.push(publication); + // Only a document change of a loaded incarnation carries held operations. + stored.push(publication.type === "document" ? { ...publication, ops: held.ops } : publication); } - return publications; + return { observed: publications, stored: stored ?? publications }; } async #assemble(): Promise { @@ -1062,6 +1170,38 @@ function planDocument(document: DocumentEntry): DocumentPlan | undefined { } } +/** The publication of one adopted plan, if it publishes (`publishes()`), with `record` as committed. */ +function describe(plan: DocumentPlan, record: DocumentRecord): DocumentCommitChange | undefined { + const conversationId = plan.conversationId; + if (plan.retire) + return { type: "document", record, conversationId, version: undefined, value: null, ops: EMPTY_OPERATIONS }; + if (plan.content?.type === "document.copy") { + return { type: "document.copy", record, conversationId: conversationId!, source: plan.content.source }; + } + const change = plan.change; + if (change === undefined || !publishes(plan)) return undefined; + const ops = change.loaded === undefined ? EMPTY_OPERATIONS : change.prepared.ops; + return { type: "document", record, conversationId, version: change.version, value: change.prepared.value, ops }; +} + +/** + * Take a hold's changes after `ops`, a commit's own on top of them, as one stored content for an incarnation now at + * `value`, and the operations a reader of the stored state applies to reach it: the folded ones, or a root replacement + * after the queue overflowed, which stores a base, as does a hold whose commits selected one. + */ +export function storeHeld( + hold: HeldDocument, + ops: readonly Op[], + value: JsonObject, + version: number, +): { readonly content: DocumentContent; readonly ops: readonly Op[] } { + hold.pending.push(ops); + const taken = hold.pending.take(); + if (taken.kind === "replace") return { content: { version, kind: "base", value }, ops: [["r", value]] }; + if (hold.base) return { content: { version, kind: "base", value }, ops: taken.ops }; + return { content: { version, kind: "delta", ops: taken.ops }, ops: taken.ops }; +} + /** * Whether adoption publishes the plan: every creation, copy, and retirement, and a loaded incarnation that writes * content, which includes a migration-only base so observers of the older shape receive the new value. diff --git a/packages/durable/src/types.ts b/packages/durable/src/types.ts index 24c44f9ec..5844fccb1 100644 --- a/packages/durable/src/types.ts +++ b/packages/durable/src/types.ts @@ -268,6 +268,8 @@ export interface TaskRuntime extends DocumentObserver * Harness is closing, or, in a run invocation, the task carries an abort mark. A returned state replaces the task's * state in the same commit; returning nothing leaves it unchanged. `tx.createTask()` defaults to the task's * conversation. Inside the callback read only through `tx`: `snapshot()`, `watchDoc()`, `getTask()`, and the other Session and runtime reads may queue on the line the callback holds, and then never resolve. + * `options.persistence` may defer storing a commit that changes only documents (`CommitPersistence`); one that returns + * a state is immediate. */ commit( change: ( @@ -275,6 +277,7 @@ export interface TaskRuntime extends DocumentObserver current: RunningTask, ) => NextTaskState | undefined | Promise | undefined>, context: Context, + options?: CommitOptions, ): Promise; /** Read a durable memo of this task. */ memo(name: string, context: Context): Promise; @@ -897,7 +900,12 @@ export type TableCommitChange = Extract< export type CommitChange = TableCommitChange | DocumentCommitChange; -/** Every immutable change from one successful Session commit. Change order is unspecified. */ +/** + * Every immutable change one Session storage transaction stored, with its `seq`. Change order is unspecified. A commit + * held in memory (`CommitPersistence`) is published once it is stored: by the next commit that reads or writes its + * document, alone (`seq` of its own), or with that commit, whose document change then carries the held operations + * followed by its own. + */ export type CommitPublication = { readonly seq: Seq; readonly changes: readonly CommitChange[]; @@ -1048,12 +1056,30 @@ export interface DocumentObserver { /** Why a Session ended: `close()`, or a failed Storage call, whose error it carries. */ export type SessionEnd = { readonly reason: "closed" } | { readonly reason: "failed"; readonly error: unknown }; +/** + * When a commit reaches Storage. `immediate`, the default: before `commit()` resolves. `deferred`: when every write of + * the commit changes an existing conversation document with `history: "latest"` and `fork: "initial"` (no creation, + * migration, or retirement), the Session adopts it in memory, shows it to watches and views, and resolves, but stores it + * later: with the next commit that reads (`tx.doc()`) or writes that document, before the document leaves memory or + * retires, at `close()`, and at the latest about `withinMs` after the commit (default `settings.progress.persistWithinMs`, + * 1,000 ms; `Infinity` waits for one of the others). A commit that writes anything else is immediate. A crash loses what + * is held, so only state whose loss is acceptable, such as streamed progress, should be deferred, and nothing durable may + * depend on it without reading it in a commit. + */ +export type CommitPersistence = + | { readonly kind: "immediate" } + | { readonly kind: "deferred"; readonly withinMs?: number }; + +/** Options of one Session commit. */ +export type CommitOptions = { readonly persistence?: CommitPersistence }; + /** Owner of one mutation line, its records, and its tracked documents. */ export interface Session extends DocumentObserver { /** * Run one atomic transaction on the Session mutation line. Inside the callback read only through `tx`: `snapshot()`, `watchDoc()`, `getTask()`, and the other Session and runtime reads may queue on the line the callback holds, and then never resolve. + * `options.persistence` may defer storing it (`CommitPersistence`). */ - commit(change: (tx: Tx) => T | Promise, context: Context): Promise; + commit(change: (tx: Tx) => T | Promise, context: Context, options?: CommitOptions): Promise; /** Seal admission, settle admitted commits, then close storage. */ close(context: Context): Promise; /** @@ -1061,7 +1087,10 @@ export interface Session extends DocumentObserver { * which closes the Session itself. A failed Session is reopened from Storage; nothing else is left to clean up. */ readonly closed: Promise; - /** Observe complete commits synchronously after adoption. The listener must not block or call Session APIs. */ + /** + * Observe stored commits synchronously after adoption, each with exactly what it stored; a commit held in memory is + * published when it is stored (`CommitPublication`). The listener must not block or call Session APIs. + */ subscribeCommits(listener: (publication: CommitPublication, context: Context) => void): () => void; /** * Observe close synchronously when it begins, also when a failed Storage call closes the Session. The listener must diff --git a/packages/durable/test/harness-deferred-progress.test.ts b/packages/durable/test/harness-deferred-progress.test.ts new file mode 100644 index 000000000..8f6221af7 --- /dev/null +++ b/packages/durable/test/harness-deferred-progress.test.ts @@ -0,0 +1,444 @@ +// Deferred progress commits (`PROGRESS_COMMIT`): the partial answer and tool output reach watches at once and Storage at +// the next commit that reads or writes `pi.live` or within `settings.progress.persistWithinMs`. Crashes are simulated +// by failing the Harness: from that moment nothing more reaches Storage, as when the process dies; the reopened Harness +// resumes from what was stored. +import { applyImmutable } from "@earendil-works/chord/delta"; +import { fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { + AssistantEntry, + type ConversationId, + defineTask, + defineTool, + type EntryRecord, + type Harness, + LiveDoc, + type LiveState, + type TaskId, + ToolResultEntry, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import type { PartialMessage } from "../src/harness/partial-changes.ts"; +import type { SessionImpl } from "../src/session/session.ts"; +import { allEntries, type ChatSetup, chatSetup, openChat, textOf, waitFor } from "./chat-support.ts"; +import { addTask, addTool } from "./harness-support.ts"; +import { ControlledStorage, context, observeCommits } from "./session-support.ts"; +import { aborted, deferred } from "./task-support.ts"; + +/** A slow stream: 4 characters 80 times a second. */ +function slowSetup(persistWithinMs: number): ChatSetup { + const setup = chatSetup({ tokensPerSecond: 80, tokenSize: { min: 4, max: 4 } }); + setup.settings.progress = { persistWithinMs }; + return setup; +} + +const LONG = "streamed ".repeat(400); + +/** Open (or reopen) a Harness over `storage` and let it run tasks. */ +async function open(storage: ControlledStorage, setup: ChatSetup) { + const opened = await openChat(storage.reopen(), setup); + opened.harness.resume(); + return { ...opened, memory: memoryPartials(opened.harness) }; +} + +/** The partial text of every `pi.live` revision memory adopted, the last one last. */ +function memoryPartials(harness: Harness): string[] { + const texts: string[] = []; + observeCommits(harness, (publication) => { + for (const change of publication.changes) { + if (change.type !== "document" || change.record.kind !== LiveDoc.definition.kind) continue; + // Publications carry untyped document values; this one is `pi.live`. + const message = (change.value as LiveState | null)?.generation?.message; + if (message !== undefined) texts.push(partialText(message) ?? ""); + } + }); + return texts; +} + +/** Fail the Harness now: nothing more is stored, held changes are lost with it. */ +async function crash(harness: Harness): Promise { + // `fail()` is internal, so the public type lacks it; every Harness is a `SessionImpl`. + (harness as unknown as SessionImpl).fail(new Error("crash")); + await harness.closed; +} + +/** `pi.live` as Storage has it, read past the Session, also after a crash closed it. */ +async function storedLive(storage: ControlledStorage, conversationId: ConversationId): Promise { + storage.reopen(); + const address = { kind: LiveDoc.definition.kind, scope: { kind: "conversation", conversationId } } as const; + const record = await storage.findDocument(address, "current", context); + if (record === undefined) return undefined; + // Storage values are untyped; this one is `pi.live`. + return (await storage.document(record.id, "current", context))?.value as LiveState | undefined; +} + +/** The text of a partial message's first text block. */ +function partialText(message: PartialMessage | undefined): string | undefined { + const block = message?.content.find((candidate) => candidate.type === "text"); + return block?.type === "text" ? block.text : undefined; +} + +function storedText(live: LiveState | undefined): string | undefined { + const message = live?.generation?.message; + return message === undefined ? undefined : (partialText(message) ?? ""); +} + +/** Assistant entries: their text and stop reason. */ +function answers(entries: readonly EntryRecord[]): { text: string | undefined; stopReason: string | undefined }[] { + return entries + .filter((entry) => AssistantEntry.is(entry)) + .map((entry) => { + const message = entry.model![0]!; + return { text: textOf(message), stopReason: message.role === "assistant" ? message.stopReason : undefined }; + }); +} + +async function generationId(harness: Harness, conversationId: ConversationId): Promise { + return (await harness.snapshot(LiveDoc, conversationId, context))!.run!.taskId; +} + +describe("generation partials", () => { + it("hold partials in memory and store them together, at most every persistWithinMs", async () => { + const storage = new ControlledStorage(); + const setup = slowSetup(400); + setup.faux.setResponses([fauxAssistantMessage(LONG.slice(0, 640))]); + const { harness, root, memory } = await open(storage, setup); + const from = storage.commits.length; + await (await root.submit({ type: "input", content: "hi" }, context)).wait(context); + const writesLive = storage.commits + .slice(from) + .filter((batch) => batch.some((write) => write.type === "document.change")); + // About 2 s of streaming at 100 ms per partial: some 20 partials in memory, a store about every 400 ms. + expect(memory.length).toBeGreaterThan(12); + expect(writesLive.length).toBeLessThan(memory.length / 1.5); + await harness.close(context); + }); + + it("converts what was stored into the aborted entry after a crash, and resends", async () => { + const storage = new ControlledStorage(); + const setup = slowSetup(150); + setup.faux.setResponses([fauxAssistantMessage(LONG)]); + let opened = await open(storage, setup); + // A replica fed by subscribeCommits() alone, as a replication feed would be. + let replica = await storedLive(storage, opened.root.id); + opened.harness.subscribeCommits((publication) => { + for (const change of publication.changes) { + if (change.type === "document" && change.record.kind === LiveDoc.definition.kind) { + replica = applyImmutable(replica, change.ops); + } + } + }); + const submission = await opened.root.submit({ type: "input", content: "hi" }, context); + await waitFor(async () => (storedText(await storedLive(storage, opened.root.id))?.length ?? 0) > 40); + const inMemory = opened.memory.at(-1)!; + await crash(opened.harness); + const stored = storedText(await storedLive(storage, opened.root.id))!; + // What Storage has is a state memory had, behind it by at most the window, and what the replica has. + expect(opened.memory).toContain(stored); + expect(inMemory.startsWith(stored)).toBe(true); + expect(replica).toEqual(await storedLive(storage, opened.root.id)); + + setup.faux.setResponses([fauxAssistantMessage("answer")]); + opened = await open(storage, setup); + expect((await (await opened.harness.submission(submission.id, context))!.wait(context)).status).toBe("done"); + expect(answers(await allEntries(opened.root))).toEqual([ + { text: stored, stopReason: "aborted" }, + { text: "answer", stopReason: "stop" }, + ]); + expect(await opened.harness.snapshot(LiveDoc, opened.root.id, context)).toEqual({}); + await opened.harness.close(context); + }); + + it("leaves no aborted entry for a stream that crashed before its first store", async () => { + const storage = new ControlledStorage(); + const setup = slowSetup(Number.POSITIVE_INFINITY); + setup.faux.setResponses([fauxAssistantMessage(LONG)]); + let opened = await open(storage, setup); + const submission = await opened.root.submit({ type: "input", content: "hi" }, context); + await waitFor(() => (opened.memory.at(-1)?.length ?? 0) > 40); + await crash(opened.harness); + expect(await storedLive(storage, opened.root.id)).toMatchObject({ generation: { attempt: 1 } }); + expect(storedText(await storedLive(storage, opened.root.id))).toBeUndefined(); + + setup.faux.setResponses([fauxAssistantMessage("answer")]); + opened = await open(storage, setup); + expect((await (await opened.harness.submission(submission.id, context))!.wait(context)).status).toBe("done"); + expect(answers(await allEntries(opened.root))).toEqual([{ text: "answer", stopReason: "stop" }]); + await opened.harness.close(context); + }); + + it("converts a stored partial once through two crashes, before and after the conversion", async () => { + const storage = new ControlledStorage(); + const setup = slowSetup(150); + setup.faux.setResponses([fauxAssistantMessage(LONG)]); + let opened = await open(storage, setup); + const submission = await opened.root.submit({ type: "input", content: "hi" }, context); + await waitFor(async () => (storedText(await storedLive(storage, opened.root.id))?.length ?? 0) > 40); + await crash(opened.harness); + const stored = storedText(await storedLive(storage, opened.root.id))!; + + // Crash again before anything ran: nothing was converted, so nothing is lost or doubled. + opened = { ...(await openChat(storage.reopen(), setup)), memory: [] }; + await crash(opened.harness); + expect(storedText(await storedLive(storage, opened.root.id))).toBe(stored); + + // Crash again while the resend streams: its conversion is stored, its own partials only held. + setup.settings.progress = { persistWithinMs: Number.POSITIVE_INFINITY }; + setup.faux.setResponses([fauxAssistantMessage(LONG)]); + opened = await open(storage, setup); + await waitFor(() => (opened.memory.at(-1)?.length ?? 0) > 40); + await crash(opened.harness); + expect(storedText(await storedLive(storage, opened.root.id))).toBeUndefined(); + + setup.faux.setResponses([fauxAssistantMessage("answer")]); + opened = await open(storage, setup); + expect((await (await opened.harness.submission(submission.id, context))!.wait(context)).status).toBe("done"); + expect(answers(await allEntries(opened.root))).toEqual([ + { text: stored, stopReason: "aborted" }, + { text: "answer", stopReason: "stop" }, + ]); + await opened.harness.close(context); + }); + + it("stores everything held at a clean close", async () => { + const storage = new ControlledStorage(); + const setup = slowSetup(Number.POSITIVE_INFINITY); + setup.faux.setResponses([fauxAssistantMessage(LONG)]); + let opened = await open(storage, setup); + const submission = await opened.root.submit({ type: "input", content: "hi" }, context); + await waitFor(() => (opened.memory.at(-1)?.length ?? 0) > 40); + await opened.harness.close(context); + const atClose = opened.memory.at(-1)!; + expect(storedText(await storedLive(storage, opened.root.id))).toBe(atClose); + + setup.faux.setResponses([fauxAssistantMessage("answer")]); + opened = await open(storage, setup); + await (await opened.harness.submission(submission.id, context))!.wait(context); + expect(answers(await allEntries(opened.root))[0]).toEqual({ text: atClose, stopReason: "aborted" }); + await opened.harness.close(context); + }); + + it("never counts partial commits as progress of a task cut off again and again", async () => { + const storage = new ControlledStorage(); + const setup = slowSetup(50); + let opened = await open(storage, setup); + setup.faux.setResponses([fauxAssistantMessage(LONG)]); + await opened.root.submit({ type: "input", content: "hi" }, context); + const id = await generationId(opened.harness, opened.root.id); + for (let crashes = 1; crashes <= 2; crashes++) { + await waitFor(async () => (storedText(await storedLive(storage, opened.root.id))?.length ?? 0) > 40); + await crash(opened.harness); + setup.faux.setResponses([fauxAssistantMessage(LONG)]); + opened = await open(storage, setup); + expect((await opened.harness.getTask(id, context))?.restarts).toBe(crashes); + } + await opened.harness.close(context); + }); + + it("resumes polling a deferred response after a crash", async () => { + const storage = new ControlledStorage(); + const setup = chatSetup({ deferred: { pollAfterMs: 60_000 } }); + let now = 1_000; + setup.now = () => now; + setup.faux.setResponses([fauxAssistantMessage("deferred answer")]); + let opened = await open(storage, setup); + setup.settings.stream = { deferred: true }; + const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; + await waitFor(async () => (await storedLive(storage, opened.root.id))?.generation?.deferred !== undefined); + await crash(opened.harness); + opened = await open(storage, setup); + now = 61_000; + opened.harness.resume(); + const settled = await (await opened.harness.submission(id, context))!.wait(context); + if (settled.status !== "done" || settled.type !== "input") throw new Error(`Unexpected ${settled.status}`); + const answer = await opened.root.commit((tx) => tx.entry(AssistantEntry, settled.answer), context); + expect(textOf(answer?.model?.[0])).toBe("deferred answer"); + expect(setup.faux.state.deferredFetchCount).toBe(1); + await opened.harness.close(context); + }); +}); + +describe("tool output", () => { + /** + * A tool that writes a line every 20 ms until its invocation is cancelled on its first run. A later run waits for + * cancellation while `stall` is set (`stalled` resolves then), and otherwise writes one line and returns. + */ + function streamingTool(replay: "safe" | "unsafe") { + const lines: string[] = []; + const state = { runs: 0, lines, stall: false, stalled: deferred() }; + const registration = defineTool({ + name: "work", + description: "work", + parameters: Type.Object({}), + replay, + execute: async (_args, api, callContext) => { + const run = ++state.runs; + const signal = callContext.abortSignal!; + if (run > 1 && state.stall) { + state.stalled.resolve(); + return aborted(signal); + } + if (run > 1) { + api.output(`rerun ${run}\n`); + return {}; + } + for (let line = 0; !signal.aborted; line++) { + api.output(`line ${line}\n`); + state.lines.push(`line ${line}\n`); + await new Promise((resolve) => setTimeout(resolve, 20)); + } + return aborted(signal); + }, + }); + return { registration, state }; + } + + async function storedOutput(storage: ControlledStorage, conversationId: ConversationId) { + return (await storedLive(storage, conversationId))?.tools?.[0]?.output; + } + + function results(entries: readonly EntryRecord[]): string[] { + return entries + .filter((entry) => ToolResultEntry.is(entry)) + .map((entry) => { + const message = entry.model![0]!; + if (message.role !== "toolResult") return ""; + return message.content.map((item) => (item.type === "text" ? item.text : "")).join("|"); + }); + } + + it("answers a tool without replay that crashed with the output stored by then", async () => { + const storage = new ControlledStorage(); + const setup = slowSetup(150); + const { registration, state } = streamingTool("unsafe"); + addTool(setup.registry, registration); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("work", {}, { id: "c1" })], { stopReason: "toolUse" }), + ]); + let opened = await open(storage, setup); + const submission = await opened.root.submit({ type: "input", content: "go" }, context); + await waitFor(async () => ((await storedOutput(storage, opened.root.id))?.length ?? 0) > 20); + await crash(opened.harness); + const stored = (await storedOutput(storage, opened.root.id))!; + expect(state.lines.join("").startsWith(stored)).toBe(true); + + setup.faux.setResponses([fauxAssistantMessage([fauxText("done")])]); + opened = await open(storage, setup); + expect((await (await opened.harness.submission(submission.id, context))!.wait(context)).status).toBe("done"); + expect(state.runs).toBe(1); + expect(results(await allEntries(opened.root))).toEqual([ + `${stored}|\n[error] Tool work was interrupted and may have partially run\n`, + ]); + await opened.harness.close(context); + }); + + it("reruns a replay-safe tool after a crash with its stored progress cleared, also through a second crash", async () => { + const storage = new ControlledStorage(); + const setup = slowSetup(150); + const { registration, state } = streamingTool("safe"); + addTool(setup.registry, registration); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("work", {}, { id: "c1" })], { stopReason: "toolUse" }), + ]); + let opened = await open(storage, setup); + const submission = await opened.root.submit({ type: "input", content: "go" }, context); + await waitFor(async () => ((await storedOutput(storage, opened.root.id))?.length ?? 0) > 20); + await crash(opened.harness); + + // The rerun's first commit stores the cleared progress; it crashes again before writing any of its own. + state.stall = true; + opened = await open(storage, setup); + await state.stalled.promise; + expect(await storedOutput(storage, opened.root.id)).toBeUndefined(); + await crash(opened.harness); + expect(await storedOutput(storage, opened.root.id)).toBeUndefined(); + + state.stall = false; + setup.faux.setResponses([fauxAssistantMessage([fauxText("done")])]); + opened = await open(storage, setup); + expect((await (await opened.harness.submission(submission.id, context))!.wait(context)).status).toBe("done"); + expect(state.runs).toBe(3); + expect(results(await allEntries(opened.root))).toEqual(["rerun 3\n"]); + await opened.harness.close(context); + }); +}); + +describe("compaction", () => { + it("reruns a summarization cut off by a crash from its checkpoint", async () => { + const storage = new ControlledStorage(); + const setup = chatSetup(); + setup.settings.compaction = { enabled: false, reserveTokens: 1000, keepRecentTokens: 150, backgroundTokens: 0 }; + const summarizing = deferred(); + setup.faux.setResponses([ + ...["a1", "a2", "a3"].map((label) => fauxAssistantMessage(`${label} ${"word ".repeat(100)}`)), + async (_request, options) => { + summarizing.resolve(); + return aborted(options!.signal!); + }, + ]); + let opened = await open(storage, setup); + for (const label of ["u1", "u2", "u3"]) { + await (await opened.root.submit({ type: "input", content: `${label} ${"word ".repeat(100)}` }, context)).wait( + context, + ); + } + const id = await opened.root.compact(undefined, context); + await summarizing.promise; + await crash(opened.harness); + + setup.faux.setResponses([fauxAssistantMessage("SUMMARY")]); + opened = await open(storage, setup); + expect((await opened.harness.waitForTask(id, context)).state.outcome.status).toBe("completed"); + await waitFor(async () => (await allEntries(opened.root)).some((entry) => entry.kind === "pi.compaction")); + expect(await opened.harness.snapshot(LiveDoc, opened.root.id, context)).toEqual({}); + await opened.harness.close(context); + }); +}); + +describe("reading pi.live in a commit", () => { + it("stores the held partial a third-party task's commit read, with what it derived", async () => { + const storage = new ControlledStorage(); + const setup = slowSetup(Number.POSITIVE_INFINITY); + const Audit = defineTask<{ conversationId: ConversationId }, { phase: "run" }, null>({ + name: "test.audit", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (task, runtime, ctx) => { + await runtime.commit(async (tx) => { + const live = await tx.doc(LiveDoc, task.input.conversationId); + const text = partialText(live.generation?.message) ?? ""; + await tx.appendEntry(task.input.conversationId, { kind: "test.audit", data: text }); + return { status: "terminal", outcome: { status: "completed", result: null } }; + }, ctx); + }, + }, + abort: async () => {}, + }); + addTask(setup.registry, Audit); + setup.faux.setResponses([fauxAssistantMessage(LONG)]); + let opened = await open(storage, setup); + const submission = await opened.root.submit({ type: "input", content: "hi" }, context); + await waitFor(() => (opened.memory.at(-1)?.length ?? 0) > 40); + const conversationId = opened.root.id; + const audit = await opened.harness.commit( + (tx) => tx.createTask(Audit, { conversationId }, { ownership: { kind: "conversation" }, conversationId }), + context, + ); + await opened.harness.waitForTask(audit, context); + const derived = String((await allEntries(opened.root)).find((entry) => entry.kind === "test.audit")!.data); + expect(derived.length).toBeGreaterThan(40); + // The read stored the partial it saw, in the same storage transaction as the entry. + const batch = storage.commits.find((writes) => + writes.some((write) => write.type === "entry" && write.value.kind === "test.audit"), + )!; + expect(batch.some((write) => write.type === "document.change")).toBe(true); + await crash(opened.harness); + expect(storedText(await storedLive(storage, conversationId))).toBe(derived); + + setup.faux.setResponses([fauxAssistantMessage("answer")]); + opened = await open(storage, setup); + await (await opened.harness.submission(submission.id, context))!.wait(context); + expect(answers(await allEntries(opened.root))[0]).toEqual({ text: derived, stopReason: "aborted" }); + await opened.harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-events.test.ts b/packages/durable/test/harness-events.test.ts index 8f561a205..029a6a53d 100644 --- a/packages/durable/test/harness-events.test.ts +++ b/packages/durable/test/harness-events.test.ts @@ -28,7 +28,7 @@ import { import { describe, expect, it } from "vitest"; import { type ChatSetup, chatSetup, openChat, textOf, waitFor } from "./chat-support.ts"; import { addTool } from "./harness-support.ts"; -import { context, documentChanges } from "./session-support.ts"; +import { context, documentChanges, observeCommits } from "./session-support.ts"; import { aborted, deferred } from "./task-support.ts"; /** Attach and start an event stream that records every delivered batch. */ @@ -95,7 +95,7 @@ function applyChanges(message: AssistantMessage, changes: readonly MessageChange /** Committed partials of `conversation`'s generation, one per commit that has one. */ function partialsOf(harness: Harness): AssistantMessage[] { const partials: AssistantMessage[] = []; - harness.subscribeCommits((publication) => { + observeCommits(harness, (publication) => { for (const change of documentChanges(publication)) { const message = (change.value as LiveState | null)?.generation?.message as AssistantMessage | undefined; if (change.record.kind === "pi.live" && message !== undefined) partials.push(message); @@ -113,7 +113,7 @@ describe("agent events", () => { const { stream, events } = await listen(harness, root); expect(stream.snapshot).toMatchObject({ type: "snapshot", entries: [], tools: [], inbox: [] }); const partials: string[] = []; - harness.subscribeCommits((publication) => { + observeCommits(harness, (publication) => { for (const change of documentChanges(publication)) { const message = (change.value as LiveState | null)?.generation?.message as AssistantMessage | undefined; if (change.record.kind === "pi.live" && message !== undefined) partials.push(textOf(message) ?? ""); @@ -354,7 +354,7 @@ describe("agent events", () => { ]); const { harness, root } = await openChat(new MemoryStorage(), setup); const outputs: string[] = []; - harness.subscribeCommits((publication) => { + observeCommits(harness, (publication) => { for (const change of documentChanges(publication)) { const output = (change.value as LiveState | null)?.tools?.[0]?.output; if (change.record.kind === "pi.live" && output !== undefined && output !== outputs.at(-1)) { @@ -724,7 +724,7 @@ describe("streamed partials", () => { ]); const { harness, root } = await openChat(new MemoryStorage(), setup); let commits = 0; - harness.subscribeCommits(() => commits++); + observeCommits(harness, () => commits++); const bytes = await allocatedBytes(async () => { await (await root.submit({ type: "input", content: "go" }, context)).wait(context); }); diff --git a/packages/durable/test/harness-generation.test.ts b/packages/durable/test/harness-generation.test.ts index 50bae0a58..3cb077264 100644 --- a/packages/durable/test/harness-generation.test.ts +++ b/packages/durable/test/harness-generation.test.ts @@ -9,7 +9,6 @@ import { } from "@earendil-works/pi-ai"; import { AssistantEntry, - type CommitPublication, type Conversation, createRegistry, defineDoc, @@ -31,7 +30,7 @@ import { resolveSettings } from "../src/harness/agent.ts"; import type { SessionImpl } from "../src/session/session.ts"; import { allEntries, type ChatSetup, chatSetup, openChat, textOf, unanswered, waitFor } from "./chat-support.ts"; import { addSection } from "./harness-support.ts"; -import { ControlledStorage, context } from "./session-support.ts"; +import { ControlledStorage, context, observeCommits } from "./session-support.ts"; const ERROR_503 = fauxAssistantMessage([], { stopReason: "error", errorMessage: "503 Service Unavailable" }); @@ -76,7 +75,7 @@ function invalidFinalStream(): ReturnType { function livePublications(harness: Harness): LiveState[] { const values: LiveState[] = []; - harness.subscribeCommits((publication: CommitPublication) => { + observeCommits(harness, (publication) => { for (const change of publication.changes) { if (change.type === "document" && change.record.kind === "pi.live" && change.value !== null) { values.push(change.value as LiveState); @@ -509,7 +508,7 @@ describe("generation", () => { stream: {}, retry: { enabled: true, maxRetries: 3, baseDelayMs: 2000, maxAgentDelayMs: 60000 }, compaction: { enabled: true, reserveTokens: 16384, keepRecentTokens: 20000, backgroundTokens: 32768 }, - progress: { partialIntervalMs: 100, outputIntervalMs: 100 }, + progress: { partialIntervalMs: 100, outputIntervalMs: 100, persistWithinMs: 1000 }, toolExecution: "parallel", steeringMode: "one-at-a-time", followUpMode: "one-at-a-time", @@ -527,7 +526,11 @@ describe("generation", () => { expect(resolveSettings({ progress: { outputIntervalMs: 500 } }).progress).toEqual({ partialIntervalMs: 100, outputIntervalMs: 500, + persistWithinMs: 1000, }); + // Not a number of milliseconds, 0 or more: the default. + expect(resolveSettings({ progress: { persistWithinMs: -1 } }).progress.persistWithinMs).toBe(1000); + expect(resolveSettings({ progress: { persistWithinMs: 0 } }).progress.persistWithinMs).toBe(0); }); it("commits partials no more often than progress.partialIntervalMs", async () => { diff --git a/packages/durable/test/harness-live-deltas.test.ts b/packages/durable/test/harness-live-deltas.test.ts index df5593384..26db00d26 100644 --- a/packages/durable/test/harness-live-deltas.test.ts +++ b/packages/durable/test/harness-live-deltas.test.ts @@ -13,7 +13,7 @@ import { import { describe, expect, it } from "vitest"; import { chatSetup, openChat, waitFor } from "./chat-support.ts"; import { addTool } from "./harness-support.ts"; -import { context, documentChanges } from "./session-support.ts"; +import { context, documentChanges, observeCommits } from "./session-support.ts"; type Action = (api: ToolExecutionApi) => void | Promise; @@ -52,7 +52,7 @@ async function drive(outputLimits: ToolRegistration["outputLimits"] = {}) { ]); const { harness, root } = await openChat(new MemoryStorage(), setup); const commits: Op[][] = []; - harness.subscribeCommits((publication) => { + observeCommits(harness, (publication) => { for (const change of documentChanges(publication)) { if (change.record.kind === "pi.live" && change.ops.length > 0) commits.push([...change.ops]); } @@ -261,7 +261,7 @@ describe("pi.live deltas", () => { setup.faux.setResponses([fauxAssistantMessage([fauxText("word ".repeat(250))])]); const { harness, root } = await openChat(new MemoryStorage(), setup); const commits: Op[][] = []; - harness.subscribeCommits((publication) => { + observeCommits(harness, (publication) => { for (const change of documentChanges(publication)) { if (change.record.kind === "pi.live" && change.ops.length > 0) commits.push([...change.ops]); } diff --git a/packages/durable/test/harness-partial-changes.test.ts b/packages/durable/test/harness-partial-changes.test.ts index 3b61133b7..4e2ebf78d 100644 --- a/packages/durable/test/harness-partial-changes.test.ts +++ b/packages/durable/test/harness-partial-changes.test.ts @@ -1,4 +1,4 @@ -import type { JsonValue } from "@earendil-works/chord"; +import { copyJson } from "@earendil-works/chord"; import { type Op, track } from "@earendil-works/chord/delta"; import { type AssistantMessage, @@ -7,11 +7,12 @@ import { ToolCallArgumentsStream, } from "@earendil-works/pi-ai"; import { describe, expect, it } from "vitest"; -import { PartialChanges } from "../src/harness/partial-changes.ts"; +import type { LiveState } from "../src/harness/live.ts"; +import { PartialChanges, type PartialMessage } from "../src/harness/partial-changes.ts"; // #9265: partial commits reassigned the growing strings of a streamed answer, and Chord compared each one whole. -type Live = { generation: Record }; +type Live = { generation: NonNullable }; /** * A response streaming one text block (started empty) and then one tool call, as a provider produces it: the partial @@ -63,10 +64,11 @@ function setup() { }, /** One partial commit, as the generation's throttle makes it; `fail` aborts it like a rejected commit. */ commit(fail = false) { - const message = structuredClone(partial) as unknown as { content: JsonValue[] }; + // As the generation copies it: `copyJson()` returns untyped JSON, here a message's JSON representation. + const message = copyJson(partial) as PartialMessage; const taken = changes.take(message); const change = tracker.beginChange(); - changes.commit(change.state.generation, message as unknown as JsonValue, taken); + changes.commit(change.state.generation, message, taken); if (fail) { change.abort(); changes.restore(taken); diff --git a/packages/durable/test/harness-registry.test.ts b/packages/durable/test/harness-registry.test.ts index bf2edf002..af852b0c1 100644 --- a/packages/durable/test/harness-registry.test.ts +++ b/packages/durable/test/harness-registry.test.ts @@ -142,6 +142,7 @@ describe("settings", () => { expect(resolveSettings({ progress: { partialIntervalMs: undefined, outputIntervalMs: 250 } }).progress).toEqual({ partialIntervalMs: 100, outputIntervalMs: 250, + persistWithinMs: 1000, }); }); }); diff --git a/packages/durable/test/harness-structured.test.ts b/packages/durable/test/harness-structured.test.ts index 7bc47916d..77fa781cc 100644 --- a/packages/durable/test/harness-structured.test.ts +++ b/packages/durable/test/harness-structured.test.ts @@ -1336,6 +1336,9 @@ describe("tool rounds", () => { }, }); const setup = { ...base, models }; + // The partial is stored at once, so the failed final commit leaves it for the reopened run to convert; held, it would + // be lost with the failed Session. + setup.settings.progress = { persistWithinMs: 0 }; addTask(setup.registry, Node); let opened: Harness | undefined; addHooks(setup.registry, GenerationTask, { diff --git a/packages/durable/test/harness-tools-recovery.test.ts b/packages/durable/test/harness-tools-recovery.test.ts index 226bade81..31485ab25 100644 --- a/packages/durable/test/harness-tools-recovery.test.ts +++ b/packages/durable/test/harness-tools-recovery.test.ts @@ -83,7 +83,7 @@ async function toolTaskId(harness: Harness): Promise { /** * A tool that writes output, then blocks until its invocation is cancelled the first `blocking` times it runs. - * `started` resolves once the output is durable. + * `started` resolves once the output is committed; a close stores it. */ function blockingTool(name: string, extra: Partial = {}) { const started = deferred(); diff --git a/packages/durable/test/session-deferred-commits.test.ts b/packages/durable/test/session-deferred-commits.test.ts new file mode 100644 index 000000000..e4281c5e8 --- /dev/null +++ b/packages/durable/test/session-deferred-commits.test.ts @@ -0,0 +1,653 @@ +// Deferred commits (`CommitPersistence`): what a deferred commit holds in memory, what stores it, and what Storage and +// each kind of listener then sees. +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import type { JsonValue } from "@earendil-works/chord"; +import { applyImmutable } from "@earendil-works/chord/delta"; +import { + type CommitChange, + type CommitOptions, + type CommitPublication, + type ConversationId, + type DocumentId, + defineDoc, + type JsonObject, + ReadOnlySession, + type SetTimer, + type StorageWrite, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import { type ObservedCommit, SessionImpl } from "../src/session/session.ts"; +import { ControlledStorage, context, createConversation, flush } from "./session-support.ts"; + +type Text = { text: string; base?: boolean; explode?: boolean; keys?: Record }; + +/** A latest, initial-fork conversation document, the kind a deferred commit may hold. */ +const TextDoc = defineDoc({ + kind: "test.deferred.text", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ text: "" }), + checkpointWhen: (value) => { + if (value.explode === true) throw new Error("checkpoint failed"); + return value.base === true; + }, +}); + +/** `TextDoc` at version 2, which a reload migrates to. */ +const TextDocV2 = defineDoc({ + kind: "test.deferred.text", + version: 2, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ text: "", migrated: true }), + // `migrate()` gets the older version's untyped value: version 1's `Text`. + migrate: (value) => ({ ...(value as Text), migrated: true }), + checkpointWhen: () => false, +}); + +const RewindableDoc = defineDoc({ + kind: "test.deferred.rewindable", + version: 1, + scope: "conversation", + history: "rewindable", + fork: "asOf", + initial: () => ({ text: "" }), + checkpointWhen: () => false, +}); + +const CurrentForkDoc = defineDoc({ + kind: "test.deferred.current-fork", + version: 1, + scope: "conversation", + history: "latest", + fork: "current", + initial: () => ({ text: "" }), + checkpointWhen: () => false, +}); + +const SessionTextDoc = defineDoc({ + kind: "test.deferred.session", + version: 1, + scope: "session", + initial: () => ({ text: "" }), + checkpointWhen: () => false, +}); + +/** Chord's `MAX_PENDING_OPS`: held operations past which a store writes the whole value. */ +const MAX_PENDING_OPS = 10_000; + +const DEFERRED: CommitOptions = { persistence: { kind: "deferred" } }; + +const IMMEDIATE: CommitOptions = { persistence: { kind: "immediate" } }; + +const within = (withinMs: number): CommitOptions => ({ persistence: { kind: "deferred", withinMs } }); + +/** One foreground timer the Session armed. */ +type Armed = { readonly callback: () => void; readonly delayMs: number; disarmed: boolean; fired: boolean }; + +/** + * A Session over controlled storage with a manual clock and manual foreground timers (background ones are declined, so + * commits evict), recording what `subscribeCommits()` and the Session's own observers get. + */ +async function open(options: { readonly retentionMs?: number; readonly persistWithinMs?: number } = {}) { + const clock = { now: 1_000 }; + const timers: Armed[] = []; + const setTimer: SetTimer = (callback, delayMs, { background }) => { + if (background) return undefined; + const timer: Armed = { callback, delayMs, disarmed: false, fired: false }; + timers.push(timer); + return () => { + timer.disarmed = true; + }; + }; + const storage = new ControlledStorage(); + const session = new SessionImpl( + storage, + () => clock.now, + () => options.retentionMs ?? 600_000, + setTimer, + false, + () => options.persistWithinMs, + ); + const stored: CommitPublication[] = []; + session.subscribeCommits((publication) => { + stored.push(publication); + }); + const observed: ObservedCommit[] = []; + session.observeCommits((publication) => { + observed.push(publication); + }); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + await tx.doc(TextDoc, conversationId); + }, context); + const recordId = (await storage.findDocument(address(conversationId), "current", context))!.id; + return { + clock, + storage, + session, + stored, + observed, + conversationId, + recordId, + /** Timers armed and neither disarmed nor fired. */ + armed: () => timers.filter((timer) => !timer.disarmed && !timer.fired), + /** Fire every armed timer and let the line run what they queued. */ + async fire(): Promise { + for (const timer of timers) { + if (timer.disarmed || timer.fired) continue; + timer.fired = true; + timer.callback(); + } + await session.readOnLine(async () => {}); + }, + /** Batches admitted to Storage from here on. */ + mark(): () => (readonly StorageWrite[])[] { + const from = storage.commits.length; + return () => storage.commits.slice(from); + }, + async storedText(id: DocumentId = recordId): Promise { + return (await storage.document(id, "current", context))?.value; + }, + sizes: () => session.documentCacheSizes, + }; +} + +function address(conversationId: ConversationId) { + return { kind: TextDoc.definition.kind, scope: { kind: "conversation", conversationId } } as const; +} + +function append( + session: SessionImpl, + conversationId: ConversationId, + text: string, + options: CommitOptions = DEFERRED, +): Promise { + return session.commit( + async (tx) => { + (await tx.doc(TextDoc, conversationId)).text += text; + }, + context, + options, + ); +} + +function changesOf( + batches: readonly (readonly StorageWrite[])[], +): Extract[] { + return batches.flat().filter((write) => write.type === "document.change"); +} + +describe("holding a deferred commit", () => { + it("adopts a commit that only changes an existing latest, initial-fork conversation document in memory, unstored", async () => { + const { session, storage, stored, observed, conversationId, sizes, storedText } = await open(); + const admitted = storage.commits.length; + const publications = stored.length; + await append(session, conversationId, "a"); + await append(session, conversationId, "b"); + expect(storage.commits.length).toBe(admitted); + expect(stored.length).toBe(publications); + expect(await session.snapshot(TextDoc, conversationId, context)).toEqual({ text: "ab" }); + expect(await storedText()).toEqual({ text: "" }); + expect(sizes().held).toBe(1); + expect(observed.slice(-2).map((publication) => publication.kind)).toEqual(["memory", "memory"]); + }); + + it("stores a deferred commit at once when it writes anything else", async () => { + const { session, storage, conversationId, sizes } = await open(); + const stores = async (change: Parameters[0]): Promise => { + const admitted = storage.commits.length; + await session.commit(change, context, DEFERRED); + return storage.commits.length > admitted; + }; + // An entry next to the held document's change. + expect( + await stores(async (tx) => { + (await tx.doc(TextDoc, conversationId)).text += "x"; + await tx.appendEntry(conversationId, { kind: "test.note" }); + }), + ).toBe(true); + // Creation, then other document semantics. + expect(await stores(async (tx) => void (await tx.doc(RewindableDoc, conversationId)))).toBe(true); + expect( + await stores(async (tx) => { + (await tx.doc(RewindableDoc, conversationId)).text = "r"; + }), + ).toBe(true); + expect(await stores(async (tx) => void (await tx.doc(CurrentForkDoc, conversationId)))).toBe(true); + expect( + await stores(async (tx) => { + (await tx.doc(CurrentForkDoc, conversationId)).text = "c"; + }), + ).toBe(true); + expect(await stores(async (tx) => void (await tx.doc(SessionTextDoc)))).toBe(true); + expect( + await stores(async (tx) => { + (await tx.doc(SessionTextDoc)).text = "s"; + }), + ).toBe(true); + // A held document next to an immediate one: the whole commit is stored. + expect( + await stores(async (tx) => { + (await tx.doc(TextDoc, conversationId)).text += "y"; + (await tx.doc(SessionTextDoc)).text += "s"; + }), + ).toBe(true); + // Retirement. + expect(await stores((tx) => tx.retireDoc(CurrentForkDoc, conversationId))).toBe(true); + expect(sizes().held).toBe(0); + // The control: the same change alone is held. + expect( + await stores(async (tx) => { + (await tx.doc(TextDoc, conversationId)).text += "z"; + }), + ).toBe(false); + }); + + it("stores a deferred commit that migrates its document at once, as a base", async () => { + const { session, conversationId, mark, sizes } = await open(); + const batches = mark(); + await session.commit( + async (tx) => { + (await tx.doc(TextDocV2, conversationId)).text = "v2"; + }, + context, + DEFERRED, + ); + expect(changesOf(batches()).map((write) => write.content.kind)).toEqual(["base"]); + expect(sizes().held).toBe(0); + }); + + it("rejects a deferred commit on a read-only Session like any commit with writes", async () => { + const { storage, conversationId } = await open(); + const viewer = new SessionImpl(storage, Date.now, () => 600_000, undefined, true); + await expect(append(viewer, conversationId, "a")).rejects.toBeInstanceOf(ReadOnlySession); + }); + + it("rejects a withinMs that is not a number of milliseconds, 0 or more", async () => { + const { session, conversationId } = await open(); + await expect(append(session, conversationId, "a", within(-1))).rejects.toBeInstanceOf(TypeError); + await expect(append(session, conversationId, "a", within(Number.NaN))).rejects.toBeInstanceOf(TypeError); + await expect(append(session, conversationId, "a", { persistence: { kind: "immediate" } })).resolves.toBe( + undefined, + ); + }); +}); + +describe("storing held changes", () => { + it("stores them first, folded with its own, in a commit that writes the document", async () => { + const { session, conversationId, mark, sizes, storedText } = await open(); + await append(session, conversationId, "a"); + await append(session, conversationId, "b"); + const batches = mark(); + await append(session, conversationId, "c", IMMEDIATE); + expect(batches()).toHaveLength(1); + expect(changesOf(batches()).map((write) => write.content)).toEqual([ + { version: 1, kind: "delta", ops: [["a", ["text"], "abc"]] }, + ]); + expect(await storedText()).toEqual({ text: "abc" }); + expect(sizes().held).toBe(0); + }); + + it("stores them in a commit that only reads the document, with its other writes", async () => { + const { session, conversationId, mark, storedText } = await open(); + await append(session, conversationId, "held"); + const batches = mark(); + await session.commit(async (tx) => { + const text = (await tx.doc(TextDoc, conversationId)).text; + await tx.appendEntry(conversationId, { kind: "test.derived", data: text }); + }, context); + expect(batches()).toHaveLength(1); + expect( + batches()[0]! + .map((write) => write.type) + .sort(), + ).toEqual(["document.change", "entry"]); + expect(await storedText()).toEqual({ text: "held" }); + }); + + it("leaves them held through commits that neither read nor write the document, or write nothing", async () => { + const { session, conversationId, mark, sizes } = await open(); + await append(session, conversationId, "held"); + const batches = mark(); + await session.commit((tx) => tx.appendEntry(conversationId, { kind: "test.unrelated" }).then(() => {}), context); + await session.commit(async (tx) => void (await tx.doc(TextDoc, conversationId)), context); + expect(changesOf(batches())).toEqual([]); + expect(sizes().held).toBe(1); + }); + + it("keeps them when a commit that reads the document fails before Storage", async () => { + const { session, conversationId, sizes, storedText, fire } = await open(); + await append(session, conversationId, "held"); + await expect( + session.commit(async (tx) => { + (await tx.doc(TextDoc, conversationId)).text += "!"; + await tx.appendEntry(conversationId, { kind: "test.note" }); + throw new Error("callback failed"); + }, context), + ).rejects.toThrow("callback failed"); + // Its checkpoint predicate throws: settlement fails after every other step. + await expect( + session.commit(async (tx) => { + (await tx.doc(TextDoc, conversationId)).explode = true; + }, context), + ).rejects.toThrow("checkpoint failed"); + expect(sizes().held).toBe(1); + await fire(); + expect(await storedText()).toEqual({ text: "held" }); + }); + + it("stores every held document in one transaction when the earliest deadline comes", async () => { + const { session, conversationId, mark, armed, fire, stored, sizes, storage } = await open(); + const other = await createConversation(session); + await session.commit(async (tx) => void (await tx.doc(TextDoc, other)), context); + await append(session, conversationId, "one"); + await append(session, other, "two"); + expect(armed().map((timer) => timer.delayMs)).toEqual([1_000]); + const publications = stored.length; + const batches = mark(); + await fire(); + expect(batches()).toHaveLength(1); + expect(changesOf(batches())).toHaveLength(2); + expect(stored.length).toBe(publications + 1); + expect(sizes().held).toBe(0); + expect(armed()).toEqual([]); + const otherId = (await storage.findDocument(address(other), "current", context))!.id; + expect((await storage.document(otherId, "current", context))?.value).toEqual({ text: "two" }); + }); + + it("arms the earliest deadline of the held commits, each with its withinMs or the default", async () => { + const { session, conversationId, armed, clock } = await open({ persistWithinMs: 2_000 }); + await append(session, conversationId, "a"); + expect(armed().map((timer) => timer.delayMs)).toEqual([2_000]); + clock.now += 500; + // Earlier: the timer moves up. + await append(session, conversationId, "b", within(100)); + expect(armed().map((timer) => timer.delayMs)).toEqual([100]); + // Later: the deadline stays. + await append(session, conversationId, "c", within(60_000)); + expect(armed().map((timer) => timer.delayMs)).toEqual([100]); + }); + + it("arms no timer for an infinite window, and moves the timer on when a commit stores the held document", async () => { + const { session, conversationId, armed, sizes } = await open(); + await append(session, conversationId, "a", within(Number.POSITIVE_INFINITY)); + expect(armed()).toEqual([]); + expect(sizes().held).toBe(1); + await append(session, conversationId, "b"); + expect(armed().map((timer) => timer.delayMs)).toEqual([1_000]); + await append(session, conversationId, "c", IMMEDIATE); + expect(armed()).toEqual([]); + expect(sizes().held).toBe(0); + }); + + it("stores them at close, after the commits admitted before it, and closes Storage when that store fails", async () => { + const { session, storage, conversationId, recordId } = await open(); + await append(session, conversationId, "held"); + await session.close(context); + expect((await storage.reopen().document(recordId, "current", context))?.value).toEqual({ text: "held" }); + + const failing = await open(); + await append(failing.session, failing.conversationId, "lost"); + const failure = new Error("disk gone"); + failing.storage.failNextCommit(failure); + await expect(failing.session.close(context)).rejects.toBe(failure); + expect(await failing.session.closed).toEqual({ reason: "failed", error: failure }); + // Closed: the backend refuses calls until reopened. + await expect(failing.storage.document(failing.recordId, "current", context)).rejects.toThrow("closed"); + expect((await failing.storage.reopen().document(failing.recordId, "current", context))?.value).toEqual({ + text: "", + }); + }); + + it("stores nothing once the Session failed", async () => { + const { session, storage, conversationId, recordId, mark } = await open(); + await append(session, conversationId, "lost"); + const batches = mark(); + session.fail(new Error("crash")); + await session.closed; + expect(batches()).toEqual([]); + expect((await storage.reopen().document(recordId, "current", context))?.value).toEqual({ text: "" }); + }); + + it("stores them before unloadDocuments() drops the trackers", async () => { + const { session, conversationId, storedText, sizes } = await open(); + await append(session, conversationId, "held"); + await session.unloadDocuments(); + expect(await storedText()).toEqual({ text: "held" }); + expect(sizes()).toMatchObject({ cached: 0, held: 0 }); + expect(await session.snapshot(TextDoc, conversationId, context)).toEqual({ text: "held" }); + }); + + it("stores them before a token of another definition version reloads the document", async () => { + const { session, conversationId, storedText, mark } = await open(); + await append(session, conversationId, "held"); + const batches = mark(); + expect(await session.snapshot(TextDocV2, conversationId, context)).toEqual({ text: "held", migrated: true }); + expect(changesOf(batches()).map((write) => write.content.kind)).toEqual(["delta"]); + expect(await storedText()).toEqual({ text: "held" }); + // The same inside a commit, which then migrates with a base. + await session.commit(async (tx) => void (await tx.doc(TextDoc, conversationId)), context); + await append(session, conversationId, "+"); + await session.commit(async (tx) => { + (await tx.doc(TextDocV2, conversationId)).text += "v2"; + }, context); + expect(await storedText()).toEqual({ text: "held+v2", migrated: true }); + }); + + it("stores them before the document's retirement, in the retiring commit", async () => { + const { session, conversationId, mark, sizes } = await open(); + await append(session, conversationId, "held"); + const batches = mark(); + await session.commit((tx) => tx.retireDoc(TextDoc, conversationId), context); + expect(batches()[0]!.map((write) => write.type)).toEqual(["document.change", "document.retire"]); + expect(changesOf(batches())[0]!.content).toEqual({ version: 1, kind: "delta", ops: [["a", ["text"], "held"]] }); + expect(sizes().held).toBe(0); + expect(await session.snapshot(TextDoc, conversationId, context)).toBeUndefined(); + }); +}); + +describe("bases and overflow", () => { + it("stores a base when a held commit selected one, and counts the store as one delta otherwise", async () => { + const { session, conversationId, mark, fire } = await open(); + const batches = mark(); + await session.commit( + async (tx) => { + const doc = await tx.doc(TextDoc, conversationId); + doc.text = "x"; + doc.base = true; + }, + context, + DEFERRED, + ); + await append(session, conversationId, "y"); + await fire(); + expect(changesOf(batches()).map((write) => write.content)).toEqual([ + { version: 1, kind: "base", value: { text: "xy", base: true } }, + ]); + }); + + it("stores the whole value once more than MAX_PENDING_OPS operations are held", async () => { + const { session, conversationId, mark, fire, stored } = await open(); + await overflow(session, conversationId); + const batches = mark(); + await fire(); + const value = (await session.snapshot(TextDoc, conversationId, context))!; + expect(changesOf(batches()).map((write) => write.content)).toEqual([{ version: 1, kind: "base", value }]); + // A durable listener replaces its copy with the stored value. + expect(stored.at(-1)!.changes).toMatchObject([{ type: "document", ops: [["r", value]] }]); + }); + + it("stores a base of the commit's own value when it stores an overflowed hold", async () => { + const { session, conversationId, mark } = await open(); + await overflow(session, conversationId); + await append(session, conversationId, "a"); + const batches = mark(); + await append(session, conversationId, "b", IMMEDIATE); + const value = (await session.snapshot(TextDoc, conversationId, context))!; + expect(value.text).toBe("ab"); + expect(changesOf(batches()).map((write) => write.content)).toEqual([{ version: 1, kind: "base", value }]); + }); +}); + +/** + * Hold more than `MAX_PENDING_OPS` operations: deferred commits of one set per key, each below the tracker's own limit, + * past which a commit is one root replacement. + */ +async function overflow(session: SessionImpl, conversationId: ConversationId): Promise { + await session.commit(async (tx) => { + (await tx.doc(TextDoc, conversationId)).keys = {}; + }, context); + for (let from = 0; from <= MAX_PENDING_OPS; from += 4_000) { + await session.commit( + async (tx) => { + const keys = (await tx.doc(TextDoc, conversationId)).keys!; + for (let index = from; index < from + 4_000; index++) keys[`k${index}`] = index; + }, + context, + DEFERRED, + ); + } +} + +describe("publications", () => { + it("give subscribeCommits() exactly what was stored, and nothing for a held commit", async () => { + const { session, storage, conversationId, recordId, stored, fire } = await open(); + const from = storage.commits.length; + const publications = stored.length; + await append(session, conversationId, "a"); + await append(session, conversationId, "b"); + await append(session, conversationId, "c", IMMEDIATE); + await append(session, conversationId, "d"); + await fire(); + await append(session, conversationId, "e"); + await session.commit(async (tx) => { + await tx.doc(TextDoc, conversationId); + await tx.appendEntry(conversationId, { kind: "test.note" }); + }, context); + await append(session, conversationId, "f"); + await session.close(context); + const batches = storage.commits.slice(from); + const published = stored.slice(publications); + expect(published).toHaveLength(batches.length); + // A replica built from the publications alone always equals what Storage holds. + let fromStorage: JsonValue = { text: "" }; + let fromPublications: JsonValue = { text: "" }; + for (const [index, batch] of batches.entries()) { + const write = changesOf([batch]).find((candidate) => candidate.id === recordId)!; + const change = published[index]!.changes.find( + (candidate): candidate is Extract => + candidate.type === "document" && candidate.record.id === recordId, + )!; + fromStorage = + write.content.kind === "base" ? write.content.value : applyImmutable(fromStorage, write.content.ops); + fromPublications = applyImmutable(fromPublications, change.ops); + expect(fromPublications).toEqual(fromStorage); + expect(change.value).toEqual(fromStorage); + } + expect(fromStorage).toEqual({ text: "abcdef" }); + expect(published.map((publication) => publication.changes.map((change) => change.type).sort())).toEqual([ + ["document"], + ["document"], + ["document", "entry"], + ["document"], + ]); + }); + + it("show every held commit to watches and the Session's own observers, once", async () => { + const { session, conversationId, observed, fire } = await open(); + const watch = (await session.watchDoc(TextDoc, conversationId, context))!; + const values: string[] = []; + watch.start(async (value) => { + values.push(value?.text ?? ""); + }); + const from = observed.length; + for (const text of ["a", "b", "c"]) { + await append(session, conversationId, text); + await flush(); + } + await append(session, conversationId, "d", IMMEDIATE); + await flush(); + await fire(); + expect(values).toEqual(["a", "ab", "abc", "abcd"]); + const seen = observed.slice(from); + expect(seen.map((publication) => publication.kind)).toEqual(["memory", "memory", "memory", "stored"]); + // The storing commit shows its own operations; the held ones were shown already. The timer showed nothing. + expect( + seen.map((publication) => publication.changes.map((change) => change.type === "document" && change.ops)), + ).toEqual([ + [[["a", ["text"], "a"]]], + [[["a", ["text"], "b"]]], + [[["a", ["text"], "c"]]], + [[["a", ["text"], "d"]]], + ]); + await watch.stop(); + }); +}); + +describe("pinning", () => { + it("keeps a held document cached with retention 0 until it is stored, then lets it go", async () => { + const { session, conversationId, sizes, fire } = await open({ retentionMs: 0 }); + await session.readOnLine(async () => {}); + expect(sizes()).toMatchObject({ cached: 0, held: 0, attached: 0 }); + await append(session, conversationId, "held"); + await session.readOnLine(async () => {}); + expect(sizes()).toMatchObject({ cached: 1, held: 1, attached: 1 }); + await fire(); + await session.commit(async () => {}, context); + expect(sizes()).toMatchObject({ cached: 0, held: 0, attached: 0 }); + expect(await session.snapshot(TextDoc, conversationId, context)).toEqual({ text: "held" }); + }); + + it("retains a held revision only until it is stored", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const collect = async (): Promise => { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + }; + const { session, conversationId, fire, stored, observed } = await open({ retentionMs: 0 }); + await append(session, conversationId, "x".repeat(1_000)); + const revision = new WeakRef((await session.snapshot(TextDoc, conversationId, context))!); + await collect(); + expect(revision.deref()).toBeDefined(); + await fire(); + await session.commit(async () => {}, context); + // The recorded publications would keep it. + stored.length = 0; + observed.length = 0; + await collect(); + expect(revision.deref()).toBeUndefined(); + }); +}); + +describe("crash", () => { + it("leaves Storage at the last stored state, which memory had, and nothing held after reopen", async () => { + const { session, storage, conversationId, recordId, fire } = await open(); + const states: string[] = []; + session.observeCommits((publication) => { + for (const change of publication.changes) { + // Publications carry untyped document values; this one is `TextDoc`'s. + if (change.type === "document" && change.record.id === recordId) states.push((change.value as Text).text); + } + }); + await append(session, conversationId, "a"); + await fire(); + await append(session, conversationId, "b"); + await append(session, conversationId, "c"); + session.fail(new Error("crash")); + await session.closed; + const reopened = new SessionImpl(storage.reopen(), Date.now, () => 600_000); + const value = (await reopened.snapshot(TextDoc, conversationId, context))!; + expect(value).toEqual({ text: "a" }); + expect(states).toContain(value.text); + expect(reopened.documentCacheSizes.held).toBe(0); + await reopened.close(context); + }); +}); diff --git a/packages/durable/test/session-support.ts b/packages/durable/test/session-support.ts index b5c4df8b6..e3bd17d44 100644 --- a/packages/durable/test/session-support.ts +++ b/packages/durable/test/session-support.ts @@ -1,6 +1,7 @@ import type { Context } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; import { + type CommitChange, type CommitPublication, type ConversationId, type DocumentAddress, @@ -11,9 +12,10 @@ import { type Id, MemoryStorage, type Seq, + type Session, type StorageWrite, } from "@earendil-works/pi-durable"; -import { SessionImpl } from "../src/session/session.ts"; +import { type ObservedCommit, SessionImpl } from "../src/session/session.ts"; export const context: Context = BACKGROUND_CONTEXT; @@ -138,14 +140,26 @@ export function openTestSession(): { return { storage, session, publications }; } -export function documentChanges( - publication: CommitPublication, -): readonly Extract[] { +export function documentChanges(publication: { + readonly changes: readonly CommitChange[]; +}): readonly Extract[] { return publication.changes.filter( (change): change is Extract => change.type === "document", ); } +/** + * Observe every commit as memory adopts it, deferred ones before they are stored, as views and document watches do; + * `subscribeCommits()` sees only what was stored. + */ +export function observeCommits( + session: Session, + listener: (publication: ObservedCommit, context: Context) => void, +): () => void { + // `observeCommits()` is internal, so the public type lacks it; every Session and Harness is a `SessionImpl`. + return (session as unknown as SessionImpl).observeCommits(listener); +} + export function documentCopyChanges( publication: CommitPublication, ): readonly Extract[] { From 38ff99177fba21e9c57a670f4e95adcf407c99c7 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 24 Oct 2026 22:48:11 +0200 Subject: [PATCH 005/271] fix(durable): continue runs after tool handoffs --- packages/durable/CHANGELOG.md | 1 + packages/durable/README.md | 4 +- packages/durable/docs/spec.md | 46 +++-- packages/durable/src/harness/generation.ts | 18 +- packages/durable/src/harness/types.ts | 4 + packages/durable/test/harness-inbox.test.ts | 184 ++++++++++++++++++-- 6 files changed, 219 insertions(+), 38 deletions(-) diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index eff3b87a9..15e4d75e4 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -22,6 +22,7 @@ ### Fixed +- Fixed a tool round with `control.handoff` ending the run on the tool-calling answer: the handoff's `pi.reset` entry is written and the same run continues with a new generation in the new context, so the submission settles with the eventual answer. Queued writes and selected steers are placed after the handoff text, follow-ups wait for the run's final boundary, and a queued reset still ends the run `unanswered` with reason `reset`. `terminate` remains the way to end a round without another model request. - Fixed a conversation watcher (`watchEvents()`, conversation views) holding a copy of a streamed text or tool call per partial commit, through Chord append ops that kept the whole string alive; a 140 KB answer followed by a 140 KB tool call held well over 100 MB ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed a nested call's error summary keeping the call's whole output alive through a slice of it ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed task span error messages and codemode call previews keeping the whole text they were cut from alive ([#9265](https://github.com/earendil-works/pi/issues/9265)) diff --git a/packages/durable/README.md b/packages/durable/README.md index 452b7c740..0ce9885bb 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -538,7 +538,9 @@ await root.reset(undefined, context); // start await root.reset("We were fixing the flaky login test. Continue.", context); // start from a handoff note ``` -While busy, the reset is queued like a write. When it is placed during a tool round, the current run ends. A tool can request the same with `control: { handoff: "..." }`. +While busy, the reset is queued like a write. When it is placed during a tool round, the current run ends: its inputs settle `unanswered` with reason `reset`, and queued inputs run in the new context. + +A tool can return `control: { handoff: "..." }` to write the same reset with its text, but the run continues: the next model request starts in the new context, and the submission settles with that eventual answer. Queued writes and steers land after the handoff text, follow-ups wait for the run's end, and a queued `reset()` still ends the run. When every call of the round also returns `terminate: true`, the run ends after the handoff instead. ## Compaction diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index 1b95481b1..3af8ff115 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -2730,8 +2730,11 @@ A reset placed at a `final` boundary follows the answer, so the inputs are already `done`. At ordinary `postTools`, generation continues even with no queued trigger; -selected steer IDs join that continuation. A terminating or handoff tool round -(section 8.5) uses final behavior instead. At `final`, the current run's placed +selected steer IDs join that continuation. A handoff tool round (section 8.5) +first appends its reset and then applies this ordinary `postTools` boundary, so +queued writes and selected steers land in the new context and follow-ups wait +for the run's final boundary. A terminating tool round uses final behavior +instead. At `final`, the current run's placed input submissions settle first, unless an `onYield` continuation keeps them open (below); selected user IDs start one successor generation with those IDs as the new run's inputs. Writes never trigger generation by @@ -2739,10 +2742,9 @@ themselves. A final boundary without user triggers leaves the conversation idle, except for an `onYield` continuation (section 8.3), which applies only when the boundary selected no user item and no reset. -Only successful run ends apply the final boundary: an answer, `terminate`, or -`handoff`. Failure, a run task's abort handler, fault, and orphan settle the -run's inputs `unanswered` and leave the inbox alone (`Conversation.abort()` -separately withdraws queued user items); the queued items stay visible in +Only successful run ends apply the final boundary: an answer or `terminate`. +Failure, a run task's abort handler, fault, and orphan settle the run's inputs +`unanswered` and leave the inbox alone (`Conversation.abort()` separately withdraws queued user items); the queued items stay visible in `pi.inbox` until the next input or queued write runs a boundary, or their withdrawal. Selected and stale items are removed positionally while retained item order is @@ -3703,10 +3705,13 @@ generation's `tools` phase (section 8.5) adds the named tools to the conversation's stored tools filter; they take effect at the next preparation. The round terminates only when every result of the round requests `terminate`, as in the pi agent loop; the `tools` phase then uses a final boundary. Any -`handoff` in the round, the last one in call order when several ask, ends the -run the same way after appending a `pi.reset` entry with `head: "self"` and the -handoff text as a user message (section 8.1), exactly what `reset(handoff)` -writes. +`handoff` in the round, the last one in call order when several ask, appends a +`pi.reset` entry with `head: "self"` and the handoff text as a user message +(section 8.1), exactly what `reset(handoff)` writes, and the run continues: the +next generation answers in the new context, and the run's inputs settle with +that eventual answer or error, not with the tool-calling answer. A later round +may hand off again. A round that also terminates writes the reset and then ends +the run as `terminate` does. A result's `usage` is stored on the tool-result message and added to the conversation's `pi.usage.tools[toolName]` in the result commit (section 8.6). A @@ -4516,15 +4521,20 @@ and then commits once: no effect for others. Names are written without consulting the registry, so a tool that is not registered now is offered once it is again. The next preparation offers the tool when it resolves and the model may call it. -- When every result of the round requests `terminate`, or any requests - `handoff`, it appends the handoff's `pi.reset` entry, if any, settles the - run's inputs `done` with the tool-calling answer, removes `run` and `tools`, - and applies the final boundary. +- When any result requests `handoff`, it first appends the `pi.reset` entry of + the last one in call order and advances the boundary's active range start to + it, so the boundary below places its writes and user items after the handoff + text, and a queued head write targeting an entry before it is stale. +- When every result of the round requests `terminate`, it settles the run's + inputs `done` with the tool-calling answer, removes `run` and `tools`, and + applies the final boundary. - Otherwise it removes `tools` and applies the `postTools` boundary. When that - boundary placed a reset, the run's inputs settle `unanswered` with `reset` and - selected user items start a successor run (section 6). Otherwise selected - steer IDs join `pi.live.run`, and it creates the next generation, owned by the - conversation, and hands it the run. + boundary placed a queued reset, which wins over a handoff written before it, + the run's inputs settle `unanswered` with `reset` and selected user items + start a successor run (section 6). Otherwise selected steer IDs join + `pi.live.run`, and it creates the next generation, owned by the conversation, + and hands it the run; after a handoff, that generation prepares and requests + in the new context. It completes with `{ entryId }` of the tool-calling answer. Its abort handler runs only after its tool tasks are terminal (section 5.5). For every call still diff --git a/packages/durable/src/harness/generation.ts b/packages/durable/src/harness/generation.ts index 087ec9a97..11bf946f2 100644 --- a/packages/durable/src/harness/generation.ts +++ b/packages/durable/src/harness/generation.ts @@ -690,8 +690,9 @@ async function startToolRound( } /** - * The round's tools are terminal: apply their controls and either end the run at the final boundary (`terminate`, - * `handoff`, or a queued reset) or hand it to the next generation at the `postTools` boundary (spec §8.5). + * The round's tools are terminal: apply their controls, append a `handoff`'s reset, and then end the run at the final + * boundary (`terminate`), end it at a queued reset, or hand it to the next generation at the `postTools` boundary, in + * the handoff's new context when there is one (spec §8.5). */ async function finishToolRound( runtime: Runtime, @@ -725,12 +726,13 @@ async function finishToolRound( if (added.length > 0) await addTools(tx, conversationId, added); const live = await tx.doc(LiveDoc, conversationId); const now = runtime.now(); - if (terminate || handoff !== undefined) { - if (handoff !== undefined) { - const message = { role: "user", content: handoff, timestamp: now } as const; - const entry = await tx.appendEntry(ResetEntry, conversationId, { head: "self", model: [message] }); - boundary.head = entry.id; - } + if (handoff !== undefined) { + // The handoff's reset precedes every queued item, so queued writes and selected steers land in its context. + const message = { role: "user", content: handoff, timestamp: now } as const; + const entry = await tx.appendEntry(ResetEntry, conversationId, { head: "self", model: [message] }); + boundary.head = entry.id; + } + if (terminate) { const { users } = await applyBoundary(tx, boundary, "final", now); endRun(tx, live, runtime.taskId, { status: "done", answer: assistant }); if (users.length > 0) await startRun(tx, conversationId, live, users); diff --git a/packages/durable/src/harness/types.ts b/packages/durable/src/harness/types.ts index fbe636dd7..748696f8c 100644 --- a/packages/durable/src/harness/types.ts +++ b/packages/durable/src/harness/types.ts @@ -142,6 +142,10 @@ export type ToolControl = { readonly addTools?: readonly string[]; /** End the run without another model request when every result of the round sets it to `true`; `false` is a no-op. */ readonly terminate?: boolean; + /** + * Start a new context with this text as its user message, as `reset(handoff)` writes, and continue the run there. + * The last handoff in call order wins. + */ readonly handoff?: string; }; diff --git a/packages/durable/test/harness-inbox.test.ts b/packages/durable/test/harness-inbox.test.ts index 19a719286..396c94e59 100644 --- a/packages/durable/test/harness-inbox.test.ts +++ b/packages/durable/test/harness-inbox.test.ts @@ -82,6 +82,14 @@ function queuedNote(text: string) { return { kind: "note", model: [{ role: "user", content: text, timestamp: 0 }] } as const; } +/** A faux response that records the request's messages as `role:text` and answers `text`. */ +function recordRequest(requests: string[][], text: string): FauxResponseStep { + return (request) => { + requests.push(request.messages.map((message) => `${message.role}:${textOf(message) ?? ""}`)); + return answer(text); + }; +} + const HOLD = fauxAssistantMessage([fauxToolCall("hold", {}, { id: "c1" })], { stopReason: "toolUse" }); async function status(submission: Submission): Promise { @@ -239,11 +247,7 @@ describe("inbox", () => { const gate = deferred(); holdTool(setup, gate); const requests: string[][] = []; - const record: FauxResponseStep = (request) => { - requests.push(request.messages.map((message) => `${message.role}:${textOf(message) ?? ""}`)); - return answer("fresh"); - }; - setup.faux.setResponses([HOLD, record]); + setup.faux.setResponses([HOLD, recordRequest(requests, "fresh")]); const { harness, root } = await openChat(new MemoryStorage(), setup); const input = await root.submit({ type: "input", content: "a" }, context); await toolRunning(harness, root); @@ -348,11 +352,164 @@ describe("inbox", () => { await harness.close(context); }); - it("ends the run with a pi.reset entry when a tool requests a handoff", async () => { + it("continues the run in a new context when a tool requests a handoff", async () => { const setup = chatSetup(); const gate = deferred(); gate.resolve(); holdTool(setup, gate, { output: [], control: { handoff: "continue here" } }); + const requests: string[][] = []; + setup.faux.setResponses([HOLD, recordRequest(requests, "after handoff")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const settled = await (await root.submit({ type: "input", content: "a" }, context)).wait(context); + const entries = await allEntries(root); + expect(settled).toMatchObject({ status: "done", answer: entries.at(-1)!.id }); + expect(transcript(entries)).toEqual([ + "pi.user:a", + "pi.assistant", + "pi.tool-result", + "pi.reset:continue here", + "pi.assistant:after handoff", + ]); + const reset = entries.find((entry) => ResetEntry.is(entry))!; + expect(reset.head).toBe(reset.id); + expect(requests).toEqual([["system:", "user:continue here"]]); + expect(setup.faux.state.callCount).toBe(2); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + await harness.close(context); + }); + + it("places queued writes and steers after the handoff text and adds the steers to the continuing run", async () => { + const setup = chatSetup(); + const gate = deferred(); + holdTool(setup, gate, { output: [], control: { handoff: "continue here" } }); + const requests: string[][] = []; + setup.faux.setResponses([HOLD, recordRequest(requests, "after handoff")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const input = await root.submit({ type: "input", content: "a" }, context); + await toolRunning(harness, root); + const steer = await root.submit({ type: "input", content: "s", whenBusy: "steer" }, context); + const write = await root.submit({ type: "write", entry: queuedNote("w") }, context); + gate.resolve(); + await input.wait(context); + const entries = await allEntries(root); + expect(transcript(entries)).toEqual([ + "pi.user:a", + "pi.assistant", + "pi.tool-result", + "pi.reset:continue here", + "note:w", + "pi.user:s", + "pi.assistant:after handoff", + ]); + expect(requests).toEqual([["system:", "user:continue here", "user:w", "user:s"]]); + for (const submission of [input, steer]) { + expect(await status(submission)).toMatchObject({ status: "done", answer: entries.at(-1)!.id }); + } + expect(await status(write)).toMatchObject({ status: "done" }); + await harness.close(context); + }); + + it("holds a follow-up queued across a handoff until the continuing run's final boundary", async () => { + const setup = chatSetup(); + const gate = deferred(); + holdTool(setup, gate, { output: [], control: { handoff: "continue here" } }); + const after = gated(answer("after handoff")); + setup.faux.setResponses([HOLD, after.step, answer("follow-up")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const input = await root.submit({ type: "input", content: "a" }, context); + await toolRunning(harness, root); + const followUp = await root.submit({ type: "input", content: "f" }, context); + gate.resolve(); + await after.reached; + expect((await status(input)).status).toBe("placed"); + expect(await inbox(harness, root)).toEqual([[followUp.id, "followUp"]]); + after.release(); + await followUp.wait(context); + const entries = await allEntries(root); + expect(transcript(entries)).toEqual([ + "pi.user:a", + "pi.assistant", + "pi.tool-result", + "pi.reset:continue here", + "pi.assistant:after handoff", + "pi.user:f", + "pi.assistant:follow-up", + ]); + const answers = entries.filter((entry) => entry.kind === "pi.assistant"); + expect(await status(input)).toMatchObject({ status: "done", answer: answers[1]!.id }); + expect(await status(followUp)).toMatchObject({ status: "done", answer: answers[2]!.id }); + await harness.close(context); + }); + + it("ends a handed-off run at a queued reset and runs earlier follow-ups after it", async () => { + const setup = chatSetup(); + const gate = deferred(); + holdTool(setup, gate, { output: [], control: { handoff: "continue here" } }); + const requests: string[][] = []; + setup.faux.setResponses([HOLD, recordRequest(requests, "fresh")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const input = await root.submit({ type: "input", content: "a" }, context); + await toolRunning(harness, root); + const followUp = await root.submit({ type: "input", content: "f" }, context); + await root.reset(undefined, context); + gate.resolve(); + await followUp.wait(context); + expect(await status(input)).toMatchObject({ status: "unanswered", reason: "reset" }); + expect(transcript(await allEntries(root))).toEqual([ + "pi.user:a", + "pi.assistant", + "pi.tool-result", + "pi.reset:continue here", + "pi.reset", + "pi.user:f", + "pi.assistant:fresh", + ]); + expect(requests).toEqual([["system:", "user:f"]]); + expect(setup.faux.state.callCount).toBe(2); + await harness.close(context); + }); + + it("hands off again in a later round of the same run", async () => { + const setup = chatSetup(); + let handoffs = 0; + addTool( + setup.registry, + defineTool({ + name: "hold", + description: "Hands off", + parameters: Type.Object({}), + execute: async () => ({ output: [], control: { handoff: `h${++handoffs}` } }), + }), + ); + setup.faux.setResponses([HOLD, HOLD, answer("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const settled = await (await root.submit({ type: "input", content: "a" }, context)).wait(context); + const entries = await allEntries(root); + expect(transcript(entries)).toEqual([ + "pi.user:a", + "pi.assistant", + "pi.tool-result", + "pi.reset:h1", + "pi.assistant", + "pi.tool-result", + "pi.reset:h2", + "pi.assistant:done", + ]); + expect(settled).toMatchObject({ status: "done", answer: entries.at(-1)!.id }); + const messages = (await root.context(context)).messages; + expect(messages.map((message) => `${message.role}:${textOf(message) ?? ""}`)).toEqual([ + "system:", + "user:h2", + "assistant:done", + ]); + await harness.close(context); + }); + + it("ends the run after the handoff when the round also terminates", async () => { + const setup = chatSetup(); + const gate = deferred(); + gate.resolve(); + holdTool(setup, gate, { output: [], control: { handoff: "continue here", terminate: true } }); setup.faux.setResponses([HOLD]); const { harness, root } = await openChat(new MemoryStorage(), setup); const settled = await (await root.submit({ type: "input", content: "a" }, context)).wait(context); @@ -360,7 +517,6 @@ describe("inbox", () => { const calling = entries.find((entry) => entry.kind === "pi.assistant")!; expect(settled).toMatchObject({ status: "done", answer: calling.id }); expect(transcript(entries)).toEqual(["pi.user:a", "pi.assistant", "pi.tool-result", "pi.reset:continue here"]); - expect(entries.at(-1)!.head).toBe(entries.at(-1)!.id); expect(setup.faux.state.callCount).toBe(1); expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); await harness.close(context); @@ -630,7 +786,7 @@ describe("inbox", () => { await harness.close(context); }); - it("writes the last handoff in call order and then runs queued follow-ups in the new context", async () => { + it("writes only the last handoff in call order and runs queued follow-ups after the continuing run", async () => { const setup = chatSetup(); const firstGate = deferred(); const secondGate = deferred(); @@ -651,7 +807,7 @@ describe("inbox", () => { [fauxToolCall("hold", {}, { id: "c1" }), fauxToolCall("later", {}, { id: "c2" })], { stopReason: "toolUse" }, ); - setup.faux.setResponses([round, answer("follow-up")]); + setup.faux.setResponses([round, answer("after handoff"), answer("follow-up")]); const { harness, root } = await openChat(new MemoryStorage(), setup); const input = await root.submit({ type: "input", content: "a" }, context); await toolRunning(harness, root); @@ -660,13 +816,19 @@ describe("inbox", () => { await waitFor(async () => (await harness.snapshot(LiveDoc, root.id, context))?.tools?.[1]?.status === "done"); firstGate.resolve(); await f.wait(context); - expect(await status(input)).toMatchObject({ status: "done" }); - expect(transcript(await allEntries(root)).slice(-4)).toEqual([ + const entries = await allEntries(root); + expect(transcript(entries)).toEqual([ + "pi.user:a", + "pi.assistant", + "pi.tool-result", "pi.tool-result", "pi.reset:two", + "pi.assistant:after handoff", "pi.user:f", "pi.assistant:follow-up", ]); + const answers = entries.filter((entry) => entry.kind === "pi.assistant"); + expect(await status(input)).toMatchObject({ status: "done", answer: answers[1]!.id }); await harness.close(context); }); From 911fbab32f2b4f98214c9043e788ad8aeb81c96e Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:43:53 +0100 Subject: [PATCH 006/271] fix: unset all supported provider env vars in pi-test.sh --no-env (#11395) `./pi-test.sh --no-env` is supposed to run pi without any API keys, but its hard-coded `unset` list had fallen behind `packages/ai/src/env-api-keys.ts`. Credentials such as `DEEPSEEK_API_KEY`, `MOONSHOT_API_KEY` and `ANTHROPIC_AUTH_TOKEN` were still picked up from the environment. This change adds the 26 missing variables to the list: - the Anthropic auth token and federation variables - the API keys for Ant Ling, Baseten, Cloudflare, DeepSeek, Fireworks, Google Cloud, Kimi, Meta, Moonshot, Nvidia, Qwen token plan, Radius, Together, Typesafe, Xiaomi (all plans) and ZAI coding CN The list now covers every variable name in `env-api-keys.ts`. Testing: - No automated test covers this shell script, so none was added. - A loop over every variable name in `env-api-keys.ts` confirmed each now has an `unset` in `pi-test.sh`. - Before the change, `PI_CODING_AGENT_DIR=$(mktemp -d) DEEPSEEK_API_KEY=x ./pi-test.sh --no-env --offline -ne --list-models deepseek` listed DeepSeek models. - After the change, the same command with both `DEEPSEEK_API_KEY` and `MOONSHOT_API_KEY` set prints "No models available." - `bash -n pi-test.sh` and `npm run check` pass. Fixes #11359 Requested by @badlogic Co-authored-by: triagent --- pi-test.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/pi-test.sh b/pi-test.sh index 95383308e..c77d7287d 100755 --- a/pi-test.sh +++ b/pi-test.sh @@ -51,6 +51,32 @@ if [[ "$NO_ENV" == "true" ]]; then unset AZURE_OPENAI_API_KEY unset AZURE_OPENAI_BASE_URL unset AZURE_OPENAI_RESOURCE_NAME + unset ANTHROPIC_AUTH_TOKEN + unset ANTHROPIC_FEDERATION_RULE_ID + unset ANTHROPIC_IDENTITY_TOKEN_FILE + unset ANTHROPIC_ORGANIZATION_ID + unset ANTHROPIC_SERVICE_ACCOUNT_ID + unset ANTHROPIC_WORKSPACE_ID + unset ANT_LING_API_KEY + unset BASETEN_API_KEY + unset CLOUDFLARE_API_KEY + unset DEEPSEEK_API_KEY + unset FIREWORKS_API_KEY + unset GOOGLE_CLOUD_API_KEY + unset KIMI_API_KEY + unset META_API_KEY + unset MOONSHOT_API_KEY + unset NVIDIA_API_KEY + unset QWEN_TOKEN_PLAN_API_KEY + unset QWEN_TOKEN_PLAN_CN_API_KEY + unset RADIUS_API_KEY + unset TOGETHER_API_KEY + unset TYPESAFE_API_KEY + unset XIAOMI_API_KEY + unset XIAOMI_TOKEN_PLAN_CN_API_KEY + unset XIAOMI_TOKEN_PLAN_AMS_API_KEY + unset XIAOMI_TOKEN_PLAN_SGP_API_KEY + unset ZAI_CODING_CN_API_KEY echo "Running without API keys..." fi From 5f960a0e2b9decdbe9183f28be947798756abecd Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:47:34 +0100 Subject: [PATCH 007/271] fix(coding-agent): pass real path as npm prefix when agent dir is a symlink (#11396) When the agent directory is reached through a symlink (for example `~/.pi` linked into a dotfiles repo), pi passed the symlinked path as `--prefix` to `npm install` and `npm uninstall`. npm then writes lock entries such as `../../../dotfiles/pi/agent/npm/node_modules/foo` instead of `node_modules/foo`. Each later install adds one more nesting level, so `package-lock.json` keeps growing. This change resolves the install root with `realpathSync` before passing it to the package manager: - in `getNpmInstallArgs()`, which covers install and update for npm, pnpm and bun; - in `uninstallNpm()`. The paths pi loads extensions from are unchanged. **Testing** - Added a regression test in `test/package-manager.test.ts`. It uses a symlinked agent dir and checks that install and uninstall get the real path as `--prefix`. The test fails without the fix and passes with it. - Wrapped the existing `--prefix`/`--cwd` expectations in `realpathSync(...)`. They now match on systems where the temp dir is itself a symlink, such as macOS. - Ran `npm run check` (clean) and `test/package-manager.test.ts` plus `test/package-manager-ssh.test.ts` (all pass). - Manual reproduction with `~/.pi` as a symlink, running `pi install npm:ms` several times: - before: lock keys grew to `../../../dotfiles/pi/agent/npm/node_modules/ms`, `../../../dotfiles/dotfiles/pi/...`, and so on; - after: `[ '', 'node_modules/ms' ]`, and `pi remove npm:ms` leaves `[ '' ]`. Fixes #11358 Requested by @badlogic Co-authored-by: triagent --- packages/coding-agent/CHANGELOG.md | 1 + .../coding-agent/src/core/package-manager.ts | 11 +++- .../coding-agent/test/package-manager.test.ts | 57 ++++++++++++++++--- 3 files changed, 57 insertions(+), 12 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 246c1225e..080bb8913 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -13,6 +13,7 @@ ### Fixed +- Fixed `pi install` and `pi remove` adding ever deeper nested entries to the npm `package-lock.json` when the agent directory is reached through a symlink, such as `~/.pi` linked into a dotfiles repo. pi now passes the resolved real path to npm ([#11358](https://github.com/earendil-works/pi/issues/11358)) - MCP sign-in now rejects authorization endpoints and other OAuth metadata URLs that are not `http:` or `https:`. Before, a malicious MCP server could supply a `file:` or custom protocol URL, which pi passed to the platform URL handler. - Automatic `fd` and `rg` downloads no longer fetch x86-64 binaries on ARM32 and other architectures without a prebuilt release. pi now warns and recommends installing the tool with the system package manager ([#11209](https://github.com/earendil-works/pi/issues/11209)) - Fixed `/model` search ranking letters scattered over an earlier part of an entry above a real match, for example `opus` listing `gpt-oss-120b` and `claude-sonnet-4.5` before `claude-opus-4.1` ([#11220](https://github.com/earendil-works/pi/issues/11220), [#11296](https://github.com/earendil-works/pi/pull/11296)) diff --git a/packages/coding-agent/src/core/package-manager.ts b/packages/coding-agent/src/core/package-manager.ts index b5015b5ba..2535b1f93 100644 --- a/packages/coding-agent/src/core/package-manager.ts +++ b/packages/coding-agent/src/core/package-manager.ts @@ -7,6 +7,7 @@ import { mkdirSync, readdirSync, readFileSync, + realpathSync, rmSync, statSync, writeFileSync, @@ -1868,7 +1869,9 @@ export class DefaultPackageManager implements PackageManager { return this.runCommandSync(npmCommand.command, [...npmCommand.args, ...args]); } - private getNpmInstallArgs(specs: string[], installRoot: string): string[] { + private getNpmInstallArgs(specs: string[], root: string): string[] { + // npm writes broken nested package-lock.json paths when --prefix goes through a symlink. + const installRoot = existsSync(root) ? realpathSync(root) : root; const packageManagerName = this.getPackageManagerName(); // Extension packages run inside pi and resolve pi APIs through loader aliases/virtual modules. // Disable peer dependency resolution for managed installs (npm's --legacy-peer-deps, and @@ -1898,10 +1901,12 @@ export class DefaultPackageManager implements PackageManager { } private async uninstallNpm(source: NpmSource, scope: SourceScope): Promise { - const installRoot = this.getNpmInstallRoot(scope, false); - if (!existsSync(installRoot)) { + const root = this.getNpmInstallRoot(scope, false); + if (!existsSync(root)) { return; } + // npm writes broken nested package-lock.json paths when --prefix goes through a symlink. + const installRoot = realpathSync(root); const packageManagerName = this.getPackageManagerName(); if (packageManagerName === "bun") { await this.runNpmCommand(["uninstall", source.name, "--cwd", installRoot]); diff --git a/packages/coding-agent/test/package-manager.test.ts b/packages/coding-agent/test/package-manager.test.ts index 4aaa60ab3..f8446b657 100644 --- a/packages/coding-agent/test/package-manager.test.ts +++ b/packages/coding-agent/test/package-manager.test.ts @@ -1,5 +1,5 @@ import { EventEmitter } from "node:events"; -import { existsSync, mkdirSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, realpathSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, relative } from "node:path"; import { PassThrough } from "node:stream"; @@ -754,7 +754,7 @@ Content`, "install", "@scope/pkg", "--prefix", - join(agentDir, "npm"), + realpathSync(join(agentDir, "npm")), "--legacy-peer-deps", ], undefined, @@ -769,7 +769,36 @@ Content`, expect(runCommandSpy).toHaveBeenCalledWith( "npm", - ["uninstall", "@scope/pkg", "--prefix", join(agentDir, "npm"), "--legacy-peer-deps"], + ["uninstall", "@scope/pkg", "--prefix", realpathSync(join(agentDir, "npm")), "--legacy-peer-deps"], + undefined, + ); + }); + + // Regression test for #11358: npm writes nested package-lock.json entries for a symlinked --prefix + it("should pass the real path as --prefix when the agent dir is a symlink", async () => { + const realAgentDir = join(tempDir, "dotfiles", "agent"); + mkdirSync(realAgentDir, { recursive: true }); + const linkedAgentDir = join(tempDir, "linked-agent"); + symlinkSync(realAgentDir, linkedAgentDir, "dir"); + packageManager = new DefaultPackageManager({ + cwd: tempDir, + agentDir: linkedAgentDir, + settingsManager, + }); + const runCommandSpy = vi.spyOn(packageManager as any, "runCommand").mockResolvedValue(undefined); + const realNpmRoot = join(realpathSync(realAgentDir), "npm"); + + await packageManager.install("npm:@scope/pkg"); + await packageManager.remove("npm:@scope/pkg"); + + expect(runCommandSpy).toHaveBeenCalledWith( + "npm", + ["install", "@scope/pkg", "--prefix", realNpmRoot, "--legacy-peer-deps"], + undefined, + ); + expect(runCommandSpy).toHaveBeenCalledWith( + "npm", + ["uninstall", "@scope/pkg", "--prefix", realNpmRoot, "--legacy-peer-deps"], undefined, ); }); @@ -790,7 +819,17 @@ Content`, expect(runCommandSpy).toHaveBeenCalledWith( "mise", - ["exec", "bun@1", "--", "bun", "install", "@scope/pkg", "--cwd", join(agentDir, "npm"), "--omit=peer"], + [ + "exec", + "bun@1", + "--", + "bun", + "install", + "@scope/pkg", + "--cwd", + realpathSync(join(agentDir, "npm")), + "--omit=peer", + ], undefined, ); }); @@ -1206,7 +1245,7 @@ Content`, "install", "pnpm-pkg", "--prefix", - join(agentDir, "npm"), + realpathSync(join(agentDir, "npm")), "--config.auto-install-peers=false", "--config.strict-peer-dependencies=false", "--config.strict-dep-builds=false", @@ -2467,7 +2506,7 @@ export default function(api) { api.registerTool({ name: "test", description: "te ); expect(runCommandSpy).toHaveBeenCalledWith( "npm", - ["install", "example@^1.0.0", "--prefix", join(tempDir, ".pi", "npm"), "--legacy-peer-deps"], + ["install", "example@^1.0.0", "--prefix", realpathSync(join(tempDir, ".pi", "npm")), "--legacy-peer-deps"], undefined, ); }); @@ -2531,7 +2570,7 @@ export default function(api) { api.registerTool({ name: "test", description: "te "install", "legacy-pkg@latest", "--prefix", - join(agentDir, "npm"), + realpathSync(join(agentDir, "npm")), "--legacy-peer-deps", ]); mkdirSync(managedPath, { recursive: true }); @@ -2648,7 +2687,7 @@ export default function(api) { api.registerTool({ name: "test", description: "te "user-old@latest", "user-unknown@latest", "--prefix", - join(agentDir, "npm"), + realpathSync(join(agentDir, "npm")), "--legacy-peer-deps", ], undefined, @@ -2661,7 +2700,7 @@ export default function(api) { api.registerTool({ name: "test", description: "te "project-old@latest", "project-missing@latest", "--prefix", - join(tempDir, ".pi", "npm"), + realpathSync(join(tempDir, ".pi", "npm")), "--legacy-peer-deps", ], undefined, From 984cf6643d303cd414a9fe883a01c212caf40b22 Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:48:27 +0100 Subject: [PATCH 008/271] fix(coding-agent): resolve provider/id references when the provider id contains a slash (#11397) `--model "/"` returned "not found" when the provider id itself contains a `/`. URL-derived ids such as `llama-server=http://host:10800` are one example. `splitProviderPrefix()` in `model-resolver.ts` split the reference at the first `/`, which turned the prefix into `llama-server=http:`. No provider has that id, so resolution fell through to not-found. `splitProviderPrefix()` now compares the text against the provider ids in the catalog. A provider matches when the text starts with its id followed by `/`, and the longest matching id wins. Ordinary provider ids resolve exactly as before. Testing: - I added two cases to the `resolveModelReference` table in `test/model-resolver.test.ts`, using a provider `llama=http://host:10800`: one plain, one with a `:high` thinking suffix. Both fail without the change and pass with it. The test runs every case against the catalog in both orders. - `node ../../node_modules/vitest/dist/cli.js --run test/model-resolver.test.ts`: 64 passed. - `npm run check`: passes. - Before the change, a standalone check of `resolveModelReference` with the reporter's exact provider and model (`llama-server=http://host:10800/unsloth/Qwen3.8-27B-GGUF:BF16`) returned `not-found`. It now returns `resolved`. - One test, `test/model-runtime-classifiers.test.ts` ("applies provider compat from models.json to classifier models"), fails in my environment both with and without this change. It is unrelated to this fix. Fixes #11394 Requested by @badlogic Co-authored-by: triagent --- .../coding-agent/src/core/model-resolver.ts | 19 ++++++++++++------- .../coding-agent/test/model-resolver.test.ts | 12 ++++++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/packages/coding-agent/src/core/model-resolver.ts b/packages/coding-agent/src/core/model-resolver.ts index e6657e9eb..bd331f690 100644 --- a/packages/coding-agent/src/core/model-resolver.ts +++ b/packages/coding-agent/src/core/model-resolver.ts @@ -128,13 +128,18 @@ function splitProviderPrefix( text: string, models: readonly Model[], ): { provider: string; modelText: string } | undefined { - const slashIndex = text.indexOf("/"); - if (slashIndex === -1) return undefined; - const prefix = text.substring(0, slashIndex).trim().toLowerCase(); - const modelText = text.substring(slashIndex + 1).trim(); - if (!prefix || !modelText) return undefined; - const provider = models.find((model) => model.provider.toLowerCase() === prefix)?.provider; - return provider ? { provider, modelText } : undefined; + // Provider ids may contain "/" themselves (e.g. URL-derived ids), so match known ids, longest first + const lowerText = text.toLowerCase(); + let best: { provider: string; modelText: string } | undefined; + for (const { provider } of models) { + const prefix = provider.trim().toLowerCase(); + if (!prefix || (best && best.provider.length >= provider.length)) continue; + const slashIndex = lowerText.indexOf("/", prefix.length); + if (slashIndex === -1 || lowerText.substring(0, slashIndex).trim() !== prefix) continue; + const modelText = text.substring(slashIndex + 1).trim(); + if (modelText) best = { provider, modelText }; + } + return best; } /** diff --git a/packages/coding-agent/test/model-resolver.test.ts b/packages/coding-agent/test/model-resolver.test.ts index f8226a2be..018f4c8d1 100644 --- a/packages/coding-agent/test/model-resolver.test.ts +++ b/packages/coding-agent/test/model-resolver.test.ts @@ -115,6 +115,8 @@ const referenceCatalog = [ referenceModel("gw-b", "claude-opus-4-8"), referenceModel("gw-b", "gw-b/free"), referenceModel("gw-b", "zz:free"), + // URL-derived provider id containing "/" (#11394) + referenceModel("llama=http://host:10800", "unsloth/qwen-gguf:BF16"), ]; type ReferenceExpected = @@ -152,6 +154,16 @@ const referenceCases: ReferenceCase[] = [ text: "openrouter/qwen/qwen3-coder:exacto:high", expected: { model: "openrouter/qwen/qwen3-coder:exacto", thinkingLevel: "high" }, }, + { + name: "provider id containing a slash", + text: "llama=http://host:10800/unsloth/qwen-gguf:BF16", + expected: { model: "llama=http://host:10800/unsloth/qwen-gguf:BF16" }, + }, + { + name: "provider id containing a slash, with thinking suffix", + text: "llama=http://host:10800/unsloth/qwen-gguf:BF16:high", + expected: { model: "llama=http://host:10800/unsloth/qwen-gguf:BF16", thinkingLevel: "high" }, + }, { name: "raw id wins when the provider prefix names no matching model", text: "openai/gpt-4o:extended", From 0eea54f11c2c8d45ec3dfaa40522030539ab28f1 Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:54:24 +0100 Subject: [PATCH 009/271] fix: normalize path separators in check-entry-graphs on Windows (#11399) `scripts/check-entry-graphs.mjs` built the list of files each export reaches with `path.relative()`, which returns paths with `\` separators on Windows. The allow and forbid patterns are written with `/`, which caused two problems on Windows: - The allow entry `packages/telemetry/src/index.ts` never matched, so the bare `index.ts` forbid pattern flagged it and the check failed even though nothing was wrong. - Forbid patterns that contain a `/` (`providers/`, `runtime/host.ts`, ...) never matched, so real violations went unreported. The fix converts the relative paths to `/` before any comparison. On Linux and macOS nothing changes. Testing: - The scripts have no unit tests, so no test was added. - `node scripts/check-entry-graphs.mjs` and `npm run check` pass on Linux. - I simulated Windows path handling with `path.win32`, without running Windows itself: - Before the fix: `relative(...)` gives `packages\telemetry\src\index.ts`, and `endsWith("packages/telemetry/src/index.ts")` is false. - After the fix: the path becomes `packages/telemetry/src/index.ts`, and the suffix check is true. Fixes #11333 Requested by @badlogic Co-authored-by: triagent --- scripts/check-entry-graphs.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/check-entry-graphs.mjs b/scripts/check-entry-graphs.mjs index a26291094..b3b033a73 100644 --- a/scripts/check-entry-graphs.mjs +++ b/scripts/check-entry-graphs.mjs @@ -12,7 +12,7 @@ */ import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; -import { dirname, relative, resolve } from "node:path"; +import { dirname, relative, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); @@ -153,7 +153,7 @@ for (const [pkgDir, budgets] of Object.entries(BUDGETS)) { failures += 1; continue; } - const graph = [...walk(source)].map((file) => relative(ROOT, file)); + const graph = [...walk(source)].map((file) => relative(ROOT, file).split(sep).join("/")); if (graph.length > budget.maxFiles) { console.error( `${pkgDir} export "${name}" reaches ${graph.length} files, budget ${budget.maxFiles}\n` + From d5b3de22d0e24a3e9c38a9410d3329b099717283 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sun, 25 Oct 2026 08:04:10 +0000 Subject: [PATCH 010/271] fix(durable): record the requested thinking level on assistant messages --- packages/durable/CHANGELOG.md | 1 + packages/durable/src/harness/generation.ts | 14 +++++-- .../test/harness-generation-recovery.test.ts | 37 +++++++++++++++++++ .../durable/test/harness-generation.test.ts | 18 +++++++++ 4 files changed, 66 insertions(+), 4 deletions(-) diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 15e4d75e4..e061eb6b7 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -27,6 +27,7 @@ - Fixed a nested call's error summary keeping the call's whole output alive through a slice of it ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed task span error messages and codemode call previews keeping the whole text they were cut from alive ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed partial commits of a streamed answer copying every growing text, thinking and tool-call argument string in full on each commit (about 20 s of CPU and a 100 MB heap peak for a 500 KB tool call): they now append new text with Chord's `appendText()`, and apply the argument changes of their own parse of the streamed argument text ([#9265](https://github.com/earendil-works/pi/issues/9265)) +- Fixed stored assistant messages lacking `thinkingLevel`: a generation records the Pi thinking level its request asked for on the response, also after polling a deferred one, as pi-agent-core's agent loop does ## [1.4.0] - 2026-10-21 diff --git a/packages/durable/src/harness/generation.ts b/packages/durable/src/harness/generation.ts index 11bf946f2..210ae981e 100644 --- a/packages/durable/src/harness/generation.ts +++ b/packages/durable/src/harness/generation.ts @@ -81,6 +81,8 @@ export type GenerationCheckpoint = attempt: number; compacted?: TaskId; model: ModelRef; + /** Absent in checkpoints of earlier versions. */ + thinkingLevel?: ModelThinkingLevel; cutoff: EntryId; handle: DeferredHandle; pollAt: number; @@ -106,6 +108,7 @@ type Request = { readonly attempt: number; readonly compacted: TaskId | undefined; readonly model: ModelRef; + readonly thinkingLevel: ModelThinkingLevel | undefined; readonly cutoff: EntryId; /** Committed model context through `cutoff`, when the phase already derived it. */ readonly messages?: readonly Message[]; @@ -231,7 +234,7 @@ export const GenerationTask = defineTask { - const { attempt, compacted, model: ref, cutoff, handle, pollAt } = task.state.checkpoint; + const { attempt, compacted, model: ref, thinkingLevel, cutoff, handle, pollAt } = task.state.checkpoint; if (pollAt > runtime.now()) return runtime.sleepUntil(pollAt, task.state.checkpoint, context); const model = runtime.models.getModel(ref.provider, ref.modelId); if (model === undefined) return failNoModel(runtime, ref, context); @@ -258,7 +261,7 @@ export const GenerationTask = defineTask { @@ -522,7 +525,9 @@ async function classify( // An abort mark or close: the abort invocation or the reopened run handles the committed state. runtime.signal.throwIfAborted(); const conversationId = runtime.conversationId; - const { attempt, compacted, model: ref, cutoff } = request; + const { attempt, compacted, model: ref, thinkingLevel, cutoff } = request; + // Record the requested level, as pi-agent-core's agent loop does. + if (thinkingLevel !== undefined) message.thinkingLevel = thinkingLevel; if (message.stopReason === "deferred" && message.deferred !== undefined) { const handle = message.deferred; const pollAt = Math.max( @@ -536,6 +541,7 @@ async function classify( attempt, ...(compacted === undefined ? {} : { compacted }), model: ref, + ...(thinkingLevel === undefined ? {} : { thinkingLevel }), cutoff, handle, pollAt, diff --git a/packages/durable/test/harness-generation-recovery.test.ts b/packages/durable/test/harness-generation-recovery.test.ts index ef104ac4c..468723ce4 100644 --- a/packages/durable/test/harness-generation-recovery.test.ts +++ b/packages/durable/test/harness-generation-recovery.test.ts @@ -287,6 +287,43 @@ describe("generation recovery", () => { await opened.harness.close(context); }); + it("polls a checkpoint stored without a thinking level and records none", async () => { + const path = await sqlitePath(); + const setup = chatSetup({ deferred: { pollAfterMs: 60_000 } }); + let now = 1_000; + setup.now = () => now; + setup.faux.setResponses([fauxAssistantMessage("deferred answer")]); + let opened = await open(path, setup); + await opened.root.configure({ thinkingLevel: "high" }, context); + opened.harness.resume(); + setup.settings.stream = { deferred: true }; + const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; + await waitFor(async () => { + const live = await opened.harness.snapshot(LiveDoc, opened.root.id, context); + return live?.generation?.deferred !== undefined; + }); + const taskId = await runTaskId(opened.harness); + expect(await checkpoint(opened.harness, taskId)).toMatchObject({ phase: "poll", thinkingLevel: "high" }); + await opened.harness.close(context); + // Earlier versions stored poll checkpoints without the field. + const database = new DatabaseSync(path); + database + .prepare("UPDATE tasks SET record = json_remove(record, '$.state.checkpoint.thinkingLevel') WHERE id = ?") + .run(taskId); + database.close(); + + opened = await open(path, setup); + expect(await checkpoint(opened.harness, taskId)).not.toHaveProperty("thinkingLevel"); + now = 61_000; + opened.harness.resume(); + const settled = await (await opened.harness.submission(id, context))!.wait(context); + if (settled.status !== "done" || settled.type !== "input") throw new Error(`Unexpected ${settled.status}`); + const answer = (await opened.root.commit((tx) => tx.entry(AssistantEntry, settled.answer), context))?.model?.[0]; + expect(textOf(answer)).toBe("deferred answer"); + expect(answer).not.toHaveProperty("thinkingLevel"); + await opened.harness.close(context); + }); + it("fails no_model when the pinned model is gone after reopen, in request and in poll", async () => { const path = await sqlitePath(); const setup = chatSetup({ deferred: { pollAfterMs: 60_000 } }); diff --git a/packages/durable/test/harness-generation.test.ts b/packages/durable/test/harness-generation.test.ts index 3cb077264..05a5f5359 100644 --- a/packages/durable/test/harness-generation.test.ts +++ b/packages/durable/test/harness-generation.test.ts @@ -385,6 +385,24 @@ describe("generation", () => { await harness.close(context); }); + it("records the requested thinking level on answers, streamed and polled", async () => { + const setup = chatSetup({ deferred: { pendingFetches: 1, pollAfterMs: 1 } }); + setup.faux.setResponses([fauxAssistantMessage("streamed"), fauxAssistantMessage("polled")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + harness.resume(); + const answer = async (content: string) => { + const settled = await (await root.submit({ type: "input", content }, context)).wait(context); + if (settled.status !== "done" || settled.type !== "input") throw new Error(`Unexpected ${settled.status}`); + return (await root.commit((tx) => tx.entry(AssistantEntry, settled.answer), context))?.model?.[0]; + }; + await root.configure({ thinkingLevel: "high" }, context); + expect(await answer("one")).toMatchObject({ thinkingLevel: "high" }); + setup.settings.stream = { deferred: true }; + expect(await answer("two")).toMatchObject({ thinkingLevel: "high" }); + expect(setup.faux.state.deferredFetchCount).toBe(2); + await harness.close(context); + }); + it("attributes model requests to pi by default and merges the Harness identity over it", async () => { const seen: (SimpleStreamOptions["identity"] | undefined)[] = []; const capture = (_context: unknown, options?: SimpleStreamOptions) => { From b84d5d1bc315dac7e2d384190cfad112e63deb9d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 26 Oct 2026 11:15:34 +0000 Subject: [PATCH 011/271] chore: approve contributors from issue #11132 --- .github/APPROVED_CONTRIBUTORS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/APPROVED_CONTRIBUTORS b/.github/APPROVED_CONTRIBUTORS index 23ba87df7..9f4a04a02 100644 --- a/.github/APPROVED_CONTRIBUTORS +++ b/.github/APPROVED_CONTRIBUTORS @@ -407,3 +407,5 @@ Jay-Madden issue lalyos pr Lincoln504 pr + +vimtor pr From 66c2fa3d7ac5b14059767f4e6178a645e65cdf27 Mon Sep 17 00:00:00 2001 From: triagent Date: Mon, 26 Oct 2026 02:45:47 +0000 Subject: [PATCH 012/271] fix(coding-agent): update undici to 8.12.3 for security advisories --- package-lock.json | 8 ++++---- packages/coding-agent/install-lock/package-lock.json | 8 ++++---- packages/coding-agent/package.json | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index 62665c62f..74451ff75 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5294,9 +5294,9 @@ } }, "node_modules/undici": { - "version": "8.10.2", - "resolved": "https://registry.npmjs.org/undici/-/undici-8.10.2.tgz", - "integrity": "sha512-/y4/bH9YNU5hi9NIrpOuvGXFcxrj3CMrV+/AYpowAYTpHn8gX/XPFjNy766FPoYY0miQhdW977JFWKGNhBdwyQ==", + "version": "8.12.3", + "resolved": "https://registry.npmjs.org/undici/-/undici-8.12.3.tgz", + "integrity": "sha512-+fdndUTwBP86cVjihTp21hZHo7d6oj1c/47XPA3lL8mM0XHs+C4EkcjmgRsFnFk2f/UONGDy9ItBItmXsLOQHw==", "license": "MIT", "engines": { "node": ">=22.19.0" @@ -5865,7 +5865,7 @@ "quickjs-wasi": "3.6.2", "semver": "7.8.5", "typebox": "1.3.27", - "undici": "8.10.2", + "undici": "8.12.3", "yaml": "2.9.0" }, "bin": { diff --git a/packages/coding-agent/install-lock/package-lock.json b/packages/coding-agent/install-lock/package-lock.json index 5d242776c..25e273cf6 100644 --- a/packages/coding-agent/install-lock/package-lock.json +++ b/packages/coding-agent/install-lock/package-lock.json @@ -516,7 +516,7 @@ "quickjs-wasi": "3.6.2", "semver": "7.8.5", "typebox": "1.3.27", - "undici": "8.10.2", + "undici": "8.12.3", "yaml": "2.9.0" }, "bin": { @@ -1429,9 +1429,9 @@ "license": "MIT" }, "node_modules/undici": { - "version": "8.10.2", - "resolved": "https://registry.npmjs.org/undici/-/undici-8.10.2.tgz", - "integrity": "sha512-/y4/bH9YNU5hi9NIrpOuvGXFcxrj3CMrV+/AYpowAYTpHn8gX/XPFjNy766FPoYY0miQhdW977JFWKGNhBdwyQ==", + "version": "8.12.3", + "resolved": "https://registry.npmjs.org/undici/-/undici-8.12.3.tgz", + "integrity": "sha512-+fdndUTwBP86cVjihTp21hZHo7d6oj1c/47XPA3lL8mM0XHs+C4EkcjmgRsFnFk2f/UONGDy9ItBItmXsLOQHw==", "license": "MIT", "engines": { "node": ">=22.19.0" diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index 616d6a475..0e452d9b5 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -61,7 +61,7 @@ "quickjs-wasi": "3.6.2", "semver": "7.8.5", "typebox": "1.3.27", - "undici": "8.10.2", + "undici": "8.12.3", "yaml": "2.9.0" }, "overrides": { From c17be1e9826b0eb3bf29d5a53c1dc0edbf860f78 Mon Sep 17 00:00:00 2001 From: triagent Date: Sat, 24 Oct 2026 12:41:35 +0000 Subject: [PATCH 013/271] fix: skip custom-message runs in prompt-url-widget --- .pi/extensions/prompt-url-widget.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pi/extensions/prompt-url-widget.ts b/.pi/extensions/prompt-url-widget.ts index 20a41a509..6901663b1 100644 --- a/.pi/extensions/prompt-url-widget.ts +++ b/.pi/extensions/prompt-url-widget.ts @@ -218,7 +218,7 @@ export default function promptUrlWidgetExtension(pi: ExtensionAPI) { }; pi.on("before_agent_start", async (event, ctx) => { - if (!ctx.hasUI) return; + if (!ctx.hasUI || event.messageType !== "user") return; const match = extractPromptMatch(event.prompt); if (!match) { return; From ae823bc0bb1da5af6ad74f3d1b2fc89846b8502e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Hou=C5=A1ka?= Date: Mon, 26 Oct 2026 14:07:52 +0100 Subject: [PATCH 014/271] fix(coding-agent): rank tools first when the search query is their name A query that equals a tool's full name or the part after a __ separator (thread_read for mcp__app__thread_read) now ranks that tool first in tool_search and codemode searchTools(). BM25 length normalization could otherwise rank a shorter description sharing the name's words above it. closes #11331 --- .../src/extensions/tool-search/tool.ts | 33 ++++++++++++++++--- .../coding-agent/test/tool-search.test.ts | 23 +++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/packages/coding-agent/src/extensions/tool-search/tool.ts b/packages/coding-agent/src/extensions/tool-search/tool.ts index 1841ebd97..b9fa4ade0 100644 --- a/packages/coding-agent/src/extensions/tool-search/tool.ts +++ b/packages/coding-agent/src/extensions/tool-search/tool.ts @@ -115,7 +115,28 @@ export function createToolSearchDocument( return { name: tool.name, text: parts.filter((part) => part.trim()).join(" ") }; } -/** Okapi BM25 with the usual parameters. Ties keep document order. */ +/** Lowercase with characters invalid in script identifiers as `_`, so `dev-radius` matches `dev_radius`. */ +function normalizeName(name: string): string { + return name + .trim() + .toLowerCase() + .replace(/[^a-z0-9_$]/g, "_"); +} + +/** + * Whether `query` names the tool: its full name or a suffix after a `__` separator, so `thread_read` + * and `app__thread_read` both name `mcp__app__thread_read`. + */ +function isNameQuery(query: string, name: string): boolean { + const normalized = normalizeName(name); + return normalized === query || normalized.endsWith(`__${query}`); +} + +/** + * Okapi BM25 with the usual parameters. A query that names a tool ranks that tool first, because + * length normalization can otherwise rank a short description sharing the name's words above it. + * Ties keep document order. + */ export class Bm25Ranker implements ToolRanker { private readonly k1: number; private readonly b: number; @@ -141,7 +162,8 @@ export class Bm25Ranker implements ToolRanker { return [term, Math.log(1 + (documents.length - frequency + 0.5) / (frequency + 0.5))] as const; }), ); - const matches: ToolSearchMatch[] = []; + const nameQuery = normalizeName(query); + const matches: (ToolSearchMatch & { named: boolean })[] = []; documents.forEach((document, index) => { let score = 0; for (const term of queryTerms) { @@ -150,9 +172,12 @@ export class Bm25Ranker implements ToolRanker { const norm = this.k1 * (1 - this.b + (this.b * lengths[index]) / averageLength); score += (idf.get(term) ?? 0) * ((count * (this.k1 + 1)) / (count + norm)); } - if (score > 0) matches.push({ name: document.name, score }); + if (score > 0) matches.push({ name: document.name, score, named: isNameQuery(nameQuery, document.name) }); }); - return matches.sort((a, b) => b.score - a.score).slice(0, limit); + return matches + .sort((a, b) => Number(b.named) - Number(a.named) || b.score - a.score) + .slice(0, limit) + .map(({ name, score }) => ({ name, score })); } } diff --git a/packages/coding-agent/test/tool-search.test.ts b/packages/coding-agent/test/tool-search.test.ts index ba07448a0..fd55b5c56 100644 --- a/packages/coding-agent/test/tool-search.test.ts +++ b/packages/coding-agent/test/tool-search.test.ts @@ -53,6 +53,29 @@ describe("Bm25Ranker", () => { expect(ranker.rank("tickets", documents, 8)).toEqual([]); }); + // https://github.com/earendil-works/pi/issues/11331 + it("ranks a tool first when the query is its name", () => { + const named = [ + tool("mcp__app__queue_read", "Read a queued message of this thread."), + tool( + "mcp__dev-app__thread_read", + "Read durable state and a paginated timeline from any thread. The default view returns user messages, assistant messages and plans. Continue with afterPosition until hasMore is false.", + ), + ].map((entry) => createToolSearchDocument(entry)); + const ranker = new Bm25Ranker(); + for (const query of [ + "thread_read", + "dev_app__thread_read", + "mcp__dev-app__thread_read", + "Thread_Read", + "thread read", + ]) { + expect(ranker.rank(query, named, 1).map((match) => match.name)).toEqual(["mcp__dev-app__thread_read"]); + } + // Partial names still rank by BM25. + expect(ranker.rank("read", named, 1).map((match) => match.name)).toEqual(["mcp__app__queue_read"]); + }); + it("includes the namespace in the search text", () => { const document = createToolSearchDocument(tool("mcp__x__run", "Run it."), { name: "mcp__x", From f9daca3480732d0042b1df3a6e27221f597a620d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Hou=C5=A1ka?= Date: Mon, 26 Oct 2026 14:03:35 +0100 Subject: [PATCH 015/271] ci: run tests in parallel jobs and drop unused apt packages Split build-check-test into build-check plus a test matrix (coding-agent, durable, other). Install only fd and ripgrep, and only in jobs whose tests use them; the cairo/pango/jpeg/gif/rsvg dev libraries were for compiling canvas, which npm ci --ignore-scripts never does. --- .github/workflows/ci.yml | 60 ++++++++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8c980e5a..71e526c62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ concurrency: cancel-in-progress: true jobs: - build-check-test: + build-check: runs-on: ubuntu-latest steps: - name: Checkout @@ -23,12 +23,6 @@ jobs: node-version: 22 cache: npm - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev fd-find ripgrep - sudo ln -s $(which fdfind) /usr/local/bin/fd - - name: Install dependencies run: npm ci --ignore-scripts @@ -55,11 +49,61 @@ jobs: - name: Check run: npm run check + # Tests are split across parallel jobs so the slowest suites do not run back to back. + test: + name: test (${{ matrix.name }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: coding-agent + - name: durable + - name: other + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + cache: npm + + # fd and ripgrep are used by coding-agent tools tests and tui autocomplete tests. + - name: Install system dependencies + if: matrix.name != 'durable' + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends fd-find ripgrep + sudo ln -s $(which fdfind) /usr/local/bin/fd + + - name: Install dependencies + run: npm ci --ignore-scripts + + - name: Build + run: npm run build + - name: Build the pi-env daemon + if: matrix.name == 'other' run: npm run build:daemon --workspace packages/env - name: Test - run: npm test + env: + SUITE: ${{ matrix.name }} + run: | + if [ "$SUITE" != other ]; then + npm run test --workspace "packages/$SUITE" + exit 0 + fi + # Every other workspace, so new packages are tested without editing this workflow. + args=() + for dir in packages/*/; do + case "$dir" in packages/coding-agent/ | packages/durable/) continue ;; esac + args+=(--workspace "${dir%/}") + done + npm run test:scripts + npm run test --if-present "${args[@]}" mcp-conformance: runs-on: ubuntu-latest From 89c921d8b56b59e768b26e7f48b6832978119310 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Mon, 26 Oct 2026 20:46:31 +0100 Subject: [PATCH 016/271] fix: harden handling of untrusted URLs, terminal text, and streamed data - ai: OAuth logins only accept http(s) URLs from servers (shared trustedHttpUrl, now also checked by Radius); pi-messages rejects contentIndex values that are not the next or an existing block; the CLI auth file is written with mode 0600 - ai: OpenCode DeepSeek V4.1 Flash (renamed from V4 Flash upstream) no longer sends prompt_cache_retention - tui: setTitle() replaces control characters; export replaceControlCharacters() - coding-agent: the login dialog only opens web URLs and never prints control characters from sign-in, device-code, or info URLs - durable: partial commits no longer merge a new __proto__ member into Object.prototype --- packages/ai/CHANGELOG.md | 4 ++ packages/ai/scripts/generate-models.ts | 1 + packages/ai/src/api/pi-messages.ts | 9 ++++ packages/ai/src/auth/oauth/github-copilot.ts | 14 ++--- packages/ai/src/auth/oauth/kimi-coding.ts | 26 ++-------- packages/ai/src/auth/oauth/meta.ts | 13 +---- packages/ai/src/auth/oauth/radius.ts | 12 +++-- packages/ai/src/auth/oauth/trusted-url.ts | 10 ++++ packages/ai/src/cli.ts | 7 ++- packages/ai/test/cache-retention.test.ts | 2 +- packages/ai/test/pi-messages.test.ts | 28 ++++++++++ packages/ai/test/radius-oauth.test.ts | 33 ++++++++++++ packages/coding-agent/CHANGELOG.md | 1 + .../interactive/components/login-dialog.ts | 52 ++++++++++++++----- .../test/login-dialog-device-code.test.ts | 44 ++++++++++++++++ packages/durable/CHANGELOG.md | 1 + packages/durable/src/harness/json.ts | 4 +- packages/durable/test/harness-json.test.ts | 23 ++++++++ packages/tui/CHANGELOG.md | 5 ++ packages/tui/src/index.ts | 1 + packages/tui/src/program-status.ts | 5 +- packages/tui/src/terminal.ts | 6 ++- packages/tui/src/utils.ts | 7 +++ packages/tui/test/terminal.test.ts | 20 +++++++ 24 files changed, 260 insertions(+), 68 deletions(-) create mode 100644 packages/ai/src/auth/oauth/trusted-url.ts create mode 100644 packages/durable/test/harness-json.test.ts diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 920d227e7..244d96f74 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -23,6 +23,10 @@ - Fixed grammar (custom) tool input streaming copying the whole input so far on every delta ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed bounded error texts (telemetry span messages, truncated provider error bodies, pi-messages diagnostics, Radius gateway errors) keeping the whole text they were cut from alive ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed Kimi K3.1 Pro and K3.1 Pro Highspeed on Kimi Coding, Moonshot, and OpenCode Go using Kimi K2 request settings. They now send `reasoning_effort` (low/high/max), return earlier reasoning, accept tools added mid-conversation, and report Moonshot rates on Kimi Coding. Kimi Coding `k3-256k` now reports Kimi K3 rates. +- Fixed OAuth logins opening server-supplied URLs that are not `http:` or `https:` URLs. Radius now checks its browser authorization endpoint and device verification URI, and GitHub Copilot, Kimi Coding, and Meta share one check +- Fixed pi-messages streams accepting a `contentIndex` that is not the next or an existing content block, such as `__proto__` or a huge index +- Fixed `pi-ai` CLI auth file being readable by other users. It is now written with mode `0600` +- Fixed OpenCode DeepSeek V4.1 Flash sending `prompt_cache_retention`, which the OpenCode gateway rejects, with long cache retention ## [1.4.0] - 2026-10-21 diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index ee01fbe1f..e9027559e 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -485,6 +485,7 @@ const XAI_RESPONSES_COMPAT: OpenAIResponsesCompat = { const OPENCODE_OPENAI_COMPLETIONS_LONG_CACHE_RETENTION_UNSUPPORTED_MODELS = new Set([ "opencode:deepseek-v4-flash", + "opencode:deepseek-v4.1-flash", "opencode:deepseek-v4-pro", "opencode:kimi-k2.5", "opencode:kimi-k2.6", diff --git a/packages/ai/src/api/pi-messages.ts b/packages/ai/src/api/pi-messages.ts index 59b54567e..3316a56ae 100644 --- a/packages/ai/src/api/pi-messages.ts +++ b/packages/ai/src/api/pi-messages.ts @@ -195,6 +195,15 @@ function createEventConverter(model: Model<"pi-messages">) { const toolArguments = new Map(); return (event: PiMessagesEvent): AssistantMessageEvent => { + if ("contentIndex" in event) { + // The index comes from the wire and keys the content array: `__proto__` would reach Array.prototype, and a huge + // index would make a sparse array of billions of holes. A start adds the next block; other events need one. + const index: unknown = event.contentIndex; + const limit = event.type.endsWith("_start") ? partial.content.length : partial.content.length - 1; + if (typeof index !== "number" || !Number.isInteger(index) || index < 0 || index > limit) { + throw new Error(`Invalid contentIndex ${JSON.stringify(index)} in pi-messages ${event.type} event`); + } + } switch (event.type) { case "done": Object.assign(partial, { diff --git a/packages/ai/src/auth/oauth/github-copilot.ts b/packages/ai/src/auth/oauth/github-copilot.ts index 5a17d07f0..5620bba9e 100644 --- a/packages/ai/src/auth/oauth/github-copilot.ts +++ b/packages/ai/src/auth/oauth/github-copilot.ts @@ -6,6 +6,7 @@ import { GITHUB_COPILOT_MODELS } from "../../providers/github-copilot.models.ts" import { sleep } from "../../utils/sleep.ts"; import type { OAuthAuth, OAuthCredential, ProviderAuthInteraction } from "../types.ts"; import { pollOAuthDeviceCodeFlow } from "./device-code.ts"; +import { trustedHttpUrl } from "./trusted-url.ts"; const decode = (s: string) => atob(s); const CLIENT_ID = decode("SXYxLmI1MDdhMDhjODdlY2ZlOTg="); @@ -239,22 +240,15 @@ async function startDeviceFlow(domain: string, signal: AbortSignal): Promise | n } } -/** The verification URI is opened in the user's browser; only http(s) URLs are trusted. */ -function trustedHttpUrl(value: unknown): string | null { - if (typeof value !== "string" || !value) return null; - try { - const url = new URL(value); - if (url.protocol !== "https:" && url.protocol !== "http:") return null; - return url.href; - } catch { - return null; - } -} - async function startDeviceAuthorization(oauthHost: string, signal: AbortSignal): Promise { const response = await fetch(`${oauthHost}/api/oauth/device_authorization`, { method: "POST", @@ -86,16 +75,9 @@ async function startDeviceAuthorization(oauthHost: string, signal: AbortSignal): const json = await readJson(response); const deviceCode = json?.device_code; const userCode = json?.user_code; - const verificationUri = json?.verification_uri; - const verificationUriComplete = json?.verification_uri_complete; - if ( - typeof deviceCode !== "string" || - typeof userCode !== "string" || - typeof verificationUri !== "string" || - typeof verificationUriComplete !== "string" || - !trustedHttpUrl(verificationUriComplete) || - !trustedHttpUrl(verificationUri) - ) { + const verificationUri = trustedHttpUrl(json?.verification_uri); + const verificationUriComplete = trustedHttpUrl(json?.verification_uri_complete); + if (typeof deviceCode !== "string" || typeof userCode !== "string" || !verificationUri || !verificationUriComplete) { throw new Error(`Invalid Kimi Code device authorization response: ${JSON.stringify(json)}`); } diff --git a/packages/ai/src/auth/oauth/meta.ts b/packages/ai/src/auth/oauth/meta.ts index b9f331cc8..24a692f8f 100644 --- a/packages/ai/src/auth/oauth/meta.ts +++ b/packages/ai/src/auth/oauth/meta.ts @@ -15,6 +15,7 @@ import type { OAuthAuth, OAuthCredential, ProviderAuthInteraction } from "../types.ts"; import { pollOAuthDeviceCodeFlow } from "./device-code.ts"; +import { trustedHttpUrl } from "./trusted-url.ts"; // Muse Code CLI client id. const CLIENT_ID = "1031625952748946"; @@ -54,18 +55,6 @@ function errorDetail(json: Record | null): string { return ""; } -/** The verification URI is opened in the user's browser; only http(s) URLs are trusted. */ -function trustedHttpUrl(value: unknown): string | null { - if (typeof value !== "string" || !value) return null; - try { - const url = new URL(value); - if (url.protocol !== "https:" && url.protocol !== "http:") return null; - return url.href; - } catch { - return null; - } -} - function positiveNumber(value: unknown): number | undefined { return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : undefined; } diff --git a/packages/ai/src/auth/oauth/radius.ts b/packages/ai/src/auth/oauth/radius.ts index 0b82b9d78..811410d8a 100644 --- a/packages/ai/src/auth/oauth/radius.ts +++ b/packages/ai/src/auth/oauth/radius.ts @@ -14,6 +14,7 @@ import type { OAuthAuth, OAuthCredential, ProviderAuthInteraction } from "../typ import { startOAuthCallbackServer } from "./callback-server.ts"; import { pollOAuthDeviceCodeFlow } from "./device-code.ts"; import { generatePKCE } from "./pkce.ts"; +import { trustedHttpUrl } from "./trusted-url.ts"; const CALLBACK_HOST = "127.0.0.1"; const CALLBACK_PORT = 1456; @@ -51,10 +52,11 @@ async function loadRadiusOAuthDiscovery(gateway: string, signal: AbortSignal): P } const discovery = (await response.json()) as Partial; - if (typeof discovery.authorizationEndpoint !== "string") { + const authorizationEndpoint = trustedHttpUrl(discovery.authorizationEndpoint); + if (!authorizationEndpoint) { throw new Error(`Invalid Radius OAuth config from ${gateway}`); } - return { authorizationEndpoint: discovery.authorizationEndpoint }; + return { authorizationEndpoint }; } class OAuthResponseError extends Error { @@ -210,11 +212,15 @@ async function requestDeviceAuthorization(gateway: string, signal: AbortSignal): if (!data.device_code || !data.user_code || !data.verification_uri || !data.expires_in) { throw new Error("Radius OAuth device authorization response is missing required fields"); } + const verificationUri = trustedHttpUrl(data.verification_uri); + if (!verificationUri) { + throw new Error("Untrusted verification_uri in Radius OAuth device authorization response"); + } return { device_code: data.device_code, user_code: data.user_code, - verification_uri: data.verification_uri, + verification_uri: verificationUri, expires_in: data.expires_in, interval: data.interval, }; diff --git a/packages/ai/src/auth/oauth/trusted-url.ts b/packages/ai/src/auth/oauth/trusted-url.ts new file mode 100644 index 000000000..a0a9450b7 --- /dev/null +++ b/packages/ai/src/auth/oauth/trusted-url.ts @@ -0,0 +1,10 @@ +/** + * The `http:` or `https:` URL `value` names, normalized, or null. Login flows open server-supplied URLs with the + * platform URL handler, which would also launch a `file:` URL or a custom protocol handler. Normalizing + * percent-encodes control characters, so the URL can also be embedded in terminal hyperlinks. + */ +export function trustedHttpUrl(value: unknown): string | null { + if (typeof value !== "string" || !URL.canParse(value)) return null; + const url = new URL(value); + return url.protocol === "https:" || url.protocol === "http:" ? url.href : null; +} diff --git a/packages/ai/src/cli.ts b/packages/ai/src/cli.ts index 04fdbaa25..8ad800f9f 100644 --- a/packages/ai/src/cli.ts +++ b/packages/ai/src/cli.ts @@ -1,7 +1,7 @@ #!/usr/bin/env node import { randomUUID } from "node:crypto"; -import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { chmodSync, existsSync, readFileSync, writeFileSync } from "node:fs"; import { createInterface } from "node:readline"; import type { AuthPrompt, OAuthCredential, Provider } from "./index.ts"; import { builtinProviders } from "./providers/all.ts"; @@ -26,7 +26,10 @@ function loadAuth(): Record { } function saveAuth(auth: Record): void { - writeFileSync(AUTH_FILE, JSON.stringify(auth, null, 2), "utf-8"); + // OAuth tokens: readable by the user only. `mode` applies when the file is created; chmod covers a file an older + // version created, before the new tokens are in it. + if (existsSync(AUTH_FILE)) chmodSync(AUTH_FILE, 0o600); + writeFileSync(AUTH_FILE, JSON.stringify(auth, null, 2), { encoding: "utf-8", mode: 0o600 }); } async function answerPrompt(rl: ReturnType, authPrompt: AuthPrompt): Promise { diff --git a/packages/ai/test/cache-retention.test.ts b/packages/ai/test/cache-retention.test.ts index f943f0529..8e06cea93 100644 --- a/packages/ai/test/cache-retention.test.ts +++ b/packages/ai/test/cache-retention.test.ts @@ -505,7 +505,7 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { }); it.each([ - MODELS.opencode["deepseek-v4-flash"], + MODELS.opencode["deepseek-v4.1-flash"], MODELS.opencode["deepseek-v4-pro"], MODELS.opencode["kimi-k2.5"], MODELS.opencode["kimi-k2.6"], diff --git a/packages/ai/test/pi-messages.test.ts b/packages/ai/test/pi-messages.test.ts index 54b3d4d9e..78d7328bf 100644 --- a/packages/ai/test/pi-messages.test.ts +++ b/packages/ai/test/pi-messages.test.ts @@ -267,6 +267,34 @@ describe("pi-messages", () => { expect(message.errorMessage).toContain("No API key provided"); }); + it("rejects content indexes that are not the next or an existing block", async () => { + const cases: unknown[][] = [ + [{ type: "text_start", contentIndex: 4294967000 }], + [{ type: "text_start", contentIndex: "__proto__" }], + [ + { type: "text_start", contentIndex: 0 }, + { + type: "toolcall_end", + contentIndex: "__proto__", + toolCall: { polluted: true }, + }, + ], + [{ type: "text_delta", contentIndex: 0, delta: "x" }], + ]; + for (const events of cases) { + const { baseUrl } = await startServer({ + events: [{ type: "start" }, ...events, { type: "done", reason: "stop", usage }], + }); + const message = await stream(createModel(baseUrl), normalizeContext(context), { apiKey: "test-key" }).result(); + server?.close(); + + expect(message.stopReason).toBe("error"); + expect(message.errorMessage).toContain("Invalid contentIndex"); + expect(message.content.length).toBeLessThanOrEqual(1); + } + expect(([] as unknown as Record).polluted).toBeUndefined(); + }); + it("errors when the stream ends without a terminal event", async () => { const { baseUrl } = await startServer({ events: [ diff --git a/packages/ai/test/radius-oauth.test.ts b/packages/ai/test/radius-oauth.test.ts index 793647762..73b2c0c86 100644 --- a/packages/ai/test/radius-oauth.test.ts +++ b/packages/ai/test/radius-oauth.test.ts @@ -128,6 +128,39 @@ describe("Radius OAuth", () => { expect(fetchMock).toHaveBeenCalledOnce(); }); + it("rejects an authorization endpoint that is not an http(s) URL", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => jsonResponse({ authorizationEndpoint: "file:///Users/victim/Downloads/payload.app" })), + ); + const events: AuthEvent[] = []; + + const oauth = createRadiusOAuth({ name: "Radius", gateway: GATEWAY }); + await expect(oauth.login(interaction("browser", events))).rejects.toThrow( + `Invalid Radius OAuth config from ${GATEWAY}`, + ); + expect(events).toEqual([]); + }); + + it("rejects a device verification URI that is not an http(s) URL", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => + jsonResponse({ + device_code: "device-code", + user_code: "ABCD-1234", + verification_uri: "file:///Users/victim/Downloads/payload.app", + expires_in: 600, + }), + ), + ); + const events: AuthEvent[] = []; + + const oauth = createRadiusOAuth({ name: "Radius", gateway: GATEWAY }); + await expect(oauth.login(interaction("device-code", events))).rejects.toThrow("Untrusted verification_uri"); + expect(events).toEqual([]); + }); + it("exchanges the browser callback code before showing the sign-in page", async () => { let tokenStatus = 400; vi.stubGlobal( diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 080bb8913..59827b2af 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -22,6 +22,7 @@ - Fixed HTML export not showing system messages: the system prompt and tools, and later section updates such as after `/reload`. `pi --export` now includes the system prompt and tools too ([#11261](https://github.com/earendil-works/pi/issues/11261), [#3464](https://github.com/earendil-works/pi/issues/3464), [#11290](https://github.com/earendil-works/pi/pull/11290)) - Fixed aborting a bash command on macOS sometimes running the next command of a list such as `long_step; next_step`. pi now stops the process group before killing it ([#11252](https://github.com/earendil-works/pi/issues/11252), [#11289](https://github.com/earendil-works/pi/pull/11289)) - Fixed streamed tool calls re-running their call renderer on every argument delta; the interactive mode now renders the latest arguments once per frame, so a large streamed `write` no longer re-highlights its whole content per delta ([#9265](https://github.com/earendil-works/pi/issues/9265)) +- Fixed the login dialog opening sign-in URLs that are not web URLs, and printing control characters from sign-in, device-code, and info URLs, which could end a terminal hyperlink and send escape sequences to the terminal ## [1.4.0] - 2026-10-21 diff --git a/packages/coding-agent/src/modes/interactive/components/login-dialog.ts b/packages/coding-agent/src/modes/interactive/components/login-dialog.ts index 0803ff865..95e676803 100644 --- a/packages/coding-agent/src/modes/interactive/components/login-dialog.ts +++ b/packages/coding-agent/src/modes/interactive/components/login-dialog.ts @@ -3,7 +3,9 @@ import { Container, type Focusable, getKeybindings, + hyperlink, Input, + replaceControlCharacters, Spacer, Text, TruncatedText, @@ -16,6 +18,31 @@ import { AuthUrlComponent } from "./auth-url.ts"; import { DynamicBorder } from "./dynamic-border.ts"; import { keyHint } from "./keybinding-hints.ts"; +/** + * The `http:` or `https:` URL `value` names, normalized, or undefined. The shared launcher also opens files and custom + * protocols, so login only opens web URLs. Normalizing percent-encodes control characters, which would otherwise end a + * terminal hyperlink early. + */ +function webUrl(value: string): string | undefined { + if (!URL.canParse(value)) return undefined; + const url = new URL(value); + return url.protocol === "https:" || url.protocol === "http:" ? url.href : undefined; +} + +/** + * `value` normalized as a web URL, else with control characters replaced: an untrusted string printed as is could end a + * hyperlink early or send its own escape sequences to the terminal. + */ +function displayUrl(value: string): string { + return webUrl(value) ?? replaceControlCharacters(value); +} + +/** `text` as a terminal hyperlink to `url` when that is a web URL, else plain `text`. */ +function webLink(text: string, url: string): string { + const target = webUrl(url); + return target === undefined ? text : hyperlink(text, target); +} + /** * Login dialog component - replaces editor during OAuth login flow */ @@ -121,7 +148,8 @@ export class LoginDialogComponent extends Container implements Focusable { this.clearDeviceCodeAction(); this.contentContainer.clear(); this.contentContainer.addChild(new Spacer(1)); - this.authUrl = new AuthUrlComponent(this.tui, url); + const target = webUrl(url); + this.authUrl = new AuthUrlComponent(this.tui, target ?? replaceControlCharacters(url)); this.contentContainer.addChild(this.authUrl); if (instructions) { @@ -129,7 +157,7 @@ export class LoginDialogComponent extends Container implements Focusable { this.contentContainer.addChild(new Text(theme.fg("warning", instructions), 1, 0)); } - openBrowser(url); + if (target !== undefined) openBrowser(target); this.tui.requestRender(); } @@ -146,12 +174,11 @@ export class LoginDialogComponent extends Container implements Focusable { this.clearDeviceCodeAction(); this.contentContainer.clear(); this.contentContainer.addChild(new Spacer(1)); - const linkedUrl = `\x1b]8;;${info.verificationUri}\x07${info.verificationUri}\x1b]8;;\x07`; - this.contentContainer.addChild(new Text(theme.fg("accent", linkedUrl), 1, 0)); + const shownUrl = displayUrl(info.verificationUri); + this.contentContainer.addChild(new Text(theme.fg("accent", webLink(shownUrl, shownUrl)), 1, 0)); const clickHint = process.platform === "darwin" ? "Cmd+click to open" : "Ctrl+click to open"; - const hyperlink = `\x1b]8;;${info.verificationUri}\x07${clickHint}\x1b]8;;\x07`; - this.contentContainer.addChild(new Text(theme.fg("dim", hyperlink), 1, 0)); + this.contentContainer.addChild(new Text(theme.fg("dim", webLink(clickHint, info.verificationUri)), 1, 0)); this.contentContainer.addChild(new Spacer(1)); this.contentContainer.addChild(new Text(theme.fg("warning", `Enter code: ${info.userCode}`), 1, 0)); @@ -187,11 +214,10 @@ export class LoginDialogComponent extends Container implements Focusable { } if (info.openBrowserOnConfirm) { try { - // The shared launcher also opens files; device-code login must only open web URLs. - const url = new URL(info.verificationUri); - if (url.protocol === "https:" || url.protocol === "http:") openBrowser(url.href); + const target = webUrl(info.verificationUri); + if (target !== undefined) openBrowser(target); } catch { - // Invalid URLs or launcher failures must not interrupt login. + // Launcher failures must not interrupt login. } } } @@ -260,9 +286,9 @@ export class LoginDialogComponent extends Container implements Focusable { this.contentContainer.addChild(new Spacer(1)); this.contentContainer.addChild(new Text(theme.fg("text", message), 1, 0)); for (const link of links) { - const text = link.label ? `${link.label}: ${link.url}` : link.url; - const hyperlink = `\x1b]8;;${link.url}\x07${text}\x1b]8;;\x07`; - this.contentContainer.addChild(new Text(theme.fg("accent", hyperlink), 1, 0)); + const url = displayUrl(link.url); + const text = link.label ? `${link.label}: ${url}` : url; + this.contentContainer.addChild(new Text(theme.fg("accent", webLink(text, url)), 1, 0)); } if (showCloseHint) { this.contentContainer.addChild(new Spacer(1)); diff --git a/packages/coding-agent/test/login-dialog-device-code.test.ts b/packages/coding-agent/test/login-dialog-device-code.test.ts index 7d5222ec5..269326c43 100644 --- a/packages/coding-agent/test/login-dialog-device-code.test.ts +++ b/packages/coding-agent/test/login-dialog-device-code.test.ts @@ -140,3 +140,47 @@ describe("device-code login", () => { expect(onComplete).not.toHaveBeenCalled(); }); }); + +describe("login URLs", () => { + beforeAll(() => initTheme("dark")); + beforeEach(() => { + vi.resetAllMocks(); + setKeybindings(new KeybindingsManager()); + }); + + it("opens only http(s) sign-in URLs", () => { + const { dialog } = createDialog(); + dialog.showAuth("file:///Users/victim/Downloads/payload.app"); + expect(openBrowser).not.toHaveBeenCalled(); + + dialog.showAuth("https://example.com/authorize"); + expect(openBrowser).toHaveBeenCalledExactlyOnceWith("https://example.com/authorize"); + }); + + it("keeps control characters in URLs from ending terminal hyperlinks", () => { + const { dialog } = createDialog(); + dialog.showDeviceCode({ ...device, verificationUri: "https://example.com/device\x07\x1b]52;c;Zm9v\x07" }); + dialog.showInfo("Docs", [{ label: "Help", url: "https://example.com/help\x07\x1b]52;c;Zm9v\x07" }]); + const output = dialog.render(120).join("\n"); + + expect(output).not.toContain("\x1b]52"); + expect(render(dialog)).toContain("https://example.com/device%07%1B]52;c;Zm9v"); + expect(render(dialog)).toContain("Help: https://example.com/help%07%1B]52;c;Zm9v"); + }); + + it("replaces control characters in URLs that are not web URLs", () => { + const injected = "x\x07\x1b]52;c;Zm9v\x07"; + const { dialog } = createDialog(); + dialog.showAuth(`file:///tmp/${injected}`); + expect(dialog.render(120).join("\n")).not.toContain("\x1b]52"); + + dialog.showDeviceCode({ ...device, verificationUri: injected }); + expect(dialog.render(120).join("\n")).not.toContain("\x1b]52"); + expect(render(dialog)).toContain("x ]52;c;Zm9v"); + + dialog.showInfo("Docs", [{ label: "Help", url: injected }]); + expect(dialog.render(120).join("\n")).not.toContain("\x1b]52"); + expect(render(dialog)).toContain("Help: x ]52;c;Zm9v"); + expect(openBrowser).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index e061eb6b7..a6431c209 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -28,6 +28,7 @@ - Fixed task span error messages and codemode call previews keeping the whole text they were cut from alive ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed partial commits of a streamed answer copying every growing text, thinking and tool-call argument string in full on each commit (about 20 s of CPU and a 100 MB heap peak for a 500 KB tool call): they now append new text with Chord's `appendText()`, and apply the argument changes of their own parse of the streamed argument text ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed stored assistant messages lacking `thinkingLevel`: a generation records the Pi thinking level its request asked for on the response, also after polling a deferred one, as pi-agent-core's agent loop does +- Fixed partial commits polluting `Object.prototype` when a streamed tool call's arguments gained a `__proto__` member after an earlier commit: the merge read the missing member through to `Object.prototype` and wrote into it ## [1.4.0] - 2026-10-21 diff --git a/packages/durable/src/harness/json.ts b/packages/durable/src/harness/json.ts index 0922fe45c..5b218fb43 100644 --- a/packages/durable/src/harness/json.ts +++ b/packages/durable/src/harness/json.ts @@ -26,7 +26,9 @@ export function assignJson; - const current: JsonValue | undefined = slots[key]; + // A `__proto__` member the target lacks reads through to Object.prototype; merging into that would pollute every + // object in the process. Other inherited members are functions, which are not merged into. + const current: JsonValue | undefined = key === "__proto__" && !Object.hasOwn(slots, key) ? undefined : slots[key]; const fields = plan?.kind === "fields" ? plan.fields : undefined; if (isRecord(current) && isRecord(value)) { for (const name of Object.keys(current)) if (!Object.hasOwn(value, name)) delete current[name]; diff --git a/packages/durable/test/harness-json.test.ts b/packages/durable/test/harness-json.test.ts new file mode 100644 index 000000000..963699770 --- /dev/null +++ b/packages/durable/test/harness-json.test.ts @@ -0,0 +1,23 @@ +import { type JsonValue, track } from "@earendil-works/chord/delta"; +import { describe, expect, it } from "vitest"; +import { assignJson } from "../src/harness/json.ts"; + +type Doc = { message?: JsonValue }; + +function assign(tracker: ReturnType>, value: JsonValue): void { + const change = tracker.beginChange(); + assignJson(change.state as Doc, "message", value); + tracker.adopt(change.prepare()); +} + +describe("assignJson", () => { + it("does not merge a new __proto__ member into Object.prototype", () => { + const tracker = track({}); + assign(tracker, { args: JSON.parse('{"path":"x"}') }); + assign(tracker, { args: JSON.parse('{"path":"xy","__proto__":{"polluted":true}}') }); + + expect(({} as Record).polluted).toBeUndefined(); + const args = (tracker.value.message as { args: object }).args; + expect(Object.getOwnPropertyDescriptor(args, "__proto__")?.value).toEqual({ polluted: true }); + }); +}); diff --git a/packages/tui/CHANGELOG.md b/packages/tui/CHANGELOG.md index f3e85bc4f..bb7634ec6 100644 --- a/packages/tui/CHANGELOG.md +++ b/packages/tui/CHANGELOG.md @@ -2,10 +2,15 @@ ## [Unreleased] +### Added + +- Added `replaceControlCharacters()` to replace runs of C0 and C1 control characters with a space + ### Fixed - Fixed fullscreen double-click selection splitting paths and tokens at `~`, `.`, `\`, and `+`. Word characters now match iTerm2's defaults ([#10980](https://github.com/earendil-works/pi/issues/10980)) - Fixed `fuzzyMatch()` ranking a query scattered over an earlier part of the text above a contiguous match later in it. It now also scores each contiguous occurrence of the query and keeps the best alignment ([#11220](https://github.com/earendil-works/pi/issues/11220), [#11296](https://github.com/earendil-works/pi/pull/11296)) +- Fixed `ProcessTerminal.setTitle()` writing control characters from the title, which could end the title sequence and send the rest to the terminal as escape sequences ## [1.4.0] - 2026-10-21 diff --git a/packages/tui/src/index.ts b/packages/tui/src/index.ts index eb962609f..61e436795 100644 --- a/packages/tui/src/index.ts +++ b/packages/tui/src/index.ts @@ -186,6 +186,7 @@ export { TuiMainScreen, type TuiMainScreenRenderState } from "./tui-main-screen. // Utilities export { getOsc8LinkAtColumn, + replaceControlCharacters, sliceByColumn, stripTerminalSequences, truncateToWidth, diff --git a/packages/tui/src/program-status.ts b/packages/tui/src/program-status.ts index 3b71e268d..919b6fc48 100644 --- a/packages/tui/src/program-status.ts +++ b/packages/tui/src/program-status.ts @@ -5,6 +5,8 @@ * Spec: https://www.superlogical.com/rex/docs/build/program-status */ +import { replaceControlCharacters } from "./utils.ts"; + export interface ProgramStatus { /** `clear` removes the status instead of reporting one. */ state: "idle" | "working" | "blocked" | "done" | "error" | "clear"; @@ -25,7 +27,6 @@ export function isProgramStatusReply(sequence: string): boolean { } const APP_PATTERN = /^[A-Za-z0-9_.+-]{1,32}$/; -const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f]+/g; /** Decoded `msg` limit. Its base64 encoding stays under the 2732-byte encoded limit. */ const MAX_MESSAGE_BYTES = 2048; @@ -47,7 +48,7 @@ export function formatProgramStatus(status: ProgramStatus): string { const pairs = [`state=${status.state}`]; if (status.app !== undefined && APP_PATTERN.test(status.app)) pairs.push(`app=${status.app}`); if (status.state === "blocked" && status.kind) pairs.push(`kind=${status.kind}`); - const message = truncateUtf8((status.message ?? "").replace(CONTROL_CHARACTERS, " ").trim(), MAX_MESSAGE_BYTES); + const message = truncateUtf8(replaceControlCharacters(status.message ?? "").trim(), MAX_MESSAGE_BYTES); if (message) pairs.push(`msg=${Buffer.from(message, "utf8").toString("base64")}`); return `\x1b]7501;${pairs.join(":")}\x1b\\`; } diff --git a/packages/tui/src/terminal.ts b/packages/tui/src/terminal.ts index 80d62b3e4..8e56022e9 100644 --- a/packages/tui/src/terminal.ts +++ b/packages/tui/src/terminal.ts @@ -10,6 +10,7 @@ import { type ProgramStatus, } from "./program-status.ts"; import { StdinBuffer } from "./stdin-buffer.ts"; +import { replaceControlCharacters } from "./utils.ts"; const TERMINAL_PROGRESS_KEEPALIVE_MS = 1000; const TERMINAL_PROGRESS_ACTIVE_SEQUENCE = "\x1b]9;4;3\x07"; @@ -590,8 +591,9 @@ export class ProcessTerminal implements Terminal { } setTitle(title: string): void { - // OSC 0;title BEL - set terminal window title - process.stdout.write(`\x1b]0;${title}\x07`); + // OSC 0;title BEL - set terminal window title. A BEL or ESC in the title would end the sequence early and + // send the rest to the terminal as escape sequences (session names come from session files). + process.stdout.write(`\x1b]0;${replaceControlCharacters(title)}\x07`); } setProgramStatus(status: ProgramStatus): void { diff --git a/packages/tui/src/utils.ts b/packages/tui/src/utils.ts index 272366899..e8a49e24c 100644 --- a/packages/tui/src/utils.ts +++ b/packages/tui/src/utils.ts @@ -326,6 +326,13 @@ export function stripTerminalSequences(str: string): string { return result; } +const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f]+/g; + +/** Replace each run of C0 and C1 control characters (ESC, BEL, newlines, ...) with one space. */ +export function replaceControlCharacters(str: string): string { + return str.replace(CONTROL_CHARACTERS, " "); +} + interface GraphemeCellRange { start: number; end: number; diff --git a/packages/tui/test/terminal.test.ts b/packages/tui/test/terminal.test.ts index 437994a38..de3bc40eb 100644 --- a/packages/tui/test/terminal.test.ts +++ b/packages/tui/test/terminal.test.ts @@ -395,6 +395,26 @@ describe("ProcessTerminal progress", () => { }); }); +describe("ProcessTerminal title", () => { + it("replaces control characters so the title cannot end the OSC sequence", () => { + const terminal = new ProcessTerminal(); + const writes: string[] = []; + const previousWrite = process.stdout.write; + + process.stdout.write = ((chunk: string | Uint8Array) => { + writes.push(String(chunk)); + return true; + }) as typeof process.stdout.write; + + try { + terminal.setTitle("pi - work\x07\x1b]52;c;Zm9v\x07\u009b2J - repo"); + assert.deepEqual(writes, ["\x1b]0;pi - work ]52;c;Zm9v 2J - repo\x07"]); + } finally { + process.stdout.write = previousWrite; + } + }); +}); + describe("ProcessTerminal dimensions", () => { it("falls back to COLUMNS and LINES before default dimensions", () => { const previousColumnsDescriptor = Object.getOwnPropertyDescriptor(process.stdout, "columns"); From 363ad78bf0625e1dfbe848429782f5e07bb1584e Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Mon, 26 Oct 2026 21:27:08 +0100 Subject: [PATCH 017/271] feat(ai): add GPT-6.1 Terra Add gpt-6.1-terra to the OpenAI, Azure OpenAI Responses, and OpenAI Codex providers with official pricing ($1 input, $0.05 cached input, $1.25 cache writes, $5 output, long-context tier above 272k), tool search, additional tools, and mid-conversation effort changes, as for GPT-6.1 Sol. Unlike Sol it accepts reasoning.effort "none", so off is supported; both reject "minimal" (verified against Codex). Codex serves it with a 500K context window. --- packages/ai/CHANGELOG.md | 1 + packages/ai/scripts/generate-models.ts | 47 +++++++++++++++++++++--- packages/ai/test/cache-retention.test.ts | 17 ++++++--- packages/ai/test/max-thinking.test.ts | 2 +- packages/ai/test/supports-xhigh.test.ts | 14 ++++++- 5 files changed, 68 insertions(+), 13 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 244d96f74..882ce5609 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -10,6 +10,7 @@ - Added optional `openBrowserOnConfirm` and `copyCodeOnConfirm` to `device_code` auth events and `OAuthDeviceCodeInfo`. Both default to `true`; set them to `false` to stop clients from offering to open the verification page or copy the code ([#9282](https://github.com/earendil-works/pi/issues/9282), [#9301](https://github.com/earendil-works/pi/pull/9301)) - Added `StreamingJsonParser`, an incremental, tolerant parser for JSON text that arrives in pieces (each piece is read once; snapshots of the partial value never change once returned), and `ToolCallArgumentsStream`, which every provider now uses for streamed tool-call arguments; its `append()` also returns the changes each piece made (`JsonChange`: a value set at a path, or text appended to a string), for writers that keep their own copy ([#9265](https://github.com/earendil-works/pi/issues/9265)) +- Added GPT-6.1 Terra to the OpenAI, Azure OpenAI Responses, and OpenAI Codex providers with official pricing ($1 input, $0.05 cached input, $1.25 cache writes, $5 output, long-context tier above 272k), tool search, and mid-conversation effort changes. Unlike GPT-6.1 Sol it supports `off` (reasoning effort `none`); Codex serves it with a 500K context window ### Changed diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index e9027559e..96d324653 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -378,6 +378,7 @@ const OPENAI_TOOL_SEARCH_MODEL_IDS = new Set([ "gpt-6-sol", "gpt-6-luna", "gpt-6.1-sol", + "gpt-6.1-terra", ]); const OPENAI_ADDITIONAL_TOOLS_MODEL_IDS = OPENAI_TOOL_SEARCH_MODEL_IDS; const OPENAI_MID_CONVO_SYSTEM_MESSAGE_MODEL_IDS = OPENAI_TOOL_SEARCH_MODEL_IDS; @@ -389,10 +390,17 @@ const OPENAI_CODEX_ADDITIONAL_TOOLS_MODEL_IDS = new Set([ "gpt-6-sol", "gpt-6-luna", "gpt-6.1-sol", + "gpt-6.1-terra", ]); // OpenAI documents configuration_update for the GPT-6 family. // https://developers.openai.com/api/docs/guides/reasoning -const OPENAI_MID_CONVO_EFFORT_MODEL_IDS = new Set(["gpt-6-astra", "gpt-6-sol", "gpt-6-luna", "gpt-6.1-sol"]); +const OPENAI_MID_CONVO_EFFORT_MODEL_IDS = new Set([ + "gpt-6-astra", + "gpt-6-sol", + "gpt-6-luna", + "gpt-6.1-sol", + "gpt-6.1-terra", +]); const OPENAI_LONG_CONTEXT_INPUT_THRESHOLD = 272000; const OPENAI_SHORT_CONTEXT_CAPPED_MODEL_IDS = new Set([ "gpt-5.4", @@ -404,6 +412,7 @@ const OPENAI_SHORT_CONTEXT_CAPPED_MODEL_IDS = new Set([ "gpt-6-sol", "gpt-6-luna", "gpt-6.1-sol", + "gpt-6.1-terra", ]); const OPENAI_LONG_CONTEXT_PRICING_MODEL_IDS = new Set([ "gpt-5.4", @@ -417,6 +426,7 @@ const OPENAI_LONG_CONTEXT_PRICING_MODEL_IDS = new Set([ "gpt-6-sol", "gpt-6-luna", "gpt-6.1-sol", + "gpt-6.1-terra", ]); // Keep the generated default no less restrictive than coding-agent's historical @@ -455,6 +465,7 @@ const OPENAI_STANDARD_COSTS: Record = { "gpt-6-luna": { input: 0.1, output: 0.5, cacheRead: 0.01, cacheWrite: 0.125 }, "gpt-6-sol": { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 2.5 }, "gpt-6.1-sol": { input: 2, output: 10, cacheRead: 0.1, cacheWrite: 2.5 }, + "gpt-6.1-terra": { input: 1, output: 5, cacheRead: 0.05, cacheWrite: 1.25 }, }; const OPENAI_RESPONSES_NONE_REASONING_MODELS = new Set([ @@ -1072,13 +1083,15 @@ function applyThinkingLevelMetadata(model: Model): void { (model.id === "gpt-6-astra" || model.id === "gpt-6-sol" || model.id === "gpt-6-luna" || - model.id === "gpt-6.1-sol") && + model.id === "gpt-6.1-sol" || + model.id === "gpt-6.1-terra") && (model.api === "openai-responses" || model.api === "azure-openai-responses" || model.api === "openai-codex-responses") ) { mergeThinkingLevelMap(model, { - // GPT-6 Astra and GPT-6.1 Sol reject reasoning.effort "none". + // GPT-6 Astra and GPT-6.1 Sol reject reasoning.effort "none"; GPT-6.1 Terra accepts it. All reject + // "minimal" (verified on Codex, October 2026). off: model.id === "gpt-6-astra" || model.id === "gpt-6.1-sol" ? null : "none", minimal: null, low: "low", @@ -2957,6 +2970,18 @@ async function generateModels() { contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, maxTokens: 128000, }, + { + id: "gpt-6.1-terra", + name: "GPT-6.1 Terra", + api: "openai-responses", + baseUrl: "https://api.openai.com/v1", + provider: "openai", + reasoning: true, + input: ["text", "image"], + cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6.1-terra"]), + contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, + maxTokens: 128000, + }, { id: "gpt-6-astra", name: "GPT-6 Astra", @@ -3177,7 +3202,7 @@ async function generateModels() { // GPT-6.1 Sol uses the 500k default returned by /backend-api/codex/models. const CODEX_BASE_URL = "https://chatgpt.com/backend-api"; const CODEX_CONTEXT = 272000; - const CODEX_GPT_61_SOL_CONTEXT = 500000; + const CODEX_GPT_61_CONTEXT = 500000; const CODEX_GPT_56_CONTEXT = 272000; const CODEX_MAX_TOKENS = 128000; const codexModels: Model<"openai-codex-responses">[] = [ @@ -3190,7 +3215,19 @@ async function generateModels() { reasoning: true, input: ["text", "image"], cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6.1-sol"]), - contextWindow: CODEX_GPT_61_SOL_CONTEXT, + contextWindow: CODEX_GPT_61_CONTEXT, + maxTokens: CODEX_MAX_TOKENS, + }, + { + id: "gpt-6.1-terra", + name: "GPT-6.1 Terra", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: CODEX_BASE_URL, + reasoning: true, + input: ["text", "image"], + cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6.1-terra"]), + contextWindow: CODEX_GPT_61_CONTEXT, maxTokens: CODEX_MAX_TOKENS, }, { diff --git a/packages/ai/test/cache-retention.test.ts b/packages/ai/test/cache-retention.test.ts index 8e06cea93..89ce3d8f6 100644 --- a/packages/ai/test/cache-retention.test.ts +++ b/packages/ai/test/cache-retention.test.ts @@ -240,12 +240,17 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { }); describe("OpenAI Responses Provider", () => { - it.each(["gpt-5.6-luna", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-6-astra", "gpt-6-luna", "gpt-6-sol"] as const)( - "does not enable cache warming from the documented TTL alone for %s", - (modelId) => { - expect(getModel("openai", modelId).promptCache).toBeUndefined(); - }, - ); + it.each([ + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-terra", + ] as const)("does not enable cache warming from the documented TTL alone for %s", (modelId) => { + expect(getModel("openai", modelId).promptCache).toBeUndefined(); + }); it.skipIf(!process.env.OPENAI_API_KEY)( "should not set prompt_cache_retention when PI_CACHE_RETENTION is not set", diff --git a/packages/ai/test/max-thinking.test.ts b/packages/ai/test/max-thinking.test.ts index 653d9a820..bc76d0d08 100644 --- a/packages/ai/test/max-thinking.test.ts +++ b/packages/ai/test/max-thinking.test.ts @@ -67,7 +67,7 @@ describe("max thinking level", () => { expect(clampThinkingLevel(model, "xhigh")).toBe("max"); }); - it.each(["gpt-5.6-sol", "gpt-6-astra", "gpt-6-sol", "gpt-6-luna", "gpt-6.1-sol"] as const)( + it.each(["gpt-5.6-sol", "gpt-6-astra", "gpt-6-sol", "gpt-6-luna", "gpt-6.1-sol", "gpt-6.1-terra"] as const)( "sends max to the Codex Responses API for %s", async (modelId) => { const model = getModel("openai-codex", modelId)!; diff --git a/packages/ai/test/supports-xhigh.test.ts b/packages/ai/test/supports-xhigh.test.ts index d16fdb61d..91f5c72f0 100644 --- a/packages/ai/test/supports-xhigh.test.ts +++ b/packages/ai/test/supports-xhigh.test.ts @@ -93,13 +93,14 @@ describe("getSupportedThinkingLevels", () => { "gpt-6-sol", "gpt-6-luna", "gpt-6.1-sol", + "gpt-6.1-terra", ] as const)("includes xhigh for openai-codex %s models", (modelId) => { const model = getModel("openai-codex", modelId); expect(model).toBeDefined(); expect(getSupportedThinkingLevels(model!)).toContain("xhigh"); }); - it.each(["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-6-sol", "gpt-6-luna"] as const)( + it.each(["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-6-sol", "gpt-6-luna", "gpt-6.1-terra"] as const)( "includes xhigh and max for OpenAI %s models", (modelId) => { const model = getModel("openai", modelId); @@ -123,11 +124,22 @@ describe("getSupportedThinkingLevels", () => { } }); + // GPT-6.1 Terra accepts reasoning.effort "none" but, like GPT-6.1 Sol, rejects "minimal". + it("supports off but not minimal for GPT-6.1 Terra", () => { + for (const provider of ["openai", "azure", "openai-codex"] as const) { + const model = getModel(provider, "gpt-6.1-terra"); + expect(model).toBeDefined(); + expect(model!.thinkingLevelMap?.off).toBe("none"); + expect(model!.thinkingLevelMap?.minimal).toBe(provider === "openai-codex" ? "low" : null); + } + }); + it.each([ ["gpt-6-sol", { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 2.5 }, 272000], ["gpt-6-luna", { input: 0.1, output: 0.5, cacheRead: 0.01, cacheWrite: 0.125 }, 272000], // Codex serves GPT-6.1 Sol with a 500K default context window (#10840) ["gpt-6.1-sol", { input: 2, output: 10, cacheRead: 0.1, cacheWrite: 2.5 }, 500000], + ["gpt-6.1-terra", { input: 1, output: 5, cacheRead: 0.05, cacheWrite: 1.25 }, 500000], ] as const)("includes official metadata for OpenAI and Codex %s", (modelId, cost, codexContextWindow) => { for (const provider of ["openai", "openai-codex"] as const) { const model = getModel(provider, modelId); From 8b0caefc2260a28d2cec0a7ce3daef1d3de24776 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 26 Oct 2026 20:44:20 +0000 Subject: [PATCH 018/271] feat(durable,chord): publish commits at adoption, address-following watches, synchronous observers Durable publishes a stored commit inside Storage.commit's adopt: the telemetry events, its own observers (document watches, conversation views, task graph, scheduler, waits), and a dispatch of host subscribeCommits listeners one microtask later, before every wait and abort the commit triggers. A commit the storage rejects after adoption is published before the Session fails; a throwing telemetry adapter is reported as host_error "telemetry" instead of rejecting a durable commit. Chord's watch listeners take one change object, { value, ops, retired } ({ value } for native watches); retirement is a flag, not an end. Durable's document watches follow their address, and a task document's watch ends after its retirement call. New synchronous observers for adapters: observeDoc, observeConversation, observeTaskGraph; the watches are built on them. The env file watcher's change type is renamed FileChange. --- packages/chord/CHANGELOG.md | 8 +- packages/chord/README.md | 38 +- packages/chord/src/index.ts | 2 + packages/chord/src/watch.ts | 226 +++++--- packages/chord/test/watch.test.ts | 424 +++++++++++--- packages/durable/CHANGELOG.md | 17 +- packages/durable/README.md | 16 +- packages/durable/docs/spec.md | 293 ++++++++-- packages/durable/src/env/index.ts | 6 +- packages/durable/src/env/node-watch.ts | 10 +- packages/durable/src/env/node.ts | 4 +- packages/durable/src/harness/events.ts | 11 +- packages/durable/src/harness/harness.ts | 33 +- packages/durable/src/harness/scheduler.ts | 29 +- packages/durable/src/harness/task-graph.ts | 47 +- packages/durable/src/harness/types.ts | 33 +- packages/durable/src/harness/view.ts | 40 +- packages/durable/src/index.ts | 4 +- packages/durable/src/session/session.ts | 533 ++++++++++++++---- .../durable/src/testing/env-conformance.ts | 10 +- packages/durable/src/types.ts | 99 +++- .../durable/test/env-node-conformance.test.ts | 4 +- packages/durable/test/examples/04-watches.ts | 2 +- .../durable/test/examples/15-real-model.ts | 2 +- packages/durable/test/examples/18-json.ts | 4 +- .../durable/test/examples/20-late-join.ts | 5 +- .../test/examples/21-subagent-foreground.ts | 2 +- .../test/examples/22-subagent-background.ts | 3 +- .../durable/test/harness-compaction.test.ts | 12 +- packages/durable/test/harness-events.test.ts | 8 +- .../test/harness-generation-recovery.test.ts | 4 +- .../test/harness-input-entries.test.ts | 2 +- .../test/harness-nested-tools-restart.test.ts | 2 +- .../durable/test/harness-nested-tools.test.ts | 12 +- .../durable/test/harness-observers.test.ts | 459 +++++++++++++++ .../test/harness-storage-failure.test.ts | 2 +- .../test/harness-structured-output.test.ts | 2 +- .../durable/test/harness-structured.test.ts | 4 +- .../durable/test/harness-submissions.test.ts | 18 +- .../durable/test/harness-task-graph.test.ts | 33 +- packages/durable/test/harness-tasks.test.ts | 276 ++++++++- .../durable/test/harness-telemetry.test.ts | 92 ++- packages/durable/test/harness-view.test.ts | 4 +- .../durable/test/session-adoption.test.ts | 268 ++++++++- .../session-checkpoints-migrations.test.ts | 6 +- .../test/session-deferred-commits.test.ts | 4 +- .../test/session-document-retention.test.ts | 25 +- .../durable/test/session-observers.test.ts | 411 ++++++++++++++ packages/durable/test/session-support.ts | 50 +- packages/durable/test/session-watches.test.ts | 328 +++++++++-- packages/durable/test/task-support.ts | 2 + packages/env/src/remote-env.ts | 4 +- packages/env/src/watch.ts | 10 +- 53 files changed, 3389 insertions(+), 554 deletions(-) create mode 100644 packages/durable/test/harness-observers.test.ts create mode 100644 packages/durable/test/session-observers.test.ts diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index e8b74b5e9..584aaddd0 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -5,14 +5,16 @@ ### Breaking Changes - `traceIn(options, fn, context)` takes the context last (was `traceIn(context, options, fn)`), like the rest of the API. +- Watch listeners take one change object before the context: `WatchHandle.start((change, context) => …)` with `change: WatchChange` = `{ value, ops, retired }`, and `NativeWatchHandle.start(({ value }, context) => …)` with `NativeWatchChange`. A change is plain data, those fields only; the frame's context is the second argument. `Watch.advance(change, context)` takes the same object, and a frame delivered alone, with nothing folded into it, reaches the listener as the object the source passed. Migration: `start(async (value, ops, ctx) => …)` becomes `start(async ({ value, ops }, ctx) => …)`, `start(async (value, ctx) => …)` on a native watch `start(async ({ value }, ctx) => …)`, and `advance(value, ops, ctx)` becomes `advance({ value, ops, retired: false }, ctx)`. +- Retirement is a flag, not an end: `WatchEnd` loses `retired`, and `Watch.retire(context)` is removed. A source reports a retirement with `advance({ value, ops, retired: true }, context)`, whose value is what the address shows after it, and the watch goes on: the next call has `retired: true`, folded with every other pending frame, so a document created at the address afterwards arrives in the same or a later call as `[["r", value]]`. `endAfterDelivery()` tells a watch its source sends nothing more: it takes no more frames, delivers what is pending, and ends `ended` after its last call returned, at once when nothing is pending and no call runs; it stays attached until then, so the source's `end()` (such as at its close), `stop()`, cancellation, or a throwing listener still end it first. Migration: a listener that waited for `closed` to give `retired` or tested for a `null` value tests `change.retired`; a source that called `retire(ctx)` calls `advance({ value, ops, retired: true }, ctx)`, then `endAfterDelivery()` when its owner ends with the retirement. ### Added - `NativeWatchable`: an in-process value that changes and can be watched (`value`, `watch(context)` returning a `NativeWatchHandle`), the in-process counterpart of `Watchable`. -- `WatchHandle` and `WatchEnd`, the one watch API, moved from Durable: a serialized observation with `value`, `start(listener)`, `stop()`, and `closed`. `WatchEnd` reasons are `stopped`, `cancelled`, `retired`, `ended`, `listener_error`, and `failed`. -- `Watch`, the one watch handle implementation, moved from Durable: a source creates one per watch and feeds it with `advance(value, ops, context)`, `retire(context)` (last frame `[["r", null]]`, then `retired`), `end()` or `end({ error })`, and `observeCancellation(signal)`. Frames that arrive while the listener runs (or before `start`) are delivered in one call with the newest value, their ops folded by `PendingOps` and the newest frame's context; past 10,000 pending ops after folding, the call gets one root replacement `[["r", newest]]` instead (replaces the collapse after 100 undelivered frames); termination drops the listener and every reference into the source. +- `WatchHandle` and `WatchEnd`, the one watch API, moved from Durable: a serialized observation with `value`, `start(listener)`, `stop()`, and `closed`; the listener gets a `WatchChange` (`{ value, ops, retired }`) and the context. `WatchEnd` reasons are `stopped`, `cancelled`, `ended`, `listener_error`, and `failed`. +- `Watch`, the one watch handle implementation, moved from Durable: a source creates one per watch and feeds it with `advance(change, context)` (`retired: true` for a retirement), `end()` or `end({ error })`, `endAfterDelivery()`, and `observeCancellation(signal)`. Frames that arrive while the listener runs (or before `start`) are delivered in one call with the newest value, their ops folded by `PendingOps` and the newest frame's context; past 10,000 pending ops after folding, the call gets one root replacement `[["r", newest]]` instead (replaces the collapse after 100 undelivered frames); termination drops the listener and every reference into the source. - `Watchable`: a value that changes and can be watched (`value`, `watch(context)` returning a `WatchHandle`), the common interface of documents, derives and other observable values. -- `NativeWatchHandle` and its implementation `NativeWatch`: the same lifecycle for in-process values, with listener `(value, context)` and no ops: one call per value in order, and past 100 pending values one call with the value of the optional `replace()` or the newest value. +- `NativeWatchHandle` and its implementation `NativeWatch`: the same lifecycle for in-process values, with listener `({ value }, context)` (`NativeWatchChange`) and no ops: one call per value in order, and past 100 pending values one call with the value of the optional `replace()` or the newest value. - `PendingOps` and `PendingTake` in `@earendil-works/chord/delta`: the pending ops of one slow consumer, kept raw and folded while they wait (appends to one path join, a primitive set replaces earlier leaf ops on its path, `r` drops everything before it); `size` counts the pending ops after folding; `take()` empties the queue and hands over the folded ops, or `replace` past 10,000 pending ops. Received batches and op tuples are never mutated. - `ownedSlice(text, start, end?)` in `@earendil-works/chord/delta`: a slice of a string that does not keep the whole string alive ([#9265](https://github.com/earendil-works/pi/issues/9265)) - `appendText(draft, key, text)` in `@earendil-works/chord/delta`: append to a string in a draft at O(text) cost, recorded as an append op without comparing the old and new strings (a plain `draft[key] += text` makes `prepare()` read the whole string, O(length) per change); `StringKeys` names the keys it accepts ([#9265](https://github.com/earendil-works/pi/issues/9265)) diff --git a/packages/chord/README.md b/packages/chord/README.md index 4239568bc..376b3625d 100644 --- a/packages/chord/README.md +++ b/packages/chord/README.md @@ -14,7 +14,7 @@ Contracts are runtime values described with [TypeBox](https://github.com/sinclai | Import | Contents | Runs in | |--------|----------|---------| -| `@earendil-works/chord` | Contracts (`local`, `remote`, `method`, `opaque`), declarations (through the factory's `self`: `state`, `derive`, `resource`, `on`, `task`), hosts (`createHost`, `defineFacet`, `defineRegistry`), links (`memoryLinkPair`, `ipcLink`, `webSocketLink`, `acceptWebSockets`), errors (`ChordError`, `isChordError`, `forbidden`, `invalid`). Also the shared base: `Context`, `JsonValue`, `JsonRepresentation`, `Draft`, `copyJson`, `isJsonValue`, `replicatedState` with its types, and the watch types `WatchHandle`, `NativeWatchHandle`, and `WatchEnd` with their implementations `Watch` and `NativeWatch`. | Node and browsers | +| `@earendil-works/chord` | Contracts (`local`, `remote`, `method`, `opaque`), declarations (through the factory's `self`: `state`, `derive`, `resource`, `on`, `task`), hosts (`createHost`, `defineFacet`, `defineRegistry`), links (`memoryLinkPair`, `ipcLink`, `webSocketLink`, `acceptWebSockets`), errors (`ChordError`, `isChordError`, `forbidden`, `invalid`). Also the shared base: `Context`, `JsonValue`, `JsonRepresentation`, `Draft`, `copyJson`, `isJsonValue`, `replicatedState` with its types, and the watch types `WatchHandle`, `NativeWatchHandle`, `WatchChange`, `NativeWatchChange`, and `WatchEnd` with their implementations `Watch` and `NativeWatch`. | Node and browsers | | `@earendil-works/chord/node` | Child host processes: `createHostManager`, `forkDriver`, `childControl`, `processLock`; the facet module loader `nodeLoader`. | Node | | `@earendil-works/chord/context` | Explicit cancellation, telemetry, and invocation values: `BACKGROUND_CONTEXT`, `withCancel`, `withAbortSignal`, `withoutAbortSignal`, `withTelemetry`, `traceIn`, `withContextValue`, `awaitWithContext`. | Node and browsers | | `@earendil-works/chord/delta` | The transactional JSON tracker, structural diff, immutable apply, and the wire encoder/decoder for operation batches. | Node and browsers | @@ -209,17 +209,26 @@ model; see [State](docs/guide.md#state) in the usage guide. `WatchHandle` is the one watch interface, implemented by watch sources such as Durable's documents, conversation views, and task graph. `watch.value` is the value at acquisition until `start(listener)` installs the sole asynchronous listener; each -later change arrives as `(value, ops, context)`, one call at a time, and `value` -advances to it. Changes made while the listener runs (or before `start`) arrive in -one call: the newest value, the ops of all of them folded by Delta's `PendingOps`, -and the newest change's context. Past 10,000 pending ops after folding the call -gets one root replacement `[["r", newest]]` instead. `stop()` is idempotent, drops -pending frames, and neither aborts nor awaits a running call; it and `closed` give -the `WatchEnd`: +later change arrives as `(change, context)`, one call at a time, with +`change: WatchChange` = `{ value, ops, retired }` (plain data; the context is +the second argument), and `value` advances to it. +Changes made while the listener runs (or before `start`) arrive in one call: the +newest value, the ops of all of them folded by Delta's `PendingOps`, and the newest +change's context. Past 10,000 pending ops after folding the call gets one root +replacement `[["r", newest]]` instead. + +`retired` says the document the watch follows was retired since the previous call. +It does not end the watch: the call carries what the address shows after the +retirement (the source decides; a Durable document keeps the last value the watch +was given, with no ops), and a document created there later arrives as +`[["r", value]]`, folded with what is pending, so one call can report a retirement +and carry the new value. + +`stop()` is idempotent, drops pending frames, and neither aborts nor awaits a +running call; it and `closed` give the `WatchEnd`: - `stopped`: `stop()` was called; - `cancelled`: the context the watch was acquired with was cancelled; -- `retired`: the source was retired, an explicit event after a last frame `[["r", null]]`; - `ended`: the source closed (its owner ended or closed); - `listener_error` with `error`: the listener threw or rejected; - `failed` with `error`: the source failed. @@ -227,9 +236,12 @@ the `WatchEnd`: An ended watch drops every reference into its source, so a stopped watch that is still referenced keeps only its last value alive. -Sources implement handles with `Watch`, fed through `advance(value, ops, context)`, -`retire(context)`, `end()` or `end({ error })`, and `observeCancellation(signal)`. +Sources implement handles with `Watch`, fed through `advance(change, context)` +(a `WatchChange`, `retired` set for a retirement), `end()` or `end({ error })` +(at once, dropping what is pending), `endAfterDelivery()` (take no more frames, +deliver what is pending, then end `ended` once the last call returned; the +source's `end()` still ends it first), and `observeCancellation(signal)`. `NativeWatch` implements `NativeWatchHandle`, the same lifecycle for -in-process values: the listener gets `(value, context)` without ops, one call per -value in order; past 100 pending values they collapse to one, the value of the +in-process values: the listener gets `({ value }, context)` without ops, one call +per value in order; past 100 pending values they collapse to one, the value of the optional `replace()` or the newest. diff --git a/packages/chord/src/index.ts b/packages/chord/src/index.ts index 8b8bd47d6..30ba50455 100644 --- a/packages/chord/src/index.ts +++ b/packages/chord/src/index.ts @@ -81,9 +81,11 @@ export type { Ctx, GetOptions, Readable, RefStatus, Shape, StateEvent } from "./ export { NativeWatch, type NativeWatchable, + type NativeWatchChange, type NativeWatchHandle, Watch, type Watchable, + type WatchChange, type WatchEnd, type WatchHandle, } from "./watch.ts"; diff --git a/packages/chord/src/watch.ts b/packages/chord/src/watch.ts index 76373de2f..74a7ef15b 100644 --- a/packages/chord/src/watch.ts +++ b/packages/chord/src/watch.ts @@ -8,16 +8,28 @@ import type { JsonValue } from "./json.ts"; * * - `stopped`: `stop()` was called. * - `cancelled`: the context the watch was acquired with was cancelled. - * - `retired`: the source was retired; its last frame was `[["r", null]]`. * - `ended`: the source closed (its owner ended or closed). * - `listener_error`: the listener threw or rejected with `error`. * - `failed`: the source failed with `error`. */ export type WatchEnd = - | { readonly reason: "stopped" | "cancelled" | "retired" | "ended" } + | { readonly reason: "stopped" | "cancelled" | "ended" } | { readonly reason: "listener_error"; readonly error: Error } | { readonly reason: "failed"; readonly error: unknown }; +/** + * One call of a JSON watch's listener, and one change a source hands its watch: the newest value, the ops that turn the + * previous value into it, and whether the document the watch follows was retired since the previous call. A retirement + * does not end the watch: `value` is then what the address shows after it (the source decides; a Durable document keeps + * the last value the watch was given, with no ops), and a document created there later arrives as `[["r", value]]`, + * possibly in the same call. Plain data: the frame's context is the listener's second argument, never part of the + * change. + */ +export type WatchChange = { readonly value: T; readonly ops: readonly Op[]; readonly retired: boolean }; + +/** One call of a native watch's listener: the new value. */ +export type NativeWatchChange = { readonly value: T }; + /** * Serialized observation of an immutable value: one listener call at a time; changes made meanwhile arrive folded into * the next call. @@ -26,7 +38,7 @@ export interface WatchHandle { /** Acquisition revision before start; latest delivered immutable revision afterward. */ readonly value: T; /** Install the sole asynchronous listener. Never invokes it inline. */ - start(listener: (value: T, ops: readonly Op[], context: Context) => Promise): void; + start(listener: (change: WatchChange, context: Context) => Promise): void; /** Idempotently stop future callbacks and return this watch's terminal result. */ stop(): Promise; /** Settle when the watch terminates; an already-running callback remains caller-owned. */ @@ -64,39 +76,32 @@ export interface NativeWatchHandle { /** Acquisition value before start; latest delivered value afterward. */ readonly value: T; /** Install the sole asynchronous listener. Never invokes it inline. */ - start(listener: (value: T, context: Context) => Promise): void; + start(listener: (change: NativeWatchChange, context: Context) => Promise): void; /** Idempotently stop future callbacks and return this watch's terminal result. */ stop(): Promise; /** Settle when the watch terminates; an already-running callback remains caller-owned. */ readonly closed: Promise; } -type WatchListener = (value: T, ops: readonly Op[], context: Context) => Promise; - -type NativeWatchListener = (value: T, context: Context) => Promise; - -/** One listener call: the value it delivers and the context of the newest frame it covers. */ -type Delivery = { - readonly value: T; - readonly context: Context; - /** The retirement: the watch ends `retired` once it is delivered. */ - readonly retires: boolean; -}; +type Listener = (change: C, context: Context) => Promise; /** * The lifecycle both handles share: serialized delivery and termination. A source feeds it and ends it explicitly; * termination drops every reference the watch holds into its source, so a stopped watch that is still referenced keeps - * nothing but its last value alive. Subclasses keep what is pending and say how a listener is called. + * nothing but its last value alive. Subclasses keep what is pending, each change apart from its frame's context, so the + * change a listener gets carries nothing but its data. */ -abstract class WatchLifecycle> { +abstract class WatchLifecycle { #detach: (() => void) | undefined; #report: ((error: unknown) => void) | undefined; readonly #closed = Promise.withResolvers(); #value: T; - #listener: L | undefined; + #listener: Listener | undefined; #started = false; #scheduled = false; #running = false; + /** Set by `endAfterDelivery()`: the source sends nothing more; the watch ends `ended` once it delivered the rest. */ + #ending = false; /** Set once, by `#terminate()`. */ #end: WatchEnd | undefined; #cancellationSignal: AbortSignal | undefined; @@ -116,7 +121,7 @@ abstract class WatchLifecycle> { return this.#closed.promise; } - start(listener: L): void { + start(listener: Listener): void { if (this.#started) throw new Error("Watch is already started"); if (this.#end !== undefined) throw new Error("Watch is stopped"); this.#started = true; @@ -144,21 +149,36 @@ abstract class WatchLifecycle> { this.#terminate({ reason: "cancelled" }); } - /** The source closed: end with `ended`, or with `failed` and the error when the source failed. */ + /** + * The source closed: end with `ended`, or with `failed` and the error when the source failed, at once, dropping what + * is pending. + */ end(failure?: { readonly error: unknown }): void { this.#terminate(failure === undefined ? { reason: "ended" } : { reason: "failed", error: failure.error }); } - /** Whether the watch still takes frames: it has not ended. */ - protected get open(): boolean { - return this.#end === undefined; + /** + * The source sends nothing more after its last frames, such as a retirement: take no more frames, deliver what is + * pending, then end `ended`, after the running call returns, if one runs. Until then the watch stays attached, so + * `end()`, `stop()`, cancellation, and a throwing listener still end it first; a watch never started keeps what is + * pending until one of them or `start()`. + */ + endAfterDelivery(): void { + if (this.#end !== undefined || this.#ending) return; + this.#ending = true; + if (!this.#running && this.pendingContext === undefined) this.#terminate({ reason: "ended" }); } - /** Take the next delivery, or undefined when nothing is pending. */ - protected abstract next(): D | undefined; + /** Whether the watch still takes frames: it has not ended, and its source has not ended. */ + protected get open(): boolean { + return this.#end === undefined && !this.#ending; + } - /** Call `listener` with one delivery. */ - protected abstract invoke(listener: L, delivery: D, context: Context): Promise; + /** The context of the newest frame the next call covers, or undefined when nothing is pending. */ + protected abstract get pendingContext(): Context | undefined; + + /** Take the next change; called only while something is pending. */ + protected abstract next(): C; /** Drop everything pending and every reference into the source; called once, on termination. */ protected abstract release(): void; @@ -177,7 +197,10 @@ abstract class WatchLifecycle> { }); } - /** Deliver until nothing is pending; what arrives during a call is taken after it returns. */ + /** + * Deliver until nothing is pending; what arrives during a call is taken after it returns. A watch whose source ended + * ends once it delivered the rest and its last call returned. + */ async #drain(): Promise { if (this.#running || this.#end !== undefined || !this.#started) return; this.#running = true; @@ -185,11 +208,15 @@ abstract class WatchLifecycle> { while (this.#end === undefined) { const listener = this.#listener; if (listener === undefined) break; - const delivery = this.next(); - if (delivery === undefined) break; - this.#value = delivery.value; + const frameContext = this.pendingContext; + if (frameContext === undefined) { + if (this.#ending) this.#terminate({ reason: "ended" }); + break; + } + const change = this.next(); + this.#value = change.value; try { - await this.invoke(listener, delivery, withoutAbortSignal(delivery.context)); + await listener(change, withoutAbortSignal(frameContext)); } catch (error) { if (this.#end === undefined) { this.#report?.(error); @@ -197,10 +224,6 @@ abstract class WatchLifecycle> { } break; } - if (delivery.retires) { - this.#terminate({ reason: "retired" }); - break; - } } } finally { this.#running = false; @@ -238,61 +261,67 @@ function toError(error: unknown): Error { } } -type WatchDelivery = Delivery & { readonly ops: readonly Op[] }; - -/** Canonical terminal frame of a retired source. */ -const RETIREMENT_OPERATIONS: readonly Op[] = [["r", null]]; - /** * The watch handle of a JSON source. Frames that arrive before `start()` or while the listener runs wait in one * `PendingOps` queue; the next call gets the newest value, the folded ops of every frame since the previous call (or - * `[["r", newest]]` after the queue overflowed) and the newest frame's context. `detach` unsubscribes it from its - * source on termination; `report` gets a listener's throw, which also ends the watch. + * `[["r", newest]]` after the queue overflowed), `retired` when any of them was a retirement, and the newest frame's + * context. A frame delivered alone is passed on as the source made it. `detach` unsubscribes it from its source on + * termination; `report` gets a listener's throw, which also ends the watch. */ -export class Watch extends WatchLifecycle, WatchDelivery> implements WatchHandle { +export class Watch extends WatchLifecycle> implements WatchHandle { readonly #ops = new PendingOps(); - /** The newest frame, while a call is pending. */ - #newest: { readonly value: T; readonly context: Context } | undefined; - #retiring = false; - - /** Queue the frame `ops`, which turn the previous value into `value`. */ - advance(value: T, ops: readonly Op[], context: Context): void { - if (!this.open || this.#retiring) return; - this.#ops.push(ops); - this.#newest = { value, context }; - this.pending(); - } + /** + * The source's change while it is the only frame pending: the call passes it on, with no allocation. A second frame + * drops it, so its ops stay only as long as `PendingOps` keeps them (folded, or dropped by an overflow). + */ + #lone: WatchChange | undefined; + /** The newest frame's value and context, while a call is pending; the context also marks that one is. */ + #newestValue: T | undefined; + #newestContext: Context | undefined; + /** A pending frame was a retirement. */ + #retired = false; /** - * The source retired: queue the last frame, `[["r", null]]` with value `null`, then end `retired`. Typed for watches - * whose values include `null`. + * Queue the frame `change`: its ops turn the previous value into its value, and `retired` says the document the + * watch follows was retired, which the next call reports however many frames it folds. */ - retire(this: Watch, context: Context): void { - if (!this.open || this.#retiring) return; - this.#retiring = true; - this.#ops.push(RETIREMENT_OPERATIONS); - this.#newest = { value: null, context }; + advance(change: WatchChange, context: Context): void { + if (!this.open) return; + this.#lone = this.#newestContext === undefined ? change : undefined; + this.#ops.push(change.ops); + this.#newestValue = change.value; + this.#newestContext = context; + if (change.retired) this.#retired = true; this.pending(); } - protected next(): WatchDelivery | undefined { - const newest = this.#newest; - if (newest === undefined) return undefined; - this.#newest = undefined; - const batch = this.#ops.take(); - // Values of a JSON watch are JSON; `T` is the source's narrower or wider view of them (e.g. a typed document). - const ops: readonly Op[] = batch.kind === "ops" ? batch.ops : [["r", newest.value as JsonValue]]; - // Nothing is queued after a retirement, so a retiring watch's pending call is its last. - return { value: newest.value, ops, context: newest.context, retires: this.#retiring }; + protected get pendingContext(): Context | undefined { + return this.#newestContext; } - protected invoke(listener: WatchListener, delivery: WatchDelivery, context: Context): Promise { - return listener(delivery.value, delivery.ops, context); + protected next(): WatchChange { + const lone = this.#lone; + // Set with the context, which says a frame is pending. + const value = this.#newestValue as T; + const batch = this.#ops.take(); + const retired = this.#retired; + this.#lone = undefined; + this.#newestValue = undefined; + this.#newestContext = undefined; + this.#retired = false; + // One frame never folds or overflows: its change is the call's. + if (lone !== undefined) return lone; + // Values of a JSON watch are JSON; `T` is the source's narrower or wider view of them (e.g. a typed document). + const ops: readonly Op[] = batch.kind === "ops" ? batch.ops : [["r", value as JsonValue]]; + return { value, ops, retired }; } protected release(): void { this.#ops.take(); - this.#newest = undefined; + this.#lone = undefined; + this.#newestValue = undefined; + this.#newestContext = undefined; + this.#retired = false; } } @@ -303,11 +332,15 @@ const MAX_PENDING_NATIVE_VALUES = 100; * The watch handle of a native source: every value in order, one call each; past 100 pending values they collapse to * one, `replace()` or by default the newest value. `detach` and `report` as for `Watch`. */ -export class NativeWatch - extends WatchLifecycle, Delivery> - implements NativeWatchHandle -{ - readonly #pending: Delivery[] = []; +export class NativeWatch extends WatchLifecycle> implements NativeWatchHandle { + /** + * Pending values in order, each followed by its frame's context, from `#head` on. Values are kept raw: the change a + * call gets is built when it is taken, so a value an overflow drops costs no object. A read index instead of + * `shift()` takes a backlog in constant time per value; taken slots are cleared at once, the array is emptied once + * all of it was taken, and its taken prefix dropped after 100 values. + */ + readonly #pending: Array = []; + #head = 0; #replace: (() => T) | undefined; constructor(value: T, detach: () => void, report: (error: unknown) => void, replace?: () => T) { @@ -318,26 +351,45 @@ export class NativeWatch /** Queue `value`, the source's new value. */ advance(value: T, context: Context): void { if (!this.open) return; - if (this.#pending.length < MAX_PENDING_NATIVE_VALUES) { - this.#pending.push({ value, context, retires: false }); + if (this.#pending.length - this.#head < 2 * MAX_PENDING_NATIVE_VALUES) { + this.#pending.push(value, context); } else { this.#pending.length = 0; - const replacement = this.#replace === undefined ? value : this.#replace(); - this.#pending.push({ value: replacement, context, retires: false }); + this.#head = 0; + this.#pending.push(this.#replace === undefined ? value : this.#replace(), context); } this.pending(); } - protected next(): Delivery | undefined { - return this.#pending.shift(); + protected get pendingContext(): Context | undefined { + // The slot after a value holds its context; past the end it is empty. + return this.#pending[this.#head + 1] as Context | undefined; } - protected invoke(listener: NativeWatchListener, delivery: Delivery, context: Context): Promise { - return listener(delivery.value, context); + protected next(): NativeWatchChange { + const head = this.#head; + const pending = this.#pending; + // The slot at `#head` holds a value. + const value = pending[head] as T; + if (head + 2 === pending.length) { + pending.length = 0; + this.#head = 0; + } else if (head + 2 >= 2 * MAX_PENDING_NATIVE_VALUES) { + // A backlog that never drains: drop the taken prefix, at most once per 100 values, so the array stays bounded. + pending.copyWithin(0, head + 2); + pending.length -= head + 2; + this.#head = 0; + } else { + pending[head] = undefined; + pending[head + 1] = undefined; + this.#head = head + 2; + } + return { value }; } protected release(): void { this.#pending.length = 0; + this.#head = 0; this.#replace = undefined; } } diff --git a/packages/chord/test/watch.test.ts b/packages/chord/test/watch.test.ts index 76b54401b..5ef530d60 100644 --- a/packages/chord/test/watch.test.ts +++ b/packages/chord/test/watch.test.ts @@ -1,7 +1,7 @@ // Tests for Chord's watch handles: the shared lifecycle through `Watch`, and what `NativeWatch` does differently. import { setFlagsFromString } from "node:v8"; import { runInNewContext } from "node:vm"; -import { afterEach, describe, expect, expectTypeOf, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { BACKGROUND_CONTEXT, type Context, @@ -12,7 +12,7 @@ import { import { applyImmutable, type Op, track } from "../src/delta/index.ts"; import { MAX_PENDING_OPS } from "../src/delta/pending.ts"; import type { JsonValue } from "../src/json.ts"; -import { NativeWatch, Watch, type WatchEnd } from "../src/watch.ts"; +import { NativeWatch, type NativeWatchChange, Watch, type WatchChange, type WatchEnd } from "../src/watch.ts"; const context = BACKGROUND_CONTEXT; @@ -25,6 +25,11 @@ function deferred(): { readonly promise: Promise; readonly resolve: () => return { promise, resolve: () => resolve() }; } +/** A source's frame that is not a retirement. */ +function frame(value: T, ops: readonly Op[]): WatchChange { + return { value, ops, retired: false }; +} + /** A watch whose detach and reports are recorded. */ function recordedWatch(value: T) { const recorded = { detached: 0, reported: [] as unknown[] }; @@ -39,9 +44,9 @@ function recordedWatch(value: T) { /** A started watch collecting its calls. */ function collectingWatch(value: T) { const { watch, recorded } = recordedWatch(value); - const frames: Array<{ value: T; ops: readonly Op[] }> = []; - watch.start(async (delivered, ops) => { - frames.push({ value: delivered, ops }); + const frames: Array<{ value: T; ops: readonly Op[]; retired?: true }> = []; + watch.start(async ({ value: delivered, ops, retired }) => { + frames.push(retired ? { value: delivered, ops, retired } : { value: delivered, ops }); }); return { watch, recorded, frames }; } @@ -69,21 +74,20 @@ const collect = (() => { describe("Watch delivery", () => { it("keeps the acquisition value until start, then delivers the frames since in one call", async () => { const { watch } = recordedWatch({ n: 0, text: "" }); - watch.advance({ n: 1, text: "" }, [["s", ["n"], 1]], context); + watch.advance(frame({ n: 1, text: "" }, [["s", ["n"], 1]]), context); watch.advance( - { n: 2, text: "a" }, - [ + frame({ n: 2, text: "a" }, [ ["s", ["n"], 2], ["a", ["text"], "a"], - ], + ]), context, ); - watch.advance({ n: 2, text: "ab" }, [["a", ["text"], "b"]], context); + watch.advance(frame({ n: 2, text: "ab" }, [["a", ["text"], "b"]]), context); await flush(); expect(watch.value).toEqual({ n: 0, text: "" }); const frames: Array<{ value: JsonValue; ops: readonly Op[] }> = []; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { frames.push({ value, ops }); }); expect(frames).toEqual([]); @@ -103,14 +107,14 @@ describe("Watch delivery", () => { it("passes a frame's ops on by identity to a listener that keeps up", async () => { const { watch } = recordedWatch(0); const delivered: Array = []; - watch.start(async (_value, ops) => { + watch.start(async ({ ops }) => { delivered.push(ops); }); const first: readonly Op[] = [["r", 1]]; const second: readonly Op[] = [["r", 2]]; - watch.advance(1, first, context); + watch.advance(frame(1, first), context); await flush(); - watch.advance(2, second, context); + watch.advance(frame(2, second), context); await flush(); expect(delivered).toHaveLength(2); expect(delivered[0]).toBe(first); @@ -119,7 +123,7 @@ describe("Watch delivery", () => { it("never invokes the listener inline", async () => { const { watch, frames } = collectingWatch(0); - watch.advance(1, [["r", 1]], context); + watch.advance(frame(1, [["r", 1]]), context); expect(frames).toEqual([]); await flush(); expect(frames).toEqual([{ value: 1, ops: [["r", 1]] }]); @@ -130,16 +134,16 @@ describe("Watch delivery", () => { const gate = deferred(); const calls: string[] = []; const frames: Array<{ value: JsonValue; ops: readonly Op[] }> = []; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { calls.push(`start ${JSON.stringify(value)}`); frames.push({ value, ops }); if (frames.length === 1) await gate.promise; calls.push(`end ${JSON.stringify(value)}`); }); - watch.advance({ text: "a" }, [["a", ["text"], "a"]], context); + watch.advance(frame({ text: "a" }, [["a", ["text"], "a"]]), context); await flush(); - watch.advance({ text: "ab" }, [["a", ["text"], "b"]], context); - watch.advance({ text: "abc" }, [["a", ["text"], "c"]], context); + watch.advance(frame({ text: "ab" }, [["a", ["text"], "b"]]), context); + watch.advance(frame({ text: "abc" }, [["a", ["text"], "c"]]), context); await flush(); expect(calls).toEqual(['start {"text":"a"}']); gate.resolve(); @@ -153,11 +157,11 @@ describe("Watch delivery", () => { const key = createContextKey("frame"); const controller = new AbortController(); const delivered: Context[] = []; - watch.start(async (_value, _ops, deliveryContext) => { + watch.start(async (_change, deliveryContext) => { delivered.push(deliveryContext); }); - watch.advance(1, [["r", 1]], withContextValue(key, "first", context)); - watch.advance(2, [["r", 2]], withContextValue(key, "second", withAbortSignal(controller.signal, context))); + watch.advance(frame(1, [["r", 1]]), withContextValue(key, "first", context)); + watch.advance(frame(2, [["r", 2]]), withContextValue(key, "second", withAbortSignal(controller.signal, context))); await flush(); expect(delivered).toHaveLength(1); expect(delivered[0]?.value(key)).toBe("second"); @@ -180,12 +184,12 @@ describe("Watch with a slow listener", () => { edit(change.state as Doc); const prepared = change.prepare(); tracker.adopt(prepared); - watch.advance(tracker.value, prepared.ops, context); + watch.advance(frame(tracker.value, prepared.ops), context); }; let replica: Doc = tracker.value; const calls: Array<{ commits: number; ops: number }> = []; let commits = 0; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { replica = applyImmutable(replica, ops); expect(replica).toEqual(value); calls.push({ commits, ops: ops.length }); @@ -217,7 +221,7 @@ function advanceDistinct(watch: Watch, count: number): JsonValue { const values: number[] = []; for (let index = 0; index < count; index++) { values.push(index); - watch.advance({ k: [...values] }, [["s", ["k", index], index]], context); + watch.advance(frame({ k: [...values] }, [["s", ["k", index], index]]), context); } return { k: values }; } @@ -240,11 +244,11 @@ describe("Watch overflow", () => { const { watch } = recordedWatch({ k: [] }); const gate = deferred(); const frames: Array<{ value: JsonValue; ops: readonly Op[] }> = []; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { frames.push({ value, ops }); if (frames.length === 1) await gate.promise; }); - watch.advance({ k: [], first: true }, [["s", ["first"], true]], context); + watch.advance(frame({ k: [], first: true }, [["s", ["first"], true]]), context); await flush(); const newest = advanceDistinct(watch, MAX_PENDING_OPS + 1); gate.resolve(); @@ -259,8 +263,8 @@ describe("Watch overflow", () => { const { watch, frames } = collectingWatch({ k: [] }); advanceDistinct(watch, MAX_PENDING_OPS + 1); await flush(); - watch.advance({ n: 1 }, [["r", { n: 1 }]], context); - watch.advance({ n: 2 }, [["s", ["n"], 2]], context); + watch.advance(frame({ n: 1 }, [["r", { n: 1 }]]), context); + watch.advance(frame({ n: 2 }, [["s", ["n"], 2]]), context); await flush(); expect(frames.at(-1)).toEqual({ value: { n: 2 }, @@ -273,43 +277,62 @@ describe("Watch overflow", () => { }); describe("Watch retirement", () => { - it('delivers `[["r", null]]` as the last call and then ends retired', async () => { + it("reports a retirement in the next call with the value the source gives, and goes on", async () => { const { watch, frames, recorded } = collectingWatch(1); const settled = endOf(watch); - watch.advance(2, [["r", 2]], context); + watch.advance(frame(2, [["r", 2]]), context); await flush(); - watch.retire(context); - watch.advance(3, [["r", 3]], context); + watch.advance({ value: 2, ops: [], retired: true }, context); + await flush(); + watch.advance(frame(3, [["r", 3]]), context); await flush(); expect(frames).toEqual([ { value: 2, ops: [["r", 2]] }, - { value: null, ops: [["r", null]] }, + { value: 2, ops: [], retired: true }, + { value: 3, ops: [["r", 3]] }, ]); - expect(watch.value).toBeNull(); - expect(settled.end).toEqual({ reason: "retired" }); - expect(recorded.detached).toBe(1); + expect(watch.value).toBe(3); + expect(settled.end).toBeUndefined(); + expect(recorded.detached).toBe(0); + await watch.stop(); }); - it("drops pending frames, also an overflow, for the retirement", async () => { - for (const pending of [3, MAX_PENDING_OPS + 1]) { - const { watch } = recordedWatch({ k: [] }); - advanceDistinct(watch, pending); - watch.retire(context); - const frames: Array<{ value: JsonValue; ops: readonly Op[] }> = []; - watch.start(async (value, ops) => { - frames.push({ value, ops }); - }); - expect(await watch.closed).toEqual({ reason: "retired" }); - expect(frames).toEqual([{ value: null, ops: [["r", null]] }]); - } + it("folds a retirement and the frames around it into one call that reports it", async () => { + const { watch } = recordedWatch({ n: 0 }); + watch.advance(frame({ n: 1 }, [["s", ["n"], 1]]), context); + watch.advance({ value: { n: 1 }, ops: [], retired: true }, context); + watch.advance(frame({ m: 1 }, [["r", { m: 1 }]]), context); + const changes: WatchChange[] = []; + watch.start(async (change) => { + changes.push(change); + }); + await flush(); + expect(changes).toEqual([{ value: { m: 1 }, ops: [["r", { m: 1 }]], retired: true }]); + watch.advance(frame({ m: 2 }, [["s", ["m"], 2]]), context); + await flush(); + expect(changes[1]).toEqual({ value: { m: 2 }, ops: [["s", ["m"], 2]], retired: false }); + await watch.stop(); }); - it("ends retired only on its source's retirement, never on a null value", async () => { + it("keeps reporting the retirement past an overflow", async () => { + const { watch } = recordedWatch({ k: [] }); + watch.advance({ value: { k: [] }, ops: [], retired: true }, context); + const newest = advanceDistinct(watch, MAX_PENDING_OPS + 1); + const changes: WatchChange[] = []; + watch.start(async (change) => { + changes.push(change); + }); + await flush(); + expect(changes).toEqual([{ value: newest, ops: [["r", newest]], retired: true }]); + await watch.stop(); + }); + + it("treats a null value as a value, never as a retirement", async () => { const { watch, frames } = collectingWatch(1); const settled = endOf(watch); - watch.advance(null, [["r", null]], context); + watch.advance(frame(null, [["r", null]]), context); await flush(); - watch.advance(2, [["r", 2]], context); + watch.advance(frame(2, [["r", 2]]), context); await flush(); expect(frames).toEqual([ { value: null, ops: [["r", null]] }, @@ -319,24 +342,204 @@ describe("Watch retirement", () => { expect(await watch.stop()).toEqual({ reason: "stopped" }); }); - it("is only available to watches whose values include null", () => { - const check = (numbers: Watch, documents: Watch<{ n: number } | null>) => { - // @ts-expect-error `retire` delivers `null`, which a number watch cannot hold. - numbers.retire(context); - documents.retire(context); - }; - expectTypeOf(check).toBeFunction(); - }); - it("discards a retirement still pending when the source ends", async () => { const { watch } = recordedWatch(1); - watch.retire(context); + watch.advance({ value: 1, ops: [], retired: true }, context); watch.end(); expect(await watch.closed).toEqual({ reason: "ended" }); expect(watch.value).toBe(1); }); }); +describe("Watch end after delivery", () => { + it("takes no more frames, delivers what is pending, then ends `ended` and only then detaches", async () => { + const { watch, frames, recorded } = collectingWatch(1); + const settled = endOf(watch); + watch.advance({ value: 1, ops: [], retired: true }, context); + watch.endAfterDelivery(); + expect(recorded.detached).toBe(0); + // The source sends nothing more: a later frame is not taken. + watch.advance(frame(2, [["r", 2]]), context); + await flush(); + expect(frames).toEqual([{ value: 1, ops: [], retired: true }]); + expect(settled.end).toEqual({ reason: "ended" }); + expect(recorded.detached).toBe(1); + }); + + it("waits for start to deliver, and ends at once with nothing pending", async () => { + const { watch, recorded } = recordedWatch(1); + watch.advance(frame(2, [["r", 2]]), context); + watch.endAfterDelivery(); + await flush(); + const settled = endOf(watch); + await flush(); + expect(settled.end).toBeUndefined(); + expect(recorded.detached).toBe(0); + const values: JsonValue[] = []; + watch.start(async ({ value }) => { + values.push(value); + }); + expect(await watch.closed).toEqual({ reason: "ended" }); + expect(values).toEqual([2]); + expect(recorded.detached).toBe(1); + + const idle = collectingWatch(1); + idle.watch.endAfterDelivery(); + expect(await idle.watch.closed).toEqual({ reason: "ended" }); + expect(idle.frames).toEqual([]); + expect(idle.recorded.detached).toBe(1); + }); + + it("ends a never-started watch at its source's end, dropping what is pending", async () => { + const { watch, recorded } = recordedWatch(1); + watch.advance({ value: 1, ops: [], retired: true }, context); + watch.endAfterDelivery(); + const failure = new Error("source failed"); + watch.end({ error: failure }); + expect(await watch.closed).toEqual({ reason: "failed", error: failure }); + expect(recorded.detached).toBe(1); + expect(() => watch.start(async () => {})).toThrow("Watch is stopped"); + }); + + /** A started watch whose listener is in a call held by `gate`, with nothing pending, then `endAfterDelivery()`. */ + async function endingWhileRunning(settle: () => Promise) { + const { watch, recorded } = recordedWatch(0); + const gate = deferred(); + const values: JsonValue[] = []; + let returned = false; + watch.start(async ({ value }) => { + values.push(value); + await gate.promise; + await settle(); + returned = true; + }); + watch.advance({ value: 1, ops: [["r", 1]], retired: true }, context); + await flush(); + expect(values).toEqual([1]); + watch.endAfterDelivery(); + const settled = endOf(watch); + await flush(); + return { watch, recorded, values, settled, release: gate.resolve, returned: () => returned }; + } + + it("ends `ended` only after the running call returns when nothing is pending, detached then", async () => { + const ending = await endingWhileRunning(async () => {}); + expect(ending.settled.end).toBeUndefined(); + expect(ending.recorded.detached).toBe(0); + ending.release(); + expect(await ending.watch.closed).toEqual({ reason: "ended" }); + expect(ending.returned()).toBe(true); + expect(ending.values).toEqual([1]); + expect(ending.recorded.detached).toBe(1); + }); + + it("ends `listener_error` and reports it when the running call throws after `endAfterDelivery()`", async () => { + const failure = new Error("listener failed"); + const ending = await endingWhileRunning(async () => { + throw failure; + }); + expect(ending.settled.end).toBeUndefined(); + ending.release(); + expect(await ending.watch.closed).toEqual({ reason: "listener_error", error: failure }); + expect(ending.recorded.reported).toEqual([failure]); + expect(ending.recorded.detached).toBe(1); + }); + + it("ends `stopped` when stopped while ending, without awaiting the running call", async () => { + const ending = await endingWhileRunning(async () => {}); + expect(await ending.watch.stop()).toEqual({ reason: "stopped" }); + expect(ending.recorded.detached).toBe(1); + expect(ending.returned()).toBe(false); + ending.release(); + await flush(); + expect(await ending.watch.closed).toEqual({ reason: "stopped" }); + }); +}); + +describe("Watch change objects", () => { + /** A frame context carrying `payload` and an abort signal, as a producer's is. */ + function producerContext(payload: object) { + const controller = new AbortController(); + return withAbortSignal( + controller.signal, + withContextValue(createContextKey("payload"), payload, context), + ); + } + + it("hands a listener plain data only: `{ value, ops, retired }`, cloneable and serializable", async () => { + const { watch } = recordedWatch({ n: 0 }); + const changes: WatchChange[] = []; + watch.start(async (change) => { + changes.push(change); + }); + watch.advance(frame({ n: 1 }, [["s", ["n"], 1]]), producerContext({})); + await flush(); + // Folded: the watch builds the change. + watch.advance(frame({ n: 2 }, [["s", ["n"], 2]]), producerContext({})); + watch.advance({ value: { n: 2 }, ops: [], retired: true }, producerContext({})); + await flush(); + expect(changes).toHaveLength(2); + for (const change of changes) { + expect(Object.keys(change)).toEqual(["value", "ops", "retired"]); + expect(structuredClone(change)).toEqual(change); + expect(JSON.parse(JSON.stringify(change))).toEqual(change); + } + expect(changes[1]).toEqual({ value: { n: 2 }, ops: [["s", ["n"], 2]], retired: true }); + await watch.stop(); + }); + + it("passes a frame delivered alone on as the source made it", async () => { + const { watch } = recordedWatch(0); + const changes: WatchChange[] = []; + watch.start(async (change) => { + changes.push(change); + }); + const alone = frame(1, [["r", 1]]); + watch.advance(alone, context); + await flush(); + expect(changes[0]).toBe(alone); + await watch.stop(); + }); + + it("keeps no frame context alive through a change a listener keeps", async () => { + const kept: Array | NativeWatchChange> = []; + /** Deliver one frame whose context alone references `payload`, keep the change, and stop the watch. */ + const deliver = async (native: boolean): Promise> => { + const payload = { big: "x".repeat(1000) }; + if (native) { + const watch = new NativeWatch( + 0, + () => {}, + () => {}, + ); + watch.start(async (change) => { + kept.push(change); + }); + watch.advance(1, producerContext(payload)); + await flush(); + expect(Object.keys(kept.at(-1)!)).toEqual(["value"]); + expect(structuredClone(kept.at(-1))).toEqual({ value: 1 }); + await watch.stop(); + } else { + const { watch } = recordedWatch(0); + watch.start(async (change) => { + kept.push(change); + }); + watch.advance(frame(1, [["r", 1]]), producerContext(payload)); + await flush(); + await watch.stop(); + } + return new WeakRef(payload); + }; + for (const native of [false, true]) { + const payload = await deliver(native); + await collect(); + expect(kept).toHaveLength(native ? 2 : 1); + expect(payload.deref()).toBeUndefined(); + } + }); +}); + describe("Watch ends", () => { it("stops idempotently and rejects a repeated or late start", async () => { const { watch, recorded } = recordedWatch(0); @@ -353,7 +556,7 @@ describe("Watch ends", () => { it("ends `ended` or `failed` from its source, dropping pending frames", async () => { const { watch, frames } = collectingWatch(0); - watch.advance(1, [["r", 1]], context); + watch.advance(frame(1, [["r", 1]]), context); watch.end(); await flush(); expect(frames).toEqual([]); @@ -370,14 +573,14 @@ describe("Watch ends", () => { const { watch } = recordedWatch(0); const gate = deferred(); const calls: string[] = []; - watch.start(async (value) => { + watch.start(async ({ value }) => { calls.push(`start ${value}`); await gate.promise; calls.push(`end ${value}`); }); - watch.advance(1, [["r", 1]], context); + watch.advance(frame(1, [["r", 1]]), context); await flush(); - watch.advance(2, [["r", 2]], context); + watch.advance(frame(2, [["r", 2]]), context); expect(await watch.stop()).toEqual({ reason: "stopped" }); gate.resolve(); await flush(); @@ -407,14 +610,14 @@ describe("Watch ends", () => { const failure = new Error("listener failed"); const gate = deferred(); const values: JsonValue[] = []; - watch.start(async (value) => { + watch.start(async ({ value }) => { values.push(value); await gate.promise; throw failure; }); - watch.advance(1, [["r", 1]], context); + watch.advance(frame(1, [["r", 1]]), context); await flush(); - watch.advance(2, [["r", 2]], context); + watch.advance(frame(2, [["r", 2]]), context); gate.resolve(); expect(await watch.closed).toEqual({ reason: "listener_error", error: failure }); expect(values).toEqual([1]); @@ -425,7 +628,7 @@ describe("Watch ends", () => { it("wraps a thrown non-Error in an Error", async () => { const { watch } = recordedWatch(0); watch.start(() => Promise.reject("plain")); - watch.advance(1, [["r", 1]], context); + watch.advance(frame(1, [["r", 1]]), context); const end = await watch.closed; expect(end.reason).toBe("listener_error"); expect(end.reason === "listener_error" && end.error.message).toBe("plain"); @@ -446,7 +649,7 @@ describe("Watch references", () => { const listener = async (): Promise => {}; watch.observeCancellation(controller.signal); watch.start(listener); - watch.advance(1, [["r", 1]], context); + watch.advance(frame(1, [["r", 1]]), context); await flush(); end(watch); return { @@ -469,7 +672,7 @@ describe("Watch references", () => { (watch: Watch) => void watch.stop(), (watch: Watch) => watch.cancel(), (watch: Watch) => watch.end(), - (watch: Watch) => watch.retire(context), + (watch: Watch) => watch.endAfterDelivery(), ]) { const ended = await acquire(end); await flush(); @@ -480,6 +683,59 @@ describe("Watch references", () => { } }); + it("keeps no ops a fold or an overflow dropped, while it waits", async () => { + /** Feed an unstarted watch four tracker batches of 4,000 sets each: the third overflows, the fourth is ignored. */ + const overflow = () => { + const initial: Record = {}; + for (let i = 0; i < 16_000; i++) initial[`k${i}`] = 0; + const tracker = track>(initial); + const watch = new Watch( + initial, + () => {}, + () => {}, + ); + const batches: Array> = []; + for (let round = 0; round < 4; round++) { + const change = tracker.beginChange(); + for (let i = round * 4_000; i < (round + 1) * 4_000; i++) change.state[`k${i}`] = 1; + const prepared = change.prepare(); + tracker.adopt(prepared); + batches.push(new WeakRef(prepared.ops)); + watch.advance(frame(tracker.value, prepared.ops), context); + } + return { watch, batches, newest: tracker.value }; + }; + const { watch, batches, newest } = overflow(); + await collect(); + expect(batches.map((batch) => batch.deref() === undefined)).toEqual([true, true, true, true]); + const changes: WatchChange[] = []; + watch.start(async (change) => { + changes.push(change); + }); + await flush(); + expect(changes).toEqual([{ value: newest, ops: [["r", newest]], retired: false }]); + await watch.stop(); + }); + + it("delivers a native backlog that never drains in order", async () => { + const watch = new NativeWatch( + 0, + () => {}, + () => {}, + ); + const values: number[] = []; + let next = 50; + // Each call queues one more value behind a backlog of 50, so the queue never empties for 1,000 values. + watch.start(async ({ value }) => { + values.push(value); + if (next < 1_000) watch.advance(next++, context); + }); + for (let n = 0; n < 50; n++) watch.advance(n, context); + await flush(); + expect(values).toEqual(Array.from({ length: 1_000 }, (_, n) => n)); + await watch.stop(); + }); + it("releases its pending queue and leaf index once delivered or ended", async () => { /** Two pending frames, folded, whose ops alone reference `payload`; `settle` delivers or ends them. */ const pending = async (settle: (watch: Watch, gate: () => void) => Promise) => { @@ -494,10 +750,10 @@ describe("Watch references", () => { watch.start(async () => { if (++calls === 1) await gate.promise; }); - watch.advance(1, [["s", ["n"], 1]], context); + watch.advance(frame(1, [["s", ["n"], 1]]), context); await flush(); - watch.advance(2, [["s", ["o"], payload]], context); - watch.advance(3, [["s", ["n"], 3]], context); + watch.advance(frame(2, [["s", ["o"], payload]]), context); + watch.advance(frame(3, [["s", ["n"], 3]]), context); await settle(watch, gate.resolve); return { watch, payload: new WeakRef(payload) }; }; @@ -529,7 +785,7 @@ describe("NativeWatch", () => { ); const key = createContextKey("frame"); const delivered: Array<{ value: { run: () => number }; frame: string | undefined }> = []; - watch.start(async (value, deliveryContext) => { + watch.start(async ({ value }, deliveryContext) => { delivered.push({ value, frame: deliveryContext.value(key) }); }); watch.advance(first, withContextValue(key, "first", context)); @@ -551,7 +807,7 @@ describe("NativeWatch", () => { ); for (let n = 1; n <= 101; n++) watch.advance(n, context); const values: number[] = []; - watch.start(async (value) => { + watch.start(async ({ value }) => { values.push(value); }); await flush(); @@ -567,7 +823,7 @@ describe("NativeWatch", () => { ); for (let n = 1; n <= 101; n++) watch.advance(`value ${n}`, context); const values: string[] = []; - watch.start(async (value) => { + watch.start(async ({ value }) => { values.push(value); }); await flush(); @@ -592,6 +848,18 @@ describe("NativeWatch", () => { ended.end(); expect(await ended.closed).toEqual({ reason: "ended" }); + const finishing = create(); + const finished: number[] = []; + finishing.advance(1, context); + finishing.advance(2, context); + finishing.endAfterDelivery(); + finishing.advance(3, context); + finishing.start(async ({ value }) => { + finished.push(value); + }); + expect(await finishing.closed).toEqual({ reason: "ended" }); + expect(finished).toEqual([1, 2]); + const error = new Error("source failed"); const failed = create(); failed.end({ error }); diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index a6431c209..ed67bba7b 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -4,21 +4,30 @@ ### Breaking Changes -- `Storage.commit(writes, adopt, context)` takes an `adopt(seq)` callback, the Session's in-memory adoption of the batch: a storage calls it exactly once, synchronously, once the batch is durable and before its next commit starts, then resolves with `seq`, and never for a commit that fails before the batch is durable. Memory, SQLite, JSONL, and IndexedDB storages do; custom storages and wrappers must call or forward it, and the storage conformance suite checks it. A commit that rejects after its adoption fails the Session without rolling the adopted changes back. +- `Storage.commit(writes, adopt, context)` takes an `adopt(seq)` callback, the Session's in-memory adoption and publication of the batch: a storage calls it exactly once, synchronously, once the batch is durable and before its next commit starts, then resolves with `seq`, and never for a commit that fails before the batch is durable. Memory, SQLite, JSONL, and IndexedDB storages do; custom storages and wrappers must call or forward it, and the storage conformance suite checks it. A commit that rejects after its adoption is published and fails the Session without rolling the adopted changes back. - `Storage.mintId()`'s contract is narrowed to IDs greater than every ID minted since the storage was opened and every ID stored, also across reopen. The bundled storages already behave so. -- Watches use Chord's `WatchHandle` and `WatchEnd` (Durable re-exports both). End reasons are `stopped`, `cancelled`, `retired`, `ended` (was `session_closed`), `listener_error`, and `failed` (was `session_failed`). Retirement is an explicit event of the watched document incarnation (last frame `[["r", null]]`, then `retired`), never inferred from a `null` value. An ended watch drops its listener and every reference into its Session, so a stopped watch that is still referenced keeps only its last value. +- Watches use Chord's `WatchHandle` and `WatchEnd` (Durable re-exports both, with `WatchChange` and `NativeWatchChange`). End reasons are `stopped`, `cancelled`, `ended` (was `session_closed`), `listener_error`, and `failed` (was `session_failed`); `retired` is gone (see below). An ended watch drops its listener and every reference into its Session, so a stopped watch that is still referenced keeps only its last value. +- Watch listeners take one change object before the context: `watch.start(async ({ value, ops, retired }, context) => …)` for document, conversation view (`Conversation.watch()`), and task graph watches, and `stream.start(async ({ value: events }, context) => …)` for `watchEvents()`. Migration: `start(async (value, ops, ctx) => …)` becomes `start(async ({ value, ops }, ctx) => …)`, and an event listener `(events, ctx)` becomes `({ value: events }, ctx)`. +- Document watches follow their address (kind, scope, and key) instead of one incarnation: `DocumentWatch` is `WatchHandle>` (no `null`). A retirement no longer ends the watch with a last frame `[["r", null]]` and `retired`: the next call has `retired: true`, the last value the watch was given, and no ops (folded with what is pending; a change made in the retiring commit itself is not shown), and a document created at the address later arrives on the same watch as `[["r", value]]`, possibly in the same call. A task document's watch ends `ended` once the retirement call of the commit that makes its task terminal has returned; Session close or failure still ends it first if that call was not delivered yet (a watch never started). Acquiring an absent document still gives `undefined`. A watch pins the incarnation at its address, nothing while it is absent. Migration: test `change.retired` instead of a `null` value or a `retired` end, and stop a watch whose document may be retired for good, as it otherwise stays until the Session ends. - Document, conversation view, and task graph watches deliver the commits made while their listener runs (or before `start()`) in one callback: the newest value, the commits' operations folded (appends to one string join, sets of one field keep the last) and the newest commit's Context. Past 10,000 pending operations the callback gets one root replacement `[["r", newest]]`. Before, each commit was its own callback and more than 100 pending commits became a root replacement. `watchEvents()` still delivers every event batch in order, with one `snapshot` past 100 pending batches. - Removed the Chord state getters `Session.documentState()`, `Conversation.viewState()`, and `Harness.taskGraph()`, and the `DocumentState` type; use `watchDoc()`, `Conversation.watch()`, and `Harness.watchTaskGraph()`. Durable no longer uses Chord's `replicatedState()`. -- Partial answers and running tool output are stored later than they are shown: their throttled commits are deferred (`CommitPersistence`, spec §4.1), so watches, views, and agent events see each at once, while Storage gets them together with the next commit that reads `pi.live` (the end of the answer, a tool settling, a run ending) and at the latest after `settings.progress.persistWithinMs` (default 1,000 ms). With 32 streams over SQLite this cuts storage transactions about 5.7x and disk writes about 2.9x for turns with parallel tools, 27x and 9x for long answers, and CPU by about a quarter. A crash now loses up to about a second of streamed content instead of 100 ms; a stream that crashes before its first store leaves no aborted entry, and that attempt's usage is not recorded. `subscribeCommits()` gets these commits only once they are stored, with exactly what was stored (held operations first), and gets a publication for a deadline's store too; it no longer sees every partial. Migration: a `subscribeCommits()` listener that showed live progress should use `watchDoc()`, `Conversation.watch()`, or `watchEvents()`; one that mirrors Storage keeps working; set `settings.progress.persistWithinMs: 0` to store progress right after each commit, about as before (one storage transaction per partial). +- Partial answers and running tool output are stored later than they are shown: their throttled commits are deferred (`CommitPersistence`, spec §4.1), so watches, views, and agent events see each at once, while Storage gets them together with the next commit that reads `pi.live` (the end of the answer, a tool settling, a run ending) and at the latest after `settings.progress.persistWithinMs` (default 1,000 ms). With 32 streams over SQLite this cuts storage transactions about 5.7x and disk writes about 2.9x for turns with parallel tools, 27x and 9x for long answers, and CPU by about a quarter. A crash now loses up to about a second of streamed content instead of 100 ms; a stream that crashes before its first store leaves no aborted entry, and that attempt's usage is not recorded. `subscribeCommits()` gets these commits only once they are stored, with exactly what was stored (held operations first; a document the commit retires shows as its retirement only, without the operations stored before it), and gets a publication for a deadline's store too; it no longer sees every partial. Migration: a `subscribeCommits()` listener that showed live progress should use `watchDoc()`, `Conversation.watch()`, or `watchEvents()`; one that mirrors Storage keeps working; set `settings.progress.persistWithinMs: 0` to store progress right after each commit, about as before (one storage transaction per partial). +- The change type of `FileSystem.watch()` in `@earendil-works/pi-durable/env` is renamed from `WatchChange` to `FileChange` (same shape: `{ paths }`, `{ overflow: true }`, or `{ error }`), so `WatchChange` names only a watch listener's change. Migration: import `type FileChange` from `@earendil-works/pi-durable/env` where `WatchChange` was imported from it. +- `HostCallback` gains `"telemetry"`: a `host_error` report about a telemetry adapter that threw while a commit was published. Exhaustive switches over `HostCallback` need a case for it. ### Added - Deferred commits: `Session.commit()`, `Conversation.commit()`, `TaskRuntime.commit()`, and a tool's `api.commit()` take `{ persistence: { kind: "deferred", withinMs? } }`. A commit that changes only existing conversation documents with `history: "latest"` and `fork: "initial"` is then adopted in memory and shown to watches at once, and stored with the next commit that reads or writes the document, before the document leaves memory, at close, or within `withinMs` (default `settings.progress.persistWithinMs`, or `createSession(storage, { persistWithinMs })`); any other write makes it immediate. A crash loses what is held. See spec §4.1. +- Synchronous observers for adapters, such as a Chord source over a Durable view: `Session.observeDoc(token, ...scope, observer, context)`, `Harness.observeConversation(id, observer, context)`, and `Harness.observeTaskGraph(observer, context)` attach an `Observer` (`advance(change, context)`, `end(failure)`) atomically on the Session line, like a watch, and return an `Observation` (`value`, `detach()`; `observeDoc` gives `undefined` for an absent document). `advance` gets every change unfolded as memory adopts it, for a stored commit inside Storage's `adopt` and for a deferred one before it is stored, so it must only record it: no awaits, no Session calls, no throws. A document observer gets one call per change at its address (two for a commit that retires and recreates it), the task graph also a call outside any commit when a registry change or a failed migration changes a `blocked` reason, and a change can arrive before the attaching promise resolves. The attaching context governs only the attachment; a closed Session or an invalid argument rejects the promise. One that throws is reported as `listener_error`, detached, and ended with the error. `end` comes once, at Session close or failure, or after a task document's retirement with its task. Document, view, and task graph watches are built on them (spec §9.1). - `HarnessOptions.readOnly` opens a Harness as a viewer of a session another process drives: open leaves running tasks as committed instead of recovering them, every commit with writes rejects with `ReadOnlySession` before it reaches Storage, and `resume()` and every call that would run tasks reject with it. ### Changed - Document, conversation view, task graph, and event watches use Chord's watch handles (`Watch`, and `NativeWatch` for event batches). +- A stored commit is published when Storage adopts it, inside `Storage.commit()`: the commit's telemetry events and Durable's own observers (document watches, conversation views, the task graph, the scheduler, and waits) run inside `adopt`, so two views never show one commit apart. `subscribeCommits()` listeners run one microtask later, outside the storage's synchronous step, before every wait the commit resolves and every abort signal it fires; they have no order relative to watch listeners and `onWake`. With a storage that works after adopting (JSONL's sidecar reclaim), all of these run before `Storage.commit()` resolves. +- `subscribeCommits()` listeners are read when a commit's dispatch runs: a listener added between adoption and the dispatch gets the commit only when one was registered at adoption; one added after the dispatch ran misses it. +- The abort signal of a newly marked running task fires after `subscribeCommits()` listeners instead of before them. +- A throwing Durable commit observer fails the Session after `Storage.commit()` returned, after the other observers and the host listeners got the commit; the commit still stands. When Storage also rejects that commit after adopting it, the storage error fails the Session and the observer's error is reported as `listener_error`. ### Fixed @@ -29,6 +38,8 @@ - Fixed partial commits of a streamed answer copying every growing text, thinking and tool-call argument string in full on each commit (about 20 s of CPU and a 100 MB heap peak for a 500 KB tool call): they now append new text with Chord's `appendText()`, and apply the argument changes of their own parse of the streamed argument text ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed stored assistant messages lacking `thinkingLevel`: a generation records the Pi thinking level its request asked for on the response, also after polling a deferred one, as pi-agent-core's agent loop does - Fixed partial commits polluting `Object.prototype` when a streamed tool call's arguments gained a `__proto__` member after an earlier commit: the merge read the missing member through to `Object.prototype` and wrote into it +- Fixed a commit that Storage rejects after adopting it (a failed JSONL reclaim) publishing nothing: it is published and its waits resolve before the Session fails, so a replica built from `subscribeCommits()` stays equal to Storage and `pi.durable.tasks.active` stays balanced. +- Fixed a throwing telemetry adapter (a commit's `pi.durable.entry` or `pi.durable.submission` event, `pi.durable.tasks.active`) rejecting a durable commit and leaving the views and the scheduler without it: it is reported as `host_error` with callback `"telemetry"`, and the commit resolves. A throwing report metric or report event is dropped. ## [1.4.0] - 2026-10-21 diff --git a/packages/durable/README.md b/packages/durable/README.md index 0ce9885bb..d10263776 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -478,7 +478,7 @@ Everything a UI needs is committed state. `watch()` returns a watch of the conve ```typescript const watch = await root.watch(context); render(watch.value); // the state at attachment -watch.start(async (value, ops) => { +watch.start(async ({ value, ops }) => { // value.entries: the active transcript // value.docs["pi.live"]: the running generation (streamed partial, retry, deferred) and tool calls (output, details) // value.docs["pi.inbox"], value.docs["pi.usage"], value.docs["pi.agent"], value.docs["pi.provider"] @@ -488,7 +488,9 @@ watch.start(async (value, ops) => { // later: await watch.stop(); ``` -Watches are Chord's `WatchHandle` (exported by `@earendil-works/chord`; Durable re-exports it). A watch ends once, and `closed` (or `stop()`) gives the reason: `stopped`, `cancelled` (its acquisition context was cancelled), `retired` (a document watch whose incarnation retired, after it delivered `[["r", null]]`), `ended` (the Harness closed), `listener_error`, or `failed` (the Harness failed). An ended watch keeps its last value and no longer references the Harness. +Watches are Chord's `WatchHandle` (exported by `@earendil-works/chord`; Durable re-exports it). The listener gets one change per call, `{ value, ops, retired }`, and the newest commit's context. A watch ends once, and `closed` (or `stop()`) gives the reason: `stopped`, `cancelled` (its acquisition context was cancelled), `ended` (the Harness closed, or a task document's task ended), `listener_error`, or `failed` (the Harness failed). An ended watch keeps its last value and no longer references the Harness. + +Adapters that must change in the same step as Durable's memory, such as a Chord source over a view, attach a synchronous observer instead of a watch: `harness.observeConversation(id, observer, context)`, `harness.observeTaskGraph(observer, context)`, and `harness.observeDoc(token, ...scope, observer, context)` return the current `value` and `detach()`, and call `observer.advance(change, context)` with every later change, unfolded, as memory adopts it: inside the storage's commit for a stored commit, at once for a deferred one (before it is stored). A document observer gets one call per change at its address, so a commit that retires and recreates the document gives two; the task graph also advances without a commit when a registry change alters a task's `blocked` reason. A change can arrive before the `observeXXX()` promise resolves; its ops apply from the observation's `value`. `observer.end(failure)` comes once, when the Harness closes or fails, or, for a task document, right after the retirement of the commit that ends its task. `advance` only records the change: no awaits, no Harness calls, no throws, and quick, since every commit waits for it. One that throws is reported, detached, and ended with the error. Watches are built on these observers. Commits that land while a callback runs are delivered together in the next callback: the newest view and the operations of all of them, folded (appends to one string join, sets of one field keep the last). Past 10,000 pending operations the callback gets one root replacement `[["r", view]]` holding the whole newest view instead. A client that joins late or reconnects starts from the current view; nothing is replayed. @@ -591,7 +593,7 @@ import { watchEvents } from "@earendil-works/pi-durable"; const stream = await watchEvents(harness, root.id, context); initialize(stream.snapshot); // entries, run, in-flight generation, tools, compactions, inbox, agent, usage -stream.start(async (events) => { +stream.start(async ({ value: events }) => { for (const event of events) console.log(JSON.stringify(event)); }); ``` @@ -708,7 +710,7 @@ decide: async (task, runtime, context) => { ```typescript const graph = await harness.watchTaskGraph(context); -graph.start(async (value) => { +graph.start(async ({ value }) => { for (const node of Object.values(value.tasks)) { const status = node.state.status === "waiting" ? `waiting on ${node.state.on.join(", ")}` : node.state.status; console.log(`${node.id} ${node.kind} ${status}`, node.owner ?? `conversation ${node.conversationId}`); @@ -753,9 +755,9 @@ await root.commit( ); ``` -Watches and the view see it at once. Storage gets it with the next commit that reads or writes the document, before the document leaves memory, at close, and at the latest about `withinMs` later; `subscribeCommits()` sees it then, with exactly what was stored. A crash loses what was not stored yet, so derive nothing durable from such a value outside a commit that reads it. Only a commit that changes nothing but existing documents with `history: "latest"` and `fork: "initial"` is deferred; anything else in it, such as an entry, a new document, or a task state, stores it at once. +Watches and the view see it at once. Storage gets it with the next commit that reads or writes the document, before the document leaves memory, at close, and at the latest about `withinMs` later; `subscribeCommits()` sees it then, with exactly what was stored (a document the commit retires shows as its retirement only). A crash loses what was not stored yet, so derive nothing durable from such a value outside a commit that reads it. Only a commit that changes nothing but existing documents with `history: "latest"` and `fork: "initial"` is deferred; anything else in it, such as an entry, a new document, or a task state, stores it at once. -`harness.watchDoc()` watches one document like the view above. `HarnessOptions.conversationCreated(tx, conversation)` runs in every commit that creates or forks a conversation, including a tool's raw `tx.createConversation()`, so every conversation gets your documents; `init` in `createConversation()`, `fork()`, and `root()` writes per-call data in the same commit. An extension's tools, sections, and hooks read their own documents through `api` or `input.read`, and treat an absent one as its default ([`10-extension-state.ts`](test/examples/10-extension-state.ts)). +`harness.watchDoc()` watches one document like the view above. A document watch follows its address: a retirement is a call with `retired: true`, the last value the watch was given (a change made in the retiring commit itself is not shown), and no ops, and a document created there later arrives as `[["r", value]]` on the same watch. A task document's watch ends `ended` with its task: once the retirement call of the commit that ends the task has returned, or at the Harness's close if that call was never delivered (a watch never started); an observer of it gets `end(undefined)` right after the retirement. `HarnessOptions.conversationCreated(tx, conversation)` runs in every commit that creates or forks a conversation, including a tool's raw `tx.createConversation()`, so every conversation gets your documents; `init` in `createConversation()`, `fork()`, and `root()` writes per-call data in the same commit. An extension's tools, sections, and hooks read their own documents through `api` or `input.read`, and treat an absent one as its default ([`10-extension-state.ts`](test/examples/10-extension-state.ts)). ## Usage and Cost @@ -807,7 +809,7 @@ await database.transaction(async (transaction) => { }); ``` -A custom backend's `commit(writes, adopt, context)` calls `adopt(seq)` once the batch is durable, synchronously and before its next commit starts, and then resolves with `seq`; a commit that fails before the batch is durable never calls it. `adopt` is the Session's in-memory adoption of the batch, so memory never shows a partial log. Custom backends can run the shared conformance suite, which checks this too, with any Vitest- or Jest-compatible runner: +A custom backend's `commit(writes, adopt, context)` calls `adopt(seq)` once the batch is durable, synchronously and before its next commit starts, and then resolves with `seq`; a commit that fails before the batch is durable never calls it. `adopt` is the Session's in-memory adoption and publication of the batch, so memory and every view never show a partial log: inside it the Session's own observers run (views, task graph, scheduler, waits), and `subscribeCommits()` listeners, watch listeners and the code the commit wakes follow in microtasks, so at the backend's next `await` if it still works after `adopt` (as JSONL reclaims sidecars), before its `commit` resolves. `adopt` calls no Storage method and never throws; once it ran, the commit is published even if the backend then rejects, and the Session then fails. Custom backends can run the shared conformance suite, which checks this too, with any Vitest- or Jest-compatible runner: ```typescript import { registerStorageConformance } from "@earendil-works/pi-durable/testing"; diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index 3af8ff115..f82c2ac37 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -64,10 +64,11 @@ The core rule is: Required invariants: 1. One Session commit is atomic across all record and document writes. -2. A document update is published only after its storage commit succeeds, - except a deferred commit's (section 4.1): it is published to watches and the - conversation view when memory adopts it, and to `subscribeCommits()` once it - is stored. +2. A stored commit is published once Storage made it durable, when Storage + adopts it (section 4), also when `Storage.commit()` then rejects; a commit + that fails before it is durable publishes nothing. A deferred commit + (section 4.1) is published to watches and the conversation view when memory + adopts it, and to `subscribeCommits()` once it is stored. 3. All visible progress is durable, except what a deferred commit holds: what watches see can be ahead of Storage by up to that commit's window (`withinMs`), and a crash loses it. A commit is deferred only when its caller @@ -81,9 +82,13 @@ Required invariants: Values assigned into a draft are copied by value and must be strict JSON. 7. The mutation line remains held through storage settlement and committed-state adoption. Commit/close observers run synchronously only to capture immutable - state; document-state and watch user callbacks run later, off the line. -8. A storage error is fatal to the open Session (section 10.1). It publishes - nothing and must be reopened. Preparation and checkpoint failures occur before + state; document-state and watch user callbacks run later, as microtasks the + line never waits for, and never inside Storage's synchronous `adopt`. With a + storage that works after `adopt` (JSONL), they may run while the line is + still held by the commit that published. +8. A storage error is fatal to the open Session (section 10.1), which must be + reopened. A commit it fails publishes nothing, unless Storage had already + made it durable and adopted it. Preparation and checkpoint failures occur before storage admission and roll back normally. ## 2. Core records @@ -397,7 +402,8 @@ type HarnessOptions = { ) => (() => void) | undefined; /** * Replaces the timer for sleeping tasks (section 5.6): called with `Harness.wake()` after open and whenever it - * changes, off the Session line; the host arms its own wakeup and calls `resume()` when it fires. + * changes, in a microtask after the commit that changed it was adopted; the host arms its own wakeup and calls + * `resume()` when it fires. */ readonly onWake?: (wake: Wake) => void; /** @@ -607,7 +613,7 @@ type Wake = { readonly active: boolean; readonly at?: number }; /** What a report is about (section 10.1). */ type ReportTask = { readonly id: TaskId; readonly kind: string; readonly conversationId: ConversationId }; -type HostCallback = "now" | "settings" | "registry" | "env" | "models" | "setTimer" | "onWake"; +type HostCallback = "now" | "settings" | "registry" | "env" | "models" | "setTimer" | "onWake" | "telemetry"; type ReportInfo = | { readonly type: "session_failed" } | { @@ -712,6 +718,9 @@ interface Harness extends Session { usage(context: Context): Promise; /** Live tasks as a structural view (section 9.5), as a watch. */ watchTaskGraph(context: Context): Promise; + /** Synchronous observers of a conversation view and of the task graph, for adapters (section 9.1). */ + observeConversation(id: ConversationId, observer: Observer, context: Context): Promise>; + observeTaskGraph(observer: Observer, context: Context): Promise>; } declare const Harness: { @@ -1320,6 +1329,14 @@ interface Session extends DocumentObserver { snapshotAsOf(token: RewindableConversationDocToken, conversationId: ConversationId, at: EntryId, context: Context): Promise | undefined>; snapshotAsOf(token: RewindableConversationDocFamilyToken, conversationId: ConversationId, key: string, at: EntryId, context: Context): Promise | undefined>; + + /** A synchronous observer of one document address, for adapters (section 9.1); the scope arguments as for `snapshot()`. */ + observeDoc(token: SessionDocToken, observer: Observer>, context: Context): Promise> | undefined>; + observeDoc(token: ConversationDocToken, conversationId: ConversationId, observer: Observer>, context: Context): Promise> | undefined>; + observeDoc(token: TaskDocToken, taskId: TaskId, observer: Observer>, context: Context): Promise> | undefined>; + observeDoc(token: SessionDocFamilyToken, key: string, observer: Observer>, context: Context): Promise> | undefined>; + observeDoc(token: ConversationDocFamilyToken, conversationId: ConversationId, key: string, observer: Observer>, context: Context): Promise> | undefined>; + observeDoc(token: TaskDocFamilyToken, taskId: TaskId, key: string, observer: Observer>, context: Context): Promise> | undefined>; } interface Tx { @@ -1505,11 +1522,17 @@ commit deferred and holdable (section 4.1) release the line; invoke listeners later otherwise, right before Storage admission take the held ops of every document the commit acquired and put them in front of its own -storage succeeds - adopt every prepared change by pointer swap and enqueue immutable revision/ops publication - commit listeners, including conversation view mounts (section 9.3), capture it synchronously - release the line; invoke listeners later -storage fails +storage makes it durable and calls adopt(seq), inside Storage.commit() (section 4) + adopt every prepared change by pointer swap; record its telemetry events + queue one microtask that calls subscribeCommits() listeners with what was stored + the Session's own observers, including watches and view mounts (section 9.3), capture it + synchronously; the waits they resolve and the aborts they queue run after that microtask +storage resolves + fail the Session if an own observer threw (the commit stands); release the line +storage rejects after adopt(seq) + the commit stays adopted and published; the storage error fails the Session (section 10.1), + and an own observer's throw is reported +storage fails before adopt(seq) abort every prepared change, fail the Session (section 10.1), and publish nothing ``` @@ -1721,24 +1744,67 @@ sets them for a plain Session; the Harness reads its settings at each use. A Session commit callback may be asynchronous. It owns the Session mutation line through callback execution, preparation, storage settlement, committed -baseline adoption, and publication enqueue. Adoption runs inside +baseline adoption, and publication. Adoption runs inside `Storage.commit()`: the Session passes an `adopt(seq)` callback, which the storage calls synchronously once the batch is durable and before its next commit (section 10), so memory always shows a complete prefix of the log, plus -what deferred commits hold (section 4.1). The callback keeps its own error: a failed adoption fails the Session once -`Storage.commit()` returns. A commit that rejects after its adoption fails the -Session too, and its adopted changes stay adopted. Commit observers run on the -line; watch listeners run later. -External model, process, tool, network, and human effects run outside it. -`subscribeCommits()` observes stored commits synchronously on the line after -adoption, each with exactly what that storage transaction stored (section 4.1). `subscribeClose()` observes close synchronously when it +what deferred commits hold (section 4.1). External model, process, tool, +network, and human effects run outside the line. + +The commit is published when it is adopted, so every view changes in the same +step as memory. Inside `adopt`, after the trackers, come the commit's telemetry +events (`pi.durable.entry`, `pi.durable.submission`), then the Session's own +observers, synchronously: document watches, conversation view mounts, the task +graph, the scheduler, and submission waits, with the observers adapters attach +(`observeDoc()`, `observeConversation()`, `observeTaskGraph()`, section 9.1). They +advance watches and resolve waits, but run no task code: the abort signal of a task the commit newly marks +follows in a microtask. Host `subscribeCommits()` listeners run one microtask +after `adopt`, outside the storage's synchronous step, before every wait the +commit resolves and every abort it queues: those continuations are queued by +the own observers, after the dispatch. There is no order between host listeners +and watch listeners, nor `onWake`: they are independent observers of the +commit, and a watch delivery already queued for an earlier frame may deliver +this commit first. The host listeners run before the Session sees +`Storage.commit()` resolve, while the line is still held. A storage that works +after `adopt` before it resolves, as JSONL does when it reclaims sidecars, runs +what the commit wakes during that work too; its next commit still waits. + +```text +adopt(41): trackers, telemetry events, own observers (views advance, waits resolve) +microtask: subscribeCommits() listeners +microtasks after it: wait continuations, abort listeners +microtasks, in no order relative to the above: watch listeners, onWake +Storage.commit() resolves; session.commit() resolves +``` + +`adopt` never throws. A throwing telemetry adapter is reported (`host_error`, +callback `telemetry`; a plain Session drops it, as it drops every report) and +the commit resolves; a host listener's throw is reported (section 10.1), and an +adapter's observer that throws is also detached and ended (section 9.1). The +first throw of an own observer is kept, the other observers and the host +listeners still get the commit, and it fails the Session once +`Storage.commit()` returned; the commit stands and resolves. A throw while +memory adopts the batch is different: nothing is published, the Session fails +once `Storage.commit()` returned, and the commit rejects with that error. A +commit that Storage rejects after its adoption is durable: it stays adopted and +published, waits it resolved stay resolved, the commit rejects with the storage +error, and the Session fails with it; an own observer's throw in that commit is +then reported as a `listener_error`. + +`subscribeCommits()` observes stored commits in one synchronous call per +listener, on the line, each with exactly what that storage transaction stored (section 4.1). Adoption +queues the one dispatch of a commit only when a listener is registered; the +dispatch calls the listeners registered when it runs. So a listener added +between `adopt` and the dispatch gets the commit only when some listener was +registered at `adopt`; one removed in between does not get it; one added after +the dispatch ran misses it. `subscribeClose()` observes close synchronously when it begins, after admission is sealed, also when a failure closes the Session; the Harness ends waits, watches, and streams there, and signals task invocations after every close listener has run, so what a task holds ends with the Session's reason rather than as cancelled. Both return idempotent disposers; their listeners must not block or call Session APIs, and one that throws is reported without affecting the others (section 10.1). Watch listeners -still run later, off the line. +still run later, as microtasks the line never waits for (invariant 7). ```ts await session.commit(async tx => { @@ -1834,10 +1900,13 @@ A failed Session stores nothing; after a reopen nothing is held. Publication: the Session's own observers see each held commit once, when memory adopts it, and of a normal commit that stores held changes only its own operations. `subscribeCommits()` sees every storage transaction with exactly -what it stored: nothing for a held commit, and the stored operations, held -first, for a commit or a deadline that stores them. A consumer that applies +what it stored, published when Storage adopts it (section 4): nothing for a +held commit, and the stored operations, held first, for a commit or a deadline +that stores them. A document the commit retires is published as its retirement +only: the held operations and the commit's own change stored before the +retirement are not. A consumer that applies `subscribeCommits()` publications to stored state, such as a replica, stays -equal to Storage. Its `seq` values have gaps where nothing was published. +equal to Storage in every live document. Its `seq` values have gaps where nothing was published. Consistency: every stored document is a state it had in memory, and a normal commit that reads a held document through `tx` stores it with whatever it @@ -2577,12 +2646,12 @@ paused, when the Session next needs a running process: Blocked tasks (section 5.4), `completing` tasks, and waits on live tasks count for neither; the work they wait for does. With `HarnessOptions.onWake`, the Harness arms no timer for sleeps. It calls `onWake` with `wake()` after open and -whenever the value changes, off the Session line; a throw is reported. The host +whenever the value changes, in a microtask the line never waits for; a throw is reported. The host arms its own wakeup and calls `resume()` when it fires, which makes due sleeps runnable. A host whose process can be evicted, such as a Durable Object with its one alarm, arms the alarm at `at`, or at `now + heartbeat` while `active` if that is earlier, and deletes it when neither is set. `onWake` runs after the -commit that changes the value, so a host call that can make an idle Session +commit that changes the value was adopted (section 4), so a host call that can make an idle Session active arms the recovery wakeup first: `submit()`, `compact()`, `abortTask()`, `Conversation.abort()`, a registry change, and `resume()` after open. An early wakeup is harmless; a missing one leaves work stranded. A failed alarm write @@ -4761,23 +4830,26 @@ re-exports both): ```ts type WatchEnd = - | { readonly reason: "stopped" | "cancelled" | "retired" | "ended" } + | { readonly reason: "stopped" | "cancelled" | "ended" } | { readonly reason: "listener_error"; readonly error: Error } /** The source failed; for Durable, what failed the Session, usually a storage error (section 10.1). */ | { readonly reason: "failed"; readonly error: unknown }; +/** One listener call: the newest value, the ops from the previous value, and a retirement since the previous call. */ +type WatchChange = { readonly value: T; readonly ops: readonly Op[]; readonly retired: boolean }; + interface WatchHandle { /** Acquisition revision before start; latest delivered immutable revision afterward. */ readonly value: T; /** Installs the sole serialized asynchronous listener. */ - start(listener: (value: T, ops: readonly Op[], context: Context) => Promise): void; + start(listener: (change: WatchChange, context: Context) => Promise): void; /** Idempotently stops future callbacks and returns this watch's terminal result. */ stop(): Promise; /** Settles when the watch terminates; an already-running callback remains caller-owned. */ readonly closed: Promise; } -type DocumentWatch = WatchHandle | null>; +type DocumentWatch = WatchHandle>; interface DocumentObserver { watchDoc(token: SessionDocToken, context: Context): Promise | undefined>; @@ -4790,15 +4862,33 @@ interface DocumentObserver { ``` A watch ends once, with the first reason that applies: `stopped` by `stop()`; -`cancelled` when its acquisition `Context` is cancelled; `retired` when its -document incarnation retired; `ended` when the Session closed; `listener_error` -when its listener threw or rejected; `failed` when the Session failed. +`cancelled` when its acquisition `Context` is cancelled; `ended` when the owner +of what it watches ends (the Session closed, or a task document's task became +terminal); `listener_error` when its listener threw or rejected; `failed` when +the Session failed. -Watch acquisition never creates; absent lookup returns `undefined`. Successful -acquisition binds one concrete incarnation. Retirement is an explicit event of -that incarnation, never inferred from a `null` value: it delivers a JSON `null` -root replacement `[["r", null]]` as the last frame, then ends the watch as -`retired`. A later recreation requires acquiring a new watch. +Watch acquisition never creates; absent lookup returns `undefined`. A document +watch follows its address (kind, scope, and key) from the incarnation found at +acquisition. A retirement does not end it: the next call has `retired` set, +with the last value the watch was given and no ops (Durable has no value for an +absent document; `null` never appears). A commit that changes a document and +retires it publishes the retirement only, so that call carries the value from +before the commit, not the content stored with the retirement. A document created at the address later arrives as a +root replacement `[["r", value]]` of the new incarnation, folded with what is +pending, so one call may report a retirement and carry the next incarnation: + +```text +commit: retireDoc(StateDoc); doc(StateDoc).value = 10 +call: { value: { value: 10, ... }, ops: [["r", { value: 10, ... }]], retired: true } +``` + +Task documents retire when their task becomes terminal and are never created +again, so a task document's watch gets the retirement of that commit as its last +call and then ends `ended`, once that call returned. Until then Session close or +failure still ends it at once, dropping the retirement if it was not delivered +yet, as for a watch never started. A watch of a session or conversation document that +is retired for good stays until it is stopped, its context is cancelled, or the +Session ends; it pins nothing while its address is absent. Session close ends every watch at its seal: the watch keeps its last value and receives no frame from a commit that settles during close. @@ -4816,15 +4906,72 @@ acquisition, and every later commit is buffered until `start()`. This prevents a value from being paired with operations based on a newer unseen revision. +**Observers.** Watches are for application code. Adapters that must change in +the same step as Durable's memory, such as a Chord source over a Durable view, +attach a synchronous observer instead; every watch is built on one: + +```ts +type Observer = { + advance(change: WatchChange, context: Context): void; + end(failure: { readonly error: unknown } | undefined): void; +}; +type Observation = { readonly value: T; detach(): void }; + +interface Session { + observeDoc(token: SessionDocToken, observer: Observer>, context: Context): Promise> | undefined>; + // ... and the other five scopes (section 3.3), the observer before the context +} +interface Harness { + observeConversation(id: ConversationId, observer: Observer, context: Context): Promise>; + observeTaskGraph(observer: Observer, context: Context): Promise>; +} +``` + +Attaching works like acquiring a watch, atomically on the Session line: the +observation's `value` includes every commit before it, and `advance` gets every +later change that a watch would get, unfolded, as memory adopts it: for a +stored commit inside Storage's `adopt` (section 4), for a deferred one when +memory adopts it, before it is stored (section 4.1). A commit gives a view or +the graph at most one call, and a document observer one call per change at its +address: a commit that retires the document and creates it again gives two, the +retirement and then the new incarnation's `[["r", value]]`. The task graph also +advances without a commit, when a registry change or a failed migration changes +a task's `blocked` reason (section 9.5): that call runs on the Session line, +outside any `adopt`, with a context of its own. A document observer follows its +address as a document watch does, and `observeDoc()` gives `undefined` for an +absent one. `end` comes once: when the Session closes (`undefined`) or fails +(`{ error }`), or, for a task document, right after the retirement change of +the commit that makes its task terminal, inside that commit's `adopt`. +`detach()` is idempotent, and nothing follows it, not even the rest of the +commit underway; a detached observer is referenced by nothing of the Session. +`context` governs only the attachment: cancelling it while attaching rejects +the attachment and leaves nothing attached; cancelling it afterwards does +nothing, and `detach()` ends the observation. + +The attachment completes on the line, and the caller's continuation runs +later, so a commit adopted in between reaches `advance` before the promise of +`observeXXX()` resolves. Each change carries the whole new value, and its +operations apply to the value before it, starting from `observation.value`: an +adapter that gets a change before its observation records it, and applies it +once it has the observation, or takes `change.value`. + +`advance` runs inside the storage's commit, so an observer records the change +and returns: no awaits, no Session calls, no throws, and quick, as its time is +part of every commit; so does `end` when a task ends. One that throws anyway is +reported as `listener_error`, detached, and ended with the error; the commit and +the other observers are not affected. The change object is plain data, shared by +every observer of a view and by the watches built on it, and must not be +mutated. + ### 9.2 `watchDoc` Tasks, hooks, and tools may observe any existing document for which their code has a token and owner/key. There is no additional subtree permission system inside trusted Session code. -On the Session line, `watchDoc()` resolves one existing concrete incarnation, +On the Session line, `watchDoc()` resolves the incarnation at the address, captures its current immutable tracker revision in O(1), and registers for every -later committed revision and its operation batch. It returns `undefined` when +later committed revision at that address and its operation batch. It returns `undefined` when absent. Before `start()`, `watch.value` remains the acquisition revision while later frames buffer. @@ -4833,7 +4980,7 @@ const watch = await api.watchDoc(JobOutputDoc, producerTaskId, context); if (watch === undefined) return; try { await initializeConsumer(watch.value, context); - watch.start(async (value, ops, deliveryContext) => { + watch.start(async ({ value, ops }, deliveryContext) => { await consume(value, ops, deliveryContext); }); } catch (error) { @@ -4868,10 +5015,12 @@ the trusted immutability contract. Consumers must not mutate them or retained descendants. The selected commit Context's values are preserved without inheriting the producer's cancellation. A callback owns its own asynchronous work. -A watch remains bound to its original incarnation. Retirement drops what is -pending and queues the terminal `[["r", null]]` with value `null`. After that -callback settles, the watch closes as `retired` and never -follows a replacement incarnation. +A watch follows its address (section 9.1). A retirement folds into the +pending call like any commit and sets its `retired` flag; its value is the last +one and its operations are those still pending. A later incarnation arrives as +`[["r", value]]`. A retirement that ends a task document's task is that watch's +last call: it is delivered, then the watch ends `ended` once the call returned. While its address is +absent a watch pins nothing; a new incarnation is pinned while it is watched. `stop()` is idempotent. It synchronously unregisters, discards pending frames, prevents another callback from starting, and returns the common terminal promise. @@ -4904,7 +5053,9 @@ The Harness keeps at most one mount per conversation. The first `watch()` or event attachment builds its revision on the Session line from the committed active transcript and documents; the mount is dropped when its last observer detaches. A mount derives each next revision from the Session's -commits as memory adopts them, like `watchDoc()`: it shows committed state, +commits as memory adopts them, for a stored commit inside Storage's `adopt` +(section 4) and for a held one when memory adopts it (section 4.1), like +`watchDoc()`: it shows committed state, including deferred commits before they are stored (section 4.1). One publication that touches the conversation yields one Chord batch: @@ -4929,7 +5080,9 @@ are structurally shared under the trusted immutability contract. The mount performs no semantic projection and owns no persistence. `Conversation.watch()` exposes the mount through the serialized watch of -section 9.2, with folded pending commits and a full-value overflow replacement. +section 9.2, with folded pending commits and a full-value overflow replacement; +`Harness.observeConversation()` attaches a synchronous observer of the same mount +(section 9.1). A view never retires, so its changes never set `retired`. ### 9.4 Agent events (experimental) @@ -5014,7 +5167,8 @@ type MessageChange = interface AgentEventStream { /** The `snapshot` event at attachment. */ readonly snapshot: Extract; - start(listener: (events: readonly AgentEvent[], context: Context) => Promise): void; + /** Each call's `value` is one commit's batch, or a `snapshot` after an overflow (Chord's `NativeWatchChange`). */ + start(listener: (change: { readonly value: readonly AgentEvent[] }, context: Context) => Promise): void; stop(): Promise; readonly closed: Promise; } @@ -5144,6 +5298,7 @@ type TaskGraphWatch = WatchHandle; interface Harness { watchTaskGraph(context: Context): Promise; + observeTaskGraph(observer: Observer, context: Context): Promise>; } ``` @@ -5170,19 +5325,25 @@ A UI that places it under its parent reads the conversation's owner edge from The Harness keeps at most one graph mount. The first `watchTaskGraph()` builds its revision on the Session line from the committed live tasks and the conversations they own, one owner scan per live task, and holds the line while it does; the mount is dropped when its last observer -detaches. It advances from `subscribeCommits()` publications like the -conversation view: one publication that changes a node yields one Chord batch +detaches. It advances from the Session's stored commits as memory adopts them, +inside Storage's `adopt` (section 4), like the conversation view: a commit that +touches a conversation view and the graph changes both in that one step. A +`blocked` reason that changes without a commit (a registry change, a failed +migration) is derived again in a job on the Session line and published there, +outside any `adopt` and any commit, with the background context. One +publication that changes a node yields one Chord batch that sets or deletes `tasks[id]`, or sets a node's `conversations`. A publication that changes no node creates no revision. Task and conversation records are immutable values; a node shares nothing with them. `watchTaskGraph()` returns the mount through the serialized watch of section -9.2. It reads only: it never enables scheduling. Close ends it at its +9.2, and `observeTaskGraph()` attaches a synchronous observer of it (section +9.1). Both read only: they never enable scheduling. Close ends them at its seal like every other watch (section 2.2). ```ts const graph = await harness.watchTaskGraph(context); -graph.start(async (value) => { +graph.start(async ({ value }) => { for (const node of Object.values(value.tasks)) { const where = node.owner === undefined ? `conversation ${node.conversationId}` : `task ${node.owner}`; render(`${node.id} ${node.kind} ${node.state.status} under ${where}`); @@ -5292,7 +5453,8 @@ interface Storage { /** * Persist the batch atomically. Once it is durable, call `adopt(seq)` exactly once, synchronously and before this * storage's next commit starts, then resolve with `seq`; a commit that fails before the batch is durable never calls - * it. `adopt` must not throw. + * it. `adopt` adopts and publishes the batch: it runs the Session's own observers (section 4), which call no Storage + * method, and it never throws. */ commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, context: Context): Promise; /** @@ -5437,8 +5599,12 @@ exempt: once a batch is admitted, whether it committed is unknown. A commit callback that catches a failed read cannot commit afterwards: the Session checks its health after the callback. Adopting a committed batch in memory that fails also fails the Session, since memory would be behind storage; so does a -`Storage.commit()` that resolves without adopting, or rejects after it adopted -(the adopted changes are not rolled back). +`Storage.commit()` that resolves without adopting, or rejects after it adopted. +A batch Storage adopted is durable, so it is not rolled back: it was published +at adoption (section 4), `subscribeCommits()` listeners get it and the waits it +settles resolve; then the commit rejects with the storage error and the Session +fails as above, with the storage error. An own observer that threw on that +commit is reported as a `listener_error`. **The Session's own bookkeeping fails it too.** A throw in a commit the scheduler makes for itself (reconciliation, reservation, a fault or terminal @@ -5448,7 +5614,9 @@ storage failure, a host callback such as a `RegistryReader` whose `snapshot()` throws, or a bug, and running the commit again fixes none of them. A throw in a commit listener of the Session's own components (scheduler, submissions, views, task graph, document observers) fails it as well, since their memory would fall -behind storage; the commit itself stands. +behind storage; the commit itself stands. Those listeners run inside Storage's +`adopt` (section 4), so the Session fails once `Storage.commit()` returned, +after the other observers and the `subscribeCommits()` listeners got the commit. **User code fails only its unit.** A task phase that throws ends that task `faulted`; a throwing `migrate` blocks it as `migration_failed`; a tool that @@ -5461,8 +5629,12 @@ Callbacks that must not fail their caller are contained: a host view observer that throws, or returns a promise that rejects, are reported, and the others still run; a watch whose listener throws ends with `listener_error`. A throwing or rejecting `onReport` is -dropped, since nothing is left to report it to. A throwing `now()` is reported -once, and `Date.now` serves instead. +dropped, since nothing is left to report it to, and so is a report's own +metric or event that throws. A throwing `now()` is reported once, and +`Date.now` serves instead. A telemetry adapter that throws on a commit's events +or on the active-task metric is reported as `host_error` with callback +`telemetry` (a plain Session drops it); the commit resolves, and memory and the +views keep it. **Every report says what happened.** `onReport(error, info)` gets a `ReportInfo` whose `type` is one of: @@ -5482,7 +5654,8 @@ once, and `Date.now` serves instead. outside a task the conversation whose agent was resolved; - `host_error`: a host callback, by its `HarnessOptions` name: `now`, `settings`, `registry`, `env`, `models` for provider calls such as cancelling - a deferred response, `setTimer`, or `onWake`; with the task that called it, + a deferred response, `setTimer`, `onWake`, or `telemetry` for a telemetry + adapter that threw while a commit was published; with the task that called it, if one did; - `listener_error`: a commit, close, watch, view, or document listener. diff --git a/packages/durable/src/env/index.ts b/packages/durable/src/env/index.ts index 0d8da86f7..170c43579 100644 --- a/packages/durable/src/env/index.ts +++ b/packages/durable/src/env/index.ts @@ -124,7 +124,7 @@ export interface WatchTarget { * - `error`: the watcher stopped, for example because the watched tree grew past the environment's limit; no calls * follow. */ -export type WatchChange = { paths: string[] } | { overflow: true } | { error: FileError }; +export type FileChange = { paths: string[] } | { overflow: true } | { error: FileError }; export interface FileWatcher { /** @@ -257,11 +257,11 @@ export interface FileSystem { /** * Report changes to files and directories, for hosts that load resources from the environment. When the returned * watcher exists, coverage is established: a host that watches before it loads cannot miss a change made during the - * load. See `WatchChange` for what is reported and `FileWatcher.mode` for how reliably. + * load. See `FileChange` for what is reported and `FileWatcher.mode` for how reliably. */ watch( targets: readonly WatchTarget[], - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, context: Context, ): Promise>; canonicalPath(path: string, context: Context): Promise>; diff --git a/packages/durable/src/env/node-watch.ts b/packages/durable/src/env/node-watch.ts index 7f3b8ab55..84d635e6f 100644 --- a/packages/durable/src/env/node-watch.ts +++ b/packages/durable/src/env/node-watch.ts @@ -3,7 +3,7 @@ import { type FSWatcher, watch as fsWatch, type Stats } from "node:fs"; import { lstat, readdir, readFile, stat, statfs } from "node:fs/promises"; import { dirname, relative, sep } from "node:path"; import type { Context } from "@earendil-works/chord"; -import { FileError, type FileWatcher, type WatchChange, type WatchTarget } from "./index.ts"; +import { type FileChange, FileError, type FileWatcher, type WatchTarget } from "./index.ts"; /** How `NodeExecutionEnv` watches. */ export interface NodeWatchOptions { @@ -138,7 +138,7 @@ function excluded(target: ResolvedTarget, name: string): boolean { */ export class NodeFileWatcher implements FileWatcher { readonly #targets: readonly ResolvedTarget[]; - readonly #onChange: (change: WatchChange) => void; + readonly #onChange: (change: FileChange) => void; readonly #pollIntervalMs: number; readonly #maxDirectories: number; readonly #watchers = new Map(); @@ -153,7 +153,7 @@ export class NodeFileWatcher implements FileWatcher { private constructor( targets: readonly ResolvedTarget[], - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, mode: "native" | "polling", options: NodeWatchOptions, ) { @@ -168,7 +168,7 @@ export class NodeFileWatcher implements FileWatcher { static async open( targets: readonly WatchTarget[], resolvePath: (path: string) => string, - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, options: NodeWatchOptions, ): Promise { const resolved = targets.map((target) => ({ @@ -216,7 +216,7 @@ export class NodeFileWatcher implements FileWatcher { this.#watchers.clear(); } - #deliver(change: WatchChange): void { + #deliver(change: FileChange): void { if (this.#closed) return; try { this.#onChange(change); diff --git a/packages/durable/src/env/node.ts b/packages/durable/src/env/node.ts index fe93ce4ed..d45ad2cae 100644 --- a/packages/durable/src/env/node.ts +++ b/packages/durable/src/env/node.ts @@ -28,6 +28,7 @@ import { type ExecutionEnv, ExecutionError, err, + type FileChange, FileError, type FileInfo, type FileKind, @@ -41,7 +42,6 @@ import { type TextLine, type TextLineReader, toError, - type WatchChange, type WatchTarget, } from "./index.ts"; import { LineScanner } from "./line-scan.ts"; @@ -627,7 +627,7 @@ export class NodeExecutionEnv implements ExecutionEnv { async watch( targets: readonly WatchTarget[], - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, context: Context, ): Promise> { const aborted = abortResult(context.abortSignal); diff --git a/packages/durable/src/harness/events.ts b/packages/durable/src/harness/events.ts index ae3e4de9d..f34af4659 100644 --- a/packages/durable/src/harness/events.ts +++ b/packages/durable/src/harness/events.ts @@ -1,4 +1,10 @@ -import { type Context, type JsonValue, NativeWatch, type WatchEnd } from "@earendil-works/chord"; +import { + type Context, + type JsonValue, + NativeWatch, + type NativeWatchChange, + type WatchEnd, +} from "@earendil-works/chord"; import type { Op, Path } from "@earendil-works/chord/delta"; import type { AssistantMessage, Message, Usage } from "@earendil-works/pi-ai"; import type { ObservedCommit } from "../session/session.ts"; @@ -114,7 +120,8 @@ export type ToolEventCall = { export interface AgentEventStream { /** The `snapshot` event at attachment. */ readonly snapshot: SnapshotEvent; - start(listener: (events: readonly AgentEvent[], context: Context) => Promise): void; + /** Install the sole listener; each call's `value` is one commit's batch, or a `snapshot` after an overflow. */ + start(listener: (change: NativeWatchChange, context: Context) => Promise): void; stop(): Promise; readonly closed: Promise; } diff --git a/packages/durable/src/harness/harness.ts b/packages/durable/src/harness/harness.ts index 7ce8de9c8..1df3422d7 100644 --- a/packages/durable/src/harness/harness.ts +++ b/packages/durable/src/harness/harness.ts @@ -28,6 +28,8 @@ import type { EntryId, EntryQuery, EntryRecord, + Observation, + Observer, Page, ReportInfo, Storage, @@ -46,7 +48,7 @@ import { ProviderDoc } from "./provider.ts"; import { BUILTIN_TASKS } from "./registry.ts"; import { type InvocationBinding, TaskScheduler } from "./scheduler.ts"; import { Submissions } from "./submissions.ts"; -import { TaskGraphView, type TaskGraphWatch } from "./task-graph.ts"; +import { type TaskGraph, TaskGraphView, type TaskGraphWatch } from "./task-graph.ts"; import { DURABLE_TELEMETRY_SCHEMA } from "./telemetry.ts"; import type { Agent, @@ -72,7 +74,7 @@ import type { } from "./types.ts"; import { addUsageState, UsageDoc, type UsageState } from "./usage.ts"; import { scanAll } from "./util.ts"; -import { ConversationViews } from "./view.ts"; +import { type ConversationView, ConversationViews } from "./view.ts"; const SCAN_PAGE_SIZE = 256; @@ -373,6 +375,20 @@ class HarnessImpl extends SessionImpl implements return this.#taskGraph.watch(context); } + async observeTaskGraph(observer: Observer, context: Context): Promise> { + this.assertUsable(); + return this.#taskGraph.observe(observer, context); + } + + async observeConversation( + id: ConversationId, + observer: Observer, + context: Context, + ): Promise> { + this.assertUsable(); + return this.#host.views.observe(id, observer, context); + } + root( context: Context, options?: { readonly agent?: AgentChange; readonly init?: ConversationInit }, @@ -433,7 +449,10 @@ class HarnessImpl extends SessionImpl implements } } -/** Preserve the host callback's error containment; telemetry adapters are responsible for their own failures. */ +/** + * Report to `onReport`, then count it and record its event. Never throws, since reports also come from inside Storage's + * `adopt`: a throwing `onReport` is dropped, and so is a throwing metric or event, as reporting it would report again. + */ function safeReport( onReport: ((error: unknown, info: ReportInfo) => void) | undefined, telemetry: TelemetryContext, @@ -467,8 +486,12 @@ function safeReport( attributes["task.id"] = info.task?.id; break; } - reports.add(1, { type: info.type }); - context?.telemetry.addEvent("pi.durable.report", attributes); + try { + reports.add(1, { type: info.type }); + context?.telemetry.addEvent("pi.durable.report", attributes); + } catch { + // Dropped: see above. + } }; } diff --git a/packages/durable/src/harness/scheduler.ts b/packages/durable/src/harness/scheduler.ts index 9a7fcf2e2..0840533a8 100644 --- a/packages/durable/src/harness/scheduler.ts +++ b/packages/durable/src/harness/scheduler.ts @@ -664,9 +664,10 @@ export class TaskScheduler { } if (record.abortRequested && previous?.abortRequested !== true) { this.#cascadePending = true; - // Signal a run invocation of the newly marked task; its next step ends it. + // Signal a run invocation of the newly marked task; its next step ends it. Abort listeners are task code, which + // never runs inside Storage's `adopt`: the signal follows in a microtask, after host commit listeners. const invocation = this.#invocations.get(record.id); - if (invocation?.mode === "run") invocation.controller.abort(); + if (invocation?.mode === "run") queueMicrotask(() => invocation.controller.abort()); } // A request replacing a `restart` mark upgrades the marks of the work below. if (previous?.abortReason !== undefined && record.abortReason === undefined) this.#cascadePending = true; @@ -1023,10 +1024,10 @@ export class TaskScheduler { const invocation = this.#invocations.get(id); if (invocation !== undefined) invocation.record = record as TaskRecord; if (record.state.status === "terminal") { - if (previous !== undefined && !this.#closing) this.#activeMetric.add(-1, { task: record.kind }); + if (previous !== undefined && !this.#closing) this.#countActive(-1, record.kind); this.#resumed.delete(id); } else if (previous === undefined && !this.#closing) { - this.#activeMetric.add(1, { task: record.kind }); + this.#countActive(1, record.kind); } if (previous?.state.status === "waiting") for (const on of previous.state.on) deleteFrom(this.#waiters, on, id); if (record.state.status === "terminal") { @@ -1304,6 +1305,18 @@ export class TaskScheduler { return undefined; } + /** + * Count a task in or out of `pi.durable.tasks.active`. Commit observers count inside Storage's `adopt`, which must not + * throw, so a throwing adapter is reported. + */ + #countActive(delta: 1 | -1, kind: string): void { + try { + this.#activeMetric.add(delta, { task: kind }); + } catch (error) { + this.#report(error, TELEMETRY); + } + } + /** Fail the Session, which reports it, for a throw in one of the scheduler's own commits; ignored once closing. */ #failSession(error: unknown, context?: Context): void { if (!this.#closing) this.#session.fail(error, context); @@ -1311,7 +1324,7 @@ export class TaskScheduler { /** Close listener: runs synchronously once admission is sealed, before `join()`, or when the Session fails. */ #seal(): void { - for (const record of this.#live.values()) this.#activeMetric.add(-1, { task: record.kind }); + for (const record of this.#live.values()) this.#countActive(-1, record.kind); this.#resumed.clear(); this.#closing = true; try { @@ -1329,7 +1342,10 @@ export class TaskScheduler { this.#timer = undefined; } - /** Recompute the wake in a microtask, off the Session line: arm the built-in timer, or tell `onWake` of a change. */ + /** + * Recompute the wake in a microtask the line never waits for, with no order relative to host commit listeners (one + * already queued serves later changes too): arm the built-in timer, or tell `onWake` of a change. + */ #scheduleWake(): void { if (this.#wakeScheduled || this.#closing || this.#readOnly) return; this.#wakeScheduled = true; @@ -2385,6 +2401,7 @@ export class TaskScheduler { const REGISTRY: ReportInfo = { type: "host_error", callback: "registry" }; const SETTINGS: ReportInfo = { type: "host_error", callback: "settings" }; const ON_WAKE: ReportInfo = { type: "host_error", callback: "onWake" }; +const TELEMETRY: ReportInfo = { type: "host_error", callback: "telemetry" }; /** The extension that installed `task` in `snapshot`, the last one that lists it; none for a built-in task. */ function installerOf(snapshot: RegistrySnapshot, task: AnyTask | undefined): string | undefined { diff --git a/packages/durable/src/harness/task-graph.ts b/packages/durable/src/harness/task-graph.ts index 55ee659ed..828e7077e 100644 --- a/packages/durable/src/harness/task-graph.ts +++ b/packages/durable/src/harness/task-graph.ts @@ -1,8 +1,17 @@ -import { type Context, type JsonValue, Watch, type WatchHandle } from "@earendil-works/chord"; +import { type Context, type JsonValue, Watch, type WatchChange, type WatchHandle } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT, withoutAbortSignal } from "@earendil-works/chord/context"; import { applyImmutable, type Op } from "@earendil-works/chord/delta"; -import { contained, type ObservedCommit, type SessionImpl } from "../session/session.ts"; -import type { ConversationId, JoinPolicy, Storage, TaskId, TaskOutcome, TaskRecord } from "../types.ts"; +import { contained, guardObserver, type ObservedCommit, observation, type SessionImpl } from "../session/session.ts"; +import type { + ConversationId, + JoinPolicy, + Observation, + Observer, + Storage, + TaskId, + TaskOutcome, + TaskRecord, +} from "../types.ts"; import { type BlockedReason, type BlockedSubject, blockedSubject } from "./scheduler.ts"; import { closedError, scanAll } from "./util.ts"; @@ -47,10 +56,10 @@ export type TaskGraph = { export type TaskGraphWatch = WatchHandle; type AnyTaskRecord = TaskRecord; -type Observer = Watch; type Mount = { value: TaskGraph; - readonly observers: Set; + /** Watches and guarded host observers (`observeTaskGraph()`). */ + readonly observers: Set>; /** What the blocked reason reads of each live task, so a registry change can derive it again without payloads. */ readonly records: Map; }; @@ -97,7 +106,25 @@ export class TaskGraphView { }); } - /** A serialized watch of the graph; cancelling `context` stops it. */ + /** + * Attach a synchronous observer of the graph (`Harness.observeTaskGraph()`), behind `guardObserver()`; `context` + * governs only the attachment. + */ + async observe(observer: Observer, context: Context): Promise> { + let value!: TaskGraph; + const attached = await this.#attach((current, release) => { + value = current; + return guardObserver(observer, release, (error) => this.#session.report(error)); + }, context); + const signal = context.abortSignal; + if (signal?.aborted) { + attached.detach(); + throw signal.reason; + } + return observation(value, attached.observer); + } + + /** A serialized watch of the graph, an observer of its mount; cancelling `context` stops it. */ async watch(context: Context): Promise { const { observer } = await this.#attach( (value, release) => new Watch(value, release, (error) => this.#session.report(error)), @@ -113,12 +140,12 @@ export class TaskGraphView { } /** Register an observer created from the current revision, atomically on the Session line. */ - #attach( + #attach>( create: (value: TaskGraph, release: () => void) => O, context: Context, ): Promise<{ observer: O; detach: () => void }> { return this.#session.readOnLine(async () => { - const mount = this.#mount ?? { ...(await this.#build(context)), observers: new Set() }; + const mount = this.#mount ?? { ...(await this.#build(context)), observers: new Set>() }; const detach = (): void => { mount.observers.delete(observer); if (mount.observers.size === 0 && this.#mount === mount) this.#mount = undefined; @@ -232,8 +259,8 @@ function publish(mount: Mount, ops: Op[], context: Context, report: (error: unkn if (ops.length === 0) return; mount.value = applyImmutable(mount.value, ops); const frameContext = withoutAbortSignal(context); - for (const observer of [...mount.observers]) - contained(() => observer.advance(mount.value, ops, frameContext), report); + const change: WatchChange = { value: mount.value, ops, retired: false }; + for (const observer of [...mount.observers]) contained(() => observer.advance(change, frameContext), report); } function nodeOf( diff --git a/packages/durable/src/harness/types.ts b/packages/durable/src/harness/types.ts index 748696f8c..61a9c0ca6 100644 --- a/packages/durable/src/harness/types.ts +++ b/packages/durable/src/harness/types.ts @@ -30,6 +30,8 @@ import type { EntryQuery, EntryRecord, JsonObject, + Observation, + Observer, Page, ReportInfo, Session, @@ -43,7 +45,7 @@ import type { TaskState, Tx, } from "../types.ts"; -import type { TaskGraphWatch } from "./task-graph.ts"; +import type { TaskGraph, TaskGraphWatch } from "./task-graph.ts"; import type { UsageState } from "./usage.ts"; import type { ConversationView } from "./view.ts"; @@ -661,9 +663,10 @@ export type HarnessOptions = { * Wakes the process for sleeping tasks instead of a `setTimer` timer. Without it, while scheduling runs, a timer that * keeps the process alive resumes sleeping tasks when they are due. With it, the Harness arms no timer for them: it * calls `onWake` after open and whenever `Harness.wake()` changes, and the host arms its own wakeup, such as a - * Durable Object alarm, and calls `resume()` when it fires. Called off the Session line, after the commit that - * changed the value, so a host arms a recovery wakeup before calls that can start work. It may call `wake()` and - * `resume()`, and must not block. A throw is reported, and the value is offered again at the next change. + * Durable Object alarm, and calls `resume()` when it fires. Called in a microtask the Session line never waits for, + * after the commit that changed the value was adopted, so a host arms a recovery wakeup before calls that can start + * work. It may call `wake()` and `resume()`, and must not block. A throw is reported, and the value is offered again + * at the next change. */ readonly onWake?: (wake: Wake) => void; /** @@ -791,7 +794,10 @@ export interface Conversation { * owned, transitively, by its non-background tasks. */ waitForIdle(context: Context): Promise; - /** The structural view (spec §9.3) as a serialized watch; commits made while its listener runs arrive folded. */ + /** + * The structural view (spec §9.3) as a serialized watch; commits made while its listener runs arrive folded. Built on + * the same mount as `Harness.observeConversation()`. + */ watch(context: Context): Promise; } @@ -854,6 +860,23 @@ export interface Harness extends Session { * made while its listener runs arrive folded. */ watchTaskGraph(context: Context): Promise; + /** + * Attach a synchronous `Observer` to a conversation's structural view (spec §9.3), atomically on the Session line, + * for adapters: `advance` gets each next revision with its operations as memory adopts the commit, inside Storage's + * `adopt` for a stored one and before it is stored for a deferred one, so it only records it. `context` governs only + * the attachment. + */ + observeConversation( + id: ConversationId, + observer: Observer, + context: Context, + ): Promise>; + /** + * Attach a synchronous `Observer` to the task graph (spec §9.5), as `observeConversation()` does to a view. A + * `blocked` reason that changes without a commit (a registry change, a failed migration) advances it on the Session + * line outside any commit and `adopt`, with the background context. + */ + observeTaskGraph(observer: Observer, context: Context): Promise>; } /** What a hook may use: committed reads and the asking task's memos, which hooks and the task share. */ diff --git a/packages/durable/src/harness/view.ts b/packages/durable/src/harness/view.ts index 7f6f1a408..d12632b24 100644 --- a/packages/durable/src/harness/view.ts +++ b/packages/durable/src/harness/view.ts @@ -1,7 +1,7 @@ -import { type Context, type JsonValue, Watch, type WatchHandle } from "@earendil-works/chord"; +import { type Context, type JsonValue, Watch, type WatchChange, type WatchHandle } from "@earendil-works/chord"; import { withoutAbortSignal } from "@earendil-works/chord/context"; import { applyImmutable, type NonEmptyPath, type Op, type Path } from "@earendil-works/chord/delta"; -import { contained, type ObservedCommit, type SessionImpl } from "../session/session.ts"; +import { contained, guardObserver, type ObservedCommit, observation, type SessionImpl } from "../session/session.ts"; import type { ConversationDocToken, ConversationId, @@ -9,6 +9,8 @@ import type { DocumentId, EntryRecord, JsonObject, + Observation, + Observer, Storage, } from "../types.ts"; import { AgentDoc } from "./agent.ts"; @@ -30,7 +32,8 @@ export type ConversationView = { /** Receives each next revision of a mount, and the Session's close. */ export type ViewObserver = { - advance?(value: ConversationView, ops: readonly Op[], context: Context): void; + /** Each next revision; one change object is shared by every observer of the mount. */ + advance?(change: WatchChange, context: Context): void; /** Every publication, after the mount took it; `ops` are the mount's, possibly none. */ publication?( before: ConversationView, @@ -106,7 +109,33 @@ export class ConversationViews { this.#session.report(error); } - /** A serialized watch of the view; cancelling `context` stops it. */ + /** + * Attach a synchronous observer of the view (`Harness.observeConversation()`), behind `guardObserver()`; `context` + * governs only the attachment. + */ + async observe( + id: ConversationId, + observer: Observer, + context: Context, + ): Promise> { + let value!: ConversationView; + const attached = await this.attach( + id, + (current, release) => { + value = current; + return guardObserver(observer, release, (error) => this.report(error)); + }, + context, + ); + const signal = context.abortSignal; + if (signal?.aborted) { + attached.detach(); + throw signal.reason; + } + return observation(value, attached.observer); + } + + /** A serialized watch of the view, an observer of its mount; cancelling `context` stops it. */ async watch(id: ConversationId, context: Context): Promise> { const { observer } = await this.attach( id, @@ -219,8 +248,9 @@ function advance( if (ops.length > 0) { const value = docOps.length === 0 ? mount.value : applyImmutable(mount.value, docOps); mount.value = entries === value.entries ? value : { ...value, entries }; + const change: WatchChange = { value: mount.value, ops, retired: false }; for (const observer of [...mount.observers]) { - contained(() => observer.advance?.(mount.value, ops, frameContext), report); + contained(() => observer.advance?.(change, frameContext), report); } } for (const observer of [...mount.observers]) { diff --git a/packages/durable/src/index.ts b/packages/durable/src/index.ts index b26a50316..3baf2e684 100644 --- a/packages/durable/src/index.ts +++ b/packages/durable/src/index.ts @@ -1,4 +1,4 @@ -export type { WatchEnd, WatchHandle } from "@earendil-works/chord"; +export type { NativeWatchChange, WatchChange, WatchEnd, WatchHandle } from "@earendil-works/chord"; export { defineDoc, defineDocFamily } from "./documents.ts"; export { AssistantEntry, @@ -171,6 +171,8 @@ export type { JsonObject, LatestConversationSemantics, NextTaskState, + Observation, + Observer, Page, PhaseHandler, ReportFrom, diff --git a/packages/durable/src/session/session.ts b/packages/durable/src/session/session.ts index d66045b19..1c0f3fd0c 100644 --- a/packages/durable/src/session/session.ts +++ b/packages/durable/src/session/session.ts @@ -1,6 +1,7 @@ import { type Context, type JsonValue, Watch } from "@earendil-works/chord"; import { awaitWithContext, BACKGROUND_CONTEXT, withoutAbortSignal } from "@earendil-works/chord/context"; import { type Op, PendingOps, track } from "@earendil-works/chord/delta"; +import type { SpanAttributes } from "@earendil-works/pi-telemetry"; import { type AnyDocToken, addressId, @@ -35,6 +36,8 @@ import type { EntryRecord, Id, JsonObject, + Observation, + Observer, Page, ReportInfo, RewindableConversationDocFamilyToken, @@ -68,9 +71,6 @@ import { type TransactionScope, } from "./transaction.ts"; -/** What a document watch observes: the committed value of its incarnation, `null` once it retired. */ -type ObservedDocumentValue = Readonly | null; - /** * What the Session's own components observe of one commit (`observeCommits()`): a stored commit with its own changes, or * a deferred one adopted in memory only (`CommitPersistence`), which changes documents only. Memory is never behind: @@ -94,9 +94,18 @@ const NO_HOLDS: readonly Hold[] = []; const NO_OPERATIONS: readonly Op[] = []; +/** + * Queues a commit's host dispatch as one microtask through `then`: in the same queue as `queueMicrotask()`, which Node + * wraps in an async resource that costs about twice the memory and three times the time per stored commit. + */ +const SETTLED: Promise = Promise.resolve(); + /** The default source of a report: a listener, which never fails what ran it. */ const LISTENER: ReportInfo = { type: "listener_error" }; +/** A telemetry adapter that threw on a commit's event, which must not fail the durable commit. */ +const TELEMETRY: ReportInfo = { type: "host_error", callback: "telemetry" }; + /** * Open a Session kernel over one storage backend. `now` is the wall clock for task times; default `Date.now`. * `setTimer` arms the timers that evict idle documents; default `setTimeout` (see `HarnessOptions.setTimer`). The first @@ -159,10 +168,12 @@ export function persistWithin(ms: number | undefined): number { * line, so no transaction ever holds a tracker that leaves. * * Only committed state is observable. A deferred commit (`CommitPersistence`) is committed in memory first: it is - * adopted and shown to the Session's own observers, and its changes wait, folded per document and pinned in the cache, for - * the first commit that reads or writes the document, the document's deadline, a drop of its tracker, or close, which - * store them. Every commit callback, preparation, Storage settlement, adoption, and publication enqueue runs while the - * line is held; listeners run later. + * adopted and shown to the Session's own observers, and its changes wait, folded per document and pinned in the cache, + * for the first commit that reads or writes the document, the document's deadline, a drop of its tracker, or close, + * which store them. Every commit callback, preparation, Storage settlement, and adoption runs while the line is held. A + * stored commit is published when Storage adopts it: inside `adopt` its telemetry events and the Session's own + * observers, one microtask later `subscribeCommits()` listeners, before every wait the commit resolves and every abort + * it queues, all still on the line. Host listeners and watch listeners have no order. * * Storage failures are final. Every Storage call goes through one guard: the first error a Storage method throws, other * than for a cancelled caller, fails the Session, as does a commit it cannot adopt. That call gets the error; every later @@ -437,35 +448,92 @@ export class SessionImpl implements Session { key: string, context: Context, ): Promise | undefined>; - async watchDoc(token: AnyDocToken, ...args: readonly unknown[]): Promise | undefined> { - this.assertUsable(); - const definition = token.definition; - const resolved = resolveAddress(definition, args); - const context = args[resolved.nextArgument] as Context; - const signal = context.abortSignal; - let cancelled = signal?.aborted ?? false; - const markCancelled = (): void => { - cancelled = true; - }; - signal?.addEventListener("abort", markCancelled, { once: true }); + watchDoc(token: AnyDocToken, ...args: readonly unknown[]): Promise | undefined> { + // Not async, so an attachment costs one async frame, `#attachDocument`'s; a throw here still rejects. try { - const watch = await this.#enqueue(async () => { - this.#assertHealthy(); - if (cancelled) throw cancellationError(signal!); - const loaded = await this.#loadDocument(definition, resolved.id, resolved.address, context); - if (cancelled) throw cancellationError(signal!); - if (loaded === undefined) return undefined; - return this.#attachDocument(definition, loaded); - }); - if (watch === undefined) return undefined; - if (cancelled) { - watch.cancel(); - throw cancellationError(signal!); - } - if (signal !== undefined) watch.observeCancellation(signal); - return watch as DocumentWatch; - } finally { - signal?.removeEventListener("abort", markCancelled); + this.assertUsable(); + const definition = token.definition; + const resolved = resolveAddress(definition, args); + const context = args[resolved.nextArgument] as Context; + const signal = context.abortSignal; + return this.#attachDocument( + definition, + resolved, + context, + (loaded) => { + const watch = this.#observeDocument(definition, loaded, createWatch, endWatch); + if (signal !== undefined) watch.observeCancellation(signal); + return watch; + }, + cancelWatch, + ); + } catch (error) { + return Promise.reject(error); + } + } + + observeDoc( + token: SessionDocToken, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: ConversationDocToken, + conversationId: ConversationId, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: TaskDocToken, + taskId: TaskId, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: SessionDocFamilyToken, + key: string, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: ConversationDocFamilyToken, + conversationId: ConversationId, + key: string, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: TaskDocFamilyToken, + taskId: TaskId, + key: string, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc(token: AnyDocToken, ...args: readonly unknown[]): Promise> | undefined> { + // Not async, as `watchDoc()`; a throw here still rejects. + try { + this.assertUsable(); + const definition = token.definition; + const resolved = resolveAddress(definition, args); + const observer = args[resolved.nextArgument] as Observer>; + const context = args[resolved.nextArgument + 1] as Context; + return this.#attachDocument( + definition, + resolved, + context, + (loaded) => { + const guarded = this.#observeDocument( + definition, + loaded, + (_value, detach, report) => guardObserver(observer, detach, report), + endGuarded, + ); + return observation(loaded.tracker.value, guarded); + }, + detachObservation, + ); + } catch (error) { + return Promise.reject(error); } } @@ -607,8 +675,10 @@ export class SessionImpl implements Session { } /** - * Register a synchronous listener for every stored commit, with exactly what it stored (`CommitPublication`). It must - * not block or call Session operations; a throw is reported. + * Register a synchronous listener for every stored commit, with exactly what it stored (`CommitPublication`). It runs + * one microtask after Storage adopted the commit, on the line, before every wait the commit resolves and every abort + * it queues (in no order with watch listeners); it must not block or call Session operations, and a throw is + * reported. */ subscribeCommits(listener: (publication: CommitPublication, context: Context) => void): () => void { this.assertUsable(); @@ -618,8 +688,9 @@ export class SessionImpl implements Session { /** * Internal: observe every commit as memory adopts it (`ObservedCommit`), for the Session's own components, whose state - * follows memory. A throw leaves that state behind, so it fails the Session instead of being reported. They run - * before host listeners. + * follows memory. A throw leaves that state behind, so it fails the Session instead of being reported. For a stored + * commit they run inside Storage's `adopt`, so they call no Storage method, run no task code, and contain every host + * callback; the waits they resolve and the aborts they queue run after host listeners. */ observeCommits(listener: (publication: ObservedCommit, context: Context) => void): () => void { this.assertUsable(); @@ -805,23 +876,35 @@ export class SessionImpl implements Session { } // The last step before Storage: held changes leave their queue only once nothing but Storage can fail. const taken = this.#holds.size === 0 ? NO_HOLDS : tx.takeHeld((id) => this.#holds.get(id)); - // Storage adopts the batch synchronously once it is durable, before its next commit; a throw here would reach - // Storage, so the callback keeps its own error. - let adoption: { readonly documents: AdoptedDocuments } | { readonly error: unknown } | undefined; + // Storage adopts the batch synchronously once it is durable, before its next commit, and the commit is published in + // the same step, so every view changes with memory. A throw here would reach Storage, so the callback keeps its own + // error, and the first error of an own observer. + let adoption: + | { readonly documents: AdoptedDocuments; readonly failure?: { readonly error: unknown } } + | { readonly error: unknown } + | undefined; const adopt = (committed: Seq): void => { + let documents: AdoptedDocuments; try { - adoption = { documents: tx.adopt(committed) }; + documents = tx.adopt(committed); } catch (error) { adoption = { error }; + return; } + const failure = this.#publish(committed, writes, documents, context); + adoption = failure === undefined ? { documents } : { documents, failure }; }; - let seq: Seq; try { // Once admitted, caller cancellation does not interrupt Storage settlement. - seq = await this.#storage.commit(writes, adopt, withoutAbortSignal(context)); + await this.#storage.commit(writes, adopt, withoutAbortSignal(context)); } catch (error) { - // The guard has failed the Session. A rejection after adoption leaves nothing staged to discard. + // The guard has failed the Session with the storage error. A rejection after adoption leaves nothing staged to + // discard, and the commit, durable, stays published; an own observer's error that would have failed the Session + // is still reported. if (adoption === undefined) tx.discard(); + else if ("documents" in adoption && adoption.failure !== undefined) { + this.report(adoption.failure.error, LISTENER, context); + } throw error; } if (adoption === undefined) { @@ -834,10 +917,10 @@ export class SessionImpl implements Session { this.fail(adoption.error); throw adoption.error; } - const documents = adoption.documents; if (taken.length > 0) this.#release(taken); - for (const document of documents.stored) this.#touchDocument(addressId(document.record)); - this.#publish(seq, writes, documents, context); + for (const document of adoption.documents.stored) this.#touchDocument(addressId(document.record)); + // The commit is durable and published, so it stands; memory behind an own observer's state fails the Session. + if (adoption.failure !== undefined) this.fail(adoption.failure.error, context); this.#sweepDocuments(); return result; } @@ -869,16 +952,19 @@ export class SessionImpl implements Session { this.#armHoldTimer(); const publication: ObservedCommit = { kind: "memory", changes: documents }; for (const listener of [...this.#internalListeners]) { - contained( - () => listener(publication, context), - (error) => this.fail(error, context), - ); + // Our own observers are synchronous. + try { + listener(publication, context); + } catch (error) { + this.fail(error, context); + } } } /** * Store the held changes of `holds` in one storage transaction, on the line. Memory already has them, so only - * `subscribeCommits()` listeners see it. A failure fails the Session, as every Storage failure does. + * `subscribeCommits()` listeners see it, published as Storage adopts it. A failure fails the Session, as every Storage + * failure does. */ async #storeHolds(holds: readonly Hold[], context: Context): Promise { if (holds.length === 0) return; @@ -894,27 +980,21 @@ export class SessionImpl implements Session { changes.push({ type: "document", record, conversationId, version, value, ops: stored.ops }); } let adopted = false; - const adopt = (): void => { + const adopt = (seq: Seq): void => { adopted = true; for (const [index, hold] of holds.entries()) { if (writes[index]!.content.kind === "base") hold.loaded.deltasSinceBase = 0; else hold.loaded.deltasSinceBase++; } + if (this.#commitListeners.size > 0) this.#notifyHosts({ seq, changes }, context); }; - const seq = await this.#storage.commit(writes, adopt, withoutAbortSignal(context)); + await this.#storage.commit(writes, adopt, withoutAbortSignal(context)); if (!adopted) { const error = new Error("Storage committed without adopting the batch"); this.fail(error); throw error; } this.#release(holds); - const publication: CommitPublication = { seq, changes }; - for (const listener of [...this.#commitListeners]) { - contained( - () => listener(publication, context), - (error) => this.report(error, LISTENER, context), - ); - } } /** After their store: forget `holds`, unpin their documents, and move the timer to the next deadline. */ @@ -954,25 +1034,36 @@ export class SessionImpl implements Session { timer.disarm = this.#timers.foreground(run, Math.max(0, at - this.#clock())); } - #publish(seq: Seq, writes: readonly StorageWrite[], documents: AdoptedDocuments, context: Context): void { + /** + * Inside `adopt`: publish a stored commit. Its telemetry events first, then the dispatch to `subscribeCommits()` + * listeners, queued before the waits and aborts the observers wake, then the Session's own observers. Never throws: + * returns the first error of an own observer, which fails the Session once `Storage.commit()` returned. + */ + #publish( + seq: Seq, + writes: readonly StorageWrite[], + documents: AdoptedDocuments, + context: Context, + ): { readonly error: unknown } | undefined { const changes: CommitChange[] | undefined = this.#commitListeners.size > 0 || this.#internalListeners.size > 0 ? [] : undefined; for (const write of writes) { switch (write.type) { case "entry": - context.telemetry.addEvent("pi.durable.entry", { - "entry.id": write.value.id, - "entry.kind": write.value.kind, - }); + this.#event("pi.durable.entry", { "entry.id": write.value.id, "entry.kind": write.value.kind }, context); changes?.push(write); break; case "submission": if (write.value.status === "done" || write.value.status === "unanswered") { - context.telemetry.addEvent("pi.durable.submission", { - "submission.id": write.value.id, - status: write.value.status, - "answer.entry_id": write.value.answer, - }); + this.#event( + "pi.durable.submission", + { + "submission.id": write.value.id, + status: write.value.status, + "answer.entry_id": write.value.answer, + }, + context, + ); } changes?.push(write); break; @@ -981,64 +1072,192 @@ export class SessionImpl implements Session { changes?.push(write); } } - if (changes === undefined) return; + if (changes === undefined) return undefined; const tables = changes.length; for (const document of documents.observed) changes.push(document); const observed: ObservedCommit = { kind: "stored", seq, changes }; - // One publication for both, unless the commit also stored held changes, which only durable listeners get. - const stored: CommitPublication = - documents.stored === documents.observed - ? observed - : { seq, changes: [...changes.slice(0, tables), ...documents.stored] }; - // The commit is durable: a failing listener neither fails the commit nor skips the others. - for (const listener of [...this.#internalListeners]) { - contained( - () => listener(observed, context), - (error) => this.fail(error, context), - ); + if (this.#commitListeners.size > 0) { + // One publication for both, unless the commit also stored held changes, which only host listeners get. + const stored: CommitPublication = + documents.stored === documents.observed + ? observed + : { seq, changes: [...changes.slice(0, tables), ...documents.stored] }; + this.#notifyHosts(stored, context); } - for (const listener of [...this.#commitListeners]) { - contained( - () => listener(stored, context), - (error) => this.report(error, LISTENER, context), - ); + // The commit is durable: a failing observer neither fails the commit nor skips the others. + let failure: { readonly error: unknown } | undefined; + for (const listener of [...this.#internalListeners]) { + // Our own observers are synchronous. + try { + listener(observed, context); + } catch (error) { + failure ??= { error }; + } + } + return failure; + } + + /** Inside `adopt`: record a commit's telemetry event. A throwing adapter is reported, as `adopt` must not throw. */ + #event(name: string, attributes: SpanAttributes, context: Context): void { + try { + context.telemetry.addEvent(name, attributes); + } catch (error) { + this.report(error, TELEMETRY, context); } } /** - * Attach a watch to one committed incarnation: check the definition, then forward this incarnation's committed - * changes, its retirement, and close. The watch's detach removes both subscriptions. + * Inside `adopt`: call the `subscribeCommits()` listeners with `publication` one microtask from now, outside the + * storage's synchronous step and, queued before the Session's own observers run, before the waits they resolve and + * the aborts they queue. + * The listeners are those registered when it runs; a throw is reported. */ - #attachDocument(definition: AnyDocToken["definition"], loaded: LoadedDocument): Watch { + #notifyHosts(publication: CommitPublication, context: Context): void { + void SETTLED.then(() => { + for (const listener of [...this.#commitListeners]) { + contained( + () => listener(publication, context), + (error) => this.report(error, LISTENER, context), + ); + } + }); + } + + /** + * On the line, as one job: load the document at `resolved` and `attach` to it, so the attachment sees every commit + * after the value it starts from; `undefined` when absent. Cancelling `context` meanwhile rejects, after `cancel` + * undid an attachment made. + */ + async #attachDocument( + definition: AnyDocToken["definition"], + resolved: { readonly id: string; readonly address: DocumentAddress }, + context: Context, + attach: (loaded: LoadedDocument) => A, + cancel: (attached: A) => void, + ): Promise { + const signal = context.abortSignal; + let cancelled = signal?.aborted ?? false; + const markCancelled = (): void => { + cancelled = true; + }; + signal?.addEventListener("abort", markCancelled, { once: true }); + try { + const attached = await this.#enqueue(async () => { + this.#assertHealthy(); + if (cancelled) throw cancellationError(signal!); + const loaded = await this.#loadDocument(definition, resolved.id, resolved.address, context); + if (cancelled) throw cancellationError(signal!); + return loaded === undefined ? undefined : attach(loaded); + }); + if (attached === undefined) return undefined; + if (cancelled) { + cancel(attached); + throw cancellationError(signal!); + } + return attached; + } finally { + signal?.removeEventListener("abort", markCancelled); + } + } + + /** + * Follow the address of the incarnation `loaded`, after checking the definition, with the observer `create` makes + * from its value, the idempotent detach, and the report of a listener error; returns that observer. It gets each + * committed change there, a retirement as a `retired` change with the last value it was given and no ops (a change + * in the retiring commit is not published), and a later incarnation as a root replacement. When the commit makes the + * owner task of a task document terminal, no later commit can change the address: after its changes the commit + * subscription and the pin go, and `ownerEnded` tells the observer, which stays subscribed to the Session's close + * until it ends (its detach drops that). `end` gets the Session's close. The current incarnation is pinned in the + * cache. The detach runs before `end`; once it ran, nothing more of the commit underway reaches the observer or pins + * anything for it. `create` and `ownerEnded` are module functions where they can be, so a watch keeps no closure of + * its caller. + */ + #observeDocument>>( + definition: AnyDocToken["definition"], + loaded: LoadedDocument, + create: (value: Readonly, detach: () => void, report: (error: unknown) => void) => O, + ownerEnded: (observer: O) => void, + ): O { checkRecordScope(definition, loaded.record); checkRecordVersion(definition, loaded.record, loaded.storedVersion); + // The closures below keep only the address, as the cache's ID and first record, which the cache shares while the + // document is cached: `loaded` holds its incarnation's tracker, which a retirement frees. + const addressId = loaded.addressId; + const address: DocumentAddress = loaded.record; + let incarnation: DocumentId | undefined = loaded.record.id; + // The value of the incarnation followed; none while the address is absent, so the Session keeps no retired value. + let value: Readonly | undefined = loaded.tracker.value; + // The definition version the observer's value has: a change of another version replaces it whole. + let version = loaded.valueVersion; + // The incarnation observed stays cached; once it retires or the observer detaches, it is idle again. + let unpin = this.#pinDocument(addressId, incarnation); let unsubscribeCommit = (): void => {}; let unsubscribeClose = (): void => {}; - // Attached, the incarnation stays cached; once its last observer detaches, or it retires, it is idle again. - const unpin = this.#pinDocument(loaded.addressId, loaded.record.id); + let attached = true; const detach = (): void => { + if (!attached) return; + attached = false; unsubscribeCommit(); unsubscribeClose(); unpin(); }; - const observer = new Watch(loaded.tracker.value, detach, (error) => this.report(error)); - const observed = { version: loaded.valueVersion }; + const observer = create(loaded.tracker.value, detach, (error) => this.report(error)); unsubscribeCommit = this.observeCommits((publication, context) => { + // Listeners run from a copy of the set: another observer's call may have detached this one. + if (!attached) return; + let ownerEnds = false; for (const change of publication.changes) { - if (change.type !== "document" || change.record.id !== loaded.record.id) continue; - if (change.value === null) { - // A retired incarnation never changes again: its watch delivers the retirement but pins nothing. - unpin(); - observer.retire(context); + if (change.type === "task") { + const scope = address.scope; + if ( + scope.kind === "task" && + change.value.id === scope.taskId && + change.value.state.status === "terminal" + ) { + ownerEnds = true; + } continue; } - const ops = observedOperations(observed, change); - // A migration-only base changes nothing for an observer of the new version. - if (ops.length === 0) continue; - observer.advance(change.value, ops, context); + // The incarnation followed is at the address by definition; only another record needs the address compared. + if (change.type !== "document") continue; + if (change.record.id !== incarnation && !atAddress(change.record, address)) continue; + if (change.value === null) { + if (change.record.id !== incarnation) continue; + // Set with every incarnation. + const last = value!; + incarnation = undefined; + value = undefined; + unpin(); + observer.advance({ value: last, ops: NO_OPERATIONS, retired: true }, context); + } else if (change.record.id !== incarnation || change.version !== version) { + // A new incarnation at the address, or a value of another definition version, whose shape the observer's + // value does not have: shown whole. + if (change.record.id !== incarnation) { + incarnation = change.record.id; + unpin(); + unpin = this.#pinDocument(addressId, incarnation); + } + version = change.version!; + value = change.value; + observer.advance({ value, ops: [["r", value]], retired: false }, context); + } else { + // A migration-only base changes nothing for an observer of the new version. + if (change.ops.length === 0) continue; + value = change.value; + observer.advance({ value, ops: change.ops, retired: false }, context); + } + // The observer may detach in its call, or throw and be detached: the rest of the commit is not its. + if (!attached) return; } + if (!ownerEnds) return; + unsubscribeCommit(); + unpin(); + ownerEnded(observer); + }); + unsubscribeClose = this.subscribeClose(() => { + detach(); + observer.end(this.#failure); }); - unsubscribeClose = this.subscribeClose(() => observer.end(this.#failure)); return observer; } @@ -1113,16 +1332,98 @@ export function contained(callback: () => unknown, onError: (error: unknown) => } /** - * Operations an observer applies for one committed change. An observer hydrated under another definition version holds - * a differently shaped value, so it receives the new value as a root replacement instead of operations for that shape. + * `observer`, a host's, behind the rule for observers that run inside Storage's `adopt`: one that throws is reported + * through `report`, detached, and ended with the error, so the throw never reaches `adopt`; nothing reaches it after + * its detach or end, and from then on the guard references neither it nor its source. `detach` is the source's; the + * returned `detach()` is the observation's. */ -function observedOperations( - observed: { version: number }, - change: Extract, -): readonly Op[] { - if (change.version === observed.version) return change.ops; - observed.version = change.version!; - return [["r", change.value]]; +export function guardObserver( + observer: Observer, + detach: () => void, + report: (error: unknown) => void, +): Observer & { detach(): void } { + type Attached = { + readonly observer: Observer; + readonly detach: () => void; + readonly report: (error: unknown) => void; + }; + let attached: Attached | undefined = { observer, detach, report }; + /** Detach from the source once, and hand back what it was attached with. */ + const release = (): Attached | undefined => { + const released = attached; + attached = undefined; + released?.detach(); + return released; + }; + return { + advance: (change, context) => { + const current = attached; + if (current === undefined) return; + try { + current.observer.advance(change, context); + } catch (error) { + current.report(error); + if (release() !== undefined) contained(() => current.observer.end({ error }), current.report); + } + }, + end: (failure) => { + const ending = release(); + if (ending !== undefined) contained(() => ending.observer.end(failure), ending.report); + }, + detach: () => { + release(); + }, + }; +} + +/** + * The `Observation` of a guarded observer attached at `value`. Built outside the attaching closure, so a detached + * observation keeps nothing but `value` and the spent guard alive. + */ +export function observation(value: T, guarded: { detach(): void }): Observation { + return { value, detach: () => guarded.detach() }; +} + +/** A document watch over its source's detach and report. */ +function createWatch( + value: Readonly, + detach: () => void, + report: (error: unknown) => void, +): Watch> { + return new Watch>(value, detach, report); +} + +/** Undo a watch attached while its acquisition was cancelled. */ +function cancelWatch(watch: Watch>): void { + watch.cancel(); +} + +/** Undo an observation attached while its attachment was cancelled. */ +function detachObservation(attached: Observation>): void { + attached.detach(); +} + +/** A task document's watch delivers the retirement that ends its task, then ends. */ +function endWatch(watch: Watch>): void { + watch.endAfterDelivery(); +} + +/** A task document's observer ends right after the retirement that ends its task. */ +function endGuarded(guarded: Observer>): void { + guarded.end(undefined); +} + +/** Whether `record` is an incarnation at `address`. */ +function atAddress(record: DocumentRecord, address: DocumentAddress): boolean { + if (record.kind !== address.kind || record.key !== address.key) return false; + switch (record.scope.kind) { + case "session": + return address.scope.kind === "session"; + case "conversation": + return address.scope.kind === "conversation" && address.scope.conversationId === record.scope.conversationId; + case "task": + return address.scope.kind === "task" && address.scope.taskId === record.scope.taskId; + } } function cancellationError(signal: AbortSignal): Error { diff --git a/packages/durable/src/testing/env-conformance.ts b/packages/durable/src/testing/env-conformance.ts index a2c7d43ba..1e6d1ad1a 100644 --- a/packages/durable/src/testing/env-conformance.ts +++ b/packages/durable/src/testing/env-conformance.ts @@ -2,12 +2,12 @@ import type { Context } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT, withAbortSignal } from "@earendil-works/chord/context"; import { type ExecutionEnv, + type FileChange, type FileInfo, getOrThrow, type ListFilesOptions, type Result, type ShellOutputInfo, - type WatchChange, type WatchTarget, } from "../env/index.ts"; import type { EnvConformanceCase, EnvConformanceOptions } from "./types.ts"; @@ -63,7 +63,7 @@ async function listed( } /** Whether a change reports `path`: an overflow, or a reported path at or above it. */ -function covers(change: WatchChange, path: string): boolean { +function covers(change: FileChange, path: string): boolean { if ("overflow" in change) return true; if (!("paths" in change)) return false; return change.paths.some( @@ -76,13 +76,13 @@ async function watching( env: ExecutionEnv, targets: readonly WatchTarget[], run: (helpers: { - readonly changes: WatchChange[]; + readonly changes: FileChange[]; /** Wait up to three seconds for a change, after the call, that reports `path`. */ expectChange(path: string, change: () => Promise): Promise; absolute(path: string): Promise; }) => Promise, ): Promise { - const changes: WatchChange[] = []; + const changes: FileChange[] = []; const watcher = getOrThrow(await env.watch(targets, (change) => changes.push(change), context)); try { const absolute = async (path: string) => getOrThrow(await env.absolutePath(path, context)); @@ -384,7 +384,7 @@ export function createEnvConformance(options: EnvConformanceOptions): readonly E }), watchCase("watch stops reporting once closed", async (env) => { - const changes: WatchChange[] = []; + const changes: FileChange[] = []; const watcher = getOrThrow(await env.watch([{ path: "file.txt" }], (change) => changes.push(change), context)); assert.ok(watcher.mode === "native" || watcher.mode === "polling", "watcher reports its mode"); await watcher.close(context); diff --git a/packages/durable/src/types.ts b/packages/durable/src/types.ts index 5844fccb1..97bdcb75e 100644 --- a/packages/durable/src/types.ts +++ b/packages/durable/src/types.ts @@ -1,4 +1,4 @@ -import type { Context, Draft, JsonValue, WatchHandle } from "@earendil-works/chord"; +import type { Context, Draft, JsonValue, WatchChange, WatchHandle } from "@earendil-works/chord"; import type { Op } from "@earendil-works/chord/delta"; import type { Message, Models } from "@earendil-works/pi-ai"; import type { TelemetryCarrier } from "@earendil-works/pi-telemetry"; @@ -142,8 +142,11 @@ export type ReportTask = { readonly conversationId: ConversationId; }; -/** A host callback, by its `HarnessOptions` name; `env` is the environment builder, `models` a provider call. */ -export type HostCallback = "now" | "settings" | "registry" | "env" | "models" | "setTimer" | "onWake"; +/** + * A host callback, by its `HarnessOptions` name; `env` is the environment builder, `models` a provider call, and + * `telemetry` a telemetry adapter call made while a commit is published, which must not fail it. + */ +export type HostCallback = "now" | "settings" | "registry" | "env" | "models" | "setTimer" | "onWake" | "telemetry"; /** * What a report to `HarnessOptions.onReport` is about. `extension`, where present, names the extension whose code @@ -1013,10 +1016,38 @@ export interface Tx { } /** - * Watch of one committed document incarnation. A retirement delivers `[["r", null]]` with value `null`, then ends the - * watch as `retired`; the Session's close ends it as `ended`, its failure as `failed` with the error that failed it. + * Watch of one document address (kind, scope, and key), from the incarnation found at acquisition on. A retirement is a + * call with `retired` set, the last value the watch was given and no ops (a change made in the retiring commit itself + * is not shown: that commit publishes the retirement only); the watch goes on, and a document created at the address + * later arrives as `[["r", value]]`, folded with what is pending. It ends `ended` when its scope's owner ends: a task + * document's watch once the retirement call of the commit that makes its task terminal returned, every watch at Session + * close (a pending retirement dropped); `failed` with the error that failed the Session. */ -export type DocumentWatch = WatchHandle | null>; +export type DocumentWatch = WatchHandle>; + +/** + * A synchronous observer of committed Durable state, for adapters such as a Chord source (`observeDoc()`, + * `Harness.observeConversation()`, `Harness.observeTaskGraph()`). `advance` gets every change, unfolded, as memory + * adopts it: for a stored commit inside Storage's `adopt`, for a deferred one before it is stored. A view or the graph + * gets at most one call per commit, a document observer one per change at its address (two for a commit that retires + * and recreates it); the task graph also advances outside any commit and `adopt`, on the Session line, when a registry + * change or a failed migration changes a `blocked` reason. Calls may come before the attaching promise resolves; their + * ops apply from `Observation.value`. `advance` must only record the change, synchronously and quickly, never await, + * call the Session, or throw. A throw is reported as `listener_error`, detaches the observer, and ends it with the + * error. `end` is called once, when the Session closes or fails (`failure`), or when the observed scope's owner ends (a + * task document's task, after its retirement change, inside that commit's `adopt`), and never after `detach()`. + */ +export type Observer = { + advance(change: WatchChange, context: Context): void; + end(failure: { readonly error: unknown } | undefined): void; +}; + +/** An attached `Observer`: the value at attachment, which every later `advance` starts from, and how to detach it. */ +export type Observation = { + readonly value: T; + /** Idempotent: no `advance` or `end` follows it, not even for the rest of a commit underway. */ + detach(): void; +}; /** Committed document reads. */ export type DocumentReader = Pick; @@ -1088,8 +1119,55 @@ export interface Session extends DocumentObserver { */ readonly closed: Promise; /** - * Observe stored commits synchronously after adoption, each with exactly what it stored; a commit held in memory is - * published when it is stored (`CommitPublication`). The listener must not block or call Session APIs. + * Attach a synchronous `Observer` to one document address, atomically on the Session line, as `watchDoc()` acquires + * a watch: the observation's value includes every commit before it, and `advance` gets every later one, retirements + * and later incarnations included. `undefined` when the address is absent. `context` governs only the attachment: + * cancelled while attaching, the promise rejects and nothing stays attached. + */ + observeDoc( + token: SessionDocToken, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: ConversationDocToken, + conversationId: ConversationId, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: TaskDocToken, + taskId: TaskId, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: SessionDocFamilyToken, + key: string, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: ConversationDocFamilyToken, + conversationId: ConversationId, + key: string, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: TaskDocFamilyToken, + taskId: TaskId, + key: string, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + /** + * Observe stored commits, each with exactly what it stored, except that a document the commit retires is published + * as its retirement only; a commit held in memory is published when it is stored (`CommitPublication`). Storage's + * adoption of the commit queues one dispatch, which runs a microtask later, before every wait the commit resolves + * and every abort it queues, and calls the listeners registered by then; a commit adopted while none was registered + * has no dispatch. There is no order relative to watch listeners. The listener runs synchronously and must not block + * or call Session APIs. */ subscribeCommits(listener: (publication: CommitPublication, context: Context) => void): () => void; /** @@ -1159,8 +1237,9 @@ export interface Storage { /** * Atomically persist one batch and return its sequence. Once the batch is durable, call `adopt(seq)` exactly once, * synchronously and before this storage's next commit starts, then resolve with `seq`; a commit that fails before it - * is durable never calls `adopt`. `adopt` is the caller's in-memory adoption of the batch, so memory always shows a - * complete prefix of the log; it must not throw. Once resolved, later reads through this storage observe the batch. + * is durable never calls `adopt`. `adopt` is the caller's in-memory adoption and publication of the batch, so memory + * and the Session's views always show a complete prefix of the log; it runs the Session's own observers, which call + * no Storage method, and never throws. Once resolved, later reads through this storage observe the batch. */ commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, context: Context): Promise; diff --git a/packages/durable/test/env-node-conformance.test.ts b/packages/durable/test/env-node-conformance.test.ts index 38e8de61b..5bdbe15c5 100644 --- a/packages/durable/test/env-node-conformance.test.ts +++ b/packages/durable/test/env-node-conformance.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; import { afterEach, describe, expect, it } from "vitest"; -import { getOrThrow, type WatchChange } from "../src/env/index.ts"; +import { type FileChange, getOrThrow } from "../src/env/index.ts"; import { NodeExecutionEnv } from "../src/env/node.ts"; import { registerEnvConformance } from "../src/testing/index.ts"; @@ -72,7 +72,7 @@ describe("NodeExecutionEnv watch limits", () => { }); getOrThrow(await env.remove("tree/c", { recursive: true }, context)); - const changes: WatchChange[] = []; + const changes: FileChange[] = []; const watcher = getOrThrow( await env.watch([{ path: "tree", recursive: true }], (change) => changes.push(change), context), ); diff --git a/packages/durable/test/examples/04-watches.ts b/packages/durable/test/examples/04-watches.ts index bc76fc6ec..eed3a17cc 100644 --- a/packages/durable/test/examples/04-watches.ts +++ b/packages/durable/test/examples/04-watches.ts @@ -32,7 +32,7 @@ console.log("watch baseline:", notesWatch.value); const busy = Promise.withResolvers(); const done = Promise.withResolvers(); let calls = 0; -notesWatch.start(async (value, ops, _deliveryContext) => { +notesWatch.start(async ({ value, ops }, _deliveryContext) => { console.log("watch update:", value, JSON.stringify(ops)); // The first call stays busy, as a slow consumer would, while three more commits land. if (++calls === 1) await busy.promise; diff --git a/packages/durable/test/examples/15-real-model.ts b/packages/durable/test/examples/15-real-model.ts index b5cbe1244..8adb2bf50 100644 --- a/packages/durable/test/examples/15-real-model.ts +++ b/packages/durable/test/examples/15-real-model.ts @@ -45,7 +45,7 @@ if (process.env.OPENAI_API_KEY === undefined) { process.stdout.write(text.slice(printed.length)); printed = text; }; - liveWatch.start(async (value) => { + liveWatch.start(async ({ value }) => { const block = value?.generation?.message?.content.find((content) => content.type === "text"); if (block?.type === "text") printText(block.text); }); diff --git a/packages/durable/test/examples/18-json.ts b/packages/durable/test/examples/18-json.ts index f1348cfe1..4e07a1768 100644 --- a/packages/durable/test/examples/18-json.ts +++ b/packages/durable/test/examples/18-json.ts @@ -82,14 +82,14 @@ let stop: () => Promise; if (mode === "events") { const stream = await watchEvents(harness, root.id, context); print(stream.snapshot); - stream.start(async (events) => { + stream.start(async ({ value: events }) => { for (const event of events) print(event); }); stop = () => stream.stop(); } else { const watch = await root.watch(context); print({ view: watch.value }); - watch.start(async (_value, ops) => print({ ops })); + watch.start(async ({ ops }) => print({ ops })); stop = () => watch.stop(); } diff --git a/packages/durable/test/examples/20-late-join.ts b/packages/durable/test/examples/20-late-join.ts index 122ad70ee..53c7118cf 100644 --- a/packages/durable/test/examples/20-late-join.ts +++ b/packages/durable/test/examples/20-late-join.ts @@ -7,7 +7,6 @@ import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; import { - type AgentEvent, createRegistry, defineExtension, defineTool, @@ -63,7 +62,7 @@ console.log( view.value.entries.map((entry) => entry.kind), ); console.log("view tool slot:", live.tools?.[0]?.status, JSON.stringify(live.tools?.[0]?.output)); -view.start(async (value) => { +view.start(async ({ value }) => { const slot = (value.docs["pi.live"] as LiveState).tools?.[0]; if (slot?.status === "running") console.log("view output now:", JSON.stringify(slot.output)); }); @@ -75,7 +74,7 @@ console.log( stream.snapshot.tools.map((slot) => `${slot.name} ${slot.status}`), ); let output = stream.snapshot.tools[0]?.output ?? ""; -stream.start(async (events: readonly AgentEvent[]) => { +stream.start(async ({ value: events }) => { for (const event of events) { if (event.type === "tool_execution_update" && event.output !== undefined) { output = diff --git a/packages/durable/test/examples/21-subagent-foreground.ts b/packages/durable/test/examples/21-subagent-foreground.ts index c766803fb..e9404bb3f 100644 --- a/packages/durable/test/examples/21-subagent-foreground.ts +++ b/packages/durable/test/examples/21-subagent-foreground.ts @@ -114,7 +114,7 @@ const attached = new Set(); const attach = async (id: ConversationId, indent: string): Promise => { attached.add(id); const stream = await watchEvents(harness, id, context); - stream.start(async (events) => { + stream.start(async ({ value: events }) => { for (const event of events) { print(indent, event); if (event.type !== "tool_execution_update") continue; diff --git a/packages/durable/test/examples/22-subagent-background.ts b/packages/durable/test/examples/22-subagent-background.ts index 19fba5dc5..0b3925d9e 100644 --- a/packages/durable/test/examples/22-subagent-background.ts +++ b/packages/durable/test/examples/22-subagent-background.ts @@ -14,7 +14,6 @@ import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; import { openaiProvider } from "@earendil-works/pi-ai/providers/openai"; import { - type AgentEvent, AssistantEntry, type ConversationId, configure, @@ -285,7 +284,7 @@ function contentText(content: string | readonly { type: string; text?: string }[ /** Print the main conversation's messages as they are committed. Subagents work in their own conversations. */ async function follow(harness: Harness, id: ConversationId): Promise { const stream = await watchEvents(harness, id, context); - stream.start(async (events: readonly AgentEvent[]) => { + stream.start(async ({ value: events }) => { for (const event of events) { if (event.type !== "message_end") continue; const message = event.entry.model?.[0]; diff --git a/packages/durable/test/harness-compaction.test.ts b/packages/durable/test/harness-compaction.test.ts index 61f1f1d14..0cfa7426b 100644 --- a/packages/durable/test/harness-compaction.test.ts +++ b/packages/durable/test/harness-compaction.test.ts @@ -1012,7 +1012,7 @@ describe("background threshold compaction", () => { chat.setup.settings.compaction = BACKGROUND; const events: AgentEvent[] = []; const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); chat.faux.summaries.push(failure("bad request")); @@ -1181,7 +1181,7 @@ describe("blocking threshold compaction", () => { chat.faux.summaries.push(gated(deferred(), summary(), reached)); const events: AgentEvent[] = []; const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); const input = await chat.root.submit({ type: "input", content: text("u4", 200) }, context); @@ -1572,7 +1572,7 @@ describe("compaction events and live status", () => { chat.setup.settings.retry = { enabled: true, maxRetries: 2, baseDelayMs: 60_000 }; const events: AgentEvent[] = []; const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); chat.faux.summaries.push(failure("overloaded")); @@ -2033,7 +2033,7 @@ describe("compaction edge cases", () => { await history(chat); const events: AgentEvent[] = []; const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); chat.faux.summaries.push(summary()); @@ -2052,7 +2052,7 @@ describe("compaction edge cases", () => { chat.setup.settings.compaction = BACKGROUND; const events: AgentEvent[][] = []; const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push([...batch]); }); // A new section makes the preparation commit append a system entry next to the compaction's status. @@ -2358,7 +2358,7 @@ describe("blocked compaction", () => { const chat = { harness, root, setup, faux: script(setup) }; const events: AgentEvent[] = []; const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); const inspection = await chat.harness.inspect(context); diff --git a/packages/durable/test/harness-events.test.ts b/packages/durable/test/harness-events.test.ts index 029a6a53d..194da5515 100644 --- a/packages/durable/test/harness-events.test.ts +++ b/packages/durable/test/harness-events.test.ts @@ -35,7 +35,7 @@ import { aborted, deferred } from "./task-support.ts"; async function listen(harness: Harness, conversation: Conversation) { const stream = await watchEvents(harness, conversation.id, context); const batches: AgentEvent[][] = []; - stream.start(async (events) => { + stream.start(async ({ value: events }) => { batches.push([...events]); }); return { stream, batches, events: () => batches.flat() }; @@ -259,7 +259,7 @@ describe("agent events", () => { await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: { index } }), context); } const batches: AgentEvent[][] = []; - stream.start(async (events) => { + stream.start(async ({ value: events }) => { batches.push([...events]); }); await drained(); @@ -278,7 +278,7 @@ describe("agent events", () => { await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); } const batches: AgentEvent[][] = []; - stream.start(async (events) => { + stream.start(async ({ value: events }) => { batches.push([...events]); }); await drained(); @@ -540,7 +540,7 @@ describe("agent events", () => { await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); } const batches: AgentEvent[][] = []; - stream.start(async (events) => { + stream.start(async ({ value: events }) => { batches.push([...events]); }); await drained(); diff --git a/packages/durable/test/harness-generation-recovery.test.ts b/packages/durable/test/harness-generation-recovery.test.ts index 468723ce4..f517bbe12 100644 --- a/packages/durable/test/harness-generation-recovery.test.ts +++ b/packages/durable/test/harness-generation-recovery.test.ts @@ -130,8 +130,8 @@ describe("generation recovery", () => { opened.harness.resume(); const watch = (await opened.harness.watchDoc(LiveDoc, opened.root.id, context))!; let watched: string | undefined; - watch.start(async (value) => { - watched = textOf(value?.generation?.message as Message) ?? watched; + watch.start(async ({ value }) => { + watched = textOf(value.generation?.message as Message) ?? watched; }); const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; await waitFor(() => watched !== undefined); diff --git a/packages/durable/test/harness-input-entries.test.ts b/packages/durable/test/harness-input-entries.test.ts index 019172d03..7c954e6f1 100644 --- a/packages/durable/test/harness-input-entries.test.ts +++ b/packages/durable/test/harness-input-entries.test.ts @@ -265,7 +265,7 @@ describe("data-only writes", () => { const { harness, root } = await openChat(new MemoryStorage(), setup); const events: AgentEvent[] = []; const stream = await watchEvents(harness, root.id, context); - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); const input = await root.submit({ type: "input", content: "a" }, context); diff --git a/packages/durable/test/harness-nested-tools-restart.test.ts b/packages/durable/test/harness-nested-tools-restart.test.ts index 342e103c0..68a96b854 100644 --- a/packages/durable/test/harness-nested-tools-restart.test.ts +++ b/packages/durable/test/harness-nested-tools-restart.test.ts @@ -79,7 +79,7 @@ function results(entries: Awaited>): ToolResultMes async function collectEnds(harness: Harness, conversationId: Parameters[0]) { const ends: Extract[] = []; const stream = await watchEvents(harness, conversationId, context); - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { for (const event of batch) if (event.type === "tool_execution_end") ends.push(event); }); return { ends, stop: () => stream.stop() }; diff --git a/packages/durable/test/harness-nested-tools.test.ts b/packages/durable/test/harness-nested-tools.test.ts index 5d68eb34a..d8354fcf2 100644 --- a/packages/durable/test/harness-nested-tools.test.ts +++ b/packages/durable/test/harness-nested-tools.test.ts @@ -374,7 +374,7 @@ describe("nested tool calls", () => { const { harness, root } = await openChat(new MemoryStorage(), setup); const stream = await watchEvents(harness, root.id, context); const ends: AgentEvent[] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { ends.push(...batch.filter((event) => event.type === "tool_execution_end")); }); const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); @@ -455,7 +455,7 @@ describe("nested tool calls", () => { const stream = await watchEvents(harness, root.id, context); expect(stream.snapshot.nestedTools).toEqual([]); const events: AgentEvent[] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); const submission = await root.submit({ type: "input", content: "go" }, context); @@ -833,7 +833,7 @@ describe("nested tool calls", () => { const { harness, root } = await openChat(new MemoryStorage(), setup); const stream = await watchEvents(harness, root.id, context); const tools: (string | undefined)[][] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { for (const event of batch) { if (event.type === "tool_execution_start" || event.type === "tool_execution_end") { tools.push([event.type, event.toolCallId, event.parentToolCallId]); @@ -895,7 +895,7 @@ describe("nested tool calls", () => { ["c1/2", "running"], ]); const updates: [string, unknown][] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { for (const event of batch) { if (event.type === "tool_execution_update" && event.output !== undefined) { updates.push([event.toolCallId, event.output]); @@ -990,7 +990,7 @@ describe("nested tool calls", () => { const { harness, root } = await openChat(new MemoryStorage(), setup); const stream = await watchEvents(harness, root.id, context); const ends: AgentEvent[] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { ends.push(...batch.filter((event) => event.type === "tool_execution_end")); }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); @@ -1040,7 +1040,7 @@ describe("nested tool calls", () => { const { harness, root } = await openChat(new MemoryStorage(), setup); const stream = await watchEvents(harness, root.id, context); const starts: AgentEvent[] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { starts.push(...batch.filter((event) => event.type === "tool_execution_start")); }); const submission = await root.submit({ type: "input", content: "go" }, context); diff --git a/packages/durable/test/harness-observers.test.ts b/packages/durable/test/harness-observers.test.ts new file mode 100644 index 000000000..c99e1aa3c --- /dev/null +++ b/packages/durable/test/harness-observers.test.ts @@ -0,0 +1,459 @@ +// Tests for the Harness's synchronous observers (`observeConversation()`, `observeTaskGraph()`, `observeDoc()` of a task +// document) and for task document watches, which end with their task. +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import type { WatchChange } from "@earendil-works/chord"; +import { withAbortSignal } from "@earendil-works/chord/context"; +import { + type ConversationView, + defineDoc, + defineExtension, + defineTask, + type Observer, + type TaskGraph, + type TaskId, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import type { SessionImpl } from "../src/session/session.ts"; +import { ControlledStorage, context, flush } from "./session-support.ts"; +import { completed, deferred, eventually, openTasks, settled } from "./task-support.ts"; + +type Step = { phase: "run" }; + +const Progress = defineDoc<{ lines: string[] }>({ + checkpointWhen: () => false, + kind: "test.observers.progress", + version: 1, + scope: "task", + initial: () => ({ lines: [] }), +}); + +/** A task that writes its progress document, waits for `gate`, then completes, which retires the document. */ +function writer(name: string, gate: Promise, written: () => void) { + return defineTask({ + name, + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (task, runtime, ctx) => { + await runtime.commit(async (tx) => { + (await tx.doc(Progress, task.id)).lines.push("started"); + }, ctx); + written(); + await gate; + await runtime.commit(() => completed(null), ctx); + }, + }, + abort: async () => {}, + }); +} + +/** An observer recording each call and whether it ran inside the storage's `adopt`. */ +function recorder(storage: ControlledStorage) { + const calls: Array<{ readonly advance: WatchChange; readonly adopting: boolean } | { readonly end: unknown }> = + []; + const observer: Observer = { + advance: (change) => { + calls.push({ advance: change, adopting: storage.adopting }); + }, + end: (failure) => { + calls.push({ end: failure }); + }, + }; + return { calls, observer }; +} + +/** A Harness with a writer task started in the root conversation and its progress document written. */ +async function openWriter() { + const storage = new ControlledStorage(); + const gate = deferred(); + const written = deferred(); + const Writer = writer("test.observers.writer", gate.promise, written.resolve); + const opened = await openTasks(storage, [Writer]); + const root = await opened.harness.root(context); + const id: TaskId = await root.commit( + (tx) => tx.createTask(Writer, null, { ownership: { kind: "conversation" } }), + context, + ); + opened.harness.resume(); + await written.promise; + return { ...opened, storage, root, id, finish: gate.resolve }; +} + +const collect = (() => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + return async (): Promise => { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + }; +})(); + +describe("observeConversation and observeTaskGraph", () => { + it("start at the current value and advance inside adopt with the same change a watch gets", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, []); + const root = await harness.root(context); + const view = recorder(storage); + const viewObservation = await harness.observeConversation(root.id, view.observer, context); + const graph = recorder(storage); + const graphObservation = await harness.observeTaskGraph(graph.observer, context); + expect(viewObservation.value.conversation.id).toBe(root.id); + expect(graphObservation.value).toEqual({ tasks: {} }); + const watch = await root.watch(context); + const watched: WatchChange[] = []; + watch.start(async (change) => { + watched.push(change); + }); + const entry = await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + expect(view.calls).toHaveLength(1); + const [call] = view.calls as [{ advance: WatchChange; adopting: boolean }]; + expect(call.adopting).toBe(true); + expect(call.advance.retired).toBe(false); + expect(call.advance.value.entries.at(-1)?.id).toBe(entry.id); + await flush(); + expect(watched).toEqual([call.advance]); + // No task changed: the graph got nothing. + expect(graph.calls).toEqual([]); + viewObservation.detach(); + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + expect(view.calls).toHaveLength(1); + await watch.stop(); + await harness.close(context); + expect(graph.calls).toEqual([{ end: undefined }]); + }); + + it("end with the error that failed the Harness", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, []); + const root = await harness.root(context); + const graph = recorder(storage); + await harness.observeTaskGraph(graph.observer, context); + const failure = new Error("disk gone"); + storage.failNextCommit(failure); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context)).rejects.toThrow( + "disk gone", + ); + await harness.closed; + expect(graph.calls).toEqual([{ end: { error: failure } }]); + }); + + it("report a throwing observer, detach it and end it with the error, and keep the view going", async () => { + const storage = new ControlledStorage(); + const { harness, reports, infos } = await openTasks(storage, []); + const root = await harness.root(context); + const bug = new Error("observer bug"); + const ends: unknown[] = []; + await harness.observeConversation( + root.id, + { + advance: () => { + throw bug; + }, + end: (failure) => ends.push(failure), + }, + context, + ); + const healthy = recorder(storage); + await harness.observeConversation(root.id, healthy.observer, context); + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + expect(reports).toEqual([bug]); + expect(infos).toEqual([{ type: "listener_error" }]); + expect(ends).toEqual([{ error: bug }]); + expect(healthy.calls).toHaveLength(2); + await harness.close(context); + expect(ends).toHaveLength(1); + expect(await harness.closed).toEqual({ reason: "closed" }); + }); + + it("reject, never throw, on a closed Harness", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, []); + const root = await harness.root(context); + await harness.close(context); + const view = harness.observeConversation(root.id, recorder(storage).observer, context); + const graph = harness.observeTaskGraph(recorder(storage).observer, context); + await expect(view).rejects.toThrow("Session is closed"); + await expect(graph).rejects.toThrow("Session is closed"); + }); + + it("reject an attachment cancelled while it waits on the line, and attach nothing", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, []); + const root = await harness.root(context); + const view = recorder(storage); + const graph = recorder(storage); + // A commit holds the line, so both attachments wait behind it. + const held = storage.holdCommits(); + const committing = root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + await held.entered; + const controller = new AbortController(); + const cancelled = withAbortSignal(controller.signal, context); + const attachingView = harness.observeConversation(root.id, view.observer, cancelled); + const attachingGraph = harness.observeTaskGraph(graph.observer, cancelled); + controller.abort(new Error("gone")); + held.release(); + await committing; + await expect(attachingView).rejects.toThrow("gone"); + await expect(attachingGraph).rejects.toThrow("gone"); + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + await harness.close(context); + expect(view.calls).toEqual([]); + expect(graph.calls).toEqual([]); + }); + + it("advance a graph observer with a blocked refresh, outside any commit", async () => { + const Missing = defineTask({ + name: "test.observers.missing", + version: 1, + initial: () => ({ phase: "run" }), + phases: { run: (_task, runtime, ctx) => runtime.commit(() => completed(null), ctx) }, + abort: async () => {}, + }); + const storage = new ControlledStorage(); + const { harness, registry } = await openTasks(storage, []); + const graph = recorder(storage); + const observation = await harness.observeTaskGraph(graph.observer, context); + const root = await harness.root(context); + // Scheduling is not resumed: the task stays pending, blocked while its definition is missing. + const id = await root.commit( + (tx) => tx.createTask(Missing, null, { ownership: { kind: "conversation" } }), + context, + ); + const key = String(id); + expect(graph.calls).toEqual([ + { + advance: { + value: { tasks: { [key]: expect.objectContaining({ blocked: "missing_task" }) } }, + ops: [["s", ["tasks", key], expect.objectContaining({ blocked: "missing_task" })]], + retired: false, + }, + adopting: true, + }, + ]); + registry.install(defineExtension({ name: "late", tasks: [Missing] })); + await eventually(() => graph.calls.length === 2); + // The registry change re-derives the reason on the line, with no commit and outside the storage's `adopt`. + expect(graph.calls[1]).toEqual({ + advance: { value: expect.anything(), ops: [["d", ["tasks", key, "blocked"]]], retired: false }, + adopting: false, + }); + expect((graph.calls[1] as { advance: WatchChange }).advance.value.tasks[key]?.blocked).toBeUndefined(); + harness.resume(); + await harness.waitForTask(id, context); + observation.detach(); + await harness.close(context); + }); + + it("keep no detached observer alive", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, []); + const root = await harness.root(context); + /** Attach observers only the Harness references; `detach` says whether to detach them before returning. */ + const attach = async (detach: boolean) => { + const view: Observer = { advance: () => {}, end: () => {} }; + const graph: Observer = { advance: () => {}, end: () => {} }; + const observations = [ + await harness.observeConversation(root.id, view, context), + await harness.observeTaskGraph(graph, context), + ]; + if (detach) for (const observation of observations) observation.detach(); + return { observations, view: new WeakRef(view), graph: new WeakRef(graph) }; + }; + // Control: attached observers stay, so the collection below is real. + const attached = await attach(false); + await collect(); + expect(attached.view.deref()).toBeDefined(); + expect(attached.graph.deref()).toBeDefined(); + for (const observation of attached.observations) observation.detach(); + + const detached = await attach(true); + await collect(); + expect(detached.view.deref()).toBeUndefined(); + expect(detached.graph.deref()).toBeUndefined(); + await harness.close(context); + }); +}); + +describe("task documents", () => { + it("a watch delivers the retirement of the commit that ends its task, then ends `ended`", async () => { + const { harness, id, finish } = await openWriter(); + const watch = (await harness.watchDoc(Progress, id, context))!; + expect(watch.value).toEqual({ lines: ["started"] }); + const changes: WatchChange>[] = []; + watch.start(async (change) => { + changes.push(change); + }); + finish(); + await harness.waitForTask(id, context); + expect(await watch.closed).toEqual({ reason: "ended" }); + expect(changes).toEqual([{ value: { lines: ["started"] }, ops: [], retired: true }]); + expect(changes[0]!.value).toBe(watch.value); + // The task is terminal: its address is absent for good. + expect(await harness.watchDoc(Progress, id, context)).toBeUndefined(); + await harness.close(context); + }); + + it("an observer gets the retirement, then its end, inside the adopt of the commit that ends the task", async () => { + const { harness, storage, id, finish } = await openWriter(); + const progress = recorder>(storage); + let endedAdopting: boolean | undefined; + const observer: Observer> = { + advance: progress.observer.advance, + end: (failure) => { + endedAdopting = storage.adopting; + progress.observer.end(failure); + }, + }; + await harness.observeDoc(Progress, id, observer, context); + finish(); + await harness.waitForTask(id, context); + expect(progress.calls).toEqual([ + { advance: { value: { lines: ["started"] }, ops: [], retired: true }, adopting: true }, + { end: undefined }, + ]); + expect(endedAdopting).toBe(true); + await harness.close(context); + expect(progress.calls).toHaveLength(2); + }); + + it("keeps no task document watch alive once its task ended", async () => { + const { harness, id, finish } = await openWriter(); + /** A started watch of the task's document, which only the Harness references. */ + const acquire = async (): Promise> => { + const watch = (await harness.watchDoc(Progress, id, context))!; + watch.start(async () => {}); + return new WeakRef(watch); + }; + const watch = await acquire(); + // Control: the watch stays while its task runs, so the collection below is real. + await collect(); + expect(watch.deref()).toBeDefined(); + finish(); + await harness.waitForTask(id, context); + await collect(); + expect(watch.deref()).toBeUndefined(); + // The Harness is the Session kernel; nothing stays pinned for the ended watch. + expect((harness as unknown as SessionImpl).documentCacheSizes.attached).toBe(0); + await harness.close(context); + }); + + it("a watch never started when its task ends still ends with the Harness's close, keeping nothing of it", async () => { + /** Acquire a task document's watch, never start it, end its task, and close the Harness. */ + const acquire = async () => { + const { harness, id, finish } = await openWriter(); + const watch = (await harness.watchDoc(Progress, id, context))!; + finish(); + await harness.waitForTask(id, context); + // Its retirement is pending until a start that never comes: the watch is open. + await flush(); + expect(await settled(watch.closed)).toBe(false); + await harness.close(context); + return { watch, harness: new WeakRef(harness) }; + }; + const { watch, harness } = await acquire(); + // Closed at the Harness's close, the pending retirement dropped, as for every watch. + expect(await watch.closed).toEqual({ reason: "ended" }); + expect(() => watch.start(async () => {})).toThrow("Watch is stopped"); + await collect(); + expect(harness.deref()).toBeUndefined(); + }); + + it("a watch never started when its task ends fails with the Harness", async () => { + const { harness, storage, root, id, finish } = await openWriter(); + const watch = (await harness.watchDoc(Progress, id, context))!; + finish(); + await harness.waitForTask(id, context); + const failure = new Error("disk gone"); + storage.failNextCommit(failure); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context)).rejects.toThrow( + "disk gone", + ); + expect(await watch.closed).toEqual({ reason: "failed", error: failure }); + }); + + it("a watch busy in an earlier call when its task ends fails with the Harness before its retirement call", async () => { + const { harness, storage, root, id, finish } = await openWriter(); + const watch = (await harness.watchDoc(Progress, id, context))!; + const entered = deferred(); + const release = deferred(); + const values: unknown[] = []; + watch.start(async ({ value }) => { + values.push(value); + entered.resolve(); + await release.promise; + }); + await harness.commit(async (tx) => { + (await tx.doc(Progress, id)).lines.push("more"); + }, context); + await entered.promise; + // The task ends while the listener is busy: its retirement waits behind the running call. + finish(); + await harness.waitForTask(id, context); + const failure = new Error("disk gone"); + storage.failNextCommit(failure); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context)).rejects.toThrow( + "disk gone", + ); + expect(await watch.closed).toEqual({ reason: "failed", error: failure }); + release.resolve(); + await flush(); + expect(values).toEqual([{ lines: ["started", "more"] }]); + }); + + it("a watch whose retirement call runs when its task ends ends only after that call returns", async () => { + const { harness, id, finish } = await openWriter(); + const watch = (await harness.watchDoc(Progress, id, context))!; + const entered = deferred(); + const release = deferred(); + let returned = false; + watch.start(async () => { + entered.resolve(); + await release.promise; + returned = true; + }); + // An explicit retirement first: its call runs while the task ends, and nothing more is pending. + await harness.commit((tx) => tx.retireDoc(Progress, id), context); + await entered.promise; + finish(); + await harness.waitForTask(id, context); + await flush(); + expect(await settled(watch.closed)).toBe(false); + release.resolve(); + expect(await watch.closed).toEqual({ reason: "ended" }); + expect(returned).toBe(true); + await harness.close(context); + }); + + it("hands listeners changes that keep nothing of the Harness", async () => { + /** Watch a task document until its task ends, keeping every change, then close the Harness. */ + const acquire = async () => { + const { harness, id, finish } = await openWriter(); + const watch = (await harness.watchDoc(Progress, id, context))!; + const changes: WatchChange>[] = []; + watch.start(async (change) => { + changes.push(change); + }); + await harness.commit(async (tx) => { + (await tx.doc(Progress, id)).lines.push("more"); + }, context); + await flush(); + finish(); + await harness.waitForTask(id, context); + expect(await watch.closed).toEqual({ reason: "ended" }); + await harness.close(context); + return { changes, harness: new WeakRef(harness) }; + }; + const { changes, harness } = await acquire(); + expect(changes).toHaveLength(2); + for (const change of changes) { + expect(Object.keys(change)).toEqual(["value", "ops", "retired"]); + expect(structuredClone(change)).toEqual(change); + } + await collect(); + expect(harness.deref()).toBeUndefined(); + }); +}); diff --git a/packages/durable/test/harness-storage-failure.test.ts b/packages/durable/test/harness-storage-failure.test.ts index 9169e62c7..35604a9ee 100644 --- a/packages/durable/test/harness-storage-failure.test.ts +++ b/packages/durable/test/harness-storage-failure.test.ts @@ -497,7 +497,7 @@ describe("listener and report errors", () => { ); const watch = await root.watch(context); const seen: unknown[] = []; - watch.start(async (value) => { + watch.start(async ({ value }) => { seen.push(value); }); await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); diff --git a/packages/durable/test/harness-structured-output.test.ts b/packages/durable/test/harness-structured-output.test.ts index 92ef991a5..2c3d2ba5d 100644 --- a/packages/durable/test/harness-structured-output.test.ts +++ b/packages/durable/test/harness-structured-output.test.ts @@ -165,7 +165,7 @@ describe("structured output", () => { }); const stream = await watchEvents(harness, root.id, context); const ends: NestedToolExecutionResult[] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { for (const event of batch) { if (event.type === "tool_execution_end" && event.parentToolCallId !== undefined) { ends.push(event.result as NestedToolExecutionResult); diff --git a/packages/durable/test/harness-structured.test.ts b/packages/durable/test/harness-structured.test.ts index 77fa781cc..12860f89b 100644 --- a/packages/durable/test/harness-structured.test.ts +++ b/packages/durable/test/harness-structured.test.ts @@ -1310,7 +1310,7 @@ describe("tool rounds", () => { opened = harness; const stream = await watchEvents(harness, root.id, context); const events: AgentEvent[] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); @@ -1761,7 +1761,7 @@ describe("tool rounds and events", () => { async function listen(harness: Harness, conversation: Conversation) { const stream = await watchEvents(harness, conversation.id, context); const events: AgentEvent[] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); return { stream, events }; diff --git a/packages/durable/test/harness-submissions.test.ts b/packages/durable/test/harness-submissions.test.ts index 07afaa547..43831b59c 100644 --- a/packages/durable/test/harness-submissions.test.ts +++ b/packages/durable/test/harness-submissions.test.ts @@ -18,7 +18,7 @@ import { afterEach, describe, expect, it } from "vitest"; import type { SessionImpl } from "../src/session/session.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, chatSetup, openChat, unanswered } from "./chat-support.ts"; -import { ControlledStorage, context } from "./session-support.ts"; +import { ControlledStorage, context, LateResolvingStorage } from "./session-support.ts"; import { deferred } from "./task-support.ts"; const directories = new Set(); @@ -275,4 +275,20 @@ describe("submissions", () => { expect([UserEntry.kind, AssistantEntry.kind]).toEqual(["pi.user", "pi.assistant"]); await harness.close(context); }); + + it("resumes a wait only once subscribeCommits() listeners saw the commit that settled it", async () => { + const setup = chatSetup(); + setup.faux.setResponses([fauxAssistantMessage("hi")]); + const { harness, root } = await openChat(new LateResolvingStorage(), setup); + const settled = new Set(); + harness.subscribeCommits((publication) => { + for (const change of publication.changes) { + if (change.type === "submission" && change.value.status === "done") settled.add(change.value.id); + } + }); + const submission = await root.submit({ type: "input", content: "hello" }, context); + await submission.wait(context); + expect(settled.has(submission.id)).toBe(true); + await harness.close(context); + }); }); diff --git a/packages/durable/test/harness-task-graph.test.ts b/packages/durable/test/harness-task-graph.test.ts index 85dd66cd4..375b0ec66 100644 --- a/packages/durable/test/harness-task-graph.test.ts +++ b/packages/durable/test/harness-task-graph.test.ts @@ -5,9 +5,11 @@ import type { JsonValue } from "@earendil-works/chord"; import { applyImmutable, type Op } from "@earendil-works/chord/delta"; import { type ConversationId, + type ConversationView, defineTask, type EntryId, MemoryStorage, + type Observer, type TaskGraph, type TaskId, } from "@earendil-works/pi-durable"; @@ -90,7 +92,7 @@ describe("task graph view", () => { const seen: TaskGraph[] = []; const observe = async () => { const opened = await harness.watchTaskGraph(context); - opened.start(async (value) => { + opened.start(async ({ value }) => { seen.push(value); }); return opened; @@ -215,7 +217,7 @@ describe("task graph view", () => { // Exact frames: replaying their operations from the acquisition revision gives each delivered value. let replica: JsonValue = watch.value as unknown as JsonValue; const frames: (readonly Op[])[] = []; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { replica = applyImmutable(replica, ops); expect(replica).toEqual(value); frames.push(ops); @@ -334,7 +336,7 @@ describe("task graph view", () => { const second = await harness.watchTaskGraph(context); expect(second.value).toBe(first.value); const updates: string[] = []; - first.start(async (value) => { + first.start(async ({ value }) => { updates.push(value.tasks[String(id)]?.state.status ?? "gone"); }); harness.resume(); @@ -356,4 +358,29 @@ describe("task graph view", () => { await rebuilt.stop(); await harness.close(context); }); + + it("advances with the conversation view inside the one storage commit that changes both", async () => { + const storage = new ControlledStorage(); + const { Parent, Child } = family(gatesNever()); + const { harness } = await openTasks(storage, [Parent, Child]); + const root = await harness.root(context); + const seen: string[] = []; + const record = (name: string): Observer => ({ + advance: () => { + seen.push(`${name} adopting=${storage.adopting}`); + }, + end: () => {}, + }); + const view = await harness.observeConversation(root.id, record("view"), context); + const graph = await harness.observeTaskGraph(record("graph"), context); + await root.commit(async (tx) => { + await tx.appendEntry(root.id, { kind: "note" }); + await tx.createTask(Parent, null, { ownership: { kind: "conversation" } }); + }, context); + // Both inside the one adopt, in no promised order. + expect([...seen].sort()).toEqual(["graph adopting=true", "view adopting=true"]); + view.detach(); + graph.detach(); + await harness.close(context); + }); }); diff --git a/packages/durable/test/harness-tasks.test.ts b/packages/durable/test/harness-tasks.test.ts index 23c9acec8..96cf23911 100644 --- a/packages/durable/test/harness-tasks.test.ts +++ b/packages/durable/test/harness-tasks.test.ts @@ -2,6 +2,7 @@ import type { Context } from "@earendil-works/chord"; import { withAbortSignal, withCancel } from "@earendil-works/chord/context"; import { createModels, Type } from "@earendil-works/pi-ai"; import { + type CommitPublication, type Conversation, createRegistry, defineDoc, @@ -13,12 +14,14 @@ import { MemoryStorage, type RegistryReader, type RegistrySnapshot, + type StorageWrite, type TaskId, type TaskRuntime, } from "@earendil-works/pi-durable"; +import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; import { describe, expect, it } from "vitest"; import { addHooks, addTask, addTool, user } from "./harness-support.ts"; -import { ControlledStorage, context, flush } from "./session-support.ts"; +import { ControlledStorage, context, flush, LateResolvingStorage, observeCommits } from "./session-support.ts"; import { aborted, abortedWith, completed, deferred, eventually, openTasks, settled } from "./task-support.ts"; type Step = { phase: "run" }; @@ -545,8 +548,8 @@ describe("task runtime", () => { handlerContext = ctx; absent = await runtime.watchDoc(Absent, ctx); const watch = await runtime.watchDoc(Notes, ctx); - watch!.start(async (value) => { - delivered.push(value?.text ?? "retired"); + watch!.start(async ({ value }) => { + delivered.push(value.text); }); watchClosed = watch?.closed; await runtime.commit(() => completed(null), ctx); @@ -1364,3 +1367,270 @@ describe("task close", () => { expect(storage.commits.at(-1)).toMatchObject([{ type: "task", value: { id, state: { status: "pending" } } }]); }); }); + +describe("publication at adoption", () => { + /** Both storages: one that resolves right after adopting, one that still works after it (JSONL's sidecar reclaim). */ + const storages = [ + { name: "memory", open: () => new ControlledStorage() }, + { name: "work after adopt", open: () => new LateResolvingStorage() }, + ] as const; + + /** Whether `publication` makes task `id` terminal. */ + const terminal = (publication: CommitPublication, id: TaskId): boolean => + publication.changes.some( + (change) => change.type === "task" && change.value.id === id && change.value.state.status === "terminal", + ); + + /** Whether `writes` make a task terminal. */ + const endsTask = (writes: readonly StorageWrite[]): boolean => + writes.some((write) => write.type === "task" && write.value.state.status === "terminal"); + + it("resumes task and idle waits only once subscribeCommits() listeners saw the terminal commit", async () => { + const gate = deferred(); + const Gated = gated("test.wake-gated", gate.promise); + const { harness, root } = await openRoot([Gated], { storage: new LateResolvingStorage() }); + const seen = new Set(); + harness.subscribeCommits((publication) => { + for (const change of publication.changes) { + if (change.type === "task" && change.value.state.status === "terminal") seen.add(change.value.id); + } + }); + const id = await start(root, Gated); + harness.resume(); + const task = harness.waitForTask(id, context).then(() => seen.has(id)); + const idle = harness.waitForIdle(context).then(() => seen.has(id)); + const conversationIdle = root.waitForIdle(context).then(() => seen.has(id)); + gate.resolve(); + expect(await Promise.all([task, idle, conversationIdle])).toEqual([true, true, true]); + await harness.close(context); + }); + + for (const { name, open } of storages) { + it(`runs host listeners before the waits a terminal commit resolves (${name})`, async () => { + const storage = open(); + const gate = deferred(); + const Gated = gated("test.order-gated", gate.promise); + const { harness, root } = await openRoot([Gated], { storage }); + const order: string[] = []; + const id = await start(root, Gated); + harness.subscribeCommits((publication) => { + if (terminal(publication, id)) order.push(`host adopting=${storage.adopting}`); + }); + harness.resume(); + const waited = harness.waitForTask(id, context).then(() => order.push("wait")); + gate.resolve(); + await waited; + expect(order).toEqual(["host adopting=false", "wait"]); + await harness.close(context); + }); + + it(`signals a newly marked run after host listeners saw the mark, outside the storage's adopt (${name})`, async () => { + const storage = open(); + const reached = deferred(); + const order: string[] = []; + const Signalled = oneStep("test.order-signalled", async (_task, runtime) => { + runtime.signal.addEventListener("abort", () => order.push(`abort adopting=${storage.adopting}`)); + reached.resolve(); + await aborted(runtime.signal); + }); + const { harness, root } = await openRoot([Signalled], { storage }); + const id = await start(root, Signalled); + harness.subscribeCommits((publication) => { + for (const change of publication.changes) + if (change.type === "task" && change.value.id === id && change.value.abortRequested) order.push("host"); + }); + harness.resume(); + await reached.promise; + expect(await harness.abortTask(id, context)).toBe("marked"); + expect(order).toEqual(["host", "abort adopting=false"]); + await harness.close(context); + }); + } + + it("resolves a waitForIdle() called after adoption, before the storage resolved, after host listeners", async () => { + const storage = new LateResolvingStorage(); + const gate = deferred(); + const Gated = gated("test.gap-idle", gate.promise); + const { harness, root } = await openRoot([Gated], { storage }); + const order: string[] = []; + const id = await start(root, Gated); + harness.subscribeCommits((publication) => { + if (terminal(publication, id)) order.push("host"); + }); + const gapped = deferred(); + let idle: Promise | undefined; + storage.gap = (writes) => { + if (!endsTask(writes)) return false; + idle = harness.waitForIdle(context).then(() => order.push(`idle committing=${storage.committing}`)); + gapped.resolve(); + return true; + }; + harness.resume(); + gate.resolve(); + await gapped.promise; + await idle; + expect(order).toEqual(["host", "idle committing=true"]); + await harness.close(context); + }); + + it("publishes a commit the storage rejects after adopting it, resolves its waits, then fails the Session", async () => { + const storage = new LateResolvingStorage(); + const telemetry = new InMemoryTelemetryContext(); + const gate = deferred(); + const Gated = gated("test.reject-after-adopt", gate.promise); + const { harness, root, infos } = await openRoot([Gated], { storage, telemetry }); + const id = await start(root, Gated); + const seen: string[] = []; + harness.subscribeCommits((publication) => { + if (terminal(publication, id)) seen.push("terminal"); + }); + storage.rejectAfterAdopt = endsTask; + harness.resume(); + const waited = harness.waitForTask(id, context); + gate.resolve(); + expect((await waited).state.outcome).toEqual({ status: "completed", result: null }); + expect(seen).toEqual(["terminal"]); + expect(await harness.closed).toEqual({ reason: "failed", error: new Error("reclaim failed") }); + expect(infos).toContainEqual({ type: "session_failed" }); + const active = telemetry.getMetrics().filter((metric) => metric.name === "pi.durable.tasks.active"); + expect(active.map((metric) => metric.value)).toEqual([1, -1]); + }); + + it("counts a task created by a commit the storage rejects after adopting it, and its close balances it", async () => { + const storage = new LateResolvingStorage(); + const telemetry = new InMemoryTelemetryContext(); + const Idle = oneStep("test.reject-created", async () => {}); + const { harness, root } = await openRoot([Idle], { storage, telemetry }); + storage.rejectAfterAdopt = (writes) => writes.some((write) => write.type === "task"); + await expect(start(root, Idle)).rejects.toThrow("reclaim failed"); + expect(await harness.closed).toMatchObject({ reason: "failed" }); + const active = telemetry.getMetrics().filter((metric) => metric.name === "pi.durable.tasks.active"); + expect(active.map((metric) => metric.value)).toEqual([1, -1]); + }); + + it("keeps a task wait resolved when a host listener closes the Harness on the terminal commit", async () => { + const gate = deferred(); + const Gated = gated("test.close-in-listener", gate.promise); + const { harness, root } = await openRoot([Gated]); + const id = await start(root, Gated); + let closing: Promise | undefined; + harness.subscribeCommits((publication) => { + if (terminal(publication, id)) closing = harness.close(context); + }); + harness.resume(); + const waited = harness.waitForTask(id, context); + gate.resolve(); + expect((await waited).state.outcome).toEqual({ status: "completed", result: null }); + await closing; + }); + + it("keeps a task wait resolved when the Harness closes after adoption, before the storage resolved", async () => { + const storage = new LateResolvingStorage(); + const gate = deferred(); + const Gated = gated("test.close-in-gap", gate.promise); + const { harness, root } = await openRoot([Gated], { storage }); + const id = await start(root, Gated); + const gapped = deferred(); + let closing: Promise | undefined; + storage.gap = (writes) => { + if (!endsTask(writes)) return false; + closing = harness.close(context); + gapped.resolve(); + return true; + }; + harness.resume(); + const waited = harness.waitForTask(id, context).then( + (record) => record.state.outcome, + (error: unknown) => error, + ); + gate.resolve(); + await gapped.promise; + await closing; + expect(await waited).toEqual({ status: "completed", result: null }); + expect(await harness.closed).toEqual({ reason: "closed" }); + }); + + it("fails the Session with the storage error when it rejects after adopting, and reports an observer's throw", async () => { + const storage = new LateResolvingStorage(); + const { harness, root, reports, infos } = await openRoot([], { storage }); + const bug = new Error("observer bug"); + observeCommits(harness, (publication) => { + if (publication.kind === "stored" && publication.changes.some((change) => change.type === "entry")) throw bug; + }); + storage.rejectAfterAdopt = (writes) => writes.some((write) => write.type === "entry"); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context)).rejects.toThrow( + "reclaim failed", + ); + expect(await harness.closed).toEqual({ reason: "failed", error: new Error("reclaim failed") }); + expect(reports).toContain(bug); + expect(infos[reports.indexOf(bug)]).toEqual({ type: "listener_error" }); + expect(infos).toContainEqual({ type: "session_failed" }); + }); + + it("stores a host listener's own commit before the reconcile commit that the same publication asks for", async () => { + const storage = new ControlledStorage(); + const gate = deferred(); + const Child = gated("test.order-child", gate.promise); + const Parent = oneStep("test.order-parent", async (task, runtime, ctx) => { + await runtime.commit(async (tx) => { + await tx.createTask(Child, null, { ownership: { kind: "task", taskId: task.id } }); + }, ctx); + // The child is still live, so the outcome is held as `completing` until a reconcile pass finalizes it. + await runtime.commit(() => completed(null), ctx); + }); + const { harness, root } = await openRoot([Parent, Child], { storage }); + const parent = await start(root, Parent); + harness.resume(); + await eventually(async () => (await harness.getTask(parent, context))?.state.status === "completing"); + harness.subscribeCommits((publication) => { + const child = publication.changes.find( + (change) => + change.type === "task" && change.value.id !== parent && change.value.state.status === "terminal", + ); + if (child !== undefined) + void root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + }); + const before = storage.commits.length; + gate.resolve(); + await harness.waitForTask(parent, context); + const kinds = storage.commits.slice(before).map((writes) => { + if (writes.some((write) => write.type === "entry")) return "host entry"; + const task = writes.find((write) => write.type === "task"); + if (task?.type !== "task" || task.value.state.status !== "terminal") return "other"; + return task.value.id === parent ? "parent terminal" : "child terminal"; + }); + expect(kinds.filter((kind) => kind !== "other")).toEqual(["child terminal", "host entry", "parent terminal"]); + await harness.close(context); + }); + + it("calls a host listener added on a Harness between adopt and the dispatch only when one was registered at adopt", async () => { + const storage = new ControlledStorage(); + const { harness, root } = await openRoot([], { storage }); + const note = () => root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + const late: number[] = []; + let stopLate = (): void => {}; + // No host listener at adopt, although the Harness's own observers run: no dispatch, so the late one misses it. + storage.afterAdopt = () => { + stopLate = harness.subscribeCommits((publication) => { + late.push(publication.seq); + }); + }; + await note(); + await flush(); + expect(late).toEqual([]); + await note(); + expect(late).toHaveLength(1); + stopLate(); + // One registered at adopt: the dispatch reads the listeners when it runs, the late one included. + harness.subscribeCommits(() => {}); + const second: number[] = []; + storage.afterAdopt = () => { + harness.subscribeCommits((publication) => { + second.push(publication.seq); + }); + }; + await note(); + expect(second).toHaveLength(1); + await harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-telemetry.test.ts b/packages/durable/test/harness-telemetry.test.ts index 20bf7f15f..c802568fa 100644 --- a/packages/durable/test/harness-telemetry.test.ts +++ b/packages/durable/test/harness-telemetry.test.ts @@ -11,13 +11,18 @@ import { type Storage, type TaskId, } from "@earendil-works/pi-durable"; -import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; +import { + InMemoryTelemetryContext, + NOOP_TELEMETRY_CONTEXT, + NOOP_TELEMETRY_METRICS, + type TelemetryContext, +} from "@earendil-works/pi-telemetry"; import { describe, expect, it, vi } from "vitest"; import { createCodemodeTool } from "../src/codemode/codemode.ts"; import { type ChatSetup, chatSetup, waitFor } from "./chat-support.ts"; import { addHooks, addTask, addTool } from "./harness-support.ts"; -import { ControlledStorage, context } from "./session-support.ts"; -import { aborted } from "./task-support.ts"; +import { ControlledStorage, context, flush } from "./session-support.ts"; +import { aborted, openTasks } from "./task-support.ts"; async function openTracked(storage: Storage, setup: ChatSetup, telemetry: InMemoryTelemetryContext) { const harness = await Harness.open( @@ -985,4 +990,85 @@ describe("Durable telemetry", () => { await harness.close(context); await sandbox.close(); }); + + it("reports telemetry adapters that throw while a commit is published, and the commit stands", async () => { + const Idle = defineTask({ + name: "test.telemetry-idle", + version: 1, + initial: () => ({ phase: "run" }), + phases: { run: async () => {} }, + abort: async () => {}, + }); + const throwingMetric = { + add: () => { + throw new Error("metrics down"); + }, + }; + const telemetry: TelemetryContext = { + ...NOOP_TELEMETRY_CONTEXT, + metrics: { ...NOOP_TELEMETRY_METRICS, upDownCounter: () => throwingMetric }, + }; + const { harness, infos } = await openTasks(new MemoryStorage(), [Idle], { telemetry }); + const root = await harness.root(context); + const view = await root.watch(context); + const shown: (number | undefined)[] = []; + view.start(async ({ value }) => { + shown.push(value.entries.at(-1)?.id); + }); + const hosts: number[] = []; + harness.subscribeCommits((publication) => { + hosts.push(publication.seq); + }); + const failing = withTelemetry( + { + ...NOOP_TELEMETRY_CONTEXT, + addEvent: () => { + throw new Error("telemetry down"); + }, + }, + context, + ); + const { entry, task } = await root.commit(async (tx) => { + const entry = await tx.appendEntry(root.id, { kind: "note", data: {} }); + const task = await tx.createTask(Idle, null, { ownership: { kind: "conversation" } }); + return { entry: entry.id, task }; + }, failing); + // The entry event, then the active-task metric; the report's own event throws too and is dropped. + expect(infos).toEqual([ + { type: "host_error", callback: "telemetry" }, + { type: "host_error", callback: "telemetry" }, + ]); + expect(hosts).toHaveLength(1); + await flush(); + expect(shown).toEqual([entry]); + expect((await harness.getTask(task, context))?.state.status).toBe("pending"); + await view.stop(); + await harness.close(context); + expect(await harness.closed).toEqual({ reason: "closed" }); + }); + + it("drops a report's own metric that throws, after onReport got the report", async () => { + const throwingCounter = { + add: () => { + throw new Error("metrics down"); + }, + }; + const telemetry: TelemetryContext = { + ...NOOP_TELEMETRY_CONTEXT, + metrics: { ...NOOP_TELEMETRY_METRICS, counter: () => throwingCounter }, + }; + const { harness, reports, infos } = await openTasks(new MemoryStorage(), [], { telemetry }); + const root = await harness.root(context); + const bug = new Error("listener bug"); + harness.subscribeCommits(() => { + throw bug; + }); + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + // The listener runs in the commit's dispatch microtask; a throw escaping its report would be unhandled there. + await flush(); + expect(reports).toEqual([bug]); + expect(infos).toEqual([{ type: "listener_error" }]); + await harness.close(context); + expect(await harness.closed).toEqual({ reason: "closed" }); + }); }); diff --git a/packages/durable/test/harness-view.test.ts b/packages/durable/test/harness-view.test.ts index a2214c762..f07ce2bb9 100644 --- a/packages/durable/test/harness-view.test.ts +++ b/packages/durable/test/harness-view.test.ts @@ -26,7 +26,7 @@ async function record(conversation: Conversation) { const watch = await conversation.watch(context); const initial = watch.value; const frames: Frame[] = []; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { frames.push({ value, ops }); }); return { initial, frames, stop: () => watch.stop() }; @@ -236,7 +236,7 @@ describe("conversation view", () => { await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); } const frames: Frame[] = []; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { frames.push({ value, ops }); }); await drained(); diff --git a/packages/durable/test/session-adoption.test.ts b/packages/durable/test/session-adoption.test.ts index 205c30066..f19ae976f 100644 --- a/packages/durable/test/session-adoption.test.ts +++ b/packages/durable/test/session-adoption.test.ts @@ -1,9 +1,21 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; import type { Context } from "@earendil-works/chord"; -import { defineDoc, MemoryStorage, type Seq, type StorageWrite } from "@earendil-works/pi-durable"; +import { + type CommitPublication, + defineDoc, + MemoryStorage, + type Seq, + type StorageWrite, +} from "@earendil-works/pi-durable"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { SessionImpl } from "../src/session/session.ts"; +import type { FileError, Result } from "../src/env/index.ts"; +import { NodeExecutionEnv } from "../src/env/node.ts"; +import { type ObservedCommit, SessionImpl } from "../src/session/session.ts"; import { Transaction } from "../src/session/transaction.ts"; -import { context } from "./session-support.ts"; +import { JsonlStorage } from "../src/storage/jsonl/index.ts"; +import { ControlledStorage, context, createConversation, documentChanges } from "./session-support.ts"; const CounterDoc = defineDoc<{ count: number }>({ checkpointWhen: () => false, @@ -13,8 +25,19 @@ const CounterDoc = defineDoc<{ count: number }>({ initial: () => ({ count: 0 }), }); -/** Memory storage whose next commit, once durable, misbehaves as `fault` says. */ -class FaultyStorage extends MemoryStorage { +/** A latest, initial-fork conversation document, the kind a deferred commit may hold. */ +const NoteDoc = defineDoc<{ a?: number; b?: number }>({ + checkpointWhen: () => false, + kind: "adoption.note", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({}), +}); + +/** Controlled storage that records `adopt` and its own resolution in `order`; its next commit misbehaves as `fault` says. */ +class FaultyStorage extends ControlledStorage { fault: "reject_after_adopt" | "skip_adopt" | undefined; readonly order: string[] = []; @@ -104,3 +127,238 @@ describe("Session adoption inside Storage.commit()", () => { expect(await session.closed).toMatchObject({ reason: "failed" }); }); }); + +describe("publication at adoption", () => { + it("runs its own observers inside adopt and host listeners one microtask later, outside it, before Storage resolves", async () => { + const { storage, session } = open(); + session.observeCommits(() => { + storage.order.push(`observed adopting=${storage.adopting}`); + }); + session.subscribeCommits(() => { + storage.order.push(`host adopting=${storage.adopting}`); + }); + await bump(session); + expect(storage.order).toEqual(["adopt", "observed adopting=true", "adopted", "host adopting=false", "resolve"]); + await session.close(context); + }); + + it("calls the host listeners registered when its dispatch runs, and none for a commit adopted without one", async () => { + const { storage, session } = open(); + const calls: string[] = []; + const removed = session.subscribeCommits(() => { + calls.push("removed"); + }); + session.subscribeCommits(() => { + calls.push("kept"); + }); + storage.afterAdopt = () => { + removed(); + session.subscribeCommits(() => { + calls.push("added"); + }); + }; + await bump(session); + expect(calls).toEqual(["kept", "added"]); + // A Session with no host listener at adoption queues no dispatch, so one added before Storage resolves misses it. + const bare = open(); + const late: Seq[] = []; + bare.storage.afterAdopt = () => { + bare.session.subscribeCommits((publication) => { + late.push(publication.seq); + }); + }; + await bump(bare.session); + expect(late).toEqual([]); + await bump(bare.session); + expect(late).toHaveLength(1); + await session.close(context); + await bare.session.close(context); + }); + + it("fails the Session after Storage returned for a throw of its own observers; the others and hosts got the commit", async () => { + const { storage, session } = open(); + const bug = new Error("observer bug"); + session.observeCommits(() => { + storage.order.push("observed"); + throw bug; + }); + session.observeCommits(() => { + storage.order.push("observed after"); + }); + session.subscribeCommits(() => { + storage.order.push(`host failed=${session.failure !== undefined}`); + }); + session.subscribeClose(() => { + storage.order.push("close"); + }); + await expect(bump(session)).resolves.toBeUndefined(); + // `adopted` and `resolve`: adopt returned to Storage without throwing, and the Session failed only after that. + expect(storage.order).toEqual([ + "adopt", + "observed", + "observed after", + "adopted", + "host failed=false", + "resolve", + "close", + ]); + expect(await session.closed).toEqual({ reason: "failed", error: bug }); + const reopened = new SessionImpl(storage.reopen()); + expect(await reopened.snapshot(CounterDoc, context)).toEqual({ count: 1 }); + await reopened.close(context); + }); + + it("publishes a commit that Storage rejects after adopting it, then fails the Session", async () => { + const { storage, session } = open(); + await bump(session); + const stored: Seq[] = []; + session.observeCommits(() => { + storage.order.push("observed"); + }); + session.subscribeCommits((publication) => { + storage.order.push("host"); + stored.push(publication.seq); + }); + storage.order.length = 0; + storage.fault = "reject_after_adopt"; + await expect(bump(session)).rejects.toThrow("reclaim failed"); + expect(storage.order).toEqual(["adopt", "observed", "adopted", "host", "resolve"]); + expect(stored).toHaveLength(1); + expect(await session.closed).toMatchObject({ reason: "failed" }); + const reopened = new SessionImpl(storage.reopen()); + expect(await reopened.snapshot(CounterDoc, context)).toEqual({ count: 2 }); + await reopened.close(context); + }); + + it("keeps what host listeners get: exactly what was stored, held changes included", async () => { + const { storage, session } = open(); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + await tx.doc(NoteDoc, conversationId); + }, context); + const observed: ObservedCommit[] = []; + const stored: CommitPublication[] = []; + session.observeCommits((publication) => { + storage.order.push("observed"); + observed.push(publication); + }); + session.subscribeCommits((publication) => { + storage.order.push("host"); + stored.push(publication); + }); + storage.order.length = 0; + await session.commit( + async (tx) => { + (await tx.doc(NoteDoc, conversationId)).a = 1; + }, + context, + { persistence: { kind: "deferred", withinMs: Infinity } }, + ); + // Held in memory: observed at once, nothing stored. + expect(storage.order).toEqual(["observed"]); + await session.commit(async (tx) => { + (await tx.doc(NoteDoc, conversationId)).b = 2; + }, context); + expect(storage.order).toEqual(["observed", "adopt", "observed", "adopted", "host", "resolve"]); + expect(documentChanges(observed[1]!).map((change) => change.ops)).toEqual([[["s", ["b"], 2]]]); + expect(stored).toHaveLength(1); + expect(documentChanges(stored[0]!).map((change) => change.ops)).toEqual([ + [ + ["s", ["a"], 1], + ["s", ["b"], 2], + ], + ]); + expect(stored[0]!.seq).toBe((observed[1] as Extract).seq); + await session.close(context); + }); + + it("publishes a store of held changes to host listeners one microtask after its adopt", async () => { + const { storage, session } = open(); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + await tx.doc(NoteDoc, conversationId); + }, context); + session.subscribeCommits(() => { + storage.order.push(`host adopting=${storage.adopting}`); + }); + await session.commit( + async (tx) => { + (await tx.doc(NoteDoc, conversationId)).a = 1; + }, + context, + { persistence: { kind: "deferred", withinMs: Infinity } }, + ); + storage.order.length = 0; + await session.unloadDocuments(); + expect(storage.order).toEqual(["adopt", "adopted", "host adopting=false", "resolve"]); + await session.close(context); + }); + + it("publishes a JSONL commit before the storage reclaims its sidecar and resolves", async () => { + /** Node environment that records each sidecar rename, the last step of a reclaim. */ + class RenameLog extends NodeExecutionEnv { + readonly order: string[] = []; + + override renameFile( + source: string, + destination: string, + renameContext: Context, + ): Promise> { + this.order.push(`rename ${basename(destination)}`); + return super.renameFile(source, destination, renameContext); + } + } + // A base at the second count makes JSONL rewrite the document's sidecar after adopting the commit. + const Rebased = defineDoc<{ count: number }>({ + checkpointWhen: (value) => value.count === 2, + kind: "adoption.rebased", + version: 1, + scope: "session", + initial: () => ({ count: 0 }), + }); + const directory = await mkdtemp(join(tmpdir(), "pi-durable-adoption-")); + const env = new RenameLog({ cwd: directory }); + const session = new SessionImpl(await JsonlStorage.open(directory, env, context)); + const count = () => + session.commit(async (tx) => { + (await tx.doc(Rebased)).count++; + }, context); + try { + await count(); + session.subscribeCommits(() => { + env.order.push("host"); + }); + await count(); + env.order.push("resolved"); + expect(env.order).toEqual(["host", expect.stringMatching(/^rename doc-\d+\.jsonl$/), "resolved"]); + } finally { + await session.close(context); + await rm(directory, { recursive: true, force: true }); + } + }); + + it("publishes a store of held changes that Storage rejects after adopting it, then fails the Session", async () => { + const { storage, session } = open(); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + await tx.doc(NoteDoc, conversationId); + }, context); + const stored: (readonly unknown[])[] = []; + session.subscribeCommits((publication) => { + stored.push(documentChanges(publication).map((change) => change.ops)); + }); + await session.commit( + async (tx) => { + (await tx.doc(NoteDoc, conversationId)).a = 1; + }, + context, + { persistence: { kind: "deferred", withinMs: Infinity } }, + ); + storage.order.length = 0; + storage.fault = "reject_after_adopt"; + await expect(session.unloadDocuments()).rejects.toThrow("reclaim failed"); + expect(storage.order).toEqual(["adopt", "adopted", "resolve"]); + expect(stored).toEqual([[[["s", ["a"], 1]]]]); + expect(await session.closed).toMatchObject({ reason: "failed", error: new Error("reclaim failed") }); + }); +}); diff --git a/packages/durable/test/session-checkpoints-migrations.test.ts b/packages/durable/test/session-checkpoints-migrations.test.ts index 98bb062ab..f5433f1a5 100644 --- a/packages/durable/test/session-checkpoints-migrations.test.ts +++ b/packages/durable/test/session-checkpoints-migrations.test.ts @@ -402,7 +402,7 @@ describe("Session document migrations", () => { // An observer of the older shape. const watch = (await session.watchDoc(Old, context))!; const frames: unknown[] = []; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { frames.push({ value, ops }); }); expect(await session.snapshot(Current, context)).toEqual({ count: 4 }); @@ -410,7 +410,7 @@ describe("Session document migrations", () => { // An observer of the new shape: the migration changes nothing it sees. const current = (await session.watchDoc(Current, context))!; const currentFrames: unknown[] = []; - current.start(async (value) => { + current.start(async ({ value }) => { currentFrames.push(value); }); @@ -836,7 +836,7 @@ describe("Session tracker cache across definition versions", () => { const watch = (await session.watchDoc(V1Doc, context))!; const frames: (readonly unknown[])[] = []; const delivered = new Promise((resolve) => { - watch.start(async (_value, ops) => { + watch.start(async ({ ops }) => { frames.push(ops); resolve(); }); diff --git a/packages/durable/test/session-deferred-commits.test.ts b/packages/durable/test/session-deferred-commits.test.ts index e4281c5e8..bbf72d002 100644 --- a/packages/durable/test/session-deferred-commits.test.ts +++ b/packages/durable/test/session-deferred-commits.test.ts @@ -562,8 +562,8 @@ describe("publications", () => { const { session, conversationId, observed, fire } = await open(); const watch = (await session.watchDoc(TextDoc, conversationId, context))!; const values: string[] = []; - watch.start(async (value) => { - values.push(value?.text ?? ""); + watch.start(async ({ value }) => { + values.push(value.text); }); const from = observed.length; for (const text of ["a", "b", "c"]) { diff --git a/packages/durable/test/session-document-retention.test.ts b/packages/durable/test/session-document-retention.test.ts index 6e2eed770..c2cec11c8 100644 --- a/packages/durable/test/session-document-retention.test.ts +++ b/packages/durable/test/session-document-retention.test.ts @@ -155,14 +155,14 @@ describe("document cache retention", () => { const kept = (await session.watchDoc(CounterDoc, conversationId, context))!; kept.start(async () => {}); const seen: number[] = []; - watch.start(async (value) => { - seen.push(value?.count ?? -1); + watch.start(async ({ value }) => { + seen.push(value.count); }); clock.now += 1_000_000; await bump(session, conversationId); await settleLine(session); // Still cached: a read long after its retention passed loads nothing from Storage. - await eventually(() => seen.includes(2) && kept.value?.count === 2); + await eventually(() => seen.includes(2) && kept.value.count === 2); const reads = storage.documentReadCount; clock.now += 1_000_000; await session.commit(async (tx) => { @@ -175,7 +175,7 @@ describe("document cache retention", () => { clock.now += 1_000_000; await bump(session, conversationId); await settleLine(session); - await eventually(() => kept.value?.count === 3); + await eventually(() => kept.value.count === 3); await kept.stop(); clock.now += 1_000_000; await session.commit(async () => {}, context); @@ -350,22 +350,27 @@ describe("document cache retention", () => { await session.close(context); }); - it("does not keep a replacement cached for an observer of the retired incarnation", async () => { + it("pins only what is at the address a watch follows: nothing while retired, then the new incarnation", async () => { const { session, sizes } = open(0); const conversationId = await createConversation(session); await bump(session, conversationId); const watch = (await session.watchDoc(CounterDoc, conversationId, context))!; watch.start(async () => {}); await session.commit((tx) => tx.retireDoc(CounterDoc, conversationId), context); - expect(await watch.closed).toEqual({ reason: "retired" }); - expect(watch.value).toBeNull(); - await bump(session, conversationId, 5); await eventually(async () => { await settleLine(session); return sizes().cached === 0; }); - expect(await session.snapshot(CounterDoc, conversationId, context)).toEqual({ count: 5 }); + expect(sizes().attached).toBe(0); + await bump(session, conversationId, 5); + await settleLine(session); + expect(sizes()).toMatchObject({ cached: 1, attached: 1 }); + await eventually(async () => watch.value.count === 5); await watch.stop(); + await eventually(async () => { + await settleLine(session); + return sizes().cached === 0; + }); await session.close(context); }); @@ -531,7 +536,7 @@ describe("document cache retention", () => { const { harness, root } = await openChat(new MemoryStorage(), setup); const stream = await watchEvents(harness, root.id, context); const events: AgentEvent[] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); diff --git a/packages/durable/test/session-observers.test.ts b/packages/durable/test/session-observers.test.ts new file mode 100644 index 000000000..c0d854bb9 --- /dev/null +++ b/packages/durable/test/session-observers.test.ts @@ -0,0 +1,411 @@ +// Tests for `Session.observeDoc()`, the synchronous observer of one document address that watches are built on. +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import type { WatchChange } from "@earendil-works/chord"; +import { withAbortSignal } from "@earendil-works/chord/context"; +import { + type ConversationId, + defineDoc, + MemoryStorage, + type Observation, + type Observer, + type ReportInfo, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import { SessionImpl } from "../src/session/session.ts"; +import { ControlledStorage, context, createConversation, flush } from "./session-support.ts"; + +type Counter = { count: number }; + +const SessionCounter = defineDoc({ + checkpointWhen: () => false, + kind: "observe.session-counter", + version: 1, + scope: "session", + initial: () => ({ count: 0 }), +}); + +/** A latest, initial-fork conversation document, the kind a deferred commit may hold. */ +const ConversationCounter = defineDoc({ + checkpointWhen: () => false, + kind: "observe.conversation-counter", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ count: 0 }), +}); + +/** A Session that keeps its reports, which a plain Session drops. */ +class ReportingSession extends SessionImpl { + readonly reports: { readonly error: unknown; readonly info: ReportInfo }[] = []; + + override report(error: unknown, info: ReportInfo = { type: "listener_error" }): void { + this.reports.push({ error, info }); + } +} + +/** An observer that records each call, and whether it ran inside the storage's `adopt`. */ +function recorder(storage: ControlledStorage) { + const calls: Array< + | { readonly advance: WatchChange>; readonly adopting: boolean } + | { readonly end: { readonly error: unknown } | undefined } + > = []; + const observer: Observer> = { + advance: (change) => { + calls.push({ advance: change, adopting: storage.adopting }); + }, + end: (failure) => { + calls.push({ end: failure }); + }, + }; + return { calls, observer }; +} + +function open() { + const storage = new ControlledStorage(); + const session = new ReportingSession(storage); + return { storage, session }; +} + +const add = (session: SessionImpl, by = 1) => + session.commit(async (tx) => { + (await tx.doc(SessionCounter)).count += by; + }, context); + +describe("observeDoc", () => { + it("returns undefined for an absent address and attaches nothing", async () => { + const { storage, session } = open(); + const { calls, observer } = recorder(storage); + expect(await session.observeDoc(SessionCounter, observer, context)).toBeUndefined(); + await add(session); + expect(calls).toEqual([]); + expect(session.documentCacheSizes.attached).toBe(0); + await session.close(context); + }); + + it("starts at the committed value and advances inside adopt with each stored commit's ops", async () => { + const { storage, session } = open(); + await add(session); + const { calls, observer } = recorder(storage); + const observation = (await session.observeDoc(SessionCounter, observer, context))!; + expect(observation.value).toEqual({ count: 1 }); + const committing = add(session, 2); + await committing; + expect(calls).toEqual([ + { advance: { value: { count: 3 }, ops: [["s", ["count"], 3]], retired: false }, adopting: true }, + ]); + observation.detach(); + await session.close(context); + }); + + it("advances when memory adopts a held commit, before anything is stored", async () => { + const { storage, session } = open(); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + await tx.doc(ConversationCounter, conversationId); + }, context); + const { calls, observer } = recorder(storage); + const observation = (await session.observeDoc(ConversationCounter, conversationId, observer, context))!; + const stored = storage.commits.length; + await session.commit( + async (tx) => { + (await tx.doc(ConversationCounter, conversationId)).count = 5; + }, + context, + { persistence: { kind: "deferred", withinMs: Number.POSITIVE_INFINITY } }, + ); + expect(storage.commits).toHaveLength(stored); + expect(calls).toEqual([ + { advance: { value: { count: 5 }, ops: [["s", ["count"], 5]], retired: false }, adopting: false }, + ]); + observation.detach(); + await session.close(context); + }); + + it("follows a conversation document's address: a retirement with the last value, then the new incarnation", async () => { + const { storage, session } = open(); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + (await tx.doc(ConversationCounter, conversationId)).count = 2; + }, context); + const { calls, observer } = recorder(storage); + const observation = (await session.observeDoc(ConversationCounter, conversationId, observer, context))!; + await session.commit((tx) => tx.retireDoc(ConversationCounter, conversationId), context); + // The address is absent now: a new attachment finds nothing, the existing one goes on. + expect(await session.observeDoc(ConversationCounter, conversationId, recorder(storage).observer, context)).toBe( + undefined, + ); + await session.commit(async (tx) => { + (await tx.doc(ConversationCounter, conversationId)).count = 9; + }, context); + expect(calls).toEqual([ + { advance: { value: observation.value, ops: [], retired: true }, adopting: true }, + { advance: { value: { count: 9 }, ops: [["r", { count: 9 }]], retired: false }, adopting: true }, + ]); + expect((calls[0] as { advance: WatchChange }).advance.value).toBe(observation.value); + observation.detach(); + await session.close(context); + }); + + it("gets nothing after detach, which is idempotent and unpins the document", async () => { + const { storage, session } = open(); + await add(session); + const { calls, observer } = recorder(storage); + const observation = (await session.observeDoc(SessionCounter, observer, context))!; + expect(session.documentCacheSizes.attached).toBe(1); + observation.detach(); + observation.detach(); + expect(session.documentCacheSizes.attached).toBe(0); + await add(session); + await session.close(context); + expect(calls).toEqual([]); + }); + + it("ends once when the Session closes", async () => { + const { storage, session } = open(); + await add(session); + const { calls, observer } = recorder(storage); + await session.observeDoc(SessionCounter, observer, context); + await session.close(context); + expect(calls).toEqual([{ end: undefined }]); + }); + + it("ends with the error that failed the Session", async () => { + const { storage, session } = open(); + await add(session); + const { calls, observer } = recorder(storage); + await session.observeDoc(SessionCounter, observer, context); + const failure = new Error("disk gone"); + storage.failNextCommit(failure); + await expect(add(session)).rejects.toThrow("disk gone"); + await session.closed; + expect(calls).toEqual([{ end: { error: failure } }]); + }); + + it("reports a throwing observer, detaches it and ends it with the error; the commit and other observers go on", async () => { + const { storage, session } = open(); + await add(session); + const bug = new Error("observer bug"); + const ends: unknown[] = []; + const throwing: Observer> = { + advance: () => { + throw bug; + }, + end: (failure) => ends.push(failure), + }; + await session.observeDoc(SessionCounter, throwing, context); + const healthy = recorder(storage); + await session.observeDoc(SessionCounter, healthy.observer, context); + await add(session); + await add(session); + expect(session.reports).toEqual([{ error: bug, info: { type: "listener_error" } }]); + expect(ends).toEqual([{ error: bug }]); + expect(healthy.calls).toHaveLength(2); + expect(await session.snapshot(SessionCounter, context)).toEqual({ count: 3 }); + await session.close(context); + // Detached at its throw: the close does not end it again. + expect(ends).toHaveLength(1); + expect(session.failure).toBeUndefined(); + }); + + it("attaches atomically on the line: a commit queued before the attachment is in its value", async () => { + const { storage, session } = open(); + await add(session); + const { calls, observer } = recorder(storage); + const committing = add(session, 10); + const observation = (await session.observeDoc(SessionCounter, observer, context))!; + await committing; + expect(observation.value).toEqual({ count: 11 }); + expect(calls).toEqual([]); + observation.detach(); + await session.close(context); + }); + + it("rejects an attachment whose context is cancelled, and attaches nothing", async () => { + const { storage, session } = open(); + await add(session); + const controller = new AbortController(); + controller.abort(new Error("gone")); + const cancelled = withAbortSignal(controller.signal, context); + await expect(session.observeDoc(SessionCounter, recorder(storage).observer, cancelled)).rejects.toThrow("gone"); + expect(session.documentCacheSizes.attached).toBe(0); + await session.close(context); + }); + + it("keeps no detached observer alive", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const collect = async (): Promise => { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + }; + const session = new SessionImpl(new MemoryStorage()); + await add(session); + /** Attach an observer only the Session references; `detach` says whether to detach it before returning. */ + const attach = async (detach: boolean) => { + const observer: Observer> = { advance: () => {}, end: () => {} }; + const observation = (await session.observeDoc(SessionCounter, observer, context))!; + if (detach) observation.detach(); + return { observation, observer: new WeakRef(observer) }; + }; + // Control: an attached observer stays, so the collection below is real. + const attached = await attach(false); + await collect(); + expect(attached.observer.deref()).toBeDefined(); + attached.observation.detach(); + + const detached = await attach(true); + await collect(); + expect(detached.observer.deref()).toBeUndefined(); + await flush(); + await session.close(context); + }); + + it("rejects, never throws, on a closed Session or a missing scope ID", async () => { + const { storage, session } = open(); + const missing = session.observeDoc( + ConversationCounter, + undefined as unknown as ConversationId, + recorder(storage).observer, + context, + ); + await expect(missing).rejects.toThrow(TypeError); + await session.close(context); + const closed = session.observeDoc(SessionCounter, recorder(storage).observer, context); + await expect(closed).rejects.toThrow("Session is closed"); + }); + it("keeps no value of a document retired for good, while still attached", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const collect = async (): Promise => { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + }; + const session = new SessionImpl(new MemoryStorage()); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + await tx.doc(ConversationCounter, conversationId); + }, context); + /** Attach an observer that keeps nothing, change the document, and retire it: only a WeakRef to its last value. */ + const retire = async () => { + let last: WeakRef | undefined; + const observation = (await session.observeDoc( + ConversationCounter, + conversationId, + { + advance: (change) => { + if (!change.retired) last = new WeakRef(change.value); + }, + end: () => {}, + }, + context, + ))!; + await session.commit(async (tx) => { + (await tx.doc(ConversationCounter, conversationId)).count = 7; + }, context); + await session.commit((tx) => tx.retireDoc(ConversationCounter, conversationId), context); + await session.unloadDocuments(); + return { observation, last: last! }; + }; + const { observation, last } = await retire(); + await collect(); + expect(last.deref()).toBeUndefined(); + observation.detach(); + await session.close(context); + }); +}); + +describe("observeDoc detached during a commit", () => { + /** A Session with a conversation counter at 1, and the commit that retires it and creates it again at 2. */ + async function retiring() { + const { storage, session } = open(); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + (await tx.doc(ConversationCounter, conversationId)).count = 1; + }, context); + const recreate = () => + session.commit(async (tx) => { + await tx.retireDoc(ConversationCounter, conversationId); + (await tx.doc(ConversationCounter, conversationId)).count = 2; + }, context); + return { storage, session, conversationId, recreate }; + } + + it("gets nothing more of the commit once it throws on the retirement, and pins nothing for the recreation", async () => { + const { session, conversationId, recreate } = await retiring(); + const bug = new Error("observer bug"); + const changes: WatchChange>[] = []; + const ends: unknown[] = []; + const observation = (await session.observeDoc( + ConversationCounter, + conversationId, + { + advance: (change) => { + changes.push(change); + throw bug; + }, + end: (failure) => ends.push(failure), + }, + context, + ))!; + await recreate(); + expect(changes).toEqual([{ value: observation.value, ops: [], retired: true }]); + expect(ends).toEqual([{ error: bug }]); + expect(session.reports).toEqual([{ error: bug, info: { type: "listener_error" } }]); + expect(session.documentCacheSizes.attached).toBe(0); + observation.detach(); + expect(session.documentCacheSizes.attached).toBe(0); + await session.close(context); + }); + + it("gets nothing more of the commit once it detaches in the retirement call, and pins nothing", async () => { + const { session, conversationId, recreate } = await retiring(); + const changes: WatchChange>[] = []; + const ends: unknown[] = []; + let observation: Observation> | undefined; + observation = await session.observeDoc( + ConversationCounter, + conversationId, + { + advance: (change) => { + changes.push(change); + observation?.detach(); + }, + end: (failure) => ends.push(failure), + }, + context, + ); + await recreate(); + expect(changes).toEqual([{ value: observation!.value, ops: [], retired: true }]); + expect(session.documentCacheSizes.attached).toBe(0); + await session.close(context); + expect(ends).toEqual([]); + }); + + it("gets nothing of the commit when another observer's call detached it first, and pins nothing", async () => { + const { storage, session, conversationId, recreate } = await retiring(); + let second: Observation> | undefined; + const first = (await session.observeDoc( + ConversationCounter, + conversationId, + { + advance: () => second?.detach(), + end: () => {}, + }, + context, + ))!; + const detached = recorder(storage); + second = await session.observeDoc(ConversationCounter, conversationId, detached.observer, context); + await recreate(); + expect(detached.calls).toEqual([]); + // Only the first observer pins the new incarnation; its detach leaves nothing pinned. + first.detach(); + expect(session.documentCacheSizes.attached).toBe(0); + await session.close(context); + }); +}); diff --git a/packages/durable/test/session-support.ts b/packages/durable/test/session-support.ts index e3bd17d44..98e3a54ce 100644 --- a/packages/durable/test/session-support.ts +++ b/packages/durable/test/session-support.ts @@ -43,6 +43,10 @@ export class ControlledStorage extends MemoryStorage { readonly commits: (readonly StorageWrite[])[] = []; mintCount = 0; documentReadCount = 0; + /** Set while `adopt` runs, the storage's synchronous step: code that sees it set runs inside `Storage.commit()`'s adopt. */ + adopting = false; + /** Runs once, in the next commit's synchronous step right after `adopt` returned. */ + afterAdopt: (() => void) | undefined; #commitGate: { gate: Deferred; entered: Deferred } | undefined; #findGate: { gate: Deferred; entered: Deferred } | undefined; #commitFailure: Error | undefined; @@ -97,7 +101,18 @@ export class ControlledStorage extends MemoryStorage { this.#commitFailure = undefined; throw failure; } - return super.commit(writes, adopt, commitContext); + const step = (seq: Seq): void => { + this.adopting = true; + try { + adopt(seq); + } finally { + this.adopting = false; + } + const after = this.afterAdopt; + this.afterAdopt = undefined; + after?.(); + }; + return super.commit(writes, step, commitContext); } override mintId>(): Promise { @@ -124,6 +139,39 @@ export class ControlledStorage extends MemoryStorage { } } +/** + * Controlled storage whose commits resolve a macrotask after they adopted the batch, as a storage with work after + * durability does (JSONL reclaims sidecars): code woken inside `adopt` runs before the commit resolves. `gap` runs in + * that work, a macrotask after adoption, and `rejectAfterAdopt` makes a commit it matches reject there, durable and + * adopted, as a failed reclaim does; each fires once. + */ +export class LateResolvingStorage extends ControlledStorage { + /** Set from a commit's start until it resolves. */ + committing = false; + gap: ((writes: readonly StorageWrite[]) => boolean) | undefined; + rejectAfterAdopt: ((writes: readonly StorageWrite[]) => boolean) | undefined; + + override async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, commitContext: Context) { + this.committing = true; + try { + const seq = await super.commit(writes, adopt, commitContext); + await flush(); + if (this.gap?.(writes) === true) { + this.gap = undefined; + // What the gap started runs before the commit resolves. + await flush(); + } + if (this.rejectAfterAdopt?.(writes) === true) { + this.rejectAfterAdopt = undefined; + throw new Error("reclaim failed"); + } + return seq; + } finally { + this.committing = false; + } + } +} + /** Session kernel plus its controlled storage and every committed publication. */ export function openTestSession(): { readonly storage: ControlledStorage; diff --git a/packages/durable/test/session-watches.test.ts b/packages/durable/test/session-watches.test.ts index 79a7f5b39..cb06268a7 100644 --- a/packages/durable/test/session-watches.test.ts +++ b/packages/durable/test/session-watches.test.ts @@ -1,6 +1,6 @@ import { setFlagsFromString } from "node:v8"; import { runInNewContext } from "node:vm"; -import type { Context } from "@earendil-works/chord"; +import type { Context, WatchChange, WatchEnd } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT, createContextKey, @@ -9,10 +9,10 @@ import { withContextValue, } from "@earendil-works/chord/context"; import { applyImmutable } from "@earendil-works/chord/delta"; -import { type DocumentWatch, defineDoc, MemoryStorage } from "@earendil-works/pi-durable"; +import { type DocumentWatch, defineDoc, defineDocFamily, MemoryStorage } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; import { SessionImpl } from "../src/session/session.ts"; -import { context, documentChanges, flush, openTestSession } from "./session-support.ts"; +import { context, createConversation, documentChanges, flush, openTestSession } from "./session-support.ts"; type State = { value: number; items: string[]; retained: { label: string } }; @@ -44,6 +44,15 @@ function deferred(): Deferred { return { promise, resolve }; } +/** The end of `closed`, or undefined while the watch is open. */ +function endOf(watch: { readonly closed: Promise }): { readonly end: WatchEnd | undefined } { + const settled: { end: WatchEnd | undefined } = { end: undefined }; + void watch.closed.then((end) => { + settled.end = end; + }); + return settled; +} + async function createState(): Promise> { const harness = openTestSession(); await harness.session.commit((tx) => tx.doc(StateDoc).then(() => undefined), context); @@ -76,8 +85,8 @@ describe("Session document watches", () => { expect(watch.value).toBe(initial); let inline = true; - const deliveries: Array<{ value: Readonly | null; ops: unknown }> = []; - watch.start(async (value, ops) => { + const deliveries: Array<{ value: Readonly; ops: unknown }> = []; + watch.start(async ({ value, ops }) => { expect(inline).toBe(false); expect(watch.value).toBe(value); deliveries.push({ value, ops }); @@ -112,10 +121,10 @@ describe("Session document watches", () => { const batches: unknown[] = []; let active = 0; let maxActive = 0; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { active++; maxActive = Math.max(maxActive, active); - values.push(value?.value ?? -1); + values.push(value.value); batches.push(ops); if (values.length === 1) { entered.resolve(); @@ -146,8 +155,8 @@ describe("Session document watches", () => { const watch = (await session.watchDoc(StateDoc, context))!; const completed = deferred(); const values: number[] = []; - watch.start(async (value) => { - const current = value?.value ?? -1; + watch.start(async ({ value }) => { + const current = value.value; values.push(current); if (current === 1) { await session.commit(async (tx) => { @@ -174,8 +183,8 @@ describe("Session document watches", () => { }, context); } const deliveries: Array<{ value: number; ops: unknown }> = []; - watch.start(async (value, ops) => { - deliveries.push({ value: value?.value ?? -1, ops }); + watch.start(async ({ value, ops }) => { + deliveries.push({ value: value.value, ops }); }); await flush(); expect(deliveries).toEqual([{ value: 101, ops: [["s", ["value"], 101]] }]); @@ -197,13 +206,13 @@ describe("Session document watches", () => { for (let index = from; index < to; index++) state.keys[`k${index}`] = index; }, context); } - const deliveries: Array<{ value: Readonly | null; ops: unknown }> = []; - watch.start(async (value, ops) => { + const deliveries: Array<{ value: Readonly; ops: unknown }> = []; + watch.start(async ({ value, ops }) => { deliveries.push({ value, ops }); }); await flush(); expect(deliveries).toEqual([{ value: watch.value, ops: [["r", watch.value]] }]); - expect(Object.keys(watch.value?.keys ?? {})).toHaveLength(10_001); + expect(Object.keys(watch.value.keys)).toHaveLength(10_001); await watch.stop(); }); @@ -213,8 +222,8 @@ describe("Session document watches", () => { const entered = deferred(); const release = deferred(); const deliveries: Array<{ value: number; ops: unknown }> = []; - watch.start(async (value, ops) => { - deliveries.push({ value: value?.value ?? -1, ops }); + watch.start(async ({ value, ops }) => { + deliveries.push({ value: value.value, ops }); if (deliveries.length === 1) { entered.resolve(); await release.promise; @@ -237,7 +246,7 @@ describe("Session document watches", () => { await watch.stop(); }); - it("folds retirement over pending commits and then closes", async () => { + it("folds a retirement into the pending commits' call, which reports it with the last value, and goes on", async () => { const { session } = await createState(); const watch = (await session.watchDoc(StateDoc, context))!; for (let value = 1; value <= 100; value++) { @@ -245,13 +254,22 @@ describe("Session document watches", () => { (await tx.doc(StateDoc)).value = value; }, context); } + const last = (await session.snapshot(StateDoc, context))!; await session.commit((tx) => tx.retireDoc(StateDoc), context); - const deliveries: Array<{ value: Readonly | null; ops: unknown }> = []; - watch.start(async (value, ops) => { - deliveries.push({ value, ops }); + const changes: WatchChange>[] = []; + watch.start(async (change) => { + changes.push(change); }); - expect(await watch.closed).toEqual({ reason: "retired" }); - expect(deliveries).toEqual([{ value: null, ops: [["r", null]] }]); + await flush(); + expect(changes).toEqual([{ value: last, ops: [["s", ["value"], 100]], retired: true }]); + expect(changes[0]!.value).toBe(last); + expect(watch.value).toBe(last); + await session.commit(async (tx) => { + (await tx.doc(StateDoc)).value = 7; + }, context); + await flush(); + expect(changes[1]).toEqual({ value: { ...last, value: 7 }, ops: [["r", { ...last, value: 7 }]], retired: false }); + await watch.stop(); }); it("delivers replayable structural no-op commits instead of suppressing them", async () => { @@ -267,7 +285,7 @@ describe("Session document watches", () => { expect(newest).not.toBe(initial); expect(newest).toEqual(initial); const batches: unknown[] = []; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { expect(value).toBe(newest); batches.push(ops); }); @@ -290,7 +308,7 @@ describe("Session document watches", () => { const entered = deferred(); const release = deferred(); let deliveryContext: Context | undefined; - watch.start(async (_value, _ops, delivered) => { + watch.start(async (_change, delivered) => { deliveryContext = delivered; entered.resolve(); await release.promise; @@ -312,8 +330,8 @@ describe("Session document watches", () => { const { session } = await createState(); const watch = (await session.watchDoc(StateDoc, context))!; const initial = watch.value; - let delivered: Readonly | null | undefined; - watch.start(async (value) => { + let delivered: Readonly | undefined; + watch.start(async ({ value }) => { delivered = value; }); await session.commit(async (tx) => { @@ -322,34 +340,55 @@ describe("Session document watches", () => { await flush(); expect(delivered).toBe(watch.value); expect(delivered).not.toBe(initial); - expect(delivered?.retained).toBe(initial?.retained); - expect(initial?.value).toBe(0); + expect(delivered?.retained).toBe(initial.retained); + expect(initial.value).toBe(0); await watch.stop(); }); - it("delivers retirement and does not follow recreation", async () => { + it("reports a change and a retirement in one commit as the retirement, with the value it was given before", async () => { + const { session, storage } = await createState(); + const watch = (await session.watchDoc(StateDoc, context))!; + const before = watch.value; + const changes: WatchChange>[] = []; + watch.start(async (change) => { + changes.push(change); + }); + await session.commit(async (tx) => { + (await tx.doc(StateDoc)).value = 5; + await tx.retireDoc(StateDoc); + }, context); + await flush(); + // Storage got the change, then the retirement; the commit publishes the retirement only. + expect(storage.commits.at(-1)!.map((write) => write.type)).toEqual(["document.change", "document.retire"]); + expect(changes).toEqual([{ value: before, ops: [], retired: true }]); + expect(changes[0]!.value).toBe(before); + await watch.stop(); + }); + + it("follows its address across a retirement and a recreation in one commit, in one call", async () => { const { session } = await createState(); - const oldWatch = (await session.watchDoc(StateDoc, context))!; - const values: Array | null> = []; - oldWatch.start(async (value) => { - values.push(value); + const watch = (await session.watchDoc(StateDoc, context))!; + const settled = endOf(watch); + const changes: WatchChange>[] = []; + watch.start(async (change) => { + changes.push(change); }); await session.commit(async (tx) => { await tx.retireDoc(StateDoc); (await tx.doc(StateDoc)).value = 10; }, context); - expect(await oldWatch.closed).toEqual({ reason: "retired" }); - expect(values).toEqual([null]); - expect(oldWatch.value).toBeNull(); - - const replacement = (await session.watchDoc(StateDoc, context))!; - expect(replacement.value?.value).toBe(10); + await flush(); + const recreated = (await session.snapshot(StateDoc, context))!; + expect(changes).toHaveLength(1); + expect(changes[0]).toEqual({ value: recreated, ops: [["r", recreated]], retired: true }); await session.commit(async (tx) => { (await tx.doc(StateDoc)).value = 11; }, context); await flush(); - expect(oldWatch.value).toBeNull(); - await replacement.stop(); + expect(changes[1]).toEqual({ value: watch.value, ops: [["s", ["value"], 11]], retired: false }); + expect(watch.value.value).toBe(11); + expect(settled.end).toBeUndefined(); + await watch.stop(); }); it("Session close discards retirement buffered before start", async () => { @@ -368,9 +407,9 @@ describe("Session document watches", () => { const watch = (await session.watchDoc(StateDoc, context))!; const entered = deferred(); const release = deferred(); - const values: Array = []; - watch.start(async (value) => { - values.push(value?.value ?? null); + const values: number[] = []; + watch.start(async ({ value }) => { + values.push(value.value); entered.resolve(); await release.promise; }); @@ -421,7 +460,7 @@ describe("Session document watches", () => { const entered = deferred(); const release = deferred(); let deliveryContext: Context | undefined; - watch.start(async (_value, _ops, delivered) => { + watch.start(async (_change, delivered) => { deliveryContext = delivered; entered.resolve(); await release.promise; @@ -442,7 +481,7 @@ describe("Session document watches", () => { const entered = deferred(); const release = deferred(); let deliveryContext: Context | undefined; - watch.start(async (_value, _ops, delivered) => { + watch.start(async (_change, delivered) => { deliveryContext = delivered; entered.resolve(); await release.promise; @@ -503,8 +542,8 @@ describe("Session document watches", () => { expect(watch.value).toEqual({ value: 3, migrated: true }); expect(storage.commits).toHaveLength(commits); const values: number[] = []; - watch.start(async (value) => { - values.push(value?.value ?? -1); + watch.start(async ({ value }) => { + values.push(value.value); }); await session.commit((tx) => tx.doc(Current).then(() => undefined), context); await flush(); @@ -520,8 +559,8 @@ describe("Session document watches", () => { it("can replay every delivered exact operation batch from the acquisition revision", async () => { const { session } = await createState(); const watch = (await session.watchDoc(StateDoc, context))!; - let replica: Readonly | null = watch.value; - watch.start(async (value, ops) => { + let replica: Readonly = watch.value; + watch.start(async ({ value, ops }) => { replica = applyImmutable(replica, ops); expect(replica).toEqual(value); }); @@ -550,10 +589,195 @@ describe("Session document watches", () => { (await tx.doc(StateDoc)).value = 7; }, context); await flush(); - expect(watch.value?.value).toBe(7); + expect(watch.value.value).toBe(7); await watch.stop(); }); + it("ignores the same kind in another conversation and under another family key", async () => { + const Notes = defineDoc<{ text: string }>({ + checkpointWhen: () => false, + kind: "watch.scoped-notes", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ text: "" }), + }); + const Members = defineDocFamily<{ text: string }, null>({ + checkpointWhen: () => false, + kind: "watch.members", + version: 1, + family: true, + scope: "session", + initial: () => ({ text: "" }), + }); + const { session } = openTestSession(); + const watched = await createConversation(session); + const other = await createConversation(session); + await session.commit(async (tx) => { + (await tx.doc(Notes, watched)).text = "a"; + (await tx.doc(Members, "watched", null)).text = "a"; + }, context); + const notes = (await session.watchDoc(Notes, watched, context))!; + const member = (await session.watchDoc(Members, "watched", context))!; + const changes: WatchChange>[] = []; + notes.start(async (change) => { + changes.push(change); + }); + member.start(async (change) => { + changes.push(change); + }); + // The same kinds at other addresses: created, changed, retired, and created again. + await session.commit(async (tx) => { + (await tx.doc(Notes, other)).text = "other"; + (await tx.doc(Members, "other", null)).text = "other"; + }, context); + await session.commit(async (tx) => { + await tx.retireDoc(Notes, other); + await tx.retireDoc(Members, "other"); + }, context); + await session.commit(async (tx) => { + (await tx.doc(Notes, other)).text = "again"; + (await tx.doc(Members, "other", null)).text = "again"; + }, context); + await flush(); + expect(changes).toEqual([]); + expect(session.documentCacheSizes.attached).toBe(2); + await notes.stop(); + await member.stop(); + await session.close(context); + }); + + it("takes the version of a recreation written by a newer definition, then its ops", async () => { + const Old = defineDoc<{ value: number }>({ + checkpointWhen: () => false, + kind: "watch.recreated-version", + version: 1, + scope: "session", + initial: () => ({ value: 1 }), + }); + const Current = defineDoc<{ value: number; migrated: boolean }>({ + checkpointWhen: () => false, + kind: "watch.recreated-version", + version: 2, + scope: "session", + initial: () => ({ value: 0, migrated: false }), + migrate: (value) => ({ value: value.value as number, migrated: true }), + }); + const { session } = openTestSession(); + await session.commit((tx) => tx.doc(Old).then(() => undefined), context); + const watch = (await session.watchDoc(Old, context))!; + const changes: WatchChange[] = []; + watch.start(async (change) => { + changes.push(change); + }); + await session.commit(async (tx) => { + await tx.retireDoc(Current); + (await tx.doc(Current)).value = 5; + }, context); + await flush(); + await session.commit(async (tx) => { + (await tx.doc(Current)).value = 6; + }, context); + await flush(); + expect(changes).toEqual([ + { + value: { value: 5, migrated: false }, + ops: [["r", { value: 5, migrated: false }]], + retired: true, + }, + { value: { value: 6, migrated: false }, ops: [["s", ["value"], 6]], retired: false }, + ]); + await watch.stop(); + await session.close(context); + }); + + it("keeps no earlier incarnation alive after a retirement and a recreation, while it goes on", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const collect = async (): Promise => { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + }; + /** Watch the document, retire and recreate it, and deliver the new value; `drop` then unloads the cache. */ + const follow = async (session: SessionImpl, drop: boolean) => { + await session.commit(async (tx) => { + (await tx.doc(StateDoc)).retained = { label: "first incarnation" }; + }, context); + const watch = (await session.watchDoc(StateDoc, context))!; + const first = new WeakRef(watch.value.retained); + watch.start(async () => {}); + await session.commit(async (tx) => { + await tx.retireDoc(StateDoc); + (await tx.doc(StateDoc)).value = 2; + }, context); + await flush(); + if (drop) await session.unloadDocuments(); + return { watch, first }; + }; + // Retention 0 evicts the retired incarnation at once; the default keeps the cache until it is unloaded. + for (const [session, drop] of [ + [new SessionImpl(new MemoryStorage(), Date.now, () => 0), false], + [new SessionImpl(new MemoryStorage(), Date.now, () => 600_000), true], + ] as const) { + const { watch, first } = await follow(session, drop); + await collect(); + expect(watch.value.value).toBe(2); + expect(watch.value.retained).toEqual({ label: "stable" }); + expect(first.deref()).toBeUndefined(); + await watch.stop(); + await session.close(context); + } + }); + + it("keeps a watch of a retired conversation document until it is stopped, then nothing of it", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const collect = async (): Promise => { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + }; + const Notes = defineDoc<{ text: string }>({ + checkpointWhen: () => false, + kind: "watch.notes", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ text: "" }), + }); + const { session } = openTestSession(); + const conversationId = await createConversation(session); + /** A started watch of a document that is then retired; `stop` says whether to stop it before returning. */ + const acquire = async (stop: boolean) => { + await session.commit(async (tx) => { + (await tx.doc(Notes, conversationId)).text = "kept"; + }, context); + const watch = (await session.watchDoc(Notes, conversationId, context))!; + const listener = async (): Promise => {}; + watch.start(listener); + await session.commit((tx) => tx.retireDoc(Notes, conversationId), context); + await flush(); + if (stop) await watch.stop(); + return { watch: new WeakRef(watch), listener: new WeakRef(listener) }; + }; + // A watch follows its address, so a forgotten one stays until it is stopped or the Session ends. + const forgotten = await acquire(false); + await collect(); + expect(forgotten.watch.deref()).toBeDefined(); + await forgotten.watch.deref()!.stop(); + + const stopped = await acquire(true); + await collect(); + expect(stopped.watch.deref()).toBeUndefined(); + expect(stopped.listener.deref()).toBeUndefined(); + await session.close(context); + }); + it("retains neither its Session nor its listener once stopped, while still referenced", async () => { setFlagsFromString("--expose-gc"); const gc = runInNewContext("gc") as () => void; @@ -595,7 +819,7 @@ describe("Session document watches", () => { expect(stopped.session.deref()).toBeUndefined(); expect(stopped.listener.deref()).toBeUndefined(); // What it delivered last stays readable. - expect(stopped.watch.value?.value).toBe(1); + expect(stopped.watch.value.value).toBe(1); expect(await stopped.watch.closed).toEqual({ reason: "stopped" }); }); }); diff --git a/packages/durable/test/task-support.ts b/packages/durable/test/task-support.ts index a94373050..77cac5897 100644 --- a/packages/durable/test/task-support.ts +++ b/packages/durable/test/task-support.ts @@ -71,6 +71,7 @@ export async function openTasks( readonly settings?: HarnessSettings; readonly onWake?: HarnessOptions["onWake"]; readonly setTimer?: HarnessOptions["setTimer"]; + readonly telemetry?: HarnessOptions["telemetry"]; } = {}, ): Promise<{ readonly harness: Harness; @@ -95,6 +96,7 @@ export async function openTasks( ...(options.settings === undefined ? {} : { settings: options.settings }), ...(options.onWake === undefined ? {} : { onWake: options.onWake }), ...(options.setTimer === undefined ? {} : { setTimer: options.setTimer }), + ...(options.telemetry === undefined ? {} : { telemetry: options.telemetry }), }, context, ); diff --git a/packages/env/src/remote-env.ts b/packages/env/src/remote-env.ts index c77ce201e..f3740994b 100644 --- a/packages/env/src/remote-env.ts +++ b/packages/env/src/remote-env.ts @@ -9,6 +9,7 @@ import { type ExecutionEnv, ExecutionError, err, + type FileChange, FileError, type FileInfo, type FileKind, @@ -25,7 +26,6 @@ import { StreamDecoder, type TextLine, type TextLineReader, - type WatchChange, type WatchTarget, } from "@earendil-works/pi-durable/env"; import { type Connection, type Json, nodePlatform, RemoteError, type RemoteInfo } from "./connection.ts"; @@ -826,7 +826,7 @@ export class RemoteExecutionEnv implements ExecutionEnv { async watch( targets: readonly WatchTarget[], - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, context: Context, ): Promise> { const aborted = abortResult(context.abortSignal); diff --git a/packages/env/src/watch.ts b/packages/env/src/watch.ts index 402869891..3ee9c0fd8 100644 --- a/packages/env/src/watch.ts +++ b/packages/env/src/watch.ts @@ -1,5 +1,5 @@ import type { Context } from "@earendil-works/chord"; -import { FileError, type FileWatcher, type WatchChange, type WatchTarget } from "@earendil-works/pi-durable/env"; +import { type FileChange, FileError, type FileWatcher, type WatchTarget } from "@earendil-works/pi-durable/env"; import { type Connection, isConnectionLost } from "./connection.ts"; import { toFileError } from "./errors.ts"; @@ -23,7 +23,7 @@ const RECONNECT_MAX_MS = 30_000; export class RemoteWatcher implements FileWatcher { readonly #connection: Connection; readonly #targets: readonly WatchTarget[]; - readonly #onChange: (change: WatchChange) => void; + readonly #onChange: (change: FileChange) => void; readonly #options: RemoteWatchOptions; #mode: "native" | "polling" = "native"; #closed = false; @@ -34,7 +34,7 @@ export class RemoteWatcher implements FileWatcher { private constructor( connection: Connection, targets: readonly WatchTarget[], - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, options: RemoteWatchOptions, ) { this.#connection = connection; @@ -47,7 +47,7 @@ export class RemoteWatcher implements FileWatcher { static async open( connection: Connection, targets: readonly WatchTarget[], - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, options: RemoteWatchOptions = {}, ): Promise { const watcher = new RemoteWatcher(connection, targets, onChange, options); @@ -67,7 +67,7 @@ export class RemoteWatcher implements FileWatcher { await this.#request; } - #deliver(change: WatchChange): void { + #deliver(change: FileChange): void { if (this.#closed) return; try { this.#onChange(change); From 6c384aa30f619d7b28963f2bab24d8dacbe73cb6 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 26 Oct 2026 21:05:54 +0000 Subject: [PATCH 019/271] fix(ai): resolve pi-telemetry from source in workspace tests The root and ./testing exports of pi-telemetry had no source condition, so Durable's and pi-env's tests, which resolve with source, failed to import it without a build: the Environments workflow has failed on every run since 2026-10-18. --- packages/telemetry/CHANGELOG.md | 4 ++++ packages/telemetry/package.json | 2 ++ 2 files changed, 6 insertions(+) diff --git a/packages/telemetry/CHANGELOG.md b/packages/telemetry/CHANGELOG.md index b71b6c86e..52617bb21 100644 --- a/packages/telemetry/CHANGELOG.md +++ b/packages/telemetry/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- The package root and `./testing` exports have a `source` condition, like `./render`, so workspace tests that resolve with `source` (Durable's and pi-env's, as the Environments workflow runs them) work without a build. + ## [1.4.0] - 2026-10-21 ### Breaking Changes diff --git a/packages/telemetry/package.json b/packages/telemetry/package.json index 92b1f04e4..c571023b6 100644 --- a/packages/telemetry/package.json +++ b/packages/telemetry/package.json @@ -7,6 +7,7 @@ "types": "./dist/index.d.ts", "exports": { ".": { + "source": "./src/index.ts", "types": "./dist/index.d.ts", "import": "./dist/index.js" }, @@ -16,6 +17,7 @@ "import": "./dist/render.js" }, "./testing": { + "source": "./src/testing/index.ts", "types": "./dist/testing/index.d.ts", "import": "./dist/testing/index.js" } From fecc19e4e9b60774da44b4c3ea22285faf79ba92 Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:07:23 +0100 Subject: [PATCH 020/271] fix(coding-agent): keep grep context lines aligned with ripgrep when a file has a lone CR (#11465) When `context` is set, the grep tool reads the file again to print the lines around each match. It turned CRLF into LF and also turned every lone CR into LF before splitting the text into lines. ripgrep, which supplies the match line numbers, counts lines only at LF. A lone CR therefore added an extra line on pi's side, so every later line was off by one. For example, in a file containing `before\rprogress\nneedle\nafter`, a search for `needle` with `context: 1` showed `progress` as the line-2 match and left out `after`. The fix keeps the CRLF normalization and splits only on LF, so pi numbers lines the same way ripgrep does. Displayed lines already have `\r` removed, so the line before the match now shows as `beforeprogress`. **Testing** - Added a test in `packages/coding-agent/test/tools.test.ts` for a file with a lone CR searched with `context: 1`. It fails without the fix and passes with it. - `test/tools.test.ts`: all 87 tests pass. - `npm run check` passes. - Reproduction from the issue. Before the fix: ``` f.txt-1- before f.txt:2: progress f.txt-3- needle ``` After the fix: ``` f.txt-1- beforeprogress f.txt:2: needle f.txt-3- after ``` Fixes #11398 Requested by @badlogic Co-authored-by: triagent --- packages/coding-agent/src/core/tools/grep.ts | 2 +- packages/coding-agent/test/tools.test.ts | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/coding-agent/src/core/tools/grep.ts b/packages/coding-agent/src/core/tools/grep.ts index a9562c69f..77d830cd1 100644 --- a/packages/coding-agent/src/core/tools/grep.ts +++ b/packages/coding-agent/src/core/tools/grep.ts @@ -150,7 +150,7 @@ export function createGrepToolDefinition( if (!lines) { try { const content = await ops.readFile(filePath); - lines = content.replace(/\r\n/g, "\n").replace(/\r/g, "\n").split("\n"); + lines = content.replace(/\r\n/g, "\n").split("\n"); } catch { lines = []; } diff --git a/packages/coding-agent/test/tools.test.ts b/packages/coding-agent/test/tools.test.ts index 91432623f..a096c2f91 100644 --- a/packages/coding-agent/test/tools.test.ts +++ b/packages/coding-agent/test/tools.test.ts @@ -908,6 +908,23 @@ describe("Coding Agent Tools", () => { expect(output).not.toContain("match two"); }); + // https://github.com/earendil-works/pi/issues/11398 + it("should keep context line numbers aligned with ripgrep when a file contains a lone CR", async () => { + const testFile = join(testDir, "lone-cr.txt"); + writeFileSync(testFile, "before\rprogress\nneedle\nafter"); + + const result = await grepTool.execute("test-call-lone-cr", { + pattern: "needle", + path: testFile, + context: 1, + }); + + const output = getTextOutput(result); + expect(output).toContain("lone-cr.txt-1- beforeprogress"); + expect(output).toContain("lone-cr.txt:2: needle"); + expect(output).toContain("lone-cr.txt-3- after"); + }); + // https://github.com/earendil-works/pi/issues/11203 it("should ignore RIPGREP_CONFIG_PATH", async () => { const configFile = join(testDir, "rg-config"); From 7853186d14c33ed4a983d694622ba2ac4cd62202 Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:11:18 +0100 Subject: [PATCH 021/271] fix(coding-agent): keep open extension dialog visible when the editor component is replaced (#11466) `ctx.ui.setEditorComponent()` always cleared the editor slot and put the new editor there with focus. If an extension dialog (`confirm`/`select`/`input`/`editor`, or an inline `custom()` component) was open at that moment, the editor replaced it on screen. The dialog was still on the dialog stack and its promise still pending, but nobody could see or answer it. It hung until its timeout, or forever without one. Now `setCustomEditorComponent` leaves the editor slot and focus alone while a dialog is open. The new editor is still created and wired up, and `showTopExtensionDialog` shows it when the last dialog closes. Without an open dialog nothing changes. Testing: - Added a regression test to `test/interactive-mode-extension-dialogs.test.ts`. It opens a confirm, replaces the editor, then checks that the confirm stays visible and answerable and that the new editor is focused afterwards. It fails without the fix and passes with it. - `npm run check` passes. - Ran the reproduction from the issue in tmux (`pi -e ./a.ts -e ./b.ts`, where a.ts opens a non-blocking `confirm` in `session_start` and b.ts calls `setEditorComponent`). - Before: only the editor shows, and the confirm times out. - After: the confirm shows, and Enter gives `answer: true`. Fixes #11431 Requested by @badlogic Co-authored-by: triagent --- .../src/modes/interactive/interactive-mode.ts | 12 ++++++---- ...interactive-mode-extension-dialogs.test.ts | 23 +++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/packages/coding-agent/src/modes/interactive/interactive-mode.ts b/packages/coding-agent/src/modes/interactive/interactive-mode.ts index a596eea88..7b3b45ba9 100644 --- a/packages/coding-agent/src/modes/interactive/interactive-mode.ts +++ b/packages/coding-agent/src/modes/interactive/interactive-mode.ts @@ -2839,8 +2839,12 @@ export class InteractiveMode { // Save text from current editor before switching const currentText = this.editor.getText(); - this.disposeActiveSelector(); - this.editorContainer.clear(); + // An open extension dialog keeps the editor slot; the new editor is shown when the last dialog closes. + const dialogOpen = this.extensionDialogs.length > 0; + if (!dialogOpen) { + this.disposeActiveSelector(); + this.editorContainer.clear(); + } if (factory) { // Create the custom editor with tui, theme, and keybindings @@ -2898,7 +2902,7 @@ export class InteractiveMode { this.editor = this.defaultEditor; } - this.editorContainer.addChild(this.editor as Component); + if (!dialogOpen) this.editorContainer.addChild(this.editor as Component); if (this.activeStatusIndicator) { this.statusContainer.clear(); this.activeWorkingIndicatorEmbedded = this.setEditorWorkingStatusIndicator(this.activeStatusIndicator); @@ -2906,7 +2910,7 @@ export class InteractiveMode { this.statusContainer.addChild(this.activeStatusIndicator); } } - this.ui.setFocus(this.editor as Component); + if (!dialogOpen) this.ui.setFocus(this.editor as Component); this.ui.requestRender(); } diff --git a/packages/coding-agent/test/interactive-mode-extension-dialogs.test.ts b/packages/coding-agent/test/interactive-mode-extension-dialogs.test.ts index d296ff2b4..2a59f9189 100644 --- a/packages/coding-agent/test/interactive-mode-extension-dialogs.test.ts +++ b/packages/coding-agent/test/interactive-mode-extension-dialogs.test.ts @@ -19,6 +19,9 @@ type DialogMethods = { showExtensionCustom( factory: (tui: unknown, theme: unknown, keybindings: unknown, done: (result: T) => void) => Component, ): Promise; + setCustomEditorComponent( + factory: ((tui: unknown, theme: unknown, keybindings: unknown) => Component) | undefined, + ): void; }; function createContext(): DialogContext & DialogMethods { @@ -149,4 +152,24 @@ describe("InteractiveMode extension dialogs", () => { await expect(custom).resolves.toBe("kept"); expect(context.focused).toBe(context.editor); }); + + // Regression for #11431: replacing the editor while a dialog was open hid the dialog and left it unanswerable. + it("keeps an open dialog visible when the editor component is replaced", async () => { + const context = createContext(); + (context as unknown as { defaultEditor: unknown }).defaultEditor = context.editor; + const confirmed = context.showExtensionConfirm("Title", "Continue?"); + const customEditor: Component & { getText(): string; setText(text: string): void } = { + render: () => ["custom editor"], + invalidate: () => {}, + getText: () => "", + setText: () => {}, + }; + context.setCustomEditorComponent(() => customEditor); + + expect(visible(context)).toContain("Title"); + context.focused?.handleInput?.("\r"); + await expect(confirmed).resolves.toBe(true); + expect(context.focused).toBe(customEditor); + expect(context.editorContainer.children).toEqual([customEditor]); + }); }); From f049787d9e7a3ae04605df8d41981cb4b8eb5225 Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:13:12 +0100 Subject: [PATCH 022/271] fix(coding-agent): default openai provider to gpt-6.1-sol (#11467) The `openai` provider still defaulted to `gpt-5.5`. The same default applies to API-key logins and to "Sign in with ChatGPT", so logging in through ChatGPT selected `openai/gpt-5.5`. The reporter says that model no longer works with a ChatGPT subscription. The `openai-codex` and `github-copilot` providers already default to `gpt-6.1-sol`. This change sets `defaultModelPerProvider.openai` to `gpt-6.1-sol` in `packages/coding-agent/src/core/model-resolver.ts`. The new value applies in three places: the model picked after `/login` when no model was set, the fallback model at startup, and the default model for `--provider openai`. API-key users get the new default too. I did not test against a real ChatGPT account, so the claim that the subscription rejects `gpt-5.5` is not checked here. Testing: - Updated the assertion in `test/model-resolver.test.ts` ("openai defaults track current models"). It now expects `gpt-6.1-sol`; the model-resolver tests pass. - `npm run check` passes. - `pi --list-models gpt-6.1` still lists `openai/gpt-6.1-sol`, so the new default is in the catalog. Fixes #11442 Requested by @badlogic Co-authored-by: triagent --- packages/coding-agent/src/core/model-resolver.ts | 2 +- packages/coding-agent/test/model-resolver.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/coding-agent/src/core/model-resolver.ts b/packages/coding-agent/src/core/model-resolver.ts index bd331f690..d1abd57cf 100644 --- a/packages/coding-agent/src/core/model-resolver.ts +++ b/packages/coding-agent/src/core/model-resolver.ts @@ -21,7 +21,7 @@ export const defaultModelPerProvider: Partial> = { "amazon-bedrock": "us.anthropic.claude-opus-4-6-v1", "ant-ling": "Ring-2.6-1T", anthropic: "claude-opus-4-8", - openai: "gpt-5.5", + openai: "gpt-6.1-sol", azure: "gpt-5.4", "openai-codex": "gpt-6.1-sol", radius: "balanced", diff --git a/packages/coding-agent/test/model-resolver.test.ts b/packages/coding-agent/test/model-resolver.test.ts index 018f4c8d1..71cd27f0c 100644 --- a/packages/coding-agent/test/model-resolver.test.ts +++ b/packages/coding-agent/test/model-resolver.test.ts @@ -698,7 +698,7 @@ describe("resolveCliModel", () => { describe("default model selection", () => { test("openai defaults track current models", () => { - expect(defaultModelPerProvider.openai).toBe("gpt-5.5"); + expect(defaultModelPerProvider.openai).toBe("gpt-6.1-sol"); expect(defaultModelPerProvider["openai-codex"]).toBe("gpt-6.1-sol"); }); From deb12d78ebbd2b65a4a884334cad2f26f3847636 Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:16:36 +0100 Subject: [PATCH 023/271] fix(tui): recognize kitty F1-F12 sequences and ignore F13+ keys (#11468) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the kitty keyboard protocol active, these function keys were not recognized: - F1, F2 and F4 arrive as `CSI P`/`Q`/`S`, or as `CSI 1; P/Q/S`. - Any F-key pressed with a modifier, e.g. `CSI 15;5~` for Ctrl+F5. `matchesKey` returned false whenever a modifier was set and only compared against the legacy sequences. `parseKittySequence` did not know any F-key forms. So shortcuts bound to keys like `f1` or `ctrl+f5` never fired. Separately, F13–F35 (and other kitty functional keys) arrive as CSI-u codepoints in the Private Use Area. `decodeKittyPrintable` returned them as text, so the editor inserted invisible U+E020+ characters into the prompt. Changes in `packages/tui/src/keys.ts`: - Add F1–F12 internal codepoints. Map `~` numbers 11–15, 17–21, 23 and 24 to them. - Parse the `CSI [1;mod[:event]] P/Q/S` form. F3 is left out because `CSI 1;R` collides with cursor position reports, so kitty sends F3 as `CSI 13~`. - Let the `f1`…`f12` cases in `matchesKey` fall back to `matchesKittySequence` so modifiers work. Name the keys in `parseKey` output. - Make `isKeyRelease` and `isKeyRepeat` recognize `:3`/`:2` events on P/Q/S, so F1/F2/F4 releases don't trigger shortcuts a second time. - `decodeKittyPrintable` now rejects the kitty functional-key PUA range (57344–63743). Keypad keys are normalized to ASCII first, so they still decode. Testing: - Added tests in `packages/tui/test/keys.test.ts`. They cover kitty F1–F12 with and without modifiers, F3 not being confused with a cursor position report, release and repeat detection, and F13+ not decoding as text. Two of the new tests fail without the change; all pass with it. - Ran `npm test` in `packages/tui` and `npm run check`; both pass. - Before the change: `parseKey("\x1b[P")` and `parseKey("\x1b[1;5P")` returned `undefined`, `matchesKey("\x1b[15;5~", "ctrl+f5")` returned false, and `decodePrintableKey("\x1b[57376u")` returned `"\ue020"`. - After the change: these give `f1`, `ctrl+f1`, true and `undefined`. Fixes #11419 Fixes #11345 Requested by @badlogic Co-authored-by: triagent --- packages/tui/src/keys.ts | 68 +++++++++++++++++++++++++++++++--- packages/tui/test/keys.test.ts | 28 ++++++++++++++ 2 files changed, 90 insertions(+), 6 deletions(-) diff --git a/packages/tui/src/keys.ts b/packages/tui/src/keys.ts index 734fbfae5..a3ce39726 100644 --- a/packages/tui/src/keys.ts +++ b/packages/tui/src/keys.ts @@ -325,8 +325,25 @@ const FUNCTIONAL_CODEPOINTS = { pageDown: -13, home: -14, end: -15, + f1: -21, + f2: -22, + f3: -23, + f4: -24, + f5: -25, + f6: -26, + f7: -27, + f8: -28, + f9: -29, + f10: -30, + f11: -31, + f12: -32, } as const; +// Kitty reserves this Private Use Area range for functional keys (F13-F35, +// media keys, modifier keys, ...). CSI-u codepoints in it are never text. +const KITTY_FUNCTIONAL_PUA_START = 57344; +const KITTY_FUNCTIONAL_PUA_END = 63743; + const KITTY_FUNCTIONAL_KEY_EQUIVALENTS = new Map([ [57399, 48], // KP_0 -> 0 [57400, 49], // KP_1 -> 1 @@ -558,7 +575,10 @@ export function isKeyRelease(data: string): boolean { data.includes(":3C") || data.includes(":3D") || data.includes(":3H") || - data.includes(":3F") + data.includes(":3F") || + data.includes(":3P") || + data.includes(":3Q") || + data.includes(":3S") ) { return true; } @@ -584,7 +604,10 @@ export function isKeyRepeat(data: string): boolean { data.includes(":2C") || data.includes(":2D") || data.includes(":2H") || - data.includes(":2F") + data.includes(":2F") || + data.includes(":2P") || + data.includes(":2Q") || + data.includes(":2S") ) { return true; } @@ -644,6 +667,18 @@ function parseKittySequence(data: string): ParsedKittySequence | null { 6: FUNCTIONAL_CODEPOINTS.pageDown, 7: FUNCTIONAL_CODEPOINTS.home, 8: FUNCTIONAL_CODEPOINTS.end, + 11: FUNCTIONAL_CODEPOINTS.f1, + 12: FUNCTIONAL_CODEPOINTS.f2, + 13: FUNCTIONAL_CODEPOINTS.f3, + 14: FUNCTIONAL_CODEPOINTS.f4, + 15: FUNCTIONAL_CODEPOINTS.f5, + 17: FUNCTIONAL_CODEPOINTS.f6, + 18: FUNCTIONAL_CODEPOINTS.f7, + 19: FUNCTIONAL_CODEPOINTS.f8, + 20: FUNCTIONAL_CODEPOINTS.f9, + 21: FUNCTIONAL_CODEPOINTS.f10, + 23: FUNCTIONAL_CODEPOINTS.f11, + 24: FUNCTIONAL_CODEPOINTS.f12, }; const codepoint = funcCodes[keyNum]; if (codepoint !== undefined) { @@ -662,6 +697,21 @@ function parseKittySequence(data: string): ParsedKittySequence | null { return { codepoint, modifier: modValue - 1, eventType }; } + // F1, F2, F4: \x1b[P/Q/S or \x1b[1;P/Q/S or \x1b[1;:P/Q/S + // F3 is sent as \x1b[13~ because \x1b[1;R collides with cursor position reports. + const fKeyMatch = data.match(/^\x1b\[(?:1;(\d+)(?::(\d+))?)?([PQS])$/); + if (fKeyMatch) { + const modValue = fKeyMatch[1] ? parseInt(fKeyMatch[1], 10) : 1; + const eventType = parseEventType(fKeyMatch[2]); + const fKeyCodes: Record = { + P: FUNCTIONAL_CODEPOINTS.f1, + Q: FUNCTIONAL_CODEPOINTS.f2, + S: FUNCTIONAL_CODEPOINTS.f4, + }; + _lastEventType = eventType; + return { codepoint: fKeyCodes[fKeyMatch[3]!]!, modifier: modValue - 1, eventType }; + } + return null; } @@ -1176,11 +1226,11 @@ export function matchesKey(data: string, keyId: KeyId): boolean { case "f10": case "f11": case "f12": { - if (modifier !== 0) { - return false; + const functionKey = key as keyof typeof LEGACY_KEY_SEQUENCES & keyof typeof FUNCTIONAL_CODEPOINTS; + if (modifier === 0 && matchesLegacySequence(data, LEGACY_KEY_SEQUENCES[functionKey])) { + return true; } - const functionKey = key as keyof typeof LEGACY_KEY_SEQUENCES; - return matchesLegacySequence(data, LEGACY_KEY_SEQUENCES[functionKey]); + return matchesKittySequence(data, FUNCTIONAL_CODEPOINTS[functionKey], modifier); } } @@ -1277,6 +1327,8 @@ function formatParsedKey(codepoint: number, modifier: number, baseLayoutKey?: nu else if (effectiveCodepoint === FUNCTIONAL_CODEPOINTS.end) keyName = "end"; else if (effectiveCodepoint === FUNCTIONAL_CODEPOINTS.pageUp) keyName = "pageUp"; else if (effectiveCodepoint === FUNCTIONAL_CODEPOINTS.pageDown) keyName = "pageDown"; + else if (effectiveCodepoint <= FUNCTIONAL_CODEPOINTS.f1 && effectiveCodepoint >= FUNCTIONAL_CODEPOINTS.f12) + keyName = `f${FUNCTIONAL_CODEPOINTS.f1 - effectiveCodepoint + 1}`; else if (effectiveCodepoint === ARROW_CODEPOINTS.up) keyName = "up"; else if (effectiveCodepoint === ARROW_CODEPOINTS.down) keyName = "down"; else if (effectiveCodepoint === ARROW_CODEPOINTS.left) keyName = "left"; @@ -1428,6 +1480,10 @@ export function decodeKittyPrintable(data: string): string | undefined { effectiveCodepoint = normalizeKittyFunctionalCodepoint(effectiveCodepoint); // Drop control characters or invalid codepoints. if (!Number.isFinite(effectiveCodepoint) || effectiveCodepoint < 32) return undefined; + // Drop unmapped Kitty functional keys (F13-F35, media keys, ...). + if (effectiveCodepoint >= KITTY_FUNCTIONAL_PUA_START && effectiveCodepoint <= KITTY_FUNCTIONAL_PUA_END) { + return undefined; + } try { return String.fromCodePoint(effectiveCodepoint); diff --git a/packages/tui/test/keys.test.ts b/packages/tui/test/keys.test.ts index 0045610bc..4494508e1 100644 --- a/packages/tui/test/keys.test.ts +++ b/packages/tui/test/keys.test.ts @@ -7,6 +7,8 @@ import { describe, it } from "node:test"; import { decodeKittyPrintable, decodePrintableKey, + isKeyRelease, + isKeyRepeat, Key, matchesKey, parseKey, @@ -509,6 +511,26 @@ describe("matchesKey", () => { assert.strictEqual(matchesKey("\x1b[E", "clear"), true); }); + it("should match Kitty function keys F1-F12 with and without modifiers", () => { + assert.strictEqual(matchesKey("\x1b[P", "f1"), true); + assert.strictEqual(matchesKey("\x1b[Q", "f2"), true); + assert.strictEqual(matchesKey("\x1b[13~", "f3"), true); + assert.strictEqual(matchesKey("\x1b[S", "f4"), true); + assert.strictEqual(matchesKey("\x1b[1;5P", "ctrl+f1"), true); + assert.strictEqual(matchesKey("\x1b[1;5P", "f1"), false); + assert.strictEqual(matchesKey("\x1b[1;2S", "shift+f4"), true); + assert.strictEqual(matchesKey("\x1b[15;5~", "ctrl+f5"), true); + assert.strictEqual(matchesKey("\x1b[24;3~", "alt+f12"), true); + assert.strictEqual(matchesKey("\x1b[15;1:2~", "f5"), true); + assert.strictEqual(parseKey("\x1b[P"), "f1"); + assert.strictEqual(parseKey("\x1b[1;5P"), "ctrl+f1"); + assert.strictEqual(parseKey("\x1b[13~"), "f3"); + assert.strictEqual(parseKey("\x1b[21;2~"), "shift+f10"); + assert.strictEqual(parseKey("\x1b[1;5R"), undefined); + assert.strictEqual(isKeyRelease("\x1b[1;1:3P"), true); + assert.strictEqual(isKeyRepeat("\x1b[1;1:2S"), true); + }); + it("should match alt+arrows", () => { assert.strictEqual(matchesKey("\x1bp", "alt+up"), true); assert.strictEqual(matchesKey("\x1bp", "up"), false); @@ -525,6 +547,12 @@ describe("matchesKey", () => { }); describe("decodeKittyPrintable", () => { + it("ignores unmapped Kitty functional keys such as F13-F35", () => { + assert.strictEqual(decodeKittyPrintable("\x1b[57376u"), undefined); + assert.strictEqual(decodeKittyPrintable("\x1b[57398;2u"), undefined); + assert.strictEqual(decodePrintableKey("\x1b[57382u"), undefined); + }); + it("should decode Kitty keypad functional keys to printable characters", () => { assert.strictEqual(decodeKittyPrintable("\x1b[57399u"), "0"); assert.strictEqual(decodeKittyPrintable("\x1b[57400u"), "1"); From 5fb9a0b11040feaec4dc4928bf699327bd0b33cd Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:17:54 +0100 Subject: [PATCH 024/271] fix(coding-agent): skip npm audit and fund output when installing pi packages (#11469) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `pi install`, `pi remove` and `pi update --extensions` run npm without `--no-audit`, so npm prints its "N vulnerabilities … run `npm audit fix`" advice after every install. pi gives users no way to clear that warning, and following it causes harm. In `~/.pi/agent/npm`, `npm audit fix` runs without `--legacy-peer-deps`, so npm installs the host-provided `@earendil-works/pi-*` peers there: a second copy of pi. The comment above `getNpmInstallArgs` warns against exactly that. The change adds `--no-audit --no-fund` wherever pi already passes `--legacy-peer-deps` to npm: - `getNpmInstallArgs` (npm package installs and updates) - `getGitDependencyInstallArgs` (dependency installs for git packages) - npm uninstall pi's managed self-update (`runManagedNpmCi`) already passes both flags. bun and pnpm arguments are unchanged. I did not edit `CHANGELOG.md`, per CONTRIBUTING.md. **Testing** - Updated the expected npm argument arrays in `test/package-manager.test.ts`. With the source change reverted, 11 tests fail; with it, all 132 pass. - `npm run check` passes. - Ran the built CLI by hand with `PI_CODING_AGENT_DIR=/tmp/pa pi install npm:lodash@4.17.20`: - Before the change, the output ended with npm's "To address all issues, run: npm audit fix" block. - After, it shows only "added 1 package" and "Installed npm:lodash@4.17.20". Fixes #11409 Requested by @badlogic Co-authored-by: triagent --- .../coding-agent/src/core/package-manager.ts | 7 +- .../coding-agent/test/package-manager.test.ts | 82 ++++++++++++++----- 2 files changed, 67 insertions(+), 22 deletions(-) diff --git a/packages/coding-agent/src/core/package-manager.ts b/packages/coding-agent/src/core/package-manager.ts index 2535b1f93..9b7a734d4 100644 --- a/packages/coding-agent/src/core/package-manager.ts +++ b/packages/coding-agent/src/core/package-manager.ts @@ -1858,7 +1858,7 @@ export class DefaultPackageManager implements PackageManager { "--config.strict-dep-builds=false", ]; case "npm": - return ["install", "--omit=dev", "--legacy-peer-deps"]; + return ["install", "--omit=dev", "--legacy-peer-deps", "--no-audit", "--no-fund"]; default: return ["install"]; } @@ -1891,7 +1891,8 @@ export class DefaultPackageManager implements PackageManager { "--config.strict-dep-builds=false", ]; } - return ["install", ...specs, "--prefix", installRoot, "--legacy-peer-deps"]; + // npm audit fix would run without --legacy-peer-deps and install pi peers, so skip audit advice. + return ["install", ...specs, "--prefix", installRoot, "--legacy-peer-deps", "--no-audit", "--no-fund"]; } private async installNpm(source: NpmSource, scope: SourceScope, temporary: boolean): Promise { @@ -1914,7 +1915,7 @@ export class DefaultPackageManager implements PackageManager { } const args = ["uninstall", source.name, "--prefix", installRoot]; if (packageManagerName !== "pnpm") { - args.push("--legacy-peer-deps"); + args.push("--legacy-peer-deps", "--no-audit", "--no-fund"); } await this.runNpmCommand(args); } diff --git a/packages/coding-agent/test/package-manager.test.ts b/packages/coding-agent/test/package-manager.test.ts index f8446b657..38d93b232 100644 --- a/packages/coding-agent/test/package-manager.test.ts +++ b/packages/coding-agent/test/package-manager.test.ts @@ -756,6 +756,8 @@ Content`, "--prefix", realpathSync(join(agentDir, "npm")), "--legacy-peer-deps", + "--no-audit", + "--no-fund", ], undefined, ); @@ -769,7 +771,15 @@ Content`, expect(runCommandSpy).toHaveBeenCalledWith( "npm", - ["uninstall", "@scope/pkg", "--prefix", realpathSync(join(agentDir, "npm")), "--legacy-peer-deps"], + [ + "uninstall", + "@scope/pkg", + "--prefix", + realpathSync(join(agentDir, "npm")), + "--legacy-peer-deps", + "--no-audit", + "--no-fund", + ], undefined, ); }); @@ -793,12 +803,12 @@ Content`, expect(runCommandSpy).toHaveBeenCalledWith( "npm", - ["install", "@scope/pkg", "--prefix", realNpmRoot, "--legacy-peer-deps"], + ["install", "@scope/pkg", "--prefix", realNpmRoot, "--legacy-peer-deps", "--no-audit", "--no-fund"], undefined, ); expect(runCommandSpy).toHaveBeenCalledWith( "npm", - ["uninstall", "@scope/pkg", "--prefix", realNpmRoot, "--legacy-peer-deps"], + ["uninstall", "@scope/pkg", "--prefix", realNpmRoot, "--legacy-peer-deps", "--no-audit", "--no-fund"], undefined, ); }); @@ -849,9 +859,13 @@ Content`, await packageManager.install(source); - expect(runCommandSpy).toHaveBeenCalledWith("npm", ["install", "--omit=dev", "--legacy-peer-deps"], { - cwd: targetDir, - }); + expect(runCommandSpy).toHaveBeenCalledWith( + "npm", + ["install", "--omit=dev", "--legacy-peer-deps", "--no-audit", "--no-fund"], + { + cwd: targetDir, + }, + ); }); it("should remove a newly created checkout when git clone fails", async () => { @@ -915,9 +929,13 @@ Content`, cwd: targetDir, }); expect(runCommandSpy).toHaveBeenCalledWith("git", ["clean", "-fdx"], { cwd: targetDir }); - expect(runCommandSpy).toHaveBeenCalledWith("npm", ["install", "--omit=dev", "--legacy-peer-deps"], { - cwd: targetDir, - }); + expect(runCommandSpy).toHaveBeenCalledWith( + "npm", + ["install", "--omit=dev", "--legacy-peer-deps", "--no-audit", "--no-fund"], + { + cwd: targetDir, + }, + ); }); it("should reconcile an existing git checkout to its update target when installing without a ref", async () => { @@ -1066,9 +1084,13 @@ Content`, await packageManager.update(source); - expect(runCommandSpy).toHaveBeenCalledWith("npm", ["install", "--omit=dev", "--legacy-peer-deps"], { - cwd: targetDir, - }); + expect(runCommandSpy).toHaveBeenCalledWith( + "npm", + ["install", "--omit=dev", "--legacy-peer-deps", "--no-audit", "--no-fund"], + { + cwd: targetDir, + }, + ); }); it("should repair missing git package dependencies when the checkout is already current", async () => { @@ -1093,9 +1115,13 @@ Content`, await packageManager.update(source); - expect(runCommandSpy).toHaveBeenCalledWith("npm", ["install", "--omit=dev", "--legacy-peer-deps"], { - cwd: targetDir, - }); + expect(runCommandSpy).toHaveBeenCalledWith( + "npm", + ["install", "--omit=dev", "--legacy-peer-deps", "--no-audit", "--no-fund"], + { + cwd: targetDir, + }, + ); expect(runCommandSpy).not.toHaveBeenCalledWith("git", ["clean", "-fdx"], { cwd: targetDir }); }); @@ -1127,9 +1153,13 @@ Content`, await expect(packageManager.update(source)).rejects.toThrow("simulated clean failure"); - expect(runCommandSpy).toHaveBeenCalledWith("npm", ["install", "--omit=dev", "--legacy-peer-deps"], { - cwd: targetDir, - }); + expect(runCommandSpy).toHaveBeenCalledWith( + "npm", + ["install", "--omit=dev", "--legacy-peer-deps", "--no-audit", "--no-fund"], + { + cwd: targetDir, + }, + ); }); it("should disable peer installation through wrapped pnpm when updating git dependencies", async () => { @@ -2506,7 +2536,15 @@ export default function(api) { api.registerTool({ name: "test", description: "te ); expect(runCommandSpy).toHaveBeenCalledWith( "npm", - ["install", "example@^1.0.0", "--prefix", realpathSync(join(tempDir, ".pi", "npm")), "--legacy-peer-deps"], + [ + "install", + "example@^1.0.0", + "--prefix", + realpathSync(join(tempDir, ".pi", "npm")), + "--legacy-peer-deps", + "--no-audit", + "--no-fund", + ], undefined, ); }); @@ -2572,6 +2610,8 @@ export default function(api) { api.registerTool({ name: "test", description: "te "--prefix", realpathSync(join(agentDir, "npm")), "--legacy-peer-deps", + "--no-audit", + "--no-fund", ]); mkdirSync(managedPath, { recursive: true }); writeFileSync( @@ -2689,6 +2729,8 @@ export default function(api) { api.registerTool({ name: "test", description: "te "--prefix", realpathSync(join(agentDir, "npm")), "--legacy-peer-deps", + "--no-audit", + "--no-fund", ], undefined, ); @@ -2702,6 +2744,8 @@ export default function(api) { api.registerTool({ name: "test", description: "te "--prefix", realpathSync(join(tempDir, ".pi", "npm")), "--legacy-peer-deps", + "--no-audit", + "--no-fund", ], undefined, ); From a6a7bac868d9a47cdda8cf26c2257047bb0307dc Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:19:37 +0100 Subject: [PATCH 025/271] fix(coding-agent): hidden thinking with an empty label renders no rows (#11470) When thinking was hidden and the hidden-thinking label was set to `""` (for example with `ctx.ui.setHiddenThinkingLabel("")`), every assistant message that contained thinking still drew blank rows: - thinking followed by a tool call drew 2 blank rows; - thinking followed by text drew 3 blank rows before the text. `AssistantMessageComponent.updateContent()` had three causes: - It counted hidden thinking as visible content when deciding on the leading and trailing spacers. - It always added the label component. With an empty label the theme's escape codes are still there, so the component rendered one blank row. **What changed** - A hidden thinking run with an empty label is now skipped entirely. Its run index is still advanced, so click overrides on other runs keep their indices. - Spacers are now added lazily, just before the next visible block. The result is the same as before: one spacer before the first visible block, and one after a visible thinking run that is followed by more visible content. But spacing now follows what actually renders. - Default behaviour (the "Thinking..." label, or expanded thinking) is unchanged. **Testing** - Added two tests to `test/assistant-message.test.ts`: - a hidden thinking run with an empty label renders nothing, both before a tool call (`[]`) and before text (`["", "answer"]`); - with the default label, rendering still gives `["", "Thinking...", "", "answer"]`. - The empty-label test fails without the fix and passes with it. - `npm run check` passes. The `test/assistant-message.test.ts` and `test/interactive-mode*` tests pass. - Rendering the component directly, with hidden thinking and an empty label: - thinking + tool call: before, 2 blank rows; after, `[]`; - thinking + text: before, 3 blank rows then the text; after, `["", " hi"]`. Fixes #11402 Requested by @badlogic Co-authored-by: triagent --- .../components/assistant-message.ts | 48 +++++++++++-------- .../test/assistant-message.test.ts | 42 ++++++++++++++++ 2 files changed, 71 insertions(+), 19 deletions(-) diff --git a/packages/coding-agent/src/modes/interactive/components/assistant-message.ts b/packages/coding-agent/src/modes/interactive/components/assistant-message.ts index c710c7df7..67bb71a3f 100644 --- a/packages/coding-agent/src/modes/interactive/components/assistant-message.ts +++ b/packages/coding-agent/src/modes/interactive/components/assistant-message.ts @@ -1,5 +1,13 @@ import type { AssistantMessage } from "@earendil-works/pi-ai"; -import { Container, Markdown, type MarkdownTheme, MouseRegion, Spacer, Text } from "@earendil-works/pi-tui"; +import { + type Component, + Container, + Markdown, + type MarkdownTheme, + MouseRegion, + Spacer, + Text, +} from "@earendil-works/pi-tui"; import type { MarkdownTransformer } from "../../../core/extensions/types.ts"; import { getMarkdownTheme, theme } from "../theme/theme.ts"; import { createMarkdownTransform } from "./markdown-transform.ts"; @@ -101,13 +109,17 @@ export class AssistantMessageComponent extends Container { // Clear content container this.contentContainer.clear(); - const hasVisibleContent = message.content.some( - (c) => (c.type === "text" && c.text.trim()) || (c.type === "thinking" && c.thinking.trim()), - ); - - if (hasVisibleContent) { - this.contentContainer.addChild(new Spacer(1)); - } + // Spacers are added lazily, right before the next visible content block. This puts a blank + // line before the first visible block and after each visible thinking run that is followed + // by more visible content, while hidden thinking runs without a label contribute nothing. + let needsSpacer = true; + const addVisibleChild = (child: Component) => { + if (needsSpacer) { + this.contentContainer.addChild(new Spacer(1)); + needsSpacer = false; + } + this.contentContainer.addChild(child); + }; // Render content in order let thinkingRunIndex = 0; @@ -116,7 +128,7 @@ export class AssistantMessageComponent extends Container { if (content.type === "text" && content.text.trim()) { // Assistant text messages with no background - trim the text // Set paddingY=0 to avoid extra spacing before tool executions - this.contentContainer.addChild( + addVisibleChild( new Markdown(content.text.trim(), this.outputPad, 0, this.markdownTheme, undefined, { transform: createMarkdownTransform( "assistant", @@ -145,14 +157,12 @@ export class AssistantMessageComponent extends Container { continue; } - // Add spacing only when another visible assistant content block follows. - // This avoids a superfluous blank line before separately-rendered tool execution blocks. - const hasVisibleContentAfter = message.content - .slice(i + 1) - .some((c) => (c.type === "text" && c.text.trim()) || (c.type === "thinking" && c.thinking.trim())); - const runIndex = thinkingRunIndex++; const hidden = this.thinkingVisibilityOverrides.get(runIndex) ?? this.hideThinkingBlock; + // A hidden run with an empty label renders nothing, including no spacing. + if (hidden && !this.hiddenThinkingLabel.trim()) { + continue; + } const thinkingComponent = hidden ? new Text(theme.italic(theme.fg("thinkingText", this.hiddenThinkingLabel)), this.outputPad, 0) : new Markdown( @@ -174,7 +184,7 @@ export class AssistantMessageComponent extends Container { ), }, ); - this.contentContainer.addChild( + addVisibleChild( new MouseRegion(thinkingComponent, (event) => { if (event.type !== "click" || event.button !== "left") return undefined; this.thinkingVisibilityOverrides.set(runIndex, !hidden); @@ -182,9 +192,9 @@ export class AssistantMessageComponent extends Container { return { handled: true }; }), ); - if (hasVisibleContentAfter) { - this.contentContainer.addChild(new Spacer(1)); - } + // Add spacing only when another visible assistant content block follows. + // This avoids a superfluous blank line before separately-rendered tool execution blocks. + needsSpacer = true; } } diff --git a/packages/coding-agent/test/assistant-message.test.ts b/packages/coding-agent/test/assistant-message.test.ts index ea2618aac..42f6e496d 100644 --- a/packages/coding-agent/test/assistant-message.test.ts +++ b/packages/coding-agent/test/assistant-message.test.ts @@ -92,6 +92,48 @@ describe("AssistantMessageComponent", () => { expect(rendered).toContain("answer"); }); + test("renders nothing for hidden thinking with an empty label", () => { + initTheme("dark"); + + const thinkingOnly = new AssistantMessageComponent( + createAssistantMessage( + [ + { type: "thinking", thinking: "reasoning" }, + { type: "toolCall", id: "tool-1", name: "read", arguments: { path: "file.txt" } }, + ], + { stopReason: "toolUse" }, + ), + true, + undefined, + "", + ); + expect(thinkingOnly.render(80)).toEqual([]); + + const withText = new AssistantMessageComponent( + createAssistantMessage([ + { type: "thinking", thinking: "reasoning" }, + { type: "text", text: "answer" }, + ]), + true, + undefined, + "", + ); + expect(withText.render(80).map((line) => stripAnsi(line).trim())).toEqual(["", "answer"]); + }); + + test("keeps label row and spacing for hidden thinking with a label", () => { + initTheme("dark"); + + const component = new AssistantMessageComponent( + createAssistantMessage([ + { type: "thinking", thinking: "reasoning" }, + { type: "text", text: "answer" }, + ]), + true, + ); + expect(component.render(80).map((line) => stripAnsi(line).trim())).toEqual(["", "Thinking...", "", "answer"]); + }); + test("collapses individual thinking runs when clicked", () => { initTheme("dark"); const component = new AssistantMessageComponent( From c1810bf70f5e75558f12bdbff51bf94921ae686b Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:27:28 +0100 Subject: [PATCH 026/271] feat(coding-agent): let inline ctx.ui.custom() components report blocked program status (#11471) Inline `ctx.ui.custom()` components never report pi as blocked through OSC 7501 program status. This is on purpose, because the same API hosts loaders. As a result, a custom dialog that waits for the user (an approval prompt, the `question.ts` example) shows up as `working`, and the terminal cannot tell that pi needs input. The extension dialog stack already stores a blocked status for each dialog and reports the visible one. This change adds an optional `blocked` option to `ctx.ui.custom()`: ```ts ctx.ui.custom(factory, { blocked: { kind: "question", message: "Pick a color" } }); ``` The inline branch of `showExtensionCustom()` passes it to `pushExtensionDialog()`. Components without the option behave as before. Overlays ignore the option because they are not part of the dialog stack. `docs/tui.md` documents the option. Testing: - Added a test to `test/interactive-mode-extension-dialogs.test.ts`: - A custom component that opts in reports `blocked`. - A plain custom component stacked on top of it clears the blocked status. - Closing the plain component restores the blocked status. - Closing the opted-in component clears it. - The test fails without the change to `interactive-mode.ts` and passes with it. - `npm run check`: biome and lint pass, and there are no type errors in coding-agent. The only errors are existing type errors in `packages/ai/test` from locally generated model data. They are not related to this change. Fixes #11428 Requested by @badlogic Co-authored-by: triagent --- packages/coding-agent/docs/tui.md | 2 + .../coding-agent/src/core/extensions/types.ts | 6 +++ .../src/modes/interactive/interactive-mode.ts | 3 +- ...interactive-mode-extension-dialogs.test.ts | 38 +++++++++++++++++++ 4 files changed, 48 insertions(+), 1 deletion(-) diff --git a/packages/coding-agent/docs/tui.md b/packages/coding-agent/docs/tui.md index 704a78633..c3fb30939 100644 --- a/packages/coding-agent/docs/tui.md +++ b/packages/coding-agent/docs/tui.md @@ -135,6 +135,8 @@ export default function (pi: ExtensionAPI) { This component does not display a text cursor, so it does not need `Focusable`. A component containing `Input` or `Editor` must implement `Focusable` and propagate its `focused` value to that child. +Custom components do not report pi as waiting for the user, because `ctx.ui.custom()` also hosts loaders and other non-blocking UI. When an inline component waits for an answer, pass `blocked: { kind: "question", message: "Pick a color" }` so terminals that support [program status](terminal-setup.md#program-status) show pi as blocked while the component is visible. `kind` is `"question"`, `"permission"`, or `"auth"`. Overlays ignore this option. + Pass `overlay: true` to draw above existing content. Overlay options control size, anchors, offsets, margins, and responsive visibility. An overlay handle can change focus or temporarily hide and show the overlay with `setHidden()` while the interaction remains active. Focused overlays retain input ownership across ordinary renders. If another component should receive input while an overlay remains visible, explicitly release or redirect focus through the handle. diff --git a/packages/coding-agent/src/core/extensions/types.ts b/packages/coding-agent/src/core/extensions/types.ts index b29e2d4b0..8bb7fbb5e 100644 --- a/packages/coding-agent/src/core/extensions/types.ts +++ b/packages/coding-agent/src/core/extensions/types.ts @@ -52,6 +52,7 @@ import type { KeyId, OverlayHandle, OverlayOptions, + ProgramStatus, TUI, } from "@earendil-works/pi-tui"; import type { Static, TSchema } from "typebox"; @@ -224,6 +225,11 @@ export interface ExtensionUIContext { overlayOptions?: OverlayOptions | (() => OverlayOptions); /** Called with the overlay handle after the overlay is shown. Use to control visibility. */ onHandle?: (handle: OverlayHandle) => void; + /** + * Report pi as blocked on user input (OSC 7501 program status) while this component is visible. + * Only applies to inline components, not overlays. + */ + blocked?: { kind: NonNullable; message: string }; }, ): Promise; diff --git a/packages/coding-agent/src/modes/interactive/interactive-mode.ts b/packages/coding-agent/src/modes/interactive/interactive-mode.ts index 7b3b45ba9..f6b0f7a58 100644 --- a/packages/coding-agent/src/modes/interactive/interactive-mode.ts +++ b/packages/coding-agent/src/modes/interactive/interactive-mode.ts @@ -2939,6 +2939,7 @@ export class InteractiveMode { overlay?: boolean; overlayOptions?: OverlayOptions | (() => OverlayOptions); onHandle?: (handle: OverlayHandle) => void; + blocked?: BlockedStatus; }, ): Promise { const savedText = this.editor.getText(); @@ -2992,7 +2993,7 @@ export class InteractiveMode { // Expose handle to caller for visibility control options?.onHandle?.(overlayHandle); } else { - this.pushExtensionDialog(c); + this.pushExtensionDialog(c, options?.blocked); } }) .catch((err) => { diff --git a/packages/coding-agent/test/interactive-mode-extension-dialogs.test.ts b/packages/coding-agent/test/interactive-mode-extension-dialogs.test.ts index 2a59f9189..91a9d1966 100644 --- a/packages/coding-agent/test/interactive-mode-extension-dialogs.test.ts +++ b/packages/coding-agent/test/interactive-mode-extension-dialogs.test.ts @@ -18,6 +18,7 @@ type DialogMethods = { showExtensionConfirm(title: string, message: string, opts?: { signal?: AbortSignal }): Promise; showExtensionCustom( factory: (tui: unknown, theme: unknown, keybindings: unknown, done: (result: T) => void) => Component, + options?: { blocked?: { kind: "permission" | "question" | "auth"; message: string } }, ): Promise; setCustomEditorComponent( factory: ((tui: unknown, theme: unknown, keybindings: unknown) => Component) | undefined, @@ -153,6 +154,43 @@ describe("InteractiveMode extension dialogs", () => { expect(context.focused).toBe(context.editor); }); + // Regression for #11428: inline custom dialogs could not report blocked program status. + it("reports blocked status for an inline custom component that opts in", async () => { + const context = createContext(); + const dones: Array<(result: string) => void> = []; + const first = context.showExtensionCustom( + (_tui, _theme, _keybindings, done) => { + dones.push(done); + return new Text("approval"); + }, + { blocked: { kind: "permission", message: "Approve?" } }, + ); + await vi.waitFor(() => expect(visible(context)).toContain("approval")); + expect(context.programStatus.setBlocked).toHaveBeenLastCalledWith("extension-dialog", { + kind: "permission", + message: "Approve?", + }); + + const loader = context.showExtensionCustom((_tui, _theme, _keybindings, done) => { + dones.push(done); + return new Text("loading"); + }); + await vi.waitFor(() => expect(visible(context)).toContain("loading")); + expect(context.programStatus.setBlocked).toHaveBeenLastCalledWith("extension-dialog", undefined); + + dones[1]("loaded"); + await expect(loader).resolves.toBe("loaded"); + expect(context.programStatus.setBlocked).toHaveBeenLastCalledWith("extension-dialog", { + kind: "permission", + message: "Approve?", + }); + + dones[0]("yes"); + await expect(first).resolves.toBe("yes"); + expect(context.focused).toBe(context.editor); + expect(context.programStatus.setBlocked).toHaveBeenLastCalledWith("extension-dialog", undefined); + }); + // Regression for #11431: replacing the editor while a dialog was open hid the dialog and left it unanswerable. it("keeps an open dialog visible when the editor component is replaced", async () => { const context = createContext(); From 531340f100c6b682a602657b699a0ac27d522c30 Mon Sep 17 00:00:00 2001 From: Christian Klotz Date: Tue, 27 Oct 2026 10:26:07 +0200 Subject: [PATCH 027/271] fix(ai): remove unused root bedrock-provider shims ./bedrock-provider has resolved to dist/ since a7559f01e (v0.59.0), and the root files are not in the published package. --- packages/ai/bedrock-provider.d.ts | 1 - packages/ai/bedrock-provider.js | 1 - 2 files changed, 2 deletions(-) delete mode 100644 packages/ai/bedrock-provider.d.ts delete mode 100644 packages/ai/bedrock-provider.js diff --git a/packages/ai/bedrock-provider.d.ts b/packages/ai/bedrock-provider.d.ts deleted file mode 100644 index a66eabee6..000000000 --- a/packages/ai/bedrock-provider.d.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./dist/bedrock-provider.js"; diff --git a/packages/ai/bedrock-provider.js b/packages/ai/bedrock-provider.js deleted file mode 100644 index a66eabee6..000000000 --- a/packages/ai/bedrock-provider.js +++ /dev/null @@ -1 +0,0 @@ -export * from "./dist/bedrock-provider.js"; From 8025c414d102e1658ddd61e9d79626e9a105b105 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Tue, 27 Oct 2026 08:54:53 +0000 Subject: [PATCH 028/271] feat(chord): SqliteExecutor.runCounted resolves with the rows a statement changed The Node driver takes the count from node:sqlite; the Durable Object driver reads SELECT changes() after the statement, since the cursor's rowsWritten also counts index writes (verified in workerd). run is unchanged and costs nothing more. --- packages/chord/CHANGELOG.md | 1 + packages/chord/src/sqlite/cloudflare.ts | 12 +++++++ packages/chord/src/sqlite/database.ts | 7 ++-- packages/chord/src/sqlite/node.ts | 4 +++ packages/chord/test/sqlite/cloudflare.test.ts | 11 ++++++- packages/chord/test/sqlite/helpers.ts | 32 +++++++++++++++++++ packages/chord/test/sqlite/node.test.ts | 14 ++++++++ .../chord/test/telemetry-sqlite-store.test.ts | 1 + packages/durable/CHANGELOG.md | 1 + packages/durable/test-workers/storage.test.ts | 19 +++++++++++ packages/durable/test/sqlite-facade.test.ts | 4 +++ 11 files changed, 103 insertions(+), 3 deletions(-) diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index 584aaddd0..28fd35e9d 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -6,6 +6,7 @@ - `traceIn(options, fn, context)` takes the context last (was `traceIn(context, options, fn)`), like the rest of the API. - Watch listeners take one change object before the context: `WatchHandle.start((change, context) => …)` with `change: WatchChange` = `{ value, ops, retired }`, and `NativeWatchHandle.start(({ value }, context) => …)` with `NativeWatchChange`. A change is plain data, those fields only; the frame's context is the second argument. `Watch.advance(change, context)` takes the same object, and a frame delivered alone, with nothing folded into it, reaches the listener as the object the source passed. Migration: `start(async (value, ops, ctx) => …)` becomes `start(async ({ value, ops }, ctx) => …)`, `start(async (value, ctx) => …)` on a native watch `start(async ({ value }, ctx) => …)`, and `advance(value, ops, ctx)` becomes `advance({ value, ops, retired: false }, ctx)`. +- `SqliteExecutor.runCounted(sql, ...params)` runs one statement like `run` and resolves with the rows it inserted, updated or deleted, as SQLite's `changes()` counts them (index writes and rows changed by triggers do not count). The Node driver takes it from `node:sqlite`; the Durable Object driver reads `SELECT changes()` after the statement (the cursor's `rowsWritten` also counts index writes), so `run` stays as cheap as before. Migration: custom `SqliteDatabase` and `SqliteExecutor` implementations add `runCounted`. - Retirement is a flag, not an end: `WatchEnd` loses `retired`, and `Watch.retire(context)` is removed. A source reports a retirement with `advance({ value, ops, retired: true }, context)`, whose value is what the address shows after it, and the watch goes on: the next call has `retired: true`, folded with every other pending frame, so a document created at the address afterwards arrives in the same or a later call as `[["r", value]]`. `endAfterDelivery()` tells a watch its source sends nothing more: it takes no more frames, delivers what is pending, and ends `ended` after its last call returned, at once when nothing is pending and no call runs; it stays attached until then, so the source's `end()` (such as at its close), `stop()`, cancellation, or a throwing listener still end it first. Migration: a listener that waited for `closed` to give `retired` or tested for a `null` value tests `change.retired`; a source that called `retire(ctx)` calls `advance({ value, ops, retired: true }, ctx)`, then `endAfterDelivery()` when its owner ends with the retirement. ### Added diff --git a/packages/chord/src/sqlite/cloudflare.ts b/packages/chord/src/sqlite/cloudflare.ts index 897a660b7..29d4345d0 100644 --- a/packages/chord/src/sqlite/cloudflare.ts +++ b/packages/chord/src/sqlite/cloudflare.ts @@ -103,6 +103,14 @@ class DurableObjectSqliteExecutor implements SqliteExecutor { this.storage.sql.exec(sql, ...bind(params)); } + async runCounted(sql: string, ...params: SqliteValue[]): Promise { + this.check(); + this.storage.sql.exec(sql, ...bind(params)); + // The cursor's `rowsWritten` is a billing count that includes index writes; SQLite's own count is exact. + const counted = this.storage.sql.exec("SELECT changes() AS changes").next(); + return counted.done ? 0 : Number(counted.value.changes); + } + async get(sql: string, ...params: SqliteValue[]): Promise { this.check(); const result = this.storage.sql.exec(sql, ...bind(params)).next(); @@ -151,6 +159,10 @@ export class DurableObjectSqliteDatabase extends DurableObjectSqliteExecutor imp return this.queue.run(() => super.run(sql, ...params)); } + override runCounted(sql: string, ...params: SqliteValue[]): Promise { + return this.queue.run(() => super.runCounted(sql, ...params)); + } + override get(sql: string, ...params: SqliteValue[]): Promise { return this.queue.run(() => super.get(sql, ...params)); } diff --git a/packages/chord/src/sqlite/database.ts b/packages/chord/src/sqlite/database.ts index fc5a814c6..6a8e6715b 100644 --- a/packages/chord/src/sqlite/database.ts +++ b/packages/chord/src/sqlite/database.ts @@ -5,12 +5,15 @@ export type SqliteValue = null | number | bigint | string | Uint8Array; * Asynchronous SQL operations shared by a database and its transaction handles. * * `exec` runs SQL text without bindings and may contain several statements. `run`, `get`, and `all` - * execute one statement with positional bindings. Adapters may cache prepared statements by SQL text, - * so callers pass values as bindings instead of interpolating them. + * execute one statement with positional bindings. `runCounted` runs one like `run` and resolves with the rows it + * inserted, updated or deleted, as SQLite's `changes()` counts them (index writes and rows changed by triggers do not + * count); it may cost a statement more than `run`, so callers that need no count use `run`. Adapters may cache + * prepared statements by SQL text, so callers pass values as bindings instead of interpolating them. */ export interface SqliteExecutor { exec(sql: string): Promise; run(sql: string, ...params: SqliteValue[]): Promise; + runCounted(sql: string, ...params: SqliteValue[]): Promise; get(sql: string, ...params: SqliteValue[]): Promise; all(sql: string, ...params: SqliteValue[]): Promise; } diff --git a/packages/chord/src/sqlite/node.ts b/packages/chord/src/sqlite/node.ts index b156477e8..202c37c4f 100644 --- a/packages/chord/src/sqlite/node.ts +++ b/packages/chord/src/sqlite/node.ts @@ -93,6 +93,10 @@ abstract class NodeSqliteExecutor implements SqliteExecutor { }); } + runCounted(sql: string, ...params: SqliteValue[]): Promise { + return this.runOperation(() => Number(this.statement(sql).run(...params).changes)); + } + get(sql: string, ...params: SqliteValue[]): Promise { return this.runOperation(() => this.statement(sql).get(...params) as T | undefined); } diff --git a/packages/chord/test/sqlite/cloudflare.test.ts b/packages/chord/test/sqlite/cloudflare.test.ts index 0e7521d25..41a270524 100644 --- a/packages/chord/test/sqlite/cloudflare.test.ts +++ b/packages/chord/test/sqlite/cloudflare.test.ts @@ -1,7 +1,7 @@ import { DatabaseSync } from "node:sqlite"; import { describe, expect, it } from "vitest"; import { DurableObjectSqliteDatabase } from "../../src/sqlite/cloudflare.ts"; -import { durableObjectStorage } from "./helpers.ts"; +import { durableObjectStorage, RUN_CHANGES, runChanges } from "./helpers.ts"; describe("Durable Object SQLite driver", () => { it("binds and returns BLOB values as Uint8Array", async () => { @@ -14,6 +14,15 @@ describe("Durable Object SQLite driver", () => { }); }); + it("reports the rows a statement changed, without index writes", async () => { + const database = new DurableObjectSqliteDatabase(durableObjectStorage()); + expect(await runChanges(database)).toEqual(RUN_CHANGES); + const inTransaction = await database.transaction((handle) => + handle.runCounted("INSERT INTO changes_probe VALUES (?, ?, ?)", "t", "t/1", 1), + ); + expect(inTransaction).toBe(1); + }); + it("reads only the cursor's first row for get", async () => { const operations: string[] = []; const database = new DurableObjectSqliteDatabase( diff --git a/packages/chord/test/sqlite/helpers.ts b/packages/chord/test/sqlite/helpers.ts index 042696e45..48e855eab 100644 --- a/packages/chord/test/sqlite/helpers.ts +++ b/packages/chord/test/sqlite/helpers.ts @@ -1,5 +1,6 @@ import { DatabaseSync } from "node:sqlite"; import type { DurableObjectSqliteStorage } from "../../src/sqlite/cloudflare.ts"; +import type { SqliteExecutor } from "../../src/sqlite/index.ts"; type SqlValue = ArrayBuffer | string | number | null; @@ -63,3 +64,34 @@ export function durableObjectStorage( }, }; } + +/** + * The rows each kind of statement changes, as `runCounted` reports them: on a table with two indexes, so a driver that counted + * index writes (a Durable Object cursor's `rowsWritten`) would report more. + */ +export async function runChanges(database: SqliteExecutor): Promise> { + await database.exec( + "CREATE TABLE changes_probe (id TEXT PRIMARY KEY, path TEXT NOT NULL, n INTEGER NOT NULL) STRICT, WITHOUT ROWID", + ); + await database.exec("CREATE INDEX changes_probe_path ON changes_probe (path, id)"); + await database.exec("CREATE INDEX changes_probe_n ON changes_probe (n, id)"); + for (let i = 0; i < 10; i++) await database.run("INSERT INTO changes_probe VALUES (?, ?, ?)", `k${i}`, `a/${i}`, i); + const changes = (sql: string, ...params: (string | number)[]) => database.runCounted(sql, ...params); + return { + insert: await changes("INSERT INTO changes_probe VALUES (?, ?, ?)", "x", "a/x", 99), + update: await changes("UPDATE changes_probe SET path = ? WHERE id = ?", "b/x", "x"), + none: await changes("UPDATE changes_probe SET n = 0 WHERE id = ?", "absent"), + bulk: await changes("UPDATE changes_probe SET path = 'c/' || substr(path, 3) WHERE path >= 'a/' AND path < 'a0'"), + upsert: await changes( + "INSERT INTO changes_probe VALUES (?, ?, ?) ON CONFLICT (id) DO UPDATE SET n = excluded.n", + "x", + "b/x", + 5, + ), + delete: await changes("DELETE FROM changes_probe WHERE id = ?", "x"), + bulkDelete: await changes("DELETE FROM changes_probe WHERE path >= 'c/' AND path < 'c0'"), + }; +} + +/** What `runChanges` reports on every driver. */ +export const RUN_CHANGES = { insert: 1, update: 1, none: 0, bulk: 10, upsert: 1, delete: 1, bulkDelete: 10 }; diff --git a/packages/chord/test/sqlite/node.test.ts b/packages/chord/test/sqlite/node.test.ts index 11098d1c5..c7b72b073 100644 --- a/packages/chord/test/sqlite/node.test.ts +++ b/packages/chord/test/sqlite/node.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import type { SqliteExecutor } from "../../src/sqlite/index.ts"; import { openNodeSqliteDatabase } from "../../src/sqlite/node.ts"; +import { RUN_CHANGES, runChanges } from "./helpers.ts"; describe("Node SQLite driver", () => { it("configures connection defaults and accepts explicit settings", async () => { @@ -18,6 +19,19 @@ describe("Node SQLite driver", () => { } }); + it("reports the rows a statement changed, without index writes", async () => { + const database = await openNodeSqliteDatabase(":memory:"); + try { + expect(await runChanges(database)).toEqual(RUN_CHANGES); + const inTransaction = await database.transaction((handle) => + handle.runCounted("INSERT INTO changes_probe VALUES (?, ?, ?)", "t", "t/1", 1), + ); + expect(inTransaction).toBe(1); + } finally { + await database.close(); + } + }); + it("commits work done through the transaction handle and closes idempotently", async () => { const database = await openNodeSqliteDatabase(":memory:"); await database.transaction(async (transaction) => { diff --git a/packages/chord/test/telemetry-sqlite-store.test.ts b/packages/chord/test/telemetry-sqlite-store.test.ts index 4da872546..34e142c57 100644 --- a/packages/chord/test/telemetry-sqlite-store.test.ts +++ b/packages/chord/test/telemetry-sqlite-store.test.ts @@ -59,6 +59,7 @@ describe("SQLite telemetry reads", () => { const counted: SqliteExecutor = { exec: (sql) => tx.exec(sql), run: (sql, ...params) => tx.run(sql, ...params), + runCounted: (sql, ...params) => tx.runCounted(sql, ...params), get: (sql: string, ...params: SqliteValue[]) => tx.get(sql, ...params), all: (sql: string, ...params: SqliteValue[]) => { queries++; diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index ed67bba7b..0f584aaed 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -4,6 +4,7 @@ ### Breaking Changes +- `SqliteExecutor` (re-exported from Chord) gains `runCounted(sql, ...params)`, resolving with the rows the statement changed; custom `SqliteDatabase` implementations add it. - `Storage.commit(writes, adopt, context)` takes an `adopt(seq)` callback, the Session's in-memory adoption and publication of the batch: a storage calls it exactly once, synchronously, once the batch is durable and before its next commit starts, then resolves with `seq`, and never for a commit that fails before the batch is durable. Memory, SQLite, JSONL, and IndexedDB storages do; custom storages and wrappers must call or forward it, and the storage conformance suite checks it. A commit that rejects after its adoption is published and fails the Session without rolling the adopted changes back. - `Storage.mintId()`'s contract is narrowed to IDs greater than every ID minted since the storage was opened and every ID stored, also across reopen. The bundled storages already behave so. - Watches use Chord's `WatchHandle` and `WatchEnd` (Durable re-exports both, with `WatchChange` and `NativeWatchChange`). End reasons are `stopped`, `cancelled`, `ended` (was `session_closed`), `listener_error`, and `failed` (was `session_failed`); `retired` is gone (see below). An ended watch drops its listener and every reference into its Session, so a stopped watch that is still referenced keeps only its last value. diff --git a/packages/durable/test-workers/storage.test.ts b/packages/durable/test-workers/storage.test.ts index 9f45ec600..70df76dea 100644 --- a/packages/durable/test-workers/storage.test.ts +++ b/packages/durable/test-workers/storage.test.ts @@ -52,3 +52,22 @@ describe("Durable Object SQLite schema", () => { }); }); }); + +describe("Durable Object SQLite driver in workerd", () => { + // The cursor's `rowsWritten` counts index writes too (3 for this insert, 20 for the bulk update); `runCounted` must not. + it("reports the rows a statement changed, without index writes", async () => { + await runInDurableObject(fresh(), async (_instance, state) => { + const database = new DurableObjectSqliteDatabase(state.storage); + await database.exec("CREATE TABLE probe (id TEXT PRIMARY KEY, path TEXT NOT NULL, n INTEGER NOT NULL) STRICT, WITHOUT ROWID"); + await database.exec("CREATE INDEX probe_path ON probe (path, id)"); + await database.exec("CREATE INDEX probe_n ON probe (n, id)"); + for (let i = 0; i < 10; i++) await database.run("INSERT INTO probe VALUES (?, ?, ?)", `k${i}`, `a/${i}`, i); + expect(await database.runCounted("INSERT INTO probe VALUES (?, ?, ?)", "x", "a/x", 99)).toBe(1); + expect(await database.runCounted("UPDATE probe SET n = 0 WHERE id = ?", "absent")).toBe(0); + expect( + await database.runCounted("UPDATE probe SET path = 'c/' || substr(path, 3) WHERE path >= 'a/' AND path < 'a0'"), + ).toBe(10); + expect(await database.runCounted("DELETE FROM probe WHERE path >= 'c/' AND path < 'c0'")).toBe(10); + }); + }); +}); diff --git a/packages/durable/test/sqlite-facade.test.ts b/packages/durable/test/sqlite-facade.test.ts index ab8b9da8e..7ba878edb 100644 --- a/packages/durable/test/sqlite-facade.test.ts +++ b/packages/durable/test/sqlite-facade.test.ts @@ -35,6 +35,10 @@ class ControlledSettlementDatabase implements SqliteDatabase { return this.delegate.run(sql, ...params); } + runCounted(sql: string, ...params: SqliteValue[]): Promise { + return this.delegate.runCounted(sql, ...params); + } + get(sql: string, ...params: SqliteValue[]): Promise { return this.delegate.get(sql, ...params); } From 973b74c55fcf096ad5976f8d8850c147ee2e62d4 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Tue, 27 Oct 2026 08:58:50 +0000 Subject: [PATCH 029/271] fix(chord): keep the workerd runCounted probe row outside the bulk range --- packages/durable/test-workers/storage.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/durable/test-workers/storage.test.ts b/packages/durable/test-workers/storage.test.ts index 70df76dea..210014285 100644 --- a/packages/durable/test-workers/storage.test.ts +++ b/packages/durable/test-workers/storage.test.ts @@ -62,7 +62,8 @@ describe("Durable Object SQLite driver in workerd", () => { await database.exec("CREATE INDEX probe_path ON probe (path, id)"); await database.exec("CREATE INDEX probe_n ON probe (n, id)"); for (let i = 0; i < 10; i++) await database.run("INSERT INTO probe VALUES (?, ?, ?)", `k${i}`, `a/${i}`, i); - expect(await database.runCounted("INSERT INTO probe VALUES (?, ?, ?)", "x", "a/x", 99)).toBe(1); + // Outside the bulk statements' range below. + expect(await database.runCounted("INSERT INTO probe VALUES (?, ?, ?)", "x", "b/x", 99)).toBe(1); expect(await database.runCounted("UPDATE probe SET n = 0 WHERE id = ?", "absent")).toBe(0); expect( await database.runCounted("UPDATE probe SET path = 'c/' || substr(path, 3) WHERE path >= 'a/' AND path < 'a0'"), From 177e2ad7e41e85c16c859f844a0569a3c76f2666 Mon Sep 17 00:00:00 2001 From: Victor Navarro Date: Tue, 27 Oct 2026 10:40:41 +0100 Subject: [PATCH 030/271] fix(ai): use OpenCode protocol-specific endpoints (#11434) closes #11132 pi sends every opencode and opencode-go request to the shared base (`https://opencode.ai/inference/v1`, `/inference/go/v1`), Jev included. this switches them to the documented protocol-specific endpoints, which is also what models.dev lists per model: | api | before | after | |---|---|---| | openai-completions / openai-responses | `/inference/v1` | `/inference/openai/v1` | | anthropic-messages | `/inference` | `/inference/anthropic` (SDK appends `/v1/messages`) | | google-generative-ai | `/inference/v1` | `/inference/google/v1beta` | | jev-1.13 / jev-1.13-free | `/inference/v1` | `/inference/systemone/v1` | go uses the same paths under `/inference/go/...`. changes: base URLs in `generate-models.ts`, the jev URL expectations in `test/classifier-models.test.ts`, and a stale comment in `providers/opencode.ts`. `models.generated.ts` has no diff since the catalog data is gitignored. verified with the regenerated catalog: - all 111 opencode/opencode-go models hit their new URL, none 404 - real requests with an OpenCode key returned replies on anthropic-messages, google-generative-ai, openai-responses, openai-completions and jev - go returned 403 "subscription required" from the new paths (my key has no Go sub), so routing works there too `npm run check` passes. `./test.sh` passes for ai and coding-agent; the only remaining failures were timeouts that pass in isolation and pi-env tests that need cargo locally. --- _Drafted with AI assistance, reviewed by me._ --- packages/ai/scripts/generate-models.ts | 18 +++++++++--------- packages/ai/src/providers/opencode.ts | 2 +- packages/ai/test/classifier-models.test.ts | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 96d324653..b556d2505 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -2272,23 +2272,23 @@ async function loadModelsDevData(): Promise[]> { if (npm === "@ai-sdk/openai") { api = "openai-responses"; - baseUrl = `${variant.basePath}/v1`; + baseUrl = `${variant.basePath}/openai/v1`; compat = { sessionAffinityFormat: "openai-nosession" }; } else if (npm === "@ai-sdk/anthropic") { api = "anthropic-messages"; // Anthropic SDK appends /v1/messages to baseURL - baseUrl = variant.basePath; + baseUrl = `${variant.basePath}/anthropic`; } else if (npm === "@ai-sdk/google") { api = "google-generative-ai"; - baseUrl = `${variant.basePath}/v1`; + baseUrl = `${variant.basePath}/google/v1beta`; } else if (npm === "@ai-sdk/alibaba") { api = "openai-completions"; - baseUrl = `${variant.basePath}/v1`; + baseUrl = `${variant.basePath}/openai/v1`; compat = { cacheControlFormat: "anthropic" }; } else { // null, undefined, or @ai-sdk/openai-compatible api = "openai-completions"; - baseUrl = `${variant.basePath}/v1`; + baseUrl = `${variant.basePath}/openai/v1`; } if (variant.provider === "opencode" && modelId === "grok-build-0.1") { @@ -2304,11 +2304,11 @@ async function loadModelsDevData(): Promise[]> { // Fix known mismatches between models.dev npm data and actual // OpenCode Go endpoint behaviour. models.dev reports these models // as @ai-sdk/anthropic, but OpenCode Go serves them through the - // OpenAI-compatible /v1/chat/completions path. + // OpenAI-compatible /openai/v1/chat/completions path. if (variant.provider === "opencode-go") { if (modelId === "qwen3.5-plus" || modelId === "qwen3.6-plus") { api = "openai-completions"; - baseUrl = `${variant.basePath}/v1`; + baseUrl = `${variant.basePath}/openai/v1`; // Qwen/DashScope uses enable_thinking at the top level. compat = { ...(compat ?? {}), thinkingFormat: "qwen" }; } @@ -2681,7 +2681,7 @@ const OPENCODE_CLASSIFIER_MODELS: ClassifierModel<"typesafe-system-one">[] = [ name: "Jev 1.13", api: "typesafe-system-one", provider: "opencode", - baseUrl: "https://opencode.ai/inference/v1", + baseUrl: "https://opencode.ai/inference/systemone/v1", input: ["text"], cost: { input: 0.042, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 32000, @@ -2692,7 +2692,7 @@ const OPENCODE_CLASSIFIER_MODELS: ClassifierModel<"typesafe-system-one">[] = [ name: "Jev 1.13 Free", api: "typesafe-system-one", provider: "opencode", - baseUrl: "https://opencode.ai/inference/v1", + baseUrl: "https://opencode.ai/inference/systemone/v1", input: ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 32000, diff --git a/packages/ai/src/providers/opencode.ts b/packages/ai/src/providers/opencode.ts index 7b3aae419..4e87cedb5 100644 --- a/packages/ai/src/providers/opencode.ts +++ b/packages/ai/src/providers/opencode.ts @@ -22,7 +22,7 @@ export function opencodeProvider(): Provider { "openai-completions": withOpenCodeSessionHeader(openAICompletionsApi()), "openai-responses": withOpenCodeSessionHeader(openAIResponsesApi()), }, - // OpenCode Zen serves TypeSafe's System One protocol at /zen/v1/systemone. + // OpenCode Zen serves TypeSafe's System One protocol at /inference/systemone/v1/systemone. classifiers: { "typesafe-system-one": typesafeSystemOneApi() }, }); } diff --git a/packages/ai/test/classifier-models.test.ts b/packages/ai/test/classifier-models.test.ts index 4a5592501..cae1df4a5 100644 --- a/packages/ai/test/classifier-models.test.ts +++ b/packages/ai/test/classifier-models.test.ts @@ -130,8 +130,8 @@ describe("Models with classifier models", () => { it.each([ ["vercel-ai-gateway", "typesafe-ai/jev", "https://ai-gateway.vercel.sh/typesafe/v1/systemone"], - ["opencode", "jev-1.13", "https://opencode.ai/inference/v1/systemone"], - ["opencode", "jev-1.13-free", "https://opencode.ai/inference/v1/systemone"], + ["opencode", "jev-1.13", "https://opencode.ai/inference/systemone/v1/systemone"], + ["opencode", "jev-1.13-free", "https://opencode.ai/inference/systemone/v1/systemone"], ])("routes %s Jev (%s) to its TypeSafe-compatible endpoint", async (provider, id, url) => { const models = builtinModels(); const jev = models.getModelOfType("classifier", provider, id); From db14038bd847c5bad57de217e0ac0c5ad1ae60ba Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Tue, 27 Oct 2026 11:28:28 +0000 Subject: [PATCH 031/271] fix(chord): bound the Node SQLite driver's prepared-statement cache The driver kept every statement it ever prepared. It now keeps the 256 most recently used per connection; below half the limit a hit stays one lookup. --- packages/chord/CHANGELOG.md | 1 + packages/chord/src/sqlite/node.ts | 24 +++++++++++++++++++----- packages/chord/test/sqlite/node.test.ts | 19 ++++++++++++++++++- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index 28fd35e9d..fd2fac9ff 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -26,6 +26,7 @@ ### Fixed +- The Node SQLite driver keeps the 256 most recently used prepared statements per connection instead of every statement it ever prepared, so callers generating SQL of unbounded shapes (a query builder) no longer grow memory without bound. - Fixed append ops from the delta tracker and `diffRevisions()` keeping the whole string they were cut from alive (V8 represents a slice as a view of its source), so a delta chain, a watch backlog or a replica built from appends to a growing text held one copy of the text per commit ([#9265](https://github.com/earendil-works/pi/issues/9265)) ## [1.4.0] - 2026-10-21 diff --git a/packages/chord/src/sqlite/node.ts b/packages/chord/src/sqlite/node.ts index 202c37c4f..269249027 100644 --- a/packages/chord/src/sqlite/node.ts +++ b/packages/chord/src/sqlite/node.ts @@ -68,9 +68,14 @@ class SerialOperationQueue { } } +/** How many prepared statements one connection keeps. */ +const STATEMENT_CACHE_LIMIT = 256; + /** * Executes SQL on one connection. Prepared statements are cached per connection by SQL text, so the - * database and its transaction handles share them across transactions. + * database and its transaction handles share them across transactions. The cache keeps the + * `STATEMENT_CACHE_LIMIT` most recently used statements, so callers generating SQL of unbounded shapes + * cannot grow it without bound. */ abstract class NodeSqliteExecutor implements SqliteExecutor { protected readonly database: DatabaseSync; @@ -108,11 +113,20 @@ abstract class NodeSqliteExecutor implements SqliteExecutor { protected abstract runOperation(operation: () => T): Promise; private statement(sql: string): StatementSync { - let statement = this.statements.get(sql); - if (statement === undefined) { - statement = this.database.prepare(sql); - this.statements.set(sql, statement); + const statements = this.statements; + let statement = statements.get(sql); + if (statement !== undefined) { + // Recency matters only once evictions come near: a small cache keeps one lookup per statement. + if (statements.size > STATEMENT_CACHE_LIMIT / 2) { + statements.delete(sql); + statements.set(sql, statement); + } + return statement; } + statement = this.database.prepare(sql); + statements.set(sql, statement); + // A Map iterates in insertion order: its first key is the least recently used statement. + if (statements.size > STATEMENT_CACHE_LIMIT) statements.delete(statements.keys().next().value!); return statement; } } diff --git a/packages/chord/test/sqlite/node.test.ts b/packages/chord/test/sqlite/node.test.ts index c7b72b073..96ad09a96 100644 --- a/packages/chord/test/sqlite/node.test.ts +++ b/packages/chord/test/sqlite/node.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { DatabaseSync } from "node:sqlite"; +import { describe, expect, it, vi } from "vitest"; import type { SqliteExecutor } from "../../src/sqlite/index.ts"; import { openNodeSqliteDatabase } from "../../src/sqlite/node.ts"; import { RUN_CHANGES, runChanges } from "./helpers.ts"; @@ -19,6 +20,22 @@ describe("Node SQLite driver", () => { } }); + it("keeps the most recently used prepared statements and prepares evicted ones again", async () => { + const database = await openNodeSqliteDatabase(":memory:"); + const prepare = vi.spyOn(DatabaseSync.prototype, "prepare"); + try { + for (let i = 0; i < 300; i++) await database.get(`SELECT ${i} AS n`); + const prepared = prepare.mock.calls.length; + expect(await database.get("SELECT 299 AS n")).toEqual({ n: 299 }); + expect(prepare.mock.calls.length).toBe(prepared); + expect(await database.get("SELECT 0 AS n")).toEqual({ n: 0 }); + expect(prepare.mock.calls.length).toBe(prepared + 1); + } finally { + prepare.mockRestore(); + await database.close(); + } + }); + it("reports the rows a statement changed, without index writes", async () => { const database = await openNodeSqliteDatabase(":memory:"); try { From 81d6759fc90439857b3ed8a020abe34ccdc8d3eb Mon Sep 17 00:00:00 2001 From: David Brailovsky Date: Tue, 27 Oct 2026 14:25:23 +0100 Subject: [PATCH 032/271] fix(ai): declare GPT-5.4 and GPT-5.5 tools at the top level GPT-5.5 and older namespace tools loaded through additional_tools or tool search and reject replayed calls without the namespace, so switching to them after another model's tool calls failed with 400. Drop dynamic tool loading for these models on OpenAI and OpenCode; keep mid-conversation system messages. closes #11415 --- packages/ai/CHANGELOG.md | 1 + packages/ai/scripts/generate-models.ts | 17 +++++++++++------ .../ai/test/openai-responses-compat.test.ts | 4 ++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 882ce5609..9d1b49309 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -28,6 +28,7 @@ - Fixed pi-messages streams accepting a `contentIndex` that is not the next or an existing content block, such as `__proto__` or a huge index - Fixed `pi-ai` CLI auth file being readable by other users. It is now written with mode `0600` - Fixed OpenCode DeepSeek V4.1 Flash sending `prompt_cache_retention`, which the OpenCode gateway rejects, with long cache retention +- Fixed GPT-5.4 and GPT-5.5 failing with `Missing namespace for function_call` after switching from another model that called tools. On the OpenAI and OpenCode providers they now declare their tools in the top-level tool list instead of loading them through `additional_tools` or tool search ([#11415](https://github.com/earendil-works/pi/issues/11415)) ## [1.4.0] - 2026-10-21 diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index b556d2505..966604358 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -366,11 +366,10 @@ const ANT_LING_RING_THINKING_LEVEL_MAP = { const BEDROCK_INFERENCE_PROFILE_ONLY_MODEL_IDS = new Set(["anthropic.claude-opus-5"]); const MODELS_DEV_OPENAI_UNSUPPORTED_MODEL_IDS = new Set(["gpt-5.6"]); +// GPT-5.5 and older put tools loaded through tool search or additional_tools in a namespace named +// after the tool and reject replayed calls without it, so history from other models fails there +// (#11415). Like Codex, dynamic tool loading therefore starts with GPT-5.6, which does not do this. const OPENAI_TOOL_SEARCH_MODEL_IDS = new Set([ - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-pro", - "gpt-5.5", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", @@ -381,7 +380,13 @@ const OPENAI_TOOL_SEARCH_MODEL_IDS = new Set([ "gpt-6.1-terra", ]); const OPENAI_ADDITIONAL_TOOLS_MODEL_IDS = OPENAI_TOOL_SEARCH_MODEL_IDS; -const OPENAI_MID_CONVO_SYSTEM_MESSAGE_MODEL_IDS = OPENAI_TOOL_SEARCH_MODEL_IDS; +const OPENAI_MID_CONVO_SYSTEM_MESSAGE_MODEL_IDS = new Set([ + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-pro", + "gpt-5.5", + ...OPENAI_TOOL_SEARCH_MODEL_IDS, +]); const OPENAI_CODEX_ADDITIONAL_TOOLS_MODEL_IDS = new Set([ "gpt-5.6-sol", "gpt-5.6-terra", @@ -985,7 +990,7 @@ function applyOpenAIResponsesTranscriptMetadata(model: Model): void { model.compat = { ...(model.compat as OpenAIResponsesCompat | undefined), supportsMidConvoSystemMessages: true, - ...(isProxiedResponses ? { supportsAdditionalTools: true } : {}), + ...(isProxiedResponses && OPENAI_ADDITIONAL_TOOLS_MODEL_IDS.has(model.id) ? { supportsAdditionalTools: true } : {}), }; } diff --git a/packages/ai/test/openai-responses-compat.test.ts b/packages/ai/test/openai-responses-compat.test.ts index 3b61686e8..ec7783313 100644 --- a/packages/ai/test/openai-responses-compat.test.ts +++ b/packages/ai/test/openai-responses-compat.test.ts @@ -119,7 +119,7 @@ describe("openai-responses provider defaults", () => { ); const stream = streamOpenAIResponses( - getModel("openai", "gpt-5.4"), + getModel("openai", "gpt-5.6-luna"), normalizeContext({ messages: [ { @@ -149,7 +149,7 @@ describe("openai-responses provider defaults", () => { if (event.type === "done" || event.type === "error") break; } - // gpt-5.4 declares its tools in additional_tools items; tool_choice applies to them. + // gpt-5.6-luna declares its tools in additional_tools items; tool_choice applies to them. expect(capturedPayload).toMatchObject({ tool_choice: "required", input: [{ type: "additional_tools", tools: [expect.objectContaining({ name: "ping" })] }, expect.anything()], From 842c73170a3658d4e3b39fcc84690211ed503e09 Mon Sep 17 00:00:00 2001 From: David Brailovsky Date: Tue, 27 Oct 2026 14:41:57 +0100 Subject: [PATCH 033/271] fix(ai): fall back on Bedrock strict tool limits Retry Bedrock strict tool limit rejections once with strict: "prefer" tools non-strict, as the Anthropic provider already does. Share the limit error messages and the prefer check between both providers. closes #11416 --- packages/ai/CHANGELOG.md | 1 + packages/ai/src/api/anthropic-messages.ts | 18 +- .../ai/src/api/bedrock-converse-stream.ts | 88 ++++++--- packages/ai/src/api/constrained-sampling.ts | 21 +++ .../test/bedrock-strict-tool-fallback.test.ts | 173 ++++++++++++++++++ 5 files changed, 265 insertions(+), 36 deletions(-) create mode 100644 packages/ai/test/bedrock-strict-tool-fallback.test.ts diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 9d1b49309..60baa1c16 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -29,6 +29,7 @@ - Fixed `pi-ai` CLI auth file being readable by other users. It is now written with mode `0600` - Fixed OpenCode DeepSeek V4.1 Flash sending `prompt_cache_retention`, which the OpenCode gateway rejects, with long cache retention - Fixed GPT-5.4 and GPT-5.5 failing with `Missing namespace for function_call` after switching from another model that called tools. On the OpenAI and OpenCode providers they now declare their tools in the top-level tool list instead of loading them through `additional_tools` or tool search ([#11415](https://github.com/earendil-works/pi/issues/11415)) +- Fixed Bedrock requests failing when `strict: "prefer"` tools exceed Claude's strict tool limits, such as more than 16 optional parameters across all strict tools; requests now retry once with preferred tools non-strict while keeping `strict: "require"` tools strict, as the Anthropic provider already does ([#11416](https://github.com/earendil-works/pi/issues/11416)) ## [1.4.0] - 2026-10-21 diff --git a/packages/ai/src/api/anthropic-messages.ts b/packages/ai/src/api/anthropic-messages.ts index 53764b406..9d6a8bdb7 100644 --- a/packages/ai/src/api/anthropic-messages.ts +++ b/packages/ai/src/api/anthropic-messages.ts @@ -59,6 +59,8 @@ import { import { getJsonSchemaToolParameters, + isStrictToolLimitErrorMessage, + prefersJsonSchemaStrictSampling, resolveJsonSchemaStrictSampling, type UnsupportedStrictSchemaKeywordCheck, } from "./constrained-sampling.ts"; @@ -674,11 +676,7 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = ( !(error instanceof Anthropic.APIError) || error.status !== 400 || error.type !== "invalid_request_error" || - ![ - "The compiled grammar is too large", - "Too many strict tools (", - "Schemas contains too many parameters with union types (", - ].some((message) => error.message.includes(message)) + !isStrictToolLimitErrorMessage(error.message) ) { throw error; } @@ -1185,10 +1183,9 @@ function buildParams( disablePreferredStrictTools, ); if ( - definitions.some((definition, index) => { - const config = tools[index].constrainedSampling; - return definition.strict === true && config && config.type === "json_schema" && config.strict === "prefer"; - }) + definitions.some( + (definition, index) => definition.strict === true && prefersJsonSchemaStrictSampling(tools[index]), + ) ) { hasPreferredStrictTools = true; } @@ -1630,9 +1627,8 @@ function convertTools( if (!tools) return []; return tools.map((tool, index) => { - const config = tool.constrainedSampling; const strict = - disablePreferredStrictTools && config && config.type === "json_schema" && config.strict === "prefer" + disablePreferredStrictTools && prefersJsonSchemaStrictSampling(tool) ? undefined : resolveJsonSchemaStrictSampling(tool, supportsStrictTools, isAnthropicStrictUnsupportedKeyword); const parameters = getJsonSchemaToolParameters(tool, strict); diff --git a/packages/ai/src/api/bedrock-converse-stream.ts b/packages/ai/src/api/bedrock-converse-stream.ts index 7a7740f06..0ed88be24 100644 --- a/packages/ai/src/api/bedrock-converse-stream.ts +++ b/packages/ai/src/api/bedrock-converse-stream.ts @@ -13,6 +13,7 @@ import { type ContentBlockStopEvent, ConversationRole, ConverseStreamCommand, + type ConverseStreamCommandOutput, type ConverseStreamMetadataEvent, ImageFormat, type Message, @@ -64,7 +65,12 @@ import { type TranscriptContext, withoutInitialSystemMessage, } from "../utils/transcript.ts"; -import { getJsonSchemaToolParameters, resolveJsonSchemaStrictSampling } from "./constrained-sampling.ts"; +import { + getJsonSchemaToolParameters, + isStrictToolLimitErrorMessage, + prefersJsonSchemaStrictSampling, + resolveJsonSchemaStrictSampling, +} from "./constrained-sampling.ts"; import { adjustMaxTokensForThinking, buildBaseOptions, @@ -269,29 +275,55 @@ export const stream: StreamFunction<"bedrock-converse-stream", BedrockOptions> = const inferenceMaxTokens = options.maxTokens ?? (isAnthropicClaudeModel(model) ? model.maxTokens : undefined); const initialSystemMessage = getInitialSystemMessage(normalizedContext.messages); const initialSystemPrompt = initialSystemMessage ? getSystemMessageText(initialSystemMessage) : undefined; - let commandInput = { - modelId: model.id, - messages: convertMessages(normalizedContext, model, cacheRetention, options.env), - system: buildSystemPrompt(initialSystemPrompt, model, cacheRetention, options.env), - inferenceConfig: { - ...(inferenceMaxTokens !== undefined && { maxTokens: inferenceMaxTokens }), - ...(options.temperature !== undefined && { temperature: options.temperature }), - }, - toolConfig: convertToolConfig( + let disablePreferredStrictTools = false; + let response: ConverseStreamCommandOutput; + for (;;) { + const tools = convertToolConfig( getCurrentTools(normalizedContext.messages), options.toolChoice, supportsStrictMode, - ), - additionalModelRequestFields: buildAdditionalModelRequestFields(model, options), - ...(options.requestMetadata !== undefined && { requestMetadata: options.requestMetadata }), - }; - const nextCommandInput = await options?.onPayload?.(commandInput, model); - if (nextCommandInput !== undefined) { - commandInput = nextCommandInput as typeof commandInput; + disablePreferredStrictTools, + ); + let commandInput = { + modelId: model.id, + messages: convertMessages(normalizedContext, model, cacheRetention, options.env), + system: buildSystemPrompt(initialSystemPrompt, model, cacheRetention, options.env), + inferenceConfig: { + ...(inferenceMaxTokens !== undefined && { maxTokens: inferenceMaxTokens }), + ...(options.temperature !== undefined && { temperature: options.temperature }), + }, + toolConfig: tools.toolConfig, + additionalModelRequestFields: buildAdditionalModelRequestFields(model, options), + ...(options.requestMetadata !== undefined && { requestMetadata: options.requestMetadata }), + }; + const nextCommandInput = await options?.onPayload?.(commandInput, model); + if (nextCommandInput !== undefined) { + commandInput = nextCommandInput as typeof commandInput; + } + try { + response = await client.send(new ConverseStreamCommand(commandInput), { abortSignal: options.signal }); + break; + } catch (error) { + // Bedrock checks the tool schemas before it starts the stream, so a retry never repeats streamed output. + if ( + options.signal?.aborted || + disablePreferredStrictTools || + !tools.hasPreferredStrictTools || + !(error instanceof BedrockRuntimeServiceException) || + error.name !== "ValidationException" || + !isStrictToolLimitErrorMessage(error.message) + ) { + throw error; + } + disablePreferredStrictTools = true; + appendAssistantMessageDiagnostic(output, { + type: "bedrock_strict_tools_fallback", + timestamp: Date.now(), + error: { message: error.message }, + details: { reason: "strict_tool_limits_exceeded" }, + }); + } } - const command = new ConverseStreamCommand(commandInput); - - const response = await client.send(command, { abortSignal: options.signal }); responseRequestId = normalizeDiagnosticValue(response.$metadata.requestId); if (!observedRawResponse && response.$metadata.httpStatusCode !== undefined) { const responseHeaders: Record = {}; @@ -1155,12 +1187,18 @@ function convertToolConfig( tools: Tool[] | undefined, toolChoice: BedrockOptions["toolChoice"], supportsStrictMode: boolean, -): ToolConfiguration | undefined { - if (!tools?.length) return undefined; - if (toolChoice === "none") return undefined; + disablePreferredStrictTools: boolean, +): { toolConfig: ToolConfiguration | undefined; hasPreferredStrictTools: boolean } { + if (!tools?.length || toolChoice === "none") return { toolConfig: undefined, hasPreferredStrictTools: false }; + let hasPreferredStrictTools = false; const bedrockTools: BedrockTool[] = tools.map((tool) => { - const strict = resolveJsonSchemaStrictSampling(tool, supportsStrictMode); + const preferred = prefersJsonSchemaStrictSampling(tool); + const strict = + disablePreferredStrictTools && preferred + ? undefined + : resolveJsonSchemaStrictSampling(tool, supportsStrictMode); + if (strict === true && preferred) hasPreferredStrictTools = true; return { toolSpec: { name: tool.name, @@ -1185,7 +1223,7 @@ function convertToolConfig( } } - return { tools: bedrockTools, toolChoice: bedrockToolChoice }; + return { toolConfig: { tools: bedrockTools, toolChoice: bedrockToolChoice }, hasPreferredStrictTools }; } function mapStopReason(reason: string | undefined): { stopReason: StopReason; errorMessage?: string } { diff --git a/packages/ai/src/api/constrained-sampling.ts b/packages/ai/src/api/constrained-sampling.ts index 62e03a105..adb729432 100644 --- a/packages/ai/src/api/constrained-sampling.ts +++ b/packages/ai/src/api/constrained-sampling.ts @@ -139,6 +139,27 @@ export function makeStrictJsonSchema( return cloned; } +/** Returns true for a tool that asks for strict JSON-schema sampling only when possible (`strict: "prefer"`). */ +export function prefersJsonSchemaStrictSampling(tool: Tool): boolean { + const config = tool.constrainedSampling; + return !!config && config.type === "json_schema" && config.strict === "prefer"; +} + +/** + * Claude rejects (on the Anthropic API and on Bedrock) a request whose strict tools exceed its limits with one of these + * messages. Observed limits: 20 strict tools, and 16 optional or union-typed parameters across all strict tools. + */ +const STRICT_TOOL_LIMIT_ERROR_MESSAGES = [ + "The compiled grammar is too large", + "Too many strict tools (", + "Schemas contains too many parameters with union types (", +]; + +/** Returns true when a provider error message says the request's strict tools exceed the provider's limits. */ +export function isStrictToolLimitErrorMessage(message: string): boolean { + return STRICT_TOOL_LIMIT_ERROR_MESSAGES.some((limitMessage) => message.includes(limitMessage)); +} + export function getJsonSchemaToolParameters(tool: Tool, strict: boolean | undefined): Tool["parameters"] { return (strict === true ? makeStrictJsonSchema(tool.parameters) : tool.parameters) as Tool["parameters"]; } diff --git a/packages/ai/test/bedrock-strict-tool-fallback.test.ts b/packages/ai/test/bedrock-strict-tool-fallback.test.ts new file mode 100644 index 000000000..c3853b461 --- /dev/null +++ b/packages/ai/test/bedrock-strict-tool-fallback.test.ts @@ -0,0 +1,173 @@ +import { Type } from "typebox"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const bedrockMock = vi.hoisted(() => ({ + inputs: [] as unknown[], + respond: undefined as ((attempt: number) => Promise) | undefined, +})); + +vi.mock("@aws-sdk/client-bedrock-runtime", () => { + class BedrockRuntimeServiceException extends Error { + constructor(options: { name: string; message: string }) { + super(options.message); + this.name = options.name; + } + } + + class BedrockRuntimeClient { + send(command: { input: unknown }): Promise { + bedrockMock.inputs.push(structuredClone(command.input)); + return bedrockMock.respond!(bedrockMock.inputs.length); + } + } + + class ConverseStreamCommand { + readonly input: unknown; + + constructor(input: unknown) { + this.input = input; + } + } + + return { + BedrockRuntimeClient, + BedrockRuntimeServiceException, + ConverseStreamCommand, + StopReason: { END_TURN: "end_turn" }, + CachePointType: { DEFAULT: "default" }, + CacheTTL: { ONE_HOUR: "ONE_HOUR" }, + ConversationRole: { ASSISTANT: "assistant", USER: "user" }, + ImageFormat: { JPEG: "jpeg", PNG: "png", GIF: "gif", WEBP: "webp" }, + ToolResultStatus: { ERROR: "error", SUCCESS: "success" }, + }; +}); + +import { BedrockRuntimeServiceException } from "@aws-sdk/client-bedrock-runtime"; +import { stream as streamBedrock } from "../src/api/bedrock-converse-stream.ts"; +import type { Model, Tool } from "../src/types.ts"; +import { normalizeContext } from "../src/utils/transcript.ts"; + +const model: Model<"bedrock-converse-stream"> = { + id: "us.anthropic.claude-opus-5-5", + name: "Claude Opus 5.5 (US)", + api: "bedrock-converse-stream", + provider: "amazon-bedrock", + baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200000, + maxTokens: 100, + compat: { supportsStrictMode: true }, +}; + +// Observed from live Bedrock with 20 optional parameters across strict tools. +const unionLimitMessage = + "The model returned the following errors: Schemas contains too many parameters with union types (20 parameters with type arrays or anyOf). This causes exponential compilation cost. Reduce the number of nullable or union-typed parameters (limit: 16 parameters with unions)."; + +type ToolSpecInput = { toolSpec: { name: string; strict?: boolean; inputSchema: { json: unknown } } }; +type CommandInput = { toolConfig: { tools: ToolSpecInput[] }; requestMetadata?: Record }; + +function tool(name: string, strict?: "prefer" | "require"): Tool { + return { + name, + description: "Look up a value", + parameters: Type.Object({ value: Type.String(), optional: Type.Optional(Type.Number()) }), + ...(strict ? { constrainedSampling: { type: "json_schema", strict } } : {}), + }; +} + +function sdkError(name: string, message: string): Error { + return new BedrockRuntimeServiceException({ name, message, $fault: "client", $metadata: { httpStatusCode: 400 } }); +} + +function successResponse(): Promise { + return Promise.resolve({ + $metadata: { httpStatusCode: 200 }, + stream: (async function* () { + yield { messageStart: { role: "assistant" } }; + yield { messageStop: { stopReason: "end_turn" } }; + })(), + }); +} + +function run(tools: Tool[], onPayload?: (payload: unknown) => unknown) { + return streamBedrock( + model, + normalizeContext({ tools, messages: [{ role: "user", content: "Hello", timestamp: 1 }] }), + { cacheRetention: "none", onPayload }, + ).result(); +} + +// Regression tests for https://github.com/earendil-works/pi/issues/11416. +describe("Bedrock preferred strict tool fallback", () => { + beforeEach(() => { + bedrockMock.inputs = []; + }); + + it("retries a strict tool limit rejection with preferred tools non-strict and keeps required tools strict", async () => { + bedrockMock.respond = (attempt) => + attempt === 1 ? Promise.reject(sdkError("ValidationException", unionLimitMessage)) : successResponse(); + const onPayload = vi.fn((payload: unknown) => ({ ...(payload as object), requestMetadata: { team: "a" } })); + + const result = await run([tool("preferred", "prefer"), tool("required", "require"), tool("ordinary")], onPayload); + + expect(result.stopReason).toBe("stop"); + expect(onPayload).toHaveBeenCalledTimes(2); + const [first, second] = bedrockMock.inputs as CommandInput[]; + expect(first.toolConfig.tools[0].toolSpec).toMatchObject({ + strict: true, + inputSchema: { + json: { + required: ["value", "optional"], + properties: { optional: { anyOf: [{ type: "number" }, { type: "null" }] } }, + }, + }, + }); + expect(second.toolConfig.tools[0].toolSpec.strict).toBeUndefined(); + expect(second.toolConfig.tools[0].toolSpec.inputSchema.json).toEqual({ + type: "object", + properties: { value: { type: "string" }, optional: { type: "number" } }, + required: ["value"], + }); + expect(second.toolConfig.tools[1]).toEqual(first.toolConfig.tools[1]); + expect(second.toolConfig.tools[1].toolSpec.strict).toBe(true); + expect(second.toolConfig.tools[2]).toEqual(first.toolConfig.tools[2]); + expect(second.requestMetadata).toEqual({ team: "a" }); + expect(result.diagnostics).toEqual([ + expect.objectContaining({ + type: "bedrock_strict_tools_fallback", + error: { message: unionLimitMessage }, + details: { reason: "strict_tool_limits_exceeded" }, + }), + ]); + }); + + it("does not retry without preferred strict tools", async () => { + bedrockMock.respond = () => Promise.reject(sdkError("ValidationException", unionLimitMessage)); + + const result = await run([tool("required", "require"), tool("ordinary")]); + + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toBe(`Validation error: ${unionLimitMessage}`); + expect(bedrockMock.inputs).toHaveLength(1); + }); + + it("does not retry other validation errors", async () => { + bedrockMock.respond = () => Promise.reject(sdkError("ValidationException", "messages: field required")); + + const result = await run([tool("preferred", "prefer")]); + + expect(result.stopReason).toBe("error"); + expect(bedrockMock.inputs).toHaveLength(1); + }); + + it("retries only once", async () => { + bedrockMock.respond = () => Promise.reject(sdkError("ValidationException", unionLimitMessage)); + + const result = await run([tool("preferred", "prefer")]); + + expect(result.stopReason).toBe("error"); + expect(bedrockMock.inputs).toHaveLength(2); + }); +}); From 269afc66abfd5fdd37ff0eedfe2b749b6050a420 Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:38:06 +0100 Subject: [PATCH 034/271] fix(coding-agent): pass thinking level to branch summarization (#11482) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `/tree` "Summarize" option ignored the session's thinking level. `AgentSession` already passes `thinkingLevel` into the `generateBranchSummary` options, but `generateBranchSummary` never read it, so the request was built without `reasoning`. For reasoning models on OpenAI Responses and Azure Responses, a missing `reasoning` becomes `effort: thinkingLevelMap.off ?? "none"`. Models that don't accept `"none"` then reject the branch summary with a 400 error: ``` Unsupported value: 'none' is not supported with the 'gpt-6-astra-…' model. ``` This change adds `thinkingLevel` to `GenerateBranchSummaryOptions`. It sets `reasoning` on the request when the model supports reasoning and the level is not `off`, the same rule compaction uses in `createSummarizationOptions`. **Testing** - Added a test to `packages/coding-agent/test/branch-summarization.test.ts` that records the stream options for three cases: a reasoning model at `xhigh`, a reasoning model at `off`, and a non-reasoning model. The expected `reasoning` values are `["xhigh", undefined, undefined]`. Without the fix, the test fails and every case gets `undefined`. With the fix, it passes. - Ran `branch-summarization`, `agent-session-branching`, `branch-summary-extensions` and `compaction-summary-reasoning`: all pass. - `npm run check` passes. - Not tested against a real provider endpoint. Fixes #11444 Requested by @davidbrai Co-authored-by: triagent --- .../core/compaction/branch-summarization.ts | 8 +++++++- .../test/branch-summarization.test.ts | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/packages/coding-agent/src/core/compaction/branch-summarization.ts b/packages/coding-agent/src/core/compaction/branch-summarization.ts index f1db25e14..69bd75133 100644 --- a/packages/coding-agent/src/core/compaction/branch-summarization.ts +++ b/packages/coding-agent/src/core/compaction/branch-summarization.ts @@ -5,7 +5,7 @@ * a summary of the branch being left so context isn't lost. */ -import type { AgentMessage, StreamFn } from "@earendil-works/pi-agent-core"; +import type { AgentMessage, StreamFn, ThinkingLevel } from "@earendil-works/pi-agent-core"; import type { RetryCallbacks, RetryPolicy } from "@earendil-works/pi-ai"; import { contentText, normalizeContext } from "@earendil-works/pi-ai"; import type { Model, SimpleStreamOptions, Usage } from "@earendil-works/pi-ai/compat"; @@ -75,6 +75,8 @@ export interface GenerateBranchSummaryOptions { env?: Record; /** Abort signal for cancellation */ signal: AbortSignal; + /** Thinking level for reasoning-capable models */ + thinkingLevel?: ThinkingLevel; /** Optional custom instructions for summarization */ customInstructions?: string; /** If true, customInstructions replaces the default prompt instead of being appended */ @@ -300,6 +302,7 @@ export async function generateBranchSummary( headers, env, signal, + thinkingLevel, customInstructions, replaceInstructions, reserveTokens = 16384, @@ -348,6 +351,9 @@ export async function generateBranchSummary( // so transient stream drops reuse the configured retry policy. const context = normalizeContext({ systemPrompt: SUMMARIZATION_SYSTEM_PROMPT, messages: summarizationMessages }); const requestOptions: SimpleStreamOptions = { apiKey, headers, env, signal }; + if (model.reasoning && thinkingLevel && thinkingLevel !== "off") { + requestOptions.reasoning = thinkingLevel; + } const response = await completeSummarization(model, context, requestOptions, streamFn, retry, callbacks); // Check if aborted or errored diff --git a/packages/coding-agent/test/branch-summarization.test.ts b/packages/coding-agent/test/branch-summarization.test.ts index 7018e969f..ade25dbfd 100644 --- a/packages/coding-agent/test/branch-summarization.test.ts +++ b/packages/coding-agent/test/branch-summarization.test.ts @@ -44,6 +44,26 @@ function response(content: AssistantMessage["content"]): AssistantMessage { } describe("branch summarization", () => { + it("passes the thinking level only for reasoning-capable models", async () => { + const seen: Array = []; + const streamFn: StreamFn = (_model, _context, options) => { + seen.push(options?.reasoning); + const stream = createAssistantMessageEventStream(); + queueMicrotask(() => + stream.push({ type: "done", reason: "stop", message: response([{ type: "text", text: "summary" }]) }), + ); + return stream; + }; + const reasoningModel = { ...model, reasoning: true }; + const signal = new AbortController().signal; + + await generateBranchSummary(entries, { model: reasoningModel, signal, streamFn, thinkingLevel: "xhigh" }); + await generateBranchSummary(entries, { model: reasoningModel, signal, streamFn, thinkingLevel: "off" }); + await generateBranchSummary(entries, { model, signal, streamFn, thinkingLevel: "high" }); + + expect(seen).toEqual(["xhigh", undefined, undefined]); + }); + it("does not override tool choice for branch summaries", async () => { let requestOptions: SimpleStreamOptions | undefined; const streamFn: StreamFn = (_model, _context, options) => { From 807a5c051f86bcbe707c59d6b954e4529058bcef Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:58:08 +0100 Subject: [PATCH 035/271] fix(coding-agent): keep session memory in line with the file when a write fails (#11486) When writing a session entry failed (ENOSPC, EBUSY, ETIMEDOUT on network drives), `SessionManager._appendEntry` had already added the entry to memory and made it the current leaf. The process kept running, and the next entry written to disk pointed to a parent that was not in the file. On resume, pi follows the history only back to that gap: everything before it disappears. In #11460 that also dropped the first system message, which declares the tools, so resume turned off every built-in tool. `_appendEntry` now undoes the in-memory append if writing fails, then rethrows. The error reaches the caller as before; later entries chain to the last entry actually on disk. Reports of this problem: - #11460: ENOSPC; resume dropped all built-in tools. - #10166: failure injected with EBUSY; the next entry on disk points to a missing parent. - #10661: ETIMEDOUT on Windows + NFS; resumed sessions showed only the last few messages. That report also describes NUL holes in the file, which this change does not address. **Testing** - Two tests in `test/session-manager/file-operations.test.ts` make writes fail with real file permissions: a read-only session file (failed append) and a read-only session directory (failed first file creation). They check that the entry is not kept, the leaf is unchanged, the next entry chains to the last written one, and the reopened session's branch contains both entries. Both fail without the change and pass with it. - `test/session-manager/` and `test/default-tools-setting.test.ts` pass (140 tests). - `npm run check` passes. Fixes #11460 Requested by @davidbrai Co-authored-by: triagent --- .../coding-agent/src/core/session-manager.ts | 12 +++++- .../session-manager/file-operations.test.ts | 39 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/packages/coding-agent/src/core/session-manager.ts b/packages/coding-agent/src/core/session-manager.ts index 57f9655ba..89431f2a3 100644 --- a/packages/coding-agent/src/core/session-manager.ts +++ b/packages/coding-agent/src/core/session-manager.ts @@ -1189,10 +1189,20 @@ export class SessionManager { } private _appendEntry(entry: SessionEntry): void { + const previousLeafId = this.leafId; this.fileEntries.push(entry); this.byId.set(entry.id, entry); this.leafId = entry.id; - this._persist(entry); + try { + this._persist(entry); + } catch (error) { + // Keep memory in line with the file: an entry that was not written must not become the + // parent of later entries, or the file would hold entries pointing to a missing parent. + this.fileEntries.pop(); + this.byId.delete(entry.id); + this.leafId = previousLeafId; + throw error; + } } /** Append a message as child of current leaf, then advance leaf. Returns entry id. diff --git a/packages/coding-agent/test/session-manager/file-operations.test.ts b/packages/coding-agent/test/session-manager/file-operations.test.ts index 8353ccdb7..ac6441490 100644 --- a/packages/coding-agent/test/session-manager/file-operations.test.ts +++ b/packages/coding-agent/test/session-manager/file-operations.test.ts @@ -1,6 +1,7 @@ import { constants as bufferConstants } from "buffer"; import { appendFileSync, + chmodSync, closeSync, existsSync, mkdirSync, @@ -471,4 +472,42 @@ describe("SessionManager session file creation", () => { expect(readSessionFileRoles(session.getSessionFile()!)).toEqual(["session", "user", "custom", "assistant"]); }); + + // #10166, #10661, #11460: a failed write must not leave later entries pointing to a missing parent + it("does not keep an entry in memory when writing it fails", () => { + const session = SessionManager.create(tempDir, tempDir); + const firstId = session.appendMessage(userMsg("first question")); + const file = session.getSessionFile()!; + + chmodSync(file, 0o444); + expect(() => session.appendMessage(assistantMsg("lost answer"))).toThrow(); + expect(session.getLeafId()).toBe(firstId); + expect(session.getEntries().map((entry) => entry.id)).toEqual([firstId]); + + chmodSync(file, 0o644); + const secondId = session.appendMessage(assistantMsg("second answer")); + expect(session.getEntry(secondId)?.parentId).toBe(firstId); + + const reopened = SessionManager.open(file, tempDir); + expect(reopened.getBranch().map((entry) => entry.id)).toEqual([firstId, secondId]); + }); + + it("does not keep an entry in memory when creating the file fails", () => { + const sessionDir = join(tempDir, "sessions"); + mkdirSync(sessionDir); + const session = SessionManager.create(tempDir, sessionDir); + const modelChangeId = session.appendModelChange("anthropic", "claude-sonnet-4-5"); + + chmodSync(sessionDir, 0o555); + try { + expect(() => session.appendMessage(userMsg("first question"))).toThrow(); + } finally { + chmodSync(sessionDir, 0o755); + } + expect(session.getLeafId()).toBe(modelChangeId); + expect(session.getEntries().map((entry) => entry.id)).toEqual([modelChangeId]); + + session.appendMessage(userMsg("second question")); + expect(readSessionFileRoles(session.getSessionFile()!)).toEqual(["session", "model_change", "user"]); + }); }); From cd1ea342aea61c6c77127c7a249763dc5227c98d Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:09:08 +0100 Subject: [PATCH 036/271] fix(tui): fall back to raw LaTeX when display math is wider than the content width (#11487) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Display math (`$$…$$`, `\[…\]`) renders as several 2D rows: numerator, fraction bar and denominator are separate lines. When the layout was wider than the Markdown content width, `render()` wrapped each row with `wrapTextWithAnsi` on its own. Only the overflowing row broke, so the rows below moved down and the equation no longer lined up. For example, `dx` landed between the fraction bar and the denominators. The `latexBlock` case in `Markdown.renderToken()` now checks the rendered layout against the available width. If any row is too wide, it uses the raw LaTeX source, the same fallback already used when parsing fails. The raw source then wraps like normal text. Layouts that fit render exactly as before. **Testing** - Added a regression test in `packages/tui/test/markdown.test.ts` using the equation from the issue. At width 40 it renders the 2D layout unchanged. At width 30 it falls back to the raw `\[ … \]` source, with no fraction bars and no line wider than 30. The test fails without the fix and passes with it. - `node --test test/markdown.test.ts`: 93 pass. `node --test test/latex.test.ts`: 111 pass. - `npm run check` reports no errors in the touched files. The only errors are 6 type errors in `packages/ai/test`, caused by stale generated model data, and they are the same on `main` without this change. - Reproduction at width 30, before the fix: ``` ∂² u ∂² u n 1 1 ──── + ──── = ∑ ── +∫ f(x) dx ∂ x² ∂ y² i=1 i² 0 ``` After the fix, the raw source, wrapped: ``` \[ \frac{\partial^2 u}{\partial x^2} + \frac{\partial^2 ... \] ``` Fixes #11483 Requested by @badlogic Co-authored-by: triagent --- packages/tui/CHANGELOG.md | 1 + packages/tui/src/components/markdown.ts | 10 +++++++--- packages/tui/test/markdown.test.ts | 23 +++++++++++++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/packages/tui/CHANGELOG.md b/packages/tui/CHANGELOG.md index bb7634ec6..23cb8dc2f 100644 --- a/packages/tui/CHANGELOG.md +++ b/packages/tui/CHANGELOG.md @@ -11,6 +11,7 @@ - Fixed fullscreen double-click selection splitting paths and tokens at `~`, `.`, `\`, and `+`. Word characters now match iTerm2's defaults ([#10980](https://github.com/earendil-works/pi/issues/10980)) - Fixed `fuzzyMatch()` ranking a query scattered over an earlier part of the text above a contiguous match later in it. It now also scores each contiguous occurrence of the query and keeps the best alignment ([#11220](https://github.com/earendil-works/pi/issues/11220), [#11296](https://github.com/earendil-works/pi/pull/11296)) - Fixed `ProcessTerminal.setTitle()` writing control characters from the title, which could end the title sequence and send the rest to the terminal as escape sequences +- Fixed display math wider than the content width being wrapped row by row, which split fractions and sums apart. It now falls back to the raw LaTeX source ([#11483](https://github.com/earendil-works/pi/issues/11483)) ## [1.4.0] - 2026-10-21 diff --git a/packages/tui/src/components/markdown.ts b/packages/tui/src/components/markdown.ts index 9194c2251..a3b526d73 100644 --- a/packages/tui/src/components/markdown.ts +++ b/packages/tui/src/components/markdown.ts @@ -536,10 +536,14 @@ export class Markdown implements Component { case "latexBlock": { const latexToken = token as LatexToken; - const rendered = + let rendered = !latexToken.pending && this.options.renderLatex !== false - ? (renderLatex(latexToken.text, { display: true }) ?? latexToken.raw.trim()) - : latexToken.raw.trim(); + ? renderLatex(latexToken.text, { display: true }) + : undefined; + // A 2D layout cannot be wrapped row by row; fall back to the raw source when it does not fit. + if (rendered === undefined || rendered.split("\n").some((line) => visibleWidth(line) > width)) { + rendered = latexToken.raw.trim(); + } for (const line of rendered.split("\n")) { lines.push(this.applyDefaultStyle(line)); } diff --git a/packages/tui/test/markdown.test.ts b/packages/tui/test/markdown.test.ts index 38e6ceb5a..b20d84dfe 100644 --- a/packages/tui/test/markdown.test.ts +++ b/packages/tui/test/markdown.test.ts @@ -966,6 +966,29 @@ Inline \(A_{sf}\)`, ); }); + // Regression test for https://github.com/earendil-works/pi/issues/11483 + it("falls back to raw source when display math is wider than the content width", () => { + const source = [ + "\\[", + String.raw`\frac{\partial^2 u}{\partial x^2} + \frac{\partial^2 u}{\partial y^2} = \sum_{i=1}^{n} \frac{1}{i^2} + \int_0^1 f(x)\,dx`, + "\\]", + ].join("\n"); + const render = (width: number) => + new Markdown(source, 0, 0, defaultMarkdownTheme).render(width).map((line) => stripAnsi(line).trimEnd()); + + assert.deepStrictEqual(render(40), [ + "∂² u ∂² u n 1 1", + "──── + ──── = ∑ ── +∫ f(x) dx", + "∂ x² ∂ y² i=1 i² 0", + ]); + + const narrow = render(30); + assert.strictEqual(narrow[0], "\\["); + assert.strictEqual(narrow[narrow.length - 1], "\\]"); + assert.ok(!narrow.some((line) => line.includes("─"))); + assert.ok(narrow.every((line) => line.length <= 30)); + }); + // Regression tests for https://github.com/earendil-works/pi/issues/11159 describe("unclosed display math before tables (#11159)", () => { const table = ["| A | B |", "|---|---|", "| 1 | 2 |"]; From 479d6e5e298821088cfd77566a139097e73a888b Mon Sep 17 00:00:00 2001 From: Michael Yu Date: Tue, 27 Oct 2026 23:16:49 +0800 Subject: [PATCH 037/271] fix(env): include stderr in pi-env startup errors (#10716) When the daemon exits before `hello`, the error has only the exit code. For example, with `loginShell: true` and a profile that aborts on a missing file: ``` pi-env exited with code 3 before it was ready ``` `sshConnection`'s probe and deploy steps already report ssh's stderr, so this is about the launch itself: a login profile, a daemon that can't execute, or a `Connection` created with its own command, which has no probe or deploy step, so auth and network errors also end up here. This keeps a short stderr tail until `hello` and adds it to startup errors: ``` pi-env exited with code 3 before it was ready remote stderr: profile: /opt/toolchain/env.sh: No such file or directory ``` The tail is capped at 20 lines / 2000 chars, with terminal escapes and control characters stripped. Ready sessions are unchanged. I hit it while testing `loginShell` against an sshd in Docker. Tests are in `test/connection.test.ts`, and `npm run check` and `./test.sh` pass. --- packages/env/src/connection.ts | 50 +++++- packages/env/test/connection.test.ts | 223 +++++++++++++++++++++++++++ 2 files changed, 268 insertions(+), 5 deletions(-) create mode 100644 packages/env/test/connection.test.ts diff --git a/packages/env/src/connection.ts b/packages/env/src/connection.ts index 484e7662c..527d90b19 100644 --- a/packages/env/src/connection.ts +++ b/packages/env/src/connection.ts @@ -1,5 +1,6 @@ import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process"; import { randomBytes } from "node:crypto"; +import { stripVTControlCharacters } from "node:util"; /** Frame types (docs/protocol.md). */ const REQUEST = 1; @@ -15,6 +16,9 @@ const PING_INTERVAL_MS = 5000; const SILENCE_LIMIT_MS = 30_000; /** How long starting the daemon and its `hello` may take. */ const START_TIMEOUT_MS = 60_000; +/** How much stderr a startup error shows: short, because a tool error can reach the model. */ +const STDERR_TAIL_CHARS = 2000; +const STDERR_TAIL_LINES = 20; export type Json = Record; @@ -131,6 +135,24 @@ function closed(): RemoteError { return new RemoteError({ code: "unknown", message: "Connection closed" }); } +/** + * `message` plus the last lines of `stderr`. The remote machine wrote that text, so terminal escapes and control + * characters are removed. + */ +function withStderr(message: string, stderr: string): string { + let text = ""; + for (const char of stripVTControlCharacters(stderr.replace(/\r\n?/g, "\n"))) { + const code = char.charCodeAt(0); + if (char === "\n" || char === "\t" || (code >= 0x20 && (code < 0x7f || code > 0x9f))) text += char; + } + const lines = text + .split("\n") + .map((line) => line.trimEnd()) + .filter((line) => line !== ""); + if (lines.length === 0) return message; + return `${message}\nremote stderr:\n${lines.slice(-STDERR_TAIL_LINES).join("\n")}`; +} + /** One running daemon. */ class Session { readonly id: number; @@ -144,6 +166,7 @@ class Session { lastSeen = Date.now(); timer: ReturnType | undefined; live = true; + stderrTail = ""; constructor(id: number, child: ChildProcessWithoutNullStreams, token: string) { this.id = id; @@ -281,6 +304,7 @@ export class Connection { const child = spawn(program, [...args, "serve", "--token", token], { stdio: ["pipe", "pipe", "pipe"] }); const session = new Session(++this.#sessions, child, token); this.#session = session; + let ready = false; let startTimer: ReturnType | undefined; const failed = new Promise((_resolve, reject) => { startTimer = setTimeout( @@ -290,13 +314,20 @@ export class Connection { child.once("error", (error) => reject(new RemoteError({ code: "spawn_error", message: error.message, lost: true })), ); - child.once("exit", (code) => reject(lost(`pi-env exited with code ${code} before it was ready`))); + child.once("close", (code) => reject(lost(`pi-env exited with code ${code} before it was ready`))); }); failed.catch(() => {}); child.stdout.on("data", (chunk: Buffer) => this.#onData(session, chunk)); - child.stderr.on("data", (chunk: Buffer) => this.#options.onLog?.(chunk.toString("utf8"))); + child.stderr.setEncoding("utf8").on("data", (text: string) => { + if (!ready) session.stderrTail = (session.stderrTail + text).slice(-STDERR_TAIL_CHARS); + this.#options.onLog?.(text); + }); child.on("error", () => this.#teardown(session, lost("pi-env connection lost"))); - child.on("exit", () => this.#teardown(session, lost("pi-env connection lost"))); + // A ready session fails as soon as the daemon exits, even while a process it started keeps a pipe open. + child.on("exit", () => { + if (ready) this.#teardown(session, lost("pi-env connection lost")); + }); + child.on("close", () => this.#teardown(session, lost("pi-env connection lost"))); child.stdin.on("error", () => {}); session.timer = setInterval(() => { this.#write(session, frame(PING, 0, {})); @@ -308,10 +339,19 @@ export class Connection { try { const { json } = await Promise.race([this.#send(session, "hello", { protocol: 1 }, {}), failed]); if (json.protocol !== 1) throw lost(`Unsupported protocol ${json.protocol}`); + ready = true; + session.stderrTail = ""; return { info: json as unknown as RemoteInfo, session }; } catch (error) { - this.#teardown(session, error instanceof Error ? error : lost(String(error))); - throw error; + // Add the captured stderr to whatever ended the start; only a pre-ready exit waits for it to drain. + const failure = + error instanceof RemoteError + ? new RemoteError({ ...error.fields, message: withStderr(error.message, session.stderrTail) }) + : error instanceof Error + ? error + : lost(String(error)); + this.#teardown(session, failure); + throw failure; } finally { clearTimeout(startTimer); } diff --git a/packages/env/test/connection.test.ts b/packages/env/test/connection.test.ts new file mode 100644 index 000000000..8b6eb35d6 --- /dev/null +++ b/packages/env/test/connection.test.ts @@ -0,0 +1,223 @@ +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { Connection, isConnectionLost, RemoteError } from "../src/connection.ts"; + +const connections: Connection[] = []; +afterAll(() => { + for (const connection of connections) connection.close(); +}); + +/** A connection whose "daemon" is a Node script; the script receives `serve --token ` as its arguments. */ +function connect(script: string | (() => string), onLog?: (text: string) => void): Connection { + const connection = new Connection({ + command: async () => [process.execPath, "-e", typeof script === "string" ? script : script()], + ...(onLog === undefined ? {} : { onLog }), + }); + connections.push(connection); + return connection; +} + +/** A script that writes `stderr` and exits with `code` once the write is flushed. */ +function failing(stderr: string, code = 1): string { + return `process.stderr.write(${JSON.stringify(stderr)}, () => process.exit(${code}));`; +} + +async function startError(connection: Connection): Promise { + try { + await connection.info(); + } catch (error) { + if (error instanceof Error) return error; + throw error; + } + throw new Error("The start did not fail"); +} + +/** + * Speaks the daemon's sync line and framing. `hello` gets `helloReply` (a result, or an error frame). The next request + * makes it exit while a process it started keeps stdout and stderr open for a few seconds. + */ +function fakeDaemon(helloReply: "result" | "error"): string { + return ` +const { spawn } = require("node:child_process"); +const token = process.argv[process.argv.indexOf("--token") + 1]; +process.stdout.write("PI-ENV " + token + "\\n"); +function frame(type, id, json) { + const body = Buffer.from(JSON.stringify(json)); + const header = Buffer.alloc(13); + header.writeUInt32BE(9 + body.length, 0); + header.writeUInt8(type, 4); + header.writeUInt32BE(id, 5); + header.writeUInt32BE(body.length, 9); + return Buffer.concat([header, body]); +} +let buffered = Buffer.alloc(0); +let requests = 0; +process.stdin.on("data", (chunk) => { + buffered = Buffer.concat([buffered, chunk]); + while (buffered.length >= 4 && buffered.length >= 4 + buffered.readUInt32BE(0)) { + const length = buffered.readUInt32BE(0); + const type = buffered.readUInt8(4); + const id = buffered.readUInt32BE(5); + buffered = buffered.subarray(4 + length); + if (type !== 1) continue; + if (++requests === 1) { + const info = { protocol: 1, version: "test", os: "linux", arch: "x86_64", home: "/", tmpdir: "/tmp", + separator: "/", cwd: "/", driveCwds: {}, pid: process.pid }; + const refusal = { code: "EINVAL", message: "hello refused", path: "/daemon", extra: 1 }; + process.stdout.write(${JSON.stringify(helloReply)} === "result" ? frame(2, id, info) : frame(3, id, refusal)); + continue; + } + spawn(process.execPath, ["-e", "setTimeout(() => {}, 3000)"], { stdio: ["ignore", "inherit", "inherit"] }).unref(); + process.exit(0); + } +}); +`; +} + +describe("Connection start failures", () => { + it("includes the end of stderr when the daemon exits before it is ready", async () => { + const error = await startError(connect(failing("ssh: Permission denied (publickey).\n", 255))); + expect(isConnectionLost(error)).toBe(true); + expect(error.message).toBe( + "pi-env exited with code 255 before it was ready\nremote stderr:\nssh: Permission denied (publickey).", + ); + }); + + it("keeps the message unchanged when the daemon wrote nothing printable to stderr", async () => { + expect((await startError(connect("process.exit(3);"))).message).toBe( + "pi-env exited with code 3 before it was ready", + ); + expect((await startError(connect(failing(" \n\u0007\u001b[0m\n", 3)))).message).toBe( + "pi-env exited with code 3 before it was ready", + ); + }); + + it("drops terminal escapes and C0 and C1 control characters, keeping other text", async () => { + const stderr = + "\u001b[31mred\u001b[0m\r\nbell\u0007 del\u007f\tc1:\u0085\u009d end\nÜberprüfung fehlgeschlagen ✗\n\n"; + const error = await startError(connect(failing(stderr))); + expect(error.message).toBe( + "pi-env exited with code 1 before it was ready\nremote stderr:\nred\nbell del\tc1: end\nÜberprüfung fehlgeschlagen ✗", + ); + }); + + it("keeps only the last 20 lines of stderr", async () => { + const lines = Array.from({ length: 30 }, (_, index) => `line ${index}`); + const error = await startError(connect(failing(`${lines.join("\n")}\n`))); + expect(error.message).toBe( + `pi-env exited with code 1 before it was ready\nremote stderr:\n${lines.slice(10).join("\n")}`, + ); + }); + + it("keeps only the last 2000 characters of stderr", async () => { + const error = await startError(connect(failing(`first\n${"x".repeat(5000)}\n`))); + const tail = error.message.split("\nremote stderr:\n")[1] ?? ""; + expect(tail).toBe("x".repeat(1999)); + }); + + it("passes stderr to onLog unchanged", async () => { + const logged: string[] = []; + await startError(connect(failing("\u001b[31mred\u001b[0m\n"), (text) => logged.push(text))); + expect(logged.join("")).toBe("\u001b[31mred\u001b[0m\n"); + }); + + // #10716: acknowledge the first stderr read before writing the remaining UTF-8 bytes. + it.each(["\u00e9", "\u9519", "\u{1f600}"])("decodes %s split across stderr chunks", async (character) => { + const directory = mkdtempSync(join(tmpdir(), "pi-env-stderr-")); + const acknowledged = join(directory, "logged"); + const stderr = `prefix ${character} suffix\n`; + const bytes = Buffer.from(stderr); + const split = Buffer.byteLength("prefix ") + 1; + const logged: string[] = []; + const connection = connect( + ` +const { existsSync } = require("node:fs"); +const bytes = Buffer.from(${JSON.stringify([...bytes])}); +const timeout = setTimeout(() => process.exit(99), 3000); +process.stderr.write(bytes.subarray(0, ${split})); +const poll = setInterval(() => { + if (!existsSync(${JSON.stringify(acknowledged)})) return; + clearInterval(poll); + clearTimeout(timeout); + process.stderr.write(bytes.subarray(${split}), () => process.exit(7)); +}, 5); +`, + (text) => { + logged.push(text); + if (!existsSync(acknowledged)) writeFileSync(acknowledged, ""); + }, + ); + try { + const error = await startError(connection); + expect(logged.length).toBeGreaterThanOrEqual(2); + expect(logged.join("")).toBe(stderr); + expect(error.message).toBe( + `pi-env exited with code 7 before it was ready\nremote stderr:\n${stderr.trimEnd()}`, + ); + } finally { + connection.close(); + rmSync(directory, { recursive: true, force: true }); + } + }); + + it("flushes an incomplete UTF-8 character when stderr ends", async () => { + const logged: string[] = []; + const error = await startError( + connect("process.stderr.write(Buffer.from([0x78, 0xf0, 0x9f]), () => process.exit(7));", (text) => + logged.push(text), + ), + ); + expect(logged.join("")).toBe("x\uFFFD"); + expect(error.message).toBe("pi-env exited with code 7 before it was ready\nremote stderr:\nx\uFFFD"); + }); + + it("reports only the stderr of the attempt that failed", async () => { + let attempt = 0; + const connection = connect(() => failing(`failure ${++attempt}\n`)); + expect((await startError(connection)).message).toContain("failure 1"); + const second = await startError(connection); + expect(second.message).toContain("failure 2"); + expect(second.message).not.toContain("failure 1"); + }); + + it("keeps the fields of a hello the daemon refused", async () => { + const connection = connect(fakeDaemon("error")); + for (let attempt = 0; attempt < 2; attempt++) { + const error = await startError(connection); + if (!(error instanceof RemoteError)) throw error; + expect(error.code).toBe("EINVAL"); + expect(error.path).toBe("/daemon"); + expect(error.fields.extra).toBe(1); + expect(error.message.startsWith("hello refused")).toBe(true); + expect(isConnectionLost(error)).toBe(false); + } + }); + + it("reports a daemon that cannot be started as a spawn error", async () => { + const connection = new Connection({ command: ["/nonexistent/pi-env"] }); + connections.push(connection); + const error = await startError(connection); + if (!(error instanceof RemoteError)) throw error; + expect(error.code).toBe("spawn_error"); + expect(isConnectionLost(error)).toBe(true); + }); +}); + +describe("Connection of a ready daemon", () => { + it("fails pending requests as soon as the daemon exits, even while its pipes stay open", async () => { + const connection = connect(fakeDaemon("result")); + await connection.info(); + const started = Date.now(); + const outcome = await connection.request("lstat", { path: "/" }).then( + () => undefined, + (error: unknown) => error, + ); + if (!(outcome instanceof Error)) throw new Error("The request did not fail"); + expect(isConnectionLost(outcome)).toBe(true); + expect(outcome.message).toBe("pi-env connection lost"); + // The process the daemon started holds its pipes for 3 s; the request must not wait for them. + expect(Date.now() - started).toBeLessThan(2000); + }); +}); From 59ca40cf5890f3d72056d3c1ed644db7339bb930 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 27 Oct 2026 16:21:37 +0100 Subject: [PATCH 038/271] docs(coding-agent): update llama.cpp setup for llama.app (#10179) update the llama.cpp docs to use the llama.app installer and the new `llama serve` command commit by @hanouticelina disclaimer: my julien-cto agent helped me write this --------- Co-authored-by: Celina Hanouti Co-authored-by: Mario Zechner --- packages/coding-agent/docs/llama-cpp.md | 35 ++++++++++++++++--------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/packages/coding-agent/docs/llama-cpp.md b/packages/coding-agent/docs/llama-cpp.md index d4c41ab2e..6684bb4df 100644 --- a/packages/coding-agent/docs/llama-cpp.md +++ b/packages/coding-agent/docs/llama-cpp.md @@ -1,31 +1,42 @@ # Local Models with llama.cpp -Pi supports the [llama.cpp](https://github.com/ggml-org/llama.cpp) router server. The router discovers multiple GGUF models and loads or unloads them on demand. +Pi supports the [llama.cpp](https://llama.app) router server. The router discovers multiple GGUF models and loads or unloads them on demand. -Use a current llama.cpp build with router support. Follow the [build instructions](https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md) or install a [prebuilt release](https://github.com/ggml-org/llama.cpp/releases) for your platform. +## Install llama.cpp + +Install llama.cpp with the [official installer](https://llama.app/docs/installation): + +```bash +curl -LsSf https://llama.app/install.sh | sh +``` + +Verify the installation: + +```bash +llama cli --version +``` + +See [llama.app](https://llama.app) for other installation methods, including package managers and building from source. ## Start the router -Start `llama-server` without `--model` or `-m`. Passing a model starts single-model mode instead of router mode. +Start `llama serve` without `--model`, `-m`, or `-hf`. Passing a model starts single-model mode instead of router mode. See the [llama.cpp server guide](https://llama.app/docs/serve#serving-multiple-models-router-mode) for router configuration. ```bash -llama-server \ - --models-dir ~/models \ +llama serve \ --no-models-autoload \ - --jinja \ --host 127.0.0.1 \ --port 8080 \ - -ngl 999 \ + -ngl all \ -c 32768 ``` Important options: -- `--models-dir ~/models` discovers local GGUF files. - `--no-models-autoload` keeps loading explicit through `/llama`. - `--jinja` enables compatible chat templates and tool calling. -- `-ngl 999` offloads as many layers as possible to the GPU. -- `-c 32768` sets the context window for each loaded model. Omit it to use the model's native context, which may require substantially more memory. +- `-ngl all` offloads all layers to the GPU. Omit it to use automatic GPU offloading. +- `-c 32768` sets the context size in tokens. Use `-c 0` for the model's full context window, which may require substantially more memory. Context is shared across parallel server slots. A single-file model can sit directly in the model directory. Put multimodal and multi-shard models in separate subdirectories: @@ -41,7 +52,7 @@ A single-file model can sit directly in the model directory. Put multimodal and └── large-model-Q4_K_M-00003-of-00003.gguf ``` -Restart the router after manually adding files. For per-model context sizes and other options, use [llama.cpp model presets](https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md#model-presets). For settings tuned to a specific NVIDIA GPU and model, see [Run Pi with NVIDIA RTX and llama.cpp](nvidia-rtx-llama-cpp.md). +Restart the router after manually adding files. The router also supports cached models and preset files via `--models-preset`, as described in the [server guide](https://llama.app/docs/serve#serving-multiple-models-router-mode). ## Configure Pi @@ -63,7 +74,7 @@ export LLAMA_API_KEY=optional-secret pi ``` -If the server uses an API key, start `llama-server` with the matching `--api-key` value. Keep `--host 127.0.0.1` for local-only access. +If the server uses an API key, start `llama serve` with the matching `--api-key` value. Keep `--host 127.0.0.1` for local-only access. ## Manage models From 72363383ff79d3f12cfa91b03cddb3eb333c6ebc Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Tue, 27 Oct 2026 15:42:21 +0000 Subject: [PATCH 039/271] docs: allow npm run build when tests need dist/; npm run check stays the check --- AGENTS.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 190d394f1..9876d8031 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,7 +30,8 @@ ## Commands - After code changes (not docs): `npm run check` (full output, no tail). Fix all errors, warnings, and infos before committing. Does not run tests. -- Never run `npm run build` or `npm test` unless requested by the user. +- Run `npm run build` whenever something needs built output (`dist/`): tests or benchmarks that import built packages, Durable's workerd suite. Never use it to type check or lint: that is `npm run check`, the preferred check. +- Never run `npm test` unless requested by the user. - Never run the full vitest suite directly: it includes e2e tests that activate when endpoint/auth env vars are present. For all non-e2e tests, run `./test.sh` from the repo root. Otherwise run specific tests from the package root: - Vitest: `node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run test/specific.test.ts` - `packages/tui` (`node:test`): `node --test test/specific.test.ts` From f3255de76581447d17db78c9d6a2c5dbb9eee762 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Tue, 27 Oct 2026 16:57:24 +0000 Subject: [PATCH 040/271] feat(durable): windowed conversation view, ConversationTranscript and runFailure; remove watchEvents ConversationView.entries is a window over the fork-aware history with before; conversation.watch(ctx, { from, min, max }) and observeConversation take the window. ConversationTranscript merges windows into a client's transcript with gaps, loadEarlier, fillGap and resume. pi.live.runFailure says why a run ended without an answer. watchEvents and its event types are removed; the changelog has the migration guide. closes #11327 --- packages/durable/CHANGELOG.md | 77 +- packages/durable/README.md | 60 +- packages/durable/docs/spec.md | 403 ++++----- packages/durable/package.json | 1 + packages/durable/src/harness/context.ts | 11 - packages/durable/src/harness/events.ts | 512 ------------ packages/durable/src/harness/generation.ts | 25 +- packages/durable/src/harness/harness.ts | 9 +- packages/durable/src/harness/live.ts | 72 +- packages/durable/src/harness/task-graph.ts | 4 +- packages/durable/src/harness/transcript.ts | 285 +++++++ packages/durable/src/harness/types.ts | 18 +- packages/durable/src/harness/view.ts | 232 ++++-- packages/durable/src/index.ts | 19 +- .../test/conversation-transcript.test.ts | 356 ++++++++ .../durable/test/conversation-view-bench.ts | 234 ++++++ packages/durable/test/examples/18-json.ts | 45 +- .../durable/test/examples/20-late-join.ts | 43 +- .../test/examples/21-subagent-foreground.ts | 38 +- .../test/examples/22-subagent-background.ts | 13 +- .../durable/test/examples/31-transcript.ts | 106 +++ .../durable/test/harness-compaction.test.ts | 161 ++-- packages/durable/test/harness-events.test.ts | 779 ------------------ .../durable/test/harness-generation.test.ts | 36 +- .../test/harness-input-entries.test.ts | 32 +- .../test/harness-nested-tools-restart.test.ts | 20 +- .../durable/test/harness-nested-tools.test.ts | 164 ++-- .../durable/test/harness-ownership.test.ts | 2 +- .../durable/test/harness-run-failure.test.ts | 156 ++++ .../test/harness-storage-failure.test.ts | 12 +- .../test/harness-structured-output.test.ts | 13 +- .../durable/test/harness-structured.test.ts | 142 ++-- .../durable/test/harness-view-run.test.ts | 607 ++++++++++++++ packages/durable/test/harness-view.test.ts | 297 ++++++- packages/durable/test/live-support.ts | 110 +++ .../test/session-document-retention.test.ts | 25 +- packages/durable/test/spec-usage.test.ts | 20 + 37 files changed, 3074 insertions(+), 2065 deletions(-) delete mode 100644 packages/durable/src/harness/events.ts create mode 100644 packages/durable/src/harness/transcript.ts create mode 100644 packages/durable/test/conversation-transcript.test.ts create mode 100644 packages/durable/test/conversation-view-bench.ts create mode 100644 packages/durable/test/examples/31-transcript.ts delete mode 100644 packages/durable/test/harness-events.test.ts create mode 100644 packages/durable/test/harness-run-failure.test.ts create mode 100644 packages/durable/test/harness-view-run.test.ts create mode 100644 packages/durable/test/live-support.ts diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 0f584aaed..471b12b79 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -8,23 +8,90 @@ - `Storage.commit(writes, adopt, context)` takes an `adopt(seq)` callback, the Session's in-memory adoption and publication of the batch: a storage calls it exactly once, synchronously, once the batch is durable and before its next commit starts, then resolves with `seq`, and never for a commit that fails before the batch is durable. Memory, SQLite, JSONL, and IndexedDB storages do; custom storages and wrappers must call or forward it, and the storage conformance suite checks it. A commit that rejects after its adoption is published and fails the Session without rolling the adopted changes back. - `Storage.mintId()`'s contract is narrowed to IDs greater than every ID minted since the storage was opened and every ID stored, also across reopen. The bundled storages already behave so. - Watches use Chord's `WatchHandle` and `WatchEnd` (Durable re-exports both, with `WatchChange` and `NativeWatchChange`). End reasons are `stopped`, `cancelled`, `ended` (was `session_closed`), `listener_error`, and `failed` (was `session_failed`); `retired` is gone (see below). An ended watch drops its listener and every reference into its Session, so a stopped watch that is still referenced keeps only its last value. -- Watch listeners take one change object before the context: `watch.start(async ({ value, ops, retired }, context) => …)` for document, conversation view (`Conversation.watch()`), and task graph watches, and `stream.start(async ({ value: events }, context) => …)` for `watchEvents()`. Migration: `start(async (value, ops, ctx) => …)` becomes `start(async ({ value, ops }, ctx) => …)`, and an event listener `(events, ctx)` becomes `({ value: events }, ctx)`. +- Watch listeners take one change object before the context: `watch.start(async ({ value, ops, retired }, context) => …)` for document, conversation view (`Conversation.watch()`), and task graph watches. Migration: `start(async (value, ops, ctx) => …)` becomes `start(async ({ value, ops }, ctx) => …)`. - Document watches follow their address (kind, scope, and key) instead of one incarnation: `DocumentWatch` is `WatchHandle>` (no `null`). A retirement no longer ends the watch with a last frame `[["r", null]]` and `retired`: the next call has `retired: true`, the last value the watch was given, and no ops (folded with what is pending; a change made in the retiring commit itself is not shown), and a document created at the address later arrives on the same watch as `[["r", value]]`, possibly in the same call. A task document's watch ends `ended` once the retirement call of the commit that makes its task terminal has returned; Session close or failure still ends it first if that call was not delivered yet (a watch never started). Acquiring an absent document still gives `undefined`. A watch pins the incarnation at its address, nothing while it is absent. Migration: test `change.retired` instead of a `null` value or a `retired` end, and stop a watch whose document may be retired for good, as it otherwise stays until the Session ends. -- Document, conversation view, and task graph watches deliver the commits made while their listener runs (or before `start()`) in one callback: the newest value, the commits' operations folded (appends to one string join, sets of one field keep the last) and the newest commit's Context. Past 10,000 pending operations the callback gets one root replacement `[["r", newest]]`. Before, each commit was its own callback and more than 100 pending commits became a root replacement. `watchEvents()` still delivers every event batch in order, with one `snapshot` past 100 pending batches. +- Document, conversation view, and task graph watches deliver the commits made while their listener runs (or before `start()`) in one callback: the newest value, the commits' operations folded (appends to one string join, sets of one field keep the last) and the newest commit's Context. Past 10,000 pending operations the callback gets one root replacement `[["r", newest]]`. Before, each commit was its own callback and more than 100 pending commits became a root replacement. - Removed the Chord state getters `Session.documentState()`, `Conversation.viewState()`, and `Harness.taskGraph()`, and the `DocumentState` type; use `watchDoc()`, `Conversation.watch()`, and `Harness.watchTaskGraph()`. Durable no longer uses Chord's `replicatedState()`. -- Partial answers and running tool output are stored later than they are shown: their throttled commits are deferred (`CommitPersistence`, spec §4.1), so watches, views, and agent events see each at once, while Storage gets them together with the next commit that reads `pi.live` (the end of the answer, a tool settling, a run ending) and at the latest after `settings.progress.persistWithinMs` (default 1,000 ms). With 32 streams over SQLite this cuts storage transactions about 5.7x and disk writes about 2.9x for turns with parallel tools, 27x and 9x for long answers, and CPU by about a quarter. A crash now loses up to about a second of streamed content instead of 100 ms; a stream that crashes before its first store leaves no aborted entry, and that attempt's usage is not recorded. `subscribeCommits()` gets these commits only once they are stored, with exactly what was stored (held operations first; a document the commit retires shows as its retirement only, without the operations stored before it), and gets a publication for a deadline's store too; it no longer sees every partial. Migration: a `subscribeCommits()` listener that showed live progress should use `watchDoc()`, `Conversation.watch()`, or `watchEvents()`; one that mirrors Storage keeps working; set `settings.progress.persistWithinMs: 0` to store progress right after each commit, about as before (one storage transaction per partial). +- Partial answers and running tool output are stored later than they are shown: their throttled commits are deferred (`CommitPersistence`, spec §4.1), so watches and views see each at once, while Storage gets them together with the next commit that reads `pi.live` (the end of the answer, a tool settling, a run ending) and at the latest after `settings.progress.persistWithinMs` (default 1,000 ms). With 32 streams over SQLite this cuts storage transactions about 5.7x and disk writes about 2.9x for turns with parallel tools, 27x and 9x for long answers, and CPU by about a quarter. A crash now loses up to about a second of streamed content instead of 100 ms; a stream that crashes before its first store leaves no aborted entry, and that attempt's usage is not recorded. `subscribeCommits()` gets these commits only once they are stored, with exactly what was stored (held operations first; a document the commit retires shows as its retirement only, without the operations stored before it), and gets a publication for a deadline's store too; it no longer sees every partial. Migration: a `subscribeCommits()` listener that showed live progress should use `watchDoc()` or `Conversation.watch()`; one that mirrors Storage keeps working; set `settings.progress.persistWithinMs: 0` to store progress right after each commit, about as before (one storage transaction per partial). - The change type of `FileSystem.watch()` in `@earendil-works/pi-durable/env` is renamed from `WatchChange` to `FileChange` (same shape: `{ paths }`, `{ overflow: true }`, or `{ error }`), so `WatchChange` names only a watch listener's change. Migration: import `type FileChange` from `@earendil-works/pi-durable/env` where `WatchChange` was imported from it. - `HostCallback` gains `"telemetry"`: a `host_error` report about a telemetry adapter that threw while a commit was published. Exhaustive switches over `HostCallback` need a case for it. +- `ConversationView.entries` is a window over the transcript instead of the active transcript: the fork-aware history `Conversation.entries()` scans, in entry ID order, with head markers (compaction summaries, resets, handoffs) appended in place instead of cutting the entries before them. A watch starts with the newest 50 entries; new entries are appends, and past 100 the oldest leave in one splice down to 50, which also sets the new `ConversationView.before`, the ID of the entry just before the window (absent when the window starts the transcript). First values and overflow replacements (`[["r", view]]`) hold at most `max` entries, never the whole transcript. Migration: read the active transcript the model sees from `Conversation.context()`; render a transcript longer than the window with `ConversationTranscript`, or page older entries with `Conversation.entries()`; a client that rebuilt the head cut from the view's operations no longer gets one ([#11327](https://github.com/earendil-works/pi/issues/11327)). +- `Conversation.watch(context, window?)` and `Harness.observeConversation(id, observer, context, window?)` take a `ConversationWindow` (`from`, `min` default 50, `max` default `2 * min`): `min`/`max` bound the window, and `from` starts the first value at that entry ID when at most `max` entries are at or after it, otherwise at the newest `min`; it picks only that first value, so every entry committed after the attachment is appended, and a `from` past the newest entry starts empty. Bounds that are not integers, or `max` below `min`, reject with a `RangeError`. Each watch and observer has its own window, so the observers of one conversation no longer share one change object or one value; they share the document values ([#11327](https://github.com/earendil-works/pi/issues/11327)). +- `pi.live` is no longer empty after a run that ended without an answer: it keeps `runFailure` (below) until a later run starts. Code that compares an idle `pi.live` with `{}`, or tests for idleness that way, checks `run === undefined` instead ([#11327](https://github.com/earendil-works/pi/issues/11327)). +- Removed `watchEvents()` and its types `AgentEvent`, `AgentEventStream`, `SnapshotEvent`, `MessageChange`, and `ToolEventCall`, and spec §9.4. A renderer renders state instead of assembling it from events: everything the events carried that a renderer draws is in the conversation view, typed for producer and consumer, and new information becomes a field there rather than an event type every client must learn ([#11327](https://github.com/earendil-works/pi/issues/11327)). Migration, for a client of `watchEvents()`: + + Before: + + ```ts + const stream = await watchEvents(harness, conversation.id, context); + initialize(stream.snapshot); // entries, run, generation, tools, compactions, inbox, agent, usage + stream.start(async ({ value: events }) => { + for (const event of events) { + if (event.type === "message_update") applyChanges(event.changes); // streamed text + else if (event.type === "message_end") appendEntry(event.entry); + else if (event.type === "tool_execution_update") applyOutput(event.toolCallId, event.output); + else if (event.type === "run_end") showIdle(); + // ... and every other event type + } + }); + // After a drop: attach again; the new snapshot replaces everything held. + ``` + + After: + + ```ts + const transcript = new ConversationTranscript((query, limit, ctx) => conversation.entries(query, limit, undefined, ctx)); + const watch = await conversation.watch(context); + render(transcript.merge(watch.value)); + watch.start(async ({ value }) => render(transcript.merge(value))); + + function render(view: TranscriptView): void { + const live = view.docs["pi.live"] as LiveState; + drawItems(view.items); // entries, a "load more" row per gap, "load earlier" at the top + if (live.generation?.message !== undefined) drawStreaming(live.generation.message); + for (const slot of live.tools ?? []) drawTool(slot); // status, running output, details, result entry + if (live.runFailure !== undefined) drawFailure(live.runFailure); // why the last run ended without an answer + setBusy(live.run !== undefined); + } + // A "load earlier" or "load more" row scrolled into view: + render(await transcript.loadEarlier(context)); + render(await transcript.fillGap(gap, context)); + // After a drop: only what was missed when it fits the window, otherwise the newest entries and a gap. + const resumed = await conversation.watch(context, transcript.resume); + ``` + + The watch's operations carry the same deltas the events did (text and thinking appends, tool-call argument appends, output trims and appends), for a client that forwards them, for example to a remote replica. Where each event went: + + | event | state | + |---|---| + | `snapshot` | the watch's first value, `watch.value`, its window bounded by `max` | + | `run_start`, `run_end` | `pi.live.run` appears, disappears | + | `turn_start`, `turn_end` | `pi.live.run.taskId` moves to the next generation; the task graph's `pi.generation` nodes, `completing` at their hold | + | `message_start`, `message_update` | `pi.live.generation.message`, the streamed answer, with its `usage` | + | `message_end`, `entry_appended` | the entry appended to `entries` | + | `tool_execution_start`, `tool_execution_update`, `tool_execution_end` | the slot in `pi.live.tools` or `nestedTools`: `status` pending, running, done; `output`, `details`, `diagnostics`; `entry` once done (absent after a fault); a nested call's `arguments`, its `summary`, and its result in the caller's `NestedCallDoc` | + | `auto_retry_start`, `auto_retry_end` | `pi.live.generation.retry` set, removed | + | `deferred_poll` | `pi.live.generation.deferred.pollAt` | + | `compaction_start`, `compaction_end` | a status in `pi.live.compactions` appears, disappears; `pi.live.compactionFailure` when it failed; the summary entry when it is placed | + | `inbox_update` | `pi.inbox` | + | `agent_changed`, `usage_changed` | `pi.agent`, `pi.usage` | + | `submission` | `Submission.status()` and `wait()`, `Harness.submission(id)`; queued items in `pi.inbox`, the run's inputs in `pi.live.run.inputs` | + | `task_failed` | the task graph (`completing` with the outcome's status), `Harness.getTask()`, and `onReport`'s `task_faulted`; for a run, `pi.live.runFailure`; for a compaction, `compactionFailure`; for a tool, a done slot without `entry` | + | a run that ended without an answer (`run_end` with an `unanswered` `submission`) | `pi.live.runFailure`: `reason` (`model_error`, `no_model`, `aborted`, `reset`, `faulted`, `orphaned`), `detail`, `taskId`, `inputs` | + + Some event data has no state in the view. A tool's arguments after repair and `beforeTool` at its start: in the nested slot's `arguments` for a nested call, only in the tool task's checkpoint for a model-issued one. A submission's settlement: `Submission.status()`/`wait()` and `Harness.submission(id)`; an input settled without a run (withdrawn, `stale`, aborted while queued, `run_failed`) just leaves `pi.inbox`. A failed task's error: `Harness.getTask()` and `onReport`; the task graph keeps only live tasks, not terminal outcomes. A nested call's full result: its caller's `NestedCallDoc` (`watchDoc()`), which retires when the caller settles. Exact turn boundaries: the run's `taskId` and the task graph, with no explicit event. Whether a compaction did anything: its `CompactionResult` (`waitForTask()`), and the summary entry when it is placed. ### Added - Deferred commits: `Session.commit()`, `Conversation.commit()`, `TaskRuntime.commit()`, and a tool's `api.commit()` take `{ persistence: { kind: "deferred", withinMs? } }`. A commit that changes only existing conversation documents with `history: "latest"` and `fork: "initial"` is then adopted in memory and shown to watches at once, and stored with the next commit that reads or writes the document, before the document leaves memory, at close, or within `withinMs` (default `settings.progress.persistWithinMs`, or `createSession(storage, { persistWithinMs })`); any other write makes it immediate. A crash loses what is held. See spec §4.1. - Synchronous observers for adapters, such as a Chord source over a Durable view: `Session.observeDoc(token, ...scope, observer, context)`, `Harness.observeConversation(id, observer, context)`, and `Harness.observeTaskGraph(observer, context)` attach an `Observer` (`advance(change, context)`, `end(failure)`) atomically on the Session line, like a watch, and return an `Observation` (`value`, `detach()`; `observeDoc` gives `undefined` for an absent document). `advance` gets every change unfolded as memory adopts it, for a stored commit inside Storage's `adopt` and for a deferred one before it is stored, so it must only record it: no awaits, no Session calls, no throws. A document observer gets one call per change at its address (two for a commit that retires and recreates it), the task graph also a call outside any commit when a registry change or a failed migration changes a `blocked` reason, and a change can arrive before the attaching promise resolves. The attaching context governs only the attachment; a closed Session or an invalid argument rejects the promise. One that throws is reported as `listener_error`, detached, and ended with the error. `end` comes once, at Session close or failure, or after a task document's retirement with its task. Document, view, and task graph watches are built on them (spec §9.1). +- `ConversationTranscript` keeps what a client has of one conversation over any `ConversationView`: `merge(view)` returns the view with `items`, the client's whole transcript (entries, a `gap` where a later window did not continue it, and an `earlier` marker until the start is loaded), instead of the window; `loadEarlier(context)` and `fillGap(gap, context)` page entries through a loader such as `Conversation.entries()`, and `resume` is the window that continues it after a drop: `conversation.watch(context, transcript.resume)`. A merge costs O(items) and returns the same `items` array when it adds nothing; concurrent loads of the same page are shared ([#11327](https://github.com/earendil-works/pi/issues/11327)). +- `pi.live.runFailure` (`RunFailure`, `RunFailureReason`) records why the newest run ended without an answer, `model_error`, `no_model`, `aborted`, `reset`, `faulted`, or `orphaned`, with the run's task and inputs and a `detail` where one exists (the error text, the missing model, the fault, the orphan reason). The commit that ends the run sets it; a later commit that starts a run removes it ([#11327](https://github.com/earendil-works/pi/issues/11327)). - `HarnessOptions.readOnly` opens a Harness as a viewer of a session another process drives: open leaves running tasks as committed instead of recovering them, every commit with writes rejects with `ReadOnlySession` before it reaches Storage, and `resume()` and every call that would run tasks reject with it. ### Changed -- Document, conversation view, task graph, and event watches use Chord's watch handles (`Watch`, and `NativeWatch` for event batches). +- Document, conversation view, and task graph watches use Chord's watch handles (`Watch`). - A stored commit is published when Storage adopts it, inside `Storage.commit()`: the commit's telemetry events and Durable's own observers (document watches, conversation views, the task graph, the scheduler, and waits) run inside `adopt`, so two views never show one commit apart. `subscribeCommits()` listeners run one microtask later, outside the storage's synchronous step, before every wait the commit resolves and every abort signal it fires; they have no order relative to watch listeners and `onWake`. With a storage that works after adopting (JSONL's sidecar reclaim), all of these run before `Storage.commit()` resolves. - `subscribeCommits()` listeners are read when a commit's dispatch runs: a listener added between adoption and the dispatch gets the commit only when one was registered at adoption; one added after the dispatch ran misses it. - The abort signal of a newly marked running task fires after `subscribeCommits()` listeners instead of before them. @@ -33,7 +100,7 @@ ### Fixed - Fixed a tool round with `control.handoff` ending the run on the tool-calling answer: the handoff's `pi.reset` entry is written and the same run continues with a new generation in the new context, so the submission settles with the eventual answer. Queued writes and selected steers are placed after the handoff text, follow-ups wait for the run's final boundary, and a queued reset still ends the run `unanswered` with reason `reset`. `terminate` remains the way to end a round without another model request. -- Fixed a conversation watcher (`watchEvents()`, conversation views) holding a copy of a streamed text or tool call per partial commit, through Chord append ops that kept the whole string alive; a 140 KB answer followed by a 140 KB tool call held well over 100 MB ([#9265](https://github.com/earendil-works/pi/issues/9265)) +- Fixed a conversation watcher (conversation views) holding a copy of a streamed text or tool call per partial commit, through Chord append ops that kept the whole string alive; a 140 KB answer followed by a 140 KB tool call held well over 100 MB ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed a nested call's error summary keeping the call's whole output alive through a slice of it ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed task span error messages and codemode call previews keeping the whole text they were cut from alive ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed partial commits of a streamed answer copying every growing text, thinking and tool-call argument string in full on each commit (about 20 s of CPU and a 100 MB heap peak for a 500 KB tool call): they now append new text with Chord's `appendText()`, and apply the argument changes of their own parse of the streamed argument text ([#9265](https://github.com/earendil-works/pi/issues/9265)) diff --git a/packages/durable/README.md b/packages/durable/README.md index d10263776..6b4237ba3 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -324,7 +324,7 @@ const testsPass = defineTool({ - The caller gets the nested call's `taskId`, `structuredOutput`, `details`, `diagnostics`, `usage`, and `isError`, not the `output` meant for the model. A blocked, invalid, failed, or aborted nested call returns an `isError` result. `executeTool()` itself rejects once `execute()` has returned, when the caller is aborted, or when the Harness closes; cancelling its `context` stops only the wait. - Hooks see a nested call's `call.parent`, so a guard can treat calls from scripts differently. -- While the caller runs, its nested calls show in `docs["pi.live"].nestedTools`, with the arguments they run with and a `summary` (error, duration, usage) once done, and as tool events with `parentToolCallId` and `parentTaskId`. `progress: false` keeps a nested call's running output out of `pi.live`. Before the caller settles, nested calls it left running are aborted, also those an earlier attempt made. The caller's result message does not list its nested calls; their task records keep them. +- While the caller runs, its nested calls show in `docs["pi.live"].nestedTools`, with their caller's `parentCallId` and `parentTaskId`, the arguments they run with, and a `summary` (error, duration, usage) once done; a nested call's full result is in its caller's `NestedCallDoc` while the caller runs. `progress: false` keeps a nested call's running output out of `pi.live`. Before the caller settles, nested calls it left running are aborted, also those an earlier attempt made. The caller's result message does not list its nested calls; their task records keep them. - Each nested call has a key, its position among the caller's nested calls unless the caller passes `{ key }`. A replay-safe caller that reruns after a crash and calls in the same order, or passes the same keys, gets the nested calls it already made back, finished or still running, instead of starting them again. A nested call the crash interrupted before its result was committed follows its own `replay` policy, so a replay-safe tool with external effects still needs its own idempotency. A caller that is not replay-safe never reruns, so after a restart its unfinished nested calls are abandoned, with everything they own, before they run again. - A nested call's `usage` is counted under its own tool; passing it on in the caller's result counts it twice. - Nested results are not kept: they live in the caller's task documents (`NestedCallDoc`, one per nested call, keyed by its key, holding `{ taskId, result? }`) only until the caller settles. The nested call's task record keeps only a small receipt. A stored result is exactly what `executeTool()` returns, without the model's `output`, so an image or a `bash` tail is stored once. @@ -479,8 +479,9 @@ Everything a UI needs is committed state. `watch()` returns a watch of the conve const watch = await root.watch(context); render(watch.value); // the state at attachment watch.start(async ({ value, ops }) => { - // value.entries: the active transcript - // value.docs["pi.live"]: the running generation (streamed partial, retry, deferred) and tool calls (output, details) + // value.entries: a window over the transcript, newest last; value.before: the entry just before it + // value.docs["pi.live"]: the run, the running generation (streamed partial, retry, deferred), tool calls (output, + // details), compactions, and why the last run ended without an answer (runFailure) // value.docs["pi.inbox"], value.docs["pi.usage"], value.docs["pi.agent"], value.docs["pi.provider"] render(value); await send(ops); // for example to a remote client that applies them @@ -488,11 +489,33 @@ watch.start(async ({ value, ops }) => { // later: await watch.stop(); ``` +The view's `entries` are a window, so neither the first value nor a reconnect costs the whole transcript. A watch starts with the newest 50 entries; new entries are appended, and once the window would hold more than 100, its oldest leave in one splice, leaving the newest 50. `watch(context, { min, max })` changes the bounds. `from` picks where the first value starts (as `transcript.resume` does after a drop); every entry committed after the watch attached is appended, so a `from` past the newest entry starts empty and receives what comes next. Compaction summaries and resets are entries like any other, in place; a summary's `head` says where the entries it summarizes end. `conversation.context()` gives the active transcript the model sees. + +`ConversationTranscript` keeps what a client has seen beyond the window: call `merge()` with every value of the view, and render the `items` it returns, the client's whole transcript with a "load earlier" marker at the top and a "load more" gap wherever the client missed entries, after a stall or a reconnect: + +```typescript +import { ConversationTranscript } from "@earendil-works/pi-durable"; + +const transcript = new ConversationTranscript((query, limit, ctx) => root.entries(query, limit, undefined, ctx)); +const watch = await root.watch(context); +render(transcript.merge(watch.value)); +watch.start(async ({ value }) => render(transcript.merge(value))); + +// When a marker scrolls into view: +render(await transcript.loadEarlier(context)); // the page before the oldest entry +render(await transcript.fillGap(gap, context)); // the next page of a gap item + +// Reconnect after a drop: only what was missed, when it fits the window, otherwise the newest entries and a gap. +const resumed = await root.watch(context, transcript.resume); +``` + +Entries never change once committed, so a merge only adds; one that adds nothing returns the same `items` array. See `test/examples/31-transcript.ts`. + Watches are Chord's `WatchHandle` (exported by `@earendil-works/chord`; Durable re-exports it). The listener gets one change per call, `{ value, ops, retired }`, and the newest commit's context. A watch ends once, and `closed` (or `stop()`) gives the reason: `stopped`, `cancelled` (its acquisition context was cancelled), `ended` (the Harness closed, or a task document's task ended), `listener_error`, or `failed` (the Harness failed). An ended watch keeps its last value and no longer references the Harness. Adapters that must change in the same step as Durable's memory, such as a Chord source over a view, attach a synchronous observer instead of a watch: `harness.observeConversation(id, observer, context)`, `harness.observeTaskGraph(observer, context)`, and `harness.observeDoc(token, ...scope, observer, context)` return the current `value` and `detach()`, and call `observer.advance(change, context)` with every later change, unfolded, as memory adopts it: inside the storage's commit for a stored commit, at once for a deferred one (before it is stored). A document observer gets one call per change at its address, so a commit that retires and recreates the document gives two; the task graph also advances without a commit when a registry change alters a task's `blocked` reason. A change can arrive before the `observeXXX()` promise resolves; its ops apply from the observation's `value`. `observer.end(failure)` comes once, when the Harness closes or fails, or, for a task document, right after the retirement of the commit that ends its task. `advance` only records the change: no awaits, no Harness calls, no throws, and quick, since every commit waits for it. One that throws is reported, detached, and ended with the error. Watches are built on these observers. -Commits that land while a callback runs are delivered together in the next callback: the newest view and the operations of all of them, folded (appends to one string join, sets of one field keep the last). Past 10,000 pending operations the callback gets one root replacement `[["r", view]]` holding the whole newest view instead. A client that joins late or reconnects starts from the current view; nothing is replayed. +Commits that land while a callback runs are delivered together in the next callback: the newest view and the operations of all of them, folded (appends to one string join, sets of one field keep the last). Past 10,000 pending operations the callback gets one root replacement `[["r", view]]` holding the whole newest view instead, its window included. A client that joins late or reconnects starts from the current view; nothing is replayed. Partial answers and tool output are committed at most every 100 ms by default, and watches see each commit at once. Storage gets them later and together, as deferred commits (see [Your Own State](#your-own-state)): with the next commit that reads `pi.live`, such as the end of the answer or of a tool, and at the latest after `settings.progress.persistWithinMs` (default 1,000 ms). A crash loses at most about that window of an answer or of tool output; a stream that crashes in its first second may leave no aborted entry, and that attempt's usage is not recorded. `settings.progress` changes the intervals; a host whose storage is remote can commit less often, for example `{ partialIntervalMs: 500, outputIntervalMs: 500 }`. @@ -515,6 +538,8 @@ await root.submit({ type: "write", entry: { kind: "app.note", data: "user opened If a run fails, queued items stay in the inbox until the next input or queued write places them, oldest first. +When a run ends without an answer, `docs["pi.live"].runFailure` says why until the next run starts: `model_error` and `no_model` (with the error text or the missing model in `detail`), `aborted`, `reset`, or `faulted` and `orphaned` (the run's task could not go on). The run's inputs are in it too, settled `unanswered`; the settlement's reason is the same, except that an orphaned run's inputs carry why it was orphaned (`missing_task`, `task_too_old`, `migration_failed`) and `no_model` has no `detail` there (spec §8.2). + An input is placed as a `pi.user` entry. To show it as something else, give it an `entry` with your kind and data; the model still sees `content` as the user message: ```typescript @@ -582,23 +607,7 @@ When a provider rejects a request because the context is too long, generation co When a compaction that a generation waits for at the blocking threshold fails, the generation fails too, without sending a request: its inputs settle `unanswered` with `model_error` and detail `Compaction failed: `. One that finds nothing to compact or is declined lets the request go ahead. -Running compactions are listed in `docs["pi.live"].compactions` with their reason, attempt, and retry backoff. The newest failure stays in `docs["pi.live"].compactionFailure` (task, reason, error) until a compaction produces a summary. The agent events add `compaction_start` and `compaction_end` (with `error` when the compaction failed), and `compactions` and `compactionFailure` fields in the snapshot. - -## Agent Events (Experimental) - -For consumers that want coding-agent style events (`message_start`, `message_update`, `tool_execution_start`, ...) instead of structural state: - -```typescript -import { watchEvents } from "@earendil-works/pi-durable"; - -const stream = await watchEvents(harness, root.id, context); -initialize(stream.snapshot); // entries, run, in-flight generation, tools, compactions, inbox, agent, usage -stream.start(async ({ value: events }) => { - for (const event of events) console.log(JSON.stringify(event)); -}); -``` - -Events are derived from commits, one batch per commit, and apply on top of the snapshot. Message and tool updates carry deltas: text and thinking appends, appended tool-call argument text, and output trims and appends. When a consumer falls more than 100 batches behind, it receives a fresh `snapshot` event instead. See `test/examples/18-json.ts` for the full stream of one run. +Running compactions are listed in `docs["pi.live"].compactions` with their reason, attempt, and retry backoff. The newest failure stays in `docs["pi.live"].compactionFailure` (task, reason, error) until a compaction produces a summary. ## Hooks @@ -831,7 +840,7 @@ The package root loads TypeBox, because the tool task validates arguments with p Three rules decide what an error does: -1. **A storage error ends the Harness.** The call that hit it rejects with the error. Everything after it, and every pending wait, rejects with `SessionFailed` (its `cause` is that error); watches and event streams end with `failed`; running tasks get their abort signal; `onReport` gets the error once. The Harness then closes itself. Nothing is retried; a custom storage retries its own transient failures. Reopen to recover: commits are atomic, so the store is consistent, and resubmitting with the same `requestId` finds a submission that was already admitted. +1. **A storage error ends the Harness.** The call that hit it rejects with the error. Everything after it, and every pending wait, rejects with `SessionFailed` (its `cause` is that error); watches end with `failed`; running tasks get their abort signal; `onReport` gets the error once. The Harness then closes itself. Nothing is retried; a custom storage retries its own transient failures. Reopen to recover: commits are atomic, so the store is consistent, and resubmitting with the same `requestId` finds a submission that was already admitted. 2. **An invalid read fails only that call:** a storage read throws `StorageRequestError` for a bad cursor, an unknown conversation, or history a document does not keep. A read you cancelled fails only itself too. The exemption covers reads only: any error from `commit()` or `mintId()`, `StorageRequestError` included, ends the Harness as in rule 1. 3. **Your code fails only its own unit:** a throwing task phase faults that task, a throwing tool gets an error result, a throwing hook is reported and skipped (`beforeTool` blocks the call), and a throwing commit callback rolls back that commit. Listeners that throw, or return a rejected promise, are reported and the others still run. The exception is a host `RegistryReader` whose `snapshot()` throws: the Harness cannot schedule without it, so it fails the Harness like a storage error. @@ -859,10 +868,10 @@ node --conditions=source --experimental-strip-types test/examples/13-chat.ts | [15-real-model](test/examples/15-real-model.ts) | Streaming an answer from OpenAI | | [16-coding-tools](test/examples/16-coding-tools.ts) | A tool-using turn on JSONL storage | | [17-print](test/examples/17-print.ts) | Print mode: submit a prompt, print the answer | -| [18-json](test/examples/18-json.ts) | JSON mode: agent events or raw view operations, on SQLite, JSONL, or memory | +| [18-json](test/examples/18-json.ts) | JSON mode: raw view operations, or appended entries and `pi.live` per change, on SQLite, JSONL, or memory | | [19-inbox](test/examples/19-inbox.ts) | Steers, follow-ups, writes, and withdrawal while busy | -| [20-late-join](test/examples/20-late-join.ts) | Attaching a view and an event stream mid-run | -| [21-subagent-foreground](test/examples/21-subagent-foreground.ts) | A replay-safe subagent tool whose child the call owns, with the child's events under the call | +| [20-late-join](test/examples/20-late-join.ts) | Attaching a view mid-run: the running tool's output, then the streamed answer | +| [21-subagent-foreground](test/examples/21-subagent-foreground.ts) | A replay-safe subagent tool whose child the call owns, with the child's transcript under the call | | [22-subagent-background](test/examples/22-subagent-background.ts) | Persistent subagents: spawn, steer, stop, list, answers reported back, restart-safe | | [23-child-tasks](test/examples/23-child-tasks.ts) | A checkout that owns and waits for four payments: failFast, abort, restart | | [24-compaction](test/examples/24-compaction.ts) | A long chat compacted in the background, manually, and after a context overflow | @@ -872,6 +881,7 @@ node --conditions=source --experimental-strip-types test/examples/13-chat.ts | [28-sandbox-per-conversation](test/examples/28-sandbox-per-conversation.ts) | An environment per conversation, looked up from an app document | | [29-tool-override](test/examples/29-tool-override.ts) | A same-name bash for some conversations, and a wrapper that times whichever bash won | | [30-reload-and-restart](test/examples/30-reload-and-restart.ts) | Reloading an extension mid-call, and stored choices surviving a restart | +| [31-transcript](test/examples/31-transcript.ts) | A client's transcript over the windowed view: live appends, a stall with a gap, loading it and earlier pages, and resuming with `from` | | [00](test/examples/00-conversation.ts)–[12](test/examples/12-recovery.ts) | The layers underneath: sessions, documents, forks, watches, the Harness, agent configuration, reload, extension state, tasks, recovery | Examples that call OpenAI need `OPENAI_API_KEY`; most use the faux provider otherwise. diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index f82c2ac37..e61912761 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -315,8 +315,10 @@ Context derivation: kind ahead of the baseline therefore keeps the baseline from leading. Stored entries and `contributions` keep the committed order. -Views carry raw active entries. UI reduction and model-context reduction are -separate consumers. Older stored history is available through the owning +The conversation view (section 9.3) carries a window over the visible +entries in ID order, head markers in place; `Conversation.context()` derives +the active transcript. UI reduction and model-context reduction are separate +consumers. Entries outside the window are available through the owning `Conversation` object's cursor-based `entries()` scan. ### 2.2 Public Harness surface @@ -685,8 +687,8 @@ interface Conversation { /** `{ background: true }` also aborts background work under the conversation (section 5.4). */ abort(context: Context, options?: { readonly background?: boolean }): Promise; waitForIdle(context: Context): Promise; - /** The structural view (section 9.3) as a watch. */ - watch(context: Context): Promise; + /** The structural view (section 9.3) as a watch; `window` bounds its entries. */ + watch(context: Context, window?: ConversationWindow): Promise; } interface Harness extends Session { @@ -716,10 +718,10 @@ interface Harness extends Session { waitForIdle(context: Context): Promise; /** Session total of every conversation's `pi.usage` (section 8.6). */ usage(context: Context): Promise; - /** Live tasks as a structural view (section 9.5), as a watch. */ + /** Live tasks as a structural view (section 9.4), as a watch. */ watchTaskGraph(context: Context): Promise; /** Synchronous observers of a conversation view and of the task graph, for adapters (section 9.1). */ - observeConversation(id: ConversationId, observer: Observer, context: Context): Promise>; + observeConversation(id: ConversationId, observer: Observer, context: Context, window?: ConversationWindow): Promise>; observeTaskGraph(observer: Observer, context: Context): Promise>; } @@ -924,7 +926,7 @@ replace its model configuration without reopening the Harness. Settings are sync modes are read on the Session line. `stream` is forwarded to generation requests; `stream.maxRetries` are provider retries inside one request, while `retry` governs durable generation attempts (section 8.3). Settings changes make -no commit and emit no event. +no commit and change no view. ```ts // The user's settings, read live through getters. @@ -1025,7 +1027,7 @@ wait aborts only that waiter. Conversation handles are stateless; compare them by `id`. Hosts discover conversations through lookups and scans. There is no creation listener; the task -graph view (section 9.5), `inspect()`, and the idle waits report live work. +graph view (section 9.4), `inspect()`, and the idle waits report live work. `submit()` returns after durable admission, not settlement. An input submission creates a user message with the Harness clock's timestamp at placement, which is @@ -1867,7 +1869,7 @@ at its stored version. Anything else, such as an entry, a task state, a submission, another document, a document's creation, migration, or retirement, makes it immediate, a normal commit. A held commit is adopted in memory by pointer swap, resolves, and is published to the Session's own observers: document -watches, the conversation view and agent events (section 9). Its stored +watches and the conversation view (section 9). Its stored version and delta count stay; its operations wait, folded per incarnation (Chord's `PendingOps`), and the incarnation is pinned in the cache, so idle eviction skips it at any retention. @@ -2404,7 +2406,7 @@ runtime state, not a persisted task status. A blocked task is not active in `wake()` (section 5.6): only a registry change, or `retryTask()` for a parked one, can let it run, and either makes `onWake` report it active, so a host need not poll it. The task graph shows the reason and follows registry changes -(section 9.5). Document migration remains +(section 9.4). Document migration remains access-driven. Aborting a blocked task cannot run its abort handler, because that code is @@ -2694,7 +2696,7 @@ An input is placed as one entry whose `model` is the user message of its `content`: a `pi.user` entry, or, with `entry: { kind, data }`, an entry of that kind carrying that data. Only `kind` and `data` are taken from `entry`; admission rejects a kind that is not a non-empty string or starts with `pi.`. Nothing in -the Harness reads input entries by kind; context, compaction, events, and run +the Harness reads input entries by kind; context, compaction, and run accounting go by `model` and the submission records. A write of only `kind` and `data`, with no `model`, `head`, or `edits` (present @@ -3076,8 +3078,8 @@ await conversation.configure({ extensions: { add: [Venv] } }, context); | boundaries and idle admission, on the Session line | queue modes | per boundary (section 6) | | tool task, `prepare`, `runtime.env()` | environment | at each call (section 2.2) | -Registry installs and settings changes make no commit, so they emit nothing -(section 9.4). A conversation on the default selection changes when the +Registry installs and settings changes make no commit, so no view changes +(section 9.3). A conversation on the default selection changes when the settings or the installed extensions change; its next preparation appends the system delta (section 7.4). A UI showing resolved tools or a resolved model resolves again through `Conversation.agent()`. @@ -3689,7 +3691,7 @@ that submission's result. Provider call IDs are not unique across a Session and must not serve as request IDs. Aborting or abnormally terminalizing the tool task cascades through the owned scope. The tool reports the child in its running details, for example `api.details({ conversationId })`, -so a UI that sees the call can attach to the child's view or events. +so a UI that sees the call can attach to the child's view. ```ts export const Subagent = defineExtension({ @@ -4206,7 +4208,7 @@ type LiveState = { * IDs ascend. */ nestedTools?: { - /** `/`: the call's ID in tool events, unique because keys contain no `/`. */ + /** `/`: the call's ID, unique because keys contain no `/`. */ callId: string; parentCallId: string; /** The calling tool task, model-issued or nested; the Harness relates slots by task ID. */ @@ -4252,6 +4254,25 @@ type LiveState = { * after the task ended. */ compactionFailure?: CompactionFailure; + /** + * Why the newest run ended without an answer, so a client can show it after the run ended. Set in the commit that + * ends the run; removed when a later commit starts a run. + */ + runFailure?: RunFailure; +}; + +/** The newest run that ended without an answer. */ +type RunFailure = { + /** The run's task at its end. */ + taskId: TaskId; + /** The run's inputs, settled `unanswered`. */ + inputs: SubmissionId[]; + reason: "model_error" | "no_model" | "aborted" | "reset" | "faulted" | "orphaned"; + /** + * `model_error`: the error text; `no_model`: which model is missing; `faulted`: the error message; `orphaned`: why + * the task could not run (`missing_task`, `task_too_old`, `migration_failed`). + */ + detail?: string; }; type CompactionFailure = { @@ -4321,7 +4342,26 @@ that outcome holds as `completing` (section 5.5), or the scheduler's cleanup It stays small: the summary text lives in the task's placement, never here. Partials are normalized to strict JSON before assignment. Every terminal path of a run task removes `run`, `generation`, and `tools` in the commit that settles the run's -inputs. `tx.settleSubmission()` stages each input's new status +inputs. + +A run that ends without an answer records why in `runFailure`, in the commit +that settles its inputs `unanswered`: + +| reason | the run ended because | input settlement | +|---|---|---| +| `model_error` | the response failed after any retries, or a blocking or overflow compaction it needed failed; `detail` is the error text | `model_error` with the text | +| `no_model` | no model is configured, or the configured one is not available; `detail` names it | `no_model` | +| `aborted` | its task was aborted (`Conversation.abort()`, `abortTask()`) | `aborted` | +| `reset` | a queued reset started a new context at the post-tools boundary (a tool's handoff continues the run) | `reset` | +| `faulted` | the scheduler faulted its task (section 5.4); `detail` is the error message | `faulted` with the message | +| `orphaned` | its task was aborted while no definition could take it; `detail` is the reason | the reason | + +The next commit that starts a run removes it. A run started in the same commit, +the queue's next run under `afterFailedRun: "continue"` (section 6) or after a +reset, keeps it, so the failure is shown until a later run starts. A run that +ends with an answer, including a round whose tools all ask to terminate, leaves +it alone. Inputs settled without a run (withdrawn, stale, aborted while +queued, `run_failed`) are not run failures; their submission records say why. `tx.settleSubmission()` stages each input's new status and resolves the transaction's latest candidate submission record, falling back to committed state, during assembly, like task-document validation (section 3.3), so it is not a caller table read and works after the commit's first table write. @@ -4922,7 +4962,7 @@ interface Session { // ... and the other five scopes (section 3.3), the observer before the context } interface Harness { - observeConversation(id: ConversationId, observer: Observer, context: Context): Promise>; + observeConversation(id: ConversationId, observer: Observer, context: Context, window?: ConversationWindow): Promise>; observeTaskGraph(observer: Observer, context: Context): Promise>; } ``` @@ -4936,7 +4976,7 @@ the graph at most one call, and a document observer one call per change at its address: a commit that retires the document and creates it again gives two, the retirement and then the new incarnation's `[["r", value]]`. The task graph also advances without a commit, when a registry change or a failed migration changes -a task's `blocked` reason (section 9.5): that call runs on the Session line, +a task's `blocked` reason (section 9.4): that call runs on the Session line, outside any `adopt`, with a context of its own. A document observer follows its address as a document watch does, and `observeDoc()` gives `undefined` for an absent one. `end` comes once: when the Session closes (`undefined`) or fails @@ -4960,8 +5000,9 @@ and returns: no awaits, no Session calls, no throws, and quick, as its time is part of every commit; so does `end` when a task ends. One that throws anyway is reported as `listener_error`, detached, and ended with the error; the commit and the other observers are not affected. The change object is plain data, shared by -every observer of a view and by the watches built on it, and must not be -mutated. +every observer of a document or the task graph and by the watches built on it, +and must not be mutated; each observer of a conversation view gets its own, for +its own window (section 9.3), sharing the document values. ### 9.2 `watchDoc` @@ -5033,227 +5074,188 @@ rejection is observed. ### 9.3 Conversation view -The public view is a fixed structural mount of the conversation's active +The public view is a fixed structural mount of a window over the conversation's transcript and its built-in documents: ```ts type ConversationView = { readonly conversation: ConversationRecord; - /** Raw active entries, as `ContextView.entries` (section 2.1): the head marker, then the non-head entries from its head. */ + /** + * A window over the transcript, its fork-aware history (`Conversation.entries()`), in entry ID order. Head markers + * such as compaction summaries sit where they were written; their `head` tells where the entries they summarize end. + */ readonly entries: readonly EntryRecord[]; + /** ID of the entry just before the window; absent when the window starts the transcript. */ + readonly before?: EntryId; /** `pi.agent`, `pi.live`, `pi.inbox`, `pi.provider`, and `pi.usage`, keyed by kind; absent documents are absent. */ readonly docs: Readonly>; }; + +type ConversationWindow = { + /** + * Where the first value's window starts, inclusive, such as right after the newest entry a client has + * (`ConversationTranscript.resume`). It selects only that first value. + */ + readonly from?: EntryId; + /** Default 50; an integer, at least 1. */ + readonly min?: number; + /** Default `2 * min`; an integer, at least `min`. */ + readonly max?: number; +}; + +interface Conversation { + watch(context: Context, window?: ConversationWindow): Promise; +} +interface Harness { + observeConversation(id: ConversationId, observer: Observer, context: Context, window?: ConversationWindow): Promise>; +} ``` These kinds and their fields are public protocol. Third-party documents are -exposed through their own Chord objects, not mounted. +exposed through their own Chord objects, not mounted. Everything a renderer +draws is state in this value: the transcript in `entries`, the run and its +presentation in `pi.live` (`run`, `generation`, `tools`, `nestedTools`, +`compactions`, `compactionFailure`, `runFailure`, section 8.2), the queue in +`pi.inbox`, the agent in `pi.agent`, spend in `pi.usage`. A renderer renders +the newest value; it never assembles state from a stream of events. + +**The window.** Each watch and observer has its own window. It starts with the +newest `min` entries and the ID of the entry before them in `before`. With +`from`, it starts at `from` instead, when at most `max` entries are at or after +it; then `before` is the newest entry before `from`, and a window that resumes +right after the newest entry starts empty. Otherwise it starts at the newest +`min`, and `before` shows the client the entries it missed. `from` only picks +that first value: every entry committed after the attachment is appended, +whatever its ID, so a `from` past the newest entry, such as one a client +computed against another replica, gives an empty window whose `before` is the +newest entry, and then every entry that follows. The window has a stable start, +so new entries are appends. When it would hold more than `max` +entries, its oldest leave in one splice down to `min`, and `before` moves to +the last one that left: + +```text +min 3, max 5; the window holds 4..7, before 3 +commit appends 8: ["p", ["entries"], 4, 0, [8]] window 4..8 +commit appends 9: ["p", ["entries"], 5, 0, [9]], ["p", ["entries"], 0, 3, []], ["s", ["before"], 6] + window 7..9, before 6 +``` + +A commit that appends more than `max` entries at once keeps its newest `min`. +A first value, and an overflow replacement (section 9.2), therefore hold at +most `max` entries, never the whole transcript. Entry IDs are ordered but not +contiguous: a client never counts entries or infers ranges from IDs. `before` +tells whether a window continues what the client has, and +`Conversation.entries()` with `minEntryId`/`maxEntryId` and its cursor fetches +what lies in between; resuming from right after the newest entry and +`maxEntryId: oldest - 1` are exclusive bounds, the only arithmetic on IDs a +client does, and `ConversationTranscript` does both. A window with +bounds that are not integers, or `max` below `min`, rejects with a +`RangeError`. + +Head markers stay in place: a compaction summary, a reset, or a handoff is an +appended entry like any other. The view does not derive the active +transcript; `Conversation.context()` does (section 2.1). The Harness keeps at most one mount per conversation. The first `watch()` or -event attachment builds its revision on the Session line from the -committed active transcript and documents; the mount is dropped when its last +observer builds its documents on the Session line, and each attachment reads +its window there with bounded scans; the mount is dropped when its last observer detaches. A mount derives each next revision from the Session's commits as memory adopts them, for a stored commit inside Storage's `adopt` (section 4) and for a held one when memory adopts it (section 4.1), like -`watchDoc()`: it shows committed state, -including deferred commits before they are stored (section 4.1). One publication that -touches the conversation yields one Chord batch: +`watchDoc()`: it shows committed state, including deferred commits before they +are stored (section 4.1). One publication that touches the conversation yields +one Chord batch per observer: the document operations, applied once and +shared, then that observer's entry splices: ```text document op ["s", ["generation", "message"], value] -> view op ["s", ["docs", "pi.live", "generation", "message"], value] ``` -An appended entry without a head is a splice at the end of `entries`. An -appended head marker `H` makes the entries `H` followed by the current non-head -entries at or after `H.head`: the kept entries stay, and splices remove the -others and insert `H` at the front. Every kept entry is already mounted because -Harness writers never target a head before the current range: resets and -handoffs head themselves, compaction summaries target a kept entry of the -current range, and head writes that reach further back are stale (section 6). A raw head write that targets further back shows only the -mounted entries (section 12). Document creation and retirement set and delete the `docs` key. Parent entries through `parent.at` are immutable, so a child's mount follows only its own conversation. A publication that touches nothing mounted creates no revision; a redundant nonempty batch remains a real publication. Mounted document revisions -are structurally shared under the trusted immutability contract. The mount -performs no semantic projection and owns no persistence. +are structurally shared under the trusted immutability contract, and an entry +that stays in a window keeps its identity across revisions. The cost of a +commit is its operations plus, per observer that gets entries, one copy of its +window, at most `max` entries. The mount performs no semantic projection and +owns no persistence. `Conversation.watch()` exposes the mount through the serialized watch of section 9.2, with folded pending commits and a full-value overflow replacement; `Harness.observeConversation()` attaches a synchronous observer of the same mount (section 9.1). A view never retires, so its changes never set `retired`. -### 9.4 Agent events (experimental) - -`watchEvents(harness, conversationId, context): Promise`, -exported from the package root, is an experimental adapter that -translates one conversation's committed publications into agent events shaped -like the coding agent's `AgentSessionEvent`s. It owns no tracker or persistence, -covers exactly one conversation (not its owned subtree), and emits an event only -after the commit that makes it true. Its protocol may change without notice. +**A client's transcript.** A window only ever adds to what a client has seen: +entries never change once committed, and those that leave the window stay with +the client. `ConversationTranscript` keeps them, as a layer over any view, +wherever it comes from (a watch, or a replica of it on another host): ```ts -/** - * Which call a tool event is about. `toolCallId` is the provider's ID for a model-issued call and - * `/` for a nested one; consumers match it and never parse it. `taskId` is the call's tool task, - * absent for a call that never got one. A nested call also names its caller by call ID and by tool task. - */ -type ToolEventCall = { - toolCallId: string; - toolName: string; - taskId?: TaskId; - parentToolCallId?: string; - parentTaskId?: TaskId; -}; +type TranscriptItem = + | { readonly kind: "entry"; readonly entry: EntryRecord } + /** Entries strictly between `after` and `upTo` are missing: render "load more". */ + | { readonly kind: "gap"; readonly after: EntryId; readonly upTo: EntryId } + /** The transcript continues above: render "load earlier". */ + | { readonly kind: "earlier" }; -type AgentEvent = - | { - type: "snapshot"; - entries: readonly EntryRecord[]; - run?: { inputs: readonly SubmissionId[] }; - /** Current generation attempt: its in-flight partial, retry backoff, or deferred poll. */ - generation?: { attempt: number; message?: AssistantMessage; retry?: { at: number; error: string }; deferred?: { pollAt: number } }; - tools: readonly ToolSlot[]; - /** `pi.live.nestedTools` (section 8.2). */ - nestedTools: readonly NestedToolSlot[]; - /** `pi.live.compactions` (section 8.2). */ - compactions: readonly CompactionStatus[]; - /** `pi.live.compactionFailure` (section 8.2). */ - compactionFailure?: CompactionFailure; - inbox: readonly { id: SubmissionId; mode: InboxItem["mode"] }[]; - /** `pi.agent`; `{}` when absent. */ - agent: AgentState; - usage: UsageState; - } - | { type: "run_start"; inputs: readonly SubmissionId[] } - | { type: "run_end"; inputs: readonly SubmissionId[] } - | { type: "turn_start" } - | { type: "turn_end" } - | { type: "message_start"; message: Message } - | { type: "message_update"; usage: Usage; changes: readonly MessageChange[] } - | { type: "message_end"; entry: EntryRecord } - | ({ type: "tool_execution_start"; args: JsonObject } & ToolEventCall) - | ({ - type: "tool_execution_update"; - /** A front trim and then an append of the retained window, or its replacement. */ - output?: { trimStart?: number; append?: string } | { set: string }; - details?: JsonValue; - diagnostics?: readonly ToolDiagnostic[]; - } & ToolEventCall) - /** A model-issued call ends with its `entry`, a nested call with its `result`; see below for when neither. */ - | ({ type: "tool_execution_end"; entry?: EntryRecord; result?: NestedToolExecutionResult } & ToolEventCall) - | { type: "inbox_update"; items: readonly { id: SubmissionId; mode: InboxItem["mode"] }[] } - | { type: "submission"; record: SubmissionRecord } - | { type: "auto_retry_start"; attempt: number; at: number; errorMessage: string } - | { type: "auto_retry_end"; attempt: number } - | { type: "deferred_poll"; pollAt: number } - | { type: "entry_appended"; entry: EntryRecord } - | { type: "agent_changed"; agent: AgentState } - | { type: "usage_changed"; usage: UsageState } - | { type: "task_failed"; taskId: TaskId; kind: string; message: string } - | { type: "compaction_start"; taskId: TaskId; reason: CompactionReason; blocking: boolean } - /** The task's receipt tells whether it produced a summary; the summary entry has its own events. */ - | { type: "compaction_end"; taskId: TaskId; reason: CompactionReason; error?: string }; +/** The conversation view as a renderer sees it: `items` instead of the window. */ +type TranscriptView = Omit & { readonly items: readonly TranscriptItem[] }; -/** One change to the in-flight assistant message, relative to that message. */ -type MessageChange = - | { type: "text_start" | "thinking_start" | "toolcall_start"; contentIndex: number; block: AssistantMessage["content"][number] } - | { type: "text_delta" | "thinking_delta"; contentIndex: number; delta: string } - | { type: "toolcall_delta"; contentIndex: number; path: readonly (string | number)[]; delta: string } - | { type: "block"; contentIndex: number; block: AssistantMessage["content"][number] } - | { type: "message"; message: AssistantMessage }; +type EntryLoader = (query: Omit, limit: number, context: Context) => Promise>; -interface AgentEventStream { - /** The `snapshot` event at attachment. */ - readonly snapshot: Extract; - /** Each call's `value` is one commit's batch, or a `snapshot` after an overflow (Chord's `NativeWatchChange`). */ - start(listener: (change: { readonly value: readonly AgentEvent[] }, context: Context) => Promise): void; - stop(): Promise; - readonly closed: Promise; +class ConversationTranscript { + constructor(load: EntryLoader, options?: { readonly pageSize?: number }); // default 50 + /** Call with every value of the view. */ + merge(view: ConversationView): TranscriptView; + /** The page before the oldest entry held. */ + loadEarlier(context: Context): Promise; + /** The next page of a gap, from its start. */ + fillGap(gap: Extract, context: Context): Promise; + /** The ID the transcript reaches: its newest entry, or a later empty window's `before`. */ + readonly newest: EntryId | undefined; + /** `{ from }` right after `newest`, `{}` for an empty transcript, undefined before the first merge. */ + readonly resume: ConversationWindow | undefined; } ``` -Every event is relative to the `snapshot` and the events before it, never to -Durable's document layout. `snapshot` replaces everything a consumer holds; every -other event applies on top of it. Attachment captures the snapshot and -registers for later publications atomically on the Session line. +```ts +const transcript = new ConversationTranscript((query, limit, ctx) => conversation.entries(query, limit, undefined, ctx)); +const watch = await conversation.watch(context); +render(transcript.merge(watch.value)); +watch.start(async ({ value }) => render(transcript.merge(value))); +// A marker scrolled into view: +render(await transcript.loadEarlier(context)); +// Reconnect after a drop: +const resumed = await conversation.watch(context, transcript.resume); +``` -Events derive from committed changes: +A window whose `before` lies past the newest entry the client has does not +continue it: after a stall, an overflow, or a reconnect, the entries in between +become a gap. While the client keeps up there is never one: the window only +appends, and its splice removes entries the client already holds. +`fillGap()` loads `pageSize` entries from the gap's start and shrinks it, or +removes it once nothing is missing; `loadEarlier()` loads the page before the +oldest entry held, and the `earlier` item disappears once a page reaches the +transcript's start. A window, or a page, that covers part of a gap fills that +part. Items are in ID order: `earlier` first, then entries, each gap before +the entry after it. -- `run_start`/`run_end`: `pi.live.run` appears, is removed, or is replaced by a - successor run whose first input differs. Steers joining the current run and - handovers between its tasks are not run events. The inputs' outcomes are - `submission` events. -- `turn_start`: a `pi.generation` task is created. `turn_end`: a generation's - outcome is committed, when it holds `completing` or becomes terminal, - whichever comes first, so a successor created at hold starts after it. -- `message_start`: the first committed partial of an attempt, or, for a message - entry without a committed partial, the entry itself; `message_end` follows for - every entry with `model` messages. Generation commits a partial only once it has - content, and every built-in path that clears one appends its entry, so each - started message ends with its entry. Entries without messages are - `entry_appended`. -- `message_update`: the Chord operations on the committed partial, translated: - an insertion into `content` starts blocks, an append to a block's `text` or - `thinking` is a delta, an append to a string inside a tool call's `arguments` - is a `toolcall_delta` with the path relative to `arguments`, any other change - inside a block sends that `block`, which already holds the batch's later - changes to it, and any other change to `content` sends the whole `message`. - Each update carries the partial's current `usage`, as the coding agent's JSON - mode does; a change of only `usage` sends no changes. Only deltas travel, so throttled - partials cause no write amplification on the wire. -- `tool_execution_start`: a slot becomes `running`; `args` come from the tool - task's intent checkpoint in the same commit. A slot that becomes `done` - without running (a call not offered, invalid arguments, a block, a fault - before intent) gets only `tool_execution_end`. `tool_execution_update`: output - appends, front trims, and replacements of the slot's retained window, and the - slot's current `details` and `diagnostics` when they change; removed ones, as - when a safe replay restarts the tool, send `null` and `[]`. - `tool_execution_end`: the slot becomes `done`, with its `pi.tool-result` - entry, which carries the diagnostics, or without one after a fault or - orphan. An unfinished slot that disappears because its run ended ends with the - result entry appended in the same commit, as for the unstarted calls of an - aborted round, directly before its `message_start`, or without an entry. - Nested slots produce the same events with `parentToolCallId`; a nested call - ends with its `result`, taken from the `NestedCallDoc` value its terminal - commit published, or without one after a fault or orphan or when its slot - disappears unfinished. Snapshots show done nested slots with their summary, - not their result; a late subscriber reads a live caller's results from its - `NestedCallDoc`, keyed by call key (the slot's `callId` after `/`). -- `inbox_update`, `agent_changed`, `usage_changed`: the document changed; a - retired one reads as its initial value, as in a snapshot. Registry installs - and settings changes make no commit and emit nothing; a UI showing resolved - tools or a resolved model resolves again through `Conversation.agent()`. -- `auto_retry_start`/`auto_retry_end`, `deferred_poll`: `pi.live.generation` - gains or drops `retry`, or gains `deferred` or moves its `pollAt`. -- `task_failed`: a task of the conversation settles `faulted` or `orphaned`. -- `compaction_start`/`compaction_end`: a status appears in or disappears from - `pi.live.compactions`. `compaction_end` carries `error` when the same commit - recorded the task in `pi.live.compactionFailure`. A retry backoff shows only - in the snapshot's `compactions`. A queued summary is placed later with its own - `message_start`/`message_end` and `submission` events. +`items` is a plain array. A merge costs O(items), never re-sorts or +re-fetches, and one that adds nothing returns the previous `items` array; with +the same `docs` and `conversation` it returns the previous view itself. +Concurrent `loadEarlier()` calls, and concurrent `fillGap()` calls for one gap, +share one load, also when one passes a gap descriptor that has shrunk since; a +load that rejects, or whose loader throws, is not kept, so the next call loads +again. A page without a next cursor, or without items, ends the range: the gap +closes, or the transcript's start is reached. An empty window adds nothing. +`loadEarlier()` before the first merge rejects; after the start is reached, and +`fillGap()` for a gap that is gone, return the current view. +Keeping the scroll position when items are inserted above is the renderer's +job. -One commit produces one batch, in this order: `tool_execution_start`, -`message_start` of a first partial, `message_update`, `tool_execution_update`, -and retry/deferred events; then entries in append order with their -`message_start`/`message_end` or `entry_appended`, where a tool result's -`tool_execution_end` directly precedes its `message_start`, as in the coding -agent, and the `tool_execution_end` of nested calls, children first, precede -the entries; then the `tool_execution_end` of tools ending without an entry; -then `compaction_end`, `task_failed`, `turn_end`, `run_end`; then `submission` -events in ID order; then `inbox_update`, `agent_changed`, and `usage_changed`; -and last `compaction_start`, `run_start`, and `turn_start`. A stream buffers at most 100 undelivered -batches; adding another replaces every undelivered batch with one `snapshot` of -the newest committed state. The stream therefore converges but does not promise -every transition; `Submission.wait()` reports exact outcomes. A reconnecting -consumer attaches again and starts from its `snapshot`; nothing is replayed. -Transport backpressure and disconnect policy belong to the consumer. - -Print mode awaits its own input `Submission` and prints its answer. A TUI -renders `ConversationView`; events may drive transient animation. - -### 9.5 Task graph view +### 9.4 Task graph view The task graph view shows every live task of the Session as one structural value, for UIs and debugging: what runs, what waits for what, and which @@ -5574,8 +5576,7 @@ may not have committed, so a host that resubmits after reopening reuses its and every wait that starts later rejects with `SessionFailed`, whose `cause` is the first error; so does every Storage call still underway when it returns, whatever it returned. -- Watches and event streams end with `{ reason: "failed", error }` and keep - their last value. +- Watches end with `{ reason: "failed", error }` and keep their last value. - Running task invocations get their abort signal. `HarnessOptions.onReport` gets the error once, after the Session is sealed, so a report handler that calls back finds it failed. @@ -5785,8 +5786,8 @@ These are contracts, not invitations to add defensive machinery: messages, can misjudge; the Harness estimate uses entry order (section 8.3). - **Raw head writes into the past:** a head written directly with `tx.appendEntry()` that targets an entry before the conversation's active range changes model - context, but a mounted view keeps only the entries it already holds until the - mount is rebuilt. Use a write submission, whose stale check rejects it. + context in ways a queued write would not. Use a write submission, whose stale + check rejects it. - **Double-counted subagent spend:** a tool that runs an owned conversation must not report that conversation's usage in its result; the child's `pi.usage` already counts it, and subtree sums would count it twice. diff --git a/packages/durable/package.json b/packages/durable/package.json index d40ddfbd8..106512d21 100644 --- a/packages/durable/package.json +++ b/packages/durable/package.json @@ -131,6 +131,7 @@ "bench:document-cache": "node --conditions=source --experimental-strip-types test/session-document-cache-bench.ts", "bench:document-append": "node --conditions=source --experimental-strip-types test/document-append-bench.ts", "bench:codemode": "node --conditions=source --experimental-strip-types test/codemode-bench.ts", + "bench:conversation-view": "node --conditions=source --experimental-strip-types --expose-gc test/conversation-view-bench.ts", "prepublishOnly": "npm run clean && npm run build" }, "keywords": [ diff --git a/packages/durable/src/harness/context.ts b/packages/durable/src/harness/context.ts index c818b3cb5..aa5660801 100644 --- a/packages/durable/src/harness/context.ts +++ b/packages/durable/src/harness/context.ts @@ -253,17 +253,6 @@ function settle( return { settled: [...settled, ...orderToolResults(open.slice(0, last))], open: open.slice(last) }; } -/** The raw active entries within captured bounds, without deriving model context. */ -export async function activeEntries( - storage: Storage, - conversationId: ConversationId, - bounds: ContextBounds | undefined, - context: Context, -): Promise { - if (bounds === undefined) return []; - return selectActive(bounds.head, await scanRange(storage, conversationId, rangeQuery(bounds), context)); -} - /** The visible range of `bounds`: from the head marker's head, or transcript start, through the tail. */ function rangeQuery(bounds: ContextBounds): Omit { const head = bounds.head; diff --git a/packages/durable/src/harness/events.ts b/packages/durable/src/harness/events.ts deleted file mode 100644 index f34af4659..000000000 --- a/packages/durable/src/harness/events.ts +++ /dev/null @@ -1,512 +0,0 @@ -import { - type Context, - type JsonValue, - NativeWatch, - type NativeWatchChange, - type WatchEnd, -} from "@earendil-works/chord"; -import type { Op, Path } from "@earendil-works/chord/delta"; -import type { AssistantMessage, Message, Usage } from "@earendil-works/pi-ai"; -import type { ObservedCommit } from "../session/session.ts"; -import type { - CommitChange, - ConversationId, - EntryRecord, - JsonObject, - SubmissionId, - SubmissionRecord, - TaskId, -} from "../types.ts"; -import { AgentDoc } from "./agent.ts"; -import type { InboxItem, InboxState } from "./inbox.ts"; -import type { CompactionFailure, CompactionStatus, LiveState, NestedToolSlot, ToolSlot } from "./live.ts"; -import { NestedCallDoc, type NestedCallState } from "./tool.ts"; -import type { AgentState, CompactionReason, Harness, NestedToolExecutionResult, ToolDiagnostic } from "./types.ts"; -import { UsageDoc, type UsageState } from "./usage.ts"; -import { scanAll } from "./util.ts"; -import { type ConversationView, conversationViews } from "./view.ts"; - -type Block = AssistantMessage["content"][number]; -type QueuedItem = { id: SubmissionId; mode: InboxItem["mode"] }; - -/** One change to the in-flight assistant message, relative to that message. */ -export type MessageChange = - | { type: "text_start" | "thinking_start" | "toolcall_start"; contentIndex: number; block: Block } - | { type: "text_delta" | "thinking_delta"; contentIndex: number; delta: string } - | { type: "toolcall_delta"; contentIndex: number; path: readonly (string | number)[]; delta: string } - | { type: "block"; contentIndex: number; block: Block } - | { type: "message"; message: AssistantMessage }; - -export type SnapshotEvent = { - type: "snapshot"; - entries: readonly EntryRecord[]; - run?: { inputs: readonly SubmissionId[] }; - /** Current generation attempt: its in-flight partial, retry backoff, or deferred poll. */ - generation?: { - attempt: number; - message?: AssistantMessage; - retry?: { at: number; error: string }; - deferred?: { pollAt: number }; - }; - tools: readonly ToolSlot[]; - /** `pi.live.nestedTools`: nested calls of running tool calls. */ - nestedTools: readonly NestedToolSlot[]; - /** `pi.live.compactions`: live compactions with their attempt and retry backoff. */ - compactions: readonly CompactionStatus[]; - /** `pi.live.compactionFailure`: the newest compaction failure since a compaction last produced a summary. */ - compactionFailure?: CompactionFailure; - inbox: readonly QueuedItem[]; - /** `pi.agent`; `{}` when absent. */ - agent: AgentState; - usage: UsageState; -}; - -/** Experimental agent event, shaped like the coding agent's session events (spec §9.4). */ -export type AgentEvent = - | SnapshotEvent - | { type: "run_start"; inputs: readonly SubmissionId[] } - | { type: "run_end"; inputs: readonly SubmissionId[] } - | { type: "turn_start" } - | { type: "turn_end" } - | { type: "message_start"; message: Message } - /** `usage` is the partial's current usage, as in the coding agent's JSON mode. */ - | { type: "message_update"; usage: Usage; changes: readonly MessageChange[] } - | { type: "message_end"; entry: EntryRecord } - | ({ type: "tool_execution_start"; args: JsonObject } & ToolEventCall) - | ({ - type: "tool_execution_update"; - /** A front trim and then an append of the retained window, or its replacement. */ - output?: { trimStart?: number; append?: string } | { set: string }; - details?: JsonValue; - diagnostics?: readonly ToolDiagnostic[]; - } & ToolEventCall) - /** - * A model-issued call ends with its result `entry`, a nested call with its `result`. Both are absent when the tool - * task faulted or was orphaned, or when the call's slot left `pi.live` unfinished: its run ended, or its parent - * settled first. - */ - | ({ type: "tool_execution_end"; entry?: EntryRecord; result?: NestedToolExecutionResult } & ToolEventCall) - | { type: "inbox_update"; items: readonly QueuedItem[] } - | { type: "submission"; record: SubmissionRecord } - | { type: "auto_retry_start"; attempt: number; at: number; errorMessage: string } - | { type: "auto_retry_end"; attempt: number } - | { type: "deferred_poll"; pollAt: number } - | { type: "entry_appended"; entry: EntryRecord } - | { type: "agent_changed"; agent: AgentState } - | { type: "usage_changed"; usage: UsageState } - | { type: "task_failed"; taskId: TaskId; kind: string; message: string } - | { type: "compaction_start"; taskId: TaskId; reason: CompactionReason; blocking: boolean } - /** - * The task's receipt tells whether it produced a summary; the summary entry has its own events. `error` is set when - * the compaction failed, faulted, or was orphaned, as in `pi.live.compactionFailure`. - */ - | { type: "compaction_end"; taskId: TaskId; reason: CompactionReason; error?: string }; - -/** - * Which call a tool event is about. `toolCallId` is the call's ID in the transcript and in tool events: the provider's - * ID for a model-issued call, `/` for a nested one. Match events by it; do not parse it. `taskId` - * is the call's tool task, absent for a call that never got one (not offered, or not started in a sequential round). - * A nested call also names the call that made it, by call ID and by tool task. - */ -export type ToolEventCall = { - toolCallId: string; - toolName: string; - taskId?: TaskId; - parentToolCallId?: string; - parentTaskId?: TaskId; -}; - -/** Serialized stream of one conversation's event batches, one per commit. */ -export interface AgentEventStream { - /** The `snapshot` event at attachment. */ - readonly snapshot: SnapshotEvent; - /** Install the sole listener; each call's `value` is one commit's batch, or a `snapshot` after an overflow. */ - start(listener: (change: NativeWatchChange, context: Context) => Promise): void; - stop(): Promise; - readonly closed: Promise; -} - -/** The typed parts of a view the events read. */ -type Parts = { - live: LiveState; - inbox: InboxState | undefined; - agent: AgentState | undefined; - usage: UsageState | undefined; -}; - -function parts(view: ConversationView): Parts { - return { - live: (view.docs["pi.live"] ?? {}) as LiveState, - inbox: view.docs["pi.inbox"] as InboxState | undefined, - agent: view.docs["pi.agent"] as AgentState | undefined, - usage: view.docs["pi.usage"] as UsageState | undefined, - }; -} - -function snapshotOf(view: ConversationView): SnapshotEvent { - const { live, inbox, agent, usage } = parts(view); - return { - type: "snapshot", - entries: view.entries, - ...(live.run === undefined ? {} : { run: { inputs: live.run.inputs } }), - ...(live.generation === undefined ? {} : { generation: live.generation as SnapshotEvent["generation"] }), - tools: live.tools ?? [], - nestedTools: live.nestedTools ?? [], - compactions: live.compactions ?? [], - ...(live.compactionFailure === undefined ? {} : { compactionFailure: live.compactionFailure }), - inbox: queued(inbox), - agent: agent ?? AgentDoc.definition.initial(), - usage: usage ?? UsageDoc.definition.initial(), - }; -} - -function queued(inbox: InboxState | undefined): QueuedItem[] { - return (inbox?.items ?? []).map((item) => ({ id: item.id, mode: item.mode })); -} - -/** - * Experimental: attach to one conversation's agent events (spec §9.4). The snapshot and the registration for later - * commits are captured atomically on the Session line; overflow replaces undelivered batches with one snapshot. - */ -export async function watchEvents( - harness: Harness, - conversationId: ConversationId, - context: Context, -): Promise { - const views = conversationViews(harness); - let watch!: NativeWatch; - let snapshot!: SnapshotEvent; - await views.attach( - conversationId, - async (initial, release, storage) => { - // Generations whose held outcome already ended their turn, read on the line with the snapshot. - const query = { conversationId, kind: "pi.generation", status: "completing" } as const; - const completing = await scanAll((cursor) => storage.scanTasks(query, 100, cursor, context)); - const held = new Set(completing.map((record) => record.id)); - let current = initial; - snapshot = snapshotOf(initial); - // Batches are the watch's values, each delivered (folding would drop events); an overflow delivers a snapshot of - // the newest view instead. - watch = new NativeWatch( - [], - release, - (error) => views.report(error), - () => [snapshotOf(current)], - ); - return { - publication: (before, after, ops, publication, commitContext) => { - current = after; - const events = translate(conversationId, before, after, ops, publication, held); - if (events.length > 0) watch.advance(events, commitContext); - }, - end: (failure) => watch.end(failure), - }; - }, - context, - ); - // Like a watch, the acquisition context governs the stream's lifetime. - const signal = context.abortSignal; - if (signal?.aborted) { - watch.cancel(); - throw signal.reason; - } - if (signal !== undefined) watch.observeCancellation(signal); - return { - snapshot, - start: (listener) => watch.start(listener), - stop: () => watch.stop(), - closed: watch.closed, - }; -} - -type TaskChange = Extract; - -/** The tool result for `callId` among `entries`. */ -function resultOf(entries: readonly EntryRecord[], callId: string): EntryRecord | undefined { - return entries.find((entry) => { - const message = entry.model?.[0]; - return message?.role === "toolResult" && message.toolCallId === callId; - }); -} - -/** Every event one publication causes, in the order of spec §9.4. */ -function translate( - conversationId: ConversationId, - before: ConversationView, - after: ConversationView, - viewOps: readonly Op[], - publication: ObservedCommit, - held: Set, -): AgentEvent[] { - const entries: EntryRecord[] = []; - const tasks = new Map(); - const submissions: SubmissionRecord[] = []; - // Nested results this commit stored, by nested task ID; read now, since the caller's documents may retire next. - const nestedResults = new Map(); - for (const change of publication.changes) { - if ( - change.type === "document" && - change.record.kind === NestedCallDoc.definition.kind && - change.value !== null && - change.conversationId === conversationId - ) { - const state = change.value as NestedCallState; - if (state.taskId !== undefined && state.result !== undefined) { - nestedResults.set(state.taskId, state.result as unknown as NestedToolExecutionResult); - } - } - if (change.type === "entry" && change.value.conversationId === conversationId) entries.push(change.value); - if (change.type === "task" && change.value.conversationId === conversationId) - tasks.set(change.value.id, change.value); - if (change.type === "submission" && change.value.conversationId === conversationId) { - submissions.push(change.value); - } - } - if (viewOps.length === 0 && entries.length === 0 && tasks.size === 0 && submissions.length === 0) return []; - // Entries are appended in ID order; submission records are published in the order the commit first touched them. - submissions.sort((a, b) => a.id - b.id); - const was = parts(before); - const now = parts(after); - const events: AgentEvent[] = []; - - // Progress: tool starts, the in-flight message, tool updates, retry and deferred state. - const slotsBefore = new Map((was.live.tools ?? []).map((slot) => [slot.callId, slot])); - const slots = now.live.tools ?? []; - const nestedBefore = new Map((was.live.nestedTools ?? []).map((slot) => [slot.taskId, slot])); - const nested = now.live.nestedTools ?? []; - for (const slot of [...slots, ...nested]) { - const previous = "parentCallId" in slot ? nestedBefore.get(slot.taskId) : slotsBefore.get(slot.callId); - if (slot.status !== "running" || previous?.status === "running") continue; - // A nested slot carries the arguments the call runs with; a model-issued call's are in its intent checkpoint. - const checkpoint = slot.taskId === undefined ? undefined : tasks.get(slot.taskId)?.state.checkpoint; - const args = - "parentCallId" in slot - ? slot.arguments - : ((checkpoint as { arguments?: JsonObject } | undefined)?.arguments ?? {}); - events.push({ type: "tool_execution_start", ...callOf(slot), args }); - } - const partialBefore = was.live.generation?.message as AssistantMessage | undefined; - const partial = now.live.generation?.message as AssistantMessage | undefined; - if (partial !== undefined && partialBefore === undefined) events.push({ type: "message_start", message: partial }); - else if (partial !== undefined && partial !== partialBefore) { - events.push({ type: "message_update", usage: partial.usage, changes: messageChanges(viewOps, partial) }); - } - for (const [index, slot] of slots.entries()) { - const previous = slotsBefore.get(slot.callId); - if (slot.status !== "running" || previous?.status !== "running") continue; - const update = toolUpdate(viewOps, ["tools", index], slot, previous); - if (update === undefined) continue; - events.push({ type: "tool_execution_update", ...callOf(slot), ...update }); - } - for (const [index, slot] of nested.entries()) { - const previous = nestedBefore.get(slot.taskId); - if (slot.status !== "running" || previous?.status !== "running") continue; - const update = toolUpdate(viewOps, ["nestedTools", index], slot, previous); - if (update === undefined) continue; - events.push({ type: "tool_execution_update", ...callOf(slot), ...update }); - } - const generation = now.live.generation; - const generationBefore = was.live.generation; - if (generation?.retry !== undefined && generationBefore?.retry === undefined) { - const { at, error } = generation.retry; - events.push({ type: "auto_retry_start", attempt: generation.attempt, at, errorMessage: error }); - } - if (generationBefore?.retry !== undefined && generation?.retry === undefined) { - events.push({ type: "auto_retry_end", attempt: generationBefore.attempt }); - } - if (generation?.deferred !== undefined && generation.deferred.pollAt !== generationBefore?.deferred?.pollAt) { - events.push({ type: "deferred_poll", pollAt: generation.deferred.pollAt }); - } - - // Tools that end in this commit: a slot that becomes done, one created done (a call not offered), or an unfinished - // one that vanishes because its run ended. A done slot that vanishes ended earlier. - const toolEnds: Extract[] = []; - const endTool = (slot: ToolSlot, entryId: number | undefined): void => { - const entry = entries.find((candidate) => candidate.id === entryId); - toolEnds.push({ type: "tool_execution_end", ...callOf(slot), ...(entry === undefined ? {} : { entry }) }); - }; - for (const previous of slotsBefore.values()) { - if (previous.status === "done") continue; - const slot = slots.find((candidate) => candidate.callId === previous.callId); - if (slot?.status === "done") endTool(slot, slot.entry); - // A slot whose run ended in this commit may have had its result appended with it, as for unstarted calls. - else if (slot === undefined) endTool(previous, resultOf(entries, previous.callId)?.id); - } - for (const slot of slots) { - if (slot.status === "done" && !slotsBefore.has(slot.callId)) endTool(slot, slot.entry); - } - // Nested calls that end in this commit, the same way, children before the calls that made them: the lists hold - // parents first, so walk them backwards. - const endNested = (slot: NestedToolSlot): void => { - const result = nestedResults.get(slot.taskId); - events.push({ type: "tool_execution_end", ...callOf(slot), ...(result === undefined ? {} : { result }) }); - }; - const nestedNow = new Map(nested.map((slot) => [slot.taskId, slot])); - for (const previous of [...nestedBefore.values()].reverse()) { - if (previous.status === "done") continue; - const slot = nestedNow.get(previous.taskId); - if (slot?.status === "done" || slot === undefined) endNested(slot ?? previous); - } - for (const slot of [...nested].reverse()) { - if (slot.status === "done" && !nestedBefore.has(slot.taskId)) endNested(slot); - } - - // Entries in append order; a tool's end directly precedes its result's message, as in the coding agent. - let assistantAppended = false; - for (const entry of entries) { - events.push(...toolEnds.filter((end) => end.entry === entry)); - const message = entry.model?.[0]; - if (message === undefined) { - events.push({ type: "entry_appended", entry }); - continue; - } - // A streamed answer already started with its first partial. - const streamed = message.role === "assistant" && partialBefore !== undefined && !assistantAppended; - if (message.role === "assistant") assistantAppended = true; - if (!streamed) events.push({ type: "message_start", message }); - events.push({ type: "message_end", entry }); - } - // Ends without a result entry: a faulted or orphaned tool, or one whose run ended. - events.push(...toolEnds.filter((end) => end.entry === undefined)); - - // Compaction ends, task failures, then turn and run ends. - const compactionsBefore = was.live.compactions ?? []; - const compactions = now.live.compactions ?? []; - const failure = now.live.compactionFailure; - for (const { taskId, reason } of compactionsBefore) { - if (!compactions.some((status) => status.taskId === taskId)) { - const error = failure?.taskId === taskId ? failure.error : undefined; - events.push({ type: "compaction_end", taskId, reason, ...(error === undefined ? {} : { error }) }); - } - } - // A generation's turn ends when its outcome is committed: at a `completing` hold or at terminal, whichever comes - // first, so a successor created at the hold starts after it. - let turnEnded = false; - for (const task of tasks.values()) { - const status = task.state.status; - if (task.kind === "pi.generation" && status === "completing" && !held.has(task.id)) { - held.add(task.id); - turnEnded = true; - } - if (status !== "terminal") continue; - if (task.kind === "pi.generation" && !held.delete(task.id)) turnEnded = true; - const outcome = task.state.outcome; - if (outcome.status === "faulted" || outcome.status === "orphaned") { - const message = outcome.status === "faulted" ? outcome.error.message : outcome.reason; - events.push({ type: "task_failed", taskId: task.id, kind: task.kind, message }); - } - } - if (turnEnded) events.push({ type: "turn_end" }); - const run = now.live.run; - const runBefore = was.live.run; - const runChanged = run?.inputs[0] !== runBefore?.inputs[0]; - if (runBefore !== undefined && runChanged) events.push({ type: "run_end", inputs: runBefore.inputs }); - - // Submissions, document state, then what began. - for (const record of submissions) events.push({ type: "submission", record }); - if (now.inbox !== was.inbox) events.push({ type: "inbox_update", items: queued(now.inbox) }); - // A retired document reads as its initial value, as in a snapshot. - if (now.agent !== was.agent) { - events.push({ type: "agent_changed", agent: now.agent ?? AgentDoc.definition.initial() }); - } - if (now.usage !== was.usage) - events.push({ type: "usage_changed", usage: now.usage ?? UsageDoc.definition.initial() }); - for (const { taskId, reason, blocking } of compactions) { - if (!compactionsBefore.some((status) => status.taskId === taskId)) { - events.push({ type: "compaction_start", taskId, reason, blocking }); - } - } - if (run !== undefined && runChanged) events.push({ type: "run_start", inputs: run.inputs }); - if (run !== undefined && run.taskId !== runBefore?.taskId && tasks.get(run.taskId)?.kind === "pi.generation") { - events.push({ type: "turn_start" }); - } - return events; -} - -const PARTIAL_PATH = ["docs", "pi.live", "generation", "message"]; - -/** Translate the view operations on the in-flight message into message changes (spec §9.4). */ -function messageChanges(viewOps: readonly Op[], message: AssistantMessage): MessageChange[] { - const changes: MessageChange[] = []; - // A block sent whole already holds every later change to it in this batch. - const whole = new Set(); - for (const op of viewOps) { - // View operations never replace the root. - const path = op[1] as Path; - if (!startsWith(path, PARTIAL_PATH)) { - // The whole message or generation was replaced. - if (startsWith(PARTIAL_PATH, path)) return [{ type: "message", message }]; - continue; - } - const rest = path.slice(PARTIAL_PATH.length); - if (rest[0] === "usage") continue; - if (rest[0] !== "content") return [{ type: "message", message }]; - if (rest.length === 1) { - if (op[0] !== "p" || op[3] !== 0) return [{ type: "message", message }]; - for (const [offset, block] of (op[4] as unknown as Block[]).entries()) { - const type = - block.type === "text" ? "text_start" : block.type === "thinking" ? "thinking_start" : "toolcall_start"; - changes.push({ type, contentIndex: op[2] + offset, block }); - } - continue; - } - const contentIndex = rest[1] as number; - const field = rest[2]; - if (whole.has(contentIndex)) continue; - if (op[0] === "a" && rest.length === 3 && (field === "text" || field === "thinking")) { - changes.push({ type: field === "text" ? "text_delta" : "thinking_delta", contentIndex, delta: op[2] }); - } else if (op[0] === "a" && field === "arguments") { - changes.push({ type: "toolcall_delta", contentIndex, path: rest.slice(3), delta: op[2] }); - } else { - whole.add(contentIndex); - changes.push({ type: "block", contentIndex, block: message.content[contentIndex]! }); - } - } - return changes; -} - -/** The identifying fields of a slot's tool events. */ -function callOf(slot: ToolSlot | NestedToolSlot): ToolEventCall { - return { - toolCallId: slot.callId, - toolName: slot.name, - ...(slot.taskId === undefined ? {} : { taskId: slot.taskId }), - ...("parentCallId" in slot ? { parentToolCallId: slot.parentCallId, parentTaskId: slot.parentTaskId } : {}), - }; -} - -/** Output, details, and diagnostics changes of a running slot at `at` in `pi.live`, from the view operations on it. */ -function toolUpdate( - viewOps: readonly Op[], - at: readonly ["tools" | "nestedTools", number], - slot: ToolSlot | NestedToolSlot, - previous: ToolSlot | NestedToolSlot, -): Omit, "type" | keyof ToolEventCall> | undefined { - const outputPath = ["docs", "pi.live", ...at, "output"]; - let trimStart = 0; - let append = ""; - let set = false; - for (const op of viewOps) { - if (!startsWith(op[1] as Path, outputPath)) continue; - if (op[0] === "t") trimStart += op[2]; - else if (op[0] === "a") append += op[2]; - else set = true; - } - let output: { trimStart?: number; append?: string } | { set: string } | undefined; - if (set || (slot.output !== previous.output && trimStart === 0 && append === "")) - output = { set: slot.output ?? "" }; - else if (trimStart > 0 || append !== "") { - output = { ...(trimStart > 0 ? { trimStart } : {}), ...(append === "" ? {} : { append }) }; - } - // A safe replay clears a running slot's progress: removed details send `null`, removed diagnostics `[]`. - const update = { - ...(output === undefined ? {} : { output }), - ...(slot.details === previous.details ? {} : { details: slot.details ?? null }), - ...(slot.diagnostics === previous.diagnostics ? {} : { diagnostics: slot.diagnostics ?? [] }), - }; - return Object.keys(update).length === 0 ? undefined : update; -} - -function startsWith(path: Path, prefix: readonly (string | number)[]): boolean { - return prefix.length <= path.length && prefix.every((segment, index) => path[index] === segment); -} diff --git a/packages/durable/src/harness/generation.ts b/packages/durable/src/harness/generation.ts index 210ae981e..53d75fd73 100644 --- a/packages/durable/src/harness/generation.ts +++ b/packages/durable/src/harness/generation.ts @@ -21,7 +21,6 @@ import type { EntryId, NextTaskState, SubmissionId, - SubmissionSettlement, TaskId, TaskOutcome, TaskRuntime, @@ -31,7 +30,7 @@ import type { import { addTools, sectionExtension } from "./agent.ts"; import { compactionPolicy, createCompaction, estimateContext, selectCut } from "./compaction.ts"; import { applyBoundary, type Boundary, prepareBoundary, withdrawQueuedInputs } from "./inbox.ts"; -import { endRun, LiveDoc, type LiveState, PROGRESS_COMMIT, type ToolSlot } from "./live.ts"; +import { endRun, LiveDoc, type LiveState, PROGRESS_COMMIT, type RunEnd, type ToolSlot } from "./live.ts"; import { PartialChanges, type PartialMessage } from "./partial-changes.ts"; import { planSystemEntries, renderSections, replaySections } from "./prompt.ts"; import { ensureProviderSessionId } from "./provider.ts"; @@ -396,7 +395,12 @@ async function failNoModel(runtime: Runtime, ref: ModelRef | undefined, context: await runtime.commit(async (tx) => { const failure = await prepareFailure(tx, runtime); const live = await tx.doc(LiveDoc, runtime.conversationId); - await endFailedRun(tx, runtime, live, failure, { status: "unanswered", reason: "no_model" }); + await endFailedRun(tx, runtime, live, failure, { + status: "unanswered", + reason: "no_model", + detail: message, + settlement: { status: "unanswered", reason: "no_model" }, + }); return { status: "terminal", outcome: { status: "failed", error: { message, detail: { reason: "no_model" } } } }; }, context); } @@ -420,14 +424,14 @@ async function endFailedRun( runtime: Runtime, live: Draft, failure: Failure, - settlement: SubmissionSettlement, + end: Extract, ): Promise { const conversationId = runtime.conversationId; - endRun(tx, live, runtime.taskId, settlement); + endRun(tx, live, runtime.taskId, end); if (failure.policy === "withdraw") await withdrawQueuedInputs(tx, conversationId, "run_failed"); if (failure.boundary === undefined) return; const { users } = await applyBoundary(tx, failure.boundary, "final", runtime.now()); - if (users.length > 0) await startRun(tx, conversationId, live, users); + if (users.length > 0) await startRun(tx, conversationId, live, users, { keepFailure: true }); } /** @@ -747,7 +751,7 @@ async function finishToolRound( if (reset) { // The queued reset cut the run's context before an answer. endRun(tx, live, runtime.taskId, { status: "unanswered", reason: "reset" }); - if (users.length > 0) await startRun(tx, conversationId, live, users); + if (users.length > 0) await startRun(tx, conversationId, live, users, { keepFailure: true }); } else { delete live.tools; delete live.nestedTools; @@ -772,14 +776,19 @@ async function appendAssistant( return tx.appendEntry(AssistantEntry, conversationId, { model: [message] }); } -/** Start a run for `inputs`, placed input submissions: a new generation takes `pi.live.run`. */ +/** + * Start a run for `inputs`, placed input submissions: a new generation takes `pi.live.run`. Removes `runFailure`, unless + * `keepFailure`: a run started in the commit that ended the failed one keeps its failure. + */ export async function startRun( tx: Tx, conversationId: ConversationId, live: Draft, inputs: SubmissionId[], + options?: { readonly keepFailure?: boolean }, ): Promise { live.run = { taskId: await createGeneration(tx, conversationId, { submissionIds: inputs }), inputs }; + if (options?.keepFailure !== true) delete live.runFailure; } /** A generation owned by its conversation. */ diff --git a/packages/durable/src/harness/harness.ts b/packages/durable/src/harness/harness.ts index 1df3422d7..9ca24f550 100644 --- a/packages/durable/src/harness/harness.ts +++ b/packages/durable/src/harness/harness.ts @@ -74,7 +74,7 @@ import type { } from "./types.ts"; import { addUsageState, UsageDoc, type UsageState } from "./usage.ts"; import { scanAll } from "./util.ts"; -import { type ConversationView, ConversationViews } from "./view.ts"; +import { type ConversationView, ConversationViews, type ConversationWindow } from "./view.ts"; const SCAN_PAGE_SIZE = 256; @@ -178,8 +178,8 @@ class ConversationImpl implements Conversation { return this.#host.tasks.waitForIdle(this.id, context); } - watch(context: Context): Promise { - return this.#host.views.watch(this.id, context); + watch(context: Context, window?: ConversationWindow): Promise { + return this.#host.views.watch(this.id, context, window); } } @@ -384,9 +384,10 @@ class HarnessImpl extends SessionImpl implements id: ConversationId, observer: Observer, context: Context, + window?: ConversationWindow, ): Promise> { this.assertUsable(); - return this.#host.views.observe(id, observer, context); + return this.#host.views.observe(id, observer, context, window); } root( diff --git a/packages/durable/src/harness/live.ts b/packages/durable/src/harness/live.ts index 55759479f..41efa804a 100644 --- a/packages/durable/src/harness/live.ts +++ b/packages/durable/src/harness/live.ts @@ -44,7 +44,7 @@ export type ToolSlot = SlotProgress & { /** * Presentation of one nested call: a call a running tool made through `executeTool()`. `callId` is - * `/`, the call's ID in tool events and unique because keys contain no `/`. The Harness relates + * `/`, the call's ID, unique because keys contain no `/`. The Harness relates * slots by task ID: `taskId` is the call's tool task, `parentTaskId` the calling one, model-issued or nested. */ export type NestedToolSlot = SlotProgress & { @@ -87,6 +87,32 @@ export type CompactionFailure = { error: string; }; +/** + * Why a run ended without an answer: + * - `model_error`: the model's response failed, after any retries, or the compaction a request needed failed; + * - `no_model`: no model is configured, or the configured one is not available; + * - `aborted`: the run was aborted (`Conversation.abort()`, `abortTask()`); + * - `reset`: a queued reset started a new context at the post-tools boundary, before an answer (a tool's handoff + * continues the run); + * - `faulted`: the run's task threw where it cannot recover, or storage rejected its commit; + * - `orphaned`: the run's task was aborted while no registered definition could take it. + */ +export type RunFailureReason = "model_error" | "no_model" | "aborted" | "reset" | "faulted" | "orphaned"; + +/** The newest run that ended without an answer (spec §8.2). */ +export type RunFailure = { + /** The task that ended the run: the run's task at its end. */ + taskId: TaskId; + /** The run's inputs, settled `unanswered`. */ + inputs: SubmissionId[]; + reason: RunFailureReason; + /** + * `model_error`: the error text; `no_model`: which model is missing; `faulted`: the error message; `orphaned`: why + * the task could not run (`missing_task`, `task_too_old`, `migration_failed`). + */ + detail?: string; +}; + /** Built-in live conversation state: run control and presentation of the current generation and tool round. */ export type LiveState = { /** Run control: the task that settles the run's inputs, and those inputs; present exactly while busy. */ @@ -115,6 +141,12 @@ export type LiveState = { * removes it; one that is aborted or finds nothing to compact leaves it. */ compactionFailure?: CompactionFailure; + /** + * Why the newest run ended without an answer, so a client can show it after the run ended. Set in the commit that + * ends the run; removed when a later commit starts a run. A run started in the same commit, by + * `afterFailedRun: "continue"` or after a reset, keeps it. + */ + runFailure?: RunFailure; }; export const LiveDoc = defineDoc({ @@ -147,13 +179,36 @@ const RUN_TASK_KINDS: ReadonlySet = new Set(["pi.generation"]); const TOOL_TASK_KIND = "pi.tool"; const COMPACTION_TASK_KIND = "pi.compaction"; +/** How a run ends: with an answer, or without one, settling its inputs `unanswered` and recording why. */ +export type RunEnd = + | { readonly status: "done"; readonly answer: EntryId } + | { + readonly status: "unanswered"; + readonly reason: RunFailureReason; + readonly detail?: string; + /** The inputs' settlement; default `unanswered` with `reason` and `detail`. */ + readonly settlement?: SubmissionSettlement; + }; + /** - * End the run owned by `taskId`: settle each of its inputs and remove `run`. Always removes `generation`, `tools`, and - * `nestedTools`, whose presentation belongs to the ending run. + * End the run owned by `taskId`: settle each of its inputs, remove `run`, and, without an answer, record + * `runFailure`. Always removes `generation`, `tools`, and `nestedTools`, whose presentation belongs to the ending run. */ -export function endRun(tx: Tx, live: Draft, taskId: TaskId, settlement: SubmissionSettlement): void { +export function endRun(tx: Tx, live: Draft, taskId: TaskId, end: RunEnd): void { if (live.run?.taskId === taskId) { - for (const id of live.run.inputs) tx.settleSubmission(id, settlement); + const inputs = [...live.run.inputs]; + if (end.status === "done") { + for (const id of inputs) tx.settleSubmission(id, { status: "done", answer: end.answer }); + } else { + const { reason, detail } = end; + const settlement = end.settlement ?? { + status: "unanswered", + reason, + ...(detail === undefined ? {} : { detail }), + }; + for (const id of inputs) tx.settleSubmission(id, settlement); + live.runFailure = { taskId, inputs, reason, ...(detail === undefined ? {} : { detail }) }; + } delete live.run; } delete live.generation; @@ -279,6 +334,11 @@ export async function settleSchedulerOutcome( record.id, outcome.status === "faulted" ? { status: "unanswered", reason: "faulted", detail: outcome.error.message } - : { status: "unanswered", reason: outcome.reason }, + : { + status: "unanswered", + reason: "orphaned", + detail: outcome.reason, + settlement: { status: "unanswered", reason: outcome.reason }, + }, ); } diff --git a/packages/durable/src/harness/task-graph.ts b/packages/durable/src/harness/task-graph.ts index 828e7077e..04913a5ac 100644 --- a/packages/durable/src/harness/task-graph.ts +++ b/packages/durable/src/harness/task-graph.ts @@ -15,7 +15,7 @@ import type { import { type BlockedReason, type BlockedSubject, blockedSubject } from "./scheduler.ts"; import { closedError, scanAll } from "./util.ts"; -/** A live task's durable status without its checkpoint and outcome payloads (spec §9.5). */ +/** A live task's durable status without its checkpoint and outcome payloads (spec §9.4). */ export type TaskGraphState = | { readonly status: "pending" | "running"; readonly phase: string } | { @@ -47,7 +47,7 @@ export type TaskGraphNode = { readonly conversations: readonly ConversationId[]; }; -/** Every live task of the Session (spec §9.5). */ +/** Every live task of the Session (spec §9.4). */ export type TaskGraph = { /** Every live task, keyed by its decimal ID. */ readonly tasks: Readonly>; diff --git a/packages/durable/src/harness/transcript.ts b/packages/durable/src/harness/transcript.ts new file mode 100644 index 000000000..d2463b0d9 --- /dev/null +++ b/packages/durable/src/harness/transcript.ts @@ -0,0 +1,285 @@ +import type { Context } from "@earendil-works/chord"; +import { idFromNumber } from "../ids.ts"; +import type { Cursor, EntryId, EntryQuery, EntryRecord, Page } from "../types.ts"; +import type { ConversationView, ConversationWindow } from "./view.ts"; + +/** One row of a client's transcript. */ +export type TranscriptItem = + | { readonly kind: "entry"; readonly entry: EntryRecord } + /** Entries strictly between `after` and `upTo` are missing: render "load more" and call `fillGap()`. */ + | { readonly kind: "gap"; readonly after: EntryId; readonly upTo: EntryId } + /** The transcript continues above: render "load earlier" and call `loadEarlier()`. Always the first item. */ + | { readonly kind: "earlier" }; + +/** The conversation view as a renderer sees it: `items`, the client's whole transcript, instead of the window. */ +export type TranscriptView = Omit & { + readonly items: readonly TranscriptItem[]; +}; + +/** Pages a conversation's entries, as `Conversation.entries()` does without a cursor. */ +export type EntryLoader = ( + query: Omit, + limit: number, + context: Context, +) => Promise>; + +export type ConversationTranscriptOptions = { + /** Entries `loadEarlier()` and `fillGap()` load per call; default 50. */ + readonly pageSize?: number; +}; + +type Gap = Extract; +type EntryItem = Extract; + +const EARLIER: TranscriptItem = Object.freeze({ kind: "earlier" }); + +/** + * What a client has of one conversation: every entry it got from a view or fetched, and the gaps between (spec §9.3). + * A layer over any `ConversationView`, wherever it comes from: call `merge()` with every value of the view; it returns + * the view with the client's whole transcript as `items` instead of the window. Entries never change once committed, + * so the transcript only grows; a window that dropped its oldest entries keeps them here. + * + * A window whose `before` lies past the newest entry the client has does not continue it, after a stall, an overflow, + * or a reconnect: the entries in between become a gap. Nothing counts entries or computes ranges from entry IDs, which + * have holes; only exclusive bounds are turned into the inclusive bounds of `EntryQuery`. After a drop, a new watch + * with `resume` as its window continues the transcript. + * + * A merge costs O(items) and never re-sorts or re-fetches; a merge, `loadEarlier()`, or `fillGap()` that adds nothing + * returns the previous `items` array. Concurrent calls of `loadEarlier()`, or of `fillGap()` for the same gap, share one + * load, so a placeholder that stays visible does not fetch twice. + */ +export class ConversationTranscript { + readonly #load: EntryLoader; + readonly #pageSize: number; + /** Every entry held, in ID order, as items. */ + #entries: EntryItem[] = []; + readonly #ids = new Set(); + /** Gaps in ID order; each is an immutable item, replaced when it shrinks. */ + #gaps: Gap[] = []; + /** Whether anything was merged yet. */ + #started = false; + /** Exclusive lower bound of the known range: entries at or below it are not loaded; `-Infinity` at the start. */ + #low = Number.NEGATIVE_INFINITY; + /** Inclusive upper bound of the known range: the newest entry, or a later empty window's `before`. */ + #tail = Number.NEGATIVE_INFINITY; + #items: readonly TranscriptItem[] = []; + #last: ConversationView | undefined; + #view: TranscriptView | undefined; + #earlier: Promise | undefined; + readonly #filling = new Map>(); + + constructor(load: EntryLoader, options?: ConversationTranscriptOptions) { + const pageSize = options?.pageSize ?? 50; + if (!Number.isSafeInteger(pageSize) || pageSize < 1) { + throw new RangeError(`Page size must be an integer >= 1, got ${pageSize}`); + } + this.#load = load; + this.#pageSize = pageSize; + } + + /** + * The ID the transcript reaches: its newest entry, or the `before` of a later window that was empty. Undefined before + * the first merge and for an empty transcript. + */ + get newest(): EntryId | undefined { + return this.#tail === Number.NEGATIVE_INFINITY ? undefined : idFromNumber(this.#tail); + } + + /** + * The window that resumes this transcript after a drop: `{ from }` right after `newest`, so a new watch returns only + * what was missed when it fits its window, otherwise its newest entries and a gap; `{}` for an empty transcript; + * undefined before the first merge. `conversation.watch(context, transcript.resume)`. + */ + get resume(): ConversationWindow | undefined { + if (!this.#started) return undefined; + const newest = this.newest; + return newest === undefined ? {} : { from: idFromNumber(newest + 1) }; + } + + /** Take one value of the view: its window's entries join the transcript. */ + merge(view: ConversationView): TranscriptView { + this.#last = view; + const { entries, before } = view; + const hi = entries.at(-1)?.id ?? before ?? Number.NEGATIVE_INFINITY; + this.#cover(before ?? Number.NEGATIVE_INFINITY, hi, entries); + return this.#result(); + } + + /** Load the page before the oldest entry held. Without anything earlier, returns the current view. */ + loadEarlier(context: Context): Promise { + const last = this.#last; + if (last === undefined) return Promise.reject(new Error("ConversationTranscript: merge a view first")); + if (this.#low === Number.NEGATIVE_INFINITY) return Promise.resolve(this.#result()); + if (this.#earlier !== undefined) return this.#earlier; + const low = this.#low; + // The loader runs in a later microtask, so even its synchronous throw settles a promise that is already shared. + const loading = Promise.resolve().then(async () => { + const query = { maxEntryId: idFromNumber(low), order: "descending" } as const; + const page = await this.#load(query, this.#pageSize, context); + const items = [...page.items].reverse(); + const oldest = items[0]; + // No next page, or an empty one: the transcript starts here. + if (page.next === undefined || oldest === undefined) this.#cover(Number.NEGATIVE_INFINITY, low, items); + else this.#cover(oldest.id - 1, low, items); + return this.#result(); + }); + this.#earlier = loading; + const settled = (): void => { + if (this.#earlier === loading) this.#earlier = undefined; + }; + loading.then(settled, settled); + return loading; + } + + /** + * Load the next page of `gap`, a gap item of `items`, shrinking it from its start, or removing it once nothing is + * missing. A gap that has shrunk since loads the gap it became; one that is no longer there returns the current view. + */ + fillGap(gap: Gap, context: Context): Promise { + const current = + this.#gaps.find((each) => each.after === gap.after && each.upTo === gap.upTo) ?? + this.#gaps.find((each) => each.after < gap.upTo && gap.after < each.upTo); + if (current === undefined) { + const last = this.#last; + if (last === undefined) return Promise.reject(new Error("ConversationTranscript: merge a view first")); + return Promise.resolve(this.#result()); + } + // Keyed by the gap the call resolved to, so a stale descriptor and the current one share the load. + const key = `${current.after}:${current.upTo}`; + const running = this.#filling.get(key); + if (running !== undefined) return running; + const filling = Promise.resolve().then(async () => { + const query = { + minEntryId: idFromNumber(current.after + 1), + maxEntryId: idFromNumber(current.upTo - 1), + order: "ascending", + } as const; + const page = await this.#load(query, this.#pageSize, context); + const newest = page.items.at(-1); + // No next page, or an empty one: nothing else is missing. + if (page.next === undefined || newest === undefined) this.#cover(current.after, current.upTo - 1, page.items); + else this.#cover(current.after, newest.id, page.items); + return this.#result(); + }); + this.#filling.set(key, filling); + const settled = (): void => { + if (this.#filling.get(key) === filling) this.#filling.delete(key); + }; + filling.then(settled, settled); + return filling; + } + + /** + * Record that every entry with an ID in (`lo`, `hi`] is known, `added` being those entries in ID order: hold the new + * ones, shrink the gaps they fill, extend the known range, and open a gap where the range does not continue it. + */ + #cover(lo: number, hi: number, added: readonly EntryRecord[]): void { + const fresh = added.filter((entry) => !this.#ids.has(entry.id)); + for (const entry of fresh) this.#ids.add(entry.id); + const held = this.#entries; + // New entries past every held one: the items grow at their end. + const appended = fresh.length === 0 || held.length === 0 || fresh[0]!.id > held.at(-1)!.entry.id; + const items = fresh.map((entry): EntryItem => ({ kind: "entry", entry })); + if (items.length > 0) this.#entries = appended ? held.concat(items) : mergeById(held, items); + let rebuild = false; + if (!this.#started || (held.length === 0 && this.#gaps.length === 0 && lo >= this.#tail)) { + // The first range, or one after an empty transcript: it is all the client knows. + rebuild = !this.#started || lo !== this.#low; + this.#started = true; + this.#low = lo; + this.#tail = Math.max(this.#tail, hi); + } else if (lo > this.#tail) { + // Past the newest known ID: what lies between is missing. + const upTo = added[0]?.id ?? lo + 1; + this.#gaps = [...this.#gaps, gapOf(this.#tail, upTo)]; + this.#tail = hi; + rebuild = true; + } else if (hi !== Number.NEGATIVE_INFINITY && hi < this.#low) { + // Below the oldest known ID: what lies between is missing. + const upTo = held.find((item) => item.entry.id > hi)?.entry.id ?? this.#low + 1; + this.#gaps = [gapOf(hi, upTo), ...this.#gaps]; + this.#low = lo; + rebuild = true; + } else { + const gaps = this.#subtract(lo, hi); + if (gaps !== this.#gaps) { + this.#gaps = gaps; + rebuild = true; + } + if (lo < this.#low) { + this.#low = lo; + rebuild = true; + } + if (hi > this.#tail) this.#tail = hi; + } + if (rebuild || !appended) this.#items = this.#build(); + else if (items.length > 0) this.#items = this.#items.concat(items); + } + + /** The gaps without (`lo`, `hi`]; the same array when none changes, as when the range is empty. */ + #subtract(lo: number, hi: number): Gap[] { + if (hi <= lo) return this.#gaps; + let changed = false; + const gaps: Gap[] = []; + for (const gap of this.#gaps) { + // Missing: IDs in (after, upTo); known now: IDs in (lo, hi]. + if (hi <= gap.after || lo + 1 >= gap.upTo) { + gaps.push(gap); + continue; + } + changed = true; + if (lo > gap.after) { + const firstKnown = this.#entries.find((item) => item.entry.id > lo)?.entry.id ?? lo + 1; + gaps.push(gapOf(gap.after, Math.min(gap.upTo, firstKnown))); + } + if (hi + 1 < gap.upTo) gaps.push(gapOf(hi, gap.upTo)); + } + return changed ? gaps : this.#gaps; + } + + /** Items in ID order: "earlier" until the start is known, then entries with each gap before the entry after it. */ + #build(): TranscriptItem[] { + const items: TranscriptItem[] = this.#low === Number.NEGATIVE_INFINITY ? [] : [EARLIER]; + const gaps = this.#gaps; + let next = 0; + for (const item of this.#entries) { + while (next < gaps.length && gaps[next]!.after < item.entry.id) items.push(gaps[next++]!); + items.push(item); + } + while (next < gaps.length) items.push(gaps[next++]!); + return items; + } + + /** The view with the current items; the previous one when nothing changed. */ + #result(): TranscriptView { + const last = this.#last!; + const previous = this.#view; + if ( + previous !== undefined && + previous.items === this.#items && + previous.docs === last.docs && + previous.conversation === last.conversation + ) { + return previous; + } + this.#view = { conversation: last.conversation, docs: last.docs, items: this.#items }; + return this.#view; + } +} + +function gapOf(after: number, upTo: number): Gap { + return Object.freeze({ kind: "gap", after: idFromNumber(after), upTo: idFromNumber(upTo) }); +} + +/** Two ID-ordered item lists merged in one pass. */ +function mergeById(left: readonly EntryItem[], right: readonly EntryItem[]): EntryItem[] { + const merged: EntryItem[] = []; + let i = 0; + let j = 0; + while (i < left.length && j < right.length) { + merged.push(left[i]!.entry.id < right[j]!.entry.id ? left[i++]! : right[j++]!); + } + while (i < left.length) merged.push(left[i++]!); + while (j < right.length) merged.push(right[j++]!); + return merged; +} diff --git a/packages/durable/src/harness/types.ts b/packages/durable/src/harness/types.ts index 61a9c0ca6..248c36a4e 100644 --- a/packages/durable/src/harness/types.ts +++ b/packages/durable/src/harness/types.ts @@ -47,7 +47,7 @@ import type { } from "../types.ts"; import type { TaskGraph, TaskGraphWatch } from "./task-graph.ts"; import type { UsageState } from "./usage.ts"; -import type { ConversationView } from "./view.ts"; +import type { ConversationView, ConversationWindow } from "./view.ts"; /** Provider and model ID resolved through pi-ai `Models`. */ export type ModelRef = { @@ -795,10 +795,11 @@ export interface Conversation { */ waitForIdle(context: Context): Promise; /** - * The structural view (spec §9.3) as a serialized watch; commits made while its listener runs arrive folded. Built on - * the same mount as `Harness.observeConversation()`. + * The structural view (spec §9.3) as a serialized watch; commits made while its listener runs arrive folded. Its + * `entries` are a window over the transcript, as `window` says: the newest 50 by default, then appends, dropping the + * oldest past 100. Built on the same mount as `Harness.observeConversation()`. */ - watch(context: Context): Promise; + watch(context: Context, window?: ConversationWindow): Promise; } export type ConversationWatch = WatchHandle; @@ -856,23 +857,24 @@ export interface Harness extends Session { /** Session total: every conversation's `pi.usage` summed. */ usage(context: Context): Promise; /** - * Every live task with its owner edge, status, and owned conversations (spec §9.5), as a serialized watch; commits + * Every live task with its owner edge, status, and owned conversations (spec §9.4), as a serialized watch; commits * made while its listener runs arrive folded. */ watchTaskGraph(context: Context): Promise; /** * Attach a synchronous `Observer` to a conversation's structural view (spec §9.3), atomically on the Session line, * for adapters: `advance` gets each next revision with its operations as memory adopts the commit, inside Storage's - * `adopt` for a stored one and before it is stored for a deferred one, so it only records it. `context` governs only - * the attachment. + * `adopt` for a stored one and before it is stored for a deferred one, so it only records it. `window` as for + * `Conversation.watch()`; each observer has its own. `context` governs only the attachment. */ observeConversation( id: ConversationId, observer: Observer, context: Context, + window?: ConversationWindow, ): Promise>; /** - * Attach a synchronous `Observer` to the task graph (spec §9.5), as `observeConversation()` does to a view. A + * Attach a synchronous `Observer` to the task graph (spec §9.4), as `observeConversation()` does to a view. A * `blocked` reason that changes without a commit (a registry change, a failed migration) advances it on the Session * line outside any commit and `adopt`, with the background context. */ diff --git a/packages/durable/src/harness/view.ts b/packages/durable/src/harness/view.ts index d12632b24..ce4c16b99 100644 --- a/packages/durable/src/harness/view.ts +++ b/packages/durable/src/harness/view.ts @@ -1,12 +1,16 @@ import { type Context, type JsonValue, Watch, type WatchChange, type WatchHandle } from "@earendil-works/chord"; import { withoutAbortSignal } from "@earendil-works/chord/context"; import { applyImmutable, type NonEmptyPath, type Op, type Path } from "@earendil-works/chord/delta"; +import { idFromNumber } from "../ids.ts"; import { contained, guardObserver, type ObservedCommit, observation, type SessionImpl } from "../session/session.ts"; import type { ConversationDocToken, ConversationId, ConversationRecord, + Cursor, DocumentId, + EntryId, + EntryQuery, EntryRecord, JsonObject, Observation, @@ -14,34 +18,50 @@ import type { Storage, } from "../types.ts"; import { AgentDoc } from "./agent.ts"; -import { activeEntries, captureContextBounds } from "./context.ts"; import { InboxDoc } from "./inbox.ts"; import { LiveDoc } from "./live.ts"; import { ProviderDoc } from "./provider.ts"; import { UsageDoc } from "./usage.ts"; import { closedError } from "./util.ts"; -/** Structural mount of one conversation's active transcript and built-in documents (spec §9.3). */ +/** Structural view of one conversation: a window over its transcript and its built-in documents (spec §9.3). */ export type ConversationView = { readonly conversation: ConversationRecord; - /** Raw active entries, as `ContextView.entries`: the head marker, then the non-head entries from its head. */ + /** + * A window over the transcript, its fork-aware history (`Conversation.entries()`), in entry ID order. Head markers + * such as compaction summaries sit where they were written; their `head` tells where the entries they summarize end. + */ readonly entries: readonly EntryRecord[]; + /** ID of the entry just before the window; absent when the window starts the transcript. */ + readonly before?: EntryId; /** Built-in conversation documents keyed by kind; absent documents are absent. */ readonly docs: Readonly>; }; -/** Receives each next revision of a mount, and the Session's close. */ +/** + * Which entries a view's window holds (spec §9.3). The window holds between `min` and `max` entries: it starts with + * the newest `min` entries, takes appends, and once it would hold more than `max`, drops its oldest in one splice down + * to `min`. With `from`, it starts at `from` instead when at most `max` entries are at or after it. + */ +export type ConversationWindow = { + /** + * Where the first value's window starts, inclusive, such as right after the newest entry a client has + * (`ConversationTranscript.resume`). It selects only that first value: every entry committed after the attachment is + * appended, whatever its ID. A `from` past the newest entry starts the window empty, with `before` the newest entry. + */ + readonly from?: EntryId; + /** Default 50; an integer, at least 1. */ + readonly min?: number; + /** Default `2 * min`; an integer, at least `min`. */ + readonly max?: number; +}; + +/** Default `ConversationWindow.min`. */ +const DEFAULT_WINDOW_MIN = 50; + +/** Receives each next revision of one observer's window, and the Session's close. */ export type ViewObserver = { - /** Each next revision; one change object is shared by every observer of the mount. */ - advance?(change: WatchChange, context: Context): void; - /** Every publication, after the mount took it; `ops` are the mount's, possibly none. */ - publication?( - before: ConversationView, - after: ConversationView, - ops: readonly Op[], - publication: ObservedCommit, - context: Context, - ): void; + advance(change: WatchChange, context: Context): void; /** The Session closed, or failed with `failure`. */ end(failure: { readonly error: unknown } | undefined): void; }; @@ -54,13 +74,26 @@ const MOUNTED = [ UsageDoc, ] as unknown as readonly ConversationDocToken[]; const MOUNTED_KINDS: ReadonlySet = new Set(MOUNTED.map((token) => token.definition.kind)); +const SCAN_PAGE_SIZE = 256; -/** One conversation's mount: its current revision, the document incarnations it shows, and its observers. */ -type Mount = { +/** One observer of a mount and its window: the bounds and the revision it was last given. */ +type Windowed = { + readonly min: number; + readonly max: number; value: ConversationView; + readonly observer: ViewObserver; +}; + +/** + * One conversation's mount: the documents it shows, shared by every observer, and the observers, each with its own + * window over the entries. + */ +type Mount = { + readonly conversation: ConversationRecord; + docs: Readonly>; /** Mounted incarnation and definition version per kind; another incarnation or version is set whole. */ - readonly docs: Map; - readonly observers: Set; + readonly incarnations: Map; + readonly observers: Set; }; const VIEWS = new WeakMap(); @@ -72,10 +105,28 @@ export function conversationViews(harness: object): ConversationViews { return views; } +/** A window with its defaults applied. */ +type WindowBounds = { readonly from: EntryId | undefined; readonly min: number; readonly max: number }; + +/** `window` with its defaults; throws a `RangeError` for bounds that are not integers or `max` below `min`. */ +function resolveWindow(window: ConversationWindow | undefined): WindowBounds { + const min = window?.min ?? DEFAULT_WINDOW_MIN; + if (!Number.isSafeInteger(min) || min < 1) throw new RangeError(`Window min must be an integer >= 1, got ${min}`); + const max = window?.max ?? 2 * min; + if (!Number.isSafeInteger(max) || max < min) { + throw new RangeError(`Window max must be an integer >= min (${min}), got ${max}`); + } + const from = window?.from; + if (from !== undefined && !Number.isSafeInteger(from)) { + throw new RangeError(`Window from must be an entry ID, got ${from}`); + } + return { from, min, max }; +} + /** * The Harness's conversation view mounts: at most one per conversation, built on the Session line by its first * observer and dropped with its last. Each mount advances from the Session's commits as memory adopts them, deferred ones - * (`CommitPersistence`) before they are stored. + * (`CommitPersistence`) before they are stored, and hands each observer the next revision of its own window. */ export class ConversationViews { readonly #session: SessionImpl; @@ -94,9 +145,9 @@ export class ConversationViews { session.subscribeClose(() => { this.#closed = true; for (const mount of this.#mounts.values()) - for (const observer of [...mount.observers]) { + for (const windowed of [...mount.observers]) { contained( - () => observer.end(this.#session.failure), + () => windowed.observer.end(this.#session.failure), (error) => this.report(error), ); } @@ -117,10 +168,12 @@ export class ConversationViews { id: ConversationId, observer: Observer, context: Context, + window?: ConversationWindow, ): Promise> { let value!: ConversationView; const attached = await this.attach( id, + window, (current, release) => { value = current; return guardObserver(observer, release, (error) => this.report(error)); @@ -136,9 +189,14 @@ export class ConversationViews { } /** A serialized watch of the view, an observer of its mount; cancelling `context` stops it. */ - async watch(id: ConversationId, context: Context): Promise> { + async watch( + id: ConversationId, + context: Context, + window?: ConversationWindow, + ): Promise> { const { observer } = await this.attach( id, + window, (value, release) => new Watch(value, release, (error) => this.#session.report(error)), context, ); @@ -152,27 +210,32 @@ export class ConversationViews { } /** - * Register an observer created from the current revision, atomically on the Session line: it sees every later - * publication and nothing earlier. `create` may read committed Storage, still on the line. `release` drops it, and - * the mount with its last observer. + * Register an observer created from its window's first revision, atomically on the Session line: it sees every later + * publication and nothing earlier. `release` drops it, and the mount with its last observer. */ - attach( + async attach( id: ConversationId, - create: (value: ConversationView, release: () => void, storage: Storage) => O | Promise, + window: ConversationWindow | undefined, + create: (value: ConversationView, release: () => void) => O, context: Context, ): Promise<{ observer: O; detach: () => void }> { + const bounds = resolveWindow(window); return this.#session.readOnLine(async () => { const mount = this.#mounts.get(id) ?? (await this.#build(id, context)); + const { entries, before } = await loadWindow(this.#storage, id, bounds, context); + const value = view(mount.conversation, entries, before, mount.docs); + let windowed: Windowed | undefined; const detach = (): void => { - mount.observers.delete(observer); + if (windowed !== undefined) mount.observers.delete(windowed); if (mount.observers.size === 0 && this.#mounts.get(id) === mount) this.#mounts.delete(id); }; - const observer = await create(mount.value, detach, this.#storage); + const observer = create(value, detach); // Close or cancellation may begin while the mount hydrates; register nothing then. if (this.#closed) throw closedError(this.#session); context.abortSignal?.throwIfAborted(); + windowed = { min: bounds.min, max: bounds.max, value, observer }; this.#mounts.set(id, mount); - mount.observers.add(observer); + mount.observers.add(windowed); return { observer, detach }; }); } @@ -180,8 +243,6 @@ export class ConversationViews { async #build(id: ConversationId, context: Context): Promise { const conversation = await this.#storage.conversation(id, context); if (conversation === undefined) throw new Error(`Conversation ${id} does not exist`); - const bounds = await captureContextBounds(this.#storage, id, context); - const entries = await activeEntries(this.#storage, id, bounds, context); const docs: Record = {}; const incarnations = new Map(); for (const token of MOUNTED) { @@ -190,13 +251,60 @@ export class ConversationViews { docs[token.definition.kind] = loaded.value; incarnations.set(token.definition.kind, { id: loaded.record.id, version: loaded.version }); } - return { value: { conversation, entries, docs }, docs: incarnations, observers: new Set() }; + return { conversation, docs, incarnations, observers: new Set() }; } } +/** A view revision; `before` is a key only when set. */ +function view( + conversation: ConversationRecord, + entries: readonly EntryRecord[], + before: EntryId | undefined, + docs: Readonly>, +): ConversationView { + return before === undefined ? { conversation, entries, docs } : { conversation, entries, before, docs }; +} + /** - * Derive the mount's operations from one publication, apply them, and hand the revision to every observer; one that - * throws is reported through `report`, and the others still get it. + * A new observer's window, read on the Session line with bounded scans: the entries from `from` when at most `max` are + * at or after it, otherwise the newest `min`, and the entry before them. + */ +async function loadWindow( + storage: Storage, + conversationId: ConversationId, + bounds: WindowBounds, + context: Context, +): Promise<{ entries: readonly EntryRecord[]; before: EntryId | undefined }> { + const { from, min, max } = bounds; + if (from !== undefined) { + const since = await scanUpTo(storage, { conversationId, minEntryId: from, order: "ascending" }, max + 1, context); + if (since.length <= max) { + const query = { conversationId, maxEntryId: idFromNumber(from - 1) }; + const previous = (await storage.scanEntries(query, 1, undefined, context)).items[0]; + return { entries: since, before: previous?.id }; + } + } + const newest = await scanUpTo(storage, { conversationId }, min + 1, context); + const before = newest.length > min ? newest.pop()!.id : undefined; + return { entries: newest.reverse(), before }; +} + +/** Up to `limit` entries of `query`, following cursors past short pages. */ +async function scanUpTo(storage: Storage, query: EntryQuery, limit: number, context: Context): Promise { + const entries: EntryRecord[] = []; + let cursor: Cursor | undefined; + do { + const page = await storage.scanEntries(query, Math.min(SCAN_PAGE_SIZE, limit - entries.length), cursor, context); + entries.push(...page.items); + cursor = page.next; + } while (cursor !== undefined && entries.length < limit); + return entries; +} + +/** + * Derive the mount's operations from one publication and hand each observer the next revision of its window; one that + * throws is reported through `report`, and the others still get theirs. Documents are applied once and shared; each + * window takes the appended entries and, past its `max`, drops its oldest down to its `min` in one splice. */ function advance( id: ConversationId, @@ -206,55 +314,51 @@ function advance( report: (error: unknown) => void, ): void { const docOps: Op[] = []; - const entryOps: Op[] = []; - let entries = mount.value.entries; // Entry writes are published in ID order. + const appended: EntryRecord[] = []; for (const change of publication.changes) { if (change.type === "entry" && change.value.conversationId === id) { - const entry = change.value; - const value = entry as unknown as JsonValue; - if (entry.head === undefined) { - entryOps.push(["p", ["entries"], entries.length, 0, [value]]); - entries = [...entries, entry]; - continue; - } - // A head marker keeps the non-head entries from its head, which are always a suffix, and goes in front. - const target = entry.head; - let kept = entries.findIndex((candidate) => candidate.head === undefined && candidate.id >= target); - if (kept < 0) kept = entries.length; - entryOps.push(["p", ["entries"], 0, kept, [value]]); - entries = [entry, ...entries.slice(kept)]; + appended.push(change.value); continue; } if (change.type !== "document" || change.conversationId !== id) continue; const kind = change.record.kind; if (!MOUNTED_KINDS.has(kind) || change.record.key !== undefined) continue; const path: NonEmptyPath = ["docs", kind]; - const mounted = mount.docs.get(kind); + const mounted = mount.incarnations.get(kind); if (change.value === null) { if (mounted?.id !== change.record.id) continue; - mount.docs.delete(kind); + mount.incarnations.delete(kind); docOps.push(["d", path]); } else if (mounted?.id === change.record.id && mounted.version === change.version) { for (const op of change.ops) docOps.push(prefixed(op, path)); } else { - mount.docs.set(kind, { id: change.record.id, version: change.version! }); + mount.incarnations.set(kind, { id: change.record.id, version: change.version! }); docOps.push(["s", path, change.value]); } } - const before = mount.value; - const ops = [...docOps, ...entryOps]; + if (docOps.length === 0 && appended.length === 0) return; + if (docOps.length > 0) mount.docs = applyImmutable({ docs: mount.docs }, docOps).docs; const frameContext = withoutAbortSignal(context); - if (ops.length > 0) { - const value = docOps.length === 0 ? mount.value : applyImmutable(mount.value, docOps); - mount.value = entries === value.entries ? value : { ...value, entries }; - const change: WatchChange = { value: mount.value, ops, retired: false }; - for (const observer of [...mount.observers]) { - contained(() => observer.advance?.(change, frameContext), report); + const inserted = appended as unknown as JsonValue[]; + for (const windowed of [...mount.observers]) { + const previous = windowed.value; + let entries = previous.entries; + let before = previous.before; + let ops: Op[] = docOps; + if (appended.length > 0) { + ops = [...docOps, ["p", ["entries"], entries.length, 0, inserted]]; + entries = entries.concat(appended); + if (entries.length > windowed.max) { + const drop = entries.length - windowed.min; + before = entries[drop - 1]!.id; + entries = entries.slice(drop); + ops.push(["p", ["entries"], 0, drop, []], ["s", ["before"], before]); + } } - } - for (const observer of [...mount.observers]) { - contained(() => observer.publication?.(before, mount.value, ops, publication, frameContext), report); + windowed.value = view(mount.conversation, entries, before, mount.docs); + const change: WatchChange = { value: windowed.value, ops, retired: false }; + contained(() => windowed.observer.advance(change, frameContext), report); } } diff --git a/packages/durable/src/index.ts b/packages/durable/src/index.ts index 3baf2e684..dbe9e1596 100644 --- a/packages/durable/src/index.ts +++ b/packages/durable/src/index.ts @@ -25,14 +25,6 @@ export { type SummaryRequest, } from "./harness/compaction.ts"; export { defineExtension, defineTool, hook, section, wrapSection, wrapTool } from "./harness/define.ts"; -export { - type AgentEvent, - type AgentEventStream, - type MessageChange, - type SnapshotEvent, - type ToolEventCall, - watchEvents, -} from "./harness/events.ts"; export { type GenerationCheckpoint, type GenerationInput, @@ -48,6 +40,8 @@ export { type LiveState, type NestedToolSlot, type NestedToolSummary, + type RunFailure, + type RunFailureReason, type ToolSlot, } from "./harness/live.ts"; export { ProviderDoc, type ProviderState } from "./harness/provider.ts"; @@ -63,6 +57,13 @@ export { type ToolTaskInput, type ToolTaskResult, } from "./harness/tool.ts"; +export { + ConversationTranscript, + type ConversationTranscriptOptions, + type EntryLoader, + type TranscriptItem, + type TranscriptView, +} from "./harness/transcript.ts"; export type { AfterFailedRun, Agent, @@ -123,7 +124,7 @@ export type { Wrap, } from "./harness/types.ts"; export { UsageDoc, type UsageState } from "./harness/usage.ts"; -export type { ConversationView } from "./harness/view.ts"; +export type { ConversationView, ConversationWindow } from "./harness/view.ts"; export { createSession } from "./session/session.ts"; export { MemoryStorage } from "./storage/memory.ts"; export { defineTask } from "./tasks.ts"; diff --git a/packages/durable/test/conversation-transcript.test.ts b/packages/durable/test/conversation-transcript.test.ts new file mode 100644 index 000000000..d5fb3568c --- /dev/null +++ b/packages/durable/test/conversation-transcript.test.ts @@ -0,0 +1,356 @@ +// Tests for `ConversationTranscript`, the client-side layer that merges view windows into one transcript (#11327). +import { + ConversationTranscript, + type ConversationView, + type Cursor, + type EntryId, + type EntryLoader, + type EntryRecord, + MemoryStorage, + type TranscriptItem, + type TranscriptView, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import { allEntries, chatSetup, openChat } from "./chat-support.ts"; +import { context } from "./session-support.ts"; +import { deferred } from "./task-support.ts"; + +const CONVERSATION = { id: 1 } as unknown as ConversationView["conversation"]; + +/** Entries with IDs that have holes: every third ID is missing. */ +function transcript(count: number): EntryRecord[] { + const entries: EntryRecord[] = []; + for (let id = 1; entries.length < count; id++) { + if (id % 3 !== 0) entries.push({ id, conversationId: CONVERSATION.id, kind: "note" } as unknown as EntryRecord); + } + return entries; +} + +/** A loader over `entries` with `Conversation.entries()` semantics, counting its calls. */ +function loader(entries: readonly EntryRecord[]): EntryLoader & { calls: number } { + const load = async (query: Parameters[0], limit: number) => { + load.calls++; + const matching = entries.filter( + (entry) => + (query.minEntryId === undefined || entry.id >= query.minEntryId) && + (query.maxEntryId === undefined || entry.id <= query.maxEntryId), + ); + const ordered = query.order === "ascending" ? matching : [...matching].reverse(); + const items = ordered.slice(0, limit); + return { items, ...(ordered.length > limit ? { next: "more" as unknown as Cursor } : {}) }; + }; + load.calls = 0; + return load; +} + +/** The view of a window: `entries` with the entry before them in `all`. */ +function windowOf( + all: readonly EntryRecord[], + first: number, + last: number, + docs: ConversationView["docs"] = {}, +): ConversationView { + const before = all[first - 1]?.id; + return { + conversation: CONVERSATION, + entries: all.slice(first, last + 1), + ...(before === undefined ? {} : { before }), + docs, + }; +} + +/** Items as a compact list: entry IDs, `gap after..upTo`, and `earlier`. */ +function shape(view: TranscriptView): (number | string)[] { + return view.items.map((item) => + item.kind === "entry" ? item.entry.id : item.kind === "gap" ? `gap ${item.after}..${item.upTo}` : "earlier", + ); +} + +const idsOf = (entries: readonly EntryRecord[]) => entries.map((entry) => entry.id); +const gapOf = (view: TranscriptView) => + view.items.find((item): item is Extract => item.kind === "gap")!; + +describe("ConversationTranscript", () => { + it("shows a first window under an `earlier` marker, or alone when it starts the transcript", () => { + const all = transcript(20); + const later = new ConversationTranscript(loader(all)); + expect(shape(later.merge(windowOf(all, 15, 19)))).toEqual(["earlier", ...idsOf(all.slice(15))]); + expect(later.newest).toBe(all[19]!.id); + const whole = new ConversationTranscript(loader(all)); + expect(shape(whole.merge(windowOf(all, 0, 19)))).toEqual(idsOf(all)); + }); + + it("appends, keeps entries the window dropped, and returns the same items when nothing is added", () => { + const all = transcript(30); + const helper = new ConversationTranscript(loader(all)); + const docsA = { a: { n: 1 } }; + const first = helper.merge(windowOf(all, 10, 19, docsA)); + const grown = helper.merge(windowOf(all, 10, 21, docsA)); + expect(shape(grown)).toEqual(["earlier", ...idsOf(all.slice(10, 22))]); + // The window dropped its oldest; the transcript keeps them. + const spliced = helper.merge(windowOf(all, 18, 21, docsA)); + expect(spliced.items).toBe(grown.items); + expect(spliced).toBe(grown); + // A docs-only change: a new view, the same items. + const docs = { a: { n: 2 } }; + const streamed = helper.merge({ ...windowOf(all, 18, 21), docs }); + expect(streamed).not.toBe(grown); + expect(streamed.items).toBe(grown.items); + expect(streamed.docs).toBe(docs); + expect(first.items).not.toBe(grown.items); + }); + + it("opens a gap when a window does not continue the transcript, and fills it page by page", async () => { + const all = transcript(200); + const load = loader(all); + const helper = new ConversationTranscript(load, { pageSize: 50 }); + helper.merge(windowOf(all, 150, 159)); + // A stall: the next window starts well past the newest entry held. + const stalled = helper.merge(windowOf(all, 190, 199)); + const gap = gapOf(stalled); + expect(gap).toEqual({ kind: "gap", after: all[159]!.id, upTo: all[190]!.id }); + expect(shape(stalled)).toEqual([ + "earlier", + ...idsOf(all.slice(150, 160)), + `gap ${all[159]!.id}..${all[190]!.id}`, + ...idsOf(all.slice(190)), + ]); + // 30 entries are missing; one page loads them all and closes the gap. + const filled = await helper.fillGap(gap, context); + expect(shape(filled)).toEqual(["earlier", ...idsOf(all.slice(150))]); + expect(load.calls).toBe(1); + }); + + it("shrinks a gap from its start while more than a page is missing", async () => { + const all = transcript(200); + const helper = new ConversationTranscript(loader(all), { pageSize: 10 }); + helper.merge(windowOf(all, 100, 109)); + let view = helper.merge(windowOf(all, 190, 199)); + view = await helper.fillGap(gapOf(view), context); + expect(gapOf(view)).toEqual({ kind: "gap", after: all[119]!.id, upTo: all[190]!.id }); + while (view.items.some((item) => item.kind === "gap")) view = await helper.fillGap(gapOf(view), context); + expect(shape(view)).toEqual(["earlier", ...idsOf(all.slice(100))]); + }); + + it("loads earlier pages until the start, then drops the marker", async () => { + const all = transcript(45); + const load = loader(all); + const helper = new ConversationTranscript(load, { pageSize: 20 }); + helper.merge(windowOf(all, 40, 44)); + let view = await helper.loadEarlier(context); + expect(shape(view)).toEqual(["earlier", ...idsOf(all.slice(20))]); + // A page without a next cursor reaches the start. + view = await helper.loadEarlier(context); + expect(shape(view)).toEqual(idsOf(all)); + const calls = load.calls; + expect(await helper.loadEarlier(context)).toBe(view); + expect(load.calls).toBe(calls); + }); + + it("shares one load between concurrent calls of loadEarlier, and of fillGap for one gap", async () => { + const all = transcript(100); + const inner = loader(all); + const gate = deferred(); + const load: EntryLoader = async (query, limit, callContext) => { + await gate.promise; + return inner(query, limit, callContext); + }; + const helper = new ConversationTranscript(load); + helper.merge(windowOf(all, 40, 49)); + const stalled = helper.merge(windowOf(all, 90, 99)); + const earlier = [helper.loadEarlier(context), helper.loadEarlier(context)]; + const fills = [helper.fillGap(gapOf(stalled), context), helper.fillGap(gapOf(stalled), context)]; + expect(earlier[0]).toBe(earlier[1]); + expect(fills[0]).toBe(fills[1]); + gate.resolve(); + await Promise.all([...earlier, ...fills]); + expect(inner.calls).toBe(2); + expect(shape(await helper.loadEarlier(context))).toEqual(idsOf(all)); + }); + + it("resumes after `newest`: an empty window right after it adds nothing; one past it opens a gap", () => { + const all = transcript(30); + const helper = new ConversationTranscript(loader(all)); + const view = helper.merge(windowOf(all, 10, 19)); + const resumed: ConversationView = { conversation: CONVERSATION, entries: [], before: helper.newest!, docs: {} }; + expect(helper.merge(resumed).items).toBe(view.items); + // A window that is empty but starts past `newest`: the entries up to its `before` are missing. + const past = helper.merge({ ...resumed, before: all[25]!.id }); + expect(shape(past).at(-1)).toBe(`gap ${all[19]!.id}..${all[25]!.id + 1}`); + expect(helper.newest).toBe(all[25]!.id); + // The next window continues it. + expect(shape(helper.merge(windowOf(all, 26, 27))).slice(-3)).toEqual([ + `gap ${all[19]!.id}..${all[25]!.id + 1}`, + all[26]!.id, + all[27]!.id, + ]); + }); + + it("fills a gap from a window that covers it", () => { + const all = transcript(100); + const helper = new ConversationTranscript(loader(all)); + helper.merge(windowOf(all, 40, 49)); + helper.merge(windowOf(all, 90, 99)); + // A wider window, such as a new watch with a larger `max`, covers the second half of the gap. + const view = helper.merge(windowOf(all, 70, 99)); + expect(shape(view)).toEqual([ + "earlier", + ...idsOf(all.slice(40, 50)), + `gap ${all[49]!.id}..${all[70]!.id}`, + ...idsOf(all.slice(70)), + ]); + }); + + it("rejects loadEarlier before the first merge and returns the view for a gap that is gone", async () => { + const all = transcript(10); + const helper = new ConversationTranscript(loader(all)); + await expect(helper.loadEarlier(context)).rejects.toThrow("merge a view first"); + const view = helper.merge(windowOf(all, 0, 9)); + const stale = { kind: "gap", after: 1 as EntryId, upTo: 2 as EntryId } as const; + expect(await helper.fillGap(stale, context)).toBe(view); + }); +}); + +describe("ConversationTranscript loads", () => { + it("retries a load whose loader threw synchronously, for loadEarlier and fillGap", async () => { + const all = transcript(100); + const inner = loader(all); + let offline = true; + let calls = 0; + const load: EntryLoader = (query, limit, callContext) => { + calls++; + // An adapter that checks its connection before it returns a promise. + if (offline) throw new Error("temporarily offline"); + return inner(query, limit, callContext); + }; + const helper = new ConversationTranscript(load); + helper.merge(windowOf(all, 40, 49)); + const stalled = helper.merge(windowOf(all, 90, 99)); + await expect(helper.loadEarlier(context)).rejects.toThrow("temporarily offline"); + await expect(helper.fillGap(gapOf(stalled), context)).rejects.toThrow("temporarily offline"); + offline = false; + expect(shape(await helper.loadEarlier(context))).toEqual([ + ...idsOf(all.slice(0, 50)), + `gap ${all[49]!.id}..${all[90]!.id}`, + ...idsOf(all.slice(90)), + ]); + expect(shape(await helper.fillGap(gapOf(stalled), context))).toEqual(idsOf(all)); + expect(calls).toBe(4); + }); + + it("retries a load that rejected", async () => { + const all = transcript(30); + const inner = loader(all); + let fail = true; + const load: EntryLoader = async (query, limit, callContext) => { + if (fail) throw new Error("timeout"); + return inner(query, limit, callContext); + }; + const helper = new ConversationTranscript(load); + helper.merge(windowOf(all, 20, 29)); + await expect(helper.loadEarlier(context)).rejects.toThrow("timeout"); + fail = false; + expect(shape(await helper.loadEarlier(context))).toEqual(idsOf(all)); + }); + + it("keeps one gap for an empty window whose `before` lies inside it", () => { + const all = transcript(100); + const helper = new ConversationTranscript(loader(all)); + helper.merge(windowOf(all, 40, 49)); + const before = helper.merge(windowOf(all, 90, 99)); + const view = helper.merge({ conversation: CONVERSATION, entries: [], before: all[65]!.id, docs: before.docs }); + expect(view.items).toBe(before.items); + expect(view.items.filter((item) => item.kind === "gap")).toHaveLength(1); + }); + + it("shares one load between a stale gap descriptor and the gap it shrank into", async () => { + const all = transcript(100); + const inner = loader(all); + const gate = deferred(); + const load: EntryLoader = async (query, limit, callContext) => { + await gate.promise; + return inner(query, limit, callContext); + }; + const helper = new ConversationTranscript(load, { pageSize: 10 }); + helper.merge(windowOf(all, 10, 19)); + const stale = gapOf(helper.merge(windowOf(all, 90, 99))); + gate.resolve(); + const current = gapOf(await helper.fillGap(stale, context)); + expect(current).not.toEqual(stale); + const loads = inner.calls; + const [a, b] = [helper.fillGap(stale, context), helper.fillGap(current, context)]; + expect(a).toBe(b); + await a; + expect(inner.calls).toBe(loads + 1); + }); + + it("treats an empty page with a next cursor as the end of the range", async () => { + const all = transcript(60); + // A remote loader that always returns a cursor. + const inner = loader(all); + const load: EntryLoader = async (query, limit, callContext) => ({ + ...(await inner(query, limit, callContext)), + next: "more" as unknown as Cursor, + }); + const helper = new ConversationTranscript(load, { pageSize: 100 }); + helper.merge(windowOf(all, 10, 19)); + let view = helper.merge(windowOf(all, 40, 49)); + view = await helper.fillGap(gapOf(view), context); + // The first page held every missing entry; the next one is empty and closes the gap. + view = await helper.fillGap(gapOf(view), context); + expect(gapOf(view)).toBeUndefined(); + await helper.loadEarlier(context); + view = await helper.loadEarlier(context); + expect(shape(view)).toEqual(idsOf(all.slice(0, 50))); + }); + + it("resumes with `resume`: undefined before a merge, `{}` for an empty transcript, `{ from }` after the newest", () => { + const all = transcript(30); + const helper = new ConversationTranscript(loader(all)); + expect(helper.resume).toBeUndefined(); + helper.merge({ conversation: CONVERSATION, entries: [], docs: {} }); + expect(helper.resume).toEqual({}); + helper.merge(windowOf(all, 0, 9)); + expect(helper.resume).toEqual({ from: all[9]!.id + 1 }); + }); +}); + +describe("ConversationTranscript over a Harness", () => { + it("follows the walkthrough: live appends, a stall, fillGap, loadEarlier, and a resume with `from`", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + const note = () => root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + for (let index = 0; index < 120; index++) await note(); + const helper = new ConversationTranscript((query, limit, callContext) => + root.entries(query, limit, undefined, callContext), + ); + const window = { min: 10, max: 20 }; + const watch = await root.watch(context, window); + let view = helper.merge(watch.value); + watch.start(async ({ value }) => { + view = helper.merge(value); + }); + for (let index = 0; index < 15; index++) await note(); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(view.items.some((item) => item.kind === "gap")).toBe(false); + expect(view.items.filter((item) => item.kind === "entry")).toHaveLength(25); + // A drop: the client misses 40 entries, more than `max`, and resumes after the newest it has. + await watch.stop(); + for (let index = 0; index < 40; index++) await note(); + const resumed = await root.watch(context, { ...window, ...helper.resume }); + view = helper.merge(resumed.value); + expect(view.items.filter((item) => item.kind === "gap")).toHaveLength(1); + while (view.items.some((item) => item.kind === "gap")) view = await helper.fillGap(gapOf(view), context); + while (view.items[0]?.kind === "earlier") view = await helper.loadEarlier(context); + const entries = view.items.flatMap((item) => (item.kind === "entry" ? [item.entry.id] : [])); + expect(entries).toEqual(idsOf(await allEntries(root))); + // A short drop resumes without a gap. + for (let index = 0; index < 3; index++) await note(); + const again = await root.watch(context, { ...window, ...helper.resume }); + expect(again.value.entries).toHaveLength(3); + view = helper.merge(again.value); + expect(view.items.some((item) => item.kind !== "entry")).toBe(false); + await resumed.stop(); + await again.stop(); + await harness.close(context); + }); +}); diff --git a/packages/durable/test/conversation-view-bench.ts b/packages/durable/test/conversation-view-bench.ts new file mode 100644 index 000000000..48d5ec565 --- /dev/null +++ b/packages/durable/test/conversation-view-bench.ts @@ -0,0 +1,234 @@ +/** + * Conversation view benchmark: a long conversation (thousands of entries) and the watches of its view. + * + * node --conditions=source --experimental-strip-types --expose-gc test/conversation-view-bench.ts + * + * Scenarios, each in its own child process so heap numbers are its own, on SQLite so history stays out of the heap: + * - `start`: acquire a watch of a conversation of N entries (the mount is built every time; the watch is stopped). + * - `append`: K started watches; commit entries one per commit, or in bursts of 50 per commit, and wait for delivery. + * - `resume`: a watch with `from` 20 entries back, as a client that reconnects after a short drop. + * - `stream`: K started watches; commits that change only `pi.live` (a streamed answer), no entries. + * - `long-lived`: one started watch while 20,000 entries are appended; heap after GC, and whether the first entry + * delivered is still reachable. + */ +import { execFile } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import type { Context } from "@earendil-works/chord"; +import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; +import { createModels } from "@earendil-works/pi-ai/models"; +import { + type Conversation, + type ConversationView, + createRegistry, + type EntryId, + Harness, + LiveDoc, + type Storage, +} from "../src/index.ts"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; + +type Scenario = + | { kind: "start"; entries: number } + | { kind: "append"; entries: number; watches: number; burst: number; commits: number } + | { kind: "resume"; entries: number } + | { kind: "stream"; entries: number; watches: number; commits: number } + | { kind: "long-lived"; entries: number }; + +const TEXT = "The quick brown fox jumps over the lazy dog. ".repeat(10); +type Watch = Awaited>; +/** `watch(context, options)`; an older Harness ignores the options. */ +type WatchWith = (context: Context, options?: { from?: EntryId }) => Promise; + +async function main(): Promise { + const [, , encoded] = process.argv; + if (encoded !== undefined) { + const result = await run(JSON.parse(encoded) as Scenario); + process.stdout.write(`${JSON.stringify(result)}\n`); + return; + } + const scenarios: Scenario[] = [ + { kind: "start", entries: 5000 }, + { kind: "resume", entries: 5000 }, + { kind: "append", entries: 5000, watches: 1, burst: 1, commits: 2000 }, + { kind: "append", entries: 5000, watches: 4, burst: 1, commits: 2000 }, + { kind: "append", entries: 5000, watches: 4, burst: 50, commits: 200 }, + { kind: "stream", entries: 5000, watches: 4, commits: 5000 }, + { kind: "long-lived", entries: 20000 }, + ]; + const rows: Record[] = []; + for (const scenario of scenarios) { + const { stdout } = await promisify(execFile)( + process.execPath, + [...process.execArgv, fileURLToPath(import.meta.url), JSON.stringify(scenario)], + { maxBuffer: 1 << 20 }, + ); + rows.push({ scenario: label(scenario), ...JSON.parse(stdout.trim().split("\n").at(-1)!) }); + } + console.table(rows); +} + +function label(scenario: Scenario): string { + switch (scenario.kind) { + case "start": + return `watch start, ${scenario.entries} entries`; + case "resume": + return `resume 20 back, ${scenario.entries} entries`; + case "append": + return `append x${scenario.burst} per commit, ${scenario.watches} watches`; + case "stream": + return `pi.live only, ${scenario.watches} watches`; + case "long-lived": + return `long-lived watch, ${scenario.entries} appends`; + } +} + +function gc(): void { + (globalThis as { gc?: () => void }).gc?.(); +} + +function heapMiB(): number { + gc(); + gc(); + return Math.round((process.memoryUsage().heapUsed / 2 ** 20) * 10) / 10; +} + +async function appendEntries(conversation: Conversation, count: number, perCommit: number): Promise { + for (let done = 0; done < count; done += perCommit) { + await conversation.commit(async (tx) => { + for (let index = done; index < Math.min(count, done + perCommit); index++) { + await tx.appendEntry(conversation.id, { kind: "note", data: { text: `${index} ${TEXT}` } }); + } + }, context); + } +} + +/** Resolve once every watch delivered a value whose newest entry is `id`. */ +function delivered(watches: readonly Watch[], id: () => EntryId | undefined): Promise { + return new Promise((resolve) => { + const check = (): void => { + const target = id(); + if (watches.every((watch) => watch.value.entries.at(-1)?.id === target)) resolve(); + else setTimeout(check, 1); + }; + check(); + }); +} + +async function run(scenario: Scenario): Promise> { + const directory = mkdtempSync(join(tmpdir(), "pi-durable-view-bench-")); + let storage: Storage | undefined; + try { + storage = await openNodeSqliteStorage(join(directory, "session.sqlite")); + const models = createModels(); + const harness = await Harness.open(storage, { models, registry: createRegistry() }, context); + const root = await harness.root(context); + await appendEntries(root, scenario.entries, 500); + const newest = async (): Promise => (await root.entries({}, 1, undefined, context)).items[0]!.id; + const watchWith = root.watch.bind(root) as unknown as WatchWith; + const heapBefore = heapMiB(); + const cpu = process.cpuUsage(); + const started = performance.now(); + const result: Record = {}; + if (scenario.kind === "start" || scenario.kind === "resume") { + const from = scenario.kind === "resume" ? (((await newest()) - 19) as EntryId) : undefined; + const times: number[] = []; + let size = 0; + for (let round = 0; round < 20; round++) { + const at = performance.now(); + const watch = await watchWith(context, from === undefined ? undefined : { from }); + times.push(performance.now() - at); + size = watch.value.entries.length; + await watch.stop(); + } + times.sort((a, b) => a - b); + result.p50Ms = round(times[10]!); + result.windowEntries = size; + } else if (scenario.kind === "append") { + const watches = await Promise.all(Array.from({ length: scenario.watches }, () => root.watch(context))); + for (const watch of watches) watch.start(async () => {}); + let last: EntryId | undefined; + const at = performance.now(); + for (let index = 0; index < scenario.commits; index++) { + const ids = await root.commit(async (tx) => { + const appended: EntryId[] = []; + for (let n = 0; n < scenario.burst; n++) { + appended.push((await tx.appendEntry(root.id, { kind: "note", data: { text: TEXT } })).id); + } + return appended; + }, context); + last = ids.at(-1); + } + await delivered(watches, () => last); + result.perCommitUs = round(((performance.now() - at) * 1000) / scenario.commits); + result.windowEntries = watches[0]!.value.entries.length; + result.heapWithWatchesMiB = heapMiB(); + for (const watch of watches) await watch.stop(); + } else if (scenario.kind === "stream") { + const watches = await Promise.all(Array.from({ length: scenario.watches }, () => root.watch(context))); + let calls = 0; + for (const watch of watches) + watch.start(async () => { + calls++; + }); + const at = performance.now(); + for (let index = 0; index < scenario.commits; index++) { + await root.commit(async (tx) => { + (await tx.doc(LiveDoc, root.id)).compactionFailure = { + taskId: 1 as never, + reason: "manual", + error: `${index}`, + }; + }, context); + } + // Every watch delivered the final value, not just one of them. + const last = `${scenario.commits - 1}`; + const done = () => + watches.every( + (watch) => + (watch.value.docs["pi.live"] as { compactionFailure?: { error: string } } | undefined) + ?.compactionFailure?.error === last, + ); + while (calls < scenario.watches || !done()) await new Promise((resolve) => setTimeout(resolve, 1)); + result.perCommitUs = round(((performance.now() - at) * 1000) / scenario.commits); + for (const watch of watches) await watch.stop(); + } else { + const watch = await root.watch(context); + let first: WeakRef | undefined; + watch.start(async ({ value }: { value: ConversationView }) => { + const entry = value.entries.at(-1); + if (first === undefined && entry !== undefined) first = new WeakRef(entry); + }); + await appendEntries(root, 1, 1); + const id = await newest(); + await delivered([watch], () => id); + await appendEntries(root, scenario.entries, 1); + const last = await newest(); + await delivered([watch], () => last); + result.windowEntries = watch.value.entries.length; + result.heapWithWatchMiB = heapMiB(); + // A WeakRef's target survives the job that created or read it; collect again in a later task. + await new Promise((resolve) => setTimeout(resolve, 10)); + gc(); + result.firstEntryCollected = first?.deref() === undefined ? 1 : 0; + await watch.stop(); + } + const usage = process.cpuUsage(cpu); + result.cpuMs = round((usage.user + usage.system) / 1000); + result.wallMs = round(performance.now() - started); + result.heapBeforeMiB = heapBefore; + await harness.close(context); + return result; + } finally { + rmSync(directory, { recursive: true, force: true }); + } +} + +function round(value: number): number { + return Math.round(value * 100) / 100; +} + +await main(); diff --git a/packages/durable/test/examples/18-json.ts b/packages/durable/test/examples/18-json.ts index 4e07a1768..15e79b795 100644 --- a/packages/durable/test/examples/18-json.ts +++ b/packages/durable/test/examples/18-json.ts @@ -1,11 +1,11 @@ -// JSON mode: stream one prompt's run as JSON lines, like `pi --mode json`. Two modes: -// --events (default): the experimental agent events of watchEvents(), starting with a `snapshot` event. -// --ops: the raw ConversationView frames, starting with the view itself; each later line holds one commit's ops. +// JSON mode: stream one prompt's run as JSON lines, like `pi --mode json`. Two modes, both of the ConversationView: +// --ops (default): the raw frames, starting with the view itself; each later line holds the ops of the commits since. +// --state: per call, the entries appended since the previous one and the current `pi.live`, state a renderer draws. // --storage sqlite (default) | jsonl | memory: sqlite writes a temporary database file, jsonl a temporary directory; // neither is deleted, and the path is printed to stderr at the end. // Uses OpenAI when OPENAI_API_KEY is set, and a scripted faux model otherwise. // Run from packages/durable: -// node --conditions=source --experimental-strip-types test/examples/18-json.ts --events "What is in this directory?" +// node --conditions=source --experimental-strip-types test/examples/18-json.ts --state "What is in this directory?" import { mkdtemp } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -14,22 +14,14 @@ import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; import { openaiProvider } from "@earendil-works/pi-ai/providers/openai"; import { NodeExecutionEnv } from "../../src/env/node.ts"; -import { - createRegistry, - defineExtension, - Harness, - MemoryStorage, - type Storage, - section, - watchEvents, -} from "../../src/index.ts"; +import { createRegistry, defineExtension, Harness, MemoryStorage, type Storage, section } from "../../src/index.ts"; import { openNodeJsonlStorage } from "../../src/storage/jsonl/node.ts"; import { openNodeSqliteStorage } from "../../src/storage/sqlite/node.ts"; import { createBashTool, createReadTool } from "../../src/tools/index.ts"; const context = BACKGROUND_CONTEXT; const args = process.argv.slice(2); -const mode = args.includes("--ops") ? "ops" : "events"; +const mode = args.includes("--state") ? "state" : "ops"; const storageIndex = args.indexOf("--storage"); const storageKind = storageIndex < 0 ? "sqlite" : args[storageIndex + 1]; const prompt = @@ -77,20 +69,19 @@ const env = new NodeExecutionEnv({ cwd: process.cwd() }); const harness = await Harness.open(storage, { models, registry, env: () => env }, context); const root = await harness.root(context, { agent: { model } }); -// Attach before submitting, so the stream covers the whole run. -let stop: () => Promise; -if (mode === "events") { - const stream = await watchEvents(harness, root.id, context); - print(stream.snapshot); - stream.start(async ({ value: events }) => { - for (const event of events) print(event); - }); - stop = () => stream.stop(); -} else { - const watch = await root.watch(context); +// Attach before submitting, so the watch covers the whole run. +const watch = await root.watch(context); +if (mode === "ops") { print({ view: watch.value }); watch.start(async ({ ops }) => print({ ops })); - stop = () => watch.stop(); +} else { + let newest = watch.value.entries.at(-1)?.id ?? 0; + print({ entries: watch.value.entries, live: watch.value.docs["pi.live"] }); + watch.start(async ({ value }) => { + const entries = value.entries.filter((entry) => entry.id > newest); + newest = value.entries.at(-1)?.id ?? newest; + print({ ...(entries.length === 0 ? {} : { entries }), live: value.docs["pi.live"] }); + }); } const submission = await root.submit({ type: "input", content: prompt }, context); @@ -98,6 +89,6 @@ await submission.wait(context); await harness.waitForIdle(context); // Let the last batch reach the listener before stopping. await new Promise((resolve) => setTimeout(resolve, 0)); -await stop(); +await watch.stop(); await harness.close(context); if (location !== undefined) console.error(`${storageKind} storage: ${location}`); diff --git a/packages/durable/test/examples/20-late-join.ts b/packages/durable/test/examples/20-late-join.ts index 53c7118cf..af7b24919 100644 --- a/packages/durable/test/examples/20-late-join.ts +++ b/packages/durable/test/examples/20-late-join.ts @@ -1,5 +1,5 @@ // Late join: a client attaches while a run is already underway. It gets the current state first, the conversation -// view or the snapshot event, and then only what changes after that. +// view, and then only what changes after that. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/20-late-join.ts import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; @@ -13,7 +13,6 @@ import { Harness, type LiveState, MemoryStorage, - watchEvents, } from "../../src/index.ts"; const context = BACKGROUND_CONTEXT; @@ -54,7 +53,8 @@ const submission = await root.submit({ type: "input", content: "Count to ten, th // Join while the tool is halfway through. await pause(500); -// Structural client: the view holds the committed transcript and documents, including the running tool's output. +// The view holds the newest entries and the documents, including the running tool's output and, later, the answer +// being streamed. const view = await root.watch(context); const live = view.value.docs["pi.live"] as LiveState; console.log( @@ -62,38 +62,23 @@ console.log( view.value.entries.map((entry) => entry.kind), ); console.log("view tool slot:", live.tools?.[0]?.status, JSON.stringify(live.tools?.[0]?.output)); +let shown = ""; view.start(async ({ value }) => { - const slot = (value.docs["pi.live"] as LiveState).tools?.[0]; + const state = value.docs["pi.live"] as LiveState; + const slot = state.tools?.[0]; if (slot?.status === "running") console.log("view output now:", JSON.stringify(slot.output)); -}); - -// Event client: the snapshot event carries the same state; later events apply on top of it. -const stream = await watchEvents(harness, root.id, context); -console.log( - "snapshot tools:", - stream.snapshot.tools.map((slot) => `${slot.name} ${slot.status}`), -); -let output = stream.snapshot.tools[0]?.output ?? ""; -stream.start(async ({ value: events }) => { - for (const event of events) { - if (event.type === "tool_execution_update" && event.output !== undefined) { - output = - "set" in event.output - ? event.output.set - : output.slice(event.output.trimStart ?? 0) + (event.output.append ?? ""); - console.log("event output now:", JSON.stringify(output)); - } else if (event.type === "message_update") { - const deltas = event.changes.flatMap((change) => (change.type === "text_delta" ? [change.delta] : [])); - console.log("event text delta:", JSON.stringify(deltas.join(""))); - } else { - console.log("event:", event.type); - } - } + const partial = state.generation?.message?.content.find((block) => block.type === "text")?.text ?? ""; + if (partial !== shown && partial !== "") console.log("view answer now:", JSON.stringify(partial)); + shown = partial; + if (state.run === undefined) + console.log( + "view entries:", + value.entries.map((entry) => entry.kind), + ); }); await submission.wait(context); await harness.waitForIdle(context); await pause(0); -await stream.stop(); await view.stop(); await harness.close(context); diff --git a/packages/durable/test/examples/21-subagent-foreground.ts b/packages/durable/test/examples/21-subagent-foreground.ts index e9404bb3f..cdf8183c9 100644 --- a/packages/durable/test/examples/21-subagent-foreground.ts +++ b/packages/durable/test/examples/21-subagent-foreground.ts @@ -1,6 +1,6 @@ // A foreground subagent tool: the parent's tool call creates a child conversation it owns, runs one task there, and // returns the child's answer. Aborting the tool call aborts the child. A UI finds the child through the tool's running -// details and shows its events under the call. +// details and shows its view under the call. // Uses OpenAI when OPENAI_API_KEY is set, and a scripted faux model otherwise. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/21-subagent-foreground.ts @@ -11,7 +11,6 @@ import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; import { openaiProvider } from "@earendil-works/pi-ai/providers/openai"; import { - type AgentEvent, AssistantEntry, type ConversationId, configure, @@ -19,11 +18,12 @@ import { defineExtension, defineTool, type EntryId, + type EntryRecord, type Extension, Harness, + type LiveState, MemoryStorage, type ToolExecutionApi, - watchEvents, } from "../../src/index.ts"; const context = BACKGROUND_CONTEXT; @@ -99,26 +99,28 @@ registry.install(Subagent); const harness = await Harness.open(new MemoryStorage(), { models, registry }, context); const root = await harness.root(context, { agent: { model } }); -// ─── UI: the parent's events, with each subagent's events indented under its call ─── +// ─── UI: the parent's transcript, with each subagent's indented under its call ─── -const print = (indent: string, event: AgentEvent): void => { - if (event.type === "message_end" && event.entry.kind === "pi.assistant") { - const message = event.entry.model?.[0] as AssistantMessage; - const text = message.content.flatMap((c) => (c.type === "text" ? [c.text] : [])).join(""); - if (text !== "") console.log(`${indent}assistant: ${text}`); - } else if (event.type === "tool_execution_start") { - console.log(`${indent}tool ${event.toolName}(${JSON.stringify(event.args)})`); +/** Print an assistant entry's text and tool calls. */ +const print = (indent: string, entry: EntryRecord): void => { + const message = entry.model?.[0]; + if (entry.kind !== "pi.assistant" || message?.role !== "assistant") return; + for (const block of (message as AssistantMessage).content) { + if (block.type === "text" && block.text !== "") console.log(`${indent}assistant: ${block.text}`); + if (block.type === "toolCall") console.log(`${indent}tool ${block.name}(${JSON.stringify(block.arguments)})`); } }; const attached = new Set(); const attach = async (id: ConversationId, indent: string): Promise => { attached.add(id); - const stream = await watchEvents(harness, id, context); - stream.start(async ({ value: events }) => { - for (const event of events) { - print(indent, event); - if (event.type !== "tool_execution_update") continue; - const child = (event.details as { conversationId?: ConversationId } | undefined)?.conversationId; + const watch = await (await harness.conversation(id, context))!.watch(context); + let newest = watch.value.entries.at(-1)?.id ?? 0; + watch.start(async ({ value }) => { + for (const entry of value.entries) if (entry.id > newest) print(indent, entry); + newest = value.entries.at(-1)?.id ?? newest; + // A running subagent call names its child conversation in its details. + for (const slot of (value.docs["pi.live"] as LiveState).tools ?? []) { + const child = (slot.details as { conversationId?: ConversationId } | undefined)?.conversationId; if (child !== undefined && !attached.has(child)) await attach(child, `${indent} `); } }); @@ -131,6 +133,6 @@ const submission = await root.submit( ); await submission.wait(context); await harness.waitForIdle(context); -// Event callbacks run after their commit; let the last ones print. +// Watch callbacks run after their commit; let the last ones print. await new Promise((resolve) => setTimeout(resolve, 0)); await harness.close(context); diff --git a/packages/durable/test/examples/22-subagent-background.ts b/packages/durable/test/examples/22-subagent-background.ts index 0b3925d9e..7bdb67470 100644 --- a/packages/durable/test/examples/22-subagent-background.ts +++ b/packages/durable/test/examples/22-subagent-background.ts @@ -26,7 +26,6 @@ import { Harness, LiveDoc, type TaskId, - watchEvents, } from "../../src/index.ts"; import { openNodeSqliteStorage } from "../../src/storage/sqlite/node.ts"; @@ -283,11 +282,13 @@ function contentText(content: string | readonly { type: string; text?: string }[ /** Print the main conversation's messages as they are committed. Subagents work in their own conversations. */ async function follow(harness: Harness, id: ConversationId): Promise { - const stream = await watchEvents(harness, id, context); - stream.start(async ({ value: events }) => { - for (const event of events) { - if (event.type !== "message_end") continue; - const message = event.entry.model?.[0]; + const watch = await (await harness.conversation(id, context))!.watch(context); + let newest = watch.value.entries.at(-1)?.id ?? 0; + watch.start(async ({ value }) => { + const entries = value.entries.filter((entry) => entry.id > newest); + newest = value.entries.at(-1)?.id ?? newest; + for (const entry of entries) { + const message = entry.model?.[0]; if (message?.role === "user") { const text = contentText(message.content); // Input is either the user or a subagent's report, which arrives whenever the subagent is done. diff --git a/packages/durable/test/examples/31-transcript.ts b/packages/durable/test/examples/31-transcript.ts new file mode 100644 index 000000000..49bf75915 --- /dev/null +++ b/packages/durable/test/examples/31-transcript.ts @@ -0,0 +1,106 @@ +// A client's transcript over the windowed conversation view: `ConversationTranscript` merges every value of a watch +// into the client's whole transcript, marks entries a stall skipped as a gap, and loads them, and earlier pages, on +// demand. A client that drops resumes after the newest entry it has. +// Run from packages/durable: +// node --conditions=source --experimental-strip-types test/examples/31-transcript.ts +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { createModels } from "@earendil-works/pi-ai/models"; +import { + ConversationTranscript, + createRegistry, + Harness, + MemoryStorage, + type TranscriptItem, + type TranscriptView, +} from "../../src/index.ts"; + +const context = BACKGROUND_CONTEXT; +const harness = await Harness.open( + new MemoryStorage(), + { models: createModels(), registry: createRegistry() }, + context, +); +const root = await harness.root(context); +let count = 0; +/** Commit `n` chat lines, one entry each. */ +const chat = async (n: number): Promise => { + for (let index = 0; index < n; index++) { + await root.commit((tx) => tx.appendEntry(root.id, { kind: "chat", data: `line ${++count}` }), context); + } +}; +const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); + +/** One line per item: ranges of entries, gaps, and the "load earlier" marker. */ +function render(label: string, view: TranscriptView): void { + const rows: string[] = []; + let run: [number, number] | undefined; + const flush = () => { + if (run !== undefined) rows.push(run[0] === run[1] ? `${run[0]}` : `${run[0]}..${run[1]}`); + run = undefined; + }; + for (const item of view.items) { + if (item.kind === "entry") { + const line = Number(String(item.entry.data).slice("line ".length)); + run = run === undefined ? [line, line] : [run[0], line]; + continue; + } + flush(); + rows.push(item.kind === "gap" ? "[load more]" : "[load earlier]"); + } + flush(); + console.log(`${label.padEnd(28)} ${rows.join(" ")}`); +} + +const firstGap = (view: TranscriptView) => + view.items.find((item): item is Extract => item.kind === "gap"); + +await chat(200); + +// Attach: the window holds the newest `min` entries; the transcript shows "load earlier" above them. +const transcript = new ConversationTranscript((query, limit, ctx) => root.entries(query, limit, undefined, ctx), { + pageSize: 20, +}); +const window = { min: 10, max: 20 }; +let watch = await root.watch(context, window); +let view = transcript.merge(watch.value); +render("attach", view); +watch.start(async ({ value }) => { + view = transcript.merge(value); +}); + +// Live: appends. The window drops its oldest past `max`; the transcript keeps them. +await chat(15); +await settle(); +render("15 more, live", view); + +// The user scrolls up: one page. +view = await transcript.loadEarlier(context); +render("load earlier", view); + +// A drop: the client misses 40 entries, more than `max`, and resumes after the newest it has. The window starts at the +// newest `min` again, and `before` shows the transcript a gap. +await watch.stop(); +await chat(40); +watch = await root.watch(context, { ...window, ...transcript.resume }); +view = transcript.merge(watch.value); +render("resume after 40 missed", view); +watch.start(async ({ value }) => { + view = transcript.merge(value); +}); + +// "load more" fills the gap a page at a time. +for (let gap = firstGap(view); gap !== undefined; gap = firstGap(view)) { + view = await transcript.fillGap(gap, context); + render("fill gap", view); +} + +// A short drop resumes without a gap: the window starts right after the newest entry. +await watch.stop(); +await chat(3); +watch = await root.watch(context, { ...window, ...transcript.resume }); +console.log(`${"resume after 3 missed".padEnd(28)} window: ${watch.value.entries.length} entries`); +view = transcript.merge(watch.value); +render("", view); + +await watch.stop(); +await harness.close(context); diff --git a/packages/durable/test/harness-compaction.test.ts b/packages/durable/test/harness-compaction.test.ts index 0cfa7426b..caece1213 100644 --- a/packages/durable/test/harness-compaction.test.ts +++ b/packages/durable/test/harness-compaction.test.ts @@ -13,7 +13,6 @@ import { type UserMessage, } from "@earendil-works/pi-ai"; import { - type AgentEvent, type CompactionPolicy, type CompactionResult, CompactionTask, @@ -24,11 +23,11 @@ import { type EntryRecord, type Harness, LiveDoc, + type LiveState, MemoryStorage, ProviderDoc, type TaskId, UsageDoc, - watchEvents, } from "@earendil-works/pi-durable"; import { afterEach, describe, expect, it } from "vitest"; import { selectCut, serializeConversation } from "../src/harness/compaction.ts"; @@ -36,6 +35,7 @@ import { orderToolResults } from "../src/harness/context.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, type ChatSetup, chatSetup, openChat, textOf, toolsNamed, waitFor } from "./chat-support.ts"; import { addHooks, addSection, addTask, addTool } from "./harness-support.ts"; +import { viewHistory } from "./live-support.ts"; import { ControlledStorage, context } from "./session-support.ts"; import { aborted, type Deferred, deferred } from "./task-support.ts"; @@ -1006,40 +1006,37 @@ describe("background threshold compaction", () => { await chat.harness.close(context); }); - it("records a failure in pi.live, the snapshot, and compaction_end until a summary is produced", async () => { + it("records a failure in pi.live, in the commit that ends the compaction, until a summary is produced", async () => { const chat = await open({ contextWindow: 2000 }); await history(chat); chat.setup.settings.compaction = BACKGROUND; - const events: AgentEvent[] = []; - const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async ({ value: batch }) => { - events.push(...batch); - }); + const views = await viewHistory(chat.harness, chat.root.id); chat.faux.summaries.push(failure("bad request")); await turn(chat, text("u4", 100), text("a4", 100)); - await waitFor(() => events.some((event) => event.type === "compaction_end")); + await waitFor(async () => (await live(chat)).compactionFailure !== undefined); const recorded = (await live(chat)).compactionFailure; expect(recorded).toMatchObject({ reason: "threshold", error: "Summarization failed: bad request" }); - expect(events.find((event) => event.type === "compaction_end")).toEqual({ - type: "compaction_end", - taskId: recorded!.taskId, - reason: "threshold", - error: "Summarization failed: bad request", - }); - const reopened = await watchEvents(chat.harness, chat.root.id, context); - expect(reopened.snapshot.compactionFailure).toEqual(recorded); - await reopened.stop(); - // A compaction that produces a summary removes the failure, and ends without an error. + // The value that drops the compaction's status records its failure. + const ended = (taskId: TaskId) => { + const states = views.live(); + const listed = states.findIndex((state) => state.compactions?.some((status) => status.taskId === taskId)); + return states.find( + (state, index) => index > listed && !state.compactions?.some((status) => status.taskId === taskId), + )!; + }; + expect(ended(recorded!.taskId).compactionFailure).toEqual(recorded); + // A late watch starts with it. + const late = await chat.root.watch(context); + expect((late.value.docs["pi.live"] as LiveState).compactionFailure).toEqual(recorded); + await late.stop(); + // A compaction that produces a summary removes the failure in the commit that ends it. chat.setup.settings.compaction = MANUAL; chat.faux.summaries.push(summary()); const manual = await chat.root.compact(undefined, context); expect((await result(chat, manual)).status).toBe("completed"); expect((await live(chat)).compactionFailure).toBeUndefined(); - await waitFor(() => events.some((event) => event.type === "compaction_end" && event.taskId === manual)); - expect(events.find((event) => event.type === "compaction_end" && event.taskId === manual)).not.toHaveProperty( - "error", - ); - await stream.stop(); + expect(ended(manual).compactionFailure).toBeUndefined(); + views.detach(); await chat.harness.close(context); }); @@ -1179,21 +1176,22 @@ describe("blocking threshold compaction", () => { chat.setup.settings.compaction = BLOCKING; const reached = deferred(); chat.faux.summaries.push(gated(deferred(), summary(), reached)); - const events: AgentEvent[] = []; - const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async ({ value: batch }) => { - events.push(...batch); - }); + const views = await viewHistory(chat.harness, chat.root.id); const input = await chat.root.submit({ type: "input", content: text("u4", 200) }, context); await reached.promise; const [child] = await compactionTasks(chat); await chat.root.abort(context); expect(await input.wait(context)).toMatchObject({ status: "unanswered", reason: "aborted" }); expect((await chat.harness.getTask(child!.id, context))?.state).toMatchObject({ outcome: { status: "aborted" } }); - await waitFor(() => events.some((event) => event.type === "run_end")); - const types = events.map((event) => event.type); - expect(types.indexOf("compaction_end")).toBeLessThan(types.indexOf("run_end")); - await stream.stop(); + views.detach(); + // The compaction's status leaves before the run ends. + const states = views.live(); + const busy = states.findIndex((state) => state.compactions !== undefined); + const compactionEnd = states.findIndex((state, index) => index > busy && state.compactions === undefined); + const runEnd = states.findIndex((state, index) => index > busy && state.run === undefined); + expect(busy).toBeGreaterThan(0); + expect(compactionEnd).toBeLessThan(runEnd); + expect(states[runEnd]!.runFailure).toMatchObject({ inputs: [input.id], reason: "aborted" }); await chat.harness.close(context); }); @@ -1448,9 +1446,11 @@ describe("compaction estimates and interactions", () => { expect(view.messages.slice(1).map(userText)).toEqual([text("u3", 100), text("a3", 100)]); // The parent is untouched. expect((await kinds(chat.root)).includes("pi.compaction")).toBe(false); - // The fork's view keeps the parent entries the summary kept. + // The fork's view shows the parent entries in ID order, the summary after them, its head on a parent entry. const watch = await fork.watch(context); - expect(watch.value.entries).toEqual(view.entries); + expect(watch.value.entries).toEqual(await allEntries(fork)); + expect(watch.value.entries.at(-1)).toEqual(view.head); + expect(watch.value.entries.some((entry) => entry.id === u3.id)).toBe(true); await watch.stop(); await chat.harness.close(context); }); @@ -1565,21 +1565,17 @@ function defineChildTask(gate: Deferred) { }); } -describe("compaction events and live status", () => { - it("reports start and end, and the retry backoff in a late joiner's snapshot", async () => { +describe("compaction live status", () => { + it("lists a compaction from start to end, with its retry backoff in a late joiner's view", async () => { const chat = await open(); await history(chat); chat.setup.settings.retry = { enabled: true, maxRetries: 2, baseDelayMs: 60_000 }; - const events: AgentEvent[] = []; - const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async ({ value: batch }) => { - events.push(...batch); - }); + const views = await viewHistory(chat.harness, chat.root.id); chat.faux.summaries.push(failure("overloaded")); const id = await chat.root.compact(undefined, context); await waitFor(async () => (await live(chat)).compactions?.[0]?.retry !== undefined); - const late = await watchEvents(chat.harness, chat.root.id, context); - expect(late.snapshot.compactions).toEqual([ + const late = await chat.root.watch(context); + expect((late.value.docs["pi.live"] as LiveState).compactions).toEqual([ { taskId: id, reason: "manual", @@ -1590,12 +1586,12 @@ describe("compaction events and live status", () => { ]); await late.stop(); await chat.harness.abortTask(id, context); - await waitFor(() => events.some((event) => event.type === "compaction_end")); - expect(events.filter((event) => event.type.startsWith("compaction_"))).toEqual([ - { type: "compaction_start", taskId: id, reason: "manual", blocking: false }, - { type: "compaction_end", taskId: id, reason: "manual" }, - ]); - await stream.stop(); + await waitFor(async () => (await live(chat)).compactions === undefined); + views.detach(); + const listed = views.live().flatMap((state) => state.compactions ?? []); + expect(listed[0]).toEqual({ taskId: id, reason: "manual", blocking: false, attempt: 1 }); + // An abort is not a failure. + expect(views.live().at(-1)!.compactionFailure).toBeUndefined(); await chat.harness.close(context); }); @@ -1893,15 +1889,21 @@ describe("compaction and the inbox", () => { await chat.harness.close(context); }); - it("keeps the kept entries mounted in the conversation view", async () => { + it("appends the summary to the conversation view in place, its head on a kept entry", async () => { const chat = await open(); await history(chat); const watch = await chat.root.watch(context); + const before = watch.value.entries; watch.start(async () => {}); chat.faux.summaries.push(summary()); await result(chat, await chat.root.compact(undefined, context)); - await waitFor(() => watch.value.entries[0]?.kind === "pi.compaction"); - expect(watch.value.entries).toEqual((await chat.root.context(context)).entries); + await waitFor(() => watch.value.entries.at(-1)?.kind === "pi.compaction"); + // Nothing leaves the window: the summary is appended where it was written. + expect(watch.value.entries.slice(0, before.length)).toEqual(before); + expect(watch.value.entries).toEqual(await allEntries(chat.root)); + const head = (await chat.root.context(context)).head!; + expect(watch.value.entries.at(-1)).toEqual(head); + expect(watch.value.entries.some((entry) => entry.id === head.head)).toBe(true); await watch.stop(); await chat.harness.close(context); }); @@ -2028,41 +2030,36 @@ describe("compaction edge cases", () => { await chat.harness.close(context); }); - it("orders the events of a summary placed at once", async () => { + it("places a summary at once in the commit that ends its compaction", async () => { const chat = await open(); await history(chat); - const events: AgentEvent[] = []; - const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async ({ value: batch }) => { - events.push(...batch); - }); + const views = await viewHistory(chat.harness, chat.root.id); chat.faux.summaries.push(summary()); await result(chat, await chat.root.compact(undefined, context)); - await waitFor(() => events.some((event) => event.type === "compaction_end")); - const end = events.findIndex((event) => event.type === "compaction_end"); - const batch = events.slice(end - 2, end + 3).map((event) => event.type); - expect(batch).toEqual(["message_start", "message_end", "compaction_end", "submission", "usage_changed"]); - await stream.stop(); + views.detach(); + const states = views.live(); + const listed = states.findIndex((state) => state.compactions !== undefined); + const ended = states.findIndex((state, index) => index > listed && state.compactions === undefined); + const before = views.views[ended - 1]!; + const after = views.views[ended]!; + expect(after.entries.slice(before.entries.length).map((entry) => entry.kind)).toEqual(["pi.compaction"]); await chat.harness.close(context); }); - it("puts compaction_start last in the batch of the preparation commit", async () => { + it("lists a background compaction in the preparation commit, with the entries it appends", async () => { const chat = await open({ contextWindow: 2000 }); await history(chat); chat.setup.settings.compaction = BACKGROUND; - const events: AgentEvent[][] = []; - const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async ({ value: batch }) => { - events.push([...batch]); - }); + const views = await viewHistory(chat.harness, chat.root.id); // A new section makes the preparation commit append a system entry next to the compaction's status. addSection(chat.setup.registry, "extra", () => "extra"); chat.faux.summaries.push(gated(deferred(), summary())); await turn(chat, text("u4", 100), "a4"); - await waitFor(() => events.some((batch) => batch.some((event) => event.type === "compaction_start"))); - const batch = events.find((candidate) => candidate.some((event) => event.type === "compaction_start"))!; - expect(batch.map((event) => event.type)).toEqual(["message_start", "message_end", "compaction_start"]); - await stream.stop(); + await waitFor(async () => (await live(chat)).compactions !== undefined); + views.detach(); + const started = views.live().findIndex((state) => state.compactions !== undefined); + const appended = views.views[started]!.entries.slice(views.views[started - 1]!.entries.length); + expect(appended.map((entry) => entry.kind)).toEqual(["pi.system"]); await chat.harness.abortTask((await compactionTasks(chat))[0]!.id, context); await chat.harness.close(context); }); @@ -2283,9 +2280,11 @@ describe("compaction pinning, silent overflow, and late policy changes", () => { chat.faux.summaries.push(gated(deferred(), summary(), reached)); const id = await chat.root.compact(undefined, context); await reached.promise; - const stream = await watchEvents(chat.harness, chat.root.id, context); - expect(stream.snapshot.compactions).toEqual([{ taskId: id, reason: "manual", blocking: false, attempt: 1 }]); - await stream.stop(); + const watch = await chat.root.watch(context); + expect((watch.value.docs["pi.live"] as LiveState).compactions).toEqual([ + { taskId: id, reason: "manual", blocking: false, attempt: 1 }, + ]); + await watch.stop(); await chat.harness.abortTask(id, context); await chat.harness.close(context); }); @@ -2356,11 +2355,6 @@ describe("blocked compaction", () => { ({ harness, root } = await openChat(await openNodeSqliteStorage(path), setup)); harness.resume(); const chat = { harness, root, setup, faux: script(setup) }; - const events: AgentEvent[] = []; - const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async ({ value: batch }) => { - events.push(...batch); - }); const inspection = await chat.harness.inspect(context); expect(inspection.tasks.find((task) => task.record.id === id)?.state).toEqual({ kind: "blocked", @@ -2372,8 +2366,7 @@ describe("blocked compaction", () => { reason: "task_too_old", }); expect((await live(chat)).compactions).toBeUndefined(); - await waitFor(() => events.some((event) => event.type === "compaction_end")); - await stream.stop(); + expect((await live(chat)).compactionFailure).toEqual({ taskId: id, reason: "manual", error: "task_too_old" }); await chat.harness.close(context); }); }); diff --git a/packages/durable/test/harness-events.test.ts b/packages/durable/test/harness-events.test.ts deleted file mode 100644 index 194da5515..000000000 --- a/packages/durable/test/harness-events.test.ts +++ /dev/null @@ -1,779 +0,0 @@ -import { GCProfiler, getHeapStatistics, setFlagsFromString } from "node:v8"; -import { runInNewContext } from "node:vm"; -import type { Draft } from "@earendil-works/chord"; -import { - type AssistantMessage, - type FauxResponseStep, - fauxAssistantMessage, - fauxText, - fauxThinking, - fauxToolCall, - Type, -} from "@earendil-works/pi-ai"; -import { - AgentDoc, - type AgentEvent, - type AgentEventStream, - type Conversation, - defineTool, - type Harness, - LiveDoc, - type LiveState, - MemoryStorage, - type MessageChange, - type SnapshotEvent, - UsageDoc, - watchEvents, -} from "@earendil-works/pi-durable"; -import { describe, expect, it } from "vitest"; -import { type ChatSetup, chatSetup, openChat, textOf, waitFor } from "./chat-support.ts"; -import { addTool } from "./harness-support.ts"; -import { context, documentChanges, observeCommits } from "./session-support.ts"; -import { aborted, deferred } from "./task-support.ts"; - -/** Attach and start an event stream that records every delivered batch. */ -async function listen(harness: Harness, conversation: Conversation) { - const stream = await watchEvents(harness, conversation.id, context); - const batches: AgentEvent[][] = []; - stream.start(async ({ value: events }) => { - batches.push([...events]); - }); - return { stream, batches, events: () => batches.flat() }; -} - -async function drained(): Promise { - await new Promise((resolve) => setTimeout(resolve, 0)); -} - -function slow(): ChatSetup { - return chatSetup({ tokensPerSecond: 400, tokenSize: { min: 1, max: 1 } }); -} - -/** Rebuild the streamed text of block 0 from `message_start` and the text deltas that follow it. */ -function streamedText(events: readonly AgentEvent[]): string { - let text = ""; - for (const event of events) { - if (event.type === "message_start" && event.message.role === "assistant") { - const block = event.message.content[0]; - text = block?.type === "text" ? block.text : ""; - } - if (event.type !== "message_update") continue; - for (const change of event.changes) { - if (change.type === "text_start" && change.contentIndex === 0 && change.block.type === "text") { - text = change.block.text; - } - if (change.type === "text_delta" && change.contentIndex === 0) text += change.delta; - } - } - return text; -} - -type Mutable = { [key: string]: unknown }; - -/** Apply message changes to a copy of `message`, as an events-only consumer would. */ -function applyChanges(message: AssistantMessage, changes: readonly MessageChange[]): AssistantMessage { - let next = structuredClone(message); - for (const change of changes) { - if (change.type === "message") next = structuredClone(change.message); - else if (change.type === "block") next.content[change.contentIndex] = structuredClone(change.block); - else if (change.type.endsWith("_start") && "block" in change) { - next.content.splice(change.contentIndex, 0, structuredClone(change.block)); - } else if (change.type === "text_delta" || change.type === "thinking_delta") { - const block = next.content[change.contentIndex] as unknown as Mutable; - const field = change.type === "text_delta" ? "text" : "thinking"; - block[field] = `${block[field] as string}${change.delta}`; - } else if (change.type === "toolcall_delta") { - let target = (next.content[change.contentIndex] as unknown as Mutable).arguments as Mutable; - for (const segment of change.path.slice(0, -1)) target = target[segment] as Mutable; - const last = change.path.at(-1)!; - target[last] = `${target[last] as string}${change.delta}`; - } - } - return next; -} - -/** Committed partials of `conversation`'s generation, one per commit that has one. */ -function partialsOf(harness: Harness): AssistantMessage[] { - const partials: AssistantMessage[] = []; - observeCommits(harness, (publication) => { - for (const change of documentChanges(publication)) { - const message = (change.value as LiveState | null)?.generation?.message as AssistantMessage | undefined; - if (change.record.kind === "pi.live" && message !== undefined) partials.push(message); - } - }); - return partials; -} - -describe("agent events", () => { - it("streams a run as lifecycle events and text deltas that rebuild the answer", async () => { - const setup = slow(); - const text = "streamed answer text ".repeat(20); - setup.faux.setResponses([fauxAssistantMessage([fauxText(text)])]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const { stream, events } = await listen(harness, root); - expect(stream.snapshot).toMatchObject({ type: "snapshot", entries: [], tools: [], inbox: [] }); - const partials: string[] = []; - observeCommits(harness, (publication) => { - for (const change of documentChanges(publication)) { - const message = (change.value as LiveState | null)?.generation?.message as AssistantMessage | undefined; - if (change.record.kind === "pi.live" && message !== undefined) partials.push(textOf(message) ?? ""); - } - }); - const submission = await root.submit({ type: "input", content: "hi" }, context); - await submission.wait(context); - await drained(); - const all = events(); - const types = all.map((event) => event.type).filter((type, index, list) => type !== list[index - 1]); - expect(types).toEqual([ - "message_start", - "message_end", - "submission", - "run_start", - "turn_start", - "message_start", - "message_update", - "message_end", - "turn_end", - "run_end", - "submission", - "usage_changed", - ]); - expect(all.find((event) => event.type === "run_start")).toEqual({ type: "run_start", inputs: [submission.id] }); - // After each event, the rebuilt text equals the committed partial of that commit. - const rebuilt = all.flatMap((event, index) => - (event.type === "message_start" && event.message.role === "assistant") || event.type === "message_update" - ? [streamedText(all.slice(0, index + 1))] - : [], - ); - expect(rebuilt).toEqual(partials); - expect(partials.length).toBeGreaterThan(1); - const end = all.filter((event) => event.type === "message_end").at(-1)!; - expect(end.type === "message_end" && textOf(end.entry!.model![0])).toBe(text); - await stream.stop(); - await harness.close(context); - }); - - it("reports tool start, output appends, and the result entry", async () => { - const setup = chatSetup(); - const gate = deferred(); - addTool( - setup.registry, - defineTool({ - name: "print", - description: "Prints", - parameters: Type.Object({ n: Type.Number() }), - execute: async (_args, api) => { - api.output("one\n"); - await new Promise((resolve) => setTimeout(resolve, 150)); - api.output("two\n"); - await gate.promise; - return {}; - }, - }), - ); - setup.faux.setResponses([ - fauxAssistantMessage([fauxToolCall("print", { n: 1 }, { id: "c1" })], { stopReason: "toolUse" }), - fauxAssistantMessage([fauxText("done")]), - ]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const { stream, events } = await listen(harness, root); - const submission = await root.submit({ type: "input", content: "go" }, context); - // The output events rebuild the slot's retained window. - const output = (): string => { - let text = ""; - for (const event of events()) { - if (event.type !== "tool_execution_update" || event.output === undefined) continue; - if ("set" in event.output) text = event.output.set; - else text = text.slice(event.output.trimStart ?? 0) + (event.output.append ?? ""); - } - return text; - }; - await waitFor(() => output() === "one\ntwo\n"); - gate.resolve(); - await submission.wait(context); - await drained(); - const tool = events().filter((event) => event.type.startsWith("tool_execution")); - expect(tool[0]).toEqual({ - type: "tool_execution_start", - toolCallId: "c1", - toolName: "print", - taskId: expect.any(Number), - args: { n: 1 }, - }); - const end = tool.at(-1)!; - expect(end).toMatchObject({ type: "tool_execution_end", toolCallId: "c1", entry: { kind: "pi.tool-result" } }); - // As in the coding agent, the tool ends directly before its result message. - const all = events(); - const endIndex = all.indexOf(end); - expect(all.slice(endIndex, endIndex + 3).map((event) => event.type)).toEqual([ - "tool_execution_end", - "message_start", - "message_end", - ]); - expect(all[endIndex + 1]).toMatchObject({ message: { role: "toolResult", toolCallId: "c1" } }); - // Two turns: the tool round, and the answer. - expect(events().filter((event) => event.type === "turn_start")).toHaveLength(2); - expect(events().filter((event) => event.type === "turn_end")).toHaveLength(2); - await stream.stop(); - await harness.close(context); - }); - - it("reports queued submissions, inbox changes, and retries", async () => { - const setup = chatSetup(); - const release = deferred(); - const held: FauxResponseStep = async (_request, options) => { - await Promise.race([release.promise, aborted(options!.signal!)]); - return fauxAssistantMessage([fauxText("first")]); - }; - const error = fauxAssistantMessage([], { stopReason: "error", errorMessage: "503 Service Unavailable" }); - setup.faux.setResponses([held, error, fauxAssistantMessage([fauxText("second")])]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - setup.settings.retry = { enabled: true, maxRetries: 1, baseDelayMs: 1 }; - const { stream, events } = await listen(harness, root); - await root.submit({ type: "input", content: "a" }, context); - const followUp = await root.submit({ type: "input", content: "f" }, context); - await drained(); - expect(events().filter((event) => event.type === "inbox_update")).toEqual([ - { type: "inbox_update", items: [{ id: followUp.id, mode: "followUp" }] }, - ]); - release.resolve(); - await followUp.wait(context); - await drained(); - const types = events().map((event) => event.type); - expect(types).toContain("auto_retry_start"); - expect(types).toContain("auto_retry_end"); - expect(types.filter((type) => type === "run_start")).toHaveLength(2); - expect( - events() - .filter((event) => event.type === "submission") - .map((event) => event.record.status), - ).toEqual(["placed", "queued", "done", "placed", "done"]); - await stream.stop(); - await harness.close(context); - }); - - it("delivers each of 100 pending batches in its own call, in order, unlike state watches", async () => { - const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); - const stream = await watchEvents(harness, root.id, context); - for (let index = 0; index < 100; index++) { - await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: { index } }), context); - } - const batches: AgentEvent[][] = []; - stream.start(async ({ value: events }) => { - batches.push([...events]); - }); - await drained(); - expect(batches).toHaveLength(100); - expect( - batches.map((events) => events.map((event) => (event.type === "entry_appended" ? event.entry.data : event))), - ).toEqual(Array.from({ length: 100 }, (_, index) => [{ index }])); - await stream.stop(); - await harness.close(context); - }); - - it("replaces undelivered batches with one snapshot after 100 pending batches", async () => { - const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); - const stream = await watchEvents(harness, root.id, context); - for (let index = 0; index < 101; index++) { - await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); - } - const batches: AgentEvent[][] = []; - stream.start(async ({ value: events }) => { - batches.push([...events]); - }); - await drained(); - expect(batches).toHaveLength(1); - expect(batches[0]).toHaveLength(1); - expect(batches[0]![0]).toMatchObject({ type: "snapshot" }); - expect((batches[0]![0] as Extract).entries).toHaveLength(101); - await stream.stop(); - await harness.close(context); - }); - - it("rebuilds every committed partial of thinking, text, and tool-call arguments from message changes", async () => { - const setup = chatSetup({ tokensPerSecond: 150, tokenSize: { min: 1, max: 1 } }); - const message = fauxAssistantMessage( - [ - fauxThinking("thinking about it ".repeat(10)), - fauxText("some text ".repeat(10)), - fauxToolCall("missing", { path: "a/long/path/".repeat(10), note: "x".repeat(60) }, { id: "c1" }), - ], - { stopReason: "toolUse" }, - ); - setup.faux.setResponses([message, fauxAssistantMessage([fauxText("done")])]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const partials = partialsOf(harness); - const { stream, events } = await listen(harness, root); - await (await root.submit({ type: "input", content: "go" }, context)).wait(context); - await drained(); - const rebuilt: AssistantMessage[] = []; - let current: AssistantMessage | undefined; - // The streamed tool-calling message, up to its end; the short final answer commits no partial. - for (const event of events()) { - if (event.type === "message_end" && event.entry?.kind === "pi.assistant") break; - if (event.type === "message_start" && event.message.role === "assistant") current = event.message; - else if (event.type === "message_update") current = applyChanges(current!, event.changes); - else continue; - rebuilt.push(current!); - } - expect(partials.length).toBeGreaterThan(2); - expect(rebuilt.map((partial) => partial.content)).toEqual(partials.map((partial) => partial.content)); - const types = new Set( - events() - .flatMap((event) => (event.type === "message_update" ? event.changes : [])) - .map((c) => c.type), - ); - expect(types.has("thinking_delta") || types.has("text_delta")).toBe(true); - await stream.stop(); - await harness.close(context); - }); - - it("rebuilds a sliding tail window from output trims and appends", async () => { - const setup = chatSetup(); - const gate = deferred(); - addTool( - setup.registry, - defineTool({ - name: "tail", - description: "Prints lines", - parameters: Type.Object({}), - outputLimits: { maxLines: 3, retain: "tail" }, - execute: async (_args, api) => { - for (let line = 0; line < 6; line++) { - api.output(`line ${line}\n`); - await new Promise((resolve) => setTimeout(resolve, 120)); - } - await gate.promise; - return {}; - }, - }), - ); - setup.faux.setResponses([ - fauxAssistantMessage([fauxToolCall("tail", {}, { id: "c1" })], { stopReason: "toolUse" }), - fauxAssistantMessage([fauxText("done")]), - ]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const outputs: string[] = []; - observeCommits(harness, (publication) => { - for (const change of documentChanges(publication)) { - const output = (change.value as LiveState | null)?.tools?.[0]?.output; - if (change.record.kind === "pi.live" && output !== undefined && output !== outputs.at(-1)) { - outputs.push(output); - } - } - }); - const { stream, events } = await listen(harness, root); - const submission = await root.submit({ type: "input", content: "go" }, context); - await waitFor(() => outputs.at(-1)?.endsWith("line 5\n") === true); - gate.resolve(); - await submission.wait(context); - await drained(); - const rebuilt: string[] = []; - let text = ""; - for (const event of events()) { - if (event.type !== "tool_execution_update" || event.output === undefined) continue; - if ("set" in event.output) text = event.output.set; - else text = text.slice(event.output.trimStart ?? 0) + (event.output.append ?? ""); - rebuilt.push(text); - } - expect(rebuilt).toEqual(outputs); - expect( - events().some((event) => event.type === "tool_execution_update" && "trimStart" in (event.output ?? {})), - ).toBe(true); - await stream.stop(); - await harness.close(context); - }); - - it("emits one exact batch when a run ends and a queued follow-up starts the next", async () => { - const setup = chatSetup(); - const release = deferred(); - const held: FauxResponseStep = async (_request, options) => { - await Promise.race([release.promise, aborted(options!.signal!)]); - return fauxAssistantMessage([fauxText("first")]); - }; - setup.faux.setResponses([held, fauxAssistantMessage([fauxText("second")])]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const input = await root.submit({ type: "input", content: "a" }, context); - const followUp = await root.submit({ type: "input", content: "f" }, context); - const { stream, batches } = await listen(harness, root); - release.resolve(); - await followUp.wait(context); - await drained(); - const boundary = batches.find((batch) => batch.some((event) => event.type === "run_end"))!; - expect(boundary.map((event) => event.type)).toEqual([ - "message_start", - "message_end", - "message_start", - "message_end", - "turn_end", - "run_end", - "submission", - "submission", - "inbox_update", - "usage_changed", - "run_start", - "turn_start", - ]); - expect(boundary.filter((event) => event.type === "submission").map((event) => event.record.id)).toEqual([ - input.id, - followUp.id, - ]); - await stream.stop(); - await harness.close(context); - }); - - it("ends a call that never runs and a tool aborted with its generation", async () => { - const setup = chatSetup(); - addTool( - setup.registry, - defineTool({ - name: "wait", - description: "Waits until aborted", - parameters: Type.Object({}), - execute: (_args, _api, callContext) => aborted(callContext.abortSignal!), - }), - ); - setup.faux.setResponses([ - fauxAssistantMessage([fauxToolCall("ghost", {}, { id: "c1" }), fauxToolCall("wait", {}, { id: "c2" })], { - stopReason: "toolUse", - }), - ]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const { stream, events } = await listen(harness, root); - const submission = await root.submit({ type: "input", content: "go" }, context); - await waitFor(() => events().some((event) => event.type === "tool_execution_start")); - // Aborting the generation aborts its round: the tool ends with its aborted result first (spec §8.5). - await harness.abortTask((await harness.snapshot(LiveDoc, root.id, context))!.run!.taskId, context); - await submission.wait(context); - await drained(); - // The call not offered ends after its calling message and directly before its result message. - const round = events().map((event) => - event.type === "message_end" - ? `end:${event.entry.kind}` - : event.type === "message_start" - ? `start:${event.message.role}` - : event.type, - ); - const ghostEnd = round.indexOf("tool_execution_end"); - expect(round.slice(ghostEnd - 1, ghostEnd + 3)).toEqual([ - "end:pi.assistant", - "tool_execution_end", - "start:toolResult", - "end:pi.tool-result", - ]); - const tool = events().filter((event) => event.type.startsWith("tool_execution")); - expect(tool.map((event) => [event.type, "toolCallId" in event && event.toolCallId, "entry" in event])).toEqual([ - ["tool_execution_end", "c1", true], - ["tool_execution_start", "c2", false], - ["tool_execution_end", "c2", true], - ]); - await stream.stop(); - await harness.close(context); - }); - - it("reports a steer without run events, a reset as an appended entry, and nothing for other conversations", async () => { - const setup = chatSetup(); - const gate = deferred(); - addTool( - setup.registry, - defineTool({ - name: "hold", - description: "Waits", - parameters: Type.Object({}), - execute: async () => { - await gate.promise; - return {}; - }, - }), - ); - setup.faux.setResponses([ - fauxAssistantMessage([fauxToolCall("hold", {}, { id: "c1" })], { stopReason: "toolUse" }), - fauxAssistantMessage([fauxText("done")]), - ]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const other = await harness.createConversation({ ownership: { kind: "ownerless" } }, context); - const { stream, events, batches } = await listen(harness, root); - const input = await root.submit({ type: "input", content: "a" }, context); - await waitFor(() => events().some((event) => event.type === "tool_execution_start")); - await root.submit({ type: "input", content: "s", whenBusy: "steer" }, context); - const before = batches.length; - await other.commit((tx) => tx.appendEntry(other.id, { kind: "note" }), context); - await drained(); - expect(batches.length).toBe(before); - gate.resolve(); - await input.wait(context); - await root.reset(undefined, context); - await drained(); - expect(events().filter((event) => event.type === "run_start")).toHaveLength(1); - expect(events().filter((event) => event.type === "run_end")).toHaveLength(1); - expect(batches.at(-1)!.map((event) => event.type)).toEqual(["entry_appended", "submission"]); - await stream.stop(); - await harness.close(context); - }); - - it("rejects an attachment cancelled while it waits for the Session line", async () => { - const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); - const release = deferred(); - const blocking = root.commit(async () => { - await release.promise; - }, context); - const controller = new AbortController(); - const attaching = watchEvents(harness, root.id, { ...context, abortSignal: controller.signal }); - controller.abort(new Error("cancelled")); - release.resolve(); - await blocking; - await expect(attaching).rejects.toThrow("cancelled"); - await harness.close(context); - }); - - it("applies deltas after an overflow snapshot that holds an in-flight partial", async () => { - const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); - const partial = fauxAssistantMessage([fauxText("hel")]); - const stream = await watchEvents(harness, root.id, context); - await root.commit(async (tx) => { - (await tx.doc(LiveDoc, root.id)).generation = { - attempt: 1, - message: JSON.parse(JSON.stringify(partial)), - }; - }, context); - for (let index = 0; index < 101; index++) { - await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); - } - const batches: AgentEvent[][] = []; - stream.start(async ({ value: events }) => { - batches.push([...events]); - }); - await drained(); - const snapshot = batches[0]![0] as SnapshotEvent; - expect(snapshot.type).toBe("snapshot"); - await root.commit(async (tx) => { - const block = (await tx.doc(LiveDoc, root.id)).generation!.message!.content[0] as { text: string }; - block.text += "lo"; - }, context); - await drained(); - const update = batches.at(-1)![0]!; - if (update.type !== "message_update") throw new Error(`Unexpected ${update.type}`); - expect(update.changes).toEqual([{ type: "text_delta", contentIndex: 0, delta: "lo" }]); - const rebuilt = applyChanges(snapshot.generation!.message!, update.changes); - expect(rebuilt.content).toEqual([{ type: "text", text: "hello" }]); - await stream.stop(); - await harness.close(context); - }); - - it("reports usage-only updates, cleared tool progress, tools ending without entries, and deferred polls", async () => { - const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); - const { stream, batches } = await listen(harness, root); - const change = (edit: (live: Draft) => void) => - root.commit(async (tx) => edit(await tx.doc(LiveDoc, root.id)), context); - const partial = JSON.parse(JSON.stringify(fauxAssistantMessage([fauxText("partial")]))); - await change((live) => { - live.generation = { attempt: 1, message: partial }; - }); - // A usage-only change sends the usage and no changes. - await change((live) => { - live.generation!.message!.usage.input = 42; - }); - await change((live) => { - live.generation = { attempt: 1, deferred: { pollAt: 1 } }; - }); - await change((live) => { - live.generation!.deferred!.pollAt = 2; - }); - await change((live) => { - live.tools = [{ callId: "c1", name: "t", status: "running", details: { n: 1 }, diagnostics: [] }]; - }); - // A safe replay clears the running slot's progress. - await change((live) => { - delete live.tools![0]!.details; - delete live.tools![0]!.diagnostics; - }); - // A fault marks the slot done without an entry. - await change((live) => { - live.tools![0]!.status = "done"; - }); - await root.commit((tx) => tx.retireDoc(UsageDoc, root.id), context); - await root.commit((tx) => tx.retireDoc(AgentDoc, root.id), context); - await drained(); - expect(batches.map((batch) => batch.filter((event) => event.type !== "task_failed"))).toEqual([ - [{ type: "message_start", message: partial }], - [{ type: "message_update", usage: { ...partial.usage, input: 42 }, changes: [] }], - [{ type: "deferred_poll", pollAt: 1 }], - [{ type: "deferred_poll", pollAt: 2 }], - [{ type: "tool_execution_start", toolCallId: "c1", toolName: "t", args: {} }], - [{ type: "tool_execution_update", toolCallId: "c1", toolName: "t", details: null, diagnostics: [] }], - [{ type: "tool_execution_end", toolCallId: "c1", toolName: "t" }], - // Retired documents read as their initial values. - [{ type: "usage_changed", usage: { models: {}, tools: {} } }], - [{ type: "agent_changed", agent: {} }], - ]); - await stream.stop(); - await harness.close(context); - }); - - it("ends the stream with the Harness", async () => { - const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); - const stream: AgentEventStream = await watchEvents(harness, root.id, context); - await harness.close(context); - expect(await stream.closed).toEqual({ reason: "ended" }); - }); - - it("starts a message at the first committed partial and ends it with the converted entry on abort", async () => { - const setup = chatSetup({ tokensPerSecond: 100, tokenSize: { min: 1, max: 1 } }); - setup.faux.setResponses([fauxAssistantMessage([fauxText("x".repeat(400))])]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const { stream, events } = await listen(harness, root); - const submission = await root.submit({ type: "input", content: "hi" }, context); - await waitFor(() => - events().some((event) => event.type === "message_start" && event.message.role === "assistant"), - ); - const live = await harness.snapshot(LiveDoc, root.id, context); - await harness.abortTask(live!.run!.taskId, context); - await submission.wait(context); - await drained(); - const assistantEnds = events().filter( - (event) => event.type === "message_end" && event.entry?.kind === "pi.assistant", - ); - expect(assistantEnds).toHaveLength(1); - expect( - events().filter((event) => event.type === "message_start" && event.message.role === "assistant"), - ).toHaveLength(1); - await stream.stop(); - await harness.close(context); - }); -}); - -const gc = (() => { - setFlagsFromString("--expose-gc"); - return runInNewContext("gc") as () => void; -})(); - -/** Bytes `run` allocates, counting what collections during it freed. */ -async function allocatedBytes(run: () => Promise): Promise { - gc(); - const profiler = new GCProfiler(); - profiler.start(); - const before = getHeapStatistics().used_heap_size; - await run(); - const after = getHeapStatistics().used_heap_size; - let freed = 0; - for (const collection of profiler.stop().statistics) { - freed += collection.beforeGC.heapStatistics.usedHeapSize - collection.afterGC.heapStatistics.usedHeapSize; - } - return after - before + freed; -} - -/** Heap bytes in use after a full collection. */ -async function heapInUse(): Promise { - for (let round = 0; round < 3; round++) { - await drained(); - gc(); - } - return getHeapStatistics().used_heap_size; -} - -// #9265 -describe("streamed partials", () => { - it("streams a tool-call partial as toolcall_delta appends at its argument paths", async () => { - const setup = chatSetup({ tokensPerSecond: 500, tokenSize: { min: 1, max: 1 } }); - setup.settings.progress = { partialIntervalMs: 20, outputIntervalMs: 100 }; - const args = { path: "src/file.ts", content: "const a = 1;\n".repeat(150) }; - setup.faux.setResponses([ - fauxAssistantMessage([fauxToolCall("missing", args, { id: "c1" })], { stopReason: "toolUse" }), - fauxAssistantMessage([fauxText("done")]), - ]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const partials = partialsOf(harness); - const { stream, events } = await listen(harness, root); - await (await root.submit({ type: "input", content: "go" }, context)).wait(context); - await drained(); - const changes = events().flatMap((event) => (event.type === "message_update" ? event.changes : [])); - const deltas = changes.filter((change) => change.type === "toolcall_delta"); - expect(deltas.length).toBeGreaterThan(10); - expect(deltas.every((change) => ["path", "content"].includes(String(change.path)))).toBe(true); - // The call is resent whole only when a member appears (at most once per argument); growing strings are appends. - // With the raw argument text on the block, every commit resent the whole call. - expect(changes.filter((change) => change.type === "message")).toEqual([]); - expect(changes.filter((change) => change.type === "block").length).toBeLessThanOrEqual(2); - // The committed partials carry only the parsed arguments, never the raw argument text. - for (const partial of partials) { - for (const block of partial.content) { - expect(Object.keys(block).sort()).toEqual(["arguments", "id", "name", "type"]); - } - } - await stream.stop(); - await harness.close(context); - }); - - it("commits a growing text and tool call at a cost that follows the new text, not the string lengths", async () => { - /** Bytes allocated while streaming one answer of `size` characters of text and as much tool-call content. */ - const stream = async (size: number) => { - const setup = chatSetup({ tokensPerSecond: 64_000, tokenSize: { min: 64, max: 64 } }); - setup.settings.progress = { partialIntervalMs: 1, outputIntervalMs: 100 }; - // Keep `pi.live` cached between commits, as the default retention does. Evicted after each use (the suite's - // evicting run), every commit replays the document's delta chain: a cost of eviction, not of appending. - setup.settings.documentRetentionMs = 60_000; - const text = 'streamed text with "quotes" and \\ escapes\n'.repeat(size / 40); - setup.faux.setResponses([ - fauxAssistantMessage( - [fauxText(text), fauxToolCall("missing", { path: "a.ts", content: text }, { id: "c1" })], - { - stopReason: "toolUse", - }, - ), - fauxAssistantMessage([fauxText("done")]), - ]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - let commits = 0; - observeCommits(harness, () => commits++); - const bytes = await allocatedBytes(async () => { - await (await root.submit({ type: "input", content: "go" }, context)).wait(context); - }); - await harness.close(context); - return { bytes, commits }; - }; - const small = await stream(40_000); - const large = await stream(400_000); - expect(large.commits).toBeGreaterThan(small.commits * 5); - // Per commit, the cost stays flat: comparing the whole strings on every commit made it grow with their length. - expect(large.bytes / large.commits / (small.bytes / small.commits)).toBeLessThan(2); - }, 60_000); - - it("keeps a watcher's memory proportional to a long streamed text and tool call, not to their commits", async () => { - const setup = chatSetup({ tokensPerSecond: 16_000, tokenSize: { min: 16, max: 16 } }); - setup.settings.progress = { partialIntervalMs: 1, outputIntervalMs: 100 }; - const line = (index: number) => `line ${index}: some streamed source text with "quotes" and \\ escapes\n`; - const text = Array.from({ length: 2_000 }, (_, index) => line(index)).join(""); - const content = Array.from({ length: 2_000 }, (_, index) => line(index)).join(""); - setup.faux.setResponses([ - fauxAssistantMessage([fauxText(text), fauxToolCall("missing", { path: "a.ts", content }, { id: "c1" })], { - stopReason: "toolUse", - }), - fauxAssistantMessage([fauxText("done")]), - ]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - const { stream, batches } = await listen(harness, root); - const before = await heapInUse(); - const submission = await root.submit({ type: "input", content: "go" }, context); - const partialContent = async () => - (await harness.snapshot(LiveDoc, root.id, context))?.generation?.message?.content; - const shown = async (index: number): Promise => { - const block = (await partialContent())?.[index]; - if (block?.type === "text") return block.text.length; - return block?.type === "toolCall" ? String(block.arguments.content ?? "").length : 0; - }; - const retained: number[] = []; - // Measure while the text, then the tool call, is almost complete: the watcher's view holds the most then. - await waitFor(async () => (await shown(0)) > text.length * 0.95, 30_000); - retained.push((await heapInUse()) - before); - await waitFor(async () => (await shown(1)) > content.length * 0.95, 30_000); - retained.push((await heapInUse()) - before); - await submission.wait(context); - const commits = batches.length; - expect(commits).toBeGreaterThan(500); - // The two 140 KB strings, the watcher's recorded batches and the rest of the stream: about 3 MB. Sliced append ops - // made the watcher's view hold a copy of the text per commit: well over 100 MB. - for (const bytes of retained) expect(bytes).toBeLessThan(20_000_000); - await stream.stop(); - await harness.close(context); - }, 60_000); -}); diff --git a/packages/durable/test/harness-generation.test.ts b/packages/durable/test/harness-generation.test.ts index 05a5f5359..28b55fa4f 100644 --- a/packages/durable/test/harness-generation.test.ts +++ b/packages/durable/test/harness-generation.test.ts @@ -155,7 +155,8 @@ describe("generation", () => { expect((await harness.waitForTask(taskId, context)).state.outcome).toEqual({ status: "aborted" }); // The earlier settlement stays; the run's own settlement leaves it unchanged. expect(await submission.status(context)).toMatchObject({ status: "unanswered", reason: "withdrawn" }); - expect(await live(harness, root)).toEqual({}); + // A run ends without an answer even when its inputs were settled already. + expect(await live(harness, root)).toEqual({ runFailure: { taskId, inputs: [submission.id], reason: "aborted" } }); await harness.close(context); }); @@ -179,8 +180,18 @@ describe("generation", () => { error: { message: "Model faux/missing is not available", detail: { reason: "no_model" } }, }, }); - expect(await live(harness, root)).toEqual({}); - expect(await live(harness, plain)).toEqual({}); + expect(await live(harness, root)).toEqual({ + runFailure: { + taskId: tasks.items[0]!.id, + inputs: [expect.any(Number)], + reason: "no_model", + detail: "Model faux/missing is not available", + }, + }); + expect((await live(harness, plain))?.runFailure).toMatchObject({ + reason: "no_model", + detail: "No model is configured", + }); await harness.close(context); }); @@ -304,7 +315,7 @@ describe("generation", () => { const converted = entries[1]!.model![0] as AssistantMessage; expect(converted.stopReason).toBe("aborted"); expect(textOf(converted)!.startsWith(textOf(partial as Message)!)).toBe(true); - expect(await live(harness, root)).toEqual({}); + expect(await live(harness, root)).toEqual({ runFailure: { taskId, inputs: [submission.id], reason: "aborted" } }); expect((await harness.waitForTask(taskId, context)).state.outcome).toEqual({ status: "aborted" }); await harness.close(context); }); @@ -321,7 +332,7 @@ describe("generation", () => { await harness.abortTask(taskId, context); expect(await submission.wait(context)).toMatchObject({ status: "unanswered", reason: "aborted" }); expect(setup.faux.state.cancelledDeferred).toHaveLength(1); - expect(await live(harness, root)).toEqual({}); + expect(await live(harness, root)).toEqual({ runFailure: { taskId, inputs: [submission.id], reason: "aborted" } }); await harness.close(context); }); @@ -345,7 +356,7 @@ describe("generation", () => { await harness.abortTask(taskId, context); expect(await submission.wait(context)).toMatchObject({ status: "unanswered", reason: "aborted" }); expect(setup.reports).toContainEqual(new Error("cancel failed")); - expect(await live(harness, root)).toEqual({}); + expect(await live(harness, root)).toEqual({ runFailure: { taskId, inputs: [submission.id], reason: "aborted" } }); await harness.close(context); }); @@ -674,7 +685,14 @@ describe("generation", () => { expect(entries.map((entry) => entry.kind)).toEqual(["pi.user", "pi.assistant"]); expect(entries[1]!.model![0]).toMatchObject({ role: "assistant", stopReason: "aborted" }); expect(textOf(entries[1]!.model![0])).toBe("partial"); - expect(await live(harness, root)).toEqual({}); + expect(await live(harness, root)).toEqual({ + runFailure: { + taskId: expect.any(Number), + inputs: [submission.id], + reason: "faulted", + detail: expect.stringContaining("non-JSON function"), + }, + }); const tasks = await harness.commit((tx) => tx.scanTasks({ conversationId: root.id }, 10), context); expect(tasks.items[0]!.state).toMatchObject({ status: "terminal", outcome: { status: "faulted" } }); await harness.close(context); @@ -715,7 +733,9 @@ describe("generation", () => { status: "unanswered", reason: "task_too_old", }); - expect(await live(harness, root)).toEqual({}); + expect(await live(harness, root)).toEqual({ + runFailure: { taskId, inputs: [submissionId], reason: "orphaned", detail: "task_too_old" }, + }); await harness.close(context); }); diff --git a/packages/durable/test/harness-input-entries.test.ts b/packages/durable/test/harness-input-entries.test.ts index 7c954e6f1..44d68a9ac 100644 --- a/packages/durable/test/harness-input-entries.test.ts +++ b/packages/durable/test/harness-input-entries.test.ts @@ -12,7 +12,6 @@ import { Type, } from "@earendil-works/pi-ai"; import { - type AgentEvent, type Conversation, ConversationBusy, defineTool, @@ -20,14 +19,15 @@ import { type Harness, InboxDoc, LiveDoc, + type LiveState, MemoryStorage, type SubmissionDraft, - watchEvents, } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, type ChatSetup, chatSetup, openChat, textOf, waitFor } from "./chat-support.ts"; import { addSection, addTool } from "./harness-support.ts"; +import { viewHistory } from "./live-support.ts"; import { context } from "./session-support.ts"; import { aborted, type Deferred, deferred } from "./task-support.ts"; @@ -263,11 +263,7 @@ describe("data-only writes", () => { const requests: string[][] = []; setup.faux.setResponses([requestLog(requests, HOLD), requestLog(requests, answer("done"))]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const events: AgentEvent[] = []; - const stream = await watchEvents(harness, root.id, context); - stream.start(async ({ value: batch }) => { - events.push(...batch); - }); + const history = await viewHistory(harness, root.id); const input = await root.submit({ type: "input", content: "a" }, context); await toolRunning(harness, root); const chat = { kind: "chat.message", data: { author: "bob", text: "lunch?" } }; @@ -281,12 +277,13 @@ describe("data-only writes", () => { ["pi.assistant"], ["chat.message", { author: "bob", text: "lunch?" }], ]); - await waitFor(() => events.some((event) => event.type === "entry_appended")); - const appended = events.find((event) => event.type === "entry_appended"); - expect(appended).toMatchObject({ - entry: { kind: "chat.message", id: settled.status === "done" && settled.entry }, + // The view shows it while the run goes on. + const shown = history.views.find((view) => view.entries.some((entry) => entry.kind === "chat.message")); + expect(shown?.entries.at(-1)).toMatchObject({ + kind: "chat.message", + id: settled.status === "done" && settled.entry, }); - expect(events.some((event) => event.type === "run_end")).toBe(false); + expect((shown?.docs["pi.live"] as LiveState).run?.inputs).toEqual([input.id]); // A repeated request ID returns the same write without a second entry. expect((await root.submit({ type: "write", entry: chat, requestId: "bob:1" }, context)).id).toBe(write.id); @@ -303,7 +300,7 @@ describe("data-only writes", () => { ["system:", "user:a"], ["system:", "user:a", "assistant:", "toolResult:held"], ]); - await stream.stop(); + history.detach(); await harness.close(context); }); @@ -374,7 +371,14 @@ describe("data-only writes", () => { await harness.waitForIdle(context); expect((await f.status(context)).status).toBe("queued"); expect((await inboxItems(harness, root)).map((item) => item.id)).toEqual([f.id]); - expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({ + runFailure: { + taskId: expect.any(Number), + inputs: [input.id], + reason: "model_error", + detail: "invalid request", + }, + }); expect(setup.faux.state.callCount).toBe(1); // The next input places the waiting mention first. const g = await root.submit({ type: "input", content: "g" }, context); diff --git a/packages/durable/test/harness-nested-tools-restart.test.ts b/packages/durable/test/harness-nested-tools-restart.test.ts index 68a96b854..44c508f70 100644 --- a/packages/durable/test/harness-nested-tools-restart.test.ts +++ b/packages/durable/test/harness-nested-tools-restart.test.ts @@ -11,7 +11,6 @@ import { Type, } from "@earendil-works/pi-ai"; import { - type AgentEvent, defineExtension, defineTask, defineTool, @@ -23,12 +22,12 @@ import { ToolResultEntry, ToolTask, type ToolTaskInput, - watchEvents, } from "@earendil-works/pi-durable"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; import { addHooks, addTool } from "./harness-support.ts"; +import { nestedResultOf, nestedResults, viewHistory } from "./live-support.ts"; import { context } from "./session-support.ts"; import { aborted, deferred, eventually } from "./task-support.ts"; @@ -75,16 +74,6 @@ function results(entries: Awaited>): ToolResultMes return entries.filter((entry) => ToolResultEntry.is(entry)).map((entry) => entry.model![0] as ToolResultMessage); } -/** Collect tool end events from now on. */ -async function collectEnds(harness: Harness, conversationId: Parameters[0]) { - const ends: Extract[] = []; - const stream = await watchEvents(harness, conversationId, context); - stream.start(async ({ value: batch }) => { - for (const event of batch) if (event.type === "tool_execution_end") ends.push(event); - }); - return { ends, stop: () => stream.stop() }; -} - function codes(result: NestedToolExecutionResult | undefined): string[] { return (result?.diagnostics ?? []).flatMap((diagnostic) => (diagnostic.code === undefined ? [] : [diagnostic.code])); } @@ -142,18 +131,19 @@ describe("nested tool calls across a restart", () => { await opened.harness.close(context); opened = await open(path, setup); - const { ends, stop } = await collectEnds(opened.harness, opened.root.id); + const history = await viewHistory(opened.harness, opened.root.id); + const nested = nestedResults(opened.harness); opened.harness.resume(); expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); await opened.harness.waitForIdle(context); - await stop(); + history.detach(); expect(runs).toEqual({ started: 1, gated: 0 }); const [started, gate] = await nestedTasks(opened.harness); for (const task of [started, gate]) { expect(task).toMatchObject({ abandonOnRestart: true, abortRequested: true, abortReason: "restart" }); expect(task!.state).toMatchObject({ status: "terminal", outcome: { status: "aborted" } }); } - const end = (callId: string) => ends.find((event) => event.toolCallId === callId)?.result; + const end = (callId: string) => nestedResultOf(history.live(), nested, callId); expect(codes(end("c1/1"))).toEqual(["interrupted"]); expect(end("c1/1")).toMatchObject({ isError: true, details: { step: 1 } }); expect(end("c1/1")).not.toHaveProperty("structuredOutput"); diff --git a/packages/durable/test/harness-nested-tools.test.ts b/packages/durable/test/harness-nested-tools.test.ts index d8354fcf2..2db53ada5 100644 --- a/packages/durable/test/harness-nested-tools.test.ts +++ b/packages/durable/test/harness-nested-tools.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import type { JsonValue } from "@earendil-works/chord"; import { withAbortSignal } from "@earendil-works/chord/context"; +import { applyImmutable } from "@earendil-works/chord/delta"; import { type AssistantMessage, fauxAssistantMessage, @@ -13,12 +14,13 @@ import { type Usage, } from "@earendil-works/pi-ai"; import { - type AgentEvent, AssistantEntry, + type ConversationView, defineTool, type EntryRecord, type Harness, LiveDoc, + type LiveState, MemoryStorage, NestedCallDoc, type NestedToolExecutionResult, @@ -30,12 +32,12 @@ import { type ToolTaskInput, type ToolTaskResult, UsageDoc, - watchEvents, } from "@earendil-works/pi-durable"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; import { addHooks, addTool } from "./harness-support.ts"; +import { endedAt, nestedResultOf, nestedResults, slotsOf, startOrder, viewHistory } from "./live-support.ts"; import { context } from "./session-support.ts"; import { aborted, deferred, eventually } from "./task-support.ts"; @@ -372,15 +374,12 @@ describe("nested tool calls", () => { ); setup.faux.setResponses([call("batch"), DONE]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const stream = await watchEvents(harness, root.id, context); - const ends: AgentEvent[] = []; - stream.start(async ({ value: batch }) => { - ends.push(...batch.filter((event) => event.type === "tool_execution_end")); - }); + const history = await viewHistory(harness, root.id); + const nested = nestedResults(harness); const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); expect(settled.status).toBe("done"); await harness.waitForIdle(context); - await stream.stop(); + history.detach(); const [result] = results(await allEntries(root)); expect(text(result)).toBe("returned early"); const [, child] = await toolTasks(harness); @@ -389,16 +388,13 @@ describe("nested tool calls", () => { outcome: { status: "aborted", result: { kind: "nested" } }, }); // The aborted call's result keeps its details; programs get no partial output, which only the model read. - const childEnd = ends.find((event) => event.type === "tool_execution_end" && event.toolCallId === "c1/1"); - expect(childEnd).toMatchObject({ result: { isError: true, details: { step: 1 } } }); - expect((childEnd as { result?: NestedToolExecutionResult }).result).not.toHaveProperty("structuredOutput"); - // The nested call ends, with its result, before its caller. - expect(ends.map((event) => [event.type === "tool_execution_end" && event.toolCallId, "result" in event])).toEqual( - [ - ["c1/1", true], - ["c1", false], - ], - ); + const childResult = nestedResultOf(history.live(), nested, "c1/1"); + expect(childResult).toMatchObject({ isError: true, details: { step: 1 } }); + expect(childResult).not.toHaveProperty("structuredOutput"); + // The nested call ends before its caller. + const live = history.live(); + expect(endedAt(live, "c1/1")).toBeGreaterThan(0); + expect(endedAt(live, "c1/1")).toBeLessThan(endedAt(live, "c1")); expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); await harness.close(context); }); @@ -429,7 +425,7 @@ describe("nested tool calls", () => { await harness.close(context); }); - it("reports nested calls in pi.live.nestedTools and as tool events with their parent", async () => { + it("reports nested calls in pi.live.nestedTools with their parent", async () => { const setup = chatSetup(); const release = deferred(); const running = deferred(); @@ -452,12 +448,9 @@ describe("nested tool calls", () => { ); setup.faux.setResponses([call("batch"), DONE]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const stream = await watchEvents(harness, root.id, context); - expect(stream.snapshot.nestedTools).toEqual([]); - const events: AgentEvent[] = []; - stream.start(async ({ value: batch }) => { - events.push(...batch); - }); + const history = await viewHistory(harness, root.id); + expect(history.live()[0]!.nestedTools).toBeUndefined(); + const nested = nestedResults(harness); const submission = await root.submit({ type: "input", content: "go" }, context); await running.promise; const live = await harness.snapshot(LiveDoc, root.id, context); @@ -477,32 +470,19 @@ describe("nested tool calls", () => { release.resolve(); await submission.wait(context); await harness.waitForIdle(context); - await stream.stop(); - const tools = events.flatMap((event) => - event.type === "tool_execution_start" || event.type === "tool_execution_end" - ? [[event.type, event.toolCallId, event.parentToolCallId]] - : [], - ); - expect(tools).toEqual([ - ["tool_execution_start", "c1", undefined], - ["tool_execution_start", "c1/1", "c1"], - ["tool_execution_end", "c1/1", "c1"], - ["tool_execution_end", "c1", undefined], - ]); - const nestedEnd = events.find((event) => event.type === "tool_execution_end" && event.toolCallId === "c1/1"); - expect(nestedEnd).toMatchObject({ result: { structuredOutput: "slow done", isError: false } }); - // Events name the call's task and, for a nested call, the calling one. + history.detach(); + const states = history.live(); + expect(startOrder(states)).toEqual(["c1", "c1/1"]); + expect(endedAt(states, "c1/1")).toBeLessThan(endedAt(states, "c1")); + expect(nestedResultOf(states, nested, "c1/1")).toMatchObject({ structuredOutput: "slow done", isError: false }); + // Slots name the call's task and, for a nested call, the calling one. const [parent, child] = await toolTasks(harness); - const parentEnd = events.find((event) => event.type === "tool_execution_end" && event.toolCallId === "c1"); - expect(nestedEnd).toMatchObject({ taskId: child!.id, parentTaskId: parent!.id }); - expect(parentEnd).toMatchObject({ taskId: parent!.id }); - expect(parentEnd).not.toHaveProperty("parentTaskId"); - expect( - events.some( - (event) => - event.type === "tool_execution_update" && event.toolCallId === "c1/1" && event.parentToolCallId === "c1", - ), - ).toBe(true); + expect(slotsOf(states, "c1/1").find((slot) => slot !== undefined)).toMatchObject({ + taskId: child!.id, + parentCallId: "c1", + parentTaskId: parent!.id, + }); + expect(slotsOf(states, "c1").find((slot) => slot?.taskId !== undefined)).toMatchObject({ taskId: parent!.id }); expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); await harness.close(context); }); @@ -535,7 +515,9 @@ describe("nested tool calls", () => { record!.state.status === "terminal" ? record!.state.outcome : undefined; expect(outcome(child)).toMatchObject({ status: "aborted", result: { kind: "nested" } }); expect(outcome(parent)).toMatchObject({ status: "aborted", result: { kind: "model" } }); - expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({ + runFailure: expect.objectContaining({ reason: "aborted" }), + }); await harness.close(context); }); @@ -831,29 +813,19 @@ describe("nested tool calls", () => { ); setup.faux.setResponses([call("outer"), DONE]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const stream = await watchEvents(harness, root.id, context); - const tools: (string | undefined)[][] = []; - stream.start(async ({ value: batch }) => { - for (const event of batch) { - if (event.type === "tool_execution_start" || event.type === "tool_execution_end") { - tools.push([event.type, event.toolCallId, event.parentToolCallId]); - } - } - }); + const history = await viewHistory(harness, root.id); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); await harness.waitForIdle(context); - await stream.stop(); + history.detach(); expect(text(results(await allEntries(root))[0])).toBe("echo deep"); const [outer, inner, deep] = await toolTasks(harness); expect([inner!.owner, deep!.owner]).toEqual([outer!.id, inner!.id]); - expect(tools).toEqual([ - ["tool_execution_start", "c1", undefined], - ["tool_execution_start", "c1/1", "c1"], - ["tool_execution_start", "c1/1/x", "c1/1"], - ["tool_execution_end", "c1/1/x", "c1/1"], - ["tool_execution_end", "c1/1", "c1"], - ["tool_execution_end", "c1", undefined], - ]); + const states = history.live(); + expect(startOrder(states)).toEqual(["c1", "c1/1", "c1/1/x"]); + expect(slotsOf(states, "c1/1/x").find((slot) => slot !== undefined)).toMatchObject({ parentCallId: "c1/1" }); + // Children end first. + expect(endedAt(states, "c1/1/x")).toBeLessThan(endedAt(states, "c1/1")); + expect(endedAt(states, "c1/1")).toBeLessThan(endedAt(states, "c1")); expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); await harness.close(context); }); @@ -889,31 +861,31 @@ describe("nested tool calls", () => { const { harness, root } = await openChat(new MemoryStorage(), setup); const submission = await root.submit({ type: "input", content: "go" }, context); await Promise.all([running.one.promise, running.two.promise]); - const stream = await watchEvents(harness, root.id, context); - expect(stream.snapshot.nestedTools.map((slot) => [slot.callId, slot.status])).toEqual([ + const watch = await root.watch(context); + const nestedOf = (view: ConversationView) => (view.docs["pi.live"] as LiveState).nestedTools ?? []; + expect(nestedOf(watch.value).map((slot) => [slot.callId, slot.status])).toEqual([ ["c1/1", "running"], ["c1/2", "running"], ]); - const updates: [string, unknown][] = []; - stream.start(async ({ value: batch }) => { - for (const event of batch) { - if (event.type === "tool_execution_update" && event.output !== undefined) { - updates.push([event.toolCallId, event.output]); - } - } + // Each value's operations rebuild the next one, the late subscriber's starting value included. + let replayed = watch.value; + const outputs: [string | undefined, string | undefined][] = []; + watch.start(async ({ value, ops }) => { + replayed = applyImmutable(replayed, ops); + expect(replayed).toEqual(value); + const [one, two] = nestedOf(value); + outputs.push([one?.output, two?.output]); }); // The second slot writes first, so a wrong slot index in the update path would show. wrote.two.resolve(); - await eventually(() => updates.length === 1); + await eventually(() => outputs.at(-1)?.[1] === "two\n"); + expect(outputs.at(-1)).toEqual([undefined, "two\n"]); wrote.one.resolve(); - await eventually(() => updates.length === 2); + await eventually(() => outputs.at(-1)?.[0] === "one\n"); + expect(outputs.at(-1)).toEqual(["one\n", "two\n"]); release.resolve(); await submission.wait(context); - await stream.stop(); - expect(updates).toEqual([ - ["c1/2", { set: "two\n" }], - ["c1/1", { set: "one\n" }], - ]); + await watch.stop(); await harness.close(context); }); @@ -988,17 +960,13 @@ describe("nested tool calls", () => { ); setup.faux.setResponses([call("batch"), DONE]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const stream = await watchEvents(harness, root.id, context); - const ends: AgentEvent[] = []; - stream.start(async ({ value: batch }) => { - ends.push(...batch.filter((event) => event.type === "tool_execution_end")); - }); + const history = await viewHistory(harness, root.id); + const nested = nestedResults(harness); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); await harness.waitForIdle(context); - await stream.stop(); + history.detach(); // The abort's result is built from the slot, which never got the output. - const end = ends.find((event) => event.type === "tool_execution_end" && event.toolCallId === "c1/1"); - const result = (end as { result?: NestedToolExecutionResult }).result!; + const result = nestedResultOf(history.live(), nested, "c1/1")!; expect(result).toMatchObject({ isError: true }); expect(result).not.toHaveProperty("structuredOutput"); expect(result).not.toHaveProperty("details"); @@ -1038,11 +1006,6 @@ describe("nested tool calls", () => { ); setup.faux.setResponses([call("batch"), DONE]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const stream = await watchEvents(harness, root.id, context); - const starts: AgentEvent[] = []; - stream.start(async ({ value: batch }) => { - starts.push(...batch.filter((event) => event.type === "tool_execution_start")); - }); const submission = await root.submit({ type: "input", content: "go" }, context); const slot = async () => (await harness.snapshot(LiveDoc, root.id, context))!.nestedTools![0]!; await hookEntered.promise; @@ -1053,10 +1016,7 @@ describe("nested tool calls", () => { release.resolve(); await submission.wait(context); await harness.waitForIdle(context); - await stream.stop(); - // The start event carries the arguments the call runs with; the task input keeps the ones it was made with. - const start = starts.find((event) => event.type === "tool_execution_start" && event.toolCallId === "c1/1"); - expect(start).toMatchObject({ args: { text: large } }); + // The slot showed the arguments the call runs with; the task input keeps the ones it was made with. const nested = (await toolTasks(harness)).find((task) => (task.input as ToolTaskInput).kind === "nested"); expect(nested!.input as ToolTaskInput).toMatchObject({ call: { arguments: { text: "asked" } } }); await harness.close(context); diff --git a/packages/durable/test/harness-ownership.test.ts b/packages/durable/test/harness-ownership.test.ts index d363f9c76..d03806c9e 100644 --- a/packages/durable/test/harness-ownership.test.ts +++ b/packages/durable/test/harness-ownership.test.ts @@ -1153,7 +1153,7 @@ describe("owned conversations from tools and supervisors", () => { const childConversation = (await harness.conversation(child!, context))!; await childConversation.waitForIdle(context); const childLive = await harness.snapshot(LiveDoc, child!, context); - expect(childLive).toEqual({}); + expect(childLive).toEqual({ runFailure: expect.objectContaining({ reason: "aborted" }) }); const childInputs = (await harness.inspect(context)).submissions.filter((s) => s.conversationId === child); expect(childInputs).toEqual([]); await harness.waitForIdle(context); diff --git a/packages/durable/test/harness-run-failure.test.ts b/packages/durable/test/harness-run-failure.test.ts new file mode 100644 index 000000000..4c21cc970 --- /dev/null +++ b/packages/durable/test/harness-run-failure.test.ts @@ -0,0 +1,156 @@ +// `pi.live.runFailure`: why the newest run ended without an answer, as state a renderer shows (#11327). +import { type FauxResponseStep, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { + type AfterFailedRun, + type Conversation, + defineTool, + type Harness, + LiveDoc, + type LiveState, + MemoryStorage, + type RunFailure, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import { chatSetup, openChat, unanswered } from "./chat-support.ts"; +import { addTool } from "./harness-support.ts"; +import { context } from "./session-support.ts"; +import { deferred } from "./task-support.ts"; + +async function live(harness: Harness, conversation: Conversation): Promise> { + return (await harness.snapshot(LiveDoc, conversation.id, context)) ?? {}; +} + +/** A response that fails for good once `gate` opens; `reached` resolves when it was requested. */ +function failingOnce() { + const gate = deferred(); + const reached = deferred(); + const step: FauxResponseStep = async () => { + reached.resolve(); + await gate.promise; + return fauxAssistantMessage([], { stopReason: "error", errorMessage: "Invalid request" }); + }; + return { step, gate, reached }; +} + +describe("pi.live.runFailure", () => { + it("records a model error with its text, and a later run clears it", async () => { + const setup = chatSetup(); + setup.faux.setResponses([ + fauxAssistantMessage([], { stopReason: "error", errorMessage: "Invalid request" }), + fauxAssistantMessage([fauxText("fine")]), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "hi" }, context); + const settled = await submission.wait(context); + expect(settled).toMatchObject({ status: "unanswered", reason: "model_error", detail: "Invalid request" }); + const failure = (await live(harness, root)).runFailure; + expect(failure).toEqual({ + taskId: expect.any(Number), + inputs: [submission.id], + reason: "model_error", + detail: "Invalid request", + } satisfies RunFailure); + const next = await root.submit({ type: "input", content: "again" }, context); + // The commit that starts the next run removes it. + expect((await live(harness, root)).runFailure).toBeUndefined(); + expect((await next.wait(context)).status).toBe("done"); + expect(await live(harness, root)).toEqual({}); + await harness.close(context); + }); + + it("records a missing model with which one", async () => { + const setup = chatSetup(); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await root.configure({ model: { provider: "faux", modelId: "gone" } }, context); + const submission = await root.submit({ type: "input", content: "hi" }, context); + const settled = await submission.wait(context); + expect(settled).toMatchObject({ status: "unanswered", reason: "no_model" }); + // The settlement is unchanged: no detail. + expect(settled).not.toHaveProperty("detail"); + expect((await live(harness, root)).runFailure).toMatchObject({ + inputs: [submission.id], + reason: "no_model", + detail: "Model faux/gone is not available", + }); + await harness.close(context); + }); + + it("records an abort", async () => { + const setup = chatSetup(); + const busy = unanswered(); + setup.faux.setResponses([busy.step]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "hi" }, context); + await busy.reached; + const taskId = (await live(harness, root)).run!.taskId; + await root.abort(context); + expect(await submission.wait(context)).toMatchObject({ status: "unanswered", reason: "aborted" }); + expect((await live(harness, root)).runFailure).toEqual({ taskId, inputs: [submission.id], reason: "aborted" }); + await harness.close(context); + }); + + it("records a reset that cut the run before an answer, kept by the run its queued input starts", async () => { + const setup = chatSetup(); + const gate = deferred(); + const started = deferred(); + addTool( + setup.registry, + defineTool({ + name: "hold", + description: "Waits", + parameters: Type.Object({}), + execute: async () => { + started.resolve(); + await gate.promise; + return {}; + }, + }), + ); + const second = unanswered(); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("hold", {}, { id: "c1" })], { stopReason: "toolUse" }), + second.step, + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const first = await root.submit({ type: "input", content: "go" }, context); + await started.promise; + await root.reset(undefined, context); + const followUp = await root.submit({ type: "input", content: "next", whenBusy: "followUp" }, context); + gate.resolve(); + expect(await first.wait(context)).toMatchObject({ status: "unanswered", reason: "reset" }); + await second.reached; + const state = await live(harness, root); + // The follow-up's run started in the commit that ended the first; the failure stays until a later run starts. + expect(state.run?.inputs).toEqual([followUp.id]); + expect(state.runFailure).toMatchObject({ inputs: [first.id], reason: "reset" }); + await root.abort(context); + expect((await live(harness, root)).runFailure).toMatchObject({ inputs: [followUp.id], reason: "aborted" }); + await harness.close(context); + }); + + it("keeps the failure while a run continues the queue in the same commit, and replaces it with the next", async () => { + for (const policy of ["continue", "keep"] as const satisfies readonly AfterFailedRun[]) { + const setup = chatSetup(); + setup.settings.afterFailedRun = policy; + const failing = failingOnce(); + const busy = unanswered(); + setup.faux.setResponses([failing.step, busy.step, fauxAssistantMessage([fauxText("third")])]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const first = await root.submit({ type: "input", content: "first" }, context); + await failing.reached.promise; + const second = await root.submit({ type: "input", content: "second", whenBusy: "followUp" }, context); + failing.gate.resolve(); + await first.wait(context); + if (policy === "continue") { + await busy.reached; + const state = await live(harness, root); + expect(state.run?.inputs).toEqual([second.id]); + expect(state.runFailure).toMatchObject({ inputs: [first.id], reason: "model_error" }); + } else { + await harness.waitForIdle(context); + expect((await live(harness, root)).runFailure).toMatchObject({ inputs: [first.id] }); + } + await harness.close(context); + } + }); +}); diff --git a/packages/durable/test/harness-storage-failure.test.ts b/packages/durable/test/harness-storage-failure.test.ts index 35604a9ee..12b08a476 100644 --- a/packages/durable/test/harness-storage-failure.test.ts +++ b/packages/durable/test/harness-storage-failure.test.ts @@ -11,7 +11,6 @@ import { MemoryStorage, StorageRequestError, type TaskId, - watchEvents, } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; import { safeNow } from "../src/harness/harness.ts"; @@ -95,8 +94,8 @@ describe("a failed Storage call", () => { const task = harness.waitForTask(id, context); const idle = harness.waitForIdle(context); const watch = (await harness.watchDoc(LiveDoc, root.id, context))!; - const stream = await watchEvents(harness, root.id, context); - stream.start(async () => {}); + const view = await root.watch(context); + view.start(async () => {}); const failure = new Error("disk gone"); storage.failNextCommit(failure); @@ -107,7 +106,7 @@ describe("a failed Storage call", () => { await expect(task).rejects.toMatchObject(failed); await expect(idle).rejects.toMatchObject(failed); expect(await watch.closed).toEqual({ reason: "failed", error: failure }); - expect(await stream.closed).toEqual({ reason: "failed", error: failure }); + expect(await view.closed).toEqual({ reason: "failed", error: failure }); // Every later call, and a wait that starts now. await expect(harness.getTask(id, context)).rejects.toMatchObject(failed); await expect(harness.waitForIdle(context)).rejects.toMatchObject(failed); @@ -487,8 +486,9 @@ describe("listener and report errors", () => { const failure = new Error("observer broke"); await views.attach( root.id, - async () => ({ - publication: () => { + undefined, + () => ({ + advance: () => { throw failure; }, end: () => {}, diff --git a/packages/durable/test/harness-structured-output.test.ts b/packages/durable/test/harness-structured-output.test.ts index 2c3d2ba5d..aac0df8a9 100644 --- a/packages/durable/test/harness-structured-output.test.ts +++ b/packages/durable/test/harness-structured-output.test.ts @@ -20,7 +20,6 @@ import { type ToolRegistration, ToolResultEntry, ToolTask, - watchEvents, } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; import { NodeExecutionEnv } from "../src/env/node.ts"; @@ -163,25 +162,15 @@ describe("structured output", () => { if (result !== undefined && change.ops.length > 0) stored.push(result as unknown as JsonValue); } }); - const stream = await watchEvents(harness, root.id, context); - const ends: NestedToolExecutionResult[] = []; - stream.start(async ({ value: batch }) => { - for (const event of batch) { - if (event.type === "tool_execution_end" && event.parentToolCallId !== undefined) { - ends.push(event.result as NestedToolExecutionResult); - } - } - }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); await harness.waitForIdle(context); - await stream.stop(); await harness.close(context); expect(received[0]).toMatchObject({ structuredOutput: "a", isError: false }); expect(received[1]).toMatchObject({ structuredOutput: null, isError: false }); expect(received[0]).not.toHaveProperty("output"); + // A client reads a live caller's nested results from the same documents. expect(stored).toEqual(received); - expect(ends).toEqual(received); }); it("gives callers a schema tool's validated structured output, which the transcript does not store", async () => { diff --git a/packages/durable/test/harness-structured.test.ts b/packages/durable/test/harness-structured.test.ts index 12860f89b..771bb9837 100644 --- a/packages/durable/test/harness-structured.test.ts +++ b/packages/durable/test/harness-structured.test.ts @@ -11,9 +11,9 @@ import { Type, } from "@earendil-works/pi-ai"; import { - type AgentEvent, type Conversation, type ConversationId, + type ConversationView, createSession, defineDoc, defineTask, @@ -26,18 +26,19 @@ import { type NextTaskState, type Registry, type Storage, + type TaskGraph, type TaskId, type TaskOptions, type TaskRuntime, type ToolRegistration, ToolResultEntry, type Tx, - watchEvents, } from "@earendil-works/pi-durable"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, chatSetup, openChat, waitFor } from "./chat-support.ts"; import { addHooks, addTask, addTool } from "./harness-support.ts"; +import { generations, graphHistory, slotsOf, viewHistory } from "./live-support.ts"; import { ControlledStorage, context } from "./session-support.ts"; import { aborted, type Deferred, deferred, openTasks, settled } from "./task-support.ts"; @@ -1251,7 +1252,9 @@ describe("tool rounds", () => { expect(await submission.wait(context)).toMatchObject({ status: "unanswered", reason: "aborted" }); const results = (await allEntries(root)).filter((entry) => ToolResultEntry.is(entry)); expect(results.map((entry) => (entry.model![0] as ToolResultMessage).toolCallId)).toEqual(["c1", "c2"]); - expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({ + runFailure: { taskId: generation, inputs: [submission.id], reason: "aborted" }, + }); expect((await harness.waitForTask(generation, context)).state.outcome.status).toBe("aborted"); await harness.close(context); }); @@ -1286,7 +1289,7 @@ describe("tool rounds", () => { await harness.close(context); }); - it("ends a turn at the generation's hold, before its successor's turn starts", async () => { + it("hands run control to the successor while the generation holds, and keeps it in the graph until it ends", async () => { const setup = chatSetup(); addTask(setup.registry, Node); addTool( @@ -1308,21 +1311,23 @@ describe("tool rounds", () => { setup.faux.setResponses([toolCalls(["noop", "c1"]), fauxAssistantMessage([fauxText("done")])]); const { harness, root } = await openChat(new MemoryStorage(), setup); opened = harness; - const stream = await watchEvents(harness, root.id, context); - const events: AgentEvent[] = []; - stream.start(async ({ value: batch }) => { - events.push(...batch); - }); + const graph = await graphHistory(harness); + const history = await viewHistory(harness, root.id); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); const first = (await harness.commit((tx) => tx.scanTasks({ kind: "pi.generation" }, 1), context)).items[0]!; expect((await state(harness, first.id)).status).toBe("completing"); - const turns = () => events.filter((event) => event.type.startsWith("turn_")).map((event) => event.type); - await waitFor(() => turns().length === 4); - expect(turns()).toEqual(["turn_start", "turn_end", "turn_start", "turn_end"]); + // The first generation's outcome is committed at its hold: the successor already runs the run, and the graph shows + // the first one completing beside it. + const handed = graph.graphs.find((value) => generations(value).length === 2)!; + const [held, successor] = generations(handed); + expect(held).toMatchObject({ id: first.id, state: { status: "completing" } }); + expect(successor!.state.status).not.toBe("completing"); + expect(history.live().some((live) => live.run?.taskId === successor!.id)).toBe(true); open("hooked"); expect(await outcomeOf(harness, first.id)).toBe("completed"); - await waitFor(() => turns().length === 4); - await stream.stop(); + await waitFor(() => graph.graphs.at(-1)!.tasks[String(first.id)] === undefined); + graph.detach(); + history.detach(); await harness.close(context); }); @@ -1396,7 +1401,14 @@ describe("tool rounds", () => { opened = reopened.harness; const again = (await reopened.harness.submission(submission.id, context))!; expect(await again.wait(context)).toMatchObject({ status: "unanswered", reason: "faulted" }); - expect(await reopened.harness.snapshot(LiveDoc, reopened.root.id, context)).toEqual({}); + expect(await reopened.harness.snapshot(LiveDoc, reopened.root.id, context)).toEqual({ + runFailure: { + taskId: expect.any(Number), + inputs: [submission.id], + reason: "faulted", + detail: "Value contains a non-JSON function; expected strict JSON", + }, + }); expect((await allEntries(reopened.root)).map((entry) => entry.kind)).toEqual(["pi.user", "pi.assistant"]); await reopened.harness.close(context); }); @@ -1757,25 +1769,12 @@ describe("recovery through ownership edges", () => { }); }); -describe("tool rounds and events", () => { - async function listen(harness: Harness, conversation: Conversation) { - const stream = await watchEvents(harness, conversation.id, context); - const events: AgentEvent[] = []; - stream.start(async ({ value: batch }) => { - events.push(...batch); - }); - return { stream, events }; - } - - /** Event types, with tool ends and message ends labelled by call ID and whether an entry came along. */ - function labels(events: readonly AgentEvent[]): string[] { - return events.flatMap((event) => { - if (event.type === "tool_execution_end") return [`end:${event.toolCallId}:${event.entry !== undefined}`]; - if (event.type === "message_end") { - const message = event.entry.model?.[0]; - return [message?.role === "toolResult" ? `result:${message.toolCallId}` : `message:${message?.role}`]; - } - return []; +describe("tool rounds as the view shows them", () => { + /** The tool result entries of a view, by call ID, in transcript order. */ + function resultsOf(view: ConversationView): string[] { + return view.entries.flatMap((entry) => { + const message = entry.model?.[0]; + return message?.role === "toolResult" ? [message.toolCallId] : []; }); } @@ -1786,25 +1785,36 @@ describe("tool rounds and events", () => { addTool(setup.registry, defineTool({ ...blockingTool("two").registration, name: "two" })); setup.faux.setResponses([toolCalls(["one", "c1"], ["two", "c2"], ["two", "c3"])]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const { stream, events } = await listen(harness, root); + const history = await viewHistory(harness, root.id); const submission = await root.submit({ type: "input", content: "go" }, context); await one.started; await harness.abortTask((await harness.snapshot(LiveDoc, root.id, context))!.run!.taskId, context); await submission.wait(context); - await waitFor(() => labels(events).includes("result:c3")); - expect(labels(events).filter((label) => !label.startsWith("message:"))).toEqual([ - "end:c1:true", - "result:c1", - "end:c2:true", - "result:c2", - "end:c3:true", - "result:c3", + history.detach(); + // The started call ends with its result entry; the commit that ends the run appends the unstarted calls' results, + // in call order, and removes the round. + const states = history.live(); + const c1Done = slotsOf(states, "c1").find((slot) => slot?.status === "done"); + const last = history.views.at(-1)!; + expect(resultsOf(last)).toEqual(["c1", "c2", "c3"]); + const c1Result = last.entries.find( + (entry) => + entry.model?.[0]?.role === "toolResult" && (entry.model[0] as ToolResultMessage).toolCallId === "c1", + ); + expect(c1Done?.entry).toBe(c1Result?.id); + const ending = history.views.findIndex((view) => resultsOf(view).includes("c2")); + expect(resultsOf(history.views[ending]!)).toEqual(["c1", "c2", "c3"]); + expect(states[ending]!.run).toBeUndefined(); + expect(states[ending]!.tools).toBeUndefined(); + expect(states[ending - 1]!.tools?.map((slot) => [slot.callId, slot.status])).toEqual([ + ["c1", "done"], + ["c2", "pending"], + ["c3", "pending"], ]); - await stream.stop(); await harness.close(context); }); - it("emits one turn_end per generation, also for a stream attached while it holds", async () => { + it("shows a held generation as completing to an observer attached while it holds", async () => { const setup = chatSetup(); addTask(setup.registry, Node); addTool( @@ -1825,19 +1835,22 @@ describe("tool rounds and events", () => { setup.faux.setResponses([toolCalls(["noop", "c1"]), fauxAssistantMessage([fauxText("done")])]); const { harness, root } = await openChat(new MemoryStorage(), setup); opened = harness; - const early = await listen(harness, root); + const early = await graphHistory(harness); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); - const late = await listen(harness, root); + const late = await graphHistory(harness); open("hooked"); const [first] = (await harness.commit((tx) => tx.scanTasks({ kind: "pi.generation" }, 1), context)).items; await harness.waitForTask(first!.id, context); - await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); - await waitFor(() => late.events.some((event) => event.type === "entry_appended")); - const turns = (events: readonly AgentEvent[]) => events.filter((event) => event.type === "turn_end").length; - expect(turns(early.events)).toBe(2); - expect(turns(late.events)).toBe(0); - await early.stream.stop(); - await late.stream.stop(); + // The early observer saw both generations; the late one starts with the held one, completing, and sees it leave. + const seen = (graphs: readonly TaskGraph[]) => + new Set(graphs.flatMap((graph) => generations(graph).map((node) => node.id))); + expect(seen(early.graphs).size).toBe(2); + expect(generations(late.graphs[0]!)).toEqual([ + expect.objectContaining({ id: first!.id, state: expect.objectContaining({ status: "completing" }) }), + ]); + expect(late.graphs.at(-1)!.tasks[String(first!.id)]).toBeUndefined(); + early.detach(); + late.detach(); await harness.close(context); }); @@ -1861,7 +1874,6 @@ describe("tool rounds and events", () => { ); setup.faux.setResponses([toolCalls(["delegate", "c1"]), fauxAssistantMessage([fauxText("done")])]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const { stream, events } = await listen(harness, root); const submission = await root.submit({ type: "input", content: "go" }, context); await waitFor(async () => (await harness.snapshot(LiveDoc, root.id, context))?.tools?.[0]?.status === "done"); const live = (await harness.snapshot(LiveDoc, root.id, context))!; @@ -1869,16 +1881,14 @@ describe("tool rounds and events", () => { expect(live.tools![0]!.entry).toBeDefined(); expect((await state(harness, tool)).status).toBe("completing"); expect((await state(harness, live.run!.taskId)).status).toBe("waiting"); - await waitFor(() => labels(events).includes("end:c1:true")); expect(await settled(submission.wait(context))).toBe(false); open("sub"); expect((await submission.wait(context)).status).toBe("done"); expect(await outcomeOf(harness, tool)).toBe("completed"); - await stream.stop(); await harness.close(context); }); - it("holds a faulted tool's slot and task_failed until the work it owns drained", async () => { + it("holds a faulted tool's slot and its graph node until the work it owns drained", async () => { const setup = chatSetup(); addTask(setup.registry, Node); script("held", { @@ -1901,20 +1911,26 @@ describe("tool rounds and events", () => { ); setup.faux.setResponses([toolCalls(["broken", "c1"]), fauxAssistantMessage([fauxText("done")])]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const { stream, events } = await listen(harness, root); + const history = await viewHistory(harness, root.id); + const graph = await graphHistory(harness); const submission = await root.submit({ type: "input", content: "go" }, context); await waitFor(() => log.includes("abort:held")); const live = (await harness.snapshot(LiveDoc, root.id, context))!; const tool = live.tools![0]!.taskId!; expect(await state(harness, tool)).toMatchObject({ status: "completing", outcome: { status: "faulted" } }); expect(live.tools![0]!.status).not.toBe("done"); - expect(events.some((event) => event.type === "task_failed")).toBe(false); + // The graph shows the held fault; the task is not terminal yet. + expect(graph.graphs.at(-1)!.tasks[String(tool)]?.state).toEqual({ status: "completing", outcome: "faulted" }); open("abort.held"); expect((await submission.wait(context)).status).toBe("done"); expect(await outcomeOf(harness, tool)).toBe("faulted"); - await waitFor(() => events.some((event) => event.type === "task_failed")); - expect(labels(events)).toContain("end:c1:false"); - await stream.stop(); + await waitFor(() => graph.graphs.at(-1)!.tasks[String(tool)] === undefined); + // The slot ends done without a result entry. + const done = slotsOf(history.live(), "c1").find((slot) => slot?.status === "done"); + expect(done).toBeDefined(); + expect(done).not.toHaveProperty("entry"); + history.detach(); + graph.detach(); await harness.close(context); }); }); diff --git a/packages/durable/test/harness-view-run.test.ts b/packages/durable/test/harness-view-run.test.ts new file mode 100644 index 000000000..05d9635f3 --- /dev/null +++ b/packages/durable/test/harness-view-run.test.ts @@ -0,0 +1,607 @@ +// What a renderer sees of a run through the conversation view: the transcript window and `pi.live`, every change a +// commit makes, rebuilt from operations alone. This replaced the event stream (`watchEvents`, #11327). +import { GCProfiler, getHeapStatistics, setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import type { Draft } from "@earendil-works/chord"; +import { applyImmutable, type Op } from "@earendil-works/chord/delta"; +import { + type AssistantMessage, + type FauxResponseStep, + fauxAssistantMessage, + fauxText, + fauxThinking, + fauxToolCall, + Type, +} from "@earendil-works/pi-ai"; +import { + type Conversation, + type ConversationView, + defineTool, + type Harness, + LiveDoc, + type LiveState, + MemoryStorage, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import { type ChatSetup, chatSetup, openChat, textOf, waitFor } from "./chat-support.ts"; +import { addTool } from "./harness-support.ts"; +import { slotsOf, viewHistory } from "./live-support.ts"; +import { context, documentChanges, observeCommits } from "./session-support.ts"; +import { aborted, deferred } from "./task-support.ts"; + +type Frame = { value: ConversationView; ops: readonly Op[] }; + +/** + * Start a watch of `conversation` that records every call and, with `check`, checks that its operations turn the value + * before it into its value. The frames are what a renderer that keeps up gets. + */ +async function listen(conversation: Conversation, check = true) { + const watch = await conversation.watch(context); + const frames: Frame[] = []; + let replayed = watch.value; + let mismatches = 0; + watch.start(async ({ value, ops }) => { + if (check) { + replayed = applyImmutable(replayed, ops); + if (JSON.stringify(replayed) !== JSON.stringify(value)) mismatches++; + } + frames.push({ value, ops }); + }); + return { watch, frames, mismatches: () => mismatches }; +} + +const liveOf = (view: ConversationView) => (view.docs["pi.live"] ?? {}) as LiveState; + +async function drained(): Promise { + await new Promise((resolve) => setTimeout(resolve, 0)); +} + +function slow(): ChatSetup { + return chatSetup({ tokensPerSecond: 400, tokenSize: { min: 1, max: 1 } }); +} + +/** Committed partials of the Harness's generations, one per commit that has one. */ +function partialsOf(harness: Harness): AssistantMessage[] { + const partials: AssistantMessage[] = []; + observeCommits(harness, (publication) => { + for (const change of documentChanges(publication)) { + const message = (change.value as LiveState | null)?.generation?.message as AssistantMessage | undefined; + if (change.record.kind === "pi.live" && message !== undefined) partials.push(message); + } + }); + return partials; +} + +/** The distinct in-flight partials in `states`, in order. */ +function partialsIn(states: readonly LiveState[]): AssistantMessage[] { + const partials: AssistantMessage[] = []; + for (const state of states) { + const message = state.generation?.message as AssistantMessage | undefined; + if (message !== undefined && message !== partials.at(-1)) partials.push(message); + } + return partials; +} + +describe("a run as the conversation view shows it", () => { + it("streams the answer as operations that rebuild every committed partial, then appends it", async () => { + const setup = slow(); + const text = "streamed answer text ".repeat(20); + setup.faux.setResponses([fauxAssistantMessage([fauxText(text)])]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const partials = partialsOf(harness); + const history = await viewHistory(harness, root.id); + const { watch, mismatches } = await listen(root); + const submission = await root.submit({ type: "input", content: "hi" }, context); + await submission.wait(context); + await drained(); + history.detach(); + expect(mismatches()).toBe(0); + const states = history.live(); + expect(partials.length).toBeGreaterThan(1); + expect(partialsIn(states).map(textOf)).toEqual(partials.map(textOf)); + // The run shows from the input's commit until the commit that appends the answer and clears the partial. + const started = states.findIndex((state) => state.run !== undefined); + expect(states[started]!.run).toEqual({ taskId: expect.any(Number), inputs: [submission.id] }); + expect(history.views[started]!.entries.at(-1)!.kind).toBe("pi.user"); + const ended = states.findIndex((state, index) => index > started && state.run === undefined); + expect(states[ended]!.generation).toBeUndefined(); + expect(textOf(history.views[ended]!.entries.at(-1)!.model![0])).toBe(text); + expect(states.at(-1)).toEqual({}); + await watch.stop(); + await harness.close(context); + }); + + it("shows a tool call's slot from pending through its running output to its result entry", async () => { + const setup = chatSetup(); + const gate = deferred(); + addTool( + setup.registry, + defineTool({ + name: "print", + description: "Prints", + parameters: Type.Object({ n: Type.Number() }), + execute: async (_args, api) => { + api.output("one\n"); + await new Promise((resolve) => setTimeout(resolve, 150)); + api.output("two\n"); + await gate.promise; + return {}; + }, + }), + ); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("print", { n: 1 }, { id: "c1" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("done")]), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const history = await viewHistory(harness, root.id); + const submission = await root.submit({ type: "input", content: "go" }, context); + await waitFor(() => slotsOf(history.live(), "c1").at(-1)?.output === "one\ntwo\n"); + gate.resolve(); + await submission.wait(context); + history.detach(); + const slots = slotsOf(history.live(), "c1"); + const listed = slots.findIndex((slot) => slot !== undefined); + // Listed pending with the tool-calling answer, in the same commit. + expect(slots[listed]).toEqual({ callId: "c1", name: "print", status: "pending", taskId: expect.any(Number) }); + expect(history.views[listed]!.entries.at(-1)!.kind).toBe("pi.assistant"); + const outputs = slots.flatMap((slot) => (slot?.status === "running" ? [slot.output ?? ""] : [])); + expect([...new Set(outputs)]).toEqual(["", "one\n", "one\ntwo\n"]); + // Done with its result entry, appended in the same commit. + const done = slots.findIndex((slot) => slot?.status === "done"); + expect(slots[done]).toEqual({ + callId: "c1", + name: "print", + status: "done", + taskId: expect.any(Number), + entry: history.views[done]!.entries.at(-1)!.id, + }); + expect(history.views[done]!.entries.at(-1)!.model![0]).toMatchObject({ role: "toolResult", toolCallId: "c1" }); + // The run's two generations, the tool round and the answer, hand the run over. + const runTasks = new Set(history.live().flatMap((state) => (state.run === undefined ? [] : [state.run.taskId]))); + expect(runTasks.size).toBe(2); + await harness.close(context); + }); + + it("shows queued inputs in the inbox, a retry backoff, and the next run", async () => { + const setup = chatSetup(); + const release = deferred(); + const held: FauxResponseStep = async (_request, options) => { + await Promise.race([release.promise, aborted(options!.signal!)]); + return fauxAssistantMessage([fauxText("first")]); + }; + const error = fauxAssistantMessage([], { stopReason: "error", errorMessage: "503 Service Unavailable" }); + setup.faux.setResponses([held, error, fauxAssistantMessage([fauxText("second")])]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + setup.settings.retry = { enabled: true, maxRetries: 1, baseDelayMs: 1 }; + const history = await viewHistory(harness, root.id); + const first = await root.submit({ type: "input", content: "a" }, context); + const followUp = await root.submit({ type: "input", content: "f" }, context); + const inbox = (view: ConversationView) => + (view.docs["pi.inbox"] as { items: { id: number; mode: string }[] }).items; + expect(inbox(history.views.at(-1)!)).toEqual([expect.objectContaining({ id: followUp.id, mode: "followUp" })]); + release.resolve(); + await followUp.wait(context); + history.detach(); + const states = history.live(); + expect(states.some((state) => state.generation?.retry?.error === "503 Service Unavailable")).toBe(true); + const retried = states.findIndex((state) => state.generation?.retry !== undefined); + expect(states.slice(retried).some((state) => state.generation?.attempt === 2)).toBe(true); + const runs = states.flatMap((state, index) => + state.run !== undefined && state.run.inputs !== states[index - 1]?.run?.inputs ? [state.run.inputs] : [], + ); + expect(runs.at(0)).toEqual([first.id]); + expect(runs.at(-1)).toEqual([followUp.id]); + expect(inbox(history.views.at(-1)!)).toEqual([]); + await harness.close(context); + }); + + it("rebuilds every committed partial of thinking, text, and tool-call arguments from the view's operations", async () => { + const setup = chatSetup({ tokensPerSecond: 150, tokenSize: { min: 1, max: 1 } }); + const message = fauxAssistantMessage( + [ + fauxThinking("thinking about it ".repeat(10)), + fauxText("some text ".repeat(10)), + fauxToolCall("missing", { path: "a/long/path/".repeat(10), note: "x".repeat(60) }, { id: "c1" }), + ], + { stopReason: "toolUse" }, + ); + setup.faux.setResponses([message, fauxAssistantMessage([fauxText("done")])]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const partials = partialsOf(harness); + const views: ConversationView[] = []; + const ops: Op[][] = []; + const observation = await harness.observeConversation( + root.id, + { + advance: (change) => { + views.push(change.value); + ops.push([...change.ops]); + }, + end: () => {}, + }, + context, + ); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + observation.detach(); + // Replaying the operations alone, from the observation's value, gives every committed partial. + let replayed = observation.value; + const rebuilt: unknown[] = []; + for (const batch of ops) { + replayed = applyImmutable(replayed, batch); + const partial = liveOf(replayed).generation?.message; + if (partial !== undefined && partial !== rebuilt.at(-1)) rebuilt.push(partial); + } + expect(replayed).toEqual(views.at(-1)); + expect(partials.length).toBeGreaterThan(2); + expect(rebuilt.map((partial) => (partial as AssistantMessage).content)).toEqual( + partials.map((partial) => partial.content), + ); + // Growing text and thinking travel as appends. + const appends = ops.flat().filter((op) => op[0] === "a" && op[1][2] === "generation"); + expect(appends.length).toBeGreaterThan(2); + await harness.close(context); + }); + + it("rebuilds a sliding tail window from output trims and appends", async () => { + const setup = chatSetup(); + const gate = deferred(); + addTool( + setup.registry, + defineTool({ + name: "tail", + description: "Prints lines", + parameters: Type.Object({}), + outputLimits: { maxLines: 3, retain: "tail" }, + execute: async (_args, api) => { + for (let line = 0; line < 6; line++) { + api.output(`line ${line}\n`); + await new Promise((resolve) => setTimeout(resolve, 120)); + } + await gate.promise; + return {}; + }, + }), + ); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("tail", {}, { id: "c1" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("done")]), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const outputs: string[] = []; + observeCommits(harness, (publication) => { + for (const change of documentChanges(publication)) { + const output = (change.value as LiveState | null)?.tools?.[0]?.output; + if (change.record.kind === "pi.live" && output !== undefined && output !== outputs.at(-1)) { + outputs.push(output); + } + } + }); + const { watch, frames, mismatches } = await listen(root); + const submission = await root.submit({ type: "input", content: "go" }, context); + await waitFor(() => outputs.at(-1)?.endsWith("line 5\n") === true); + gate.resolve(); + await submission.wait(context); + await drained(); + await watch.stop(); + expect(mismatches()).toBe(0); + const shown: string[] = []; + for (const frame of frames) { + const output = liveOf(frame.value).tools?.[0]?.output; + if (output !== undefined && output !== shown.at(-1)) shown.push(output); + } + expect(shown).toEqual(outputs); + expect(frames.some((frame) => frame.ops.some((op) => op[0] === "t"))).toBe(true); + await harness.close(context); + }); + + it("ends a run and starts the queued follow-up's in one value", async () => { + const setup = chatSetup(); + const release = deferred(); + const held: FauxResponseStep = async (_request, options) => { + await Promise.race([release.promise, aborted(options!.signal!)]); + return fauxAssistantMessage([fauxText("first")]); + }; + setup.faux.setResponses([held, fauxAssistantMessage([fauxText("second")])]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const input = await root.submit({ type: "input", content: "a" }, context); + const followUp = await root.submit({ type: "input", content: "f" }, context); + const history = await viewHistory(harness, root.id); + release.resolve(); + await followUp.wait(context); + history.detach(); + const states = history.live(); + const boundary = states.findIndex((state) => state.run?.inputs[0] === followUp.id); + expect(states[boundary - 1]!.run?.inputs).toEqual([input.id]); + const before = history.views[boundary - 1]!; + const after = history.views[boundary]!; + const appended = after.entries.slice(before.entries.length); + expect(appended.map((entry) => [entry.kind, textOf(entry.model?.[0])])).toEqual([ + ["pi.assistant", "first"], + ["pi.user", "f"], + ]); + expect((after.docs["pi.inbox"] as { items: unknown[] }).items).toEqual([]); + await harness.close(context); + }); + + it("ends a call that never runs with its result at once, and a tool aborted with its generation", async () => { + const setup = chatSetup(); + addTool( + setup.registry, + defineTool({ + name: "wait", + description: "Waits until aborted", + parameters: Type.Object({}), + execute: (_args, _api, callContext) => aborted(callContext.abortSignal!), + }), + ); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("ghost", {}, { id: "c1" }), fauxToolCall("wait", {}, { id: "c2" })], { + stopReason: "toolUse", + }), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const history = await viewHistory(harness, root.id); + const submission = await root.submit({ type: "input", content: "go" }, context); + await waitFor(() => slotsOf(history.live(), "c2").some((slot) => slot?.status === "running")); + // Aborting the generation aborts its round: the tool ends with its aborted result first (spec §8.5). + await harness.abortTask((await harness.snapshot(LiveDoc, root.id, context))!.run!.taskId, context); + await submission.wait(context); + history.detach(); + const states = history.live(); + // The call not offered is listed done, with its result entry, in the commit of its calling answer. + const listed = slotsOf(states, "c1").findIndex((slot) => slot !== undefined); + const ghost = slotsOf(states, "c1")[listed]!; + expect(ghost.status).toBe("done"); + const listing = history.views[listed]!.entries; + expect(listing.map((entry) => entry.kind).slice(-2)).toEqual(["pi.assistant", "pi.tool-result"]); + expect(ghost.entry).toBe(listing.at(-1)!.id); + // The aborted tool ends with its result entry, then the run ends without an answer. + const done = slotsOf(states, "c2").findIndex((slot) => slot?.status === "done"); + expect(slotsOf(states, "c2")[done]!.entry).toBe(history.views[done]!.entries.at(-1)!.id); + expect(states.at(-1)!.runFailure).toMatchObject({ inputs: [submission.id], reason: "aborted" }); + await harness.close(context); + }); + + it("keeps a steer in the current run, shows nothing for other conversations, and appends a reset", async () => { + const setup = chatSetup(); + const gate = deferred(); + addTool( + setup.registry, + defineTool({ + name: "hold", + description: "Waits", + parameters: Type.Object({}), + execute: async () => { + await gate.promise; + return {}; + }, + }), + ); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("hold", {}, { id: "c1" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("done")]), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const other = await harness.createConversation({ ownership: { kind: "ownerless" } }, context); + const history = await viewHistory(harness, root.id); + const input = await root.submit({ type: "input", content: "a" }, context); + await waitFor(() => slotsOf(history.live(), "c1").some((slot) => slot?.status === "running")); + const steer = await root.submit({ type: "input", content: "s", whenBusy: "steer" }, context); + const count = history.views.length; + await other.commit((tx) => tx.appendEntry(other.id, { kind: "note" }), context); + expect(history.views.length).toBe(count); + gate.resolve(); + await input.wait(context); + await root.reset(undefined, context); + history.detach(); + const states = history.live(); + // One run: the steer joined it. + const runs = new Set(states.flatMap((state) => (state.run === undefined ? [] : [state.run.inputs[0]]))); + expect([...runs]).toEqual([input.id]); + expect(states.some((state) => state.run?.inputs.includes(steer.id))).toBe(true); + // The reset is appended in place, with no run. + expect(history.views.at(-1)!.entries.at(-1)!.kind).toBe("pi.reset"); + expect(states.at(-1)!.run).toBeUndefined(); + await harness.close(context); + }); + + it("converts the streamed partial into one aborted answer entry when the run is aborted", async () => { + const setup = chatSetup({ tokensPerSecond: 100, tokenSize: { min: 1, max: 1 } }); + setup.faux.setResponses([fauxAssistantMessage([fauxText("x".repeat(400))])]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const history = await viewHistory(harness, root.id); + const submission = await root.submit({ type: "input", content: "hi" }, context); + await waitFor(() => history.live().some((state) => state.generation?.message !== undefined)); + const live = await harness.snapshot(LiveDoc, root.id, context); + await harness.abortTask(live!.run!.taskId, context); + await submission.wait(context); + history.detach(); + const states = history.live(); + const ended = states.findIndex((state, index) => index > 0 && state.run === undefined); + expect(states[ended - 1]!.generation?.message).toBeDefined(); + const appended = history.views[ended]!.entries.slice(history.views[ended - 1]!.entries.length); + expect(appended.map((entry) => entry.kind)).toEqual(["pi.assistant"]); + expect(appended[0]!.model![0]).toMatchObject({ role: "assistant", stopReason: "aborted" }); + expect(states[ended]!.runFailure).toMatchObject({ reason: "aborted" }); + await harness.close(context); + }); + + it("applies later operations to an overflow replacement that holds an in-flight partial", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + const partial = fauxAssistantMessage([fauxText("hel")]); + const watch = await root.watch(context, { min: 1, max: 1 }); + const change = (edit: (live: Draft) => void) => + root.commit(async (tx) => edit(await tx.doc(LiveDoc, root.id)), context); + await change((live) => { + live.generation = { attempt: 1, message: JSON.parse(JSON.stringify(partial)) }; + }); + // Past 10,000 pending operations: an append, a splice, and a `before` per commit. + for (let index = 0; index < 3400; index++) { + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + } + const frames: Frame[] = []; + watch.start(async ({ value, ops }) => { + frames.push({ value, ops }); + }); + await drained(); + expect(frames[0]!.ops).toEqual([["r", frames[0]!.value]]); + await change((live) => { + (live.generation!.message!.content[0] as { text: string }).text += "lo"; + }); + await drained(); + const update = frames.at(-1)!; + expect(update.ops).toEqual([["a", ["docs", "pi.live", "generation", "message", "content", 0, "text"], "lo"]]); + const rebuilt = applyImmutable(frames[0]!.value, update.ops); + expect(liveOf(rebuilt).generation?.message?.content).toEqual([{ type: "text", text: "hello" }]); + await watch.stop(); + await harness.close(context); + }, 30_000); +}); + +const gc = (() => { + setFlagsFromString("--expose-gc"); + return runInNewContext("gc") as () => void; +})(); + +/** Bytes `run` allocates, counting what collections during it freed. */ +async function allocatedBytes(run: () => Promise): Promise { + gc(); + const profiler = new GCProfiler(); + profiler.start(); + const before = getHeapStatistics().used_heap_size; + await run(); + const after = getHeapStatistics().used_heap_size; + let freed = 0; + for (const collection of profiler.stop().statistics) { + freed += collection.beforeGC.heapStatistics.usedHeapSize - collection.afterGC.heapStatistics.usedHeapSize; + } + return after - before + freed; +} + +/** Heap bytes in use after a full collection. */ +async function heapInUse(): Promise { + for (let round = 0; round < 3; round++) { + await drained(); + gc(); + } + return getHeapStatistics().used_heap_size; +} + +// #9265 +describe("streamed partials", () => { + it("streams a tool-call partial as appends at its argument paths", async () => { + const setup = chatSetup({ tokensPerSecond: 500, tokenSize: { min: 1, max: 1 } }); + setup.settings.progress = { partialIntervalMs: 20, outputIntervalMs: 100 }; + const args = { path: "src/file.ts", content: "const a = 1;\n".repeat(150) }; + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("missing", args, { id: "c1" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("done")]), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const partials = partialsOf(harness); + const ops: Op[] = []; + const observation = await harness.observeConversation( + root.id, + { advance: (change) => ops.push(...change.ops), end: () => {} }, + context, + ); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + observation.detach(); + const MESSAGE = ["docs", "pi.live", "generation", "message"]; + // Operations on the in-flight message, with their paths relative to it. + const onMessage = ops.flatMap((op) => { + const path = op[0] === "r" ? [] : (op[1] as readonly (string | number)[]); + return MESSAGE.every((segment, index) => path[index] === segment) + ? [{ kind: op[0], path: path.slice(MESSAGE.length) }] + : []; + }); + const argumentAppends = onMessage.filter((op) => op.kind === "a" && op.path[2] === "arguments"); + expect(argumentAppends.length).toBeGreaterThan(10); + expect(argumentAppends.every((op) => ["path", "content"].includes(String(op.path.at(-1))))).toBe(true); + // The call is set whole only when a member appears (at most once per argument); growing strings are appends. + // With the raw argument text on the block, every commit resent the whole call. + const blockSets = onMessage.filter((op) => op.kind !== "a" && op.path.length === 2); + expect(blockSets.length).toBeLessThanOrEqual(2); + // The committed partials carry only the parsed arguments, never the raw argument text. + for (const partial of partials) { + for (const block of partial.content) { + expect(Object.keys(block).sort()).toEqual(["arguments", "id", "name", "type"]); + } + } + await harness.close(context); + }); + + it("commits a growing text and tool call at a cost that follows the new text, not the string lengths", async () => { + /** Bytes allocated while streaming one answer of `size` characters of text and as much tool-call content. */ + const stream = async (size: number) => { + const setup = chatSetup({ tokensPerSecond: 64_000, tokenSize: { min: 64, max: 64 } }); + setup.settings.progress = { partialIntervalMs: 1, outputIntervalMs: 100 }; + // Keep `pi.live` cached between commits, as the default retention does. Evicted after each use (the suite's + // evicting run), every commit replays the document's delta chain: a cost of eviction, not of appending. + setup.settings.documentRetentionMs = 60_000; + const text = 'streamed text with "quotes" and \\ escapes\n'.repeat(size / 40); + setup.faux.setResponses([ + fauxAssistantMessage( + [fauxText(text), fauxToolCall("missing", { path: "a.ts", content: text }, { id: "c1" })], + { + stopReason: "toolUse", + }, + ), + fauxAssistantMessage([fauxText("done")]), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + let commits = 0; + observeCommits(harness, () => commits++); + const bytes = await allocatedBytes(async () => { + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + }); + await harness.close(context); + return { bytes, commits }; + }; + const small = await stream(40_000); + const large = await stream(400_000); + expect(large.commits).toBeGreaterThan(small.commits * 5); + // Per commit, the cost stays flat: comparing the whole strings on every commit made it grow with their length. + expect(large.bytes / large.commits / (small.bytes / small.commits)).toBeLessThan(2); + }, 60_000); + + it("keeps a watcher's memory proportional to a long streamed text and tool call, not to their commits", async () => { + const setup = chatSetup({ tokensPerSecond: 16_000, tokenSize: { min: 16, max: 16 } }); + setup.settings.progress = { partialIntervalMs: 1, outputIntervalMs: 100 }; + const line = (index: number) => `line ${index}: some streamed source text with "quotes" and \\ escapes\n`; + const text = Array.from({ length: 2_000 }, (_, index) => line(index)).join(""); + const content = Array.from({ length: 2_000 }, (_, index) => line(index)).join(""); + setup.faux.setResponses([ + fauxAssistantMessage([fauxText(text), fauxToolCall("missing", { path: "a.ts", content }, { id: "c1" })], { + stopReason: "toolUse", + }), + fauxAssistantMessage([fauxText("done")]), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const { watch, frames } = await listen(root, false); + const before = await heapInUse(); + const submission = await root.submit({ type: "input", content: "go" }, context); + const partialContent = async () => + (await harness.snapshot(LiveDoc, root.id, context))?.generation?.message?.content; + const shown = async (index: number): Promise => { + const block = (await partialContent())?.[index]; + if (block?.type === "text") return block.text.length; + return block?.type === "toolCall" ? String(block.arguments.content ?? "").length : 0; + }; + const retained: number[] = []; + // Measure while the text, then the tool call, is almost complete: the watcher's view holds the most then. + await waitFor(async () => (await shown(0)) > text.length * 0.95, 30_000); + retained.push((await heapInUse()) - before); + await waitFor(async () => (await shown(1)) > content.length * 0.95, 30_000); + retained.push((await heapInUse()) - before); + await submission.wait(context); + const commits = frames.length; + expect(commits).toBeGreaterThan(500); + // The two 140 KB strings, the watcher's recorded frames and the rest of the stream: about 3 MB. Sliced append ops + // made the watcher's view hold a copy of the text per commit: well over 100 MB. + for (const bytes of retained) expect(bytes).toBeLessThan(20_000_000); + await watch.stop(); + await harness.close(context); + }, 60_000); +}); diff --git a/packages/durable/test/harness-view.test.ts b/packages/durable/test/harness-view.test.ts index f07ce2bb9..f5a5d7f1b 100644 --- a/packages/durable/test/harness-view.test.ts +++ b/packages/durable/test/harness-view.test.ts @@ -1,10 +1,17 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; import { applyImmutable, type Op } from "@earendil-works/chord/delta"; import { type AssistantMessage, type FauxResponseStep, fauxAssistantMessage, fauxText } from "@earendil-works/pi-ai"; import { AgentDoc, type Conversation, type ConversationView, + type ConversationWindow, defineDoc, + type EntryRecord, type Harness, InboxDoc, LiveDoc, @@ -13,6 +20,7 @@ import { UsageDoc, } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, chatSetup, openChat, waitFor } from "./chat-support.ts"; import { context } from "./session-support.ts"; import { aborted, deferred } from "./task-support.ts"; @@ -22,8 +30,8 @@ type Frame = { value: ConversationView; ops: readonly Op[] }; const MOUNTED = ["pi.agent", "pi.inbox", "pi.live", "pi.provider", "pi.usage"]; /** Start a watch of `conversation` that records its acquisition revision and every delivered frame. */ -async function record(conversation: Conversation) { - const watch = await conversation.watch(context); +async function record(conversation: Conversation, window?: ConversationWindow) { + const watch = await conversation.watch(context, window); const initial = watch.value; const frames: Frame[] = []; watch.start(async ({ value, ops }) => { @@ -33,21 +41,31 @@ async function record(conversation: Conversation) { } /** A freshly built view of `conversation`, for comparing with an advanced one. */ -async function fresh(conversation: Conversation): Promise { - const watch = await conversation.watch(context); +async function fresh(conversation: Conversation, window?: ConversationWindow): Promise { + const watch = await conversation.watch(context, window); const value = watch.value; await watch.stop(); return value; } -/** The view as committed state defines it, read without any mount: the active entries and the built-in documents. */ -async function committed(harness: Harness, conversation: Conversation, record: ConversationView["conversation"]) { +/** + * The view as committed state defines it, read without any mount: the newest `size` entries, the entry before them, and + * the built-in documents. + */ +async function committed( + harness: Harness, + conversation: Conversation, + record: ConversationView["conversation"], + size: number, +) { const docs: Record = {}; for (const token of [AgentDoc, LiveDoc, InboxDoc, ProviderDoc, UsageDoc] as const) { const value = await harness.snapshot(token as typeof LiveDoc, conversation.id, context); if (value !== undefined) docs[token.definition.kind] = value; } - return { conversation: record, entries: (await conversation.context(context)).entries, docs }; + const newest = [...(await conversation.entries({}, size + 1, undefined, context)).items]; + const before = newest.length > size ? newest.pop()!.id : undefined; + return { conversation: record, entries: newest.reverse(), ...(before === undefined ? {} : { before }), docs }; } /** Replay every frame's operations from `initial`, checking each delivered revision on the way. */ @@ -86,7 +104,7 @@ function touches(harness: Harness, conversation: Conversation): { count: number } describe("conversation view", () => { - it("hydrates the active entries and the built-in documents", async () => { + it("hydrates the newest entries and the built-in documents", async () => { const setup = chatSetup(); setup.faux.setResponses([answer("hello")]); const { harness, root } = await openChat(new MemoryStorage(), setup); @@ -94,6 +112,7 @@ describe("conversation view", () => { const view = await fresh(root); expect(view.conversation).toEqual({ id: root.id }); expect(view.entries).toEqual(await allEntries(root)); + expect(view.before).toBeUndefined(); expect(Object.keys(view.docs).sort()).toEqual(MOUNTED); expect(view.docs["pi.live"]).toEqual({}); expect(view.docs["pi.inbox"]).toEqual({ items: [] }); @@ -118,7 +137,8 @@ describe("conversation view", () => { await harness.waitForIdle(context); await drained(); expect(frames).toHaveLength(touching.count); - expect(replay(initial, frames)).toEqual(await committed(harness, root, initial.conversation)); + const last = replay(initial, frames); + expect(last).toEqual(await committed(harness, root, initial.conversation, last.entries.length)); expect(frames[0]!.ops).toContainEqual(["p", ["entries"], 0, 0, [expect.objectContaining({ kind: "pi.user" })]]); // Document operations keep their exact shape under the mount path. expect(frames.flatMap((frame) => frame.ops)).toContainEqual([ @@ -148,7 +168,7 @@ describe("conversation view", () => { await harness.close(context); }); - it("cuts the entries at a head marker, keeping the entries from its head", async () => { + it("keeps head markers in place: a summary and a reset are appended entries", async () => { const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); const note = (kind: string) => root.commit((tx) => tx.appendEntry(root.id, { kind }), context); await note("a"); @@ -160,28 +180,21 @@ describe("conversation view", () => { await root.reset(undefined, context); await drained(); const kinds = frames.map((frame) => frame.value.entries.map((entry) => entry.kind)); - expect(kinds).toEqual([["summary", "b", "c"], ["summary", "b", "c", "d"], ["pi.reset"]]); - expect(frames[0]!.ops).toEqual([["p", ["entries"], 0, 1, [summary]]]); - expect(replay(initial, frames)).toEqual(await committed(harness, root, initial.conversation)); + expect(kinds).toEqual([ + ["a", "b", "c", "summary"], + ["a", "b", "c", "summary", "d"], + ["a", "b", "c", "summary", "d", "pi.reset"], + ]); + expect(frames[0]!.ops).toEqual([["p", ["entries"], 3, 0, [summary]]]); + // The summary's `head` tells a renderer where the entries it summarizes end. + expect(frames[0]!.value.entries[3]!.head).toBe(b.id); + const last = replay(initial, frames); + expect(last).toEqual(await committed(harness, root, initial.conversation, 6)); await stop(); await harness.close(context); }); - it("keeps only mounted entries for a raw head write that targets before the active range", async () => { - const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); - const old = await root.commit((tx) => tx.appendEntry(root.id, { kind: "old" }), context); - await root.reset(undefined, context); - const { frames, stop } = await record(root); - await root.commit((tx) => tx.appendEntry(root.id, { kind: "summary", head: old.id }), context); - await drained(); - // Model context now starts at `old` again, but the mount never held it (spec §12); a rebuilt mount shows it. - expect(frames[0]!.value.entries.map((entry) => entry.kind)).toEqual(["summary"]); - await stop(); - expect((await fresh(root)).entries.map((entry) => entry.kind)).toEqual(["summary", "old"]); - await harness.close(context); - }); - - it("cuts a fork's view into its inherited entries", async () => { + it("shows a fork's inherited entries in ID order, its head markers in place", async () => { const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); const a = await root.commit((tx) => tx.appendEntry(root.id, { kind: "a" }), context); const b = await root.commit((tx) => tx.appendEntry(root.id, { kind: "b" }), context); @@ -190,8 +203,8 @@ describe("conversation view", () => { await fork.commit((tx) => tx.appendEntry(fork.id, { kind: "summary", head: b.id }), context); await drained(); expect(initial.entries.map((entry) => entry.id)).toEqual([a.id, b.id]); - expect(frames[0]!.value.entries.map((entry) => entry.kind)).toEqual(["summary", "b"]); - expect(frames[0]!.value).toEqual(await committed(harness, fork, initial.conversation)); + expect(frames[0]!.value.entries.map((entry) => entry.kind)).toEqual(["a", "b", "summary"]); + expect(frames[0]!.value).toEqual(await committed(harness, fork, initial.conversation, 3)); await stop(); await harness.close(context); }); @@ -243,7 +256,8 @@ describe("conversation view", () => { expect(frames).toHaveLength(1); expect(frames[0]!.ops[0]?.[0]).not.toBe("r"); expect(applyImmutable(initial, frames[0]!.ops)).toEqual(frames[0]!.value); - expect(frames[0]!.value.entries).toHaveLength(101); + // The default window took 100 appends, then dropped its oldest 51 at the 101st, down to 50. + expect(frames[0]!.value).toEqual(await committed(harness, root, initial.conversation, 50)); await watch.stop(); await harness.close(context); }); @@ -309,7 +323,9 @@ describe("conversation view", () => { const failing = await root.watch(context); const { frames, stop } = await record(root); const shared = await root.watch(context); - expect(failing.value).toBe(shared.value); + // Each observer has its own window; the documents are the mount's. + expect(failing.value).toEqual(shared.value); + expect(failing.value.docs).toBe(shared.value.docs); await shared.stop(); failing.start(async () => { throw new Error("listener failed"); @@ -347,8 +363,223 @@ describe("conversation view", () => { await drained(); expect(frames).toHaveLength(1); expect(frames[0]!.value.entries.map((entry) => entry.kind)).toEqual(["a", "b"]); - expect(replay(initial, frames)).toEqual(await committed(harness, root, initial.conversation)); + expect(replay(initial, frames)).toEqual(await committed(harness, root, initial.conversation, 2)); await stop(); await harness.close(context); }); }); + +/** Append one entry of `kind` per commit; returns the entries. */ +async function notes(conversation: Conversation, count: number, kind = "note"): Promise { + const entries: EntryRecord[] = []; + for (let index = 0; index < count; index++) { + entries.push(await conversation.commit((tx) => tx.appendEntry(conversation.id, { kind, data: index }), context)); + } + return entries; +} + +const ids = (entries: readonly EntryRecord[]) => entries.map((entry) => entry.id); + +const gc = (() => { + setFlagsFromString("--expose-gc"); + return runInNewContext("gc") as () => void; +})(); + +describe("conversation view window", () => { + it("starts with the newest `min` entries and the entry before them, or the whole short transcript", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + const written = await notes(root, 10); + const view = await fresh(root, { min: 4 }); + expect(ids(view.entries)).toEqual(ids(written.slice(6))); + expect(view.before).toBe(written[5]!.id); + // The default window: the newest 50, here all ten, starting the transcript. + const all = await fresh(root); + expect(ids(all.entries)).toEqual(ids(written)); + expect(all.before).toBeUndefined(); + expect("before" in all).toBe(false); + await harness.close(context); + }); + + it("takes appends up to `max`, then drops its oldest in one splice down to `min`", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + const written = await notes(root, 4); + const { initial, frames, stop } = await record(root, { min: 3, max: 5 }); + expect(ids(initial.entries)).toEqual(ids(written.slice(1))); + const appended = await notes(root, 3); + await drained(); + expect(frames.map((frame) => frame.value.entries.length)).toEqual([4, 5, 3]); + const [e5, e6, e7] = appended as [EntryRecord, EntryRecord, EntryRecord]; + expect(frames[0]!.ops).toEqual([["p", ["entries"], 3, 0, [e5]]]); + expect(frames[2]!.ops).toEqual([ + ["p", ["entries"], 5, 0, [e7]], + ["p", ["entries"], 0, 3, []], + ["s", ["before"], written[3]!.id], + ]); + expect(ids(frames[2]!.value.entries)).toEqual(ids([e5, e6, e7])); + expect(frames[2]!.value.before).toBe(written[3]!.id); + const last = replay(initial, frames); + expect(last).toEqual(await committed(harness, root, initial.conversation, 3)); + // Entries that stay keep their identity: a splice moves them, it never copies them. + expect(frames[2]!.value.entries[0]).toBe(frames[1]!.value.entries[3]); + await stop(); + await harness.close(context); + }); + + it("keeps the newest `min` of a commit that appends more than `max` at once", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + await notes(root, 2); + const { initial, frames, stop } = await record(root, { min: 2, max: 3 }); + const burst = await root.commit(async (tx) => { + const entries: EntryRecord[] = []; + for (let index = 0; index < 5; index++) entries.push(await tx.appendEntry(root.id, { kind: "burst" })); + return entries; + }, context); + await drained(); + expect(frames).toHaveLength(1); + expect(ids(frames[0]!.value.entries)).toEqual(ids(burst.slice(3))); + expect(frames[0]!.value.before).toBe(burst[2]!.id); + expect(replay(initial, frames)).toEqual(await committed(harness, root, initial.conversation, 2)); + await stop(); + await harness.close(context); + }); + + it("starts at `from` when the entries since fit in `max`, otherwise at the newest `min`", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + const written = await notes(root, 30); + const window = { min: 5, max: 10 }; + // Ten entries since `from`: all of them, with the entry before `from`. + const within = await fresh(root, { ...window, from: written[20]!.id }); + expect(ids(within.entries)).toEqual(ids(written.slice(20))); + expect(within.before).toBe(written[19]!.id); + // Fewer than `min` since `from`: only those. + const short = await fresh(root, { ...window, from: written[28]!.id }); + expect(ids(short.entries)).toEqual(ids(written.slice(28))); + expect(short.before).toBe(written[27]!.id); + // Resuming after the newest entry: an empty window right after it. + const resumed = await fresh(root, { ...window, from: (written[29]!.id + 1) as EntryRecord["id"] }); + expect(resumed.entries).toEqual([]); + expect(resumed.before).toBe(written[29]!.id); + // Eleven since `from`: more than `max`, so the newest `min`; `before` shows the client a gap. + const beyond = await fresh(root, { ...window, from: written[19]!.id }); + expect(ids(beyond.entries)).toEqual(ids(written.slice(25))); + expect(beyond.before).toBe(written[24]!.id); + // From the transcript's start. + const whole = await fresh(root, { min: 40, from: written[0]!.id }); + expect(ids(whole.entries)).toEqual(ids(written)); + expect(whole.before).toBeUndefined(); + await harness.close(context); + }); + + it("advances a resumed window by appends from its start", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + const written = await notes(root, 5); + const { initial, frames, stop } = await record(root, { from: (written[4]!.id + 1) as EntryRecord["id"] }); + const later = await notes(root, 2); + await drained(); + expect(ids(frames.at(-1)!.value.entries)).toEqual(ids(later)); + expect(frames.at(-1)!.value.before).toBe(written[4]!.id); + expect(replay(initial, frames)).toEqual(frames.at(-1)!.value); + await stop(); + await harness.close(context); + }); + + it("uses `from` only for the first value: a `from` past the newest entry starts empty and takes every append", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + const written = await notes(root, 3); + const from = (written[2]!.id + 1000) as EntryRecord["id"]; + const { initial, frames, stop } = await record(root, { from }); + expect(initial.entries).toEqual([]); + expect(initial.before).toBe(written[2]!.id); + // An entry below `from` is still appended: the window continues from its attachment. + const [next] = await notes(root, 1); + expect(next!.id).toBeLessThan(from); + await drained(); + expect(frames.at(-1)!.value.entries).toEqual([next]); + expect(frames.at(-1)!.value.before).toBe(written[2]!.id); + await stop(); + await harness.close(context); + }); + + it("rejects window bounds that are not integers or put `max` below `min`", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + await expect(root.watch(context, { min: 0 })).rejects.toThrow(RangeError); + await expect(root.watch(context, { min: 1.5 })).rejects.toThrow(RangeError); + await expect(root.watch(context, { min: 10, max: 9 })).rejects.toThrow(RangeError); + await expect(root.watch(context, { from: Number.NaN as EntryRecord["id"] })).rejects.toThrow(RangeError); + const recorder = { advance: () => {}, end: () => {} }; + await expect(harness.observeConversation(root.id, recorder, context, { max: 0 })).rejects.toThrow(RangeError); + // `max` may equal `min`. + await (await root.watch(context, { min: 3, max: 3 })).stop(); + await harness.close(context); + }); + + it("gives each observer its own window over one mount", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + const written = await notes(root, 6); + const changes: ConversationView[] = []; + const observation = await harness.observeConversation( + root.id, + { advance: ({ value }) => changes.push(value), end: () => {} }, + context, + { min: 2, max: 3 }, + ); + const wide = await record(root, { min: 5 }); + expect(ids(observation.value.entries)).toEqual(ids(written.slice(4))); + expect(ids(wide.initial.entries)).toEqual(ids(written.slice(1))); + const [next] = await notes(root, 1); + await drained(); + expect(ids(changes[0]!.entries)).toEqual(ids([...written.slice(4), next!])); + expect(ids(wide.frames[0]!.value.entries)).toEqual(ids([...written.slice(1), next!])); + expect(changes[0]!.docs).toBe(wide.frames[0]!.value.docs); + observation.detach(); + await wide.stop(); + await harness.close(context); + }); + + it("bounds an overflow replacement by `max`", async () => { + const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); + const watch = await root.watch(context, { min: 5, max: 6 }); + // Past 10,000 pending operations, the next call is one root replacement: here an append per commit, and a splice + // and a `before` every other commit. + for (let index = 0; index < 6000; index++) { + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + } + const frames: Frame[] = []; + watch.start(async ({ value, ops }) => { + frames.push({ value, ops }); + }); + await drained(); + expect(frames).toHaveLength(1); + expect(frames[0]!.ops).toEqual([["r", frames[0]!.value]]); + expect(frames[0]!.value.entries.length).toBeLessThanOrEqual(6); + await watch.stop(); + await harness.close(context); + }, 30_000); + + it("lets a long-lived watch drop the entries that left its window", async () => { + // On SQLite, which keeps no entry objects, so only the watch can hold the first one. + const directory = await mkdtemp(join(tmpdir(), "pi-durable-view-")); + const storage = await openNodeSqliteStorage(join(directory, "session.sqlite")); + const { harness, root } = await openChat(storage, chatSetup()); + const watch = await root.watch(context, { min: 5, max: 10 }); + let first: WeakRef | undefined; + let count = 0; + watch.start(async ({ value }) => { + count = value.entries.length; + if (first === undefined && value.entries[0] !== undefined) first = new WeakRef(value.entries[0]); + }); + await notes(root, 1); + await waitFor(() => first !== undefined); + await notes(root, 200); + await drained(); + expect(count).toBeLessThanOrEqual(10); + for (let round = 0; round < 3 && first!.deref() !== undefined; round++) { + await drained(); + gc(); + } + expect(first!.deref()).toBeUndefined(); + await watch.stop(); + await harness.close(context); + await rm(directory, { recursive: true, force: true }); + }); +}); diff --git a/packages/durable/test/live-support.ts b/packages/durable/test/live-support.ts new file mode 100644 index 000000000..562d13e56 --- /dev/null +++ b/packages/durable/test/live-support.ts @@ -0,0 +1,110 @@ +// What tests of renderers' state read: every `pi.live` value of a conversation, and the results of nested calls. +import { + type ConversationId, + type ConversationView, + type Harness, + type LiveState, + NestedCallDoc, + type NestedCallState, + type NestedToolExecutionResult, + type NestedToolSlot, + type TaskGraph, + type TaskGraphNode, + type TaskId, + type ToolSlot, +} from "@earendil-works/pi-durable"; +import { context, documentChanges, observeCommits } from "./session-support.ts"; + +/** + * Every view of `conversationId` from now on, one per commit that changes it, unfolded: a synchronous observer, so no + * transition is skipped. `live()` lists their `pi.live` values. + */ +export async function viewHistory(harness: Harness, conversationId: ConversationId) { + const views: ConversationView[] = []; + const observation = await harness.observeConversation( + conversationId, + { advance: ({ value }) => views.push(value), end: () => {} }, + context, + ); + views.push(observation.value); + return { + views, + live: (): LiveState[] => views.map((view) => (view.docs["pi.live"] ?? {}) as LiveState), + detach: () => observation.detach(), + }; +} + +/** A model-issued or nested slot, read through the fields of both. */ +type Slot = ToolSlot & Partial>; + +/** The slots of tool call `callId`, model-issued or nested, in each `pi.live` value; `undefined` where it is absent. */ +export function slotsOf(history: readonly LiveState[], callId: string): (Slot | undefined)[] { + return history.map( + (live) => [...(live.tools ?? []), ...(live.nestedTools ?? [])].find((slot) => slot.callId === callId) as Slot, + ); +} + +/** The index of the first value where call `callId` ended: its slot is done, or gone after it was listed. */ +export function endedAt(history: readonly LiveState[], callId: string): number { + const slots = slotsOf(history, callId); + const listed = slots.findIndex((slot) => slot !== undefined); + if (listed < 0) return -1; + return slots.findIndex((slot, index) => index >= listed && (slot === undefined || slot.status === "done")); +} + +/** The calls whose slot went `running`, in the order they started. */ +export function startOrder(history: readonly LiveState[]): string[] { + const started: string[] = []; + for (const live of history) { + for (const slot of [...(live.tools ?? []), ...(live.nestedTools ?? [])]) { + if (slot.status === "running" && !started.includes(slot.callId)) started.push(slot.callId); + } + } + return started; +} + +/** + * The result of every nested call from now on, by its tool task, read from the `NestedCallDoc` its terminal commit + * writes (spec §7.3), the place a client reads a live caller's nested results from. + */ +export function nestedResults(harness: Harness): Map { + const results = new Map(); + observeCommits(harness, (publication) => { + for (const change of documentChanges(publication)) { + if (change.record.kind !== NestedCallDoc.definition.kind || change.value === null) continue; + const state = change.value as NestedCallState; + if (state.taskId !== undefined && state.result !== undefined) { + results.set(state.taskId, state.result as unknown as NestedToolExecutionResult); + } + } + }); + return results; +} + +/** The result of nested call `callId`, found by the task its slot names. */ +export function nestedResultOf( + history: readonly LiveState[], + results: ReadonlyMap, + callId: string, +): NestedToolExecutionResult | undefined { + const slot = slotsOf(history, callId).find((each) => each?.taskId !== undefined); + return slot?.taskId === undefined ? undefined : results.get(slot.taskId); +} + +/** Every task graph from now on, one per change, unfolded. */ +export async function graphHistory(harness: Harness) { + const graphs: TaskGraph[] = []; + const observation = await harness.observeTaskGraph( + { advance: ({ value }) => graphs.push(value), end: () => {} }, + context, + ); + graphs.push(observation.value); + return { graphs, detach: () => observation.detach() }; +} + +/** The live `pi.generation` nodes of `graph`, in task ID order. */ +export function generations(graph: TaskGraph): TaskGraphNode[] { + return Object.values(graph.tasks) + .filter((node) => node.kind === "pi.generation") + .sort((a, b) => a.id - b.id); +} diff --git a/packages/durable/test/session-document-retention.test.ts b/packages/durable/test/session-document-retention.test.ts index c2cec11c8..56cf9bd7d 100644 --- a/packages/durable/test/session-document-retention.test.ts +++ b/packages/durable/test/session-document-retention.test.ts @@ -3,21 +3,21 @@ import type { Context } from "@earendil-works/chord"; import { fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; import { - type AgentEvent, type ConversationId, defineDoc, defineDocFamily, defineTool, type HarnessSettings, + type LiveState, MemoryStorage, type Seq, type StorageWrite, - watchEvents, } from "@earendil-works/pi-durable"; import { describe, expect, it, onTestFinished } from "vitest"; import { SessionImpl, setDefaultDocumentRetention } from "../src/session/session.ts"; import { chatSetup, openChat } from "./chat-support.ts"; import { addTool } from "./harness-support.ts"; +import { viewHistory } from "./live-support.ts"; import { ControlledStorage, context, createConversation } from "./session-support.ts"; import { deferred, eventually } from "./task-support.ts"; @@ -509,7 +509,7 @@ describe("document cache retention", () => { await harness.close(context); }); - it("gives the same events with documents evicted right after use as with documents kept", async () => { + it("gives the same views with documents evicted right after use as with documents kept", async () => { const run = async (documentRetentionMs: number): Promise => { const setup = chatSetup(); setup.settings.documentRetentionMs = documentRetentionMs; @@ -534,19 +534,18 @@ describe("document cache retention", () => { fauxAssistantMessage([fauxText("done")]), ]); const { harness, root } = await openChat(new MemoryStorage(), setup); - const stream = await watchEvents(harness, root.id, context); - const events: AgentEvent[] = []; - stream.start(async ({ value: batch }) => { - events.push(...batch); - }); + const history = await viewHistory(harness, root.id); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); await harness.waitForIdle(context); - await stream.stop(); + history.detach(); await harness.close(context); - // Ids and times differ between runs; the sequence of event types and tool outputs must not. - return events.map((event) => - event.type === "tool_execution_update" ? `update:${JSON.stringify(event.output)}` : event.type, - ); + // Ids and times differ between runs; the sequence of entries, run states, and tool outputs must not. + return history.views.map((view) => { + const live = (view.docs["pi.live"] ?? {}) as LiveState; + const tools = (live.tools ?? []).map((slot) => `${slot.callId}:${slot.status}:${slot.output ?? ""}`); + const kinds = view.entries.map((entry) => entry.kind).join(","); + return `${kinds} run:${live.run !== undefined} generation:${live.generation !== undefined} ${tools.join(" ")}`; + }); }; expect(await run(0)).toEqual(await run(600_000)); }); diff --git a/packages/durable/test/spec-usage.test.ts b/packages/durable/test/spec-usage.test.ts index 489f48204..5dc763339 100644 --- a/packages/durable/test/spec-usage.test.ts +++ b/packages/durable/test/spec-usage.test.ts @@ -6,7 +6,9 @@ import type { Context, Draft } from "@earendil-works/chord"; import { type AssistantMessage, type Models, type ToolCall, Type } from "@earendil-works/pi-ai"; import { + type Conversation, type ConversationId, + ConversationTranscript, configure, createRegistry, defineDoc, @@ -27,6 +29,7 @@ import { type TaskId, type ToolExecutionApi, ToolTask, + type TranscriptView, type Tx, type UserInput, wrapTool, @@ -75,6 +78,8 @@ declare const manager: SettingsManager; declare const Anchor: ReturnType>; declare const Reporter: ReturnType>; declare const subagentTool: ReturnType; +declare const conversation: Conversation; +declare function render(view: TranscriptView): void; /** Never called: the declared names above have no values. */ function examples() { @@ -351,6 +356,21 @@ function examples() { }); }, + // Section 9.3: a client's transcript over the windowed view. + transcript: async () => { + const transcript = new ConversationTranscript((query, limit, ctx) => + conversation.entries(query, limit, undefined, ctx), + ); + const watch = await conversation.watch(context); + render(transcript.merge(watch.value)); + watch.start(async ({ value }) => render(transcript.merge(value))); + // A marker scrolled into view: + render(await transcript.loadEarlier(context)); + // Reconnect after a drop: + const resumed = await conversation.watch(context, transcript.resume); + return resumed; + }, + // Section 4: table reads before the first table write; documents stay usable. tableRules: async () => { await session.commit(async (tx) => { From b3a29f1111114fac696e2cfcd65acdb382a6ed5e Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Tue, 27 Oct 2026 16:57:59 +0000 Subject: [PATCH 041/271] fix(durable): end restart loops visibly; a host death charges a generation's attempt A generation whose request or poll fetch was in flight at a host death counts one failed attempt of the retry policy (backoff, the partial kept, the same deferred handle polled again); without retries left the run ends with runFailure interrupted. restartLimit ends every other task visibly instead of parking it (tool error result, compactionFailure, runFailure restart_limit, faulted extension tasks) and reports task_interrupted; retryTask, task_parked and the restart_limit blocked reason are removed. closes #11139 --- packages/durable/CHANGELOG.md | 5 + packages/durable/README.md | 12 +- packages/durable/docs/spec.md | 125 ++- packages/durable/src/harness/agent.ts | 2 +- packages/durable/src/harness/compaction.ts | 17 + packages/durable/src/harness/generation.ts | 175 +++- packages/durable/src/harness/harness.ts | 10 +- packages/durable/src/harness/live.ts | 68 +- packages/durable/src/harness/scheduler.ts | 127 ++- packages/durable/src/harness/task-graph.ts | 2 +- packages/durable/src/harness/telemetry.ts | 4 +- packages/durable/src/harness/tool.ts | 28 +- packages/durable/src/harness/types.ts | 23 +- packages/durable/src/types.ts | 19 +- packages/durable/test-workers/wake.test.ts | 2 +- packages/durable/test-workers/worker.ts | 2 + packages/durable/test/chat-support.ts | 10 +- packages/durable/test/harness-blocked.test.ts | 68 +- .../test/harness-deferred-progress.test.ts | 67 +- .../test/harness-restart-failure.test.ts | 775 ++++++++++++++++++ 20 files changed, 1295 insertions(+), 246 deletions(-) create mode 100644 packages/durable/test/harness-restart-failure.test.ts diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 471b12b79..17cfc25d6 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -80,6 +80,8 @@ | a run that ended without an answer (`run_end` with an `unanswered` `submission`) | `pi.live.runFailure`: `reason` (`model_error`, `no_model`, `aborted`, `reset`, `faulted`, `orphaned`), `detail`, `taskId`, `inputs` | Some event data has no state in the view. A tool's arguments after repair and `beforeTool` at its start: in the nested slot's `arguments` for a nested call, only in the tool task's checkpoint for a model-issued one. A submission's settlement: `Submission.status()`/`wait()` and `Harness.submission(id)`; an input settled without a run (withdrawn, `stale`, aborted while queued, `run_failed`) just leaves `pi.inbox`. A failed task's error: `Harness.getTask()` and `onReport`; the task graph keeps only live tasks, not terminal outcomes. A nested call's full result: its caller's `NestedCallDoc` (`watchDoc()`), which retires when the caller settles. Exact turn boundaries: the run's `taskId` and the task graph, with no explicit event. Whether a compaction did anything: its `CompactionResult` (`waitForTask()`), and the summary entry when it is placed. +- Restart-limited tasks no longer park: a task cut off `settings.restartLimit` times in a row ends instead, visibly. A tool call ends with an error result (diagnostic code `restart_limit`) and its task fails with detail `{ reason: "restart_limit" }`, a compaction fails into `pi.live.compactionFailure`, a generation ends its run with `pi.live.runFailure` reason `restart_limit` (its inputs settle `unanswered` with that reason), and a task of an extension's kind ends `faulted` with detail `{ reason: "restart_limit" }`, reported as `task_faulted` with reason `restart_limit`. The built-in tasks are reported as `task_interrupted` (below). An abort handler cut off at the limit still faults its task, but a run someone aborted now ends with `runFailure` reason `aborted`, the cut-off as `detail`, and an aborted compaction leaves no `compactionFailure`. `Harness.retryTask()`, the `restart_limit` blocked reason of `TaskInspection` and `TaskGraphNode`, and the `task_parked` report (and its telemetry value) are removed. Migration: a parked task kept its conversation busy until `retryTask()`; it now ends, so show `runFailure`, `compactionFailure`, the tool's error result, or the `task_interrupted` and `task_faulted` reports instead, and to run the work again, submit it again (a new input, `compact()`, or a new task); raise `restartLimit` before the task is next reserved to let it run longer ([#11139](https://github.com/earendil-works/pi/issues/11139)). +- A generation whose model request was in flight when the host died no longer resends it on open: the attempt counts as failed under `settings.retry`. Its partial becomes an aborted entry, as before; the next attempt waits for the backoff, shown in `pi.live.generation.retry`, and with no retry left the run ends with `pi.live.runFailure` reason `interrupted` and the inputs settle `unanswered` with that reason. A deferred poll whose fetch was in flight is charged the same way, even when the replacement host's clock is behind, and its next attempt polls the same deferred response again instead of submitting a new request; a run that ends instead cancels it. A close still resends. `restartLimit` still bounds these phases: a charge that itself keeps being cut off ends the run `restart_limit`, past the limit without converting the partial ([#11139](https://github.com/earendil-works/pi/issues/11139)). ### Added @@ -87,6 +89,8 @@ - Synchronous observers for adapters, such as a Chord source over a Durable view: `Session.observeDoc(token, ...scope, observer, context)`, `Harness.observeConversation(id, observer, context)`, and `Harness.observeTaskGraph(observer, context)` attach an `Observer` (`advance(change, context)`, `end(failure)`) atomically on the Session line, like a watch, and return an `Observation` (`value`, `detach()`; `observeDoc` gives `undefined` for an absent document). `advance` gets every change unfolded as memory adopts it, for a stored commit inside Storage's `adopt` and for a deferred one before it is stored, so it must only record it: no awaits, no Session calls, no throws. A document observer gets one call per change at its address (two for a commit that retires and recreates it), the task graph also a call outside any commit when a registry change or a failed migration changes a `blocked` reason, and a change can arrive before the attaching promise resolves. The attaching context governs only the attachment; a closed Session or an invalid argument rejects the promise. One that throws is reported as `listener_error`, detached, and ended with the error. `end` comes once, at Session close or failure, or after a task document's retirement with its task. Document, view, and task graph watches are built on them (spec §9.1). - `ConversationTranscript` keeps what a client has of one conversation over any `ConversationView`: `merge(view)` returns the view with `items`, the client's whole transcript (entries, a `gap` where a later window did not continue it, and an `earlier` marker until the start is loaded), instead of the window; `loadEarlier(context)` and `fillGap(gap, context)` page entries through a loader such as `Conversation.entries()`, and `resume` is the window that continues it after a drop: `conversation.watch(context, transcript.resume)`. A merge costs O(items) and returns the same `items` array when it adds nothing; concurrent loads of the same page are shared ([#11327](https://github.com/earendil-works/pi/issues/11327)). - `pi.live.runFailure` (`RunFailure`, `RunFailureReason`) records why the newest run ended without an answer, `model_error`, `no_model`, `aborted`, `reset`, `faulted`, or `orphaned`, with the run's task and inputs and a `detail` where one exists (the error text, the missing model, the fault, the orphan reason). The commit that ends the run sets it; a later commit that starts a run removes it ([#11327](https://github.com/earendil-works/pi/issues/11327)). +- `RunFailureReason` gains `interrupted` (the host stopped while the model was answering, with no retry left) and `restart_limit` (the run's task was cut off `settings.restartLimit` times in a row) ([#11139](https://github.com/earendil-works/pi/issues/11139)). +- `onReport` gets `task_interrupted` (`reason` `restart_limit` or `interrupted`, `restarts`, and the task) when crashes end a built-in task: a tool call, a compaction, or a generation at the restart limit, or a run that ends `interrupted`. The telemetry report attributes gain the type `task_interrupted` and the reason `interrupted` ([#11139](https://github.com/earendil-works/pi/issues/11139)). - `HarnessOptions.readOnly` opens a Harness as a viewer of a session another process drives: open leaves running tasks as committed instead of recovering them, every commit with writes rejects with `ReadOnlySession` before it reaches Storage, and `resume()` and every call that would run tasks reject with it. ### Changed @@ -108,6 +112,7 @@ - Fixed partial commits polluting `Object.prototype` when a streamed tool call's arguments gained a `__proto__` member after an earlier commit: the merge read the missing member through to `Object.prototype` and wrote into it - Fixed a commit that Storage rejects after adopting it (a failed JSONL reclaim) publishing nothing: it is published and its waits resolve before the Session fails, so a replica built from `subscribeCommits()` stays equal to Storage and `pi.durable.tasks.active` stays balanced. - Fixed a throwing telemetry adapter (a commit's `pi.durable.entry` or `pi.durable.submission` event, `pi.durable.tasks.active`) rejecting a durable commit and leaving the views and the scheduler without it: it is reported as `host_error` with callback `"telemetry"`, and the commit resolves. A throwing report metric or report event is dropped. +- Fixed a host that crashes during a model request resending it forever, adding an aborted entry each time: the interrupted request is a failed attempt under the retry policy, and the run ends with `runFailure` reason `interrupted` once no retry is left; other work cut off `restartLimit` times ends instead of keeping its conversation busy ([#11139](https://github.com/earendil-works/pi/issues/11139)) ## [1.4.0] - 2026-10-21 diff --git a/packages/durable/README.md b/packages/durable/README.md index 6b4237ba3..f57c2629b 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -2,7 +2,7 @@ > **Experimental.** The API changes without notice between releases. -A durable agent harness. Conversations, model turns, tool calls, and your own state are committed to storage before anything is shown. If the process dies mid-turn, reopening the storage picks the work up where it stopped. +A durable agent harness. Conversations, model turns, tool calls, and your own state are committed to storage before anything is shown. If the process dies mid-turn, reopening the storage picks the work up where it stopped; a model request it died during counts as a failed attempt and is retried, so a request that keeps killing the process ends (see [Persist and Resume](#persist-and-resume)). Built on [`@earendil-works/pi-ai`](../ai/README.md) for model access and `@earendil-works/chord` for document state. @@ -213,11 +213,11 @@ The default `setTimer` declines the timers that only free memory in Workers, so ### Failures and Recovery -**Crash loops.** Work that deterministically crashes its process, by exhausting memory or CPU before its next checkpoint, would otherwise be resumed and crash again forever. Open counts each task a crash, eviction, or reset cut off; a close is not counted, and a new state the task commits starts the count over. Once the count reaches `settings.restartLimit` (default 5; `Infinity` never parks), the task is parked: `inspect()` and the task graph show it as blocked with reason `restart_limit`, `wake()` does not count it as active, and `onReport` gets it once as `task_parked`. Every task running when a process dies is counted, so show parked tasks and offer `harness.retryTask(id)`, which runs one again. `abortTask()` still runs its abort handler; an abort handler cut off that many times ends the task `faulted`. +**Crash loops.** Work that deterministically crashes its process, by exhausting memory or CPU before its next checkpoint, would otherwise be resumed and crash again forever. A model request the host died during is a failed attempt under `settings.retry`: its partial answer is kept as an aborted entry, the next attempt waits for the backoff (shown in `docs["pi.live"].generation.retry`), and with no retry left the run ends with `docs["pi.live"].runFailure` reason `interrupted`. A deferred response that was being fetched is polled again as the next attempt, not requested again, and cancelled when the run ends. A close is not a crash: a request it stopped is sent again. Other work is counted: open counts each task a crash, eviction, or reset cut off, and a new state the task commits starts the count over. Once the count reaches `settings.restartLimit` (default 5; `Infinity` never ends a task for it), the task ends instead of running again: a tool call with an error result (diagnostic code `restart_limit`), a compaction failed into `compactionFailure`, a run with `runFailure` reason `restart_limit` (also when charging an interrupted request keeps crashing), and a task of your own kinds `faulted`, reported as `task_faulted` with reason `restart_limit`. The built-in ones are reported as `task_interrupted`, as is a run that ends `interrupted`, so a crash loop always reaches `onReport`. Every task running when a process dies is counted, not only the one that killed it. An abort handler cut off that many times ends its task `faulted` too; a run someone aborted still ends `aborted`, with the cut-off in `detail`. -**Reports.** `onReport(error, info)` says what happened in `info.type`: `session_failed` for the error that failed the Harness (a Storage error, often a platform reset); `task_faulted`, `task_parked`, `migration_failed`, `task_outdated`, or `task_error` for a task, with `info.task` (its `id`, `kind`, and `conversationId`) and the extension that installed it in `info.extension`; `extension_error` for hooks, sections, wrappers, and tools, with the extension's name; `host_error` for a host callback, named in `info.callback`; and `listener_error`. A host can feed `task_faulted` reports of one kind to its own breaker. +**Reports.** `onReport(error, info)` says what happened in `info.type`: `session_failed` for the error that failed the Harness (a Storage error, often a platform reset); `task_faulted`, `task_interrupted` (crashes ended it, with `info.reason` and `info.restarts`), `migration_failed`, `task_outdated`, or `task_error` for a task, with `info.task` (its `id`, `kind`, and `conversationId`) and the extension that installed it in `info.extension`; `extension_error` for hooks, sections, wrappers, and tools, with the extension's name; `host_error` for a host callback, named in `info.callback`; and `listener_error`. A host can feed `task_faulted` reports of one kind to its own breaker. -**Queued inputs after a failed run.** A run that fails with a model error leaves the inputs queued behind it waiting until the next input arrives. `settings.afterFailedRun` changes that: `continue` starts the next run with them, as after an answer, and `withdraw` settles them `unanswered` with reason `run_failed`. +**Queued inputs after a failed run.** A run that fails with a model error, a missing model, or crashes (`interrupted`, `restart_limit`) leaves the inputs queued behind it waiting until the next input arrives. `settings.afterFailedRun` changes that: `continue` starts the next run with them, as after an answer, and `withdraw` settles them `unanswered` with reason `run_failed`. **Blocked tasks.** Tasks blocked on a missing or unmigratable definition show their reason in the task graph too, and a registry change that unblocks them makes `onWake` report the Session active, so a host need not poll them. @@ -274,7 +274,7 @@ const count = defineTool({ registry.install(defineExtension({ name: "count", tools: [count] })); ``` -Each call runs as its own durable task. Its intent is committed before `execute()` runs. If the process dies mid-call, the tool reruns on reopen only when it is declared `replay: "safe"`; otherwise the model gets an `interrupted` error result with the output committed so far. Throwing from `execute()` gives the model an error result. A result can also return `usage`, which is added to the conversation's [usage](#usage-and-cost). It can also return `control: { terminate: true }`: when every result of the round asks for it, the run ends without another model request. +Each call runs as its own durable task. Its intent is committed before `execute()` runs. If the process dies mid-call, the tool reruns on reopen only when it is declared `replay: "safe"`; otherwise the model gets an `interrupted` error result with the output committed so far. A replay-safe call cut off `settings.restartLimit` times in a row ends with a `restart_limit` error result instead of running again. Throwing from `execute()` gives the model an error result. A result can also return `usage`, which is added to the conversation's [usage](#usage-and-cost). It can also return `control: { terminate: true }`: when every result of the round asks for it, the run ends without another model request. A result has three channels: `output` for the model, `structuredOutput` for programs that call the tool (see [Calling tools from tools](#calling-tools-from-tools)), and `details` for UIs. A tool that declares `structuredOutputSchema` returns a matching `structuredOutput`, which the Harness validates. Programs calling a tool without one get its bounded output: one text item as a string, one image as its `ImageContent`, nothing as `""`, and anything else as the content list, for errors too. `structuredOutput` is always the tool's own value: a schema tool's error result keeps the value it returned, and the text of one without is in a `tool_error` diagnostic, as a thrown error's message is. The transcript stores only `output` and `details`: @@ -538,7 +538,7 @@ await root.submit({ type: "write", entry: { kind: "app.note", data: "user opened If a run fails, queued items stay in the inbox until the next input or queued write places them, oldest first. -When a run ends without an answer, `docs["pi.live"].runFailure` says why until the next run starts: `model_error` and `no_model` (with the error text or the missing model in `detail`), `aborted`, `reset`, or `faulted` and `orphaned` (the run's task could not go on). The run's inputs are in it too, settled `unanswered`; the settlement's reason is the same, except that an orphaned run's inputs carry why it was orphaned (`missing_task`, `task_too_old`, `migration_failed`) and `no_model` has no `detail` there (spec §8.2). +When a run ends without an answer, `docs["pi.live"].runFailure` says why until the next run starts: `model_error` and `no_model` (with the error text or the missing model in `detail`), `aborted`, `reset`, `interrupted` (the host stopped while the model answered, with no retry left), or `faulted`, `orphaned`, and `restart_limit` (the run's task could not go on). The run's inputs are in it too, settled `unanswered`; the settlement's reason is the same, except that an orphaned run's inputs carry why it was orphaned (`missing_task`, `task_too_old`, `migration_failed`) and `no_model` has no `detail` there (spec §8.2). An input is placed as a `pi.user` entry. To show it as something else, give it an `entry` with your kind and data; the model still sees `content` as the user message: diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index e61912761..647a2822b 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -477,7 +477,7 @@ type HarnessSettings = { readonly documentRetentionMs?: number; /** What a failed run does with the inputs queued behind it (section 6). */ readonly afterFailedRun?: "keep" | "continue" | "withdraw"; - /** Restarts in a row without progress after which a task is parked (section 5.4); `Infinity` never parks. */ + /** Restarts in a row without progress after which a task ends instead of running again (section 5.4). */ readonly restartLimit?: number; }; @@ -605,7 +605,7 @@ type TaskInspection = { | { readonly kind: "completing" } | { readonly kind: "blocked"; - readonly reason: "missing_task" | "task_too_old" | "migration_failed" | "restart_limit"; + readonly reason: "missing_task" | "task_too_old" | "migration_failed"; readonly error?: unknown; }; }; @@ -624,7 +624,13 @@ type ReportInfo = readonly extension?: string; readonly reason: "threw" | "restart_limit"; } - | { readonly type: "task_parked"; readonly task: ReportTask; readonly extension?: string } + | { + readonly type: "task_interrupted"; + readonly task: ReportTask; + readonly extension?: string; + readonly reason: "restart_limit" | "interrupted"; + readonly restarts: number; + } | { readonly type: "migration_failed"; readonly task: ReportTask; readonly extension?: string } | { readonly type: "task_outdated"; @@ -712,8 +718,6 @@ interface Harness extends Session { conversationId?: ConversationId, ): Promise<"aborted" | "already_placed" | "settled" | "not_found">; abortTask(id: TaskId, context: Context): Promise<"marked" | "terminal">; - /** Run a task parked by the restart limit again (section 5.4). */ - retryTask(id: TaskId, context: Context): Promise; waitForTask(id: TaskId, context: Context): Promise>; waitForIdle(context: Context): Promise; /** Session total of every conversation's `pi.usage` (section 8.6). */ @@ -2360,7 +2364,7 @@ Every run invocation start resolves the task's definition by `TaskRecord.kind` from the scheduler's current registry snapshot: ```text -restarts at restartLimit -> stay pending, blocked: restart_limit +restarts at restartLimit -> a kind of an extension: terminal faulted (built-in tasks end themselves) definition missing -> stay pending, blocked: missing_task stored version newer than it -> stay pending, blocked: task_too_old stored version older than it -> migrate(input, checkpoint, storedVersion) @@ -2376,20 +2380,43 @@ after joining the invocations it stopped, close writes those tasks back as reset leaves `running` records. A state the task commits itself that differs from its current one (a new checkpoint, a wait, a sleep, an outcome) is progress and removes the count; the same checkpoint committed again, -memos, and document writes are not. Once `restarts` reaches -`settings.restartLimit` (default 5; `Infinity` never parks), the task is parked: -blocked with reason `restart_limit`, reported once as `task_parked` at the open -that parks it, shown in `inspect()` and the task graph, and not active in -`wake()`. Work that deterministically exhausts memory or CPU before its next -checkpoint thus stops after a few attempts instead of running forever. Every -task running when a process dies is counted, not only the one that killed it, -so a long stream next to the culprit can be parked too; a host shows parked -tasks and offers `Harness.retryTask(id)`, which removes the count and runs the -task again, and does nothing for a task that is not parked. A run task that is -parked keeps its conversation busy. An abort mark starts the count over and -still runs a parked task's abort handler; an abort handler cut off -`restartLimit` times is not run again, and the task ends `faulted` with detail -`{ reason: "restart_limit" }`, reported as `task_faulted` with reason `restart_limit`. +memos, and document writes are not. Work that deterministically exhausts memory +or CPU before its next checkpoint thus shows as a growing count, and once +`restarts` reaches `settings.restartLimit` (default 5; `Infinity` never ends a +task for it), the task ends visibly instead of running again: + +| task | at the limit | +|---|---| +| tool call (`pi.tool`) | ends with an error result, diagnostic code `restart_limit` ("Tool X was interrupted 3 times in a row; it is not run again"), built from its slot like an interrupted call's; its task fails with detail `{ reason: "restart_limit" }` | +| compaction | fails with that error and detail, recorded in `pi.live.compactionFailure` | +| generation | ends its run `unanswered` with `pi.live.runFailure` reason `restart_limit`, failing with that detail | +| a kind of an extension | the scheduler ends it `faulted` with detail `{ reason: "restart_limit" }`, reported as `task_faulted` with reason `restart_limit` | + +A built-in task that ends itself this way, and a generation whose run ends +`interrupted` (section 8.3), is reported as `task_interrupted` with that reason, +the task, and its `restarts`, so a host keeps a crash-loop signal. The built-in +tasks check the count when a phase starts, a tool call after reading its call; +reading the setting only when the task has a count, so the normal path does no +more work. The scheduler checks an extension's task when it reserves it, +reading the current setting, so a raised limit lets the task run; a task that +already ended stays ended. A generation's request or deferred poll in flight +when the host died is a failed attempt under the retry policy (section 8.3) +first; the limit still bounds it, so a charge that itself keeps cutting the +task off (converting a huge partial, say) ends the run. At the limit a task +still does its usual cleanup, such as converting a partial or cancelling a +deferred response; past it, after that cleanup was cut off too, it ends +without. Every task running when a process dies is counted, not only the one +that killed it, so a long stream next to the culprit can end too. A nested +call runs only while its replay-safe caller runs, so a crash counts both and +the caller reaches the limit no later than its nested calls; the caller's own +`restart_limit` result is what the model sees. An abort mark starts the count +over and still runs the abort handler; an abort handler cut off `restartLimit` +times is not run again, and the task ends `faulted` with detail +`{ reason: "restart_limit" }`, reported as `task_faulted` with reason +`restart_limit`. Someone aborted that task, so its conversation shows the +abort: a run ends with `runFailure` reason `aborted`, the cut-off as `detail`, +without converting its partial, and a compaction's status is removed without a +failure. An abort invocation resolves the definition the same way. When the definition can take the task, its abort handler runs; otherwise the task is orphaned as @@ -2403,9 +2430,8 @@ incompatible, neither at open nor later. A blocked task without an abort mark keeps its durable record unchanged, remains live, still blocks ordinary idle waits, and is reconsidered whenever the registry changes. The blocked reason is derived runtime state, not a persisted task status. A blocked task is not active in -`wake()` (section 5.6): only a registry change, or `retryTask()` for a parked -one, can let it run, and either makes `onWake` report it active, so a host need -not poll it. The task graph shows the reason and follows registry changes +`wake()` (section 5.6): only a registry change can let it run, and it makes +`onWake` report it active, so a host need not poll it. The task graph shows the reason and follows registry changes (section 9.4). Document migration remains access-driven. @@ -2740,8 +2766,8 @@ for the other submission type rejects. A busy input with `whenBusy: "reject"` writes no record and reports `ConversationBusy`. An idle input queues with mode `steer` when `whenBusy` is `steer` and `followUp` otherwise. -`settings.afterFailedRun` decides what a run that fails with `model_error` or -`no_model` does with the inputs queued behind it, in the commit that settles its +`settings.afterFailedRun` decides what a run that fails with `model_error`, +`no_model`, `interrupted`, or `restart_limit` does with the inputs queued behind it, in the commit that settles its own inputs: `keep` (the default) leaves them queued; `continue` applies the final boundary, so the next run starts with them, as after an answer; `withdraw` settles them `unanswered` with reason `run_failed`, keeping queued @@ -3431,7 +3457,8 @@ error diagnostic already says what went wrong, so programs get it, for example diagnostic too. So a hook that redacts a schema-less tool's output redacts what programs get. An error result the Harness writes itself (unavailable, invalid arguments, -blocked, interrupted, aborted, abandoned, or a scheduler fault) has no +blocked, interrupted, aborted, abandoned, ended at the restart limit, or a +scheduler fault) has no `structuredOutput`; its diagnostics say what went wrong. A thrown error becomes a `tool_error` diagnostic with its message. A nested call whose result breaks this gets an error result with an `invalid_structured_output` diagnostic, without the broken value: a @@ -3799,7 +3826,8 @@ retain their tool-result entry ID for the generation's `tools` phase. Error results the Harness writes itself set `isError` and carry an `error` diagnostic with one of the codes `tool_unavailable`, `invalid_arguments`, -`blocked`, `interrupted`, `aborted`, or `tool_error` (a throw from `execute()`) +`blocked`, `interrupted`, `abandoned`, `restart_limit` (section 5.4), +`aborted`, or `tool_error` (a throw from `execute()`) and the error text as its message. Their content is the durable partial output, if any, and `details` is the tool's last reported value, if any. @@ -4267,7 +4295,7 @@ type RunFailure = { taskId: TaskId; /** The run's inputs, settled `unanswered`. */ inputs: SubmissionId[]; - reason: "model_error" | "no_model" | "aborted" | "reset" | "faulted" | "orphaned"; + reason: "model_error" | "no_model" | "aborted" | "reset" | "faulted" | "orphaned" | "interrupted" | "restart_limit"; /** * `model_error`: the error text; `no_model`: which model is missing; `faulted`: the error message; `orphaned`: why * the task could not run (`missing_task`, `task_too_old`, `migration_failed`). @@ -4351,10 +4379,12 @@ that settles its inputs `unanswered`: |---|---|---| | `model_error` | the response failed after any retries, or a blocking or overflow compaction it needed failed; `detail` is the error text | `model_error` with the text | | `no_model` | no model is configured, or the configured one is not available; `detail` names it | `no_model` | -| `aborted` | its task was aborted (`Conversation.abort()`, `abortTask()`) | `aborted` | +| `aborted` | its task was aborted (`Conversation.abort()`, `abortTask()`); `detail` says when its abort handler was cut off `restartLimit` times (section 5.4) | `aborted`, with that detail | | `reset` | a queued reset started a new context at the post-tools boundary (a tool's handoff continues the run) | `reset` | | `faulted` | the scheduler faulted its task (section 5.4); `detail` is the error message | `faulted` with the message | | `orphaned` | its task was aborted while no definition could take it; `detail` is the reason | the reason | +| `interrupted` | the host stopped while the model was answering, with no retry left (section 8.3); `detail` names the attempt | `interrupted` with the text | +| `restart_limit` | its generation was cut off `restartLimit` times in a row (section 5.4), also while an interrupted request was charged; `detail` says in which phase | `restart_limit` with the text | The next commit that starts a run removes it. A run started in the same commit, the queue's next run under `afterFailedRun: "continue"` (section 6) or after a @@ -4469,7 +4499,22 @@ The run's inputs live in `pi.live.run`, not in the task input. `prepare`. `request` converts a leftover partial (below) before it resolves the model. - `request` first converts a committed partial left in `pi.live` by an - interrupted attempt into an aborted `pi.assistant` entry. It then streams the + interrupted attempt into an aborted `pi.assistant` entry. When the host died + while this request was in flight (the record's `restarts`, section 5.4), the + attempt failed: that commit also either sleeps at `retry` until + `now + retryDelayMs(policy, attempt)`, with `pi.live.generation.retry` saying + "The host stopped while the model was answering, on attempt N", when the + retry policy allows another attempt, or settles the inputs `unanswered` and + ends the run with `runFailure` reason `interrupted` and that text, failing the + task. A request a close stopped is not counted and is sent again, below. A + `poll` whose fetch was in flight is charged the same way ("The host stopped + while the deferred answer was fetched"), checked before its `pollAt`, which a + replacement host's clock may not have reached: its `retry` keeps the deferred + response and polls it again as the next attempt instead of sending another + request, and a run that ends instead cancels it, as the abort handler does. + A generation asleep in `retry` or `poll`, or waiting for its tools, is not + running and never charged. Past `restartLimit` the run ends `restart_limit` + instead (section 5.4). Otherwise it streams the model context through `cutoff` with the invocation signal, the thinking level as `reasoning` (omitted for `off`), the conversation's persisted provider `sessionId`, and the pinned `streamOptions`, committing throttled partials at @@ -4477,9 +4522,10 @@ The run's inputs live in `pi.live.run`, not in the task input. (section 4.1), so Storage gets them at the next commit that reads `pi.live` or within `settings.progress.persistWithinMs`. Before streaming, the `beforeRequest` chain may - replace the messages for this request only. Recovery resends the same - committed messages with the same pinned model, thinking level, and stream - options, and reruns `beforeRequest`. + replace the messages for this request only. After a close, recovery resends + the same committed messages with the same pinned model, thinking level, and + stream options, and reruns `beforeRequest`; after a crash the attempt is + charged instead, as above. - `afterResponse` observes every terminal message, from `request` or `poll`, before classification; a still deferred result is not terminal. - Before classifying, the handler stops the partial throttle and awaits any @@ -4790,7 +4836,7 @@ Phases: `sessionId`, and the pinned stream options without `deferred`, with `cacheRetention: "none"` and the pinned `maxTokens`. Providers such as Codex may suppress that identity when caching is disabled. Recovery resends the same - request. The response is classified in one commit + request, until the task was cut off `restartLimit` times in a row (section 5.4). The response is classified in one commit that adds its usage to `pi.usage` (section 8.6): - `stop` with non-empty text and no tool call: the text is the summary, placed as below. @@ -5288,7 +5334,7 @@ type TaskGraphNode = { /** Conversations this task owns, in ID order. */ readonly conversations: readonly ConversationId[]; /** Why it cannot run here, when it cannot (section 5.4); follows registry changes. */ - readonly blocked?: "missing_task" | "task_too_old" | "migration_failed" | "restart_limit"; + readonly blocked?: "missing_task" | "task_too_old" | "migration_failed"; }; type TaskGraph = { @@ -5643,9 +5689,12 @@ views keep it. - `session_failed`: the error that failed the Session (a Storage error, often a platform reset, or the Session's own bookkeeping); - `task_faulted`: a task ended `faulted`, because its phase threw (reason - `threw`) or its abort handler was cut off `restartLimit` times (reason - `restart_limit`); -- `task_parked`: the restart limit parked a task (section 5.4); + `threw`), or it or its abort handler was cut off `restartLimit` times (reason + `restart_limit`, section 5.4); +- `task_interrupted`: crashes ended a task: a built-in task ended itself at the + restart limit (reason `restart_limit`), or a generation's run ended because + the host kept stopping while the model answered (reason `interrupted`), with + the task's `restarts` (section 5.4); - `migration_failed`: a task's migration threw, and the task is blocked; - `task_outdated`: a running task keeps its old definition, which is missing (`missing_task`) or cannot take the task (`incompatible_task`); diff --git a/packages/durable/src/harness/agent.ts b/packages/durable/src/harness/agent.ts index 7706d945b..d996bec6f 100644 --- a/packages/durable/src/harness/agent.ts +++ b/packages/durable/src/harness/agent.ts @@ -57,7 +57,7 @@ const DEFAULT_CONTEXT_RETENTION_MS = 600_000; /** Default `settings.restartLimit`. */ export const DEFAULT_RESTART_LIMIT = 5; -/** A restart limit as given when it is a number of restarts, `1` or more, rounded down (`Infinity` never parks); else the default. */ +/** A restart limit as given when it is a number of restarts, `1` or more, rounded down (`Infinity`: no task ends for restarts); else the default. */ function restartLimit(limit: number | undefined): number { return typeof limit === "number" && limit >= 1 ? Math.floor(limit) : DEFAULT_RESTART_LIMIT; } diff --git a/packages/durable/src/harness/compaction.ts b/packages/durable/src/harness/compaction.ts index b7376a882..b5f480fae 100644 --- a/packages/durable/src/harness/compaction.ts +++ b/packages/durable/src/harness/compaction.ts @@ -18,6 +18,8 @@ import { orderToolResults } from "./context.ts"; import { addCompactionStatus, compactionStatus, + cutOffCount, + cutOffTooOften, failCompactionStatus, LiveDoc, type LiveState, @@ -115,6 +117,7 @@ export const CompactionTask = defineTask ({ phase: "select" }), phases: { select: async (task, runtime, context) => { + if (cutOffTooOften(task, runtime)) return failCutOff(runtime, task, context); const { conversationId } = runtime; const agent = await runtime.agent(context); const settings = runtime.settings; @@ -155,6 +158,7 @@ export const CompactionTask = defineTask ({ status: "running", checkpoint: { phase: "summarize", ...request } }), context); }, summarize: async (task, runtime, context) => { + if (cutOffTooOften(task, runtime)) return failCutOff(runtime, task, context); const { phase: _, ...request } = task.state.checkpoint; const { model: ref, thinkingLevel, streamOptions, maxTokens, tail, firstKept, attempt } = request; const model = runtime.models.getModel(ref.provider, ref.modelId); @@ -219,6 +223,7 @@ export const CompactionTask = defineTask { const { phase: _, until, ...request } = task.state.checkpoint; + if (cutOffTooOften(task, runtime)) return failCutOff(runtime, task, context); if (until > runtime.now()) return runtime.sleepUntil(until, task.state.checkpoint, context); const attempt = request.attempt + 1; await runtime.commit(async (tx) => { @@ -473,6 +478,18 @@ async function complete(runtime: Runtime, context: Context): Promise { }, context); } +/** Fail a compaction cut off `settings.restartLimit` times in a row (spec §5.4) instead of running it again. */ +async function failCutOff(runtime: Runtime, task: { readonly restarts?: number }, context: Context): Promise { + const message = `Compaction was interrupted ${cutOffCount(task)}`; + await runtime.commit(async (tx) => { + failCompactionStatus(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId, message); + return { + status: "terminal", + outcome: { status: "failed", error: { message, detail: { reason: "restart_limit" } } }, + }; + }, context); +} + async function failNoModel(runtime: Runtime, ref: ModelRef | undefined, context: Context): Promise { const message = ref === undefined ? "No model is configured" : `Model ${ref.provider}/${ref.modelId} is not available`; diff --git a/packages/durable/src/harness/generation.ts b/packages/durable/src/harness/generation.ts index 53d75fd73..dc2a9877a 100644 --- a/packages/durable/src/harness/generation.ts +++ b/packages/durable/src/harness/generation.ts @@ -30,7 +30,17 @@ import type { import { addTools, sectionExtension } from "./agent.ts"; import { compactionPolicy, createCompaction, estimateContext, selectCut } from "./compaction.ts"; import { applyBoundary, type Boundary, prepareBoundary, withdrawQueuedInputs } from "./inbox.ts"; -import { endRun, LiveDoc, type LiveState, PROGRESS_COMMIT, type RunEnd, type ToolSlot } from "./live.ts"; +import { + cutOffCount, + cutOffTooOften, + endRun, + LiveDoc, + type LiveState, + PROGRESS_COMMIT, + type RunEnd, + salvageable, + type ToolSlot, +} from "./live.ts"; import { PartialChanges, type PartialMessage } from "./partial-changes.ts"; import { planSystemEntries, renderSections, replaySections } from "./prompt.ts"; import { ensureProviderSessionId } from "./provider.ts"; @@ -74,7 +84,17 @@ export type GenerationCheckpoint = /** Newest entry included in the request. */ cutoff: EntryId; } - | { phase: "retry"; attempt: number; compacted?: TaskId; until: number } + | { + phase: "retry"; + attempt: number; + compacted?: TaskId; + until: number; + /** + * The deferred response a poll was fetching when the host died: the next attempt polls it again instead of + * sending another request, so the provider's job is neither lost nor paid twice. Absent otherwise. + */ + poll?: { model: ModelRef; thinkingLevel?: ModelThinkingLevel; cutoff: EntryId; handle: DeferredHandle }; + } | { phase: "poll"; attempt: number; @@ -133,6 +153,7 @@ export const GenerationTask = defineTask { + if (cutOffTooOften(task, runtime)) return endCutOff(runtime, task, context); const { conversationId } = runtime; const agent = await runtime.agent(context); const settings = runtime.settings; @@ -209,6 +230,10 @@ export const GenerationTask = defineTask { + // Charging the attempt may itself be what keeps failing, such as converting a huge partial: the limit ends it. + if (cutOffTooOften(task, runtime)) return endCutOff(runtime, task, context); + // The host stopped while this request was in flight: a failed attempt, not a resend (spec §8.3). + if (task.restarts !== undefined) return interrupted(runtime, task.state.checkpoint, context); const { attempt, compacted, model: ref, thinkingLevel, streamOptions, cutoff } = task.state.checkpoint; const conversationId = runtime.conversationId; await runtime.commit(async (tx) => { @@ -238,21 +263,32 @@ export const GenerationTask = defineTask { - const { attempt, compacted, until } = task.state.checkpoint; + const { attempt, compacted, until, poll } = task.state.checkpoint; + if (cutOffTooOften(task, runtime)) return endCutOff(runtime, task, context); if (until > runtime.now()) return runtime.sleepUntil(until, task.state.checkpoint, context); await runtime.commit(async (tx) => { - (await tx.doc(LiveDoc, runtime.conversationId)).generation = { attempt: attempt + 1 }; - const checkpoint: GenerationCheckpoint = { - phase: "prepare", - attempt: attempt + 1, - ...(compacted === undefined ? {} : { compacted }), - }; - return { status: "running", checkpoint }; + const live = await tx.doc(LiveDoc, runtime.conversationId); + const kept = compacted === undefined ? {} : { compacted }; + // A poll the host died during polls its response again, as the next attempt. + if (poll !== undefined) { + const pollAt = runtime.now(); + live.generation = { attempt: attempt + 1, deferred: { pollAt } }; + return { + status: "running", + checkpoint: { phase: "poll", attempt: attempt + 1, ...kept, ...poll, pollAt }, + }; + } + live.generation = { attempt: attempt + 1 }; + return { status: "running", checkpoint: { phase: "prepare", attempt: attempt + 1, ...kept } }; }, context); }, /** After the sleep until `pollAt`; a run before it (an early wake, or a record of an earlier version) sleeps again. */ poll: async (task, runtime, context) => { const { attempt, compacted, model: ref, thinkingLevel, cutoff, handle, pollAt } = task.state.checkpoint; + if (cutOffTooOften(task, runtime)) return endCutOff(runtime, task, context); + // The host stopped while the poll's fetch was in flight: a failed attempt, as for a request. Checked before the + // time, which may have moved backwards on a replacement host; a sleeping poll is never cut off. + if (task.restarts !== undefined) return interrupted(runtime, task.state.checkpoint, context); if (pollAt > runtime.now()) return runtime.sleepUntil(pollAt, task.state.checkpoint, context); const model = runtime.models.getModel(ref.provider, ref.modelId); if (model === undefined) return failNoModel(runtime, ref, context); @@ -264,6 +300,7 @@ export const GenerationTask = defineTask { + if (cutOffTooOften(task, runtime)) return endCutOff(runtime, task, context); const { assistant, tools, pending } = task.state.checkpoint; const [next, ...rest] = pending; if (next === undefined) return finishToolRound(runtime, assistant, tools, context); @@ -285,19 +322,8 @@ export const GenerationTask = defineTask { const checkpoint = task.state.checkpoint; - if (checkpoint.phase === "poll") { - const model = runtime.models.getModel(checkpoint.model.provider, checkpoint.model.modelId); - if (model !== undefined) { - try { - await runtime.models.cancelDeferred(model, checkpoint.handle, { - signal: runtime.signal, - telemetry: context.telemetry, - }); - } catch (error) { - runtime.report(error, { type: "host", callback: "models" }); - } - } - } + const polled = deferredOf(checkpoint); + if (polled !== undefined) await cancelDeferred(runtime, polled, context); const conversationId = runtime.conversationId; // Runs after the round's tool tasks are terminal; calls never started get `aborted` results (spec §8.5). const unstarted = @@ -388,6 +414,109 @@ async function failModelError(runtime: Runtime, text: string, context: Context): }, context); } +/** + * The host stopped while this attempt's request or poll was in flight (`TaskRecord.restarts`): a failed attempt under + * `settings.retry` (spec §8.3). The committed partial becomes an aborted entry; the next attempt follows the backoff, a + * poll's by polling its deferred response again, or, with no retry left, the run ends `interrupted`. + */ +async function interrupted( + runtime: Runtime, + checkpoint: Extract, + context: Context, +): Promise { + const { attempt, compacted } = checkpoint; + const policy = runtime.settings.retry; + const retry = policy.enabled && attempt <= policy.maxRetries; + const until = retry ? runtime.now() + retryDelayMs(policy, attempt) : 0; + const polled = checkpoint.phase === "poll" ? checkpoint : undefined; + const text = + polled === undefined + ? `The host stopped while the model was answering, on attempt ${attempt}` + : `The host stopped while the deferred answer was fetched, on attempt ${attempt}`; + // A run that ends does not leave the provider's job running. + if (!retry && polled !== undefined) await cancelDeferred(runtime, polled, context); + await runtime.commit(async (tx): Promise => { + // Read before the first table write; only a failure uses it. + const failure = retry ? undefined : await prepareFailure(tx, runtime); + const live = await tx.doc(LiveDoc, runtime.conversationId); + await convertPartial(tx, live, runtime.conversationId); + if (failure === undefined) { + live.generation = { attempt, retry: { at: until, error: text } }; + const poll = + polled === undefined + ? {} + : { + poll: { + model: polled.model, + ...(polled.thinkingLevel === undefined ? {} : { thinkingLevel: polled.thinkingLevel }), + cutoff: polled.cutoff, + handle: polled.handle, + }, + }; + const next = { phase: "retry", attempt, ...(compacted === undefined ? {} : { compacted }), until, ...poll }; + return { status: "sleeping", checkpoint: next as GenerationCheckpoint, until }; + } + await endFailedRun(tx, runtime, live, failure, { status: "unanswered", reason: "interrupted", detail: text }); + return { + status: "terminal", + outcome: { status: "failed", error: { message: text, detail: { reason: "interrupted" } } }, + }; + }, context); +} + +/** + * End a run whose generation was cut off `settings.restartLimit` times in a row (spec §5.4). At the limit it still + * keeps a committed partial as an aborted entry and cancels a deferred response it polled; past the limit that cleanup + * may be what cut it off, so it ends without it. + */ +async function endCutOff( + runtime: Runtime, + task: { readonly restarts?: number; readonly state: { readonly checkpoint: GenerationCheckpoint } }, + context: Context, +): Promise { + const checkpoint = task.state.checkpoint; + const text = `The generation was interrupted ${cutOffCount(task)} in its ${checkpoint.phase} phase`; + const salvage = salvageable(task, runtime); + const polled = deferredOf(checkpoint); + if (salvage && polled !== undefined) await cancelDeferred(runtime, polled, context); + await runtime.commit(async (tx): Promise => { + const failure = await prepareFailure(tx, runtime); + const live = await tx.doc(LiveDoc, runtime.conversationId); + if (salvage) await convertPartial(tx, live, runtime.conversationId); + await endFailedRun(tx, runtime, live, failure, { status: "unanswered", reason: "restart_limit", detail: text }); + return { + status: "terminal", + outcome: { status: "failed", error: { message: text, detail: { reason: "restart_limit" } } }, + }; + }, context); +} + +/** The deferred response a checkpoint polls, or will poll again after a retry's backoff. */ +function deferredOf( + checkpoint: GenerationCheckpoint, +): { readonly model: ModelRef; readonly handle: DeferredHandle } | undefined { + if (checkpoint.phase === "poll") return checkpoint; + return checkpoint.phase === "retry" ? checkpoint.poll : undefined; +} + +/** Cancel a deferred response no attempt will poll again; a failure is reported, never thrown. */ +async function cancelDeferred( + runtime: Runtime, + checkpoint: { readonly model: ModelRef; readonly handle: DeferredHandle }, + context: Context, +): Promise { + const model = runtime.models.getModel(checkpoint.model.provider, checkpoint.model.modelId); + if (model === undefined) return; + try { + await runtime.models.cancelDeferred(model, checkpoint.handle, { + signal: runtime.signal, + telemetry: context.telemetry, + }); + } catch (error) { + runtime.report(error, { type: "host", callback: "models" }); + } +} + /** Settle the run's inputs `unanswered` with `no_model` and fail. */ async function failNoModel(runtime: Runtime, ref: ModelRef | undefined, context: Context): Promise { const message = diff --git a/packages/durable/src/harness/harness.ts b/packages/durable/src/harness/harness.ts index 9ca24f550..5908e0262 100644 --- a/packages/durable/src/harness/harness.ts +++ b/packages/durable/src/harness/harness.ts @@ -43,7 +43,7 @@ import { AgentDoc, configure, createAgent, resolveAgent, resolveSettings } from import { createCompaction } from "./compaction.ts"; import { readContext } from "./context.ts"; import { InboxDoc, withdrawQueuedInputs } from "./inbox.ts"; -import { LiveDoc, settleSchedulerOutcome } from "./live.ts"; +import { LIMITS_RESTARTS, LiveDoc, settleSchedulerOutcome } from "./live.ts"; import { ProviderDoc } from "./provider.ts"; import { BUILTIN_TASKS } from "./registry.ts"; import { type InvocationBinding, TaskScheduler } from "./scheduler.ts"; @@ -245,6 +245,7 @@ class HarnessImpl extends SessionImpl implements timers: hostTimers(options.setTimer, (error) => report(error, { type: "host_error", callback: "setTimer" })), writeAtClose: (writes) => this.writeAtClose(writes), report: this.#report, + limitsRestarts: (kind) => LIMITS_RESTARTS.has(kind), settleOutcome: settleSchedulerOutcome, withdrawInputs: withdrawQueuedInputs, conversation: async (id, binding, callContext) => { @@ -343,11 +344,6 @@ class HarnessImpl extends SessionImpl implements return this.#tasks.abort(id, context); } - retryTask(id: TaskId, context: Context): Promise { - this.assertUsable(); - return this.#tasks.retry(id, context); - } - async waitForTask(id: TaskId, context: Context): Promise> { this.#tasks.resume(); return this.#tasks.waitForTask(id, context) as Promise>; @@ -471,12 +467,12 @@ function safeReport( }; switch (info.type) { case "task_faulted": + case "task_interrupted": case "task_outdated": attributes.reason = info.reason; attributes.extension = info.extension; attributes["task.id"] = info.task.id; break; - case "task_parked": case "migration_failed": case "task_error": case "extension_error": diff --git a/packages/durable/src/harness/live.ts b/packages/durable/src/harness/live.ts index 41efa804a..d9b09cde6 100644 --- a/packages/durable/src/harness/live.ts +++ b/packages/durable/src/harness/live.ts @@ -91,13 +91,25 @@ export type CompactionFailure = { * Why a run ended without an answer: * - `model_error`: the model's response failed, after any retries, or the compaction a request needed failed; * - `no_model`: no model is configured, or the configured one is not available; - * - `aborted`: the run was aborted (`Conversation.abort()`, `abortTask()`); + * - `aborted`: the run was aborted (`Conversation.abort()`, `abortTask()`); `detail` says when its abort handler was + * cut off `settings.restartLimit` times and did not finish; * - `reset`: a queued reset started a new context at the post-tools boundary, before an answer (a tool's handoff * continues the run); * - `faulted`: the run's task threw where it cannot recover, or storage rejected its commit; - * - `orphaned`: the run's task was aborted while no registered definition could take it. + * - `orphaned`: the run's task was aborted while no registered definition could take it; + * - `interrupted`: the host stopped while the model was answering, more often than `settings.retry` retries; + * - `restart_limit`: the run's generation was cut off `settings.restartLimit` times in a row; a request or poll it was + * cut off during counts as a failed attempt first, so this ends one whose charge itself keeps failing. */ -export type RunFailureReason = "model_error" | "no_model" | "aborted" | "reset" | "faulted" | "orphaned"; +export type RunFailureReason = + | "model_error" + | "no_model" + | "aborted" + | "reset" + | "faulted" + | "orphaned" + | "interrupted" + | "restart_limit"; /** The newest run that ended without an answer (spec §8.2). */ export type RunFailure = { @@ -108,7 +120,8 @@ export type RunFailure = { reason: RunFailureReason; /** * `model_error`: the error text; `no_model`: which model is missing; `faulted`: the error message; `orphaned`: why - * the task could not run (`missing_task`, `task_too_old`, `migration_failed`). + * the task could not run (`missing_task`, `task_too_old`, `migration_failed`); `interrupted` and `restart_limit`: what + * was cut off, and how often. */ detail?: string; }; @@ -179,6 +192,38 @@ const RUN_TASK_KINDS: ReadonlySet = new Set(["pi.generation"]); const TOOL_TASK_KIND = "pi.tool"; const COMPACTION_TASK_KIND = "pi.compaction"; +/** + * Built-in task kinds that end themselves once cut off `settings.restartLimit` times (`TaskRecord.restarts`); the + * scheduler faults a task of any other kind instead (spec §5.4). + */ +export const LIMITS_RESTARTS: ReadonlySet = new Set([...RUN_TASK_KINDS, TOOL_TASK_KIND, COMPACTION_TASK_KIND]); + +/** Whether `task` was cut off `settings.restartLimit` times in a row, so its phase ends it instead of running. */ +export function cutOffTooOften( + task: { readonly restarts?: number }, + runtime: { readonly settings: { readonly restartLimit: number } }, +): boolean { + // The settings are read only for a task that was cut off at all, so the normal path resolves nothing more. + return task.restarts !== undefined && task.restarts >= runtime.settings.restartLimit; +} + +/** + * Whether a task at the limit may still do the cleanup it does when it ends, such as converting a partial: only at the + * limit. Past it, that cleanup may be what cut the task off, so the task ends without it. + */ +export function salvageable( + task: { readonly restarts?: number }, + runtime: { readonly settings: { readonly restartLimit: number } }, +): boolean { + return (task.restarts ?? 0) <= runtime.settings.restartLimit; +} + +/** How often a task was cut off, for a message: "once", or "3 times in a row". */ +export function cutOffCount(task: { readonly restarts?: number }): string { + const restarts = task.restarts ?? 0; + return restarts === 1 ? "once" : `${restarts} times in a row`; +} + /** How a run ends: with an answer, or without one, settling its inputs `unanswered` and recording why. */ export type RunEnd = | { readonly status: "done"; readonly answer: EntryId } @@ -319,21 +364,28 @@ export async function settleSchedulerOutcome( removeNestedSlots(live, record.id); return; } + // Only an abort handler is faulted for the limit here (the built-in phases end themselves): the task was aborted, + // and keeps that meaning. Its run ends `aborted`, with the limit as detail, and an aborted compaction is no failure. + const detail = outcome.status === "faulted" ? outcome.error.detail : undefined; + const cutOff = + typeof detail === "object" && detail !== null && "reason" in detail && detail.reason === "restart_limit"; if (record.kind === COMPACTION_TASK_KIND) { - const error = outcome.status === "faulted" ? outcome.error.message : outcome.reason; - failCompactionStatus(await tx.doc(LiveDoc, record.conversationId), record.id, error); + const live = await tx.doc(LiveDoc, record.conversationId); + if (cutOff) removeCompactionStatus(live, record.id); + else failCompactionStatus(live, record.id, outcome.status === "faulted" ? outcome.error.message : outcome.reason); return; } if (!RUN_TASK_KINDS.has(record.kind)) return; const live = await tx.doc(LiveDoc, record.conversationId); if (live.run?.taskId !== record.id) return; - await convertPartial(tx, live, record.conversationId); + // Cut off, the abort handler may have died converting the partial: the run ends without it. + if (!cutOff) await convertPartial(tx, live, record.conversationId); endRun( tx, live, record.id, outcome.status === "faulted" - ? { status: "unanswered", reason: "faulted", detail: outcome.error.message } + ? { status: "unanswered", reason: cutOff ? "aborted" : "faulted", detail: outcome.error.message } : { status: "unanswered", reason: "orphaned", diff --git a/packages/durable/src/harness/scheduler.ts b/packages/durable/src/harness/scheduler.ts index 0840533a8..06fda6a3c 100644 --- a/packages/durable/src/harness/scheduler.ts +++ b/packages/durable/src/harness/scheduler.ts @@ -31,6 +31,7 @@ import type { TaskDefinition, TaskId, TaskOutcome, + TaskOutcomeError, TaskRecord, TaskRuntime, TaskState, @@ -68,10 +69,7 @@ const SCAN_PAGE_SIZE = 256; const LIVE_STATUSES = ["pending", "running", "waiting", "sleeping", "completing"] as const; /** What `blockedReason()` reads of a task: the task graph keeps only this of each record. */ -export type BlockedSubject = Pick< - AnyTaskRecord, - "id" | "kind" | "version" | "abortRequested" | "abortReason" | "restarts" -> & { +export type BlockedSubject = Pick & { readonly status: TaskState["status"]; }; @@ -83,16 +81,12 @@ export function blockedSubject(record: AnyTaskRecord): BlockedSubject { version: record.version, abortRequested: record.abortRequested, ...(record.abortReason === undefined ? {} : { abortReason: record.abortReason }), - ...(record.restarts === undefined ? {} : { restarts: record.restarts }), status: record.state.status, }; } -/** - * Why a task cannot be reserved: no registered definition can take it, or it was cut off `restartLimit` times in a row. - * Derived from the registry, the settings, and the record's `restarts`. - */ -export type BlockedReason = "missing_task" | "task_too_old" | "migration_failed" | "restart_limit"; +/** Why a task cannot be reserved: no registered definition can take it. Derived from the registry. */ +export type BlockedReason = "missing_task" | "task_too_old" | "migration_failed"; type Resolution = | { readonly kind: "ready"; readonly task: AnyTask; readonly record: RunnableTaskRecord } @@ -215,6 +209,11 @@ export type TaskSchedulerOptions = { /** While closing, on the line after admitted commits settle, write what the callback returns straight to Storage. */ readonly writeAtClose: (writes: () => readonly StorageWrite[]) => Promise; readonly report: (error: unknown, info: ReportInfo, context?: Context) => void; + /** + * Task kinds whose run phases end the task themselves once it was cut off `settings.restartLimit` times (the + * built-in tasks, from `TaskRecord.restarts`); the scheduler ends a task of any other kind `faulted` instead. + */ + readonly limitsRestarts: (kind: string) => boolean; /** Harness cleanup staged in the commit that makes an outcome the scheduler wrote itself terminal. */ readonly settleOutcome: (tx: Transaction, record: AnyTaskRecord, outcome: SchedulerOutcome) => Promise; /** Withdraw a conversation's queued inputs, for conversation abort and abort cascades. */ @@ -274,6 +273,7 @@ export class TaskScheduler { readonly #writeAtClose: (writes: () => readonly StorageWrite[]) => Promise; readonly #report: (error: unknown, info: ReportInfo, context?: Context) => void; readonly #settleOutcome: TaskSchedulerOptions["settleOutcome"]; + readonly #limitsRestarts: TaskSchedulerOptions["limitsRestarts"]; readonly #withdrawInputs: TaskSchedulerOptions["withdrawInputs"]; readonly #conversation: TaskSchedulerOptions["conversation"]; readonly #context: Context; @@ -337,8 +337,6 @@ export class TaskScheduler { readonly #blockedListeners = new Set<() => void>(); /** A registry snapshot that threw for the task graph was reported; it is read per record, so only once. */ #snapshotReported = false; - /** Tasks a pass left out as parked; each pass checks them again, as `settings.restartLimit` may have risen. */ - readonly #parkedTasks = new Set(); #reconcileScheduled = false; /** Reconcile passes underway with work (they may overlap): their commits finalize held outcomes and mark cascades. */ #reconcilingWork = 0; @@ -369,6 +367,7 @@ export class TaskScheduler { this.#writeAtClose = options.writeAtClose; this.#report = options.report; this.#settleOutcome = options.settleOutcome; + this.#limitsRestarts = options.limitsRestarts; this.#withdrawInputs = options.withdrawInputs; this.#conversation = options.conversation; this.#context = options.context; @@ -376,9 +375,9 @@ export class TaskScheduler { } /** - * Load live tasks and change surviving `running` tasks back to `pending`, counting the restart. Dispatches nothing. A - * task that reaches the restart limit this way is parked and reported. Read-only, it only loads them: a `running` - * task stays as committed, running in another process. + * Load live tasks and change surviving `running` tasks back to `pending`, counting the restart. Dispatches nothing; + * a task cut off `settings.restartLimit` times ends when it is next reserved. Read-only, it only loads them: a + * `running` task stays as committed, running in another process. */ async open(context: Context): Promise { this.#session.observeCommits((publication) => this.#observe(publication)); @@ -387,8 +386,6 @@ export class TaskScheduler { this.#notifyBlocked(); this.#kick(); }); - const limit = this.#restartLimit(); - const parked: AnyTaskRecord[] = []; await this.#session.commitWith(async (tx) => { // Every table read before the first write. const scans = []; @@ -405,7 +402,6 @@ export class TaskScheduler { const restarts = (record.restarts ?? 0) + 1; const pending = withState(record, { status: "pending", checkpoint: record.state.checkpoint }); tx.setTask({ ...pending, restarts } as AnyTaskRecord); - if (restarts >= limit && !record.abortRequested) parked.push(record); } if (record.state.status === "waiting" && record.state.policy === "failFast") { this.#failFastChecks.add(record.id); @@ -413,13 +409,6 @@ export class TaskScheduler { } } }, context); - for (const record of parked) { - const restarts = (record.restarts ?? 0) + 1; - const error = new Error( - `Task ${record.id} (${record.kind}) was cut off ${restarts} times without progress; it is parked until retryTask()`, - ); - this.#report(error, { type: "task_parked", ...this.#aboutTask(recordTask(record)) }); - } if (this.#readOnly) return; // Derive abort marks a crash left unapplied below cancelled owners, and finalize held outcomes. this.#cascadePending = true; @@ -439,30 +428,14 @@ export class TaskScheduler { } /** - * Run a task parked by the restart limit again: clears its restart count and enables scheduling. Does nothing for a - * task that is not parked; rejects for an unknown one. - */ - async retry(id: TaskId, context: Context): Promise { - await this.#session.commitWith(async (tx) => { - const current = await tx.task(id); - if (current === undefined) throw new Error(`Task ${id} does not exist`); - const status = current.state.status; - if (status === "terminal" || status === "completing" || !this.#parked(current)) return; - tx.setTask(progressed(current)); - }, context); - this.resume(); - } - - /** - * Why a live task cannot run here, if it cannot: its definition (missing, too old, failed migration) or the restart - * limit. A running task, an abort-marked one, and one holding its outcome are never blocked. + * Why a live task cannot run here, if it cannot: its definition is missing, too old, or failed to migrate. A running + * task, an abort-marked one, and one holding its outcome are never blocked. */ blockedReason(record: BlockedSubject): BlockedReason | undefined { const status = record.status; if (status === "running" || status === "completing" || status === "terminal") return undefined; // An abort request runs its handler or orphans the task; a `restart` mark waits for the definition, blocked. if (record.abortRequested && record.abortReason === undefined) return undefined; - if (this.#parked(record)) return "restart_limit"; let snapshot: RegistrySnapshot; try { snapshot = this.#registry.snapshot(); @@ -506,7 +479,6 @@ export class TaskScheduler { continue; } if (active) continue; - if (this.#parked(record)) continue; // An abort invocation waits for the live work below it, which counts itself. if (record.abortRequested && this.#waitingOn(record, true).length > 0) continue; // A task no definition can take waits for a registry change, except an abort-marked one, which is orphaned. @@ -1395,11 +1367,6 @@ export class TaskScheduler { } } - /** A task cut off `restartLimit` times in a row; an abort mark still runs its abort handler. */ - #parked(record: Pick): boolean { - return !record.abortRequested && record.restarts !== undefined && record.restarts >= this.#restartLimit(); - } - /** * The report info of a task's failure, with the extension that installed its definition: the code that failed is * that extension's. Built-in tasks have none. @@ -1425,20 +1392,6 @@ export class TaskScheduler { return { task: reportTask(task), ...optionalExtension(extension) }; } - /** Make parked tasks that a raised `settings.restartLimit` no longer parks candidates again, and tell the graph. */ - #unparkRaised(): void { - let unparked = false; - for (const id of this.#parkedTasks) { - const record = this.#live.get(id); - if (record !== undefined && this.#parked(record)) continue; - this.#parkedTasks.delete(id); - if (record === undefined) continue; - this.#runnable.add(id); - unparked = true; - } - if (unparked) queueMicrotask(() => this.#notifyBlocked()); - } - /** `settings.restartLimit`, or the default when the host's settings throw. */ #restartLimit(): number { try { @@ -1515,7 +1468,6 @@ export class TaskScheduler { // Every pass picks up sleeps whose time has come: a pass started by any change may be the first to run after // a deadline, and the wake it schedules no longer keeps a timer for a due sleep. this.#wakeDue(); - this.#unparkRaised(); // Taken once per pass, and only when some task is a candidate. let snapshot: RegistrySnapshot | undefined; for (const id of [...this.#runnable]) { @@ -1534,14 +1486,6 @@ export class TaskScheduler { this.#runnable.delete(id); continue; } - // It returns when `retry()` clears its restart count, or at a pass after `settings.restartLimit` rose. - if (this.#parked(record)) { - this.#runnable.delete(id); - // Newly parked without a commit, as after a lowered limit: the graph must hear it. - if (!this.#parkedTasks.has(id)) queueMicrotask(() => this.#notifyBlocked()); - this.#parkedTasks.add(id); - continue; - } const runnable = record as RunnableTaskRecord; const mode = record.abortRequested ? "abort" : "run"; // Work below an owner with cancellation intent waits for its cascade mark instead of running a phase; schedule @@ -1551,9 +1495,15 @@ export class TaskScheduler { this.#scheduleReconcile(); continue; } - // An abort handler cut off `restartLimit` times in a row would run forever; nothing is left to park it for. - if (mode === "abort" && (record.restarts ?? 0) >= this.#restartLimit()) { - const message = `Abort handler of task ${id} (${record.kind}) was cut off ${record.restarts} times`; + // Work cut off `restartLimit` times in a row would run forever: an abort handler, or a phase of a kind that + // does not end itself at the limit, ends the task `faulted` instead. + // The settings are read only for a task that was cut off at all. + const cutOff = record.restarts !== undefined && record.restarts >= this.#restartLimit(); + if (cutOff && (mode === "abort" || !this.#limitsRestarts(record.kind))) { + const message = + mode === "abort" + ? `Abort handler of task ${id} (${record.kind}) was cut off ${record.restarts} times` + : `Task ${id} (${record.kind}) was cut off ${record.restarts} times without progress`; await this.#terminate(tx, record, { status: "faulted", error: { message, detail: { reason: "restart_limit" } }, @@ -1689,7 +1639,6 @@ export class TaskScheduler { const on = this.#waitingOn(record); if (on.length > 0) return { kind: "waiting", on }; if (this.#notDue(record)) return { kind: "sleeping", until: record.state.until }; - if (this.#parked(record)) return { kind: "blocked", reason: "restart_limit" }; const fit = this.#fit(record, snapshot.task(record.kind)); if ("reason" in fit) return { kind: "blocked", ...fit }; if (fit.migrates && fit.task.definition.migrate === undefined) { @@ -2165,6 +2114,7 @@ export class TaskScheduler { }, commit: async (change, context, options) => { let checkpoint: string | undefined; + let interrupted: { readonly error: TaskOutcomeError; readonly restarts: number } | undefined; await this.#gated( invocation, async (tx, current) => { @@ -2173,10 +2123,28 @@ export class TaskScheduler { if (next?.status === "running" && current.state.checkpoint.phase !== next.checkpoint.phase) { checkpoint = next.checkpoint.phase; } + // A task that ends for crashes says so (`task_interrupted`), so a host keeps a crash-loop signal. + if (next?.status === "terminal" && next.outcome.status === "failed") { + if (interruptionOf(next.outcome.error) !== undefined) { + interrupted = { error: next.outcome.error, restarts: current.restarts ?? 0 }; + } + } }, context, options, ); + if (interrupted !== undefined) { + this.#report( + new Error(interrupted.error.message), + { + type: "task_interrupted", + ...this.#aboutTask(invocation), + reason: interruptionOf(interrupted.error)!, + restarts: interrupted.restarts, + }, + invocation.context, + ); + } if (checkpoint !== undefined) { invocation.context.telemetry.addEvent("pi.durable.checkpoint", { phase: checkpoint, @@ -2427,6 +2395,13 @@ function recordTask(record: Pick { const call = await readCall(runtime, task.input, context); + if (cutOffTooOften(task, runtime)) return endCutOff(runtime, task, call, context); const tool = await resolveTool(runtime, task.input, call.name, context); if (tool === undefined) { const error = harnessError("tool_unavailable", `Tool ${call.name} is not available`); @@ -184,6 +187,7 @@ export const ToolTask = defineTask { const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId); @@ -212,6 +216,21 @@ export const ToolTask = defineTask { + const message = `Tool ${call.name} was interrupted ${cutOffCount(task)}; it is not run again`; + const ending = { status: "failed", message, reason: "restart_limit" } as const; + await settle(runtime, task.input, call, ending, (slot) => fromSlot(slot, "restart_limit", message), context); +} + /** The called tool: among the tools the model is offered for a model-issued call, else among those tools may call. */ async function resolveTool( runtime: Runtime, @@ -699,7 +718,10 @@ async function settle( if (madeNested) removeNestedSlots(live, runtime.taskId); if (ending.status === "aborted") return { status: "terminal", outcome: { status: "aborted", result: settled } }; if (ending.status === "failed") { - const error = { message: ending.message }; + const error = { + message: ending.message, + ...(ending.reason === undefined ? {} : { detail: { reason: ending.reason } }), + }; return { status: "terminal", outcome: { status: "failed", error, result: settled } }; } return { status: "terminal", outcome: { status: "completed", result: settled } }; @@ -826,7 +848,9 @@ function fallbackResult(taskId: TaskId, name: string, outcome: TaskOutcome = { /** * Receives failures that do not fail the calling operation, such as a throwing hook or listener, a task that faulted, * and the error that fails the Session, once. `info` says what happened: a host can tell a failed Session - * (`session_failed`, often a platform reset) from a faulted task (`task_faulted`) or a parked one (`task_parked`). + * (`session_failed`, often a platform reset) from a faulted task (`task_faulted`). * Its own throw or rejection is dropped. */ readonly onReport?: (error: unknown, info: ReportInfo) => void; @@ -711,13 +714,10 @@ export type TaskInspection = { | { readonly kind: "sleeping"; readonly until: number } /** Outcome held until its ordinary owned work drains. */ | { readonly kind: "completing" } - /** - * Cannot run here: no registered definition can take it (aborting it settles it as `orphaned`), or it is parked by - * the restart limit (`retryTask()` runs it again). - */ + /** Cannot run here: no registered definition can take it (aborting it settles it as `orphaned`). */ | { readonly kind: "blocked"; - readonly reason: "missing_task" | "task_too_old" | "migration_failed" | "restart_limit"; + readonly reason: "missing_task" | "task_too_old" | "migration_failed"; readonly error?: unknown; }; }; @@ -845,11 +845,6 @@ export interface Harness extends Session { * definition cannot take it settles as `orphaned` instead. */ abortTask(id: TaskId, context: Context): Promise<"marked" | "terminal">; - /** - * Run a task parked by `settings.restartLimit` again: clears its restart count and enables scheduling. Does nothing - * for a task that is not parked; rejects for an unknown one. - */ - retryTask(id: TaskId, context: Context): Promise; /** Resolve with the terminal receipt; cancelling `context` cancels only this wait. */ waitForTask(id: TaskId, context: Context): Promise>; /** Resolve when the ordinary ownership scope of every ownerless conversation has no live non-background task. */ diff --git a/packages/durable/src/types.ts b/packages/durable/src/types.ts index 97bdcb75e..e1c9c6453 100644 --- a/packages/durable/src/types.ts +++ b/packages/durable/src/types.ts @@ -153,8 +153,10 @@ export type HostCallback = "now" | "settings" | "registry" | "env" | "models" | * failed, when known; for a task's own code, the one that installed the definition the task ran. * - `session_failed`: the error failed the Session, which closes: a Storage error, often a platform reset. * - `task_faulted`: the task ended `faulted`: its phase threw (`reason: "threw"`), or its abort handler was cut off - * `settings.restartLimit` times (`reason: "restart_limit"`). - * - `task_parked`: the restart limit parked the task until `Harness.retryTask()`. + * `settings.restartLimit` times (`reason: "restart_limit"`), as was a phase of a task of an extension's kind. + * - `task_interrupted`: crashes ended the task: it ended itself after being cut off `settings.restartLimit` times in a + * row (`reason: "restart_limit"`; a tool call, a compaction, a generation), or a generation's run ended because the + * host kept stopping while the model answered (`reason: "interrupted"`). `restarts` is the task's count when it ended. * - `migration_failed`: the task's stored state could not be migrated; it is blocked until the registry changes. * - `task_outdated`: a running task keeps running under its old definition, which was uninstalled (`missing_task`) * or replaced by one that cannot take over its state (`incompatible_task`). @@ -172,7 +174,13 @@ export type ReportInfo = readonly extension?: string; readonly reason: "threw" | "restart_limit"; } - | { readonly type: "task_parked"; readonly task: ReportTask; readonly extension?: string } + | { + readonly type: "task_interrupted"; + readonly task: ReportTask; + readonly extension?: string; + readonly reason: "restart_limit" | "interrupted"; + readonly restarts: number; + } | { readonly type: "migration_failed"; readonly task: ReportTask; readonly extension?: string } | { readonly type: "task_outdated"; @@ -686,8 +694,9 @@ type TaskRecordBase = { readonly endedAt?: number; /** * How many times in a row a crash, eviction, or reset cut the task off while it ran, without the task changing its - * state since; open counts them, and a close does not. At `HarnessSettings.restartLimit` the task is parked as blocked - * until `Harness.retryTask()`. + * state since; open counts them, and a close does not. At `HarnessSettings.restartLimit` the task ends (the built-in + * tasks end themselves, visibly; the scheduler faults any other kind), and a generation reads it to charge an + * interrupted model request as a failed attempt. */ readonly restarts?: number; }; diff --git a/packages/durable/test-workers/wake.test.ts b/packages/durable/test-workers/wake.test.ts index a28a739c9..fb991edf7 100644 --- a/packages/durable/test-workers/wake.test.ts +++ b/packages/durable/test-workers/wake.test.ts @@ -59,7 +59,7 @@ describe("a Harness woken by its Durable Object alarm", () => { await eventually(stub, async (_instance, state) => (await state.storage.getAlarm()) === null); }); - it("recovers a turn an eviction cut off mid-stream through the heartbeat alarm", async () => { + it("recovers a turn an eviction cut off mid-stream through the alarms, retrying the charged attempt", async () => { const stub = fresh(); await runInDurableObject(stub, async (instance: TestSession, state) => { instance.script(["hang", "recovered"]); diff --git a/packages/durable/test-workers/worker.ts b/packages/durable/test-workers/worker.ts index 24d0c5ae2..ce6ea8e94 100644 --- a/packages/durable/test-workers/worker.ts +++ b/packages/durable/test-workers/worker.ts @@ -66,6 +66,8 @@ export class TestSession extends DurableObject { { models, registry, + // An eviction during a request is a failed attempt (#11139); its retry need not wait long here. + settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } }, now: () => this.now(), onWake: (wake) => { this.wakes.push(wake); diff --git a/packages/durable/test/chat-support.ts b/packages/durable/test/chat-support.ts index 06b019d3d..1e9538ee1 100644 --- a/packages/durable/test/chat-support.ts +++ b/packages/durable/test/chat-support.ts @@ -16,6 +16,7 @@ import { type HarnessOptions, type HarnessSettings, type Registry, + type ReportInfo, type Storage, type ToolRegistration, } from "@earendil-works/pi-durable"; @@ -28,6 +29,8 @@ export type ChatSetup = { readonly models: Models; readonly registry: Registry; readonly reports: unknown[]; + /** What each report was about, in the order of `reports`. */ + readonly infos: ReportInfo[]; /** Live Harness settings; tests assign fields to change them between decisions. */ readonly settings: { -readonly [K in keyof HarnessSettings]: HarnessSettings[K] }; now: () => number; @@ -39,7 +42,7 @@ export function chatSetup(options: RegisterFauxProviderOptions = {}): ChatSetup const faux = fauxProvider(options); const models = createModels(); models.setProvider(faux.provider); - return { faux, models, registry: createRegistry(), reports: [], settings: {}, now: () => Date.now() }; + return { faux, models, registry: createRegistry(), reports: [], infos: [], settings: {}, now: () => Date.now() }; } /** Open a Harness over `storage` and return its root, configured with the faux model on first creation. */ @@ -60,7 +63,10 @@ export async function openChat( ? {} : { env: typeof options.env === "function" ? options.env : () => options.env as ExecutionEnv }), now: () => setup.now(), - onReport: (error) => setup.reports.push(error), + onReport: (error, info) => { + setup.reports.push(error); + setup.infos.push(info); + }, }, context, ); diff --git a/packages/durable/test/harness-blocked.test.ts b/packages/durable/test/harness-blocked.test.ts index 6cd234cc6..50ab0cb43 100644 --- a/packages/durable/test/harness-blocked.test.ts +++ b/packages/durable/test/harness-blocked.test.ts @@ -132,7 +132,8 @@ describe("recovery at open", () => { }); describe("tasks that cannot run", () => { - it("parks a task cut off restartLimit times in a row, shows why, and runs it again on retryTask()", async () => { + // #11139 + it("faults an extension's task cut off restartLimit times in a row instead of running it again", async () => { const path = await sqlitePath(); const state = { crashing: true, started: 0 }; const registry = createRegistry(); @@ -150,31 +151,29 @@ describe("tasks that cannot run", () => { } const { harness, infos } = opened; expect((await harness.getTask(id, context))?.restarts).toBe(3); - expect(infos).toEqual([ - { type: "task_parked", task: { id, kind: "test.crash-loop", conversationId: root.id }, extension: "loop" }, - ]); + // Nothing is parked: the task is ready, and the host is asked to run, which ends it. const inspection = await harness.inspect(context); - expect(inspection.tasks.find((task) => task.record.id === id)?.state).toEqual({ - kind: "blocked", - reason: "restart_limit", - }); - const graph = await harness.watchTaskGraph(context); - // A started watch keeps its value current. - graph.start(async () => {}); - expect(graph.value.tasks[String(id)]?.blocked).toBe("restart_limit"); - // Parked work needs no process: nothing is active. - await eventually(() => !harness.wake().active); - expect(harness.wake()).toEqual({ active: false }); + expect(inspection.tasks.find((task) => task.record.id === id)?.state).toEqual({ kind: "ready", migrates: false }); + expect(harness.wake().active).toBe(true); harness.resume(); - // A scheduling pass runs within a few turns; it leaves the parked task out. - for (let turn = 0; turn < 5; turn++) await flush(); + const settled = await harness.waitForTask(id, context); + expect(settled.state.outcome).toEqual({ + status: "faulted", + error: { + message: `Task ${id} (test.crash-loop) was cut off 3 times without progress`, + detail: { reason: "restart_limit" }, + }, + }); expect(state.started).toBe(3); - - state.crashing = false; - await harness.retryTask(id, context); - expect((await harness.waitForTask(id, context)).state.outcome.status).toBe("completed"); - expect(state.started).toBe(4); - await graph.stop(); + await eventually(() => infos.length > 0); + expect(infos).toEqual([ + { + type: "task_faulted", + task: { id, kind: "test.crash-loop", conversationId: root.id }, + extension: "loop", + reason: "restart_limit", + }, + ]); await harness.close(context); }); @@ -268,7 +267,7 @@ describe("tasks that cannot run", () => { await opened.harness.close(context); }); - it("runs a parked task once a raised restartLimit no longer parks it", async () => { + it("reads restartLimit when it reserves the task: a raised one runs it again", async () => { const path = await sqlitePath(); const state = { crashing: true, started: 0 }; const registry = createRegistry(); @@ -284,24 +283,17 @@ describe("tasks that cannot run", () => { await eventually(() => state.started === 1); opened.kill(); // a crash: a close would not count opened = await openWith(path, registry, settings); - const graph = await opened.harness.watchTaskGraph(context); - // A started watch keeps its value current. - graph.start(async () => {}); - opened.harness.resume(); - await eventually(() => graph.value.tasks[String(id)]?.blocked === "restart_limit"); - // Let a scheduling pass leave the parked task out. - for (let turn = 0; turn < 5; turn++) await flush(); + expect((await opened.harness.getTask(id, context))?.restarts).toBe(1); state.crashing = false; - // The host's live settings change; the next pass runs the task. + // The host's live settings change before the task is reserved; it runs instead of ending. settings.restartLimit = Number.POSITIVE_INFINITY; opened.harness.resume(); expect((await opened.harness.waitForTask(id, context)).state.outcome.status).toBe("completed"); expect(state.started).toBe(2); - await graph.stop(); await opened.harness.close(context); }); - it("still runs the abort handler of a parked task", async () => { + it("runs the abort handler of a task at the limit", async () => { const path = await sqlitePath(); const state = { crashing: true, started: 0 }; const aborted: TaskId[] = []; @@ -362,10 +354,12 @@ describe("tasks that cannot run", () => { opened = await openWith(path, registry, { restartLimit: 2 }); } expect((await opened.harness.getTask(id, context))?.restarts).toBe(2); - expect((await opened.harness.inspect(context)).tasks[0]?.state).toEqual({ - kind: "blocked", - reason: "restart_limit", + opened.harness.resume(); + expect((await opened.harness.waitForTask(id, context)).state.outcome).toMatchObject({ + status: "faulted", + error: { detail: { reason: "restart_limit" } }, }); + expect(started).toBe(2); await opened.harness.close(context); }); diff --git a/packages/durable/test/harness-deferred-progress.test.ts b/packages/durable/test/harness-deferred-progress.test.ts index 8f6221af7..0a410a7af 100644 --- a/packages/durable/test/harness-deferred-progress.test.ts +++ b/packages/durable/test/harness-deferred-progress.test.ts @@ -13,7 +13,6 @@ import { type Harness, LiveDoc, type LiveState, - type TaskId, ToolResultEntry, } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; @@ -28,6 +27,8 @@ import { aborted, deferred } from "./task-support.ts"; function slowSetup(persistWithinMs: number): ChatSetup { const setup = chatSetup({ tokensPerSecond: 80, tokenSize: { min: 4, max: 4 } }); setup.settings.progress = { persistWithinMs }; + // A request a crash interrupted is a failed attempt (#11139): retry at once. + setup.settings.retry = { enabled: true, maxRetries: 5, baseDelayMs: 1 }; return setup; } @@ -92,10 +93,6 @@ function answers(entries: readonly EntryRecord[]): { text: string | undefined; s }); } -async function generationId(harness: Harness, conversationId: ConversationId): Promise { - return (await harness.snapshot(LiveDoc, conversationId, context))!.run!.taskId; -} - describe("generation partials", () => { it("hold partials in memory and store them together, at most every persistWithinMs", async () => { const storage = new ControlledStorage(); @@ -113,7 +110,7 @@ describe("generation partials", () => { await harness.close(context); }); - it("converts what was stored into the aborted entry after a crash, and resends", async () => { + it("converts what was stored into the aborted entry after a crash, and retries the charged attempt", async () => { const storage = new ControlledStorage(); const setup = slowSetup(150); setup.faux.setResponses([fauxAssistantMessage(LONG)]); @@ -217,23 +214,6 @@ describe("generation partials", () => { await opened.harness.close(context); }); - it("never counts partial commits as progress of a task cut off again and again", async () => { - const storage = new ControlledStorage(); - const setup = slowSetup(50); - let opened = await open(storage, setup); - setup.faux.setResponses([fauxAssistantMessage(LONG)]); - await opened.root.submit({ type: "input", content: "hi" }, context); - const id = await generationId(opened.harness, opened.root.id); - for (let crashes = 1; crashes <= 2; crashes++) { - await waitFor(async () => (storedText(await storedLive(storage, opened.root.id))?.length ?? 0) > 40); - await crash(opened.harness); - setup.faux.setResponses([fauxAssistantMessage(LONG)]); - opened = await open(storage, setup); - expect((await opened.harness.getTask(id, context))?.restarts).toBe(crashes); - } - await opened.harness.close(context); - }); - it("resumes polling a deferred response after a crash", async () => { const storage = new ControlledStorage(); const setup = chatSetup({ deferred: { pollAfterMs: 60_000 } }); @@ -306,6 +286,47 @@ describe("tool output", () => { }); } + // #11139: a generation's request is charged at once after a crash, so its partials never stream with a count; a + // replay-safe tool reruns and streams its output with one. + it("never counts progress commits as progress of a task cut off again and again", async () => { + const storage = new ControlledStorage(); + const setup = slowSetup(50); + let runs = 0; + addTool( + setup.registry, + defineTool({ + name: "work", + description: "work", + parameters: Type.Object({}), + replay: "safe", + execute: async (_args, api, callContext) => { + runs++; + for (let line = 0; !callContext.abortSignal!.aborted; line++) { + api.output(`run ${runs} line ${line}\n`); + await new Promise((resolve) => setTimeout(resolve, 20)); + } + return aborted(callContext.abortSignal!); + }, + }), + ); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("work", {}, { id: "c1" })], { stopReason: "toolUse" }), + ]); + let opened = await open(storage, setup); + await opened.root.submit({ type: "input", content: "go" }, context); + for (let crashes = 1; crashes <= 2; crashes++) { + // This run stored output of its own, a progress commit, before it was cut off. + await waitFor( + async () => (await storedOutput(storage, opened.root.id))?.startsWith(`run ${crashes} `) === true, + ); + await crash(opened.harness); + opened = await open(storage, setup); + const tool = (await opened.harness.commit((tx) => tx.scanTasks({ kind: "pi.tool" }, 1), context)).items[0]!; + expect(tool.restarts).toBe(crashes); + } + await opened.harness.close(context); + }); + it("answers a tool without replay that crashed with the output stored by then", async () => { const storage = new ControlledStorage(); const setup = slowSetup(150); diff --git a/packages/durable/test/harness-restart-failure.test.ts b/packages/durable/test/harness-restart-failure.test.ts new file mode 100644 index 000000000..9bc220a1e --- /dev/null +++ b/packages/durable/test/harness-restart-failure.test.ts @@ -0,0 +1,775 @@ +// What a crash (a host that dies, is evicted, or is reset) does to work that was running: a generation's interrupted +// request is a failed attempt under the retry policy, and every other task ends visibly once it was cut off +// `settings.restartLimit` times in a row (#11139). +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + type AssistantMessage, + type FauxResponseStep, + fauxAssistantMessage, + fauxText, + fauxToolCall, + type Message, + type Models, + type ToolResultMessage, + Type, +} from "@earendil-works/pi-ai"; +import { + defineTool, + type GenerationCheckpoint, + type Harness, + LiveDoc, + type LiveState, + MemoryStorage, + ROOT_CONVERSATION_ID, + type Storage, + type SubmissionId, + type TaskId, +} from "@earendil-works/pi-durable"; +import { afterEach, describe, expect, it } from "vitest"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { allEntries, type ChatSetup, chatSetup, openChat, textOf, unanswered, waitFor } from "./chat-support.ts"; +import { addSection, addTool } from "./harness-support.ts"; +import { context, observeCommits } from "./session-support.ts"; +import { aborted, deferred } from "./task-support.ts"; + +const directories = new Set(); +/** Storages of killed Harnesses, never closed by them: closed after each test. */ +const killed = new Set(); + +afterEach(async () => { + for (const storage of killed) await storage.close(context).catch(() => {}); + killed.clear(); + for (const directory of directories) await rm(directory, { recursive: true, force: true }); + directories.clear(); +}); + +async function sqlitePath(): Promise { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-restart-")); + directories.add(directory); + return join(directory, "session.sqlite"); +} + +/** + * Open a chat over the SQLite file at `path`. `kill()` simulates a crash: every later storage call hangs, so the Harness + * writes nothing more, not even the `pending` records a close writes for the tasks it stops. + */ +async function open(path: string, setup: ChatSetup) { + const inner = await openNodeSqliteStorage(path); + let dead = false; + const storage = new Proxy(inner, { + get(target, property) { + const value = Reflect.get(target, property, target); + if (typeof value !== "function") return value; + return (...args: unknown[]) => (dead ? new Promise(() => {}) : value.apply(target, args)); + }, + }); + const opened = await openChat(storage, setup); + const kill = (): void => { + dead = true; + killed.add(inner); + }; + return { ...opened, kill }; +} + +async function live(harness: Harness): Promise> { + return (await harness.snapshot(LiveDoc, ROOT_CONVERSATION_ID, context)) ?? {}; +} + +/** Every `pi.live` value from now on, as memory adopts it. */ +function liveValues(harness: Harness): LiveState[] { + const values: LiveState[] = []; + observeCommits(harness, (publication) => { + for (const change of publication.changes) { + if (change.type === "document" && change.record.kind === "pi.live" && change.value !== null) { + values.push(change.value as LiveState); + } + } + }); + return values; +} + +/** Let deferred progress commits (`persistWithinMs: 0`) reach Storage before a crash. */ +function stored(): Promise { + return new Promise((resolve) => setTimeout(resolve, 50)); +} + +/** A setup whose model streams a long answer slowly; partials are stored as soon as they are committed. */ +function streaming(): ChatSetup { + const setup = chatSetup({ tokensPerSecond: 20, tokenSize: { min: 1, max: 1 } }); + setup.faux.setResponses([fauxAssistantMessage("z".repeat(400))]); + setup.settings.progress = { persistWithinMs: 0 }; + return setup; +} + +/** Submit an input to a fresh chat at `path` and crash once its streamed partial is stored. */ +async function crashWhileStreaming(path: string, setup: ChatSetup) { + const opened = await open(path, setup); + opened.harness.resume(); + const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; + await waitFor(async () => (await live(opened.harness)).generation?.message !== undefined); + const taskId = (await live(opened.harness)).run!.taskId; + await stored(); + opened.kill(); + return { id, taskId }; +} + +/** Reopen after a crash in the middle of a request, with another streaming answer, and crash again. */ +async function crashAgain(path: string, setup: ChatSetup, taskId: TaskId) { + const opened = await open(path, setup); + opened.harness.resume(); + // The first attempt was charged; the second one streams. + await waitFor(async () => { + const state = (await opened.harness.getTask(taskId, context))?.state; + if (state?.status !== "running") return false; + const checkpoint = state.checkpoint as GenerationCheckpoint; + return checkpoint.phase === "request" && checkpoint.attempt === 2; + }); + await waitFor(async () => (await live(opened.harness)).generation?.message !== undefined); + await stored(); + opened.kill(); +} + +async function settled(harness: Harness, id: SubmissionId) { + return (await harness.submission(id, context))!.wait(context); +} + +describe("a generation interrupted while the model answers (#11139)", () => { + it("charges the attempt: keeps the partial, waits for the retry backoff, and answers on the next attempt", async () => { + const path = await sqlitePath(); + const first = streaming(); + first.settings.retry = { enabled: true, maxRetries: 2, baseDelayMs: 1 }; + const { id, taskId } = await crashWhileStreaming(path, first); + + const setup = chatSetup(); + setup.settings.retry = { enabled: true, maxRetries: 2, baseDelayMs: 1 }; + const sent: string[][] = []; + setup.faux.setResponses([ + (request) => { + sent.push(request.messages.map((message) => message.role)); + return fauxAssistantMessage("answer"); + }, + ]); + const opened = await open(path, setup); + const values = liveValues(opened.harness); + const partial = textOf((await live(opened.harness)).generation?.message as Message)!; + expect((await opened.harness.getTask(taskId, context))?.restarts).toBe(1); + opened.harness.resume(); + expect((await settled(opened.harness, id)).status).toBe("done"); + // The backoff showed in `pi.live` with why. + const retry = values.find((value) => value.generation?.retry !== undefined)?.generation; + expect(retry).toEqual({ + attempt: 1, + retry: { at: expect.any(Number), error: "The host stopped while the model was answering, on attempt 1" }, + }); + expect(values.some((value) => value.generation?.attempt === 2)).toBe(true); + const entries = await allEntries(opened.root); + expect(entries.map((entry) => entry.kind)).toEqual(["pi.user", "pi.assistant", "pi.assistant"]); + const converted = entries[1]!.model![0] as AssistantMessage; + expect(converted.stopReason).toBe("aborted"); + expect(textOf(converted)).toBe(partial); + // The aborted partial is excluded from the next request. + expect(sent).toEqual([["user"]]); + expect(await live(opened.harness)).toEqual({}); + await opened.harness.close(context); + }); + + it("sleeps the backoff of `settings.retry` before the next attempt", async () => { + const path = await sqlitePath(); + const first = streaming(); + first.settings.retry = { enabled: true, maxRetries: 2, baseDelayMs: 60_000 }; + const { taskId } = await crashWhileStreaming(path, first); + + const setup = chatSetup(); + setup.settings.retry = { enabled: true, maxRetries: 2, baseDelayMs: 60_000 }; + const now = 1_000_000; + setup.now = () => now; + const opened = await open(path, setup); + opened.harness.resume(); + await waitFor(async () => (await opened.harness.getTask(taskId, context))?.state.status === "sleeping"); + const record = await opened.harness.getTask(taskId, context); + expect(record?.state).toMatchObject({ status: "sleeping", until: now + 60_000, checkpoint: { phase: "retry" } }); + expect(record?.restarts).toBeUndefined(); + expect((await live(opened.harness)).generation).toEqual({ + attempt: 1, + retry: { at: now + 60_000, error: "The host stopped while the model was answering, on attempt 1" }, + }); + expect(opened.harness.wake()).toEqual({ active: false, at: now + 60_000 }); + await opened.harness.close(context); + }); + + it("ends the run interrupted once no retry is left, keeping every partial", async () => { + const path = await sqlitePath(); + const first = streaming(); + first.settings.retry = { enabled: true, maxRetries: 1, baseDelayMs: 1 }; + const { id, taskId } = await crashWhileStreaming(path, first); + // Attempt 2 is interrupted too. + const second = streaming(); + second.settings.retry = { enabled: true, maxRetries: 1, baseDelayMs: 1 }; + await crashAgain(path, second, taskId); + + const setup = chatSetup(); + setup.settings.retry = { enabled: true, maxRetries: 1, baseDelayMs: 1 }; + const opened = await open(path, setup); + opened.harness.resume(); + const text = "The host stopped while the model was answering, on attempt 2"; + expect(await settled(opened.harness, id)).toMatchObject({ + status: "unanswered", + reason: "interrupted", + detail: text, + }); + expect(await live(opened.harness)).toEqual({ + runFailure: { taskId, inputs: [id], reason: "interrupted", detail: text }, + }); + expect((await opened.harness.waitForTask(taskId, context)).state.outcome).toEqual({ + status: "failed", + error: { message: text, detail: { reason: "interrupted" } }, + }); + const entries = await allEntries(opened.root); + expect(entries.map((entry) => entry.kind)).toEqual(["pi.user", "pi.assistant", "pi.assistant"]); + expect(entries.slice(1).map((entry) => (entry.model![0] as AssistantMessage).stopReason)).toEqual([ + "aborted", + "aborted", + ]); + // No request was sent after the second crash. + expect(setup.faux.state.callCount).toBe(0); + await opened.harness.close(context); + }); + + it("ends the run at once when retries are disabled", async () => { + const path = await sqlitePath(); + const first = streaming(); + const { id } = await crashWhileStreaming(path, first); + const setup = chatSetup(); + setup.settings.retry = { enabled: false }; + const opened = await open(path, setup); + opened.harness.resume(); + expect(await settled(opened.harness, id)).toMatchObject({ status: "unanswered", reason: "interrupted" }); + await opened.harness.close(context); + }); + + it("resends after a clean close: a close is not a crash", async () => { + const path = await sqlitePath(); + const first = streaming(); + first.settings.retry = { enabled: false }; + const opened = await open(path, first); + opened.harness.resume(); + const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; + await waitFor(async () => (await live(opened.harness)).generation?.message !== undefined); + const taskId = (await live(opened.harness)).run!.taskId; + await opened.harness.close(context); + + const setup = chatSetup(); + setup.settings.retry = { enabled: false }; + setup.faux.setResponses([fauxAssistantMessage("answer")]); + const reopened = await open(path, setup); + expect((await reopened.harness.getTask(taskId, context))?.restarts).toBeUndefined(); + reopened.harness.resume(); + expect((await settled(reopened.harness, id)).status).toBe("done"); + await reopened.harness.close(context); + }); + + it("does not charge a generation that sleeps in a retry or waits for its tools", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.retry = { enabled: true, maxRetries: 1, baseDelayMs: 60_000 }; + const gate = deferred(); + const started = deferred(); + addTool( + setup.registry, + defineTool({ + name: "hold", + description: "Waits", + parameters: Type.Object({}), + replay: "safe", + execute: async () => { + started.resolve(); + await gate.promise; + return {}; + }, + }), + ); + const error = fauxAssistantMessage([], { stopReason: "error", errorMessage: "503 Service Unavailable" }); + setup.faux.setResponses([error]); + let opened = await open(path, setup); + opened.harness.resume(); + const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; + const taskId = (await live(opened.harness)).run!.taskId; + await waitFor(async () => (await opened.harness.getTask(taskId, context))?.state.status === "sleeping"); + opened.kill(); + + // Asleep in its retry: not running, so not cut off. + const later = chatSetup(); + later.settings.retry = { enabled: true, maxRetries: 1, baseDelayMs: 60_000 }; + later.now = () => Date.now() + 120_000; + addTool(later.registry, setup.registry.snapshot().tools()[0]!.tool); + later.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("hold", {}, { id: "c1" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("done")]), + ]); + opened = await open(path, later); + expect((await opened.harness.getTask(taskId, context))?.restarts).toBeUndefined(); + opened.harness.resume(); + // Waiting for its tool round: not running either. + await started.promise; + const waiting = await opened.harness.getTask(taskId, context); + expect(waiting?.state).toMatchObject({ status: "waiting", checkpoint: { phase: "tools" } }); + opened.kill(); + + opened = await open(path, later); + expect((await opened.harness.getTask(taskId, context))?.restarts).toBeUndefined(); + gate.resolve(); + opened.harness.resume(); + expect((await settled(opened.harness, id)).status).toBe("done"); + await opened.harness.close(context); + }); + + it("ends a run whose generation is cut off restartLimit times outside a request", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.restartLimit = 1; + const reached = deferred(); + // Preparation never finishes, as one that exhausts the host's memory would not. + addSection(setup.registry, "preamble", async (_input, ctx) => { + reached.resolve(); + return aborted(ctx.abortSignal!); + }); + let opened = await open(path, setup); + opened.harness.resume(); + const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; + await reached.promise; + const taskId = (await live(opened.harness)).run!.taskId; + opened.kill(); + + opened = await open(path, setup); + opened.harness.resume(); + const text = "The generation was interrupted once in its prepare phase"; + expect(await settled(opened.harness, id)).toMatchObject({ + status: "unanswered", + reason: "restart_limit", + detail: text, + }); + expect(await live(opened.harness)).toEqual({ + runFailure: { taskId, inputs: [id], reason: "restart_limit", detail: text }, + }); + await waitFor(() => setup.infos.length > 0); + expect(setup.infos).toContainEqual( + expect.objectContaining({ type: "task_interrupted", reason: "restart_limit", restarts: 1 }), + ); + await opened.harness.close(context); + }); +}); + +describe("other tasks cut off restartLimit times in a row (#11139)", () => { + it("ends a tool call with an error result instead of running it again", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.restartLimit = 2; + setup.settings.progress = { outputIntervalMs: 0, persistWithinMs: 0 }; + let runs = 0; + const running = { current: deferred() }; + addTool( + setup.registry, + defineTool({ + name: "heavy", + description: "Never finishes", + parameters: Type.Object({}), + replay: "safe", + execute: async (_args, api, ctx) => { + runs++; + api.output("partial\n"); + running.current.resolve(); + return aborted(ctx.abortSignal!); + }, + }), + ); + const answers: FauxResponseStep[] = [ + () => fauxAssistantMessage([fauxToolCall("heavy", {}, { id: "c1" })], { stopReason: "toolUse" }), + (request) => { + const result = request.messages.at(-1) as ToolResultMessage; + return fauxAssistantMessage(`saw: ${textOf(result)}`); + }, + ]; + setup.faux.setResponses(answers); + let opened = await open(path, setup); + opened.harness.resume(); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + for (let crash = 1; crash <= 2; crash++) { + await running.current.promise; + running.current = deferred(); + await stored(); + opened.kill(); + opened = await open(path, setup); + opened.harness.resume(); + } + expect((await settled(opened.harness, id)).status).toBe("done"); + expect(runs).toBe(2); + const result = (await allEntries(opened.root)).find((entry) => entry.kind === "pi.tool-result")!; + const message = result.model![0] as ToolResultMessage; + expect(message.isError).toBe(true); + expect(result.data).toMatchObject({ + diagnostics: [ + { + severity: "error", + code: "restart_limit", + message: "Tool heavy was interrupted 2 times in a row; it is not run again", + }, + ], + }); + // The slot's partial output is kept, as for an interrupted call. + expect(textOf(message)).toContain("partial"); + const toolTask = (await opened.harness.commit((tx) => tx.scanTasks({ kind: "pi.tool" }, 10), context)).items[0]!; + expect(toolTask.state).toMatchObject({ + status: "terminal", + outcome: { status: "failed", error: { detail: { reason: "restart_limit" } } }, + }); + expect(setup.infos).toContainEqual({ + type: "task_interrupted", + task: { id: toolTask.id, kind: "pi.tool", conversationId: ROOT_CONVERSATION_ID }, + reason: "restart_limit", + restarts: 2, + }); + await opened.harness.close(context); + }); + + it("fails a compaction into compactionFailure", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.restartLimit = 1; + // Little recent context is kept, so the first turn is summarized. + setup.settings.compaction = { enabled: false, keepRecentTokens: 10 }; + setup.faux.setResponses([fauxAssistantMessage("a1"), fauxAssistantMessage("a2")]); + let opened = await open(path, setup); + opened.harness.resume(); + await (await opened.root.submit({ type: "input", content: "u1 ".repeat(200) }, context)).wait(context); + await (await opened.root.submit({ type: "input", content: "u2 ".repeat(200) }, context)).wait(context); + // The summary request never returns. + const hanging = unanswered(); + setup.faux.setResponses([hanging.step]); + const compaction = await opened.root.compact(undefined, context); + await hanging.reached; + opened.kill(); + + opened = await open(path, setup); + opened.harness.resume(); + const error = "Compaction was interrupted once"; + expect((await opened.harness.waitForTask(compaction, context)).state.outcome).toEqual({ + status: "failed", + error: { message: error, detail: { reason: "restart_limit" } }, + }); + expect((await live(opened.harness)).compactionFailure).toEqual({ + taskId: compaction, + reason: "manual", + error, + }); + expect((await live(opened.harness)).compactions).toBeUndefined(); + await waitFor(() => setup.infos.length > 0); + expect(setup.infos).toContainEqual({ + type: "task_interrupted", + task: { id: compaction, kind: "pi.compaction", conversationId: ROOT_CONVERSATION_ID }, + reason: "restart_limit", + restarts: 1, + }); + await opened.harness.close(context); + }); +}); + +/** + * A setup whose model answers deferred, and whose first fetch of each deferred response is in flight until the Harness + * stops; later fetches answer. `fetching` resolves when the first fetch began. + */ +function deferredSetup(pollAfterMs = 10) { + const base = chatSetup({ deferred: { pollAfterMs } }); + base.settings.stream = { deferred: true }; + base.faux.setResponses([fauxAssistantMessage("deferred answer")]); + const fetching = deferred(); + let fetches = 0; + const fetchDeferred: Models["fetchDeferred"] = (model, handle, options) => { + fetches++; + if (fetches > 1) return base.models.fetchDeferred(model, handle, options); + fetching.resolve(); + return aborted(options!.signal!); + }; + const models = new Proxy(base.models, { + get(target, property) { + if (property === "fetchDeferred") return fetchDeferred; + const value = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + const setup: ChatSetup = { ...base, models }; + return { setup, fetching, fetches: () => fetches }; +} + +/** Submit to a fresh chat at `path`, let its deferred response be fetched, and crash during that fetch. */ +async function crashWhilePolling(path: string, setup: ChatSetup, fetching: Promise) { + const opened = await open(path, setup); + opened.harness.resume(); + const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; + await fetching; + const taskId = (await live(opened.harness)).run!.taskId; + expect((await opened.harness.getTask(taskId, context))?.state).toMatchObject({ + status: "running", + checkpoint: { phase: "poll" }, + }); + opened.kill(); + return { id, taskId }; +} + +describe("crash edges (#11139)", () => { + it("charges a crash during a poll's fetch, then polls the same deferred response again", async () => { + const path = await sqlitePath(); + const { setup, fetching, fetches } = deferredSetup(); + setup.settings.retry = { enabled: true, maxRetries: 2, baseDelayMs: 1 }; + const { id, taskId } = await crashWhilePolling(path, setup, fetching.promise); + + const opened = await open(path, setup); + const values = liveValues(opened.harness); + expect((await opened.harness.getTask(taskId, context))?.restarts).toBe(1); + opened.harness.resume(); + const answer = await settled(opened.harness, id); + expect(answer.status).toBe("done"); + expect(values.find((value) => value.generation?.retry !== undefined)?.generation).toEqual({ + attempt: 1, + retry: { + at: expect.any(Number), + error: "The host stopped while the deferred answer was fetched, on attempt 1", + }, + }); + expect(values.some((value) => value.generation?.attempt === 2 && value.generation.deferred !== undefined)).toBe( + true, + ); + // One request, polled twice: the provider's job is neither dropped nor submitted again. + expect(setup.faux.state.callCount).toBe(1); + expect(fetches()).toBe(2); + expect(setup.faux.state.cancelledDeferred).toEqual([]); + const entries = await allEntries(opened.root); + expect(textOf(entries.at(-1)!.model![0])).toBe("deferred answer"); + await opened.harness.close(context); + }); + + it("cancels the deferred response when a crash during its poll ends the run", async () => { + const path = await sqlitePath(); + const { setup, fetching } = deferredSetup(); + setup.settings.retry = { enabled: false }; + const { id, taskId } = await crashWhilePolling(path, setup, fetching.promise); + const opened = await open(path, setup); + opened.harness.resume(); + const text = "The host stopped while the deferred answer was fetched, on attempt 1"; + expect(await settled(opened.harness, id)).toMatchObject({ + status: "unanswered", + reason: "interrupted", + detail: text, + }); + expect(await live(opened.harness)).toEqual({ + runFailure: { taskId, inputs: [id], reason: "interrupted", detail: text }, + }); + expect(setup.faux.state.cancelledDeferred).toHaveLength(1); + await opened.harness.close(context); + }); + + it("charges an interrupted poll even when the clock moved backwards", async () => { + const path = await sqlitePath(); + const { setup, fetching } = deferredSetup(10); + let now = 1000; + setup.now = () => now; + setup.settings.retry = { enabled: false }; + const first = await open(path, setup); + first.harness.resume(); + const id = (await first.root.submit({ type: "input", content: "hi" }, context)).id; + await waitFor(async () => (await live(first.harness)).generation?.deferred !== undefined); + now = 1010; + first.harness.resume(); + await fetching.promise; + first.kill(); + // A replacement host whose clock is behind: the poll looks not due yet, but its fetch was cut off. + now = 1000; + const opened = await open(path, setup); + opened.harness.resume(); + expect(await settled(opened.harness, id)).toMatchObject({ status: "unanswered", reason: "interrupted" }); + await opened.harness.close(context); + }); + + it("charges a request once when a crash is followed by a clean close before the charge", async () => { + const path = await sqlitePath(); + const first = streaming(); + first.settings.retry = { enabled: true, maxRetries: 3, baseDelayMs: 1 }; + const { id, taskId } = await crashWhileStreaming(path, first); + const setup = chatSetup(); + setup.settings.retry = { enabled: true, maxRetries: 3, baseDelayMs: 1 }; + setup.faux.setResponses([fauxAssistantMessage("answer")]); + let opened = await open(path, setup); + await opened.harness.close(context); + opened = await open(path, setup); + expect((await opened.harness.getTask(taskId, context))?.restarts).toBe(1); + const values = liveValues(opened.harness); + opened.harness.resume(); + expect((await settled(opened.harness, id)).status).toBe("done"); + const charged = values.filter((value) => value.generation?.retry !== undefined); + expect(charged.map((value) => value.generation?.attempt)).toEqual([1]); + expect((await allEntries(opened.root)).map((entry) => entry.kind)).toEqual([ + "pi.user", + "pi.assistant", + "pi.assistant", + ]); + await opened.harness.close(context); + }); + + it("ends a run restart_limit when charging the interrupted request keeps being cut off", async () => { + const path = await sqlitePath(); + const first = streaming(); + first.settings.restartLimit = 1; + const { id, taskId } = await crashWhileStreaming(path, first); + // At the limit: the partial is still kept. + const setup = chatSetup(); + setup.settings.restartLimit = 1; + const opened = await open(path, setup); + const partial = textOf((await live(opened.harness)).generation?.message as Message)!; + opened.harness.resume(); + const text = "The generation was interrupted once in its request phase"; + expect(await settled(opened.harness, id)).toMatchObject({ + status: "unanswered", + reason: "restart_limit", + detail: text, + }); + expect(await live(opened.harness)).toEqual({ + runFailure: { taskId, inputs: [id], reason: "restart_limit", detail: text }, + }); + const entries = await allEntries(opened.root); + expect(entries.map((entry) => entry.kind)).toEqual(["pi.user", "pi.assistant"]); + expect(textOf(entries[1]!.model![0])).toBe(partial); + expect(setup.faux.state.callCount).toBe(0); + await opened.harness.close(context); + }); + + it("drops the partial past the limit, where converting it may be what keeps failing", async () => { + const path = await sqlitePath(); + const first = streaming(); + const { id, taskId } = await crashWhileStreaming(path, first); + // As if the charge had been cut off twice before: the open after this crash counts 3, past the limit of 2. + const raw = await openNodeSqliteStorage(path); + const record = await raw.task(taskId, context); + await raw.commit([{ type: "task", value: { ...record!, restarts: 2 } }], () => {}, context); + await raw.close(context); + const setup = chatSetup(); + setup.settings.restartLimit = 2; + const opened = await open(path, setup); + opened.harness.resume(); + expect(await settled(opened.harness, id)).toMatchObject({ + status: "unanswered", + reason: "restart_limit", + detail: "The generation was interrupted 3 times in a row in its request phase", + }); + expect((await allEntries(opened.root)).map((entry) => entry.kind)).toEqual(["pi.user"]); + expect((await live(opened.harness)).generation).toBeUndefined(); + await opened.harness.close(context); + }); + + it("ends a run someone aborted as aborted when its abort handler is cut off restartLimit times", async () => { + const path = await sqlitePath(); + const setup = chatSetup({ deferred: { pollAfterMs: 3_600_000 } }); + setup.settings.restartLimit = 2; + setup.settings.stream = { deferred: true }; + setup.faux.setResponses([fauxAssistantMessage("never")]); + let cancels = 0; + // The abort handler of a polling generation cancels its response: make that hang. + setup.models.cancelDeferred = () => { + cancels++; + return new Promise(() => {}); + }; + let opened = await open(path, setup); + opened.harness.resume(); + const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; + const taskId = (await live(opened.harness)).run!.taskId; + await waitFor(async () => (await opened.harness.getTask(taskId, context))?.state.status === "sleeping"); + void opened.root.abort(context).catch(() => {}); + for (let crash = 1; crash <= 2; crash++) { + await waitFor(() => cancels === crash); + opened.kill(); + opened = await open(path, setup); + expect((await opened.harness.getTask(taskId, context))?.restarts).toBe(crash); + opened.harness.resume(); + } + const detail = `Abort handler of task ${taskId} (pi.generation) was cut off 2 times`; + expect(await settled(opened.harness, id)).toMatchObject({ status: "unanswered", reason: "aborted", detail }); + expect(await live(opened.harness)).toEqual({ runFailure: { taskId, inputs: [id], reason: "aborted", detail } }); + expect((await opened.harness.waitForTask(taskId, context)).state.outcome).toMatchObject({ + status: "faulted", + error: { detail: { reason: "restart_limit" } }, + }); + await waitFor(() => setup.infos.some((info) => info.type === "task_faulted")); + expect(setup.infos).toContainEqual(expect.objectContaining({ type: "task_faulted", reason: "restart_limit" })); + await opened.harness.close(context); + }); +}); + +describe("reports of work crashes ended (#11139)", () => { + it("reports task_interrupted for a run that ends interrupted and for a task that ends at the limit", async () => { + const path = await sqlitePath(); + const first = streaming(); + first.settings.retry = { enabled: false }; + const { taskId } = await crashWhileStreaming(path, first); + const setup = chatSetup(); + setup.settings.retry = { enabled: false }; + const opened = await open(path, setup); + opened.harness.resume(); + await opened.harness.waitForTask(taskId, context); + await waitFor(() => setup.infos.length > 0); + expect(setup.infos).toEqual([ + { + type: "task_interrupted", + task: { id: taskId, kind: "pi.generation", conversationId: ROOT_CONVERSATION_ID }, + reason: "interrupted", + restarts: 1, + }, + ]); + expect(String(setup.reports[0])).toContain("The host stopped while the model was answering, on attempt 1"); + await opened.harness.close(context); + }); +}); + +describe("the restart limit on the normal path (#11139)", () => { + it("resolves the settings no more often than before: only a task that was cut off reads the limit", async () => { + const setup = chatSetup(); + let resolutions = 0; + // Every resolution of the host settings reads `extensions` once. + (setup as { settings: object }).settings = new Proxy( + {}, + { + get(target, key) { + if (key === "extensions") resolutions++; + return Reflect.get(target, key); + }, + }, + ); + addTool( + setup.registry, + defineTool({ + name: "t", + description: "t", + parameters: Type.Object({}), + replay: "safe", + execute: async () => ({}), + }), + ); + const rounds = 5; + const responses: FauxResponseStep[] = []; + for (let round = 0; round <= rounds; round++) { + responses.push(() => + fauxAssistantMessage([fauxToolCall("t", {}, { id: `c${round}` })], { stopReason: "toolUse" }), + ); + responses.push(() => fauxAssistantMessage([fauxText("done")])); + } + setup.faux.setResponses(responses); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "warm up" }, context)).wait(context); + resolutions = 0; + for (let round = 0; round < rounds; round++) { + await (await root.submit({ type: "input", content: `round ${round}` }, context)).wait(context); + } + // One generation round with one tool call, as before the restart limit applied to every kind (#11327: 49). + expect(resolutions / rounds).toBe(49); + await harness.close(context); + }); +}); From d3398f826ae2654b52a7988830d533a354754c13 Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:27:30 +0100 Subject: [PATCH 042/271] fix(ai): detect LiteLLM ContextWindowExceededError as context overflow (#11494) LiteLLM's pre-call context check rejects requests that are too large with HTTP 400 and a message like: ``` litellm.ContextWindowExceededError: litellm.BadRequestError: litellm._pre_call_checks: Context Window exceeded for given call. ... Max Input Tokens=262144, Got=265262 ... ``` None of the `OVERFLOW_PATTERNS` matched this wording, so `isContextOverflow()` returned false. As a result, the agent session never ran overflow compaction and retry, and the turn failed with the 400 error. This change adds `/context window exceeded/i` to `OVERFLOW_PATTERNS` and documents the error in the doc comment that lists providers. Testing: - Added a regression test in `packages/ai/test/overflow.test.ts` with the exact error message from the issue. It fails without the pattern and passes with it. - `node ../../node_modules/vitest/dist/cli.js --run test/overflow.test.ts` in `packages/ai`: 32 passed. - `npm run check` passes. - Running `isContextOverflow()` directly on the reported message returned `false` before this change and returns `true` after it. Fixes #11492 Requested by @rwachtler Co-authored-by: triagent --- packages/ai/src/utils/overflow.ts | 2 ++ packages/ai/test/overflow.test.ts | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/packages/ai/src/utils/overflow.ts b/packages/ai/src/utils/overflow.ts index 03f82f61b..36ae26b49 100644 --- a/packages/ai/src/utils/overflow.ts +++ b/packages/ai/src/utils/overflow.ts @@ -12,6 +12,7 @@ import type { AssistantMessage } from "../types.ts"; * - Anthropic: "413 {\"error\":{\"type\":\"request_too_large\",\"message\":\"Request exceeds the maximum size\"}}" * - OpenAI: "Your input exceeds the context window of this model" * - OpenAI/LiteLLM: "Requested token count exceeds the model's maximum context length of 131072 tokens" + * - LiteLLM pre-call check: "litellm.ContextWindowExceededError: ... Context Window exceeded for given call." * - OpenAI-compatible: "Input length (265330) exceeds model's maximum context length (262144)." * - Google: "The input token count (1196265) exceeds the maximum number of tokens allowed (1048575)" * - xAI: "This model's maximum prompt length is 131072 but the request contains 537812 tokens" @@ -42,6 +43,7 @@ const OVERFLOW_PATTERNS = [ /input is too long for requested model/i, // Amazon Bedrock /exceeds the context window/i, // OpenAI (Completions & Responses API) /exceeds (?:the )?(?:model'?s )?maximum context length(?: of [\d,]+ tokens?|\s*\([\d,]+\))/i, // OpenAI-compatible proxies (LiteLLM) + /context window exceeded/i, // LiteLLM pre-call check (ContextWindowExceededError) /input token count.*exceeds the maximum/i, // Google (Gemini) /maximum prompt length is \d+/i, // xAI (Grok) /reduce the length of the messages/i, // Groq diff --git a/packages/ai/test/overflow.test.ts b/packages/ai/test/overflow.test.ts index e7ae90548..1cab5e92e 100644 --- a/packages/ai/test/overflow.test.ts +++ b/packages/ai/test/overflow.test.ts @@ -91,6 +91,14 @@ describe("isContextOverflow", () => { expect(isContextOverflow(message, 131072)).toBe(true); }); + it("detects LiteLLM pre-call ContextWindowExceededError", () => { + // Regression for #11492. + const message = createErrorMessage( + '400: {"message":"litellm.ContextWindowExceededError: litellm.BadRequestError: litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.\nModel=Qwen3.8-Flash-Next, Max Input Tokens=262144, Got=265262\nmodel=Qwen3.8-Flash-Next. context_window_fallbacks=None. fallbacks=[]."}', + ); + expect(isContextOverflow(message, 262144)).toBe(true); + }); + it("detects OpenAI-compatible parenthesized maximum context length errors", () => { const message = createErrorMessage( "Error: 400 Input length (265330) exceeds model's maximum context length (262144).", From 5f312dac4c4b18ad6b4af05dbd751d2668a4d6d9 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Tue, 27 Oct 2026 20:38:08 +0000 Subject: [PATCH 043/271] feat(chord,durable): failures are watch calls; one vocabulary for watches, observers and reports Chord's shared watch: listeners take a tagged call, { kind: "value", value, ops, retired } or { kind: "failed", error }; a recoverable failure keeps the watch open and the next value arrives as [["r", value]]; WatchEnd is tagged by kind, and an aborted context ends a watch "aborted". Durable: observers take the watch's shapes (advance(call, ctx), end(end)); onReport(report) with report.kind and report.error, session_failed is storage_failed; subscribeCommits is observeCommits; SessionEnd and ReportFrom are tagged by kind; FileChange is tagged and file watchers stop with stop(ctx); CompactionFailure and CompactionStatus name their trigger (pi.live version 2 with a migrate); the task_outdated report uses the scheduler's reasons; TaskInspection's ready is pending; DocumentCommitChange is tagged. Stored formats are unchanged. --- packages/chord/CHANGELOG.md | 2 + packages/chord/README.md | 56 +- packages/chord/src/index.ts | 4 +- packages/chord/src/watch.ts | 257 +++++++--- packages/chord/test/watch.test.ts | 478 +++++++++++++----- packages/durable/CHANGELOG.md | 10 + packages/durable/README.md | 29 +- packages/durable/docs/spec.md | 215 +++++--- packages/durable/src/env/index.ts | 13 +- packages/durable/src/env/node-watch.ts | 34 +- packages/durable/src/env/node.ts | 2 +- packages/durable/src/harness/compaction.ts | 2 +- packages/durable/src/harness/generation.ts | 6 +- packages/durable/src/harness/harness.ts | 27 +- packages/durable/src/harness/live.ts | 25 +- packages/durable/src/harness/scheduler.ts | 47 +- packages/durable/src/harness/submissions.ts | 2 +- packages/durable/src/harness/task-graph.ts | 34 +- packages/durable/src/harness/telemetry.ts | 8 +- packages/durable/src/harness/tool.ts | 2 +- packages/durable/src/harness/types.ts | 12 +- packages/durable/src/harness/view.ts | 44 +- packages/durable/src/index.ts | 5 +- packages/durable/src/session/session.ts | 160 +++--- packages/durable/src/session/transaction.ts | 24 +- .../durable/src/testing/env-conformance.ts | 14 +- packages/durable/src/types.ts | 113 +++-- packages/durable/test/chat-support.ts | 2 +- packages/durable/test/codemode-bench.ts | 7 +- packages/durable/test/codemode.test.ts | 11 +- .../test/conversation-transcript.test.ts | 10 +- .../durable/test/conversation-view-bench.ts | 17 +- .../durable/test/env-node-conformance.test.ts | 6 +- packages/durable/test/examples/04-watches.ts | 6 +- .../durable/test/examples/15-real-model.ts | 4 +- packages/durable/test/examples/18-json.ts | 8 +- .../durable/test/examples/20-late-join.ts | 4 +- .../test/examples/21-subagent-foreground.ts | 4 +- .../test/examples/22-subagent-background.ts | 4 +- .../durable/test/examples/31-transcript.ts | 8 +- packages/durable/test/harness-blocked.test.ts | 11 +- .../durable/test/harness-compaction.test.ts | 80 ++- .../test/harness-deferred-progress.test.ts | 28 +- .../test/harness-generation-recovery.test.ts | 10 +- .../durable/test/harness-generation.test.ts | 6 +- packages/durable/test/harness-inbox.test.ts | 48 +- packages/durable/test/harness-inspect.test.ts | 8 +- .../durable/test/harness-lifecycle.test.ts | 15 +- .../durable/test/harness-live-deltas.test.ts | 60 ++- .../test/harness-nested-tools-restart.test.ts | 26 +- .../durable/test/harness-nested-tools.test.ts | 16 +- .../durable/test/harness-observers.test.ts | 82 +-- .../durable/test/harness-ownership.test.ts | 4 +- .../durable/test/harness-read-only.test.ts | 2 +- packages/durable/test/harness-reports.test.ts | 33 +- .../test/harness-restart-failure.test.ts | 20 +- .../test/harness-storage-failure.test.ts | 62 ++- .../test/harness-structured-output.test.ts | 20 +- .../durable/test/harness-structured.test.ts | 4 +- .../durable/test/harness-submissions.test.ts | 13 +- packages/durable/test/harness-support.ts | 6 +- .../durable/test/harness-task-graph.test.ts | 30 +- .../test/harness-tasks-recovery.test.ts | 27 +- packages/durable/test/harness-tasks.test.ts | 122 +++-- .../durable/test/harness-telemetry.test.ts | 73 ++- .../durable/test/harness-view-run.test.ts | 49 +- packages/durable/test/harness-view.test.ts | 66 ++- packages/durable/test/live-support.ts | 10 +- .../durable/test/session-adoption.test.ts | 113 +++-- .../session-checkpoints-migrations.test.ts | 40 +- .../test/session-deferred-commits.test.ts | 38 +- .../test/session-document-retention.test.ts | 10 +- .../durable/test/session-documents.test.ts | 34 +- packages/durable/test/session-forks.test.ts | 2 +- .../durable/test/session-observers.test.ts | 158 +++++- packages/durable/test/session-support.ts | 40 +- packages/durable/test/session-tables.test.ts | 3 +- packages/durable/test/session-watches.test.ts | 200 ++++---- packages/durable/test/spec-usage.test.ts | 3 +- packages/durable/test/task-support.ts | 2 +- packages/durable/test/tool-output-bench.ts | 18 +- packages/env/CHANGELOG.md | 4 + packages/env/src/remote-env.ts | 2 +- packages/env/src/watch.ts | 38 +- 84 files changed, 2196 insertions(+), 1156 deletions(-) diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index fd2fac9ff..145418f1b 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -8,6 +8,8 @@ - Watch listeners take one change object before the context: `WatchHandle.start((change, context) => …)` with `change: WatchChange` = `{ value, ops, retired }`, and `NativeWatchHandle.start(({ value }, context) => …)` with `NativeWatchChange`. A change is plain data, those fields only; the frame's context is the second argument. `Watch.advance(change, context)` takes the same object, and a frame delivered alone, with nothing folded into it, reaches the listener as the object the source passed. Migration: `start(async (value, ops, ctx) => …)` becomes `start(async ({ value, ops }, ctx) => …)`, `start(async (value, ctx) => …)` on a native watch `start(async ({ value }, ctx) => …)`, and `advance(value, ops, ctx)` becomes `advance({ value, ops, retired: false }, ctx)`. - `SqliteExecutor.runCounted(sql, ...params)` runs one statement like `run` and resolves with the rows it inserted, updated or deleted, as SQLite's `changes()` counts them (index writes and rows changed by triggers do not count). The Node driver takes it from `node:sqlite`; the Durable Object driver reads `SELECT changes()` after the statement (the cursor's `rowsWritten` also counts index writes), so `run` stays as cheap as before. Migration: custom `SqliteDatabase` and `SqliteExecutor` implementations add `runCounted`. - Retirement is a flag, not an end: `WatchEnd` loses `retired`, and `Watch.retire(context)` is removed. A source reports a retirement with `advance({ value, ops, retired: true }, context)`, whose value is what the address shows after it, and the watch goes on: the next call has `retired: true`, folded with every other pending frame, so a document created at the address afterwards arrives in the same or a later call as `[["r", value]]`. `endAfterDelivery()` tells a watch its source sends nothing more: it takes no more frames, delivers what is pending, and ends `ended` after its last call returned, at once when nothing is pending and no call runs; it stays attached until then, so the source's `end()` (such as at its close), `stop()`, cancellation, or a throwing listener still end it first. Migration: a listener that waited for `closed` to give `retired` or tested for a `null` value tests `change.retired`; a source that called `retire(ctx)` calls `advance({ value, ops, retired: true }, ctx)`, then `endAfterDelivery()` when its owner ends with the retirement. +- Watch listeners get a tagged call, and a watch stays open through a failure its source recovers from: `WatchCall` is `{ kind: "value", value, ops, retired }` or `{ kind: "failed", error }`, and `NativeWatchCall` is `{ kind: "value", value }` or `{ kind: "failed", error }` (replacing `WatchChange` and `NativeWatchChange`). After a `failed` call `watch.value` keeps the last value and the next value arrives as `[["r", value]]`; while the listener is busy, a failure drops the pending values (`value` then `failed` folds into the failure) and a value replaces a pending failure (`failed` then `value` folds into the value, as `r`). `Watch.advance(call, context)` takes a `WatchCall`, a value call delivered alone still reaches the listener as the object the source passed (no allocation), and `Watch.fail(error, context)` and `NativeWatch.fail(error, context)` send a failure. Migration: `start(async ({ value, ops }, ctx) => …)` becomes `start(async (call, ctx) => { if (call.kind === "failed") return; const { value, ops } = call; … })`, a native `start(async ({ value }, ctx) => …)` likewise; `advance({ value, ops, retired }, ctx)` becomes `advance({ kind: "value", value, ops, retired }, ctx)`. +- `WatchEnd` is tagged by `kind` instead of `reason`, and a watch whose acquisition context is aborted ends `aborted` instead of `cancelled`: `{ kind: "stopped" | "aborted" | "ended" }`, `{ kind: "listener_error", error }`, or `{ kind: "failed", error }` (a final failure only). `Watch.cancel()` is `abort()`, `observeCancellation(signal)` is `observeAbort(signal)`, and `end(failure?)` takes the end: `end({ kind: "ended" })` or `end({ kind: "failed", error })`. Migration: `end.reason === "cancelled"` becomes `end.kind === "aborted"`, every other `end.reason` becomes `end.kind`; `watch.end()` becomes `watch.end({ kind: "ended" })`, `watch.end({ error })` becomes `watch.end({ kind: "failed", error })`. ### Added diff --git a/packages/chord/README.md b/packages/chord/README.md index 376b3625d..9d9d95587 100644 --- a/packages/chord/README.md +++ b/packages/chord/README.md @@ -14,7 +14,7 @@ Contracts are runtime values described with [TypeBox](https://github.com/sinclai | Import | Contents | Runs in | |--------|----------|---------| -| `@earendil-works/chord` | Contracts (`local`, `remote`, `method`, `opaque`), declarations (through the factory's `self`: `state`, `derive`, `resource`, `on`, `task`), hosts (`createHost`, `defineFacet`, `defineRegistry`), links (`memoryLinkPair`, `ipcLink`, `webSocketLink`, `acceptWebSockets`), errors (`ChordError`, `isChordError`, `forbidden`, `invalid`). Also the shared base: `Context`, `JsonValue`, `JsonRepresentation`, `Draft`, `copyJson`, `isJsonValue`, `replicatedState` with its types, and the watch types `WatchHandle`, `NativeWatchHandle`, `WatchChange`, `NativeWatchChange`, and `WatchEnd` with their implementations `Watch` and `NativeWatch`. | Node and browsers | +| `@earendil-works/chord` | Contracts (`local`, `remote`, `method`, `opaque`), declarations (through the factory's `self`: `state`, `derive`, `resource`, `on`, `task`), hosts (`createHost`, `defineFacet`, `defineRegistry`), links (`memoryLinkPair`, `ipcLink`, `webSocketLink`, `acceptWebSockets`), errors (`ChordError`, `isChordError`, `forbidden`, `invalid`). Also the shared base: `Context`, `JsonValue`, `JsonRepresentation`, `Draft`, `copyJson`, `isJsonValue`, `replicatedState` with its types, and the watch types `WatchHandle`, `NativeWatchHandle`, `WatchCall`, `NativeWatchCall`, and `WatchEnd` with their implementations `Watch` and `NativeWatch`. | Node and browsers | | `@earendil-works/chord/node` | Child host processes: `createHostManager`, `forkDriver`, `childControl`, `processLock`; the facet module loader `nodeLoader`. | Node | | `@earendil-works/chord/context` | Explicit cancellation, telemetry, and invocation values: `BACKGROUND_CONTEXT`, `withCancel`, `withAbortSignal`, `withoutAbortSignal`, `withTelemetry`, `traceIn`, `withContextValue`, `awaitWithContext`. | Node and browsers | | `@earendil-works/chord/delta` | The transactional JSON tracker, structural diff, immutable apply, and the wire encoder/decoder for operation batches. | Node and browsers | @@ -209,39 +209,59 @@ model; see [State](docs/guide.md#state) in the usage guide. `WatchHandle` is the one watch interface, implemented by watch sources such as Durable's documents, conversation views, and task graph. `watch.value` is the value at acquisition until `start(listener)` installs the sole asynchronous listener; each -later change arrives as `(change, context)`, one call at a time, with -`change: WatchChange` = `{ value, ops, retired }` (plain data; the context is -the second argument), and `value` advances to it. +later change arrives as `(call, context)`, one call at a time, with +`call: WatchCall` tagged by `kind` (plain data; the context is the second +argument): + +- `{ kind: "value", value, ops, retired }`: the new value, and `watch.value` + advances to it; +- `{ kind: "failed", error }`: the source cannot compute its value now. The watch + stays open and `watch.value` keeps the last value; the next value arrives as + `[["r", value]]`. + +```ts +watch.start(async (call) => { + if (call.kind === "failed") return showError(call.error); + render(call.value); +}); +``` + Changes made while the listener runs (or before `start`) arrive in one call: the newest value, the ops of all of them folded by Delta's `PendingOps`, and the newest change's context. Past 10,000 pending ops after folding the call gets one root -replacement `[["r", newest]]` instead. +replacement `[["r", newest]]` instead. A failure drops the pending values and waits +as one `failed` call (the newest failure); a value after it replaces it, as one +`[["r", value]]` call. -`retired` says the document the watch follows was retired since the previous call. -It does not end the watch: the call carries what the address shows after the +`retired` says the document the watch follows was retired since the previous value +call. It does not end the watch: the call carries what the address shows after the retirement (the source decides; a Durable document keeps the last value the watch was given, with no ops), and a document created there later arrives as `[["r", value]]`, folded with what is pending, so one call can report a retirement and carry the new value. -`stop()` is idempotent, drops pending frames, and neither aborts nor awaits a -running call; it and `closed` give the `WatchEnd`: +A watch ends only for good. `stop()` is idempotent, drops pending frames, and +neither aborts nor awaits a running call; it and `closed` give the `WatchEnd`, +tagged by `kind`: - `stopped`: `stop()` was called; -- `cancelled`: the context the watch was acquired with was cancelled; +- `aborted`: the context the watch was acquired with was aborted; - `ended`: the source closed (its owner ended or closed); - `listener_error` with `error`: the listener threw or rejected; -- `failed` with `error`: the source failed. +- `failed` with `error`: the source failed for good. An ended watch drops every reference into its source, so a stopped watch that is still referenced keeps only its last value alive. -Sources implement handles with `Watch`, fed through `advance(change, context)` -(a `WatchChange`, `retired` set for a retirement), `end()` or `end({ error })` -(at once, dropping what is pending), `endAfterDelivery()` (take no more frames, -deliver what is pending, then end `ended` once the last call returned; the -source's `end()` still ends it first), and `observeCancellation(signal)`. +Sources implement handles with `Watch`, fed through `advance(call, context)` +(a `WatchCall`: a value with `retired` set for a retirement, or a failure), +`fail(error, context)` (a failure the source may recover from), `end(end)` (a +`WatchEnd`, at once, dropping what is pending), `endAfterDelivery()` (take no more +frames, deliver what is pending, then end `ended` once the last call returned; the +source's `end()` still ends it first), and `observeAbort(signal)`. `NativeWatch` implements `NativeWatchHandle`, the same lifecycle for -in-process values: the listener gets `({ value }, context)` without ops, one call +in-process values: the listener gets a `NativeWatchCall` +(`{ kind: "value", value }` without ops, or `{ kind: "failed", error }`), one call per value in order; past 100 pending values they collapse to one, the value of the -optional `replace()` or the newest. +optional `replace()` or the newest. A failure drops the pending values; values +after it are delivered in order. diff --git a/packages/chord/src/index.ts b/packages/chord/src/index.ts index 30ba50455..34b3613ea 100644 --- a/packages/chord/src/index.ts +++ b/packages/chord/src/index.ts @@ -81,11 +81,11 @@ export type { Ctx, GetOptions, Readable, RefStatus, Shape, StateEvent } from "./ export { NativeWatch, type NativeWatchable, - type NativeWatchChange, + type NativeWatchCall, type NativeWatchHandle, Watch, type Watchable, - type WatchChange, + type WatchCall, type WatchEnd, type WatchHandle, } from "./watch.ts"; diff --git a/packages/chord/src/watch.ts b/packages/chord/src/watch.ts index 74a7ef15b..da6dc115e 100644 --- a/packages/chord/src/watch.ts +++ b/packages/chord/src/watch.ts @@ -4,41 +4,50 @@ import { type Op, PendingOps } from "./delta/index.ts"; import type { JsonValue } from "./json.ts"; /** - * Terminal result of one watch. + * Terminal result of one watch, tagged by `kind`. A watch ends only for good; a failure its source recovers from is a + * `failed` call (`WatchCall`), and the watch stays open. * * - `stopped`: `stop()` was called. - * - `cancelled`: the context the watch was acquired with was cancelled. + * - `aborted`: the context the watch was acquired with was aborted. * - `ended`: the source closed (its owner ended or closed). * - `listener_error`: the listener threw or rejected with `error`. - * - `failed`: the source failed with `error`. + * - `failed`: the source failed for good with `error`. */ export type WatchEnd = - | { readonly reason: "stopped" | "cancelled" | "ended" } - | { readonly reason: "listener_error"; readonly error: Error } - | { readonly reason: "failed"; readonly error: unknown }; + | { readonly kind: "stopped" | "aborted" | "ended" } + | { readonly kind: "listener_error"; readonly error: Error } + | { readonly kind: "failed"; readonly error: unknown }; /** - * One call of a JSON watch's listener, and one change a source hands its watch: the newest value, the ops that turn the - * previous value into it, and whether the document the watch follows was retired since the previous call. A retirement - * does not end the watch: `value` is then what the address shows after it (the source decides; a Durable document keeps - * the last value the watch was given, with no ops), and a document created there later arrives as `[["r", value]]`, - * possibly in the same call. Plain data: the frame's context is the listener's second argument, never part of the - * change. + * One call of a JSON watch's listener, and one frame a source hands its watch, tagged by `kind`: + * + * - `value`: the newest value, the ops that turn the previous value into it, and whether the document the watch follows + * was retired since the previous value call. A retirement does not end the watch: `value` is then what the address + * shows after it (the source decides; a Durable document keeps the last value the watch was given, with no ops), and a + * document created there later arrives as `[["r", value]]`, possibly in the same call. + * - `failed`: the source cannot compute its value now, with `error`; the watch stays open, and the next value arrives as + * `[["r", value]]`. `WatchHandle.value` keeps the last value. + * + * Plain data: the frame's context is the listener's second argument, never part of the call. */ -export type WatchChange = { readonly value: T; readonly ops: readonly Op[]; readonly retired: boolean }; +export type WatchCall = + | { readonly kind: "value"; readonly value: T; readonly ops: readonly Op[]; readonly retired: boolean } + | { readonly kind: "failed"; readonly error: unknown }; -/** One call of a native watch's listener: the new value. */ -export type NativeWatchChange = { readonly value: T }; +/** One call of a native watch's listener: the new value, or a failure its source may recover from, as in `WatchCall`. */ +export type NativeWatchCall = + | { readonly kind: "value"; readonly value: T } + | { readonly kind: "failed"; readonly error: unknown }; /** * Serialized observation of an immutable value: one listener call at a time; changes made meanwhile arrive folded into * the next call. */ export interface WatchHandle { - /** Acquisition revision before start; latest delivered immutable revision afterward. */ + /** Acquisition revision before start; latest delivered immutable revision afterward (a `failed` call keeps it). */ readonly value: T; /** Install the sole asynchronous listener. Never invokes it inline. */ - start(listener: (change: WatchChange, context: Context) => Promise): void; + start(listener: (call: WatchCall, context: Context) => Promise): void; /** Idempotently stop future callbacks and return this watch's terminal result. */ stop(): Promise; /** Settle when the watch terminates; an already-running callback remains caller-owned. */ @@ -53,7 +62,7 @@ export interface WatchHandle { export interface Watchable { /** The current value. */ readonly value: T; - /** A watch holding the current value, then its changes; the context's cancellation ends it. */ + /** A watch holding the current value, then its changes; aborting the context ends it. */ watch(context: Context): Promise>; } @@ -64,7 +73,7 @@ export interface Watchable { export interface NativeWatchable { /** The current value. */ readonly value: T; - /** A watch holding the current value, then one frame per new value; the context's cancellation ends it. */ + /** A watch holding the current value, then one frame per new value; aborting the context ends it. */ watch(context: Context): Promise>; } @@ -73,25 +82,32 @@ export interface NativeWatchable { * ops. */ export interface NativeWatchHandle { - /** Acquisition value before start; latest delivered value afterward. */ + /** Acquisition value before start; latest delivered value afterward (a `failed` call keeps it). */ readonly value: T; /** Install the sole asynchronous listener. Never invokes it inline. */ - start(listener: (change: NativeWatchChange, context: Context) => Promise): void; + start(listener: (call: NativeWatchCall, context: Context) => Promise): void; /** Idempotently stop future callbacks and return this watch's terminal result. */ stop(): Promise; /** Settle when the watch terminates; an already-running callback remains caller-owned. */ readonly closed: Promise; } -type Listener = (change: C, context: Context) => Promise; +type Listener = (call: C, context: Context) => Promise; + +const STOPPED: WatchEnd = { kind: "stopped" }; +const ABORTED: WatchEnd = { kind: "aborted" }; +const ENDED: WatchEnd = { kind: "ended" }; + +/** A failure call, as both call types have it. */ +type FailedCall = { readonly kind: "failed"; readonly error: unknown }; /** * The lifecycle both handles share: serialized delivery and termination. A source feeds it and ends it explicitly; * termination drops every reference the watch holds into its source, so a stopped watch that is still referenced keeps - * nothing but its last value alive. Subclasses keep what is pending, each change apart from its frame's context, so the - * change a listener gets carries nothing but its data. + * nothing but its last value alive. Subclasses keep what is pending, each call apart from its frame's context, so the + * call a listener gets carries nothing but its data. */ -abstract class WatchLifecycle { +abstract class WatchLifecycle { #detach: (() => void) | undefined; #report: ((error: unknown) => void) | undefined; readonly #closed = Promise.withResolvers(); @@ -104,8 +120,8 @@ abstract class WatchLifecycle { #ending = false; /** Set once, by `#terminate()`. */ #end: WatchEnd | undefined; - #cancellationSignal: AbortSignal | undefined; - #cancellationListener: (() => void) | undefined; + #abortSignal: AbortSignal | undefined; + #abortListener: (() => void) | undefined; constructor(value: T, detach: () => void, report: (error: unknown) => void) { this.#value = value; @@ -130,43 +146,43 @@ abstract class WatchLifecycle { } stop(): Promise { - this.#terminate({ reason: "stopped" }); + this.#terminate(STOPPED); return this.#closed.promise; } - /** End the watch `cancelled` when `signal` aborts; installed once. */ - observeCancellation(signal: AbortSignal): void { - if (this.#cancellationSignal !== undefined) throw new Error("Watch cancellation is already installed"); + /** End the watch `aborted` when `signal` aborts; installed once. */ + observeAbort(signal: AbortSignal): void { + if (this.#abortSignal !== undefined) throw new Error("Watch abort is already installed"); if (this.#end !== undefined) return; - this.#cancellationSignal = signal; - this.#cancellationListener = () => this.cancel(); - signal.addEventListener("abort", this.#cancellationListener, { once: true }); - if (signal.aborted) this.cancel(); + this.#abortSignal = signal; + this.#abortListener = () => this.abort(); + signal.addEventListener("abort", this.#abortListener, { once: true }); + if (signal.aborted) this.abort(); } - /** End the watch `cancelled`: its acquisition context was cancelled. */ - cancel(): void { - this.#terminate({ reason: "cancelled" }); + /** End the watch `aborted`: its acquisition context was aborted. */ + abort(): void { + this.#terminate(ABORTED); } /** - * The source closed: end with `ended`, or with `failed` and the error when the source failed, at once, dropping what - * is pending. + * End the watch with `end` at once, dropping what is pending: the source closed (`ended`), or failed for good + * (`failed`). A failure the source may recover from is a `failed` call instead (`fail()`). */ - end(failure?: { readonly error: unknown }): void { - this.#terminate(failure === undefined ? { reason: "ended" } : { reason: "failed", error: failure.error }); + end(end: WatchEnd): void { + this.#terminate(end); } /** * The source sends nothing more after its last frames, such as a retirement: take no more frames, deliver what is * pending, then end `ended`, after the running call returns, if one runs. Until then the watch stays attached, so - * `end()`, `stop()`, cancellation, and a throwing listener still end it first; a watch never started keeps what is + * `end()`, `stop()`, an abort, and a throwing listener still end it first; a watch never started keeps what is * pending until one of them or `start()`. */ endAfterDelivery(): void { if (this.#end !== undefined || this.#ending) return; this.#ending = true; - if (!this.#running && this.pendingContext === undefined) this.#terminate({ reason: "ended" }); + if (!this.#running && this.pendingContext === undefined) this.#terminate(ENDED); } /** Whether the watch still takes frames: it has not ended, and its source has not ended. */ @@ -177,7 +193,7 @@ abstract class WatchLifecycle { /** The context of the newest frame the next call covers, or undefined when nothing is pending. */ protected abstract get pendingContext(): Context | undefined; - /** Take the next change; called only while something is pending. */ + /** Take the next call; called only while something is pending. */ protected abstract next(): C; /** Drop everything pending and every reference into the source; called once, on termination. */ @@ -210,17 +226,17 @@ abstract class WatchLifecycle { if (listener === undefined) break; const frameContext = this.pendingContext; if (frameContext === undefined) { - if (this.#ending) this.#terminate({ reason: "ended" }); + if (this.#ending) this.#terminate(ENDED); break; } - const change = this.next(); - this.#value = change.value; + const call = this.next(); + if (call.kind === "value") this.#value = call.value; try { - await listener(change, withoutAbortSignal(frameContext)); + await listener(call, withoutAbortSignal(frameContext)); } catch (error) { if (this.#end === undefined) { this.#report?.(error); - this.#terminate({ reason: "listener_error", error: toError(error) }); + this.#terminate({ kind: "listener_error", error: toError(error) }); } break; } @@ -242,11 +258,9 @@ abstract class WatchLifecycle { this.#report = undefined; this.#listener = undefined; this.release(); - if (this.#cancellationListener !== undefined) { - this.#cancellationSignal?.removeEventListener("abort", this.#cancellationListener); - } - this.#cancellationSignal = undefined; - this.#cancellationListener = undefined; + if (this.#abortListener !== undefined) this.#abortSignal?.removeEventListener("abort", this.#abortListener); + this.#abortSignal = undefined; + this.#abortListener = undefined; detach?.(); this.#closed.resolve(end); } @@ -265,33 +279,66 @@ function toError(error: unknown): Error { * The watch handle of a JSON source. Frames that arrive before `start()` or while the listener runs wait in one * `PendingOps` queue; the next call gets the newest value, the folded ops of every frame since the previous call (or * `[["r", newest]]` after the queue overflowed), `retired` when any of them was a retirement, and the newest frame's - * context. A frame delivered alone is passed on as the source made it. `detach` unsubscribes it from its source on - * termination; `report` gets a listener's throw, which also ends the watch. + * context. A frame delivered alone is passed on as the source made it. A failure (`fail()`) drops the pending frames and + * waits as one `failed` call; a value after it replaces it, and the first value call after a failure is `[["r", + * value]]`. `detach` unsubscribes it from its source on termination; `report` gets a listener's throw, which also ends + * the watch. */ -export class Watch extends WatchLifecycle> implements WatchHandle { +export class Watch extends WatchLifecycle> implements WatchHandle { readonly #ops = new PendingOps(); /** - * The source's change while it is the only frame pending: the call passes it on, with no allocation. A second frame + * The source's call while it is the only frame pending: the call passes it on, with no allocation. A second frame * drops it, so its ops stay only as long as `PendingOps` keeps them (folded, or dropped by an overflow). */ - #lone: WatchChange | undefined; + #lone: WatchValueCall | undefined; /** The newest frame's value and context, while a call is pending; the context also marks that one is. */ #newestValue: T | undefined; #newestContext: Context | undefined; /** A pending frame was a retirement. */ #retired = false; + /** The newest frame was a failure: the next call is this. */ + #failure: FailedCall | undefined; + /** A failure came after the last value call: the next value call replaces the root, and ops are not kept. */ + #replace = false; /** - * Queue the frame `change`: its ops turn the previous value into its value, and `retired` says the document the - * watch follows was retired, which the next call reports however many frames it folds. + * Queue the frame `call`. A value's ops turn the previous value into it, and `retired` says the document the watch + * follows was retired, which the next value call reports however many frames it folds; a failure is `fail()`. */ - advance(change: WatchChange, context: Context): void { + advance(call: WatchCall, context: Context): void { + if (call.kind === "failed") { + this.#fail(call, context); + return; + } if (!this.open) return; - this.#lone = this.#newestContext === undefined ? change : undefined; - this.#ops.push(change.ops); - this.#newestValue = change.value; + if (this.#replace) { + this.#failure = undefined; + } else { + this.#lone = this.#newestContext === undefined ? call : undefined; + this.#ops.push(call.ops); + } + this.#newestValue = call.value; this.#newestContext = context; - if (change.retired) this.#retired = true; + if (call.retired) this.#retired = true; + this.pending(); + } + + /** + * Queue a failure the source may recover from: the pending frames are dropped (`value` keeps the last delivered + * value), the listener gets `{ kind: "failed", error }`, and the next value arrives as `[["r", value]]`. + */ + fail(error: unknown, context: Context): void { + this.#fail({ kind: "failed", error }, context); + } + + #fail(call: FailedCall, context: Context): void { + if (!this.open) return; + this.#ops.take(); + this.#lone = undefined; + this.#newestValue = undefined; + this.#failure = call; + this.#newestContext = context; + this.#replace = true; this.pending(); } @@ -299,7 +346,9 @@ export class Watch extends WatchLifecycle> implements Watch return this.#newestContext; } - protected next(): WatchChange { + protected next(): WatchCall { + // After a failure the call is that failure, or the value after it; otherwise one field read decides. + if (this.#replace) return this.#nextAfterFailure(); const lone = this.#lone; // Set with the context, which says a frame is pending. const value = this.#newestValue as T; @@ -309,11 +358,27 @@ export class Watch extends WatchLifecycle> implements Watch this.#newestValue = undefined; this.#newestContext = undefined; this.#retired = false; - // One frame never folds or overflows: its change is the call's. + // One frame never folds or overflows: its call is the listener's. if (lone !== undefined) return lone; // Values of a JSON watch are JSON; `T` is the source's narrower or wider view of them (e.g. a typed document). const ops: readonly Op[] = batch.kind === "ops" ? batch.ops : [["r", value as JsonValue]]; - return { value, ops, retired }; + return { kind: "value", value, ops, retired }; + } + + /** The pending failure, or the value after one as a root replacement (no ops were kept since the failure). */ + #nextAfterFailure(): WatchCall { + const failure = this.#failure; + this.#newestContext = undefined; + if (failure !== undefined) { + this.#failure = undefined; + return failure; + } + const value = this.#newestValue as T; + const retired = this.#retired; + this.#newestValue = undefined; + this.#retired = false; + this.#replace = false; + return { kind: "value", value, ops: [["r", value as JsonValue]], retired }; } protected release(): void { @@ -322,35 +387,46 @@ export class Watch extends WatchLifecycle> implements Watch this.#newestValue = undefined; this.#newestContext = undefined; this.#retired = false; + this.#failure = undefined; + this.#replace = false; } } +/** The value call of a JSON watch. */ +type WatchValueCall = Extract, { readonly kind: "value" }>; + /** Maximum values retained behind one unavailable native watch listener. */ const MAX_PENDING_NATIVE_VALUES = 100; /** * The watch handle of a native source: every value in order, one call each; past 100 pending values they collapse to - * one, `replace()` or by default the newest value. `detach` and `report` as for `Watch`. + * one, `replace()` or by default the newest value. A failure (`fail()`) drops the pending values and waits as one + * `failed` call; a value after it replaces it. `detach` and `report` as for `Watch`. */ -export class NativeWatch extends WatchLifecycle> implements NativeWatchHandle { +export class NativeWatch extends WatchLifecycle> implements NativeWatchHandle { /** - * Pending values in order, each followed by its frame's context, from `#head` on. Values are kept raw: the change a - * call gets is built when it is taken, so a value an overflow drops costs no object. A read index instead of + * Pending values in order, each followed by its frame's context, from `#head` on. Values are kept raw: the call a + * listener gets is built when it is taken, so a value an overflow drops costs no object. A read index instead of * `shift()` takes a backlog in constant time per value; taken slots are cleared at once, the array is emptied once * all of it was taken, and its taken prefix dropped after 100 values. */ readonly #pending: Array = []; #head = 0; #replace: (() => T) | undefined; + /** A pending failure and its frame's context; nothing else is pending meanwhile. */ + #failure: FailedCall | undefined; + #failureContext: Context | undefined; constructor(value: T, detach: () => void, report: (error: unknown) => void, replace?: () => T) { super(value, detach, report); this.#replace = replace; } - /** Queue `value`, the source's new value. */ + /** Queue `value`, the source's new value; it replaces a pending failure. */ advance(value: T, context: Context): void { if (!this.open) return; + this.#failure = undefined; + this.#failureContext = undefined; if (this.#pending.length - this.#head < 2 * MAX_PENDING_NATIVE_VALUES) { this.#pending.push(value, context); } else { @@ -361,12 +437,31 @@ export class NativeWatch extends WatchLifecycle> impl this.pending(); } - protected get pendingContext(): Context | undefined { - // The slot after a value holds its context; past the end it is empty. - return this.#pending[this.#head + 1] as Context | undefined; + /** + * Queue a failure the source may recover from: the pending values are dropped (`value` keeps the last delivered + * value), and the listener gets `{ kind: "failed", error }`. + */ + fail(error: unknown, context: Context): void { + if (!this.open) return; + this.#pending.length = 0; + this.#head = 0; + this.#failure = { kind: "failed", error }; + this.#failureContext = context; + this.pending(); } - protected next(): NativeWatchChange { + protected get pendingContext(): Context | undefined { + // The slot after a value holds its context; past the end it is empty. + return this.#failureContext ?? (this.#pending[this.#head + 1] as Context | undefined); + } + + protected next(): NativeWatchCall { + const failure = this.#failure; + if (failure !== undefined) { + this.#failure = undefined; + this.#failureContext = undefined; + return failure; + } const head = this.#head; const pending = this.#pending; // The slot at `#head` holds a value. @@ -384,12 +479,14 @@ export class NativeWatch extends WatchLifecycle> impl pending[head + 1] = undefined; this.#head = head + 2; } - return { value }; + return { kind: "value", value }; } protected release(): void { this.#pending.length = 0; this.#head = 0; this.#replace = undefined; + this.#failure = undefined; + this.#failureContext = undefined; } } diff --git a/packages/chord/test/watch.test.ts b/packages/chord/test/watch.test.ts index 5ef530d60..d3b9c2416 100644 --- a/packages/chord/test/watch.test.ts +++ b/packages/chord/test/watch.test.ts @@ -12,7 +12,7 @@ import { import { applyImmutable, type Op, track } from "../src/delta/index.ts"; import { MAX_PENDING_OPS } from "../src/delta/pending.ts"; import type { JsonValue } from "../src/json.ts"; -import { NativeWatch, type NativeWatchChange, Watch, type WatchChange, type WatchEnd } from "../src/watch.ts"; +import { NativeWatch, type NativeWatchCall, Watch, type WatchCall, type WatchEnd } from "../src/watch.ts"; const context = BACKGROUND_CONTEXT; @@ -26,8 +26,25 @@ function deferred(): { readonly promise: Promise; readonly resolve: () => } /** A source's frame that is not a retirement. */ -function frame(value: T, ops: readonly Op[]): WatchChange { - return { value, ops, retired: false }; +function frame(value: T, ops: readonly Op[]): WatchValue { + return { kind: "value", value, ops, retired: false }; +} + +/** A source's retirement frame. */ +function retirement(value: T, ops: readonly Op[] = []): WatchValue { + return { kind: "value", value, ops, retired: true }; +} + +type WatchValue = Extract, { kind: "value" }>; + +/** A listener for value calls only: a `failed` call throws. */ +function onValue( + listener: (call: Extract, context: Context) => Promise, +): (call: C, context: Context) => Promise { + return (call, deliveryContext) => { + if (call.kind !== "value") throw new Error(`Unexpected ${call.kind} call`); + return listener(call as Extract, deliveryContext); + }; } /** A watch whose detach and reports are recorded. */ @@ -45,9 +62,11 @@ function recordedWatch(value: T) { function collectingWatch(value: T) { const { watch, recorded } = recordedWatch(value); const frames: Array<{ value: T; ops: readonly Op[]; retired?: true }> = []; - watch.start(async ({ value: delivered, ops, retired }) => { - frames.push(retired ? { value: delivered, ops, retired } : { value: delivered, ops }); - }); + watch.start( + onValue(async ({ value: delivered, ops, retired }) => { + frames.push(retired ? { value: delivered, ops, retired } : { value: delivered, ops }); + }), + ); return { watch, recorded, frames }; } @@ -87,9 +106,11 @@ describe("Watch delivery", () => { expect(watch.value).toEqual({ n: 0, text: "" }); const frames: Array<{ value: JsonValue; ops: readonly Op[] }> = []; - watch.start(async ({ value, ops }) => { - frames.push({ value, ops }); - }); + watch.start( + onValue(async ({ value, ops }) => { + frames.push({ value, ops }); + }), + ); expect(frames).toEqual([]); await flush(); expect(frames).toEqual([ @@ -107,9 +128,11 @@ describe("Watch delivery", () => { it("passes a frame's ops on by identity to a listener that keeps up", async () => { const { watch } = recordedWatch(0); const delivered: Array = []; - watch.start(async ({ ops }) => { - delivered.push(ops); - }); + watch.start( + onValue(async ({ ops }) => { + delivered.push(ops); + }), + ); const first: readonly Op[] = [["r", 1]]; const second: readonly Op[] = [["r", 2]]; watch.advance(frame(1, first), context); @@ -134,12 +157,14 @@ describe("Watch delivery", () => { const gate = deferred(); const calls: string[] = []; const frames: Array<{ value: JsonValue; ops: readonly Op[] }> = []; - watch.start(async ({ value, ops }) => { - calls.push(`start ${JSON.stringify(value)}`); - frames.push({ value, ops }); - if (frames.length === 1) await gate.promise; - calls.push(`end ${JSON.stringify(value)}`); - }); + watch.start( + onValue(async ({ value, ops }) => { + calls.push(`start ${JSON.stringify(value)}`); + frames.push({ value, ops }); + if (frames.length === 1) await gate.promise; + calls.push(`end ${JSON.stringify(value)}`); + }), + ); watch.advance(frame({ text: "a" }, [["a", ["text"], "a"]]), context); await flush(); watch.advance(frame({ text: "ab" }, [["a", ["text"], "b"]]), context); @@ -189,13 +214,15 @@ describe("Watch with a slow listener", () => { let replica: Doc = tracker.value; const calls: Array<{ commits: number; ops: number }> = []; let commits = 0; - watch.start(async ({ value, ops }) => { - replica = applyImmutable(replica, ops); - expect(replica).toEqual(value); - calls.push({ commits, ops: ops.length }); - // A listener taking a second per call, as a slow client would. - await new Promise((resolve) => setTimeout(resolve, 1000)); - }); + watch.start( + onValue(async ({ value, ops }) => { + replica = applyImmutable(replica, ops); + expect(replica).toEqual(value); + calls.push({ commits, ops: ops.length }); + // A listener taking a second per call, as a slow client would. + await new Promise((resolve) => setTimeout(resolve, 1000)); + }), + ); // Ten seconds of a streamed reply at 60 tokens per second, a new entry every second. for (let tick = 0; tick < 600; tick++) { if (tick % 60 === 0) commit((draft) => void draft.entries.push({ text: "", status: "streaming" })); @@ -244,10 +271,12 @@ describe("Watch overflow", () => { const { watch } = recordedWatch({ k: [] }); const gate = deferred(); const frames: Array<{ value: JsonValue; ops: readonly Op[] }> = []; - watch.start(async ({ value, ops }) => { - frames.push({ value, ops }); - if (frames.length === 1) await gate.promise; - }); + watch.start( + onValue(async ({ value, ops }) => { + frames.push({ value, ops }); + if (frames.length === 1) await gate.promise; + }), + ); watch.advance(frame({ k: [], first: true }, [["s", ["first"], true]]), context); await flush(); const newest = advanceDistinct(watch, MAX_PENDING_OPS + 1); @@ -282,7 +311,7 @@ describe("Watch retirement", () => { const settled = endOf(watch); watch.advance(frame(2, [["r", 2]]), context); await flush(); - watch.advance({ value: 2, ops: [], retired: true }, context); + watch.advance(retirement(2), context); await flush(); watch.advance(frame(3, [["r", 3]]), context); await flush(); @@ -300,30 +329,30 @@ describe("Watch retirement", () => { it("folds a retirement and the frames around it into one call that reports it", async () => { const { watch } = recordedWatch({ n: 0 }); watch.advance(frame({ n: 1 }, [["s", ["n"], 1]]), context); - watch.advance({ value: { n: 1 }, ops: [], retired: true }, context); + watch.advance(retirement({ n: 1 }), context); watch.advance(frame({ m: 1 }, [["r", { m: 1 }]]), context); - const changes: WatchChange[] = []; + const changes: WatchCall[] = []; watch.start(async (change) => { changes.push(change); }); await flush(); - expect(changes).toEqual([{ value: { m: 1 }, ops: [["r", { m: 1 }]], retired: true }]); + expect(changes).toEqual([{ kind: "value", value: { m: 1 }, ops: [["r", { m: 1 }]], retired: true }]); watch.advance(frame({ m: 2 }, [["s", ["m"], 2]]), context); await flush(); - expect(changes[1]).toEqual({ value: { m: 2 }, ops: [["s", ["m"], 2]], retired: false }); + expect(changes[1]).toEqual({ kind: "value", value: { m: 2 }, ops: [["s", ["m"], 2]], retired: false }); await watch.stop(); }); it("keeps reporting the retirement past an overflow", async () => { const { watch } = recordedWatch({ k: [] }); - watch.advance({ value: { k: [] }, ops: [], retired: true }, context); + watch.advance(retirement({ k: [] }), context); const newest = advanceDistinct(watch, MAX_PENDING_OPS + 1); - const changes: WatchChange[] = []; + const changes: WatchCall[] = []; watch.start(async (change) => { changes.push(change); }); await flush(); - expect(changes).toEqual([{ value: newest, ops: [["r", newest]], retired: true }]); + expect(changes).toEqual([{ kind: "value", value: newest, ops: [["r", newest]], retired: true }]); await watch.stop(); }); @@ -339,30 +368,159 @@ describe("Watch retirement", () => { { value: 2, ops: [["r", 2]] }, ]); expect(settled.end).toBeUndefined(); - expect(await watch.stop()).toEqual({ reason: "stopped" }); + expect(await watch.stop()).toEqual({ kind: "stopped" }); }); it("discards a retirement still pending when the source ends", async () => { const { watch } = recordedWatch(1); - watch.advance({ value: 1, ops: [], retired: true }, context); - watch.end(); - expect(await watch.closed).toEqual({ reason: "ended" }); + watch.advance(retirement(1), context); + watch.end({ kind: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); expect(watch.value).toBe(1); }); }); +describe("Watch failure calls", () => { + /** A started watch recording every call, failures included. */ + function recordingWatch(value: T) { + const { watch, recorded } = recordedWatch(value); + const calls: WatchCall[] = []; + watch.start(async (call) => { + calls.push(call); + }); + return { watch, recorded, calls }; + } + + it("delivers a failure as a call, stays open, keeps the last value, and replaces the root on recovery", async () => { + const { watch, recorded, calls } = recordingWatch({ n: 0 }); + const settled = endOf(watch); + watch.advance(frame({ n: 1 }, [["s", ["n"], 1]]), context); + await flush(); + const error = new Error("cannot compute"); + watch.fail(error, context); + await flush(); + expect(calls[1]).toEqual({ kind: "failed", error }); + expect(watch.value).toEqual({ n: 1 }); + expect(settled.end).toBeUndefined(); + expect(recorded.detached).toBe(0); + watch.advance(frame({ n: 2 }, [["s", ["n"], 2]]), context); + await flush(); + expect(calls[2]).toEqual({ kind: "value", value: { n: 2 }, ops: [["r", { n: 2 }]], retired: false }); + // Recovered: frames pass on as the source made them again. + const next = frame({ n: 3 }, [["s", ["n"], 3]]); + watch.advance(next, context); + await flush(); + expect(calls[3]).toBe(next); + expect(watch.value).toEqual({ n: 3 }); + await watch.stop(); + }); + + it("takes a failure through `advance()` as through `fail()`", async () => { + const { watch, calls } = recordingWatch(0); + const failure: WatchCall = { kind: "failed", error: "down" }; + watch.advance(failure, context); + await flush(); + expect(calls).toEqual([failure]); + expect(calls[0]).toBe(failure); + await watch.stop(); + }); + + it("folds pending values followed by a failure into the failure", async () => { + const { watch } = recordedWatch({ n: 0 }); + watch.advance(frame({ n: 1 }, [["s", ["n"], 1]]), context); + watch.advance(frame({ n: 2 }, [["s", ["n"], 2]]), context); + const first = new Error("first"); + const second = new Error("second"); + watch.fail(first, context); + watch.fail(second, context); + const calls: WatchCall[] = []; + watch.start(async (call) => { + calls.push(call); + }); + await flush(); + expect(calls).toEqual([{ kind: "failed", error: second }]); + expect(watch.value).toEqual({ n: 0 }); + await watch.stop(); + }); + + it("folds a pending failure followed by values into one root replacement with the newest value", async () => { + const { watch } = recordedWatch({ n: 0 }); + const key = createContextKey("frame"); + watch.fail(new Error("down"), withContextValue(key, "failure", context)); + watch.advance(frame({ n: 1 }, [["s", ["n"], 1]]), context); + watch.advance(retirement({ n: 1 }), context); + watch.advance(frame({ m: 2 }, [["r", { m: 2 }]]), withContextValue(key, "newest", context)); + const calls: Array<{ call: WatchCall; frame: string | undefined }> = []; + watch.start(async (call, deliveryContext) => { + calls.push({ call, frame: deliveryContext.value(key) }); + }); + await flush(); + expect(calls).toEqual([ + { call: { kind: "value", value: { m: 2 }, ops: [["r", { m: 2 }]], retired: true }, frame: "newest" }, + ]); + await watch.stop(); + }); + + it("delivers a failure that arrives during a call after it, then the recovery", async () => { + const { watch } = recordedWatch(0); + const gate = deferred(); + const calls: WatchCall[] = []; + watch.start(async (call) => { + calls.push(call); + if (calls.length === 1) await gate.promise; + }); + watch.advance(frame(1, [["r", 1]]), context); + await flush(); + watch.advance(frame(2, [["r", 2]]), context); + watch.fail("down", context); + gate.resolve(); + await flush(); + watch.advance(frame(3, [["r", 3]]), context); + await flush(); + expect(calls.map((call) => (call.kind === "value" ? call.value : call.error))).toEqual([1, "down", 3]); + expect(watch.value).toBe(3); + await watch.stop(); + }); + + it("drops a pending failure when the source ends, and ignores one after", async () => { + const { watch, calls } = recordingWatch(0); + watch.fail("down", context); + watch.end({ kind: "ended" }); + watch.fail("later", context); + await flush(); + expect(calls).toEqual([]); + expect(await watch.closed).toEqual({ kind: "ended" }); + }); + + it("passes every lone value call on by identity, so a value call costs the watch no allocation", async () => { + const { watch, calls } = recordingWatch(0); + const sent: WatchCall[] = []; + for (let n = 1; n <= 1_000; n++) { + const call = frame(n, [["r", n]]); + sent.push(call); + watch.advance(call, context); + await Promise.resolve(); + await Promise.resolve(); + } + await flush(); + expect(calls).toHaveLength(sent.length); + for (const [index, call] of calls.entries()) expect(call).toBe(sent[index]); + await watch.stop(); + }); +}); + describe("Watch end after delivery", () => { it("takes no more frames, delivers what is pending, then ends `ended` and only then detaches", async () => { const { watch, frames, recorded } = collectingWatch(1); const settled = endOf(watch); - watch.advance({ value: 1, ops: [], retired: true }, context); + watch.advance(retirement(1), context); watch.endAfterDelivery(); expect(recorded.detached).toBe(0); // The source sends nothing more: a later frame is not taken. watch.advance(frame(2, [["r", 2]]), context); await flush(); expect(frames).toEqual([{ value: 1, ops: [], retired: true }]); - expect(settled.end).toEqual({ reason: "ended" }); + expect(settled.end).toEqual({ kind: "ended" }); expect(recorded.detached).toBe(1); }); @@ -376,27 +534,29 @@ describe("Watch end after delivery", () => { expect(settled.end).toBeUndefined(); expect(recorded.detached).toBe(0); const values: JsonValue[] = []; - watch.start(async ({ value }) => { - values.push(value); - }); - expect(await watch.closed).toEqual({ reason: "ended" }); + watch.start( + onValue(async ({ value }) => { + values.push(value); + }), + ); + expect(await watch.closed).toEqual({ kind: "ended" }); expect(values).toEqual([2]); expect(recorded.detached).toBe(1); const idle = collectingWatch(1); idle.watch.endAfterDelivery(); - expect(await idle.watch.closed).toEqual({ reason: "ended" }); + expect(await idle.watch.closed).toEqual({ kind: "ended" }); expect(idle.frames).toEqual([]); expect(idle.recorded.detached).toBe(1); }); it("ends a never-started watch at its source's end, dropping what is pending", async () => { const { watch, recorded } = recordedWatch(1); - watch.advance({ value: 1, ops: [], retired: true }, context); + watch.advance(retirement(1), context); watch.endAfterDelivery(); const failure = new Error("source failed"); - watch.end({ error: failure }); - expect(await watch.closed).toEqual({ reason: "failed", error: failure }); + watch.end({ kind: "failed", error: failure }); + expect(await watch.closed).toEqual({ kind: "failed", error: failure }); expect(recorded.detached).toBe(1); expect(() => watch.start(async () => {})).toThrow("Watch is stopped"); }); @@ -407,13 +567,15 @@ describe("Watch end after delivery", () => { const gate = deferred(); const values: JsonValue[] = []; let returned = false; - watch.start(async ({ value }) => { - values.push(value); - await gate.promise; - await settle(); - returned = true; - }); - watch.advance({ value: 1, ops: [["r", 1]], retired: true }, context); + watch.start( + onValue(async ({ value }) => { + values.push(value); + await gate.promise; + await settle(); + returned = true; + }), + ); + watch.advance(retirement(1, [["r", 1]]), context); await flush(); expect(values).toEqual([1]); watch.endAfterDelivery(); @@ -427,7 +589,7 @@ describe("Watch end after delivery", () => { expect(ending.settled.end).toBeUndefined(); expect(ending.recorded.detached).toBe(0); ending.release(); - expect(await ending.watch.closed).toEqual({ reason: "ended" }); + expect(await ending.watch.closed).toEqual({ kind: "ended" }); expect(ending.returned()).toBe(true); expect(ending.values).toEqual([1]); expect(ending.recorded.detached).toBe(1); @@ -440,23 +602,23 @@ describe("Watch end after delivery", () => { }); expect(ending.settled.end).toBeUndefined(); ending.release(); - expect(await ending.watch.closed).toEqual({ reason: "listener_error", error: failure }); + expect(await ending.watch.closed).toEqual({ kind: "listener_error", error: failure }); expect(ending.recorded.reported).toEqual([failure]); expect(ending.recorded.detached).toBe(1); }); it("ends `stopped` when stopped while ending, without awaiting the running call", async () => { const ending = await endingWhileRunning(async () => {}); - expect(await ending.watch.stop()).toEqual({ reason: "stopped" }); + expect(await ending.watch.stop()).toEqual({ kind: "stopped" }); expect(ending.recorded.detached).toBe(1); expect(ending.returned()).toBe(false); ending.release(); await flush(); - expect(await ending.watch.closed).toEqual({ reason: "stopped" }); + expect(await ending.watch.closed).toEqual({ kind: "stopped" }); }); }); -describe("Watch change objects", () => { +describe("Watch call objects", () => { /** A frame context carrying `payload` and an abort signal, as a producer's is. */ function producerContext(payload: object) { const controller = new AbortController(); @@ -466,9 +628,9 @@ describe("Watch change objects", () => { ); } - it("hands a listener plain data only: `{ value, ops, retired }`, cloneable and serializable", async () => { + it("hands a listener plain data only: `{ kind, value, ops, retired }`, cloneable and serializable", async () => { const { watch } = recordedWatch({ n: 0 }); - const changes: WatchChange[] = []; + const changes: WatchCall[] = []; watch.start(async (change) => { changes.push(change); }); @@ -476,21 +638,21 @@ describe("Watch change objects", () => { await flush(); // Folded: the watch builds the change. watch.advance(frame({ n: 2 }, [["s", ["n"], 2]]), producerContext({})); - watch.advance({ value: { n: 2 }, ops: [], retired: true }, producerContext({})); + watch.advance(retirement({ n: 2 }), producerContext({})); await flush(); expect(changes).toHaveLength(2); for (const change of changes) { - expect(Object.keys(change)).toEqual(["value", "ops", "retired"]); + expect(Object.keys(change)).toEqual(["kind", "value", "ops", "retired"]); expect(structuredClone(change)).toEqual(change); expect(JSON.parse(JSON.stringify(change))).toEqual(change); } - expect(changes[1]).toEqual({ value: { n: 2 }, ops: [["s", ["n"], 2]], retired: true }); + expect(changes[1]).toEqual({ kind: "value", value: { n: 2 }, ops: [["s", ["n"], 2]], retired: true }); await watch.stop(); }); it("passes a frame delivered alone on as the source made it", async () => { const { watch } = recordedWatch(0); - const changes: WatchChange[] = []; + const changes: WatchCall[] = []; watch.start(async (change) => { changes.push(change); }); @@ -501,8 +663,8 @@ describe("Watch change objects", () => { await watch.stop(); }); - it("keeps no frame context alive through a change a listener keeps", async () => { - const kept: Array | NativeWatchChange> = []; + it("keeps no frame context alive through a call a listener keeps", async () => { + const kept: Array | NativeWatchCall> = []; /** Deliver one frame whose context alone references `payload`, keep the change, and stop the watch. */ const deliver = async (native: boolean): Promise> => { const payload = { big: "x".repeat(1000) }; @@ -517,8 +679,8 @@ describe("Watch change objects", () => { }); watch.advance(1, producerContext(payload)); await flush(); - expect(Object.keys(kept.at(-1)!)).toEqual(["value"]); - expect(structuredClone(kept.at(-1))).toEqual({ value: 1 }); + expect(Object.keys(kept.at(-1)!)).toEqual(["kind", "value"]); + expect(structuredClone(kept.at(-1))).toEqual({ kind: "value", value: 1 }); await watch.stop(); } else { const { watch } = recordedWatch(0); @@ -545,8 +707,8 @@ describe("Watch ends", () => { const { watch, recorded } = recordedWatch(0); watch.start(async () => {}); expect(() => watch.start(async () => {})).toThrow("Watch is already started"); - expect(await watch.stop()).toEqual({ reason: "stopped" }); - expect(await watch.stop()).toEqual({ reason: "stopped" }); + expect(await watch.stop()).toEqual({ kind: "stopped" }); + expect(await watch.stop()).toEqual({ kind: "stopped" }); expect(recorded.detached).toBe(1); const late = recordedWatch(0).watch; @@ -557,52 +719,54 @@ describe("Watch ends", () => { it("ends `ended` or `failed` from its source, dropping pending frames", async () => { const { watch, frames } = collectingWatch(0); watch.advance(frame(1, [["r", 1]]), context); - watch.end(); + watch.end({ kind: "ended" }); await flush(); expect(frames).toEqual([]); - expect(await watch.closed).toEqual({ reason: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); const failure = new Error("source failed"); const failed = recordedWatch(0).watch; - failed.end({ error: failure }); - failed.end(); - expect(await failed.closed).toEqual({ reason: "failed", error: failure }); + failed.end({ kind: "failed", error: failure }); + failed.end({ kind: "ended" }); + expect(await failed.closed).toEqual({ kind: "failed", error: failure }); }); it("stops future delivery without aborting or awaiting an in-flight callback", async () => { const { watch } = recordedWatch(0); const gate = deferred(); const calls: string[] = []; - watch.start(async ({ value }) => { - calls.push(`start ${value}`); - await gate.promise; - calls.push(`end ${value}`); - }); + watch.start( + onValue(async ({ value }) => { + calls.push(`start ${value}`); + await gate.promise; + calls.push(`end ${value}`); + }), + ); watch.advance(frame(1, [["r", 1]]), context); await flush(); watch.advance(frame(2, [["r", 2]]), context); - expect(await watch.stop()).toEqual({ reason: "stopped" }); + expect(await watch.stop()).toEqual({ kind: "stopped" }); gate.resolve(); await flush(); expect(calls).toEqual(["start 1", "end 1"]); }); - it("ends `cancelled` when its acquisition signal aborts, also when already aborted", async () => { + it("ends `aborted` when its acquisition signal aborts, also when already aborted", async () => { const controller = new AbortController(); const { watch, recorded } = recordedWatch(0); - watch.observeCancellation(controller.signal); - expect(() => watch.observeCancellation(controller.signal)).toThrow("Watch cancellation is already installed"); + watch.observeAbort(controller.signal); + expect(() => watch.observeAbort(controller.signal)).toThrow("Watch abort is already installed"); controller.abort(); - expect(await watch.closed).toEqual({ reason: "cancelled" }); + expect(await watch.closed).toEqual({ kind: "aborted" }); expect(recorded.detached).toBe(1); const aborted = recordedWatch(0).watch; - aborted.observeCancellation(AbortSignal.abort()); - expect(await aborted.closed).toEqual({ reason: "cancelled" }); + aborted.observeAbort(AbortSignal.abort()); + expect(await aborted.closed).toEqual({ kind: "aborted" }); - const cancelled = recordedWatch(0).watch; - cancelled.cancel(); - expect(await cancelled.closed).toEqual({ reason: "cancelled" }); + const direct = recordedWatch(0).watch; + direct.abort(); + expect(await direct.closed).toEqual({ kind: "aborted" }); }); it("ends `listener_error` on a throw, reports it, and delivers nothing more", async () => { @@ -610,16 +774,18 @@ describe("Watch ends", () => { const failure = new Error("listener failed"); const gate = deferred(); const values: JsonValue[] = []; - watch.start(async ({ value }) => { - values.push(value); - await gate.promise; - throw failure; - }); + watch.start( + onValue(async ({ value }) => { + values.push(value); + await gate.promise; + throw failure; + }), + ); watch.advance(frame(1, [["r", 1]]), context); await flush(); watch.advance(frame(2, [["r", 2]]), context); gate.resolve(); - expect(await watch.closed).toEqual({ reason: "listener_error", error: failure }); + expect(await watch.closed).toEqual({ kind: "listener_error", error: failure }); expect(values).toEqual([1]); expect(recorded.reported).toEqual([failure]); expect(recorded.detached).toBe(1); @@ -630,8 +796,8 @@ describe("Watch ends", () => { watch.start(() => Promise.reject("plain")); watch.advance(frame(1, [["r", 1]]), context); const end = await watch.closed; - expect(end.reason).toBe("listener_error"); - expect(end.reason === "listener_error" && end.error.message).toBe("plain"); + expect(end.kind).toBe("listener_error"); + expect(end.kind === "listener_error" && end.error.message).toBe("plain"); }); }); @@ -647,7 +813,7 @@ describe("Watch references", () => { () => void source, ); const listener = async (): Promise => {}; - watch.observeCancellation(controller.signal); + watch.observeAbort(controller.signal); watch.start(listener); watch.advance(frame(1, [["r", 1]]), context); await flush(); @@ -670,8 +836,8 @@ describe("Watch references", () => { for (const end of [ (watch: Watch) => void watch.stop(), - (watch: Watch) => watch.cancel(), - (watch: Watch) => watch.end(), + (watch: Watch) => watch.abort(), + (watch: Watch) => watch.end({ kind: "ended" }), (watch: Watch) => watch.endAfterDelivery(), ]) { const ended = await acquire(end); @@ -708,12 +874,12 @@ describe("Watch references", () => { const { watch, batches, newest } = overflow(); await collect(); expect(batches.map((batch) => batch.deref() === undefined)).toEqual([true, true, true, true]); - const changes: WatchChange[] = []; + const changes: WatchCall[] = []; watch.start(async (change) => { changes.push(change); }); await flush(); - expect(changes).toEqual([{ value: newest, ops: [["r", newest]], retired: false }]); + expect(changes).toEqual([{ kind: "value", value: newest, ops: [["r", newest]], retired: false }]); await watch.stop(); }); @@ -726,10 +892,12 @@ describe("Watch references", () => { const values: number[] = []; let next = 50; // Each call queues one more value behind a backlog of 50, so the queue never empties for 1,000 values. - watch.start(async ({ value }) => { - values.push(value); - if (next < 1_000) watch.advance(next++, context); - }); + watch.start( + onValue(async ({ value }) => { + values.push(value); + if (next < 1_000) watch.advance(next++, context); + }), + ); for (let n = 0; n < 50; n++) watch.advance(n, context); await flush(); expect(values).toEqual(Array.from({ length: 1_000 }, (_, n) => n)); @@ -785,9 +953,11 @@ describe("NativeWatch", () => { ); const key = createContextKey("frame"); const delivered: Array<{ value: { run: () => number }; frame: string | undefined }> = []; - watch.start(async ({ value }, deliveryContext) => { - delivered.push({ value, frame: deliveryContext.value(key) }); - }); + watch.start( + onValue(async ({ value }, deliveryContext) => { + delivered.push({ value, frame: deliveryContext.value(key) }); + }), + ); watch.advance(first, withContextValue(key, "first", context)); watch.advance(second, withContextValue(key, "second", context)); expect(delivered).toEqual([]); @@ -807,9 +977,11 @@ describe("NativeWatch", () => { ); for (let n = 1; n <= 101; n++) watch.advance(n, context); const values: number[] = []; - watch.start(async ({ value }) => { - values.push(value); - }); + watch.start( + onValue(async ({ value }) => { + values.push(value); + }), + ); await flush(); expect(values).toEqual([101]); }); @@ -823,9 +995,11 @@ describe("NativeWatch", () => { ); for (let n = 1; n <= 101; n++) watch.advance(`value ${n}`, context); const values: string[] = []; - watch.start(async ({ value }) => { - values.push(value); - }); + watch.start( + onValue(async ({ value }) => { + values.push(value); + }), + ); await flush(); expect(values).toEqual(["replacement"]); }); @@ -838,15 +1012,15 @@ describe("NativeWatch", () => { () => {}, ); const stopped = create(); - expect(await stopped.stop()).toEqual({ reason: "stopped" }); + expect(await stopped.stop()).toEqual({ kind: "stopped" }); - const cancelled = create(); - cancelled.observeCancellation(AbortSignal.abort()); - expect(await cancelled.closed).toEqual({ reason: "cancelled" }); + const aborted = create(); + aborted.observeAbort(AbortSignal.abort()); + expect(await aborted.closed).toEqual({ kind: "aborted" }); const ended = create(); - ended.end(); - expect(await ended.closed).toEqual({ reason: "ended" }); + ended.end({ kind: "ended" }); + expect(await ended.closed).toEqual({ kind: "ended" }); const finishing = create(); const finished: number[] = []; @@ -854,16 +1028,18 @@ describe("NativeWatch", () => { finishing.advance(2, context); finishing.endAfterDelivery(); finishing.advance(3, context); - finishing.start(async ({ value }) => { - finished.push(value); - }); - expect(await finishing.closed).toEqual({ reason: "ended" }); + finishing.start( + onValue(async ({ value }) => { + finished.push(value); + }), + ); + expect(await finishing.closed).toEqual({ kind: "ended" }); expect(finished).toEqual([1, 2]); const error = new Error("source failed"); const failed = create(); - failed.end({ error }); - expect(await failed.closed).toEqual({ reason: "failed", error }); + failed.end({ kind: "failed", error }); + expect(await failed.closed).toEqual({ kind: "failed", error }); const reported: unknown[] = []; const throwing = new NativeWatch( @@ -874,10 +1050,44 @@ describe("NativeWatch", () => { const failure = new Error("listener failed"); throwing.start(() => Promise.reject(failure)); throwing.advance(1, context); - expect(await throwing.closed).toEqual({ reason: "listener_error", error: failure }); + expect(await throwing.closed).toEqual({ kind: "listener_error", error: failure }); expect(reported).toEqual([failure]); }); + it("delivers a failure as a call, stays open, and folds it with the values around it", async () => { + const reported: unknown[] = []; + const watch = new NativeWatch( + 0, + () => {}, + (error) => reported.push(error), + ); + const calls: NativeWatchCall[] = []; + watch.start(async (call) => { + calls.push(call); + }); + watch.advance(1, context); + await flush(); + // Values followed by a failure: the failure; a failure followed by values: the values, in order. + watch.advance(2, context); + watch.advance(3, context); + watch.fail("down", context); + await flush(); + expect(watch.value).toBe(1); + watch.fail("again", context); + watch.advance(4, context); + watch.advance(5, context); + await flush(); + expect(calls).toEqual([ + { kind: "value", value: 1 }, + { kind: "failed", error: "down" }, + { kind: "value", value: 4 }, + { kind: "value", value: 5 }, + ]); + expect(watch.value).toBe(5); + expect(reported).toEqual([]); + await watch.stop(); + }); + it("retains neither its source nor its listener once stopped, while still referenced", async () => { const acquire = async (stop: boolean) => { const source = { value: 1 }; diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 17cfc25d6..a913ce222 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -82,6 +82,16 @@ Some event data has no state in the view. A tool's arguments after repair and `beforeTool` at its start: in the nested slot's `arguments` for a nested call, only in the tool task's checkpoint for a model-issued one. A submission's settlement: `Submission.status()`/`wait()` and `Harness.submission(id)`; an input settled without a run (withdrawn, `stale`, aborted while queued, `run_failed`) just leaves `pi.inbox`. A failed task's error: `Harness.getTask()` and `onReport`; the task graph keeps only live tasks, not terminal outcomes. A nested call's full result: its caller's `NestedCallDoc` (`watchDoc()`), which retires when the caller settles. Exact turn boundaries: the run's `taskId` and the task graph, with no explicit event. Whether a compaction did anything: its `CompactionResult` (`waitForTask()`), and the summary entry when it is placed. - Restart-limited tasks no longer park: a task cut off `settings.restartLimit` times in a row ends instead, visibly. A tool call ends with an error result (diagnostic code `restart_limit`) and its task fails with detail `{ reason: "restart_limit" }`, a compaction fails into `pi.live.compactionFailure`, a generation ends its run with `pi.live.runFailure` reason `restart_limit` (its inputs settle `unanswered` with that reason), and a task of an extension's kind ends `faulted` with detail `{ reason: "restart_limit" }`, reported as `task_faulted` with reason `restart_limit`. The built-in tasks are reported as `task_interrupted` (below). An abort handler cut off at the limit still faults its task, but a run someone aborted now ends with `runFailure` reason `aborted`, the cut-off as `detail`, and an aborted compaction leaves no `compactionFailure`. `Harness.retryTask()`, the `restart_limit` blocked reason of `TaskInspection` and `TaskGraphNode`, and the `task_parked` report (and its telemetry value) are removed. Migration: a parked task kept its conversation busy until `retryTask()`; it now ends, so show `runFailure`, `compactionFailure`, the tool's error result, or the `task_interrupted` and `task_faulted` reports instead, and to run the work again, submit it again (a new input, `compact()`, or a new task); raise `restartLimit` before the task is next reserved to let it run longer ([#11139](https://github.com/earendil-works/pi/issues/11139)). - A generation whose model request was in flight when the host died no longer resends it on open: the attempt counts as failed under `settings.retry`. Its partial becomes an aborted entry, as before; the next attempt waits for the backoff, shown in `pi.live.generation.retry`, and with no retry left the run ends with `pi.live.runFailure` reason `interrupted` and the inputs settle `unanswered` with that reason. A deferred poll whose fetch was in flight is charged the same way, even when the replacement host's clock is behind, and its next attempt polls the same deferred response again instead of submitting a new request; a run that ends instead cancels it. A close still resends. `restartLimit` still bounds these phases: a charge that itself keeps being cut off ends the run `restart_limit`, past the limit without converting the partial ([#11139](https://github.com/earendil-works/pi/issues/11139)). +- Watch listeners and synchronous observers take Chord's tagged watch shapes. Durable re-exports `WatchCall` and `NativeWatchCall` instead of `WatchChange` and `NativeWatchChange`; a listener gets `{ kind: "value", value, ops, retired }`. Chord's `{ kind: "failed", error }` call, for a source that recovers, never comes from Durable, whose failures end the watch. `WatchEnd` is tagged by `kind`, and a watch whose acquisition context is aborted ends `aborted` (was `cancelled`). Before: `watch.start(async ({ value, ops }) => render(value, ops))`, `if (end.reason === "cancelled")`. After: `watch.start(async (call) => { if (call.kind === "value") render(call.value, call.ops); })`, `if (end.kind === "aborted")`. +- `Observer` (`observeDoc()`, `observeConversation()`, `observeTaskGraph()`) has the watch's shapes: `advance(call: WatchCall, context)` (always a `value` call) and `end(end: WatchEnd)`: `ended` at close or after a task document's retirement with its task (was `undefined`), `failed` with the error that failed the Session (was `{ error }`), and `listener_error` with the error for an observer that threw (was `{ error }`), now also for one whose `advance` returns a promise that rejects. Before: `{ advance: ({ value }) => push(value), end: (failure) => failure && fail(failure.error) }`. After: `{ advance: (call) => call.kind === "value" && push(call.value), end: (end) => end.kind === "failed" && fail(end.error) }`. +- `subscribeCommits(listener)` is `observeCommits(observer)` with the observer shape: `{ advance(publication, context), end(end) }`, returning `{ detach() }`. Its observation has no `value`: commits are events, and the Session does not know Storage's newest `seq` before its own first commit. `end` comes once, `ended` after the last publication at close or `failed` when the Session fails; an observer that throws (or whose `advance` rejects) is reported, detached, and ended `listener_error` (a throwing listener stayed subscribed). Before: `const unsubscribe = harness.subscribeCommits((publication) => apply(publication)); unsubscribe();`. After: `const observation = harness.observeCommits({ advance: (publication) => apply(publication), end: () => {} }); observation.detach();`. +- `HarnessOptions.onReport(report)` takes one `Report`: the fields of `ReportInfo`, tagged by `kind` (was `type`), and the `error` (was the first argument). `session_failed` is `storage_failed`, and `task_outdated`'s reason `incompatible_task` is the scheduler's own word for why the replacement cannot take the task: `task_too_old` (older than the task) or `migration_failed` (newer without a `migrate`), as a blocked task shows. The `pi.durable.report` event and `pi.durable.reports` metric keep their `type` attribute, with the new values. Before: `onReport: (error, info) => { if (info.type === "session_failed") restart(error); }`. After: `onReport: (report) => { if (report.kind === "storage_failed") restart(report.error); }`. +- `DocumentCommitChange` is tagged: a document change is `{ type: "document", kind: "change", record, conversationId, version, value, ops }` and a retirement `{ type: "document", kind: "retire", record, conversationId }` (was `value: null` and `version: undefined`). Before: `if (change.type === "document" && change.value !== null) use(change.value)`. After: `if (change.type === "document" && change.kind === "change") use(change.value)`. +- `CompactionStatus.reason` and `CompactionFailure.reason` in `pi.live` are `trigger` (what started the compaction; a run failure's `reason` is why it failed). `pi.live` is version 2: its `migrate` renames the field in a stored version 1 value when it is read, and the next commit that writes `pi.live` stores it at version 2, so code built against version 1 cannot read a conversation this version wrote to. Before: `live.compactionFailure?.reason`. After: `live.compactionFailure?.trigger`. +- `TaskInspection.state` uses `TaskState`'s word: `{ kind: "pending", migrates }` (was `ready`). Before: `state.kind === "ready"`. After: `state.kind === "pending"`. +- `FileChange` (`FileSystem.watch()` in `@earendil-works/pi-durable/env`) is tagged by `kind`: `{ kind: "paths", paths }`, `{ kind: "overflow" }`, or `{ kind: "failed", error }` (was `{ paths }`, `{ overflow: true }`, `{ error }`), and `FileWatcher.close(context)` is `stop(context)`. Before: `if ("overflow" in change) rescan(); else if ("paths" in change) reload(change.paths); await watcher.close(ctx);`. After: `if (change.kind === "overflow") rescan(); else if (change.kind === "paths") reload(change.paths); await watcher.stop(ctx);`. +- `SessionEnd` (`Session.closed`, `Harness.closed`) is tagged by `kind`: `{ kind: "closed" }` or `{ kind: "failed", error }` (was `reason`). Before: `harness.closed.then((end) => { if (end.reason === "failed") reopen(end.error); })`. After: `harness.closed.then((end) => { if (end.kind === "failed") reopen(end.error); })`. +- `ReportFrom`, the second argument of a task's `runtime.report(error, from)`, is tagged by `kind`: `{ kind: "extension", extension? }` or `{ kind: "host", callback }` (was `type`). Before: `runtime.report(error, { type: "host", callback: "env" })`. After: `runtime.report(error, { kind: "host", callback: "env" })`. ### Added diff --git a/packages/durable/README.md b/packages/durable/README.md index f57c2629b..9b2882e22 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -84,7 +84,7 @@ Every async call takes a Chord `Context`, which carries cancellation and its cur - `pi.durable.submit` covers the public submit call and its commit. Each input starts a new root trace with a link to its caller's current span; a write is a child of its caller. Duplicate `requestId` preserves the original submission and carrier. Internal admissions, such as compaction writes, emit events on their phase span without a submit span. - `pi.durable.phase` covers one run or abort phase of every task, including its decision commit, never the next phase, a durable wait, or a restart. Its attributes identify the task, phase, attempt when present, `resumed`, and outcome. Tool phases also identify the tool and call. Nested tasks and background compactions inherit the creating phase; model requests receive that phase context. Hooks add no spans. - Failed phase outcomes set error status with a fixed category (`task_faulted`, `task_failed`, or `task_orphaned`) and the outcome's diagnostic message, truncated to 500 characters (the reason for an orphaned task). Diagnostics can contain caller content; no error detail or exception name is recorded. An `aborted` outcome keeps status `ok`, including interruption by close or cancellation. -- Every `onReport` report also emits a `pi.durable.report` event on its current span. Without a span, the NOOP span drops the event; no root report span is created. Only `type`, optional fixed `reason`, `extension`, and `task.id` are recorded; no error payload is included. `pi.durable.reports` counts all reports by `type` only, including those without a span. Clock failures report without a phase context. The `onReport` callback is unchanged. +- Every `onReport` report also emits a `pi.durable.report` event on its current span. Without a span, the NOOP span drops the event; no root report span is created. Only `type` (the report's `kind`), optional fixed `reason`, `extension`, and `task.id` are recorded; no error payload is included. `pi.durable.reports` counts all reports by `type` only, including those without a span. Clock failures report without a phase context. The `onReport` callback is unchanged. - Committed entries and settled submissions emit `pi.durable.entry` and `pi.durable.submission`. Committed running phase changes emit `pi.durable.checkpoint`; a nested call answered from its stored result emits `pi.durable.replay`. - Propagation stores optional opaque `trace` carriers in submission and task JSON without a storage migration. Phase spans extract their task's stored carrier, or start at the adapter root when absent or unknown. `TaskOptions.trace` inherits the creator's carrier when absent, accepts an explicit carrier override, or uses `"root"` to start at the adapter root without storing a carrier. A run started for several queued inputs takes the first input's trace; its first invocation's phase spans link to the other inputs through immutable task `links`. A steer stays in the running trace and its submit span links to that run's stored carrier. A reopened task's first invocation has `resumed: true` in the same trace. - Metrics record `pi.durable.phase.duration` (milliseconds; task, phase, outcome) and `pi.durable.tasks.active` (live tasks; task name), without record IDs. Closing the Harness releases its active counts. @@ -215,7 +215,7 @@ The default `setTimer` declines the timers that only free memory in Workers, so **Crash loops.** Work that deterministically crashes its process, by exhausting memory or CPU before its next checkpoint, would otherwise be resumed and crash again forever. A model request the host died during is a failed attempt under `settings.retry`: its partial answer is kept as an aborted entry, the next attempt waits for the backoff (shown in `docs["pi.live"].generation.retry`), and with no retry left the run ends with `docs["pi.live"].runFailure` reason `interrupted`. A deferred response that was being fetched is polled again as the next attempt, not requested again, and cancelled when the run ends. A close is not a crash: a request it stopped is sent again. Other work is counted: open counts each task a crash, eviction, or reset cut off, and a new state the task commits starts the count over. Once the count reaches `settings.restartLimit` (default 5; `Infinity` never ends a task for it), the task ends instead of running again: a tool call with an error result (diagnostic code `restart_limit`), a compaction failed into `compactionFailure`, a run with `runFailure` reason `restart_limit` (also when charging an interrupted request keeps crashing), and a task of your own kinds `faulted`, reported as `task_faulted` with reason `restart_limit`. The built-in ones are reported as `task_interrupted`, as is a run that ends `interrupted`, so a crash loop always reaches `onReport`. Every task running when a process dies is counted, not only the one that killed it. An abort handler cut off that many times ends its task `faulted` too; a run someone aborted still ends `aborted`, with the cut-off in `detail`. -**Reports.** `onReport(error, info)` says what happened in `info.type`: `session_failed` for the error that failed the Harness (a Storage error, often a platform reset); `task_faulted`, `task_interrupted` (crashes ended it, with `info.reason` and `info.restarts`), `migration_failed`, `task_outdated`, or `task_error` for a task, with `info.task` (its `id`, `kind`, and `conversationId`) and the extension that installed it in `info.extension`; `extension_error` for hooks, sections, wrappers, and tools, with the extension's name; `host_error` for a host callback, named in `info.callback`; and `listener_error`. A host can feed `task_faulted` reports of one kind to its own breaker. +**Reports.** `onReport(report)` gets one `Report`: the `error` and what happened in `report.kind`: `storage_failed` for the error that failed the Harness (a Storage error, often a platform reset); `task_faulted`, `task_interrupted` (crashes ended it, with `report.reason` and `report.restarts`), `migration_failed`, `task_outdated` (`reason` `missing_task`, `task_too_old`, or `migration_failed`, the words a blocked task shows), or `task_error` for a task, with `report.task` (its `id`, `kind`, and `conversationId`) and the extension that installed it in `report.extension`; `extension_error` for hooks, sections, wrappers, and tools, with the extension's name; `host_error` for a host callback, named in `report.callback`; and `listener_error`. A host can feed `task_faulted` reports of one kind to its own breaker. **Queued inputs after a failed run.** A run that fails with a model error, a missing model, or crashes (`interrupted`, `restart_limit`) leaves the inputs queued behind it waiting until the next input arrives. `settings.afterFailedRun` changes that: `continue` starts the next run with them, as after an answer, and `withdraw` settles them `unanswered` with reason `run_failed`. @@ -478,7 +478,9 @@ Everything a UI needs is committed state. `watch()` returns a watch of the conve ```typescript const watch = await root.watch(context); render(watch.value); // the state at attachment -watch.start(async ({ value, ops }) => { +watch.start(async (call) => { + if (call.kind !== "value") return; // Durable's watches end on a failure; they never send a failed call + const { value, ops } = call; // value.entries: a window over the transcript, newest last; value.before: the entry just before it // value.docs["pi.live"]: the run, the running generation (streamed partial, retry, deferred), tool calls (output, // details), compactions, and why the last run ended without an answer (runFailure) @@ -499,7 +501,9 @@ import { ConversationTranscript } from "@earendil-works/pi-durable"; const transcript = new ConversationTranscript((query, limit, ctx) => root.entries(query, limit, undefined, ctx)); const watch = await root.watch(context); render(transcript.merge(watch.value)); -watch.start(async ({ value }) => render(transcript.merge(value))); +watch.start(async (call) => { + if (call.kind === "value") render(transcript.merge(call.value)); +}); // When a marker scrolls into view: render(await transcript.loadEarlier(context)); // the page before the oldest entry @@ -511,9 +515,9 @@ const resumed = await root.watch(context, transcript.resume); Entries never change once committed, so a merge only adds; one that adds nothing returns the same `items` array. See `test/examples/31-transcript.ts`. -Watches are Chord's `WatchHandle` (exported by `@earendil-works/chord`; Durable re-exports it). The listener gets one change per call, `{ value, ops, retired }`, and the newest commit's context. A watch ends once, and `closed` (or `stop()`) gives the reason: `stopped`, `cancelled` (its acquisition context was cancelled), `ended` (the Harness closed, or a task document's task ended), `listener_error`, or `failed` (the Harness failed). An ended watch keeps its last value and no longer references the Harness. +Watches are Chord's `WatchHandle` (exported by `@earendil-works/chord`; Durable re-exports it). The listener gets one `WatchCall` per call, tagged by `kind`, and the newest commit's context: `{ kind: "value", value, ops, retired }`. Chord's watches also have `{ kind: "failed", error }` calls for failures a source recovers from; Durable's failures are final, so a Durable watch never sends one and ends instead. A watch ends once, and `closed` (or `stop()`) gives a `WatchEnd` tagged by `kind`: `stopped`, `aborted` (its acquisition context was aborted), `ended` (the Harness closed, or a task document's task ended), `listener_error`, or `failed` (the Harness failed). An ended watch keeps its last value and no longer references the Harness. -Adapters that must change in the same step as Durable's memory, such as a Chord source over a view, attach a synchronous observer instead of a watch: `harness.observeConversation(id, observer, context)`, `harness.observeTaskGraph(observer, context)`, and `harness.observeDoc(token, ...scope, observer, context)` return the current `value` and `detach()`, and call `observer.advance(change, context)` with every later change, unfolded, as memory adopts it: inside the storage's commit for a stored commit, at once for a deferred one (before it is stored). A document observer gets one call per change at its address, so a commit that retires and recreates the document gives two; the task graph also advances without a commit when a registry change alters a task's `blocked` reason. A change can arrive before the `observeXXX()` promise resolves; its ops apply from the observation's `value`. `observer.end(failure)` comes once, when the Harness closes or fails, or, for a task document, right after the retirement of the commit that ends its task. `advance` only records the change: no awaits, no Harness calls, no throws, and quick, since every commit waits for it. One that throws is reported, detached, and ended with the error. Watches are built on these observers. +Adapters that must change in the same step as Durable's memory, such as a Chord source over a view, attach a synchronous observer instead of a watch: `harness.observeConversation(id, observer, context)`, `harness.observeTaskGraph(observer, context)`, and `harness.observeDoc(token, ...scope, observer, context)` return the current `value` and `detach()`, and call `observer.advance(call, context)` with every later change as a `WatchCall` (always a `value` call), unfolded, as memory adopts it: inside the storage's commit for a stored commit, at once for a deferred one (before it is stored). A document observer gets one call per change at its address, so a commit that retires and recreates the document gives two; the task graph also advances without a commit when a registry change alters a task's `blocked` reason. A change can arrive before the `observeXXX()` promise resolves; its ops apply from the observation's `value`. `observer.end(end)` comes once with a `WatchEnd`: `ended` when the Harness closes or, for a task document, right after the retirement of the commit that ends its task, `failed` when the Harness fails. `advance` only records the change: no awaits, no Harness calls, no throws, and quick, since every commit waits for it. One that throws (or returns a rejected promise) is reported, detached, and ended `listener_error`. Watches are built on these observers. Commits that land while a callback runs are delivered together in the next callback: the newest view and the operations of all of them, folded (appends to one string join, sets of one field keep the last). Past 10,000 pending operations the callback gets one root replacement `[["r", view]]` holding the whole newest view instead, its window included. A client that joins late or reconnects starts from the current view; nothing is replayed. @@ -607,7 +611,7 @@ When a provider rejects a request because the context is too long, generation co When a compaction that a generation waits for at the blocking threshold fails, the generation fails too, without sending a request: its inputs settle `unanswered` with `model_error` and detail `Compaction failed: `. One that finds nothing to compact or is declined lets the request go ahead. -Running compactions are listed in `docs["pi.live"].compactions` with their reason, attempt, and retry backoff. The newest failure stays in `docs["pi.live"].compactionFailure` (task, reason, error) until a compaction produces a summary. +Running compactions are listed in `docs["pi.live"].compactions` with their `trigger` (`manual`, `threshold`, or `overflow`), attempt, and retry backoff. The newest failure stays in `docs["pi.live"].compactionFailure` (task, trigger, error) until a compaction produces a summary. ## Hooks @@ -719,8 +723,9 @@ decide: async (task, runtime, context) => { ```typescript const graph = await harness.watchTaskGraph(context); -graph.start(async ({ value }) => { - for (const node of Object.values(value.tasks)) { +graph.start(async (call) => { + if (call.kind !== "value") return; + for (const node of Object.values(call.value.tasks)) { const status = node.state.status === "waiting" ? `waiting on ${node.state.on.join(", ")}` : node.state.status; console.log(`${node.id} ${node.kind} ${status}`, node.owner ?? `conversation ${node.conversationId}`); } @@ -764,9 +769,9 @@ await root.commit( ); ``` -Watches and the view see it at once. Storage gets it with the next commit that reads or writes the document, before the document leaves memory, at close, and at the latest about `withinMs` later; `subscribeCommits()` sees it then, with exactly what was stored (a document the commit retires shows as its retirement only). A crash loses what was not stored yet, so derive nothing durable from such a value outside a commit that reads it. Only a commit that changes nothing but existing documents with `history: "latest"` and `fork: "initial"` is deferred; anything else in it, such as an entry, a new document, or a task state, stores it at once. +Watches and the view see it at once. Storage gets it with the next commit that reads or writes the document, before the document leaves memory, at close, and at the latest about `withinMs` later; `observeCommits()` sees it then, with exactly what was stored (a document the commit retires shows as its retirement only). A crash loses what was not stored yet, so derive nothing durable from such a value outside a commit that reads it. Only a commit that changes nothing but existing documents with `history: "latest"` and `fork: "initial"` is deferred; anything else in it, such as an entry, a new document, or a task state, stores it at once. -`harness.watchDoc()` watches one document like the view above. A document watch follows its address: a retirement is a call with `retired: true`, the last value the watch was given (a change made in the retiring commit itself is not shown), and no ops, and a document created there later arrives as `[["r", value]]` on the same watch. A task document's watch ends `ended` with its task: once the retirement call of the commit that ends the task has returned, or at the Harness's close if that call was never delivered (a watch never started); an observer of it gets `end(undefined)` right after the retirement. `HarnessOptions.conversationCreated(tx, conversation)` runs in every commit that creates or forks a conversation, including a tool's raw `tx.createConversation()`, so every conversation gets your documents; `init` in `createConversation()`, `fork()`, and `root()` writes per-call data in the same commit. An extension's tools, sections, and hooks read their own documents through `api` or `input.read`, and treat an absent one as its default ([`10-extension-state.ts`](test/examples/10-extension-state.ts)). +`harness.watchDoc()` watches one document like the view above. A document watch follows its address: a retirement is a call with `retired: true`, the last value the watch was given (a change made in the retiring commit itself is not shown), and no ops, and a document created there later arrives as `[["r", value]]` on the same watch. A task document's watch ends `ended` with its task: once the retirement call of the commit that ends the task has returned, or at the Harness's close if that call was never delivered (a watch never started); an observer of it gets `end({ kind: "ended" })` right after the retirement. `HarnessOptions.conversationCreated(tx, conversation)` runs in every commit that creates or forks a conversation, including a tool's raw `tx.createConversation()`, so every conversation gets your documents; `init` in `createConversation()`, `fork()`, and `root()` writes per-call data in the same commit. An extension's tools, sections, and hooks read their own documents through `api` or `input.read`, and treat an absent one as its default ([`10-extension-state.ts`](test/examples/10-extension-state.ts)). ## Usage and Cost @@ -848,7 +853,7 @@ Three rules decide what an error does: ```typescript harness.closed.then((end) => { - if (end.reason === "failed") reopen(end.error); + if (end.kind === "failed") reopen(end.error); }); ``` diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index 647a2822b..70a185a86 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -9,6 +9,7 @@ import type { Draft, JsonRepresentation, JsonValue, + WatchCall, WatchEnd, WatchHandle, } from "@earendil-works/chord"; @@ -68,7 +69,7 @@ Required invariants: adopts it (section 4), also when `Storage.commit()` then rejects; a commit that fails before it is durable publishes nothing. A deferred commit (section 4.1) is published to watches and the conversation view when memory - adopts it, and to `subscribeCommits()` once it is stored. + adopts it, and to `observeCommits()` once it is stored. 3. All visible progress is durable, except what a deferred commit holds: what watches see can be ahead of Storage by up to that commit's window (`withinMs`), and a crash loses it. A commit is deferred only when its caller @@ -409,10 +410,10 @@ type HarnessOptions = { */ readonly onWake?: (wake: Wake) => void; /** - * Failures that do not fail the calling operation, and the Session's failure, once; `info` says where each came - * from (section 10.1). + * Failures that do not fail the calling operation, and the Session's failure, once; `report.kind` says where each + * came from (section 10.1). */ - readonly onReport?: (error: unknown, info: ReportInfo) => void; + readonly onReport?: (report: Report) => void; }; /** Curated pi-ai request options; absent fields use pi-ai defaults. */ @@ -597,7 +598,7 @@ type TaskInspection = { readonly record: TaskRecord; readonly state: | { readonly kind: "running" } - | { readonly kind: "ready"; readonly migrates: boolean } + | { readonly kind: "pending"; readonly migrates: boolean } | { readonly kind: "waiting"; readonly on: readonly TaskId[] } /** A sleep whose `until` the clock has not reached (section 5.6). */ | { readonly kind: "sleeping"; readonly until: number } @@ -617,36 +618,38 @@ type Wake = { readonly active: boolean; readonly at?: number }; type ReportTask = { readonly id: TaskId; readonly kind: string; readonly conversationId: ConversationId }; type HostCallback = "now" | "settings" | "registry" | "env" | "models" | "setTimer" | "onWake" | "telemetry"; type ReportInfo = - | { readonly type: "session_failed" } + | { readonly kind: "storage_failed" } | { - readonly type: "task_faulted"; + readonly kind: "task_faulted"; readonly task: ReportTask; readonly extension?: string; readonly reason: "threw" | "restart_limit"; } | { - readonly type: "task_interrupted"; + readonly kind: "task_interrupted"; readonly task: ReportTask; readonly extension?: string; readonly reason: "restart_limit" | "interrupted"; readonly restarts: number; } - | { readonly type: "migration_failed"; readonly task: ReportTask; readonly extension?: string } + | { readonly kind: "migration_failed"; readonly task: ReportTask; readonly extension?: string } | { - readonly type: "task_outdated"; + readonly kind: "task_outdated"; readonly task: ReportTask; readonly extension?: string; - readonly reason: "missing_task" | "incompatible_task"; + readonly reason: "missing_task" | "task_too_old" | "migration_failed"; } - | { readonly type: "task_error"; readonly task: ReportTask; readonly extension?: string } + | { readonly kind: "task_error"; readonly task: ReportTask; readonly extension?: string } | { - readonly type: "extension_error"; + readonly kind: "extension_error"; readonly extension?: string; readonly task?: ReportTask; readonly conversationId?: ConversationId; } - | { readonly type: "host_error"; readonly callback: HostCallback; readonly task?: ReportTask } - | { readonly type: "listener_error" }; + | { readonly kind: "host_error"; readonly callback: HostCallback; readonly task?: ReportTask } + | { readonly kind: "listener_error" }; +/** One report: what it is about, and the error. */ +type Report = ReportInfo & { readonly error: unknown }; type HarnessInspection = { readonly scheduling: "paused" | "running" | "closing"; @@ -1062,7 +1065,7 @@ under the current registry: `running` with an active invocation; `completing` for a held outcome; `waiting` with the live tasks it still waits for, the live part of its `on`, or, when abort-marked, its live ordinary owned work (section 5.5); `sleeping` with its `until` while the clock has not reached it (section -5.6); `ready` when the next scheduling pass would reserve it, with `migrates` +5.6); `pending` when the next scheduling pass would reserve it, with `migrates` when its definition is newer and has `migrate`; or `blocked` (section 5.4). A migration shows as failed only after the scheduler tried it, or when the newer definition has no `migrate`; inspection never runs one to find out. Blocked reasons are derived, never stored. Queued and placed @@ -1310,20 +1313,55 @@ without storage-assigned lifetime fields. There is no mutable `session.document()` API. ```ts -type SessionEnd = { readonly reason: "closed" } | { readonly reason: "failed"; readonly error: unknown }; +type SessionEnd = { readonly kind: "closed" } | { readonly kind: "failed"; readonly error: unknown }; type CommitPersistence = | { readonly kind: "immediate" } | { readonly kind: "deferred"; readonly withinMs?: number }; type CommitOptions = { readonly persistence?: CommitPersistence }; +/** One stored commit as `observeCommits()` publishes it (section 4.1); change order is unspecified. */ +type CommitPublication = { readonly seq: Seq; readonly changes: readonly CommitChange[] }; +type CommitChange = + | Extract + | DocumentCommitChange; +/** A creation or update (`kind: "change"`), a retirement (`kind: "retire"`), or a definition-free copy. */ +type DocumentCommitChange = + | { + readonly type: "document"; + readonly kind: "change"; + readonly record: DocumentRecord; + readonly conversationId: ConversationId | undefined; + readonly version: number; + readonly value: JsonObject; + /** Empty for a creation. */ + readonly ops: readonly Op[]; + } + | { + readonly type: "document"; + readonly kind: "retire"; + readonly record: DocumentRecord; + readonly conversationId: ConversationId | undefined; + } + | { + readonly type: "document.copy"; + readonly record: DocumentRecord; + readonly conversationId: ConversationId; + readonly source: { readonly id: DocumentId; readonly at: DocumentPoint }; + }; +type CommitObserver = { + advance(publication: CommitPublication, context: Context): void; + end(end: WatchEnd): void; +}; +type CommitObservation = { detach(): void }; + interface Session extends DocumentObserver { /** `options.persistence` may defer storing the commit (section 4.1). */ commit(change: (tx: Tx) => T | Promise, context: Context, options?: CommitOptions): Promise; close(context: Context): Promise; - /** Settles once closed, Storage included: `{ reason: "closed" }`, or `{ reason: "failed", error }` (section 10.1). */ + /** Settles once closed, Storage included: `{ kind: "closed" }`, or `{ kind: "failed", error }` (section 10.1). */ readonly closed: Promise; - subscribeCommits(listener: (publication: CommitPublication, context: Context) => void): () => void; + observeCommits(observer: CommitObserver): CommitObservation; subscribeClose(listener: () => void): () => void; snapshot(token: SessionDocToken, context: Context): Promise | undefined>; @@ -1524,13 +1562,13 @@ callback succeeds with no pending acquisition Storage.commit persists the atomic batch while the Session line remains held commit deferred and holdable (section 4.1) adopt every prepared change by pointer swap; queue its ops per document, unstored - watches and view mounts capture it synchronously; subscribeCommits() does not + watches and view mounts capture it synchronously; observeCommits() does not release the line; invoke listeners later otherwise, right before Storage admission take the held ops of every document the commit acquired and put them in front of its own storage makes it durable and calls adopt(seq), inside Storage.commit() (section 4) adopt every prepared change by pointer swap; record its telemetry events - queue one microtask that calls subscribeCommits() listeners with what was stored + queue one microtask that calls observeCommits() observers with what was stored the Session's own observers, including watches and view mounts (section 9.3), capture it synchronously; the waits they resolve and the aborts they queue run after that microtask storage resolves @@ -1764,7 +1802,7 @@ observers, synchronously: document watches, conversation view mounts, the task graph, the scheduler, and submission waits, with the observers adapters attach (`observeDoc()`, `observeConversation()`, `observeTaskGraph()`, section 9.1). They advance watches and resolve waits, but run no task code: the abort signal of a task the commit newly marks -follows in a microtask. Host `subscribeCommits()` listeners run one microtask +follows in a microtask. Host `observeCommits()` observers run one microtask after `adopt`, outside the storage's synchronous step, before every wait the commit resolves and every abort it queues: those continuations are queued by the own observers, after the dispatch. There is no order between host listeners @@ -1777,7 +1815,7 @@ what the commit wakes during that work too; its next commit still waits. ```text adopt(41): trackers, telemetry events, own observers (views advance, waits resolve) -microtask: subscribeCommits() listeners +microtask: observeCommits() observers microtasks after it: wait continuations, abort listeners microtasks, in no order relative to the above: watch listeners, onWake Storage.commit() resolves; session.commit() resolves @@ -1797,18 +1835,24 @@ published, waits it resolved stay resolved, the commit rejects with the storage error, and the Session fails with it; an own observer's throw in that commit is then reported as a `listener_error`. -`subscribeCommits()` observes stored commits in one synchronous call per -listener, on the line, each with exactly what that storage transaction stored (section 4.1). Adoption -queues the one dispatch of a commit only when a listener is registered; the -dispatch calls the listeners registered when it runs. So a listener added -between `adopt` and the dispatch gets the commit only when some listener was -registered at `adopt`; one removed in between does not get it; one added after -the dispatch ran misses it. `subscribeClose()` observes close synchronously when it +`observeCommits(observer)` observes stored commits in one synchronous +`observer.advance(publication, context)` call per observer, on the line, each with exactly what that storage +transaction stored (section 4.1). It attaches synchronously and returns a `CommitObservation` with an idempotent +`detach()`; unlike the other `observeXXX()` it has no `value`: commits are events, not a state, and the Session does +not know Storage's newest `seq` before its own first commit. Adoption +queues the one dispatch of a commit only when an observer is attached; the +dispatch calls the observers attached when it runs. So an observer attached +between `adopt` and the dispatch gets the commit only when some observer was +attached at `adopt`; one detached in between does not get it; one attached after +the dispatch ran misses it. `observer.end(end)` comes once with a `WatchEnd`: `ended` when the Session has closed, +after its last publication, or `failed` with the error that failed it; never after `detach()`. An observer whose +`advance` throws, or returns a promise that rejects, is reported (section 10.1), detached, and ended +`listener_error`; the others still get the commit. `subscribeClose()` observes close synchronously when it begins, after admission is sealed, also when a failure closes the Session; the Harness ends waits, watches, and streams there, and signals task invocations after every close listener has run, so what a task holds ends with the Session's -reason rather than as cancelled. Both return idempotent -disposers; their listeners must not block or call Session APIs, and one that +reason rather than as aborted. It returns an idempotent +disposer; its listeners must not block or call Session APIs, and one that throws is reported without affecting the others (section 10.1). Watch listeners still run later, as microtasks the line never waits for (invariant 7). @@ -1905,13 +1949,13 @@ A failed Session stores nothing; after a reopen nothing is held. Publication: the Session's own observers see each held commit once, when memory adopts it, and of a normal commit that stores held changes only its own -operations. `subscribeCommits()` sees every storage transaction with exactly +operations. `observeCommits()` sees every storage transaction with exactly what it stored, published when Storage adopts it (section 4): nothing for a held commit, and the stored operations, held first, for a commit or a deadline that stores them. A document the commit retires is published as its retirement only: the held operations and the commit's own change stored before the retirement are not. A consumer that applies -`subscribeCommits()` publications to stored state, such as a replica, stays +`observeCommits()` publications to stored state, such as a replica, stays equal to Storage in every live document. Its `seq` values have gaps where nothing was published. Consistency: every stored document is a state it had in memory, and a normal @@ -2060,7 +2104,7 @@ interface TaskRuntime extends DocumentObserver, Docum /** A `task_error`, or with `from` an `extension_error` or `host_error`, with this task (section 10.1). */ report( error: unknown, - from?: { type: "extension"; extension?: string } | { type: "host"; callback: HostCallback }, + from?: { kind: "extension"; extension?: string } | { kind: "host"; callback: HostCallback }, ): void; /** Commit `{ status: "sleeping", until, checkpoint }`; resolves once stored (section 5.6). */ sleepUntil(until: number, checkpoint: S, context: Context): Promise; @@ -4305,14 +4349,16 @@ type RunFailure = { type CompactionFailure = { taskId: TaskId; - reason: CompactionReason; + /** What started the compaction that failed. */ + trigger: CompactionReason; /** The failed or faulted outcome's error message, or why the scheduler orphaned the task. */ error: string; }; type CompactionStatus = { taskId: TaskId; - reason: CompactionReason; + /** What started it: `compact()`, a threshold, or a context overflow. */ + trigger: CompactionReason; /** Whether a generation waits for it: a compaction the generation owns. */ blocking: boolean; attempt: number; @@ -4324,9 +4370,10 @@ type CompactionStatus = { | field | value | |---|---| | kind | `pi.live` | -| version | `1` | +| version | `2` | | scope/history/fork | conversation, `latest`, `initial` | | `initial()` | `{}` | +| `migrate()` | from version 1: `reason` of each `compactions` status and of `compactionFailure` renamed `trigger` (section 3.6) | | checkpoint | complete base whenever nothing runs: `generation` absent and no `running` tool or nested slot | | view mount | `docs["pi.live"]` | | created | with every Harness conversation (section 2.2) | @@ -4893,7 +4940,7 @@ aborted with its generation, before the generation's abort handler runs conversation abort keeps it, and `Submission.abort()` withdraws it. **Failure.** The commit that fails a compaction (`model_error`, `no_model`) -removes its status and sets `pi.live.compactionFailure` to its task, reason, +removes its status and sets `pi.live.compactionFailure` to its task, trigger, and error message, replacing an older failure; the Harness cleanup does the same for a compaction the scheduler faults or orphans, with the fault's message or the orphan reason. Only a compaction that produces a summary @@ -4916,19 +4963,24 @@ re-exports both): ```ts type WatchEnd = - | { readonly reason: "stopped" | "cancelled" | "ended" } - | { readonly reason: "listener_error"; readonly error: Error } - /** The source failed; for Durable, what failed the Session, usually a storage error (section 10.1). */ - | { readonly reason: "failed"; readonly error: unknown }; + | { readonly kind: "stopped" | "aborted" | "ended" } + | { readonly kind: "listener_error"; readonly error: Error } + /** The source failed for good; for Durable, what failed the Session, usually a storage error (section 10.1). */ + | { readonly kind: "failed"; readonly error: unknown }; -/** One listener call: the newest value, the ops from the previous value, and a retirement since the previous call. */ -type WatchChange = { readonly value: T; readonly ops: readonly Op[]; readonly retired: boolean }; +/** + * One listener call: the newest value, the ops from the previous value, and a retirement since the previous value + * call; or a failure the source may recover from, after which the watch stays open. Durable sends no `failed` call. + */ +type WatchCall = + | { readonly kind: "value"; readonly value: T; readonly ops: readonly Op[]; readonly retired: boolean } + | { readonly kind: "failed"; readonly error: unknown }; interface WatchHandle { /** Acquisition revision before start; latest delivered immutable revision afterward. */ readonly value: T; /** Installs the sole serialized asynchronous listener. */ - start(listener: (change: WatchChange, context: Context) => Promise): void; + start(listener: (call: WatchCall, context: Context) => Promise): void; /** Idempotently stops future callbacks and returns this watch's terminal result. */ stop(): Promise; /** Settles when the watch terminates; an already-running callback remains caller-owned. */ @@ -4947,11 +4999,12 @@ interface DocumentObserver { } ``` -A watch ends once, with the first reason that applies: `stopped` by `stop()`; -`cancelled` when its acquisition `Context` is cancelled; `ended` when the owner +A watch ends once, with the first end that applies, tagged by `kind`: `stopped` by `stop()`; +`aborted` when its acquisition `Context` is aborted; `ended` when the owner of what it watches ends (the Session closed, or a task document's task became terminal); `listener_error` when its listener threw or rejected; `failed` when -the Session failed. +the Session failed. Durable's failures are final, so a Durable watch only gets +`value` calls; Chord's `failed` call, for a source that recovers, never comes. Watch acquisition never creates; absent lookup returns `undefined`. A document watch follows its address (kind, scope, and key) from the incarnation found at @@ -4965,7 +5018,7 @@ pending, so one call may report a retirement and carry the next incarnation: ```text commit: retireDoc(StateDoc); doc(StateDoc).value = 10 -call: { value: { value: 10, ... }, ops: [["r", { value: 10, ... }]], retired: true } +call: { kind: "value", value: { value: 10, ... }, ops: [["r", { value: 10, ... }]], retired: true } ``` Task documents retire when their task becomes terminal and are never created @@ -4973,7 +5026,7 @@ again, so a task document's watch gets the retirement of that commit as its last call and then ends `ended`, once that call returned. Until then Session close or failure still ends it at once, dropping the retirement if it was not delivered yet, as for a watch never started. A watch of a session or conversation document that -is retired for good stays until it is stopped, its context is cancelled, or the +is retired for good stays until it is stopped, its context is aborted, or the Session ends; it pins nothing while its address is absent. Session close ends every watch at its seal: the watch keeps its last value and @@ -4981,7 +5034,7 @@ receives no frame from a commit that settles during close. Termination drops every reference the watch holds into its source: its listener, its subscriptions and the closures that detach, report, and replace, and its -cancellation hook. A stopped watch that is still referenced keeps only its last +abort hook. A stopped watch that is still referenced keeps only its last value alive, never the Session or the listener; a callback already running keeps what it needs until it settles. Watches retain immutable revisions independently of the loaded tracker cache. Reopen creates a new watch lifetime. @@ -4997,9 +5050,10 @@ the same step as Durable's memory, such as a Chord source over a Durable view, attach a synchronous observer instead; every watch is built on one: ```ts +/** The shapes of Chord's watch: `advance` gets a `WatchCall` (always a `value` call), `end` a `WatchEnd`. */ type Observer = { - advance(change: WatchChange, context: Context): void; - end(failure: { readonly error: unknown } | undefined): void; + advance(call: WatchCall, context: Context): void; + end(end: WatchEnd): void; }; type Observation = { readonly value: T; detach(): void }; @@ -5025,9 +5079,9 @@ advances without a commit, when a registry change or a failed migration changes a task's `blocked` reason (section 9.4): that call runs on the Session line, outside any `adopt`, with a context of its own. A document observer follows its address as a document watch does, and `observeDoc()` gives `undefined` for an -absent one. `end` comes once: when the Session closes (`undefined`) or fails -(`{ error }`), or, for a task document, right after the retirement change of -the commit that makes its task terminal, inside that commit's `adopt`. +absent one. `end` comes once: when the Session closes (`ended`) or fails +(`failed` with the error), or, for a task document, right after the retirement change of +the commit that makes its task terminal, inside that commit's `adopt` (`ended`). `detach()` is idempotent, and nothing follows it, not even the rest of the commit underway; a detached observer is referenced by nothing of the Session. `context` governs only the attachment: cancelling it while attaching rejects @@ -5039,13 +5093,14 @@ later, so a commit adopted in between reaches `advance` before the promise of `observeXXX()` resolves. Each change carries the whole new value, and its operations apply to the value before it, starting from `observation.value`: an adapter that gets a change before its observation records it, and applies it -once it has the observation, or takes `change.value`. +once it has the observation, or takes `call.value`. `advance` runs inside the storage's commit, so an observer records the change and returns: no awaits, no Session calls, no throws, and quick, as its time is -part of every commit; so does `end` when a task ends. One that throws anyway is -reported as `listener_error`, detached, and ended with the error; the commit and -the other observers are not affected. The change object is plain data, shared by +part of every commit; so does `end` when a task ends. One that throws anyway, or +returns a promise that rejects, is reported as `listener_error`, detached, and ended +`listener_error` with the error; the commit and +the other observers are not affected. The call object is plain data, shared by every observer of a document or the task graph and by the watches built on it, and must not be mutated; each observer of a conversation view gets its own, for its own window (section 9.3), sharing the document values. @@ -5067,8 +5122,8 @@ const watch = await api.watchDoc(JobOutputDoc, producerTaskId, context); if (watch === undefined) return; try { await initializeConsumer(watch.value, context); - watch.start(async ({ value, ops }, deliveryContext) => { - await consume(value, ops, deliveryContext); + watch.start(async (call, deliveryContext) => { + if (call.kind === "value") await consume(call.value, call.ops, deliveryContext); }); } catch (error) { await watch.stop(); @@ -5270,7 +5325,9 @@ class ConversationTranscript { const transcript = new ConversationTranscript((query, limit, ctx) => conversation.entries(query, limit, undefined, ctx)); const watch = await conversation.watch(context); render(transcript.merge(watch.value)); -watch.start(async ({ value }) => render(transcript.merge(value))); +watch.start(async (call) => { + if (call.kind === "value") render(transcript.merge(call.value)); +}); // A marker scrolled into view: render(await transcript.loadEarlier(context)); // Reconnect after a drop: @@ -5391,8 +5448,9 @@ seal like every other watch (section 2.2). ```ts const graph = await harness.watchTaskGraph(context); -graph.start(async ({ value }) => { - for (const node of Object.values(value.tasks)) { +graph.start(async (call) => { + if (call.kind !== "value") return; + for (const node of Object.values(call.value.tasks)) { const where = node.owner === undefined ? `conversation ${node.conversationId}` : `task ${node.owner}`; render(`${node.id} ${node.kind} ${node.state.status} under ${where}`); } @@ -5622,14 +5680,14 @@ may not have committed, so a host that resubmits after reopening reuses its and every wait that starts later rejects with `SessionFailed`, whose `cause` is the first error; so does every Storage call still underway when it returns, whatever it returned. -- Watches end with `{ reason: "failed", error }` and keep their last value. +- Watches end with `{ kind: "failed", error }` and keep their last value. - Running task invocations get their abort signal. `HarnessOptions.onReport` gets the error once, after the Session is sealed, so a report handler that calls back finds it failed. -- The Session closes itself. `closed` settles `{ reason: "failed", error }` once +- The Session closes itself. `closed` settles `{ kind: "failed", error }` once that close has run, also when the backend could not close cleanly; after a plain `close()` it settles - `{ reason: "closed" }`. Closing the backend waits for every Storage call + `{ kind: "closed" }`. Closing the backend waits for every Storage call underway, refusing new ones, and for running task code to return: code that ignores its abort signal keeps the backend open and `closed` pending. - A failing `Storage.close()` fails the Session too, unless it already failed; @@ -5648,7 +5706,7 @@ checks its health after the callback. Adopting a committed batch in memory that fails also fails the Session, since memory would be behind storage; so does a `Storage.commit()` that resolves without adopting, or rejects after it adopted. A batch Storage adopted is durable, so it is not rolled back: it was published -at adoption (section 4), `subscribeCommits()` listeners get it and the waits it +at adoption (section 4), `observeCommits()` observers get it and the waits it settles resolve; then the commit rejects with the storage error and the Session fails as above, with the storage error. An own observer that threw on that commit is reported as a `listener_error`. @@ -5663,7 +5721,7 @@ listener of the Session's own components (scheduler, submissions, views, task graph, document observers) fails it as well, since their memory would fall behind storage; the commit itself stands. Those listeners run inside Storage's `adopt` (section 4), so the Session fails once `Storage.commit()` returned, -after the other observers and the `subscribeCommits()` listeners got the commit. +after the other observers and the `observeCommits()` observers got the commit. **User code fails only its unit.** A task phase that throws ends that task `faulted`; a throwing `migrate` blocks it as `migration_failed`; a tool that @@ -5672,10 +5730,10 @@ throw are reported and skipped, except `beforeTool`, whose throw blocks the call a commit callback, `conversationCreated`, `init`, or a document's `initial`, `migrate`, or `checkpointWhen` that throws rolls its commit back and rejects it. Callbacks that must not fail their caller are contained: a host -`subscribeCommits()` or `subscribeClose()` listener, a watch listener, and a -view observer that throws, or returns a +`observeCommits()` observer or `subscribeClose()` listener, a watch listener, and a +view or document observer that throws, or returns a promise that rejects, are reported, and the others still run; a watch whose -listener throws ends with `listener_error`. A throwing or rejecting `onReport` is +listener throws, and an observer that throws, end `listener_error`. A throwing or rejecting `onReport` is dropped, since nothing is left to report it to, and so is a report's own metric or event that throws. A throwing `now()` is reported once, and `Date.now` serves instead. A telemetry adapter that throws on a commit's events @@ -5683,10 +5741,10 @@ or on the active-task metric is reported as `host_error` with callback `telemetry` (a plain Session drops it); the commit resolves, and memory and the views keep it. -**Every report says what happened.** `onReport(error, info)` gets a -`ReportInfo` whose `type` is one of: +**Every report says what happened.** `onReport(report)` gets one `Report`: the +`error`, and the fields of a `ReportInfo` whose `kind` is one of: -- `session_failed`: the error that failed the Session (a Storage error, often a +- `storage_failed`: the error that failed the Session (a Storage error, often a platform reset, or the Session's own bookkeeping); - `task_faulted`: a task ended `faulted`, because its phase threw (reason `threw`), or it or its abort handler was cut off `restartLimit` times (reason @@ -5697,7 +5755,8 @@ views keep it. the task's `restarts` (section 5.4); - `migration_failed`: a task's migration threw, and the task is blocked; - `task_outdated`: a running task keeps its old definition, which is missing - (`missing_task`) or cannot take the task (`incompatible_task`); + (`missing_task`), older than the task (`task_too_old`), or newer without a + `migrate` (`migration_failed`); - `task_error`: a task's own `runtime.report()`, or a failure running it; - `extension_error`: hooks, wrappers, system prompt sections, and a tool's invalid structured output, with the task that ran them, or for a wrapper diff --git a/packages/durable/src/env/index.ts b/packages/durable/src/env/index.ts index 170c43579..f2d97d12b 100644 --- a/packages/durable/src/env/index.ts +++ b/packages/durable/src/env/index.ts @@ -117,14 +117,17 @@ export interface WatchTarget { } /** - * What changed: + * What changed, tagged by `kind`: * - `paths`: something at or below each path may have changed (a directory path covers its whole subtree). Calls may be * spurious; a change is never missed while the watcher is healthy. * - `overflow`: coverage was uncertain for a while (lost events, reconnect); rescan everything that is watched. - * - `error`: the watcher stopped, for example because the watched tree grew past the environment's limit; no calls - * follow. + * - `failed`: the watcher stopped with `error`, for example because the watched tree grew past the environment's limit; + * no calls follow. */ -export type FileChange = { paths: string[] } | { overflow: true } | { error: FileError }; +export type FileChange = + | { readonly kind: "paths"; readonly paths: string[] } + | { readonly kind: "overflow" } + | { readonly kind: "failed"; readonly error: FileError }; export interface FileWatcher { /** @@ -134,7 +137,7 @@ export interface FileWatcher { */ readonly mode: "native" | "polling"; /** Stop watching; no `onChange` call starts after this resolves. Idempotent. */ - close(context: Context): Promise; + stop(context: Context): Promise; } /** Where lines of a file are, as `BinaryReader.scanLines` found them. */ diff --git a/packages/durable/src/env/node-watch.ts b/packages/durable/src/env/node-watch.ts index 84d635e6f..e56645bb4 100644 --- a/packages/durable/src/env/node-watch.ts +++ b/packages/durable/src/env/node-watch.ts @@ -52,6 +52,8 @@ const DEBOUNCE_MS = 50; const FSEVENTS_SETTLE_MS = 500; const DEFAULT_POLL_MS = 2000; const DEFAULT_MAX_DIRECTORIES = 10_000; +/** The one `overflow` change; it carries nothing else. */ +const OVERFLOW: FileChange = { kind: "overflow" }; /** In polling mode, recently modified small files are also compared by content: a second write within the file system's * timestamp granularity can keep size and modification time. */ const HASH_MAX_BYTES = 256 * 1024; @@ -149,7 +151,7 @@ export class NodeFileWatcher implements FileWatcher { #settleTimer: ReturnType | undefined; #running: Promise | undefined; #dirty = false; - #closed = false; + #stopped = false; private constructor( targets: readonly ResolvedTarget[], @@ -200,14 +202,14 @@ export class NodeFileWatcher implements FileWatcher { return this.#mode; } - async close(_context: Context): Promise { - if (this.#closed) return; + async stop(_context: Context): Promise { + if (this.#stopped) return; this.#stop(); await this.#running; } #stop(): void { - this.#closed = true; + this.#stopped = true; clearTimeout(this.#timer); this.#timer = undefined; clearTimeout(this.#settleTimer); @@ -217,7 +219,7 @@ export class NodeFileWatcher implements FileWatcher { } #deliver(change: FileChange): void { - if (this.#closed) return; + if (this.#stopped) return; try { this.#onChange(change); } catch { @@ -226,7 +228,7 @@ export class NodeFileWatcher implements FileWatcher { } #schedulePoll(): void { - if (this.#closed || this.#mode !== "polling") return; + if (this.#stopped || this.#mode !== "polling") return; this.#timer = setTimeout(() => { this.#timer = undefined; void this.#flush().finally(() => this.#schedulePoll()); @@ -234,7 +236,7 @@ export class NodeFileWatcher implements FileWatcher { } #scheduleFlush(): void { - if (this.#closed || this.#timer !== undefined) return; + if (this.#stopped || this.#timer !== undefined) return; this.#timer = setTimeout(() => { this.#timer = undefined; void this.#flush(); @@ -242,7 +244,7 @@ export class NodeFileWatcher implements FileWatcher { } #scheduleSettle(): void { - if (this.#closed) return; + if (this.#stopped) return; clearTimeout(this.#settleTimer); this.#settleTimer = setTimeout(() => { this.#settleTimer = undefined; @@ -263,15 +265,15 @@ export class NodeFileWatcher implements FileWatcher { this.#events.clear(); const changed = await this.#sync(true); for (const path of events) changed.add(path); - if (changed.size > 0) this.#deliver({ paths: [...changed].sort() }); - } while (this.#dirty && !this.#closed); + if (changed.size > 0) this.#deliver({ kind: "paths", paths: [...changed].sort() }); + } while (this.#dirty && !this.#stopped); } catch (error) { const message = error instanceof Error ? error.message : String(error); const fileError = error instanceof FileError ? error : new FileError(isDenied(error) ? "permission_denied" : "invalid", message); - this.#deliver({ error: fileError }); + this.#deliver({ kind: "failed", error: fileError }); this.#stop(); } finally { this.#running = undefined; @@ -283,10 +285,10 @@ export class NodeFileWatcher implements FileWatcher { /** Rescan, report differences, and install watchers for new directories, rescanning until none are new. */ async #sync(report: boolean): Promise> { const changed = new Set(); - for (let round = 0; round < 10 && !this.#closed; round++) { + for (let round = 0; round < 10 && !this.#stopped; round++) { const scan = await this.#scan(); // Closed during the scan: installing watchers now would leak them. - if (this.#closed) break; + if (this.#stopped) break; const next = scan.snapshot; if (report) for (const path of this.#diff(this.#snapshot, next)) changed.add(path); this.#snapshot = next; @@ -471,14 +473,14 @@ export class NodeFileWatcher implements FileWatcher { this.#mode = "polling"; for (const { watcher } of this.#watchers.values()) watcher.close(); this.#watchers.clear(); - this.#deliver({ overflow: true }); + this.#deliver(OVERFLOW); clearTimeout(this.#timer); this.#timer = undefined; this.#schedulePoll(); } #onEvent(directory: string, filename: string | undefined): void { - if (this.#closed) return; + if (this.#stopped) return; const path = filename === undefined ? directory : `${directory.endsWith(sep) ? directory : directory + sep}${filename}`; // Events about unrelated siblings of an ancestor, or about excluded entries, are ignored. @@ -489,7 +491,7 @@ export class NodeFileWatcher implements FileWatcher { /** The file a target links to changed: report the target. */ #onLinkedFileEvent(target: string): void { - if (this.#closed) return; + if (this.#stopped) return; this.#events.add(target); this.#scheduleFlush(); } diff --git a/packages/durable/src/env/node.ts b/packages/durable/src/env/node.ts index d45ad2cae..178255731 100644 --- a/packages/durable/src/env/node.ts +++ b/packages/durable/src/env/node.ts @@ -641,7 +641,7 @@ export class NodeExecutionEnv implements ExecutionEnv { ); const afterOpenAbort = abortResult(context.abortSignal); if (afterOpenAbort) { - await watcher.close(context); + await watcher.stop(context); return afterOpenAbort; } return ok(watcher); diff --git a/packages/durable/src/harness/compaction.ts b/packages/durable/src/harness/compaction.ts index b5f480fae..e11830f6e 100644 --- a/packages/durable/src/harness/compaction.ts +++ b/packages/durable/src/harness/compaction.ts @@ -258,7 +258,7 @@ export async function createCompaction( owner === undefined ? ({ kind: "conversation" } as const) : { kind: "task" as const, taskId: owner }; const background = owner === undefined && input.reason !== "manual"; const taskId = await tx.createTask(CompactionTask, input, { ownership, conversationId, background }); - const status = { taskId, reason: input.reason, blocking: owner !== undefined, attempt: 1 }; + const status = { taskId, trigger: input.reason, blocking: owner !== undefined, attempt: 1 }; addCompactionStatus(await tx.doc(LiveDoc, conversationId), status); return taskId; } diff --git a/packages/durable/src/harness/generation.ts b/packages/durable/src/harness/generation.ts index dc2a9877a..462f73e23 100644 --- a/packages/durable/src/harness/generation.ts +++ b/packages/durable/src/harness/generation.ts @@ -182,7 +182,7 @@ export const GenerationTask = defineTask runtime.report(error, { - type: "extension", + kind: "extension", ...optionalExtension(sectionExtension(agent, section.key)), }), context, @@ -513,7 +513,7 @@ async function cancelDeferred( telemetry: context.telemetry, }); } catch (error) { - runtime.report(error, { type: "host", callback: "models" }); + runtime.report(error, { kind: "host", callback: "models" }); } } diff --git a/packages/durable/src/harness/harness.ts b/packages/durable/src/harness/harness.ts index 5908e0262..a82e40fe7 100644 --- a/packages/durable/src/harness/harness.ts +++ b/packages/durable/src/harness/harness.ts @@ -31,6 +31,7 @@ import type { Observation, Observer, Page, + Report, ReportInfo, Storage, SubmissionId, @@ -242,7 +243,7 @@ class HarnessImpl extends SessionImpl implements env: (id, callContext) => this.buildEnv(id, callContext), now, onWake: options.onWake, - timers: hostTimers(options.setTimer, (error) => report(error, { type: "host_error", callback: "setTimer" })), + timers: hostTimers(options.setTimer, (error) => report(error, { kind: "host_error", callback: "setTimer" })), writeAtClose: (writes) => this.writeAtClose(writes), report: this.#report, limitsRestarts: (kind) => LIMITS_RESTARTS.has(kind), @@ -280,7 +281,7 @@ class HarnessImpl extends SessionImpl implements const registry = snapshot ?? this.#options.registry.snapshot(); const state = await this.snapshot(AgentDoc, id, context); return resolveAgent(state, registry, resolveSettings(this.#hostSettings()), (error, extension) => - this.#report(error, { type: "extension_error", conversationId: id, extension }, context), + this.#report(error, { kind: "extension_error", conversationId: id, extension }, context), ); } @@ -403,7 +404,7 @@ class HarnessImpl extends SessionImpl implements return this.#create({ kind: "independent", ownership: options.ownership }, options, context); } - override report(error: unknown, info: ReportInfo = { type: "listener_error" }, context?: Context): void { + override report(error: unknown, info: ReportInfo = { kind: "listener_error" }, context?: Context): void { this.#report(error, info, context); } @@ -451,21 +452,23 @@ class HarnessImpl extends SessionImpl implements * `adopt`: a throwing `onReport` is dropped, and so is a throwing metric or event, as reporting it would report again. */ function safeReport( - onReport: ((error: unknown, info: ReportInfo) => void) | undefined, + onReport: ((report: Report) => void) | undefined, telemetry: TelemetryContext, ): (error: unknown, info: ReportInfo, context?: Context) => void { const reports = createTypedMetrics(telemetry.metrics ?? NOOP_TELEMETRY_METRICS, [DURABLE_TELEMETRY_SCHEMA]).counter( "pi.durable.reports", ); return (error, info, context) => { - contained( - () => onReport?.(error, info), - () => {}, - ); + if (onReport !== undefined) { + contained( + () => onReport({ ...info, error }), + () => {}, + ); + } const attributes: SpanAttributes = { - type: info.type, + type: info.kind, }; - switch (info.type) { + switch (info.kind) { case "task_faulted": case "task_interrupted": case "task_outdated": @@ -484,7 +487,7 @@ function safeReport( break; } try { - reports.add(1, { type: info.type }); + reports.add(1, { type: info.kind }); context?.telemetry.addEvent("pi.durable.report", attributes); } catch { // Dropped: see above. @@ -509,7 +512,7 @@ export function safeNow( // Set first: a report handler that reads the clock again must not report again. if (!reported) { reported = true; - report(error, { type: "host_error", callback: "now" }); + report(error, { kind: "host_error", callback: "now" }); } return Date.now(); } diff --git a/packages/durable/src/harness/live.ts b/packages/durable/src/harness/live.ts index d9b09cde6..837225a04 100644 --- a/packages/durable/src/harness/live.ts +++ b/packages/durable/src/harness/live.ts @@ -71,7 +71,8 @@ export type NestedToolSummary = { /** Presentation of one live compaction task (spec §8.7). */ export type CompactionStatus = { taskId: TaskId; - reason: CompactionReason; + /** What started it: `compact()`, a threshold, or a context overflow. */ + trigger: CompactionReason; /** Whether a generation waits for it: a compaction the generation owns. */ blocking: boolean; attempt: number; @@ -82,7 +83,8 @@ export type CompactionStatus = { /** The newest compaction that failed since a compaction last produced a summary (spec §8.7). */ export type CompactionFailure = { taskId: TaskId; - reason: CompactionReason; + /** What started the compaction that failed. */ + trigger: CompactionReason; /** The failed or faulted outcome's error message, or why the scheduler orphaned the task. */ error: string; }; @@ -164,11 +166,21 @@ export type LiveState = { export const LiveDoc = defineDoc({ kind: "pi.live", - version: 1, + version: 2, scope: "conversation", history: "latest", fork: "initial", initial: () => ({}), + // Version 1 named a compaction's trigger `reason`, in `compactions` and `compactionFailure`. + migrate: (value) => { + const live = value as { compactions?: JsonObject[]; compactionFailure?: JsonObject }; + const { compactions, compactionFailure } = live; + return { + ...value, + ...(compactions === undefined ? {} : { compactions: compactions.map(renameReason) }), + ...(compactionFailure === undefined ? {} : { compactionFailure: renameReason(compactionFailure) }), + } as LiveState; + }, // REMINDER: a complete base whenever nothing runs (spec §8.2): no generation and no running tool slot. That holds // while idle, in the commit handing a generation over to its tool round, and between tools, so the delta chain // spans at most one generation or the overlapping execution of one round's tools. A slot holds output only while @@ -180,6 +192,11 @@ export const LiveDoc = defineDoc({ !(value.nestedTools ?? []).some((slot) => slot.status === "running"), }); +/** A copy of a version 1 compaction status or failure with `reason` renamed `trigger`, its keys in their order. */ +function renameReason(value: JsonObject): JsonObject { + return Object.fromEntries(Object.entries(value).map(([key, field]) => [key === "reason" ? "trigger" : key, field])); +} + /** * How the progress throttles commit a partial answer and running tool output into `pi.live`: deferred, so storage gets * them together, at the next commit that reads or writes `pi.live` or after `settings.progress.persistWithinMs`. Every @@ -284,7 +301,7 @@ export function removeCompactionStatus(live: Draft, taskId: TaskId): /** Remove the status of compaction task `taskId` and record that it failed with `error`. */ export function failCompactionStatus(live: Draft, taskId: TaskId, error: string): void { const status = compactionStatus(live, taskId); - if (status !== undefined) live.compactionFailure = { taskId: status.taskId, reason: status.reason, error }; + if (status !== undefined) live.compactionFailure = { taskId: status.taskId, trigger: status.trigger, error }; removeCompactionStatus(live, taskId); } diff --git a/packages/durable/src/harness/scheduler.ts b/packages/durable/src/harness/scheduler.ts index 06fda6a3c..1fc981be6 100644 --- a/packages/durable/src/harness/scheduler.ts +++ b/packages/durable/src/harness/scheduler.ts @@ -380,7 +380,7 @@ export class TaskScheduler { * `running` task stays as committed, running in another process. */ async open(context: Context): Promise { - this.#session.observeCommits((publication) => this.#observe(publication)); + this.#session.followCommits((publication) => this.#observe(publication)); this.#session.subscribeClose(() => this.#seal()); this.#unsubscribeRegistry = this.#registry.subscribe(() => { this.#notifyBlocked(); @@ -1407,7 +1407,7 @@ export class TaskScheduler { try { listener(); } catch (error) { - this.#report(error, { type: "listener_error" }); + this.#report(error, { kind: "listener_error" }); } } } @@ -1545,7 +1545,7 @@ export class TaskScheduler { } for (const { record, message } of faulted) { this.#report(new Error(message), { - type: "task_faulted", + kind: "task_faulted", ...this.#aboutTask(recordTask(record)), reason: "restart_limit", }); @@ -1586,7 +1586,7 @@ export class TaskScheduler { queueMicrotask(() => this.#notifyBlocked()); this.#scheduleWake(); const extension = installerOf(snapshot, fit.task); - this.#report(error, { type: "migration_failed", ...this.#aboutTask({ ...recordTask(record), extension }) }); + this.#report(error, { kind: "migration_failed", ...this.#aboutTask({ ...recordTask(record), extension }) }); return { kind: "blocked", reason: "migration_failed" }; } } @@ -1644,7 +1644,7 @@ export class TaskScheduler { if (fit.migrates && fit.task.definition.migrate === undefined) { return { kind: "blocked", reason: "migration_failed", error: missingMigration(record, erased(fit.task)) }; } - return { kind: "ready", migrates: fit.migrates }; + return { kind: "pending", migrates: fit.migrates }; } #createInvocation(record: RunnableTaskRecord, mode: "run" | "abort", extension: string | undefined): Invocation { @@ -1677,7 +1677,7 @@ export class TaskScheduler { if (invocation.mode === "run") await this.#run(reservation); else await this.#runAbort(reservation); } catch (error) { - this.#report(error, { type: "task_error", ...this.#aboutTask(invocation) }, invocation.context); + this.#report(error, { kind: "task_error", ...this.#aboutTask(invocation) }, invocation.context); } finally { this.#end(invocation); invocation.record = undefined; @@ -1825,9 +1825,9 @@ export class TaskScheduler { this.#report( new Error(`Task ${current.id} keeps running under its old ${current.kind} definition`), { - type: "task_outdated", + kind: "task_outdated", ...this.#aboutTask({ ...recordTask(current), extension }), - reason: next === undefined ? "missing_task" : "incompatible_task", + reason: outdatedReason(next, current), }, context, ); @@ -1889,7 +1889,7 @@ export class TaskScheduler { if (fault !== undefined) this.#report( fault.error, - { type: "task_faulted", ...this.#aboutTask(invocation), reason: "threw" }, + { kind: "task_faulted", ...this.#aboutTask(invocation), reason: "threw" }, invocation.context, ); return continued; @@ -2086,7 +2086,7 @@ export class TaskScheduler { if (invocation.controller.signal.aborted) throw error; this.#report( error, - { type: "extension_error", extension, task: reportTask(invocation) }, + { kind: "extension_error", extension, task: reportTask(invocation) }, invocation.context, ); } @@ -2137,7 +2137,7 @@ export class TaskScheduler { this.#report( new Error(interrupted.error.message), { - type: "task_interrupted", + kind: "task_interrupted", ...this.#aboutTask(invocation), reason: interruptionOf(interrupted.error)!, restarts: interrupted.restarts, @@ -2309,10 +2309,10 @@ export class TaskScheduler { this.#report( error, from === undefined - ? { type: "task_error", ...this.#aboutTask(invocation) } - : from.type === "extension" - ? { type: "extension_error", task, ...optionalExtension(from.extension) } - : { type: "host_error", callback: from.callback, task }, + ? { kind: "task_error", ...this.#aboutTask(invocation) } + : from.kind === "extension" + ? { kind: "extension_error", task, ...optionalExtension(from.extension) } + : { kind: "host_error", callback: from.callback, task }, invocation.context, ); }, @@ -2366,10 +2366,10 @@ export class TaskScheduler { } } -const REGISTRY: ReportInfo = { type: "host_error", callback: "registry" }; -const SETTINGS: ReportInfo = { type: "host_error", callback: "settings" }; -const ON_WAKE: ReportInfo = { type: "host_error", callback: "onWake" }; -const TELEMETRY: ReportInfo = { type: "host_error", callback: "telemetry" }; +const REGISTRY: ReportInfo = { kind: "host_error", callback: "registry" }; +const SETTINGS: ReportInfo = { kind: "host_error", callback: "settings" }; +const ON_WAKE: ReportInfo = { kind: "host_error", callback: "onWake" }; +const TELEMETRY: ReportInfo = { kind: "host_error", callback: "telemetry" }; /** The extension that installed `task` in `snapshot`, the last one that lists it; none for a built-in task. */ function installerOf(snapshot: RegistrySnapshot, task: AnyTask | undefined): string | undefined { @@ -2490,6 +2490,15 @@ function withState(record: AnyTaskRecord, state: TaskState } /** Whether a definition can take the task at reservation: same version, or newer with a migration. */ +/** + * Why a replacement definition cannot take a running task, in `#fit()`'s words: none is registered, it is older than + * the task, or it is newer without a `migrate`. + */ +function outdatedReason(task: AnyTask | undefined, record: AnyTaskRecord): BlockedReason { + if (task === undefined) return "missing_task"; + return task.definition.version < record.version ? "task_too_old" : "migration_failed"; +} + function canReserve(task: AnyTask, record: AnyTaskRecord): boolean { const definition = task.definition; return ( diff --git a/packages/durable/src/harness/submissions.ts b/packages/durable/src/harness/submissions.ts index 8f44e0793..419c6786d 100644 --- a/packages/durable/src/harness/submissions.ts +++ b/packages/durable/src/harness/submissions.ts @@ -57,7 +57,7 @@ export class Submissions { this.#now = now; this.#queueModes = queueModes; this.#resume = resume; - session.observeCommits((publication) => this.#observe(publication)); + session.followCommits((publication) => this.#observe(publication)); session.subscribeClose(() => { this.#closed = true; this.#waiters.rejectAll(closedError(this.#session)); diff --git a/packages/durable/src/harness/task-graph.ts b/packages/durable/src/harness/task-graph.ts index 9fdde3967..2e0b78875 100644 --- a/packages/durable/src/harness/task-graph.ts +++ b/packages/durable/src/harness/task-graph.ts @@ -1,7 +1,21 @@ -import { type Context, type JsonValue, Watch, type WatchChange, type WatchHandle } from "@earendil-works/chord"; +import { + type Context, + type JsonValue, + Watch, + type WatchCall, + type WatchEnd, + type WatchHandle, +} from "@earendil-works/chord"; import { BACKGROUND_CONTEXT, withoutAbortSignal } from "@earendil-works/chord/context"; import { applyImmutable, type Op } from "@earendil-works/chord/delta"; -import { contained, guardObserver, type ObservedCommit, observation, type SessionImpl } from "../session/session.ts"; +import { + contained, + ENDED, + guardObserver, + type ObservedCommit, + observation, + type SessionImpl, +} from "../session/session.ts"; import type { ConversationId, JoinPolicy, @@ -88,7 +102,7 @@ export class TaskGraphView { this.#session = session; this.#storage = storage; this.#blocking = blocking; - session.observeCommits((publication, context) => { + session.followCommits((publication, context) => { // A commit held in memory changes documents only, which the graph does not show. if (this.#mount === undefined || publication.kind === "memory") return; advance(this.#mount, publication, context, blocking.blocked, (error) => session.report(error)); @@ -96,9 +110,11 @@ export class TaskGraphView { blocking.subscribeBlocked(() => this.#refreshBlocked()); session.subscribeClose(() => { this.#closed = true; + const failure = this.#session.failure; + const end: WatchEnd = failure === undefined ? ENDED : { kind: "failed", error: failure.error }; for (const observer of [...(this.#mount?.observers ?? [])]) { contained( - () => observer.end(this.#session.failure), + () => observer.end(end), (error) => session.report(error), ); } @@ -124,7 +140,7 @@ export class TaskGraphView { return observation(value, attached.observer); } - /** A serialized watch of the graph, an observer of its mount; cancelling `context` stops it. */ + /** A serialized watch of the graph, an observer of its mount; aborting `context` ends it `aborted`. */ async watch(context: Context): Promise { const { observer } = await this.#attach( (value, release) => new Watch(value, release, (error) => this.#session.report(error)), @@ -132,10 +148,10 @@ export class TaskGraphView { ); const signal = context.abortSignal; if (signal?.aborted) { - observer.cancel(); + observer.abort(); throw signal.reason; } - if (signal !== undefined) observer.observeCancellation(signal); + if (signal !== undefined) observer.observeAbort(signal); return observer; } @@ -259,8 +275,8 @@ function publish(mount: Mount, ops: Op[], context: Context, report: (error: unkn if (ops.length === 0) return; mount.value = applyImmutable(mount.value, ops); const frameContext = withoutAbortSignal(context); - const change: WatchChange = { value: mount.value, ops, retired: false }; - for (const observer of [...mount.observers]) contained(() => observer.advance(change, frameContext), report); + const call: WatchCall = { kind: "value", value: mount.value, ops, retired: false }; + for (const observer of [...mount.observers]) contained(() => observer.advance(call, frameContext), report); } function nodeOf( diff --git a/packages/durable/src/harness/telemetry.ts b/packages/durable/src/harness/telemetry.ts index 108a2459c..1cf66ef73 100644 --- a/packages/durable/src/harness/telemetry.ts +++ b/packages/durable/src/harness/telemetry.ts @@ -11,9 +11,9 @@ const reportAttributes = { type: { type: "string", required: true, - description: "ReportInfo type.", + description: "Report kind (`Report.kind`).", values: [ - "session_failed", + "storage_failed", "task_faulted", "task_interrupted", "migration_failed", @@ -28,7 +28,7 @@ const reportAttributes = { type: "string", required: false, description: "Fixed report reason, when present.", - values: ["threw", "restart_limit", "interrupted", "missing_task", "incompatible_task"], + values: ["threw", "restart_limit", "interrupted", "missing_task", "task_too_old", "migration_failed"], }, extension: { type: "string", required: false, description: "Registered extension name, when present." }, "task.id": { type: "number", required: false, description: "Task ID, when present.", cardinality: "high" }, @@ -117,7 +117,7 @@ export const DURABLE_TELEMETRY_SCHEMA = defineTelemetrySchema({ metrics: { "pi.durable.reports": { kind: "counter", - description: "Reports raised, grouped by their fixed ReportInfo type.", + description: "Reports raised, grouped by their fixed kind.", attributes: { type: reportAttributes.type }, }, "pi.durable.phase.duration": { diff --git a/packages/durable/src/harness/tool.ts b/packages/durable/src/harness/tool.ts index d23faf76e..946294524 100644 --- a/packages/durable/src/harness/tool.ts +++ b/packages/durable/src/harness/tool.ts @@ -607,7 +607,7 @@ async function finalResult( (agent) => toolExtension(agent, tool.name), () => undefined, ); - runtime.report(new Error(broken), { type: "extension", ...optionalExtension(extension) }); + runtime.report(new Error(broken), { kind: "extension", ...optionalExtension(extension) }); } } // Programs get the tool's own value: without a schema, its output. The model reads only `output`, so model-issued diff --git a/packages/durable/src/harness/types.ts b/packages/durable/src/harness/types.ts index 91f8192d0..d787dd3cd 100644 --- a/packages/durable/src/harness/types.ts +++ b/packages/durable/src/harness/types.ts @@ -33,7 +33,7 @@ import type { Observation, Observer, Page, - ReportInfo, + Report, Session, SetTimer, SubmissionId, @@ -674,11 +674,11 @@ export type HarnessOptions = { readonly onWake?: (wake: Wake) => void; /** * Receives failures that do not fail the calling operation, such as a throwing hook or listener, a task that faulted, - * and the error that fails the Session, once. `info` says what happened: a host can tell a failed Session - * (`session_failed`, often a platform reset) from a faulted task (`task_faulted`). - * Its own throw or rejection is dropped. + * and the error that fails the Session, once: one `Report`, the `error` and what happened (`kind` and the fields of + * its `ReportInfo`). A host can tell a failed Session (`storage_failed`, often a platform reset) from a faulted task + * (`task_faulted`). Its own throw or rejection is dropped. */ - readonly onReport?: (error: unknown, info: ReportInfo) => void; + readonly onReport?: (report: Report) => void; /** * Open for reading only, as a viewer of a session that another process drives, for example a synced copy on a device * that only shows it. Open recovers nothing: a `running` task stays as committed. Every commit with writes rejects @@ -704,7 +704,7 @@ export type TaskInspection = { readonly state: /** An invocation is active. */ | { readonly kind: "running" } /** The next scheduling pass reserves it; `migrates` when its definition is newer and has `migrate`. */ - | { readonly kind: "ready"; readonly migrates: boolean } + | { readonly kind: "pending"; readonly migrates: boolean } /** * Waits for these live tasks: the live part of its `on`, or, when abort-marked, its live ordinary owned work, which * must end before its abort handler starts. diff --git a/packages/durable/src/harness/view.ts b/packages/durable/src/harness/view.ts index ce4c16b99..d2999c47f 100644 --- a/packages/durable/src/harness/view.ts +++ b/packages/durable/src/harness/view.ts @@ -1,8 +1,22 @@ -import { type Context, type JsonValue, Watch, type WatchChange, type WatchHandle } from "@earendil-works/chord"; +import { + type Context, + type JsonValue, + Watch, + type WatchCall, + type WatchEnd, + type WatchHandle, +} from "@earendil-works/chord"; import { withoutAbortSignal } from "@earendil-works/chord/context"; import { applyImmutable, type NonEmptyPath, type Op, type Path } from "@earendil-works/chord/delta"; import { idFromNumber } from "../ids.ts"; -import { contained, guardObserver, type ObservedCommit, observation, type SessionImpl } from "../session/session.ts"; +import { + contained, + ENDED, + guardObserver, + type ObservedCommit, + observation, + type SessionImpl, +} from "../session/session.ts"; import type { ConversationDocToken, ConversationId, @@ -61,9 +75,9 @@ const DEFAULT_WINDOW_MIN = 50; /** Receives each next revision of one observer's window, and the Session's close. */ export type ViewObserver = { - advance(change: WatchChange, context: Context): void; - /** The Session closed, or failed with `failure`. */ - end(failure: { readonly error: unknown } | undefined): void; + advance(call: WatchCall, context: Context): void; + /** The Session closed (`ended`), or failed (`failed`). */ + end(end: WatchEnd): void; }; const MOUNTED = [ @@ -138,16 +152,18 @@ export class ConversationViews { this.#session = session; this.#storage = storage; VIEWS.set(session, this); - session.observeCommits((publication, context) => { + session.followCommits((publication, context) => { for (const [id, mount] of this.#mounts) advance(id, mount, publication, context, (error) => this.report(error)); }); session.subscribeClose(() => { this.#closed = true; + const failure = this.#session.failure; + const end: WatchEnd = failure === undefined ? ENDED : { kind: "failed", error: failure.error }; for (const mount of this.#mounts.values()) for (const windowed of [...mount.observers]) { contained( - () => windowed.observer.end(this.#session.failure), + () => windowed.observer.end(end), (error) => this.report(error), ); } @@ -188,7 +204,7 @@ export class ConversationViews { return observation(value, attached.observer); } - /** A serialized watch of the view, an observer of its mount; cancelling `context` stops it. */ + /** A serialized watch of the view, an observer of its mount; aborting `context` ends it `aborted`. */ async watch( id: ConversationId, context: Context, @@ -202,10 +218,10 @@ export class ConversationViews { ); const signal = context.abortSignal; if (signal?.aborted) { - observer.cancel(); + observer.abort(); throw signal.reason; } - if (signal !== undefined) observer.observeCancellation(signal); + if (signal !== undefined) observer.observeAbort(signal); return observer; } @@ -326,14 +342,14 @@ function advance( if (!MOUNTED_KINDS.has(kind) || change.record.key !== undefined) continue; const path: NonEmptyPath = ["docs", kind]; const mounted = mount.incarnations.get(kind); - if (change.value === null) { + if (change.kind === "retire") { if (mounted?.id !== change.record.id) continue; mount.incarnations.delete(kind); docOps.push(["d", path]); } else if (mounted?.id === change.record.id && mounted.version === change.version) { for (const op of change.ops) docOps.push(prefixed(op, path)); } else { - mount.incarnations.set(kind, { id: change.record.id, version: change.version! }); + mount.incarnations.set(kind, { id: change.record.id, version: change.version }); docOps.push(["s", path, change.value]); } } @@ -357,8 +373,8 @@ function advance( } } windowed.value = view(mount.conversation, entries, before, mount.docs); - const change: WatchChange = { value: windowed.value, ops, retired: false }; - contained(() => windowed.observer.advance(change, frameContext), report); + const call: WatchCall = { kind: "value", value: windowed.value, ops, retired: false }; + contained(() => windowed.observer.advance(call, frameContext), report); } } diff --git a/packages/durable/src/index.ts b/packages/durable/src/index.ts index dbe9e1596..be0ea2317 100644 --- a/packages/durable/src/index.ts +++ b/packages/durable/src/index.ts @@ -1,4 +1,4 @@ -export type { NativeWatchChange, WatchChange, WatchEnd, WatchHandle } from "@earendil-works/chord"; +export type { NativeWatchCall, WatchCall, WatchEnd, WatchHandle } from "@earendil-works/chord"; export { defineDoc, defineDocFamily } from "./documents.ts"; export { AssistantEntry, @@ -131,6 +131,8 @@ export { defineTask } from "./tasks.ts"; export type { CheckpointInfo, CommitChange, + CommitObservation, + CommitObserver, CommitOptions, CommitPersistence, CommitPublication, @@ -176,6 +178,7 @@ export type { Observer, Page, PhaseHandler, + Report, ReportFrom, ReportInfo, ReportTask, diff --git a/packages/durable/src/session/session.ts b/packages/durable/src/session/session.ts index 1c0f3fd0c..36fd435d0 100644 --- a/packages/durable/src/session/session.ts +++ b/packages/durable/src/session/session.ts @@ -1,4 +1,4 @@ -import { type Context, type JsonValue, Watch } from "@earendil-works/chord"; +import { type Context, type JsonValue, Watch, type WatchEnd } from "@earendil-works/chord"; import { awaitWithContext, BACKGROUND_CONTEXT, withoutAbortSignal } from "@earendil-works/chord/context"; import { type Op, PendingOps, track } from "@earendil-works/chord/delta"; import type { SpanAttributes } from "@earendil-works/pi-telemetry"; @@ -15,6 +15,8 @@ import { idFromNumber } from "../ids.ts"; import { hostTimers, type Timers } from "../timers.ts"; import type { CommitChange, + CommitObservation, + CommitObserver, CommitOptions, CommitPersistence, CommitPublication, @@ -72,7 +74,7 @@ import { } from "./transaction.ts"; /** - * What the Session's own components observe of one commit (`observeCommits()`): a stored commit with its own changes, or + * What the Session's own components observe of one commit (`followCommits()`): a stored commit with its own changes, or * a deferred one adopted in memory only (`CommitPersistence`), which changes documents only. Memory is never behind: * a later store of held changes is not observed again. */ @@ -101,10 +103,10 @@ const NO_OPERATIONS: readonly Op[] = []; const SETTLED: Promise = Promise.resolve(); /** The default source of a report: a listener, which never fails what ran it. */ -const LISTENER: ReportInfo = { type: "listener_error" }; +const LISTENER: ReportInfo = { kind: "listener_error" }; /** A telemetry adapter that threw on a commit's event, which must not fail the durable commit. */ -const TELEMETRY: ReportInfo = { type: "host_error", callback: "telemetry" }; +const TELEMETRY: ReportInfo = { kind: "host_error", callback: "telemetry" }; /** * Open a Session kernel over one storage backend. `now` is the wall clock for task times; default `Date.now`. @@ -172,7 +174,7 @@ export function persistWithin(ms: number | undefined): number { * for the first commit that reads or writes the document, the document's deadline, a drop of its tracker, or close, * which store them. Every commit callback, preparation, Storage settlement, and adoption runs while the line is held. A * stored commit is published when Storage adopts it: inside `adopt` its telemetry events and the Session's own - * observers, one microtask later `subscribeCommits()` listeners, before every wait the commit resolves and every abort + * observers, one microtask later `observeCommits()` observers, before every wait the commit resolves and every abort * it queues, all still on the line. Host listeners and watch listeners have no order. * * Storage failures are final. Every Storage call goes through one guard: the first error a Storage method throws, other @@ -202,7 +204,8 @@ export class SessionImpl implements Session { #sweepJobs = 0; /** Set once the host declined a background timer: then none is kept, and the end of each commit evicts what expired. */ #documentTimersUnavailable = false; - readonly #commitListeners = new Set<(publication: CommitPublication, context: Context) => void>(); + /** Host commit observers (`observeCommits()`), each behind `guardObserver()`. */ + readonly #commitListeners = new Set>(); /** The Session's own listeners, such as its scheduler's: they keep memory in step with storage, so a throw fails it. */ readonly #internalListeners = new Set<(publication: ObservedCommit, context: Context) => void>(); readonly #closeListeners = new Set<() => void>(); @@ -240,7 +243,7 @@ export class SessionImpl implements Session { this.#failed.promise.catch(() => {}); this.#readOnly = readOnly; this.#storage = new GuardedStorage(storage, this); - this.#timers = hostTimers(setTimer, (error) => this.report(error, { type: "host_error", callback: "setTimer" })); + this.#timers = hostTimers(setTimer, (error) => this.report(error, { kind: "host_error", callback: "setTimer" })); // Cache bookkeeping must never change the outcome of the operation it follows: a clock or a retention setting // that throws counts as now, and as the default. this.#clock = () => { @@ -305,7 +308,7 @@ export class SessionImpl implements Session { // Seal first, so a report handler that calls back finds the Session failed. Closing runs the close listeners // synchronously; a failing backend close is the caller's to see, and here nobody waits. this.close(BACKGROUND_CONTEXT).catch(() => {}); - this.report(error, { type: "session_failed" }, context); + this.report(error, { kind: "storage_failed" }, context); } /** @@ -462,10 +465,10 @@ export class SessionImpl implements Session { context, (loaded) => { const watch = this.#observeDocument(definition, loaded, createWatch, endWatch); - if (signal !== undefined) watch.observeCancellation(signal); + if (signal !== undefined) watch.observeAbort(signal); return watch; }, - cancelWatch, + abortWatch, ); } catch (error) { return Promise.reject(error); @@ -601,7 +604,12 @@ export class SessionImpl implements Session { unstored = { error }; }); } + const observers = [...this.#commitListeners]; this.#commitListeners.clear(); + // After the last publication: the commits admitted before the seal have settled, held changes are stored. + const end: WatchEnd = + this.#failure === undefined ? ENDED : { kind: "failed", error: this.#failure.error }; + for (const observer of observers) observer.end(end); this.#internalListeners.clear(); this.#documents.clear(); this.#idleDocuments.clear(); @@ -613,7 +621,7 @@ export class SessionImpl implements Session { // A Storage that cannot close is a failed one; an earlier failure stays the cause. if (this.#failure === undefined) { this.#failure = { error }; - this.report(error, { type: "session_failed" }); + this.report(error, { kind: "storage_failed" }); } throw error; } @@ -623,7 +631,7 @@ export class SessionImpl implements Session { .finally(() => { const failure = this.#failure; this.#closed.resolve( - failure === undefined ? { reason: "closed" } : { reason: "failed", error: failure.error }, + failure === undefined ? { kind: "closed" } : { kind: "failed", error: failure.error }, ); }); this.#notifyClose(); @@ -675,24 +683,29 @@ export class SessionImpl implements Session { } /** - * Register a synchronous listener for every stored commit, with exactly what it stored (`CommitPublication`). It runs - * one microtask after Storage adopted the commit, on the line, before every wait the commit resolves and every abort - * it queues (in no order with watch listeners); it must not block or call Session operations, and a throw is - * reported. + * Attach a synchronous observer of every stored commit, with exactly what it stored (`CommitPublication`). Its + * `advance` runs one microtask after Storage adopted the commit, on the line, before every wait the commit resolves + * and every abort it queues (in no order with watch listeners); it must not block or call Session operations. A throw + * is reported, detaches it, and ends it `listener_error`; `end` gets the Session's close after its last publication. */ - subscribeCommits(listener: (publication: CommitPublication, context: Context) => void): () => void { + observeCommits(observer: CommitObserver): CommitObservation { this.assertUsable(); - this.#commitListeners.add(listener); - return () => this.#commitListeners.delete(listener); + const guarded = guardObserver( + observer, + () => this.#commitListeners.delete(guarded), + (error, context) => this.report(error, LISTENER, context), + ); + this.#commitListeners.add(guarded); + return { detach: () => guarded.detach() }; } /** - * Internal: observe every commit as memory adopts it (`ObservedCommit`), for the Session's own components, whose state + * Internal: follow every commit as memory adopts it (`ObservedCommit`), for the Session's own components, whose state * follows memory. A throw leaves that state behind, so it fails the Session instead of being reported. For a stored * commit they run inside Storage's `adopt`, so they call no Storage method, run no task code, and contain every host - * callback; the waits they resolve and the aborts they queue run after host listeners. + * callback; the waits they resolve and the aborts they queue run after host observers. */ - observeCommits(listener: (publication: ObservedCommit, context: Context) => void): () => void { + followCommits(listener: (publication: ObservedCommit, context: Context) => void): () => void { this.assertUsable(); this.#internalListeners.add(listener); return () => this.#internalListeners.delete(listener); @@ -963,7 +976,7 @@ export class SessionImpl implements Session { /** * Store the held changes of `holds` in one storage transaction, on the line. Memory already has them, so only - * `subscribeCommits()` listeners see it, published as Storage adopts it. A failure fails the Session, as every Storage + * `observeCommits()` observers see it, published as Storage adopts it. A failure fails the Session, as every Storage * failure does. */ async #storeHolds(holds: readonly Hold[], context: Context): Promise { @@ -977,7 +990,7 @@ export class SessionImpl implements Session { writes.push({ type: "document.change", id: record.id, content: stored.content }); // Held documents are conversation documents. const conversationId = record.scope.kind === "conversation" ? record.scope.conversationId : undefined; - changes.push({ type: "document", record, conversationId, version, value, ops: stored.ops }); + changes.push({ type: "document", kind: "change", record, conversationId, version, value, ops: stored.ops }); } let adopted = false; const adopt = (seq: Seq): void => { @@ -1035,8 +1048,8 @@ export class SessionImpl implements Session { } /** - * Inside `adopt`: publish a stored commit. Its telemetry events first, then the dispatch to `subscribeCommits()` - * listeners, queued before the waits and aborts the observers wake, then the Session's own observers. Never throws: + * Inside `adopt`: publish a stored commit. Its telemetry events first, then the dispatch to `observeCommits()` + * observers, queued before the waits and aborts the observers wake, then the Session's own observers. Never throws: * returns the first error of an own observer, which fails the Session once `Storage.commit()` returned. */ #publish( @@ -1107,19 +1120,14 @@ export class SessionImpl implements Session { } /** - * Inside `adopt`: call the `subscribeCommits()` listeners with `publication` one microtask from now, outside the + * Inside `adopt`: call the `observeCommits()` observers with `publication` one microtask from now, outside the * storage's synchronous step and, queued before the Session's own observers run, before the waits they resolve and * the aborts they queue. - * The listeners are those registered when it runs; a throw is reported. + * The observers are those attached when it runs; their guards report a throw. */ #notifyHosts(publication: CommitPublication, context: Context): void { void SETTLED.then(() => { - for (const listener of [...this.#commitListeners]) { - contained( - () => listener(publication, context), - (error) => this.report(error, LISTENER, context), - ); - } + for (const observer of [...this.#commitListeners]) observer.advance(publication, context); }); } @@ -1202,7 +1210,7 @@ export class SessionImpl implements Session { unpin(); }; const observer = create(loaded.tracker.value, detach, (error) => this.report(error)); - unsubscribeCommit = this.observeCommits((publication, context) => { + unsubscribeCommit = this.followCommits((publication, context) => { // Listeners run from a copy of the set: another observer's call may have detached this one. if (!attached) return; let ownerEnds = false; @@ -1221,14 +1229,14 @@ export class SessionImpl implements Session { // The incarnation followed is at the address by definition; only another record needs the address compared. if (change.type !== "document") continue; if (change.record.id !== incarnation && !atAddress(change.record, address)) continue; - if (change.value === null) { + if (change.kind === "retire") { if (change.record.id !== incarnation) continue; // Set with every incarnation. const last = value!; incarnation = undefined; value = undefined; unpin(); - observer.advance({ value: last, ops: NO_OPERATIONS, retired: true }, context); + observer.advance({ kind: "value", value: last, ops: NO_OPERATIONS, retired: true }, context); } else if (change.record.id !== incarnation || change.version !== version) { // A new incarnation at the address, or a value of another definition version, whose shape the observer's // value does not have: shown whole. @@ -1237,14 +1245,14 @@ export class SessionImpl implements Session { unpin(); unpin = this.#pinDocument(addressId, incarnation); } - version = change.version!; + version = change.version; value = change.value; - observer.advance({ value, ops: [["r", value]], retired: false }, context); + observer.advance({ kind: "value", value, ops: [["r", value]], retired: false }, context); } else { // A migration-only base changes nothing for an observer of the new version. if (change.ops.length === 0) continue; value = change.value; - observer.advance({ value, ops: change.ops, retired: false }, context); + observer.advance({ kind: "value", value, ops: change.ops, retired: false }, context); } // The observer may detach in its call, or throw and be detached: the rest of the commit is not its. if (!attached) return; @@ -1256,7 +1264,7 @@ export class SessionImpl implements Session { }); unsubscribeClose = this.subscribeClose(() => { detach(); - observer.end(this.#failure); + observer.end(this.#failure === undefined ? ENDED : { kind: "failed", error: this.#failure.error }); }); return observer; } @@ -1331,21 +1339,27 @@ export function contained(callback: () => unknown, onError: (error: unknown) => } } +/** A synchronous observer as Durable attaches it: `Observer` (`advance` gets a `WatchCall`) or `CommitObserver`. */ +type SyncObserver = { advance(call: A, context: Context): void; end(end: WatchEnd): void }; + +/** A guarded observer (`guardObserver()`) with its observation's detach. */ +export type GuardedObserver = SyncObserver & { detach(): void }; + /** - * `observer`, a host's, behind the rule for observers that run inside Storage's `adopt`: one that throws is reported - * through `report`, detached, and ended with the error, so the throw never reaches `adopt`; nothing reaches it after - * its detach or end, and from then on the guard references neither it nor its source. `detach` is the source's; the - * returned `detach()` is the observation's. + * `observer`, a host's, behind the rule for observers that run inside Storage's `adopt`: one that throws, or whose + * `advance` returns a promise that rejects, is reported through `report`, detached, and ended `listener_error`, so the + * throw never reaches `adopt`; nothing reaches it after its detach or end, and from then on the guard references + * neither it nor its source. `detach` is the source's; the returned `detach()` is the observation's. */ -export function guardObserver( - observer: Observer, +export function guardObserver( + observer: SyncObserver, detach: () => void, - report: (error: unknown) => void, -): Observer & { detach(): void } { + report: (error: unknown, context?: Context) => void, +): GuardedObserver { type Attached = { - readonly observer: Observer; + readonly observer: SyncObserver; readonly detach: () => void; - readonly report: (error: unknown) => void; + readonly report: (error: unknown, context?: Context) => void; }; let attached: Attached | undefined = { observer, detach, report }; /** Detach from the source once, and hand back what it was attached with. */ @@ -1355,20 +1369,31 @@ export function guardObserver( released?.detach(); return released; }; + /** Report a throw of `current`'s `advance`, then detach and end it, unless that happened meanwhile. */ + const fault = (current: Attached, error: unknown, context: Context): void => { + current.report(error, context); + if (release() !== undefined) { + const end: WatchEnd = { kind: "listener_error", error: toError(error) }; + contained(() => current.observer.end(end), current.report); + } + }; return { - advance: (change, context) => { + advance: (call, context) => { const current = attached; if (current === undefined) return; try { - current.observer.advance(change, context); + const result: unknown = current.observer.advance(call, context); + // Any thenable, as `contained()` takes it: a rejection is the observer's throw, after the fact. + if (typeof (result as { then?: unknown } | null)?.then === "function") { + Promise.resolve(result).catch((error: unknown) => fault(current, error, context)); + } } catch (error) { - current.report(error); - if (release() !== undefined) contained(() => current.observer.end({ error }), current.report); + fault(current, error, context); } }, - end: (failure) => { + end: (end) => { const ending = release(); - if (ending !== undefined) contained(() => ending.observer.end(failure), ending.report); + if (ending !== undefined) contained(() => ending.observer.end(end), ending.report); }, detach: () => { release(); @@ -1376,6 +1401,19 @@ export function guardObserver( }; } +/** A listener's throw as the `Error` a `listener_error` end carries, as Chord's watch makes it. */ +function toError(error: unknown): Error { + if (error instanceof Error) return error; + try { + return new Error(String(error)); + } catch { + return new Error(Object.prototype.toString.call(error)); + } +} + +/** The end of an observer whose source closed. */ +export const ENDED: WatchEnd = { kind: "ended" }; + /** * The `Observation` of a guarded observer attached at `value`. Built outside the attaching closure, so a detached * observation keeps nothing but `value` and the spent guard alive. @@ -1393,9 +1431,9 @@ function createWatch( return new Watch>(value, detach, report); } -/** Undo a watch attached while its acquisition was cancelled. */ -function cancelWatch(watch: Watch>): void { - watch.cancel(); +/** Undo a watch attached while its acquisition was aborted. */ +function abortWatch(watch: Watch>): void { + watch.abort(); } /** Undo an observation attached while its attachment was cancelled. */ @@ -1410,7 +1448,7 @@ function endWatch(watch: Watch>): void { /** A task document's observer ends right after the retirement that ends its task. */ function endGuarded(guarded: Observer>): void { - guarded.end(undefined); + guarded.end(ENDED); } /** Whether `record` is an incarnation at `address`. */ diff --git a/packages/durable/src/session/transaction.ts b/packages/durable/src/session/transaction.ts index 7a62c440b..cd59bba27 100644 --- a/packages/durable/src/session/transaction.ts +++ b/packages/durable/src/session/transaction.ts @@ -109,7 +109,7 @@ export type HeldDocument = { base: boolean; }; -/** What adoption publishes: to the Session's own observers, and the stored changes to `subscribeCommits()`. */ +/** What adoption publishes: to the Session's own observers, and the stored changes to `observeCommits()`. */ export type AdoptedDocuments = { /** This commit's own changes, as memory adopted them. */ readonly observed: readonly DocumentCommitChange[]; @@ -791,6 +791,7 @@ export class Transaction implements Tx { const conversationId = plan.conversationId; publications.push({ type: "document", + kind: "change", record, conversationId, version, @@ -883,8 +884,12 @@ export class Transaction implements Tx { } stored ??= [...publications]; if (held.observed) publications.push(publication); - // Only a document change of a loaded incarnation carries held operations. - stored.push(publication.type === "document" ? { ...publication, ops: held.ops } : publication); + // Only a document change of a loaded incarnation carries held operations; a retirement is published alone. + stored.push( + publication.type === "document" && publication.kind === "change" + ? { ...publication, ops: held.ops } + : publication, + ); } return { observed: publications, stored: stored ?? publications }; } @@ -1173,15 +1178,22 @@ function planDocument(document: DocumentEntry): DocumentPlan | undefined { /** The publication of one adopted plan, if it publishes (`publishes()`), with `record` as committed. */ function describe(plan: DocumentPlan, record: DocumentRecord): DocumentCommitChange | undefined { const conversationId = plan.conversationId; - if (plan.retire) - return { type: "document", record, conversationId, version: undefined, value: null, ops: EMPTY_OPERATIONS }; + if (plan.retire) return { type: "document", kind: "retire", record, conversationId }; if (plan.content?.type === "document.copy") { return { type: "document.copy", record, conversationId: conversationId!, source: plan.content.source }; } const change = plan.change; if (change === undefined || !publishes(plan)) return undefined; const ops = change.loaded === undefined ? EMPTY_OPERATIONS : change.prepared.ops; - return { type: "document", record, conversationId, version: change.version, value: change.prepared.value, ops }; + return { + type: "document", + kind: "change", + record, + conversationId, + version: change.version, + value: change.prepared.value, + ops, + }; } /** diff --git a/packages/durable/src/testing/env-conformance.ts b/packages/durable/src/testing/env-conformance.ts index 1e6d1ad1a..596ea95ad 100644 --- a/packages/durable/src/testing/env-conformance.ts +++ b/packages/durable/src/testing/env-conformance.ts @@ -64,8 +64,8 @@ async function listed( /** Whether a change reports `path`: an overflow, or a reported path at or above it. */ function covers(change: FileChange, path: string): boolean { - if ("overflow" in change) return true; - if (!("paths" in change)) return false; + if (change.kind === "overflow") return true; + if (change.kind !== "paths") return false; return change.paths.some( (reported) => path === reported || path.startsWith(`${reported}/`) || path.startsWith(`${reported}\\`), ); @@ -103,7 +103,7 @@ async function watching( }, }); } finally { - await watcher.close(context); + await watcher.stop(context); } } @@ -346,7 +346,7 @@ export function createEnvConformance(options: EnvConformanceOptions): readonly E }); const hidden = [await absolute("skills/node_modules"), await absolute("skills/.SKILL.md.tmp")]; for (const change of changes) { - if (!("paths" in change)) continue; + if (change.kind !== "paths") continue; for (const path of change.paths) { assert.ok( !hidden.some((excluded) => path === excluded || path.startsWith(excluded)), @@ -383,12 +383,12 @@ export function createEnvConformance(options: EnvConformanceOptions): readonly E }); }), - watchCase("watch stops reporting once closed", async (env) => { + watchCase("watch stops reporting once stopped", async (env) => { const changes: FileChange[] = []; const watcher = getOrThrow(await env.watch([{ path: "file.txt" }], (change) => changes.push(change), context)); assert.ok(watcher.mode === "native" || watcher.mode === "polling", "watcher reports its mode"); - await watcher.close(context); - await watcher.close(context); + await watcher.stop(context); + await watcher.stop(context); getOrThrow(await env.writeFile("file.txt", "x", context)); await new Promise((resolve) => setTimeout(resolve, 300)); assert.deepEqual(changes, []); diff --git a/packages/durable/src/types.ts b/packages/durable/src/types.ts index e1c9c6453..3e20ff6ac 100644 --- a/packages/durable/src/types.ts +++ b/packages/durable/src/types.ts @@ -1,4 +1,4 @@ -import type { Context, Draft, JsonValue, WatchChange, WatchHandle } from "@earendil-works/chord"; +import type { Context, Draft, JsonValue, WatchCall, WatchEnd, WatchHandle } from "@earendil-works/chord"; import type { Op } from "@earendil-works/chord/delta"; import type { Message, Models } from "@earendil-works/pi-ai"; import type { TelemetryCarrier } from "@earendil-works/pi-telemetry"; @@ -149,9 +149,10 @@ export type ReportTask = { export type HostCallback = "now" | "settings" | "registry" | "env" | "models" | "setTimer" | "onWake" | "telemetry"; /** - * What a report to `HarnessOptions.onReport` is about. `extension`, where present, names the extension whose code - * failed, when known; for a task's own code, the one that installed the definition the task ran. - * - `session_failed`: the error failed the Session, which closes: a Storage error, often a platform reset. + * What a report to `HarnessOptions.onReport` is about, by `kind` (`Report` adds the error). `extension`, where present, + * names the extension whose code failed, when known; for a task's own code, the one that installed the definition the + * task ran. + * - `storage_failed`: the error failed the Session, which closes: a Storage error, often a platform reset. * - `task_faulted`: the task ended `faulted`: its phase threw (`reason: "threw"`), or its abort handler was cut off * `settings.restartLimit` times (`reason: "restart_limit"`), as was a phase of a task of an extension's kind. * - `task_interrupted`: crashes ended the task: it ended itself after being cut off `settings.restartLimit` times in a @@ -159,7 +160,8 @@ export type HostCallback = "now" | "settings" | "registry" | "env" | "models" | * host kept stopping while the model answered (`reason: "interrupted"`). `restarts` is the task's count when it ended. * - `migration_failed`: the task's stored state could not be migrated; it is blocked until the registry changes. * - `task_outdated`: a running task keeps running under its old definition, which was uninstalled (`missing_task`) - * or replaced by one that cannot take over its state (`incompatible_task`). + * or replaced by one that cannot take over its state: older than the task (`task_too_old`), or newer without a + * `migrate` (`migration_failed`). * - `task_error`: the task's own `runtime.report()`, or a failure in running it. * - `extension_error`: a hook, a section, a wrapper, or a tool's invalid structured output; with the task that ran it, * or, outside a task, the conversation whose agent was resolved. @@ -167,41 +169,44 @@ export type HostCallback = "now" | "settings" | "registry" | "env" | "models" | * - `listener_error`: a commit, close, watch, view, or document listener threw. */ export type ReportInfo = - | { readonly type: "session_failed" } + | { readonly kind: "storage_failed" } | { - readonly type: "task_faulted"; + readonly kind: "task_faulted"; readonly task: ReportTask; readonly extension?: string; readonly reason: "threw" | "restart_limit"; } | { - readonly type: "task_interrupted"; + readonly kind: "task_interrupted"; readonly task: ReportTask; readonly extension?: string; readonly reason: "restart_limit" | "interrupted"; readonly restarts: number; } - | { readonly type: "migration_failed"; readonly task: ReportTask; readonly extension?: string } + | { readonly kind: "migration_failed"; readonly task: ReportTask; readonly extension?: string } | { - readonly type: "task_outdated"; + readonly kind: "task_outdated"; readonly task: ReportTask; readonly extension?: string; - readonly reason: "missing_task" | "incompatible_task"; + readonly reason: "missing_task" | "task_too_old" | "migration_failed"; } - | { readonly type: "task_error"; readonly task: ReportTask; readonly extension?: string } + | { readonly kind: "task_error"; readonly task: ReportTask; readonly extension?: string } | { - readonly type: "extension_error"; + readonly kind: "extension_error"; readonly extension?: string; readonly task?: ReportTask; readonly conversationId?: ConversationId; } - | { readonly type: "host_error"; readonly callback: HostCallback; readonly task?: ReportTask } - | { readonly type: "listener_error" }; + | { readonly kind: "host_error"; readonly callback: HostCallback; readonly task?: ReportTask } + | { readonly kind: "listener_error" }; + +/** One report to `HarnessOptions.onReport`: what it is about (`ReportInfo`, by `kind`) and the `error` reported. */ +export type Report = ReportInfo & { readonly error: unknown }; /** Whose code a task's `runtime.report()` is about when not the task's own: an extension's, or a host callback. */ export type ReportFrom = - | { readonly type: "extension"; readonly extension?: string } - | { readonly type: "host"; readonly callback: HostCallback }; + | { readonly kind: "extension"; readonly extension?: string } + | { readonly kind: "host"; readonly callback: HostCallback }; /** * A host timer, given as `HarnessOptions.setTimer`: call `callback` once, about `delayMs` from now, and return @@ -882,20 +887,31 @@ export type StorageWrite = } | { readonly type: "document.retire"; readonly id: DocumentId }; -/** Committed change of one document incarnation. */ +/** + * Committed change of one document incarnation: `type: "document"` with `kind: "change"` (a creation or an update) or + * `kind: "retire"` (this commit retired the incarnation), or a `document.copy`. + */ export type DocumentCommitChange = | { readonly type: "document"; + readonly kind: "change"; readonly record: DocumentRecord; /** Conversation owning the document; task documents derive it from their task record. Undefined only for Session documents. */ readonly conversationId: ConversationId | undefined; - /** Definition version of `value`; absent when this commit retired the incarnation. */ - readonly version: number | undefined; - /** Exact adopted immutable revision, or `null` when this commit retired the incarnation. */ - readonly value: JsonObject | null; - /** Exact adopted operations for an ordinary update; empty for creation and retirement. */ + /** Definition version of `value`. */ + readonly version: number; + /** Exact adopted immutable revision. */ + readonly value: JsonObject; + /** Exact adopted operations for an ordinary update; empty for creation. */ readonly ops: readonly Op[]; } + | { + readonly type: "document"; + readonly kind: "retire"; + readonly record: DocumentRecord; + /** Conversation owning the document; task documents derive it from their task record. Undefined only for Session documents. */ + readonly conversationId: ConversationId | undefined; + } | { /** Definition-free child initialization; consumers hydrate through watch acquisition. */ readonly type: "document.copy"; @@ -1036,21 +1052,35 @@ export type DocumentWatch = WatchHandle>; /** * A synchronous observer of committed Durable state, for adapters such as a Chord source (`observeDoc()`, - * `Harness.observeConversation()`, `Harness.observeTaskGraph()`). `advance` gets every change, unfolded, as memory - * adopts it: for a stored commit inside Storage's `adopt`, for a deferred one before it is stored. A view or the graph - * gets at most one call per commit, a document observer one per change at its address (two for a commit that retires - * and recreates it); the task graph also advances outside any commit and `adopt`, on the Session line, when a registry - * change or a failed migration changes a `blocked` reason. Calls may come before the attaching promise resolves; their - * ops apply from `Observation.value`. `advance` must only record the change, synchronously and quickly, never await, - * call the Session, or throw. A throw is reported as `listener_error`, detaches the observer, and ends it with the - * error. `end` is called once, when the Session closes or fails (`failure`), or when the observed scope's owner ends (a - * task document's task, after its retirement change, inside that commit's `adopt`), and never after `detach()`. + * `Harness.observeConversation()`, `Harness.observeTaskGraph()`), with the shapes of Chord's watch: `advance` gets a + * `WatchCall`, `end` a `WatchEnd`. `advance` gets every change, unfolded, as memory adopts it: for a stored commit inside + * Storage's `adopt`, for a deferred one before it is stored. A view or the graph gets at most one call per commit, a + * document observer one per change at its address (two for a commit that retires and recreates it); the task graph also + * advances outside any commit and `adopt`, on the Session line, when a registry change or a failed migration changes a + * `blocked` reason. Durable's failures are final, so every call is a `value` call; a failure ends the observer. Calls + * may come before the attaching promise resolves; their ops apply from `Observation.value`. `advance` must only record + * the call, synchronously and quickly, never await, call the Session, or throw. A throw is reported as + * `listener_error`, detaches the observer, and ends it `listener_error` with the error. `end` is called once: `ended` + * when the Session closes or the observed scope's owner ends (a task document's task, after its retirement change, + * inside that commit's `adopt`), `failed` with the error that failed the Session; never after `detach()`. */ export type Observer = { - advance(change: WatchChange, context: Context): void; - end(failure: { readonly error: unknown } | undefined): void; + advance(call: WatchCall, context: Context): void; + end(end: WatchEnd): void; }; +/** + * A synchronous observer of stored commits (`Session.observeCommits()`): `advance` gets each `CommitPublication`, `end` + * the Session's close. + */ +export type CommitObserver = { + advance(publication: CommitPublication, context: Context): void; + end(end: WatchEnd): void; +}; + +/** An attached `CommitObserver`: how to detach it. Idempotent: no `advance` or `end` follows it. */ +export type CommitObservation = { detach(): void }; + /** An attached `Observer`: the value at attachment, which every later `advance` starts from, and how to detach it. */ export type Observation = { readonly value: T; @@ -1093,8 +1123,8 @@ export interface DocumentObserver { ): Promise | undefined>; } -/** Why a Session ended: `close()`, or a failed Storage call, whose error it carries. */ -export type SessionEnd = { readonly reason: "closed" } | { readonly reason: "failed"; readonly error: unknown }; +/** Why a Session ended, by `kind`: `close()`, or a failed Storage call, whose error it carries. */ +export type SessionEnd = { readonly kind: "closed" } | { readonly kind: "failed"; readonly error: unknown }; /** * When a commit reaches Storage. `immediate`, the default: before `commit()` resolves. `deferred`: when every write of @@ -1174,11 +1204,14 @@ export interface Session extends DocumentObserver { * Observe stored commits, each with exactly what it stored, except that a document the commit retires is published * as its retirement only; a commit held in memory is published when it is stored (`CommitPublication`). Storage's * adoption of the commit queues one dispatch, which runs a microtask later, before every wait the commit resolves - * and every abort it queues, and calls the listeners registered by then; a commit adopted while none was registered - * has no dispatch. There is no order relative to watch listeners. The listener runs synchronously and must not block - * or call Session APIs. + * and every abort it queues, and calls the observers attached by then; a commit adopted while none was attached has + * no dispatch. There is no order relative to watch listeners. `advance` runs synchronously and must not block or + * call Session APIs; a throw is reported as `listener_error`, detaches the observer, and ends it `listener_error`. + * `end` is called once, when the Session closes (`ended`, after its last publication) or fails (`failed`), never + * after `detach()`. Attaches synchronously; the observation has no `value`: commits are events, not a state, and the + * Session does not know Storage's newest `seq` before its own first commit. */ - subscribeCommits(listener: (publication: CommitPublication, context: Context) => void): () => void; + observeCommits(observer: CommitObserver): CommitObservation; /** * Observe close synchronously when it begins, also when a failed Storage call closes the Session. The listener must * not block or call Session APIs. diff --git a/packages/durable/test/chat-support.ts b/packages/durable/test/chat-support.ts index 1e9538ee1..1c2946f4a 100644 --- a/packages/durable/test/chat-support.ts +++ b/packages/durable/test/chat-support.ts @@ -63,7 +63,7 @@ export async function openChat( ? {} : { env: typeof options.env === "function" ? options.env : () => options.env as ExecutionEnv }), now: () => setup.now(), - onReport: (error, info) => { + onReport: ({ error, ...info }) => { setup.reports.push(error); setup.infos.push(info); }, diff --git a/packages/durable/test/codemode-bench.ts b/packages/durable/test/codemode-bench.ts index 982443cdb..c9c3c2b17 100644 --- a/packages/durable/test/codemode-bench.ts +++ b/packages/durable/test/codemode-bench.ts @@ -66,8 +66,11 @@ async function run(mode: "codemode" | "plain", shape: Shape, n: number): Promise const harness = await Harness.open(storage, { models, registry, settings: {} }, context); const root = await harness.root(context, { agent: { model: { provider: "faux", modelId: "faux-1" } } }); let commits = 0; - harness.subscribeCommits(() => { - commits++; + harness.observeCommits({ + advance: () => { + commits++; + }, + end: () => {}, }); const started = performance.now(); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); diff --git a/packages/durable/test/codemode.test.ts b/packages/durable/test/codemode.test.ts index 650255e43..460836d72 100644 --- a/packages/durable/test/codemode.test.ts +++ b/packages/durable/test/codemode.test.ts @@ -928,10 +928,13 @@ describe("durable code mode", () => { ]); const { harness, root } = await openChat(await openNodeSqliteStorage(await sqlitePath()), chat.setup); let journalCommits = 0; - harness.subscribeCommits((publication) => { - for (const change of publication.changes) { - if (change.type === "document" && change.record.kind === "pi.codemode.journal") journalCommits++; - } + harness.observeCommits({ + advance: (publication) => { + for (const change of publication.changes) { + if (change.type === "document" && change.record.kind === "pi.codemode.journal") journalCommits++; + } + }, + end: () => {}, }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); expect(await resultText(chat.setup, root)).toEqual({ text: "50", isError: false }); diff --git a/packages/durable/test/conversation-transcript.test.ts b/packages/durable/test/conversation-transcript.test.ts index d5fb3568c..985941ef9 100644 --- a/packages/durable/test/conversation-transcript.test.ts +++ b/packages/durable/test/conversation-transcript.test.ts @@ -12,7 +12,7 @@ import { } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; import { allEntries, chatSetup, openChat } from "./chat-support.ts"; -import { context } from "./session-support.ts"; +import { context, onValue } from "./session-support.ts"; import { deferred } from "./task-support.ts"; const CONVERSATION = { id: 1 } as unknown as ConversationView["conversation"]; @@ -326,9 +326,11 @@ describe("ConversationTranscript over a Harness", () => { const window = { min: 10, max: 20 }; const watch = await root.watch(context, window); let view = helper.merge(watch.value); - watch.start(async ({ value }) => { - view = helper.merge(value); - }); + watch.start( + onValue(async ({ value }) => { + view = helper.merge(value); + }), + ); for (let index = 0; index < 15; index++) await note(); await new Promise((resolve) => setTimeout(resolve, 0)); expect(view.items.some((item) => item.kind === "gap")).toBe(false); diff --git a/packages/durable/test/conversation-view-bench.ts b/packages/durable/test/conversation-view-bench.ts index 48d5ec565..d9096293f 100644 --- a/packages/durable/test/conversation-view-bench.ts +++ b/packages/durable/test/conversation-view-bench.ts @@ -20,15 +20,7 @@ import { promisify } from "node:util"; import type { Context } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; import { createModels } from "@earendil-works/pi-ai/models"; -import { - type Conversation, - type ConversationView, - createRegistry, - type EntryId, - Harness, - LiveDoc, - type Storage, -} from "../src/index.ts"; +import { type Conversation, createRegistry, type EntryId, Harness, LiveDoc, type Storage } from "../src/index.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; type Scenario = @@ -179,7 +171,7 @@ async function run(scenario: Scenario): Promise> { await root.commit(async (tx) => { (await tx.doc(LiveDoc, root.id)).compactionFailure = { taskId: 1 as never, - reason: "manual", + trigger: "manual", error: `${index}`, }; }, context); @@ -198,8 +190,9 @@ async function run(scenario: Scenario): Promise> { } else { const watch = await root.watch(context); let first: WeakRef | undefined; - watch.start(async ({ value }: { value: ConversationView }) => { - const entry = value.entries.at(-1); + watch.start(async (call) => { + if (call.kind !== "value") return; + const entry = call.value.entries.at(-1); if (first === undefined && entry !== undefined) first = new WeakRef(entry); }); await appendEntries(root, 1, 1); diff --git a/packages/durable/test/env-node-conformance.test.ts b/packages/durable/test/env-node-conformance.test.ts index 5bdbe15c5..b7f36b1c1 100644 --- a/packages/durable/test/env-node-conformance.test.ts +++ b/packages/durable/test/env-node-conformance.test.ts @@ -78,11 +78,11 @@ describe("NodeExecutionEnv watch limits", () => { ); getOrThrow(await env.createDir("tree/d", undefined, context)); const deadline = Date.now() + 3000; - while (!changes.some((change) => "error" in change) && Date.now() < deadline) { + while (!changes.some((change) => change.kind === "failed") && Date.now() < deadline) { await new Promise((resolve) => setTimeout(resolve, 20)); } - expect(changes.at(-1)).toMatchObject({ error: { code: "invalid" } }); - await watcher.close(context); + expect(changes.at(-1)).toMatchObject({ kind: "failed", error: { code: "invalid" } }); + await watcher.stop(context); } finally { await removeTempDir(cwd); } diff --git a/packages/durable/test/examples/04-watches.ts b/packages/durable/test/examples/04-watches.ts index eed3a17cc..be3c0f405 100644 --- a/packages/durable/test/examples/04-watches.ts +++ b/packages/durable/test/examples/04-watches.ts @@ -32,7 +32,11 @@ console.log("watch baseline:", notesWatch.value); const busy = Promise.withResolvers(); const done = Promise.withResolvers(); let calls = 0; -notesWatch.start(async ({ value, ops }, _deliveryContext) => { +notesWatch.start(async (call, _deliveryContext) => { + // Calls are tagged: a `failed` call is a failure a source recovers from, which Durable never sends (its failures + // end the watch, see `closed`). + if (call.kind !== "value") return; + const { value, ops } = call; console.log("watch update:", value, JSON.stringify(ops)); // The first call stays busy, as a slow consumer would, while three more commits land. if (++calls === 1) await busy.promise; diff --git a/packages/durable/test/examples/15-real-model.ts b/packages/durable/test/examples/15-real-model.ts index 8adb2bf50..e43c6b427 100644 --- a/packages/durable/test/examples/15-real-model.ts +++ b/packages/durable/test/examples/15-real-model.ts @@ -45,7 +45,9 @@ if (process.env.OPENAI_API_KEY === undefined) { process.stdout.write(text.slice(printed.length)); printed = text; }; - liveWatch.start(async ({ value }) => { + liveWatch.start(async (call) => { + if (call.kind !== "value") return; + const { value } = call; const block = value?.generation?.message?.content.find((content) => content.type === "text"); if (block?.type === "text") printText(block.text); }); diff --git a/packages/durable/test/examples/18-json.ts b/packages/durable/test/examples/18-json.ts index 15e79b795..2e2b801c7 100644 --- a/packages/durable/test/examples/18-json.ts +++ b/packages/durable/test/examples/18-json.ts @@ -73,11 +73,15 @@ const root = await harness.root(context, { agent: { model } }); const watch = await root.watch(context); if (mode === "ops") { print({ view: watch.value }); - watch.start(async ({ ops }) => print({ ops })); + watch.start(async (call) => { + if (call.kind === "value") print({ ops: call.ops }); + }); } else { let newest = watch.value.entries.at(-1)?.id ?? 0; print({ entries: watch.value.entries, live: watch.value.docs["pi.live"] }); - watch.start(async ({ value }) => { + watch.start(async (call) => { + if (call.kind !== "value") return; + const { value } = call; const entries = value.entries.filter((entry) => entry.id > newest); newest = value.entries.at(-1)?.id ?? newest; print({ ...(entries.length === 0 ? {} : { entries }), live: value.docs["pi.live"] }); diff --git a/packages/durable/test/examples/20-late-join.ts b/packages/durable/test/examples/20-late-join.ts index af7b24919..73b53076b 100644 --- a/packages/durable/test/examples/20-late-join.ts +++ b/packages/durable/test/examples/20-late-join.ts @@ -63,7 +63,9 @@ console.log( ); console.log("view tool slot:", live.tools?.[0]?.status, JSON.stringify(live.tools?.[0]?.output)); let shown = ""; -view.start(async ({ value }) => { +view.start(async (call) => { + if (call.kind !== "value") return; + const { value } = call; const state = value.docs["pi.live"] as LiveState; const slot = state.tools?.[0]; if (slot?.status === "running") console.log("view output now:", JSON.stringify(slot.output)); diff --git a/packages/durable/test/examples/21-subagent-foreground.ts b/packages/durable/test/examples/21-subagent-foreground.ts index cdf8183c9..af4a31823 100644 --- a/packages/durable/test/examples/21-subagent-foreground.ts +++ b/packages/durable/test/examples/21-subagent-foreground.ts @@ -115,7 +115,9 @@ const attach = async (id: ConversationId, indent: string): Promise => { attached.add(id); const watch = await (await harness.conversation(id, context))!.watch(context); let newest = watch.value.entries.at(-1)?.id ?? 0; - watch.start(async ({ value }) => { + watch.start(async (call) => { + if (call.kind !== "value") return; + const { value } = call; for (const entry of value.entries) if (entry.id > newest) print(indent, entry); newest = value.entries.at(-1)?.id ?? newest; // A running subagent call names its child conversation in its details. diff --git a/packages/durable/test/examples/22-subagent-background.ts b/packages/durable/test/examples/22-subagent-background.ts index 7bdb67470..7aaf64d8b 100644 --- a/packages/durable/test/examples/22-subagent-background.ts +++ b/packages/durable/test/examples/22-subagent-background.ts @@ -284,7 +284,9 @@ function contentText(content: string | readonly { type: string; text?: string }[ async function follow(harness: Harness, id: ConversationId): Promise { const watch = await (await harness.conversation(id, context))!.watch(context); let newest = watch.value.entries.at(-1)?.id ?? 0; - watch.start(async ({ value }) => { + watch.start(async (call) => { + if (call.kind !== "value") return; + const { value } = call; const entries = value.entries.filter((entry) => entry.id > newest); newest = value.entries.at(-1)?.id ?? newest; for (const entry of entries) { diff --git a/packages/durable/test/examples/31-transcript.ts b/packages/durable/test/examples/31-transcript.ts index 49bf75915..bdbf7f1f2 100644 --- a/packages/durable/test/examples/31-transcript.ts +++ b/packages/durable/test/examples/31-transcript.ts @@ -64,7 +64,9 @@ const window = { min: 10, max: 20 }; let watch = await root.watch(context, window); let view = transcript.merge(watch.value); render("attach", view); -watch.start(async ({ value }) => { +watch.start(async (call) => { + if (call.kind !== "value") return; + const { value } = call; view = transcript.merge(value); }); @@ -84,7 +86,9 @@ await chat(40); watch = await root.watch(context, { ...window, ...transcript.resume }); view = transcript.merge(watch.value); render("resume after 40 missed", view); -watch.start(async ({ value }) => { +watch.start(async (call) => { + if (call.kind !== "value") return; + const { value } = call; view = transcript.merge(value); }); diff --git a/packages/durable/test/harness-blocked.test.ts b/packages/durable/test/harness-blocked.test.ts index 50ab0cb43..bc85d0e52 100644 --- a/packages/durable/test/harness-blocked.test.ts +++ b/packages/durable/test/harness-blocked.test.ts @@ -125,7 +125,7 @@ describe("recovery at open", () => { }); const wakes: Wake[] = []; const { harness } = await openTasks(failing, [], { registry, onWake: (wake) => wakes.push(wake) }); - expect((await harness.closed).reason).toBe("failed"); + expect((await harness.closed).kind).toBe("failed"); // The host was told work is underway, and never that it ended: its recovery alarm stays. expect(wakes).toEqual([{ active: true }]); }); @@ -153,7 +153,10 @@ describe("tasks that cannot run", () => { expect((await harness.getTask(id, context))?.restarts).toBe(3); // Nothing is parked: the task is ready, and the host is asked to run, which ends it. const inspection = await harness.inspect(context); - expect(inspection.tasks.find((task) => task.record.id === id)?.state).toEqual({ kind: "ready", migrates: false }); + expect(inspection.tasks.find((task) => task.record.id === id)?.state).toEqual({ + kind: "pending", + migrates: false, + }); expect(harness.wake().active).toBe(true); harness.resume(); const settled = await harness.waitForTask(id, context); @@ -168,7 +171,7 @@ describe("tasks that cannot run", () => { await eventually(() => infos.length > 0); expect(infos).toEqual([ { - type: "task_faulted", + kind: "task_faulted", task: { id, kind: "test.crash-loop", conversationId: root.id }, extension: "loop", reason: "restart_limit", @@ -263,7 +266,7 @@ describe("tasks that cannot run", () => { expect(settled.restarts).toBeUndefined(); expect(aborts).toBe(1); await eventually(() => opened.infos.length > 0); - expect(opened.infos).toMatchObject([{ type: "task_faulted", task: { id }, reason: "restart_limit" }]); + expect(opened.infos).toMatchObject([{ kind: "task_faulted", task: { id }, reason: "restart_limit" }]); await opened.harness.close(context); }); diff --git a/packages/durable/test/harness-compaction.test.ts b/packages/durable/test/harness-compaction.test.ts index caece1213..afe74558a 100644 --- a/packages/durable/test/harness-compaction.test.ts +++ b/packages/durable/test/harness-compaction.test.ts @@ -18,10 +18,13 @@ import { CompactionTask, type ContextView, type Conversation, + createSession, + defineDoc, defineTask, defineTool, type EntryRecord, type Harness, + type JsonObject, LiveDoc, type LiveState, MemoryStorage, @@ -962,7 +965,7 @@ describe("background threshold compaction", () => { expect(task).toMatchObject({ background: true, input: { reason: "threshold" } }); expect(task!.owner).toBeUndefined(); expect((await live(chat)).compactions).toEqual([ - { taskId: task!.id, reason: "threshold", blocking: false, attempt: 1 }, + { taskId: task!.id, trigger: "threshold", blocking: false, attempt: 1 }, ]); // Background work: neither conversation idle nor Esc waits for or stops it. await chat.root.waitForIdle(context); @@ -1015,7 +1018,7 @@ describe("background threshold compaction", () => { await turn(chat, text("u4", 100), text("a4", 100)); await waitFor(async () => (await live(chat)).compactionFailure !== undefined); const recorded = (await live(chat)).compactionFailure; - expect(recorded).toMatchObject({ reason: "threshold", error: "Summarization failed: bad request" }); + expect(recorded).toMatchObject({ trigger: "threshold", error: "Summarization failed: bad request" }); // The value that drops the compaction's status records its failure. const ended = (taskId: TaskId) => { const states = views.live(); @@ -1081,7 +1084,7 @@ describe("blocking threshold compaction", () => { checkpoint: { phase: "prepare", attempt: 1, compacted: child!.id }, }); expect((await live(chat)).compactions).toEqual([ - { taskId: child!.id, reason: "threshold", blocking: true, attempt: 1 }, + { taskId: child!.id, trigger: "threshold", blocking: true, attempt: 1 }, ]); // Nothing was appended before the wait. expect((await kinds(chat.root)).at(-1)).toBe("pi.user"); @@ -1141,7 +1144,7 @@ describe("blocking threshold compaction", () => { const state = await live(chat); expect(state.compactions).toBeUndefined(); expect(state.compactionFailure).toMatchObject({ - reason: "threshold", + trigger: "threshold", error: "Summarization failed: bad request", }); await chat.harness.close(context); @@ -1578,7 +1581,7 @@ describe("compaction live status", () => { expect((late.value.docs["pi.live"] as LiveState).compactions).toEqual([ { taskId: id, - reason: "manual", + trigger: "manual", blocking: false, attempt: 1, retry: { at: expect.any(Number), error: "overloaded" }, @@ -1589,7 +1592,7 @@ describe("compaction live status", () => { await waitFor(async () => (await live(chat)).compactions === undefined); views.detach(); const listed = views.live().flatMap((state) => state.compactions ?? []); - expect(listed[0]).toEqual({ taskId: id, reason: "manual", blocking: false, attempt: 1 }); + expect(listed[0]).toEqual({ taskId: id, trigger: "manual", blocking: false, attempt: 1 }); // An abort is not a failure. expect(views.live().at(-1)!.compactionFailure).toBeUndefined(); await chat.harness.close(context); @@ -1990,7 +1993,7 @@ describe("compaction edge cases", () => { expect((await kinds(chat.root)).includes("pi.compaction")).toBe(false); const state = await live(chat); expect(state.compactions).toBeUndefined(); - expect(state.compactionFailure).toMatchObject({ reason: "threshold", error: "no credentials" }); + expect(state.compactionFailure).toMatchObject({ trigger: "threshold", error: "no credentials" }); await chat.harness.close(context); }); @@ -2126,7 +2129,7 @@ describe("compaction edge cases", () => { return summary(); }); const id = await chat.root.compact(undefined, context); - expect(await chat.harness.closed).toEqual({ reason: "failed", error: failure }); + expect(await chat.harness.closed).toEqual({ kind: "failed", error: failure }); await chat.harness.close(context); // Nothing of the failed commit landed: the task is still live, without usage, entry, or submission. storage.reopen(); @@ -2282,7 +2285,7 @@ describe("compaction pinning, silent overflow, and late policy changes", () => { await reached.promise; const watch = await chat.root.watch(context); expect((watch.value.docs["pi.live"] as LiveState).compactions).toEqual([ - { taskId: id, reason: "manual", blocking: false, attempt: 1 }, + { taskId: id, trigger: "manual", blocking: false, attempt: 1 }, ]); await watch.stop(); await chat.harness.abortTask(id, context); @@ -2348,7 +2351,7 @@ describe("blocked compaction", () => { const id = await root.commit(async (tx) => { const taskId = await tx.createTask(newer, { reason: "manual" }, { ownership: { kind: "conversation" } }); const live = await tx.doc(LiveDoc, root.id); - live.compactions = [{ taskId, reason: "manual", blocking: false, attempt: 1 }]; + live.compactions = [{ taskId, trigger: "manual", blocking: false, attempt: 1 }]; return taskId; }, context); await harness.close(context); @@ -2366,7 +2369,7 @@ describe("blocked compaction", () => { reason: "task_too_old", }); expect((await live(chat)).compactions).toBeUndefined(); - expect((await live(chat)).compactionFailure).toEqual({ taskId: id, reason: "manual", error: "task_too_old" }); + expect((await live(chat)).compactionFailure).toEqual({ taskId: id, trigger: "manual", error: "task_too_old" }); await chat.harness.close(context); }); }); @@ -2449,3 +2452,58 @@ describe("per-model compaction overrides", () => { await chat.harness.close(context); }); }); + +describe("pi.live version 2", () => { + it("reads a version 1 value with each compaction's `reason` as `trigger`, and stores it migrated on its next write", async () => { + const storage = new MemoryStorage(); + const session = createSession(storage); + // `pi.live` as version 1 wrote it. + const LiveV1 = defineDoc({ + kind: "pi.live", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({}), + checkpointWhen: () => false, + }); + const conversationId = await session.commit( + async (tx) => (await tx.createConversation({ ownership: { kind: "ownerless" } })).id, + context, + ); + await session.commit(async (tx) => { + const live = await tx.doc(LiveV1, conversationId); + live.compactions = [{ taskId: 7, reason: "threshold", blocking: true, attempt: 2 }]; + live.compactionFailure = { taskId: 5, reason: "manual", error: "no model" }; + }, context); + // The version 2 token reloads the stored value and migrates it. + const stored = async () => { + const record = await storage.findDocument( + { kind: "pi.live", scope: { kind: "conversation", conversationId } }, + "current", + context, + ); + return (await storage.document(record!.id, "current", context))!; + }; + const migrated: LiveState = { + compactions: [{ taskId: 7 as TaskId, trigger: "threshold", blocking: true, attempt: 2 }], + compactionFailure: { taskId: 5 as TaskId, trigger: "manual", error: "no model" }, + }; + const read = await session.snapshot(LiveDoc, conversationId, context); + expect(read).toEqual(migrated); + expect(Object.keys(read!.compactions![0]!)).toEqual(["taskId", "trigger", "blocking", "attempt"]); + // A read migrates in memory only: Storage keeps the version 1 value. + expect(await stored()).toMatchObject({ + version: 1, + value: { compactions: [{ reason: "threshold" }], compactionFailure: { reason: "manual" } }, + }); + await session.commit(async (tx) => { + (await tx.doc(LiveDoc, conversationId)).compactions![0]!.attempt = 3; + }, context); + expect(await stored()).toMatchObject({ + version: 2, + value: { compactions: [{ trigger: "threshold", attempt: 3 }], compactionFailure: { trigger: "manual" } }, + }); + await session.close(context); + }); +}); diff --git a/packages/durable/test/harness-deferred-progress.test.ts b/packages/durable/test/harness-deferred-progress.test.ts index 0a410a7af..da77f6f36 100644 --- a/packages/durable/test/harness-deferred-progress.test.ts +++ b/packages/durable/test/harness-deferred-progress.test.ts @@ -20,7 +20,7 @@ import type { PartialMessage } from "../src/harness/partial-changes.ts"; import type { SessionImpl } from "../src/session/session.ts"; import { allEntries, type ChatSetup, chatSetup, openChat, textOf, waitFor } from "./chat-support.ts"; import { addTask, addTool } from "./harness-support.ts"; -import { ControlledStorage, context, observeCommits } from "./session-support.ts"; +import { ControlledStorage, context, followCommits } from "./session-support.ts"; import { aborted, deferred } from "./task-support.ts"; /** A slow stream: 4 characters 80 times a second. */ @@ -44,11 +44,12 @@ async function open(storage: ControlledStorage, setup: ChatSetup) { /** The partial text of every `pi.live` revision memory adopted, the last one last. */ function memoryPartials(harness: Harness): string[] { const texts: string[] = []; - observeCommits(harness, (publication) => { + followCommits(harness, (publication) => { for (const change of publication.changes) { - if (change.type !== "document" || change.record.kind !== LiveDoc.definition.kind) continue; + if (change.type !== "document" || change.kind !== "change" || change.record.kind !== LiveDoc.definition.kind) + continue; // Publications carry untyped document values; this one is `pi.live`. - const message = (change.value as LiveState | null)?.generation?.message; + const message = (change.value as LiveState)?.generation?.message; if (message !== undefined) texts.push(partialText(message) ?? ""); } }); @@ -115,14 +116,21 @@ describe("generation partials", () => { const setup = slowSetup(150); setup.faux.setResponses([fauxAssistantMessage(LONG)]); let opened = await open(storage, setup); - // A replica fed by subscribeCommits() alone, as a replication feed would be. + // A replica fed by observeCommits() alone, as a replication feed would be. let replica = await storedLive(storage, opened.root.id); - opened.harness.subscribeCommits((publication) => { - for (const change of publication.changes) { - if (change.type === "document" && change.record.kind === LiveDoc.definition.kind) { - replica = applyImmutable(replica, change.ops); + opened.harness.observeCommits({ + advance: (publication) => { + for (const change of publication.changes) { + if ( + change.type === "document" && + change.kind === "change" && + change.record.kind === LiveDoc.definition.kind + ) { + replica = applyImmutable(replica, change.ops); + } } - } + }, + end: () => {}, }); const submission = await opened.root.submit({ type: "input", content: "hi" }, context); await waitFor(async () => (storedText(await storedLive(storage, opened.root.id))?.length ?? 0) > 40); diff --git a/packages/durable/test/harness-generation-recovery.test.ts b/packages/durable/test/harness-generation-recovery.test.ts index f517bbe12..6e044a9fb 100644 --- a/packages/durable/test/harness-generation-recovery.test.ts +++ b/packages/durable/test/harness-generation-recovery.test.ts @@ -8,7 +8,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, type ChatSetup, chatSetup, openChat, textOf, unanswered, waitFor } from "./chat-support.ts"; import { addSection } from "./harness-support.ts"; -import { context } from "./session-support.ts"; +import { context, onValue } from "./session-support.ts"; import { aborted, deferred, eventually } from "./task-support.ts"; const directories = new Set(); @@ -130,9 +130,11 @@ describe("generation recovery", () => { opened.harness.resume(); const watch = (await opened.harness.watchDoc(LiveDoc, opened.root.id, context))!; let watched: string | undefined; - watch.start(async ({ value }) => { - watched = textOf(value.generation?.message as Message) ?? watched; - }); + watch.start( + onValue(async ({ value }) => { + watched = textOf(value.generation?.message as Message) ?? watched; + }), + ); const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; await waitFor(() => watched !== undefined); await opened.harness.close(context); diff --git a/packages/durable/test/harness-generation.test.ts b/packages/durable/test/harness-generation.test.ts index 28b55fa4f..fa6b0d21e 100644 --- a/packages/durable/test/harness-generation.test.ts +++ b/packages/durable/test/harness-generation.test.ts @@ -30,7 +30,7 @@ import { resolveSettings } from "../src/harness/agent.ts"; import type { SessionImpl } from "../src/session/session.ts"; import { allEntries, type ChatSetup, chatSetup, openChat, textOf, unanswered, waitFor } from "./chat-support.ts"; import { addSection } from "./harness-support.ts"; -import { ControlledStorage, context, observeCommits } from "./session-support.ts"; +import { ControlledStorage, context, followCommits } from "./session-support.ts"; const ERROR_503 = fauxAssistantMessage([], { stopReason: "error", errorMessage: "503 Service Unavailable" }); @@ -75,9 +75,9 @@ function invalidFinalStream(): ReturnType { function livePublications(harness: Harness): LiveState[] { const values: LiveState[] = []; - observeCommits(harness, (publication) => { + followCommits(harness, (publication) => { for (const change of publication.changes) { - if (change.type === "document" && change.record.kind === "pi.live" && change.value !== null) { + if (change.type === "document" && change.record.kind === "pi.live" && change.kind === "change") { values.push(change.value as LiveState); } } diff --git a/packages/durable/test/harness-inbox.test.ts b/packages/durable/test/harness-inbox.test.ts index 396c94e59..a96ab0f6c 100644 --- a/packages/durable/test/harness-inbox.test.ts +++ b/packages/durable/test/harness-inbox.test.ts @@ -35,7 +35,7 @@ import { recordUsage } from "../src/harness/usage.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, type ChatSetup, chatSetup, openChat, textOf, waitFor } from "./chat-support.ts"; import { addHooks, addTool } from "./harness-support.ts"; -import { ControlledStorage, context, documentChanges, flush } from "./session-support.ts"; +import { ControlledStorage, context, documentChanges, documentValues, flush } from "./session-support.ts"; import { aborted, type Deferred, deferred } from "./task-support.ts"; const directories = new Set(); @@ -616,10 +616,13 @@ describe("inbox", () => { setup.faux.setResponses([first.step, answer("second")]); const { harness, root } = await openChat(new MemoryStorage(), setup); const ops: Op[][] = []; - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) { - if (change.record.kind === "pi.inbox" && change.ops.length > 0) ops.push([...change.ops]); - } + harness.observeCommits({ + advance: (publication) => { + for (const change of documentValues(publication)) { + if (change.record.kind === "pi.inbox" && change.ops.length > 0) ops.push([...change.ops]); + } + }, + end: () => {}, }); const input = await root.submit({ type: "input", content: "a" }, context); await first.reached; @@ -873,10 +876,13 @@ describe("inbox", () => { const items = []; for (const text of ["x", "y", "z"]) items.push(await root.submit({ type: "input", content: text }, context)); const ops: Op[][] = []; - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) { - if (change.record.kind === "pi.inbox" && change.ops.length > 0) ops.push([...change.ops]); - } + harness.observeCommits({ + advance: (publication) => { + for (const change of documentValues(publication)) { + if (change.record.kind === "pi.inbox" && change.ops.length > 0) ops.push([...change.ops]); + } + }, + end: () => {}, }); await items[1]!.abort(context); expect(ops).toEqual([[["p", ["items"], 1, 1, []]]]); @@ -909,9 +915,12 @@ describe("inbox", () => { const first = gated(answer("first")); setup.faux.setResponses([first.step, answer("second")]); const { harness, root } = await openChat(new RecordingStorage(), setup); - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) - if (change.record.kind === "pi.inbox") inboxId = change.record.id; + harness.observeCommits({ + advance: (publication) => { + for (const change of documentChanges(publication)) + if (change.record.kind === "pi.inbox") inboxId = change.record.id; + }, + end: () => {}, }); const input = await root.submit({ type: "input", content: "a" }, context); await first.reached; @@ -1074,12 +1083,15 @@ describe("usage", () => { setup.faux.setResponses([answer("one"), answer("two")]); const { harness, root } = await openChat(new MemoryStorage(), setup); const commits: { entries: number; ops: readonly Op[] }[] = []; - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) { - if (change.record.kind !== "pi.usage") continue; - const entries = publication.changes.filter((other) => other.type === "entry").length; - commits.push({ entries, ops: change.ops }); - } + harness.observeCommits({ + advance: (publication) => { + for (const change of documentValues(publication)) { + if (change.record.kind !== "pi.usage") continue; + const entries = publication.changes.filter((other) => other.type === "entry").length; + commits.push({ entries, ops: change.ops }); + } + }, + end: () => {}, }); await (await root.submit({ type: "input", content: "a" }, context)).wait(context); await (await root.submit({ type: "input", content: "b" }, context)).wait(context); diff --git a/packages/durable/test/harness-inspect.test.ts b/packages/durable/test/harness-inspect.test.ts index 78aa8f20d..bec664694 100644 --- a/packages/durable/test/harness-inspect.test.ts +++ b/packages/durable/test/harness-inspect.test.ts @@ -94,11 +94,11 @@ describe("Harness.inspect()", () => { const paused = await harness.inspect(context); expect(paused.scheduling).toBe("paused"); expect(paused.tasks.map((entry) => entry.record.id)).toEqual(Object.values(ids)); - expect(stateOf(paused.tasks, ids.gate)).toEqual({ kind: "ready", migrates: false }); - expect(stateOf(paused.tasks, ids.dependent)).toEqual({ kind: "ready", migrates: false }); - expect(stateOf(paused.tasks, ids.migrating)).toEqual({ kind: "ready", migrates: true }); + expect(stateOf(paused.tasks, ids.gate)).toEqual({ kind: "pending", migrates: false }); + expect(stateOf(paused.tasks, ids.dependent)).toEqual({ kind: "pending", migrates: false }); + expect(stateOf(paused.tasks, ids.migrating)).toEqual({ kind: "pending", migrates: true }); // A migration that was never tried is not run to find out. - expect(stateOf(paused.tasks, ids.failing)).toEqual({ kind: "ready", migrates: true }); + expect(stateOf(paused.tasks, ids.failing)).toEqual({ kind: "pending", migrates: true }); expect(stateOf(paused.tasks, ids.noMigration)).toMatchObject({ kind: "blocked", reason: "migration_failed", diff --git a/packages/durable/test/harness-lifecycle.test.ts b/packages/durable/test/harness-lifecycle.test.ts index be422e526..0c0049599 100644 --- a/packages/durable/test/harness-lifecycle.test.ts +++ b/packages/durable/test/harness-lifecycle.test.ts @@ -117,7 +117,7 @@ describe("Harness open", () => { const reports: unknown[] = []; const opening = Harness.open( storage, - { models: createModels(), registry: createRegistry(), onReport: (error) => reports.push(error) }, + { models: createModels(), registry: createRegistry(), onReport: ({ error }) => reports.push(error) }, context, ); await expect(opening).rejects.toThrow("disk full"); @@ -318,9 +318,9 @@ describe("Harness close", () => { expect(docWatch.value).toBe(docValue); expect(viewWatch.value).toBe(viewValue); expect(graphWatch.value).toEqual({ tasks: {} }); - expect(await graphWatch.closed).toEqual({ reason: "ended" }); - expect(await docWatch.closed).toEqual({ reason: "ended" }); - expect(await viewWatch.closed).toEqual({ reason: "ended" }); + expect(await graphWatch.closed).toEqual({ kind: "ended" }); + expect(await docWatch.closed).toEqual({ kind: "ended" }); + expect(await viewWatch.closed).toEqual({ kind: "ended" }); // The commit itself settled. expect(storage.commits.at(-1)).toContainEqual(expect.objectContaining({ type: "entry" })); }); @@ -341,8 +341,9 @@ describe("Harness close", () => { }, context); // Count commit subscriptions of document observers. let subscriptions = 0; - const subscribe = harness.subscribeCommits.bind(harness); - harness.subscribeCommits = (listener) => { + const session = harness as unknown as SessionImpl; + const subscribe = session.followCommits.bind(session); + session.followCommits = (listener) => { subscriptions++; const unsubscribe = subscribe(listener); let active = true; @@ -586,7 +587,7 @@ describe("registry changes before resume", () => { { state: { kind: "blocked", reason: "missing_task" } }, ]); const installed = addTask(first.registry, define("v1")); - expect((await first.harness.inspect(context)).tasks).toMatchObject([{ state: { kind: "ready" } }]); + expect((await first.harness.inspect(context)).tasks).toMatchObject([{ state: { kind: "pending" } }]); // The same extension name replaces the definition in place, still before resume. installed.dispose(); addTask(first.registry, define("v2")); diff --git a/packages/durable/test/harness-live-deltas.test.ts b/packages/durable/test/harness-live-deltas.test.ts index 26db00d26..0802a324e 100644 --- a/packages/durable/test/harness-live-deltas.test.ts +++ b/packages/durable/test/harness-live-deltas.test.ts @@ -13,7 +13,7 @@ import { import { describe, expect, it } from "vitest"; import { chatSetup, openChat, waitFor } from "./chat-support.ts"; import { addTool } from "./harness-support.ts"; -import { context, documentChanges, observeCommits } from "./session-support.ts"; +import { context, documentChanges, documentValues, followCommits } from "./session-support.ts"; type Action = (api: ToolExecutionApi) => void | Promise; @@ -52,8 +52,8 @@ async function drive(outputLimits: ToolRegistration["outputLimits"] = {}) { ]); const { harness, root } = await openChat(new MemoryStorage(), setup); const commits: Op[][] = []; - observeCommits(harness, (publication) => { - for (const change of documentChanges(publication)) { + followCommits(harness, (publication) => { + for (const change of documentValues(publication)) { if (change.record.kind === "pi.live" && change.ops.length > 0) commits.push([...change.ops]); } }); @@ -114,10 +114,13 @@ describe("pi.live deltas", () => { ]); const { harness, root } = await openChat(new MemoryStorage(), setup); const commits: Op[][] = []; - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) { - if (change.record.kind === "pi.live" && change.ops.length > 0) commits.push([...change.ops]); - } + harness.observeCommits({ + advance: (publication) => { + for (const change of documentValues(publication)) { + if (change.record.kind === "pi.live" && change.ops.length > 0) commits.push([...change.ops]); + } + }, + end: () => {}, }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); const tasks = await harness.commit((tx) => tx.scanTasks({ conversationId: root.id }, 20), context); @@ -261,8 +264,8 @@ describe("pi.live deltas", () => { setup.faux.setResponses([fauxAssistantMessage([fauxText("word ".repeat(250))])]); const { harness, root } = await openChat(new MemoryStorage(), setup); const commits: Op[][] = []; - observeCommits(harness, (publication) => { - for (const change of documentChanges(publication)) { + followCommits(harness, (publication) => { + for (const change of documentValues(publication)) { if (change.record.kind === "pi.live" && change.ops.length > 0) commits.push([...change.ops]); } }); @@ -321,12 +324,15 @@ describe("pi.live deltas", () => { setup.settings.toolExecution = "sequential"; const { harness, root } = await openChat(new RecordingStorage(), setup); const values = new Map(); - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) { - if (change.record.kind !== "pi.live") continue; - liveId = change.record.id; - if (change.value !== null) values.set(publication.seq, change.value as LiveState); - } + harness.observeCommits({ + advance: (publication) => { + for (const change of documentChanges(publication)) { + if (change.record.kind !== "pi.live") continue; + liveId = change.record.id; + if (change.kind === "change") values.set(publication.seq, change.value as LiveState); + } + }, + end: () => {}, }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); const nothingRuns = (value: LiveState) => @@ -361,10 +367,13 @@ describe("pi.live deltas", () => { ]); const { harness, root } = await openChat(new MemoryStorage(), setup); const commits: Op[][] = []; - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) { - if (change.record.kind === "pi.live" && change.ops.length > 0) commits.push([...change.ops]); - } + harness.observeCommits({ + advance: (publication) => { + for (const change of documentValues(publication)) { + if (change.record.kind === "pi.live" && change.ops.length > 0) commits.push([...change.ops]); + } + }, + end: () => {}, }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); const entries = await root.entries({}, 10, undefined, context); @@ -402,12 +411,15 @@ describe("pi.live deltas", () => { ]); const { harness, root } = await openChat(new MemoryStorage(), setup); let handover: CommitPublication | undefined; - harness.subscribeCommits((publication) => { - for (const change of documentChanges(publication)) { - if (change.record.kind === "pi.live" && (change.value as LiveState | null)?.tools?.length === 2) { - handover ??= publication; + harness.observeCommits({ + advance: (publication) => { + for (const change of documentValues(publication)) { + if (change.record.kind === "pi.live" && (change.value as LiveState)?.tools?.length === 2) { + handover ??= publication; + } } - } + }, + end: () => {}, }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); const changes = handover!.changes; diff --git a/packages/durable/test/harness-nested-tools-restart.test.ts b/packages/durable/test/harness-nested-tools-restart.test.ts index 44c508f70..0eb29ff76 100644 --- a/packages/durable/test/harness-nested-tools-restart.test.ts +++ b/packages/durable/test/harness-nested-tools-restart.test.ts @@ -487,15 +487,23 @@ describe("nested tool calls across a restart", () => { opened = await open(path, setup); let slotsAfterCaller: number | undefined; - opened.harness.subscribeCommits((publication) => { - for (const change of publication.changes) { - if (change.type !== "task" || change.value.kind !== ToolTask.definition.name) continue; - if ((change.value.input as ToolTaskInput).kind !== "model" || change.value.state.status !== "terminal") - continue; - const live = publication.changes.find((each) => each.type === "document" && each.record.kind === "pi.live"); - const value = live?.type === "document" ? (live.value as { nestedTools?: unknown[] } | null) : undefined; - slotsAfterCaller = value?.nestedTools?.length ?? 0; - } + opened.harness.observeCommits({ + advance: (publication) => { + for (const change of publication.changes) { + if (change.type !== "task" || change.value.kind !== ToolTask.definition.name) continue; + if ((change.value.input as ToolTaskInput).kind !== "model" || change.value.state.status !== "terminal") + continue; + const live = publication.changes.find( + (each) => each.type === "document" && each.record.kind === "pi.live", + ); + const value = + live?.type === "document" && live.kind === "change" + ? (live.value as { nestedTools?: unknown[] }) + : undefined; + slotsAfterCaller = value?.nestedTools?.length ?? 0; + } + }, + end: () => {}, }); opened.harness.resume(); expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); diff --git a/packages/durable/test/harness-nested-tools.test.ts b/packages/durable/test/harness-nested-tools.test.ts index 2db53ada5..de61c3ec4 100644 --- a/packages/durable/test/harness-nested-tools.test.ts +++ b/packages/durable/test/harness-nested-tools.test.ts @@ -38,7 +38,7 @@ import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; import { addHooks, addTool } from "./harness-support.ts"; import { endedAt, nestedResultOf, nestedResults, slotsOf, startOrder, viewHistory } from "./live-support.ts"; -import { context } from "./session-support.ts"; +import { context, onValue } from "./session-support.ts"; import { aborted, deferred, eventually } from "./task-support.ts"; const Parameters = Type.Object({ text: Type.Optional(Type.String()) }); @@ -870,12 +870,14 @@ describe("nested tool calls", () => { // Each value's operations rebuild the next one, the late subscriber's starting value included. let replayed = watch.value; const outputs: [string | undefined, string | undefined][] = []; - watch.start(async ({ value, ops }) => { - replayed = applyImmutable(replayed, ops); - expect(replayed).toEqual(value); - const [one, two] = nestedOf(value); - outputs.push([one?.output, two?.output]); - }); + watch.start( + onValue(async ({ value, ops }) => { + replayed = applyImmutable(replayed, ops); + expect(replayed).toEqual(value); + const [one, two] = nestedOf(value); + outputs.push([one?.output, two?.output]); + }), + ); // The second slot writes first, so a wrong slot index in the update path would show. wrote.two.resolve(); await eventually(() => outputs.at(-1)?.[1] === "two\n"); diff --git a/packages/durable/test/harness-observers.test.ts b/packages/durable/test/harness-observers.test.ts index c99e1aa3c..fdd6d7787 100644 --- a/packages/durable/test/harness-observers.test.ts +++ b/packages/durable/test/harness-observers.test.ts @@ -2,7 +2,7 @@ // document) and for task document watches, which end with their task. import { setFlagsFromString } from "node:v8"; import { runInNewContext } from "node:vm"; -import type { WatchChange } from "@earendil-works/chord"; +import type { WatchCall } from "@earendil-works/chord"; import { withAbortSignal } from "@earendil-works/chord/context"; import { type ConversationView, @@ -15,7 +15,7 @@ import { } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; import type { SessionImpl } from "../src/session/session.ts"; -import { ControlledStorage, context, flush } from "./session-support.ts"; +import { ControlledStorage, context, flush, onValue } from "./session-support.ts"; import { completed, deferred, eventually, openTasks, settled } from "./task-support.ts"; type Step = { phase: "run" }; @@ -48,16 +48,19 @@ function writer(name: string, gate: Promise, written: () => void) { }); } +type WatchValue = Extract, { readonly kind: "value" }>; + /** An observer recording each call and whether it ran inside the storage's `adopt`. */ function recorder(storage: ControlledStorage) { - const calls: Array<{ readonly advance: WatchChange; readonly adopting: boolean } | { readonly end: unknown }> = - []; + const calls: Array<{ readonly advance: WatchValue; readonly adopting: boolean } | { readonly end: unknown }> = []; const observer: Observer = { - advance: (change) => { - calls.push({ advance: change, adopting: storage.adopting }); + advance: (call) => { + // Durable never sends a failure call. + if (call.kind !== "value") throw new Error("Unexpected failure call"); + calls.push({ advance: call, adopting: storage.adopting }); }, - end: (failure) => { - calls.push({ end: failure }); + end: (end) => { + calls.push({ end }); }, }; return { calls, observer }; @@ -103,13 +106,13 @@ describe("observeConversation and observeTaskGraph", () => { expect(viewObservation.value.conversation.id).toBe(root.id); expect(graphObservation.value).toEqual({ tasks: {} }); const watch = await root.watch(context); - const watched: WatchChange[] = []; + const watched: WatchCall[] = []; watch.start(async (change) => { watched.push(change); }); const entry = await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); expect(view.calls).toHaveLength(1); - const [call] = view.calls as [{ advance: WatchChange; adopting: boolean }]; + const [call] = view.calls as [{ advance: WatchValue; adopting: boolean }]; expect(call.adopting).toBe(true); expect(call.advance.retired).toBe(false); expect(call.advance.value.entries.at(-1)?.id).toBe(entry.id); @@ -122,7 +125,7 @@ describe("observeConversation and observeTaskGraph", () => { expect(view.calls).toHaveLength(1); await watch.stop(); await harness.close(context); - expect(graph.calls).toEqual([{ end: undefined }]); + expect(graph.calls).toEqual([{ end: { kind: "ended" } }]); }); it("end with the error that failed the Harness", async () => { @@ -137,7 +140,7 @@ describe("observeConversation and observeTaskGraph", () => { "disk gone", ); await harness.closed; - expect(graph.calls).toEqual([{ end: { error: failure } }]); + expect(graph.calls).toEqual([{ end: { kind: "failed", error: failure } }]); }); it("report a throwing observer, detach it and end it with the error, and keep the view going", async () => { @@ -152,7 +155,7 @@ describe("observeConversation and observeTaskGraph", () => { advance: () => { throw bug; }, - end: (failure) => ends.push(failure), + end: (end) => ends.push(end), }, context, ); @@ -161,12 +164,12 @@ describe("observeConversation and observeTaskGraph", () => { await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); expect(reports).toEqual([bug]); - expect(infos).toEqual([{ type: "listener_error" }]); - expect(ends).toEqual([{ error: bug }]); + expect(infos).toEqual([{ kind: "listener_error" }]); + expect(ends).toEqual([{ kind: "listener_error", error: bug }]); expect(healthy.calls).toHaveLength(2); await harness.close(context); expect(ends).toHaveLength(1); - expect(await harness.closed).toEqual({ reason: "closed" }); + expect(await harness.closed).toEqual({ kind: "closed" }); }); it("reject, never throw, on a closed Harness", async () => { @@ -227,6 +230,7 @@ describe("observeConversation and observeTaskGraph", () => { expect(graph.calls).toEqual([ { advance: { + kind: "value", value: { tasks: { [key]: expect.objectContaining({ blocked: "missing_task" }) } }, ops: [["s", ["tasks", key], expect.objectContaining({ blocked: "missing_task" })]], retired: false, @@ -238,10 +242,10 @@ describe("observeConversation and observeTaskGraph", () => { await eventually(() => graph.calls.length === 2); // The registry change re-derives the reason on the line, with no commit and outside the storage's `adopt`. expect(graph.calls[1]).toEqual({ - advance: { value: expect.anything(), ops: [["d", ["tasks", key, "blocked"]]], retired: false }, + advance: { kind: "value", value: expect.anything(), ops: [["d", ["tasks", key, "blocked"]]], retired: false }, adopting: false, }); - expect((graph.calls[1] as { advance: WatchChange }).advance.value.tasks[key]?.blocked).toBeUndefined(); + expect((graph.calls[1] as { advance: WatchValue }).advance.value.tasks[key]?.blocked).toBeUndefined(); harness.resume(); await harness.waitForTask(id, context); observation.detach(); @@ -283,15 +287,15 @@ describe("task documents", () => { const { harness, id, finish } = await openWriter(); const watch = (await harness.watchDoc(Progress, id, context))!; expect(watch.value).toEqual({ lines: ["started"] }); - const changes: WatchChange>[] = []; + const changes: WatchCall>[] = []; watch.start(async (change) => { changes.push(change); }); finish(); await harness.waitForTask(id, context); - expect(await watch.closed).toEqual({ reason: "ended" }); - expect(changes).toEqual([{ value: { lines: ["started"] }, ops: [], retired: true }]); - expect(changes[0]!.value).toBe(watch.value); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(changes).toEqual([{ kind: "value", value: { lines: ["started"] }, ops: [], retired: true }]); + expect((changes[0] as { value: unknown }).value).toBe(watch.value); // The task is terminal: its address is absent for good. expect(await harness.watchDoc(Progress, id, context)).toBeUndefined(); await harness.close(context); @@ -303,17 +307,17 @@ describe("task documents", () => { let endedAdopting: boolean | undefined; const observer: Observer> = { advance: progress.observer.advance, - end: (failure) => { + end: (end) => { endedAdopting = storage.adopting; - progress.observer.end(failure); + progress.observer.end(end); }, }; await harness.observeDoc(Progress, id, observer, context); finish(); await harness.waitForTask(id, context); expect(progress.calls).toEqual([ - { advance: { value: { lines: ["started"] }, ops: [], retired: true }, adopting: true }, - { end: undefined }, + { advance: { kind: "value", value: { lines: ["started"] }, ops: [], retired: true }, adopting: true }, + { end: { kind: "ended" } }, ]); expect(endedAdopting).toBe(true); await harness.close(context); @@ -356,7 +360,7 @@ describe("task documents", () => { }; const { watch, harness } = await acquire(); // Closed at the Harness's close, the pending retirement dropped, as for every watch. - expect(await watch.closed).toEqual({ reason: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); expect(() => watch.start(async () => {})).toThrow("Watch is stopped"); await collect(); expect(harness.deref()).toBeUndefined(); @@ -372,7 +376,7 @@ describe("task documents", () => { await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context)).rejects.toThrow( "disk gone", ); - expect(await watch.closed).toEqual({ reason: "failed", error: failure }); + expect(await watch.closed).toEqual({ kind: "failed", error: failure }); }); it("a watch busy in an earlier call when its task ends fails with the Harness before its retirement call", async () => { @@ -381,11 +385,13 @@ describe("task documents", () => { const entered = deferred(); const release = deferred(); const values: unknown[] = []; - watch.start(async ({ value }) => { - values.push(value); - entered.resolve(); - await release.promise; - }); + watch.start( + onValue(async ({ value }) => { + values.push(value); + entered.resolve(); + await release.promise; + }), + ); await harness.commit(async (tx) => { (await tx.doc(Progress, id)).lines.push("more"); }, context); @@ -398,7 +404,7 @@ describe("task documents", () => { await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context)).rejects.toThrow( "disk gone", ); - expect(await watch.closed).toEqual({ reason: "failed", error: failure }); + expect(await watch.closed).toEqual({ kind: "failed", error: failure }); release.resolve(); await flush(); expect(values).toEqual([{ lines: ["started", "more"] }]); @@ -423,7 +429,7 @@ describe("task documents", () => { await flush(); expect(await settled(watch.closed)).toBe(false); release.resolve(); - expect(await watch.closed).toEqual({ reason: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); expect(returned).toBe(true); await harness.close(context); }); @@ -433,7 +439,7 @@ describe("task documents", () => { const acquire = async () => { const { harness, id, finish } = await openWriter(); const watch = (await harness.watchDoc(Progress, id, context))!; - const changes: WatchChange>[] = []; + const changes: WatchCall>[] = []; watch.start(async (change) => { changes.push(change); }); @@ -443,14 +449,14 @@ describe("task documents", () => { await flush(); finish(); await harness.waitForTask(id, context); - expect(await watch.closed).toEqual({ reason: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); await harness.close(context); return { changes, harness: new WeakRef(harness) }; }; const { changes, harness } = await acquire(); expect(changes).toHaveLength(2); for (const change of changes) { - expect(Object.keys(change)).toEqual(["value", "ops", "retired"]); + expect(Object.keys(change)).toEqual(["kind", "value", "ops", "retired"]); expect(structuredClone(change)).toEqual(change); } await collect(); diff --git a/packages/durable/test/harness-ownership.test.ts b/packages/durable/test/harness-ownership.test.ts index d03806c9e..e7e796133 100644 --- a/packages/durable/test/harness-ownership.test.ts +++ b/packages/durable/test/harness-ownership.test.ts @@ -959,7 +959,7 @@ describe("ownership", () => { // The failed cascade commit fails the Harness. `closed` settles once its invocations have ended, and the owner's // abort handler ignores its signal: let it return. open("abort.owner", "completed"); - expect(await opened.harness.closed).toMatchObject({ reason: "failed" }); + expect(await opened.harness.closed).toMatchObject({ kind: "failed" }); await opened.harness.close(context); // Reopening derives the mark again. opened = await openHarness(await openNodeSqliteStorage(path)); @@ -991,7 +991,7 @@ describe("ownership", () => { const tree = await ownedChild(root, "owner"); failMark = tree.inner; open("owner", "failed"); - expect(await harness.closed).toEqual({ reason: "failed", error: failure }); + expect(await harness.closed).toEqual({ kind: "failed", error: failure }); expect(reports).toEqual([failure]); await harness.close(context); const reopened = await openHarness(storage.reopen()); diff --git a/packages/durable/test/harness-read-only.test.ts b/packages/durable/test/harness-read-only.test.ts index da6539789..b45b14b0f 100644 --- a/packages/durable/test/harness-read-only.test.ts +++ b/packages/durable/test/harness-read-only.test.ts @@ -30,7 +30,7 @@ async function openReadOnly(path: string, setup: ChatSetup, onWake?: (wake: Wake models: setup.models, registry: setup.registry, readOnly: true, - onReport: (error) => setup.reports.push(error), + onReport: ({ error }) => setup.reports.push(error), ...(onWake === undefined ? {} : { onWake }), }, context, diff --git a/packages/durable/test/harness-reports.test.ts b/packages/durable/test/harness-reports.test.ts index 0bc2304cd..623fafcdb 100644 --- a/packages/durable/test/harness-reports.test.ts +++ b/packages/durable/test/harness-reports.test.ts @@ -34,7 +34,7 @@ async function open( { models: createModels(), registry, - onReport: (error, info) => reports.push({ message: (error as Error).message, info }), + onReport: ({ error, ...info }) => reports.push({ message: (error as Error).message, info }), ...(options.onWake === undefined ? {} : { onWake: options.onWake }), ...(options.setTimer === undefined ? {} : { setTimer: options.setTimer }), }, @@ -69,8 +69,8 @@ describe("reports", () => { // The task's own failures name the extension that installed it. const task = { task: { id, kind: "test.faulty", conversationId: root.id }, extension: "test" }; expect(reports).toEqual([ - { message: "noted", info: { type: "task_error", ...task } }, - { message: "broken", info: { type: "task_faulted", ...task, reason: "threw" } }, + { message: "noted", info: { kind: "task_error", ...task } }, + { message: "broken", info: { kind: "task_faulted", ...task, reason: "threw" } }, ]); await harness.close(context); }); @@ -104,7 +104,7 @@ describe("reports", () => { registry.install(defineExtension({ name: "new-extension", tasks: [definition("new")] })); gate.resolve(); await harness.waitForTask(id, context); - const faulted = (report: Report) => report.info.type === "task_faulted"; + const faulted = (report: Report) => report.info.kind === "task_faulted"; await eventually(() => reports.some(faulted)); expect(reports.find(faulted)).toMatchObject({ message: "broken in old", @@ -147,7 +147,7 @@ describe("reports", () => { env: () => { throw new Error("env"); }, - onReport: (error, info) => reports.push({ message: (error as Error).message, info }), + onReport: ({ error, ...info }) => reports.push({ message: (error as Error).message, info }), }, context, ); @@ -155,9 +155,9 @@ describe("reports", () => { await (await root.submit({ type: "input", content: "hi" }, context)).wait(context); const infoOf = (message: string) => reports.find((report) => report.message === message)?.info; const generation = { kind: "pi.generation", conversationId: root.id }; - expect(infoOf("env")).toMatchObject({ type: "host_error", callback: "env", task: generation }); - expect(infoOf("section")).toMatchObject({ type: "extension_error", extension: "sections", task: generation }); - expect(infoOf("hook")).toMatchObject({ type: "extension_error", extension: "hooks", task: generation }); + expect(infoOf("env")).toMatchObject({ kind: "host_error", callback: "env", task: generation }); + expect(infoOf("section")).toMatchObject({ kind: "extension_error", extension: "sections", task: generation }); + expect(infoOf("hook")).toMatchObject({ kind: "extension_error", extension: "hooks", task: generation }); await harness.close(context); }); @@ -196,8 +196,11 @@ describe("reports", () => { registry.install( defineExtension({ name: "wraps", wraps: [wrapTool(echo, (tool) => ({ ...tool, name: "renamed" }))] }), ); - harness.subscribeCommits(() => { - throw new Error("listener"); + harness.observeCommits({ + advance: () => { + throw new Error("listener"); + }, + end: () => {}, }); const root = await harness.root(context); await root.agent(context); @@ -208,18 +211,18 @@ describe("reports", () => { ); const infoOf = (message: string) => reports.find((report) => report.message === message)?.info; expect(infoOf("Wrapper renamed echo to renamed")).toEqual({ - type: "extension_error", + kind: "extension_error", conversationId: root.id, extension: "wraps", }); - expect(infoOf("listener")).toEqual({ type: "listener_error" }); - expect(infoOf("alarm")).toEqual({ type: "host_error", callback: "onWake" }); - expect(infoOf("no timers")).toEqual({ type: "host_error", callback: "setTimer" }); + expect(infoOf("listener")).toEqual({ kind: "listener_error" }); + expect(infoOf("alarm")).toEqual({ kind: "host_error", callback: "onWake" }); + expect(infoOf("no timers")).toEqual({ kind: "host_error", callback: "setTimer" }); const failure = new Error("disk gone"); storage.failNextCommit(failure); await root.commit((tx) => tx.createConversation({ ownership: { kind: "ownerless" } }), context).catch(() => {}); await harness.closed; - expect(infoOf("disk gone")).toEqual({ type: "session_failed" }); + expect(infoOf("disk gone")).toEqual({ kind: "storage_failed" }); }); }); diff --git a/packages/durable/test/harness-restart-failure.test.ts b/packages/durable/test/harness-restart-failure.test.ts index 9bc220a1e..f0552a953 100644 --- a/packages/durable/test/harness-restart-failure.test.ts +++ b/packages/durable/test/harness-restart-failure.test.ts @@ -31,7 +31,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, type ChatSetup, chatSetup, openChat, textOf, unanswered, waitFor } from "./chat-support.ts"; import { addSection, addTool } from "./harness-support.ts"; -import { context, observeCommits } from "./session-support.ts"; +import { context, followCommits } from "./session-support.ts"; import { aborted, deferred } from "./task-support.ts"; const directories = new Set(); @@ -80,9 +80,9 @@ async function live(harness: Harness): Promise> { /** Every `pi.live` value from now on, as memory adopts it. */ function liveValues(harness: Harness): LiveState[] { const values: LiveState[] = []; - observeCommits(harness, (publication) => { + followCommits(harness, (publication) => { for (const change of publication.changes) { - if (change.type === "document" && change.record.kind === "pi.live" && change.value !== null) { + if (change.type === "document" && change.record.kind === "pi.live" && change.kind === "change") { values.push(change.value as LiveState); } } @@ -355,7 +355,7 @@ describe("a generation interrupted while the model answers (#11139)", () => { }); await waitFor(() => setup.infos.length > 0); expect(setup.infos).toContainEqual( - expect.objectContaining({ type: "task_interrupted", reason: "restart_limit", restarts: 1 }), + expect.objectContaining({ kind: "task_interrupted", reason: "restart_limit", restarts: 1 }), ); await opened.harness.close(context); }); @@ -425,7 +425,7 @@ describe("other tasks cut off restartLimit times in a row (#11139)", () => { outcome: { status: "failed", error: { detail: { reason: "restart_limit" } } }, }); expect(setup.infos).toContainEqual({ - type: "task_interrupted", + kind: "task_interrupted", task: { id: toolTask.id, kind: "pi.tool", conversationId: ROOT_CONVERSATION_ID }, reason: "restart_limit", restarts: 2, @@ -460,13 +460,13 @@ describe("other tasks cut off restartLimit times in a row (#11139)", () => { }); expect((await live(opened.harness)).compactionFailure).toEqual({ taskId: compaction, - reason: "manual", + trigger: "manual", error, }); expect((await live(opened.harness)).compactions).toBeUndefined(); await waitFor(() => setup.infos.length > 0); expect(setup.infos).toContainEqual({ - type: "task_interrupted", + kind: "task_interrupted", task: { id: compaction, kind: "pi.compaction", conversationId: ROOT_CONVERSATION_ID }, reason: "restart_limit", restarts: 1, @@ -698,8 +698,8 @@ describe("crash edges (#11139)", () => { status: "faulted", error: { detail: { reason: "restart_limit" } }, }); - await waitFor(() => setup.infos.some((info) => info.type === "task_faulted")); - expect(setup.infos).toContainEqual(expect.objectContaining({ type: "task_faulted", reason: "restart_limit" })); + await waitFor(() => setup.infos.some((info) => info.kind === "task_faulted")); + expect(setup.infos).toContainEqual(expect.objectContaining({ kind: "task_faulted", reason: "restart_limit" })); await opened.harness.close(context); }); }); @@ -718,7 +718,7 @@ describe("reports of work crashes ended (#11139)", () => { await waitFor(() => setup.infos.length > 0); expect(setup.infos).toEqual([ { - type: "task_interrupted", + kind: "task_interrupted", task: { id: taskId, kind: "pi.generation", conversationId: ROOT_CONVERSATION_ID }, reason: "interrupted", restarts: 1, diff --git a/packages/durable/test/harness-storage-failure.test.ts b/packages/durable/test/harness-storage-failure.test.ts index 12b08a476..4e066b77f 100644 --- a/packages/durable/test/harness-storage-failure.test.ts +++ b/packages/durable/test/harness-storage-failure.test.ts @@ -16,7 +16,7 @@ import { describe, expect, it } from "vitest"; import { safeNow } from "../src/harness/harness.ts"; import { conversationViews } from "../src/harness/view.ts"; import { contained } from "../src/session/session.ts"; -import { ControlledStorage, context } from "./session-support.ts"; +import { ControlledStorage, context, onValue } from "./session-support.ts"; import { aborted, deferred, openTasks } from "./task-support.ts"; /** Waits for its signal, records that it saw it, and ends `aborted` through its abort handler. */ @@ -105,13 +105,13 @@ describe("a failed Storage call", () => { const failed = { name: "SessionFailed", cause: failure }; await expect(task).rejects.toMatchObject(failed); await expect(idle).rejects.toMatchObject(failed); - expect(await watch.closed).toEqual({ reason: "failed", error: failure }); - expect(await view.closed).toEqual({ reason: "failed", error: failure }); + expect(await watch.closed).toEqual({ kind: "failed", error: failure }); + expect(await view.closed).toEqual({ kind: "failed", error: failure }); // Every later call, and a wait that starts now. await expect(harness.getTask(id, context)).rejects.toMatchObject(failed); await expect(harness.waitForIdle(context)).rejects.toMatchObject(failed); await expect(root.submit({ type: "input", content: "x" }, context)).rejects.toMatchObject(failed); - expect(await harness.closed).toEqual({ reason: "failed", error: failure }); + expect(await harness.closed).toEqual({ kind: "failed", error: failure }); expect(signals).toEqual(["aborted"]); expect(reports).toEqual([failure]); await harness.close(context); @@ -123,7 +123,7 @@ describe("a failed Storage call", () => { await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); storage.failing.add("scanRange"); await expect(root.context(context)).rejects.toBe(storage.error); - expect(await harness.closed).toEqual({ reason: "failed", error: storage.error }); + expect(await harness.closed).toEqual({ kind: "failed", error: storage.error }); expect(reports).toEqual([storage.error]); await harness.close(context); }); @@ -148,7 +148,7 @@ describe("a failed Storage call", () => { // The read succeeds at the backend, yet its caller gets the failure: nothing it read is used. release.resolve(); await expect(reading).rejects.toMatchObject({ name: "SessionFailed", cause: storage.error }); - expect(await harness.closed).toEqual({ reason: "failed", error: storage.error }); + expect(await harness.closed).toEqual({ kind: "failed", error: storage.error }); await harness.close(context); }); @@ -185,7 +185,7 @@ describe("a failed Storage call", () => { await expect(reading).rejects.toThrow("is closed"); await closing; expect(pages).toEqual(["first", "close"]); - expect(await harness.closed).toEqual({ reason: "closed" }); + expect(await harness.closed).toEqual({ kind: "closed" }); }); it("fails it when Storage cannot close; an earlier failure stays the cause", async () => { @@ -198,7 +198,7 @@ describe("a failed Storage call", () => { } const healthy = await started(new FailingClose()); await expect(healthy.harness.close(context)).rejects.toBe(closeError); - expect(await healthy.harness.closed).toEqual({ reason: "failed", error: closeError }); + expect(await healthy.harness.closed).toEqual({ kind: "failed", error: closeError }); expect(healthy.reports).toEqual([closeError]); const storage = new FailingClose(); @@ -208,7 +208,7 @@ describe("a failed Storage call", () => { await expect(failed.root.commit((tx) => tx.appendEntry(failed.root.id, { kind: "note" }), context)).rejects.toBe( failure, ); - expect(await failed.harness.closed).toEqual({ reason: "failed", error: failure }); + expect(await failed.harness.closed).toEqual({ kind: "failed", error: failure }); expect(failed.reports).toEqual([failure]); }); @@ -259,7 +259,7 @@ describe("a failed Storage call", () => { storage.failNextCommit(failure); await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).rejects.toBe(failure); await opened.harness.closed; - expect(ends).toEqual([{ reason: "failed", error: failure }, "SessionFailed"]); + expect(ends).toEqual([{ kind: "failed", error: failure }, "SessionFailed"]); await opened.harness.close(context); }); @@ -268,7 +268,7 @@ describe("a failed Storage call", () => { const { harness, root } = await started(storage); storage.failing.add("mintId"); await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).rejects.toBe(storage.error); - expect(await harness.closed).toEqual({ reason: "failed", error: storage.error }); + expect(await harness.closed).toEqual({ kind: "failed", error: storage.error }); await harness.close(context); }); @@ -293,7 +293,7 @@ describe("a failed Storage call", () => { const rejection = new StorageRequestError("bad batch"); storage.failNextCommit(rejection); await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).rejects.toBe(rejection); - expect(await harness.closed).toEqual({ reason: "failed", error: rejection }); + expect(await harness.closed).toEqual({ kind: "failed", error: rejection }); await harness.close(context); }); @@ -328,12 +328,12 @@ describe("a failed Storage call", () => { const storage = new ControlledStorage(); const { harness, root, reports } = await started(storage); const bug = new Error("listener bug"); - (harness as unknown as { observeCommits(listener: () => void): void }).observeCommits(() => { + (harness as unknown as { followCommits(listener: () => void): void }).followCommits(() => { throw bug; }); // The commit itself is durable and resolves; the Session fails behind it. await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).resolves.toBeDefined(); - expect(await harness.closed).toEqual({ reason: "failed", error: bug }); + expect(await harness.closed).toEqual({ kind: "failed", error: bug }); expect(reports).toEqual([bug]); await harness.close(context); }); @@ -350,7 +350,7 @@ describe("a failed Storage call", () => { held.release(); await expect(committing).rejects.toBe(failure); await closing; - expect(await harness.closed).toEqual({ reason: "failed", error: failure }); + expect(await harness.closed).toEqual({ kind: "failed", error: failure }); expect(reports).toEqual([failure]); }); @@ -436,7 +436,7 @@ describe("listener and report errors", () => { const asyncThrow = async () => { throw new Error("async listener"); }; - harness.subscribeCommits(asyncThrow); + harness.observeCommits({ advance: asyncThrow, end: () => {} }); harness.subscribeClose(asyncThrow); await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); await harness.close(context); @@ -449,8 +449,11 @@ describe("listener and report errors", () => { { models: createModels(), registry, onReport: asyncThrow }, context, ); - rejecting.subscribeCommits(() => { - throw new Error("sync listener"); + rejecting.observeCommits({ + advance: () => { + throw new Error("sync listener"); + }, + end: () => {}, }); const again = await rejecting.root(context); await again.commit((tx) => tx.appendEntry(again.id, { kind: "note" }), context); @@ -466,10 +469,13 @@ describe("listener and report errors", () => { const storage = new MemoryStorage(); const { harness, root, reports } = await started(storage); const seen: string[] = []; - harness.subscribeCommits(() => { - throw new Error("commit listener"); + harness.observeCommits({ + advance: () => { + throw new Error("commit listener"); + }, + end: () => {}, }); - harness.subscribeCommits(() => seen.push("commit")); + harness.observeCommits({ advance: () => seen.push("commit"), end: () => {} }); harness.subscribeClose(() => { throw new Error("close listener"); }); @@ -497,14 +503,16 @@ describe("listener and report errors", () => { ); const watch = await root.watch(context); const seen: unknown[] = []; - watch.start(async ({ value }) => { - seen.push(value); - }); + watch.start( + onValue(async ({ value }) => { + seen.push(value); + }), + ); await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); while (seen.length === 0) await new Promise((resolve) => setTimeout(resolve, 1)); expect(reports).toEqual([failure]); await harness.close(context); - expect(await watch.closed).toEqual({ reason: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); }); it("contains a fulfilled thenable that calls only its fulfillment handler", async () => { @@ -559,7 +567,7 @@ describe("listener and report errors", () => { await root.commit(async (tx) => { (await tx.doc(LiveDoc, root.id)).compactions = []; }, context); - expect(await watch.closed).toEqual({ reason: "listener_error", error: failure }); + expect(await watch.closed).toEqual({ kind: "listener_error", error: failure }); expect(reports).toEqual([failure]); await harness.close(context); }); @@ -595,7 +603,7 @@ describe("listener and report errors", () => { await root.commit(async (tx) => { (await tx.doc(LiveDoc, root.id)).compactions = []; }, context); - expect(await watch.closed).toMatchObject({ reason: "listener_error" }); + expect(await watch.closed).toMatchObject({ kind: "listener_error" }); await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).resolves.toBeDefined(); await opened.harness.close(context); }); diff --git a/packages/durable/test/harness-structured-output.test.ts b/packages/durable/test/harness-structured-output.test.ts index aac0df8a9..829932323 100644 --- a/packages/durable/test/harness-structured-output.test.ts +++ b/packages/durable/test/harness-structured-output.test.ts @@ -155,12 +155,20 @@ describe("structured output", () => { const { harness, root } = await openChat(storage, setup); // The result each nested call's document got, as committed. const stored: JsonValue[] = []; - harness.subscribeCommits((publication) => { - for (const change of publication.changes) { - if (change.type !== "document" || change.record.kind !== NestedCallDoc.definition.kind) continue; - const result = (change.value as NestedCallState | null)?.result; - if (result !== undefined && change.ops.length > 0) stored.push(result as unknown as JsonValue); - } + harness.observeCommits({ + advance: (publication) => { + for (const change of publication.changes) { + if ( + change.type !== "document" || + change.kind !== "change" || + change.record.kind !== NestedCallDoc.definition.kind + ) + continue; + const result = (change.value as NestedCallState)?.result; + if (result !== undefined && change.ops.length > 0) stored.push(result as unknown as JsonValue); + } + }, + end: () => {}, }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); await harness.waitForIdle(context); diff --git a/packages/durable/test/harness-structured.test.ts b/packages/durable/test/harness-structured.test.ts index 771bb9837..87fa7c72d 100644 --- a/packages/durable/test/harness-structured.test.ts +++ b/packages/durable/test/harness-structured.test.ts @@ -1393,7 +1393,7 @@ describe("tool rounds", () => { reject = generation; open("abort.hooked"); // The final commit, with the run's cleanup, fails: nothing of the cleanup lands, and the Harness fails. - expect(await harness.closed).toEqual({ reason: "failed", error: failure }); + expect(await harness.closed).toEqual({ kind: "failed", error: failure }); await expect(submission.wait(context)).rejects.toMatchObject({ name: "SessionFailed", cause: failure }); await harness.close(context); // Reopening finalizes the run; the partial becomes one aborted entry. @@ -1668,7 +1668,7 @@ describe("conversation abort and boundaries", () => { await until(async () => (await state(harness, parent)).status === "completing"); reject = parent; open("child"); - expect(await harness.closed).toEqual({ reason: "failed", error: failure }); + expect(await harness.closed).toEqual({ kind: "failed", error: failure }); expect(reports).toEqual([failure]); await harness.close(context); const reopened = await openNodes(storage.reopen()); diff --git a/packages/durable/test/harness-submissions.test.ts b/packages/durable/test/harness-submissions.test.ts index 43831b59c..bae654985 100644 --- a/packages/durable/test/harness-submissions.test.ts +++ b/packages/durable/test/harness-submissions.test.ts @@ -276,15 +276,18 @@ describe("submissions", () => { await harness.close(context); }); - it("resumes a wait only once subscribeCommits() listeners saw the commit that settled it", async () => { + it("resumes a wait only once observeCommits() observers saw the commit that settled it", async () => { const setup = chatSetup(); setup.faux.setResponses([fauxAssistantMessage("hi")]); const { harness, root } = await openChat(new LateResolvingStorage(), setup); const settled = new Set(); - harness.subscribeCommits((publication) => { - for (const change of publication.changes) { - if (change.type === "submission" && change.value.status === "done") settled.add(change.value.id); - } + harness.observeCommits({ + advance: (publication) => { + for (const change of publication.changes) { + if (change.type === "submission" && change.value.status === "done") settled.add(change.value.id); + } + }, + end: () => {}, }); const submission = await root.submit({ type: "input", content: "hello" }, context); await submission.wait(context); diff --git a/packages/durable/test/harness-support.ts b/packages/durable/test/harness-support.ts index 956b2e675..499f81846 100644 --- a/packages/durable/test/harness-support.ts +++ b/packages/durable/test/harness-support.ts @@ -40,7 +40,11 @@ export async function openHarness( registry.install(defineExtension({ name: "tools", tools: toolNames.map((name) => tool(name)) })); const harness = await Harness.open( storage, - { models: createModels(), registry, ...(options.onReport === undefined ? {} : { onReport: options.onReport }) }, + { + models: createModels(), + registry, + ...(options.onReport === undefined ? {} : { onReport: (report) => options.onReport?.(report.error) }), + }, context, ); return { harness, registry }; diff --git a/packages/durable/test/harness-task-graph.test.ts b/packages/durable/test/harness-task-graph.test.ts index 375b0ec66..c2d63694b 100644 --- a/packages/durable/test/harness-task-graph.test.ts +++ b/packages/durable/test/harness-task-graph.test.ts @@ -15,7 +15,7 @@ import { } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; -import { ControlledStorage, context, flush } from "./session-support.ts"; +import { ControlledStorage, context, flush, onValue } from "./session-support.ts"; import { aborted, completed, deferred, eventually, openTasks } from "./task-support.ts"; type ParentState = { phase: "spawn" } | { phase: "join"; child: TaskId } | { phase: "finish" }; @@ -92,9 +92,11 @@ describe("task graph view", () => { const seen: TaskGraph[] = []; const observe = async () => { const opened = await harness.watchTaskGraph(context); - opened.start(async ({ value }) => { - seen.push(value); - }); + opened.start( + onValue(async ({ value }) => { + seen.push(value); + }), + ); return opened; }; let graph = await observe(); @@ -217,11 +219,13 @@ describe("task graph view", () => { // Exact frames: replaying their operations from the acquisition revision gives each delivered value. let replica: JsonValue = watch.value as unknown as JsonValue; const frames: (readonly Op[])[] = []; - watch.start(async ({ value, ops }) => { - replica = applyImmutable(replica, ops); - expect(replica).toEqual(value); - frames.push(ops); - }); + watch.start( + onValue(async ({ value, ops }) => { + replica = applyImmutable(replica, ops); + expect(replica).toEqual(value); + frames.push(ops); + }), + ); harness.resume(); await eventually(() => watch.value.tasks[String(background)]?.state.status === "running"); expect(await harness.abortTask(background, context)).toBe("marked"); @@ -336,9 +340,11 @@ describe("task graph view", () => { const second = await harness.watchTaskGraph(context); expect(second.value).toBe(first.value); const updates: string[] = []; - first.start(async ({ value }) => { - updates.push(value.tasks[String(id)]?.state.status ?? "gone"); - }); + first.start( + onValue(async ({ value }) => { + updates.push(value.tasks[String(id)]?.state.status ?? "gone"); + }), + ); harness.resume(); await reached.promise; await flush(); diff --git a/packages/durable/test/harness-tasks-recovery.test.ts b/packages/durable/test/harness-tasks-recovery.test.ts index 29b98d8d9..714fdc202 100644 --- a/packages/durable/test/harness-tasks-recovery.test.ts +++ b/packages/durable/test/harness-tasks-recovery.test.ts @@ -644,11 +644,11 @@ describe("definition handover", () => { }); } - async function startHandover(log: string[], gates: Gates, storage = new MemoryStorage()) { + async function startHandover(log: string[], gates: Gates, storage = new MemoryStorage(), version = 1) { const registry = createRegistry(); - const old = addTask(registry, handoverTask("old", 1, log, { gates })); + const old = addTask(registry, handoverTask("old", version, log, { gates })); const opened = await openTasks(storage, [], { registry }); - const id = await createIn(opened.harness, handoverTask("old", 1, log)); + const id = await createIn(opened.harness, handoverTask("old", version, log)); opened.harness.resume(); await eventually(() => log.length === 1); return { ...opened, old, id }; @@ -734,7 +734,7 @@ describe("definition handover", () => { ); gate.resolve(); await eventually(() => infos.length === 1); - expect(infos).toMatchObject([{ type: "migration_failed", task: { id } }]); + expect(infos).toMatchObject([{ kind: "migration_failed", task: { id } }]); await flush(); expect(await harness.getTask(id, context)).toMatchObject({ version: 1, @@ -753,17 +753,30 @@ describe("definition handover", () => { gateA.resolve(); await eventually(() => log.includes("old:b start")); // A newer definition without a migration cannot take the task either. - addTask(registry, handoverTask("incompatible", 2, log)); + addTask(registry, handoverTask("unmigratable", 2, log)); gateB.resolve(); await harness.waitForTask(id, context); expect(log).toEqual(["old:a start", "old:a end", "old:b start", "old:b end", "old:c"]); expect(infos).toMatchObject([ - { type: "task_outdated", task: { id }, reason: "missing_task" }, - { type: "task_outdated", task: { id }, reason: "incompatible_task" }, + { kind: "task_outdated", task: { id }, reason: "missing_task" }, + { kind: "task_outdated", task: { id }, reason: "migration_failed" }, ]); await harness.close(context); }); + it("reports a replacement older than the running task as `task_too_old` and keeps the old definition", async () => { + const log: string[] = []; + const gate = deferred(); + const { harness, registry, id, infos } = await startHandover(log, { a: gate.promise }, new MemoryStorage(), 2); + // The same extension name replaces the running definition with an older version. + addTask(registry, handoverTask("older", 1, log)); + gate.resolve(); + await harness.waitForTask(id, context); + expect(log).toEqual(["old:a start", "old:a end", "old:b start", "old:b end", "old:c"]); + expect(infos).toMatchObject([{ kind: "task_outdated", task: { id }, reason: "task_too_old" }]); + await harness.close(context); + }); + it("rejects a runtime commit of the old invocation queued behind its handover commit", async () => { const log: string[] = []; const gate = deferred(); diff --git a/packages/durable/test/harness-tasks.test.ts b/packages/durable/test/harness-tasks.test.ts index 96cf23911..5cce1635b 100644 --- a/packages/durable/test/harness-tasks.test.ts +++ b/packages/durable/test/harness-tasks.test.ts @@ -21,7 +21,7 @@ import { import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; import { describe, expect, it } from "vitest"; import { addHooks, addTask, addTool, user } from "./harness-support.ts"; -import { ControlledStorage, context, flush, LateResolvingStorage, observeCommits } from "./session-support.ts"; +import { ControlledStorage, context, flush, followCommits, LateResolvingStorage, onValue } from "./session-support.ts"; import { aborted, abortedWith, completed, deferred, eventually, openTasks, settled } from "./task-support.ts"; type Step = { phase: "run" }; @@ -548,9 +548,11 @@ describe("task runtime", () => { handlerContext = ctx; absent = await runtime.watchDoc(Absent, ctx); const watch = await runtime.watchDoc(Notes, ctx); - watch!.start(async ({ value }) => { - delivered.push(value.text); - }); + watch!.start( + onValue(async ({ value }) => { + delivered.push(value.text); + }), + ); watchClosed = watch?.closed; await runtime.commit(() => completed(null), ctx); }); @@ -563,7 +565,7 @@ describe("task runtime", () => { await harness.waitForTask(id, context); expect(absent).toBeUndefined(); // The watch stops when the step after the phase ends the invocation; later commits deliver nothing. - expect(await watchClosed).toEqual({ reason: "stopped" }); + expect(await watchClosed).toEqual({ kind: "stopped" }); await harness.commit(async (tx) => { (await tx.doc(Notes)).text = "after"; }, context); @@ -712,7 +714,7 @@ describe("task scheduling", () => { storage.failNextCommit(failure); harness.resume(); const end = await harness.closed; - expect(end).toEqual({ reason: "failed", error: failure }); + expect(end).toEqual({ kind: "failed", error: failure }); expect(reports).toEqual([failure]); // The pending wait and every later call get SessionFailed with the storage error. await expect(waiting).rejects.toMatchObject({ name: "SessionFailed", cause: failure }); @@ -743,7 +745,7 @@ describe("task scheduling", () => { const { harness, root, reports } = await openRoot([Throws], { storage }); const id = await start(root, Throws); harness.resume(); - expect(await harness.closed).toEqual({ reason: "failed", error: failure }); + expect(await harness.closed).toEqual({ kind: "failed", error: failure }); expect(runs).toBe(1); expect(reports).toEqual([failure]); await harness.close(context); @@ -1201,7 +1203,7 @@ describe("task close", () => { await reached.promise; await harness.close(context); await expect(fromListener).rejects.toThrow("closed"); - expect(watchEnd).toEqual({ reason: "ended" }); + expect(watchEnd).toEqual({ kind: "ended" }); }); it("writes no fault when close seals while the step after a failed phase is queued", async () => { @@ -1385,15 +1387,18 @@ describe("publication at adoption", () => { const endsTask = (writes: readonly StorageWrite[]): boolean => writes.some((write) => write.type === "task" && write.value.state.status === "terminal"); - it("resumes task and idle waits only once subscribeCommits() listeners saw the terminal commit", async () => { + it("resumes task and idle waits only once observeCommits() observers saw the terminal commit", async () => { const gate = deferred(); const Gated = gated("test.wake-gated", gate.promise); const { harness, root } = await openRoot([Gated], { storage: new LateResolvingStorage() }); const seen = new Set(); - harness.subscribeCommits((publication) => { - for (const change of publication.changes) { - if (change.type === "task" && change.value.state.status === "terminal") seen.add(change.value.id); - } + harness.observeCommits({ + advance: (publication) => { + for (const change of publication.changes) { + if (change.type === "task" && change.value.state.status === "terminal") seen.add(change.value.id); + } + }, + end: () => {}, }); const id = await start(root, Gated); harness.resume(); @@ -1413,8 +1418,11 @@ describe("publication at adoption", () => { const { harness, root } = await openRoot([Gated], { storage }); const order: string[] = []; const id = await start(root, Gated); - harness.subscribeCommits((publication) => { - if (terminal(publication, id)) order.push(`host adopting=${storage.adopting}`); + harness.observeCommits({ + advance: (publication) => { + if (terminal(publication, id)) order.push(`host adopting=${storage.adopting}`); + }, + end: () => {}, }); harness.resume(); const waited = harness.waitForTask(id, context).then(() => order.push("wait")); @@ -1435,9 +1443,13 @@ describe("publication at adoption", () => { }); const { harness, root } = await openRoot([Signalled], { storage }); const id = await start(root, Signalled); - harness.subscribeCommits((publication) => { - for (const change of publication.changes) - if (change.type === "task" && change.value.id === id && change.value.abortRequested) order.push("host"); + harness.observeCommits({ + advance: (publication) => { + for (const change of publication.changes) + if (change.type === "task" && change.value.id === id && change.value.abortRequested) + order.push("host"); + }, + end: () => {}, }); harness.resume(); await reached.promise; @@ -1454,8 +1466,11 @@ describe("publication at adoption", () => { const { harness, root } = await openRoot([Gated], { storage }); const order: string[] = []; const id = await start(root, Gated); - harness.subscribeCommits((publication) => { - if (terminal(publication, id)) order.push("host"); + harness.observeCommits({ + advance: (publication) => { + if (terminal(publication, id)) order.push("host"); + }, + end: () => {}, }); const gapped = deferred(); let idle: Promise | undefined; @@ -1481,8 +1496,11 @@ describe("publication at adoption", () => { const { harness, root, infos } = await openRoot([Gated], { storage, telemetry }); const id = await start(root, Gated); const seen: string[] = []; - harness.subscribeCommits((publication) => { - if (terminal(publication, id)) seen.push("terminal"); + harness.observeCommits({ + advance: (publication) => { + if (terminal(publication, id)) seen.push("terminal"); + }, + end: () => {}, }); storage.rejectAfterAdopt = endsTask; harness.resume(); @@ -1490,8 +1508,8 @@ describe("publication at adoption", () => { gate.resolve(); expect((await waited).state.outcome).toEqual({ status: "completed", result: null }); expect(seen).toEqual(["terminal"]); - expect(await harness.closed).toEqual({ reason: "failed", error: new Error("reclaim failed") }); - expect(infos).toContainEqual({ type: "session_failed" }); + expect(await harness.closed).toEqual({ kind: "failed", error: new Error("reclaim failed") }); + expect(infos).toContainEqual({ kind: "storage_failed" }); const active = telemetry.getMetrics().filter((metric) => metric.name === "pi.durable.tasks.active"); expect(active.map((metric) => metric.value)).toEqual([1, -1]); }); @@ -1503,7 +1521,7 @@ describe("publication at adoption", () => { const { harness, root } = await openRoot([Idle], { storage, telemetry }); storage.rejectAfterAdopt = (writes) => writes.some((write) => write.type === "task"); await expect(start(root, Idle)).rejects.toThrow("reclaim failed"); - expect(await harness.closed).toMatchObject({ reason: "failed" }); + expect(await harness.closed).toMatchObject({ kind: "failed" }); const active = telemetry.getMetrics().filter((metric) => metric.name === "pi.durable.tasks.active"); expect(active.map((metric) => metric.value)).toEqual([1, -1]); }); @@ -1514,8 +1532,11 @@ describe("publication at adoption", () => { const { harness, root } = await openRoot([Gated]); const id = await start(root, Gated); let closing: Promise | undefined; - harness.subscribeCommits((publication) => { - if (terminal(publication, id)) closing = harness.close(context); + harness.observeCommits({ + advance: (publication) => { + if (terminal(publication, id)) closing = harness.close(context); + }, + end: () => {}, }); harness.resume(); const waited = harness.waitForTask(id, context); @@ -1547,24 +1568,24 @@ describe("publication at adoption", () => { await gapped.promise; await closing; expect(await waited).toEqual({ status: "completed", result: null }); - expect(await harness.closed).toEqual({ reason: "closed" }); + expect(await harness.closed).toEqual({ kind: "closed" }); }); it("fails the Session with the storage error when it rejects after adopting, and reports an observer's throw", async () => { const storage = new LateResolvingStorage(); const { harness, root, reports, infos } = await openRoot([], { storage }); const bug = new Error("observer bug"); - observeCommits(harness, (publication) => { + followCommits(harness, (publication) => { if (publication.kind === "stored" && publication.changes.some((change) => change.type === "entry")) throw bug; }); storage.rejectAfterAdopt = (writes) => writes.some((write) => write.type === "entry"); await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context)).rejects.toThrow( "reclaim failed", ); - expect(await harness.closed).toEqual({ reason: "failed", error: new Error("reclaim failed") }); + expect(await harness.closed).toEqual({ kind: "failed", error: new Error("reclaim failed") }); expect(reports).toContain(bug); - expect(infos[reports.indexOf(bug)]).toEqual({ type: "listener_error" }); - expect(infos).toContainEqual({ type: "session_failed" }); + expect(infos[reports.indexOf(bug)]).toEqual({ kind: "listener_error" }); + expect(infos).toContainEqual({ kind: "storage_failed" }); }); it("stores a host listener's own commit before the reconcile commit that the same publication asks for", async () => { @@ -1582,13 +1603,16 @@ describe("publication at adoption", () => { const parent = await start(root, Parent); harness.resume(); await eventually(async () => (await harness.getTask(parent, context))?.state.status === "completing"); - harness.subscribeCommits((publication) => { - const child = publication.changes.find( - (change) => - change.type === "task" && change.value.id !== parent && change.value.state.status === "terminal", - ); - if (child !== undefined) - void root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + harness.observeCommits({ + advance: (publication) => { + const child = publication.changes.find( + (change) => + change.type === "task" && change.value.id !== parent && change.value.state.status === "terminal", + ); + if (child !== undefined) + void root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + }, + end: () => {}, }); const before = storage.commits.length; gate.resolve(); @@ -1608,11 +1632,14 @@ describe("publication at adoption", () => { const { harness, root } = await openRoot([], { storage }); const note = () => root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); const late: number[] = []; - let stopLate = (): void => {}; + let stopLate = { detach: (): void => {} }; // No host listener at adopt, although the Harness's own observers run: no dispatch, so the late one misses it. storage.afterAdopt = () => { - stopLate = harness.subscribeCommits((publication) => { - late.push(publication.seq); + stopLate = harness.observeCommits({ + advance: (publication) => { + late.push(publication.seq); + }, + end: () => {}, }); }; await note(); @@ -1620,13 +1647,16 @@ describe("publication at adoption", () => { expect(late).toEqual([]); await note(); expect(late).toHaveLength(1); - stopLate(); + stopLate.detach(); // One registered at adopt: the dispatch reads the listeners when it runs, the late one included. - harness.subscribeCommits(() => {}); + harness.observeCommits({ advance: () => {}, end: () => {} }); const second: number[] = []; storage.afterAdopt = () => { - harness.subscribeCommits((publication) => { - second.push(publication.seq); + harness.observeCommits({ + advance: (publication) => { + second.push(publication.seq); + }, + end: () => {}, }); }; await note(); diff --git a/packages/durable/test/harness-telemetry.test.ts b/packages/durable/test/harness-telemetry.test.ts index c802568fa..59537179e 100644 --- a/packages/durable/test/harness-telemetry.test.ts +++ b/packages/durable/test/harness-telemetry.test.ts @@ -21,7 +21,7 @@ import { describe, expect, it, vi } from "vitest"; import { createCodemodeTool } from "../src/codemode/codemode.ts"; import { type ChatSetup, chatSetup, waitFor } from "./chat-support.ts"; import { addHooks, addTask, addTool } from "./harness-support.ts"; -import { ControlledStorage, context, flush } from "./session-support.ts"; +import { ControlledStorage, context, flush, onValue } from "./session-support.ts"; import { aborted, openTasks } from "./task-support.ts"; async function openTracked(storage: Storage, setup: ChatSetup, telemetry: InMemoryTelemetryContext) { @@ -33,7 +33,7 @@ async function openTracked(storage: Storage, setup: ChatSetup, telemetry: InMemo settings: setup.settings, now: () => setup.now(), telemetry, - onReport: (error) => setup.reports.push(error), + onReport: ({ error }) => setup.reports.push(error), }, context, ); @@ -722,8 +722,8 @@ describe("Durable telemetry", () => { phases: { report: async (_task, runtime, ctx) => { runtime.report(error); - runtime.report(error, { type: "extension", extension: "reporting-extension" }); - runtime.report(error, { type: "host", callback: "models" }); + runtime.report(error, { kind: "extension", extension: "reporting-extension" }); + runtime.report(error, { kind: "host", callback: "models" }); await runtime.commit( () => ({ status: "terminal", outcome: { status: "completed", result: null } }), ctx, @@ -856,7 +856,7 @@ describe("Durable telemetry", () => { .getMetrics() .filter((metric) => metric.name === "pi.durable.reports") .map((metric) => metric.attributes), - ).toEqual([{ type: "session_failed" }]); + ).toEqual([{ type: "storage_failed" }]); }); it("counts reports raised with a NOOP span without creating a root", async () => { @@ -864,8 +864,11 @@ describe("Durable telemetry", () => { const telemetry = new InMemoryTelemetryContext(); const { harness, root } = await openTracked(new MemoryStorage(), setup, telemetry); const error = new Error("private-noop-report"); - const unsubscribe = harness.subscribeCommits(() => { - throw error; + const observation = harness.observeCommits({ + advance: () => { + throw error; + }, + end: () => {}, }); await root.commit((tx) => tx.appendEntry(root.id, { kind: "app.note" }), context); expect(setup.reports).toEqual([error]); @@ -876,7 +879,7 @@ describe("Durable telemetry", () => { .filter((metric) => metric.name === "pi.durable.reports") .map((metric) => metric.attributes), ).toEqual([{ type: "listener_error" }]); - unsubscribe(); + observation.detach(); await harness.close(context); }); @@ -886,9 +889,12 @@ describe("Durable telemetry", () => { setup.faux.setResponses([fauxAssistantMessage("answer")]); const { harness, root } = await openTracked(new MemoryStorage(), setup, telemetry); const error = new Error("private-listener-error"); - const unsubscribe = harness.subscribeCommits((publication) => { - if (publication.changes.some((change) => change.type === "entry" && change.value.kind === "pi.user")) - throw error; + const observation = harness.observeCommits({ + advance: (publication) => { + if (publication.changes.some((change) => change.type === "entry" && change.value.kind === "pi.user")) + throw error; + }, + end: () => {}, }); await (await root.submit({ type: "input", content: "input" }, context)).wait(context); await harness.waitForIdle(context); @@ -897,7 +903,7 @@ describe("Durable telemetry", () => { const span = telemetry.getSpans().find((span) => span.name === "pi.durable.submit")!; expect(span.events).toContainEqual({ name: "pi.durable.report", attributes: { type: "listener_error" } }); expect(JSON.stringify(telemetry.getSpans())).not.toContain("private-listener-error"); - unsubscribe(); + observation.detach(); await harness.close(context); }); @@ -907,9 +913,12 @@ describe("Durable telemetry", () => { setup.faux.setResponses([fauxAssistantMessage("answer")]); const { harness, root } = await openTracked(new MemoryStorage(), setup, telemetry); const failed = Promise.withResolvers(); - const unsubscribe = harness.subscribeCommits((publication) => { - if (publication.changes.some((change) => change.type === "entry" && change.value.kind === "pi.user")) - return failed.promise; + const observation = harness.observeCommits({ + advance: (publication) => { + if (publication.changes.some((change) => change.type === "entry" && change.value.kind === "pi.user")) + return failed.promise; + }, + end: () => {}, }); await (await root.submit({ type: "input", content: "input" }, context)).wait(context); await harness.waitForIdle(context); @@ -925,7 +934,7 @@ describe("Durable telemetry", () => { .map((metric) => metric.attributes), ).toEqual([{ type: "listener_error" }]); expect(JSON.stringify(telemetry.getSpans())).not.toContain("private-async-listener-error"); - unsubscribe(); + observation.detach(); await harness.close(context); }); @@ -1012,12 +1021,17 @@ describe("Durable telemetry", () => { const root = await harness.root(context); const view = await root.watch(context); const shown: (number | undefined)[] = []; - view.start(async ({ value }) => { - shown.push(value.entries.at(-1)?.id); - }); + view.start( + onValue(async ({ value }) => { + shown.push(value.entries.at(-1)?.id); + }), + ); const hosts: number[] = []; - harness.subscribeCommits((publication) => { - hosts.push(publication.seq); + harness.observeCommits({ + advance: (publication) => { + hosts.push(publication.seq); + }, + end: () => {}, }); const failing = withTelemetry( { @@ -1035,8 +1049,8 @@ describe("Durable telemetry", () => { }, failing); // The entry event, then the active-task metric; the report's own event throws too and is dropped. expect(infos).toEqual([ - { type: "host_error", callback: "telemetry" }, - { type: "host_error", callback: "telemetry" }, + { kind: "host_error", callback: "telemetry" }, + { kind: "host_error", callback: "telemetry" }, ]); expect(hosts).toHaveLength(1); await flush(); @@ -1044,7 +1058,7 @@ describe("Durable telemetry", () => { expect((await harness.getTask(task, context))?.state.status).toBe("pending"); await view.stop(); await harness.close(context); - expect(await harness.closed).toEqual({ reason: "closed" }); + expect(await harness.closed).toEqual({ kind: "closed" }); }); it("drops a report's own metric that throws, after onReport got the report", async () => { @@ -1060,15 +1074,18 @@ describe("Durable telemetry", () => { const { harness, reports, infos } = await openTasks(new MemoryStorage(), [], { telemetry }); const root = await harness.root(context); const bug = new Error("listener bug"); - harness.subscribeCommits(() => { - throw bug; + harness.observeCommits({ + advance: () => { + throw bug; + }, + end: () => {}, }); await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); // The listener runs in the commit's dispatch microtask; a throw escaping its report would be unhandled there. await flush(); expect(reports).toEqual([bug]); - expect(infos).toEqual([{ type: "listener_error" }]); + expect(infos).toEqual([{ kind: "listener_error" }]); await harness.close(context); - expect(await harness.closed).toEqual({ reason: "closed" }); + expect(await harness.closed).toEqual({ kind: "closed" }); }); }); diff --git a/packages/durable/test/harness-view-run.test.ts b/packages/durable/test/harness-view-run.test.ts index 05d9635f3..62c70b850 100644 --- a/packages/durable/test/harness-view-run.test.ts +++ b/packages/durable/test/harness-view-run.test.ts @@ -26,7 +26,7 @@ import { describe, expect, it } from "vitest"; import { type ChatSetup, chatSetup, openChat, textOf, waitFor } from "./chat-support.ts"; import { addTool } from "./harness-support.ts"; import { slotsOf, viewHistory } from "./live-support.ts"; -import { context, documentChanges, observeCommits } from "./session-support.ts"; +import { context, documentValues, followCommits, onValue } from "./session-support.ts"; import { aborted, deferred } from "./task-support.ts"; type Frame = { value: ConversationView; ops: readonly Op[] }; @@ -40,13 +40,15 @@ async function listen(conversation: Conversation, check = true) { const frames: Frame[] = []; let replayed = watch.value; let mismatches = 0; - watch.start(async ({ value, ops }) => { - if (check) { - replayed = applyImmutable(replayed, ops); - if (JSON.stringify(replayed) !== JSON.stringify(value)) mismatches++; - } - frames.push({ value, ops }); - }); + watch.start( + onValue(async ({ value, ops }) => { + if (check) { + replayed = applyImmutable(replayed, ops); + if (JSON.stringify(replayed) !== JSON.stringify(value)) mismatches++; + } + frames.push({ value, ops }); + }), + ); return { watch, frames, mismatches: () => mismatches }; } @@ -63,9 +65,9 @@ function slow(): ChatSetup { /** Committed partials of the Harness's generations, one per commit that has one. */ function partialsOf(harness: Harness): AssistantMessage[] { const partials: AssistantMessage[] = []; - observeCommits(harness, (publication) => { - for (const change of documentChanges(publication)) { - const message = (change.value as LiveState | null)?.generation?.message as AssistantMessage | undefined; + followCommits(harness, (publication) => { + for (const change of documentValues(publication)) { + const message = (change.value as LiveState)?.generation?.message as AssistantMessage | undefined; if (change.record.kind === "pi.live" && message !== undefined) partials.push(message); } }); @@ -214,9 +216,10 @@ describe("a run as the conversation view shows it", () => { const observation = await harness.observeConversation( root.id, { - advance: (change) => { - views.push(change.value); - ops.push([...change.ops]); + advance: (call) => { + if (call.kind !== "value") return; + views.push(call.value); + ops.push([...call.ops]); }, end: () => {}, }, @@ -269,9 +272,9 @@ describe("a run as the conversation view shows it", () => { ]); const { harness, root } = await openChat(new MemoryStorage(), setup); const outputs: string[] = []; - observeCommits(harness, (publication) => { - for (const change of documentChanges(publication)) { - const output = (change.value as LiveState | null)?.tools?.[0]?.output; + followCommits(harness, (publication) => { + for (const change of documentValues(publication)) { + const output = (change.value as LiveState)?.tools?.[0]?.output; if (change.record.kind === "pi.live" && output !== undefined && output !== outputs.at(-1)) { outputs.push(output); } @@ -441,9 +444,11 @@ describe("a run as the conversation view shows it", () => { await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); } const frames: Frame[] = []; - watch.start(async ({ value, ops }) => { - frames.push({ value, ops }); - }); + watch.start( + onValue(async ({ value, ops }) => { + frames.push({ value, ops }); + }), + ); await drained(); expect(frames[0]!.ops).toEqual([["r", frames[0]!.value]]); await change((live) => { @@ -503,7 +508,7 @@ describe("streamed partials", () => { const ops: Op[] = []; const observation = await harness.observeConversation( root.id, - { advance: (change) => ops.push(...change.ops), end: () => {} }, + { advance: (call) => call.kind === "value" && ops.push(...call.ops), end: () => {} }, context, ); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); @@ -552,7 +557,7 @@ describe("streamed partials", () => { ]); const { harness, root } = await openChat(new MemoryStorage(), setup); let commits = 0; - observeCommits(harness, () => commits++); + followCommits(harness, () => commits++); const bytes = await allocatedBytes(async () => { await (await root.submit({ type: "input", content: "go" }, context)).wait(context); }); diff --git a/packages/durable/test/harness-view.test.ts b/packages/durable/test/harness-view.test.ts index f5a5d7f1b..2f0913fc0 100644 --- a/packages/durable/test/harness-view.test.ts +++ b/packages/durable/test/harness-view.test.ts @@ -22,7 +22,7 @@ import { import { describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, chatSetup, openChat, waitFor } from "./chat-support.ts"; -import { context } from "./session-support.ts"; +import { context, onValue } from "./session-support.ts"; import { aborted, deferred } from "./task-support.ts"; type Frame = { value: ConversationView; ops: readonly Op[] }; @@ -34,9 +34,11 @@ async function record(conversation: Conversation, window?: ConversationWindow) { const watch = await conversation.watch(context, window); const initial = watch.value; const frames: Frame[] = []; - watch.start(async ({ value, ops }) => { - frames.push({ value, ops }); - }); + watch.start( + onValue(async ({ value, ops }) => { + frames.push({ value, ops }); + }), + ); return { initial, frames, stop: () => watch.stop() }; } @@ -89,16 +91,20 @@ async function drained(): Promise { function touches(harness: Harness, conversation: Conversation): { count: number } { const counter = { count: 0 }; - harness.subscribeCommits((publication) => { - const touched = publication.changes.some( - (change) => - (change.type === "entry" && change.value.conversationId === conversation.id) || - (change.type === "document" && - change.conversationId === conversation.id && - MOUNTED.includes(change.record.kind) && - change.ops.length > 0), - ); - if (touched) counter.count++; + harness.observeCommits({ + advance: (publication) => { + const touched = publication.changes.some( + (change) => + (change.type === "entry" && change.value.conversationId === conversation.id) || + (change.type === "document" && + change.kind === "change" && + change.conversationId === conversation.id && + MOUNTED.includes(change.record.kind) && + change.ops.length > 0), + ); + if (touched) counter.count++; + }, + end: () => {}, }); return counter; } @@ -249,9 +255,11 @@ describe("conversation view", () => { await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); } const frames: Frame[] = []; - watch.start(async ({ value, ops }) => { - frames.push({ value, ops }); - }); + watch.start( + onValue(async ({ value, ops }) => { + frames.push({ value, ops }); + }), + ); await drained(); expect(frames).toHaveLength(1); expect(frames[0]!.ops[0]?.[0]).not.toBe("r"); @@ -287,7 +295,7 @@ describe("conversation view", () => { const { harness, root } = await openChat(new MemoryStorage(), chatSetup()); const watch = await root.watch(context); await harness.close(context); - expect(await watch.closed).toEqual({ reason: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); expect(watch.value.entries).toEqual([]); await expect(root.watch(context)).rejects.toThrow(); }); @@ -333,7 +341,7 @@ describe("conversation view", () => { await root.commit((tx) => tx.appendEntry(root.id, { kind: "one" }), context); await root.commit((tx) => tx.appendEntry(root.id, { kind: "two" }), context); await drained(); - expect(await failing.closed).toMatchObject({ reason: "listener_error" }); + expect(await failing.closed).toMatchObject({ kind: "listener_error" }); expect(frames).toHaveLength(2); await stop(); await harness.close(context); @@ -519,7 +527,7 @@ describe("conversation view window", () => { const changes: ConversationView[] = []; const observation = await harness.observeConversation( root.id, - { advance: ({ value }) => changes.push(value), end: () => {} }, + { advance: (call) => call.kind === "value" && changes.push(call.value), end: () => {} }, context, { min: 2, max: 3 }, ); @@ -545,9 +553,11 @@ describe("conversation view window", () => { await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); } const frames: Frame[] = []; - watch.start(async ({ value, ops }) => { - frames.push({ value, ops }); - }); + watch.start( + onValue(async ({ value, ops }) => { + frames.push({ value, ops }); + }), + ); await drained(); expect(frames).toHaveLength(1); expect(frames[0]!.ops).toEqual([["r", frames[0]!.value]]); @@ -564,10 +574,12 @@ describe("conversation view window", () => { const watch = await root.watch(context, { min: 5, max: 10 }); let first: WeakRef | undefined; let count = 0; - watch.start(async ({ value }) => { - count = value.entries.length; - if (first === undefined && value.entries[0] !== undefined) first = new WeakRef(value.entries[0]); - }); + watch.start( + onValue(async ({ value }) => { + count = value.entries.length; + if (first === undefined && value.entries[0] !== undefined) first = new WeakRef(value.entries[0]); + }), + ); await notes(root, 1); await waitFor(() => first !== undefined); await notes(root, 200); diff --git a/packages/durable/test/live-support.ts b/packages/durable/test/live-support.ts index 562d13e56..794da6beb 100644 --- a/packages/durable/test/live-support.ts +++ b/packages/durable/test/live-support.ts @@ -13,7 +13,7 @@ import { type TaskId, type ToolSlot, } from "@earendil-works/pi-durable"; -import { context, documentChanges, observeCommits } from "./session-support.ts"; +import { context, documentChanges, followCommits } from "./session-support.ts"; /** * Every view of `conversationId` from now on, one per commit that changes it, unfolded: a synchronous observer, so no @@ -23,7 +23,7 @@ export async function viewHistory(harness: Harness, conversationId: Conversation const views: ConversationView[] = []; const observation = await harness.observeConversation( conversationId, - { advance: ({ value }) => views.push(value), end: () => {} }, + { advance: (call) => call.kind === "value" && views.push(call.value), end: () => {} }, context, ); views.push(observation.value); @@ -69,9 +69,9 @@ export function startOrder(history: readonly LiveState[]): string[] { */ export function nestedResults(harness: Harness): Map { const results = new Map(); - observeCommits(harness, (publication) => { + followCommits(harness, (publication) => { for (const change of documentChanges(publication)) { - if (change.record.kind !== NestedCallDoc.definition.kind || change.value === null) continue; + if (change.record.kind !== NestedCallDoc.definition.kind || change.kind === "retire") continue; const state = change.value as NestedCallState; if (state.taskId !== undefined && state.result !== undefined) { results.set(state.taskId, state.result as unknown as NestedToolExecutionResult); @@ -95,7 +95,7 @@ export function nestedResultOf( export async function graphHistory(harness: Harness) { const graphs: TaskGraph[] = []; const observation = await harness.observeTaskGraph( - { advance: ({ value }) => graphs.push(value), end: () => {} }, + { advance: (call) => call.kind === "value" && graphs.push(call.value), end: () => {} }, context, ); graphs.push(observation.value); diff --git a/packages/durable/test/session-adoption.test.ts b/packages/durable/test/session-adoption.test.ts index f19ae976f..ddd7dd1fd 100644 --- a/packages/durable/test/session-adoption.test.ts +++ b/packages/durable/test/session-adoption.test.ts @@ -15,7 +15,7 @@ import { NodeExecutionEnv } from "../src/env/node.ts"; import { type ObservedCommit, SessionImpl } from "../src/session/session.ts"; import { Transaction } from "../src/session/transaction.ts"; import { JsonlStorage } from "../src/storage/jsonl/index.ts"; -import { ControlledStorage, context, createConversation, documentChanges } from "./session-support.ts"; +import { ControlledStorage, context, createConversation, documentValues } from "./session-support.ts"; const CounterDoc = defineDoc<{ count: number }>({ checkpointWhen: () => false, @@ -93,7 +93,7 @@ describe("Session adoption inside Storage.commit()", () => { storage.fault = "reject_after_adopt"; await expect(bump(session)).rejects.toThrow("reclaim failed"); expect(discard).not.toHaveBeenCalled(); - expect(await session.closed).toMatchObject({ reason: "failed", error: new Error("reclaim failed") }); + expect(await session.closed).toMatchObject({ kind: "failed", error: new Error("reclaim failed") }); }); it("discards the staged changes when Storage rejects before adoption", async () => { @@ -104,7 +104,7 @@ describe("Session adoption inside Storage.commit()", () => { vi.spyOn(storage, "commit").mockRejectedValueOnce(new Error("disk full")); await expect(bump(session)).rejects.toThrow("disk full"); expect(discard).toHaveBeenCalledTimes(1); - expect(await session.closed).toMatchObject({ reason: "failed" }); + expect(await session.closed).toMatchObject({ kind: "failed" }); }); it("fails the Session with its own adoption error, which never reaches Storage", async () => { @@ -117,25 +117,28 @@ describe("Session adoption inside Storage.commit()", () => { await expect(bump(session)).rejects.toBe(failure); // Storage saw a normal adoption and resolved. expect(storage.order.slice(-3)).toEqual(["adopt", "adopted", "resolve"]); - expect(await session.closed).toEqual({ reason: "failed", error: failure }); + expect(await session.closed).toEqual({ kind: "failed", error: failure }); }); it("fails the Session when Storage resolves without adopting", async () => { const { storage, session } = open(); storage.fault = "skip_adopt"; await expect(bump(session)).rejects.toThrow("without adopting"); - expect(await session.closed).toMatchObject({ reason: "failed" }); + expect(await session.closed).toMatchObject({ kind: "failed" }); }); }); describe("publication at adoption", () => { it("runs its own observers inside adopt and host listeners one microtask later, outside it, before Storage resolves", async () => { const { storage, session } = open(); - session.observeCommits(() => { + session.followCommits(() => { storage.order.push(`observed adopting=${storage.adopting}`); }); - session.subscribeCommits(() => { - storage.order.push(`host adopting=${storage.adopting}`); + session.observeCommits({ + advance: () => { + storage.order.push(`host adopting=${storage.adopting}`); + }, + end: () => {}, }); await bump(session); expect(storage.order).toEqual(["adopt", "observed adopting=true", "adopted", "host adopting=false", "resolve"]); @@ -145,16 +148,25 @@ describe("publication at adoption", () => { it("calls the host listeners registered when its dispatch runs, and none for a commit adopted without one", async () => { const { storage, session } = open(); const calls: string[] = []; - const removed = session.subscribeCommits(() => { - calls.push("removed"); + const removed = session.observeCommits({ + advance: () => { + calls.push("removed"); + }, + end: () => {}, }); - session.subscribeCommits(() => { - calls.push("kept"); + session.observeCommits({ + advance: () => { + calls.push("kept"); + }, + end: () => {}, }); storage.afterAdopt = () => { - removed(); - session.subscribeCommits(() => { - calls.push("added"); + removed.detach(); + session.observeCommits({ + advance: () => { + calls.push("added"); + }, + end: () => {}, }); }; await bump(session); @@ -163,8 +175,11 @@ describe("publication at adoption", () => { const bare = open(); const late: Seq[] = []; bare.storage.afterAdopt = () => { - bare.session.subscribeCommits((publication) => { - late.push(publication.seq); + bare.session.observeCommits({ + advance: (publication) => { + late.push(publication.seq); + }, + end: () => {}, }); }; await bump(bare.session); @@ -178,15 +193,18 @@ describe("publication at adoption", () => { it("fails the Session after Storage returned for a throw of its own observers; the others and hosts got the commit", async () => { const { storage, session } = open(); const bug = new Error("observer bug"); - session.observeCommits(() => { + session.followCommits(() => { storage.order.push("observed"); throw bug; }); - session.observeCommits(() => { + session.followCommits(() => { storage.order.push("observed after"); }); - session.subscribeCommits(() => { - storage.order.push(`host failed=${session.failure !== undefined}`); + session.observeCommits({ + advance: () => { + storage.order.push(`host failed=${session.failure !== undefined}`); + }, + end: () => {}, }); session.subscribeClose(() => { storage.order.push("close"); @@ -202,7 +220,7 @@ describe("publication at adoption", () => { "resolve", "close", ]); - expect(await session.closed).toEqual({ reason: "failed", error: bug }); + expect(await session.closed).toEqual({ kind: "failed", error: bug }); const reopened = new SessionImpl(storage.reopen()); expect(await reopened.snapshot(CounterDoc, context)).toEqual({ count: 1 }); await reopened.close(context); @@ -212,19 +230,22 @@ describe("publication at adoption", () => { const { storage, session } = open(); await bump(session); const stored: Seq[] = []; - session.observeCommits(() => { + session.followCommits(() => { storage.order.push("observed"); }); - session.subscribeCommits((publication) => { - storage.order.push("host"); - stored.push(publication.seq); + session.observeCommits({ + advance: (publication) => { + storage.order.push("host"); + stored.push(publication.seq); + }, + end: () => {}, }); storage.order.length = 0; storage.fault = "reject_after_adopt"; await expect(bump(session)).rejects.toThrow("reclaim failed"); expect(storage.order).toEqual(["adopt", "observed", "adopted", "host", "resolve"]); expect(stored).toHaveLength(1); - expect(await session.closed).toMatchObject({ reason: "failed" }); + expect(await session.closed).toMatchObject({ kind: "failed" }); const reopened = new SessionImpl(storage.reopen()); expect(await reopened.snapshot(CounterDoc, context)).toEqual({ count: 2 }); await reopened.close(context); @@ -238,13 +259,16 @@ describe("publication at adoption", () => { }, context); const observed: ObservedCommit[] = []; const stored: CommitPublication[] = []; - session.observeCommits((publication) => { + session.followCommits((publication) => { storage.order.push("observed"); observed.push(publication); }); - session.subscribeCommits((publication) => { - storage.order.push("host"); - stored.push(publication); + session.observeCommits({ + advance: (publication) => { + storage.order.push("host"); + stored.push(publication); + }, + end: () => {}, }); storage.order.length = 0; await session.commit( @@ -260,9 +284,9 @@ describe("publication at adoption", () => { (await tx.doc(NoteDoc, conversationId)).b = 2; }, context); expect(storage.order).toEqual(["observed", "adopt", "observed", "adopted", "host", "resolve"]); - expect(documentChanges(observed[1]!).map((change) => change.ops)).toEqual([[["s", ["b"], 2]]]); + expect(documentValues(observed[1]!).map((change) => change.ops)).toEqual([[["s", ["b"], 2]]]); expect(stored).toHaveLength(1); - expect(documentChanges(stored[0]!).map((change) => change.ops)).toEqual([ + expect(documentValues(stored[0]!).map((change) => change.ops)).toEqual([ [ ["s", ["a"], 1], ["s", ["b"], 2], @@ -278,8 +302,11 @@ describe("publication at adoption", () => { await session.commit(async (tx) => { await tx.doc(NoteDoc, conversationId); }, context); - session.subscribeCommits(() => { - storage.order.push(`host adopting=${storage.adopting}`); + session.observeCommits({ + advance: () => { + storage.order.push(`host adopting=${storage.adopting}`); + }, + end: () => {}, }); await session.commit( async (tx) => { @@ -325,8 +352,11 @@ describe("publication at adoption", () => { }, context); try { await count(); - session.subscribeCommits(() => { - env.order.push("host"); + session.observeCommits({ + advance: () => { + env.order.push("host"); + }, + end: () => {}, }); await count(); env.order.push("resolved"); @@ -344,8 +374,11 @@ describe("publication at adoption", () => { await tx.doc(NoteDoc, conversationId); }, context); const stored: (readonly unknown[])[] = []; - session.subscribeCommits((publication) => { - stored.push(documentChanges(publication).map((change) => change.ops)); + session.observeCommits({ + advance: (publication) => { + stored.push(documentValues(publication).map((change) => change.ops)); + }, + end: () => {}, }); await session.commit( async (tx) => { @@ -359,6 +392,6 @@ describe("publication at adoption", () => { await expect(session.unloadDocuments()).rejects.toThrow("reclaim failed"); expect(storage.order).toEqual(["adopt", "adopted", "resolve"]); expect(stored).toEqual([[[["s", ["a"], 1]]]]); - expect(await session.closed).toMatchObject({ reason: "failed", error: new Error("reclaim failed") }); + expect(await session.closed).toMatchObject({ kind: "failed", error: new Error("reclaim failed") }); }); }); diff --git a/packages/durable/test/session-checkpoints-migrations.test.ts b/packages/durable/test/session-checkpoints-migrations.test.ts index f5433f1a5..cc60b232e 100644 --- a/packages/durable/test/session-checkpoints-migrations.test.ts +++ b/packages/durable/test/session-checkpoints-migrations.test.ts @@ -7,7 +7,15 @@ import { type StorageWrite, } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; -import { context, createConversation, documentChanges, flush, openTestSession } from "./session-support.ts"; +import { + context, + createConversation, + documentChanges, + documentValues, + flush, + onValue, + openTestSession, +} from "./session-support.ts"; function documentWrites(writes: readonly StorageWrite[]): readonly StorageWrite[] { return writes.filter( @@ -94,7 +102,7 @@ describe("Session document checkpoints", () => { expect(falseCalls).toHaveLength(1); const snapshot = (await session.snapshot(BaseDoc, context))!; const deltaSnapshot = (await session.snapshot(DeltaDoc, context))!; - const [published, publishedDelta] = documentChanges(publications.at(-1)!); + const [published, publishedDelta] = documentValues(publications.at(-1)!); expect(calls[0]!.value).toBe(snapshot); expect(calls[0]!.ops).toBe(published!.ops); expect(falseCalls[0]!.value).toBe(deltaSnapshot); @@ -402,17 +410,21 @@ describe("Session document migrations", () => { // An observer of the older shape. const watch = (await session.watchDoc(Old, context))!; const frames: unknown[] = []; - watch.start(async ({ value, ops }) => { - frames.push({ value, ops }); - }); + watch.start( + onValue(async ({ value, ops }) => { + frames.push({ value, ops }); + }), + ); expect(await session.snapshot(Current, context)).toEqual({ count: 4 }); const snapshot = await session.snapshot(Current, context); // An observer of the new shape: the migration changes nothing it sees. const current = (await session.watchDoc(Current, context))!; const currentFrames: unknown[] = []; - current.start(async ({ value }) => { - currentFrames.push(value); - }); + current.start( + onValue(async ({ value }) => { + currentFrames.push(value); + }), + ); await session.commit((tx) => tx.doc(Current).then(() => undefined), context); await flush(); @@ -488,7 +500,7 @@ describe("Session document migrations", () => { }), ]); await flush(); - const published = documentChanges(publications.at(-1)!)[0]!; + const published = documentValues(publications.at(-1)!)[0]!; const admitted = storage.admittedCommits.at(-1)![0]!; if (admitted.type !== "document.change" || admitted.content.kind !== "base") { throw new Error("Expected migration base"); @@ -836,10 +848,12 @@ describe("Session tracker cache across definition versions", () => { const watch = (await session.watchDoc(V1Doc, context))!; const frames: (readonly unknown[])[] = []; const delivered = new Promise((resolve) => { - watch.start(async ({ ops }) => { - frames.push(ops); - resolve(); - }); + watch.start( + onValue(async ({ ops }) => { + frames.push(ops); + resolve(); + }), + ); }); await session.commit(async (tx) => { (await tx.doc(V2Doc)).names.push("second"); diff --git a/packages/durable/test/session-deferred-commits.test.ts b/packages/durable/test/session-deferred-commits.test.ts index bbf72d002..7de47e659 100644 --- a/packages/durable/test/session-deferred-commits.test.ts +++ b/packages/durable/test/session-deferred-commits.test.ts @@ -18,7 +18,7 @@ import { } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; import { type ObservedCommit, SessionImpl } from "../src/session/session.ts"; -import { ControlledStorage, context, createConversation, flush } from "./session-support.ts"; +import { ControlledStorage, context, createConversation, flush, onValue } from "./session-support.ts"; type Text = { text: string; base?: boolean; explode?: boolean; keys?: Record }; @@ -91,7 +91,7 @@ type Armed = { readonly callback: () => void; readonly delayMs: number; disarmed /** * A Session over controlled storage with a manual clock and manual foreground timers (background ones are declined, so - * commits evict), recording what `subscribeCommits()` and the Session's own observers get. + * commits evict), recording what `observeCommits()` and the Session's own observers get. */ async function open(options: { readonly retentionMs?: number; readonly persistWithinMs?: number } = {}) { const clock = { now: 1_000 }; @@ -114,11 +114,14 @@ async function open(options: { readonly retentionMs?: number; readonly persistWi () => options.persistWithinMs, ); const stored: CommitPublication[] = []; - session.subscribeCommits((publication) => { - stored.push(publication); + session.observeCommits({ + advance: (publication) => { + stored.push(publication); + }, + end: () => {}, }); const observed: ObservedCommit[] = []; - session.observeCommits((publication) => { + session.followCommits((publication) => { observed.push(publication); }); const conversationId = await createConversation(session); @@ -396,7 +399,7 @@ describe("storing held changes", () => { const failure = new Error("disk gone"); failing.storage.failNextCommit(failure); await expect(failing.session.close(context)).rejects.toBe(failure); - expect(await failing.session.closed).toEqual({ reason: "failed", error: failure }); + expect(await failing.session.closed).toEqual({ kind: "failed", error: failure }); // Closed: the backend refuses calls until reopened. await expect(failing.storage.document(failing.recordId, "current", context)).rejects.toThrow("closed"); expect((await failing.storage.reopen().document(failing.recordId, "current", context))?.value).toEqual({ @@ -515,7 +518,7 @@ async function overflow(session: SessionImpl, conversationId: ConversationId): P } describe("publications", () => { - it("give subscribeCommits() exactly what was stored, and nothing for a held commit", async () => { + it("give observeCommits() exactly what was stored, and nothing for a held commit", async () => { const { session, storage, conversationId, recordId, stored, fire } = await open(); const from = storage.commits.length; const publications = stored.length; @@ -540,8 +543,8 @@ describe("publications", () => { for (const [index, batch] of batches.entries()) { const write = changesOf([batch]).find((candidate) => candidate.id === recordId)!; const change = published[index]!.changes.find( - (candidate): candidate is Extract => - candidate.type === "document" && candidate.record.id === recordId, + (candidate): candidate is Extract => + candidate.type === "document" && candidate.kind === "change" && candidate.record.id === recordId, )!; fromStorage = write.content.kind === "base" ? write.content.value : applyImmutable(fromStorage, write.content.ops); @@ -562,9 +565,11 @@ describe("publications", () => { const { session, conversationId, observed, fire } = await open(); const watch = (await session.watchDoc(TextDoc, conversationId, context))!; const values: string[] = []; - watch.start(async ({ value }) => { - values.push(value.text); - }); + watch.start( + onValue(async ({ value }) => { + values.push(value.text); + }), + ); const from = observed.length; for (const text of ["a", "b", "c"]) { await append(session, conversationId, text); @@ -578,7 +583,9 @@ describe("publications", () => { expect(seen.map((publication) => publication.kind)).toEqual(["memory", "memory", "memory", "stored"]); // The storing commit shows its own operations; the held ones were shown already. The timer showed nothing. expect( - seen.map((publication) => publication.changes.map((change) => change.type === "document" && change.ops)), + seen.map((publication) => + publication.changes.map((change) => change.type === "document" && change.kind === "change" && change.ops), + ), ).toEqual([ [[["a", ["text"], "a"]]], [[["a", ["text"], "b"]]], @@ -631,10 +638,11 @@ describe("crash", () => { it("leaves Storage at the last stored state, which memory had, and nothing held after reopen", async () => { const { session, storage, conversationId, recordId, fire } = await open(); const states: string[] = []; - session.observeCommits((publication) => { + session.followCommits((publication) => { for (const change of publication.changes) { // Publications carry untyped document values; this one is `TextDoc`'s. - if (change.type === "document" && change.record.id === recordId) states.push((change.value as Text).text); + if (change.type === "document" && change.kind === "change" && change.record.id === recordId) + states.push((change.value as Text).text); } }); await append(session, conversationId, "a"); diff --git a/packages/durable/test/session-document-retention.test.ts b/packages/durable/test/session-document-retention.test.ts index 56cf9bd7d..639d695b8 100644 --- a/packages/durable/test/session-document-retention.test.ts +++ b/packages/durable/test/session-document-retention.test.ts @@ -18,7 +18,7 @@ import { SessionImpl, setDefaultDocumentRetention } from "../src/session/session import { chatSetup, openChat } from "./chat-support.ts"; import { addTool } from "./harness-support.ts"; import { viewHistory } from "./live-support.ts"; -import { ControlledStorage, context, createConversation } from "./session-support.ts"; +import { ControlledStorage, context, createConversation, onValue } from "./session-support.ts"; import { deferred, eventually } from "./task-support.ts"; type Counter = { count: number }; @@ -155,9 +155,11 @@ describe("document cache retention", () => { const kept = (await session.watchDoc(CounterDoc, conversationId, context))!; kept.start(async () => {}); const seen: number[] = []; - watch.start(async ({ value }) => { - seen.push(value.count); - }); + watch.start( + onValue(async ({ value }) => { + seen.push(value.count); + }), + ); clock.now += 1_000_000; await bump(session, conversationId); await settleLine(session); diff --git a/packages/durable/test/session-documents.test.ts b/packages/durable/test/session-documents.test.ts index b92f67776..0ae614d1d 100644 --- a/packages/durable/test/session-documents.test.ts +++ b/packages/durable/test/session-documents.test.ts @@ -8,7 +8,14 @@ import { } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; import { idFromNumber } from "../src/ids.ts"; -import { context, createConversation, documentChanges, flush, openTestSession } from "./session-support.ts"; +import { + context, + createConversation, + documentChanges, + documentValues, + flush, + openTestSession, +} from "./session-support.ts"; type Live = { message?: string; items: string[]; nested: { count: number }; other: { label: string } }; @@ -103,7 +110,7 @@ describe("Session document transactions", () => { expect(snapshot).toEqual({ message: "hello", items: [], nested: { count: 0 }, other: { label: "x" } }); await flush(); const publication = publications.at(-1)!; - const published = documentChanges(publication)[0]!; + const published = documentValues(publication)[0]!; expect(published.record.createdAt).toBe(publication.seq); expect(published.value).toBe(snapshot); expect(published.conversationId).toBe(conversationId); @@ -148,7 +155,7 @@ describe("Session document transactions", () => { }, context); await flush(); const snapshot = (await session.snapshot(LiveDoc, conversationId, context))!; - const published = documentChanges(publications.at(-1)!)[0]!; + const published = documentValues(publications.at(-1)!)[0]!; expect(published.value).toBe(snapshot); const admitted = storage.admittedCommits.at(-1)!.find((write) => write.type === "document.change")!; expect(admitted.content.kind).toBe("delta"); @@ -206,7 +213,7 @@ describe("Session document transactions", () => { const after = (await session.snapshot(LiveDoc, conversationId, context))!; expect(after).toEqual(before); expect(after).not.toBe(before); - expect(documentChanges(publications.at(-1)!)[0]!.ops.length).toBeGreaterThan(0); + expect(documentValues(publications.at(-1)!)[0]!.ops.length).toBeGreaterThan(0); }); it("revokes escaped drafts when the callback settles", async () => { @@ -506,12 +513,14 @@ describe("Session document transactions", () => { await flush(); const publication = publications.at(-1)!; const [retired, created] = documentChanges(publication); - expect(retired).toMatchObject({ record: { id: oldId }, value: null, ops: [] }); - expect(retired!.record.retiredAt).toBe(publication.seq); + expect(retired).toEqual({ type: "document", kind: "retire", record: retired!.record, conversationId }); + expect(retired!.record).toMatchObject({ id: oldId, retiredAt: publication.seq }); expect(created!.record.id).not.toBe(oldId); expect(created!.record.createdAt).toBe(publication.seq); - expect(created).toMatchObject({ value: { message: "new", items: [] }, ops: [] }); - expect(await session.snapshot(LiveDoc, conversationId, context)).toBe(created!.value); + expect(created).toMatchObject({ kind: "change", value: { message: "new", items: [] }, ops: [] }); + expect(await session.snapshot(LiveDoc, conversationId, context)).toBe( + created?.kind === "change" ? created.value : undefined, + ); await session.commit((tx) => tx.retireDoc(LiveDoc, conversationId), context); expect(await session.snapshot(LiveDoc, conversationId, context)).toBeUndefined(); @@ -613,8 +622,11 @@ describe("Session document transactions", () => { const { session, publications, conversationId } = await setupLive(); const published = publications.length; let listenerContext: typeof context | undefined; - const unsubscribe = session.subscribeCommits((_publication, deliveredContext) => { - listenerContext = deliveredContext; + const observation = session.observeCommits({ + advance: (_publication, deliveredContext) => { + listenerContext = deliveredContext; + }, + end: () => {}, }); const result = await session.commit(async (tx) => { (await tx.doc(LiveDoc, conversationId)).message = "m"; @@ -623,7 +635,7 @@ describe("Session document transactions", () => { expect(result).toBe("done"); expect(listenerContext).toBe(context); expect(publications.length).toBe(published + 1); - unsubscribe(); + observation.detach(); }); it("publishes close synchronously and supports unsubscription", async () => { diff --git a/packages/durable/test/session-forks.test.ts b/packages/durable/test/session-forks.test.ts index 3c541b548..0755abe57 100644 --- a/packages/durable/test/session-forks.test.ts +++ b/packages/durable/test/session-forks.test.ts @@ -595,7 +595,7 @@ describe("Session conversation document forks", () => { session.commit((tx) => tx.createConversation({ ownership: { kind: "ownerless" } }), context), ).rejects.toMatchObject({ name: "SessionFailed", cause: rejection }); // The failed Session closed its Storage; reopened, it holds nothing of the batch. - expect(await session.closed).toEqual({ reason: "failed", error: rejection }); + expect(await session.closed).toEqual({ kind: "failed", error: rejection }); expect(await storage.reopen().conversation(rejectedChildId, context)).toBeUndefined(); }); diff --git a/packages/durable/test/session-observers.test.ts b/packages/durable/test/session-observers.test.ts index c0d854bb9..0dcbc324a 100644 --- a/packages/durable/test/session-observers.test.ts +++ b/packages/durable/test/session-observers.test.ts @@ -1,9 +1,10 @@ // Tests for `Session.observeDoc()`, the synchronous observer of one document address that watches are built on. import { setFlagsFromString } from "node:v8"; import { runInNewContext } from "node:vm"; -import type { WatchChange } from "@earendil-works/chord"; +import type { WatchCall, WatchEnd } from "@earendil-works/chord"; import { withAbortSignal } from "@earendil-works/chord/context"; import { + type CommitObserver, type ConversationId, defineDoc, MemoryStorage, @@ -40,7 +41,7 @@ const ConversationCounter = defineDoc({ class ReportingSession extends SessionImpl { readonly reports: { readonly error: unknown; readonly info: ReportInfo }[] = []; - override report(error: unknown, info: ReportInfo = { type: "listener_error" }): void { + override report(error: unknown, info: ReportInfo = { kind: "listener_error" }): void { this.reports.push({ error, info }); } } @@ -48,15 +49,14 @@ class ReportingSession extends SessionImpl { /** An observer that records each call, and whether it ran inside the storage's `adopt`. */ function recorder(storage: ControlledStorage) { const calls: Array< - | { readonly advance: WatchChange>; readonly adopting: boolean } - | { readonly end: { readonly error: unknown } | undefined } + { readonly advance: WatchCall>; readonly adopting: boolean } | { readonly end: WatchEnd } > = []; const observer: Observer> = { advance: (change) => { calls.push({ advance: change, adopting: storage.adopting }); }, - end: (failure) => { - calls.push({ end: failure }); + end: (end) => { + calls.push({ end }); }, }; return { calls, observer }; @@ -93,7 +93,10 @@ describe("observeDoc", () => { const committing = add(session, 2); await committing; expect(calls).toEqual([ - { advance: { value: { count: 3 }, ops: [["s", ["count"], 3]], retired: false }, adopting: true }, + { + advance: { kind: "value", value: { count: 3 }, ops: [["s", ["count"], 3]], retired: false }, + adopting: true, + }, ]); observation.detach(); await session.close(context); @@ -117,7 +120,10 @@ describe("observeDoc", () => { ); expect(storage.commits).toHaveLength(stored); expect(calls).toEqual([ - { advance: { value: { count: 5 }, ops: [["s", ["count"], 5]], retired: false }, adopting: false }, + { + advance: { kind: "value", value: { count: 5 }, ops: [["s", ["count"], 5]], retired: false }, + adopting: false, + }, ]); observation.detach(); await session.close(context); @@ -140,10 +146,15 @@ describe("observeDoc", () => { (await tx.doc(ConversationCounter, conversationId)).count = 9; }, context); expect(calls).toEqual([ - { advance: { value: observation.value, ops: [], retired: true }, adopting: true }, - { advance: { value: { count: 9 }, ops: [["r", { count: 9 }]], retired: false }, adopting: true }, + { advance: { kind: "value", value: observation.value, ops: [], retired: true }, adopting: true }, + { + advance: { kind: "value", value: { count: 9 }, ops: [["r", { count: 9 }]], retired: false }, + adopting: true, + }, ]); - expect((calls[0] as { advance: WatchChange }).advance.value).toBe(observation.value); + expect((calls[0] as { advance: Extract, { kind: "value" }> }).advance.value).toBe( + observation.value, + ); observation.detach(); await session.close(context); }); @@ -168,7 +179,7 @@ describe("observeDoc", () => { const { calls, observer } = recorder(storage); await session.observeDoc(SessionCounter, observer, context); await session.close(context); - expect(calls).toEqual([{ end: undefined }]); + expect(calls).toEqual([{ end: { kind: "ended" } }]); }); it("ends with the error that failed the Session", async () => { @@ -180,7 +191,7 @@ describe("observeDoc", () => { storage.failNextCommit(failure); await expect(add(session)).rejects.toThrow("disk gone"); await session.closed; - expect(calls).toEqual([{ end: { error: failure } }]); + expect(calls).toEqual([{ end: { kind: "failed", error: failure } }]); }); it("reports a throwing observer, detaches it and ends it with the error; the commit and other observers go on", async () => { @@ -192,15 +203,15 @@ describe("observeDoc", () => { advance: () => { throw bug; }, - end: (failure) => ends.push(failure), + end: (end) => ends.push(end), }; await session.observeDoc(SessionCounter, throwing, context); const healthy = recorder(storage); await session.observeDoc(SessionCounter, healthy.observer, context); await add(session); await add(session); - expect(session.reports).toEqual([{ error: bug, info: { type: "listener_error" } }]); - expect(ends).toEqual([{ error: bug }]); + expect(session.reports).toEqual([{ error: bug, info: { kind: "listener_error" } }]); + expect(ends).toEqual([{ kind: "listener_error", error: bug }]); expect(healthy.calls).toHaveLength(2); expect(await session.snapshot(SessionCounter, context)).toEqual({ count: 3 }); await session.close(context); @@ -299,7 +310,7 @@ describe("observeDoc", () => { conversationId, { advance: (change) => { - if (!change.retired) last = new WeakRef(change.value); + if (change.kind === "value" && !change.retired) last = new WeakRef(change.value); }, end: () => {}, }, @@ -320,6 +331,103 @@ describe("observeDoc", () => { }); }); +describe("observeDoc throwing asynchronously", () => { + it("reports an observer whose call returns a rejected promise, then detaches it and ends it `listener_error`", async () => { + const { session } = open(); + await add(session); + const bug = new Error("async observer bug"); + const ends: WatchEnd[] = []; + let calls = 0; + const observer = { + advance: async () => { + calls++; + throw bug; + }, + end: (end: WatchEnd) => ends.push(end), + }; + await session.observeDoc(SessionCounter, observer, context); + await add(session); + await flush(); + await add(session); + expect(calls).toBe(1); + expect(session.reports).toEqual([{ error: bug, info: { kind: "listener_error" } }]); + expect(ends).toEqual([{ kind: "listener_error", error: bug }]); + await session.close(context); + expect(ends).toHaveLength(1); + }); +}); + +describe("observeCommits", () => { + /** A commit observer recording each seq and its end. */ + function commits() { + const calls: Array<{ readonly seq: number } | { readonly end: WatchEnd }> = []; + const observer: CommitObserver = { + advance: (publication) => calls.push({ seq: publication.seq }), + end: (end) => calls.push({ end }), + }; + return { calls, observer }; + } + + it("gets every stored commit, then `ended` once the Session closed, after its last publication", async () => { + const { session } = open(); + const { calls, observer } = commits(); + session.observeCommits(observer); + await add(session); + await add(session); + await session.close(context); + expect(calls).toEqual([{ seq: expect.any(Number) }, { seq: expect.any(Number) }, { end: { kind: "ended" } }]); + }); + + it("ends `failed` with the error that failed the Session", async () => { + const { storage, session } = open(); + const { calls, observer } = commits(); + session.observeCommits(observer); + const failure = new Error("disk gone"); + storage.failNextCommit(failure); + await expect(add(session)).rejects.toThrow("disk gone"); + await session.closed; + expect(calls).toEqual([{ end: { kind: "failed", error: failure } }]); + }); + + it("gets nothing after detach, which is idempotent, not even its end", async () => { + const { session } = open(); + const { calls, observer } = commits(); + const observation = session.observeCommits(observer); + await add(session); + await flush(); + observation.detach(); + observation.detach(); + await add(session); + await session.close(context); + expect(calls).toHaveLength(1); + }); + + it("reports a throwing observer, detaches it and ends it `listener_error`; the others go on", async () => { + const { session } = open(); + const bug = new Error("commit observer bug"); + const ends: WatchEnd[] = []; + let thrown = 0; + session.observeCommits({ + advance: () => { + thrown++; + throw bug; + }, + end: (end) => ends.push(end), + }); + const healthy = commits(); + session.observeCommits(healthy.observer); + await add(session); + await add(session); + await flush(); + expect(thrown).toBe(1); + expect(session.reports).toEqual([{ error: bug, info: { kind: "listener_error" } }]); + expect(ends).toEqual([{ kind: "listener_error", error: bug }]); + expect(healthy.calls).toHaveLength(2); + await session.close(context); + expect(ends).toHaveLength(1); + }); +}); + describe("observeDoc detached during a commit", () => { /** A Session with a conversation counter at 1, and the commit that retires it and creates it again at 2. */ async function retiring() { @@ -339,7 +447,7 @@ describe("observeDoc detached during a commit", () => { it("gets nothing more of the commit once it throws on the retirement, and pins nothing for the recreation", async () => { const { session, conversationId, recreate } = await retiring(); const bug = new Error("observer bug"); - const changes: WatchChange>[] = []; + const changes: WatchCall>[] = []; const ends: unknown[] = []; const observation = (await session.observeDoc( ConversationCounter, @@ -349,14 +457,14 @@ describe("observeDoc detached during a commit", () => { changes.push(change); throw bug; }, - end: (failure) => ends.push(failure), + end: (end) => ends.push(end), }, context, ))!; await recreate(); - expect(changes).toEqual([{ value: observation.value, ops: [], retired: true }]); - expect(ends).toEqual([{ error: bug }]); - expect(session.reports).toEqual([{ error: bug, info: { type: "listener_error" } }]); + expect(changes).toEqual([{ kind: "value", value: observation.value, ops: [], retired: true }]); + expect(ends).toEqual([{ kind: "listener_error", error: bug }]); + expect(session.reports).toEqual([{ error: bug, info: { kind: "listener_error" } }]); expect(session.documentCacheSizes.attached).toBe(0); observation.detach(); expect(session.documentCacheSizes.attached).toBe(0); @@ -365,7 +473,7 @@ describe("observeDoc detached during a commit", () => { it("gets nothing more of the commit once it detaches in the retirement call, and pins nothing", async () => { const { session, conversationId, recreate } = await retiring(); - const changes: WatchChange>[] = []; + const changes: WatchCall>[] = []; const ends: unknown[] = []; let observation: Observation> | undefined; observation = await session.observeDoc( @@ -376,12 +484,12 @@ describe("observeDoc detached during a commit", () => { changes.push(change); observation?.detach(); }, - end: (failure) => ends.push(failure), + end: (end) => ends.push(end), }, context, ); await recreate(); - expect(changes).toEqual([{ value: observation!.value, ops: [], retired: true }]); + expect(changes).toEqual([{ kind: "value", value: observation!.value, ops: [], retired: true }]); expect(session.documentCacheSizes.attached).toBe(0); await session.close(context); expect(ends).toEqual([]); diff --git a/packages/durable/test/session-support.ts b/packages/durable/test/session-support.ts index 98e3a54ce..f31e4b266 100644 --- a/packages/durable/test/session-support.ts +++ b/packages/durable/test/session-support.ts @@ -19,6 +19,19 @@ import { type ObservedCommit, SessionImpl } from "../src/session/session.ts"; export const context: Context = BACKGROUND_CONTEXT; +/** + * A watch listener for value calls: Durable's watches never send a `failed` call (their failures end them), so one here + * throws, which ends the watch `listener_error`. + */ +export function onValue( + listener: (call: Extract, context: Context) => Promise, +): (call: C, context: Context) => Promise { + return (call, deliveryContext) => { + if (call.kind !== "value") throw new Error(`Unexpected ${call.kind} watch call`); + return listener(call as Extract, deliveryContext); + }; +} + type Deferred = { readonly promise: Promise; readonly resolve: () => void }; function deferred(): Deferred { @@ -182,8 +195,11 @@ export function openTestSession(): { // These tests pin the cache's identities and loads, so documents stay as long as the default keeps them. const session = new SessionImpl(storage, Date.now, () => 600_000); const publications: CommitPublication[] = []; - session.subscribeCommits((publication) => { - publications.push(publication); + session.observeCommits({ + advance: (publication) => { + publications.push(publication); + }, + end: () => {}, }); return { storage, session, publications }; } @@ -197,15 +213,25 @@ export function documentChanges(publication: { } /** - * Observe every commit as memory adopts it, deferred ones before they are stored, as views and document watches do; - * `subscribeCommits()` sees only what was stored. + * Follow every commit as memory adopts it, deferred ones before they are stored, as views and document watches do; + * `observeCommits()` sees only what was stored. */ -export function observeCommits( +export function followCommits( session: Session, listener: (publication: ObservedCommit, context: Context) => void, ): () => void { - // `observeCommits()` is internal, so the public type lacks it; every Session and Harness is a `SessionImpl`. - return (session as unknown as SessionImpl).observeCommits(listener); + // `followCommits()` is internal, so the public type lacks it; every Session and Harness is a `SessionImpl`. + return (session as unknown as SessionImpl).followCommits(listener); +} + +/** The document changes of `publication` that carry a value: creations and updates, no retirements. */ +export function documentValues(publication: { + readonly changes: readonly CommitChange[]; +}): readonly Extract[] { + return publication.changes.filter( + (change): change is Extract => + change.type === "document" && change.kind === "change", + ); } export function documentCopyChanges( diff --git a/packages/durable/test/session-tables.test.ts b/packages/durable/test/session-tables.test.ts index fe017504f..3713b5cb9 100644 --- a/packages/durable/test/session-tables.test.ts +++ b/packages/durable/test/session-tables.test.ts @@ -432,8 +432,7 @@ describe("Session transaction tables", () => { const publication = publications.at(-1)!; const documents = documentChanges(publication); expect(publication.changes.filter((change) => change.type === "task")).toHaveLength(1); - expect(documents.map((document) => document.value)).toEqual([null, null, null]); - for (const document of documents) expect(document.ops).toEqual([]); + expect(documents.map((document) => document.kind)).toEqual(["retire", "retire", "retire"]); for (const document of documents) expect(document.conversationId).toBe(conversationId); expect(await session.snapshot(ProgressDoc, taskId, context)).toBeUndefined(); expect(await session.snapshot(StepDoc, taskId, "committed", context)).toBeUndefined(); diff --git a/packages/durable/test/session-watches.test.ts b/packages/durable/test/session-watches.test.ts index cb06268a7..256c27b51 100644 --- a/packages/durable/test/session-watches.test.ts +++ b/packages/durable/test/session-watches.test.ts @@ -1,6 +1,6 @@ import { setFlagsFromString } from "node:v8"; import { runInNewContext } from "node:vm"; -import type { Context, WatchChange, WatchEnd } from "@earendil-works/chord"; +import type { Context, WatchCall, WatchEnd } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT, createContextKey, @@ -12,7 +12,7 @@ import { applyImmutable } from "@earendil-works/chord/delta"; import { type DocumentWatch, defineDoc, defineDocFamily, MemoryStorage } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; import { SessionImpl } from "../src/session/session.ts"; -import { context, createConversation, documentChanges, flush, openTestSession } from "./session-support.ts"; +import { context, createConversation, documentValues, flush, onValue, openTestSession } from "./session-support.ts"; type State = { value: number; items: string[]; retained: { label: string } }; @@ -76,21 +76,23 @@ describe("Session document watches", () => { (await tx.doc(StateDoc)).value = 1; }, context); await flush(); - const firstPublished = documentChanges(publications.at(-1)!)[0]!; + const firstPublished = documentValues(publications.at(-1)!)[0]!; await session.commit(async (tx) => { (await tx.doc(StateDoc)).value = 2; }, context); await flush(); - const secondPublished = documentChanges(publications.at(-1)!)[0]!; + const secondPublished = documentValues(publications.at(-1)!)[0]!; expect(watch.value).toBe(initial); let inline = true; const deliveries: Array<{ value: Readonly; ops: unknown }> = []; - watch.start(async ({ value, ops }) => { - expect(inline).toBe(false); - expect(watch.value).toBe(value); - deliveries.push({ value, ops }); - }); + watch.start( + onValue(async ({ value, ops }) => { + expect(inline).toBe(false); + expect(watch.value).toBe(value); + deliveries.push({ value, ops }); + }), + ); inline = false; expect(deliveries).toHaveLength(0); await flush(); @@ -105,7 +107,7 @@ describe("Session document watches", () => { (await tx.doc(StateDoc)).value = 3; }, context); await flush(); - const thirdPublished = documentChanges(publications.at(-1)!)[0]!; + const thirdPublished = documentValues(publications.at(-1)!)[0]!; expect(deliveries).toHaveLength(2); expect(deliveries[1]!.value).toBe(thirdPublished.value); expect(deliveries[1]!.ops).toBe(thirdPublished.ops); @@ -121,17 +123,19 @@ describe("Session document watches", () => { const batches: unknown[] = []; let active = 0; let maxActive = 0; - watch.start(async ({ value, ops }) => { - active++; - maxActive = Math.max(maxActive, active); - values.push(value.value); - batches.push(ops); - if (values.length === 1) { - entered.resolve(); - await release.promise; - } - active--; - }); + watch.start( + onValue(async ({ value, ops }) => { + active++; + maxActive = Math.max(maxActive, active); + values.push(value.value); + batches.push(ops); + if (values.length === 1) { + entered.resolve(); + await release.promise; + } + active--; + }), + ); await session.commit(async (tx) => { (await tx.doc(StateDoc)).value = 1; }, context); @@ -155,17 +159,19 @@ describe("Session document watches", () => { const watch = (await session.watchDoc(StateDoc, context))!; const completed = deferred(); const values: number[] = []; - watch.start(async ({ value }) => { - const current = value.value; - values.push(current); - if (current === 1) { - await session.commit(async (tx) => { - (await tx.doc(StateDoc)).value = 2; - }, context); - } else { - completed.resolve(); - } - }); + watch.start( + onValue(async ({ value }) => { + const current = value.value; + values.push(current); + if (current === 1) { + await session.commit(async (tx) => { + (await tx.doc(StateDoc)).value = 2; + }, context); + } else { + completed.resolve(); + } + }), + ); await session.commit(async (tx) => { (await tx.doc(StateDoc)).value = 1; }, context); @@ -183,9 +189,11 @@ describe("Session document watches", () => { }, context); } const deliveries: Array<{ value: number; ops: unknown }> = []; - watch.start(async ({ value, ops }) => { - deliveries.push({ value: value.value, ops }); - }); + watch.start( + onValue(async ({ value, ops }) => { + deliveries.push({ value: value.value, ops }); + }), + ); await flush(); expect(deliveries).toEqual([{ value: 101, ops: [["s", ["value"], 101]] }]); await watch.stop(); @@ -207,9 +215,11 @@ describe("Session document watches", () => { }, context); } const deliveries: Array<{ value: Readonly; ops: unknown }> = []; - watch.start(async ({ value, ops }) => { - deliveries.push({ value, ops }); - }); + watch.start( + onValue(async ({ value, ops }) => { + deliveries.push({ value, ops }); + }), + ); await flush(); expect(deliveries).toEqual([{ value: watch.value, ops: [["r", watch.value]] }]); expect(Object.keys(watch.value.keys)).toHaveLength(10_001); @@ -222,13 +232,15 @@ describe("Session document watches", () => { const entered = deferred(); const release = deferred(); const deliveries: Array<{ value: number; ops: unknown }> = []; - watch.start(async ({ value, ops }) => { - deliveries.push({ value: value.value, ops }); - if (deliveries.length === 1) { - entered.resolve(); - await release.promise; - } - }); + watch.start( + onValue(async ({ value, ops }) => { + deliveries.push({ value: value.value, ops }); + if (deliveries.length === 1) { + entered.resolve(); + await release.promise; + } + }), + ); await session.commit(async (tx) => { (await tx.doc(StateDoc)).value = 1; }, context); @@ -256,19 +268,24 @@ describe("Session document watches", () => { } const last = (await session.snapshot(StateDoc, context))!; await session.commit((tx) => tx.retireDoc(StateDoc), context); - const changes: WatchChange>[] = []; + const changes: WatchCall>[] = []; watch.start(async (change) => { changes.push(change); }); await flush(); - expect(changes).toEqual([{ value: last, ops: [["s", ["value"], 100]], retired: true }]); - expect(changes[0]!.value).toBe(last); + expect(changes).toEqual([{ kind: "value", value: last, ops: [["s", ["value"], 100]], retired: true }]); + expect((changes[0] as { value: unknown }).value).toBe(last); expect(watch.value).toBe(last); await session.commit(async (tx) => { (await tx.doc(StateDoc)).value = 7; }, context); await flush(); - expect(changes[1]).toEqual({ value: { ...last, value: 7 }, ops: [["r", { ...last, value: 7 }]], retired: false }); + expect(changes[1]).toEqual({ + kind: "value", + value: { ...last, value: 7 }, + ops: [["r", { ...last, value: 7 }]], + retired: false, + }); await watch.stop(); }); @@ -285,10 +302,12 @@ describe("Session document watches", () => { expect(newest).not.toBe(initial); expect(newest).toEqual(initial); const batches: unknown[] = []; - watch.start(async ({ value, ops }) => { - expect(value).toBe(newest); - batches.push(ops); - }); + watch.start( + onValue(async ({ value, ops }) => { + expect(value).toBe(newest); + batches.push(ops); + }), + ); await flush(); expect(batches).toHaveLength(1); expect((batches[0] as readonly unknown[]).length).toBeGreaterThan(0); @@ -321,7 +340,7 @@ describe("Session document watches", () => { expect(deliveryContext!.abortSignal).toBeUndefined(); parentController.abort("caller finished"); const stopped = watch.stop(); - expect(await stopped).toEqual({ reason: "stopped" }); + expect(await stopped).toEqual({ kind: "stopped" }); expect(deliveryContext!.abortSignal).toBeUndefined(); release.resolve(); }); @@ -331,9 +350,11 @@ describe("Session document watches", () => { const watch = (await session.watchDoc(StateDoc, context))!; const initial = watch.value; let delivered: Readonly | undefined; - watch.start(async ({ value }) => { - delivered = value; - }); + watch.start( + onValue(async ({ value }) => { + delivered = value; + }), + ); await session.commit(async (tx) => { (await tx.doc(StateDoc)).value = 1; }, context); @@ -349,7 +370,7 @@ describe("Session document watches", () => { const { session, storage } = await createState(); const watch = (await session.watchDoc(StateDoc, context))!; const before = watch.value; - const changes: WatchChange>[] = []; + const changes: WatchCall>[] = []; watch.start(async (change) => { changes.push(change); }); @@ -360,8 +381,8 @@ describe("Session document watches", () => { await flush(); // Storage got the change, then the retirement; the commit publishes the retirement only. expect(storage.commits.at(-1)!.map((write) => write.type)).toEqual(["document.change", "document.retire"]); - expect(changes).toEqual([{ value: before, ops: [], retired: true }]); - expect(changes[0]!.value).toBe(before); + expect(changes).toEqual([{ kind: "value", value: before, ops: [], retired: true }]); + expect((changes[0] as { value: unknown }).value).toBe(before); await watch.stop(); }); @@ -369,7 +390,7 @@ describe("Session document watches", () => { const { session } = await createState(); const watch = (await session.watchDoc(StateDoc, context))!; const settled = endOf(watch); - const changes: WatchChange>[] = []; + const changes: WatchCall>[] = []; watch.start(async (change) => { changes.push(change); }); @@ -380,12 +401,12 @@ describe("Session document watches", () => { await flush(); const recreated = (await session.snapshot(StateDoc, context))!; expect(changes).toHaveLength(1); - expect(changes[0]).toEqual({ value: recreated, ops: [["r", recreated]], retired: true }); + expect(changes[0]).toEqual({ kind: "value", value: recreated, ops: [["r", recreated]], retired: true }); await session.commit(async (tx) => { (await tx.doc(StateDoc)).value = 11; }, context); await flush(); - expect(changes[1]).toEqual({ value: watch.value, ops: [["s", ["value"], 11]], retired: false }); + expect(changes[1]).toEqual({ kind: "value", value: watch.value, ops: [["s", ["value"], 11]], retired: false }); expect(watch.value.value).toBe(11); expect(settled.end).toBeUndefined(); await watch.stop(); @@ -397,7 +418,7 @@ describe("Session document watches", () => { const baseline = watch.value; await session.commit((tx) => tx.retireDoc(StateDoc), context); await session.close(context); - expect(await watch.closed).toEqual({ reason: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); expect(watch.value).toBe(baseline); expect(() => watch.start(async () => undefined)).toThrow("stopped"); }); @@ -408,18 +429,20 @@ describe("Session document watches", () => { const entered = deferred(); const release = deferred(); const values: number[] = []; - watch.start(async ({ value }) => { - values.push(value.value); - entered.resolve(); - await release.promise; - }); + watch.start( + onValue(async ({ value }) => { + values.push(value.value); + entered.resolve(); + await release.promise; + }), + ); await session.commit(async (tx) => { (await tx.doc(StateDoc)).value = 1; }, context); await entered.promise; await session.commit((tx) => tx.retireDoc(StateDoc), context); await session.close(context); - expect(await watch.closed).toEqual({ reason: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); release.resolve(); await flush(); expect(values).toEqual([1]); @@ -433,7 +456,7 @@ describe("Session document watches", () => { const first = started.stop(); const second = started.stop(); expect(second).toBe(first); - expect(await first).toEqual({ reason: "stopped" }); + expect(await first).toEqual({ kind: "stopped" }); const stopped = (await session.watchDoc(StateDoc, context))!; await stopped.stop(); @@ -470,7 +493,7 @@ describe("Session document watches", () => { }, context); await entered.promise; child.cancel(); - expect(await watch.closed).toEqual({ reason: "cancelled" }); + expect(await watch.closed).toEqual({ kind: "aborted" }); expect(deliveryContext!.abortSignal).toBeUndefined(); release.resolve(); }); @@ -492,7 +515,7 @@ describe("Session document watches", () => { await entered.promise; await session.close(context); expect(deliveryContext!.abortSignal).toBeUndefined(); - expect(await watch.closed).toEqual({ reason: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); release.resolve(); }); @@ -511,8 +534,8 @@ describe("Session document watches", () => { (await tx.doc(StateDoc)).value = 1; }, context); const end = await failed.closed; - expect(end.reason).toBe("listener_error"); - if (end.reason === "listener_error") expect(end.error.message).toBe("listener failed"); + expect(end.kind).toBe("listener_error"); + if (end.kind === "listener_error") expect(end.error.message).toBe("listener failed"); await flush(); expect(healthyCalls).toBe(1); await healthy.stop(); @@ -542,9 +565,11 @@ describe("Session document watches", () => { expect(watch.value).toEqual({ value: 3, migrated: true }); expect(storage.commits).toHaveLength(commits); const values: number[] = []; - watch.start(async ({ value }) => { - values.push(value.value); - }); + watch.start( + onValue(async ({ value }) => { + values.push(value.value); + }), + ); await session.commit((tx) => tx.doc(Current).then(() => undefined), context); await flush(); expect(values).toEqual([]); @@ -560,10 +585,12 @@ describe("Session document watches", () => { const { session } = await createState(); const watch = (await session.watchDoc(StateDoc, context))!; let replica: Readonly = watch.value; - watch.start(async ({ value, ops }) => { - replica = applyImmutable(replica, ops); - expect(replica).toEqual(value); - }); + watch.start( + onValue(async ({ value, ops }) => { + replica = applyImmutable(replica, ops); + expect(replica).toEqual(value); + }), + ); await session.commit(async (tx) => { const state = await tx.doc(StateDoc); state.value = 3; @@ -620,7 +647,7 @@ describe("Session document watches", () => { }, context); const notes = (await session.watchDoc(Notes, watched, context))!; const member = (await session.watchDoc(Members, "watched", context))!; - const changes: WatchChange>[] = []; + const changes: WatchCall>[] = []; notes.start(async (change) => { changes.push(change); }); @@ -667,7 +694,7 @@ describe("Session document watches", () => { const { session } = openTestSession(); await session.commit((tx) => tx.doc(Old).then(() => undefined), context); const watch = (await session.watchDoc(Old, context))!; - const changes: WatchChange[] = []; + const changes: WatchCall[] = []; watch.start(async (change) => { changes.push(change); }); @@ -682,11 +709,12 @@ describe("Session document watches", () => { await flush(); expect(changes).toEqual([ { + kind: "value", value: { value: 5, migrated: false }, ops: [["r", { value: 5, migrated: false }]], retired: true, }, - { value: { value: 6, migrated: false }, ops: [["s", ["value"], 6]], retired: false }, + { kind: "value", value: { value: 6, migrated: false }, ops: [["s", ["value"], 6]], retired: false }, ]); await watch.stop(); await session.close(context); @@ -820,6 +848,6 @@ describe("Session document watches", () => { expect(stopped.listener.deref()).toBeUndefined(); // What it delivered last stays readable. expect(stopped.watch.value.value).toBe(1); - expect(await stopped.watch.closed).toEqual({ reason: "stopped" }); + expect(await stopped.watch.closed).toEqual({ kind: "stopped" }); }); }); diff --git a/packages/durable/test/spec-usage.test.ts b/packages/durable/test/spec-usage.test.ts index 5dc763339..c50cc255e 100644 --- a/packages/durable/test/spec-usage.test.ts +++ b/packages/durable/test/spec-usage.test.ts @@ -37,6 +37,7 @@ import { import { expect, expectTypeOf, it } from "vitest"; import type { ExecutionEnv } from "../src/env/index.ts"; import { CodingTools, createBashTool, createEditTool, createReadTool } from "../src/tools/index.ts"; +import { onValue } from "./session-support.ts"; // ─── Names the spec leaves to the application ──────────────────────────────── @@ -363,7 +364,7 @@ function examples() { ); const watch = await conversation.watch(context); render(transcript.merge(watch.value)); - watch.start(async ({ value }) => render(transcript.merge(value))); + watch.start(onValue(async ({ value }) => render(transcript.merge(value)))); // A marker scrolled into view: render(await transcript.loadEarlier(context)); // Reconnect after a drop: diff --git a/packages/durable/test/task-support.ts b/packages/durable/test/task-support.ts index 77cac5897..fc0258955 100644 --- a/packages/durable/test/task-support.ts +++ b/packages/durable/test/task-support.ts @@ -88,7 +88,7 @@ export async function openTasks( { models: createModels(), registry, - onReport: (error, info) => { + onReport: ({ error, ...info }) => { reports.push(error); infos.push(info); }, diff --git a/packages/durable/test/tool-output-bench.ts b/packages/durable/test/tool-output-bench.ts index 040c3c7b4..88fec589e 100644 --- a/packages/durable/test/tool-output-bench.ts +++ b/packages/durable/test/tool-output-bench.ts @@ -195,13 +195,17 @@ async function runScenario(scenario: Scenario): Promise { let liveOpBytes = 0; let outputSets = 0; const observe = (target: Harness): void => { - target.subscribeCommits((publication) => { - for (const change of publication.changes) { - if (change.type !== "document" || change.record.kind !== "pi.live") continue; - liveCommits++; - liveOpBytes += JSON.stringify(change.ops).length; - for (const op of change.ops) if (op[0] === "s" && op[1].at(-1) === "output") outputSets++; - } + target.observeCommits({ + advance: (publication) => { + for (const change of publication.changes) { + if (change.type !== "document" || change.kind !== "change" || change.record.kind !== "pi.live") + continue; + liveCommits++; + liveOpBytes += JSON.stringify(change.ops).length; + for (const op of change.ops) if (op[0] === "s" && op[1].at(-1) === "output") outputSets++; + } + }, + end: () => {}, }); }; observe(harness); diff --git a/packages/env/CHANGELOG.md b/packages/env/CHANGELOG.md index 4d2a1c0e1..372e1aaea 100644 --- a/packages/env/CHANGELOG.md +++ b/packages/env/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Breaking Changes + +- `RemoteWatcher` follows Durable's `FileWatcher`: `close(context)` is `stop(context)`, and its changes are tagged (`{ kind: "paths", paths }`, `{ kind: "overflow" }`, `{ kind: "failed", error }`). The daemon's `watch` protocol is unchanged. Migration: `await watcher.close(ctx)` becomes `await watcher.stop(ctx)`; test `change.kind` instead of `"overflow" in change` or `"error" in change`. + ## [1.4.0] - 2026-10-21 ## [1.3.0] - 2026-10-15 diff --git a/packages/env/src/remote-env.ts b/packages/env/src/remote-env.ts index f3740994b..c2656f747 100644 --- a/packages/env/src/remote-env.ts +++ b/packages/env/src/remote-env.ts @@ -840,7 +840,7 @@ export class RemoteExecutionEnv implements ExecutionEnv { const watcher = await RemoteWatcher.open(this.connection, resolved, onChange, this.#watchOptions); const afterOpen = abortResult(context.abortSignal); if (afterOpen) { - await watcher.close(context); + await watcher.stop(context); return afterOpen; } return ok(watcher); diff --git a/packages/env/src/watch.ts b/packages/env/src/watch.ts index 3ee9c0fd8..b80d41e2c 100644 --- a/packages/env/src/watch.ts +++ b/packages/env/src/watch.ts @@ -13,6 +13,9 @@ export interface RemoteWatchOptions { maxDirectories?: number; } +/** The one `overflow` change; it carries nothing else. */ +const OVERFLOW: FileChange = { kind: "overflow" }; + const RECONNECT_FIRST_MS = 1000; const RECONNECT_MAX_MS = 30_000; @@ -26,7 +29,7 @@ export class RemoteWatcher implements FileWatcher { readonly #onChange: (change: FileChange) => void; readonly #options: RemoteWatchOptions; #mode: "native" | "polling" = "native"; - #closed = false; + #stopped = false; #controller: AbortController | undefined; #request: Promise | undefined; #wake: (() => void) | undefined; @@ -59,16 +62,16 @@ export class RemoteWatcher implements FileWatcher { return this.#mode; } - async close(_context: Context): Promise { - if (this.#closed) return; - this.#closed = true; + async stop(_context: Context): Promise { + if (this.#stopped) return; + this.#stopped = true; this.#wake?.(); this.#controller?.abort(); await this.#request; } #deliver(change: FileChange): void { - if (this.#closed) return; + if (this.#stopped) return; try { this.#onChange(change); } catch { @@ -101,12 +104,13 @@ export class RemoteWatcher implements FileWatcher { resolve(); } else if (event.kind === "change") { if (event.mode === "polling") this.#mode = "polling"; - if (event.overflow === true) this.#deliver({ overflow: true }); - else this.#deliver({ paths: event.paths as string[] }); + // The daemon's wire format: `{ overflow: true }` or `{ paths }`. + if (event.overflow === true) this.#deliver(OVERFLOW); + else this.#deliver({ kind: "paths", paths: event.paths as string[] }); } else if (event.kind === "error") { const code = event.code === "permission_denied" ? "permission_denied" : "invalid"; - this.#deliver({ error: new FileError(code, String(event.message)) }); - this.#closed = true; + this.#deliver({ kind: "failed", error: new FileError(code, String(event.message)) }); + this.#stopped = true; } }, }, @@ -118,20 +122,20 @@ export class RemoteWatcher implements FileWatcher { reject(error); return; } - if (this.#closed) return; + if (this.#stopped) return; if (isConnectionLost(error)) { void this.#reconnect(); return; } - this.#deliver({ error: toFileError(error) }); - this.#closed = true; + this.#deliver({ kind: "failed", error: toFileError(error) }); + this.#stopped = true; }, ); }); } async #reconnect(): Promise { - for (let delay = RECONNECT_FIRST_MS; !this.#closed; delay = Math.min(delay * 2, RECONNECT_MAX_MS)) { + for (let delay = RECONNECT_FIRST_MS; !this.#stopped; delay = Math.min(delay * 2, RECONNECT_MAX_MS)) { await new Promise((resolve) => { const timer = setTimeout(resolve, delay); timer.unref(); @@ -141,16 +145,16 @@ export class RemoteWatcher implements FileWatcher { }; }); this.#wake = undefined; - if (this.#closed) return; + if (this.#stopped) return; try { await this.#start(); // Changes while disconnected were not seen. - this.#deliver({ overflow: true }); + this.#deliver(OVERFLOW); return; } catch (error) { if (isConnectionLost(error)) continue; - this.#deliver({ error: toFileError(error) }); - this.#closed = true; + this.#deliver({ kind: "failed", error: toFileError(error) }); + this.#stopped = true; return; } } From bb65e3e49710fbd14f10c7792d0eaa6e2b2c5923 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Tue, 27 Oct 2026 21:31:47 +0000 Subject: [PATCH 044/271] fix(durable): import toCodemodeIdentifier from pi-codemode/declarations The Code Mode extension imported it from /portable, which loads the WASM QuickJS runtime the extension never uses. closes #11254 --- packages/durable/CHANGELOG.md | 1 + packages/durable/src/codemode/codemode.ts | 14 +++++++------- packages/durable/test/codemode-imports.test.ts | 11 +++++++++++ 3 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 packages/durable/test/codemode-imports.test.ts diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index a913ce222..cf45dd551 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -123,6 +123,7 @@ - Fixed a commit that Storage rejects after adopting it (a failed JSONL reclaim) publishing nothing: it is published and its waits resolve before the Session fails, so a replica built from `subscribeCommits()` stays equal to Storage and `pi.durable.tasks.active` stays balanced. - Fixed a throwing telemetry adapter (a commit's `pi.durable.entry` or `pi.durable.submission` event, `pi.durable.tasks.active`) rejecting a durable commit and leaving the views and the scheduler without it: it is reported as `host_error` with callback `"telemetry"`, and the commit resolves. A throwing report metric or report event is dropped. - Fixed a host that crashes during a model request resending it forever, adding an aborted entry each time: the interrupted request is a failed attempt under the retry policy, and the run ends with `runFailure` reason `interrupted` once no retry is left; other work cut off `restartLimit` times ends instead of keeping its conversation busy ([#11139](https://github.com/earendil-works/pi/issues/11139)) +- Fixed the Code Mode extension loading pi-codemode's WASM QuickJS runtime (`quickjs-wasi`) although it only runs scripts in the sandbox the host passes: it imports `toCodemodeIdentifier` from `@earendil-works/pi-codemode/declarations` instead of `/portable` ([#11254](https://github.com/earendil-works/pi/issues/11254)) ## [1.4.0] - 2026-10-21 diff --git a/packages/durable/src/codemode/codemode.ts b/packages/durable/src/codemode/codemode.ts index dba1d1e57..2b308e2da 100644 --- a/packages/durable/src/codemode/codemode.ts +++ b/packages/durable/src/codemode/codemode.ts @@ -37,13 +37,13 @@ import { type Context, copyJson, type JsonValue } from "@earendil-works/chord"; import { ownedSlice } from "@earendil-works/chord/delta"; import type { TextContent } from "@earendil-works/pi-ai"; -import { - type CodemodeHostCall, - type CodemodeOutcome, - type CodemodeResult, - type CodemodeRun, - type CodemodeSandboxBase, - toCodemodeIdentifier, +import { toCodemodeIdentifier } from "@earendil-works/pi-codemode/declarations"; +import type { + CodemodeHostCall, + CodemodeOutcome, + CodemodeResult, + CodemodeRun, + CodemodeSandboxBase, } from "@earendil-works/pi-codemode/portable"; import { CODEMODE_SOURCE_GRAMMAR, parseCodemodeSource } from "@earendil-works/pi-codemode/source"; import { type Static, Type } from "typebox"; diff --git a/packages/durable/test/codemode-imports.test.ts b/packages/durable/test/codemode-imports.test.ts new file mode 100644 index 000000000..df74da0e2 --- /dev/null +++ b/packages/durable/test/codemode-imports.test.ts @@ -0,0 +1,11 @@ +import { expect, test, vi } from "vitest"; +import { createCodemodeExtension } from "../src/codemode/codemode.ts"; + +// The extension runs scripts in the sandbox the host passes, so loading it must not load the WASM VM (#11254). +vi.mock("quickjs-wasi", () => { + throw new Error("the Code Mode extension loaded quickjs-wasi"); +}); + +test("the Code Mode extension loads without pi-codemode's WASM runtime", () => { + expect(createCodemodeExtension).toBeTypeOf("function"); +}); From 85c2ee3e761e4fc4f901ff0ed584fb2465aa552e Mon Sep 17 00:00:00 2001 From: Christian Klotz Date: Wed, 28 Oct 2026 00:31:24 +0200 Subject: [PATCH 045/271] docs(coding-agent): restore the full extensions guide, corrected against current source (#11498) ## Summary The docs refresh (#9898) cut `docs/extensions.md` from about 3,100 lines to 130, dropping API reference and tool guidance. #11039 restored part of it. This PR restores the full guide and corrects it against current source. - **Structure:** the original sections, with the ASCII lifecycle diagram replaced by three Mermaid diagrams (run, turn, session) and ordered hook lists, plus a handler composition table. - **Merged:** content added after the cut: tool exposure, calling other tools, structured output, MCP servers, `registerToolRenderer`, virtual models, `provider_stream_event`, `before_agent_start` `messageType`. - **Corrected:** removed or renamed APIs (`modelRegistry.stream()`, prompt sections are ``/``, `builtin:read` paths, `highlightCode`), wrong behavior claims (errors only by throwing, `model_select` on restore, `sendUserMessage` always triggering a turn, override warning, `truncateLine` bytes), examples that did not type-check, links to removed `tui.md` patterns and `rpc.md` anchors, and wrong API lists in the examples table. - **Kept:** anchors that other docs and changelogs link to, as `` tags. - **Fixed example:** `reload-runtime.ts` tool queued `/reload-runtime` with `deliverAs: "followUp"`. Queued text reaches the model as a plain user message, so the command never ran and Pi never reloaded. The tool now sets a flag and returns `terminate: true`; an `agent_settled` handler sends `/reload-runtime` with `expandPromptTemplates: true`, which runs after the settled handlers finish, while the agent is idle. In the long run it may be better to revisit how API reference is, provided to the agent, especially since more capable models do explore the source without it. ## Validation - `npm run check` - Edited TypeScript examples type-check against current types - All relative links and anchors in `extensions.md` resolve, including inbound anchors from other docs - `reload-runtime.ts`: faux-provider harness test (temporary, not committed) shows one reload after the run settles and no extra model request; the old version never reloads. Also verified in interactive mode with a real model. - Reviewed by GPT-6 Astra and GPT-5.6 Sol against source; all findings applied --- packages/coding-agent/docs/extensions.md | 3347 ++++++++++++++++- .../examples/extensions/reload-runtime.ts | 22 +- 2 files changed, 3156 insertions(+), 213 deletions(-) diff --git a/packages/coding-agent/docs/extensions.md b/packages/coding-agent/docs/extensions.md index 7580137f6..2439d1b45 100644 --- a/packages/coding-agent/docs/extensions.md +++ b/packages/coding-agent/docs/extensions.md @@ -1,65 +1,311 @@ +> Pi can create extensions. Ask it to build one for your use case. + # Extensions -Extensions are TypeScript modules that add executable behavior to Pi. Use one when a workflow needs tools, commands, event handlers, model providers, session state, or terminal UI rather than instructions alone. +Extensions are TypeScript modules that extend Pi's behavior. They can subscribe to lifecycle events, register custom tools callable by the LLM, add commands, and more. An extension runs inside the Pi process with the same operating-system permissions. It can inspect prompts, tool calls, files, credentials, and session history, so load extensions only from sources you trust. -Typical extensions add an agent tool, protect paths, confirm dangerous commands, react to session events, modify context, expose a command, or display persistent status. +> **Placement for /reload:** Put extensions in `~/.pi/agent/extensions/` (global) or `.pi/extensions/` (project-local) for auto-discovery. Use `pi -e ./path.ts` only for quick tests. Extensions in auto-discovered locations can be hot-reloaded with `/reload`. + +**Key capabilities:** +- **Custom tools** - Register tools the LLM can call via `pi.registerTool()` +- **Event interception** - Block or modify tool calls, inject context, customize compaction +- **User interaction** - Prompt users via `ctx.ui` (select, confirm, input, notify) +- **Custom UI components** - Full TUI components with keyboard input via `ctx.ui.custom()` for complex interactions +- **Custom commands** - Register commands like `/mycommand` via `pi.registerCommand()` +- **Session persistence** - Store state that survives restarts via `pi.appendEntry()` +- **Custom rendering** - Control how tool calls/results and messages appear in TUI +- **Providers and MCP servers** - Register model providers, virtual models, and MCP servers + +**Example use cases:** +- Permission gates (confirm before `rm -rf`, `sudo`, etc.) +- Git checkpointing (stash at each turn, restore on branch) +- Path protection (block writes to `.env`, `node_modules/`) +- Custom compaction (summarize conversation your way) +- Conversation summaries (see `summarize.ts` example) +- Interactive tools (questions, wizards, custom dialogs) +- Stateful tools (todo lists, connection pools) +- External integrations (file watchers, webhooks, CI triggers) +- Games while you wait (see `snake.ts` example) + +See [examples/extensions/](../examples/extensions/) for working implementations. + +## Table of Contents + +- [Quick Start](#quick-start) +- [Extension Locations](#extension-locations) +- [Available Imports](#available-imports) +- [Writing an Extension](#writing-an-extension) + - [Extension Styles](#extension-styles) +- [Events](#events) + - [Lifecycle Overview](#lifecycle-overview) + - [Handler Composition](#handler-composition) + - [Startup Events](#startup-events) + - [Resource Events](#resource-events) + - [Session Events](#session-events) + - [Agent Events](#agent-events) + - [Model Events](#model-events) + - [Tool Events](#tool-events) + - [User Bash Events](#user-bash-events) + - [Input Events](#input-events) +- [ExtensionContext](#extensioncontext) +- [ExtensionCommandContext](#extensioncommandcontext) +- [ExtensionAPI Methods](#extensionapi-methods) +- [State Management](#state-management) +- [Custom Tools](#custom-tools) + - [Tool Exposure](#tool-exposure) + - [Calling Other Tools](#calling-other-tools) + - [Dynamic Tool Loading](#dynamic-tool-loading) +- [MCP Servers](#mcp-servers) +- [Custom UI](#custom-ui) +- [Error Handling](#error-handling) +- [Mode Behavior](#mode-behavior) +- [Examples Reference](#examples-reference) - - -## Create and load an extension +## Quick Start -An extension exports a default factory that receives `ExtensionAPI`. The factory registers capabilities for the current extension runtime. - -Create `~/.pi/agent/extensions/hello.ts`: +Create `~/.pi/agent/extensions/my-extension.ts`: ```typescript -import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { type ExtensionAPI, isToolCallEventType } from "@earendil-works/pi-coding-agent"; +import { Type } from "typebox"; export default function (pi: ExtensionAPI) { + // React to events + pi.on("session_start", async (_event, ctx) => { + ctx.ui.notify("Extension loaded!", "info"); + }); + + pi.on("tool_call", async (event, ctx) => { + if (isToolCallEventType("bash", event) && event.input.command.includes("rm -rf")) { + const ok = await ctx.ui.confirm("Dangerous!", "Allow rm -rf?"); + if (!ok) return { block: true, reason: "Blocked by user" }; + } + }); + + // Register a custom tool + pi.registerTool({ + name: "greet", + label: "Greet", + description: "Greet someone by name", + parameters: Type.Object({ + name: Type.String({ description: "Name to greet" }), + }), + async execute(toolCallId, params, signal, onUpdate, ctx) { + return { + content: [{ type: "text", text: `Hello, ${params.name}!` }], + details: {}, + }; + }, + }); + + // Register a command pi.registerCommand("hello", { - description: "Show a greeting", - handler: async (name, ctx) => { - ctx.ui.notify(`Hello, ${name || "world"}!`, "info"); + description: "Say hello", + handler: async (args, ctx) => { + ctx.ui.notify(`Hello ${args || "world"}!`, "info"); }, }); } ``` -Start Pi and run `/hello`. During development, load a file directly: +Test with `--extension` (or `-e`) flag: ```bash -pi --extension ./hello.ts +pi -e ./my-extension.ts ``` -Pi uses `jiti`, so local TypeScript extensions do not need a separate compilation step. Use [Pi packages](packages.md) for distributed extensions and dependencies. - - - -## Add it to Pi +## Extension Locations -Place the extension in your user or project extensions directory. Pi loads direct TypeScript or JavaScript files and subdirectories containing an `index.ts` or `index.js` entry point. +> **Security:** Extensions run with your full system permissions and can execute arbitrary code. Only install from sources you trust. -Use a single file for a small extension and a directory for a multi-file implementation. Put npm dependencies in a nearby `package.json`. See [Configuration](configuration.md) for conventional locations and [Settings](settings.md#resources) for additional paths. +Extensions are auto-discovered from trusted locations. Project-local `.pi/extensions` entries load only after the project is trusted. -Reload replaces the extension runtime, so code after `await ctx.reload()` must not reuse state from the old runtime. Only personal and explicit command-line extensions can participate in the `project_trust` event that runs before project extensions load. +| Location | Scope | +|----------|-------| +| `~/.pi/agent/extensions/*.ts` or `*.js` | Global (all projects) | +| `~/.pi/agent/extensions/*/index.ts` or `index.js` | Global (subdirectory) | +| `~/.pi/agent/extensions/*/package.json` with a `pi.extensions` field | Global (subdirectory with manifest) | +| `.pi/extensions/...` | Project-local, same layouts | + +Discovery goes one level deep. Larger packages declare their entry points in a `package.json` manifest. + +Additional paths via `settings.json`: + +```json +{ + "packages": [ + "npm:@foo/bar@1.0.0", + "git:github.com/user/repo@v1" + ], + "extensions": [ + "/path/to/local/extension.ts", + "/path/to/local/extension/dir" + ] +} +``` + +Settings also accept built-in extensions as `builtin:` entries; see [Settings](settings.md#resources). To share extensions via npm or git as Pi packages, see [packages.md](packages.md). + +## Available Imports + +| Package | Purpose | +|---------|---------| +| `@earendil-works/pi-coding-agent` | Extension types (`ExtensionAPI`, `ExtensionContext`, events) and helpers | +| `@earendil-works/pi-agent-core` | Agent message and tool types | +| `typebox`, `typebox/compile`, `typebox/value` | Schema definitions for tool parameters | +| `@earendil-works/pi-ai` | AI utilities (`StringEnum` for string enums, transcript helpers, model types) | +| `@earendil-works/pi-ai/compat`, `/models`, `/oauth`, `/providers/all` | Provider construction, built-in APIs, and OAuth helpers | +| `@earendil-works/pi-tui` | TUI components for custom rendering | + +Pi maps these imports to its own copies, so extensions share Pi's classes and registries. The legacy `@mariozechner/*` and `@sinclair/typebox` names map to the same modules. + +npm dependencies work too. Add a `package.json` next to your extension (or in a parent directory), run `npm install`, and imports from `node_modules/` are resolved automatically. + +For distributed Pi packages installed with `pi install` (npm or git), runtime deps must be in `dependencies`; `devDependencies` are not installed. Do not list the host-provided packages above in `dependencies`. See [Declare dependencies](packages.md#declare-dependencies). + +Node.js built-ins (`node:fs`, `node:path`, etc.) are also available. + +## Writing an Extension + +An extension exports a default factory function that receives `ExtensionAPI`. The factory can be synchronous or asynchronous: + +```typescript +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; + +export default function (pi: ExtensionAPI) { + // Subscribe to events + pi.on("event_name", async (event, ctx) => { + // ctx.ui for user interaction + const ok = await ctx.ui.confirm("Title", "Are you sure?"); + ctx.ui.notify("Done!", "info"); + ctx.ui.setStatus("my-ext", "Processing..."); // Footer status + ctx.ui.setWidget("my-ext", ["Line 1", "Line 2"]); // Widget above editor (default) + }); + + // Register tools, commands, shortcuts, flags + pi.registerTool({ ... }); + pi.registerCommand("name", { ... }); + pi.registerShortcut("ctrl+x", { ... }); + pi.registerFlag("my-flag", { ... }); +} +``` + +Extensions are loaded via [jiti](https://github.com/unjs/jiti), so TypeScript works without compilation. + +If the factory returns a `Promise`, Pi awaits it before continuing startup. That means async initialization completes before `session_start`, before `resources_discover`, and before provider registrations queued via `pi.registerProvider()` are flushed. + +### Async factory functions + +Use an async factory for one-time startup work such as fetching remote configuration or dynamically discovering available models. + +```typescript +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; + +export default async function (pi: ExtensionAPI) { + const response = await fetch("http://localhost:1234/v1/models"); + const payload = (await response.json()) as { + data: Array<{ + id: string; + name?: string; + context_window?: number; + max_tokens?: number; + }>; + }; + + pi.registerProvider("local-openai", { + baseUrl: "http://localhost:1234/v1", + apiKey: "$LOCAL_OPENAI_API_KEY", + api: "openai-completions", + models: payload.data.map((model) => ({ + id: model.id, + name: model.name ?? model.id, + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: model.context_window ?? 128000, + maxTokens: model.max_tokens ?? 4096, + })), + }); +} +``` + +This pattern makes the fetched models available during normal startup and to `pi --list-models`. For model lists that change while Pi runs, implement `refreshModels` instead; see [pi.registerProvider()](#piregisterprovidername-config). + +### Long-lived resources and shutdown + +Extension factories may run in invocations that never start a session. Do not start background resources such as processes, sockets, file watchers, or timers from the factory. + +Defer background resource startup until `session_start` or the command/tool/event that needs the resource. Register an idempotent `session_shutdown` handler to close any session-scoped resources you start. + +### Extension Styles + +**Single file** - simplest, for small extensions: + +``` +~/.pi/agent/extensions/ +└── my-extension.ts +``` + +**Directory with index.ts** - for multi-file extensions: + +``` +~/.pi/agent/extensions/ +└── my-extension/ + ├── index.ts # Entry point (exports default function) + ├── tools.ts # Helper module + └── utils.ts # Helper module +``` + +**Package with dependencies** - for extensions that need npm packages: + +``` +~/.pi/agent/extensions/ +└── my-extension/ + ├── package.json # Declares dependencies and entry points + ├── package-lock.json + ├── node_modules/ # After npm install + └── src/ + └── index.ts +``` + +```json +// package.json +{ + "name": "my-extension", + "dependencies": { + "zod": "^3.0.0", + "chalk": "^5.0.0" + }, + "pi": { + "extensions": ["./src/index.ts"] + } +} +``` + +Run `npm install` in the extension directory, then imports from `node_modules/` work automatically. + + + +## Events -## Understand the runtime lifecycle +### Lifecycle Overview -The factory can be synchronous or asynchronous. Pi waits for an asynchronous factory before startup continues, allowing it to fetch configuration or register providers needed during startup. +A prompt starts a run. Input handling comes first: -Do not start processes, sockets, watchers, or timers in the factory because some invocations load extensions without starting a session. -Start long-lived resources from `session_start` or from the command or tool that needs them. -Close session-scoped resources from an idempotent `session_shutdown` handler. +1. Extension commands (`/name`) run their handler. The `input` event does not fire. Typed and RPC prompts always check for commands; `pi.sendUserMessage()` checks only with `expandPromptTemplates: true`. +2. `input` handlers can intercept, transform, or handle the text. +3. Skill commands (`/skill:name`) and prompt templates (`/template`) expand. +4. `before_agent_start` runs, then the agent loop starts. -A normal prompt starts a run. The run repeats turns, then checks whether more work starts another low-level agent run: +The run repeats turns, then checks whether more work starts another low-level agent run: ```mermaid flowchart TD @@ -75,7 +321,7 @@ flowchart TD Continue -->|no| Settled["agent_settled"] ``` -More work means a retry, recovery, compaction retry, or queued work. +More work means a retry, final-attempt recovery, a compaction retry, or queued messages. `agent_end` is therefore not a final-idle signal; use `agent_settled` for that. An aborted run skips `agent_before_settle` and goes from `agent_end` to `agent_settled`. Pi counts as busy while a run starts, including the compaction check and `before_agent_start`, so messages sent meanwhile go into the starting run. Each turn sends one model request and executes the tool calls in the assistant response: @@ -93,36 +339,24 @@ flowchart TD The model request runs these hooks in order: -1. `context` -2. `context_with_system` -3. `before_provider_headers` -4. `before_provider_request` -5. `after_provider_response` -6. `provider_stream_event` and assistant message events +1. `context` (can modify conversation messages) +2. `context_with_system` (can modify the full transcript) +3. `before_provider_headers` (can mutate headers) +4. `before_provider_request` (can inspect or replace the payload) +5. `after_provider_response` (status and headers, before the stream is consumed) +6. `provider_stream_event` (raw parsed stream events) and `message_start` / `message_update` / `message_end` for the assistant message -Tool execution runs these hooks in order for each tool call: +A tool call that executes runs these hooks in order: 1. `tool_execution_start` -2. `tool_call` +2. `tool_call` (can block or mutate input) 3. `tool_execution_update` while the tool executes -4. `tool_result` +4. `tool_result` (can modify the result) 5. `tool_execution_end` -The lifecycle involves these event groups: +Calls that do not execute end early. An unknown tool or invalid arguments skip `tool_call`; a blocked or aborted call skips execution and `tool_result`. Each of these still gets `tool_execution_end` with an error result, so use `tool_execution_end` to observe every finalized call. -| Area | Events | -|---|---| -| Input preparation | `input`, `before_agent_start` | -| Agent run | `agent_start`, `agent_end` | -| Model turn | `turn_start`, `context`, `context_with_system`, `turn_end` | -| Provider call | `before_provider_headers`, `before_provider_request`, `after_provider_response`, `provider_stream_event` | -| Message lifecycle | `message_start`, `message_update`, `message_end` | -| Tool execution | `tool_execution_start`, `tool_call`, `tool_execution_update`, `tool_result`, `tool_execution_end` | -| Settlement | `agent_before_settle`, `agent_settled` | - -Pi counts as busy while a run starts, including the compaction check and `before_agent_start`, so messages sent meanwhile go into the starting run. Message lifecycle events also cover user and tool-result messages; the model request list places only the assistant stream where it occurs in each turn. - -Tool hooks run for each tool call, and one assistant response can start another turn by returning tool calls. Provider hooks run only when that transport exposes the corresponding stage. See the exported declarations in [`extensions/types.ts`](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/extensions/types.ts) for exact event payloads and result types. +`turn_end` can append entries and request a continuation. Threshold compaction runs before a naturally required next turn. Message events also fire for other messages, such as user, tool-result, and custom messages. Provider hooks run only when the transport exposes the corresponding stage. Session startup and replacement follow a separate lifecycle: @@ -134,73 +368,38 @@ flowchart TD Fork["/fork or /clone"] --> BeforeFork["session_before_fork"] BeforeSwitch --> Shutdown["session_shutdown"] BeforeFork --> Shutdown - Reload["reload"] --> Shutdown + Reload["/reload"] --> Shutdown + Exit["exit"] --> Shutdown Shutdown --> Replacing{"Replacing the runtime?"} - Replacing -->|yes| Replacement["load replacement extension runtime"] + Replacing -->|yes| ReplacementTrust["project_trust when the working directory is new"] + ReplacementTrust --> Replacement["load replacement extension runtime"] Replacement --> SessionStart Replacing -->|no| Stop["process exits"] SessionStart --> Discover["resources_discover"] ``` -`project_trust` runs before trusted project resources load and can run again when session replacement enters a new working directory. Retries, recovery, compaction, and queued messages can start another agent run after `agent_end`, so `agent_end` is not a final-idle signal. Reload and session replacement invalidate the old runtime. Code after `await ctx.reload()` still belongs to the old runtime and must not reuse its session-bound state. +`session_start` carries `reason: "startup" | "reload" | "new" | "resume" | "fork"`, and `session_shutdown` carries `reason: "quit" | "reload" | "new" | "resume" | "fork"`. `resources_discover` uses `reason: "reload"` after a reload and `"startup"` otherwise. Exit covers Ctrl+C, Ctrl+D, SIGHUP, and SIGTERM. `project_trust` runs before trusted project resources load and can run again when session replacement enters a new working directory. -### Choose an event by intent +Other operations change the current session without replacing the runtime: -| Goal | Event | +| Operation | Events | |---|---| -| Inspect, transform, or consume raw user input | `input` | -| Change prompt sections, active tools, or add context before a run | `before_agent_start` | -| Observe one low-level agent run | `agent_start`, `agent_end` | -| Act after one model response and its tool calls | `turn_end` | -| Filter conversation messages for one provider request | `context` | -| Transform the complete transcript, including system messages | `context_with_system` | -| Change request headers or the serialized provider payload | `before_provider_headers`, `before_provider_request` | -| Observe provider response metadata or parsed stream events | `after_provider_response`, `provider_stream_event` | -| Authorize or modify a tool call | `tool_call` | -| Redact, annotate, or replace a tool result | `tool_result` | -| Append final entries or request one last continuation | `agent_before_settle` | -| Run notification work after automatic processing has stopped | `agent_settled` | -| Initialize or release session-scoped resources | `session_start`, `session_shutdown` | +| `/compact` or automatic compaction | `session_before_compact` (can cancel or customize), then `session_compact` or `session_compact_failed` | +| `/tree` navigation | `session_before_tree` (can cancel or customize), then `session_tree` | +| `/name` or `pi.setSessionName()` | `session_info_changed` | +| `/model` or model cycling (`Ctrl+P`) | `thinking_level_select` if the level changes or is clamped, then `model_select` | +| Thinking-level change (settings, keybinding, `pi.setThinkingLevel()`) | `thinking_level_select` | +| `!` or `!!` command | `user_bash` (can intercept) | +| Extension dialog or custom screen waits for the user | `ui_prompt_start`, `ui_prompt_end` | +| Idle prompt-cache refresh | `cache_warming_decision` | +| Extension registers or removes an MCP server after startup | `mcp_servers_change` | - - -Use `agent_before_settle` when an extension must append entries or request one final continuation. Use `agent_settled` for notification-only work that should run after Pi has no automatic continuation remaining. - - - -## Choose an integration point - -| Capability | Main API | -|---|---| -| Observe or modify lifecycle behavior | `pi.on()` | -| Add a model-callable operation | `pi.registerTool()` | -| Add a `/` command | `pi.registerCommand()` | -| Add a shortcut or CLI flag | `pi.registerShortcut()` or `pi.registerFlag()` | -| Send user or custom messages | `pi.sendUserMessage()` or `pi.sendMessage()` | -| Persist non-context session data | `pi.appendEntry()` | -| Change active tools, model, or thinking level | Session control methods on `pi` | -| Add a model provider | `pi.registerProvider()` | -| Add an MCP server | `pi.registerMcpServer()` | -| Route each request to a model | [`pi.registerVirtualModel()`](virtual-models.md) | -| Add terminal rendering | Renderer registration and `ctx.ui` | -| Communicate with another extension | `pi.events` | - -Use the exported declarations in [`extensions/types.ts`](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/extensions/types.ts) for exact event, context, tool, and result types. - -## Follow the extension contracts - - + -### Events and concurrency +### Handler Composition -Handlers run in extension load and registration order. `pi.on()` returns a function that unsubscribes that registration; changes do not affect a dispatch already in progress. -Some events notify; others transform data, replace results, or cancel an operation. -Use each event’s declared result type rather than assuming every return value has an effect. - -Events cover resource discovery, sessions, agent and message lifecycle, providers, tools, and raw input. - -#### Understand handler composition +Handlers run in extension load order, then registration order. Some events notify; others transform data, replace results, or cancel an operation: | Pattern | Behavior | |---|---| @@ -213,56 +412,2315 @@ Events cover resource discovery, sessions, agent and message lifecycle, provider Notification handlers should not perform slow work on a streaming event. Transform handlers should preserve fields they do not intentionally own, and gate handlers should fail closed when their safety decision cannot be made. -`before_agent_start` exposes both the current prompt and its structured `systemPromptOptions`. `event.messageType` says what started the run: `"user"` for a prompt or `pi.sendUserMessage()`, with `prompt` and `images`; `"custom"` for `pi.sendMessage(..., { triggerTurn: true })`, with the starting `message` and no `prompt`. A returned `message` is added to either kind of run; check `messageType` to skip runs started by custom messages. Prefer changing prompt sections, selected tools, or guidelines so Pi can append a transcript delta. Returning `systemPrompt`, or setting `forceSystemPrompt`, replaces the whole prompt for that run while the transcript continues recording the structured sections. Providers receive the forced text as their leading system prompt. +### Startup Events -`message_end` can replace a finalized message while preserving its role. `tool_call` can mutate input or block execution. `tool_result` handlers compose, with each handler seeing prior changes. +#### project_trust - +Fired before Pi decides whether to trust a project with dynamic configs (`.pi` or `.agents/skills`). It runs during startup and when session replacement (for example `/resume`) enters a cwd whose trust has not been resolved in the current process. Only user/global extensions and CLI `-e` extensions participate; project-local extensions are not loaded until after trust is resolved. -`provider_stream_event` fires for each parsed provider stream event before Pi normalizes it. The event identifies the provider, API, and model; `event.data` is the earliest structured value available to Pi, not necessarily the original HTTP bytes or SSE frame. Treat it as read-only because mutation can affect normalization. The event is notification-only and is not persisted. +```typescript +pi.on("project_trust", async (event, ctx) => { + // event.cwd - current working directory + // ctx has a limited trust context: cwd, mode, hasUI, and select/confirm/input/notify UI helpers + if (await ctx.ui.confirm("Trust project?", event.cwd)) { + return { trusted: "yes", remember: true }; + } + return { trusted: "undecided" }; +}); +``` -Handlers are awaited in stream order, so slow handlers delay stream consumption. Handler errors are reported without changing the provider response. See [`debug-provider.ts`](../examples/extensions/debug-provider.ts) for an opt-in viewer that groups raw events by assistant message. +A `project_trust` handler must return `{ trusted: "yes" | "no" | "undecided" }`. A user/global or CLI extension that returns `"yes"` or `"no"` owns the decision; the first yes/no decision wins and suppresses the built-in trust prompt. Use `remember: true` to persist a yes/no decision; otherwise it applies only to the current process. Return `"undecided"` to let later handlers or the built-in trust flow decide. Check `ctx.hasUI` before prompting. If no handler returns yes/no, normal trust resolution continues: saved `trust.json` decisions apply first, then `defaultProjectTrust` controls whether Pi asks, trusts, or declines by default. - +### Resource Events -`context` transforms conversation messages without prompt and tool system messages; Pi restores that state afterward. Use `context_with_system` only when a request-local transformation must own the complete transcript, and keep a system message at index zero. +#### resources_discover -`turn_end` and `agent_before_settle` are actionable boundaries. Their handlers can chain proposed `custom`, `custom_message`, `context_edit`, or `compaction` entries and return `continue: true` for one next model request. Guard continuation conditions because an unconditional continuation can loop. Use the exported event declarations for the complete validation and ordering contract. +Fired after `session_start` so extensions can contribute additional skill, prompt, and theme paths. +The startup path uses `reason: "startup"`. Reload uses `reason: "reload"`. - +```typescript +pi.on("resources_discover", async (event, _ctx) => { + // event.cwd - current working directory + // event.reason - "startup" | "reload" + return { + skillPaths: ["/path/to/skills"], + promptPaths: ["/path/to/prompts"], + themePaths: ["/path/to/themes"], + }; +}); +``` -`cache_warming_decision` can override an idle prompt-cache refresh with `{ action: "warm" }` or `{ action: "stop" }`. The last handler that returns an action wins. +#### mcp_servers_change -Tool calls from one assistant message can run in parallel. -Do not assume a sibling call or result exists when another tool event runs. -Use `ctx.signal` for nested work owned by an active turn; commands and idle session events often have no operation signal. +Fired when an extension registers or unregisters an MCP server after extensions are bound. `event.servers` lists every registered server after the change. Handling this event marks the extension as the one that connects registered servers; see [MCP Servers](#mcp-servers). -A `user_bash` handler that returns `undefined` passes the command to the next handler and then to local execution if no handler handles it. Returning `operations` or `result` stops propagation. A handler failure blocks the command rather than falling through to local execution. +### Session Events + +See [Session Format](session-format.md) for session storage internals and the SessionManager API. + +#### session_start + +Fired when a session is started, loaded, or reloaded. + +```typescript +pi.on("session_start", async (event, ctx) => { + // event.reason - "startup" | "reload" | "new" | "resume" | "fork" + // event.previousSessionFile - present for "new", "resume", and "fork" + ctx.ui.notify(`Session: ${ctx.sessionManager.getSessionFile() ?? "ephemeral"}`, "info"); +}); +``` + +#### session_info_changed + +Fired when the current session display name is set via `/name`, RPC, or `pi.setSessionName()`. + +```typescript +pi.on("session_info_changed", async (event, ctx) => { + // event.name - current normalized name, or undefined if cleared + ctx.ui.notify(`Session renamed: ${event.name ?? "(none)"}`, "info"); +}); +``` + +#### session_before_switch + +Fired before starting a new session (`/new`) or switching sessions (`/resume`). + +```typescript +pi.on("session_before_switch", async (event, ctx) => { + // event.reason - "new" or "resume" + // event.targetSessionFile - session we're switching to (only for "resume") + + if (event.reason === "new") { + const ok = await ctx.ui.confirm("Clear?", "Delete all messages?"); + if (!ok) return { cancel: true }; + } +}); +``` + +After a successful switch or new-session action, Pi emits `session_shutdown` for the old extension instance, reloads and rebinds extensions for the new session, then emits `session_start` with `reason: "new" | "resume"` and `previousSessionFile`. +Do cleanup work in `session_shutdown`, then reestablish any in-memory state in `session_start`. + +#### session_before_fork + +Fired when forking via `/fork` or cloning via `/clone`. + +```typescript +pi.on("session_before_fork", async (event, ctx) => { + // event.entryId - ID of the selected entry + // event.position - "before" for /fork, "at" for /clone + return { cancel: true }; // Cancel fork/clone + // OR + return { skipConversationRestore: true }; // Reserved for future conversation restore control +}); +``` + +After a successful fork or clone, Pi emits `session_shutdown` for the old extension instance, reloads and rebinds extensions for the new session, then emits `session_start` with `reason: "fork"` and `previousSessionFile`. +Do cleanup work in `session_shutdown`, then reestablish any in-memory state in `session_start`. + + + +#### session_before_compact / session_compact / session_compact_failed + +Fired on compaction. See [compaction.md](compaction.md) for details. + +```typescript +pi.on("session_before_compact", async (event, ctx) => { + const { preparation, branchEntries, customInstructions, reason, willRetry, signal } = event; + + // reason - "manual" (/compact), "threshold", or "overflow" + // willRetry - whether the aborted turn is retried after compaction (overflow recovery) + + // Cancel: + return { cancel: true }; + + // Custom summary: + return { + compaction: { + summary: "...", + firstKeptEntryId: preparation.firstKeptEntryId, + tokensBefore: preparation.tokensBefore, + // usage: summaryResponse.usage, // Optional; included in session totals + } + }; +}); + +pi.on("session_compact", async (event, ctx) => { + // event.compactionEntry - the saved compaction + // event.fromExtension - whether extension provided it + // event.reason - "manual" (/compact), "threshold", or "overflow" + // event.willRetry - whether the aborted turn is retried after compaction (overflow recovery) +}); + +pi.on("session_compact_failed", async (event, ctx) => { + // event.reason - "manual" (/compact), "threshold", or "overflow" + // event.errorMessage - present for non-abort failures + // event.aborted - true for cancelled/aborted compactions + // event.willRetry - whether the aborted turn would have retried after compaction + // event.fromExtension - whether extension-provided compaction content was being used +}); +``` + +#### session_before_tree / session_tree + +Fired on `/tree` navigation. See [Sessions](sessions.md) for tree navigation concepts. + +```typescript +pi.on("session_before_tree", async (event, ctx) => { + const { preparation, signal } = event; + // preparation.targetId, oldLeafId, commonAncestorId, entriesToSummarize, + // userWantsSummary, customInstructions, replaceInstructions, label + return { cancel: true }; + // OR provide custom summary (used only when preparation.userWantsSummary is true): + return { + summary: { + summary: "...", + // usage: summaryResponse.usage, // Optional; included in session totals + details: {}, + }, + }; + // OR override summarization options: customInstructions, replaceInstructions, label +}); + +pi.on("session_tree", async (event, ctx) => { + // event.newLeafId, oldLeafId, summaryEntry, fromExtension +}); +``` + +#### session_shutdown + +Fired before a started session runtime is torn down. Use this to clean up resources opened from `session_start` or other session-scoped hooks. + +```typescript +pi.on("session_shutdown", async (event, ctx) => { + // event.reason - "quit" | "reload" | "new" | "resume" | "fork" + // event.targetSessionFile - destination session for session replacement flows + // Cleanup, save state, etc. +}); +``` + +### Agent Events + +#### before_agent_start + +Fired before a run starts, after input handling and expansion. Can inject a message and/or modify the system prompt. + +`event.messageType` says what started the run: `"user"` for a prompt or `pi.sendUserMessage()`, with `prompt` and `images`; `"custom"` for `pi.sendMessage(..., { triggerTurn: true })` while idle, with the starting `message` and no `prompt`. + +```typescript +pi.on("before_agent_start", async (event, ctx) => { + // event.messageType - "user" | "custom" + // event.prompt, event.images - user's prompt text and images (messageType "user") + // event.message - the custom message that starts the run (messageType "custom") + // event.systemPrompt - current chained system prompt for this handler + // (includes changes from earlier before_agent_start handlers) + // event.systemPromptOptions - structured options used to build the system prompt + // .customPrompt - exact prompt prefix from --system-prompt, SYSTEM.md, or custom templates + // .forceSystemPrompt - optional exact replacement for the complete prompt + // .selectedTools - tools currently active in the prompt + // .hiddenTools - selected tools whose declarations requests leave out + // .toolSnippets - one-line descriptions for each tool + // .toolGuidelines - rule bullets keyed by tool name + // .promptGuidelines - additional custom rule bullets + // .sections - custom XML-wrapped sections keyed by tag name + // .appendSystemPrompt - text from --append-system-prompt flags + // .cwd - working directory + // .contextFiles - AGENTS.md files and other loaded context files + // .skills - loaded skills + + return { + // Inject a persistent message (stored in session, sent to LLM) + message: { + customType: "my-extension", + content: "Additional context for the LLM", + display: true, + }, + // Replace the system prompt for this turn (chained across extensions) + systemPrompt: event.systemPrompt + "\n\nExtra instructions for this turn...", + }; +}); +``` + +A returned `message` is added to either kind of run; check `messageType` to skip runs started by custom messages. + +The `systemPromptOptions` field gives extensions access to the same structured data Pi uses to build the system prompt. Collections are mutable. Prefer changing `sections`, `selectedTools`, or `promptGuidelines`: Pi diffs the resulting prompt sections against what the model already has and appends one system message patching only the changed sections. Returning `systemPrompt`, or setting `forceSystemPrompt`, replaces the whole prompt for the run: every provider receives the forced text as its leading system prompt (a cache miss when it changes), and the session transcript keeps recording the structured sections. Tool selection changes update both the prompt contributions and executable provider tools; calling `pi.setActiveTools()` inside the handler has the same effect as editing `selectedTools`. Models that accept system messages mid-conversation receive the patch in place and keep their cached prefix; other models get the replayed prompt as their system prompt, which is a cache miss once per change. + +Inside `before_agent_start`, `event.systemPrompt` and `ctx.getSystemPrompt()` both reflect the chained system prompt as of the current handler. Later `before_agent_start` handlers can still modify it again. + + + +#### agent_start / agent_end / agent_before_settle / agent_settled + +`agent_start` fires when a low-level agent run begins. `agent_end` fires when that run ends, but Pi may still auto-retry, auto-compact and retry, or continue with queued follow-up messages. `agent_before_settle` is the final actionable boundary: it can append session entries and request one continuation. `agent_settled` is final and notification-only; use it for status integrations that need to know Pi will not continue running automatically. + +```typescript +pi.on("agent_start", async (_event, ctx) => {}); + +pi.on("agent_end", async (event, ctx) => { + // event.messages - messages from this low-level run +}); + +let addedReviewReminder = false; +pi.on("agent_before_settle", async (event, ctx) => { + if (addedReviewReminder) return; + addedReviewReminder = true; + return { + entries: [...event.entries, { + type: "custom_message", + customType: "review-reminder", + content: "Review the final diff before replying.", + display: false, + }], + continue: true, + }; +}); + +pi.on("agent_settled", async (event, ctx) => { + // event.aborted - true when the run ended because it was aborted, for example with Escape + // ctx.isIdle() is true; runs requested here start after all settled handlers finish. +}); +``` + +If the run is aborted while `agent_before_settle` handlers are running, valid returned entries are still committed, but requested continuation is suppressed. Work requested from `agent_settled` is deferred until every settled handler completes, so notification dispatch is non-reentrant. + +#### ui_prompt_start / ui_prompt_end + +Notification-only lifecycle events for blocking user-facing extension UI prompts. They fire around `ctx.ui.select()`, `ctx.ui.confirm()`, `ctx.ui.input()`, `ctx.ui.editor()`, and `ctx.ui.custom()` so host/status integrations can report "waiting for user" instead of just "running". + +Nested or overlapping prompts are coalesced into one outer waiting span. Handlers are invoked best-effort and are not awaited before showing or closing the prompt. + +```typescript +pi.on("ui_prompt_start", async (event, ctx) => { + // event.reason === "ui_prompt" + // event.kind: "select" | "confirm" | "input" | "editor" | "custom" + // event.title: prompt title when available +}); + +pi.on("ui_prompt_end", async (event, ctx) => { + // Pi is no longer waiting on that UI prompt span. +}); +``` + +#### turn_start / turn_end + +Fired for each turn (one LLM response + tool calls). + +```typescript +pi.on("turn_start", async (event, ctx) => { + // event.turnIndex, event.timestamp +}); + +let replacedResponse = false; +pi.on("turn_end", async (event, ctx) => { + // event.turnIndex, event.message, event.toolResults + // event.entries contains the structural entries proposed so far. + if (replacedResponse || event.outcome !== "completed" || event.toolResults.length > 0) return; + replacedResponse = true; + return { + entries: [ + ...event.entries, + { type: "context_edit", targetId: event.messageEntryId, replacement: null }, + { + type: "custom_message", + customType: "replacement-instruction", + content: "Answer again using the persisted user request.", + display: false, + }, + ], + continue: true, + }; +}); +``` + +`turn_end` runs after the assistant and tool-result messages have been persisted and before the low-level `turn_end` event. Retry backoff and final-attempt recovery still happen after `agent_end`, preserving their existing lifecycle and queue ordering; `agent_before_settle` sees the repaired projection after that work completes. Boundary handlers run in extension load and registration order. Each handler sees prior proposals in `event.entries` and sees `event.context` rebuilt from them. Returning `entries` or `continue` replaces only that field; omitted fields preserve the current proposal. Allowed draft entry types are `custom`, `custom_message`, `context_edit`, and `compaction`. The complete proposal is validated before it is appended in list order after all handlers finish; a handler error is reported and later handlers still run. Validation prevents partially applied semantic errors, but persistence is not transactional. + +`continue: true` ensures one next provider request for that boundary invocation. If tool results, steering, or a follow-up already cause that request, they satisfy the decision and no additional request is made; otherwise Pi makes one context-only request. Error and aborted responses remain hard exits. `continue: false` never suppresses natural work. Guard continuation conditions: an unconditional `continue: true` is evaluated again after the next response and can create an endless loop. A `custom_message` draft contributes a user-role model message but is extension-authored: it does not run human input hooks, slash commands, skills, or prompt templates. + +Host integrations that construct `TurnEndEvent` values must provide `messageEntryId`, `toolResultEntryIds`, `outcome`, `entries`, `continue`, and `context`. `ExtensionEvent` exhaustive switches must also handle `agent_before_settle`. `ExtensionRunner.emit()` excludes actionable turn boundaries; dispatch `turn_end` and `agent_before_settle` through `emitBoundary(baseEvent, buildContext)` so handlers receive chained previews. Other dedicated runner methods still return results for events such as `session_before_*`. + +#### message_start / message_update / message_end + +Fired for message lifecycle updates. + +- `message_start` and `message_end` fire for every message, including user, assistant, tool-result, and custom messages. +- `message_update` fires for assistant streaming updates. +- `message_end` handlers can return `{ message }` to replace the finalized message. The replacement must keep the same `role`. + +```typescript +pi.on("message_start", async (event, ctx) => { + // event.message +}); + +pi.on("message_update", async (event, ctx) => { + // event.message + // event.assistantMessageEvent (token-by-token stream event) +}); + +pi.on("message_end", async (event, ctx) => { + if (event.message.role !== "assistant") return; + + return { + message: { + ...event.message, + usage: { + ...event.message.usage, + cost: { + ...event.message.usage.cost, + total: 0.123, + }, + }, + }, + }; +}); +``` + +#### tool_execution_start / tool_execution_update / tool_execution_end + +Fired for tool execution lifecycle updates. + +In parallel tool mode: +- `tool_execution_start` is emitted in assistant source order during the preflight phase +- `tool_execution_update` events may interleave across tools +- `tool_execution_end` is emitted in tool completion order after each tool is finalized +- final `toolResult` message events are still emitted later in assistant source order + +```typescript +pi.on("tool_execution_start", async (event, ctx) => { + // event.toolCallId, event.toolName, event.args +}); + +pi.on("tool_execution_update", async (event, ctx) => { + // event.toolCallId, event.toolName, event.args, event.partialResult +}); + +pi.on("tool_execution_end", async (event, ctx) => { + // event.toolCallId, event.toolName, event.result, event.isError + // event.durationMs - milliseconds execute() took; absent when the tool did not run +}); +``` + +All three events, `tool_call`, and `tool_result` carry `parentToolCallId` when another tool made the call; see [Calling Other Tools](#calling-other-tools). + +#### context + +Fired before each LLM call. Modify messages non-destructively. See [Session Format](session-format.md) for message types. + +```typescript +pi.on("context", async (event, ctx) => { + // event.messages - deep copy without system messages, safe to modify + const filtered = event.messages.filter(m => !shouldPrune(m)); + return { messages: filtered }; +}); +``` + +`event.messages` holds the conversation without system messages. The prompt and tool declarations belong to Pi and are not part of this hook: when the handler returns a changed list, Pi replays the current prompt sections and tool declarations into one leading system message ahead of the returned messages. Filtering, windowing, or slicing from a compaction summary therefore cannot drop the prompt or the tools. An unchanged list keeps mid-conversation system messages in place, so models that accept them retain their cached prefix. System messages a handler adds are kept after Pi's head. To change the prompt or the tool set durably, use [`before_agent_start`](#before_agent_start) or `pi.setActiveTools()`; to edit system messages for one request, use [`context_with_system`](#context_with_system). + +#### context_with_system + +Fired before each LLM call, after every `context` handler has run and Pi has restored the prompt and tool state. `event.messages` is the full transcript, including the leading system message and any mid-conversation prompt or tool patches (see [Session Format](session-format.md#sessionmessageentry)). The returned messages are sent as they are: this hook owns the prompt and tool declarations for the request. + +```typescript +import { getCurrentSystemMessage } from "@earendil-works/pi-ai"; + +pi.on("context_with_system", async (event, ctx) => { + const cut = findCutIndex(event.messages); + // Fold the dropped prefix so its prompt and tool state survives as the new head. + const head = getCurrentSystemMessage(event.messages.slice(0, cut)); + return { messages: head ? [head, ...event.messages.slice(cut)] : event.messages.slice(cut) }; +}); +``` + +Rules: keep a system message at index 0 (providers read the prompt and initial tool declarations there; Pi reports an error if a handler drops it). Removing a system message removes the tool declarations and section patches it carries. Check your output with `getCurrentSystemPrompt()` and `getCurrentTools()` from `@earendil-works/pi-ai`. Handlers run in extension load order; a `systemPrompt` forced from `before_agent_start` is still projected onto the request afterwards. + +#### before_provider_headers + +Fired after the outgoing HTTP headers are assembled. Use it to add, override, or remove request headers. + +Handlers mutate `event.headers` in place. Set a key to a string to add or override it, or to `null` to delete it. + +```typescript +pi.on("before_provider_headers", (event, ctx) => { + // Add or override — e.g. a session id for gateway tracing/attribution + event.headers["x-session-id"] = ctx.sessionManager.getSessionId(); + + // Drop a tracking header pi adds for this call + event.headers["X-OpenRouter-Title"] = null; +}); +``` + +Runs once per provider request; retries reuse the same headers rather than re-firing the hook. + +#### before_provider_request + +Fired after the provider-specific payload is built, right before the request is sent. Handlers run in extension load order. Returning `undefined` keeps the payload unchanged. Returning any other value replaces the payload for later handlers and for the actual request. + +This hook can rewrite provider-level system instructions or remove them entirely. Those payload-level changes are not reflected by `ctx.getSystemPrompt()`, which reports Pi's system prompt string rather than the final serialized provider payload. + +```typescript +pi.on("before_provider_request", (event, ctx) => { + console.log(JSON.stringify(event.payload, null, 2)); + + // Optional: replace payload + // return { ...event.payload, temperature: 0 }; +}); +``` + +This is mainly useful for debugging provider serialization and cache behavior. + +#### after_provider_response + +Fired after an HTTP response is received and before its stream body is consumed. Handlers run in extension load order. + +```typescript +pi.on("after_provider_response", (event, ctx) => { + // event.status - HTTP status code + // event.headers - normalized response headers + if (event.status === 429) { + console.log("rate limited", event.headers["retry-after"]); + } +}); +``` + +Header availability depends on provider and transport. Providers that abstract HTTP responses may not expose headers. + +#### provider_stream_event + +Fired for each parsed provider stream event before Pi normalizes it. Notification-only and not persisted. + +```typescript +pi.on("provider_stream_event", (event, ctx) => { + // event.provider, event.api, event.model + // event.data - earliest structured value available to Pi (read-only) +}); +``` + +`event.data` is not necessarily the original HTTP bytes or SSE frame. Treat it as read-only because mutation can affect normalization. Handlers are awaited in stream order, so slow handlers delay stream consumption. Handler errors are reported without changing the provider response. See [`debug-provider.ts`](../examples/extensions/debug-provider.ts) for an opt-in viewer that groups raw events by assistant message. + +#### cache_warming_decision + +Fired before each prompt-cache refresh with Pi's decision filled in. The event carries only Pi's cost estimates; use `ctx.model`, `ctx.isIdle()`, and `ctx.getContextUsage()` for everything else. + +```typescript +pi.on("cache_warming_decision", (event, ctx) => { + // event.warmCost: price of this refresh + // event.missCost: extra price of the next request if the entry is lost + // event.continuationProbability: pi's estimate that a request arrives in time + // event.action: "warm" | "stop", pi's decision + + if (ctx.model?.provider === "my-provider") { + return { action: "stop" }; + } +}); +``` + +Return `{ action: "warm" }` or `{ action: "stop" }` to override; the last handler that returns an action wins. `"stop"` ends warming until the next real request. + +### Model Events + +#### model_select + +Fired when the model changes via `/model` command, model cycling (`Ctrl+P`), or `pi.setModel()`. Restoring a session does not fire it; read `ctx.model` in `session_start` to initialize model-dependent state. + +```typescript +pi.on("model_select", async (event, ctx) => { + // event.model - newly selected model + // event.previousModel - previous model (undefined if first selection) + // event.source - "set" | "cycle" + + const prev = event.previousModel + ? `${event.previousModel.provider}/${event.previousModel.id}` + : "none"; + const next = `${event.model.provider}/${event.model.id}`; + + ctx.ui.notify(`Model changed (${event.source}): ${prev} -> ${next}`, "info"); +}); +``` + +Use this to update UI elements (status bars, footers) or perform model-specific initialization when the active model changes. + +#### thinking_level_select + +Fired when the thinking level changes. This is notification-only; handler return values are ignored. + +```typescript +pi.on("thinking_level_select", async (event, ctx) => { + // event.level - newly selected thinking level + // event.previousLevel - previous thinking level + + ctx.ui.setStatus("thinking", `thinking: ${event.level}`); +}); +``` + +Use this to update extension UI when `pi.setThinkingLevel()`, model changes, or built-in thinking-level controls change the active thinking level. + + + +### Tool Events + +#### tool_call + +Fired after `tool_execution_start`, before the tool executes. **Can block.** Use `isToolCallEventType` to narrow and get typed inputs. + +Before `tool_call` runs, Pi waits for previously emitted Agent events to finish draining through `AgentSession`. This means `ctx.sessionManager` is up to date through the current assistant tool-calling message. + +In the default parallel tool execution mode, sibling tool calls from the same assistant message are preflighted sequentially, then executed concurrently. `tool_call` is not guaranteed to see sibling tool results from that same assistant message in `ctx.sessionManager`. + +`event.input` is mutable. Mutate it in place to patch tool arguments before execution. + +Behavior guarantees: +- Mutations to `event.input` affect the actual tool execution +- Later `tool_call` handlers see mutations made by earlier handlers +- No re-validation is performed after your mutation +- Return values from `tool_call` control blocking via `{ block: true, reason?: string, terminate?: boolean }` +- `terminate` only applies to a blocked call; the agent stops early only when every finalized result in the batch is terminating + +```typescript +import { isToolCallEventType } from "@earendil-works/pi-coding-agent"; + +pi.on("tool_call", async (event, ctx) => { + // event.toolName - "bash", "read", "write", "edit", etc. + // event.toolCallId + // event.input - tool parameters (mutable) + + // Built-in tools: no type params needed + if (isToolCallEventType("bash", event)) { + // event.input is { command: string; timeout?: number } + event.input.command = `source ~/.profile\n${event.input.command}`; + + if (event.input.command.includes("rm -rf")) { + return { block: true, reason: "Dangerous command", terminate: true }; + } + } + + if (isToolCallEventType("read", event)) { + // event.input is { path: string; offset?: number; limit?: number } + console.log(`Reading: ${event.input.path}`); + } +}); +``` + +#### Typing custom tool input + +Custom tools should export their input type: + +```typescript +// my-tool.ts +import { type Static, Type } from "typebox"; + +export const myToolSchema = Type.Object({ action: Type.String() }); +export type MyToolInput = Static; +``` + +Use `isToolCallEventType` with explicit type parameters: + +```typescript +import { isToolCallEventType } from "@earendil-works/pi-coding-agent"; +import type { MyToolInput } from "./my-tool.ts"; + +pi.on("tool_call", (event) => { + if (isToolCallEventType<"my_tool", MyToolInput>("my_tool", event)) { + event.input.action; // typed + } +}); +``` + +#### tool_result + +Fired after tool execution finishes and before `tool_execution_end` plus the final tool result message events are emitted. **Can modify result.** + +In parallel tool mode, `tool_result` and `tool_execution_end` may interleave in tool completion order, while final `toolResult` message events are still emitted later in assistant source order. + +`tool_result` handlers chain like middleware: +- Handlers run in extension load order +- Each handler sees the latest result after previous handler changes +- Handlers can return partial patches (`content`, `details`, `structuredContent`, `isError`, or `usage`); omitted fields keep their current values +- Replacing `content` without returning `structuredContent` drops the structured content, because it may no longer match + +Use `ctx.signal` for nested async work inside the handler. This lets Esc cancel model calls, `fetch()`, and other abort-aware operations started by the extension. + +```typescript +import { isBashToolResult } from "@earendil-works/pi-coding-agent"; + +pi.on("tool_result", async (event, ctx) => { + // event.toolName, event.toolCallId, event.input, event.parentToolCallId + // event.content, event.details, event.structuredContent, event.isError, event.usage + + if (isBashToolResult(event)) { + // event.details is typed as BashToolDetails + } + + const response = await fetch("https://example.com/summarize", { + method: "POST", + body: JSON.stringify({ content: event.content }), + signal: ctx.signal, + }); + + // Modify result: + return { content: [...], details: {...}, isError: false, usage: nestedModelUsage }; +}); +``` + +### User Bash Events + +#### user_bash + +Fired when user executes `!` or `!!` commands. **Can intercept.** + +```typescript +import { createLocalBashOperations } from "@earendil-works/pi-coding-agent"; + +pi.on("user_bash", (event, ctx) => { + // event.command - the bash command + // event.excludeFromContext - true if !! prefix + // event.cwd - working directory + + // Option 1: Provide custom operations (e.g., SSH) + return { operations: remoteBashOps }; + + // Option 2: Wrap pi's built-in local bash backend + const local = createLocalBashOperations(); + return { + operations: { + exec(command, cwd, options) { + return local.exec(`source ~/.profile\n${command}`, cwd, options); + } + } + }; + + // Option 3: Full replacement - return result directly + return { result: { output: "...", exitCode: 0, cancelled: false, truncated: false } }; +}); +``` + +Returning `undefined` continues to the next handler, then local execution if none handles the event. A valid result stops propagation: `operations` executes the command through the supplied backend, while `result` records the completed command without executing it. + +### Input Events + +#### input + +Fired when user input is received, after extension commands are checked but before skill and template expansion. The event sees the raw input text, so `/skill:foo` and `/template` are not yet expanded. + +**Processing order:** +1. Extension commands (`/cmd`) checked first - if found, handler runs and input event is skipped. Messages from `pi.sendUserMessage()` are checked only with `expandPromptTemplates: true`. +2. `input` event fires - can intercept, transform, or handle +3. If not handled: skill commands (`/skill:name`) expanded to skill content +4. If not handled: prompt templates (`/template`) expanded to template content +5. Agent processing begins (`before_agent_start`, etc.) + +```typescript +pi.on("input", async (event, ctx) => { + // event.text - raw input (before skill/template expansion) + // event.images - attached images, if any + // event.source - "interactive" (typed), "rpc" (API), or "extension" (via sendUserMessage) + // event.streamingBehavior - "steer" | "followUp" | undefined + // undefined when idle, "steer" for mid-stream interrupts, + // "followUp" for messages queued until the agent finishes + + // Transform: rewrite input before expansion + if (event.text.startsWith("?quick ")) + return { action: "transform", text: `Respond briefly: ${event.text.slice(7)}` }; + + // Handle: respond without LLM (extension shows its own feedback) + if (event.text === "ping") { + ctx.ui.notify("pong", "info"); + return { action: "handled" }; + } + + // Route by source: skip processing for extension-injected messages + if (event.source === "extension") return { action: "continue" }; + + // Intercept skill commands before expansion + if (event.text.startsWith("/skill:")) { + // Could transform, block, or let pass through + } + + return { action: "continue" }; // Default: pass through to expansion +}); +``` + +**Results:** +- `continue` - pass through unchanged (default if handler returns nothing) +- `transform` - modify text/images, then continue to expansion +- `handled` - skip agent entirely (first handler to return this wins) + +Transforms chain across handlers. See [input-transform.ts](../examples/extensions/input-transform.ts) and [input-transform-streaming.ts](../examples/extensions/input-transform-streaming.ts) for `streamingBehavior`-aware routing. + + + +## ExtensionContext + +All handlers receive `ctx: ExtensionContext`. Tool `execute()` receives `ExtensionToolContext`, which adds `ctx.tools` and `ctx.executeTool()` (see [Calling Other Tools](#calling-other-tools)). Command handlers receive [`ExtensionCommandContext`](#extensioncommandcontext). + +### ctx.ui + +UI methods for user interaction. See [Custom UI](#custom-ui) for full details. + +### ctx.mode + +Current run mode: `"tui"`, `"rpc"`, `"json"`, or `"print"`. Use `ctx.mode === "tui"` to guard terminal-only features such as `custom()`, component factories, terminal input, and direct TUI rendering. + +### ctx.hasUI + +`true` in TUI and RPC modes. `false` in print mode (`-p`) and JSON mode. Use this to guard dialog methods (`select`, `confirm`, `input`, `editor`) and fire-and-forget methods (`notify`, `setStatus`, `setWidget`, `setTitle`, `setEditorText`) that work in both TUI and RPC modes. In RPC mode, some TUI-specific methods are no-ops or return defaults (see [RPC Extension UI](rpc-extension-ui.md)). + +### ctx.cwd + +Current working directory. + +Use `CONFIG_DIR_NAME` instead of hardcoding `.pi` when constructing project-local config paths. Rebranded distributions can use a different config directory name. + +```typescript +import { CONFIG_DIR_NAME, type ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { join } from "node:path"; + +export default function (pi: ExtensionAPI) { + pi.on("session_start", (_event, ctx) => { + const projectConfigPath = join(ctx.cwd, CONFIG_DIR_NAME, "my-extension.json"); + // ... + }); +} +``` + +### ctx.isProjectTrusted() + +Returns whether project-local trust is active for the current session context. This includes temporary trust decisions and CLI trust overrides, not just saved decisions in the global trust store. + +Use this before reading project-local extension configuration that should only be honored for trusted projects. + +### ctx.sessionManager + +Read-only access to session state: `getCwd()`, `getSessionDir()`, `getSessionId()`, `getSessionFile()`, `getSessionName()`, `getHeader()`, `getEntries()`, `getEntry()`, `getBranch()`, `getTree()`, `getLeafId()`, `getLeafEntry()`, `getLabel()`, `buildContextEntries()`, and `buildSessionProjection()`. See [Session Format](session-format.md) for entry types. + +For `tool_call`, this state is synchronized through the current assistant message before handlers run. In parallel tool execution mode it is still not guaranteed to include sibling tool results from the same assistant message. + +```typescript +ctx.sessionManager.getEntries() // All entries +ctx.sessionManager.getBranch() // Current branch +ctx.sessionManager.buildContextEntries() // Active branch entries with compaction applied +ctx.sessionManager.getLeafId() // Current leaf entry ID +``` + +### ctx.modelRegistry / ctx.model / ctx.thinkingLevel / ctx.scopedModels + +Access to models, providers, and resolved authentication. `ctx.modelRegistry.getProvider(id)` returns the effective pi-ai provider, while `getProviderAuth(id)` resolves its current API key, headers, base URL, and provider-scoped environment without requiring a loaded model. `ctx.model` is the active model, and `ctx.thinkingLevel` is its current effective thinking level. + +`ctx.scopedModels` is the read-only list of models scoped to the current session — the same set the `/scoped-models` command shows. It is resolved at session start from the `--models` CLI flag and the `enabledModels` setting (matched against the available catalogue with minimatch on `provider/modelId` or a bare `modelId`). It is empty when no scoping is configured, meaning every available model is usable. Each entry is `{ model, thinkingLevel? }`, where `thinkingLevel` is set only when a pattern pinned it (e.g. `anthropic/*:high`). Use it to populate a model picker that mirrors the built-in one instead of enumerating the whole catalogue via `ctx.modelRegistry.getAvailable()`. + +#### Streaming model calls + +Use `ctx.modelRegistry.streamSimple(model, context, options)` for provider-neutral options such as `reasoning`, or `complete(model, context, options)` to await the final assistant message with API-specific options. Both use configured providers and resolve authentication, including for providers registered with `pi.registerProvider()`. Use these instead of `pi-ai/compat` streaming functions, which cannot see extension provider registrations. + +`streamSimple()` returns an `AssistantMessageEventStream`. Iterate it for response events and await `.result()` for the final message. Setup failures produce error events and error results. + +For non-chat models, `findOfType()`, `getModelsOfType()`, `getAvailableOfType()`, and `getModelOfType()` look up image and classifier models, and `classify()` and `generateImages()` call them. See [Use classifier models](models.md#use-classifier-models) and [Use image models](models.md#use-image-models). + +### ctx.signal + +The current agent abort signal, or `undefined` when no agent turn is active. + +Use this for abort-aware nested work started by extension handlers, for example: +- `fetch(..., { signal: ctx.signal })` +- model calls that accept `signal` +- file or process helpers that accept `AbortSignal` + +`ctx.signal` is typically defined during active turn events such as `tool_call`, `tool_result`, `message_update`, and `turn_end`. +It is usually `undefined` in idle or non-turn contexts such as session events, extension commands, and shortcuts fired while Pi is idle. + +```typescript +pi.on("tool_result", async (event, ctx) => { + const response = await fetch("https://example.com/api", { + method: "POST", + body: JSON.stringify(event), + signal: ctx.signal, + }); + + const data = await response.json(); + return { details: data }; +}); +``` + +### ctx.isIdle() / ctx.abort() / ctx.hasPendingMessages() + +Control flow helpers. `ctx.isIdle()` is false while Pi is processing an agent run, automatic retry, auto-compaction retry, or queued continuation. + +### ctx.shutdown() + +Request a graceful shutdown of pi. + +- **Interactive mode:** Deferred until the agent becomes idle (after processing all queued steering and follow-up messages). +- **RPC mode:** Deferred until the next idle state (after completing the current command response, when waiting for the next command). +- **Print mode:** No-op. The process exits automatically when all prompts are processed. + +Emits `session_shutdown` event to all extensions before exiting. Available in all contexts (event handlers, tools, commands, shortcuts). + +```typescript +pi.on("tool_call", (event, ctx) => { + if (isFatal(event.input)) { + ctx.shutdown(); + } +}); +``` + +### ctx.getContextUsage() + +Returns current context usage for the active model. Uses last assistant usage when available, then estimates tokens for trailing messages. + +```typescript +const usage = ctx.getContextUsage(); +if (usage?.tokens != null && usage.tokens > 100_000) { + // ... +} +``` + +### ctx.compact() + +Trigger compaction without awaiting completion. Use `onComplete` and `onError` for follow-up actions. + +```typescript +ctx.compact({ + customInstructions: "Focus on recent changes", + onComplete: (result) => { + ctx.ui.notify("Compaction completed", "info"); + }, + onError: (error) => { + ctx.ui.notify(`Compaction failed: ${error.message}`, "error"); + }, +}); +``` + +### ctx.getSystemPrompt() + +Returns Pi's current system prompt string. + +- During `before_agent_start`, this reflects chained system-prompt changes made so far for the current turn. +- It does not include later `context` message mutations. +- It does not include `before_provider_request` payload rewrites. +- If later-loaded extensions run after yours, they can still change what is ultimately sent. + +```typescript +pi.on("before_agent_start", (event, ctx) => { + const prompt = ctx.getSystemPrompt(); + console.log(`System prompt length: ${prompt.length}`); +}); +``` + +## ExtensionCommandContext + +Command handlers receive `ExtensionCommandContext`, which extends `ExtensionContext` with session control methods. These are only available in commands because they can deadlock if called from event handlers. + +### ctx.getSystemPromptOptions() + +Returns the base inputs Pi currently uses to build the system prompt. + +```typescript +const options = ctx.getSystemPromptOptions(); +const contextPaths = options.contextFiles?.map((file) => file.path) ?? []; +``` + +This has the same shape and mutability as `before_agent_start` `event.systemPromptOptions`: custom or forced prompt, active tools, tool snippets, per-tool and custom rules, custom sections, appended prompt text, cwd, loaded context files, and loaded skills. It may include full context file contents, so treat it as sensitive extension-local data and avoid exposing it through command lists, logs, or autocomplete metadata. + +This reports the current base prompt inputs. It does not include per-turn `before_agent_start` chained system-prompt changes, later `context` event message mutations, or `before_provider_request` payload rewrites. + +### ctx.waitForIdle() + +Wait for the agent to fully settle, including automatic retries, auto-compaction retries, and queued continuations: + +```typescript +pi.registerCommand("my-cmd", { + handler: async (args, ctx) => { + await ctx.waitForIdle(); + // Agent is now idle, safe to modify session + }, +}); +``` + +### ctx.newSession(options?) + +Create a new session: + +```typescript +const parentSession = ctx.sessionManager.getSessionFile(); +const kickoff = "Continue in the replacement session"; + +const result = await ctx.newSession({ + parentSession, + setup: async (sm) => { + sm.appendMessage({ + role: "user", + content: [{ type: "text", text: "Context from previous session..." }], + timestamp: Date.now(), + }); + }, + withSession: async (ctx) => { + // Use only the replacement-session ctx here. + await ctx.sendUserMessage(kickoff); + }, +}); + +if (result.cancelled) { + // An extension cancelled the new session +} +``` + +Options: +- `parentSession`: parent session file to record in the new session header +- `setup`: mutate the new session's `SessionManager` before `withSession` runs +- `withSession`: run post-switch work against a fresh replacement-session context. Do not use captured old `pi` / command `ctx`; see [Session replacement lifecycle and footguns](#session-replacement-lifecycle-and-footguns). + +### ctx.fork(entryId, options?) + +Fork from a specific entry, creating a new session file: + +```typescript +const result = await ctx.fork("entry-id-123", { + withSession: async (ctx) => { + // Use only the replacement-session ctx here. + ctx.ui.notify("Now in the forked session", "info"); + }, +}); +if (result.cancelled) { + // An extension cancelled the fork +} + +const cloneResult = await ctx.fork("entry-id-456", { position: "at" }); +if (cloneResult.cancelled) { + // An extension cancelled the clone +} +``` + +Options: +- `position`: `"before"` (default) forks before the selected user message, restoring that prompt into the editor +- `position`: `"at"` duplicates the active path through the selected entry without restoring editor text +- `withSession`: run post-switch work against a fresh replacement-session context. Do not use captured old `pi` / command `ctx`; see [Session replacement lifecycle and footguns](#session-replacement-lifecycle-and-footguns). + +### ctx.navigateTree(targetId, options?) + +Navigate to a different point in the session tree. Rejects while an agent response, manual or automatic compaction, or another tree navigation is active, even with `summarize: false`. These conflicts leave the active branch unchanged and reject the promise rather than returning `{ cancelled: true }`. Wait for the active operation to finish (for example, with `await ctx.waitForIdle()` in a command handler) and retry. It also rejects when `summarize` is set without an available model, or when the target entry does not exist: + +```typescript +const result = await ctx.navigateTree("entry-id-456", { + summarize: true, + customInstructions: "Focus on error handling changes", + replaceInstructions: false, // true = replace default prompt entirely + label: "review-checkpoint", +}); +``` + +Options: +- `summarize`: Whether to generate a summary of the abandoned branch +- `customInstructions`: Custom instructions for the summarizer +- `replaceInstructions`: If true, `customInstructions` replaces the default prompt instead of being appended +- `label`: Label to attach to the branch summary entry (or target entry if not summarizing) + +### ctx.switchSession(sessionPath, options?) + +Switch to a different session file: + +```typescript +const result = await ctx.switchSession("/path/to/session.jsonl", { + withSession: async (ctx) => { + await ctx.sendUserMessage("Resume work in the replacement session"); + }, +}); +if (result.cancelled) { + // An extension cancelled the switch via session_before_switch +} +``` + +Options: +- `withSession`: run post-switch work against a fresh replacement-session context. Do not use captured old `pi` / command `ctx`; see [Session replacement lifecycle and footguns](#session-replacement-lifecycle-and-footguns). + +To discover available sessions, use the static `SessionManager.list()` or `SessionManager.listAll()` methods: + +```typescript +import { SessionManager } from "@earendil-works/pi-coding-agent"; + +pi.registerCommand("switch", { + description: "Switch to another session", + handler: async (args, ctx) => { + const sessions = await SessionManager.list(ctx.cwd); + if (sessions.length === 0) return; + const choice = await ctx.ui.select( + "Pick session:", + sessions.map((s) => s.path), + ); + if (choice) { + await ctx.switchSession(choice, { + withSession: async (ctx) => { + ctx.ui.notify("Switched session", "info"); + }, + }); + } + }, +}); +``` + +### Session replacement lifecycle and footguns + +`withSession` receives a fresh `ReplacedSessionContext`, which extends `ExtensionCommandContext` with async `sendMessage()` and `sendUserMessage()` helpers bound to the replacement session. + +Lifecycle and footguns: +- `withSession` runs only after the old session has emitted `session_shutdown`, the old runtime has been torn down, the replacement session has been rebound, and the new extension instance has already received `session_start`. +- The callback still executes in the original closure, not inside the new extension instance. That means your old extension instance may already have run its shutdown cleanup before `withSession` starts. +- Captured old `pi` / old command `ctx` session-bound objects are stale after replacement and will throw if used. Use only the `ctx` passed to `withSession` for session-bound work. +- Previously extracted raw objects are still your responsibility. For example, if you capture `const sm = ctx.sessionManager` before replacement, `sm` is still the old `SessionManager` object. Do not reuse it after replacement. +- Code in `withSession` should assume any state invalidated by your `session_shutdown` handler is already gone. Only capture plain data that survives shutdown cleanly, such as strings, ids, and serialized config. + +Safe pattern: + +```typescript +pi.registerCommand("handoff", { + handler: async (_args, ctx) => { + const kickoff = "Continue from the replacement session"; + await ctx.newSession({ + withSession: async (ctx) => { + await ctx.sendUserMessage(kickoff); + }, + }); + }, +}); +``` + +Unsafe pattern: + +```typescript +pi.registerCommand("handoff", { + handler: async (_args, ctx) => { + const oldSessionManager = ctx.sessionManager; + await ctx.newSession({ + withSession: async (_ctx) => { + // stale old objects: do not do this + oldSessionManager.getSessionFile(); + pi.sendUserMessage("wrong"); + }, + }); + }, +}); +``` + +### ctx.reload() + +Run the same reload flow as `/reload`. + +```typescript +pi.registerCommand("reload-runtime", { + description: "Reload extensions, skills, prompts, themes, and context files", + handler: async (_args, ctx) => { + await ctx.reload(); + return; + }, +}); +``` + +Important behavior: +- `await ctx.reload()` emits `session_shutdown` for the current extension runtime +- It then reloads resources and emits `session_start` with `reason: "reload"` and `resources_discover` with reason `"reload"` +- The currently running command handler still continues in the old call frame +- Code after `await ctx.reload()` still runs from the pre-reload version +- Code after `await ctx.reload()` must not assume old in-memory extension state is still valid +- After the handler returns, future commands/events/tool calls use the new extension version + +For predictable behavior, treat reload as terminal for that handler (`await ctx.reload(); return;`). + +Tools run with `ExtensionToolContext`, not `ExtensionCommandContext`, so they cannot call `ctx.reload()`, and Pi cannot reload while the agent is running. To let the LLM trigger a reload, have the tool record the request and run the command from `agent_settled`, where messages sent by handlers are processed after all settled handlers finish: + +```typescript +let reloadRequested = false; + +pi.registerTool({ + name: "reload_runtime", + label: "Reload Runtime", + description: "Reload extensions, skills, prompts, themes, and context files", + parameters: Type.Object({}), + async execute() { + reloadRequested = true; + return { + content: [{ type: "text", text: "Reload requested. Pi reloads after this response." }], + details: {}, + terminate: true, + }; + }, +}); + +pi.on("agent_settled", () => { + if (!reloadRequested) return; + reloadRequested = false; + pi.sendUserMessage("/reload-runtime", { expandPromptTemplates: true }); +}); +``` + +`expandPromptTemplates: true` dispatches the text as a command. Without it, or when queued with `deliverAs`, the text reaches the model as a plain user message. See [reload-runtime.ts](../examples/extensions/reload-runtime.ts). + +## ExtensionAPI Methods + +### pi.on(event, handler) + +Subscribe to events. Returns an unsubscribe function that removes only that registration. See [Events](#events) for event types and return values. + +```typescript +const unsubscribe = pi.on("agent_end", async (event) => { + unsubscribe(); + await updateIntegration(event.messages); +}); +``` + +Handlers run in extension load order, then registration order within each extension. Adding or removing a handler does not affect a dispatch already in progress. + +### pi.registerTool(definition) + +Register a custom tool callable by the LLM. See [Custom Tools](#custom-tools) for full details. + +`pi.registerTool()` works both during extension load and after startup. You can call it inside `session_start`, command handlers, or other event handlers. New tools are refreshed immediately in the same session, so they appear in `pi.getAllTools()` without `/reload`. Registering a tool activates it unless it sets `defaultActive: false` or an [exposure](#tool-exposure) other than `direct` or `model-only`. + +Use `pi.setActiveTools()` to enable or disable tools (including dynamically added tools) at runtime. + +Use `promptSnippet` to opt a custom tool into a one-line entry in the `` section of the default system prompt, and `promptGuidelines` to append tool-specific bullets to its `` section when the tool is active. + +**Important:** `promptGuidelines` bullets are appended flat to the `` section with no tool name prefix. Each guideline must name the tool it refers to — avoid "Use this tool when..." because the LLM cannot tell which tool "this" means. Write "Use my_tool when..." instead. + +See [dynamic-tools.ts](../examples/extensions/dynamic-tools.ts) for a full example. + +```typescript +import { Type } from "typebox"; +import { StringEnum } from "@earendil-works/pi-ai"; + +pi.registerTool({ + name: "my_tool", + label: "My Tool", + description: "What this tool does", + promptSnippet: "Summarize or transform text according to action", + promptGuidelines: ["Use my_tool when the user asks to summarize previously generated text."], + parameters: Type.Object({ + action: StringEnum(["list", "add"] as const), + text: Type.Optional(Type.String()), + }), + + async execute(toolCallId, params, signal, onUpdate, ctx) { + // Stream progress; partial results also need details + onUpdate?.({ content: [{ type: "text", text: "Working..." }], details: { result: "pending" } }); + + return { + content: [{ type: "text", text: "Done" }], + details: { result: "..." }, + }; + }, + + // Optional: Custom rendering + renderCall(args, theme, context) { ... }, + renderResult(result, options, theme, context) { ... }, +}); +``` + +### pi.sendMessage(message, options?) + +Inject a custom message into the session. Custom messages participate in LLM context. For durable TUI-only content that should not be sent to the LLM, use [`pi.appendEntry()`](#piappendentrycustomtype-data) with [`pi.registerEntryRenderer()`](#piregisterentryrenderercustomtype-renderer). + +```typescript +pi.sendMessage({ + customType: "my-extension", + content: "Message text", + display: true, + details: { ... }, +}, { + triggerTurn: true, + deliverAs: "steer", +}); +``` + +**Options:** +- `deliverAs` - Delivery mode: + - `"steer"` (default) - Queues the message while streaming. Delivered after the current assistant turn finishes executing its tool calls, before the next LLM call. + - `"followUp"` - Waits for agent to finish. Delivered only when agent has no more tool calls. + - `"nextTurn"` - Queued for next user prompt. Does not interrupt or trigger anything. +- `triggerTurn: true` - If agent is idle, trigger an LLM response immediately. Only applies to `"steer"` and `"followUp"` modes (ignored for `"nextTurn"`). Without it, an idle session stores the message without starting a turn. +- `triggerTurn: false` while streaming - The message is not steered or queued as a follow-up; it is appended after the current turn ends. + +### pi.sendUserMessage(content, options?) + +Send a user message to the agent. Unlike `sendMessage()` which sends custom messages, this sends an actual user message that appears as if typed by the user. It starts or queues a turn unless an `input` handler handles it or, with `expandPromptTemplates: true`, it runs an extension command. + +```typescript +// Simple text message +pi.sendUserMessage("What is 2+2?"); + +// With content array (text + images) +pi.sendUserMessage([ + { type: "text", text: "Describe this image:" }, + { type: "image", data: "...", mimeType: "image/png" }, +]); + +// During streaming - must specify delivery mode +pi.sendUserMessage("Focus on error handling", { deliverAs: "steer" }); +pi.sendUserMessage("And then summarize", { deliverAs: "followUp" }); + +// Opt in to extension command dispatch and skill/prompt template expansion +pi.sendUserMessage("/review src/index.ts", { expandPromptTemplates: true }); +``` + +**Options:** +- `deliverAs` - Required when agent is streaming: + - `"steer"` - Queues the message for delivery after the current assistant turn finishes executing its tool calls + - `"followUp"` - Waits for agent to finish all tools +- `expandPromptTemplates` - Dispatch extension commands and expand skill commands and prompt templates. Defaults to `false`. + +When not streaming, the message is sent immediately and triggers a new turn. When streaming without `deliverAs`, the message is rejected and Pi reports an extension error. + +See [send-user-message.ts](../examples/extensions/send-user-message.ts) for a complete example. + +### pi.appendEntry(customType, data?) + +Persist extension data. Custom entries do NOT participate in LLM context. In interactive mode, they can also render inside the chat transcript when paired with `pi.registerEntryRenderer()`. + +```typescript +pi.appendEntry("my-state", { count: 42 }); +pi.appendEntry("status-card", { title: "Indexed files", count: 17 }); + +// Restore on reload +pi.on("session_start", async (_event, ctx) => { + for (const entry of ctx.sessionManager.getEntries()) { + if (entry.type === "custom" && entry.customType === "my-state") { + // Reconstruct from entry.data + } + } +}); +``` + +### pi.setSessionName(name) + +Set the session display name (shown in session selector instead of first message). + +```typescript +pi.setSessionName("Refactor auth module"); +``` + +### pi.getSessionName() + +Get the current session name, if set. + +```typescript +const name = pi.getSessionName(); +if (name) { + console.log(`Session: ${name}`); +} +``` + +### pi.setLabel(entryId, label) + +Set or clear a label on an entry. Labels are user-defined markers for bookmarking and navigation (shown in `/tree` selector). + +```typescript +// Set a label +pi.setLabel(entryId, "checkpoint-before-refactor"); + +// Clear a label +pi.setLabel(entryId, undefined); + +// Read labels via sessionManager +const label = ctx.sessionManager.getLabel(entryId); +``` + +Labels persist in the session and survive restarts. Use them to mark important points (turns, checkpoints) in the conversation tree. + +### pi.registerCommand(name, options) + +Register a command. + +If multiple extensions register the same command name, Pi keeps them all and assigns numeric invocation suffixes in load order, for example `/review:1` and `/review:2`. + +```typescript +pi.registerCommand("stats", { + description: "Show session statistics", + handler: async (args, ctx) => { + const count = ctx.sessionManager.getEntries().length; + ctx.ui.notify(`${count} entries`, "info"); + } +}); +``` + +Optional: add argument auto-completion for `/command ...`. `getArgumentCompletions` can also return a `Promise`: + +```typescript +import type { AutocompleteItem } from "@earendil-works/pi-tui"; + +pi.registerCommand("deploy", { + description: "Deploy to an environment", + getArgumentCompletions: (prefix: string): AutocompleteItem[] | null => { + const envs = ["dev", "staging", "prod"]; + const items = envs.map((e) => ({ value: e, label: e })); + const filtered = items.filter((i) => i.value.startsWith(prefix)); + return filtered.length > 0 ? filtered : null; + }, + handler: async (args, ctx) => { + ctx.ui.notify(`Deploying: ${args}`, "info"); + }, +}); +``` + +### pi.getCommands() + +Get the slash commands available for invocation via `prompt` in the current session. Includes extension commands, prompt templates, and skill commands. +The list matches the RPC `get_commands` ordering: extensions first, then templates, then skills. Skill commands are named `skill:`. + +```typescript +const commands = pi.getCommands(); +const bySource = commands.filter((command) => command.source === "extension"); +const userScoped = commands.filter((command) => command.sourceInfo.scope === "user"); +``` + +Each entry has this shape: + +```typescript +{ + name: string; // Invokable command name without the leading slash. May be suffixed like "review:1" + description?: string; + source: "extension" | "prompt" | "skill"; + sourceInfo: { + path: string; + source: string; + scope: "user" | "project" | "temporary"; + origin: "package" | "top-level"; + baseDir?: string; + }; +} +``` + +Use `sourceInfo` as the canonical provenance field. Do not infer ownership from command names or from ad hoc path parsing. + +Built-in interactive commands (like `/model` and `/settings`) are not included here. They are handled only in interactive +mode and would not execute if sent via `prompt`. + +### pi.registerMessageRenderer(customType, renderer) + +Register a custom TUI renderer for custom messages with your `customType`. Custom messages are created with `pi.sendMessage()` and participate in LLM context. See [Custom UI](#custom-ui). + +### pi.registerMarkdownTransformer(transformer) + +Register a transformer for the Markdown in normal user text, assistant text, and thinking blocks. Transformers run in extension load order, and each transformer receives the Markdown returned by the previous transformer. After the chain finishes, Pi renders the transformed content with its built-in renderer. + +The transformer receives the Markdown string and a context with: + +- `messageType` — `"user"`, `"assistant"`, or `"assistant-thinking"` +- `isStreaming` — `true` for partial assistant updates; `false` for user, finalized assistant, and restored messages +- `availableWidth` — exact terminal columns available for the transformed Markdown content +- `messageId`, `timestamp` — session-entry ID and ISO timestamp when rendering from persisted session entries; `undefined` for live messages + +Return the transformed Markdown: + +```typescript +pi.registerMarkdownTransformer((markdown, { messageType, isStreaming }) => { + if (isStreaming || messageType === "assistant-thinking") return markdown; + return markdown.replaceAll("-->", "→"); +}); +``` + +If a transformer throws, Pi keeps the Markdown produced so far and continues with the next transformer. The hook is display-only: the original message remains unchanged in the session and model context. It runs for new user messages, assistant streaming updates, restored session messages, and terminal width changes, so transformers should remain synchronous and inexpensive. + +### pi.registerEntryRenderer(customType, renderer) + +Register a custom TUI renderer for custom entries with your `customType`. Custom entries are created with `pi.appendEntry()` and do not participate in LLM context. + +```typescript +import { Box, Text } from "@earendil-works/pi-tui"; + +pi.registerEntryRenderer("status-card", (entry, { expanded }, theme) => { + const data = entry.data as { title: string; count: number }; + const box = new Box(1, 1, (text) => theme.bg("customMessageBg", text)); + box.addChild(new Text(`${theme.bold(data.title)}: ${data.count}`)); + if (expanded) { + box.addChild(new Text(theme.fg("dim", JSON.stringify(data, null, 2)))); + } + return box; +}); + +pi.appendEntry("status-card", { title: "Indexed files", count: 17 }); +``` + +### pi.registerShortcut(shortcut, options) + +Register a keyboard shortcut. See [keybindings.md](keybindings.md) for the shortcut format and built-in keybindings. + +```typescript +pi.registerShortcut("ctrl+shift+p", { + description: "Toggle plan mode", + handler: async (ctx) => { + ctx.ui.notify("Toggled!"); + }, +}); +``` + +### pi.registerFlag(name, options) + +Register a CLI flag. + +```typescript +pi.registerFlag("plan", { + description: "Start in plan mode", + type: "boolean", + default: false, +}); + +// Check the value after loading; in the factory, getFlag() returns the default +pi.on("session_start", () => { + if (pi.getFlag("plan")) { + // Plan mode enabled + } +}); +``` + +### pi.exec(command, args, options?) + +Execute a shell command. + +```typescript +const result = await pi.exec("git", ["status"], { signal, timeout: 5000 }); +// result.stdout, result.stderr, result.code, result.killed +``` + +### pi.getActiveTools() / pi.getAllTools() / pi.setActiveTools(names) + +Manage active tools. This works for both built-in tools and dynamically registered tools. `pi.getActiveTools()` returns the active tool names as `string[]`; `pi.getAllTools()` returns metadata for all configured tools. + +```typescript +const active = pi.getActiveTools(); // ["read", "bash", ...] +const all = pi.getAllTools(); +// all = [{ +// name: "read", +// description: "Read file contents...", +// parameters: ..., +// promptGuidelines: ["Use read to examine files instead of cat or sed."], +// exposure: "direct", +// sourceInfo: { path: "builtin:read", source: "builtin", scope: "temporary", origin: "top-level" } +// }, ...] +// Tools shipped with Pi: core tools and tools from built-in extensions such as builtin:mcp +const shippedTools = all.filter((t) => t.sourceInfo.source === "builtin"); +// Tools from installed or local extensions +const extensionTools = all.filter((t) => t.sourceInfo.source !== "builtin" && t.sourceInfo.source !== "sdk"); +pi.setActiveTools([...new Set([...active, "my_custom_tool"])]); // Keep current tools and enable my_custom_tool +pi.setActiveTools(["read", "bash"]); // Only read and bash +``` + +`pi.getAllTools()` returns `name`, `description`, `parameters`, `promptGuidelines`, `exposure`, `sourceInfo`, and, when set, `namespace`, `annotations`, and `reachesTools`. + +Typical `sourceInfo` values: +- `source: "builtin"` for tools shipped with Pi: core tools (`path: "builtin:"`) and tools registered by built-in extensions such as `builtin:mcp` (`path: "builtin:"`) +- `source: "sdk"` for tools passed via `createAgentSession({ customTools })` +- extension source metadata for tools registered by extensions + +`pi.setActiveTools()` ignores unknown and `hidden` tools. Tools with `codemode` or `deferred` exposure stay callable from codemode scripts whether active or not. + +### pi.setModel(model) + +Set the model for the current session. The change is recorded in session history and restored when that session is resumed, but it does not change the configured `defaultProvider` or `defaultModel` used by new sessions. Returns `false` if authentication is not configured for the model's provider. See [models.md](models.md) for configuring custom models. + +```typescript +const model = ctx.modelRegistry.find("anthropic", "claude-sonnet-4-5"); +if (model) { + const success = await pi.setModel(model); + if (!success) { + ctx.ui.notify("No API key for this model", "error"); + } +} +``` + +### pi.getThinkingLevel() / pi.setThinkingLevel(level) + +Get the current thinking level. Level is clamped to model capabilities (non-reasoning models always use "off"). Changes emit `thinking_level_select`. + +`pi.setThinkingLevel()` changes the thinking level for the current session. The change is recorded in session history and restored when that session is resumed, but it does not change the configured default used by new sessions. + +```typescript +const current = pi.getThinkingLevel(); // "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max" +pi.setThinkingLevel("high"); +``` + +### pi.events + +Shared event bus for communication between extensions: + +```typescript +pi.events.on("my:event", (data) => { ... }); +pi.events.emit("my:event", { ... }); +``` + +### pi.registerProvider(name, config) + +Register or override a model provider dynamically. Useful for proxies, custom endpoints, or team-wide model configurations. + +Calls made during the extension factory function are queued and applied once the runner initialises. Calls made after that — for example from a command handler following a user setup flow — take effect immediately without requiring a `/reload`. + +Dynamic providers can implement `refreshModels`. Pi calls it during model refresh, publishes the returned list synchronously through the provider, and passes the canonical credential/stored-catalog/network/signal context. The extension decides whether to persist catalog metadata through generation-checked `context.publish({ persist: entry })`; live servers such as llama.cpp can return models without persisting them. + +`context.signal` is always a concrete signal and provider callbacks must pass it to blocking I/O. Public `ModelRuntime.refresh()` and `ModelRegistry.refresh()` calls accept an optional signal and are unbounded when it is omitted; extensions and applications choose their own deadlines. Cancellation stops the caller waiting even if a provider ignores the signal, but cooperation is still required to stop the underlying work. + +Extensions that need native provider auth, filtering, refresh, or stream behavior can register a complete `Provider` from `@earendil-works/pi-ai`. The provider becomes the composition base and `models.json` overrides still apply above it. + +```typescript +import { createProvider, openAICompletionsApi } from "@earendil-works/pi-ai/compat"; + +const provider = createProvider({ + id: "local-server", + name: "Local Server", + baseUrl: "http://localhost:8080/v1", + auth: { + apiKey: { + name: "Local server setup", + async login(interaction) { + return { + type: "api_key", + key: await interaction.prompt({ type: "secret", message: "API key" }), + }; + }, + async resolve({ credential }) { + return credential?.key + ? { auth: { apiKey: credential.key }, source: "stored API key" } + : undefined; + }, + }, + }, + models: [], + api: openAICompletionsApi(), +}); + +pi.registerProvider(provider); + +// Register a new provider with custom models +pi.registerProvider("my-proxy", { + name: "My Proxy", + baseUrl: "https://proxy.example.com", + apiKey: "$PROXY_API_KEY", // env var reference + api: "anthropic-messages", + models: [ + { + id: "claude-sonnet-4-20250514", + name: "Claude 4 Sonnet (proxy)", + reasoning: false, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200000, + maxTokens: 16384 + } + ] +}); + +// Register a live llama.cpp catalog without persisting discovered models +pi.registerProvider("llama.cpp", { + baseUrl: "http://localhost:8080/v1", + apiKey: "local", + api: "openai-completions", + async refreshModels({ signal }) { + const response = await fetch("http://localhost:8080/v1/models", { signal }); + const { data } = (await response.json()) as { data: Array<{ id: string }> }; + return data.map(({ id }) => ({ + id, + name: id, + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 16384 + })); + } +}); + +// Override baseUrl for an existing provider (keeps all models) +pi.registerProvider("anthropic", { + baseUrl: "https://proxy.example.com" +}); + +// Register provider with OAuth support for /login +pi.registerProvider("corporate-ai", { + baseUrl: "https://ai.corp.com", + api: "openai-responses", + models: [...], + oauth: { + name: "Corporate AI (SSO)", + async login(callbacks) { + // Custom OAuth flow + callbacks.onAuth({ url: "https://sso.corp.com/..." }); + const code = await callbacks.onPrompt({ message: "Enter code:" }); + return { refresh: code, access: code, expires: Date.now() + 3600000 }; + }, + async refreshToken(credentials, signal) { + signal.throwIfAborted(); + // Refresh logic + return credentials; + }, + getApiKey(credentials) { + return credentials.access; + } + } +}); +``` + +The object form accepts a complete pi-ai `Provider`, including native `auth`, `getModels`, `refreshModels`, `filterModels`, `stream`, and `streamSimple` behavior. + +**Legacy config options:** +- `name` - Display name for the provider in UI such as `/login`. +- `baseUrl` - API endpoint URL. Each defined model needs one, from the model definition or the provider. +- `apiKey` - API key literal, environment interpolation (`$ENV_VAR` or `${ENV_VAR}`), or leading `!command`. Optional: without it, users store a key with `/login`. `$$` escapes `$`, and `$!` escapes a literal `!` without triggering command execution. +- `api` - API type: `"anthropic-messages"`, `"openai-completions"`, `"openai-responses"`, etc. +- `headers` - Custom headers to include in requests. +- `authHeader` - If true, adds `Authorization: Bearer` header automatically. +- `models` - Array of model definitions. If provided, replaces all existing models for this provider. Model definitions can set `baseUrl` to override the provider endpoint for that model. +- `refreshModels` - Async dynamic discovery callback. Its returned models replace extension-provided models. `context.stored` contains the persisted provider snapshot; use generation-checked `context.publish({ persist: entry })` only when updated catalog data should persist. Use `persist: null` to delete that snapshot. +- `oauth` - OAuth provider config for `/login` support. When provided, the provider appears in the login menu. +- `streamSimple` - Custom streaming implementation for non-standard APIs. + +See [custom-provider.md](custom-provider.md) for advanced topics: custom streaming APIs, OAuth details, model definition reference. + +### pi.unregisterProvider(name) + +Remove a previously registered provider and its models. Built-in models that were overridden by the provider are restored. Has no effect if the provider was not registered. + +Like `registerProvider`, this takes effect immediately when called after the initial load phase, so a `/reload` is not required. + +```typescript +pi.registerCommand("my-setup-teardown", { + description: "Remove the custom proxy provider", + handler: async (_args, _ctx) => { + pi.unregisterProvider("my-proxy"); + }, +}); +``` + +### pi.registerVirtualModel(model) / pi.unregisterVirtualModel(provider, id) + +Register a selectable catalog entry that routes each request to a physical model. `ctx.model` and the session's model selection name the virtual model; assistant messages record the physical model and thinking level the router picked. Registering the same provider and ID again replaces the virtual model. See [Virtual Models](virtual-models.md). + +### pi.registerMcpServer(name, config) / pi.unregisterMcpServer(name) / pi.getMcpServers() + +Register an MCP server for the current session. See [MCP Servers](#mcp-servers). + +### pi.registerToolRenderer(resolver) + +Choose how calls to any tool are drawn, including tools that are not registered yet, such as MCP tools in a resumed session before their server connected. The resolver receives the tool name and `next()`, which returns what the remaining resolvers (in extension load order), then the registered tool, would use, so `next() ?? mine` only fills in. + +```typescript +pi.registerToolRenderer((toolName, next) => { + if (!toolName.startsWith("mcp__")) return next(); + return next() ?? { renderCall: (args, theme) => new Text(theme.fg("toolTitle", toolName), 0, 0) }; +}); +``` + +### pi.getSettings() + +Return a copy of the effective settings, with global and project settings merged and overrides applied. + + + +## State Management + +Extensions with state should store it in tool result `details` for proper branching support: + +```typescript +export default function (pi: ExtensionAPI) { + let items: string[] = []; + + // Reconstruct state from the active branch + const reconstruct = (ctx: ExtensionContext) => { + items = []; + for (const entry of ctx.sessionManager.getBranch()) { + if (entry.type === "message" && entry.message.role === "toolResult") { + if (entry.message.toolName === "my_tool") { + items = (entry.message.details as { items?: string[] } | undefined)?.items ?? []; + } + } + } + }; + pi.on("session_start", async (_event, ctx) => reconstruct(ctx)); + pi.on("session_tree", async (_event, ctx) => reconstruct(ctx)); // /tree changes the branch + + pi.registerTool({ + name: "my_tool", + // ... + async execute(toolCallId, params, signal, onUpdate, ctx) { + items.push("new item"); + return { + content: [{ type: "text", text: "Added" }], + details: { items: [...items] }, // Store for reconstruction + }; + }, + }); +} +``` + +Rebuild from `getBranch()`, not from every entry: abandoned branches represent alternative histories. Use [`pi.appendEntry()`](#piappendentrycustomtype-data) for durable data that should not reach the model. - -### Tools +## Custom Tools -A custom tool defines a name, model-facing description, TypeBox parameter schema, and `execute()` function. -Its result requires model-facing `content` and a `details` field for rendering or state reconstruction. -Use `details: undefined` when there are no structured details. If the tool makes nested model calls, include their `usage` in the result so session totals remain accurate. +Register tools the LLM can call via `pi.registerTool()`. Tools appear in the system prompt and can have custom rendering. -Throw from `execute()` to produce a failed tool result. -Returning an object does not mark it as an error. -Return `terminate: true` only when the agent should skip its automatic follow-up after every completed tool in that batch agrees to terminate. +Use `promptSnippet` for a short one-line entry in the `` section of the default system prompt. If omitted, custom tools are left out of that section. -Use sequential execution when tools share mutable in-memory state. -File-mutating tools should wrap the complete read-modify-write operation with `withFileMutationQueue()`. -Truncate large model-facing results and tell the model where to read the complete output. +Use `promptGuidelines` to add tool-specific bullets to the `` section of the default system prompt. These bullets are included only while the tool is active (for example, after `pi.setActiveTools([...])`). -Declare `outputSchema` and return a matching `structuredContent` when the result is data. The model still receives `content`; programmatic callers such as codemode scripts receive `structuredContent` instead of the text. Tools without `outputSchema` are passed to scripts as their text content. To report a failure that still carries data, return the result with `isError: true` instead of throwing: the model sees an error, and scripts still receive `structuredContent`. +**Important:** `promptGuidelines` bullets are appended flat to the `` section with no tool name prefix or grouping. Each guideline must name the tool it refers to — avoid "Use this tool when..." because the LLM cannot tell which tool "this" means. Write "Use my_tool when..." instead. -A tool can run other tools with `ctx.executeTool(name, args, { signal, onUpdate })`. Nested calls go through argument validation and the `tool_call` and `tool_result` handlers like model-issued calls, and emit `tool_execution_start`, `tool_execution_update`, and `tool_execution_end`; all of these events carry `parentToolCallId`, and their `toolCallId` is assigned by pi as `/`. These ids do not appear as tool calls or tool results in the transcript. Nested calls do not add transcript entries: their results only reach the calling tool, which reports them itself, for example through `onUpdate` and `details`. The session keeps a bounded record of them (name, arguments, status, duration, error; never results) as `nestedCalls` on the calling tool's result message. It is used for compaction file lists and shown in HTML exports. Arguments over 8 KiB per call or 32 KiB per tool result are omitted, at most 256 calls are kept, and `complete: false` marks a record that lost anything. The `usage` of nested results, at every depth, is added to the calling tool's result `usage`, so a tool reports only its own usage, not that of the tools it called. `ctx.tools` lists the tools `ctx.executeTool()` can call. `tool_result` handlers that redact `content` should also replace `structuredContent`; replacing only `content` drops it. +Note: Some models include the @ prefix in tool path arguments. Built-in tools strip a leading @ before resolving paths. If your custom tool accepts a path, normalize a leading @ as well. -See [`hello.ts`](../examples/extensions/hello.ts), [`todo.ts`](../examples/extensions/todo.ts), [`dynamic-tools.ts`](../examples/extensions/dynamic-tools.ts), and [`truncated-tool.ts`](../examples/extensions/truncated-tool.ts). +If your custom tool mutates files, use `withFileMutationQueue()` so it participates in the same per-file queue as built-in `edit` and `write`. This matters because tool calls run in parallel by default. Without the queue, two tools can read the same old file contents, compute different updates, and then whichever write lands last overwrites the other. -### Tool exposure +Example failure case: your custom tool edits `foo.ts` while built-in `edit` also changes `foo.ts` in the same assistant turn. If your tool does not participate in the queue, both can read the original `foo.ts`, apply separate changes, and one of those changes is lost. + +Pass the real target file path to `withFileMutationQueue()`, not the raw user argument. Resolve it to an absolute path first, relative to `ctx.cwd` or your tool's working directory. For existing files, the helper canonicalizes through `realpath()`, so symlink aliases for the same file share one queue. For new files, it falls back to the resolved absolute path because there is nothing to `realpath()` yet. + +Queue the entire mutation window on that target path. That includes read-modify-write logic, not just the final write. + +```typescript +import { withFileMutationQueue } from "@earendil-works/pi-coding-agent"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; + +async execute(_toolCallId, params, _signal, _onUpdate, ctx) { + const absolutePath = resolve(ctx.cwd, params.path); + + return withFileMutationQueue(absolutePath, async () => { + await mkdir(dirname(absolutePath), { recursive: true }); + const current = await readFile(absolutePath, "utf8"); + const next = current.replace(params.oldText, params.newText); + await writeFile(absolutePath, next, "utf8"); + + return { + content: [{ type: "text", text: `Updated ${params.path}` }], + details: {}, + }; + }); +} +``` + +### Tool Definition + +```typescript +import { type Static, Type } from "typebox"; +import { StringEnum } from "@earendil-works/pi-ai"; + +const myToolSchema = Type.Object({ + action: StringEnum(["list", "add"] as const), + text: Type.Optional(Type.String()), +}); + +pi.registerTool({ + name: "my_tool", + label: "My Tool", + description: "What this tool does (shown to LLM)", + promptSnippet: "List or add items in the project todo list", + promptGuidelines: [ + "Use my_tool for todo planning instead of direct file edits when the user asks for a task list." + ], + parameters: myToolSchema, + prepareArguments(args) { + // Fold a legacy `oldAction` field into `action`. Schema validation still runs afterwards. + const input = (args ?? {}) as { action?: unknown; oldAction?: unknown }; + if (input.action === undefined && input.oldAction !== undefined) { + return { ...input, action: input.oldAction } as Static; + } + return args as Static; + }, + + async execute(toolCallId, params, signal, onUpdate, ctx) { + // Check for cancellation + if (signal?.aborted) { + return { content: [{ type: "text", text: "Cancelled" }], details: {} }; + } + + // Stream progress updates; partial results also need details + onUpdate?.({ + content: [{ type: "text", text: "Working..." }], + details: {}, + }); + + // Run commands via pi.exec (captured from extension closure) + const result = await pi.exec("some-command", [], { signal }); + + // Return result + return { + content: [{ type: "text", text: "Done" }], // Sent to LLM + details: { data: result }, // For rendering & state + // usage: nestedModelResponse.usage, // Optional nested LLM usage + // Optional: stop after this tool batch when every finalized tool result + // in the batch also returns terminate: true. + terminate: true, + }; + }, + + // Optional: Custom rendering + renderCall(args, theme, context) { ... }, + renderResult(result, options, theme, context) { ... }, +}); +``` + +**Result fields:** Every result, including partial results passed to `onUpdate`, needs model-facing `content` and a `details` value. Use `details: undefined` when there is nothing to store; the type of `details` must match across all results of the tool. + +**Usage accounting:** If a tool makes nested LLM calls, return their combined `Usage` as `usage`. Pi persists it on the tool result and includes it in footer, `/session`, and RPC session totals. `tool_result` handlers can inspect or replace this value. + +**Signaling errors:** To mark a tool execution as failed, throw an error from `execute`, or return a result with `isError: true`. Both report an error result to the LLM; a returned result also keeps `details` and `structuredContent` for rendering and scripts. Other returned values never set the error flag. + +**Early termination:** Return `terminate: true` from `execute()` to hint that the automatic follow-up LLM call should be skipped after the current tool batch. This only takes effect when every finalized tool result in that batch is terminating. See [examples/extensions/structured-output.ts](../examples/extensions/structured-output.ts) for a minimal example where the agent ends on a final structured-output tool call. + +```typescript +// Correct: throw to signal an error +async execute(toolCallId, params) { + if (!isValid(params.input)) { + throw new Error(`Invalid input: ${params.input}`); + } + return { content: [{ type: "text", text: "OK" }], details: {} }; +} +``` + +**String enums:** Prefer `StringEnum` from `@earendil-works/pi-ai`. It produces a plain `enum` schema that every provider accepts. + +**Structured output:** Declare `outputSchema` and return a matching `structuredContent` when the result is data. The model still receives `content`; programmatic callers such as codemode scripts receive `structuredContent` instead of the text. Tools without `outputSchema` are passed to scripts as their text content. To report a failure that still carries data, return the result with `isError: true` instead of throwing: the model sees an error, and scripts still receive `structuredContent`. + +**Execution mode:** Tool calls from one assistant message run in parallel by default. Set `executionMode: "sequential"` when a tool must not run concurrently with other calls, for example because it shares mutable in-memory state. + +**Other definition fields:** + +| Field | Purpose | +|---|---| +| `defaultActive` | Set `false` to register the tool inactive; `--tools`, the `defaultTools` setting, or `pi.setActiveTools()` activates it | +| `exposure`, `namespace`, `annotations`, `reachesTools`, `prepareLoadout` | Control how the model and other tools reach the tool; see [Tool Exposure](#tool-exposure) | +| `renderShell`, `renderCall`, `renderResult` | Custom rendering; see [Custom Rendering](#custom-rendering) | +| `constrainedSampling` | Optional provider-side constrained sampling request for the tool's arguments | + +Use `defineTool()` to keep parameter type inference when you assign a tool definition to a variable before registering it. + +**Argument preparation:** `prepareArguments(args)` is optional. If defined, it runs before schema validation and before `execute()`. Use it to mimic an older accepted input shape when Pi resumes an older session whose stored tool call arguments no longer match the current schema. Return the object you want validated against `parameters`. Keep the public schema strict. Do not add deprecated compatibility fields to `parameters` just to keep old resumed sessions working. + +Example: an older session may contain an `edit` tool call with top-level `oldText` and `newText`, while the current schema only accepts `edits: [{ oldText, newText }]`. + +```typescript +const editSchema = Type.Object({ + path: Type.String(), + edits: Type.Array( + Type.Object({ + oldText: Type.String(), + newText: Type.String(), + }), + ), +}); +type EditInput = Static; + +pi.registerTool({ + name: "edit", + label: "Edit", + description: "Edit a single file using exact text replacement", + parameters: editSchema, + prepareArguments(args) { + const input = (args ?? {}) as Partial & { oldText?: unknown; newText?: unknown }; + if (typeof input.oldText !== "string" || typeof input.newText !== "string") { + return args as EditInput; // validation reports anything still invalid + } + + const { oldText, newText, ...rest } = input; + return { ...rest, edits: [...(input.edits ?? []), { oldText, newText }] } as EditInput; + }, + async execute(toolCallId, params, signal, onUpdate, ctx) { + // params now matches the current schema + return { + content: [{ type: "text", text: `Applying ${params.edits.length} edit block(s)` }], + details: {}, + }; + }, +}); +``` + +### Overriding Built-in Tools + +Extensions can override built-in tools (`read`, `bash`, `powershell`, `edit`, `write`, `grep`, `find`, `ls`) by registering a tool with the same name. The extension tool replaces the built-in without a warning. + +```bash +# Extension's read tool replaces built-in read +pi -e ./tool-override.ts +``` + +Alternatively, use `--no-builtin-tools` to start without any built-in tools while keeping extension tools enabled: +```bash +# No built-in tools, only extension tools +pi --no-builtin-tools -e ./my-extension.ts +``` + +See [examples/extensions/tool-override.ts](../examples/extensions/tool-override.ts) for a complete example that overrides `read` with logging and access control. + +**Rendering:** Built-in renderer inheritance is resolved per slot. Execution override and rendering override are independent. If your override omits `renderCall`, the built-in `renderCall` is used. If your override omits `renderResult`, the built-in `renderResult` is used. If your override omits both, the built-in renderer is used automatically (syntax highlighting, diffs, etc.). This lets you wrap built-in tools for logging or access control without reimplementing the UI. + +**Prompt metadata:** `promptSnippet` and `promptGuidelines` are not inherited from the built-in tool. If your override should keep those prompt instructions, define them on the override explicitly. + +**Your implementation must match the exact result shape**, including the `details` type. The UI and session logic depend on these shapes for rendering and state tracking. + +Built-in tool implementations: +- [read.ts](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/tools/read.ts) - `ReadToolDetails` +- [bash.ts](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/tools/bash.ts) - `BashToolDetails` +- [powershell.ts](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/tools/powershell.ts) - `PowerShellToolDetails` +- [edit.ts](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/tools/edit.ts) +- [write.ts](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/tools/write.ts) +- [grep.ts](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/tools/grep.ts) - `GrepToolDetails` +- [find.ts](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/tools/find.ts) - `FindToolDetails` +- [ls.ts](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/tools/ls.ts) - `LsToolDetails` + +### Remote Execution + +Built-in tools support pluggable operations for delegating to remote systems (SSH, containers, etc.): + +```typescript +import { createReadTool, type ReadOperations } from "@earendil-works/pi-coding-agent"; + +const localRead = createReadTool(cwd); + +// Register, checking the remote configuration at execution time +pi.registerTool({ + ...localRead, + async execute(id, params, signal, onUpdate, _ctx) { + const ssh = getSshConfig(); + if (ssh) { + const operations: ReadOperations = { + readFile: (path) => sshReadBuffer(ssh, path), // must resolve to a Buffer + access: (path) => sshExec(ssh, `test -r ${path}`).then(() => {}), + }; + return createReadTool(cwd, { operations }).execute(id, params, signal, onUpdate); + } + return localRead.execute(id, params, signal, onUpdate); + }, +}); +``` + +The `create*ToolDefinition()` factories, such as `createReadToolDefinition()`, return complete tool definitions, including the built-in prompt snippet and guidelines, for the same purpose. Rendering does not need copying: a tool registered under a built-in name inherits the built-in renderer for each slot it leaves out (see [Overriding Built-in Tools](#overriding-built-in-tools)). + +**Operations interfaces:** `ReadOperations`, `WriteOperations`, `EditOperations`, `BashOperations`, `PowerShellOperations`, `LsOperations`, `GrepOperations`, `FindOperations` + +For `user_bash`, extensions can reuse Pi's local shell backend via `createLocalBashOperations()` (or `createLocalPowerShellOperations()`) instead of reimplementing local process spawning, shell resolution, and process-tree termination. + +The `bash` and `powershell` tools also support a spawn hook to adjust the command, cwd, or env before execution: + +```typescript +import { createBashTool } from "@earendil-works/pi-coding-agent"; + +const bashTool = createBashTool(cwd, { + spawnHook: ({ command, cwd, env }) => ({ + command: `source ~/.profile\n${command}`, + cwd: `/mnt/sandbox${cwd}`, + env: { ...env, CI: "1" }, + }), +}); +``` + +`createBashTool()` and `createPowerShellTool()` expose the current session to commands through `PI_SESSION_ID`, `PI_SESSION_FILE`, `PI_PROVIDER`, `PI_MODEL`, and `PI_REASONING_LEVEL`. Injection happens before `spawnHook`, so hooks receive these values in `env` and preserve them when they spread the existing environment as above. Set `exposeSessionEnvironment: false` to disable them: + +```typescript +const bashTool = createBashTool(cwd, { + exposeSessionEnvironment: false, +}); +``` + +See [Shell tool session environment](environment-variables.md#shell-tool-session-environment) for variable semantics. See [examples/extensions/ssh.ts](../examples/extensions/ssh.ts) for a complete SSH example with `--ssh` flag. + +### Output Truncation + +**Tools MUST truncate their output** to avoid overwhelming the LLM context. Large outputs can cause: +- Context overflow errors (prompt too long) +- Compaction failures +- Degraded model performance + +The built-in limit is **50KB** (~10k tokens) and **2000 lines**, whichever is hit first. Use the exported truncation utilities: + +```typescript +import { + truncateHead, // Keep first N lines/bytes (good for file reads, search results) + truncateTail, // Keep last N lines/bytes (good for logs, command output) + truncateLine, // Truncate a single line to a character count; returns { text, wasTruncated } + formatSize, // Human-readable size (e.g., "50KB", "1.5MB") + DEFAULT_MAX_BYTES, // 50KB + DEFAULT_MAX_LINES, // 2000 +} from "@earendil-works/pi-coding-agent"; + +async execute(toolCallId, params, signal, onUpdate, ctx) { + const output = await runCommand(); + + // Apply truncation + const truncation = truncateHead(output, { + maxLines: DEFAULT_MAX_LINES, + maxBytes: DEFAULT_MAX_BYTES, + }); + + let result = truncation.content; + + if (truncation.truncated) { + // Write full output to temp file + const tempFile = writeTempFile(output); + + // Inform the LLM where to find complete output + result += `\n\n[Output truncated: ${truncation.outputLines} of ${truncation.totalLines} lines`; + result += ` (${formatSize(truncation.outputBytes)} of ${formatSize(truncation.totalBytes)}).`; + result += ` Full output saved to: ${tempFile}]`; + } + + return { content: [{ type: "text", text: result }], details: { truncation } }; +} +``` + +**Key points:** +- Use `truncateHead` for content where the beginning matters (search results, file reads) +- Use `truncateTail` for content where the end matters (logs, command output) +- Always inform the LLM when output is truncated and where to find the full version +- Document the truncation limits in your tool's description + +See [examples/extensions/truncated-tool.ts](../examples/extensions/truncated-tool.ts) for a complete example wrapping `rg` (ripgrep) with proper truncation. + +### Multiple Tools + +One extension can register multiple tools with shared state: + +```typescript +export default function (pi: ExtensionAPI) { + let connection = null; + + pi.registerTool({ name: "db_connect", ... }); + pi.registerTool({ name: "db_query", ... }); + pi.registerTool({ name: "db_close", ... }); + + pi.on("session_shutdown", async () => { + connection?.close(); + }); +} +``` + + + +### Custom Rendering + +Tools can provide `renderCall` and `renderResult` for custom TUI display. See [tui.md](tui.md) for the full component API and [tool-execution.ts](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/modes/interactive/components/tool-execution.ts) for how tool rows are composed. + +By default, tool output is wrapped in a `Box` that handles background and horizontal padding from the `outputPad` setting. Tool renderers are also used in HTML exports. A defined `renderCall` or `renderResult` must return a `Component`. If a slot renderer is not defined, `tool-execution.ts` uses fallback rendering for that slot. + +Set `renderShell: "self"` when the tool should render its own shell instead of using the default `Box`. This is useful for tools that need complete control over framing or background behavior, for example large previews that must stay visually stable after the tool settles. + +```typescript +pi.registerTool({ + name: "my_tool", + label: "My Tool", + description: "Custom shell example", + parameters: Type.Object({}), + renderShell: "self", + async execute() { + return { content: [{ type: "text", text: "ok" }], details: undefined }; + }, + renderCall(args, theme, context) { + return new Text(theme.fg("accent", "my custom shell"), 0, 0); + }, +}); +``` + +`renderCall` and `renderResult` each receive a `context` object with: +- `args` - the current tool call arguments +- `state` - shared row-local state across `renderCall` and `renderResult` +- `lastComponent` - the previously returned component for that slot, if any +- `invalidate()` - request a rerender of this tool row +- `toolCallId`, `cwd`, `executionStarted`, `argsComplete`, `isPartial`, `expanded`, `showImages`, `isError` +- `durationMs` - how long execution took, from the final result; `undefined` while running +- `outputPad` - horizontal padding from the `outputPad` setting; renderers with `renderShell: "self"` apply it themselves + +Use `context.state` for cross-slot shared state. Keep slot-local caches on the returned component instance when you want to reuse and mutate the same component across renders. + +#### renderCall + +Renders the tool call or header: + +```typescript +import { Text } from "@earendil-works/pi-tui"; + +renderCall(args, theme, context) { + const text = (context.lastComponent as Text | undefined) ?? new Text("", 0, 0); + let content = theme.fg("toolTitle", theme.bold("my_tool ")); + content += theme.fg("muted", args.action); + if (args.text) { + content += " " + theme.fg("dim", `"${args.text}"`); + } + text.setText(content); + return text; +} +``` + +#### renderResult + +Renders the tool result or output: + +```typescript +renderResult(result, { expanded, isPartial }, theme, context) { + if (isPartial) { + return new Text(theme.fg("warning", "Processing..."), 0, 0); + } + + if (result.details?.error) { + return new Text(theme.fg("error", `Error: ${result.details.error}`), 0, 0); + } + + let text = theme.fg("success", "✓ Done"); + if (expanded && result.details?.items) { + for (const item of result.details.items) { + text += "\n " + theme.fg("dim", item); + } + } + return new Text(text, 0, 0); +} +``` + +If a slot intentionally has no visible content, return an empty `Component` such as an empty `Container`. + +#### Keybinding Hints + +Use `keyHint()` to display keybinding hints that respect the active keybinding configuration: + +```typescript +import { keyHint } from "@earendil-works/pi-coding-agent"; + +renderResult(result, { expanded }, theme, context) { + let text = theme.fg("success", "✓ Done"); + if (!expanded) { + text += ` (${keyHint("app.tools.expand", "to expand")})`; + } + return new Text(text, 0, 0); +} +``` + +Available functions: +- `keyHint(keybinding, description)` - Formats a configured keybinding id such as `"app.tools.expand"` or `"tui.select.confirm"` +- `keyText(keybinding)` - Returns the raw configured key text for a keybinding id +- `rawKeyHint(key, description)` - Format a raw key string + +Use namespaced keybinding ids: +- Coding-agent ids use the `app.*` namespace, for example `app.tools.expand`, `app.editor.external`, `app.session.rename` +- Shared TUI ids use the `tui.*` namespace, for example `tui.select.confirm`, `tui.select.cancel`, `tui.input.tab` + +For the exhaustive list of keybinding ids and defaults, see [keybindings.md](keybindings.md). `keybindings.json` uses those same namespaced ids. + +Custom editors and `ctx.ui.custom()` components receive `keybindings: KeybindingsManager` as an injected argument. They should use that injected manager directly instead of calling `getKeybindings()` or `setKeybindings()`. + +#### Best Practices + +- Use `Text` with padding `(0, 0)`. The default Box handles padding. +- Use `\n` for multi-line content. +- Handle `isPartial` for streaming progress. +- Support `expanded` for detail on demand. +- Keep default view compact. +- Read `context.args` in `renderResult` instead of copying args into `context.state`. +- Use `context.state` only for data that must be shared across call and result slots. +- Reuse `context.lastComponent` when the same component instance can be updated in place. +- Use `renderShell: "self"` only when the default boxed shell gets in the way. In self-shell mode the tool is responsible for its own framing, padding, and background. + +#### Fallback + +If a slot renderer is not defined or throws: +- `renderCall`: Shows the tool name +- `renderResult`: Shows raw text from `content` + +### Dynamic Tool Loading + +Extensions can register many tools while keeping only a small initial set active. A tool can then change the active set with `pi.setActiveTools()` during execution. Pi stores the initial prompt and tool loadout in the transcript's first system message, then appends tool and prompt deltas before the next model request. Providers that cannot represent a transition receive a complete transcript checkpoint, which may invalidate the cached prefix. + +The lifecycle is: + +1. Register every tool with `pi.registerTool()` so it appears in `pi.getAllTools()`. Register searchable tools with `defaultActive: false` so they start inactive. +2. Keep loader tools, such as `search_tools`, active. +3. During loader execution, call `pi.setActiveTools()` with the desired active tool names. Names must already be registered; unknown and `hidden` names are ignored. + +Pi also ships a built-in `tool_search` tool. Register tools with `exposure: "deferred"` to make them findable by it instead of writing your own loader; see [Tool Exposure](#tool-exposure). + +#### Search tool example + +The following extension registers two inactive searchable tools and keeps `search_tools` active as their loader. The example uses simple keyword matching, but the search implementation could use BM25, embeddings, a remote catalog, or project-specific routing. + +```typescript +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { Type } from "typebox"; + +const SEARCHABLE_TOOL_NAMES = new Set(["lookup_weather", "search_issues"]); + +export default function (pi: ExtensionAPI) { + pi.registerTool({ + name: "lookup_weather", + label: "Lookup Weather", + defaultActive: false, + description: "Look up the current weather for a city", + parameters: Type.Object({ city: Type.String() }), + async execute(_toolCallId, params) { + return { + content: [{ type: "text", text: `Weather for ${params.city}: sunny` }], + details: {}, + }; + }, + }); + + pi.registerTool({ + name: "search_issues", + label: "Search Issues", + defaultActive: false, + description: "Search project issues by keyword", + parameters: Type.Object({ query: Type.String() }), + async execute(_toolCallId, params) { + return { + content: [{ type: "text", text: `No open issues matching ${params.query}` }], + details: {}, + }; + }, + }); + + pi.registerTool({ + name: "search_tools", + label: "Search Tools", + description: "Search for and enable tools relevant to a task", + promptSnippet: "Search for additional tools when the active tools cannot perform the task", + promptGuidelines: [ + "Use search_tools when a task requires a capability that is not currently available.", + ], + parameters: Type.Object({ + query: Type.String({ description: "Capability or task to search for" }), + limit: Type.Optional(Type.Integer({ minimum: 1, maximum: 10 })), + }), + async execute(_toolCallId, params) { + const terms = params.query.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean); + const matches = pi.getAllTools() + .filter((tool) => SEARCHABLE_TOOL_NAMES.has(tool.name)) + .map((tool) => ({ + tool, + score: terms.reduce( + (score, term) => + score + (`${tool.name} ${tool.description}`.toLowerCase().includes(term) ? 1 : 0), + 0, + ), + })) + .filter((match) => match.score > 0) + .sort((a, b) => b.score - a.score) + .slice(0, params.limit ?? 3) + .map((match) => match.tool.name); + + if (matches.length === 0) { + return { + content: [{ type: "text", text: `No tools found for: ${params.query}` }], + details: { matches: [] as string[], added: [] as string[] }, + }; + } + + const active = pi.getActiveTools(); + const added = matches.filter((name) => !active.includes(name)); + pi.setActiveTools([...new Set([...active, ...added])]); + + return { + content: [{ + type: "text", + text: added.length > 0 + ? `Loaded tools: ${added.join(", ")}` + : `Matching tools already active: ${matches.join(", ")}`, + }], + details: { matches, added }, + }; + }, + }); +} +``` + +When `search_tools` adds a match, the model receives the complete updated tool list on the immediately following request. + +### Tool Exposure `exposure` controls how the model reaches a tool. "Callable" means callable from other tools through `ctx.executeTool()` (`ctx.tools`), as the `codemode` tool's scripts do: @@ -274,7 +2732,7 @@ See [`hello.ts`](../examples/extensions/hello.ts), [`todo.ts`](../examples/exten `namespace: { name, aliases, description, instructions }` groups related tools, as MCP servers do. Codemode tools list a namespace under one heading with its `description`. `instructions` holds longer usage guidance; it is not listed, and codemode scripts read it with `describeNamespace(name)`. `aliases` are other names scripts may look the namespace up by, for example the server name `docs` for `mcp__docs`; a name and the same name as a script identifier (`my-docs`, `my_docs`) match alike. -Registering a `direct` or `model-only` tool activates it; the other exposures are not activated on registration. The active set (`pi.getActiveTools()`, `pi.setActiveTools()`) is the set of tools declared to the model. `pi.getAllTools()` reports each tool's `exposure`, `namespace`, `annotations`, and `reachesTools`. +Registering a `direct` or `model-only` tool activates it unless it sets `defaultActive: false`; the other exposures are not activated on registration. The active set (`pi.getActiveTools()`, `pi.setActiveTools()`) is the set of tools declared to the model, except active tools whose declarations `prepareLoadout` hides. `pi.getAllTools()` reports each tool's `exposure`, `namespace`, `annotations`, and `reachesTools`. `annotations` are hints about what a tool does, with the meaning of MCP tool annotations: `readOnlyHint`, `destructiveHint`, `idempotentHint`, and `openWorldHint`. MCP tools carry the hints their server declares. Missing hints take the MCP defaults: a tool is not read-only, and may be destructive and reach an open world. The hints are not verified, but a permission extension can use them to decide which calls to confirm. This confirms the calls Codex asks approval for: @@ -294,17 +2752,17 @@ A tool that orchestrates other tools can adjust what the model sees while it is A tool that reaches other tools declares which ones a call may reach with `reachesTools(input)`: `"all"`, or the words of the input that may name them, where a word refers to every tool or namespace whose name it contains. `codemode` returns `"all"` for scripts that search or list tools and the words of the script otherwise; `tool_search` always returns `"all"`. Extensions that register tools late use it from their `tool_call` handler to finish registering the tools a call needs before it runs, as the MCP extension does for servers that are still connecting. The MCP extension also activates `codemode` or `tool_search` for MCP tools only when the tool of that name declares `reachesTools`. -### Activate tools dynamically +### Calling Other Tools -Register every tool first, keep optional tools inactive, and use `pi.setActiveTools()` from a loader tool to select the desired active tools. Names must already be registered; unknown names are ignored. +A tool can run other tools with `ctx.executeTool(name, args, { signal, onUpdate })`. `ctx.tools` lists the tools it can call. The call never rejects for tool failures: unknown tools, validation errors, blocked calls, and thrown errors come back as results with `isError: true`. -Pi records the initial prompt and tool set in the transcript's first system message, then appends tool and prompt changes before the next model request. Providers that cannot represent the transition receive a complete transcript checkpoint, which can invalidate the cached prefix. +Nested calls go through argument validation and the `tool_call` and `tool_result` handlers like model-issued calls, and emit `tool_execution_start`, `tool_execution_update`, and `tool_execution_end`. All of these events carry `parentToolCallId`, and their `toolCallId` is assigned by Pi as `/`. These IDs do not appear as tool calls or tool results in the transcript. -### Tool rendering +Nested calls do not add transcript entries: their results only reach the calling tool, which reports them itself, for example through `onUpdate` and `details`. The session keeps a bounded record of them (name, arguments, status, duration, error; never results) as `nestedCalls` on the calling tool's result message. It is used for compaction file lists and shown in HTML exports. Arguments over 8 KiB per call or 32 KiB per tool result are omitted, at most 256 calls are kept, and `complete: false` marks a record that lost anything. -A tool's `renderCall` and `renderResult` draw its calls in the interactive transcript and in HTML exports. `pi.registerToolRenderer((toolName, next) => renderers)` chooses renderers for calls to any tool, including tools that are not registered yet, such as MCP tools in a resumed session before their server connected. `next()` returns what the remaining resolvers (in extension load order), then the registered tool, would use, so `next() ?? mine` only fills in. +The `usage` of nested results, at every depth, is added to the calling tool's result `usage`, so a tool reports only its own usage, not that of the tools it called. -### MCP servers +## MCP Servers `pi.registerMcpServer(name, config)` adds an MCP server for the current session. `config` has the shape of an `mcpServers` entry in [`mcp.json`](mcp.md): `command`, `args`, `env`, and `cwd` for stdio servers, `url`, `headers`, and `oauth` for HTTP servers, plus `exposure`, `toolExposure`, `description`, `enabled`, and `timeout`. @@ -317,84 +2775,557 @@ Servers registered while the extension loads connect when the session starts, to The built-in MCP support connects registered servers. When nothing does, because another extension replaced it (see [MCP](mcp.md#replace-the-built-in-mcp-support)), each registration is reported as an extension error. Other MCP extensions can connect registered servers too: read them with `pi.getMcpServers()` on `session_start` and handle the `mcp_servers_change` event for later changes. - - - - -### Context and session changes - -`ExtensionContext` provides the working directory, mode, UI, session manager, model runtime, abort signal, context usage, and controls for compaction and shutdown. -Use `ctx.modelRegistry.streamSimple()` for provider-neutral nested model calls. - -Command handlers receive `ExtensionCommandContext`, which adds operations for waiting until idle, reloading, tree navigation, and session replacement. -These operations are command-only because calling them from lifecycle handlers can deadlock the runtime. - -Session replacement invalidates the old context. Capture only plain data before switching, then use the fresh context supplied to `withSession` for session-bound work. - - - - -### State - -Choose storage based on how state participates in the conversation: - -| State | Storage | -|---|---| -| Tool state that follows the active branch | Tool-result `details` | -| Durable data excluded from model context | `pi.appendEntry()` | -| Custom content stored and sent to the model | `pi.sendMessage()` | -| Data outside one session | External storage | - -Reconstruct branch-sensitive state from `ctx.sessionManager.getBranch()` during `session_start`. -Do not rebuild it from every file entry because abandoned branches represent alternative histories. -Register an entry or message renderer when custom stored content should appear in the transcript. - - - - -### UI and modes +## Custom UI -`ctx.ui` provides dialogs, notifications, status text, widgets, titles, editor access, and custom components. -Use `ctx.ui.custom()` only when the interaction needs its own rendering and input. -See [Terminal UI](tui.md) for component, focus, overlay, theme, and performance guidance. +Extensions can interact with users via `ctx.ui` methods and customize how messages/tools render. -Extensions load in interactive, RPC, JSON, and print modes. -Interactive mode provides the complete terminal UI. -RPC can forward supported dialogs and notifications through the [RPC Extension UI protocol](rpc-extension-ui.md), but not custom terminal components; JSON and print modes have no UI. -Guard terminal-only behavior with `ctx.mode === "tui"` and use `ctx.hasUI` for interactions supported by interactive and RPC clients. +For custom components, focus, overlays, themes, and rendering performance, see [Terminal UI](tui.md). The examples it lists show selection lists, settings toggles, cancellable loaders, footers, and widgets. -Keep tool and event behavior independent from rendering so non-interactive modes remain functional. +### Dialogs + +```typescript +// Select from options +const choice = await ctx.ui.select("Pick one:", ["A", "B", "C"]); + +// Confirm dialog +const ok = await ctx.ui.confirm("Delete?", "This cannot be undone"); + +// Text input +const name = await ctx.ui.input("Name:", "placeholder"); + +// Multi-line editor +const text = await ctx.ui.editor("Edit:", "prefilled text"); + +// Notification (non-blocking) +ctx.ui.notify("Done!", "info"); // "info" | "warning" | "error" +``` + +#### Timed Dialogs with Countdown + +Dialogs support a `timeout` option that auto-dismisses with a live countdown display: + +```typescript +// Dialog shows "Title (5s)" → "Title (4s)" → ... → auto-dismisses at 0 +const confirmed = await ctx.ui.confirm( + "Timed Confirmation", + "This dialog will auto-cancel in 5 seconds. Confirm?", + { timeout: 5000 } +); + +if (confirmed) { + // User confirmed +} else { + // User cancelled or timed out +} +``` + +**Return values on timeout:** +- `select()` returns `undefined` +- `confirm()` returns `false` +- `input()` returns `undefined` + +#### Manual Dismissal with AbortSignal + +For more control (e.g., to distinguish timeout from user cancel), use `AbortSignal`: + +```typescript +const controller = new AbortController(); +const timeoutId = setTimeout(() => controller.abort(), 5000); + +const confirmed = await ctx.ui.confirm( + "Timed Confirmation", + "This dialog will auto-cancel in 5 seconds. Confirm?", + { signal: controller.signal } +); + +clearTimeout(timeoutId); + +if (confirmed) { + // User confirmed +} else if (controller.signal.aborted) { + // Dialog timed out +} else { + // User cancelled (pressed Escape or selected "No") +} +``` + +See [examples/extensions/timed-confirm.ts](../examples/extensions/timed-confirm.ts) for complete examples. + +### Widgets, Status, and Footer + +```typescript +// Status in footer (persistent until cleared) +ctx.ui.setStatus("my-ext", "Processing..."); +ctx.ui.setStatus("my-ext", undefined); // Clear + +// Working loader (shown during streaming) +ctx.ui.setWorkingMessage("Thinking deeply..."); +ctx.ui.setWorkingMessage(); // Restore default +ctx.ui.setWorkingVisible(false); // Hide the built-in working loader row entirely +ctx.ui.setWorkingVisible(true); // Show the built-in working loader row + +// Working indicator (shown during streaming) +ctx.ui.setWorkingIndicator({ frames: [ctx.ui.theme.fg("accent", "●")] }); // Static dot +ctx.ui.setWorkingIndicator({ + frames: [ + ctx.ui.theme.fg("dim", "·"), + ctx.ui.theme.fg("muted", "•"), + ctx.ui.theme.fg("accent", "●"), + ctx.ui.theme.fg("muted", "•"), + ], + intervalMs: 120, +}); +ctx.ui.setWorkingIndicator({ frames: [] }); // Hide indicator +ctx.ui.setWorkingIndicator(); // Restore default spinner + +// Widget above editor (default) +ctx.ui.setWidget("my-widget", ["Line 1", "Line 2"]); +// Widget below editor +ctx.ui.setWidget("my-widget", ["Line 1", "Line 2"], { placement: "belowEditor" }); +ctx.ui.setWidget("my-widget", (tui, theme) => new Text(theme.fg("accent", "Custom"), 0, 0)); +ctx.ui.setWidget("my-widget", undefined); // Clear + +// Custom footer (replaces built-in footer entirely) +// footerData provides the git branch and extension statuses from setStatus() +ctx.ui.setFooter((tui, theme, footerData) => ({ + render(width) { return [theme.fg("dim", "Custom footer")]; }, + invalidate() {}, +})); +ctx.ui.setFooter(undefined); // Restore built-in footer + +// Custom header shown at startup above the chat +ctx.ui.setHeader((tui, theme) => new Text(theme.fg("accent", "my-project"), 0, 0)); +ctx.ui.setHeader(undefined); // Restore built-in header + +// Label shown for hidden thinking blocks +ctx.ui.setHiddenThinkingLabel("Reasoning..."); +ctx.ui.setHiddenThinkingLabel(); // Restore default + +// Raw terminal input (interactive mode only); return { consume: true } to swallow it +const unsubscribe = ctx.ui.onTerminalInput((data) => undefined); + +// Terminal title +ctx.ui.setTitle("pi - my-project"); + +// Editor text +ctx.ui.setEditorText("Prefill text"); +const current = ctx.ui.getEditorText(); + +// Paste into editor (triggers paste handling, including collapse for large content) +ctx.ui.pasteToEditor("pasted content"); + +// Stack custom autocomplete behavior on top of the built-in provider +ctx.ui.addAutocompleteProvider((current) => ({ + triggerCharacters: ["#"], + async getSuggestions(lines, line, col, options) { + const beforeCursor = (lines[line] ?? "").slice(0, col); + const match = beforeCursor.match(/(?:^|[ \t])#([^\s#]*)$/); + if (!match) { + return current.getSuggestions(lines, line, col, options); + } + + return { + prefix: `#${match[1] ?? ""}`, + items: [{ value: "#2983", label: "#2983", description: "Extension API for autocomplete" }], + }; + }, + applyCompletion(lines, line, col, item, prefix) { + return current.applyCompletion(lines, line, col, item, prefix); + }, + shouldTriggerFileCompletion(lines, line, col) { + return current.shouldTriggerFileCompletion?.(lines, line, col) ?? true; + }, +})); + +// Tool output expansion +const wasExpanded = ctx.ui.getToolsExpanded(); +ctx.ui.setToolsExpanded(true); +ctx.ui.setToolsExpanded(wasExpanded); + +// Custom editor (vim mode, emacs mode, etc.) +ctx.ui.setEditorComponent((tui, theme, keybindings) => new VimEditor(tui, theme, keybindings)); +const currentEditor = ctx.ui.getEditorComponent(); +ctx.ui.setEditorComponent((tui, theme, keybindings) => + new WrappedEditor(tui, theme, keybindings, currentEditor?.(tui, theme, keybindings)) +); +ctx.ui.setEditorComponent(undefined); // Restore default editor + +// Theme management (see themes.md for creating themes) +const themes = ctx.ui.getAllThemes(); // [{ name: "dark", path: "/..." | undefined }, ...] +const lightTheme = ctx.ui.getTheme("light"); // Load without switching +const result = ctx.ui.setTheme("light"); // Switch by name +if (!result.success) { + ctx.ui.notify(`Failed: ${result.error}`, "error"); +} +ctx.ui.setTheme(lightTheme!); // Or switch by Theme object +ctx.ui.theme.fg("accent", "styled text"); // Access current theme +``` + +Custom working-indicator frames are rendered verbatim. If you want colors, add them to the frame strings yourself, for example with `ctx.ui.theme.fg(...)`. + +### Autocomplete Providers + +Use `ctx.ui.addAutocompleteProvider()` to stack custom autocomplete logic on top of the built-in slash-command and path provider. Set `triggerCharacters` for custom natural triggers such as `$`. + +Typical pattern: + +- inspect the text before the cursor +- return your own suggestions when your extension-specific syntax matches +- otherwise delegate to `current.getSuggestions(...)` +- delegate `applyCompletion(...)` unless you need custom insertion behavior + +```typescript +pi.on("session_start", (_event, ctx) => { + ctx.ui.addAutocompleteProvider((current) => ({ + triggerCharacters: ["#"], + async getSuggestions(lines, cursorLine, cursorCol, options) { + const line = lines[cursorLine] ?? ""; + const beforeCursor = line.slice(0, cursorCol); + const match = beforeCursor.match(/(?:^|[ \t])#([^\s#]*)$/); + if (!match) { + return current.getSuggestions(lines, cursorLine, cursorCol, options); + } + + return { + prefix: `#${match[1] ?? ""}`, + items: [ + { value: "#2983", label: "#2983", description: "Extension API for registering custom @ autocomplete providers" }, + { value: "#2753", label: "#2753", description: "Reload stale resource settings" }, + ], + }; + }, + + applyCompletion(lines, cursorLine, cursorCol, item, prefix) { + return current.applyCompletion(lines, cursorLine, cursorCol, item, prefix); + }, + + shouldTriggerFileCompletion(lines, cursorLine, cursorCol) { + return current.shouldTriggerFileCompletion?.(lines, cursorLine, cursorCol) ?? true; + }, + })); +}); +``` + +See [github-issue-autocomplete.ts](../examples/extensions/github-issue-autocomplete.ts) for a complete example that preloads the latest open GitHub issues with `gh issue list` and filters them locally for fast `#...` completion. It requires GitHub CLI (`gh`) and a GitHub repository checkout. + +### Custom Components + +For complex UI, use `ctx.ui.custom()`. This temporarily replaces the editor with your component until `done()` is called: + +```typescript +import { truncateToWidth } from "@earendil-works/pi-tui"; + +const result = await ctx.ui.custom((tui, theme, keybindings, done) => ({ + render(width: number): string[] { + return [truncateToWidth(theme.fg("accent", "Press Enter to confirm, Escape to cancel"), width)]; + }, + handleInput(data: string): void { + if (keybindings.matches(data, "tui.select.confirm")) done(true); + else if (keybindings.matches(data, "tui.select.cancel")) done(false); + }, + invalidate() {}, +})); + +if (result) { + // User pressed Enter +} +``` + +The callback receives: +- `tui` - TUI instance (for screen dimensions, focus management) +- `theme` - Current theme for styling +- `keybindings` - App keybinding manager (for checking shortcuts) +- `done(value)` - Call to close component and return value + +The factory can also return a `Promise` of a component. See [Terminal UI](tui.md#build-a-minimal-custom-screen) for a complete custom screen with cached rendering. + +#### Overlay Mode (Experimental) + +Pass `{ overlay: true }` to render the component as a floating modal on top of existing content, without clearing the screen: + +```typescript +const result = await ctx.ui.custom( + (tui, theme, keybindings, done) => new MyOverlayComponent({ onClose: done }), + { overlay: true } +); +``` + +For advanced positioning (anchors, margins, percentages, responsive visibility), pass `overlayOptions`. Use `onHandle` to control focus or visibility programmatically: + +```typescript +const result = await ctx.ui.custom( + (tui, theme, keybindings, done) => new MyOverlayComponent({ onClose: done }), + { + overlay: true, + overlayOptions: { anchor: "top-right", width: "50%", margin: 2 }, + onHandle: (handle) => { + handle.focus(); // focus this overlay and bring it to the visual front + // handle.unfocus({ target: editorComponent }); // release input to a specific component + // handle.setHidden(true/false); // toggle visibility + // handle.isHidden(), handle.isFocused(), handle.getBounds() + } + } +); +``` + +`overlayOptions` can also be a function, re-evaluated for dynamic positioning. Finish the interaction with `done()`; do not call `handle.hide()` on an overlay created by `ctx.ui.custom()`. + +A focused visible overlay can reclaim input after temporary non-overlay custom UI closes. If you intentionally want another component to keep input while the overlay stays visible, call `handle.unfocus({ target })`. Passing `{ target: null }` releases the overlay without focusing another component. + +See [`OverlayOptions` and `OverlayHandle`](https://github.com/earendil-works/pi/blob/main/packages/tui/src/tui.ts) for the full API and [overlay-qa-tests.ts](../examples/extensions/overlay-qa-tests.ts) for examples. + +### Custom Editor + +Replace the main input editor with a custom implementation (vim mode, emacs mode, etc.): + +```typescript +import { CustomEditor, type ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { matchesKey } from "@earendil-works/pi-tui"; + +class VimEditor extends CustomEditor { + private mode: "normal" | "insert" = "insert"; + + handleInput(data: string): void { + if (matchesKey(data, "escape") && this.mode === "insert") { + this.mode = "normal"; + return; + } + if (this.mode === "normal" && data === "i") { + this.mode = "insert"; + return; + } + super.handleInput(data); // App keybindings + text editing + } +} + +export default function (pi: ExtensionAPI) { + pi.on("session_start", (_event, ctx) => { + ctx.ui.setEditorComponent((tui, theme, keybindings) => + new VimEditor(tui, theme, keybindings) + ); + }); +} +``` + +**Key points:** +- Extend `CustomEditor` (not base `Editor`) to get app keybindings (escape to abort, ctrl+d, model switching) +- Call `super.handleInput(data)` for keys you don't handle +- Custom editors keep the standalone working row by default. Pass `{ embedWorkingStatus: true }` as the fourth `CustomEditor` constructor argument to use the built-in editor-border spinner instead. +- Factory receives `tui`, `theme`, and `keybindings` from the app +- Use `ctx.ui.getEditorComponent()` before `setEditorComponent()` to wrap the previously configured custom editor +- Pass `undefined` to restore default: `ctx.ui.setEditorComponent(undefined)` + +To compose with another extension that already replaced the editor, capture the previous factory before setting yours: + +```typescript +const previous = ctx.ui.getEditorComponent(); +ctx.ui.setEditorComponent((tui, theme, keybindings) => + new MyEditor(tui, theme, keybindings, { base: previous?.(tui, theme, keybindings) }) +); +``` + +See [modal-editor.ts](../examples/extensions/modal-editor.ts) for a complete example with a mode indicator. + +### Message and Entry Rendering + +Register a custom renderer for messages with your `customType`. Use message renderers for content that should participate in LLM context: + +```typescript +import { Text } from "@earendil-works/pi-tui"; + +pi.registerMessageRenderer("my-extension", (message, options, theme) => { + const { expanded, outputPad } = options; + let text = theme.fg("accent", `[${message.customType}] `); + text += message.content; + + if (expanded && message.details) { + text += "\n" + theme.fg("dim", JSON.stringify(message.details, null, 2)); + } + + return new Text(text, outputPad, 0); +}); +``` + +Messages are sent via `pi.sendMessage()`: + +```typescript +pi.sendMessage({ + customType: "my-extension", // Matches registerMessageRenderer + content: "Status update", + display: true, // Show in TUI + details: { ... }, // Available in renderer +}); +``` + +For TUI-only content that should not be sent to the LLM, render custom entries instead: + +```typescript +pi.registerEntryRenderer("my-card", (entry, options, theme) => { + return new Text(theme.fg("accent", JSON.stringify(entry.data))); +}); + +pi.appendEntry("my-card", { status: "done" }); +``` + +### Theme Colors + +All render functions receive a `theme` object. See [themes.md](themes.md) for creating custom themes and the full color palette. + +```typescript +// Foreground colors +theme.fg("toolTitle", text) // Tool names +theme.fg("accent", text) // Highlights +theme.fg("success", text) // Success (green) +theme.fg("error", text) // Errors (red) +theme.fg("warning", text) // Warnings (yellow) +theme.fg("muted", text) // Secondary text +theme.fg("dim", text) // Tertiary text + +// Text styles +theme.bold(text) +theme.italic(text) +theme.underline(text) +theme.strikethrough(text) + +// Combined foreground, background, and attributes +theme.style(text, { fg: "success", bg: "toolSuccessBg", bold: true }) +``` + +`theme.colors` exposes concrete colors and `theme.appearance` is `"dark"` or `"light"`. See [Apply themes correctly](tui.md#apply-themes-correctly). + +For syntax highlighting in custom tool renderers: + +```typescript +import { highlightCode, getLanguageFromPath } from "@earendil-works/pi-coding-agent"; + +// Highlight code with an explicit language; returns one string per line +const lines = highlightCode("const x = 1;", "typescript"); + +// Auto-detect language from file path +const lang = getLanguageFromPath("/path/to/file.rs"); // "rust", or undefined +const highlighted = highlightCode(code, lang); +``` - -### Errors and cleanup +## Error Handling -Pi reports handler errors and continues where possible. A `tool_call` handler failure blocks the tool as a fail-safe; a tool execution failure becomes an error result for the model. +- Extension errors are logged, agent continues +- `tool_call` errors block the tool (fail-safe) +- `user_bash` errors block the command instead of falling through to local execution +- Tool `execute` errors are signaled by throwing or by returning `isError: true`; a thrown error is caught, reported to the LLM as an error result, and execution continues -Release resources in `session_shutdown` even when normal operation attempted cleanup. -Keep cleanup idempotent because cancellation, reload, session replacement, and process exit can converge on the same path. -Use `ctx.shutdown()` to request an orderly process shutdown. + + + +## Mode Behavior + +| Mode | `ctx.mode` | `ctx.hasUI` | Notes | +|------|------------|-------------|-------| +| Interactive | `"tui"` | `true` | Full TUI with terminal rendering | +| RPC (`--mode rpc`) | `"rpc"` | `true` | Dialogs and notifications via JSON protocol; `custom()` returns `undefined`, component factories are ignored. See [RPC Extension UI](rpc-extension-ui.md) | +| JSON (`--mode json`) | `"json"` | `false` | Event stream to stdout; UI methods are no-ops | +| Print (`-p`) | `"print"` | `false` | Extensions run but can't prompt | + +Use `ctx.mode === "tui"` before TUI-specific features (`custom()`, component factories, terminal input). Use `ctx.hasUI` before dialog and notification methods that work in both TUI and RPC modes. - -## Examples and reference +## Examples Reference -The checked [extension examples](../examples/extensions/) are the implementation reference. Start with the smallest example matching the behavior you need: +All examples in [examples/extensions/](../examples/extensions/). -| Goal | Start with | Why | -|---|---|---| -| Confirm or block tool calls | [`permission-gate.ts`](../examples/extensions/permission-gate.ts) | Combines `tool_call`, mode-aware UI, and fail-closed behavior. | -| Modify prompt sections for a run | [`prompt-customizer.ts`](../examples/extensions/prompt-customizer.ts) | Uses structured prompt options instead of replacing the complete prompt. | -| Keep branch-aware tool state | [`todo.ts`](../examples/extensions/todo.ts) | Reconstructs state from persisted entries and renders it in the TUI. | -| Register tools after startup | [`dynamic-tools.ts`](../examples/extensions/dynamic-tools.ts) | Shares registration logic between lifecycle and command handlers. | -| Run cancellable model work from a command | [`qna.ts`](../examples/extensions/qna.ts) | Combines nested model calls, cancellation, and custom UI. | -| Replace the main editor | [`modal-editor.ts`](../examples/extensions/modal-editor.ts) | Extends `CustomEditor` while preserving application controls. | -| Run finalization at shutdown | [`auto-commit-on-exit.ts`](../examples/extensions/auto-commit-on-exit.ts) | Uses the session shutdown hook for finalization work. | -| Route built-in tools to another machine | [`ssh.ts`](../examples/extensions/ssh.ts) | Reuses tool operation interfaces instead of recreating tool contracts. | -| Add OAuth and provider-specific streaming | [`custom-provider-gitlab-duo/`](../examples/extensions/custom-provider-gitlab-duo/) | Shows a complete provider integration while delegating supported protocols. | - -Use [Custom Providers](custom-provider.md) for model-service integrations, [Terminal UI](tui.md) for custom components, and [Pi Packages](packages.md) to install or distribute extensions with other resources. Use the exported declarations for exact signatures, payloads, and result schemas; this guide focuses on lifecycle choices, composition, and durable implementation patterns. +| Example | Description | Key APIs | +|---------|-------------|----------| +| **Tools** ||| +| `hello.ts` | Minimal tool registration | `registerTool` | +| `question.ts` | Tool with user interaction | `registerTool`, `ui.custom`, `renderResult` | +| `questionnaire.ts` | Multi-step wizard tool | `registerTool`, `ui.custom` | +| `todo.ts` | Stateful tool with branch-aware state in `details` | `registerTool`, `renderResult`, `registerCommand`, `on("session_start")`, `on("session_tree")` | +| `dynamic-tools.ts` | Register tools after startup and during commands | `registerTool`, `session_start`, `registerCommand` | +| `structured-output.ts` | Final structured-output tool with `terminate: true` | `registerTool`, terminating tool results | +| `truncated-tool.ts` | Output truncation example | `registerTool`, `truncateHead` | +| `tic-tac-toe.ts` | Tool that must run sequentially | `registerTool`, `executionMode: "sequential"`, `ui.custom` | +| `tool-override.ts` | Override built-in read tool | `registerTool` (same name as built-in) | +| `built-in-tool-renderer.ts` | Custom rendering for built-in tools | `create*ToolDefinition`, `renderCall`, `renderResult`, `renderShell` | +| `minimal-mode.ts` | Collapsed display for built-in tools | Built-in tool overrides, `renderResult` | +| **Commands** ||| +| `pirate.ts` | Modify system prompt per-turn | `registerCommand`, `before_agent_start` | +| `summarize.ts` | Conversation summary command | `registerCommand`, `ui.custom` | +| `handoff.ts` | Cross-provider model handoff | `registerCommand`, `ui.editor`, `ui.custom` | +| `qna.ts` | Q&A with custom UI | `registerCommand`, `ui.custom`, `setEditorText` | +| `send-user-message.ts` | Inject user messages | `registerCommand`, `sendUserMessage` | +| `reload-runtime.ts` | Reload command and LLM tool handoff | `registerCommand`, `ctx.reload()`, `on("agent_settled")`, `sendUserMessage` | +| `shutdown-command.ts` | Graceful shutdown command | `registerCommand`, `shutdown()` | +| **Events & Gates** ||| +| `permission-gate.ts` | Block dangerous commands | `on("tool_call")`, `ui.select` | +| `project-trust.ts` | Decide or defer project trust from a user/global or CLI extension | `on("project_trust")`, trust UI, required trust result | +| `protected-paths.ts` | Block writes to specific paths | `on("tool_call")` | +| `confirm-destructive.ts` | Confirm session changes | `on("session_before_switch")`, `on("session_before_fork")` | +| `dirty-repo-guard.ts` | Warn on dirty git repo | `on("session_before_*")`, `exec` | +| `input-transform.ts` | Transform user input | `on("input")` | +| `input-transform-streaming.ts` | Streaming-aware input transform | `on("input")`, `streamingBehavior` | +| `model-status.ts` | React to model changes | `on("model_select")`, `setStatus` | +| `provider-payload.ts` | Inspect payloads and provider response headers | `on("before_provider_request")`, `on("after_provider_response")` | +| `debug-provider.ts` | View raw provider stream events | `on("provider_stream_event")`, `appendEntry`, `registerEntryRenderer` | +| `system-prompt-header.ts` | Display system prompt info | `on("agent_start")`, `getSystemPrompt` | +| `claude-rules.ts` | Load rules from files | `on("session_start")`, `on("before_agent_start")` | +| `prompt-customizer.ts` | Add context-aware tool guidance using `systemPromptOptions` | `on("before_agent_start")`, `BuildSystemPromptOptions` | +| `file-trigger.ts` | File watcher triggers messages | `sendMessage` | +| **Compaction & Sessions** ||| +| `custom-compaction.ts` | Custom compaction summary | `on("session_before_compact")` | +| `trigger-compact.ts` | Trigger compaction manually | `compact()` | +| `git-checkpoint.ts` | Git stash on turns | `on("turn_start")`, `on("tool_result")`, `on("agent_settled")`, `on("session_before_fork")`, `exec` | +| `git-merge-and-resolve.ts` | Fetch, merge, and resolve conflicts | `on("agent_end")`, `exec`, `sendUserMessage` | +| `auto-commit-on-exit.ts` | Commit on shutdown | `on("session_shutdown")`, `exec` | +| **UI Components** ||| +| `status-line.ts` | Footer status indicator | `setStatus`, session events | +| `working-indicator.ts` | Customize the streaming working indicator | `setWorkingIndicator`, `registerCommand` | +| `github-issue-autocomplete.ts` | Add `#1234` issue completions on top of built-in autocomplete by preloading recent open issues from `gh issue list` | `addAutocompleteProvider`, `on("session_start")`, `exec` | +| `custom-footer.ts` | Replace footer entirely | `registerCommand`, `setFooter` | +| `custom-header.ts` | Replace startup header | `on("session_start")`, `setHeader` | +| `modal-editor.ts` | Vim-style modal editor | `setEditorComponent`, `CustomEditor` | +| `rainbow-editor.ts` | Custom editor styling | `setEditorComponent` | +| `widget-placement.ts` | Widget above/below editor | `setWidget` | +| `overlay-test.ts` | Overlay components | `ui.custom` with overlay options | +| `overlay-qa-tests.ts` | Comprehensive overlay tests | `ui.custom`, all overlay options | +| `notify.ts` | Native terminal notification when the agent is done | `on("agent_settled")` | +| `titlebar-spinner.ts` | Spinner in the terminal title while working | `setTitle` | +| `hidden-thinking-label.ts` | Label for hidden thinking blocks | `setHiddenThinkingLabel` | +| `border-status-editor.ts` | Status in the editor border | `setEditorComponent`, `CustomEditor` | +| `rpc-demo.ts` | Every RPC-supported UI method | `ui.*` in RPC mode | +| `timed-confirm.ts` | Dialogs with timeout | `ui.confirm` with timeout/signal | +| `mac-system-theme.ts` | Auto-switch theme | `setTheme`, `exec` | +| **Complex Extensions** ||| +| `plan-mode/` | Full plan mode implementation | All event types, `registerCommand`, `registerShortcut`, `registerFlag`, `setStatus`, `setWidget`, `sendMessage`, `setActiveTools` | +| `preset.ts` | Saveable presets (model, tools, thinking) | `registerCommand`, `registerShortcut`, `registerFlag`, `setModel`, `setActiveTools`, `setThinkingLevel`, `appendEntry` | +| `tools.ts` | Toggle tools on/off UI | `registerCommand`, `setActiveTools`, `SettingsList`, session events | +| **Remote & Sandbox** ||| +| `ssh.ts` | SSH remote execution | `registerFlag`, `on("user_bash")`, `on("before_agent_start")`, tool operations | +| `interactive-shell.ts` | Persistent shell session | `on("user_bash")` | +| `sandbox/` | Sandboxed tool execution | Tool operations | +| `gondolin/` | Route built-in tools and `!` commands into a Gondolin micro-VM | Tool operations, built-in tool overrides, `on("user_bash")` | +| `subagent/` | Spawn sub-agents | `registerTool`, `exec` | +| **Games** ||| +| `snake.ts` | Snake game | `registerCommand`, `ui.custom`, keyboard handling | +| `space-invaders.ts` | Space Invaders game | `registerCommand`, `ui.custom` | +| `doom-overlay/` | Doom in overlay | `ui.custom` with overlay | +| **Providers** ||| +| `custom-provider-anthropic/` | Custom Anthropic proxy | `registerProvider` | +| `custom-provider-gitlab-duo/` | GitLab Duo integration | `registerProvider` with OAuth | +| `jev-router.ts` | Virtual model that routes planning and implementation to different models | `registerVirtualModel` | +| **Messages & Communication** ||| +| `message-renderer.ts` | Custom message rendering | `registerMessageRenderer`, `sendMessage` | +| `entry-renderer.ts` | TUI-only custom entry rendering | `registerEntryRenderer`, `appendEntry` | +| `event-bus.ts` | Inter-extension events | `pi.events` | +| **Session Metadata** ||| +| `session-name.ts` | Name sessions for selector | `setSessionName`, `getSessionName` | +| `bookmark.ts` | Bookmark entries for /tree | `setLabel` | +| **Misc** ||| +| `commands.ts` | List available slash commands | `getCommands` | +| `dynamic-resources/` | Add skills, prompts, and themes at runtime | `on("resources_discover")` | +| `inline-bash.ts` | Expand inline `!{command}` in user prompts | `on("input")`, `exec` | +| `bash-spawn-hook.ts` | Adjust bash command, cwd, and env before execution | `createBashTool`, `spawnHook` | +| `with-deps/` | Extension with npm dependencies | Package structure with `package.json` | diff --git a/packages/coding-agent/examples/extensions/reload-runtime.ts b/packages/coding-agent/examples/extensions/reload-runtime.ts index d0e9abd47..938e31dfe 100644 --- a/packages/coding-agent/examples/extensions/reload-runtime.ts +++ b/packages/coding-agent/examples/extensions/reload-runtime.ts @@ -2,7 +2,7 @@ * Reload Runtime Extension * * Demonstrates ctx.reload() from ExtensionCommandContext and an LLM-callable - * tool that queues a follow-up command to trigger reload. + * tool that requests a reload once the agent has settled. */ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; @@ -19,19 +19,31 @@ export default function (pi: ExtensionAPI) { }, }); - // LLM-callable tool. Tools get ExtensionContext, so they cannot call ctx.reload() directly. - // Instead, queue a follow-up user command that executes the command above. + // LLM-callable tool. Tools do not receive ExtensionCommandContext, so they cannot call + // ctx.reload(), and Pi cannot reload while the agent is running. The tool records the + // request and ends the run; the agent_settled handler then runs the command above. + let reloadRequested = false; + pi.registerTool({ name: "reload_runtime", label: "Reload Runtime", description: "Reload extensions, skills, prompts, themes, and context files", parameters: Type.Object({}), async execute() { - pi.sendUserMessage("/reload-runtime", { deliverAs: "followUp" }); + reloadRequested = true; return { - content: [{ type: "text", text: "Queued /reload-runtime as a follow-up command." }], + content: [{ type: "text", text: "Reload requested. Pi reloads after this response." }], details: {}, + terminate: true, }; }, }); + + pi.on("agent_settled", () => { + if (!reloadRequested) return; + reloadRequested = false; + // Sent from agent_settled, the command runs after every settled handler finishes, while + // the agent is idle. expandPromptTemplates dispatches it as a command, not as model input. + pi.sendUserMessage("/reload-runtime", { expandPromptTemplates: true }); + }); } From a92283056d915c758799fcd904948ff0cb42628c Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Tue, 27 Oct 2026 23:46:11 +0000 Subject: [PATCH 046/271] fix(chord): Delta's tracker holds its one open change without a WeakRef Chord and Durable change a document one transaction at a time, so a tracker registered a WeakRef per change for competitors that never existed on their hot path. The WeakRef targets survived scavenges and V8 could pretenure every overlay context into old space (47.5 MB promoted against 2 MB in a tracker loop). The tracker now keeps its open change in a plain field and moves it into the weak registry only when a second change opens; stale semantics are unchanged. A draft dropped without prepare or abort stays referenced until the tracker's next change or adopt (at most one per tracker). `stored` and `rawNodes` are created on first use. --- packages/chord/CHANGELOG.md | 1 + packages/chord/src/delta/README.md | 2 + packages/chord/src/delta/tracker.ts | 95 +++++++++------ .../test/delta-tracker/retention.test.ts | 2 + .../test/delta-tracker/retention.worker.ts | 51 +++++++- .../chord/test/delta-tracker/tracker.test.ts | 114 ++++++++++++++++++ 6 files changed, 227 insertions(+), 38 deletions(-) diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index 145418f1b..cf4082fa9 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -24,6 +24,7 @@ ### Changed +- The delta tracker keeps its one open change in a plain field and needs a `WeakRef` only for a second change open at the same time (one per change before). A `WeakRef` target survives the scavenges of its job, so a run of changes in one job promoted every change's overlay to old space: 200,000 small changes in one job now promote 2 MB instead of 47.5 MB and take 3.1 µs each instead of 3.8 µs (3.0 instead of 3.5 µs when they span jobs). A draft dropped without `prepare()` or `abort()` now stays referenced by its tracker until the tracker's next `beginChange()` or `adopt()`. - `applyImmutable()`, `applyImmutableBatches()` and tracker revisions copy each container on a changed path with spread (or `Object.assign()` onto a null-prototype object) instead of one `Object.defineProperty()` per key, about 25 times faster per container ([#9265](https://github.com/earendil-works/pi/issues/9265)) ### Fixed diff --git a/packages/chord/src/delta/README.md b/packages/chord/src/delta/README.md index 0db4a8103..8dbf06f75 100644 --- a/packages/chord/src/delta/README.md +++ b/packages/chord/src/delta/README.md @@ -154,6 +154,8 @@ Growing `length` inserts `null`; shrinking it removes elements. `tracker.value` is the latest adopted revision. `beginChange()` opens an overlay draft over it and never modifies it. A draft may stay open across `await`. +A draft dropped without `prepare()` or `abort()` stays referenced by its +tracker until the tracker's next `beginChange()` or `adopt()`. `prepare()` materializes the candidate and ops and does not change authority. `adopt()` checks the preparation and swaps the root pointer. diff --git a/packages/chord/src/delta/tracker.ts b/packages/chord/src/delta/tracker.ts index db5eb6b66..731cc49a8 100644 --- a/packages/chord/src/delta/tracker.ts +++ b/packages/chord/src/delta/tracker.ts @@ -229,9 +229,20 @@ class ChangeImpl implements Change { } } +/** + * Open drafts are tracked so that adopting one change makes its open competitors stale at once. Prepared contexts need + * no tracking: `prepare()` releases their overlay, and `adopt()` rejects them by revision. + * + * The most recent open context is held strongly in `#open`: with one transaction at a time per tracker, the common case, + * that is the only one, and it costs no `WeakRef` (each `WeakRef` target survives the scavenges of the current job, and + * V8 then pretenures the context allocation site). A second `beginChange()` while one is open moves the older context + * into `#competitors` through a `WeakRef`, so a draft the caller drops can be collected. A dropped draft that was never + * settled stays held by `#open` until the tracker's next `beginChange()` or `adopt()`: at most one per tracker. + */ class TrackerImpl implements Tracker { readonly #owner = {}; - readonly #contexts = new Set>(); + #open: OverlayContext | undefined; + #competitors: Set> | undefined; readonly #selfRef: WeakRef>; #value: T; #revision = 0; @@ -252,14 +263,15 @@ class TrackerImpl implements Tracker { beginChange(): Change { const context = createContext(this.#selfRef, this.#owner, this.#revision, this.#value, false, this.#value); - this.#register(context); + if (this.#open !== undefined) this.#demote(this.#open); + this.#open = context; return new ChangeImpl(context); } prepareReplace(value: T): Prepared { + // Not tracked: materialization below releases the context before this returns, as `prepare()` does. const context = createContext(this.#selfRef, this.#owner, this.#revision, value, true, this.#value); context.status.value = "prepared"; - this.#register(context); try { return materializePrepared(context); } catch (error) { @@ -295,39 +307,37 @@ class TrackerImpl implements Tracker { } releaseContext(context: OverlayContext): void { - if (context.registryRef !== undefined) this.#contexts.delete(context.registryRef); + if (this.#open === context) this.#open = undefined; + else if (context.registryRef !== undefined) this.#competitors?.delete(context.registryRef); context.registryRef = undefined; } - #register(context: OverlayContext): void { + #demote(context: OverlayContext): void { const reference = new WeakRef(context); context.registryRef = reference; - this.#contexts.add(reference); + this.#competitors ??= new Set(); + this.#competitors.add(reference); this.#pruneBudget -= 1; if (this.#pruneBudget === 0) { - this.#prune(); - this.#pruneBudget = Math.max(256, this.#contexts.size); - } - } - - #prune(): void { - for (const reference of this.#contexts) { - if (reference.deref() === undefined) this.#contexts.delete(reference); + for (const entry of this.#competitors) { + if (entry.deref() === undefined) this.#competitors.delete(entry); + } + this.#pruneBudget = Math.max(256, this.#competitors.size); } } #invalidate(winner: OverlayContext): void { - for (const reference of this.#contexts) { - const context = reference.deref(); - if (context === undefined || context === winner) continue; - if (context.status.value === "open" || context.status.value === "prepared") context.status.value = "stale"; - // Adoption must not walk every node touched by a competing draft. Drop the - // context's strong overlay references in O(1); externally retained proxies - // still see the stale status and are reclaimed with their holders. - releaseOverlayReferences(context); - } - this.#contexts.clear(); + const open = this.#open; + const competitors = this.#competitors; + this.#open = undefined; + this.#competitors = undefined; this.#pruneBudget = 256; + if (open !== undefined) staleCompetitor(open, winner); + if (competitors === undefined) return; + for (const reference of competitors) { + const context = reference.deref(); + if (context !== undefined) staleCompetitor(context, winner); + } } replacementBase(context: OverlayContext): T | undefined { @@ -335,6 +345,16 @@ class TrackerImpl implements Tracker { } } +function staleCompetitor(context: OverlayContext, winner: OverlayContext): void { + if (context === winner) return; + if (context.status.value === "open" || context.status.value === "prepared") context.status.value = "stale"; + // Adoption must not walk every node touched by a competing draft. Drop the + // context's strong overlay references in O(1); externally retained proxies + // still see the stale status and are reclaimed with their holders. + context.registryRef = undefined; + releaseOverlayReferences(context); +} + /** Take immutable ownership of an alias-free strict-JSON root in O(1). */ export function track(initial: T): Tracker { return new TrackerImpl(initial); @@ -408,10 +428,10 @@ function createContext( status: { value: "open" }, root: undefined, bases: [], - stored: [], + stored: undefined, dirty: [], nodes: [], - rawNodes: new WeakMap(), + rawNodes: undefined, ops: undefined, replacement, replacementNoop: false, @@ -483,7 +503,7 @@ function createNode( parentSource: InsertSource | undefined = undefined, parentPlacement = false, ): OverlayNode { - const existing = context.rawNodes!.get(base); + const existing = context.rawNodes?.get(base); if (existing !== undefined) return existing; const target: object = Array.isArray(base) ? [] : {}; const baseIndex = context.bases!.length; @@ -502,7 +522,12 @@ function createNode( if (parentSource !== undefined) node.parentSource = parentSource; (target as Record)[NODE] = node; node.proxy = new Proxy(target, Array.isArray(base) ? sharedArrayHandler : sharedObjectHandler); - context.rawNodes!.set(base, node); + // The root is never reached again through a child value (tracked values are alias-free), so a change that touches + // only the root needs no map. + if (parent !== undefined) { + context.rawNodes ??= new WeakMap(); + context.rawNodes.set(base, node); + } context.nodes.push(node); return node; } @@ -517,8 +542,10 @@ function nodeParent(node: OverlayNode): OverlayNode | undefined { function storeValue(context: OverlayContext, value: Stored): StoredRef { if (!isContainer(value)) return value; - const index = context.stored!.length; - context.stored!.push(value); + // Created on the first container write: primitive writes never store. + context.stored ??= []; + const index = context.stored.length; + context.stored.push(value); return { index }; } @@ -1121,12 +1148,12 @@ function insertPiece(node: OverlayNode, items: Stored[], start = 0): InsertPiece function insertPlacementPiece(node: OverlayNode, values: unknown[], start = 0): InsertPiece[] { if (values.length === start) return []; - const storedLength = node.context.stored!.length; + const storedLength = node.context.stored?.length ?? 0; try { for (let index = start; index < values.length; index++) values[index] = storeValue(node.context, clonePlacement(values[index])); } catch (error) { - node.context.stored!.length = storedLength; + if (node.context.stored !== undefined) node.context.stored.length = storedLength; throw error; } return [ @@ -1611,7 +1638,7 @@ function cloneNode(node: OverlayNode): Container { function cloneStored(value: Stored, context: OverlayContext): Stored { if (!isContainer(value)) return value; - const node = context.rawNodes!.get(value); + const node = context.rawNodes?.get(value); return node === undefined || (!node.dirty && !node.subtreeDirty) ? value : cloneNode(node); } @@ -1636,7 +1663,7 @@ function clonePlacementNode(node: OverlayNode): Container { function clonePlacementStored(value: Stored, context: OverlayContext): Stored { if (!isContainer(value)) return copyJson(value); - const node = context.rawNodes!.get(value); + const node = context.rawNodes?.get(value); return node === undefined ? copyJson(value) : clonePlacementNode(node); } diff --git a/packages/chord/test/delta-tracker/retention.test.ts b/packages/chord/test/delta-tracker/retention.test.ts index fe39b0dd0..443db1126 100644 --- a/packages/chord/test/delta-tracker/retention.test.ts +++ b/packages/chord/test/delta-tracker/retention.test.ts @@ -19,6 +19,8 @@ describe("delta tracker retention across GC jobs", () => { "same-job-folded-ops-cleanup", "lifecycle-churn", "adopted-changes-flat", + "abandoned-draft-released", + "tracker-with-open-draft", ])( "validates %s retention semantics", (scenario) => { diff --git a/packages/chord/test/delta-tracker/retention.worker.ts b/packages/chord/test/delta-tracker/retention.worker.ts index 23e9efa85..57d111d55 100644 --- a/packages/chord/test/delta-tracker/retention.worker.ts +++ b/packages/chord/test/delta-tracker/retention.worker.ts @@ -39,13 +39,16 @@ function draftProxies(tracker: Tracker<{ payload: { rows: { value: number }[] } return reference; } -async function assertRetained(reference: WeakRef): Promise { +async function assertRetained( + reference: WeakRef, + message = "retained Prepared must retain its immutable revisions", +): Promise { assert.ok(global.gc, "worker requires --expose-gc"); for (let attempt = 0; attempt < 5; attempt++) { await setImmediate(); global.gc(); } - assert.notEqual(reference.deref(), undefined, "retained Prepared must retain its immutable revisions"); + assert.notEqual(reference.deref(), undefined, message); } function settledLifecycle(): { tracker: WeakRef; future: WeakRef } { @@ -261,8 +264,8 @@ if (scenario === "settled-lifecycle") { assert.ok(retainedMiB < 8, `same-job folded operations retained ${retainedMiB.toFixed(2)} MiB`); assert.equal(tracker.value.field0, 100); } else if (scenario === "adopted-changes-flat") { - // Each change registers its context with the tracker through a WeakRef; adoption must drop every registration, so - // a long run of small changes keeps only the document, not a record per change. + // Each change's context is tracked by the tracker until it is settled; adoption must drop every one, so a long run + // of small changes keeps only the document, not a record per change. const gc = global.gc; assert.ok(gc, "worker requires --expose-gc"); const tracker = track({ entries: [{ id: 0, text: "" }] }); @@ -289,6 +292,46 @@ if (scenario === "settled-lifecycle") { const perChange = ((await settledHeap()) - readyHeap) / 10_000 - 4; assert.ok(perChange < 48, `each adopted change retained ${perChange.toFixed(1)} bytes`); assert.equal(tracker.value.entries[0]!.text.length, 4 * 11_000); +} else if (scenario === "abandoned-draft-released") { + // The tracker holds its most recent open draft strongly: one dropped without settling stays until the tracker's next + // change or adoption, which releases it. + type Root = { value: number; payload: { rows: { value: number }[] } | null }; + const abandon = (tracker: Tracker): WeakRef => { + const change = tracker.beginChange(); + change.state.payload = { rows: Array.from({ length: 50_000 }, (_, value) => ({ value })) }; + return new WeakRef(change.state.payload.rows[0]!); + }; + const byChange = track({ value: 0, payload: null }); + const first = abandon(byChange); + await assertRetained(first, "the tracker holds its latest open draft"); + const next = byChange.beginChange(); + await collect(first); + // A chain of abandoned drafts keeps only the latest. + next.abort(); + const chain = Array.from({ length: 20 }, () => abandon(byChange)); + for (const reference of chain.slice(0, -1)) await collect(reference); + await assertRetained(chain.at(-1)!, "the tracker holds its latest open draft"); + byChange.adopt(byChange.prepareReplace({ value: 1, payload: null })); + await collect(chain.at(-1)!); + + const byAdopt = track({ value: 0, payload: null }); + const dropped = abandon(byAdopt); + byAdopt.adopt(byAdopt.prepareReplace({ value: 1, payload: null })); + await collect(dropped); + assert.deepEqual(byAdopt.value, { value: 1, payload: null }); +} else if (scenario === "tracker-with-open-draft") { + const open = (): WeakRef => { + const tracker = track({ value: 0, rows: Array.from({ length: 50_000 }, (_, value) => ({ value })) }); + const change = tracker.beginChange(); + change.state.rows[0]!.value = -1; + retainedSettledChanges.push(change); + const competitor = tracker.beginChange(); + competitor.state.value = 1; + retainedSettledChanges.push(competitor); + return new WeakRef(tracker); + }; + await collect(open()); + assert.equal(retainedSettledChanges.length, 2); } else if (scenario === "lifecycle-churn") { const tracker = track({ value: 0 }); for (let index = 0; index < 100_000; index++) { diff --git a/packages/chord/test/delta-tracker/tracker.test.ts b/packages/chord/test/delta-tracker/tracker.test.ts index fddbd8c61..e0e779b4d 100644 --- a/packages/chord/test/delta-tracker/tracker.test.ts +++ b/packages/chord/test/delta-tracker/tracker.test.ts @@ -347,6 +347,120 @@ describe("delta tracker transactional overlay lifecycle", () => { }); }); +/** Runs `body` with `WeakRef` construction counted; returns the count. */ +function countWeakRefs(body: () => void): number { + const original = globalThis.WeakRef; + let created = 0; + globalThis.WeakRef = class extends original { + constructor(target: T) { + super(target); + created += 1; + } + }; + try { + body(); + } finally { + globalThis.WeakRef = original; + } + return created; +} + +describe("delta tracker competing changes", () => { + it("tracks one change at a time without a WeakRef", () => { + const tracker = track({ value: 0, child: { rows: [0] } }); + const created = countWeakRefs(() => { + for (let index = 1; index <= 100; index++) { + const change = tracker.beginChange(); + change.state.value = index; + change.state.child.rows.push(index); + change.state.child = { rows: [index] }; + tracker.adopt(change.prepare()); + tracker.beginChange().abort(); + const aborted = tracker.beginChange(); + aborted.state.value = -1; + aborted.prepare().abort(); + tracker.adopt(tracker.prepareReplace({ value: index, child: { rows: [index] } })); + } + }); + expect(created).toBe(0); + expect(tracker.value).toEqual({ value: 100, child: { rows: [100] } }); + expect(tracker.revision).toBe(200); + }); + + it("makes the older of two open drafts stale when the newer is adopted", () => { + const tracker = track({ value: 0, child: { value: 0 } }); + const older = tracker.beginChange(); + const held = older.state.child; + held.value = 1; + const newer = tracker.beginChange(); + newer.state.value = 2; + tracker.adopt(newer.prepare()); + expect(tracker.value).toEqual({ value: 2, child: { value: 0 } }); + expect(() => held.value).toThrow(/settled/); + expect(() => older.state).toThrow(/settled/); + expect(() => older.prepare()).toThrow(/settled/); + older.abort(); + }); + + it("makes the newer of two open drafts stale when the older is adopted", () => { + const tracker = track({ value: 0, child: { value: 0 } }); + const older = tracker.beginChange(); + older.state.value = 1; + const newer = tracker.beginChange(); + const held = newer.state.child; + held.value = 2; + tracker.adopt(older.prepare()); + expect(tracker.value).toEqual({ value: 1, child: { value: 0 } }); + expect(() => held.value).toThrow(/settled/); + expect(() => newer.prepare()).toThrow(/settled/); + // The tracker moved on: the next change is tracked as the only one again. + const next = tracker.beginChange(); + next.state.value = 3; + tracker.adopt(next.prepare()); + expect(tracker.value).toEqual({ value: 3, child: { value: 0 } }); + }); + + it("makes every other of three drafts stale, open or prepared, whichever is adopted", () => { + for (const winner of [0, 1, 2]) { + const tracker = track({ value: 0, rows: [{ value: 0 }] }); + const changes = [tracker.beginChange(), tracker.beginChange(), tracker.beginChange()]; + const held = changes.map((change) => change.state.rows[0]!); + held.forEach((row, index) => { + row.value = index + 1; + }); + const prepared = changes[(winner + 1) % 3]!.prepare(); + tracker.adopt(changes[winner]!.prepare()); + expect(tracker.value).toEqual({ value: 0, rows: [{ value: winner + 1 }] }); + expect(() => tracker.adopt(prepared)).toThrow(/stale/); + const open = changes[(winner + 2) % 3]!; + expect(() => held[(winner + 2) % 3]!.value).toThrow(/settled/); + expect(() => open.prepare()).toThrow(/settled/); + expect(prepared.value).toEqual({ value: 0, rows: [{ value: ((winner + 1) % 3) + 1 }] }); + } + }); + + it("keeps an abandoned draft a competitor of the next change, through one WeakRef", () => { + const tracker = track({ value: 0, child: { value: 0 } }); + let abandonedChild: { value: number } | undefined; + let abandoned: ReturnType | undefined; + const created = countWeakRefs(() => { + abandoned = tracker.beginChange(); + abandonedChild = abandoned.state.child; + abandonedChild.value = 1; + const next = tracker.beginChange(); + next.state.value = 2; + tracker.adopt(next.prepare()); + const after = tracker.beginChange(); + after.state.value = 3; + tracker.adopt(after.prepare()); + }); + expect(created).toBe(1); + expect(tracker.value).toEqual({ value: 3, child: { value: 0 } }); + expect(() => abandonedChild!.value).toThrow(/settled/); + expect(() => abandoned!.prepare()).toThrow(/settled/); + }); +}); + describe("delta tracker policy view", () => { it("supports native reads, descriptors, keys, iteration, map, and JSON", () => { const tracker = track({ values: [1, 2, 3], object: { a: 1 } as { a: number; b?: number } }); From 44f6851b7f27ad5c18ccc94f6b4321650e757e3e Mon Sep 17 00:00:00 2001 From: David Brailovsky Date: Wed, 28 Oct 2026 09:45:08 +0100 Subject: [PATCH 047/271] fix(ai): use Anthropic Messages for OpenRouter ~anthropic latest aliases The ~anthropic/claude-*-latest aliases used Chat Completions, where pi dropped the thinking signature and OpenRouter ignored the unsigned thinking on replay. Route them through Anthropic Messages with adaptive thinking, like the versioned Claude models. refs #9534 --- packages/ai/CHANGELOG.md | 1 + packages/ai/scripts/generate-models.ts | 3 +++ packages/ai/scripts/openrouter-catalog.ts | 3 ++- .../openrouter-cache-control-models.test.ts | 18 ------------------ 4 files changed, 6 insertions(+), 19 deletions(-) delete mode 100644 packages/ai/test/openrouter-cache-control-models.test.ts diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 60baa1c16..444c3be23 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -30,6 +30,7 @@ - Fixed OpenCode DeepSeek V4.1 Flash sending `prompt_cache_retention`, which the OpenCode gateway rejects, with long cache retention - Fixed GPT-5.4 and GPT-5.5 failing with `Missing namespace for function_call` after switching from another model that called tools. On the OpenAI and OpenCode providers they now declare their tools in the top-level tool list instead of loading them through `additional_tools` or tool search ([#11415](https://github.com/earendil-works/pi/issues/11415)) - Fixed Bedrock requests failing when `strict: "prefer"` tools exceed Claude's strict tool limits, such as more than 16 optional parameters across all strict tools; requests now retry once with preferred tools non-strict while keeping `strict: "require"` tools strict, as the Anthropic provider already does ([#11416](https://github.com/earendil-works/pi/issues/11416)) +- Fixed OpenRouter's `~anthropic/claude-*-latest` aliases losing Claude's earlier thinking after each tool call. They used the Chat Completions API, where pi dropped the thinking signature, and OpenRouter then ignored the unsigned thinking. They now use the Anthropic Messages API with adaptive thinking, like the versioned Claude models ([#9534](https://github.com/earendil-works/pi/issues/9534)) ## [1.4.0] - 2026-10-21 diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 966604358..c63afb6e5 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -650,6 +650,9 @@ function supportsAnthropicMidConvoSystemMessages(modelId: string): boolean { function isAnthropicAdaptiveThinkingModel(modelId: string): boolean { return ( + // OpenRouter's `~anthropic/claude-*-latest` aliases follow the newest model of each family, + // all of which use adaptive thinking. + /^~anthropic\/claude-[a-z]+-latest$/.test(modelId) || modelId.includes("opus-4-6") || modelId.includes("opus-4.6") || modelId.includes("opus-4-7") || diff --git a/packages/ai/scripts/openrouter-catalog.ts b/packages/ai/scripts/openrouter-catalog.ts index 8f01a14e6..f0b7c0b03 100644 --- a/packages/ai/scripts/openrouter-catalog.ts +++ b/packages/ai/scripts/openrouter-catalog.ts @@ -45,7 +45,8 @@ export function buildOpenRouterCatalog( } const thinkingLevelMap = getOpenRouterThinkingLevelMap(model.reasoning); - const useAnthropicMessages = /^anthropic\//.test(model.id) && !model.id.endsWith(":batch"); + // Includes the `~anthropic/claude-*-latest` aliases, which OpenRouter's Messages endpoint accepts. + const useAnthropicMessages = /^~?anthropic\//.test(model.id) && !model.id.endsWith(":batch"); chat.push({ type: "chat", id: model.id, diff --git a/packages/ai/test/openrouter-cache-control-models.test.ts b/packages/ai/test/openrouter-cache-control-models.test.ts deleted file mode 100644 index f8db1794f..000000000 --- a/packages/ai/test/openrouter-cache-control-models.test.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { getModel } from "../src/compat.ts"; - -const OPENROUTER_ANTHROPIC_LATEST_MODEL_IDS = [ - "~anthropic/claude-fable-latest", - "~anthropic/claude-haiku-latest", - "~anthropic/claude-opus-latest", - "~anthropic/claude-sonnet-latest", -] as const; - -describe("OpenRouter Anthropic latest alias metadata", () => { - it.each(OPENROUTER_ANTHROPIC_LATEST_MODEL_IDS)("keeps completions cache control for %s", (modelId) => { - const model = getModel("openrouter", modelId); - expect(model.api).toBe("openai-completions"); - if (model.api !== "openai-completions") throw new Error(`Unexpected API for ${modelId}`); - expect(model.compat?.cacheControlFormat).toBe("anthropic"); - }); -}); From 88ffcdc734605d80bc383476950856b1242081fa Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:51:39 +0100 Subject: [PATCH 048/271] fix(coding-agent): report a deleted working directory instead of crashing on startup (#11511) Starting `pi` from a directory that has been deleted crashed with an uncaught Node `ENOENT ... uv_cwd` stack trace. This also happened for `--version` and `--help`. The cause is the unguarded `process.cwd()` call in `main()` (`packages/coding-agent/src/main.ts`). `main()` now catches `ENOENT` from that call, prints one line telling the user to change to an existing directory, and exits with code 1. Other errors are rethrown unchanged. Argument handling is otherwise unchanged: `--version` still needs a valid cwd, but it now gets the readable error instead of a crash. **Testing** - Added `test/deleted-cwd.test.ts`. It stubs `process.cwd()` to throw ENOENT and checks that `main()` prints the message and exits with code 1. It also checks that a non-ENOENT error is rethrown. The first test fails without the change and passes with it. - `npm run check` passes. - Manual check with the built CLI. Before, all three commands below printed the `uv_cwd` stack trace. After, each prints `Error: The current working directory no longer exists. Change to an existing directory and try again.` and exits 1: ```sh d=$(mktemp -d); cd "$d"; rmdir "$d" pi -ne --version; pi -ne --help; pi -ne ``` Fixes #11509 Requested by @badlogic Co-authored-by: triagent --- packages/coding-agent/src/main.ts | 13 ++++++- .../coding-agent/test/deleted-cwd.test.ts | 35 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 packages/coding-agent/test/deleted-cwd.test.ts diff --git a/packages/coding-agent/src/main.ts b/packages/coding-agent/src/main.ts index 723fc910b..faba51f48 100644 --- a/packages/coding-agent/src/main.ts +++ b/packages/coding-agent/src/main.ts @@ -591,7 +591,18 @@ export async function main(args: string[], options?: MainOptions) { } cleanupManagedInstall(); - const cwd = process.cwd(); + let cwd: string; + try { + cwd = process.cwd(); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + console.error( + chalk.red( + "Error: The current working directory no longer exists. Change to an existing directory and try again.", + ), + ); + process.exit(1); + } const agentDir = getAgentDir(); const bootstrapSettingsManager = SettingsManager.create(cwd, agentDir, { projectTrusted: false }); applyHttpProxySettings(bootstrapSettingsManager.getGlobalSettings().httpProxy); diff --git a/packages/coding-agent/test/deleted-cwd.test.ts b/packages/coding-agent/test/deleted-cwd.test.ts new file mode 100644 index 000000000..3bf60a8dd --- /dev/null +++ b/packages/coding-agent/test/deleted-cwd.test.ts @@ -0,0 +1,35 @@ +import { afterEach, describe, expect, test, vi } from "vitest"; +import { main } from "../src/main.ts"; + +// Regression test for https://github.com/earendil-works/pi/issues/11509 +describe("startup in a deleted working directory", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + test("prints an actionable error and exits 1 instead of throwing", async () => { + vi.spyOn(process, "cwd").mockImplementation(() => { + throw Object.assign(new Error("ENOENT: no such file or directory, uv_cwd"), { + code: "ENOENT", + syscall: "uv_cwd", + }); + }); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`process.exit(${code})`); + }) as typeof process.exit); + + await expect(main(["--no-extensions", "--version"])).rejects.toThrow("process.exit(1)"); + expect(exitSpy).toHaveBeenCalledWith(1); + const stderr = errorSpy.mock.calls.map(([message]) => String(message)).join("\n"); + expect(stderr).toContain("The current working directory no longer exists"); + }); + + test("rethrows unrelated cwd errors", async () => { + vi.spyOn(process, "cwd").mockImplementation(() => { + throw Object.assign(new Error("EACCES"), { code: "EACCES" }); + }); + + await expect(main(["--no-extensions", "--version"])).rejects.toThrow("EACCES"); + }); +}); From 7391dbb0f849bc8d08fc070a8cc5acba0866f7b3 Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:54:06 +0100 Subject: [PATCH 049/271] fix(coding-agent): show model scope in the fullscreen header instead of the shell (#11512) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In fullscreen mode, the startup `Model scope: …` notice was still in the shell after pi exited. With `fullscreenExitOutput: "resume-hint"`, the shell showed it above the resume command. Cause: `InteractiveMode.init()` printed the notice with `console.log` before the renderer started. In fullscreen mode that output goes to the terminal's normal screen. Pi then draws on the alternate screen, and when it exits the terminal switches back to the normal screen, where the notice is still visible. The change: - Building the notice text moves into `getModelScopeNotice()`. - Regular mode prints it with `console.log` before the TUI starts, as before. - Fullscreen mode adds it to the header container below the built-in header. It uses a `ThemedText`, so the text is built after the terminal reports its colors and follows theme changes. Since fullscreen is the default `tuiMode`, this applies to every user who has `enabledModels` set and quiet startup off. Testing: - Added `packages/coding-agent/test/interactive-mode-model-scope.test.ts`. It checks two things for fullscreen: the notice renders in the header and `console.log` is not called, and nothing is added when startup details are hidden. - `npm run check` passes. - Manual check in tmux with `enabledModels` set: - fullscreen + `resume-hint`: before the change, `Model scope: claude-sonnet-4-5, claude-opus-4-5 (Ctrl+P to cycle)` was left in the shell after `/quit`. Now the line shows inside pi's header and the shell is clean after exit. - `tuiMode: "regular"`: unchanged, the line is still printed above the header. Fixes #11508 Requested by @badlogic Co-authored-by: triagent --- .../src/modes/interactive/interactive-mode.ts | 41 ++++++++----- .../test/interactive-mode-model-scope.test.ts | 58 +++++++++++++++++++ 2 files changed, 86 insertions(+), 13 deletions(-) create mode 100644 packages/coding-agent/test/interactive-mode-model-scope.test.ts diff --git a/packages/coding-agent/src/modes/interactive/interactive-mode.ts b/packages/coding-agent/src/modes/interactive/interactive-mode.ts index f6b0f7a58..57bd91f25 100644 --- a/packages/coding-agent/src/modes/interactive/interactive-mode.ts +++ b/packages/coding-agent/src/modes/interactive/interactive-mode.ts @@ -950,19 +950,11 @@ export class InteractiveMode { // Load changelog (only show new entries, skip for resumed sessions) this.changelogMarkdown = this.getChangelogForDisplay(); - if (this.session.scopedModels.length > 0 && this.shouldShowStartupDetails()) { - const modelList = this.session.scopedModels - .map((sm) => { - const thinkingStr = sm.thinkingLevel ? `:${sm.thinkingLevel}` : ""; - return `${sm.model.id}${thinkingStr}`; - }) - .join(", "); - const cycleKeys = this.keybindings.getKeys("app.model.cycleForward"); - const cycleHint = - cycleKeys.length > 0 - ? theme.fg("muted", ` (${formatKeyText(cycleKeys.join("/"), { capitalize: true })} to cycle)`) - : ""; - console.log(theme.fg("dim", `Model scope: ${modelList}${cycleHint}`)); + // Fullscreen renders on the alternate screen, so text printed to stdout here would stay in the shell + // after exit. Fullscreen shows the notice in the header instead (#11508). + if (this.renderer.mode !== "fullscreen") { + const modelScopeNotice = this.getModelScopeNotice(); + if (modelScopeNotice) console.log(modelScopeNotice); } // Keep one component tree and remount it when changing renderers. @@ -1082,6 +1074,7 @@ export class InteractiveMode { this.builtInHeader = new Text("", 0, 0); this.headerContainer.addChild(this.builtInHeader); } + if (this.renderer.mode === "fullscreen") this.showFullscreenModelScopeNotice(); this.ui.requestRender(); // Ensure fd and rg are available after mounting the TUI (downloads if missing, adds to PATH via getBinDir) @@ -1427,6 +1420,28 @@ export class InteractiveMode { } /** Startup details (model scope, loaded resources). Hidden by quietStartup: true or "header". */ + private getModelScopeNotice(): string | undefined { + if (this.session.scopedModels.length === 0 || !this.shouldShowStartupDetails()) return undefined; + const modelList = this.session.scopedModels + .map((sm) => { + const thinkingStr = sm.thinkingLevel ? `:${sm.thinkingLevel}` : ""; + return `${sm.model.id}${thinkingStr}`; + }) + .join(", "); + const cycleKeys = this.keybindings.getKeys("app.model.cycleForward"); + const cycleHint = + cycleKeys.length > 0 + ? theme.fg("muted", ` (${formatKeyText(cycleKeys.join("/"), { capitalize: true })} to cycle)`) + : ""; + return theme.fg("dim", `Model scope: ${modelList}${cycleHint}`); + } + + private showFullscreenModelScopeNotice(): void { + if (!this.getModelScopeNotice()) return; + this.headerContainer.addChild(new ThemedText(() => this.getModelScopeNotice() ?? "", 1, 0)); + this.headerContainer.addChild(new Spacer(1)); + } + private shouldShowStartupDetails(): boolean { return this.options.verbose === true || this.settingsManager.getQuietStartup() === false; } diff --git a/packages/coding-agent/test/interactive-mode-model-scope.test.ts b/packages/coding-agent/test/interactive-mode-model-scope.test.ts new file mode 100644 index 000000000..66a2ef82a --- /dev/null +++ b/packages/coding-agent/test/interactive-mode-model-scope.test.ts @@ -0,0 +1,58 @@ +import { Container } from "@earendil-works/pi-tui"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { InteractiveMode } from "../src/modes/interactive/interactive-mode.ts"; +import { initTheme } from "../src/modes/interactive/theme/theme.ts"; + +type ModelScopeContext = { + session: { scopedModels: Array<{ model: { id: string }; thinkingLevel?: string }> }; + keybindings: { getKeys: (keybinding: string) => string[] }; + headerContainer: Container; + shouldShowStartupDetails: () => boolean; + getModelScopeNotice: () => string | undefined; +}; + +type InteractiveModePrivate = { + getModelScopeNotice(this: ModelScopeContext): string | undefined; + showFullscreenModelScopeNotice(this: ModelScopeContext): void; +}; + +const interactiveModePrototype = InteractiveMode.prototype as unknown as InteractiveModePrivate; + +function createContext(showDetails = true): ModelScopeContext { + return { + session: { scopedModels: [{ model: { id: "model-a" }, thinkingLevel: "high" }, { model: { id: "model-b" } }] }, + keybindings: { getKeys: () => ["ctrl+p"] }, + headerContainer: new Container(), + shouldShowStartupDetails: () => showDetails, + getModelScopeNotice: interactiveModePrototype.getModelScopeNotice, + }; +} + +function stripAnsi(text: string): string { + return text.replace(/\x1b\[[0-9;]*m/g, ""); +} + +describe("InteractiveMode model scope notice", () => { + beforeEach(() => initTheme("dark")); + + // Regression test for #11508: fullscreen must not print the notice to the shell's main screen. + it("renders the model scope inside the fullscreen header instead of printing it", () => { + const context = createContext(); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + try { + interactiveModePrototype.showFullscreenModelScopeNotice.call(context); + expect(log).not.toHaveBeenCalled(); + } finally { + log.mockRestore(); + } + + const rendered = stripAnsi(context.headerContainer.render(120).join("\n")); + expect(rendered).toContain("Model scope: model-a:high, model-b (Ctrl+P to cycle)"); + }); + + it("adds nothing when startup details are hidden", () => { + const context = createContext(false); + interactiveModePrototype.showFullscreenModelScopeNotice.call(context); + expect(context.headerContainer.children).toHaveLength(0); + }); +}); From f6ee9ce9c16a6f2765a39728f3fbd601a9a83967 Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:58:20 +0100 Subject: [PATCH 050/271] fix(coding-agent): summarize union and multi-line codemode output types in tool descriptions (#11513) With `codemode.mode: "on"`, each declared tool gets a description suffix that says what a script call resolves to. For an object output schema, `describeOutput` shortens this to field names. Any other schema fell back to the whole rendered TypeScript type, collapsed onto one line. A tool with a union output schema therefore put its full output type, nested fields and comments included, into every request. The reporter's `process` tool, a union of 7 variants, added about 8,000 characters. Collapsing the type onto one line also left `//` comments in the middle of the text. This change: - **Unions:** for `anyOf`/`oneOf` schemas where every variant is an object, the suffix lists each variant's field names, joined by ` | `. - **`const` fields:** `describeFields` now shows the value of `const` fields, so a discriminator like `action` stays visible. For example: `` `{ action: "start", process } | { action: "list", processes, truncated? }` ``. - **Other multi-line types**, such as an array of objects: instead of the inlined type, the suffix says `` a value; `await describeTool("name")` shows its type ``. `describeTool()` already returns the full declaration. - **Single-line types** such as `number` are unchanged. Testing: - I added a test in `test/suite/agent-session-codemode.test.ts` that checks the suffix for three tools: one with a union output, one with an array-of-objects output, and one with a `number` output. It fails without the change and passes with it. - `node ../../node_modules/vitest/dist/cli.js --run test/suite/agent-session-codemode.test.ts test/suite/agent-session-mcp.test.ts` passes, including the existing MCP description assertions. - `npm run check` is clean. Fixes #11501 Requested by @badlogic Co-authored-by: triagent --- .../src/extensions/codemode/tool.ts | 39 +++++++++++++---- .../test/suite/agent-session-codemode.test.ts | 43 +++++++++++++++++++ 2 files changed, 74 insertions(+), 8 deletions(-) diff --git a/packages/coding-agent/src/extensions/codemode/tool.ts b/packages/coding-agent/src/extensions/codemode/tool.ts index b55a117a7..e28ea999f 100644 --- a/packages/coding-agent/src/extensions/codemode/tool.ts +++ b/packages/coding-agent/src/extensions/codemode/tool.ts @@ -271,8 +271,8 @@ export function createCodemodeDescription( /** * The field names of an object schema, `{ output, exit_code, full_output_path? }`, with the fields of - * object-typed fields `depth` levels deep: `{ content, structuredContent?: { results } }`. Undefined - * for other schemas. + * object-typed fields `depth` levels deep: `{ content, structuredContent?: { results } }`, and the + * value of `const` fields: `{ action: "start", id }`. Undefined for other schemas. */ function describeFields(schema: unknown, depth: number): string | undefined { if (typeof schema !== "object" || schema === null) return undefined; @@ -280,7 +280,13 @@ function describeFields(schema: unknown, depth: number): string | undefined { if (type !== "object" || typeof properties !== "object" || properties === null) return undefined; const requiredNames = new Set(Array.isArray(required) ? required : []); const fields = Object.entries(properties).map(([name, field]) => { - const nested = depth > 0 ? describeFields(field, depth - 1) : undefined; + const constValue = (field as Record | null)?.const; + const nested = + constValue !== undefined + ? JSON.stringify(constValue) + : depth > 0 + ? describeFields(field, depth - 1) + : undefined; return `${name}${requiredNames.has(name) ? "" : "?"}${nested ? `: ${nested}` : ""}`; }); return `{ ${fields.join(", ")} }`; @@ -288,13 +294,30 @@ function describeFields(schema: unknown, depth: number): string | undefined { /** * What a script call resolves to, in one line: `a string`, the field names of an object (see - * {@link describeFields}), or the rendered type for anything else. + * {@link describeFields}), the field names of each object in a union, the rendered type when it + * fits on one line, or a pointer to `describeTool()` for anything longer. The full type of a large + * schema would otherwise be sent with every request (#11501). */ -function describeOutput(schema: CodemodeJsonSchema | undefined): string { +function describeOutput(name: string, schema: CodemodeJsonSchema | undefined): string { const type = renderToolOutputType(schema); if (type === "string") return "a string"; - const fields = describeFields(schema, 1); - return `\`${fields ?? type.replace(/\s+/g, " ")}\``; + const fields = describeFields(schema, 1) ?? describeUnionFields(schema); + if (fields) return `\`${fields}\``; + if (!type.includes("\n")) return `\`${type}\``; + return `a value; \`await describeTool(${JSON.stringify(name)})\` shows its type`; +} + +/** + * The field names of each variant of an `anyOf`/`oneOf` whose variants are all objects, joined by + * `|`. + */ +function describeUnionFields(schema: unknown): string | undefined { + if (typeof schema !== "object" || schema === null) return undefined; + const { anyOf, oneOf } = schema as Record; + const variants = anyOf ?? oneOf; + if (!Array.isArray(variants) || variants.length === 0) return undefined; + const described = variants.map((variant) => describeFields(variant, 0)); + return described.every((fields) => fields !== undefined) ? described.join(" | ") : undefined; } /** @@ -302,7 +325,7 @@ function describeOutput(schema: CodemodeJsonSchema | undefined): string { * arguments are the tool's declared parameters, so they are not repeated. */ function describeScriptCall(tool: AgentTool): string { - return `${tool.description.trim()}\n\nCodemode: \`tools.${toCodemodeIdentifier(tool.name)}(args)\` resolves to ${describeOutput(toCodemodeDeclaration(tool).outputSchema)}.`; + return `${tool.description.trim()}\n\nCodemode: \`tools.${toCodemodeIdentifier(tool.name)}(args)\` resolves to ${describeOutput(tool.name, toCodemodeDeclaration(tool).outputSchema)}.`; } /** diff --git a/packages/coding-agent/test/suite/agent-session-codemode.test.ts b/packages/coding-agent/test/suite/agent-session-codemode.test.ts index 4aeeb9940..ff39f1cd6 100644 --- a/packages/coding-agent/test/suite/agent-session-codemode.test.ts +++ b/packages/coding-agent/test/suite/agent-session-codemode.test.ts @@ -184,6 +184,49 @@ describe("AgentSession codemode tool", () => { expect(description("echo")).toBe("Echo text back.\n\nSecond paragraph."); }); + // #11501 + it("summarizes union and multi-line output types instead of inlining them", async () => { + const outputTool = (name: string, outputSchema: AgentTool["outputSchema"]): AgentTool => ({ + name, + label: name, + description: `${name} tool`, + parameters: Type.Object({}), + outputSchema, + execute: async () => ({ content: [], details: {} }), + }); + const process = outputTool( + "process", + Type.Union([ + Type.Object({ + action: Type.Literal("start"), + process: Type.Object({ id: Type.String({ description: "Process id" }), pid: Type.Integer() }), + }), + Type.Object({ + action: Type.Literal("list"), + processes: Type.Array(Type.Object({ id: Type.String() })), + truncated: Type.Optional(Type.Boolean()), + }), + ]), + ); + const entries = outputTool("entries", Type.Array(Type.Object({ id: Type.String(), size: Type.Integer() }))); + const count = outputTool("count", Type.Number()); + const harness = await setup([ + (pi: ExtensionAPI) => { + for (const tool of [process, entries, count]) pi.registerTool(createToolDefinitionFromAgentTool(tool)); + }, + ]); + harness.session.setActiveToolsByName(["process", "entries", "count", "codemode"]); + const description = (name: string) => + harness.session.agent.state.tools.find((tool) => tool.name === name)?.description ?? ""; + expect(description("process")).toBe( + 'process tool\n\nCodemode: `tools.process(args)` resolves to `{ action: "start", process } | { action: "list", processes, truncated? }`.', + ); + expect(description("entries")).toBe( + 'entries tool\n\nCodemode: `tools.entries(args)` resolves to a value; `await describeTool("entries")` shows its type.', + ); + expect(description("count")).toBe("count tool\n\nCodemode: `tools.count(args)` resolves to `number`."); + }); + // #10343 it("shows the guidelines of tools that do not fit the inline budget through describeTool()", async () => { const harness = await setup(); From b9e3eef6c2e192f69f387c8f95245639c413d549 Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:09:41 +0100 Subject: [PATCH 051/271] fix(coding-agent): keep active tool order when the same tools are set again (#11514) The system prompt lists tool snippets and per-tool guidelines in the order of the active tools. When an extension called `pi.setActiveTools()` with the same tools in a different order (for example `[...pi.getActiveTools()].reverse()`), the "tools" and "rules" sections came out different. pi then recorded a "System Prompt Update: tools, rules" message even though the active tools had not changed. On providers without mid-conversation system messages, this also rewrites the prompt head and loses the cache. `_applyToolLoadout` in `agent-session.ts` now treats the requested names as a set: - tools that stay active keep their current order; - newly activated tools are appended at the end. Reordering the same set therefore leaves the prompt unchanged. Adding a tool only appends its bullet. The same rule covers tool lists changed by `before_agent_start` handlers, since they go through the same function. Testing: - Added a test in `test/system-prompt-updates.test.ts`: it reverses the active tools between two prompts and expects a single system message. It fails without the change (a second message patches `tools` and `rules`) and passes with it. - `npm run check` passes. - `./test.sh`: all coding-agent suites pass. The only failure is an unrelated Anthropic model metadata test in `packages/ai`. Fixes #11495 Requested by @badlogic Co-authored-by: triagent --- packages/coding-agent/CHANGELOG.md | 1 + packages/coding-agent/src/core/agent-session.ts | 7 ++++++- .../test/system-prompt-updates.test.ts | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 59827b2af..2ec3d9bd4 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -13,6 +13,7 @@ ### Fixed +- Fixed a redundant system prompt update (tools, rules) when an extension set the same active tools in a different order, for example a reversed `pi.setActiveTools()` list. Tools that stay active now keep their order and newly activated tools are appended ([#11495](https://github.com/earendil-works/pi/issues/11495)) - Fixed `pi install` and `pi remove` adding ever deeper nested entries to the npm `package-lock.json` when the agent directory is reached through a symlink, such as `~/.pi` linked into a dotfiles repo. pi now passes the resolved real path to npm ([#11358](https://github.com/earendil-works/pi/issues/11358)) - MCP sign-in now rejects authorization endpoints and other OAuth metadata URLs that are not `http:` or `https:`. Before, a malicious MCP server could supply a `file:` or custom protocol URL, which pi passed to the platform URL handler. - Automatic `fd` and `rg` downloads no longer fetch x86-64 binaries on ARM32 and other architectures without a prebuilt release. pi now warns and recommends installing the tool with the system package manager ([#11209](https://github.com/earendil-works/pi/issues/11209)) diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index 3a9174cf6..c1529f019 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -1591,9 +1591,14 @@ export class AgentSession { * the registered, non-hidden ones; they are declared to the model. Active tools with a * `prepareLoadout` hook can change the declared descriptions and hide declarations from * requests (see {@link _installHiddenDeclarationsProjection}). + * + * The active tools are a set: tools that stay active keep their current order and newly active + * ones are appended, so reordering the same names does not change the prompt (#11495). */ private _applyToolLoadout(toolNames: string[]): AgentTool[] { - const tools = [...new Set(toolNames)].flatMap((name) => { + const requested = new Set(toolNames); + const kept = this.getActiveToolNames().filter((name) => requested.has(name)); + const tools = [...new Set([...kept, ...requested])].flatMap((name) => { const tool = this._toolRegistry.get(name); return tool && this._getToolExposure(name) !== "hidden" && this._isActivatable(name) ? [tool] : []; }); diff --git a/packages/coding-agent/test/system-prompt-updates.test.ts b/packages/coding-agent/test/system-prompt-updates.test.ts index a55c493d9..bbe5fb9f0 100644 --- a/packages/coding-agent/test/system-prompt-updates.test.ts +++ b/packages/coding-agent/test/system-prompt-updates.test.ts @@ -53,6 +53,23 @@ describe("system prompt updates", () => { } }); + // #11495 + test("does not patch the prompt when the active tools are only reordered", async () => { + const harness = await createHarness(); + try { + harness.setResponses([fauxAssistantMessage("first"), fauxAssistantMessage("second")]); + await harness.session.prompt("one"); + const before = harness.session.getActiveToolNames(); + harness.session.setActiveToolsByName([...before].reverse()); + expect(harness.session.getActiveToolNames()).toEqual(before); + await harness.session.prompt("two"); + const systemMessages = harness.session.messages.filter((message) => message.role === "system"); + expect(systemMessages).toHaveLength(1); + } finally { + harness.cleanup(); + } + }); + test("opens a transcript without a system message and declares the prompt on the first request", async () => { const tempDir = mkdtempSync(join(tmpdir(), "pi-system-prompt-migration-")); try { From e509d7edc78a51e241f7525c451012db17818179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Hou=C5=A1ka?= Date: Wed, 28 Oct 2026 10:10:44 +0100 Subject: [PATCH 052/271] docs(quickstart): add Windows installer tab (#11231) The Quickstart only showed the macOS/Linux `curl` installer, even though `install.ps1` installs Pi the same way: a managed install pinned by the published lockfile, updated with `pi update`. This PR adds the Windows installer as a second tab next to the existing one. All other sections stay the same. ````markdown Install Pi with the installer on macOS, Linux, or Windows: ```bash tab="macOS / Linux" curl -fsSL https://pi.dev/install.sh | sh ``` ```powershell tab="Windows" powershell -c "irm https://pi.dev/install.ps1 | iex" ``` Both installers pin all dependencies and update Pi with `pi update`. ... ```` The tabs are labelled by platform rather than by shell: newcomers know their OS, and the command also works when pasted into cmd. Tab rendering on pi.dev comes from earendil-works/pi.dev#183. This PR can merge before or after it. Until the renderer ships, and on GitHub, the page shows two ordinary code blocks, and the prose still reads correctly. ## Screenshots (with earendil-works/pi.dev#183) | | Default | Windows tab selected | |---|---|---| | Dark | ![dark default](https://gist.githubusercontent.com/petrroll/27308d9d8522c90ea5b290b0c3ba9862/raw/cad05d1cf643903955af7282763e22ff2a101e73/dark-unix.png) | ![dark windows](https://gist.githubusercontent.com/petrroll/27308d9d8522c90ea5b290b0c3ba9862/raw/cad05d1cf643903955af7282763e22ff2a101e73/dark-windows-tab.png) | | Light | ![light default](https://gist.githubusercontent.com/petrroll/27308d9d8522c90ea5b290b0c3ba9862/raw/cad05d1cf643903955af7282763e22ff2a101e73/light-unix.png) | ![light windows](https://gist.githubusercontent.com/petrroll/27308d9d8522c90ea5b290b0c3ba9862/raw/cad05d1cf643903955af7282763e22ff2a101e73/light-windows-tab.png) | --- packages/coding-agent/docs/quickstart.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/coding-agent/docs/quickstart.md b/packages/coding-agent/docs/quickstart.md index 89b56f07e..44054e949 100644 --- a/packages/coding-agent/docs/quickstart.md +++ b/packages/coding-agent/docs/quickstart.md @@ -6,13 +6,17 @@ For native Windows setup, read [Windows Setup](windows.md). For Android, read [T ## 1. Install Pi -On macOS or Linux, you can use the installer: +Install Pi with the installer on macOS, Linux, or Windows: -```bash +```bash tab="macOS / Linux" curl -fsSL https://pi.dev/install.sh | sh ``` -The installer pins all dependencies and updates Pi with `pi update`. Alternatively, install Pi from npm, which does not pin transitive dependencies. This requires Node.js 22.19 or newer: +```powershell tab="Windows" +powershell -c "irm https://pi.dev/install.ps1 | iex" +``` + +Both installers pin all dependencies and update Pi with `pi update`. Alternatively, install Pi from npm, which does not pin transitive dependencies. This requires Node.js 22.19 or newer: ```bash npm install -g --ignore-scripts @earendil-works/pi-coding-agent From fa16851520f8c5edf930d14ca04cdb4cb71d7a67 Mon Sep 17 00:00:00 2001 From: xu0o0 Date: Wed, 28 Oct 2026 17:11:40 +0800 Subject: [PATCH 053/271] fix(tui): stop bare URLs at CJK punctuation (#10934) This PR: - Stop bare URLs at CJK punctuation, preserving the following text. - Leave CJK characters without punctuation, explicit Markdown links, and angle autolinks unchanged. - Add regression tests. Fixes #10933 Before: screenshot-2026-10-14_17-13-48 After: screenshot-2026-10-14_17-14-14 --- packages/tui/src/components/markdown.ts | 16 +++++++-- packages/tui/test/markdown.test.ts | 43 +++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/packages/tui/src/components/markdown.ts b/packages/tui/src/components/markdown.ts index a3b526d73..bacfe29f8 100644 --- a/packages/tui/src/components/markdown.ts +++ b/packages/tui/src/components/markdown.ts @@ -3,11 +3,21 @@ import markedCjkFriendly from "marked-cjk-friendly"; import { renderLatex } from "../latex.ts"; import { getCapabilities, hyperlink, isImageLine } from "../terminal-image.ts"; import type { Component } from "../tui.ts"; -import { applyBackgroundToLine, flattenLines, visibleWidth, wrapTextWithAnsi } from "../utils.ts"; +import { applyBackgroundToLine, cjkPunctuationRegex, flattenLines, visibleWidth, wrapTextWithAnsi } from "../utils.ts"; const STRICT_STRIKETHROUGH_REGEX = /^(~~)(?=[^\s~])((?:\\.|[^\\])*?(?:\\.|[^\s~\\]))\1(?=[^~]|$)/; -class StrictStrikethroughTokenizer extends Tokenizer { +class MarkdownTokenizer extends Tokenizer { + override url(src: string): Tokens.Link | undefined { + const token = super.url(src); + if (!token) return undefined; + + // Bare URLs stop at CJK punctuation; explicit links and angle autolinks + // use separate tokenizers and can still contain these characters. + const punctuationIndex = token.raw.search(cjkPunctuationRegex); + return punctuationIndex < 0 ? token : super.url(token.raw.slice(0, punctuationIndex)); + } + override del(src: string): Tokens.Del | undefined { const match = STRICT_STRIKETHROUGH_REGEX.exec(src); if (!match) { @@ -189,7 +199,7 @@ function trimPartialClosingFences(tokens: readonly Token[]): void { const markdownParser = new Marked(); markdownParser.setOptions({ - tokenizer: new StrictStrikethroughTokenizer(), + tokenizer: new MarkdownTokenizer(), }); markdownParser.use({ tokenizer: { emStrong: markedCjkFriendly().tokenizer?.emStrong }, diff --git a/packages/tui/test/markdown.test.ts b/packages/tui/test/markdown.test.ts index b20d84dfe..0cdcf172b 100644 --- a/packages/tui/test/markdown.test.ts +++ b/packages/tui/test/markdown.test.ts @@ -1716,6 +1716,49 @@ bar`, resetCapabilitiesCache(); }); + it("should stop bare URLs at CJK punctuation and preserve the following text", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: true }); + for (const punctuation of "。,;:!?、“”‘’()【】《》「」『』…—·・〈〉[]{}.。「」、・") { + for (const url of ["https://example.com", "https://example.com/中文?q=搜索#片段", "www.example.com"]) { + const source = `${url}${punctuation}后面的中文`; + const output = new Markdown(source, 0, 0, defaultMarkdownTheme).render(160).join(""); + const href = url.startsWith("www.") ? `http://${url}` : url; + assert.ok(output.includes(`\x1b]8;;${href}\x1b\\`), source); + assert.ok(stripAnsi(output).includes(`\x1b]8;;\x1b\\${punctuation}后面的中文`), source); + assert.strictEqual(stripAnsi(output.replace(/\x1b\]8;;[^\x1b]*\x1b\\/g, "")).trimEnd(), source); + } + } + }); + + it("should preserve Chinese characters without punctuation in bare URLs", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: true }); + for (const url of [ + "https://example.com紧接着中文", + "https://example.com/中文?q=搜索#片段", + "https://example.com/テストー한글", + ]) { + const output = new Markdown(url, 0, 0, defaultMarkdownTheme).render(160).join(""); + assert.ok(output.includes(`\x1b]8;;${url}\x1b\\`)); + } + }); + + it("should preserve CJK punctuation in explicit links and angle autolinks", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: true }); + const url = "https://example.com/中文,路径·テスト・한글…—。"; + for (const source of [`[文档](${url})`, `<${url}>`]) { + const output = new Markdown(source, 0, 0, defaultMarkdownTheme).render(160).join(""); + assert.ok(output.includes(`\x1b]8;;${url}\x1b\\`), source); + } + }); + + it("should preserve English punctuation trimming before Chinese punctuation", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: true }); + const source = "https://example.com.,后面的中文"; + const output = new Markdown(source, 0, 0, defaultMarkdownTheme).render(160).join(""); + assert.ok(output.includes("\x1b]8;;https://example.com\x1b\\")); + assert.ok(stripAnsi(output).includes("\x1b]8;;\x1b\\.,后面的中文")); + }); + it("should not duplicate URL for autolinked emails", () => { // Hyperlinks capability does not affect the mailto: display check. setCapabilities({ images: null, trueColor: false, hyperlinks: false }); From 068e972c7baefac2db3a245f1a634ce21a90a2d9 Mon Sep 17 00:00:00 2001 From: David Brailovsky Date: Wed, 28 Oct 2026 10:37:22 +0100 Subject: [PATCH 054/271] fix(ai): remove adaptive thinking metadata test that duplicated generator logic The test copied the generator's model-ID rule and snapshotted upstream catalog IDs, so it broke on catalog changes without describing behavior beyond the code. --- ...anthropic-adaptive-thinking-models.test.ts | 50 ------------------- 1 file changed, 50 deletions(-) delete mode 100644 packages/ai/test/anthropic-adaptive-thinking-models.test.ts diff --git a/packages/ai/test/anthropic-adaptive-thinking-models.test.ts b/packages/ai/test/anthropic-adaptive-thinking-models.test.ts deleted file mode 100644 index a3a1c4414..000000000 --- a/packages/ai/test/anthropic-adaptive-thinking-models.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { getModels, getProviders } from "../src/compat.ts"; -import type { Api, Model } from "../src/types.ts"; - -const EXPECTED_CURRENT_ADAPTIVE_THINKING_MODELS = [ - "anthropic/claude-fable-5", - "anthropic/claude-haiku-5-5", - "anthropic/claude-opus-4-8", - "anthropic/claude-opus-5", - "anthropic/claude-sonnet-5", - "cloudflare-ai-gateway/claude-fable-5", - "fireworks/accounts/fireworks/models/deepseek-v4p1-flash", - "fireworks/accounts/fireworks/models/gpt-oss-120b", - "fireworks/accounts/fireworks/models/qwen3p8-max", - "minimax-cn/MiniMax-M3.1-Flash", - "minimax/MiniMax-M3.1-Flash", - "opencode/claude-opus-4-8", - "opencode/claude-opus-5", - "vercel-ai-gateway/anthropic/claude-opus-4.8", - "vercel-ai-gateway/anthropic/claude-opus-5", - "vercel-ai-gateway/anthropic/claude-sonnet-5", -]; - -function getAllModels(): Model[] { - return getProviders().flatMap((provider) => getModels(provider) as Model[]); -} - -describe("Anthropic adaptive thinking model metadata", () => { - it("marks built-in Anthropic Messages models that use adaptive thinking", () => { - const flaggedModels = getAllModels() - .filter((model): model is Model<"anthropic-messages"> => model.api === "anthropic-messages") - .filter((model) => model.compat?.forceAdaptiveThinking === true) - .map((model) => `${model.provider}/${model.id}`) - .sort(); - - expect(flaggedModels).toEqual(expect.arrayContaining([...EXPECTED_CURRENT_ADAPTIVE_THINKING_MODELS].sort())); - expect(flaggedModels).toEqual( - flaggedModels.filter( - (modelId) => - // Regression for #9323: Fireworks uses catalog effort metadata and - // verified fallbacks, not a fixed set of adaptive model names. - modelId.startsWith("fireworks/") || - // #11121: MiniMax models with catalog effort levels require adaptive thinking. - modelId.startsWith("minimax/") || - modelId.startsWith("minimax-cn/") || - /(opus[-.](4[-.][678]|5)|sonnet[-.]4[-.]6|sonnet[-.]5|haiku[-.]5[-.]5|fable[-.]5)/.test(modelId), - ), - ); - }); -}); From 2e0f175bd58752ad4123aa12c2f3979839003dc7 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Wed, 28 Oct 2026 11:06:10 +0100 Subject: [PATCH 055/271] feat(ai): refactor provider definitions (#11368) This builds the model catalog from per-provider definitions and models.dev v2 data. It eplacer `generate-models.ts` with one `defineProvider()` file per built-in provider, shared model-family rules (Claude, GPT, DeepSeek, Kimi, Google), and source adapters. models.dev is read through its v2 API. Outputs and CLI flags are unchanged. --- .pi/skills/add-llm-provider.md | 8 +- AGENTS.md | 2 +- biome.json | 1 + packages/ai/CHANGELOG.md | 1 + packages/ai/README.md | 6 +- packages/ai/catalog/README.md | 47 + packages/ai/catalog/build.ts | 164 + packages/ai/catalog/define.ts | 142 + packages/ai/catalog/families/anthropic.ts | 134 + packages/ai/catalog/families/deepseek.ts | 67 + packages/ai/catalog/families/google.ts | 22 + packages/ai/catalog/families/kimi.ts | 38 + packages/ai/catalog/families/openai.ts | 141 + packages/ai/catalog/generate.ts | 308 ++ .../ai/catalog/providers/amazon-bedrock.ts | 32 + packages/ai/catalog/providers/ant-ling.ts | 52 + packages/ai/catalog/providers/anthropic.ts | 66 + packages/ai/catalog/providers/azure.ts | 61 + packages/ai/catalog/providers/baseten.ts | 89 + packages/ai/catalog/providers/cerebras.ts | 8 + .../providers/cloudflare-ai-gateway.ts | 84 + .../providers/cloudflare-workers-ai.ts | 61 + packages/ai/catalog/providers/deepseek.ts | 57 + packages/ai/catalog/providers/fireworks.ts | 100 + .../ai/catalog/providers/github-copilot.ts | 122 + .../ai/catalog/providers/google-vertex.ts | 43 + packages/ai/catalog/providers/google.ts | 36 + packages/ai/catalog/providers/groq.ts | 13 + packages/ai/catalog/providers/huggingface.ts | 12 + packages/ai/catalog/providers/index.ts | 90 + packages/ai/catalog/providers/kimi-coding.ts | 55 + packages/ai/catalog/providers/meta.ts | 6 + packages/ai/catalog/providers/minimax-cn.ts | 7 + packages/ai/catalog/providers/minimax.ts | 26 + packages/ai/catalog/providers/mistral.ts | 46 + .../ai/catalog/providers/moonshotai-cn.ts | 9 + packages/ai/catalog/providers/moonshotai.ts | 71 + packages/ai/catalog/providers/nvidia.ts | 64 + packages/ai/catalog/providers/openai-codex.ts | 68 + packages/ai/catalog/providers/openai.ts | 141 + packages/ai/catalog/providers/opencode-go.ts | 9 + packages/ai/catalog/providers/opencode.ts | 206 + packages/ai/catalog/providers/openrouter.ts | 108 + .../catalog/providers/qwen-token-plan-cn.ts | 13 + .../providers/qwen-token-plan-individual.ts | 34 + .../ai/catalog/providers/qwen-token-plan.ts | 53 + packages/ai/catalog/providers/radius.ts | 7 + packages/ai/catalog/providers/together.ts | 58 + packages/ai/catalog/providers/typesafe.ts | 32 + .../ai/catalog/providers/vercel-ai-gateway.ts | 14 + packages/ai/catalog/providers/xai.ts | 29 + .../providers/xiaomi-token-plan-ams.ts | 8 + .../catalog/providers/xiaomi-token-plan-cn.ts | 8 + .../providers/xiaomi-token-plan-sgp.ts | 8 + packages/ai/catalog/providers/xiaomi.ts | 46 + .../ai/catalog/providers/zai-coding-cn.ts | 8 + packages/ai/catalog/providers/zai.ts | 50 + .../reasoning.ts} | 9 +- packages/ai/catalog/sources/index.ts | 95 + packages/ai/catalog/sources/models-dev.ts | 242 ++ .../sources/openrouter.ts} | 46 +- .../ai/catalog/sources/vercel-ai-gateway.ts | 140 + packages/ai/package.json | 6 +- packages/ai/scripts/ai-gateway-pricing.ts | 67 - packages/ai/scripts/generate-models.ts | 3688 ----------------- .../scripts/openrouter-reasoning-options.ts | 23 - packages/ai/src/models.generated.ts | 2 +- .../ai/src/providers/amazon-bedrock.models.ts | 2 +- packages/ai/src/providers/ant-ling.models.ts | 2 +- packages/ai/src/providers/anthropic.models.ts | 2 +- packages/ai/src/providers/azure.models.ts | 2 +- packages/ai/src/providers/baseten.models.ts | 2 +- packages/ai/src/providers/cerebras.models.ts | 2 +- .../providers/cloudflare-ai-gateway.models.ts | 2 +- .../providers/cloudflare-workers-ai.models.ts | 2 +- packages/ai/src/providers/deepseek.models.ts | 2 +- packages/ai/src/providers/fireworks.models.ts | 2 +- .../ai/src/providers/github-copilot.models.ts | 2 +- .../ai/src/providers/google-vertex.models.ts | 2 +- packages/ai/src/providers/google.models.ts | 2 +- packages/ai/src/providers/groq.models.ts | 2 +- .../ai/src/providers/huggingface.models.ts | 2 +- .../ai/src/providers/kimi-coding.models.ts | 2 +- packages/ai/src/providers/meta.models.ts | 2 +- .../ai/src/providers/minimax-cn.models.ts | 2 +- packages/ai/src/providers/minimax.models.ts | 2 +- packages/ai/src/providers/mistral.models.ts | 2 +- .../ai/src/providers/moonshotai-cn.models.ts | 2 +- .../ai/src/providers/moonshotai.models.ts | 2 +- packages/ai/src/providers/nvidia.models.ts | 2 +- .../ai/src/providers/openai-codex.models.ts | 2 +- packages/ai/src/providers/openai.models.ts | 2 +- .../ai/src/providers/opencode-go.models.ts | 2 +- packages/ai/src/providers/opencode.models.ts | 2 +- .../ai/src/providers/openrouter.models.ts | 2 +- .../providers/qwen-token-plan-cn.models.ts | 2 +- .../qwen-token-plan-individual.models.ts | 2 +- .../src/providers/qwen-token-plan.models.ts | 2 +- packages/ai/src/providers/radius.models.ts | 2 +- packages/ai/src/providers/together.models.ts | 2 +- packages/ai/src/providers/typesafe.models.ts | 2 +- .../src/providers/vercel-ai-gateway.models.ts | 2 +- packages/ai/src/providers/xai.models.ts | 2 +- .../providers/xiaomi-token-plan-ams.models.ts | 2 +- .../providers/xiaomi-token-plan-cn.models.ts | 2 +- .../providers/xiaomi-token-plan-sgp.models.ts | 2 +- packages/ai/src/providers/xiaomi.models.ts | 2 +- .../ai/src/providers/zai-coding-cn.models.ts | 2 +- packages/ai/src/providers/zai.models.ts | 2 +- packages/ai/test/catalog-generation-utils.ts | 52 + .../test/fireworks-model-generation.test.ts | 59 +- .../ai/test/generate-models-strict.test.ts | 30 +- packages/ai/test/image-model-data.test.ts | 2 +- packages/ai/test/model-cost-tiers.test.ts | 4 +- .../ai/test/opencode-model-generation.test.ts | 97 + .../test/openrouter-reasoning-options.test.ts | 2 +- packages/ai/test/reasoning-options.test.ts | 2 +- scripts/diff-model-catalog.mjs | 6 +- scripts/issues/categories/models.md | 2 +- tsconfig.json | 2 +- 120 files changed, 3952 insertions(+), 3899 deletions(-) create mode 100644 packages/ai/catalog/README.md create mode 100644 packages/ai/catalog/build.ts create mode 100644 packages/ai/catalog/define.ts create mode 100644 packages/ai/catalog/families/anthropic.ts create mode 100644 packages/ai/catalog/families/deepseek.ts create mode 100644 packages/ai/catalog/families/google.ts create mode 100644 packages/ai/catalog/families/kimi.ts create mode 100644 packages/ai/catalog/families/openai.ts create mode 100644 packages/ai/catalog/generate.ts create mode 100644 packages/ai/catalog/providers/amazon-bedrock.ts create mode 100644 packages/ai/catalog/providers/ant-ling.ts create mode 100644 packages/ai/catalog/providers/anthropic.ts create mode 100644 packages/ai/catalog/providers/azure.ts create mode 100644 packages/ai/catalog/providers/baseten.ts create mode 100644 packages/ai/catalog/providers/cerebras.ts create mode 100644 packages/ai/catalog/providers/cloudflare-ai-gateway.ts create mode 100644 packages/ai/catalog/providers/cloudflare-workers-ai.ts create mode 100644 packages/ai/catalog/providers/deepseek.ts create mode 100644 packages/ai/catalog/providers/fireworks.ts create mode 100644 packages/ai/catalog/providers/github-copilot.ts create mode 100644 packages/ai/catalog/providers/google-vertex.ts create mode 100644 packages/ai/catalog/providers/google.ts create mode 100644 packages/ai/catalog/providers/groq.ts create mode 100644 packages/ai/catalog/providers/huggingface.ts create mode 100644 packages/ai/catalog/providers/index.ts create mode 100644 packages/ai/catalog/providers/kimi-coding.ts create mode 100644 packages/ai/catalog/providers/meta.ts create mode 100644 packages/ai/catalog/providers/minimax-cn.ts create mode 100644 packages/ai/catalog/providers/minimax.ts create mode 100644 packages/ai/catalog/providers/mistral.ts create mode 100644 packages/ai/catalog/providers/moonshotai-cn.ts create mode 100644 packages/ai/catalog/providers/moonshotai.ts create mode 100644 packages/ai/catalog/providers/nvidia.ts create mode 100644 packages/ai/catalog/providers/openai-codex.ts create mode 100644 packages/ai/catalog/providers/openai.ts create mode 100644 packages/ai/catalog/providers/opencode-go.ts create mode 100644 packages/ai/catalog/providers/opencode.ts create mode 100644 packages/ai/catalog/providers/openrouter.ts create mode 100644 packages/ai/catalog/providers/qwen-token-plan-cn.ts create mode 100644 packages/ai/catalog/providers/qwen-token-plan-individual.ts create mode 100644 packages/ai/catalog/providers/qwen-token-plan.ts create mode 100644 packages/ai/catalog/providers/radius.ts create mode 100644 packages/ai/catalog/providers/together.ts create mode 100644 packages/ai/catalog/providers/typesafe.ts create mode 100644 packages/ai/catalog/providers/vercel-ai-gateway.ts create mode 100644 packages/ai/catalog/providers/xai.ts create mode 100644 packages/ai/catalog/providers/xiaomi-token-plan-ams.ts create mode 100644 packages/ai/catalog/providers/xiaomi-token-plan-cn.ts create mode 100644 packages/ai/catalog/providers/xiaomi-token-plan-sgp.ts create mode 100644 packages/ai/catalog/providers/xiaomi.ts create mode 100644 packages/ai/catalog/providers/zai-coding-cn.ts create mode 100644 packages/ai/catalog/providers/zai.ts rename packages/ai/{scripts/models-dev-reasoning-options.ts => catalog/reasoning.ts} (79%) create mode 100644 packages/ai/catalog/sources/index.ts create mode 100644 packages/ai/catalog/sources/models-dev.ts rename packages/ai/{scripts/openrouter-catalog.ts => catalog/sources/openrouter.ts} (66%) create mode 100644 packages/ai/catalog/sources/vercel-ai-gateway.ts delete mode 100644 packages/ai/scripts/ai-gateway-pricing.ts delete mode 100644 packages/ai/scripts/generate-models.ts delete mode 100644 packages/ai/scripts/openrouter-reasoning-options.ts create mode 100644 packages/ai/test/catalog-generation-utils.ts create mode 100644 packages/ai/test/opencode-model-generation.test.ts diff --git a/.pi/skills/add-llm-provider.md b/.pi/skills/add-llm-provider.md index 884d3da7f..d137e548e 100644 --- a/.pi/skills/add-llm-provider.md +++ b/.pi/skills/add-llm-provider.md @@ -23,11 +23,11 @@ Most providers reuse an existing API such as `openai-completions`; skip this ste - Register the lazy wrapper in the builtin API list in `src/compat.ts`. - Add `export type` re-exports in `src/index.ts` for option types that should be available from the root entry. The `./api/*` subpath export in `package.json` already covers the new module. -## 3. Model Generation (`packages/ai/scripts/generate-models.ts`) +## 3. Model Catalog (`packages/ai/catalog/providers/.ts`) -- Add logic to fetch/parse models from the provider source. -- Map to the standardized `Model` interface. -- Regenerate. This writes the stable catalog wrapper `src/providers/.models.ts` (exporting `_MODELS`) that the provider factory imports, and updates `src/models.generated.ts`. +- Add a provider definition with `defineProvider()` and register it in `catalog/providers/index.ts`. Read `packages/ai/catalog/README.md` first. +- For a models.dev provider, use `modelsDevModels()`; set compat defaults per API in `compat` and provider-specific behavior in `finalize`. +- Regenerate with `npm run generate-models` in `packages/ai`. This writes the stable catalog wrapper `src/providers/.models.ts` (exporting `_MODELS`) that the provider factory imports, and updates `src/models.generated.ts`. ## 4. Provider Factory and Auth (`packages/ai/src/providers/`) diff --git a/AGENTS.md b/AGENTS.md index 9876d8031..04c9c32be 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,7 +25,7 @@ - Always ask before removing functionality or code that appears intentional. - Do not preserve backward compatibility unless the user asks for it. - Never hardcode key checks (e.g. `matchesKey(keyData, "ctrl+x")`). Add defaults to `DEFAULT_EDITOR_KEYBINDINGS` or `DEFAULT_APP_KEYBINDINGS` so they stay configurable. -- Never modify `packages/ai/src/models.generated.ts` directly; update `packages/ai/scripts/generate-models.ts` instead, then regenerate. Including the resulting `models.generated.ts` diff is always OK, even if regeneration includes unrelated upstream model metadata changes. The generated model data in `packages/ai/src/providers/data/` is gitignored: never stage it, and don't look for it in git history. +- Never modify `packages/ai/src/models.generated.ts` directly; update the catalog definitions in `packages/ai/catalog/` instead (see its README), then regenerate. Including the resulting `models.generated.ts` diff is always OK, even if regeneration includes unrelated upstream model metadata changes. The generated model data in `packages/ai/src/providers/data/` is gitignored: never stage it, and don't look for it in git history. ## Commands diff --git a/biome.json b/biome.json index a27fb2366..2c5b8a2fa 100644 --- a/biome.json +++ b/biome.json @@ -29,6 +29,7 @@ "scripts/model-catalog-protocol.test.ts", "packages/*/src/**/*.ts", "packages/*/test/**/*.ts", + "packages/ai/catalog/**/*.ts", "packages/coding-agent/examples/**/*.ts", "packages/agent/examples/**/*.ts", "!**/node_modules/**/*", diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 444c3be23..996207fbd 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -15,6 +15,7 @@ ### Changed - The faux provider streams partial tool-call arguments, so `toolcall_delta` partials show growing arguments as real providers do ([#9265](https://github.com/earendil-works/pi/issues/9265)) +- The built-in model catalog is generated from models.dev's v2 catalog, and its per-provider rules moved from `scripts/generate-models.ts` into one TypeScript definition per provider under `catalog/`. Models that models.dev types as realtime or image models (`gpt-realtime-2.1`, `gemini-3.1-flash-live-preview`, `gemini-3.1-flash-lite-image`) are no longer offered as chat models, and models.dev's newer OpenAI and Google chat models (`chat-latest`, `gpt-5.6-cyber`, `gemini-robotics-er-2-preview`) are now included ### Fixed diff --git a/packages/ai/README.md b/packages/ai/README.md index 37fbcb7e2..583d55bc3 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -1928,12 +1928,12 @@ Create a new API implementation file (for example `bedrock-converse-stream.ts`) Add a lazy wrapper `src/api/.lazy.ts` (`Api()` via `lazyApi()`) so providers can reference the implementation without importing its SDK. Add any root-level `export type` re-exports in `src/index.ts` that should remain available from `@earendil-works/pi-ai`. -#### 3. Model Generation (`scripts/generate-models.ts`) +#### 3. Model Catalog (`catalog/providers/.ts`) -- Add logic to fetch and parse models from the provider's source (e.g., models.dev API) +- Add a provider definition and register it in `catalog/providers/index.ts`; see `catalog/README.md` for the build order and helpers - Map chat/tool-capable provider data to `Model`, image-generation data to `ImageModel`, and models.dev `type: "decision"` entries to `ClassifierModel`; hydration groups the ignored `src/providers/data/.json` values by API while stable `src/providers/.models.ts` wrappers derive exact model/API types directly from those JSON keys - Keep model ids unique within each provider and model type; emit separate entries when an upstream model supports multiple operations -- Handle provider-specific quirks (pricing format, capability flags, model ID transformations) +- Handle provider-specific quirks (pricing format, capability flags, model ID transformations) in the provider definition #### 4. Provider Factory (`src/providers/.ts`) diff --git a/packages/ai/catalog/README.md b/packages/ai/catalog/README.md new file mode 100644 index 000000000..24f3641a4 --- /dev/null +++ b/packages/ai/catalog/README.md @@ -0,0 +1,47 @@ +# Model catalog + +`node catalog/generate.ts` (`npm run generate-models`) fetches upstream catalogs and writes the built-in model catalog: + +- `src/providers/data/.json`: model values, grouped by API (gitignored) +- `src/providers/.models.ts` and `src/models.generated.ts`: typed wrappers (committed) +- with `--json-output `: the public catalog JSON served by pi.dev + +## Layout + +| Path | Contents | +| --- | --- | +| `sources/` | Upstream fetchers. `models-dev.ts` normalizes the [models.dev v2](https://models.dev/experimental/v2.0/api.json) catalog; OpenRouter, Vercel AI Gateway, NVIDIA NIM, and Radius have their own modules. All sources are fetched before anything is built. | +| `providers/.ts` | One file per built-in provider: where its models come from, its API and base URL, compat defaults, corrections, and models upstream does not list. Register new files in `providers/index.ts`. | +| `families/` | Facts about a model family that hold on every host, such as Claude effort levels or GPT-6 thinking levels. They apply to every provider's models. | +| `define.ts` | `defineProvider()` and the helpers provider files use. | +| `build.ts` | Runs the provider definitions in a fixed order. | + +## Build order + +For each provider: + +1. `models(ctx)` returns its chat models: upstream entries mapped to pi models, data corrections (limits, prices), and authored models. `ctx.providerModels(id)` returns another provider's models from this step, e.g. Azure derives from OpenAI. +2. `compat` defaults for each API are merged under each model's own compat. Then the shared rules run: models.dev effort levels for APIs that send them natively, and the rules in `families/`. +3. `finalize(model, ctx)` applies provider-specific behavior. It runs after the shared rules, so its values win. +4. Image input limits get the default resize profile, then `finalizeAll(models, ctx)` runs cross-model logic such as Anthropic's allowed fallback models. + +## Adding a provider + +Most providers are a models.dev provider served on one API: + +```ts +import { defineProvider, modelsDevModels } from "../define.ts"; + +export default defineProvider({ + id: "example", + models: (ctx) => + modelsDevModels(ctx, "example", { api: "openai-completions", baseUrl: "https://api.example.com/v1" }), + compat: { "openai-completions": { supportsStore: false, supportsDeveloperRole: false } }, +}); +``` + +`modelsDevModels()` keeps tool-capable chat models and maps names, input modalities, costs (including context tiers), and limits. Use its `include` and `map` options for provider-specific filtering and fields. Prefer fixing wrong upstream data in models.dev; keep local corrections small and say why they exist. + +## Checking changes + +`npm run diff:model-catalog -- [provider ...]` from the repository root generates the catalog at `HEAD` and in the worktree and prints the JSON differences. diff --git a/packages/ai/catalog/build.ts b/packages/ai/catalog/build.ts new file mode 100644 index 000000000..1a47dce05 --- /dev/null +++ b/packages/ai/catalog/build.ts @@ -0,0 +1,164 @@ +import type { + AnyModel, + Api, + ClassifierApi, + ClassifierModel, + ImageApi, + ImageModel, + KnownApi, + Model, + OpenAICompletionsCompat, +} from "../src/types.ts"; +import type { ApiCompatDefaults, ChatModel, ProviderContext, ProviderDefinition } from "./define.ts"; +import { mergeThinking } from "./define.ts"; +import { applyClaudeFamilyRules } from "./families/anthropic.ts"; +import { applyDeepSeekCompatRules, applyDeepSeekFamilyRules } from "./families/deepseek.ts"; +import { applyOpenAiFamilyRules } from "./families/openai.ts"; +import { getEffortThinkingLevelMap } from "./reasoning.ts"; +import type { CatalogSources } from "./sources/index.ts"; +import type { ModelsDevModel } from "./sources/models-dev.ts"; + +export interface ProviderCatalog { + chat: (Model & { type: "chat" })[]; + image: ImageModel[]; + classifier: ClassifierModel[]; +} + +// Keep the generated default no less restrictive than coding-agent's historical +// image preprocessing. Provider limits can narrow this profile, but unknown +// providers retain the cache-safe 2000px / 4.5 MiB behavior. +const DEFAULT_IMAGE_RESIZE = { + maxWidth: 2000, + maxHeight: 2000, + maxBytes: 4.5 * 1024 * 1024, + jpegQuality: 80, +} as const; + +function modelKey(model: Pick): string { + return `${model.provider}:${model.id}`; +} + +function applyCompatDefaults(model: ChatModel, defaults: ApiCompatDefaults | undefined): void { + const entry = defaults?.[model.api as KnownApi] as + | Model["compat"] + | ((model: Model) => Model["compat"]) + | undefined; + if (!entry) return; + const compat = typeof entry === "function" ? entry(model) : entry; + model.compat = { ...compat, ...model.compat } as Model["compat"]; +} + +/** Whether the model's API sends models.dev effort values to the host as-is. */ +function sendsNativeEffort(model: ChatModel): boolean { + if (model.api === "anthropic-messages") { + return (model as Model<"anthropic-messages">).compat?.forceAdaptiveThinking === true; + } + if ( + model.api === "openai-responses" || + model.api === "azure-openai-responses" || + model.api === "openai-codex-responses" + ) { + return true; + } + if (model.api !== "openai-completions") return false; + const compat: OpenAICompletionsCompat = (model as Model<"openai-completions">).compat ?? {}; + return (compat.thinkingFormat ?? "openai") === "openai" && (compat.supportsReasoningEffort ?? true); +} + +function applyModelsDevEffort(model: ChatModel, source: ModelsDevModel | undefined): void { + if (!source?.reasoningOptions || !sendsNativeEffort(model)) return; + const map = getEffortThinkingLevelMap(source.reasoningOptions); + if (map) mergeThinking(model, map); +} + +function applyImageResizeDefaults(model: AnyModel): void { + if (!model.input.includes("image")) return; + const images = model.inputLimits?.images; + model.inputLimits = { + ...model.inputLimits, + images: { ...images, resize: { ...DEFAULT_IMAGE_RESIZE, ...images?.resize } }, + }; +} + +/** Build every provider's catalog from the fetched sources. */ +export function buildCatalog( + definitions: readonly ProviderDefinition[], + sources: CatalogSources, + strict: boolean, +): Record { + const definitionsById = new Map(definitions.map((definition) => [definition.id, definition])); + const modelsDevSources = new Map(); + const stageOne = new Map(); + const building = new Set(); + + function context(providerId: string): ProviderContext { + return { + providerId, + sources, + strict, + providerModels: (id) => structuredClone(buildModels(id)), + modelsDevSource: (model) => modelsDevSources.get(modelKey(model)), + recordModelsDevSource: (model, source) => { + modelsDevSources.set(modelKey(model), source); + }, + }; + } + + function buildModels(providerId: string): ChatModel[] { + const built = stageOne.get(providerId); + if (built) return built; + const definition = definitionsById.get(providerId); + if (!definition) throw new Error(`Unknown provider: ${providerId}`); + if (building.has(providerId)) throw new Error(`Provider ${providerId} depends on itself`); + building.add(providerId); + const models = definition.models?.(context(providerId)) ?? []; + building.delete(providerId); + for (const model of models) { + if (model.provider !== providerId) { + throw new Error(`${providerId} produced ${modelKey(model)} for another provider`); + } + } + stageOne.set(providerId, models); + return models; + } + + for (const definition of definitions) buildModels(definition.id); + + const catalogs: Record = {}; + for (const definition of definitions) { + const ctx = context(definition.id); + const chat: ChatModel[] = []; + const seen = new Set(); + for (const model of stageOne.get(definition.id) ?? []) { + // The first source listing a model wins. + if (seen.has(model.id)) continue; + seen.add(model.id); + chat.push(model); + } + + for (const model of chat) { + applyCompatDefaults(model, definition.compat); + applyDeepSeekCompatRules(model); + applyModelsDevEffort(model, ctx.modelsDevSource(model)); + applyOpenAiFamilyRules(model); + applyClaudeFamilyRules(model); + applyDeepSeekFamilyRules(model); + } + for (const model of chat) definition.finalize?.(model, ctx); + for (const model of chat) { + if (model.compat && Object.keys(model.compat).length === 0) delete model.compat; + applyImageResizeDefaults(model); + } + definition.finalizeAll?.(chat, ctx); + + const image = definition.images?.(ctx) ?? []; + for (const model of image) applyImageResizeDefaults(model); + const classifier = definition.classifiers?.(ctx) ?? []; + catalogs[definition.id] = { + chat: chat.map((model) => ({ ...model, type: "chat" as const })), + image, + classifier, + }; + } + return catalogs; +} diff --git a/packages/ai/catalog/define.ts b/packages/ai/catalog/define.ts new file mode 100644 index 000000000..71517c6e1 --- /dev/null +++ b/packages/ai/catalog/define.ts @@ -0,0 +1,142 @@ +import type { + Api, + ClassifierApi, + ClassifierModel, + ImageApi, + ImageModel, + KnownApi, + Model, + ModelInputLimits, + ThinkingLevelMap, +} from "../src/types.ts"; +import type { CatalogSources } from "./sources/index.ts"; +import { type ModelsDevModel, modelsDevChatModels, modelsDevCost } from "./sources/models-dev.ts"; + +export type ChatModel = Model; +export type ModelCompat = NonNullable["compat"]>; + +/** + * Compat defaults per API. They sit under each model's own compat, so model + * values win. A function receives the model to vary defaults by model. + */ +export type ApiCompatDefaults = { + [TApi in KnownApi]?: ModelCompat | ((model: Model) => ModelCompat); +}; + +/** Passed to every provider hook. */ +export interface ProviderContext { + /** The provider being built. */ + readonly providerId: string; + readonly sources: CatalogSources; + /** Fail instead of degrading when upstream data is unexpected. */ + readonly strict: boolean; + /** Copies of another provider's chat models before shared rules run. */ + providerModels(providerId: string): ChatModel[]; + /** The models.dev entry a model was built from, if it was recorded. */ + modelsDevSource(model: ChatModel): ModelsDevModel | undefined; + /** Record the models.dev entry so shared rules can use its reasoning metadata. */ + recordModelsDevSource(model: ChatModel, source: ModelsDevModel): void; +} + +/** + * One built-in provider's catalog. The build runs, in order: + * 1. `models()`: upstream models plus authored ones, with data corrections. + * 2. `compat` defaults and the shared model-family rules in `families/`. + * 3. `finalize()`: provider-specific behavior that must win over shared rules. + * 4. Shared normalization (image input limits), then `finalizeAll()`. + */ +export interface ProviderDefinition { + id: string; + models?(ctx: ProviderContext): ChatModel[]; + images?(ctx: ProviderContext): ImageModel[]; + classifiers?(ctx: ProviderContext): ClassifierModel[]; + compat?: ApiCompatDefaults; + finalize?(model: ChatModel, ctx: ProviderContext): void; + finalizeAll?(models: ChatModel[], ctx: ProviderContext): void; +} + +export function defineProvider(definition: ProviderDefinition): ProviderDefinition { + return definition; +} + +/** Shallow-merge compat flags into a model. */ +export function mergeCompat(model: Model, compat: ModelCompat): void { + model.compat = { ...model.compat, ...compat } as ModelCompat; +} + +/** Shallow-merge thinking level mappings into a model. Later values win. */ +export function mergeThinking(model: ChatModel, map: ThinkingLevelMap): void { + model.thinkingLevelMap = { ...model.thinkingLevelMap, ...map }; +} + +/** Provider input limits for image-capable models. Limits the model already carries take precedence. */ +export function withInputLimits(model: ChatModel, limits: ModelInputLimits): void { + if (!model.input.includes("image")) return; + model.inputLimits = { + ...limits, + ...model.inputLimits, + images: { ...limits.images, ...model.inputLimits?.images }, + }; +} + +/** Add models only when upstream has not listed them (yet). */ +export function addMissing(models: ChatModel[], additions: readonly ChatModel[]): ChatModel[] { + for (const addition of additions) { + if (!models.some((model) => model.id === addition.id)) models.push(addition); + } + return models; +} + +export interface ModelsDevOptions { + api: TApi; + baseUrl: string; + compat?: ModelCompat; + headers?: Record; + /** Only models models.dev lists on this API protocol (e.g. "responses"). */ + protocol?: string; + /** Skip models models.dev marks deprecated. */ + skipDeprecated?: boolean; + /** Expose models.dev effort levels through the shared rules. Defaults to true. */ + recordReasoning?: boolean; + /** Fallback limits when models.dev has none. Defaults to 4096 each. */ + defaultContextWindow?: number; + defaultMaxTokens?: number; + /** Extra filter. Models without tool support are always skipped. */ + include?(source: ModelsDevModel): boolean; + /** Adjust or replace the mapped model. Return undefined to skip it. */ + map?(model: Model, source: ModelsDevModel): ChatModel | undefined; +} + +/** Map one models.dev provider's tool-capable chat models onto pi models. */ +export function modelsDevModels( + ctx: ProviderContext, + sourceProvider: string, + options: ModelsDevOptions, +): ChatModel[] { + const models: ChatModel[] = []; + for (const source of modelsDevChatModels(ctx.sources.modelsDev, sourceProvider)) { + if (!source.toolCall) continue; + if (options.protocol && !source.apis.some((api) => api.protocol === options.protocol)) continue; + if (options.skipDeprecated && source.status === "deprecated") continue; + if (options.include && !options.include(source)) continue; + const base: Model = { + id: source.id, + name: source.name || source.id, + api: options.api, + provider: ctx.providerId, + baseUrl: options.baseUrl, + reasoning: source.reasoning, + input: source.imageInput ? ["text", "image"] : ["text"], + cost: modelsDevCost(source.cost), + contextWindow: source.contextWindow || options.defaultContextWindow || 4096, + maxTokens: source.maxTokens || options.defaultMaxTokens || 4096, + ...(options.headers ? { headers: { ...options.headers } } : {}), + ...(options.compat ? { compat: { ...options.compat } } : {}), + }; + const model = options.map ? options.map(base, source) : base; + if (!model) continue; + if (options.recordReasoning !== false) ctx.recordModelsDevSource(model, source); + models.push(model); + } + return models; +} diff --git a/packages/ai/catalog/families/anthropic.ts b/packages/ai/catalog/families/anthropic.ts new file mode 100644 index 000000000..f9d6d8a64 --- /dev/null +++ b/packages/ai/catalog/families/anthropic.ts @@ -0,0 +1,134 @@ +// Anthropic Claude models: facts that hold on every host that serves them. +// Ids appear dashed (claude-opus-4-8), dotted (claude-opus-4.8), prefixed +// (anthropic/claude-opus-4.8), and dated (claude-opus-5-20260723). + +import type { Model, ThinkingLevelMap } from "../../src/types.ts"; +import { type ChatModel, mergeCompat, mergeThinking } from "../define.ts"; + +function stripPrefix(modelId: string): string { + return modelId.toLowerCase().replace(/^~?anthropic\//, ""); +} + +/** Models that accept per-message `output_config` effort changes. */ +export function supportsClaudeMidConvoEffort(modelId: string): boolean { + const id = stripPrefix(modelId); + return ( + /^claude-opus-(?:5|5[.-]5)(?:-\d{8})?$/.test(id) || + /^claude-(?:sonnet|haiku)-5[.-]5(?:-\d{8})?$/.test(id) || + /^claude-(?:fable|mythos)-5(?:[.-]1)(?:-\d{8})?$/.test(id) + ); +} + +// Claude models that accept `thinking: {type: "disabled"}` alongside per-message effort. +// Opus 5.5, Sonnet 5.5, Fable 5.1, and Mythos 5.1 reject it. +// https://platform.claude.com/docs/en/build-with-claude/thinking#configuring-thinking +const DISABLED_THINKING_MODELS = new Set(["claude-opus-5", "claude-haiku-5-5", "claude-haiku-5.5"]); + +/** Models that accept system messages and tool changes after the conversation starts. */ +export function supportsClaudeMidConvoSystemMessages(modelId: string): boolean { + return ( + /^claude-opus-(?:4[.-]8|5(?:[.-]5)?)(?:-\d{8})?$/.test(modelId) || + /^claude-(?:sonnet|haiku)-5[.-]5(?:-\d{8})?$/.test(modelId) || + /^claude-(?:fable|mythos)-5(?:[.-]1)?(?:-\d{8})?$/.test(modelId) + ); +} + +/** Enable per-message effort changes. Models that cannot disable thinking with it lose "off". */ +export function enableClaudeMidConvoEffort(model: Model<"anthropic-messages">): void { + mergeCompat(model, { supportsMidConvoEffort: true }); + if (!DISABLED_THINKING_MODELS.has(stripPrefix(model.id).replace(/-\d{8}$/, ""))) { + mergeThinking(model, { off: null }); + } +} + +function idIncludesAny(modelId: string, fragments: readonly string[]): boolean { + return fragments.some((fragment) => modelId.includes(fragment)); +} + +const ADAPTIVE_THINKING = [ + "opus-4-6", + "opus-4.6", + "opus-4-7", + "opus-4.7", + "opus-4-8", + "opus-4.8", + "opus-5", + "opus.5", + "sonnet-4-6", + "sonnet-4.6", + "sonnet-5", + "sonnet.5", + "haiku-5", + "haiku.5", + "fable-5", + "mythos-5", +]; +/** + * OpenRouter's `~anthropic/claude-*-latest` aliases follow the newest model of each family, + * all of which use adaptive thinking. + */ +function isClaudeLatestAlias(modelId: string): boolean { + return /^~anthropic\/claude-[a-z]+-latest$/.test(modelId); +} + +const TEMPERATURE_UNSUPPORTED = [ + "opus-4-7", + "opus-4.7", + "opus-4-8", + "opus-4.8", + "opus-5", + "opus.5", + "sonnet-5-5", + "sonnet-5.5", + "haiku-5-5", + "haiku-5.5", +]; +const MAX_EFFORT_ONLY = ["opus-4-6", "opus-4.6", "sonnet-4-6", "sonnet-4.6"]; +const XHIGH_AND_MAX_EFFORT = [ + "opus-4-7", + "opus-4.7", + "opus-4-8", + "opus-4.8", + "opus-5", + "opus.5", + "sonnet-5", + "sonnet.5", + "haiku-5", + "haiku.5", +]; + +/** + * Adaptive thinking effort levels (per Anthropic adaptive thinking docs): + * "max" is available on all adaptive-thinking Claude models; "xhigh" only on + * Opus 4.7/4.8/5, Sonnet 5, Haiku 5.5, and Fable 5. + */ +export function applyClaudeFamilyRules(model: ChatModel): void { + if (idIncludesAny(model.id, MAX_EFFORT_ONLY)) mergeThinking(model, { max: "max" }); + if (idIncludesAny(model.id, XHIGH_AND_MAX_EFFORT)) mergeThinking(model, { xhigh: "xhigh", max: "max" }); + if (model.id.includes("fable-5")) mergeThinking(model, { off: null, xhigh: "xhigh", max: "max" }); + if (model.api !== "anthropic-messages") return; + const messagesModel = model as Model<"anthropic-messages">; + if (isClaudeLatestAlias(model.id) || idIncludesAny(model.id, ADAPTIVE_THINKING)) { + mergeCompat(messagesModel, { forceAdaptiveThinking: true }); + } + if (idIncludesAny(model.id.toLowerCase(), TEMPERATURE_UNSUPPORTED)) { + mergeCompat(messagesModel, { supportsTemperature: false }); + } +} + +/** + * models.dev may list Opus 5.5, Sonnet 5.5, and Haiku 5.5 before their effort + * metadata is complete. Haiku 5.5 accepts disabled thinking; Opus 5.5 and + * Sonnet 5.5 reject it. + */ +export function claude55ThinkingLevels(modelId: string): ThinkingLevelMap { + return { + ...(modelId.includes("haiku") ? {} : { off: null }), + minimal: null, + low: "low", + medium: "medium", + high: "high", + xhigh: "xhigh", + max: "max", + }; +} diff --git a/packages/ai/catalog/families/deepseek.ts b/packages/ai/catalog/families/deepseek.ts new file mode 100644 index 000000000..630ed51c9 --- /dev/null +++ b/packages/ai/catalog/families/deepseek.ts @@ -0,0 +1,67 @@ +// DeepSeek V4 models on OpenAI-compatible hosts. + +import type { Model, OpenAICompletionsCompat } from "../../src/types.ts"; +import { type ChatModel, mergeCompat, mergeThinking } from "../define.ts"; + +export const DEEPSEEK_COMPAT: OpenAICompletionsCompat = { + requiresReasoningContentOnAssistantMessages: true, + thinkingFormat: "deepseek", +}; + +export const DEEPSEEK_V4_THINKING_LEVEL_MAP = { + minimal: null, + low: null, + medium: null, + high: "high", + max: "max", +} as const; +export const DEEPSEEK_V4_FLASH_THINKING_LEVEL_MAP = { + ...DEEPSEEK_V4_THINKING_LEVEL_MAP, + low: "low", +} as const; +// Azure Foundry rejects DeepSeek's own max effort. +const AZURE_DEEPSEEK_V4_THINKING_LEVEL_MAP = { + minimal: null, + low: "low", + medium: "medium", + high: "high", + xhigh: null, + max: null, +} as const; + +// Qwen Token Plan serves DeepSeek V4 through Qwen's own thinking format. +const QWEN_FORMAT_PROVIDERS = new Set(["qwen-token-plan", "qwen-token-plan-cn", "qwen-token-plan-individual"]); +// These hosts accept native reasoning efforts instead of DeepSeek's thinking toggle. +const NATIVE_EFFORT_PROVIDERS = new Set(["openrouter", "opencode"]); + +function isDeepSeekV4(model: ChatModel): model is Model<"openai-completions"> { + return model.api === "openai-completions" && model.id.includes("deepseek-v4"); +} + +/** DeepSeek V4 needs its reasoning content replayed on every host. */ +export function applyDeepSeekCompatRules(model: ChatModel): void { + if (!isDeepSeekV4(model) || QWEN_FORMAT_PROVIDERS.has(model.provider)) return; + mergeCompat( + model, + NATIVE_EFFORT_PROVIDERS.has(model.provider) + ? { requiresReasoningContentOnAssistantMessages: DEEPSEEK_COMPAT.requiresReasoningContentOnAssistantMessages } + : DEEPSEEK_COMPAT, + ); +} + +/** DeepSeek V4 effort levels when the host did not report its own. */ +export function applyDeepSeekFamilyRules(model: ChatModel): void { + if (!isDeepSeekV4(model) || model.thinkingLevelMap !== undefined) return; + if (model.provider === "openrouter") { + mergeThinking(model, { ...DEEPSEEK_V4_THINKING_LEVEL_MAP, xhigh: "xhigh", max: null }); + } else if (model.provider === "azure") { + mergeThinking(model, AZURE_DEEPSEEK_V4_THINKING_LEVEL_MAP); + } else if ( + (model.provider === "deepseek" || model.provider === "opencode" || model.provider === "opencode-go") && + model.id.includes("deepseek-v4-flash") + ) { + mergeThinking(model, DEEPSEEK_V4_FLASH_THINKING_LEVEL_MAP); + } else { + mergeThinking(model, DEEPSEEK_V4_THINKING_LEVEL_MAP); + } +} diff --git a/packages/ai/catalog/families/google.ts b/packages/ai/catalog/families/google.ts new file mode 100644 index 000000000..8e6bd9936 --- /dev/null +++ b/packages/ai/catalog/families/google.ts @@ -0,0 +1,22 @@ +// Google Gemini and Gemma models. + +import type { ThinkingLevelMap } from "../../src/types.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; +import type { ModelsDevReasoningOption } from "../sources/models-dev.ts"; + +function isGemma4Model(modelId: string): boolean { + return /gemma-?4/.test(modelId.toLowerCase()); +} + +/** Gemini effort levels from models.dev, with a Gemma 4 fallback. */ +export function googleThinkingLevels( + modelId: string, + reasoningOptions: readonly ModelsDevReasoningOption[] = [], +): ThinkingLevelMap | undefined { + const effortMap = getEffortThinkingLevelMap(reasoningOptions); + if (effortMap) return effortMap; + if (isGemma4Model(modelId)) { + return { off: null, minimal: "MINIMAL", low: null, medium: null, high: "HIGH" }; + } + return undefined; +} diff --git a/packages/ai/catalog/families/kimi.ts b/packages/ai/catalog/families/kimi.ts new file mode 100644 index 000000000..01e9f9067 --- /dev/null +++ b/packages/ai/catalog/families/kimi.ts @@ -0,0 +1,38 @@ +// Moonshot Kimi models, shared by Moonshot, Kimi Coding, and gateways. + +import type { ModelCost, OpenAICompletionsCompat } from "../../src/types.ts"; + +/** + * Kimi K3 and its point releases (kimi-k3.1-pro, kimi-k3.1-pro-highspeed) share + * K3's request format. https://api.moonshot.ai/v1/models reports effort levels + * and dynamic tool loading for all of them. + */ +export function isKimiK3FamilyId(modelId: string): boolean { + return /^kimi-k3(?:\.\d+)?(?:-|$)/.test(modelId); +} + +export const KIMI_K3_MAX_TOKENS = 131072; +export const KIMI_K3_COST: ModelCost = { + input: 3, + output: 15, + cacheRead: 0.3, + cacheWrite: 0, +}; + +// Kimi Chat Completions settings shared by Moonshot and Kimi Coding. +export const KIMI_OPENAI_COMPAT: OpenAICompletionsCompat = { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + supportsStrictMode: false, + thinkingFormat: "deepseek", +}; + +// Kimi K3-style models take reasoning_effort and need earlier reasoning sent back. +export const KIMI_K3_OPENAI_COMPAT: OpenAICompletionsCompat = { + ...KIMI_OPENAI_COMPAT, + requiresReasoningContentOnAssistantMessages: true, + thinkingFormat: "openai", + supportsReasoningEffort: true, +}; diff --git a/packages/ai/catalog/families/openai.ts b/packages/ai/catalog/families/openai.ts new file mode 100644 index 000000000..3f3a6fa11 --- /dev/null +++ b/packages/ai/catalog/families/openai.ts @@ -0,0 +1,141 @@ +// OpenAI GPT models: facts that hold on every host that serves them. + +import type { Api, Model, ModelCost } from "../../src/types.ts"; +import { type ChatModel, mergeCompat, mergeThinking } from "../define.ts"; +import { roundCost } from "../sources/models-dev.ts"; + +export const OPENAI_LONG_CONTEXT_INPUT_THRESHOLD = 272000; + +/** OpenAI bills prompts above 272K input tokens at 2x input and 1.5x output rates. */ +export function withOpenAiLongContextPricing(cost: ModelCost): ModelCost { + return { + ...cost, + tiers: [ + { + inputTokensAbove: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, + input: roundCost(cost.input * 2), + output: roundCost(cost.output * 1.5), + cacheRead: roundCost(cost.cacheRead * 2), + cacheWrite: roundCost(cost.cacheWrite * 2), + }, + ], + }; +} + +// Keep current OpenAI prices authoritative until models.dev and passthrough +// catalogs catch up. +// https://developers.openai.com/api/docs/pricing +export const OPENAI_STANDARD_COSTS: Record = { + "gpt-5.6-luna": { input: 0.2, output: 1.2, cacheRead: 0.02, cacheWrite: 0.25 }, + "gpt-5.6-sol": { input: 4, output: 20, cacheRead: 0.4, cacheWrite: 5 }, + "gpt-5.6-terra": { input: 2, output: 12, cacheRead: 0.2, cacheWrite: 2.5 }, + "gpt-6-astra": { input: 10, output: 50, cacheRead: 1, cacheWrite: 12.5 }, + "gpt-6-luna": { input: 0.1, output: 0.5, cacheRead: 0.01, cacheWrite: 0.125 }, + "gpt-6-sol": { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 2.5 }, + "gpt-6.1-sol": { input: 2, output: 10, cacheRead: 0.1, cacheWrite: 2.5 }, + "gpt-6.1-terra": { input: 1, output: 5, cacheRead: 0.05, cacheWrite: 1.25 }, +}; + +/** The standard price of an OpenAI model with long-context pricing. */ +export function openAiStandardCost(id: string): ModelCost { + return withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS[id]); +} + +// GPT-5.5 and older put tools loaded through tool search or additional_tools in a namespace named +// after the tool and reject replayed calls without it, so history from other models fails there +// (#11415). Like Codex, dynamic tool loading therefore starts with GPT-5.6, which does not do this. +/** Models that accept tool search and `additional_tools`. */ +export const OPENAI_TOOL_SEARCH_MODEL_IDS = new Set([ + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-5.6-luna", + "gpt-6-astra", + "gpt-6-sol", + "gpt-6-luna", + "gpt-6.1-sol", + "gpt-6.1-terra", +]); +/** Models that accept mid-conversation developer messages. */ +export const OPENAI_MID_CONVO_MODEL_IDS = new Set([ + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-pro", + "gpt-5.5", + ...OPENAI_TOOL_SEARCH_MODEL_IDS, +]); + +const RESPONSES_APIS = new Set(["openai-responses", "azure-openai-responses", "openai-codex-responses"]); + +function isResponsesApi(model: ChatModel): boolean { + return RESPONSES_APIS.has(model.api); +} + +export function supportsOpenAiXhigh(modelId: string): boolean { + return ( + modelId.includes("gpt-5.2") || + modelId.includes("gpt-5.3") || + modelId.includes("gpt-5.4") || + modelId.includes("gpt-5.5") || + modelId.includes("gpt-5.6") || + modelId.includes("gpt-6") + ); +} + +const GPT_6_MODEL_IDS = new Set(["gpt-6-astra", "gpt-6-sol", "gpt-6-luna", "gpt-6.1-sol", "gpt-6.1-terra"]); + +/** Thinking levels of GPT models on any host. */ +export function applyOpenAiFamilyRules(model: ChatModel): void { + if ((model.api === "openai-responses" || model.api === "azure-openai-responses") && model.id.startsWith("gpt-5")) { + mergeThinking(model, { off: null }); + } + if (GPT_6_MODEL_IDS.has(model.id) && isResponsesApi(model)) { + mergeThinking(model, { + // GPT-6 Astra and GPT-6.1 Sol reject reasoning.effort "none"; GPT-6.1 Terra accepts it. All reject + // "minimal" (verified on Codex, October 2026). + off: model.id === "gpt-6-astra" || model.id === "gpt-6.1-sol" ? null : "none", + minimal: null, + low: "low", + medium: "medium", + high: "high", + xhigh: "xhigh", + max: "max", + }); + } + if (supportsOpenAiXhigh(model.id)) { + mergeThinking(model, { xhigh: "xhigh" }); + } + if ( + (model.id.includes("gpt-5.6") || model.id.includes("gpt-6")) && + (isResponsesApi(model) || model.api === "openai-completions") + ) { + mergeThinking(model, { max: "max" }); + } + if (model.id.endsWith("gpt-5.5-pro")) { + mergeThinking(model, { off: null, minimal: null, low: null }); + } +} + +/** + * OpenAI custom grammar tools on Responses endpoints. OpenAI rejects + * `type: "custom"` tools for pre-GPT-5 models (gpt-4.x, gpt-4o, o-series). + */ +export function enableOpenAiGrammarTools(model: ChatModel): void { + if (!isResponsesApi(model)) return; + const match = /^gpt-(\d+)/.exec(model.id); + if (!match || Number(match[1]) < 5) return; + mergeCompat(model as Model<"openai-responses">, { supportsOpenAIGrammarTools: true }); +} + +/** + * Newer OpenAI Responses models accept developer messages after the + * conversation has started. Proxies that pass them through also pass + * `additional_tools` items through unchanged, for the models that accept them. + */ +export function enableOpenAiMidConvoSystemMessages(model: ChatModel, options: { proxy: boolean }): void { + if (model.api !== "openai-responses" && model.api !== "openai-codex-responses") return; + if (!OPENAI_MID_CONVO_MODEL_IDS.has(model.id)) return; + mergeCompat(model as Model<"openai-responses">, { + supportsMidConvoSystemMessages: true, + ...(options.proxy && OPENAI_TOOL_SEARCH_MODEL_IDS.has(model.id) ? { supportsAdditionalTools: true } : {}), + }); +} diff --git a/packages/ai/catalog/generate.ts b/packages/ai/catalog/generate.ts new file mode 100644 index 000000000..718159674 --- /dev/null +++ b/packages/ai/catalog/generate.ts @@ -0,0 +1,308 @@ +#!/usr/bin/env node + +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + createModelDataManifest, + groupProviderModelData, + MODEL_DATA_MANIFEST_FILE, + type ModelDataStructure, + readModelDataProviderIds, + validateGeneratedModelData, + validateModelDataDirectory, +} from "../scripts/model-data.ts"; +import type { + AnyModel, + Api, + ClassifierApi, + ClassifierModel, + ImageApi, + ImageModel, + Model, + ModelType, +} from "../src/types.ts"; +import { buildCatalog, type ProviderCatalog } from "./build.ts"; +import { PROVIDERS } from "./providers/index.ts"; +import { loadCatalogSources } from "./sources/index.ts"; + +/** A catalog entry with its model type spelled out. */ +type CatalogEntry = AnyModel & { type: ModelType }; + +const packageRoot = join(dirname(fileURLToPath(import.meta.url)), ".."); + +interface GeneratorOptions { + strict: boolean; + dataOnly: boolean; + jsonOnly: boolean; + jsonOutputDir: string | undefined; + pretty: boolean; +} + +function readGeneratorOptions(args: string[]): GeneratorOptions { + let strict = false; + let dataOnly = false; + let jsonOnly = false; + let jsonOutputDir: string | undefined; + let pretty = false; + + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (arg === "--strict") { + strict = true; + continue; + } + if (arg === "--data-only") { + dataOnly = true; + continue; + } + if (arg === "--json-only") { + jsonOnly = true; + continue; + } + if (arg === "--pretty") { + pretty = true; + continue; + } + if (arg === "--json-output") { + const value = args[++index]; + if (!value) throw new Error("--json-output requires a directory"); + jsonOutputDir = resolve(value); + continue; + } + throw new Error(`Unknown argument: ${arg}`); + } + + if (jsonOnly && !jsonOutputDir) throw new Error("--json-only requires --json-output"); + if (dataOnly && (jsonOnly || jsonOutputDir)) + throw new Error("--data-only cannot be combined with JSON catalog output"); + return { strict, dataOnly, jsonOnly, jsonOutputDir, pretty }; +} + +/** Key models by id, keeping the first entry per id, sorted by id. */ +function sortedById(models: readonly T[]): [string, T][] { + const byId = new Map(); + for (const model of models) if (!byId.has(model.id)) byId.set(model.id, model); + return [...byId].sort(([left], [right]) => left.localeCompare(right)); +} + +async function generateModels(generatorOptions: GeneratorOptions): Promise { + const sources = await loadCatalogSources(generatorOptions.strict); + const catalogs = buildCatalog(PROVIDERS, sources, generatorOptions.strict); + // Providers without any models are left out, as if they did not exist. + const providers: Record = Object.fromEntries( + Object.entries(catalogs).filter( + ([, catalog]) => catalog.chat.length + catalog.image.length + catalog.classifier.length > 0, + ), + ); + + const sortedProviderIds = Object.keys(providers).sort(); + const jsonChatProviders: Record & { type: "chat" }>> = {}; + const jsonImageProviders: Record>> = {}; + const jsonClassifierProviders: Record>> = {}; + const jsonAllProviders: Record = {}; + for (const providerId of sortedProviderIds) { + jsonChatProviders[providerId] = Object.fromEntries(sortedById(providers[providerId].chat)); + jsonImageProviders[providerId] = Object.fromEntries(sortedById(providers[providerId].image)); + jsonClassifierProviders[providerId] = Object.fromEntries(sortedById(providers[providerId].classifier)); + jsonAllProviders[providerId] = [ + ...Object.values(jsonChatProviders[providerId]), + ...Object.values(jsonImageProviders[providerId]), + ...Object.values(jsonClassifierProviders[providerId]), + ]; + } + + const serializeJson = (value: unknown) => + `${JSON.stringify(value, null, generatorOptions.pretty ? 2 : undefined)}\n`; + const writeJson = (path: string, value: unknown) => writeFileSync(path, serializeJson(value)); + const generatedDataProviderIds = generatorOptions.dataOnly + ? readModelDataProviderIds(packageRoot) + : sortedProviderIds; + const missingProviderIds = generatedDataProviderIds.filter((providerId) => !jsonAllProviders[providerId]); + if (missingProviderIds.length > 0) { + throw new Error(`Cannot hydrate missing providers: ${missingProviderIds.join(", ")}`); + } + + // Only the ignored internal data is grouped by API for type derivation. + const generatedDataProviders: Record>> = {}; + const modelDataStructure: ModelDataStructure = {}; + for (const providerId of generatedDataProviderIds) { + const { groups, structure } = groupProviderModelData(providerId, jsonAllProviders[providerId]); + generatedDataProviders[providerId] = groups; + modelDataStructure[providerId] = structure; + } + + const generatedAt = new Date().toISOString(); + + if (!generatorOptions.jsonOnly) { + // Stage and validate all provider values before replacing the current generated data. + const providersDir = join(packageRoot, "src/providers"); + const dataDir = join(providersDir, "data"); + const stagingRoot = mkdtempSync(join(providersDir, ".model-generation-")); + const stagedDataDir = join(stagingRoot, "data"); + const previousDataDir = join(stagingRoot, "previous-data"); + let restoreGeneratedCatalog: (() => void) | undefined; + try { + mkdirSync(stagedDataDir, { recursive: true }); + const fileContents: Record = {}; + for (const providerId of generatedDataProviderIds) { + const filename = `${providerId}.json`; + const content = serializeJson(generatedDataProviders[providerId]); + fileContents[filename] = content; + writeFileSync(join(stagedDataDir, filename), content); + } + writeJson( + join(stagedDataDir, MODEL_DATA_MANIFEST_FILE), + createModelDataManifest(modelDataStructure, fileContents, generatedAt), + ); + validateModelDataDirectory(modelDataStructure, stagedDataDir); + + if (!generatorOptions.dataOnly) { + const previousShardContents = new Map( + readdirSync(providersDir) + .filter((entry) => entry.endsWith(".models.ts")) + .map((entry) => [entry, readFileSync(join(providersDir, entry), "utf8")] as const), + ); + const aggregatorPath = join(packageRoot, "src/models.generated.ts"); + const previousAggregator = readFileSync(aggregatorPath, "utf8"); + restoreGeneratedCatalog = () => { + for (const entry of readdirSync(providersDir)) { + if (entry.endsWith(".models.ts")) rmSync(join(providersDir, entry)); + } + for (const [entry, content] of previousShardContents) { + writeFileSync(join(providersDir, entry), content); + } + writeFileSync(aggregatorPath, previousAggregator); + }; + + const generatedHeader = `// This file is auto-generated by catalog/generate.ts +// Do not edit manually - run 'npm run generate-models' to update + +`; + const catalogConstName = (providerId: string) => + `${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_MODELS`; + const imageCatalogConstName = (providerId: string) => + `${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_IMAGE_MODELS`; + const classifierCatalogConstName = (providerId: string) => + `${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_CLASSIFIER_MODELS`; + const generatedShardFiles = new Set(); + for (const providerId of sortedProviderIds) { + let output = generatedHeader; + output += `import values from "./data/${providerId}.json" with { type: "json" };\n`; + output += `import { flattenChatModelCatalog, flattenClassifierModelCatalog, flattenImageModelCatalog, type ChatModelCatalog, type ClassifierModelCatalog, type ImageModelCatalog } from "../model-catalog.ts";\n\n`; + output += `export const ${catalogConstName(providerId)}: ChatModelCatalog =\n`; + output += `\tflattenChatModelCatalog(${JSON.stringify(providerId)}, values);\n\n`; + output += `export const ${imageCatalogConstName(providerId)}: ImageModelCatalog =\n`; + output += `\tflattenImageModelCatalog(${JSON.stringify(providerId)}, values);\n\n`; + output += `export const ${classifierCatalogConstName(providerId)}: ClassifierModelCatalog =\n`; + output += `\tflattenClassifierModelCatalog(${JSON.stringify(providerId)}, values);\n`; + const filename = `${providerId}.models.ts`; + generatedShardFiles.add(filename); + writeFileSync(join(providersDir, filename), output); + } + for (const entry of readdirSync(providersDir)) { + if (entry.endsWith(".models.ts") && !generatedShardFiles.has(entry)) rmSync(join(providersDir, entry)); + } + + let output = generatedHeader; + for (const providerId of sortedProviderIds) { + output += `import { ${classifierCatalogConstName(providerId)}, ${imageCatalogConstName(providerId)}, ${catalogConstName(providerId)} } from "./providers/${providerId}.models.ts";\n`; + } + output += `\nexport const MODELS: {\n`; + for (const providerId of sortedProviderIds) { + output += `\treadonly ${JSON.stringify(providerId)}: typeof ${catalogConstName(providerId)};\n`; + } + output += `} = {\n`; + for (const providerId of sortedProviderIds) { + output += `\t${JSON.stringify(providerId)}: ${catalogConstName(providerId)},\n`; + } + output += `};\n\nexport const IMAGE_MODELS: {\n`; + for (const providerId of sortedProviderIds) { + output += `\treadonly ${JSON.stringify(providerId)}: typeof ${imageCatalogConstName(providerId)};\n`; + } + output += `} = {\n`; + for (const providerId of sortedProviderIds) { + output += `\t${JSON.stringify(providerId)}: ${imageCatalogConstName(providerId)},\n`; + } + output += `};\n\nexport const CLASSIFIER_MODELS: {\n`; + for (const providerId of sortedProviderIds) { + output += `\treadonly ${JSON.stringify(providerId)}: typeof ${classifierCatalogConstName(providerId)};\n`; + } + output += `} = {\n`; + for (const providerId of sortedProviderIds) { + output += `\t${JSON.stringify(providerId)}: ${classifierCatalogConstName(providerId)},\n`; + } + output += `};\n`; + writeFileSync(aggregatorPath, output); + console.log("Generated provider catalogs and src/models.generated.ts"); + } + + const hadPreviousData = existsSync(dataDir); + if (hadPreviousData) renameSync(dataDir, previousDataDir); + try { + renameSync(stagedDataDir, dataDir); + validateGeneratedModelData(packageRoot); + } catch (error) { + rmSync(dataDir, { recursive: true, force: true }); + if (hadPreviousData && existsSync(previousDataDir)) renameSync(previousDataDir, dataDir); + throw error; + } + restoreGeneratedCatalog = undefined; + console.log( + generatorOptions.dataOnly + ? "Hydrated JSON model values under src/providers/data/" + : "Generated JSON model values under src/providers/data/", + ); + } catch (error) { + restoreGeneratedCatalog?.(); + throw error; + } finally { + rmSync(stagingRoot, { recursive: true, force: true }); + } + } + + if (generatorOptions.jsonOutputDir) { + // `models.json` and `providers/{id}.json` retain the legacy keyed chat catalog. + // The `.all` variants are arrays so the same upstream id can appear once per type. + const providerOutputDir = join(generatorOptions.jsonOutputDir, "providers"); + rmSync(generatorOptions.jsonOutputDir, { recursive: true, force: true }); + mkdirSync(providerOutputDir, { recursive: true }); + writeJson(join(generatorOptions.jsonOutputDir, "models.json"), jsonChatProviders); + writeJson(join(generatorOptions.jsonOutputDir, "models.all.json"), jsonAllProviders); + writeJson(join(generatorOptions.jsonOutputDir, "providers.json"), sortedProviderIds); + for (const providerId of sortedProviderIds) { + writeJson(join(providerOutputDir, `${providerId}.json`), jsonChatProviders[providerId]); + writeJson(join(providerOutputDir, `${providerId}.all.json`), jsonAllProviders[providerId]); + } + console.log(`Generated JSON model catalog under ${generatorOptions.jsonOutputDir}`); + } + + // Print statistics + const chatModels = Object.values(providers).flatMap((catalog) => catalog.chat); + const totalModels = chatModels.length; + const reasoningModels = chatModels.filter((model) => model.reasoning).length; + + console.log(`\nModel Statistics:`); + console.log(` Total tool-capable models: ${totalModels}`); + console.log(` Reasoning-capable models: ${reasoningModels}`); + + for (const [provider, models] of Object.entries(providers)) { + console.log( + ` ${provider}: ${models.chat.length} chat models, ${models.image.length} image models, ${models.classifier.length} classifier models`, + ); + } +} + +generateModels(readGeneratorOptions(process.argv.slice(2))).catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/packages/ai/catalog/providers/amazon-bedrock.ts b/packages/ai/catalog/providers/amazon-bedrock.ts new file mode 100644 index 000000000..28bc0c71c --- /dev/null +++ b/packages/ai/catalog/providers/amazon-bedrock.ts @@ -0,0 +1,32 @@ +import { defineProvider, modelsDevModels, withInputLimits } from "../define.ts"; + +// Opus 5 is only callable through cross-region inference profiles. +const INFERENCE_PROFILE_ONLY_MODEL_IDS = new Set(["anthropic.claude-opus-5"]); + +function bedrockBaseUrl(modelId: string): string { + return modelId.startsWith("eu.") + ? "https://bedrock-runtime.eu-central-1.amazonaws.com" + : "https://bedrock-runtime.us-east-1.amazonaws.com"; +} + +export default defineProvider({ + id: "amazon-bedrock", + models: (ctx) => + modelsDevModels(ctx, "amazon-bedrock", { + api: "bedrock-converse-stream", + baseUrl: "", + include: (source) => + !INFERENCE_PROFILE_ONLY_MODEL_IDS.has(source.id) && + // Jamba does not support tool use in streaming mode. + !source.id.startsWith("ai21.jamba") && + // Mistral 7B Instruct does not support system messages. + !source.id.startsWith("mistral.mistral-7b-instruct-v0"), + // Includes models.dev pricing tiers, e.g. the long-context tier for OpenAI models (#10326). + map: (model, source) => ({ + ...model, + baseUrl: bedrockBaseUrl(model.id), + ...(source.structuredOutput ? { compat: { supportsStrictMode: true } } : {}), + }), + }), + finalize: (model) => withInputLimits(model, { images: { maxPerMessage: 20 } }), +}); diff --git a/packages/ai/catalog/providers/ant-ling.ts b/packages/ai/catalog/providers/ant-ling.ts new file mode 100644 index 000000000..b9cf58cc2 --- /dev/null +++ b/packages/ai/catalog/providers/ant-ling.ts @@ -0,0 +1,52 @@ +import type { Model, OpenAICompletionsCompat } from "../../src/types.ts"; +import { defineProvider, mergeThinking } from "../define.ts"; + +const COMPAT: OpenAICompletionsCompat = { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + supportsLongCacheRetention: false, +}; + +function model(id: string, name: string, reasoning: boolean, cost: Model<"openai-completions">["cost"]) { + return { + id, + name, + api: "openai-completions", + baseUrl: "https://api.ant-ling.com/v1", + provider: "ant-ling", + reasoning, + input: ["text"], + cost, + contextWindow: 262144, + maxTokens: 65536, + compat: reasoning ? { ...COMPAT, thinkingFormat: "ant-ling" } : COMPAT, + } satisfies Model<"openai-completions">; +} + +export default defineProvider({ + id: "ant-ling", + models: () => [ + model("Ling-2.6-flash", "Ling 2.6 Flash", false, { input: 0.01, output: 0.02, cacheRead: 0, cacheWrite: 0 }), + model("Ling-2.6-1T", "Ling 2.6 1T", false, { input: 0.06, output: 0.25, cacheRead: 0, cacheWrite: 0 }), + model("Ring-2.6-1T", "Ring 2.6 1T", true, { input: 0.06, output: 0.25, cacheRead: 0, cacheWrite: 0 }), + ], + compat: { + "openai-completions": { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + thinkingFormat: "ant-ling", + supportsStrictMode: true, + supportsLongCacheRetention: false, + }, + }, + finalize: (model) => { + // Ring reasons by default. Only high/xhigh have documented explicit effort controls. + if (model.reasoning) { + mergeThinking(model, { off: null, minimal: null, low: null, medium: null, high: "high", xhigh: "xhigh" }); + } + }, +}); diff --git a/packages/ai/catalog/providers/anthropic.ts b/packages/ai/catalog/providers/anthropic.ts new file mode 100644 index 000000000..5b97aff2d --- /dev/null +++ b/packages/ai/catalog/providers/anthropic.ts @@ -0,0 +1,66 @@ +import type { Model } from "../../src/types.ts"; +import { defineProvider, mergeCompat, mergeThinking, modelsDevModels, withInputLimits } from "../define.ts"; +import { + claude55ThinkingLevels, + enableClaudeMidConvoEffort, + supportsClaudeMidConvoEffort, + supportsClaudeMidConvoSystemMessages, +} from "../families/anthropic.ts"; + +// Models the API may fall back to, with the price the caller is billed. +const ALLOWED_FALLBACK_MODELS: Record = { + "claude-fable-5": ["claude-opus-4-8", "claude-opus-5"], + "claude-opus-5": ["claude-opus-4-8"], +}; + +const MODELS_WITH_1M_CONTEXT = new Set(["claude-opus-4-6", "claude-sonnet-4-6"]); +const MODELS_55 = new Set(["claude-opus-5-5", "claude-sonnet-5-5", "claude-haiku-5-5"]); + +export default defineProvider({ + id: "anthropic", + models: (ctx) => + modelsDevModels(ctx, "anthropic", { + api: "anthropic-messages", + baseUrl: "https://api.anthropic.com", + map: (model) => { + if (MODELS_55.has(model.id)) mergeThinking(model, claude55ThinkingLevels(model.id)); + if (MODELS_WITH_1M_CONTEXT.has(model.id)) model.contextWindow = 1000000; + return model; + }, + }), + finalize: (model) => { + if (model.api !== "anthropic-messages") return; + const messagesModel = model as Model<"anthropic-messages">; + if (supportsClaudeMidConvoEffort(model.id)) enableClaudeMidConvoEffort(messagesModel); + if (supportsClaudeMidConvoSystemMessages(model.id)) { + mergeCompat(messagesModel, { supportsMidConvoSystemMessages: true, supportsMidConvoToolChanges: true }); + } + mergeCompat(messagesModel, { supportsStrictTools: true }); + // Ephemeral entries have a hard five-minute lifetime; `ttl: "1h"` extends + // it to one hour. Only direct Anthropic is annotated so cache warming does + // not assume equivalent behavior through proxies. + // https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching + model.promptCache = { short: 300, long: 3600 }; + withInputLimits(model, { + maxRequestBytes: 32 * 1024 * 1024, + images: { maxPerRequest: model.contextWindow === 200000 ? 100 : 600 }, + }); + }, + finalizeAll: (models) => { + const modelsById = new Map(models.map((model) => [model.id, model as Model<"anthropic-messages">])); + for (const [modelId, fallbackModelIds] of Object.entries(ALLOWED_FALLBACK_MODELS)) { + const model = modelsById.get(modelId); + if (!model) continue; + const compatibleFallbackModelIds = model.compat?.supportsMidConvoEffort + ? fallbackModelIds.filter(supportsClaudeMidConvoEffort) + : fallbackModelIds; + const allowedFallbackModels = compatibleFallbackModelIds.flatMap((fallbackModelId) => { + const fallbackModel = modelsById.get(fallbackModelId); + return fallbackModel + ? [{ provider: fallbackModel.provider, model: fallbackModel.id, cost: fallbackModel.cost }] + : []; + }); + if (allowedFallbackModels.length > 0) mergeCompat(model, { allowedFallbackModels }); + } + }, +}); diff --git a/packages/ai/catalog/providers/azure.ts b/packages/ai/catalog/providers/azure.ts new file mode 100644 index 000000000..94bd77a65 --- /dev/null +++ b/packages/ai/catalog/providers/azure.ts @@ -0,0 +1,61 @@ +import type { Model, ModelCost } from "../../src/types.ts"; +import { type ChatModel, defineProvider, mergeCompat } from "../define.ts"; +import { enableOpenAiGrammarTools } from "../families/openai.ts"; + +// Azure Foundry deploys these with larger context windows than OpenAI's own short-tier defaults. +// See models-sold-directly-by-azure docs. +const CONTEXT_WINDOW_OVERRIDES: Record = { + "gpt-5.4": 1050000, + "gpt-5.5": 1050000, + "gpt-5.6-luna": 1050000, + "gpt-5.6-sol": 1050000, + "gpt-5.6-terra": 1050000, +}; + +// Azure resells DeepSeek at its own rates. US data zone, checked 2026-09-16. +// https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/ +const DEEPSEEK_V4_PRO_COST: ModelCost = { input: 1.925, output: 3.828, cacheRead: 0.165, cacheWrite: 0 }; + +/** OpenAI models as Azure deploys them. Azure bills without OpenAI's long-context tier. */ +function openAiModels(models: ChatModel[]): ChatModel[] { + return models + .filter((model) => model.api === "openai-responses") + .map((model) => ({ + ...model, + api: "azure-openai-responses", + provider: "azure", + baseUrl: "", + cost: { + input: model.cost.input, + output: model.cost.output, + cacheRead: model.cost.cacheRead, + cacheWrite: model.cost.cacheWrite, + }, + contextWindow: CONTEXT_WINDOW_OVERRIDES[model.id] ?? model.contextWindow, + })); +} + +export default defineProvider({ + id: "azure", + models: (ctx) => [ + ...openAiModels(ctx.providerModels("openai")), + ...ctx + .providerModels("deepseek") + .filter((model) => model.id === "deepseek-v4-pro") + .map((model) => ({ ...model, provider: "azure", baseUrl: "", cost: DEEPSEEK_V4_PRO_COST })), + ], + compat: { "openai-completions": { supportsStrictMode: true } }, + finalize: (model) => { + enableOpenAiGrammarTools(model); + if (model.api === "openai-completions") { + // Azure 400s on DeepSeek's `thinking` field and on every prompt cache parameter, discards a + // `developer` system message unbilled once reasoning_effort is set, and honours mid-convo ones (#9645). + mergeCompat(model as Model<"openai-completions">, { + supportsDeveloperRole: false, + supportsMidConvoSystemMessages: true, + thinkingFormat: "openai", + supportsLongCacheRetention: false, + }); + } + }, +}); diff --git a/packages/ai/catalog/providers/baseten.ts b/packages/ai/catalog/providers/baseten.ts new file mode 100644 index 000000000..4aaaaca13 --- /dev/null +++ b/packages/ai/catalog/providers/baseten.ts @@ -0,0 +1,89 @@ +import type { OpenAICompletionsCompat, ThinkingLevelMap } from "../../src/types.ts"; +import { defineProvider, modelsDevModels } from "../define.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; + +const BASE_COMPAT: OpenAICompletionsCompat = { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + supportsUsageInStreaming: true, + maxTokensField: "max_tokens", + supportsStrictMode: true, + // Baseten automatic prompt caching needs session affinity so related + // requests land on the same replica. See: + // https://docs.baseten.co/inference/model-apis/pricing-and-limits + sendSessionAffinityHeaders: true, + supportsLongCacheRetention: false, +}; +const REASONING_EFFORT_COMPAT: OpenAICompletionsCompat = { + ...BASE_COMPAT, + supportsReasoningEffort: true, + thinkingFormat: "openai", +}; +const TOGGLE_REASONING_COMPAT: OpenAICompletionsCompat = { + ...BASE_COMPAT, + thinkingFormat: "baseten", + chatTemplateArgs: { enable_thinking: { $var: "thinking.enabled" } }, +}; +const TOGGLE_REASONING_EFFORT_COMPAT: OpenAICompletionsCompat = { + ...REASONING_EFFORT_COMPAT, + thinkingFormat: "baseten", + chatTemplateArgs: { enable_thinking: { $var: "thinking.enabled" } }, +}; +const TOGGLE_LEVELS: ThinkingLevelMap = { + off: "off", + minimal: null, + low: null, + medium: null, + high: "high", + xhigh: null, + max: null, +}; +const GLM_52_LEVELS: ThinkingLevelMap = { + off: "none", + minimal: null, + low: null, + medium: null, + high: "high", + xhigh: null, + max: "max", +}; +const GLM_52_MODEL_IDS = new Set(["zai-org/GLM-5.2", "zai-org/GLM-5.2-Fast"]); + +export default defineProvider({ + id: "baseten", + models: (ctx) => + modelsDevModels(ctx, "baseten", { + api: "openai-completions", + baseUrl: "https://inference.baseten.co/v1", + skipDeprecated: true, + recordReasoning: false, + map: (model, source) => { + const options = source.reasoningOptions ?? []; + const isGlm52 = GLM_52_MODEL_IDS.has(model.id); + const supportsToggle = isGlm52 || options.some((option) => option.type === "toggle"); + const supportsEffort = isGlm52 || options.some((option) => option.type === "effort"); + const compat = + supportsToggle && supportsEffort + ? TOGGLE_REASONING_EFFORT_COMPAT + : supportsToggle + ? TOGGLE_REASONING_COMPAT + : supportsEffort + ? REASONING_EFFORT_COMPAT + : BASE_COMPAT; + const thinkingLevelMap = isGlm52 + ? GLM_52_LEVELS + : supportsToggle + ? TOGGLE_LEVELS + : getEffortThinkingLevelMap(options); + return { + ...model, + ...(thinkingLevelMap ? { thinkingLevelMap } : {}), + // Baseten's GLM-5.2 endpoints are text-only despite models.dev reporting image input. + input: !isGlm52 && source.imageInput ? ["text", "image"] : ["text"], + compat, + }; + }, + }), + compat: { "openai-completions": { supportsStrictMode: true } }, +}); diff --git a/packages/ai/catalog/providers/cerebras.ts b/packages/ai/catalog/providers/cerebras.ts new file mode 100644 index 000000000..18340cbd7 --- /dev/null +++ b/packages/ai/catalog/providers/cerebras.ts @@ -0,0 +1,8 @@ +import { defineProvider, modelsDevModels } from "../define.ts"; + +export default defineProvider({ + id: "cerebras", + models: (ctx) => + modelsDevModels(ctx, "cerebras", { api: "openai-completions", baseUrl: "https://api.cerebras.ai/v1" }), + compat: { "openai-completions": { supportsStore: false, supportsDeveloperRole: false } }, +}); diff --git a/packages/ai/catalog/providers/cloudflare-ai-gateway.ts b/packages/ai/catalog/providers/cloudflare-ai-gateway.ts new file mode 100644 index 000000000..ed4fb95d5 --- /dev/null +++ b/packages/ai/catalog/providers/cloudflare-ai-gateway.ts @@ -0,0 +1,84 @@ +import { + CLOUDFLARE_AI_GATEWAY_ANTHROPIC_BASE_URL, + CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL, + CLOUDFLARE_AI_GATEWAY_OPENAI_BASE_URL, +} from "../../src/api/cloudflare.ts"; +import type { Model } from "../../src/types.ts"; +import { type ChatModel, defineProvider, mergeCompat, modelsDevModels } from "../define.ts"; +import { enableOpenAiGrammarTools, OPENAI_STANDARD_COSTS, withOpenAiLongContextPricing } from "../families/openai.ts"; + +/** models.dev lists gateway models as `/`; route each upstream to its passthrough. */ +function routeGatewayModel(model: ChatModel): ChatModel | undefined { + const slash = model.id.indexOf("/"); + if (slash === -1) return undefined; + const upstream = model.id.slice(0, slash); + const nativeId = model.id.slice(slash + 1); + if (upstream === "openai") { + // Cloudflare AI Gateway passes OpenAI usage through at OpenAI list prices. + const standardCost = OPENAI_STANDARD_COSTS[nativeId]; + return { + ...model, + id: nativeId, + api: "openai-responses", + baseUrl: CLOUDFLARE_AI_GATEWAY_OPENAI_BASE_URL, + ...(standardCost ? { cost: withOpenAiLongContextPricing(standardCost) } : {}), + }; + } + if (upstream === "anthropic") { + // The /anthropic passthrough forwards the model ID to Anthropic unchanged. + // models.dev lists dotted versions (claude-opus-5.5), but Anthropic only + // accepts dashed IDs (claude-opus-5-5). + return { + ...model, + id: nativeId.replaceAll(".", "-"), + api: "anthropic-messages", + baseUrl: CLOUDFLARE_AI_GATEWAY_ANTHROPIC_BASE_URL, + // Gateway passthroughs forward session affinity headers to upstreams that + // use them for cache/routing affinity. + compat: { sendSessionAffinityHeaders: true }, + }; + } + if (upstream === "workers-ai") { + return { ...model, compat: { sendSessionAffinityHeaders: true } }; + } + return undefined; +} + +export default defineProvider({ + id: "cloudflare-ai-gateway", + models: (ctx) => { + const models = modelsDevModels(ctx, "cloudflare-ai-gateway", { + api: "openai-completions", + baseUrl: CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL, + map: routeGatewayModel, + }); + // The gateway proxies Workers AI through its OpenAI-compatible /compat endpoint, + // but models.dev may omit or intermittently drop those `workers-ai/*` entries + // from the AI Gateway catalog. Mirror the Workers AI catalog under the documented + // prefix so the gateway keeps its OpenAI-compatible models stable. + const ids = new Set(models.map((model) => model.id)); + const mirrored = modelsDevModels(ctx, "cloudflare-workers-ai", { + api: "openai-completions", + baseUrl: CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL, + compat: { sendSessionAffinityHeaders: true }, + include: (source) => !ids.has(`workers-ai/${source.id}`), + map: (model) => ({ ...model, id: `workers-ai/${model.id}` }), + }); + return [...models, ...mirrored]; + }, + compat: { + "openai-completions": { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + supportsLongCacheRetention: false, + }, + }, + finalize: (model) => { + if (model.api === "openai-responses") { + mergeCompat(model as Model<"openai-responses">, { supportsStrictMode: true }); + } + enableOpenAiGrammarTools(model); + }, +}); diff --git a/packages/ai/catalog/providers/cloudflare-workers-ai.ts b/packages/ai/catalog/providers/cloudflare-workers-ai.ts new file mode 100644 index 000000000..6274661b6 --- /dev/null +++ b/packages/ai/catalog/providers/cloudflare-workers-ai.ts @@ -0,0 +1,61 @@ +import { CLOUDFLARE_WORKERS_AI_BASE_URL, CLOUDFLARE_WORKERS_AI_REST_BASE_URL } from "../../src/api/cloudflare.ts"; +import type { ClassifierModel } from "../../src/types.ts"; +import { defineProvider, modelsDevModels } from "../define.ts"; + +function classifier( + id: string, + name: string, + input: number, + contextWindow: number, + imageInput: "images" | "assets", +): ClassifierModel<"cloudflare-workers-ai-system-one"> { + return { + type: "classifier", + id, + name, + api: "cloudflare-workers-ai-system-one", + provider: "cloudflare-workers-ai", + baseUrl: CLOUDFLARE_WORKERS_AI_REST_BASE_URL, + input: ["text", "image"], + cost: { input, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow, + compat: { imageInput }, + }; +} + +// Workers AI has no unauthenticated catalog, so its System One models are listed here from +// Cloudflare's model pages. +const CLASSIFIER_MODELS = [ + // Cloudflare-hosted Clef decision models. They take up to four PNG, JPEG, or WebP images as data URLs. + // Clef Omni also takes audio and video, which classifier contexts do not carry. + // Pricing and context: https://developers.cloudflare.com/workers-ai/models/clef/, + // https://developers.cloudflare.com/workers-ai/models/clef-flash/, + // and https://developers.cloudflare.com/workers-ai/models/clef-omni/ + classifier("@cf/cloudflare/clef", "Clef", 0.24, 65536, "images"), + classifier("@cf/cloudflare/clef-flash", "Clef Flash", 0.038, 24576, "images"), + classifier("@cf/cloudflare/clef-omni", "Clef Omni", 0.15, 64000, "images"), + // TypeSafe's Jev models take PNG and JPEG images as System One assets. + // https://developers.cloudflare.com/ai/models/typesafe/jev/ (jev-latest) + // and https://developers.cloudflare.com/ai/models/typesafe/jev-1.14.0/ + classifier("typesafe/jev", "Jev", 0.042, 32000, "assets"), + classifier("typesafe/jev-1.14.0", "Jev 1.14", 0.042, 32000, "assets"), +]; + +export default defineProvider({ + id: "cloudflare-workers-ai", + models: (ctx) => + modelsDevModels(ctx, "cloudflare-workers-ai", { + api: "openai-completions", + baseUrl: CLOUDFLARE_WORKERS_AI_BASE_URL, + compat: { sendSessionAffinityHeaders: true }, + }), + classifiers: () => CLASSIFIER_MODELS, + compat: { + "openai-completions": { + supportsStore: false, + supportsDeveloperRole: false, + supportsStrictMode: true, + supportsLongCacheRetention: false, + }, + }, +}); diff --git a/packages/ai/catalog/providers/deepseek.ts b/packages/ai/catalog/providers/deepseek.ts new file mode 100644 index 000000000..af048844d --- /dev/null +++ b/packages/ai/catalog/providers/deepseek.ts @@ -0,0 +1,57 @@ +import type { Model } from "../../src/types.ts"; +import { defineProvider, mergeCompat } from "../define.ts"; +import { DEEPSEEK_COMPAT, DEEPSEEK_V4_FLASH_THINKING_LEVEL_MAP } from "../families/deepseek.ts"; + +// models.dev's DeepSeek entries lag the API; pricing from +// https://api-docs.deepseek.com/quick_start/pricing. DeepSeek also offers +// time-based off-peak rates, which the cost schema cannot represent yet. +const MODELS: Model<"openai-completions">[] = [ + { + id: "deepseek-flash", + name: "DeepSeek V4.1 Flash", + api: "openai-completions", + baseUrl: "https://api.deepseek.com", + provider: "deepseek", + reasoning: true, + thinkingLevelMap: DEEPSEEK_V4_FLASH_THINKING_LEVEL_MAP, + input: ["text", "image"], + cost: { input: 0.3, output: 1.2, cacheRead: 0.006, cacheWrite: 0 }, + contextWindow: 1000000, + maxTokens: 384000, + compat: DEEPSEEK_COMPAT, + }, + { + id: "deepseek-v4-pro", + name: "DeepSeek V4 Pro", + api: "openai-completions", + baseUrl: "https://api.deepseek.com", + provider: "deepseek", + reasoning: true, + input: ["text"], + cost: { input: 1.32, output: 3.96, cacheRead: 0.044, cacheWrite: 0 }, + contextWindow: 1000000, + maxTokens: 384000, + compat: DEEPSEEK_COMPAT, + }, +]; + +export default defineProvider({ + id: "deepseek", + models: () => structuredClone(MODELS), + compat: { + "openai-completions": { + supportsStore: false, + supportsDeveloperRole: false, + maxTokensField: "max_tokens", + requiresReasoningContentOnAssistantMessages: true, + thinkingFormat: "deepseek", + supportsStrictMode: true, + }, + }, + finalize: (model) => { + // DeepSeek V4 Pro accepts plain system text in place mid-conversation. + if (model.id === "deepseek-v4-pro") { + mergeCompat(model as Model<"openai-completions">, { supportsMidConvoSystemMessages: true }); + } + }, +}); diff --git a/packages/ai/catalog/providers/fireworks.ts b/packages/ai/catalog/providers/fireworks.ts new file mode 100644 index 000000000..9751c022b --- /dev/null +++ b/packages/ai/catalog/providers/fireworks.ts @@ -0,0 +1,100 @@ +import type { AnthropicMessagesCompat, Model, OpenAICompletionsCompat } from "../../src/types.ts"; +import { defineProvider, mergeCompat, mergeThinking, modelsDevModels } from "../define.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; + +// Fireworks Anthropic-compatible API. Prompt caching uses automatic prefix +// matching and session affinity: x-session-affinity routes requests to the same +// replica for cache hits. cache_control on tools and eager_input_streaming are +// not supported. +// https://docs.fireworks.ai/tools-sdks/anthropic-compatibility +const ANTHROPIC_COMPAT: AnthropicMessagesCompat = { + allowEmptySignature: true, + sendSessionAffinityHeaders: true, + supportsEagerToolInputStreaming: false, + supportsCacheControlOnTools: false, + supportsLongCacheRetention: false, +}; +const OPENAI_COMPAT: OpenAICompletionsCompat = { + supportsStore: false, + supportsDeveloperRole: false, + sendSessionAffinityHeaders: true, + supportsLongCacheRetention: false, +}; +const KIMI_K3_COMPAT: OpenAICompletionsCompat = { + ...OPENAI_COMPAT, + requiresReasoningContentOnAssistantMessages: true, + thinkingFormat: "openai", +}; + +// Verified against Fireworks Messages raw_output on 2026-09-10 (#9323). +// Fall back to verified support when models.dev omits effort metadata; this is +// not an allowlist. Any Fireworks Messages model advertising effort uses adaptive thinking. +const ADAPTIVE_THINKING_FALLBACK_MODELS = new Set([ + "accounts/fireworks/models/deepseek-v4-flash-0731", + "accounts/fireworks/models/deepseek-v4-flash-vision-exp", + "accounts/fireworks/models/deepseek-v4-pro-0813", + "accounts/fireworks/models/qwen3p8-max", + "accounts/fireworks/models/qwen3p8-2p4t-a95b", +]); + +export default defineProvider({ + id: "fireworks", + models: (ctx) => + modelsDevModels(ctx, "fireworks-ai", { + api: "anthropic-messages", + // SDK appends /v1/messages. + baseUrl: "https://api.fireworks.ai/inference", + map: (model, source) => { + if (model.id.includes("glm-") || model.id.includes("kimi-k3")) { + return { + ...model, + api: "openai-completions", + baseUrl: "https://api.fireworks.ai/inference/v1", + compat: model.id.includes("kimi-k3") ? KIMI_K3_COMPAT : OPENAI_COMPAT, + }; + } + // Use adaptive thinking for cataloged effort controls, with verified + // fallbacks where models.dev is incomplete. New models need no allowlist entry. + const adaptive = + source.reasoningOptions?.some((option) => option.type === "effort") || + ADAPTIVE_THINKING_FALLBACK_MODELS.has(model.id); + return { ...model, compat: { ...ANTHROPIC_COMPAT, ...(adaptive ? { forceAdaptiveThinking: true } : {}) } }; + }, + }), + compat: { "openai-completions": { supportsStrictMode: true } }, + finalize: (model, ctx) => { + if (model.api === "anthropic-messages" && (model as Model<"anthropic-messages">).compat?.forceAdaptiveThinking) { + // Qwen Max currently advertises only a toggle. Prefer upstream effort + // metadata once available instead of replacing it with this fallback. + if (model.id === "accounts/fireworks/models/qwen3p8-max" && !model.thinkingLevelMap) { + model.thinkingLevelMap = getEffortThinkingLevelMap([ + { type: "effort", values: ["low", "medium", "xhigh"] }, + ]); + } + if ( + ctx.modelsDevSource(model)?.reasoningOptions?.some((option) => option.type === "toggle") || + // The 2.4T alias omits the verified toggle in models.dev. + model.id === "accounts/fireworks/models/qwen3p8-2p4t-a95b" + ) { + mergeThinking(model, { off: "none" }); + } + if (model.id === "accounts/fireworks/models/deepseek-v4-pro-0813") mergeThinking(model, { low: "low" }); + } + if (model.id.includes("glm-5p2")) { + // GLM 5.2 and its fast router support off/high/max. Fireworks maps low + // and medium to high, so do not expose those aliases as distinct levels. + mergeThinking(model, { off: "none", minimal: null, low: null, medium: null, max: "max" }); + } + if (model.id.includes("kimi-k3")) { + // Fireworks maps medium to high on both APIs; do not expose it as a distinct level. + mergeThinking(model, { medium: null }); + // Kimi K3 accepts tool-bearing system messages; Fireworks passes them through. + if (model.api === "openai-completions") { + mergeCompat(model as Model<"openai-completions">, { + supportsMidConvoSystemMessages: true, + supportsMidConvoToolAdditions: true, + }); + } + } + }, +}); diff --git a/packages/ai/catalog/providers/github-copilot.ts b/packages/ai/catalog/providers/github-copilot.ts new file mode 100644 index 000000000..2bea3586b --- /dev/null +++ b/packages/ai/catalog/providers/github-copilot.ts @@ -0,0 +1,122 @@ +import type { Api, Model, ThinkingLevelMap } from "../../src/types.ts"; +import { addMissing, type ChatModel, defineProvider, mergeCompat, mergeThinking, modelsDevModels } from "../define.ts"; +import { claude55ThinkingLevels, supportsClaudeMidConvoSystemMessages } from "../families/anthropic.ts"; +import { + enableOpenAiGrammarTools, + enableOpenAiMidConvoSystemMessages, + openAiStandardCost, +} from "../families/openai.ts"; + +const BASE_URL = "https://api.individual.githubcopilot.com"; +const STATIC_HEADERS = { + "User-Agent": "GitHubCopilotChat/0.35.0", + "Editor-Version": "vscode/1.107.0", + "Editor-Plugin-Version": "copilot-chat/0.35.0", + "Copilot-Integration-Id": "vscode-chat", +} as const; + +// GitHub's "Models with extended capabilities" table lists these Copilot models as supporting +// the extended 1 million token context window. +const EXTENDED_CONTEXT_MODEL_IDS = new Set([ + "claude-fable-5", + "claude-opus-4.8", + "claude-opus-5", + "claude-opus-5.5", + "claude-sonnet-4.6", + "claude-sonnet-5", + "gpt-5.3-codex", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", +]); + +const EAGER_TOOL_INPUT_STREAMING_UNSUPPORTED_MODEL_IDS = new Set([ + "claude-haiku-4.5", + "claude-sonnet-4", + "claude-sonnet-4.5", +]); + +// Checked manually against the authenticated GitHub Copilot /models endpoint on 2026-06-15. +// Keep this to narrow corrections over models.dev metadata instead of snapshotting Copilot's catalog. +const THINKING_LEVEL_OVERRIDES: Record = { + "claude-opus-4.8": { minimal: "low" }, + "claude-opus-5": { minimal: "low" }, + "claude-sonnet-4.6": { minimal: "low", max: "max" }, +}; + +/** Claude 4.x and 5.x use Messages; GPT, Grok, OSWE, and MAI-Code are only served on /responses. */ +function copilotApi(modelId: string): Api { + if (/^claude-(haiku|sonnet|opus|fable)-[45]([.-]|$)/.test(modelId)) return "anthropic-messages"; + if (/^(gpt-|grok-|oswe|mai-)/.test(modelId)) return "openai-responses"; + return "openai-completions"; +} + +// The authenticated Copilot catalog advertised these models on 2026-09-22, +// but models.dev did not include them yet. +const GPT_6_LUNA: Model<"openai-responses"> = { + id: "gpt-6-luna", + name: "GPT-6 Luna", + api: "openai-responses", + provider: "github-copilot", + baseUrl: BASE_URL, + reasoning: true, + input: ["text", "image"], + cost: openAiStandardCost("gpt-6-luna"), + contextWindow: 1000000, + maxTokens: 128000, + headers: { ...STATIC_HEADERS }, +}; + +export default defineProvider({ + id: "github-copilot", + models: (ctx) => + addMissing( + modelsDevModels(ctx, "github-copilot", { + api: "openai-completions", + baseUrl: BASE_URL, + headers: STATIC_HEADERS, + skipDeprecated: true, + defaultContextWindow: 128000, + defaultMaxTokens: 8192, + map: (model): ChatModel => { + const api = copilotApi(model.id); + const result: ChatModel = { ...model, api }; + if (api === "openai-completions") { + result.compat = { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + }; + } + if (model.id === "claude-opus-5.5") mergeThinking(result, claude55ThinkingLevels(model.id)); + return result; + }, + }), + [GPT_6_LUNA], + ).map((model) => (EXTENDED_CONTEXT_MODEL_IDS.has(model.id) ? { ...model, contextWindow: 1000000 } : model)), + compat: { "openai-completions": { supportsStrictMode: true } }, + finalize: (model) => { + if (model.id.startsWith("gpt-5")) mergeThinking(model, { minimal: "low" }); + const override = THINKING_LEVEL_OVERRIDES[model.id]; + if (override) mergeThinking(model, override); + if (model.api === "anthropic-messages") { + const messagesModel = model as Model<"anthropic-messages">; + // Copilot forwards mid-conversation system messages but rejects + // `tool_addition`/`tool_removal` blocks, so tool changes stay top-level. + if (supportsClaudeMidConvoSystemMessages(model.id)) { + mergeCompat(messagesModel, { supportsMidConvoSystemMessages: true }); + } + if (EAGER_TOOL_INPUT_STREAMING_UNSUPPORTED_MODEL_IDS.has(model.id)) { + mergeCompat(messagesModel, { supportsEagerToolInputStreaming: false }); + } + } + // Copilot forwards Kimi K3's system text but silently drops its tool-bearing message. + if (model.api === "openai-completions" && model.id === "kimi-k3") { + mergeCompat(model as Model<"openai-completions">, { supportsMidConvoSystemMessages: true }); + } + // Copilot passes mid-conversation developer messages and `additional_tools` + // through to OpenAI unchanged; tool search is not verified through it. + enableOpenAiMidConvoSystemMessages(model, { proxy: true }); + enableOpenAiGrammarTools(model); + }, +}); diff --git a/packages/ai/catalog/providers/google-vertex.ts b/packages/ai/catalog/providers/google-vertex.ts new file mode 100644 index 000000000..e74a4ed5f --- /dev/null +++ b/packages/ai/catalog/providers/google-vertex.ts @@ -0,0 +1,43 @@ +import { defineProvider, modelsDevModels } from "../define.ts"; +import { googleThinkingLevels } from "../families/google.ts"; +import { modelsDevChatModels } from "../sources/models-dev.ts"; +import { GOOGLE_ALIAS_TARGETS } from "./google.ts"; + +export default defineProvider({ + id: "google-vertex", + models: (ctx) => { + const sources = new Map( + modelsDevChatModels(ctx.sources.modelsDev, "google-vertex").map((model) => [model.id, model]), + ); + return modelsDevModels(ctx, "google-vertex", { + api: "google-vertex", + baseUrl: "https://{location}-aiplatform.googleapis.com", + recordReasoning: false, + // The models.dev catalog also includes Claude, OpenAI, and other MaaS models + // that do not use the @google/genai Gemini streaming path. + include: (source) => source.id.startsWith("gemini-") && source.id !== "gemini-3.1-flash-lite-preview", + map: (model, aliasSource) => { + const target = GOOGLE_ALIAS_TARGETS[model.id]; + const source = (target && sources.get(target)) || aliasSource; + const thinkingLevelMap = googleThinkingLevels(model.id, source.reasoningOptions); + return { + ...model, + reasoning: source.reasoning, + ...(thinkingLevelMap ? { thinkingLevelMap } : {}), + input: source.imageInput ? ["text", "image"] : ["text"], + cost: { + input: source.cost.input || 0, + output: source.cost.output || 0, + // models.dev reports Vertex cache_read/cache_write values for Gemini 2.5 Flash that + // do not match the official Gemini API standard pricing table. pi only accounts + // cachedContentTokenCount as cacheRead. + cacheRead: model.id === "gemini-2.5-flash" ? 0.03 : source.cost.cacheRead || 0, + cacheWrite: 0, + }, + contextWindow: source.contextWindow || 4096, + maxTokens: source.maxTokens || 4096, + }; + }, + }); + }, +}); diff --git a/packages/ai/catalog/providers/google.ts b/packages/ai/catalog/providers/google.ts new file mode 100644 index 000000000..15cac1f93 --- /dev/null +++ b/packages/ai/catalog/providers/google.ts @@ -0,0 +1,36 @@ +import { defineProvider, modelsDevModels, withInputLimits } from "../define.ts"; +import { googleThinkingLevels } from "../families/google.ts"; +import { modelsDevChatModels, modelsDevCost } from "../sources/models-dev.ts"; + +// Rolling aliases take their metadata from the model they currently point to. +export const GOOGLE_ALIAS_TARGETS: Record = { + "gemini-flash-latest": "gemini-3.5-flash", + "gemini-flash-lite-latest": "gemini-3.1-flash-lite", +}; + +export default defineProvider({ + id: "google", + models: (ctx) => { + const sources = new Map(modelsDevChatModels(ctx.sources.modelsDev, "google").map((model) => [model.id, model])); + return modelsDevModels(ctx, "google", { + api: "google-generative-ai", + baseUrl: "https://generativelanguage.googleapis.com/v1beta", + recordReasoning: false, + map: (model, aliasSource) => { + const target = GOOGLE_ALIAS_TARGETS[model.id]; + const source = (target && sources.get(target)) || aliasSource; + const thinkingLevelMap = googleThinkingLevels(model.id, source.reasoningOptions); + return { + ...model, + reasoning: source.reasoning, + ...(thinkingLevelMap ? { thinkingLevelMap } : {}), + input: source.imageInput ? ["text", "image"] : ["text"], + cost: modelsDevCost(source.cost), + contextWindow: source.contextWindow || 4096, + maxTokens: source.maxTokens || 4096, + }; + }, + }); + }, + finalize: (model) => withInputLimits(model, { maxRequestBytes: 20 * 1024 * 1024, images: { maxPerRequest: 3600 } }), +}); diff --git a/packages/ai/catalog/providers/groq.ts b/packages/ai/catalog/providers/groq.ts new file mode 100644 index 000000000..7f6e15d76 --- /dev/null +++ b/packages/ai/catalog/providers/groq.ts @@ -0,0 +1,13 @@ +import { defineProvider, mergeThinking, modelsDevModels } from "../define.ts"; + +export default defineProvider({ + id: "groq", + models: (ctx) => + modelsDevModels(ctx, "groq", { api: "openai-completions", baseUrl: "https://api.groq.com/openai/v1" }), + compat: { "openai-completions": { supportsStrictMode: true } }, + finalize: (model) => { + if (model.id === "qwen/qwen3.6-27b") { + mergeThinking(model, { minimal: null, low: null, medium: null, high: "default" }); + } + }, +}); diff --git a/packages/ai/catalog/providers/huggingface.ts b/packages/ai/catalog/providers/huggingface.ts new file mode 100644 index 000000000..69ab9104c --- /dev/null +++ b/packages/ai/catalog/providers/huggingface.ts @@ -0,0 +1,12 @@ +import { defineProvider, modelsDevModels } from "../define.ts"; + +export default defineProvider({ + id: "huggingface", + models: (ctx) => + modelsDevModels(ctx, "huggingface", { + api: "openai-completions", + baseUrl: "https://router.huggingface.co/v1", + compat: { supportsDeveloperRole: false }, + }), + compat: { "openai-completions": { supportsStrictMode: true } }, +}); diff --git a/packages/ai/catalog/providers/index.ts b/packages/ai/catalog/providers/index.ts new file mode 100644 index 000000000..60b29cbd5 --- /dev/null +++ b/packages/ai/catalog/providers/index.ts @@ -0,0 +1,90 @@ +// Every built-in provider catalog. Add new providers here. + +import type { ProviderDefinition } from "../define.ts"; +import amazonBedrock from "./amazon-bedrock.ts"; +import antLing from "./ant-ling.ts"; +import anthropic from "./anthropic.ts"; +import azure from "./azure.ts"; +import baseten from "./baseten.ts"; +import cerebras from "./cerebras.ts"; +import cloudflareAiGateway from "./cloudflare-ai-gateway.ts"; +import cloudflareWorkersAi from "./cloudflare-workers-ai.ts"; +import deepseek from "./deepseek.ts"; +import fireworks from "./fireworks.ts"; +import githubCopilot from "./github-copilot.ts"; +import google from "./google.ts"; +import googleVertex from "./google-vertex.ts"; +import groq from "./groq.ts"; +import huggingface from "./huggingface.ts"; +import kimiCoding from "./kimi-coding.ts"; +import meta from "./meta.ts"; +import minimax from "./minimax.ts"; +import minimaxCn from "./minimax-cn.ts"; +import mistral from "./mistral.ts"; +import moonshotai from "./moonshotai.ts"; +import moonshotaiCn from "./moonshotai-cn.ts"; +import nvidia from "./nvidia.ts"; +import openai from "./openai.ts"; +import openaiCodex from "./openai-codex.ts"; +import opencode from "./opencode.ts"; +import opencodeGo from "./opencode-go.ts"; +import openrouter from "./openrouter.ts"; +import qwenTokenPlan from "./qwen-token-plan.ts"; +import qwenTokenPlanCn from "./qwen-token-plan-cn.ts"; +import qwenTokenPlanIndividual from "./qwen-token-plan-individual.ts"; +import radius from "./radius.ts"; +import together from "./together.ts"; +import typesafe from "./typesafe.ts"; +import vercelAiGateway from "./vercel-ai-gateway.ts"; +import xai from "./xai.ts"; +import xiaomi from "./xiaomi.ts"; +import xiaomiTokenPlanAms from "./xiaomi-token-plan-ams.ts"; +import xiaomiTokenPlanCn from "./xiaomi-token-plan-cn.ts"; +import xiaomiTokenPlanSgp from "./xiaomi-token-plan-sgp.ts"; +import zai from "./zai.ts"; +import zaiCodingCn from "./zai-coding-cn.ts"; + +export const PROVIDERS: readonly ProviderDefinition[] = [ + amazonBedrock, + antLing, + anthropic, + azure, + baseten, + cerebras, + cloudflareAiGateway, + cloudflareWorkersAi, + deepseek, + fireworks, + githubCopilot, + googleVertex, + google, + groq, + huggingface, + kimiCoding, + meta, + minimaxCn, + minimax, + mistral, + moonshotaiCn, + moonshotai, + nvidia, + openaiCodex, + openai, + opencodeGo, + opencode, + openrouter, + qwenTokenPlanCn, + qwenTokenPlanIndividual, + qwenTokenPlan, + radius, + together, + typesafe, + vercelAiGateway, + xai, + xiaomiTokenPlanAms, + xiaomiTokenPlanCn, + xiaomiTokenPlanSgp, + xiaomi, + zaiCodingCn, + zai, +]; diff --git a/packages/ai/catalog/providers/kimi-coding.ts b/packages/ai/catalog/providers/kimi-coding.ts new file mode 100644 index 000000000..782b44455 --- /dev/null +++ b/packages/ai/catalog/providers/kimi-coding.ts @@ -0,0 +1,55 @@ +import type { Model, ModelCost } from "../../src/types.ts"; +import { defineProvider, modelsDevModels } from "../define.ts"; +import { KIMI_K3_COST, KIMI_K3_OPENAI_COMPAT, KIMI_OPENAI_COMPAT } from "../families/kimi.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; + +// Kimi Coding is subscription-backed, so models.dev reports zero cost. Use the +// equivalent Moonshot API rates to estimate the value of subscription usage. +const IMPLIED_COSTS: Record = { + k3: KIMI_K3_COST, + "k3-256k": KIMI_K3_COST, + "k3.1-pro": { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 0 }, + "k3.1-pro-highspeed": { input: 4, output: 20, cacheRead: 0.2, cacheWrite: 0 }, + "kimi-for-coding": { input: 0.95, output: 4, cacheRead: 0.19, cacheWrite: 0 }, + "kimi-for-coding-highspeed": { input: 1.9, output: 8, cacheRead: 0.38, cacheWrite: 0 }, +}; + +// Kimi Coding uses Chat Completions, the same protocol as the official Kimi Code client. +export default defineProvider({ + id: "kimi-coding", + models: (ctx) => + modelsDevModels(ctx, "kimi-code-plan-global", { + api: "openai-completions", + baseUrl: "https://api.kimi.com/coding/v1", + recordReasoning: false, + map: (model, source): Model<"openai-completions"> => { + // K3-style models report effort levels and dynamic tool loading (tools declared in + // later system messages) in https://api.kimi.com/coding/v1/models; models.dev mirrors + // the effort levels. kimi-for-coding-highspeed reports neither and rejects + // tool-bearing system messages with a 400. + const isK3Style = source.reasoningOptions?.some((option) => option.type === "effort") === true; + const impliedCost = IMPLIED_COSTS[model.id]; + // K3-style models send reasoning_effort, so expose the effort values models.dev lists. + const effortLevels = + isK3Style && source.reasoningOptions ? getEffortThinkingLevelMap(source.reasoningOptions) : undefined; + return { + ...model, + compat: { + ...(isK3Style ? KIMI_K3_OPENAI_COMPAT : KIMI_OPENAI_COMPAT), + // All models accept system text after the conversation starts. + supportsMidConvoSystemMessages: true, + ...(isK3Style ? { supportsMidConvoToolAdditions: true } : {}), + }, + cost: { + input: source.cost.input || impliedCost?.input || 0, + output: source.cost.output || impliedCost?.output || 0, + cacheRead: source.cost.cacheRead || impliedCost?.cacheRead || 0, + cacheWrite: source.cost.cacheWrite || impliedCost?.cacheWrite || 0, + }, + // Kimi Code reports every model as always-thinking. + thinkingLevelMap: { ...effortLevels, off: null }, + }; + }, + }), + compat: { "openai-completions": { supportsStrictMode: true } }, +}); diff --git a/packages/ai/catalog/providers/meta.ts b/packages/ai/catalog/providers/meta.ts new file mode 100644 index 000000000..7d4bba68d --- /dev/null +++ b/packages/ai/catalog/providers/meta.ts @@ -0,0 +1,6 @@ +import { defineProvider, modelsDevModels } from "../define.ts"; + +export default defineProvider({ + id: "meta", + models: (ctx) => modelsDevModels(ctx, "meta", { api: "openai-responses", baseUrl: "https://api.meta.ai/v1" }), +}); diff --git a/packages/ai/catalog/providers/minimax-cn.ts b/packages/ai/catalog/providers/minimax-cn.ts new file mode 100644 index 000000000..c34cd2316 --- /dev/null +++ b/packages/ai/catalog/providers/minimax-cn.ts @@ -0,0 +1,7 @@ +import { defineProvider } from "../define.ts"; +import { minimaxModels } from "./minimax.ts"; + +export default defineProvider({ + id: "minimax-cn", + models: (ctx) => minimaxModels(ctx, "minimax-cn", "https://api.minimaxi.com/anthropic"), +}); diff --git a/packages/ai/catalog/providers/minimax.ts b/packages/ai/catalog/providers/minimax.ts new file mode 100644 index 000000000..7504b7683 --- /dev/null +++ b/packages/ai/catalog/providers/minimax.ts @@ -0,0 +1,26 @@ +import { defineProvider, modelsDevModels, type ProviderContext } from "../define.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; + +// MiniMax asked to hide older models from its direct providers (#2596). +const HIDDEN_MODEL_IDS = new Set(["MiniMax-M2", "MiniMax-M2.1", "MiniMax-M2.5", "MiniMax-M2.5-highspeed"]); + +/** MiniMax's Anthropic-compatible API. The SDK appends /v1/messages. */ +export function minimaxModels(ctx: ProviderContext, source: string, baseUrl: string) { + return modelsDevModels(ctx, source, { + api: "anthropic-messages", + baseUrl, + recordReasoning: false, + include: (model) => !HIDDEN_MODEL_IDS.has(model.id), + map: (model, source) => { + // Models with effort levels (e.g. MiniMax-M3.1-Flash) take adaptive thinking with + // output_config.effort and reject thinking.type="disabled" (no "none" effort, so off is null). + const thinkingLevelMap = getEffortThinkingLevelMap(source.reasoningOptions ?? []); + return thinkingLevelMap ? { ...model, thinkingLevelMap, compat: { forceAdaptiveThinking: true } } : model; + }, + }); +} + +export default defineProvider({ + id: "minimax", + models: (ctx) => minimaxModels(ctx, "minimax", "https://api.minimax.io/anthropic"), +}); diff --git a/packages/ai/catalog/providers/mistral.ts b/packages/ai/catalog/providers/mistral.ts new file mode 100644 index 000000000..8009e4650 --- /dev/null +++ b/packages/ai/catalog/providers/mistral.ts @@ -0,0 +1,46 @@ +import { addMissing, defineProvider, modelsDevModels } from "../define.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; +import { roundCost } from "../sources/models-dev.ts"; + +export default defineProvider({ + id: "mistral", + models: (ctx) => + addMissing( + modelsDevModels(ctx, "mistral", { + api: "mistral-conversations", + baseUrl: "https://api.mistral.ai", + recordReasoning: false, + map: (model, source) => { + // Models with effort values use `reasoning_effort` with these levels. + // Reasoning models without them (Magistral) use `prompt_mode`. + const thinkingLevelMap = getEffortThinkingLevelMap(source.reasoningOptions ?? []); + return { + ...model, + ...(thinkingLevelMap ? { thinkingLevelMap } : {}), + cost: { + input: source.cost.input || 0, + output: source.cost.output || 0, + // Mistral bills cached input at 10% when models.dev does not say otherwise. + cacheRead: source.cost.cacheRead ?? (source.cost.input ? roundCost(source.cost.input * 0.1) : 0), + cacheWrite: source.cost.cacheWrite || 0, + }, + }; + }, + }), + [ + { + id: "mistral-medium-3.5", + name: "Mistral Medium 3.5", + api: "mistral-conversations", + provider: "mistral", + baseUrl: "https://api.mistral.ai", + reasoning: true, + thinkingLevelMap: getEffortThinkingLevelMap([{ type: "effort", values: ["none", "high"] }]), + input: ["text", "image"], + cost: { input: 1.5, output: 7.5, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 262144, + maxTokens: 262144, + }, + ], + ), +}); diff --git a/packages/ai/catalog/providers/moonshotai-cn.ts b/packages/ai/catalog/providers/moonshotai-cn.ts new file mode 100644 index 000000000..e8f3d91e1 --- /dev/null +++ b/packages/ai/catalog/providers/moonshotai-cn.ts @@ -0,0 +1,9 @@ +import { defineProvider } from "../define.ts"; +import { finalizeMoonshotModel, MOONSHOT_COMPLETIONS_COMPAT, moonshotModels } from "./moonshotai.ts"; + +export default defineProvider({ + id: "moonshotai-cn", + models: (ctx) => moonshotModels(ctx, "moonshotai-cn", "https://api.moonshot.cn/v1"), + compat: { "openai-completions": MOONSHOT_COMPLETIONS_COMPAT }, + finalize: finalizeMoonshotModel, +}); diff --git a/packages/ai/catalog/providers/moonshotai.ts b/packages/ai/catalog/providers/moonshotai.ts new file mode 100644 index 000000000..11ba2a5cc --- /dev/null +++ b/packages/ai/catalog/providers/moonshotai.ts @@ -0,0 +1,71 @@ +import type { Model } from "../../src/types.ts"; +import { + type ChatModel, + defineProvider, + mergeCompat, + mergeThinking, + modelsDevModels, + type ProviderContext, +} from "../define.ts"; +import { isKimiK3FamilyId, KIMI_K3_COST, KIMI_K3_OPENAI_COMPAT, KIMI_OPENAI_COMPAT } from "../families/kimi.ts"; + +const ALWAYS_THINKING_MODEL_IDS = new Set(["kimi-k2.7-code", "kimi-k2.7-code-highspeed"]); +// Kimi K2.6/K2.7 accept system text after the conversation starts but reject +// tool-bearing system messages. +const TEXT_ONLY_MID_CONVO_MODEL_IDS = new Set(["kimi-k2.6", ...ALWAYS_THINKING_MODEL_IDS]); + +export function moonshotModels(ctx: ProviderContext, source: string, baseUrl: string) { + return modelsDevModels(ctx, source, { + api: "openai-completions", + baseUrl, + map: (model, source): Model<"openai-completions"> => { + const isKimiK3 = isKimiK3FamilyId(model.id); + return { + ...model, + reasoning: isKimiK3 || source.reasoning, + // Moonshot does not bill cache writes; models.dev lists the input rate as cache_write for the K3 family. + cost: + model.id === "kimi-k3" + ? { ...KIMI_K3_COST } + : { + input: source.cost.input || 0, + output: source.cost.output || 0, + cacheRead: source.cost.cacheRead || 0, + cacheWrite: isKimiK3 ? 0 : source.cost.cacheWrite || 0, + }, + compat: { ...(isKimiK3 ? KIMI_K3_OPENAI_COMPAT : KIMI_OPENAI_COMPAT) }, + }; + }, + }); +} + +export const MOONSHOT_COMPLETIONS_COMPAT = { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", +} as const; + +export function finalizeMoonshotModel(model: ChatModel): void { + if (ALWAYS_THINKING_MODEL_IDS.has(model.id)) { + // Kimi K2.7 Code is always-thinking. Official docs say + // `thinking: { type: "disabled" }` is rejected, and callers can omit + // the thinking parameter to use the enabled default. + mergeThinking(model, { off: null }); + } + const isKimiK3 = isKimiK3FamilyId(model.id); + if (isKimiK3 || TEXT_ONLY_MID_CONVO_MODEL_IDS.has(model.id)) { + mergeCompat(model as Model<"openai-completions">, { + supportsMidConvoSystemMessages: true, + // The Kimi K3 family also accepts tool-bearing system messages. + ...(isKimiK3 ? { supportsMidConvoToolAdditions: true } : {}), + }); + } +} + +export default defineProvider({ + id: "moonshotai", + models: (ctx) => moonshotModels(ctx, "moonshotai", "https://api.moonshot.ai/v1"), + compat: { "openai-completions": MOONSHOT_COMPLETIONS_COMPAT }, + finalize: finalizeMoonshotModel, +}); diff --git a/packages/ai/catalog/providers/nvidia.ts b/packages/ai/catalog/providers/nvidia.ts new file mode 100644 index 000000000..1d492ff29 --- /dev/null +++ b/packages/ai/catalog/providers/nvidia.ts @@ -0,0 +1,64 @@ +import { defineProvider, modelsDevModels } from "../define.ts"; +import { NVIDIA_BASE_URL, normalizeNvidiaModelId } from "../sources/index.ts"; + +// Listed by NIM but unusable through pi (no tool calls, broken streaming, or retired). +const UNSUPPORTED_MODEL_IDS = new Set([ + "abacusai/dracarys-llama-3.1-70b-instruct", + "bytedance/seed-oss-36b-instruct", + "deepseek-ai/deepseek-v4-flash", + "deepseek-ai/deepseek-v4-pro", + "google/gemma-2-2b-it", + "google/gemma-3n-e2b-it", + "google/gemma-3n-e4b-it", + "google/gemma-4-31b-it", + "meta/llama-3.2-1b-instruct", + "meta/llama-4-maverick-17b-128e-instruct", + "microsoft/phi-4-mini-instruct", + "minimaxai/minimax-m2.7", + "mistralai/mistral-nemotron", + "nvidia/nemotron-mini-4b-instruct", + "qwen/qwen3-next-80b-a3b-instruct", + "qwen/qwen3.5-397b-a17b", + "sarvamai/sarvam-m", + "upstage/solar-10.7b-instruct", +]); + +export default defineProvider({ + id: "nvidia", + models: (ctx) => { + const liveIds = ctx.sources.nvidiaModelIds; + const liveId = (id: string) => liveIds.get(id) ?? liveIds.get(normalizeNvidiaModelId(id)); + return modelsDevModels(ctx, "nvidia", { + api: "openai-completions", + baseUrl: NVIDIA_BASE_URL, + headers: { "NVCF-POLL-SECONDS": "3600" }, + compat: { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + supportsStrictMode: false, + supportsLongCacheRetention: false, + }, + // Only models the live NIM API serves, under the id it uses. + include: (source) => { + if (!source.textInput || !source.textOutput) return false; + const id = liveId(source.id); + return id !== undefined && !UNSUPPORTED_MODEL_IDS.has(id); + }, + map: (model) => { + const id = liveId(model.id) ?? model.id; + return { ...model, id, name: model.name === model.id ? id : model.name }; + }, + }); + }, + compat: { + "openai-completions": { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + supportsLongCacheRetention: false, + }, + }, +}); diff --git a/packages/ai/catalog/providers/openai-codex.ts b/packages/ai/catalog/providers/openai-codex.ts new file mode 100644 index 000000000..fc120d039 --- /dev/null +++ b/packages/ai/catalog/providers/openai-codex.ts @@ -0,0 +1,68 @@ +import type { Model } from "../../src/types.ts"; +import { defineProvider, mergeCompat, mergeThinking } from "../define.ts"; +import { + enableOpenAiGrammarTools, + enableOpenAiMidConvoSystemMessages, + OPENAI_TOOL_SEARCH_MODEL_IDS, + openAiStandardCost, + supportsOpenAiXhigh, +} from "../families/openai.ts"; + +// ChatGPT OAuth models. models.dev does not list them; keep a small, explicit +// list without aliases. Only list models the Codex backend accepts for ChatGPT +// accounts: GPT-5.5 and GPT-5.3 Codex Spark are rejected with "not supported +// when using Codex with a ChatGPT account". +const MODELS = [ + // GPT-6.1 uses the 500k default returned by /backend-api/codex/models. + { id: "gpt-6.1-sol", name: "GPT-6.1 Sol", contextWindow: 500000 }, + { id: "gpt-6.1-terra", name: "GPT-6.1 Terra", contextWindow: 500000 }, + // GPT-5.6 and GPT-6 use Codex's 272k default catalog limit. + { id: "gpt-6-astra", name: "GPT-6 Astra", contextWindow: 272000 }, + { id: "gpt-6-sol", name: "GPT-6 Sol", contextWindow: 272000 }, + { id: "gpt-6-luna", name: "GPT-6 Luna", contextWindow: 272000 }, + { id: "gpt-5.6-luna", name: "GPT-5.6 Luna", contextWindow: 272000 }, + { id: "gpt-5.6-sol", name: "GPT-5.6 Sol", contextWindow: 272000 }, + { id: "gpt-5.6-terra", name: "GPT-5.6 Terra", contextWindow: 272000 }, +]; + +// Codex CLI keeps additional_tools behind its own model list. +const ADDITIONAL_TOOLS_MODEL_IDS = new Set([ + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-5.6-luna", + "gpt-6-astra", + "gpt-6-sol", + "gpt-6-luna", + "gpt-6.1-sol", + "gpt-6.1-terra", +]); + +export default defineProvider({ + id: "openai-codex", + models: () => + MODELS.map( + ({ id, name, contextWindow }): Model<"openai-codex-responses"> => ({ + id, + name, + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text", "image"], + cost: openAiStandardCost(id), + contextWindow, + maxTokens: 128000, + }), + ), + finalize: (model) => { + if (supportsOpenAiXhigh(model.id)) mergeThinking(model, { minimal: "low" }); + enableOpenAiGrammarTools(model); + if (OPENAI_TOOL_SEARCH_MODEL_IDS.has(model.id)) { + mergeCompat(model as Model<"openai-codex-responses">, { + ...(ADDITIONAL_TOOLS_MODEL_IDS.has(model.id) ? { supportsAdditionalTools: true } : {}), + supportsToolSearch: true, + }); + } + enableOpenAiMidConvoSystemMessages(model, { proxy: false }); + }, +}); diff --git a/packages/ai/catalog/providers/openai.ts b/packages/ai/catalog/providers/openai.ts new file mode 100644 index 000000000..873dc11ec --- /dev/null +++ b/packages/ai/catalog/providers/openai.ts @@ -0,0 +1,141 @@ +import type { ClassifierModel, Model } from "../../src/types.ts"; +import { + addMissing, + type ChatModel, + defineProvider, + mergeCompat, + mergeThinking, + modelsDevModels, + withInputLimits, +} from "../define.ts"; +import { + enableOpenAiGrammarTools, + enableOpenAiMidConvoSystemMessages, + OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, + OPENAI_STANDARD_COSTS, + OPENAI_TOOL_SEARCH_MODEL_IDS, + withOpenAiLongContextPricing, +} from "../families/openai.ts"; + +const BASE_URL = "https://api.openai.com/v1"; + +// models.dev lists this alias, but it is not accepted by OpenAI APIs. +const UNSUPPORTED_MODEL_IDS = new Set(["gpt-5.6"]); + +// Keep direct OpenAI requests in the short-context pricing tier by default. Users can opt into the +// larger context through model overrides, so retain long-context cost metadata on the capped models. +const SHORT_CONTEXT_CAPPED_MODEL_IDS = new Set([ + "gpt-5.4", + "gpt-5.5", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-5.6-luna", + "gpt-6-astra", + "gpt-6-sol", + "gpt-6-luna", + "gpt-6.1-sol", + "gpt-6.1-terra", +]); +const LONG_CONTEXT_PRICING_MODEL_IDS = new Set([...SHORT_CONTEXT_CAPPED_MODEL_IDS, "gpt-5.4-pro", "gpt-5.5-pro"]); + +const NONE_REASONING_MODEL_IDS = new Set([ + "gpt-5.1", + "gpt-5.2", + "gpt-5.3-codex", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.5", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-5.6-luna", + "gpt-6-sol", + "gpt-6-luna", +]); +// OpenAI documents configuration_update for the GPT-6 family. +// https://developers.openai.com/api/docs/guides/reasoning +const MID_CONVO_EFFORT_MODEL_IDS = new Set(["gpt-6-astra", "gpt-6-sol", "gpt-6-luna", "gpt-6.1-sol", "gpt-6.1-terra"]); + +// Not listed by models.dev. +const GPT_5_CHAT_LATEST: Model<"openai-responses"> = { + id: "gpt-5-chat-latest", + name: "GPT-5 Chat Latest", + api: "openai-responses", + baseUrl: BASE_URL, + provider: "openai", + reasoning: false, + input: ["text", "image"], + cost: { input: 1.25, output: 10, cacheRead: 0.125, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 16384, +}; + +// OpenAI Decisions API (public beta): gpt-6-luna is its only model. It bills input tokens only, +// with the same long-context multiplier as chat requests. Only API keys work: Sign in with ChatGPT +// tokens are rejected on /v1/decisions, and the Codex backend has no Decisions route. +// The endpoint rejects inputs above 922K tokens (the model's documented maximum input), but +// requests running longer than about five seconds, currently above roughly 600K input tokens, +// fail with a gateway timeout. +// https://developers.openai.com/api/docs/guides/decisions +const CLASSIFIER_MODELS: ClassifierModel<"openai-decisions">[] = [ + { + type: "classifier", + id: "gpt-6-luna", + name: "GPT-6 Luna", + api: "openai-decisions", + provider: "openai", + baseUrl: BASE_URL, + input: ["text", "image"], + cost: withOpenAiLongContextPricing({ input: 0.1, output: 0, cacheRead: 0, cacheWrite: 0 }), + contextWindow: 922000, + }, +]; + +function correctUpstream(model: ChatModel): ChatModel { + if (SHORT_CONTEXT_CAPPED_MODEL_IDS.has(model.id)) { + model.contextWindow = OPENAI_LONG_CONTEXT_INPUT_THRESHOLD; + model.maxTokens = 128000; + } + if (LONG_CONTEXT_PRICING_MODEL_IDS.has(model.id)) { + model.cost = withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS[model.id] ?? model.cost); + } + // models.dev reports gpt-5-pro output as 272000 (a duplicate of the input sub-limit); + // the actual max output is 128000. Also propagates to the derived Azure clone. + if (model.id === "gpt-5-pro") model.maxTokens = 128000; + return model; +} + +export default defineProvider({ + id: "openai", + models: (ctx) => + addMissing( + modelsDevModels(ctx, "openai", { + api: "openai-responses", + baseUrl: BASE_URL, + // Audio models are only served on Chat Completions. + protocol: "responses", + include: (source) => !UNSUPPORTED_MODEL_IDS.has(source.id), + map: correctUpstream, + }), + [GPT_5_CHAT_LATEST], + ), + classifiers: () => CLASSIFIER_MODELS, + finalize: (model) => { + if (model.api !== "openai-responses") return; + const responsesModel = model as Model<"openai-responses">; + if (NONE_REASONING_MODEL_IDS.has(model.id)) mergeThinking(model, { off: "none" }); + if (model.id === "gpt-5.5") mergeThinking(model, { minimal: null }); + mergeCompat(responsesModel, { supportsStrictMode: true }); + enableOpenAiGrammarTools(model); + if (OPENAI_TOOL_SEARCH_MODEL_IDS.has(model.id)) { + mergeCompat(responsesModel, { supportsAdditionalTools: true, supportsToolSearch: true }); + } + enableOpenAiMidConvoSystemMessages(model, { proxy: false }); + // OpenAI charges prompt-cache writes starting with the GPT-5.6 family, and exactly + // those models accept `prompt_cache_options`; older models reject the parameter. + // https://developers.openai.com/api/docs/guides/prompt-caching + if (model.cost.cacheWrite > 0) mergeCompat(responsesModel, { supportsExplicitPromptCacheMode: true }); + if (MID_CONVO_EFFORT_MODEL_IDS.has(model.id)) mergeCompat(responsesModel, { supportsMidConvoEffort: true }); + withInputLimits(model, { maxRequestBytes: 512 * 1024 * 1024, images: { maxPerRequest: 1500 } }); + }, +}); diff --git a/packages/ai/catalog/providers/opencode-go.ts b/packages/ai/catalog/providers/opencode-go.ts new file mode 100644 index 000000000..b13b778e7 --- /dev/null +++ b/packages/ai/catalog/providers/opencode-go.ts @@ -0,0 +1,9 @@ +import { defineProvider } from "../define.ts"; +import { finalizeOpencodeModel, OPENCODE_COMPLETIONS_COMPAT, opencodeModels } from "./opencode.ts"; + +export default defineProvider({ + id: "opencode-go", + models: opencodeModels, + compat: { "openai-completions": OPENCODE_COMPLETIONS_COMPAT }, + finalize: finalizeOpencodeModel, +}); diff --git a/packages/ai/catalog/providers/opencode.ts b/packages/ai/catalog/providers/opencode.ts new file mode 100644 index 000000000..36057c764 --- /dev/null +++ b/packages/ai/catalog/providers/opencode.ts @@ -0,0 +1,206 @@ +import type { Api, ClassifierModel, Model, OpenAICompletionsCompat, OpenAIResponsesCompat } from "../../src/types.ts"; +import { + type ChatModel, + defineProvider, + mergeCompat, + mergeThinking, + modelsDevModels, + type ProviderContext, +} from "../define.ts"; +import { supportsClaudeMidConvoSystemMessages } from "../families/anthropic.ts"; +import { googleThinkingLevels } from "../families/google.ts"; +import { isKimiK3FamilyId } from "../families/kimi.ts"; +import { enableOpenAiGrammarTools, enableOpenAiMidConvoSystemMessages } from "../families/openai.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; +import type { ModelsDevModel } from "../sources/models-dev.ts"; + +// OpenAI-compatible endpoints reject the long prompt-cache retention parameter for these models. +const LONG_CACHE_RETENTION_UNSUPPORTED_MODELS = new Set([ + "opencode:deepseek-v4-flash", + "opencode:deepseek-v4.1-flash", + "opencode:deepseek-v4-pro", + "opencode:kimi-k2.5", + "opencode:kimi-k2.6", + "opencode:minimax-m2.7", + "opencode-go:kimi-k2.6", +]); + +// OpenCode serves each model on one protocol-specific endpoint. Native protocols +// come before generic Chat Completions if models.dev ever lists several. +const OPENCODE_PROTOCOLS = ["responses", "messages", "generate-content", "chat-completions"] as const; + +interface OpencodeApi { + api: Api; + baseUrl: string; + compat?: OpenAICompletionsCompat | OpenAIResponsesCompat; +} + +/** + * The API and base URL models.dev lists for a model. Models served only on a + * protocol pi does not support fail strict generation and are skipped otherwise. + */ +function opencodeApi(ctx: ProviderContext, source: ModelsDevModel): OpencodeApi | undefined { + const entry = OPENCODE_PROTOCOLS.map((protocol) => + source.apis.find((candidate) => candidate.protocol === protocol), + ).find((candidate) => candidate !== undefined); + if (!entry) { + const protocols = source.apis.map((candidate) => candidate.protocol).join(", ") || "none"; + const message = `models.dev lists no supported API for ${ctx.providerId}/${source.id} (${protocols})`; + if (ctx.strict) throw new Error(message); + console.warn(`Skipping model: ${message}`); + return undefined; + } + switch (entry.protocol) { + case "responses": + return { + api: "openai-responses", + baseUrl: entry.baseUrl, + compat: { sessionAffinityFormat: "openai-nosession" }, + }; + case "messages": + // The Anthropic SDK appends /v1/messages itself. + return { api: "anthropic-messages", baseUrl: entry.baseUrl.replace(/\/v1\/?$/, "") }; + case "generate-content": + return { api: "google-generative-ai", baseUrl: entry.baseUrl }; + case "chat-completions": + return { + api: "openai-completions", + baseUrl: entry.baseUrl, + ...(entry.aisdk === "@ai-sdk/alibaba" ? { compat: { cacheControlFormat: "anthropic" } } : {}), + }; + } +} + +/** OpenCode Zen and Go models, on the endpoints models.dev lists for them. */ +export function opencodeModels(ctx: ProviderContext): ChatModel[] { + const provider = ctx.providerId; + return modelsDevModels(ctx, provider, { + api: "openai-completions", + // Replaced by the base URL of the API models.dev lists for each model. + baseUrl: "", + skipDeprecated: true, + // Codex Spark is not usable through OpenCode. + include: (source) => source.id !== "gpt-5.3-codex-spark", + map: (model, source) => { + const served = opencodeApi(ctx, source); + if (!served) return undefined; + let { api, baseUrl, compat } = served; + if (provider === "opencode" && model.id === "grok-build-0.1") { + compat = { ...compat, supportsReasoningEffort: false }; + } + if (model.id === "kimi-k2.6") { + // OpenCode Kimi K2.6 accepts Anthropic-style thinking objects + // and rejects string thinking values or combined reasoning_effort. + compat = { ...compat, thinkingFormat: "deepseek", supportsReasoningEffort: false }; + } + // models.dev reports these as @ai-sdk/anthropic, but OpenCode Go serves + // them through the OpenAI-compatible /openai/v1/chat/completions path. + if ( + provider === "opencode-go" && + api === "anthropic-messages" && + (model.id === "qwen3.5-plus" || model.id === "qwen3.6-plus") + ) { + api = "openai-completions"; + baseUrl = baseUrl.replace(/\/anthropic$/, "/openai/v1"); + // Qwen/DashScope uses enable_thinking at the top level. + compat = { ...compat, thinkingFormat: "qwen" }; + } + if (api === "openai-completions") { + compat = { + ...compat, + maxTokensField: "max_tokens", + ...(LONG_CACHE_RETENTION_UNSUPPORTED_MODELS.has(`${provider}:${model.id}`) + ? { supportsLongCacheRetention: false } + : {}), + }; + } + const thinkingLevelMap = + api === "google-generative-ai" + ? googleThinkingLevels(model.id, source.reasoningOptions) + : provider === "opencode-go" && model.id === "deepseek-v4.1-flash" + ? getEffortThinkingLevelMap(source.reasoningOptions ?? []) + : undefined; + const result: ChatModel = { ...model, api, baseUrl, ...(thinkingLevelMap ? { thinkingLevelMap } : {}) }; + if (compat) result.compat = compat; + // OpenCode variants list Claude Sonnet 4/4.5 with 1M context, actual limit is 200K. + if (model.id === "claude-sonnet-4-5" || model.id === "claude-sonnet-4") result.contextWindow = 200000; + if (["claude-opus-4-6", "claude-sonnet-4-6", "claude-opus-4.6", "claude-sonnet-4.6"].includes(model.id)) { + result.contextWindow = 1000000; + } + if (model.id === "gpt-5.4") { + result.contextWindow = 272000; + result.maxTokens = 128000; + } + return result; + }, + }); +} + +export const OPENCODE_COMPLETIONS_COMPAT = { + supportsStore: false, + supportsDeveloperRole: false, + supportsStrictMode: true, +} as const; + +/** Behavior shared by OpenCode Zen and Go. */ +export function finalizeOpencodeModel(model: ChatModel): void { + if (model.provider === "opencode-go" && model.id === "glm-5.2") { + mergeThinking(model, { off: null, minimal: null, low: null, medium: null, high: "high", max: "max" }); + } + if (model.provider === "opencode-go" && model.id === "kimi-k2.6") { + // OpenCode Go exposes Kimi K2.6 thinking as on/off, not distinct effort tiers. + mergeThinking(model, { minimal: null, low: null, medium: null }); + } + if (model.provider === "opencode" && model.id === "grok-build-0.1") { + // OpenCode Zen Grok Build reasons by default but rejects explicit reasoningEffort. + mergeThinking(model, { off: null, minimal: null, low: null, medium: null }); + } + if (model.api === "anthropic-messages") { + const messagesModel = model as Model<"anthropic-messages">; + // OpenCode Zen forwards mid-conversation system messages but rejects + // `tool_addition`/`tool_removal` blocks, so tool changes stay top-level. + if (model.provider === "opencode" && supportsClaudeMidConvoSystemMessages(model.id)) { + mergeCompat(messagesModel, { supportsMidConvoSystemMessages: true }); + } + // OpenCode Qwen 3.8 Flash emits and accepts thinking blocks with empty signatures. + if (model.id === "qwen3.8-flash") mergeCompat(messagesModel, { allowEmptySignature: true }); + } + // The Kimi K3 family accepts tool-bearing system messages; OpenCode passes them through. + if (model.api === "openai-completions" && isKimiK3FamilyId(model.id)) { + mergeCompat(model as Model<"openai-completions">, { + supportsMidConvoSystemMessages: true, + supportsMidConvoToolAdditions: true, + }); + } + // OpenCode passes mid-conversation developer messages and `additional_tools` + // through to OpenAI unchanged; tool search is not verified through it. + enableOpenAiMidConvoSystemMessages(model, { proxy: true }); +} + +// OpenCode Zen serves Jev through its TypeSafe-compatible System One endpoint. +// Neither its /inference/openai/v1/models listing nor models.dev carries metadata for it. +// https://opencode.ai/docs/zen +function jev(id: string, name: string, input: number): ClassifierModel<"typesafe-system-one"> { + return { + type: "classifier", + id, + name, + api: "typesafe-system-one", + provider: "opencode", + baseUrl: "https://opencode.ai/inference/systemone/v1", + input: ["text"], + cost: { input, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 32000, + }; +} + +export default defineProvider({ + id: "opencode", + models: opencodeModels, + classifiers: () => [jev("jev-1.13", "Jev 1.13", 0.042), jev("jev-1.13-free", "Jev 1.13 Free", 0)], + compat: { "openai-completions": OPENCODE_COMPLETIONS_COMPAT }, + finalize: (model) => { + finalizeOpencodeModel(model); + enableOpenAiGrammarTools(model); + }, +}); diff --git a/packages/ai/catalog/providers/openrouter.ts b/packages/ai/catalog/providers/openrouter.ts new file mode 100644 index 000000000..6d705af93 --- /dev/null +++ b/packages/ai/catalog/providers/openrouter.ts @@ -0,0 +1,108 @@ +import type { Model, OpenAICompletionsCompat } from "../../src/types.ts"; +import { addMissing, type ChatModel, defineProvider, mergeCompat, mergeThinking } from "../define.ts"; +import { enableClaudeMidConvoEffort, supportsClaudeMidConvoEffort } from "../families/anthropic.ts"; +import { isKimiK3FamilyId, KIMI_K3_MAX_TOKENS } from "../families/kimi.ts"; +import { OPENAI_MID_CONVO_MODEL_IDS } from "../families/openai.ts"; + +const BASE_URL = "https://openrouter.ai/api/v1"; +// OpenRouter rejects `configuration_update` system messages on Opus 5 ("Mid-conversation +// reasoning effort (configuration_update) is not supported on anthropic/claude-opus-5-20260723") +// while accepting them on Fable 5.1. +const MID_CONVO_EFFORT_UNSUPPORTED_MODEL_IDS = new Set(["anthropic/claude-opus-5"]); + +/** Keep selected model metadata stable until upstream settles. */ +function correctUpstream(model: ChatModel): ChatModel { + // Keep Kimi K3's canonical output limit when gateway metadata is missing or incorrect. + if ( + model.id === "~moonshotai/kimi-latest" || + (model.id.startsWith("moonshotai/") && isKimiK3FamilyId(model.id.slice("moonshotai/".length))) + ) { + model.maxTokens = KIMI_K3_MAX_TOKENS; + } + if (model.id === "moonshotai/kimi-k2.5") { + model.cost = { ...model.cost, input: 0.41, output: 2.06, cacheRead: 0.07 }; + model.maxTokens = 4096; + } + if (model.id.startsWith("moonshotai/kimi-k2.6")) { + mergeCompat(model as Model<"openai-completions">, { + supportsDeveloperRole: false, + requiresReasoningContentOnAssistantMessages: true, + }); + } + if (model.id === "z-ai/glm-5") model.cost = { ...model.cost, input: 0.6, output: 1.9, cacheRead: 0.119 }; + return model; +} + +// Router aliases. Their price depends on the model they route to, so it is unknown here. +const ROUTER_MODELS: Model<"openai-completions">[] = [ + { + id: "auto", + name: "Auto", + api: "openai-completions", + provider: "openrouter", + baseUrl: BASE_URL, + reasoning: true, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 2000000, + maxTokens: 30000, + }, + // OpenRouter exposes Fusion as a router alias/plugin entry point; its model + // metadata does not advertise tools, but the alias resolves to a concrete model + // that can invoke caller tools and has the openrouter:fusion server tool auto-injected. + { + id: "openrouter/fusion", + name: "OpenRouter: Fusion", + api: "openai-completions", + provider: "openrouter", + baseUrl: BASE_URL, + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1000000, + maxTokens: 30000, + }, +]; + +export default defineProvider({ + id: "openrouter", + models: (ctx) => + addMissing(structuredClone(ctx.sources.openRouter.chat).map(correctUpstream), structuredClone(ROUTER_MODELS)), + images: (ctx) => structuredClone(ctx.sources.openRouter.images), + classifiers: (ctx) => structuredClone(ctx.sources.openRouter.classifiers), + compat: { + "openai-completions": (model): OpenAICompletionsCompat => ({ + // OpenRouter passes developer messages through to Anthropic and OpenAI models only. + ...(/^(anthropic|openai)\//.test(model.id) ? {} : { supportsDeveloperRole: false }), + thinkingFormat: "openrouter", + supportsStrictMode: true, + ...(/^~?anthropic\//.test(model.id) ? { cacheControlFormat: "anthropic" } : {}), + sendSessionAffinityHeaders: true, + }), + }, + finalize: (model) => { + if ( + model.api === "anthropic-messages" && + supportsClaudeMidConvoEffort(model.id) && + !MID_CONVO_EFFORT_UNSUPPORTED_MODEL_IDS.has(model.id) + ) { + enableClaudeMidConvoEffort(model as Model<"anthropic-messages">); + } + if (model.id.startsWith("inception/mercury-2")) { + // Mercury 2 in instant mode (reasoning_effort: "none") disables tool calling. + // Mark "off" unsupported so the openai-completions provider omits the reasoning param + // instead of defaulting to {reasoning:{effort:"none"}}. + // Pi's low/medium/high pass through verbatim; OpenRouter normalizes to Mercury's vocabulary. + mergeThinking(model, { off: null }); + } + if (model.id === "z-ai/glm-5.2") mergeThinking(model, { xhigh: "xhigh" }); + // OpenAI models behind OpenRouter accept plain system text in place. + if ( + model.api === "openai-completions" && + model.id.startsWith("openai/") && + OPENAI_MID_CONVO_MODEL_IDS.has(model.id.slice("openai/".length)) + ) { + mergeCompat(model as Model<"openai-completions">, { supportsMidConvoSystemMessages: true }); + } + }, +}); diff --git a/packages/ai/catalog/providers/qwen-token-plan-cn.ts b/packages/ai/catalog/providers/qwen-token-plan-cn.ts new file mode 100644 index 000000000..33b210dd9 --- /dev/null +++ b/packages/ai/catalog/providers/qwen-token-plan-cn.ts @@ -0,0 +1,13 @@ +import { defineProvider } from "../define.ts"; +import { QWEN_COMPLETIONS_COMPAT, qwenTokenPlanModels } from "./qwen-token-plan.ts"; + +export default defineProvider({ + id: "qwen-token-plan-cn", + models: (ctx) => + qwenTokenPlanModels( + ctx, + "alibaba-token-plan-cn", + "https://token-plan.cn-beijing.maas.aliyuncs.com/compatible-mode/v1", + ), + compat: { "openai-completions": QWEN_COMPLETIONS_COMPAT }, +}); diff --git a/packages/ai/catalog/providers/qwen-token-plan-individual.ts b/packages/ai/catalog/providers/qwen-token-plan-individual.ts new file mode 100644 index 000000000..c172748d1 --- /dev/null +++ b/packages/ai/catalog/providers/qwen-token-plan-individual.ts @@ -0,0 +1,34 @@ +import { assertExactModelIds } from "../../scripts/model-data.ts"; +import { defineProvider } from "../define.ts"; +import { INTERNATIONAL_BASE_URL, QWEN_COMPLETIONS_COMPAT, qwenTokenPlanModels } from "./qwen-token-plan.ts"; + +// QwenCloud Token Plan Individual text-model allowlist, verified 2026-09-03. +// It shares the international endpoint with a narrower catalog. +// https://docs.qwencloud.com/token-plan/personal/token-plan-personal-overview +const MODEL_IDS = new Set([ + "deepseek-v4-flash-0731", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "glm-5.2", + "qwen3.6-flash", + "qwen3.7-max", + "qwen3.7-plus", + "qwen3.8-flash", + "qwen3.8-max", +]); + +export default defineProvider({ + id: "qwen-token-plan-individual", + models: (ctx) => { + const models = qwenTokenPlanModels(ctx, "alibaba-token-plan", INTERNATIONAL_BASE_URL, MODEL_IDS); + if (ctx.strict) { + assertExactModelIds( + ctx.providerId, + MODEL_IDS, + models.map((model) => model.id), + ); + } + return models; + }, + compat: { "openai-completions": QWEN_COMPLETIONS_COMPAT }, +}); diff --git a/packages/ai/catalog/providers/qwen-token-plan.ts b/packages/ai/catalog/providers/qwen-token-plan.ts new file mode 100644 index 000000000..b526a9e28 --- /dev/null +++ b/packages/ai/catalog/providers/qwen-token-plan.ts @@ -0,0 +1,53 @@ +import type { Model, OpenAICompletionsCompat } from "../../src/types.ts"; +import { defineProvider, modelsDevModels, type ProviderContext } from "../define.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; + +// Alibaba Cloud Model Studio Token Plan. International and China use separate +// endpoints and API keys (sk-sp- prefix). models.dev lists them as +// "alibaba-token-plan[-cn]". +export const INTERNATIONAL_BASE_URL = "https://token-plan.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1"; + +const COMPAT: OpenAICompletionsCompat = { + thinkingFormat: "qwen", + supportsDeveloperRole: false, + supportsStore: false, + supportsReasoningEffort: true, +}; +const FALLBACK_THINKING_LEVELS = { + minimal: null, + low: null, + medium: null, + high: "high", + xhigh: null, + max: "max", +} as const; +const REASONING_EFFORT_FALLBACK_MODEL_IDS = new Set(["glm-5", "glm-5.1"]); +// Retired preview id — models.dev may still list it after GA ships. +const EXCLUDED_MODEL_IDS = new Set(["qwen3.8-max-preview"]); + +export function qwenTokenPlanModels(ctx: ProviderContext, source: string, baseUrl: string, modelIds?: Set) { + return modelsDevModels(ctx, source, { + api: "openai-completions", + baseUrl, + recordReasoning: false, + include: (model) => !EXCLUDED_MODEL_IDS.has(model.id) && (!modelIds || modelIds.has(model.id)), + map: (model, source): Model<"openai-completions"> => { + const thinkingLevelMap = + getEffortThinkingLevelMap(source.reasoningOptions ?? []) ?? + (REASONING_EFFORT_FALLBACK_MODEL_IDS.has(model.id) ? FALLBACK_THINKING_LEVELS : undefined); + return { + ...model, + compat: thinkingLevelMap ? COMPAT : { ...COMPAT, supportsReasoningEffort: false }, + ...(thinkingLevelMap ? { thinkingLevelMap } : {}), + }; + }, + }); +} + +export const QWEN_COMPLETIONS_COMPAT = { supportsStrictMode: true } as const; + +export default defineProvider({ + id: "qwen-token-plan", + models: (ctx) => qwenTokenPlanModels(ctx, "alibaba-token-plan", INTERNATIONAL_BASE_URL), + compat: { "openai-completions": QWEN_COMPLETIONS_COMPAT }, +}); diff --git a/packages/ai/catalog/providers/radius.ts b/packages/ai/catalog/providers/radius.ts new file mode 100644 index 000000000..3c1b9deb4 --- /dev/null +++ b/packages/ai/catalog/providers/radius.ts @@ -0,0 +1,7 @@ +import { defineProvider } from "../define.ts"; + +// The unauthenticated public catalog; authenticated clients overlay it at runtime. +export default defineProvider({ + id: "radius", + models: (ctx) => structuredClone(ctx.sources.radius), +}); diff --git a/packages/ai/catalog/providers/together.ts b/packages/ai/catalog/providers/together.ts new file mode 100644 index 000000000..30cea723b --- /dev/null +++ b/packages/ai/catalog/providers/together.ts @@ -0,0 +1,58 @@ +import type { OpenAICompletionsCompat, ThinkingLevelMap } from "../../src/types.ts"; +import { defineProvider, modelsDevModels } from "../define.ts"; + +const BASE_COMPAT: OpenAICompletionsCompat = { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + supportsStrictMode: false, + supportsLongCacheRetention: false, +}; +const TOGGLE_REASONING_COMPAT: OpenAICompletionsCompat = { ...BASE_COMPAT, thinkingFormat: "together" }; +const REASONING_EFFORT_COMPAT: OpenAICompletionsCompat = { + ...BASE_COMPAT, + supportsReasoningEffort: true, + thinkingFormat: "openai", +}; +const TOGGLE_REASONING_EFFORT_COMPAT: OpenAICompletionsCompat = { + ...TOGGLE_REASONING_COMPAT, + supportsReasoningEffort: true, +}; + +const REASONING_EFFORT_MODELS = new Set(["openai/gpt-oss-20b", "openai/gpt-oss-120b"]); +const TOGGLE_REASONING_EFFORT_MODELS = new Set(["deepseek-ai/DeepSeek-V4-Pro-0813"]); + +const REASONING_EFFORT_LEVELS: ThinkingLevelMap = { off: null, minimal: null }; +const DEEPSEEK_V4_LEVELS: ThinkingLevelMap = { minimal: null, low: null, medium: null, high: "high", xhigh: null }; +const TOGGLE_REASONING_LEVELS: ThinkingLevelMap = { minimal: null, low: null, medium: null }; + +export default defineProvider({ + id: "together", + models: (ctx) => + modelsDevModels(ctx, "togetherai", { + api: "openai-completions", + baseUrl: "https://api.together.ai/v1", + skipDeprecated: true, + map: (model) => { + if (!model.reasoning) return { ...model, compat: BASE_COMPAT }; + if (REASONING_EFFORT_MODELS.has(model.id)) { + return { ...model, compat: REASONING_EFFORT_COMPAT, thinkingLevelMap: { ...REASONING_EFFORT_LEVELS } }; + } + if (TOGGLE_REASONING_EFFORT_MODELS.has(model.id)) { + return { ...model, compat: TOGGLE_REASONING_EFFORT_COMPAT, thinkingLevelMap: { ...DEEPSEEK_V4_LEVELS } }; + } + return { ...model, compat: TOGGLE_REASONING_COMPAT, thinkingLevelMap: { ...TOGGLE_REASONING_LEVELS } }; + }, + }), + compat: { + "openai-completions": { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + thinkingFormat: "together", + supportsLongCacheRetention: false, + }, + }, +}); diff --git a/packages/ai/catalog/providers/typesafe.ts b/packages/ai/catalog/providers/typesafe.ts new file mode 100644 index 000000000..384c19831 --- /dev/null +++ b/packages/ai/catalog/providers/typesafe.ts @@ -0,0 +1,32 @@ +import type { ClassifierModel } from "../../src/types.ts"; +import { defineProvider } from "../define.ts"; + +export default defineProvider({ + id: "typesafe", + classifiers: (ctx): ClassifierModel<"typesafe-system-one">[] => { + const metadata = ctx.sources.modelsDevDecisions["typesafe/jev-latest"]; + if (!metadata || metadata.type !== "decision") { + if (ctx.strict) throw new Error("models.dev did not return decision model typesafe/jev-latest"); + return []; + } + return [ + { + type: "classifier", + id: "jev-latest", + name: metadata.name, + api: "typesafe-system-one", + provider: "typesafe", + baseUrl: "https://api.typesafe.ai/v1/", + // models.dev lists text input only, but the API accepts image assets as an experimental + // feature behind an opt-in header, which the typesafe-system-one API sends. + // https://github.com/typesafe-ai/typesafe-sdk-js (v0.7.0 changelog) + input: ["text", "image"], + // The canonical models.dev entry has no direct-provider pricing. System One reports token usage, + // so classify() results carry token counts but price them at zero. + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: metadata.limit?.context || 64000, + compat: { imageInput: "assets", sendImageOptInHeader: true }, + }, + ]; + }, +}); diff --git a/packages/ai/catalog/providers/vercel-ai-gateway.ts b/packages/ai/catalog/providers/vercel-ai-gateway.ts new file mode 100644 index 000000000..e414e235c --- /dev/null +++ b/packages/ai/catalog/providers/vercel-ai-gateway.ts @@ -0,0 +1,14 @@ +import { defineProvider } from "../define.ts"; +import { isKimiK3FamilyId, KIMI_K3_MAX_TOKENS } from "../families/kimi.ts"; + +export default defineProvider({ + id: "vercel-ai-gateway", + models: (ctx) => + structuredClone(ctx.sources.aiGateway.chat).map((model) => + // Keep Kimi K3's canonical output limit when gateway metadata is missing or incorrect. + model.id.startsWith("moonshotai/") && isKimiK3FamilyId(model.id.slice("moonshotai/".length)) + ? { ...model, maxTokens: KIMI_K3_MAX_TOKENS } + : model, + ), + classifiers: (ctx) => structuredClone(ctx.sources.aiGateway.classifiers), +}); diff --git a/packages/ai/catalog/providers/xai.ts b/packages/ai/catalog/providers/xai.ts new file mode 100644 index 000000000..1a189f82b --- /dev/null +++ b/packages/ai/catalog/providers/xai.ts @@ -0,0 +1,29 @@ +import { defineProvider, mergeThinking, modelsDevModels } from "../define.ts"; + +// Retired or unsuitable for pi's built-in catalog. +const EXCLUDED_MODEL_IDS = new Set([ + "grok-3", + "grok-3-fast", + "grok-4.20-0309-non-reasoning", + "grok-4.20-0309-reasoning", + "grok-build-0.1", + "grok-code-fast-1", +]); + +export default defineProvider({ + id: "xai", + models: (ctx) => + modelsDevModels(ctx, "xai", { + api: "openai-responses", + baseUrl: "https://api.x.ai/v1", + compat: { supportsLongCacheRetention: false }, + include: (source) => !EXCLUDED_MODEL_IDS.has(source.id), + }), + finalize: (model) => { + // Models without verified effort options must not send the undocumented + // "none"/"minimal" efforts. + if (model.api === "openai-responses" && model.thinkingLevelMap === undefined) { + mergeThinking(model, { off: null, minimal: null }); + } + }, +}); diff --git a/packages/ai/catalog/providers/xiaomi-token-plan-ams.ts b/packages/ai/catalog/providers/xiaomi-token-plan-ams.ts new file mode 100644 index 000000000..6f8c6ce12 --- /dev/null +++ b/packages/ai/catalog/providers/xiaomi-token-plan-ams.ts @@ -0,0 +1,8 @@ +import { defineProvider } from "../define.ts"; +import { XIAOMI_COMPLETIONS_COMPAT, xiaomiModels } from "./xiaomi.ts"; + +export default defineProvider({ + id: "xiaomi-token-plan-ams", + models: (ctx) => xiaomiModels(ctx, "https://token-plan-ams.xiaomimimo.com/v1"), + compat: { "openai-completions": XIAOMI_COMPLETIONS_COMPAT }, +}); diff --git a/packages/ai/catalog/providers/xiaomi-token-plan-cn.ts b/packages/ai/catalog/providers/xiaomi-token-plan-cn.ts new file mode 100644 index 000000000..a7a371860 --- /dev/null +++ b/packages/ai/catalog/providers/xiaomi-token-plan-cn.ts @@ -0,0 +1,8 @@ +import { defineProvider } from "../define.ts"; +import { XIAOMI_COMPLETIONS_COMPAT, xiaomiModels } from "./xiaomi.ts"; + +export default defineProvider({ + id: "xiaomi-token-plan-cn", + models: (ctx) => xiaomiModels(ctx, "https://token-plan-cn.xiaomimimo.com/v1"), + compat: { "openai-completions": XIAOMI_COMPLETIONS_COMPAT }, +}); diff --git a/packages/ai/catalog/providers/xiaomi-token-plan-sgp.ts b/packages/ai/catalog/providers/xiaomi-token-plan-sgp.ts new file mode 100644 index 000000000..d3614b3ad --- /dev/null +++ b/packages/ai/catalog/providers/xiaomi-token-plan-sgp.ts @@ -0,0 +1,8 @@ +import { defineProvider } from "../define.ts"; +import { XIAOMI_COMPLETIONS_COMPAT, xiaomiModels } from "./xiaomi.ts"; + +export default defineProvider({ + id: "xiaomi-token-plan-sgp", + models: (ctx) => xiaomiModels(ctx, "https://token-plan-sgp.xiaomimimo.com/v1"), + compat: { "openai-completions": XIAOMI_COMPLETIONS_COMPAT }, +}); diff --git a/packages/ai/catalog/providers/xiaomi.ts b/packages/ai/catalog/providers/xiaomi.ts new file mode 100644 index 000000000..3129bf139 --- /dev/null +++ b/packages/ai/catalog/providers/xiaomi.ts @@ -0,0 +1,46 @@ +import type { Model } from "../../src/types.ts"; +import { defineProvider, modelsDevModels, type ProviderContext } from "../define.ts"; + +/** + * Xiaomi MiMo. The `xiaomi` provider targets the API billing endpoint; the + * `xiaomi-token-plan-*` providers cover prepaid Token Plan endpoints per region. + * MiMo Chat Completions only documents thinking.type enabled/disabled and + * rejects some reasoning_effort values (e.g. "minimal"). Xiaomi documents all + * effort levels as identical, so expose only off and high. + */ +export function xiaomiModels(ctx: ProviderContext, baseUrl: string) { + return modelsDevModels(ctx, ctx.providerId, { + api: "openai-completions", + baseUrl, + skipDeprecated: true, + recordReasoning: false, + compat: { + supportsReasoningEffort: false, + requiresReasoningContentOnAssistantMessages: true, + thinkingFormat: "deepseek", + }, + map: (model): Model<"openai-completions"> => + model.reasoning + ? { + ...model, + thinkingLevelMap: { + off: "off", + minimal: null, + low: null, + medium: null, + high: "high", + xhigh: null, + max: null, + }, + } + : model, + }); +} + +export const XIAOMI_COMPLETIONS_COMPAT = { supportsStrictMode: true } as const; + +export default defineProvider({ + id: "xiaomi", + models: (ctx) => xiaomiModels(ctx, "https://api.xiaomimimo.com/v1"), + compat: { "openai-completions": XIAOMI_COMPLETIONS_COMPAT }, +}); diff --git a/packages/ai/catalog/providers/zai-coding-cn.ts b/packages/ai/catalog/providers/zai-coding-cn.ts new file mode 100644 index 000000000..2321572ec --- /dev/null +++ b/packages/ai/catalog/providers/zai-coding-cn.ts @@ -0,0 +1,8 @@ +import { defineProvider } from "../define.ts"; +import { ZAI_COMPLETIONS_COMPAT, zaiCodingPlanModels } from "./zai.ts"; + +export default defineProvider({ + id: "zai-coding-cn", + models: (ctx) => zaiCodingPlanModels(ctx, "zhipuai-coding-plan", "https://open.bigmodel.cn/api/coding/paas/v4"), + compat: { "openai-completions": ZAI_COMPLETIONS_COMPAT }, +}); diff --git a/packages/ai/catalog/providers/zai.ts b/packages/ai/catalog/providers/zai.ts new file mode 100644 index 000000000..540d30d79 --- /dev/null +++ b/packages/ai/catalog/providers/zai.ts @@ -0,0 +1,50 @@ +import type { Model } from "../../src/types.ts"; +import { defineProvider, modelsDevModels, type ProviderContext } from "../define.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; +import { modelsDevChatModels, modelsDevCost } from "../sources/models-dev.ts"; + +// Tool-call streaming (`tool_stream`) is rejected by the GLM-4.5 generation. +const TOOL_STREAM_UNSUPPORTED_MODEL_IDS = new Set(["glm-4.5", "glm-4.5-air", "glm-4.5-flash", "glm-4.5v"]); + +/** The Z.AI coding plans. The international and China plans list the same models. */ +export function zaiCodingPlanModels(ctx: ProviderContext, source: string, baseUrl: string) { + // Coding-plan entries carry plan prices; show the pay-as-you-go API rates. + const apiModels = new Map(modelsDevChatModels(ctx.sources.modelsDev, "zai").map((model) => [model.id, model])); + return modelsDevModels(ctx, source, { + api: "openai-completions", + baseUrl, + map: (model, source): Model<"openai-completions"> => { + const thinkingLevelMap = getEffortThinkingLevelMap(source.reasoningOptions ?? []); + if (thinkingLevelMap && (model.id === "glm-5.2" || model.id === "glm-5.2-highspeed")) { + thinkingLevelMap.off = "none"; + } + const apiModel = apiModels.get(model.id); + return { + ...model, + ...(thinkingLevelMap ? { thinkingLevelMap } : {}), + cost: modelsDevCost(apiModel?.source.cost ? apiModel.cost : source.cost), + compat: { + supportsDeveloperRole: false, + thinkingFormat: "zai", + ...(thinkingLevelMap ? { supportsReasoningEffort: true } : {}), + ...(!TOOL_STREAM_UNSUPPORTED_MODEL_IDS.has(model.id) ? { zaiToolStream: true } : {}), + }, + }; + }, + }); +} + +export const ZAI_COMPLETIONS_COMPAT = { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + thinkingFormat: "zai", + supportsStrictMode: true, +} as const; + +export default defineProvider({ + id: "zai", + models: (ctx) => zaiCodingPlanModels(ctx, "zai-coding-plan", "https://api.z.ai/api/coding/paas/v4"), + compat: { "openai-completions": ZAI_COMPLETIONS_COMPAT }, +}); diff --git a/packages/ai/scripts/models-dev-reasoning-options.ts b/packages/ai/catalog/reasoning.ts similarity index 79% rename from packages/ai/scripts/models-dev-reasoning-options.ts rename to packages/ai/catalog/reasoning.ts index 9d0711395..389f39382 100644 --- a/packages/ai/scripts/models-dev-reasoning-options.ts +++ b/packages/ai/catalog/reasoning.ts @@ -1,12 +1,5 @@ import type { ThinkingLevel, ThinkingLevelMap } from "../src/types.ts"; - -export type ModelsDevReasoningOption = - | { type: "toggle" } - | { - type: "effort"; - values: Array<"none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max" | "default" | null>; - } - | { type: "budget_tokens"; min?: number; max?: number }; +import type { ModelsDevReasoningOption } from "./sources/models-dev.ts"; const THINKING_LEVELS: readonly ThinkingLevel[] = ["minimal", "low", "medium", "high", "xhigh", "max"]; diff --git a/packages/ai/catalog/sources/index.ts b/packages/ai/catalog/sources/index.ts new file mode 100644 index 000000000..b043778b9 --- /dev/null +++ b/packages/ai/catalog/sources/index.ts @@ -0,0 +1,95 @@ +import { + DEFAULT_RADIUS_GATEWAY, + getRadiusModelsFromConfig, + loadRadiusGatewayConfig, +} from "../../src/providers/radius-config.ts"; +import type { Model } from "../../src/types.ts"; +import { + MODELS_DEV_DECISIONS_URL, + MODELS_DEV_URL, + type ModelsDevDecisionMetadata, + type ModelsDevV2Catalog, +} from "./models-dev.ts"; +import { fetchOpenRouterCatalog, type OpenRouterCatalog } from "./openrouter.ts"; +import { type AiGatewayCatalog, fetchAiGatewayCatalog } from "./vercel-ai-gateway.ts"; + +export const NVIDIA_BASE_URL = "https://integrate.api.nvidia.com/v1"; + +/** Everything fetched from upstream before the catalog is built. */ +export interface CatalogSources { + modelsDev: ModelsDevV2Catalog; + modelsDevDecisions: Record; + openRouter: OpenRouterCatalog; + aiGateway: AiGatewayCatalog; + /** Live NVIDIA NIM ids, keyed by both the raw and the normalized id. */ + nvidiaModelIds: Map; + radius: Model<"pi-messages">[]; +} + +async function fetchJson(url: string, label: string): Promise { + const response = await fetch(url); + if (!response.ok) throw new Error(`${label} returned ${response.status}`); + return (await response.json()) as T; +} + +export function normalizeNvidiaModelId(modelId: string): string { + return modelId.toLowerCase().replaceAll("_", "."); +} + +async function fetchNvidiaModelIds(): Promise> { + const data = await fetchJson<{ data?: { id: string }[] }>(`${NVIDIA_BASE_URL}/models`, "NVIDIA NIM API"); + const modelIds = new Map(); + for (const model of data.data ?? []) { + modelIds.set(model.id, model.id); + modelIds.set(normalizeNvidiaModelId(model.id), model.id); + } + return modelIds; +} + +async function fetchRadiusModels(): Promise[]> { + const config = await loadRadiusGatewayConfig(DEFAULT_RADIUS_GATEWAY); + const models = getRadiusModelsFromConfig("radius", config); + if (models.length === 0) throw new Error("Radius API returned no models"); + return models; +} + +/** + * Fetch every upstream source. In strict mode any failure aborts generation; + * otherwise a failed source contributes nothing. + */ +export async function loadCatalogSources(strict: boolean): Promise { + async function load(label: string, fetcher: () => Promise, fallback: T): Promise { + try { + console.log(`Fetching ${label}...`); + return await fetcher(); + } catch (error) { + console.error(`Failed to fetch ${label}:`, error); + if (strict) throw error; + return fallback; + } + } + + const modelsDev = await load( + "models.dev catalog", + () => fetchJson(MODELS_DEV_URL, "models.dev API"), + {}, + ); + const modelsDevDecisions = await load( + "models.dev decision models", + () => fetchJson>(MODELS_DEV_DECISIONS_URL, "models.dev decision API"), + {}, + ); + const nvidiaModelIds = modelsDev.nvidia?.models + ? await load("NVIDIA NIM models", fetchNvidiaModelIds, new Map()) + : new Map(); + const openRouter = await load("OpenRouter models", fetchOpenRouterCatalog, { + chat: [], + images: [], + classifiers: [], + }); + if (strict && openRouter.images.length === 0) throw new Error("OpenRouter API returned no usable image models"); + const aiGateway = await load("Vercel AI Gateway models", fetchAiGatewayCatalog, { chat: [], classifiers: [] }); + const radius = await load("Radius models", fetchRadiusModels, []); + + return { modelsDev, modelsDevDecisions, openRouter, aiGateway, nvidiaModelIds, radius }; +} diff --git a/packages/ai/catalog/sources/models-dev.ts b/packages/ai/catalog/sources/models-dev.ts new file mode 100644 index 000000000..3928dc438 --- /dev/null +++ b/packages/ai/catalog/sources/models-dev.ts @@ -0,0 +1,242 @@ +import type { ModelCost } from "../../src/types.ts"; + +// models.dev v2 catalog. The schema is published as experimental, so everything +// the catalog reads goes through `normalizeModelsDevModel()` below. +// https://github.com/anomalyco/models.dev/blob/dev/packages/core/src/schema-v2.ts +export const MODELS_DEV_URL = "https://models.dev/experimental/v2.0/api.json"; +// v2 has no provider-agnostic decision metadata endpoint; v1 still serves it. +export const MODELS_DEV_DECISIONS_URL = "https://models.dev/models.json?type=decision"; + +export type ModelsDevReasoningOption = + | { type: "toggle" } + | { + type: "effort"; + values: Array<"none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max" | "default" | null>; + } + | { type: "budget_tokens"; min?: number; max?: number }; + +type ModelsDevEffort = Extract["values"][number]; + +interface V2CachePrices { + read?: number; + write?: number | Record; + explicit?: { read?: number; write?: number | Record }; +} + +interface V2CostOverride { + when: Record; + price?: number; + cache?: V2CachePrices; +} + +interface V2ModalityCost { + input?: { unit?: string; price: number; cache?: V2CachePrices; overrides?: V2CostOverride[] }; + output?: { unit?: string; price: number; overrides?: V2CostOverride[] }; +} + +interface V2ApiEntry { + base_url: string; + path?: string; + sdk?: { aisdk?: string }; + compatibility?: Record; + capabilities?: Record; +} + +/** The subset of a models.dev v2 model the catalog reads. */ +export interface ModelsDevV2Model { + id: string; + type: string; + name: string; + status?: string; + modalities: { input: string[]; output: string[] }; + capabilities: { + tools: { supported: boolean }; + reasoning: { + supported: boolean; + toggle?: boolean; + effort?: ModelsDevEffort[]; + budget?: { min?: number; max?: number }; + }; + structured_output?: boolean; + }; + limit?: { context?: number; input?: number; output?: number }; + cost?: { text?: V2ModalityCost }; + api: Record; +} + +export type ModelsDevV2Catalog = Record }>; + +/** Prices as models.dev lists them. Missing prices stay undefined. */ +export interface ModelsDevCost { + input?: number; + output?: number; + cacheRead?: number; + cacheWrite?: number; + /** Context-size tiers. Unlisted rates fall back to the base rates. */ + tiers?: Array<{ above: number; input?: number; output?: number; cacheRead?: number; cacheWrite?: number }>; +} + +/** One API a model is served on, in the order models.dev lists them. */ +export interface ModelsDevApi { + protocol: string; + baseUrl: string; + aisdk?: string; +} + +/** A models.dev model normalized for the catalog. */ +export interface ModelsDevModel { + id: string; + name: string; + type: string; + status?: string; + toolCall: boolean; + structuredOutput: boolean; + reasoning: boolean; + /** Undefined when the model does not reason. */ + reasoningOptions?: ModelsDevReasoningOption[]; + imageInput: boolean; + textInput: boolean; + textOutput: boolean; + contextWindow?: number; + maxTokens?: number; + cost: ModelsDevCost; + apis: ModelsDevApi[]; + /** The full v2 entry, for fields not normalized above. */ + source: ModelsDevV2Model; +} + +/** Provider-agnostic decision model metadata from models.dev. */ +export interface ModelsDevDecisionMetadata { + id: string; + type?: string; + name: string; + limit?: { context?: number }; +} + +function firstPrice(write: number | Record | undefined): number | undefined { + return typeof write === "object" ? Object.values(write)[0] : write; +} + +// models.dev lists one cache write price per TTL; the first is the host's default. +function cacheWritePrice(cache: V2CachePrices | undefined): number | undefined { + return firstPrice(cache?.write) ?? firstPrice(cache?.explicit?.write); +} + +function contextThreshold(when: Record): number | undefined { + const keys = Object.keys(when); + if (keys.length !== 1 || keys[0] !== "context") return undefined; + const context = when.context as { gt?: unknown; gte?: unknown }; + const size = context.gt ?? context.gte; + return typeof size === "number" ? size : undefined; +} + +function normalizeCost(cost: ModelsDevV2Model["cost"]): ModelsDevCost { + const input = cost?.text?.input; + const output = cost?.text?.output; + const perToken = (unit: string | undefined) => unit === undefined || unit === "token"; + if (!input || !output || !perToken(input.unit) || !perToken(output.unit)) return {}; + + // Only context-size overrides map onto pi's cost tiers. Region, service-tier, + // and time-based prices have no pi equivalent yet. + const tiers = new Map[number]>(); + const tier = (above: number) => { + let entry = tiers.get(above); + if (!entry) { + entry = { above }; + tiers.set(above, entry); + } + return entry; + }; + for (const override of input.overrides ?? []) { + const above = contextThreshold(override.when); + if (above === undefined) continue; + const entry = tier(above); + if (override.price !== undefined) entry.input = override.price; + if (override.cache?.read !== undefined) entry.cacheRead = override.cache.read; + const cacheWrite = cacheWritePrice(override.cache); + if (cacheWrite !== undefined) entry.cacheWrite = cacheWrite; + } + for (const override of output.overrides ?? []) { + const above = contextThreshold(override.when); + if (above === undefined) continue; + if (override.price !== undefined) tier(above).output = override.price; + } + + return { + input: input.price, + output: output.price, + cacheRead: input.cache?.read, + cacheWrite: cacheWritePrice(input.cache), + ...(tiers.size > 0 ? { tiers: [...tiers.values()] } : {}), + }; +} + +function reasoningOptions(reasoning: ModelsDevV2Model["capabilities"]["reasoning"]): ModelsDevReasoningOption[] { + const options: ModelsDevReasoningOption[] = []; + if (reasoning.toggle) options.push({ type: "toggle" }); + if (reasoning.effort !== undefined) options.push({ type: "effort", values: reasoning.effort }); + if (reasoning.budget !== undefined) options.push({ type: "budget_tokens", ...reasoning.budget }); + return options; +} + +export function normalizeModelsDevModel(model: ModelsDevV2Model): ModelsDevModel { + const input = model.modalities?.input ?? []; + const reasoning = model.capabilities?.reasoning; + return { + id: model.id, + name: model.name, + type: model.type, + status: model.status, + toolCall: model.capabilities?.tools?.supported === true, + structuredOutput: model.capabilities?.structured_output === true, + reasoning: reasoning?.supported === true, + reasoningOptions: reasoning?.supported ? reasoningOptions(reasoning) : undefined, + imageInput: input.some((modality) => modality === "image" || modality.startsWith("image/")), + textInput: input.includes("text"), + textOutput: (model.modalities?.output ?? []).includes("text"), + contextWindow: model.limit?.context, + maxTokens: model.limit?.output, + cost: normalizeCost(model.cost), + apis: Object.entries(model.api ?? {}).map(([protocol, entry]) => ({ + protocol, + baseUrl: entry.base_url, + aisdk: entry.sdk?.aisdk, + })), + source: model, + }; +} + +export function roundCost(value: number): number { + return Number(value.toFixed(6)); +} + +/** Converts models.dev prices to pi's cost shape. Missing rates are free. */ +export function modelsDevCost(cost: ModelsDevCost): ModelCost { + const base = { + input: cost.input || 0, + output: cost.output || 0, + cacheRead: cost.cacheRead || 0, + cacheWrite: cost.cacheWrite || 0, + }; + const tiers = cost.tiers?.map((tier) => ({ + inputTokensAbove: tier.above, + input: tier.input ?? base.input, + output: tier.output ?? base.output, + cacheRead: tier.cacheRead ?? base.cacheRead, + cacheWrite: tier.cacheWrite ?? base.cacheWrite, + })); + return tiers && tiers.length > 0 ? { ...base, tiers } : base; +} + +/** + * Chat models of one models.dev provider. Specialized types (image, speech, + * realtime, decision, ...) only appear in v2 and are left out here. + */ +export function modelsDevChatModels(catalog: ModelsDevV2Catalog, provider: string): ModelsDevModel[] { + return ( + Object.values(catalog[provider]?.models ?? {}) + // Unlike pi models, models.dev v2 always states the type. + .filter(({ type }) => type === "chat") + .map(normalizeModelsDevModel) + ); +} diff --git a/packages/ai/scripts/openrouter-catalog.ts b/packages/ai/catalog/sources/openrouter.ts similarity index 66% rename from packages/ai/scripts/openrouter-catalog.ts rename to packages/ai/catalog/sources/openrouter.ts index f0b7c0b03..a7e82a735 100644 --- a/packages/ai/scripts/openrouter-catalog.ts +++ b/packages/ai/catalog/sources/openrouter.ts @@ -1,6 +1,29 @@ -import { type OpenRouterModel, openRouterCost } from "../src/providers/openrouter-api.ts"; -import type { ClassifierModel, ImageModel, Model } from "../src/types.ts"; -import { getOpenRouterThinkingLevelMap, type OpenRouterReasoningMetadata } from "./openrouter-reasoning-options.ts"; +import { type OpenRouterModel, openRouterCost } from "../../src/providers/openrouter-api.ts"; +import type { ClassifierModel, ImageModel, Model, ThinkingLevel, ThinkingLevelMap } from "../../src/types.ts"; +import { getEffortThinkingLevelMap } from "../reasoning.ts"; + +const OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models"; + +export interface OpenRouterReasoningMetadata { + mandatory?: boolean; + default_enabled?: boolean; + supported_efforts?: Array; + default_effort?: ThinkingLevel | "none"; +} + +/** Convert OpenRouter's reasoning metadata into Pi model capabilities. */ +export function getOpenRouterThinkingLevelMap( + reasoning: OpenRouterReasoningMetadata | undefined, +): ThinkingLevelMap | undefined { + if (!reasoning) return undefined; + if (!reasoning.supported_efforts?.length) return reasoning.mandatory === true ? { off: null } : undefined; + + // OpenRouter's supported_efforts uses the same effort values as models.dev reasoning_options, + // so both sources can share the same Pi thinking-level conversion. + const map = getEffortThinkingLevelMap([{ type: "effort", values: reasoning.supported_efforts }]); + if (!map) return reasoning.mandatory === true ? { off: null } : undefined; + return { ...map, off: reasoning.mandatory === true ? null : "none" }; +} export interface OpenRouterModelListItem extends OpenRouterModel { name: string; @@ -106,3 +129,20 @@ export function buildOpenRouterCatalog( return { chat, images, classifiers }; } + +async function fetchOpenRouterList(query: string): Promise { + const response = await fetch(`${OPENROUTER_MODELS_URL}${query}`); + if (!response.ok) throw new Error(`OpenRouter API returned ${response.status}`); + const data = (await response.json()) as { data?: OpenRouterModelListItem[] }; + return data.data ?? []; +} + +/** Fetch the tool-capable chat, image, and decision listings. */ +export async function fetchOpenRouterCatalog(): Promise { + const [listed, imageListed, decisionListed] = await Promise.all([ + fetchOpenRouterList(""), + fetchOpenRouterList("?output_modalities=image"), + fetchOpenRouterList("?output_modalities=decisions"), + ]); + return buildOpenRouterCatalog(listed, imageListed, decisionListed); +} diff --git a/packages/ai/catalog/sources/vercel-ai-gateway.ts b/packages/ai/catalog/sources/vercel-ai-gateway.ts new file mode 100644 index 000000000..3da44b2d8 --- /dev/null +++ b/packages/ai/catalog/sources/vercel-ai-gateway.ts @@ -0,0 +1,140 @@ +import type { ClassifierModel, Model, ModelCost, ModelCostRates, ModelCostTier } from "../../src/types.ts"; + +const AI_GATEWAY_MODELS_URL = "https://ai-gateway.vercel.sh/v1/models"; +const AI_GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh"; +// TypeSafe-compatible System One endpoint for evaluation models. +// https://vercel.com/docs/ai-gateway/sdks-and-apis/typesafe +const AI_GATEWAY_TYPESAFE_BASE_URL = "https://ai-gateway.vercel.sh/typesafe/v1"; + +/** A prompt-length bracket. `min` is inclusive and `max` exclusive, both in prompt tokens. */ +export interface AiGatewayPriceTier { + cost?: string | number; + min?: number; + max?: number; +} + +/** Vercel AI Gateway pricing in $/token. */ +export interface AiGatewayPricing { + input?: string | number; + output?: string | number; + input_cache_read?: string | number; + input_cache_write?: string | number; + input_tiers?: AiGatewayPriceTier[]; + output_tiers?: AiGatewayPriceTier[]; + input_cache_read_tiers?: AiGatewayPriceTier[]; + input_cache_write_tiers?: AiGatewayPriceTier[]; +} + +const RATE_FIELDS = [ + { rate: "input", tiers: "input_tiers" }, + { rate: "output", tiers: "output_tiers" }, + { rate: "cacheRead", tiers: "input_cache_read_tiers" }, + { rate: "cacheWrite", tiers: "input_cache_write_tiers" }, +] as const; + +function perMillion(value: string | number | undefined): number { + const parsed = typeof value === "number" ? value : parseFloat(value ?? "0"); + return Number.isFinite(parsed) ? Number((parsed * 1_000_000).toFixed(6)) : 0; +} + +/** + * Convert AI Gateway pricing to $/million tokens. Each rate lists its own prompt-length brackets; + * every bracket start above zero becomes a request-wide tier with the rates in effect there. + */ +export function getAiGatewayCost(pricing: AiGatewayPricing | undefined): ModelCost { + const base: ModelCostRates = { + input: perMillion(pricing?.input), + output: perMillion(pricing?.output), + cacheRead: perMillion(pricing?.input_cache_read), + cacheWrite: perMillion(pricing?.input_cache_write), + }; + + const starts = new Set(); + for (const field of RATE_FIELDS) { + for (const bracket of pricing?.[field.tiers] ?? []) { + if (bracket.min !== undefined && bracket.min > 0) starts.add(bracket.min); + } + } + + const tiers = [...starts] + .sort((a, b) => a - b) + .map((start): ModelCostTier => { + const tier: ModelCostTier = { inputTokensAbove: start - 1, ...base }; + for (const field of RATE_FIELDS) { + const bracket = pricing?.[field.tiers]?.find( + (candidate) => (candidate.min ?? 0) <= start && (candidate.max === undefined || start < candidate.max), + ); + if (bracket?.cost !== undefined) tier[field.rate] = perMillion(bracket.cost); + } + return tier; + }); + + return tiers.length > 0 ? { ...base, tiers } : base; +} + +interface AiGatewayModel { + id: string; + name?: string; + type?: string; + context_window?: number; + max_tokens?: number; + tags?: string[]; + pricing?: AiGatewayPricing; +} + +export interface AiGatewayCatalog { + chat: Model<"anthropic-messages">[]; + classifiers: ClassifierModel<"typesafe-system-one">[]; +} + +/** Fetch tool-capable chat models and evaluation (classifier) models. */ +export async function fetchAiGatewayCatalog(): Promise { + const response = await fetch(AI_GATEWAY_MODELS_URL); + if (!response.ok) throw new Error(`Vercel AI Gateway API returned ${response.status}`); + const data = (await response.json()) as { data?: unknown }; + const items = Array.isArray(data.data) ? (data.data as AiGatewayModel[]) : []; + const chat: AiGatewayCatalog["chat"] = []; + const classifiers: AiGatewayCatalog["classifiers"] = []; + + for (const model of items) { + // Evaluation models such as TypeSafe's Jev are served through the + // TypeSafe-compatible System One endpoint. + if (model.type === "evaluation") { + classifiers.push({ + type: "classifier", + id: model.id, + name: model.name || model.id, + api: "typesafe-system-one", + provider: "vercel-ai-gateway", + baseUrl: AI_GATEWAY_TYPESAFE_BASE_URL, + input: ["text"], + cost: { + input: perMillion(model.pricing?.input), + output: perMillion(model.pricing?.output), + cacheRead: 0, + cacheWrite: 0, + }, + contextWindow: model.context_window || 4096, + }); + continue; + } + const tags = Array.isArray(model.tags) ? model.tags : []; + // Only include models that support tools + if (!tags.includes("tool-use")) continue; + + chat.push({ + id: model.id, + name: model.name || model.id, + api: "anthropic-messages", + baseUrl: AI_GATEWAY_BASE_URL, + provider: "vercel-ai-gateway", + reasoning: tags.includes("reasoning"), + input: tags.includes("vision") ? ["text", "image"] : ["text"], + compat: { allowEmptySignature: true }, + cost: getAiGatewayCost(model.pricing), + contextWindow: model.context_window || 4096, + maxTokens: model.max_tokens || 4096, + }); + } + return { chat, classifiers }; +} diff --git a/packages/ai/package.json b/packages/ai/package.json index ac6f865c6..6e04980f0 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -57,9 +57,9 @@ ], "scripts": { "clean": "shx rm -rf dist", - "generate-models": "node scripts/generate-models.ts --strict", - "hydrate-model-data": "node scripts/generate-models.ts --strict --data-only", - "generate-model-catalog": "node scripts/generate-models.ts --strict --json-only --json-output ../../.artifacts/model-catalog", + "generate-models": "node catalog/generate.ts --strict", + "hydrate-model-data": "node catalog/generate.ts --strict --data-only", + "generate-model-catalog": "node catalog/generate.ts --strict --json-only --json-output ../../.artifacts/model-catalog", "check:model-data": "node scripts/check-model-data.ts", "build": "npm run generate-models && npm run build:offline", "build:offline": "npm run check:model-data && tsc -p tsconfig.build.json && shx rm -rf dist/providers/data && shx cp -r src/providers/data dist/providers/data", diff --git a/packages/ai/scripts/ai-gateway-pricing.ts b/packages/ai/scripts/ai-gateway-pricing.ts deleted file mode 100644 index 74ac54b84..000000000 --- a/packages/ai/scripts/ai-gateway-pricing.ts +++ /dev/null @@ -1,67 +0,0 @@ -import type { ModelCost, ModelCostRates, ModelCostTier } from "../src/types.ts"; - -/** A prompt-length bracket. `min` is inclusive and `max` exclusive, both in prompt tokens. */ -export interface AiGatewayPriceTier { - cost?: string | number; - min?: number; - max?: number; -} - -/** Vercel AI Gateway pricing in $/token. */ -export interface AiGatewayPricing { - input?: string | number; - output?: string | number; - input_cache_read?: string | number; - input_cache_write?: string | number; - input_tiers?: AiGatewayPriceTier[]; - output_tiers?: AiGatewayPriceTier[]; - input_cache_read_tiers?: AiGatewayPriceTier[]; - input_cache_write_tiers?: AiGatewayPriceTier[]; -} - -const RATE_FIELDS = [ - { rate: "input", tiers: "input_tiers" }, - { rate: "output", tiers: "output_tiers" }, - { rate: "cacheRead", tiers: "input_cache_read_tiers" }, - { rate: "cacheWrite", tiers: "input_cache_write_tiers" }, -] as const; - -function perMillion(value: string | number | undefined): number { - const parsed = typeof value === "number" ? value : parseFloat(value ?? "0"); - return Number.isFinite(parsed) ? Number((parsed * 1_000_000).toFixed(6)) : 0; -} - -/** - * Convert AI Gateway pricing to $/million tokens. Each rate lists its own prompt-length brackets; - * every bracket start above zero becomes a request-wide tier with the rates in effect there. - */ -export function getAiGatewayCost(pricing: AiGatewayPricing | undefined): ModelCost { - const base: ModelCostRates = { - input: perMillion(pricing?.input), - output: perMillion(pricing?.output), - cacheRead: perMillion(pricing?.input_cache_read), - cacheWrite: perMillion(pricing?.input_cache_write), - }; - - const starts = new Set(); - for (const field of RATE_FIELDS) { - for (const bracket of pricing?.[field.tiers] ?? []) { - if (bracket.min !== undefined && bracket.min > 0) starts.add(bracket.min); - } - } - - const tiers = [...starts] - .sort((a, b) => a - b) - .map((start): ModelCostTier => { - const tier: ModelCostTier = { inputTokensAbove: start - 1, ...base }; - for (const field of RATE_FIELDS) { - const bracket = pricing?.[field.tiers]?.find( - (candidate) => (candidate.min ?? 0) <= start && (candidate.max === undefined || start < candidate.max), - ); - if (bracket?.cost !== undefined) tier[field.rate] = perMillion(bracket.cost); - } - return tier; - }); - - return tiers.length > 0 ? { ...base, tiers } : base; -} diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts deleted file mode 100644 index c63afb6e5..000000000 --- a/packages/ai/scripts/generate-models.ts +++ /dev/null @@ -1,3688 +0,0 @@ -#!/usr/bin/env node - -import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from "fs"; -import { dirname, join, resolve } from "path"; -import { fileURLToPath } from "url"; -import { getEffortThinkingLevelMap, type ModelsDevReasoningOption } from "./models-dev-reasoning-options.ts"; -import { type AiGatewayPricing, getAiGatewayCost } from "./ai-gateway-pricing.ts"; -import { buildOpenRouterCatalog, type OpenRouterCatalog, type OpenRouterModelListItem } from "./openrouter-catalog.ts"; -import { - CLOUDFLARE_AI_GATEWAY_ANTHROPIC_BASE_URL, - CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL, - CLOUDFLARE_AI_GATEWAY_OPENAI_BASE_URL, - CLOUDFLARE_WORKERS_AI_BASE_URL, - CLOUDFLARE_WORKERS_AI_REST_BASE_URL, -} from "../src/api/cloudflare.ts"; -import type { - AnthropicMessagesCompat, - AnyModel, - Api, - ClassifierApi, - ClassifierModel, - ImageApi, - ImageModel, - KnownProvider, - Model, - ModelCost, - ModelPromptCache, - OpenAICompletionsCompat, - OpenAIResponsesCompat, -} from "../src/types.ts"; -import { - assertExactModelIds, - createModelDataManifest, - groupProviderModelData, - type ModelDataStructure, - MODEL_DATA_MANIFEST_FILE, - readModelDataProviderIds, - validateGeneratedModelData, - validateModelDataDirectory, -} from "./model-data.ts"; -import { - DEFAULT_RADIUS_GATEWAY, - getRadiusModelsFromConfig, - loadRadiusGatewayConfig, -} from "../src/providers/radius-config.ts"; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); -const packageRoot = join(__dirname, ".."); - -function readGeneratorOptions(args: string[]): { - strict: boolean; - dataOnly: boolean; - jsonOnly: boolean; - jsonOutputDir: string | undefined; - pretty: boolean; -} { - let strict = false; - let dataOnly = false; - let jsonOnly = false; - let jsonOutputDir: string | undefined; - let pretty = false; - - for (let index = 0; index < args.length; index++) { - const arg = args[index]; - if (arg === "--strict") { - strict = true; - continue; - } - if (arg === "--data-only") { - dataOnly = true; - continue; - } - if (arg === "--json-only") { - jsonOnly = true; - continue; - } - if (arg === "--pretty") { - pretty = true; - continue; - } - if (arg === "--json-output") { - const value = args[++index]; - if (!value) throw new Error("--json-output requires a directory"); - jsonOutputDir = resolve(value); - continue; - } - throw new Error(`Unknown argument: ${arg}`); - } - - if (jsonOnly && !jsonOutputDir) throw new Error("--json-only requires --json-output"); - if (dataOnly && (jsonOnly || jsonOutputDir)) throw new Error("--data-only cannot be combined with JSON catalog output"); - return { strict, dataOnly, jsonOnly, jsonOutputDir, pretty }; -} - -const generatorOptions = readGeneratorOptions(process.argv.slice(2)); - -interface ModelsDevModel { - id: string; - name: string; - tool_call?: boolean; - structured_output?: boolean; - reasoning?: boolean; - reasoning_options?: ModelsDevReasoningOption[]; - status?: string; - limit?: { - context?: number; - output?: number; - }; - cost?: { - input?: number; - output?: number; - cache_read?: number; - cache_write?: number; - tiers?: { - input?: number; - output?: number; - cache_read?: number; - cache_write?: number; - tier?: { - type?: string; - size?: number; - }; - }[]; - }; - modalities?: { - input?: string[]; - output?: string[]; - }; - provider?: { - npm?: string; - }; -} - -interface ModelsDevProvider { - models?: Record; -} - -interface ModelsDevMetadata { - id: string; - type?: string; - name: string; - limit?: { context?: number }; - modalities?: { input?: string[] }; -} - -type ModelsDevCatalog = Record; - -interface NvidiaNimModelListItem { - id: string; -} - -interface AiGatewayModel { - id: string; - name?: string; - type?: string; - context_window?: number; - max_tokens?: number; - tags?: string[]; - pricing?: AiGatewayPricing; -} - -const COPILOT_STATIC_HEADERS = { - "User-Agent": "GitHubCopilotChat/0.35.0", - "Editor-Version": "vscode/1.107.0", - "Editor-Plugin-Version": "copilot-chat/0.35.0", - "Copilot-Integration-Id": "vscode-chat", -} as const; - -const TOGETHER_BASE_URL = "https://api.together.ai/v1"; -const TOGETHER_BASE_COMPAT: OpenAICompletionsCompat = { - supportsStore: false, - supportsDeveloperRole: false, - supportsReasoningEffort: false, - maxTokensField: "max_tokens", - supportsStrictMode: false, - supportsLongCacheRetention: false, -}; -const TOGETHER_TOGGLE_REASONING_COMPAT: OpenAICompletionsCompat = { - ...TOGETHER_BASE_COMPAT, - thinkingFormat: "together", -}; -const TOGETHER_REASONING_EFFORT_COMPAT: OpenAICompletionsCompat = { - ...TOGETHER_BASE_COMPAT, - supportsReasoningEffort: true, - thinkingFormat: "openai", -}; -const TOGETHER_TOGGLE_REASONING_EFFORT_COMPAT: OpenAICompletionsCompat = { - ...TOGETHER_TOGGLE_REASONING_COMPAT, - supportsReasoningEffort: true, -}; -const TOGETHER_REASONING_EFFORT_MODELS = new Set(["openai/gpt-oss-20b", "openai/gpt-oss-120b"]); -const TOGETHER_TOGGLE_REASONING_EFFORT_MODELS = new Set(["deepseek-ai/DeepSeek-V4-Pro-0813"]); -const TOGETHER_REASONING_EFFORT_LEVEL_MAP = { - off: null, - minimal: null, -} as const; -const TOGETHER_DEEPSEEK_V4_THINKING_LEVEL_MAP = { - minimal: null, - low: null, - medium: null, - high: "high", - xhigh: null, -} as const; -const TOGETHER_TOGGLE_REASONING_LEVEL_MAP = { - minimal: null, - low: null, - medium: null, -} as const; - -const AI_GATEWAY_MODELS_URL = "https://ai-gateway.vercel.sh/v1"; -const AI_GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh"; -// TypeSafe-compatible System One endpoint for evaluation models. -// https://vercel.com/docs/ai-gateway/sdks-and-apis/typesafe -const AI_GATEWAY_TYPESAFE_BASE_URL = "https://ai-gateway.vercel.sh/typesafe/v1"; -const VERTEX_BASE_URL = "https://{location}-aiplatform.googleapis.com"; -const NVIDIA_BASE_URL = "https://integrate.api.nvidia.com/v1"; -const NVIDIA_HEADERS = { - "NVCF-POLL-SECONDS": "3600", -} as const; -const NVIDIA_OPENAI_COMPAT: OpenAICompletionsCompat = { - supportsStore: false, - supportsDeveloperRole: false, - supportsReasoningEffort: false, - maxTokensField: "max_tokens", - supportsStrictMode: false, - supportsLongCacheRetention: false, -}; -const NVIDIA_NIM_UNSUPPORTED_MODELS = new Set([ - "abacusai/dracarys-llama-3.1-70b-instruct", - "bytedance/seed-oss-36b-instruct", - "deepseek-ai/deepseek-v4-flash", - "deepseek-ai/deepseek-v4-pro", - "google/gemma-2-2b-it", - "google/gemma-3n-e2b-it", - "google/gemma-3n-e4b-it", - "google/gemma-4-31b-it", - "meta/llama-3.2-1b-instruct", - "meta/llama-4-maverick-17b-128e-instruct", - "microsoft/phi-4-mini-instruct", - "minimaxai/minimax-m2.7", - "mistralai/mistral-nemotron", - "nvidia/nemotron-mini-4b-instruct", - "qwen/qwen3-next-80b-a3b-instruct", - "qwen/qwen3.5-397b-a17b", - "sarvamai/sarvam-m", - "upstage/solar-10.7b-instruct", -]); -const ZAI_TOOL_STREAM_UNSUPPORTED_MODELS = new Set(["glm-4.5", "glm-4.5-air", "glm-4.5-flash", "glm-4.5v"]); -const OPENCODE_GO_GLM52_THINKING_LEVEL_MAP = { - off: null, - minimal: null, - low: null, - medium: null, - high: "high", - max: "max", -} as const; -const EAGER_TOOL_INPUT_STREAMING_UNSUPPORTED_ANTHROPIC_MODELS = new Set([ - "github-copilot:claude-haiku-4.5", - "github-copilot:claude-sonnet-4", - "github-copilot:claude-sonnet-4.5", -]); -const ANTHROPIC_ALLOWED_FALLBACK_MODELS = { - "claude-fable-5": ["claude-opus-4-8", "claude-opus-5"], - "claude-opus-5": ["claude-opus-4-8"], -} satisfies Record; - -const DEEPSEEK_V4_THINKING_LEVEL_MAP = { - minimal: null, - low: null, - medium: null, - high: "high", - max: "max", -} as const; -const DEEPSEEK_V4_FLASH_THINKING_LEVEL_MAP = { - ...DEEPSEEK_V4_THINKING_LEVEL_MAP, - low: "low", -} as const; -// Azure Foundry rejects DeepSeek's own max effort. -const AZURE_DEEPSEEK_V4_THINKING_LEVEL_MAP = { - minimal: null, - low: "low", - medium: "medium", - high: "high", - xhigh: null, - max: null, -} as const; -// Verified against Fireworks Messages raw_output on 2026-09-10 (#9323). -// Fall back to verified support when models.dev omits effort metadata; this is -// not an allowlist. Any Fireworks Messages model advertising effort uses adaptive thinking. -const FIREWORKS_ADAPTIVE_THINKING_FALLBACK_MODELS = new Set([ - "accounts/fireworks/models/deepseek-v4-flash-0731", - "accounts/fireworks/models/deepseek-v4-flash-vision-exp", - "accounts/fireworks/models/deepseek-v4-pro-0813", - "accounts/fireworks/models/qwen3p8-max", - "accounts/fireworks/models/qwen3p8-2p4t-a95b", -]); -const QWEN_TOKEN_PLAN_FALLBACK_THINKING_LEVEL_MAP = { - minimal: null, - low: null, - medium: null, - high: "high", - xhigh: null, - max: "max", -} as const; -const QWEN_TOKEN_PLAN_REASONING_EFFORT_FALLBACK_MODEL_IDS = new Set(["glm-5", "glm-5.1"]); -// Retired preview id — models.dev may still list it after GA ships. -const QWEN_TOKEN_PLAN_EXCLUDED_MODEL_IDS = new Set(["qwen3.8-max-preview"]); -const QWEN_TOKEN_PLAN_PROVIDER_IDS = new Set([ - "qwen-token-plan", - "qwen-token-plan-cn", - "qwen-token-plan-individual", -]); -// QwenCloud Token Plan Individual text-model allowlist, verified 2026-09-03. -// Retired models remain excluded above even if the public catalog lags. -// https://docs.qwencloud.com/token-plan/personal/token-plan-personal-overview -const QWEN_TOKEN_PLAN_INDIVIDUAL_MODEL_IDS = new Set([ - "deepseek-v4-flash-0731", - "deepseek-v4-pro", - "deepseek-v4-pro-0813", - "glm-5.2", - "qwen3.6-flash", - "qwen3.7-max", - "qwen3.7-plus", - "qwen3.8-flash", - "qwen3.8-max", -]); - -const KIMI_K3_MAX_TOKENS = 131072; -const KIMI_K3_COST = { - input: 3, - output: 15, - cacheRead: 0.3, - cacheWrite: 0, -} as const; -// Kimi Chat Completions settings shared by Moonshot and Kimi Coding. -const KIMI_OPENAI_COMPAT: OpenAICompletionsCompat = { - supportsStore: false, - supportsDeveloperRole: false, - supportsReasoningEffort: false, - maxTokensField: "max_tokens", - supportsStrictMode: false, - thinkingFormat: "deepseek", -}; -// Kimi K3-style models take reasoning_effort and need earlier reasoning sent back. -const KIMI_K3_OPENAI_COMPAT: OpenAICompletionsCompat = { - ...KIMI_OPENAI_COMPAT, - requiresReasoningContentOnAssistantMessages: true, - thinkingFormat: "openai", - supportsReasoningEffort: true, -}; -// Kimi K3 and its point releases (kimi-k3.1-pro, kimi-k3.1-pro-highspeed) share K3's request -// format. https://api.moonshot.ai/v1/models reports effort levels and dynamic tool loading for all. -function isKimiK3FamilyId(modelId: string): boolean { - return /^kimi-k3(?:\.\d+)?(?:-|$)/.test(modelId); -} - -const ANT_LING_RING_THINKING_LEVEL_MAP = { - off: null, - minimal: null, - low: null, - medium: null, - high: "high", - xhigh: "xhigh", -} as const; - -const BEDROCK_INFERENCE_PROFILE_ONLY_MODEL_IDS = new Set(["anthropic.claude-opus-5"]); -const MODELS_DEV_OPENAI_UNSUPPORTED_MODEL_IDS = new Set(["gpt-5.6"]); -// GPT-5.5 and older put tools loaded through tool search or additional_tools in a namespace named -// after the tool and reject replayed calls without it, so history from other models fails there -// (#11415). Like Codex, dynamic tool loading therefore starts with GPT-5.6, which does not do this. -const OPENAI_TOOL_SEARCH_MODEL_IDS = new Set([ - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-6.1-sol", - "gpt-6.1-terra", -]); -const OPENAI_ADDITIONAL_TOOLS_MODEL_IDS = OPENAI_TOOL_SEARCH_MODEL_IDS; -const OPENAI_MID_CONVO_SYSTEM_MESSAGE_MODEL_IDS = new Set([ - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-pro", - "gpt-5.5", - ...OPENAI_TOOL_SEARCH_MODEL_IDS, -]); -const OPENAI_CODEX_ADDITIONAL_TOOLS_MODEL_IDS = new Set([ - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-6.1-sol", - "gpt-6.1-terra", -]); -// OpenAI documents configuration_update for the GPT-6 family. -// https://developers.openai.com/api/docs/guides/reasoning -const OPENAI_MID_CONVO_EFFORT_MODEL_IDS = new Set([ - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-6.1-sol", - "gpt-6.1-terra", -]); -const OPENAI_LONG_CONTEXT_INPUT_THRESHOLD = 272000; -const OPENAI_SHORT_CONTEXT_CAPPED_MODEL_IDS = new Set([ - "gpt-5.4", - "gpt-5.5", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-6.1-sol", - "gpt-6.1-terra", -]); -const OPENAI_LONG_CONTEXT_PRICING_MODEL_IDS = new Set([ - "gpt-5.4", - "gpt-5.4-pro", - "gpt-5.5", - "gpt-5.5-pro", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-6-astra", - "gpt-6-sol", - "gpt-6-luna", - "gpt-6.1-sol", - "gpt-6.1-terra", -]); - -// Keep the generated default no less restrictive than coding-agent's historical -// image preprocessing. Provider limits can narrow this profile, but unknown -// providers retain the cache-safe 2000px / 4.5 MiB behavior. -const DEFAULT_IMAGE_RESIZE = { - maxWidth: 2000, - maxHeight: 2000, - maxBytes: 4.5 * 1024 * 1024, - jpegQuality: 80, -} as const; - -function withOpenAiLongContextPricing(cost: Model["cost"]): Model["cost"] { - return { - ...cost, - tiers: [ - { - inputTokensAbove: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, - input: roundCost(cost.input * 2), - output: roundCost(cost.output * 1.5), - cacheRead: roundCost(cost.cacheRead * 2), - cacheWrite: roundCost(cost.cacheWrite * 2), - }, - ], - }; -} - -// Keep current OpenAI prices authoritative until models.dev and passthrough -// catalogs catch up. -// https://developers.openai.com/api/docs/pricing -const OPENAI_STANDARD_COSTS: Record = { - "gpt-5.6-luna": { input: 0.2, output: 1.2, cacheRead: 0.02, cacheWrite: 0.25 }, - "gpt-5.6-sol": { input: 4, output: 20, cacheRead: 0.4, cacheWrite: 5 }, - "gpt-5.6-terra": { input: 2, output: 12, cacheRead: 0.2, cacheWrite: 2.5 }, - "gpt-6-astra": { input: 10, output: 50, cacheRead: 1, cacheWrite: 12.5 }, - "gpt-6-luna": { input: 0.1, output: 0.5, cacheRead: 0.01, cacheWrite: 0.125 }, - "gpt-6-sol": { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 2.5 }, - "gpt-6.1-sol": { input: 2, output: 10, cacheRead: 0.1, cacheWrite: 2.5 }, - "gpt-6.1-terra": { input: 1, output: 5, cacheRead: 0.05, cacheWrite: 1.25 }, -}; - -const OPENAI_RESPONSES_NONE_REASONING_MODELS = new Set([ - "gpt-5.1", - "gpt-5.2", - "gpt-5.3-codex", - "gpt-5.4", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.5", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-6-sol", - "gpt-6-luna", -]); -const XAI_BUILTIN_EXCLUDED_MODEL_IDS = new Set([ - "grok-3", - "grok-3-fast", - "grok-4.20-0309-non-reasoning", - "grok-4.20-0309-reasoning", - "grok-build-0.1", - "grok-code-fast-1", -]); -const XAI_RESPONSES_COMPAT: OpenAIResponsesCompat = { - supportsLongCacheRetention: false, -}; - -const OPENCODE_OPENAI_COMPLETIONS_LONG_CACHE_RETENTION_UNSUPPORTED_MODELS = new Set([ - "opencode:deepseek-v4-flash", - "opencode:deepseek-v4.1-flash", - "opencode:deepseek-v4-pro", - "opencode:kimi-k2.5", - "opencode:kimi-k2.6", - "opencode:minimax-m2.7", - "opencode-go:kimi-k2.6", -]); - -// GitHub's "Models with extended capabilities" table lists these Copilot models as supporting -// the extended 1 million token context window. -const GITHUB_COPILOT_EXTENDED_CONTEXT_MODELS = new Set([ - "claude-fable-5", - "claude-opus-4.8", - "claude-opus-5", - "claude-opus-5.5", - "claude-sonnet-4.6", - "claude-sonnet-5", - "gpt-5.3-codex", - "gpt-6-astra", - "gpt-6-luna", - "gpt-6-sol", -]); - -// Checked manually against the authenticated GitHub Copilot /models endpoint on 2026-06-15. -// Keep this to narrow corrections over models.dev metadata instead of snapshotting Copilot's catalog. -const GITHUB_COPILOT_THINKING_LEVEL_OVERRIDES = { - "claude-opus-4.8": { minimal: "low" }, - "claude-opus-5": { minimal: "low" }, - "claude-sonnet-4.6": { minimal: "low", max: "max" }, -} satisfies Record["thinkingLevelMap"]>>; - -function mergeThinkingLevelMap(model: Model, map: NonNullable["thinkingLevelMap"]>): void { - model.thinkingLevelMap = { ...model.thinkingLevelMap, ...map }; -} - -const modelsDevReasoningOptions = new Map(); - -function getModelKey(model: Pick, "provider" | "id">): string { - return `${model.provider}:${model.id}`; -} - -function recordModelsDevReasoningOptions(provider: string, id: string, sourceModel: ModelsDevModel): void { - if (sourceModel.reasoning_options !== undefined) { - modelsDevReasoningOptions.set(`${provider}:${id}`, sourceModel.reasoning_options); - } -} - -function supportsDirectReasoningEffort(model: Model): boolean { - if (model.api === "anthropic-messages") return model.compat?.forceAdaptiveThinking === true; - if ( - model.api === "openai-responses" || - model.api === "azure-openai-responses" || - model.api === "openai-codex-responses" - ) { - return true; - } - if (model.api !== "openai-completions") return false; - - const compat = { - ...detectOpenAICompletionsCompat(model as Model<"openai-completions">), - ...(model.compat as OpenAICompletionsCompat | undefined), - }; - return compat.thinkingFormat === "openai" && compat.supportsReasoningEffort; -} - -function applyModelsDevReasoningOptionMetadata(model: Model): void { - const reasoningOptions = modelsDevReasoningOptions.get(getModelKey(model)); - if (!reasoningOptions || !supportsDirectReasoningEffort(model)) return; - const thinkingLevelMap = getEffortThinkingLevelMap(reasoningOptions); - if (thinkingLevelMap) mergeThinkingLevelMap(model, thinkingLevelMap); -} - -function getTogetherCompat(modelId: string, reasoning: boolean): OpenAICompletionsCompat { - if (!reasoning) return TOGETHER_BASE_COMPAT; - if (TOGETHER_REASONING_EFFORT_MODELS.has(modelId)) return TOGETHER_REASONING_EFFORT_COMPAT; - if (TOGETHER_TOGGLE_REASONING_EFFORT_MODELS.has(modelId)) return TOGETHER_TOGGLE_REASONING_EFFORT_COMPAT; - return TOGETHER_TOGGLE_REASONING_COMPAT; -} - -function getTogetherThinkingLevelMap( - modelId: string, - reasoning: boolean, -): NonNullable["thinkingLevelMap"]> | undefined { - if (!reasoning) return undefined; - if (TOGETHER_REASONING_EFFORT_MODELS.has(modelId)) return { ...TOGETHER_REASONING_EFFORT_LEVEL_MAP }; - if (TOGETHER_TOGGLE_REASONING_EFFORT_MODELS.has(modelId)) return { ...TOGETHER_DEEPSEEK_V4_THINKING_LEVEL_MAP }; - return { ...TOGETHER_TOGGLE_REASONING_LEVEL_MAP }; -} - -function supportsOpenAiXhigh(modelId: string): boolean { - return ( - modelId.includes("gpt-5.2") || - modelId.includes("gpt-5.3") || - modelId.includes("gpt-5.4") || - modelId.includes("gpt-5.5") || - modelId.includes("gpt-5.6") || - modelId.includes("gpt-6") - ); -} - -function supportsOpenAiMax(model: Model): boolean { - return ( - (model.id.includes("gpt-5.6") || model.id.includes("gpt-6")) && - (model.api === "openai-responses" || - model.api === "azure-openai-responses" || - model.api === "openai-codex-responses" || - model.api === "openai-completions") - ); -} - -const VERIFIED_ANTHROPIC_MID_CONVO_EFFORT_PROVIDERS = new Set(["anthropic", "openrouter"]); -// OpenRouter rejects `configuration_update` system messages on Opus 5 ("Mid-conversation -// reasoning effort (configuration_update) is not supported on anthropic/claude-opus-5-20260723") -// while accepting them on Fable 5.1, so gate that model there. -const MID_CONVO_EFFORT_UNSUPPORTED_ANTHROPIC_MODELS = new Set(["openrouter:anthropic/claude-opus-5"]); - -function supportsAnthropicMidConvoEffort(modelId: string): boolean { - const id = modelId.toLowerCase().replace(/^~?anthropic\//, ""); - return ( - /^claude-opus-(?:5|5[.-]5)(?:-\d{8})?$/.test(id) || - /^claude-(?:sonnet|haiku)-5[.-]5(?:-\d{8})?$/.test(id) || - /^claude-(?:fable|mythos)-5(?:[.-]1)(?:-\d{8})?$/.test(id) - ); -} - -// Claude models that accept `thinking: {type: "disabled"}` alongside per-message effort. -// Opus 5.5, Sonnet 5.5, Fable 5.1, and Mythos 5.1 reject it. -// https://platform.claude.com/docs/en/build-with-claude/thinking#configuring-thinking -const ANTHROPIC_DISABLED_THINKING_MODELS = new Set(["claude-opus-5", "claude-haiku-5-5", "claude-haiku-5.5"]); - -function supportsAnthropicDisabledThinkingWithMidConvoEffort(modelId: string): boolean { - // Strip OpenRouter's "anthropic/" or "~anthropic/" prefix and any dated snapshot suffix. - const id = modelId - .toLowerCase() - .replace(/^~?anthropic\//, "") - .replace(/-\d{8}$/, ""); - return ANTHROPIC_DISABLED_THINKING_MODELS.has(id); -} - -function supportsAnthropicMidConvoSystemMessages(modelId: string): boolean { - return ( - /^claude-opus-(?:4[.-]8|5(?:[.-]5)?)(?:-\d{8})?$/.test(modelId) || - /^claude-(?:sonnet|haiku)-5[.-]5(?:-\d{8})?$/.test(modelId) || - /^claude-(?:fable|mythos)-5(?:[.-]1)?(?:-\d{8})?$/.test(modelId) - ); -} - -function isAnthropicAdaptiveThinkingModel(modelId: string): boolean { - return ( - // OpenRouter's `~anthropic/claude-*-latest` aliases follow the newest model of each family, - // all of which use adaptive thinking. - /^~anthropic\/claude-[a-z]+-latest$/.test(modelId) || - modelId.includes("opus-4-6") || - modelId.includes("opus-4.6") || - modelId.includes("opus-4-7") || - modelId.includes("opus-4.7") || - modelId.includes("opus-4-8") || - modelId.includes("opus-4.8") || - modelId.includes("opus-5") || - modelId.includes("opus.5") || - modelId.includes("sonnet-4-6") || - modelId.includes("sonnet-4.6") || - modelId.includes("sonnet-5") || - modelId.includes("sonnet.5") || - modelId.includes("haiku-5") || - modelId.includes("haiku.5") || - modelId.includes("fable-5") || - modelId.includes("mythos-5") - ); -} - -function isAnthropicTemperatureUnsupportedModel(modelId: string): boolean { - const id = modelId.toLowerCase(); - return ( - id.includes("opus-4-7") || - id.includes("opus-4.7") || - id.includes("opus-4-8") || - id.includes("opus-4.8") || - id.includes("opus-5") || - id.includes("opus.5") || - id.includes("sonnet-5-5") || - id.includes("sonnet-5.5") || - id.includes("haiku-5-5") || - id.includes("haiku-5.5") - ); -} - -const OPENAI_COMPLETIONS_DEFAULT_COMPAT = { - supportsStore: true, - supportsDeveloperRole: true, - supportsReasoningEffort: true, - supportsUsageInStreaming: true, - supportsFinishReason: true, - maxTokensField: "max_completion_tokens", - requiresToolResultName: false, - requiresAssistantAfterToolResult: false, - requiresThinkingAsText: false, - requiresReasoningContentOnAssistantMessages: false, - thinkingFormat: "openai", - openRouterRouting: {}, - vercelGatewayRouting: {}, - chatTemplateKwargs: {}, - chatTemplateArgs: {}, - zaiToolStream: false, - supportsStrictMode: false, - supportsOpenAIGrammarTools: false, - supportsMidConvoSystemMessages: false, - supportsMidConvoToolAdditions: false, - sendSessionAffinityHeaders: false, - supportsLongCacheRetention: true, -} satisfies Required< - Omit< - OpenAICompletionsCompat, - "cacheControlFormat" | "supportsThinkingTokenBudget" | "thinkingTokenBudgetField" - > -> & { - cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"]; -}; - -type OpenAICompletionsResolvedCompat = typeof OPENAI_COMPLETIONS_DEFAULT_COMPAT & { - cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"]; -}; - -function mergeAnthropicMessagesCompat(model: Model, compat: AnthropicMessagesCompat): void { - model.compat = { ...(model.compat as AnthropicMessagesCompat | undefined), ...compat }; -} - -function detectOpenAICompletionsCompat(model: Model<"openai-completions">): OpenAICompletionsResolvedCompat { - const provider = model.provider; - const baseUrl = model.baseUrl; - - const isZai = - provider === "zai" || - provider === "zai-coding-cn" || - baseUrl.includes("api.z.ai") || - baseUrl.includes("open.bigmodel.cn"); - const isTogether = - provider === "together" || baseUrl.includes("api.together.ai") || baseUrl.includes("api.together.xyz"); - const isMoonshot = provider === "moonshotai" || provider === "moonshotai-cn" || baseUrl.includes("api.moonshot."); - const isOpenRouter = provider === "openrouter" || baseUrl.includes("openrouter.ai"); - const isCloudflareWorkersAI = provider === "cloudflare-workers-ai" || baseUrl.includes("api.cloudflare.com"); - const isCloudflareAiGateway = provider === "cloudflare-ai-gateway" || baseUrl.includes("gateway.ai.cloudflare.com"); - const isNvidia = provider === "nvidia" || baseUrl.includes("integrate.api.nvidia.com"); - const isAntLing = provider === "ant-ling" || baseUrl.includes("api.ant-ling.com"); - const isCerebras = provider === "cerebras" || baseUrl.includes("cerebras.ai"); - const isDeepSeek = provider === "deepseek" || baseUrl.toLowerCase().includes("deepseek.com"); - - const isNonStandard = - isNvidia || - isCerebras || - provider === "xai" || - baseUrl.includes("api.x.ai") || - isTogether || - baseUrl.includes("chutes.ai") || - isDeepSeek || - isZai || - isMoonshot || - provider === "opencode" || - baseUrl.includes("opencode.ai") || - isCloudflareWorkersAI || - isCloudflareAiGateway || - isAntLing; - - const useMaxTokens = - baseUrl.includes("chutes.ai") || - isDeepSeek || - isMoonshot || - isCloudflareAiGateway || - isTogether || - isNvidia || - isAntLing || - isZai; - - const isGrok = provider === "xai" || baseUrl.includes("api.x.ai"); - const isOpenRouterDeveloperRoleModel = - isOpenRouter && (model.id.startsWith("anthropic/") || model.id.startsWith("openai/")); - const cacheControlFormat = - provider === "openrouter" && /^~?anthropic\//.test(model.id) ? "anthropic" : undefined; - - return { - supportsStore: !isNonStandard, - supportsDeveloperRole: isOpenRouterDeveloperRoleModel || (!isNonStandard && !isOpenRouter), - supportsReasoningEffort: - !isGrok && !isZai && !isMoonshot && !isTogether && !isCloudflareAiGateway && !isNvidia && !isAntLing, - supportsUsageInStreaming: true, - supportsFinishReason: true, - maxTokensField: useMaxTokens ? "max_tokens" : "max_completion_tokens", - requiresToolResultName: false, - requiresAssistantAfterToolResult: false, - requiresThinkingAsText: false, - requiresReasoningContentOnAssistantMessages: isDeepSeek, - thinkingFormat: isDeepSeek - ? "deepseek" - : isZai - ? "zai" - : isTogether - ? "together" - : isAntLing - ? "ant-ling" - : isOpenRouter - ? "openrouter" - : "openai", - openRouterRouting: {}, - vercelGatewayRouting: {}, - chatTemplateKwargs: {}, - chatTemplateArgs: {}, - zaiToolStream: false, - // Preserve built-in behavior as explicit metadata against the conservative runtime default. - supportsStrictMode: !isMoonshot && !isTogether && !isCloudflareAiGateway && !isNvidia && !isCerebras, - supportsOpenAIGrammarTools: false, - supportsMidConvoSystemMessages: false, - supportsMidConvoToolAdditions: false, - ...(cacheControlFormat ? { cacheControlFormat } : {}), - sendSessionAffinityHeaders: isOpenRouter, - supportsLongCacheRetention: !( - isTogether || - isCloudflareWorkersAI || - isCloudflareAiGateway || - isNvidia || - isAntLing - ), - }; -} - -function isPlainEmptyObject(value: unknown): boolean { - return typeof value === "object" && value !== null && !Array.isArray(value) && Object.keys(value).length === 0; -} - -function openAICompletionsCompatDelta(compat: OpenAICompletionsResolvedCompat): OpenAICompletionsCompat { - const delta: OpenAICompletionsCompat = {}; - for (const [key, value] of Object.entries(compat)) { - const defaultValue = OPENAI_COMPLETIONS_DEFAULT_COMPAT[key as keyof typeof OPENAI_COMPLETIONS_DEFAULT_COMPAT]; - if (isPlainEmptyObject(value) && isPlainEmptyObject(defaultValue)) continue; - if (value !== defaultValue) { - (delta as Record)[key] = value; - } - } - return delta; -} - -function mergeOpenAICompletionsCompat(model: Model, compat: OpenAICompletionsCompat): void { - model.compat = { ...(model.compat as OpenAICompletionsCompat | undefined), ...compat }; -} - -function applyOpenAICompletionsCompatMetadata(model: Model): void { - if (model.api !== "openai-completions") return; - const detected = openAICompletionsCompatDelta(detectOpenAICompletionsCompat(model as Model<"openai-completions">)); - model.compat = { ...detected, ...(model.compat as OpenAICompletionsCompat | undefined) }; - if (Object.keys(model.compat).length === 0) { - delete model.compat; - } -} - -function applyAnthropicMessagesCompatMetadata(model: Model): void { - if (model.api !== "anthropic-messages") return; - const compat = getAnthropicMessagesCompat(model.provider, model.id); - if (compat) { - mergeAnthropicMessagesCompat(model, compat); - if (compat.supportsMidConvoEffort && !supportsAnthropicDisabledThinkingWithMidConvoEffort(model.id)) { - mergeThinkingLevelMap(model, { off: null }); - } - } -} - -function isAnthropicFallbackMetadataModel(model: Model): model is Model<"anthropic-messages"> { - if (model.provider !== "anthropic" || model.api !== "anthropic-messages") return false; - return ( - model.id in ANTHROPIC_ALLOWED_FALLBACK_MODELS || - Object.values(ANTHROPIC_ALLOWED_FALLBACK_MODELS).some((fallbackModelIds) => fallbackModelIds.includes(model.id)) - ); -} - -function applyAnthropicAllowedFallbackModelMetadata(models: readonly Model<"anthropic-messages">[]): void { - const modelsById = new Map(models.map((model) => [model.id, model])); - for (const [modelId, fallbackModelIds] of Object.entries(ANTHROPIC_ALLOWED_FALLBACK_MODELS)) { - const model = modelsById.get(modelId); - if (!model) continue; - - const compatibleFallbackModelIds = model.compat?.supportsMidConvoEffort - ? fallbackModelIds.filter(supportsAnthropicMidConvoEffort) - : fallbackModelIds; - const allowedFallbackModels = compatibleFallbackModelIds.flatMap((fallbackModelId) => { - const fallbackModel = modelsById.get(fallbackModelId); - return fallbackModel - ? [{ provider: fallbackModel.provider, model: fallbackModel.id, cost: fallbackModel.cost }] - : []; - }); - if (allowedFallbackModels.length > 0) { - mergeAnthropicMessagesCompat(model, { allowedFallbackModels }); - } - } -} - -function applyStrictToolCompatMetadata(model: Model): void { - if ( - (model.provider === "openai" || model.provider === "cloudflare-ai-gateway") && - model.api === "openai-responses" - ) { - model.compat = { ...(model.compat as OpenAIResponsesCompat | undefined), supportsStrictMode: true }; - } else if (model.provider === "anthropic" && model.api === "anthropic-messages") { - mergeAnthropicMessagesCompat(model, { supportsStrictTools: true }); - } -} - -// Responses endpoints verified (OpenAI, ChatGPT Codex backend, GitHub Copilot, -// opencode zen) or documented (Azure OpenAI, Cloudflare AI Gateway) to pass -// OpenAI custom grammar tools through. OpenAI rejects `type: "custom"` tools -// for pre-GPT-5 models (gpt-4.x, gpt-4o, o-series). -const OPENAI_GRAMMAR_TOOL_PROVIDERS = new Set([ - "openai", - "openai-codex", - "azure", - "github-copilot", - "opencode", - "cloudflare-ai-gateway", -]); -const OPENAI_GRAMMAR_TOOL_APIS = new Set([ - "openai-responses", - "azure-openai-responses", - "openai-codex-responses", -]); - -function applyOpenAIGrammarToolCompatMetadata(model: Model): void { - if (!OPENAI_GRAMMAR_TOOL_APIS.has(model.api) || !OPENAI_GRAMMAR_TOOL_PROVIDERS.has(model.provider)) return; - const match = /^gpt-(\d+)/.exec(model.id); - if (!match || Number(match[1]) < 5) return; - model.compat = { ...(model.compat as OpenAIResponsesCompat | undefined), supportsOpenAIGrammarTools: true }; -} - -function applyOpenAIToolSearchMetadata(model: Model): void { - const isOpenAIResponses = model.provider === "openai" && model.api === "openai-responses"; - const isOpenAICodex = model.provider === "openai-codex" && model.api === "openai-codex-responses"; - if (!(isOpenAIResponses || isOpenAICodex) || !OPENAI_TOOL_SEARCH_MODEL_IDS.has(model.id)) return; - const supportsAdditionalTools = - (isOpenAIResponses && OPENAI_ADDITIONAL_TOOLS_MODEL_IDS.has(model.id)) || - (isOpenAICodex && OPENAI_CODEX_ADDITIONAL_TOOLS_MODEL_IDS.has(model.id)); - model.compat = { - ...(model.compat as OpenAIResponsesCompat | undefined), - ...(supportsAdditionalTools ? { supportsAdditionalTools: true } : {}), - supportsToolSearch: true, - }; -} - -// Moonshot Kimi K2.6/K2.7 accept system text after the conversation starts but reject -// tool-bearing system messages. Kimi K3 accepts both forms; Fireworks and OpenCode pass -// its tool-bearing form through. GitHub Copilot forwards K3 text but silently drops its -// tool-bearing message. DeepSeek V4 Pro and OpenAI models behind OpenRouter also accept -// plain system text in place. Kimi Coding sets these flags in createKimiCodingModels(). -function applyOpenAICompletionsTranscriptMetadata(model: Model): void { - if (model.api !== "openai-completions") return; - const isKimiK3 = - (model.provider.startsWith("moonshot") && isKimiK3FamilyId(model.id)) || - (model.provider === "fireworks" && model.id.includes("kimi-k3")) || - ((model.provider === "opencode" || model.provider === "opencode-go") && isKimiK3FamilyId(model.id)); - const isMoonshotKimiK2 = - model.provider.startsWith("moonshot") && - (model.id === "kimi-k2.6" || model.id === "kimi-k2.7-code" || model.id === "kimi-k2.7-code-highspeed"); - const isTextOnly = - isMoonshotKimiK2 || - (model.provider === "github-copilot" && model.id === "kimi-k3") || - (model.provider === "deepseek" && model.id === "deepseek-v4-pro") || - (model.provider === "openrouter" && - model.id.startsWith("openai/") && - OPENAI_MID_CONVO_SYSTEM_MESSAGE_MODEL_IDS.has(model.id.slice("openai/".length))); - if (!isKimiK3 && !isTextOnly) return; - model.compat = { - ...(model.compat as OpenAICompletionsCompat | undefined), - supportsMidConvoSystemMessages: true, - ...(isKimiK3 ? { supportsMidConvoToolAdditions: true } : {}), - }; -} - -// Newer OpenAI Responses models accept developer messages after the conversation has started. -// OpenCode Zen, OpenCode Go, and GitHub Copilot pass both those messages and -// `additional_tools` items through to OpenAI unchanged; tool search is not verified -// through those proxies. -const OPENAI_RESPONSES_PROXY_PROVIDERS = new Set(["opencode", "opencode-go", "github-copilot"]); - -function applyOpenAIResponsesTranscriptMetadata(model: Model): void { - const isOpenAIResponses = model.provider === "openai" && model.api === "openai-responses"; - const isOpenAICodex = model.provider === "openai-codex" && model.api === "openai-codex-responses"; - const isProxiedResponses = - OPENAI_RESPONSES_PROXY_PROVIDERS.has(model.provider) && model.api === "openai-responses"; - if ( - !(isOpenAIResponses || isOpenAICodex || isProxiedResponses) || - !OPENAI_MID_CONVO_SYSTEM_MESSAGE_MODEL_IDS.has(model.id) - ) { - return; - } - model.compat = { - ...(model.compat as OpenAIResponsesCompat | undefined), - supportsMidConvoSystemMessages: true, - ...(isProxiedResponses && OPENAI_ADDITIONAL_TOOLS_MODEL_IDS.has(model.id) ? { supportsAdditionalTools: true } : {}), - }; -} - -// OpenAI charges prompt-cache writes starting with the GPT-5.6 family, and exactly -// those models accept `prompt_cache_options`; older models reject the parameter. -// https://developers.openai.com/api/docs/guides/prompt-caching -function applyOpenAIExplicitPromptCacheMetadata(model: Model): void { - if (model.provider !== "openai" || model.api !== "openai-responses") return; - if (!(model.cost.cacheWrite > 0)) return; - model.compat = { - ...(model.compat as OpenAIResponsesCompat | undefined), - supportsExplicitPromptCacheMode: true, - }; -} - -// GPT-6 models accept `configuration_update` input items that change the reasoning effort -// without changing the cached prompt prefix. Not enabled for openai-codex (Codex CLI keeps it -// behind a flag that is off by default) or proxies. -function applyOpenAIMidConvoEffortMetadata(model: Model): void { - if (model.provider !== "openai" || model.api !== "openai-responses") return; - if (!OPENAI_MID_CONVO_EFFORT_MODEL_IDS.has(model.id)) return; - model.compat = { - ...(model.compat as OpenAIResponsesCompat | undefined), - supportsMidConvoEffort: true, - }; -} - -// Anthropic ephemeral entries have a hard five-minute lifetime; `ttl: "1h"` -// extends it to one hour. Only direct Anthropic is annotated so cache warming -// does not assume equivalent behavior through proxies. -// https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching -const ANTHROPIC_PROMPT_CACHE: ModelPromptCache = { short: 300, long: 3600 }; - -function applyPromptCacheMetadata(model: Model): void { - if (model.provider === "anthropic" && model.api === "anthropic-messages") { - model.promptCache = ANTHROPIC_PROMPT_CACHE; - } - // Do not add OpenAI lifetimes yet. Before enabling warming for explicit - // OpenAI caches, re-evaluate it using observed expiry, replay, and billing - // behavior; a documented TTL alone does not establish full cache loss. -} - -function applyImageInputMetadata(model: AnyModel): void { - if (!model.input.includes("image")) return; - - const providerLimits: AnyModel["inputLimits"] = - model.provider === "anthropic" - ? { - maxRequestBytes: 32 * 1024 * 1024, - images: { maxPerRequest: model.type !== "image" && model.contextWindow === 200000 ? 100 : 600 }, - } - : model.provider === "amazon-bedrock" - ? { images: { maxPerMessage: 20 } } - : model.provider === "openai" - ? { maxRequestBytes: 512 * 1024 * 1024, images: { maxPerRequest: 1500 } } - : model.provider === "google" - ? { maxRequestBytes: 20 * 1024 * 1024, images: { maxPerRequest: 3600 } } - : undefined; - const configuredImages = model.inputLimits?.images; - model.inputLimits = { - ...providerLimits, - ...model.inputLimits, - images: { - ...providerLimits?.images, - ...configuredImages, - resize: { ...DEFAULT_IMAGE_RESIZE, ...configuredImages?.resize }, - }, - }; -} - -function isGemma4Model(modelId: string): boolean { - return /gemma-?4/.test(modelId.toLowerCase()); -} - -function getGoogleThinkingLevelMap( - modelId: string, - reasoningOptions: readonly ModelsDevReasoningOption[], -): NonNullable["thinkingLevelMap"]> | undefined { - const effortMap = getEffortThinkingLevelMap(reasoningOptions); - if (effortMap) return effortMap; - if (isGemma4Model(modelId)) { - return { off: null, minimal: "MINIMAL", low: null, medium: null, high: "HIGH" }; - } - return undefined; -} - -function applyThinkingLevelMetadata(model: Model): void { - if ( - (model.api === "openai-responses" || model.api === "azure-openai-responses") && - model.id.startsWith("gpt-5") - ) { - mergeThinkingLevelMap(model, { off: null }); - } - if ( - (model.id === "gpt-6-astra" || - model.id === "gpt-6-sol" || - model.id === "gpt-6-luna" || - model.id === "gpt-6.1-sol" || - model.id === "gpt-6.1-terra") && - (model.api === "openai-responses" || - model.api === "azure-openai-responses" || - model.api === "openai-codex-responses") - ) { - mergeThinkingLevelMap(model, { - // GPT-6 Astra and GPT-6.1 Sol reject reasoning.effort "none"; GPT-6.1 Terra accepts it. All reject - // "minimal" (verified on Codex, October 2026). - off: model.id === "gpt-6-astra" || model.id === "gpt-6.1-sol" ? null : "none", - minimal: null, - low: "low", - medium: "medium", - high: "high", - xhigh: "xhigh", - max: "max", - }); - } - if (model.provider === "github-copilot" && model.id.startsWith("gpt-5")) { - mergeThinkingLevelMap(model, { minimal: "low" }); - } - if ( - model.api === "openai-responses" && - model.provider === "openai" && - OPENAI_RESPONSES_NONE_REASONING_MODELS.has(model.id) - ) { - mergeThinkingLevelMap(model, { off: "none" }); - } - // xAI models without verified effort options must not send the undocumented - // "none"/"minimal" efforts. - if (model.provider === "xai" && model.api === "openai-responses" && model.thinkingLevelMap === undefined) { - mergeThinkingLevelMap(model, { off: null, minimal: null }); - } - if (supportsOpenAiXhigh(model.id)) { - mergeThinkingLevelMap(model, { xhigh: "xhigh" }); - } - if (supportsOpenAiMax(model)) { - mergeThinkingLevelMap(model, { max: "max" }); - } - if (model.provider === "openai" && model.id === "gpt-5.5") { - mergeThinkingLevelMap(model, { minimal: null }); - } - if (model.id.endsWith("gpt-5.5-pro")) { - mergeThinkingLevelMap(model, { off: null, minimal: null, low: null }); - } - // Anthropic adaptive-thinking effort support (per Anthropic adaptive thinking docs): - // - "max" is available on all adaptive-thinking Claude models. - // - "xhigh" is only available on Opus 4.7/4.8/5, Sonnet 5, Haiku 5.5, and Fable 5. - if ( - model.id.includes("opus-4-6") || - model.id.includes("opus-4.6") || - model.id.includes("sonnet-4-6") || - model.id.includes("sonnet-4.6") - ) { - mergeThinkingLevelMap(model, { max: "max" }); - } - if ( - model.id.includes("opus-4-7") || - model.id.includes("opus-4.7") || - model.id.includes("opus-4-8") || - model.id.includes("opus-4.8") || - model.id.includes("opus-5") || - model.id.includes("opus.5") || - model.id.includes("sonnet-5") || - model.id.includes("sonnet.5") || - model.id.includes("haiku-5") || - model.id.includes("haiku.5") - ) { - mergeThinkingLevelMap(model, { xhigh: "xhigh", max: "max" }); - } - if (model.id.includes("fable-5")) { - mergeThinkingLevelMap(model, { off: null, xhigh: "xhigh", max: "max" }); - } - if (model.api === "anthropic-messages" && isAnthropicAdaptiveThinkingModel(model.id)) { - mergeAnthropicMessagesCompat(model, { forceAdaptiveThinking: true }); - } - if (model.api === "anthropic-messages" && isAnthropicTemperatureUnsupportedModel(model.id)) { - mergeAnthropicMessagesCompat(model, { supportsTemperature: false }); - } - if ( - model.api === "openai-completions" && - model.id.includes("deepseek-v4") && - model.thinkingLevelMap === undefined - ) { - mergeThinkingLevelMap( - model, - model.provider === "openrouter" - ? { ...DEEPSEEK_V4_THINKING_LEVEL_MAP, xhigh: "xhigh", max: null } - : model.provider === "azure" - ? AZURE_DEEPSEEK_V4_THINKING_LEVEL_MAP - : (model.provider === "deepseek" || model.provider === "opencode" || model.provider === "opencode-go") && - model.id.includes("deepseek-v4-flash") - ? DEEPSEEK_V4_FLASH_THINKING_LEVEL_MAP - : DEEPSEEK_V4_THINKING_LEVEL_MAP, - ); - } - if (model.provider === "groq" && model.id === "qwen/qwen3.6-27b") { - mergeThinkingLevelMap(model, { minimal: null, low: null, medium: null, high: "default" }); - } - if (model.provider === "openai-codex" && supportsOpenAiXhigh(model.id)) { - mergeThinkingLevelMap(model, { minimal: "low" }); - } - if ( - (model.provider === "moonshotai" || model.provider === "moonshotai-cn") && - (model.id === "kimi-k2.7-code" || model.id === "kimi-k2.7-code-highspeed") - ) { - // Kimi K2.7 Code is always-thinking. Official docs say - // `thinking: { type: "disabled" }` is rejected, and callers can omit - // the thinking parameter to use the enabled default. - mergeThinkingLevelMap(model, { off: null }); - } - if (model.provider === "openrouter" && model.id.startsWith("inception/mercury-2")) { - // Mercury 2 in instant mode (reasoning_effort: "none") disables tool calling. - // Mark "off" unsupported so the openai-completions provider omits the reasoning param - // instead of defaulting to {reasoning:{effort:"none"}} (see openai-completions.ts:575). - // Pi's low/medium/high pass through verbatim; OpenRouter normalizes to Mercury's vocabulary. - mergeThinkingLevelMap(model, { off: null }); - } - if (model.provider === "openrouter" && model.id === "z-ai/glm-5.2") { - mergeThinkingLevelMap(model, { xhigh: "xhigh" }); - } - if (model.provider === "fireworks") { - if (model.api === "anthropic-messages" && model.compat?.forceAdaptiveThinking) { - // Qwen Max currently advertises only a toggle. Prefer upstream effort - // metadata once available instead of replacing it with this fallback. - if (model.id === "accounts/fireworks/models/qwen3p8-max" && !model.thinkingLevelMap) { - model.thinkingLevelMap = getEffortThinkingLevelMap([ - { type: "effort", values: ["low", "medium", "xhigh"] }, - ]); - } - const reasoningOptions = modelsDevReasoningOptions.get(getModelKey(model)); - if ( - reasoningOptions?.some((option) => option.type === "toggle") || - // The 2.4T alias omits the verified toggle in models.dev. - model.id === "accounts/fireworks/models/qwen3p8-2p4t-a95b" - ) { - mergeThinkingLevelMap(model, { off: "none" }); - } - if (model.id === "accounts/fireworks/models/deepseek-v4-pro-0813") { - mergeThinkingLevelMap(model, { low: "low" }); - } - } - if (model.id.includes("glm-5p2")) { - // GLM 5.2 and its fast router support off/high/max. Fireworks maps low - // and medium to high, so do not expose those aliases as distinct levels. - mergeThinkingLevelMap(model, { off: "none", minimal: null, low: null, medium: null, max: "max" }); - } - if (model.id.includes("kimi-k3")) { - // Fireworks maps medium to high on both APIs; do not expose it as a distinct level. - mergeThinkingLevelMap(model, { medium: null }); - } - } - if (model.provider === "opencode-go" && model.id === "glm-5.2") { - mergeThinkingLevelMap(model, OPENCODE_GO_GLM52_THINKING_LEVEL_MAP); - } - if (model.provider === "opencode-go" && model.id === "kimi-k2.6") { - // OpenCode Go exposes Kimi K2.6 thinking as on/off, not distinct effort tiers. - mergeThinkingLevelMap(model, { minimal: null, low: null, medium: null }); - } - if (model.provider === "opencode" && model.id === "grok-build-0.1") { - // OpenCode Zen Grok Build reasons by default but rejects explicit reasoningEffort. - mergeThinkingLevelMap(model, { off: null, minimal: null, low: null, medium: null }); - } - if (model.provider === "ant-ling" && model.reasoning) { - // Ring reasons by default. Only high/xhigh have documented explicit effort controls. - mergeThinkingLevelMap(model, ANT_LING_RING_THINKING_LEVEL_MAP); - } - if (model.provider === "github-copilot") { - const override = GITHUB_COPILOT_THINKING_LEVEL_OVERRIDES[model.id]; - if (override) { - mergeThinkingLevelMap(model, override); - } - } -} - -function getAnthropicMessagesCompat(provider: string, modelId: string): AnthropicMessagesCompat | undefined { - const compat: AnthropicMessagesCompat = {}; - if ( - VERIFIED_ANTHROPIC_MID_CONVO_EFFORT_PROVIDERS.has(provider) && - supportsAnthropicMidConvoEffort(modelId) && - !MID_CONVO_EFFORT_UNSUPPORTED_ANTHROPIC_MODELS.has(`${provider}:${modelId}`) - ) { - compat.supportsMidConvoEffort = true; - } - if (provider === "anthropic" && supportsAnthropicMidConvoSystemMessages(modelId)) { - compat.supportsMidConvoSystemMessages = true; - compat.supportsMidConvoToolChanges = true; - } - // OpenCode Zen and GitHub Copilot forward mid-conversation system messages but reject - // `tool_addition`/`tool_removal` blocks, so tool changes stay top-level there. - if ((provider === "opencode" || provider === "github-copilot") && supportsAnthropicMidConvoSystemMessages(modelId)) { - compat.supportsMidConvoSystemMessages = true; - } - if (EAGER_TOOL_INPUT_STREAMING_UNSUPPORTED_ANTHROPIC_MODELS.has(`${provider}:${modelId}`)) { - compat.supportsEagerToolInputStreaming = false; - } - if (provider === "xiaomi" || provider.startsWith("xiaomi-token-plan-")) { - compat.allowEmptySignature = true; - } - // OpenCode Qwen 3.8 Flash emits and accepts thinking blocks with empty signatures. - if ((provider === "opencode" || provider === "opencode-go") && modelId === "qwen3.8-flash") { - compat.allowEmptySignature = true; - } - return Object.keys(compat).length > 0 ? compat : undefined; -} - -function getBedrockBaseUrl(modelId: string): string { - return modelId.startsWith("eu.") - ? "https://bedrock-runtime.eu-central-1.amazonaws.com" - : "https://bedrock-runtime.us-east-1.amazonaws.com"; -} - -function normalizeNvidiaModelId(modelId: string): string { - return modelId.toLowerCase().replaceAll("_", "."); -} - -function roundCost(value: number): number { - return Number(value.toFixed(6)); -} - -function getModelsDevCost(cost: ModelsDevModel["cost"]): ModelCost { - const base = { - input: cost?.input || 0, - output: cost?.output || 0, - cacheRead: cost?.cache_read || 0, - cacheWrite: cost?.cache_write || 0, - }; - // Rates a tier does not list keep the base price. - const tiers = cost?.tiers?.flatMap((tier) => { - const context = tier.tier; - if (context?.type !== "context" || context.size === undefined) return []; - return [ - { - inputTokensAbove: context.size, - input: tier.input ?? base.input, - output: tier.output ?? base.output, - cacheRead: tier.cache_read ?? base.cacheRead, - cacheWrite: tier.cache_write ?? base.cacheWrite, - }, - ]; - }); - - return tiers && tiers.length > 0 ? { ...base, tiers } : base; -} - -async function fetchNvidiaNimModelIds(): Promise> { - try { - console.log("Fetching models from NVIDIA NIM API..."); - const response = await fetch(`${NVIDIA_BASE_URL}/models`); - if (!response.ok) throw new Error(`NVIDIA NIM API returned ${response.status}`); - const data = (await response.json()) as { data?: NvidiaNimModelListItem[] }; - const modelIds = new Map(); - - for (const model of data.data ?? []) { - modelIds.set(model.id, model.id); - modelIds.set(normalizeNvidiaModelId(model.id), model.id); - } - - console.log(`Fetched ${data.data?.length ?? 0} model IDs from NVIDIA NIM`); - return modelIds; - } catch (error) { - console.error("Failed to fetch NVIDIA NIM models:", error); - if (generatorOptions.strict) throw error; - return new Map(); - } -} - -async function fetchOpenRouterList(query: string): Promise { - const response = await fetch(`https://openrouter.ai/api/v1/models${query}`); - if (!response.ok) throw new Error(`OpenRouter API returned ${response.status}`); - const data = (await response.json()) as { data?: OpenRouterModelListItem[] }; - return data.data ?? []; -} - -async function fetchOpenRouterModels(): Promise { - try { - console.log("Fetching models from OpenRouter API..."); - const [listed, imageListed, decisionListed] = await Promise.all([ - fetchOpenRouterList(""), - fetchOpenRouterList("?output_modalities=image"), - fetchOpenRouterList("?output_modalities=decisions"), - ]); - const catalog = buildOpenRouterCatalog(listed, imageListed, decisionListed); - console.log( - `Fetched ${catalog.chat.length} tool-capable, ${catalog.images.length} image, and ${catalog.classifiers.length} classifier models from OpenRouter`, - ); - if (generatorOptions.strict && catalog.images.length === 0) { - throw new Error("OpenRouter API returned no usable image models"); - } - return catalog; - } catch (error) { - console.error("Failed to fetch OpenRouter models:", error); - if (generatorOptions.strict) throw error; - return { chat: [], images: [], classifiers: [] }; - } -} - -async function fetchRadiusModels(): Promise[]> { - try { - console.log("Fetching models from Radius API..."); - const config = await loadRadiusGatewayConfig(DEFAULT_RADIUS_GATEWAY); - const models = getRadiusModelsFromConfig("radius", config); - if (models.length === 0) throw new Error("Radius API returned no models"); - console.log(`Fetched ${models.length} models from Radius`); - return models; - } catch (error) { - console.error("Failed to fetch Radius models:", error); - if (generatorOptions.strict) throw error; - return []; - } -} - -async function fetchAiGatewayModels(): Promise<{ - chat: Model[]; - classifiers: ClassifierModel<"typesafe-system-one">[]; -}> { - try { - console.log("Fetching models from Vercel AI Gateway API..."); - const response = await fetch(`${AI_GATEWAY_MODELS_URL}/models`); - if (!response.ok) throw new Error(`Vercel AI Gateway API returned ${response.status}`); - const data = await response.json(); - const models: Model[] = []; - const classifiers: ClassifierModel<"typesafe-system-one">[] = []; - - const toNumber = (value: string | number | undefined): number => { - if (typeof value === "number") { - return Number.isFinite(value) ? value : 0; - } - const parsed = parseFloat(value ?? "0"); - return Number.isFinite(parsed) ? parsed : 0; - }; - - const items = Array.isArray(data.data) ? (data.data as AiGatewayModel[]) : []; - for (const model of items) { - // Evaluation models such as TypeSafe's Jev are served through the - // TypeSafe-compatible System One endpoint. - if (model.type === "evaluation") { - classifiers.push({ - type: "classifier", - id: model.id, - name: model.name || model.id, - api: "typesafe-system-one", - provider: "vercel-ai-gateway", - baseUrl: AI_GATEWAY_TYPESAFE_BASE_URL, - input: ["text"], - cost: { - input: roundCost(toNumber(model.pricing?.input) * 1_000_000), - output: roundCost(toNumber(model.pricing?.output) * 1_000_000), - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: model.context_window || 4096, - }); - continue; - } - const tags = Array.isArray(model.tags) ? model.tags : []; - // Only include models that support tools - if (!tags.includes("tool-use")) continue; - - const input: ("text" | "image")[] = ["text"]; - if (tags.includes("vision")) { - input.push("image"); - } - - models.push({ - id: model.id, - name: model.name || model.id, - api: "anthropic-messages", - baseUrl: AI_GATEWAY_BASE_URL, - provider: "vercel-ai-gateway", - reasoning: tags.includes("reasoning"), - input, - compat: { allowEmptySignature: true }, - cost: getAiGatewayCost(model.pricing), - contextWindow: model.context_window || 4096, - maxTokens: model.max_tokens || 4096, - }); - } - - console.log( - `Fetched ${models.length} tool-capable and ${classifiers.length} classifier models from Vercel AI Gateway`, - ); - return { chat: models, classifiers }; - } catch (error) { - console.error("Failed to fetch Vercel AI Gateway models:", error); - if (generatorOptions.strict) throw error; - return { chat: [], classifiers: [] }; - } -} - -function processZaiModels(data: ModelsDevCatalog): Model[] { - const variants = [ - { - source: "zai-coding-plan", - provider: "zai", - baseUrl: "https://api.z.ai/api/coding/paas/v4", - }, - { - source: "zhipuai-coding-plan", - provider: "zai-coding-cn", - baseUrl: "https://open.bigmodel.cn/api/coding/paas/v4", - }, - ] as const; - const models: Model[] = []; - - for (const { source, provider, baseUrl } of variants) { - for (const [modelId, model] of Object.entries(data[source]?.models ?? {})) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - const supportsImage = m.modalities?.input?.includes("image"); - - const thinkingLevelMap = getEffortThinkingLevelMap(m.reasoning_options ?? []); - const isGlm52 = modelId === "glm-5.2" || modelId === "glm-5.2-highspeed"; - if (thinkingLevelMap && isGlm52) { - thinkingLevelMap.off = "none"; - } - const supportsReasoningEffort = thinkingLevelMap !== undefined; - const referenceCost = data.zai?.models[modelId]?.cost ?? m.cost; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider, - baseUrl, - reasoning: m.reasoning === true, - ...(thinkingLevelMap ? { thinkingLevelMap } : {}), - input: supportsImage ? ["text", "image"] : ["text"], - cost: getModelsDevCost(referenceCost), - compat: { - supportsDeveloperRole: false, - thinkingFormat: "zai", - ...(supportsReasoningEffort ? { supportsReasoningEffort: true } : {}), - ...(!ZAI_TOOL_STREAM_UNSUPPORTED_MODELS.has(modelId) ? { zaiToolStream: true } : {}), - }, - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions(provider, modelId, m); - } - } - - return models; -} - -function processBasetenModels(provider: ModelsDevProvider | undefined): Model[] { - if (!provider?.models) return []; - - const baseUrl = "https://inference.baseten.co/v1"; - const baseCompat: OpenAICompletionsCompat = { - supportsStore: false, - supportsDeveloperRole: false, - supportsReasoningEffort: false, - supportsUsageInStreaming: true, - maxTokensField: "max_tokens", - supportsStrictMode: true, - // Baseten automatic prompt caching needs session affinity so related - // requests land on the same replica. See: - // https://docs.baseten.co/inference/model-apis/pricing-and-limits - sendSessionAffinityHeaders: true, - supportsLongCacheRetention: false, - }; - const reasoningEffortCompat: OpenAICompletionsCompat = { - ...baseCompat, - supportsReasoningEffort: true, - thinkingFormat: "openai", - }; - const toggleReasoningCompat: OpenAICompletionsCompat = { - ...baseCompat, - thinkingFormat: "baseten", - chatTemplateArgs: { enable_thinking: { $var: "thinking.enabled" } }, - }; - const toggleReasoningEffortCompat: OpenAICompletionsCompat = { - ...reasoningEffortCompat, - thinkingFormat: "baseten", - chatTemplateArgs: { enable_thinking: { $var: "thinking.enabled" } }, - }; - const toggleThinkingLevelMap = { - off: "off", - minimal: null, - low: null, - medium: null, - high: "high", - xhigh: null, - max: null, - } as const; - const glm52ThinkingLevelMap = { - off: "none", - minimal: null, - low: null, - medium: null, - high: "high", - xhigh: null, - max: "max", - } as const; - const models: Model[] = []; - - for (const [modelId, model] of Object.entries(provider.models)) { - if (model.status === "deprecated") continue; - - const reasoning = model.reasoning === true; - const reasoningOptions = model.reasoning_options ?? []; - const isGlm52 = modelId === "zai-org/GLM-5.2" || modelId === "zai-org/GLM-5.2-Fast"; - const supportsToggle = reasoningOptions.some((option) => option.type === "toggle") || isGlm52; - const supportsEffort = reasoningOptions.some((option) => option.type === "effort") || isGlm52; - const compat = - supportsToggle && supportsEffort - ? toggleReasoningEffortCompat - : supportsToggle - ? toggleReasoningCompat - : supportsEffort - ? reasoningEffortCompat - : baseCompat; - const thinkingLevelMap = isGlm52 - ? glm52ThinkingLevelMap - : supportsToggle - ? toggleThinkingLevelMap - : getEffortThinkingLevelMap(reasoningOptions); - // Baseten's GLM-5.2 endpoints are text-only despite models.dev reporting image input. - const supportsImageInput = !isGlm52 && model.modalities?.input?.includes("image"); - - models.push({ - id: modelId, - name: model.name || modelId, - api: "openai-completions", - provider: "baseten", - baseUrl, - reasoning, - ...(thinkingLevelMap ? { thinkingLevelMap } : {}), - input: supportsImageInput ? ["text", "image"] : ["text"], - cost: getModelsDevCost(model.cost), - compat, - contextWindow: model.limit?.context || 4096, - maxTokens: model.limit?.output || 4096, - }); - } - - return models; -} - -function processGoogleModels(data: ModelsDevCatalog): Model[] { - const models: Model[] = []; - const googleModels = data.google?.models; - if (googleModels) { - for (const [modelId, model] of Object.entries(googleModels)) { - if (model.tool_call !== true) continue; - const source = - modelId === "gemini-flash-latest" - ? (googleModels["gemini-3.5-flash"] ?? model) - : modelId === "gemini-flash-lite-latest" - ? (googleModels["gemini-3.1-flash-lite"] ?? model) - : model; - const thinkingLevelMap = getGoogleThinkingLevelMap(modelId, source.reasoning_options ?? []); - - models.push({ - id: modelId, - name: model.name || modelId, - api: "google-generative-ai", - provider: "google", - baseUrl: "https://generativelanguage.googleapis.com/v1beta", - reasoning: source.reasoning === true, - ...(thinkingLevelMap ? { thinkingLevelMap } : {}), - input: source.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(source.cost), - contextWindow: source.limit?.context || 4096, - maxTokens: source.limit?.output || 4096, - }); - } - } - - // The google-vertex models.dev catalog also includes Claude, OpenAI, and other - // MaaS models that do not use the @google/genai Gemini streaming path. - const vertexModels = data["google-vertex"]?.models; - if (vertexModels) { - for (const [modelId, model] of Object.entries(vertexModels)) { - if (model.tool_call !== true || !modelId.startsWith("gemini-")) continue; - if (modelId === "gemini-3.1-flash-lite-preview") continue; - const source = - modelId === "gemini-flash-latest" - ? (vertexModels["gemini-3.5-flash"] ?? model) - : modelId === "gemini-flash-lite-latest" - ? (vertexModels["gemini-3.1-flash-lite"] ?? model) - : model; - const thinkingLevelMap = getGoogleThinkingLevelMap(modelId, source.reasoning_options ?? []); - // models.dev reports Vertex cache_read/cache_write values for Gemini 2.5 Flash that - // do not match the official Gemini API standard pricing table. pi only accounts - // cachedContentTokenCount as cacheRead. - const cacheRead = modelId === "gemini-2.5-flash" ? 0.03 : source.cost?.cache_read || 0; - - models.push({ - id: modelId, - name: model.name || modelId, - api: "google-vertex", - provider: "google-vertex", - baseUrl: VERTEX_BASE_URL, - reasoning: source.reasoning === true, - ...(thinkingLevelMap ? { thinkingLevelMap } : {}), - input: source.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: { - input: source.cost?.input || 0, - output: source.cost?.output || 0, - cacheRead, - cacheWrite: 0, - }, - contextWindow: source.limit?.context || 4096, - maxTokens: source.limit?.output || 4096, - }); - } - } - - return models; -} - -function processFireworksModels(provider: ModelsDevProvider | undefined): Model[] { - if (!provider?.models) return []; - - const anthropicCompat: AnthropicMessagesCompat = { - allowEmptySignature: true, - sendSessionAffinityHeaders: true, - supportsEagerToolInputStreaming: false, - supportsCacheControlOnTools: false, - supportsLongCacheRetention: false, - }; - const openAICompat: OpenAICompletionsCompat = { - supportsStore: false, - supportsDeveloperRole: false, - sendSessionAffinityHeaders: true, - supportsLongCacheRetention: false, - }; - const kimiK3Compat: OpenAICompletionsCompat = { - ...openAICompat, - requiresReasoningContentOnAssistantMessages: true, - thinkingFormat: "openai", - }; - const models: Model[] = []; - - for (const [modelId, model] of Object.entries(provider.models)) { - if (model.tool_call !== true) continue; - - const input: ("text" | "image")[] = model.modalities?.input?.includes("image") - ? ["text", "image"] - : ["text"]; - const common = { - id: modelId, - name: model.name || modelId, - provider: "fireworks", - reasoning: model.reasoning === true, - input, - cost: getModelsDevCost(model.cost), - contextWindow: model.limit?.context || 4096, - maxTokens: model.limit?.output || 4096, - }; - - if (modelId.includes("glm-")) { - models.push({ - ...common, - api: "openai-completions", - baseUrl: "https://api.fireworks.ai/inference/v1", - compat: openAICompat, - }); - } else if (modelId.includes("kimi-k3")) { - models.push({ - ...common, - api: "openai-completions", - baseUrl: "https://api.fireworks.ai/inference/v1", - compat: kimiK3Compat, - }); - } else { - models.push({ - ...common, - api: "anthropic-messages", - // Fireworks Anthropic-compatible API - SDK appends /v1/messages. - baseUrl: "https://api.fireworks.ai/inference", - // Fireworks prompt caching uses automatic prefix matching + session affinity. - // x-session-affinity routes requests to the same replica for cache hits. - // cache_control on tools and eager_input_streaming are not supported. - // See: https://docs.fireworks.ai/tools-sdks/anthropic-compatibility - // Use adaptive thinking for cataloged effort controls, with verified - // fallbacks where models.dev is incomplete. New models need no allowlist entry. - compat: { - ...anthropicCompat, - ...(model.reasoning_options?.some((option) => option.type === "effort") || - FIREWORKS_ADAPTIVE_THINKING_FALLBACK_MODELS.has(modelId) - ? { forceAdaptiveThinking: true } - : {}), - }, - }); - } - recordModelsDevReasoningOptions("fireworks", modelId, model); - } - - return models; -} - -// Kimi Coding is subscription-backed, so models.dev reports zero cost. Use the -// equivalent Moonshot API rates to estimate the value of subscription usage. -const KIMI_CODING_IMPLIED_COSTS: Record["cost"]> = { - k3: KIMI_K3_COST, - "k3-256k": KIMI_K3_COST, - "k3.1-pro": { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 0 }, - "k3.1-pro-highspeed": { input: 4, output: 20, cacheRead: 0.2, cacheWrite: 0 }, - "kimi-for-coding": { input: 0.95, output: 4, cacheRead: 0.19, cacheWrite: 0 }, - "kimi-for-coding-highspeed": { input: 1.9, output: 8, cacheRead: 0.38, cacheWrite: 0 }, -}; - -// Kimi Coding uses Chat Completions, the same protocol as the official Kimi Code client. -function createKimiCodingModels(kimiModels: Record): Model<"openai-completions">[] { - const models: Model<"openai-completions">[] = []; - for (const [modelId, m] of Object.entries(kimiModels)) { - if (m.tool_call !== true) continue; - - // K3-style models report effort levels and dynamic tool loading (tools declared in - // later system messages) in https://api.kimi.com/coding/v1/models; models.dev mirrors - // the effort levels. kimi-for-coding-highspeed reports neither and rejects - // tool-bearing system messages with a 400. - const isK3Style = m.reasoning_options?.some((option) => option.type === "effort") === true; - const impliedCost = KIMI_CODING_IMPLIED_COSTS[modelId]; - // K3-style models send reasoning_effort, so expose the effort values models.dev lists. - const effortLevels = isK3Style && m.reasoning_options ? getEffortThinkingLevelMap(m.reasoning_options) : undefined; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider: "kimi-coding", - baseUrl: "https://api.kimi.com/coding/v1", - compat: { - ...(isK3Style ? KIMI_K3_OPENAI_COMPAT : KIMI_OPENAI_COMPAT), - // All models accept system text after the conversation starts. - supportsMidConvoSystemMessages: true, - ...(isK3Style ? { supportsMidConvoToolAdditions: true } : {}), - }, - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: { - input: m.cost?.input || impliedCost?.input || 0, - output: m.cost?.output || impliedCost?.output || 0, - cacheRead: m.cost?.cache_read || impliedCost?.cacheRead || 0, - cacheWrite: m.cost?.cache_write || impliedCost?.cacheWrite || 0, - }, - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - // Kimi Code reports every model as always-thinking. - thinkingLevelMap: { ...effortLevels, off: null }, - }); - } - return models; -} - -async function loadModelsDevData(): Promise[]> { - try { - console.log("Fetching models from models.dev API..."); - const response = await fetch("https://models.dev/api.json"); - if (!response.ok) throw new Error(`models.dev API returned ${response.status}`); - const data = (await response.json()) as ModelsDevCatalog; - - const models: Model[] = []; - const nvidiaNimModelIds = data.nvidia?.models ? await fetchNvidiaNimModelIds() : new Map(); - - // Process Amazon Bedrock models - if (data["amazon-bedrock"]?.models) { - for (const [modelId, model] of Object.entries(data["amazon-bedrock"].models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - if (BEDROCK_INFERENCE_PROFILE_ONLY_MODEL_IDS.has(modelId)) continue; - - let id = modelId; - - if (id.startsWith("ai21.jamba")) { - // These models doesn't support tool use in streaming mode - continue; - } - - if (id.startsWith("mistral.mistral-7b-instruct-v0")) { - // These models doesn't support system messages - continue; - } - - models.push({ - id, - name: m.name || id, - api: "bedrock-converse-stream" as const, - provider: "amazon-bedrock" as const, - baseUrl: getBedrockBaseUrl(id), - reasoning: m.reasoning === true, - input: (m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"]) as ("text" | "image")[], - // Includes models.dev pricing tiers, e.g. the long-context tier for OpenAI models (#10326). - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - ...(m.structured_output === true && { compat: { supportsStrictMode: true } }), - }); - recordModelsDevReasoningOptions("amazon-bedrock" as const, id, m); - } - } - - // Process Anthropic models - if (data.anthropic?.models) { - for (const [modelId, model] of Object.entries(data.anthropic.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "anthropic-messages", - provider: "anthropic", - baseUrl: "https://api.anthropic.com", - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions("anthropic", modelId, m); - } - } - - models.push(...processGoogleModels(data)); - - // Process OpenAI models - if (data.openai?.models) { - for (const [modelId, model] of Object.entries(data.openai.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - // models.dev lists this alias, but it is not accepted by OpenAI APIs. - if (MODELS_DEV_OPENAI_UNSUPPORTED_MODEL_IDS.has(modelId)) continue; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-responses", - provider: "openai", - baseUrl: "https://api.openai.com/v1", - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions("openai", modelId, m); - } - } - - // Process Groq models - if (data.groq?.models) { - for (const [modelId, model] of Object.entries(data.groq.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider: "groq", - baseUrl: "https://api.groq.com/openai/v1", - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions("groq", modelId, m); - } - } - - // Process Cerebras models - if (data.cerebras?.models) { - for (const [modelId, model] of Object.entries(data.cerebras.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider: "cerebras", - baseUrl: "https://api.cerebras.ai/v1", - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions("cerebras", modelId, m); - } - } - - // Process Cloudflare Workers AI models - if (data["cloudflare-workers-ai"]?.models) { - for (const [modelId, model] of Object.entries(data["cloudflare-workers-ai"].models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider: "cloudflare-workers-ai", - baseUrl: CLOUDFLARE_WORKERS_AI_BASE_URL, - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - compat: { sendSessionAffinityHeaders: true }, - }); - recordModelsDevReasoningOptions("cloudflare-workers-ai", modelId, m); - } - } - - // Process Cloudflare AI Gateway models - const cloudflareAIGatewayModelIds = new Set(); - if (data["cloudflare-ai-gateway"]?.models) { - for (const [prefixedId, model] of Object.entries(data["cloudflare-ai-gateway"].models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - const slashIdx = prefixedId.indexOf("/"); - if (slashIdx === -1) continue; - const upstream = prefixedId.slice(0, slashIdx); - const nativeId = prefixedId.slice(slashIdx + 1); - - let api: "anthropic-messages" | "openai-completions" | "openai-responses"; - let baseUrl: string; - let id: string; - if (upstream === "openai") { - api = "openai-responses"; - baseUrl = CLOUDFLARE_AI_GATEWAY_OPENAI_BASE_URL; - id = nativeId; - } else if (upstream === "anthropic") { - api = "anthropic-messages"; - baseUrl = CLOUDFLARE_AI_GATEWAY_ANTHROPIC_BASE_URL; - // The /anthropic passthrough forwards the model ID to Anthropic unchanged. - // models.dev lists dotted versions (claude-opus-5.5), but Anthropic only - // accepts dashed IDs (claude-opus-5-5). - id = nativeId.replaceAll(".", "-"); - } else if (upstream === "workers-ai") { - api = "openai-completions"; - baseUrl = CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL; - id = prefixedId; - } else { - continue; - } - - // Gateway passthroughs forward session affinity headers to upstreams that - // use them for cache/routing affinity. - const compat = - upstream === "anthropic" || upstream === "workers-ai" ? { sendSessionAffinityHeaders: true } : undefined; - - cloudflareAIGatewayModelIds.add(id); - models.push({ - id, - name: m.name || id, - api, - provider: "cloudflare-ai-gateway", - baseUrl, - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - ...(compat ? { compat } : {}), - }); - recordModelsDevReasoningOptions("cloudflare-ai-gateway", id, m); - } - } - - // The gateway proxies Workers AI through its OpenAI-compatible /compat endpoint, - // but models.dev may omit or intermittently drop those `workers-ai/*` entries - // from the AI Gateway catalog. Mirror the Workers AI catalog under the documented - // prefix so the gateway keeps its OpenAI-compatible models stable. - if (data["cloudflare-workers-ai"]?.models) { - for (const [modelId, model] of Object.entries(data["cloudflare-workers-ai"].models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - const id = `workers-ai/${modelId}`; - if (cloudflareAIGatewayModelIds.has(id)) continue; - cloudflareAIGatewayModelIds.add(id); - - models.push({ - id, - name: m.name || id, - api: "openai-completions", - provider: "cloudflare-ai-gateway", - baseUrl: CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL, - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - compat: { sendSessionAffinityHeaders: true }, - }); - recordModelsDevReasoningOptions("cloudflare-ai-gateway", id, m); - } - } - - // Process xAi models - if (data.xai?.models) { - for (const [modelId, model] of Object.entries(data.xai.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-responses", - provider: "xai", - baseUrl: "https://api.x.ai/v1", - compat: { ...XAI_RESPONSES_COMPAT }, - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions("xai", modelId, m); - } - } - - // Process Meta models - if (data.meta?.models) { - for (const [modelId, model] of Object.entries(data.meta.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-responses", - provider: "meta", - baseUrl: "https://api.meta.ai/v1", - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions("meta", modelId, m); - } - } - - models.push(...processZaiModels(data)); - - // Process Mistral models - if (data.mistral?.models) { - for (const [modelId, model] of Object.entries(data.mistral.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - // Models with effort values use `reasoning_effort` with these levels. - // Reasoning models without them (Magistral) use `prompt_mode`. - const thinkingLevelMap = getEffortThinkingLevelMap(m.reasoning_options ?? []); - - models.push({ - id: modelId, - name: m.name || modelId, - api: "mistral-conversations", - provider: "mistral", - baseUrl: "https://api.mistral.ai", - reasoning: m.reasoning === true, - ...(thinkingLevelMap ? { thinkingLevelMap } : {}), - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: { - input: m.cost?.input || 0, - output: m.cost?.output || 0, - cacheRead: m.cost?.cache_read ?? (m.cost?.input ? roundCost(m.cost.input * 0.1) : 0), - cacheWrite: m.cost?.cache_write || 0, - }, - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - } - } - - // Process Hugging Face models - if (data.huggingface?.models) { - for (const [modelId, model] of Object.entries(data.huggingface.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider: "huggingface", - baseUrl: "https://router.huggingface.co/v1", - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - compat: { - supportsDeveloperRole: false, - }, - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions("huggingface", modelId, m); - } - } - - models.push(...processFireworksModels(data["fireworks-ai"])); - - // Process NVIDIA NIM models - if (data.nvidia?.models) { - for (const [modelId, model] of Object.entries(data.nvidia.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - if (!m.modalities?.input?.includes("text")) continue; - if (!m.modalities?.output?.includes("text")) continue; - - const liveModelId = nvidiaNimModelIds.get(modelId) ?? nvidiaNimModelIds.get(normalizeNvidiaModelId(modelId)); - if (!liveModelId) continue; - if (NVIDIA_NIM_UNSUPPORTED_MODELS.has(liveModelId)) continue; - - models.push({ - id: liveModelId, - name: m.name || liveModelId, - api: "openai-completions", - provider: "nvidia", - baseUrl: NVIDIA_BASE_URL, - headers: { ...NVIDIA_HEADERS }, - reasoning: m.reasoning === true, - input: m.modalities.input.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - compat: NVIDIA_OPENAI_COMPAT, - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions("nvidia", liveModelId, m); - } - } - - // Process Together AI models - const togetherProvider = data.together ?? data.togetherai ?? data["together-ai"]; - if (togetherProvider?.models) { - for (const [modelId, model] of Object.entries(togetherProvider.models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - if (m.status === "deprecated") continue; - - const reasoning = m.reasoning === true; - const thinkingLevelMap = getTogetherThinkingLevelMap(modelId, reasoning); - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider: "together", - baseUrl: TOGETHER_BASE_URL, - reasoning, - ...(thinkingLevelMap ? { thinkingLevelMap } : {}), - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - compat: getTogetherCompat(modelId, reasoning), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions("together", modelId, m); - } - } - - models.push(...processBasetenModels(data.baseten)); - - // Process OpenCode models (Zen and Go) - // API mapping based on provider.npm field: - // - @ai-sdk/openai → openai-responses - // - @ai-sdk/anthropic → anthropic-messages - // - @ai-sdk/google → google-generative-ai - // - null/undefined/@ai-sdk/openai-compatible → openai-completions - const opencodeVariants = [ - { key: "opencode", provider: "opencode", basePath: "https://opencode.ai/inference" }, - { key: "opencode-go", provider: "opencode-go", basePath: "https://opencode.ai/inference/go" }, - ] as const; - - for (const variant of opencodeVariants) { - if (!data[variant.key]?.models) continue; - - for (const [modelId, model] of Object.entries(data[variant.key].models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - if (m.status === "deprecated") continue; - - const npm = m.provider?.npm; - let api: Api; - let baseUrl: string; - let compat: OpenAICompletionsCompat | OpenAIResponsesCompat | undefined; - - if (npm === "@ai-sdk/openai") { - api = "openai-responses"; - baseUrl = `${variant.basePath}/openai/v1`; - compat = { sessionAffinityFormat: "openai-nosession" }; - } else if (npm === "@ai-sdk/anthropic") { - api = "anthropic-messages"; - // Anthropic SDK appends /v1/messages to baseURL - baseUrl = `${variant.basePath}/anthropic`; - } else if (npm === "@ai-sdk/google") { - api = "google-generative-ai"; - baseUrl = `${variant.basePath}/google/v1beta`; - } else if (npm === "@ai-sdk/alibaba") { - api = "openai-completions"; - baseUrl = `${variant.basePath}/openai/v1`; - compat = { cacheControlFormat: "anthropic" }; - } else { - // null, undefined, or @ai-sdk/openai-compatible - api = "openai-completions"; - baseUrl = `${variant.basePath}/openai/v1`; - } - - if (variant.provider === "opencode" && modelId === "grok-build-0.1") { - compat = { ...(compat ?? {}), supportsReasoningEffort: false }; - } - - if ((variant.provider === "opencode" || variant.provider === "opencode-go") && modelId === "kimi-k2.6") { - // OpenCode Kimi K2.6 accepts Anthropic-style thinking objects - // and rejects string thinking values or combined reasoning_effort. - compat = { ...(compat ?? {}), thinkingFormat: "deepseek", supportsReasoningEffort: false }; - } - - // Fix known mismatches between models.dev npm data and actual - // OpenCode Go endpoint behaviour. models.dev reports these models - // as @ai-sdk/anthropic, but OpenCode Go serves them through the - // OpenAI-compatible /openai/v1/chat/completions path. - if (variant.provider === "opencode-go") { - if (modelId === "qwen3.5-plus" || modelId === "qwen3.6-plus") { - api = "openai-completions"; - baseUrl = `${variant.basePath}/openai/v1`; - // Qwen/DashScope uses enable_thinking at the top level. - compat = { ...(compat ?? {}), thinkingFormat: "qwen" }; - } - } - - if (api === "openai-completions") { - compat = { ...(compat ?? {}), maxTokensField: "max_tokens" }; - if ( - OPENCODE_OPENAI_COMPLETIONS_LONG_CACHE_RETENTION_UNSUPPORTED_MODELS.has( - `${variant.provider}:${modelId}`, - ) - ) { - compat = { ...compat, supportsLongCacheRetention: false }; - } - } - - let thinkingLevelMap: NonNullable["thinkingLevelMap"]> | undefined; - if (api === "google-generative-ai") { - thinkingLevelMap = getGoogleThinkingLevelMap(modelId, m.reasoning_options ?? []); - } else if (variant.provider === "opencode-go" && modelId === "deepseek-v4.1-flash") { - thinkingLevelMap = getEffortThinkingLevelMap(m.reasoning_options ?? []); - } - models.push({ - id: modelId, - name: m.name || modelId, - api, - provider: variant.provider, - baseUrl, - reasoning: m.reasoning === true, - ...(thinkingLevelMap ? { thinkingLevelMap } : {}), - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - ...(compat ? { compat } : {}), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - recordModelsDevReasoningOptions(variant.provider, modelId, m); - } - } - - // Process GitHub Copilot models - if (data["github-copilot"]?.models) { - for (const [modelId, model] of Object.entries(data["github-copilot"].models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - if (m.status === "deprecated") continue; - - // Claude 4.x and 5.x models route to Anthropic Messages API - const isCopilotClaude = /^claude-(haiku|sonnet|opus|fable)-[45]([.\-]|$)/.test(modelId); - // GPT, Grok, OSWE, and MAI-Code models are only served through - // the Copilot /responses endpoint. - const needsResponsesApi = - modelId.startsWith("gpt-") || - modelId.startsWith("grok-") || - modelId.startsWith("oswe") || - modelId.startsWith("mai-"); - - const api: Api = isCopilotClaude - ? "anthropic-messages" - : needsResponsesApi - ? "openai-responses" - : "openai-completions"; - - const anthropicCompat = - api === "anthropic-messages" ? getAnthropicMessagesCompat("github-copilot", modelId) : undefined; - - const copilotModel: Model = { - id: modelId, - name: m.name || modelId, - api, - provider: "github-copilot", - baseUrl: "https://api.individual.githubcopilot.com", - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 128000, - maxTokens: m.limit?.output || 8192, - headers: { ...COPILOT_STATIC_HEADERS }, - ...(anthropicCompat ? { compat: anthropicCompat } : {}), - // compat only applies to openai-completions - ...(api === "openai-completions" ? { - compat: { - supportsStore: false, - supportsDeveloperRole: false, - supportsReasoningEffort: false, - }, - } : {}), - }; - - models.push(copilotModel); - recordModelsDevReasoningOptions("github-copilot", modelId, m); - } - } - - // Process MiniMax models - const minimaxVariants = [ - { key: "minimax", provider: "minimax", baseUrl: "https://api.minimax.io/anthropic" }, - { key: "minimax-cn", provider: "minimax-cn", baseUrl: "https://api.minimaxi.com/anthropic" }, - ] as const; - // MiniMax asked to hide older models from its direct providers (#2596). - const minimaxHiddenIds = new Set(["MiniMax-M2", "MiniMax-M2.1", "MiniMax-M2.5", "MiniMax-M2.5-highspeed"]); - - for (const { key, provider, baseUrl } of minimaxVariants) { - if (data[key]?.models) { - for (const [modelId, model] of Object.entries(data[key].models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true || minimaxHiddenIds.has(modelId)) continue; - // Models with effort levels (e.g. MiniMax-M3.1-Flash) take adaptive thinking with - // output_config.effort and reject thinking.type="disabled" (no "none" effort, so off is null). - const thinkingLevelMap = getEffortThinkingLevelMap(m.reasoning_options ?? []); - - models.push({ - id: modelId, - name: m.name || modelId, - api: "anthropic-messages", - provider, - // MiniMax's Anthropic-compatible API - SDK appends /v1/messages - baseUrl, - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - ...(thinkingLevelMap ? { thinkingLevelMap, compat: { forceAdaptiveThinking: true } } : {}), - }); - } - } - } - - // Process Kimi For Coding models - if (data["kimi-code-plan-global"]?.models) { - models.push(...createKimiCodingModels(data["kimi-code-plan-global"].models as Record)); - } - - // Process Moonshot AI models - const moonshotVariants = [ - { key: "moonshotai", provider: "moonshotai", baseUrl: "https://api.moonshot.ai/v1" }, - { key: "moonshotai-cn", provider: "moonshotai-cn", baseUrl: "https://api.moonshot.cn/v1" }, - ] as const; - const getMoonshotProviderModels = (key: "moonshotai" | "moonshotai-cn"): Record => { - const providerModels = data[key]?.models as Record | undefined; - return providerModels ? { ...providerModels } : {}; - }; - const moonshotModels = { - moonshotai: getMoonshotProviderModels("moonshotai"), - "moonshotai-cn": getMoonshotProviderModels("moonshotai-cn"), - }; - - for (const { key, provider, baseUrl } of moonshotVariants) { - for (const [modelId, m] of Object.entries(moonshotModels[key])) { - if (m.tool_call !== true) continue; - - const isKimiK3 = isKimiK3FamilyId(modelId); - const compat = { ...(isKimiK3 ? KIMI_K3_OPENAI_COMPAT : KIMI_OPENAI_COMPAT) }; - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider, - baseUrl, - reasoning: isKimiK3 || m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - // Moonshot does not bill cache writes; models.dev lists the input rate as cache_write for Kimi K3. - cost: - modelId === "kimi-k3" - ? { ...KIMI_K3_COST } - : { - input: m.cost?.input || 0, - output: m.cost?.output || 0, - cacheRead: m.cost?.cache_read || 0, - cacheWrite: isKimiK3 ? 0 : m.cost?.cache_write || 0, - }, - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - compat, - }); - recordModelsDevReasoningOptions(provider, modelId, m); - } - } - - // Process Xiaomi MiMo models - // Built-in `xiaomi` targets the API billing endpoint (single stable URL, - // keys from platform.xiaomimimo.com). The three `xiaomi-token-plan-*` - // providers cover prepaid Token Plan endpoints in cn / ams / sgp. - // MiMo Chat Completions only documents thinking.type enabled/disabled and - // rejects some reasoning_effort values (e.g. "minimal"). Xiaomi documents - // all effort levels as identical, so expose only off and high. - const xiaomiCompat: OpenAICompletionsCompat = { - supportsReasoningEffort: false, - requiresReasoningContentOnAssistantMessages: true, - thinkingFormat: "deepseek", - }; - const xiaomiThinkingLevelMap = { - off: "off", - minimal: null, - low: null, - medium: null, - high: "high", - xhigh: null, - max: null, - } as const; - const xiaomiVariants = [ - { source: "xiaomi", provider: "xiaomi", baseUrl: "https://api.xiaomimimo.com/v1" }, - { - source: "xiaomi-token-plan-cn", - provider: "xiaomi-token-plan-cn", - baseUrl: "https://token-plan-cn.xiaomimimo.com/v1", - }, - { - source: "xiaomi-token-plan-ams", - provider: "xiaomi-token-plan-ams", - baseUrl: "https://token-plan-ams.xiaomimimo.com/v1", - }, - { - source: "xiaomi-token-plan-sgp", - provider: "xiaomi-token-plan-sgp", - baseUrl: "https://token-plan-sgp.xiaomimimo.com/v1", - }, - ] as const; - - for (const { source, provider, baseUrl } of xiaomiVariants) { - const providerModels = data[source]?.models; - if (!providerModels) continue; - - for (const [modelId, model] of Object.entries(providerModels)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - if (m.status === "deprecated") continue; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider, - baseUrl, - compat: xiaomiCompat, - reasoning: m.reasoning === true, - ...(m.reasoning === true ? { thinkingLevelMap: xiaomiThinkingLevelMap } : {}), - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - } - } - - // Process Alibaba Cloud Model Studio Token Plan models. International and - // China use separate endpoints and API keys (sk-sp- prefix). The Individual - // provider reuses the international source and endpoint with a narrower catalog. - // models.dev keys are "alibaba-token-plan[-cn]"; pi exposes them as - // "qwen-token-plan[-cn]" plus the Individual catalog view. - const qwenTokenPlanCompat: OpenAICompletionsCompat = { - thinkingFormat: "qwen", - supportsDeveloperRole: false, - supportsStore: false, - supportsReasoningEffort: true, - }; - const qwenTokenPlanVariants = [ - { - source: "alibaba-token-plan", - provider: "qwen-token-plan", - baseUrl: "https://token-plan.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1", - modelIds: undefined, - }, - { - source: "alibaba-token-plan", - provider: "qwen-token-plan-individual", - baseUrl: "https://token-plan.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1", - modelIds: QWEN_TOKEN_PLAN_INDIVIDUAL_MODEL_IDS, - }, - { - source: "alibaba-token-plan-cn", - provider: "qwen-token-plan-cn", - baseUrl: "https://token-plan.cn-beijing.maas.aliyuncs.com/compatible-mode/v1", - modelIds: undefined, - }, - ] as const; - - for (const { source, provider, baseUrl, modelIds } of qwenTokenPlanVariants) { - const providerModels = data[source]?.models; - const emittedModelIds = modelIds ? new Set() : undefined; - - for (const [modelId, model] of Object.entries(providerModels ?? {})) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - if (QWEN_TOKEN_PLAN_EXCLUDED_MODEL_IDS.has(modelId)) continue; - if (modelIds && !modelIds.has(modelId)) continue; - const thinkingLevelMap = - getEffortThinkingLevelMap(m.reasoning_options ?? []) ?? - (QWEN_TOKEN_PLAN_REASONING_EFFORT_FALLBACK_MODEL_IDS.has(modelId) - ? QWEN_TOKEN_PLAN_FALLBACK_THINKING_LEVEL_MAP - : undefined); - - models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider, - baseUrl, - compat: thinkingLevelMap - ? qwenTokenPlanCompat - : { ...qwenTokenPlanCompat, supportsReasoningEffort: false }, - ...(thinkingLevelMap ? { thinkingLevelMap } : {}), - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: getModelsDevCost(m.cost), - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - }); - emittedModelIds?.add(modelId); - } - - if (modelIds && emittedModelIds && generatorOptions.strict) { - assertExactModelIds(provider, modelIds, emittedModelIds); - } - } - - console.log(`Loaded ${models.length} tool-capable models from models.dev`); - return models; - } catch (error) { - console.error("Failed to load models.dev data:", error); - if (generatorOptions.strict) throw error; - return []; - } -} - -async function loadModelsDevClassifierModels(): Promise[]> { - try { - console.log("Fetching classifier models from models.dev API..."); - const response = await fetch("https://models.dev/models.json?type=decision"); - if (!response.ok) throw new Error(`models.dev classifier API returned ${response.status}`); - const data = (await response.json()) as Record; - const metadata = data["typesafe/jev-latest"]; - if (!metadata || metadata.type !== "decision") { - throw new Error("models.dev did not return decision model typesafe/jev-latest"); - } - return [ - { - type: "classifier", - id: "jev-latest", - name: metadata.name, - api: "typesafe-system-one", - provider: "typesafe", - baseUrl: "https://api.typesafe.ai/v1/", - // models.dev lists text input only, but the API accepts image assets as an experimental - // feature behind an opt-in header, which the typesafe-system-one API sends. - // https://github.com/typesafe-ai/typesafe-sdk-js (v0.7.0 changelog) - input: ["text", "image"], - // The canonical models.dev entry has no direct-provider pricing. System One reports token usage, - // so classify() results carry token counts but price them at zero. - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: metadata.limit?.context || 64000, - compat: { imageInput: "assets", sendImageOptInHeader: true }, - }, - ]; - } catch (error) { - console.error("Failed to load models.dev classifier data:", error); - if (generatorOptions.strict) throw error; - return []; - } -} - -// OpenCode Zen serves Jev through its TypeSafe-compatible System One endpoint. -// Neither its /inference/v1/models listing nor models.dev carries metadata for it. -// https://opencode.ai/docs/zen -const OPENCODE_CLASSIFIER_MODELS: ClassifierModel<"typesafe-system-one">[] = [ - { - type: "classifier", - id: "jev-1.13", - name: "Jev 1.13", - api: "typesafe-system-one", - provider: "opencode", - baseUrl: "https://opencode.ai/inference/systemone/v1", - input: ["text"], - cost: { input: 0.042, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 32000, - }, - { - type: "classifier", - id: "jev-1.13-free", - name: "Jev 1.13 Free", - api: "typesafe-system-one", - provider: "opencode", - baseUrl: "https://opencode.ai/inference/systemone/v1", - input: ["text"], - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 32000, - }, -]; - -// Workers AI has no unauthenticated catalog, so its System One models are listed here from -// Cloudflare's model pages. -const CLOUDFLARE_WORKERS_AI_CLASSIFIER_MODELS: ClassifierModel<"cloudflare-workers-ai-system-one">[] = [ - // Cloudflare-hosted Clef decision models. They take up to four PNG, JPEG, or WebP images as data URLs. - // Clef Omni also takes audio and video, which classifier contexts do not carry. - // Pricing and context: https://developers.cloudflare.com/workers-ai/models/clef/, - // https://developers.cloudflare.com/workers-ai/models/clef-flash/, - // and https://developers.cloudflare.com/workers-ai/models/clef-omni/ - { - type: "classifier", - id: "@cf/cloudflare/clef", - name: "Clef", - api: "cloudflare-workers-ai-system-one", - provider: "cloudflare-workers-ai", - baseUrl: CLOUDFLARE_WORKERS_AI_REST_BASE_URL, - input: ["text", "image"], - cost: { input: 0.24, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 65536, - compat: { imageInput: "images" }, - }, - { - type: "classifier", - id: "@cf/cloudflare/clef-flash", - name: "Clef Flash", - api: "cloudflare-workers-ai-system-one", - provider: "cloudflare-workers-ai", - baseUrl: CLOUDFLARE_WORKERS_AI_REST_BASE_URL, - input: ["text", "image"], - cost: { input: 0.038, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 24576, - compat: { imageInput: "images" }, - }, - { - type: "classifier", - id: "@cf/cloudflare/clef-omni", - name: "Clef Omni", - api: "cloudflare-workers-ai-system-one", - provider: "cloudflare-workers-ai", - baseUrl: CLOUDFLARE_WORKERS_AI_REST_BASE_URL, - input: ["text", "image"], - cost: { input: 0.15, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 64000, - compat: { imageInput: "images" }, - }, - // TypeSafe's Jev models take PNG and JPEG images as System One assets. - // https://developers.cloudflare.com/ai/models/typesafe/jev/ (jev-latest) - // and https://developers.cloudflare.com/ai/models/typesafe/jev-1.14.0/ - { - type: "classifier", - id: "typesafe/jev", - name: "Jev", - api: "cloudflare-workers-ai-system-one", - provider: "cloudflare-workers-ai", - baseUrl: CLOUDFLARE_WORKERS_AI_REST_BASE_URL, - input: ["text", "image"], - cost: { input: 0.042, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 32000, - compat: { imageInput: "assets" }, - }, - { - type: "classifier", - id: "typesafe/jev-1.14.0", - name: "Jev 1.14", - api: "cloudflare-workers-ai-system-one", - provider: "cloudflare-workers-ai", - baseUrl: CLOUDFLARE_WORKERS_AI_REST_BASE_URL, - input: ["text", "image"], - cost: { input: 0.042, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 32000, - compat: { imageInput: "assets" }, - }, -]; - -// OpenAI Decisions API (public beta): gpt-6-luna is its only model. It bills input tokens only, -// with the same long-context multiplier as chat requests. Only API keys work: Sign in with ChatGPT -// tokens are rejected on /v1/decisions, and the Codex backend has no Decisions route. -// The endpoint rejects inputs above 922K tokens (the model's documented maximum input), but -// requests running longer than about five seconds, currently above roughly 600K input tokens, -// fail with a gateway timeout. -// https://developers.openai.com/api/docs/guides/decisions -const OPENAI_CLASSIFIER_MODELS: ClassifierModel<"openai-decisions">[] = [ - { - type: "classifier", - id: "gpt-6-luna", - name: "GPT-6 Luna", - api: "openai-decisions", - provider: "openai", - baseUrl: "https://api.openai.com/v1", - input: ["text", "image"], - cost: withOpenAiLongContextPricing({ input: 0.1, output: 0, cacheRead: 0, cacheWrite: 0 }), - contextWindow: 922000, - }, -]; - -async function generateModels() { - // Fetch models from all upstream catalogs. - // models.dev: Anthropic, Google, OpenAI, Groq, Cerebras, and others - // OpenRouter: its tool-capable routed catalog - // AI Gateway: OpenAI-compatible catalog with tool-capable models - // Radius: its unauthenticated public catalog; authenticated clients overlay it at runtime - const modelsDevModels = await loadModelsDevData(); - const modelsDevClassifierModels = await loadModelsDevClassifierModels(); - const openRouterCatalog = await fetchOpenRouterModels(); - const aiGatewayCatalog = await fetchAiGatewayModels(); - const radiusModels = await fetchRadiusModels(); - - // Combine chat models (models.dev has priority where sources overlap). - const allModels = [...modelsDevModels, ...openRouterCatalog.chat, ...aiGatewayCatalog.chat, ...radiusModels].filter( - (model) => - !(model.provider === "xai" && XAI_BUILTIN_EXCLUDED_MODEL_IDS.has(model.id)) && - !((model.provider === "opencode" || model.provider === "opencode-go") && model.id === "gpt-5.3-codex-spark"), - ); - - // The authenticated Copilot catalog advertised these models on 2026-09-22, - // but models.dev did not include them yet. - const missingCopilotModels: Model[] = [ - { - id: "claude-opus-5.5", - name: "Claude Opus 5.5", - api: "anthropic-messages", - provider: "github-copilot", - baseUrl: "https://api.individual.githubcopilot.com", - reasoning: true, - thinkingLevelMap: { - off: null, - minimal: null, - low: "low", - medium: "medium", - high: "high", - xhigh: "xhigh", - max: "max", - }, - input: ["text", "image"], - cost: { input: 4, output: 20, cacheRead: 0.2, cacheWrite: 5 }, - contextWindow: 1000000, - maxTokens: 128000, - headers: { ...COPILOT_STATIC_HEADERS }, - }, - ...(["gpt-6-sol", "gpt-6-luna"] as const).map((id) => ({ - id, - name: id === "gpt-6-sol" ? "GPT-6 Sol" : "GPT-6 Luna", - api: "openai-responses" as const, - provider: "github-copilot" as const, - baseUrl: "https://api.individual.githubcopilot.com", - reasoning: true, - input: ["text", "image"] as ("text" | "image")[], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS[id]), - contextWindow: 1000000, - maxTokens: 128000, - headers: { ...COPILOT_STATIC_HEADERS }, - })), - ]; - for (const model of missingCopilotModels) { - if (!allModels.some((candidate) => candidate.provider === model.provider && candidate.id === model.id)) { - allModels.push(model); - } - } - - // Temporary overrides until upstream model metadata is corrected. - for (const candidate of allModels) { - if (candidate.provider === "github-copilot" && GITHUB_COPILOT_EXTENDED_CONTEXT_MODELS.has(candidate.id)) { - candidate.contextWindow = 1000000; - } - - // models.dev may list Opus 5.5, Sonnet 5.5, and Haiku 5.5 before their effort metadata is complete. - if ( - (candidate.provider === "anthropic" && - (candidate.id === "claude-opus-5-5" || - candidate.id === "claude-sonnet-5-5" || - candidate.id === "claude-haiku-5-5")) || - (candidate.provider === "github-copilot" && candidate.id === "claude-opus-5.5") - ) { - mergeThinkingLevelMap(candidate, { - // Haiku 5.5 accepts disabled thinking; Opus 5.5 and Sonnet 5.5 reject it. - ...(candidate.id === "claude-haiku-5-5" ? {} : { off: null }), - minimal: null, - low: "low", - medium: "medium", - high: "high", - xhigh: "xhigh", - max: "max", - }); - } - - if ( - (candidate.provider === "anthropic" || - candidate.provider === "opencode" || - candidate.provider === "opencode-go") && - (candidate.id === "claude-opus-4-6" || - candidate.id === "claude-sonnet-4-6" || - candidate.id === "claude-opus-4.6" || - candidate.id === "claude-sonnet-4.6") - ) { - candidate.contextWindow = 1000000; - } - - // OpenCode variants list Claude Sonnet 4/4.5 with 1M context, actual limit is 200K - if ( - (candidate.provider === "opencode" || candidate.provider === "opencode-go") && - (candidate.id === "claude-sonnet-4-5" || candidate.id === "claude-sonnet-4") - ) { - candidate.contextWindow = 200000; - } - if ((candidate.provider === "opencode" || candidate.provider === "opencode-go") && candidate.id === "gpt-5.4") { - candidate.contextWindow = 272000; - candidate.maxTokens = 128000; - } - // Keep direct OpenAI requests in the short-context pricing tier by default. Users can opt into the - // larger context through model overrides, so retain long-context cost metadata on the capped models. - if (candidate.provider === "openai" && OPENAI_SHORT_CONTEXT_CAPPED_MODEL_IDS.has(candidate.id)) { - candidate.contextWindow = OPENAI_LONG_CONTEXT_INPUT_THRESHOLD; - candidate.maxTokens = 128000; - } - if (candidate.provider === "openai" && OPENAI_LONG_CONTEXT_PRICING_MODEL_IDS.has(candidate.id)) { - const standardCost = OPENAI_STANDARD_COSTS[candidate.id]; - candidate.cost = withOpenAiLongContextPricing(standardCost ?? candidate.cost); - } - // Cloudflare AI Gateway passes OpenAI usage through at OpenAI list prices. - if (candidate.provider === "cloudflare-ai-gateway") { - const standardCost = OPENAI_STANDARD_COSTS[candidate.id]; - if (standardCost) candidate.cost = withOpenAiLongContextPricing(standardCost); - } - // models.dev reports gpt-5-pro output as 272000 (a duplicate of the input sub-limit); - // the actual max output is 128000. Also propagates to the derived Azure clone. - if (candidate.provider === "openai" && candidate.id === "gpt-5-pro") { - candidate.maxTokens = 128000; - } - // Keep Kimi K3's canonical output limit when gateway metadata is missing or incorrect. - if ( - (candidate.provider === "openrouter" && candidate.id === "~moonshotai/kimi-latest") || - ((candidate.provider === "openrouter" || candidate.provider === "vercel-ai-gateway") && - candidate.id.startsWith("moonshotai/") && - isKimiK3FamilyId(candidate.id.slice("moonshotai/".length))) - ) { - candidate.maxTokens = KIMI_K3_MAX_TOKENS; - } - // Keep selected OpenRouter model metadata stable until upstream settles. - if (candidate.provider === "openrouter" && candidate.id === "moonshotai/kimi-k2.5") { - candidate.cost.input = 0.41; - candidate.cost.output = 2.06; - candidate.cost.cacheRead = 0.07; - candidate.maxTokens = 4096; - } - if (candidate.provider === "openrouter" && candidate.id.startsWith("moonshotai/kimi-k2.6")) { - candidate.compat = { - ...candidate.compat, - supportsDeveloperRole: false, - requiresReasoningContentOnAssistantMessages: true, - }; - } - if (candidate.provider === "openrouter" && candidate.id === "z-ai/glm-5") { - candidate.cost.input = 0.6; - candidate.cost.output = 1.9; - candidate.cost.cacheRead = 0.119; - } - } - - // Add missing gpt models - const missingOpenAiModels: Model<"openai-responses">[] = [ - { - id: "gpt-6.1-sol", - name: "GPT-6.1 Sol", - api: "openai-responses", - baseUrl: "https://api.openai.com/v1", - provider: "openai", - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6.1-sol"]), - contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, - maxTokens: 128000, - }, - { - id: "gpt-6.1-terra", - name: "GPT-6.1 Terra", - api: "openai-responses", - baseUrl: "https://api.openai.com/v1", - provider: "openai", - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6.1-terra"]), - contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, - maxTokens: 128000, - }, - { - id: "gpt-6-astra", - name: "GPT-6 Astra", - api: "openai-responses", - baseUrl: "https://api.openai.com/v1", - provider: "openai", - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6-astra"]), - contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, - maxTokens: 128000, - }, - { - id: "gpt-6-sol", - name: "GPT-6 Sol", - api: "openai-responses", - baseUrl: "https://api.openai.com/v1", - provider: "openai", - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6-sol"]), - contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, - maxTokens: 128000, - }, - { - id: "gpt-6-luna", - name: "GPT-6 Luna", - api: "openai-responses", - baseUrl: "https://api.openai.com/v1", - provider: "openai", - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6-luna"]), - contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, - maxTokens: 128000, - }, - { - id: "gpt-5.6-sol", - name: "GPT-5.6 Sol", - api: "openai-responses", - baseUrl: "https://api.openai.com/v1", - provider: "openai", - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-5.6-sol"]), - contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, - maxTokens: 128000, - }, - { - id: "gpt-5.6-terra", - name: "GPT-5.6 Terra", - api: "openai-responses", - baseUrl: "https://api.openai.com/v1", - provider: "openai", - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-5.6-terra"]), - contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, - maxTokens: 128000, - }, - { - id: "gpt-5.6-luna", - name: "GPT-5.6 Luna", - api: "openai-responses", - baseUrl: "https://api.openai.com/v1", - provider: "openai", - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-5.6-luna"]), - contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD, - maxTokens: 128000, - }, - { - id: "gpt-5-chat-latest", - name: "GPT-5 Chat Latest", - api: "openai-responses", - baseUrl: "https://api.openai.com/v1", - provider: "openai", - reasoning: false, - input: ["text", "image"], - cost: { - input: 1.25, - output: 10, - cacheRead: 0.125, - cacheWrite: 0, - }, - contextWindow: 128000, - maxTokens: 16384, - }, - ]; - for (const model of missingOpenAiModels) { - if (!allModels.some((m) => m.provider === model.provider && m.id === model.id)) { - allModels.push(model); - } - } - - const deepseekCompat: OpenAICompletionsCompat = { - requiresReasoningContentOnAssistantMessages: true, - thinkingFormat: "deepseek", - }; - const deepseekModels: Model<"openai-completions">[] = [ - { - id: "deepseek-flash", - name: "DeepSeek V4.1 Flash", - api: "openai-completions", - baseUrl: "https://api.deepseek.com", - provider: "deepseek", - reasoning: true, - thinkingLevelMap: DEEPSEEK_V4_FLASH_THINKING_LEVEL_MAP, - input: ["text", "image"], - cost: { - // DeepSeek also offers time-based off-peak rates, which the cost schema cannot represent yet. - input: 0.3, - output: 1.2, - cacheRead: 0.006, - cacheWrite: 0, - }, - contextWindow: 1000000, - maxTokens: 384000, - compat: deepseekCompat, - }, - { - id: "deepseek-v4-pro", - name: "DeepSeek V4 Pro", - api: "openai-completions", - baseUrl: "https://api.deepseek.com", - provider: "deepseek", - reasoning: true, - input: ["text"], - cost: { - // DeepSeek also offers time-based off-peak rates, which the cost schema cannot represent yet. - input: 1.32, - output: 3.96, - cacheRead: 0.044, - cacheWrite: 0, - }, - contextWindow: 1000000, - maxTokens: 384000, - compat: deepseekCompat, - }, - ]; - allModels.push(...deepseekModels); - - const antLingCompat: OpenAICompletionsCompat = { - supportsStore: false, - supportsDeveloperRole: false, - supportsReasoningEffort: false, - maxTokensField: "max_tokens", - supportsLongCacheRetention: false, - }; - const antLingModels: Model<"openai-completions">[] = [ - { - id: "Ling-2.6-flash", - name: "Ling 2.6 Flash", - api: "openai-completions", - baseUrl: "https://api.ant-ling.com/v1", - provider: "ant-ling", - reasoning: false, - input: ["text"], - cost: { input: 0.01, output: 0.02, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 262144, - maxTokens: 65536, - compat: antLingCompat, - }, - { - id: "Ling-2.6-1T", - name: "Ling 2.6 1T", - api: "openai-completions", - baseUrl: "https://api.ant-ling.com/v1", - provider: "ant-ling", - reasoning: false, - input: ["text"], - cost: { input: 0.06, output: 0.25, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 262144, - maxTokens: 65536, - compat: antLingCompat, - }, - { - id: "Ring-2.6-1T", - name: "Ring 2.6 1T", - api: "openai-completions", - baseUrl: "https://api.ant-ling.com/v1", - provider: "ant-ling", - reasoning: true, - input: ["text"], - cost: { input: 0.06, output: 0.25, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 262144, - maxTokens: 65536, - compat: { ...antLingCompat, thinkingFormat: "ant-ling" }, - }, - ]; - allModels.push(...antLingModels); - - for (const candidate of allModels) { - if ( - candidate.api === "openai-completions" && - candidate.id.includes("deepseek-v4") && - !QWEN_TOKEN_PLAN_PROVIDER_IDS.has(candidate.provider) - ) { - const preservesNativeReasoningEffort = candidate.provider === "openrouter" || candidate.provider === "opencode"; - candidate.compat = { - ...candidate.compat, - ...(preservesNativeReasoningEffort - ? { - requiresReasoningContentOnAssistantMessages: - deepseekCompat.requiresReasoningContentOnAssistantMessages, - } - : deepseekCompat), - }; - } - } - - // OpenAI Codex (ChatGPT OAuth) models - // NOTE: These are not fetched from models.dev; we keep a small, explicit list to avoid aliases. - // Only list models the Codex backend accepts for ChatGPT accounts. GPT-5.5 and GPT-5.3 Codex Spark are rejected - // with "not supported when using Codex with a ChatGPT account". - // GPT-5.6 and GPT-6 use Codex's 272k default catalog limit. - // GPT-6.1 Sol uses the 500k default returned by /backend-api/codex/models. - const CODEX_BASE_URL = "https://chatgpt.com/backend-api"; - const CODEX_CONTEXT = 272000; - const CODEX_GPT_61_CONTEXT = 500000; - const CODEX_GPT_56_CONTEXT = 272000; - const CODEX_MAX_TOKENS = 128000; - const codexModels: Model<"openai-codex-responses">[] = [ - { - id: "gpt-6.1-sol", - name: "GPT-6.1 Sol", - api: "openai-codex-responses", - provider: "openai-codex", - baseUrl: CODEX_BASE_URL, - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6.1-sol"]), - contextWindow: CODEX_GPT_61_CONTEXT, - maxTokens: CODEX_MAX_TOKENS, - }, - { - id: "gpt-6.1-terra", - name: "GPT-6.1 Terra", - api: "openai-codex-responses", - provider: "openai-codex", - baseUrl: CODEX_BASE_URL, - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6.1-terra"]), - contextWindow: CODEX_GPT_61_CONTEXT, - maxTokens: CODEX_MAX_TOKENS, - }, - { - id: "gpt-6-astra", - name: "GPT-6 Astra", - api: "openai-codex-responses", - provider: "openai-codex", - baseUrl: CODEX_BASE_URL, - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6-astra"]), - contextWindow: CODEX_CONTEXT, - maxTokens: CODEX_MAX_TOKENS, - }, - { - id: "gpt-6-sol", - name: "GPT-6 Sol", - api: "openai-codex-responses", - provider: "openai-codex", - baseUrl: CODEX_BASE_URL, - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6-sol"]), - contextWindow: CODEX_CONTEXT, - maxTokens: CODEX_MAX_TOKENS, - }, - { - id: "gpt-6-luna", - name: "GPT-6 Luna", - api: "openai-codex-responses", - provider: "openai-codex", - baseUrl: CODEX_BASE_URL, - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-6-luna"]), - contextWindow: CODEX_CONTEXT, - maxTokens: CODEX_MAX_TOKENS, - }, - { - id: "gpt-5.6-luna", - name: "GPT-5.6 Luna", - api: "openai-codex-responses", - provider: "openai-codex", - baseUrl: CODEX_BASE_URL, - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-5.6-luna"]), - contextWindow: CODEX_GPT_56_CONTEXT, - maxTokens: CODEX_MAX_TOKENS, - }, - { - id: "gpt-5.6-sol", - name: "GPT-5.6 Sol", - api: "openai-codex-responses", - provider: "openai-codex", - baseUrl: CODEX_BASE_URL, - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-5.6-sol"]), - contextWindow: CODEX_GPT_56_CONTEXT, - maxTokens: CODEX_MAX_TOKENS, - }, - { - id: "gpt-5.6-terra", - name: "GPT-5.6 Terra", - api: "openai-codex-responses", - provider: "openai-codex", - baseUrl: CODEX_BASE_URL, - reasoning: true, - input: ["text", "image"], - cost: withOpenAiLongContextPricing(OPENAI_STANDARD_COSTS["gpt-5.6-terra"]), - contextWindow: CODEX_GPT_56_CONTEXT, - maxTokens: CODEX_MAX_TOKENS, - }, - ]; - allModels.push(...codexModels); - - // Add missing Mistral Medium 3.5 model until models.dev includes it - if (!allModels.some(m => m.provider === "mistral" && m.id === "mistral-medium-3.5")) { - allModels.push({ - id: "mistral-medium-3.5", - name: "Mistral Medium 3.5", - api: "mistral-conversations", - provider: "mistral", - baseUrl: "https://api.mistral.ai", - reasoning: true, - thinkingLevelMap: getEffortThinkingLevelMap([{ type: "effort", values: ["none", "high"] }]), - input: ["text", "image"], - cost: { - input: 1.5, - output: 7.5, - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: 262144, // 256k tokens - maxTokens: 262144, - }); - } - - // Add "auto" alias for openrouter/auto - if (!allModels.some(m => m.provider === "openrouter" && m.id === "auto")) { - allModels.push({ - id: "auto", - name: "Auto", - api: "openai-completions", - provider: "openrouter", - baseUrl: "https://openrouter.ai/api/v1", - reasoning: true, - input: ["text", "image"], - cost: { - // we dont know about the costs because OpenRouter auto routes to different models - // and then charges you for the underlying used model - input:0, - output:0, - cacheRead:0, - cacheWrite:0, - }, - contextWindow: 2000000, - maxTokens: 30000, - }); - } - - // Add "fusion" alias for openrouter/fusion. OpenRouter exposes Fusion as a - // router alias/plugin entry point; its model metadata does not advertise - // tools, but the alias resolves to a concrete model that can invoke caller - // tools and has the openrouter:fusion server tool auto-injected. - if (!allModels.some(m => m.provider === "openrouter" && m.id === "openrouter/fusion")) { - allModels.push({ - id: "openrouter/fusion", - name: "OpenRouter: Fusion", - api: "openai-completions", - provider: "openrouter", - baseUrl: "https://openrouter.ai/api/v1", - reasoning: true, - input: ["text"], - cost: { - // we dont know about the costs because Fusion routes to multiple models - // and then charges you for the underlying used models - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: 1000000, - maxTokens: 30000, - }); - } - - // Azure Foundry deploys these with larger context windows than OpenAI's own short-tier defaults. - // See models-sold-directly-by-azure docs. - const AZURE_CONTEXT_WINDOW_OVERRIDES: Record = { - "gpt-5.4": 1050000, - "gpt-5.5": 1050000, - "gpt-5.6-luna": 1050000, - "gpt-5.6-sol": 1050000, - "gpt-5.6-terra": 1050000, - }; - const azureOpenAiModels: Model[] = allModels - .filter((model) => model.provider === "openai" && model.api === "openai-responses") - .map((model) => ({ - ...model, - api: "azure-openai-responses", - provider: "azure", - baseUrl: "", - cost: { - input: model.cost.input, - output: model.cost.output, - cacheRead: model.cost.cacheRead, - cacheWrite: model.cost.cacheWrite, - }, - contextWindow: AZURE_CONTEXT_WINDOW_OVERRIDES[model.id] ?? model.contextWindow, - })); - allModels.push(...azureOpenAiModels); - - // Azure resells DeepSeek at its own rates. US data zone, checked 2026-09-16. - // https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/ - const AZURE_DEEPSEEK_V4_PRO_COST: ModelCost = { input: 1.925, output: 3.828, cacheRead: 0.165, cacheWrite: 0 }; - // Azure 400s on DeepSeek's `thinking` field and on every prompt cache parameter, discards a - // `developer` system message unbilled once reasoning_effort is set, and honours mid-convo ones (#9645). - const azureDeepSeekModels: Model[] = allModels - .filter((model) => model.provider === "deepseek" && model.id === "deepseek-v4-pro") - .map((model) => ({ - ...model, - provider: "azure", - baseUrl: "", - cost: AZURE_DEEPSEEK_V4_PRO_COST, - compat: { - ...(model.compat as OpenAICompletionsCompat), - supportsDeveloperRole: false, - supportsMidConvoSystemMessages: true, - thinkingFormat: "openai", - supportsLongCacheRetention: false, - }, - })); - allModels.push(...azureDeepSeekModels); - - for (const model of allModels) { - applyOpenAICompletionsCompatMetadata(model); - applyAnthropicMessagesCompatMetadata(model); - applyModelsDevReasoningOptionMetadata(model); - applyThinkingLevelMetadata(model); - applyStrictToolCompatMetadata(model); - applyOpenAIGrammarToolCompatMetadata(model); - applyOpenAIToolSearchMetadata(model); - applyOpenAICompletionsTranscriptMetadata(model); - applyOpenAIResponsesTranscriptMetadata(model); - applyOpenAIExplicitPromptCacheMetadata(model); - applyOpenAIMidConvoEffortMetadata(model); - applyPromptCacheMetadata(model); - applyImageInputMetadata(model); - } - applyAnthropicAllowedFallbackModelMetadata(allModels.filter(isAnthropicFallbackMetadataModel)); - - // Keep chat and image catalogs separate so one upstream ID can expose both - // operations with different API implementations. - type ProviderCatalog = { - chat: Record>; - image: Record>; - classifier: Record>; - }; - const providers: Record = {}; - for (const model of allModels) { - providers[model.provider] ??= { chat: {}, image: {}, classifier: {} }; - // Only add if not already present (models.dev takes priority over OpenRouter). - providers[model.provider].chat[model.id] ??= { ...model, type: "chat" }; - } - for (const model of openRouterCatalog.images) { - applyImageInputMetadata(model); - providers[model.provider] ??= { chat: {}, image: {}, classifier: {} }; - providers[model.provider].image[model.id] ??= model; - } - const classifierModels: ClassifierModel[] = [ - ...modelsDevClassifierModels, - ...openRouterCatalog.classifiers, - ...aiGatewayCatalog.classifiers, - ...OPENCODE_CLASSIFIER_MODELS, - ...CLOUDFLARE_WORKERS_AI_CLASSIFIER_MODELS, - ...OPENAI_CLASSIFIER_MODELS, - ]; - for (const model of classifierModels) { - providers[model.provider] ??= { chat: {}, image: {}, classifier: {} }; - providers[model.provider].classifier[model.id] ??= model; - } - - const sortedProviderIds = Object.keys(providers).sort(); - const jsonChatProviders: Record>> = {}; - const jsonImageProviders: Record>> = {}; - const jsonClassifierProviders: Record>> = {}; - const jsonAllProviders: Record = {}; - for (const providerId of sortedProviderIds) { - jsonChatProviders[providerId] = Object.fromEntries( - Object.entries(providers[providerId].chat).sort(([left], [right]) => left.localeCompare(right)), - ); - jsonImageProviders[providerId] = Object.fromEntries( - Object.entries(providers[providerId].image).sort(([left], [right]) => left.localeCompare(right)), - ); - jsonClassifierProviders[providerId] = Object.fromEntries( - Object.entries(providers[providerId].classifier).sort(([left], [right]) => left.localeCompare(right)), - ); - jsonAllProviders[providerId] = [ - ...Object.values(jsonChatProviders[providerId]), - ...Object.values(jsonImageProviders[providerId]), - ...Object.values(jsonClassifierProviders[providerId]), - ]; - } - - const serializeJson = (value: unknown) => `${JSON.stringify(value, null, generatorOptions.pretty ? 2 : undefined)}\n`; - const writeJson = (path: string, value: unknown) => writeFileSync(path, serializeJson(value)); - const generatedDataProviderIds = generatorOptions.dataOnly - ? readModelDataProviderIds(packageRoot) - : sortedProviderIds; - const missingProviderIds = generatedDataProviderIds.filter((providerId) => !jsonAllProviders[providerId]); - if (missingProviderIds.length > 0) { - throw new Error(`Cannot hydrate missing providers: ${missingProviderIds.join(", ")}`); - } - - // Only the ignored internal data is grouped by API for type derivation. - const generatedDataProviders: Record>> = {}; - const modelDataStructure: ModelDataStructure = {}; - for (const providerId of generatedDataProviderIds) { - const { groups, structure } = groupProviderModelData(providerId, jsonAllProviders[providerId]); - generatedDataProviders[providerId] = groups; - modelDataStructure[providerId] = structure; - } - - const generatedAt = new Date().toISOString(); - - if (!generatorOptions.jsonOnly) { - // Stage and validate all provider values before replacing the current generated data. - const providersDir = join(packageRoot, "src/providers"); - const dataDir = join(providersDir, "data"); - const stagingRoot = mkdtempSync(join(providersDir, ".model-generation-")); - const stagedDataDir = join(stagingRoot, "data"); - const previousDataDir = join(stagingRoot, "previous-data"); - let restoreGeneratedCatalog: (() => void) | undefined; - try { - mkdirSync(stagedDataDir, { recursive: true }); - const fileContents: Record = {}; - for (const providerId of generatedDataProviderIds) { - const filename = `${providerId}.json`; - const content = serializeJson(generatedDataProviders[providerId]); - fileContents[filename] = content; - writeFileSync(join(stagedDataDir, filename), content); - } - writeJson( - join(stagedDataDir, MODEL_DATA_MANIFEST_FILE), - createModelDataManifest(modelDataStructure, fileContents, generatedAt), - ); - validateModelDataDirectory(modelDataStructure, stagedDataDir); - - if (!generatorOptions.dataOnly) { - const previousShardContents = new Map( - readdirSync(providersDir) - .filter((entry) => entry.endsWith(".models.ts")) - .map((entry) => [entry, readFileSync(join(providersDir, entry), "utf8")] as const), - ); - const aggregatorPath = join(packageRoot, "src/models.generated.ts"); - const previousAggregator = readFileSync(aggregatorPath, "utf8"); - restoreGeneratedCatalog = () => { - for (const entry of readdirSync(providersDir)) { - if (entry.endsWith(".models.ts")) rmSync(join(providersDir, entry)); - } - for (const [entry, content] of previousShardContents) { - writeFileSync(join(providersDir, entry), content); - } - writeFileSync(aggregatorPath, previousAggregator); - }; - - const generatedHeader = `// This file is auto-generated by scripts/generate-models.ts -// Do not edit manually - run 'npm run generate-models' to update - -`; - const catalogConstName = (providerId: string) => - `${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_MODELS`; - const imageCatalogConstName = (providerId: string) => - `${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_IMAGE_MODELS`; - const classifierCatalogConstName = (providerId: string) => - `${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_CLASSIFIER_MODELS`; - const generatedShardFiles = new Set(); - for (const providerId of sortedProviderIds) { - let output = generatedHeader; - output += `import values from "./data/${providerId}.json" with { type: "json" };\n`; - output += `import { flattenChatModelCatalog, flattenClassifierModelCatalog, flattenImageModelCatalog, type ChatModelCatalog, type ClassifierModelCatalog, type ImageModelCatalog } from "../model-catalog.ts";\n\n`; - output += `export const ${catalogConstName(providerId)}: ChatModelCatalog =\n`; - output += `\tflattenChatModelCatalog(${JSON.stringify(providerId)}, values);\n\n`; - output += `export const ${imageCatalogConstName(providerId)}: ImageModelCatalog =\n`; - output += `\tflattenImageModelCatalog(${JSON.stringify(providerId)}, values);\n\n`; - output += `export const ${classifierCatalogConstName(providerId)}: ClassifierModelCatalog =\n`; - output += `\tflattenClassifierModelCatalog(${JSON.stringify(providerId)}, values);\n`; - const filename = `${providerId}.models.ts`; - generatedShardFiles.add(filename); - writeFileSync(join(providersDir, filename), output); - } - for (const entry of readdirSync(providersDir)) { - if (entry.endsWith(".models.ts") && !generatedShardFiles.has(entry)) rmSync(join(providersDir, entry)); - } - - let output = generatedHeader; - for (const providerId of sortedProviderIds) { - output += `import { ${classifierCatalogConstName(providerId)}, ${imageCatalogConstName(providerId)}, ${catalogConstName(providerId)} } from "./providers/${providerId}.models.ts";\n`; - } - output += `\nexport const MODELS: {\n`; - for (const providerId of sortedProviderIds) { - output += `\treadonly ${JSON.stringify(providerId)}: typeof ${catalogConstName(providerId)};\n`; - } - output += `} = {\n`; - for (const providerId of sortedProviderIds) { - output += `\t${JSON.stringify(providerId)}: ${catalogConstName(providerId)},\n`; - } - output += `};\n\nexport const IMAGE_MODELS: {\n`; - for (const providerId of sortedProviderIds) { - output += `\treadonly ${JSON.stringify(providerId)}: typeof ${imageCatalogConstName(providerId)};\n`; - } - output += `} = {\n`; - for (const providerId of sortedProviderIds) { - output += `\t${JSON.stringify(providerId)}: ${imageCatalogConstName(providerId)},\n`; - } - output += `};\n\nexport const CLASSIFIER_MODELS: {\n`; - for (const providerId of sortedProviderIds) { - output += `\treadonly ${JSON.stringify(providerId)}: typeof ${classifierCatalogConstName(providerId)};\n`; - } - output += `} = {\n`; - for (const providerId of sortedProviderIds) { - output += `\t${JSON.stringify(providerId)}: ${classifierCatalogConstName(providerId)},\n`; - } - output += `};\n`; - writeFileSync(aggregatorPath, output); - console.log("Generated provider catalogs and src/models.generated.ts"); - } - - const hadPreviousData = existsSync(dataDir); - if (hadPreviousData) renameSync(dataDir, previousDataDir); - try { - renameSync(stagedDataDir, dataDir); - validateGeneratedModelData(packageRoot); - } catch (error) { - rmSync(dataDir, { recursive: true, force: true }); - if (hadPreviousData && existsSync(previousDataDir)) renameSync(previousDataDir, dataDir); - throw error; - } - restoreGeneratedCatalog = undefined; - console.log( - generatorOptions.dataOnly - ? "Hydrated JSON model values under src/providers/data/" - : "Generated JSON model values under src/providers/data/", - ); - } catch (error) { - restoreGeneratedCatalog?.(); - throw error; - } finally { - rmSync(stagingRoot, { recursive: true, force: true }); - } - } - - if (generatorOptions.jsonOutputDir) { - // `models.json` and `providers/{id}.json` retain the legacy keyed chat catalog. - // The `.all` variants are arrays so the same upstream id can appear once per type. - const providerOutputDir = join(generatorOptions.jsonOutputDir, "providers"); - rmSync(generatorOptions.jsonOutputDir, { recursive: true, force: true }); - mkdirSync(providerOutputDir, { recursive: true }); - writeJson(join(generatorOptions.jsonOutputDir, "models.json"), jsonChatProviders); - writeJson(join(generatorOptions.jsonOutputDir, "models.all.json"), jsonAllProviders); - writeJson(join(generatorOptions.jsonOutputDir, "providers.json"), sortedProviderIds); - for (const providerId of sortedProviderIds) { - writeJson(join(providerOutputDir, `${providerId}.json`), jsonChatProviders[providerId]); - writeJson(join(providerOutputDir, `${providerId}.all.json`), jsonAllProviders[providerId]); - } - console.log(`Generated JSON model catalog under ${generatorOptions.jsonOutputDir}`); - } - - // Print statistics - const totalModels = allModels.length; - const reasoningModels = allModels.filter((model) => model.reasoning).length; - - console.log(`\nModel Statistics:`); - console.log(` Total tool-capable models: ${totalModels}`); - console.log(` Reasoning-capable models: ${reasoningModels}`); - - for (const [provider, models] of Object.entries(providers)) { - console.log( - ` ${provider}: ${Object.keys(models.chat).length} chat models, ${Object.keys(models.image).length} image models, ${Object.keys(models.classifier).length} classifier models`, - ); - } -} - -// Run the generator -generateModels().catch((error) => { - console.error(error); - process.exitCode = 1; -}); diff --git a/packages/ai/scripts/openrouter-reasoning-options.ts b/packages/ai/scripts/openrouter-reasoning-options.ts deleted file mode 100644 index 6fc4a5c6c..000000000 --- a/packages/ai/scripts/openrouter-reasoning-options.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { ThinkingLevel, ThinkingLevelMap } from "../src/types.ts"; -import { getEffortThinkingLevelMap } from "./models-dev-reasoning-options.ts"; - -export interface OpenRouterReasoningMetadata { - mandatory?: boolean; - default_enabled?: boolean; - supported_efforts?: Array; - default_effort?: ThinkingLevel | "none"; -} - -/** Convert OpenRouter's reasoning metadata into Pi model capabilities. */ -export function getOpenRouterThinkingLevelMap( - reasoning: OpenRouterReasoningMetadata | undefined, -): ThinkingLevelMap | undefined { - if (!reasoning) return undefined; - if (!reasoning.supported_efforts?.length) return reasoning.mandatory === true ? { off: null } : undefined; - - // OpenRouter's supported_efforts uses the same effort values as models.dev reasoning_options, - // so both sources can share the same Pi thinking-level conversion. - const map = getEffortThinkingLevelMap([{ type: "effort", values: reasoning.supported_efforts }]); - if (!map) return reasoning.mandatory === true ? { off: null } : undefined; - return { ...map, off: reasoning.mandatory === true ? null : "none" }; -} diff --git a/packages/ai/src/models.generated.ts b/packages/ai/src/models.generated.ts index 0890c4cee..93f2889fd 100644 --- a/packages/ai/src/models.generated.ts +++ b/packages/ai/src/models.generated.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import { AMAZON_BEDROCK_CLASSIFIER_MODELS, AMAZON_BEDROCK_IMAGE_MODELS, AMAZON_BEDROCK_MODELS } from "./providers/amazon-bedrock.models.ts"; diff --git a/packages/ai/src/providers/amazon-bedrock.models.ts b/packages/ai/src/providers/amazon-bedrock.models.ts index 37b348ef7..0d4030b6a 100644 --- a/packages/ai/src/providers/amazon-bedrock.models.ts +++ b/packages/ai/src/providers/amazon-bedrock.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/amazon-bedrock.json" with { type: "json" }; diff --git a/packages/ai/src/providers/ant-ling.models.ts b/packages/ai/src/providers/ant-ling.models.ts index bc42a8c62..4f66c20ba 100644 --- a/packages/ai/src/providers/ant-ling.models.ts +++ b/packages/ai/src/providers/ant-ling.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/ant-ling.json" with { type: "json" }; diff --git a/packages/ai/src/providers/anthropic.models.ts b/packages/ai/src/providers/anthropic.models.ts index c70b3160c..656c9fabb 100644 --- a/packages/ai/src/providers/anthropic.models.ts +++ b/packages/ai/src/providers/anthropic.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/anthropic.json" with { type: "json" }; diff --git a/packages/ai/src/providers/azure.models.ts b/packages/ai/src/providers/azure.models.ts index f86701758..0fb15ec40 100644 --- a/packages/ai/src/providers/azure.models.ts +++ b/packages/ai/src/providers/azure.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/azure.json" with { type: "json" }; diff --git a/packages/ai/src/providers/baseten.models.ts b/packages/ai/src/providers/baseten.models.ts index 864eb1f99..d68edd3e1 100644 --- a/packages/ai/src/providers/baseten.models.ts +++ b/packages/ai/src/providers/baseten.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/baseten.json" with { type: "json" }; diff --git a/packages/ai/src/providers/cerebras.models.ts b/packages/ai/src/providers/cerebras.models.ts index 39b1d2900..e3454f2fa 100644 --- a/packages/ai/src/providers/cerebras.models.ts +++ b/packages/ai/src/providers/cerebras.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/cerebras.json" with { type: "json" }; diff --git a/packages/ai/src/providers/cloudflare-ai-gateway.models.ts b/packages/ai/src/providers/cloudflare-ai-gateway.models.ts index 95e656542..ff524e062 100644 --- a/packages/ai/src/providers/cloudflare-ai-gateway.models.ts +++ b/packages/ai/src/providers/cloudflare-ai-gateway.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/cloudflare-ai-gateway.json" with { type: "json" }; diff --git a/packages/ai/src/providers/cloudflare-workers-ai.models.ts b/packages/ai/src/providers/cloudflare-workers-ai.models.ts index d84e49d16..0d34f6fae 100644 --- a/packages/ai/src/providers/cloudflare-workers-ai.models.ts +++ b/packages/ai/src/providers/cloudflare-workers-ai.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/cloudflare-workers-ai.json" with { type: "json" }; diff --git a/packages/ai/src/providers/deepseek.models.ts b/packages/ai/src/providers/deepseek.models.ts index 5ece8a190..9c92fb885 100644 --- a/packages/ai/src/providers/deepseek.models.ts +++ b/packages/ai/src/providers/deepseek.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/deepseek.json" with { type: "json" }; diff --git a/packages/ai/src/providers/fireworks.models.ts b/packages/ai/src/providers/fireworks.models.ts index 44cfa0856..53144268b 100644 --- a/packages/ai/src/providers/fireworks.models.ts +++ b/packages/ai/src/providers/fireworks.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/fireworks.json" with { type: "json" }; diff --git a/packages/ai/src/providers/github-copilot.models.ts b/packages/ai/src/providers/github-copilot.models.ts index 4b853d477..a71210730 100644 --- a/packages/ai/src/providers/github-copilot.models.ts +++ b/packages/ai/src/providers/github-copilot.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/github-copilot.json" with { type: "json" }; diff --git a/packages/ai/src/providers/google-vertex.models.ts b/packages/ai/src/providers/google-vertex.models.ts index a1778eecd..b081a3c71 100644 --- a/packages/ai/src/providers/google-vertex.models.ts +++ b/packages/ai/src/providers/google-vertex.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/google-vertex.json" with { type: "json" }; diff --git a/packages/ai/src/providers/google.models.ts b/packages/ai/src/providers/google.models.ts index 726a47440..ef5bec067 100644 --- a/packages/ai/src/providers/google.models.ts +++ b/packages/ai/src/providers/google.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/google.json" with { type: "json" }; diff --git a/packages/ai/src/providers/groq.models.ts b/packages/ai/src/providers/groq.models.ts index 75a84c36f..0af13338e 100644 --- a/packages/ai/src/providers/groq.models.ts +++ b/packages/ai/src/providers/groq.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/groq.json" with { type: "json" }; diff --git a/packages/ai/src/providers/huggingface.models.ts b/packages/ai/src/providers/huggingface.models.ts index 3bc971b6a..b8028a480 100644 --- a/packages/ai/src/providers/huggingface.models.ts +++ b/packages/ai/src/providers/huggingface.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/huggingface.json" with { type: "json" }; diff --git a/packages/ai/src/providers/kimi-coding.models.ts b/packages/ai/src/providers/kimi-coding.models.ts index 5b6ea5404..af36f3b47 100644 --- a/packages/ai/src/providers/kimi-coding.models.ts +++ b/packages/ai/src/providers/kimi-coding.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/kimi-coding.json" with { type: "json" }; diff --git a/packages/ai/src/providers/meta.models.ts b/packages/ai/src/providers/meta.models.ts index 81cd83917..d51b53db5 100644 --- a/packages/ai/src/providers/meta.models.ts +++ b/packages/ai/src/providers/meta.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/meta.json" with { type: "json" }; diff --git a/packages/ai/src/providers/minimax-cn.models.ts b/packages/ai/src/providers/minimax-cn.models.ts index b6cab80c6..db78342f4 100644 --- a/packages/ai/src/providers/minimax-cn.models.ts +++ b/packages/ai/src/providers/minimax-cn.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/minimax-cn.json" with { type: "json" }; diff --git a/packages/ai/src/providers/minimax.models.ts b/packages/ai/src/providers/minimax.models.ts index 4a46241a8..0adf27526 100644 --- a/packages/ai/src/providers/minimax.models.ts +++ b/packages/ai/src/providers/minimax.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/minimax.json" with { type: "json" }; diff --git a/packages/ai/src/providers/mistral.models.ts b/packages/ai/src/providers/mistral.models.ts index 0f7bed430..b59abdde8 100644 --- a/packages/ai/src/providers/mistral.models.ts +++ b/packages/ai/src/providers/mistral.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/mistral.json" with { type: "json" }; diff --git a/packages/ai/src/providers/moonshotai-cn.models.ts b/packages/ai/src/providers/moonshotai-cn.models.ts index e1cde46dc..b9183981a 100644 --- a/packages/ai/src/providers/moonshotai-cn.models.ts +++ b/packages/ai/src/providers/moonshotai-cn.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/moonshotai-cn.json" with { type: "json" }; diff --git a/packages/ai/src/providers/moonshotai.models.ts b/packages/ai/src/providers/moonshotai.models.ts index 19ac3d66e..41bd93e39 100644 --- a/packages/ai/src/providers/moonshotai.models.ts +++ b/packages/ai/src/providers/moonshotai.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/moonshotai.json" with { type: "json" }; diff --git a/packages/ai/src/providers/nvidia.models.ts b/packages/ai/src/providers/nvidia.models.ts index f7dfc374f..73e3a7c18 100644 --- a/packages/ai/src/providers/nvidia.models.ts +++ b/packages/ai/src/providers/nvidia.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/nvidia.json" with { type: "json" }; diff --git a/packages/ai/src/providers/openai-codex.models.ts b/packages/ai/src/providers/openai-codex.models.ts index 996f1fd2f..b5e910527 100644 --- a/packages/ai/src/providers/openai-codex.models.ts +++ b/packages/ai/src/providers/openai-codex.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/openai-codex.json" with { type: "json" }; diff --git a/packages/ai/src/providers/openai.models.ts b/packages/ai/src/providers/openai.models.ts index 06e0d4123..32faca489 100644 --- a/packages/ai/src/providers/openai.models.ts +++ b/packages/ai/src/providers/openai.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/openai.json" with { type: "json" }; diff --git a/packages/ai/src/providers/opencode-go.models.ts b/packages/ai/src/providers/opencode-go.models.ts index cb1dcc343..6231ee805 100644 --- a/packages/ai/src/providers/opencode-go.models.ts +++ b/packages/ai/src/providers/opencode-go.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/opencode-go.json" with { type: "json" }; diff --git a/packages/ai/src/providers/opencode.models.ts b/packages/ai/src/providers/opencode.models.ts index 812c584e3..c671fa275 100644 --- a/packages/ai/src/providers/opencode.models.ts +++ b/packages/ai/src/providers/opencode.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/opencode.json" with { type: "json" }; diff --git a/packages/ai/src/providers/openrouter.models.ts b/packages/ai/src/providers/openrouter.models.ts index 75d808b59..80bdc1c08 100644 --- a/packages/ai/src/providers/openrouter.models.ts +++ b/packages/ai/src/providers/openrouter.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/openrouter.json" with { type: "json" }; diff --git a/packages/ai/src/providers/qwen-token-plan-cn.models.ts b/packages/ai/src/providers/qwen-token-plan-cn.models.ts index 6b609faf7..4c6907031 100644 --- a/packages/ai/src/providers/qwen-token-plan-cn.models.ts +++ b/packages/ai/src/providers/qwen-token-plan-cn.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/qwen-token-plan-cn.json" with { type: "json" }; diff --git a/packages/ai/src/providers/qwen-token-plan-individual.models.ts b/packages/ai/src/providers/qwen-token-plan-individual.models.ts index 8d48afd89..bb2175425 100644 --- a/packages/ai/src/providers/qwen-token-plan-individual.models.ts +++ b/packages/ai/src/providers/qwen-token-plan-individual.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/qwen-token-plan-individual.json" with { type: "json" }; diff --git a/packages/ai/src/providers/qwen-token-plan.models.ts b/packages/ai/src/providers/qwen-token-plan.models.ts index c546c44d3..bb035d2e3 100644 --- a/packages/ai/src/providers/qwen-token-plan.models.ts +++ b/packages/ai/src/providers/qwen-token-plan.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/qwen-token-plan.json" with { type: "json" }; diff --git a/packages/ai/src/providers/radius.models.ts b/packages/ai/src/providers/radius.models.ts index c8f3b12d7..81070bd0a 100644 --- a/packages/ai/src/providers/radius.models.ts +++ b/packages/ai/src/providers/radius.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/radius.json" with { type: "json" }; diff --git a/packages/ai/src/providers/together.models.ts b/packages/ai/src/providers/together.models.ts index d8d5c919b..3e1c5d2e3 100644 --- a/packages/ai/src/providers/together.models.ts +++ b/packages/ai/src/providers/together.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/together.json" with { type: "json" }; diff --git a/packages/ai/src/providers/typesafe.models.ts b/packages/ai/src/providers/typesafe.models.ts index 05429c335..2be16f483 100644 --- a/packages/ai/src/providers/typesafe.models.ts +++ b/packages/ai/src/providers/typesafe.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/typesafe.json" with { type: "json" }; diff --git a/packages/ai/src/providers/vercel-ai-gateway.models.ts b/packages/ai/src/providers/vercel-ai-gateway.models.ts index e1c25832b..d9aff7dd9 100644 --- a/packages/ai/src/providers/vercel-ai-gateway.models.ts +++ b/packages/ai/src/providers/vercel-ai-gateway.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/vercel-ai-gateway.json" with { type: "json" }; diff --git a/packages/ai/src/providers/xai.models.ts b/packages/ai/src/providers/xai.models.ts index b1fb5d302..654b7e524 100644 --- a/packages/ai/src/providers/xai.models.ts +++ b/packages/ai/src/providers/xai.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/xai.json" with { type: "json" }; diff --git a/packages/ai/src/providers/xiaomi-token-plan-ams.models.ts b/packages/ai/src/providers/xiaomi-token-plan-ams.models.ts index 72b691d02..6c514b5d1 100644 --- a/packages/ai/src/providers/xiaomi-token-plan-ams.models.ts +++ b/packages/ai/src/providers/xiaomi-token-plan-ams.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/xiaomi-token-plan-ams.json" with { type: "json" }; diff --git a/packages/ai/src/providers/xiaomi-token-plan-cn.models.ts b/packages/ai/src/providers/xiaomi-token-plan-cn.models.ts index cae4bc12a..df833e743 100644 --- a/packages/ai/src/providers/xiaomi-token-plan-cn.models.ts +++ b/packages/ai/src/providers/xiaomi-token-plan-cn.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/xiaomi-token-plan-cn.json" with { type: "json" }; diff --git a/packages/ai/src/providers/xiaomi-token-plan-sgp.models.ts b/packages/ai/src/providers/xiaomi-token-plan-sgp.models.ts index 13db578e9..e965bd8e4 100644 --- a/packages/ai/src/providers/xiaomi-token-plan-sgp.models.ts +++ b/packages/ai/src/providers/xiaomi-token-plan-sgp.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/xiaomi-token-plan-sgp.json" with { type: "json" }; diff --git a/packages/ai/src/providers/xiaomi.models.ts b/packages/ai/src/providers/xiaomi.models.ts index 0ade9200d..8fd6f6949 100644 --- a/packages/ai/src/providers/xiaomi.models.ts +++ b/packages/ai/src/providers/xiaomi.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/xiaomi.json" with { type: "json" }; diff --git a/packages/ai/src/providers/zai-coding-cn.models.ts b/packages/ai/src/providers/zai-coding-cn.models.ts index 7dbd6a27c..bb30aba5d 100644 --- a/packages/ai/src/providers/zai-coding-cn.models.ts +++ b/packages/ai/src/providers/zai-coding-cn.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/zai-coding-cn.json" with { type: "json" }; diff --git a/packages/ai/src/providers/zai.models.ts b/packages/ai/src/providers/zai.models.ts index 7de6890f7..8bdb1ca58 100644 --- a/packages/ai/src/providers/zai.models.ts +++ b/packages/ai/src/providers/zai.models.ts @@ -1,4 +1,4 @@ -// This file is auto-generated by scripts/generate-models.ts +// This file is auto-generated by catalog/generate.ts // Do not edit manually - run 'npm run generate-models' to update import values from "./data/zai.json" with { type: "json" }; diff --git a/packages/ai/test/catalog-generation-utils.ts b/packages/ai/test/catalog-generation-utils.ts new file mode 100644 index 000000000..4e68f390d --- /dev/null +++ b/packages/ai/test/catalog-generation-utils.ts @@ -0,0 +1,52 @@ +import { writeFileSync } from "node:fs"; +import type { ModelsDevV2Catalog } from "../catalog/sources/models-dev.ts"; + +/** + * Write a `node --import` preload that serves every upstream catalog the + * generator fetches: the given models.dev v2 catalog, and minimal responses for + * the other sources. Any other request fails. + */ +export function writeUpstreamMock(preloadPath: string, modelsDev: ModelsDevV2Catalog): void { + const responses = { + "https://models.dev/experimental/v2.0/api.json": modelsDev, + "https://models.dev/models.json?type=decision": { + "typesafe/jev-latest": { id: "typesafe/jev-latest", name: "Jev", type: "decision", limit: { context: 64000 } }, + }, + "https://openrouter.ai/api/v1/models": { data: [] }, + "https://openrouter.ai/api/v1/models?output_modalities=image": { + data: [ + { + id: "test/image", + name: "Test Image", + architecture: { input_modalities: ["text"], output_modalities: ["image"] }, + pricing: {}, + }, + ], + }, + "https://openrouter.ai/api/v1/models?output_modalities=decisions": { data: [] }, + "https://ai-gateway.vercel.sh/v1/models": { data: [] }, + "https://radius.pi.dev/v1/config": { + baseUrl: "https://radius.pi.dev", + models: [ + { + id: "test", + name: "Test", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 4096, + maxTokens: 4096, + }, + ], + }, + }; + writeFileSync( + preloadPath, + `const responses = ${JSON.stringify(responses)};\n` + + `globalThis.fetch = async (input) => {\n` + + ` const url = String(input);\n` + + ` if (url in responses) return Response.json(responses[url]);\n` + + ` throw new Error(\`Unexpected fetch: \${url}\`);\n` + + `};\n`, + ); +} diff --git a/packages/ai/test/fireworks-model-generation.test.ts b/packages/ai/test/fireworks-model-generation.test.ts index 4b497bb77..189f80ac5 100644 --- a/packages/ai/test/fireworks-model-generation.test.ts +++ b/packages/ai/test/fireworks-model-generation.test.ts @@ -1,14 +1,15 @@ import { spawnSync } from "node:child_process"; -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { afterEach, describe, expect, it } from "vitest"; -import type { ModelsDevReasoningOption } from "../scripts/models-dev-reasoning-options.ts"; +import type { ModelsDevReasoningOption, ModelsDevV2Model } from "../catalog/sources/models-dev.ts"; import { streamSimple } from "../src/api/anthropic-messages.ts"; import { getSupportedThinkingLevels, hasApi } from "../src/models.ts"; import type { Api, Model } from "../src/types.ts"; import { normalizeContext } from "../src/utils/transcript.ts"; +import { writeUpstreamMock } from "./catalog-generation-utils.ts"; const packageRoot = fileURLToPath(new URL("..", import.meta.url)); const temporaryRoots: string[] = []; @@ -24,38 +25,32 @@ function generateFireworksModels( temporaryRoots.push(root); const preloadPath = join(root, "mock-catalog.mjs"); const outputPath = join(root, "catalog"); - const catalog = { - "fireworks-ai": { - models: Object.fromEntries( - Object.entries(options).map(([id, reasoning_options]) => [ - `accounts/fireworks/models/${id}`, - { id, tool_call: true, reasoning: true, reasoning_options }, - ]), - ), - }, - }; - writeFileSync( - preloadPath, - `const catalog = ${JSON.stringify(catalog)};\n` + - `globalThis.fetch = async (input) => {\n` + - ` const url = String(input);\n` + - ` if (url === "https://models.dev/api.json") return Response.json(catalog);\n` + - ` if (url === "https://models.dev/models.json?type=decision") return Response.json({ "typesafe/jev-latest": { name: "Jev", type: "decision", limit: { context: 64000, output: 0 } } });\n` + - ` if (url.startsWith("https://openrouter.ai/api/v1/models") || url === "https://ai-gateway.vercel.sh/v1/models") return Response.json({ data: [] });\n` + - ` if (url === "https://radius.pi.dev/v1/config") return Response.json({ baseUrl: "https://radius.pi.dev", models: [{ id: "test", name: "Test", reasoning: false, input: ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 4096, maxTokens: 4096 }] });\n` + - ` throw new Error(\`Unexpected fetch: \${url}\`);\n` + - `};\n`, - ); + const models = Object.entries(options).map(([id, reasoningOptions]): ModelsDevV2Model => { + const effort = reasoningOptions?.find((option) => option.type === "effort"); + const budget = reasoningOptions?.find((option) => option.type === "budget_tokens"); + return { + id: `accounts/fireworks/models/${id}`, + type: "chat", + name: id, + modalities: { input: ["text"], output: ["text"] }, + capabilities: { + tools: { supported: true }, + reasoning: { + supported: true, + ...(reasoningOptions?.some((option) => option.type === "toggle") ? { toggle: true } : {}), + ...(effort ? { effort: effort.values } : {}), + ...(budget ? { budget: { min: budget.min, max: budget.max } } : {}), + }, + }, + api: {}, + }; + }); + writeUpstreamMock(preloadPath, { + "fireworks-ai": { models: Object.fromEntries(models.map((model) => [model.id, model])) }, + }); const result = spawnSync( process.execPath, - [ - "--import", - pathToFileURL(preloadPath).href, - "scripts/generate-models.ts", - "--json-only", - "--json-output", - outputPath, - ], + ["--import", pathToFileURL(preloadPath).href, "catalog/generate.ts", "--json-only", "--json-output", outputPath], { cwd: packageRoot, encoding: "utf8", timeout: 10_000 }, ); expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); diff --git a/packages/ai/test/generate-models-strict.test.ts b/packages/ai/test/generate-models-strict.test.ts index 78d27f4ec..a5f3ffa8e 100644 --- a/packages/ai/test/generate-models-strict.test.ts +++ b/packages/ai/test/generate-models-strict.test.ts @@ -1,9 +1,11 @@ import { spawnSync } from "node:child_process"; -import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { afterEach, describe, expect, it } from "vitest"; +import type { ModelsDevV2Model } from "../catalog/sources/models-dev.ts"; +import { writeUpstreamMock } from "./catalog-generation-utils.ts"; const packageRoot = fileURLToPath(new URL("..", import.meta.url)); const temporaryRoots: string[] = []; @@ -18,7 +20,7 @@ describe("strict model generation", () => { temporaryRoots.push(fixtureRoot); const isolatedPackageRoot = join(fixtureRoot, "package"); mkdirSync(isolatedPackageRoot); - for (const entry of ["package.json", "scripts", "src"]) { + for (const entry of ["package.json", "catalog", "scripts", "src"]) { cpSync(join(packageRoot, entry), join(isolatedPackageRoot, entry), { recursive: true }); } const preloadPath = join(fixtureRoot, "mock-models-dev.mjs"); @@ -35,26 +37,22 @@ describe("strict model generation", () => { "qwen3.8-max-preview", ]; const sourceModels = Object.fromEntries( - modelIds.map((id) => [ + modelIds.map((id): [string, ModelsDevV2Model] => [ id, { id, + type: "chat", name: id, - tool_call: id !== "deepseek-v4-flash-0731", + modalities: { input: ["text"], output: ["text"] }, + capabilities: { + tools: { supported: id !== "deepseek-v4-flash-0731" }, + reasoning: { supported: false }, + }, + api: {}, }, ]), ); - const catalog = { "alibaba-token-plan": { models: sourceModels } }; - writeFileSync( - preloadPath, - `const catalog = ${JSON.stringify(catalog)};\n` + - `globalThis.fetch = async (input) => {\n` + - ` if (String(input) === "https://models.dev/api.json") {\n` + - ` return new Response(JSON.stringify(catalog), { status: 200 });\n` + - ` }\n` + - ` throw new Error(\`Unexpected fetch: \${String(input)}\`);\n` + - `};\n`, - ); + writeUpstreamMock(preloadPath, { "alibaba-token-plan": { models: sourceModels } }); const generatedPaths = [ "src/models.generated.ts", @@ -67,7 +65,7 @@ describe("strict model generation", () => { const result = spawnSync( process.execPath, - ["--import", pathToFileURL(preloadPath).href, "scripts/generate-models.ts", "--strict"], + ["--import", pathToFileURL(preloadPath).href, "catalog/generate.ts", "--strict"], { cwd: isolatedPackageRoot, encoding: "utf8", diff --git a/packages/ai/test/image-model-data.test.ts b/packages/ai/test/image-model-data.test.ts index 21545fe8f..645fdc242 100644 --- a/packages/ai/test/image-model-data.test.ts +++ b/packages/ai/test/image-model-data.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { buildOpenRouterCatalog, type OpenRouterModelListItem } from "../scripts/openrouter-catalog.ts"; +import { buildOpenRouterCatalog, type OpenRouterModelListItem } from "../catalog/sources/openrouter.ts"; const imageOnly: OpenRouterModelListItem = { id: "example/image-model", diff --git a/packages/ai/test/model-cost-tiers.test.ts b/packages/ai/test/model-cost-tiers.test.ts index ac74ea7c6..9eade534e 100644 --- a/packages/ai/test/model-cost-tiers.test.ts +++ b/packages/ai/test/model-cost-tiers.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { getAiGatewayCost } from "../scripts/ai-gateway-pricing.ts"; -import { buildOpenRouterCatalog, type OpenRouterModelListItem } from "../scripts/openrouter-catalog.ts"; +import { buildOpenRouterCatalog, type OpenRouterModelListItem } from "../catalog/sources/openrouter.ts"; +import { getAiGatewayCost } from "../catalog/sources/vercel-ai-gateway.ts"; function openRouterChatCost(pricing: OpenRouterModelListItem["pricing"]) { const catalog = buildOpenRouterCatalog( diff --git a/packages/ai/test/opencode-model-generation.test.ts b/packages/ai/test/opencode-model-generation.test.ts new file mode 100644 index 000000000..6bf861697 --- /dev/null +++ b/packages/ai/test/opencode-model-generation.test.ts @@ -0,0 +1,97 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import type { ModelsDevV2Model } from "../catalog/sources/models-dev.ts"; +import type { Api, Model } from "../src/types.ts"; +import { writeUpstreamMock } from "./catalog-generation-utils.ts"; + +const packageRoot = fileURLToPath(new URL("..", import.meta.url)); +const temporaryRoots: string[] = []; + +afterEach(() => { + for (const root of temporaryRoots.splice(0)) rmSync(root, { force: true, recursive: true }); +}); + +function sourceModel(id: string, api: ModelsDevV2Model["api"]): ModelsDevV2Model { + return { + id, + type: "chat", + name: id, + modalities: { input: ["text"], output: ["text"] }, + capabilities: { tools: { supported: true }, reasoning: { supported: false } }, + api, + }; +} + +function generateOpencodeModels(models: ModelsDevV2Model[], strict = false) { + const root = mkdtempSync(join(tmpdir(), "pi-opencode-generation-")); + temporaryRoots.push(root); + const preloadPath = join(root, "mock-catalog.mjs"); + const outputPath = join(root, "catalog"); + writeUpstreamMock(preloadPath, { + opencode: { models: Object.fromEntries(models.map((model) => [model.id, model])) }, + }); + const result = spawnSync( + process.execPath, + [ + "--import", + pathToFileURL(preloadPath).href, + "catalog/generate.ts", + ...(strict ? ["--strict"] : []), + "--json-only", + "--json-output", + outputPath, + ], + { cwd: packageRoot, encoding: "utf8", timeout: 10_000 }, + ); + return { + result, + models: () => + JSON.parse(readFileSync(join(outputPath, "providers/opencode.json"), "utf8")) as Record>, + }; +} + +describe("OpenCode model generation", () => { + it("uses the protocol and base URL models.dev lists for each model", () => { + const base = "https://example.test/zen"; + const { result, models } = generateOpencodeModels([ + sourceModel("chat", { "chat-completions": { base_url: `${base}/openai/v1` } }), + sourceModel("responses", { responses: { base_url: `${base}/openai/v1` } }), + sourceModel("messages", { messages: { base_url: `${base}/anthropic/v1` } }), + sourceModel("gemini", { "generate-content": { base_url: `${base}/google/v1beta` } }), + // Native protocols win over Chat Completions regardless of listing order. + sourceModel("both", { + "chat-completions": { base_url: `${base}/openai/v1` }, + messages: { base_url: `${base}/anthropic/v1` }, + }), + sourceModel("live", { "bidi-generate-content": { base_url: `${base}/google/v1beta` } }), + ]); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stderr).toContain("models.dev lists no supported API for opencode/live (bidi-generate-content)"); + + const generated = models(); + const endpoints = Object.fromEntries( + Object.values(generated).map((model) => [model.id, { api: model.api, baseUrl: model.baseUrl }]), + ); + expect(endpoints).toEqual({ + chat: { api: "openai-completions", baseUrl: `${base}/openai/v1` }, + responses: { api: "openai-responses", baseUrl: `${base}/openai/v1` }, + // The Anthropic SDK appends /v1/messages itself. + messages: { api: "anthropic-messages", baseUrl: `${base}/anthropic` }, + gemini: { api: "google-generative-ai", baseUrl: `${base}/google/v1beta` }, + both: { api: "anthropic-messages", baseUrl: `${base}/anthropic` }, + }); + }); + + it("fails strict generation for a model without a supported protocol", () => { + const { result } = generateOpencodeModels( + [sourceModel("live", { "bidi-generate-content": { base_url: "https://example.test/google/v1beta" } })], + true, + ); + expect(result.status).toBe(1); + expect(result.stderr).toContain("models.dev lists no supported API for opencode/live (bidi-generate-content)"); + }); +}); diff --git a/packages/ai/test/openrouter-reasoning-options.test.ts b/packages/ai/test/openrouter-reasoning-options.test.ts index dbeb1140e..743f58d6c 100644 --- a/packages/ai/test/openrouter-reasoning-options.test.ts +++ b/packages/ai/test/openrouter-reasoning-options.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { getOpenRouterThinkingLevelMap } from "../scripts/openrouter-reasoning-options.ts"; +import { getOpenRouterThinkingLevelMap } from "../catalog/sources/openrouter.ts"; import { streamSimple } from "../src/api/openai-completions.ts"; import type { Model, ThinkingLevelMap } from "../src/types.ts"; import { normalizeContext } from "../src/utils/transcript.ts"; diff --git a/packages/ai/test/reasoning-options.test.ts b/packages/ai/test/reasoning-options.test.ts index fafa9fa5e..135a0b9ec 100644 --- a/packages/ai/test/reasoning-options.test.ts +++ b/packages/ai/test/reasoning-options.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { getEffortThinkingLevelMap } from "../scripts/models-dev-reasoning-options.ts"; +import { getEffortThinkingLevelMap } from "../catalog/reasoning.ts"; describe("getEffortThinkingLevelMap", () => { it("exposes only verified effort values and none", () => { diff --git a/scripts/diff-model-catalog.mjs b/scripts/diff-model-catalog.mjs index e00563f76..1d2f10a4e 100644 --- a/scripts/diff-model-catalog.mjs +++ b/scripts/diff-model-catalog.mjs @@ -63,7 +63,11 @@ const currentThinkingOutput = join(temporaryRoot, "after-thinking"); let worktreeAdded = false; function generateCatalog(cwd, outputDir, pretty = false) { - const args = ["packages/ai/scripts/generate-models.ts", "--strict", "--json-only", "--json-output", outputDir]; + // Baselines from before the catalog moved still have the old generator. + const generator = existsSync(join(cwd, "packages/ai/catalog/generate.ts")) + ? "packages/ai/catalog/generate.ts" + : "packages/ai/scripts/generate-models.ts"; + const args = [generator, "--strict", "--json-only", "--json-output", outputDir]; if (pretty) args.push("--pretty"); run(process.execPath, args, { cwd, capture: true }); } diff --git a/scripts/issues/categories/models.md b/scripts/issues/categories/models.md index cc9d32953..774f24e99 100644 --- a/scripts/issues/categories/models.md +++ b/scripts/issues/categories/models.md @@ -6,7 +6,7 @@ owner: davidbrai Which models exist and which one is used. In: -- Built-in model catalog (generate-models, models.dev, pi.dev registry). +- Built-in model catalog (packages/ai/catalog, models.dev, pi.dev registry). - Model metadata: context window, max tokens, thinking level map, pricing. - models.json custom providers and models. - Model resolution: --model, --provider, defaultModel, startup fallback, scoped models, virtual models. diff --git a/tsconfig.json b/tsconfig.json index b530a76e4..8a90126b1 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -49,6 +49,6 @@ "@earendil-works/pi-agent-old/*": ["./packages/agent-old/src/*"] } }, - "include": ["scripts/model-catalog-protocol.ts", "scripts/model-catalog-protocol.test.ts", "packages/*/src/**/*", "packages/*/test/**/*", "packages/durable/test-workers/**/*", "packages/coding-agent/examples/**/*", "packages/agent/examples/**/*"], + "include": ["scripts/model-catalog-protocol.ts", "scripts/model-catalog-protocol.test.ts", "packages/*/src/**/*", "packages/*/test/**/*", "packages/ai/catalog/**/*", "packages/durable/test-workers/**/*", "packages/coding-agent/examples/**/*", "packages/agent/examples/**/*"], "exclude": ["**/dist/**", "packages/coding-agent/examples/extensions/gondolin/**"] } From 9628c84809ab657ff47237bc63e3052cac4587fa Mon Sep 17 00:00:00 2001 From: Christian Klotz Date: Wed, 28 Oct 2026 12:12:19 +0200 Subject: [PATCH 056/271] feat(coding-agent): add prompt template documentation eval (#10261) ## Summary - add live documentation comparisons for project-scoped and user-scoped `/current-time` prompt templates - require the generated command to expand to the exact requested prompt text - select the requested case from multi-case Vitest reports so skipped siblings do not invalidate observations - strengthen documentation-audit evidence and move the expensive audit behind an explicit `eval:audit` command - remove redundant eval oracles and test-only exports found during the eval-suite audit ## Testing - `npm run check` - `./test.sh` - `npm --prefix packages/evals test` (54 passed, also covered by `./test.sh`) - failed-status mutation: focused report test passed before the fixture repair, failed after it, then passed with production restored - prompt-template documentation eval, three repetitions per variant (6/6 pairs eligible) - project scope: with docs 66.7%, without docs 100%; negative delta and treatment flakiness - user scope: with docs 100%, without docs 100%; no lift - model documentation eval (1/1 pair eligible; treatment 100%, control 0%) - representative live documentation audit (`cli.md` passed) --- packages/evals/README.md | 12 +-- packages/evals/evals/configured-runtime.ts | 10 --- .../evals/evals/documentation-audit.eval.ts | 18 ++++- packages/evals/evals/extensions.docs.eval.ts | 2 - .../evals/evals/prompt-templates.docs.eval.ts | 79 +++++++++++++++++++ packages/evals/package.json | 1 + packages/evals/src/report.ts | 13 +-- packages/evals/test/comparison.test.ts | 2 + .../evals/test/configured-runtime.test.ts | 12 ++- packages/evals/test/report.test.ts | 79 ++++++++++--------- packages/evals/vitest.evals.config.ts | 12 ++- 11 files changed, 174 insertions(+), 66 deletions(-) create mode 100644 packages/evals/evals/prompt-templates.docs.eval.ts diff --git a/packages/evals/README.md b/packages/evals/README.md index 74c4904b0..406dec988 100644 --- a/packages/evals/README.md +++ b/packages/evals/README.md @@ -7,7 +7,7 @@ Behavioral evals for Pi's coding agent, built with `vitest-evals`. Eval definitions are flat under `evals/`: - `*.docs.eval.ts` is a documentation-lift eval. `eval:docs` runs each case in isolated `without_docs` and `with_docs` containers and reports lift. -- Other `*.eval.ts` files are host evals. `eval:host` runs them with Vitest on this machine. They are ordinary vitest-evals suites, not paired comparisons. +- Other `*.eval.ts` files are host evals. `eval:host` runs the smoke suite; `eval:audit` runs the documentation audit. They are ordinary vitest-evals suites, not paired comparisons. Runner code lives in `src/`: @@ -21,25 +21,25 @@ Eval suites and their fixtures live under `evals/`. Image build files live in `d ## Run evals -Host evals (smoke, documentation audit) and documentation-lift evals need `PI_PROVIDER` and `PI_MODEL`. +The host smoke eval, on-demand documentation audit, and documentation-lift evals need `PI_PROVIDER` and `PI_MODEL`. ```bash PI_PROVIDER=openai-codex PI_MODEL=gpt-5.6-sol npm run eval -w packages/evals ``` -That runs host evals, then the documentation comparison. Extra CLI flags after `--` go to `eval:docs` only. +That runs the host smoke eval, then the documentation comparison. The documentation audit remains explicit because it scans every documentation page. Extra CLI flags after `--` go to `eval:docs` only. -Host only: +Host smoke only: ```bash PI_PROVIDER=openai-codex PI_MODEL=gpt-5.6-sol npm run eval:host -w packages/evals ``` -One host suite: +Documentation audit: ```bash PI_PROVIDER=openai-codex PI_MODEL=gpt-5.6-sol \ - npm run eval:host -w packages/evals -- evals/documentation-audit.eval.ts + npm run eval:audit -w packages/evals ``` ## Run documentation comparisons diff --git a/packages/evals/evals/configured-runtime.ts b/packages/evals/evals/configured-runtime.ts index 8aaca0eee..c343634d5 100644 --- a/packages/evals/evals/configured-runtime.ts +++ b/packages/evals/evals/configured-runtime.ts @@ -1,4 +1,3 @@ -import { join } from "node:path"; import { type Api, type Context, contentText, type Model, type ModelsSimpleStreamOptions } from "@earendil-works/pi-ai"; import { ModelRuntime } from "@earendil-works/pi-coding-agent"; @@ -53,15 +52,6 @@ function modelFields(model: Model): ModelFields { }; } -export function loadConfiguredModelRuntime(agentDir: string): Promise { - return ModelRuntime.create({ - modelsPath: join(agentDir, "models.json"), - authPath: join(agentDir, "auth.json"), - modelsStorePath: join(agentDir, "models-store.json"), - allowModelNetwork: false, - }); -} - export async function inspectProvider( runtime: ModelRuntime, scenario: ProviderScenario, diff --git a/packages/evals/evals/documentation-audit.eval.ts b/packages/evals/evals/documentation-audit.eval.ts index dc2d6a582..baf8c6f0f 100644 --- a/packages/evals/evals/documentation-audit.eval.ts +++ b/packages/evals/evals/documentation-audit.eval.ts @@ -1,5 +1,5 @@ import { globSync } from "node:fs"; -import { resolve } from "node:path"; +import { isAbsolute, relative, resolve } from "node:path"; import { Type } from "@earendil-works/pi-ai"; import { defineTool } from "@earendil-works/pi-coding-agent"; import { expect } from "vitest"; @@ -31,6 +31,7 @@ const submitAudit = defineTool({ const repositoryRoot = resolve(import.meta.dirname, "../../.."); const docsRoot = resolve(repositoryRoot, "packages/coding-agent/docs"); +const sourceRoot = resolve(repositoryRoot, "packages/coding-agent/src"); const pages = globSync("**/*.md", { cwd: docsRoot }) .map((path) => ({ path: path.replaceAll("\\", "/") })) .sort((left, right) => left.path.localeCompare(right.path)); @@ -56,7 +57,20 @@ For a mismatch, quote the claim, cite the implementation path and symbol, and st Call ${TOOL_NAME} exactly once as your final action. Do not return prose.`); - const auditCalls = toolCalls(result).filter((call) => call.name === TOOL_NAME); + const calls = toolCalls(result); + const readPaths = calls.flatMap((call) => { + const path = call.name === "read" && call.status === "ok" ? call.arguments?.path : undefined; + return typeof path === "string" ? [resolve(path)] : []; + }); + expect(readPaths).toContain(documentationPath); + expect( + readPaths.some((readPath) => { + const sourceRelative = relative(sourceRoot, readPath); + return sourceRelative !== "" && !sourceRelative.startsWith("..") && !isAbsolute(sourceRelative); + }), + ).toBe(true); + + const auditCalls = calls.filter((call) => call.name === TOOL_NAME); expect(auditCalls).toHaveLength(1); const auditCall = auditCalls[0]; expect(auditCall?.status).toBe("ok"); diff --git a/packages/evals/evals/extensions.docs.eval.ts b/packages/evals/evals/extensions.docs.eval.ts index ef21c8f60..5fd51666d 100644 --- a/packages/evals/evals/extensions.docs.eval.ts +++ b/packages/evals/evals/extensions.docs.eval.ts @@ -16,7 +16,6 @@ const harness = createPiDocumentationEvalHarness({ return { response, extensionErrors: extensions.errors, - toolLoaded: extensions.extensions.some(({ tools }) => tools.has(TOOL_NAME)), toolResult: result?.role === "toolResult" ? contentText(result.content) : null, }; }, @@ -31,7 +30,6 @@ describeEval( expected: { response: TOOL_RESULT, extensionErrors: [], - toolLoaded: true, toolResult: TOOL_RESULT, }, match: "strict", diff --git a/packages/evals/evals/prompt-templates.docs.eval.ts b/packages/evals/evals/prompt-templates.docs.eval.ts new file mode 100644 index 000000000..08edd2238 --- /dev/null +++ b/packages/evals/evals/prompt-templates.docs.eval.ts @@ -0,0 +1,79 @@ +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { contentText } from "@earendil-works/pi-ai"; +import { describeEval, StructuredOutputJudge } from "vitest-evals"; +import { createPiDocumentationEvalHarness } from "../src/harness.ts"; + +const TEMPLATE_NAME = "current-time"; +const COMMAND = `/${TEMPLATE_NAME}`; +const TEMPLATE_PROMPT = "Tell me the current time and timezone."; + +const SCENARIOS = [ + { + title: "Create a project prompt template", + request: `This project is already trusted. Add a ${COMMAND} prompt template for this project containing exactly: ${TEMPLATE_PROMPT}`, + scope: "project", + }, + { + title: "Create a user prompt template", + request: `Make a ${COMMAND} prompt template available in all my Pi sessions containing exactly: ${TEMPLATE_PROMPT}`, + scope: "user", + }, +] as const; + +function registerScenario(scenario: (typeof SCENARIOS)[number]): void { + const harness = createPiDocumentationEvalHarness({ + output: ({ session, agentDir }) => { + const workspace = session.sessionManager.getCwd(); + const projectPath = join(workspace, ".pi", "prompts", `${TEMPLATE_NAME}.md`); + const userPath = join(agentDir, "prompts", `${TEMPLATE_NAME}.md`); + const expectedPath = scenario.scope === "project" ? projectPath : userPath; + const otherPath = scenario.scope === "project" ? userPath : projectPath; + const { prompts, diagnostics } = session.resourceLoader.getPrompts(); + const extensions = session.resourceLoader.getExtensions(); + const matchingTemplates = prompts.filter(({ name }) => name === TEMPLATE_NAME); + const template = matchingTemplates.length === 1 ? matchingTemplates[0] : undefined; + const expandedPrompt = [...session.messages] + .reverse() + .find((message) => message.role === "user"); + const expandedText = expandedPrompt?.role === "user" ? contentText(expandedPrompt.content).trim() : ""; + + return { + template: template + ? { scope: template.sourceInfo.scope, atExpectedPath: template.filePath === expectedPath } + : null, + promptDiagnostics: diagnostics.map(({ message }) => message), + extensions: extensions.extensions.filter(({ hidden }) => !hidden).map(({ path }) => path), + extensionErrors: extensions.errors.map(({ error }) => error), + skills: session.resourceLoader.getSkills().skills.map(({ name }) => name), + otherScopeTemplateExists: existsSync(otherPath), + expandedToTimeRequest: expandedText === TEMPLATE_PROMPT, + }; + }, + }); + const judge = StructuredOutputJudge({ + expected: { + template: { scope: scenario.scope, atExpectedPath: true }, + promptDiagnostics: [], + extensions: [], + extensionErrors: [], + skills: [], + otherScopeTemplateExists: false, + expandedToTimeRequest: true, + }, + match: "strict", + allowExtras: false, + }); + + describeEval(scenario.title, { harness, judges: [judge], judgeThreshold: null }, (it) => { + it(`creates, reloads, and invokes ${COMMAND}`, async ({ run }) => { + await run([ + { type: "prompt", content: scenario.request }, + { type: "reload" }, + { type: "prompt", content: COMMAND }, + ]); + }); + }); +} + +for (const scenario of SCENARIOS) registerScenario(scenario); diff --git a/packages/evals/package.json b/packages/evals/package.json index b36546214..e7cc7bde7 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -7,6 +7,7 @@ "clean": "shx rm -rf .eval", "eval": "npm run eval:host && npm run eval:docs --", "eval:host": "vitest run --config vitest.evals.config.ts --project host", + "eval:audit": "vitest run --config vitest.evals.config.ts --project audit", "eval:docs": "node --experimental-strip-types src/cli.ts", "test": "vitest run --config vitest.test.config.ts" }, diff --git a/packages/evals/src/report.ts b/packages/evals/src/report.ts index 7c5e1d093..cd7192d39 100644 --- a/packages/evals/src/report.ts +++ b/packages/evals/src/report.ts @@ -98,7 +98,7 @@ function validateScore(value: unknown): number | undefined { return value; } -export function classifyCaseStatus(status: ReportCase["status"]): "errored" | "skipped" | "pending" | undefined { +function classifyCaseStatus(status: ReportCase["status"]): "errored" | "skipped" | "pending" | undefined { if (status === "failed") return "errored"; if (status === "skipped" || status === "todo" || status === "disabled") return "skipped"; if (status === "pending") return "pending"; @@ -145,15 +145,16 @@ export async function readTaskObservation( if (!loaded) return { ...identity, outcome: "errored" }; const [{ workspace }, rawReport] = loaded; const reportedFullName = `${task.evalSet} ${task.caseId}`; - const assertions = rawReport.testResults.flatMap(({ assertionResults }) => assertionResults); + const assertions = rawReport.testResults + .flatMap(({ assertionResults }) => assertionResults) + .filter(({ fullName }) => fullName === reportedFullName); if (assertions.length !== 1) return { ...identity, outcome: "errored" }; const assertion = assertions[0]; - if (assertion.fullName !== reportedFullName) return { ...identity, outcome: "errored" }; const statusOutcome = classifyCaseStatus(assertion.status); if (statusOutcome === "skipped" || statusOutcome === "pending") return { ...identity, outcome: statusOutcome }; - if (workspace.cases.length !== 1) return { ...identity, outcome: "errored" }; - const caseResult = workspace.cases[0]; - if (caseResult.fullName !== reportedFullName) return { ...identity, outcome: "errored" }; + const caseResults = workspace.cases.filter(({ fullName }) => fullName === reportedFullName); + if (caseResults.length !== 1) return { ...identity, outcome: "errored" }; + const caseResult = caseResults[0]; if (caseResult.status !== assertion.status) return { ...identity, outcome: "errored" }; const run = caseResult.harness?.run; if (!run) return { ...identity, outcome: "errored" }; diff --git a/packages/evals/test/comparison.test.ts b/packages/evals/test/comparison.test.ts index ba2ad550d..4a00bac8e 100644 --- a/packages/evals/test/comparison.test.ts +++ b/packages/evals/test/comparison.test.ts @@ -78,6 +78,8 @@ describe("summarizeEvalObservations", () => { lift: 0.5, totalTokens: { eligiblePairs: 2, controlMean: 150, treatmentMean: 150, meanDelta: 0 }, toolCalls: { eligiblePairs: 2, controlMean: 2.5, treatmentMean: 2, meanDelta: -0.5 }, + totalMs: { eligiblePairs: 2, controlMean: 1000, treatmentMean: 900, meanDelta: -100 }, + estimatedCostUsd: { eligiblePairs: 2, controlMean: 0.01, treatmentMean: 0.01, meanDelta: 0 }, }), ]); }); diff --git a/packages/evals/test/configured-runtime.test.ts b/packages/evals/test/configured-runtime.test.ts index 7b74c94d3..267151486 100644 --- a/packages/evals/test/configured-runtime.test.ts +++ b/packages/evals/test/configured-runtime.test.ts @@ -1,6 +1,7 @@ import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { ModelRuntime } from "@earendil-works/pi-coding-agent"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { createAcmeServer, @@ -9,7 +10,7 @@ import { OPENAI_PROBE_RESPONSE, OPENAI_PROVIDER_ID, } from "../evals/acme-server.ts"; -import { inspectAddedModel, inspectProvider, loadConfiguredModelRuntime } from "../evals/configured-runtime.ts"; +import { inspectAddedModel, inspectProvider } from "../evals/configured-runtime.ts"; const PROVIDER_ID = OPENAI_PROVIDER_ID; const MODEL = { @@ -43,6 +44,15 @@ async function runtimeWith(modelsJson: unknown) { return loadConfiguredModelRuntime(await agentDirWith(modelsJson)); } +function loadConfiguredModelRuntime(agentDir: string): Promise { + return ModelRuntime.create({ + modelsPath: join(agentDir, "models.json"), + authPath: join(agentDir, "auth.json"), + modelsStorePath: join(agentDir, "models-store.json"), + allowModelNetwork: false, + }); +} + function acmeModelsJson() { return { providers: { diff --git a/packages/evals/test/report.test.ts b/packages/evals/test/report.test.ts index f872ea4f3..594f2b96a 100644 --- a/packages/evals/test/report.test.ts +++ b/packages/evals/test/report.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import type { EvalTask } from "../src/plan.ts"; -import { classifyCaseStatus, PI_SESSION_SNAPSHOT_ARTIFACT, readTaskObservation } from "../src/report.ts"; +import { PI_SESSION_SNAPSHOT_ARTIFACT, readTaskObservation } from "../src/report.ts"; const temporaryDirectories: string[] = []; afterEach(async () => { @@ -22,40 +22,41 @@ const task: EvalTask = { const SESSION = '{"type":"session"}\n'; type ReportAssertion = { - status?: "passed" | "skipped" | "pending" | "failed"; + caseId?: string; + status?: "passed" | "skipped" | "pending" | "todo" | "disabled" | "failed"; meta?: Record; }; -async function writeTaskReport(assertion: ReportAssertion = {}) { +async function writeTaskReport(assertions: ReportAssertion[]) { const directory = await mkdtemp(join(tmpdir(), "pi-eval-report-test-")); temporaryDirectories.push(directory); const reportPath = join(directory, "vitest.json"); - const status = assertion.status ?? "passed"; + const results = assertions.map(({ caseId = task.caseId, status = "passed", meta = {} }) => ({ + ancestorTitles: [task.evalSet], + fullName: `${task.evalSet} ${caseId}`, + status, + title: caseId, + failureMessages: [], + meta, + })); + const count = (status: ReportAssertion["status"]): number => + results.filter((result) => result.status === status).length; await writeFile( reportPath, JSON.stringify({ - numFailedTests: 0, - numPassedTests: status === "passed" ? 1 : 0, - numPendingTests: status === "pending" || status === "skipped" ? 1 : 0, - numTodoTests: 0, - numTotalTests: 1, + numFailedTests: count("failed"), + numPassedTests: count("passed"), + numPendingTests: count("pending") + count("skipped") + count("disabled"), + numTodoTests: count("todo"), + numTotalTests: results.length, startTime: 0, - success: true, + success: count("failed") === 0, testResults: [ { message: "", name: "/repo/packages/evals/evals/example.docs.eval.ts", - status: "passed", - assertionResults: [ - { - ancestorTitles: [task.evalSet], - fullName: `${task.evalSet} ${task.caseId}`, - status, - title: task.caseId, - failureMessages: [], - meta: assertion.meta ?? {}, - }, - ], + status: count("failed") === 0 ? "passed" : "failed", + assertionResults: results, }, ], }), @@ -97,26 +98,19 @@ function scoredMeta(overrides?: { }; } -async function readObservation(assertion?: ReportAssertion) { - const { directory, reportPath } = await writeTaskReport(assertion); +async function readObservation(assertion: ReportAssertion = {}) { + const { directory, reportPath } = await writeTaskReport([assertion]); return { directory, observation: await readTaskObservation(task, reportPath, directory) }; } -describe("classifyCaseStatus", () => { - it.each(["skipped", "todo", "disabled"] as const)("maps %s to skipped", (status) => { - expect(classifyCaseStatus(status)).toBe("skipped"); - }); - - it("maps failed infrastructure to errored", () => { - expect(classifyCaseStatus("failed")).toBe("errored"); - }); -}); - describe("readTaskObservation", () => { - it("preserves a skipped outcome when no harness run exists", async () => { - const { observation } = await readObservation({ status: "skipped" }); - expect(observation).toMatchObject({ outcome: "skipped" }); - }); + it.each(["skipped", "todo", "disabled"] as const)( + "preserves a skipped outcome for %s cases without harness runs", + async (status) => { + const { observation } = await readObservation({ status }); + expect(observation).toMatchObject({ outcome: "skipped" }); + }, + ); it("preserves a pending outcome when no harness run exists", async () => { const { observation } = await readObservation({ status: "pending" }); @@ -126,7 +120,7 @@ describe("readTaskObservation", () => { it("preserves metrics from a failed eval with a partial harness run", async () => { const { directory, observation } = await readObservation({ status: "failed", - meta: scoredMeta({ errors: [{ message: "Prompt verification failed" }] }), + meta: scoredMeta(), }); expect(observation).toEqual({ evalSet: task.evalSet, @@ -181,6 +175,15 @@ describe("readTaskObservation", () => { ).resolves.toBe(SESSION); }); + it("selects the requested case when a sibling case is skipped", async () => { + const { directory, reportPath } = await writeTaskReport([ + { caseId: "sibling case", status: "skipped" }, + { caseId: task.caseId, meta: scoredMeta() }, + ]); + const observation = await readTaskObservation(task, reportPath, directory); + expect(observation).toMatchObject({ outcome: "scored", score: 0.5, totalTokens: 15 }); + }); + it("treats a zero score as scored data", async () => { const { observation } = await readObservation({ meta: scoredMeta({ avgScore: 0 }) }); expect(observation).toMatchObject({ outcome: "scored", score: 0 }); diff --git a/packages/evals/vitest.evals.config.ts b/packages/evals/vitest.evals.config.ts index ced768eaf..20ecd73e7 100644 --- a/packages/evals/vitest.evals.config.ts +++ b/packages/evals/vitest.evals.config.ts @@ -24,7 +24,17 @@ const evalConfig = defineConfig({ test: { name: "host", include: ["evals/**/*.eval.ts"], - exclude: ["evals/**/*.docs.eval.ts"], + exclude: ["evals/**/*.docs.eval.ts", "evals/documentation-audit.eval.ts"], + sequence: { concurrent: false }, + testTimeout: 300_000, + hookTimeout: 300_000, + }, + }, + { + extends: true, + test: { + name: "audit", + include: ["evals/documentation-audit.eval.ts"], sequence: { concurrent: false }, testTimeout: 300_000, hookTimeout: 300_000, From b4b72a351048668d13ac3b42cc568589612460cb Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:25:04 +0100 Subject: [PATCH 057/271] feat(coding-agent): extensions changelog (#8790) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses #5958. Adds `changelogPath` to `PiManifest`. Safe package-relative changelog resolution, with `CHANGELOG.md` fallback, and carries it through `PathMetadata`/`SourceInfo`. Added extension changelog handling like Pi’s own changelog. If the package exposes `pi.changelogPath` or has `CHANGELOG.md`, Pi stores the last seen package version in settings. On each fresh interactive launch, we compare the stored version to the installed package version. Then parses the changelog entries in between, shows an Extension Updates section, then marks the current version as seen. The first time an extension is seen, its version is recorded without showing a changelog, same as Pi's own on a fresh install. `collapseChangelog` condenses this to one line per package. Any update that bumps the extension's version shows this on the next launch, e.g.: ```bash pi update --extensions pi update --all pi update pi update --extension pi update --self --extensions pi update pi --extensions pi update self --extensions ``` Screenshot 2026-08-28 at 16 34 15 --- packages/coding-agent/CHANGELOG.md | 1 + packages/coding-agent/docs/packages.md | 20 +++ packages/coding-agent/docs/settings.md | 2 +- .../coding-agent/schemas/settings.schema.json | 9 ++ .../coding-agent/src/core/package-manager.ts | 51 ++++++- packages/coding-agent/src/core/pi-manifest.ts | 7 + .../coding-agent/src/core/settings-manager.ts | 13 ++ .../coding-agent/src/core/settings-schema.ts | 5 + packages/coding-agent/src/core/source-info.ts | 16 ++ .../src/modes/interactive/interactive-mode.ts | 122 +++++++++++++-- ...teractive-mode-extension-changelog.test.ts | 139 ++++++++++++++++++ .../coding-agent/test/package-manager.test.ts | 86 +++++++++++ 12 files changed, 449 insertions(+), 22 deletions(-) create mode 100644 packages/coding-agent/test/interactive-mode-extension-changelog.test.ts diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 2ec3d9bd4..ae86551c1 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added +- Extension packages can now show their changelog after an update. Pi reads `pi.changelogPath` from the package's `package.json`, or `CHANGELOG.md` in the package root, and on the next fresh launch shows entries newer than the last seen version under "Extension Updates". The `collapseChangelog` setting condenses this to one line per package ([#5958](https://github.com/earendil-works/pi/issues/5958), [#8790](https://github.com/earendil-works/pi/pull/8790)) - Device-code login (for example GitHub Copilot) now opens the verification page and copies the code when you press Enter. Providers can turn either action off with `openBrowserOnConfirm: false` or `copyCodeOnConfirm: false` ([#9282](https://github.com/earendil-works/pi/issues/9282), [#9301](https://github.com/earendil-works/pi/pull/9301)) ### Changed diff --git a/packages/coding-agent/docs/packages.md b/packages/coding-agent/docs/packages.md index a4f609db8..6cf306318 100644 --- a/packages/coding-agent/docs/packages.md +++ b/packages/coding-agent/docs/packages.md @@ -75,6 +75,26 @@ Paths are relative to the package root. Arrays accept glob patterns and exclusio The `pi-package` keyword makes an npm package eligible for discovery in the [Pi package gallery](https://pi.dev/packages). Optional `pi.image` and `pi.video` fields add gallery previews. +## Show a changelog + +After a package update, Pi can show what changed. On the next fresh launch, Pi reads the package changelog and shows entries newer than the last seen version under "Extension Updates", the same way it shows its own changelog. The first time Pi sees a package, it records the version without showing anything. + +This applies to packages that load at least one extension. Packages that only provide skills, prompt templates, or themes do not show a changelog. + +By default Pi reads `CHANGELOG.md` from the package root. Point `pi.changelogPath` at another file inside the package, or set it to `""` to disable the changelog: + +```json +{ + "name": "my-pi-package", + "version": "1.2.0", + "pi": { + "changelogPath": "./docs/CHANGELOG.md" + } +} +``` + +Use version headings in the same format as Pi's changelog, such as `## [1.2.0]` or `## 1.2.0 - 2026-01-01`. Entries are matched against the `version` field in `package.json`, so a git package only shows a changelog when that field increases. The `collapseChangelog` setting condenses the section to one line per package, such as `Updated my-pi-package to v1.2.0.` Unlike Pi's own changelog, `/changelog` does not show package changelogs, so read the package's changelog file or repository for details. + ## Declare dependencies Put runtime packages imported by extensions in `dependencies`. Pi installs package dependencies when it installs an npm or git source. diff --git a/packages/coding-agent/docs/settings.md b/packages/coding-agent/docs/settings.md index 150eaf4d7..538a61392 100644 --- a/packages/coding-agent/docs/settings.md +++ b/packages/coding-agent/docs/settings.md @@ -166,7 +166,7 @@ The built-in extensions are named `builtin:mcp`, `builtin:llama.cpp`, `builtin:c | Setting | Type | Default | Description | |---|---|---|---| -| `collapseChangelog` | boolean | `false` | Show a condensed changelog after an update. | +| `collapseChangelog` | boolean | `false` | Show a condensed changelog after an update. Also condenses extension package changelogs to one line per package. | | `enableInstallTelemetry` | boolean | `true` | Enable anonymous install/update reporting and selected provider attribution headers. Does not control update checks. | | `enableAnalytics` | boolean | `false` | Opt in to analytics data sharing. Currently used only by the experimental first-run setup. | | `warnings.anthropicExtraUsage` | boolean | `true` | Warn when Anthropic subscription authentication may use paid extra usage. | diff --git a/packages/coding-agent/schemas/settings.schema.json b/packages/coding-agent/schemas/settings.schema.json index 4d6c23a6d..90eaaa3c0 100644 --- a/packages/coding-agent/schemas/settings.schema.json +++ b/packages/coding-agent/schemas/settings.schema.json @@ -11,6 +11,15 @@ "lastChangelogVersion": { "type": "string" }, + "extensionChangelogVersions": { + "type": "object", + "patternProperties": { + "^.*$": { + "type": "string" + } + }, + "description": "Last seen changelog version per extension package." + }, "defaultProvider": { "type": "string" }, diff --git a/packages/coding-agent/src/core/package-manager.ts b/packages/coding-agent/src/core/package-manager.ts index 9b7a734d4..3bed70557 100644 --- a/packages/coding-agent/src/core/package-manager.ts +++ b/packages/coding-agent/src/core/package-manager.ts @@ -33,7 +33,7 @@ function getEnv(): NodeJS.ProcessEnv { } } -import { basename, dirname, join, relative, resolve, sep } from "node:path"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import type { Readable } from "node:stream"; import ignore from "ignore"; import { minimatch } from "minimatch"; @@ -77,6 +77,7 @@ export interface PathMetadata { scope: SourceScope; origin: "package" | "top-level"; baseDir?: string; + changelogPath?: string; packageRoot?: string; } @@ -674,6 +675,37 @@ function collectResourceFiles(dir: string, resourceType: ResourceType): string[] return collectFiles(dir, FILE_PATTERNS[resourceType]); } +// A `pi` object counts as an explicit resource manifest only when it lists resources. Packages whose +// `pi` object holds only metadata (changelogPath, gallery image/video) still use directory auto-discovery. +function hasManifestResourceFields(manifest: PiManifest): boolean { + return RESOURCE_TYPES.some((resourceType) => manifest[resourceType] !== undefined); +} + +function isPathInside(parent: string, child: string): boolean { + const relativePath = relative(parent, child); + return ( + relativePath !== "" && relativePath !== ".." && !relativePath.startsWith(`..${sep}`) && !isAbsolute(relativePath) + ); +} + +function resolvePackageChangelogPath(packageRoot: string, manifest: PiManifest | null): string | undefined { + const changelogPath = manifest?.changelogPath ?? "CHANGELOG.md"; + if (!changelogPath) return undefined; + + const resolvedPath = resolve(packageRoot, changelogPath); + if (!existsSync(resolvedPath)) return undefined; + + const canonicalPackageRoot = canonicalizePath(packageRoot); + const canonicalChangelogPath = canonicalizePath(resolvedPath); + if (!isPathInside(canonicalPackageRoot, canonicalChangelogPath)) return undefined; + + try { + return statSync(resolvedPath).isFile() ? resolvedPath : undefined; + } catch { + return undefined; + } +} + function matchesAnyPattern(filePath: string, patterns: string[], baseDir: string): boolean { const rel = toPosixPath(relative(baseDir, filePath)); const name = basename(filePath); @@ -2249,6 +2281,14 @@ export class DefaultPackageManager implements PackageManager { filter: PackageFilter | undefined, metadata: PathMetadata, ): boolean { + const manifest = readPiManifest(join(packageRoot, "package.json")); + const changelogPath = resolvePackageChangelogPath(packageRoot, manifest); + if (changelogPath) { + metadata.changelogPath = changelogPath; + } else { + delete metadata.changelogPath; + } + if (filter) { for (const resourceType of RESOURCE_TYPES) { const patterns = filter[resourceType]; @@ -2264,10 +2304,9 @@ export class DefaultPackageManager implements PackageManager { return true; } - const manifest = readPiManifest(join(packageRoot, "package.json")); - if (manifest) { + if (manifest && hasManifestResourceFields(manifest)) { for (const resourceType of RESOURCE_TYPES) { - const entries = manifest[resourceType as keyof PiManifest]; + const entries = manifest[resourceType]; this.addManifestEntries( entries, packageRoot, @@ -2301,7 +2340,7 @@ export class DefaultPackageManager implements PackageManager { metadata: PathMetadata, ): void { const manifest = readPiManifest(join(packageRoot, "package.json")); - const entries = manifest?.[resourceType as keyof PiManifest]; + const entries = manifest?.[resourceType]; if (entries) { this.addManifestEntries(entries, packageRoot, resourceType, target, metadata); return; @@ -2370,7 +2409,7 @@ export class DefaultPackageManager implements PackageManager { resourceType: ResourceType, ): { allFiles: string[]; enabledByManifest: Set } { const manifest = readPiManifest(join(packageRoot, "package.json")); - const entries = manifest?.[resourceType as keyof PiManifest]; + const entries = manifest?.[resourceType]; if (entries && entries.length > 0) { const allFiles = this.collectFilesFromManifestEntries(entries, packageRoot, resourceType); const manifestPatterns = entries.filter(isOverridePattern); diff --git a/packages/coding-agent/src/core/pi-manifest.ts b/packages/coding-agent/src/core/pi-manifest.ts index fd7dd5edb..0dca39385 100644 --- a/packages/coding-agent/src/core/pi-manifest.ts +++ b/packages/coding-agent/src/core/pi-manifest.ts @@ -6,6 +6,7 @@ export interface PiManifest { skills?: string[]; prompts?: string[]; themes?: string[]; + changelogPath?: string; } const RESOURCE_FIELDS = ["extensions", "skills", "prompts", "themes"] as const; @@ -28,6 +29,12 @@ export function readPiManifest(packageJsonPath: string): PiManifest | null { manifest[field] = entries; } } + + const changelogPath = pkg.pi.changelogPath; + if (typeof changelogPath === "string") { + manifest.changelogPath = changelogPath; + } + return manifest; } catch { return null; diff --git a/packages/coding-agent/src/core/settings-manager.ts b/packages/coding-agent/src/core/settings-manager.ts index ad8b82c9e..99bbeade0 100644 --- a/packages/coding-agent/src/core/settings-manager.ts +++ b/packages/coding-agent/src/core/settings-manager.ts @@ -687,6 +687,19 @@ export class SettingsManager { this.save(); } + getExtensionChangelogVersion(key: string): string | undefined { + return this.settings.extensionChangelogVersions?.[key]; + } + + setExtensionChangelogVersion(key: string, version: string): void { + this.globalSettings.extensionChangelogVersions = { + ...(this.globalSettings.extensionChangelogVersions ?? {}), + [key]: version, + }; + this.markModified("extensionChangelogVersions", key); + this.save(); + } + getSessionDir(): string | undefined { const sessionDir = this.settings.sessionDir; return sessionDir ? normalizePath(sessionDir) : sessionDir; diff --git a/packages/coding-agent/src/core/settings-schema.ts b/packages/coding-agent/src/core/settings-schema.ts index b17bab563..87a3b71a5 100644 --- a/packages/coding-agent/src/core/settings-schema.ts +++ b/packages/coding-agent/src/core/settings-schema.ts @@ -207,6 +207,11 @@ export const SettingsSchema = Type.Object( { $schema: Type.Optional(Type.String({ description: "JSON Schema reference." })), lastChangelogVersion: Type.Optional(Type.String()), + extensionChangelogVersions: Type.Optional( + Type.Record(Type.String(), Type.String(), { + description: "Last seen changelog version per extension package.", + }), + ), defaultProvider: Type.Optional(Type.String()), defaultModel: Type.Optional(Type.String()), defaultThinkingLevel: Type.Optional(ModelThinkingLevelSchema), diff --git a/packages/coding-agent/src/core/source-info.ts b/packages/coding-agent/src/core/source-info.ts index 765395d64..e73a0471d 100644 --- a/packages/coding-agent/src/core/source-info.ts +++ b/packages/coding-agent/src/core/source-info.ts @@ -9,6 +9,19 @@ export interface SourceInfo { scope: SourceScope; origin: SourceOrigin; baseDir?: string; + changelogPath?: string; +} + +export interface SourcePackageIdentity { + source: string; + scope: SourceScope; +} + +export function getSourcePackageKey(identity: SourcePackageIdentity): string { + return JSON.stringify({ + source: identity.source, + scope: identity.scope, + }); } /** Prefix of built-in tool and extension paths, such as `builtin:read` or `builtin:mcp`. */ @@ -35,6 +48,7 @@ export function createSourceInfo(path: string, metadata: PathMetadata): SourceIn scope: metadata.scope, origin: metadata.origin, baseDir: metadata.baseDir, + ...(metadata.changelogPath ? { changelogPath: metadata.changelogPath } : {}), }; } @@ -45,6 +59,7 @@ export function createSyntheticSourceInfo( scope?: SourceScope; origin?: SourceOrigin; baseDir?: string; + changelogPath?: string; }, ): SourceInfo { return { @@ -53,5 +68,6 @@ export function createSyntheticSourceInfo( scope: options.scope ?? "temporary", origin: options.origin ?? "top-level", baseDir: options.baseDir, + ...(options.changelogPath ? { changelogPath: options.changelogPath } : {}), }; } diff --git a/packages/coding-agent/src/modes/interactive/interactive-mode.ts b/packages/coding-agent/src/modes/interactive/interactive-mode.ts index 57bd91f25..9707bf5dc 100644 --- a/packages/coding-agent/src/modes/interactive/interactive-mode.ts +++ b/packages/coding-agent/src/modes/interactive/interactive-mode.ts @@ -115,7 +115,7 @@ import { } from "../../core/session-manager.ts"; import type { FullscreenExitOutput, TuiMode } from "../../core/settings-manager.ts"; import { BUILTIN_SLASH_COMMANDS } from "../../core/slash-commands.ts"; -import type { SourceInfo } from "../../core/source-info.ts"; +import { getSourcePackageKey, type SourceInfo } from "../../core/source-info.ts"; import { isInstallTelemetryEnabled } from "../../core/telemetry.ts"; import { withBuiltInRenderers } from "../../core/tools/renderers/index.ts"; import type { TruncationResult } from "../../core/tools/truncate.ts"; @@ -131,6 +131,7 @@ import { getCwdRelativePath } from "../../utils/paths.ts"; import { getPiUserAgent } from "../../utils/pi-user-agent.ts"; import { killTrackedDetachedChildren } from "../../utils/shell.ts"; import { loadAllHighlightLanguages } from "../../utils/syntax-highlight.ts"; +import { stripBom } from "../../utils/text.ts"; import { ensureTool, type ToolStatus } from "../../utils/tools-manager.ts"; import { checkForNewPiVersion, type LatestPiRelease } from "../../utils/version-check.ts"; import { reportBug } from "./bug-report.ts"; @@ -450,6 +451,12 @@ export interface InteractiveModeOptions { terminal?: Terminal; } +interface ExtensionChangelogUpdate { + name: string; + version: string; + markdown: string; +} + export class InteractiveMode { private runtimeHost: AgentSessionRuntime; private renderer: TuiMainScreen | TuiAltScreen; @@ -493,6 +500,7 @@ export class InteractiveMode { private lastSigintTime = 0; private lastEscapeTime = 0; private changelogMarkdown: string | undefined = undefined; + private extensionChangelogUpdates: ExtensionChangelogUpdate[] = []; private startupNoticesShown = false; private anthropicSubscriptionWarningShown = false; @@ -848,7 +856,7 @@ export class InteractiveMode { } this.startupNoticesShown = true; - if (!this.changelogMarkdown) { + if (!this.changelogMarkdown && this.extensionChangelogUpdates.length === 0) { return; } @@ -856,18 +864,39 @@ export class InteractiveMode { this.chatContainer.addChild(new Spacer(1)); } this.chatContainer.addChild(new DynamicBorder()); - if (this.settingsManager.getCollapseChangelog()) { - const versionMatch = this.changelogMarkdown.match(/##\s+\[?(\d+\.\d+\.\d+)\]?/); - const latestVersion = versionMatch ? versionMatch[1] : this.version; - const condensedText = `Updated to v${latestVersion}. Use ${theme.bold("/changelog")} to view full changelog.`; - this.chatContainer.addChild(new Text(condensedText, 1, 0)); - } else { - this.chatContainer.addChild(new ThemedText(() => theme.bold(theme.fg("accent", "What's New")), 1, 0)); - this.chatContainer.addChild(new Spacer(1)); - this.chatContainer.addChild( - new Markdown(this.changelogMarkdown.trim(), 1, 0, this.getMarkdownThemeWithSettings()), - ); - this.chatContainer.addChild(new Spacer(1)); + if (this.changelogMarkdown) { + if (this.settingsManager.getCollapseChangelog()) { + const versionMatch = this.changelogMarkdown.match(/##\s+\[?(\d+\.\d+\.\d+)\]?/); + const latestVersion = versionMatch ? versionMatch[1] : this.version; + const condensedText = `Updated to v${latestVersion}. Use ${theme.bold("/changelog")} to view full changelog.`; + this.chatContainer.addChild(new Text(condensedText, 1, 0)); + } else { + this.chatContainer.addChild(new ThemedText(() => theme.bold(theme.fg("accent", "What's New")), 1, 0)); + this.chatContainer.addChild(new Spacer(1)); + this.chatContainer.addChild( + new Markdown(this.changelogMarkdown.trim(), 1, 0, this.getMarkdownThemeWithSettings()), + ); + this.chatContainer.addChild(new Spacer(1)); + } + } + if (this.extensionChangelogUpdates.length > 0) { + if (this.changelogMarkdown) { + this.chatContainer.addChild(new Spacer(1)); + } + if (this.settingsManager.getCollapseChangelog()) { + const condensedText = this.extensionChangelogUpdates + .map((update) => `Updated ${update.name} to v${update.version}.`) + .join("\n"); + this.chatContainer.addChild(new Text(condensedText, 1, 0)); + } else { + this.chatContainer.addChild( + new ThemedText(() => theme.bold(theme.fg("accent", "Extension Updates")), 1, 0), + ); + this.chatContainer.addChild(new Spacer(1)); + const markdown = this.extensionChangelogUpdates.map((update) => update.markdown).join("\n\n"); + this.chatContainer.addChild(new Markdown(markdown, 1, 0, this.getMarkdownThemeWithSettings())); + this.chatContainer.addChild(new Spacer(1)); + } } this.chatContainer.addChild(new DynamicBorder()); } @@ -947,8 +976,9 @@ export class InteractiveMode { this.registerSignalHandlers(); - // Load changelog (only show new entries, skip for resumed sessions) + // Load changelogs (only show new entries, skip for resumed sessions) this.changelogMarkdown = this.getChangelogForDisplay(); + this.extensionChangelogUpdates = this.getExtensionChangelogUpdates(); // Fullscreen renders on the alternate screen, so text printed to stdout here would stay in the shell // after exit. Fullscreen shows the notice in the header instead (#11508). @@ -1351,6 +1381,68 @@ export class InteractiveMode { return undefined; } + private getExtensionChangelogUpdates(): ExtensionChangelogUpdate[] { + // Skip changelog for resumed/continued sessions (already have messages) + if (this.session.state.messages.length > 0) { + return []; + } + + const seenKeys = new Set(); + const updates: ExtensionChangelogUpdate[] = []; + for (const { sourceInfo } of this.session.resourceLoader.getExtensions().extensions) { + if (sourceInfo.origin !== "package" || !sourceInfo.baseDir || !sourceInfo.changelogPath) { + continue; + } + + const key = getSourcePackageKey(sourceInfo); + if (seenKeys.has(key)) { + continue; + } + seenKeys.add(key); + + const pkg = this.readPackageJson(sourceInfo.baseDir); + if (!pkg?.version) { + continue; + } + + const lastVersion = this.settingsManager.getExtensionChangelogVersion(key); + if (!lastVersion) { + // Fresh install - record the version, don't show changelog + this.settingsManager.setExtensionChangelogVersion(key, pkg.version); + continue; + } + + const newEntries = getNewEntries(parseChangelog(sourceInfo.changelogPath), lastVersion); + if (newEntries.length > 0) { + this.settingsManager.setExtensionChangelogVersion(key, pkg.version); + const name = pkg.name || sourceInfo.source; + // Prefix each "## [x.y.z]" heading with the package name so entries render like Pi's own. + const markdown = newEntries + .map((entry) => entry.content.replace(/^##\s+/, `## ${name} `)) + .join("\n\n") + .trim(); + updates.push({ name, version: pkg.version, markdown }); + } + } + + return updates; + } + + private readPackageJson(packageRoot: string): { name?: string; version?: string } | undefined { + try { + const pkg = JSON.parse(stripBom(fs.readFileSync(path.join(packageRoot, "package.json"), "utf-8"))) as { + name?: unknown; + version?: unknown; + }; + return { + name: typeof pkg.name === "string" ? pkg.name : undefined, + version: typeof pkg.version === "string" ? pkg.version : undefined, + }; + } catch { + return undefined; + } + } + private reportInstallTelemetry(version: string): void { if (process.env.PI_OFFLINE) { return; diff --git a/packages/coding-agent/test/interactive-mode-extension-changelog.test.ts b/packages/coding-agent/test/interactive-mode-extension-changelog.test.ts new file mode 100644 index 000000000..f588ba474 --- /dev/null +++ b/packages/coding-agent/test/interactive-mode-extension-changelog.test.ts @@ -0,0 +1,139 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { InMemorySettingsStorage, SettingsManager } from "../src/core/settings-manager.ts"; +import { getSourcePackageKey, type SourceInfo } from "../src/core/source-info.ts"; +import { InteractiveMode } from "../src/modes/interactive/interactive-mode.ts"; + +type ExtensionChangelogUpdate = { name: string; version: string; markdown: string }; + +type ExtensionChangelogContext = { + session: { + state: { messages: unknown[] }; + resourceLoader: { getExtensions(): { extensions: Array<{ sourceInfo: SourceInfo }> } }; + }; + settingsManager: SettingsManager; + readPackageJson(packageRoot: string): { name?: string; version?: string } | undefined; +}; + +const prototype = InteractiveMode.prototype as unknown as { + getExtensionChangelogUpdates(this: ExtensionChangelogContext): ExtensionChangelogUpdate[]; + readPackageJson(packageRoot: string): { name?: string; version?: string } | undefined; +}; + +const CHANGELOG = `# Changelog + +## [Unreleased] + +- Not released yet + +## [1.2.0] + +- Added feature B + +## [1.1.0] + +- Added feature A + +## [1.0.0] + +- Initial release +`; + +describe("InteractiveMode extension changelog", () => { + let tempDir: string; + let packageRoot: string; + let sourceInfo: SourceInfo; + let settingsManager: SettingsManager; + let messages: unknown[]; + + function writePackage(version: string): void { + writeFileSync(join(packageRoot, "package.json"), JSON.stringify({ name: "pi-demo-extension", version })); + } + + function createContext(): ExtensionChangelogContext { + return { + session: { + state: { messages }, + resourceLoader: { + getExtensions: () => ({ + // Two extensions from the same package must only produce one changelog section. + extensions: [{ sourceInfo }, { sourceInfo: { ...sourceInfo, path: join(packageRoot, "b.ts") } }], + }), + }, + }, + settingsManager, + readPackageJson: prototype.readPackageJson, + }; + } + + function display(): ExtensionChangelogUpdate[] { + return prototype.getExtensionChangelogUpdates.call(createContext()); + } + + beforeEach(() => { + tempDir = mkdtempSync(join(tmpdir(), "pi-extension-changelog-")); + packageRoot = join(tempDir, "pkg"); + mkdirSync(packageRoot); + writeFileSync(join(packageRoot, "CHANGELOG.md"), CHANGELOG); + sourceInfo = { + path: join(packageRoot, "a.ts"), + source: "npm:pi-demo-extension", + scope: "user", + origin: "package", + baseDir: packageRoot, + changelogPath: join(packageRoot, "CHANGELOG.md"), + }; + settingsManager = SettingsManager.fromStorage(new InMemorySettingsStorage()); + messages = []; + }); + + afterEach(() => { + rmSync(tempDir, { recursive: true, force: true }); + }); + + test("records the version on first launch without showing the changelog", () => { + writePackage("1.0.0"); + + expect(display()).toEqual([]); + expect(settingsManager.getExtensionChangelogVersion(getSourcePackageKey(sourceInfo))).toBe("1.0.0"); + }); + + test("shows entries newer than the last seen version, headed by package name", () => { + writePackage("1.0.0"); + display(); + writePackage("1.2.0"); + + expect(display()).toEqual([ + { + name: "pi-demo-extension", + version: "1.2.0", + markdown: + "## pi-demo-extension [1.2.0]\n\n- Added feature B\n\n## pi-demo-extension [1.1.0]\n\n- Added feature A", + }, + ]); + expect(settingsManager.getExtensionChangelogVersion(getSourcePackageKey(sourceInfo))).toBe("1.2.0"); + expect(display()).toEqual([]); + }); + + test("skips resumed sessions", () => { + writePackage("1.0.0"); + display(); + writePackage("1.2.0"); + messages.push({ role: "user", content: "hi" }); + + expect(display()).toEqual([]); + expect(settingsManager.getExtensionChangelogVersion(getSourcePackageKey(sourceInfo))).toBe("1.0.0"); + }); + + test("ignores non-package extensions and packages without a changelog", () => { + writePackage("1.0.0"); + sourceInfo = { ...sourceInfo, origin: "top-level" }; + expect(display()).toEqual([]); + + sourceInfo = { ...sourceInfo, origin: "package", changelogPath: undefined }; + expect(display()).toEqual([]); + expect(settingsManager.getExtensionChangelogVersion(getSourcePackageKey(sourceInfo))).toBeUndefined(); + }); +}); diff --git a/packages/coding-agent/test/package-manager.test.ts b/packages/coding-agent/test/package-manager.test.ts index 38d93b232..6aaeb7773 100644 --- a/packages/coding-agent/test/package-manager.test.ts +++ b/packages/coding-agent/test/package-manager.test.ts @@ -671,6 +671,92 @@ Content`, expect(result.skills.some((r) => r.path === slashSkillPath && r.enabled)).toBe(true); }); + it("should expose manifest changelogPath on package resources", async () => { + const pkgDir = join(tempDir, "changelog-manifest-package"); + const extensionPath = join(pkgDir, "src", "index.ts"); + const changelogPath = join(pkgDir, "docs", "CHANGELOG.md"); + mkdirSync(join(pkgDir, "src"), { recursive: true }); + mkdirSync(join(pkgDir, "docs"), { recursive: true }); + writeFileSync(extensionPath, "export default function() {}"); + writeFileSync(changelogPath, "## [1.0.0]\n\n- Initial release"); + writeFileSync( + join(pkgDir, "package.json"), + JSON.stringify({ + name: "changelog-manifest-package", + pi: { + extensions: ["src/index.ts"], + changelogPath: "docs/CHANGELOG.md", + }, + }), + ); + + const result = await packageManager.resolveExtensionSources([pkgDir]); + const extension = result.extensions.find((r) => r.path === extensionPath); + + expect(extension?.metadata.changelogPath).toBe(changelogPath); + }); + + it("should expose package CHANGELOG.md by convention", async () => { + const pkgDir = join(tempDir, "changelog-convention-package"); + const extensionPath = join(pkgDir, "extensions", "index.ts"); + const changelogPath = join(pkgDir, "CHANGELOG.md"); + mkdirSync(join(pkgDir, "extensions"), { recursive: true }); + writeFileSync(extensionPath, "export default function() {}"); + writeFileSync(changelogPath, "## [1.0.0]\n\n- Initial release"); + + const result = await packageManager.resolveExtensionSources([pkgDir]); + const extension = result.extensions.find((r) => r.path === extensionPath); + + expect(extension?.metadata.changelogPath).toBe(changelogPath); + }); + + it("should still auto-discover resources when pi manifest only declares changelogPath", async () => { + const pkgDir = join(tempDir, "changelog-only-manifest-package"); + const extensionPath = join(pkgDir, "extensions", "index.ts"); + const changelogPath = join(pkgDir, "CHANGELOG.md"); + mkdirSync(join(pkgDir, "extensions"), { recursive: true }); + writeFileSync(extensionPath, "export default function() {}"); + writeFileSync(changelogPath, "## [1.0.0]\n\n- Initial release"); + writeFileSync( + join(pkgDir, "package.json"), + JSON.stringify({ + name: "changelog-only-manifest-package", + pi: { + changelogPath: "CHANGELOG.md", + }, + }), + ); + + const result = await packageManager.resolveExtensionSources([pkgDir]); + const extension = result.extensions.find((r) => r.path === extensionPath); + + expect(extension?.metadata.changelogPath).toBe(changelogPath); + }); + + it("should ignore manifest changelogPath outside the package root", async () => { + const pkgDir = join(tempDir, "unsafe-changelog-package"); + const extensionPath = join(pkgDir, "src", "index.ts"); + mkdirSync(join(pkgDir, "src"), { recursive: true }); + writeFileSync(extensionPath, "export default function() {}"); + writeFileSync(join(tempDir, "CHANGELOG.md"), "## [1.0.0]\n\n- Outside"); + writeFileSync(join(pkgDir, "CHANGELOG.md"), "## [1.0.0]\n\n- Fallback should not be used"); + writeFileSync( + join(pkgDir, "package.json"), + JSON.stringify({ + name: "unsafe-changelog-package", + pi: { + extensions: ["src/index.ts"], + changelogPath: "../CHANGELOG.md", + }, + }), + ); + + const result = await packageManager.resolveExtensionSources([pkgDir]); + const extension = result.extensions.find((r) => r.path === extensionPath); + + expect(extension?.metadata.changelogPath).toBeUndefined(); + }); + it("should handle directories with auto-discovery layout", async () => { const pkgDir = join(tempDir, "auto-pkg"); mkdirSync(join(pkgDir, "extensions"), { recursive: true }); From 93029a0f1e7949cd75a6bb2989f4fecb7dc5e78c Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 10:30:19 +0000 Subject: [PATCH 058/271] feat(chord): Chord v2 in src/next, experimental and not exported yet The next object runtime moves from the private repository into pi: src/next (contracts, hosts and objects, documents, derives, links and relays, routines and jobs, the store) with its tests and examples in test/next, src/next-node (Node compression), and its design in docs/next (design.md, the decision log decisions.md, the slice plan slices.md, slice specs, books and benchmarks). AGENTS.md gets its code style. Not exported: the API changes with every slice until it has an entry point. --- AGENTS.md | 14 + packages/chord/CHANGELOG.md | 1 + packages/chord/docs/next/benchmarks.md | 492 ++ packages/chord/docs/next/decisions.md | 2168 +++++++ packages/chord/docs/next/design.md | 2199 +++++++ .../chord/docs/next/durable-observers/book.md | 432 ++ .../docs/next/durable-observers/index.html | 517 ++ packages/chord/docs/next/explainer/book.md | 4507 ++++++++++++++ packages/chord/docs/next/explainer/index.html | 5409 +++++++++++++++++ .../chord/docs/next/explainer/src/00-front.md | 111 + .../chord/docs/next/explainer/src/01-shape.md | 246 + .../chord/docs/next/explainer/src/02-first.md | 187 + .../docs/next/explainer/src/03-principals.md | 190 + .../chord/docs/next/explainer/src/04-store.md | 132 + .../docs/next/explainer/src/05-documents.md | 258 + .../docs/next/explainer/src/06-references.md | 185 + .../docs/next/explainer/src/07-revocation.md | 113 + .../docs/next/explainer/src/08-derives.md | 114 + .../docs/next/explainer/src/09-natives.md | 160 + .../docs/next/explainer/src/10-routines.md | 145 + .../chord/docs/next/explainer/src/11-jobs.md | 171 + .../chord/docs/next/explainer/src/12-flows.md | 141 + .../docs/next/explainer/src/13-registries.md | 185 + .../chord/docs/next/explainer/src/14-agent.md | 192 + .../docs/next/explainer/src/15-reload.md | 131 + .../docs/next/explainer/src/16-contracts.md | 310 + .../chord/docs/next/explainer/src/17-wire.md | 251 + .../docs/next/explainer/src/18-p2-core.md | 324 + .../docs/next/explainer/src/19-p2-sessions.md | 276 + .../docs/next/explainer/src/20-p2-machines.md | 245 + .../docs/next/explainer/src/20a-compared.md | 146 + .../docs/next/explainer/src/21-epilogue.md | 104 + .../docs/next/explainer/src/22-appendix.md | 190 + .../chord/docs/next/explainer/walkthrough.md | 284 + packages/chord/docs/next/flow-control/book.md | 333 + .../docs/next/flow-control/fold/index.html | 265 + .../chord/docs/next/flow-control/index.html | 548 ++ .../docs/next/flow-control/window/index.html | 228 + packages/chord/docs/next/layer-1.md | 311 + packages/chord/docs/next/layer-2.md | 216 + packages/chord/docs/next/slice-1.md | 236 + packages/chord/docs/next/slice-2.md | 174 + packages/chord/docs/next/slice-3a.md | 296 + packages/chord/docs/next/slice-3b.md | 197 + packages/chord/docs/next/slice-4.md | 146 + packages/chord/docs/next/slice-4a.md | 418 ++ packages/chord/docs/next/slice-4b.md | 208 + packages/chord/docs/next/slice-5.md | 281 + packages/chord/docs/next/slice-6a.md | 676 ++ packages/chord/docs/next/slice-6b.md | 244 + packages/chord/docs/next/slices.md | 269 + packages/chord/docs/next/store/book.md | 665 ++ packages/chord/docs/next/store/index.html | 861 +++ packages/chord/docs/next/store/walkthrough.md | 193 + packages/chord/src/next-node/compression.ts | 24 + packages/chord/src/next/contracts/README.md | 80 + packages/chord/src/next/contracts/accepts.ts | 244 + packages/chord/src/next/contracts/contract.ts | 178 + packages/chord/src/next/contracts/declare.ts | 237 + packages/chord/src/next/contracts/evolve.ts | 310 + .../chord/src/next/contracts/fingerprint.ts | 69 + packages/chord/src/next/contracts/history.ts | 66 + .../chord/src/next/contracts/host-kind.ts | 68 + packages/chord/src/next/contracts/index.ts | 30 + packages/chord/src/next/contracts/internal.ts | 25 + packages/chord/src/next/contracts/names.ts | 14 + packages/chord/src/next/contracts/params.ts | 77 + packages/chord/src/next/contracts/schema.ts | 180 + packages/chord/src/next/contracts/snapshot.ts | 206 + packages/chord/src/next/data/README.md | 60 + packages/chord/src/next/data/args.ts | 142 + packages/chord/src/next/data/errors.ts | 142 + packages/chord/src/next/data/failure.ts | 26 + packages/chord/src/next/data/index.ts | 11 + packages/chord/src/next/data/internal.ts | 14 + packages/chord/src/next/data/params.ts | 43 + packages/chord/src/next/data/principal.ts | 57 + packages/chord/src/next/data/validator.ts | 72 + packages/chord/src/next/documents/README.md | 91 + .../src/next/documents/address-source.ts | 56 + .../chord/src/next/documents/definition.ts | 157 + .../chord/src/next/documents/documents.ts | 237 + packages/chord/src/next/documents/fence.ts | 17 + packages/chord/src/next/documents/handles.ts | 133 + packages/chord/src/next/documents/index.ts | 5 + packages/chord/src/next/documents/internal.ts | 9 + .../chord/src/next/documents/keyed-slot.ts | 414 ++ packages/chord/src/next/documents/line.ts | 15 + .../chord/src/next/documents/persistence.ts | 156 + .../chord/src/next/documents/residency.ts | 110 + packages/chord/src/next/documents/slot.ts | 239 + .../chord/src/next/documents/transaction.ts | 510 ++ packages/chord/src/next/host/README.md | 479 ++ packages/chord/src/next/host/activity.ts | 56 + packages/chord/src/next/host/attempt.ts | 620 ++ packages/chord/src/next/host/backoff.ts | 83 + packages/chord/src/next/host/ctx.ts | 132 + packages/chord/src/next/host/deadline.ts | 43 + packages/chord/src/next/host/facet.ts | 71 + packages/chord/src/next/host/host.ts | 862 +++ packages/chord/src/next/host/index.ts | 24 + packages/chord/src/next/host/internal.ts | 11 + packages/chord/src/next/host/keyed-members.ts | 268 + packages/chord/src/next/host/member-source.ts | 221 + packages/chord/src/next/host/members.ts | 124 + packages/chord/src/next/host/native.ts | 151 + packages/chord/src/next/host/object.ts | 691 +++ packages/chord/src/next/host/parked.ts | 98 + packages/chord/src/next/host/placement.ts | 112 + packages/chord/src/next/host/presence.ts | 132 + packages/chord/src/next/host/ref-state.ts | 91 + packages/chord/src/next/host/reference.ts | 674 ++ packages/chord/src/next/host/revocation.ts | 35 + packages/chord/src/next/host/route.ts | 160 + packages/chord/src/next/host/routines.ts | 432 ++ packages/chord/src/next/host/sleep.ts | 43 + packages/chord/src/next/host/uuid.ts | 38 + packages/chord/src/next/jobs/README.md | 112 + packages/chord/src/next/jobs/definition.ts | 123 + packages/chord/src/next/jobs/index.ts | 13 + packages/chord/src/next/jobs/internal.ts | 4 + packages/chord/src/next/jobs/job.ts | 319 + packages/chord/src/next/jobs/records.ts | 446 ++ packages/chord/src/next/jobs/run.ts | 137 + packages/chord/src/next/jobs/running.ts | 143 + packages/chord/src/next/link/README.md | 141 + packages/chord/src/next/link/binary.ts | 565 ++ packages/chord/src/next/link/channel.ts | 243 + packages/chord/src/next/link/errors.ts | 23 + packages/chord/src/next/link/index.ts | 10 + packages/chord/src/next/link/internal.ts | 35 + packages/chord/src/next/link/ipc.ts | 78 + packages/chord/src/next/link/link.ts | 106 + packages/chord/src/next/link/message.ts | 182 + packages/chord/src/next/link/part.ts | 208 + packages/chord/src/next/link/text.ts | 356 ++ packages/chord/src/next/link/values.ts | 126 + packages/chord/src/next/link/websocket.ts | 106 + packages/chord/src/next/remote/README.md | 172 + packages/chord/src/next/remote/calls.ts | 70 + packages/chord/src/next/remote/consumer.ts | 355 ++ packages/chord/src/next/remote/hold.ts | 434 ++ packages/chord/src/next/remote/host-link.ts | 219 + packages/chord/src/next/remote/internal.ts | 8 + packages/chord/src/next/remote/path.ts | 68 + packages/chord/src/next/remote/provider.ts | 510 ++ packages/chord/src/next/remote/relay.ts | 130 + .../chord/src/next/remote/remote-object.ts | 428 ++ packages/chord/src/next/remote/replica.ts | 141 + packages/chord/src/next/state/README.md | 70 + packages/chord/src/next/state/batch.ts | 118 + packages/chord/src/next/state/derive.ts | 191 + packages/chord/src/next/state/index.ts | 5 + packages/chord/src/next/state/internal.ts | 5 + packages/chord/src/next/state/watchable.ts | 190 + packages/chord/src/next/store/README.md | 67 + packages/chord/src/next/store/applier.ts | 163 + packages/chord/src/next/store/archive.ts | 106 + packages/chord/src/next/store/backend.ts | 19 + packages/chord/src/next/store/catalog.ts | 37 + packages/chord/src/next/store/columns.ts | 153 + packages/chord/src/next/store/engine.ts | 295 + packages/chord/src/next/store/evaluate.ts | 131 + packages/chord/src/next/store/export.ts | 207 + packages/chord/src/next/store/expression.ts | 356 ++ packages/chord/src/next/store/import.ts | 236 + packages/chord/src/next/store/index.ts | 49 + packages/chord/src/next/store/json-column.ts | 78 + packages/chord/src/next/store/jsonl.ts | 60 + packages/chord/src/next/store/outbox.ts | 100 + packages/chord/src/next/store/query.ts | 118 + packages/chord/src/next/store/rows.ts | 213 + packages/chord/src/next/store/schema.ts | 89 + packages/chord/src/next/store/sql.ts | 266 + packages/chord/src/next/store/step.ts | 69 + packages/chord/src/next/store/store.ts | 347 ++ .../chord/src/next/store/table-changes.ts | 147 + packages/chord/src/next/store/table-evolve.ts | 306 + packages/chord/src/next/store/table-handle.ts | 149 + .../chord/src/next/store/table-history.ts | 98 + .../chord/src/next/store/table-migration.ts | 245 + packages/chord/src/next/store/table-writes.ts | 260 + packages/chord/src/next/store/table.ts | 157 + packages/chord/src/next/store/text.ts | 51 + packages/chord/src/next/store/writes.ts | 8 + .../chord/test/next-node/compression.test.ts | 39 + packages/chord/test/next/bench/compression.ts | 155 + packages/chord/test/next/bench/derives.ts | 73 + packages/chord/test/next/bench/links.ts | 469 ++ packages/chord/test/next/bench/relay.ts | 619 ++ packages/chord/test/next/bench/store.ts | 450 ++ .../chord/test/next/contracts/accepts.test.ts | 247 + .../test/next/contracts/contract.test.ts | 115 + .../chord/test/next/contracts/declare.test.ts | 228 + .../chord/test/next/contracts/evolve.test.ts | 217 + .../test/next/contracts/fingerprint.test.ts | 80 + .../chord/test/next/contracts/history.test.ts | 37 + .../test/next/contracts/host-kind.test.ts | 80 + .../chord/test/next/contracts/params.test.ts | 41 + .../chord/test/next/contracts/schema.test.ts | 99 + .../test/next/contracts/snapshot.test.ts | 190 + packages/chord/test/next/data/args.test.ts | 223 + packages/chord/test/next/data/errors.test.ts | 135 + packages/chord/test/next/data/failure.test.ts | 27 + packages/chord/test/next/data/params.test.ts | 130 + .../chord/test/next/data/principal.test.ts | 75 + .../chord/test/next/data/validator.test.ts | 106 + .../next/documents/address-source.test.ts | 114 + .../test/next/documents/definition.test.ts | 90 + .../test/next/documents/documents.test.ts | 111 + .../chord/test/next/documents/fence.test.ts | 14 + .../chord/test/next/documents/handles.test.ts | 75 + .../test/next/documents/keyed-slot.test.ts | 704 +++ .../chord/test/next/documents/line.test.ts | 45 + .../test/next/documents/persistence.test.ts | 292 + .../test/next/documents/residency.test.ts | 77 + .../chord/test/next/documents/slot.test.ts | 375 ++ .../test/next/documents/transaction.test.ts | 731 +++ .../chord/test/next/examples/01-objects.ts | 68 + .../chord/test/next/examples/02-documents.ts | 79 + .../chord/test/next/examples/03-derives.ts | 66 + .../chord/test/next/examples/04-watches.ts | 101 + .../chord/test/next/examples/05-errors.ts | 93 + .../chord/test/next/examples/06-natives.ts | 96 + .../chord/test/next/examples/07-failures.ts | 96 + .../chord/test/next/examples/08-remote.ts | 111 + packages/chord/test/next/examples/09-calls.ts | 107 + packages/chord/test/next/examples/10-drops.ts | 107 + .../test/next/examples/11-linger-and-retry.ts | 109 + .../chord/test/next/examples/12-routes.ts | 160 + packages/chord/test/next/examples/13-relay.ts | 146 + .../test/next/examples/14-flow-control.ts | 208 + .../chord/test/next/examples/15-revocation.ts | 142 + .../next/examples/16-presence-and-activity.ts | 89 + .../examples/17-limits-and-compression.ts | 189 + .../chord/test/next/examples/18-routines.ts | 115 + packages/chord/test/next/examples/19-jobs.ts | 211 + .../chord/test/next/examples/20-tables.ts | 101 + .../next/examples/21-persisted-documents.ts | 91 + .../chord/test/next/examples/22-migrations.ts | 100 + .../test/next/examples/23-keyed-watches.ts | 84 + .../test/next/examples/24-keyed-members.ts | 64 + .../test/next/examples/25-export-import.ts | 102 + .../chord/test/next/host/activity.test.ts | 146 + packages/chord/test/next/host/attempt.test.ts | 848 +++ packages/chord/test/next/host/backoff.test.ts | 87 + packages/chord/test/next/host/ctx.test.ts | 173 + .../chord/test/next/host/deadline.test.ts | 42 + packages/chord/test/next/host/facet.test.ts | 39 + packages/chord/test/next/host/host.test.ts | 944 +++ .../test/next/host/keyed-members.test.ts | 452 ++ .../test/next/host/member-source.test.ts | 507 ++ packages/chord/test/next/host/members.test.ts | 31 + packages/chord/test/next/host/native.test.ts | 244 + packages/chord/test/next/host/object.test.ts | 1047 ++++ packages/chord/test/next/host/parked.test.ts | 213 + .../chord/test/next/host/presence.test.ts | 328 + .../chord/test/next/host/ref-state.test.ts | 189 + .../chord/test/next/host/reference.test.ts | 656 ++ .../chord/test/next/host/revocation.test.ts | 430 ++ packages/chord/test/next/host/route.test.ts | 368 ++ .../chord/test/next/host/routines.test.ts | 1148 ++++ packages/chord/test/next/host/sleep.test.ts | 74 + .../chord/test/next/jobs/definition.test.ts | 106 + packages/chord/test/next/jobs/job.test.ts | 1378 +++++ packages/chord/test/next/jobs/records.test.ts | 205 + packages/chord/test/next/jobs/run.test.ts | 49 + packages/chord/test/next/jobs/running.test.ts | 124 + packages/chord/test/next/link/channel.test.ts | 342 ++ packages/chord/test/next/link/errors.test.ts | 35 + .../chord/test/next/link/fixtures/ipc-echo.ts | 10 + packages/chord/test/next/link/ipc.test.ts | 143 + packages/chord/test/next/link/link.test.ts | 96 + packages/chord/test/next/link/message.test.ts | 540 ++ packages/chord/test/next/link/part.test.ts | 196 + packages/chord/test/next/link/values.test.ts | 64 + .../chord/test/next/link/websocket.test.ts | 159 + packages/chord/test/next/remote/calls.test.ts | 74 + .../chord/test/next/remote/consumer.test.ts | 718 +++ packages/chord/test/next/remote/hold.test.ts | 694 +++ .../chord/test/next/remote/host-link.test.ts | 429 ++ packages/chord/test/next/remote/path.test.ts | 70 + .../chord/test/next/remote/provider.test.ts | 974 +++ packages/chord/test/next/remote/relay.test.ts | 732 +++ .../test/next/remote/remote-object.test.ts | 768 +++ .../chord/test/next/remote/replica.test.ts | 89 + packages/chord/test/next/state/batch.test.ts | 154 + packages/chord/test/next/state/derive.test.ts | 484 ++ .../chord/test/next/state/watchable.test.ts | 133 + .../chord/test/next/store/applier.test.ts | 114 + .../chord/test/next/store/archive.test.ts | 42 + .../chord/test/next/store/catalog.test.ts | 65 + .../chord/test/next/store/columns.test.ts | 73 + packages/chord/test/next/store/engine.test.ts | 142 + .../chord/test/next/store/evaluate.test.ts | 211 + packages/chord/test/next/store/export.test.ts | 230 + .../chord/test/next/store/expression.test.ts | 152 + packages/chord/test/next/store/import.test.ts | 436 ++ .../chord/test/next/store/json-column.test.ts | 58 + packages/chord/test/next/store/jsonl.test.ts | 56 + packages/chord/test/next/store/outbox.test.ts | 165 + packages/chord/test/next/store/query.test.ts | 86 + packages/chord/test/next/store/rows.test.ts | 77 + packages/chord/test/next/store/sql.test.ts | 106 + packages/chord/test/next/store/store.test.ts | 250 + .../test/next/store/table-changes.test.ts | 84 + .../test/next/store/table-evolve.test.ts | 225 + .../test/next/store/table-handle.test.ts | 136 + .../test/next/store/table-history.test.ts | 73 + .../test/next/store/table-migration.test.ts | 329 + .../test/next/store/table-writes.test.ts | 233 + packages/chord/test/next/store/table.test.ts | 66 + packages/chord/test/next/store/text.test.ts | 62 + packages/chord/test/next/support.ts | 973 +++ 314 files changed, 79114 insertions(+) create mode 100644 packages/chord/docs/next/benchmarks.md create mode 100644 packages/chord/docs/next/decisions.md create mode 100644 packages/chord/docs/next/design.md create mode 100644 packages/chord/docs/next/durable-observers/book.md create mode 100644 packages/chord/docs/next/durable-observers/index.html create mode 100644 packages/chord/docs/next/explainer/book.md create mode 100644 packages/chord/docs/next/explainer/index.html create mode 100644 packages/chord/docs/next/explainer/src/00-front.md create mode 100644 packages/chord/docs/next/explainer/src/01-shape.md create mode 100644 packages/chord/docs/next/explainer/src/02-first.md create mode 100644 packages/chord/docs/next/explainer/src/03-principals.md create mode 100644 packages/chord/docs/next/explainer/src/04-store.md create mode 100644 packages/chord/docs/next/explainer/src/05-documents.md create mode 100644 packages/chord/docs/next/explainer/src/06-references.md create mode 100644 packages/chord/docs/next/explainer/src/07-revocation.md create mode 100644 packages/chord/docs/next/explainer/src/08-derives.md create mode 100644 packages/chord/docs/next/explainer/src/09-natives.md create mode 100644 packages/chord/docs/next/explainer/src/10-routines.md create mode 100644 packages/chord/docs/next/explainer/src/11-jobs.md create mode 100644 packages/chord/docs/next/explainer/src/12-flows.md create mode 100644 packages/chord/docs/next/explainer/src/13-registries.md create mode 100644 packages/chord/docs/next/explainer/src/14-agent.md create mode 100644 packages/chord/docs/next/explainer/src/15-reload.md create mode 100644 packages/chord/docs/next/explainer/src/16-contracts.md create mode 100644 packages/chord/docs/next/explainer/src/17-wire.md create mode 100644 packages/chord/docs/next/explainer/src/18-p2-core.md create mode 100644 packages/chord/docs/next/explainer/src/19-p2-sessions.md create mode 100644 packages/chord/docs/next/explainer/src/20-p2-machines.md create mode 100644 packages/chord/docs/next/explainer/src/20a-compared.md create mode 100644 packages/chord/docs/next/explainer/src/21-epilogue.md create mode 100644 packages/chord/docs/next/explainer/src/22-appendix.md create mode 100644 packages/chord/docs/next/explainer/walkthrough.md create mode 100644 packages/chord/docs/next/flow-control/book.md create mode 100644 packages/chord/docs/next/flow-control/fold/index.html create mode 100644 packages/chord/docs/next/flow-control/index.html create mode 100644 packages/chord/docs/next/flow-control/window/index.html create mode 100644 packages/chord/docs/next/layer-1.md create mode 100644 packages/chord/docs/next/layer-2.md create mode 100644 packages/chord/docs/next/slice-1.md create mode 100644 packages/chord/docs/next/slice-2.md create mode 100644 packages/chord/docs/next/slice-3a.md create mode 100644 packages/chord/docs/next/slice-3b.md create mode 100644 packages/chord/docs/next/slice-4.md create mode 100644 packages/chord/docs/next/slice-4a.md create mode 100644 packages/chord/docs/next/slice-4b.md create mode 100644 packages/chord/docs/next/slice-5.md create mode 100644 packages/chord/docs/next/slice-6a.md create mode 100644 packages/chord/docs/next/slice-6b.md create mode 100644 packages/chord/docs/next/slices.md create mode 100644 packages/chord/docs/next/store/book.md create mode 100644 packages/chord/docs/next/store/index.html create mode 100644 packages/chord/docs/next/store/walkthrough.md create mode 100644 packages/chord/src/next-node/compression.ts create mode 100644 packages/chord/src/next/contracts/README.md create mode 100644 packages/chord/src/next/contracts/accepts.ts create mode 100644 packages/chord/src/next/contracts/contract.ts create mode 100644 packages/chord/src/next/contracts/declare.ts create mode 100644 packages/chord/src/next/contracts/evolve.ts create mode 100644 packages/chord/src/next/contracts/fingerprint.ts create mode 100644 packages/chord/src/next/contracts/history.ts create mode 100644 packages/chord/src/next/contracts/host-kind.ts create mode 100644 packages/chord/src/next/contracts/index.ts create mode 100644 packages/chord/src/next/contracts/internal.ts create mode 100644 packages/chord/src/next/contracts/names.ts create mode 100644 packages/chord/src/next/contracts/params.ts create mode 100644 packages/chord/src/next/contracts/schema.ts create mode 100644 packages/chord/src/next/contracts/snapshot.ts create mode 100644 packages/chord/src/next/data/README.md create mode 100644 packages/chord/src/next/data/args.ts create mode 100644 packages/chord/src/next/data/errors.ts create mode 100644 packages/chord/src/next/data/failure.ts create mode 100644 packages/chord/src/next/data/index.ts create mode 100644 packages/chord/src/next/data/internal.ts create mode 100644 packages/chord/src/next/data/params.ts create mode 100644 packages/chord/src/next/data/principal.ts create mode 100644 packages/chord/src/next/data/validator.ts create mode 100644 packages/chord/src/next/documents/README.md create mode 100644 packages/chord/src/next/documents/address-source.ts create mode 100644 packages/chord/src/next/documents/definition.ts create mode 100644 packages/chord/src/next/documents/documents.ts create mode 100644 packages/chord/src/next/documents/fence.ts create mode 100644 packages/chord/src/next/documents/handles.ts create mode 100644 packages/chord/src/next/documents/index.ts create mode 100644 packages/chord/src/next/documents/internal.ts create mode 100644 packages/chord/src/next/documents/keyed-slot.ts create mode 100644 packages/chord/src/next/documents/line.ts create mode 100644 packages/chord/src/next/documents/persistence.ts create mode 100644 packages/chord/src/next/documents/residency.ts create mode 100644 packages/chord/src/next/documents/slot.ts create mode 100644 packages/chord/src/next/documents/transaction.ts create mode 100644 packages/chord/src/next/host/README.md create mode 100644 packages/chord/src/next/host/activity.ts create mode 100644 packages/chord/src/next/host/attempt.ts create mode 100644 packages/chord/src/next/host/backoff.ts create mode 100644 packages/chord/src/next/host/ctx.ts create mode 100644 packages/chord/src/next/host/deadline.ts create mode 100644 packages/chord/src/next/host/facet.ts create mode 100644 packages/chord/src/next/host/host.ts create mode 100644 packages/chord/src/next/host/index.ts create mode 100644 packages/chord/src/next/host/internal.ts create mode 100644 packages/chord/src/next/host/keyed-members.ts create mode 100644 packages/chord/src/next/host/member-source.ts create mode 100644 packages/chord/src/next/host/members.ts create mode 100644 packages/chord/src/next/host/native.ts create mode 100644 packages/chord/src/next/host/object.ts create mode 100644 packages/chord/src/next/host/parked.ts create mode 100644 packages/chord/src/next/host/placement.ts create mode 100644 packages/chord/src/next/host/presence.ts create mode 100644 packages/chord/src/next/host/ref-state.ts create mode 100644 packages/chord/src/next/host/reference.ts create mode 100644 packages/chord/src/next/host/revocation.ts create mode 100644 packages/chord/src/next/host/route.ts create mode 100644 packages/chord/src/next/host/routines.ts create mode 100644 packages/chord/src/next/host/sleep.ts create mode 100644 packages/chord/src/next/host/uuid.ts create mode 100644 packages/chord/src/next/jobs/README.md create mode 100644 packages/chord/src/next/jobs/definition.ts create mode 100644 packages/chord/src/next/jobs/index.ts create mode 100644 packages/chord/src/next/jobs/internal.ts create mode 100644 packages/chord/src/next/jobs/job.ts create mode 100644 packages/chord/src/next/jobs/records.ts create mode 100644 packages/chord/src/next/jobs/run.ts create mode 100644 packages/chord/src/next/jobs/running.ts create mode 100644 packages/chord/src/next/link/README.md create mode 100644 packages/chord/src/next/link/binary.ts create mode 100644 packages/chord/src/next/link/channel.ts create mode 100644 packages/chord/src/next/link/errors.ts create mode 100644 packages/chord/src/next/link/index.ts create mode 100644 packages/chord/src/next/link/internal.ts create mode 100644 packages/chord/src/next/link/ipc.ts create mode 100644 packages/chord/src/next/link/link.ts create mode 100644 packages/chord/src/next/link/message.ts create mode 100644 packages/chord/src/next/link/part.ts create mode 100644 packages/chord/src/next/link/text.ts create mode 100644 packages/chord/src/next/link/values.ts create mode 100644 packages/chord/src/next/link/websocket.ts create mode 100644 packages/chord/src/next/remote/README.md create mode 100644 packages/chord/src/next/remote/calls.ts create mode 100644 packages/chord/src/next/remote/consumer.ts create mode 100644 packages/chord/src/next/remote/hold.ts create mode 100644 packages/chord/src/next/remote/host-link.ts create mode 100644 packages/chord/src/next/remote/internal.ts create mode 100644 packages/chord/src/next/remote/path.ts create mode 100644 packages/chord/src/next/remote/provider.ts create mode 100644 packages/chord/src/next/remote/relay.ts create mode 100644 packages/chord/src/next/remote/remote-object.ts create mode 100644 packages/chord/src/next/remote/replica.ts create mode 100644 packages/chord/src/next/state/README.md create mode 100644 packages/chord/src/next/state/batch.ts create mode 100644 packages/chord/src/next/state/derive.ts create mode 100644 packages/chord/src/next/state/index.ts create mode 100644 packages/chord/src/next/state/internal.ts create mode 100644 packages/chord/src/next/state/watchable.ts create mode 100644 packages/chord/src/next/store/README.md create mode 100644 packages/chord/src/next/store/applier.ts create mode 100644 packages/chord/src/next/store/archive.ts create mode 100644 packages/chord/src/next/store/backend.ts create mode 100644 packages/chord/src/next/store/catalog.ts create mode 100644 packages/chord/src/next/store/columns.ts create mode 100644 packages/chord/src/next/store/engine.ts create mode 100644 packages/chord/src/next/store/evaluate.ts create mode 100644 packages/chord/src/next/store/export.ts create mode 100644 packages/chord/src/next/store/expression.ts create mode 100644 packages/chord/src/next/store/import.ts create mode 100644 packages/chord/src/next/store/index.ts create mode 100644 packages/chord/src/next/store/json-column.ts create mode 100644 packages/chord/src/next/store/jsonl.ts create mode 100644 packages/chord/src/next/store/outbox.ts create mode 100644 packages/chord/src/next/store/query.ts create mode 100644 packages/chord/src/next/store/rows.ts create mode 100644 packages/chord/src/next/store/schema.ts create mode 100644 packages/chord/src/next/store/sql.ts create mode 100644 packages/chord/src/next/store/step.ts create mode 100644 packages/chord/src/next/store/store.ts create mode 100644 packages/chord/src/next/store/table-changes.ts create mode 100644 packages/chord/src/next/store/table-evolve.ts create mode 100644 packages/chord/src/next/store/table-handle.ts create mode 100644 packages/chord/src/next/store/table-history.ts create mode 100644 packages/chord/src/next/store/table-migration.ts create mode 100644 packages/chord/src/next/store/table-writes.ts create mode 100644 packages/chord/src/next/store/table.ts create mode 100644 packages/chord/src/next/store/text.ts create mode 100644 packages/chord/src/next/store/writes.ts create mode 100644 packages/chord/test/next-node/compression.test.ts create mode 100644 packages/chord/test/next/bench/compression.ts create mode 100644 packages/chord/test/next/bench/derives.ts create mode 100644 packages/chord/test/next/bench/links.ts create mode 100644 packages/chord/test/next/bench/relay.ts create mode 100644 packages/chord/test/next/bench/store.ts create mode 100644 packages/chord/test/next/contracts/accepts.test.ts create mode 100644 packages/chord/test/next/contracts/contract.test.ts create mode 100644 packages/chord/test/next/contracts/declare.test.ts create mode 100644 packages/chord/test/next/contracts/evolve.test.ts create mode 100644 packages/chord/test/next/contracts/fingerprint.test.ts create mode 100644 packages/chord/test/next/contracts/history.test.ts create mode 100644 packages/chord/test/next/contracts/host-kind.test.ts create mode 100644 packages/chord/test/next/contracts/params.test.ts create mode 100644 packages/chord/test/next/contracts/schema.test.ts create mode 100644 packages/chord/test/next/contracts/snapshot.test.ts create mode 100644 packages/chord/test/next/data/args.test.ts create mode 100644 packages/chord/test/next/data/errors.test.ts create mode 100644 packages/chord/test/next/data/failure.test.ts create mode 100644 packages/chord/test/next/data/params.test.ts create mode 100644 packages/chord/test/next/data/principal.test.ts create mode 100644 packages/chord/test/next/data/validator.test.ts create mode 100644 packages/chord/test/next/documents/address-source.test.ts create mode 100644 packages/chord/test/next/documents/definition.test.ts create mode 100644 packages/chord/test/next/documents/documents.test.ts create mode 100644 packages/chord/test/next/documents/fence.test.ts create mode 100644 packages/chord/test/next/documents/handles.test.ts create mode 100644 packages/chord/test/next/documents/keyed-slot.test.ts create mode 100644 packages/chord/test/next/documents/line.test.ts create mode 100644 packages/chord/test/next/documents/persistence.test.ts create mode 100644 packages/chord/test/next/documents/residency.test.ts create mode 100644 packages/chord/test/next/documents/slot.test.ts create mode 100644 packages/chord/test/next/documents/transaction.test.ts create mode 100644 packages/chord/test/next/examples/01-objects.ts create mode 100644 packages/chord/test/next/examples/02-documents.ts create mode 100644 packages/chord/test/next/examples/03-derives.ts create mode 100644 packages/chord/test/next/examples/04-watches.ts create mode 100644 packages/chord/test/next/examples/05-errors.ts create mode 100644 packages/chord/test/next/examples/06-natives.ts create mode 100644 packages/chord/test/next/examples/07-failures.ts create mode 100644 packages/chord/test/next/examples/08-remote.ts create mode 100644 packages/chord/test/next/examples/09-calls.ts create mode 100644 packages/chord/test/next/examples/10-drops.ts create mode 100644 packages/chord/test/next/examples/11-linger-and-retry.ts create mode 100644 packages/chord/test/next/examples/12-routes.ts create mode 100644 packages/chord/test/next/examples/13-relay.ts create mode 100644 packages/chord/test/next/examples/14-flow-control.ts create mode 100644 packages/chord/test/next/examples/15-revocation.ts create mode 100644 packages/chord/test/next/examples/16-presence-and-activity.ts create mode 100644 packages/chord/test/next/examples/17-limits-and-compression.ts create mode 100644 packages/chord/test/next/examples/18-routines.ts create mode 100644 packages/chord/test/next/examples/19-jobs.ts create mode 100644 packages/chord/test/next/examples/20-tables.ts create mode 100644 packages/chord/test/next/examples/21-persisted-documents.ts create mode 100644 packages/chord/test/next/examples/22-migrations.ts create mode 100644 packages/chord/test/next/examples/23-keyed-watches.ts create mode 100644 packages/chord/test/next/examples/24-keyed-members.ts create mode 100644 packages/chord/test/next/examples/25-export-import.ts create mode 100644 packages/chord/test/next/host/activity.test.ts create mode 100644 packages/chord/test/next/host/attempt.test.ts create mode 100644 packages/chord/test/next/host/backoff.test.ts create mode 100644 packages/chord/test/next/host/ctx.test.ts create mode 100644 packages/chord/test/next/host/deadline.test.ts create mode 100644 packages/chord/test/next/host/facet.test.ts create mode 100644 packages/chord/test/next/host/host.test.ts create mode 100644 packages/chord/test/next/host/keyed-members.test.ts create mode 100644 packages/chord/test/next/host/member-source.test.ts create mode 100644 packages/chord/test/next/host/members.test.ts create mode 100644 packages/chord/test/next/host/native.test.ts create mode 100644 packages/chord/test/next/host/object.test.ts create mode 100644 packages/chord/test/next/host/parked.test.ts create mode 100644 packages/chord/test/next/host/presence.test.ts create mode 100644 packages/chord/test/next/host/ref-state.test.ts create mode 100644 packages/chord/test/next/host/reference.test.ts create mode 100644 packages/chord/test/next/host/revocation.test.ts create mode 100644 packages/chord/test/next/host/route.test.ts create mode 100644 packages/chord/test/next/host/routines.test.ts create mode 100644 packages/chord/test/next/host/sleep.test.ts create mode 100644 packages/chord/test/next/jobs/definition.test.ts create mode 100644 packages/chord/test/next/jobs/job.test.ts create mode 100644 packages/chord/test/next/jobs/records.test.ts create mode 100644 packages/chord/test/next/jobs/run.test.ts create mode 100644 packages/chord/test/next/jobs/running.test.ts create mode 100644 packages/chord/test/next/link/channel.test.ts create mode 100644 packages/chord/test/next/link/errors.test.ts create mode 100644 packages/chord/test/next/link/fixtures/ipc-echo.ts create mode 100644 packages/chord/test/next/link/ipc.test.ts create mode 100644 packages/chord/test/next/link/link.test.ts create mode 100644 packages/chord/test/next/link/message.test.ts create mode 100644 packages/chord/test/next/link/part.test.ts create mode 100644 packages/chord/test/next/link/values.test.ts create mode 100644 packages/chord/test/next/link/websocket.test.ts create mode 100644 packages/chord/test/next/remote/calls.test.ts create mode 100644 packages/chord/test/next/remote/consumer.test.ts create mode 100644 packages/chord/test/next/remote/hold.test.ts create mode 100644 packages/chord/test/next/remote/host-link.test.ts create mode 100644 packages/chord/test/next/remote/path.test.ts create mode 100644 packages/chord/test/next/remote/provider.test.ts create mode 100644 packages/chord/test/next/remote/relay.test.ts create mode 100644 packages/chord/test/next/remote/remote-object.test.ts create mode 100644 packages/chord/test/next/remote/replica.test.ts create mode 100644 packages/chord/test/next/state/batch.test.ts create mode 100644 packages/chord/test/next/state/derive.test.ts create mode 100644 packages/chord/test/next/state/watchable.test.ts create mode 100644 packages/chord/test/next/store/applier.test.ts create mode 100644 packages/chord/test/next/store/archive.test.ts create mode 100644 packages/chord/test/next/store/catalog.test.ts create mode 100644 packages/chord/test/next/store/columns.test.ts create mode 100644 packages/chord/test/next/store/engine.test.ts create mode 100644 packages/chord/test/next/store/evaluate.test.ts create mode 100644 packages/chord/test/next/store/export.test.ts create mode 100644 packages/chord/test/next/store/expression.test.ts create mode 100644 packages/chord/test/next/store/import.test.ts create mode 100644 packages/chord/test/next/store/json-column.test.ts create mode 100644 packages/chord/test/next/store/jsonl.test.ts create mode 100644 packages/chord/test/next/store/outbox.test.ts create mode 100644 packages/chord/test/next/store/query.test.ts create mode 100644 packages/chord/test/next/store/rows.test.ts create mode 100644 packages/chord/test/next/store/sql.test.ts create mode 100644 packages/chord/test/next/store/store.test.ts create mode 100644 packages/chord/test/next/store/table-changes.test.ts create mode 100644 packages/chord/test/next/store/table-evolve.test.ts create mode 100644 packages/chord/test/next/store/table-handle.test.ts create mode 100644 packages/chord/test/next/store/table-history.test.ts create mode 100644 packages/chord/test/next/store/table-migration.test.ts create mode 100644 packages/chord/test/next/store/table-writes.test.ts create mode 100644 packages/chord/test/next/store/table.test.ts create mode 100644 packages/chord/test/next/store/text.test.ts create mode 100644 packages/chord/test/next/support.ts diff --git a/AGENTS.md b/AGENTS.md index 04c9c32be..43a9cf8c9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,6 +27,20 @@ - Never hardcode key checks (e.g. `matchesKey(keyData, "ctrl+x")`). Add defaults to `DEFAULT_EDITOR_KEYBINDINGS` or `DEFAULT_APP_KEYBINDINGS` so they stay configurable. - Never modify `packages/ai/src/models.generated.ts` directly; update the catalog definitions in `packages/ai/catalog/` instead (see its README), then regenerate. Including the resulting `models.generated.ts` diff is always OK, even if regeneration includes unrelated upstream model metadata changes. The generated model data in `packages/ai/src/providers/data/` is gitignored: never stage it, and don't look for it in git history. +## Chord v2 Code Style + +Binding for `packages/chord/src/next/` and its tests, when writing and when reviewing. The model is pi-durable's source. + +- Files: one concept per file, named after it; no one-helper files, no grab-bag files. First line: a comment saying what the file is for. Order: imports, exported types, then the public entry point first and its helpers below it in the order they are used; constants right above their first user. A blank line between top-level declarations. Split files by concept, never by size: one concept may take a long file (up to about 1,000 lines is fine); a second concept in a file is the reason to split. +- Functions: every export has a one-line doc comment saying what it does or guarantees. Inline comments explain a non-obvious why (cost, race, decided rule), never what the next line does. Short functions with one job (about 40 lines needs a reason; one `switch` over a tagged union is fine). No boolean flag parameters that switch behaviour: two functions or a tagged value. Never write the same lines twice. Context last: required args, then `context`/`ctx` (always required, as in Durable; optional only on error-reporting paths), then optional args. +- Types and checks: tagged unions for every state and result. Casts only where TypeScript cannot express a decided invariant, each with a short why. Every remaining runtime check names the trust boundary it serves (wire input, schema-checked params and arguments, declaration policy the types cannot express). Same-host values are plain data, immutable by convention: never copied, frozen or walked to enforce it; freeze only Chord-owned metadata when a decision says so. No defensive code against broken adapters, callers or our own modules. +- Cost and memory: no hot-path work a decision did not ask for (walks, copies, compilations, per-call allocations). The function doing counted work counts it through its context; call sites never count for others. Caches are `WeakMap`s keyed by their owner, each with a GC test; no module-level strong caches. +- Names and API: the design's vocabulary (`params`, `binding`, `principal key`), no invented synonyms; decided `ChordError` codes and reasons. Export only what a later layer or app needs. No dev or test modes, no unrequested options. +- Tests: one test file per source file, one `describe` per mechanism, the layer spec's test list case by case; cost guards through the in-memory telemetry adapter; GC tests for every cache and binding. Every slice also adds one or more runnable examples in `packages/chord/test/next/examples/NN-topic.ts`, one topic each, numbered across slices, written as references for Chord's users (Durable's `test/examples/` format: header with what it shows and how to run it, commented steps, printed results). +- Slices: Chord v2 is built in vertical slices (`packages/chord/docs/next/slices.md`). Implementers and reviewers read the slice spec, the feature map in `slices.md` and the whole of `packages/chord/docs/next/design.md` in full, and check the work against all three; slice specs cite design sections for orientation, not as a reading limit. +- Deferred work: anything left to a later slice goes into the section "Carry-over" of `packages/chord/docs/next/slices.md` under that slice, in the same change; a slice's spec and build include every entry of its section. +- Before reporting: re-read every changed file top to bottom as a stranger would; each file must be understandable without jumping around. Then `npm run check` and the layer's tests. + ## Commands - After code changes (not docs): `npm run check` (full output, no tail). Fix all errors, warnings, and infos before committing. Does not run tests. diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index cf4082fa9..5724e1a42 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -13,6 +13,7 @@ ### Added +- Chord v2, experimental and not exported yet: the next object runtime in `src/next` (contracts, hosts and objects, documents, derives, links and relays, routines and jobs, the store), with its tests and runnable examples in `test/next` and its design in `docs/next` (`design.md`, the decision log `decisions.md`, the slice plan `slices.md`). Its API changes with every slice until it gets an entry point. - `NativeWatchable`: an in-process value that changes and can be watched (`value`, `watch(context)` returning a `NativeWatchHandle`), the in-process counterpart of `Watchable`. - `WatchHandle` and `WatchEnd`, the one watch API, moved from Durable: a serialized observation with `value`, `start(listener)`, `stop()`, and `closed`; the listener gets a `WatchChange` (`{ value, ops, retired }`) and the context. `WatchEnd` reasons are `stopped`, `cancelled`, `ended`, `listener_error`, and `failed`. - `Watch`, the one watch handle implementation, moved from Durable: a source creates one per watch and feeds it with `advance(change, context)` (`retired: true` for a retirement), `end()` or `end({ error })`, `endAfterDelivery()`, and `observeCancellation(signal)`. Frames that arrive while the listener runs (or before `start`) are delivered in one call with the newest value, their ops folded by `PendingOps` and the newest frame's context; past 10,000 pending ops after folding, the call gets one root replacement `[["r", newest]]` instead (replaces the collapse after 100 undelivered frames); termination drops the listener and every reference into the source. diff --git a/packages/chord/docs/next/benchmarks.md b/packages/chord/docs/next/benchmarks.md new file mode 100644 index 000000000..4add7d22b --- /dev/null +++ b/packages/chord/docs/next/benchmarks.md @@ -0,0 +1,492 @@ +# Chord v2 benchmarks + +Light benchmarks of hot paths, run by hand (not in CI), recorded per slice for later slices to compare against +(owner, 2026-10-22). Each section names its script, the machine and what the numbers include. + +## Slice 3a: remote objects over one link + +Script: `test/next/bench/links.ts`, run from `packages/chord` with +`node --conditions=source --experimental-strip-types --expose-gc test/next/bench/links.ts` (about 15 s). Node 24.4.1, +Intel Core i9-13900, one process: server and client host on one memory link (`queueMicrotask`), every case once per +codec, side by side. + +Payloads, modelled on p2: entries `{ id, text, author, at }` in one document of a local store, exposed by a remote +view; "1,000 items" are entries with 80-character texts (the notes list), "500 KB" are 500 entries of 960 characters +(a conversation document). Commits are server-side method calls (one transaction each) on the store. + +What the numbers include: + +- *first hold to live*: the first `client.get`, which also activates the view on the server; one sample, so it moves + by a few hundred µs between runs (JIT and collector state). +- *hold to live (mean of 100)*: further `client.get`s of the live view, each released: the `hold`, the snapshot, the + `bound` message encoded, decoded and checked, the replicas built (two link hops). +- *wire bytes*: one whole message as its codec writes it (memory links add nothing). +- *encode / decode*: that message through the codec alone (`decode` without the schema check), averaged over 10,000 + runs (1,000 for snapshots and the 1 MiB call), the fastest of five such loops (one loop of a microsecond-sized + message is at the mercy of the collector and the JIT). +- *heap per encode + decode*: heap bytes allocated per message while encoding and decoding it that many times, + measured with `GCProfiler` (the growth plus what the collections freed); `ArrayBuffer` backing stores (the message + bytes themselves) are outside the heap. +- *array buffer kept per encoded message*: the `process.memoryUsage().arrayBuffers` growth while 50 encoded copies of + the message are kept, per message: the backing store a queued or held message keeps. Freed buffers are not reported + reliably inside a loop, so allocation of temporary buffers is not measured. V8 keeps very small typed arrays on the + heap, so the smallest binary messages show 0 B here. +- *commit to frame*: from the server's method call to the client's watch listener, averaged over 1,000 commits: the + call and its transaction, the encodes, the hop, decode, check, apply, and the watch's delivery. +- *heap / array buffers retained per hold*: the heap and `arrayBuffers` growth of holding the 1,000-item view, after full collections, divided by the + holds; both ends (the provider's member encoders and subscriptions, the consumer's member decoders and replicas, + each its own decoded copy of the snapshot), median of seven trials. + +| Case | binary | text | +| ------------------------------------------------------------------------------- | --- | --- | +| notes list of 1,000 items: first hold to live (activation, one sample) | 0.958 ms | 1.041 ms | +| notes list of 1,000 items: hold to live (two hops, mean of 100) | 0.666 ms | 0.686 ms | +| notes list of 1,000 items: snapshot: wire bytes | 133.7 KiB | 133.7 KiB | +| notes list of 1,000 items: snapshot: encode / decode | 291.1 µs / 266.4 µs | 312.3 µs / 302.8 µs | +| notes list of 1,000 items: snapshot: heap per encode + decode | 441.5 KiB | 441.8 KiB | +| notes list of 1,000 items: snapshot: array buffer kept per encoded message | 137.7 KiB | 133.7 KiB | +| 500 KB document: first hold to live (activation, one sample) | 1.461 ms | 1.555 ms | +| 500 KB document: hold to live (two hops, mean of 100) | 1.138 ms | 1.309 ms | +| 500 KB document: snapshot: wire bytes | 496.5 KiB | 496.5 KiB | +| 500 KB document: snapshot: encode / decode | 615.4 µs / 390.8 µs | 687.5 µs / 528.3 µs | +| 500 KB document: snapshot: heap per encode + decode | 1511.2 KiB | 1511.2 KiB | +| 500 KB document: snapshot: array buffer kept per encoded message | 500.5 KiB | 496.5 KiB | +| one appended item: commit to frame | 0.037 ms | 0.031 ms | +| one appended item: messages per commit, encodes per hold | 1, 1 | 1, 1 | +| one appended item: state message: wire bytes | 165 B | 175 B | +| one appended item: state message: encode / decode | 1.4 µs / 1.4 µs | 0.9 µs / 1.1 µs | +| one appended item: state message: heap per encode + decode | 2607 B | 3034 B | +| one appended item: state message: array buffer kept per encoded message | 165 B | 175 B | +| one changed field: commit to frame | 0.036 ms | 0.031 ms | +| one changed field: messages per commit, encodes per hold | 1, 1 | 1, 1 | +| one changed field: state message: wire bytes | 27 B | 38 B | +| one changed field: state message: encode / decode | 0.6 µs / 1.0 µs | 0.5 µs / 0.7 µs | +| one changed field: state message: heap per encode + decode | 2005 B | 2527 B | +| one changed field: state message: array buffer kept per encoded message | 0 B | 38 B | +| one removed item: commit to frame | 0.031 ms | 0.029 ms | +| one removed item: messages per commit, encodes per hold | 1, 1 | 1, 1 | +| one removed item: state message: wire bytes | 23 B | 34 B | +| one removed item: state message: encode / decode | 0.6 µs / 1.0 µs | 0.5 µs / 0.7 µs | +| one removed item: state message: heap per encode + decode | 1704 B | 2575 B | +| one removed item: state message: array buffer kept per encoded message | 0 B | 34 B | +| streamed reply, 2,000 appends: time per append | 0.031 ms | 0.027 ms | +| streamed reply: bytes per append, total | 23 B, 45.0 KiB | 34 B, 66.5 KiB | +| streamed token: wire bytes | 23 B | 34 B | +| streamed token: encode / decode | 0.5 µs / 1.0 µs | 0.5 µs / 0.6 µs | +| streamed token: heap per encode + decode | 1696 B | 2527 B | +| streamed token: array buffer kept per encoded message | 0 B | 34 B | +| 1 hold(s) of 1,000 items: heap / array buffers retained per hold (both ends) | 154.1 KiB / 47 B | 159.7 KiB / 21 B | +| 1 hold(s): messages per commit | 1 | 1 | +| 10 hold(s) of 1,000 items: heap / array buffers retained per hold (both ends) | 175.2 KiB / 41 B | 175.2 KiB / 44 B | +| 10 hold(s): messages per commit | 1 | 1 | +| call, small argument: round trip | 0.016 ms | 0.015 ms | +| call, small argument: call message: wire bytes | 16 B | 23 B | +| call, small argument: call message: encode / decode | 0.6 µs / 0.4 µs | 0.5 µs / 0.6 µs | +| call, small argument: call message: heap per encode + decode | 1580 B | 1603 B | +| call, small argument: call message: array buffer kept per encoded message | 0 B | 23 B | +| call, small argument: result message: wire bytes | 9 B | 14 B | +| call, small argument: result message: encode / decode | 0.4 µs / 0.3 µs | 0.5 µs / 0.5 µs | +| call, small argument: result message: heap per encode + decode | 1219 B | 1509 B | +| call, small argument: result message: array buffer kept per encoded message | 0 B | 14 B | +| call, 1 MiB bytes argument: round trip | 0.192 ms | 0.159 ms | +| call, 1 MiB bytes argument: call message: wire bytes | 1024.0 KiB | 1024.0 KiB | +| call, 1 MiB bytes argument: call message: encode / decode | 113.7 µs / 1.4 µs | 123.5 µs / 1.5 µs | +| call, 1 MiB bytes argument: call message: heap per encode + decode | 3042 B | 3383 B | +| call, 1 MiB bytes argument: call message: array buffer kept per encoded message | 1028.0 KiB | 1024.0 KiB | + +Observations: + +- One message per commit and one encode per hold in every case (W1). +- The binary codec makes small messages 30-45 % smaller (a streamed token 23 B against 34 B; a changed field 27 B + against 38 B) and allocates less heap per message; both are about 1 µs per small message each way. Binary decodes + small messages a little slower than text: it makes one call per field (varints, strings, one `JSON.parse` per value) + where text makes one `JSON.parse` of the whole tuple; the difference is under 1 µs per message and not worth + optimizing. Snapshots are JSON in both and about the same size; binary decodes them faster, text encodes them a + little faster (binary encodes the JSON text, then copies it into the message once). +- A binary message keeps a right-sized backing store (a 500 KB snapshot keeps about 500 KiB, not the 1.5 MB a buffer + reserved for the worst-case UTF-8 length kept before the review). +- Kept backing stores of small binary messages (owner, 2026-10-22): a message is written into a 256-byte starting + buffer, whose exact size is not known before the message is written, so in-place encoding does not right-size it. + `finish` copies to an exact-size buffer when the spare room is at least 64 bytes and more than an eighth of the + message. The appended-item state message (165 B) now keeps 165 B instead of 256 B, for about 0.2 µs more per encode + (1.1 against 1.3-1.4 µs); messages up to about 100 B were copied before already (they show 0 B: V8 keeps such small + arrays on the heap); large ones end within their 4 KiB of room and are not copied (the 1 MiB call keeps 1028 KiB). +- Large snapshots (owner's profiling request, 2026-10-22): the earlier "hold to live" gap (2.67 ms binary, 1.67 ms + text) was one sample each; averaged over 200 holds both codecs took about 1.2 ms. The remaining real gap was the + encode (886 against 718 µs): binary encoded the JSON text into its own buffer, then copied it into the message. + Long strings (from 4,096 UTF-16 units) are now encoded straight into the message buffer behind a fixed five-byte + length (a padded varint), and short ones (up to 42 units, a one-byte length) too: binary now encodes a 500 KB + snapshot in 650 µs against text's 690 µs and holds it in 1.14 ms against 1.30 ms. +- A 1 MiB `bytes()` argument travels as 1 MiB plus a few bytes in both codecs (not base64's 1.33 MiB); its decode is a + view (under 2 µs), its encode the one copy into the message. +- Encoding is not the bottleneck at these sizes; no CBOR. +- Each hold's replica is its own decoded copy (about 160-175 KiB for 1,000 entries); two references from one host are + two holds by design. + +## Slice 3b: surviving drops + +Script and machine as for slice 3a (`test/next/bench/links.ts` now runs the 3b cases after the 3a ones, about 37 s). + +New cases, both on the 1,000-item notes list view (133.7 KiB snapshot): + +- *reconnect, N reference(s)*: N references held on the client (N holds of one provider object); the server's end + of the link closes, the client sees the drop, then a new link is accepted and connected. *Time to all live*: from + `connect` to every reference's `ref.state` being `live` again (the re-holds, N snapshots encoded, decoded and + adopted as one frame `[["r", value]]` per member); the mean of 20 reconnects for one reference, 5 for 100. *Bytes*: + what the new link carried both ways until then (the handshake included). +- *1,000 calls parked while down*: a small call (`ping`) made 1,000 times right after `connect`, before the re-hold is + bound, so every call parks; time from `connect` to the last result (the re-hold, then 1,000 calls sent in order and + answered), mean of 10; the order is checked at the provider and in the results. + +| Case | binary | text | +| --- | --- | --- | +| reconnect, 1 reference of 1,000 items: time to all live (mean of 20) | 0.870 ms | 0.791 ms | +| reconnect, 1 reference of 1,000 items: bytes down / up | 133.7 KiB / 63 B | 133.8 KiB / 110 B | +| reconnect, 100 references of 1,000 items: time to all live (mean of 5) | 63.1 ms | 66.4 ms | +| reconnect, 100 references of 1,000 items: bytes down / up | 13370.5 KiB / 4718 B | 13371.1 KiB / 6736 B | +| 1,000 calls parked while down, flushed on reconnect: time to the last result | 14.9 ms | 13.8 ms | +| 1,000 parked calls: run in the order they were made | yes | yes | + +Regression rows (the live paths of 3a, same run): + +| Case | binary | text | +| --- | --- | --- | +| notes list of 1,000 items: hold to live (two hops, mean of 100) | 0.750 ms | 0.682 ms | +| 500 KB document: hold to live (two hops, mean of 100) | 1.134 ms | 1.308 ms | +| one appended item: commit to frame | 0.037 ms | 0.031 ms | +| one changed field: commit to frame | 0.036 ms | 0.029 ms | +| streamed reply, 2,000 appends: time per append | 0.031 ms | 0.027 ms | +| 1 hold of 1,000 items: heap / array buffers retained per hold (both ends) | 154.6 KiB / 47 B | 160.1 KiB / 21 B | +| 10 holds of 1,000 items: heap / array buffers retained per hold (both ends) | 175.1 KiB / 41 B | 175.4 KiB / 44 B | +| call, small argument: round trip | 0.016 ms | 0.014 ms | +| call, 1 MiB bytes argument: round trip | 0.192 ms | 0.191 ms | +| messages per commit, encodes per hold (every commit case) | 1, 1 | 1, 1 | + +Observations: + +- The live paths are unchanged within noise (3a: 0.67 / 0.69 ms per hold, 0.037 / 0.031 ms per appended item, 154.1 / + 159.7 KiB per hold); message sizes and encodes per commit are identical. A live reference's call checks its state + once and makes no queue; `ref.state` is not made until read. +- A reconnect costs what the holds cost the first time: about 0.63 ms and one 133.7 KiB snapshot per reference of + this size (100 references: 63 ms, 13 MiB). The consumer sends only the `hold`s (47 B each binary). +- 1,000 parked calls take about 14 ms to flush and answer, a little less than 1,000 sequential round trips (16 ms): + they leave in one turn once the re-hold is bound. Each parked call keeps its arguments and one timer until sent. + +## Slice 4a: links and relays + +Script: `test/next/bench/relay.ts`, run from `packages/chord` with +`node --conditions=source --experimental-strip-types --expose-gc --no-flush-bytecode test/next/bench/relay.ts` (about +5 s; the forked workers are the same file run with the argument `worker`). Machine as for slice 3a. Binary codec +throughout. The topology is the one of slice 4a's example: a tab (a client host), a server with an address and a +route for `Session`, and per session a worker host whose `Store` (a local document) backs a remote `View` placed on +the session; the tab holds the view through the server, which relays (no part parsed). + +What the numbers include: + +- *call to frame*: the tab calls `stream` (one commit appending `" tok"` on the worker) and waits for both the result + and its watch frame; mean of 1,000 after 100 warm-up calls. Through a relay that is four hops for the call and four + for the frame and result (tab → server → worker → server → tab); the baseline is a tab directly on the worker. + Slice 3a's *commit to frame* started at the provider, so these are not comparable with it. +- *bytes per commit*: bytes the server sent to the tab, and received from the worker, per commit (the `state` + message and the call's result; acks and the call go the other way). From the worker the messages carry the tab + link's origin (two bytes each). +- *first hold, `resolve` included*: a `tab.get` of a new session: the route resolves (an in-process worker host is + created, connects, loads its facet with 1,000 entries and activates the view), the relayed hold, the 133.7 KiB + snapshot through two links. The IPC row's first hold includes forking a Node process with strip-types (one sample). +- *the throttled tab*: the tab's end of its link delivers at 64 KiB/s (whole messages, in order; idle time banks no + budget) in **virtual time**, 10 ms steps, so 60 s stalls run in well under a second. Per step: one token committed + on the worker (100 tokens/s), the throttle's delivery, then every host's turn. Phases: the first snapshot of the + 500 KB view (8 s of virtual time at 64 KiB/s), 2 s of streaming, the pause (a stall: the tab receives nothing, so + it acknowledges nothing; or the control: the same streaming without a stall), then 10 s more of streaming and + until the tab's text equals the worker's. *Caught up after the stall*: virtual time from the end of the pause to + the first step where the tab's view equals the worker's document; *bytes until then*: what the throttle delivered + meanwhile. *Heap growth during the stall*: the + process's heap after full collections at the end of the pause minus at its start (all hosts share one process; + compare with the control row, which is the same streaming without a stall). *Commit at the worker*: the mean time + of `store.stream` (the transaction, its flush: an encode when the window has room, a push into the pending queues + when it is full), outside the pause and during it. *Longest message handling at the provider*: the longest single + received message handled after the pause at the worker, the provider of the tab's state: its end of the tab link + directly, of the route link relayed (the `ack` that sends the folded `state`). +- *calls*: tab B's sequential `ping`s through the relay, alone and while tab A has issued 10,000 `ping`s at once on + the same session (two holds, two origins at the worker, each with its N = 256 running calls; the relay forwards at + once). + +| Topology (1,000 streamed tokens into the 1,000-item notes list view) | call to frame | bytes per commit: to the tab / from the worker | +| --- | --- | --- | +| memory tab, in-process worker (memory links) | 0.053 ms | 27 B / 29 B | +| memory tab, forked worker (IPC) | 0.097 ms | 27 B / 29 B | +| first hold to live, the fork included (one sample) | 240 ms | - | +| loopback WebSocket tab (`ws` serving, Node's `WebSocket`), forked worker (IPC) | 0.148 ms | 27 B / 29 B | +| no relay: memory tab on the worker (baseline) | 0.034 ms | 27 B / - | + +| Hold to live, the 1,000-item notes list view (133.7 KiB snapshot) | time | +| --- | --- | +| first hold, `resolve` of an in-process worker included (mean of 20) | 1.375 ms | +| hold of a resolved session through the relay (four hops, mean of 100) | 0.763 ms | +| no relay: hold on the worker (two hops, mean of 100, baseline) | 0.703 ms | + +| Throttled tab (64 KiB/s), 100 tokens/s into the 500 KB view | caught up after the pause | bytes to the tab until then | heap growth during the pause | commit at the worker, outside / during the pause | longest message handling at the worker after the pause | +| --- | --- | --- | --- | --- | --- | +| direct, 1 s stall | 0.02 s | 485 B | 28.0 KiB | 0.018 ms / 0.030 ms | 0.169 ms | +| direct, 10 s stall | 0.08 s | 4147 B | 134.2 KiB | 0.029 ms / 0.023 ms | 0.053 ms | +| direct, 60 s stall | 0.38 s | 23.7 KiB | 392.2 KiB | 0.028 ms / 0.017 ms | 0.138 ms | +| direct, 60 s without a stall (control) | 0.01 s | 23 B | 482.7 KiB | 0.025 ms / 0.017 ms | 0.005 ms | +| relayed, 1 s stall | 0.02 s | 485 B | 7.5 KiB | 0.025 ms / 0.025 ms | 0.057 ms | +| relayed, 10 s stall | 0.08 s | 4147 B | -25.3 KiB | 0.030 ms / 0.015 ms | 0.059 ms | +| relayed, 60 s stall | 0.38 s | 23.7 KiB | 513.5 KiB | 0.026 ms / 0.015 ms | 0.146 ms | +| relayed, 60 s without a stall (control) | 0.01 s | 23 B | 257.1 KiB | 0.023 ms / 0.021 ms | 0.003 ms | + +| Calls through the relay (in-process worker, memory links) | p50 | p95 | max | calls | +| --- | --- | --- | --- | --- | +| tab B alone | 0.012 ms | 0.018 ms | 0.600 ms | 1,000 | +| tab B while tab A has 10,000 calls in flight or queued | 4.851 ms | 10.701 ms | 10.708 ms | 27 | +| tab A: 10,000 calls at once, until the last result | 657 ms | - | - | 10,000 | + +Observations: + +- A relay hop costs about 0.019 ms per commit-to-frame on memory links (0.053 against 0.034 ms for call, commit, frame + and result) and adds no bytes to the tab: the server forwards the worker's message with the origin taken off (ids + unchanged, parts copied once); the origin costs two bytes per message on the route link. IPC to a forked worker adds about 0.04 ms, a loopback WebSocket to the tab another 0.05 ms. +- Hold to live through the relay is within 9% of a direct hold (0.763 against 0.703 ms): the relay never parses the + 133.7 KiB snapshot. A first hold that resolves an in-process worker (create, connect, load, activate) takes 1.4 ms; + forking a strip-types Node process takes about 240 ms, which is the app's to hide (start workers ahead). +- Directly, the provider's pending queue folds a stall of any length into one append (24 KiB of text after 60 s, + sent in 0.38 s at 64 KiB/s); committing while the window is full costs no more than committing with room (about + 0.02 ms), and the folded message is encoded in under 0.2 ms. The stall adds no heap beyond the control's, which is + the streamed text itself on the worker and the tab. +- Through the relay a stall now costs what it costs directly (decision "Relay flow control per origin"): the worker + keeps the tab origin's window and folds its commits into its pending queues, the relay forwards and acknowledges + per message. Before this rework the relay ran the tab's window itself and recovered with `reset`: every stall, even + 1 s, cost three full snapshots (about 1.5 MiB) and about 24 s to catch up under sustained streaming. +- Tab B's calls are their own origin at the worker, with their own N, but B's call still waits behind the up to 256 + calls of A running there (one worker, one event loop): about 4.9 ms median, 11 ms p95, against 0.01 ms alone. A's + 10,000 calls take 0.07 ms each (its window of 256 refills one reply round trip through the relay at a time). +- A first run of this script found about 350 B retained per call or commit: `AbortSignal.any` on Node 24.4.1 keeps + cleanup entries per call while its sources live (nodejs/node#62367, fixed in 24.16.0). Calls and gets now link + their signals only until they settle (`withLinkedAbortSignal`); the numbers above are after that fix (heap during a + 60 s stall: 2.3 MiB before, 0.4 MiB after). + +## Slice 4b: authority and housekeeping + +Script: `test/next/bench/compression.ts`, run from `packages/chord` with +`node --conditions=source --experimental-strip-types --expose-gc test/next/bench/compression.ts` (about 5 s). Node +24.4.1, Intel Core i9-13900, one process. + +Payloads: the 500 KB document of slice 3a (500 entries `{ id, text, author, at }` of 960 characters), but with +prose-like text (words from a small vocabulary in a fixed pseudo-random order, some with numbers: it compresses about +3.6x at level 1, as conversation JSON does; the `"x".repeat` texts of the earlier benchmarks would compress about +100x and say nothing), and a 1 MiB string as a call's result (a file export). Compression is option (b) as built: +`nodeCompression` (zlib's raw deflate at level 1) at the origin; the owner inflates with `nodeCompression` +(synchronous, a Node client) or `DecompressionStream` (asynchronous, a browser). + +- *deflate at the origin / inflate*: one part through the adapter alone, mean of 20, the fastest of three loops. +- *end to end*: a server and a client host on a memory link (binary codec); *hold to live* is a further `get` of the + live view (snapshot encoded, deflated, sent, inflated, parsed, replicas built), *call to result* a call returning + the 1 MiB string; *bytes received* is the one message as the client's link delivered it. + +| Payload | JSON | deflated (level 1) | deflate at the origin | inflate, sync (`nodeCompression`) | inflate, async (`DecompressionStream`) | +| --- | --- | --- | --- | --- | --- | +| 500 KB snapshot | 494.9 KiB | 137.4 KiB | 3.308 ms | 0.915 ms | 1.400 ms | +| 1 MiB result | 1024.0 KiB | 286.1 KiB | 6.856 ms | 1.757 ms | 3.113 ms | + +| End to end over a memory link (binary codec) | snapshot: bytes received | hold to live (mean of 20) | 1 MiB result: bytes received | call to result (mean of 20) | +| --- | --- | --- | --- | --- | +| uncompressed | 494.9 KiB | 1.197 ms | 1024.0 KiB | 1.939 ms | +| deflated, inflated synchronously (Node client) | 137.5 KiB | 5.632 ms | 286.1 KiB | 10.868 ms | +| deflated, inflated asynchronously (browser client) | 137.5 KiB | 6.356 ms | 286.1 KiB | 12.198 ms | + +Rerun after the review fixes (inflate only after admission by the part's owner, a tagged adapter result, byte counts +for the threshold): the same within noise (the asynchronous inflate of the 500 KB part varies between 1.4 and 2.0 ms +from run to run). + +Observations: + +- Level 1 cuts prose-like JSON to 28% (494.9 to 137.4 KiB, 1 MiB to 286 KiB) at about 6.7 ms per MiB deflated at the + origin and 1.8 ms per MiB inflated synchronously; `DecompressionStream` takes about twice as long as zlib's + synchronous inflate (3.1 against 1.8 ms per MiB) and adds its own turns, which the in-order holding of the link's + later messages absorbs. +- On a memory link (no transport time) compression only costs: hold to live 1.2 → 5.6 ms, the 1 MiB call 2.0 → + 10.9 ms. It pays where bytes take time: at 10 MB/s a 500 KB snapshot takes about 50 ms uncompressed and 14 ms + compressed (plus 4.3 ms of deflate and inflate); at 1 MB/s (a phone) 500 ms against 140 ms. On IPC to a worker + (gigabytes per second) it does not pay for itself on that hop: it pays on the tab's WebSocket, where the relay + forwards the worker's deflated part as it is (a test counts no inflate at the relay). +- Parts at or below 16 KiB (UTF-8 bytes) are never compressed (a state change, most call arguments), nor is a part + that deflating does not make smaller (sent as the bytes encoded for the attempt, never encoded twice), nor hold + params, nor call arguments sent up a parent link: the cost per message of the slices 3a-4a benchmarks is unchanged. The binary codec's part prefix (length doubled, plus one when deflated) costs + a byte more for parts between 64 and 127 bytes. + +## Slice 6a: the store + +`test/next/bench/store.ts` (run by hand: `node --conditions=source --experimental-strip-types --expose-gc +test/next/bench/store.ts` from `packages/chord`): Node 24.4.1, `chord/sqlite/node` on a fresh SQLite file per burst +for both sides (the same pragmas and file system), 7 bursts of 1,000 commits after a warm-up burst; per commit, the +median and p95 of single commits and the burst's mean. + +The gate (decision "The commit-step benchmark"): the store without outbox no slower than Durable's `SqliteStorage`. + +| Case | Durable | Chord store | Chord store + outbox | +| --- | --- | --- | --- | +| (a) one document delta per commit (step level) | 23.7 / 31.4 / 32.9 µs | 13.4 / 18.7 / 20.9 µs | 16.6 / 29.2 / 26.7 µs | +| (b) one table `put` (Durable: one entry append) | 19.3 / 27.1 / 29.5 µs | 13.2 / 23.6 / 20.6 µs | 17.5 / 28.1 / 26.3 µs | +| (c) whole transactions back to back, no watch (Durable `Session`, Chord host) | 35.0 / 61.1 / 44.4 µs | 18.1 / 27.2 / 25.7 µs | 20.3 / 38.8 / 30.7 µs | +| (c) the same with a watched document | 34.1 / 55.4 / 42.4 µs | 20.6 / 26.3 / 27.1 µs | 20.2 / 39.0 / 29.9 µs | +| (d) one in-memory document per transaction (no store) | | 2.5 / 2.9 / 2.7 µs | | +| (d) the same, watched | | 2.6 / 4.4 / 2.7 µs | | + +Every gate case passes. The outbox adds 3-5 µs per commit (one `INSERT` of the commit's writes as JSON). The heap after +GC at the end of the whole script is about 3 MiB above its start (the modules and statement caches loaded on the way). + +In-memory transactions against HEAD (before slice 6a), after the second fix round (`Transaction.draft` returns its +draft's promise as it is: an in-memory `change` settles in 5 microtask turns, HEAD 6, a test guards it): one `change` +per transaction through a host, 15 bursts of 1,000 per process with a timer around each `change`, HEAD's source +extracted beside this checkout, the same script on both, alternating processes pinned to one CPU (`taskset -c 4`), +three runs; median / p95 / burst mean: + +| Case | HEAD | now | +| --- | --- | --- | +| unwatched document | 2.52-2.55 / 4.96-5.02 / 5.41-5.44 µs | 2.56-2.60 / 5.02-5.06 / 5.69-5.71 µs | +| watched document | 2.96-3.07 / 6.44-6.54 / 6.81-6.90 µs | 2.96-3.01 / 6.35-6.42 / 6.88-6.95 µs | +| watched derive through a reference | 3.14-3.29 / 6.77-6.93 / 8.38-8.50 µs | 3.06-3.08 / 6.49-6.53 / 7.52-7.54 µs | + +What the differences are (the second review's measurements, Opus): + +- The change path itself is at HEAD's level or below. Without the per-operation timer, a tight loop of 200,000 changes + costs 3.67-3.69 µs against HEAD's 3.70-3.78 µs (and the loop of the timed harness 4.85-5.06 µs against + 5.15-5.24 µs). Allocation per change is lower (7,703 B against 8,035 B, measured with a semi-space of 128 MB so no + scavenge lands in a burst), and so are the collections (40 hosts of 2,000 changes: 34 scavenges against 38, 2 + mark-compacts on both). +- The median difference of the timed harness (+1-2% here, up to +3-7% elsewhere) comes from the per-operation timer: + it stays with zero collections in the bursts and disappears without the timer, so it is harness-sensitive (code + layout, inline caches), not work in the change path. +- The burst means differ by where a mark-compact lands: the worst burst per process coincides with a mark-compact over + a larger old space (67 MB against 45 MB before the collection, about 18 MB after on both; a pause of 5.2-7.1 ms + against 3.0-3.7 ms), which is heap-growth timing, not extra garbage. Host setup is about 3% above HEAD (21.9-23.2 µs + against 21.1-21.9 µs per host, create, load and get; close equal), which the burst means include too. + +Calls and references (`taskset -c 4`, five alternating pairs of processes; one host, 21 bursts of 3,000 operations +after 3 warm-up bursts, a timer around each; reads: 200,000 per burst, burst means). Every call admits its handler to +the drain and its cleanup (call count, `ctx.get` references) in one map entry with one reaction: 19 promises per call +against HEAD's 20, a test guards it. A reference's state members are its object's member sources (one per object and +member, re-bound when a new attempt goes live): a watch through a reference gets the member's frame passed on as it is +(one call more per frame), a read goes through one getter more. + +| Case | HEAD | now | +| --- | --- | --- | +| local call: median / p95 / burst median / burst p95 | 1.49-1.51 / 1.93-1.99 / 1.64-1.67 / 2.51-2.75 µs | 1.47-1.48 / 1.93-1.96 / 1.59-1.61 / 2.40-2.47 µs | +| `change` under a watch through a reference, derive member: median / p95; frames per second | 2.80-2.92 / 4.64-4.80 µs; 268,000-282,000 | 2.76-2.87 / 4.75-4.84 µs; 278,000-288,000 | +| the same, document member | 2.68-2.79 / 4.58-4.79 µs; 285,000-306,000 | 2.67-2.76 / 4.57-4.79 µs; 285,000-306,000 | +| `ref.member.value`, derive member | 9.24-10.82 ns | 8.53-8.57 ns | +| `ref.member.value`, document member | 10.02-10.10 ns | 10.30-10.32 ns | + +Calls are 1-3% faster than HEAD (bursts' p95 10% lower), watches through a reference at HEAD's level (p95 +1-2% for a +derive member, more frames per second), a derive member's read faster; a document member's read costs 0.25 ns more +(the member source's getter in front of the slot's). + +A local document member re-exposed by a remote object and held over a memory link (one `change` and its state message +per operation; Astra's third review harness, `taskset -c 4`, alternating pairs, 41 bursts of 10,000 after 3 warm-up +bursts): a member source passes its subscribers (holds, member sources re-exposing it) on to the member it follows, so +their frames take no extra hop. Burst median 7.80-8.38 µs against HEAD's 7.82-8.35 µs, burst p95 8.84-9.44 against +9.52-10.06 µs, median per operation 6.76-7.34 against 6.82-7.29 µs (before the pass-through, one run: 8.51 / 13.05 µs +against 7.99 / 10.36 µs). Host setup (+2-4%) and a document member's read through a reference (+0.25 ns) are accepted +as they are. + +After Delta's tracker stopped allocating a `WeakRef` per change (pi a92283056, merged as 95ec93a08), against HEAD +95ec93a08 (which has that change too) extracted at a path as long as this checkout's (44 characters). Sol's tight +harness without a per-operation timer (`taskset -c 4`, five alternating pairs of processes, 31 bursts of 10,000 after +3 warm-up bursts; medians of the burst medians, ranges in parentheses): + +| Case | HEAD | now | +| --- | --- | --- | +| unwatched `change` | 1.96 µs | 1.96 µs (0.0%) | +| `change` under a watch of the document | 2.24 µs | 2.19 µs (-2.1%) | +| `change` under a watch through a reference | 2.23 µs | 2.21 µs (-0.8%) | +| `change` under a watch of a derive through a reference | 2.33 µs | 2.26 µs (-2.9%) | +| local call | 1.36 µs | 1.33 µs (-1.9%) | +| a document member forwarded into a remote object, held over a memory link | 6.60 µs (6.54-6.63) | 6.52 µs (6.48-6.59) (-1.2%) | +| the same, yielding a macrotask every 1,000 operations | 6.67 µs (6.57-7.02) | 6.54 µs (6.42-6.79) (-1.9%) | + +Collections over one forwarded-link run (`--trace-gc-nvp`, 340,000 operations, two processes each; scavenges and +promotion were the same in both): + +| Run | Promoted | Scavenges | Mark-compacts | GC pauses | +| --- | --- | --- | --- | --- | +| HEAD | 6 MB | 158 | 2 | 154 ms | +| now | 7 MB | 148 | 2 | 147-148 ms | +| HEAD, yielding every 1,000 | 6 MB | 186 | 2-3 | 149-162 ms | +| now, yielding every 1,000 | 7 MB | 178 | 2-3 | 145-149 ms | + +History: the fifth fix round's measurement, before the Delta change (failure calls once per episode, watches on +failed sources, keyed pins still loading). Sol's tight harness without a per-operation timer (`taskset -c 4`, five +alternating pairs of processes, 31 bursts of 10,000 after 3 warm-up bursts; burst medians, HEAD's source at +`/tmp/chord-head4`, this checkout at its own path): + +| Case | HEAD | now | +| --- | --- | --- | +| unwatched `change` | 2.91 µs | 2.95 µs (+1.5%) | +| `change` under a watch of the document | 3.14 µs | 3.10 µs (-1.4%) | +| `change` under a watch through a reference | 3.10 µs | 3.08 µs (-0.4%) | +| `change` under a watch of a derive through a reference | 3.21 µs | 3.26 µs (+1.6%) | +| local call | 1.37 µs | 1.35 µs (-2.0%) | +| a document member forwarded into a remote object, held over a memory link | 7.80 µs | 8.32 µs (+6.6%) | +| the same, `--no-allocation-site-pretenuring` | 8.12 µs | 8.04 µs (-1.0%) | +| the same, yielding a macrotask every 1,000 operations | 7.17 µs | 8.61 µs (+20%) | +| the same, yielding, `--no-allocation-site-pretenuring` | 7.21 µs | 7.52 µs (+4.2%) | + +Allocation per operation, with a fixed semi-space of 256 MB so no scavenge lands in the measured 20,000 operations, is +below HEAD's in every case: 6,942 B against 7,317 B unwatched, 8,383-8,393 B against 8,764 B watched, 8,935-8,939 B +against 9,338 B for a derive, 8,405-8,413 B against 8,764 B through a reference, 2,885 B against 3,089 B over the +link. What stays after the link drains and a GC is equal (18.9 against 19.0 B per operation). + +The forwarded-link gap is not work or retention in 6a's code. It is a V8 decision that depends on heap layout: + +- With huge semi-spaces (no scavenges) the two take the same time (10.01 against 10.00 µs). Without pretenuring (the + flag above) they are equal too. +- An identical copy of this checkout's source runs at HEAD's level (7.1 µs, 7 MB promoted over the run) when its path + is at most 40 characters long (`/tmp/wexp/packages/chord/src`). From 44 characters on it flips: 46 MB promoted, 4 + mark-compacts instead of 2, about 9 µs. The repository's own path is 44 characters. HEAD's source does not flip at + any tested path length (28 to 62 characters). +- `--trace-deopt` shows the flip: V8 marks the allocation site of the delta tracker's per-change overlay context + (`createContext`, inlined into `beginChange` and `Transaction.#begin`) as tenured. From then on every change's context + is allocated in old space and dies there. +- The contexts survive scavenges because each change registers its context as a `WeakRef` target + (`TrackerImpl.#register`). A `WeakRef` keeps its target alive until the current job ends, and scavenges do not + clear it. So a site that allocates one context per change shows a high survival rate. +- Heap snapshots taken during and after the run are equal between the two trees, and no 6a object outlives its + change. +- pi a92283056 ("Delta's tracker holds its one open change without a WeakRef") removed the cause: the numbers above + it show the forwarded link at HEAD's level with 6-7 MB promoted and 2 mark-compacts on both. + +A retirement under a diamond of derives (`test/next/bench/derives.ts`: 40 derives in 13 layers of two derives joined +by a third, 2^13 paths from top to bottom, the bottom watched; one step retires and recreates the document in one +transaction): 9.8 µs median, 17.7 µs p95 per step with one visit per derive per batch; 349 µs without the visited set +(one visit per path). + +Binary `state` messages (`test/next/bench/links.ts`, against HEAD, alternating, pinned): an entry without retirements +costs no extra byte (165 / 27 / 23 B for one appended item, one changed field, one removed item, as at HEAD); encode +and decode are at HEAD's level (one changed field, 200,000 runs, best of 9: encode 1,167-1,203 ns against +1,172-1,275 ns, decode 692 against 685-705 ns); heap per encode and decode 1,592 / 1,631 / 1,363 B against 1,684 / +1,726 / 1,355 B; commit to frame within HEAD's range (0.112-0.166 ms against 0.111-0.164 ms). After the second fix +round (`taskset -c 4`, two runs each): commit to frame 0.086-0.087 / 0.097-0.099 / 0.065 ms against 0.084-0.089 / +0.098-0.099 / 0.064 ms, a streamed append 0.053 ms against 0.051-0.053 ms, the same bytes and heap. + +Other costs (one run each): + +| Cost | Result | +| --- | --- | +| 100 commits submitted at once: time waiting in the queue against the step itself | median wait 956 µs, step 11.3 µs | +| a query page of 100 rows in an index's order (10,000 rows, cursors) | 113 µs per page | +| one `updateWhere` over 10,000 rows | one step of 10.6 ms (every owner's commits wait behind it) | +| `put` with 10-character text cells / 10,000-character cells (2,000 rows, one commit) | 2.4 / 58.4 µs per row | +| a `rows` migration of 10,000 rows (100 chunks; `taskset -c 4`) | 69-71 ms in all, 7.0 µs per row; chunk commits: first 0.34 ms, last 0.21 ms, median 0.22 ms | + +Observations: + +- The commit step is cheaper than Durable's per commit: Chord keeps no generic entry log beside the documents engine + and its tables, and serializes writes before the queue (rule 8). +- Between chunks a migration yields one macrotask without the 1 ms clamp of `setTimeout(0)`: `setImmediate` where the + runtime has it, else a `MessageChannel` message (Workers, browsers). The same 10,000-row migration takes 191 ms with + `setTimeout(0)` (99 clamped yields) and 100 ms with messages in Node (0.27 ms per yield there, against 13 µs for + `setImmediate`). +- Late migration chunks cost what early ones do (paging forward by the last key keeps each chunk's read bounded); the + spec's estimate of about 15 µs per migrated row holds with room. +- A bulk write's step is the one place a single commit holds every owner's commits for milliseconds; the guide says so. diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md new file mode 100644 index 000000000..c87c860ca --- /dev/null +++ b/packages/chord/docs/next/decisions.md @@ -0,0 +1,2168 @@ +# Redesign decisions (input for target design v2) + +Decisions made with the owner after the plugin review round on `target-design.md` (6559589bd), in the order they were +made. Later entries supersede earlier ones; superseded entries are marked inline and point to their replacement. The target design v2 is `packages/chord/docs/next/design.md`, written from this log; new decisions land here +first and win until the design is updated. + +## Plan + +- **p3 next to p2, fully isolated.** The new design is built as a new app (`apps/p3`) with its own data directory. p2 + keeps running unchanged with its data. Plugins are ported to p3 one at a time. +- **No legacy data migration.** p3 starts empty: no import of p2's `catalog.sqlite`, `shares.sqlite` or subagents table. +- **Chord v2 lives in `packages/chord/src/next`** (export `@earendil-works/chord/next`), developed on `private` only + (owner override of the "pi first" rule for this directory) and upstreamed to pi once stable; then `next/` replaces + today's Chord and p2 is deleted. `next/` imports `../context`, `../delta`, `../json`, `../output` directly, so + Durable's `Context`, `Draft` and `JsonValue` are the same modules and types in v2 (no copies, no type conflicts). + (Replaces "Chord v2 lives inside p3".) +- **Build a playground before p3:** a small system with p3's shape (server, worker processes over IPC, clients over + WebSockets) and toy plugins, one per concept; it proves the concepts and reload and is the e2e test ground. + Location `apps/chord-playground` (private). Client: a web page first (a CLI client possibly later). Order: kernel + (hosts, links, objects, references, principals), then plugin loading and reload with the dev loop, then the web shell + with a private preview, then each feature built live as a plugin. Plugin changes reload; Chord changes restart the + hosts (fine for the owner). +- **Chord v2 is written fresh** in `packages/chord/src/next`, copying what fits from today's Chord (link, wire, cell, + delta and the like). It needs the same test coverage as today's Chord (tests under `packages/chord/test/next`). +- **Playground web stack:** React, built with esbuild (loader D) instead of Vite, if feasible. +- **Reload control in the playground's web shell:** when sources changed, a control pops up listing the changed + plugins (reload them) or base code (restart all hosts); the owner triggers it manually. The agent can trigger the + same actions (command or tool). Never automatic on save. +- **Preview:** `https://p3.mariozechner.at` on slayer (wildcard DNS already resolves), set up like + `~/p2.mariozechner.at` (Caddy container on `caddy-network`, unix-socket bridge, `run.sh` in tmux) with `basic_auth`; + the password is chosen by the owner and never stored in the repo. `p2.mariozechner.at` stays the live p2. + +## Natives + +- **Natives only in objects.** An object opens a native in its synchronous factory with + `self.open(async ({ signal }) => value, close)`; Chord opens it during activation (the object goes live after all + opens) and closes it when the object ends, in reverse order. The value never changes during the object's life: no + `self.resource`, `version`, `replace`, `deps`, reopen, stale handles or "read `.value` at use". +- **The factory principle:** synchronous, declares only (references, documents, natives, derives, routines, jobs, + contributions); Chord performs activation; the factory receives handles usable in everything it registers; handles + are valid for the object's whole life. +- **No natives across plugins:** other plugins reach a native only through the owning object's methods and state, or + through the owner's registry entries ("Registries and HTTP"). No callbacks as method arguments, also not on + `local` types ("Object kinds"). +- **The facet is a pure object.** (**superseded in part** by "Facet documents are removed" and "Facets only declare" (2026-10-26): no documents, no routines) It may hold documents, references, routines and contributions, and may not open + natives (`self.open` does not exist on the facet). Served objects of a facet may read and write the facet's documents; + a served object's transaction may span its own documents and its facet's documents (lock order: own line, then the + facet's). The facet may not reach into its served objects' documents. + +## API style + +- **Tagged unions, always,** instead of a status string plus fields that only mean something in some states ("field + soup"). Each state carries exactly its own fields, and consumers narrow on `kind`. Example: `ref.state` is + `{ kind: "failed"; cause; retryAt } | { kind: "live" } | ...`, not `status` plus `error` plus `retryAt`. + +- **Context last (owner, 2026-10-21):** parameters are the required arguments, then the context (`ctx` or a Durable + `Context`), then optional arguments, as Durable already does (`Storage.scanEntries(query, limit, cursor, context)`, + `transaction(change, context)`). Examples: `self.tx(fn, ctx)`, `handle.change(fn, ctx)` and keyed + `handle.change(key, fn, ctx)`, `handle.retire(ctx)` and keyed `handle.retire(key, ctx)`, `chord(fn, context)`, + `job.start(args, ctx, { key })`. Applied to design.md, the guide and the walkthrough; the frozen `step-*.md` and + `kernel.md` document the frozen code and keep its signatures. pi's `traceIn(context, options, fn)` (implemented in + `chord/context`) becomes `traceIn(options, fn, context)` in the same pi change as S1 (owner, 2026-10-21). + +## Hosts and placement + +- **Placement is a function** of the object's typed input, not data naming an input field (`hostFamily(prefix, + key)` looked a key up in the input untyped). A function is type-checked and composable. The caller runs it and + sends the result; the provider runs it too and refuses an object that is not its own. Placement is then no longer + part of the contract fingerprint; a changed rule surfaces as that refusal. +- **Incarnations are minted by Chord** (uuidv7, the code copied from pi-ai into Chord: no dependency), never passed + in; the link handshake carries it, so the other side sees a restart as a new incarnation of the same identity. + +- **The low-level host API** (no topology layer; Radius and DO show entry points vary, so the kernel has no + "listen"): + - host kinds are typed: `hostKind("server")`, `hostKind("session", Type.Object({ id: Type.String() }))`; an address + is `Session({ id: "S1" })`; placement functions return addresses; no host-name strings in app code; + - `createHost

({ address?, principal? })`: a host with an address is addressable and acts as `principal` on its + own host; no options is a client (not addressable, no own principal); the incarnation is minted inside; + - (**superseded in part** by "Host values are removed" (Review round 1): start data is a core object) `host.provide(Key, value)` before `load`; factories read host values with `self.value(Key)`; + - `host.load(facets)` takes imported facets (loaders are adapters); + - `host.connect(link, { principal })`: the link to the parent (at most one), `principal` = who the parent is; + today's duplicate `upstream` option is gone; + - `host.accept(link, { principal })`: a link from a client; every entry point (loopback/tunnel WebSocket server, + Radius channels, a DO edge, memory) is app or adapter code that authenticates and calls `accept`; + - `host.route(Kind, { authorize(principal, address), resolve(address, { signal }) => { link, principal } })`: + how to reach hosts of a kind; `authorize` runs before anything starts; Chord calls `resolve` once per address + while its link lives and again after a drop; the returned principal is stamped on that link (defined once, in + the parent; the app sends it to the child in its own start message, with any init data); + - starting processes, init data and keep-alive are app code (helpers later if apps repeat them). + Close to today's shape (`createHost`, `connect`, `accept`, `route`, `load`); gone: string host names and prefix + parsing, app-managed generations, `upstream`, `opaque` runtime objects, the loader inside the kernel. + +## Quick decisions on the design's open points (design.md section 26) + +- **Call-scoped natives (#4):** no `ctx.open`; a native that lives for one call is opened and closed in the method with + `try/finally` (documented pattern). Revisit only if it shows up often. +- **Documents across plugins (#6):** no plugin reads or writes another plugin's documents (Chord or Durable). Data + crosses plugins only through state and methods. ~~Within a plugin, the facet's documents are readable and writable + by the objects that facet serves (decided earlier).~~ (**superseded** by "Facet documents are removed") Data core enforces lives in a format core owns and is handed to + core through its API when the data is created (e.g. `createConversation` with `init`), never written into core's + documents directly. +- **`overlap` for routines (#8):** no `overlap: "latest"` option; a handler compares with the current value itself. +- **Async activation guard (#3):** none; authority checks live in the per-principal view's methods (and in + `route.authorize` before a host starts). +- **`close` with a reason (#7):** `self.open(open, (value, reason) => …)` with `reason: { kind: "reload" } | { kind: + "stop" } | { kind: "end" }`, decided by Chord (its plugin is in the reload set and the object is native / the host is + shutting down / the last reference ended or its type was removed). Owners decide what to do with it. Chord aborts + only the object's own calls and jobs; work elsewhere that uses the native (e.g. Durable tool executions using a code + mode sandbox) is the owner's to drain or end in `close` ("old code keeps running" applies to natives too). +- **Code mode in p3 drains its sandbox in `close`** (**superseded in part** by R16 (`close` detaches the sandbox and disposes of it in the background)) (not moved into Durable): the plugin keeps owning the + `pi-codemode` sandbox; on `reload` (and `stop`) `close` seals it (no new scripts), running scripts finish or hit + their deadline, then its VMs and worker threads end (`closeWhenIdle` in `pi-codemode`, or a counter in the plugin). + New scripts start on the new sandbox through the registry entry. `RunningCalls`, the removal message and + `abortTask` on reload go away; after a worker stop Durable replays as today. +- **Invariants across owners (#5):** the owner of the dependent data enforces the rule by reconciling from the other + owner's state with an idempotent routine; no cross-owner transactions, no "notify the other owner" calls that can + be lost. Example: ports' forward store watches the catalog and closes forwards of archived sessions (also after a + crash, from the current value); sessions does not know ports exists. + +## Wire (#15) + +- ~~**Compression:** large JSON envelopes are deflated on WebSocket links (relayed and mobile links benefit); byte + segments stay raw and uncompressed; IPC never compresses.~~ **Superseded** by R1 and "Compression at the origin". +- **Frame size limit per link adapter** (**superseded in part** by Q2 (limits per part from the header; `maxPayload` only as the hard cap)) (e.g. 16 MiB on client WebSockets, none on IPC): a frame over the limit fails + that call or member with `too_large`, the link stays up. Large uploads go through owner chunking anyway. +- **One batch per commit per link:** a commit that touches several holds on one link travels as one `state` frame; + the consumer applies every member update of it before listeners run. Remote consumers never see a torn commit, + matching local subscribers, who are notified once after every document of a commit is adopted. +- **Adapted bindings get diffs:** a consumer bound through `accepts` (an older contract version) is served like a + derive of the provider's state through the adapter: Chord keeps the last adapted value per binding, applies the + adapter to the new value and sends the diff (`diffRevisions`), never the whole adapted value per commit. + +## Gaps closed after the walkthrough rework + +- **Clients** serve only `local` types (not addressable); local calls have no principal; what a client calls on the + server is stamped with the client's authenticated principal. +- **Placement refusal** is `invalid/placement`; during a rollout it is handled like a mismatch (provider on another + revision: park; same revision: fail). +- **HTTP route signature:** `(req, ctx) => Promise`; `ctx.principal` is the request's principal (from + the request resolvers), so `ctx.get(Notes, …)` gives that principal's view. +- **Start barrier** (**superseded in part** by R17 (the harness starts after the barrier, which waits until every plugin loaded or failed)) counts facets and `lifetime: "host"` objects; an object that must contribute at start (the model + runtime) is `lifetime: "host"`. +- **`persist` only in `local` objects and `lifetime: "host"` objects** (**superseded** by P3: only in input-less `local` + objects) (identities without a principal); `storeKey` + is dropped (two principals' objects writing one document would be two lines on one document). Enforced at the type + level (`self.state` in a remote factory takes only non-persisted definitions) and as a declaration error at runtime + ("persist needs a stable identity: put the documents into a local core"). +- **Call deadlines:** a call ends through the caller's signal; there is no per-call deadline of its own. Parked calls + (link drop, loading, rollout) share one default deadline (60 s, like `readyTimeout`), then fail `unavailable`. +- ~~Lingering through `ref.release({ linger: true })`~~ **superseded** by D8 (no consumer-side linger). Was: a closing view releases with `ref.release({ linger: true })`; the hold stays, marked lingering, until + its timeout. A link drop is still a departure (presence details stay open). +- **Contributor order** (**superseded in part** by "Contributor order is sorted by plugin name (never load order)" in the design, section 17) is load order (alphabetical by plugin name), deterministic on every host and across reloads, + and carries no meaning: wraps in one phase must not depend on each other; meaningful order comes only from phases. +- **Revocation without field patterns:** the app defines a typed revocation type and a function + `createHost({ …, revokedBy: (p: P) => R[] })` (the same app code on every host); `host.revoke(r)` ends every + object and closes every link whose principal `p` has `r` in `revokedBy(p)`, and propagates `r` (a small JSON value) + to the children, which apply their own `revokedBy`. No `PrincipalPattern`, no local-only predicates, no re-stamping: + a changed grant (mode `write` → `read`) is a revocation of the old one; the client reconnects and authenticates + anew, and its references re-hold. +- **Per-session native owners:** (**superseded in part** by "Host values are removed": the session is read from the core object `SessionInfo`) the facet cannot `self.get` a type it serves, so a worker-wide native owner is a + `lifetime: "host"` object without input that reads its session from a host value (`self.value(SessionInit)`). + +## Documents step + +- **Plugin identity:** every plugin has a `package.json` whose `name` is its globally unique, stable plugin name (like + an npm package name); that name identifies its facets (~~persisted facet documents,~~ reload, diagnostics; **superseded in part** by "Facet documents are removed"). Loader D + reads it; static composition passes the same name. Never a directory name, never load order. +- **Document roots are `JsonObject`,** as in Durable (`defineDoc`); lists live in a field. Documents + hold JSON only. +- **In-memory documents end with their object** (last reference released, linger expired); they survive reloads and + native restarts of a live object. What must outlive the object is persisted or lives in a `lifetime: "host"` object. +- **Chord ships its own document storages, no adapter over Durable's storage:** (**superseded** by "The store" + (2026-10-21): Chord's persisted documents live in the per-session store, next to Durable's adapter and plugin tables) in-memory and SQLite (JSONL was cut later) + implementations of `DocumentStorage`, inspired by Durable's storage implementations (atomic commits, base plus + deltas, incarnations) but with Chord's own schema: ordered keys per owner and definition, a tag index. Durable's + storage is shaped for Durable (scans by id, no tags); Chord documents and Durable transcripts never commit together, + so they need not share a storage. Node-specific storages live outside the portable core; a Durable Object SQLite + storage can follow. (Replaces the earlier "adapter over Durable's node SQLite storage" for the playground.) +- **Durable's SQLite driver layer moves into Chord** (`@earendil-works/chord/sqlite`, `/sqlite/node`, + `/sqlite/cloudflare`): the `SqliteDatabase`/`SqliteExecutor` interfaces, the `node:sqlite` and Durable Object + drivers, the migration runner and schema-prefix helpers. Durable imports them from Chord (it already depends on + Chord, so no cycle) and re-exports them under its old paths so no Durable user breaks. Chord's SQLite document + storage uses the same drivers (`next/` imports `../sqlite` like `../output`). Done in pi first, then merged. + +## Derives and natives step + +- **`self.open` returns a handle** whose `.value` is the native (throws before activation); no forwarding proxy + (private fields and identity break through proxies). The value never changes during the object's life. +- **A partly failed activation** closes the natives already open (reason `end`) and opens all of them again on the + next attempt. +- **A broken part fails the whole object** (owner): a derive that throws, or a document whose hydration or `migrate` + fails, makes the object `failed`; an incomplete object could break invariants that rely on atomic commits across its + documents. (Replaces "member-local" errors for documents.) +- **`previous` of derives** is tested through failed/retry until the reload step exists. +- **`self.retryOn` may appear anywhere in the factory:** the factory specifies, the host orders and executes (watches + are installed before natives open). +- **Failure causes are tagged:** `{ kind: "transient" }` (an `open` throws, a reference is unreachable) gets the + automatic backoff; `{ kind: "broken" }` (factory, derive, document hydration or `migrate` throws; or an `open` that + throws `broken(…)` because retrying is pointless, e.g. invalid config) gets no automatic retries, only reload, + `ref.retry()` or `retryOn`. Chord decides by where it failed; only an `open` can choose `broken` itself. +- **Activation waits for its dependencies:** an object whose references are not live yet stays `loading` and goes live + as soon as they are (bounded by `readyTimeout`, then it fails `transient` with the dependency as cause); ~~a `broken` + dependency fails it at once. No retained references while failed.~~ **Superseded** by R18 (dependency failures are + `transient`, references are kept as an automatic retry trigger). +- **Timeouts** are host defaults with per-thing overrides: `closeTimeout` 30 s (per native), call drain on teardown + = `closeTimeout` (per type), `readyTimeout` 60 s (per `get`), park deadline 60 s (per type), routine handover 10 s + (per routine), job `finish` retry backoff 1 s to 60 s (per job), rollout install 60 s, linger per type; route + `resolve` is bounded by the app's signal. +- **Inbox path:** the notes plugin computes its own inbox path from generic start data (the data directory; host + values were removed later, start data is the core object `SessionInfo`). + Natives open sequentially in declaration order and close in reverse; a throwing `close` does not stop the others + (`host.close` rejects with all errors, background cleanup reports to `onError`). Type names `Native`, + `NativeCloseReason`. +- **`retryOn` sources** must live independently of the attempt (facet documents, registry values; host values were + removed later; + readables the host provides); a reference's member is not allowed (type error). Dependency waiting covers references. +- ~~**A facet that fails transiently in activation** retries automatically~~ (**superseded** by "Facets only declare": a facet cannot fail after `load`) with backoff, like an object; it does not wait + for a reload. +- **Park deadline per consumer:** `get(..., { parkTimeout })` overrides the host default for that reference; the + provider's per-type override applies to its own local bindings. +- **Playground:** the inbox watcher only opens and closes for now (the import comes with routines); a note-count derive + and the reader's redaction as a derive are visible to clients. + +## Telemetry + +- **Context:** (**superseded in part** by "Telemetry types and review cleanup": `Context.telemetry` is always a + `TelemetrySpan` and `withTelemetry` takes spans) Chord's `Context` gets `telemetry: TelemetryContext` as a fixed member, like `abortSignal` (never + undefined; `NOOP_TELEMETRY_CONTEXT` by default), with `withTelemetry(telemetry, context)` and + `traceIn(context, options, fn)` (child span, `fn` gets the derived context). `value()` and keys stay, forever, without + deprecation. Durable gets telemetry everywhere through the `Context` it already passes; principal and other app + values stay out of Durable's context (Durable work runs in the scheduler's contexts and resumes without a caller; + origin travels as data). In Chord v2, `Ctx` extends `Context`: `ctx.telemetry` is the current span, `ctx.trace(name, + attributes, fn)` gives `fn` a child `ctx` (principal and `get` kept), so calls inside are parented correctly. +- **One adapter per host:** (**superseded in part** by the adapter is a `TelemetryContext` with readonly `metrics`/`propagation` members, held only by hosts ("Telemetry types and review cleanup")) `createHost({ telemetry })`, the adapter implements `TelemetryContext` and optionally + `TelemetryPropagation` and `TelemetryMetrics` (no split). Default NOOP. +- **pi-telemetry additions** (pi first): `TelemetryCarrier = Record` (opaque, W3C-compatible); + `TelemetryPropagation { inject(context) → carrier | undefined; extract(carrier) → TelemetryContext }` (absent, + invalid or unknown carriers give the adapter's root, never throw; carriers are best effort: after an adapter change + only links are lost); span links (`SpanOptions.links`, required `TelemetrySpan.addLink(link)`, a link targets a + context); `NOOP_TELEMETRY_PROPAGATION`; the in-memory adapter implements propagation, links and metrics (carrier + snapshots, no forced W3C fields); conformance fixture with optional `propagation`. No `isRecording` for now (it would + break adapters, as would closure attributes). +- **Metrics** (in the same pi change): `TelemetryMetrics { counter, upDownCounter, histogram }` with `add`/`record` + (synchronous, cheap, never throw; histogram `record` takes an optional context for exemplars), `InstrumentOptions` + (description, unit), histogram boundaries, `NOOP_TELEMETRY_METRICS`, typed through `metrics` in telemetry schemas, + low-cardinality attributes only. +- **Across links:** (**superseded in part** by R5 (the carrier is a bounded field of the header schema)) `call` and `hold` frames carry the caller's carrier; the provider extracts it and runs the method + under it; the server adds a `chord.relay` span. Cancellation already travels as `cancel` frames. +- **Shared work** (an activation awaited by several getters, a worker start, the per-object line) is its own root span + with a link to its first trigger; every waiting span links to it. Work without a running trigger (routine runs, job + runs, eviction, backoff retries, background closes) is a root span, linked to its trigger where one exists. Chord + sets all of these; authors may add links through `ctx.telemetry.addLink`. +- **Tiers:** spans for calls, methods, relays, holds, activations, opens, closes, loads, revocations, resolves, routine + runs, job runs and finishes; events on the enclosing span for commits inside a method (`chord.commit` with docs, ops, + duration), retries and parking; metrics for background high-frequency work (state frames, bytes on WebSocket links, + commit durations, in-flight calls, active holds, open links, park durations, backpressure waits). Verbose per-commit + or per-frame spans are an adapter decision (filter by name); Chord, Durable and pi-ai only report. Sampling, + buffering and limits belong to the adapter. +- **No measuring for telemetry:** only sizes the code already has are reported (encoded WebSocket frames, `byteLength` + of bytes, lengths a storage computed while writing); nothing is serialized just to count it. +- **A Chord telemetry schema now** (typed span, event and metric names; used by tests and debugging). +- **Debugging:** (**superseded in part** by the SQLite telemetry store and `TelemetryStore` ("Telemetry store")) a JSONL trace adapter (bounded rotating file, batched asynchronous writes, drops when full, never + blocks) and a text renderer of one trace now; a web trace viewer in the playground (span tree, durations, hosts, + links, attributes, events; also usable standalone on a JSONL file) with the web shell. +- **Durable** (pi): (**superseded in part** by "Durable and AI spans" and "Telemetry implementation decisions" (phase spans under the stored carrier; every input submission starts its own trace)) `HarnessOptions.telemetry`; an optional `trace` carrier field on submission and task records (the + submitting call's or the parent task's carrier, rewritten with the record as other immutable fields are; JSON records, + no migration, no format bump, old records simply have no link); task runs are child spans while their trigger runs in + the same process, otherwise root spans linked to the stored carrier; Durable spans get their own schema. Persisted + Chord jobs store their start carrier the same way. +- **Own schema, no OTel GenAI (owner, 2026-10-18):** pi's AI and Durable telemetry uses its own typed schema + (`pi.ai.*`, `pi.durable.*`); the OTel GenAI conventions (development status, renames, streaming still `TODO`) are not + followed. An OTel adapter may translate pi's names to `gen_ai.*` at export. Model stream details are span events, never + per delta. The pi part of telemetry (propagation, links, metrics, typed metric schemas, `chord/context` telemetry + member, `HarnessOptions.telemetry` type) is approved as diffed in `/home/badlogic/pi-telemetry`. +- **Durable and AI spans (owner, 2026-10-18), partly open:** (**superseded in part** by pi-ai's option is `telemetry` (a span), not `telemetryContext` ("Telemetry types and review cleanup")) every span is short and in one process. Spans: + `pi.durable.submit` (admission), `pi.durable.phase` (one invocation of any task's phase; fixed name, `task.name`, + `phase`, `kind`, `attempt`, `resumed`, outcome as attributes), `pi.ai.request` (one model request). Events: + `pi.ai.first_chunk`, `pi.ai.block` (per `*_end`: type, index, chars, tool; never per delta), `pi.durable.entry`, + `pi.durable.submission`, `pi.durable.replay` (a codemode call answered from its journal). No content ever; entry ids + point to the transcript. Submissions and tasks store their creator's carrier; phase spans are children of it. Every + input submission starts its own trace, linked to its trigger (bounded traces; subagent report chains do not nest). + No hook spans for now. Metrics: `pi.ai.request.duration`, `pi.ai.first_chunk.duration`, `pi.ai.tokens`, `pi.ai.cost`, + `pi.durable.phase.duration`, `pi.durable.tasks.active` (ids never on metrics). `pi.ai.request` is emitted by pi-ai + in one wrapper around `lazyStream` in `models.ts` (both `stream` and `streamSimple`; reads the event stream, changes + nothing; deferred results get `pi.ai.poll` the same way); Durable passes `telemetryContext`. Background tasks + (e.g. compaction) are children of the creating phase like every task. A task picked up by a new process sets + `resumed=true` and stays under its stored carrier (same trace). +- **Telemetry implementation decisions (owner, 2026-10-18):** `TelemetryContext` carries optional `metrics` and + `propagation` members (every span from an adapter carries its capabilities), so one type is used everywhere + (`HarnessOptions.telemetry`, `createHost({ telemetry })`, `Context.telemetry`, pi-ai's option, renamed `telemetry`). + Entry and submission events only after a successful commit. Several queued inputs started as one run: the run takes + the first input's carrier and stores the others' as immutable creation-time links on its first phase span; a steer's + submit span links to the running run's carrier. A phase closed or cancelled without a committed state ends + `aborted`. Submit spans only on the public submit path; internal admissions inside another commit show as events. + ToolTask sets `tool.name`/`tool.call_id` on its own span. Write submissions are children of the caller; only input + submissions start traces. Polls count tokens and cost only for final results. Import budgets raised (pi-ai models + 18, Durable root 71). +- **Errors in telemetry (owner, 2026-10-18):** a failed span carries `status: "error"` with a fixed category name and the + error message truncated to 500 characters (`provider_error` for a model request, `task_faulted` for a faulted phase). + Cancelling is not an error (`stop=aborted`/`outcome=aborted` keep status ok). Durable's `onReport` stays unchanged; + every report is additionally a `pi.durable.report` event on the current span (its own short root span outside any + span) plus a `pi.durable.reports { type }` counter. (Superseded in part below: no root span for reports.) +- **Telemetry types and review cleanup (owner, 2026-10-18):** `Context.telemetry` is always a `TelemetrySpan` (default: + the NOOP span); the adapter (`TelemetryContext` with readonly `metrics`/`propagation`) is held only by hosts + (`createHost`, `HarnessOptions`) to start roots, never put into a Context; pi-ai's option is `telemetry?: + TelemetrySpan`. Code that only has a Context calls the generic `addEvent` (no typed-events helper, no casts, no + duck typing). A report is an event on the current span plus the counter and `onReport`; outside any span only the + latter two. `TaskOptions.trace?: TelemetryCarrier | "root"` (absent: inherit). No protection against broken adapters. + Telemetry never walks or reads data just to measure (checkpoint event compares the phase only). Instrument caches are + `WeakMap`s keyed by the adapter's metrics object. +- **Telemetry store (owner, 2026-10-18):** a SQLite adapter in Chord (`@earendil-works/chord/telemetry-sqlite`, reusing + `chord/sqlite`, Node and Cloudflare): settled spans one row each (events and links inline), metrics aggregated per + series per 10 s, a 1 s batched flush that also deletes rows older than the retention (7 days), a bounded queue of + settled spans (dropped and counted), no retries. A shared read interface `TelemetryStore` in pi-telemetry, modeled on + Jaeger's span reader: `findTraces({ name, attributes, errors, start window, limit, cursor })` returns complete traces + (newest trace start first, paged by an opaque cursor), `getTrace(id)`, `findMetrics({ name, window, attributes })`. + Spans carry global ids and a trace id (a root's own id), links carry trace and span id. Both the SQLite and the + in-memory adapter implement it; one renderer `renderTrace(spans)` (subpath `pi-telemetry/render`) prints links and + never follows them. Live spans, incoming-link search and duration search are left out. The store's metric read method is + `findMetrics` (`metrics` is the recording capability). Adapters trust typed inputs (no validation of malformed + caller values, no try/catch around recording); only carriers from another process are checked (keys present, + strings), else the root. + +## Review round 1 (Astra, Fable): kernel fixes and decisions + +Source: `p3-reviews/v2-review-summary.md` (K = kernel fixes, D = decisions). + +- **Variants (D11):** a member name has the same kind (state or method) in every variant that has it, and a method has + the same signature (arguments and result) in every variant; state schemas may differ per variant (redaction). + Enforced at the type level in `remote(...)` and at runtime for contracts arriving as data. K7 becomes that check. +- **Registry composition is a runtime state, not a staging check (D1; reverses "compose runs in staging").** + Contributions can be dynamic (only known once a native is open, or changing at runtime), so no conflict check can be + complete in staging. `compose(table)` is pure and runs whenever the table changes (after a commit, when a contributor + goes live or ends, when a dynamic contribution changes; contributions may be readables). The composed value is a + plain value for readers: `compose(table, { report })` always delivers a value, builds the fallback in (e.g. the + first by plugin name wins) and reports conflicts through `report` (they become owner notices and telemetry events); + readers never handle conflicts. Only if `compose` throws is the registry in an error state, and readers get an error + when they use it (like a failed reference). Staging rollback stays for code errors only (load errors, throwing factories). A contribution + is in the table exactly while its contributor is live (a facet while its plugin is loaded; an object from `live` + until it ends, fails or restarts). The HTTP router follows the same rule: a duplicate route is a conflict state, not + a rollback. +- **Code mode reload vs. a pinned Durable phase (D2):** (**superseded in part** by R16 (the old sandbox is detached and disposed of in the background, not drained in `close`)) the code mode tool entry looks up the current sandbox when + `execute()` starts (not the one it was created with), so a call pinned by a phase before a reload (e.g. waiting in + an approval hook) starts on the new sandbox; the old sandbox is still sealed and drained in `close` (running scripts + finish or hit their deadline, then it ends). +- **Natives that follow a relocation (D3):** documented pattern, no new mechanism: the native is a stable manager + (`self.open` once); a routine on the session location switches its internal resource (e.g. the watched directory) + and bumps a generation, so late events from the old location are dropped. Used for watchers, prompt resources and + OS probes in p3. +- **Jobs keep their owner alive (D4):** (**superseded in part** by R11 (revocation ends the objects whatever their jobs)) a job that is running, finishing or blocked in `finish` counts as a reference + to its object; the object ends only once all its jobs are settled. Starting a job and releasing the reference (an + HTTP route returning the job id, a tab closing) no longer aborts it. Keep-alive already counts running jobs. +- **Reloading native objects (D5):** stays as decided: native objects of a reloaded plugin restart. In practice this + costs little: the Durable harness is core (base code, restarts only with the host), terminal PTYs live in the + separate terminal host process (the server's native is only the connection), model runtimes, sandboxes and the + search index rebuild in milliseconds; only working on envs (running remote commands are killed) or radius (relayed + connections drop) hurts, and only on reloads of exactly that plugin. Keeping natives by name was rejected: natives + have no line, no transactions and no schema, so old and new code would interleave unchecked. +- **Old handles stay valid until their calls settle:** when a pure object is swapped in place, a call that started on + the old code (and its transaction) keeps working with the old handles until it finishes; only new calls get the new + handles. Matches "running calls finish on old code". +- **Core vs. plugin is decided by where the code lives:** `src/` (base code, imported statically by the start code + under a fixed name such as `@p3/core`, not switchable, a change restarts the host) or a plugin directory (found by + listing, loaded through loader D, named by its `package.json`, switchable, a change reloads it). Core is what the app + always needs and everything builds on (harness, catalog, access, session location); optional features are plugins + (code mode stays a plugin). +- **Host values are removed** (no `hostValue`, `provide`, `self.value`, no host-value versions). Start data (session + id, init, data directory) is a `local` core object with `lifetime: "host"` and constant state, served by the core + facet from the start message; platform bindings the host owns (a Durable Object's `state`/`env`) are core objects + whose `self.open` returns the binding and whose `close` does nothing. Plugins read start data through a reference, + in openers, derives, routines and methods (after activation); nothing reads it synchronously in a factory. A + long-lived native from the host is a `lifetime: "host"` object in a core facet. +- **`accepts` adapters map values (D6):** (**superseded in part** by R6 (provider-owned old schemas plus mappings)) per member, `state: { member: (newValue) => oldValue }` (pure; Chord wraps it + as a derive and diffs per binding) and `methods: { name: (oldArgs) => newArgs }` (plus an optional result mapping + back); members an old consumer does not know do not appear. Authors never build readables for adapters. +- **Revocation and work in cores (D7):** no revocation hook. A core that runs work on behalf of a principal stores the + grant (or device) as data with the work and reconciles from the grant owner's state (the sharing core's live grants): + a routine aborts jobs, withdraws questions and drops queued input whose grant no longer exists. The same pattern as + invariants across owners; it also holds after a crash. +- **Lingering and keep-alive (D8, decides open #1):** (**superseded in part** by R26 (the counts are one readable, `host.activity`)) the consumer-side linger is removed (`release({ linger })`, the + lingering mark on holds, the consumer linger cache): a closed tab closes its link, so it could never survive the + case it was made for. What remains is provider-side `serve(…, { linger })` (an object stays alive for a while after + its last reference, whoever held it; this is what keeps a just-closed conversation warm). Presence is the live + references (`host.references`). Worker keep-alive works as in p2 (the worker counts, the app's supervisor decides): + core code on the worker reports viewers (from `host.references`), active calls and running jobs (Chord's counters) + and Durable tasks (the core `Harness`), plus provider-side lingering objects; the server's supervisor stops the + worker when all are zero for the idle time (p2: 5 minutes, checked every second). Chord provides the counts, no idle + logic. +- **Validation cost (D9):** (**superseded in part** by R9 (no dev walks: one behavior everywhere) and R7 (arguments validated for every caller, inputs normalized)) document values are not walked against their schema in production. Instead each + document record stores the fingerprint of the schema it was written with (computed once per definition from the + structural schema, ignoring descriptions and titles; written at creation and with the new base after a `migrate`). + On access Chord compares the token's fingerprint with the record's: same version but a different fingerprint fails + the object `broken` ("schema changed without a version bump"). Always on, O(1). Every structural schema change needs + a version bump (an additive change gets a trivial `migrate`). Derives and readables are not validated in + production; in dev, values may be walked as a bug finder (documents at commit, state at publication). Inputs from + peers (method arguments, hold inputs, with the provider's schemas) are always validated. +- **Storage failure (D10):** Chord never ends the process. A storage error ends the store and closes the host, and + `onError` receives a tagged error (`{ kind: "storage_failed", … }`); the app's start code must handle it (log and + exit with its own code for a supervisor, or reopen). The design states that every app has to; the playground shows + how (`process.exit` with a distinct code). Fixes the server zombie (a closed host with a live listener). +- **Reading without starting a worker (D14):** (**superseded in part** by R20 (Chord stamps the caller's incarnation as `ctx.incarnation`)) plain pub/sub in an ordinary plugin, no Chord mechanism: the plugin's + worker facet records the summary locally and publishes it to the plugin's server object (a method call, from a + routine on the source); clients read the server object, which never starts a worker. The server object keeps the + latest value per session (a document if it must survive a server restart) and ignores reports from an older + worker incarnation. Session list, status, children, unread, artifact lists follow this. +- **The K list is confirmed** with these changes: + - K9: (**superseded in part** by R14 (`checkpointWhen` is required on every definition)) like Durable: storages do not validate deltas against content (the store already holds and validated the + authoritative value); a commit writes the delta; replay only when loading; checkpoints stay optional per + document, no default. + - K16: (**superseded in part** by "Flow control: an ack-clocked window" (call window N, not credits; no byte window)) no count limits that refuse; call credits as backpressure (throttle only, never an error); the byte window + applies to every frame including first snapshots; holds unlimited (protected by authorization and input + validation). + - K17: dropped (an IPC peer producing deep JSON shows up at once in development). + - K21: (**superseded in part** by "Only `tx.draft`" (`tx.get` is gone) and R12 (drafts are revoked; reads after the commit)) values are immutable by convention (Chord Delta's rule, as in Durable), documented for `tx.get`, + `handle.value`, `handle.initial()`; no copy, no freeze; the `keys()` index is fixed; the write on a read-only + transaction after a version bump is documented. + +## The state path: minimal compute and memory (after the cost review) + +Source: `p3-reviews/v2-cost-model.md` (before/after per hop, measured by Fable and Astra). Replaces the per-change +validation and copies the v2 kernel added around Delta. + +- **The model, end to end, as Delta and `replicatedState` (which Durable already uses):** (**superseded in part** by R1 and Q4 (parts encoded once at the origin per hold; relays rewrite headers; one binary format)) values are immutable + revisions with structural sharing; commits produce exact ops; subscribers on the same host get the same value and + ops by reference; derives publish through `diffRevisions` (identity short-circuit; bytes are atomic leaves; + equality never by `canonicalJson`); each frame is encoded once per link and sized from its encoded bytes; relays + read only the frame header and forward the body opaquely; receivers check a frame once by its header (`JSON.parse` + already yields strict JSON; bytes recognized in O(1)); consumers apply with `applyImmutable`. No validation of + state, no clone of snapshots or ops, no repeated walks. +- **A v2 readable is `replicatedState`:** today's `chord/src/state.ts` semantics (immutable value, ordered ops, + gapless attach with snapshot and cursor) is the one readable implementation, imported by `next/` like `../delta`, + so Durable's views and document watches are state members without conversion. +- **The store is a port of Durable's store design** (`durable/src/session/transaction.ts`, `session.ts`): one Delta + tracker per document, prepare and adopt by pointer swap, observers get the commit's ops, storages write the delta + (or a base when `checkpointWhen` says so) and check only metadata; replay only on load. +- **Only `tx.draft`:** it returns a draft of the current document, or a draft over `initial()` if the document is + absent; an unchanged draft (empty ops) commits nothing, so an absent document is never stored by being read. + `tx.get` is removed. +- **`derive` stays as it is,** documented with its footgun (a derive that builds new objects for unchanged data costs + O(value) per commit and per view) and a cookbook of efficient derives for p2's categories: select a field, look up + by key, small aggregates (just compute); filter/map a list (an identity cache per source element); redaction + (preferably separate documents per visibility, otherwise the identity cache); joins (identity cache plus the + looked-up source's revision); a window over a large list (reuse unchanged parts, as p2's chat window). No `project`, + no view algebra, no `memo` API. +- **Diffing and derived values come from Delta and today's Chord, not new code:** Delta's `diffRevisions` (tested, + identity short-circuit) gets bytes as atomic leaves (compared by identity, replaced when different), and v2 calls + it directly; `next/diff.ts` is removed. Derives reuse today's derived cells (`chord/src/cell.ts`) on top of the + readable. Delta gets a small, pure, tested helper `mapShared(f)` (an identity cache per source element: a new object + per changed element, the previous result for unchanged ones), used by derives that build a new object per source + element; the cookbook documents it alongside the patterns. +- **Same-host calls on `remote` types pass values by reference** (**superseded in part** by R9 (no freezing anywhere, no development mode)) (immutable by convention, as `local`); the dev freeze + catches mutation. (Replaces "validate and copy arguments and results also on the same host".) +- **Arguments and hold inputs are validated at the provider by the caller's trust:** (**superseded in part** by R7 (arguments validated for every caller; inputs always normalized)) whenever the originating + principal is not a host principal (guests, owners, any client, also when relayed over trusted IPC), once, with the + provider's schema; calls from server and worker code are not validated in production. Results are not validated in + production. Provider schemas are compiled lazily at their first use and cached in a `WeakMap` keyed by the schema + object; old tokens and their compiled schemas are collected by the GC after a reload. +- **Contract descriptions travel once per (link, fingerprint):** (**superseded in part** by R6 (no descriptions on the wire; strict fingerprints)) a hold sends the fingerprint; the provider asks for + the description once and caches the parsed token per link; the cache dies with the link. +- **Cost guards:** (**superseded in part** by R9 (the counters are ordinary telemetry metrics, always on; no test mode) and T7 (one encode per hold part at its origin, none at relays)) counting tests in the normal test run (in test mode Chord counts full-value walks, deep copies, + encodes, schema validations and materializations; per hop a test asserts the counts, e.g. one appended note to a + 500 KB document with 10 viewers: no walks, no copies, one encode per link, no validations) and a `vitest bench` + suite outside CI (500 KB and 1 MB values, 0/1/10 viewers, memory/SQLite, 1 MiB bytes) with results recorded + in `packages/chord/docs/next/benchmarks.md`. +- **Naming:** the pattern of a `local` object holding shared data behind per-principal views is the **shared store** + (not "core"); base code and plugins use exactly the same primitives. + +## Cuts and closed deferred points (review round 1) + +- **Cut:** the cross-relay reset coordination (`#collectReset`, `#resyncs`, `#resetRequests`): on an overflow a relay + releases the hold and re-holds, and the provider sends a fresh snapshot. The JSONL document storage (Chord ships + memory and SQLite). `MemoryDocumentStorage.prepareCommit`/`snapshot` (unused after K9). +- **Kept:** (**superseded in part** by R9 (no dev freeze)) the dev freeze (the safety net for passing values by reference). The Questions service stays a documented + pattern, not built. **Deferred:** the HTTP and channel components, middleware phases and the Durable Object profile, + until the kernel fixes and a first real proof (harness, one tool with an approval pause, a code mode reload) are + done; the playground's web shell serves its pages with plain app code until then. +- **Presence (#10):** a link drop is a departure (accurate and simple); smoothing (a phone switching networks) is the + UI's job, e.g. a debounced viewer count. Counts per person or origin are derived by the app from + `host.references` (principal, origin). Workers do not stop on a flicker because of provider-side linger (D8). +- **Optional dependencies (#2):** a rule, no mechanism: an optional dependency is a registry owned by the dependent + (ports contributes its forwards to `radius.forwards`; without ports the table is empty). No `optional` on `get`. +- **Rollout fixes (#12), built with the rollout step:** (a) a host checks its own install status before waiting on a + provider (a rejected own install fails at once with the reason); (b) installs are serialized per host; (c) a + revision also carries the code-set digest (the hash over the facet bundles' content hashes that loader D computes + anyway; no module analysis), and hosts compare the digest on reconnect, so a server restart with unchanged code + triggers no reinstall. +- **UI reload vs. leaving (#13):** "leaving" is never derived from an unmount, only from explicit navigation or + presence (a dropped reference); a component unmounted by a reload has not left. + +## Process (after review round 2) + +- **Layer by layer, as pi-durable was built:** the existing `next/` code (about 10k lines, kernel steps 1–6) is frozen as + a reference and not built on; Chord v2 is rebuilt bottom-up in small layers (data and contracts; readable and derive; + documents and store; objects; references and `ctx`; protocol and links; relay, routes, revocation; routines and + jobs; registries; reload and rollout; telemetry). Per layer: a one-page spec with data flow, cost per hop, a line + budget, footguns and what is reused, approved by the owner; a minimal build with counting and GC tests; a cleanup + loop with fresh reviewers and the owner until he is happy; only then the next layer. Target: 6–7k lines without + tests. The handoff for continuing after a compaction is `apps/p2/docs/p3-handoff.md`. + +## Review round 2: decided items + +- **R1, the message protocol and relays:** (**superseded in part** by Q4 (one binary message format on every link), R6 (no `describe` kind) and compression at the origin) a message is a header plus opaque parts, one part per hold. A part's payload + is encoded once by its origin with that hold's own Delta encoder (path interning per hold, end to end between + provider and consumer), packed with its byte segments, compressed at the origin above a threshold (`codec`). WebSocket: + `u32 header length | header JSON | part bytes`; IPC: structured clone of `{ header, parts: Uint8Array[] }`; memory: + the object. Kinds: `hold`, `release`, `call` (1 part: arguments), `cancel` (consumer to provider); `bound` (1 part: + snapshot), `state` (n parts: one per hold the commit touched on this link), `result` (1 part), `gone`, `describe` + (provider to consumer). Relays parse and rewrite only headers: map hold and call ids per link, stamp the principal on + `hold` and `call` (a principal field from a client link is always overwritten; only a trusted parent link may carry + one), group parts per destination link (each link gets only its own holds' parts); payloads are never parsed. One + `state` message per commit per link. Encoding happens when sending: while a link has credits a commit is encoded and + sent at once; under backpressure its ops are collected per hold and encoded into one part when credits return (still + atomic for the consumer). Flow control counts the parts' sizes from the header. A re-hold sends a fresh snapshot and + restarts that hold's encoder and decoder. Two holds of the same principal (two tabs) encode the same ops separately + (O(change) each). Replaces "relays forward bodies opaquely" from "The state path". + +- **R2, frame size limits (owner, 2026-10-19):** (**superseded in part** by Q2 (limits per part; `maxPayload` only as the hard cap)) configurable per link kind: the compressed (wire) size through the + WebSocket server's `maxPayload`, the decoded size through zlib's `maxOutputLength` in the one decode function, + wherever decoding happens (relays do not decode). Defaults for client WebSockets 16 MiB compressed, 64 MiB decoded; + none on IPC. An oversized message fails only its own call (`too_large`); the link stays up. +- **R4 and R5, protocol header schema (owner, 2026-10-19):** (**superseded in part** by Q1 (unknown ids are ignored; only a header failing the schema closes the link)) the message header is a TypeBox schema, a tagged union + over `kind` (`hold`, `release`, `call`, `cancel`, `bound`, `state`, `result`, `gone`); protocol types derive from + it, and headers from untrusted links are checked with the schema compiled once (trusted links such as IPC are not + checked, as for arguments). No kind allow-list: every message resolves against the receiving link's own tables; a + header that fails the schema, or an unknown kind, hold or call id, is a protocol error that closes that link. The + trace carrier is an optional bounded field (key count, string lengths) of `call` and `hold` in that schema; relays + read it from the header. Payloads stay opaque. +- **R7, inputs and arguments (owner, 2026-10-19):** (**superseded in part** by "Params instead of inputs": flat required + params, no normalization) object inputs are always normalized (defaults, unknown fields + removed, a canonical identity key), whoever the caller is, so equal inputs give one identity; inputs stay arbitrary + (nested objects, arrays, records allowed). (**Superseded in part** by Q12: closed schemas, unknown fields rejected.) + Method arguments are always stripped to the provider's schema and + validated with it (compiled once), whoever the caller is: trusted principals are not type-checked code (plugins are + bundled by esbuild without type checks, agents write JavaScript plugins). Bytes are leaves: `instanceof Uint8Array` + plus an optional length bound, never walked. Results and state are not validated (they come from the provider). + Replaces "arguments validated only for callers that are not host principals". +- **R8, migration output (owner, 2026-10-19; `initial()` part **refined** by "Simplification pass after 4a" D3: once per definition):** the result of `migrate` and of `initial()` is validated against the new + schema once, when produced (per document per version change, on load, off the commit path); a failure refuses the + access, keeps the old data and fails the object. Commits are still never validated. +- **R9, no development mode (owner, 2026-10-19):** one behavior everywhere; the host's `dev` option is removed. Values + are immutable by convention and never frozen (deep freezing was too costly in Delta), and state is never validated + (schema changes are caught by fingerprints, migration results by R8). The cost-guard counters (walks, copies, + encodes, validations, materializations) are ordinary telemetry metrics, always on; tests assert them through the + in-memory adapter. Replaces "values immutable by convention, with a dev freeze". +- **R10, revocation window (owner, 2026-10-19):** accepted. Work started on behalf of a revoked principal stops when + the reconciling routine runs; a step started in between still runs. No extra checks before tool calls or approvals; + there is always some window. +- **R11, jobs and revocation (owner, 2026-10-19):** "jobs keep their owner alive" stays for the normal end (the last + reference released: a closed tab does not abort a running job). Revocation takes precedence: revoking a principal ends + its objects at once, whatever references or jobs remain; their jobs get the abort signal and the normal + `closeTimeout`. A job runs as its object's principal. +- **R12, returning data from transactions (owner, 2026-10-19):** option B, as in Durable: drafts are revoked when the + transaction settles; transactions write and return at most plain values; data is read after the commit from the + document (`handle.value`, `handle.get`). No `tx.read`. +- **R13, existence inside transactions (owner, 2026-10-19):** as in Durable, no existence check: `tx.retire` of an + absent document is a no-op, `tx.draft(…, { tags })` creates a missing document with its tags, existence is read + outside the transaction. The R12 and R13 patterns are documented in `design.md` (section 9). +- **R14, checkpoints (owner, 2026-10-19; **refined** by "Slice plan and slice-1 review decisions": required only for persisted documents):** no default, but `checkpointWhen` is required on every document + definition, in Chord and in Durable (a pi breaking change), so forgetting it is a type error. Durable's task-scoped + documents (nested call, codemode run and journal) return false (they retire with their task); the codemode store + checkpoints after 64 deltas; p2's small settings documents checkpoint on every write, its growing ones after 64. +- **R15, old calls and migration (owner, 2026-10-19):** as in Durable: running calls keep their old handles until they + settle, except for documents whose version rose; old code touching a document migrated past its version throws, the + document stays intact, the call fails once. +- **R16, native restarts (owner, 2026-10-19):** no Chord change: a restart closes the old native, then opens the new. + Code mode's `close` detaches the old sandbox, returns at once and disposes of it when its scripts finish or after + the default `closeTimeout` (30 s, no override); its tool entry finds the current sandbox at `execute()`. Documented + as two patterns for natives: drain in the background, and swap through a registry contribution. +- **R17, harness start (owner, 2026-10-19):** (**superseded in part** by "The harness is an ordinary object") the harness is not part of the start barrier; it starts after it (the + barrier waits until every plugin loaded or failed, so the registries hold everything that will be there). No list of + required contributions and no paused state: a missing tool gives its call an error result; a missing model makes the + generation report "model unavailable" and wait until a model appears; failed plugins are reported with their error. +- **Note for p3, moving a session between instances (owner, 2026-10-19):** copying a session's database to an instance + with other plugins already works as Durable does: missing tools, sections and hooks are skipped; tasks without their + definition stay blocked (`missing_task`) until installed or aborted; documents of missing plugins stay untouched; + stored selections are never rewritten. The UI must show blocked tasks with their reason and offer to abort them (a + turn waiting on such a task hangs until aborted). +- **R18, dependency recovery (owner, 2026-10-19):** a dependency that fails (or does not become live in time) fails + the dependent `transient` with the dependency's cause inside, never `broken`. The dependent keeps its references + while failed; their state is an automatic retry trigger, so the dependent is retried the moment the dependency goes + live (backoff cancelled; it stays as a safety net). Invisible to factory authors. After `live`, a dependency leaving + `live` leaves the dependent live; its calls using the dependency get its error or park while it loads. +- **R19, flow control and large snapshots (owner, 2026-10-19):** (**superseded** by "Flow control: an ack-clocked window") snapshots are always admitted, even when larger + than the window (they consume it); overflow is decided on changes queued after them; after an overflow a re-hold + waits until the credits cover the last snapshot's size, with a minimum interval. +- **R20, incarnation fence (owner, 2026-10-19):** Chord stamps the calling host's incarnation (from the link handshake) + on calls arriving over a link: `ctx.incarnation` next to `ctx.principal`, not part of the principal or its key; a + server object ignores reports from an older worker incarnation by comparing it. Nothing is sent as data. +- **Testing the core (owner, 2026-10-19):** every mechanism decided in review round 2 gets exhaustive, individual tests + with its edge cases; every layer spec lists them (design section 28). +- **R21, who contributes to registries (owner, 2026-10-19):** only facets and `lifetime: "host"` objects, enforced by + the type of the factory's `self` (only theirs has `contribute`). Reading registries stays open to every object. +- **R23 (owner, 2026-10-19):** `constant(value)`, a readable that never changes, used for start data. +- **R22 (owner, 2026-10-19):** a host-lifetime object without input gets session-scoped objects with `ctx.get` in a + routine or method (documented pattern); a `get` with an input computed after activation only if needed. +- **R24 (owner, 2026-10-19):** the container-width cost of structural sharing is documented (one array slot copy and + one element visit per element per change); a benchmark of long arrays sets the limit; large collections are keyed + documents or bounded windows. No new mechanism. +- **R25 (owner, 2026-10-19):** rejected are registries created at runtime and entries resolved through mutable + indirection; a contributor may change its contribution at any time (a readable contribution recomposes on change), + and readers may look up the current value at use. +- **Changing contributions, `self.writable` (owner, 2026-10-19):** a contribution is fixed for its contributor's life + or a readable; Chord subscribes to readable contributions and reruns `compose` on every change; use sites follow the + composed readable or read its current value. `self.writable(initial)` is a host-local readable of any in-process + value (identity-compared, no JSON, no wire) with `set`, for contributions such as MCP tool lists. Tools, sections, + hooks, wraps and tasks reach the harness as one Durable `Extension` per plugin; skills and other lists likewise. +- **Removing a contribution (owner, 2026-10-19):** no remove call; an entry leaves when its contributor ends or a reload + stops contributing; a live contributor with nothing to offer sets its readable to the registry's empty value (no + `undefined` entries). +- **R26, drift (2026-10-19):** fixed in the text (the description frame is gone with R6; §9 no longer says values read + in a transaction are shared). Keep-alive counters: one readable `host.activity` (`{ references, calls, jobs, lingering }`; owner). +- **Q1, unknown ids (owner, 2026-10-19; review round 3):** unknown hold and call ids are ignored on both sides (races + such as `result` crossing `cancel` or `call` crossing `gone`); a consumer receiving `gone(h)` rejects its own pending + calls on `h`. Only a header that fails the schema closes the link. Replaces "an unknown hold or call id closes the + link" (R4). +- **Q2, size limits per part (owner, 2026-10-19):** limits per part from the header sizes, before decoding; an oversized + part fails only its hold or call, other parts apply; an oversized snapshot fails its hold without automatic re-hold; + the admitting link's decoded limit is stamped on relayed work and enforced where it is decoded; `maxPayload` is only + the transport's hard cap (closes the connection). Replaces R2's per-message wording. +- **Q3, re-hold after overflow (owner, 2026-10-19):** (**superseded** by "Flow control: an ack-clocked window") re-hold once the previous snapshot's debt has drained (credits at + the full window), with the minimum interval; replaces "once the credits cover the last snapshot's size" (R19), + which stalls for snapshots larger than the window. +- **Q4, one message format and window (owner, 2026-10-19):** (**superseded in part** by "Wire protocol" (codecs, not the JSON header framing) and "Flow control: an ack-clocked window" (no byte window)) every link carries the same binary message + (`u32 header length | header JSON | part bytes`, one `Uint8Array`): WebSocket as a binary frame, IPC through + `serialization: "advanced"`, memory delivered asynchronously. Encoding, framing, checks, ids, flow control and limits + are one shared protocol layer; adapters only move bytes; one byte window everywhere from `byteLength`. Replaces the + IPC frame-count window and R1's per-adapter formats ("IPC: structured clone of { header, parts }; memory: the + object"). Tests on memory links run the production path. +- **Q5, dependency cycles (owner, 2026-10-19):** rejected before activation waits: on one host by following the + waiting edges of activating objects; across hosts through the activation chain of uuidv7 instance ids carried in the + header of holds made during an activation (bounded; ignored from untrusted links). A cycle fails the activation + `broken` naming it; no references are kept; checked again after reloads. +- **Q6, Chord from Durable tools (owner, 2026-10-19):** (**superseded in part** by "Context last": `chord(fn, context)`) p3's core puts the host into the harness's base context under + a typed key; `chord(context, fn)` opens a call scope (host principal, the call's abort and span, references released + when `fn` settles) in tools, hooks and sections; no wrapping, no p3 `defineTool`. Needs good docs: the pattern, + lifetime and principal, and the Questions service as the example (tools reach users only through owner state; + clients are not addressable). Replaces "hooks open a call scope with `host.root`" (plugins never hold the host). +- **Q7, code mode on reload (owner, 2026-10-19):** R16 stands: the detached old sandbox is disposed when its scripts + finish or after 30 s at most; a reload during a longer script aborts it and that turn fails. After a reload at most + 30 s of old code keeps running. +- **Q8, registry ownership and versions (owner, 2026-10-19):** one authoritative declaration per registry, from the + defining facet (`compose`, version, `accepts`); other tokens only identify it; a reload replaces it atomically. + `accepts` stays (third-party plugins are an explicit goal of p3), owned by the defining facet, pure, following the + owner-adapter pattern of documents, object types and jobs; a contribution without a matching version or adapter is + left out and reported, not a load error; contributions are not schema-checked. Every use site of a contribution + catches throws and rejections, reports naming the contributor, and continues. Replaces "newer ones are refused at + load". +- **Q9, job dedupe and retention (owner, 2026-10-19):** dedupe covers unsettled runs only; after `finish` commits, + Chord drops the run's record and key (a persisted record in the same commit) and releases waiters; dedupe after + completion is the owner's, through the result in its own document. Chord keeps nothing beyond a run's lifetime. +- **Q10, contract evolution (owner, 2026-10-19/20):** forward evolution in code, no build step: a base written in + full with an explicit version (`remote()`/`local()`; `local` keeps its own kind with flat `members`, C1 rejected), + `evolve(prev, e => …)` always continues the history with structured edits (`state`, `addState`, `removeState`, + `args`, `result`, `method(name, args, result)`, `addMethod`, `removeMethod`); every version is materialized and + fingerprinted at declaration. Input or variant-set changes start a new history (a new declaration with an explicit + version; no older accepts). Mappings only with the provider, in the pure `checkAccepts(contract, accepts)` (no + `serve` in this layer): required at type level and at runtime for every member of the old version that changed or + was removed; removed state from the whole current variant state or `"refuse"`. Fingerprint: `hash64` over a + schema-aware data-only normal form (`NORMAL_FORM_VERSION`; union and intersection order kept; `required`/`enum` + sorted; TypeBox keyword whitelist; bytes as a `{ type: "bytes" }` data marker); history private, deep-copied and + frozen (**superseded in part** by "L1 rewrite review": nothing is frozen). Schema helpers `str num int bool nul lit oneOf union obj arr map opt bytes unknown` + `type Static`. + Implemented and reviewed (Fable, Astra: thumbs up) in `packages/chord/src/next/contracts/` (`7d2bb8ea8`). +- **Q11, absent singletons (owner, 2026-10-21):** a singleton always has a value: `handle.value` is the written value + or `initial()`, never `undefined` (an exposed singleton is typed `T`); `undefined` only for absent keyed documents + (`handle.get(key)`); singleton existence has no API. A deliberate difference from Durable's `snapshot()`. +- **Facets typed by the app's principal (owner, 2026-10-21):** base code exports `defineFacet` bound to its `P` + (`bindFacet

()`, working name) next to the principal union; every facet imports it from there; `host.load` + accepts only facets typed for the host's `P`. No runtime check. +- **`placement`, not `host` (owner, 2026-10-21):** (**superseded in part** by "Params instead of inputs") the contract field naming where an object lives is `placement` + (a typed function of the input); `input` stays (the object's key among objects of its type, part of its identity). +- **Q-B2, in-memory documents across input changes (owner, 2026-10-21):** not carried over. An input change of a + type ends its live objects on reload; their in-memory documents, flows and jobs are lost; documented ("in-memory + documents survive a reload only while the object identity is unchanged; persist what must survive"). +- **The store (owner, 2026-10-21; backends **superseded in part** by W14: no IndexedDB; direction accepted, details under review in `p3-reviews/store-proposal.md`):** + P1 recorded: a session is self-contained (its store plus its directory); export from the session alone, import + into another instance, sync to Radius and continue elsewhere; the server only mirrors (catalog) and keeps instance + data. One transactional table store per session, layered backend (SQLite, memory, IndexedDB) / store (tables, a + small query DSL, controlled writes, commit queue, seq, outbox, export/replay) / adapters: Durable's `Storage` as a + new `StoreStorage` adapter (Durable's existing storages stay unchanged), Chord's persisted documents, and plugin + tables (`self.table`), each table owned by one object like a document. Writes are only `put`/`delete` + (`deleteWhere` resolved to keys), recorded as row operations `{ table, key, v, row }` in the same backend + transaction as the rows, so a replica or an import needs no plugin code; indexes are local (**superseded** by "Index + definitions are recorded"); migrations run only + where the owning code runs and ship as ordinary puts. Lines stay per Chord object and per Durable Session; the + store has one short commit queue per store that assigns the seq; Durable's seq is the store's seq (Durable sees + gaps; it only compares seqs). Rejected on the way: derived-only plugin data, opaque plugin directories, export + hooks, plugin record logs, per-store outboxes with a global position counter, a SQLite-changeset or SQL-statement + store, raw SQL reads, an IndexedDB-only API, Chord documents inside Durable's `Storage`. Open: physical layout + (generic rows or real tables), DSL scope, migration timing, `next_id`, backend details, the ordering review + ("seq 40 after seq 50"). +- **S1, adoption inside the commit step (owner, 2026-10-21; store review):** every owner's in-memory adoption runs + synchronously inside the store's commit step, after the backend commit and before the next commit starts, so memory + always shows a complete prefix of the log. Durable's `Storage.commit(writes, adopt, context)` gains a required `adopt` + callback (pi breaking change): every storage calls it synchronously once the commit is durable and before its next + commit; the Session catches its own adoption error and fails; a rejection after adoption fails the Session without + `tx.discard()`; `writeAtClose` passes a no-op; `StoreStorage` forwards `adopt` into the store's step; Chord objects + adopt (pointer swap) the same way. Listeners stay where they are: after adoption, in each owner's continuation + (unchanged), so their order across owners may differ from the seq order while every listener reads a complete + prefix. Checked against Durable's code (memory, JSONL, SQLite, GuardedStorage, writeAtClose); the pi change and its + test run come with the store work. +- **S2, references to later commits (owner, 2026-10-21; store review):** no mechanism and no new rule. Ids are minted + inside the transaction that creates them (Durable `tx.appendEntry` → `mintId`) and reach other code only after that + commit, so a reference to a later commit only arises when code leaks data out of an uncommitted transaction, which + is already wrong today (the transaction may still fail). The guide states it once: ids and data leave a transaction + only after it commits; reference what is committed. +- **S3, commit completion (owner, 2026-10-21; store review; IndexedDB parts **superseded** by W14):** the commit step finishes only on the backend's terminal + success (SQLite: `COMMIT` returned; IndexedDB: the transaction's `complete` event); only then adoption (S1) and the + next commit run. The memory backend stages a commit and swaps it in at the end. An ambiguous outcome (an error during + `COMMIT`) stops the store (`storage_failed`, D10), as Durable's `GuardedStorage` does. After a restart a local head + below the uploaded head is normal: the driver pulls its own uploaded batches back before writing. SQLite keeps + `synchronous=NORMAL` (no extra cost: a crash loses nothing, a power loss the last commits, recovered from Radius); + IndexedDB uses `durability: "strict"` where supported. Condition (owner): SQLite must not get slower. +- **S4, consistent reads (owner, 2026-10-21; store review):** an export pauses commits (holds the commit queue) while + it reads, so the archive is the state at one head; streaming from a snapshot comes only if exports get large. Reads + that need several queries (a document from its base and revisions) run in one backend read transaction, as + Durable's SQLite storage does. Ordinary reads in object code (separate awaits) are not snapshot reads; the guide + says so; no global snapshot guarantee. +- **S5, replica apply (owner, 2026-10-21; store review; a sync-layer rule):** a replica applies a whole batch in one + backend transaction together with its applied head. Uploads may run in parallel (owner); the driver resubmits a + batch until Radius acknowledges it and trims a batch from its outbox only after its acknowledgement. Every batch + carries `from` and `to`; a replica applies a batch only when its `from` equals the replica's applied head, so it + never applies past a gap: a later batch waits until the missing one has arrived. +- **S6, compaction (owner, 2026-10-21; store review):** (**superseded** by W9: no compaction at first) covered by S5: a compacted batch is applied as a whole, so + its intermediate states are never visible; takeover happens at batch boundaries. Nothing to add. +- **S7, losing the driver role (owner, 2026-10-21; store review):** no rollback (tools ran, files changed, natives and + derived data followed the tail). When an upload is refused for an old epoch, the local copy becomes a separate + session with a new id and keeps running; commits in flight land in it; the old id's sync stops there; the new session + may sync on its own from a full snapshot; the two never sync again. The catalog shows both (the split-off copy marked + with its origin and seq). No `not_driver` error. To work out with sync: changing the id (store metadata, session + directory, catalog), links from other sessions keep the original id. +- **S9, S12, S14, S17 (owner, 2026-10-21; store review):** S9: the store persists its head with every commit; the + prototype's `Sequencer` and safe cut go away (prototype bug, nothing to decide). S12: the query DSL starts without + `join`, `groupBy` and `select`; plugins aggregate in JS over indexed scans; added when a real case needs them. S14: + tables and documents of an object are written in one transaction API, `self.tx(fn, ctx)` (no separate + `table.transaction`). S17: bulk operations are designed now, together with migrations, after the remaining S items + (full-row puts make a 10k-row rewrite about 40 MB in the outbox; revision pruning is one record per row). +- **S8, work that finishes out of order (owner, 2026-10-21; store review):** an app concern, no store or job mechanism. + §14 states that the handler calls of one routine never overlap; the guide shows two patterns: chain through the + routine (await the run's outcome, skip values that are no longer current) or drop stale results in `finish` by + comparing the source position inside the destination's transaction. +- **S10, ids (owner, 2026-10-21; store review):** no persisted reservations. Durable's `mintId` contract is narrowed to + "greater than every id minted since this storage was opened, and every id stored, also across reopen" (an id from a + transaction that never committed was never seen outside it, S2; Durable's transactions are serialized on the + Session line). In the store each owner's id counter starts at open above that owner's highest stored id (one index + lookup per table; a replica taking over gets it the same way). Durable's existing storages already behave so; the + comment changes in the same pi change as S1. +- **S11, copying documents into a fork (owner, 2026-10-21; store review):** `StoreStorage` reconstructs the source + documents of a `document.copy` before entering the commit step (reads of committed history outside the step) and + enters it with plain creates carrying the base; Durable is unchanged. Safe because the source is a past state and + only Durable's Session line writes Durable's tables. The outbox only ever carries the resulting rows. Forks of forks + and selecting an entry in the parent work as today (rewindable documents keep their history). +- **S13, one documents engine (owner, 2026-10-21; store review):** the store contains one documents engine used by + Durable's `StoreStorage` and directly by Chord (Chord keeps no Durable dependency). Address `(namespace, owner, kind, + key?)` with an opaque `owner` string chosen by the adapter (Durable: `session`, `conversation:7`, `task:31`; Chord: + host/object type); records per incarnation with `history: "latest" | "full"`, `createdAt`, `retiredAt`, opaque + `meta`; revisions (base with version and optional fingerprint, deltas). Reads outside the step: `find(address, at, + context)`, `load(id, at, context)` (reconstructs base plus deltas once; no cache), `list(query, limit, cursor, + context)` ordered by id or key. Writes inside the step: `create(record, base)`, `change(id, content)`, `retire(id)`, + with the engine's checks (one live per address, create before change, delta on a base of the same version, nothing + after retire); `latest` prunes revisions on a new base and on retire. Callers decide base or delta + (`checkpointWhen`) and keep their in-memory trackers as today (Durable's Session caches, pins and evicts unchanged). + Fork copies stay in `StoreStorage` (S11). Chord's frozen storage layer and its conformance suite are not ported. +- **S15, Durable's existing storages (owner, 2026-10-21; store review):** kept, as decided; no plan to remove them. + Revisit whenever it becomes worth it. +- **S16, layout and text order (owner, 2026-10-21; store review, Astra's design; **superseded in part** by W8 (every text + column), W14 (no IndexedDB) and "Text is plain UTF-8, in code point order" (the whole text order and encoding)):** real tables per logical table + (definitions and changes recorded in the outbox, the store renders DDL); keys and indexed columns of type text or + integer, the rest of a row in a JSON column (**superseded in part** by "Flat columns"). Text order is JavaScript's (UTF-16 code units) on every backend: memory + and IndexedDB use plain strings natively; SQLite stores key and indexed text columns as `BLOB`s of UTF-16 code + units, high byte first (`Buffer.from(s, "utf16le").swap16()`), so its bytewise comparison equals JS order; text that + is not indexed stays `TEXT`. Lossless (lone surrogates, NUL round-trip), no comparator, no normalization ("é" and + "e\u0301" are two keys). The SQLite backend encodes writes and query bounds and decodes results; outbox records, + exports and cursors keep logical strings. Prefix ranges come from one store helper (increment the last code unit + below `FFFF`). Every index ends with the primary key; cursors are opaque `{ dir, last }` with logical values and are + refused by a scan in the other direction; a row whose indexed field is missing or null is not in that index. Plugin + rule: text compares exactly like JavaScript strings; page with the store's cursors. Verified on Node 24.4: lossless + round trip, SQLite BLOB order equals JS sort, ~0.4 µs per encode plus decode. Cost: indexed text shows as hex in + `sqlite3` and takes two bytes per code unit. Rejected: generic JSON rows, store-maintained index entries, code point + order with `toWellFormed()` (Fable), WTF-8, text equality-only. +- **S17, bulk operations and migrations (owner, 2026-10-21; `p3-reviews/store-bulk-migrations.md`; JSON-part wording + **superseded in part** by "Flat columns"):** writes are `put` + (a whole row), `update(table, key, set)` (some columns of one row), `delete`, `updateWhere` and `deleteWhere` (many + rows, one record). Bulk operations are recorded as operations with the driver's affected-row `count`; every replica + runs the same operation (one SQL statement, or a loop in the memory backend) without plugin code; a different count + stops applying that store and restores it from a snapshot. `set` expressions touch real columns only: `value`, + `col`, `add`, `sub`, `max`, `min`, `concat`, `replacePrefix`, `coalesce`, `when`; no JSON-field operands (added + when a real case needs them); no subqueries, joins, aggregates or insert-from-query (read first, then `inArray` or + puts); no case folding, `substr` on unindexed text or floats; one conformance suite for both backends. Migrations: + shape changes of the JSON part migrate on access (each row stores its version `v`; `migrate(row, from)` on read; the + next write stores the new shape; old code refuses newer rows); changes to key or indexed columns run eagerly when + the owning object starts on the driver, before it goes live: recorded table changes (`alter` add/rename/drop, a + rebuild for key or type changes), then either an `update` step (one `updateWhere`) or a `rows` step (scan in key + order, `tx.update` per row, chunks of N rows per commit; table version `migrating` → `ready`; a crash resumes the + scan). Indexes are local and created after `ready` (**superseded in part** by "Index definitions are recorded": the + index changes after `ready` are recorded). +- **Blobs and derived data (owner, 2026-10-21):** blobs (artifact versions, uploads, sent files) are app layer: files in + the session directory named by the hash of their content, never changed, written (tmp, fsync, rename) before the row + or document that names them; any process may write one. Export copies `blobs/` with the store snapshot; import + copies them back and checks each hash. Sync: before uploading a batch, the sync layer uploads every blob file present + at that moment that has not been uploaded yet (no reference tracking; a replica never sees a hash without its file). + No blob is deleted before its session is deleted (an unreferenced blob only costs disk). Derived data (full-text + indexes, caches, a plugin's search database) is never exported or synced and is rebuilt where needed, typically by a + catch-up routine comparing it with the store. +- **Index definitions are recorded (owner, 2026-10-21):** index contents are never recorded (every backend maintains + them from the rows), but their definitions are: when the owning object starts on the driver, the store compares the + declared indexes with the existing ones and creates or drops the difference, recorded as `{ op: "index", table, name, + columns }` and `{ op: "dropIndex", table, name }` (after a migration's `ready`); every replica applies them. Every + copy of a store, an export and the file seen by outside tools is then identical and self-describing. Indexes never + affect results (queries order by index columns plus the primary key; rows without the field are excluded by rule). + Cost: replicas without the plugin maintain indexes they never use; adding an index builds it during apply. +- **P3, where persisted data lives (owner, 2026-10-21):** persisted documents and tables only in input-less `local` + objects (one per host and type); their address is (host, object type, name, key), with no input, so an input change + can never orphan persisted data. Per-entity data (per conversation, per session) is keyed documents or table rows in + such a store with an app-chosen key (data, not identity); objects with an input keep only in-memory state and read + persisted data from the input-less store by key. Not adopted (proposals of 2026-10-21): `previous`/`rehome`, the + split question, identity hashes as storage addresses. A persisted store's type and definition names stay stable; + renaming one, or changing how keys are formed, is an explicit one-off migration in the store. p2 fits: its four + stores with an input persist no Chord state. Enforced at the type level (`self.state(…, persist)` and `self.table` + only in input-less local factories) and as a declaration error at runtime. +- **Flat columns (owner, 2026-10-21):** `defineTable` takes `columns`, a flat record: each column is a scalar (`str`, + `int`, `bool`, `num`) or `json(...)`; no nested `obj`/`arr` outside `json(...)` (refused by the types and at + declaration). Every column is a real SQLite column; scalars can be filtered and ordered; keys and indexed columns are + `str` or `int` (S16); a `json(...)` column is one JSON `TEXT` column, opaque to queries, typed when read. Supersedes in + part S16 ("the rest of a row in a JSON column") and S17 ("shape changes of the JSON part"): adding, renaming or + dropping a column is a recorded `alter`; shape changes inside `json(...)` columns migrate on access. +- **P2, in-memory documents are instance state (owner, 2026-10-21):** an in-memory document is attached to the live + object: no storage, no address, no seq, never in the outbox. Same machinery as persisted documents (drafts, `self.tx`, + atomic commits, ops for subscribers and the wire); its commit is prepare and adopt. A transaction mixing both kinds + adopts the in-memory part inside the store's commit step (S1). It lives from activation to the end of the object: + survives plugin reloads (pure objects swapped in place) and native restarts of the live object, ends with the object + (last reference, linger, an input change on reload, Q-B2). Deletes the memory `DocumentStorage` for in-memory + documents and identity hashes as their keys. +- **Query runner (owner, 2026-10-21):** a query is immutable data built with `query(Table).where(…).orderBy(…)`; + `limit` is an argument, not part of the query; the same `where` serves `updateWhere`/`deleteWhere`. The table handle + from `self.table(Def)` runs it: `page(q, limit, ctx, { after })` returns `{ rows, next? }` (an opaque cursor with + direction and last key as logical values, bound to the query's order, refused with another query), `rows(q, ctx)` + iterates all pages, `get(key, ctx)`, `count(q, ctx)`. Outside a transaction it reads committed state, one query at a + time (S4). Reads inside `self.tx` (owner, 2026-10-21): `tx.page(table, q, limit, { after })` and `tx.rows(table, q)` + return committed rows, stable for the whole transaction (only the owning object writes its tables, P3, and its + transactions run one at a time on its line), without the transaction's own writes; `tx.get(table, key)` overlays + the transaction's own pending writes (put, update, delete) on the committed row (**superseded in part** by W10: after + a bulk operation on the table, `tx.get` throws `read_after_bulk`). + Documented rule: queries inside a transaction see the state before it; own writes become visible after the commit. +- **P4, plugins' own databases and files (owner, 2026-10-21):** state written by code lives in a store: the session's + store for session data, the server's store for instance data (p2's `catalog.sqlite` becomes a rebuildable mirror table, + `shares.sqlite` server tables, `subagents.sqlite` part of the session's metadata, the code-written JSON files such as + `models-store.json`, `forwards.json`, `owner-revoked.json`, `radius.json`, `device.json`, `prefs.json`, `envs.json`, + `voice.json` server documents or tables); a plugin's own SQLite file holds only derived data (`search.sqlite`); blobs + are files named by hash; files a person edits by hand (`settings.json`, `models.json`, `auth.json`) stay files + (open point #21). Logs are not data. +- **Params instead of inputs (owner, 2026-10-21):** an object type's `input` is renamed `params`: the fields that select + an instance among objects of its type (like route params), part of its identity. Params are a flat record of + required fields, each `str()`, `int()`, `bool()` or a union of string literals (`oneOf("chat", "agent")`); no + defaults, no optional, nested or unknown fields (a `get` with a missing or extra field fails `invalid/params`); the + identity key is the field values in declared order (no canonical JSON walk, no normalization). p2's 78 object types + fit (52 empty, the rest one or two required scalars). Supersedes in part R7 ("inputs stay arbitrary", "inputs are + always normalized") and the placement entry ("`input` stays"); Q12 (stripping) now concerns method arguments only. + "Input-less" in P3 and Q-A reads "with empty params". +- **Q-A, lifetime from kind and params (owner, 2026-10-21; both reviewers up):** no `lifetime` option and no lazy flag. + A `local` type with empty params lives as long as the host (started eagerly at host start, never ended by reference + count; it may hold persisted data, P3, and contribute, R21); every other `local` type and every `remote` type lives + while referenced (`linger` only for those). No remote object has host lifetime (its identity includes the + principal; long-lived parts go into a local shared store). A lazy flag was rejected for now: a lazy store would + contribute and run routines only after first access and escape the start barrier; p2's 11 such stores open only + cheap natives. Every "`lifetime: \"host\"` object" in earlier entries reads "`local` object with empty params". +- **The harness is an ordinary object (owner, 2026-10-21; Astra's C17):** no harness exception in the start barrier. + The harness is a host-lifetime object (`local`, empty params) that opens Durable during the barrier with scheduling + paused (Durable opens paused until `resume()`); base code calls `harness.resume()` after `await host.load(...)` and + before the worker connects to its parent (**superseded in part** by W6: connect first), so no request arrives earlier and every registry is filled. Chord knows + nothing about Durable's start. +- **Q12, closed argument schemas (owner, 2026-10-21; review round 3):** every object schema in a contract is closed + (`obj()` sets `additionalProperties: false`); an unknown argument field fails the call with `invalid/args` instead of + being stripped; the provider validates the caller's object in place, never copies or mutates it; arguments have no + defaults (optional fields are `opt()`, handlers write `args.limit ?? 50`). Supersedes in part R7 ("arguments are + always stripped") and the earlier "undeclared argument fields are stripped" (step 2); `checkArgs` is one compiled + validation. +- **Q15, `local` arguments are validated (owner, 2026-10-21; review round 3):** one rule for every method call: the + provider validates arguments against its closed schema (Q12) for `local` types too (callers are often other plugins, + bundled without type checks; `local` stores hold persisted data, P3). Never copied. Supersedes the `local` column + "neither validated nor copied" (design §5) and the `local` part of "Object kinds". +- **Q13, origin incarnation on relayed calls (owner, 2026-10-21; review round 3):** relays stamp the origin host's + incarnation into the `call`/`hold` header, taken from the link the call came in on and overwriting any value a client + sent (like the principal), so `ctx.incarnation` always names the host where the call started. One header field. + Refines R20. +- **Q14, exposable state (owner, 2026-10-21; review round 3):** a state member of a contract may be backed only by a + JSON readable: a document handle (persisted or in memory), a derive, `constant(...)` or a state readable + (`replicatedState`, e.g. Durable's views). `self.writable` (host-local, any in-process value, for registry + contributions) is refused as a state member: by type, and when the object is served (a kind tag check, no value + walk; the object fails `broken` naming the member). Values are still not validated (R9). +- **`reset` instead of release and re-hold (owner, 2026-10-21):** (**superseded in part** by "Flow control: an ack-clocked window": consumers never fall behind a window; only a relay sends `reset { hold }` upstream, for a downstream that fell behind; no debt or minimum interval) when a hold falls behind (the link's window + overflows), the consumer or relay sends `reset(hold)` on the existing hold, never a release: the reference count is + unchanged, so a slow connection can never end an object. The provider answers with `bound` (a fresh snapshot) and + restarts that hold's encoder; the consumer drops everything for the hold that arrived before the new `bound` + (per-direction order). Q3's rules stay (one snapshot in flight per hold, after the previous debt drained, minimum + interval). A link drop remains a departure. Supersedes "releases the hold and re-holds" (R19, design §3). +- **Routines get the changes since their last run (owner, 2026-10-21):** (**superseded** by "Watchable and one watch API": per-commit frames) `self.routines.on(readable, (value, changes, + ctx) => …)` runs one call at a time with the latest value; `changes` is `{ kind: "ops", ops }` (every op since the + previous run, in order, the commits' op arrays kept by reference, never re-diffed) or `{ kind: "reset" }` (the first + run, or more than N ops piled up). Same rule as Durable's watch overflow and the link's `reset`, applied per run; + routines that only want the value ignore `changes`. +- **Watchable and one watch API (owner, 2026-10-21; **superseded in part** by W1 (flush after the step), W2b (routines + start with the current value), W4 (two kinds) and "One pending queue for slow consumers of state" (the 100-frame + overflow)):** v2 has no `replicatedState`. Everything observable is a + `Watchable` = `{ value; watch(ctx): Promise> }` with `T` limited to JSON plus bytes at the type + level: document handles (singletons; keyed per key), derives, reference members, `constant(v)`, and Durable's + conversation view, task graph and documents. Durable's `WatchHandle` and `WatchEnd` move into Chord as the one + observation API and Durable imports them: `watch()` atomically takes the current value and buffers later commits; + `start(listener)` installs the one listener; each commit is delivered as `(value, ops, ctx)`, in order, async + listeners awaited; after 100 pending frames the backlog becomes one root replacement `[["r", newest]]`; a retired + document ends with `[["r", null]]`; `stop()` is idempotent, drops pending frames, never aborts or awaits a running + call, and returns the end; `closed` resolves to the same. `WatchEnd` reasons are generic: `stopped`, `cancelled`, + `retired`, `ended` (source closed: object ended, reference released, Session closed), `listener_error { error }`, + `failed { error }` (source failed); Durable adopts them (was `session_closed`, `session_failed`). `routines.on` is a + watch owned by the object (per-commit frames; Chord awaits a running call on reload), replacing "Routines get the + changes since their last run". Durable's state getters (`documentState`, `viewState`, `taskGraph()`) are removed + (unused by p2). Derives and the wire encoder use an internal synchronous hook inside the commit step so a derive's + change travels in the same wire batch as its source (no torn state); this needs exhaustive tests. Memory: each watch + holds at most 100 frames by reference; every watch ends (object end, reload, ctx, source end); GC tests count + retained revisions. +- **W4, two watchable kinds (owner, 2026-10-21; both reviewers agree; review round 4; JSON-plus-bytes **superseded in part** by "L2 spec decisions": watchables are JSON only):** Delta's `Op` is typed over + JSON, so natives never ride in ops. `Watchable` (JSON plus bytes): `value` + `watch(ctx)` → + `WatchHandle`, listener `(value, ops, ctx)`, overflow → one `[["r", newest]]` frame; documents, derives, reference + members, `constant`, Durable's views; the only kind that can be a contract state member or go on the wire. + `NativeWatchable` (any in-process value): `value` + `watch(ctx)` → `NativeWatchHandle`, listener `(value, + ctx)`, a change is a new value by identity (`!==`; mutating an existing native value notifies nobody), overflow + collapses to the newest value, never ops; `self.writable(initial)` (with `set`) and composed registry values. Both + share one lifecycle implementation (`watch`, `start`, `stop`, `closed`, `WatchEnd`) and carry explicit runtime kind + tags; a `NativeWatchable` is rejected as contract state even when its current value happens to be JSON. Derives may + read either kind (mixed sources) and always return JSON plus bytes: a native change marks the derive dirty without + ops; the derive recomputes and its result is diffed as usual. Native changes use the same synchronous dirty-marking + hook (W1) with the same reentrancy flag; `writable.set` never goes through the commit queue and has no seq; + registries recompose in the owner's flush, never inside another owner's commit step. Supersedes the JSON limit in + "Watchable and one watch API" and the earlier `Writable` interface. +- **W1, adopt and mark in the step, one synchronous flush after it (owner, 2026-10-21; review round 4):** the commit + step writes, adopts (pointer swap of every changed value: the owner's documents; Durable's views and documents + touched by the commit, dispatched by conversation/document id) and marks (dependent derives dirty; ops appended by + reference to each touched hold's batch and each watch's queue), O(1) per dependent. Right after the step, before any + other code runs and without any await, the owner's flush recomputes the dirty derives that someone holds or watches, + in dependency order (derives are synchronous pure functions), encodes one part per touched hold (a hold covers all + members of the held object's variant) into one state message per link, and queues watch callbacks. A derive that + throws fails only its member (`broken`); the commit and its sources' parts still go out. A dirty derive nobody + flushes recomputes lazily on its next read. During adopt and flush, `self.tx`, `watch` and `release` throw + `reentrant`; a derive cycle throws once. The same adopt/mark/flush boundary runs without a store seq for in-memory + commits (P2), `writable.set`, registry changes and inbound wire batches. Durable calls the adopt synchronously + inside its commit step (no microtask). Async work belongs in routines writing in-memory documents. Supersedes + "derives and the wire encoder run inside the commit step" ("Watchable and one watch API"). Exhaustive tests: a + throwing derive, a chain A→B→C where B throws, two owners alternating with a derive over both, reentrancy, a watch + started after 150 frames, stop during a running call, a retire with a derive over it, a reload during a routine call, + GC counts of retained revisions. +- **W2, W3, watch fixes (owner, 2026-10-21; review round 4):** W2a: retirement is an explicit source event (last frame + `[["r", null]]`, then the end `retired`), never inferred from `value === null`; in Chord and in Durable alike (a pi + change; Durable's `CommittedWatch` stops treating `null` as retirement). W2b: only `routines.on` delivers the + acquisition value: it calls its handler once with `watch.value` (ops `[["r", value]]`), then starts the watch and + drains the buffered commits; a direct `watch()`/`start()` user keeps Durable's pattern (use `watch.value` first, then + `start`), unchanged. W3: termination clears every reference a watch holds (listener, detach/report/replace closures, + cancellation hooks); a running call keeps only what it needs; GC tests cover stopped but still referenced handles. +- **W6, worker start order (owner, 2026-10-21; review round 4):** connect to the parent first, then `load`, then + `harness.resume()`. A link carries nothing until a hold or call; a host-lifetime store that references a server type + in its factory no longer waits 60 s for a missing link; requests arriving during `load` park like any call to a + loading object; no turn starts before every plugin loaded. +- **W7, store addresses carry no host instance (owner, 2026-10-21; review round 4):** automatic: Chord builds the owner + string of persisted documents and tables from the host kind and object type (`session/notes.store`), never from a host + address (no session id), so a split (S7) or an import under a new id keeps every document reachable; each session has + its own store, so the session is the store, not part of each address. Durable's owner strings (`session`, + `conversation:7`, `task:31`) already qualify. Plugin code never writes addresses; the guide says keys need no session + id. +- ~~**W8, every text column as UTF-16 BLOB (owner, 2026-10-21; review round 4):**~~ (**superseded** by "Text is plain + UTF-8, in code point order") every `str()` column, indexed or not, + is stored as a UTF-16 `BLOB` (high byte first), so text has one order everywhere (JavaScript's), adding an index never + changes results, and lone surrogates and NUL survive; `json(...)` columns stay JSON `TEXT`. The SQL generator casts + `concat`/`replacePrefix` results back to `BLOB` and uses only even byte offsets (whole code units); conformance tests + cover emoji prefixes, NUL and lone surrogates. Cost: twice the bytes for ASCII text, hex in `sqlite3`, ~0.4 µs per + encode. Supersedes S16's "only key and indexed text". Revisit note (owner): Armin is not happy with the UTF-16 + encoding; no better option is known yet. +- **W9, no compaction at first (owner, 2026-10-21; review round 4):** batches carry the ordered operations exactly as + committed; a replica applies them in order, as a whole, in one transaction (S5). "Keep the last write per row" is + dropped (it loses rows with `update` and bulk operations). A later compaction must merge `update`s into the preceding + `put` and treat bulk operations as barriers. Supersedes S6's compaction note and the book's compaction text. +- **W10, no reads after a bulk operation (owner, 2026-10-21; review round 4):** once a transaction has an + `updateWhere` or `deleteWhere` on a table, `tx.get` on that table throws `read_after_bulk` (no JS evaluator of `where` + and `set` on any backend); other tables and reads before the bulk operation are unaffected. Like Durable's + `ReadAfterWrite` (spec §4: no table read after the first table write), but narrower. The overlay for `put`, + `update` and `delete` stays (owner): helpers doing read-modify-write compose within one transaction. Durable keeps its + stricter rule (point reads could get the same overlay, scans could not; not needed today). +- **W11, two version levels for migrations (owner, 2026-10-21; review round 4):** each `json(...)` column has its own + version and a value-only migration (`json(Schema, { version, migrate: (old, from) => new })`), stored with its value; + it migrates on access, and a write of that column stores the new shape and its version (writes of other columns never + touch it). The row's `v` means only "eager steps applied up to v": only an eager step and a full-row `put` set it; + `update`, `updateWhere` and `deleteWhere` never do. An eager `rows` step applies its row function to rows with + `v < target` in key order with a cursor, writes the changed columns plus `v`, one commit per chunk; after a crash it + scans from the first key again, skipping rows with `v >= target` (one pass, no temporary index, no progress record). + No ordinary writes happen between chunks (the owner is not live until `ready`); old code refuses a newer row `v` or a + newer JSON column version independently; replicas apply the shipped writes and alters without code; a new index is + created after `ready`. Key changes are a recorded `rebuild`. Supersedes in part S17 ("each row stores its version `v`; + `migrate(row, from)` on read") and "Flat columns" (shape changes inside `json(...)`). +- **W12, W13 (owner, 2026-10-21; review round 4):** an in-memory commit (P2) takes the owner's next publication + number, never a store seq, never enters the commit queue, and runs adopt/mark/flush (W1) when it happens. Durable's id + counter at open is the maximum over all five of its tables (S10). One message per link per commit comes from the W1 + flush, also when one Durable commit touches several holds on a link. `count(q)` stays in the query runner although + S12 starts without aggregates. +- **W14, cuts (owner, 2026-10-21; review round 4):** (a) no IndexedDB backend: the browser runs SQLite in WebAssembly + (official build 3.54.1: `sqlite3.wasm` 879 KB, ~406 KB gzipped), persisting into IndexedDB (a storage adapter such as + wa-sqlite's) or the browser's private file system; serverless targets use SQLite where it exists (Durable Objects, + Turso, D1) and a Postgres backend (a second SQL generator for the query language) when a real target needs it; a + backend needs atomic transactions, ordered indexes, bytewise binary keys and one writer per session. FoundationDB was + considered (Armin): the right model (ordered key-value with transactions) but the wrong shape (an operated cluster, + no embedding, 5 s / 10 MB transaction limits, no self-contained session files); at most a later backend for a hosted + multi-tenant service. (b) The `replicatedState` family (`replicatedState()`, `ReplicatedStateSource`, + `AttachedReplicatedState`, the microtask scheduling in Durable's source attachment) is removed as public API in Chord + and Durable. (c) `stopped` and `cancelled` stay two end reasons. (d) Derive `previous` is removed (`mapShared` and + lazy recompute cover it). (e) The per-row `rows` migration step is the explicit exception (done by W11). +- **No document tags (owner, 2026-10-21; Fable's consistency check, Astra earlier):** `defineTag`, the `tags` option of + `tx.draft` and `retireByTag` are removed. They existed only to delete a session's data; with one store per session, + deleting the session deletes its store and directory, and the documents engine (S13) has no tag index. Instance data + on the server that refers to a session is a table with a `session` column, cleaned up by its owner (`deleteWhere`, + reconciling from the catalog, "invariants across owners"). +- **Principals stay unversioned (owner, 2026-10-22; L1 part 4):** the app's `P` is a plain base-code type with no + schema or version: Chord never persists principals, base code changes every host at once, links stamp principals. + Objects persist a projection with its own schema, never the principal itself (p2: chat's `Author { kind, id, name }` + from `authorOf(principal, ownerName)`, stored in every input entry); the guide states the rule. +- **Direct mappings stay; params checked by TypeBox (owner, 2026-10-22; L1 build):** `accepts` maps each accepted old + version directly to the current one (no chains in Chord; one argument mapping per call and one state projection per + commit per old version); when a new version arrives, the author updates every still-accepted version's mappings, and + the guide shows composing step functions (`(old) => v2to3(v1to2(old))`). Params are checked with a compiled TypeBox + schema (a closed object), so constraints such as `str({ pattern })` or `int({ minimum: 0 })` are allowed and + enforced; the type level and the declaration check allow only scalar helpers (`str`, `int`, `bool`, `oneOf`) without + nesting. Refines L1 part 2 ("field by field, no TypeBox"). +- **`bindProvider` (owner, 2026-10-22; L1 build):** `serve` (L4) calls `bindProvider(Type, accepts)` once per load and + reload: it runs `checkAccepts` once and keeps the validated mappings (frozen); per call `checkArgs(provider, method, + raw, boundVersion)` checks with the bound version's schema and maps directly to the current version. Condition + (owner): no leak. The binding is held only by the served type's registration and by calls in flight; no global + strong cache keyed by contracts or bindings (compiled schemas only in `WeakMap`s keyed by schema objects); a GC test + shows a binding, its mappings and compiled schemas are collected once dropped (reload) and the last call settled. +- **L1 review round 1 decisions (owner, 2026-10-22):** same-host arguments are not checked for plain data (no + prototype, getter or `unknown()`-subtree checks): the documented convention is that callers pass plain data (JSON + plus bytes), immutable by convention. Params are never copied or frozen: the object keeps the caller's params + object, immutable by convention (no freezing anywhere). A method without arguments takes `null`: dispatch passes + `null`, the README says so. Cuts (owner): the error reason type is a closed union of the decided reasons (later layers + add theirs; plugin errors never use Chord reasons, shape decided in L4); `failure.ts` and its activation helpers stay; + the `copyJson` wrapper goes; `Provider` keeps only its runtime brand; `revocationKeys` stays (**superseded** by "Simplification pass after 4a": cut until 4b). +- **L1 review round 2 decisions (owner, 2026-10-22):** error reasons are part of the link protocol: adding a reason + bumps Chord's protocol version (hosts with different versions refuse each other, an old tab reloads), so a receiver + keeps rejecting unknown reasons; a test pins the reason list with the protocol version. Failed checks take TypeBox's + public error list (the internal first-error wrapper is removed). The provider's dispatch table is private (a + provider-keyed `WeakMap`). Accepted versions' argument validators compile on first use. The closed-object argument + check runs once per declared or edited method. Owner rule: no runtime checks of values TypeScript already guarantees + (AGENTS.md); declaration checks of decided rules the types cannot express stay. +- **Cost counters (owner, 2026-10-22):** the function that does the work counts it through its context (validation, + compilation, canonical JSON, a future counted `copyJson` with an optional context added in pi); call sites never + count work done elsewhere. Counters stay plain totals without site labels: mainly a test guard (a test isolates one + operation), plus a rough production number; attribution, if ever needed, comes from an event on the context's span. + The error rebuild from the wire does not copy (fresh decoder output) and checks only the envelope. +- **Code style and L1 cuts (owner, 2026-10-22):** all Chord v2 code follows the section "Chord v2 Code Style" of the + root AGENTS.md (one concept per file, top-down order, doc comments, no flag parameters, no typed-value checks, weak + caches with GC tests), so every agent has it in context. Cuts: the params descriptor pre-check is removed (params and + arguments follow one rule: the compiled closed schema; wire params are parsed JSON, same-host callers pass plain data + by convention); `bindProvider` keeps the app's accepts table as is (immutable by convention), no copy or freeze. +- **L1 rewrite questions (owner, 2026-10-22):** `checkArgs` returns a tagged value, `{ kind: "method"; args; result? }` + (args in the current shape, `result` maps back for an old-version caller) or `{ kind: "call"; args; call }` (the + checked old args and the provider's replacement, invoked by L4); no second lookup function. Failed-check paths are + TypeBox's instance path unchanged; keys containing `/` or `~` make it ambiguous (a diagnostic; not worth recovering). + The JSON Schema `oneOf` keyword is rejected (the `oneOf()` helper emits `anyOf`); local accepts tables and + `Implementation` are flat. +- **L1 rewrite review (owner, 2026-10-22):** nothing in L1 is frozen: contract handles, schema snapshots, params + schemas and providers are Chord-owned data, immutable by convention like everything else; the snapshot copy stays + (it keeps validation in sync with the fingerprint when an app changes its schema objects). Supersedes "frozen" in Q10 + ("history private, deep-copied and frozen") and the frozen metadata in the round-1 and round-2 entries. Size accepted + (about 1,900 lines, 1,450 without blanks and comments; about 420 type-only). Failed checks say "unknown field" and + "missing" for those two cases, TypeBox's message otherwise. A malformed error envelope from a peer throws a plain + `Error`. `Principal` is a type alias (apps declare principals as type aliases, not interfaces). +- **One watch-handle implementation, upstream first (owner, 2026-10-22; L2 spec):** Durable's `CommittedWatch` moves + into Chord's shared `src/watch.ts` (pi change, upstream first), generalized to serve native watchables too + (`NativeWatchHandle`, listener `(value, context)`, overflow to the newest value); Durable uses it and its + `observation.ts` goes away; Chord v2's L2 builds on it. Publications, derives, sinks, `constant` and `writable` stay in + `next/` (v2 only). Durable's synchronous view-change hook (so a derive over a Durable view is marked in the commit + step) is a later pi change, with L3d. +- **L2 spec decisions (owner, 2026-10-22):** watchable state is JSON only (no bytes in documents, derives, reference + members or any watchable; bytes only in method arguments and results and chunked transfers; Delta stays JSON-only; + supersedes W4's "JSON plus bytes" for watchables). `mapShared` lives in `next/watch/` until v2 goes upstream. C10 is + rejected (`constant` and `writable` stay two kinds, Q14). One publication object per host, passed to its sources; + no module-level singleton. No CBOR: the wire stays JSON plus raw byte segments (R1, Q4), revisited only if L6 + benchmarks show encoding as a bottleneck. +- **Contexts are required (owner, 2026-10-22):** as in Durable (356 required context parameters, none defaulted; + optional only on error-reporting paths), every Chord v2 function that takes a context requires it: no + `= BACKGROUND_CONTEXT` defaults, no `context?`. A getter that cannot take one (a lazy derive read) uses its owner's + context (the publication's). Applied to L1 and L2; written into the AGENTS.md style section. L2 review: the + publication resets its phase and queues in a `finally`, so a throwing sink cannot leave it stuck; L2's size (about + 680 lines) is accepted. +- **Carry-over ledger (owner, 2026-10-22):** obligations deferred to later layers are recorded in + `packages/chord/docs/next/carry-over.md`, under the layer that must do them, by whoever defers them, in the same + change; a layer's spec includes its section; done entries are deleted. The AGENTS.md style section states the rule. +- **Vertical slices instead of layers (owner, 2026-10-22):** building strictly bottom-up made L2 grow generic machinery + for callers that did not exist yet (nodes, sinks, deliveries, "publications"), hard to follow; its code was deleted, + not kept. From now on Chord v2 is built in vertical slices: each slice is the thinnest end-to-end path through + every layer it needs (slice 1: one host, one local object with `self.state`, `self.tx`, `self.derive`, a checked + method call and a watch), so every API has a real caller. Same process per slice: spec by this session, owner + approval, an Opus 5.5 build, review, owner review, commit. L1 stays (stateless checks, committed with required + contexts and an `index.ts`/`internal.ts` split, `ff4f4456d`). Kept from L2: Chord's shared watch handle + (`src/watch.ts`) and the decisions (JSON-only state, one batch per host, no CBOR, C10 rejected). Plain names from + the design's vocabulary or common usage (a signals library's `batch`), never invented internal jargon in APIs. + Plan: `packages/chord/docs/next/slices.md`. +- **`Implementation` maps state members to watchables (owner, 2026-10-22; slice 1):** L1's `Implementation` types a + state member as `Watchable>` (the document handle, derive or reference member the factory returns), + not a getter `() => value`: a getter cannot tell Chord when a value changed (design §11). +- **Slice plan and slice-1 review decisions (owner, 2026-10-22):** slice order 1 local objects, 2 failures and + natives, 3 two hosts, 4 topology, 5 routines and jobs, 6 persistence, 7 reload and rollout, 8 registries and the + harness, 9 telemetry (routines and jobs before the store and reload). A feature map in `slices.md` assigns every + design feature to a slice; implementers and reviewers check the slice spec, the map and the design (AGENTS.md). + Slice process includes reviews by Astra and Fable. An object that ends rejects new calls, aborts its running calls' + contexts and makes its `self.tx` throw `released` (a running method can change nothing after the end); no drain + without natives (the drain, design §3, is decided with natives and C13 in slice 2). Keyed in-memory documents, + `mapShared` and `defineFacet

` stay in slice 1 (base infrastructure), each with a real use in its example. + `checkpointWhen` is required only for persisted documents (refines R14: in-memory documents are never written, so + checkpointing cannot apply). `retryOn` waits for the first independently living source (registries, slice 8); + slice 2 has `ref.retry()` and dependency recovery. +- **Slice 1 approved (owner, 2026-10-22):** cycle rejection on one host (Q5) is in slice 1; watches belong to the + reference they came through and end when it is released; at `get` the consumer's contract version and fingerprint + must equal the provider's (`contract_mismatch` otherwise); `then`, `release`, `state` and `retry` are reserved member + names, refused at declaration; cut C14 rejected (the `handle.change`/`handle.retire` shorthands are built); cut C16 (**superseded**: accepted on 2026-10-26, "Facet documents are removed") + rejected (facet documents are built: served objects may draft them in their own transaction). +- **Callers when an object ends (owner, 2026-10-22; slice-1 review):** as design §3 says, callers are rejected at + once: when an object ends, every caller of a running call gets `released` immediately; the handler keeps running in + the background under an aborted context and cannot commit (`self.tx` throws `released`). A call whose caller context + is already aborted is rejected before its handler runs. Each call runs under `withAbortSignal(objectEndingSignal, + callerContext)` (one controller per object; a caller's cancellation aborts only its own call). The frozen v2 kernel, + its tests, the playground built on it and the kernel step docs were removed (`4a54fc38b`); `Watchable` went upstream + into `src/watch.ts` (`a2fd25f56`). Facets are typed by exactly the host's principal type (invariant). +- **No derive names, no `null` for no-argument methods (owner, 2026-10-22; slice-1 review):** `self.derive(sources, + compute)` takes no name: the name existed for `previous` (removed, W14); on reload exposed derives are matched by their + member name, internal ones are rebuilt (slice 7 adds matching only if a real need appears). A method without + arguments omits them in the API: references call `ref.bump(ctx)` and implementations are `bump: async (ctx) => …`; + the shape is fixed per method by the contract; `null` remains the internal and wire form. Refines the L1 round-1 + decision "no-argument methods take `null`". +- **Slice 2 approved (owner, 2026-10-22):** natives with per-native readiness, ordered opens and reverse closes, close + reasons and a close context aborting at `closeTimeout`; failure causes by where they failed; every attempt bounded + by `readyTimeout`; backoff per identity (a map holding no object); only R18 failures keep references; gap-free + dependency handover on retries; the start barrier waits for each host-lifetime object's first outcome; `host.get`: + broken rejects at once, transient waits up to the getter's own deadline; the drain waits for handlers' own promises + before natives close. Cuts: C5 rejected (no natives in facets); C6: `broken()` kept, `retryOn` in slice 8; C7: + `linger` kept, built in slice 3; C13: two host timeouts (`closeTimeout`, `readyTimeout`), the drain uses + `closeTimeout`. `ref.state`, `ref.retry()` and `host.hold` move to slice 3; the plugin error shape is decided in + slice 3. Openers take a `Context` (not `({ signal })`). +- **Slice 3a approved (owner, 2026-10-22):** slice 3 is split into 3a (remote objects over one link) and 3b (surviving + drops: re-holds, parking, `ref.state`, `host.hold`, `ref.retry()`, `linger`, `host.root`, `state: false`). Guiding + rule for every choice: simple, fast, correct. A remote factory returns `{ variant, members }`; `ref.variant` is the + discriminant (C15 accepted: no `inVariant`/`expectVariant`; `variant` reserved). Calls are checked against the + binding's variant. A hold's state is one object with one Delta encoder; failed members travel in the header. A + release rejects the calls made through that reference, locally and over links (refines slices 1-2). `$bytes` is the + one reserved key in values crossing a link (byte leaves travel as raw segments referenced by `{"$bytes": i}`). + Plugin errors: `PluginError(code, message, details?)`, `internal` for anything else, never a stack. Headers are + checked on every link until IPC (slice 4). Hot-path slices add light benchmarks (timing, memory, wire size) recorded + in `benchmarks.md`. +- **Wire protocol (owner, 2026-10-22; slice 3a review):** the protocol works on a typed `Message` (one TypeBox + schema, `MESSAGE`, also checked after every decode); codecs are pluggable per link: `binaryCodec` (kind byte, varints, + raw ids, JSON values; about 23 B for a streamed token instead of 98) and `textCodec` (Delta-style JSON tuples, no field + names), compared by benchmarks for wire size, compute and memory. Member, method and variant names travel as indexes + into the hold's bound version; a reload that drops that version ends its holds first (slice 7). Checks by layer: the + codec's structural decode, the `MESSAGE` envelope, the bound version (ids, indexes, order), then the owners + (`checkParams`, `checkArgs`, Delta's decoder). The memory link delivers with `queueMicrotask` (no Node APIs). A + 2,000-token reply drops from about 196 KiB to about 46 KiB. Supersedes R1/Q4's JSON header framing. +- **Slice 3b approved (owner, 2026-10-22):** references survive link drops (`loading`, values readable, calls queued + with their own deadline and sent in order, sent calls fail), one owner of remote references across links re-holding + on every `connect`, a re-hold may bind another variant; `ref.state` (`loading`/`live`/`failed`/`released`) is a + `NativeWatchable` (interface added to `src/watch.ts` upstream first); `host.hold`; remote retries through two new + messages `retry { hold }` and `loading { hold }`; parked calls wait at most `readyTimeout` (C13 keeps two host + timeouts); `linger` for live objects; C8 (`state: false`) cut; `host.root` moves to slice 8. +- **Flow control: an ack-clocked window (owner, 2026-10-23):** supersedes Q4's byte window, R19/Q3 (snapshot admission + and debt), C12's credit accounting and K16's call credits. Measured by Astra and Fable (Node 24, `ws` 8.22, separate + processes, proxy at 64 KiB/s): a `bufferedAmount` cap plus latest-state coalescing still delivered state 9.4 s old + (kernel and proxy buffers hide megabytes); application acks with 1-4 messages in flight plus coalescing delivered + 0.0 s old state, heap +0.4 MB, 98% fewer messages; no control buffered 7 MB (+18.5 MB heap; a client that stops + reading: 23.4 MiB, RSS +121 MB in 15 s). IPC `send()` returns false from about 64 KB yet 623k sends cost +1,035 MB; + its callback acknowledges the write, not the handler. Rules: + - At most K=2 unacked `bound`/`state` messages per link; the consumer sends a small `ack` after applying each one. + - Ops queued meanwhile are merged per hold; a snapshot replaces them only when the merged ops exceed the member's + value size (2,000 appends: 46,024 B one by one, 26,021 B merged, 8,035 B as a snapshot; but a member with 500 KB + of history would cost 508,035 B as a snapshot). + - Calls: a call window N (default 256) announced in `hello`; the caller parks extra calls in 3b's parked-call queue, + FIFO, refusing nothing (K16 kept). Exceeding N is a protocol error, enforced by the provider. 100k calls of 10 ms: + a caller-side cap of 64 took 17.1 s, +4 MB, another client's p95 0.66 ms; unlimited ran 43,824 at once (+56.5 MB, + p95 129 ms); a provider semaphore queued 96,736 on the provider. + - Holds are unlimited by count; size limits stay a separate check. Dropped: the byte window and credit messages, + snapshot admission and debt, C12's credit accounting, an adapter `backlog()` API, `reset` from the consumer. + - **Relays ack per hop (option b):** the server acks upstream on receipt. When a downstream tab falls behind, the + server drops that tab's queued parts and sends `reset { hold }` upstream; the provider answers with a fresh + snapshot of that hold (a second `bound`). One slow tab never coarsens the other tabs of the same worker link + (option a, end-to-end acks, rejected for that). About 40 more lines than (a). + - Open for the slice-4 spec: Astra's circuit breaker (a hard transport-buffer cap plus a stall or heartbeat + deadline that closes the link). + Explainer: `packages/chord/docs/next/flow-control/` (Radius artifact 01m5n82vsbe5rtn4nm6wzaq0mp). +- **One pending queue for slow consumers of state (owner, 2026-10-23):** refines the snapshot fallback of "Flow control: + an ack-clocked window" and supersedes "Watchable and one watch API"'s "after 100 pending frames the backlog becomes + one root replacement" for state watches. Measured by Astra and Fable (`/tmp/fc2-*.md`, `/tmp/fc3-*.md`; real p2 entries plus synthetic stress + cases): size-based rules fail ("snapshot bytes + sent op bytes" is no bound: an interned path lets a value grow + faster than its ops; it drifts 1,254x under churn; exact size needs per-node bookkeeping), a time trigger punishes a + stalled stream into a large value, and a count of unfolded commits resends whole histories. Folding alone came + within about 10% of an exact oracle. Rules: + - Pending ops stay raw (never encoded per commit while waiting); `r` drops everything before it. + - **Leaf index:** a map from path key to the last pending leaf op (`a`, `s` of a primitive); a new leaf op on a + path in the map merges into that op in place (`a`+`a` joined, `s` replaces `s` or `a`, `a` after a string `s` + appends into it). Any other op (`p`, `m`, `d`, `t`, `s` of an object or array) clears the map. Correct because + leaf ops on different paths commute, except an ancestor primitive `s`, after which the descendant cannot be + written until a structural write clears the map (fuzzed: about 1.8M tracker and `diffRevisions` ops, arrays with + splices and reorders, adversarial cases; a broken index without the clear on `p` fails). Cost about 0.2-0.4 us and + 60-80 B per pending leaf path. Ops come only from Chord's tracker and `diffRevisions`: no defences against + hand-written ops (no path-key normalization). No folding of list appends (1% gain, quadratic copies) or trims. + - **Cap: 10,000 pending ops after folding;** past it only "replace with the current value" stays pending (nothing + more accumulates). Worst case about 2.2-2.7 MB per member per slow consumer. No size or text caps, no time trigger. + - Used by link hold members while the K=2 window is full (encoded once when an ack frees a slot) and by state + watches (`Watch` over `(value, ops)`): a slow listener gets one call per batch with the folded ops and the newest + value. Measured, listener taking 1 s: 438 calls and 3.39 MB of ops become 171 and 371 KB. The watch change is in + Chord's shared `src/watch.ts` (pi first); tests and `examples/04-watches` change with the contract. + - Unchanged: Durable's event stream (`watchEvents` batches are events: they keep their own queue until removed), + `NativeWatch` (newest value), relays (drop + `reset { hold }`: parts are opaque). + Animation: `packages/chord/docs/next/flow-control/fold/` (Radius 01m5nc9s24e9kscppen2fxgwye). +- **Slice 4 split and first decisions (owner, 2026-10-23):** slice 4 is split into 4a (links and relays: IPC and + WebSocket links, trusted links, parts as opaque bytes, flow control, host kinds and placement, routes, relayed holds + and calls) and 4b (revocation, cycles across hosts, `host.references`, `host.activity`, size limits, compression); + draft with structural changes and races: `packages/chord/docs/next/slice-4.md`. Decided: trusted links are an + explicit option (`trusted: true` on `accept` and in `resolve`'s result; default untrusted; stamping only from the + parent link). ~~Resets and queue overflow travel as `[["r", value]]` ops in an ordinary `state` message~~ (**superseded** by + "Slice 4a review decisions": resets are a list-`bound`; overflow stays an `r` op): a relay's + `reset { holds }` (a list) is answered with one message carrying every reset hold's members as `r` ops (snapshots + taken in one step, applied in one batch), entries flagged `reset: true` in the header for relays; a second `bound` + only for slice 7's restarted objects. ~~Relayed calls are sent round-robin per origin while the worker's call window + is full (each origin keeps its order).~~ (**superseded** by "Relay flow control per origin": a relay forwards calls at + once; each origin has its own N at the provider). The Durable conversation view's size on re-hold is a known cost until pi + #11327 lands. +- **Relay flow control per origin (owner, 2026-10-24):** supersedes per-hop acks, `reset { holds }`, relay recovery + and id renumbering ("Flow control: an ack-clocked window" relay part, "Slice 4a review decisions" reset part). The + provider keeps the window (K=2), its pending queues and its call queue per **origin**; the relay never drops, folds or + resets. The relay assigns each downstream link an origin number (never reused on an upstream link; 0 = its own + work); upstream messages keep their ids and carry `origin` (read only on the parent link); the provider keys holds, + calls, windows and acks by (origin, id); replies carry the origin. The relay keeps per downstream hold its upstream + link and its original `hold` message (routing calls and releases; re-holds after a worker restart), and per downstream + link a FIFO of the upstream link of each message in flight (each in-order `ack` goes to the right worker). A new + message "origin closed" releases an origin's holds and calls when its link closes. A tab served by W workers may have + 2W messages in flight. Measured motivation: a slow tab through the relay needed ~24 s and 3 resets to catch up after a + 1 s stall (provider folding: 0.02-0.38 s). After the rework and the review fixes: a simplification pass with a budget + of 1,200 lines of net growth for 4a (owner). +- **Docs (owner, 2026-10-24):** `design.md` becomes the one current statement of all rules, updated in the same commit + as each decision; `mechanisms.md` folds into it, `carry-over.md` into `slices.md`; the decision log stays complete as + the history and the reasons; slice specs stay as history after their slice. Nothing is dropped in the move. +- **Slice 4b: `host.activity` is a watchable (owner, 2026-10-24):** a JSON `Watchable` of `{ references, calls, jobs, + lingering }`, published at most once per microtask turn with changes (design §3's "one readable"). +- **Docs consolidation (owner, 2026-10-24):** applied as decided in "Docs": `design.md` now states every current rule + (`mechanisms.md` folded in section by section, its 17 stale points fixed, then deleted); `carry-over.md` is the + section "Carry-over" of `slices.md` (AGENTS.md updated). Settled by what slices 1-4a built, as approved: resets do + not exist (per-origin flow control); trusted links skip only the `MESSAGE` check; a parked call waits its + reference's `readyTimeout` (no per-`get` `parkTimeout`); the object backoff is per identity; `host.get(Type, params, + context, options)` takes a context and `host.hold` returns at once and never rejects; a native's `close` gets + `(value, reason, context)`; a local object's handlers act as the host's principal. Still open, listed in design + §26 (#24-#29): the timeouts of later mechanisms, `host.root`'s signature, how a migration commit reaches consumers, + compression on browser links, the circuit breaker, cuts C3/C4/C11. +- **Simplification pass after 4a (owner, 2026-10-24):** two cutters (Opus 5.5, GPT-6.1 Sol) read all of `src/next` + without a line target (a target is not meaningful); the merged small cuts, two bug fixes (relays strip a client's + principal fields on every path; a closed link delivers no queued message) and D1-D4 are applied by a fresh builder. + D1: the unreachable "binding back to pending" branch of a served hold is cut from 4a; slice 7 builds it with reload + (carry-over). D2: L1's unused `revocationKeys` is cut; slice 4b writes revocation keys with their first user. D3: + `initial()` is checked once per definition at its first value (refines R8's "per document": `initial()` takes no + input, so a wrong shape shows at the first call); a failing keyed `initial()` fails only its transaction, a + singleton's fails the activation. D4: object instance ids (Q5) come with slice 4b, their first user; one-host cycles + follow the waiting objects without ids. +- **Slice 5 spec drafted (owner, 2026-10-24):** straight from the discussion to `slice-5.md` (the walkthrough + `routines-and-jobs.md` deleted, owner); design §14 and §15 state routines and jobs as decided; jobs in memory in + slice 5, persistence in 6, reload rules in 7 (carry-over); jobs as objects (#9) rejected. `tx.start` returns the + run's id (R12), `job.outcome(id, ctx)` waits by id; `unavailable/busy` is a new error reason. + On the draft: routines start right after the object went live (same turn); `abort(id)` drops a blocked run (no + `finish`, reported); `running` carries every run's args (views filter). Process: spec reviews, then the build. + After the spec reviews (Astra, Fable, Opus, Sol): `tx.start` returns a `Run` handle (its outcome rejects if the + transaction is discarded) instead of an id; `ctx.state.change(fn)` takes no context (changes while stopping after an + abort still commit); running and finishing runs are a count, not references, and an object without holders ends + (after `linger`) when its last run settles, dropping blocked runs; `finish` may chain the same job (its own run is + excluded from the concurrency check); a `run` throwing after an abort ends `aborted`; `job_failed` reported; + ~~routines also in facets~~ (**superseded** by "Facets only declare"); `persist: { resume: "rerun" | "abandon" }` is the final shape. + After the build reviews: the builder's ten points accepted (every object end reports dropped blocked runs; their + waiters get the end's error; `accepts` needs both mappings per older version; job args without bytes; routine labels + per kind; `start` before live is `not_ready`; a run cancels a linger; design fixes; rejected handles keep their stack + until read, documented); native `on` sources also get only the newest value after a busy call; the finishing run is + excluded from dedupe too (`finish` may start the same job with the same key); the retirement frame in `on` is a + slice-6 carry-over. +- **Routines and jobs open points (owner, 2026-10-24; walkthrough `routines-and-jobs.md` and its reviews):** at the + `concurrency` limit `start` rejects; the routine handover is bounded by `closeTimeout`; a persisted job's state + document gets the deferred persistence option (as in Durable), and authors throttle frequent state changes themselves + (Chord does not coalesce them: every change is a commit and a message). Jobs keep `accepts` (owner: everything + versioned has adapters; the walkthrough's "no job adapters" was never decided): a run started on an older version + finishes on its old code, its args and result are mapped to the current version for `finish` and for `running`, its + state document migrates through its own `migrate`; without an adapter the run is `blocked` and reported until a + reload brings one. So a version change aborts nothing and discards nothing. + Crash handling for persisted jobs (owner, with all four reviewers): a restart limit as Durable's (five crash + interruptions, counted and persisted, clean closes not counted; then parked as `blocked`, cleared by `abort` or a + reload); a `finish` that keeps throwing is retried with backoff and then `blocked` after five consecutive failures, + its outcome, state and key kept, retried on a reload; how a persisted run resumes after a crash (rerun, replay-safe, + or abandoned with an aborted outcome, wording aligned with Durable) is required, with no default. +- **Routines for slice 5 (owner, 2026-10-24; C4 decided):** two kinds, declared in the factory, no names and no + `routines` namespace: `self.on(source, fn)` (first call with the current value, W2b; then one call per change, calls + never overlap, changes meanwhile folded; a running call is not aborted by a change; a throwing call is reported and + called again with the current value after a backoff of 1 s doubling to 60 s, a change during the backoff calls at + once; the source ending ends the routine) and `self.onLive(fn)` (called once when the object goes live; returning + ends it until the next reload or activation; throwing is reported and restarts it after the same backoff, reset + after a run of a minute; an abort is no error). Both start when the object is live and stop when it ends (`ctx` + aborted, then the drain); on reload all old routine calls of the object finish (bounded) before the new routines + start: a handover per object, which needs no names. Routines are idempotent reconcilers; Chord keeps no state for + them; what must survive is a document. Helpers `sleep(ms, ctx)` and `untilAborted(ctx)`; routines and handlers wait + only through calls that take `ctx`. Errors: `HostError` `routine_failed { object, routine, error }` (`routine` = + kind and position). Routines do not count in `host.activity`. Dropped: `every` (an `onLive` with `sleep`; a due + time that must survive is a document), `custom` and `run(fn)` (an `onLive` with `try`/`finally`), names, the due + time across reloads. Kept separate from jobs (different lifecycle, reload behaviour and output). Open: the + handover's bound (`closeTimeout`, whose default may drop from 30 s to 10 s, or its own). +- **Slice 6 first decisions (owner, 2026-10-26):** (1) **Durable's internal commit observers run inside `adopt`** + (pi change, upstream first, built and reviewed before the slice): the Session's `observeCommits` listeners (view + mounts, task graph, scheduler, submissions) move from `#publish`, after `Storage.commit` returned, into the `adopt` + callback, so every Durable view changes inside the storage's commit step; with `StoreStorage` Durable's `adopt` runs + inside the host's batch, and Chord follows Durable's views as sources marked in the step (W1). Reason: today a + derive over two Durable views (conversation view and task graph), fed by two watches, computes between their frames + of one Durable commit ("tool call issued, no task running"). `adopt` stays throw-free (a listener's throw is kept and + fails the Session after `Storage.commit` returned, as today); `subscribeCommits` and telemetry events stay in + `#publish`; nothing changes for users of Durable's own storages. (2) **Persisted singletons load sequentially** right + after the factory, all singletons of an object in one read transaction (S4), before its references are bound; a + storage error is `storage_failed`, a fingerprint mismatch, a throwing `migrate` or an invalid migration result is + `broken`. (3) **Durable's ids in the store:** `StoreStorage` keeps a `record_ids` table (Durable's one namespace + across its five tables); the counter at open is `coalesce(max(id), 1) + 1` over it (1 is the root conversation). (4) + **Few statements per commit:** the store keeps seq and head in memory (read at open, the head persisted with each + commit, S9) and folds the documents engine's checks into the writes (a unique `record_ids` key, a partial unique + index on the live address, `UPDATE … WHERE id = ? AND version = ?`), so a delta commit is about three statements + (revision, version, outbox); `StoreStorage` maps constraint errors to the messages Durable's conformance suite + expects. A benchmark compares `SqliteStorage` and `StoreStorage` on the same workload. Still open: the split of slice + 6, and retirement in watches (point 3 of the discussion: no `null` frame, Chord documents that retire). + **Refined (owner, 2026-10-26), after four reviews of the first build:** the first build moved only Durable's internal + observers into `adopt` and deferred host-waking work (`afterPublish`); the reviews found host-visible regressions in + the gap between `adopt` and the end of `Storage.commit` (watch listeners before `subscribeCommits`, `waitForIdle` + early, a late subscriber missing a commit, the active-task metric drifting after a rejection). Under review now, + nothing built: "publication at adoption" (the whole publication, host `subscribeCommits` listeners and telemetry + events included, inside `adopt`; no `afterPublish`; `adopt` runs no task code, so a marked run's `abort()` is + queued). **No Durable special cases in Chord's store** (owner): the store never knows or waits for Durable's + continuation. + **Decided (owner, 2026-10-26), after four checks of the proposal (`/tmp/observers-proposal-*.md`, explainer + `packages/chord/docs/next/durable-observers/`):** publication at adoption in Opus's variant. Inside `adopt`, in + HEAD's order: the trackers, the telemetry events, the queueing of one microtask that dispatches the host + `subscribeCommits` listeners (the listener list read inside that microtask), the internal observers. Microtasks run + in order, so host listeners still run before every wait continuation, abort and `onWake` of the commit (before + every watch delivery too, as first stated: **superseded**, see "Superseded (owner, 2026-10-26)" below), but after the storage's synchronous step, so never inside Chord's batch (Chord calls from them work; Chord + needs only its step and flush to be synchronous). `afterPublish` is dropped. `adopt` runs no task code (a newly + marked run's `abort()` is queued) and is fully throw-free (listeners, reports, metrics and telemetry contained; an + internal observer's error fails the Session after `Storage.commit` returned). Accepted changes from HEAD: abort + listeners fire after host listeners; a commit the storage rejects after `adopt` is published (it is durable; a + replica stays equal to storage; spec invariant 2 changes) and its waits resolve before the Session fails; with a + storage that works after `adopt` (JSONL) the publication runs before `Storage.commit` resolves; a throwing telemetry + call is reported as `host_error` with callback `"telemetry"` and no longer rejects a durable commit. Nothing in pi or + p2 depends on the old orders (checked: p2's three `subscribeCommits` users). + **After the reviews of the second build (owner, 2026-10-26):** one delivery rule for every watch: a watch delivers a + frame in a microtask queued after the frame arrived, also when its listener was busy (Chord's shared `src/watch.ts`: + after a call the drain reschedules instead of taking the next frame in place). Without it a busy watch's + continuation, queued before `adopt`, delivered a commit before its host listeners (Astra, Opus). Cost measured: + about 170 ns per back-to-back delivery with a promise, less with a plain `queueMicrotask`; only busy watches pay it. + Also: a host listener added between `adopt` and the dispatch gets the commit only when a listener existed at + `adopt` (documented, accepted); a rejection after `adopt` keeps the storage error as the failure and still reports an + observer's error. + **Superseded (owner, 2026-10-26), after the third reviews:** the delivery rule did not hold (Astra, Opus: a delivery + already queued folds frames that arrive later, so a watch can still deliver a commit before its host listeners, also + in a shared commit step chained through promises); a complete fix (a late delivery queues itself again) would stall + watchers behind a source that changes in every microtask. Decided instead (option b): **no ordering promise between + host `subscribeCommits` listeners and watch listeners** (independent observers; nothing in pi or p2 depends on their + order); waits and aborts still follow the host listeners by construction; the hop in `src/watch.ts` is reverted. +- **Durable's synchronous observers (owner, 2026-10-26):** `observeConversation(id, observer, ctx)`, + `observeTaskGraph(observer, ctx)` and `observeDoc(token, …, observer, ctx)` become Durable's public primitive: the + observer's `advance(change, ctx)` (the listener bag `{ value, ops, retired }`) runs synchronously inside `adopt`, + `end(failure)` when the Session closes or fails; attaching returns the current value and `detach`, taken atomically + on the line. `watchXXX` stays, built on it (a `Watch` is such an observer), for app code; `observeXXX` is documented + for adapters (record only: no awaits, no Session calls, no throws, quick). Chord's `durableView` (slice 6b) is a + source over it, marked inside Chord's batch in the store and grouped into one batch per Durable commit on the next + microtask outside it. + **After the reviews of its build (owner, 2026-10-26: fix everything to HEAD's level of correctness, performance and + memory; the owner delegated the details):** `onWake` is no longer ordered after host listeners (same cause as + watches); `Watch.endAfterDelivery()` exists (a task document's watch delivers its retirement, then ends `ended`), + detaching only when the watch really ends and never while a listener call runs; the `observeXXX` context covers the + attachment only; observers get `end(undefined)` when their task document's task ends; the change object a listener + gets carries no context (the context is a separate field of the watch); a watch keeps only its address, never a + loaded document; `observeXXX` rejects like `watchXXX`; Durable's file-watcher type in `/env` is renamed + `FileChange` so `WatchChange` names only the listener bag. +- **One retirement mechanism for Chord and Durable, watches follow the address (owner, 2026-10-26; option A):** + supersedes W2a's last frame `[["r", null]]` and the end reason `retired`, and Q11's silent reset as the only meaning + of retiring a singleton. A document lives at an address (Chord: definition and key; Durable: kind, scope and key); + every document, singleton or keyed, Chord or Durable, is watchable; a retirement is signalled to the watch listener + as one last call with the last good value, no ops and a `retired` flag (no `null` anywhere); the watch goes on, and a + later write delivers the new incarnation as `[["r", value]]` on the same watch. A watch ends when its owner ends + (`ended`): a Chord document's object, a Durable task document's task (the retirement call, then `ended`, in the same + commit), the Session; and by `stop`, cancellation, failure or a throwing listener. Durable's watches change from + incarnation to address (breaking; nothing in pi or p2 watches a document that retires). Leaks: none new for task + documents (they end with their task and invocation); a forgotten watch of a session or conversation document that + is retired for good stays until its taker's context, `stop` or the Session ends it (GC tests). Still to decide: the + shape of the flag, the value an absent address shows to members, derives and `.value`, and the per-key source in a + factory. + **Details decided (owner, 2026-10-26):** (1) a watch listener (and an `on` handler) takes one object before the + context, `(change, ctx)`: `{ value, ops, retired }` for JSON watches, `{ value }` for native ones (a breaking change + of the shared `WatchHandle`/`NativeWatchHandle` and of `self.on`, accepted). (2) A retirement call carries the + address's value after the retirement: `initial()` for a Chord document (as Q11 reads an absent singleton), so + `watch.value` equals the source's value for every reader; for Durable, which has no value for an absent document, + the last value with no ops. (4) Watching an absent address gives `undefined` (`notes.watch(key, ctx)` as Durable's + `watchDoc`). The wire carries a `retired` list per hold entry in `state`, like `failed`. (3) **A per-key source in a + factory:** `handle.at(key)` declares a singleton per key: loaded at activation with the object's singletons, it reads + `initial()` while the key is absent, and is a state member, a derive source and an `on` source; its watchers learn + of a retirement through the flag. Telling "absent" from "empty" is `await handle.get(key)` (`undefined`) or the + flag. Not adopted: the key as an activation dependency (fails when absent, ends `gone` on retirement). +- **Text is plain UTF-8, in code point order (owner, 2026-10-26; reviews `/tmp/text-review-{astra,fable,opus,sol}.md`):** + supersedes S16's JavaScript order and W8's UTF-16 BLOB. `str()` columns are plain UTF-8 `TEXT` (`STRICT` tables; + strings bound and read as they are), and text compares by Unicode code point on every backend (SQLite `BINARY`, + Durable Objects, sqlite-wasm; Postgres `COLLATE "C"`, MySQL `utf8mb4_0900_bin` later). Written values and every + query operand (`where` values, `inArray`, prefixes, cursors, `set` operands) of a `str()` column go through + `toWellFormed()`: a lone surrogate becomes U+FFFD (what `TextEncoder` and Node's SQLite binding do), documented; a + value containing NUL fails `invalid/args` (Node 24.4.1 truncates text at NUL on read; Postgres refuses it; content + goes into `json(...)` columns, which escape both). A leading U+FEFF stays allowed: sqlite-wasm's reader drops it, so + its backend reads values starting with `EF BB BF` as bytes and decodes them keeping it. The store exports + `compareText(a, b)`; the memory backend uses it for ordering, ranges, `max`/`min` and `when`; a prefix's upper bound is + the next code point after its last (portable); `substr` offsets count code points; a SQLite file must use the UTF-8 + encoding (checked at open). The guide states the rule: store order differs from JS `sort()` only for U+E000-U+FFFF + against characters above U+FFFF; sort store text with `compareText`, page with the store's cursors, use `prefix()`, + cut strings with `[...s].slice()` or `toWellFormed()`. Durable's strings in `StoreStorage` (`requestId`, kinds) keep + Durable's JSON encoding in indexed columns, so Durable stays lossless. Rejected: CESU-8 in `TEXT` (a read path of + special cases on best-effort SQLite behaviour; Postgres refuses it), CESU-8 in a BLOB, keeping W8. +- **Slice 6 is split (owner, 2026-10-26):** 6a, the store with Chord's persisted documents and tables (backends, + tables, query runner, writes, commit queue, outbox, documents engine, index definitions, migrations, export and + import, `self.table`, persisted `self.state`, P3, the host's store option, `storage_failed`, W12, keyed watches and + retirement); 6b, what builds on it (persisted jobs, `StoreStorage` with Durable's conformance suite, the Chord source + over Durable's views). The API points (the host option's name, how a Chord transaction enters the commit step, the + table API in `self.tx`, the Durable view source, the commit benchmark) are proposed in the 6a and 6b specs. +- **The host's store and how it is opened (owner, 2026-10-26; slice 6a point 1):** the host option is `store` + (`createHost({ …, store })`), not `storage`: it is the store itself, and in 6b Durable's `Storage` is `StoreStorage` + over the same store. One entry point, `openStore(backend, ctx)`, with `sqliteBackend(db)` over any `SqliteDatabase` + (Node through `chord/sqlite/node`, Durable Objects, sqlite-wasm later) and `memoryBackend()` for tests; the backend + interface stays internal; `importStore(backend, archive, ctx)` takes the same values. `openStore` runs the store's + own migrations, checks the file is UTF-8, and reads the head, the definitions and the id counters once. The app owns + the store's lifetime: the host never closes it (in 6b the host and the Durable Session share it); the app calls + `store.close()` after `host.close(ctx)`; on `storage_failed` the store stops, the host closes, and the app closes or + reopens. No `openNodeStore(path)` shortcut. + + ```ts + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(join(dir, "session.sqlite"))), ctx); + const worker = createHost

({ address: Session({ id }), principal, store, onError }); + … + await worker.close(ctx); + await store.close(); + ``` +- **A Chord transaction enters the commit step as Durable's does (owner, 2026-10-26; slice 6a point 2: "if this is + Durable's proven flow we should use it here too"; checked against Durable's `#runCommit`):** the object's line ~~(and + the facet's)~~ (facet documents removed the same day) is held from the callback until the step returned; an abort or error before the step discards + everything; then every draft is prepared, `checkpointWhen` picks base or delta per persisted document, and + `await store.commit(writes, adopt, ctx)` runs without the caller's abort signal (once admitted, a commit runs to its + end). `adopt(seq)` is one `host.batch.commit(...)` adopting the persisted documents, the in-memory documents and the + staged changes together (Chord's addition: Durable has no in-memory documents; adopting them apart would show half + a transaction, or publish an in-memory change of a commit SQLite then refused). An error before `COMMIT` returned + rejects the transaction with nothing adopted; an error during or after `COMMIT`, or a failed `adopt`, is + `storage_failed` (the commit stays published if it was adopted). A transaction with only in-memory writes never + enters the queue and commits synchronously as before (W12): no extra await or promise. + + ``` + line -> fn -> (abort/error: discard) -> prepare, checkpointWhen + -> await store.commit(writes, adopt) // queue; BEGIN; revision, version, rows, outbox, head; COMMIT + adopt(seq) = host.batch.commit(() => { persisted; in-memory; staged }) + -> release line(s) -> settle staged -> drafts revoked + ``` +- **Facet documents are removed; persisted data stays in objects (owner, 2026-10-26; slice 6a point 5; reviews + `/tmp/facet-persist-review-{astra,fable,opus,sol}.md`):** cut C16 accepted (supersedes its rejection in "Slice 1 + approved"): a facet has no documents (`FacetSelf.state` goes), so no facet line, no lock order across two lines, no + `Documents.facet` and no `Transaction.#facetLine`; a served object's transaction covers only its own object. Shared + plugin state lives in a `local` store object with empty params, reached through its methods (its own failure domain, + R18 recovers its dependents). Point 5 (persisted facet data) is moot: persisted documents and tables stay only in + `local` objects with empty params (P3 unchanged). Why: the four reviewers found no stated reason in the history (the + frozen kernel let facets persist; P3 was written for objects only), but a facet cannot fail at runtime (slice 2: a + facet setup or `initial()` that throws rejects `load`, closing the host; C5 was rejected because a plugin's always-on + parts would share one failure domain), and persisted data fails at runtime (fingerprint mismatch, throwing `migrate`, + a newer version after a rollback, no store on the host): one plugin's stale document would close the worker, or need + a second failable lifecycle for facets. In-memory facet documents had no real use either: none of p2's 46 `self.state` + calls is in a facet; their only ability, one transaction over an object's and its facet's documents, is a + cross-owner transaction, which the design rejects elsewhere (§9 "Rules across owners"). Other findings kept for a + later facet lifecycle: reads of shared tables need the writer's line; an owner string `facet:` collides with + type names. Built in slice 6a (carry-over). +- **The table API in `self.tx` (owner, 2026-10-26; slice 6a point 3):** (a) the first argument is the table handle + from `self.table(Def)`, never the definition (the handle belongs to one object, so `tx` refuses another object's + table, as it refuses its documents); (b) keys are objects with exactly the key columns, `{ path, entry }`, not tuples + (two swapped `str` columns of a tuple are silent at the type level; other or missing fields fail `invalid/args`, a + trust boundary); (c) reads are async (`await tx.get(t, key)`, `tx.page(t, q, limit, { after? })`, `tx.rows(t, q)`), + writes synchronous and buffered like a draft's ops (`tx.put(t, row)`, `tx.update(t, key, set)`, `tx.delete(t, key)`, + `tx.updateWhere(t, where, set)`, `tx.deleteWhere(t, where)`): they check the row and stage it, no I/O, no line, so a + bad row throws at its own call; the buffer goes into the commit step's backend transaction or is dropped on failure, + like aborted drafts (writing at once would hold SQLite's one write transaction for the whole callback). As Durable + (`setTask`, `settleSubmission` synchronous, reads async) and Chord's `tx.start`. (d) No write shorthands on the table + handle (`edits.put(row, ctx)`) until real code repeats the one-liner. Outside a transaction the handle reads + committed state: `edits.get(key, ctx)`, `edits.page(q, limit, ctx, { after? })`, `edits.rows(q, ctx)`, + `edits.count(q, ctx)`. `tx.get` exists for tables only (documents keep `tx.draft`). + + ```ts + self.tx(async (tx) => { + const old = await tx.get(edits, { id }); + tx.put(edits, { ...row, at: now }); + tx.updateWhere(edits, (e) => prefix(e.path, from), { path: replacePrefix(from, to) }); + }, ctx); + ``` +- **Chunks of `rows` migrations (owner, 2026-10-26; slice 6a point 6):** 100 rows per commit, a fixed constant. + Measured on slayer (`node:sqlite`, WAL, 20,000 rows of about 3 KB, a primary-key `UPDATE` plus an outbox `INSERT` + per row): the total is flat from 100 to 1000 rows per chunk (about 15 µs per row); the step's median is 0.94 ms at + 100, 2.4 ms at 250, 9 ms at 500, 14.5 ms at 1000, 32 ms at 2000, and every other owner's commit waits for it. The + store benchmark measures it again with the real store. Can change later (owner), e.g. a table definition naming its + own chunk size; not built until a case needs it. +- **`lastSeq` instead of "head" (owner, 2026-10-26):** the seq of a store's last commit is called `lastSeq`, never + "head" (Durable's "head" is a conversation's head: head markers, `findLatestHeadMarker`). Every "head" of the store + in earlier entries (S3, S4, S5, S9, "Slice 6 first decisions" (4), "The host's store and how it is opened", the + commit-step trace) reads `lastSeq`; a replica's "applied head" is its `lastSeq`. Renamed in design.md, the store + book and the slice 6a spec. +- **Export archives are operations, the encoding is pluggable (owner, 2026-10-26; slice 6a point 7):** + `store.export(ctx)` yields `AsyncIterable`: the state after its `lastSeq`, written in the outbox's + operations as one synthetic batch from 0 to `lastSeq` (a manifest `{ kind, format: 1, lastSeq, schema }`; `table`, + `index`, `put` per row; `document.create` per live record with its base, plus its later `document.change` records + for `history: "full"` ones (Durable, 6b); an `end` record with counts). Retired documents, blobs, the outbox and sync + state are not in it. `importStore(backend, records, ctx)` refuses a non-empty backend, an unknown format, records out + of order and a missing or wrong `end`, and applies everything in one backend transaction with the replica's applier + (import, bootstrap and batch apply are one code path). Export holds the commit queue until the iteration ends, + throws, is closed or `ctx` aborts (a slow consumer stalls commits; the guide: export to a local file, then upload; + snapshot reads on a second connection only if exports get large, S4). The encoding is pluggable, as the link's + codecs: the store yields and takes records only and ships plain JSONL (`encodeJsonl`, `decodeJsonl`) with no size + optimizations; gzip or a Delta-style dictionary encoder come later as formats without touching the applier (owner: + "then it doesn't matter that the first thing we do is plain JSONL"). Sync is not export: it ships outbox batches as + commits happen (S5); a full archive is needed only where no history can be replayed (a session moved as a file, a + split-off session, a replica restored after divergence). Open for the sync layer: a session that never syncs keeps + every outbox record (trimmed only after Radius acknowledges it), so its outbox grows without bound. +- **State and job names are restricted too (2026-10-27, slice 6a fix round, under the owner's delegation):** + `defineState` and `defineJob` names follow the pattern of object type and table names, `[a-z0-9@/._~-]+`, lowercase, + checked at declaration: they become the documents engine's `kind` and keys of job records, where NUL or lone + surrogates would be truncated or replaced by SQLite. +- **Slice 6a build reviews (2026-10-27, under the owner's delegation to the reviewers' majority):** two rounds of four + build reviews (`/tmp/s6a-build-review{,2}-{astra,fable,opus,sol}.md`), merged into two fix lists + (`/tmp/s6a-fix{1,2}-brief.md`, 28 and 16 items) by majority, a point raised by one reviewer taking that reviewer's + recommendation. Rules they settle beyond fixes of the spec: (1) **members across attempts:** a local object's state + member is followed by its references and by other objects' derives through one member source per object and member + name; when a new attempt goes live every member re-binds by name (one diff frame from the last value, consumers' + derives recompute, a member the new variant lacks fails `invalid/member`), as a remote reference re-holds and gets a + fresh snapshot (Opus B1); supersedes slice-6a §1.2's "derive members' watches end with their attempt". (2) While an + object that was live is failed or activating again, same-host reads and new watches return the last values and get + no frames (was `not_ready` while activating again). (3) A live failure freezes the members only after the commits + admitted before the seal adopted and published once; a retry's factory and hydration wait for that, so no old commit + lands in a new attempt. (4) Re-exposed and forwarded members freeze at the re-exposing object; the provider's source + stays untouched. (5) A transaction cut at the seal while queued for the step rejects `released` at once; line jobs + queued before the seal are refused. (6) The seq, document id and revision counters stop at + `Number.MAX_SAFE_INTEGER` (`storage_failed`). (7) `evolveTable` checks each edit when it is made; import checks + `meta` with the text rules; an export never started rejects `released` after `store.close()`. (8) The migration + yield between chunks: `setImmediate` where the runtime has it, otherwise a `MessageChannel` message, never + `setTimeout(0)` (clamped to 1 ms in Node; a message costs about 0.27 ms per yield there: a 10,000-row migration takes + 70 ms against 100 ms and 191 ms); supersedes round 1's `setTimeout(…, 0)`. (9) Round 1: keyed keys follow the + `str()` rules (the empty key valid); same-host reads of a failed live object return the last value; stuck handlers' + counts, references and presence released at teardown; refusing a new job version on a live object kept for 6a + (slice 7 carry-over); in-memory keyed slots migrate on access. Measured against HEAD (`benchmarks.md`): calls 1-3% + faster (19 promises per call against 20, guarded), watches through a reference at HEAD's level, a document member's + read through a reference 0.25 ns slower, host setup about 3% slower. +- **Derives heal on a re-bind (owner, 2026-10-27: "a all the way, self healing is good"; Opus's F2 in the third + build review):** a consumer's derive that failed on a provider member's earlier error recomputes when that member + re-binds to the provider's new attempt (and every derive downstream of it), so it recovers once its source works + again; watches over it get the recovered value as a fresh frame. Before, a failed derive ignored later changes (as at + HEAD) and recovered only when its own object restarted. +- **Failures are watch calls; derives heal on every change (owner, 2026-10-27: "B", "beautiful, let's do it"):** + extends "Derives heal on a re-bind". (1) A failed derive recomputes on every later change of its sources (and on a + re-bind), so it heals as soon as it gets a value it can compute. (2) A watch stays open through a recoverable failure: + the listener gets one tagged call `{ kind: "failed", error }`, and the next good value as `{ kind: "value", value, + ops: [["r", value]], retired }`; the call is a tagged union (`WatchCall`: `value | failed`; native watches + `NativeWatchCall`: `{ kind: "value", value } | { kind: "failed", error }`), so the compiler makes every listener + handle the failure (an optional `error` field would let `({ value }) => render(value)` render the old value). + Supersedes the listener bags `({ value, ops, retired }, ctx)` and `({ value }, ctx)` of "One retirement mechanism" + (1). (3) Applies to derives, replica members a hold lists as failed, a member a new variant lacks (it comes back with + a later attempt), registry values whose `compose` throws (slice 8). (4) A watch ends only for good: its owner ended, + `stop()` or a cancelled context, a throwing listener, `storage_failed`; Durable's watches fail only for such causes + and only take the new shape. (5) `self.on` handlers are not called for failures (a reconciler acting on an error call + would act on the old value); the routine waits for the next good value; the failure is reported where it happens: + a derive's owner reports `derive_failed { object, member?, error }` once per failure episode (the first failure after + a value or after creation; a heal ends it), a new `HostError` kind (2026-10-27, slice 6a fix round 4: there was no + report for a throwing derive besides a routine's `routine_failed` on its ended watch). One failure call per + episode (a value turning into a failure), not one per source change while failed; a hold keeps only a member's + newest pending failure. Watching a failed source (2026-10-27, slice 6a fix round 5, under the rule "the routine + waits for the next good value"): `watch(ctx)` opens at the last good value and delivers one `failed` call first; + a source that never had a good value rejects `watch()` with its error; `self.on` on a failed source waits for the + first good value instead of ending. + (6) `ref.state` stays a separate watchable of reachability (`loading | live | failed | released`; `ended` since + "One vocabulary, groups 2 and 3"), same words: a + value's watch says whether it can be computed, `ref.state` whether its object is reachable (values stay readable + while loading). No new leaks: an open watch on a failed source holds what it held before and ends with its owner, + reference or context (GC test). Built: the shared `src/watch.ts` upstream in pi first (Durable's listeners adapt), + then slice 6a (points 1-3 except registries; registries in slice 8). +- **One vocabulary, group 1: the shared watch (owner, 2026-10-27: "yes"; vocabulary audit `/tmp/vocab-audit.md`, + A1, A3, A8):** built in the same upstream change as the failure calls. (a) `WatchEnd` is tagged by `kind`, not + `reason` (`end.kind === "failed"`). (b) A watch whose context is aborted ends `aborted`, not `cancelled` (one word + for a requested end, as `AbortSignal`, the `aborted` code and `JobError.aborted`). (c) Durable's synchronous + observers take the watch's shapes: `advance(call, ctx)` with a `WatchCall`, `end(end: WatchEnd)` (Durable never + sends a failure call: its failures are final). (d) A member a new variant lacks behaves the same locally and over a + link: a failure call, the watch stays open (today local `failed`, remote `ended`). (e) Durable's `FileChange` is + tagged: `{ kind: "paths", paths } | { kind: "overflow" } | { kind: "failed", error }`. The audit also found that + "Slice 6a small items" (1) is wrong: Durable's `Storage.close(context)` takes a context (decided with group 2). +- **One vocabulary, groups 2 and 3 (owner, 2026-10-27: "lgtm"; Durable's stored formats stay the same):** + Chord v2 (built in slice 6a): (1) `ref.state` `{ kind: "ended", error }` replaces `released`; the `released` code + gets reasons `ended` (object ended), `closed` (host, store, native), `settled` (`ctx.get` after its call), + `version` (an old handle), `discarded` (a transaction cut at the seal); a holder's own release stays `released` + without a reason. (2) Internal results are `done | failed { error }`; the attempt's `Outcome` is `Activation`, + the jobs' `Ended` is `Settled`. (3) `DocumentSlot.remove` is `retire`; `StoredChange` is `create | change | + retire`. (4) The wire's `cancel` message is `abort`; a native's close reason `stop` is `close`; ~~`job.abort(id, ctx)` + returns `"marked" | "terminal"` (Durable's words)~~ (**superseded**, owner 2026-10-27: Durable's `abortTask` stores a + mark and joins the run, Chord's 6a abort only fires a signal in memory; `job.abort(id)` stays in 6a, and becomes async + and stored with persisted jobs in 6b, its result chosen there). (5) Homonyms: the store's `Staged` is `StagedWrite`, its `Phase` + `WriteOrder`, the host's `KeyedMember` `KeyedBinding`, `KeyedMembers.hold` `pin`, `Presence.origin` `incarnation`. + (6) Context last: `runState.change(fn, ctx)`, `host.route(Kind, options, ctx)`, `store.close(ctx)` (as Durable's + `Storage.close(context)`; supersedes "Slice 6a small items" (1)'s "no context"). (7) `failed` and `gone` carry the + wire error format (`WireError`). (8) `HostError`: `failure` where the value is a `FailureCause`, `error` otherwise; + positions `{ kind: "on" | "onLive" | "native", position }`. (9) Store types as Durable's: `history: "latest" | + "rewindable"`, pages `{ items, next }`, a singleton's address has no key (`""` only in the column). (10) The + listener parameter is `call`; holds `served | relayed`; `put.v` is `version`. Durable's public API (pi upstream, + with group 1): (11) `onReport(report)` with `report.kind` and `report.error`; `session_failed` is `storage_failed`. + (12) `subscribeCommits` is `observeCommits` with the observer shape (`{ value, detach }`); a file watcher stops with + `stop(ctx)`. (13) `CompactionFailure.reason` and `CompactionStatus.reason` are `trigger` (`pi.live`'s definition + bumps its version with a `migrate`). (14) ~~One word for a definition that cannot take the stored state: + `incompatible`~~ (**superseded**, owner 2026-10-27: `task_too_old` is stored in orphaned outcomes and settlements, and the + report's `incompatible_task` covered two cases): the `task_outdated` report uses the scheduler's own words, + `missing_task | task_too_old | migration_failed` (newer without `migrate`: `migration_failed`); Chord's jobs name + their blocked reasons in slice 7. + (15) `TaskInspection` uses `TaskState`'s words (`ready` is `pending`). (16) `DocumentCommitChange` is tagged + (`{ kind: "retire" }` instead of `value: null`). Not done: Durable's stored tags (`status`, `type`; group 4), Durable + throwing Chord's codes (6b), one retirement value ("One retirement mechanism" (2) stands). + Added (owner, 2026-10-27, "lgtm"): Durable's `SessionEnd` and the `ReportFrom` argument of `runtime.report()` are + tagged by `kind` too (neither is stored). +- **Slice 6b D1: `StoreStorage` and `durableView` live in Durable (owner, 2026-10-27: "they are only used if durable + is used"):** in `packages/durable/src/next/`, private-only like `packages/chord/src/next`, importing + `@earendil-works/chord/next` (Chord's `./next` export returns, private-only). No package cycle: Durable already + depends on Chord, Chord imports nothing of Durable (S13). The folder goes upstream with Chord v2. +- **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as + HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within + about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the + owner with numbers. Memory: allocation per operation and memory retained after GC no worse than HEAD beyond noise, + with GC probes for everything that must be collected (ended objects, released references, closed hosts, watches on + failed sources). No super bad regressions; small differences within noise are not chased. Reviews converge: a fix + round adds no new decisions or features; the same reviewers (restored, with their context) re-review their own + findings; when only small items are left, the slice is committed and they are carried over. +- **Size is not a goal by itself (owner, 2026-10-27):** no cut pass after slice 6a: "if the user-facing API is great + and the underlying implementation fast and correct, we're ok". `src/next` is about 13.2k lines without blank lines + and comments after 6a's build (about 8.5k before), against the earlier target of 6-7k for all of Chord v2, which this + supersedes; the bar stays a great API, HEAD-level speed and memory, and correctness. +- **Second spec review: the majority (owner, 2026-10-27: "pick the majority vote for everything, get building"):** the + second review of slice 6a (`/tmp/s6a-review2-{astra,fable,opus,sol}.md`) is decided by the reviewers' majority; a + point raised by one reviewer takes that reviewer's recommendation; ties take the simpler single mechanism. Folded + into slice-6a.md (section 9 and section 15, 27 rules) and design.md. The main points: (1) **migrations by + `evolveTable`** (all four): a base `defineTable` and one `evolveTable(prev, edits)` per version, each materialized and + fingerprinted (`TABLE_FORM_VERSION`), edits `rename add drop index dropIndex json` and at most one data step (`rows` or + `update`), the editor's types ordering them, per version an (A) commit (DDL, `migrating`) and a (Z) commit (last + chunk, later edits, `ready`), recorded as `alter`/`index`/`dropIndex`/`table` writes; fresh tables created at the + newest version; shared with contracts only `hash64`, the normal form per column, `Increment` and the editor-return + check (supersedes the derived adds and drops of "The migration API: the reviewers' consensus"). (2) The store's + admission check through `store.commit(writes, { admit, adopt }, ctx)`. (3) An admitted commit's staged job changes are + adopted into the object (supersedes "The seal fence"'s "staged job changes of the sealed attempt abandoned"); a run cut + off by its attempt's seal ends `aborted { reason: "restart" }`. (4) A transaction at the seal is discarded; pending + async `tx.*` operations tracked as in Durable. (5) Key columns immutable in `set`. (6) Writes serialized before + `BEGIN`; canonical `set` vocabulary with an explicit null for clears. (7) `int` columns kept in the safe range by a SQL + `CHECK` (an out-of-range result in the step is fatal, accepted), the JS evaluator checking the same. (8) The outbox + floor (`read(afterSeq)` valid from the floor to `lastSeq`). (9) The manifest carries `nextDocumentId`. (10) A bounded + LRU for `chord/sqlite/node`'s prepared statements (pi change). (11) Slots keep `valueVersion` and `storedVersion` (as + Durable's `LoadedDocument`). (12) An admitted commit of a failing object adopts and publishes once (Sol, Fable). (13) + Retiring a `latest` record deletes its row. (14) One host per store. The rest: section 15 of the spec. Carry-over to + slice 7 recorded in slices.md. +- **Slice 6a small items (owner, 2026-10-27: "all yes"; from the four spec reviews):** (1) `store.close()` (no + context, as Durable's `Storage.close()`) waits for a running step, rejects queued commits, running exports and reads + `released`, closes the backend; idempotent, also after `storage_failed`. (2) The store tells its host of + `storage_failed` through an internal listener, once; the host reports once and closes itself. (3) An omitted outbox + option keeps the persisted setting; switching needs an explicit `outbox: true` or `false` (supersedes in part "The + outbox is optional": "opening with the other setting is that explicit switch"). (4) `store.outbox` is `undefined` + without an outbox (typed); `read(afterSeq)` before the first stored seq and `trim` beyond `lastSeq` reject; `trim` is + idempotent. (5) **No hand-written memory backend:** tests run on SQLite `:memory:` through `chord/sqlite/node`; one + semantics (`NULL`, text order, the `set` evaluator, ordering), no conformance suite across two backends; every real + target is SQLite (supersedes the memory backend of "The store", S3 and "The host's store and how it is opened"). + (6) Export takes the queue at its first `next()`, an abort listener releases it while the consumer is suspended (a + later `next()` rejects `aborted`); it reads page by page with plain reads, not one long read transaction. (7) Parts in + order: manifest; per table its `table`, `put`s, `index`es; the `document.create`s; `end`; parts after `end` refused; + `importStore(backend, parts, ctx, { outbox? })`, the outbox starting after `lastSeq`. (8) A live object failing is + reported as `object_failed { object, cause }`. (9) `keys(ctx)` is in code point order, in memory too (`compareText`). + (10) A migrated base is written back by the next commit that changes the document (supersedes design §9's "a + read-only transaction right after a version bump writes the migrated base"); until then a load re-migrates (once per + object). (11) Document ids are assigned in the step, monotonic with seq, given to `adopt`. (12) A run waiting for a + live attempt keeps its object alive only while an attempt is live or activating. (13) Chord's own loads take no read + transaction (the object holds its line, so plain reads are consistent); S4's read transaction stays for reads outside + a line (Durable's, 6b); supersedes "Slice 6 first decisions" (2)'s "in one read transaction". (14) `put` is an upsert + (the whole row replaced); `update`/`delete` of an absent key change nothing and are recorded with their count (0). + (15) At write `int` is a safe integer, `num` finite, `bool` stored 0/1; one compiled closed check per table. (16) The + column helpers are L1's (`str int bool num opt`), a restricted subset. (17) Default order by key ascending; the key is + appended to every `orderBy` as the tie-break; one direction per query in 6a. (18) Cursors are opaque strings bound to + table, order columns and direction; decoding them is a trust boundary (`invalid/args`). (19) Generated SQL has a + bounded number of texts (the facade caches every prepared statement): `limit` and cursor values bound, `inArray` as + one JSON parameter through `json_each`. (20) Cut to 6b: the engine's reads at an older seq (`find`/`load` with `at`) + and exporting `history: "full"` (the format defines the fields now). +- **Cold keyed loads on the line; Chord aligns with Durable (owner, 2026-10-27; slice 6a spec review blocker 9):** + a cold read of a keyed document (`get`, `watch`, an `at` subscription) is a job on the object's line, as Durable's + `snapshot` loads on the Session line: it waits for the line, loads, installs and releases it; cached reads take no + line; `tx.draft` loads inside its transaction, which holds the line; one load in flight per key, shared, each + caller's `ctx` ending only its own wait; a load installs its tracker only while the key is still absent (`adopt` + always wins). Why: a load outside the line could install a value from before a transaction that committed meanwhile, + and the next delta would be computed on it. **General rule (owner): Chord and Durable align as much as possible** + (the same rules and mechanisms where they solve the same problem: commits, lines, loads, errors, watches). +- **The migration API: the reviewers' consensus (owner, 2026-10-27; slice 6a spec review blocker 8):** the owner + delegates the shape to the spec reviewers ("whatever the reviewers all like and find consistent and easy to implement + correctly"); only a disagreement among them comes back. Proposal in the spec (Opus's shape, Sol's protocol): adds and + drops derived from the declared versus the recorded columns (a new column `opt(...)` or with a `default`; a dropped + column loses its indexes first); `migrations: { N: { rename?, rows? | update? } }` with renames explicit; a version + bump without an entry is a schema-only change; columns or key differing at the same version fail `broken`; key and + column-type changes (`rebuild`) cut from 6a; `rows` synchronous and pure, chunks of 100, writing changed columns plus + `$v` through an internal write form (`tx.update` never writes `$v`); `update` one `updateWhere` stamping `$v`; one + version at a time (a DDL commit recording `version N, migrating`, the data step, `ready`), indexes diffed after all + versions; resume continues `migrating` skipping rows with `$v >= N`; a `rows` function gets the stored row with + `json(...)` columns migrated on access. +- **Nullable columns and the `set` form (owner, 2026-10-27; slice 6a spec review blocker 6 and the DSL):** columns + are non-null by default; a nullable column is L1's `opt(...)` (`note: opt(str())`): in a row an optional field, + absent when `NULL`, never an explicit `null`. A `put` carries every non-`opt` column (a missing or unknown field + fails `invalid/args`). Queries follow SQL's rules for `NULL` (a comparison with `NULL` never matches, `ne`/`not` + included; ask with `isNull` or `or(isNull(...), ne(...))`), on every backend. Keys, index columns and `orderBy` + columns must be non-null (types and `defineTable`), so cursors are total. `alter` adding a column: `opt(...)` or a + declared `default`. The `set` of `update`/`updateWhere` is a plain object of values (`{ note: "fix" }`) or a callback + getting typed column references (`(e) => ({ hits: add(e.hits, 1) })`), like `where`; `col()` and `value()` are gone; + `undefined` clears an `opt` column. The semantics the reviewers listed (simultaneous evaluation of a multi-column + `set`, integer arithmetic and overflow, `replacePrefix` without the prefix, empty and U+10FFFF prefixes, `inArray([])` + and long lists bound as one JSON parameter, `when`'s condition, default order and key tie-breaks, one evaluator for + `where` and `set`) are specified in the 6a spec; no separate DSL review (owner: the spec review is enough). +- **Restricted names instead of assigned table names (owner, 2026-10-27; slice 6a spec review blocker 5):** names + that become SQL names are restricted at declaration (and checked on import) instead of being mapped. Host kinds, + object types (every contract, not only stores) and table names use npm package-name characters `[a-z0-9@/._~-]`, + lowercase only (SQLite compares identifiers case-insensitively), so plugin-prefixed type names (`@pi/file-history.store`) + stay valid; the separator is `:`, which they cannot contain, so the rendering `"session:@pi/file-history.store:edits"` + is injective (`a/b` + `c` and `a` + `b/c` differ) and readable in `sqlite3`. Index names `[a-z0-9_]+`, rendered + `:`. Column names `[A-Za-z_][A-Za-z0-9_]*` (camelCase allowed), two columns equal ignoring ASCII case + refused at `defineTable`, no `$` (the hidden `$v` and `$v` cannot collide). Import is a trust boundary: every + name in an archive is checked against these rules (no `"` can occur: quoting is always safe), every row against its + table's recorded columns, the parts against one compiled `ArchivePart` schema; a failure refuses the whole import. + Rejected: physical names assigned by the store (`t1`, `t2` with a mapping table: unreadable, owner), escaping logical + names into physical ones. +- **`runCounted` in `chord/sqlite` (owner, 2026-10-27; slice 6a spec review blocker 4):** `SqliteExecutor` gains + `runCounted(sql, ...params): Promise`, running one statement like `run` and resolving with the rows it + inserted, updated or deleted (SQLite's `changes()`: index writes and trigger rows do not count); `run` stays + `Promise`. The store uses `runCounted` only where a count matters (bulk writes' `count`, the engine's + conditional `UPDATE`, single `update`/`delete`). Node takes it from `StatementSync.run()` (free); the Durable Object + driver reads `SELECT changes()` after the statement, because the cursor's `rowsWritten` is Cloudflare's billing + count and includes index writes (verified in workerd 2026-10-12: an insert with two indexes 3, a 10-row bulk update + 20, `changes()` 1 and 10). Measured in workerd: `SELECT changes()` costs about 1.5 µs per statement and reads or + writes no billed rows, so making every `run` count would nearly double a small write on Durable Objects for callers + that need no count (Durable's `SqliteStorage`): hence a separate method (option 2, owner). Name `runCounted` + (owner). pi `8025c414d` (upstream first), with a real Durable Object test in Durable's workerd suite. +- **Store errors: Durable's rule (owner, 2026-10-26; slice 6a spec review blocker 3):** any backend error in a commit + step is `storage_failed` (the store stops, every later commit and read rejects `unavailable/storage_failed`, a new + reason in the protocol list; the host closes; `onError` once), as Durable's `GuardedStorage`: the SQLite facade cannot + tell a failed statement from a failed `COMMIT`, and what reaches the step is a real storage problem or memory and disk + disagreeing. Plugin mistakes never reach the step: they are refused when the write is staged (`invalid/args`). A + commit refused by the seal fence is rejected `released` before `BEGIN` and is no error of the backend. A backend + error on a read is `storage_failed` unless the read's `ctx` was aborted (`aborted`); invalid arguments are checked + before the backend (`invalid/args`). An `adopt` that throws is `storage_failed` (owners never throw from it). + Accepted: integer overflow past 2^63 in a bulk `add` is fatal (a `STRICT` integer column refuses SQLite's REAL). + Supersedes the 6a spec's "an error before `COMMIT` returned rejects that commit". +- **A live object can fail (owner, 2026-10-26; slice 6a spec review blocker 7):** a runtime failure of the owner (a + `migrate` that fails on access, a stored version newer than the code) seals its live attempt (the seal fence) and + makes the object `failed` with a `broken` cause: calls on it reject at once with the cause, its state members keep + their last values and get no frames, its natives close, its documents stay on the object (the failed data intact in + the store), `onError` gets one report (the kind is settled in the 6a spec; `object_failed { object, cause }` + proposed); recovery only by a reload or `ref.retry()` (a new attempt; only the failed load is redone). Nothing + cascades: dependents that are live stay live (§6), only their calls into it fail. +- **A native restart drains first (owner, 2026-10-26; general-model discussion):** when a reload restarts an object + with natives, new calls park, the running calls are awaited (the drain, bounded by `closeTimeout`), then the natives + close (reason `reload`), the new factory runs and the natives reopen; only a call that outlives the drain is cut off + by the seal fence. Supersedes §21's "running work ends `aborted`" for calls (running job runs: slice 7 decides with + the job ownership change). The general-model discussion is closed with this entry (owner: "model discussion done"). +- **Keyed state members, `local` contracts only (owner, 2026-10-26; reviews + `/tmp/keyed-members-review-{astra,fable,opus,sol}.md`, all four for A):** the gap: across objects live state travels + only as state members, one value per object, so a view with params (`NoteView { session, id }`) could not follow one + keyed document of a persisted store (P3 keeps persisted data in `local` objects with empty params). A contract member + may be `keyed(schema)`, a third member kind (part of the fingerprint), only in `local` contracts (`remote()` refuses + it): a provider returns a keyed document handle for it (in memory or persisted) or re-exposes another reference's + keyed member; a consumer on the same host takes `ref.member.at(key)` in its factory (only there: a counted pin on the + provider's per-key source while the consumer lives, loaded before the consumer goes live and never evicted while + pinned) and `ref.member.watch(key, ctx)` / `get(key, ctx)` anywhere (`watch` on an absent key gives `undefined`). An + absent key reads `initial()`; a retirement arrives as the `retired` flag; a key whose `migrate` fails breaks the store + (6a's keyed rule). Nothing on the wire changes: tabs and other hosts hold the view, an ordinary remote object with + singleton members. A derive over a retired source carries the `retired` flag on (a reader's redacted member learns of + the deletion). Versions of keyed members are mapped per key in slice 7 (`"refuse"` only for removed ones). Rejected: + B (persisted `local` objects with params: one object per entity, nothing lists or retires across them). Kept for a + real case: keyed members across links (Fable's design: keyed entries inside a hold's `bound`/`state`, a `keys { hold, + add, drop }` message, re-holds carrying their keys; or Sol's selector holds), as a later slice. Option C (a singleton + map plus a lookup derive, p2's `DraftStore`, `Status`) stays for small collections (§12 cookbook). Built in slice 6a + (about 300-350 lines), which already builds the per-key source behind `handle.at(key)`. +- **The seal fence (owner, 2026-10-26; the fence review `/tmp/fence-review-{astra,fable,opus,sol}.md`):** leftovers of + an ended or replaced attempt (handlers past the drain, queued commits, routine calls, watch deliveries) are cut off + by one token per attempt, and one per job run invocation for its state changes and outcome. (1) Sealed handles: + after the seal, new work through the attempt's handles is refused at use with `released` (`self.tx`, `ctx.get`, + which checks the abort signal first, a job start, a routine start); the attempt's references stay usable for its + own natives' closes, then are released. (2) Commits are checked at two points: when they enter the store's commit + step (sealed: rejected `released`, nothing written; not a storage failure) and at adopt (admitted while current, + sealed during the SQL: durable, adopted into the object's documents, staged job changes of the sealed attempt + abandoned); in-memory commits are checked inside the batch. (3) At the seal, a transaction of the attempt that has + not entered the step yet is aborted and the object's line released; a step already started finishes before the + attempt's teardown completes. (4) Cleanup still runs after the seal (releasing what stuck calls hold, `ctx.get` + references, presence entries, `host.activity` counts), so a stuck handler cannot keep a worker busy forever. The + existing start guards stay (an old activation finishing after the object ended, `Job.#invoke`, a `ctx.get` resolving + after its call settled). Unit: the attempt (every way an attempt ends already goes through `Attempt.seal`); a pure + swap is not a replacement (the same attempt continues), so a stuck old routine call past the handover is fenced per + routine invocation (slice 7). Natives' effects outside Chord are the plugin's; their `close` is honoured, and their + writes to Chord state go through transactions and are fenced. Cost: one boolean read per commit at one (in memory) or + two (persisted) points. Replaces slice 6a's per-path rules ("`adopt` of a dropped handle is a no-op", race 4, staged + job runs adopted after `Job.end()`); `Documents.seal`/`assertWritable` become the token's check. +- **Job runs belong to the object; singletons stay resident (owner, 2026-10-26; the four fence reviewers' + follow-ups):** (1) A job's records, keys, active count, `running`, run state documents and pending actions belong to + the object (per job name), with its documents; the attempt keeps the job's implementation (`run`, `finish`) and the + running invocations (its drain, its seal). A run's state changes and its outcome are fenced by the run's own + invocation, not the attempt; `finish` runs on the object's current live attempt (a finishing run waits for one); a + run whose job the new code no longer declares stays `blocked`. Why: with runs per attempt a native restart breaks key + dedupe (a run can start twice), `concurrency` (undercounts), `running` (loses its runs), the state documents (a dead + attempt's cleanup would clear them) and §15's rule that the current code finishes a run aborted by a restart. Open + for the spec: whether a run waiting for a live attempt keeps a `broken` object alive (proposal: it counts only while + an attempt is live or activating). (2) Persisted singletons and `at(key)` documents stay in memory while their object + lives; only keyed persisted documents are evicted. Why: `handle.value` is a synchronous read (methods, derives, + snapshots for new holds), so the value must be in memory when read; an evicted singleton would need an async read, + as keyed documents have. Memory is bounded by the number of declared singletons, one value each; data that grows + belongs in keyed documents or tables. +- **Documents and the line belong to the object, not the attempt (owner, 2026-10-26; general-model discussion after + the slice 6a spec reviews and the fence review `/tmp/fence-review-{astra,fable,opus,sol}.md`):** an object (one + identity) owns its documents (in memory and persisted, their trackers) and its line for its whole life, across + retries, failures and native restarts; an attempt (one try at activating the object: factory, references, natives, + members, derives, routines) keeps only what belongs to one try: its natives, routines, derives and its seal token. + A new attempt's factory gets handles to the same documents (`self.state(def)` by name). Persisted documents load + once per object, and again only after a load failed (a `migrate` that needs fixed code). In-memory state survives a + failure and retry too; ending the object clears it. Why: per-attempt documents let a late commit of a dead attempt + leave the next one on a stale base (the review's race), contradict §9 ("in-memory documents survive native restarts + of a live object"), and were an accident of slice 2 (each activation a fresh instance, documents created inside it). + Built with slice 6a (carry-over), which needs it; replaces slice 7's carry-over "run records and state documents + belong to the object across attempts (documents belong to an attempt today)" for documents. Decided the same + day: the seal token's rule ("The seal fence") and job runs ("Job runs belong to the object"). +- **Writes carry their versions; archives are lossless (owner, 2026-10-26; spec review blocker 1, all four reviewers):** + every `put` write carries the row's `$v` and each `json(...)` column's version explicitly, in the outbox and in + archives alike (`{ kind: "put", table, row, v: 3, columns: { meta: 1 } }`); the applier never infers a version from + the table definition (an archive's `table` write has the current version while cells may be stored at older ones; an + inferred stamp would label an old shape current, and an export runs no plugin code, so it cannot migrate). A `table` + write carries the table's `state` (`migrating` | `ready`), so an export between migration chunks imports as + `migrating` and resumes. A `document.create` carries the record's stored version and fingerprint; for + `history: "latest"` its base is the current value (base plus deltas, as a load gives it); for `history: "full"` + (Durable, 6b) the archive carries every revision with its seq, retired `full` records included (format 1 defines the + fields now, 6b builds them); retired `latest` records stay out. Supersedes in part "Export archives are operations" + ("a `document.create` per live document record (with its base)", "Retired documents … are not in it"). +- **Slice 6a's three code questions (owner, 2026-10-26):** (1) keyed reads take a context: `handle.get(key, ctx)` and + `handle.keys(ctx)` for every keyed handle, in memory too (one signature; persisted, they read the store; "Contexts + are required"). (2) The activation attempt's `readyTimeout` deadline does not run while an eager migration step runs + (a `rows` step over 10M rows takes longer than 60 s; the scan resumes after a crash anyway); getters still time out + by their own `readyTimeout`. (3) A host without an address uses `client` as its host kind in store addresses (owner + strings, table names), so a browser tab with sqlite-wasm can have a store later. +- **The commit-step benchmark (owner, 2026-10-26; S3's condition "SQLite must not get slower"):** in 6a + `test/next/bench/store.ts` measures the store without outbox against Durable's `SqliteStorage` per commit on (a) a + commit of one document delta, (b) a commit with one table `put`, (c) 1,000 commits back to back, in bursts (median + and p95 over several runs) with memory after GC; each must be no slower. The outbox's extra cost per commit is + reported separately. A failed gate goes to the owner with numbers and options before the build continues. 6b repeats + the gate with `StoreStorage` on Durable's own storage benchmark. +- **The outbox is optional, internal and versioned (owner, 2026-10-26):** supersedes in part "The store" and S1/S5 where + every commit writes the outbox. (1) A store has an outbox only when opened with one, `openStore(backend, ctx, + { outbox: true })`; the setting is persisted in the store's metadata. Without it the commit step writes data and + `lastSeq` only (tests, web tabs, a server's instance store, a session before sync). Turning it on records "outbox from + seq N+1" and the sync layer starts from an archive at N; turning it off drops its commits (a later start needs a + snapshot again). Opening with the other setting is an explicit switch with these rules, never a silent gap. (2) The + outbox is a table in the store's own backend, written in the commit step's backend transaction: SQLite commits data + and outbox atomically even with `synchronous=NORMAL`, for one `INSERT`. An external outbox would work only as Durable's + JSONL storage commits across files (outbox entry first, SQLite `COMMIT` as the commit point, recovery cutting entries + above `lastSeq`), only for a local, truncatable outbox (never Radius: an uploaded entry cannot be taken back), and only + with an `fsync` of the outbox before every `COMMIT` (otherwise a power loss keeps the commit and loses its entry, which + no pull-back recovers); can be retrofitted later if needed (owner). (3) No pluggable outbox encoding: the store keeps + commits as it likes and exposes them typed, `store.outbox.read(afterSeq, limit, ctx)` -> `{ seq, writes }[]` and + `store.outbox.trim(throughSeq, ctx)`; the sync layer encodes for the wire. (4) Versioned on two levels: the outbox + table's layout is part of the store's schema (the store's own migrations at `openStore`); the write vocabulary has a + `format` number, the archive manifest's `format`, stored per outbox commit (`outbox(seq, format, writes)`); + `outbox.read` hands out the current format (older commits through the store's pure step functions), a newer format + than the code is refused with the data intact, and batches and archives carry it. (5) The benchmark gate (S3) is + measured without the outbox; its cost per commit is reported separately. Replaces the sync-layer carry-over "outbox + growth without sync". +- **Writes, commits, batches, archives (owner, 2026-10-26):** the names of what the outbox, sync and export carry. A + **write** (`StoreWrite`, tagged by `kind`, Chord's rule for tagged unions) is one change of the store: `put`, + `update`, `delete`, `updateWhere`, `deleteWhere` (with `count`), `table`, `alter`, `rebuild`, `index`, `dropIndex`, + `document.create`, `document.change`, `document.retire` (as Durable's `StorageWrite`, "one record or document + mutation in an atomic storage commit"; S17's "five writes"). A **commit** is the writes of one seq as the outbox keeps + them (`{ seq, writes }`); a **batch** the commits `from..to` that sync uploads and a replica applies; an **archive** a + manifest, the writes from 0 to `lastSeq` and an `end` (its parts are `ArchivePart`). Not "record" (document records, + run records, Durable's records), "op" (Delta ops inside document changes), "entry" (Durable's conversation entries) + or "change" (`WatchChange`, Delta's `Change`). Earlier entries' "outbox record" reads "the outbox's commit" and + "operation" reads "write"; the book's `op:` tags read `kind:`. `StoreStorage` (6b) turns Durable's `StorageWrite`s + into `StoreWrite`s. +- **The `json(...)` column helper (owner, 2026-10-26; slice 6a point 8):** `json(schema, { version, migrate? })`. + The schema is required (reads are typed from it; a migration's result is checked against it once, R8; written values + are not walked, R9); `version` is required, as for `defineState`; `migrate(old, from)` is optional and pure: without + it, or when it throws or returns an invalid value, reading an older value is refused, the data stays intact and the + owning object fails `broken` (R8, R15). New: each `json(...)` column's version and schema fingerprint are recorded in + the table definition (the `table` write); at activation, the same version with another fingerprint fails the object + `broken` (D9; one comparison per column per activation, none per row). A read does not write back: an old row pays + the migration and the check on every read until its column is written; an eager `rows` step rewrites a table at once. +- **Facets only declare (owner, 2026-10-26: "remove facet routines, everything done through objects seems sensible"):** + facet routines (`self.on`, `self.onLive` in `FacetSelf`, added in slice 5 because §16 listed routines) are removed + with the documents. A facet's setup declares only: `host.serve`, `self.contribute` (slice 8), `host.references`; + no natives, documents, routines or jobs, so a facet has no runtime behaviour and cannot fail after `load`. Without + documents a facet routine owned nothing it could write and acted only through other objects' methods; standing + behaviour belongs to the object whose state it reconciles. p2 already works so: all 63 facets are + `defineFacet((host) => …)` with no `self`; its routines and loops live in host-lifetime objects. Built in slice 6a + (carry-over): the routines in `FacetSelf`, their start in `host.ts#setUp`, their drain at close, tests, the facet + part of example 18. +- **Relays do not check answers (owner, 2026-10-24; second 4b review):** supersedes the third fix-round choice. A + relay forwards answers from upstream (`bound`, `state`, results) unchecked and unparsed; the end that parses a part + checks it against its own link's limits. Reason: the relay's check had to release the hold upstream and hold it + again with the same id on `retry`, and an old `state` or `gone` still in flight then looked like the new hold's + (reproduced by three reviewers). The relay still checks everything arriving from downstream. A reference failed + `too_large/part` leaves its path (a reconnect does not hold it again) until `ref.retry()`; this lives only in the + consumer's memory. +- **Slice 4b fix round (owner, 2026-10-24):** the builder's five choices accepted and folded into design.md: call + arguments sent up a parent link are never compressed (a host cannot tell whether its parent sees it as a child or a + route); `revoke` goes on route links before revoked child links close (worker objects end `forbidden/revoked`, not + `released`); ~~a relayed answer over the route link's wire limit fails its call, or releases its hold upstream and + sends `loading` and a broken `failed` downstream (`retry` re-holds with the same id)~~ (**superseded** by "Relays do not check answers"); compressed hold params are a + protocol error on every link; the asynchronous inflate runs one part at a time (its queue is bounded by the parent's + or route peer's window; a misbehaving one of the app's own hosts is accepted). No extra performance regression tests + (cost guards, GC tests and manual benchmarks suffice). A second, non-pedantic review (correctness and performance) + before the commit. +- **Slice 4b review decisions (owner, 2026-10-24; reviews `/tmp/s4b-review-{astra,fable,opus,sol}.md`):** (1) a + dependency failing with `invalid/cycle` makes its dependent broken (exception to R18). (2) Cycles that enter an + already activating object (simultaneous starts) are a documented limitation: they time out transient and retry; no + probe message. (3) Child links refuse compressed parts (protocol error): clients never send them, and a small part + could inflate to 64 MiB on the server; asynchronous inflate then happens only on a host receiving from its parent. + (4) The text codec tags every part, so no JSON value is reserved for compression. (5) A departure discovered while a + batch is sending lands in the next batch (one commit stays one message per link). (6) Presence `origin` is one rule: + the caller's incarnation, also for `ctx.get`. (7) Revocation's error is `forbidden` with a new reason `revoked` (a UI + can say "access revoked"; the protocol table changes, version stays 1 until p3 ships). (8) Compression details: a + tagged adapter result (too large or corrupt), compression also on IPC, `./next/node` export later, hold params never + compressed. (9) `host.references` keeps a type's list from its first ask (entries come and go with references). + (10) Instance ids on restart are slice 7's (carry-over). Revocation propagates over route links; the decoded limit is + stamped on holds only (design §4, §22 aligned). +- **Slice 4b to building (owner, 2026-10-24):** `slice-4b.md` revised for flow control per origin goes to a builder + directly; fresh reviewers (Astra, Fable, Opus, Sol) review the build; the owner then decides the open points the + spec marks as proposals (a cycle error makes the dependent broken; compression as built; `host.references` lazily + per type). Compression is built as option (b) (owner leaning, decided after the build): hosts with a synchronous + compressor compress parts above 16 KiB at their origin; browsers send uncompressed and inflate asynchronously with + `DecompressionStream`, holding later messages of the link in order (bounded by the window and N; acks after apply). +- **Slice 4a approved (owner, 2026-10-24):** `packages/chord/docs/next/slice-4a.md` as revised after the reviews. Build + order: the pending queue in Chord's shared code first (pi, upstream), then the slice by a fresh Opus builder (`s4`). +- **Slice 4a review decisions (owner, 2026-10-23; Astra's and Fable's spec reviews, `/tmp/s4a-review-*.md`):** + supersedes in part "Slice 4 split and first decisions" (resets as `r` ops). A `hold` carries the address the caller + computed; relays route by address and never need the contract or parse params. A `hold` carries `principal?` and + `incarnation?`, honoured only on the parent link (calls inherit their hold's, so calls carry none; refines R1/Q13 + "hold and call"); a reference made on a host sends its maker's principal over route links (`ctx.get` in a guest's + handler acts as the guest); upward, a server ignores a child's fields (a worker acts as itself; documented + boundary). Resets deliver full bindings: `bound` carries a list of holds (`bound { holds: [...] }`, one batch), a + `bound` on a bound hold replaces its replicas (another variant allowed; moved from slice 7 to 4a); overflow stays an + `r` op. A provider answers `reset` through its pending path (never exceeding K). A relay recovers per (downstream + link, upstream link): once something from U cannot reach D, everything from U to D is dropped, and the next free slot + requests one reset of every hold D has on U; an answer that finds no slot is requested again. Heartbeat: one + `lastReceived` timestamp per link and one host timer; the 30 s silence covers the handshake. Transports: a connecting + WebSocket buffers; IPC `send() === false` is backpressure. A hold from the parent is never routed back to it. +- **Slice 4a spec decisions (owner, 2026-10-23; `packages/chord/docs/next/slice-4a.md`):** a heartbeat on every link + kind, in the protocol layer (`ping`/`pong`; a side that received nothing for 15 s pings, a link silent for 30 s + closes; host options with these defaults); supersedes design §3's "heartbeats are the adapter's policy". The text + codec may carry relayed links (one parse per part per hop, accepted for a debugging codec). `resolve(address, + context)` takes a Context. Upstream reconnects at a relay are `loading` downstream, then the new `bound` (3b's + re-hold path, another variant allowed); flow-control resets stay `r` ops. +- **Compression at the origin (owner, 2026-10-19):** a JSON part above a threshold (e.g. 16 KiB) is compressed once + at its origin, whatever the link; byte segments stay raw; relays forward it compressed; IPC carries compressed parts + too. No per-link compression. +- **R6 and contract compatibility (owner, 2026-10-18), strict, like documents:** a hold carries only the consumer's + contract version and fingerprint, never a description. The provider binds when they equal its current version and + fingerprint, or an accepted old version's fingerprint, computed from the provider's own old schemas. Every contract + change, additive ones included, bumps `version`. (**Superseded in part** by Q10: forward evolution in code and + `checkAccepts`; old versions come from the materialized history, not from copies in `accepts`.) `accepts[v]` carries the provider's own copy of version v's member + schemas plus value mappings: old arguments are validated against the provider's old schema, then mapped (state: + new to old; methods: old arguments to new, optional result mapping back); an additive change needs the old schemas + and no mappings. Anything else is `contract_mismatch` (or `updating` during a rollout). Consequences: no + descriptions on the wire (the `describe` message kind of R1 and the per-link description cache go away), R3 is moot, + the coverage check across fingerprints of one version is gone. + +## Portability boundaries in the kernel + +From `research/chord-on-durable-objects.md`, adopted for the Chord v2 kernel because they cost little and help tests: + +- **A host is a logical identity plus an incarnation**, not an OS process. +- **The loader is a host adapter:** the kernel also accepts facets imported ahead of time (static composition, as on a + Durable Object or in tests); loader D is one adapter. +- **Values are JSON plus bytes, defined independently of the transport;** (**superseded in part** by Q4 (IPC carries the same binary message, V8-serialized bytes)) IPC structured clone and WebSocket binary + frames are carriers, not the definition. +- **Durable plugin state:** whatever must survive a host teardown (a DO eviction, a crash, a restart) is a persisted + document written in transactions (`DocumentStorage`). Everything else is rebuilt on activation: in-memory documents, + natives, derives (`previous` is a cache), routines, presence and references (re-held by clients); job runs end + `lost` unless their records persist. + +## Object kinds + +- **`local` stays** next to `remote` (owner, after the plugin scouts judged "one kind only"; all four would have + accepted it with conditions). +- **`local` means:** (**superseded in part** by R7 (validation for every caller; `local`'s exception is Q15, open) and R9 (no dev freeze)) same host, no validation, no copy, no principals; plain data only; **no `opaque`, no callbacks** + as arguments. Behaviour goes through registries, multi-step work through jobs, input through flows (or Questions), + runtime services through host values (**superseded:** host values were removed; runtime services are core objects, + "Hosts and placement" and "Review round 1"). Without a copy caller and object share values: neither may mutate arguments + or results (the playground freezes them in dev mode to check). +- **Kernel semantics decided while building step 2:** + - `local` factories are `(input, self)`; a local method's `ctx` has no principal; references a local object makes + run as the host principal; attribution is passed as data. + - ~~Calls on `remote` types validate and copy arguments and results also on the same host.~~ **Superseded** by "The + state path": same-host calls pass values by reference; arguments are validated only for callers that are not + host principals, at the provider. + - (**superseded in part** by R9 (no dev freeze)) `Uint8Array` leaves are never copied in process (shared, immutable by convention); the dev freeze of `local` + values freezes objects and arrays only. Transports carry bytes raw (IPC structured clone, WebSocket binary + segments), never base64. + - (**Superseded** by Q12: unknown fields are rejected.) Undeclared argument fields are stripped as in today's Chord: the consumer strips to its schema, the provider to + its own, then validates; a handler receives exactly its declared shape (additive rule; no mass assignment). + - A failed reference stays alive: `ref.state` is a tagged value (`loading`, `live`, `failed` with `cause` and + `retryAt`, `released`); while held, Chord retries per the per-address backoff; `ref.retry()` tries at once; calls + on a failed reference reject at once. Calls park on link drops, loading or restarting objects and rollout + `updating`. +- **Pattern: shared core, per-principal views** (confirmed; renamed "shared store" in "The state path"). Identity stays (type, input, principal key): the + factory receives the principal and builds its variant and principal-specific state (e.g. a read-only guest's + filtered view). Heavy or shared parts (a Durable watch, a prompt watcher, a sandbox, a store) live once in a + `local` core object (no principals: one per (type, input) on the host; no copy); per-principal remote objects + `self.get` it and derive their view from it. Same for stores that serve different views per principal. +- From the scouts' review of "one kind only", still valid for `remote`: same-host state as immutable snapshot plus + deltas (never a full clone per change); bytes transferred, not cloned; real schemas instead of `Type.Unsafe`; double + copies through facade layers are fixed by merging owners. + +## Reload + +- **Per host:** load (bundle; a load error rejects) → staging (run the facet setup and every live object's new factory + with a recording `self`, declarations only, nothing activated; a throw rejects) → commit (atomic) → after. +- **Rollback is all-or-nothing per host:** any load or staging error leaves the host entirely on its previous code. + A host is completely at revision R or R−1, so status per host suffices. +- **Pure objects** (**superseded in part** by R15 (old code touching a document migrated past its version throws)) (no `self.open`, old or new) are swapped in place: new methods and derives; documents, references + and jobs stay; routines restart serialized; new references become live; then the object's gate opens. Running calls + finish with their old code. +- **Native objects of the plugins in the reload restart:** running work in them is aborted (outcome `aborted`), + natives close, the factory runs, natives reopen. ~~No per-module digest: if a native is expensive to restart, the + author splits it into its own plugin.~~ **Superseded** by R16 (restart patterns: drain in the background, swap + through a registry). An open that fails makes that object `failed` with backoff; the rest of the + host runs on the new code. +- **Remote consumers** of a restarting object see what they see for any object that briefly goes away: `loading`, + values readable, calls park, re-hold; calls running on the old instance end with an error. +- Rollback covers code errors (load, setup, factories); activation errors (opens, new references, migrations) stay per + object; a bug in a method fails that call. + +## API additions + +- `self.derive(name, sources, (values, previous) => …)`: `previous` is the derive's last value, kept across factory + re-runs by name. +- `job.start(args, ctx, opts)` returns a run with `outcome(ctx)`: the caller can await the outcome; the caller's abort + ends only its wait, never the run. +- Jobs stay as designed (intent / run / finish). Jobs as objects with an "until done" lifetime are an option for later. +- **Bytes are a schema type** (**superseded in part** by Q4 (IPC carries the same binary message)) (`Uint8Array`), transported natively: structured clone on IPC (`fork` with + `serialization: "advanced"`), binary frames on WebSockets. No base64 in JSON. (Today an IPC payload is JSON, deflated + and base64-encoded; a 1 MiB chunk of compressed data grows by a third or more.) +- **Large transfers are chunked by the owner, not by Chord:** the owning object offers `begin` / `append({ upload, + offset, bytes })` / `finish` and `read({ id, offset, length })`, chunks of about 1 MiB. Each side holds one chunk; + the awaited `append` is the backpressure; after a link drop the writer asks for the stored offset and resumes; an + unfinished upload is removed after a timeout by the owner. A small helper wraps the loop. Chord gets no stream type + (it would need its own rules for link drops, cancellation, relays and restarts); reconsider only if many owners + repeat the protocol. + +## Registries and HTTP + +- **Registries work like Durable's** (option c): (**superseded in part** by R21 (only facets and `lifetime: "host"` objects contribute) and "Changing contributions, `self.writable`") each contributor (a facet or an object, by its identity) contributes + at most one value per registry; the table is an immutable, ordered list of `(contributor, value)`; Chord has no + generic keys and no `conflict` flag. The owner composes the table (in the registry's `compose`, below) and decides + conflicts and order (**superseded:** composition runs at runtime and reports conflicts, D1 in "Review round 1"). Chord ships helpers for the common compositions + (`keyed(...)` with a conflict rule, `ordered(...)`). Several contributors (model providers, environment kinds) are + the owner's composition, not a special case; selection (which contributions apply where) stays possible. +- ~~The composition runs in staging and a conflict rolls the host back.~~ **Superseded** by D1 ("Review round 1"): + `compose(table, { report })` runs at runtime, always delivers a value and reports conflicts. + +- **Registries may carry natives.** An entry lives exactly as long as the object or facet that registered it; the + table changes when it goes away. Consumers look up at each use or follow the table and keep nothing from an entry + beyond a use (a pinned Durable phase holding an old table is fine). Natives reach other plugins only this way: no + `opaque` members, no natives in method results. What a native's old value does after its owner restarts is the + owner's `close` (a model runtime keeps working until collected; a closed SSH connection fails its in-flight `exec`). +- **HTTP is an optional Chord component** (`@earendil-works/chord/next/http`): a route gets a plain request summary + (method, path params, query, principal, selected headers; parsed and capped JSON bodies; uploads handed over in + chunks) and returns a response description (`json`, `page`, `bytes` with `size`, `mime` and `read(offset, length)`, + `redirect`, `error`). The component does ranges (206/416), HEAD, download headers, streaming, abort on disconnect and + the CSP nonce once, instead of every plugin. Request resolvers (who sent a request) are a hook taking the summary. + A **channel component** provides WebSocket byte channels (`send`, `onData`, `close`) for terminals. +- **HTTP routes and wraps (middleware):** a contribution holds `routes` (pattern to route) and `wraps`. Routes are + composed with `keyed` (two contributors with one pattern: a reported conflict, D1). Wraps run in **phases the owner defines** (e.g. + `observe`, `headers`, `transform`; outermost first), within a phase in contributor order; no numeric priorities + (plugins have no single registration order, unlike Express or Hono; Fastify's lifecycle phases are the model). A wrap + applies by default only to its own contributor's routes (`scope: "own"`, Fastify's encapsulation); `scope: "all"` + is explicit; `route` narrows it to one route, and a `route` that does not exist is a reported conflict (D1). Wraps + may add a security boundary on top of the owner, never instead of the owner's own checks. A request uses the router + as it was when the request started. + +## Durable harness inputs + +- **`HarnessOptions.models` and `settings` accept a function** (`() => Models`, `() => HarnessSettings`), like + `env`; a plain object is still accepted. Internally both are stored as functions; generation, hooks and tools get + the function's current return value. Replaces p2's `Proxy` around the registry entry. Done in pi main first, then + merged into `private`. +- Models and environments reach the harness through registry entries (functions) registered by the objects that own + their natives; the `Harness` object opens Durable itself (no host value) and looks the entries up at each use. + +## Input into running work + +- **Questions (`questions.md`) is one possible pattern, not the required one.** Another: the owner of a flow keeps it + as state keyed by a uuidv7. Example login: `start({ provider })` returns an id; the flow's live state holds the + current step (`{ id, kind, ... }`), notices and status; `answer({ id, step, value })` resolves the waiting step (a + late or second answer to a step that is gone gets "no longer active"); `cancel`. The client watches the state, so no + `next()` is needed. pi-ai's promise-based `prompt` callback stays inside the owner (the job keeps the resolver); + nothing crosses an object boundary. Open: the flow as a keyed document of `Login`, or an object `LoginFlow {id}` that + lives until done ("jobs as objects"). +- **Imperative consumers get a client helper over the same flow state,** (**superseded in part** by design section 20 (flow state is tagged by `kind`, not `status`)) not a second server protocol. Tests, + scripts, a CLI or an agent tool loop over the steps; the web UI watches the state; both see the same steps: + + ```ts + async function* steps(flow, ctx) { // generic: any flow with { status, step?: { id, ... } } + let last = 0; + for (;;) { + const s = await until(flow.state, (v) => v.status !== "running" || (v.step && v.step.id > last), ctx); + if (s.status !== "running") return s.status; // done | failed | cancelled + last = s.step.id; + yield s.step; + } + } + + const { id } = await login.start({ provider }, ctx); + for await (const step of steps(await ctx.get(LoginFlow, { id }), ctx)) + await login.answer({ id, step: step.id, value: answerFor(step) }, ctx); + ``` + + Why not a long-held server `step()` call: a link drop fails it and the consumer cannot tell whether it missed a + step; the helper reads state that re-holds by itself, and `last` keeps steps from being lost or repeated. The + same shape fits login, tool approval and a question from the agent. + +## Ownership principle + +- **Every thing has exactly one owning object: the smallest object it belongs to.** The owner holds its data + (documents), enforces its rules (in its methods) and offers the operations others need. Nobody reaches past the + owner into its data, writes its files or documents, or attaches rules to it from outside. +- Ownership nests: the session owns the set of its conversations and its location (creation, relocation); each + conversation owns its content and rules (read-only, follows the session cwd) and enforces them in its own methods. + Rules set at creation are written in the creating commit (Durable `createConversation` with `init`). +- Methods return data, never objects; an object is handed out as its address, and the receiver gets it with + `ctx.get(Type, address)` (identity, lifecycle and principal stay with Chord). +- Registries are extension points that add behaviour (tools, routes, renderers, commands, send routers, input gates); + they never guard owned data. +- Core owners are always present; a plugin owner may be missing (optional dependencies are registries owned by the + dependent, "Cuts and closed deferred points"). +- Risk to avoid: a core god object. Each core owner owns only what belongs to it. + +## Porting guidelines + +- **What core enforces on data belongs in a format core owns,** written by the plugin when it creates the data; a + plugin's private document is read only by that plugin. Example: subagents/agent mark one-shot and cwd-following + child conversations in a core per-conversation policy document (`readOnly`, `followsSessionCwd`) in the creating + commit; chat and relocation read it directly; `ReadOnlyConversations` and `SessionCwdFollowers` go away. (This was + plugin design, not a missing mechanism: today the rule is lost when subagents is not loaded.) + +## Still valid from `target-design.md` + +Documents and the store (async `tx.draft`, keys, tags, eviction, Durable's transaction rule), job sandwich, routines +(`self.routines.on/every/loop/custom`), registries (versioned, immutable tables, no late binding), the +Questions service (`questions.md`), retry triggers and `retryOn`, revisions over the complete code set, mismatch rules +during a rollout, start barrier. (**Superseded in part** by R25 (contributions may change; rejected are registries +created at runtime and mutable indirection), R17 (the harness starts after the barrier) and R6 (strict compatibility).) + +## Open + +Everything else from earlier lists is decided (see the entries above and `packages/chord/docs/next/design.md` §26). + +- **Future goal: deploy Chord apps to Cloudflare Durable Objects and similar platforms.** Hot reload there is not + needed; composition through plugins is (plugin set fixed at deploy). Research in `research/chord-on-durable-objects.md`. +- Deferred until a concrete case: jobs as objects and where a flow lives (when login is ported); project plugins. +- p3 domain points (decided per domain when p3 is designed): `SessionFiles {session}` (proposed), session delete, + connection pools, config files with a single writer, file formats as versioned interfaces, the terminal host + process as a revision artifact. diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md new file mode 100644 index 000000000..af87a6489 --- /dev/null +++ b/packages/chord/docs/next/design.md @@ -0,0 +1,2199 @@ +# Chord v2: target design + +The design of Chord v2 (`packages/chord/src/next`, export `@earendil-works/chord/next`) and of how apps (the +playground, p3) use it: **the one current statement of every rule** (owner, 2026-10-24). Every decision is recorded in +the decision log `packages/chord/docs/next/decisions.md` (the history and the reasons, kept complete) and folded into this +document in the same commit. Where a rule is built, the slice is named (`slices.md`); the code follows this document. +It replaces `apps/p2/docs/target-design.md`. + +Marks: everything is decided unless marked **Decision needed** (problem with an example, then options; nothing in such +a block is settled; section 26 lists them all). **(carried over)** marks behaviour of today's Chord that v2 keeps +without a new decision. API names are working names. + +Contents: 1 goals · 2 concepts · 3 hosts and links · 4 principals · 5 object types · 6 objects · 7 references · 8 ctx · +9 documents · 10 store and storage · 11 kinds of state · 12 derives · 13 natives · 14 routines · 15 jobs · 16 the facet +· 17 registries · 18 start data and platform bindings · 19 presence · 20 input into running work · 21 reload and rollout +· 22 wire · 23 optional components · 24 Durable integration · 25 apps: ownership and porting · 26 decisions needed · 27 +versions and adapters · 28 building: slices · 29 telemetry · 30 cost model · 31 rejected + +--- + +## 1. Goals and principles + +1. **Reload without restarts.** A plugin's new code takes over inside the running app: hosts keep running; data, open + connections and running work survive where they can. Reloads are explicit (a button or a command), never on save. + Base code (Chord, the app runtime) restarts the hosts. +2. **Ownership.** Every thing has exactly one owning object, the smallest it belongs to. The owner holds the data, + enforces its rules in its methods and offers the operations others need. Nobody reaches past it. +3. **Factories declare, Chord activates.** A factory is synchronous and only declares; Chord performs activation. + Handles a factory gets are valid for the object's whole life. +4. **Natives only in objects.** OS-level values are opened and closed by the object that owns them, and reach other + plugins only through that owner's methods, state or registry entries. +5. **Errors look like disconnects.** No request-id framework; consumers handle a failed call as they handle a dropped + link. References re-hold by themselves. +6. **Tagged unions, always.** States, results and outcomes are discriminated by `kind`; each state carries exactly its + own fields. +7. **Portable kernel.** A host is a logical identity plus an incarnation; facets may come pre-imported; values are + JSON plus bytes independent of the transport. Node first; Cloudflare Durable Objects and similar stay possible. +8. **Context last.** Parameters are the required arguments, then the context (`ctx` or a Durable `Context`), then + optional arguments: `self.tx(fn, ctx)`, `handle.change(key, fn, ctx)`, `chord(fn, context)`, `job.start(args, ctx, + { key })` (owner, 2026-10-21). +9. **Few concepts, no ceremony.** No special directories, no `defineNative`, no lints standing in for design. +10. **Aligned with Durable** (owner, 2026-10-27): where Chord and Durable solve the same problem (commits, lines, + loads, storage errors, watches), they use the same rules and mechanisms. + +--- + +## 2. Concepts + +| Concept | What it is | +| --- | --- | +| host | one participant (a process, a tab, a DO): a logical identity plus an incarnation | +| link | a connection between two hosts | +| host kind / address | typed description of addressable hosts / one host of a kind (`Session({ id })`) | +| facet | a plugin's declarations for one host kind: the types it serves, its contributions; no state or behaviour of its own | +| object type | a contract: name, params (which instance), host placement, members, version | +| object | one instance per (type, params, principal key); `local`: per (type, params) | +| principal | who calls; stamped by links, typed by the app | +| reference | what `get` returns; keeps its object alive; live and hydrated when awaited | +| document | a named JSON value an object owns, written in transactions | +| derive | a value computed from watchables | +| native | an OS-level value opened with `self.open`; only in objects | +| routine | standing behaviour of an object: `self.on(source, fn)`, `self.onLive(fn)` | +| job | owned work a call starts that outlives it: intent, run, finish; optional state document per run | +| registry | an owner's extension point: one contribution per contributor, composed by the owner | +| shared store | a `local` object holding shared data once, behind per-principal `remote` views | +| `ctx` | per call: cancellation, principal, call-scoped references | +| revision | (server incarnation, sequence): the complete code set a host runs | + +--- + +## 3. Hosts and links + +### Roles + +A Node app has one shape: + +``` +client client client browsers, TUIs: not addressable + └──────┼──────┘ + server exactly one, addressable + ┌──────┼──────┐ +worker worker worker addressable by a typed address, started on demand +``` + +| Role | Addressable | Principal comes from | Lifecycle | +| --- | --- | --- | --- | +| client | no | authentication of its link | connects by itself | +| server | yes, one | its own declaration | runs | +| worker | yes, many | stamped by the parent | started on demand by the app | + +Clients serve only `local` types (they are not addressable): a host without a principal refuses to serve a remote type +(`load` throws; a remote object is owned by a principal of the host serving it; owner, 2026-10-24). A client's local +calls have no principal; what a client calls on the server is stamped with the client's authenticated principal. +Clients reach workers only through the server, which relays; workers reach each other through the server too. A +detached process (e.g. a terminal host) would be another child kind of the server. Things that are not hosts: a code +mode sandbox (a worker thread inside a worker), remote environment daemons (their own protocol, owned by an object), +the TUI sandbox behind its launcher (the launcher only passes frames). A Durable Object deployment is a mesh routed by +the edge (section 24 and the research note). + +### Host kinds, addresses and placement + +```ts +export const Server = hostKind("server"); +export const Session = hostKind("session", { id: str() }); // params like a contract's + +export const Notes = remote("notes", { + version: 1, + params: { session: str() }, // flat record of required scalars + placement: ({ session }) => Session({ id: session }), // a typed function of the params + variants: { … }, +}); +export const Catalog = remote("catalog", { placement: () => Server(), … }); +``` + +- `hostKind(name, params?)` declares a kind of addressable host; its params follow the params policy of contracts + (section 5). `Session({ id: "S1" })` is an **address**: `{ kind, params, key }`, the key built like an identity key + (`session|S1`); two addresses are equal when their keys are. No host-name strings appear in app code. +- **`placement`** (owner, 2026-10-21) is a typed function of the params returning an address; `local` contracts have + none. It is not part of the contract fingerprint; a changed rule surfaces as a refusal (below). +- **`params`** (owner, 2026-10-21; was `input`) select the instance among objects of its type and are part of its + identity (section 5). +- **The caller computes the address** and sends it in the `hold`; a host that relays routes by that address alone and + never needs the contract or parses params. **The provider computes it again** from the params and refuses an object + that is not its own: broken `invalid/placement` (misrouted holds are never forwarded). During a rollout the refusal + is handled like a mismatch (provider on another revision: park; same revision: fail; slice 7). +- **Where a reference made on a host, or a hold arriving on one of its links, goes** (first match; slice 4a): + + | Case | Goes | + | --- | --- | + | the type is served here and its placement (computed here from the params) is this host's address, or the host has no address, or the type no placement | here | + | a reference made here (`host.get`/`hold`, `self.get`, `ctx.get`) to a served type placed on another host | on, as if not served here (next rows): worker S1 reaches `Notes { session: "S2" }` through the parent | + | an arriving hold for a served type whose params place it on another host | broken `invalid/placement`; the address the hold names only routes it | + | the hold names an address of a kind this host has a route for | over that route's link (the address's params checked against the kind, then `authorize`) | + | otherwise (a remote type) | over the parent link; a hold that arrived *on* the parent link is never sent back to it: `unavailable` | + | no parent link | `unavailable` | + + A `placement` or `authorize` that throws fails the hold broken with that error; an address whose params fail the + kind's check is broken `invalid/placement`. + +### The low-level host API + +```ts +// server.ts +const server = createHost

({ address: Server(), principal: { kind: "server" }, onError }); +server.route(Session, { + authorize: (p, address) => p.kind === "owner" || p.kind === "server" || (p.kind === "guest" && p.session === address.params.id), + resolve: async (address, context) => { + const child = fork("worker.ts", [], { serialization: "advanced" }); // no `signal`: see below + child.send({ id: address.params.id, init: await initFor(address.params.id) }); // the app's own start message + return { link: ipcLink(child), principal: { kind: "worker", session: address.params.id }, trusted: true }; + }, +}); +await server.load([{ name: "@app/core", facet: coreServerFacet(config) }, ...serverPlugins], ctx); + +wss.on("connection", (ws, req) => { // an entry point: app or adapter code + const principal = authenticate(req); + if (!principal) return ws.close(4401); + server.accept(webSocketLink(ws), { principal }, ctx); +}); +radius.onChannel((ws) => server.accept(webSocketLink(ws), { principal: { kind: "owner", access: "relay" } }, ctx)); +``` + +```ts +// worker.ts: connect first (W6), then load; start data through the core facet (section 18) +const { id, init } = await startMessage(process); +const worker = createHost

({ address: Session({ id }), principal: { kind: "worker", session: id }, onError }); +worker.connect(ipcLink(process), { principal: { kind: "server" }, trusted: true }, ctx); // stores may reference server types +await worker.load([{ name: "@app/core", facet: coreWorkerFacet(init) }, ...workerPlugins], ctx); // serves SessionInfo +await (await worker.get(Harness, {}, ctx)).resume(ctx); // tasks run only after every plugin loaded + +// a browser tab +const tab = createHost

({ onError }); +await tab.load(webPlugins, ctx); +tab.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }, ctx); +const notes = await tab.get(Notes, { session: "S1" }, ctx); // tab → server → route(Session) → worker +``` + +```ts +// a test: everything in one process +server.route(Session, { + authorize: () => true, + resolve: async (address) => { + const [up, down] = memoryLinkPair(); + const principal = { kind: "worker", session: address.params.id } as const; + const worker = createHost

({ address, principal, onError }); + worker.connect(up, { principal: { kind: "server" } }, ctx); // W6: connect, load, resume + void worker.load([{ name: "@app/core", facet: coreWorkerFacet({ id: address.params.id }) }, ...workerPlugins], ctx); + return { link: down, principal }; + }, +}); +``` + +Do not pass `context.abortSignal` to `fork`: `host.close` aborts it, which kills the worker and makes Node emit an +`error` event that crashes an app without a handler. Chord closes the link at `host.close`; the worker exits when its +parent link closes. `resolve` returns the link before the worker has loaded (the worker connects first, W6): a +`resolve` waiting for the worker's `load` while the worker's facets hold server types would deadlock. + +| Call | Meaning | +| --- | --- | +| `createHost

({ address?, principal?, ...options })` | with an address: addressable, serves only objects placed there, acts as `principal` on its own host; without: a client. `P` is the app's principal type | +| `load([{ name, facet }], context)` | facets with their plugin names (the plugin's `package.json` name, npm style; duplicates and invalid names are refused); runs every facet's setup, then waits for the start barrier (section 21); a throwing setup rejects and closes the host; runs once. Loaders are adapters; base code passes its core facet the same way | +| `connect(link, { principal, trusted?, codec? }, context)` | the link to the parent, at most one open; once it closed (a drop), `connect` takes the next, and every remote reference over the parent is held again on it. `principal` = who the parent is | +| `accept(link, { principal, trusted?, codec? }, context)` | a link from a child; holds and calls on it act as `principal` | +| `route(Kind, { authorize, resolve })` | how to reach hosts of a kind (below); called before `load`, once per kind | +| `get(Type, params, context, { readyTimeout? })` | a reference once its object is live (section 7) | +| `hold(Type, params, context, { readyTimeout? })` | a reference at once that follows its object until released (section 7) | +| `close(context)` | closes the links (children, routes, then the parent), then ends every object at once and tears them down, the last to go live first (reason `close`); rejects with an `AggregateError` of the native closes that threw | +| `revoke(r, context)` | revocation (section 4; slice 4b) | +| `root(fn, …)` | a call scope for code outside objects (a Durable hook, a CLI); built in slice 8 (**decision needed** on its signature, section 26) | + +**Options of `createHost`:** `address`, `principal`, `principalKey` (section 4; the same function on every host), +`telemetry` (section 29), `onError`, `closeTimeout`, `readyTimeout`, `heartbeat: { ping?, close? }`; later `revokedBy` +(section 4, slice 4b) and `store` (section 10, slice 6a: the store, opened by the app with `openStore(backend, ctx)` +and closed by the app with `store.close(ctx)` after `host.close`; the host never closes it). **`onError`** receives everything that fails in the +background, as a tagged `HostError`: `listener_error` (a watch listener threw), `delivery_failed` (a frame delivery +threw), `activation_failed { object, failure }`, `close_failed { object, native, error }`, `close_timeout { object, +native }`, `drain_timeout { object }`, `protocol_error { error }`, `resolve_failed { address, error }`; later routine +errors (slice 5), `derive_failed { object, member?, error }` (a derive's compute threw, once per failure episode, slice 6a), `object_failed { object, failure }` (a live object failing, slice 6a; `failure` where the value is a `FailureCause`, `error` otherwise, 2026-10-27) and `storage_failed { error }` +(slice 6a), which every app must handle: Chord never ends the process, +the app exits with its own code for its supervisor or reopens. + +**Timeouts** are host options; no fixed numbers in the code. Two host timeouts (C13, owner, 2026-10-22): + +| Timeout | Bounds | Host default | Override | +| --- | --- | --- | --- | +| `closeTimeout` | a native's `close` (then reported as a possible leak; the rest continues); the drain of an ending object's running handlers (callers are rejected at once) | 30 s | per native: `self.open(open, close, { closeTimeout })` | +| `readyTimeout` | an activation attempt (its wait for the previous instance's teardown included); a getter's whole wait; how long a call made while its reference is loading waits (the park deadline) | 60 s | per `get`/`hold`: `{ readyTimeout }` | + +The park deadline's per-type override is not built (only if a real case needs it). Timeouts of later slices +(routine handover 10 s, job `finish` retry backoff 1 s to 60 s, rollout install 60 s, provider-side `linger` per type) +belong to their mechanisms (**decision needed** on which stay host options, section 26). A route's `resolve` is +bounded by the app. + +- **Incarnations** are minted by Chord (uuidv7, code copied from pi-ai, no dependency), never passed in, and carried + in the link handshake (`hello`): the other side sees a restart as a new incarnation of the same identity; + references to it re-hold. +- **Entry points vary** (loopback or tunnel WebSocket servers, Radius dialing out and delivering channels, a DO edge, + memory), so the kernel has no `listen`; each entry point authenticates and calls `accept`. Radius no longer needs to + pose as a loopback request with a secret header. +- **App code, not Chord:** starting processes, init data, authentication, keep-alive decisions. Helpers come only when + apps repeat the same code. +- Compared with today's Chord: same shape (`createHost`, `connect`, `accept`, `route`, `load`); gone are string host + names and prefix parsing, app-managed generations, the duplicate `upstream` option, `opaque` runtime objects and the + loader inside the kernel. +- **Worker keep-alive** works as in p2: the worker counts, the app's supervisor decides. **`host.activity`** (slice 4b; + owner, 2026-10-24) is a JSON `Watchable` of `{ references, calls, jobs, lingering }`: holds arriving over links, + method calls running on this host, running jobs, provider-side lingering objects. Counters are kept where the work + happens; a change marks it dirty and it is published at most once per microtask turn with changes. Core code + reports it plus Durable tasks (the core `Harness`); the server's supervisor stops the worker when all are zero for + the idle time (p2: 5 minutes, checked every second). Chord provides the counts, no idle logic. + +### Routes + +```ts +host.route(Session, { + authorize(principal: P, address: Address): boolean; + resolve(address: Address, context: Context): + Promise<{ link: Link; principal: P; trusted?: boolean; codec?: Codec }>; +}); +``` + +- **`authorize`** runs for every hold routed over the route, with the hold's principal (section 4), before anything + starts; `false` fails the hold broken `forbidden`. +- **`resolve`** runs at most once at a time per address: the first hold starts it, later holds wait for the same + link. The returned principal is stamped on that link; `trusted` and `codec` as for `accept`. Its `context` is + aborted only when the host closes; a link it returns after that is closed at once. A `resolve` whose holds were all + released meanwhile finishes and its link is kept (a worker start that produced a link finishes even if its first + requester left). A `resolve` that never settles is the app's (it bounds its own `fork`); getters still time out. +- **Failure and re-resolve:** a rejected `resolve` (reported `resolve_failed`), a returned link already closed, or the + address's link closing while holds over it exist, starts the **per-address backoff** (1 s doubling to 60 s; a map + keyed by address, separate from objects' identity backoff). Meanwhile the holds are `loading`. Only a link that + completed its handshake resets the backoff. Without holds for the address nothing is resolved again, so passive + viewers recover without a user action while they hold. +- Chord never closes a route link itself except at `host.close`; stopping idle workers is the app's (`host.activity`). +- Route links have no app context yet: work arriving on them runs under the background context without telemetry + (slice 4b gives routes a context). + +### Links + +- **A link is delivery only:** an ordered, bidirectional channel of `Uint8Array` messages; adapters only move bytes + and close both ends. Session recovery is the adapter's policy. Implementations (slices 3a, 4a): + - `memoryLinkPair()`: delivers with `queueMicrotask` (no Node APIs); closing one end closes the peer a microtask + later. + - `ipcLink(port)` over a Node IPC channel (`fork` with `serialization: "advanced"`, or `process` in the child), + typed structurally (no Node import in Chord). A `Uint8Array` crosses as one message; other messages on the + channel (the app's start message) are skipped. It listens from the moment it is made and queues until its + listener is set. `send` returning `false` is backpressure, not failure; a thrown `send`, an error callback or + `disconnect` closes the link once; listeners are removed on close. + - `webSocketLink(socket)` over a WebSocket (`ws` in Node, the browser's), typed structurally: one binary frame per + message, `binaryType = "arraybuffer"`; messages sent while the socket connects are buffered until `open`; a socket + closing or closed already gives a closed link; `close`, `error` and a text frame close it. The transport's + `maxPayload` stays its hard cap (code 1009 closes the connection). +- **One link object per link carries both directions** (slice 4a): the channel (handshake, codec, checks, + heartbeat), a **provider side per origin** (holds and calls the peer sent, its own or those of a relay's downstream + link, by the peer's ids) and a **consumer side** (holds and calls this host's own references sent, by its own + counters). Message kinds are directional, so each goes to its side. A link carries nothing until a hold or call + (W6). +- **Roles:** `parent` (made by `connect`), `child` (`accept`), `route` (`resolve`). The role decides whose principal a + hold acts as (section 4). **Trusted links** are an explicit option (`trusted: true` on `accept`, `connect` and in + `resolve`'s result; default untrusted; never inferred from the link kind; owner, 2026-10-23): they skip only the + `MESSAGE` schema check; framing, the codec's decode, `checkParams`, `checkArgs` and Delta decoding still run. Trust + never grants the principal fields (they are read only on the parent link). +- **Remote objects belong to a path:** the parent, or a route address. Each new link of a path holds every remote + object of that path again (with new hold ids); a closed link drops its tables and the objects go `loading`. +- **Heartbeat on every link kind, in the protocol layer** (owner, 2026-10-23): each link keeps a `lastReceived` + timestamp (every received message; sends do not count); the host runs one interval timer for all its links (none per + message). A link silent for 15 s sends `ping` (after its handshake); the peer answers `pong` at once, never queued + behind the window. A link silent for 30 s, the handshake included, is closed (its references go `loading`). Both + numbers are host options (`heartbeat`). The heartbeat detects a silent transport (TCP does not notice a sleeping + laptop), not a peer that answers pings but withholds acks (bounded by the window and the pending cap instead). A + handler that blocks the event loop longer than 30 s gets its links closed (documented). +- **Flow control: an ack-clocked window per origin** (owner, 2026-10-23; per origin 2026-10-24). Per (link, origin), at + most **K = 2** of the provider's own `bound` and `state` messages are unacknowledged; only these two kinds count + (everything else is control, never queued behind them). The consumer sends `ack` (with the message's origin) after + applying each, after its batch; an `ack` crossing nothing in flight is ignored. While the window is full: + - each touched hold member keeps its ops in its **pending queue** (section 11: raw, never mutated, folded through + the leaf index, at most 10,000 ops, then only "replace with the current value"); when an `ack` frees a slot, one + `state` message carries every waiting hold's entry, its members' pending ops encoded once (or `[["r", value]]` + after an overflow), all reflecting the latest commit, still atomic for the consumer. So a slow consumer gets + coarser updates, never stale ones (measured: a `bufferedAmount` cap alone left state 9.4 s old behind kernel and + proxy buffers; acks kept it 0.0 s old); + - a hold that binds waits for a slot; its snapshot and subscription are taken in one synchronous step when its + `bound` is sent; bindings and waiting state take turns when both wait, so neither starves; + - a hold whose object leaves `live` while its `bound` waits must go back to pending; only a reload makes a live + object leave `live`, so slice 7 builds it (D1, owner, 2026-10-24). + Not built, by decision: a byte window, credit messages, snapshot admission or debt, an adapter `backlog()` API, + `reset` from anyone. +- **Calls: a call window N** (default 256, the same on every host) announced in `hello`: how many concurrent calls a + side accepts **per origin**. A side sends no call before the peer's `hello`. A host sends at most N of its own calls + per link in flight; further calls wait in the link's **call queue**, one FIFO, sent in order as replies free slots; + nothing is refused for being "too many" (K16). A slot is freed once per call: by its `result` or `error`, its + `abort` (when sent, at the caller; when received, at the provider), its hold's `release`, `gone`, its origin's + `closed` or the link closing. A call still queued when its hold rebinds (`loading`) is rejected `unavailable` (it may + name a method of the old variant). A provider seeing more than N running calls of one origin closes the link + (protocol error); a call naming a hold it does not know is ignored before it is counted (Q1). A per-link N, if ever + needed, must bound a relay's forwarding too. +- Holds are not limited by count (authorization and params checks protect them); size limits are a separate check + (section 22). Byte-heavy traffic (terminals) uses separate channels (section 23), so a large state message never + holds up keystrokes. + +### Relays (slice 4a; decision "Relay flow control per origin", owner, 2026-10-24) + +A host that receives a hold it cannot serve (the placement table above: a route or the parent) **relays** it: it holds +it again on that link (upstream) with the arriving hold's principal and incarnation, and forwards between the two +without parsing payloads. The provider keeps the window, the pending queues and the calls per origin; the relay never +drops, folds or resets. + +- **Origins:** the relay gives each downstream origin (a downstream link, and the origin its messages carry when that + link is itself a relay's) a number, one counter per host, never reused; 0 is a host's own work. Every message + upstream for that origin carries `origin`; the provider keys holds, calls, windows and acks by (link, origin, id), and + its answers carry the origin back. `origin` is honoured on every link (a worker may relay a child's hold to its + parent); principals and incarnations stay parent-only. +- **Ids are not renumbered:** an upstream hold has the downstream hold's id under the origin's number; calls the same. + Two tabs (two holds) of one principal are two upstream holds, each with its own encoders (R1). +- **Nothing is parsed:** the relay routes by the hold's address and authorizes it; params, arguments, results, + snapshots and ops travel as opaque parts. Cost per message: decode the envelope, swap the origin, encode the + envelope; parts copied once into the outgoing buffer. +- **What the relay keeps:** per relayed hold, its upstream link and its original `hold` message (params copied once), + which routes `call`, `abort`, `retry` and `release` upstream at once and holds again on the next link; per + downstream origin, the FIFO of the sender of each `bound`/`state` message in flight and the set of upstream links it + relayed to. +- **Forwarding up:** `hold`, `call`, `abort`, `retry`, `release` with the origin's number; a hold waiting for its + route is sent when the link exists; a `release` before that drops it. A call is forwarded at once; with no upstream + link it is answered `unavailable` (it crossed the `loading` notice). Each downstream consumer is bounded by its own + N, so the upstream sees at most N per origin. +- **Forwarding down:** a message from upstream with an origin goes to that origin's provider side, which forwards it + with its own origin (none for a tab), ids and parts unchanged. **A link answers only for what was relayed over it** + (owner, 2026-10-24): every entry of a `bound` or `state`, every `failed`, `loading` and `gone` by its hold, every + `result` and `error` by its call is checked against the relay's table; an answer for a hold or call not relayed over + the link it came from is dropped (Q1: also one crossing a release). A `bound` or `state` left with some entries is + forwarded with those; left with none, it is acknowledged to that link at once. A message for an origin from a link + the origin relayed nothing to is ignored whole. +- **Acks:** the relay forwards each upstream `bound`/`state` at once and returns its `ack` to the upstream link that + sent it when the downstream acknowledges it (the senders FIFO). A tab served by W workers may have 2W messages in + flight (plus the relay's own 2), each worker's window bounding its own. A slow tab stalls only its origin at each + worker; other tabs are other origins. Measured: a slow tab through the relay catches up like a direct one (a 60 s + stall: 0.38 s, 23.7 KiB at 64 KiB/s; before the rework, 24 s and three full snapshots). +- **Upstream drop:** the relay sends `loading { hold }` downstream for every relayed hold on the dropped link, fails + their relayed calls in flight `unavailable` (never resent), re-resolves and holds again upstream from the kept + message (same id, same origin); the new `bound` is forwarded. Acks for messages of the dropped link still in flight + downstream go nowhere. +- **Downstream drop:** the origin's provider side ends: its served holds are released, its relayed holds forgotten, + and every upstream link it relayed to gets one `closed { origin }`, which releases the origin's holds and aborts its + calls there; the origin's number is never used again; messages crossing it are dropped. +- **`gone`** from upstream is forwarded; the relayed hold is forgotten. Relays never make a `gone`. + +### Recovery + +- After a drop the consumer re-holds every hold of the path over the new link and receives snapshots (a `bound` on a + bound hold replaces its replicas in one batch, another variant allowed); calls in flight fail like on a disconnect; + calls made meanwhile park (section 7). +- A slow connection never ends an object: nothing releases a hold for falling behind (the reference count is unchanged). +- A link drop is a departure (section 19). + +### Values + +- **JSON plus `Uint8Array` leaves**, with a bytes schema type, defined independently of the transport. Every link + carries the same `Message` through its codec (section 22); parts are encoded at their origin; byte segments travel + raw, never base64. Same-host calls (no link) pass values by reference. +- Schemas are TypeBox and must describe data (no callbacks, no native types, no runtime codecs). Method arguments are + validated at the provider for every caller against closed schemas (section 5, R7, Q12, Q15); results and state are + never validated (section 30). Values are immutable by convention once handed over (Delta's rule): nobody mutates a + value it got, and nothing is copied or frozen to enforce it. There is no development mode (R9): one behavior + everywhere. + +--- + +## 4. Principals + +- The app defines its principal type `P`: JSON, every variant with a `kind`, a type alias (not an interface), + unversioned (owner, 2026-10-22): Chord never persists principals, base code changes every host at once, links stamp + them. Objects persist a projection with its own schema, never the principal (p2: chat's `Author { kind, id, name }`). + p2/p3: `owner` (`access: "local" | "relay" | "device"`), `guest` (`session`, `mode`, grant revision), `server`, + `worker` (`session`). +- **Links stamp principals.** A caller's own data never sets its principal. +- **The principal key** maps a principal to its object identity (`principalKey`, default canonical JSON; cached per + principal object; the same function on every host). It should include connection provenance and grant revision, so + a revoked grant's objects are distinct from a new grant's **(carried over)**. +- **Principals on holds** (slice 4a; owner, 2026-10-23): a `hold` carries `principal?` and `incarnation?`. The receiver + uses them **only on its parent link**; everywhere else they are ignored and the link's principal and peer + incarnation apply (a tab cannot claim to be the owner; a relay overwrites what a client sent). Calls carry neither: a + call runs as its hold's principal and with its hold's incarnation (refines R1/Q13, which stamped calls too). What a + host sends in those fields when it holds over a route link (towards a child that honours them): + + | The reference was made by | `principal` | `incarnation` | + | --- | --- | --- | + | `host.get` / `host.hold` | the host's own | this host's | + | `ctx.get` in a handler | the handler's `ctx.principal` (the caller's) | the handler's `ctx.incarnation` | + | `self.get` in a remote object | that object's principal | this host's | + | `self.get` in a local object or a facet | the host's own | this host's | + | a relayed hold | the arriving hold's principal | the arriving hold's incarnation | + + So a guest's call on the server whose handler `ctx.get`s a worker type reaches the worker as the guest and gets the + guest's variant; two references with different principals are two remote objects. +- **Upward is a boundary:** a worker's hold of a server type goes over its parent link, and the server ignores + principal fields from a child, so it acts as the worker, whoever the worker's handler serves. A worker that acts for + a user towards the server passes the user as data and the server object decides. +- **The caller's incarnation** (R20, Q13): `ctx.incarnation` is the incarnation of the host where the call started + (from the handshake, or from the hold's field on a parent link), next to `ctx.principal`, not part of the principal + or its key, never sent as data by apps; it is this host's own for local calls (section 25 uses it as a fence). +- **Revocation** (slice 4b) is a typed app function, not field matching: + + ```ts + type Revocation = { grant: string } | { device: string } | { session: string }; + const server = createHost({ + address: Server(), principal: { kind: "server" }, onError, + revokedBy: (p): Revocation[] => + p.kind === "guest" ? [{ grant: p.grant }, { session: p.session }] + : p.kind === "owner" && p.access === "device" ? [{ device: p.device }] + : [], + }); + server.revoke({ grant: "g1" }, ctx); + ``` + + `host.revoke(r)` ends every object and closes every link whose principal `p` has `r` in `revokedBy(p)` (compared by + canonical JSON of each value `revokedBy` returns, computed once per principal; never the host's own principal), and propagates `r` to the children as a `revoke` message on its route links, + honoured only on a child's parent link, where each applies its own `revokedBy` (the same app code; nothing is + replayed to a child connected later). Holders of a revoked object get `gone` with `forbidden/revoked` (owner, + 2026-10-24: a reason of its own, so a UI can say "access revoked"); holds still waiting for `load` are ended too. + Order: the `revoke` message goes out on the route links before the revoked child links close, so a worker's objects + end `forbidden/revoked`, not `released` by the tab's `closed`. Revocation ends a principal's objects at + once, whatever references or jobs remain (R11); their jobs get the abort signal and the normal `closeTimeout`. + There is no re-stamping: a changed grant (mode `write` → `read`) is a revocation of the old one; the client + reconnects and authenticates anew, and its references re-hold. Reconnecting never restores revoked authority. Work + started on behalf of a revoked principal in a shared store stops when its reconciling routine runs (R10's window, + accepted; section 25); there is no revocation hook. +- `local` objects have no principal in their identity; their handlers and references act as the host (section 5). + +--- + +## 5. Object types + +```ts +export const Notes = remote("notes", { version, placement, params, variants }); +export const NoteStore = local("notes.store", { version, params, members }); +``` + +| | `remote` | `local` | +| --- | --- | --- | +| reachable from | any host, across links | its own host only | +| identity | (type, params, principal key) | (type, params) | +| principals, variants | yes | no | +| factory | `(params, principal, self)` returning `{ variant, members }` | `(params, self)` returning the members | +| arguments | validated once at the provider for every caller (compiled closed schema, R7, Q12); unknown fields rejected; passed by reference on the same host | validated the same way (Q15), never copied | +| results | not validated; passed by reference on the same host | not validated, not copied | +| members | state (JSON) and methods (JSON plus bytes) | plain-data state and methods; no `opaque`, no callbacks | +| `ctx.principal` in methods | the caller | the host's principal (none on a client); references a local object makes act as the host | + +```ts +variants: { + editor: { state: { notes: NotesSchema }, methods: { add: method(AddArgs, NoteId), remove: method(RemoveArgs) } }, + reader: { state: { notes: PublicNotesSchema } }, +}, +// the factory chooses one variant per object (slice 3a, C15): +host.serve(Notes, ({ session }, principal, self) => + principal.kind === "owner" ? { variant: "editor", members: { notes, add, remove } } : { variant: "reader", members: { notes: publicNotes } }); +``` + +- **Names** (owner, 2026-10-27): object type names, host kind names, document (`defineState`) and job (`defineJob`) + names use `[a-z0-9@/._~-]`, lowercase, checked at declaration (they become SQL table names in the store, joined with `:`). +- **Keyed state members** (owner, 2026-10-26; `local` contracts only): `members: { notes: keyed(NoteSchema) }`; the + provider returns a keyed document handle; a consumer on the same host takes `ref.notes.at(key)` in its factory (a + counted pin: the key stays loaded while the consumer lives) or `ref.notes.watch(key, ctx)`/`get(key, ctx)` anywhere; + an absent key reads `initial()`, a retirement arrives as the `retired` flag (a derive over it carries the flag on). + Across links, a view per entity (`NoteView { session, id }`) is held as an ordinary remote object. +- **Methods:** `method()` (no arguments, no result), `method(args)`, `method(args, result)`, `method(undefined, + result)`. A method without arguments omits them in the API (`ref.bump(ctx)`, implemented as `bump: async (ctx) => + …`); `null` is their internal and wire form (owner, 2026-10-22). +- **Implementations** (owner, 2026-10-22): a state member is a JSON `Watchable` the factory returns (a document handle, + a derive, a reference member, `constant(...)`), never a getter; a method is a function `(args, ctx)` or `(ctx)`. + Activation checks what the factory returned (plugin code): exactly the variant's members, methods as functions, + state members as JSON watchables (a `NativeWatchable` or anything else fails the object `broken` naming the member, + Q14). +- **Reserved member names:** `then`, `release`, `state`, `retry` and `variant` are refused at declaration (a reference + has them; `then` would make an awaited reference thenable). +- **Compatibility (strict, like documents; R6):** a hold carries only the consumer's contract `version` and + fingerprint, never a description. The provider binds when both equal its current ones, or (slice 7) when the + version is one it accepts and the fingerprint equals the one computed from its own history (section 27). Every + contract change, additive ones included, bumps `version`. Anything else is `contract_mismatch` (or `updating` during + a rollout, section 21). The fingerprint covers params, variants and members, not placement. A consumer whose contract + does not fit gets a mismatch on its own binding; the provider object and its other bindings are not affected. +- **Variants:** a member name has the same kind (state or method) in every variant that has it, and a method the same + signature (arguments and result) in every variant; state schemas may differ per variant (redaction). Enforced at + the type level in `remote(...)` and at declaration at runtime. A remote factory returns one variant, `{ variant, + members }`; `ref.variant` is the discriminant (section 7). Calls are checked against the binding's variant + (`invalid/member` outside it). +- **Params** (owner, 2026-10-21/22): a flat record of required fields, each `str()`, `int()`, `bool()` or `oneOf(...)` + of string literals, constraints allowed (`str({ pattern })`, `int({ min: 0 })`); no defaults, optional, nested or + unknown fields. Checked for every caller with one compiled closed-object TypeBox check (a missing or extra field + fails `invalid/params`). The identity key is the field values in declared order, each escaped, joined with `|`. + Params are never copied or frozen: the object keeps the caller's object, immutable by convention. +- **Arguments (R7, Q12, Q15):** always validated with the provider's schema, whoever the caller is (trusted code is + not type-checked code: plugins are bundled without type checks), never with a schema a peer sent; object schemas are + closed (`obj()` sets `additionalProperties: false`), so an unknown field fails `invalid/args` instead of being + stripped; nothing is copied or mutated; arguments have no defaults (optional fields are `opt()`, handlers write + `args.limit ?? 50`). Same-host arguments are not checked for plain data (no prototype or getter checks): callers pass + plain data by convention. Provider schemas compile lazily at first use, cached in `WeakMap`s keyed by the schema + owner; accepted versions' validators also on first use. Bytes are leaves (`instanceof Uint8Array`, optional length + bound), never walked. Results and state are not validated. +- **Providers** (`bindProvider(Type, accepts)`, L1): runs `checkAccepts` once per load and reload and keeps the + validated mappings; per call `checkArgs(provider, method, raw, boundVersion, context)` returns a tagged value, + `{ kind: "method"; args; result? }` (current-shape arguments and, for an old-version caller, the result mapping back) + or `{ kind: "call"; args; call }` (the provider's own replacement of a removed method, invoked by Chord, slice 7). The + binding is held only by the served type's registration and by calls in flight (GC-tested; no global strong cache). +- **Errors** (L1, slice 3a): `ChordError` with a code and, for some codes, a reason, a closed union that is part of the + link protocol (adding one bumps the protocol version; a receiver rejects unknown reasons; a test pins the list): + `invalid` (`args`, `params`, `member`, `placement`, `reentrant`, `read_after_bulk`, `cycle`), `contract_mismatch` + (`updating`), `forbidden` (`revoked`), `gone`, `not_ready`, `released` (`ended`, `closed`, `settled`, `version`, `discarded`; none for a holder's own release), `too_large` (`part`), `unavailable` (`locked`, + `park_timeout`, `busy`, `storage_failed`), `aborted`. A failed check names TypeBox's instance path ("unknown field", "missing", else TypeBox's + message; keys containing `/` or `~` make the path ambiguous, documented). Plugins throw `PluginError(code, message, + details?)` for codes of their own; anything else crosses a link as the plugin code `internal` with its message, never + a stack or a cause; plugin errors never use Chord's codes. An error rebuilt from the wire is not copied and only its + envelope is checked; a malformed envelope is a protocol error. +- **Methods return data,** never objects; an object is handed out as its address and the receiver does `ctx.get`. +- **No callbacks as arguments,** on either kind: behaviour goes through registries, multi-step work through jobs, + input through flows, runtime services through core objects. Real schemas instead of `Type.Unsafe`. + +--- + +## 6. Objects + +### Lifecycle + +``` +(get) → activating → live → ending → ended + └→ failed (backoff) → activating … + live → failed (a runtime failure of the owner, slice 6a: broken, by reload or retry only) +``` + +- **One object per identity.** The first `get` runs the factory; later gets share the object. +- **Lifetime follows from kind and params (Q-A, owner, 2026-10-21):** there is no `lifetime` option and no lazy flag. + A `local` type with empty params lives as long as the host: started eagerly at host start, never ended by reference + count (it may hold persisted data, P3, and contribute to registries, R21). Every other `local` type and every + `remote` type lives while referenced (`linger` keeps it briefly after the last reference). No remote object has host + lifetime: its identity includes the principal, and long-lived parts belong in a local shared store. Below, + "host-lifetime object" means a `local` object with empty params. Running and finishing job runs keep their + object alive as a count, not as references; when the count reaches zero with no holders, the object ends (§15). +- **Activation** (one fresh attempt each time, bounded by `readyTimeout`, its wait for the previous instance's + teardown included): the factory runs; singleton documents take `initial()` and, on the object's first activation, + persisted documents load and its tables' definitions are checked and migrated (slice 6a; the deadline paused during + eager migration work); references are bound (before the + previous attempt's are released, so a dependency held only by it never ends in between) and waited for until live + and hydrated; natives open one after another; the returned members are checked (section 5); derives compute; + routines start (slice 5). The object goes live when all of that succeeded. Calls wait behind its **gate** until + then. +- **The seal fence** (owner, 2026-10-26): every attempt has a seal token (a job run invocation its own, for its state + changes and outcome). After the seal, new work through the attempt's handles is refused at use (`self.tx`, `ctx.get`, + a job or routine start); commits are checked when they enter the store's commit step (sealed: rejected `released`, + nothing written) and at adopt (sealed during the SQL: durable, adopted into the object's documents, staged job + changes abandoned); at the seal a transaction not yet in the step is aborted and the line released, a started step + finishes before teardown completes; cleanup (references, presence, activity counts) still runs. A pure swap is not a + replacement. +- **Ending** (slice 1, 2): new calls are rejected; every caller of a running call gets `released` at once; the + handler keeps running in the background under an aborted context and its `self.tx` throws `released` (a running + method can change nothing after the end). Each call runs under a context aborted by its caller and by the object's + end (one controller per attempt, linked until the call settles; a caller's cancellation aborts only its own call); a + call whose caller context is already aborted is rejected before its handler runs. Then the **drain** waits for the + handlers' own promises, bounded by `closeTimeout` (`drain_timeout` reported), natives close in reverse order, + documents are dropped, derives detached, and references released last (closes may use them). Routines stop and jobs + follow their rules (slice 5). +- **A live object can fail** (owner, 2026-10-26): a runtime failure of the owner (a `migrate` failing on access, a + stored version newer than the code) seals the live attempt and makes the object `failed` (`broken`): calls reject + with the cause, state members keep their last values, natives close, documents stay on the object; recovery by + reload or `ref.retry()`; live dependents stay live. The members freeze only after the commits admitted before the + seal adopted and published once, and a retry's factory waits for that (2026-10-27, slice 6a build reviews). +- **Waiting for dependencies (R18):** an object whose references are not live yet stays `loading` and goes live as + soon as they are (bounded by `readyTimeout`). A dependency that fails (`broken` or `transient`), or does not become + live in time, fails the dependent **`transient`** with the dependency's cause inside, never `broken`: the cause is + the dependency's. The dependent keeps its references while failed, and Chord installs their state as an automatic + retry trigger: the moment the dependency goes live, the dependent is retried at once (the remaining backoff is + cancelled; the backoff stays as a safety net). Factory authors write `self.get` and know nothing of this. When a + dependent's last reference goes, it ends and releases its dependencies. After `live`, a dependency leaving `live` + does not change the dependent: it stays live, and its calls that use the dependency get its error (or park while it + is loading). Only R18 failures keep references; any other failure releases them. +- **Dependency cycles are rejected (Q5):** every object instance has a uuidv7 instance id, minted at its first + attempt; an object that restarts gets a new one, so an old chain never matches (slice 4b, the first user; D4, + owner, 2026-10-24). On one host, at + `self.get(B)` during A's activation Chord follows the waiting edges of activating objects from B; reaching A is a + cycle. Across hosts, a hold made during an activation carries the activation chain of instance ids in its header + (bounded; ignored from untrusted links): the provider rejects the hold when the instance being activated is in the + chain (slice 4b; on one host slice 1; at most 32 entries; locally created objects inherit the activating chain). A + cycle fails the activation `broken invalid/cycle` with the cycle as cause ("A → B → A"), and every other activating + object on it the same way; no references are kept. A dependency failing with `invalid/cycle` fails its dependent + **broken** with the same error, an exception to R18 (owner, 2026-10-24: otherwise every backoff retry hits the same + cycle); dependents that are not on the cycle break too, and a reload or `ref.retry()` recovers them. Checked again + after a reload. Cost: O(chain) per declared reference, only during activation. + **Known limitation** (owner, 2026-10-24): a chain is fixed when its object is created, so a cycle entering an object + that is already activating is not detected. Example: A on the server and B on a worker start at the same moment, A + declares B and B declares A; the holds carry `[A]` and `[B]`, neither finds the other, both fail `transient` at + `readyTimeout` and retry with backoff (later attempts usually come in order and detect it). A probe along waiting + holds was not built. +- **Failed:** a broken part fails the whole object: a throwing factory, document hydration or `migrate`, a failed + open, a derive that throws on its first computation during activation, or a dependency that did not become live. + After the object is live, a throwing derive fails only its own member (`broken`, W1). The cause is tagged and decided by Chord from where it + failed: + + ```ts + cause: { kind: "transient"; error: ChordError } // an open threw, a reference was unreachable: automatic backoff + | { kind: "broken"; error: ChordError } // factory, derive, hydration or migrate threw: no automatic retries + ``` + + Only an `open` may choose `broken` itself (`throw broken(error)`, e.g. invalid configuration). **Transient** causes + get a **backoff per identity** (1 s doubling to 60 s; a map holding no object, so a failed object is collected after + its last reference while its window runs; concurrent gets share one attempt; success resets). **Broken** ones are + retried only by a trigger. Triggers (for both): `ref.retry()`, a plugin reload (slice 7) and wake signals the factory + declares with `self.retryOn(...readables)` (slice 8; anywhere in the factory: the factory specifies, Chord installs + the watches before natives open, so a failed open still has them). Every trigger works like `ref.retry()`: the + backoff window is skipped, an attempt starts now, holders are told `loading`. `retryOn` sources must live independently of the + attempt (registry values); a reference's member is a type error. A facet cannot fail + after `load`: its setup only declares (a throwing setup rejects `load`, slice 2; owner, 2026-10-26). + +### The factory principle + +```ts +host.serve(NoteStore, ({ session }, self) => { + const notes = self.state(NoteDoc); // section 9 + const index = self.state(Index); + const count = self.derive([index], ([i]) => i.ids.length); // section 12 + const watcher = self.open(async (context) => watch(dir, { signal: context.abortSignal }), (w) => w.close()); // section 13 + self.onLive(async (ctx) => { for (;;) { await sleep(HOUR, ctx); await cleanup(ctx); } }); // section 14 + const imports = self.job(ImportJob, { run, finish }); // section 15 + self.contribute(Http, { routes: { … } }); // section 17 + const info = self.get(SessionInfo, {}); // start data, section 18 + const catalog = self.get(Catalog, {}); // section 7 + return { index, count, add: …, startImport: … }; +}); +``` + +- Synchronous and declarative: no awaits, no side effects; async work happens in activation (opens), in calls, in + routines and jobs. +- Handles stay valid for their attempt's life (owner, 2026-10-26: documents and the line belong to the object; each + attempt's handles carry its seal token and definition version); nothing is "read at use" or re-fetched. +- On a reload the factory runs again for pure objects (new methods and derives, same documents); native + objects restart (section 21). + +### Pattern: shared store, per-principal views + +``` +NoteStore {S1} local shared store: the stored notes, once + ▲ self.get ▲ +Notes {S1} as owner Notes {S1} as read-only guest + variant "editor" variant "reader": derives a filtered view +``` + +Heavy or shared parts (a Durable watch, a prompt watcher, a sandbox, a store) live once in a `local` shared store; +per-principal `remote` views get it and expose what their principal may see. Authority is the view's; the store is +trusted and acts for the host. Views of the same variant read one value from the store (a redaction is computed once, +not per guest; section 12). Any plugin uses this pattern with the same primitives; base code is not special. + +**Worker-wide native owners:** the facet cannot `self.get` a type it serves, so an owner of a per-session native (an +inbox watcher, a sandbox) is a `local` object with empty params (host lifetime) that reads its session from the core's start +data (`self.get(SessionInfo, {})`, read in its opener). Paths are derived by the plugin from generic start data (the +data directory), never provided per plugin. +Such an object cannot `self.get` a session-scoped object in its factory (the session is known only after activation): +it gets one with `ctx.get` in a routine or method (R22). A `get` with params computed after activation is added only +if a real case needs it. + +**Natives that follow a relocation:** the native is a stable manager (`self.open` once); a routine on the session +location switches its internal resource (the watched directory) and bumps a generation, so late events from the old +location are dropped. + +--- + +## 7. References + +| Where | API | Released | +| --- | --- | --- | +| a factory | `self.get(Type, params)` | when the object ends | +| a call | `await ctx.get(Type, params, { readyTimeout? })` | when the handler settles | +| outside | `await host.get(Type, params, context, { readyTimeout? })` / `host.hold(Type, params, context, { readyTimeout? })` | `release()` | + +- **`host.get`** (and `ctx.get`) resolves with a reference once its object is live and **hydrated** (every state member + of its variant has a value). A broken failure rejects at once; through transient ones it waits up to its + `readyTimeout`, then rejects with the last failure's error; a getter that gives up releases its reference. + **`host.hold`** returns a reference at once that follows its object until released (loading, live, failed and + retried as its object is, loading again while its link is down) and never rejects; invalid params and a local type + not served here throw at once (before `load` resolved, these fail the reference broken once it did); its `variant` is + `undefined` until its first binding (slice 3b). There is no `state: false` option (C8) and no `optional` flag. +- **Narrowing by variant:** `ref.variant` is the discriminant (C15: no `inVariant`/`expectVariant`); `RefIn` + narrows to some variants (strict unions, never weakened). +- **Members across attempts** (2026-10-27, slice 6a build reviews): on the same host a reference's state member + follows its object's member across attempts, as a remote reference re-holds: one member source per object and + member name; when a new attempt goes live every member re-binds by name (watches get one diff frame from the last + value, consumers' derives recompute, also one that failed on the member's earlier error, which heals; a member the + new variant lacks fails `invalid/member`); while the object is + failed or activating again after having been live, reads and new watches return the last value and get no frames. + A re-exposed member freezes at the re-exposing object; the provider's own source is untouched. +- **Members:** each state member of a reference is a watchable of the provider's member (a replica over a link); + watches taken through a reference belong to it and end when it is released. Methods are functions (`ref.add(args, + ctx)`, `ref.bump(ctx)`). +- **`ref.state`** is a `NativeWatchable` (slice 3b) of a tagged value; watches get each new state: + + ```ts + | { kind: "loading" } | { kind: "live" } + | { kind: "failed"; cause: FailureCause; retryAt: number | undefined } // retryAt undefined: broken, no automatic retry + | { kind: "ended"; error: ChordError } // released by its holder, or its object ended + ``` + +| Situation | Calls | Values | +| --- | --- | --- | +| link drop | park until reconnect | readable (`loading`) | +| object loading or restarting | park until live | readable (`loading`) | +| rollout: provider ahead or installing | park until installed, with a deadline (slice 7) | readable | +| object `failed` | reject at once with the cause | `failed` with cause and `retryAt` | + +- A failed reference stays alive: while held, Chord retries per the backoff; `ref.retry()` tries at once (a Retry + button; over a link the `retry { hold }` message); success goes `loading`, then `live`. A provider tells a pending + hold's consumer each failure (`failed`, with the rest of its backoff window) and the next attempt (`loading`). +- **Parked calls** (slice 3b): calls made while the reference is loading are queued in order, each waiting at most the + reference's `readyTimeout` from the moment it was made (never renewed), then fail `unavailable/park_timeout`; they + are sent in order when it goes live (a new call never overtakes them) and rejected with the cause when it fails. + Otherwise a call ends only through its caller's signal; there is no per-call deadline. +- Calls already sent when a link drops fail `unavailable` and are not resent; calls running on an instance that ends + fail and are not resent. **Releasing a reference rejects the calls made through it** (`released`), locally and over + links (slice 3a). A consumer receiving `gone(h)` rejects its own pending calls on `h` (Q1). +- References survive link drops: one owner of a host's remote references holds them again on every new link of their + path; a re-hold may bind another variant (slice 3b). +- **Consumers** handle a failed call as they handle a disconnect (error shown, user retries); Chat's own submission + `requestId` stays an app concern. + +- **Optional dependencies** are registries owned by the dependent, not optional references: ports contributes its + forwards to `radius.forwards`; without ports the table is empty and radius works. + +- **Authority checks** live in the per-principal view's methods (and in `route.authorize` before a host starts); + there is no async activation guard. An object for an address that carries authority (`LoginFlow { id }`) may exist; + what a caller can see and do is decided by its view. + +--- + +## 8. `ctx` + +Every handler gets a `ctx` (a `Context`): `telemetry` (the current span, section 29), `abortSignal` (the caller +cancelled or the object ended; there is no per-call deadline), `principal` (the caller as its link stamped it; on a +local object's handler the host's principal, none on a client), `incarnation` (the calling host's, section 4), and +`get(Type, params, { readyTimeout? })` (references as the caller sees them, released when the handler settles; a +`get` after that throws `released`). Later: `ctx.trace(name, attributes, fn)` gives `fn` a child `ctx` under a child +span (slice 9, cut C11 to decide); in a job's `run` also `jobId` and `state` (slice 5). Pass `ctx` to everything you wait +on. Code outside objects gets a `ctx` from `host.root` (slice 8). Durable's `Context` is the same module +(`../context`); bridging is explicit. No code versions, no request ids. + +**Contexts are required** (owner, 2026-10-22): every function that takes a context requires it, as the parameter after +the required ones (context last); optional only on error-reporting paths. A getter that cannot take one (a lazy derive +read) uses its owner's. + +**Call-scoped natives** (a native that lives exactly as long as one call: a temporary SSH session for a directory +listing, a file handle for a download) are opened and closed inside the method with `try/finally`; there is no +`ctx.open`: + +```ts +list: async ({ path }, ctx) => { + const session = await ssh.open(config, ctx.abortSignal); + try { return await session.readDir(path, ctx.abortSignal); } + finally { session.close(); } +}, +``` + +--- + +## 9. Documents + +### Definition and handles + +```ts +const Settings = defineState("settings", { + schema: SettingsSchema, + version: 1, + initial: () => ({ theme: "dark" }), + persist: true, // default: in memory + checkpointWhen: () => true, // required for persisted documents; small: a base per write +}); +const Draft = defineState("draft", { + schema: DraftSchema, + version: 2, + migrate: (old, from) => ({ body: old.text }), // pure + initial: () => ({ body: "" }), + keyed: true, // many per object + persist: true, + checkpointWhen: (value, ops, info) => info.deltasSinceBase > 50, +}); + +host.serve(DraftStore, (_, self) => { // local, empty params: may persist (P3) + const settings = self.state(Settings); // THE settings document + const drafts = self.state(Draft); // all Draft documents of this object + return { + settings, // a singleton handle is an exposable Watchable + get: ({ conversation }, ctx) => drafts.get(conversation, ctx), + save: ({ conversation, body }, ctx) => + self.tx(async (tx) => { + const d = await tx.draft(drafts, conversation); + d.body = body; + return d.body.length; + }, ctx), + drop: ({ conversation }, ctx) => self.tx(async (tx) => tx.retire(drafts, conversation), ctx), + }; +}); +``` + +- **Ownership:** a document belongs to exactly one object identity, and so does its line (owner, 2026-10-26: across the + object's attempts, retries, failures and native restarts; an attempt keeps only its natives, routines, derives and + seal token) (persisted ones across restarts; in-memory ones for + the object's life, across reloads and native restarts). Facets have no documents (owner, 2026-10-26: C16). + Others see it only through exposed members. References to other objects are stored as data (addresses). +- **Singleton** (no `keyed`): loaded at activation, kept in memory while the object lives; `handle.value` is + synchronous; the handle is an exposable `Watchable`. +- **Keyed:** addressed by a short key local to the object, loaded on demand; `await handle.get(key, ctx)` reads (creates + nothing); `for await (const key of handle.keys(ctx))` pages keys (both take a context, owner, 2026-10-26) (indexed per owner and definition, also in memory). No + live key list: a collection shown live is a singleton (an object holding a list or map); large collections are + methods returning pages. +- **Values are immutable by convention** (Chord Delta's rule, as in Durable): `handle.value` and published values + are shared, never mutated by their readers, and never copied or frozen; a draft belongs only to its transaction. + +### Writes and transactions + +- **Only `tx.draft`** (as Durable's `tx.doc`): `await tx.draft(handle, key?)` inside `self.tx(fn, ctx)` + returns a draft of the current document, or a draft over `initial()` if it is absent. Reading is drafting (Delta's + overlay: no copy). An unchanged draft (empty ops) commits nothing, so an absent document is never stored by being + read. `tx.retire(handle, key?)` removes a document. `keyed` makes the key mandatory at the type level. There is no + `tx.get` for documents (tables have one, section 10). +- Drafts record ops (assignment, `push`, `Object.assign`); the same document returns the same draft within one + transaction. **Drafts are revoked when the transaction settles (R12, as in Durable):** a transaction writes and + returns at most plain values (an id, a count), never anything taken from a draft. To return data, read after the + commit from the document (`handle.value`, `handle.get`), which then holds the committed revision; a later commit + may slip in between, and the reader gets that newer value. +- **No existence check inside a transaction (R13, as in Durable):** `tx.retire` of an absent document is a no-op (it + never creates); `tx.draft(handle, key)` creates a missing document when the draft changes; + whether a keyed document exists is read outside the transaction (`handle.get(key, ctx)` gives `undefined` for an absent + one). **A singleton always has a value (Q11):** `handle.value` is the written value or `initial()`, never + `undefined`, so an exposed singleton is typed `T`; singleton existence has no API (unlike Durable's `snapshot()`). + + Patterns: + + ```ts + // Return data: write in the transaction, read after the commit. + await self.tx(async (tx) => { (await tx.draft(notes)).items.push(note); }, ctx); + return notes.value.items; // the committed revision (or a newer one) + + // Remove if present: retire is a no-op for an absent document. + await self.tx((tx) => tx.retire(drafts, conversation), ctx); + + // Create if missing: the draft creates it (only if it changes). + await self.tx(async (tx) => { + (await tx.draft(drafts, conversation)).text = text; + }, ctx); + + // Branch on existence (keyed documents): read first, outside the transaction. + if ((await drafts.get(conversation, ctx)) === undefined) { … } + ``` +- Document roots are `JsonObject` (as in Durable; lists live in a field); documents hold JSON only (bytes are a value + and wire type). A singleton that does not exist yet reads `initial()` through its handle; retiring resets it to + `initial()`. Handles may be used only in transactions of their own object. +- **Commit sequence** (Durable's): wait for the object's line (per object, never across objects; reads outside a + transaction do not take it exclusively); run the callback; an error or an abort before the storage write discards + everything; prepare every draft (Delta copies and checks only the placed values); write persisted documents in one + storage batch through the store's commit step, `await store.commit(writes, adopt, ctx)` without the caller's abort + signal (deltas, or a base when `checkpointWhen` says so; the line held until the step returned, as Durable's + `#runCommit`, owner, 2026-10-26); adopt by pointer swap and mark dependents in the step, persisted and in-memory + documents and staged changes in one `host.batch.commit` (any backend error in the step is `storage_failed`, + Durable's rule, owner, 2026-10-26: plugin mistakes are refused when a write is staged; a seal-fenced commit is + rejected `released` before `BEGIN`); then the owner's synchronous flush recomputes dirty derives, sends one message + per link and queues watch callbacks (W1); drafts then throw if touched. In-memory-only commits take no store seq and never + enter the commit queue (W12). No + revalidation of the whole document, no copy, no freeze. +- **Nothing leaves a transaction before it commits (S2):** ids and data reach other owners only after the commit; + reference what is committed (a reference to a later commit dangles on every replica that stops in between). +- **Durable's rule:** inside the callback only `await tx.*`: no other objects, models, processes or network, no second + `self.tx`, no reads past `tx` (otherwise a hang, not an error). Fetch first; decide and write inside. +- A transaction covers only its own object's documents and tables: one line, no transactions across owners (facet + documents were removed, owner, 2026-10-26; shared plugin state is a `local` store object's, reached by methods). +- **Shorthands:** `handle.change(fn, ctx)` (keyed: `handle.change(key, fn, ctx)`) (one document, loaded before the synchronous `fn`), + `handle.retire(ctx)` (keyed: `handle.retire(key, ctx)`); several steps atomically: `self.tx`. + +### Versions + +- A shape or meaning change raises `version` with a pure `migrate(old, from)`. A document migrates on access by the + new definition (in memory too); the migrated base is persisted with the next commit. +- **Schema fingerprints instead of value validation:** `defineState` computes the fingerprint of its structural schema + once (descriptions and titles ignored) and keeps it on the token; every document record stores the fingerprint it was + written with (at creation, and with the new base after a `migrate`). On access Chord compares the token's fingerprint + with the record's: the same version with a different fingerprint fails the object `broken` ("schema changed without + a version bump"). Always on, O(1). Every structural schema change needs a version bump (an additive change gets a + trivial `migrate`). Values are not walked against the schema in production, with two exceptions (R8): + - the result of `migrate` is validated against the new schema once, when it is produced (per document per version + change, off the commit path); a failure refuses the access, keeps the old data and fails the object (slice 6); + - `initial()` is checked once per definition, at its first value (D3, owner, 2026-10-24: `initial()` takes no input, + so a wrong shape is wrong at its first call; checking every call would cost one validation per created document + and catch nothing more). A failing singleton `initial()` fails its object's activation (`broken`); a failing + keyed one fails only the transaction that drafts the absent document, naming the definition (slice 1). +- Old code whose definition is older than the document throws on access. Without `migrate` the access is refused and + the data kept. A changed meaning without a bump cannot be detected. +- Singleton documents migrate during activation and after reload (hydration counts as access). A hydration or + migration failure keeps stored data and fails the whole owning object, not only the exposed member. Keyed + documents migrate on their next access; a migration failure there also fails their owner. A migrated base is written + back by the next commit that changes the document (owner, 2026-10-27); until then a load migrates again (once per + object, since documents stay on the object). + +### Lifetime, persistence + +- **No tags (owner, 2026-10-21):** deleting a session deletes its store and directory; instance data on the server that + refers to a session is a table with a `session` column, cleaned up by its owner (`deleteWhere`, reconciling from the + catalog). +- **In-memory documents are instance state (P2, owner, 2026-10-21):** attached to the live object, with no storage, no + address, no seq and never in the outbox; their commit is prepare and adopt (inside the store's commit step when the + transaction also writes persisted data). +- **Lifetime:** in-memory documents end with their object (last reference released, linger expired, a params change on + reload) and survive reloads and native restarts of a live object; long-lived objects creating many keyed in-memory documents retire + them themselves. Persisted documents stay until retired. +- **Persisted data only in `local` objects with empty params (P3, owner, 2026-10-21):** persisted documents and tables + live in `local` objects with empty params (one per host and type); their address is (host kind, object type, name, + key), never a host instance such as the session id (W7), + so a params change never orphans data. Per-entity data is keyed documents or table rows there, with an app-chosen + key; objects with params keep only in-memory state and read persisted data by key. Renaming a persisted store's type or + definition, or changing how keys are formed, is an explicit one-off migration in the store. + Why not elsewhere: a `remote` object's identity contains the principal key (provenance, grant revision), so its + persisted documents would scatter across access paths and be orphaned after a reconnect; a shared storage key + (`storeKey`) would put two objects with two lines on one document, so it does not exist; params would orphan data + on a params change. Enforced at the type level (`self.state(…, persist)` and `self.table` only in factories of + `local` types with empty params) and as a declaration error at runtime ("persist needs a stable identity: put the + documents into a local shared store"). +- **Durable plugin state:** whatever must survive a host teardown (crash, restart, a DO eviction) is a persisted + document; everything else is rebuilt on activation (in-memory documents, natives, derives, routines, presence, + references re-held by clients; job runs end `lost` unless their records persist). + +### Rules across owners + +- **No plugin reads or writes another plugin's documents** (Chord or Durable). Data crosses plugins only through + state and methods. Data core enforces lives in a format core owns and is handed to core through its API when the data is + created (`createConversation` with `init`), never written into core's documents directly. +- **Invariants across owners** are enforced by the owner of the dependent data, which reconciles from the other + owner's state with an idempotent routine. No cross-owner transactions, no "notify the other owner" calls that can + be lost: + + ```ts + // ports' ForwardStore: "a forward of an archived session is closed" + const catalog = self.get(Catalog, {}); + self.on(catalog.sessions, async ({ value: sessions }, ctx) => { + for (const f of forwards.value.list) if (sessions[f.session]?.archived) await close(f.id, ctx); // idempotent + }); + ``` + + After a crash the routine runs again with the current value; sessions does not know ports exists. + +--- + +## 10. Store and storage + +> **Superseded (owner, 2026-10-21):** persisted data lives in one transactional store per session, designed in +> `packages/chord/docs/next/store/book.md` and decided in the log ("The store", S1-S17, W1-W14, P2-P4, "Flat columns", +> "Query runner", "Index definitions are recorded", "Blobs and derived data"). In short: backends SQLite (Node, Durable +> Objects, sqlite-wasm in the browser; tests on SQLite `:memory:`, no hand-written memory backend, owner, 2026-10-27); real tables owned by objects with empty params (P3), flat +> columns (scalars, every `str()` as plain UTF-8 `TEXT` in code point order with lone surrogates replaced and NUL refused (owner, 2026-10-26, superseding W8), `json(...)` with its own version and on-access migration), a small +> query language, writes `put`/`update`/`delete`/`updateWhere`/`deleteWhere`; one documents engine for Durable +> (`StoreStorage`) and Chord; one commit queue per store that writes, adopts and marks (S1, W1), with `Storage.commit( +> writes, adopt, context)` in Durable; an outbox of ordered operations in the same transaction (no compaction at first); +> eager migrations by scan with row `v` for eager steps only. In-memory documents are instance state with no storage +> (P2). The text below describes the per-object lines, Delta trackers and commit sequence, which stay; its storage +> interface is replaced by the store. + +- **What stays from Durable's store design** (`durable/src/session/transaction.ts`, `session.ts`): one Delta tracker + per document held by its object, prepare and adopt by pointer swap, ops per commit, lines per object, migrate on + access, `checkpointWhen` (required for persisted documents, R14), retire, and the documents engine's invariants (create before change, no + change after retire, one live incarnation per address, `deltasSinceBase`). +- **Where persisted documents live:** rows of the store's documents engine (S13) with address `(namespace: "chord", + owner: ":", kind: , key)` (key `""` for a singleton), never a host instance (W7); ids from the store's + counter (S10). Chord ships no storage of its own: no `DocumentStorage` interface, no Chord SQLite or memory document + storage and no Chord storage conformance suite (the store's conformance suite covers the engine); the frozen `next/` + storages are not ported. The store is passed to the session's host by base code (owner, 2026-10-26): + + ```ts + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(join(dir, "session.sqlite"))), ctx); + const worker = createHost

({ address: Session({ id }), principal, store, onError }); // tests: SQLite :memory: + … + await worker.close(ctx); + await store.close(ctx); // the app owns the store; in 6b the Durable Session shares it + ``` + + `openStore` runs the store's own migrations, checks the file is UTF-8 and reads `lastSeq`, the definitions and the + id counters once; `sqliteBackend(db)` takes any `SqliteDatabase` (Node, Durable Objects, sqlite-wasm later); the + backend interface is internal; `importStore(backend, archive, ctx)` takes the same backends. No `openNodeStore`. +- **Loading at activation (owner, 2026-10-26):** persisted singletons load right after the factory, sequentially, + all singletons of the object with plain reads (owner, 2026-10-27: the object holds its line, so no read transaction + is needed), once per object, before its references are bound: find, load (base plus + deltas), fingerprint check, `migrate` with its result validated once (R8). A storage error is `storage_failed`; a + fingerprint mismatch, a throwing `migrate` or an invalid migration result fails the object `broken`. +- **Statements per commit (owner, 2026-10-26):** the store keeps `lastSeq` (the seq of its last commit) in memory (read at open, + persisted with each commit, S9) and folds the documents engine's checks into the writes (unique keys, a partial + unique index on the live address, conditional `UPDATE … WHERE id = ? AND version = ?`): a delta commit is about three + statements (revision, the record's counted conditional `UPDATE`, `lastSeq`), plus one `INSERT` with an outbox. Durable's ids in the store: `StoreStorage` keeps a `record_ids` table (one + namespace across Durable's five tables), its counter at open `coalesce(max(id), 1) + 1`. +- **Tables in a transaction (owner, 2026-10-26):** the table handle first, keys as objects with exactly the key + columns, reads async, writes synchronous and buffered like a draft's ops (checked and staged, no I/O; written in the + commit step or dropped on failure); no write shorthands on the handle: + + ```ts + self.tx(async (tx) => { + const old = await tx.get(edits, { id }); // overlays own put/update/delete (W10 after bulk) + const { items, next } = await tx.page(edits, q, 50, { after }); // committed rows, stable for the transaction + tx.put(edits, row); + tx.update(edits, { id }, { at: now }); + tx.delete(edits, { id }); + tx.updateWhere(edits, (e) => prefix(e.path, from), (e) => ({ path: replacePrefix(e.path, from, to) })); + tx.deleteWhere(edits, (e) => lt(e.at, cutoff)); + }, ctx); + await edits.get({ id }, ctx); // outside: committed state, ctx last + ``` +- **The outbox (owner, 2026-10-26):** optional, `openStore(backend, ctx, { outbox: true })` (an omitted option keeps the + persisted setting; switching needs an explicit `true` or `false`), persisted in the store's + metadata (turning it on starts sync from an archive at the current `lastSeq`); a table in the store's backend, + written in the commit step's transaction (atomic for one `INSERT`); exposed as `store.outbox.read(afterSeq, limit, + ctx)` -> `{ seq, writes }[]` and `store.outbox.trim(throughSeq, ctx)`, no pluggable encoding; versioned by the store's + schema migrations (its layout) and a `format` per outbox commit (the write vocabulary, the archive's `format`; older + formats stepped forward on read, newer ones refused). +- **Writes, commits, batches, archives (owner, 2026-10-26):** a *write* (`StoreWrite`, tagged by `kind`) is one change + of the store (`put`, `update`, `updateWhere`, `table`, `index`, `document.create`, …); a *commit* the writes of one + seq in the outbox; a *batch* the commits `from..to` that sync uploads and a replica applies; an *archive* a manifest, + the writes from 0 to `lastSeq` and an `end`. `lastSeq` is the seq of the store's last commit. +- **Export and import (owner, 2026-10-26):** `store.export(ctx)` yields the archive's parts, the state after its + `lastSeq` as writes (manifest, `table`, `index`, `put`, `document.create`, `end`), holding the commit queue + until the iteration ends or `ctx` aborts; `importStore(backend, parts, ctx)` applies them into an empty backend in + one transaction with the replica's applier. Lossless (owner, 2026-10-26): every `put` write carries its versions (`$v`, + each `json(...)` column's) in the outbox and in archives, a `table` write its state, a `latest` document its current + value; the applier never infers a version. The encoding is pluggable; plain JSONL ships first. Sync is not export: + it ships outbox batches as commits happen. +- **`json(schema, { version, migrate? })` columns (owner, 2026-10-26):** schema and version required, `migrate` + optional and pure, on access (W11), its result checked once (R8); without it (or failing) the read is refused and + the owner fails `broken`; each column's version and schema fingerprint are recorded in the table definition and + compared at activation (same version, other fingerprint: `broken`, D9); a read never writes back. +- **Table versions: `evolveTable` (2026-10-27, the second spec review's majority):** a table's history is a base + (`defineTable`) and one `evolveTable(prev, (t) => t.rename(…).add(…).rows(…).drop(…).index(…))` per version, each + materialized and fingerprinted at declaration, as contracts' `evolve`; the store replays the recorded version's + successors in order (DDL and `migrating`, the data step, `ready`); key columns are immutable in `set`. +- **Eager migrations and addresses (owner, 2026-10-26):** the activation attempt's `readyTimeout` deadline does not + run while an eager migration step runs (getters still time out by their own); a host without an address uses `client` + as its host kind in store addresses. +- **Eager `rows` migrations** commit in chunks of 100 rows (owner, 2026-10-26; a fixed constant: about 1 ms of commit + step per chunk, the total time is the same for larger chunks, and every other owner's commit waits for the step). + Between chunks the migration yields one macrotask without `setTimeout`'s 1 ms clamp: `setImmediate` where the + runtime has it, otherwise a `MessageChannel` message (2026-10-27, slice 6a build reviews). +- **Eviction as in Durable,** for keyed persisted documents only (persisted singletons and `at(key)` documents stay + in memory while their object lives: `handle.value` is a synchronous read; owner, 2026-10-26): one not read, written or watched for the retention + period (default 10 minutes) leaves memory and reloads on the next access. A storage error ends the store and closes + the host; `onError` receives `{ kind: "storage_failed" }` and the app decides (section 3). Any backend error in a + commit step, or on a read whose `ctx` was not aborted, is a storage error (Durable's rule); later calls reject + `unavailable/storage_failed`. + +--- + +## 11. Kinds of state + +| Kind | For | API | +| --- | --- | --- | +| document | data the object owns | `self.state`, `self.tx` | +| derive | computed values | `self.derive(sources, fn)` | +| Durable's views | the conversation view, task graph and Durable documents | `Watchable`s through Durable's watches; feed derives and state members | +| paged method | large or queryable collections | a method returning pages (chat history) | +| table | a plugin's own records with queries (owner, 2026-10-21) | `self.table`, the store's query DSL | +| plugin database | derived data only: full-text search, caches (rebuilt, never synced) | a native opened by its owner | + +App state lives in Chord documents; only transcript-bound data (entries, documents that fork and rewind with a +conversation) lives in Durable. + +### Watchables (owner, 2026-10-21; log "Watchable and one watch API", W1-W4) + +There is no `replicatedState` in v2. Everything observable is one of two kinds, sharing one lifecycle: + +```ts +interface Watchable { // JSON only (no bytes); documents, derives, reference members, constant(v), Durable's views + readonly value: T; + watch(ctx: Context): Promise>; // listener (call, ctx), call: WatchCall (below) +} +interface NativeWatchable { // self.writable(initial) with set(), composed registry values + readonly value: T; + watch(ctx: Context): Promise>; // listener (call, ctx), call: NativeWatchCall; a change is a new value (!==) +} +type WatchCall = // owner, 2026-10-27: a recoverable failure is a call, the watch stays open + | { kind: "value"; value: T; ops: Op[]; retired: boolean } + | { kind: "failed"; error: unknown }; +type NativeWatchCall = { kind: "value"; value: T } | { kind: "failed"; error: unknown }; +// const w = await notes.watch(ctx); send(w.value); +// w.start(async (call, ctx) => { if (call.kind === "failed") return showError(call.error); render(call.value); }); +// … await w.stop(); +``` + +- **Failures are calls, watches end only for good (owner, 2026-10-27):** a source that fails and can recover (a + derive whose compute throws, a replica member a hold lists as failed, a member a new variant lacks, a registry whose + `compose` throws, slice 8) gives its watches one `{ kind: "failed", error }` call and keeps them open; its next good + value arrives as `[["r", value]]` (folded with the failure for a busy listener). A failed derive recomputes on every + later change of its sources and on a re-bind (self-healing). A watch ends only when nothing can come back: its owner + ended (`ended`), `stop()` or a cancelled context, a throwing listener (`listener_error`), `storage_failed`. `self.on` + handlers are not called for failures: a routine waits for the next good value (the failure is reported where it + happens: the derive's owner reports `derive_failed { object, member?, error }` once per failure episode, the first + failure after a value or after creation, not for a derive failing only because a source failed). `ref.state` stays + a separate watchable of the object's reachability (`loading | live | failed | ended`), with the same words. A + replica member that vanished with a variant is kept and comes back with a later binding; while failed it accepts + only `[["r", value]]`. +- **Watch handles are Durable's, moved into Chord** (Durable imports them): `watch()` atomically takes the current + value and buffers later commits; `start(listener)` installs the one listener; commits are delivered in order, async + listeners awaited; a listener still busy with an earlier frame gets the commits made meanwhile as one call, from + the **pending queue** (owner, 2026-10-23): ops stay raw and are folded through a leaf index (a map from path to + the last pending append or primitive set; a new one on that path merges into it; any structural op clears the + map; `r` drops everything before it); past 10,000 pending ops only "replace with the current value" is kept (no + size, text or time caps; no folding of list appends or trims; ops come only from Chord's tracker and + `diffRevisions`, so there are no defences against hand-written ops). `PendingOps` is Chord's shared Delta code. The + same queue holds a link hold's ops while its window is full (section 3). Natives: the newest value. Durable's + event stream keeps its own queue (events are not state); `stop()` is idempotent, drops pending frames, never aborts or awaits a running call; `stop()` and + `closed` return the end, tagged by `kind` (owner, 2026-10-27): `stopped`, `aborted` (its context aborted), `ended`, + `listener_error { error }`, `failed { error }` (a final failure; a recoverable one is a call). Durable's synchronous + observers take the same shapes: `advance(call, ctx)`, `end(end)`. + **Retirement (owner, 2026-10-26, option A; replaces W2a's `[["r", null]]` frame and the end `retired`):** one + mechanism for Chord and Durable documents, singletons and keyed alike. A watch follows its document's address; a + retirement is one last call with a `retired` flag (a Chord document: `initial()` with `[["r", initial()]]`, which a + wire replica applies; a Durable document: the last good value, no ops); the watch goes on, and a later + write delivers the new incarnation as `[["r", value]]`. A watch ends `ended` with its owner (the object of a Chord + document; for Durable the task of a task document, right after its retirement call, or the Session). Listeners take + one object before the context (owner, 2026-10-26; tagged since 2026-10-27, above): `(call, ctx)`, a `WatchCall` or + `NativeWatchCall`. + A retirement call carries the address's value after the retirement: `initial()` for a Chord document, the last value + (no ops) for a Durable one, which has none while absent. Watching an absent address gives `undefined`. On the wire a + `state` entry lists retired members like failed ones. A per-key source in a factory, `handle.at(key)`, is a singleton per key: loaded at activation, `initial()` while absent, usable as a state member, derive source and `on` source (owner, 2026-10-26). A + stopped watch releases every reference it held (W3). +- **Only `Watchable` can be a contract state member or go on the wire**, by type and by a kind check when the + object is served (Q14); a `NativeWatchable` never can, even holding JSON. Mutating a native value in place notifies + nobody. +- **Publication (W1):** the commit step writes, adopts (pointer swaps, Durable's touched views dispatched by id) and + marks (dirty derives; ops appended by reference to each touched hold's batch and each watch's queue). Right after + the step, synchronously, the owner's flush recomputes the dirty derives someone holds or watches (dependency order), + encodes one entry per touched hold into one `state` message per link and origin, and queues watch callbacks. A + throwing derive fails only its member; reentrant `self.tx`/`watch`/`release`/`get`/`hold`/`load`/`close` throw + `invalid/reentrant`; a delivery or flush that throws is reported (`delivery_failed`) and the others still run. The + same boundary runs without a store seq for in-memory commits, `writable.set`, registry changes and inbound wire + batches (a `bound` or `state` message is applied as one batch before listeners run). There is **one batch per host**, + passed to its sources (no module-level singleton); a dirty derive nobody holds or watches recomputes lazily on its + next read. +- **Consumers:** derives and the wire use the internal flush; app code uses watches; `self.on` is a watch owned by + the object (first call with the current value, then every commit; Chord awaits a running call on reload); clients + get ordered ops over the link, folded by the provider's pending queue when they fall behind (section 3); a re-hold + gets a fresh snapshot. + +--- + +## 12. Derives + +`self.derive(sources, (values) => value)`: a synchronous pure function, recomputed when a source changes and +diffed into ops for the wire. Sources are `Watchable`s and `NativeWatchable`s (documents, other derives, reference +members, Durable's views, registry values); the result is always JSON (state is JSON only; bytes only in method arguments and results). A derive is marked dirty in the +commit step and recomputed in the owner's flush right after it, so its change travels in the same message as its +source (W1); a dirty derive nobody holds recomputes on its next read. A throwing derive fails only its member +(`broken`). There is no `previous` (W14): `mapShared` keeps unchanged elements identical. + +- **Built on what exists:** derives sit on Chord's internal publication primitive (not today's `cell.ts` or + `replicatedState`); the diff is + Delta's `diffRevisions` called directly (identity short-circuit: unchanged shared subtrees cost one `===`; bytes are + atomic leaves compared by identity; equality never by stringify). +- **Container width (R24):** structural sharing is O(change) along a path, not across a container's width: changing + one item of a flat array copies the array's slots, and `mapShared` visits every element (one identity comparison + each). Large collections are keyed documents or bounded windows; a benchmark of long arrays (allocations, element + visits per change) sets the documented limit. +- **The footgun:** a derive that builds new objects for unchanged data makes every commit cost O(size of the value), + per view, because the diff must compare contents. Unchanged parts must be the same objects as last time. +- **`mapShared(f)`** (from `@earendil-works/chord/next` state; it moves into Delta when v2 goes upstream): an identity cache per source element: `f` runs only for new or + changed source elements, unchanged ones return their previous result object. With it a list derive costs pointer + comparisons, not content comparisons: + + ```ts + const strip = mapShared((n: Note) => ({ text: n.text, at: n.at })); + self.derive([notes], ([n]) => ({ list: n.list.map(strip) })); + // one appended note: one new object, 2,000 identical references, diffRevisions emits one insert + ``` + +- **Cookbook** (documentation with examples per case): select a field, look up by key, small aggregates (just compute; + the value is small); filter or map a list (`mapShared`); redaction (preferably separate documents per visibility, + otherwise `mapShared`; computed once in the shared store, not per view); joins across two sources (`mapShared` plus + the looked-up source's revision); a window over a large list (take a section of the source elements without + rebuilding them, cache per-element measurements by identity, keep the window's start stable so changes arrive as + appends). Large combined views from several sources re-path the sources' ops (as Durable's `ConversationView`); + application code never interprets ops. + +--- + +## 13. Natives + +```ts +const watcher = self.open(async (context) => watchDir(path, { signal: context.abortSignal }), (w) => w.close()); +self.onLive(async (ctx) => { for await (const f of watcher.value.files(ctx.abortSignal)) await importFile(f, ctx); }); +``` + +- `self.open(open, close, { closeTimeout? })` (slice 2): `open(context)` gets a context aborted when the attempt is + abandoned (a timeout, a cycle, its end); `close(value, reason, context)` gets one aborted at the close timeout, after + which a possible leak is reported (`close_timeout`) and the rest continues. It returns a handle with `.value`, not a + proxy: reading it before its open resolved throws `not_ready`, after its close started `released`. A value that + arrives after its attempt was abandoned is closed at once. + Natives open during activation (the object goes live after all opens) and close in reverse order when it ends. + The opened value never changes during that attempt: no `self.resource`, versions, `replace`, `deps` or stale + handles. To change a live native, the object restarts. A partly failed activation closes already-open natives + with reason `end`; a later attempt opens them all anew. `retryOn` may appear anywhere in the synchronous factory. + Type names: `Native` (the handle), `NativeCloseReason`. +- **Order:** natives open one after another in declaration order (a later opener may use an earlier handle's + `.value`) and close in reverse. A throwing `close` does not stop the others: on `host.close()` all errors are + collected and it rejects with them; an object ending in the background reports them to `onError`. +- Behaviour around a native goes into routines; listeners on natives outside routines are forbidden. +- The facet may not open natives. +- What an old native does after its owner restarts is its owner's `close` (a model runtime keeps working until + collected; a closed SSH connection fails its in-flight `exec`). +- An open that fails makes the object `failed` (`transient` with backoff, or `broken` if it says so). + +- **`close` gets a reason:** `reason: { kind: "reload" } | { kind: "close" } | + { kind: "end" }` (the plugin is reloaded and the object is native / the host shuts down / the last reference ended or + the type was removed). Chord aborts only the object's own calls and jobs; other work using the native (Durable tool + executions on a code mode sandbox) is the owner's: `close` may drain instead of tearing down, e.g. seal the sandbox + (no new scripts) and end it when its running scripts finish or hit their deadline. +- **Restart stays sequential (R16):** close the old native, then open the new one. Two patterns for natives whose + resource has work in flight: + - **Drain in the background:** `close` detaches the old resource, returns at once, and disposes of it when its work + has finished or after `closeTimeout` (30 s at most, Q7: a longer script is aborted); Chord then opens the new one + immediately. Code mode's sandbox does + this; its tool entry looks up the current sandbox at `execute()`. + - **Swap through a registry:** the resource is a registry contribution; readers always find the current one, and a + reader that must wait for one uses a cancellable "wait until present" with the park deadline. + +--- + +## 14. Routines + +Two kinds, declared in the factory (owner, 2026-10-24; C4 decided; slice 5): + +```ts +self.on(catalog.sessions, async ({ value: sessions }, ctx) => { // reacts to a value: first the current one, then every change + for (const f of forwards.value.list) if (sessions[f.session]?.archived) await close(f.id, ctx); // idempotent +}); +self.onLive(async (ctx) => { // called once when the object goes live + for await (const file of watcher.value.files(ctx.abortSignal)) await importFile(file, ctx); +}); +self.onLive(async (ctx) => { // periodic: a loop with sleep + for (;;) { await sleep(HOUR, ctx); await dropOld(ctx); } +}); +self.onLive(async (ctx) => { // set up, wait for the end, clean up + const timer = setInterval(tick, 1000); + try { await untilAborted(ctx); } finally { clearInterval(timer); } +}); +``` + +- **Lifecycle (both kinds):** declared in objects, never in facets (owner, 2026-10-26: everything is done through + objects); they start when the object goes live (in the same turn) + and stop when it ends (their `ctx` is aborted, then + the drain waits for running calls, bounded by `closeTimeout`). Each call gets its own `ctx` (the object's principal + for a remote type, the host's for a local one; the host's incarnation; `ctx.get` released when the call settles). +- **On reload** the old routines' `ctx` is aborted; the new routines start once all old routine calls of the object + have finished (bounded; a stuck call is reported and the new routines start anyway): a handover per object, so + routines need no names. Example it prevents: an old handler midway through `restart(W1)` and a new one seeing W1 + still pending and restarting it again. The bound is `closeTimeout` (owner, 2026-10-24); calls do not wait for the + handover, only the new routines do (slice 7). +- **`on(source, fn)`:** any watchable (a document, a derive, a reference member, a registry value, `ref.state`). The + first call gets the current value (`[["r", value]]`, W2b), then one call per change; calls never overlap (S8); + changes made during a call arrive folded in the next (newest value, folded ops); a running call is not aborted by a + change (native sources too: a busy call is followed by one call with the newest value). A throwing call is + reported and called again with the current value after a backoff (1 s doubling to 60 s; + a change during the backoff calls at once; success resets it). The source ending (a released reference, its object's + end) ends the routine without a report; a retired document does not end it (the call has `retired: true`). +- **`onLive(fn)`:** called once when the object goes live. Returning ends it until the next reload or activation; + throwing is reported and restarts it after the same backoff (reset after a run of at least a minute); an abort is + no error. +- **Idempotent reconcilers:** a routine reconciles from the current value, writes no "in progress" flags and starts a + job for long work. Chord keeps no state for routines across reloads or restarts; what must survive is a document of + the object (e.g. a `lastCleanup` time, which also survives a crash). +- **Helpers:** `sleep(ms, ctx)` (rejects when `ctx` is aborted, leaves no timer or listener) and `untilAborted(ctx)`. + Routines and handlers wait only through calls that take `ctx`; otherwise Chord cannot stop them, and the handover + and the drain report them stuck. +- **Errors:** `onError` gets `routine_failed { object, routine, error }`, `routine` a `Position` (`{ kind: "on" | + "onLive", position }`; natives use the same type); never uncaught. +- **Keep-alive:** routines do not count in `host.activity` (an `onLive` runs forever). +- No bare timers or callbacks that touch state outside routines, jobs or calls. Gone (C4): `every` (an `onLive` with + `sleep`), `custom` and its `run(fn)` (an `onLive` with `try`/`finally`; callbacks that must run one at a time go + through a small queue or the source's async iterator), names, a due time kept across reloads. + +--- + +## 15. Jobs + +Owned work a call starts that outlives the call (a shell command, a session clone, a peer send, a login's work), in +Durable's sandwich: **intent** (a transaction), **run** (any effect), **finish** (a transaction on the current code). +Decided with the owner on 2026-10-24 (decision log "Routines and jobs open points", "Jobs keep accepts", "Job crash +handling"; C3 settled this way); slice 5 builds jobs in memory, slice 6 persistence, slice 7 their reload rules. + +```ts +const ShellState = defineState("chat.shell.state", { + schema: obj({ output: str() }), version: 1, initial: () => ({ output: "" }), +}); +const ShellJob = defineJob("chat.shell", { + version: 1, + args: obj({ command: str(), author: Author }), + result: obj({ exit: int(), output: str() }), + state: ShellState, // optional: one document per run + concurrency: 1, // optional: at most one active run per object +}); + +host.serve(ChatStore, ({ conversation }, self) => { // local: no principals; views filter (below) + const entries = self.state(ChatEntries); + const shell = self.job(ShellJob, { + run: async ({ command }, ctx) => { + const out = new OutputBuffer({ maxLines: 2000, retain: "tail" }); + const progress = throttle(100, () => ctx.state.change((d) => { d.output = out.snapshot().text; })); + try { + const r = await runProcess(command, ctx.abortSignal, (chunk) => { out.push(chunk); progress.schedule(); }); + return { exit: r.code, output: out.snapshot().text }; + } finally { + await progress.flush(); // the last output reaches outcome.state, also after an abort + } + }, + finish: async (outcome, tx) => { + const list = (await tx.draft(entries)).list; + list.push(outcome.kind === "done" + ? shellEntry(outcome.args, outcome.result) + : shellEntry(outcome.args, { interrupted: outcome.error.kind, output: outcome.state.output })); + }, // the run's state document is retired in this commit + }); + return { + shells: shell.running, + runShell: async ({ command, author, requestId }, ctx) => + ({ id: (await shell.start({ command, author }, ctx, { key: requestId })).id }), + abortShell: async ({ id }) => shell.abort(id), + }; +}); +``` + +- **Definition:** `defineJob(name, { version, args, result, state?, concurrency?, accepts?, persist? })`. `args` and + `result` are schemas; `state` is a document definition (a singleton, one document per run); `persist` arrives with + slice 6. `self.job(Job, { run, finish? })` is declared in an object's factory (not in a facet) and returns the job + handle: `start(args, ctx, { key? })` (a `Run { id, outcome(ctx) }` after the commit), `tx.start(job, args, { key? })` + inside `self.tx` (a `Run` handle at once, whose `outcome` rejects if the transaction is discarded), `outcome(id, ctx)` + (unsettled runs only), `abort(id)` (async and stored with persisted jobs, slice 6b) and `running`. +- **1. Intent:** one transaction on the owner's line. Arguments are checked against `args` (closed schema, like + method arguments). Dedupe first: a run with the same `key` that is not settled is returned as it is (the run whose + `finish` this transaction is never counts, so `finish` can start the same job again with the same key). Then the + `concurrency` check: at the limit `start` rejects with `unavailable/busy` (a queue is the owner's, built from a + document and a routine). Otherwise a run record `{ id: uuidv7, version, args, key }` and its state document + (`initial()`) are written in the commit, with whatever else the transaction writes (`tx.start`: a request entry and + the run, both or neither). `run` starts after the commit. +- **2. Run:** `run(args, ctx)` may do anything, outside transactions. `ctx` is a `Ctx` (the object's principal for a + remote type, the host's for a local one) plus `jobId` and `state`: `ctx.state.value` (synchronous) and + `ctx.state.change(fn)` (a transaction on the owner's line on the run's document, without the run's abort signal). + State changes are accepted until `run` settles, also after an abort (so output written while stopping still reaches `outcome.state`), and are + refused (`released`) afterwards, never recreating a retired document. Each change is a commit and, while someone + watches, a message: authors throttle frequent changes themselves (Chord does not coalesce them); awaiting or + catching each change is the author's (an unhandled rejection is the author's bug). External writes carry keys made + from `ctx.jobId`. A `run` that throws after its signal fired ends `aborted`, otherwise `threw` (reported as + `job_failed`). +- **3. Finish:** when `run` settles, Chord records the outcome on the run (`phase: "finishing"`), then calls the + current code's `finish(outcome, tx)` in a transaction (Durable's rule: only `await tx.*`). In the same commit Chord + deletes the run record and its key (Q9: dedupe covers unsettled runs only) and retires the run's state document. A + `finish` that throws changes nothing: it is retried with backoff (1 s doubling to 60 s), each failure reported + (`job_finish_failed`); after five consecutive failures the run is `blocked { finish_failed }`, its outcome, state + and key kept, retried on the next reload. `finish` is optional (then the commit only cleans up). + + ```ts + type Outcome = { readonly id: string; readonly args: Args; readonly state: State } & ( + | { readonly kind: "done"; readonly result: Result } + | { readonly kind: "failed"; readonly error: JobError } + ); + type JobError = + | { readonly kind: "threw"; readonly error: ChordError | PluginError } // run threw (reported) + | { readonly kind: "aborted"; readonly reason: "abort" | "reload" | "restart" }; + // abort(id); a native restart on reload (slice 7); "restart": a persisted run abandoned after a crash (slice 6b), + // or a run cut off by its attempt's seal while the object continues (slice 6a) + ``` +- **Abort is a request:** `abort(id)` fires `run`'s `ctx.abortSignal`; `run` decides what it returns (a result that + already arrived stays a success); a run ignoring the signal keeps its slot. Abort of a finishing run does nothing; + abort of a blocked run drops it (no `finish`, its record, key and state document go, reported; owner, 2026-10-24). +- **Waiting:** `run.outcome(ctx)` resolves with the outcome once `finish` committed; the caller's abort ends only its + wait. Across a link nobody waits on a long call: UIs watch state. +- **`running`:** a JSON watchable, one entry per unsettled run, tagged by phase: + `{ id, version, key?, args, state, phase: "running" | "finishing" } | { id, version, phase: "blocked", reason }`. A + blocked run carries no `args` or `state` (it may be of another version than the code); views filter it as they + filter anything. Per-principal views derive what each principal may see (§6, the shared store): a reader without + output, a guest only their own runs. +- **Concurrency and keep-alive:** `concurrency` counts runs that are running or finishing; blocked runs do not count. + **Runs belong to the object** (owner, 2026-10-26): records, keys, the active count, `running`, run state documents + and pending actions are the object's, across attempts and native restarts; an attempt has the implementation and + its running invocations; a run's state changes and outcome are fenced by the run's own invocation; `finish` runs + on the current live attempt. **Jobs keep their object alive** (D4): running and finishing runs are a count on the object, not references (they + never appear in `host.references`); an object whose last reference goes stays while the count is above zero, and + ends (after its type's `linger`) when it reaches zero, dropping blocked runs with a report. `host.activity.jobs` + counts them. +- **Object end:** revocation (R11), `host.close` and a reload that removes the type end the object at once: runs get + the abort signal, the drain is bounded by `closeTimeout`; an in-memory run's outcome and state go with the object + (no `finish`). Jobs in `local` stores are never revoked (no principal): a store that runs work for a principal keeps + the grant as data and reconciles with a routine (§25). +- **Versions (`accepts`, slice 7):** a job is versioned like everything else; `accepts` maps an older version's args + forwards and its result forwards (`{ 1: { args: (old) => current, result: (old) => current } }`; a producer's + direction, unlike a contract's). A run started on an older version finishes on its old code across a pure reload; + `finish` and `running` see its args and result mapped (mapped once per run, never per publication); its state + document migrates through its own `migrate` (when it may migrate while the old code still writes it is slice 7's + rule). Without an adapter, or for a run of a newer version than the code (a rollback), the run is + `blocked { version }` until a reload brings matching code. A version change aborts nothing and discards nothing. +- **Persistence (slice 6):** `persist: { resume: "rerun" | "abandon" }` keeps the run record and its state document + in the owner's store (owner: a `local` object with empty params, P3). How a run resumes after a crash is required, no + default: rerun (the current + `run`, args mapped forwards, state migrated; only work since the last state change repeats, so it must be + idempotent) or abandon (`finish` gets `failed { aborted, reason: "restart" }`); wording aligned with Durable + (replay-safe, abandon on restart). A recorded outcome resumes only `finish`, never `run`; an abort request is + persisted. A restart limit as Durable's: five crash interruptions, counted and persisted, clean closes not counted; + then `blocked { restart_limit }`, cleared by `abort` or a reload. A persisted state document may use the deferred + persistence option (stored within about a second, flushed before the outcome is recorded). +- **Chaining:** `finish` may `tx.start` the next job, atomically with the outcome (separate steps, branches). +- **Not jobs:** request/response work (a method), standing behaviour (a routine), input into running work (a flow, + §20: the owner's document plus methods), multi-step durable work with sleeps, waits or ownership trees on a host + with a harness (a Durable task). Jobs as objects (#9) are rejected: a third lifetime, two objects per run for two + principals, a `finish` across owners. + +--- + +## 16. The facet + +```ts +import { defineFacet } from "@app/core"; // base code's defineFacet, bound to the app's P + +export default defineFacet((host, self) => { // `self` (contributions) arrives with slice 8 + self.contribute(Http, { routes: { "GET /notes/export": exportNotes } }); + host.serve(Notes, (params, principal, self) => …); + host.serve(NoteStore, (params, self) => …); +}); +``` + +- **Facets are typed by the app's principal (owner, 2026-10-21):** base code exports `defineFacet` already bound to its + `P` (`export const defineFacet = bindFacet

()`, next to the principal union), and every facet (server, worker, + web, TUI) imports it from there; `host.load` accepts only facets typed for the host's `P`, so a factory's `principal` + narrows by variant. Nothing is checked at runtime: the principal a factory receives is the one the link stamped. +- A plugin's per-host setup, **declarations only** (owner, 2026-10-26): `host.serve`, `self.contribute` (slice 8), + `host.references`; no natives, no documents (C16 accepted), no routines, no jobs. Everything a plugin does at runtime + is done through objects: shared or persisted state and standing behaviour live in a `local` store object with empty + params (host lifetime, its own failure domain, R18 recovers its dependents), as p2 does (its 63 facets take only + `host`). A facet cannot fail after `load`. +- `host.serve(Type, factory, { linger? })`: `linger` (milliseconds, 0 by default; slice 3b) keeps a live object with + params alive that long after its last reference, so a new reference within that time reuses it with its documents + and dependencies; it never applies to objects that are not live. +- `host.serve` and contributions are valid only while the setup runs. The facet never `self.get`s a type it serves + itself. +- Between plugins only: object types and registries. No natives across plugins except through registry entries. + +--- + +## 17. Registries + +```ts +export const Http = defineRegistry("chord.http", { + version: 2, + accepts: { 1: (old: HttpContributionV1): HttpContribution => upgrade(old) }, + compose: (table, { report }) => buildRouter(table, report), // pure; always delivers a value; reports conflicts +}); +self.contribute(Http, { routes: { … }, wraps: [ … ] }); +const router = host.registry(Http).value; // live +``` + +- Each contributor (a facet or an object) contributes **at most one value per registry**; the table is an immutable, + ordered list of `(contributor, value)`. Contributor order is sorted by plugin name (never load order): deterministic + on every host and across reloads, and deliberately meaningless; meaningful order comes from the owner's composition + (e.g. HTTP phases). A reload replaces a contribution as a + whole. +- **`compose` is a runtime composition, not a staging check:** contributions can be dynamic (known only once a native + is open, or changing at runtime; contributions may be readables), so no conflict check can be complete in staging. + `compose(table, { report })` is pure and runs whenever the table changes (after a commit, when a contributor goes + live or ends, when a dynamic contribution changes). It always delivers a value with its fallback built in (e.g. the + first by plugin name wins) and reports conflicts through `report` (owner notices and telemetry events); readers get + a plain value and never handle conflicts. Only if `compose` throws is the registry in an error state, and readers + get an error when they use it. Work that started earlier keeps the value it read (a pinned Durable phase keeps its + old table). Helpers: `keyed(table, entriesOf)`, `ordered(table, itemsOf, orderOf?)`. +- **Who contributes (R21):** only facets and host-lifetime objects (`local`, empty params); their factory's `self` has `contribute`, a + referenced object's `self` does not (a type error). Reading a registry's composed value stays open to every object. +- **A contribution is in the table exactly while its contributor is live:** a facet's while its plugin is loaded; an + object's from `live` until it ends, fails or restarts. +- **The owner decides** conflicts, order and selection (which contributions apply where). +- **One authoritative declaration (Q8):** only the facet of the plugin that defines a registry declares it to the host + (`compose`, version, `accepts`); other plugins' tokens only identify it by name and types (contribute and read, + never bring their own `compose`). A reload of the defining plugin replaces the declaration atomically and the table + recomposes. Without a declaration (the defining plugin is not loaded) the registry has no composed value: readers + get an error like a failed reference; contributions wait in the table. +- **Versions (Q8, the owner-adapter pattern):** a contribution carries the registry version its plugin was built + against; older ones are adapted with the defining facet's pure `accepts` when the table is built. A contribution + without a matching version or adapter (older without adapter, or newer) is left out of the table and reported (an + owner notice naming plugin, registry and versions); its plugin keeps working. Contributions are in-process values + (often functions) and are not schema-checked. +- **Use sites guard contributions:** a contribution is untrusted code (third-party plugins are a goal of p3). Every + use site catches synchronous throws and rejections when it calls an entry, reports naming the contributor (the + composed table keeps each entry's plugin name), and continues with the other entries. Durable already does this for + extensions (`extension_error`; a throwing tool becomes an error result); `compose` reports through `report`. +- **Registries may carry natives.** An entry lives exactly as long as its contributor; consumers look up at each use + or follow the composed value and keep nothing from an entry beyond a use. +- **Contributions that change (owner, 2026-10-19; W4):** registries are host-local, so contributions are usually + in-process values (functions, runtimes, handles). A contribution is fixed for its contributor's life (dynamic parts + behind handles read at use; a change means a restart or reload), a `Watchable`, or a `NativeWatchable`; Chord follows + the latter two through the internal flush: on every change it replaces that entry (a new immutable table) and runs + `compose` again, in the owner's flush, never inside another owner's commit step. The composed value is itself a + `NativeWatchable`; use sites watch it or read its current value. For in-process values that change, + `self.writable(initial)` is a `NativeWatchable` with `set` (identity compared, never ops, never a state member): + + ```ts + const ext = self.writable(mcpExtension([])); // e.g. MCP: tools known only at runtime + self.contribute(HarnessExtensions, ext); + self.on(mcp.tools, ({ value: tools }) => ext.set(mcpExtension(tools))); + ``` + + Tools, sections, hooks, wraps and tasks reach the harness as one Durable `Extension` per plugin contributed to + p3's core registry; `compose` builds Durable's registry snapshot (Durable applies selection and wraps, and pins a + snapshot per task phase). Skills and other lists work the same way. +- **Removing a contribution:** there is no remove call (`contribute` is declarative, in the factory). An entry leaves + the table when its contributor ends, or after a reload whose new code no longer contributes. A live contributor that + has nothing to offer for now sets its readable to the registry's empty value (an extension without tools, empty + routes); the table keeps exactly one entry per live contributor. +- Registries add behaviour (tools, routes, renderers, commands, input gates); they never replace an owner's checks. + No lifecycle in entries: anything needing cleanup is a native of the contributing object. + +--- + +## 18. Start data and platform bindings + +There are no host values. Start data (session id, init, data directory) is a `local` object with empty params (host lifetime) +and constant state (`constant(value)`: a readable that never changes, R23), served by the base code's core facet from the start message: + +```ts +export const SessionInfo = local("app.session-info", { version: 1, params: {}, members: { init: InitSchema } }); + +// base code (apps//src): the core facet, passed to load() like any facet +export const coreWorkerFacet = (init: Init) => defineFacet((host) => { // base code's own bound defineFacet + host.serve(SessionInfo, () => ({ init: constant(init) })); // local, empty params: host lifetime +}); + +// any plugin +host.serve(NoteInbox, (_, self) => { + const info = self.get(SessionInfo, {}); + const watcher = self.open(async (context) => + watchDir(join(info.init.value.dataDir, info.init.value.id, "inbox"), { signal: context.abortSignal }), (w) => w.close()); + return {}; +}); // NoteInbox: local, empty params → host lifetime +``` + +Plugins read start data through a reference, after activation (in openers, derives, routines and methods); nothing +reads it synchronously in a factory. Platform bindings the host owns (a Durable Object's `state` and `env`) are core +objects whose `self.open` returns the binding and whose `close` does nothing. A long-lived native from the host is a +host-lifetime object in a core facet (the Durable harness, section 24). + +**Core vs. plugin is decided by where the code lives:** base code (`src/`, imported statically by the start code under +a fixed name such as `@app/core`, not switchable, a change restarts the host) or a plugin directory (found by listing, +loaded through loader D, named by its `package.json`, switchable, a change reloads it). Core is what the app always +needs and everything builds on (harness, catalog, access, session location); optional features are plugins (code +mode is a plugin). Both use exactly the same primitives. + +--- + +## 19. Presence + +```ts +const viewers = self.derive([host.references(Notes)], ([refs]) => refs.length); +``` + +`host.references(Type)` (slice 4b) is a JSON `Watchable` of the references to objects of a type on this host: +`{ params, principal?, incarnation }[]` (named `origin` until 2026-10-27). It is one rule (owner, 2026-10-24): the incarnation of the host that made +the reference as its caller sees it, so two tabs are two origins, and a guest's `ctx.get` shows the guest's tab. An +entry is added when a reference binds and removed when it ends, in that batch; a departure discovered while a batch +is sending lands in the next batch (so one commit stays one message per link). The host keeps a type's list from its +first `host.references(Type)` on (one small object per asked type; entries come and go with references). **A link drop is a departure** +(accurate and simple); smoothing (a phone switching networks) is the UI's job, e.g. a debounced viewer count. Counts +per person or origin are derived by the app from the principals and origins. There is no consumer-side linger +(a closed tab closes its link); provider-side `linger` keeps a just-closed object warm and keeps the worker alive +(section 3). Replaces the presence, viewer and login-abandon leases. "Leaving" is never derived from an unmount +(section 21). + +--- + +## 20. Input into running work + +### Flows kept by their owner + +```ts +start({ provider }, ctx) → { id } // uuidv7 +LoginFlow {id}: state | { kind: "running"; step?: { id, … }; notices } | { kind: "done" | "failed" | "cancelled"; notices } +answer({ id, step, value }, ctx) → ok | "step no longer active" +cancel({ id }, ctx) +``` + +The UI watches the state; a late or second answer to a step that is gone is refused; a promise-based callback (pi-ai's +`prompt`) stays inside the owner (its job keeps the resolver). Imperative consumers (tests, scripts, a CLI, an agent +tool) use a generic client helper over the same state, not a second protocol: + +```ts +async function* steps(flow, ctx) { + let last = 0; + for (;;) { + const s = await until(flow.state, (v) => v.kind !== "running" || (v.step && v.step.id > last), ctx); + if (s.kind !== "running") return s.kind; + last = s.step.id; + yield s.step; + } +} +for await (const step of steps(await ctx.get(LoginFlow, { id }), ctx)) + await login.answer({ id, step: step.id, value: answerFor(step) }, ctx); +``` + +A long-held server `step()` call is not used: a link drop fails it and the consumer cannot tell whether it missed a +step; state re-holds by itself and `last` keeps steps from being lost or repeated. + +### The Questions service + +Another pattern (`apps/p2/docs/questions.md`): a shared store with per-principal views (asker: `ask`; viewer: `open`, +`answer`), typed questions (`text`, `select`, `confirm`), policies over a live audience (`first`, `quorum`, `all`), +reattachment by `key`, deadlines, authorization per question, secrets redacted. Fits tool approval and a question the +agent asks everyone present. Its store must live in an object (it predates "natives only in objects"). + +> **Decision needed: where a flow lives.** Options: (a) a keyed document of its owner; (b) an object `LoginFlow {id}` +> living until done. (Jobs as objects were rejected, §15; a flow kept as its owner's document plus a job is the likely +> answer.) + +--- + +## 21. Reload and rollout + +### Per host + +``` +1. load new bundles; an import or syntax error rejects +2. staging facet setups and every live object's new factory run with a recording `self` (declarations only, + nothing activated); any throw rejects +3. commit the host switches to the new code atomically +4. after per object (below) +``` + +- **Rollback is all-or-nothing per host:** any load or staging error (code errors: load errors, throwing factories) + leaves the whole host on its previous code; a host is completely at revision R or R−1, so status per host suffices. + Runtime states (registry conflicts, failed opens, migrations) are not rollback reasons. +- **Pure objects** (no `self.open`, old or new) are swapped in place: new methods and derives; documents, references + and jobs stay; routines restart (serialized); new references become live; then the gate opens. Running calls finish + on old code, and keep their old handles valid until they settle (their transactions keep working), except for + documents whose version rose (R15, as in Durable): old code touching a document migrated past its version throws, + the document stays intact, and the call fails once (not resent). Only new calls get the new handles. +- **Native objects of the reloaded plugins restart:** new calls park, running calls are drained first (bounded by + `closeTimeout`; owner, 2026-10-26: a call outliving the drain is cut off by the seal fence), then natives close (reason `reload`), + the factory runs, natives reopen; documents stay. In practice this costs little: the Durable harness is core (base + code), terminal PTYs live in the separate terminal host process (the server's native is only the connection), model + runtimes, sandboxes and the search index rebuild in milliseconds; only reloading the plugin that owns SSH connections + (running remote commands are killed) or the relay socket (relayed connections drop) hurts. Natives are not kept + across reloads by name: natives have no line, no transactions and no schema, so old and new code would interleave + unchecked. +- **Derives on reload:** derives have no names; exposed derives are matched by their member name, internal ones are + rebuilt (matching only if a real need appears). +- **Holds across reloads:** a hold's indexes belong to its bound version. A reload that still accepts that version + keeps the hold: a pure object re-subscribes it through the version's mappings and sends `[["r", value]]` for each + member whose value changed; a restarted object sends a fresh `bound` for the same hold (a `bound` on a bound hold + replaces its replicas). A served hold whose object leaves `live` while its `bound` waits for a slot goes back to + pending (slice 7 builds it: nothing in 4a leaves `live`). A reload that drops the version ends its holds with `gone` (`contract_mismatch`) before + the new code serves anything. A consumer whose own code moves to a new version re-holds (a new hold id) and releases + the old. +- **Old-version calls:** `checkArgs`'s `{ kind: "call" }` result runs the provider's replacement; `{ kind: "method"; + result }` maps the result back for an old-version caller (section 5). +- **Retries:** a reload retries a broken object as `ref.retry()` does (section 6). +- **After the commit,** errors stay per object: a failed open or reference makes that object `failed` (backoff); a + migration that throws on a lazily loaded document refuses that access; a method bug fails that call. +- **Removing a type:** its objects end (routines stop, natives close, references end with an error); in-memory + documents go with them; persisted ones stay until retired. +- **Consumers elsewhere** see a restarting object like any object that briefly goes away. +- No factory trial, no migrate dry run, no consumer contract check during staging: incompatible consumers fail at bind + with `contract_mismatch` and a notice naming plugin and type. +- **Start barrier** (host start and first load): load facets (the core facet with its start data among them), wait + until every facet and host-lifetime object is live or failed (an object that must + contribute at start, like the model runtime, is a `local` object with empty params). Then the registries hold the contributions of every + plugin that loaded, and dependent work + resumes: base code calls `harness.resume()` after `load` (the harness is an ordinary host-lifetime object that + opened Durable paused during the barrier; owner, 2026-10-21, refines R17). Nothing + waits for specific contributions: what is missing shows where it is used (section 24). Failed plugins are reported + with their error. Plugin reloads need no barrier. + +### Rollout across hosts + +- A **revision** is (server incarnation, sequence) and names the complete desired code set (every facet with its + digest), not a batch. Every install reconciles everything missing; a host reports a revision only when it runs that + complete set. +- **Order:** server, workers, clients. Each host reports `installed`, `installed_failed` (named failed objects) or + `rejected` (the reason; the host stays on its previous code). A server that rejects stops the rollout. +- A host that starts or reconnects installs the current revision before anything is routed to it. "Installed" means + the code is published; objects activate on demand. +- **Mismatches:** a mismatch response carries the provider's revision and status. Provider ahead, or behind and + installing → `updating` (values readable, calls park, with a deadline); provider rejected → fail at once (retry on + the next revision); same revision → `contract_mismatch`, final. Calls parked on a binding that is released end with + an error and are not resent. No cross-host transaction, no votes. + +- **A host checks its own install status before waiting on a provider:** a rejected own install fails at once with + the reason. **Installs are serialized per host.** **A revision also carries the code-set digest** (the hash over the + facet bundles' content hashes, which loader D computes anyway; no module analysis); hosts compare the digest on + reconnect, so a server restart with unchanged code triggers no reinstall. + +### Loading and the dev loop + +- Plugins come from built-in, global (`/plugins`), package (npm/git, installed with `--ignore-scripts`) and + project roots, found by directory listing (no source parsing); the server selects; dependencies resolve by contract + name at runtime; names are prefixed (`.`; tools `_`). +- **Loader D:** one base bundle per surface (host packages and the app runtime) plus one script per facet (host + packages passed in, no module map); old generations are reclaimed by the GC; stable filename per (facet, digest) and + V8's compilation cache off on hosts that reload in place. The loader is an adapter: static imports (tests, Durable + Objects) work the same. +- Project plugins, trust, snapshots and scopes: `apps/p2/docs/plugin-loading.md` (still the reference for those). +- **Dev loop:** a source watcher marks plugins dirty (never applies changes); a reload control (playground web shell, + CLI, agent tool) reloads the changed plugins or restarts all hosts for base code. + +- **UI reload vs. leaving:** "leaving" is never derived from an unmount, only from explicit navigation or presence (a + dropped reference); a component unmounted by a reload has not left. + +> **Decision needed: project plugins.** Per-directory plugins with trust and snapshots (`plugin-loading.md`) need their +> own pass over this design. + +--- + +## 22. Wire + +Exposed documents, derives and readables are state members: a snapshot in `bound` when a hold binds, ops per commit +(a draft's recorded ops, no diff), ops from diffs for derives, ops of job state documents (slice 5), and failed +members in the entry of the message that carries them. **Commits are atomic for consumers:** locally, subscribers are +notified once after every document of the commit is adopted; remotely, one `state` message per commit per link and +origin carries an entry per touched hold, applied in one batch before listeners run (also when one Durable commit +touches several holds), so remote consumers never see a torn commit. Consumers apply ops with `applyImmutable`: no +validation, no clone of snapshots or ops. + +- **The protocol version** travels in `hello`; hosts of different versions refuse each other (a web page reloads for + a newer server). Error reasons and message kinds are part of it. Until p3 ships, changes keep `PROTOCOL_VERSION` 1 + (nothing is deployed); after that every message or reason change bumps it. +- **One typed `Message`** (owner, 2026-10-22, "Wire protocol"): a TypeBox schema `MESSAGE`, a tagged union over + `kind`, from which the protocol types derive; every decoded message is checked against it (compiled once) unless + the link is trusted (section 3). Kinds: + + | Kind | Direction | Fields | + | --- | --- | --- | + | `hello` | both, first | `protocol`, `incarnation`, `calls` (the call window N) | + | `hold` | to provider | `origin?`, `hold`, `type`, `version`, `fingerprint`, `params` (part), `address?`, `principal?`, `incarnation?` | + | `release`, `retry` | to provider | `origin?`, `hold` | + | `call` | to provider | `origin?`, `call`, `hold`, `method` (index), `args?` (part; absent: no arguments), `segments?` | + | `abort` | to provider | `origin?`, `call` (named `cancel` until 2026-10-27) | + | `ack` | to provider | `origin?`; one per applied `bound`/`state` message | + | `closed` | relay to provider | `origin`: the downstream origin ended | + | `bound` | to consumer | `origin?`, `holds: [{ hold, variant, members: [{ member, value }], failed }]`; one batch | + | `state` | to consumer | `origin?`, `holds: [{ hold, members: [{ member, ops }], failed, retired }]`; one batch (`retired`: members retired since the last message, slice 6a) | + | `failed` | to consumer | `origin?`, `hold`, `cause { kind, error }`, `retryIn?` (absent: broken) | + | `loading` | to consumer | `origin?`, `hold`: a new attempt after a failure, or a relay's upstream dropped | + | `result` | to consumer | `origin?`, `call`, `value?` (part; absent: `null`), `segments?` | + | `error` | to consumer | `origin?`, `call`, `error` (a ChordError's JSON or a plugin's code, message, details) | + | `gone` | to consumer | `origin?`, `hold`, `error`: the held object ended (revocation 4b, reload 7; relays never make one) | + | `ping`, `pong` | both | the heartbeat | + + A `bound` on a hold that is bound already replaces its replicas in one batch, another variant allowed (re-holds, + slice 7's restarted objects). +- **Codecs are pluggable per link** (both ends use the same): `binaryCodec` (default: a kind byte with a flag for a + following origin varint, varints, raw incarnations and fingerprints, length-prefixed strings and parts; about 23 B + for a streamed token) and `textCodec` (Delta-style JSON tuples without field names, a message with an origin wrapped + as `["@", origin, tuple]`; for debugging; `describeMessage` prints any message so). Measured: a 2,000-token reply + about 46 KiB instead of 196 KiB. No CBOR (revisit only if benchmarks show encoding as a bottleneck). +- **Parts are opaque** (slice 4a): `params`, `args`, `value`, snapshot values and ops are a `Part` (JSON text at the + origin, its UTF-8 bytes once a binary codec received it). A part is parsed only where it is owned, right when its + message is handled (the provider parses params and arguments, the consumer results, snapshots and ops), so a relay + forwards it unparsed and a malformed part is still a protocol error. A message naming an unknown id is ignored + before its parts are parsed. The text codec re-serializes parts at decode (one parse per part per hop, accepted for + a debugging codec). +- **Names travel as indexes** into the hold's bound version (each variant's state members and methods sorted; variants + sorted). A reload that drops that version ends its holds first (slice 7). +- **Checks by layer:** the codec's structural decode, the `MESSAGE` envelope (skipped on trusted links), the bound + version (ids, indexes, order), then the owners (`checkParams`, `checkArgs`, Delta's decoder). +- **Unknown ids are ignored on both sides (Q1):** they are normal races (a `result` crossing an `abort`, a `call` + crossing `gone`, a state part crossing a `release`); the side that ended a hold or call settles its own pending work + (a consumer receiving `gone(h)` rejects its pending calls on `h`; per-direction order guarantees nothing else for `h` + follows). Protocol errors close the link and are reported (`protocol_error`): a message failing the schema or the + codec, a malformed part, a reused live id, more than N running calls of one origin, a broken handshake (a first + message that is no `hello` of the link's codec, a second `hello`, another protocol version). +- **Byte segments:** a payload (arguments, a result) whose method's schema has a `bytes()` leaf carries each + `Uint8Array` as a raw segment referenced by `{"$bytes": i}` and read back as a view of the received message; `$bytes` + is the one reserved key there (`invalid/args`). Never base64. State is JSON only. +- **Encoders:** each member of a hold has its own Delta encoder and decoder (path interning end to end between + provider and consumer), restarted with every snapshot. Two holds of the same principal (two tabs) encode the same + ops separately (O(change) each). A part is encoded once at its origin; relays never re-encode it. +- **Sending:** one `state` message per commit per link and origin while the window has a slot; otherwise the ops wait + in the members' pending queues and leave as one message when an `ack` frees a slot (section 3). +- **Relays** read and rewrite only envelopes (section 3, "Relays"). +- **Size limits per part (R2, Q2; slice 4b):** limits apply to each part (wire and decoded size), configurable per + link, checked from the part's length before it is parsed or inflated; the decoded limit is enforced in the one + decode function. An oversized part fails only its own hold or call (`too_large/part`); the other parts of the message + apply; a snapshot over the limit fails its hold with no automatic re-hold. The decoded limit travels with relayed + work: the relay checks the admitting link's wire limit itself and stamps its decoded limit on the upstream `hold` + (honoured on the parent link only, never taken from a client; calls inherit their hold's), and the host that + inflates enforces the smaller of its own and the stamped one while inflating, after the part's owner and id are + known (parts for unknown ids are never inflated, Q1). A relay checks only what arrives from downstream; answers + from upstream (the app's own workers) it forwards unchecked and unparsed, and the end that parses them checks them + against its own link's limits (owner, 2026-10-24: each end checks what it parses; a relay check needed a re-hold + with the same id, which old messages in flight could mistake for the new hold). So IPC carries no limit of its own. + A reference failed `too_large/part` is never held again automatically: it leaves its path, so a reconnect does not + hold it; only `ref.retry()` puts it back (a new hold). This lives only in the consumer's memory, as long as the + reference. The + WebSocket's `maxPayload` stays only the transport's hard cap (e.g. 64 MiB; code 1009 closes the connection). + Defaults for client WebSockets: 16 MiB on the wire and 64 MiB decoded per part; none on IPC. +- **Compression at the origin** (owner, 2026-10-19; slice 4b): a JSON part above a threshold (e.g. 16 KiB) is + compressed once by a host with a synchronous compressor when it encodes it (`createHost({ compression })`, Node's + zlib adapter in a Node-only subpath), whatever the link (IPC included), only if it shrinks; hold params are never + compressed; byte segments stay raw; the codec is in the part's envelope (the text codec tags every part, so no JSON + value is reserved); relays forward it compressed; the end that owns it inflates it. Option (b) (owner, 2026-10-24): + hosts without a compressor (browsers) send uncompressed and inflate asynchronously (`DecompressionStream`), holding + later messages of that link in order (bounded by the window and N; acks go out after apply). **Child links refuse + compressed parts** (protocol error): honest clients never send them, and a 65 KB part could inflate to 64 MiB on the + server; so asynchronous inflate happens only on a host receiving from its parent or a route. Because a host cannot + tell whether its parent sees it as a child (a tab) or over a route (a worker), call arguments sent up a parent link + are never compressed; results and state going up are. Compressed hold params are a protocol error on every link. + The asynchronous inflate runs one part at a time; the queue behind it is bounded by the sending peer's window and N, + so only a misbehaving parent or route peer could grow it (accepted: those are the app's own hosts). The adapter reports "too large" + and "corrupt" as a tagged result. Limits and the threshold count bytes. No per-link compression. +- **Trace carriers (R5; slice 9)** are an optional bounded field of `call` and `hold` (key count, string lengths); + relays read them from the envelope. +- **Adapted bindings get diffs** (slice 7): a consumer bound through `accepts` is served like a derive of the + provider's state through the adapter: Chord keeps the last adapted value per binding, applies the adapter to the new + value and sends the diff (`diffRevisions`), so an old tab during a rollout never receives a large state whole on + every commit. + +--- + +## 23. Optional components + +Deferred: the HTTP and channel components and middleware phases are built after the kernel fixes and a first real +proof (harness, one tool with an approval pause, a code mode reload); the playground's web shell serves its pages +with plain app code until then. The design below stays the target. + +### HTTP (`@earendil-works/chord/next/http`) + +A route is `(req, ctx) => Promise`. `req` is a plain request summary (method, path params, query, +principal, provenance such as local or relayed, selected headers; parsed and capped JSON bodies; uploads handed over in +chunks); `ctx.principal` is the request's principal (from the request resolvers), so `ctx.get(Notes, …)` gives that +principal's view. A route returns a description: + +```ts +| { kind: "json"; value } | { kind: "page"; title; body } +| { kind: "bytes"; name; mime; size; read(offset, length) } +| { kind: "redirect"; to } | { kind: "error"; status } +``` + +The component does ranges (206/416), HEAD, download headers (safe filename, `nosniff`, CSP, caching), streaming, +abort on disconnect and the CSP nonce once, instead of every plugin. Contributions hold `routes` (composed with +`keyed`; a duplicate pattern is a reported conflict, the first by plugin name wins) and `wraps`: + +```ts +type Wrap = { + name: string; + phase: "observe" | "headers" | "transform"; // owner-defined phases, outermost first + scope?: "own" | "all"; // default "own": only this contributor's routes + route?: string; // only this route; a missing route is a reported conflict + wrap: (req, next, ctx) => Promise; +}; +``` + +Within a phase, wraps run in contributor order (no numeric priorities: plugins have no single registration order, as +Express or Hono apps do; Fastify's lifecycle phases and encapsulation are the model). Wraps may add a security boundary +on top of an owner's checks, never instead of them. A request uses the router as it was when it started. Request +resolvers (who sent a request: cookie, loopback, relay) are hooks over the summary. + +### Channels + +WebSocket byte channels (`send`, `onData`, `close`) for terminals, separate from object links, with their own flow +control (acknowledged windows). + +### Chunked transfers + +Large byte transfers are chunked by the owning object: `begin`, `append({ upload, offset, bytes })`, `finish`, +`read({ id, offset, length })`, about 1 MiB per chunk. The awaited `append` is the backpressure; after a drop the +writer asks for the stored offset and resumes; unfinished uploads expire (the owner's routine). A helper wraps the +loop. Chord has no stream type (it would need its own rules for drops, cancellation, relays and restarts). + +--- + +## 24. Durable integration and portability + +### Durable + +- Durable keeps importing today's Chord's `context`, `delta`, `json`, `output` and `sqlite`; `next/` imports the same + modules, so `Context`, `Draft` and `JsonValue` are one type. Durable's conversation view, task graph and documents + are `Watchable`s through Durable's watches (Durable's `WatchHandle` now lives in Chord); Durable adopts and dispatches + them synchronously inside the storage's commit (S1, W1; with `StoreStorage` that is the store's commit step), so a derive over a Durable view travels with it. This needs Durable to publish a commit when it adopts it (owner, 2026-10-26; a pi change, upstream first; explainer `durable-observers/`): inside `adopt`, its trackers, telemetry events and internal observers (view mounts, task graph, scheduler, submissions) run, and the host `subscribeCommits` listeners are dispatched in one microtask queued before the observers, so they run before every wait the commit resolves and every abort it queues, but after the storage's synchronous step, never inside Chord's batch; host listeners and watch listeners have no order between them (owner, 2026-10-26). Durable's public primitive for adapters is a synchronous observer, `observeConversation`, `observeTaskGraph`, `observeDoc` (its `advance({ value, ops, retired }, ctx)` runs inside `adopt`: record only, no awaits, no Session calls, no throws); `watchXXX` is built on it for app code. Chord's `durableView` (slice 6b) is a source over it. Otherwise a derive over two Durable views computes between their frames of one commit. With `StoreStorage`, Durable's `adopt` runs inside the host's batch; the store knows nothing of Durable (no special cases) and needs only its step and flush to stay synchronous. Durable's + state getters (`documentState`, `viewState`, `taskGraph()`) and the `replicatedState` family are removed (W14). Only + plain data crosses between Durable and v2 objects. +- **The harness is an object in the core facet** (base code): a `Harness` `local` object with empty params, so host lifetime (it runs for the worker's whole life, + whether or not anyone holds it) opens Durable paused with `self.open` during the barrier and gets model + access and execution environments from registry entries, looked up at each use: + + ```ts + const harness = self.open(async () => Durable.open(storage, { + models: () => host.registry(HarnessModels).value, + env: (target) => host.registry(HarnessEnvironments).value(target.cwd), + settings: () => currentSettings(), + registry: durableRegistry(host.registry(HarnessExtensions)), // a RegistryReader over the composed NativeWatchable + }), (h) => h.close()); + ``` + + Durable's `models` and `settings` accept functions (pi `7ec1f648e`). Durable opens with scheduling paused; base code + calls `harness.resume()` after `await host.load(...)`; the worker connects to its parent first (W6), so host-lifetime + stores can reference server types during the barrier; requests arriving during `load` park like any call to a + loading object (refines R17). A missing tool gives its call an error result the model sees; a missing model makes the generation + report "model unavailable" and wait (not fail) until a model contribution appears; a task whose definition is missing + stays blocked (`missing_task`) until the plugin is installed or the task is aborted, as Durable does. +- `HarnessModels` and `HarnessEnvironments` start with one contribution each, from the models and envs plugins' + objects that own the natives; several providers or environment kinds are their `compose`. After a models reload + the next request uses the new runtime; a running stream finishes on the old one. +- **Chord from Durable code (Q6):** tools, hooks and sections get Durable's `ToolExecutionApi` and a `Context`, never a + Chord `ctx`. p3's core opens the harness with a base context holding the host under a typed key + (`withContextValue(ChordHostKey, host, …)`); every phase context inherits it. A core helper opens a call scope from + any Durable context: `chord(async (ctx) => …, context)` = `host.root(fn, { signal: context.abortSignal, telemetry: + context.telemetry })` (host principal; attribution as data; references released when `fn` settles). No wrapping of + tools and no p3 `defineTool`. Clients are never reachable this way (they are not addressable): a tool that needs the + user goes through an owner's state, e.g. the Questions service (`apps/p2/docs/questions.md`): the model-facing + `ask_user` tool and any other tool call `Questions.ask`/`wait`, tabs render the open questions and `answer` them. + Documented with this example. +- **Code mode** (a plugin) keeps owning its `pi-codemode` sandbox; its tool entry looks up the current sandbox when + `execute()` starts, so a call pinned by a phase before a reload (e.g. waiting in an approval hook) starts on the new + sandbox; on `reload` and `close` its `close` detaches the old sandbox, returns at once and disposes of it when its + scripts finish or after `closeTimeout` (30 s at most, R16, Q7); after a worker stop Durable replays as today. +- **Telemetry:** `HarnessOptions.telemetry`; submissions and tasks store an optional carrier (section 29). + +### Portability + +Adopted in the kernel: host = identity plus incarnation; the loader is an adapter; values independent of the +transport; durable state only in persisted documents. A Durable Object deployment +(`apps/p2/docs/research/chord-on-durable-objects.md`) would map a logical host to a DO, compose plugins at build time +(Workers forbid `eval`), replace the relaying tree with edge routing and use alarms for routines. + +> **Decision needed: a Durable Object profile.** Problem: a DO can lose its memory while clients stay connected +> (hibernation); routines need wake-ups (one alarm per DO); links between DOs are RPC stubs. Hot reload is not needed +> there; plugin composition is. For later. + +--- + +## 25. Apps: ownership and porting + +- **Every thing has one owner, the smallest object it belongs to;** ownership nests; rules set at creation are written + in the creating commit (Durable `createConversation` with `init`). +- **Core owners are always present;** a plugin owner may be missing (optional dependencies are registries owned by the + dependent, section 7). Avoid a core god object: each core owner owns only what belongs to it. +- **Revocation of work in shared stores:** a store that runs work on behalf of a principal stores the grant (or device) + as data with the work and reconciles from the grant owner's live grants (a routine aborts jobs, withdraws questions, + drops queued input whose grant no longer exists); no revocation hook. The window between a revocation and the + routine's run is accepted (R10): a step started in it still runs; there is always some window. +- **Reading without starting a worker** is plain pub/sub in an ordinary plugin: a host-lifetime object of its worker facet + records the summary locally and publishes it to the plugin's server object (a method call from a routine on the source); clients read + the server object, which never starts a worker; it keeps the latest value per session (a document if it must + survive a server restart) and ignores reports from an older worker incarnation. **The caller's incarnation is + stamped by Chord (R20):** a call arriving over a link carries the calling host's incarnation from the link handshake + (`ctx.incarnation`, next to `ctx.principal`, not part of the principal or its key, so object identities do not + change with restarts); nobody sends it as data and it cannot be forged. On relayed calls it is the **origin** host's + incarnation (Q13, owner, 2026-10-21): the relay takes it from the link the hold came in on and sends it in + the upstream `hold` (a client's value is ignored, like its principal); calls inherit their hold's (section 4). Session list, status, children, unread and + artifact lists follow this. +- **Porting guideline:** what core enforces on data lives in a format core owns, written by the plugin that creates + the data; a plugin's private document is read only by that plugin. Example: subagents mark one-shot and + cwd-following child conversations in a core per-conversation policy document (`readOnly`, `followsSessionCwd`); + chat and relocation read it; `ReadOnlyConversations` and `SessionCwdFollowers` go away. +- **p3 is built next to p2,** fully isolated (`apps/p3`, own data directory), without migrating p2 data; plugins are + derived from the functionality map (`apps/p2/docs/p3-functions/`), not ported one to one. +- **Session lifecycle:** archive retires nothing (stops the worker, ends guest access, closes forwards); a session + delete removes the whole session (tagged documents, the worker's databases, files, the catalog entry). + +--- + +## 26. Decisions needed + +Open, decided with the slice named or a concrete case (numbers kept stable; everything else is decided and folded into +this document): + +| # | Topic | Section | When | +| --- | --- | --- | --- | +| 11 | where a flow lives | 20 | when login is ported | +| 14 | project plugins | 21 | own pass over `plugin-loading.md` | +| 16 | a Durable Object profile | 24 | when a DO target becomes concrete | +| 24 | the rollout install timeout (60 s) and a per-type drain override: host options or fixed (the routine handover uses `closeTimeout`; a failing job `finish` backs off 1-60 s and blocks after five failures, decided) | 3 | with slice 7 | +| 25 | `host.root` signature (`root(fn, context, { principal? })` proposed) and what an aborted, uncooperative `fn` means for its references (match the handler and `CallContext` rules) | 3, 24 | slice 8 | +| 26 | how a migration commit reaches consumers (`[["r", value]]` per changed member, or a fresh `bound`; adapted by `accepts` for older consumers) | 21, 22 | slice 7 | +| 28 | Astra's circuit breaker (a hard transport-buffer cap plus a stall deadline); the heartbeat is built, the cap was never decided | 3 | if a real case needs it | +| 29 | round-3 cut C11 (`ctx.trace`); C3 and C4 are decided (§14, §15) | 8 | slice 9 | + +Apps (p3), from the functionality map, decided per domain when p3 is designed: + +| # | Topic | Problem and options | +| --- | --- | --- | +| 17 | session files | one owner `SessionFiles {session}` on the server for uploads, sent-file snapshots and artifact versions (caps per kind, index readable without a worker, copy on clone/fork, delete with the session); proposed, sub-points: one preview limit, fork's artifact list, copying attachments into remote environments | +| 18 | session delete | per-session objects plus `SessionTag` plus a session directory, or owner hooks through a registry; search index per session or global plus a hook | +| 20 | connection pools | SSH connections to one environment exist four times: one owner with a pool, or per-use connections | +| 21 | config files with a single writer | settings, models and auth are watched by every process and written by two: one owner on the server, others follow its state | +| 22 | file formats as versioned interfaces | `envs.json`, downloads, uploads have no version: version them like documents, or keep each internal to one owner | +| 23 | the terminal host process | a detached process with its own code: part of the revision (reloaded or restarted with it) or deliberately outside | + +--- + +## 27. Versions and adapters + +| Surface | Version | On change | +| --- | --- | --- | +| object type | `version` plus contract fingerprint | every change bumps; forward evolution and `checkAccepts` (Q10); otherwise `contract_mismatch` | +| registry | `version` | `accepts` adapter on contributions | +| document | `version` plus schema fingerprint | pure `migrate`, on access; a schema change without a bump fails the object `broken` | +| job | `version` | `accepts` maps older args and results forwards (`finish`, `running`); the state document's own `migrate`; `blocked` without an adapter or for a newer version | +| native | none | the object restarts on reload | +| Durable tasks and documents | `version` | Durable's `migrate` | +| link protocol | protocol version | refused; hosts update together | +| UI components, Chord and app APIs | none yet | exposed as they are; versioned with an SDK later | + +**Contract evolution in code (Q10; implemented in `packages/chord/src/next/contracts/`, see its README):** + +```ts +export const NotesV1 = remote("notes", { version: 1, params, variants: { + editor: { state: { notes: arr(NoteV1) }, methods: { add: method(NoteV1, Id) } }, +} }); +export const NotesV2 = evolve(NotesV1, (e) => e.state("editor", "notes", arr(NoteV2)).args("add", NoteV2)); + +const accepted = checkAccepts(NotesV2, { + 1: { state: { editor: { notes: (current) => current.map(toV1) } }, methods: { add: { args: (old) => toV2(old) } } }, +}); +``` + +- A history starts with a base written in full with an explicit version (`remote()` / `local()`; `local` has flat + `members` and no variants). `evolve(prev, (e) => …)` always continues it with structured edits: `state`, + `addState`, `removeState`, `args`, `result`, `method(name, args, result)`, `addMethod`, `removeMethod` (local: no + variant argument). Every version is materialized and fingerprinted at declaration, so an old plugin's own copy of + v1 computes the same fingerprint; no build step, no generated files, no hand-copied fingerprints. +- **A new history base** (a params change or a change of the variant set) is a new `remote()`/`local()` with an + explicit version and no accepts for older versions: old plugins get `contract_mismatch` and must update. +- **Mappings live with the provider,** never in the contract: the pure `checkAccepts(contract, accepts)` (called by + the objects layer) requires, at type level and again at runtime, a mapping for every member that exists in the old + version and changed or was removed since: changed state `(currentMember) => oldMember`, removed state `(current + variant state) => oldMember` or `"refuse"`, `args: (old) => current`, `result: (current) => old`, removed methods + `{ call }` or `"refuse"`; added members need nothing. No-op edits need identity mappings. Untyped JavaScript + providers fail at declaration naming the member; unknown keys and wrong shapes are rejected. +- **The fingerprint** is a hash (`hash64`, portable, not a security boundary) of a schema-aware, data-only normal form + (`NORMAL_FORM_VERSION`, tied to the protocol): annotations dropped only on schema nodes, `required`/`enum` sorted, + union and intersection order kept (it is meaning: defaults), a whitelist of TypeBox keywords, executable features + rejected; bytes are a `{ type: "bytes" }` data marker. Schemas are deep-copied into a private history (never frozen; immutable by convention). +- **Schema helpers:** `str num int bool nul lit oneOf union obj arr map opt bytes unknown` and `type Static`. +- Chord wraps state mappings as a derive and diffs per binding (section 22); members an old consumer does not know do + not appear. +--- + +## 28. Building: slices + +Chord v2 is built in vertical slices; the plan, the feature map (every feature of this document assigned to the slice +that builds it) and the carry-over of deferred obligations are in `slices.md`. Slice specs (`slice-N.md`) stay as the +history of how each slice was specified; this document states the current rules. The first layered build (kernel steps +1-6) was deleted on 2026-10-22 and survives only in git history. Done in pi: `pi-telemetry` (propagation, links, +metrics, the store interface, the renderer), `chord/context` (`Context.telemetry`), pi-ai and Durable telemetry, +`chord/telemetry-sqlite` (section 29), Chord's shared `Watch`/`NativeWatch`/`PendingOps`, Delta's `appendText`. + +**Rules for every slice** (owner, 2026-10-22): the code follows the root `AGENTS.md` section "Chord v2 Code Style"; +one test file per source file, one `describe` per mechanism, the spec's test list case by case; cost guards through the +in-memory telemetry adapter (section 30); GC tests for every cache and binding; one or more runnable examples +(`packages/chord/test/next/examples/NN-topic.ts`); hot-path slices add a light benchmark script +(`test/next/bench/`, by hand) with results in `benchmarks.md`; `npm run check` and the slice's tests before reporting. +Caches are `WeakMap`s keyed by their owner; no module-level strong caches; no hot-path work no decision asked for. The +function doing counted work counts it through its context; call sites never count for others; plain totals without +site labels. + +**Testing the core:** every slice spec contains an explicit test list, one group per mechanism with its edge cases, +tested individually and exhaustively: activation and dependency recovery (R18: broken dependency, trigger, backoff as +safety net, a dependency failing while live, cycles), revocation over running jobs (R11), compatibility (R6), params +and arguments for every caller (R7), migration results and old calls on migrated documents (R8, R15), flow control +(the window per origin, folding, relays forwarding and returning acks), the incarnation fence (R20), the harness start +after the barrier (R17), native restart patterns (R16), and W1's publication list (a throwing derive, a chain A→B→C +where B throws, two owners alternating with a derive over both, reentrancy, a watch started after 150 frames, stop +during a running call, a retire with a derive over it, a reload during a routine call, GC counts of retained +revisions). + +--- + +## 29. Telemetry + +Chord reports traces, events and metrics through `@earendil-works/pi-telemetry`; one adapter per host +(`createHost({ telemetry })`, NOOP by default) decides sampling, buffering, export and filtering. + +- **Types:** the adapter is a `TelemetryContext` (`startSpan` plus optional readonly `metrics` and `propagation` + capabilities); only hosts hold it, to start roots. `Context.telemetry` is always a `TelemetrySpan` (the NOOP span by + default); every span carries its adapter's `metrics` and `propagation`, so code with a context can record metrics + and inject carriers. `withTelemetry(span, context)`, `traceIn(options, fn, context)` (context last; pi change with S1); `value()` and keys stay. + `Ctx` extends `Context`, so `ctx.telemetry` is the current span; app values such as the principal stay out of + Durable's context. `ctx.trace(name, attributes, fn)` creates a child span and gives `fn` a child `ctx`. +- **Adapters trust typed inputs:** no validation of malformed caller values and no try/catch around recording; only + carriers from another process are checked (keys present, strings), else the root. Callers never protect themselves + against broken adapters. +- **Across links:** `call` and `hold` carry the caller's carrier as a bounded header field (section 22); the provider + extracts it and runs the method under it; relays add a `chord.relay` span. + + ``` + chord.call Notes.add [tab A] + └─ chord.relay [server] + └─ chord.method Notes.add [session:S1] principal=guest + · event chord.commit docs=2 ops=5 + ``` + +- **Shared work** (one activation for several getters, a worker start) is its own root span linked to its first + trigger; each waiting span links to it. Routine runs, job runs, eviction, backoff retries and background closes are + root spans linked to their trigger where one exists. +- **Tiers:** spans for calls, methods, relays, holds, activations, opens, closes, loads, revocations, resolves, routine + runs and job runs; events for commits inside methods, retries and parking; metrics for state messages, WebSocket + bytes, commit durations, in-flight calls, active holds, open links, park durations and backpressure waits. IDs are + attributes of spans, never of metrics. +- **Nothing is measured or walked for telemetry:** only sizes the code already has. Errors carry a fixed category and + a message truncated to 500 characters; cancelling is not an error. +- **A typed telemetry schema** ships with Chord, as pi-ai (`AI_TELEMETRY_SCHEMA`) and Durable + (`DURABLE_TELEMETRY_SCHEMA`) have; call sites that only have a context use the generic `addEvent`. +- **Storage and reading:** `@earendil-works/chord/telemetry-sqlite` over `chord/sqlite` (Node and Cloudflare): one + row per settled span (events and links inline), metrics aggregated per series per 10 s, a 1 s batched flush with + retention (7 days), a bounded queue (dropped and counted). The shared read interface `TelemetryStore` (Jaeger + style): `findTraces` (name, attributes, errors, time window, limit, cursor) returns complete traces, `getTrace`, + `findMetrics`. Spans carry global ids and a trace id; links carry trace and span id. One renderer + (`pi-telemetry/render`) prints links and never follows them. A web trace viewer in the playground reads the store. +- **Durable** (done): submit and phase spans, stored carriers on submissions and tasks (`TaskOptions.trace`, creation + links), events after commit, report events; pi-ai emits `pi.ai.request`/`pi.ai.poll` from one observer in + `lazyStream`. Persisted Chord jobs store their start carrier. + +## 30. Cost model + +As little compute and memory as possible while staying correct; Delta was built for this (its own primitives are +practically free: `diffRevisions` on shared subtrees 0.018 ms, `applyImmutable` per small change 0.004 ms). Measured +before/after per hop: `apps/p2/docs/p3-reviews/v2-cost-model.md`. + +| Hop | Work | +| --- | --- | +| commit | Delta's `prepare` (copies and checks only placed values), adopt by pointer swap; storage writes the delta | +| reads in a transaction | the draft overlay, no copy; unchanged = no write | +| subscribers on the same host | the same value and ops by reference | +| derives | compute plus `diffRevisions`; unchanged parts stay the same objects (`mapShared`, cookbook) | +| send | one encode per touched hold member per commit at its origin (the member's Delta encoder); while the window is full, per op one path key and one map lookup (leaf index), and one encode per waiting member per freed slot | +| relay, per message | decode the envelope, swap the origin (one shallow copy), encode the envelope; parts copied once into the outgoing buffer, never parsed; per `bound`/`state` one FIFO entry; two bytes of origin per message upstream | +| relay, per hold | the address's params checked once (route admission, `authorize`) | +| receive | the compiled `MESSAGE` check on untrusted links; arguments validated once at the provider (compiled closed schema), params checked; one `ack` per `bound`/`state` message | +| heartbeat | one timestamp write per received message; one timer per host | +| apply | `applyImmutable`, structure shared | +| new hold | the value encoded once (unavoidable) | +| memory | one authoritative value per document per host; replicas and derived values share structure; one cached value per adapted binding | + +**Cost guards:** Chord reports full-value walks, deep copies, encodes, schema validations and materializations as +ordinary telemetry metrics, always on (no test mode; with the NOOP adapter they cost an empty call where such work +happens). Tests use the in-memory adapter and assert them per hop (e.g. one appended note to a 500 KB document with +10 viewers: no walks, no copies, one encode per hold part at its origin and none at relays, no validations); a `vitest bench` suite measures the hops. + +--- + +## 31. Rejected + +`replicatedState` and its source family as public API; a separate IndexedDB backend; generic JSON rows and +store-maintained index entries; last-write-per-row compaction; recording raw SQL statements or SQLite changesets; +plugins' own databases holding source data; rollback after losing the driver role; JSON-field operands in bulk +updates (until needed); a lazy-lifetime flag; a `lifetime` option; derive `previous`. Drain-based handover; version fences without transactions; object restart for state changes; Durable's Session as the +object store; volatile documents; per-group lines; generic call or request ids; optimistic transactions; an outbox for +transient delivery errors; revoking proxies on `.value`; resources replaced in place (`replace`, `deps`, stale +handles); registries created at runtime and entries resolved through mutable indirection (contributions that change +with their contributor, readable contributions and readers looking up the current value at use are fine, R25); generic +registry keys with a `conflict` flag; +`restart_required`; detached references counting toward presence; contracts taken from the running app; ceremony +(`native/` directories, `defineNative`, lints); weakening type checking; "magic" input-field matching for delete; a +factory trial; a migrate dry run; principal field patterns and re-stamping for revocation; `storeKey`; a consumer contract check during staging; reload votes; per-plugin rollback within a +host; `defaultProjectTrust: "always"`; loader variants A and B; a topology layer above the host API; a stream type in +Chord; callbacks as method arguments; `opaque` members; numeric middleware priorities; placement as data naming an +input field; host-name strings in app code; host values (start data and platform bindings are core objects); +consumer-side linger; registry composition as a staging check; a JSONL document storage; `tx.get` for documents; validating state and documents on commits or reads (migration results are validated once, R8); copying values on same-host calls; op-rewriting projections and a +view algebra; keeping natives across reloads by name; Chord ending the process on a storage failure; an `optional` +flag on references; cross-relay reset coordination; count limits that refuse calls or holds; relays acking per hop, relay resets +(`reset { hold }`, `reset { holds }`), relay recovery and id renumbering at relays (replaced by flow control per +origin); a byte window, credit messages, snapshot admission and debt; `reset` from a consumer; a round-robin of +relayed calls at the relay; `state: false` on references (C8); `inVariant`/`expectVariant` (C15); per-call deadlines. diff --git a/packages/chord/docs/next/durable-observers/book.md b/packages/chord/docs/next/durable-observers/book.md new file mode 100644 index 000000000..71f6f4833 --- /dev/null +++ b/packages/chord/docs/next/durable-observers/book.md @@ -0,0 +1,432 @@ +--- +title: In the Step: when Durable's views change, and who hears it +cover: In the / Step +kicker: An explainer · Chord v2 slice 6 and pi-durable · 2026-10-26 +subtitle: Why Durable's views must change inside the storage's commit, why the first build broke hosts, and what the "publication at adoption" proposal changes +stats: + - 1 | Durable commit, seen as one change + - 4 | regressions the first build had, all in one gap + - 7 | things that change for hosts compared with HEAD + - 5 | decisions for the owner +note: | + Written for the owner from the build of `/home/badlogic/pi-observers` (uncommitted), the four reviews + (`/tmp/observers-review-{astra,fable,opus,sol}.md`) and the four checks of the proposal + (`/tmp/observers-proposal-{astra,fable,opus,sol}.md`), whose probes ran against HEAD, the first build and the + proposal. Code is Durable's `session/session.ts` and `harness/*.ts` at HEAD, simplified. Nothing here is decided + beyond what the decision log says ("Slice 6 first decisions", refined 2026-10-26). +colophon: In the Step · first edition · 2026-10-26 · from the observers build, its reviews and the proposal checks +--- + +# Before we begin {.preface} + +Mario's phone shows a p3 conversation. Next to the transcript a small status line reads "last: tool call, 1 task +running". It is a Chord derive over two Durable views: the conversation view (the transcript) and the task graph (the +running tasks). + +The model asks for a tool. Durable makes one commit, seq 41, that appends the tool-call entry and creates the tool's +task. The phone should go from "last: assistant, 0 running" to "last: tool call, 1 running" in one step. Today it can +show "last: tool call, 0 running" in between: a state that never existed. Slice 6 also puts Durable and Chord into one +store with one commit step, where every owner's memory must show a complete prefix of the log (S1, W1). Both need the +same thing: Durable's views must change inside the storage's commit, not after it. + +This book explains where Durable publishes a commit today, why moving half of it broke hosts, and what moving all of +it changes. + +## Seven things to know + +:::findings +1. **Durable adopts inside the storage, but publishes after it.** `Storage.commit(writes, adopt, context)` calls + `adopt` once the batch is durable; Durable's views, scheduler and host listeners all run later, after + `Storage.commit` returned. +2. **Two watches tear one commit.** A derive over two Durable views sees seq 41 in one view before the other. +3. **The first build moved only Durable's internal observers.** Host listeners and telemetry stayed after the + storage. Everything a host could see in between came out in the wrong order. +4. **All four regressions live in one gap:** the time between `adopt` and the end of `Storage.commit`. +5. **The proposal removes the gap:** the whole publication happens inside `adopt`. +6. **It changes seven things a host could observe**, most of them timing nobody documented, one a bug fix, one a real + problem for Chord. +7. **The real problem has a small fix:** host listeners one microtask after `adopt`, outside Chord's batch (Opus's + variant). +::: + +## Words used throughout + +**Durable** is pi's agent harness; a **Session** is its one line of atomic commits. **Storage** is Durable's storage +interface; `Storage.commit(writes, adopt, context)` stores a batch and calls **`adopt(seq)`** once it is durable. +Durable's **trackers** hold each loaded document's value in memory. **Internal observers** are Durable's own commit +listeners (`observeCommits`): the conversation **view** mounts, the **task graph**, the **scheduler** and the +**submissions**. **Host listeners** are what an app registers with `subscribeCommits`. A **watch** is a +`WatchHandle`: its listener runs a microtask after a frame arrives. **The gap** is the time between `adopt` and the +moment `Storage.commit` resolves. In Chord's store, the **commit step** is the store-wide serialized part of a commit +(backend transaction plus adoption), and the **batch** is Chord's synchronous adopt, mark and flush around it. +Letters in traces: **A** = `adopt`, **R** = `Storage.commit` resolves, **H** = host listeners, **C** = +`session.commit()` resolves. + +# Today {#today lede="Where a Durable commit is adopted and where it is published, at HEAD."} + +A Durable commit runs on the Session's line from start to end. Inside, `#runCommit` hands the batch to the storage +with an `adopt` callback, waits for it, and only then publishes: + +```ts +// session.ts, #runCommit, at HEAD (simplified) +const adopt = (seq: Seq): void => { + adoption = { documents: tx.adopt(seq) }; // trackers swap to the new values +}; +seq = await this.#storage.commit(writes, adopt, context); +// ... back in the Session, after R: +this.#publish(seq, writes, documents, context); // telemetry events, internal observers, host listeners +return result; // C +``` + +The SQLite storage calls `adopt` right after `COMMIT` returned. JSONL calls it and then still awaits its own cleanup +of sidecar files before it resolves: + +```ts +// jsonl/storage.ts +adopt(seq); // A +await this.reclaimSidecars(reclamations, context); // still inside Storage.commit +return seq; // R +``` + +:::sequence animate | One Durable commit at HEAD. The trackers change inside the storage; everything that tells anyone about it happens after the storage returned. +actor session "Session" "the line" accent +actor storage "Storage" "JSONL" muted +actor obs "internal observers" "views, graph, scheduler" +actor host "host" "subscribeCommits" +actor watch "watch listeners" "microtasks" blue +session -> storage: commit(writes, adopt) +storage -> storage: write, marker durable +storage => session: A: adopt(41), trackers swap +storage -> storage: reclaim sidecars +storage --> session: R +session -> obs: telemetry events, then observers (views advance, waits resolve) +session -> host: H: subscribeCommits(41) +obs -> watch: frames queued, delivered next microtask +session --> session: C +::: + +Two things follow. A host listener always runs before any watch listener of the same commit: the observers queue the +watch deliveries as microtasks, and the host listeners run synchronously right after them. And nothing is published +until the storage is completely done. + +## The tear + +Chord follows a Durable view through its watch. Two views mean two watches, and each delivers in its own microtask: + +```ts +const status = self.derive([conversationView, taskGraph], ([view, graph]) => ({ + last: view.entries.at(-1)?.kind, + running: graph.tasks.filter((t) => t.status === "running").length, +})); +``` + +```trace +commit 41 (entry + task) + microtask 1: view watch delivers 41 → derive: last "toolCall", running 0 ← never existed + microtask 2: graph watch delivers 41 → derive: last "toolCall", running 1 +``` + +The phone receives both. Chord's rule W1 says a commit changes its sources inside one step and the derives over them +recompute once, right after. For Durable's views that means: they must change inside `adopt`, and Chord must follow +them as sources marked there, not through watches. + +## Why after is not good enough for the store + +In Chord's store, Durable's storage (`StoreStorage`) runs `adopt` inside the store's commit step, and the step lets +the next owner's commit start as soon as `adopt` returned. Durable's `#publish` runs later, after its `await`. In +between, a Chord object's commit can adopt, flush and recompute a derive that reads a Durable view still at 40. The +owner ruled out the alternative, a store that waits for Durable's continuation: no Durable special cases in the store. + +# The First Build {#first lede="Moving half the publication into adopt, and the four things that broke."} + +The first build moved Durable's internal observers into `adopt` and left the rest of `#publish` (host listeners, +telemetry events) after R. To keep host code from waking too early, it added `afterPublish(callback)`: observers queue +what wakes a host (resolving a wait, aborting a run, `onWake`), and the queue runs after the host listeners. + +```ts +const adopt = (seq: Seq): void => { + const documents = tx.adopt(seq); + this.#observeStored(seq, writes, documents); // internal observers, inside A +}; +seq = await this.#storage.commit(writes, adopt, context); +this.#publish(...); // host listeners, after R +this.#settle(); // afterPublish callbacks +``` + +Durable's state (views, the scheduler's idle state, the metric counts) now changes at A. Hosts hear about it at H, +after R. Everything that runs in between sees a commit that is half published. Four reviewers found four ways that +shows, each reproduced with a probe. + +:::steps | The four regressions ++++ Watch listeners before host listeners +The observers call `Watch.advance()` inside A; the watch delivers a microtask later. The storage's own `await` and the +Session's `await` resume later still, so the watch listener runs first. + +```trace +HEAD: A … R → H → watch +first build: A (watch queued) → watch → R → H +``` + +Even with the memory storage. With JSONL the watch listener runs before `Storage.commit` resolved; if the storage then +rejects, the watch has shown a commit whose `commit()` failed. ++++ waitForIdle answers early +The scheduler's idle state is updated in A. A host that calls `waitForIdle()` in the gap gets the fast path ("already +idle") and resumes before H. + +```ts +harness.subscribeCommits((p) => seen.push(p.seq)); +// a task-graph watch listener runs in the gap: +await harness.waitForIdle(ctx); // resolves before seen has the terminal commit +``` ++++ A late subscriber misses the commit +When nothing listened at A, the publication is never built. A host that subscribes during the gap gets nothing for +that commit; at HEAD the listener list was read after R, so it did. ++++ The metric drifts after a rejection +A storage that rejects after A (JSONL's cleanup fails): the scheduler counted the new task in A and queued its `+1` +for the metric; the failure seals the scheduler, which subtracts `-1`; the queued `+1` is dropped with the failed +commit. + +```trace +HEAD: [] (nothing observed, net 0) +first build: [-1] (net -1) +``` +::: + +Fable found three more of the same kind: a host listener that closes the Harness makes resolved waits reject; +scheduler line jobs queued in A jump ahead of a host listener's own commit; abort listeners moved after H. The +patches each reviewer suggested were more `afterPublish`. The owner called it what it is: special-casing. + +# Publication at Adoption {#proposal lede="One rule: a commit is published when it is adopted, all of it, inside adopt."} + +The gap is the bug. The first build created it by splitting the publication. The proposal moves all of it: + +```ts +const adopt = (seq: Seq): void => { + const documents = tx.adopt(seq); // trackers, as today + const publication = this.#publication(seq, writes, documents); + this.#events(writes); // pi.durable.entry / .submission, contained + this.#notifyInternal(publication); // views, graph, scheduler, submissions + this.#notifyHosts(publication); // subscribeCommits +}; +seq = await this.#storage.commit(writes, adopt, context); +// after R: touch documents, sweep, fail() for an error an observer threw; nothing is published here +``` + +The order inside is HEAD's: telemetry events, internal observers, host listeners. `afterPublish` disappears. Waits +need nothing special: an observer resolves a promise in A, and the waiter's continuation runs later by nature, after +the host listeners already ran synchronously. One rule replaces the special cases: `adopt` runs no task code. The +only synchronous path into task code is `AbortController.abort()` of a newly marked run (abort listeners run inside +`abort()`), so that one is queued with `queueMicrotask`. `adopt` never throws: every listener, every report and every +telemetry call inside it is contained. + +:::sequence animate | The same commit under the proposal. Everything happens inside A; R and C only end it. +actor session "Session" "the line" accent +actor storage "Storage" "JSONL" muted +actor obs "internal observers" "views, graph, scheduler" +actor host "host" "subscribeCommits" +actor watch "watch listeners" "microtasks" blue +session -> storage: commit(writes, adopt) +storage -> storage: write, marker durable +storage => session: A: adopt(41) +session -> session: trackers swap, telemetry events +session => obs: observers: views advance, waits resolve +session => host: H: subscribeCommits(41) +obs -> watch: watches, waits, abort: next microtasks +storage -> storage: reclaim sidecars +storage --> session: R +session --> session: C +::: + +What this does to the regressions, verified by Opus and Fable with probes against HEAD: + +| regression | HEAD | first build | proposal | +| --- | --- | --- | --- | +| watch listeners vs H | H, watch | **watch, H** | H, watch | +| `waitForIdle` in the gap | after H | **before H** | after H | +| metric after a rejection following A | 0 | **−1** | 0 (+1, −1) | +| `close()` from H, then `waitForTask` | resolves | **rejects** | resolves | +| scheduler jobs vs H's own commit | H first | **jobs first** | H first | +| stuck state, lost reports | none | yes | none | +| allocations per commit | baseline | more | about HEAD | + +HEAD's whole non-e2e Durable suite (73 files, 1404 tests) passes on a build of the proposal. The four tests the first +build added that encode its split fail, as they should, and are replaced. + +# What Changes for Hosts {#changes lede="Seven things a host, a task or a storage could observe, compared with HEAD."} + +Nothing in the public API changes. These are the differences in behaviour the four checks found. + +## 1. With JSONL, everything runs before the storage is done + +```trace +HEAD: marker → A → cleanup … → R → observers → H → watches, waits → C +proposal: marker → A { observers, H } → watches, waits, abort → cleanup … → R → C +``` + +Host and task code woken by a commit now runs while JSONL still cleans up its sidecars. It cannot reach the storage +through the Session: the line is still held, so a commit or read it starts waits. The memory, SQLite and IndexedDB +storages do no work after A, so for them only the microtask placement differs, and the order is HEAD's. No spec +section promised "after `Storage.commit` resolved". Verdict: harmless; the spec gets one sentence. + +## 2. Abort listeners fire after host listeners + +```ts +runtime.signal.addEventListener("abort", () => log(hostSaw(id))); +// HEAD: false (abort before H) proposal: true (abort after H) +``` + +At HEAD the scheduler aborted a newly marked run inside `#publish`, before the host listeners. Under the proposal the +abort is a microtask after A, so after H. No contract fixes this order. Decision for the owner. + +## 3. A commit the storage rejects after A is published + +A host keeps a replica from `subscribeCommits`. JSONL wrote the marker (the commit is durable), called `adopt`, then +failed while cleaning up and rejected. + +```trace +HEAD: durable, but no observer and no host listener ran → the replica misses it + → replica ≠ storage; waits of that commit reject; the Session fails +proposal: published at A → the replica gets it → replica = storage + → waits of that commit resolve; the Session fails after R +``` + +Durable's spec says a replica fed by `subscribeCommits` "stays equal to Storage"; HEAD breaks that, the proposal keeps +it. Spec invariant 2 changes from "published only after its storage commit succeeds" to "published once it is +durable, even when `commit()` then rejects". Decision for the owner; all four checks recommend it. + +## 4. A subscriber that arrives late misses the commit + +```ts +// host code that runs in the gap, between A and R: +harness.subscribeCommits(listener); // HEAD: gets commit 41; proposal: does not, 41 was published at A +``` + +Consistent with the rule: a listener added after the publication misses it, as one added after `#publish` did at +HEAD. Documented, nothing to decide. + +## 5. An observer that throws fails the Session after the host listeners + +At HEAD, a throwing internal observer failed the Session at once, inside `#publish`: close listeners and the +`session_failed` report came before the host listeners. Under the proposal, `adopt` records the error, lets the other +observers and the host listeners run, and the Session fails after R. The commit stands, as before. Failing inside A +instead would make Durable's storage guard reject the caller of a durable commit with `SessionFailed`. Verdict: keep +it so. + +## 6. Telemetry that throws no longer breaks memory + +At HEAD, `#publish` called `context.telemetry.addEvent` for each entry and submission without protection. A throwing +adapter rejected a commit that was already durable and adopted, skipped every observer and host listener, and left the +Session healthy: the scheduler and the views missed the commit for good. Inside `adopt` nothing may throw, so the +proposal contains it and reports it; the commit resolves. This fixes a HEAD bug. The report needs a type (`host_error` +with callback `"telemetry"` is the suggestion). The same containment must cover Durable's report path itself +(`safeReport` records a metric and a span event outside its `try`), or `adopt` is not truly throw-free. + +## 7. Host listeners would run inside Chord's batch + +This is the one that matters. In Chord's store, `StoreStorage` runs Durable's `adopt` inside the store's commit step, +inside Chord's batch, where Chord refuses reentrant calls (`self.tx`, `get`, `watch`, `close` throw +`invalid/reentrant`). Under the proposal the host listeners run inside `adopt`, so inside that batch: + +```ts +// p3 code that mirrors Durable's commits into a Chord object +harness.subscribeCommits((publication) => + chord((ctx) => audit.record(publication.seq, ctx), context)); // throws invalid/reentrant +``` + +The listener keeps every rule Durable states (synchronous, no Session calls) and still fails. Its error is reported; +an `onReport` that itself uses Chord throws too, and a throwing `onReport` is dropped, so the work vanishes without a +trace. And every host listener's run time is added to the store-wide commit step that every other object's commit +waits for. + +:::flow wide | Under the proposal, a host listener runs inside Chord's batch; Chord calls from it are refused. +row "Store commit step (one at a time for the whole store)" accent +box backend "backend transaction" "`BEGIN … COMMIT`" muted +box adopt "Durable's adopt" "trackers, views, scheduler" accent +box hostin "host listeners" "inside the batch" dashed +row "Chord" +box flush "flush" "derives, wire, watches queued" blue +box refused "`self.tx` from a listener" "`invalid/reentrant`" dashed +arrow backend -> adopt "durable" +arrow adopt -> hostin "proposal" +arrow hostin -> refused "Chord call" +arrow adopt -> flush "after the step" +::: + +# The Microtask Variant {#variant lede="Host listeners one microtask after adopt: same results, outside Chord's batch."} + +Opus proposed a variant that keeps every result of the proposal and moves only the host listeners out of the step. +The publication is still built in A and the internal observers still run there. The host listeners are dispatched in +one microtask that is queued before the internal observers run: + +```ts +// inside adopt, after tx.adopt(seq) +this.#events(writes); // contained +if (this.#commitListeners.size > 0) { + const listeners = [...this.#commitListeners]; + queueMicrotask(() => { + for (const listener of listeners) + contained(() => listener(stored, context), (error) => this.report(error, LISTENER, context)); + }); +} +for (const listener of [...this.#internalListeners]) + contained(() => listener(observed, context), (error) => { failure ??= { error }; }); +``` + +Microtasks run first in, first out. The host dispatch is queued before anything the observers queue, so the host +listeners still run before every watch delivery, every resolved wait, every abort and `onWake`. And it is a +microtask, so it runs after the storage's synchronous step: after Chord's batch and flush, where Chord calls work. + +:::sequence animate | The variant inside Chord's store. The step stays synchronous and short; host listeners run right after it, first of everything the commit woke. +actor store "store step" "one at a time" accent +actor durable "Durable adopt" "views, scheduler" +actor chord "Chord batch" "mark, flush" blue +actor host "host listeners" +actor later "watches, waits, abort" muted +store -> store: backend COMMIT +store => durable: adopt(41): trackers, events +durable -> host: queue H (microtask 1) +durable -> durable: observers: views advance (sources marked) +durable -> later: queue watches, waits, abort (microtasks 2…) +store => chord: flush: derives over both views recompute once +store --> store: step ends, next commit may start +host -> host: H runs: Chord calls allowed +later -> later: then watches, waits, abort +::: + +Chord needs nothing Durable-specific: only that its commit step and its flush are synchronous, which its design +already states. The store never knows Durable's continuation, as the owner required. + +What the variant gives up compared with the proposal: host listeners are no longer called synchronously from inside +`adopt`, but one microtask later, still on the Session line and still before C. Opus ran HEAD's full non-e2e suite +against the variant (1404 of 1404) and all twelve probes: the same results as the proposal. One detail to settle while +building: the listener list is copied at A, so a listener removed between A and the microtask is still called once; +copying it inside the microtask avoids that, but then a listener added in between is called too. + +# Decisions {.epilogue lede="What the owner decides before the rebuild."} + +1. **The variant or the proposal.** The variant (recommended by Opus) runs host listeners one microtask after `adopt`, + outside Chord's batch. The proposal runs them inside `adopt` and needs the documented rule "no Chord calls from + `subscribeCommits` or `onReport`". +2. **Abort listeners after host listeners** (change 2): acceptable, or must the abort keep HEAD's place before them. +3. **Publish a commit the storage rejects after `adopt`** (change 3): recommended by all four checks; it fixes HEAD's + replica gap and changes spec invariant 2. +4. **The report type of a throwing telemetry call** (change 6): suggested `host_error` with callback `"telemetry"`. +5. **HEAD's order inside the publication** (events, internal observers, host listeners) and a fully throw-free + `adopt`, the report path included. These are fixes rather than choices; listed so they are not lost. + +After the decisions: the spec rewrites (invariant 2, section 4 and its pseudo-code, the Storage contract, section 10 +failures), new tests in place of the four that encode the split (order with the memory storage and with a storage that +works after `adopt`, `waitForIdle` in the gap, publication on a rejection with a balanced metric, close in the gap, +Chord-like reentrancy, telemetry throws), a rebuild by a fresh builder, and the four reviewers again. + +# Sources {.appendix} + +| what | where | +| --- | --- | +| HEAD code | `packages/durable/src/session/session.ts` (`#runCommit`, `#publish`), `harness/view.ts`, `harness/task-graph.ts`, `harness/scheduler.ts`, `harness/submissions.ts`, `storage/jsonl/storage.ts` | +| first build | `/home/badlogic/pi-observers` (uncommitted) | +| reviews of the first build | `/tmp/observers-review-{astra,fable,opus,sol}.md` | +| the proposal | `/tmp/observers-proposal.md` | +| checks of the proposal | `/tmp/observers-proposal-{astra,fable,opus,sol}.md`; probes in `/tmp/obsprobe-*` | +| decisions so far | `packages/chord/docs/next/decisions.md`, "Slice 6 first decisions" | +| Chord rules | `packages/chord/docs/next/design.md` §11 (W1, the batch), §24 (Durable integration) | diff --git a/packages/chord/docs/next/durable-observers/index.html b/packages/chord/docs/next/durable-observers/index.html new file mode 100644 index 000000000..fe6961cd5 --- /dev/null +++ b/packages/chord/docs/next/durable-observers/index.html @@ -0,0 +1,517 @@ + + +In the Step: when Durable's views change, and who hears it + + + + +

+
+
An explainer · Chord v2 slice 6 and pi-durable · 2026-10-26
+

In the
Step

+

Why Durable's views must change inside the storage's commit, why the first build broke hosts, and what the "publication at adoption" proposal changes

+
1Durable commit, seen as one change
4regressions the first build had, all in one gap
7things that change for hosts compared with HEAD
5decisions for the owner
+

Written for the owner from the build of /home/badlogic/pi-observers (uncommitted), the four reviews +(/tmp/observers-review-{astra,fable,opus,sol}.md) and the four checks of the proposal +(/tmp/observers-proposal-{astra,fable,opus,sol}.md), whose probes ran against HEAD, the first build and the +proposal. Code is Durable's session/session.ts and harness/*.ts at HEAD, simplified. Nothing here is decided +beyond what the decision log says ("Slice 6 first decisions", refined 2026-10-26).

+
+
+ + +
+

Before we begin

+

Mario's phone shows a p3 conversation. Next to the transcript a small status line reads "last: tool call, 1 task +running". It is a Chord derive over two Durable views: the conversation view (the transcript) and the task graph (the +running tasks).

+

The model asks for a tool. Durable makes one commit, seq 41, that appends the tool-call entry and creates the tool's +task. The phone should go from "last: assistant, 0 running" to "last: tool call, 1 running" in one step. Today it can +show "last: tool call, 0 running" in between: a state that never existed. Slice 6 also puts Durable and Chord into one +store with one commit step, where every owner's memory must show a complete prefix of the log (S1, W1). Both need the +same thing: Durable's views must change inside the storage's commit, not after it.

+

This book explains where Durable publishes a commit today, why moving half of it broke hosts, and what moving all of +it changes.

+

Seven things to know

+
    +
  1. Durable adopts inside the storage, but publishes after it. Storage.commit(writes, adopt, context) calls +adopt once the batch is durable; Durable's views, scheduler and host listeners all run later, after +Storage.commit returned.
  2. +
  3. Two watches tear one commit. A derive over two Durable views sees seq 41 in one view before the other.
  4. +
  5. The first build moved only Durable's internal observers. Host listeners and telemetry stayed after the +storage. Everything a host could see in between came out in the wrong order.
  6. +
  7. All four regressions live in one gap: the time between adopt and the end of Storage.commit.
  8. +
  9. The proposal removes the gap: the whole publication happens inside adopt.
  10. +
  11. It changes seven things a host could observe, most of them timing nobody documented, one a bug fix, one a real +problem for Chord.
  12. +
  13. The real problem has a small fix: host listeners one microtask after adopt, outside Chord's batch (Opus's +variant).
  14. +
+

Words used throughout

+

Durable is pi's agent harness; a Session is its one line of atomic commits. Storage is Durable's storage +interface; Storage.commit(writes, adopt, context) stores a batch and calls adopt(seq) once it is durable. +Durable's trackers hold each loaded document's value in memory. Internal observers are Durable's own commit +listeners (observeCommits): the conversation view mounts, the task graph, the scheduler and the +submissions. Host listeners are what an app registers with subscribeCommits. A watch is a +WatchHandle: its listener runs a microtask after a frame arrives. The gap is the time between adopt and the +moment Storage.commit resolves. In Chord's store, the commit step is the store-wide serialized part of a commit +(backend transaction plus adoption), and the batch is Chord's synchronous adopt, mark and flush around it. +Letters in traces: A = adopt, R = Storage.commit resolves, H = host listeners, C = +session.commit() resolves.

+
+ +
I

Today

Where a Durable commit is adopted and where it is published, at HEAD.

+

A Durable commit runs on the Session's line from start to end. Inside, #runCommit hands the batch to the storage +with an adopt callback, waits for it, and only then publishes:

+
// session.ts, #runCommit, at HEAD (simplified)
+const adopt = (seq: Seq): void => {
+  adoption = { documents: tx.adopt(seq) };          // trackers swap to the new values
+};
+seq = await this.#storage.commit(writes, adopt, context);
+// ... back in the Session, after R:
+this.#publish(seq, writes, documents, context);      // telemetry events, internal observers, host listeners
+return result;                                       // C
+

The SQLite storage calls adopt right after COMMIT returned. JSONL calls it and then still awaits its own cleanup +of sidecar files before it resolves:

+
// jsonl/storage.ts
+adopt(seq);                                           // A
+await this.reclaimSidecars(reclamations, context);   // still inside Storage.commit
+return seq;                                           // R
+
Sessionthe lineStorageJSONLinternal observersviews, graph, schedulerhostsubscribeCommitswatch listenersmicrotaskscommit(writes, adopt)write, marker durableA: adopt(41), trackers swapreclaim sidecarsRtelemetry events, then observers (views advance, waits resolve)H: subscribeCommits(41)frames queued, delivered next microtaskC
Figure 1. One Durable commit at HEAD. The trackers change inside the storage; everything that tells anyone about it happens after the storage returned.
+

Two things follow. A host listener always runs before any watch listener of the same commit: the observers queue the +watch deliveries as microtasks, and the host listeners run synchronously right after them. And nothing is published +until the storage is completely done.

+

The tear

+

Chord follows a Durable view through its watch. Two views mean two watches, and each delivers in its own microtask:

+
const status = self.derive([conversationView, taskGraph], ([view, graph]) => ({
+  last: view.entries.at(-1)?.kind,
+  running: graph.tasks.filter((t) => t.status === "running").length,
+}));
+
commit 41 (entry + task)
+  microtask 1: view watch delivers 41   → derive: last "toolCall", running 0    ← never existed
+  microtask 2: graph watch delivers 41  → derive: last "toolCall", running 1
+

The phone receives both. Chord's rule W1 says a commit changes its sources inside one step and the derives over them +recompute once, right after. For Durable's views that means: they must change inside adopt, and Chord must follow +them as sources marked there, not through watches.

+

Why after is not good enough for the store

+

In Chord's store, Durable's storage (StoreStorage) runs adopt inside the store's commit step, and the step lets +the next owner's commit start as soon as adopt returned. Durable's #publish runs later, after its await. In +between, a Chord object's commit can adopt, flush and recompute a derive that reads a Durable view still at 40. The +owner ruled out the alternative, a store that waits for Durable's continuation: no Durable special cases in the store.

+
+ +
II

The First Build

Moving half the publication into adopt, and the four things that broke.

+

The first build moved Durable's internal observers into adopt and left the rest of #publish (host listeners, +telemetry events) after R. To keep host code from waking too early, it added afterPublish(callback): observers queue +what wakes a host (resolving a wait, aborting a run, onWake), and the queue runs after the host listeners.

+
const adopt = (seq: Seq): void => {
+  const documents = tx.adopt(seq);
+  this.#observeStored(seq, writes, documents);       // internal observers, inside A
+};
+seq = await this.#storage.commit(writes, adopt, context);
+this.#publish(...);                                    // host listeners, after R
+this.#settle();                                        // afterPublish callbacks
+

Durable's state (views, the scheduler's idle state, the metric counts) now changes at A. Hosts hear about it at H, +after R. Everything that runs in between sees a commit that is half published. Four reviewers found four ways that +shows, each reproduced with a probe.

+
The four regressions

Watch listeners before host listeners

The observers call Watch.advance() inside A; the watch delivers a microtask later. The storage's own await and the +Session's await resume later still, so the watch listener runs first.

+
HEAD:        A … R → H → watch
+first build: A (watch queued) → watch → R → H
+

Even with the memory storage. With JSONL the watch listener runs before Storage.commit resolved; if the storage then +rejects, the watch has shown a commit whose commit() failed.

+

waitForIdle answers early

The scheduler's idle state is updated in A. A host that calls waitForIdle() in the gap gets the fast path ("already +idle") and resumes before H.

+
harness.subscribeCommits((p) => seen.push(p.seq));
+// a task-graph watch listener runs in the gap:
+await harness.waitForIdle(ctx);          // resolves before seen has the terminal commit
+

A late subscriber misses the commit

When nothing listened at A, the publication is never built. A host that subscribes during the gap gets nothing for +that commit; at HEAD the listener list was read after R, so it did.

+

The metric drifts after a rejection

A storage that rejects after A (JSONL's cleanup fails): the scheduler counted the new task in A and queued its +1 +for the metric; the failure seals the scheduler, which subtracts -1; the queued +1 is dropped with the failed +commit.

+
HEAD:        []        (nothing observed, net 0)
+first build: [-1]      (net -1)
+
+

Fable found three more of the same kind: a host listener that closes the Harness makes resolved waits reject; +scheduler line jobs queued in A jump ahead of a host listener's own commit; abort listeners moved after H. The +patches each reviewer suggested were more afterPublish. The owner called it what it is: special-casing.

+
+ +
III

Publication at Adoption

One rule: a commit is published when it is adopted, all of it, inside adopt.

+

The gap is the bug. The first build created it by splitting the publication. The proposal moves all of it:

+
const adopt = (seq: Seq): void => {
+  const documents = tx.adopt(seq);                              // trackers, as today
+  const publication = this.#publication(seq, writes, documents);
+  this.#events(writes);                                         // pi.durable.entry / .submission, contained
+  this.#notifyInternal(publication);                            // views, graph, scheduler, submissions
+  this.#notifyHosts(publication);                               // subscribeCommits
+};
+seq = await this.#storage.commit(writes, adopt, context);
+// after R: touch documents, sweep, fail() for an error an observer threw; nothing is published here
+

The order inside is HEAD's: telemetry events, internal observers, host listeners. afterPublish disappears. Waits +need nothing special: an observer resolves a promise in A, and the waiter's continuation runs later by nature, after +the host listeners already ran synchronously. One rule replaces the special cases: adopt runs no task code. The +only synchronous path into task code is AbortController.abort() of a newly marked run (abort listeners run inside +abort()), so that one is queued with queueMicrotask. adopt never throws: every listener, every report and every +telemetry call inside it is contained.

+
Sessionthe lineStorageJSONLinternal observersviews, graph, schedulerhostsubscribeCommitswatch listenersmicrotaskscommit(writes, adopt)write, marker durableA: adopt(41)trackers swap, telemetry eventsobservers: views advance, waits resolveH: subscribeCommits(41)watches, waits, abort: next microtasksreclaim sidecarsRC
Figure 2. The same commit under the proposal. Everything happens inside A; R and C only end it.
+

What this does to the regressions, verified by Opus and Fable with probes against HEAD:

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
regressionHEADfirst buildproposal
watch listeners vs HH, watchwatch, HH, watch
waitForIdle in the gapafter Hbefore Hafter H
metric after a rejection following A0−10 (+1, −1)
close() from H, then waitForTaskresolvesrejectsresolves
scheduler jobs vs H's own commitH firstjobs firstH first
stuck state, lost reportsnoneyesnone
allocations per commitbaselinemoreabout HEAD
+

HEAD's whole non-e2e Durable suite (73 files, 1404 tests) passes on a build of the proposal. The four tests the first +build added that encode its split fail, as they should, and are replaced.

+ + +
IV

What Changes for Hosts

Seven things a host, a task or a storage could observe, compared with HEAD.

+

Nothing in the public API changes. These are the differences in behaviour the four checks found.

+

1. With JSONL, everything runs before the storage is done

+
HEAD:     marker → A → cleanup … → R → observers → H → watches, waits → C
+proposal: marker → A { observers, H } → watches, waits, abort → cleanup … → R → C
+

Host and task code woken by a commit now runs while JSONL still cleans up its sidecars. It cannot reach the storage +through the Session: the line is still held, so a commit or read it starts waits. The memory, SQLite and IndexedDB +storages do no work after A, so for them only the microtask placement differs, and the order is HEAD's. No spec +section promised "after Storage.commit resolved". Verdict: harmless; the spec gets one sentence.

+

2. Abort listeners fire after host listeners

+
runtime.signal.addEventListener("abort", () => log(hostSaw(id)));
+// HEAD: false (abort before H)    proposal: true (abort after H)
+

At HEAD the scheduler aborted a newly marked run inside #publish, before the host listeners. Under the proposal the +abort is a microtask after A, so after H. No contract fixes this order. Decision for the owner.

+

3. A commit the storage rejects after A is published

+

A host keeps a replica from subscribeCommits. JSONL wrote the marker (the commit is durable), called adopt, then +failed while cleaning up and rejected.

+
HEAD:     durable, but no observer and no host listener ran → the replica misses it
+          → replica ≠ storage; waits of that commit reject; the Session fails
+proposal: published at A → the replica gets it → replica = storage
+          → waits of that commit resolve; the Session fails after R
+

Durable's spec says a replica fed by subscribeCommits "stays equal to Storage"; HEAD breaks that, the proposal keeps +it. Spec invariant 2 changes from "published only after its storage commit succeeds" to "published once it is +durable, even when commit() then rejects". Decision for the owner; all four checks recommend it.

+

4. A subscriber that arrives late misses the commit

+
// host code that runs in the gap, between A and R:
+harness.subscribeCommits(listener);   // HEAD: gets commit 41; proposal: does not, 41 was published at A
+

Consistent with the rule: a listener added after the publication misses it, as one added after #publish did at +HEAD. Documented, nothing to decide.

+

5. An observer that throws fails the Session after the host listeners

+

At HEAD, a throwing internal observer failed the Session at once, inside #publish: close listeners and the +session_failed report came before the host listeners. Under the proposal, adopt records the error, lets the other +observers and the host listeners run, and the Session fails after R. The commit stands, as before. Failing inside A +instead would make Durable's storage guard reject the caller of a durable commit with SessionFailed. Verdict: keep +it so.

+

6. Telemetry that throws no longer breaks memory

+

At HEAD, #publish called context.telemetry.addEvent for each entry and submission without protection. A throwing +adapter rejected a commit that was already durable and adopted, skipped every observer and host listener, and left the +Session healthy: the scheduler and the views missed the commit for good. Inside adopt nothing may throw, so the +proposal contains it and reports it; the commit resolves. This fixes a HEAD bug. The report needs a type (host_error +with callback "telemetry" is the suggestion). The same containment must cover Durable's report path itself +(safeReport records a metric and a span event outside its try), or adopt is not truly throw-free.

+

7. Host listeners would run inside Chord's batch

+

This is the one that matters. In Chord's store, StoreStorage runs Durable's adopt inside the store's commit step, +inside Chord's batch, where Chord refuses reentrant calls (self.tx, get, watch, close throw +invalid/reentrant). Under the proposal the host listeners run inside adopt, so inside that batch:

+
// p3 code that mirrors Durable's commits into a Chord object
+harness.subscribeCommits((publication) =>
+  chord((ctx) => audit.record(publication.seq, ctx), context));   // throws invalid/reentrant
+

The listener keeps every rule Durable states (synchronous, no Session calls) and still fails. Its error is reported; +an onReport that itself uses Chord throws too, and a throwing onReport is dropped, so the work vanishes without a +trace. And every host listener's run time is added to the store-wide commit step that every other object's commit +waits for.

+
STORE COMMIT STEP (ONE AT A TIME FOR THE WHOLE STORE)CHORDbackend transactionBEGIN … COMMITDurable's adopttrackers, views, schedulerhost listenersinside the batchflushderives, wire, watches queued`self.tx` from a listenerinvalid/reentrantdurableproposalChord callafter the step
Figure 3. Under the proposal, a host listener runs inside Chord's batch; Chord calls from it are refused.
+
+ +
V

The Microtask Variant

Host listeners one microtask after adopt: same results, outside Chord's batch.

+

Opus proposed a variant that keeps every result of the proposal and moves only the host listeners out of the step. +The publication is still built in A and the internal observers still run there. The host listeners are dispatched in +one microtask that is queued before the internal observers run:

+
// inside adopt, after tx.adopt(seq)
+this.#events(writes);                                            // contained
+if (this.#commitListeners.size > 0) {
+  const listeners = [...this.#commitListeners];
+  queueMicrotask(() => {
+    for (const listener of listeners)
+      contained(() => listener(stored, context), (error) => this.report(error, LISTENER, context));
+  });
+}
+for (const listener of [...this.#internalListeners])
+  contained(() => listener(observed, context), (error) => { failure ??= { error }; });
+

Microtasks run first in, first out. The host dispatch is queued before anything the observers queue, so the host +listeners still run before every watch delivery, every resolved wait, every abort and onWake. And it is a +microtask, so it runs after the storage's synchronous step: after Chord's batch and flush, where Chord calls work.

+
store stepone at a timeDurable adoptviews, schedulerChord batchmark, flushhost listenerswatches, waits, abortbackend COMMITadopt(41): trackers, eventsqueue H (microtask 1)observers: views advance (sources marked)queue watches, waits, abort (microtasks 2…)flush: derives over both views recompute oncestep ends, next commit may startH runs: Chord calls allowedthen watches, waits, abort
Figure 4. The variant inside Chord's store. The step stays synchronous and short; host listeners run right after it, first of everything the commit woke.
+

Chord needs nothing Durable-specific: only that its commit step and its flush are synchronous, which its design +already states. The store never knows Durable's continuation, as the owner required.

+

What the variant gives up compared with the proposal: host listeners are no longer called synchronously from inside +adopt, but one microtask later, still on the Session line and still before C. Opus ran HEAD's full non-e2e suite +against the variant (1404 of 1404) and all twelve probes: the same results as the proposal. One detail to settle while +building: the listener list is copied at A, so a listener removed between A and the microtask is still called once; +copying it inside the microtask avoids that, but then a listener added in between is called too.

+
+ +
∴

Decisions

What the owner decides before the rebuild.

+
    +
  1. The variant or the proposal. The variant (recommended by Opus) runs host listeners one microtask after adopt, +outside Chord's batch. The proposal runs them inside adopt and needs the documented rule "no Chord calls from +subscribeCommits or onReport".
  2. +
  3. Abort listeners after host listeners (change 2): acceptable, or must the abort keep HEAD's place before them.
  4. +
  5. Publish a commit the storage rejects after adopt (change 3): recommended by all four checks; it fixes HEAD's +replica gap and changes spec invariant 2.
  6. +
  7. The report type of a throwing telemetry call (change 6): suggested host_error with callback "telemetry".
  8. +
  9. HEAD's order inside the publication (events, internal observers, host listeners) and a fully throw-free +adopt, the report path included. These are fixes rather than choices; listed so they are not lost.
  10. +
+

After the decisions: the spec rewrites (invariant 2, section 4 and its pseudo-code, the Storage contract, section 10 +failures), new tests in place of the four that encode the split (order with the memory storage and with a storage that +works after adopt, waitForIdle in the gap, publication on a rejection with a balanced metric, close in the gap, +Chord-like reentrancy, telemetry throws), a rebuild by a fresh builder, and the four reviewers again.

+
+ +
A

Sources

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
whatwhere
HEAD codepackages/durable/src/session/session.ts (#runCommit, #publish), harness/view.ts, harness/task-graph.ts, harness/scheduler.ts, harness/submissions.ts, storage/jsonl/storage.ts
first build/home/badlogic/pi-observers (uncommitted)
reviews of the first build/tmp/observers-review-{astra,fable,opus,sol}.md
the proposal/tmp/observers-proposal.md
checks of the proposal/tmp/observers-proposal-{astra,fable,opus,sol}.md; probes in /tmp/obsprobe-*
decisions so farpackages/chord/docs/next/decisions.md, "Slice 6 first decisions"
Chord rulespackages/chord/docs/next/design.md §11 (W1, the batch), §24 (Durable integration)
+
+ +

In the Step · first edition · 2026-10-26 · from the observers build, its reviews and the proposal checks

+ + + diff --git a/packages/chord/docs/next/explainer/book.md b/packages/chord/docs/next/explainer/book.md new file mode 100644 index 000000000..05a98d9aa --- /dev/null +++ b/packages/chord/docs/next/explainer/book.md @@ -0,0 +1,4507 @@ +--- +title: Chord v2: a guide to building apps that reload +cover: Chord v2 / A Guide +kicker: A guide · Chord v2 target design · 2026-10-20 +subtitle: How to build an app like p2 from nothing on Chord v2 and pi-durable, one object at a time, so that new plugin code takes over while the agent keeps running +stats: + - 11 | layers in the rebuild, L1 to L11 + - ~1,300 | lines built so far (L1 contracts) + - 6–7k | target size of Chord v2, without tests + - 0.004 ms | Delta's apply per small change +note: | + This guide teaches the **designed** API of Chord v2 (`packages/chord/src/next`, export `@earendil-works/chord/next`) + from `packages/chord/docs/next/design.md` and the newer decision log `packages/chord/docs/next/decisions.md`, which wins + where the two differ. Most of it is **not built yet**: a first kernel of about 10k lines is frozen as a reference and + Chord v2 is being rebuilt layer by layer; only the L1 contract layer exists (`src/next/contracts/`, commit + `7d2bb8ea8`). Code in this guide is therefore the target, not something you can run today. Every chapter says what is + built. API names are working names. Notes marked *review* are points that are undecided, or where the sources + disagree; they name the files. The reader is assumed to know pi-durable; Chord is explained from the start. +colophon: Chord v2 · a guide · first edition · 2026-10-20 · based on design.md (target design v2), redesign-decisions.md (review rounds 2 and 3), chord-next-walkthrough.md, layer-1.md, contracts/README.md, the p3 functionality map, questions.md and the pi-durable README +--- + +# Before we begin {.preface} + +It is late in the evening. Mario is on his phone, connected to his laptop at home through the Radius relay. In session +S1 the agent is twenty minutes into a refactoring; right now it is running a test suite through `bash`. A colleague, +Ann, follows the session read-only from her own browser. Mario notices that the notes panel sorts notes the wrong way +round. He fixes one line in the notes plugin on the laptop and presses **Reload**. + +What should happen is easy to say. The notes panel on both screens shows the new order. The test run keeps running. The +agent's turn continues as if nothing happened. Ann's view does not flicker into an error. Nothing that was written is +lost, and no process restarts. + +The obvious ways to get there fail. **Restarting the processes** loses the turn in flight, the open terminal and every +connection; pi-durable would resume the turn, but the test run and the connections are gone. **Hot-swapping modules** +in place, as front-end tooling does, leaves the old code alive in closures, timers, event listeners and sockets that +nobody knows about: the old sort function keeps running inside a callback someone stored, a file watcher opened by the +old code is never closed, and two versions of the plugin now write the same data without any order. + +Chord v2 is the runtime that makes the first outcome normal. It does so by being strict about one thing: **every piece +of state, every open OS handle and every piece of running work has exactly one owner, an object, and objects only +declare what they need.** Chord itself activates, connects, reloads and ends them. Because Chord knows every timer, +watcher, transaction and reference an object has, it can swap the object's code while its data stays, restart only +what touches the operating system, and route calls between browser tabs, the server and per-session worker processes +without the plugin author writing any of that. + +This guide teaches Chord v2 by building an app called **Studio**: a small version of p2 (Pi Mobile), with a server, +one worker process per session, browser tabs as clients, and an agent from pi-durable on each worker. The first +chapters grow one plugin, `@studio/notes`, until it uses every concept; the last three chapters assemble the rest of a +p2-like app from the same parts. + +:::review | Review: what exists today +Nothing in this guide runs yet except the contract layer. `packages/chord/src/next/contracts/` (L1 part 1: `remote`, +`local`, `method`, `evolve`, `checkAccepts`, the schema helpers) is built and reviewed. The rest of L1 (inputs, +argument checks, principals, errors) is a draft spec (`packages/chord/docs/next/layer-1.md`). The frozen kernel in +`src/next/*.ts` implements an older version of the design (host values, a development freeze, JSONL storage, +descriptions on the wire); do not read it as the target. Build on the design, not on the frozen code. +::: + +## Ten things to know + +:::findings +1. **An app is a tree of hosts.** One server, any number of workers it starts on demand, any number of clients (tabs, + TUIs) that connect to it. Clients reach workers only through the server, which relays. +2. **Objects are the unit of everything.** An object type is a contract (name, input, placement, members, version); an + object is one instance per input and caller. The first `get` creates it, later gets share it, the last release ends + it. +3. **Factories declare, Chord activates.** A factory is synchronous: it declares documents, references, derived + values, OS handles, routines and jobs through `self`, and returns the members. Chord opens, binds and starts them, + and undoes them in reverse. +4. **Links stamp who is calling.** A caller's data never says who it is; the link it arrived on does. Each caller gets + its own object and the variant of members it may use. +5. **Shared data lives once, behind views.** A `local` shared store holds the data; per-caller `remote` views expose + what each caller may see. Only stable identities persist documents. +6. **State changes only in transactions.** Documents are JSON values written in `self.tx`; commits are atomic for + every viewer and travel as small operations, never as whole values. +7. **Failures look like disconnects.** A reference survives link drops, restarts and failures: calls park or fail, + values stay readable, and Chord retries by itself. There are no request ids. +8. **Everything long-lived has a declared form.** Standing behaviour is a routine, work that outlives a call is a job, + input from a person is a flow, behaviour from other plugins comes through a registry. Nothing else may hold a timer + or a callback. +9. **Reload swaps pure objects in place and restarts native ones.** Objects without OS handles get the new code with + their data and running work intact; objects with OS handles close and reopen them. A broken change leaves the whole + host on its old code. +10. **pi-durable runs inside a core object.** The agent harness is a host-lifetime object on each worker; models, + environments and tools reach it through registries, and tools reach Chord through `chord(fn, context)`. +::: + +## Words used throughout + +A **host** is one participant of the app: a process, a browser tab or a Durable Object, identified by a logical +identity and an **incarnation** that changes on every start. A **link** connects two hosts. The **server** is the one +addressable root host; a **worker** is a host the server starts for one session; a **client** is a tab or TUI, not +addressable. A **plugin** is a directory with a `package.json`; its code for one kind of host is a **facet**. **Base +code** is the app's own code (Chord calls its facet the core facet); it restarts the hosts when it changes, while +plugins reload. An **object type** (or **contract**) describes objects; an **object** is one instance; its **input** is +its key among objects of that type, and its **placement** computes from the input on which host it lives. A **factory** builds an +object; **`self`** is what a factory declares through. A **principal** is who calls, stamped by the link; a +**variant** is the set of members a principal gets. A **reference** is what `get` returns. A **readable** is a value +plus its ordered changes. A **document** is a named JSON value an object owns, written in a **transaction**. A +**derive** is a value computed from readables. A **native** is an OS-level value (a socket, a watcher, a process) +opened with `self.open`; an object with natives is a **native object**, one without is a **pure object**. A +**routine** is standing behaviour, a **job** is owned work that outlives its call, a **flow** is input into running +work. A **registry** is an owner's extension point, filled with **contributions**. **`ctx`** is what every call +receives: cancellation, caller, call-scoped references. A **reload** swaps a plugin's code inside running hosts; a +**restart** ends a process; a **revision** names the complete code set the hosts run. + +:::note | Reading the code in this guide +Imports are shown once per file and then omitted. `obj`, `str`, `arr` and friends are Chord's schema helpers (they +return ordinary TypeBox schemas). Helper functions that belong to the app (`authenticate`, `watchDir`, `uuidv7`) are +named and not shown. Where the design gives no name, the guide says it uses a working name. +::: + +# The Shape of an App {#shape lede="Which processes a Chord app has, how they find each other, and the start code that connects them."} + +Before any plugin exists, Studio needs processes and connections between them. In Chord each participant is a host, +and the app's own start code creates every host, connects them and decides how workers start. Chord deliberately does +none of that itself: a laptop app forks child processes, a test runs everything in one process, a Cloudflare +deployment routes through the edge. This chapter writes Studio's start code: `server.ts`, `worker.ts`, the browser +entry, and a test setup. + +## Three roles + +A Node app on Chord has one shape: + +:::flow wide | The three roles. Clients connect to the one server; the server starts workers and relays between them. +row "Clients: not addressable" muted +box tab1 "browser tab" "owner, local" +box tab2 "browser tab" "guest Ann" +box tui "TUI" "owner" +row "Server: exactly one" accent +gap +box server "server" "`Server()`" accent +gap +row "Workers: one per session, on demand" blue +box w1 "worker" "`Session({ id: \"S1\" })`" blue +box w2 "worker" "`Session({ id: \"S2\" })`" blue +box w3 "worker" "`Session({ id: \"S3\" })`" dashed +arrow tab1 -> server "WebSocket" +arrow tab2 -> server "WebSocket" +arrow tui -> server "WebSocket" +arrow server -> w1 "IPC" +arrow server -> w2 "IPC" +note "A worker for S3 does not exist until something asks for an object that lives there." +::: + +| Role | Addressable | Who it is comes from | Lifecycle | +| --- | --- | --- | --- | +| client | no | the authentication of its link | connects by itself | +| server | yes, exactly one | its own declaration | runs | +| worker | yes, many, by a typed address | stamped by the server, its parent | started on demand by the app | + +The **server** holds what all sessions share (the session list, access, settings) and starts workers. A **worker** +runs one session: its agent, its files, its terminal connections. **Clients** are browser tabs and TUIs. Nobody can +send a request *to* a client, so a client can only serve objects that nobody outside it needs; chapter IV names them +`local` types. + +Things that look like hosts but are not: a code mode sandbox (a worker thread inside a worker, owned by an object), a +remote environment's daemon (it speaks its own protocol, owned by an object), the TUI's sandbox behind its launcher. +A detached process such as a terminal host could become another kind of child of the server. + +## Host kinds and addresses + +Every addressable host has a typed address. The app declares its **host kinds** once, in base code: + +```ts +// studio/src/hosts.ts +import { hostKind, obj, str } from "@earendil-works/chord/next"; + +export const Server = hostKind("server"); +export const Session = hostKind("session", obj({ id: str() })); + +Server(); // the address of the server +Session({ id: "S1" }); // the address of the worker of session S1 +``` + +There are no host-name strings anywhere in app code. Chord serializes addresses internally; your code only calls the +kinds. An object type names its host with a placement function from its input to an address (chapter II), so the +type checker knows that `Session({ id: session })` needs a `session` field. + +## Links + +A **link** is an ordered, two-way channel of messages between two hosts; it only delivers. Chord ships three: a memory +pair for tests, IPC for a Node child process forked with `serialization: "advanced"`, and WebSocket. All three carry +the same binary message format and the same flow control (chapter XVII). Recovery policy belongs to the link adapter: +IPC needs no heartbeat, a WebSocket through a relay needs one, because TCP does not notice that a laptop went to sleep. + +Each link has a **direction**. A host has at most one link to its parent (`connect`) and any number of links from +below (`accept` for clients, `route` for workers). Requests arriving on a link act as the principal the link was given; +chapter III explains why that matters. + +## The server + +```ts +// studio/src/server.ts +import { createHost, ipcLink, webSocketLink } from "@earendil-works/chord/next"; +import { fork } from "node:child_process"; +import { WebSocketServer } from "ws"; +import { Server, Session } from "./hosts.ts"; +import { coreServerFacet } from "./core/server.ts"; +import type { P } from "./principal.ts"; + +const server = createHost

({ + address: Server(), + principal: { kind: "server" }, // who this host's own code acts as + storage: await openServerStorage(dataDir), // persisted documents (chapter V); memory if omitted + onError: (e) => { if (e.kind === "storage_failed") process.exit(75); else log(e); }, +}); + +await server.load([ + { name: "@studio/core", facet: coreServerFacet(config) }, // base code, under a fixed name + ...(await loadPlugins("server")), // [{ name, facet }] from the plugin loader +]); + +const wss = new WebSocketServer({ server: http, path: "/chord", maxPayload: 64 * 1024 * 1024 }); +wss.on("connection", (ws, req) => { + const principal = authenticate(req); // app code: cookie, loopback, device token + if (!principal) return ws.close(4401); + server.accept(webSocketLink(ws), { principal }); +}); + +server.route(Session, { + authorize: (p, { id }) => + p.kind === "owner" || p.kind === "server" || (p.kind === "guest" && p.session === id), + resolve: async ({ id }, { signal }) => { + const principal: P = { kind: "worker", session: id }; + const child = fork("worker.ts", [], { serialization: "advanced", signal }); + child.send({ id, principal, init: await initFor(id, signal) }); // the app's own start message + return { link: ipcLink(child), principal }; // Chord stamps `principal` on it + }, +}); +``` + +Each call has one job: + +| Call | Meaning | +| --- | --- | +| `createHost

({ address?, principal?, ...options })` | with an address: an addressable host whose own code acts as `principal`; without: a client. `P` is the app's principal type | +| `load([{ name, facet }])` | the facets this host runs, each with its plugin's `package.json` name; duplicates and invalid names are refused. Base code passes its core facet the same way | +| `accept(link, { principal })` | a link from a client; every request on it acts as `principal` | +| `route(Kind, { authorize, resolve })` | how to reach hosts of a kind. `authorize(principal, address)` runs before anything starts; `resolve(address, { signal })` returns `{ link, principal }` | +| `connect(link, { principal })` | the link to the parent, at most one, replaced after a reconnect; `principal` says who the parent is | +| `root(fn, { principal?, signal? })` | a call scope for code outside objects (a CLI command): gives `fn` a `ctx` (chapter VI) | + +Chord has no `listen`. Entry points differ too much: a loopback WebSocket server, a tunnel, the Radius relay +delivering channels, a Durable Object edge, a memory link in a test. Each entry point authenticates and calls +`accept`. Radius, for example, is one more line: + +```ts +radius.onChannel((ws) => + server.accept(webSocketLink(ws), { principal: { kind: "owner", access: "relay" } })); +``` + +## The worker + +```ts +// studio/src/worker.ts +const { id, principal, init } = await startMessage(process); // app code: the first IPC message +const worker = createHost

({ + address: Session({ id }), + principal, // { kind: "worker", session: id } + storage: await openSessionStorage(init.dataDir, id), // one storage per session +}); +await worker.load([ + { name: "@studio/core", facet: coreWorkerFacet(init) }, // serves the start data (chapter IX) + ...(await loadPlugins("worker")), +]); +worker.connect(ipcLink(process), { principal: { kind: "server" } }); +``` + +## A browser tab + +```ts +// studio/web/main.tsx +const client = createHost

(); // no address: a client +await client.load(await loadPlugins("web")); +client.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }); +const notes = await client.get(Notes, { session: "S1" }); // tab → server → worker S1 +``` + +## The first `get`, step by step + +The tab asks for the notes of S1. No worker for S1 runs yet. + +:::sequence animate | The first get for an object on a worker that is not running. The server authorizes, the app starts the worker, Chord stamps the link. +actor tab "browser tab" "owner" +actor server "server" accent +actor app "route(Session)" "app code" muted +actor worker "worker S1" blue +tab -> tab: placement: Session({ id: "S1" }) +tab -> server: hold Notes { session: "S1" } +server -> server: not mine: route(Session) +server -> app: authorize(owner, { id: "S1" }) +app --> server: true +server => app: resolve({ id: "S1" }) +app -> worker: fork + start message +worker -> worker: createHost, load, connect +app --> server: { link, principal: worker S1 } +server => worker: hold (stamped: owner) +worker -> worker: placement check, run factory +worker --> server: bound: snapshot +server --> tab: bound: snapshot +::: + +`resolve` runs once per address while its link lives. When the link drops (the worker crashed or the supervisor +stopped it) and references to that worker still exist, Chord calls `resolve` again by itself, with a per-address +backoff, so a tab that only watches recovers without a click. A worker start that already produced a link finishes +even if the request that triggered it went away. + +**Incarnations** make restarts visible. Chord mints a fresh id (a uuidv7) on every host start and sends it in the link +handshake; the other side sees a restart as a new incarnation of the same identity, and references to it re-hold +(chapter VI). + +## Tests: everything in one process + +The same code runs in a test with memory links. `resolve` builds the worker host in memory and returns one end of the +pair: + +```ts +server.route(Session, { + authorize: () => true, + resolve: async ({ id }) => { + const [up, down] = memoryLinkPair(); + const principal: P = { kind: "worker", session: id }; + const worker = createHost

({ address: Session({ id }), principal }); + await worker.load([{ name: "@studio/core", facet: coreWorkerFacet({ id, dataDir }) }, notesWorker]); + worker.connect(up, { principal: { kind: "server" } }); + return { link: down, principal }; + }, +}); +``` + +Memory links deliver the same encoded messages asynchronously, so a test runs the production protocol path. + +## What stays app code + +Starting processes, the start message, authentication, supervising and stopping idle workers are app code. Chord +provides one readable for the last one: `host.activity`, with `{ references, calls, jobs, lingering }` (viewers, calls +in flight, unsettled jobs, objects kept warm after their last reference). Studio's core reports it to the server +together with the agent's running tasks, and the server's supervisor stops a worker when everything has been zero for +five minutes, as p2 does. Chord counts; it has no idle policy. + +## Host options and timeouts + +`createHost` takes the options introduced later in this guide: `storage` (V), `principalKey` and `revokedBy` (III, +VII), `telemetry` (XVII) and `onError`. `onError` receives background errors and `{ kind: "storage_failed" }`; every +app must handle the latter, because Chord never ends the process. Studio exits with its own code so its supervisor +restarts it. + +Timeouts are host defaults with per-thing overrides; the full table is in the appendix. The two you meet first: +`readyTimeout` (60 s: how long an activation and an awaited `get` wait) and the park deadline (60 s: how long a call +waits through a link drop or a restart before failing `unavailable`). + +:::note | Built so far +Nothing in this chapter is built in Chord v2 yet: hosts, links and routes are layers L6 and L7. The frozen kernel has +working versions with the same shape (`createHost`, `connect`, `accept`, `route`, `load`), which the rebuild +simplifies. +::: + +# A First Object {#first-object lede="A plugin, a contract, a factory, and a browser tab that calls it on another process."} + +Studio's first feature is session notes: a list of short notes per session, shown in a panel, that anyone allowed can +add to. Several places need "the notes of S1": the owner's tab, a guest's tab, later a download route and an agent +tool. If each built its own copy, the copies would drift apart and nobody would know when to clean them up. In Chord +there is **one object** for "the notes of S1", everyone asks for it by its type and input, and Chord routes them all to it. + +## The plugin's files + +A plugin is a directory with a `package.json`. Its `name` (npm style, globally unique and stable) is the plugin's +identity: Chord finds its stored documents, reports its errors and orders its contributions by it, never by a +directory name or load order. + +| File | Runs on | Holds | +| --- | --- | --- | +| `package.json` | read by the loader | `"name": "@studio/notes"` | +| `contract.ts` | every host that imports it | object types: names, shapes, method signatures; no running code | +| `worker.ts` | every session worker | the worker facet | +| `server.ts` | the server | the server facet (later: the download route) | +| `web.tsx` | browser tabs | the web facet: the panel | + +## The contract + +An object type is a contract: a name, an **input** (the object's key among objects of this type), a **placement** +(on which host it lives, computed from the input), the **members** (state you can read and methods you can call) and a **version**. + +```ts +// notes/contract.ts +import { arr, int, method, obj, remote, str, type Static } from "@earendil-works/chord/next"; +import { Session } from "@studio/core/hosts"; + +export const Note = obj({ id: str(), text: str({ max: 2000 }), by: str(), at: int() }); +export type Note = Static; + +export const Notes = remote("notes", { + version: 1, + input: obj({ session: str() }), // the key: { session: "S1" } + placement: ({ session }) => Session({ id: session }), // input → the host's address + variants: { + editor: { + state: { notes: obj({ items: arr(Note) }) }, + methods: { + add: method(obj({ text: str({ min: 1, max: 2000 }) }), str()), // returns the new id + remove: method(obj({ id: str() })), // no result + }, + }, + }, +}); +``` + +`remote` means objects of this type can be reached from any host, across links (chapter IV introduces `local`). The +schema helpers (`obj`, `str`, `int`, `arr`, `map`, `opt`, `lit`, `oneOf`, `union`, `bool`, `num`, `nul`, `bytes`, +`unknown`) return ordinary TypeBox schemas; short options such as `{ min, max }` bound lengths or values. Schemas must +describe data: no callbacks, no class instances, no custom codecs. + +`method()` has four forms: `method()` (no arguments, no result), `method(args)`, `method(args, result)` and +`method(undefined, result)`. + +A contract contains no code that runs, so every host can import it. When you declare it, Chord materializes its +description and computes a **fingerprint**, a hash of its input, variants and members. Two hosts compare contracts by +version and fingerprint (chapter XVI). Placement is not part of the fingerprint. + +## Placement + +The input says *which* object; `placement` says *where* it lives. The caller runs the placement function on the input and sends the resulting address with its request; the server +routes by it. The provider runs the function too and refuses an object that is not its own with `invalid/placement`. +So a worker for S2 never builds "Notes for S1", even if a caller has an outdated rule. Because placement is a typed +function, a `Session` placement on a type whose input has no `session` field does not compile. + +Inputs are **normalized** before anything else: defaults are applied, unknown fields removed, and a canonical key +computed. `{ session: "S1" }` and `{ session: "S1", extra: 1 }` are the same object. + +## The facet and the factory + +A **facet** is a plugin's setup for one kind of host. It runs once when the plugin loads on that host and tells the +host which object types it serves: + +```ts +// notes/worker.ts +import { defineState } from "@earendil-works/chord/next"; +import { defineFacet } from "@studio/core"; // Studio's defineFacet: principals typed as Studio's P (chapter III) +import { Note, Notes } from "./contract.ts"; + +const NoteList = defineState("notes.list", { // a document (chapter V); in memory for now + schema: obj({ items: arr(Note) }), + version: 1, + initial: () => ({ items: [] }), + checkpointWhen: () => true, +}); + +export default defineFacet((host, self) => { + host.serve(Notes, ({ session }, principal, self) => { + const list = self.state(NoteList); // THE note list of this object + return { + variant: "editor", + notes: list, // a document handle is a readable: expose it as state + add: ({ text }, ctx) => + self.tx(async (tx) => { + const id = uuidv7(); + (await tx.draft(list)).items.push({ id, text, by: "someone", at: Date.now() }); + return id; + }, ctx), + remove: ({ id }, ctx) => + list.change((d) => { d.items = d.items.filter((n) => n.id !== id); }, ctx), + }; + }); +}); +``` + +Read `host.serve(Notes, factory)` as "this host builds `Notes` objects like this". The factory receives the +normalized input, the caller's principal (chapter III) and `self`, through which it declares what the object needs. +It returns the members of one variant, tagged with `variant`. + +**Factories declare; Chord activates.** The factory is synchronous and has no side effects. It does not await, open +files or start timers. `self.state(NoteList)` does not load anything: it declares that the object has this document +and returns a **handle** that stays valid for the object's whole life. Chord then performs the activation: it loads the +document, binds references, opens OS handles, starts routines, and only then lets calls in. The rule exists for +reload: a reload runs the factory again, and anything the factory did itself would happen twice. + +## Calling it + +```tsx +// notes/web.tsx +export function NotesPanel({ session }: { session: string }) { + const notes = useRef(Notes, { session }); // app-side hook over host.get / release + const items = useReadable(notes?.notes)?.items ?? []; + return ( +

+ {items.map((n) =>

{n.text}

)} + notes?.add({ text })} /> +
+ ); +} +``` + +Underneath the hooks is the reference API: + +```ts +const notes = await client.get(Notes, { session: "S1" }); // live and hydrated once awaited +notes.notes.value.items; // the current list +notes.notes.subscribe((v) => render(v.items)); // every committed change +const id = await notes.add({ text: "check the flaky test" }); +notes.release(); // the object may end now +``` + +An awaited reference is **live and hydrated**: every state member has a value. State members are **readables**: +`.value` is the current value, `subscribe` is called after every commit. The first `get` for `{ session: "S1" }` runs +the factory; every later `get` for the same input and caller shares that object. The object ends when the last +reference is released (a type may keep it a little longer with `linger`). + +:::sequence animate | A call to add. Arguments are checked once at the provider; the commit reaches the viewer as one small change. +actor tab "browser tab" +actor server "server" muted +actor worker "worker S1" blue +actor obj "Notes { S1 }" accent +tab -> server: call add { text } +server -> worker: relay (header only) +worker -> obj: strip + validate args +obj => obj: self.tx: push note, commit +obj --> worker: result: id +worker --> server: state: ops of the commit +worker --> server: result +server --> tab: state, then result +::: + +## Two rules for members + +**Methods return data, never objects.** A method that wants to point you to another object returns its address +(`{ session: "S2" }`), and you `get` it yourself. That keeps every reference owned by someone who releases it. + +**No callbacks as arguments.** A callback carries code, and everything it closes over, from one object into another. +Suppose an inbox object passed `onFile` to another object, which stored it. After a reload of the inbox's plugin, the +other object still calls the *old* `onFile`; after the inbox ends, it calls into an object whose file watcher is +already closed. Chord forbids this by construction: members are data (JSON plus bytes). Behaviour from elsewhere goes +through registries (chapter XIII), multi-step work through jobs (XI), input from a person through flows (XII). + +## What is wrong with this version + +This first version works for one person. It has two problems, which the next two chapters fix. Everyone gets the +`editor` variant, including a read-only guest. And, less obvious, the owner and a guest do not see the same notes: as +chapter III shows, a remote object exists per caller, so each caller's `Notes` object has its own `NoteList`. + +:::note | Built so far +`remote`, `method`, the schema helpers, `Static` and fingerprints are built (`src/next/contracts/`). Serving, `get`, +references and documents are layers L3 to L5. +::: + +# Who Is Calling {#principals lede="Principals: how Chord knows who calls, why a caller cannot lie about it, and how the answer picks the object and its members."} + +The owner may add and remove notes. Ann, a read-only guest, may only read them and should not even see who wrote +what. The object must therefore know who is calling, and the caller must not be able to claim to be someone else. In +Chord the answer is a **principal**, a small JSON value the app defines and the links stamp. + +## The app's principal type + +Chord defines no principal kinds; the app does. Every variant has a `kind`. Studio uses p2's set: + +```ts +// studio/src/principal.ts +export type P = + | { kind: "owner"; access: "local" | "relay" | "device"; device?: string } + | { kind: "guest"; session: string; mode: "read" | "write"; grant: string; revision: number } + | { kind: "server" } + | { kind: "worker"; session: string }; +``` + +| `kind` | Who | +| --- | --- | +| `owner` | the person running the app, on loopback, through the relay, or on a paired device | +| `guest` | someone invited into one session, with a mode and the grant (and its revision) that let them in | +| `server` | the server's own code calling a worker | +| `worker` | a session worker calling the server | + +## Links stamp principals + +A principal never comes from a request's data. The server authenticates a client once, when its WebSocket connects, +and stamps the result on the whole link (`server.accept(link, { principal })`). The server tells Chord who each worker +is when it resolves it (`return { link, principal }`). When the server relays a client's request to a worker, Chord +writes the client's principal into the forwarded request; a principal field coming from a client is always +overwritten, and only a trusted parent link may carry one. + +Where principals come from is app code. Studio's sharing feature creates a share link per session and mode; a guest +who opens it gets a guest record (grant `g1`) and a cookie; `authenticate(req)` turns the cookie into +`{ kind: "guest", session: "S1", mode: "read", grant: "g1", revision: 1 }` on every new connection. Chord never sees +cookies or grants, only the finished principal. + +## The caller is part of the identity + +A `remote` object exists once per **(type, input, principal key)**. The **principal key** is a string computed from +the principal; by default its canonical JSON (sorted keys). The app may pass its own `principalKey` to `createHost`, +and must pass the same function on every host, or identities split silently. + +```ts +const principalKey = (p: P) => + p.kind === "owner" ? `owner:${p.access}:${p.device ?? ""}` + : p.kind === "guest" ? `guest:${p.grant}:r${p.revision}` + : p.kind === "worker" ? `worker:${p.session}` + : p.kind; +``` + +The design wants the key to include how the client is connected and the grant's revision, so that two access paths +never share objects and a changed grant never reuses the old grant's object: + +| Caller | Principal key | `Notes { S1 }` object | +| --- | --- | --- | +| owner, tab A, local | `owner:local:` | 1 | +| owner, tab B, local | `owner:local:` | 1 (shared with tab A) | +| owner through the relay | `owner:relay:` | 2 | +| guest g1, revision 1 | `guest:g1:r1` | 3 | +| guest g1 after a change to write mode, revision 2 | `guest:g1:r2` | 4 (new) | + +## Variants + +The contract lists named sets of members; the factory receives the principal and returns the variant that principal +may use: + +```ts +// notes/contract.ts (version 1, final form for now) +variants: { + editor: { + state: { notes: obj({ items: arr(Note) }) }, + methods: { add: method(obj({ text: str({ min: 1, max: 2000 }) }), str()), remove: method(obj({ id: str() })) }, + }, + reader: { + state: { notes: obj({ items: arr(PublicNote) }) }, // no `by`: authors are hidden + }, +}, +``` + +```ts +// notes/worker.ts +host.serve(Notes, ({ session }, principal, self) => { + if (canWrite(principal, session)) return { variant: "editor", notes: …, add: …, remove: … }; + return { variant: "reader", notes: … }; +}); +``` + +A reader has no `add` at all. Chord refuses a call to a member that is not in the caller's variant, so `add` needs no +check of its own. Three rules keep variants compatible: a member name has the same kind (state or method) in every +variant that has it; a method has the same arguments and result in every variant; only state schemas may differ per +variant (the reader's notes have no authors). `remote(...)` enforces them at the type level, and at runtime for +contracts that arrive as data. + +## How a facet knows `P` + +`P` is Studio's type, not Chord's, and every host is created with it (`createHost

`). A plugin's facets are written +and compiled separately, then handed to `host.load` as values, so the type checker cannot infer `P` from the host into +the factory's `principal`. Studio's base code therefore exports its own `defineFacet`, already bound to `P`, and every +facet of every plugin imports that one instead of Chord's: + +```ts +// studio/src/facet.ts (base code, exported as @studio/core) +import { bindFacet } from "@earendil-works/chord/next"; +import type { P } from "./principal.ts"; + +export const defineFacet = bindFacet

(); // a facet's principals are Studio's P +export type { P, Revocation } from "./principal.ts"; +``` + +```ts +// notes/worker.ts (server.ts, web.tsx and tui.ts import the same defineFacet) +import { defineFacet } from "@studio/core"; + +export default defineFacet((host, self) => { + host.serve(Notes, ({ session }, principal, self) => { // principal: P + if (principal.kind === "guest" && principal.mode === "read") return { variant: "reader", notes: … }; + … + }); +}); +``` + +The import is cheap: `@studio/core` is a host package plugins already import at runtime (the contract imports `Session` +from it), `P` is a type-only import, and `bindFacet` is a one-line function. The host checks the pairing: `load` +accepts only facets typed for its own `P`, so a facet compiled against another app's principal type, or one built with +Chord's untyped `defineFacet`, does not compile into Studio's start code. Client facets (`web.tsx`, `tui.ts`) use the +same `defineFacet`; they rarely see a principal, because clients serve only `local` types. + +None of this is checked at runtime, and none of it needs to be: the principal a factory receives is the one the link +stamped. + +:::review | Review: binding facets to the app's principal type (open) +The design shows `defineFacet((host, self) => …)` without saying how `P` reaches the factory. The frozen kernel takes +it as an explicit type parameter, `defineFacet

(…)`, defaulting to `{ kind: string }` +(`packages/chord/src/next/facet.ts`), and the playground writes `defineFacet(…)`. The pre-bound +`defineFacet` above (`bindFacet` is a working name) is the owner's preferred direction, recorded as an open point in +`packages/chord/docs/next/layer-1.md` §4. +::: + +## Narrowing on the caller's side + +Owner and guest run the same `web.tsx`, and which variant they get is known only at runtime. The types force every +caller to narrow before using variant members: + +```ts +const notes = await client.get(Notes, { session: "S1" }); +notes.add({ text: "hi" }); // type error: `add` exists only in "editor" +if (inVariant(notes, "editor")) await notes.add({ text: "hi" }); +expectVariant(notes, "editor"); // or assert (throws) +function Editor({ notes }: { notes: RefIn }) { … } +``` + +The protection is not the type checker; it is Chord refusing the call at the provider. A client that ignores the +types gets the same error. + +## Admission before a worker starts + +Principals also decide whether a caller may reach a host at all. `route(Session, { authorize })` runs before Chord +resolves an address, so a guest of S1 asking for anything in S2 is refused at the server, and no worker starts for +them. Finer authority lives in the per-principal view's methods; there is no asynchronous "may this caller activate +this object" guard. An object whose address carries authority (say, a login flow by id) may exist; what a caller can +see and do is decided by the variant its factory picks. + +## Inside methods + +`ctx.principal` is the caller of the current call. In Studio's notes the editor's `add` uses it to record the +author: + +```ts +add: ({ text }, ctx) => store.add({ text, by: authorOf(ctx.principal) }, ctx), +``` + +Where principals act, in summary: + +| Purpose | Where | +| --- | --- | +| identity: which object a caller gets | principal key | +| variant: which members | the factory | +| admission: whether a host is reachable | `route.authorize` | +| attribution and fine checks | `ctx.principal` in methods | +| revocation | `revokedBy` (chapter VII) | + +## The bug from chapter II + +Now the bug is visible. The owner's `Notes { S1 }` and Ann's `Notes { S1 }` are two objects (rows 1 and 3 above), and +each declared its own `NoteList`. The owner's notes are not in Ann's list. Per-caller objects must not hold the data; +they must be views over something that exists once. That is the next chapter. + +# One Store, Many Views {#shared-store lede="Local types, and the pattern every Chord plugin uses: shared data once in a store, per-caller views in front of it."} + +Every caller has its own `Notes` object, which is what makes per-caller variants possible. But data must exist once. +The fix is the most common pattern in Chord: a **shared store**, a `local` object that holds the data and does the +heavy work once, behind thin per-principal **views**, the `remote` objects callers get. + +## `local` types + +```ts +// notes/contract.ts +export const NoteStore = local("notes.store", { + version: 1, + input: obj({ session: str() }), + members: { + notes: obj({ items: arr(Note) }), // state: a schema + public: obj({ items: arr(PublicNote) }), + add: method(obj({ text: str(), by: str() }), str()), // methods: method(...) + remove: method(obj({ id: str() })), + }, +}); +``` + +A `local` contract has flat `members` and no variants. The differences: + +| | `remote` | `local` | +| --- | --- | --- | +| reachable from | any host, across links | its own host only | +| identity | (type, input, principal key) | (type, input) | +| principals, variants | yes | no | +| factory | `(input, principal, self)` | `(input, self)` | +| arguments | stripped to the provider's schema and validated, for every caller | see the review note below | +| values on the same host | passed by reference, never copied | passed by reference, never copied | +| `ctx.principal` | the caller | none | +| may persist documents | no (chapter V) | yes | + +A local object has no principal: it is trusted code on its own host, acting for the host. So attribution ("who wrote +this note") is passed as data (`by`), by the view that knows who called. Clients serve only local types, because +nobody can address a client. + +## The pattern + +```ts +// notes/worker.ts +export default defineFacet((host, self) => { + host.serve(NoteStore, ({ session }, self) => { + const list = self.state(NoteList); + const pub = self.derive("public", [list], ([l]) => ({ items: l.items.map(stripAuthor) })); + return { + notes: list, + public: pub, // computed once, for every reader + add: ({ text, by }, ctx) => + self.tx(async (tx) => { + const id = uuidv7(); + (await tx.draft(list)).items.push({ id, text, by, at: Date.now() }); + return id; + }, ctx), + remove: ({ id }, ctx) => list.change((d) => { d.items = d.items.filter((n) => n.id !== id); }, ctx), + }; + }); + + host.serve(Notes, ({ session }, principal, self) => { + const store = self.get(NoteStore, { session }); // a reference, declared: held for the view's life + if (canWrite(principal, session)) return { + variant: "editor", + notes: store.notes, // the store's readable, exposed as-is + add: ({ text }, ctx) => store.add({ text, by: authorOf(principal) }, ctx), + remove: (args, ctx) => store.remove(args, ctx), + }; + return { variant: "reader", notes: store.public }; + }); +}); +``` + +:::flow wide | One store, three views. The redaction runs once in the store; both readers read the same value. +row "Per-principal views (remote)" blue +box ve "Notes { S1 }" "owner · editor" blue +box vr1 "Notes { S1 }" "guest Ann · reader" blue +box vr2 "Notes { S1 }" "guest Bob · reader" blue +row "Shared store (local)" accent +gap +box store "NoteStore { S1 }" "notes, public (a derive)" accent +gap +arrow ve -> store "self.get" +arrow vr1 -> store "self.get" +arrow vr2 -> store "self.get" +note "Authority is the view's: the reader variant has no add. The store does not check; it is trusted and acts for the host." +::: + +`self.get(NoteStore, { session })` in a factory declares a **reference** that the view holds for its whole life; Chord +binds it during activation and waits until the store is live (chapter VI). The view exposes the store's readables +directly. Owner, Ann and Bob now give three `Notes` objects, one `NoteStore`, one note list and one redacted list. + +Any heavy or shared part belongs in a store like this: a pi-durable watch, a file watcher, a sandbox, a search index. +Base code uses the same pattern with the same primitives; nothing about it is special to core. + +## Derived values, briefly + +`self.derive(name, sources, fn)` computes a value from readables and recomputes it when a source changes; viewers get +only the difference. The reader's list above is one. Chapter VIII covers derives in depth, including the one footgun +that matters: `stripAuthor` must return the *same* object for an unchanged note, or every commit costs the size of the +whole list. The tool for that is `mapShared`. + +## Arguments are checked at the provider + +Trusted code is not type-checked code: plugins are bundled with esbuild without type checks, and agents write +JavaScript plugins. So the provider checks arguments itself (decision R7): every method's arguments are stripped to +the **provider's own** schema (unknown fields removed) and validated with it, whoever the caller is, and never with a +schema a peer sent. Schemas are compiled lazily on first use and cached per schema object. Bytes are checked as leaves +(`instanceof Uint8Array` plus an optional length bound), never walked. Stripping copies only when a field was removed; +the caller's value is never mutated (proposal Q12, in the L1 draft). Results and state are not validated: they come +from the provider. + +On the same host nothing is copied: arguments and results are passed by reference. Values are **immutable by +convention** (Delta's rule, as in Durable): nobody mutates a value it received, and Chord neither copies nor freezes +values to enforce it. There is no development mode that would freeze them; one behaviour everywhere (R9). + +:::review | Review: are local arguments validated? (Q15, open) +Design §5's table says `local` arguments are "neither validated nor copied"; §3 and R7 say arguments are validated +"for every caller". Review round 3 raised this as Q15 (apply R7 to `local` too, or keep `local` as the one exception), +and `layer-1.md` lists it as open. Until it is decided, write local methods so that they do not depend on either +answer: trust the shape, but never trust content (lengths, ids) that came from a remote caller without the view +checking it. `chord-next-walkthrough.md` concept 6 still describes the older rule ("validated only when the caller is +not a host principal"), which R7 replaced. +::: + +## Choosing between the two + +Use a `remote` type for what callers address: views, things with per-caller authority. Use a `local` type for what +exists once per input on one host: stores, caches, owners of OS handles, and the core objects a host provides +(chapter IX). If a type is reached across a link, it is remote; if a type persists documents, it is local (or a +host-lifetime object, chapter V). + +# Writing It Down {#documents lede="Documents and transactions: how an object stores data, changes it atomically, survives restarts and migrates it."} + +Notes must be stored, changed safely while others read them, sent to viewers as small changes, and survive a worker +restart. An object's data lives in **documents**: named JSON values the object owns, changed only in +**transactions**. The design is a port of pi-durable's store, so most rules will look familiar; this chapter states +them for Chord. + +## Defining a document + +```ts +// notes/worker.ts +const NoteList = defineState("notes.list", { + schema: obj({ items: arr(Note) }), + version: 1, + initial: () => ({ items: [] }), + persist: true, // stored; default: memory only + checkpointWhen: (_value, _ops, info) => info.deltasSinceBase > 64, +}); + +const Draft = defineState("notes.draft", { // one unfinished note per author + schema: obj({ text: str() }), + version: 1, + initial: () => ({ text: "" }), + keyed: true, // many per object, by key + persist: true, + checkpointWhen: () => true, // small: a full base per write +}); +``` + +`checkpointWhen` is required on every definition, with no default (R14, as in Durable since pi `5c1736f82`): it says +when a commit stores a complete base instead of a delta, which bounds how much is replayed on load. Small documents +checkpoint on every write; growing ones after a number of deltas. + +A document's root is always a JSON object (lists live in a field). Documents hold JSON only; bytes are a value and wire +type, not a document type. + +## Singleton and keyed + +```ts +host.serve(NoteStore, ({ session }, self) => { + const list = self.state(NoteList); // THE note list of this object + const drafts = self.state(Draft); // all drafts of this object + … +}); +``` + +| | singleton (no `keyed`) | keyed | +| --- | --- | --- | +| how many | one per object | many per object, by a short key | +| loaded | at activation, kept in memory | on demand | +| read | `list.value` (synchronous) | `await drafts.get(key)`: the value or `undefined` | +| list | (it is one value) | `for await (const key of drafts.keys())` pages keys | +| expose as state | yes: the handle is a readable | no: expose a method or a singleton index | + +There is no live key list. A collection shown live is a singleton (an object holding a list or a map); a large +collection is a method returning pages. + +## Transactions + +All writes go through `self.tx(fn, ctx)`. Inside, `await tx.draft(handle, key?, { tags? })` returns a draft of the +current document, or a draft over `initial()` if it is absent. Drafts record operations (assignments, `push`, +`Object.assign`); asking for the same document twice in one transaction returns the same draft. `tx.retire(handle, +key?)` removes a document. There is no `tx.get`: reading is drafting, and a draft is an overlay over the current +value, so nothing is copied. A draft you do not change commits nothing, so reading never stores an absent document. + +```ts +saveDraft: ({ author, text }, ctx) => + self.tx(async (tx) => { (await tx.draft(drafts, author)).text = text; }, ctx), +``` + +Shorthands for one document: `handle.change(fn, ctx)` (keyed: `handle.change(key, fn, ctx)`) (loads the document, then runs the synchronous `fn` on +its draft) and `handle.retire(ctx)` (keyed: `handle.retire(key, ctx)`). + +### Four patterns + +Two decisions (R12 and R13, both as in Durable) shape how transactions are written. **Drafts are revoked when the +transaction settles**, so a transaction returns at most plain values, never something taken from a draft. And +**there is no existence check inside a transaction.** + +```ts +// 1. Return data: write in the transaction, read after the commit. +await self.tx(async (tx) => { (await tx.draft(list)).items.push(note); }, ctx); +return list.value.items.length; // the committed revision, or a newer one + +// 2. Remove if present: retire of an absent document is a no-op. +await drafts.retire(author, ctx); + +// 3. Create with tags if missing: the draft creates it, only if it changes. +await self.tx(async (tx) => { + (await tx.draft(drafts, author, { tags: [SessionTag(session)] })).text = text; +}, ctx); + +// 4. Branch on existence: read first, outside the transaction. +if ((await drafts.get(author)) === undefined) { … } +``` + +### Durable's rule + +Inside the callback, only `await tx.*`. No other objects, no network, no models, no processes, no second `self.tx`, no +reads past `tx`. Breaking it is not an error but a hang: the object's line is held while the callback waits. Fetch +first; decide and write inside. + +```ts +// Wrong: a fetch inside the transaction holds the object's line. +await self.tx(async (tx) => { (await tx.draft(list)).items.push({ …, title: await fetchTitle(url) }); }, ctx); +// Right: fetch, then write. +const title = await fetchTitle(url, ctx.signal); +await self.tx(async (tx) => { (await tx.draft(list)).items.push({ …, title }); }, ctx); +``` + +## What a commit does + +:::flow wide animate | The commit sequence, Durable's. Nothing is copied, frozen or revalidated; viewers get one batch per commit. +row "Inside self.tx" accent +box line "wait for the line" "per object" +box run "run the callback" "drafts record ops" accent +box prep "prepare" "Delta checks placed values" +box write "storage batch" "deltas or a base" accent +row "After the write" blue +box adopt "adopt" "pointer swap" +box local "notify locally" "once, with the ops" blue +box wire "send" "one batch per link" +box revoke "revoke drafts" "touching one throws" +arrow line -> run +arrow run -> prep +arrow prep -> write +arrow write -> adopt "stored" +arrow adopt -> local +arrow local -> wire +arrow wire -> revoke +note "An error or an abort before the storage write discards everything. Storage commits are serialized per host." +::: + +Lines are per object, never across objects; reads outside a transaction do not take the line. Subscribers on the same +host get the new value and the commit's operations by reference. Remote viewers get the commit's operations, never the +document again; chapter XVII shows how one commit touching several holds still arrives as one message. + +## Tags and deleting by tag + +Tags are typed tokens set when a document is created: + +```ts +export const SessionTag = defineTag("studio.session"); // base code, shared by every plugin +await self.tx(async (tx) => { (await tx.draft(drafts, author, { tags: [SessionTag(session)] })).text = text; }, ctx); +// when the session is deleted +await store.retireByTag(SessionTag, "S1"); // every document tagged S1 +``` + +The design names the operation `store.retireByTag(tag, value)` without saying who holds the store; session delete is +one of the app decisions still open (#18, chapter XIX). + +## Who may persist + +`persist: true` is allowed only in `local` objects and `lifetime: "host"` objects: identities without a principal. A +`remote` object's identity contains the principal key, which changes with the access path and the grant revision. Its +stored documents would scatter across access paths and be orphaned after a reconnect. The rule is enforced at the type +level (`self.state` in a remote factory accepts only non-persisted definitions) and at declaration ("persist needs a +stable identity: put the documents into a local shared store"). This is the second reason for the shared store. + +What survives a host teardown (a crash, a restart) is exactly what is persisted. In-memory documents live as long as +their object: they survive reloads and native restarts of a live object, and go when it ends. Everything else +(natives, derives, routines, references) is rebuilt on activation. + +## Facet documents + +A facet has its own identity (its plugin's name) and may hold documents too, for plugin-wide data such as +preferences. Objects the facet serves may read and write their facet's documents, and one transaction may span the +object's own documents and its facet's (lock order: own line, then the facet's). The facet never touches a served +object's documents. + +## Ownership across plugins + +No plugin reads or writes another plugin's documents, Chord or Durable. Data crosses plugins only through state and +methods. Data that core enforces lives in a format core owns and is handed to core through its API when the data is +created (as with Durable's `createConversation({ init })`), never written into core's documents. + +## Versions and migration + +A change of shape or meaning raises `version` and adds a pure `migrate(old, from)`: + +```ts +const NoteList = defineState("notes.list", { + schema: obj({ items: arr(NoteV2) }), // NoteV2 adds `pinned` + version: 2, + migrate: (old, from) => ({ items: old.items.map((n) => ({ ...n, pinned: false })) }), + initial: () => ({ items: [] }), + persist: true, + checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64, +}); +``` + +Documents migrate on access by the new definition: singletons during activation (hydration counts as access), keyed +ones on their next access. The migrated base is persisted with the next commit; right after a bump, a read-only +transaction writes it. + +Chord does not walk values against schemas. Instead, each definition computes a **schema fingerprint** once +(descriptions and titles ignored), and every stored record keeps the fingerprint it was written with. On access Chord +compares them: the same version with a different fingerprint fails the object `broken` ("schema changed without a +version bump"). So every structural change needs a bump, an additive one with a trivial `migrate`. The one exception +to "no value checks" (R8): the result of `migrate` and of `initial()` is validated against the new schema once, when +produced. A failed check, a throwing `migrate`, or old code meeting a newer document refuses the access, keeps the +stored data and fails the owning object. A changed meaning without a schema change cannot be detected; that is on +you. + +## Storage + +Behind the store is a `DocumentStorage`, passed as `createHost({ storage })`; memory by default. The server has one +shared storage, each worker one for its session. + +```ts +interface DocumentStorage { + commit(writes: readonly DocumentWrite[]): Promise; // create/change/retire, atomic + find(address: DocumentAddress): Promise; + load(id: string): Promise; // version, value, deltasSinceBase + scan(query: { owner: string; definition: string }, limit: number, cursor?: string): + Promise<{ keys: string[]; next?: string }>; + byTag(tag: string, value: string, limit: number, cursor?: string): Promise<{ ids: string[]; next?: string }>; + mintId(): Promise; +} +``` + +Chord ships a memory storage and a SQLite storage (over `@earendil-works/chord/sqlite`, the driver layer that moved +from Durable; Node and Durable Object drivers). Storages check only metadata (incarnation, version, sequence, address +occupancy, fingerprint) and write the supplied delta or base; they never rebuild or validate content on a commit. +Chord documents never commit together with Durable transcripts: Chord has its own storage, not an adapter over +Durable's. Your own storage can prove itself with `registerDocumentStorageConformance(runner, name, withStorage)` from +`@earendil-works/chord/testing`. + +Keyed persisted documents that nobody reads, writes or watches for 10 minutes leave memory and reload on the next +access. A storage error ends the store and closes the host; `onError` receives `{ kind: "storage_failed" }` and the +app decides (chapter I). + +## Kinds of state + +| Kind | For | API | +| --- | --- | --- | +| document | data the object owns | `self.state`, `self.tx` | +| derive | computed values | `self.derive` (chapter VIII) | +| source readable | external data: Durable views, files, logs | value plus ordered changes; feeds derives | +| paged method | large or queryable collections | a method returning pages (chat history) | +| plugin database | full-text search, large blobs | a native opened by its owner (chapter IX) | + +App state lives in Chord documents. Only transcript-bound data, which must fork and rewind with a conversation, lives +in Durable. + +:::review | Review: does an absent singleton read `undefined` or `initial()`? (Q11, open) +Design §9 says "whether a document exists is read outside the transaction (`handle.value` / `handle.get` give +`undefined` for an absent one)" and also "a singleton that does not exist yet reads `initial()` through its handle; +retiring resets it to `initial()`". Review round 3 proposes: singletons always read `initial()`, only absent keyed +documents read `undefined`, and singleton existence needs no API. This guide follows the proposal (`list.value` is +never `undefined`); it is not decided. +::: + +:::note | Built so far +Documents, transactions and storages are layer L3, a port of Durable's session store; not started. The frozen kernel +has a working version with a JSONL storage, which the design rejected. +::: + +# Holding On {#references lede="References, the object lifecycle and ctx: what a caller sees while objects start, fail, restart and move, and why nobody writes retry code."} + +A tab shows the notes of S1 for an hour. During that hour the laptop sleeps, the worker is stopped for idleness and +started again, the notes plugin reloads twice and the store's storage hiccups once. The panel should show the notes the +whole time, say "reconnecting" when it cannot reach them, and recover by itself. In Chord this is what a +**reference** does; plugin code contains no retry loops and no request ids. + +## Three ways to hold + +| Where | API | Released | +| --- | --- | --- | +| a factory | `self.get(Type, input)` | when the object ends | +| a call | `await ctx.get(Type, input, { state? })` | when the call ends | +| outside objects | `await host.get(Type, input, { signal, state?, parkTimeout? })` | `ref.release()` | + +Use `self.get` for what an object needs its whole life (the view's store), `ctx.get` for what one call needs, and +`host.get` only in UI code, tests and entry points. `{ state: false }` binds for methods only, without receiving state. +A reference keeps its object alive while held. + +## The object lifecycle + +```text +(get) → activating → live → ending → ended + └→ failed (backoff) → activating … +``` + +**Activating:** Chord binds the declared references and waits until they are live and hydrated, loads singleton +documents, opens natives (one after another, in declaration order), computes derives and starts routines. The object +goes live when all of that succeeded. Calls wait behind its **gate** until then. + +**Ending:** routines stop, the object's calls and jobs are aborted as their rules say, natives close in reverse order, +references are released. An object with `lifetime: "referenced"` (the default) ends when its last reference goes, +after the type's `linger` if it has one. A `lifetime: "host"` object (only for types without a required input) lives +as long as the host. A running, finishing or blocked job counts as a reference (chapter XI). + +## `ref.state` + +Every reference reports what is going on as a tagged value; listeners get each new one: + +```ts +type RefState = + | { kind: "loading" } // first hold, reconnecting, restarting + | { kind: "live" } + | { kind: "failed"; cause: FailureCause; retryAt: number | undefined } // undefined: no automatic retry + | { kind: "released" }; +``` + +| Situation | Calls | Values | +| --- | --- | --- | +| link drop | park until reconnected | readable, `loading` | +| object loading or restarting | park until live | readable, `loading` | +| rollout: provider ahead or installing | park until installed | readable | +| object `failed` | reject at once with the cause | `failed`, with cause and `retryAt` | + +Values stay readable through all of this: the panel keeps showing the last notes. Parked calls share one deadline (60 +s by default, `parkTimeout` per `get`), then fail `unavailable`. Otherwise a call ends only through its caller's +signal; there is no per-call deadline. A call that was running on an instance that ends fails with an error and is +not resent: Chord cannot know whether it took effect. + +**Errors look like disconnects.** A consumer handles a failed call exactly as a dropped link: show the error, let the +user retry. Work that must not run twice carries its own idempotency key as data (Durable's `requestId`, a job's +`key`). + +:::steps | The worker restarts while a tab watches the notes ++++ Watching +The tab holds `Notes { S1 }`. `ref.state` is `live`; `notes.notes.value` has 40 notes. ++++ The worker goes away +The supervisor stops worker S1 (or it crashes). The server's IPC link to it drops. The server tells the tab that its +hold is gone; the tab's reference goes `loading`. The 40 notes stay readable. ++++ A click meanwhile +Ann's colleague clicks **Add**. The call **parks** on the reference; nothing is sent yet. The deadline is 60 s. ++++ Chord resolves again +Holds on S1 still exist, so Chord calls `route(Session).resolve({ id: "S1" })` again, with the per-address backoff. +The app forks a new worker: a new **incarnation** of `Session({ id: "S1" })`. ++++ Re-hold +The server re-holds every hold on S1 over the new link. The new worker activates `NoteStore { S1 }` (its notes load +from storage) and the views; it sends fresh snapshots. The tab's reference goes `live`. ++++ The parked call runs +The parked `add` is sent now and succeeds. The panel shows 41 notes. Nobody wrote recovery code. +::: + +## Failure and retry + +Any broken part fails the **whole** object: a half-working object could break rules that rely on its documents +changing together. Chord tags the cause by where it failed: + +```ts +type FailureCause = + | { kind: "transient"; error: ChordError } // an open threw, a dependency was unreachable: automatic backoff + | { kind: "broken"; error: ChordError }; // factory, derive, hydration or migrate threw: retried only by a trigger +``` + +**Transient** causes get a backoff per address: 1 s, doubling to 60 s; concurrent gets share one attempt; success +resets it. **Broken** causes would fail the same way again (a code bug, invalid data), so they are retried only by a +**trigger**: a plugin reload, `ref.retry()` (a Retry button), or a readable the factory declared with +`self.retryOn(...readables)`. Only an opener may declare its own failure broken, with `throw broken(error)`, for +example for invalid configuration. A failed reference stays held; on success it goes `loading`, then `live`. + +```ts +host.serve(EnvConnection, ({ env }, self) => { + const config = self.get(EnvConfig, {}); // a facet or registry value, outlives the attempt + self.retryOn(config.envs); // a config change retries a broken open + const ssh = self.open(async ({ signal }) => { + const c = config.envs.value[env]; + if (!c) throw broken(new Error(`unknown environment ${env}`)); + return connect(c, signal); + }, (s) => s.close()); + … +}); +``` + +`retryOn` sources must live independently of the attempt (facet documents, registry values); a member of one of the +object's own references is a type error, since it goes away with the attempt. + +## Waiting for dependencies + +An object whose references are not live yet stays `loading` and goes live the moment they are, bounded by +`readyTimeout` (60 s). The decision for the failure case (R18) is what makes recovery automatic: + +:::sequence animate | Dependency recovery. The view fails transient with the store's cause, keeps its reference, and is retried the moment the store goes live. +actor tab "tab" +actor view "Notes { S1 }" blue +actor store "NoteStore { S1 }" accent +tab -> view: hold +view -> store: self.get (activation) +store -> store: open fails: disk not mounted +store --> view: failed (transient) +view --> tab: failed: transient, cause inside +note view..store: the view keeps its reference; its state is the retry trigger +store -> store: backoff retry: open succeeds +store --> view: live +view -> view: retried at once (backoff cancelled) +view --> tab: live +::: + +A dependency that fails, or does not become live in time, fails the dependent **transient** with the dependency's +cause inside, never broken: the cause is the dependency's. The dependent keeps its references while failed, and Chord +uses their state as a retry trigger. Factory authors write `self.get` and know none of this. After an object is live, +a dependency leaving `live` does not change it: its calls that use the dependency get the dependency's error, or park +while it loads. + +**Cycles are rejected** (Q5). If A's activation gets B and B's gets A, neither would ever lose its last reference. +Every object instance has a uuidv7 instance id; on one host Chord follows the waiting edges of activating objects, and +across hosts a hold made during an activation carries the chain of instance ids. A cycle fails the activation +`broken`, naming it ("A → B → A"), and keeps no references. + +## No optional references + +`get` has no `optional` flag. When a dependency's plugin may be missing, turn the dependency around: the dependent +owns a registry and the other plugin contributes to it. Radius uses forwards from the ports plugin, but ports is +optional, so radius owns `radius.forwards` and ports contributes its forwards; without ports the table is empty and +radius works (chapter XIII). + +## `ctx` + +Every call gets a `ctx` as its last argument: + +| Field | Meaning | +| --- | --- | +| `signal` | fires when the caller cancelled or the object ended; pass it to everything you wait on | +| `principal` | the caller (remote types only) | +| `incarnation` | the calling host's incarnation, for calls that arrived over a link (chapter X) | +| `get` | references released when the call ends | +| `telemetry`, `trace(name, attributes, fn)` | the current span; a child span with a child `ctx` (chapter XVII) | +| `jobId`, `progress` | in a job's `run` (chapter XI) | + +```ts +exportText: async ({}, ctx) => { + const catalog = await ctx.get(Catalog, {}); // released when the call ends + const title = catalog.sessions.value[session]?.title ?? session; + const res = await fetch(renderUrl, { method: "POST", body: render(title, list.value), signal: ctx.signal }); + return await res.text(); +}, +``` + +There are no code versions and no request ids in `ctx`. Durable's `Context` is the same module (`chord/context`): +`ctx` extends it, so passing `ctx` into pi-durable needs no conversion. Code outside objects gets a `ctx` from +`host.root(fn, { principal?, signal? })`; inside Durable tools it uses `chord(fn, context)` (chapter XIV). + +:::note | Built so far +References, `ctx` and the lifecycle are layers L4 and L5. `ChordError` and `FailureCause` are specified in the L1 +draft (`layer-1.md`), with codes `invalid`, `contract_mismatch`, `forbidden`, `gone`, `not_ready`, `released`, +`too_large`, `unavailable` and `aborted`, and a finer `reason` such as `invalid/args`. +::: + +# Taking Access Away {#revocation lede="Revocation: how removing a guest ends their objects on every host at once, and why reconnecting does not bring access back."} + +The owner removes Ann from S1. Her tab still has a WebSocket open and a `Notes { S1 }` view on worker S1. Both must end +now, on the server and on the worker, and her tab's automatic reconnect must not restore anything. Chord provides the +mechanism; the app decides who is blocked and remembers it. + +## Labels, not predicates + +The app declares, once, which **revocation labels** each principal carries. Revoking means revoking one label; +everything carrying it is hit. + +```ts +// studio/src/principal.ts +export type Revocation = { grant: string } | { device: string } | { session: string }; + +export const revokedBy = (p: P): Revocation[] => + p.kind === "guest" ? [{ grant: p.grant }, { session: p.session }] + : p.kind === "owner" && p.access === "device" && p.device ? [{ device: p.device }] + : []; + +const server = createHost({ address: Server(), principal: { kind: "server" }, revokedBy }); +// every worker is created with the same revokedBy +``` + +| Principal | Labels | +| --- | --- | +| guest Ann (grant g1) in S1 | `{ grant: "g1" }`, `{ session: "S1" }` | +| guest Bob (grant g2) in S1 | `{ grant: "g2" }`, `{ session: "S1" }` | +| owner on the paired phone | `{ device: "phone" }` | +| owner, local | none | + +| Call | Hits | +| --- | --- | +| `server.revoke({ grant: "g1" })`: remove Ann | Ann | +| `server.revoke({ session: "S1" })`: archive S1 | Ann and Bob | +| `server.revoke({ device: "phone" })`: sign out the phone | the phone | + +A label is plain data because revocation is an event that must travel from the server to the workers; a predicate +function could not be sent. + +## What `revoke` does + +`host.revoke(r)` ends every object and closes every link whose principal `p` has `r` in `revokedBy(p)` (compared by +canonical JSON; never the host's own principal), and passes `r` to its children, which apply their own `revokedBy`. + +:::sequence animate | Removing guest Ann. The label travels down; her objects end on every host; her reconnect is refused by the app's own record. +actor owner "owner tab" +actor server "server" accent +actor worker "worker S1" blue +actor ann "Ann's tab" "guest g1" muted +owner -> server: Sharing.remove({ grant: "g1" }) +server -> server: app: mark g1 blocked (document) +server => server: revoke({ grant: "g1" }) +server -> ann: close link +server => worker: revoke { grant: "g1" } +worker -> worker: end Notes { S1 } as g1 +ann -> server: reconnect +server -> server: authenticate: g1 blocked +server --> ann: refused (4401) +::: + +On worker S1, for `revoke({ grant: "g1" })`: + +| Object | Principal | Hit | +| --- | --- | --- | +| `Notes { S1 }` | owner, local | no | +| `Notes { S1 }` | guest g1 | **yes**: ends | +| `Notes { S1 }` | guest g2 | no | +| `NoteStore { S1 }` | none (local) | no | + +Ending Ann's view releases its reference to the store, which keeps serving the others. + +## Chord persists nothing here + +`revoke` cuts live access only. Lasting denial lives in the app's data (the sharing plugin's guest records), which +`authenticate` checks on every new connection. After a server restart nothing live exists to revoke; every client +authenticates again and blocked grants are refused there. + +**No re-stamping.** A changed grant (Ann moves from read to write) is a revocation of the old grant. Her link closes, +her tab reconnects and authenticates with the new grant revision, which gives a new principal key, and her references +re-hold by themselves, now as an editor. Reconnecting never restores revoked authority. + +**Revocation beats jobs** (R11). Normally a running job keeps its object alive after the last reference is released +(a closed tab does not abort an import). Revoking a principal ends its objects at once regardless; their jobs get the +abort signal and the normal close timeout. + +## Work in shared stores + +Revocation ends objects whose principal carries the label. A shared store has no principal, so work it runs *on +behalf of* someone (a question waiting for Ann's answer, a shell command a guest started) is not ended by `revoke`. +Such a store keeps the grant as data with the work and reconciles from the grant owner's live state with a routine +(chapter X): + +```ts +// in a shared store that runs work for guests +const sharing = self.get(Grants, {}); // the sharing plugin's live grants +self.routines.on(sharing.live, async (live, ctx) => { + for (const run of shell.running.value) + if (run.args.grant && !live[run.args.grant]) shell.abort(run.id); // idempotent +}); +``` + +There is no revocation hook: a hook can be missed (a crash between revoke and hook), a routine over current state +cannot. The window between the revocation and the routine's run is accepted (R10); a step started in it still runs. + +## Who does what + +| Chord | The app | +| --- | --- | +| stamps principals on links | what a grant is; share links, cookies, `authenticate` | +| keeps one object per principal key | defines `Revocation` and `revokedBy` | +| on `revoke(r)`: ends matching objects and links, passes `r` down | decides when to revoke; stores who is blocked | + +# Computed Values {#derives lede="Derives, the one footgun that makes them expensive, mapShared, and presence: who is looking."} + +The reader's notes without authors, a note count, the number of people viewing: values computed from other values. +Storing copies would go stale; recomputing them on every read would be wasteful and could not be watched. A +**derive** is computed once per change and sent to viewers as a difference. + +## `self.derive` + +```ts +const count = self.derive("count", [list], ([l]) => l.items.length); +const summary = self.derive("summary", [list, store.pinned], ([l, pinned], previous) => ({ + total: l.items.length, + pinned: pinned.length, +})); +``` + +`self.derive(name, sources, (values, previous) => value)`. Sources can be documents, readables, other derives and +state members of references. The value is recomputed when a source changes and diffed into operations for the wire. +Derived values are adopted before ordinary observers run and share their source commit's wire batch, so a viewer +never sees a note list and a count from different commits. + +`previous` is the derive's last value, kept across factory re-runs (reloads) by name. It is a cache, empty after a +host teardown, so a derive must be correct without it. A derive that throws fails its whole owner `broken`. + +## How the difference is found + +Chord compares the new value with the previous one using Delta's `diffRevisions`. Parts that are the **same objects** +as last time cost one `===` each. Parts that are new objects must be compared by content. + +**The footgun.** A derive that builds new objects for unchanged data makes every commit cost the size of the whole +value, per view. The reader list from chapter IV, written naively: + +```ts +self.derive("public", [list], ([l]) => ({ items: l.items.map((n) => ({ id: n.id, text: n.text, at: n.at })) })); +// one note added to 2,000: 2,001 new objects, every one compared by content +``` + +**`mapShared(f)`** keeps one result per source element: `f` runs only for new or changed elements, unchanged ones +return their previous result object. + +```ts +import { mapShared } from "@earendil-works/chord/delta"; + +const stripAuthor = mapShared((n: Note): PublicNote => ({ id: n.id, text: n.text, at: n.at })); +self.derive("public", [list], ([l]) => ({ items: l.items.map(stripAuthor) })); +// one note added: one new object, 2,000 identical references; the diff emits one insert +``` + +:::review | Review: mapShared is planned, not available +Design §12 introduces `mapShared` as coming "from `@earendil-works/chord/delta`", but nothing in `packages/chord/src` +defines it yet, and Delta does not yet treat bytes as atomic leaves in `diffRevisions`. Design §28 lists both as the +step "Delta in pi", before the layered rebuild. Until then, keep a `Map` from source element to result yourself, which +is what `mapShared` will do. +::: + +## Cookbook + +| Case | How | +| --- | --- | +| select a field, look up by key, a small aggregate | compute it; the value is small | +| filter or map a list | `mapShared` over the elements | +| redaction | preferably separate documents per visibility; otherwise `mapShared`, computed once in the shared store, never per view | +| join two sources | `mapShared` plus the looked-up source's revision | +| a window over a large list | take a section of the source elements without rebuilding them; cache per-element measurements by identity; keep the window's start stable so changes arrive as appends | +| a large view combined from several sources | re-path the sources' operations, as Durable's `ConversationView` does; application code never interprets ops | + +**Container width** (R24): structural sharing is O(change) along a path, not across a container's width. Changing one +item of a flat array copies the array's slots, and `mapShared` visits every element once (one identity comparison +each). Large collections are keyed documents or bounded windows. A benchmark of long arrays is to set the documented +limit. + +## Readables that are not documents + +Three small readables round out the toolkit: + +- `constant(value)`: never changes. Start data uses it (chapter IX). +- `self.writable(initial)`: a host-local readable of any in-process value (functions included), compared by identity, + with `set`. It never goes on the wire; it exists for registry contributions that change (chapter XIII). +- Source readables from elsewhere: a Durable view or document watch is already a Chord readable, because Durable and + Chord v2 share the readable type (`chord/src/state.ts`), so it can feed a derive or be exposed as state directly. + +:::review | Review: exposing a writable (Q14, open) +A `writable` may hold functions; nothing yet stops a factory from returning one as a state member, which would reach +the wire encoder. Review round 3 proposes that only members with a JSON schema in the contract are exposable, checked +when the object is served. Undecided. +::: + +## Presence: who is looking + +`host.references(Type)` is a readable of who holds references to objects of a type on this host: input, principal and +client. Client holds carry an opaque origin id, so two tabs are two clients. + +```ts +host.serve(NoteStore, ({ session }, self) => { + const viewers = self.derive("viewers", [host.references(Notes)], ([refs]) => + refs.filter((r) => r.input.session === session).length); + return { …, viewers }; +}); +``` + +A link drop is a departure: accurate and simple. Smoothing (a phone switching networks makes the count flicker) is the +UI's job, for example a debounced count. Counts per person are derived by the app from principals and origins. There +is no consumer-side linger: a closed tab closes its link and keeps nothing. The provider's `linger` keeps a +just-closed object warm, and that also keeps the worker alive through `host.activity`. + +**"Leaving" is never derived from an unmount.** A UI reload unmounts and remounts components; a component unmounted by +a reload has not left. Leaving is explicit navigation, or presence (a dropped reference). This replaces p2's presence, +viewer and login-abandon leases with one mechanism. + +:::note | Built so far +Readables and derives are layer L2 (a readable is today's `replicatedState`, derives reuse today's derived cells over +it). Not started in the rebuild. +::: + +# Things the OS Gives You {#natives lede="Start data, host-lifetime objects and natives: how a plugin opens a file watcher or a socket so that it is closed exactly once, also across reloads."} + +Studio's notes get an inbox: a text file dropped into the session's `inbox/` directory becomes a note. That needs a +file watcher, a **native**: an OS-level value (a file handle, a socket, a process, a database connection) that cannot +be JSON, cannot be copied, must be opened, and must be closed exactly once. It also needs the session's data +directory, which the plugin does not know. This chapter covers both. + +## Start data is an object + +A worker knows things before any plugin loads: its session id, the data directory, init data from the server's start +message. There is no special mechanism for them. Start data is an ordinary `local` object with `lifetime: "host"` and +constant state, served by base code's core facet: + +```ts +// studio/src/core/contract.ts +export const Init = obj({ id: str(), dataDir: str(), cwd: str(), env: opt(str()) }); +export const SessionInfo = local("studio.session-info", { + version: 1, + input: obj({}), + members: { init: Init }, +}); + +// studio/src/core/worker.ts +export const coreWorkerFacet = (init: Static) => defineFacet((host) => { + host.serve(SessionInfo, () => ({ init: constant(init) }), { lifetime: "host" }); + // … the Harness and other core objects (chapter XIV) +}); +``` + +Any plugin on the worker declares a reference and reads it **after activation**: in an opener, a derive, a routine or +a method, never synchronously in the factory. + +```ts +const info = self.get(SessionInfo, {}); +// later: info.init.value.id, info.init.value.dataDir +``` + +Paths are derived by the plugin from generic start data (`join(dataDir, id, "notes-inbox")`), never provided per +plugin. Platform bindings work the same way: on a Durable Object, a core object's `self.open` returns the binding +(`state`, `env`) and its close does nothing. + +## `self.open` + +```ts +const watcher = self.open( + async ({ signal }) => watchDir(path, { signal }), // open: during activation + (w, reason) => w.close(), // close: when the object ends or restarts + { closeTimeout: 10_000 }, // optional; host default 30 s +); +watcher.value; // the watcher; reading it before activation throws not_ready +``` + +`self.open` returns a handle with `.value` (type `Native`), not a proxy. The rules: + +- **Natives live only in objects.** Not at module level, not in the facet. A module-level watcher would survive a + reload with nobody to close it. +- **Order:** natives open one after another in declaration order (a later opener may read an earlier handle's + `.value`) and close in reverse. The object goes live after all opens succeeded. +- **The value never changes during an attempt.** No replacement, no versions, no stale-handle logic. To change a live + native, the object restarts. +- **A failed open** makes the object `failed`: transient with backoff by default, broken if the opener says so + (`throw broken(error)`). A partly failed activation closes the natives already open, with reason `end`; the next + attempt opens all of them anew. +- **`close` gets a reason:** `{ kind: "reload" }` (the plugin reloads and this native object restarts), `{ kind: + "stop" }` (the host shuts down) or `{ kind: "end" }` (the last reference ended, or the type was removed). Each + `close` is bounded by `closeTimeout`, then reported as a possible leak while the rest continues. A throwing `close` + does not stop the others; `host.close()` collects the errors, background ends report to `onError`. +- **Behaviour around a native goes into routines** (chapter X). Listeners on natives outside routines are forbidden. + +An object with at least one `self.open` is a **native object**; one without is a **pure object**. The difference +decides what a reload does (chapter XV): pure objects are swapped in place, native objects restart. + +## The inbox + +The inbox must run as long as the worker, whether anyone looks at the notes or not. So it is a `lifetime: "host"` +object. The facet cannot `self.get` a type it serves, and the inbox has no meaningful input, so it is an object +without input that reads its session from the start data: + +```ts +// notes/contract.ts +export const NoteInbox = local("notes.inbox", { version: 1, input: obj({}), members: {} }); + +// notes/worker.ts +host.serve(NoteInbox, (_input, self) => { + const info = self.get(SessionInfo, {}); + const watcher = self.open(async ({ signal }) => { + const { id, dataDir } = info.init.value; // read in the opener: after activation + return watchDir(join(dataDir, id, "notes-inbox"), { signal }); + }, (w) => w.close()); + + self.routines.loop("import", async (ctx) => { // chapter X + const store = await ctx.get(NoteStore, { session: info.init.value.id }); + for await (const file of watcher.value.files(ctx.signal)) { + await store.add({ text: await readFile(file, "utf8"), by: "inbox" }, ctx); + await rm(file); + } + }); + return {}; +}, { lifetime: "host" }); +``` + +Note `ctx.get(NoteStore, …)` in the routine rather than `self.get` in the factory. The factory runs before the +session id is known (start data is read after activation), so a host-lifetime object without input gets +session-scoped objects with `ctx.get` in a routine or method (R22). A `get` whose input is computed after activation is +to be added only if a real case needs it. + +## Natives with work in flight + +A restart is sequential (R16): close the old native, then open the new one. When the old one still has work running, +use one of two patterns: + +- **Drain in the background.** `close` detaches the old resource and returns at once; it disposes of it when its work + finishes, or after `closeTimeout` at most. Chord opens the new one immediately. Code mode's sandbox does this: a + script running during a reload finishes on the old sandbox (for up to 30 s, Q7; a longer one is aborted and its turn + fails), new scripts start on the new one. +- **Swap through a registry.** The resource is a registry contribution (chapter XIII). Readers always look up the + current one at use; a reader that must wait for one uses a cancellable "wait until present" with the park deadline. + +Chord aborts only the object's own calls and jobs on close. Other work that uses the native (Durable tool calls on a +sandbox) is the owner's business: `close` may drain instead of tearing down. What an old native does after its owner +restarts is also the owner's `close`: a model runtime keeps working until collected, a closed SSH connection fails its +in-flight `exec`. + +## Natives that follow a relocation + +A session can move to another directory. The inbox watcher must follow. Restarting the object on every move would be +heavy-handed; instead the native is a stable manager opened once, and a routine on the session location switches its +internal resource and bumps a generation, so late events from the old location are dropped: + +```ts +const watcher = self.open(async ({ signal }) => new SwitchableWatcher({ signal }), (w) => w.close()); +self.routines.on(location.cwd, (cwd) => watcher.value.switchTo(join(cwd, ".notes-inbox"))); +``` + +## Call-scoped natives + +A native that lives exactly as long as one call (a temporary SSH session for a directory listing, a file handle for a +download) is opened and closed inside the method with `try/finally`. There is no `ctx.open`: + +```ts +list: async ({ path }, ctx) => { + const session = await ssh.open(config, ctx.signal); + try { return await session.readDir(path, ctx.signal); } + finally { session.close(); } +}, +``` + +## Core vs. plugin + +`SessionInfo` comes from base code, `NoteInbox` from a plugin; both use the same primitives. Where code lives decides +which it is. **Base code** (the app's `src/`, imported statically by the start code under a fixed name such as +`@studio/core`) is always there, not switchable, and a change restarts the hosts. **A plugin** (its own directory, +found by listing, named by its `package.json`) is switchable and reloads. Core is what the app always needs and +everything builds on: the harness, the session list, access, the session location. Optional features are plugins; +code mode is a plugin. + +:::note | Built so far +Natives and host-lifetime objects are layer L4. `constant` is decided (R23), not built. +::: + +# Standing Behaviour {#routines lede="Routines, reconciling instead of notifying, and how two owners keep a rule between them without a shared transaction."} + +Notes older than 30 days should go every hour. Inbox files should be imported as they arrive. Notes of an archived +session should become read-only. Each of these is behaviour that stands, not a call someone makes. Written as bare +`setInterval`s and event listeners, it would keep running old code after a reload and touch state from nowhere. Chord +has one declared form for it: the **routine**. + +## Four kinds + +```ts +self.routines.every("cleanup", HOUR, (ctx) => dropOlderThan(30 * DAY, ctx)); // due time survives reloads +self.routines.on(settings, async (value, ctx) => applyLimit(value, ctx)); // a readable or event source +self.routines.loop("import", async (ctx) => { // runs until the object ends + while (!ctx.signal.aborted) await importNext(ctx); +}); +self.routines.custom("ticker", ({ signal, run }) => { // anything else + const timer = setInterval(() => run(tick), 1000); + return () => clearInterval(timer); // cleanup undoes the setup +}); +``` + +Routines start when the object goes live, stop when it ends, and restart with the new code on every reload of their +plugin. Each run gets its own `ctx`; errors are caught and reported, never uncaught. Routines are named (`on` +optionally by its source) so Chord recognizes them across reloads; that is how `every` keeps its due time. + +## Serialized per routine + +The new instance of a routine starts only after the old handler in flight has settled; only that routine waits, for at +most the routine handover timeout (10 s), after which it starts anyway and the stuck handler is reported. + +The reason, as a trace: a supervisor routine reconciles pending worker restarts. The old handler is midway through +`restart(W1)` when the plugin reloads. Without serialization the new routine starts at once, sees W1 still pending, and +restarts it a second time. + +## Reconcile, don't track + +A routine **reconciles from the current value**: it makes the world match what it sees now. It writes no "in +progress" flags, and it starts a job for long work (chapter XI). A handler that only cares about the latest value +compares with the current value itself; there is no `overlap` option. Subscribing to the object's own documents is +gapless: every commit is seen. + +No bare timers or callbacks that touch state outside routines, jobs or calls. That is what lets Chord stop all of an +object's behaviour when it ends or reloads. + +## Rules between two owners + +Every thing has exactly one owner, the smallest object it belongs to. Some rules span two: "notes of an archived +session are read-only" involves the catalog (which knows archiving) and `NoteStore` (which holds notes). Two tempting +designs are wrong: + +- **A cross-owner transaction** (archive and lock the notes atomically) would make the catalog know every plugin and + couple their storage. +- **A notification** (`catalog` calls `notes.archive(session)`) can be lost: the worker is not running, the link drops, + the process crashes between the archive and the call. + +The design's rule: **the owner of the dependent data enforces the rule** by reconciling from the other owner's state +with an idempotent routine. + +```ts +// NoteStore: an archived session's notes are read-only +const flags = self.state(NoteFlags); // { archived: boolean }, NoteStore's own document +const catalog = self.get(Catalog, {}); // the server's session list: another owner +self.routines.on(catalog.sessions, async (sessions, ctx) => { + const archived = sessions[session]?.archived ?? false; + if (flags.value.archived !== archived) await flags.change((d) => { d.archived = archived; }, ctx); +}); +// add and remove refuse while flags.value.archived +``` + +After a crash the routine runs again with the current value. The catalog does not know the notes plugin exists. The +same pattern closes a forward of an archived session in the ports plugin, aborts a job whose grant was revoked +(chapter VII), and keeps any invariant between plugins. + +## Reading without starting a worker + +The session list in the sidebar shows each session's note count. Getting `NoteStore { S1 }` for every session would +start every worker. The answer is plain publishing between two objects of one plugin, no Chord mechanism: the worker +side reports a summary to a server-side object by a method call from a routine, and clients read the server object. + +```ts +// notes/contract.ts +export const NoteCounts = remote("notes.counts", { + version: 1, input: obj({}), placement: () => Server(), + variants: { + owner: { state: { counts: map(int()) } }, + worker: { methods: { report: method(obj({ session: str(), count: int() })) } }, + }, +}); + +// notes/server.ts +const Counts = defineState("notes.counts", { + schema: obj({ bySession: map(obj({ count: int(), incarnation: str() })) }), + version: 1, initial: () => ({ bySession: {} }), persist: true, // survives a server restart + checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64, +}); + +host.serve(NoteCountStore, (_input, self) => { // local, lifetime "host" + const counts = self.state(Counts); + return { + counts: self.derive("counts", [counts], ([c]) => mapValues(c.bySession, (e) => e.count)), + report: ({ session, count, incarnation }, ctx) => + counts.change((d) => { + const last = d.bySession[session]; + if (last && last.incarnation > incarnation) return; // uuidv7: an older worker, ignore + d.bySession[session] = { count, incarnation }; + }, ctx), + }; +}, { lifetime: "host" }); + +host.serve(NoteCounts, (_input, principal, self) => { + const store = self.get(NoteCountStore, {}); + if (principal.kind === "worker") return { + variant: "worker", + report: ({ session, count }, ctx) => { + if (session !== principal.session) throw forbidden(); // a worker reports only itself + return store.report({ session, count, incarnation: ctx.incarnation }, ctx); + }, + }; + return { variant: "owner", counts: store.counts }; +}); + +// notes/worker.ts, inside NoteStore +const counts = self.get(NoteCounts, {}); // a reference to the server +self.routines.on(count, (n, ctx) => inVariant(counts, "worker") && counts.report({ session, count: n }, ctx)); +``` + +The server object keeps the latest value per session (in a document if it must survive a server restart) and +**ignores reports from an older worker incarnation** (incarnations are uuidv7, so they order by start time). A worker that was replaced may still deliver a late report; the +fence is the caller's incarnation, which Chord stamps on every call from the link handshake (R20): `ctx.incarnation`, +next to `ctx.principal`, not part of the principal or its key, never sent as data, so it cannot be forged. Session +status, subagent lists, unread marks and artifact lists in p2 all follow this pattern. The store is local and has no +caller incarnation of its own, so the view passes it on as data. + +:::review | Review: `ctx.incarnation` across a relay (Q13, open) +A worker's call to the server arrives directly, so `ctx.incarnation` is the worker's. But a call relayed through the +server (a tab's call to a worker) carries the server's incarnation on the worker's side, because the worker's +handshake is with the server. Review round 3 proposes that relays stamp the origin host's incarnation, as they stamp +the principal. Undecided; the pattern above only needs the direct case. +::: + +:::note | Built so far +Routines are layer L8. The design treats `every` as one of four forms over one runner; review round 3 proposed cutting +to `on` and `loop` only (C4), not decided. +::: + +# Work That Outlives the Call {#jobs lede="Jobs: owned work started by a method that keeps running after its caller leaves, with cancel, dedupe and a recorded outcome."} + +Importing a 50 MB text file into notes takes a minute. The tab that started it may close; the import must go on, be +cancellable, not run twice when the tab retries after a link blip, and its result must be recorded. A method cannot do +this (it dies with its caller), a routine is standing behaviour rather than a request, and a Durable task is for work +that must resume after a crash. Chord's form for it is the **job**, built in pi-durable's sandwich: **intent** (a +transaction), **effect** (anything), **outcome** (a transaction in the current code). + +## Defining and running a job + +```ts +// notes/worker.ts +const ImportJob = defineJob("notes.import", { + version: 1, + args: obj({ path: str() }), + progress: obj({ lines: int() }), + result: obj({ added: int() }), + concurrency: 1, // one import at a time per NoteStore +}); + +const ImportResults = defineState("notes.import-result", { + schema: obj({ outcome: union([obj({ added: int() }), obj({ error: str() })]) }), + version: 1, initial: () => ({ outcome: { added: 0 } }), keyed: true, + checkpointWhen: () => true, +}); + +host.serve(NoteStore, ({ session }, self) => { + const list = self.state(NoteList); + const results = self.state(ImportResults); + const imports = self.job(ImportJob, { + run: async ({ path }, ctx) => { // 2. effect + let lines = 0; + for await (const line of readLines(path, ctx.signal)) { + await addNote({ id: `${ctx.jobId}:${lines}`, text: line, by: "import" }, ctx); // keyed by job id + ctx.progress({ lines: ++lines }); + } + return { added: lines }; + }, + finish: async (outcome, tx) => { // 3. outcome, current code + (await tx.draft(results, outcome.id)).outcome = + outcome.kind === "done" ? outcome.result : { error: outcome.error }; + }, + }); + return { + …, + imports: imports.running, // live list of runs + startImport: ({ path, requestId }, ctx) => + imports.start({ path }, ctx, { key: requestId }), // 1. intent (refined below) + abortImport: ({ id }) => imports.abort(id), + }; +}); +``` + +`defineJob(name, { version, args, progress, result, concurrency?, persist?, accepts? })`; `progress` and the outcome +are typed by the schemas. + +:::sequence animate | The job sandwich. The intent commits before anything runs; the effect runs outside any transaction; the outcome commits in the current code. +actor tab "tab" +actor store "NoteStore { S1 }" accent +actor run "run" "effect" blue +tab -> store: startImport({ path, requestId }) +store => store: 1. intent: busy check, dedupe by key, commit +store --> tab: { id } +store -> run: start after the commit +run -> run: read lines, add notes (keyed by jobId) +run --> store: progress { lines } (throttled) +note tab..store: the tab may close here: the job counts as a reference +run --> store: settles: { kind: "done", result } +store => store: 3. record outcome, finish(outcome, tx), ack +::: + +## The three steps + +**1. Intent.** `job.start(args, ctx, { key? })` is shorthand for `tx.start(job, args, { key? })` inside a +transaction, which may contain other writes. The busy check (`concurrency`) and deduplication (the same key returns the +existing run) are atomic and in the same commit; `run` starts only after the commit. Run ids are uuidv7. + +**2. Effect.** `run(args, ctx)` may do anything: processes, network, other objects. It writes no document itself; it +returns the outcome. External writes carry a dedupe key derived from `ctx.jobId` (above: note ids), so an interrupted +attempt cannot apply them twice. On a reload `run` keeps its old code. + +**3. Outcome.** When `run` settles, Chord records the outcome in its own record, then calls the **current** code's +`finish(outcome, tx)` and acknowledges the run in the same commit. The outcome is tagged: + +```ts +type Outcome = { id: string } & ({ kind: "done"; result: R } | { kind: "failed"; error: "threw" | "aborted" | "lost" }); +``` + +A throwing `finish` changes nothing and is retried with a backoff (1 s to 60 s), and after the next reload. `finish` +is optional and follows Durable's transaction rule. + +## Dedupe ends with the run + +Deduplication covers **unsettled runs only** (Q9). Once `finish` has committed, Chord drops the run's record and key +and releases its waiters' references; it keeps nothing beyond a run's lifetime. Dedupe after completion is the +owner's. Studio records which run a request started, in the same commit as the intent, and `finish` records the +result by run id (`ImportResults` above): + +```ts +const Requests = defineState("notes.import-request", { + schema: obj({ run: str() }), version: 1, initial: () => ({ run: "" }), keyed: true, persist: true, + checkpointWhen: () => true, +}); + +startImport: async ({ path, requestId }, ctx) => { + const prior = await requests.get(requestId); // read outside the transaction (R13) + if (prior) return { id: prior.run }; // started before, maybe finished long ago + return self.tx(async (tx) => { + const run = await tx.start(imports, { path }, { key: requestId }); // dedupes a concurrent retry + (await tx.draft(requests, requestId, { tags: [SessionTag(session)] })).run = run.id; + return { id: run.id }; + }, ctx); +}, +``` + +Two concurrent retries both miss `prior`, but the key makes `tx.start` return the same unsettled run. The client +watches `imports` for progress and reads the result by run id once it is gone from `running`. + +Why Chord does not keep completed runs itself: either 100,000 keyed imports would stay in memory forever, or a +retention limit would let a retried key rerun its effect. The owner knows which results matter. + +## Watching, waiting, aborting + +- **`running`** is a readable of `{ id, version, args, progress, phase }`. `phase` is `running`, `finishing` (outcome + recorded, `finish` not yet committed) or `blocked` (with the reason). `concurrency` counts only active runs. The + panel shows a progress bar from it. +- **Progress** is in memory, throttled and sent as deltas. For output, use `OutputBuffer` and `Progress` from + `@earendil-works/chord/output`: bounded windows, adaptive throttling, no write amplification. +- **Waiting:** `start` returns a run with `outcome(ctx)`. The caller's abort ends only its wait, never the run. +- **Abort is a request:** `job.abort(id)` fires the run's `ctx.signal`; the run decides what it returns (a result + that already arrived stays a success). A run that ignores the signal keeps its slot until it ends. `finish` runs in + Chord's own context. + +## Lifetime, persistence, revocation + +**Jobs keep their owner alive.** A running, finishing or blocked job counts as a reference to its object, which ends +only once all its jobs are settled. Starting an import and closing the tab does not abort it. Unsettled runs also +count as activity for the worker's keep-alive. + +**Revocation takes precedence** (R11). Revoking a principal ends its objects at once, whatever jobs remain; the jobs +get the abort signal and the normal close timeout. A job runs as its object's principal. + +**Persistence:** jobs are in memory by default and gone with the process. `persist: true` (only for stable +identities) keeps the record: a run live at a crash ends `lost` at the next activation and `finish` records it. +Nothing is rerun automatically. + +**Versions:** a run started under an older job version keeps its version; `finish` and `running` see it through pure +`accepts` adapters, and stored outcomes are never rewritten. A version bump without an adapter does not refuse the +reload: the old outcome is `blocked`, kept and reported until a later reload brings the adapter. + +**Native objects:** a job in a native object ends `aborted` when the object restarts on a reload (chapter XV). In a +pure object it keeps running on its old `run` code. + +## What is not a job + +| Work | Use | +| --- | --- | +| request and response that may die with its caller | a method | +| input into running work (a pasted code, an approval) | a flow (chapter XII) | +| work that must resume after a crash | a Durable task | +| standing behaviour | a routine | + +:::review | Review: jobs are the largest open simplification (#9, C3) +Decision #9 (deferred until login is ported): a login or a long relocation is a job whose state consumers want to +address and watch (`LoginFlow { id }`). Options: (a) jobs stay records inside their owner, watched through `running`; +(b) a job is an object of its own, addressed by its id, living until done. Separately, review round 3 proposed cut C3: +shrink jobs to a runner over documents (no job versions, `accepts`, `blocked`, `persist`) or move them out of the +kernel. Not decided. `chord-next-walkthrough.md` concept 17 still shows the older outcome shape `{ ok: true, result }`; +the design tags outcomes by `kind`. +::: + +# Asking a Person {#flows lede="Flows and the Questions service: how running work waits for an answer from someone who may be on any tab, may reconnect, and may answer twice."} + +A provider login asks for a pasted code, then an account choice. A dangerous tool call waits for the owner's +approval. The agent asks everyone present a question. In each case running work waits for a person, who may be on any +tab, may lose the connection halfway, may answer late or twice. Two patterns cover this: a **flow** kept as its +owner's state, and the **Questions** service, a shared store for questions. + +## A flow is state, not a call + +The tempting design is a long-held server call, `step()`, that blocks until the next step. A link drop fails it, and +the consumer cannot tell whether it missed a step. Chord's answer is that the owner keeps the flow as **state**, which +re-holds by itself after any drop: + +```ts +// models/contract.ts (sketch from the design) +start: method(obj({ provider: str() }), obj({ id: str() })) // uuidv7 +answer: method(obj({ id: str(), step: int(), value: str() }), Answered) // ok | "step no longer active" +cancel: method(obj({ id: str() })) + +// LoginFlow { id }: state +type FlowState = + | { kind: "running"; step?: { id: number; prompt: LoginPrompt }; notices: Notice[] } + | { kind: "done" | "failed" | "cancelled"; notices: Notice[] }; +``` + +The UI watches the state and renders the current step. A late or second answer to a step that is gone is refused. A +promise-based callback inside the work (pi-ai's `prompt` during a login) stays **inside** the owner: its job keeps the +resolver, and nothing crosses an object boundary. + +Imperative consumers (tests, scripts, a CLI, an agent tool) use a generic client helper over the same state, not a +second protocol: + +```ts +async function* steps(flow: LoginFlowRef, ctx: Ctx) { + let last = 0; + for (;;) { + const s = await until(flow.state, (v) => v.kind !== "running" || (v.step !== undefined && v.step.id > last), ctx); + if (s.kind !== "running") return s.kind; + last = s.step!.id; + yield s.step!; + } +} + +for await (const step of steps(await ctx.get(LoginFlow, { id }), ctx)) + await login.answer({ id, step: step.id, value: answerFor(step) }, ctx); +``` + +`last` keeps steps from being lost or repeated across re-holds. + +:::review | Review: where a flow lives (#11, deferred) +Decision #11, deferred until login is ported: (a) a keyed document of its owner, or (b) an object `LoginFlow { id }` +living until done; tied to #9 (jobs as objects). The `LoginFlow { id }` address above assumes (b). Also undecided is +how "leaving" ends a login (p2 cancels when the starting owner leaves everywhere); with presence it is a dropped +reference, never an unmount (chapter VIII). +::: + +## The Questions service + +The Questions service (`apps/p2/docs/questions.md`) is a p2 design, not a Chord feature, built from Chord's +primitives: a shared store with per-principal views. The asker gets `ask`; viewers get the open questions and +`answer`. + +```ts +export type Question = + | { kind: "text"; message: string; secret?: boolean; help?: string } + | { kind: "select"; message: string; options: readonly { id: string; label: string }[] } + | { kind: "confirm"; message: string; allow: string; deny: string }; + +type Policy = { mode: "first" } | { mode: "quorum"; count: number } | { mode: "all"; join?: "live" | "snapshot" }; + +ask(question, { + audience, // a readable or a list of principals + policy, + key?, // stable per asking flow: a re-ask reattaches to the pending question + deadline?, // then resolve with what is there; `missing` names who did not answer +}, ctx): Promise; +``` + +Its rules: + +- **The store is the truth.** Pending questions and their resolvers live in the store; `open` is a readable over it, + filtered per viewer. Nothing is a document, so a reload or migration cannot leave a ghost question. +- **One settlement point.** Register the waiter and the abort listener before publishing the question; an answer, an + abort or the deadline atomically takes the entry out before resolving. A late or second answer gets "no longer + active". Answers are validated against the stored question (a select id must be one of the options). Never await an + answer inside a transaction. +- **Policies over a live audience.** `first` resolves with the first answer; `quorum(n)` after n answers; `all` when + every member of the audience has answered. With `join: "live"` (the default), someone who joins becomes required and + someone who leaves drops out. +- **Authorization.** The asker is stamped by the host (a worker asks only in its own session's scope; secret questions + only from server askers). Each question stores who may answer it; a guest's send access does not imply approval + rights. Secret answers never appear in `open`, documents or logs. +- **Link drops and crashes.** A rejected `ask` is not a denial. Askers retry with the same `key` until their own signal + aborts; a pending `ask` counts as an active call for keep-alive. +- **Where it lives.** On the host where the asking work runs: session questions on the session's worker, login + questions on the server. UIs reach both through the usual relay. + +The store in Chord v2 terms: + +```ts +host.serve(QuestionStore, ({ scope }, self) => { + const pending = self.open(async () => new PendingQuestions(), (p) => p.rejectAll("store closed")); + const open = self.derive("open", [pending.value.readable], ([all]) => all); // data only, no resolvers + return { open, ask: (args, ctx) => pending.value.ask(args, ctx), answer: (args, ctx) => pending.value.answer(args, ctx) }; +}); + +host.serve(Questions, ({ scope }, principal, self) => { + const store = self.get(QuestionStore, { scope }); + if (isHost(principal)) return { variant: "asker", ask: (args, ctx) => store.ask({ ...args, asker: principal }, ctx) }; + if (!mayView(principal, scope)) throw forbidden(); + return { + variant: "viewer", + open: self.derive("mine", [store.open], ([all]) => openFor(all, principal)), + answer: ({ id, value }, ctx) => store.answer({ id, value, by: principal }, ctx), + }; +}); +``` + +The pending map with its resolvers is in-process state with promises, so it is a native of the store (opened and +closed with it). Its `close` rejects every waiter; askers retry with their key against the new store after a reload. + +:::review | Review: questions.md predates the v2 rules +`apps/p2/docs/questions.md` writes the store as `self.resource(...)` with a `replace` hand-over, which the design +rejected ("resources replaced in place"), and opens Chord call scopes in Durable hooks with `host.root`, which Q6 +replaced with `chord(fn, context)` (chapter XIV). Design §20 says the Questions store "must live in an object (it +predates natives only in objects)". The sketch above is this guide's translation, with the derive per viewer; filtering +per viewer in the store, once per variant, may be cheaper for large audiences. The `ask` signature passes typed +objects as arguments, which is fine (data), but the `audience` readable form cannot cross a link: across links, +audience must be data (principals) or computed by the store. +::: + +## Choosing + +| Case | Use | +| --- | --- | +| a multi-step login in one owner | a flow: the owner's job keeps the resolver, state shows the steps | +| a tool needs the owner's approval | Questions, `first`, audience the owner | +| the agent asks everyone present | Questions, `all`, audience from presence | +| deploy needs 2 of 3 approvals | Questions, `quorum(2)` | +| a notice (auth URL, device code, progress) | not a question: the asking work's own progress state | + +# Extension Points {#registries lede="The facet, registries and the HTTP component: how plugins add behaviour to each other without callbacks, and how the owner stays in charge."} + +The notes plugin wants a download route, `GET /s/:session/notes.txt`. A logs plugin wants to time every request. +Several plugins add model providers, tools for the agent, commands for the TUI. Someone must combine these +contributions, decide conflicts, and replace one plugin's part as a whole when it reloads. Between plugins exactly two +things pass: object types and **registries**. + +## The facet, completed + +The facet has appeared since chapter II as the place that serves types. It is also itself a **pure object** with the +host's lifetime: + +```ts +// notes/server.ts +export default defineFacet((host, self) => { + const prefs = self.state(NotesPrefs); // plugin-wide documents + self.contribute(Http, { routes: { "GET /s/:session/notes.txt": exportNotes } }); + host.serve(NoteCountStore, …, { lifetime: "host" }); + host.serve(NoteCounts, …); +}); +``` + +- It may hold documents, references, routines and contributions; **no natives** (those need an object with its own + lifetime). +- Its identity is the plugin's name, so its persisted documents survive restarts and reloads. +- Served objects go live after the facet and may read and write its documents (chapter V). +- `host.serve` and `self.contribute` are valid only while the setup runs. The facet never `self.get`s a type it + serves. + +## Registries + +A registry is an extension point owned by someone (a Chord component, the app, a plugin). Each contributor +contributes **at most one value** per registry; the table is an immutable, ordered list of `(contributor, value)`; +the registry's `compose` turns it into the value consumers use. + +```ts +// owned by the HTTP component +export const Http = defineRegistry("chord.http", { + version: 2, + accepts: { 1: (old: HttpContributionV1): HttpContribution => upgrade(old) }, + compose: (table, { report }) => buildRouter(table, report), // pure; always delivers a value +}); + +self.contribute(Http, { routes: { … }, wraps: [ … ] }); // a contributor +const router = host.registry(Http).value; // a consumer: live +``` + +**`compose` runs at runtime, whenever the table changes:** after a reload's commit, when a contributor goes live or +ends, when a contribution changes. It is not a staging check: contributions can be dynamic (known only once a native +is open), so no check before a reload could be complete. `compose` always delivers a value with its fallback built in +(e.g. the first by plugin name wins) and reports conflicts through `report`, which become owner notices and telemetry +events. Readers get a plain value and never handle conflicts. Only if `compose` throws is the registry in an error +state, and readers get an error when they use it. Work that started earlier keeps the value it read. + +**Rules:** + +- **Who contributes** (R21): only facets and `lifetime: "host"` objects; only their `self` has `contribute` (a type + error elsewhere). Every object may read registries. +- **A contribution is in the table exactly while its contributor is live:** a facet's while its plugin is loaded; an + object's from `live` until it ends, fails or restarts. +- **Order** is sorted by plugin name: deterministic on every host and across reloads, and deliberately meaningless. + Meaningful order comes from the owner's composition (HTTP phases, below). Helpers: `keyed(table, entriesOf)`, + `ordered(table, itemsOf, orderOf?)`. +- **One authoritative declaration** (Q8): only the defining plugin's facet declares the registry to the host + (`compose`, version, `accepts`). Other plugins' tokens only identify it by name and types. A reload of the defining + plugin replaces the declaration atomically and the table recomposes. If the defining plugin is not loaded, the + registry has no value (readers get an error like a failed reference) and contributions wait in the table. +- **Versions** (Q8): a contribution carries the registry version its plugin was built against; older ones are adapted + by the defining facet's pure `accepts`. One without a matching version or adapter is left out and reported naming + plugin, registry and versions; its plugin keeps working. Contributions are in-process values (often functions) and + are not schema-checked. +- **Use sites guard contributions.** A contribution is untrusted code (third-party plugins are a goal). Every use site + catches throws and rejections when calling an entry, reports naming the contributor, and continues with the others. +- **The owner decides** conflicts, order and selection. Registries add behaviour; they never replace an owner's checks. + +## Natives through registries + +Registries are the only way a native reaches another plugin. An entry lives exactly as long as its contributor; +consumers look up at each use (or follow the composed value) and keep nothing from an entry beyond a use. The models +plugin's host-lifetime object contributes its runtime; the harness looks it up at every request. After a models reload +the next request gets the new runtime, while a running stream finishes on the old one. + +A handler that stores `const r = entry()` in a module variable and uses it for days has a bug: after the contributor +restarts it holds a closed native. + +## Contributions that change + +Some contributions are known only at runtime: MCP tools appear when a server connects. A contribution is either fixed +for its contributor's life (dynamic parts behind handles read at use) or a **readable**, which Chord subscribes to; +on every change Chord replaces that entry and runs `compose` again. For in-process values, `self.writable(initial)` is +a host-local readable of any value (no JSON, no wire, compared by identity) with `set`: + +```ts +const ext = self.writable(mcpExtension([])); // a Durable Extension, no tools yet +self.contribute(HarnessExtensions, ext); +self.routines.on(mcp.tools, (tools) => ext.set(mcpExtension(tools))); +``` + +There is no remove call. An entry leaves when its contributor ends, or after a reload whose new code no longer +contributes. A live contributor with nothing to offer sets its readable to the registry's empty value (an extension +without tools, empty routes); the table keeps exactly one entry per live contributor. + +## Optional dependencies + +Registries also replace optional references (chapter VI). Radius forwards connections to ports the ports plugin +manages, but ports is optional. So radius owns `radius.forwards` and ports contributes its forwards; without ports the +table is empty and radius works. + +## The HTTP component + +HTTP is an optional Chord component (`@earendil-works/chord/next/http`) built on a registry. A route gets a plain +request summary (method, path params, query, principal, provenance such as local or relayed, selected headers, parsed +and capped JSON bodies, uploads in chunks) and a `ctx` whose `principal` is the request's, so `ctx.get(Notes, …)` gives +exactly the view that person may see. The route returns a description; the component does the HTTP work once (ranges +with 206 and 416, HEAD, download headers with a safe filename, `nosniff`, CSP and its nonce, caching, streaming, abort +on disconnect): + +```ts +type HttpResponse = + | { kind: "json"; value: JsonValue } | { kind: "page"; title: string; body: string } + | { kind: "bytes"; name: string; mime: string; size: number; read(offset: number, length: number): Promise } + | { kind: "redirect"; to: string } | { kind: "error"; status: number }; + +// notes/server.ts +self.contribute(Http, { + routes: { + "GET /s/:session/notes.txt": async (req, ctx) => { + const notes = await ctx.get(Notes, { session: req.params.session }); // the requester's view + const text = new TextEncoder().encode(render(notes.notes.value)); + return { kind: "bytes", name: "notes.txt", mime: "text/plain", size: text.byteLength, + read: async (offset, length) => text.subarray(offset, offset + length) }; + }, + }, +}); +``` + +Routes compose with `keyed`: two contributors with one pattern is a reported conflict, and the first by plugin name +wins. **Wraps** are middleware in phases the owner defines (`observe`, `headers`, `transform`, outermost first), +within a phase in contributor order. No numeric priorities: plugins have no single registration order, so Fastify's +lifecycle phases and encapsulation are the model, not Express. + +```ts +// logs/server.ts +self.contribute(Http, { + wraps: [{ + name: "access-log", phase: "observe", scope: "all", // default scope "own": only own routes + wrap: async (req, next, ctx) => { + const t = Date.now(); + const res = await next(req); + log(req, res, Date.now() - t); + return res; + }, + }], +}); +``` + +`route` narrows a wrap to one route; a `route` that does not exist is a reported conflict. Wraps may add a security +boundary on top of an owner's checks, never instead of them. A request uses the router as it was when it started. +Request resolvers (who sent a request: cookie, loopback, relay) are hooks over the summary. + +## Channels and big bytes + +**Channels** are WebSocket byte channels (`send`, `onData`, `close`) for terminals, separate from object links and +with their own acknowledged windows, so a large state message never holds up keystrokes. + +**Big transfers are chunked by the owner,** not by Chord: bytes are a schema type (`bytes()`), carried natively +(never base64), and a 50 MB upload is a protocol of plain calls: + +```ts +beginUpload({ name, size }, ctx) → { upload } +append({ upload, offset, bytes }, ctx) // about 1 MiB per chunk; the awaited call is the backpressure +finishUpload({ upload }, ctx) → { id } +read({ id, offset, length }, ctx) → bytes +``` + +After a link drop the `append` in flight fails like any call; the writer asks the owner for the stored offset and +resumes. Unfinished uploads expire through the owner's routine. Chord has no stream type: it would need its own rules +for drops, cancellation, relays and restarts, which this offset protocol already survives. + +:::note | Built so far +Registries are layer L9. The HTTP component, channels and the chunked-transfer helper are deferred until after the +kernel and a first real proof (the harness, one tool with an approval pause, a code mode reload); until then the +playground serves its pages with plain app code. Their design stays the target. +::: + +# The Agent {#agent lede="pi-durable inside Chord: the harness as a core object, models, environments and tools through registries, and how a tool reaches Chord objects and people."} + +Each Studio worker runs a pi-durable harness for its session. The harness must run for the worker's whole life, +whether anyone watches or not. Its models come from the models plugin, its execution environments from the envs +plugin, its tools from every plugin, and all of those reload independently. This chapter assumes you know pi-durable +and shows only how it plugs into Chord. + +## Shared foundations + +Durable and Chord v2 share modules: Durable imports Chord's `context`, `delta`, `json`, `output` and `sqlite`, and v2 +imports the same, so `Context`, `Draft` and `JsonValue` are one type each. A v2 readable is today's +`replicatedState` (`chord/src/state.ts`), so Durable's `viewState()` and document watches can be exposed as Chord state +members without conversion. Only plain data crosses between Durable and Chord objects. + +## The harness is a core object + +```ts +// studio/src/core/harness.ts (base code) +import { Harness as DurableHarness } from "@earendil-works/pi-durable"; + +export const ChordHostKey = createContextKey>("studio.chord-host"); +export const HarnessModels = defineRegistry("studio.harness-models", { version: 1, compose: firstModels }); +export const HarnessEnvironments = defineRegistry("studio.harness-envs", { version: 1, compose: firstEnv }); +export const HarnessExtensions = defineRegistry("studio.harness-extensions", { + version: 1, + compose: (table, { report }) => extensionSnapshot(table, report), // one Durable Extension per plugin +}); +const HarnessRuntime = defineRegistry<() => DurableHarness, () => DurableHarness>("studio.harness-runtime", { + version: 1, compose: onlyEntry, // core-private: one contributor +}); + +host.serve(Harness, (_input, self) => { + const info = self.get(SessionInfo, {}); + const harness = self.open(async ({ signal }) => { + const { id, dataDir } = info.init.value; + const base = withContextValue(ChordHostKey, host, withAbortSignal(signal, BACKGROUND_CONTEXT)); + return DurableHarness.open(await openNodeSqliteStorage(join(dataDir, id, "durable.sqlite")), { + models: () => host.registry(HarnessModels).value, // read at each use + env: (target) => host.registry(HarnessEnvironments).value(target), + settings: () => currentSettings(), + registry: durableRegistry(host.registry(HarnessExtensions)), // a RegistryReader over the composed value + }, base); + }, (h) => h.close(BACKGROUND_CONTEXT)); + self.contribute(HarnessRuntime, () => harness.value); // for core's conversation objects + return { … }; +}, { lifetime: "host" }); +``` + +- **Lifetime `"host"`:** it runs for the worker's whole life, also while nobody holds it, because Durable tasks + continue. +- **Base code:** it lives in the core facet, so a plugin reload never restarts it; only a base-code change (a host + restart) does. +- **Registries, looked up at each use:** Durable's `models` and `settings` accept functions (pi `7ec1f648e`), so each + generation gets the current value. After a models reload the next request uses the new runtime; a running stream + finishes on the old one. +- **Durable's storage is its own.** Chord documents never commit together with Durable transcripts. + +The design writes the call as `Durable.open(storage, { … })`; pi-durable's actual entry point is +`Harness.open(storage, options, context)`, used here as `DurableHarness` to keep it apart from the core object +`Harness`. + +## Start order + +At host start Chord loads every facet and waits until every facet and every `lifetime: "host"` object **other than the +harness** is live or failed: the **start barrier**. Then the registries hold the contributions of every plugin that +loaded, and the harness opens and resumes scheduling (R17). An object that must contribute at start, like the model +runtime, is therefore `lifetime: "host"`. Nothing waits for specific contributions; what is missing shows where it is +used: + +| Missing | What happens | +| --- | --- | +| a tool | its call gets an error result the model sees | +| a model | the generation reports "model unavailable" and waits, not fails, until a model appears | +| a task definition | the task stays blocked (`missing_task`) until the plugin is installed or the task is aborted | +| a plugin that failed | reported with its error; a facet in backoff contributes later and the registries recompose | + +The same holds when a session's database is copied to an instance with other plugins: missing tools, sections and +hooks are skipped, documents of missing plugins stay untouched, stored selections are never rewritten. The UI must +show blocked tasks with their reason and offer to abort them; a turn waiting on one hangs until then. + +:::review | Review: the harness exception to the barrier (C17, R17) +R17 makes the harness start after the barrier, a special case in the barrier's rule. Review round 3 proposed cut C17: +no harness-specific exception; the harness is an ordinary object the app's start code holds after `load` resolves. +Not decided. `chord-next-walkthrough.md` concept 28 still says the harness's activation *is* the start barrier, which +R17 replaced. +::: + +## Tools that call Chord + +Durable tools, hooks and sections receive Durable's `ToolExecutionApi` (or `HookApi`) and a `Context`, never a Chord +`ctx` (Q6). Core put the host into the harness's base context under `ChordHostKey`, and every phase context inherits +it. A core helper opens a Chord call scope from any Durable context: + +```ts +// studio/src/core/chord.ts +export function chord(context: Context, fn: (ctx: Ctx) => Promise): Promise { + const host = context.value(ChordHostKey); + if (!host) throw new Error("not inside the Studio harness"); + return host.root(fn, { signal: context.abortSignal, telemetry: context.telemetry }); +} +``` + +The scope runs as the host principal; attribution is data; references taken with `ctx.get` are released when `fn` +settles. There is no wrapping of tools and no app-specific `defineTool`. The notes plugin gives the agent a tool: + +```ts +// notes/worker.ts +const addNote = defineTool({ + name: "notes_add", // tools are named _ + description: "Add a note to this session's notes panel", + parameters: Type.Object({ text: Type.String({ maxLength: 2000 }) }), + execute: (args, api, context) => chord(async (ctx) => { + const info = await ctx.get(SessionInfo, {}); + const store = await ctx.get(NoteStore, { session: info.init.value.id }); + const id = await store.add({ text: args.text, by: `agent:${api.conversationId}` }, ctx); + return { output: [{ type: "text", text: `added note ${id}` }] }; + }, context), +}); + +export default defineFacet((host, self) => { + self.contribute(HarnessExtensions, defineExtension({ name: "@studio/notes", tools: [addNote] })); + … +}); +``` + +A reload of the notes plugin replaces its extension in the table; Durable pins a registry snapshot per task phase, so +a tool call already running finishes on the old code and the next phase uses the new one. + +## Tools that need a person + +Clients are not addressable, so a tool cannot call the user's tab. A tool that needs the user goes through an owner's +state: the Questions service (chapter XII). The model-facing `ask_user` tool and an approval hook both call +`Questions.ask`; tabs render the open questions and answer them. + +```ts +hook(ToolTask, { + beforeTool: async (call, api, context) => { + if (!needsApproval(call)) return undefined; + const decided = await api.memo("approval", context); // a rerun does not ask again + if (decided !== undefined) return decided ? undefined : { block: "Denied" }; + const allowed = await chord(async (ctx) => { + const q = await ctx.get(Questions, { scope: { session } }); + for (;;) { + try { + const r = await q.ask( + { kind: "confirm", message: `Run ${call.name}?`, allow: "Allow", deny: "Deny" }, + { audience: approvers, policy: { mode: "first" }, key: api.taskId }, + ctx, + ); + return r.answer; + } catch (error) { + if (ctx.signal.aborted) throw error; // a link blip is not a denial: retry + await backoff(ctx); + } + } + }, context); + await api.memo("approval", allowed, context); + return allowed ? undefined : { block: "Denied by the user" }; + }, +}); +``` + +Durable runs `beforeTool` in the tool task's call phase; after the execute checkpoint a crash retries execution +without hooks, so the stored decision covers that retry. The approval must cover the final arguments: put the +approval hook after hooks that rewrite arguments. + +## Conversations for the UI + +Chat panels need a conversation's live view, submit, steer, stop. The design leaves p3's conversation API to the p3 +design; one construction that follows the rules: a core `ConversationStore { conversation }` (local) opens the Durable +watch as its own native, getting the Durable harness from the core-private `HarnessRuntime` registry contribution (a +native may reach another object only through a registry entry), and exposes a bounded window as a derive. Per-principal +`Conversation { session, conversation }` views in front of it check who may send (chapter XIX). + +## Code mode + +Code mode is a plugin. It owns its `pi-codemode` sandbox as a native; its tool entry looks up the **current** sandbox +when `execute()` starts, so a call pinned by a phase before a reload (waiting in an approval hook, say) starts on the +new sandbox. On `reload` and `stop` the old sandbox is sealed and drained in `close`: running scripts finish or hit +their deadline, for at most 30 s (Q7). After a worker stop Durable replays as it does today. + +## What lives where + +| Data | Where | +| --- | --- | +| transcript-bound: entries, documents that fork and rewind with a conversation | Durable | +| app state: session list, notes, drafts, grants, settings | Chord documents | +| data core enforces on conversations (read-only one-shots, cwd followers) | a core per-conversation policy document, written when the conversation is created (`createConversation({ init })`) | + +**Telemetry:** pass the host's adapter as `HarnessOptions.telemetry`; Durable stores trace carriers on submissions and +tasks (chapter XVII). + +# Reload and Rollout {#reload lede="What happens when you press Reload: per host, per object, and across the server, the workers and every tab."} + +Back to the preface. Mario changed the notes plugin's sort order and pressed Reload. Everything the earlier chapters +asked of plugin authors (synchronous factories, natives only in objects, routines instead of timers, data in +documents) exists so that this moment is safe. This chapter shows what Chord does with it. + +## On one host + +```text +1. load new bundles; an import or syntax error rejects +2. staging facet setups and every live object's new factory run with a recording self + (declarations only, nothing activated); any throw rejects +3. commit the host switches to the new code atomically +4. after per object: pure objects swap, native objects restart +``` + +**Rollback is all-or-nothing per host.** Any load or staging error leaves the whole host on its previous code, so a +host is always completely at one revision and its status is one value. Runtime states (a registry conflict, a failed +open, a migration) are not reasons to roll back; after the commit, errors stay with one object. There is no factory +trial, no migrate dry run, and no consumer contract check in staging: an incompatible consumer fails at bind with +`contract_mismatch` and a notice naming plugin and type. + +:::steps | Reloading @studio/notes on worker S1, object by object ++++ Load and stage +The new bundles load. The facet setup and the factories of `Notes` (three views), `NoteStore { S1 }` and `NoteInbox` +run with a recording `self`: they declare, nothing activates. Nothing throws, so the host commits to the new code. ++++ NoteStore: pure, swapped in place +`NoteStore` has no `self.open`. Its new methods and derives take over; `NoteList`, the drafts, its references and the +running import job stay. Routines restart, serialized: the new `cleanup` waits for the old handler in flight. The gate +opens. ++++ Running calls finish on old code +An `add` that was running when the swap happened finishes on the old code with its old handles; its transaction still +commits. Only new calls get the new handles. Exception (R15): if the reload raised a document's version, old code that +touches the migrated document throws, the document stays intact, and that call fails once. ++++ The import job keeps its old run +The job's `run` keeps the code it started with. When it settles, the **new** `finish` records the outcome. ++++ The views: pure, swapped +The three `Notes` views swap in place. Owner, Ann and Bob keep their references; the reader derive is recomputed with +the new sort and the difference is sent. Nobody sees a reconnect. ++++ NoteInbox: native, restarts +`NoteInbox` has a watcher. Its running work ends `aborted`, the watcher closes with reason `reload`, the new factory +runs, the watcher reopens. Its documents would stay. Whether a file dropped during the restart is seen is the +watcher's business: Studio's `watchDir` scans the directory once when it opens. +::: + +**Why natives restart instead of being handed over:** a native has no line, no transactions and no schema. If old and +new code shared it, they would interleave on it unchecked. In practice restarts are cheap: the Durable harness is base +code, terminal PTYs live in a separate terminal host process, and model runtimes, sandboxes and a search index rebuild +in milliseconds. Only reloading the plugin that owns SSH connections (running remote commands die) or the relay socket +(relayed connections drop) hurts. + +**Removing a type:** its objects end (routines stop, natives close, references end with an error); in-memory documents +go with them; persisted ones stay until retired. **Consumers elsewhere** see a restarting object like any object that +briefly goes away: `loading`, parked calls, then `live`. + +**UI facets reload too.** A reload unmounts and remounts components. That is why "leaving" (abort a login, close an +overlay) is never derived from an unmount (chapter VIII). + +## Across hosts + +A **revision** is (server incarnation, sequence) and names the complete desired code set: every facet with its +digest, not a batch of changes. Every install reconciles everything missing, and a host reports a revision only when it +runs that complete set. A revision also carries the code-set digest (a hash over the facet bundles' content hashes), +so a server restart with unchanged code triggers no reinstall. + +The order is **server, workers, clients**. Each host reports `installed`, `installed_failed` (with the objects that +failed) or `rejected` (with the reason; it stays on its previous code). A server that rejects stops the rollout. A +host that starts or reconnects installs the current revision before anything is routed to it. Installs are serialized +per host and bounded (60 s). + +Calls keep flowing during a rollout. When a call reaches a provider on another revision, the mismatch response +carries the provider's revision and status: + +| Provider | The call | +| --- | --- | +| ahead, or behind and installing | `updating`: values readable, calls park (with the park deadline) | +| rejected | fails at once; retried on the next revision | +| same revision | `contract_mismatch`: a real mismatch, final | + +A host checks its own install status first: if its own install was rejected, a call fails at once with the reason +instead of waiting on a provider. Calls parked on a binding that is released end with an error and are not resent. +There are no cross-host transactions and no votes. + +:::sequence animate | A tab calls into a worker that is still installing. The call parks and runs once the worker reports the revision. +actor tab "tab" "revision 8" +actor server "server" "revision 8" accent +actor worker "worker S1" "installing 8" blue +server => worker: install revision 8 +tab -> server: call Notes.add (new contract) +server -> worker: relay +worker --> server: updating (rev 7, installing) +server --> tab: updating +tab -> tab: park (deadline 60 s) +worker --> server: installed 8 +server --> tab: provider at 8 +tab -> server: call Notes.add (resent from park) +server -> worker: relay +worker --> tab: result +::: + +## Start barrier + +At host start and first load there is one more rule (chapter XIV): load every facet (the core facet with its start +data among them), wait until every facet and `lifetime: "host"` object other than the harness is live or failed, then +resume dependent work. Plugin reloads need no barrier. + +## Loading and the dev loop + +- **Roots:** plugins come from built-in, global (`/plugins`), package (npm or git, installed with + `--ignore-scripts`) and project roots, found by directory listing; no source parsing. The server selects which load. + Dependencies between plugins resolve by contract name at runtime. Names are prefixed: objects `.`, + tools `_`. +- **Loader D:** one base bundle per surface (host packages and the app runtime) plus one script per facet, with host + packages passed in (no module map). Old generations are reclaimed by the garbage collector; a stable filename per + (facet, digest), and V8's compilation cache off on hosts that reload in place. The loader is an adapter: static + imports (tests, Durable Objects) work the same, with the app passing the plugin name to `load`. +- **Dev loop:** a source watcher marks plugins dirty and never applies anything. A reload control (the playground's web + shell, a CLI, an agent tool such as p2's `reload_plugins`) reloads the changed plugins, or restarts all hosts for + base code. Reloads are always explicit. + +:::review | Review: project plugins (#14, deferred) +Per-directory plugins with trust prompts and snapshots (`apps/p2/docs/plugin-loading.md`, still the reference for +roots, trust and scopes) need their own pass over this design. Until then, treat the project root as designed but not +settled. +::: + +:::note | Built so far +Reload and rollout are layer L10, the last before telemetry. The frozen kernel has a reload path with host values and +a development freeze, both removed from the design. +::: + +# Changing Contracts {#contracts lede="How a contract gets a new version while old tabs, old workers and old plugins are still talking to it."} + +Chapter XV ended with a tab that calls into a worker on another revision. That case is short-lived: the rollout +finishes and both sides run the same code. Other cases last longer. A phone kept a tab open overnight with yesterday's +web bundle. A plugin from another author was built against `Notes` version 1 and is not updated yet. In both, a +consumer holds a contract that the provider no longer has. This chapter is about what the provider does then. It is +also the one part of Chord v2 that is built: everything in the sections on `evolve`, `checkAccepts` and fingerprints +runs today in `packages/chord/src/next/contracts/`. + +## What has a version + +Everything that is stored or crosses a boundary has a version. Each surface handles an old version its own way: + +| Surface | Version | When an old one shows up | Chapter | +| --- | --- | --- | --- | +| object type (contract) | `version` plus fingerprint | `evolve` and `checkAccepts` on the provider; otherwise `contract_mismatch` | this one | +| document | `version` plus schema fingerprint | pure `migrate` on access; a schema change without a bump fails the object `broken` | V | +| job | `version` | `accepts` on outcomes and `running`; `blocked` without an adapter | XI | +| registry | `version` | `accepts` adapts contributions; one without a match is left out and reported | XIII | +| native | none | the object restarts on reload | IX | +| Durable tasks and documents | Durable's `version` | Durable's `migrate` | XIV | +| link protocol | protocol version | refused; hosts update together, a web page reloads | XVII | +| UI components, Chord and app APIs | none yet | exposed as they are; an SDK will version them later | | + +The rule behind every row is the same as for documents: **every change bumps the version**, additive ones included. +Chord never guesses compatibility from the shape of two schemas. + +## A history, not a file of versions + +A contract history starts with a **base**: a full `remote()` or `local()` declaration with an explicit `version`. The +`Notes` contract from chapters II and III is a base at version 1. Studio now wants pinned notes: notes get a `pinned` +flag and editors get a `pin` method. Instead of editing the declaration, the plugin continues the history: + +```ts +// notes/contract.ts +import { bool, evolve, nul } from "@earendil-works/chord/next"; + +export const NoteV2 = obj({ id: str(), text: str({ max: 2000 }), by: str(), at: int(), pinned: bool() }); +export const PublicNoteV2 = obj({ id: str(), text: str(), at: int(), pinned: bool() }); + +export const NotesV2 = evolve(Notes, (e) => + e.state("editor", "notes", obj({ items: arr(NoteV2) })) + .state("reader", "notes", obj({ items: arr(PublicNoteV2) })) + .addMethod("editor", "pin", obj({ id: str(), pinned: bool() }), nul()), +); +``` + +`evolve(prev, (e) => …)` appends one version to `prev`'s history: `NotesV2.version` is 2, and `NotesV2` still knows +version 1 exactly as it was declared. Every version is **materialized and fingerprinted at declaration**. That is what +makes old code safe: an old plugin bundle that contains its own copy of `Notes` version 1 computes the same +fingerprint as the history inside `NotesV2`. There is no build step, no generated file and no fingerprint copied by +hand. + +The edits: + +| Edit | Changes | Kind | +| --- | --- | --- | +| `e.state(variant, member, schema)` | one state member of one variant | remote | +| `e.addState(variant, member, schema)` / `e.removeState(variant, member)` | adds or removes a state member | remote | +| `e.state(member, schema)`, `e.addState(member, schema)`, `e.removeState(member)` | the same, flat | local | +| `e.args(name, schema)` / `e.result(name, schema)` | one side of a method | both | +| `e.method(name, args, result)` | both sides of a method | both | +| `e.addMethod(variant, name, args, result)` | adds a method (or an existing one to another variant, same signature) | remote | +| `e.addMethod(name, args, result)` | adds a method | local | +| `e.removeMethod(name)` | removes a method from every variant | both | + +Edits chain. The `method(...)` wrapper appears only in base declarations; edits take the schemas directly, so a +method without a result is added with `nul()`. The rules of chapter III still hold across versions: a method has one +signature in every variant, and a member's kind never changes, not even after it was removed and added again. + +**What `evolve` cannot do:** change the input or the set of variants. Both change what an object *is* (its identity, +who gets which members), and no adapter can turn one into the other. That needs a new history base, below. + +## The provider says what it accepts + +`NotesV2` says what changed. It does not say how version 1 consumers are served; the **provider** decides that, with +mappings that live in its own code, never in the contract. The worker facet declares them with `checkAccepts`: + +```ts +// notes/worker.ts +import { checkAccepts } from "@earendil-works/chord/next"; + +const toV1 = ({ pinned, ...note }: Static) => note; +const toPublicV1 = ({ pinned, ...note }: Static) => note; + +const notesAccepts = checkAccepts(NotesV2, { + 1: { + state: { + editor: { notes: (current) => ({ items: current.items.map(toV1) }) }, + reader: { notes: (current) => ({ items: current.items.map(toPublicV1) }) }, + }, + methods: {}, // add and remove did not change; pin is new + }, +}); +// notesAccepts.versions: [{ version: 1, fingerprint: Notes.fingerprint }] + +host.serve(NotesV2, notesFactory, { accepts: notesAccepts }); // working name for the objects-layer option +``` + +`checkAccepts` is **pure**: it checks the mappings without calling them and returns only the version metadata of the +old versions it lists. A provider that lists nothing accepts only its current version. The objects layer later +compares a consumer's version and fingerprint against this list. + +The mappings point in the direction the data flows: + +| Member in the old version | Mapping | Direction | +| --- | --- | --- | +| state, changed | `(currentMember) => oldMember` | provider's current value to what the old consumer expects | +| state, removed | `(currentVariantState) => oldMember`, or `"refuse"` | receives the whole current state of that variant | +| method arguments, changed | `args: (old) => current` | old caller's arguments to current ones | +| method result, changed | `result: (current) => old` | current result back to the old shape | +| method, changed or removed | `{ call: async (oldArgs, ctx) => oldResult }`, or `"refuse"` | replaces the method for old callers | +| member added since | nothing | an old consumer never asks for it | + +A changed state member cannot be refused: the old consumer still has that member and must get a value. A removed one +can be computed from what the variant has now, or refused. `e.method(name, args, result)` replaces both sides and +therefore needs both mappings, even if one side stayed the same (then an identity mapping). + +**Required, at type level and again at runtime.** For every member that exists in the old version and changed or was +removed since, the mapping is required by the type of `checkAccepts`. Because plugins are bundled without type +checks, the same check runs at declaration: a missing mapping, a key for a member that needs none, an unknown version +or a wrong shape throws, naming the member's path (for example `accepts[1].state.reader.notes`). An absent `state` or +`methods` section counts as empty and may be left out only when nothing there needs a mapping. + +**Conservative with no-op edits.** An edit that sets a schema to what it already was, or changes a member and then +changes it back, still requires an identity mapping. TypeScript erases constraints such as `{ max: 2000 }` from the +static type, so two schemas that look the same to the compiler may differ; Chord does not try to tell. + +### Old versions map straight to the current one + +Studio later replaces `remove` with `archive`, which keeps the note in a trash list: + +```ts +export const NotesV3 = evolve(NotesV2, (e) => + e.removeMethod("remove").addMethod("editor", "archive", obj({ id: str() }), nul()), +); + +const notesAccepts = checkAccepts(NotesV3, { + 1: { + state: { editor: { notes: (c) => ({ items: c.items.map(toV1) }) }, reader: { notes: (c) => ({ items: c.items.map(toPublicV1) }) } }, + methods: { remove: { call: async ({ id }, ctx) => { await archiveNote(id, ctx); return null; } } }, + }, + 2: { methods: { remove: { call: async ({ id }, ctx) => { await archiveNote(id, ctx); return null; } } } }, +}); +``` + +Each entry maps **one old version directly to the current one**, not through the versions in between. Version 1 +needs the state mappings (its notes have no `pinned`) and the `remove` replacement; version 2 needs only the +`remove` replacement, and leaves `state` out because its state is unchanged. A member that an old version never had +needs nothing for that version, even if it was added, changed and removed later. Dropping support for version 1 is +deleting its entry: version 1 consumers then get `contract_mismatch`. + +## Binding an old consumer + +A hold carries only the consumer's contract **version and fingerprint**, never a description of the contract (R6). +The provider decides with its own schemas: + +:::flow wide animate | What a provider does with a hold. The decision uses only the provider's own history; a peer's schema is never an input. +row "The hold arrives" +box hold "hold" "version 1, fingerprint F1" blue +box cur "equal to current?" "version and fingerprint" +box acc "accepted old version?" "F1 from the provider's own history" +row "Outcome" accent +box live "bind" "current members" +box adapted "bind adapted" "through the mappings" accent +box mismatch "`contract_mismatch`" "or updating, during a rollout" muted +arrow hold -> cur +arrow cur -> live "yes" +arrow cur -> acc "no" +arrow acc -> adapted "yes" +arrow acc -> mismatch "no" +note "A mismatch fails only that consumer's binding. The provider object and its other bindings are not affected." +::: + +Once a version 1 tab is bound, it is served as if version 1 were current: + +:::steps | A version 1 tab on a version 3 worker ++++ Bind +The tab holds `Notes { S1 }` with version 1 and fingerprint F1. The worker's history computes F1 for version 1, and +`notesAccepts` lists version 1, so the hold binds adapted. The tab gets the `editor` variant as version 1 defined it: +`notes`, `add`, `remove`. It never sees `pin` or `archive`. ++++ State through the mapping +Chord serves the adapted state like a derive of the provider's state: it applies the mapping to the current value +and keeps the result per binding. The snapshot the tab receives has no `pinned` fields. ++++ A commit +The owner pins a note from a current tab. The worker sends the current tab the draft's ops. For the version 1 tab, +Chord applies the mapping to the new value and sends the difference to its last adapted value (`diffRevisions`). The +mapping produced nothing new for a pin, so the version 1 tab receives no change at all. A large list is never resent +whole on every commit. ++++ A call +The tab calls `remove({ id })`. Its arguments are checked against the provider's own version 1 schema, then the +`call` replacement runs `archiveNote` on version 3 code. Its `null` result is what version 1 expects. Had the method +only changed, `args` would have mapped the arguments forward and `result` the result back. ++++ The worker moves on +A later revision drops the version 1 entry. When the tab re-holds (or the worker restarts and the tab binds again), +the hold gets `contract_mismatch`, and Studio's panel shows "reload to update" instead of failing silently. +::: + +## A new history base + +Some changes cannot be adapted. Studio decides that notes belong to a project, not a session: the input changes from +`{ session }` to `{ project }`. That is a new base, written in full, with an explicit version that continues the +published numbering: + +```ts +export const NotesV4 = remote("notes", { + version: 4, // explicit: the published history used 1 to 3 + input: obj({ project: str() }), + placement: () => Server, + variants: { + editor: { state: { notes: obj({ items: arr(NoteV2) }) }, methods: { add: method(…), pin: method(…), archive: method(…) } }, + reader: { state: { notes: obj({ items: arr(PublicNoteV2) }) } }, + }, +}); +// checkAccepts(NotesV4).versions is empty +``` + +A new base has no history before its version and **cannot accept older versions**, not even for an additive change. +Every consumer built against versions 1 to 3 gets `contract_mismatch` and must update. Later `evolve` calls continue +from version 4. The same applies to a change of the variant set. Do not spread an older declaration into a new one or +copy its fingerprint; the contract objects are branded and `evolve` and `checkAccepts` reject copies. + +## Local contracts + +`local` contracts (chapter IV) evolve the same way, with a flat members map and no variants: + +```ts +export const SessionInfoV2 = evolve(SessionInfo, (e) => e.state("init", InitV2)); +checkAccepts(SessionInfoV2, { 1: { state: { init: (current) => toInitV1(current) }, methods: {} } }); +``` + +Same-host consumers of a `local` type are usually reloaded together with their provider, but not always: another +plugin on the same host may be built against an older version. The rules are identical; only the variant argument is +missing. + +## Implementing a contract + +`Implementation` is the type of everything a provider must supply for the current version: one state +getter per member and variant, and one asynchronous method per method, each receiving `(args, ctx)`: + +```ts +const impl: Implementation = { + state: { editor: { notes: () => list.value }, reader: { notes: () => publicList.value } }, + methods: { add: async ({ text }, ctx) => addNote(text, ctx), remove: async ({ id }) => removeNote(id), pin: async (a) => pin(a) }, +}; +``` + +Keep declaration types **inferred** (no annotation on `NotesV2`): the inferred type carries the complete history, and +`checkAccepts` needs it to know what changed. + +:::review | Review: L1's implementation shape and the factory's return value +`Implementation` in `src/next/contracts/types.ts` is `{ state: { variant: { member: getter } }, methods }`. The +design's factories (`design.md` §6, used throughout this guide) return `{ variant, member: value, method: fn }`. +The objects layer (L4) has to connect the two, and also define how `serve` receives the checked accepts (this guide +uses the working name `{ accepts }` as a third argument) and what `"refuse"` does at runtime. The natural reading, +used in the stepper above, is that a refused method call and a hold that needs a refused state member end with +`contract_mismatch`; no source says so. +::: + +## The fingerprint + +The fingerprint is a 64-bit hash (`hash64`, portable across runtimes) of a **normal form** of the contract's +schemas. The normal form decides what counts as a change: + +| Ignored | Significant | +| --- | --- | +| order of object keys, `required` and `enum` (sorted) | order of union and intersection branches (`anyOf`, `oneOf`, `allOf`) and of tuples | +| annotations on schema nodes: `title`, `description`, `examples`, `$comment`, `deprecated` | defaults, formats, patterns, bounds, `$ref` names | +| placement | input, variants, member kinds and every member schema | + +Union order is significant because it is meaning: TypeBox picks defaults and decodes by the first matching branch. +Annotations are dropped only on schema nodes; an object property that happens to be named `title` is data and counts. + +Only data schemas are allowed. One walker copies, validates and normalizes each schema against a **whitelist of +TypeBox keywords**: primitives, arrays and tuples, objects and records, unions and intersections, `$defs` and `$ref` +(use `$ref` for recursive types), constraints and annotations. Conditional and dependent schemas, `not`, +`definitions`, `contentSchema`, `unevaluated*`, `contains`, `propertyNames`, unknown keywords, executable or custom +TypeBox features, arbitrary `Unsafe` and refinements, accessors, symbols and cyclic objects are rejected at +declaration. Bytes are the one special leaf: `bytes({ min, max })` checks a `Uint8Array` and has the data marker +`{ type: "bytes", minLength?, maxLength? }`, so it hashes like any schema; TypeBox's checking still works on it. + +Schemas are deep-copied into a private, frozen history; your own schema objects are copied, not frozen. +`NORMAL_FORM_VERSION` is part of every digest, so a change of the normal form is a link protocol change. + +**Not a security boundary.** A fingerprint says "we were compiled against the same contract"; it proves nothing. A +hostile client can send any fingerprint it likes. Providers therefore always validate arguments with their own +schemas (chapter IV), whatever the fingerprint says. + +## Rules of thumb + +- Treat a published contract like a published API: every change is `evolve`, every `evolve` needs a decision about + old consumers. +- Keep mappings small and pure. A state mapping runs for every commit and every adapted binding; make it a map over + the value, not a query. +- Accept the versions your users still run (one or two back during a normal rollout), and delete entries when the + metrics show no old bindings. +- Change the input or the variants only with a new base, and only when updating every consumer is acceptable. +- Contract, document and job versions are separate and each is decided on its own. Pinned notes needed `NotesV2` + for the wire and `NoteList` version 2 with a `migrate` that adds `pinned: false` (chapter V), but no new + `ImportJob` version, because its outcome did not change. + +:::note | Built so far +This chapter's contract layer is built: `packages/chord/src/next/contracts/` (commit `7d2bb8ea8`, about 1,300 lines +with `remote`, `local`, `method`, `evolve`, `checkAccepts`, `Implementation`, the schema helpers, `fingerprintOf`, +`schemaNormalForm` and `NORMAL_FORM_VERSION`), with nine test files covering evolution, fingerprints, the hash, +local contracts, long chains, schemas, types and review findings. It stands alone: it hosts nothing, calls no +mapping and does not import the frozen runtime. Binding by version and fingerprint, adapted state and calls belong to +the objects and protocol layers, not built yet. +::: + +# On the Wire {#wire lede="What travels over a link, what a relay does with it, how big things are kept in check, what each hop costs, and how to see it in traces."} + +Most plugin code never thinks about the wire: it declares state, commits transactions and calls methods, and Chord +moves the bytes. You need this chapter when something is slow, when a value is large, when a link misbehaves, or when +you want to read a trace. It also explains why the earlier chapters insisted on small operations, shared structure and +no callbacks: each of those rules removes a cost on some hop. + +## What a state member sends + +Every exposed document, derive or readable is a **state member**. Over a hold it produces one of these: + +| Update | When | +| --- | --- | +| `snap` | on hold and on re-hold: the whole value, once | +| `ops` | per commit: the draft's recorded ops, no diff | +| `ops` from a diff | per change of a derive (and of an adapted binding, chapter XVI) | +| throttled `ops` | job progress | +| `reset` | a migration commit replaced the value (adapted for older consumers) | +| `error` | the member failed | + +**Commits are atomic for consumers.** On the same host, subscribers are notified once, after every document of the +commit is adopted. Remotely, one `state` message per hold carries a batch of member updates that is applied before +listeners run, and a commit that touches several holds on one link travels as **one** `state` message on that link. +A consumer never sees half a commit. + +## Messages + +A message is a **header plus opaque parts** (R1). There is one part per hold the message concerns; the header says +which hold each part belongs to and how large it is. + +| Direction | Kind | Parts | +| --- | --- | --- | +| consumer to provider | `hold`, `release`, `call`, `cancel` | `call` has one (the arguments) | +| provider to consumer | `bound` | one: the snapshot | +| | `state` | one per hold the commit touched on this link | +| | `result` | one | +| | `gone` | none | + +There are no request ids in application code. Hold ids and call ids exist only inside the protocol, per link. + +**One binary format on every link (Q4):** `u32 header length | header JSON | part bytes`, as one `Uint8Array`. A +WebSocket sends it as a binary frame, IPC passes it with `serialization: "advanced"`, a memory link delivers it +asynchronously. Encoding, framing, header checks, id resolution, flow control and size limits are one shared protocol +layer; link adapters only move bytes. Tests on memory links therefore run the production path. + +A part is encoded **once, at its origin**, with that hold's own Delta encoder (paths are interned per hold, end to +end between provider and consumer). A JSON part above a threshold (about 16 KiB) is compressed once at its origin, +whatever the link; byte segments stay raw; relays forward compressed parts as they are; the header names the codec. + +**The header is a TypeBox schema (R4)**, a union tagged by `kind`; the protocol types derive from it. Headers from +untrusted links (client WebSockets) are checked with the schema compiled once; trusted links (IPC between server and +workers) are not checked. **Unknown ids are ignored on both sides (Q1).** They are normal races: a `result` crossing a +`cancel`, a `call` crossing a `gone`, a state part crossing a `release`. The side that ended a hold or call settles +its own pending work: a consumer that receives `gone(h)` rejects its pending calls on `h`, and the order per +direction guarantees nothing else for `h` follows. Only a header that fails the schema is a protocol error, and it +closes that link. + +The link protocol has a version. Hosts with different protocol versions refuse each other; a web page reloads to get +the server's protocol. + +## Through a relay + +A tab never talks to a worker directly (chapter I). The server **relays**, and a relay reads and rewrites only +headers: it maps hold and call ids between its two links, stamps the principal on `hold` and `call` (a client's +principal field is always overwritten; only a trusted parent link may carry one), and groups parts per destination +link, so each link gets only its own holds' parts. It never parses a payload. + +:::sequence animate | The owner adds a note while Ann watches. The commit leaves the worker as one `state` message with two parts; the server splits it by destination without decoding either part. +actor owner "owner's tab" "hold h1" blue +actor ann "Ann's tab" "hold h2" muted +actor server "server" "relay" accent +actor worker "worker S1" "NoteStore, Notes" +owner -> server: call c7 Notes.add (1 part) +server -> server: map ids, stamp owner +server => worker: call c3 (part forwarded as is) +worker -> worker: method runs, self.tx commits +worker => server: state: parts for h1', h2' +server -> server: group parts per link +server --> owner: state (h1 part) +server --> ann: state (h2 part) +worker --> server: result c3 +server --> owner: result c7 +::: + +The `state` message reaches the owner's tab before the `result`, so when `await notes.add(…)` resolves, the new note +is already in `notes.value`. Ann's view is a different variant (reader) and gets its own part from its own derive; the +relay neither knows nor cares. + +## Flow control + +Each link has **one byte window**, counted from each message's `byteLength` (parts are already encoded bytes; nothing +is measured or stringified to count it). The window covers every message, first snapshots included. + +- **Snapshots are always admitted (R19)**, even when larger than the window; they use it up. Whether a hold + **overflows** is decided on the changes queued after the snapshot. +- **Under backpressure** a commit's ops are collected per hold and encoded into one part when credits return. The + consumer still gets the commit atomically. +- **After an overflow** the hold is dropped and established again once the previous snapshot's debt has drained + (credits back at the full window, Q3), and not more often than a minimum interval. So there is at most one snapshot + in flight per hold: no re-hold storm, and no stall for a snapshot larger than the window. +- **Calls** are throttled by call credits the provider grants (a generous window). Beyond them calls wait locally. + Nothing is refused for being "too many", and holds are not limited by count; authorization and input validation + protect them. +- **Byte-heavy traffic** such as terminals uses separate channels (chapter XIII), so a large state message never + holds up keystrokes. + +**Recovery:** after a link drops, the consumer re-holds every hold over the new link and receives snapshots; calls in +flight fail as on any disconnect, calls made meanwhile park. A relay whose window overflows releases the hold and +re-holds; the provider sends a fresh snapshot. After a routed link drops, Chord re-resolves the address (with the +per-address backoff) while holds on it exist, so a passive viewer recovers without anyone pressing anything. + +## Size limits + +Limits apply **per part** (R2, Q2), both to the compressed and the decoded size, and are checked from the sizes in the +header before anything is decoded; zlib's `maxOutputLength` enforces the decoded limit in the one decode function. + +| Situation | What fails | +| --- | --- | +| a part over the limit | only its own hold or call, with `too_large`; the other parts of the message apply | +| a snapshot over the limit | its hold ends `failed`, with no automatic re-hold | +| a WebSocket frame over `maxPayload` (e.g. 64 MiB) | the connection closes (code 1009); this is the transport's hard cap | + +Defaults for client WebSockets are 16 MiB compressed and 64 MiB decoded per part, configurable per link kind. The +decoded limit travels with relayed work: the server stamps the admitting link's limit on the header at admission +(never taken from the client), and the host that decodes enforces it. IPC therefore needs no limit of its own. + +Large data is never a large state member or a large argument. Files go through chunked transfers or the HTTP +component (chapter XIII); long lists are keyed documents or bounded windows (chapter VIII). + +## What each hop costs + +Chord v2's goal is as little compute and memory as possible while staying correct: **O(size of the change)** per +commit and per viewer, independent of the size of the value. A 500 KB value is touched as a whole only when a new +viewer takes its first snapshot. Delta, the value library under documents, was built for this, and its own primitives +are almost free: `diffRevisions` on shared subtrees takes 0.018 ms, `applyImmutable` of a small change 0.004 ms. + +| Hop | Work | +| --- | --- | +| commit | Delta's `prepare` (copies and checks only the values placed in this commit), adopt by pointer swap; storage writes the delta | +| reads in a transaction | the draft overlay, no copy; an unchanged draft writes nothing | +| subscribers on the same host | the same value and ops, by reference | +| derives | compute plus `diffRevisions`; unchanged parts stay the same objects (`mapShared`, chapter VIII) | +| send | one encoding per hold part at its origin, size taken from the encoded bytes | +| relay | header only: ids mapped, principal stamped, parts grouped; parts never parsed | +| receive | compiled header schema on untrusted links; arguments stripped and validated once at the provider (compiled schema); inputs normalized | +| apply | `applyImmutable`, structure shared | +| new hold | the value encoded once (unavoidable) | +| memory | one authoritative value per document per host; replicas and derived values share structure; one cached value per adapted binding | + +These are targets. The frozen kernel measured far worse, which is why the design is so strict here: + +:::stats +- 35 ms | one small change to a 500 KB value in the frozen kernel, no viewer +- +13 ms | per viewer on top (10 viewers: 130 ms) +- 17 / 33 / 101 ms | a tiny SQLite commit after 0 / 10 / 50 earlier deltas +::: + +Nearly all of that came from layers around Delta: whole-value validation on every access, whole-value freezing, +JSON-ness walks on every received frame, double stringification to measure a frame, snapshots cloned per viewer, and +a SQLite storage that replayed history, and schemas rebuilt for every check (a compiled TypeBox check takes +0.012 ms, an interpreted one 5.6 ms). The redesign removes each of them (chapters V, VIII and this one); the +before/after per hop is in `apps/p2/docs/p3-reviews/v2-cost-model.md`. + +:::review | Review: v2-cost-model.md is partly outdated +The measurements and the per-hop "after" targets in `apps/p2/docs/p3-reviews/v2-cost-model.md` are current. Its +closing section "Decisions this needs" is not: items 6 and 7 (validation by the caller's trust; contract descriptions +once per link and fingerprint) were replaced by R7 (validate for every caller) and R6 (no descriptions on the wire), +and §10 still describes the description fetch. +::: + +### Cost guards + +Chord reports **full-value walks, deep copies, encodes, schema validations and materializations** as ordinary +telemetry metrics, always on; with the NOOP adapter they cost an empty call where such work happens. Tests use the +in-memory adapter and assert them per hop. The reference case: one appended note to a 500 KB document with 10 viewers +does no walks, no copies, one encode per hold part at its origin and none at relays, and no validations. A +`vitest bench` suite measures the hops. + +### Ways to make it slow anyway + +| Mistake | Why it costs | Instead | +| --- | --- | --- | +| a derive that builds new objects for unchanged data | every commit diffs and sends O(value) | keep unchanged items identical: `mapShared`, `previous` (chapter VIII) | +| one long array that changes often (R24) | sharing is per path; each change copies one slot per element and visits each | keyed documents or a bounded window | +| bytes as base64 strings | inflated, parsed, compressed as text | `bytes()` leaves; chunked transfers for files | +| awaiting I/O inside `self.tx` | the object's line is blocked for every caller | do the I/O first, then a short transaction | +| storing registry entries or references in documents | stale after a reload, not data | look up the current value at use | +| opening natives at module level or in a facet | no owner: never closed, never restarted | `self.open` in an object (chapter IX) | +| callbacks as arguments | cannot cross a link, keep old code alive | registries, jobs, flows | +| one big object that everybody views | one large snapshot per viewer and per reconnect | split by viewer need; views of the same variant share one value | + +## Telemetry + +Chord reports traces, events and metrics through `@earendil-works/pi-telemetry`. Each host gets **one adapter**, +`createHost({ telemetry })`, NOOP by default; the adapter decides sampling, buffering, export and filtering. Only hosts +hold the adapter, to start root spans. Code sees spans: + +- `ctx.telemetry` is the current span (`Ctx` extends `Context`, so it is the same field Durable and pi-ai read). Every + span carries its adapter's metrics and propagation, so code with a `ctx` can record a metric or inject a carrier. +- `ctx.trace(name, attributes, fn)` runs `fn` in a child span with a child `ctx`. +- App values such as the principal stay out of Durable's context. + +**Across links**, `call` and `hold` carry the caller's trace carrier as a bounded header field (R5: limited key count +and string lengths); the provider runs the method under it, and each relay adds a `chord.relay` span. The call from the +sequence above, as a trace: + +```text +chord.call Notes.add [owner's tab] +└─ chord.relay [server] + └─ chord.method Notes.add [session:S1] principal=owner + · event chord.commit docs=2 ops=5 +``` + +Adapters trust typed input: no validation of malformed values and no try/catch around recording. Only carriers from +another process are checked (keys present, strings), and a bad one falls back to a new root. + +**Shared work** (one activation serving several `get`s, a worker start) is its own root span, linked to the trigger +that started it; each waiting span links to it. Routine runs, job runs, eviction, backoff retries and background +closes are root spans too, linked to their trigger where there is one. + +| Tier | What | +| --- | --- | +| spans | calls, methods, relays, holds, activations, opens, closes, loads, revocations, resolves, routine runs, job runs | +| events | commits inside methods, retries, parking | +| metrics | state messages, WebSocket bytes, commit durations, in-flight calls, active holds, open links, park durations, backpressure waits, the cost guards | + +Ids are attributes of spans, never of metrics. **Nothing is measured or walked for telemetry**; only sizes the code +already has are reported. Errors carry a fixed category and a message cut to 500 characters; cancelling is not an +error. Chord ships a typed telemetry schema, as pi-ai (`AI_TELEMETRY_SCHEMA`) and Durable +(`DURABLE_TELEMETRY_SCHEMA`) do. + +**Storing and reading:** `@earendil-works/chord/telemetry-sqlite` over `chord/sqlite` (Node and Cloudflare) stores one +row per settled span (events and links inline) and metrics aggregated per series per 10 s, flushes in batches every +second, keeps 7 days, and drops (and counts) beyond a bounded queue. The read interface `TelemetryStore` is shared: +`findTraces` (by name, attributes, errors, time window, with limit and cursor) returns complete traces, `getTrace` +one, `findMetrics` metrics. One renderer, `pi-telemetry/render`, prints traces as above; it prints links and never +follows them. The playground's trace viewer reads the same store. + +**Durable** already emits submit and phase spans, stores carriers on submissions and tasks (`TaskOptions.trace`), +and pi-ai emits `pi.ai.request` and `pi.ai.poll`. With the harness given the host's adapter (chapter XIV), an agent +tool that calls `Notes.add` through `chord(fn, context)` shows up as one tree from the model request down to the +commit. + +:::note | Built so far +Nothing of the protocol is built in v2: messages, relays, flow control and limits are layers L6 and L7, telemetry in +Chord is L11. Built in pi: `pi-telemetry` (propagation, links, metrics, the store interface, the renderer), +`chord/context` with `Context.telemetry`, pi-ai and Durable telemetry, and `chord/telemetry-sqlite`. Delta is built; +`mapShared` and bytes as atomic leaves in `diffRevisions` are still to do. The frozen kernel sends descriptions on the +wire and closes a link on an unknown id; both are replaced. +::: + +# Building p2: the Core {#p2-core lede="From a list of what the app must do to a set of owners: the base code every p3 plugin builds on."} + +The notes plugin used every Chord concept. A real app is the same concepts many times over. This chapter and the +next two assemble **p3**, the Chord v2 version of p2, from the parts the guide introduced. p3 is built next to p2 in +`apps/p3`, fully isolated with its own data directory, and **without migrating p2's data** (no import of +`catalog.sqlite`, `shares.sqlite` or the subagents table). Its plugins are derived from what the app must do, not +ported one to one from p2's plugin layout. + +:::review | Review: p3 is not designed yet +The design fixes the rules (`design.md` §25) and lists the p3 decisions still open (#17, #18, #20 to #23); the +per-domain design of p3 has not started. What these three chapters show is one construction that follows the rules, +derived from the functionality map in `apps/p2/docs/p3-functions/`. Object names are working names. Where a domain +has an open decision, the text says which. +::: + +## Start from functions, not from plugins + +The functionality map in `apps/p2/docs/p3-functions/` lists what p2 does, in plain words, with four facts per +function: the data and its size, how durable it is, who may see or do it, and where it must be available. It groups +the functions into 22 domains: + +| # | Domain | # | Domain | +| --- | --- | --- | --- | +| 1 | sessions | 12 | access | +| 2 | session workers | 13 | models, credentials, settings | +| 3 | conversations and agent | 14 | prompt resources | +| 4 | shell commands | 15 | status and presence | +| 5 | chat mode | 16 | search | +| 6 | agents (subagents, one-shots, peer messages) | 17 | remote access and forwards | +| 7 | code mode | 18 | logs and notices | +| 8 | session files | 19 | dev and reload | +| 9 | environments and execution | 20 | web shell | +| 10 | workspace | 21 | TUI shell | +| 11 | terminals | 22 | transport | + +The four facts answer the four questions every Chord object needs, in order: + +:::flow wide animate | From a function in the map to code. Each fact in the map answers one question Chord needs answered. +row "The map says" +box data "data" "what, how big" +box dur "durability" "ephemeral · until restart · persisted · transcript-bound" +box who "who" "owner · guest read · guest write · worker" +box where "where" "server · worker · web · TUI · environment" +row "Chord needs" accent +box owner "the owner" "smallest object it belongs to" accent +box store "where it is kept" "Durable, persisted or in-memory document, tab" +box variant "variants" "and revocation" blue +box host "host and code" "placement; core or plugin" +arrow data -> owner +arrow dur -> store +arrow who -> variant +arrow where -> host +note "Durability decides storage: transcript-bound data is Durable's; persisted app data is a persisted document with a stable identity; until-restart data is an in-memory document or a native; tab-scoped UI state stays in the client." +::: + +**One owner per thing, the smallest object it belongs to.** A guest's grant belongs to the access owner, not to the +session; a draft belongs to one (conversation, author) pair, not to the chat plugin as a whole. Ownership nests, and +rules set at creation are written in the creating commit (Durable's `createConversation({ init })`). + +**Core or plugin.** Core owners are what the app always needs and everything builds on; they are always present, so +other code may depend on them directly. Plugin owners may be missing; anything that depends on an optional feature +does so through a registry it owns itself (chapter XIII). The risk to avoid is a **core god object**: each core owner +owns only what belongs to it. + +## The ownership map + +The result of that procedure for p3, as a starting point: + +| Domain | Owner object(s) | Host | Code | +| --- | --- | --- | --- | +| access | `Access` (grants, devices, share links), `authenticate` | server | core | +| sessions | `Catalog` (session records), lifecycle jobs | server | core | +| session workers | the supervisor (`route(Session)` + idle stop) | server | core (app code) | +| session location | `SessionLocation { session }` | worker | core | +| conversations and agent | `Harness`, `ConversationStore`, `Conversation` views | worker | core | +| status and presence | `SessionStatus` (server copy), reporters on workers | both | core | +| shell commands | `ShellJob` in the chat plugin | worker | plugin | +| chat mode | chat plugin | worker | plugin | +| agents | subagents plugin (Durable child conversations) | worker | plugin | +| code mode | code mode plugin (sandbox native) | worker | plugin | +| session files | `SessionFiles { session }` (proposed, #17) | server | core or plugin, open | +| environments | envs plugin (config, SSH pools #20) | server and worker | plugin | +| workspace | workspace plugin | worker | plugin | +| terminals | terminal host process (#23), terminals plugin | separate process | plugin | +| models, credentials | models plugin, login job + flow | server | plugin | +| settings | `Settings` (single writer, #21) | server | core | +| prompt resources | prompt plugin (watchers in the session's environment) | worker | plugin | +| search | search plugin (index on the server, feed on workers) | both | plugin | +| remote access, forwards | radius plugin, ports plugin | server | plugin | +| logs and notices | `Notices` (one channel), log capture | server | core | +| dev and reload | the reload control, `reload_plugins` tool | server | plugin | +| web and TUI shells | UI facets over the objects above | clients | shells | + +The rest of this chapter builds the core rows. Chapters XIX and XX build the plugins. + +## The base code + +```text +apps/p3/ + src/ + hosts.ts Server, Session host kinds + principal.ts P, Revocation, principalKey, revokedBy, can() + server/main.ts createHost, load, route(Session), WebSocket and Radius entry points, supervisor + worker/main.ts createHost with the start data, connect to the parent + core/ the core facets: Access, Catalog, SessionStatus, Settings, Notices (server); + SessionInfo, SessionLocation, Harness, ConversationStore (worker) + plugins/ chat, subagents, codemode, envs, workspace, terminals, models, login, + prompt, search, ports, radius, files, dev +``` + +`hosts.ts`, `principal.ts`, the `SessionInfo` start data and the `Harness` object are the ones the guide already +built for Studio (chapters I, III, IX and XIV). Base code restarts the hosts when it changes; everything in +`plugins/` reloads. + +## Who is calling: authentication + +Chord stamps principals; the app produces them. p3's server has four ways in, each turned into a principal by +`authenticate(req)` when a WebSocket connects (chapter III): + +| Way in | Principal | +| --- | --- | +| loopback without a cookie (TUI, CLI, first page load) | `{ kind: "owner", access: "local" }` | +| loopback owner browser cookie (a per-port secret file) | `{ kind: "owner", access: "local" }` | +| a device cookie (HMAC-signed, per port, 30 days), obtained with a one-time sign-in token | `{ kind: "owner", access: "device", device }` | +| a guest cookie from a share link | `{ kind: "guest", session, mode, grant, revision }` | +| the Radius relay channel | `{ kind: "owner", access: "relay" }` | + +The checks that are not Chord's stay in this function: origin checks and Fetch Metadata (a browser request from +another site gets no principal, except a top-level navigation), proxy trust, the sign-in token rules (10 minutes, +single use, at most 32 outstanding, memory only), cookie expiry and revocation lists. A failed check closes the socket +before Chord sees it. + +**One permission matrix.** Every function in the map names who may use it, and p2 checks those rules in several +places. p3 writes them once, in `principal.ts`, and every factory and `authorize` uses it: + +```ts +// src/principal.ts +type Action = "read" | "send" | "upload" | "stop" | "workspace" | "terminal" | "manage"; + +export function can(p: P, action: Action, session?: string): boolean { + if (p.kind === "owner" || p.kind === "server") return true; + if (p.kind === "worker") return action !== "manage" && p.session === session; + if (p.kind !== "guest" || p.session !== session) return false; + if (action === "terminal" || action === "manage") return false; // terminals: owner only + if (p.mode === "read") return action === "read"; // no workspace for guest read + return true; // guest write: send, upload, stop, workspace +} +``` + +Factories pick variants from it (`can(p, "send", session) ? "writer" : "reader"`), so the matrix is visible in one +file and the members a caller lacks are simply not there. + +## Access: grants, devices, share links + +`Access` is the core owner of everything that lets someone in: share links per session and mode, guest records, the +device list and the signing key. It lives on the server as the chapter IV pattern: a `local` store that persists the +documents (only stable identities may), and a `remote` view for the owner's settings screen: + +```ts +// src/core/access.ts (server, in the core server facet) +const Links = defineState("access.links", { schema: map(ShareLink), version: 1, persist: true, + initial: () => ({}), checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64 }); +const Guests = defineState("access.guests", { … }); // same shape: map(Guest), persisted +const Devices = defineState("access.devices", { … }); // map(Device), persisted + +host.serve(AccessStore, (_input, self) => { // local: may persist, acts for the host + const links = self.state(Links), guests = self.state(Guests), devices = self.state(Devices); + return { + links, guests, devices, + removeGuest: async ({ grant }, ctx) => { + await self.tx(async (tx) => { (await tx.draft(guests))[grant].blocked = true; }, ctx); + host.revoke({ grant }); // live access ends now + }, + resetLink: async ({ session, mode }, ctx) => { + const blocked = await self.tx((tx) => rotateLink(tx, links, guests, session, mode), ctx); + for (const grant of blocked) host.revoke({ grant }); + }, + signOut: async ({ device }, ctx) => { + await self.tx(async (tx) => { (await tx.draft(devices))[device].revoked = true; }, ctx); + host.revoke({ device }); + }, + … // shareLink, join, signOutAll + }; +}); + +host.serve(Access, (_input, principal, self) => { // remote view: owner only + if (!can(principal, "manage")) throw forbidden(); + const store = self.get(AccessStore, {}); + return { + variant: "owner", + guests: self.derive("guests", [store.guests], ([g]) => publicGuests(g)), // names, modes; no tokens + removeGuest: (args, ctx) => store.removeGuest(args, ctx), + resetLink: (args, ctx) => store.resetLink(args, ctx), + signOut: (args, ctx) => store.signOut(args, ctx), + }; +}); +``` + +`authenticate` is base code on the same host and checks `AccessStore`'s current documents on every new connection +(it holds a reference to the store, taken with `server.root` at start). Live access and lasting denial stay separate, as chapter VII explained: + +| Event | Lasting effect (data) | Live effect | +| --- | --- | --- | +| remove a guest | guest record blocked | `revoke({ grant })` | +| reset a share link | new token; its guests' grants blocked | `revoke({ grant })` for each | +| archive a session | grants of that session void while archived | `revoke({ session })` | +| restore a session | grants valid again; revision bumped | none: guests reconnect | +| sign out a device | device revoked until its expiry | `revoke({ device })` | +| sign out all devices | signing key rotated, tokens cleared | `revoke({ device })` for each | + +A guest who rejoins with the same browser keeps its identity; a removed guest cannot mint a new one, because the +block is in `Guests` and `authenticate` refuses it. Names are unverified, cleaned, at most 40 characters, unique per +session ("Ann (2)"). + +## The catalog + +`Catalog` owns the session records: id, title (and whether it was edited), location, model default, pinned, +archived, timestamps. It lives on the server, persisted, and it **never starts a worker**: listing sessions, showing +their status or their unread marks reads server-side copies only. + +```ts +export const Catalog = remote("p3.catalog", { + version: 1, + input: obj({}), + placement: () => Server(), + variants: { + owner: { + state: { sessions: obj({ items: arr(SessionRow) }) }, + methods: { + create: method(CreateArgs, str()), rename: method(RenameArgs), pin: method(PinArgs), + archive: method(obj({ id: str() })), restore: method(obj({ id: str() })), + }, + }, + guest: { state: { sessions: obj({ items: arr(SessionRow) }) } }, // only the guest's own session, if not archived + }, +}); +``` + +Behind it sits a persisted `local` `CatalogStore`; the guest variant is a derive over it, as in chapter IV. The lifecycle operations (archive, clone, fork, +relocate, delete) are chapter XIX; here only the rule that keeps the catalog small: **the catalog does not know which +plugins exist.** Archiving a session writes `archived: true` and revokes the session; every plugin that keeps +per-session things reconciles from `catalog.sessions` with a routine (chapter X), as the notes plugin did. The ports +plugin closes forwards of archived sessions that way, also after a crash. + +## On the worker: start data, location, harness + +Three core objects exist on every worker from its start: + +- **`SessionInfo`** (chapter IX): the start message as `constant(init)`, with the session id, data directory and the + location the server chose at start. +- **`SessionLocation { session }`** (working name), lifetime host: the current cwd and environment, with a location + revision. Relocation changes it while the worker runs (chapter XIX); natives that depend on the location follow it + with a routine and a generation (chapter IX), and conversations that follow the session's cwd are listed in the core + per-conversation policy document. +- **`Harness`** (chapter XIV): the Durable harness for the session's store, one SQLite database per session, opened + under an exclusive lock keyed by the real data directory and session id. It refuses to start when the saved root + model is not configured, and resumes interrupted Durable work on start. + +`ConversationStore` and the `Conversation` views in front of it are chapter XIX. + +## Workers: start, keep alive, stop + +Starting, supervising and stopping workers is app code (chapter I); Chord only routes and counts. p3's supervisor is +`server.route(Session, { authorize, resolve })` plus a small server object that keeps per-session worker state: + +:::steps | A worker's life, as the supervisor sees it ++++ Authorize, then start +A tab gets `Notes { S1 }`. `authorize` checks `can(p, "read", "S1")` and that the session exists and is not archived +(an archived session refuses, except an index rebuild that starts the worker without resuming work). `resolve` forks +`worker/main.ts` with the start data from the catalog, bounded by a 60 s readiness timeout. ++++ A start failure sticks +If the worker fails to start (an unconfigured environment, a missing model), the error is kept per session and +every `get` fails at once with it, until the owner presses Retry. That clears the error and the related notices. +Guests see that the session is unavailable, not the error text. ++++ Counting +The core facet on the worker reports `host.activity` (`{ references, calls, jobs, lingering }`) and the harness's +running Durable task count to the supervisor object, from a routine whenever either changes. The call carries +`ctx.incarnation`, so a report from an older worker incarnation is ignored (chapter X). ++++ Idle stop +When everything has been zero for five minutes (checked every second, as in p2), the supervisor stops the worker: +the worker closes its host (routines stop, natives close, storage flushes) and exits. Durable work stays persisted +and resumes on the next start. Detached work that must keep the worker alive (a shell command whose caller left) is +a job, so it counts. ++++ Crash +If the worker exits on its own, the link drops. The supervisor records the exit code and a redacted stderr tail +(at most 4,000 characters) for that incarnation and posts one notice. References that still exist make Chord call +`resolve` again with backoff; a passive viewer recovers without a click. ++++ Shutdown +On server shutdown the supervisor stops every worker, with a forced exit after three seconds. Durable work remains +persisted. +::: + +## Status without a worker + +The session list shows per session whether it is running, busy (which conversations), its task count, usage and cost, +and an error. None of that may start a worker. The pattern is chapter X's pub/sub: a reporter on each worker (a +routine over the harness's state) calls `SessionStatus.report(summary)` on the server; `SessionStatus` keeps the latest +value per session and ignores calls whose `ctx.incarnation` is older than the newest it has seen; clients read +`SessionStatus`, which lives on the server. Children lists, unread marks, search index rows and artifact lists use the +same pattern (chapter XIX). + +## Settings and notices + +**Settings** (default model and thinking, run settings, compaction, queue modes, code mode switch, owner name) have +**one writer**: a core `Settings` object on the server owns the file, validates writes and serializes them; every +other process follows its state through a reference instead of watching and rewriting the file itself. p2 watches +settings, models and auth in every process and writes them from two, which is decision #21. + +**Notices** are one channel: one core `Notices` object on the server, written by everything that fails or finishes +(worker crashes, reload results, model errors, login outcomes, grants), keeping one latest notice per source, at most +50. The owner's shell reads one list; dismissing is a method. p2 has five notice channels for the same thing. + +:::review | Review: settings and file formats (#21, #22) +Single writer for settings, models and auth (#21) and versioning on-disk formats such as `envs.json` like documents +(#22) are open p3 decisions in `design.md` §26. The construction above assumes "one owner on the server, others +follow its state" for #21. +::: + +:::note | Built so far +Nothing of p3 exists. The functionality map is complete (`apps/p2/docs/p3-functions/`, five area files plus the +condensed `README.md`), and the design's app rules (`design.md` §25) are decided. p3 starts after the layered rebuild +and the playground's first real proof. +::: + +# Sessions and Conversations {#p2-sessions lede="The session lifecycle, status without a worker, chat, shell commands, subagents and search, each mapped to an owner and the Chord parts it uses."} + +This chapter builds the domains a person touches most: the session list and its operations, the conversation view +and composer, shell commands, chat mode, subagents, messages between sessions, search and unread marks. Each section +names the owner, the host, and the Chord parts it uses; code appears only where it shows something new. + +## The session lifecycle + +The catalog (chapter XVIII) owns the session records. Its operations, by the Chord part each needs: + +| Operation | Owner and form | Notes | +| --- | --- | --- | +| create | `Catalog.create`, a method | absolute directory, `~` expanded, must be a directory; on an environment, resolved there through the envs plugin; refuses an unknown model; model and thinking default to the last used | +| rename, pin | methods on `Catalog` | title 1 to 120 characters, whitespace collapsed; an explicit title is never replaced automatically | +| automatic title | a routine on the search feed | the first user message, line breaks to spaces, 100 characters | +| agent rename | an agent tool on the worker calling `Catalog` | root conversation only | +| archive, restore | methods; revocation; reconciling routines | below | +| clone, fork, relocate | jobs on the catalog store | outlive the call, cancellable, one outcome | +| delete | a job; decision #18 | below | + +### Archive and restore + +Archiving must stop the worker whatever it is doing, end guest access, and close the session's port forwards, while +keeping its storage. The catalog does none of the plugin-specific parts: + +:::steps | Archiving S1 ++++ One write +`Catalog.archive({ id: "S1" })` sets `archived: true` in the session record, in one transaction. ++++ Revoke the session +The catalog calls `host.revoke({ session: "S1" })`. Every guest principal of S1 has `{ session: "S1" }` in its +`revokedBy`, so their objects end and their links close on the server and on the worker (chapter VII). New +connections are refused by `authenticate`, because grants of an archived session are void. ++++ Stop the worker +The supervisor reconciles from `catalog.sessions` with a routine: an archived session's worker is stopped, and +`authorize` refuses to start it again. ++++ Plugins reconcile +The ports plugin's routine on `catalog.sessions` closes forwards of archived sessions. The search plugin marks the +session's rows archived. If the server crashed between the write and these steps, the routines run again with the +current value at the next start: nothing is a notification that can be lost. ++++ Restore +`Catalog.restore` clears the flag. Grants are valid again, so guests reconnect and authenticate as before; forwards +stay closed. +::: + +### Clone, fork, relocate + +Each is work that outlives its call, can be cancelled and has one outcome, so each is a **job** (chapter XI) on the +catalog's local store, keyed by the owner's request id. They differ in what they pause: + +- **Clone** copies the session's store, uploads, downloads and artifacts under a new id, with the source worker + paused (refused while work runs) and title "(copy)"; directories `0700`, files `0600`, no links, no overwrite; a + partial copy is removed on failure. +- **Fork** before a user message copies the active context as of the entry before it into a new session (title + "(fork)"), with the message waiting in the new session's editor. It needs two workers: the source reads history, + the new one is seeded empty. A failure archives the copy. +- **Relocate** (cwd, environment or both): refused while the session is busy (naming the running subagents) or when + a subagent has its own cwd and the move crosses machines. The job fences the worker (new admissions wait), updates + the record, and moves the running worker in place by updating `SessionLocation`: natives that follow the location + switch with a generation (chapter IX), and conversations that follow the session's cwd are rewritten in one Durable + commit before work resumes. After 60 s it restarts the worker instead; on failure it releases the fence. + +:::review | Review: jobs as objects (#9) +The design defers whether a long relocation or a login is a record inside its owner (watched through `running`) or an +object of its own (`Relocation { id }`). The construction above uses records. Decide when login is ported +(`design.md` §15, §26). +::: + +### Delete + +p2 has no session delete. p3 needs one that removes the session everywhere: every document tagged with the session +(`store.retireByTag(SessionTag, id)`, chapter V), the worker's databases (the Durable store), the session's files, the +search rows, read marks, children copies, forwards, guest records, and finally the catalog entry. The design commits to +this outcome (`design.md` §25); how plugins take part is open. + +:::review | Review: session delete (#18) +Options in `design.md` §26: per-session objects plus `SessionTag` plus one session directory (delete is retire by tag +and remove a directory, plugins need do nothing), or owner hooks contributed through a catalog registry. Also open: +whether the search index is per session (deleted with it) or global with a hook. Who holds the store for +`retireByTag` is not specified either. +::: + +## Status, children, unread: reports to the server + +The session list must show status, subagents and unread marks for sessions whose workers are not running, and must +not start them. All three follow chapter X's pub/sub pattern: a routine on the worker calls a server object with the +current summary, the server object keeps the latest value per session (persisted where it must survive a restart), +ignores calls from an older `ctx.incarnation`, and clients read the server object. + +| Summary | Source on the worker | Kept on the server | +| --- | --- | --- | +| status: running, error, task count, busy conversations, usage and cost | the harness's task graph and usage | `SessionStatus`, in memory: a stopped worker is "not running" | +| children: name, model, thinking, tools, cwd, last entry, archived | the subagents plugin's child list | `Children`, persisted | +| latest root answer | the search feed | `Unread`: compared with the owner's read mark, persisted, shared by all owner devices | + +Guests see their own session's status without the error text. The owner's read mark is written when the root +conversation is visible, and a new session starts as read. + +## Conversations + +The harness (chapter XIV) owns the Durable store. The UI needs, per conversation: a live window of entries, older +pages, usage numbers, the working indicator, the inbox (queued inputs), and the actions. Two core objects provide them: + +- **`ConversationStore { conversation }`** (local, on the worker): opens the Durable watch as its native, reaching + the harness through the core-private `HarnessRuntime` registry entry, and exposes a **window** as a derive: the + newest entries (at most 100 or 512 KiB, at least 20) plus the conversation's documents. The window's start stays + put while it is within 1.5 times the limits, so new entries arrive as small appends instead of a shifted list + (chapter VIII's window pattern). +- **`Conversation { session, conversation }`** (remote, placement `Session`): the per-principal view. Variants by + `can()`: `owner` (everything, including model, thinking and handoff), `writer` (guest write: send, steer, stop, + compact, shell), `reader` (the window only). + +```ts +host.serve(Conversation, ({ session, conversation }, principal, self) => { + const store = self.get(ConversationStore, { conversation }); + const policy = self.get(ConversationPolicy, { conversation }); // core: readOnly, followsSessionCwd + const author = authorOf(principal); // structured: kind, id, display name + if (!can(principal, "send", session)) return { variant: "reader", view: store.window, page: store.page }; + return { + variant: can(principal, "manage") ? "owner" : "writer", + view: store.window, + page: store.page, // older history: returns data + send: ({ text, attachments, mode, requestId }, ctx) => { + if (policy.value.readOnly) throw forbidden("read-only conversation"); + return store.submit({ type: "input", content: text, attachments, author, whenBusy: mode, requestId }, ctx); + }, + stop: (_args, ctx) => store.stop(author, ctx), // allowed even when read-only + … + }; +}); +``` + +What each function becomes: + +| Function | How | +| --- | --- | +| send, retried send | Durable `submit` with the author's `requestId`: a retry with the same id is admitted once (Durable dedupes, chapter XIV) | +| steer or queue while busy | `whenBusy: "steer"` or a follow-up; the inbox is `docs["pi.inbox"]` in the window | +| attribution | the view derives the author from the principal; the entry stores it structured, the model sees a speaker block; nothing parses text | +| wait for prompt resources | the first input of a fresh worker waits until the prompt plugin's resources are loaded (a registry readable the store follows) | +| stop, dequeue | withdraw queued inputs, abort the run; the caller's texts return to its editor, other authors' to their **restored drafts** | +| restored drafts | a keyed document per (conversation, author), tagged `SessionTag`, persisted in the store; claimed once, first tab wins | +| read-only conversations | the core per-conversation policy document, written at `createConversation({ init })` by whoever creates the conversation; send, dequeue, configure, compact and shell refuse; stop is allowed | +| history pages | a method returning a page (100 entries, 1 MB, 2,000 scanned); compaction and reset markers always included | +| model and thinking | owner variant only; validated against the models plugin's catalog; the root conversation's model is also reported as the session default | +| compaction, handoff | Durable operations through the store | + +**Drafts in the composer** are UI state, not Chord state: the web shell keeps one draft in `sessionStorage`, the TUI +keeps one per conversation across UI replacements, and pending sends keep their request id until acknowledged +("Sending when connected"). Only restored drafts, which the worker produces, are documents. + +## Shell commands + +`!cmd` runs a command in the conversation's cwd on its environment and records the result as an entry; `!!cmd` does +the same but keeps the output out of the model's context. One command at a time per conversation; the live tail is +at most 50 KB or 2,000 lines, sampled every 200 ms; the full output spills to a file; the result entry is placed at the +run's next boundary while the agent is busy and never starts a run. This is the design's job example almost as is: + +```ts +const ShellJob = defineJob("chat.shell", { + version: 1, + args: obj({ command: str({ max: 100_000 }), include: bool(), author: Author }), + progress: obj({ output: str() }), + result: ShellResult, + concurrency: 1, // one per conversation +}); + +const results = self.state(ShellResults); // keyed by run id: dedupe after completion +const shell = self.job(ShellJob, { + run: async ({ command, include, author }, ctx) => { + const out = new OutputBuffer({ maxBytes: 50_000, maxLines: 2000, retain: "tail" }); + const env = await environmentFor(ctx); // the conversation's environment, chapter XX + const r = await env.exec(command, { cwd: env.cwd, signal: ctx.signal, spill: spillPath(ctx.jobId) }, + (chunk) => { out.push(chunk); ctx.progress({ output: out.snapshot().text }); }); + const data = { command, output: out.snapshot().text, exit: r.code, cancelled: ctx.signal.aborted }; + await conversation.submit( + { type: "write", entry: shellEntry(data, author, include), requestId: `shell:${ctx.jobId}` }, ctx); + return data; + }, + finish: async (done, tx) => { + (await tx.draft(results, done.id)).outcome = done.kind === "done" ? done.result : { error: done.error }; + }, +}); +``` + +The request id `shell:${ctx.jobId}` makes the entry idempotent: an attempt interrupted after the write cannot write +it twice. Cancel is `shell.abort(id)`, allowed for the command's author and the owner. A retried `runShell` with the +same request id returns the running job (dedupe of unsettled runs), or the stored result (the owner's own dedupe after +completion, chapter XI). p2's "cancel when nobody watched for 30 s" is dropped: a job keeps its owner alive and does +not depend on viewers. + +## Chat mode + +Chat mode is a per-conversation switch that everyone in the session sees and only the owner changes; forks and new +subagents start with it off. Because it must fork and rewind with the conversation, it is **transcript-bound**: a +Durable conversation document, not a Chord document. Messages without `@pi` become display-only Durable `write` +entries (author, text, attachments, reply quote); a message with `@pi` becomes an input with a digest of the chat +since the last mention (at most 100 messages, 20 files, 4 images), recording which entries it passed on. The reply +quote is derived on the worker from the entry id the client names: the client never supplies author or quote text. + +## Subagents and one-shot agents + +Subagents are Durable child conversations owned by tasks, as in Durable's own background-subagent example: one +`subagent` tool spawns, steers, waits for, stops and lists them; each child is owned by a background anchor task, and +each message is delivered by a reporter task that posts the answer back to the parent once, with request ids so a +restart sends nothing twice. Chord adds only three things: + +- the **child list** for the sidebar is reported to the server (`Children`, above), so it shows without a worker; +- a child that follows the session's cwd, or a read-only one-shot agent created by a code-mode script, is marked in + the core **per-conversation policy document** when it is created, so chat and relocation enforce it without knowing + the subagents plugin (`design.md` §25 porting guideline); +- archive and restore of a child from the UI are methods on the subagents plugin's view, refused for one-shots. + +Subagents cannot spawn subagents; thinking levels are clamped to what the model supports, in one helper shared with +the session's model choice. + +## Messages between sessions + +The root agent can list other sessions and send one an attributed message. The tool runs on the sender's worker and +calls a server object through `chord(fn, context)` (chapter XIV): + +```ts +const peerSend = defineTool({ + name: "session_send", + execute: (args, api, context) => chord(async (ctx) => { + const peers = await ctx.get(Peers, {}); // server, worker variant + await peers.send({ to: args.session, text: args.text, mode: args.mode, requestId: `peer:${api.taskId}` }, ctx); + return { output: [{ type: "text", text: "sent" }] }; + }, context), +}); +``` + +`Peers` checks that the sender is a root conversation of a non-archived session, refuses sending to itself and +archived recipients, then gets the recipient's `Conversation` (which starts its worker) and submits with the +sender as a structured author and the same request id (the tool task's id, stable across restarts), so a retried +tool call delivers once. The sender's +incarnation is `ctx.incarnation`; a call from a replaced worker is refused. Only owner-initiated turns may use the +tool; guests never. + +## Search + +The search plugin indexes user and assistant messages of root and subagent conversations (no shell output, no +Durable internal inputs) for title, cwd and message search across sessions, active or archived. + +:::flow wide animate | The search feed. The worker pushes pages from a cursor; the server owns the index and never starts a worker to answer a query. +row "Worker S1" blue +box commit "Durable commit" "in any conversation" +box wake "wake" "a routine on the harness's commit signal" +box page "read from cursor" "200 entries per page" +row "Server" accent +box index "SearchIndex" "native: SQLite FTS" accent +box cursor "cursor per conversation" "moves after the whole range" +box query "query" "terms, scope, at most 41 hits" +arrow commit -> wake +arrow wake -> page +arrow page -> index "report(page)" +arrow index -> cursor +note "Reports carry ctx.incarnation. Indexing is idempotent: a page sent twice changes nothing." +::: + +The index is a native owned by a host-lifetime object on the server; cursors live with the index, so after a crash +the worker asks for the cursor and continues. Rebuilding one session's index drops its rows and rescans, starting +an archived session's worker without resuming its work. Queries are methods returning data: at most 41 hits, +snippets of 300 characters, queries up to 500 characters. The same feed sets automatic titles, moves "updated at" +forward and supplies the latest root answer for unread marks. + +## Presence + +Viewers per session are `host.references(Conversation)` on the worker, counted per client (chapter VIII), and +reported with the status. Connected guests (name and mode) are the references whose principal is a guest. Presence +is not keep-alive: the supervisor uses `host.activity`, which also counts calls and jobs. + +:::note | Built so far +None of this exists in p3. The parts it relies on are Durable (built: submit with request ids, steer and queue, +child conversations, background tasks) and the Chord layers L3 to L9 (not built). +::: + +# Machines, Files, Models {#p2-machines lede="Environments, workspace, terminals, session files, ports and Radius, models and logins, prompt resources, logs, reload and the shells: the rest of p3."} + +The last domains are the ones that touch the outside world: other machines over SSH, the file system and git, +terminals, the relay, model providers and their credentials. Most of them are native objects, and most of them are +places where p2 has the same thing several times (four SSH connections to one environment, five notice channels, +three copies of the environment config parser). The rule that fixes that is the one from chapter XVIII: one owner per +thing. + +## Environments + +The **envs plugin** owns the environment configuration: name, host, user, port, identity file, shell and its +variables, plus a managed ed25519 key per environment and its own `known_hosts` (kept apart from the user's, aliased +per environment name). + +| Function | Owner and form | +| --- | --- | +| list, add, edit, delete (refused while a session, archived too, uses it) | `Envs` on the server: a persisted document behind an owner-only view | +| validation (no leading `-`, no whitespace or control characters in host and user) | the view's methods, before the write | +| a hand-edited `envs.json` is picked up; an invalid file keeps the last valid set and shows an error | a native file watcher (poll fallback) in the envs store; one writer (#21) | +| environment revision: a hash of the config, part of its identity | a derive | +| connection test (connect, deploy the daemon, platform, latency, warnings) | a job; the result per revision in a document | +| unknown or changed host key: fingerprints, trust after confirmation, forget | a flow (chapter XII) on the server | +| browse directories (`~`, relative paths, at most 10k scanned, 1,000 shown) | a method with a **call-scoped** SSH session (chapter IX) or the pooled connection | +| a changed environment restarts its sessions' workers once idle | a routine on the envs store and the status reports | + +**Execution in the agent.** The worker's harness asks `HarnessEnvironments` for an environment per target (chapter +XIV). The envs plugin contributes it from its worker facet: local execution at the session's cwd, or a remote one over +SSH through the `pi-env` daemon, deployed per platform and connecting at the first operation. A remote session never +falls back to local tools. The environment follows `SessionLocation`: after a move, the old connection closes once +the move has committed and watchers reopen at the new location. + +```ts +// envs/worker.ts +export default defineFacet((host, self) => { + const location = self.get(SessionLocation, { session }); + const pool = self.get(EnvPool, {}); // one owner of SSH connections (#20) + self.contribute(HarnessEnvironments, (target) => + target.env === "local" ? localEnv(target.cwd) : pool.environment(target.env, target.cwd)); +}); +``` + +:::review | Review: connection pools (#20) +SSH connections to one environment exist four times in p2 (worker environment, browse connection, test connection, +terminal host). `design.md` §26 asks for one owner with a pool or deliberate per-use connections. `EnvPool` above is +the first option: a native object that closes idle connections after 10 minutes and invalidates them on a config or +key change. Reloading the plugin that owns the pool kills running remote commands (chapter XV), so it should change +rarely. +::: + +## Workspace and file index + +The **workspace plugin** serves the viewed conversation's working tree (a subagent's own cwd included) on the worker: +list a folder (at most 2,000 entries, `.git` hidden), read a file (1 MiB, binary detected), git status, diff, log and +single commits, literal text search, and all paths for name filtering. Every query is a **method returning data**, not +state: results are bounded and requested by the UI. Only change notifications are state: a counter in a document, +bumped by a native watcher of the tree, `HEAD`, the index and config (debounced 100 ms); viewers reload what they show +when it changes. + +Two rules live in one place each instead of three: + +- **Confinement:** path rules, a canonical root, no `.git`, the app's private data directory denied (also excluded + from search). One helper used by listing, reading, uploads and the file index. +- **Git hardening:** no optional locks, no fsmonitor, filters disabled, no prompts or pager, no system config, 10 s + timeout, output caps. + +Downloads (25 MiB, byte ranges for media) go through the HTTP component (chapter XIII) with a route on the server that +reads chunks from the worker; uploads into the project use the chunked protocol of chapter XIII (1 MiB chunks, temp +file then rename, never overwrite, writes serialized). Guest read gets no workspace: the view has no variant for it. + +The **file index** for `@` completion belongs to the same owner, so completion consumes the index instead of scanning +on its own: built from the environment's listing (else git or a walk, at most 50k paths), rebuilt at most once per +second while files change, discarded after a move. Review comments drafted on files and diffs are browser state until +sent; sending them is an ordinary `Conversation.send`. + +## Terminals + +Terminals must survive server and worker restarts, so their PTYs live in a **detached terminal host process**, one per +data directory, with its own process lock; it exits after 10 minutes with no terminals and no clients. The terminals +plugin on the server is the owner the UI talks to: list (title, cwd, environment, size, exit status), open at the +session's cwd and environment (refused for archived sessions), attach, close (asking first if a foreground job runs). +Terminals are **owner only**. + +Bytes do not go over object links. Attaching opens a **channel** (chapter XIII) with a single-use token (30 s): JSON +control on the object, bytes on the channel, with its own acknowledged window (256 KiB, acknowledgements per 64 KiB; a +viewer that falls behind gets a fresh screen snapshot; an 8 MiB backlog disconnects). Attaching gets the current +screen from a headless emulator with 10k lines of scrollback. + +:::review | Review: the terminal host process (#23) +The terminal host runs its own code. Is it part of the revision, reloaded or restarted with it (and running terminals +end), or deliberately outside it with a "host runs older code" notice as in p2? Open in `design.md` §26. +::: + +## Session files + +Attachments (25 MiB, at most 20 per message), sent-file snapshots made by the agent's `send_file` tool, versioned +artifacts (HTML, SVG, Markdown, text; 2 MiB; each update a new immutable version) and shell spill files are all +per-session bytes with caps, immutable ids and range reads; p2 has two upload routes, two caps and two directory +layouts for them. The proposal is one owner: + +```ts +export const SessionFiles = remote("p3.session-files", { + version: 1, + input: obj({ session: str() }), + placement: () => Server(), // readable without a worker + variants: { writer: { … }, reader: { … }, worker: { … } }, +}); +``` + +`SessionFiles { session }` on the server holds the index (per kind, readable without a worker), the chunked upload +and read protocol, the caps per kind, copying on clone and fork, and deletion with the session. Attachments are copied +into a remote environment before an input is admitted, with no fallback on failure. Artifact previews run in a +sandboxed frame with its own CSP and no app identity; ordinary file previews allow only safe types. + +:::review | Review: session files (#17) +Proposed in `design.md` §26, not decided. Open sub-points: one preview limit (p2 has 256 KiB for sent files and +512 KiB for artifacts), what a fork copies of the artifact list, and how attachments are copied into remote +environments. Related: #22, whether on-disk layouts such as downloads and uploads get versions like documents. +::: + +## Ports and Radius + +The **radius plugin** keeps the outbound relay connection, a native in a host-lifetime object on the server: sign-in +by device code from the CLI (credential in a private file), token refresh, a heartbeat in the link adapter (20 s ping, +60 s timeout; TCP does not notice a sleeping laptop), backoff from 1 to 30 s, a disable switch. The relay delivers two +kinds of channels: + +```ts +// radius/server.ts (the entry point: one line, as in chapter I) +radius.onChannel((ws) => host.accept(webSocketLink(ws), { principal: { kind: "owner", access: "relay" } })); +radius.onForward((channel) => proxyToForward(channel, host.registry(RadiusForwards).value)); +``` + +Object links become owner principals with `access: "relay"`: owner, but treated as remote (no localhost-only +powers). p2 had to fake a loopback request with a secret header for this; with `accept` that is gone. + +Forward channels go to `localhost:` (Host, Origin and Location rewritten; 502 when nothing listens; anonymous +visitors only to public forwards). Which forwards exist is not radius's business: it owns a registry, +`radius.forwards`, and the **ports plugin** contributes its forwards (at most 50, random ids, a public toggle that +needs confirmation, an agent tool `ports` for top-level owner-initiated conversations). Without the ports plugin the +table is empty and radius still works: an optional dependency is a registry owned by the dependent (chapter XIII). The +ports plugin closes forwards of archived sessions with a routine on `catalog.sessions` (chapter XIX). + +:::review | Review: may a plugin call `accept`? +`design.md` §3 shows the Radius line in the server's start code and calls entry points "app or adapter code". If +radius is a plugin, its facet needs the host's `accept`, which no other part of this guide gives a plugin. The +alternative is that the start code keeps the entry point and gets channels from the radius object. Not specified. +::: + +## Models and credentials + +The **models plugin** owns the model runtime (pi-ai's models with custom providers and the model cache) in a +host-lifetime object, and the credentials. It contributes the runtime to `HarnessModels` (chapter XIV), so after a +models reload the next request uses the new runtime while a running stream finishes on the old one. **Credentials +never go to clients**: the views expose providers, login methods and which type is signed in, never tokens. Every +secret is registered for redaction in one place, the log capture. + +Views: a model catalog for pickers (names, ids, context windows, thinking levels; at most 300 matches in the web), +logout (refused during a login), refresh a provider's model list. The default model and the cycling scope are +settings (chapter XVIII). + +### Login + +A login is the guide's flow example (chapter XII): a **job** on the server that runs pi-ai's login, and a flow that +shows its steps to the starting owner. + +```ts +const LoginJob = defineJob("models.login", { + version: 1, args: obj({ provider: str(), type: AuthType }), progress: LoginState, result: obj({ ok: bool() }), + concurrency: 1, // one login at a time on the server +}); + +const login = self.job(LoginJob, { + run: async ({ provider, type }, ctx) => { + await runtime.value.login(provider, type, { + signal: ctx.signal, // 10 minute deadline, cancel, leaving + prompt: (prompt) => steps.ask(prompt, ctx), // text, secret, select, manual_code + notify: (event) => steps.notice(event, ctx), // info, auth_url, device_code, progress + }); + return { ok: true }; + }, + finish: async (done, tx) => { /* record the outcome; refresh the model list */ }, +}); +``` + +`prompt` is a promise-based callback inside the work; the resolver stays in the job (`steps.ask` records the step in +the flow state and waits for `answer`), and nothing crosses an object boundary. On a relayed connection the device +code method is selected automatically. The flow is visible only to the owner principal that started it; others are +refused. Leaving ends it: when the starting owner's references are all gone (presence, not an unmount), a routine +aborts the job. Codes, URLs and typed secrets are redacted from logs. + +## Prompt resources + +`AGENTS.md`, skills and templates are read in the session's environment, watched (poll fallback, last valid state), +and expanded on the worker before sending. The **prompt plugin** owns them per session on the worker, contributes the +system prompt section and skills to `HarnessExtensions` as readable contributions (a change recomposes), and exposes a +readiness readable: the first input of a fresh worker waits until it is ready (chapter XIX). After a move the watchers +reopen at the new location. + +## Code mode + +Code mode is a plugin (chapter XIV): its `pi-codemode` sandbox is a native; scripts call tools as nested Durable calls +and replay after a restart; the tool entry looks up the current sandbox when it starts. A reload seals and drains the +old sandbox for at most 30 s. The on/off switch is a setting. + +## Logs and notices + +**Log capture** tees stdout and stderr of the server, workers and detached hosts without changing their behaviour, +redacts known secrets and credential patterns, and keeps bounded rings (at most 128 sources, 1,000 lines or 256 KiB +each). Detached hosts publish redacted snapshots to a private file. Logs are read without waking a stopped worker. +**Notices** are the one channel from chapter XVIII; dismissal stores only notice ids. + +## Dev and reload + +The dev plugin is chapter XV's reload control for p3: a source watcher marks plugins dirty and never applies anything; +the `reload_plugins` agent tool and a control in the owner's shell reload the changed plugins, or restart all hosts for +base code (through the supervisor, reopening the sessions that were running). UI bundles are published only after a +successful backend change; a web page detects a new manifest and reloads with its drafts and navigation state saved; +the TUI swaps its sandboxed UI code with a snapshot of its state. A local Unix socket (`0700`) accepts development +commands without a network admin port. + +## The shells + +The web and TUI shells are UI facets over the objects above. Two rules matter for Chord: + +- **UI state survives navigation, reload and reconnect on purpose.** Composer drafts, pending sends with their request + ids, reading positions and review drafts are kept in tab storage or the TUI's replacement snapshot. A reconnect + re-holds and gets snapshots; nothing is lost that was not in flight. +- **Leaving is never derived from an unmount.** A plugin reload remounts components (chapter XV). Cancelling a login, + closing an overlay or giving up a lease is an explicit action or a dropped reference. + +The TUI launcher keeps downloaded UI code in a permission-dropped worker with no access to files, network or +credentials; local side effects (copy, upload, download, open a URL, an external editor) need a real user gesture or a +confirmation. That is app code around Chord, not part of it. + +## Transport + +The last domain, transport, is chapter XVII: one message format, per-part limits (16 MiB compressed and 64 MiB decoded +on client WebSockets, instead of p2's 1 MiB input cap and 32 pending calls), one byte window per link, re-holds with +snapshots after a reconnect, and a protocol version that makes an outdated web page reload. + +:::note | Built so far +None of p3 exists. pi-ai's login API, pi-codemode and the `pi-env` daemon exist and are used by p2 today; the HTTP +component and channels are deferred until after the kernel's first real proof (chapter XIII). +::: + +# Other Ways to Do It {#compared lede="Cordis and DeepSeek Harness attack the same problem, plugins that come and go while an agent runs, inside one process. What that buys, and what it costs."} + +Chord v2 is not the only answer to "replace plugin code while the app keeps running". The closest relative is +**Cordis**, a TypeScript plugin framework by Shigma that has been the plugin kernel of the Koishi chatbot framework +since 2019, and its most visible user, **DeepSeek Harness** (`dsh`), DeepSeek's open-source agent harness, whose +motto is "everything is a plugin". This chapter compares them with Chord v2 on the questions the guide cared about: +who owns side effects, how plugins depend on each other, what a reload keeps, what a crash keeps, and how far the model +reaches beyond one process. + +:::note | Sources and status +DeepSeek Harness at commit `687f282` (release 0.2.1-rc.2), which vendors Cordis as `@deepseek-ai/cordis` 4.0.5 (Cordis +RC7 with local changes, `vendor/README.md`): its Cordis tutorial (`docs/cordis-tutorial/`), `docs/architecture.md`, +`docs/subsystems/persistence.md`, `docs/subsystems/core.md` and `packages/boot/hmr/README.md`; the Cordis paper "A +Programming Paradigm for Spatiotemporal Composability" (arXiv 2608.25512). Cordis and dsh are built and in use; Chord v2 +is the design this guide describes. The comparison is between a working system and a design. +::: + +## Cordis in one example + +A Cordis plugin is a function (or an object with `apply`, or a `Service` subclass) that receives a **context**. It may +declare the services it needs with `inject`, and everything it does to the world goes through the context: + +```ts +export const name = "greeter"; +export const inject = ["tools"]; // activate only while a `tools` service exists + +export function apply(ctx: Context) { + ctx.tools.register(defineTool({ name: "greet", /* … */ })); // a registration is an effect: undone on unload + ctx.on("session/start", (s) => log(s.id)); // an effect-bound listener + ctx.effect(() => { // any other side effect, with its disposer + const timer = setInterval(work, 200); + return () => clearInterval(timer); + }); +} +``` + +Each plugin instance is a **fiber** with states `PENDING → LOADING → ACTIVE → UNLOADING → DISPOSED` (or `FAILED`). +Every effect, registration, listener, child plugin and provided service is recorded on the fiber, and unloading runs +the disposers. The paper calls this **temporal composability**: a component can be removed at any time and reverses its +own effects. Its counterpart, **spatial composability**, is the dependency side: `inject` is live, so a plugin starts +when its services appear, and unloads (and later reloads) when a service it depends on goes away or is replaced. The +paper's claim is that independently written components can therefore be combined and replaced without disturbing each +other. The next sections check how far that holds. + +**Services** are named capabilities on the context (`ctx.provide("x", value)` or `class X extends Service`). Names are +runtime strings in one flat namespace per application; TypeScript declaration merging makes `ctx.x` typecheck, but +nothing checks the service's shape at runtime, and optional dependencies are probed with `ctx.get("x")`. Scopes come +from the context tree: `ctx.extend()` derives a child context, `ctx.isolate(name)` gives one subtree its own binding of +a service, so two parts of an app can use two different `shell` providers. + +**Hot reload** follows from the two halves: the reload plugin unloads the old fiber (unwinding all its effects) and +loads the new module with the same config. Nothing held in the old instance's closures or fields survives; a plugin +that needs state across reloads must keep it in a service or on disk. Config edits are diffed by entry id and +mount, unmount or reconfigure only what changed; config fields declared volatile update in place without a remount. + +## DeepSeek Harness on Cordis + +dsh applies "everything is a plugin" to an agent: the model adapters (`ctx.llm`), the tool registry (`ctx.tools`), the +session log (`ctx.sessions`), the agents (`ctx.agents`) and the agent loop itself (`ctx.agentLoop`) are plugins and +services. A running dsh is a plugin tree composed at boot from a **profile** (`web`, `headless`, `sdk`, `acp`) that +stacks **bundles** of Cordis config rows, with user patches on top. There is no privileged core: replacing the agent +loop is a config patch. + +| Concern | How dsh does it | +| --- | --- | +| sessions | a durable, append-only event log per session (JSONL backend, single writer, `flush` as the durability barrier); model history is derived from the events | +| crash mid-turn | the log keeps everything appended; on resume the agent loop closes the open turn by appending missing tool errors and a synthetic `turn/end { reason: "interrupted" }` | +| tools | `ctx.tools.register(defineTool(…))`, an effect of the registering plugin; calls go through pre/execute/post policy waterfalls | +| approval | closed by default; `allowed-once` is the only grant; plugin installation needs full access or per-call approval | +| reload | a source watcher evicts changed modules; Loader entries running affected plugins stop and start again with the new code; a failed replacement is an activation failure, its previous code no longer runs | +| processes | one plugin tree in the Host process; the browser client and Electron's renderer talk to it over RPC streams | +| distribution | a Plugin Manager installs and enables bundles per profile; plugin markets are optional indexes | +| subagents | local subagents with their own durable sessions, and external ones | + +## What the guarantees are in practice + +Reading the Cordis source vendored in dsh and dsh's own documentation, the guarantees are narrower than the paper's +framing. An earlier review for p2 came to the same result (`apps/p2/docs/p3-reviews/dsh-compare.md`). + +- **Reversal is cooperative and unordered.** `_unload` runs all of a fiber's top-level disposers concurrently + (`Promise.all` in `vendor/cordis/src/fiber.ts`); order holds only inside one effect, and the primer tells authors to + "keep the related work in one effect" when teardown order matters. A disposer that throws is logged and skipped. A + disposer that never settles stalls the teardown; dsh's jobs service works around this by force-failing a job whose + cancel throws and warning that "the work may be orphaned" (`packages/jobs/jobs-local/README.md`). Reversal is as + good as every disposer, and nothing checks that a plugin registered all its effects. +- **Behaviour depends on registration order, and a reload changes it.** `emit`, `waterfall`, `serial` and `bail` + listeners run in registration order (`docs/cordis-primer.md`). Registering appends the listener (`push` in + `vendor/cordis/src/events.ts`, unless `prepend`), so when a plugin reloads, its listeners move to the end of the + list. A tool-policy listener that ran first runs last after its own plugin reloads; a waterfall that short-circuits + now sees a different decision first. +- **Dependencies are strings, and every provider reload restarts its dependents.** A fiber's activation key is the + concatenation of its injected providers' fiber ids (`_refresh` in `fiber.ts`), so replacing a provider unloads and + reloads every plugin that injects it, with their in-memory state. Service APIs are "pre-stable; update every + consumer" (`AGENTS.md`): there is no versioned contract between a provider and its consumers. +- **Reloading a core service ends live work.** Unloading the agent provider "stops and drains every live handle it + made" (`packages/core/agent/README.md`), so running agents stop; reloading the session persistence backend under + live sessions invalidates their write handles, and "writes fail loudly until the sessions restart" + (`.agents/notes/implemented/architecture/2026-08-27-handle-based-session-persistence.md`). +- **Registrations leave a gap.** A reloading plugin's tools and other registry entries are removed with its old fiber + and added again by the new one; anything that looks them up in between does not find them. + +None of this is a bug in the narrow sense: each point is documented, and dsh is built around it. But "components +interleave without disturbing each other" holds only for plugins that own nothing other plugins depend on, register +every effect, order nothing by registration, and hold no state that must outlive a reload. + +## Side by side + +| | Cordis and dsh | Chord v2 | +| --- | --- | --- | +| unit | a plugin instance (fiber) in one context tree | an object per (type, input, principal), placed on a host (chapters II, III) | +| side effects | recorded on the fiber with `ctx.effect` and friends; on unload the disposers run concurrently, each as good as its author made it | declared through `self` (documents, natives, routines, jobs, contributions); Chord activates and undoes them (chapters V, IX to XI) | +| dependencies | service names as strings in one flat namespace, via `inject`; a provider reload restarts every dependent | typed contracts via `get(Type, input)`; a reference parks and recovers through restarts (chapter VI); optional dependencies are registries (chapter XIII) | +| ordering | listeners run in registration order, which changes when a plugin reloads | registries compose with a declared `compose` function; nothing depends on load order (chapter XIII) | +| where code runs | one process; remote UIs over RPC to the Host | a tree of hosts (server, workers, tabs); objects reachable across links, relayed by the server (chapters I, XVII) | +| who is calling | not a framework concept | principals stamped by links, variants per caller, revocation (chapters III, VII) | +| state | the plugin's own business; dsh's session log is a service | documents written in transactions, persisted, versioned and migrated (chapter V) | +| reload | unload the old instance, load the new one: in-memory state and running work of that plugin end | pure objects swap their code in place with their documents and running work kept; only objects with natives restart (chapter XV) | +| a running agent turn during reload | entries of changed plugins stop; unloading the agent provider stops its live agents | the harness is base code; Durable pins tools per task phase, so a running call finishes on old code and the next phase uses the new (chapter XIV) | +| crash recovery of a turn | the turn is closed as `interrupted` | pi-durable continues the turn's tasks from their checkpoints; replay-safe tools rerun (chapter XIV) | +| contracts and versions | TypeScript types only, pre-stable APIs, every consumer updated together; config validated with a schema | contract versions with fingerprints and provider-side mappings for old consumers (chapter XVI) | +| status | in production in Koishi since 2019; dsh released | a design with the contract layer built | + +## What the comparison says + +**The core idea is the same.** "Factories declare, Chord activates" (chapter II) and Cordis' reversible effects are +two forms of one rule: a plugin may only change the world through its runtime, so the runtime knows what to undo. +Both reject the module-level timer and the stored callback for the same reason. + +**The difference is what survives a reload.** Cordis reloads at plugin granularity: the old instance unwinds and the +new one starts from nothing, together with every plugin that depends on it. That is simple, but a plugin's in-memory +state and its dependents' state and work in flight end with it, and correctness rests on every disposer and on +registration order. Chord v2 keeps state out of closures on purpose (documents, references, jobs as +records) so that it can swap the code of a pure object while its data and running work stay, and restarts only what +holds OS handles. That is more machinery: transactions, a store, versions on every document and contract. + +**The other difference is reach.** Cordis composes plugins inside one process, and dsh adds RPC to its own clients. +Chord v2 makes the process boundary part of the model: objects live on the host their placement names, callers are +principals, and a reload rolls out across the server, the workers and every tab. An app like p2, with a process per +session and guests in other browsers, needs that; a single-process agent does not. + +:::review | Review: what a reload does to a running tool call in dsh +The sources say that Loader entries running affected plugins stop and that unloading the agent provider stops its live +agents (`packages/boot/hmr/README.md`, `docs/subsystems/core.md`). They do not say what happens to a tool call in +flight when only the tool's plugin reloads; the comparison above does not claim either way. +::: + +# What's Next {.epilogue lede="Where Chord v2 stands, what is still undecided, and where the written sources disagree with each other."} + +The guide built Studio from one object to a p2-sized app. Every piece used the same few rules: one owner per thing, +factories that only declare, state that changes only in transactions, long-lived behaviour in declared forms, and +mappings and migrations owned by the provider. Those rules are what let Mario press Reload on his phone in the +preface while the agent keeps running. This last chapter says how much of that exists and what is still open. + +## Status + +:::stats +- ~10k | lines in the frozen first kernel (steps 1 to 6, 443 tests), kept as a reference only +- 1 of 11 | layers of the rebuild started: L1 contracts, built and reviewed +- 6–7k | target size of Chord v2 without tests +::: + +- **The design is decided** except the points marked below. `packages/chord/docs/next/design.md` is the target; + `packages/chord/docs/next/decisions.md` is the log and wins where they differ. +- **The rebuild** goes layer by layer, each with a one-page spec the owner approves, a minimal build with counting and + garbage-collection tests, the cost-guard metrics, and a cleanup loop: L1 data and contracts; L2 readable and + derive; L3 documents and store; L4 objects; L5 references and `ctx`; L6 protocol and links; L7 relay, routes, + revocation; L8 routines and jobs; L9 registries; L10 reload and rollout; L11 telemetry in Chord. +- **Built:** the contract layer (`src/next/contracts/`, commit `7d2bb8ea8`, chapter XVI). In pi: Delta, + `pi-telemetry`, `chord/context`, `chord/telemetry-sqlite`, pi-ai and Durable telemetry, Durable's required + `checkpointWhen`. +- **Next, in order** (`apps/p2/docs/p3-handoff.md`): decide Q12 and Q15, which the L1 spec needs; write the spec for + the rest of L1 (values, inputs and identity, argument checks, principals, errors; draft in + `packages/chord/docs/next/layer-1.md`); then Q11, Q13, Q14 and the cuts C3 to C17 before the layers they affect. +- **In pi:** `mapShared` and bytes as atomic leaves in Delta's `diffRevisions`. +- **Then** the playground web shell (React, esbuild) with the reload control and a preview at `p3.mariozechner.at`, + and the first real proof: the harness, one tool with an approval pause, a code mode reload. Then the HTTP and + channel components, then p3, domain by domain. + +## Deferred decisions + +Kept with stable numbers in `design.md` §26, each waiting for a concrete case: + +| # | Topic | When | Chapter | +| --- | --- | --- | --- | +| 9 | jobs as objects: a record inside the owner, or an object addressed by its id | when login is ported | XI, XIX | +| 11 | where a flow lives: a keyed document of its owner, or an object `LoginFlow { id }` | when login is ported | XII | +| 14 | project plugins: per-directory plugins with trust prompts and snapshots | own pass over `plugin-loading.md` | XV | +| 16 | a Durable Object profile | when a DO target becomes concrete | | +| 17 | session files: one `SessionFiles { session }` owner on the server (proposed) | p3 design | XX | +| 18 | session delete: retire by tag plus a session directory, or owner hooks | p3 design | XIX | +| 20 | connection pools: one owner of SSH connections per environment, or per-use connections | p3 design | XX | +| 21 | config files with a single writer: one owner on the server, others follow | p3 design | XVIII | +| 22 | file formats (`envs.json`, downloads, uploads) as versioned interfaces | p3 design | XX | +| 23 | the terminal host process: inside the revision or deliberately outside | p3 design | XX | + +## Open from review round 3 + +Decisions (`apps/p2/docs/p3-reviews/v2-review3-summary.md`; Q1 to Q10 are decided): + +| # | Problem | Proposal | +| --- | --- | --- | +| Q11 | a singleton that does not exist yet: `undefined` or `initial()`? | singletons read `initial()`; `undefined` only for absent keyed documents | +| Q12 | stripping arguments (R7) against "nothing is copied" | copy on change: a new object only when the argument had extra fields; never mutate the caller's value (proposed in the L1 draft) | +| Q13 | `ctx.incarnation` on relayed calls | relays stamp the origin host's incarnation, as they stamp the principal | +| Q14 | a writable holding functions exposed as a state member | only members with a JSON schema in the contract are exposable, checked at serve | +| Q15 | arguments of `local` methods: validated or not | apply R7 to `local` too, or keep `local` as the one exception | + +Cuts the owner may choose (each removes a primitive this guide teaches): + +| # | Cut | What it costs | +| --- | --- | --- | +| C3 | shrink jobs to a runner over documents (no versions, `accepts`, `blocked`, `persist`), or move jobs out of the kernel | job versioning, or `self.job` for core-only users | +| C4 | routines: `on` and `loop` only | `every`'s due time across reloads | +| C5 | merge `lifetime: "host"` objects into the facet | a plugin's always-on natives share one failure domain | +| C6 | remove `broken()` for opens and `retryOn` | nothing; reload and Retry still work | +| C7 | remove provider-side `linger` | ten lines of app code | +| C8 | remove `state: false` (method-only references) | a method-only consumer pays one snapshot | +| C9 | remove `previous` from derives | derives that used it move to `mapShared` | +| C10 | one `cell(initial)` for `constant` and `writable` | nothing | +| C11 | remove `ctx.trace` (use `traceIn(ctx, …)`) | nothing | +| C12 | remove call credits (the byte window covers calls) | nothing | +| C13 | two timeouts (`closeTimeout`, `readyTimeout`) instead of eight | per-thing overrides | +| C14 | remove the `handle.change` and `handle.retire` shorthands | nothing | +| C15 | remove `inVariant` and `expectVariant` (keep `RefIn`) | nothing | +| C16 | remove facet-owned mutable documents and their transaction exception | atomic writes across an object and its facet move to one owner | +| C17 | no harness-specific start-barrier exception | factories before the barrier cannot depend on the harness | + +## Where the sources disagree + +This guide follows the decision log where it is newer than the design, and the design where it is newer than +everything else. The places that need a fix in the sources: + +:::review | Review: drift in the written sources +- `design.md` §26 lists only the deferred decisions; it omits the open review-3 items Q11 to Q15 and the cuts C3 to + C17. +- `design.md` §3 has a garbled sentence, "Values are method arguments are stripped". +- `apps/p2/docs/p3-handoff.md` §3 is stale: it says all R items are decided and, two lines later, that R2 to R5 and + R7 to R26 are open, and "resume R6 next". +- `kernel.md` and `step-*.md` describe the frozen code: host values, a development freeze, consumer-side linger, + JSONL storage, descriptions on the wire, and a link closed on an unknown id. None of these is the target. +- `chord-next-walkthrough.md` lags the log: validation by trust (now R7, every caller), dev mode and freeze (removed + by R9), descriptions once per link (gone with R6), an IPC frame-count window (now one byte window, Q4), job outcome + `{ ok }` (now `{ kind }`), harness activation as the start barrier (R17), registries "newer refused at load" (Q8), + and hooks using `host.root` (now `chord(fn, context)`, Q6). +- `apps/p2/docs/questions.md` uses `self.resource` and `replace` (both rejected) and `host.root` in hooks. +- `apps/p2/docs/p3-reviews/v2-cost-model.md`: the measurements are current, its "Decisions this needs" section is + not (chapter XVII). +- L1's `Implementation` shape differs from the factory return value used in the design (chapter XVI). +::: + +# API Reference {.appendix #api lede="Every name the guide uses, grouped by where you meet it. Working names; only the contract layer is built."} + +## Hosts + +| Call | Meaning | Chapter | +| --- | --- | --- | +| `createHost({ address?, principal?, storage?, principalKey?, revokedBy?, telemetry?, onError?, timeouts })` | a host; with an address it is addressable and its own code acts as `principal`; without, a client | I | +| `host.load([{ name, facet }])` | load facets by plugin name; base code passes its core facet the same way | I | +| `host.connect(link, { principal })` | the link to the parent (at most one) | I | +| `host.accept(link, { principal })` | a link from a client; every request on it acts as `principal` | I | +| `host.route(Kind, { authorize, resolve })` | how to reach hosts of a kind; `resolve` returns `{ link, principal }` | I | +| `host.revoke(r)` | end matching objects and links here and below | VII | +| `host.root(fn, { principal?, signal?, telemetry? })` | a call scope with a `ctx` for code outside objects | VI | +| `host.get(Type, input, { signal, state?, parkTimeout? })`, `host.hold(…)` | a reference from outside objects; `release()` ends it | VI | +| `host.serve(Type, factory, { lifetime?, linger? })` | serve an object type (in a facet) | II | +| `host.registry(Registry)` | the composed value of a registry, a readable | XIII | +| `host.references(Type)` | who holds references to objects of a type on this host (presence) | VIII | +| `host.activity` | `{ references, calls, jobs, lingering }`, for keep-alive | I | +| `host.close()` | end the host: objects, natives, links | I | + +## Inside a factory: `self` + +| Member | Meaning | Chapter | +| --- | --- | --- | +| `self.state(Definition)` | a document handle | V | +| `self.tx(async (tx) => …, ctx)` | a transaction on the object's line | V | +| `self.derive(name, [sources], fn)` | a derived readable | VIII | +| `self.get(Type, input)` | a reference held for the object's life | VI | +| `self.open(open, close, { closeTimeout? })` | a native; `.value` once open | IX | +| `self.routines.on(readable, fn)`, `.every(interval, fn)`, `.loop(fn)`, `.custom(…)` | standing behaviour | X | +| `self.job(Job, { run, finish? })` | owned work that outlives its call | XI | +| `self.contribute(Registry, value)` | a contribution (facets and `lifetime: "host"` objects only) | XIII | +| `self.writable(initial)` | a host-local readable of any in-process value, with `set` | XIII | +| `self.retryOn(readable)` | a trigger that retries a broken activation | VI | + +**Document handles:** `value`, `get(key)` (keyed), `keys`, `change(fn, ctx)` / `change(key, fn, ctx)` and `retire(ctx)` / `retire(key, ctx)` (shorthands for +one-document transactions). **Transactions:** `tx.draft(handle, key?, { tags? })`, `tx.retire(handle, key?)`, +`tx.start(job, args, { key? })`. **Jobs:** `job.start(args, ctx, { key? })`, `job.abort(id)`, `job.running`, +`run.outcome(ctx)`. + +## References and `ctx` + +| Member | Meaning | +| --- | --- | +| `ref.` | a state member (readable: `.value`, subscribe) or a method | +| `ref.state`, `ref.onState(fn)` | `loading`, `live`, `failed` (cause, `retryAt`), `released` | +| `ref.retry()` | retry a failed reference now | +| `ref.release()` | end a reference taken outside objects | +| `ref.members` | the variant's members | +| `ctx.signal`, `ctx.principal`, `ctx.incarnation`, `ctx.get` | cancellation, caller, calling host's incarnation, call-scoped references | +| `ctx.telemetry`, `ctx.trace(name, attributes, fn)` | the current span; a child span with a child `ctx` | +| `ctx.jobId`, `ctx.progress(value)` | in a job's `run` | + +## Definitions + +| Name | What it declares | Chapter | +| --- | --- | --- | +| `hostKind(name, input?)` | a kind of host and its address | I | +| `remote(name, { version, input, placement, variants })` | an object type reachable across links | II | +| `local(name, { version, input, members })` | an object type on its own host | IV | +| `method(args?, result?)` | a method in a base declaration | II | +| `evolve(prev, (e) => …)`, `checkAccepts(contract, accepts)` | the next contract version; the provider's mappings (**built**) | XVI | +| `Implementation` | the type of a provider for the current version (**built**) | XVI | +| `str num int bool nul lit oneOf union obj arr map opt bytes unknown`, `Static` | schema helpers (**built**) | II | +| `defineState(name, { schema, version, initial, checkpointWhen, persist?, keyed?, ordered?, migrate? })` | a document | V | +| `defineTag(name)` | a tag; `store.retireByTag(tag, value)` | V | +| `defineJob(name, { version, args, progress, result, concurrency?, persist?, accepts? })` | a job type | XI | +| `defineRegistry(name, { version, compose, accepts? })` | an extension point | XIII | +| `bindFacet

()` | base code's `defineFacet`, with factories' principals typed as the app's `P` (working name) | III | +| `defineFacet((host, self) => …)` | a plugin's setup for one kind of host; imported from base code (`@studio/core`) | II, III | +| `constant(value)` | a readable that never changes (start data) | IX | +| `broken(error)` | a failure that is not retried by the backoff | VI | +| `inVariant(ref, …)`, `expectVariant(ref, …)`, `RefIn<…>` | narrowing a reference by variant | III | +| `mapShared(fn)` | an array map that keeps unchanged items identical (planned in Delta) | VIII | +| `OutputBuffer`, `Progress` | bounded output and throttled progress (`@earendil-works/chord/output`) | XI | +| `memoryLinkPair()`, `ipcLink(child)`, `webSocketLink(ws)` | link adapters | I | +| `DocumentStorage`, `registerDocumentStorageConformance` | the storage port and its shared test suite | V | + +# Timeouts and Defaults {.appendix #defaults} + +| Timeout or default | Bounds | Default | Override | +| --- | --- | --- | --- | +| `closeTimeout` | a native's `close`, then reported as a possible leak | 30 s | per native | +| call drain | an ending object's own call handlers | = `closeTimeout` | per type | +| `readyTimeout` | activation; an awaited `get` | 60 s | per `get` | +| park deadline | parked calls (link drop, loading, rollout) | 60 s | per `get` (`parkTimeout`); per type | +| routine handover | a new routine waiting for the old handler on reload | 10 s | per routine | +| `finish` retry | backoff of a throwing job `finish` | 1 s to 60 s | per job | +| rollout install | one host installing a revision | 60 s | none | +| linger | an object kept alive after its last reference | per type | per type | +| backoff | retries of transient failures, per address | 1 s doubling to 60 s | none | +| eviction | keyed persisted documents idle in memory | 10 minutes | | +| client WebSocket limits | per part | 16 MiB compressed, 64 MiB decoded | per link kind | +| compression | JSON parts compressed at their origin | above about 16 KiB | | +| worker keep-alive (p2, app code) | idle time before the supervisor stops a worker | 5 minutes | app | + +# Versions {.appendix #versions} + +| Surface | Version | On change | Chapter | +| --- | --- | --- | --- | +| object type | `version` plus fingerprint | `evolve` and `checkAccepts`; otherwise `contract_mismatch` | XVI | +| document | `version` plus schema fingerprint | pure `migrate` on access; no bump fails `broken` | V | +| job | `version` | `accepts` on outcomes and `running`; `blocked` without | XI | +| registry | `version` | `accepts` on contributions; unmatched ones left out and reported | XIII | +| native | none | restarts on reload | IX | +| Durable tasks and documents | Durable's `version` | Durable's `migrate` | XIV | +| link protocol | protocol version | refused; hosts update together | XVII | +| UI components, Chord and app APIs | none yet | exposed as they are | | + +# Errors and States {.appendix #errors} + +`ChordError` has a stable `code` and a finer `reason`; on the wire it is `{ code, reason?, message, details? }` +(specified in `layer-1.md`, not built). + +| Code | Meaning | +| --- | --- | +| `invalid` | input, arguments or placement rejected (`invalid/input`, `invalid/args`, `invalid/placement`) | +| `contract_mismatch` | the consumer's version and fingerprint are neither current nor accepted | +| `forbidden` | the caller may not do this | +| `gone` | the object or hold ended | +| `not_ready` | not activated in time | +| `released` | the reference was released | +| `too_large` | a part exceeded its size limit | +| `unavailable` | could not be reached before the deadline (`unavailable/locked` and others) | +| `aborted` | cancelled by a signal | + +| Type | Values | +| --- | --- | +| `FailureCause` | `{ kind: "transient"; error }` (retried with backoff) or `{ kind: "broken"; error }` (retried only by a trigger) | +| reference state | `loading`, `live`, `failed` (cause, `retryAt`), `released` | +| job outcome | `{ kind: "done"; result }` or `{ kind: "failed"; error: "threw" \| "aborted" \| "lost" }` | +| job phase | `running`, `finishing`, `blocked` | +| rollout status of a provider | `updating` (calls park), rejected (fail at once), same revision (`contract_mismatch`) | +| host install report | `installed`, `installed_failed`, `rejected` | + +# Rejected Designs {.appendix #rejected} + +From `design.md` §31, grouped. Each was considered and is not coming back without a new reason. + +| Area | Rejected | +| --- | --- | +| reload | drain-based handover; object restart for state changes; a factory trial; a migrate dry run; a consumer contract check during staging; reload votes; per-plugin rollback within a host; keeping natives across reloads by name; `restart_required`; host values and a development freeze | +| state | version fences without transactions; volatile documents; per-group lines; optimistic transactions; `tx.get`; a JSONL document storage; validating state and documents on commits or reads; copying values on same-host calls; op-rewriting projections and a view algebra; Durable's Session as the object store | +| objects and references | resources replaced in place (`replace`, `deps`, stale handles); revoking proxies on `.value`; consumer-side linger; detached references counting toward presence; an `optional` flag on references; `opaque` members; callbacks as method arguments; generic call or request ids; a stream type in Chord | +| registries | registries created at runtime; entries resolved through mutable indirection; generic registry keys with a `conflict` flag; numeric middleware priorities; registry composition as a staging check | +| principals and placement | principal field patterns and re-stamping for revocation; `storeKey`; placement as data naming an input field; host-name strings in app code; "magic" input-field matching for delete | +| wire | an outbox for transient delivery errors; cross-relay reset coordination; count limits that refuse calls or holds; contracts taken from the running app | +| process and tooling | a topology layer above the host API; loader variants A and B; Chord ending the process on a storage failure; `defaultProjectTrust: "always"`; ceremony (`native/` directories, `defineNative`, lints); weakening type checking | + +# Glossary {.appendix #glossary} + +**Adapted binding:** a consumer bound at an older accepted contract version, served through the provider's mappings. +**Base code:** the app's own code, imported statically; a change restarts the hosts. **Client:** a tab or TUI host, +not addressable. **Contract:** an object type's declaration (name, input, placement, members, version). +**Contribution:** a value a plugin adds to a registry. **`ctx`:** what every call receives. **Derive:** a value +computed from readables. **Document:** a named JSON value an object owns. **Facet:** a plugin's code for one kind of +host. **Factory:** the synchronous function that declares an object. **Fingerprint:** a hash of a contract's or +schema's normal form. **Flow:** input into running work, kept as state by its owner. **Host:** one participant of the +app, with an identity and an incarnation. **Incarnation:** the id a host gets on every start. **Input:** an object's +key among objects of its type; with the type and the caller's principal key it identifies the object. **Job:** owned work that outlives its call. **Link:** an ordered connection between +two hosts. **Local type:** an object type reachable only on its own host, without principals. **Native:** an OS-level +value opened with `self.open`. **Park:** a call waiting for a reference to become live again. **Placement:** the +function from input to host address. **Plugin:** a directory with a `package.json`, reloadable. **Principal:** who +calls, stamped by the link. **Principal key:** the string that identifies a principal in object identities. **Pure +object:** an object without natives; swapped in place on reload. **Readable:** a value plus its ordered changes. +**Reference:** what `get` returns. **Registry:** an owner's extension point. **Relay:** a host forwarding messages +between two links, rewriting only headers. **Reload:** swapping a plugin's code in running hosts. **Restart:** ending a +process. **Revision:** the complete code set the hosts should run. **Revocation:** ending live access for principals +matched by `revokedBy`. **Routine:** standing behaviour of an object. **Server:** the addressable root host. **Shared +store:** a local object that holds data once behind per-principal views. **Tag:** a typed label on documents, used to +retire them together. **Transaction:** an atomic set of document writes on the object's line. **Variant:** the set of +members a principal gets. **View:** a per-principal remote object in front of a shared store. **Worker:** a host the +server starts for one session. + +# Sources {.appendix #sources} + +| Source | Used for | Weight | +| --- | --- | --- | +| `packages/chord/docs/next/decisions.md` | every decision, review rounds 2 and 3 | wins where newer | +| `packages/chord/docs/next/design.md` | the target design, all chapters | authoritative otherwise | +| `packages/chord/src/next/contracts/` and its `README.md` | chapter XVI | built code | +| `packages/chord/docs/next/layer-1.md` | errors, inputs, argument checks | draft spec | +| `apps/p2/docs/p3-reviews/v2-review3-summary.md` | open Q11 to Q15, cuts C3 to C17 | current | +| `apps/p2/docs/p3-reviews/v2-cost-model.md` | measurements, per-hop costs | current except its last section | +| `apps/p2/docs/p3-functions/` | chapters XVIII to XX | current | +| `apps/p2/docs/p3-handoff.md` | status and order of work | §3 partly stale | +| `packages/durable/README.md` | the harness, submit, tasks, subagents | current | +| `chord-next-walkthrough.md`, `kernel.md`, `step-*.md`, `apps/p2/docs/questions.md` | background only | lag the decisions (see What's Next) | +| DeepSeek Harness at `687f282` (its Cordis tutorial, architecture, persistence, core and HMR docs), the Cordis paper (arXiv 2608.25512) | "Other Ways to Do It" | external, built systems | diff --git a/packages/chord/docs/next/explainer/index.html b/packages/chord/docs/next/explainer/index.html new file mode 100644 index 000000000..e13c03572 --- /dev/null +++ b/packages/chord/docs/next/explainer/index.html @@ -0,0 +1,5409 @@ + + +Chord v2: a guide to building apps that reload + + + + +

+
+
A guide · Chord v2 target design · 2026-10-20
+

Chord v2
A Guide

+

How to build an app like p2 from nothing on Chord v2 and pi-durable, one object at a time, so that new plugin code takes over while the agent keeps running

+
11layers in the rebuild, L1 to L11
~1,300lines built so far (L1 contracts)
6–7ktarget size of Chord v2, without tests
0.004 msDelta's apply per small change
+

This guide teaches the designed API of Chord v2 (packages/chord/src/next, export @earendil-works/chord/next) +from packages/chord/docs/next/design.md and the newer decision log packages/chord/docs/next/decisions.md, which wins +where the two differ. Most of it is not built yet: a first kernel of about 10k lines is frozen as a reference and +Chord v2 is being rebuilt layer by layer; only the L1 contract layer exists (src/next/contracts/, commit +7d2bb8ea8). Code in this guide is therefore the target, not something you can run today. Every chapter says what is +built. API names are working names. Notes marked review are points that are undecided, or where the sources +disagree; they name the files. The reader is assumed to know pi-durable; Chord is explained from the start.

+
+
+ + +
+

Before we begin

+

It is late in the evening. Mario is on his phone, connected to his laptop at home through the Radius relay. In session +S1 the agent is twenty minutes into a refactoring; right now it is running a test suite through bash. A colleague, +Ann, follows the session read-only from her own browser. Mario notices that the notes panel sorts notes the wrong way +round. He fixes one line in the notes plugin on the laptop and presses Reload.

+

What should happen is easy to say. The notes panel on both screens shows the new order. The test run keeps running. The +agent's turn continues as if nothing happened. Ann's view does not flicker into an error. Nothing that was written is +lost, and no process restarts.

+

The obvious ways to get there fail. Restarting the processes loses the turn in flight, the open terminal and every +connection; pi-durable would resume the turn, but the test run and the connections are gone. Hot-swapping modules +in place, as front-end tooling does, leaves the old code alive in closures, timers, event listeners and sockets that +nobody knows about: the old sort function keeps running inside a callback someone stored, a file watcher opened by the +old code is never closed, and two versions of the plugin now write the same data without any order.

+

Chord v2 is the runtime that makes the first outcome normal. It does so by being strict about one thing: every piece +of state, every open OS handle and every piece of running work has exactly one owner, an object, and objects only +declare what they need. Chord itself activates, connects, reloads and ends them. Because Chord knows every timer, +watcher, transaction and reference an object has, it can swap the object's code while its data stays, restart only +what touches the operating system, and route calls between browser tabs, the server and per-session worker processes +without the plugin author writing any of that.

+

This guide teaches Chord v2 by building an app called Studio: a small version of p2 (Pi Mobile), with a server, +one worker process per session, browser tabs as clients, and an agent from pi-durable on each worker. The first +chapters grow one plugin, @studio/notes, until it uses every concept; the last three chapters assemble the rest of a +p2-like app from the same parts.

+
Review: what exists today

Nothing in this guide runs yet except the contract layer. packages/chord/src/next/contracts/ (L1 part 1: remote, +local, method, evolve, checkAccepts, the schema helpers) is built and reviewed. The rest of L1 (inputs, +argument checks, principals, errors) is a draft spec (packages/chord/docs/next/layer-1.md). The frozen kernel in +src/next/*.ts implements an older version of the design (host values, a development freeze, JSONL storage, +descriptions on the wire); do not read it as the target. Build on the design, not on the frozen code.

+
+

Ten things to know

+
    +
  1. An app is a tree of hosts. One server, any number of workers it starts on demand, any number of clients (tabs, +TUIs) that connect to it. Clients reach workers only through the server, which relays.
  2. +
  3. Objects are the unit of everything. An object type is a contract (name, input, placement, members, version); an +object is one instance per input and caller. The first get creates it, later gets share it, the last release ends +it.
  4. +
  5. Factories declare, Chord activates. A factory is synchronous: it declares documents, references, derived +values, OS handles, routines and jobs through self, and returns the members. Chord opens, binds and starts them, +and undoes them in reverse.
  6. +
  7. Links stamp who is calling. A caller's data never says who it is; the link it arrived on does. Each caller gets +its own object and the variant of members it may use.
  8. +
  9. Shared data lives once, behind views. A local shared store holds the data; per-caller remote views expose +what each caller may see. Only stable identities persist documents.
  10. +
  11. State changes only in transactions. Documents are JSON values written in self.tx; commits are atomic for +every viewer and travel as small operations, never as whole values.
  12. +
  13. Failures look like disconnects. A reference survives link drops, restarts and failures: calls park or fail, +values stay readable, and Chord retries by itself. There are no request ids.
  14. +
  15. Everything long-lived has a declared form. Standing behaviour is a routine, work that outlives a call is a job, +input from a person is a flow, behaviour from other plugins comes through a registry. Nothing else may hold a timer +or a callback.
  16. +
  17. Reload swaps pure objects in place and restarts native ones. Objects without OS handles get the new code with +their data and running work intact; objects with OS handles close and reopen them. A broken change leaves the whole +host on its old code.
  18. +
  19. pi-durable runs inside a core object. The agent harness is a host-lifetime object on each worker; models, +environments and tools reach it through registries, and tools reach Chord through chord(fn, context).
  20. +
+

Words used throughout

+

A host is one participant of the app: a process, a browser tab or a Durable Object, identified by a logical +identity and an incarnation that changes on every start. A link connects two hosts. The server is the one +addressable root host; a worker is a host the server starts for one session; a client is a tab or TUI, not +addressable. A plugin is a directory with a package.json; its code for one kind of host is a facet. Base +code is the app's own code (Chord calls its facet the core facet); it restarts the hosts when it changes, while +plugins reload. An object type (or contract) describes objects; an object is one instance; its input is +its key among objects of that type, and its placement computes from the input on which host it lives. A factory builds an +object; self is what a factory declares through. A principal is who calls, stamped by the link; a +variant is the set of members a principal gets. A reference is what get returns. A readable is a value +plus its ordered changes. A document is a named JSON value an object owns, written in a transaction. A +derive is a value computed from readables. A native is an OS-level value (a socket, a watcher, a process) +opened with self.open; an object with natives is a native object, one without is a pure object. A +routine is standing behaviour, a job is owned work that outlives its call, a flow is input into running +work. A registry is an owner's extension point, filled with contributions. ctx is what every call +receives: cancellation, caller, call-scoped references. A reload swaps a plugin's code inside running hosts; a +restart ends a process; a revision names the complete code set the hosts run.

+
Reading the code in this guide

Imports are shown once per file and then omitted. obj, str, arr and friends are Chord's schema helpers (they +return ordinary TypeBox schemas). Helper functions that belong to the app (authenticate, watchDir, uuidv7) are +named and not shown. Where the design gives no name, the guide says it uses a working name.

+
+
+ +
I

The Shape of an App

Which processes a Chord app has, how they find each other, and the start code that connects them.

+

Before any plugin exists, Studio needs processes and connections between them. In Chord each participant is a host, +and the app's own start code creates every host, connects them and decides how workers start. Chord deliberately does +none of that itself: a laptop app forks child processes, a test runs everything in one process, a Cloudflare +deployment routes through the edge. This chapter writes Studio's start code: server.ts, worker.ts, the browser +entry, and a test setup.

+

Three roles

+

A Node app on Chord has one shape:

+
CLIENTS: NOT ADDRESSABLESERVER: EXACTLY ONEWORKERS: ONE PER SESSION, ON DEMANDbrowser tabowner, localbrowser tabguest AnnTUIownerserverServer()workerSession({ id: "S1" })workerSession({ id: "S2" })workerSession({ id: "S3" })WebSocketWebSocketWebSocketIPCIPCA worker for S3 does not exist until something asks for an object that lives there.
Figure 1. The three roles. Clients connect to the one server; the server starts workers and relays between them.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
RoleAddressableWho it is comes fromLifecycle
clientnothe authentication of its linkconnects by itself
serveryes, exactly oneits own declarationruns
workeryes, many, by a typed addressstamped by the server, its parentstarted on demand by the app
+

The server holds what all sessions share (the session list, access, settings) and starts workers. A worker +runs one session: its agent, its files, its terminal connections. Clients are browser tabs and TUIs. Nobody can +send a request to a client, so a client can only serve objects that nobody outside it needs; chapter IV names them +local types.

+

Things that look like hosts but are not: a code mode sandbox (a worker thread inside a worker, owned by an object), a +remote environment's daemon (it speaks its own protocol, owned by an object), the TUI's sandbox behind its launcher. +A detached process such as a terminal host could become another kind of child of the server.

+

Host kinds and addresses

+

Every addressable host has a typed address. The app declares its host kinds once, in base code:

+
// studio/src/hosts.ts
+import { hostKind, obj, str } from "@earendil-works/chord/next";
+
+export const Server = hostKind("server");
+export const Session = hostKind("session", obj({ id: str() }));
+
+Server();                  // the address of the server
+Session({ id: "S1" });     // the address of the worker of session S1
+

There are no host-name strings anywhere in app code. Chord serializes addresses internally; your code only calls the +kinds. An object type names its host with a placement function from its input to an address (chapter II), so the +type checker knows that Session({ id: session }) needs a session field.

+ +

A link is an ordered, two-way channel of messages between two hosts; it only delivers. Chord ships three: a memory +pair for tests, IPC for a Node child process forked with serialization: "advanced", and WebSocket. All three carry +the same binary message format and the same flow control (chapter XVII). Recovery policy belongs to the link adapter: +IPC needs no heartbeat, a WebSocket through a relay needs one, because TCP does not notice that a laptop went to sleep.

+

Each link has a direction. A host has at most one link to its parent (connect) and any number of links from +below (accept for clients, route for workers). Requests arriving on a link act as the principal the link was given; +chapter III explains why that matters.

+

The server

+
// studio/src/server.ts
+import { createHost, ipcLink, webSocketLink } from "@earendil-works/chord/next";
+import { fork } from "node:child_process";
+import { WebSocketServer } from "ws";
+import { Server, Session } from "./hosts.ts";
+import { coreServerFacet } from "./core/server.ts";
+import type { P } from "./principal.ts";
+
+const server = createHost<P>({
+  address: Server(),
+  principal: { kind: "server" },                 // who this host's own code acts as
+  storage: await openServerStorage(dataDir),     // persisted documents (chapter V); memory if omitted
+  onError: (e) => { if (e.kind === "storage_failed") process.exit(75); else log(e); },
+});
+
+await server.load([
+  { name: "@studio/core", facet: coreServerFacet(config) },   // base code, under a fixed name
+  ...(await loadPlugins("server")),                           // [{ name, facet }] from the plugin loader
+]);
+
+const wss = new WebSocketServer({ server: http, path: "/chord", maxPayload: 64 * 1024 * 1024 });
+wss.on("connection", (ws, req) => {
+  const principal = authenticate(req);           // app code: cookie, loopback, device token
+  if (!principal) return ws.close(4401);
+  server.accept(webSocketLink(ws), { principal });
+});
+
+server.route(Session, {
+  authorize: (p, { id }) =>
+    p.kind === "owner" || p.kind === "server" || (p.kind === "guest" && p.session === id),
+  resolve: async ({ id }, { signal }) => {
+    const principal: P = { kind: "worker", session: id };
+    const child = fork("worker.ts", [], { serialization: "advanced", signal });
+    child.send({ id, principal, init: await initFor(id, signal) });   // the app's own start message
+    return { link: ipcLink(child), principal };                       // Chord stamps `principal` on it
+  },
+});
+

Each call has one job:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CallMeaning
createHost<P>({ address?, principal?, ...options })with an address: an addressable host whose own code acts as principal; without: a client. P is the app's principal type
load([{ name, facet }])the facets this host runs, each with its plugin's package.json name; duplicates and invalid names are refused. Base code passes its core facet the same way
accept(link, { principal })a link from a client; every request on it acts as principal
route(Kind, { authorize, resolve })how to reach hosts of a kind. authorize(principal, address) runs before anything starts; resolve(address, { signal }) returns { link, principal }
connect(link, { principal })the link to the parent, at most one, replaced after a reconnect; principal says who the parent is
root(fn, { principal?, signal? })a call scope for code outside objects (a CLI command): gives fn a ctx (chapter VI)
+

Chord has no listen. Entry points differ too much: a loopback WebSocket server, a tunnel, the Radius relay +delivering channels, a Durable Object edge, a memory link in a test. Each entry point authenticates and calls +accept. Radius, for example, is one more line:

+
radius.onChannel((ws) =>
+  server.accept(webSocketLink(ws), { principal: { kind: "owner", access: "relay" } }));
+

The worker

+
// studio/src/worker.ts
+const { id, principal, init } = await startMessage(process);       // app code: the first IPC message
+const worker = createHost<P>({
+  address: Session({ id }),
+  principal,                                                       // { kind: "worker", session: id }
+  storage: await openSessionStorage(init.dataDir, id),             // one storage per session
+});
+await worker.load([
+  { name: "@studio/core", facet: coreWorkerFacet(init) },          // serves the start data (chapter IX)
+  ...(await loadPlugins("worker")),
+]);
+worker.connect(ipcLink(process), { principal: { kind: "server" } });
+

A browser tab

+
// studio/web/main.tsx
+const client = createHost<P>();                                    // no address: a client
+await client.load(await loadPlugins("web"));
+client.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } });
+const notes = await client.get(Notes, { session: "S1" });          // tab → server → worker S1
+

The first get, step by step

+

The tab asks for the notes of S1. No worker for S1 runs yet.

+
browser tabownerserverroute(Session)app codeworker S1placement: Session({ id: "S1" })hold Notes { session: "S1" }not mine: route(Session)authorize(owner, { id: "S1" })trueresolve({ id: "S1" })fork + start messagecreateHost, load, connect{ link, principal: worker S1 }hold (stamped: owner)placement check, run factorybound: snapshotbound: snapshot
Figure 2. The first get for an object on a worker that is not running. The server authorizes, the app starts the worker, Chord stamps the link.
+

resolve runs once per address while its link lives. When the link drops (the worker crashed or the supervisor +stopped it) and references to that worker still exist, Chord calls resolve again by itself, with a per-address +backoff, so a tab that only watches recovers without a click. A worker start that already produced a link finishes +even if the request that triggered it went away.

+

Incarnations make restarts visible. Chord mints a fresh id (a uuidv7) on every host start and sends it in the link +handshake; the other side sees a restart as a new incarnation of the same identity, and references to it re-hold +(chapter VI).

+

Tests: everything in one process

+

The same code runs in a test with memory links. resolve builds the worker host in memory and returns one end of the +pair:

+
server.route(Session, {
+  authorize: () => true,
+  resolve: async ({ id }) => {
+    const [up, down] = memoryLinkPair();
+    const principal: P = { kind: "worker", session: id };
+    const worker = createHost<P>({ address: Session({ id }), principal });
+    await worker.load([{ name: "@studio/core", facet: coreWorkerFacet({ id, dataDir }) }, notesWorker]);
+    worker.connect(up, { principal: { kind: "server" } });
+    return { link: down, principal };
+  },
+});
+

Memory links deliver the same encoded messages asynchronously, so a test runs the production protocol path.

+

What stays app code

+

Starting processes, the start message, authentication, supervising and stopping idle workers are app code. Chord +provides one readable for the last one: host.activity, with { references, calls, jobs, lingering } (viewers, calls +in flight, unsettled jobs, objects kept warm after their last reference). Studio's core reports it to the server +together with the agent's running tasks, and the server's supervisor stops a worker when everything has been zero for +five minutes, as p2 does. Chord counts; it has no idle policy.

+

Host options and timeouts

+

createHost takes the options introduced later in this guide: storage (V), principalKey and revokedBy (III, +VII), telemetry (XVII) and onError. onError receives background errors and { kind: "storage_failed" }; every +app must handle the latter, because Chord never ends the process. Studio exits with its own code so its supervisor +restarts it.

+

Timeouts are host defaults with per-thing overrides; the full table is in the appendix. The two you meet first: +readyTimeout (60 s: how long an activation and an awaited get wait) and the park deadline (60 s: how long a call +waits through a link drop or a restart before failing unavailable).

+
Built so far

Nothing in this chapter is built in Chord v2 yet: hosts, links and routes are layers L6 and L7. The frozen kernel has +working versions with the same shape (createHost, connect, accept, route, load), which the rebuild +simplifies.

+
+
+ +
II

A First Object

A plugin, a contract, a factory, and a browser tab that calls it on another process.

+

Studio's first feature is session notes: a list of short notes per session, shown in a panel, that anyone allowed can +add to. Several places need "the notes of S1": the owner's tab, a guest's tab, later a download route and an agent +tool. If each built its own copy, the copies would drift apart and nobody would know when to clean them up. In Chord +there is one object for "the notes of S1", everyone asks for it by its type and input, and Chord routes them all to it.

+

The plugin's files

+

A plugin is a directory with a package.json. Its name (npm style, globally unique and stable) is the plugin's +identity: Chord finds its stored documents, reports its errors and orders its contributions by it, never by a +directory name or load order.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FileRuns onHolds
package.jsonread by the loader"name": "@studio/notes"
contract.tsevery host that imports itobject types: names, shapes, method signatures; no running code
worker.tsevery session workerthe worker facet
server.tsthe serverthe server facet (later: the download route)
web.tsxbrowser tabsthe web facet: the panel
+

The contract

+

An object type is a contract: a name, an input (the object's key among objects of this type), a placement +(on which host it lives, computed from the input), the members (state you can read and methods you can call) and a version.

+
// notes/contract.ts
+import { arr, int, method, obj, remote, str, type Static } from "@earendil-works/chord/next";
+import { Session } from "@studio/core/hosts";
+
+export const Note = obj({ id: str(), text: str({ max: 2000 }), by: str(), at: int() });
+export type Note = Static<typeof Note>;
+
+export const Notes = remote("notes", {
+  version: 1,
+  input: obj({ session: str() }),                          // the key: { session: "S1" }
+  placement: ({ session }) => Session({ id: session }),    // input → the host's address
+  variants: {
+    editor: {
+      state: { notes: obj({ items: arr(Note) }) },
+      methods: {
+        add: method(obj({ text: str({ min: 1, max: 2000 }) }), str()),   // returns the new id
+        remove: method(obj({ id: str() })),                              // no result
+      },
+    },
+  },
+});
+

remote means objects of this type can be reached from any host, across links (chapter IV introduces local). The +schema helpers (obj, str, int, arr, map, opt, lit, oneOf, union, bool, num, nul, bytes, +unknown) return ordinary TypeBox schemas; short options such as { min, max } bound lengths or values. Schemas must +describe data: no callbacks, no class instances, no custom codecs.

+

method() has four forms: method() (no arguments, no result), method(args), method(args, result) and +method(undefined, result).

+

A contract contains no code that runs, so every host can import it. When you declare it, Chord materializes its +description and computes a fingerprint, a hash of its input, variants and members. Two hosts compare contracts by +version and fingerprint (chapter XVI). Placement is not part of the fingerprint.

+

Placement

+

The input says which object; placement says where it lives. The caller runs the placement function on the input and sends the resulting address with its request; the server +routes by it. The provider runs the function too and refuses an object that is not its own with invalid/placement. +So a worker for S2 never builds "Notes for S1", even if a caller has an outdated rule. Because placement is a typed +function, a Session placement on a type whose input has no session field does not compile.

+

Inputs are normalized before anything else: defaults are applied, unknown fields removed, and a canonical key +computed. { session: "S1" } and { session: "S1", extra: 1 } are the same object.

+

The facet and the factory

+

A facet is a plugin's setup for one kind of host. It runs once when the plugin loads on that host and tells the +host which object types it serves:

+
// notes/worker.ts
+import { defineState } from "@earendil-works/chord/next";
+import { defineFacet } from "@studio/core";          // Studio's defineFacet: principals typed as Studio's P (chapter III)
+import { Note, Notes } from "./contract.ts";
+
+const NoteList = defineState("notes.list", {      // a document (chapter V); in memory for now
+  schema: obj({ items: arr(Note) }),
+  version: 1,
+  initial: () => ({ items: [] }),
+  checkpointWhen: () => true,
+});
+
+export default defineFacet((host, self) => {
+  host.serve(Notes, ({ session }, principal, self) => {
+    const list = self.state(NoteList);            // THE note list of this object
+    return {
+      variant: "editor",
+      notes: list,                                // a document handle is a readable: expose it as state
+      add: ({ text }, ctx) =>
+        self.tx(async (tx) => {
+          const id = uuidv7();
+          (await tx.draft(list)).items.push({ id, text, by: "someone", at: Date.now() });
+          return id;
+        }, ctx),
+      remove: ({ id }, ctx) =>
+        list.change((d) => { d.items = d.items.filter((n) => n.id !== id); }, ctx),
+    };
+  });
+});
+

Read host.serve(Notes, factory) as "this host builds Notes objects like this". The factory receives the +normalized input, the caller's principal (chapter III) and self, through which it declares what the object needs. +It returns the members of one variant, tagged with variant.

+

Factories declare; Chord activates. The factory is synchronous and has no side effects. It does not await, open +files or start timers. self.state(NoteList) does not load anything: it declares that the object has this document +and returns a handle that stays valid for the object's whole life. Chord then performs the activation: it loads the +document, binds references, opens OS handles, starts routines, and only then lets calls in. The rule exists for +reload: a reload runs the factory again, and anything the factory did itself would happen twice.

+

Calling it

+
// notes/web.tsx
+export function NotesPanel({ session }: { session: string }) {
+  const notes = useRef(Notes, { session });                 // app-side hook over host.get / release
+  const items = useReadable(notes?.notes)?.items ?? [];
+  return (
+    <section>
+      {items.map((n) => <p key={n.id}>{n.text}</p>)}
+      <AddBox onAdd={(text) => notes?.add({ text })} />
+    </section>
+  );
+}
+

Underneath the hooks is the reference API:

+
const notes = await client.get(Notes, { session: "S1" });   // live and hydrated once awaited
+notes.notes.value.items;                                    // the current list
+notes.notes.subscribe((v) => render(v.items));              // every committed change
+const id = await notes.add({ text: "check the flaky test" });
+notes.release();                                            // the object may end now
+

An awaited reference is live and hydrated: every state member has a value. State members are readables: +.value is the current value, subscribe is called after every commit. The first get for { session: "S1" } runs +the factory; every later get for the same input and caller shares that object. The object ends when the last +reference is released (a type may keep it a little longer with linger).

+
browser tabserverworker S1Notes { S1 }call add { text }relay (header only)strip + validate argsself.tx: push note, commitresult: idstate: ops of the commitresultstate, then result
Figure 3. A call to add. Arguments are checked once at the provider; the commit reaches the viewer as one small change.
+

Two rules for members

+

Methods return data, never objects. A method that wants to point you to another object returns its address +({ session: "S2" }), and you get it yourself. That keeps every reference owned by someone who releases it.

+

No callbacks as arguments. A callback carries code, and everything it closes over, from one object into another. +Suppose an inbox object passed onFile to another object, which stored it. After a reload of the inbox's plugin, the +other object still calls the old onFile; after the inbox ends, it calls into an object whose file watcher is +already closed. Chord forbids this by construction: members are data (JSON plus bytes). Behaviour from elsewhere goes +through registries (chapter XIII), multi-step work through jobs (XI), input from a person through flows (XII).

+

What is wrong with this version

+

This first version works for one person. It has two problems, which the next two chapters fix. Everyone gets the +editor variant, including a read-only guest. And, less obvious, the owner and a guest do not see the same notes: as +chapter III shows, a remote object exists per caller, so each caller's Notes object has its own NoteList.

+
Built so far

remote, method, the schema helpers, Static and fingerprints are built (src/next/contracts/). Serving, get, +references and documents are layers L3 to L5.

+
+
+ +
III

Who Is Calling

Principals: how Chord knows who calls, why a caller cannot lie about it, and how the answer picks the object and its members.

+

The owner may add and remove notes. Ann, a read-only guest, may only read them and should not even see who wrote +what. The object must therefore know who is calling, and the caller must not be able to claim to be someone else. In +Chord the answer is a principal, a small JSON value the app defines and the links stamp.

+

The app's principal type

+

Chord defines no principal kinds; the app does. Every variant has a kind. Studio uses p2's set:

+
// studio/src/principal.ts
+export type P =
+  | { kind: "owner"; access: "local" | "relay" | "device"; device?: string }
+  | { kind: "guest"; session: string; mode: "read" | "write"; grant: string; revision: number }
+  | { kind: "server" }
+  | { kind: "worker"; session: string };
+ + + + + + + + + + + + + + + + + + + + + + + +
kindWho
ownerthe person running the app, on loopback, through the relay, or on a paired device
guestsomeone invited into one session, with a mode and the grant (and its revision) that let them in
serverthe server's own code calling a worker
workera session worker calling the server
+ +

A principal never comes from a request's data. The server authenticates a client once, when its WebSocket connects, +and stamps the result on the whole link (server.accept(link, { principal })). The server tells Chord who each worker +is when it resolves it (return { link, principal }). When the server relays a client's request to a worker, Chord +writes the client's principal into the forwarded request; a principal field coming from a client is always +overwritten, and only a trusted parent link may carry one.

+

Where principals come from is app code. Studio's sharing feature creates a share link per session and mode; a guest +who opens it gets a guest record (grant g1) and a cookie; authenticate(req) turns the cookie into +{ kind: "guest", session: "S1", mode: "read", grant: "g1", revision: 1 } on every new connection. Chord never sees +cookies or grants, only the finished principal.

+

The caller is part of the identity

+

A remote object exists once per (type, input, principal key). The principal key is a string computed from +the principal; by default its canonical JSON (sorted keys). The app may pass its own principalKey to createHost, +and must pass the same function on every host, or identities split silently.

+
const principalKey = (p: P) =>
+  p.kind === "owner" ? `owner:${p.access}:${p.device ?? ""}`
+  : p.kind === "guest" ? `guest:${p.grant}:r${p.revision}`
+  : p.kind === "worker" ? `worker:${p.session}`
+  : p.kind;
+

The design wants the key to include how the client is connected and the grant's revision, so that two access paths +never share objects and a changed grant never reuses the old grant's object:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CallerPrincipal keyNotes { S1 } object
owner, tab A, localowner:local:1
owner, tab B, localowner:local:1 (shared with tab A)
owner through the relayowner:relay:2
guest g1, revision 1guest:g1:r13
guest g1 after a change to write mode, revision 2guest:g1:r24 (new)
+

Variants

+

The contract lists named sets of members; the factory receives the principal and returns the variant that principal +may use:

+
// notes/contract.ts (version 1, final form for now)
+variants: {
+  editor: {
+    state: { notes: obj({ items: arr(Note) }) },
+    methods: { add: method(obj({ text: str({ min: 1, max: 2000 }) }), str()), remove: method(obj({ id: str() })) },
+  },
+  reader: {
+    state: { notes: obj({ items: arr(PublicNote) }) },     // no `by`: authors are hidden
+  },
+},
+
// notes/worker.ts
+host.serve(Notes, ({ session }, principal, self) => {
+  if (canWrite(principal, session)) return { variant: "editor", notes: …, add: …, remove: … };
+  return { variant: "reader", notes: … };
+});
+

A reader has no add at all. Chord refuses a call to a member that is not in the caller's variant, so add needs no +check of its own. Three rules keep variants compatible: a member name has the same kind (state or method) in every +variant that has it; a method has the same arguments and result in every variant; only state schemas may differ per +variant (the reader's notes have no authors). remote(...) enforces them at the type level, and at runtime for +contracts that arrive as data.

+

How a facet knows P

+

P is Studio's type, not Chord's, and every host is created with it (createHost<P>). A plugin's facets are written +and compiled separately, then handed to host.load as values, so the type checker cannot infer P from the host into +the factory's principal. Studio's base code therefore exports its own defineFacet, already bound to P, and every +facet of every plugin imports that one instead of Chord's:

+
// studio/src/facet.ts (base code, exported as @studio/core)
+import { bindFacet } from "@earendil-works/chord/next";
+import type { P } from "./principal.ts";
+
+export const defineFacet = bindFacet<P>();         // a facet's principals are Studio's P
+export type { P, Revocation } from "./principal.ts";
+
// notes/worker.ts (server.ts, web.tsx and tui.ts import the same defineFacet)
+import { defineFacet } from "@studio/core";
+
+export default defineFacet((host, self) => {
+  host.serve(Notes, ({ session }, principal, self) => {   // principal: P
+    if (principal.kind === "guest" && principal.mode === "read") return { variant: "reader", notes: … };
+    …
+  });
+});
+

The import is cheap: @studio/core is a host package plugins already import at runtime (the contract imports Session +from it), P is a type-only import, and bindFacet is a one-line function. The host checks the pairing: load +accepts only facets typed for its own P, so a facet compiled against another app's principal type, or one built with +Chord's untyped defineFacet, does not compile into Studio's start code. Client facets (web.tsx, tui.ts) use the +same defineFacet; they rarely see a principal, because clients serve only local types.

+

None of this is checked at runtime, and none of it needs to be: the principal a factory receives is the one the link +stamped.

+
Review: binding facets to the app's principal type (open)

The design shows defineFacet((host, self) => …) without saying how P reaches the factory. The frozen kernel takes +it as an explicit type parameter, defineFacet<P>(…), defaulting to { kind: string } +(packages/chord/src/next/facet.ts), and the playground writes defineFacet<AppPrincipal>(…). The pre-bound +defineFacet above (bindFacet is a working name) is the owner's preferred direction, recorded as an open point in +packages/chord/docs/next/layer-1.md §4.

+
+

Narrowing on the caller's side

+

Owner and guest run the same web.tsx, and which variant they get is known only at runtime. The types force every +caller to narrow before using variant members:

+
const notes = await client.get(Notes, { session: "S1" });
+notes.add({ text: "hi" });                         // type error: `add` exists only in "editor"
+if (inVariant(notes, "editor")) await notes.add({ text: "hi" });
+expectVariant(notes, "editor");                    // or assert (throws)
+function Editor({ notes }: { notes: RefIn<typeof Notes, "editor"> }) { … }
+

The protection is not the type checker; it is Chord refusing the call at the provider. A client that ignores the +types gets the same error.

+

Admission before a worker starts

+

Principals also decide whether a caller may reach a host at all. route(Session, { authorize }) runs before Chord +resolves an address, so a guest of S1 asking for anything in S2 is refused at the server, and no worker starts for +them. Finer authority lives in the per-principal view's methods; there is no asynchronous "may this caller activate +this object" guard. An object whose address carries authority (say, a login flow by id) may exist; what a caller can +see and do is decided by the variant its factory picks.

+

Inside methods

+

ctx.principal is the caller of the current call. In Studio's notes the editor's add uses it to record the +author:

+
add: ({ text }, ctx) => store.add({ text, by: authorOf(ctx.principal) }, ctx),
+

Where principals act, in summary:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
PurposeWhere
identity: which object a caller getsprincipal key
variant: which membersthe factory
admission: whether a host is reachableroute.authorize
attribution and fine checksctx.principal in methods
revocationrevokedBy (chapter VII)
+

The bug from chapter II

+

Now the bug is visible. The owner's Notes { S1 } and Ann's Notes { S1 } are two objects (rows 1 and 3 above), and +each declared its own NoteList. The owner's notes are not in Ann's list. Per-caller objects must not hold the data; +they must be views over something that exists once. That is the next chapter.

+
+ +
IV

One Store, Many Views

Local types, and the pattern every Chord plugin uses: shared data once in a store, per-caller views in front of it.

+

Every caller has its own Notes object, which is what makes per-caller variants possible. But data must exist once. +The fix is the most common pattern in Chord: a shared store, a local object that holds the data and does the +heavy work once, behind thin per-principal views, the remote objects callers get.

+

local types

+
// notes/contract.ts
+export const NoteStore = local("notes.store", {
+  version: 1,
+  input: obj({ session: str() }),
+  members: {
+    notes: obj({ items: arr(Note) }),                          // state: a schema
+    public: obj({ items: arr(PublicNote) }),
+    add: method(obj({ text: str(), by: str() }), str()),       // methods: method(...)
+    remove: method(obj({ id: str() })),
+  },
+});
+

A local contract has flat members and no variants. The differences:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
remotelocal
reachable fromany host, across linksits own host only
identity(type, input, principal key)(type, input)
principals, variantsyesno
factory(input, principal, self)(input, self)
argumentsstripped to the provider's schema and validated, for every callersee the review note below
values on the same hostpassed by reference, never copiedpassed by reference, never copied
ctx.principalthe callernone
may persist documentsno (chapter V)yes
+

A local object has no principal: it is trusted code on its own host, acting for the host. So attribution ("who wrote +this note") is passed as data (by), by the view that knows who called. Clients serve only local types, because +nobody can address a client.

+

The pattern

+
// notes/worker.ts
+export default defineFacet((host, self) => {
+  host.serve(NoteStore, ({ session }, self) => {
+    const list = self.state(NoteList);
+    const pub = self.derive("public", [list], ([l]) => ({ items: l.items.map(stripAuthor) }));
+    return {
+      notes: list,
+      public: pub,                                   // computed once, for every reader
+      add: ({ text, by }, ctx) =>
+        self.tx(async (tx) => {
+          const id = uuidv7();
+          (await tx.draft(list)).items.push({ id, text, by, at: Date.now() });
+          return id;
+        }, ctx),
+      remove: ({ id }, ctx) => list.change((d) => { d.items = d.items.filter((n) => n.id !== id); }, ctx),
+    };
+  });
+
+  host.serve(Notes, ({ session }, principal, self) => {
+    const store = self.get(NoteStore, { session });   // a reference, declared: held for the view's life
+    if (canWrite(principal, session)) return {
+      variant: "editor",
+      notes: store.notes,                             // the store's readable, exposed as-is
+      add: ({ text }, ctx) => store.add({ text, by: authorOf(principal) }, ctx),
+      remove: (args, ctx) => store.remove(args, ctx),
+    };
+    return { variant: "reader", notes: store.public };
+  });
+});
+
PER-PRINCIPAL VIEWS (REMOTE)SHARED STORE (LOCAL)Notes { S1 }owner · editorNotes { S1 }guest Ann · readerNotes { S1 }guest Bob · readerNoteStore { S1 }notes, public (a derive)self.getself.getself.getAuthority is the view's: the reader variant has no add. The store does not check; it is trusted and acts for the host.
Figure 4. One store, three views. The redaction runs once in the store; both readers read the same value.
+

self.get(NoteStore, { session }) in a factory declares a reference that the view holds for its whole life; Chord +binds it during activation and waits until the store is live (chapter VI). The view exposes the store's readables +directly. Owner, Ann and Bob now give three Notes objects, one NoteStore, one note list and one redacted list.

+

Any heavy or shared part belongs in a store like this: a pi-durable watch, a file watcher, a sandbox, a search index. +Base code uses the same pattern with the same primitives; nothing about it is special to core.

+

Derived values, briefly

+

self.derive(name, sources, fn) computes a value from readables and recomputes it when a source changes; viewers get +only the difference. The reader's list above is one. Chapter VIII covers derives in depth, including the one footgun +that matters: stripAuthor must return the same object for an unchanged note, or every commit costs the size of the +whole list. The tool for that is mapShared.

+

Arguments are checked at the provider

+

Trusted code is not type-checked code: plugins are bundled with esbuild without type checks, and agents write +JavaScript plugins. So the provider checks arguments itself (decision R7): every method's arguments are stripped to +the provider's own schema (unknown fields removed) and validated with it, whoever the caller is, and never with a +schema a peer sent. Schemas are compiled lazily on first use and cached per schema object. Bytes are checked as leaves +(instanceof Uint8Array plus an optional length bound), never walked. Stripping copies only when a field was removed; +the caller's value is never mutated (proposal Q12, in the L1 draft). Results and state are not validated: they come +from the provider.

+

On the same host nothing is copied: arguments and results are passed by reference. Values are immutable by +convention (Delta's rule, as in Durable): nobody mutates a value it received, and Chord neither copies nor freezes +values to enforce it. There is no development mode that would freeze them; one behaviour everywhere (R9).

+
Review: are local arguments validated? (Q15, open)

Design §5's table says local arguments are "neither validated nor copied"; §3 and R7 say arguments are validated +"for every caller". Review round 3 raised this as Q15 (apply R7 to local too, or keep local as the one exception), +and layer-1.md lists it as open. Until it is decided, write local methods so that they do not depend on either +answer: trust the shape, but never trust content (lengths, ids) that came from a remote caller without the view +checking it. chord-next-walkthrough.md concept 6 still describes the older rule ("validated only when the caller is +not a host principal"), which R7 replaced.

+
+

Choosing between the two

+

Use a remote type for what callers address: views, things with per-caller authority. Use a local type for what +exists once per input on one host: stores, caches, owners of OS handles, and the core objects a host provides +(chapter IX). If a type is reached across a link, it is remote; if a type persists documents, it is local (or a +host-lifetime object, chapter V).

+
+ +
V

Writing It Down

Documents and transactions: how an object stores data, changes it atomically, survives restarts and migrates it.

+

Notes must be stored, changed safely while others read them, sent to viewers as small changes, and survive a worker +restart. An object's data lives in documents: named JSON values the object owns, changed only in +transactions. The design is a port of pi-durable's store, so most rules will look familiar; this chapter states +them for Chord.

+

Defining a document

+
// notes/worker.ts
+const NoteList = defineState("notes.list", {
+  schema: obj({ items: arr(Note) }),
+  version: 1,
+  initial: () => ({ items: [] }),
+  persist: true,                                                   // stored; default: memory only
+  checkpointWhen: (_value, _ops, info) => info.deltasSinceBase > 64,
+});
+
+const Draft = defineState("notes.draft", {                         // one unfinished note per author
+  schema: obj({ text: str() }),
+  version: 1,
+  initial: () => ({ text: "" }),
+  keyed: true,                                                     // many per object, by key
+  persist: true,
+  checkpointWhen: () => true,                                      // small: a full base per write
+});
+

checkpointWhen is required on every definition, with no default (R14, as in Durable since pi 5c1736f82): it says +when a commit stores a complete base instead of a delta, which bounds how much is replayed on load. Small documents +checkpoint on every write; growing ones after a number of deltas.

+

A document's root is always a JSON object (lists live in a field). Documents hold JSON only; bytes are a value and wire +type, not a document type.

+

Singleton and keyed

+
host.serve(NoteStore, ({ session }, self) => {
+  const list = self.state(NoteList);      // THE note list of this object
+  const drafts = self.state(Draft);       // all drafts of this object
+  …
+});
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
singleton (no keyed)keyed
how manyone per objectmany per object, by a short key
loadedat activation, kept in memoryon demand
readlist.value (synchronous)await drafts.get(key): the value or undefined
list(it is one value)for await (const key of drafts.keys()) pages keys
expose as stateyes: the handle is a readableno: expose a method or a singleton index
+

There is no live key list. A collection shown live is a singleton (an object holding a list or a map); a large +collection is a method returning pages.

+

Transactions

+

All writes go through self.tx(fn, ctx). Inside, await tx.draft(handle, key?, { tags? }) returns a draft of the +current document, or a draft over initial() if it is absent. Drafts record operations (assignments, push, +Object.assign); asking for the same document twice in one transaction returns the same draft. tx.retire(handle, key?) removes a document. There is no tx.get: reading is drafting, and a draft is an overlay over the current +value, so nothing is copied. A draft you do not change commits nothing, so reading never stores an absent document.

+
saveDraft: ({ author, text }, ctx) =>
+  self.tx(async (tx) => { (await tx.draft(drafts, author)).text = text; }, ctx),
+

Shorthands for one document: handle.change(fn, ctx) (keyed: handle.change(key, fn, ctx)) (loads the document, then runs the synchronous fn on +its draft) and handle.retire(ctx) (keyed: handle.retire(key, ctx)).

+

Four patterns

+

Two decisions (R12 and R13, both as in Durable) shape how transactions are written. Drafts are revoked when the +transaction settles, so a transaction returns at most plain values, never something taken from a draft. And +there is no existence check inside a transaction.

+
// 1. Return data: write in the transaction, read after the commit.
+await self.tx(async (tx) => { (await tx.draft(list)).items.push(note); }, ctx);
+return list.value.items.length;                    // the committed revision, or a newer one
+
+// 2. Remove if present: retire of an absent document is a no-op.
+await drafts.retire(author, ctx);
+
+// 3. Create with tags if missing: the draft creates it, only if it changes.
+await self.tx(async (tx) => {
+  (await tx.draft(drafts, author, { tags: [SessionTag(session)] })).text = text;
+}, ctx);
+
+// 4. Branch on existence: read first, outside the transaction.
+if ((await drafts.get(author)) === undefined) { … }
+

Durable's rule

+

Inside the callback, only await tx.*. No other objects, no network, no models, no processes, no second self.tx, no +reads past tx. Breaking it is not an error but a hang: the object's line is held while the callback waits. Fetch +first; decide and write inside.

+
// Wrong: a fetch inside the transaction holds the object's line.
+await self.tx(async (tx) => { (await tx.draft(list)).items.push({ …, title: await fetchTitle(url) }); }, ctx);
+// Right: fetch, then write.
+const title = await fetchTitle(url, ctx.signal);
+await self.tx(async (tx) => { (await tx.draft(list)).items.push({ …, title }); }, ctx);
+

What a commit does

+
INSIDE SELF.TXAFTER THE WRITEwait for the lineper objectrun the callbackdrafts record opsprepareDelta checks placed valuesstorage batchdeltas or a baseadoptpointer swapnotify locallyonce, with the opssendone batch per linkrevoke draftstouching one throwsstoredAn error or an abort before the storage write discards everything. Storage commits are serialized per host.
Figure 5. The commit sequence, Durable's. Nothing is copied, frozen or revalidated; viewers get one batch per commit.
+

Lines are per object, never across objects; reads outside a transaction do not take the line. Subscribers on the same +host get the new value and the commit's operations by reference. Remote viewers get the commit's operations, never the +document again; chapter XVII shows how one commit touching several holds still arrives as one message.

+

Tags and deleting by tag

+

Tags are typed tokens set when a document is created:

+
export const SessionTag = defineTag<string>("studio.session");     // base code, shared by every plugin
+await self.tx(async (tx) => { (await tx.draft(drafts, author, { tags: [SessionTag(session)] })).text = text; }, ctx);
+// when the session is deleted
+await store.retireByTag(SessionTag, "S1");                          // every document tagged S1
+

The design names the operation store.retireByTag(tag, value) without saying who holds the store; session delete is +one of the app decisions still open (#18, chapter XIX).

+

Who may persist

+

persist: true is allowed only in local objects and lifetime: "host" objects: identities without a principal. A +remote object's identity contains the principal key, which changes with the access path and the grant revision. Its +stored documents would scatter across access paths and be orphaned after a reconnect. The rule is enforced at the type +level (self.state in a remote factory accepts only non-persisted definitions) and at declaration ("persist needs a +stable identity: put the documents into a local shared store"). This is the second reason for the shared store.

+

What survives a host teardown (a crash, a restart) is exactly what is persisted. In-memory documents live as long as +their object: they survive reloads and native restarts of a live object, and go when it ends. Everything else +(natives, derives, routines, references) is rebuilt on activation.

+

Facet documents

+

A facet has its own identity (its plugin's name) and may hold documents too, for plugin-wide data such as +preferences. Objects the facet serves may read and write their facet's documents, and one transaction may span the +object's own documents and its facet's (lock order: own line, then the facet's). The facet never touches a served +object's documents.

+

Ownership across plugins

+

No plugin reads or writes another plugin's documents, Chord or Durable. Data crosses plugins only through state and +methods. Data that core enforces lives in a format core owns and is handed to core through its API when the data is +created (as with Durable's createConversation({ init })), never written into core's documents.

+

Versions and migration

+

A change of shape or meaning raises version and adds a pure migrate(old, from):

+
const NoteList = defineState("notes.list", {
+  schema: obj({ items: arr(NoteV2) }),           // NoteV2 adds `pinned`
+  version: 2,
+  migrate: (old, from) => ({ items: old.items.map((n) => ({ ...n, pinned: false })) }),
+  initial: () => ({ items: [] }),
+  persist: true,
+  checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64,
+});
+

Documents migrate on access by the new definition: singletons during activation (hydration counts as access), keyed +ones on their next access. The migrated base is persisted with the next commit; right after a bump, a read-only +transaction writes it.

+

Chord does not walk values against schemas. Instead, each definition computes a schema fingerprint once +(descriptions and titles ignored), and every stored record keeps the fingerprint it was written with. On access Chord +compares them: the same version with a different fingerprint fails the object broken ("schema changed without a +version bump"). So every structural change needs a bump, an additive one with a trivial migrate. The one exception +to "no value checks" (R8): the result of migrate and of initial() is validated against the new schema once, when +produced. A failed check, a throwing migrate, or old code meeting a newer document refuses the access, keeps the +stored data and fails the owning object. A changed meaning without a schema change cannot be detected; that is on +you.

+

Storage

+

Behind the store is a DocumentStorage, passed as createHost({ storage }); memory by default. The server has one +shared storage, each worker one for its session.

+
interface DocumentStorage {
+  commit(writes: readonly DocumentWrite[]): Promise<Seq>;            // create/change/retire, atomic
+  find(address: DocumentAddress): Promise<DocumentRecord | undefined>;
+  load(id: string): Promise<StoredDocument | undefined>;             // version, value, deltasSinceBase
+  scan(query: { owner: string; definition: string }, limit: number, cursor?: string):
+    Promise<{ keys: string[]; next?: string }>;
+  byTag(tag: string, value: string, limit: number, cursor?: string): Promise<{ ids: string[]; next?: string }>;
+  mintId(): Promise<string>;
+}
+

Chord ships a memory storage and a SQLite storage (over @earendil-works/chord/sqlite, the driver layer that moved +from Durable; Node and Durable Object drivers). Storages check only metadata (incarnation, version, sequence, address +occupancy, fingerprint) and write the supplied delta or base; they never rebuild or validate content on a commit. +Chord documents never commit together with Durable transcripts: Chord has its own storage, not an adapter over +Durable's. Your own storage can prove itself with registerDocumentStorageConformance(runner, name, withStorage) from +@earendil-works/chord/testing.

+

Keyed persisted documents that nobody reads, writes or watches for 10 minutes leave memory and reload on the next +access. A storage error ends the store and closes the host; onError receives { kind: "storage_failed" } and the +app decides (chapter I).

+

Kinds of state

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KindForAPI
documentdata the object ownsself.state, self.tx
derivecomputed valuesself.derive (chapter VIII)
source readableexternal data: Durable views, files, logsvalue plus ordered changes; feeds derives
paged methodlarge or queryable collectionsa method returning pages (chat history)
plugin databasefull-text search, large blobsa native opened by its owner (chapter IX)
+

App state lives in Chord documents. Only transcript-bound data, which must fork and rewind with a conversation, lives +in Durable.

+
Review: does an absent singleton read undefined or initial()? (Q11, open)

Design §9 says "whether a document exists is read outside the transaction (handle.value / handle.get give +undefined for an absent one)" and also "a singleton that does not exist yet reads initial() through its handle; +retiring resets it to initial()". Review round 3 proposes: singletons always read initial(), only absent keyed +documents read undefined, and singleton existence needs no API. This guide follows the proposal (list.value is +never undefined); it is not decided.

+
+
Built so far

Documents, transactions and storages are layer L3, a port of Durable's session store; not started. The frozen kernel +has a working version with a JSONL storage, which the design rejected.

+
+
+ +
VI

Holding On

References, the object lifecycle and ctx: what a caller sees while objects start, fail, restart and move, and why nobody writes retry code.

+

A tab shows the notes of S1 for an hour. During that hour the laptop sleeps, the worker is stopped for idleness and +started again, the notes plugin reloads twice and the store's storage hiccups once. The panel should show the notes the +whole time, say "reconnecting" when it cannot reach them, and recover by itself. In Chord this is what a +reference does; plugin code contains no retry loops and no request ids.

+

Three ways to hold

+ + + + + + + + + + + + + + + + + + + + + + + +
WhereAPIReleased
a factoryself.get(Type, input)when the object ends
a callawait ctx.get(Type, input, { state? })when the call ends
outside objectsawait host.get(Type, input, { signal, state?, parkTimeout? })ref.release()
+

Use self.get for what an object needs its whole life (the view's store), ctx.get for what one call needs, and +host.get only in UI code, tests and entry points. { state: false } binds for methods only, without receiving state. +A reference keeps its object alive while held.

+

The object lifecycle

+
(get) → activating → live → ending → ended
+             └→ failed (backoff) → activating …
+

Activating: Chord binds the declared references and waits until they are live and hydrated, loads singleton +documents, opens natives (one after another, in declaration order), computes derives and starts routines. The object +goes live when all of that succeeded. Calls wait behind its gate until then.

+

Ending: routines stop, the object's calls and jobs are aborted as their rules say, natives close in reverse order, +references are released. An object with lifetime: "referenced" (the default) ends when its last reference goes, +after the type's linger if it has one. A lifetime: "host" object (only for types without a required input) lives +as long as the host. A running, finishing or blocked job counts as a reference (chapter XI).

+

ref.state

+

Every reference reports what is going on as a tagged value; listeners get each new one:

+
type RefState =
+  | { kind: "loading" }                                                // first hold, reconnecting, restarting
+  | { kind: "live" }
+  | { kind: "failed"; cause: FailureCause; retryAt: number | undefined } // undefined: no automatic retry
+  | { kind: "released" };
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
SituationCallsValues
link droppark until reconnectedreadable, loading
object loading or restartingpark until livereadable, loading
rollout: provider ahead or installingpark until installedreadable
object failedreject at once with the causefailed, with cause and retryAt
+

Values stay readable through all of this: the panel keeps showing the last notes. Parked calls share one deadline (60 +s by default, parkTimeout per get), then fail unavailable. Otherwise a call ends only through its caller's +signal; there is no per-call deadline. A call that was running on an instance that ends fails with an error and is +not resent: Chord cannot know whether it took effect.

+

Errors look like disconnects. A consumer handles a failed call exactly as a dropped link: show the error, let the +user retry. Work that must not run twice carries its own idempotency key as data (Durable's requestId, a job's +key).

+
The worker restarts while a tab watches the notes

Watching

The tab holds Notes { S1 }. ref.state is live; notes.notes.value has 40 notes.

+

The worker goes away

The supervisor stops worker S1 (or it crashes). The server's IPC link to it drops. The server tells the tab that its +hold is gone; the tab's reference goes loading. The 40 notes stay readable.

+

A click meanwhile

Ann's colleague clicks Add. The call parks on the reference; nothing is sent yet. The deadline is 60 s.

+

Chord resolves again

Holds on S1 still exist, so Chord calls route(Session).resolve({ id: "S1" }) again, with the per-address backoff. +The app forks a new worker: a new incarnation of Session({ id: "S1" }).

+

Re-hold

The server re-holds every hold on S1 over the new link. The new worker activates NoteStore { S1 } (its notes load +from storage) and the views; it sends fresh snapshots. The tab's reference goes live.

+

The parked call runs

The parked add is sent now and succeeds. The panel shows 41 notes. Nobody wrote recovery code.

+
+

Failure and retry

+

Any broken part fails the whole object: a half-working object could break rules that rely on its documents +changing together. Chord tags the cause by where it failed:

+
type FailureCause =
+  | { kind: "transient"; error: ChordError }   // an open threw, a dependency was unreachable: automatic backoff
+  | { kind: "broken"; error: ChordError };     // factory, derive, hydration or migrate threw: retried only by a trigger
+

Transient causes get a backoff per address: 1 s, doubling to 60 s; concurrent gets share one attempt; success +resets it. Broken causes would fail the same way again (a code bug, invalid data), so they are retried only by a +trigger: a plugin reload, ref.retry() (a Retry button), or a readable the factory declared with +self.retryOn(...readables). Only an opener may declare its own failure broken, with throw broken(error), for +example for invalid configuration. A failed reference stays held; on success it goes loading, then live.

+
host.serve(EnvConnection, ({ env }, self) => {
+  const config = self.get(EnvConfig, {});                 // a facet or registry value, outlives the attempt
+  self.retryOn(config.envs);                              // a config change retries a broken open
+  const ssh = self.open(async ({ signal }) => {
+    const c = config.envs.value[env];
+    if (!c) throw broken(new Error(`unknown environment ${env}`));
+    return connect(c, signal);
+  }, (s) => s.close());
+  …
+});
+

retryOn sources must live independently of the attempt (facet documents, registry values); a member of one of the +object's own references is a type error, since it goes away with the attempt.

+

Waiting for dependencies

+

An object whose references are not live yet stays loading and goes live the moment they are, bounded by +readyTimeout (60 s). The decision for the failure case (R18) is what makes recovery automatic:

+
tabNotes { S1 }NoteStore { S1 }holdself.get (activation)open fails: disk not mountedfailed (transient)failed: transient, cause insidethe view keeps its reference; its state is the retry triggerbackoff retry: open succeedsliveretried at once (backoff cancelled)live
Figure 6. Dependency recovery. The view fails transient with the store's cause, keeps its reference, and is retried the moment the store goes live.
+

A dependency that fails, or does not become live in time, fails the dependent transient with the dependency's +cause inside, never broken: the cause is the dependency's. The dependent keeps its references while failed, and Chord +uses their state as a retry trigger. Factory authors write self.get and know none of this. After an object is live, +a dependency leaving live does not change it: its calls that use the dependency get the dependency's error, or park +while it loads.

+

Cycles are rejected (Q5). If A's activation gets B and B's gets A, neither would ever lose its last reference. +Every object instance has a uuidv7 instance id; on one host Chord follows the waiting edges of activating objects, and +across hosts a hold made during an activation carries the chain of instance ids. A cycle fails the activation +broken, naming it ("A → B → A"), and keeps no references.

+

No optional references

+

get has no optional flag. When a dependency's plugin may be missing, turn the dependency around: the dependent +owns a registry and the other plugin contributes to it. Radius uses forwards from the ports plugin, but ports is +optional, so radius owns radius.forwards and ports contributes its forwards; without ports the table is empty and +radius works (chapter XIII).

+

ctx

+

Every call gets a ctx as its last argument:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldMeaning
signalfires when the caller cancelled or the object ended; pass it to everything you wait on
principalthe caller (remote types only)
incarnationthe calling host's incarnation, for calls that arrived over a link (chapter X)
getreferences released when the call ends
telemetry, trace(name, attributes, fn)the current span; a child span with a child ctx (chapter XVII)
jobId, progressin a job's run (chapter XI)
+
exportText: async ({}, ctx) => {
+  const catalog = await ctx.get(Catalog, {});                     // released when the call ends
+  const title = catalog.sessions.value[session]?.title ?? session;
+  const res = await fetch(renderUrl, { method: "POST", body: render(title, list.value), signal: ctx.signal });
+  return await res.text();
+},
+

There are no code versions and no request ids in ctx. Durable's Context is the same module (chord/context): +ctx extends it, so passing ctx into pi-durable needs no conversion. Code outside objects gets a ctx from +host.root(fn, { principal?, signal? }); inside Durable tools it uses chord(fn, context) (chapter XIV).

+
Built so far

References, ctx and the lifecycle are layers L4 and L5. ChordError and FailureCause are specified in the L1 +draft (layer-1.md), with codes invalid, contract_mismatch, forbidden, gone, not_ready, released, +too_large, unavailable and aborted, and a finer reason such as invalid/args.

+
+
+ +
VII

Taking Access Away

Revocation: how removing a guest ends their objects on every host at once, and why reconnecting does not bring access back.

+

The owner removes Ann from S1. Her tab still has a WebSocket open and a Notes { S1 } view on worker S1. Both must end +now, on the server and on the worker, and her tab's automatic reconnect must not restore anything. Chord provides the +mechanism; the app decides who is blocked and remembers it.

+

Labels, not predicates

+

The app declares, once, which revocation labels each principal carries. Revoking means revoking one label; +everything carrying it is hit.

+
// studio/src/principal.ts
+export type Revocation = { grant: string } | { device: string } | { session: string };
+
+export const revokedBy = (p: P): Revocation[] =>
+  p.kind === "guest" ? [{ grant: p.grant }, { session: p.session }]
+  : p.kind === "owner" && p.access === "device" && p.device ? [{ device: p.device }]
+  : [];
+
+const server = createHost<P, Revocation>({ address: Server(), principal: { kind: "server" }, revokedBy });
+// every worker is created with the same revokedBy
+ + + + + + + + + + + + + + + + + + + + + + + +
PrincipalLabels
guest Ann (grant g1) in S1{ grant: "g1" }, { session: "S1" }
guest Bob (grant g2) in S1{ grant: "g2" }, { session: "S1" }
owner on the paired phone{ device: "phone" }
owner, localnone
+ + + + + + + + + + + + + + + + + + + +
CallHits
server.revoke({ grant: "g1" }): remove AnnAnn
server.revoke({ session: "S1" }): archive S1Ann and Bob
server.revoke({ device: "phone" }): sign out the phonethe phone
+

A label is plain data because revocation is an event that must travel from the server to the workers; a predicate +function could not be sent.

+

What revoke does

+

host.revoke(r) ends every object and closes every link whose principal p has r in revokedBy(p) (compared by +canonical JSON; never the host's own principal), and passes r to its children, which apply their own revokedBy.

+
owner tabserverworker S1Ann's tabguest g1Sharing.remove({ grant: "g1" })app: mark g1 blocked (document)revoke({ grant: "g1" })close linkrevoke { grant: "g1" }end Notes { S1 } as g1reconnectauthenticate: g1 blockedrefused (4401)
Figure 7. Removing guest Ann. The label travels down; her objects end on every host; her reconnect is refused by the app's own record.
+

On worker S1, for revoke({ grant: "g1" }):

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ObjectPrincipalHit
Notes { S1 }owner, localno
Notes { S1 }guest g1yes: ends
Notes { S1 }guest g2no
NoteStore { S1 }none (local)no
+

Ending Ann's view releases its reference to the store, which keeps serving the others.

+

Chord persists nothing here

+

revoke cuts live access only. Lasting denial lives in the app's data (the sharing plugin's guest records), which +authenticate checks on every new connection. After a server restart nothing live exists to revoke; every client +authenticates again and blocked grants are refused there.

+

No re-stamping. A changed grant (Ann moves from read to write) is a revocation of the old grant. Her link closes, +her tab reconnects and authenticates with the new grant revision, which gives a new principal key, and her references +re-hold by themselves, now as an editor. Reconnecting never restores revoked authority.

+

Revocation beats jobs (R11). Normally a running job keeps its object alive after the last reference is released +(a closed tab does not abort an import). Revoking a principal ends its objects at once regardless; their jobs get the +abort signal and the normal close timeout.

+

Work in shared stores

+

Revocation ends objects whose principal carries the label. A shared store has no principal, so work it runs on +behalf of someone (a question waiting for Ann's answer, a shell command a guest started) is not ended by revoke. +Such a store keeps the grant as data with the work and reconciles from the grant owner's live state with a routine +(chapter X):

+
// in a shared store that runs work for guests
+const sharing = self.get(Grants, {});                              // the sharing plugin's live grants
+self.routines.on(sharing.live, async (live, ctx) => {
+  for (const run of shell.running.value)
+    if (run.args.grant && !live[run.args.grant]) shell.abort(run.id);   // idempotent
+});
+

There is no revocation hook: a hook can be missed (a crash between revoke and hook), a routine over current state +cannot. The window between the revocation and the routine's run is accepted (R10); a step started in it still runs.

+

Who does what

+ + + + + + + + + + + + + + + + + + + +
ChordThe app
stamps principals on linkswhat a grant is; share links, cookies, authenticate
keeps one object per principal keydefines Revocation and revokedBy
on revoke(r): ends matching objects and links, passes r downdecides when to revoke; stores who is blocked
+
+ +
VIII

Computed Values

Derives, the one footgun that makes them expensive, mapShared, and presence: who is looking.

+

The reader's notes without authors, a note count, the number of people viewing: values computed from other values. +Storing copies would go stale; recomputing them on every read would be wasteful and could not be watched. A +derive is computed once per change and sent to viewers as a difference.

+

self.derive

+
const count = self.derive("count", [list], ([l]) => l.items.length);
+const summary = self.derive("summary", [list, store.pinned], ([l, pinned], previous) => ({
+  total: l.items.length,
+  pinned: pinned.length,
+}));
+

self.derive(name, sources, (values, previous) => value). Sources can be documents, readables, other derives and +state members of references. The value is recomputed when a source changes and diffed into operations for the wire. +Derived values are adopted before ordinary observers run and share their source commit's wire batch, so a viewer +never sees a note list and a count from different commits.

+

previous is the derive's last value, kept across factory re-runs (reloads) by name. It is a cache, empty after a +host teardown, so a derive must be correct without it. A derive that throws fails its whole owner broken.

+

How the difference is found

+

Chord compares the new value with the previous one using Delta's diffRevisions. Parts that are the same objects +as last time cost one === each. Parts that are new objects must be compared by content.

+

The footgun. A derive that builds new objects for unchanged data makes every commit cost the size of the whole +value, per view. The reader list from chapter IV, written naively:

+
self.derive("public", [list], ([l]) => ({ items: l.items.map((n) => ({ id: n.id, text: n.text, at: n.at })) }));
+// one note added to 2,000: 2,001 new objects, every one compared by content
+

mapShared(f) keeps one result per source element: f runs only for new or changed elements, unchanged ones +return their previous result object.

+
import { mapShared } from "@earendil-works/chord/delta";
+
+const stripAuthor = mapShared((n: Note): PublicNote => ({ id: n.id, text: n.text, at: n.at }));
+self.derive("public", [list], ([l]) => ({ items: l.items.map(stripAuthor) }));
+// one note added: one new object, 2,000 identical references; the diff emits one insert
+
Review: mapShared is planned, not available

Design §12 introduces mapShared as coming "from @earendil-works/chord/delta", but nothing in packages/chord/src +defines it yet, and Delta does not yet treat bytes as atomic leaves in diffRevisions. Design §28 lists both as the +step "Delta in pi", before the layered rebuild. Until then, keep a Map from source element to result yourself, which +is what mapShared will do.

+
+

Cookbook

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CaseHow
select a field, look up by key, a small aggregatecompute it; the value is small
filter or map a listmapShared over the elements
redactionpreferably separate documents per visibility; otherwise mapShared, computed once in the shared store, never per view
join two sourcesmapShared plus the looked-up source's revision
a window over a large listtake a section of the source elements without rebuilding them; cache per-element measurements by identity; keep the window's start stable so changes arrive as appends
a large view combined from several sourcesre-path the sources' operations, as Durable's ConversationView does; application code never interprets ops
+

Container width (R24): structural sharing is O(change) along a path, not across a container's width. Changing one +item of a flat array copies the array's slots, and mapShared visits every element once (one identity comparison +each). Large collections are keyed documents or bounded windows. A benchmark of long arrays is to set the documented +limit.

+

Readables that are not documents

+

Three small readables round out the toolkit:

+
    +
  • constant(value): never changes. Start data uses it (chapter IX).
  • +
  • self.writable(initial): a host-local readable of any in-process value (functions included), compared by identity, +with set. It never goes on the wire; it exists for registry contributions that change (chapter XIII).
  • +
  • Source readables from elsewhere: a Durable view or document watch is already a Chord readable, because Durable and +Chord v2 share the readable type (chord/src/state.ts), so it can feed a derive or be exposed as state directly.
  • +
+
Review: exposing a writable (Q14, open)

A writable may hold functions; nothing yet stops a factory from returning one as a state member, which would reach +the wire encoder. Review round 3 proposes that only members with a JSON schema in the contract are exposable, checked +when the object is served. Undecided.

+
+

Presence: who is looking

+

host.references(Type) is a readable of who holds references to objects of a type on this host: input, principal and +client. Client holds carry an opaque origin id, so two tabs are two clients.

+
host.serve(NoteStore, ({ session }, self) => {
+  const viewers = self.derive("viewers", [host.references(Notes)], ([refs]) =>
+    refs.filter((r) => r.input.session === session).length);
+  return { …, viewers };
+});
+

A link drop is a departure: accurate and simple. Smoothing (a phone switching networks makes the count flicker) is the +UI's job, for example a debounced count. Counts per person are derived by the app from principals and origins. There +is no consumer-side linger: a closed tab closes its link and keeps nothing. The provider's linger keeps a +just-closed object warm, and that also keeps the worker alive through host.activity.

+

"Leaving" is never derived from an unmount. A UI reload unmounts and remounts components; a component unmounted by +a reload has not left. Leaving is explicit navigation, or presence (a dropped reference). This replaces p2's presence, +viewer and login-abandon leases with one mechanism.

+
Built so far

Readables and derives are layer L2 (a readable is today's replicatedState, derives reuse today's derived cells over +it). Not started in the rebuild.

+
+
+ +
IX

Things the OS Gives You

Start data, host-lifetime objects and natives: how a plugin opens a file watcher or a socket so that it is closed exactly once, also across reloads.

+

Studio's notes get an inbox: a text file dropped into the session's inbox/ directory becomes a note. That needs a +file watcher, a native: an OS-level value (a file handle, a socket, a process, a database connection) that cannot +be JSON, cannot be copied, must be opened, and must be closed exactly once. It also needs the session's data +directory, which the plugin does not know. This chapter covers both.

+

Start data is an object

+

A worker knows things before any plugin loads: its session id, the data directory, init data from the server's start +message. There is no special mechanism for them. Start data is an ordinary local object with lifetime: "host" and +constant state, served by base code's core facet:

+
// studio/src/core/contract.ts
+export const Init = obj({ id: str(), dataDir: str(), cwd: str(), env: opt(str()) });
+export const SessionInfo = local("studio.session-info", {
+  version: 1,
+  input: obj({}),
+  members: { init: Init },
+});
+
+// studio/src/core/worker.ts
+export const coreWorkerFacet = (init: Static<typeof Init>) => defineFacet((host) => {
+  host.serve(SessionInfo, () => ({ init: constant(init) }), { lifetime: "host" });
+  // … the Harness and other core objects (chapter XIV)
+});
+

Any plugin on the worker declares a reference and reads it after activation: in an opener, a derive, a routine or +a method, never synchronously in the factory.

+
const info = self.get(SessionInfo, {});
+// later: info.init.value.id, info.init.value.dataDir
+

Paths are derived by the plugin from generic start data (join(dataDir, id, "notes-inbox")), never provided per +plugin. Platform bindings work the same way: on a Durable Object, a core object's self.open returns the binding +(state, env) and its close does nothing.

+

self.open

+
const watcher = self.open(
+  async ({ signal }) => watchDir(path, { signal }),       // open: during activation
+  (w, reason) => w.close(),                               // close: when the object ends or restarts
+  { closeTimeout: 10_000 },                               // optional; host default 30 s
+);
+watcher.value;                                            // the watcher; reading it before activation throws not_ready
+

self.open returns a handle with .value (type Native<T>), not a proxy. The rules:

+
    +
  • Natives live only in objects. Not at module level, not in the facet. A module-level watcher would survive a +reload with nobody to close it.
  • +
  • Order: natives open one after another in declaration order (a later opener may read an earlier handle's +.value) and close in reverse. The object goes live after all opens succeeded.
  • +
  • The value never changes during an attempt. No replacement, no versions, no stale-handle logic. To change a live +native, the object restarts.
  • +
  • A failed open makes the object failed: transient with backoff by default, broken if the opener says so +(throw broken(error)). A partly failed activation closes the natives already open, with reason end; the next +attempt opens all of them anew.
  • +
  • close gets a reason: { kind: "reload" } (the plugin reloads and this native object restarts), { kind: "stop" } (the host shuts down) or { kind: "end" } (the last reference ended, or the type was removed). Each +close is bounded by closeTimeout, then reported as a possible leak while the rest continues. A throwing close +does not stop the others; host.close() collects the errors, background ends report to onError.
  • +
  • Behaviour around a native goes into routines (chapter X). Listeners on natives outside routines are forbidden.
  • +
+

An object with at least one self.open is a native object; one without is a pure object. The difference +decides what a reload does (chapter XV): pure objects are swapped in place, native objects restart.

+

The inbox

+

The inbox must run as long as the worker, whether anyone looks at the notes or not. So it is a lifetime: "host" +object. The facet cannot self.get a type it serves, and the inbox has no meaningful input, so it is an object +without input that reads its session from the start data:

+
// notes/contract.ts
+export const NoteInbox = local("notes.inbox", { version: 1, input: obj({}), members: {} });
+
+// notes/worker.ts
+host.serve(NoteInbox, (_input, self) => {
+  const info = self.get(SessionInfo, {});
+  const watcher = self.open(async ({ signal }) => {
+    const { id, dataDir } = info.init.value;                       // read in the opener: after activation
+    return watchDir(join(dataDir, id, "notes-inbox"), { signal });
+  }, (w) => w.close());
+
+  self.routines.loop("import", async (ctx) => {                    // chapter X
+    const store = await ctx.get(NoteStore, { session: info.init.value.id });
+    for await (const file of watcher.value.files(ctx.signal)) {
+      await store.add({ text: await readFile(file, "utf8"), by: "inbox" }, ctx);
+      await rm(file);
+    }
+  });
+  return {};
+}, { lifetime: "host" });
+

Note ctx.get(NoteStore, …) in the routine rather than self.get in the factory. The factory runs before the +session id is known (start data is read after activation), so a host-lifetime object without input gets +session-scoped objects with ctx.get in a routine or method (R22). A get whose input is computed after activation is +to be added only if a real case needs it.

+

Natives with work in flight

+

A restart is sequential (R16): close the old native, then open the new one. When the old one still has work running, +use one of two patterns:

+
    +
  • Drain in the background. close detaches the old resource and returns at once; it disposes of it when its work +finishes, or after closeTimeout at most. Chord opens the new one immediately. Code mode's sandbox does this: a +script running during a reload finishes on the old sandbox (for up to 30 s, Q7; a longer one is aborted and its turn +fails), new scripts start on the new one.
  • +
  • Swap through a registry. The resource is a registry contribution (chapter XIII). Readers always look up the +current one at use; a reader that must wait for one uses a cancellable "wait until present" with the park deadline.
  • +
+

Chord aborts only the object's own calls and jobs on close. Other work that uses the native (Durable tool calls on a +sandbox) is the owner's business: close may drain instead of tearing down. What an old native does after its owner +restarts is also the owner's close: a model runtime keeps working until collected, a closed SSH connection fails its +in-flight exec.

+

Natives that follow a relocation

+

A session can move to another directory. The inbox watcher must follow. Restarting the object on every move would be +heavy-handed; instead the native is a stable manager opened once, and a routine on the session location switches its +internal resource and bumps a generation, so late events from the old location are dropped:

+
const watcher = self.open(async ({ signal }) => new SwitchableWatcher({ signal }), (w) => w.close());
+self.routines.on(location.cwd, (cwd) => watcher.value.switchTo(join(cwd, ".notes-inbox")));
+

Call-scoped natives

+

A native that lives exactly as long as one call (a temporary SSH session for a directory listing, a file handle for a +download) is opened and closed inside the method with try/finally. There is no ctx.open:

+
list: async ({ path }, ctx) => {
+  const session = await ssh.open(config, ctx.signal);
+  try { return await session.readDir(path, ctx.signal); }
+  finally { session.close(); }
+},
+

Core vs. plugin

+

SessionInfo comes from base code, NoteInbox from a plugin; both use the same primitives. Where code lives decides +which it is. Base code (the app's src/, imported statically by the start code under a fixed name such as +@studio/core) is always there, not switchable, and a change restarts the hosts. A plugin (its own directory, +found by listing, named by its package.json) is switchable and reloads. Core is what the app always needs and +everything builds on: the harness, the session list, access, the session location. Optional features are plugins; +code mode is a plugin.

+
Built so far

Natives and host-lifetime objects are layer L4. constant is decided (R23), not built.

+
+
+ +
X

Standing Behaviour

Routines, reconciling instead of notifying, and how two owners keep a rule between them without a shared transaction.

+

Notes older than 30 days should go every hour. Inbox files should be imported as they arrive. Notes of an archived +session should become read-only. Each of these is behaviour that stands, not a call someone makes. Written as bare +setIntervals and event listeners, it would keep running old code after a reload and touch state from nowhere. Chord +has one declared form for it: the routine.

+

Four kinds

+
self.routines.every("cleanup", HOUR, (ctx) => dropOlderThan(30 * DAY, ctx));   // due time survives reloads
+self.routines.on(settings, async (value, ctx) => applyLimit(value, ctx));        // a readable or event source
+self.routines.loop("import", async (ctx) => {                                     // runs until the object ends
+  while (!ctx.signal.aborted) await importNext(ctx);
+});
+self.routines.custom("ticker", ({ signal, run }) => {                             // anything else
+  const timer = setInterval(() => run(tick), 1000);
+  return () => clearInterval(timer);                                              // cleanup undoes the setup
+});
+

Routines start when the object goes live, stop when it ends, and restart with the new code on every reload of their +plugin. Each run gets its own ctx; errors are caught and reported, never uncaught. Routines are named (on +optionally by its source) so Chord recognizes them across reloads; that is how every keeps its due time.

+

Serialized per routine

+

The new instance of a routine starts only after the old handler in flight has settled; only that routine waits, for at +most the routine handover timeout (10 s), after which it starts anyway and the stuck handler is reported.

+

The reason, as a trace: a supervisor routine reconciles pending worker restarts. The old handler is midway through +restart(W1) when the plugin reloads. Without serialization the new routine starts at once, sees W1 still pending, and +restarts it a second time.

+

Reconcile, don't track

+

A routine reconciles from the current value: it makes the world match what it sees now. It writes no "in +progress" flags, and it starts a job for long work (chapter XI). A handler that only cares about the latest value +compares with the current value itself; there is no overlap option. Subscribing to the object's own documents is +gapless: every commit is seen.

+

No bare timers or callbacks that touch state outside routines, jobs or calls. That is what lets Chord stop all of an +object's behaviour when it ends or reloads.

+

Rules between two owners

+

Every thing has exactly one owner, the smallest object it belongs to. Some rules span two: "notes of an archived +session are read-only" involves the catalog (which knows archiving) and NoteStore (which holds notes). Two tempting +designs are wrong:

+
    +
  • A cross-owner transaction (archive and lock the notes atomically) would make the catalog know every plugin and +couple their storage.
  • +
  • A notification (catalog calls notes.archive(session)) can be lost: the worker is not running, the link drops, +the process crashes between the archive and the call.
  • +
+

The design's rule: the owner of the dependent data enforces the rule by reconciling from the other owner's state +with an idempotent routine.

+
// NoteStore: an archived session's notes are read-only
+const flags = self.state(NoteFlags);                         // { archived: boolean }, NoteStore's own document
+const catalog = self.get(Catalog, {});                       // the server's session list: another owner
+self.routines.on(catalog.sessions, async (sessions, ctx) => {
+  const archived = sessions[session]?.archived ?? false;
+  if (flags.value.archived !== archived) await flags.change((d) => { d.archived = archived; }, ctx);
+});
+// add and remove refuse while flags.value.archived
+

After a crash the routine runs again with the current value. The catalog does not know the notes plugin exists. The +same pattern closes a forward of an archived session in the ports plugin, aborts a job whose grant was revoked +(chapter VII), and keeps any invariant between plugins.

+

Reading without starting a worker

+

The session list in the sidebar shows each session's note count. Getting NoteStore { S1 } for every session would +start every worker. The answer is plain publishing between two objects of one plugin, no Chord mechanism: the worker +side reports a summary to a server-side object by a method call from a routine, and clients read the server object.

+
// notes/contract.ts
+export const NoteCounts = remote("notes.counts", {
+  version: 1, input: obj({}), placement: () => Server(),
+  variants: {
+    owner: { state: { counts: map(int()) } },
+    worker: { methods: { report: method(obj({ session: str(), count: int() })) } },
+  },
+});
+
+// notes/server.ts
+const Counts = defineState("notes.counts", {
+  schema: obj({ bySession: map(obj({ count: int(), incarnation: str() })) }),
+  version: 1, initial: () => ({ bySession: {} }), persist: true,     // survives a server restart
+  checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64,
+});
+
+host.serve(NoteCountStore, (_input, self) => {                        // local, lifetime "host"
+  const counts = self.state(Counts);
+  return {
+    counts: self.derive("counts", [counts], ([c]) => mapValues(c.bySession, (e) => e.count)),
+    report: ({ session, count, incarnation }, ctx) =>
+      counts.change((d) => {
+        const last = d.bySession[session];
+        if (last && last.incarnation > incarnation) return;               // uuidv7: an older worker, ignore
+        d.bySession[session] = { count, incarnation };
+      }, ctx),
+  };
+}, { lifetime: "host" });
+
+host.serve(NoteCounts, (_input, principal, self) => {
+  const store = self.get(NoteCountStore, {});
+  if (principal.kind === "worker") return {
+    variant: "worker",
+    report: ({ session, count }, ctx) => {
+      if (session !== principal.session) throw forbidden();               // a worker reports only itself
+      return store.report({ session, count, incarnation: ctx.incarnation }, ctx);
+    },
+  };
+  return { variant: "owner", counts: store.counts };
+});
+
+// notes/worker.ts, inside NoteStore
+const counts = self.get(NoteCounts, {});                                   // a reference to the server
+self.routines.on(count, (n, ctx) => inVariant(counts, "worker") && counts.report({ session, count: n }, ctx));
+

The server object keeps the latest value per session (in a document if it must survive a server restart) and +ignores reports from an older worker incarnation (incarnations are uuidv7, so they order by start time). A worker that was replaced may still deliver a late report; the +fence is the caller's incarnation, which Chord stamps on every call from the link handshake (R20): ctx.incarnation, +next to ctx.principal, not part of the principal or its key, never sent as data, so it cannot be forged. Session +status, subagent lists, unread marks and artifact lists in p2 all follow this pattern. The store is local and has no +caller incarnation of its own, so the view passes it on as data.

+
Review: ctx.incarnation across a relay (Q13, open)

A worker's call to the server arrives directly, so ctx.incarnation is the worker's. But a call relayed through the +server (a tab's call to a worker) carries the server's incarnation on the worker's side, because the worker's +handshake is with the server. Review round 3 proposes that relays stamp the origin host's incarnation, as they stamp +the principal. Undecided; the pattern above only needs the direct case.

+
+
Built so far

Routines are layer L8. The design treats every as one of four forms over one runner; review round 3 proposed cutting +to on and loop only (C4), not decided.

+
+
+ +
XI

Work That Outlives the Call

Jobs: owned work started by a method that keeps running after its caller leaves, with cancel, dedupe and a recorded outcome.

+

Importing a 50 MB text file into notes takes a minute. The tab that started it may close; the import must go on, be +cancellable, not run twice when the tab retries after a link blip, and its result must be recorded. A method cannot do +this (it dies with its caller), a routine is standing behaviour rather than a request, and a Durable task is for work +that must resume after a crash. Chord's form for it is the job, built in pi-durable's sandwich: intent (a +transaction), effect (anything), outcome (a transaction in the current code).

+

Defining and running a job

+
// notes/worker.ts
+const ImportJob = defineJob("notes.import", {
+  version: 1,
+  args: obj({ path: str() }),
+  progress: obj({ lines: int() }),
+  result: obj({ added: int() }),
+  concurrency: 1,                                         // one import at a time per NoteStore
+});
+
+const ImportResults = defineState("notes.import-result", {
+  schema: obj({ outcome: union([obj({ added: int() }), obj({ error: str() })]) }),
+  version: 1, initial: () => ({ outcome: { added: 0 } }), keyed: true,
+  checkpointWhen: () => true,
+});
+
+host.serve(NoteStore, ({ session }, self) => {
+  const list = self.state(NoteList);
+  const results = self.state(ImportResults);
+  const imports = self.job(ImportJob, {
+    run: async ({ path }, ctx) => {                                       // 2. effect
+      let lines = 0;
+      for await (const line of readLines(path, ctx.signal)) {
+        await addNote({ id: `${ctx.jobId}:${lines}`, text: line, by: "import" }, ctx);   // keyed by job id
+        ctx.progress({ lines: ++lines });
+      }
+      return { added: lines };
+    },
+    finish: async (outcome, tx) => {                                      // 3. outcome, current code
+      (await tx.draft(results, outcome.id)).outcome =
+        outcome.kind === "done" ? outcome.result : { error: outcome.error };
+    },
+  });
+  return {
+    …,
+    imports: imports.running,                                             // live list of runs
+    startImport: ({ path, requestId }, ctx) =>
+      imports.start({ path }, ctx, { key: requestId }),                   // 1. intent (refined below)
+    abortImport: ({ id }) => imports.abort(id),
+  };
+});
+

defineJob(name, { version, args, progress, result, concurrency?, persist?, accepts? }); progress and the outcome +are typed by the schemas.

+
tabNoteStore { S1 }runeffectstartImport({ path, requestId })1. intent: busy check, dedupe by key, commit{ id }start after the commitread lines, add notes (keyed by jobId)progress { lines } (throttled)the tab may close here: the job counts as a referencesettles: { kind: "done", result }3. record outcome, finish(outcome, tx), ack
Figure 8. The job sandwich. The intent commits before anything runs; the effect runs outside any transaction; the outcome commits in the current code.
+

The three steps

+

1. Intent. job.start(args, ctx, { key? }) is shorthand for tx.start(job, args, { key? }) inside a +transaction, which may contain other writes. The busy check (concurrency) and deduplication (the same key returns the +existing run) are atomic and in the same commit; run starts only after the commit. Run ids are uuidv7.

+

2. Effect. run(args, ctx) may do anything: processes, network, other objects. It writes no document itself; it +returns the outcome. External writes carry a dedupe key derived from ctx.jobId (above: note ids), so an interrupted +attempt cannot apply them twice. On a reload run keeps its old code.

+

3. Outcome. When run settles, Chord records the outcome in its own record, then calls the current code's +finish(outcome, tx) and acknowledges the run in the same commit. The outcome is tagged:

+
type Outcome<R> = { id: string } & ({ kind: "done"; result: R } | { kind: "failed"; error: "threw" | "aborted" | "lost" });
+

A throwing finish changes nothing and is retried with a backoff (1 s to 60 s), and after the next reload. finish +is optional and follows Durable's transaction rule.

+

Dedupe ends with the run

+

Deduplication covers unsettled runs only (Q9). Once finish has committed, Chord drops the run's record and key +and releases its waiters' references; it keeps nothing beyond a run's lifetime. Dedupe after completion is the +owner's. Studio records which run a request started, in the same commit as the intent, and finish records the +result by run id (ImportResults above):

+
const Requests = defineState("notes.import-request", {
+  schema: obj({ run: str() }), version: 1, initial: () => ({ run: "" }), keyed: true, persist: true,
+  checkpointWhen: () => true,
+});
+
+startImport: async ({ path, requestId }, ctx) => {
+  const prior = await requests.get(requestId);                  // read outside the transaction (R13)
+  if (prior) return { id: prior.run };                          // started before, maybe finished long ago
+  return self.tx(async (tx) => {
+    const run = await tx.start(imports, { path }, { key: requestId });   // dedupes a concurrent retry
+    (await tx.draft(requests, requestId, { tags: [SessionTag(session)] })).run = run.id;
+    return { id: run.id };
+  }, ctx);
+},
+

Two concurrent retries both miss prior, but the key makes tx.start return the same unsettled run. The client +watches imports for progress and reads the result by run id once it is gone from running.

+

Why Chord does not keep completed runs itself: either 100,000 keyed imports would stay in memory forever, or a +retention limit would let a retried key rerun its effect. The owner knows which results matter.

+

Watching, waiting, aborting

+
    +
  • running is a readable of { id, version, args, progress, phase }. phase is running, finishing (outcome +recorded, finish not yet committed) or blocked (with the reason). concurrency counts only active runs. The +panel shows a progress bar from it.
  • +
  • Progress is in memory, throttled and sent as deltas. For output, use OutputBuffer and Progress from +@earendil-works/chord/output: bounded windows, adaptive throttling, no write amplification.
  • +
  • Waiting: start returns a run with outcome(ctx). The caller's abort ends only its wait, never the run.
  • +
  • Abort is a request: job.abort(id) fires the run's ctx.signal; the run decides what it returns (a result +that already arrived stays a success). A run that ignores the signal keeps its slot until it ends. finish runs in +Chord's own context.
  • +
+

Lifetime, persistence, revocation

+

Jobs keep their owner alive. A running, finishing or blocked job counts as a reference to its object, which ends +only once all its jobs are settled. Starting an import and closing the tab does not abort it. Unsettled runs also +count as activity for the worker's keep-alive.

+

Revocation takes precedence (R11). Revoking a principal ends its objects at once, whatever jobs remain; the jobs +get the abort signal and the normal close timeout. A job runs as its object's principal.

+

Persistence: jobs are in memory by default and gone with the process. persist: true (only for stable +identities) keeps the record: a run live at a crash ends lost at the next activation and finish records it. +Nothing is rerun automatically.

+

Versions: a run started under an older job version keeps its version; finish and running see it through pure +accepts adapters, and stored outcomes are never rewritten. A version bump without an adapter does not refuse the +reload: the old outcome is blocked, kept and reported until a later reload brings the adapter.

+

Native objects: a job in a native object ends aborted when the object restarts on a reload (chapter XV). In a +pure object it keeps running on its old run code.

+

What is not a job

+ + + + + + + + + + + + + + + + + + + + + + + +
WorkUse
request and response that may die with its callera method
input into running work (a pasted code, an approval)a flow (chapter XII)
work that must resume after a crasha Durable task
standing behavioura routine
+
Review: jobs are the largest open simplification (#9, C3)

Decision #9 (deferred until login is ported): a login or a long relocation is a job whose state consumers want to +address and watch (LoginFlow { id }). Options: (a) jobs stay records inside their owner, watched through running; +(b) a job is an object of its own, addressed by its id, living until done. Separately, review round 3 proposed cut C3: +shrink jobs to a runner over documents (no job versions, accepts, blocked, persist) or move them out of the +kernel. Not decided. chord-next-walkthrough.md concept 17 still shows the older outcome shape { ok: true, result }; +the design tags outcomes by kind.

+
+
+ +
XII

Asking a Person

Flows and the Questions service: how running work waits for an answer from someone who may be on any tab, may reconnect, and may answer twice.

+

A provider login asks for a pasted code, then an account choice. A dangerous tool call waits for the owner's +approval. The agent asks everyone present a question. In each case running work waits for a person, who may be on any +tab, may lose the connection halfway, may answer late or twice. Two patterns cover this: a flow kept as its +owner's state, and the Questions service, a shared store for questions.

+

A flow is state, not a call

+

The tempting design is a long-held server call, step(), that blocks until the next step. A link drop fails it, and +the consumer cannot tell whether it missed a step. Chord's answer is that the owner keeps the flow as state, which +re-holds by itself after any drop:

+
// models/contract.ts (sketch from the design)
+start:  method(obj({ provider: str() }), obj({ id: str() }))              // uuidv7
+answer: method(obj({ id: str(), step: int(), value: str() }), Answered)   // ok | "step no longer active"
+cancel: method(obj({ id: str() }))
+
+// LoginFlow { id }: state
+type FlowState =
+  | { kind: "running"; step?: { id: number; prompt: LoginPrompt }; notices: Notice[] }
+  | { kind: "done" | "failed" | "cancelled"; notices: Notice[] };
+

The UI watches the state and renders the current step. A late or second answer to a step that is gone is refused. A +promise-based callback inside the work (pi-ai's prompt during a login) stays inside the owner: its job keeps the +resolver, and nothing crosses an object boundary.

+

Imperative consumers (tests, scripts, a CLI, an agent tool) use a generic client helper over the same state, not a +second protocol:

+
async function* steps(flow: LoginFlowRef, ctx: Ctx) {
+  let last = 0;
+  for (;;) {
+    const s = await until(flow.state, (v) => v.kind !== "running" || (v.step !== undefined && v.step.id > last), ctx);
+    if (s.kind !== "running") return s.kind;
+    last = s.step!.id;
+    yield s.step!;
+  }
+}
+
+for await (const step of steps(await ctx.get(LoginFlow, { id }), ctx))
+  await login.answer({ id, step: step.id, value: answerFor(step) }, ctx);
+

last keeps steps from being lost or repeated across re-holds.

+
Review: where a flow lives (#11, deferred)

Decision #11, deferred until login is ported: (a) a keyed document of its owner, or (b) an object LoginFlow { id } +living until done; tied to #9 (jobs as objects). The LoginFlow { id } address above assumes (b). Also undecided is +how "leaving" ends a login (p2 cancels when the starting owner leaves everywhere); with presence it is a dropped +reference, never an unmount (chapter VIII).

+
+

The Questions service

+

The Questions service (apps/p2/docs/questions.md) is a p2 design, not a Chord feature, built from Chord's +primitives: a shared store with per-principal views. The asker gets ask; viewers get the open questions and +answer.

+
export type Question =
+  | { kind: "text"; message: string; secret?: boolean; help?: string }
+  | { kind: "select"; message: string; options: readonly { id: string; label: string }[] }
+  | { kind: "confirm"; message: string; allow: string; deny: string };
+
+type Policy = { mode: "first" } | { mode: "quorum"; count: number } | { mode: "all"; join?: "live" | "snapshot" };
+
+ask(question, {
+  audience,                // a readable or a list of principals
+  policy,
+  key?,                    // stable per asking flow: a re-ask reattaches to the pending question
+  deadline?,               // then resolve with what is there; `missing` names who did not answer
+}, ctx): Promise<Result>;
+

Its rules:

+
    +
  • The store is the truth. Pending questions and their resolvers live in the store; open is a readable over it, +filtered per viewer. Nothing is a document, so a reload or migration cannot leave a ghost question.
  • +
  • One settlement point. Register the waiter and the abort listener before publishing the question; an answer, an +abort or the deadline atomically takes the entry out before resolving. A late or second answer gets "no longer +active". Answers are validated against the stored question (a select id must be one of the options). Never await an +answer inside a transaction.
  • +
  • Policies over a live audience. first resolves with the first answer; quorum(n) after n answers; all when +every member of the audience has answered. With join: "live" (the default), someone who joins becomes required and +someone who leaves drops out.
  • +
  • Authorization. The asker is stamped by the host (a worker asks only in its own session's scope; secret questions +only from server askers). Each question stores who may answer it; a guest's send access does not imply approval +rights. Secret answers never appear in open, documents or logs.
  • +
  • Link drops and crashes. A rejected ask is not a denial. Askers retry with the same key until their own signal +aborts; a pending ask counts as an active call for keep-alive.
  • +
  • Where it lives. On the host where the asking work runs: session questions on the session's worker, login +questions on the server. UIs reach both through the usual relay.
  • +
+

The store in Chord v2 terms:

+
host.serve(QuestionStore, ({ scope }, self) => {
+  const pending = self.open(async () => new PendingQuestions(), (p) => p.rejectAll("store closed"));
+  const open = self.derive("open", [pending.value.readable], ([all]) => all);   // data only, no resolvers
+  return { open, ask: (args, ctx) => pending.value.ask(args, ctx), answer: (args, ctx) => pending.value.answer(args, ctx) };
+});
+
+host.serve(Questions, ({ scope }, principal, self) => {
+  const store = self.get(QuestionStore, { scope });
+  if (isHost(principal)) return { variant: "asker", ask: (args, ctx) => store.ask({ ...args, asker: principal }, ctx) };
+  if (!mayView(principal, scope)) throw forbidden();
+  return {
+    variant: "viewer",
+    open: self.derive("mine", [store.open], ([all]) => openFor(all, principal)),
+    answer: ({ id, value }, ctx) => store.answer({ id, value, by: principal }, ctx),
+  };
+});
+

The pending map with its resolvers is in-process state with promises, so it is a native of the store (opened and +closed with it). Its close rejects every waiter; askers retry with their key against the new store after a reload.

+
Review: questions.md predates the v2 rules

apps/p2/docs/questions.md writes the store as self.resource(...) with a replace hand-over, which the design +rejected ("resources replaced in place"), and opens Chord call scopes in Durable hooks with host.root, which Q6 +replaced with chord(fn, context) (chapter XIV). Design §20 says the Questions store "must live in an object (it +predates natives only in objects)". The sketch above is this guide's translation, with the derive per viewer; filtering +per viewer in the store, once per variant, may be cheaper for large audiences. The ask signature passes typed +objects as arguments, which is fine (data), but the audience readable form cannot cross a link: across links, +audience must be data (principals) or computed by the store.

+
+

Choosing

+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
CaseUse
a multi-step login in one ownera flow: the owner's job keeps the resolver, state shows the steps
a tool needs the owner's approvalQuestions, first, audience the owner
the agent asks everyone presentQuestions, all, audience from presence
deploy needs 2 of 3 approvalsQuestions, quorum(2)
a notice (auth URL, device code, progress)not a question: the asking work's own progress state
+
+ +
XIII

Extension Points

The facet, registries and the HTTP component: how plugins add behaviour to each other without callbacks, and how the owner stays in charge.

+

The notes plugin wants a download route, GET /s/:session/notes.txt. A logs plugin wants to time every request. +Several plugins add model providers, tools for the agent, commands for the TUI. Someone must combine these +contributions, decide conflicts, and replace one plugin's part as a whole when it reloads. Between plugins exactly two +things pass: object types and registries.

+

The facet, completed

+

The facet has appeared since chapter II as the place that serves types. It is also itself a pure object with the +host's lifetime:

+
// notes/server.ts
+export default defineFacet((host, self) => {
+  const prefs = self.state(NotesPrefs);                                  // plugin-wide documents
+  self.contribute(Http, { routes: { "GET /s/:session/notes.txt": exportNotes } });
+  host.serve(NoteCountStore, …, { lifetime: "host" });
+  host.serve(NoteCounts, …);
+});
+
    +
  • It may hold documents, references, routines and contributions; no natives (those need an object with its own +lifetime).
  • +
  • Its identity is the plugin's name, so its persisted documents survive restarts and reloads.
  • +
  • Served objects go live after the facet and may read and write its documents (chapter V).
  • +
  • host.serve and self.contribute are valid only while the setup runs. The facet never self.gets a type it +serves.
  • +
+

Registries

+

A registry is an extension point owned by someone (a Chord component, the app, a plugin). Each contributor +contributes at most one value per registry; the table is an immutable, ordered list of (contributor, value); +the registry's compose turns it into the value consumers use.

+
// owned by the HTTP component
+export const Http = defineRegistry<HttpContribution, Router>("chord.http", {
+  version: 2,
+  accepts: { 1: (old: HttpContributionV1): HttpContribution => upgrade(old) },
+  compose: (table, { report }) => buildRouter(table, report),   // pure; always delivers a value
+});
+
+self.contribute(Http, { routes: { … }, wraps: [ … ] });          // a contributor
+const router = host.registry(Http).value;                        // a consumer: live
+

compose runs at runtime, whenever the table changes: after a reload's commit, when a contributor goes live or +ends, when a contribution changes. It is not a staging check: contributions can be dynamic (known only once a native +is open), so no check before a reload could be complete. compose always delivers a value with its fallback built in +(e.g. the first by plugin name wins) and reports conflicts through report, which become owner notices and telemetry +events. Readers get a plain value and never handle conflicts. Only if compose throws is the registry in an error +state, and readers get an error when they use it. Work that started earlier keeps the value it read.

+

Rules:

+
    +
  • Who contributes (R21): only facets and lifetime: "host" objects; only their self has contribute (a type +error elsewhere). Every object may read registries.
  • +
  • A contribution is in the table exactly while its contributor is live: a facet's while its plugin is loaded; an +object's from live until it ends, fails or restarts.
  • +
  • Order is sorted by plugin name: deterministic on every host and across reloads, and deliberately meaningless. +Meaningful order comes from the owner's composition (HTTP phases, below). Helpers: keyed(table, entriesOf), +ordered(table, itemsOf, orderOf?).
  • +
  • One authoritative declaration (Q8): only the defining plugin's facet declares the registry to the host +(compose, version, accepts). Other plugins' tokens only identify it by name and types. A reload of the defining +plugin replaces the declaration atomically and the table recomposes. If the defining plugin is not loaded, the +registry has no value (readers get an error like a failed reference) and contributions wait in the table.
  • +
  • Versions (Q8): a contribution carries the registry version its plugin was built against; older ones are adapted +by the defining facet's pure accepts. One without a matching version or adapter is left out and reported naming +plugin, registry and versions; its plugin keeps working. Contributions are in-process values (often functions) and +are not schema-checked.
  • +
  • Use sites guard contributions. A contribution is untrusted code (third-party plugins are a goal). Every use site +catches throws and rejections when calling an entry, reports naming the contributor, and continues with the others.
  • +
  • The owner decides conflicts, order and selection. Registries add behaviour; they never replace an owner's checks.
  • +
+

Natives through registries

+

Registries are the only way a native reaches another plugin. An entry lives exactly as long as its contributor; +consumers look up at each use (or follow the composed value) and keep nothing from an entry beyond a use. The models +plugin's host-lifetime object contributes its runtime; the harness looks it up at every request. After a models reload +the next request gets the new runtime, while a running stream finishes on the old one.

+

A handler that stores const r = entry() in a module variable and uses it for days has a bug: after the contributor +restarts it holds a closed native.

+

Contributions that change

+

Some contributions are known only at runtime: MCP tools appear when a server connects. A contribution is either fixed +for its contributor's life (dynamic parts behind handles read at use) or a readable, which Chord subscribes to; +on every change Chord replaces that entry and runs compose again. For in-process values, self.writable(initial) is +a host-local readable of any value (no JSON, no wire, compared by identity) with set:

+
const ext = self.writable(mcpExtension([]));                   // a Durable Extension, no tools yet
+self.contribute(HarnessExtensions, ext);
+self.routines.on(mcp.tools, (tools) => ext.set(mcpExtension(tools)));
+

There is no remove call. An entry leaves when its contributor ends, or after a reload whose new code no longer +contributes. A live contributor with nothing to offer sets its readable to the registry's empty value (an extension +without tools, empty routes); the table keeps exactly one entry per live contributor.

+

Optional dependencies

+

Registries also replace optional references (chapter VI). Radius forwards connections to ports the ports plugin +manages, but ports is optional. So radius owns radius.forwards and ports contributes its forwards; without ports the +table is empty and radius works.

+

The HTTP component

+

HTTP is an optional Chord component (@earendil-works/chord/next/http) built on a registry. A route gets a plain +request summary (method, path params, query, principal, provenance such as local or relayed, selected headers, parsed +and capped JSON bodies, uploads in chunks) and a ctx whose principal is the request's, so ctx.get(Notes, …) gives +exactly the view that person may see. The route returns a description; the component does the HTTP work once (ranges +with 206 and 416, HEAD, download headers with a safe filename, nosniff, CSP and its nonce, caching, streaming, abort +on disconnect):

+
type HttpResponse =
+  | { kind: "json"; value: JsonValue } | { kind: "page"; title: string; body: string }
+  | { kind: "bytes"; name: string; mime: string; size: number; read(offset: number, length: number): Promise<Uint8Array> }
+  | { kind: "redirect"; to: string } | { kind: "error"; status: number };
+
+// notes/server.ts
+self.contribute(Http, {
+  routes: {
+    "GET /s/:session/notes.txt": async (req, ctx) => {
+      const notes = await ctx.get(Notes, { session: req.params.session });   // the requester's view
+      const text = new TextEncoder().encode(render(notes.notes.value));
+      return { kind: "bytes", name: "notes.txt", mime: "text/plain", size: text.byteLength,
+               read: async (offset, length) => text.subarray(offset, offset + length) };
+    },
+  },
+});
+

Routes compose with keyed: two contributors with one pattern is a reported conflict, and the first by plugin name +wins. Wraps are middleware in phases the owner defines (observe, headers, transform, outermost first), +within a phase in contributor order. No numeric priorities: plugins have no single registration order, so Fastify's +lifecycle phases and encapsulation are the model, not Express.

+
// logs/server.ts
+self.contribute(Http, {
+  wraps: [{
+    name: "access-log", phase: "observe", scope: "all",          // default scope "own": only own routes
+    wrap: async (req, next, ctx) => {
+      const t = Date.now();
+      const res = await next(req);
+      log(req, res, Date.now() - t);
+      return res;
+    },
+  }],
+});
+

route narrows a wrap to one route; a route that does not exist is a reported conflict. Wraps may add a security +boundary on top of an owner's checks, never instead of them. A request uses the router as it was when it started. +Request resolvers (who sent a request: cookie, loopback, relay) are hooks over the summary.

+

Channels and big bytes

+

Channels are WebSocket byte channels (send, onData, close) for terminals, separate from object links and +with their own acknowledged windows, so a large state message never holds up keystrokes.

+

Big transfers are chunked by the owner, not by Chord: bytes are a schema type (bytes()), carried natively +(never base64), and a 50 MB upload is a protocol of plain calls:

+
beginUpload({ name, size }, ctx)               → { upload }
+append({ upload, offset, bytes }, ctx)         // about 1 MiB per chunk; the awaited call is the backpressure
+finishUpload({ upload }, ctx)                  → { id }
+read({ id, offset, length }, ctx)              → bytes
+

After a link drop the append in flight fails like any call; the writer asks the owner for the stored offset and +resumes. Unfinished uploads expire through the owner's routine. Chord has no stream type: it would need its own rules +for drops, cancellation, relays and restarts, which this offset protocol already survives.

+
Built so far

Registries are layer L9. The HTTP component, channels and the chunked-transfer helper are deferred until after the +kernel and a first real proof (the harness, one tool with an approval pause, a code mode reload); until then the +playground serves its pages with plain app code. Their design stays the target.

+
+
+ +
XIV

The Agent

pi-durable inside Chord: the harness as a core object, models, environments and tools through registries, and how a tool reaches Chord objects and people.

+

Each Studio worker runs a pi-durable harness for its session. The harness must run for the worker's whole life, +whether anyone watches or not. Its models come from the models plugin, its execution environments from the envs +plugin, its tools from every plugin, and all of those reload independently. This chapter assumes you know pi-durable +and shows only how it plugs into Chord.

+

Shared foundations

+

Durable and Chord v2 share modules: Durable imports Chord's context, delta, json, output and sqlite, and v2 +imports the same, so Context, Draft and JsonValue are one type each. A v2 readable is today's +replicatedState (chord/src/state.ts), so Durable's viewState() and document watches can be exposed as Chord state +members without conversion. Only plain data crosses between Durable and Chord objects.

+

The harness is a core object

+
// studio/src/core/harness.ts (base code)
+import { Harness as DurableHarness } from "@earendil-works/pi-durable";
+
+export const ChordHostKey = createContextKey<Host<P>>("studio.chord-host");
+export const HarnessModels = defineRegistry<Models, Models>("studio.harness-models", { version: 1, compose: firstModels });
+export const HarnessEnvironments = defineRegistry<EnvFactory, EnvFactory>("studio.harness-envs", { version: 1, compose: firstEnv });
+export const HarnessExtensions = defineRegistry<Extension, RegistrySnapshotSource>("studio.harness-extensions", {
+  version: 1,
+  compose: (table, { report }) => extensionSnapshot(table, report),      // one Durable Extension per plugin
+});
+const HarnessRuntime = defineRegistry<() => DurableHarness, () => DurableHarness>("studio.harness-runtime", {
+  version: 1, compose: onlyEntry,                                        // core-private: one contributor
+});
+
+host.serve(Harness, (_input, self) => {
+  const info = self.get(SessionInfo, {});
+  const harness = self.open(async ({ signal }) => {
+    const { id, dataDir } = info.init.value;
+    const base = withContextValue(ChordHostKey, host, withAbortSignal(signal, BACKGROUND_CONTEXT));
+    return DurableHarness.open(await openNodeSqliteStorage(join(dataDir, id, "durable.sqlite")), {
+      models: () => host.registry(HarnessModels).value,                  // read at each use
+      env: (target) => host.registry(HarnessEnvironments).value(target),
+      settings: () => currentSettings(),
+      registry: durableRegistry(host.registry(HarnessExtensions)),      // a RegistryReader over the composed value
+    }, base);
+  }, (h) => h.close(BACKGROUND_CONTEXT));
+  self.contribute(HarnessRuntime, () => harness.value);                 // for core's conversation objects
+  return { … };
+}, { lifetime: "host" });
+
    +
  • Lifetime "host": it runs for the worker's whole life, also while nobody holds it, because Durable tasks +continue.
  • +
  • Base code: it lives in the core facet, so a plugin reload never restarts it; only a base-code change (a host +restart) does.
  • +
  • Registries, looked up at each use: Durable's models and settings accept functions (pi 7ec1f648e), so each +generation gets the current value. After a models reload the next request uses the new runtime; a running stream +finishes on the old one.
  • +
  • Durable's storage is its own. Chord documents never commit together with Durable transcripts.
  • +
+

The design writes the call as Durable.open(storage, { … }); pi-durable's actual entry point is +Harness.open(storage, options, context), used here as DurableHarness to keep it apart from the core object +Harness.

+

Start order

+

At host start Chord loads every facet and waits until every facet and every lifetime: "host" object other than the +harness is live or failed: the start barrier. Then the registries hold the contributions of every plugin that +loaded, and the harness opens and resumes scheduling (R17). An object that must contribute at start, like the model +runtime, is therefore lifetime: "host". Nothing waits for specific contributions; what is missing shows where it is +used:

+ + + + + + + + + + + + + + + + + + + + + + + +
MissingWhat happens
a toolits call gets an error result the model sees
a modelthe generation reports "model unavailable" and waits, not fails, until a model appears
a task definitionthe task stays blocked (missing_task) until the plugin is installed or the task is aborted
a plugin that failedreported with its error; a facet in backoff contributes later and the registries recompose
+

The same holds when a session's database is copied to an instance with other plugins: missing tools, sections and +hooks are skipped, documents of missing plugins stay untouched, stored selections are never rewritten. The UI must +show blocked tasks with their reason and offer to abort them; a turn waiting on one hangs until then.

+
Review: the harness exception to the barrier (C17, R17)

R17 makes the harness start after the barrier, a special case in the barrier's rule. Review round 3 proposed cut C17: +no harness-specific exception; the harness is an ordinary object the app's start code holds after load resolves. +Not decided. chord-next-walkthrough.md concept 28 still says the harness's activation is the start barrier, which +R17 replaced.

+
+

Tools that call Chord

+

Durable tools, hooks and sections receive Durable's ToolExecutionApi (or HookApi) and a Context, never a Chord +ctx (Q6). Core put the host into the harness's base context under ChordHostKey, and every phase context inherits +it. A core helper opens a Chord call scope from any Durable context:

+
// studio/src/core/chord.ts
+export function chord<T>(context: Context, fn: (ctx: Ctx) => Promise<T>): Promise<T> {
+  const host = context.value(ChordHostKey);
+  if (!host) throw new Error("not inside the Studio harness");
+  return host.root(fn, { signal: context.abortSignal, telemetry: context.telemetry });
+}
+

The scope runs as the host principal; attribution is data; references taken with ctx.get are released when fn +settles. There is no wrapping of tools and no app-specific defineTool. The notes plugin gives the agent a tool:

+
// notes/worker.ts
+const addNote = defineTool({
+  name: "notes_add",                                         // tools are named <plugin>_<name>
+  description: "Add a note to this session's notes panel",
+  parameters: Type.Object({ text: Type.String({ maxLength: 2000 }) }),
+  execute: (args, api, context) => chord(async (ctx) => {
+    const info = await ctx.get(SessionInfo, {});
+    const store = await ctx.get(NoteStore, { session: info.init.value.id });
+    const id = await store.add({ text: args.text, by: `agent:${api.conversationId}` }, ctx);
+    return { output: [{ type: "text", text: `added note ${id}` }] };
+  }, context),
+});
+
+export default defineFacet((host, self) => {
+  self.contribute(HarnessExtensions, defineExtension({ name: "@studio/notes", tools: [addNote] }));
+  …
+});
+

A reload of the notes plugin replaces its extension in the table; Durable pins a registry snapshot per task phase, so +a tool call already running finishes on the old code and the next phase uses the new one.

+

Tools that need a person

+

Clients are not addressable, so a tool cannot call the user's tab. A tool that needs the user goes through an owner's +state: the Questions service (chapter XII). The model-facing ask_user tool and an approval hook both call +Questions.ask; tabs render the open questions and answer them.

+
hook(ToolTask, {
+  beforeTool: async (call, api, context) => {
+    if (!needsApproval(call)) return undefined;
+    const decided = await api.memo<boolean>("approval", context);       // a rerun does not ask again
+    if (decided !== undefined) return decided ? undefined : { block: "Denied" };
+    const allowed = await chord(async (ctx) => {
+      const q = await ctx.get(Questions, { scope: { session } });
+      for (;;) {
+        try {
+          const r = await q.ask(
+            { kind: "confirm", message: `Run ${call.name}?`, allow: "Allow", deny: "Deny" },
+            { audience: approvers, policy: { mode: "first" }, key: api.taskId },
+            ctx,
+          );
+          return r.answer;
+        } catch (error) {
+          if (ctx.signal.aborted) throw error;                          // a link blip is not a denial: retry
+          await backoff(ctx);
+        }
+      }
+    }, context);
+    await api.memo("approval", allowed, context);
+    return allowed ? undefined : { block: "Denied by the user" };
+  },
+});
+

Durable runs beforeTool in the tool task's call phase; after the execute checkpoint a crash retries execution +without hooks, so the stored decision covers that retry. The approval must cover the final arguments: put the +approval hook after hooks that rewrite arguments.

+

Conversations for the UI

+

Chat panels need a conversation's live view, submit, steer, stop. The design leaves p3's conversation API to the p3 +design; one construction that follows the rules: a core ConversationStore { conversation } (local) opens the Durable +watch as its own native, getting the Durable harness from the core-private HarnessRuntime registry contribution (a +native may reach another object only through a registry entry), and exposes a bounded window as a derive. Per-principal +Conversation { session, conversation } views in front of it check who may send (chapter XIX).

+

Code mode

+

Code mode is a plugin. It owns its pi-codemode sandbox as a native; its tool entry looks up the current sandbox +when execute() starts, so a call pinned by a phase before a reload (waiting in an approval hook, say) starts on the +new sandbox. On reload and stop the old sandbox is sealed and drained in close: running scripts finish or hit +their deadline, for at most 30 s (Q7). After a worker stop Durable replays as it does today.

+

What lives where

+ + + + + + + + + + + + + + + + + + + +
DataWhere
transcript-bound: entries, documents that fork and rewind with a conversationDurable
app state: session list, notes, drafts, grants, settingsChord documents
data core enforces on conversations (read-only one-shots, cwd followers)a core per-conversation policy document, written when the conversation is created (createConversation({ init }))
+

Telemetry: pass the host's adapter as HarnessOptions.telemetry; Durable stores trace carriers on submissions and +tasks (chapter XVII).

+
+ +
XV

Reload and Rollout

What happens when you press Reload: per host, per object, and across the server, the workers and every tab.

+

Back to the preface. Mario changed the notes plugin's sort order and pressed Reload. Everything the earlier chapters +asked of plugin authors (synchronous factories, natives only in objects, routines instead of timers, data in +documents) exists so that this moment is safe. This chapter shows what Chord does with it.

+

On one host

+
1. load      new bundles; an import or syntax error rejects
+2. staging   facet setups and every live object's new factory run with a recording self
+             (declarations only, nothing activated); any throw rejects
+3. commit    the host switches to the new code atomically
+4. after     per object: pure objects swap, native objects restart
+

Rollback is all-or-nothing per host. Any load or staging error leaves the whole host on its previous code, so a +host is always completely at one revision and its status is one value. Runtime states (a registry conflict, a failed +open, a migration) are not reasons to roll back; after the commit, errors stay with one object. There is no factory +trial, no migrate dry run, and no consumer contract check in staging: an incompatible consumer fails at bind with +contract_mismatch and a notice naming plugin and type.

+
Reloading @studio/notes on worker S1, object by object

Load and stage

The new bundles load. The facet setup and the factories of Notes (three views), NoteStore { S1 } and NoteInbox +run with a recording self: they declare, nothing activates. Nothing throws, so the host commits to the new code.

+

NoteStore: pure, swapped in place

NoteStore has no self.open. Its new methods and derives take over; NoteList, the drafts, its references and the +running import job stay. Routines restart, serialized: the new cleanup waits for the old handler in flight. The gate +opens.

+

Running calls finish on old code

An add that was running when the swap happened finishes on the old code with its old handles; its transaction still +commits. Only new calls get the new handles. Exception (R15): if the reload raised a document's version, old code that +touches the migrated document throws, the document stays intact, and that call fails once.

+

The import job keeps its old run

The job's run keeps the code it started with. When it settles, the new finish records the outcome.

+

The views: pure, swapped

The three Notes views swap in place. Owner, Ann and Bob keep their references; the reader derive is recomputed with +the new sort and the difference is sent. Nobody sees a reconnect.

+

NoteInbox: native, restarts

NoteInbox has a watcher. Its running work ends aborted, the watcher closes with reason reload, the new factory +runs, the watcher reopens. Its documents would stay. Whether a file dropped during the restart is seen is the +watcher's business: Studio's watchDir scans the directory once when it opens.

+
+

Why natives restart instead of being handed over: a native has no line, no transactions and no schema. If old and +new code shared it, they would interleave on it unchecked. In practice restarts are cheap: the Durable harness is base +code, terminal PTYs live in a separate terminal host process, and model runtimes, sandboxes and a search index rebuild +in milliseconds. Only reloading the plugin that owns SSH connections (running remote commands die) or the relay socket +(relayed connections drop) hurts.

+

Removing a type: its objects end (routines stop, natives close, references end with an error); in-memory documents +go with them; persisted ones stay until retired. Consumers elsewhere see a restarting object like any object that +briefly goes away: loading, parked calls, then live.

+

UI facets reload too. A reload unmounts and remounts components. That is why "leaving" (abort a login, close an +overlay) is never derived from an unmount (chapter VIII).

+

Across hosts

+

A revision is (server incarnation, sequence) and names the complete desired code set: every facet with its +digest, not a batch of changes. Every install reconciles everything missing, and a host reports a revision only when it +runs that complete set. A revision also carries the code-set digest (a hash over the facet bundles' content hashes), +so a server restart with unchanged code triggers no reinstall.

+

The order is server, workers, clients. Each host reports installed, installed_failed (with the objects that +failed) or rejected (with the reason; it stays on its previous code). A server that rejects stops the rollout. A +host that starts or reconnects installs the current revision before anything is routed to it. Installs are serialized +per host and bounded (60 s).

+

Calls keep flowing during a rollout. When a call reaches a provider on another revision, the mismatch response +carries the provider's revision and status:

+ + + + + + + + + + + + + + + + + + + +
ProviderThe call
ahead, or behind and installingupdating: values readable, calls park (with the park deadline)
rejectedfails at once; retried on the next revision
same revisioncontract_mismatch: a real mismatch, final
+

A host checks its own install status first: if its own install was rejected, a call fails at once with the reason +instead of waiting on a provider. Calls parked on a binding that is released end with an error and are not resent. +There are no cross-host transactions and no votes.

+
tabrevision 8serverrevision 8worker S1installing 8install revision 8call Notes.add (new contract)relayupdating (rev 7, installing)updatingpark (deadline 60 s)installed 8provider at 8call Notes.add (resent from park)relayresult
Figure 9. A tab calls into a worker that is still installing. The call parks and runs once the worker reports the revision.
+

Start barrier

+

At host start and first load there is one more rule (chapter XIV): load every facet (the core facet with its start +data among them), wait until every facet and lifetime: "host" object other than the harness is live or failed, then +resume dependent work. Plugin reloads need no barrier.

+

Loading and the dev loop

+
    +
  • Roots: plugins come from built-in, global (<data>/plugins), package (npm or git, installed with +--ignore-scripts) and project roots, found by directory listing; no source parsing. The server selects which load. +Dependencies between plugins resolve by contract name at runtime. Names are prefixed: objects <plugin>.<name>, +tools <plugin>_<name>.
  • +
  • Loader D: one base bundle per surface (host packages and the app runtime) plus one script per facet, with host +packages passed in (no module map). Old generations are reclaimed by the garbage collector; a stable filename per +(facet, digest), and V8's compilation cache off on hosts that reload in place. The loader is an adapter: static +imports (tests, Durable Objects) work the same, with the app passing the plugin name to load.
  • +
  • Dev loop: a source watcher marks plugins dirty and never applies anything. A reload control (the playground's web +shell, a CLI, an agent tool such as p2's reload_plugins) reloads the changed plugins, or restarts all hosts for +base code. Reloads are always explicit.
  • +
+
Review: project plugins (#14, deferred)

Per-directory plugins with trust prompts and snapshots (apps/p2/docs/plugin-loading.md, still the reference for +roots, trust and scopes) need their own pass over this design. Until then, treat the project root as designed but not +settled.

+
+
Built so far

Reload and rollout are layer L10, the last before telemetry. The frozen kernel has a reload path with host values and +a development freeze, both removed from the design.

+
+
+ +
XVI

Changing Contracts

How a contract gets a new version while old tabs, old workers and old plugins are still talking to it.

+

Chapter XV ended with a tab that calls into a worker on another revision. That case is short-lived: the rollout +finishes and both sides run the same code. Other cases last longer. A phone kept a tab open overnight with yesterday's +web bundle. A plugin from another author was built against Notes version 1 and is not updated yet. In both, a +consumer holds a contract that the provider no longer has. This chapter is about what the provider does then. It is +also the one part of Chord v2 that is built: everything in the sections on evolve, checkAccepts and fingerprints +runs today in packages/chord/src/next/contracts/.

+

What has a version

+

Everything that is stored or crosses a boundary has a version. Each surface handles an old version its own way:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
SurfaceVersionWhen an old one shows upChapter
object type (contract)version plus fingerprintevolve and checkAccepts on the provider; otherwise contract_mismatchthis one
documentversion plus schema fingerprintpure migrate on access; a schema change without a bump fails the object brokenV
jobversionaccepts on outcomes and running; blocked without an adapterXI
registryversionaccepts adapts contributions; one without a match is left out and reportedXIII
nativenonethe object restarts on reloadIX
Durable tasks and documentsDurable's versionDurable's migrateXIV
link protocolprotocol versionrefused; hosts update together, a web page reloadsXVII
UI components, Chord and app APIsnone yetexposed as they are; an SDK will version them later
+

The rule behind every row is the same as for documents: every change bumps the version, additive ones included. +Chord never guesses compatibility from the shape of two schemas.

+

A history, not a file of versions

+

A contract history starts with a base: a full remote() or local() declaration with an explicit version. The +Notes contract from chapters II and III is a base at version 1. Studio now wants pinned notes: notes get a pinned +flag and editors get a pin method. Instead of editing the declaration, the plugin continues the history:

+
// notes/contract.ts
+import { bool, evolve, nul } from "@earendil-works/chord/next";
+
+export const NoteV2 = obj({ id: str(), text: str({ max: 2000 }), by: str(), at: int(), pinned: bool() });
+export const PublicNoteV2 = obj({ id: str(), text: str(), at: int(), pinned: bool() });
+
+export const NotesV2 = evolve(Notes, (e) =>
+  e.state("editor", "notes", obj({ items: arr(NoteV2) }))
+    .state("reader", "notes", obj({ items: arr(PublicNoteV2) }))
+    .addMethod("editor", "pin", obj({ id: str(), pinned: bool() }), nul()),
+);
+

evolve(prev, (e) => …) appends one version to prev's history: NotesV2.version is 2, and NotesV2 still knows +version 1 exactly as it was declared. Every version is materialized and fingerprinted at declaration. That is what +makes old code safe: an old plugin bundle that contains its own copy of Notes version 1 computes the same +fingerprint as the history inside NotesV2. There is no build step, no generated file and no fingerprint copied by +hand.

+

The edits:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
EditChangesKind
e.state(variant, member, schema)one state member of one variantremote
e.addState(variant, member, schema) / e.removeState(variant, member)adds or removes a state memberremote
e.state(member, schema), e.addState(member, schema), e.removeState(member)the same, flatlocal
e.args(name, schema) / e.result(name, schema)one side of a methodboth
e.method(name, args, result)both sides of a methodboth
e.addMethod(variant, name, args, result)adds a method (or an existing one to another variant, same signature)remote
e.addMethod(name, args, result)adds a methodlocal
e.removeMethod(name)removes a method from every variantboth
+

Edits chain. The method(...) wrapper appears only in base declarations; edits take the schemas directly, so a +method without a result is added with nul(). The rules of chapter III still hold across versions: a method has one +signature in every variant, and a member's kind never changes, not even after it was removed and added again.

+

What evolve cannot do: change the input or the set of variants. Both change what an object is (its identity, +who gets which members), and no adapter can turn one into the other. That needs a new history base, below.

+

The provider says what it accepts

+

NotesV2 says what changed. It does not say how version 1 consumers are served; the provider decides that, with +mappings that live in its own code, never in the contract. The worker facet declares them with checkAccepts:

+
// notes/worker.ts
+import { checkAccepts } from "@earendil-works/chord/next";
+
+const toV1 = ({ pinned, ...note }: Static<typeof NoteV2>) => note;
+const toPublicV1 = ({ pinned, ...note }: Static<typeof PublicNoteV2>) => note;
+
+const notesAccepts = checkAccepts(NotesV2, {
+  1: {
+    state: {
+      editor: { notes: (current) => ({ items: current.items.map(toV1) }) },
+      reader: { notes: (current) => ({ items: current.items.map(toPublicV1) }) },
+    },
+    methods: {},                                   // add and remove did not change; pin is new
+  },
+});
+// notesAccepts.versions: [{ version: 1, fingerprint: Notes.fingerprint }]
+
+host.serve(NotesV2, notesFactory, { accepts: notesAccepts });    // working name for the objects-layer option
+

checkAccepts is pure: it checks the mappings without calling them and returns only the version metadata of the +old versions it lists. A provider that lists nothing accepts only its current version. The objects layer later +compares a consumer's version and fingerprint against this list.

+

The mappings point in the direction the data flows:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Member in the old versionMappingDirection
state, changed(currentMember) => oldMemberprovider's current value to what the old consumer expects
state, removed(currentVariantState) => oldMember, or "refuse"receives the whole current state of that variant
method arguments, changedargs: (old) => currentold caller's arguments to current ones
method result, changedresult: (current) => oldcurrent result back to the old shape
method, changed or removed{ call: async (oldArgs, ctx) => oldResult }, or "refuse"replaces the method for old callers
member added sincenothingan old consumer never asks for it
+

A changed state member cannot be refused: the old consumer still has that member and must get a value. A removed one +can be computed from what the variant has now, or refused. e.method(name, args, result) replaces both sides and +therefore needs both mappings, even if one side stayed the same (then an identity mapping).

+

Required, at type level and again at runtime. For every member that exists in the old version and changed or was +removed since, the mapping is required by the type of checkAccepts. Because plugins are bundled without type +checks, the same check runs at declaration: a missing mapping, a key for a member that needs none, an unknown version +or a wrong shape throws, naming the member's path (for example accepts[1].state.reader.notes). An absent state or +methods section counts as empty and may be left out only when nothing there needs a mapping.

+

Conservative with no-op edits. An edit that sets a schema to what it already was, or changes a member and then +changes it back, still requires an identity mapping. TypeScript erases constraints such as { max: 2000 } from the +static type, so two schemas that look the same to the compiler may differ; Chord does not try to tell.

+

Old versions map straight to the current one

+

Studio later replaces remove with archive, which keeps the note in a trash list:

+
export const NotesV3 = evolve(NotesV2, (e) =>
+  e.removeMethod("remove").addMethod("editor", "archive", obj({ id: str() }), nul()),
+);
+
+const notesAccepts = checkAccepts(NotesV3, {
+  1: {
+    state: { editor: { notes: (c) => ({ items: c.items.map(toV1) }) }, reader: { notes: (c) => ({ items: c.items.map(toPublicV1) }) } },
+    methods: { remove: { call: async ({ id }, ctx) => { await archiveNote(id, ctx); return null; } } },
+  },
+  2: { methods: { remove: { call: async ({ id }, ctx) => { await archiveNote(id, ctx); return null; } } } },
+});
+

Each entry maps one old version directly to the current one, not through the versions in between. Version 1 +needs the state mappings (its notes have no pinned) and the remove replacement; version 2 needs only the +remove replacement, and leaves state out because its state is unchanged. A member that an old version never had +needs nothing for that version, even if it was added, changed and removed later. Dropping support for version 1 is +deleting its entry: version 1 consumers then get contract_mismatch.

+

Binding an old consumer

+

A hold carries only the consumer's contract version and fingerprint, never a description of the contract (R6). +The provider decides with its own schemas:

+
THE HOLD ARRIVESOUTCOMEholdversion 1, fingerprint F1equal to current?version and fingerprintaccepted old version?F1 from the provider's own historybindcurrent membersbind adaptedthrough the mappingscontract_mismatchor updating, during a rolloutyesnoyesnoA mismatch fails only that consumer's binding. The provider object and its other bindings are not affected.
Figure 10. What a provider does with a hold. The decision uses only the provider's own history; a peer's schema is never an input.
+

Once a version 1 tab is bound, it is served as if version 1 were current:

+
A version 1 tab on a version 3 worker

Bind

The tab holds Notes { S1 } with version 1 and fingerprint F1. The worker's history computes F1 for version 1, and +notesAccepts lists version 1, so the hold binds adapted. The tab gets the editor variant as version 1 defined it: +notes, add, remove. It never sees pin or archive.

+

State through the mapping

Chord serves the adapted state like a derive of the provider's state: it applies the mapping to the current value +and keeps the result per binding. The snapshot the tab receives has no pinned fields.

+

A commit

The owner pins a note from a current tab. The worker sends the current tab the draft's ops. For the version 1 tab, +Chord applies the mapping to the new value and sends the difference to its last adapted value (diffRevisions). The +mapping produced nothing new for a pin, so the version 1 tab receives no change at all. A large list is never resent +whole on every commit.

+

A call

The tab calls remove({ id }). Its arguments are checked against the provider's own version 1 schema, then the +call replacement runs archiveNote on version 3 code. Its null result is what version 1 expects. Had the method +only changed, args would have mapped the arguments forward and result the result back.

+

The worker moves on

A later revision drops the version 1 entry. When the tab re-holds (or the worker restarts and the tab binds again), +the hold gets contract_mismatch, and Studio's panel shows "reload to update" instead of failing silently.

+
+

A new history base

+

Some changes cannot be adapted. Studio decides that notes belong to a project, not a session: the input changes from +{ session } to { project }. That is a new base, written in full, with an explicit version that continues the +published numbering:

+
export const NotesV4 = remote("notes", {
+  version: 4,                                      // explicit: the published history used 1 to 3
+  input: obj({ project: str() }),
+  placement: () => Server,
+  variants: {
+    editor: { state: { notes: obj({ items: arr(NoteV2) }) }, methods: { add: method(…), pin: method(…), archive: method(…) } },
+    reader: { state: { notes: obj({ items: arr(PublicNoteV2) }) } },
+  },
+});
+// checkAccepts(NotesV4).versions is empty
+

A new base has no history before its version and cannot accept older versions, not even for an additive change. +Every consumer built against versions 1 to 3 gets contract_mismatch and must update. Later evolve calls continue +from version 4. The same applies to a change of the variant set. Do not spread an older declaration into a new one or +copy its fingerprint; the contract objects are branded and evolve and checkAccepts reject copies.

+

Local contracts

+

local contracts (chapter IV) evolve the same way, with a flat members map and no variants:

+
export const SessionInfoV2 = evolve(SessionInfo, (e) => e.state("init", InitV2));
+checkAccepts(SessionInfoV2, { 1: { state: { init: (current) => toInitV1(current) }, methods: {} } });
+

Same-host consumers of a local type are usually reloaded together with their provider, but not always: another +plugin on the same host may be built against an older version. The rules are identical; only the variant argument is +missing.

+

Implementing a contract

+

Implementation<typeof NotesV2> is the type of everything a provider must supply for the current version: one state +getter per member and variant, and one asynchronous method per method, each receiving (args, ctx):

+
const impl: Implementation<typeof NotesV2> = {
+  state: { editor: { notes: () => list.value }, reader: { notes: () => publicList.value } },
+  methods: { add: async ({ text }, ctx) => addNote(text, ctx), remove: async ({ id }) => removeNote(id), pin: async (a) => pin(a) },
+};
+

Keep declaration types inferred (no annotation on NotesV2): the inferred type carries the complete history, and +checkAccepts needs it to know what changed.

+
Review: L1's implementation shape and the factory's return value

Implementation<C> in src/next/contracts/types.ts is { state: { variant: { member: getter } }, methods }. The +design's factories (design.md §6, used throughout this guide) return { variant, member: value, method: fn }. +The objects layer (L4) has to connect the two, and also define how serve receives the checked accepts (this guide +uses the working name { accepts } as a third argument) and what "refuse" does at runtime. The natural reading, +used in the stepper above, is that a refused method call and a hold that needs a refused state member end with +contract_mismatch; no source says so.

+
+

The fingerprint

+

The fingerprint is a 64-bit hash (hash64, portable across runtimes) of a normal form of the contract's +schemas. The normal form decides what counts as a change:

+ + + + + + + + + + + + + + + + + + + +
IgnoredSignificant
order of object keys, required and enum (sorted)order of union and intersection branches (anyOf, oneOf, allOf) and of tuples
annotations on schema nodes: title, description, examples, $comment, deprecateddefaults, formats, patterns, bounds, $ref names
placementinput, variants, member kinds and every member schema
+

Union order is significant because it is meaning: TypeBox picks defaults and decodes by the first matching branch. +Annotations are dropped only on schema nodes; an object property that happens to be named title is data and counts.

+

Only data schemas are allowed. One walker copies, validates and normalizes each schema against a whitelist of +TypeBox keywords: primitives, arrays and tuples, objects and records, unions and intersections, $defs and $ref +(use $ref for recursive types), constraints and annotations. Conditional and dependent schemas, not, +definitions, contentSchema, unevaluated*, contains, propertyNames, unknown keywords, executable or custom +TypeBox features, arbitrary Unsafe and refinements, accessors, symbols and cyclic objects are rejected at +declaration. Bytes are the one special leaf: bytes({ min, max }) checks a Uint8Array and has the data marker +{ type: "bytes", minLength?, maxLength? }, so it hashes like any schema; TypeBox's checking still works on it.

+

Schemas are deep-copied into a private, frozen history; your own schema objects are copied, not frozen. +NORMAL_FORM_VERSION is part of every digest, so a change of the normal form is a link protocol change.

+

Not a security boundary. A fingerprint says "we were compiled against the same contract"; it proves nothing. A +hostile client can send any fingerprint it likes. Providers therefore always validate arguments with their own +schemas (chapter IV), whatever the fingerprint says.

+

Rules of thumb

+
    +
  • Treat a published contract like a published API: every change is evolve, every evolve needs a decision about +old consumers.
  • +
  • Keep mappings small and pure. A state mapping runs for every commit and every adapted binding; make it a map over +the value, not a query.
  • +
  • Accept the versions your users still run (one or two back during a normal rollout), and delete entries when the +metrics show no old bindings.
  • +
  • Change the input or the variants only with a new base, and only when updating every consumer is acceptable.
  • +
  • Contract, document and job versions are separate and each is decided on its own. Pinned notes needed NotesV2 +for the wire and NoteList version 2 with a migrate that adds pinned: false (chapter V), but no new +ImportJob version, because its outcome did not change.
  • +
+
Built so far

This chapter's contract layer is built: packages/chord/src/next/contracts/ (commit 7d2bb8ea8, about 1,300 lines +with remote, local, method, evolve, checkAccepts, Implementation, the schema helpers, fingerprintOf, +schemaNormalForm and NORMAL_FORM_VERSION), with nine test files covering evolution, fingerprints, the hash, +local contracts, long chains, schemas, types and review findings. It stands alone: it hosts nothing, calls no +mapping and does not import the frozen runtime. Binding by version and fingerprint, adapted state and calls belong to +the objects and protocol layers, not built yet.

+
+
+ +
XVII

On the Wire

What travels over a link, what a relay does with it, how big things are kept in check, what each hop costs, and how to see it in traces.

+

Most plugin code never thinks about the wire: it declares state, commits transactions and calls methods, and Chord +moves the bytes. You need this chapter when something is slow, when a value is large, when a link misbehaves, or when +you want to read a trace. It also explains why the earlier chapters insisted on small operations, shared structure and +no callbacks: each of those rules removes a cost on some hop.

+

What a state member sends

+

Every exposed document, derive or readable is a state member. Over a hold it produces one of these:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
UpdateWhen
snapon hold and on re-hold: the whole value, once
opsper commit: the draft's recorded ops, no diff
ops from a diffper change of a derive (and of an adapted binding, chapter XVI)
throttled opsjob progress
reseta migration commit replaced the value (adapted for older consumers)
errorthe member failed
+

Commits are atomic for consumers. On the same host, subscribers are notified once, after every document of the +commit is adopted. Remotely, one state message per hold carries a batch of member updates that is applied before +listeners run, and a commit that touches several holds on one link travels as one state message on that link. +A consumer never sees half a commit.

+

Messages

+

A message is a header plus opaque parts (R1). There is one part per hold the message concerns; the header says +which hold each part belongs to and how large it is.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
DirectionKindParts
consumer to providerhold, release, call, cancelcall has one (the arguments)
provider to consumerboundone: the snapshot
stateone per hold the commit touched on this link
resultone
gonenone
+

There are no request ids in application code. Hold ids and call ids exist only inside the protocol, per link.

+

One binary format on every link (Q4): u32 header length | header JSON | part bytes, as one Uint8Array. A +WebSocket sends it as a binary frame, IPC passes it with serialization: "advanced", a memory link delivers it +asynchronously. Encoding, framing, header checks, id resolution, flow control and size limits are one shared protocol +layer; link adapters only move bytes. Tests on memory links therefore run the production path.

+

A part is encoded once, at its origin, with that hold's own Delta encoder (paths are interned per hold, end to +end between provider and consumer). A JSON part above a threshold (about 16 KiB) is compressed once at its origin, +whatever the link; byte segments stay raw; relays forward compressed parts as they are; the header names the codec.

+

The header is a TypeBox schema (R4), a union tagged by kind; the protocol types derive from it. Headers from +untrusted links (client WebSockets) are checked with the schema compiled once; trusted links (IPC between server and +workers) are not checked. Unknown ids are ignored on both sides (Q1). They are normal races: a result crossing a +cancel, a call crossing a gone, a state part crossing a release. The side that ended a hold or call settles +its own pending work: a consumer that receives gone(h) rejects its pending calls on h, and the order per +direction guarantees nothing else for h follows. Only a header that fails the schema is a protocol error, and it +closes that link.

+

The link protocol has a version. Hosts with different protocol versions refuse each other; a web page reloads to get +the server's protocol.

+

Through a relay

+

A tab never talks to a worker directly (chapter I). The server relays, and a relay reads and rewrites only +headers: it maps hold and call ids between its two links, stamps the principal on hold and call (a client's +principal field is always overwritten; only a trusted parent link may carry one), and groups parts per destination +link, so each link gets only its own holds' parts. It never parses a payload.

+
owner's tabhold h1Ann's tabhold h2serverrelayworker S1NoteStore, Notescall c7 Notes.add (1 part)map ids, stamp ownercall c3 (part forwarded as is)method runs, self.tx commitsstate: parts for h1', h2'group parts per linkstate (h1 part)state (h2 part)result c3result c7
Figure 11. The owner adds a note while Ann watches. The commit leaves the worker as one state message with two parts; the server splits it by destination without decoding either part.
+

The state message reaches the owner's tab before the result, so when await notes.add(…) resolves, the new note +is already in notes.value. Ann's view is a different variant (reader) and gets its own part from its own derive; the +relay neither knows nor cares.

+

Flow control

+

Each link has one byte window, counted from each message's byteLength (parts are already encoded bytes; nothing +is measured or stringified to count it). The window covers every message, first snapshots included.

+
    +
  • Snapshots are always admitted (R19), even when larger than the window; they use it up. Whether a hold +overflows is decided on the changes queued after the snapshot.
  • +
  • Under backpressure a commit's ops are collected per hold and encoded into one part when credits return. The +consumer still gets the commit atomically.
  • +
  • After an overflow the hold is dropped and established again once the previous snapshot's debt has drained +(credits back at the full window, Q3), and not more often than a minimum interval. So there is at most one snapshot +in flight per hold: no re-hold storm, and no stall for a snapshot larger than the window.
  • +
  • Calls are throttled by call credits the provider grants (a generous window). Beyond them calls wait locally. +Nothing is refused for being "too many", and holds are not limited by count; authorization and input validation +protect them.
  • +
  • Byte-heavy traffic such as terminals uses separate channels (chapter XIII), so a large state message never +holds up keystrokes.
  • +
+

Recovery: after a link drops, the consumer re-holds every hold over the new link and receives snapshots; calls in +flight fail as on any disconnect, calls made meanwhile park. A relay whose window overflows releases the hold and +re-holds; the provider sends a fresh snapshot. After a routed link drops, Chord re-resolves the address (with the +per-address backoff) while holds on it exist, so a passive viewer recovers without anyone pressing anything.

+

Size limits

+

Limits apply per part (R2, Q2), both to the compressed and the decoded size, and are checked from the sizes in the +header before anything is decoded; zlib's maxOutputLength enforces the decoded limit in the one decode function.

+ + + + + + + + + + + + + + + + + + + +
SituationWhat fails
a part over the limitonly its own hold or call, with too_large; the other parts of the message apply
a snapshot over the limitits hold ends failed, with no automatic re-hold
a WebSocket frame over maxPayload (e.g. 64 MiB)the connection closes (code 1009); this is the transport's hard cap
+

Defaults for client WebSockets are 16 MiB compressed and 64 MiB decoded per part, configurable per link kind. The +decoded limit travels with relayed work: the server stamps the admitting link's limit on the header at admission +(never taken from the client), and the host that decodes enforces it. IPC therefore needs no limit of its own.

+

Large data is never a large state member or a large argument. Files go through chunked transfers or the HTTP +component (chapter XIII); long lists are keyed documents or bounded windows (chapter VIII).

+

What each hop costs

+

Chord v2's goal is as little compute and memory as possible while staying correct: O(size of the change) per +commit and per viewer, independent of the size of the value. A 500 KB value is touched as a whole only when a new +viewer takes its first snapshot. Delta, the value library under documents, was built for this, and its own primitives +are almost free: diffRevisions on shared subtrees takes 0.018 ms, applyImmutable of a small change 0.004 ms.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
HopWork
commitDelta's prepare (copies and checks only the values placed in this commit), adopt by pointer swap; storage writes the delta
reads in a transactionthe draft overlay, no copy; an unchanged draft writes nothing
subscribers on the same hostthe same value and ops, by reference
derivescompute plus diffRevisions; unchanged parts stay the same objects (mapShared, chapter VIII)
sendone encoding per hold part at its origin, size taken from the encoded bytes
relayheader only: ids mapped, principal stamped, parts grouped; parts never parsed
receivecompiled header schema on untrusted links; arguments stripped and validated once at the provider (compiled schema); inputs normalized
applyapplyImmutable, structure shared
new holdthe value encoded once (unavoidable)
memoryone authoritative value per document per host; replicas and derived values share structure; one cached value per adapted binding
+

These are targets. The frozen kernel measured far worse, which is why the design is so strict here:

+
35 msone small change to a 500 KB value in the frozen kernel, no viewer
+13 msper viewer on top (10 viewers: 130 ms)
17 / 33 / 101 msa tiny SQLite commit after 0 / 10 / 50 earlier deltas
+

Nearly all of that came from layers around Delta: whole-value validation on every access, whole-value freezing, +JSON-ness walks on every received frame, double stringification to measure a frame, snapshots cloned per viewer, and +a SQLite storage that replayed history, and schemas rebuilt for every check (a compiled TypeBox check takes +0.012 ms, an interpreted one 5.6 ms). The redesign removes each of them (chapters V, VIII and this one); the +before/after per hop is in apps/p2/docs/p3-reviews/v2-cost-model.md.

+
Review: v2-cost-model.md is partly outdated

The measurements and the per-hop "after" targets in apps/p2/docs/p3-reviews/v2-cost-model.md are current. Its +closing section "Decisions this needs" is not: items 6 and 7 (validation by the caller's trust; contract descriptions +once per link and fingerprint) were replaced by R7 (validate for every caller) and R6 (no descriptions on the wire), +and §10 still describes the description fetch.

+
+

Cost guards

+

Chord reports full-value walks, deep copies, encodes, schema validations and materializations as ordinary +telemetry metrics, always on; with the NOOP adapter they cost an empty call where such work happens. Tests use the +in-memory adapter and assert them per hop. The reference case: one appended note to a 500 KB document with 10 viewers +does no walks, no copies, one encode per hold part at its origin and none at relays, and no validations. A +vitest bench suite measures the hops.

+

Ways to make it slow anyway

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MistakeWhy it costsInstead
a derive that builds new objects for unchanged dataevery commit diffs and sends O(value)keep unchanged items identical: mapShared, previous (chapter VIII)
one long array that changes often (R24)sharing is per path; each change copies one slot per element and visits eachkeyed documents or a bounded window
bytes as base64 stringsinflated, parsed, compressed as textbytes() leaves; chunked transfers for files
awaiting I/O inside self.txthe object's line is blocked for every callerdo the I/O first, then a short transaction
storing registry entries or references in documentsstale after a reload, not datalook up the current value at use
opening natives at module level or in a facetno owner: never closed, never restartedself.open in an object (chapter IX)
callbacks as argumentscannot cross a link, keep old code aliveregistries, jobs, flows
one big object that everybody viewsone large snapshot per viewer and per reconnectsplit by viewer need; views of the same variant share one value
+

Telemetry

+

Chord reports traces, events and metrics through @earendil-works/pi-telemetry. Each host gets one adapter, +createHost({ telemetry }), NOOP by default; the adapter decides sampling, buffering, export and filtering. Only hosts +hold the adapter, to start root spans. Code sees spans:

+
    +
  • ctx.telemetry is the current span (Ctx extends Context, so it is the same field Durable and pi-ai read). Every +span carries its adapter's metrics and propagation, so code with a ctx can record a metric or inject a carrier.
  • +
  • ctx.trace(name, attributes, fn) runs fn in a child span with a child ctx.
  • +
  • App values such as the principal stay out of Durable's context.
  • +
+

Across links, call and hold carry the caller's trace carrier as a bounded header field (R5: limited key count +and string lengths); the provider runs the method under it, and each relay adds a chord.relay span. The call from the +sequence above, as a trace:

+
chord.call Notes.add            [owner's tab]
+└─ chord.relay                  [server]
+   └─ chord.method Notes.add    [session:S1] principal=owner
+      · event chord.commit docs=2 ops=5
+

Adapters trust typed input: no validation of malformed values and no try/catch around recording. Only carriers from +another process are checked (keys present, strings), and a bad one falls back to a new root.

+

Shared work (one activation serving several gets, a worker start) is its own root span, linked to the trigger +that started it; each waiting span links to it. Routine runs, job runs, eviction, backoff retries and background +closes are root spans too, linked to their trigger where there is one.

+ + + + + + + + + + + + + + + + + + + +
TierWhat
spanscalls, methods, relays, holds, activations, opens, closes, loads, revocations, resolves, routine runs, job runs
eventscommits inside methods, retries, parking
metricsstate messages, WebSocket bytes, commit durations, in-flight calls, active holds, open links, park durations, backpressure waits, the cost guards
+

Ids are attributes of spans, never of metrics. Nothing is measured or walked for telemetry; only sizes the code +already has are reported. Errors carry a fixed category and a message cut to 500 characters; cancelling is not an +error. Chord ships a typed telemetry schema, as pi-ai (AI_TELEMETRY_SCHEMA) and Durable +(DURABLE_TELEMETRY_SCHEMA) do.

+

Storing and reading: @earendil-works/chord/telemetry-sqlite over chord/sqlite (Node and Cloudflare) stores one +row per settled span (events and links inline) and metrics aggregated per series per 10 s, flushes in batches every +second, keeps 7 days, and drops (and counts) beyond a bounded queue. The read interface TelemetryStore is shared: +findTraces (by name, attributes, errors, time window, with limit and cursor) returns complete traces, getTrace +one, findMetrics metrics. One renderer, pi-telemetry/render, prints traces as above; it prints links and never +follows them. The playground's trace viewer reads the same store.

+

Durable already emits submit and phase spans, stores carriers on submissions and tasks (TaskOptions.trace), +and pi-ai emits pi.ai.request and pi.ai.poll. With the harness given the host's adapter (chapter XIV), an agent +tool that calls Notes.add through chord(fn, context) shows up as one tree from the model request down to the +commit.

+
Built so far

Nothing of the protocol is built in v2: messages, relays, flow control and limits are layers L6 and L7, telemetry in +Chord is L11. Built in pi: pi-telemetry (propagation, links, metrics, the store interface, the renderer), +chord/context with Context.telemetry, pi-ai and Durable telemetry, and chord/telemetry-sqlite. Delta is built; +mapShared and bytes as atomic leaves in diffRevisions are still to do. The frozen kernel sends descriptions on the +wire and closes a link on an unknown id; both are replaced.

+
+
+ +
XVIII

Building p2: the Core

From a list of what the app must do to a set of owners: the base code every p3 plugin builds on.

+

The notes plugin used every Chord concept. A real app is the same concepts many times over. This chapter and the +next two assemble p3, the Chord v2 version of p2, from the parts the guide introduced. p3 is built next to p2 in +apps/p3, fully isolated with its own data directory, and without migrating p2's data (no import of +catalog.sqlite, shares.sqlite or the subagents table). Its plugins are derived from what the app must do, not +ported one to one from p2's plugin layout.

+
Review: p3 is not designed yet

The design fixes the rules (design.md §25) and lists the p3 decisions still open (#17, #18, #20 to #23); the +per-domain design of p3 has not started. What these three chapters show is one construction that follows the rules, +derived from the functionality map in apps/p2/docs/p3-functions/. Object names are working names. Where a domain +has an open decision, the text says which.

+
+

Start from functions, not from plugins

+

The functionality map in apps/p2/docs/p3-functions/ lists what p2 does, in plain words, with four facts per +function: the data and its size, how durable it is, who may see or do it, and where it must be available. It groups +the functions into 22 domains:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#Domain#Domain
1sessions12access
2session workers13models, credentials, settings
3conversations and agent14prompt resources
4shell commands15status and presence
5chat mode16search
6agents (subagents, one-shots, peer messages)17remote access and forwards
7code mode18logs and notices
8session files19dev and reload
9environments and execution20web shell
10workspace21TUI shell
11terminals22transport
+

The four facts answer the four questions every Chord object needs, in order:

+
THE MAP SAYSCHORD NEEDSdatawhat, how bigdurabilityephemeral · until restart· persisted ·transcript-boundwhoowner · guest read · guestwrite · workerwhereserver · worker · web ·TUI · environmentthe ownersmallest object it belongstowhere it is keptDurable, persisted orin-memory document, tabvariantsand revocationhost and codeplacement; core or pluginDurability decides storage: transcript-bound data is Durable's; persisted app data is a persisted document with astable identity; until-restart data is an in-memory document or a native; tab-scoped UI state stays in the client.
Figure 12. From a function in the map to code. Each fact in the map answers one question Chord needs answered.
+

One owner per thing, the smallest object it belongs to. A guest's grant belongs to the access owner, not to the +session; a draft belongs to one (conversation, author) pair, not to the chat plugin as a whole. Ownership nests, and +rules set at creation are written in the creating commit (Durable's createConversation({ init })).

+

Core or plugin. Core owners are what the app always needs and everything builds on; they are always present, so +other code may depend on them directly. Plugin owners may be missing; anything that depends on an optional feature +does so through a registry it owns itself (chapter XIII). The risk to avoid is a core god object: each core owner +owns only what belongs to it.

+

The ownership map

+

The result of that procedure for p3, as a starting point:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
DomainOwner object(s)HostCode
accessAccess (grants, devices, share links), authenticateservercore
sessionsCatalog (session records), lifecycle jobsservercore
session workersthe supervisor (route(Session) + idle stop)servercore (app code)
session locationSessionLocation { session }workercore
conversations and agentHarness, ConversationStore, Conversation viewsworkercore
status and presenceSessionStatus (server copy), reporters on workersbothcore
shell commandsShellJob in the chat pluginworkerplugin
chat modechat pluginworkerplugin
agentssubagents plugin (Durable child conversations)workerplugin
code modecode mode plugin (sandbox native)workerplugin
session filesSessionFiles { session } (proposed, #17)servercore or plugin, open
environmentsenvs plugin (config, SSH pools #20)server and workerplugin
workspaceworkspace pluginworkerplugin
terminalsterminal host process (#23), terminals pluginseparate processplugin
models, credentialsmodels plugin, login job + flowserverplugin
settingsSettings (single writer, #21)servercore
prompt resourcesprompt plugin (watchers in the session's environment)workerplugin
searchsearch plugin (index on the server, feed on workers)bothplugin
remote access, forwardsradius plugin, ports pluginserverplugin
logs and noticesNotices (one channel), log captureservercore
dev and reloadthe reload control, reload_plugins toolserverplugin
web and TUI shellsUI facets over the objects aboveclientsshells
+

The rest of this chapter builds the core rows. Chapters XIX and XX build the plugins.

+

The base code

+
apps/p3/
+  src/
+    hosts.ts           Server, Session host kinds
+    principal.ts       P, Revocation, principalKey, revokedBy, can()
+    server/main.ts     createHost, load, route(Session), WebSocket and Radius entry points, supervisor
+    worker/main.ts     createHost with the start data, connect to the parent
+    core/              the core facets: Access, Catalog, SessionStatus, Settings, Notices (server);
+                       SessionInfo, SessionLocation, Harness, ConversationStore (worker)
+  plugins/             chat, subagents, codemode, envs, workspace, terminals, models, login,
+                       prompt, search, ports, radius, files, dev
+

hosts.ts, principal.ts, the SessionInfo start data and the Harness object are the ones the guide already +built for Studio (chapters I, III, IX and XIV). Base code restarts the hosts when it changes; everything in +plugins/ reloads.

+

Who is calling: authentication

+

Chord stamps principals; the app produces them. p3's server has four ways in, each turned into a principal by +authenticate(req) when a WebSocket connects (chapter III):

+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
Way inPrincipal
loopback without a cookie (TUI, CLI, first page load){ kind: "owner", access: "local" }
loopback owner browser cookie (a per-port secret file){ kind: "owner", access: "local" }
a device cookie (HMAC-signed, per port, 30 days), obtained with a one-time sign-in token{ kind: "owner", access: "device", device }
a guest cookie from a share link{ kind: "guest", session, mode, grant, revision }
the Radius relay channel{ kind: "owner", access: "relay" }
+

The checks that are not Chord's stay in this function: origin checks and Fetch Metadata (a browser request from +another site gets no principal, except a top-level navigation), proxy trust, the sign-in token rules (10 minutes, +single use, at most 32 outstanding, memory only), cookie expiry and revocation lists. A failed check closes the socket +before Chord sees it.

+

One permission matrix. Every function in the map names who may use it, and p2 checks those rules in several +places. p3 writes them once, in principal.ts, and every factory and authorize uses it:

+
// src/principal.ts
+type Action = "read" | "send" | "upload" | "stop" | "workspace" | "terminal" | "manage";
+
+export function can(p: P, action: Action, session?: string): boolean {
+  if (p.kind === "owner" || p.kind === "server") return true;
+  if (p.kind === "worker") return action !== "manage" && p.session === session;
+  if (p.kind !== "guest" || p.session !== session) return false;
+  if (action === "terminal" || action === "manage") return false;     // terminals: owner only
+  if (p.mode === "read") return action === "read";                     // no workspace for guest read
+  return true;                                                         // guest write: send, upload, stop, workspace
+}
+

Factories pick variants from it (can(p, "send", session) ? "writer" : "reader"), so the matrix is visible in one +file and the members a caller lacks are simply not there.

+ +

Access is the core owner of everything that lets someone in: share links per session and mode, guest records, the +device list and the signing key. It lives on the server as the chapter IV pattern: a local store that persists the +documents (only stable identities may), and a remote view for the owner's settings screen:

+
// src/core/access.ts (server, in the core server facet)
+const Links = defineState("access.links", { schema: map(ShareLink), version: 1, persist: true,
+  initial: () => ({}), checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64 });
+const Guests = defineState("access.guests", { … });         // same shape: map(Guest), persisted
+const Devices = defineState("access.devices", { … });       // map(Device), persisted
+
+host.serve(AccessStore, (_input, self) => {                 // local: may persist, acts for the host
+  const links = self.state(Links), guests = self.state(Guests), devices = self.state(Devices);
+  return {
+    links, guests, devices,
+    removeGuest: async ({ grant }, ctx) => {
+      await self.tx(async (tx) => { (await tx.draft(guests))[grant].blocked = true; }, ctx);
+      host.revoke({ grant });                                 // live access ends now
+    },
+    resetLink: async ({ session, mode }, ctx) => {
+      const blocked = await self.tx((tx) => rotateLink(tx, links, guests, session, mode), ctx);
+      for (const grant of blocked) host.revoke({ grant });
+    },
+    signOut: async ({ device }, ctx) => {
+      await self.tx(async (tx) => { (await tx.draft(devices))[device].revoked = true; }, ctx);
+      host.revoke({ device });
+    },
+    …                                                         // shareLink, join, signOutAll
+  };
+});
+
+host.serve(Access, (_input, principal, self) => {           // remote view: owner only
+  if (!can(principal, "manage")) throw forbidden();
+  const store = self.get(AccessStore, {});
+  return {
+    variant: "owner",
+    guests: self.derive("guests", [store.guests], ([g]) => publicGuests(g)),   // names, modes; no tokens
+    removeGuest: (args, ctx) => store.removeGuest(args, ctx),
+    resetLink: (args, ctx) => store.resetLink(args, ctx),
+    signOut: (args, ctx) => store.signOut(args, ctx),
+  };
+});
+

authenticate is base code on the same host and checks AccessStore's current documents on every new connection +(it holds a reference to the store, taken with server.root at start). Live access and lasting denial stay separate, as chapter VII explained:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
EventLasting effect (data)Live effect
remove a guestguest record blockedrevoke({ grant })
reset a share linknew token; its guests' grants blockedrevoke({ grant }) for each
archive a sessiongrants of that session void while archivedrevoke({ session })
restore a sessiongrants valid again; revision bumpednone: guests reconnect
sign out a devicedevice revoked until its expiryrevoke({ device })
sign out all devicessigning key rotated, tokens clearedrevoke({ device }) for each
+

A guest who rejoins with the same browser keeps its identity; a removed guest cannot mint a new one, because the +block is in Guests and authenticate refuses it. Names are unverified, cleaned, at most 40 characters, unique per +session ("Ann (2)").

+

The catalog

+

Catalog owns the session records: id, title (and whether it was edited), location, model default, pinned, +archived, timestamps. It lives on the server, persisted, and it never starts a worker: listing sessions, showing +their status or their unread marks reads server-side copies only.

+
export const Catalog = remote("p3.catalog", {
+  version: 1,
+  input: obj({}),
+  placement: () => Server(),
+  variants: {
+    owner: {
+      state: { sessions: obj({ items: arr(SessionRow) }) },
+      methods: {
+        create: method(CreateArgs, str()), rename: method(RenameArgs), pin: method(PinArgs),
+        archive: method(obj({ id: str() })), restore: method(obj({ id: str() })),
+      },
+    },
+    guest: { state: { sessions: obj({ items: arr(SessionRow) }) } },     // only the guest's own session, if not archived
+  },
+});
+

Behind it sits a persisted local CatalogStore; the guest variant is a derive over it, as in chapter IV. The lifecycle operations (archive, clone, fork, +relocate, delete) are chapter XIX; here only the rule that keeps the catalog small: the catalog does not know which +plugins exist. Archiving a session writes archived: true and revokes the session; every plugin that keeps +per-session things reconciles from catalog.sessions with a routine (chapter X), as the notes plugin did. The ports +plugin closes forwards of archived sessions that way, also after a crash.

+

On the worker: start data, location, harness

+

Three core objects exist on every worker from its start:

+
    +
  • SessionInfo (chapter IX): the start message as constant(init), with the session id, data directory and the +location the server chose at start.
  • +
  • SessionLocation { session } (working name), lifetime host: the current cwd and environment, with a location +revision. Relocation changes it while the worker runs (chapter XIX); natives that depend on the location follow it +with a routine and a generation (chapter IX), and conversations that follow the session's cwd are listed in the core +per-conversation policy document.
  • +
  • Harness (chapter XIV): the Durable harness for the session's store, one SQLite database per session, opened +under an exclusive lock keyed by the real data directory and session id. It refuses to start when the saved root +model is not configured, and resumes interrupted Durable work on start.
  • +
+

ConversationStore and the Conversation views in front of it are chapter XIX.

+

Workers: start, keep alive, stop

+

Starting, supervising and stopping workers is app code (chapter I); Chord only routes and counts. p3's supervisor is +server.route(Session, { authorize, resolve }) plus a small server object that keeps per-session worker state:

+
A worker's life, as the supervisor sees it

Authorize, then start

A tab gets Notes { S1 }. authorize checks can(p, "read", "S1") and that the session exists and is not archived +(an archived session refuses, except an index rebuild that starts the worker without resuming work). resolve forks +worker/main.ts with the start data from the catalog, bounded by a 60 s readiness timeout.

+

A start failure sticks

If the worker fails to start (an unconfigured environment, a missing model), the error is kept per session and +every get fails at once with it, until the owner presses Retry. That clears the error and the related notices. +Guests see that the session is unavailable, not the error text.

+

Counting

The core facet on the worker reports host.activity ({ references, calls, jobs, lingering }) and the harness's +running Durable task count to the supervisor object, from a routine whenever either changes. The call carries +ctx.incarnation, so a report from an older worker incarnation is ignored (chapter X).

+

Idle stop

When everything has been zero for five minutes (checked every second, as in p2), the supervisor stops the worker: +the worker closes its host (routines stop, natives close, storage flushes) and exits. Durable work stays persisted +and resumes on the next start. Detached work that must keep the worker alive (a shell command whose caller left) is +a job, so it counts.

+

Crash

If the worker exits on its own, the link drops. The supervisor records the exit code and a redacted stderr tail +(at most 4,000 characters) for that incarnation and posts one notice. References that still exist make Chord call +resolve again with backoff; a passive viewer recovers without a click.

+

Shutdown

On server shutdown the supervisor stops every worker, with a forced exit after three seconds. Durable work remains +persisted.

+
+

Status without a worker

+

The session list shows per session whether it is running, busy (which conversations), its task count, usage and cost, +and an error. None of that may start a worker. The pattern is chapter X's pub/sub: a reporter on each worker (a +routine over the harness's state) calls SessionStatus.report(summary) on the server; SessionStatus keeps the latest +value per session and ignores calls whose ctx.incarnation is older than the newest it has seen; clients read +SessionStatus, which lives on the server. Children lists, unread marks, search index rows and artifact lists use the +same pattern (chapter XIX).

+

Settings and notices

+

Settings (default model and thinking, run settings, compaction, queue modes, code mode switch, owner name) have +one writer: a core Settings object on the server owns the file, validates writes and serializes them; every +other process follows its state through a reference instead of watching and rewriting the file itself. p2 watches +settings, models and auth in every process and writes them from two, which is decision #21.

+

Notices are one channel: one core Notices object on the server, written by everything that fails or finishes +(worker crashes, reload results, model errors, login outcomes, grants), keeping one latest notice per source, at most +50. The owner's shell reads one list; dismissing is a method. p2 has five notice channels for the same thing.

+
Review: settings and file formats (#21, #22)

Single writer for settings, models and auth (#21) and versioning on-disk formats such as envs.json like documents +(#22) are open p3 decisions in design.md §26. The construction above assumes "one owner on the server, others +follow its state" for #21.

+
+
Built so far

Nothing of p3 exists. The functionality map is complete (apps/p2/docs/p3-functions/, five area files plus the +condensed README.md), and the design's app rules (design.md §25) are decided. p3 starts after the layered rebuild +and the playground's first real proof.

+
+
+ +
XIX

Sessions and Conversations

The session lifecycle, status without a worker, chat, shell commands, subagents and search, each mapped to an owner and the Chord parts it uses.

+

This chapter builds the domains a person touches most: the session list and its operations, the conversation view +and composer, shell commands, chat mode, subagents, messages between sessions, search and unread marks. Each section +names the owner, the host, and the Chord parts it uses; code appears only where it shows something new.

+

The session lifecycle

+

The catalog (chapter XVIII) owns the session records. Its operations, by the Chord part each needs:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
OperationOwner and formNotes
createCatalog.create, a methodabsolute directory, ~ expanded, must be a directory; on an environment, resolved there through the envs plugin; refuses an unknown model; model and thinking default to the last used
rename, pinmethods on Catalogtitle 1 to 120 characters, whitespace collapsed; an explicit title is never replaced automatically
automatic titlea routine on the search feedthe first user message, line breaks to spaces, 100 characters
agent renamean agent tool on the worker calling Catalogroot conversation only
archive, restoremethods; revocation; reconciling routinesbelow
clone, fork, relocatejobs on the catalog storeoutlive the call, cancellable, one outcome
deletea job; decision #18below
+

Archive and restore

+

Archiving must stop the worker whatever it is doing, end guest access, and close the session's port forwards, while +keeping its storage. The catalog does none of the plugin-specific parts:

+
Archiving S1

One write

Catalog.archive({ id: "S1" }) sets archived: true in the session record, in one transaction.

+

Revoke the session

The catalog calls host.revoke({ session: "S1" }). Every guest principal of S1 has { session: "S1" } in its +revokedBy, so their objects end and their links close on the server and on the worker (chapter VII). New +connections are refused by authenticate, because grants of an archived session are void.

+

Stop the worker

The supervisor reconciles from catalog.sessions with a routine: an archived session's worker is stopped, and +authorize refuses to start it again.

+

Plugins reconcile

The ports plugin's routine on catalog.sessions closes forwards of archived sessions. The search plugin marks the +session's rows archived. If the server crashed between the write and these steps, the routines run again with the +current value at the next start: nothing is a notification that can be lost.

+

Restore

Catalog.restore clears the flag. Grants are valid again, so guests reconnect and authenticate as before; forwards +stay closed.

+
+

Clone, fork, relocate

+

Each is work that outlives its call, can be cancelled and has one outcome, so each is a job (chapter XI) on the +catalog's local store, keyed by the owner's request id. They differ in what they pause:

+
    +
  • Clone copies the session's store, uploads, downloads and artifacts under a new id, with the source worker +paused (refused while work runs) and title "(copy)"; directories 0700, files 0600, no links, no overwrite; a +partial copy is removed on failure.
  • +
  • Fork before a user message copies the active context as of the entry before it into a new session (title +"(fork)"), with the message waiting in the new session's editor. It needs two workers: the source reads history, +the new one is seeded empty. A failure archives the copy.
  • +
  • Relocate (cwd, environment or both): refused while the session is busy (naming the running subagents) or when +a subagent has its own cwd and the move crosses machines. The job fences the worker (new admissions wait), updates +the record, and moves the running worker in place by updating SessionLocation: natives that follow the location +switch with a generation (chapter IX), and conversations that follow the session's cwd are rewritten in one Durable +commit before work resumes. After 60 s it restarts the worker instead; on failure it releases the fence.
  • +
+
Review: jobs as objects (#9)

The design defers whether a long relocation or a login is a record inside its owner (watched through running) or an +object of its own (Relocation { id }). The construction above uses records. Decide when login is ported +(design.md §15, §26).

+
+

Delete

+

p2 has no session delete. p3 needs one that removes the session everywhere: every document tagged with the session +(store.retireByTag(SessionTag, id), chapter V), the worker's databases (the Durable store), the session's files, the +search rows, read marks, children copies, forwards, guest records, and finally the catalog entry. The design commits to +this outcome (design.md §25); how plugins take part is open.

+
Review: session delete (#18)

Options in design.md §26: per-session objects plus SessionTag plus one session directory (delete is retire by tag +and remove a directory, plugins need do nothing), or owner hooks contributed through a catalog registry. Also open: +whether the search index is per session (deleted with it) or global with a hook. Who holds the store for +retireByTag is not specified either.

+
+

Status, children, unread: reports to the server

+

The session list must show status, subagents and unread marks for sessions whose workers are not running, and must +not start them. All three follow chapter X's pub/sub pattern: a routine on the worker calls a server object with the +current summary, the server object keeps the latest value per session (persisted where it must survive a restart), +ignores calls from an older ctx.incarnation, and clients read the server object.

+ + + + + + + + + + + + + + + + + + + + + + + +
SummarySource on the workerKept on the server
status: running, error, task count, busy conversations, usage and costthe harness's task graph and usageSessionStatus, in memory: a stopped worker is "not running"
children: name, model, thinking, tools, cwd, last entry, archivedthe subagents plugin's child listChildren, persisted
latest root answerthe search feedUnread: compared with the owner's read mark, persisted, shared by all owner devices
+

Guests see their own session's status without the error text. The owner's read mark is written when the root +conversation is visible, and a new session starts as read.

+

Conversations

+

The harness (chapter XIV) owns the Durable store. The UI needs, per conversation: a live window of entries, older +pages, usage numbers, the working indicator, the inbox (queued inputs), and the actions. Two core objects provide them:

+
    +
  • ConversationStore { conversation } (local, on the worker): opens the Durable watch as its native, reaching +the harness through the core-private HarnessRuntime registry entry, and exposes a window as a derive: the +newest entries (at most 100 or 512 KiB, at least 20) plus the conversation's documents. The window's start stays +put while it is within 1.5 times the limits, so new entries arrive as small appends instead of a shifted list +(chapter VIII's window pattern).
  • +
  • Conversation { session, conversation } (remote, placement Session): the per-principal view. Variants by +can(): owner (everything, including model, thinking and handoff), writer (guest write: send, steer, stop, +compact, shell), reader (the window only).
  • +
+
host.serve(Conversation, ({ session, conversation }, principal, self) => {
+  const store = self.get(ConversationStore, { conversation });
+  const policy = self.get(ConversationPolicy, { conversation });         // core: readOnly, followsSessionCwd
+  const author = authorOf(principal);                                    // structured: kind, id, display name
+  if (!can(principal, "send", session)) return { variant: "reader", view: store.window, page: store.page };
+  return {
+    variant: can(principal, "manage") ? "owner" : "writer",
+    view: store.window,
+    page: store.page,                                                    // older history: returns data
+    send: ({ text, attachments, mode, requestId }, ctx) => {
+      if (policy.value.readOnly) throw forbidden("read-only conversation");
+      return store.submit({ type: "input", content: text, attachments, author, whenBusy: mode, requestId }, ctx);
+    },
+    stop: (_args, ctx) => store.stop(author, ctx),                       // allowed even when read-only
+    …
+  };
+});
+

What each function becomes:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FunctionHow
send, retried sendDurable submit with the author's requestId: a retry with the same id is admitted once (Durable dedupes, chapter XIV)
steer or queue while busywhenBusy: "steer" or a follow-up; the inbox is docs["pi.inbox"] in the window
attributionthe view derives the author from the principal; the entry stores it structured, the model sees a speaker block; nothing parses text
wait for prompt resourcesthe first input of a fresh worker waits until the prompt plugin's resources are loaded (a registry readable the store follows)
stop, dequeuewithdraw queued inputs, abort the run; the caller's texts return to its editor, other authors' to their restored drafts
restored draftsa keyed document per (conversation, author), tagged SessionTag, persisted in the store; claimed once, first tab wins
read-only conversationsthe core per-conversation policy document, written at createConversation({ init }) by whoever creates the conversation; send, dequeue, configure, compact and shell refuse; stop is allowed
history pagesa method returning a page (100 entries, 1 MB, 2,000 scanned); compaction and reset markers always included
model and thinkingowner variant only; validated against the models plugin's catalog; the root conversation's model is also reported as the session default
compaction, handoffDurable operations through the store
+

Drafts in the composer are UI state, not Chord state: the web shell keeps one draft in sessionStorage, the TUI +keeps one per conversation across UI replacements, and pending sends keep their request id until acknowledged +("Sending when connected"). Only restored drafts, which the worker produces, are documents.

+

Shell commands

+

!cmd runs a command in the conversation's cwd on its environment and records the result as an entry; !!cmd does +the same but keeps the output out of the model's context. One command at a time per conversation; the live tail is +at most 50 KB or 2,000 lines, sampled every 200 ms; the full output spills to a file; the result entry is placed at the +run's next boundary while the agent is busy and never starts a run. This is the design's job example almost as is:

+
const ShellJob = defineJob("chat.shell", {
+  version: 1,
+  args: obj({ command: str({ max: 100_000 }), include: bool(), author: Author }),
+  progress: obj({ output: str() }),
+  result: ShellResult,
+  concurrency: 1,                                                   // one per conversation
+});
+
+const results = self.state(ShellResults);                           // keyed by run id: dedupe after completion
+const shell = self.job(ShellJob, {
+  run: async ({ command, include, author }, ctx) => {
+    const out = new OutputBuffer({ maxBytes: 50_000, maxLines: 2000, retain: "tail" });
+    const env = await environmentFor(ctx);                          // the conversation's environment, chapter XX
+    const r = await env.exec(command, { cwd: env.cwd, signal: ctx.signal, spill: spillPath(ctx.jobId) },
+      (chunk) => { out.push(chunk); ctx.progress({ output: out.snapshot().text }); });
+    const data = { command, output: out.snapshot().text, exit: r.code, cancelled: ctx.signal.aborted };
+    await conversation.submit(
+      { type: "write", entry: shellEntry(data, author, include), requestId: `shell:${ctx.jobId}` }, ctx);
+    return data;
+  },
+  finish: async (done, tx) => {
+    (await tx.draft(results, done.id)).outcome = done.kind === "done" ? done.result : { error: done.error };
+  },
+});
+

The request id shell:${ctx.jobId} makes the entry idempotent: an attempt interrupted after the write cannot write +it twice. Cancel is shell.abort(id), allowed for the command's author and the owner. A retried runShell with the +same request id returns the running job (dedupe of unsettled runs), or the stored result (the owner's own dedupe after +completion, chapter XI). p2's "cancel when nobody watched for 30 s" is dropped: a job keeps its owner alive and does +not depend on viewers.

+

Chat mode

+

Chat mode is a per-conversation switch that everyone in the session sees and only the owner changes; forks and new +subagents start with it off. Because it must fork and rewind with the conversation, it is transcript-bound: a +Durable conversation document, not a Chord document. Messages without @pi become display-only Durable write +entries (author, text, attachments, reply quote); a message with @pi becomes an input with a digest of the chat +since the last mention (at most 100 messages, 20 files, 4 images), recording which entries it passed on. The reply +quote is derived on the worker from the entry id the client names: the client never supplies author or quote text.

+

Subagents and one-shot agents

+

Subagents are Durable child conversations owned by tasks, as in Durable's own background-subagent example: one +subagent tool spawns, steers, waits for, stops and lists them; each child is owned by a background anchor task, and +each message is delivered by a reporter task that posts the answer back to the parent once, with request ids so a +restart sends nothing twice. Chord adds only three things:

+
    +
  • the child list for the sidebar is reported to the server (Children, above), so it shows without a worker;
  • +
  • a child that follows the session's cwd, or a read-only one-shot agent created by a code-mode script, is marked in +the core per-conversation policy document when it is created, so chat and relocation enforce it without knowing +the subagents plugin (design.md §25 porting guideline);
  • +
  • archive and restore of a child from the UI are methods on the subagents plugin's view, refused for one-shots.
  • +
+

Subagents cannot spawn subagents; thinking levels are clamped to what the model supports, in one helper shared with +the session's model choice.

+

Messages between sessions

+

The root agent can list other sessions and send one an attributed message. The tool runs on the sender's worker and +calls a server object through chord(fn, context) (chapter XIV):

+
const peerSend = defineTool({
+  name: "session_send",
+  execute: (args, api, context) => chord(async (ctx) => {
+    const peers = await ctx.get(Peers, {});                       // server, worker variant
+    await peers.send({ to: args.session, text: args.text, mode: args.mode, requestId: `peer:${api.taskId}` }, ctx);
+    return { output: [{ type: "text", text: "sent" }] };
+  }, context),
+});
+

Peers checks that the sender is a root conversation of a non-archived session, refuses sending to itself and +archived recipients, then gets the recipient's Conversation (which starts its worker) and submits with the +sender as a structured author and the same request id (the tool task's id, stable across restarts), so a retried +tool call delivers once. The sender's +incarnation is ctx.incarnation; a call from a replaced worker is refused. Only owner-initiated turns may use the +tool; guests never.

+ +

The search plugin indexes user and assistant messages of root and subagent conversations (no shell output, no +Durable internal inputs) for title, cwd and message search across sessions, active or archived.

+
WORKER S1SERVERDurable commitin any conversationwakea routine on the harness's commitsignalread from cursor200 entries per pageSearchIndexnative: SQLite FTScursor per conversationmoves after the whole rangequeryterms, scope, at most 41 hitsreport(page)Reports carry ctx.incarnation. Indexing is idempotent: a page sent twice changes nothing.
Figure 13. The search feed. The worker pushes pages from a cursor; the server owns the index and never starts a worker to answer a query.
+

The index is a native owned by a host-lifetime object on the server; cursors live with the index, so after a crash +the worker asks for the cursor and continues. Rebuilding one session's index drops its rows and rescans, starting +an archived session's worker without resuming its work. Queries are methods returning data: at most 41 hits, +snippets of 300 characters, queries up to 500 characters. The same feed sets automatic titles, moves "updated at" +forward and supplies the latest root answer for unread marks.

+

Presence

+

Viewers per session are host.references(Conversation) on the worker, counted per client (chapter VIII), and +reported with the status. Connected guests (name and mode) are the references whose principal is a guest. Presence +is not keep-alive: the supervisor uses host.activity, which also counts calls and jobs.

+
Built so far

None of this exists in p3. The parts it relies on are Durable (built: submit with request ids, steer and queue, +child conversations, background tasks) and the Chord layers L3 to L9 (not built).

+
+
+ +
XX

Machines, Files, Models

Environments, workspace, terminals, session files, ports and Radius, models and logins, prompt resources, logs, reload and the shells: the rest of p3.

+

The last domains are the ones that touch the outside world: other machines over SSH, the file system and git, +terminals, the relay, model providers and their credentials. Most of them are native objects, and most of them are +places where p2 has the same thing several times (four SSH connections to one environment, five notice channels, +three copies of the environment config parser). The rule that fixes that is the one from chapter XVIII: one owner per +thing.

+

Environments

+

The envs plugin owns the environment configuration: name, host, user, port, identity file, shell and its +variables, plus a managed ed25519 key per environment and its own known_hosts (kept apart from the user's, aliased +per environment name).

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FunctionOwner and form
list, add, edit, delete (refused while a session, archived too, uses it)Envs on the server: a persisted document behind an owner-only view
validation (no leading -, no whitespace or control characters in host and user)the view's methods, before the write
a hand-edited envs.json is picked up; an invalid file keeps the last valid set and shows an errora native file watcher (poll fallback) in the envs store; one writer (#21)
environment revision: a hash of the config, part of its identitya derive
connection test (connect, deploy the daemon, platform, latency, warnings)a job; the result per revision in a document
unknown or changed host key: fingerprints, trust after confirmation, forgeta flow (chapter XII) on the server
browse directories (~, relative paths, at most 10k scanned, 1,000 shown)a method with a call-scoped SSH session (chapter IX) or the pooled connection
a changed environment restarts its sessions' workers once idlea routine on the envs store and the status reports
+

Execution in the agent. The worker's harness asks HarnessEnvironments for an environment per target (chapter +XIV). The envs plugin contributes it from its worker facet: local execution at the session's cwd, or a remote one over +SSH through the pi-env daemon, deployed per platform and connecting at the first operation. A remote session never +falls back to local tools. The environment follows SessionLocation: after a move, the old connection closes once +the move has committed and watchers reopen at the new location.

+
// envs/worker.ts
+export default defineFacet((host, self) => {
+  const location = self.get(SessionLocation, { session });
+  const pool = self.get(EnvPool, {});                              // one owner of SSH connections (#20)
+  self.contribute(HarnessEnvironments, (target) =>
+    target.env === "local" ? localEnv(target.cwd) : pool.environment(target.env, target.cwd));
+});
+
Review: connection pools (#20)

SSH connections to one environment exist four times in p2 (worker environment, browse connection, test connection, +terminal host). design.md §26 asks for one owner with a pool or deliberate per-use connections. EnvPool above is +the first option: a native object that closes idle connections after 10 minutes and invalidates them on a config or +key change. Reloading the plugin that owns the pool kills running remote commands (chapter XV), so it should change +rarely.

+
+

Workspace and file index

+

The workspace plugin serves the viewed conversation's working tree (a subagent's own cwd included) on the worker: +list a folder (at most 2,000 entries, .git hidden), read a file (1 MiB, binary detected), git status, diff, log and +single commits, literal text search, and all paths for name filtering. Every query is a method returning data, not +state: results are bounded and requested by the UI. Only change notifications are state: a counter in a document, +bumped by a native watcher of the tree, HEAD, the index and config (debounced 100 ms); viewers reload what they show +when it changes.

+

Two rules live in one place each instead of three:

+
    +
  • Confinement: path rules, a canonical root, no .git, the app's private data directory denied (also excluded +from search). One helper used by listing, reading, uploads and the file index.
  • +
  • Git hardening: no optional locks, no fsmonitor, filters disabled, no prompts or pager, no system config, 10 s +timeout, output caps.
  • +
+

Downloads (25 MiB, byte ranges for media) go through the HTTP component (chapter XIII) with a route on the server that +reads chunks from the worker; uploads into the project use the chunked protocol of chapter XIII (1 MiB chunks, temp +file then rename, never overwrite, writes serialized). Guest read gets no workspace: the view has no variant for it.

+

The file index for @ completion belongs to the same owner, so completion consumes the index instead of scanning +on its own: built from the environment's listing (else git or a walk, at most 50k paths), rebuilt at most once per +second while files change, discarded after a move. Review comments drafted on files and diffs are browser state until +sent; sending them is an ordinary Conversation.send.

+

Terminals

+

Terminals must survive server and worker restarts, so their PTYs live in a detached terminal host process, one per +data directory, with its own process lock; it exits after 10 minutes with no terminals and no clients. The terminals +plugin on the server is the owner the UI talks to: list (title, cwd, environment, size, exit status), open at the +session's cwd and environment (refused for archived sessions), attach, close (asking first if a foreground job runs). +Terminals are owner only.

+

Bytes do not go over object links. Attaching opens a channel (chapter XIII) with a single-use token (30 s): JSON +control on the object, bytes on the channel, with its own acknowledged window (256 KiB, acknowledgements per 64 KiB; a +viewer that falls behind gets a fresh screen snapshot; an 8 MiB backlog disconnects). Attaching gets the current +screen from a headless emulator with 10k lines of scrollback.

+
Review: the terminal host process (#23)

The terminal host runs its own code. Is it part of the revision, reloaded or restarted with it (and running terminals +end), or deliberately outside it with a "host runs older code" notice as in p2? Open in design.md §26.

+
+

Session files

+

Attachments (25 MiB, at most 20 per message), sent-file snapshots made by the agent's send_file tool, versioned +artifacts (HTML, SVG, Markdown, text; 2 MiB; each update a new immutable version) and shell spill files are all +per-session bytes with caps, immutable ids and range reads; p2 has two upload routes, two caps and two directory +layouts for them. The proposal is one owner:

+
export const SessionFiles = remote("p3.session-files", {
+  version: 1,
+  input: obj({ session: str() }),
+  placement: () => Server(),                                  // readable without a worker
+  variants: { writer: { … }, reader: { … }, worker: { … } },
+});
+

SessionFiles { session } on the server holds the index (per kind, readable without a worker), the chunked upload +and read protocol, the caps per kind, copying on clone and fork, and deletion with the session. Attachments are copied +into a remote environment before an input is admitted, with no fallback on failure. Artifact previews run in a +sandboxed frame with its own CSP and no app identity; ordinary file previews allow only safe types.

+
Review: session files (#17)

Proposed in design.md §26, not decided. Open sub-points: one preview limit (p2 has 256 KiB for sent files and +512 KiB for artifacts), what a fork copies of the artifact list, and how attachments are copied into remote +environments. Related: #22, whether on-disk layouts such as downloads and uploads get versions like documents.

+
+

Ports and Radius

+

The radius plugin keeps the outbound relay connection, a native in a host-lifetime object on the server: sign-in +by device code from the CLI (credential in a private file), token refresh, a heartbeat in the link adapter (20 s ping, +60 s timeout; TCP does not notice a sleeping laptop), backoff from 1 to 30 s, a disable switch. The relay delivers two +kinds of channels:

+
// radius/server.ts (the entry point: one line, as in chapter I)
+radius.onChannel((ws) => host.accept(webSocketLink(ws), { principal: { kind: "owner", access: "relay" } }));
+radius.onForward((channel) => proxyToForward(channel, host.registry(RadiusForwards).value));
+

Object links become owner principals with access: "relay": owner, but treated as remote (no localhost-only +powers). p2 had to fake a loopback request with a secret header for this; with accept that is gone.

+

Forward channels go to localhost:<port> (Host, Origin and Location rewritten; 502 when nothing listens; anonymous +visitors only to public forwards). Which forwards exist is not radius's business: it owns a registry, +radius.forwards, and the ports plugin contributes its forwards (at most 50, random ids, a public toggle that +needs confirmation, an agent tool ports for top-level owner-initiated conversations). Without the ports plugin the +table is empty and radius still works: an optional dependency is a registry owned by the dependent (chapter XIII). The +ports plugin closes forwards of archived sessions with a routine on catalog.sessions (chapter XIX).

+
Review: may a plugin call accept?

design.md §3 shows the Radius line in the server's start code and calls entry points "app or adapter code". If +radius is a plugin, its facet needs the host's accept, which no other part of this guide gives a plugin. The +alternative is that the start code keeps the entry point and gets channels from the radius object. Not specified.

+
+

Models and credentials

+

The models plugin owns the model runtime (pi-ai's models with custom providers and the model cache) in a +host-lifetime object, and the credentials. It contributes the runtime to HarnessModels (chapter XIV), so after a +models reload the next request uses the new runtime while a running stream finishes on the old one. Credentials +never go to clients: the views expose providers, login methods and which type is signed in, never tokens. Every +secret is registered for redaction in one place, the log capture.

+

Views: a model catalog for pickers (names, ids, context windows, thinking levels; at most 300 matches in the web), +logout (refused during a login), refresh a provider's model list. The default model and the cycling scope are +settings (chapter XVIII).

+

Login

+

A login is the guide's flow example (chapter XII): a job on the server that runs pi-ai's login, and a flow that +shows its steps to the starting owner.

+
const LoginJob = defineJob("models.login", {
+  version: 1, args: obj({ provider: str(), type: AuthType }), progress: LoginState, result: obj({ ok: bool() }),
+  concurrency: 1,                                                    // one login at a time on the server
+});
+
+const login = self.job(LoginJob, {
+  run: async ({ provider, type }, ctx) => {
+    await runtime.value.login(provider, type, {
+      signal: ctx.signal,                                            // 10 minute deadline, cancel, leaving
+      prompt: (prompt) => steps.ask(prompt, ctx),                    // text, secret, select, manual_code
+      notify: (event) => steps.notice(event, ctx),                   // info, auth_url, device_code, progress
+    });
+    return { ok: true };
+  },
+  finish: async (done, tx) => { /* record the outcome; refresh the model list */ },
+});
+

prompt is a promise-based callback inside the work; the resolver stays in the job (steps.ask records the step in +the flow state and waits for answer), and nothing crosses an object boundary. On a relayed connection the device +code method is selected automatically. The flow is visible only to the owner principal that started it; others are +refused. Leaving ends it: when the starting owner's references are all gone (presence, not an unmount), a routine +aborts the job. Codes, URLs and typed secrets are redacted from logs.

+

Prompt resources

+

AGENTS.md, skills and templates are read in the session's environment, watched (poll fallback, last valid state), +and expanded on the worker before sending. The prompt plugin owns them per session on the worker, contributes the +system prompt section and skills to HarnessExtensions as readable contributions (a change recomposes), and exposes a +readiness readable: the first input of a fresh worker waits until it is ready (chapter XIX). After a move the watchers +reopen at the new location.

+

Code mode

+

Code mode is a plugin (chapter XIV): its pi-codemode sandbox is a native; scripts call tools as nested Durable calls +and replay after a restart; the tool entry looks up the current sandbox when it starts. A reload seals and drains the +old sandbox for at most 30 s. The on/off switch is a setting.

+

Logs and notices

+

Log capture tees stdout and stderr of the server, workers and detached hosts without changing their behaviour, +redacts known secrets and credential patterns, and keeps bounded rings (at most 128 sources, 1,000 lines or 256 KiB +each). Detached hosts publish redacted snapshots to a private file. Logs are read without waking a stopped worker. +Notices are the one channel from chapter XVIII; dismissal stores only notice ids.

+

Dev and reload

+

The dev plugin is chapter XV's reload control for p3: a source watcher marks plugins dirty and never applies anything; +the reload_plugins agent tool and a control in the owner's shell reload the changed plugins, or restart all hosts for +base code (through the supervisor, reopening the sessions that were running). UI bundles are published only after a +successful backend change; a web page detects a new manifest and reloads with its drafts and navigation state saved; +the TUI swaps its sandboxed UI code with a snapshot of its state. A local Unix socket (0700) accepts development +commands without a network admin port.

+

The shells

+

The web and TUI shells are UI facets over the objects above. Two rules matter for Chord:

+
    +
  • UI state survives navigation, reload and reconnect on purpose. Composer drafts, pending sends with their request +ids, reading positions and review drafts are kept in tab storage or the TUI's replacement snapshot. A reconnect +re-holds and gets snapshots; nothing is lost that was not in flight.
  • +
  • Leaving is never derived from an unmount. A plugin reload remounts components (chapter XV). Cancelling a login, +closing an overlay or giving up a lease is an explicit action or a dropped reference.
  • +
+

The TUI launcher keeps downloaded UI code in a permission-dropped worker with no access to files, network or +credentials; local side effects (copy, upload, download, open a URL, an external editor) need a real user gesture or a +confirmation. That is app code around Chord, not part of it.

+

Transport

+

The last domain, transport, is chapter XVII: one message format, per-part limits (16 MiB compressed and 64 MiB decoded +on client WebSockets, instead of p2's 1 MiB input cap and 32 pending calls), one byte window per link, re-holds with +snapshots after a reconnect, and a protocol version that makes an outdated web page reload.

+
Built so far

None of p3 exists. pi-ai's login API, pi-codemode and the pi-env daemon exist and are used by p2 today; the HTTP +component and channels are deferred until after the kernel's first real proof (chapter XIII).

+
+
+ +
XXI

Other Ways to Do It

Cordis and DeepSeek Harness attack the same problem, plugins that come and go while an agent runs, inside one process. What that buys, and what it costs.

+

Chord v2 is not the only answer to "replace plugin code while the app keeps running". The closest relative is +Cordis, a TypeScript plugin framework by Shigma that has been the plugin kernel of the Koishi chatbot framework +since 2019, and its most visible user, DeepSeek Harness (dsh), DeepSeek's open-source agent harness, whose +motto is "everything is a plugin". This chapter compares them with Chord v2 on the questions the guide cared about: +who owns side effects, how plugins depend on each other, what a reload keeps, what a crash keeps, and how far the model +reaches beyond one process.

+
Sources and status

DeepSeek Harness at commit 687f282 (release 0.2.1-rc.2), which vendors Cordis as @deepseek-ai/cordis 4.0.5 (Cordis +RC7 with local changes, vendor/README.md): its Cordis tutorial (docs/cordis-tutorial/), docs/architecture.md, +docs/subsystems/persistence.md, docs/subsystems/core.md and packages/boot/hmr/README.md; the Cordis paper "A +Programming Paradigm for Spatiotemporal Composability" (arXiv 2608.25512). Cordis and dsh are built and in use; Chord v2 +is the design this guide describes. The comparison is between a working system and a design.

+
+

Cordis in one example

+

A Cordis plugin is a function (or an object with apply, or a Service subclass) that receives a context. It may +declare the services it needs with inject, and everything it does to the world goes through the context:

+
export const name = "greeter";
+export const inject = ["tools"];                       // activate only while a `tools` service exists
+
+export function apply(ctx: Context) {
+  ctx.tools.register(defineTool({ name: "greet", /* … */ }));   // a registration is an effect: undone on unload
+  ctx.on("session/start", (s) => log(s.id));                     // an effect-bound listener
+  ctx.effect(() => {                                              // any other side effect, with its disposer
+    const timer = setInterval(work, 200);
+    return () => clearInterval(timer);
+  });
+}
+

Each plugin instance is a fiber with states PENDING → LOADING → ACTIVE → UNLOADING → DISPOSED (or FAILED). +Every effect, registration, listener, child plugin and provided service is recorded on the fiber, and unloading runs +the disposers. The paper calls this temporal composability: a component can be removed at any time and reverses its +own effects. Its counterpart, spatial composability, is the dependency side: inject is live, so a plugin starts +when its services appear, and unloads (and later reloads) when a service it depends on goes away or is replaced. The +paper's claim is that independently written components can therefore be combined and replaced without disturbing each +other. The next sections check how far that holds.

+

Services are named capabilities on the context (ctx.provide("x", value) or class X extends Service). Names are +runtime strings in one flat namespace per application; TypeScript declaration merging makes ctx.x typecheck, but +nothing checks the service's shape at runtime, and optional dependencies are probed with ctx.get("x"). Scopes come +from the context tree: ctx.extend() derives a child context, ctx.isolate(name) gives one subtree its own binding of +a service, so two parts of an app can use two different shell providers.

+

Hot reload follows from the two halves: the reload plugin unloads the old fiber (unwinding all its effects) and +loads the new module with the same config. Nothing held in the old instance's closures or fields survives; a plugin +that needs state across reloads must keep it in a service or on disk. Config edits are diffed by entry id and +mount, unmount or reconfigure only what changed; config fields declared volatile update in place without a remount.

+

DeepSeek Harness on Cordis

+

dsh applies "everything is a plugin" to an agent: the model adapters (ctx.llm), the tool registry (ctx.tools), the +session log (ctx.sessions), the agents (ctx.agents) and the agent loop itself (ctx.agentLoop) are plugins and +services. A running dsh is a plugin tree composed at boot from a profile (web, headless, sdk, acp) that +stacks bundles of Cordis config rows, with user patches on top. There is no privileged core: replacing the agent +loop is a config patch.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConcernHow dsh does it
sessionsa durable, append-only event log per session (JSONL backend, single writer, flush as the durability barrier); model history is derived from the events
crash mid-turnthe log keeps everything appended; on resume the agent loop closes the open turn by appending missing tool errors and a synthetic turn/end { reason: "interrupted" }
toolsctx.tools.register(defineTool(…)), an effect of the registering plugin; calls go through pre/execute/post policy waterfalls
approvalclosed by default; allowed-once is the only grant; plugin installation needs full access or per-call approval
reloada source watcher evicts changed modules; Loader entries running affected plugins stop and start again with the new code; a failed replacement is an activation failure, its previous code no longer runs
processesone plugin tree in the Host process; the browser client and Electron's renderer talk to it over RPC streams
distributiona Plugin Manager installs and enables bundles per profile; plugin markets are optional indexes
subagentslocal subagents with their own durable sessions, and external ones
+

What the guarantees are in practice

+

Reading the Cordis source vendored in dsh and dsh's own documentation, the guarantees are narrower than the paper's +framing. An earlier review for p2 came to the same result (apps/p2/docs/p3-reviews/dsh-compare.md).

+
    +
  • Reversal is cooperative and unordered. _unload runs all of a fiber's top-level disposers concurrently +(Promise.all in vendor/cordis/src/fiber.ts); order holds only inside one effect, and the primer tells authors to +"keep the related work in one effect" when teardown order matters. A disposer that throws is logged and skipped. A +disposer that never settles stalls the teardown; dsh's jobs service works around this by force-failing a job whose +cancel throws and warning that "the work may be orphaned" (packages/jobs/jobs-local/README.md). Reversal is as +good as every disposer, and nothing checks that a plugin registered all its effects.
  • +
  • Behaviour depends on registration order, and a reload changes it. emit, waterfall, serial and bail +listeners run in registration order (docs/cordis-primer.md). Registering appends the listener (push in +vendor/cordis/src/events.ts, unless prepend), so when a plugin reloads, its listeners move to the end of the +list. A tool-policy listener that ran first runs last after its own plugin reloads; a waterfall that short-circuits +now sees a different decision first.
  • +
  • Dependencies are strings, and every provider reload restarts its dependents. A fiber's activation key is the +concatenation of its injected providers' fiber ids (_refresh in fiber.ts), so replacing a provider unloads and +reloads every plugin that injects it, with their in-memory state. Service APIs are "pre-stable; update every +consumer" (AGENTS.md): there is no versioned contract between a provider and its consumers.
  • +
  • Reloading a core service ends live work. Unloading the agent provider "stops and drains every live handle it +made" (packages/core/agent/README.md), so running agents stop; reloading the session persistence backend under +live sessions invalidates their write handles, and "writes fail loudly until the sessions restart" +(.agents/notes/implemented/architecture/2026-08-27-handle-based-session-persistence.md).
  • +
  • Registrations leave a gap. A reloading plugin's tools and other registry entries are removed with its old fiber +and added again by the new one; anything that looks them up in between does not find them.
  • +
+

None of this is a bug in the narrow sense: each point is documented, and dsh is built around it. But "components +interleave without disturbing each other" holds only for plugins that own nothing other plugins depend on, register +every effect, order nothing by registration, and hold no state that must outlive a reload.

+

Side by side

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Cordis and dshChord v2
unita plugin instance (fiber) in one context treean object per (type, input, principal), placed on a host (chapters II, III)
side effectsrecorded on the fiber with ctx.effect and friends; on unload the disposers run concurrently, each as good as its author made itdeclared through self (documents, natives, routines, jobs, contributions); Chord activates and undoes them (chapters V, IX to XI)
dependenciesservice names as strings in one flat namespace, via inject; a provider reload restarts every dependenttyped contracts via get(Type, input); a reference parks and recovers through restarts (chapter VI); optional dependencies are registries (chapter XIII)
orderinglisteners run in registration order, which changes when a plugin reloadsregistries compose with a declared compose function; nothing depends on load order (chapter XIII)
where code runsone process; remote UIs over RPC to the Hosta tree of hosts (server, workers, tabs); objects reachable across links, relayed by the server (chapters I, XVII)
who is callingnot a framework conceptprincipals stamped by links, variants per caller, revocation (chapters III, VII)
statethe plugin's own business; dsh's session log is a servicedocuments written in transactions, persisted, versioned and migrated (chapter V)
reloadunload the old instance, load the new one: in-memory state and running work of that plugin endpure objects swap their code in place with their documents and running work kept; only objects with natives restart (chapter XV)
a running agent turn during reloadentries of changed plugins stop; unloading the agent provider stops its live agentsthe harness is base code; Durable pins tools per task phase, so a running call finishes on old code and the next phase uses the new (chapter XIV)
crash recovery of a turnthe turn is closed as interruptedpi-durable continues the turn's tasks from their checkpoints; replay-safe tools rerun (chapter XIV)
contracts and versionsTypeScript types only, pre-stable APIs, every consumer updated together; config validated with a schemacontract versions with fingerprints and provider-side mappings for old consumers (chapter XVI)
statusin production in Koishi since 2019; dsh releaseda design with the contract layer built
+

What the comparison says

+

The core idea is the same. "Factories declare, Chord activates" (chapter II) and Cordis' reversible effects are +two forms of one rule: a plugin may only change the world through its runtime, so the runtime knows what to undo. +Both reject the module-level timer and the stored callback for the same reason.

+

The difference is what survives a reload. Cordis reloads at plugin granularity: the old instance unwinds and the +new one starts from nothing, together with every plugin that depends on it. That is simple, but a plugin's in-memory +state and its dependents' state and work in flight end with it, and correctness rests on every disposer and on +registration order. Chord v2 keeps state out of closures on purpose (documents, references, jobs as +records) so that it can swap the code of a pure object while its data and running work stay, and restarts only what +holds OS handles. That is more machinery: transactions, a store, versions on every document and contract.

+

The other difference is reach. Cordis composes plugins inside one process, and dsh adds RPC to its own clients. +Chord v2 makes the process boundary part of the model: objects live on the host their placement names, callers are +principals, and a reload rolls out across the server, the workers and every tab. An app like p2, with a process per +session and guests in other browsers, needs that; a single-process agent does not.

+
Review: what a reload does to a running tool call in dsh

The sources say that Loader entries running affected plugins stop and that unloading the agent provider stops its live +agents (packages/boot/hmr/README.md, docs/subsystems/core.md). They do not say what happens to a tool call in +flight when only the tool's plugin reloads; the comparison above does not claim either way.

+
+
+ +
∴

What's Next

Where Chord v2 stands, what is still undecided, and where the written sources disagree with each other.

+

The guide built Studio from one object to a p2-sized app. Every piece used the same few rules: one owner per thing, +factories that only declare, state that changes only in transactions, long-lived behaviour in declared forms, and +mappings and migrations owned by the provider. Those rules are what let Mario press Reload on his phone in the +preface while the agent keeps running. This last chapter says how much of that exists and what is still open.

+

Status

+
~10klines in the frozen first kernel (steps 1 to 6, 443 tests), kept as a reference only
1 of 11layers of the rebuild started: L1 contracts, built and reviewed
6–7ktarget size of Chord v2 without tests
+
    +
  • The design is decided except the points marked below. packages/chord/docs/next/design.md is the target; +packages/chord/docs/next/decisions.md is the log and wins where they differ.
  • +
  • The rebuild goes layer by layer, each with a one-page spec the owner approves, a minimal build with counting and +garbage-collection tests, the cost-guard metrics, and a cleanup loop: L1 data and contracts; L2 readable and +derive; L3 documents and store; L4 objects; L5 references and ctx; L6 protocol and links; L7 relay, routes, +revocation; L8 routines and jobs; L9 registries; L10 reload and rollout; L11 telemetry in Chord.
  • +
  • Built: the contract layer (src/next/contracts/, commit 7d2bb8ea8, chapter XVI). In pi: Delta, +pi-telemetry, chord/context, chord/telemetry-sqlite, pi-ai and Durable telemetry, Durable's required +checkpointWhen.
  • +
  • Next, in order (apps/p2/docs/p3-handoff.md): decide Q12 and Q15, which the L1 spec needs; write the spec for +the rest of L1 (values, inputs and identity, argument checks, principals, errors; draft in +packages/chord/docs/next/layer-1.md); then Q11, Q13, Q14 and the cuts C3 to C17 before the layers they affect.
  • +
  • In pi: mapShared and bytes as atomic leaves in Delta's diffRevisions.
  • +
  • Then the playground web shell (React, esbuild) with the reload control and a preview at p3.mariozechner.at, +and the first real proof: the harness, one tool with an approval pause, a code mode reload. Then the HTTP and +channel components, then p3, domain by domain.
  • +
+

Deferred decisions

+

Kept with stable numbers in design.md §26, each waiting for a concrete case:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#TopicWhenChapter
9jobs as objects: a record inside the owner, or an object addressed by its idwhen login is portedXI, XIX
11where a flow lives: a keyed document of its owner, or an object LoginFlow { id }when login is portedXII
14project plugins: per-directory plugins with trust prompts and snapshotsown pass over plugin-loading.mdXV
16a Durable Object profilewhen a DO target becomes concrete
17session files: one SessionFiles { session } owner on the server (proposed)p3 designXX
18session delete: retire by tag plus a session directory, or owner hooksp3 designXIX
20connection pools: one owner of SSH connections per environment, or per-use connectionsp3 designXX
21config files with a single writer: one owner on the server, others followp3 designXVIII
22file formats (envs.json, downloads, uploads) as versioned interfacesp3 designXX
23the terminal host process: inside the revision or deliberately outsidep3 designXX
+

Open from review round 3

+

Decisions (apps/p2/docs/p3-reviews/v2-review3-summary.md; Q1 to Q10 are decided):

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#ProblemProposal
Q11a singleton that does not exist yet: undefined or initial()?singletons read initial(); undefined only for absent keyed documents
Q12stripping arguments (R7) against "nothing is copied"copy on change: a new object only when the argument had extra fields; never mutate the caller's value (proposed in the L1 draft)
Q13ctx.incarnation on relayed callsrelays stamp the origin host's incarnation, as they stamp the principal
Q14a writable holding functions exposed as a state memberonly members with a JSON schema in the contract are exposable, checked at serve
Q15arguments of local methods: validated or notapply R7 to local too, or keep local as the one exception
+

Cuts the owner may choose (each removes a primitive this guide teaches):

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#CutWhat it costs
C3shrink jobs to a runner over documents (no versions, accepts, blocked, persist), or move jobs out of the kerneljob versioning, or self.job for core-only users
C4routines: on and loop onlyevery's due time across reloads
C5merge lifetime: "host" objects into the faceta plugin's always-on natives share one failure domain
C6remove broken() for opens and retryOnnothing; reload and Retry still work
C7remove provider-side lingerten lines of app code
C8remove state: false (method-only references)a method-only consumer pays one snapshot
C9remove previous from derivesderives that used it move to mapShared
C10one cell(initial) for constant and writablenothing
C11remove ctx.trace (use traceIn(ctx, …))nothing
C12remove call credits (the byte window covers calls)nothing
C13two timeouts (closeTimeout, readyTimeout) instead of eightper-thing overrides
C14remove the handle.change and handle.retire shorthandsnothing
C15remove inVariant and expectVariant (keep RefIn)nothing
C16remove facet-owned mutable documents and their transaction exceptionatomic writes across an object and its facet move to one owner
C17no harness-specific start-barrier exceptionfactories before the barrier cannot depend on the harness
+

Where the sources disagree

+

This guide follows the decision log where it is newer than the design, and the design where it is newer than +everything else. The places that need a fix in the sources:

+
Review: drift in the written sources
    +
  • design.md §26 lists only the deferred decisions; it omits the open review-3 items Q11 to Q15 and the cuts C3 to +C17.
  • +
  • design.md §3 has a garbled sentence, "Values are method arguments are stripped".
  • +
  • apps/p2/docs/p3-handoff.md §3 is stale: it says all R items are decided and, two lines later, that R2 to R5 and +R7 to R26 are open, and "resume R6 next".
  • +
  • kernel.md and step-*.md describe the frozen code: host values, a development freeze, consumer-side linger, +JSONL storage, descriptions on the wire, and a link closed on an unknown id. None of these is the target.
  • +
  • chord-next-walkthrough.md lags the log: validation by trust (now R7, every caller), dev mode and freeze (removed +by R9), descriptions once per link (gone with R6), an IPC frame-count window (now one byte window, Q4), job outcome +{ ok } (now { kind }), harness activation as the start barrier (R17), registries "newer refused at load" (Q8), +and hooks using host.root (now chord(fn, context), Q6).
  • +
  • apps/p2/docs/questions.md uses self.resource and replace (both rejected) and host.root in hooks.
  • +
  • apps/p2/docs/p3-reviews/v2-cost-model.md: the measurements are current, its "Decisions this needs" section is +not (chapter XVII).
  • +
  • L1's Implementation<C> shape differs from the factory return value used in the design (chapter XVI).
  • +
+
+
+ +
A

API Reference

Every name the guide uses, grouped by where you meet it. Working names; only the contract layer is built.

+

Hosts

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CallMeaningChapter
createHost<P, R>({ address?, principal?, storage?, principalKey?, revokedBy?, telemetry?, onError?, timeouts })a host; with an address it is addressable and its own code acts as principal; without, a clientI
host.load([{ name, facet }])load facets by plugin name; base code passes its core facet the same wayI
host.connect(link, { principal })the link to the parent (at most one)I
host.accept(link, { principal })a link from a client; every request on it acts as principalI
host.route(Kind, { authorize, resolve })how to reach hosts of a kind; resolve returns { link, principal }I
host.revoke(r)end matching objects and links here and belowVII
host.root(fn, { principal?, signal?, telemetry? })a call scope with a ctx for code outside objectsVI
host.get(Type, input, { signal, state?, parkTimeout? }), host.hold(…)a reference from outside objects; release() ends itVI
host.serve(Type, factory, { lifetime?, linger? })serve an object type (in a facet)II
host.registry(Registry)the composed value of a registry, a readableXIII
host.references(Type)who holds references to objects of a type on this host (presence)VIII
host.activity{ references, calls, jobs, lingering }, for keep-aliveI
host.close()end the host: objects, natives, linksI
+

Inside a factory: self

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MemberMeaningChapter
self.state(Definition)a document handleV
self.tx(async (tx) => …, ctx)a transaction on the object's lineV
self.derive(name, [sources], fn)a derived readableVIII
self.get(Type, input)a reference held for the object's lifeVI
self.open(open, close, { closeTimeout? })a native; .value once openIX
self.routines.on(readable, fn), .every(interval, fn), .loop(fn), .custom(…)standing behaviourX
self.job(Job, { run, finish? })owned work that outlives its callXI
self.contribute(Registry, value)a contribution (facets and lifetime: "host" objects only)XIII
self.writable(initial)a host-local readable of any in-process value, with setXIII
self.retryOn(readable)a trigger that retries a broken activationVI
+

Document handles: value, get(key) (keyed), keys, change(fn, ctx) / change(key, fn, ctx) and retire(ctx) / retire(key, ctx) (shorthands for +one-document transactions). Transactions: tx.draft(handle, key?, { tags? }), tx.retire(handle, key?), +tx.start(job, args, { key? }). Jobs: job.start(args, ctx, { key? }), job.abort(id), job.running, +run.outcome(ctx).

+

References and ctx

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MemberMeaning
ref.<member>a state member (readable: .value, subscribe) or a method
ref.state, ref.onState(fn)loading, live, failed (cause, retryAt), released
ref.retry()retry a failed reference now
ref.release()end a reference taken outside objects
ref.membersthe variant's members
ctx.signal, ctx.principal, ctx.incarnation, ctx.getcancellation, caller, calling host's incarnation, call-scoped references
ctx.telemetry, ctx.trace(name, attributes, fn)the current span; a child span with a child ctx
ctx.jobId, ctx.progress(value)in a job's run
+

Definitions

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameWhat it declaresChapter
hostKind(name, input?)a kind of host and its addressI
remote(name, { version, input, placement, variants })an object type reachable across linksII
local(name, { version, input, members })an object type on its own hostIV
method(args?, result?)a method in a base declarationII
evolve(prev, (e) => …), checkAccepts(contract, accepts)the next contract version; the provider's mappings (built)XVI
Implementation<typeof C>the type of a provider for the current version (built)XVI
str num int bool nul lit oneOf union obj arr map opt bytes unknown, Staticschema helpers (built)II
defineState(name, { schema, version, initial, checkpointWhen, persist?, keyed?, ordered?, migrate? })a documentV
defineTag<T>(name)a tag; store.retireByTag(tag, value)V
defineJob(name, { version, args, progress, result, concurrency?, persist?, accepts? })a job typeXI
defineRegistry(name, { version, compose, accepts? })an extension pointXIII
bindFacet<P>()base code's defineFacet, with factories' principals typed as the app's P (working name)III
defineFacet((host, self) => …)a plugin's setup for one kind of host; imported from base code (@studio/core)II, III
constant(value)a readable that never changes (start data)IX
broken(error)a failure that is not retried by the backoffVI
inVariant(ref, …), expectVariant(ref, …), RefIn<…>narrowing a reference by variantIII
mapShared(fn)an array map that keeps unchanged items identical (planned in Delta)VIII
OutputBuffer, Progressbounded output and throttled progress (@earendil-works/chord/output)XI
memoryLinkPair(), ipcLink(child), webSocketLink(ws)link adaptersI
DocumentStorage, registerDocumentStorageConformancethe storage port and its shared test suiteV
+
+ +
B

Timeouts and Defaults

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Timeout or defaultBoundsDefaultOverride
closeTimeouta native's close, then reported as a possible leak30 sper native
call drainan ending object's own call handlers= closeTimeoutper type
readyTimeoutactivation; an awaited get60 sper get
park deadlineparked calls (link drop, loading, rollout)60 sper get (parkTimeout); per type
routine handovera new routine waiting for the old handler on reload10 sper routine
finish retrybackoff of a throwing job finish1 s to 60 sper job
rollout installone host installing a revision60 snone
lingeran object kept alive after its last referenceper typeper type
backoffretries of transient failures, per address1 s doubling to 60 snone
evictionkeyed persisted documents idle in memory10 minutes
client WebSocket limitsper part16 MiB compressed, 64 MiB decodedper link kind
compressionJSON parts compressed at their originabove about 16 KiB
worker keep-alive (p2, app code)idle time before the supervisor stops a worker5 minutesapp
+
+ +
C

Versions

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
SurfaceVersionOn changeChapter
object typeversion plus fingerprintevolve and checkAccepts; otherwise contract_mismatchXVI
documentversion plus schema fingerprintpure migrate on access; no bump fails brokenV
jobversionaccepts on outcomes and running; blocked withoutXI
registryversionaccepts on contributions; unmatched ones left out and reportedXIII
nativenonerestarts on reloadIX
Durable tasks and documentsDurable's versionDurable's migrateXIV
link protocolprotocol versionrefused; hosts update togetherXVII
UI components, Chord and app APIsnone yetexposed as they are
+
+ +
D

Errors and States

+

ChordError has a stable code and a finer reason; on the wire it is { code, reason?, message, details? } +(specified in layer-1.md, not built).

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CodeMeaning
invalidinput, arguments or placement rejected (invalid/input, invalid/args, invalid/placement)
contract_mismatchthe consumer's version and fingerprint are neither current nor accepted
forbiddenthe caller may not do this
gonethe object or hold ended
not_readynot activated in time
releasedthe reference was released
too_largea part exceeded its size limit
unavailablecould not be reached before the deadline (unavailable/locked and others)
abortedcancelled by a signal
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TypeValues
FailureCause{ kind: "transient"; error } (retried with backoff) or { kind: "broken"; error } (retried only by a trigger)
reference stateloading, live, failed (cause, retryAt), released
job outcome{ kind: "done"; result } or { kind: "failed"; error: "threw" | "aborted" | "lost" }
job phaserunning, finishing, blocked
rollout status of a providerupdating (calls park), rejected (fail at once), same revision (contract_mismatch)
host install reportinstalled, installed_failed, rejected
+
+ +
E

Rejected Designs

+

From design.md §31, grouped. Each was considered and is not coming back without a new reason.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
AreaRejected
reloaddrain-based handover; object restart for state changes; a factory trial; a migrate dry run; a consumer contract check during staging; reload votes; per-plugin rollback within a host; keeping natives across reloads by name; restart_required; host values and a development freeze
stateversion fences without transactions; volatile documents; per-group lines; optimistic transactions; tx.get; a JSONL document storage; validating state and documents on commits or reads; copying values on same-host calls; op-rewriting projections and a view algebra; Durable's Session as the object store
objects and referencesresources replaced in place (replace, deps, stale handles); revoking proxies on .value; consumer-side linger; detached references counting toward presence; an optional flag on references; opaque members; callbacks as method arguments; generic call or request ids; a stream type in Chord
registriesregistries created at runtime; entries resolved through mutable indirection; generic registry keys with a conflict flag; numeric middleware priorities; registry composition as a staging check
principals and placementprincipal field patterns and re-stamping for revocation; storeKey; placement as data naming an input field; host-name strings in app code; "magic" input-field matching for delete
wirean outbox for transient delivery errors; cross-relay reset coordination; count limits that refuse calls or holds; contracts taken from the running app
process and toolinga topology layer above the host API; loader variants A and B; Chord ending the process on a storage failure; defaultProjectTrust: "always"; ceremony (native/ directories, defineNative, lints); weakening type checking
+
+ +
F

Glossary

+

Adapted binding: a consumer bound at an older accepted contract version, served through the provider's mappings. +Base code: the app's own code, imported statically; a change restarts the hosts. Client: a tab or TUI host, +not addressable. Contract: an object type's declaration (name, input, placement, members, version). +Contribution: a value a plugin adds to a registry. ctx: what every call receives. Derive: a value +computed from readables. Document: a named JSON value an object owns. Facet: a plugin's code for one kind of +host. Factory: the synchronous function that declares an object. Fingerprint: a hash of a contract's or +schema's normal form. Flow: input into running work, kept as state by its owner. Host: one participant of the +app, with an identity and an incarnation. Incarnation: the id a host gets on every start. Input: an object's +key among objects of its type; with the type and the caller's principal key it identifies the object. Job: owned work that outlives its call. Link: an ordered connection between +two hosts. Local type: an object type reachable only on its own host, without principals. Native: an OS-level +value opened with self.open. Park: a call waiting for a reference to become live again. Placement: the +function from input to host address. Plugin: a directory with a package.json, reloadable. Principal: who +calls, stamped by the link. Principal key: the string that identifies a principal in object identities. Pure +object: an object without natives; swapped in place on reload. Readable: a value plus its ordered changes. +Reference: what get returns. Registry: an owner's extension point. Relay: a host forwarding messages +between two links, rewriting only headers. Reload: swapping a plugin's code in running hosts. Restart: ending a +process. Revision: the complete code set the hosts should run. Revocation: ending live access for principals +matched by revokedBy. Routine: standing behaviour of an object. Server: the addressable root host. Shared +store: a local object that holds data once behind per-principal views. Tag: a typed label on documents, used to +retire them together. Transaction: an atomic set of document writes on the object's line. Variant: the set of +members a principal gets. View: a per-principal remote object in front of a shared store. Worker: a host the +server starts for one session.

+
+ +
G

Sources

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
SourceUsed forWeight
packages/chord/docs/next/decisions.mdevery decision, review rounds 2 and 3wins where newer
packages/chord/docs/next/design.mdthe target design, all chaptersauthoritative otherwise
packages/chord/src/next/contracts/ and its README.mdchapter XVIbuilt code
packages/chord/docs/next/layer-1.mderrors, inputs, argument checksdraft spec
apps/p2/docs/p3-reviews/v2-review3-summary.mdopen Q11 to Q15, cuts C3 to C17current
apps/p2/docs/p3-reviews/v2-cost-model.mdmeasurements, per-hop costscurrent except its last section
apps/p2/docs/p3-functions/chapters XVIII to XXcurrent
apps/p2/docs/p3-handoff.mdstatus and order of work§3 partly stale
packages/durable/README.mdthe harness, submit, tasks, subagentscurrent
chord-next-walkthrough.md, kernel.md, step-*.md, apps/p2/docs/questions.mdbackground onlylag the decisions (see What's Next)
DeepSeek Harness at 687f282 (its Cordis tutorial, architecture, persistence, core and HMR docs), the Cordis paper (arXiv 2608.25512)"Other Ways to Do It"external, built systems
+
+ +

Chord v2 · a guide · first edition · 2026-10-20 · based on design.md (target design v2), redesign-decisions.md (review rounds 2 and 3), chord-next-walkthrough.md, layer-1.md, contracts/README.md, the p3 functionality map, questions.md and the pi-durable README

+
+ + diff --git a/packages/chord/docs/next/explainer/src/00-front.md b/packages/chord/docs/next/explainer/src/00-front.md new file mode 100644 index 000000000..36d62e8d0 --- /dev/null +++ b/packages/chord/docs/next/explainer/src/00-front.md @@ -0,0 +1,111 @@ +--- +title: Chord v2: a guide to building apps that reload +cover: Chord v2 / A Guide +kicker: A guide · Chord v2 target design · 2026-10-20 +subtitle: How to build an app like p2 from nothing on Chord v2 and pi-durable, one object at a time, so that new plugin code takes over while the agent keeps running +stats: + - 11 | layers in the rebuild, L1 to L11 + - ~1,300 | lines built so far (L1 contracts) + - 6–7k | target size of Chord v2, without tests + - 0.004 ms | Delta's apply per small change +note: | + This guide teaches the **designed** API of Chord v2 (`packages/chord/src/next`, export `@earendil-works/chord/next`) + from `packages/chord/docs/next/design.md` and the newer decision log `packages/chord/docs/next/decisions.md`, which wins + where the two differ. Most of it is **not built yet**: a first kernel of about 10k lines is frozen as a reference and + Chord v2 is being rebuilt layer by layer; only the L1 contract layer exists (`src/next/contracts/`, commit + `7d2bb8ea8`). Code in this guide is therefore the target, not something you can run today. Every chapter says what is + built. API names are working names. Notes marked *review* are points that are undecided, or where the sources + disagree; they name the files. The reader is assumed to know pi-durable; Chord is explained from the start. +colophon: Chord v2 · a guide · first edition · 2026-10-20 · based on design.md (target design v2), redesign-decisions.md (review rounds 2 and 3), chord-next-walkthrough.md, layer-1.md, contracts/README.md, the p3 functionality map, questions.md and the pi-durable README +--- + +# Before we begin {.preface} + +It is late in the evening. Mario is on his phone, connected to his laptop at home through the Radius relay. In session +S1 the agent is twenty minutes into a refactoring; right now it is running a test suite through `bash`. A colleague, +Ann, follows the session read-only from her own browser. Mario notices that the notes panel sorts notes the wrong way +round. He fixes one line in the notes plugin on the laptop and presses **Reload**. + +What should happen is easy to say. The notes panel on both screens shows the new order. The test run keeps running. The +agent's turn continues as if nothing happened. Ann's view does not flicker into an error. Nothing that was written is +lost, and no process restarts. + +The obvious ways to get there fail. **Restarting the processes** loses the turn in flight, the open terminal and every +connection; pi-durable would resume the turn, but the test run and the connections are gone. **Hot-swapping modules** +in place, as front-end tooling does, leaves the old code alive in closures, timers, event listeners and sockets that +nobody knows about: the old sort function keeps running inside a callback someone stored, a file watcher opened by the +old code is never closed, and two versions of the plugin now write the same data without any order. + +Chord v2 is the runtime that makes the first outcome normal. It does so by being strict about one thing: **every piece +of state, every open OS handle and every piece of running work has exactly one owner, an object, and objects only +declare what they need.** Chord itself activates, connects, reloads and ends them. Because Chord knows every timer, +watcher, transaction and reference an object has, it can swap the object's code while its data stays, restart only +what touches the operating system, and route calls between browser tabs, the server and per-session worker processes +without the plugin author writing any of that. + +This guide teaches Chord v2 by building an app called **Studio**: a small version of p2 (Pi Mobile), with a server, +one worker process per session, browser tabs as clients, and an agent from pi-durable on each worker. The first +chapters grow one plugin, `@studio/notes`, until it uses every concept; the last three chapters assemble the rest of a +p2-like app from the same parts. + +:::review | Review: what exists today +Nothing in this guide runs yet except the contract layer. `packages/chord/src/next/contracts/` (L1 part 1: `remote`, +`local`, `method`, `evolve`, `checkAccepts`, the schema helpers) is built and reviewed. The rest of L1 (inputs, +argument checks, principals, errors) is a draft spec (`packages/chord/docs/next/layer-1.md`). The frozen kernel in +`src/next/*.ts` implements an older version of the design (host values, a development freeze, JSONL storage, +descriptions on the wire); do not read it as the target. Build on the design, not on the frozen code. +::: + +## Ten things to know + +:::findings +1. **An app is a tree of hosts.** One server, any number of workers it starts on demand, any number of clients (tabs, + TUIs) that connect to it. Clients reach workers only through the server, which relays. +2. **Objects are the unit of everything.** An object type is a contract (name, input, placement, members, version); an + object is one instance per input and caller. The first `get` creates it, later gets share it, the last release ends + it. +3. **Factories declare, Chord activates.** A factory is synchronous: it declares documents, references, derived + values, OS handles, routines and jobs through `self`, and returns the members. Chord opens, binds and starts them, + and undoes them in reverse. +4. **Links stamp who is calling.** A caller's data never says who it is; the link it arrived on does. Each caller gets + its own object and the variant of members it may use. +5. **Shared data lives once, behind views.** A `local` shared store holds the data; per-caller `remote` views expose + what each caller may see. Only stable identities persist documents. +6. **State changes only in transactions.** Documents are JSON values written in `self.tx`; commits are atomic for + every viewer and travel as small operations, never as whole values. +7. **Failures look like disconnects.** A reference survives link drops, restarts and failures: calls park or fail, + values stay readable, and Chord retries by itself. There are no request ids. +8. **Everything long-lived has a declared form.** Standing behaviour is a routine, work that outlives a call is a job, + input from a person is a flow, behaviour from other plugins comes through a registry. Nothing else may hold a timer + or a callback. +9. **Reload swaps pure objects in place and restarts native ones.** Objects without OS handles get the new code with + their data and running work intact; objects with OS handles close and reopen them. A broken change leaves the whole + host on its old code. +10. **pi-durable runs inside a core object.** The agent harness is a host-lifetime object on each worker; models, + environments and tools reach it through registries, and tools reach Chord through `chord(fn, context)`. +::: + +## Words used throughout + +A **host** is one participant of the app: a process, a browser tab or a Durable Object, identified by a logical +identity and an **incarnation** that changes on every start. A **link** connects two hosts. The **server** is the one +addressable root host; a **worker** is a host the server starts for one session; a **client** is a tab or TUI, not +addressable. A **plugin** is a directory with a `package.json`; its code for one kind of host is a **facet**. **Base +code** is the app's own code (Chord calls its facet the core facet); it restarts the hosts when it changes, while +plugins reload. An **object type** (or **contract**) describes objects; an **object** is one instance; its **input** is +its key among objects of that type, and its **placement** computes from the input on which host it lives. A **factory** builds an +object; **`self`** is what a factory declares through. A **principal** is who calls, stamped by the link; a +**variant** is the set of members a principal gets. A **reference** is what `get` returns. A **readable** is a value +plus its ordered changes. A **document** is a named JSON value an object owns, written in a **transaction**. A +**derive** is a value computed from readables. A **native** is an OS-level value (a socket, a watcher, a process) +opened with `self.open`; an object with natives is a **native object**, one without is a **pure object**. A +**routine** is standing behaviour, a **job** is owned work that outlives its call, a **flow** is input into running +work. A **registry** is an owner's extension point, filled with **contributions**. **`ctx`** is what every call +receives: cancellation, caller, call-scoped references. A **reload** swaps a plugin's code inside running hosts; a +**restart** ends a process; a **revision** names the complete code set the hosts run. + +:::note | Reading the code in this guide +Imports are shown once per file and then omitted. `obj`, `str`, `arr` and friends are Chord's schema helpers (they +return ordinary TypeBox schemas). Helper functions that belong to the app (`authenticate`, `watchDir`, `uuidv7`) are +named and not shown. Where the design gives no name, the guide says it uses a working name. +::: diff --git a/packages/chord/docs/next/explainer/src/01-shape.md b/packages/chord/docs/next/explainer/src/01-shape.md new file mode 100644 index 000000000..db18b5cfe --- /dev/null +++ b/packages/chord/docs/next/explainer/src/01-shape.md @@ -0,0 +1,246 @@ + +# The Shape of an App {#shape lede="Which processes a Chord app has, how they find each other, and the start code that connects them."} + +Before any plugin exists, Studio needs processes and connections between them. In Chord each participant is a host, +and the app's own start code creates every host, connects them and decides how workers start. Chord deliberately does +none of that itself: a laptop app forks child processes, a test runs everything in one process, a Cloudflare +deployment routes through the edge. This chapter writes Studio's start code: `server.ts`, `worker.ts`, the browser +entry, and a test setup. + +## Three roles + +A Node app on Chord has one shape: + +:::flow wide | The three roles. Clients connect to the one server; the server starts workers and relays between them. +row "Clients: not addressable" muted +box tab1 "browser tab" "owner, local" +box tab2 "browser tab" "guest Ann" +box tui "TUI" "owner" +row "Server: exactly one" accent +gap +box server "server" "`Server()`" accent +gap +row "Workers: one per session, on demand" blue +box w1 "worker" "`Session({ id: \"S1\" })`" blue +box w2 "worker" "`Session({ id: \"S2\" })`" blue +box w3 "worker" "`Session({ id: \"S3\" })`" dashed +arrow tab1 -> server "WebSocket" +arrow tab2 -> server "WebSocket" +arrow tui -> server "WebSocket" +arrow server -> w1 "IPC" +arrow server -> w2 "IPC" +note "A worker for S3 does not exist until something asks for an object that lives there." +::: + +| Role | Addressable | Who it is comes from | Lifecycle | +| --- | --- | --- | --- | +| client | no | the authentication of its link | connects by itself | +| server | yes, exactly one | its own declaration | runs | +| worker | yes, many, by a typed address | stamped by the server, its parent | started on demand by the app | + +The **server** holds what all sessions share (the session list, access, settings) and starts workers. A **worker** +runs one session: its agent, its files, its terminal connections. **Clients** are browser tabs and TUIs. Nobody can +send a request *to* a client, so a client can only serve objects that nobody outside it needs; chapter IV names them +`local` types. + +Things that look like hosts but are not: a code mode sandbox (a worker thread inside a worker, owned by an object), a +remote environment's daemon (it speaks its own protocol, owned by an object), the TUI's sandbox behind its launcher. +A detached process such as a terminal host could become another kind of child of the server. + +## Host kinds and addresses + +Every addressable host has a typed address. The app declares its **host kinds** once, in base code: + +```ts +// studio/src/hosts.ts +import { hostKind, obj, str } from "@earendil-works/chord/next"; + +export const Server = hostKind("server"); +export const Session = hostKind("session", obj({ id: str() })); + +Server(); // the address of the server +Session({ id: "S1" }); // the address of the worker of session S1 +``` + +There are no host-name strings anywhere in app code. Chord serializes addresses internally; your code only calls the +kinds. An object type names its host with a placement function from its input to an address (chapter II), so the +type checker knows that `Session({ id: session })` needs a `session` field. + +## Links + +A **link** is an ordered, two-way channel of messages between two hosts; it only delivers. Chord ships three: a memory +pair for tests, IPC for a Node child process forked with `serialization: "advanced"`, and WebSocket. All three carry +the same binary message format and the same flow control (chapter XVII). Recovery policy belongs to the link adapter: +IPC needs no heartbeat, a WebSocket through a relay needs one, because TCP does not notice that a laptop went to sleep. + +Each link has a **direction**. A host has at most one link to its parent (`connect`) and any number of links from +below (`accept` for clients, `route` for workers). Requests arriving on a link act as the principal the link was given; +chapter III explains why that matters. + +## The server + +```ts +// studio/src/server.ts +import { createHost, ipcLink, webSocketLink } from "@earendil-works/chord/next"; +import { fork } from "node:child_process"; +import { WebSocketServer } from "ws"; +import { Server, Session } from "./hosts.ts"; +import { coreServerFacet } from "./core/server.ts"; +import type { P } from "./principal.ts"; + +const server = createHost

({ + address: Server(), + principal: { kind: "server" }, // who this host's own code acts as + storage: await openServerStorage(dataDir), // persisted documents (chapter V); memory if omitted + onError: (e) => { if (e.kind === "storage_failed") process.exit(75); else log(e); }, +}); + +await server.load([ + { name: "@studio/core", facet: coreServerFacet(config) }, // base code, under a fixed name + ...(await loadPlugins("server")), // [{ name, facet }] from the plugin loader +]); + +const wss = new WebSocketServer({ server: http, path: "/chord", maxPayload: 64 * 1024 * 1024 }); +wss.on("connection", (ws, req) => { + const principal = authenticate(req); // app code: cookie, loopback, device token + if (!principal) return ws.close(4401); + server.accept(webSocketLink(ws), { principal }); +}); + +server.route(Session, { + authorize: (p, { id }) => + p.kind === "owner" || p.kind === "server" || (p.kind === "guest" && p.session === id), + resolve: async ({ id }, { signal }) => { + const principal: P = { kind: "worker", session: id }; + const child = fork("worker.ts", [], { serialization: "advanced", signal }); + child.send({ id, principal, init: await initFor(id, signal) }); // the app's own start message + return { link: ipcLink(child), principal }; // Chord stamps `principal` on it + }, +}); +``` + +Each call has one job: + +| Call | Meaning | +| --- | --- | +| `createHost

({ address?, principal?, ...options })` | with an address: an addressable host whose own code acts as `principal`; without: a client. `P` is the app's principal type | +| `load([{ name, facet }])` | the facets this host runs, each with its plugin's `package.json` name; duplicates and invalid names are refused. Base code passes its core facet the same way | +| `accept(link, { principal })` | a link from a client; every request on it acts as `principal` | +| `route(Kind, { authorize, resolve })` | how to reach hosts of a kind. `authorize(principal, address)` runs before anything starts; `resolve(address, { signal })` returns `{ link, principal }` | +| `connect(link, { principal })` | the link to the parent, at most one, replaced after a reconnect; `principal` says who the parent is | +| `root(fn, { principal?, signal? })` | a call scope for code outside objects (a CLI command): gives `fn` a `ctx` (chapter VI) | + +Chord has no `listen`. Entry points differ too much: a loopback WebSocket server, a tunnel, the Radius relay +delivering channels, a Durable Object edge, a memory link in a test. Each entry point authenticates and calls +`accept`. Radius, for example, is one more line: + +```ts +radius.onChannel((ws) => + server.accept(webSocketLink(ws), { principal: { kind: "owner", access: "relay" } })); +``` + +## The worker + +```ts +// studio/src/worker.ts +const { id, principal, init } = await startMessage(process); // app code: the first IPC message +const worker = createHost

({ + address: Session({ id }), + principal, // { kind: "worker", session: id } + storage: await openSessionStorage(init.dataDir, id), // one storage per session +}); +await worker.load([ + { name: "@studio/core", facet: coreWorkerFacet(init) }, // serves the start data (chapter IX) + ...(await loadPlugins("worker")), +]); +worker.connect(ipcLink(process), { principal: { kind: "server" } }); +``` + +## A browser tab + +```ts +// studio/web/main.tsx +const client = createHost

(); // no address: a client +await client.load(await loadPlugins("web")); +client.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }); +const notes = await client.get(Notes, { session: "S1" }); // tab → server → worker S1 +``` + +## The first `get`, step by step + +The tab asks for the notes of S1. No worker for S1 runs yet. + +:::sequence animate | The first get for an object on a worker that is not running. The server authorizes, the app starts the worker, Chord stamps the link. +actor tab "browser tab" "owner" +actor server "server" accent +actor app "route(Session)" "app code" muted +actor worker "worker S1" blue +tab -> tab: placement: Session({ id: "S1" }) +tab -> server: hold Notes { session: "S1" } +server -> server: not mine: route(Session) +server -> app: authorize(owner, { id: "S1" }) +app --> server: true +server => app: resolve({ id: "S1" }) +app -> worker: fork + start message +worker -> worker: createHost, load, connect +app --> server: { link, principal: worker S1 } +server => worker: hold (stamped: owner) +worker -> worker: placement check, run factory +worker --> server: bound: snapshot +server --> tab: bound: snapshot +::: + +`resolve` runs once per address while its link lives. When the link drops (the worker crashed or the supervisor +stopped it) and references to that worker still exist, Chord calls `resolve` again by itself, with a per-address +backoff, so a tab that only watches recovers without a click. A worker start that already produced a link finishes +even if the request that triggered it went away. + +**Incarnations** make restarts visible. Chord mints a fresh id (a uuidv7) on every host start and sends it in the link +handshake; the other side sees a restart as a new incarnation of the same identity, and references to it re-hold +(chapter VI). + +## Tests: everything in one process + +The same code runs in a test with memory links. `resolve` builds the worker host in memory and returns one end of the +pair: + +```ts +server.route(Session, { + authorize: () => true, + resolve: async ({ id }) => { + const [up, down] = memoryLinkPair(); + const principal: P = { kind: "worker", session: id }; + const worker = createHost

({ address: Session({ id }), principal }); + await worker.load([{ name: "@studio/core", facet: coreWorkerFacet({ id, dataDir }) }, notesWorker]); + worker.connect(up, { principal: { kind: "server" } }); + return { link: down, principal }; + }, +}); +``` + +Memory links deliver the same encoded messages asynchronously, so a test runs the production protocol path. + +## What stays app code + +Starting processes, the start message, authentication, supervising and stopping idle workers are app code. Chord +provides one readable for the last one: `host.activity`, with `{ references, calls, jobs, lingering }` (viewers, calls +in flight, unsettled jobs, objects kept warm after their last reference). Studio's core reports it to the server +together with the agent's running tasks, and the server's supervisor stops a worker when everything has been zero for +five minutes, as p2 does. Chord counts; it has no idle policy. + +## Host options and timeouts + +`createHost` takes the options introduced later in this guide: `storage` (V), `principalKey` and `revokedBy` (III, +VII), `telemetry` (XVII) and `onError`. `onError` receives background errors and `{ kind: "storage_failed" }`; every +app must handle the latter, because Chord never ends the process. Studio exits with its own code so its supervisor +restarts it. + +Timeouts are host defaults with per-thing overrides; the full table is in the appendix. The two you meet first: +`readyTimeout` (60 s: how long an activation and an awaited `get` wait) and the park deadline (60 s: how long a call +waits through a link drop or a restart before failing `unavailable`). + +:::note | Built so far +Nothing in this chapter is built in Chord v2 yet: hosts, links and routes are layers L6 and L7. The frozen kernel has +working versions with the same shape (`createHost`, `connect`, `accept`, `route`, `load`), which the rebuild +simplifies. +::: diff --git a/packages/chord/docs/next/explainer/src/02-first.md b/packages/chord/docs/next/explainer/src/02-first.md new file mode 100644 index 000000000..5c01e5213 --- /dev/null +++ b/packages/chord/docs/next/explainer/src/02-first.md @@ -0,0 +1,187 @@ + +# A First Object {#first-object lede="A plugin, a contract, a factory, and a browser tab that calls it on another process."} + +Studio's first feature is session notes: a list of short notes per session, shown in a panel, that anyone allowed can +add to. Several places need "the notes of S1": the owner's tab, a guest's tab, later a download route and an agent +tool. If each built its own copy, the copies would drift apart and nobody would know when to clean them up. In Chord +there is **one object** for "the notes of S1", everyone asks for it by its type and input, and Chord routes them all to it. + +## The plugin's files + +A plugin is a directory with a `package.json`. Its `name` (npm style, globally unique and stable) is the plugin's +identity: Chord finds its stored documents, reports its errors and orders its contributions by it, never by a +directory name or load order. + +| File | Runs on | Holds | +| --- | --- | --- | +| `package.json` | read by the loader | `"name": "@studio/notes"` | +| `contract.ts` | every host that imports it | object types: names, shapes, method signatures; no running code | +| `worker.ts` | every session worker | the worker facet | +| `server.ts` | the server | the server facet (later: the download route) | +| `web.tsx` | browser tabs | the web facet: the panel | + +## The contract + +An object type is a contract: a name, an **input** (the object's key among objects of this type), a **placement** +(on which host it lives, computed from the input), the **members** (state you can read and methods you can call) and a **version**. + +```ts +// notes/contract.ts +import { arr, int, method, obj, remote, str, type Static } from "@earendil-works/chord/next"; +import { Session } from "@studio/core/hosts"; + +export const Note = obj({ id: str(), text: str({ max: 2000 }), by: str(), at: int() }); +export type Note = Static; + +export const Notes = remote("notes", { + version: 1, + input: obj({ session: str() }), // the key: { session: "S1" } + placement: ({ session }) => Session({ id: session }), // input → the host's address + variants: { + editor: { + state: { notes: obj({ items: arr(Note) }) }, + methods: { + add: method(obj({ text: str({ min: 1, max: 2000 }) }), str()), // returns the new id + remove: method(obj({ id: str() })), // no result + }, + }, + }, +}); +``` + +`remote` means objects of this type can be reached from any host, across links (chapter IV introduces `local`). The +schema helpers (`obj`, `str`, `int`, `arr`, `map`, `opt`, `lit`, `oneOf`, `union`, `bool`, `num`, `nul`, `bytes`, +`unknown`) return ordinary TypeBox schemas; short options such as `{ min, max }` bound lengths or values. Schemas must +describe data: no callbacks, no class instances, no custom codecs. + +`method()` has four forms: `method()` (no arguments, no result), `method(args)`, `method(args, result)` and +`method(undefined, result)`. + +A contract contains no code that runs, so every host can import it. When you declare it, Chord materializes its +description and computes a **fingerprint**, a hash of its input, variants and members. Two hosts compare contracts by +version and fingerprint (chapter XVI). Placement is not part of the fingerprint. + +## Placement + +The input says *which* object; `placement` says *where* it lives. The caller runs the placement function on the input and sends the resulting address with its request; the server +routes by it. The provider runs the function too and refuses an object that is not its own with `invalid/placement`. +So a worker for S2 never builds "Notes for S1", even if a caller has an outdated rule. Because placement is a typed +function, a `Session` placement on a type whose input has no `session` field does not compile. + +Inputs are **normalized** before anything else: defaults are applied, unknown fields removed, and a canonical key +computed. `{ session: "S1" }` and `{ session: "S1", extra: 1 }` are the same object. + +## The facet and the factory + +A **facet** is a plugin's setup for one kind of host. It runs once when the plugin loads on that host and tells the +host which object types it serves: + +```ts +// notes/worker.ts +import { defineState } from "@earendil-works/chord/next"; +import { defineFacet } from "@studio/core"; // Studio's defineFacet: principals typed as Studio's P (chapter III) +import { Note, Notes } from "./contract.ts"; + +const NoteList = defineState("notes.list", { // a document (chapter V); in memory for now + schema: obj({ items: arr(Note) }), + version: 1, + initial: () => ({ items: [] }), + checkpointWhen: () => true, +}); + +export default defineFacet((host, self) => { + host.serve(Notes, ({ session }, principal, self) => { + const list = self.state(NoteList); // THE note list of this object + return { + variant: "editor", + notes: list, // a document handle is a readable: expose it as state + add: ({ text }, ctx) => + self.tx(async (tx) => { + const id = uuidv7(); + (await tx.draft(list)).items.push({ id, text, by: "someone", at: Date.now() }); + return id; + }, ctx), + remove: ({ id }, ctx) => + list.change((d) => { d.items = d.items.filter((n) => n.id !== id); }, ctx), + }; + }); +}); +``` + +Read `host.serve(Notes, factory)` as "this host builds `Notes` objects like this". The factory receives the +normalized input, the caller's principal (chapter III) and `self`, through which it declares what the object needs. +It returns the members of one variant, tagged with `variant`. + +**Factories declare; Chord activates.** The factory is synchronous and has no side effects. It does not await, open +files or start timers. `self.state(NoteList)` does not load anything: it declares that the object has this document +and returns a **handle** that stays valid for the object's whole life. Chord then performs the activation: it loads the +document, binds references, opens OS handles, starts routines, and only then lets calls in. The rule exists for +reload: a reload runs the factory again, and anything the factory did itself would happen twice. + +## Calling it + +```tsx +// notes/web.tsx +export function NotesPanel({ session }: { session: string }) { + const notes = useRef(Notes, { session }); // app-side hook over host.get / release + const items = useReadable(notes?.notes)?.items ?? []; + return ( +

+ {items.map((n) =>

{n.text}

)} + notes?.add({ text })} /> +
+ ); +} +``` + +Underneath the hooks is the reference API: + +```ts +const notes = await client.get(Notes, { session: "S1" }); // live and hydrated once awaited +notes.notes.value.items; // the current list +notes.notes.subscribe((v) => render(v.items)); // every committed change +const id = await notes.add({ text: "check the flaky test" }); +notes.release(); // the object may end now +``` + +An awaited reference is **live and hydrated**: every state member has a value. State members are **readables**: +`.value` is the current value, `subscribe` is called after every commit. The first `get` for `{ session: "S1" }` runs +the factory; every later `get` for the same input and caller shares that object. The object ends when the last +reference is released (a type may keep it a little longer with `linger`). + +:::sequence animate | A call to add. Arguments are checked once at the provider; the commit reaches the viewer as one small change. +actor tab "browser tab" +actor server "server" muted +actor worker "worker S1" blue +actor obj "Notes { S1 }" accent +tab -> server: call add { text } +server -> worker: relay (header only) +worker -> obj: strip + validate args +obj => obj: self.tx: push note, commit +obj --> worker: result: id +worker --> server: state: ops of the commit +worker --> server: result +server --> tab: state, then result +::: + +## Two rules for members + +**Methods return data, never objects.** A method that wants to point you to another object returns its address +(`{ session: "S2" }`), and you `get` it yourself. That keeps every reference owned by someone who releases it. + +**No callbacks as arguments.** A callback carries code, and everything it closes over, from one object into another. +Suppose an inbox object passed `onFile` to another object, which stored it. After a reload of the inbox's plugin, the +other object still calls the *old* `onFile`; after the inbox ends, it calls into an object whose file watcher is +already closed. Chord forbids this by construction: members are data (JSON plus bytes). Behaviour from elsewhere goes +through registries (chapter XIII), multi-step work through jobs (XI), input from a person through flows (XII). + +## What is wrong with this version + +This first version works for one person. It has two problems, which the next two chapters fix. Everyone gets the +`editor` variant, including a read-only guest. And, less obvious, the owner and a guest do not see the same notes: as +chapter III shows, a remote object exists per caller, so each caller's `Notes` object has its own `NoteList`. + +:::note | Built so far +`remote`, `method`, the schema helpers, `Static` and fingerprints are built (`src/next/contracts/`). Serving, `get`, +references and documents are layers L3 to L5. +::: diff --git a/packages/chord/docs/next/explainer/src/03-principals.md b/packages/chord/docs/next/explainer/src/03-principals.md new file mode 100644 index 000000000..52a4c1064 --- /dev/null +++ b/packages/chord/docs/next/explainer/src/03-principals.md @@ -0,0 +1,190 @@ + +# Who Is Calling {#principals lede="Principals: how Chord knows who calls, why a caller cannot lie about it, and how the answer picks the object and its members."} + +The owner may add and remove notes. Ann, a read-only guest, may only read them and should not even see who wrote +what. The object must therefore know who is calling, and the caller must not be able to claim to be someone else. In +Chord the answer is a **principal**, a small JSON value the app defines and the links stamp. + +## The app's principal type + +Chord defines no principal kinds; the app does. Every variant has a `kind`. Studio uses p2's set: + +```ts +// studio/src/principal.ts +export type P = + | { kind: "owner"; access: "local" | "relay" | "device"; device?: string } + | { kind: "guest"; session: string; mode: "read" | "write"; grant: string; revision: number } + | { kind: "server" } + | { kind: "worker"; session: string }; +``` + +| `kind` | Who | +| --- | --- | +| `owner` | the person running the app, on loopback, through the relay, or on a paired device | +| `guest` | someone invited into one session, with a mode and the grant (and its revision) that let them in | +| `server` | the server's own code calling a worker | +| `worker` | a session worker calling the server | + +## Links stamp principals + +A principal never comes from a request's data. The server authenticates a client once, when its WebSocket connects, +and stamps the result on the whole link (`server.accept(link, { principal })`). The server tells Chord who each worker +is when it resolves it (`return { link, principal }`). When the server relays a client's request to a worker, Chord +writes the client's principal into the forwarded request; a principal field coming from a client is always +overwritten, and only a trusted parent link may carry one. + +Where principals come from is app code. Studio's sharing feature creates a share link per session and mode; a guest +who opens it gets a guest record (grant `g1`) and a cookie; `authenticate(req)` turns the cookie into +`{ kind: "guest", session: "S1", mode: "read", grant: "g1", revision: 1 }` on every new connection. Chord never sees +cookies or grants, only the finished principal. + +## The caller is part of the identity + +A `remote` object exists once per **(type, input, principal key)**. The **principal key** is a string computed from +the principal; by default its canonical JSON (sorted keys). The app may pass its own `principalKey` to `createHost`, +and must pass the same function on every host, or identities split silently. + +```ts +const principalKey = (p: P) => + p.kind === "owner" ? `owner:${p.access}:${p.device ?? ""}` + : p.kind === "guest" ? `guest:${p.grant}:r${p.revision}` + : p.kind === "worker" ? `worker:${p.session}` + : p.kind; +``` + +The design wants the key to include how the client is connected and the grant's revision, so that two access paths +never share objects and a changed grant never reuses the old grant's object: + +| Caller | Principal key | `Notes { S1 }` object | +| --- | --- | --- | +| owner, tab A, local | `owner:local:` | 1 | +| owner, tab B, local | `owner:local:` | 1 (shared with tab A) | +| owner through the relay | `owner:relay:` | 2 | +| guest g1, revision 1 | `guest:g1:r1` | 3 | +| guest g1 after a change to write mode, revision 2 | `guest:g1:r2` | 4 (new) | + +## Variants + +The contract lists named sets of members; the factory receives the principal and returns the variant that principal +may use: + +```ts +// notes/contract.ts (version 1, final form for now) +variants: { + editor: { + state: { notes: obj({ items: arr(Note) }) }, + methods: { add: method(obj({ text: str({ min: 1, max: 2000 }) }), str()), remove: method(obj({ id: str() })) }, + }, + reader: { + state: { notes: obj({ items: arr(PublicNote) }) }, // no `by`: authors are hidden + }, +}, +``` + +```ts +// notes/worker.ts +host.serve(Notes, ({ session }, principal, self) => { + if (canWrite(principal, session)) return { variant: "editor", notes: …, add: …, remove: … }; + return { variant: "reader", notes: … }; +}); +``` + +A reader has no `add` at all. Chord refuses a call to a member that is not in the caller's variant, so `add` needs no +check of its own. Three rules keep variants compatible: a member name has the same kind (state or method) in every +variant that has it; a method has the same arguments and result in every variant; only state schemas may differ per +variant (the reader's notes have no authors). `remote(...)` enforces them at the type level, and at runtime for +contracts that arrive as data. + +## How a facet knows `P` + +`P` is Studio's type, not Chord's, and every host is created with it (`createHost

`). A plugin's facets are written +and compiled separately, then handed to `host.load` as values, so the type checker cannot infer `P` from the host into +the factory's `principal`. Studio's base code therefore exports its own `defineFacet`, already bound to `P`, and every +facet of every plugin imports that one instead of Chord's: + +```ts +// studio/src/facet.ts (base code, exported as @studio/core) +import { bindFacet } from "@earendil-works/chord/next"; +import type { P } from "./principal.ts"; + +export const defineFacet = bindFacet

(); // a facet's principals are Studio's P +export type { P, Revocation } from "./principal.ts"; +``` + +```ts +// notes/worker.ts (server.ts, web.tsx and tui.ts import the same defineFacet) +import { defineFacet } from "@studio/core"; + +export default defineFacet((host, self) => { + host.serve(Notes, ({ session }, principal, self) => { // principal: P + if (principal.kind === "guest" && principal.mode === "read") return { variant: "reader", notes: … }; + … + }); +}); +``` + +The import is cheap: `@studio/core` is a host package plugins already import at runtime (the contract imports `Session` +from it), `P` is a type-only import, and `bindFacet` is a one-line function. The host checks the pairing: `load` +accepts only facets typed for its own `P`, so a facet compiled against another app's principal type, or one built with +Chord's untyped `defineFacet`, does not compile into Studio's start code. Client facets (`web.tsx`, `tui.ts`) use the +same `defineFacet`; they rarely see a principal, because clients serve only `local` types. + +None of this is checked at runtime, and none of it needs to be: the principal a factory receives is the one the link +stamped. + +:::review | Review: binding facets to the app's principal type (open) +The design shows `defineFacet((host, self) => …)` without saying how `P` reaches the factory. The frozen kernel takes +it as an explicit type parameter, `defineFacet

(…)`, defaulting to `{ kind: string }` +(`packages/chord/src/next/facet.ts`), and the playground writes `defineFacet(…)`. The pre-bound +`defineFacet` above (`bindFacet` is a working name) is the owner's preferred direction, recorded as an open point in +`packages/chord/docs/next/layer-1.md` §4. +::: + +## Narrowing on the caller's side + +Owner and guest run the same `web.tsx`, and which variant they get is known only at runtime. The types force every +caller to narrow before using variant members: + +```ts +const notes = await client.get(Notes, { session: "S1" }); +notes.add({ text: "hi" }); // type error: `add` exists only in "editor" +if (inVariant(notes, "editor")) await notes.add({ text: "hi" }); +expectVariant(notes, "editor"); // or assert (throws) +function Editor({ notes }: { notes: RefIn }) { … } +``` + +The protection is not the type checker; it is Chord refusing the call at the provider. A client that ignores the +types gets the same error. + +## Admission before a worker starts + +Principals also decide whether a caller may reach a host at all. `route(Session, { authorize })` runs before Chord +resolves an address, so a guest of S1 asking for anything in S2 is refused at the server, and no worker starts for +them. Finer authority lives in the per-principal view's methods; there is no asynchronous "may this caller activate +this object" guard. An object whose address carries authority (say, a login flow by id) may exist; what a caller can +see and do is decided by the variant its factory picks. + +## Inside methods + +`ctx.principal` is the caller of the current call. In Studio's notes the editor's `add` uses it to record the +author: + +```ts +add: ({ text }, ctx) => store.add({ text, by: authorOf(ctx.principal) }, ctx), +``` + +Where principals act, in summary: + +| Purpose | Where | +| --- | --- | +| identity: which object a caller gets | principal key | +| variant: which members | the factory | +| admission: whether a host is reachable | `route.authorize` | +| attribution and fine checks | `ctx.principal` in methods | +| revocation | `revokedBy` (chapter VII) | + +## The bug from chapter II + +Now the bug is visible. The owner's `Notes { S1 }` and Ann's `Notes { S1 }` are two objects (rows 1 and 3 above), and +each declared its own `NoteList`. The owner's notes are not in Ann's list. Per-caller objects must not hold the data; +they must be views over something that exists once. That is the next chapter. diff --git a/packages/chord/docs/next/explainer/src/04-store.md b/packages/chord/docs/next/explainer/src/04-store.md new file mode 100644 index 000000000..44d3739ff --- /dev/null +++ b/packages/chord/docs/next/explainer/src/04-store.md @@ -0,0 +1,132 @@ + +# One Store, Many Views {#shared-store lede="Local types, and the pattern every Chord plugin uses: shared data once in a store, per-caller views in front of it."} + +Every caller has its own `Notes` object, which is what makes per-caller variants possible. But data must exist once. +The fix is the most common pattern in Chord: a **shared store**, a `local` object that holds the data and does the +heavy work once, behind thin per-principal **views**, the `remote` objects callers get. + +## `local` types + +```ts +// notes/contract.ts +export const NoteStore = local("notes.store", { + version: 1, + input: obj({ session: str() }), + members: { + notes: obj({ items: arr(Note) }), // state: a schema + public: obj({ items: arr(PublicNote) }), + add: method(obj({ text: str(), by: str() }), str()), // methods: method(...) + remove: method(obj({ id: str() })), + }, +}); +``` + +A `local` contract has flat `members` and no variants. The differences: + +| | `remote` | `local` | +| --- | --- | --- | +| reachable from | any host, across links | its own host only | +| identity | (type, input, principal key) | (type, input) | +| principals, variants | yes | no | +| factory | `(input, principal, self)` | `(input, self)` | +| arguments | stripped to the provider's schema and validated, for every caller | see the review note below | +| values on the same host | passed by reference, never copied | passed by reference, never copied | +| `ctx.principal` | the caller | none | +| may persist documents | no (chapter V) | yes | + +A local object has no principal: it is trusted code on its own host, acting for the host. So attribution ("who wrote +this note") is passed as data (`by`), by the view that knows who called. Clients serve only local types, because +nobody can address a client. + +## The pattern + +```ts +// notes/worker.ts +export default defineFacet((host, self) => { + host.serve(NoteStore, ({ session }, self) => { + const list = self.state(NoteList); + const pub = self.derive("public", [list], ([l]) => ({ items: l.items.map(stripAuthor) })); + return { + notes: list, + public: pub, // computed once, for every reader + add: ({ text, by }, ctx) => + self.tx(async (tx) => { + const id = uuidv7(); + (await tx.draft(list)).items.push({ id, text, by, at: Date.now() }); + return id; + }, ctx), + remove: ({ id }, ctx) => list.change((d) => { d.items = d.items.filter((n) => n.id !== id); }, ctx), + }; + }); + + host.serve(Notes, ({ session }, principal, self) => { + const store = self.get(NoteStore, { session }); // a reference, declared: held for the view's life + if (canWrite(principal, session)) return { + variant: "editor", + notes: store.notes, // the store's readable, exposed as-is + add: ({ text }, ctx) => store.add({ text, by: authorOf(principal) }, ctx), + remove: (args, ctx) => store.remove(args, ctx), + }; + return { variant: "reader", notes: store.public }; + }); +}); +``` + +:::flow wide | One store, three views. The redaction runs once in the store; both readers read the same value. +row "Per-principal views (remote)" blue +box ve "Notes { S1 }" "owner · editor" blue +box vr1 "Notes { S1 }" "guest Ann · reader" blue +box vr2 "Notes { S1 }" "guest Bob · reader" blue +row "Shared store (local)" accent +gap +box store "NoteStore { S1 }" "notes, public (a derive)" accent +gap +arrow ve -> store "self.get" +arrow vr1 -> store "self.get" +arrow vr2 -> store "self.get" +note "Authority is the view's: the reader variant has no add. The store does not check; it is trusted and acts for the host." +::: + +`self.get(NoteStore, { session })` in a factory declares a **reference** that the view holds for its whole life; Chord +binds it during activation and waits until the store is live (chapter VI). The view exposes the store's readables +directly. Owner, Ann and Bob now give three `Notes` objects, one `NoteStore`, one note list and one redacted list. + +Any heavy or shared part belongs in a store like this: a pi-durable watch, a file watcher, a sandbox, a search index. +Base code uses the same pattern with the same primitives; nothing about it is special to core. + +## Derived values, briefly + +`self.derive(name, sources, fn)` computes a value from readables and recomputes it when a source changes; viewers get +only the difference. The reader's list above is one. Chapter VIII covers derives in depth, including the one footgun +that matters: `stripAuthor` must return the *same* object for an unchanged note, or every commit costs the size of the +whole list. The tool for that is `mapShared`. + +## Arguments are checked at the provider + +Trusted code is not type-checked code: plugins are bundled with esbuild without type checks, and agents write +JavaScript plugins. So the provider checks arguments itself (decision R7): every method's arguments are stripped to +the **provider's own** schema (unknown fields removed) and validated with it, whoever the caller is, and never with a +schema a peer sent. Schemas are compiled lazily on first use and cached per schema object. Bytes are checked as leaves +(`instanceof Uint8Array` plus an optional length bound), never walked. Stripping copies only when a field was removed; +the caller's value is never mutated (proposal Q12, in the L1 draft). Results and state are not validated: they come +from the provider. + +On the same host nothing is copied: arguments and results are passed by reference. Values are **immutable by +convention** (Delta's rule, as in Durable): nobody mutates a value it received, and Chord neither copies nor freezes +values to enforce it. There is no development mode that would freeze them; one behaviour everywhere (R9). + +:::review | Review: are local arguments validated? (Q15, open) +Design §5's table says `local` arguments are "neither validated nor copied"; §3 and R7 say arguments are validated +"for every caller". Review round 3 raised this as Q15 (apply R7 to `local` too, or keep `local` as the one exception), +and `layer-1.md` lists it as open. Until it is decided, write local methods so that they do not depend on either +answer: trust the shape, but never trust content (lengths, ids) that came from a remote caller without the view +checking it. `chord-next-walkthrough.md` concept 6 still describes the older rule ("validated only when the caller is +not a host principal"), which R7 replaced. +::: + +## Choosing between the two + +Use a `remote` type for what callers address: views, things with per-caller authority. Use a `local` type for what +exists once per input on one host: stores, caches, owners of OS handles, and the core objects a host provides +(chapter IX). If a type is reached across a link, it is remote; if a type persists documents, it is local (or a +host-lifetime object, chapter V). diff --git a/packages/chord/docs/next/explainer/src/05-documents.md b/packages/chord/docs/next/explainer/src/05-documents.md new file mode 100644 index 000000000..2a25a1caa --- /dev/null +++ b/packages/chord/docs/next/explainer/src/05-documents.md @@ -0,0 +1,258 @@ + +# Writing It Down {#documents lede="Documents and transactions: how an object stores data, changes it atomically, survives restarts and migrates it."} + +Notes must be stored, changed safely while others read them, sent to viewers as small changes, and survive a worker +restart. An object's data lives in **documents**: named JSON values the object owns, changed only in +**transactions**. The design is a port of pi-durable's store, so most rules will look familiar; this chapter states +them for Chord. + +## Defining a document + +```ts +// notes/worker.ts +const NoteList = defineState("notes.list", { + schema: obj({ items: arr(Note) }), + version: 1, + initial: () => ({ items: [] }), + persist: true, // stored; default: memory only + checkpointWhen: (_value, _ops, info) => info.deltasSinceBase > 64, +}); + +const Draft = defineState("notes.draft", { // one unfinished note per author + schema: obj({ text: str() }), + version: 1, + initial: () => ({ text: "" }), + keyed: true, // many per object, by key + persist: true, + checkpointWhen: () => true, // small: a full base per write +}); +``` + +`checkpointWhen` is required on every definition, with no default (R14, as in Durable since pi `5c1736f82`): it says +when a commit stores a complete base instead of a delta, which bounds how much is replayed on load. Small documents +checkpoint on every write; growing ones after a number of deltas. + +A document's root is always a JSON object (lists live in a field). Documents hold JSON only; bytes are a value and wire +type, not a document type. + +## Singleton and keyed + +```ts +host.serve(NoteStore, ({ session }, self) => { + const list = self.state(NoteList); // THE note list of this object + const drafts = self.state(Draft); // all drafts of this object + … +}); +``` + +| | singleton (no `keyed`) | keyed | +| --- | --- | --- | +| how many | one per object | many per object, by a short key | +| loaded | at activation, kept in memory | on demand | +| read | `list.value` (synchronous) | `await drafts.get(key)`: the value or `undefined` | +| list | (it is one value) | `for await (const key of drafts.keys())` pages keys | +| expose as state | yes: the handle is a readable | no: expose a method or a singleton index | + +There is no live key list. A collection shown live is a singleton (an object holding a list or a map); a large +collection is a method returning pages. + +## Transactions + +All writes go through `self.tx(fn, ctx)`. Inside, `await tx.draft(handle, key?, { tags? })` returns a draft of the +current document, or a draft over `initial()` if it is absent. Drafts record operations (assignments, `push`, +`Object.assign`); asking for the same document twice in one transaction returns the same draft. `tx.retire(handle, +key?)` removes a document. There is no `tx.get`: reading is drafting, and a draft is an overlay over the current +value, so nothing is copied. A draft you do not change commits nothing, so reading never stores an absent document. + +```ts +saveDraft: ({ author, text }, ctx) => + self.tx(async (tx) => { (await tx.draft(drafts, author)).text = text; }, ctx), +``` + +Shorthands for one document: `handle.change(fn, ctx)` (keyed: `handle.change(key, fn, ctx)`) (loads the document, then runs the synchronous `fn` on +its draft) and `handle.retire(ctx)` (keyed: `handle.retire(key, ctx)`). + +### Four patterns + +Two decisions (R12 and R13, both as in Durable) shape how transactions are written. **Drafts are revoked when the +transaction settles**, so a transaction returns at most plain values, never something taken from a draft. And +**there is no existence check inside a transaction.** + +```ts +// 1. Return data: write in the transaction, read after the commit. +await self.tx(async (tx) => { (await tx.draft(list)).items.push(note); }, ctx); +return list.value.items.length; // the committed revision, or a newer one + +// 2. Remove if present: retire of an absent document is a no-op. +await drafts.retire(author, ctx); + +// 3. Create with tags if missing: the draft creates it, only if it changes. +await self.tx(async (tx) => { + (await tx.draft(drafts, author, { tags: [SessionTag(session)] })).text = text; +}, ctx); + +// 4. Branch on existence: read first, outside the transaction. +if ((await drafts.get(author)) === undefined) { … } +``` + +### Durable's rule + +Inside the callback, only `await tx.*`. No other objects, no network, no models, no processes, no second `self.tx`, no +reads past `tx`. Breaking it is not an error but a hang: the object's line is held while the callback waits. Fetch +first; decide and write inside. + +```ts +// Wrong: a fetch inside the transaction holds the object's line. +await self.tx(async (tx) => { (await tx.draft(list)).items.push({ …, title: await fetchTitle(url) }); }, ctx); +// Right: fetch, then write. +const title = await fetchTitle(url, ctx.signal); +await self.tx(async (tx) => { (await tx.draft(list)).items.push({ …, title }); }, ctx); +``` + +## What a commit does + +:::flow wide animate | The commit sequence, Durable's. Nothing is copied, frozen or revalidated; viewers get one batch per commit. +row "Inside self.tx" accent +box line "wait for the line" "per object" +box run "run the callback" "drafts record ops" accent +box prep "prepare" "Delta checks placed values" +box write "storage batch" "deltas or a base" accent +row "After the write" blue +box adopt "adopt" "pointer swap" +box local "notify locally" "once, with the ops" blue +box wire "send" "one batch per link" +box revoke "revoke drafts" "touching one throws" +arrow line -> run +arrow run -> prep +arrow prep -> write +arrow write -> adopt "stored" +arrow adopt -> local +arrow local -> wire +arrow wire -> revoke +note "An error or an abort before the storage write discards everything. Storage commits are serialized per host." +::: + +Lines are per object, never across objects; reads outside a transaction do not take the line. Subscribers on the same +host get the new value and the commit's operations by reference. Remote viewers get the commit's operations, never the +document again; chapter XVII shows how one commit touching several holds still arrives as one message. + +## Tags and deleting by tag + +Tags are typed tokens set when a document is created: + +```ts +export const SessionTag = defineTag("studio.session"); // base code, shared by every plugin +await self.tx(async (tx) => { (await tx.draft(drafts, author, { tags: [SessionTag(session)] })).text = text; }, ctx); +// when the session is deleted +await store.retireByTag(SessionTag, "S1"); // every document tagged S1 +``` + +The design names the operation `store.retireByTag(tag, value)` without saying who holds the store; session delete is +one of the app decisions still open (#18, chapter XIX). + +## Who may persist + +`persist: true` is allowed only in `local` objects and `lifetime: "host"` objects: identities without a principal. A +`remote` object's identity contains the principal key, which changes with the access path and the grant revision. Its +stored documents would scatter across access paths and be orphaned after a reconnect. The rule is enforced at the type +level (`self.state` in a remote factory accepts only non-persisted definitions) and at declaration ("persist needs a +stable identity: put the documents into a local shared store"). This is the second reason for the shared store. + +What survives a host teardown (a crash, a restart) is exactly what is persisted. In-memory documents live as long as +their object: they survive reloads and native restarts of a live object, and go when it ends. Everything else +(natives, derives, routines, references) is rebuilt on activation. + +## Facet documents + +A facet has its own identity (its plugin's name) and may hold documents too, for plugin-wide data such as +preferences. Objects the facet serves may read and write their facet's documents, and one transaction may span the +object's own documents and its facet's (lock order: own line, then the facet's). The facet never touches a served +object's documents. + +## Ownership across plugins + +No plugin reads or writes another plugin's documents, Chord or Durable. Data crosses plugins only through state and +methods. Data that core enforces lives in a format core owns and is handed to core through its API when the data is +created (as with Durable's `createConversation({ init })`), never written into core's documents. + +## Versions and migration + +A change of shape or meaning raises `version` and adds a pure `migrate(old, from)`: + +```ts +const NoteList = defineState("notes.list", { + schema: obj({ items: arr(NoteV2) }), // NoteV2 adds `pinned` + version: 2, + migrate: (old, from) => ({ items: old.items.map((n) => ({ ...n, pinned: false })) }), + initial: () => ({ items: [] }), + persist: true, + checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64, +}); +``` + +Documents migrate on access by the new definition: singletons during activation (hydration counts as access), keyed +ones on their next access. The migrated base is persisted with the next commit; right after a bump, a read-only +transaction writes it. + +Chord does not walk values against schemas. Instead, each definition computes a **schema fingerprint** once +(descriptions and titles ignored), and every stored record keeps the fingerprint it was written with. On access Chord +compares them: the same version with a different fingerprint fails the object `broken` ("schema changed without a +version bump"). So every structural change needs a bump, an additive one with a trivial `migrate`. The one exception +to "no value checks" (R8): the result of `migrate` and of `initial()` is validated against the new schema once, when +produced. A failed check, a throwing `migrate`, or old code meeting a newer document refuses the access, keeps the +stored data and fails the owning object. A changed meaning without a schema change cannot be detected; that is on +you. + +## Storage + +Behind the store is a `DocumentStorage`, passed as `createHost({ storage })`; memory by default. The server has one +shared storage, each worker one for its session. + +```ts +interface DocumentStorage { + commit(writes: readonly DocumentWrite[]): Promise; // create/change/retire, atomic + find(address: DocumentAddress): Promise; + load(id: string): Promise; // version, value, deltasSinceBase + scan(query: { owner: string; definition: string }, limit: number, cursor?: string): + Promise<{ keys: string[]; next?: string }>; + byTag(tag: string, value: string, limit: number, cursor?: string): Promise<{ ids: string[]; next?: string }>; + mintId(): Promise; +} +``` + +Chord ships a memory storage and a SQLite storage (over `@earendil-works/chord/sqlite`, the driver layer that moved +from Durable; Node and Durable Object drivers). Storages check only metadata (incarnation, version, sequence, address +occupancy, fingerprint) and write the supplied delta or base; they never rebuild or validate content on a commit. +Chord documents never commit together with Durable transcripts: Chord has its own storage, not an adapter over +Durable's. Your own storage can prove itself with `registerDocumentStorageConformance(runner, name, withStorage)` from +`@earendil-works/chord/testing`. + +Keyed persisted documents that nobody reads, writes or watches for 10 minutes leave memory and reload on the next +access. A storage error ends the store and closes the host; `onError` receives `{ kind: "storage_failed" }` and the +app decides (chapter I). + +## Kinds of state + +| Kind | For | API | +| --- | --- | --- | +| document | data the object owns | `self.state`, `self.tx` | +| derive | computed values | `self.derive` (chapter VIII) | +| source readable | external data: Durable views, files, logs | value plus ordered changes; feeds derives | +| paged method | large or queryable collections | a method returning pages (chat history) | +| plugin database | full-text search, large blobs | a native opened by its owner (chapter IX) | + +App state lives in Chord documents. Only transcript-bound data, which must fork and rewind with a conversation, lives +in Durable. + +:::review | Review: does an absent singleton read `undefined` or `initial()`? (Q11, open) +Design §9 says "whether a document exists is read outside the transaction (`handle.value` / `handle.get` give +`undefined` for an absent one)" and also "a singleton that does not exist yet reads `initial()` through its handle; +retiring resets it to `initial()`". Review round 3 proposes: singletons always read `initial()`, only absent keyed +documents read `undefined`, and singleton existence needs no API. This guide follows the proposal (`list.value` is +never `undefined`); it is not decided. +::: + +:::note | Built so far +Documents, transactions and storages are layer L3, a port of Durable's session store; not started. The frozen kernel +has a working version with a JSONL storage, which the design rejected. +::: diff --git a/packages/chord/docs/next/explainer/src/06-references.md b/packages/chord/docs/next/explainer/src/06-references.md new file mode 100644 index 000000000..b852916b5 --- /dev/null +++ b/packages/chord/docs/next/explainer/src/06-references.md @@ -0,0 +1,185 @@ + +# Holding On {#references lede="References, the object lifecycle and ctx: what a caller sees while objects start, fail, restart and move, and why nobody writes retry code."} + +A tab shows the notes of S1 for an hour. During that hour the laptop sleeps, the worker is stopped for idleness and +started again, the notes plugin reloads twice and the store's storage hiccups once. The panel should show the notes the +whole time, say "reconnecting" when it cannot reach them, and recover by itself. In Chord this is what a +**reference** does; plugin code contains no retry loops and no request ids. + +## Three ways to hold + +| Where | API | Released | +| --- | --- | --- | +| a factory | `self.get(Type, input)` | when the object ends | +| a call | `await ctx.get(Type, input, { state? })` | when the call ends | +| outside objects | `await host.get(Type, input, { signal, state?, parkTimeout? })` | `ref.release()` | + +Use `self.get` for what an object needs its whole life (the view's store), `ctx.get` for what one call needs, and +`host.get` only in UI code, tests and entry points. `{ state: false }` binds for methods only, without receiving state. +A reference keeps its object alive while held. + +## The object lifecycle + +```text +(get) → activating → live → ending → ended + └→ failed (backoff) → activating … +``` + +**Activating:** Chord binds the declared references and waits until they are live and hydrated, loads singleton +documents, opens natives (one after another, in declaration order), computes derives and starts routines. The object +goes live when all of that succeeded. Calls wait behind its **gate** until then. + +**Ending:** routines stop, the object's calls and jobs are aborted as their rules say, natives close in reverse order, +references are released. An object with `lifetime: "referenced"` (the default) ends when its last reference goes, +after the type's `linger` if it has one. A `lifetime: "host"` object (only for types without a required input) lives +as long as the host. A running, finishing or blocked job counts as a reference (chapter XI). + +## `ref.state` + +Every reference reports what is going on as a tagged value; listeners get each new one: + +```ts +type RefState = + | { kind: "loading" } // first hold, reconnecting, restarting + | { kind: "live" } + | { kind: "failed"; cause: FailureCause; retryAt: number | undefined } // undefined: no automatic retry + | { kind: "released" }; +``` + +| Situation | Calls | Values | +| --- | --- | --- | +| link drop | park until reconnected | readable, `loading` | +| object loading or restarting | park until live | readable, `loading` | +| rollout: provider ahead or installing | park until installed | readable | +| object `failed` | reject at once with the cause | `failed`, with cause and `retryAt` | + +Values stay readable through all of this: the panel keeps showing the last notes. Parked calls share one deadline (60 +s by default, `parkTimeout` per `get`), then fail `unavailable`. Otherwise a call ends only through its caller's +signal; there is no per-call deadline. A call that was running on an instance that ends fails with an error and is +not resent: Chord cannot know whether it took effect. + +**Errors look like disconnects.** A consumer handles a failed call exactly as a dropped link: show the error, let the +user retry. Work that must not run twice carries its own idempotency key as data (Durable's `requestId`, a job's +`key`). + +:::steps | The worker restarts while a tab watches the notes ++++ Watching +The tab holds `Notes { S1 }`. `ref.state` is `live`; `notes.notes.value` has 40 notes. ++++ The worker goes away +The supervisor stops worker S1 (or it crashes). The server's IPC link to it drops. The server tells the tab that its +hold is gone; the tab's reference goes `loading`. The 40 notes stay readable. ++++ A click meanwhile +Ann's colleague clicks **Add**. The call **parks** on the reference; nothing is sent yet. The deadline is 60 s. ++++ Chord resolves again +Holds on S1 still exist, so Chord calls `route(Session).resolve({ id: "S1" })` again, with the per-address backoff. +The app forks a new worker: a new **incarnation** of `Session({ id: "S1" })`. ++++ Re-hold +The server re-holds every hold on S1 over the new link. The new worker activates `NoteStore { S1 }` (its notes load +from storage) and the views; it sends fresh snapshots. The tab's reference goes `live`. ++++ The parked call runs +The parked `add` is sent now and succeeds. The panel shows 41 notes. Nobody wrote recovery code. +::: + +## Failure and retry + +Any broken part fails the **whole** object: a half-working object could break rules that rely on its documents +changing together. Chord tags the cause by where it failed: + +```ts +type FailureCause = + | { kind: "transient"; error: ChordError } // an open threw, a dependency was unreachable: automatic backoff + | { kind: "broken"; error: ChordError }; // factory, derive, hydration or migrate threw: retried only by a trigger +``` + +**Transient** causes get a backoff per address: 1 s, doubling to 60 s; concurrent gets share one attempt; success +resets it. **Broken** causes would fail the same way again (a code bug, invalid data), so they are retried only by a +**trigger**: a plugin reload, `ref.retry()` (a Retry button), or a readable the factory declared with +`self.retryOn(...readables)`. Only an opener may declare its own failure broken, with `throw broken(error)`, for +example for invalid configuration. A failed reference stays held; on success it goes `loading`, then `live`. + +```ts +host.serve(EnvConnection, ({ env }, self) => { + const config = self.get(EnvConfig, {}); // a facet or registry value, outlives the attempt + self.retryOn(config.envs); // a config change retries a broken open + const ssh = self.open(async ({ signal }) => { + const c = config.envs.value[env]; + if (!c) throw broken(new Error(`unknown environment ${env}`)); + return connect(c, signal); + }, (s) => s.close()); + … +}); +``` + +`retryOn` sources must live independently of the attempt (facet documents, registry values); a member of one of the +object's own references is a type error, since it goes away with the attempt. + +## Waiting for dependencies + +An object whose references are not live yet stays `loading` and goes live the moment they are, bounded by +`readyTimeout` (60 s). The decision for the failure case (R18) is what makes recovery automatic: + +:::sequence animate | Dependency recovery. The view fails transient with the store's cause, keeps its reference, and is retried the moment the store goes live. +actor tab "tab" +actor view "Notes { S1 }" blue +actor store "NoteStore { S1 }" accent +tab -> view: hold +view -> store: self.get (activation) +store -> store: open fails: disk not mounted +store --> view: failed (transient) +view --> tab: failed: transient, cause inside +note view..store: the view keeps its reference; its state is the retry trigger +store -> store: backoff retry: open succeeds +store --> view: live +view -> view: retried at once (backoff cancelled) +view --> tab: live +::: + +A dependency that fails, or does not become live in time, fails the dependent **transient** with the dependency's +cause inside, never broken: the cause is the dependency's. The dependent keeps its references while failed, and Chord +uses their state as a retry trigger. Factory authors write `self.get` and know none of this. After an object is live, +a dependency leaving `live` does not change it: its calls that use the dependency get the dependency's error, or park +while it loads. + +**Cycles are rejected** (Q5). If A's activation gets B and B's gets A, neither would ever lose its last reference. +Every object instance has a uuidv7 instance id; on one host Chord follows the waiting edges of activating objects, and +across hosts a hold made during an activation carries the chain of instance ids. A cycle fails the activation +`broken`, naming it ("A → B → A"), and keeps no references. + +## No optional references + +`get` has no `optional` flag. When a dependency's plugin may be missing, turn the dependency around: the dependent +owns a registry and the other plugin contributes to it. Radius uses forwards from the ports plugin, but ports is +optional, so radius owns `radius.forwards` and ports contributes its forwards; without ports the table is empty and +radius works (chapter XIII). + +## `ctx` + +Every call gets a `ctx` as its last argument: + +| Field | Meaning | +| --- | --- | +| `signal` | fires when the caller cancelled or the object ended; pass it to everything you wait on | +| `principal` | the caller (remote types only) | +| `incarnation` | the calling host's incarnation, for calls that arrived over a link (chapter X) | +| `get` | references released when the call ends | +| `telemetry`, `trace(name, attributes, fn)` | the current span; a child span with a child `ctx` (chapter XVII) | +| `jobId`, `progress` | in a job's `run` (chapter XI) | + +```ts +exportText: async ({}, ctx) => { + const catalog = await ctx.get(Catalog, {}); // released when the call ends + const title = catalog.sessions.value[session]?.title ?? session; + const res = await fetch(renderUrl, { method: "POST", body: render(title, list.value), signal: ctx.signal }); + return await res.text(); +}, +``` + +There are no code versions and no request ids in `ctx`. Durable's `Context` is the same module (`chord/context`): +`ctx` extends it, so passing `ctx` into pi-durable needs no conversion. Code outside objects gets a `ctx` from +`host.root(fn, { principal?, signal? })`; inside Durable tools it uses `chord(fn, context)` (chapter XIV). + +:::note | Built so far +References, `ctx` and the lifecycle are layers L4 and L5. `ChordError` and `FailureCause` are specified in the L1 +draft (`layer-1.md`), with codes `invalid`, `contract_mismatch`, `forbidden`, `gone`, `not_ready`, `released`, +`too_large`, `unavailable` and `aborted`, and a finer `reason` such as `invalid/args`. +::: diff --git a/packages/chord/docs/next/explainer/src/07-revocation.md b/packages/chord/docs/next/explainer/src/07-revocation.md new file mode 100644 index 000000000..1cacf921b --- /dev/null +++ b/packages/chord/docs/next/explainer/src/07-revocation.md @@ -0,0 +1,113 @@ + +# Taking Access Away {#revocation lede="Revocation: how removing a guest ends their objects on every host at once, and why reconnecting does not bring access back."} + +The owner removes Ann from S1. Her tab still has a WebSocket open and a `Notes { S1 }` view on worker S1. Both must end +now, on the server and on the worker, and her tab's automatic reconnect must not restore anything. Chord provides the +mechanism; the app decides who is blocked and remembers it. + +## Labels, not predicates + +The app declares, once, which **revocation labels** each principal carries. Revoking means revoking one label; +everything carrying it is hit. + +```ts +// studio/src/principal.ts +export type Revocation = { grant: string } | { device: string } | { session: string }; + +export const revokedBy = (p: P): Revocation[] => + p.kind === "guest" ? [{ grant: p.grant }, { session: p.session }] + : p.kind === "owner" && p.access === "device" && p.device ? [{ device: p.device }] + : []; + +const server = createHost({ address: Server(), principal: { kind: "server" }, revokedBy }); +// every worker is created with the same revokedBy +``` + +| Principal | Labels | +| --- | --- | +| guest Ann (grant g1) in S1 | `{ grant: "g1" }`, `{ session: "S1" }` | +| guest Bob (grant g2) in S1 | `{ grant: "g2" }`, `{ session: "S1" }` | +| owner on the paired phone | `{ device: "phone" }` | +| owner, local | none | + +| Call | Hits | +| --- | --- | +| `server.revoke({ grant: "g1" })`: remove Ann | Ann | +| `server.revoke({ session: "S1" })`: archive S1 | Ann and Bob | +| `server.revoke({ device: "phone" })`: sign out the phone | the phone | + +A label is plain data because revocation is an event that must travel from the server to the workers; a predicate +function could not be sent. + +## What `revoke` does + +`host.revoke(r)` ends every object and closes every link whose principal `p` has `r` in `revokedBy(p)` (compared by +canonical JSON; never the host's own principal), and passes `r` to its children, which apply their own `revokedBy`. + +:::sequence animate | Removing guest Ann. The label travels down; her objects end on every host; her reconnect is refused by the app's own record. +actor owner "owner tab" +actor server "server" accent +actor worker "worker S1" blue +actor ann "Ann's tab" "guest g1" muted +owner -> server: Sharing.remove({ grant: "g1" }) +server -> server: app: mark g1 blocked (document) +server => server: revoke({ grant: "g1" }) +server -> ann: close link +server => worker: revoke { grant: "g1" } +worker -> worker: end Notes { S1 } as g1 +ann -> server: reconnect +server -> server: authenticate: g1 blocked +server --> ann: refused (4401) +::: + +On worker S1, for `revoke({ grant: "g1" })`: + +| Object | Principal | Hit | +| --- | --- | --- | +| `Notes { S1 }` | owner, local | no | +| `Notes { S1 }` | guest g1 | **yes**: ends | +| `Notes { S1 }` | guest g2 | no | +| `NoteStore { S1 }` | none (local) | no | + +Ending Ann's view releases its reference to the store, which keeps serving the others. + +## Chord persists nothing here + +`revoke` cuts live access only. Lasting denial lives in the app's data (the sharing plugin's guest records), which +`authenticate` checks on every new connection. After a server restart nothing live exists to revoke; every client +authenticates again and blocked grants are refused there. + +**No re-stamping.** A changed grant (Ann moves from read to write) is a revocation of the old grant. Her link closes, +her tab reconnects and authenticates with the new grant revision, which gives a new principal key, and her references +re-hold by themselves, now as an editor. Reconnecting never restores revoked authority. + +**Revocation beats jobs** (R11). Normally a running job keeps its object alive after the last reference is released +(a closed tab does not abort an import). Revoking a principal ends its objects at once regardless; their jobs get the +abort signal and the normal close timeout. + +## Work in shared stores + +Revocation ends objects whose principal carries the label. A shared store has no principal, so work it runs *on +behalf of* someone (a question waiting for Ann's answer, a shell command a guest started) is not ended by `revoke`. +Such a store keeps the grant as data with the work and reconciles from the grant owner's live state with a routine +(chapter X): + +```ts +// in a shared store that runs work for guests +const sharing = self.get(Grants, {}); // the sharing plugin's live grants +self.routines.on(sharing.live, async (live, ctx) => { + for (const run of shell.running.value) + if (run.args.grant && !live[run.args.grant]) shell.abort(run.id); // idempotent +}); +``` + +There is no revocation hook: a hook can be missed (a crash between revoke and hook), a routine over current state +cannot. The window between the revocation and the routine's run is accepted (R10); a step started in it still runs. + +## Who does what + +| Chord | The app | +| --- | --- | +| stamps principals on links | what a grant is; share links, cookies, `authenticate` | +| keeps one object per principal key | defines `Revocation` and `revokedBy` | +| on `revoke(r)`: ends matching objects and links, passes `r` down | decides when to revoke; stores who is blocked | diff --git a/packages/chord/docs/next/explainer/src/08-derives.md b/packages/chord/docs/next/explainer/src/08-derives.md new file mode 100644 index 000000000..720542ef5 --- /dev/null +++ b/packages/chord/docs/next/explainer/src/08-derives.md @@ -0,0 +1,114 @@ + +# Computed Values {#derives lede="Derives, the one footgun that makes them expensive, mapShared, and presence: who is looking."} + +The reader's notes without authors, a note count, the number of people viewing: values computed from other values. +Storing copies would go stale; recomputing them on every read would be wasteful and could not be watched. A +**derive** is computed once per change and sent to viewers as a difference. + +## `self.derive` + +```ts +const count = self.derive("count", [list], ([l]) => l.items.length); +const summary = self.derive("summary", [list, store.pinned], ([l, pinned], previous) => ({ + total: l.items.length, + pinned: pinned.length, +})); +``` + +`self.derive(name, sources, (values, previous) => value)`. Sources can be documents, readables, other derives and +state members of references. The value is recomputed when a source changes and diffed into operations for the wire. +Derived values are adopted before ordinary observers run and share their source commit's wire batch, so a viewer +never sees a note list and a count from different commits. + +`previous` is the derive's last value, kept across factory re-runs (reloads) by name. It is a cache, empty after a +host teardown, so a derive must be correct without it. A derive that throws fails its whole owner `broken`. + +## How the difference is found + +Chord compares the new value with the previous one using Delta's `diffRevisions`. Parts that are the **same objects** +as last time cost one `===` each. Parts that are new objects must be compared by content. + +**The footgun.** A derive that builds new objects for unchanged data makes every commit cost the size of the whole +value, per view. The reader list from chapter IV, written naively: + +```ts +self.derive("public", [list], ([l]) => ({ items: l.items.map((n) => ({ id: n.id, text: n.text, at: n.at })) })); +// one note added to 2,000: 2,001 new objects, every one compared by content +``` + +**`mapShared(f)`** keeps one result per source element: `f` runs only for new or changed elements, unchanged ones +return their previous result object. + +```ts +import { mapShared } from "@earendil-works/chord/delta"; + +const stripAuthor = mapShared((n: Note): PublicNote => ({ id: n.id, text: n.text, at: n.at })); +self.derive("public", [list], ([l]) => ({ items: l.items.map(stripAuthor) })); +// one note added: one new object, 2,000 identical references; the diff emits one insert +``` + +:::review | Review: mapShared is planned, not available +Design §12 introduces `mapShared` as coming "from `@earendil-works/chord/delta`", but nothing in `packages/chord/src` +defines it yet, and Delta does not yet treat bytes as atomic leaves in `diffRevisions`. Design §28 lists both as the +step "Delta in pi", before the layered rebuild. Until then, keep a `Map` from source element to result yourself, which +is what `mapShared` will do. +::: + +## Cookbook + +| Case | How | +| --- | --- | +| select a field, look up by key, a small aggregate | compute it; the value is small | +| filter or map a list | `mapShared` over the elements | +| redaction | preferably separate documents per visibility; otherwise `mapShared`, computed once in the shared store, never per view | +| join two sources | `mapShared` plus the looked-up source's revision | +| a window over a large list | take a section of the source elements without rebuilding them; cache per-element measurements by identity; keep the window's start stable so changes arrive as appends | +| a large view combined from several sources | re-path the sources' operations, as Durable's `ConversationView` does; application code never interprets ops | + +**Container width** (R24): structural sharing is O(change) along a path, not across a container's width. Changing one +item of a flat array copies the array's slots, and `mapShared` visits every element once (one identity comparison +each). Large collections are keyed documents or bounded windows. A benchmark of long arrays is to set the documented +limit. + +## Readables that are not documents + +Three small readables round out the toolkit: + +- `constant(value)`: never changes. Start data uses it (chapter IX). +- `self.writable(initial)`: a host-local readable of any in-process value (functions included), compared by identity, + with `set`. It never goes on the wire; it exists for registry contributions that change (chapter XIII). +- Source readables from elsewhere: a Durable view or document watch is already a Chord readable, because Durable and + Chord v2 share the readable type (`chord/src/state.ts`), so it can feed a derive or be exposed as state directly. + +:::review | Review: exposing a writable (Q14, open) +A `writable` may hold functions; nothing yet stops a factory from returning one as a state member, which would reach +the wire encoder. Review round 3 proposes that only members with a JSON schema in the contract are exposable, checked +when the object is served. Undecided. +::: + +## Presence: who is looking + +`host.references(Type)` is a readable of who holds references to objects of a type on this host: input, principal and +client. Client holds carry an opaque origin id, so two tabs are two clients. + +```ts +host.serve(NoteStore, ({ session }, self) => { + const viewers = self.derive("viewers", [host.references(Notes)], ([refs]) => + refs.filter((r) => r.input.session === session).length); + return { …, viewers }; +}); +``` + +A link drop is a departure: accurate and simple. Smoothing (a phone switching networks makes the count flicker) is the +UI's job, for example a debounced count. Counts per person are derived by the app from principals and origins. There +is no consumer-side linger: a closed tab closes its link and keeps nothing. The provider's `linger` keeps a +just-closed object warm, and that also keeps the worker alive through `host.activity`. + +**"Leaving" is never derived from an unmount.** A UI reload unmounts and remounts components; a component unmounted by +a reload has not left. Leaving is explicit navigation, or presence (a dropped reference). This replaces p2's presence, +viewer and login-abandon leases with one mechanism. + +:::note | Built so far +Readables and derives are layer L2 (a readable is today's `replicatedState`, derives reuse today's derived cells over +it). Not started in the rebuild. +::: diff --git a/packages/chord/docs/next/explainer/src/09-natives.md b/packages/chord/docs/next/explainer/src/09-natives.md new file mode 100644 index 000000000..ed7dacbbf --- /dev/null +++ b/packages/chord/docs/next/explainer/src/09-natives.md @@ -0,0 +1,160 @@ + +# Things the OS Gives You {#natives lede="Start data, host-lifetime objects and natives: how a plugin opens a file watcher or a socket so that it is closed exactly once, also across reloads."} + +Studio's notes get an inbox: a text file dropped into the session's `inbox/` directory becomes a note. That needs a +file watcher, a **native**: an OS-level value (a file handle, a socket, a process, a database connection) that cannot +be JSON, cannot be copied, must be opened, and must be closed exactly once. It also needs the session's data +directory, which the plugin does not know. This chapter covers both. + +## Start data is an object + +A worker knows things before any plugin loads: its session id, the data directory, init data from the server's start +message. There is no special mechanism for them. Start data is an ordinary `local` object with `lifetime: "host"` and +constant state, served by base code's core facet: + +```ts +// studio/src/core/contract.ts +export const Init = obj({ id: str(), dataDir: str(), cwd: str(), env: opt(str()) }); +export const SessionInfo = local("studio.session-info", { + version: 1, + input: obj({}), + members: { init: Init }, +}); + +// studio/src/core/worker.ts +export const coreWorkerFacet = (init: Static) => defineFacet((host) => { + host.serve(SessionInfo, () => ({ init: constant(init) }), { lifetime: "host" }); + // … the Harness and other core objects (chapter XIV) +}); +``` + +Any plugin on the worker declares a reference and reads it **after activation**: in an opener, a derive, a routine or +a method, never synchronously in the factory. + +```ts +const info = self.get(SessionInfo, {}); +// later: info.init.value.id, info.init.value.dataDir +``` + +Paths are derived by the plugin from generic start data (`join(dataDir, id, "notes-inbox")`), never provided per +plugin. Platform bindings work the same way: on a Durable Object, a core object's `self.open` returns the binding +(`state`, `env`) and its close does nothing. + +## `self.open` + +```ts +const watcher = self.open( + async ({ signal }) => watchDir(path, { signal }), // open: during activation + (w, reason) => w.close(), // close: when the object ends or restarts + { closeTimeout: 10_000 }, // optional; host default 30 s +); +watcher.value; // the watcher; reading it before activation throws not_ready +``` + +`self.open` returns a handle with `.value` (type `Native`), not a proxy. The rules: + +- **Natives live only in objects.** Not at module level, not in the facet. A module-level watcher would survive a + reload with nobody to close it. +- **Order:** natives open one after another in declaration order (a later opener may read an earlier handle's + `.value`) and close in reverse. The object goes live after all opens succeeded. +- **The value never changes during an attempt.** No replacement, no versions, no stale-handle logic. To change a live + native, the object restarts. +- **A failed open** makes the object `failed`: transient with backoff by default, broken if the opener says so + (`throw broken(error)`). A partly failed activation closes the natives already open, with reason `end`; the next + attempt opens all of them anew. +- **`close` gets a reason:** `{ kind: "reload" }` (the plugin reloads and this native object restarts), `{ kind: + "stop" }` (the host shuts down) or `{ kind: "end" }` (the last reference ended, or the type was removed). Each + `close` is bounded by `closeTimeout`, then reported as a possible leak while the rest continues. A throwing `close` + does not stop the others; `host.close()` collects the errors, background ends report to `onError`. +- **Behaviour around a native goes into routines** (chapter X). Listeners on natives outside routines are forbidden. + +An object with at least one `self.open` is a **native object**; one without is a **pure object**. The difference +decides what a reload does (chapter XV): pure objects are swapped in place, native objects restart. + +## The inbox + +The inbox must run as long as the worker, whether anyone looks at the notes or not. So it is a `lifetime: "host"` +object. The facet cannot `self.get` a type it serves, and the inbox has no meaningful input, so it is an object +without input that reads its session from the start data: + +```ts +// notes/contract.ts +export const NoteInbox = local("notes.inbox", { version: 1, input: obj({}), members: {} }); + +// notes/worker.ts +host.serve(NoteInbox, (_input, self) => { + const info = self.get(SessionInfo, {}); + const watcher = self.open(async ({ signal }) => { + const { id, dataDir } = info.init.value; // read in the opener: after activation + return watchDir(join(dataDir, id, "notes-inbox"), { signal }); + }, (w) => w.close()); + + self.routines.loop("import", async (ctx) => { // chapter X + const store = await ctx.get(NoteStore, { session: info.init.value.id }); + for await (const file of watcher.value.files(ctx.signal)) { + await store.add({ text: await readFile(file, "utf8"), by: "inbox" }, ctx); + await rm(file); + } + }); + return {}; +}, { lifetime: "host" }); +``` + +Note `ctx.get(NoteStore, …)` in the routine rather than `self.get` in the factory. The factory runs before the +session id is known (start data is read after activation), so a host-lifetime object without input gets +session-scoped objects with `ctx.get` in a routine or method (R22). A `get` whose input is computed after activation is +to be added only if a real case needs it. + +## Natives with work in flight + +A restart is sequential (R16): close the old native, then open the new one. When the old one still has work running, +use one of two patterns: + +- **Drain in the background.** `close` detaches the old resource and returns at once; it disposes of it when its work + finishes, or after `closeTimeout` at most. Chord opens the new one immediately. Code mode's sandbox does this: a + script running during a reload finishes on the old sandbox (for up to 30 s, Q7; a longer one is aborted and its turn + fails), new scripts start on the new one. +- **Swap through a registry.** The resource is a registry contribution (chapter XIII). Readers always look up the + current one at use; a reader that must wait for one uses a cancellable "wait until present" with the park deadline. + +Chord aborts only the object's own calls and jobs on close. Other work that uses the native (Durable tool calls on a +sandbox) is the owner's business: `close` may drain instead of tearing down. What an old native does after its owner +restarts is also the owner's `close`: a model runtime keeps working until collected, a closed SSH connection fails its +in-flight `exec`. + +## Natives that follow a relocation + +A session can move to another directory. The inbox watcher must follow. Restarting the object on every move would be +heavy-handed; instead the native is a stable manager opened once, and a routine on the session location switches its +internal resource and bumps a generation, so late events from the old location are dropped: + +```ts +const watcher = self.open(async ({ signal }) => new SwitchableWatcher({ signal }), (w) => w.close()); +self.routines.on(location.cwd, (cwd) => watcher.value.switchTo(join(cwd, ".notes-inbox"))); +``` + +## Call-scoped natives + +A native that lives exactly as long as one call (a temporary SSH session for a directory listing, a file handle for a +download) is opened and closed inside the method with `try/finally`. There is no `ctx.open`: + +```ts +list: async ({ path }, ctx) => { + const session = await ssh.open(config, ctx.signal); + try { return await session.readDir(path, ctx.signal); } + finally { session.close(); } +}, +``` + +## Core vs. plugin + +`SessionInfo` comes from base code, `NoteInbox` from a plugin; both use the same primitives. Where code lives decides +which it is. **Base code** (the app's `src/`, imported statically by the start code under a fixed name such as +`@studio/core`) is always there, not switchable, and a change restarts the hosts. **A plugin** (its own directory, +found by listing, named by its `package.json`) is switchable and reloads. Core is what the app always needs and +everything builds on: the harness, the session list, access, the session location. Optional features are plugins; +code mode is a plugin. + +:::note | Built so far +Natives and host-lifetime objects are layer L4. `constant` is decided (R23), not built. +::: diff --git a/packages/chord/docs/next/explainer/src/10-routines.md b/packages/chord/docs/next/explainer/src/10-routines.md new file mode 100644 index 000000000..4191f21d0 --- /dev/null +++ b/packages/chord/docs/next/explainer/src/10-routines.md @@ -0,0 +1,145 @@ + +# Standing Behaviour {#routines lede="Routines, reconciling instead of notifying, and how two owners keep a rule between them without a shared transaction."} + +Notes older than 30 days should go every hour. Inbox files should be imported as they arrive. Notes of an archived +session should become read-only. Each of these is behaviour that stands, not a call someone makes. Written as bare +`setInterval`s and event listeners, it would keep running old code after a reload and touch state from nowhere. Chord +has one declared form for it: the **routine**. + +## Four kinds + +```ts +self.routines.every("cleanup", HOUR, (ctx) => dropOlderThan(30 * DAY, ctx)); // due time survives reloads +self.routines.on(settings, async (value, ctx) => applyLimit(value, ctx)); // a readable or event source +self.routines.loop("import", async (ctx) => { // runs until the object ends + while (!ctx.signal.aborted) await importNext(ctx); +}); +self.routines.custom("ticker", ({ signal, run }) => { // anything else + const timer = setInterval(() => run(tick), 1000); + return () => clearInterval(timer); // cleanup undoes the setup +}); +``` + +Routines start when the object goes live, stop when it ends, and restart with the new code on every reload of their +plugin. Each run gets its own `ctx`; errors are caught and reported, never uncaught. Routines are named (`on` +optionally by its source) so Chord recognizes them across reloads; that is how `every` keeps its due time. + +## Serialized per routine + +The new instance of a routine starts only after the old handler in flight has settled; only that routine waits, for at +most the routine handover timeout (10 s), after which it starts anyway and the stuck handler is reported. + +The reason, as a trace: a supervisor routine reconciles pending worker restarts. The old handler is midway through +`restart(W1)` when the plugin reloads. Without serialization the new routine starts at once, sees W1 still pending, and +restarts it a second time. + +## Reconcile, don't track + +A routine **reconciles from the current value**: it makes the world match what it sees now. It writes no "in +progress" flags, and it starts a job for long work (chapter XI). A handler that only cares about the latest value +compares with the current value itself; there is no `overlap` option. Subscribing to the object's own documents is +gapless: every commit is seen. + +No bare timers or callbacks that touch state outside routines, jobs or calls. That is what lets Chord stop all of an +object's behaviour when it ends or reloads. + +## Rules between two owners + +Every thing has exactly one owner, the smallest object it belongs to. Some rules span two: "notes of an archived +session are read-only" involves the catalog (which knows archiving) and `NoteStore` (which holds notes). Two tempting +designs are wrong: + +- **A cross-owner transaction** (archive and lock the notes atomically) would make the catalog know every plugin and + couple their storage. +- **A notification** (`catalog` calls `notes.archive(session)`) can be lost: the worker is not running, the link drops, + the process crashes between the archive and the call. + +The design's rule: **the owner of the dependent data enforces the rule** by reconciling from the other owner's state +with an idempotent routine. + +```ts +// NoteStore: an archived session's notes are read-only +const flags = self.state(NoteFlags); // { archived: boolean }, NoteStore's own document +const catalog = self.get(Catalog, {}); // the server's session list: another owner +self.routines.on(catalog.sessions, async (sessions, ctx) => { + const archived = sessions[session]?.archived ?? false; + if (flags.value.archived !== archived) await flags.change((d) => { d.archived = archived; }, ctx); +}); +// add and remove refuse while flags.value.archived +``` + +After a crash the routine runs again with the current value. The catalog does not know the notes plugin exists. The +same pattern closes a forward of an archived session in the ports plugin, aborts a job whose grant was revoked +(chapter VII), and keeps any invariant between plugins. + +## Reading without starting a worker + +The session list in the sidebar shows each session's note count. Getting `NoteStore { S1 }` for every session would +start every worker. The answer is plain publishing between two objects of one plugin, no Chord mechanism: the worker +side reports a summary to a server-side object by a method call from a routine, and clients read the server object. + +```ts +// notes/contract.ts +export const NoteCounts = remote("notes.counts", { + version: 1, input: obj({}), placement: () => Server(), + variants: { + owner: { state: { counts: map(int()) } }, + worker: { methods: { report: method(obj({ session: str(), count: int() })) } }, + }, +}); + +// notes/server.ts +const Counts = defineState("notes.counts", { + schema: obj({ bySession: map(obj({ count: int(), incarnation: str() })) }), + version: 1, initial: () => ({ bySession: {} }), persist: true, // survives a server restart + checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64, +}); + +host.serve(NoteCountStore, (_input, self) => { // local, lifetime "host" + const counts = self.state(Counts); + return { + counts: self.derive("counts", [counts], ([c]) => mapValues(c.bySession, (e) => e.count)), + report: ({ session, count, incarnation }, ctx) => + counts.change((d) => { + const last = d.bySession[session]; + if (last && last.incarnation > incarnation) return; // uuidv7: an older worker, ignore + d.bySession[session] = { count, incarnation }; + }, ctx), + }; +}, { lifetime: "host" }); + +host.serve(NoteCounts, (_input, principal, self) => { + const store = self.get(NoteCountStore, {}); + if (principal.kind === "worker") return { + variant: "worker", + report: ({ session, count }, ctx) => { + if (session !== principal.session) throw forbidden(); // a worker reports only itself + return store.report({ session, count, incarnation: ctx.incarnation }, ctx); + }, + }; + return { variant: "owner", counts: store.counts }; +}); + +// notes/worker.ts, inside NoteStore +const counts = self.get(NoteCounts, {}); // a reference to the server +self.routines.on(count, (n, ctx) => inVariant(counts, "worker") && counts.report({ session, count: n }, ctx)); +``` + +The server object keeps the latest value per session (in a document if it must survive a server restart) and +**ignores reports from an older worker incarnation** (incarnations are uuidv7, so they order by start time). A worker that was replaced may still deliver a late report; the +fence is the caller's incarnation, which Chord stamps on every call from the link handshake (R20): `ctx.incarnation`, +next to `ctx.principal`, not part of the principal or its key, never sent as data, so it cannot be forged. Session +status, subagent lists, unread marks and artifact lists in p2 all follow this pattern. The store is local and has no +caller incarnation of its own, so the view passes it on as data. + +:::review | Review: `ctx.incarnation` across a relay (Q13, open) +A worker's call to the server arrives directly, so `ctx.incarnation` is the worker's. But a call relayed through the +server (a tab's call to a worker) carries the server's incarnation on the worker's side, because the worker's +handshake is with the server. Review round 3 proposes that relays stamp the origin host's incarnation, as they stamp +the principal. Undecided; the pattern above only needs the direct case. +::: + +:::note | Built so far +Routines are layer L8. The design treats `every` as one of four forms over one runner; review round 3 proposed cutting +to `on` and `loop` only (C4), not decided. +::: diff --git a/packages/chord/docs/next/explainer/src/11-jobs.md b/packages/chord/docs/next/explainer/src/11-jobs.md new file mode 100644 index 000000000..e506c05df --- /dev/null +++ b/packages/chord/docs/next/explainer/src/11-jobs.md @@ -0,0 +1,171 @@ + +# Work That Outlives the Call {#jobs lede="Jobs: owned work started by a method that keeps running after its caller leaves, with cancel, dedupe and a recorded outcome."} + +Importing a 50 MB text file into notes takes a minute. The tab that started it may close; the import must go on, be +cancellable, not run twice when the tab retries after a link blip, and its result must be recorded. A method cannot do +this (it dies with its caller), a routine is standing behaviour rather than a request, and a Durable task is for work +that must resume after a crash. Chord's form for it is the **job**, built in pi-durable's sandwich: **intent** (a +transaction), **effect** (anything), **outcome** (a transaction in the current code). + +## Defining and running a job + +```ts +// notes/worker.ts +const ImportJob = defineJob("notes.import", { + version: 1, + args: obj({ path: str() }), + progress: obj({ lines: int() }), + result: obj({ added: int() }), + concurrency: 1, // one import at a time per NoteStore +}); + +const ImportResults = defineState("notes.import-result", { + schema: obj({ outcome: union([obj({ added: int() }), obj({ error: str() })]) }), + version: 1, initial: () => ({ outcome: { added: 0 } }), keyed: true, + checkpointWhen: () => true, +}); + +host.serve(NoteStore, ({ session }, self) => { + const list = self.state(NoteList); + const results = self.state(ImportResults); + const imports = self.job(ImportJob, { + run: async ({ path }, ctx) => { // 2. effect + let lines = 0; + for await (const line of readLines(path, ctx.signal)) { + await addNote({ id: `${ctx.jobId}:${lines}`, text: line, by: "import" }, ctx); // keyed by job id + ctx.progress({ lines: ++lines }); + } + return { added: lines }; + }, + finish: async (outcome, tx) => { // 3. outcome, current code + (await tx.draft(results, outcome.id)).outcome = + outcome.kind === "done" ? outcome.result : { error: outcome.error }; + }, + }); + return { + …, + imports: imports.running, // live list of runs + startImport: ({ path, requestId }, ctx) => + imports.start({ path }, ctx, { key: requestId }), // 1. intent (refined below) + abortImport: ({ id }) => imports.abort(id), + }; +}); +``` + +`defineJob(name, { version, args, progress, result, concurrency?, persist?, accepts? })`; `progress` and the outcome +are typed by the schemas. + +:::sequence animate | The job sandwich. The intent commits before anything runs; the effect runs outside any transaction; the outcome commits in the current code. +actor tab "tab" +actor store "NoteStore { S1 }" accent +actor run "run" "effect" blue +tab -> store: startImport({ path, requestId }) +store => store: 1. intent: busy check, dedupe by key, commit +store --> tab: { id } +store -> run: start after the commit +run -> run: read lines, add notes (keyed by jobId) +run --> store: progress { lines } (throttled) +note tab..store: the tab may close here: the job counts as a reference +run --> store: settles: { kind: "done", result } +store => store: 3. record outcome, finish(outcome, tx), ack +::: + +## The three steps + +**1. Intent.** `job.start(args, ctx, { key? })` is shorthand for `tx.start(job, args, { key? })` inside a +transaction, which may contain other writes. The busy check (`concurrency`) and deduplication (the same key returns the +existing run) are atomic and in the same commit; `run` starts only after the commit. Run ids are uuidv7. + +**2. Effect.** `run(args, ctx)` may do anything: processes, network, other objects. It writes no document itself; it +returns the outcome. External writes carry a dedupe key derived from `ctx.jobId` (above: note ids), so an interrupted +attempt cannot apply them twice. On a reload `run` keeps its old code. + +**3. Outcome.** When `run` settles, Chord records the outcome in its own record, then calls the **current** code's +`finish(outcome, tx)` and acknowledges the run in the same commit. The outcome is tagged: + +```ts +type Outcome = { id: string } & ({ kind: "done"; result: R } | { kind: "failed"; error: "threw" | "aborted" | "lost" }); +``` + +A throwing `finish` changes nothing and is retried with a backoff (1 s to 60 s), and after the next reload. `finish` +is optional and follows Durable's transaction rule. + +## Dedupe ends with the run + +Deduplication covers **unsettled runs only** (Q9). Once `finish` has committed, Chord drops the run's record and key +and releases its waiters' references; it keeps nothing beyond a run's lifetime. Dedupe after completion is the +owner's. Studio records which run a request started, in the same commit as the intent, and `finish` records the +result by run id (`ImportResults` above): + +```ts +const Requests = defineState("notes.import-request", { + schema: obj({ run: str() }), version: 1, initial: () => ({ run: "" }), keyed: true, persist: true, + checkpointWhen: () => true, +}); + +startImport: async ({ path, requestId }, ctx) => { + const prior = await requests.get(requestId); // read outside the transaction (R13) + if (prior) return { id: prior.run }; // started before, maybe finished long ago + return self.tx(async (tx) => { + const run = await tx.start(imports, { path }, { key: requestId }); // dedupes a concurrent retry + (await tx.draft(requests, requestId, { tags: [SessionTag(session)] })).run = run.id; + return { id: run.id }; + }, ctx); +}, +``` + +Two concurrent retries both miss `prior`, but the key makes `tx.start` return the same unsettled run. The client +watches `imports` for progress and reads the result by run id once it is gone from `running`. + +Why Chord does not keep completed runs itself: either 100,000 keyed imports would stay in memory forever, or a +retention limit would let a retried key rerun its effect. The owner knows which results matter. + +## Watching, waiting, aborting + +- **`running`** is a readable of `{ id, version, args, progress, phase }`. `phase` is `running`, `finishing` (outcome + recorded, `finish` not yet committed) or `blocked` (with the reason). `concurrency` counts only active runs. The + panel shows a progress bar from it. +- **Progress** is in memory, throttled and sent as deltas. For output, use `OutputBuffer` and `Progress` from + `@earendil-works/chord/output`: bounded windows, adaptive throttling, no write amplification. +- **Waiting:** `start` returns a run with `outcome(ctx)`. The caller's abort ends only its wait, never the run. +- **Abort is a request:** `job.abort(id)` fires the run's `ctx.signal`; the run decides what it returns (a result + that already arrived stays a success). A run that ignores the signal keeps its slot until it ends. `finish` runs in + Chord's own context. + +## Lifetime, persistence, revocation + +**Jobs keep their owner alive.** A running, finishing or blocked job counts as a reference to its object, which ends +only once all its jobs are settled. Starting an import and closing the tab does not abort it. Unsettled runs also +count as activity for the worker's keep-alive. + +**Revocation takes precedence** (R11). Revoking a principal ends its objects at once, whatever jobs remain; the jobs +get the abort signal and the normal close timeout. A job runs as its object's principal. + +**Persistence:** jobs are in memory by default and gone with the process. `persist: true` (only for stable +identities) keeps the record: a run live at a crash ends `lost` at the next activation and `finish` records it. +Nothing is rerun automatically. + +**Versions:** a run started under an older job version keeps its version; `finish` and `running` see it through pure +`accepts` adapters, and stored outcomes are never rewritten. A version bump without an adapter does not refuse the +reload: the old outcome is `blocked`, kept and reported until a later reload brings the adapter. + +**Native objects:** a job in a native object ends `aborted` when the object restarts on a reload (chapter XV). In a +pure object it keeps running on its old `run` code. + +## What is not a job + +| Work | Use | +| --- | --- | +| request and response that may die with its caller | a method | +| input into running work (a pasted code, an approval) | a flow (chapter XII) | +| work that must resume after a crash | a Durable task | +| standing behaviour | a routine | + +:::review | Review: jobs are the largest open simplification (#9, C3) +Decision #9 (deferred until login is ported): a login or a long relocation is a job whose state consumers want to +address and watch (`LoginFlow { id }`). Options: (a) jobs stay records inside their owner, watched through `running`; +(b) a job is an object of its own, addressed by its id, living until done. Separately, review round 3 proposed cut C3: +shrink jobs to a runner over documents (no job versions, `accepts`, `blocked`, `persist`) or move them out of the +kernel. Not decided. `chord-next-walkthrough.md` concept 17 still shows the older outcome shape `{ ok: true, result }`; +the design tags outcomes by `kind`. +::: diff --git a/packages/chord/docs/next/explainer/src/12-flows.md b/packages/chord/docs/next/explainer/src/12-flows.md new file mode 100644 index 000000000..cf6a986c1 --- /dev/null +++ b/packages/chord/docs/next/explainer/src/12-flows.md @@ -0,0 +1,141 @@ + +# Asking a Person {#flows lede="Flows and the Questions service: how running work waits for an answer from someone who may be on any tab, may reconnect, and may answer twice."} + +A provider login asks for a pasted code, then an account choice. A dangerous tool call waits for the owner's +approval. The agent asks everyone present a question. In each case running work waits for a person, who may be on any +tab, may lose the connection halfway, may answer late or twice. Two patterns cover this: a **flow** kept as its +owner's state, and the **Questions** service, a shared store for questions. + +## A flow is state, not a call + +The tempting design is a long-held server call, `step()`, that blocks until the next step. A link drop fails it, and +the consumer cannot tell whether it missed a step. Chord's answer is that the owner keeps the flow as **state**, which +re-holds by itself after any drop: + +```ts +// models/contract.ts (sketch from the design) +start: method(obj({ provider: str() }), obj({ id: str() })) // uuidv7 +answer: method(obj({ id: str(), step: int(), value: str() }), Answered) // ok | "step no longer active" +cancel: method(obj({ id: str() })) + +// LoginFlow { id }: state +type FlowState = + | { kind: "running"; step?: { id: number; prompt: LoginPrompt }; notices: Notice[] } + | { kind: "done" | "failed" | "cancelled"; notices: Notice[] }; +``` + +The UI watches the state and renders the current step. A late or second answer to a step that is gone is refused. A +promise-based callback inside the work (pi-ai's `prompt` during a login) stays **inside** the owner: its job keeps the +resolver, and nothing crosses an object boundary. + +Imperative consumers (tests, scripts, a CLI, an agent tool) use a generic client helper over the same state, not a +second protocol: + +```ts +async function* steps(flow: LoginFlowRef, ctx: Ctx) { + let last = 0; + for (;;) { + const s = await until(flow.state, (v) => v.kind !== "running" || (v.step !== undefined && v.step.id > last), ctx); + if (s.kind !== "running") return s.kind; + last = s.step!.id; + yield s.step!; + } +} + +for await (const step of steps(await ctx.get(LoginFlow, { id }), ctx)) + await login.answer({ id, step: step.id, value: answerFor(step) }, ctx); +``` + +`last` keeps steps from being lost or repeated across re-holds. + +:::review | Review: where a flow lives (#11, deferred) +Decision #11, deferred until login is ported: (a) a keyed document of its owner, or (b) an object `LoginFlow { id }` +living until done; tied to #9 (jobs as objects). The `LoginFlow { id }` address above assumes (b). Also undecided is +how "leaving" ends a login (p2 cancels when the starting owner leaves everywhere); with presence it is a dropped +reference, never an unmount (chapter VIII). +::: + +## The Questions service + +The Questions service (`apps/p2/docs/questions.md`) is a p2 design, not a Chord feature, built from Chord's +primitives: a shared store with per-principal views. The asker gets `ask`; viewers get the open questions and +`answer`. + +```ts +export type Question = + | { kind: "text"; message: string; secret?: boolean; help?: string } + | { kind: "select"; message: string; options: readonly { id: string; label: string }[] } + | { kind: "confirm"; message: string; allow: string; deny: string }; + +type Policy = { mode: "first" } | { mode: "quorum"; count: number } | { mode: "all"; join?: "live" | "snapshot" }; + +ask(question, { + audience, // a readable or a list of principals + policy, + key?, // stable per asking flow: a re-ask reattaches to the pending question + deadline?, // then resolve with what is there; `missing` names who did not answer +}, ctx): Promise; +``` + +Its rules: + +- **The store is the truth.** Pending questions and their resolvers live in the store; `open` is a readable over it, + filtered per viewer. Nothing is a document, so a reload or migration cannot leave a ghost question. +- **One settlement point.** Register the waiter and the abort listener before publishing the question; an answer, an + abort or the deadline atomically takes the entry out before resolving. A late or second answer gets "no longer + active". Answers are validated against the stored question (a select id must be one of the options). Never await an + answer inside a transaction. +- **Policies over a live audience.** `first` resolves with the first answer; `quorum(n)` after n answers; `all` when + every member of the audience has answered. With `join: "live"` (the default), someone who joins becomes required and + someone who leaves drops out. +- **Authorization.** The asker is stamped by the host (a worker asks only in its own session's scope; secret questions + only from server askers). Each question stores who may answer it; a guest's send access does not imply approval + rights. Secret answers never appear in `open`, documents or logs. +- **Link drops and crashes.** A rejected `ask` is not a denial. Askers retry with the same `key` until their own signal + aborts; a pending `ask` counts as an active call for keep-alive. +- **Where it lives.** On the host where the asking work runs: session questions on the session's worker, login + questions on the server. UIs reach both through the usual relay. + +The store in Chord v2 terms: + +```ts +host.serve(QuestionStore, ({ scope }, self) => { + const pending = self.open(async () => new PendingQuestions(), (p) => p.rejectAll("store closed")); + const open = self.derive("open", [pending.value.readable], ([all]) => all); // data only, no resolvers + return { open, ask: (args, ctx) => pending.value.ask(args, ctx), answer: (args, ctx) => pending.value.answer(args, ctx) }; +}); + +host.serve(Questions, ({ scope }, principal, self) => { + const store = self.get(QuestionStore, { scope }); + if (isHost(principal)) return { variant: "asker", ask: (args, ctx) => store.ask({ ...args, asker: principal }, ctx) }; + if (!mayView(principal, scope)) throw forbidden(); + return { + variant: "viewer", + open: self.derive("mine", [store.open], ([all]) => openFor(all, principal)), + answer: ({ id, value }, ctx) => store.answer({ id, value, by: principal }, ctx), + }; +}); +``` + +The pending map with its resolvers is in-process state with promises, so it is a native of the store (opened and +closed with it). Its `close` rejects every waiter; askers retry with their key against the new store after a reload. + +:::review | Review: questions.md predates the v2 rules +`apps/p2/docs/questions.md` writes the store as `self.resource(...)` with a `replace` hand-over, which the design +rejected ("resources replaced in place"), and opens Chord call scopes in Durable hooks with `host.root`, which Q6 +replaced with `chord(fn, context)` (chapter XIV). Design §20 says the Questions store "must live in an object (it +predates natives only in objects)". The sketch above is this guide's translation, with the derive per viewer; filtering +per viewer in the store, once per variant, may be cheaper for large audiences. The `ask` signature passes typed +objects as arguments, which is fine (data), but the `audience` readable form cannot cross a link: across links, +audience must be data (principals) or computed by the store. +::: + +## Choosing + +| Case | Use | +| --- | --- | +| a multi-step login in one owner | a flow: the owner's job keeps the resolver, state shows the steps | +| a tool needs the owner's approval | Questions, `first`, audience the owner | +| the agent asks everyone present | Questions, `all`, audience from presence | +| deploy needs 2 of 3 approvals | Questions, `quorum(2)` | +| a notice (auth URL, device code, progress) | not a question: the asking work's own progress state | diff --git a/packages/chord/docs/next/explainer/src/13-registries.md b/packages/chord/docs/next/explainer/src/13-registries.md new file mode 100644 index 000000000..22efc7019 --- /dev/null +++ b/packages/chord/docs/next/explainer/src/13-registries.md @@ -0,0 +1,185 @@ + +# Extension Points {#registries lede="The facet, registries and the HTTP component: how plugins add behaviour to each other without callbacks, and how the owner stays in charge."} + +The notes plugin wants a download route, `GET /s/:session/notes.txt`. A logs plugin wants to time every request. +Several plugins add model providers, tools for the agent, commands for the TUI. Someone must combine these +contributions, decide conflicts, and replace one plugin's part as a whole when it reloads. Between plugins exactly two +things pass: object types and **registries**. + +## The facet, completed + +The facet has appeared since chapter II as the place that serves types. It is also itself a **pure object** with the +host's lifetime: + +```ts +// notes/server.ts +export default defineFacet((host, self) => { + const prefs = self.state(NotesPrefs); // plugin-wide documents + self.contribute(Http, { routes: { "GET /s/:session/notes.txt": exportNotes } }); + host.serve(NoteCountStore, …, { lifetime: "host" }); + host.serve(NoteCounts, …); +}); +``` + +- It may hold documents, references, routines and contributions; **no natives** (those need an object with its own + lifetime). +- Its identity is the plugin's name, so its persisted documents survive restarts and reloads. +- Served objects go live after the facet and may read and write its documents (chapter V). +- `host.serve` and `self.contribute` are valid only while the setup runs. The facet never `self.get`s a type it + serves. + +## Registries + +A registry is an extension point owned by someone (a Chord component, the app, a plugin). Each contributor +contributes **at most one value** per registry; the table is an immutable, ordered list of `(contributor, value)`; +the registry's `compose` turns it into the value consumers use. + +```ts +// owned by the HTTP component +export const Http = defineRegistry("chord.http", { + version: 2, + accepts: { 1: (old: HttpContributionV1): HttpContribution => upgrade(old) }, + compose: (table, { report }) => buildRouter(table, report), // pure; always delivers a value +}); + +self.contribute(Http, { routes: { … }, wraps: [ … ] }); // a contributor +const router = host.registry(Http).value; // a consumer: live +``` + +**`compose` runs at runtime, whenever the table changes:** after a reload's commit, when a contributor goes live or +ends, when a contribution changes. It is not a staging check: contributions can be dynamic (known only once a native +is open), so no check before a reload could be complete. `compose` always delivers a value with its fallback built in +(e.g. the first by plugin name wins) and reports conflicts through `report`, which become owner notices and telemetry +events. Readers get a plain value and never handle conflicts. Only if `compose` throws is the registry in an error +state, and readers get an error when they use it. Work that started earlier keeps the value it read. + +**Rules:** + +- **Who contributes** (R21): only facets and `lifetime: "host"` objects; only their `self` has `contribute` (a type + error elsewhere). Every object may read registries. +- **A contribution is in the table exactly while its contributor is live:** a facet's while its plugin is loaded; an + object's from `live` until it ends, fails or restarts. +- **Order** is sorted by plugin name: deterministic on every host and across reloads, and deliberately meaningless. + Meaningful order comes from the owner's composition (HTTP phases, below). Helpers: `keyed(table, entriesOf)`, + `ordered(table, itemsOf, orderOf?)`. +- **One authoritative declaration** (Q8): only the defining plugin's facet declares the registry to the host + (`compose`, version, `accepts`). Other plugins' tokens only identify it by name and types. A reload of the defining + plugin replaces the declaration atomically and the table recomposes. If the defining plugin is not loaded, the + registry has no value (readers get an error like a failed reference) and contributions wait in the table. +- **Versions** (Q8): a contribution carries the registry version its plugin was built against; older ones are adapted + by the defining facet's pure `accepts`. One without a matching version or adapter is left out and reported naming + plugin, registry and versions; its plugin keeps working. Contributions are in-process values (often functions) and + are not schema-checked. +- **Use sites guard contributions.** A contribution is untrusted code (third-party plugins are a goal). Every use site + catches throws and rejections when calling an entry, reports naming the contributor, and continues with the others. +- **The owner decides** conflicts, order and selection. Registries add behaviour; they never replace an owner's checks. + +## Natives through registries + +Registries are the only way a native reaches another plugin. An entry lives exactly as long as its contributor; +consumers look up at each use (or follow the composed value) and keep nothing from an entry beyond a use. The models +plugin's host-lifetime object contributes its runtime; the harness looks it up at every request. After a models reload +the next request gets the new runtime, while a running stream finishes on the old one. + +A handler that stores `const r = entry()` in a module variable and uses it for days has a bug: after the contributor +restarts it holds a closed native. + +## Contributions that change + +Some contributions are known only at runtime: MCP tools appear when a server connects. A contribution is either fixed +for its contributor's life (dynamic parts behind handles read at use) or a **readable**, which Chord subscribes to; +on every change Chord replaces that entry and runs `compose` again. For in-process values, `self.writable(initial)` is +a host-local readable of any value (no JSON, no wire, compared by identity) with `set`: + +```ts +const ext = self.writable(mcpExtension([])); // a Durable Extension, no tools yet +self.contribute(HarnessExtensions, ext); +self.routines.on(mcp.tools, (tools) => ext.set(mcpExtension(tools))); +``` + +There is no remove call. An entry leaves when its contributor ends, or after a reload whose new code no longer +contributes. A live contributor with nothing to offer sets its readable to the registry's empty value (an extension +without tools, empty routes); the table keeps exactly one entry per live contributor. + +## Optional dependencies + +Registries also replace optional references (chapter VI). Radius forwards connections to ports the ports plugin +manages, but ports is optional. So radius owns `radius.forwards` and ports contributes its forwards; without ports the +table is empty and radius works. + +## The HTTP component + +HTTP is an optional Chord component (`@earendil-works/chord/next/http`) built on a registry. A route gets a plain +request summary (method, path params, query, principal, provenance such as local or relayed, selected headers, parsed +and capped JSON bodies, uploads in chunks) and a `ctx` whose `principal` is the request's, so `ctx.get(Notes, …)` gives +exactly the view that person may see. The route returns a description; the component does the HTTP work once (ranges +with 206 and 416, HEAD, download headers with a safe filename, `nosniff`, CSP and its nonce, caching, streaming, abort +on disconnect): + +```ts +type HttpResponse = + | { kind: "json"; value: JsonValue } | { kind: "page"; title: string; body: string } + | { kind: "bytes"; name: string; mime: string; size: number; read(offset: number, length: number): Promise } + | { kind: "redirect"; to: string } | { kind: "error"; status: number }; + +// notes/server.ts +self.contribute(Http, { + routes: { + "GET /s/:session/notes.txt": async (req, ctx) => { + const notes = await ctx.get(Notes, { session: req.params.session }); // the requester's view + const text = new TextEncoder().encode(render(notes.notes.value)); + return { kind: "bytes", name: "notes.txt", mime: "text/plain", size: text.byteLength, + read: async (offset, length) => text.subarray(offset, offset + length) }; + }, + }, +}); +``` + +Routes compose with `keyed`: two contributors with one pattern is a reported conflict, and the first by plugin name +wins. **Wraps** are middleware in phases the owner defines (`observe`, `headers`, `transform`, outermost first), +within a phase in contributor order. No numeric priorities: plugins have no single registration order, so Fastify's +lifecycle phases and encapsulation are the model, not Express. + +```ts +// logs/server.ts +self.contribute(Http, { + wraps: [{ + name: "access-log", phase: "observe", scope: "all", // default scope "own": only own routes + wrap: async (req, next, ctx) => { + const t = Date.now(); + const res = await next(req); + log(req, res, Date.now() - t); + return res; + }, + }], +}); +``` + +`route` narrows a wrap to one route; a `route` that does not exist is a reported conflict. Wraps may add a security +boundary on top of an owner's checks, never instead of them. A request uses the router as it was when it started. +Request resolvers (who sent a request: cookie, loopback, relay) are hooks over the summary. + +## Channels and big bytes + +**Channels** are WebSocket byte channels (`send`, `onData`, `close`) for terminals, separate from object links and +with their own acknowledged windows, so a large state message never holds up keystrokes. + +**Big transfers are chunked by the owner,** not by Chord: bytes are a schema type (`bytes()`), carried natively +(never base64), and a 50 MB upload is a protocol of plain calls: + +```ts +beginUpload({ name, size }, ctx) → { upload } +append({ upload, offset, bytes }, ctx) // about 1 MiB per chunk; the awaited call is the backpressure +finishUpload({ upload }, ctx) → { id } +read({ id, offset, length }, ctx) → bytes +``` + +After a link drop the `append` in flight fails like any call; the writer asks the owner for the stored offset and +resumes. Unfinished uploads expire through the owner's routine. Chord has no stream type: it would need its own rules +for drops, cancellation, relays and restarts, which this offset protocol already survives. + +:::note | Built so far +Registries are layer L9. The HTTP component, channels and the chunked-transfer helper are deferred until after the +kernel and a first real proof (the harness, one tool with an approval pause, a code mode reload); until then the +playground serves its pages with plain app code. Their design stays the target. +::: diff --git a/packages/chord/docs/next/explainer/src/14-agent.md b/packages/chord/docs/next/explainer/src/14-agent.md new file mode 100644 index 000000000..40b69b555 --- /dev/null +++ b/packages/chord/docs/next/explainer/src/14-agent.md @@ -0,0 +1,192 @@ + +# The Agent {#agent lede="pi-durable inside Chord: the harness as a core object, models, environments and tools through registries, and how a tool reaches Chord objects and people."} + +Each Studio worker runs a pi-durable harness for its session. The harness must run for the worker's whole life, +whether anyone watches or not. Its models come from the models plugin, its execution environments from the envs +plugin, its tools from every plugin, and all of those reload independently. This chapter assumes you know pi-durable +and shows only how it plugs into Chord. + +## Shared foundations + +Durable and Chord v2 share modules: Durable imports Chord's `context`, `delta`, `json`, `output` and `sqlite`, and v2 +imports the same, so `Context`, `Draft` and `JsonValue` are one type each. A v2 readable is today's +`replicatedState` (`chord/src/state.ts`), so Durable's `viewState()` and document watches can be exposed as Chord state +members without conversion. Only plain data crosses between Durable and Chord objects. + +## The harness is a core object + +```ts +// studio/src/core/harness.ts (base code) +import { Harness as DurableHarness } from "@earendil-works/pi-durable"; + +export const ChordHostKey = createContextKey>("studio.chord-host"); +export const HarnessModels = defineRegistry("studio.harness-models", { version: 1, compose: firstModels }); +export const HarnessEnvironments = defineRegistry("studio.harness-envs", { version: 1, compose: firstEnv }); +export const HarnessExtensions = defineRegistry("studio.harness-extensions", { + version: 1, + compose: (table, { report }) => extensionSnapshot(table, report), // one Durable Extension per plugin +}); +const HarnessRuntime = defineRegistry<() => DurableHarness, () => DurableHarness>("studio.harness-runtime", { + version: 1, compose: onlyEntry, // core-private: one contributor +}); + +host.serve(Harness, (_input, self) => { + const info = self.get(SessionInfo, {}); + const harness = self.open(async ({ signal }) => { + const { id, dataDir } = info.init.value; + const base = withContextValue(ChordHostKey, host, withAbortSignal(signal, BACKGROUND_CONTEXT)); + return DurableHarness.open(await openNodeSqliteStorage(join(dataDir, id, "durable.sqlite")), { + models: () => host.registry(HarnessModels).value, // read at each use + env: (target) => host.registry(HarnessEnvironments).value(target), + settings: () => currentSettings(), + registry: durableRegistry(host.registry(HarnessExtensions)), // a RegistryReader over the composed value + }, base); + }, (h) => h.close(BACKGROUND_CONTEXT)); + self.contribute(HarnessRuntime, () => harness.value); // for core's conversation objects + return { … }; +}, { lifetime: "host" }); +``` + +- **Lifetime `"host"`:** it runs for the worker's whole life, also while nobody holds it, because Durable tasks + continue. +- **Base code:** it lives in the core facet, so a plugin reload never restarts it; only a base-code change (a host + restart) does. +- **Registries, looked up at each use:** Durable's `models` and `settings` accept functions (pi `7ec1f648e`), so each + generation gets the current value. After a models reload the next request uses the new runtime; a running stream + finishes on the old one. +- **Durable's storage is its own.** Chord documents never commit together with Durable transcripts. + +The design writes the call as `Durable.open(storage, { … })`; pi-durable's actual entry point is +`Harness.open(storage, options, context)`, used here as `DurableHarness` to keep it apart from the core object +`Harness`. + +## Start order + +At host start Chord loads every facet and waits until every facet and every `lifetime: "host"` object **other than the +harness** is live or failed: the **start barrier**. Then the registries hold the contributions of every plugin that +loaded, and the harness opens and resumes scheduling (R17). An object that must contribute at start, like the model +runtime, is therefore `lifetime: "host"`. Nothing waits for specific contributions; what is missing shows where it is +used: + +| Missing | What happens | +| --- | --- | +| a tool | its call gets an error result the model sees | +| a model | the generation reports "model unavailable" and waits, not fails, until a model appears | +| a task definition | the task stays blocked (`missing_task`) until the plugin is installed or the task is aborted | +| a plugin that failed | reported with its error; a facet in backoff contributes later and the registries recompose | + +The same holds when a session's database is copied to an instance with other plugins: missing tools, sections and +hooks are skipped, documents of missing plugins stay untouched, stored selections are never rewritten. The UI must +show blocked tasks with their reason and offer to abort them; a turn waiting on one hangs until then. + +:::review | Review: the harness exception to the barrier (C17, R17) +R17 makes the harness start after the barrier, a special case in the barrier's rule. Review round 3 proposed cut C17: +no harness-specific exception; the harness is an ordinary object the app's start code holds after `load` resolves. +Not decided. `chord-next-walkthrough.md` concept 28 still says the harness's activation *is* the start barrier, which +R17 replaced. +::: + +## Tools that call Chord + +Durable tools, hooks and sections receive Durable's `ToolExecutionApi` (or `HookApi`) and a `Context`, never a Chord +`ctx` (Q6). Core put the host into the harness's base context under `ChordHostKey`, and every phase context inherits +it. A core helper opens a Chord call scope from any Durable context: + +```ts +// studio/src/core/chord.ts +export function chord(context: Context, fn: (ctx: Ctx) => Promise): Promise { + const host = context.value(ChordHostKey); + if (!host) throw new Error("not inside the Studio harness"); + return host.root(fn, { signal: context.abortSignal, telemetry: context.telemetry }); +} +``` + +The scope runs as the host principal; attribution is data; references taken with `ctx.get` are released when `fn` +settles. There is no wrapping of tools and no app-specific `defineTool`. The notes plugin gives the agent a tool: + +```ts +// notes/worker.ts +const addNote = defineTool({ + name: "notes_add", // tools are named _ + description: "Add a note to this session's notes panel", + parameters: Type.Object({ text: Type.String({ maxLength: 2000 }) }), + execute: (args, api, context) => chord(async (ctx) => { + const info = await ctx.get(SessionInfo, {}); + const store = await ctx.get(NoteStore, { session: info.init.value.id }); + const id = await store.add({ text: args.text, by: `agent:${api.conversationId}` }, ctx); + return { output: [{ type: "text", text: `added note ${id}` }] }; + }, context), +}); + +export default defineFacet((host, self) => { + self.contribute(HarnessExtensions, defineExtension({ name: "@studio/notes", tools: [addNote] })); + … +}); +``` + +A reload of the notes plugin replaces its extension in the table; Durable pins a registry snapshot per task phase, so +a tool call already running finishes on the old code and the next phase uses the new one. + +## Tools that need a person + +Clients are not addressable, so a tool cannot call the user's tab. A tool that needs the user goes through an owner's +state: the Questions service (chapter XII). The model-facing `ask_user` tool and an approval hook both call +`Questions.ask`; tabs render the open questions and answer them. + +```ts +hook(ToolTask, { + beforeTool: async (call, api, context) => { + if (!needsApproval(call)) return undefined; + const decided = await api.memo("approval", context); // a rerun does not ask again + if (decided !== undefined) return decided ? undefined : { block: "Denied" }; + const allowed = await chord(async (ctx) => { + const q = await ctx.get(Questions, { scope: { session } }); + for (;;) { + try { + const r = await q.ask( + { kind: "confirm", message: `Run ${call.name}?`, allow: "Allow", deny: "Deny" }, + { audience: approvers, policy: { mode: "first" }, key: api.taskId }, + ctx, + ); + return r.answer; + } catch (error) { + if (ctx.signal.aborted) throw error; // a link blip is not a denial: retry + await backoff(ctx); + } + } + }, context); + await api.memo("approval", allowed, context); + return allowed ? undefined : { block: "Denied by the user" }; + }, +}); +``` + +Durable runs `beforeTool` in the tool task's call phase; after the execute checkpoint a crash retries execution +without hooks, so the stored decision covers that retry. The approval must cover the final arguments: put the +approval hook after hooks that rewrite arguments. + +## Conversations for the UI + +Chat panels need a conversation's live view, submit, steer, stop. The design leaves p3's conversation API to the p3 +design; one construction that follows the rules: a core `ConversationStore { conversation }` (local) opens the Durable +watch as its own native, getting the Durable harness from the core-private `HarnessRuntime` registry contribution (a +native may reach another object only through a registry entry), and exposes a bounded window as a derive. Per-principal +`Conversation { session, conversation }` views in front of it check who may send (chapter XIX). + +## Code mode + +Code mode is a plugin. It owns its `pi-codemode` sandbox as a native; its tool entry looks up the **current** sandbox +when `execute()` starts, so a call pinned by a phase before a reload (waiting in an approval hook, say) starts on the +new sandbox. On `reload` and `stop` the old sandbox is sealed and drained in `close`: running scripts finish or hit +their deadline, for at most 30 s (Q7). After a worker stop Durable replays as it does today. + +## What lives where + +| Data | Where | +| --- | --- | +| transcript-bound: entries, documents that fork and rewind with a conversation | Durable | +| app state: session list, notes, drafts, grants, settings | Chord documents | +| data core enforces on conversations (read-only one-shots, cwd followers) | a core per-conversation policy document, written when the conversation is created (`createConversation({ init })`) | + +**Telemetry:** pass the host's adapter as `HarnessOptions.telemetry`; Durable stores trace carriers on submissions and +tasks (chapter XVII). diff --git a/packages/chord/docs/next/explainer/src/15-reload.md b/packages/chord/docs/next/explainer/src/15-reload.md new file mode 100644 index 000000000..09ab349de --- /dev/null +++ b/packages/chord/docs/next/explainer/src/15-reload.md @@ -0,0 +1,131 @@ + +# Reload and Rollout {#reload lede="What happens when you press Reload: per host, per object, and across the server, the workers and every tab."} + +Back to the preface. Mario changed the notes plugin's sort order and pressed Reload. Everything the earlier chapters +asked of plugin authors (synchronous factories, natives only in objects, routines instead of timers, data in +documents) exists so that this moment is safe. This chapter shows what Chord does with it. + +## On one host + +```text +1. load new bundles; an import or syntax error rejects +2. staging facet setups and every live object's new factory run with a recording self + (declarations only, nothing activated); any throw rejects +3. commit the host switches to the new code atomically +4. after per object: pure objects swap, native objects restart +``` + +**Rollback is all-or-nothing per host.** Any load or staging error leaves the whole host on its previous code, so a +host is always completely at one revision and its status is one value. Runtime states (a registry conflict, a failed +open, a migration) are not reasons to roll back; after the commit, errors stay with one object. There is no factory +trial, no migrate dry run, and no consumer contract check in staging: an incompatible consumer fails at bind with +`contract_mismatch` and a notice naming plugin and type. + +:::steps | Reloading @studio/notes on worker S1, object by object ++++ Load and stage +The new bundles load. The facet setup and the factories of `Notes` (three views), `NoteStore { S1 }` and `NoteInbox` +run with a recording `self`: they declare, nothing activates. Nothing throws, so the host commits to the new code. ++++ NoteStore: pure, swapped in place +`NoteStore` has no `self.open`. Its new methods and derives take over; `NoteList`, the drafts, its references and the +running import job stay. Routines restart, serialized: the new `cleanup` waits for the old handler in flight. The gate +opens. ++++ Running calls finish on old code +An `add` that was running when the swap happened finishes on the old code with its old handles; its transaction still +commits. Only new calls get the new handles. Exception (R15): if the reload raised a document's version, old code that +touches the migrated document throws, the document stays intact, and that call fails once. ++++ The import job keeps its old run +The job's `run` keeps the code it started with. When it settles, the **new** `finish` records the outcome. ++++ The views: pure, swapped +The three `Notes` views swap in place. Owner, Ann and Bob keep their references; the reader derive is recomputed with +the new sort and the difference is sent. Nobody sees a reconnect. ++++ NoteInbox: native, restarts +`NoteInbox` has a watcher. Its running work ends `aborted`, the watcher closes with reason `reload`, the new factory +runs, the watcher reopens. Its documents would stay. Whether a file dropped during the restart is seen is the +watcher's business: Studio's `watchDir` scans the directory once when it opens. +::: + +**Why natives restart instead of being handed over:** a native has no line, no transactions and no schema. If old and +new code shared it, they would interleave on it unchecked. In practice restarts are cheap: the Durable harness is base +code, terminal PTYs live in a separate terminal host process, and model runtimes, sandboxes and a search index rebuild +in milliseconds. Only reloading the plugin that owns SSH connections (running remote commands die) or the relay socket +(relayed connections drop) hurts. + +**Removing a type:** its objects end (routines stop, natives close, references end with an error); in-memory documents +go with them; persisted ones stay until retired. **Consumers elsewhere** see a restarting object like any object that +briefly goes away: `loading`, parked calls, then `live`. + +**UI facets reload too.** A reload unmounts and remounts components. That is why "leaving" (abort a login, close an +overlay) is never derived from an unmount (chapter VIII). + +## Across hosts + +A **revision** is (server incarnation, sequence) and names the complete desired code set: every facet with its +digest, not a batch of changes. Every install reconciles everything missing, and a host reports a revision only when it +runs that complete set. A revision also carries the code-set digest (a hash over the facet bundles' content hashes), +so a server restart with unchanged code triggers no reinstall. + +The order is **server, workers, clients**. Each host reports `installed`, `installed_failed` (with the objects that +failed) or `rejected` (with the reason; it stays on its previous code). A server that rejects stops the rollout. A +host that starts or reconnects installs the current revision before anything is routed to it. Installs are serialized +per host and bounded (60 s). + +Calls keep flowing during a rollout. When a call reaches a provider on another revision, the mismatch response +carries the provider's revision and status: + +| Provider | The call | +| --- | --- | +| ahead, or behind and installing | `updating`: values readable, calls park (with the park deadline) | +| rejected | fails at once; retried on the next revision | +| same revision | `contract_mismatch`: a real mismatch, final | + +A host checks its own install status first: if its own install was rejected, a call fails at once with the reason +instead of waiting on a provider. Calls parked on a binding that is released end with an error and are not resent. +There are no cross-host transactions and no votes. + +:::sequence animate | A tab calls into a worker that is still installing. The call parks and runs once the worker reports the revision. +actor tab "tab" "revision 8" +actor server "server" "revision 8" accent +actor worker "worker S1" "installing 8" blue +server => worker: install revision 8 +tab -> server: call Notes.add (new contract) +server -> worker: relay +worker --> server: updating (rev 7, installing) +server --> tab: updating +tab -> tab: park (deadline 60 s) +worker --> server: installed 8 +server --> tab: provider at 8 +tab -> server: call Notes.add (resent from park) +server -> worker: relay +worker --> tab: result +::: + +## Start barrier + +At host start and first load there is one more rule (chapter XIV): load every facet (the core facet with its start +data among them), wait until every facet and `lifetime: "host"` object other than the harness is live or failed, then +resume dependent work. Plugin reloads need no barrier. + +## Loading and the dev loop + +- **Roots:** plugins come from built-in, global (`/plugins`), package (npm or git, installed with + `--ignore-scripts`) and project roots, found by directory listing; no source parsing. The server selects which load. + Dependencies between plugins resolve by contract name at runtime. Names are prefixed: objects `.`, + tools `_`. +- **Loader D:** one base bundle per surface (host packages and the app runtime) plus one script per facet, with host + packages passed in (no module map). Old generations are reclaimed by the garbage collector; a stable filename per + (facet, digest), and V8's compilation cache off on hosts that reload in place. The loader is an adapter: static + imports (tests, Durable Objects) work the same, with the app passing the plugin name to `load`. +- **Dev loop:** a source watcher marks plugins dirty and never applies anything. A reload control (the playground's web + shell, a CLI, an agent tool such as p2's `reload_plugins`) reloads the changed plugins, or restarts all hosts for + base code. Reloads are always explicit. + +:::review | Review: project plugins (#14, deferred) +Per-directory plugins with trust prompts and snapshots (`apps/p2/docs/plugin-loading.md`, still the reference for +roots, trust and scopes) need their own pass over this design. Until then, treat the project root as designed but not +settled. +::: + +:::note | Built so far +Reload and rollout are layer L10, the last before telemetry. The frozen kernel has a reload path with host values and +a development freeze, both removed from the design. +::: diff --git a/packages/chord/docs/next/explainer/src/16-contracts.md b/packages/chord/docs/next/explainer/src/16-contracts.md new file mode 100644 index 000000000..51e0126cb --- /dev/null +++ b/packages/chord/docs/next/explainer/src/16-contracts.md @@ -0,0 +1,310 @@ + +# Changing Contracts {#contracts lede="How a contract gets a new version while old tabs, old workers and old plugins are still talking to it."} + +Chapter XV ended with a tab that calls into a worker on another revision. That case is short-lived: the rollout +finishes and both sides run the same code. Other cases last longer. A phone kept a tab open overnight with yesterday's +web bundle. A plugin from another author was built against `Notes` version 1 and is not updated yet. In both, a +consumer holds a contract that the provider no longer has. This chapter is about what the provider does then. It is +also the one part of Chord v2 that is built: everything in the sections on `evolve`, `checkAccepts` and fingerprints +runs today in `packages/chord/src/next/contracts/`. + +## What has a version + +Everything that is stored or crosses a boundary has a version. Each surface handles an old version its own way: + +| Surface | Version | When an old one shows up | Chapter | +| --- | --- | --- | --- | +| object type (contract) | `version` plus fingerprint | `evolve` and `checkAccepts` on the provider; otherwise `contract_mismatch` | this one | +| document | `version` plus schema fingerprint | pure `migrate` on access; a schema change without a bump fails the object `broken` | V | +| job | `version` | `accepts` on outcomes and `running`; `blocked` without an adapter | XI | +| registry | `version` | `accepts` adapts contributions; one without a match is left out and reported | XIII | +| native | none | the object restarts on reload | IX | +| Durable tasks and documents | Durable's `version` | Durable's `migrate` | XIV | +| link protocol | protocol version | refused; hosts update together, a web page reloads | XVII | +| UI components, Chord and app APIs | none yet | exposed as they are; an SDK will version them later | | + +The rule behind every row is the same as for documents: **every change bumps the version**, additive ones included. +Chord never guesses compatibility from the shape of two schemas. + +## A history, not a file of versions + +A contract history starts with a **base**: a full `remote()` or `local()` declaration with an explicit `version`. The +`Notes` contract from chapters II and III is a base at version 1. Studio now wants pinned notes: notes get a `pinned` +flag and editors get a `pin` method. Instead of editing the declaration, the plugin continues the history: + +```ts +// notes/contract.ts +import { bool, evolve, nul } from "@earendil-works/chord/next"; + +export const NoteV2 = obj({ id: str(), text: str({ max: 2000 }), by: str(), at: int(), pinned: bool() }); +export const PublicNoteV2 = obj({ id: str(), text: str(), at: int(), pinned: bool() }); + +export const NotesV2 = evolve(Notes, (e) => + e.state("editor", "notes", obj({ items: arr(NoteV2) })) + .state("reader", "notes", obj({ items: arr(PublicNoteV2) })) + .addMethod("editor", "pin", obj({ id: str(), pinned: bool() }), nul()), +); +``` + +`evolve(prev, (e) => …)` appends one version to `prev`'s history: `NotesV2.version` is 2, and `NotesV2` still knows +version 1 exactly as it was declared. Every version is **materialized and fingerprinted at declaration**. That is what +makes old code safe: an old plugin bundle that contains its own copy of `Notes` version 1 computes the same +fingerprint as the history inside `NotesV2`. There is no build step, no generated file and no fingerprint copied by +hand. + +The edits: + +| Edit | Changes | Kind | +| --- | --- | --- | +| `e.state(variant, member, schema)` | one state member of one variant | remote | +| `e.addState(variant, member, schema)` / `e.removeState(variant, member)` | adds or removes a state member | remote | +| `e.state(member, schema)`, `e.addState(member, schema)`, `e.removeState(member)` | the same, flat | local | +| `e.args(name, schema)` / `e.result(name, schema)` | one side of a method | both | +| `e.method(name, args, result)` | both sides of a method | both | +| `e.addMethod(variant, name, args, result)` | adds a method (or an existing one to another variant, same signature) | remote | +| `e.addMethod(name, args, result)` | adds a method | local | +| `e.removeMethod(name)` | removes a method from every variant | both | + +Edits chain. The `method(...)` wrapper appears only in base declarations; edits take the schemas directly, so a +method without a result is added with `nul()`. The rules of chapter III still hold across versions: a method has one +signature in every variant, and a member's kind never changes, not even after it was removed and added again. + +**What `evolve` cannot do:** change the input or the set of variants. Both change what an object *is* (its identity, +who gets which members), and no adapter can turn one into the other. That needs a new history base, below. + +## The provider says what it accepts + +`NotesV2` says what changed. It does not say how version 1 consumers are served; the **provider** decides that, with +mappings that live in its own code, never in the contract. The worker facet declares them with `checkAccepts`: + +```ts +// notes/worker.ts +import { checkAccepts } from "@earendil-works/chord/next"; + +const toV1 = ({ pinned, ...note }: Static) => note; +const toPublicV1 = ({ pinned, ...note }: Static) => note; + +const notesAccepts = checkAccepts(NotesV2, { + 1: { + state: { + editor: { notes: (current) => ({ items: current.items.map(toV1) }) }, + reader: { notes: (current) => ({ items: current.items.map(toPublicV1) }) }, + }, + methods: {}, // add and remove did not change; pin is new + }, +}); +// notesAccepts.versions: [{ version: 1, fingerprint: Notes.fingerprint }] + +host.serve(NotesV2, notesFactory, { accepts: notesAccepts }); // working name for the objects-layer option +``` + +`checkAccepts` is **pure**: it checks the mappings without calling them and returns only the version metadata of the +old versions it lists. A provider that lists nothing accepts only its current version. The objects layer later +compares a consumer's version and fingerprint against this list. + +The mappings point in the direction the data flows: + +| Member in the old version | Mapping | Direction | +| --- | --- | --- | +| state, changed | `(currentMember) => oldMember` | provider's current value to what the old consumer expects | +| state, removed | `(currentVariantState) => oldMember`, or `"refuse"` | receives the whole current state of that variant | +| method arguments, changed | `args: (old) => current` | old caller's arguments to current ones | +| method result, changed | `result: (current) => old` | current result back to the old shape | +| method, changed or removed | `{ call: async (oldArgs, ctx) => oldResult }`, or `"refuse"` | replaces the method for old callers | +| member added since | nothing | an old consumer never asks for it | + +A changed state member cannot be refused: the old consumer still has that member and must get a value. A removed one +can be computed from what the variant has now, or refused. `e.method(name, args, result)` replaces both sides and +therefore needs both mappings, even if one side stayed the same (then an identity mapping). + +**Required, at type level and again at runtime.** For every member that exists in the old version and changed or was +removed since, the mapping is required by the type of `checkAccepts`. Because plugins are bundled without type +checks, the same check runs at declaration: a missing mapping, a key for a member that needs none, an unknown version +or a wrong shape throws, naming the member's path (for example `accepts[1].state.reader.notes`). An absent `state` or +`methods` section counts as empty and may be left out only when nothing there needs a mapping. + +**Conservative with no-op edits.** An edit that sets a schema to what it already was, or changes a member and then +changes it back, still requires an identity mapping. TypeScript erases constraints such as `{ max: 2000 }` from the +static type, so two schemas that look the same to the compiler may differ; Chord does not try to tell. + +### Old versions map straight to the current one + +Studio later replaces `remove` with `archive`, which keeps the note in a trash list: + +```ts +export const NotesV3 = evolve(NotesV2, (e) => + e.removeMethod("remove").addMethod("editor", "archive", obj({ id: str() }), nul()), +); + +const notesAccepts = checkAccepts(NotesV3, { + 1: { + state: { editor: { notes: (c) => ({ items: c.items.map(toV1) }) }, reader: { notes: (c) => ({ items: c.items.map(toPublicV1) }) } }, + methods: { remove: { call: async ({ id }, ctx) => { await archiveNote(id, ctx); return null; } } }, + }, + 2: { methods: { remove: { call: async ({ id }, ctx) => { await archiveNote(id, ctx); return null; } } } }, +}); +``` + +Each entry maps **one old version directly to the current one**, not through the versions in between. Version 1 +needs the state mappings (its notes have no `pinned`) and the `remove` replacement; version 2 needs only the +`remove` replacement, and leaves `state` out because its state is unchanged. A member that an old version never had +needs nothing for that version, even if it was added, changed and removed later. Dropping support for version 1 is +deleting its entry: version 1 consumers then get `contract_mismatch`. + +## Binding an old consumer + +A hold carries only the consumer's contract **version and fingerprint**, never a description of the contract (R6). +The provider decides with its own schemas: + +:::flow wide animate | What a provider does with a hold. The decision uses only the provider's own history; a peer's schema is never an input. +row "The hold arrives" +box hold "hold" "version 1, fingerprint F1" blue +box cur "equal to current?" "version and fingerprint" +box acc "accepted old version?" "F1 from the provider's own history" +row "Outcome" accent +box live "bind" "current members" +box adapted "bind adapted" "through the mappings" accent +box mismatch "`contract_mismatch`" "or updating, during a rollout" muted +arrow hold -> cur +arrow cur -> live "yes" +arrow cur -> acc "no" +arrow acc -> adapted "yes" +arrow acc -> mismatch "no" +note "A mismatch fails only that consumer's binding. The provider object and its other bindings are not affected." +::: + +Once a version 1 tab is bound, it is served as if version 1 were current: + +:::steps | A version 1 tab on a version 3 worker ++++ Bind +The tab holds `Notes { S1 }` with version 1 and fingerprint F1. The worker's history computes F1 for version 1, and +`notesAccepts` lists version 1, so the hold binds adapted. The tab gets the `editor` variant as version 1 defined it: +`notes`, `add`, `remove`. It never sees `pin` or `archive`. ++++ State through the mapping +Chord serves the adapted state like a derive of the provider's state: it applies the mapping to the current value +and keeps the result per binding. The snapshot the tab receives has no `pinned` fields. ++++ A commit +The owner pins a note from a current tab. The worker sends the current tab the draft's ops. For the version 1 tab, +Chord applies the mapping to the new value and sends the difference to its last adapted value (`diffRevisions`). The +mapping produced nothing new for a pin, so the version 1 tab receives no change at all. A large list is never resent +whole on every commit. ++++ A call +The tab calls `remove({ id })`. Its arguments are checked against the provider's own version 1 schema, then the +`call` replacement runs `archiveNote` on version 3 code. Its `null` result is what version 1 expects. Had the method +only changed, `args` would have mapped the arguments forward and `result` the result back. ++++ The worker moves on +A later revision drops the version 1 entry. When the tab re-holds (or the worker restarts and the tab binds again), +the hold gets `contract_mismatch`, and Studio's panel shows "reload to update" instead of failing silently. +::: + +## A new history base + +Some changes cannot be adapted. Studio decides that notes belong to a project, not a session: the input changes from +`{ session }` to `{ project }`. That is a new base, written in full, with an explicit version that continues the +published numbering: + +```ts +export const NotesV4 = remote("notes", { + version: 4, // explicit: the published history used 1 to 3 + input: obj({ project: str() }), + placement: () => Server, + variants: { + editor: { state: { notes: obj({ items: arr(NoteV2) }) }, methods: { add: method(…), pin: method(…), archive: method(…) } }, + reader: { state: { notes: obj({ items: arr(PublicNoteV2) }) } }, + }, +}); +// checkAccepts(NotesV4).versions is empty +``` + +A new base has no history before its version and **cannot accept older versions**, not even for an additive change. +Every consumer built against versions 1 to 3 gets `contract_mismatch` and must update. Later `evolve` calls continue +from version 4. The same applies to a change of the variant set. Do not spread an older declaration into a new one or +copy its fingerprint; the contract objects are branded and `evolve` and `checkAccepts` reject copies. + +## Local contracts + +`local` contracts (chapter IV) evolve the same way, with a flat members map and no variants: + +```ts +export const SessionInfoV2 = evolve(SessionInfo, (e) => e.state("init", InitV2)); +checkAccepts(SessionInfoV2, { 1: { state: { init: (current) => toInitV1(current) }, methods: {} } }); +``` + +Same-host consumers of a `local` type are usually reloaded together with their provider, but not always: another +plugin on the same host may be built against an older version. The rules are identical; only the variant argument is +missing. + +## Implementing a contract + +`Implementation` is the type of everything a provider must supply for the current version: one state +getter per member and variant, and one asynchronous method per method, each receiving `(args, ctx)`: + +```ts +const impl: Implementation = { + state: { editor: { notes: () => list.value }, reader: { notes: () => publicList.value } }, + methods: { add: async ({ text }, ctx) => addNote(text, ctx), remove: async ({ id }) => removeNote(id), pin: async (a) => pin(a) }, +}; +``` + +Keep declaration types **inferred** (no annotation on `NotesV2`): the inferred type carries the complete history, and +`checkAccepts` needs it to know what changed. + +:::review | Review: L1's implementation shape and the factory's return value +`Implementation` in `src/next/contracts/types.ts` is `{ state: { variant: { member: getter } }, methods }`. The +design's factories (`design.md` §6, used throughout this guide) return `{ variant, member: value, method: fn }`. +The objects layer (L4) has to connect the two, and also define how `serve` receives the checked accepts (this guide +uses the working name `{ accepts }` as a third argument) and what `"refuse"` does at runtime. The natural reading, +used in the stepper above, is that a refused method call and a hold that needs a refused state member end with +`contract_mismatch`; no source says so. +::: + +## The fingerprint + +The fingerprint is a 64-bit hash (`hash64`, portable across runtimes) of a **normal form** of the contract's +schemas. The normal form decides what counts as a change: + +| Ignored | Significant | +| --- | --- | +| order of object keys, `required` and `enum` (sorted) | order of union and intersection branches (`anyOf`, `oneOf`, `allOf`) and of tuples | +| annotations on schema nodes: `title`, `description`, `examples`, `$comment`, `deprecated` | defaults, formats, patterns, bounds, `$ref` names | +| placement | input, variants, member kinds and every member schema | + +Union order is significant because it is meaning: TypeBox picks defaults and decodes by the first matching branch. +Annotations are dropped only on schema nodes; an object property that happens to be named `title` is data and counts. + +Only data schemas are allowed. One walker copies, validates and normalizes each schema against a **whitelist of +TypeBox keywords**: primitives, arrays and tuples, objects and records, unions and intersections, `$defs` and `$ref` +(use `$ref` for recursive types), constraints and annotations. Conditional and dependent schemas, `not`, +`definitions`, `contentSchema`, `unevaluated*`, `contains`, `propertyNames`, unknown keywords, executable or custom +TypeBox features, arbitrary `Unsafe` and refinements, accessors, symbols and cyclic objects are rejected at +declaration. Bytes are the one special leaf: `bytes({ min, max })` checks a `Uint8Array` and has the data marker +`{ type: "bytes", minLength?, maxLength? }`, so it hashes like any schema; TypeBox's checking still works on it. + +Schemas are deep-copied into a private, frozen history; your own schema objects are copied, not frozen. +`NORMAL_FORM_VERSION` is part of every digest, so a change of the normal form is a link protocol change. + +**Not a security boundary.** A fingerprint says "we were compiled against the same contract"; it proves nothing. A +hostile client can send any fingerprint it likes. Providers therefore always validate arguments with their own +schemas (chapter IV), whatever the fingerprint says. + +## Rules of thumb + +- Treat a published contract like a published API: every change is `evolve`, every `evolve` needs a decision about + old consumers. +- Keep mappings small and pure. A state mapping runs for every commit and every adapted binding; make it a map over + the value, not a query. +- Accept the versions your users still run (one or two back during a normal rollout), and delete entries when the + metrics show no old bindings. +- Change the input or the variants only with a new base, and only when updating every consumer is acceptable. +- Contract, document and job versions are separate and each is decided on its own. Pinned notes needed `NotesV2` + for the wire and `NoteList` version 2 with a `migrate` that adds `pinned: false` (chapter V), but no new + `ImportJob` version, because its outcome did not change. + +:::note | Built so far +This chapter's contract layer is built: `packages/chord/src/next/contracts/` (commit `7d2bb8ea8`, about 1,300 lines +with `remote`, `local`, `method`, `evolve`, `checkAccepts`, `Implementation`, the schema helpers, `fingerprintOf`, +`schemaNormalForm` and `NORMAL_FORM_VERSION`), with nine test files covering evolution, fingerprints, the hash, +local contracts, long chains, schemas, types and review findings. It stands alone: it hosts nothing, calls no +mapping and does not import the frozen runtime. Binding by version and fingerprint, adapted state and calls belong to +the objects and protocol layers, not built yet. +::: diff --git a/packages/chord/docs/next/explainer/src/17-wire.md b/packages/chord/docs/next/explainer/src/17-wire.md new file mode 100644 index 000000000..67bd21118 --- /dev/null +++ b/packages/chord/docs/next/explainer/src/17-wire.md @@ -0,0 +1,251 @@ + +# On the Wire {#wire lede="What travels over a link, what a relay does with it, how big things are kept in check, what each hop costs, and how to see it in traces."} + +Most plugin code never thinks about the wire: it declares state, commits transactions and calls methods, and Chord +moves the bytes. You need this chapter when something is slow, when a value is large, when a link misbehaves, or when +you want to read a trace. It also explains why the earlier chapters insisted on small operations, shared structure and +no callbacks: each of those rules removes a cost on some hop. + +## What a state member sends + +Every exposed document, derive or readable is a **state member**. Over a hold it produces one of these: + +| Update | When | +| --- | --- | +| `snap` | on hold and on re-hold: the whole value, once | +| `ops` | per commit: the draft's recorded ops, no diff | +| `ops` from a diff | per change of a derive (and of an adapted binding, chapter XVI) | +| throttled `ops` | job progress | +| `reset` | a migration commit replaced the value (adapted for older consumers) | +| `error` | the member failed | + +**Commits are atomic for consumers.** On the same host, subscribers are notified once, after every document of the +commit is adopted. Remotely, one `state` message per hold carries a batch of member updates that is applied before +listeners run, and a commit that touches several holds on one link travels as **one** `state` message on that link. +A consumer never sees half a commit. + +## Messages + +A message is a **header plus opaque parts** (R1). There is one part per hold the message concerns; the header says +which hold each part belongs to and how large it is. + +| Direction | Kind | Parts | +| --- | --- | --- | +| consumer to provider | `hold`, `release`, `call`, `cancel` | `call` has one (the arguments) | +| provider to consumer | `bound` | one: the snapshot | +| | `state` | one per hold the commit touched on this link | +| | `result` | one | +| | `gone` | none | + +There are no request ids in application code. Hold ids and call ids exist only inside the protocol, per link. + +**One binary format on every link (Q4):** `u32 header length | header JSON | part bytes`, as one `Uint8Array`. A +WebSocket sends it as a binary frame, IPC passes it with `serialization: "advanced"`, a memory link delivers it +asynchronously. Encoding, framing, header checks, id resolution, flow control and size limits are one shared protocol +layer; link adapters only move bytes. Tests on memory links therefore run the production path. + +A part is encoded **once, at its origin**, with that hold's own Delta encoder (paths are interned per hold, end to +end between provider and consumer). A JSON part above a threshold (about 16 KiB) is compressed once at its origin, +whatever the link; byte segments stay raw; relays forward compressed parts as they are; the header names the codec. + +**The header is a TypeBox schema (R4)**, a union tagged by `kind`; the protocol types derive from it. Headers from +untrusted links (client WebSockets) are checked with the schema compiled once; trusted links (IPC between server and +workers) are not checked. **Unknown ids are ignored on both sides (Q1).** They are normal races: a `result` crossing a +`cancel`, a `call` crossing a `gone`, a state part crossing a `release`. The side that ended a hold or call settles +its own pending work: a consumer that receives `gone(h)` rejects its pending calls on `h`, and the order per +direction guarantees nothing else for `h` follows. Only a header that fails the schema is a protocol error, and it +closes that link. + +The link protocol has a version. Hosts with different protocol versions refuse each other; a web page reloads to get +the server's protocol. + +## Through a relay + +A tab never talks to a worker directly (chapter I). The server **relays**, and a relay reads and rewrites only +headers: it maps hold and call ids between its two links, stamps the principal on `hold` and `call` (a client's +principal field is always overwritten; only a trusted parent link may carry one), and groups parts per destination +link, so each link gets only its own holds' parts. It never parses a payload. + +:::sequence animate | The owner adds a note while Ann watches. The commit leaves the worker as one `state` message with two parts; the server splits it by destination without decoding either part. +actor owner "owner's tab" "hold h1" blue +actor ann "Ann's tab" "hold h2" muted +actor server "server" "relay" accent +actor worker "worker S1" "NoteStore, Notes" +owner -> server: call c7 Notes.add (1 part) +server -> server: map ids, stamp owner +server => worker: call c3 (part forwarded as is) +worker -> worker: method runs, self.tx commits +worker => server: state: parts for h1', h2' +server -> server: group parts per link +server --> owner: state (h1 part) +server --> ann: state (h2 part) +worker --> server: result c3 +server --> owner: result c7 +::: + +The `state` message reaches the owner's tab before the `result`, so when `await notes.add(…)` resolves, the new note +is already in `notes.value`. Ann's view is a different variant (reader) and gets its own part from its own derive; the +relay neither knows nor cares. + +## Flow control + +Each link has **one byte window**, counted from each message's `byteLength` (parts are already encoded bytes; nothing +is measured or stringified to count it). The window covers every message, first snapshots included. + +- **Snapshots are always admitted (R19)**, even when larger than the window; they use it up. Whether a hold + **overflows** is decided on the changes queued after the snapshot. +- **Under backpressure** a commit's ops are collected per hold and encoded into one part when credits return. The + consumer still gets the commit atomically. +- **After an overflow** the hold is dropped and established again once the previous snapshot's debt has drained + (credits back at the full window, Q3), and not more often than a minimum interval. So there is at most one snapshot + in flight per hold: no re-hold storm, and no stall for a snapshot larger than the window. +- **Calls** are throttled by call credits the provider grants (a generous window). Beyond them calls wait locally. + Nothing is refused for being "too many", and holds are not limited by count; authorization and input validation + protect them. +- **Byte-heavy traffic** such as terminals uses separate channels (chapter XIII), so a large state message never + holds up keystrokes. + +**Recovery:** after a link drops, the consumer re-holds every hold over the new link and receives snapshots; calls in +flight fail as on any disconnect, calls made meanwhile park. A relay whose window overflows releases the hold and +re-holds; the provider sends a fresh snapshot. After a routed link drops, Chord re-resolves the address (with the +per-address backoff) while holds on it exist, so a passive viewer recovers without anyone pressing anything. + +## Size limits + +Limits apply **per part** (R2, Q2), both to the compressed and the decoded size, and are checked from the sizes in the +header before anything is decoded; zlib's `maxOutputLength` enforces the decoded limit in the one decode function. + +| Situation | What fails | +| --- | --- | +| a part over the limit | only its own hold or call, with `too_large`; the other parts of the message apply | +| a snapshot over the limit | its hold ends `failed`, with no automatic re-hold | +| a WebSocket frame over `maxPayload` (e.g. 64 MiB) | the connection closes (code 1009); this is the transport's hard cap | + +Defaults for client WebSockets are 16 MiB compressed and 64 MiB decoded per part, configurable per link kind. The +decoded limit travels with relayed work: the server stamps the admitting link's limit on the header at admission +(never taken from the client), and the host that decodes enforces it. IPC therefore needs no limit of its own. + +Large data is never a large state member or a large argument. Files go through chunked transfers or the HTTP +component (chapter XIII); long lists are keyed documents or bounded windows (chapter VIII). + +## What each hop costs + +Chord v2's goal is as little compute and memory as possible while staying correct: **O(size of the change)** per +commit and per viewer, independent of the size of the value. A 500 KB value is touched as a whole only when a new +viewer takes its first snapshot. Delta, the value library under documents, was built for this, and its own primitives +are almost free: `diffRevisions` on shared subtrees takes 0.018 ms, `applyImmutable` of a small change 0.004 ms. + +| Hop | Work | +| --- | --- | +| commit | Delta's `prepare` (copies and checks only the values placed in this commit), adopt by pointer swap; storage writes the delta | +| reads in a transaction | the draft overlay, no copy; an unchanged draft writes nothing | +| subscribers on the same host | the same value and ops, by reference | +| derives | compute plus `diffRevisions`; unchanged parts stay the same objects (`mapShared`, chapter VIII) | +| send | one encoding per hold part at its origin, size taken from the encoded bytes | +| relay | header only: ids mapped, principal stamped, parts grouped; parts never parsed | +| receive | compiled header schema on untrusted links; arguments stripped and validated once at the provider (compiled schema); inputs normalized | +| apply | `applyImmutable`, structure shared | +| new hold | the value encoded once (unavoidable) | +| memory | one authoritative value per document per host; replicas and derived values share structure; one cached value per adapted binding | + +These are targets. The frozen kernel measured far worse, which is why the design is so strict here: + +:::stats +- 35 ms | one small change to a 500 KB value in the frozen kernel, no viewer +- +13 ms | per viewer on top (10 viewers: 130 ms) +- 17 / 33 / 101 ms | a tiny SQLite commit after 0 / 10 / 50 earlier deltas +::: + +Nearly all of that came from layers around Delta: whole-value validation on every access, whole-value freezing, +JSON-ness walks on every received frame, double stringification to measure a frame, snapshots cloned per viewer, and +a SQLite storage that replayed history, and schemas rebuilt for every check (a compiled TypeBox check takes +0.012 ms, an interpreted one 5.6 ms). The redesign removes each of them (chapters V, VIII and this one); the +before/after per hop is in `apps/p2/docs/p3-reviews/v2-cost-model.md`. + +:::review | Review: v2-cost-model.md is partly outdated +The measurements and the per-hop "after" targets in `apps/p2/docs/p3-reviews/v2-cost-model.md` are current. Its +closing section "Decisions this needs" is not: items 6 and 7 (validation by the caller's trust; contract descriptions +once per link and fingerprint) were replaced by R7 (validate for every caller) and R6 (no descriptions on the wire), +and §10 still describes the description fetch. +::: + +### Cost guards + +Chord reports **full-value walks, deep copies, encodes, schema validations and materializations** as ordinary +telemetry metrics, always on; with the NOOP adapter they cost an empty call where such work happens. Tests use the +in-memory adapter and assert them per hop. The reference case: one appended note to a 500 KB document with 10 viewers +does no walks, no copies, one encode per hold part at its origin and none at relays, and no validations. A +`vitest bench` suite measures the hops. + +### Ways to make it slow anyway + +| Mistake | Why it costs | Instead | +| --- | --- | --- | +| a derive that builds new objects for unchanged data | every commit diffs and sends O(value) | keep unchanged items identical: `mapShared`, `previous` (chapter VIII) | +| one long array that changes often (R24) | sharing is per path; each change copies one slot per element and visits each | keyed documents or a bounded window | +| bytes as base64 strings | inflated, parsed, compressed as text | `bytes()` leaves; chunked transfers for files | +| awaiting I/O inside `self.tx` | the object's line is blocked for every caller | do the I/O first, then a short transaction | +| storing registry entries or references in documents | stale after a reload, not data | look up the current value at use | +| opening natives at module level or in a facet | no owner: never closed, never restarted | `self.open` in an object (chapter IX) | +| callbacks as arguments | cannot cross a link, keep old code alive | registries, jobs, flows | +| one big object that everybody views | one large snapshot per viewer and per reconnect | split by viewer need; views of the same variant share one value | + +## Telemetry + +Chord reports traces, events and metrics through `@earendil-works/pi-telemetry`. Each host gets **one adapter**, +`createHost({ telemetry })`, NOOP by default; the adapter decides sampling, buffering, export and filtering. Only hosts +hold the adapter, to start root spans. Code sees spans: + +- `ctx.telemetry` is the current span (`Ctx` extends `Context`, so it is the same field Durable and pi-ai read). Every + span carries its adapter's metrics and propagation, so code with a `ctx` can record a metric or inject a carrier. +- `ctx.trace(name, attributes, fn)` runs `fn` in a child span with a child `ctx`. +- App values such as the principal stay out of Durable's context. + +**Across links**, `call` and `hold` carry the caller's trace carrier as a bounded header field (R5: limited key count +and string lengths); the provider runs the method under it, and each relay adds a `chord.relay` span. The call from the +sequence above, as a trace: + +```text +chord.call Notes.add [owner's tab] +└─ chord.relay [server] + └─ chord.method Notes.add [session:S1] principal=owner + · event chord.commit docs=2 ops=5 +``` + +Adapters trust typed input: no validation of malformed values and no try/catch around recording. Only carriers from +another process are checked (keys present, strings), and a bad one falls back to a new root. + +**Shared work** (one activation serving several `get`s, a worker start) is its own root span, linked to the trigger +that started it; each waiting span links to it. Routine runs, job runs, eviction, backoff retries and background +closes are root spans too, linked to their trigger where there is one. + +| Tier | What | +| --- | --- | +| spans | calls, methods, relays, holds, activations, opens, closes, loads, revocations, resolves, routine runs, job runs | +| events | commits inside methods, retries, parking | +| metrics | state messages, WebSocket bytes, commit durations, in-flight calls, active holds, open links, park durations, backpressure waits, the cost guards | + +Ids are attributes of spans, never of metrics. **Nothing is measured or walked for telemetry**; only sizes the code +already has are reported. Errors carry a fixed category and a message cut to 500 characters; cancelling is not an +error. Chord ships a typed telemetry schema, as pi-ai (`AI_TELEMETRY_SCHEMA`) and Durable +(`DURABLE_TELEMETRY_SCHEMA`) do. + +**Storing and reading:** `@earendil-works/chord/telemetry-sqlite` over `chord/sqlite` (Node and Cloudflare) stores one +row per settled span (events and links inline) and metrics aggregated per series per 10 s, flushes in batches every +second, keeps 7 days, and drops (and counts) beyond a bounded queue. The read interface `TelemetryStore` is shared: +`findTraces` (by name, attributes, errors, time window, with limit and cursor) returns complete traces, `getTrace` +one, `findMetrics` metrics. One renderer, `pi-telemetry/render`, prints traces as above; it prints links and never +follows them. The playground's trace viewer reads the same store. + +**Durable** already emits submit and phase spans, stores carriers on submissions and tasks (`TaskOptions.trace`), +and pi-ai emits `pi.ai.request` and `pi.ai.poll`. With the harness given the host's adapter (chapter XIV), an agent +tool that calls `Notes.add` through `chord(fn, context)` shows up as one tree from the model request down to the +commit. + +:::note | Built so far +Nothing of the protocol is built in v2: messages, relays, flow control and limits are layers L6 and L7, telemetry in +Chord is L11. Built in pi: `pi-telemetry` (propagation, links, metrics, the store interface, the renderer), +`chord/context` with `Context.telemetry`, pi-ai and Durable telemetry, and `chord/telemetry-sqlite`. Delta is built; +`mapShared` and bytes as atomic leaves in `diffRevisions` are still to do. The frozen kernel sends descriptions on the +wire and closes a link on an unknown id; both are replaced. +::: diff --git a/packages/chord/docs/next/explainer/src/18-p2-core.md b/packages/chord/docs/next/explainer/src/18-p2-core.md new file mode 100644 index 000000000..51925055d --- /dev/null +++ b/packages/chord/docs/next/explainer/src/18-p2-core.md @@ -0,0 +1,324 @@ + +# Building p2: the Core {#p2-core lede="From a list of what the app must do to a set of owners: the base code every p3 plugin builds on."} + +The notes plugin used every Chord concept. A real app is the same concepts many times over. This chapter and the +next two assemble **p3**, the Chord v2 version of p2, from the parts the guide introduced. p3 is built next to p2 in +`apps/p3`, fully isolated with its own data directory, and **without migrating p2's data** (no import of +`catalog.sqlite`, `shares.sqlite` or the subagents table). Its plugins are derived from what the app must do, not +ported one to one from p2's plugin layout. + +:::review | Review: p3 is not designed yet +The design fixes the rules (`design.md` §25) and lists the p3 decisions still open (#17, #18, #20 to #23); the +per-domain design of p3 has not started. What these three chapters show is one construction that follows the rules, +derived from the functionality map in `apps/p2/docs/p3-functions/`. Object names are working names. Where a domain +has an open decision, the text says which. +::: + +## Start from functions, not from plugins + +The functionality map in `apps/p2/docs/p3-functions/` lists what p2 does, in plain words, with four facts per +function: the data and its size, how durable it is, who may see or do it, and where it must be available. It groups +the functions into 22 domains: + +| # | Domain | # | Domain | +| --- | --- | --- | --- | +| 1 | sessions | 12 | access | +| 2 | session workers | 13 | models, credentials, settings | +| 3 | conversations and agent | 14 | prompt resources | +| 4 | shell commands | 15 | status and presence | +| 5 | chat mode | 16 | search | +| 6 | agents (subagents, one-shots, peer messages) | 17 | remote access and forwards | +| 7 | code mode | 18 | logs and notices | +| 8 | session files | 19 | dev and reload | +| 9 | environments and execution | 20 | web shell | +| 10 | workspace | 21 | TUI shell | +| 11 | terminals | 22 | transport | + +The four facts answer the four questions every Chord object needs, in order: + +:::flow wide animate | From a function in the map to code. Each fact in the map answers one question Chord needs answered. +row "The map says" +box data "data" "what, how big" +box dur "durability" "ephemeral · until restart · persisted · transcript-bound" +box who "who" "owner · guest read · guest write · worker" +box where "where" "server · worker · web · TUI · environment" +row "Chord needs" accent +box owner "the owner" "smallest object it belongs to" accent +box store "where it is kept" "Durable, persisted or in-memory document, tab" +box variant "variants" "and revocation" blue +box host "host and code" "placement; core or plugin" +arrow data -> owner +arrow dur -> store +arrow who -> variant +arrow where -> host +note "Durability decides storage: transcript-bound data is Durable's; persisted app data is a persisted document with a stable identity; until-restart data is an in-memory document or a native; tab-scoped UI state stays in the client." +::: + +**One owner per thing, the smallest object it belongs to.** A guest's grant belongs to the access owner, not to the +session; a draft belongs to one (conversation, author) pair, not to the chat plugin as a whole. Ownership nests, and +rules set at creation are written in the creating commit (Durable's `createConversation({ init })`). + +**Core or plugin.** Core owners are what the app always needs and everything builds on; they are always present, so +other code may depend on them directly. Plugin owners may be missing; anything that depends on an optional feature +does so through a registry it owns itself (chapter XIII). The risk to avoid is a **core god object**: each core owner +owns only what belongs to it. + +## The ownership map + +The result of that procedure for p3, as a starting point: + +| Domain | Owner object(s) | Host | Code | +| --- | --- | --- | --- | +| access | `Access` (grants, devices, share links), `authenticate` | server | core | +| sessions | `Catalog` (session records), lifecycle jobs | server | core | +| session workers | the supervisor (`route(Session)` + idle stop) | server | core (app code) | +| session location | `SessionLocation { session }` | worker | core | +| conversations and agent | `Harness`, `ConversationStore`, `Conversation` views | worker | core | +| status and presence | `SessionStatus` (server copy), reporters on workers | both | core | +| shell commands | `ShellJob` in the chat plugin | worker | plugin | +| chat mode | chat plugin | worker | plugin | +| agents | subagents plugin (Durable child conversations) | worker | plugin | +| code mode | code mode plugin (sandbox native) | worker | plugin | +| session files | `SessionFiles { session }` (proposed, #17) | server | core or plugin, open | +| environments | envs plugin (config, SSH pools #20) | server and worker | plugin | +| workspace | workspace plugin | worker | plugin | +| terminals | terminal host process (#23), terminals plugin | separate process | plugin | +| models, credentials | models plugin, login job + flow | server | plugin | +| settings | `Settings` (single writer, #21) | server | core | +| prompt resources | prompt plugin (watchers in the session's environment) | worker | plugin | +| search | search plugin (index on the server, feed on workers) | both | plugin | +| remote access, forwards | radius plugin, ports plugin | server | plugin | +| logs and notices | `Notices` (one channel), log capture | server | core | +| dev and reload | the reload control, `reload_plugins` tool | server | plugin | +| web and TUI shells | UI facets over the objects above | clients | shells | + +The rest of this chapter builds the core rows. Chapters XIX and XX build the plugins. + +## The base code + +```text +apps/p3/ + src/ + hosts.ts Server, Session host kinds + principal.ts P, Revocation, principalKey, revokedBy, can() + server/main.ts createHost, load, route(Session), WebSocket and Radius entry points, supervisor + worker/main.ts createHost with the start data, connect to the parent + core/ the core facets: Access, Catalog, SessionStatus, Settings, Notices (server); + SessionInfo, SessionLocation, Harness, ConversationStore (worker) + plugins/ chat, subagents, codemode, envs, workspace, terminals, models, login, + prompt, search, ports, radius, files, dev +``` + +`hosts.ts`, `principal.ts`, the `SessionInfo` start data and the `Harness` object are the ones the guide already +built for Studio (chapters I, III, IX and XIV). Base code restarts the hosts when it changes; everything in +`plugins/` reloads. + +## Who is calling: authentication + +Chord stamps principals; the app produces them. p3's server has four ways in, each turned into a principal by +`authenticate(req)` when a WebSocket connects (chapter III): + +| Way in | Principal | +| --- | --- | +| loopback without a cookie (TUI, CLI, first page load) | `{ kind: "owner", access: "local" }` | +| loopback owner browser cookie (a per-port secret file) | `{ kind: "owner", access: "local" }` | +| a device cookie (HMAC-signed, per port, 30 days), obtained with a one-time sign-in token | `{ kind: "owner", access: "device", device }` | +| a guest cookie from a share link | `{ kind: "guest", session, mode, grant, revision }` | +| the Radius relay channel | `{ kind: "owner", access: "relay" }` | + +The checks that are not Chord's stay in this function: origin checks and Fetch Metadata (a browser request from +another site gets no principal, except a top-level navigation), proxy trust, the sign-in token rules (10 minutes, +single use, at most 32 outstanding, memory only), cookie expiry and revocation lists. A failed check closes the socket +before Chord sees it. + +**One permission matrix.** Every function in the map names who may use it, and p2 checks those rules in several +places. p3 writes them once, in `principal.ts`, and every factory and `authorize` uses it: + +```ts +// src/principal.ts +type Action = "read" | "send" | "upload" | "stop" | "workspace" | "terminal" | "manage"; + +export function can(p: P, action: Action, session?: string): boolean { + if (p.kind === "owner" || p.kind === "server") return true; + if (p.kind === "worker") return action !== "manage" && p.session === session; + if (p.kind !== "guest" || p.session !== session) return false; + if (action === "terminal" || action === "manage") return false; // terminals: owner only + if (p.mode === "read") return action === "read"; // no workspace for guest read + return true; // guest write: send, upload, stop, workspace +} +``` + +Factories pick variants from it (`can(p, "send", session) ? "writer" : "reader"`), so the matrix is visible in one +file and the members a caller lacks are simply not there. + +## Access: grants, devices, share links + +`Access` is the core owner of everything that lets someone in: share links per session and mode, guest records, the +device list and the signing key. It lives on the server as the chapter IV pattern: a `local` store that persists the +documents (only stable identities may), and a `remote` view for the owner's settings screen: + +```ts +// src/core/access.ts (server, in the core server facet) +const Links = defineState("access.links", { schema: map(ShareLink), version: 1, persist: true, + initial: () => ({}), checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64 }); +const Guests = defineState("access.guests", { … }); // same shape: map(Guest), persisted +const Devices = defineState("access.devices", { … }); // map(Device), persisted + +host.serve(AccessStore, (_input, self) => { // local: may persist, acts for the host + const links = self.state(Links), guests = self.state(Guests), devices = self.state(Devices); + return { + links, guests, devices, + removeGuest: async ({ grant }, ctx) => { + await self.tx(async (tx) => { (await tx.draft(guests))[grant].blocked = true; }, ctx); + host.revoke({ grant }); // live access ends now + }, + resetLink: async ({ session, mode }, ctx) => { + const blocked = await self.tx((tx) => rotateLink(tx, links, guests, session, mode), ctx); + for (const grant of blocked) host.revoke({ grant }); + }, + signOut: async ({ device }, ctx) => { + await self.tx(async (tx) => { (await tx.draft(devices))[device].revoked = true; }, ctx); + host.revoke({ device }); + }, + … // shareLink, join, signOutAll + }; +}); + +host.serve(Access, (_input, principal, self) => { // remote view: owner only + if (!can(principal, "manage")) throw forbidden(); + const store = self.get(AccessStore, {}); + return { + variant: "owner", + guests: self.derive("guests", [store.guests], ([g]) => publicGuests(g)), // names, modes; no tokens + removeGuest: (args, ctx) => store.removeGuest(args, ctx), + resetLink: (args, ctx) => store.resetLink(args, ctx), + signOut: (args, ctx) => store.signOut(args, ctx), + }; +}); +``` + +`authenticate` is base code on the same host and checks `AccessStore`'s current documents on every new connection +(it holds a reference to the store, taken with `server.root` at start). Live access and lasting denial stay separate, as chapter VII explained: + +| Event | Lasting effect (data) | Live effect | +| --- | --- | --- | +| remove a guest | guest record blocked | `revoke({ grant })` | +| reset a share link | new token; its guests' grants blocked | `revoke({ grant })` for each | +| archive a session | grants of that session void while archived | `revoke({ session })` | +| restore a session | grants valid again; revision bumped | none: guests reconnect | +| sign out a device | device revoked until its expiry | `revoke({ device })` | +| sign out all devices | signing key rotated, tokens cleared | `revoke({ device })` for each | + +A guest who rejoins with the same browser keeps its identity; a removed guest cannot mint a new one, because the +block is in `Guests` and `authenticate` refuses it. Names are unverified, cleaned, at most 40 characters, unique per +session ("Ann (2)"). + +## The catalog + +`Catalog` owns the session records: id, title (and whether it was edited), location, model default, pinned, +archived, timestamps. It lives on the server, persisted, and it **never starts a worker**: listing sessions, showing +their status or their unread marks reads server-side copies only. + +```ts +export const Catalog = remote("p3.catalog", { + version: 1, + input: obj({}), + placement: () => Server(), + variants: { + owner: { + state: { sessions: obj({ items: arr(SessionRow) }) }, + methods: { + create: method(CreateArgs, str()), rename: method(RenameArgs), pin: method(PinArgs), + archive: method(obj({ id: str() })), restore: method(obj({ id: str() })), + }, + }, + guest: { state: { sessions: obj({ items: arr(SessionRow) }) } }, // only the guest's own session, if not archived + }, +}); +``` + +Behind it sits a persisted `local` `CatalogStore`; the guest variant is a derive over it, as in chapter IV. The lifecycle operations (archive, clone, fork, +relocate, delete) are chapter XIX; here only the rule that keeps the catalog small: **the catalog does not know which +plugins exist.** Archiving a session writes `archived: true` and revokes the session; every plugin that keeps +per-session things reconciles from `catalog.sessions` with a routine (chapter X), as the notes plugin did. The ports +plugin closes forwards of archived sessions that way, also after a crash. + +## On the worker: start data, location, harness + +Three core objects exist on every worker from its start: + +- **`SessionInfo`** (chapter IX): the start message as `constant(init)`, with the session id, data directory and the + location the server chose at start. +- **`SessionLocation { session }`** (working name), lifetime host: the current cwd and environment, with a location + revision. Relocation changes it while the worker runs (chapter XIX); natives that depend on the location follow it + with a routine and a generation (chapter IX), and conversations that follow the session's cwd are listed in the core + per-conversation policy document. +- **`Harness`** (chapter XIV): the Durable harness for the session's store, one SQLite database per session, opened + under an exclusive lock keyed by the real data directory and session id. It refuses to start when the saved root + model is not configured, and resumes interrupted Durable work on start. + +`ConversationStore` and the `Conversation` views in front of it are chapter XIX. + +## Workers: start, keep alive, stop + +Starting, supervising and stopping workers is app code (chapter I); Chord only routes and counts. p3's supervisor is +`server.route(Session, { authorize, resolve })` plus a small server object that keeps per-session worker state: + +:::steps | A worker's life, as the supervisor sees it ++++ Authorize, then start +A tab gets `Notes { S1 }`. `authorize` checks `can(p, "read", "S1")` and that the session exists and is not archived +(an archived session refuses, except an index rebuild that starts the worker without resuming work). `resolve` forks +`worker/main.ts` with the start data from the catalog, bounded by a 60 s readiness timeout. ++++ A start failure sticks +If the worker fails to start (an unconfigured environment, a missing model), the error is kept per session and +every `get` fails at once with it, until the owner presses Retry. That clears the error and the related notices. +Guests see that the session is unavailable, not the error text. ++++ Counting +The core facet on the worker reports `host.activity` (`{ references, calls, jobs, lingering }`) and the harness's +running Durable task count to the supervisor object, from a routine whenever either changes. The call carries +`ctx.incarnation`, so a report from an older worker incarnation is ignored (chapter X). ++++ Idle stop +When everything has been zero for five minutes (checked every second, as in p2), the supervisor stops the worker: +the worker closes its host (routines stop, natives close, storage flushes) and exits. Durable work stays persisted +and resumes on the next start. Detached work that must keep the worker alive (a shell command whose caller left) is +a job, so it counts. ++++ Crash +If the worker exits on its own, the link drops. The supervisor records the exit code and a redacted stderr tail +(at most 4,000 characters) for that incarnation and posts one notice. References that still exist make Chord call +`resolve` again with backoff; a passive viewer recovers without a click. ++++ Shutdown +On server shutdown the supervisor stops every worker, with a forced exit after three seconds. Durable work remains +persisted. +::: + +## Status without a worker + +The session list shows per session whether it is running, busy (which conversations), its task count, usage and cost, +and an error. None of that may start a worker. The pattern is chapter X's pub/sub: a reporter on each worker (a +routine over the harness's state) calls `SessionStatus.report(summary)` on the server; `SessionStatus` keeps the latest +value per session and ignores calls whose `ctx.incarnation` is older than the newest it has seen; clients read +`SessionStatus`, which lives on the server. Children lists, unread marks, search index rows and artifact lists use the +same pattern (chapter XIX). + +## Settings and notices + +**Settings** (default model and thinking, run settings, compaction, queue modes, code mode switch, owner name) have +**one writer**: a core `Settings` object on the server owns the file, validates writes and serializes them; every +other process follows its state through a reference instead of watching and rewriting the file itself. p2 watches +settings, models and auth in every process and writes them from two, which is decision #21. + +**Notices** are one channel: one core `Notices` object on the server, written by everything that fails or finishes +(worker crashes, reload results, model errors, login outcomes, grants), keeping one latest notice per source, at most +50. The owner's shell reads one list; dismissing is a method. p2 has five notice channels for the same thing. + +:::review | Review: settings and file formats (#21, #22) +Single writer for settings, models and auth (#21) and versioning on-disk formats such as `envs.json` like documents +(#22) are open p3 decisions in `design.md` §26. The construction above assumes "one owner on the server, others +follow its state" for #21. +::: + +:::note | Built so far +Nothing of p3 exists. The functionality map is complete (`apps/p2/docs/p3-functions/`, five area files plus the +condensed `README.md`), and the design's app rules (`design.md` §25) are decided. p3 starts after the layered rebuild +and the playground's first real proof. +::: diff --git a/packages/chord/docs/next/explainer/src/19-p2-sessions.md b/packages/chord/docs/next/explainer/src/19-p2-sessions.md new file mode 100644 index 000000000..9d40c2287 --- /dev/null +++ b/packages/chord/docs/next/explainer/src/19-p2-sessions.md @@ -0,0 +1,276 @@ + +# Sessions and Conversations {#p2-sessions lede="The session lifecycle, status without a worker, chat, shell commands, subagents and search, each mapped to an owner and the Chord parts it uses."} + +This chapter builds the domains a person touches most: the session list and its operations, the conversation view +and composer, shell commands, chat mode, subagents, messages between sessions, search and unread marks. Each section +names the owner, the host, and the Chord parts it uses; code appears only where it shows something new. + +## The session lifecycle + +The catalog (chapter XVIII) owns the session records. Its operations, by the Chord part each needs: + +| Operation | Owner and form | Notes | +| --- | --- | --- | +| create | `Catalog.create`, a method | absolute directory, `~` expanded, must be a directory; on an environment, resolved there through the envs plugin; refuses an unknown model; model and thinking default to the last used | +| rename, pin | methods on `Catalog` | title 1 to 120 characters, whitespace collapsed; an explicit title is never replaced automatically | +| automatic title | a routine on the search feed | the first user message, line breaks to spaces, 100 characters | +| agent rename | an agent tool on the worker calling `Catalog` | root conversation only | +| archive, restore | methods; revocation; reconciling routines | below | +| clone, fork, relocate | jobs on the catalog store | outlive the call, cancellable, one outcome | +| delete | a job; decision #18 | below | + +### Archive and restore + +Archiving must stop the worker whatever it is doing, end guest access, and close the session's port forwards, while +keeping its storage. The catalog does none of the plugin-specific parts: + +:::steps | Archiving S1 ++++ One write +`Catalog.archive({ id: "S1" })` sets `archived: true` in the session record, in one transaction. ++++ Revoke the session +The catalog calls `host.revoke({ session: "S1" })`. Every guest principal of S1 has `{ session: "S1" }` in its +`revokedBy`, so their objects end and their links close on the server and on the worker (chapter VII). New +connections are refused by `authenticate`, because grants of an archived session are void. ++++ Stop the worker +The supervisor reconciles from `catalog.sessions` with a routine: an archived session's worker is stopped, and +`authorize` refuses to start it again. ++++ Plugins reconcile +The ports plugin's routine on `catalog.sessions` closes forwards of archived sessions. The search plugin marks the +session's rows archived. If the server crashed between the write and these steps, the routines run again with the +current value at the next start: nothing is a notification that can be lost. ++++ Restore +`Catalog.restore` clears the flag. Grants are valid again, so guests reconnect and authenticate as before; forwards +stay closed. +::: + +### Clone, fork, relocate + +Each is work that outlives its call, can be cancelled and has one outcome, so each is a **job** (chapter XI) on the +catalog's local store, keyed by the owner's request id. They differ in what they pause: + +- **Clone** copies the session's store, uploads, downloads and artifacts under a new id, with the source worker + paused (refused while work runs) and title "(copy)"; directories `0700`, files `0600`, no links, no overwrite; a + partial copy is removed on failure. +- **Fork** before a user message copies the active context as of the entry before it into a new session (title + "(fork)"), with the message waiting in the new session's editor. It needs two workers: the source reads history, + the new one is seeded empty. A failure archives the copy. +- **Relocate** (cwd, environment or both): refused while the session is busy (naming the running subagents) or when + a subagent has its own cwd and the move crosses machines. The job fences the worker (new admissions wait), updates + the record, and moves the running worker in place by updating `SessionLocation`: natives that follow the location + switch with a generation (chapter IX), and conversations that follow the session's cwd are rewritten in one Durable + commit before work resumes. After 60 s it restarts the worker instead; on failure it releases the fence. + +:::review | Review: jobs as objects (#9) +The design defers whether a long relocation or a login is a record inside its owner (watched through `running`) or an +object of its own (`Relocation { id }`). The construction above uses records. Decide when login is ported +(`design.md` §15, §26). +::: + +### Delete + +p2 has no session delete. p3 needs one that removes the session everywhere: every document tagged with the session +(`store.retireByTag(SessionTag, id)`, chapter V), the worker's databases (the Durable store), the session's files, the +search rows, read marks, children copies, forwards, guest records, and finally the catalog entry. The design commits to +this outcome (`design.md` §25); how plugins take part is open. + +:::review | Review: session delete (#18) +Options in `design.md` §26: per-session objects plus `SessionTag` plus one session directory (delete is retire by tag +and remove a directory, plugins need do nothing), or owner hooks contributed through a catalog registry. Also open: +whether the search index is per session (deleted with it) or global with a hook. Who holds the store for +`retireByTag` is not specified either. +::: + +## Status, children, unread: reports to the server + +The session list must show status, subagents and unread marks for sessions whose workers are not running, and must +not start them. All three follow chapter X's pub/sub pattern: a routine on the worker calls a server object with the +current summary, the server object keeps the latest value per session (persisted where it must survive a restart), +ignores calls from an older `ctx.incarnation`, and clients read the server object. + +| Summary | Source on the worker | Kept on the server | +| --- | --- | --- | +| status: running, error, task count, busy conversations, usage and cost | the harness's task graph and usage | `SessionStatus`, in memory: a stopped worker is "not running" | +| children: name, model, thinking, tools, cwd, last entry, archived | the subagents plugin's child list | `Children`, persisted | +| latest root answer | the search feed | `Unread`: compared with the owner's read mark, persisted, shared by all owner devices | + +Guests see their own session's status without the error text. The owner's read mark is written when the root +conversation is visible, and a new session starts as read. + +## Conversations + +The harness (chapter XIV) owns the Durable store. The UI needs, per conversation: a live window of entries, older +pages, usage numbers, the working indicator, the inbox (queued inputs), and the actions. Two core objects provide them: + +- **`ConversationStore { conversation }`** (local, on the worker): opens the Durable watch as its native, reaching + the harness through the core-private `HarnessRuntime` registry entry, and exposes a **window** as a derive: the + newest entries (at most 100 or 512 KiB, at least 20) plus the conversation's documents. The window's start stays + put while it is within 1.5 times the limits, so new entries arrive as small appends instead of a shifted list + (chapter VIII's window pattern). +- **`Conversation { session, conversation }`** (remote, placement `Session`): the per-principal view. Variants by + `can()`: `owner` (everything, including model, thinking and handoff), `writer` (guest write: send, steer, stop, + compact, shell), `reader` (the window only). + +```ts +host.serve(Conversation, ({ session, conversation }, principal, self) => { + const store = self.get(ConversationStore, { conversation }); + const policy = self.get(ConversationPolicy, { conversation }); // core: readOnly, followsSessionCwd + const author = authorOf(principal); // structured: kind, id, display name + if (!can(principal, "send", session)) return { variant: "reader", view: store.window, page: store.page }; + return { + variant: can(principal, "manage") ? "owner" : "writer", + view: store.window, + page: store.page, // older history: returns data + send: ({ text, attachments, mode, requestId }, ctx) => { + if (policy.value.readOnly) throw forbidden("read-only conversation"); + return store.submit({ type: "input", content: text, attachments, author, whenBusy: mode, requestId }, ctx); + }, + stop: (_args, ctx) => store.stop(author, ctx), // allowed even when read-only + … + }; +}); +``` + +What each function becomes: + +| Function | How | +| --- | --- | +| send, retried send | Durable `submit` with the author's `requestId`: a retry with the same id is admitted once (Durable dedupes, chapter XIV) | +| steer or queue while busy | `whenBusy: "steer"` or a follow-up; the inbox is `docs["pi.inbox"]` in the window | +| attribution | the view derives the author from the principal; the entry stores it structured, the model sees a speaker block; nothing parses text | +| wait for prompt resources | the first input of a fresh worker waits until the prompt plugin's resources are loaded (a registry readable the store follows) | +| stop, dequeue | withdraw queued inputs, abort the run; the caller's texts return to its editor, other authors' to their **restored drafts** | +| restored drafts | a keyed document per (conversation, author), tagged `SessionTag`, persisted in the store; claimed once, first tab wins | +| read-only conversations | the core per-conversation policy document, written at `createConversation({ init })` by whoever creates the conversation; send, dequeue, configure, compact and shell refuse; stop is allowed | +| history pages | a method returning a page (100 entries, 1 MB, 2,000 scanned); compaction and reset markers always included | +| model and thinking | owner variant only; validated against the models plugin's catalog; the root conversation's model is also reported as the session default | +| compaction, handoff | Durable operations through the store | + +**Drafts in the composer** are UI state, not Chord state: the web shell keeps one draft in `sessionStorage`, the TUI +keeps one per conversation across UI replacements, and pending sends keep their request id until acknowledged +("Sending when connected"). Only restored drafts, which the worker produces, are documents. + +## Shell commands + +`!cmd` runs a command in the conversation's cwd on its environment and records the result as an entry; `!!cmd` does +the same but keeps the output out of the model's context. One command at a time per conversation; the live tail is +at most 50 KB or 2,000 lines, sampled every 200 ms; the full output spills to a file; the result entry is placed at the +run's next boundary while the agent is busy and never starts a run. This is the design's job example almost as is: + +```ts +const ShellJob = defineJob("chat.shell", { + version: 1, + args: obj({ command: str({ max: 100_000 }), include: bool(), author: Author }), + progress: obj({ output: str() }), + result: ShellResult, + concurrency: 1, // one per conversation +}); + +const results = self.state(ShellResults); // keyed by run id: dedupe after completion +const shell = self.job(ShellJob, { + run: async ({ command, include, author }, ctx) => { + const out = new OutputBuffer({ maxBytes: 50_000, maxLines: 2000, retain: "tail" }); + const env = await environmentFor(ctx); // the conversation's environment, chapter XX + const r = await env.exec(command, { cwd: env.cwd, signal: ctx.signal, spill: spillPath(ctx.jobId) }, + (chunk) => { out.push(chunk); ctx.progress({ output: out.snapshot().text }); }); + const data = { command, output: out.snapshot().text, exit: r.code, cancelled: ctx.signal.aborted }; + await conversation.submit( + { type: "write", entry: shellEntry(data, author, include), requestId: `shell:${ctx.jobId}` }, ctx); + return data; + }, + finish: async (done, tx) => { + (await tx.draft(results, done.id)).outcome = done.kind === "done" ? done.result : { error: done.error }; + }, +}); +``` + +The request id `shell:${ctx.jobId}` makes the entry idempotent: an attempt interrupted after the write cannot write +it twice. Cancel is `shell.abort(id)`, allowed for the command's author and the owner. A retried `runShell` with the +same request id returns the running job (dedupe of unsettled runs), or the stored result (the owner's own dedupe after +completion, chapter XI). p2's "cancel when nobody watched for 30 s" is dropped: a job keeps its owner alive and does +not depend on viewers. + +## Chat mode + +Chat mode is a per-conversation switch that everyone in the session sees and only the owner changes; forks and new +subagents start with it off. Because it must fork and rewind with the conversation, it is **transcript-bound**: a +Durable conversation document, not a Chord document. Messages without `@pi` become display-only Durable `write` +entries (author, text, attachments, reply quote); a message with `@pi` becomes an input with a digest of the chat +since the last mention (at most 100 messages, 20 files, 4 images), recording which entries it passed on. The reply +quote is derived on the worker from the entry id the client names: the client never supplies author or quote text. + +## Subagents and one-shot agents + +Subagents are Durable child conversations owned by tasks, as in Durable's own background-subagent example: one +`subagent` tool spawns, steers, waits for, stops and lists them; each child is owned by a background anchor task, and +each message is delivered by a reporter task that posts the answer back to the parent once, with request ids so a +restart sends nothing twice. Chord adds only three things: + +- the **child list** for the sidebar is reported to the server (`Children`, above), so it shows without a worker; +- a child that follows the session's cwd, or a read-only one-shot agent created by a code-mode script, is marked in + the core **per-conversation policy document** when it is created, so chat and relocation enforce it without knowing + the subagents plugin (`design.md` §25 porting guideline); +- archive and restore of a child from the UI are methods on the subagents plugin's view, refused for one-shots. + +Subagents cannot spawn subagents; thinking levels are clamped to what the model supports, in one helper shared with +the session's model choice. + +## Messages between sessions + +The root agent can list other sessions and send one an attributed message. The tool runs on the sender's worker and +calls a server object through `chord(fn, context)` (chapter XIV): + +```ts +const peerSend = defineTool({ + name: "session_send", + execute: (args, api, context) => chord(async (ctx) => { + const peers = await ctx.get(Peers, {}); // server, worker variant + await peers.send({ to: args.session, text: args.text, mode: args.mode, requestId: `peer:${api.taskId}` }, ctx); + return { output: [{ type: "text", text: "sent" }] }; + }, context), +}); +``` + +`Peers` checks that the sender is a root conversation of a non-archived session, refuses sending to itself and +archived recipients, then gets the recipient's `Conversation` (which starts its worker) and submits with the +sender as a structured author and the same request id (the tool task's id, stable across restarts), so a retried +tool call delivers once. The sender's +incarnation is `ctx.incarnation`; a call from a replaced worker is refused. Only owner-initiated turns may use the +tool; guests never. + +## Search + +The search plugin indexes user and assistant messages of root and subagent conversations (no shell output, no +Durable internal inputs) for title, cwd and message search across sessions, active or archived. + +:::flow wide animate | The search feed. The worker pushes pages from a cursor; the server owns the index and never starts a worker to answer a query. +row "Worker S1" blue +box commit "Durable commit" "in any conversation" +box wake "wake" "a routine on the harness's commit signal" +box page "read from cursor" "200 entries per page" +row "Server" accent +box index "SearchIndex" "native: SQLite FTS" accent +box cursor "cursor per conversation" "moves after the whole range" +box query "query" "terms, scope, at most 41 hits" +arrow commit -> wake +arrow wake -> page +arrow page -> index "report(page)" +arrow index -> cursor +note "Reports carry ctx.incarnation. Indexing is idempotent: a page sent twice changes nothing." +::: + +The index is a native owned by a host-lifetime object on the server; cursors live with the index, so after a crash +the worker asks for the cursor and continues. Rebuilding one session's index drops its rows and rescans, starting +an archived session's worker without resuming its work. Queries are methods returning data: at most 41 hits, +snippets of 300 characters, queries up to 500 characters. The same feed sets automatic titles, moves "updated at" +forward and supplies the latest root answer for unread marks. + +## Presence + +Viewers per session are `host.references(Conversation)` on the worker, counted per client (chapter VIII), and +reported with the status. Connected guests (name and mode) are the references whose principal is a guest. Presence +is not keep-alive: the supervisor uses `host.activity`, which also counts calls and jobs. + +:::note | Built so far +None of this exists in p3. The parts it relies on are Durable (built: submit with request ids, steer and queue, +child conversations, background tasks) and the Chord layers L3 to L9 (not built). +::: diff --git a/packages/chord/docs/next/explainer/src/20-p2-machines.md b/packages/chord/docs/next/explainer/src/20-p2-machines.md new file mode 100644 index 000000000..51b9e7546 --- /dev/null +++ b/packages/chord/docs/next/explainer/src/20-p2-machines.md @@ -0,0 +1,245 @@ + +# Machines, Files, Models {#p2-machines lede="Environments, workspace, terminals, session files, ports and Radius, models and logins, prompt resources, logs, reload and the shells: the rest of p3."} + +The last domains are the ones that touch the outside world: other machines over SSH, the file system and git, +terminals, the relay, model providers and their credentials. Most of them are native objects, and most of them are +places where p2 has the same thing several times (four SSH connections to one environment, five notice channels, +three copies of the environment config parser). The rule that fixes that is the one from chapter XVIII: one owner per +thing. + +## Environments + +The **envs plugin** owns the environment configuration: name, host, user, port, identity file, shell and its +variables, plus a managed ed25519 key per environment and its own `known_hosts` (kept apart from the user's, aliased +per environment name). + +| Function | Owner and form | +| --- | --- | +| list, add, edit, delete (refused while a session, archived too, uses it) | `Envs` on the server: a persisted document behind an owner-only view | +| validation (no leading `-`, no whitespace or control characters in host and user) | the view's methods, before the write | +| a hand-edited `envs.json` is picked up; an invalid file keeps the last valid set and shows an error | a native file watcher (poll fallback) in the envs store; one writer (#21) | +| environment revision: a hash of the config, part of its identity | a derive | +| connection test (connect, deploy the daemon, platform, latency, warnings) | a job; the result per revision in a document | +| unknown or changed host key: fingerprints, trust after confirmation, forget | a flow (chapter XII) on the server | +| browse directories (`~`, relative paths, at most 10k scanned, 1,000 shown) | a method with a **call-scoped** SSH session (chapter IX) or the pooled connection | +| a changed environment restarts its sessions' workers once idle | a routine on the envs store and the status reports | + +**Execution in the agent.** The worker's harness asks `HarnessEnvironments` for an environment per target (chapter +XIV). The envs plugin contributes it from its worker facet: local execution at the session's cwd, or a remote one over +SSH through the `pi-env` daemon, deployed per platform and connecting at the first operation. A remote session never +falls back to local tools. The environment follows `SessionLocation`: after a move, the old connection closes once +the move has committed and watchers reopen at the new location. + +```ts +// envs/worker.ts +export default defineFacet((host, self) => { + const location = self.get(SessionLocation, { session }); + const pool = self.get(EnvPool, {}); // one owner of SSH connections (#20) + self.contribute(HarnessEnvironments, (target) => + target.env === "local" ? localEnv(target.cwd) : pool.environment(target.env, target.cwd)); +}); +``` + +:::review | Review: connection pools (#20) +SSH connections to one environment exist four times in p2 (worker environment, browse connection, test connection, +terminal host). `design.md` §26 asks for one owner with a pool or deliberate per-use connections. `EnvPool` above is +the first option: a native object that closes idle connections after 10 minutes and invalidates them on a config or +key change. Reloading the plugin that owns the pool kills running remote commands (chapter XV), so it should change +rarely. +::: + +## Workspace and file index + +The **workspace plugin** serves the viewed conversation's working tree (a subagent's own cwd included) on the worker: +list a folder (at most 2,000 entries, `.git` hidden), read a file (1 MiB, binary detected), git status, diff, log and +single commits, literal text search, and all paths for name filtering. Every query is a **method returning data**, not +state: results are bounded and requested by the UI. Only change notifications are state: a counter in a document, +bumped by a native watcher of the tree, `HEAD`, the index and config (debounced 100 ms); viewers reload what they show +when it changes. + +Two rules live in one place each instead of three: + +- **Confinement:** path rules, a canonical root, no `.git`, the app's private data directory denied (also excluded + from search). One helper used by listing, reading, uploads and the file index. +- **Git hardening:** no optional locks, no fsmonitor, filters disabled, no prompts or pager, no system config, 10 s + timeout, output caps. + +Downloads (25 MiB, byte ranges for media) go through the HTTP component (chapter XIII) with a route on the server that +reads chunks from the worker; uploads into the project use the chunked protocol of chapter XIII (1 MiB chunks, temp +file then rename, never overwrite, writes serialized). Guest read gets no workspace: the view has no variant for it. + +The **file index** for `@` completion belongs to the same owner, so completion consumes the index instead of scanning +on its own: built from the environment's listing (else git or a walk, at most 50k paths), rebuilt at most once per +second while files change, discarded after a move. Review comments drafted on files and diffs are browser state until +sent; sending them is an ordinary `Conversation.send`. + +## Terminals + +Terminals must survive server and worker restarts, so their PTYs live in a **detached terminal host process**, one per +data directory, with its own process lock; it exits after 10 minutes with no terminals and no clients. The terminals +plugin on the server is the owner the UI talks to: list (title, cwd, environment, size, exit status), open at the +session's cwd and environment (refused for archived sessions), attach, close (asking first if a foreground job runs). +Terminals are **owner only**. + +Bytes do not go over object links. Attaching opens a **channel** (chapter XIII) with a single-use token (30 s): JSON +control on the object, bytes on the channel, with its own acknowledged window (256 KiB, acknowledgements per 64 KiB; a +viewer that falls behind gets a fresh screen snapshot; an 8 MiB backlog disconnects). Attaching gets the current +screen from a headless emulator with 10k lines of scrollback. + +:::review | Review: the terminal host process (#23) +The terminal host runs its own code. Is it part of the revision, reloaded or restarted with it (and running terminals +end), or deliberately outside it with a "host runs older code" notice as in p2? Open in `design.md` §26. +::: + +## Session files + +Attachments (25 MiB, at most 20 per message), sent-file snapshots made by the agent's `send_file` tool, versioned +artifacts (HTML, SVG, Markdown, text; 2 MiB; each update a new immutable version) and shell spill files are all +per-session bytes with caps, immutable ids and range reads; p2 has two upload routes, two caps and two directory +layouts for them. The proposal is one owner: + +```ts +export const SessionFiles = remote("p3.session-files", { + version: 1, + input: obj({ session: str() }), + placement: () => Server(), // readable without a worker + variants: { writer: { … }, reader: { … }, worker: { … } }, +}); +``` + +`SessionFiles { session }` on the server holds the index (per kind, readable without a worker), the chunked upload +and read protocol, the caps per kind, copying on clone and fork, and deletion with the session. Attachments are copied +into a remote environment before an input is admitted, with no fallback on failure. Artifact previews run in a +sandboxed frame with its own CSP and no app identity; ordinary file previews allow only safe types. + +:::review | Review: session files (#17) +Proposed in `design.md` §26, not decided. Open sub-points: one preview limit (p2 has 256 KiB for sent files and +512 KiB for artifacts), what a fork copies of the artifact list, and how attachments are copied into remote +environments. Related: #22, whether on-disk layouts such as downloads and uploads get versions like documents. +::: + +## Ports and Radius + +The **radius plugin** keeps the outbound relay connection, a native in a host-lifetime object on the server: sign-in +by device code from the CLI (credential in a private file), token refresh, a heartbeat in the link adapter (20 s ping, +60 s timeout; TCP does not notice a sleeping laptop), backoff from 1 to 30 s, a disable switch. The relay delivers two +kinds of channels: + +```ts +// radius/server.ts (the entry point: one line, as in chapter I) +radius.onChannel((ws) => host.accept(webSocketLink(ws), { principal: { kind: "owner", access: "relay" } })); +radius.onForward((channel) => proxyToForward(channel, host.registry(RadiusForwards).value)); +``` + +Object links become owner principals with `access: "relay"`: owner, but treated as remote (no localhost-only +powers). p2 had to fake a loopback request with a secret header for this; with `accept` that is gone. + +Forward channels go to `localhost:` (Host, Origin and Location rewritten; 502 when nothing listens; anonymous +visitors only to public forwards). Which forwards exist is not radius's business: it owns a registry, +`radius.forwards`, and the **ports plugin** contributes its forwards (at most 50, random ids, a public toggle that +needs confirmation, an agent tool `ports` for top-level owner-initiated conversations). Without the ports plugin the +table is empty and radius still works: an optional dependency is a registry owned by the dependent (chapter XIII). The +ports plugin closes forwards of archived sessions with a routine on `catalog.sessions` (chapter XIX). + +:::review | Review: may a plugin call `accept`? +`design.md` §3 shows the Radius line in the server's start code and calls entry points "app or adapter code". If +radius is a plugin, its facet needs the host's `accept`, which no other part of this guide gives a plugin. The +alternative is that the start code keeps the entry point and gets channels from the radius object. Not specified. +::: + +## Models and credentials + +The **models plugin** owns the model runtime (pi-ai's models with custom providers and the model cache) in a +host-lifetime object, and the credentials. It contributes the runtime to `HarnessModels` (chapter XIV), so after a +models reload the next request uses the new runtime while a running stream finishes on the old one. **Credentials +never go to clients**: the views expose providers, login methods and which type is signed in, never tokens. Every +secret is registered for redaction in one place, the log capture. + +Views: a model catalog for pickers (names, ids, context windows, thinking levels; at most 300 matches in the web), +logout (refused during a login), refresh a provider's model list. The default model and the cycling scope are +settings (chapter XVIII). + +### Login + +A login is the guide's flow example (chapter XII): a **job** on the server that runs pi-ai's login, and a flow that +shows its steps to the starting owner. + +```ts +const LoginJob = defineJob("models.login", { + version: 1, args: obj({ provider: str(), type: AuthType }), progress: LoginState, result: obj({ ok: bool() }), + concurrency: 1, // one login at a time on the server +}); + +const login = self.job(LoginJob, { + run: async ({ provider, type }, ctx) => { + await runtime.value.login(provider, type, { + signal: ctx.signal, // 10 minute deadline, cancel, leaving + prompt: (prompt) => steps.ask(prompt, ctx), // text, secret, select, manual_code + notify: (event) => steps.notice(event, ctx), // info, auth_url, device_code, progress + }); + return { ok: true }; + }, + finish: async (done, tx) => { /* record the outcome; refresh the model list */ }, +}); +``` + +`prompt` is a promise-based callback inside the work; the resolver stays in the job (`steps.ask` records the step in +the flow state and waits for `answer`), and nothing crosses an object boundary. On a relayed connection the device +code method is selected automatically. The flow is visible only to the owner principal that started it; others are +refused. Leaving ends it: when the starting owner's references are all gone (presence, not an unmount), a routine +aborts the job. Codes, URLs and typed secrets are redacted from logs. + +## Prompt resources + +`AGENTS.md`, skills and templates are read in the session's environment, watched (poll fallback, last valid state), +and expanded on the worker before sending. The **prompt plugin** owns them per session on the worker, contributes the +system prompt section and skills to `HarnessExtensions` as readable contributions (a change recomposes), and exposes a +readiness readable: the first input of a fresh worker waits until it is ready (chapter XIX). After a move the watchers +reopen at the new location. + +## Code mode + +Code mode is a plugin (chapter XIV): its `pi-codemode` sandbox is a native; scripts call tools as nested Durable calls +and replay after a restart; the tool entry looks up the current sandbox when it starts. A reload seals and drains the +old sandbox for at most 30 s. The on/off switch is a setting. + +## Logs and notices + +**Log capture** tees stdout and stderr of the server, workers and detached hosts without changing their behaviour, +redacts known secrets and credential patterns, and keeps bounded rings (at most 128 sources, 1,000 lines or 256 KiB +each). Detached hosts publish redacted snapshots to a private file. Logs are read without waking a stopped worker. +**Notices** are the one channel from chapter XVIII; dismissal stores only notice ids. + +## Dev and reload + +The dev plugin is chapter XV's reload control for p3: a source watcher marks plugins dirty and never applies anything; +the `reload_plugins` agent tool and a control in the owner's shell reload the changed plugins, or restart all hosts for +base code (through the supervisor, reopening the sessions that were running). UI bundles are published only after a +successful backend change; a web page detects a new manifest and reloads with its drafts and navigation state saved; +the TUI swaps its sandboxed UI code with a snapshot of its state. A local Unix socket (`0700`) accepts development +commands without a network admin port. + +## The shells + +The web and TUI shells are UI facets over the objects above. Two rules matter for Chord: + +- **UI state survives navigation, reload and reconnect on purpose.** Composer drafts, pending sends with their request + ids, reading positions and review drafts are kept in tab storage or the TUI's replacement snapshot. A reconnect + re-holds and gets snapshots; nothing is lost that was not in flight. +- **Leaving is never derived from an unmount.** A plugin reload remounts components (chapter XV). Cancelling a login, + closing an overlay or giving up a lease is an explicit action or a dropped reference. + +The TUI launcher keeps downloaded UI code in a permission-dropped worker with no access to files, network or +credentials; local side effects (copy, upload, download, open a URL, an external editor) need a real user gesture or a +confirmation. That is app code around Chord, not part of it. + +## Transport + +The last domain, transport, is chapter XVII: one message format, per-part limits (16 MiB compressed and 64 MiB decoded +on client WebSockets, instead of p2's 1 MiB input cap and 32 pending calls), one byte window per link, re-holds with +snapshots after a reconnect, and a protocol version that makes an outdated web page reload. + +:::note | Built so far +None of p3 exists. pi-ai's login API, pi-codemode and the `pi-env` daemon exist and are used by p2 today; the HTTP +component and channels are deferred until after the kernel's first real proof (chapter XIII). +::: diff --git a/packages/chord/docs/next/explainer/src/20a-compared.md b/packages/chord/docs/next/explainer/src/20a-compared.md new file mode 100644 index 000000000..aa4fcd2c2 --- /dev/null +++ b/packages/chord/docs/next/explainer/src/20a-compared.md @@ -0,0 +1,146 @@ + +# Other Ways to Do It {#compared lede="Cordis and DeepSeek Harness attack the same problem, plugins that come and go while an agent runs, inside one process. What that buys, and what it costs."} + +Chord v2 is not the only answer to "replace plugin code while the app keeps running". The closest relative is +**Cordis**, a TypeScript plugin framework by Shigma that has been the plugin kernel of the Koishi chatbot framework +since 2019, and its most visible user, **DeepSeek Harness** (`dsh`), DeepSeek's open-source agent harness, whose +motto is "everything is a plugin". This chapter compares them with Chord v2 on the questions the guide cared about: +who owns side effects, how plugins depend on each other, what a reload keeps, what a crash keeps, and how far the model +reaches beyond one process. + +:::note | Sources and status +DeepSeek Harness at commit `687f282` (release 0.2.1-rc.2), which vendors Cordis as `@deepseek-ai/cordis` 4.0.5 (Cordis +RC7 with local changes, `vendor/README.md`): its Cordis tutorial (`docs/cordis-tutorial/`), `docs/architecture.md`, +`docs/subsystems/persistence.md`, `docs/subsystems/core.md` and `packages/boot/hmr/README.md`; the Cordis paper "A +Programming Paradigm for Spatiotemporal Composability" (arXiv 2608.25512). Cordis and dsh are built and in use; Chord v2 +is the design this guide describes. The comparison is between a working system and a design. +::: + +## Cordis in one example + +A Cordis plugin is a function (or an object with `apply`, or a `Service` subclass) that receives a **context**. It may +declare the services it needs with `inject`, and everything it does to the world goes through the context: + +```ts +export const name = "greeter"; +export const inject = ["tools"]; // activate only while a `tools` service exists + +export function apply(ctx: Context) { + ctx.tools.register(defineTool({ name: "greet", /* … */ })); // a registration is an effect: undone on unload + ctx.on("session/start", (s) => log(s.id)); // an effect-bound listener + ctx.effect(() => { // any other side effect, with its disposer + const timer = setInterval(work, 200); + return () => clearInterval(timer); + }); +} +``` + +Each plugin instance is a **fiber** with states `PENDING → LOADING → ACTIVE → UNLOADING → DISPOSED` (or `FAILED`). +Every effect, registration, listener, child plugin and provided service is recorded on the fiber, and unloading runs +the disposers. The paper calls this **temporal composability**: a component can be removed at any time and reverses its +own effects. Its counterpart, **spatial composability**, is the dependency side: `inject` is live, so a plugin starts +when its services appear, and unloads (and later reloads) when a service it depends on goes away or is replaced. The +paper's claim is that independently written components can therefore be combined and replaced without disturbing each +other. The next sections check how far that holds. + +**Services** are named capabilities on the context (`ctx.provide("x", value)` or `class X extends Service`). Names are +runtime strings in one flat namespace per application; TypeScript declaration merging makes `ctx.x` typecheck, but +nothing checks the service's shape at runtime, and optional dependencies are probed with `ctx.get("x")`. Scopes come +from the context tree: `ctx.extend()` derives a child context, `ctx.isolate(name)` gives one subtree its own binding of +a service, so two parts of an app can use two different `shell` providers. + +**Hot reload** follows from the two halves: the reload plugin unloads the old fiber (unwinding all its effects) and +loads the new module with the same config. Nothing held in the old instance's closures or fields survives; a plugin +that needs state across reloads must keep it in a service or on disk. Config edits are diffed by entry id and +mount, unmount or reconfigure only what changed; config fields declared volatile update in place without a remount. + +## DeepSeek Harness on Cordis + +dsh applies "everything is a plugin" to an agent: the model adapters (`ctx.llm`), the tool registry (`ctx.tools`), the +session log (`ctx.sessions`), the agents (`ctx.agents`) and the agent loop itself (`ctx.agentLoop`) are plugins and +services. A running dsh is a plugin tree composed at boot from a **profile** (`web`, `headless`, `sdk`, `acp`) that +stacks **bundles** of Cordis config rows, with user patches on top. There is no privileged core: replacing the agent +loop is a config patch. + +| Concern | How dsh does it | +| --- | --- | +| sessions | a durable, append-only event log per session (JSONL backend, single writer, `flush` as the durability barrier); model history is derived from the events | +| crash mid-turn | the log keeps everything appended; on resume the agent loop closes the open turn by appending missing tool errors and a synthetic `turn/end { reason: "interrupted" }` | +| tools | `ctx.tools.register(defineTool(…))`, an effect of the registering plugin; calls go through pre/execute/post policy waterfalls | +| approval | closed by default; `allowed-once` is the only grant; plugin installation needs full access or per-call approval | +| reload | a source watcher evicts changed modules; Loader entries running affected plugins stop and start again with the new code; a failed replacement is an activation failure, its previous code no longer runs | +| processes | one plugin tree in the Host process; the browser client and Electron's renderer talk to it over RPC streams | +| distribution | a Plugin Manager installs and enables bundles per profile; plugin markets are optional indexes | +| subagents | local subagents with their own durable sessions, and external ones | + +## What the guarantees are in practice + +Reading the Cordis source vendored in dsh and dsh's own documentation, the guarantees are narrower than the paper's +framing. An earlier review for p2 came to the same result (`apps/p2/docs/p3-reviews/dsh-compare.md`). + +- **Reversal is cooperative and unordered.** `_unload` runs all of a fiber's top-level disposers concurrently + (`Promise.all` in `vendor/cordis/src/fiber.ts`); order holds only inside one effect, and the primer tells authors to + "keep the related work in one effect" when teardown order matters. A disposer that throws is logged and skipped. A + disposer that never settles stalls the teardown; dsh's jobs service works around this by force-failing a job whose + cancel throws and warning that "the work may be orphaned" (`packages/jobs/jobs-local/README.md`). Reversal is as + good as every disposer, and nothing checks that a plugin registered all its effects. +- **Behaviour depends on registration order, and a reload changes it.** `emit`, `waterfall`, `serial` and `bail` + listeners run in registration order (`docs/cordis-primer.md`). Registering appends the listener (`push` in + `vendor/cordis/src/events.ts`, unless `prepend`), so when a plugin reloads, its listeners move to the end of the + list. A tool-policy listener that ran first runs last after its own plugin reloads; a waterfall that short-circuits + now sees a different decision first. +- **Dependencies are strings, and every provider reload restarts its dependents.** A fiber's activation key is the + concatenation of its injected providers' fiber ids (`_refresh` in `fiber.ts`), so replacing a provider unloads and + reloads every plugin that injects it, with their in-memory state. Service APIs are "pre-stable; update every + consumer" (`AGENTS.md`): there is no versioned contract between a provider and its consumers. +- **Reloading a core service ends live work.** Unloading the agent provider "stops and drains every live handle it + made" (`packages/core/agent/README.md`), so running agents stop; reloading the session persistence backend under + live sessions invalidates their write handles, and "writes fail loudly until the sessions restart" + (`.agents/notes/implemented/architecture/2026-08-27-handle-based-session-persistence.md`). +- **Registrations leave a gap.** A reloading plugin's tools and other registry entries are removed with its old fiber + and added again by the new one; anything that looks them up in between does not find them. + +None of this is a bug in the narrow sense: each point is documented, and dsh is built around it. But "components +interleave without disturbing each other" holds only for plugins that own nothing other plugins depend on, register +every effect, order nothing by registration, and hold no state that must outlive a reload. + +## Side by side + +| | Cordis and dsh | Chord v2 | +| --- | --- | --- | +| unit | a plugin instance (fiber) in one context tree | an object per (type, input, principal), placed on a host (chapters II, III) | +| side effects | recorded on the fiber with `ctx.effect` and friends; on unload the disposers run concurrently, each as good as its author made it | declared through `self` (documents, natives, routines, jobs, contributions); Chord activates and undoes them (chapters V, IX to XI) | +| dependencies | service names as strings in one flat namespace, via `inject`; a provider reload restarts every dependent | typed contracts via `get(Type, input)`; a reference parks and recovers through restarts (chapter VI); optional dependencies are registries (chapter XIII) | +| ordering | listeners run in registration order, which changes when a plugin reloads | registries compose with a declared `compose` function; nothing depends on load order (chapter XIII) | +| where code runs | one process; remote UIs over RPC to the Host | a tree of hosts (server, workers, tabs); objects reachable across links, relayed by the server (chapters I, XVII) | +| who is calling | not a framework concept | principals stamped by links, variants per caller, revocation (chapters III, VII) | +| state | the plugin's own business; dsh's session log is a service | documents written in transactions, persisted, versioned and migrated (chapter V) | +| reload | unload the old instance, load the new one: in-memory state and running work of that plugin end | pure objects swap their code in place with their documents and running work kept; only objects with natives restart (chapter XV) | +| a running agent turn during reload | entries of changed plugins stop; unloading the agent provider stops its live agents | the harness is base code; Durable pins tools per task phase, so a running call finishes on old code and the next phase uses the new (chapter XIV) | +| crash recovery of a turn | the turn is closed as `interrupted` | pi-durable continues the turn's tasks from their checkpoints; replay-safe tools rerun (chapter XIV) | +| contracts and versions | TypeScript types only, pre-stable APIs, every consumer updated together; config validated with a schema | contract versions with fingerprints and provider-side mappings for old consumers (chapter XVI) | +| status | in production in Koishi since 2019; dsh released | a design with the contract layer built | + +## What the comparison says + +**The core idea is the same.** "Factories declare, Chord activates" (chapter II) and Cordis' reversible effects are +two forms of one rule: a plugin may only change the world through its runtime, so the runtime knows what to undo. +Both reject the module-level timer and the stored callback for the same reason. + +**The difference is what survives a reload.** Cordis reloads at plugin granularity: the old instance unwinds and the +new one starts from nothing, together with every plugin that depends on it. That is simple, but a plugin's in-memory +state and its dependents' state and work in flight end with it, and correctness rests on every disposer and on +registration order. Chord v2 keeps state out of closures on purpose (documents, references, jobs as +records) so that it can swap the code of a pure object while its data and running work stay, and restarts only what +holds OS handles. That is more machinery: transactions, a store, versions on every document and contract. + +**The other difference is reach.** Cordis composes plugins inside one process, and dsh adds RPC to its own clients. +Chord v2 makes the process boundary part of the model: objects live on the host their placement names, callers are +principals, and a reload rolls out across the server, the workers and every tab. An app like p2, with a process per +session and guests in other browsers, needs that; a single-process agent does not. + +:::review | Review: what a reload does to a running tool call in dsh +The sources say that Loader entries running affected plugins stop and that unloading the agent provider stops its live +agents (`packages/boot/hmr/README.md`, `docs/subsystems/core.md`). They do not say what happens to a tool call in +flight when only the tool's plugin reloads; the comparison above does not claim either way. +::: diff --git a/packages/chord/docs/next/explainer/src/21-epilogue.md b/packages/chord/docs/next/explainer/src/21-epilogue.md new file mode 100644 index 000000000..987fb3c44 --- /dev/null +++ b/packages/chord/docs/next/explainer/src/21-epilogue.md @@ -0,0 +1,104 @@ + +# What's Next {.epilogue lede="Where Chord v2 stands, what is still undecided, and where the written sources disagree with each other."} + +The guide built Studio from one object to a p2-sized app. Every piece used the same few rules: one owner per thing, +factories that only declare, state that changes only in transactions, long-lived behaviour in declared forms, and +mappings and migrations owned by the provider. Those rules are what let Mario press Reload on his phone in the +preface while the agent keeps running. This last chapter says how much of that exists and what is still open. + +## Status + +:::stats +- ~10k | lines in the frozen first kernel (steps 1 to 6, 443 tests), kept as a reference only +- 1 of 11 | layers of the rebuild started: L1 contracts, built and reviewed +- 6–7k | target size of Chord v2 without tests +::: + +- **The design is decided** except the points marked below. `packages/chord/docs/next/design.md` is the target; + `packages/chord/docs/next/decisions.md` is the log and wins where they differ. +- **The rebuild** goes layer by layer, each with a one-page spec the owner approves, a minimal build with counting and + garbage-collection tests, the cost-guard metrics, and a cleanup loop: L1 data and contracts; L2 readable and + derive; L3 documents and store; L4 objects; L5 references and `ctx`; L6 protocol and links; L7 relay, routes, + revocation; L8 routines and jobs; L9 registries; L10 reload and rollout; L11 telemetry in Chord. +- **Built:** the contract layer (`src/next/contracts/`, commit `7d2bb8ea8`, chapter XVI). In pi: Delta, + `pi-telemetry`, `chord/context`, `chord/telemetry-sqlite`, pi-ai and Durable telemetry, Durable's required + `checkpointWhen`. +- **Next, in order** (`apps/p2/docs/p3-handoff.md`): decide Q12 and Q15, which the L1 spec needs; write the spec for + the rest of L1 (values, inputs and identity, argument checks, principals, errors; draft in + `packages/chord/docs/next/layer-1.md`); then Q11, Q13, Q14 and the cuts C3 to C17 before the layers they affect. +- **In pi:** `mapShared` and bytes as atomic leaves in Delta's `diffRevisions`. +- **Then** the playground web shell (React, esbuild) with the reload control and a preview at `p3.mariozechner.at`, + and the first real proof: the harness, one tool with an approval pause, a code mode reload. Then the HTTP and + channel components, then p3, domain by domain. + +## Deferred decisions + +Kept with stable numbers in `design.md` §26, each waiting for a concrete case: + +| # | Topic | When | Chapter | +| --- | --- | --- | --- | +| 9 | jobs as objects: a record inside the owner, or an object addressed by its id | when login is ported | XI, XIX | +| 11 | where a flow lives: a keyed document of its owner, or an object `LoginFlow { id }` | when login is ported | XII | +| 14 | project plugins: per-directory plugins with trust prompts and snapshots | own pass over `plugin-loading.md` | XV | +| 16 | a Durable Object profile | when a DO target becomes concrete | | +| 17 | session files: one `SessionFiles { session }` owner on the server (proposed) | p3 design | XX | +| 18 | session delete: retire by tag plus a session directory, or owner hooks | p3 design | XIX | +| 20 | connection pools: one owner of SSH connections per environment, or per-use connections | p3 design | XX | +| 21 | config files with a single writer: one owner on the server, others follow | p3 design | XVIII | +| 22 | file formats (`envs.json`, downloads, uploads) as versioned interfaces | p3 design | XX | +| 23 | the terminal host process: inside the revision or deliberately outside | p3 design | XX | + +## Open from review round 3 + +Decisions (`apps/p2/docs/p3-reviews/v2-review3-summary.md`; Q1 to Q10 are decided): + +| # | Problem | Proposal | +| --- | --- | --- | +| Q11 | a singleton that does not exist yet: `undefined` or `initial()`? | singletons read `initial()`; `undefined` only for absent keyed documents | +| Q12 | stripping arguments (R7) against "nothing is copied" | copy on change: a new object only when the argument had extra fields; never mutate the caller's value (proposed in the L1 draft) | +| Q13 | `ctx.incarnation` on relayed calls | relays stamp the origin host's incarnation, as they stamp the principal | +| Q14 | a writable holding functions exposed as a state member | only members with a JSON schema in the contract are exposable, checked at serve | +| Q15 | arguments of `local` methods: validated or not | apply R7 to `local` too, or keep `local` as the one exception | + +Cuts the owner may choose (each removes a primitive this guide teaches): + +| # | Cut | What it costs | +| --- | --- | --- | +| C3 | shrink jobs to a runner over documents (no versions, `accepts`, `blocked`, `persist`), or move jobs out of the kernel | job versioning, or `self.job` for core-only users | +| C4 | routines: `on` and `loop` only | `every`'s due time across reloads | +| C5 | merge `lifetime: "host"` objects into the facet | a plugin's always-on natives share one failure domain | +| C6 | remove `broken()` for opens and `retryOn` | nothing; reload and Retry still work | +| C7 | remove provider-side `linger` | ten lines of app code | +| C8 | remove `state: false` (method-only references) | a method-only consumer pays one snapshot | +| C9 | remove `previous` from derives | derives that used it move to `mapShared` | +| C10 | one `cell(initial)` for `constant` and `writable` | nothing | +| C11 | remove `ctx.trace` (use `traceIn(ctx, …)`) | nothing | +| C12 | remove call credits (the byte window covers calls) | nothing | +| C13 | two timeouts (`closeTimeout`, `readyTimeout`) instead of eight | per-thing overrides | +| C14 | remove the `handle.change` and `handle.retire` shorthands | nothing | +| C15 | remove `inVariant` and `expectVariant` (keep `RefIn`) | nothing | +| C16 | remove facet-owned mutable documents and their transaction exception | atomic writes across an object and its facet move to one owner | +| C17 | no harness-specific start-barrier exception | factories before the barrier cannot depend on the harness | + +## Where the sources disagree + +This guide follows the decision log where it is newer than the design, and the design where it is newer than +everything else. The places that need a fix in the sources: + +:::review | Review: drift in the written sources +- `design.md` §26 lists only the deferred decisions; it omits the open review-3 items Q11 to Q15 and the cuts C3 to + C17. +- `design.md` §3 has a garbled sentence, "Values are method arguments are stripped". +- `apps/p2/docs/p3-handoff.md` §3 is stale: it says all R items are decided and, two lines later, that R2 to R5 and + R7 to R26 are open, and "resume R6 next". +- `kernel.md` and `step-*.md` describe the frozen code: host values, a development freeze, consumer-side linger, + JSONL storage, descriptions on the wire, and a link closed on an unknown id. None of these is the target. +- `chord-next-walkthrough.md` lags the log: validation by trust (now R7, every caller), dev mode and freeze (removed + by R9), descriptions once per link (gone with R6), an IPC frame-count window (now one byte window, Q4), job outcome + `{ ok }` (now `{ kind }`), harness activation as the start barrier (R17), registries "newer refused at load" (Q8), + and hooks using `host.root` (now `chord(fn, context)`, Q6). +- `apps/p2/docs/questions.md` uses `self.resource` and `replace` (both rejected) and `host.root` in hooks. +- `apps/p2/docs/p3-reviews/v2-cost-model.md`: the measurements are current, its "Decisions this needs" section is + not (chapter XVII). +- L1's `Implementation` shape differs from the factory return value used in the design (chapter XVI). +::: diff --git a/packages/chord/docs/next/explainer/src/22-appendix.md b/packages/chord/docs/next/explainer/src/22-appendix.md new file mode 100644 index 000000000..42eb965f4 --- /dev/null +++ b/packages/chord/docs/next/explainer/src/22-appendix.md @@ -0,0 +1,190 @@ + +# API Reference {.appendix #api lede="Every name the guide uses, grouped by where you meet it. Working names; only the contract layer is built."} + +## Hosts + +| Call | Meaning | Chapter | +| --- | --- | --- | +| `createHost({ address?, principal?, storage?, principalKey?, revokedBy?, telemetry?, onError?, timeouts })` | a host; with an address it is addressable and its own code acts as `principal`; without, a client | I | +| `host.load([{ name, facet }])` | load facets by plugin name; base code passes its core facet the same way | I | +| `host.connect(link, { principal })` | the link to the parent (at most one) | I | +| `host.accept(link, { principal })` | a link from a client; every request on it acts as `principal` | I | +| `host.route(Kind, { authorize, resolve })` | how to reach hosts of a kind; `resolve` returns `{ link, principal }` | I | +| `host.revoke(r)` | end matching objects and links here and below | VII | +| `host.root(fn, { principal?, signal?, telemetry? })` | a call scope with a `ctx` for code outside objects | VI | +| `host.get(Type, input, { signal, state?, parkTimeout? })`, `host.hold(…)` | a reference from outside objects; `release()` ends it | VI | +| `host.serve(Type, factory, { lifetime?, linger? })` | serve an object type (in a facet) | II | +| `host.registry(Registry)` | the composed value of a registry, a readable | XIII | +| `host.references(Type)` | who holds references to objects of a type on this host (presence) | VIII | +| `host.activity` | `{ references, calls, jobs, lingering }`, for keep-alive | I | +| `host.close()` | end the host: objects, natives, links | I | + +## Inside a factory: `self` + +| Member | Meaning | Chapter | +| --- | --- | --- | +| `self.state(Definition)` | a document handle | V | +| `self.tx(async (tx) => …, ctx)` | a transaction on the object's line | V | +| `self.derive(name, [sources], fn)` | a derived readable | VIII | +| `self.get(Type, input)` | a reference held for the object's life | VI | +| `self.open(open, close, { closeTimeout? })` | a native; `.value` once open | IX | +| `self.routines.on(readable, fn)`, `.every(interval, fn)`, `.loop(fn)`, `.custom(…)` | standing behaviour | X | +| `self.job(Job, { run, finish? })` | owned work that outlives its call | XI | +| `self.contribute(Registry, value)` | a contribution (facets and `lifetime: "host"` objects only) | XIII | +| `self.writable(initial)` | a host-local readable of any in-process value, with `set` | XIII | +| `self.retryOn(readable)` | a trigger that retries a broken activation | VI | + +**Document handles:** `value`, `get(key)` (keyed), `keys`, `change(fn, ctx)` / `change(key, fn, ctx)` and `retire(ctx)` / `retire(key, ctx)` (shorthands for +one-document transactions). **Transactions:** `tx.draft(handle, key?, { tags? })`, `tx.retire(handle, key?)`, +`tx.start(job, args, { key? })`. **Jobs:** `job.start(args, ctx, { key? })`, `job.abort(id)`, `job.running`, +`run.outcome(ctx)`. + +## References and `ctx` + +| Member | Meaning | +| --- | --- | +| `ref.` | a state member (readable: `.value`, subscribe) or a method | +| `ref.state`, `ref.onState(fn)` | `loading`, `live`, `failed` (cause, `retryAt`), `released` | +| `ref.retry()` | retry a failed reference now | +| `ref.release()` | end a reference taken outside objects | +| `ref.members` | the variant's members | +| `ctx.signal`, `ctx.principal`, `ctx.incarnation`, `ctx.get` | cancellation, caller, calling host's incarnation, call-scoped references | +| `ctx.telemetry`, `ctx.trace(name, attributes, fn)` | the current span; a child span with a child `ctx` | +| `ctx.jobId`, `ctx.progress(value)` | in a job's `run` | + +## Definitions + +| Name | What it declares | Chapter | +| --- | --- | --- | +| `hostKind(name, input?)` | a kind of host and its address | I | +| `remote(name, { version, input, placement, variants })` | an object type reachable across links | II | +| `local(name, { version, input, members })` | an object type on its own host | IV | +| `method(args?, result?)` | a method in a base declaration | II | +| `evolve(prev, (e) => …)`, `checkAccepts(contract, accepts)` | the next contract version; the provider's mappings (**built**) | XVI | +| `Implementation` | the type of a provider for the current version (**built**) | XVI | +| `str num int bool nul lit oneOf union obj arr map opt bytes unknown`, `Static` | schema helpers (**built**) | II | +| `defineState(name, { schema, version, initial, checkpointWhen, persist?, keyed?, ordered?, migrate? })` | a document | V | +| `defineTag(name)` | a tag; `store.retireByTag(tag, value)` | V | +| `defineJob(name, { version, args, progress, result, concurrency?, persist?, accepts? })` | a job type | XI | +| `defineRegistry(name, { version, compose, accepts? })` | an extension point | XIII | +| `bindFacet

()` | base code's `defineFacet`, with factories' principals typed as the app's `P` (working name) | III | +| `defineFacet((host, self) => …)` | a plugin's setup for one kind of host; imported from base code (`@studio/core`) | II, III | +| `constant(value)` | a readable that never changes (start data) | IX | +| `broken(error)` | a failure that is not retried by the backoff | VI | +| `inVariant(ref, …)`, `expectVariant(ref, …)`, `RefIn<…>` | narrowing a reference by variant | III | +| `mapShared(fn)` | an array map that keeps unchanged items identical (planned in Delta) | VIII | +| `OutputBuffer`, `Progress` | bounded output and throttled progress (`@earendil-works/chord/output`) | XI | +| `memoryLinkPair()`, `ipcLink(child)`, `webSocketLink(ws)` | link adapters | I | +| `DocumentStorage`, `registerDocumentStorageConformance` | the storage port and its shared test suite | V | + +# Timeouts and Defaults {.appendix #defaults} + +| Timeout or default | Bounds | Default | Override | +| --- | --- | --- | --- | +| `closeTimeout` | a native's `close`, then reported as a possible leak | 30 s | per native | +| call drain | an ending object's own call handlers | = `closeTimeout` | per type | +| `readyTimeout` | activation; an awaited `get` | 60 s | per `get` | +| park deadline | parked calls (link drop, loading, rollout) | 60 s | per `get` (`parkTimeout`); per type | +| routine handover | a new routine waiting for the old handler on reload | 10 s | per routine | +| `finish` retry | backoff of a throwing job `finish` | 1 s to 60 s | per job | +| rollout install | one host installing a revision | 60 s | none | +| linger | an object kept alive after its last reference | per type | per type | +| backoff | retries of transient failures, per address | 1 s doubling to 60 s | none | +| eviction | keyed persisted documents idle in memory | 10 minutes | | +| client WebSocket limits | per part | 16 MiB compressed, 64 MiB decoded | per link kind | +| compression | JSON parts compressed at their origin | above about 16 KiB | | +| worker keep-alive (p2, app code) | idle time before the supervisor stops a worker | 5 minutes | app | + +# Versions {.appendix #versions} + +| Surface | Version | On change | Chapter | +| --- | --- | --- | --- | +| object type | `version` plus fingerprint | `evolve` and `checkAccepts`; otherwise `contract_mismatch` | XVI | +| document | `version` plus schema fingerprint | pure `migrate` on access; no bump fails `broken` | V | +| job | `version` | `accepts` on outcomes and `running`; `blocked` without | XI | +| registry | `version` | `accepts` on contributions; unmatched ones left out and reported | XIII | +| native | none | restarts on reload | IX | +| Durable tasks and documents | Durable's `version` | Durable's `migrate` | XIV | +| link protocol | protocol version | refused; hosts update together | XVII | +| UI components, Chord and app APIs | none yet | exposed as they are | | + +# Errors and States {.appendix #errors} + +`ChordError` has a stable `code` and a finer `reason`; on the wire it is `{ code, reason?, message, details? }` +(specified in `layer-1.md`, not built). + +| Code | Meaning | +| --- | --- | +| `invalid` | input, arguments or placement rejected (`invalid/input`, `invalid/args`, `invalid/placement`) | +| `contract_mismatch` | the consumer's version and fingerprint are neither current nor accepted | +| `forbidden` | the caller may not do this | +| `gone` | the object or hold ended | +| `not_ready` | not activated in time | +| `released` | the reference was released | +| `too_large` | a part exceeded its size limit | +| `unavailable` | could not be reached before the deadline (`unavailable/locked` and others) | +| `aborted` | cancelled by a signal | + +| Type | Values | +| --- | --- | +| `FailureCause` | `{ kind: "transient"; error }` (retried with backoff) or `{ kind: "broken"; error }` (retried only by a trigger) | +| reference state | `loading`, `live`, `failed` (cause, `retryAt`), `released` | +| job outcome | `{ kind: "done"; result }` or `{ kind: "failed"; error: "threw" \| "aborted" \| "lost" }` | +| job phase | `running`, `finishing`, `blocked` | +| rollout status of a provider | `updating` (calls park), rejected (fail at once), same revision (`contract_mismatch`) | +| host install report | `installed`, `installed_failed`, `rejected` | + +# Rejected Designs {.appendix #rejected} + +From `design.md` §31, grouped. Each was considered and is not coming back without a new reason. + +| Area | Rejected | +| --- | --- | +| reload | drain-based handover; object restart for state changes; a factory trial; a migrate dry run; a consumer contract check during staging; reload votes; per-plugin rollback within a host; keeping natives across reloads by name; `restart_required`; host values and a development freeze | +| state | version fences without transactions; volatile documents; per-group lines; optimistic transactions; `tx.get`; a JSONL document storage; validating state and documents on commits or reads; copying values on same-host calls; op-rewriting projections and a view algebra; Durable's Session as the object store | +| objects and references | resources replaced in place (`replace`, `deps`, stale handles); revoking proxies on `.value`; consumer-side linger; detached references counting toward presence; an `optional` flag on references; `opaque` members; callbacks as method arguments; generic call or request ids; a stream type in Chord | +| registries | registries created at runtime; entries resolved through mutable indirection; generic registry keys with a `conflict` flag; numeric middleware priorities; registry composition as a staging check | +| principals and placement | principal field patterns and re-stamping for revocation; `storeKey`; placement as data naming an input field; host-name strings in app code; "magic" input-field matching for delete | +| wire | an outbox for transient delivery errors; cross-relay reset coordination; count limits that refuse calls or holds; contracts taken from the running app | +| process and tooling | a topology layer above the host API; loader variants A and B; Chord ending the process on a storage failure; `defaultProjectTrust: "always"`; ceremony (`native/` directories, `defineNative`, lints); weakening type checking | + +# Glossary {.appendix #glossary} + +**Adapted binding:** a consumer bound at an older accepted contract version, served through the provider's mappings. +**Base code:** the app's own code, imported statically; a change restarts the hosts. **Client:** a tab or TUI host, +not addressable. **Contract:** an object type's declaration (name, input, placement, members, version). +**Contribution:** a value a plugin adds to a registry. **`ctx`:** what every call receives. **Derive:** a value +computed from readables. **Document:** a named JSON value an object owns. **Facet:** a plugin's code for one kind of +host. **Factory:** the synchronous function that declares an object. **Fingerprint:** a hash of a contract's or +schema's normal form. **Flow:** input into running work, kept as state by its owner. **Host:** one participant of the +app, with an identity and an incarnation. **Incarnation:** the id a host gets on every start. **Input:** an object's +key among objects of its type; with the type and the caller's principal key it identifies the object. **Job:** owned work that outlives its call. **Link:** an ordered connection between +two hosts. **Local type:** an object type reachable only on its own host, without principals. **Native:** an OS-level +value opened with `self.open`. **Park:** a call waiting for a reference to become live again. **Placement:** the +function from input to host address. **Plugin:** a directory with a `package.json`, reloadable. **Principal:** who +calls, stamped by the link. **Principal key:** the string that identifies a principal in object identities. **Pure +object:** an object without natives; swapped in place on reload. **Readable:** a value plus its ordered changes. +**Reference:** what `get` returns. **Registry:** an owner's extension point. **Relay:** a host forwarding messages +between two links, rewriting only headers. **Reload:** swapping a plugin's code in running hosts. **Restart:** ending a +process. **Revision:** the complete code set the hosts should run. **Revocation:** ending live access for principals +matched by `revokedBy`. **Routine:** standing behaviour of an object. **Server:** the addressable root host. **Shared +store:** a local object that holds data once behind per-principal views. **Tag:** a typed label on documents, used to +retire them together. **Transaction:** an atomic set of document writes on the object's line. **Variant:** the set of +members a principal gets. **View:** a per-principal remote object in front of a shared store. **Worker:** a host the +server starts for one session. + +# Sources {.appendix #sources} + +| Source | Used for | Weight | +| --- | --- | --- | +| `packages/chord/docs/next/decisions.md` | every decision, review rounds 2 and 3 | wins where newer | +| `packages/chord/docs/next/design.md` | the target design, all chapters | authoritative otherwise | +| `packages/chord/src/next/contracts/` and its `README.md` | chapter XVI | built code | +| `packages/chord/docs/next/layer-1.md` | errors, inputs, argument checks | draft spec | +| `apps/p2/docs/p3-reviews/v2-review3-summary.md` | open Q11 to Q15, cuts C3 to C17 | current | +| `apps/p2/docs/p3-reviews/v2-cost-model.md` | measurements, per-hop costs | current except its last section | +| `apps/p2/docs/p3-functions/` | chapters XVIII to XX | current | +| `apps/p2/docs/p3-handoff.md` | status and order of work | §3 partly stale | +| `packages/durable/README.md` | the harness, submit, tasks, subagents | current | +| `chord-next-walkthrough.md`, `kernel.md`, `step-*.md`, `apps/p2/docs/questions.md` | background only | lag the decisions (see What's Next) | +| DeepSeek Harness at `687f282` (its Cordis tutorial, architecture, persistence, core and HMR docs), the Cordis paper (arXiv 2608.25512) | "Other Ways to Do It" | external, built systems | diff --git a/packages/chord/docs/next/explainer/walkthrough.md b/packages/chord/docs/next/explainer/walkthrough.md new file mode 100644 index 000000000..e4bfa5a9e --- /dev/null +++ b/packages/chord/docs/next/explainer/walkthrough.md @@ -0,0 +1,284 @@ +--- +title: Chord v2 +kind: walkthrough +kicker: Walkthrough · Chord v2 +subtitle: Build an app like p2 on Chord v2 and pi-durable, one concept at a time, with a tutor that draws the figures and asks the questions +model: anthropic/claude-sonnet-5-5 +sources: + - packages/chord/docs/next/explainer/book.md + - packages/chord/docs/next/design.md + - packages/chord/docs/next/decisions.md + - packages/chord/docs/next/layer-1.md + - packages/chord/src/next/contracts/README.md + - packages/chord/src/next/contracts/*.ts + - packages/durable/README.md +truth: treat Chord v2 as fully built as designed; ignore what the sources say about build status. redesign-decisions.md wins over design.md; design.md wins over the guide (book.md) +--- + +:::tutor +The learner knows pi-durable (harness, conversations, tasks, tools) and nothing about Chord. + +Teach Chord v2 as a finished, working runtime: never say a part is "not built", "planned" or "the target design", even +though the sources say so in places. Open decisions may be mentioned as "still being decided" when a concept depends on +one. + +The running example is Studio, introduced on the page before the first concept (see the introduction). Use it in every +concept and grow it step by step, as the guide book.md does chapter by chapter: read the matching guide chapter before +each concept, and check details in design.md and the decision log. Name the guide chapter in "Read more" lines. + +Keep code short and complete enough to read on its own. Prefer one figure per concept: a sequence for message orders, +a flow for structure, a stepper for lifecycles. +::: + +:::intro +#### What this is about + +Picture **p2** (Pi Mobile): a web app for working with pi agents from any device. It runs on your laptop as one +**server** process. Every chat session gets its own **worker** process, and in each worker a pi-durable agent works +through its turns. You use it from **browser tabs** on your desktop or phone, and you can invite a colleague to watch. +Most features (a notes panel, file browsing, terminals, provider logins) are **plugins**. + +Now you change one line in a plugin and press Reload, while an agent is twenty minutes into a task. The panel should +show the new code on every screen. The agent's turn, the running tool, the open connections and every bit of saved +data should simply continue. No restart, no lost work. + +**Chord v2** is the runtime that makes this normal. An app on Chord is a set of **objects** spread over the server, the +workers and the tabs. Each object owns its data and its OS handles, declares what it needs, and Chord creates, +connects, reloads and ends it. Because Chord knows every piece of state an object has, it can swap the object's code +while its data stays. + +#### The example: Studio + +This walkthrough builds **Studio**, a small p2, on Chord v2 and pi-durable. It has the same three kinds of processes: + +:::flow wide | Studio: one server, a worker per session with its agent, browser tabs as clients. The notes plugin is the first feature we build. +row "Clients" muted +box owner "owner's tab" "phone or desktop" +box guest "guest's tab" "read-only" +row "Server" accent +box server "server" "session list, access" accent +row "Workers" blue +box worker "worker S1" "agent + notes" blue +box worker2 "worker S2" "agent + notes" blue +arrow owner -> server "WebSocket" +arrow guest -> server "WebSocket" +arrow server -> worker "IPC" +arrow server -> worker2 "IPC" +::: + +The first feature is a **notes plugin**: a panel where the owner adds notes to a session, a guest reads them, and the +agent can add notes through a tool. Each concept adds one piece to Studio, until it has everything a p2 needs: who may +do what, saved data, live updates, background jobs, logins, extension points, the agent, and reload. + +#### How it progresses + +1. **The pieces:** processes, objects, callers, shared data, saved data. +2. **Living objects:** references that survive restarts, revocation, computed values, OS handles, standing behaviour, + background jobs, input from people. +3. **The app:** extension points between plugins, and the pi-durable agent inside Chord. +4. **Change:** reloading code while everything runs, evolving contracts, and what goes over the wire. +::: + +# What we're building {#goal part="The pieces"} +Guide preface ("Before we begin", "Ten things to know"). Must cover, briefly and concretely: the reload scenario (phone, +agent mid-turn, guest watching, one-line fix, Reload) and why restarting processes or hot-swapping modules fails; the +one rule behind Chord (every piece of state, OS handle and running work has one owner, an object, and objects only +declare what they need); Studio's pieces from the introduction. Do not go into APIs yet. + +:::check +1. Why is restarting the worker processes not a good way to load new plugin code? ++++ answers +1. It loses the agent's turn in flight, open terminals and connections (pi-durable would resume the turn, but the running tool and the connections are gone). +::: + +# The shape of an app {#shape part="The pieces"} +Guide chapter I. Must cover: hosts (server, workers, clients), links, host kinds and placement, createHost/load/connect/accept/route, +clients reach workers only through the server, incarnations, keep-alive is app code (host.activity). + +:::check +1. A tab wants an object that lives on worker S1, which is not running. What happens, in order? +2. Why does Chord have no listen() call? ++++ answers +1. The tab's call goes to the server; the address is not the server's, so it uses route(Session): authorize, then resolve starts the worker and returns a link plus principal; the server relays the hold, the worker builds the object and answers with a snapshot. +2. Entry points differ too much (loopback WebSocket, Radius channels, a Durable Object edge, memory links); each authenticates and calls accept. +::: + +# A first object {#first-object} +Guide chapter II. Must cover: contract (name, input, placement, variants, version): the input is the object's key +among objects of its type, `placement` computes the host from it; schema helpers, facet and factory (a facet imports +Studio's own defineFacet from @studio/core), serve and get, methods return data, placement checked on both sides. + +:::check +1. Two tabs of the owner ask for Notes of S1. How often does the factory run? +2. A caller with an outdated placement rule asks worker S2 for Notes of S1. What happens? ++++ answers +1. Once per principal key; the owner's two tabs on the same access path share the object, so once. +2. The provider runs the placement function itself and refuses with invalid/placement. +::: + +# Who is calling {#principals} +Guide chapter III. Must cover: the app's principal type, links stamp principals, principal key in the object identity, +variants per principal, narrowing with inVariant/RefIn, and how a facet knows P: base code exports defineFacet bound to +P (bindFacet

()), every facet imports it, and host.load accepts only facets typed for its P. + +:::check +1. Can a client send a principal field to claim it is the owner? +2. Why do guests not get an add method that throws "forbidden"? +3. The notes plugin's worker facet narrows principal.kind === "guest". Where does the type checker get Studio's principal union from? ++++ answers +1. No: a client's principal field is always overwritten by the link's stamp; only a trusted parent link may carry one. +2. Variants: the reader variant simply has no add member, and Chord refuses calls to members outside the caller's variant. +3. From Studio's base code: @studio/core exports defineFacet = bindFacet

(), the facet imports that one, and the worker's host.load accepts only facets typed for the same P. +::: + +# One store, many views {#shared-store} +Guide chapter IV. Must cover: local types, the shared store pattern, attribution as data, argument validation for every caller (R7, Q15 open). + +:::check +1. Where does "who wrote this note" come from if the store is local and has no principal? ++++ answers +1. The remote view knows the caller and passes the author into the store's method as data. +::: + +# Writing it down {#documents} +Guide chapter V. Must cover: defineState, self.tx, drafts and ops, persistence only for stable identities, versions with migrate, tags, storage. + +:::check +1. Why may a remote per-principal object not persist documents? +2. A schema changes but the version is not bumped. What happens? ++++ answers +1. Its identity includes the principal key, which changes (devices, grant revisions); persisted data needs a stable identity, so it lives in a local store or a host-lifetime object. +2. The stored fingerprint differs from the definition's at access, and the object fails broken ("schema changed without a version bump"). +::: + +# Holding on {#references part="Living objects"} +Guide chapter VI. Must cover: reference lifecycle, ref.state, parking, transient vs broken failures, backoff, ctx. + +:::check +1. The link to a worker drops while a tab calls a method. What does the call do? ++++ answers +1. A call already in flight fails like on a disconnect; calls made while the link is down park until the reference is live again (up to the park deadline, 60 s), then fail unavailable. +::: + +# Taking access away {#revocation} +Guide chapter VII. Must cover: revokedBy, host.revoke, propagation to children, no re-stamping, lasting denial in app data. + +:::check +1. After revoke({ grant: "g1" }), Ann reloads her tab. Why does she not get access back? ++++ answers +1. revoke cuts only live access; the app's guest record is blocked and authenticate refuses it on the new connection. +::: + +# Computed values {#derives} +Guide chapter VIII. Must cover: derives, the footgun (new objects for unchanged data), mapShared, container width, presence with host.references. + +:::check +1. Why is items.map(stripAuthor) in a derive expensive on every commit? ++++ answers +1. It builds new objects for every unchanged note, so the diff walks and sends O(value) instead of O(change); mapShared keeps unchanged items identical. +::: + +# Things the OS gives you {#natives} +Guide chapter IX. Must cover: natives with self.open, pure vs native objects, start data with constant, lifetime host, call-scoped natives. + +:::check +1. What happens to a native object on reload, and why? ++++ answers +1. It restarts: its natives close and reopen with the new code, because old and new code sharing an OS handle would interleave unchecked. +::: + +# Standing behaviour {#routines} +Guide chapter X. Must cover: routines (on, every, loop), serialized handlers, reconciling instead of notifying, pub/sub reports with ctx.incarnation. + +:::check +1. Why does archiving a session not call notes.archive(session) directly? ++++ answers +1. A notification can be lost (worker not running, link drops, crash); a routine on the catalog's sessions reconciles from the current value, also after a crash. +::: + +# Work that outlives the call {#jobs} +Guide chapter XI. Must cover: defineJob, the intent / effect / outcome sandwich, dedupe of unsettled runs (Q9), finish in current code, abort as a request. + +:::check +1. A shell job's run crashes after writing the result entry. How is a duplicate entry avoided? ++++ answers +1. The external write carries a key derived from ctx.jobId (requestId shell:jobId), so Durable dedupes it. +::: + +# Asking a person {#flows} +Guide chapter XII. Must cover: a flow is state, not a call; steps helper; the Questions service; decisions #9 and #11 open. + +:::check +1. Why not a long-held step() call that returns the next login step? ++++ answers +1. A link drop fails it and the consumer cannot tell if it missed a step; state re-holds by itself and shows the current step. +::: + +# Extension points {#registries part="The app"} +Guide chapter XIII. Must cover: registries, compose, who may contribute (R21), contributions that change, optional dependencies, HTTP component, channels, chunked transfers. + +:::check +1. Radius needs forwards from the ports plugin, which may be missing. How is that expressed? ++++ answers +1. Radius owns a registry (radius.forwards) and ports contributes to it; without ports the table is empty. +::: + +# The agent {#agent} +Guide chapter XIV. Must cover: the Harness as a core host-lifetime object, HarnessModels / HarnessEnvironments / HarnessExtensions registries, chord(fn, context) for tools, approval via Questions. + +:::check +1. How does a Durable tool get a Chord ctx? ++++ answers +1. Through chord(fn, context): the host is in Durable's context under a context key, and host.root gives fn a ctx. +::: + +# Reload and rollout {#reload part="Change"} +Guide chapter XV. Must cover: load, staging, commit, per-object swap or restart, rollback all-or-nothing per host, revisions, rollout order, updating vs contract_mismatch. + +:::check +1. A staging error happens in one of three plugins being reloaded on a host. What runs afterwards? ++++ answers +1. The whole host stays on its previous code; rollback is all-or-nothing per host. +::: + +# Changing contracts {#contracts} +Guide chapter XVI; the code is in src/next/contracts. Must cover: evolve, checkAccepts and mapping directions, binding by version and fingerprint, new history bases, the fingerprint normal form. + +:::check +1. Which way does a state mapping go, and which way an args mapping? +2. Why can an input change not be adapted? ++++ answers +1. State: current to old (the old consumer reads). Args: old to current (the old caller sends). +2. The input is the object's address; changing it changes what the object is, so it needs a new history base and old consumers get contract_mismatch. +::: + +# On the wire {#wire} +Guide chapter XVII. Must cover: messages and parts, relays rewrite only headers, one state message per commit per link, flow control (R19, Q3), size limits per part, the cost model, telemetry. + +:::check +1. What does the server do with a state part it relays from a worker to a tab? ++++ answers +1. It reads and rewrites only the header (maps ids, groups parts per destination link) and forwards the part without decoding it. +::: + +# Other ways to do it {#compared part="In context"} +Guide chapter "Other Ways to Do It" (after chapter XX). Must cover: Cordis (Koishi's plugin framework since 2019) and +DeepSeek Harness (dsh, everything is a plugin, built on Cordis): reversible effects on a plugin fiber, inject-based +services, reload by unload-and-load; dsh's durable session log and its "interrupted" crash repair. Then what the +guarantees are in practice (the chapter's section of that name): disposers run concurrently and are only as good as +their authors, listener order follows registration order and changes on reload, services are untyped strings and a +provider reload restarts all dependents, and reloading a core service stops live agents. Then the two differences that +matter: what survives a reload (Cordis: nothing of the plugin's or its dependents' in-memory state; Chord: a pure +object's data and running work) and reach (one process vs. a tree of hosts with principals). Say what Chord costs: +more concepts and machinery. A flow or table comparing a reload in both helps. + +:::check +1. A Cordis plugin keeps a list of notes in a field of its instance. What happens to the list when the plugin hot-reloads, and what would the same plugin do on Chord v2? +2. Two Cordis plugins register waterfall listeners on the same tool-policy event; the first one short-circuits. What changes after the first plugin is hot-reloaded? +3. Why does an app like p2 need Chord's multi-host model, when dsh runs its plugin tree in one Host process? ++++ answers +1. The old fiber unloads and the new one starts empty, so the list is gone unless it was kept in a service or on disk. On Chord the notes are a document of a pure object; a reload swaps the object's code and keeps the document. +2. Its listener is registered again at the end of the list, so the other plugin's listener now runs first and the decision can change, unless the listener was registered with prepend. +3. p2 has a worker process per session and clients (including guests) in other browsers; objects must live on the right host, callers must be stamped principals, and reload has to roll out across server, workers and tabs. A single-process tree does not cover that. +::: + diff --git a/packages/chord/docs/next/flow-control/book.md b/packages/chord/docs/next/flow-control/book.md new file mode 100644 index 000000000..0485fe6a0 --- /dev/null +++ b/packages/chord/docs/next/flow-control/book.md @@ -0,0 +1,333 @@ +--- +title: Flow control for Chord links +cover: Slow / Links +kicker: A design note · Chord v2 slice 4 · 2026-10-22 +subtitle: How a phone on a bad train connection keeps seeing the newest state of a streamed reply, and why the transport's own backpressure cannot do that +stats: + - 9.4 s | how old the newest state was with transport backpressure only + - 0.0 s | with an acknowledged window of two messages + - 345 ms | another client's p95 during a call flood, without a call window + - ~200 | lines for the converged design +note: | + Written from the measurements and the cross-checked reports of the reviewers Astra and Fable (2026-10-22): Node 24.4.1, + `ws` 8.22, separate server and client processes, a throttling proxy at 64 KiB/s. The numbers are single synthetic + runs: they show failure modes, not production capacity. Relays: first decided as per-hop acknowledgements (b) on + 2026-10-23, then replaced by flow control per origin on 2026-10-24 after the slice 4a benchmarks (chapter VI). +colophon: Flow control for Chord links · fourth edition · 2026-10-24 · based on the reviewers' experiments and design §3, §22 +--- + +# Before we begin {.preface} + +Mario is on a train. His phone shows a p3 session in which the agent is writing a long reply. Every token is a small +state change: the worker commits it, the server relays it, the phone applies it. The train enters a tunnel and the +phone's link drops to a trickle, 64 KB per second. The worker keeps producing tokens at full speed. + +Without flow control, every token still gets sent. The bytes pile up somewhere between the worker and the phone, and +when the train leaves the tunnel the phone has to work through a backlog of old tokens before it sees the current text. +It is not only slow: what it shows is minutes old, and nothing tells it so. + +This note explains what keeps the phone current, why the obvious answer (let the transport push back) fails, and the +design Astra and Fable converged on after measuring it. + +## Six things to know + +:::findings +1. **Transport backpressure arrives too late.** The kernel's and a proxy's buffers take megabytes before a WebSocket + reports anything buffered; the phone sees state 4 to 10 seconds old. +2. **Acknowledgements keep state fresh.** The receiving application acknowledges each message; at most two are in + flight; the newest state was 0.0 seconds old on the same 64 KB/s link. +3. **Count messages, not bytes.** Each message carries everything that piled up since the last acknowledgement, so + throughput adapts to the link by itself. +4. **Merge ops first, snapshot last.** 2,000 merged token appends cost 26 KB, sent one by one 46 KB; a snapshot of a + member with 500 KB of history would cost 508 KB. +5. **Calls need their own window.** A tab firing 100,000 calls pushed another client's p95 to 345 ms; with a window of + 64 respected by the caller it stayed at 0.66 ms and the flood waited in the buggy tab. +6. **Nothing is refused for being too many** (decision K16): extra state is merged, extra calls wait. +::: + +## Words used throughout + +A **link** connects two hosts. The **provider** owns an object and sends its state; the **consumer** holds a +reference to it. A **hold** is one reference over a link, with its own stream of **ops** (Delta's changes). A +**snapshot** is a hold's whole state. An **acknowledgement** (`ack`) is a tiny message the consumer sends after it +applied a state message. **In flight** means sent and not yet acknowledged. **Coalescing** means merging the ops that +wait to be sent. A **relay** is a host in the middle (the server between worker and tab). + +# Where the Bytes Hide {#hidden lede="Why the sender cannot see a slow phone."} + +A WebSocket in Node reports `bufferedAmount`: bytes the library still holds. It looks like the perfect signal: stop +sending when it grows. The experiment says otherwise. + +:::sequence animate | A flood of state messages to a slow phone through a proxy. The sender's buffer stays small; the kernel's and the proxy's do not. +actor worker "worker" "commits tokens" accent +actor ws "server WebSocket" "bufferedAmount" +actor kernel "kernel + proxy buffers" muted +actor phone "phone" "64 KB/s" blue +worker -> ws: token 1 … token 9,000 +ws -> kernel: written at once (bufferedAmount stays near 0) +kernel -> phone: drains at 64 KB/s +note kernel: megabytes waiting +worker -> ws: token 9,001 (the newest) +ws -> kernel: queued behind everything +kernel --> phone: arrives seconds later +::: + +| Flood, ~9,800 × 128 B per second, 64 KB/s link | Sender memory | Newest state the phone saw | +| --- | --- | --- | +| no flow control | 7.0 MB queued in the WebSocket, heap +18.5 MB | 9.4 s old | +| cap `bufferedAmount` at 64 KiB, merge pending state | +0.46 MB | **still 9.4 s old** | +| cap at 1 MiB (Fable's rerun) | small | 3.8 s, growing to 10.4 s | +| at most 1 to 4 acknowledged messages in flight, merge pending state | +0.4 MB | **0.0 s** | + +The transport's backpressure protects the sender's memory, but freshness depends on buffers the sender cannot see: +the operating system's socket buffer, autotuned to hundreds of kilobytes, and every proxy on the way. A normal +streamed reply (50 tokens per second, 23 bytes each) never fills them; a burst or a tunnel does. Only the receiving +application knows what it has applied. + +# Acknowledged Messages {#acks lede="A window of two messages in flight, and why it adapts to any link."} + +The rule: a provider has at most **K = 2** state messages in flight per link. The consumer sends `ack` (three bytes in +the binary codec) after it applied one. While two are in flight, new commits do not produce messages; their ops wait, +merged per hold, and go out in the next message once an `ack` arrives. + +:::steps | Two messages in flight on a slow link ++++ Fast link +Commits 1, 2, 3 each become a message; each is acknowledged before the next commit. Nothing waits: the window never +fills, so a normal stream behaves exactly as without flow control. ++++ The link slows +Messages 1 and 2 are in flight. Commits 3 to 400 arrive; their ops are folded into the pending queue of each member +they touch (chapter III). Memory: the net change, not the history; at most 10,000 ops per member. ++++ An ack arrives +`ack` for message 1. The provider sends message 3, carrying the folded ops of commits 3 to 400, encoded once. +The phone applies them as one batch and shows the newest text. ++++ Self-clocking +The slower the link, the more each message carries; the faster, the smaller and more frequent. Nobody measures +bandwidth: the acknowledgements pace the sender. +::: + +:::raw +

+
+ + + +
+ +
+
+ +::: + +The simulation above shows the same effect in miniature. Drag the link below the commit rate: the upper lane's queue +grows without bound and the phone falls further behind every second; the lower lane never has more than two messages +waiting, and the phone is at most a message behind. + +# Fold the Backlog {#merge lede="What a waiting hold keeps while the window is full, and why it never measures sizes."} + +While messages are in flight, each hold member keeps the ops of new commits in a **pending queue** and **folds** them: +a new op merges into the last pending op on the same path when that is safe, so a backlog keeps the net change, not +the history. When an acknowledgement frees a slot, the queue is encoded once into one message. + +| Last pending op | New op | Kept | +| --- | --- | --- | +| `a text "Hel"` | `a text "lo"` | one `a text "Hello"` | +| `s count 41` or an append | `s count 42` | the `s`: it overwrites what came before | +| anything | `r value` | only the `r`: a replacement makes everything before it moot | +| anything on a path in the index | a structural op (`p`, `m`, `d`, `t`, `s` of an object) | appended; the index is cleared | + +The merge partner is found through a **leaf index**: a map from path to the last pending append or primitive set on +that path. Leaf ops on different paths commute, so a new op may merge into an earlier one, not only into its +neighbour; a status field changing between two tokens no longer stops the reply from folding. Any structural op +clears the index, because after a splice `entries[7]` may name another entry. Past 10,000 pending ops the queue keeps +only "replace with the current value", which then costs nothing while the consumer stays slow. + +**Why no size rule.** The first idea was to send a snapshot once the merged ops outgrow the member. The reviewers +measured it on real p2 data and it failed three ways: "snapshot bytes plus sent op bytes" is no bound (an interned path +lets a value grow faster than its ops: 5,404 B of snapshot after 832 B of ops), it drifts 1,254 times too high under +churn, and an exact size needs a size on every node of the value. Folding alone came within about 10% of an exact +oracle, with no measuring at all. + +| 60 s stall, all workloads | Uncompressed | Compressed | +| --- | --- | --- | +| merged ops, no folding | 466 KB | 166 KB | +| adjacent folding | 399 KB | 147 KB | +| leaf index (the decision) | 372 KB | 145 KB | +| exact oracle (not buildable) | 335 KB | 136 KB | + +**One queue for every slow consumer.** The same queue serves Chord's shared watch handle: a listener still busy with +the last frame gets the commits made meanwhile as one call with the newest value and the folded ops, instead of +today's "after 100 frames, replace everything". With a listener taking one second, a conversation view's watch went +from 402 calls and 17.9 MB of ops to 312 calls and 257 KB. Durable's event stream keeps a queue of its own, because +events are not state: folding would drop them. + +:::raw +

Animation of the fold: How folding works.

+::: + +# Calls Get Their Own Window {#calls lede="Why acknowledged state does not protect against a call flood."} + +Calls are small and the provider acknowledges nothing until they finish, so a byte or message window on state says +nothing about how many handlers run. A tab with a bug: + +```ts +for (const note of notes) ref.add(note, ctx); // 100,000 calls, none awaited +``` + +| 100,000 calls, handlers awaiting 10 ms | Finished in | Provider memory | Another client's p95 round trip | +| --- | --- | --- | --- | +| no limit | 0.66 s | +56.5 MB, 43,824 running | 129 ms (345 ms with trivial handlers) | +| provider-side semaphore of 64 | 15.9 s | +42.5 MB, 96,736 queued on the provider | 0.38 ms | +| caller-side window of 64 | 17.1 s | **+4 MB, nothing queued on the provider** | 0.66 ms | +| handlers doing 10 ms of CPU work | | | blocked about 1 s, with or without a limit | + +The provider's `hello` announces a call window **N** (256 by default) per origin (chapter VI). A consumer keeps at +most N of its own calls unanswered per link; further calls wait in the link's call queue, in order. Nothing is refused (K16); a peer that sends +more than N is a protocol error. The flood waits in the tab that caused it. CPU-heavy handlers block the event loop +whatever the window: that stays the handler author's responsibility. + +# Relays {#relays lede="One link to a worker carries many tabs. Decided: flow control per origin."} + +The server relays every tab's holds to a session's worker over one link. Three ways to apply the acknowledged window +across that hop were considered: + +:::flow wide animate | (a) End-to-end acknowledgements: the server acknowledges the worker only after the tab did. +row "Worker" accent +box w "worker" "K = 2 per link" +row "Server" muted +box s "server" "acks upstream after downstream" +row "Tabs" blue +box fast "fast tab" "desktop" +box slow "slow tab" "phone in a tunnel" dashed +arrow w -> s "state" +arrow s -> fast "state" +arrow s -> slow "state, slow" +arrow slow -> s "ack (late)" dashed +arrow s -> w "ack only when every tab acked" accent +note "Simplest, but the slow phone holds the worker's one window, so the fast tab's updates arrive in coarser, later batches too." +::: + +:::flow wide animate | (b) Acknowledgements per hop: the server acknowledges at once and resets a slow tab. +row "Worker" accent +box w2 "worker" "K = 2 per link" +row "Server" muted +box s2 "server" "acks at once; window per tab" +row "Tabs" blue +box fast2 "fast tab" "desktop" +box slow2 "slow tab" "phone in a tunnel" dashed +arrow w2 -> s2 "state" +arrow s2 -> w2 "ack at once" accent +arrow s2 -> fast2 "state, current" +arrow s2 -> slow2 "window full: its parts dropped" dashed +arrow s2 -> w2 "reset: fresh snapshots" accent +note "The server cannot fold opaque parts, so it drops them and asks for snapshots. Built and measured in slice 4a: a 1 s stall cost three full snapshots (~1.5 MiB) and ~24 s to catch up under streaming." +::: + +:::flow wide animate | (c) Flow control per origin (decided 2026-10-24): the worker keeps a window per tab; the relay only forwards. +row "Worker" accent +box w3 "worker" "K = 2 per (link, origin)" +row "Server" muted +box s3 "server" "forwards; FIFO of senders" +row "Tabs" blue +box fast3 "fast tab" "origin 1" +box slow3 "slow tab" "origin 2, in a tunnel" dashed +arrow w3 -> s3 "state @1, state @2" +arrow s3 -> fast3 "state, current" +arrow s3 -> slow3 "state, slow" dashed +arrow slow3 -> s3 "ack (late)" dashed +arrow s3 -> w3 "ack @2 when the tab acked" accent +note "The relay numbers each tab (an origin) and forwards at once; each tab's ack goes back to the worker that sent the message. The worker folds the slow tab's commits in that origin's pending queues, exactly as for a direct tab; the fast tab is another origin with its own window." +::: + +How (c) works: + +- **Origins.** The relay gives each downstream link a number, never reused; every message upstream carries it + (`origin`), and the worker keys holds, calls, windows and acks by (link, origin, id). Ids are not renumbered. +- **What the relay keeps.** Per relayed hold its upstream link and its original `hold` message (to route calls and + hold again after a worker restart); per tab a FIFO of which worker sent each message in flight, so each in-order + `ack` goes back to the right worker. Nothing is parsed, folded, dropped or reset. +- **Calls.** N is per origin at the worker; the relay forwards calls at once, each tab bounded by its own window. +- **A tab leaves.** Its link closes; the relay sends `closed { origin }` to each worker it relayed to, which releases + that origin's holds and aborts its calls. +- **Cost.** A tab served by W workers may have 2W messages in flight (plus the relay's own 2). Two bytes of origin per + message on the worker link. + +# What Changes in the Design {.epilogue lede="The converged scheme, what it replaces, and the relay decision."} + +**Converged** (Astra and Fable, after cross-checking each other's experiments): at most K = 2 acknowledged state +messages in flight; pending ops folded per hold member through a leaf index, capped at 10,000; a call window N +announced in `hello`, extra calls queued by the caller; holds unlimited; size limits stay a separate check. + +**Replaced** from the design (§3, §22): the byte window and its credit messages, snapshot admission and debt (R19, Q3), +cut C12's call-credit accounting, any adapter backlog API. IPC and WebSocket behave the same, because acknowledgements +are messages. + +**Relays (owner):** per-hop acknowledgements (b) were decided on 2026-10-23 and built in slice 4a; the benchmarks +showed a slow tab recovering through resets far worse than a direct tab (24 s and three snapshots after a 1 s stall). +On 2026-10-24 they were replaced by **flow control per origin** (c): the worker keeps window, pending queues and calls +per (link, origin), the relay only forwards and returns acks. Measured after the rework: a slow tab through the relay +catches up exactly like a direct one. The circuit breaker stayed undecided; the heartbeat (15 s ping, 30 s close) +was built instead. + +# Measurements {.appendix} + +| Experiment | Setup | Result | +| --- | --- | --- | +| normal stream | 50 × 23 B/s through 64 KiB/s | no buffering, frames ≤ 2 ms old | +| flood, no control | ~9,800 × 128 B/s, 10 s | 7.0 MB in `bufferedAmount`, heap +18.5 MB, newest state 9.4 s old | +| flood, `bufferedAmount` ≤ 64 KiB + merging | same | heap +0.46 MB, newest state 9.4 s old | +| flood, 1 to 4 acked messages + merging | same | heap +0.4 MB, newest state 0.0 s old, 98 % fewer messages | +| client stops reading (loopback) | 20k × 90 B/s, 15 s | 23.4 MiB buffered, RSS +121 MB; merging above 1 MiB: flat | +| IPC to a busy child | 623k sends | `send()` false from ~64 KB, parent +1,035 MB; the callback acknowledges the write, not the handler | +| call flood | 100,000 calls, see chapter IV | caller-side window: +4 MB, other client p95 0.66 ms | +| slow tab through a relay, (b) resets | 64 KiB/s, 100 tokens/s into a 500 KB view, 1 s stall | ~24 s to catch up, three full snapshots (~1.5 MiB) | +| slow tab through a relay, (c) per origin | same, 1 s / 10 s / 60 s stall | caught up after 0.02 / 0.08 / 0.38 s, 485 B / 4.1 KiB / 23.7 KiB (same as direct) | diff --git a/packages/chord/docs/next/flow-control/fold/index.html b/packages/chord/docs/next/flow-control/fold/index.html new file mode 100644 index 000000000..3cb573bb5 --- /dev/null +++ b/packages/chord/docs/next/flow-control/fold/index.html @@ -0,0 +1,265 @@ + + + + + +How folding works + + + +
+

How folding works

+

What a hold does with commits while its link's window is full (two messages unacknowledged), and why folding needs no size measuring.

+ +

The rule

+

While the window is full, a hold keeps the ops of each new commit raw in a pending list. Before appending an op, it looks at the last pending op only:

+ + + + + +
last pendingnew opresult
a path "Hel"a path "lo"one a path "Hello" (text joined)
s path 41 or a path …s path 42one s path 42 (the set overwrites what came before)
anything on another pathanythingappended as it is
+

Applied in order, the folded list gives the same value as the unfolded one: only neighbours on the same path merge, so no op in between can see the difference. Cost: one path comparison per op. When an ack frees a slot, the pending list is encoded once into one message.

+
a = append texts = set valuep = insert into listsnapshot
+ +
+
+ + + + +
+
+ + + + +
+
+
link
window full: 2 unacked (tunnel)
+
commits
+
pending (raw)
+
sent
+
+
+ commits 0 + pending ops 0 + merged or dropped 0 + encodes 0 +
+
+
+ +

What the four cases show

+ + + + + + +
casepending after the stallwhy
streamed reply1 opevery token appends to the same path
counter, 100,000 sets1 opeach set overwrites the previous one
reply + status, interleavedgrowsthe neighbour is always on the other path: adjacent-only folding cannot merge
new list itemsgrowsinserts carry new data: nothing to fold, this is the real change
+

The last two cases are why a memory bound is still needed. It can be a plain count of pending ops after folding (say 1,000): past it the hold drops its list and marks “send the current value”, which then costs nothing more while the stall lasts.

+ +

Why not time since the last ack?

+

A time trigger (“no ack for T seconds: drop the list, send a snapshot later”) punishes the most common case: a phone in a tunnel while a reply streams into a long conversation. Folded, the 60 s stall costs one append of about 12 KB; a snapshot of the conversation costs its whole size (Astra's corpus: 4.6 KB of updates against 168 KB, recovery 0.2 s against 2.7 s). And before T, a fast producer on many paths still piles up rate × T ops: time bounds duration, not memory. With folding, the counter case (100,000 sets) is one op whatever the time, so time adds nothing there.

+
+ + + + diff --git a/packages/chord/docs/next/flow-control/index.html b/packages/chord/docs/next/flow-control/index.html new file mode 100644 index 000000000..a34b39a24 --- /dev/null +++ b/packages/chord/docs/next/flow-control/index.html @@ -0,0 +1,548 @@ + + +Flow control for Chord links + + + + +
+
+
A design note · Chord v2 slice 4 · 2026-10-22
+

Slow
Links

+

How a phone on a bad train connection keeps seeing the newest state of a streamed reply, and why the transport's own backpressure cannot do that

+
9.4 show old the newest state was with transport backpressure only
0.0 swith an acknowledged window of two messages
345 msanother client's p95 during a call flood, without a call window
~200lines for the converged design
+

Written from the measurements and the cross-checked reports of the reviewers Astra and Fable (2026-10-22): Node 24.4.1, +ws 8.22, separate server and client processes, a throttling proxy at 64 KiB/s. The numbers are single synthetic +runs: they show failure modes, not production capacity. Relays: first decided as per-hop acknowledgements (b) on +2026-10-23, then replaced by flow control per origin on 2026-10-24 after the slice 4a benchmarks (chapter VI).

+
+
+ + +
+

Before we begin

+

Mario is on a train. His phone shows a p3 session in which the agent is writing a long reply. Every token is a small +state change: the worker commits it, the server relays it, the phone applies it. The train enters a tunnel and the +phone's link drops to a trickle, 64 KB per second. The worker keeps producing tokens at full speed.

+

Without flow control, every token still gets sent. The bytes pile up somewhere between the worker and the phone, and +when the train leaves the tunnel the phone has to work through a backlog of old tokens before it sees the current text. +It is not only slow: what it shows is minutes old, and nothing tells it so.

+

This note explains what keeps the phone current, why the obvious answer (let the transport push back) fails, and the +design Astra and Fable converged on after measuring it.

+

Six things to know

+
    +
  1. Transport backpressure arrives too late. The kernel's and a proxy's buffers take megabytes before a WebSocket +reports anything buffered; the phone sees state 4 to 10 seconds old.
  2. +
  3. Acknowledgements keep state fresh. The receiving application acknowledges each message; at most two are in +flight; the newest state was 0.0 seconds old on the same 64 KB/s link.
  4. +
  5. Count messages, not bytes. Each message carries everything that piled up since the last acknowledgement, so +throughput adapts to the link by itself.
  6. +
  7. Merge ops first, snapshot last. 2,000 merged token appends cost 26 KB, sent one by one 46 KB; a snapshot of a +member with 500 KB of history would cost 508 KB.
  8. +
  9. Calls need their own window. A tab firing 100,000 calls pushed another client's p95 to 345 ms; with a window of +64 respected by the caller it stayed at 0.66 ms and the flood waited in the buggy tab.
  10. +
  11. Nothing is refused for being too many (decision K16): extra state is merged, extra calls wait.
  12. +
+

Words used throughout

+

A link connects two hosts. The provider owns an object and sends its state; the consumer holds a +reference to it. A hold is one reference over a link, with its own stream of ops (Delta's changes). A +snapshot is a hold's whole state. An acknowledgement (ack) is a tiny message the consumer sends after it +applied a state message. In flight means sent and not yet acknowledged. Coalescing means merging the ops that +wait to be sent. A relay is a host in the middle (the server between worker and tab).

+
+ +
I

Where the Bytes Hide

Why the sender cannot see a slow phone.

+

A WebSocket in Node reports bufferedAmount: bytes the library still holds. It looks like the perfect signal: stop +sending when it grows. The experiment says otherwise.

+
workercommits tokensserver WebSocketbufferedAmountkernel + proxy buffersphone64 KB/stoken 1 … token 9,000written at once (bufferedAmount stays near 0)drains at 64 KB/smegabytes waitingtoken 9,001 (the newest)queued behind everythingarrives seconds later
Figure 1. A flood of state messages to a slow phone through a proxy. The sender's buffer stays small; the kernel's and the proxy's do not.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Flood, ~9,800 × 128 B per second, 64 KB/s linkSender memoryNewest state the phone saw
no flow control7.0 MB queued in the WebSocket, heap +18.5 MB9.4 s old
cap bufferedAmount at 64 KiB, merge pending state+0.46 MBstill 9.4 s old
cap at 1 MiB (Fable's rerun)small3.8 s, growing to 10.4 s
at most 1 to 4 acknowledged messages in flight, merge pending state+0.4 MB0.0 s
+

The transport's backpressure protects the sender's memory, but freshness depends on buffers the sender cannot see: +the operating system's socket buffer, autotuned to hundreds of kilobytes, and every proxy on the way. A normal +streamed reply (50 tokens per second, 23 bytes each) never fills them; a burst or a tunnel does. Only the receiving +application knows what it has applied.

+
+ +
II

Acknowledged Messages

A window of two messages in flight, and why it adapts to any link.

+

The rule: a provider has at most K = 2 state messages in flight per link. The consumer sends ack (three bytes in +the binary codec) after it applied one. While two are in flight, new commits do not produce messages; their ops wait, +merged per hold, and go out in the next message once an ack arrives.

+
Two messages in flight on a slow link

Fast link

Commits 1, 2, 3 each become a message; each is acknowledged before the next commit. Nothing waits: the window never +fills, so a normal stream behaves exactly as without flow control.

+

The link slows

Messages 1 and 2 are in flight. Commits 3 to 400 arrive; their ops are folded into the pending queue of each member +they touch (chapter III). Memory: the net change, not the history; at most 10,000 ops per member.

+

An ack arrives

ack for message 1. The provider sends message 3, carrying the folded ops of commits 3 to 400, encoded once. +The phone applies them as one batch and shows the newest text.

+

Self-clocking

The slower the link, the more each message carries; the faster, the smaller and more frequent. Nobody measures +bandwidth: the acknowledgements pace the sender.

+
+
+
+ + + +
+ +
+
+ +

The simulation above shows the same effect in miniature. Drag the link below the commit rate: the upper lane's queue +grows without bound and the phone falls further behind every second; the lower lane never has more than two messages +waiting, and the phone is at most a message behind.

+
+ +
III

Fold the Backlog

What a waiting hold keeps while the window is full, and why it never measures sizes.

+

While messages are in flight, each hold member keeps the ops of new commits in a pending queue and folds them: +a new op merges into the last pending op on the same path when that is safe, so a backlog keeps the net change, not +the history. When an acknowledgement frees a slot, the queue is encoded once into one message.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Last pending opNew opKept
a text "Hel"a text "lo"one a text "Hello"
s count 41 or an appends count 42the s: it overwrites what came before
anythingr valueonly the r: a replacement makes everything before it moot
anything on a path in the indexa structural op (p, m, d, t, s of an object)appended; the index is cleared
+

The merge partner is found through a leaf index: a map from path to the last pending append or primitive set on +that path. Leaf ops on different paths commute, so a new op may merge into an earlier one, not only into its +neighbour; a status field changing between two tokens no longer stops the reply from folding. Any structural op +clears the index, because after a splice entries[7] may name another entry. Past 10,000 pending ops the queue keeps +only "replace with the current value", which then costs nothing while the consumer stays slow.

+

Why no size rule. The first idea was to send a snapshot once the merged ops outgrow the member. The reviewers +measured it on real p2 data and it failed three ways: "snapshot bytes plus sent op bytes" is no bound (an interned path +lets a value grow faster than its ops: 5,404 B of snapshot after 832 B of ops), it drifts 1,254 times too high under +churn, and an exact size needs a size on every node of the value. Folding alone came within about 10% of an exact +oracle, with no measuring at all.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
60 s stall, all workloadsUncompressedCompressed
merged ops, no folding466 KB166 KB
adjacent folding399 KB147 KB
leaf index (the decision)372 KB145 KB
exact oracle (not buildable)335 KB136 KB
+

One queue for every slow consumer. The same queue serves Chord's shared watch handle: a listener still busy with +the last frame gets the commits made meanwhile as one call with the newest value and the folded ops, instead of +today's "after 100 frames, replace everything". With a listener taking one second, a conversation view's watch went +from 402 calls and 17.9 MB of ops to 312 calls and 257 KB. Durable's event stream keeps a queue of its own, because +events are not state: folding would drop them.

+

Animation of the fold: How folding works.

+
+ +
IV

Calls Get Their Own Window

Why acknowledged state does not protect against a call flood.

+

Calls are small and the provider acknowledges nothing until they finish, so a byte or message window on state says +nothing about how many handlers run. A tab with a bug:

+
for (const note of notes) ref.add(note, ctx);   // 100,000 calls, none awaited
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
100,000 calls, handlers awaiting 10 msFinished inProvider memoryAnother client's p95 round trip
no limit0.66 s+56.5 MB, 43,824 running129 ms (345 ms with trivial handlers)
provider-side semaphore of 6415.9 s+42.5 MB, 96,736 queued on the provider0.38 ms
caller-side window of 6417.1 s+4 MB, nothing queued on the provider0.66 ms
handlers doing 10 ms of CPU workblocked about 1 s, with or without a limit
+

The provider's hello announces a call window N (256 by default) per origin (chapter VI). A consumer keeps at +most N of its own calls unanswered per link; further calls wait in the link's call queue, in order. Nothing is refused (K16); a peer that sends +more than N is a protocol error. The flood waits in the tab that caused it. CPU-heavy handlers block the event loop +whatever the window: that stays the handler author's responsibility.

+
+ +
V

Relays

One link to a worker carries many tabs. Decided: flow control per origin.

+

The server relays every tab's holds to a session's worker over one link. Three ways to apply the acknowledged window +across that hop were considered:

+
WORKERSERVERTABSworkerK = 2 per linkserveracks upstream after downstreamfast tabdesktopslow tabphone in a tunnelstatestatestate, slowack (late)ack only when every tab ackedSimplest, but the slow phone holds the worker's one window, so the fast tab's updates arrive in coarser, later batchestoo.
Figure 2. (a) End-to-end acknowledgements: the server acknowledges the worker only after the tab did.
+
WORKERSERVERTABSworkerK = 2 per linkserveracks at once; window per tabfast tabdesktopslow tabphone in a tunnelstateack at oncestate, currentwindow full: its parts droppedreset: fresh snapshotsThe server cannot fold opaque parts, so it drops them and asks for snapshots. Built and measured in slice 4a: a 1 sstall cost three full snapshots (~1.5 MiB) and ~24 s to catch up under streaming.
Figure 3. (b) Acknowledgements per hop: the server acknowledges at once and resets a slow tab.
+
WORKERSERVERTABSworkerK = 2 per (link, origin)serverforwards; FIFO of sendersfast taborigin 1slow taborigin 2, in a tunnelstate @1, state @2state, currentstate, slowack (late)ack @2 when the tab ackedThe relay numbers each tab (an origin) and forwards at once; each tab's ack goes back to the worker that sent themessage. The worker folds the slow tab's commits in that origin's pending queues, exactly as for a direct tab; thefast tab is another origin with its own window.
Figure 4. (c) Flow control per origin (decided 2026-10-24): the worker keeps a window per tab; the relay only forwards.
+

How (c) works:

+
    +
  • Origins. The relay gives each downstream link a number, never reused; every message upstream carries it +(origin), and the worker keys holds, calls, windows and acks by (link, origin, id). Ids are not renumbered.
  • +
  • What the relay keeps. Per relayed hold its upstream link and its original hold message (to route calls and +hold again after a worker restart); per tab a FIFO of which worker sent each message in flight, so each in-order +ack goes back to the right worker. Nothing is parsed, folded, dropped or reset.
  • +
  • Calls. N is per origin at the worker; the relay forwards calls at once, each tab bounded by its own window.
  • +
  • A tab leaves. Its link closes; the relay sends closed { origin } to each worker it relayed to, which releases +that origin's holds and aborts its calls.
  • +
  • Cost. A tab served by W workers may have 2W messages in flight (plus the relay's own 2). Two bytes of origin per +message on the worker link.
  • +
+
+ +
∴

What Changes in the Design

The converged scheme, what it replaces, and the relay decision.

+

Converged (Astra and Fable, after cross-checking each other's experiments): at most K = 2 acknowledged state +messages in flight; pending ops folded per hold member through a leaf index, capped at 10,000; a call window N +announced in hello, extra calls queued by the caller; holds unlimited; size limits stay a separate check.

+

Replaced from the design (§3, §22): the byte window and its credit messages, snapshot admission and debt (R19, Q3), +cut C12's call-credit accounting, any adapter backlog API. IPC and WebSocket behave the same, because acknowledgements +are messages.

+

Relays (owner): per-hop acknowledgements (b) were decided on 2026-10-23 and built in slice 4a; the benchmarks +showed a slow tab recovering through resets far worse than a direct tab (24 s and three snapshots after a 1 s stall). +On 2026-10-24 they were replaced by flow control per origin (c): the worker keeps window, pending queues and calls +per (link, origin), the relay only forwards and returns acks. Measured after the rework: a slow tab through the relay +catches up exactly like a direct one. The circuit breaker stayed undecided; the heartbeat (15 s ping, 30 s close) +was built instead.

+
+ +
A

Measurements

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ExperimentSetupResult
normal stream50 × 23 B/s through 64 KiB/sno buffering, frames ≤ 2 ms old
flood, no control~9,800 × 128 B/s, 10 s7.0 MB in bufferedAmount, heap +18.5 MB, newest state 9.4 s old
flood, bufferedAmount ≤ 64 KiB + mergingsameheap +0.46 MB, newest state 9.4 s old
flood, 1 to 4 acked messages + mergingsameheap +0.4 MB, newest state 0.0 s old, 98 % fewer messages
client stops reading (loopback)20k × 90 B/s, 15 s23.4 MiB buffered, RSS +121 MB; merging above 1 MiB: flat
IPC to a busy child623k sendssend() false from ~64 KB, parent +1,035 MB; the callback acknowledges the write, not the handler
call flood100,000 calls, see chapter IVcaller-side window: +4 MB, other client p95 0.66 ms
slow tab through a relay, (b) resets64 KiB/s, 100 tokens/s into a 500 KB view, 1 s stall~24 s to catch up, three full snapshots (~1.5 MiB)
slow tab through a relay, (c) per originsame, 1 s / 10 s / 60 s stallcaught up after 0.02 / 0.08 / 0.38 s, 485 B / 4.1 KiB / 23.7 KiB (same as direct)
+
+ +

Flow control for Chord links · fourth edition · 2026-10-24 · based on the reviewers' experiments and design §3, §22

+
+ + diff --git a/packages/chord/docs/next/flow-control/window/index.html b/packages/chord/docs/next/flow-control/window/index.html new file mode 100644 index 000000000..52ac8c39c --- /dev/null +++ b/packages/chord/docs/next/flow-control/window/index.html @@ -0,0 +1,228 @@ + + + + + +A windowed conversation view + + + +
+

A windowed conversation view

+

Proposal for Durable: the view holds a window of the newest entries with a stable start; older entries are fetched in pages; a reconnect resumes after the last entry it saw.

+ +
type ConversationView = {
+  conversation: ConversationRecord;
+  head?: EntryRecord;               // the current compaction summary, if any
+  entries: readonly EntryRecord[];  // the window: active entries with id >= from
+  earlier: boolean;                 // older active entries exist before it
+  docs: { "pi.live", "pi.agent", "pi.inbox", "pi.provider", "pi.usage" };
+};
+conversation.watch(context, { from?: EntryId, tail?: number });          // default: the last `tail`
+conversation.entries({ before, order: "descending" }, limit, cursor, ctx); // exists today
+ +

Watch it work

+

Shown with tail = 4 (the proposal: about 50). The window grows by appends until it holds twice tail; then the oldest half leaves in one splice.

+
+
+ + + + + +
+
+
store: every active entry
+
client: view window (blue), kept or fetched (purple)
+
+
+ window 0 entries + ops sent 0 + last snapshot - +
+
+
+
+ +

What each step costs

+ + + + + + + + +
stepwire
a new entryone append splice: p entries <len> 0 [entry]
window reaches 2 × tailone splice removing the oldest half: p entries 0 <tail> [], once every tail entries
streaming into the newest messageappends on docs["pi.live"].generation.message, unchanged
hold, re-hold, overflowa snapshot of at most 2 × tail entries plus the live documents, never the whole transcript
scroll upa page from entries(), outside the view: the client keeps it, the view never grows
reconnect with from: lastSeen + 1only the entries it missed plus the documents (issue #11327)
+

Why a stable start instead of “always the last 50”: a sliding window would remove one entry at the front for every entry appended at the back, two ops per entry, and every index in the window would shift. With a stable start, entries only append, and the front moves in one splice now and then.

+
+ + + + diff --git a/packages/chord/docs/next/layer-1.md b/packages/chord/docs/next/layer-1.md new file mode 100644 index 000000000..505fe7294 --- /dev/null +++ b/packages/chord/docs/next/layer-1.md @@ -0,0 +1,311 @@ +# L1: data and contracts (spec) + +Status: **approved (owner, 2026-10-22)**, rewritten 2026-10-22 as the single source for a from-scratch implementation +(the first implementation was discarded for code quality). Every rule here is decided; the decision log +(`packages/chord/docs/next/decisions.md`) has the reasons. Code style: the section "Chord v2 Code Style" of the root +`AGENTS.md`. + +## What L1 is + +Everything about **data** that the upper layers need and that does not depend on hosts, objects or links: how an +object type is declared and evolves (contracts), what a value is, how params become an identity, how arguments are +checked, what a principal key is, what an error is. Pure functions, types and weak caches; no I/O. + +Source: `packages/chord/src/next/contracts/` (declarations) and `packages/chord/src/next/data/` (per-call checks, +principals, errors), split into files by concept. Everything else under `src/next/` is the frozen old kernel: never +imported, never edited. Reused: `src/context/` (`Context`, `BACKGROUND_CONTEXT`, `context.telemetry`) and +`src/json.ts` (`JsonValue`), both unchanged. Schemas are TypeBox 1.3 (`typebox` in `node_modules`). + +## 1. Values + +```ts +type Value = null | boolean | number | string | Uint8Array | Value[] | { [key: string]: Value }; +``` + +- JSON plus byte leaves. No `undefined` inside values, no non-finite numbers, no foreign prototypes, no accessors. +- Values from links are already strict JSON plus bytes; values from same-host code are plain data **by convention**, + immutable once handed over. L1 never walks, copies or freezes a value to enforce that. State is never checked. +- `canonicalJson(value: JsonValue, context)`: sorted object keys, `-0` written as `0`. Used only for principal and + revocation keys, which are JSON (no bytes), so it lives with principals. + +## 2. Contracts: declarations + +A contract is the declared type of an object: name, kind, version, params, members. Two kinds: + +```ts +const Notes = remote("notes", { + version: 1, + params: { session: str() }, + variants: { + editor: { state: { notes: arr(Note) }, methods: { add: method(obj({ text: str() }), str()), clear: method() } }, + reader: { state: { notes: arr(PublicNote) } }, + }, +}); +const Info = local("app.session-info", { + version: 1, + params: {}, + members: { init: obj({ title: str() }), clear: method() }, // a schema is state, method(...) is a method +}); +``` + +- `remote` has variants (per-principal views, at least one); `local` has one flat `members` map and no variants. +- `method()`, `method(args)`, `method(args, result)`, `method(undefined, result)`. An omitted side is `null` on the + wire: a no-argument method is called with `null`. +- **Variants (D11):** a member name has the same kind (state or method) in every variant that has it; a method has the + same signature (args and result) in every variant; state schemas may differ per variant (redaction). Enforced in the + types of `remote(...)` and at declaration (for untyped callers). +- **Closed argument objects (Q12):** every object schema inside a method's arguments sets + `additionalProperties: false`, nested ones included, raw `Type.Object` included; record schemas (`map`) are allowed. + Checked once per declared or edited method, at declaration. `obj()` always produces closed objects and cannot be + opened. +- Declaration checks (the types cannot express them for JavaScript callers): nonempty name, version a positive safe + integer, no unknown declaration keys, params policy (section 4), D11, closed arguments, the schema vocabulary + (below). Each fails with an `Error` naming the path (`notes.editor.methods.add.args`). +- Declarations are immutable by convention: a contract holds Chord-owned snapshots of its schemas (copied once at + declaration, so later changes to the app's schema objects cannot desync validation from the fingerprint), never the + app's schema objects. Nothing in L1 is frozen (owner, 2026-10-22). Contracts are branded: hand-built or spread copies are refused by `evolve`, `checkAccepts` + and `bindProvider` (a private `WeakMap` from contract to its data). +- **Placement** is not part of L1 contracts: the `placement` field arrives with host kinds (L7) and is never part of + the fingerprint. + +### Schema helpers + +`str({ min, max, pattern })`, `num({ min, max })`, `int({ min, max })`, `bool()`, `nul()`, `lit(v)`, `oneOf(...literals)`, +`union([...])`, `obj(props)`, `arr(items, { min, max })`, `map(value)` (string-keyed record), `opt(schema)`, +`bytes({ min, max })`, `unknown()`, and `type Static`. They return ordinary TypeBox schemas, so `Static` and TypeBox's +compiler work unchanged. `min`/`max` mean string length, numeric bounds or array length. + +`bytes()` is a `Uint8Array` leaf with optional length bounds, checked by `instanceof` and length, never enumerated. +Its enumerable form is the data marker `{ type: "bytes", minLength?, maxLength? }`; the TypeBox check is attached +non-enumerably and rebuilt from the marker whenever a schema is snapshotted, so no callback is ever copied from app +data. + +**Vocabulary (rejected at declaration otherwise):** data primitives, arrays and tuples, objects and records, unions and +intersections, `$defs`/`$ref` (use `$ref` for recursion), constraints (`pattern`, `format`, bounds, `multipleOf`, +`uniqueItems`, property counts), `default`, `readOnly`/`writeOnly`, annotations, bytes. Rejected: conditional and +dependent schemas, `not`, `contains`, `propertyNames`, `unevaluated*`, `contentSchema`, `definitions`, unknown +keywords, TypeBox's executable features (`Unsafe` other than bytes, refinements, transforms), accessors, symbol keys, +cyclic objects, non-JSON data. + +### Fingerprint + +Every version gets a fingerprint at declaration: `hash64` (a portable 64-bit hash, two FNV-style lanes, 16 hex +characters; not a security boundary) of `JSON.stringify({ normalFormVersion, value })` over a **normal form**: + +- object keys sorted; `required` and `enum` sorted; union, intersection and tuple order **kept** (it is meaning); +- annotations dropped on schema nodes only (`title`, `description`, `examples`, `$comment`, `deprecated`); defaults, + formats, patterns, bounds and `additionalProperties` kept; +- params as ordered `[fieldName, schema]` pairs (declaration order is identity order); members as sorted dictionaries; +- `{ kind: "remote", params, variants }` or `{ kind: "local", params, members }`; placement excluded. + +`NORMAL_FORM_VERSION = 2` is exported; changing the normal form changes it and is a link-protocol change. +One schema-aware walk produces both the snapshot and the normal form. + +### Evolution (Q10) + +A history starts with a base written in full (`remote()`/`local()` with an explicit version). `evolve(prev, (e) => +e...)` appends the next version (`prev.version + 1`) with structured edits: + +| Edit | Meaning | +| --- | --- | +| `state(variant, name, schema)` | change one state member (local: `state(name, schema)`) | +| `addState(variant, name, schema)` | add a state member | +| `removeState(variant, name)` | remove a state member | +| `args(name, schema)`, `result(name, schema)` | change one side of a method in every variant that has it | +| `method(name, args, result)` | replace both sides | +| `addMethod(variant, name, args, result)` | add a method (local: no variant); to another variant only with the same signature | +| `removeMethod(name)` | remove a method from every variant | + +- Params and the variant set never change by evolution. A change to either is a **new history base**: a new full + declaration with an explicit version and no accepted older versions. +- A member's kind never changes, also not after a removal and re-add. Checks run after every edit. +- `evolve` must return the builder it was given (no foreign builders). Every version is materialized and + fingerprinted at declaration, so an old plugin's copy of v1 has the same fingerprint as the history's v1. +- The types track the whole history: `Contract` carries the version, the current description and every step with its + edits, so `checkAccepts` can require exactly the right mappings at compile time. + +### Accepted versions and mappings + +A provider may accept older versions of its contract with mappings it owns: + +```ts +const NotesV2 = evolve(NotesV1, (e) => e.state("editor", "notes", arr(NoteV2)).args("add", NoteV2)); +const accepts = { + 1: { + state: { editor: { notes: (current) => current.map(toV1) } }, // current member -> old member + methods: { add: { args: (old) => toV2(old) } }, // old args -> current args + }, +}; +checkAccepts(NotesV2, accepts); // { versions: [{ version: 1, fingerprint }] } +``` + +`checkAccepts(contract, accepts)` is pure: it validates the table without calling any mapping and returns the accepted +versions with their fingerprints. Mappings are **direct** (old to current, never chained). For accepted version `v`, +considering every edit after `v`, a mapping is required exactly for: + +- a changed state member: `(currentMember) => oldMember`; +- a removed state member: `(currentVariantState) => oldMember` (the whole current state of that variant), or + `"refuse"`; +- a method whose args changed: `{ args: (old) => current }`; whose result changed: `{ result: (current) => old }`; + both: both functions; +- a removed method (from any variant that had it in `v`): `{ call: async (oldArgs, context) => oldResult }` or + `"refuse"`; `call` and `"refuse"` are also allowed for changed methods. + +Added members need nothing. No-op edits and changes made and reverted still require (identity) mappings: the check is +by edit, not by schema comparison. Unknown versions, the current version, missing mappings, unnecessary keys and wrong +shapes fail at declaration with the path (`notes accepts[1].methods.add`). An absent `state` or `methods` section means +empty and is allowed only when nothing is required there. The same rules hold at the type level (`AcceptsTable`). + +`Implementation` derives the provider's shape: per variant, state getters `() => value`; methods +`(args, context) => Promise`. + +## 3. Principals + +- The app's principal type `P` is plain JSON with a `kind` field: no schema, no version, never persisted by Chord + (objects persist their own projection, e.g. chat's `Author`). +- `principalKey(principal, context, customKey?)`: default `canonicalJson(principal)`; an app may pass its own key + function, which must be the same on every host. Cached per principal object in `WeakMap`s; a custom key function's + cache is keyed by the function too, so a dropped function is never retained by a live principal. +- `revocationKeys(principal, revokedBy, context)`: the canonical JSON of each JSON value `revokedBy(principal)` + returns (`host.revoke` is L7). + +## 4. Params and identity + +```ts +const { params, key } = checkParams(Notes, { session: "S1" }, context); // throws invalid/params +const identity = identityKey(Notes, key, principalKey(principal)); // "notes|S1|" +``` + +- Declaration policy: params are a flat record of required fields, each `str()`, `int()`, `bool()` or a union of + string literals (`oneOf("chat", "agent")`), constraints allowed; no defaults, optional, nested, bytes, numbers other + than integers. Checked in the types and at declaration. +- `checkParams(contract, raw, context)`: one compiled closed-object check of `raw` (constraints enforced), the compiled + validator cached per contract in a `WeakMap`. Then the key is the field values in declared order, each escaped + (`\` and `|`), joined with `|`. It returns the caller's own object: no pre-check walk, no copy, no freeze. +- `identityKey(contract, key, principalKey)`: `name|key|principalKey`, each part escaped; `local` takes no principal + key (by type). The key is an in-process map key; links carry the params themselves. + +## 5. Arguments + +```ts +const provider = bindProvider(NotesV2, accepts); // once per registration and reload +const checked = checkArgs(provider, "add", raw, boundVersion, ctx); // invalid/args with path +// { kind: "method"; args (current shape); result?: (current) => old } | { kind: "call"; args (old shape); call } +checkArgs(provider, "clear", null, 2, ctx); // no-argument methods take null +``` + +- `bindProvider(contract, accepts)` runs `checkAccepts` once and keeps the app's accepts table as it is (immutable by + convention; no copy, no freeze). It builds a private dispatch table, held in a `WeakMap` keyed by the returned + provider: accepted version -> method -> `{ schema, validator, mapping }`. Validators compile on first use per + (version, method); a weak cache keyed by schema object lets equal schema objects share a compiled validator. +- `checkArgs(provider, method, raw, boundVersion, context)`: three map lookups, one compiled check against the + provider's schema for that version, then at most one direct `args` mapping old -> current. It returns a tagged + value (owner, 2026-10-22): `{ kind: "method"; args; result? }` (args in the current shape; `result` maps the current + result back for an old-version caller) or `{ kind: "call"; args; call }` (a replaced method: the checked old args + and the provider's `call`, which L4 invokes). Current-version calls are never mapped. Unknown method: + `invalid/member`. Version not accepted, or `"refuse"`: `contract_mismatch`. Mapping output is trusted. +- Every caller is checked, `local` included (Q15). Unknown fields fail (closed schemas); nothing is stripped, copied, + frozen or defaulted. Results and state are never checked. +- Memory: a binding lives as long as its registration and the calls in flight. No module-level strong cache holds a + contract, binding, schema or validator. +- TypeBox keeps its last build in module-level accumulators: after every `Compile`, reset them (`ResetFunctions`, + `ResetExternal`, in `finally`), so no build state is retained. + +### A failed check + +A failed compiled check uses TypeBox's public `Errors(schema, value)` and reports the first error. The error is +`invalid/args` or `invalid/params`, message `args/text: ` (fixed text for the two common cases: +`args/extra: unknown field`, `args/text: missing`), `details: { path }` with TypeBox's +instance path (`/text`, `/` for the root), including missing (`required`) and unknown (`additionalProperties`) +fields at their own path. TypeBox does not escape `/` and `~` inside keys, so a key containing them makes the path +ambiguous (owner, 2026-10-22: accepted; it is a diagnostic). + +## 6. Errors + +```ts +type ChordErrorCode = "invalid" | "contract_mismatch" | "forbidden" | "gone" | "not_ready" | "released" + | "too_large" | "unavailable" | "aborted"; +``` + +| Code | Reasons | +| --- | --- | +| `invalid` | required: `args`, `params`, `member`, `placement`, `reentrant`, `read_after_bulk`, `cycle` | +| `contract_mismatch` | optional: `updating` | +| `unavailable` | optional: `locked`, `park_timeout` | +| `too_large` | optional: `part` | +| others | none | + +- One `ChordError` class (an `Error`), with `code`, `reason`, `message`, `details?: JsonValue`. The constructor, + instances and JSON form are tagged unions: a wrong pairing is a type error, and narrowing on `code` narrows + `reason`. `toJSON()` emits `{ code, reason?, message, details? }`; stack and cause never cross the wire. +- `PROTOCOL_VERSION = 1` is exported. Reasons and pairings are link protocol: adding or moving one bumps it. A test + pins the table together with the version and fails with "bump PROTOCOL_VERSION" when they diverge. L6 compares the + version in the handshake before accepting anything. +- `chordErrorFromJson(raw)`: `raw` is fresh JSON from the link decoder (a trust boundary). Checks the envelope only: + known code, a reason allowed for that code (required for `invalid`), string message, no other keys. `details` is + kept by reference: no walk, no copy, no counters. +- Helpers: `isChordError(error, code?)`, `invalid(reason, message, details?)`, `asChordError(error, label)` (wraps a + foreign error as `unavailable`). +- `FailureCause = { kind: "transient"; error } | { kind: "broken"; error }`: the failing stage decides, never the + error code. `broken(cause)` marks an open's failure as broken (bad configuration); `openFailure(cause, label)` turns + an open's thrown value into the cause. Used by L4. +- Plugin errors have their own shape (L4); they never use Chord reasons. + +## Cost per hop + +| Hop | Work | +| --- | --- | +| a `get` | one compiled closed-object check, one joined key; no walk, no copy | +| a call | three map lookups, one compiled check, at most one direct mapping; no copy | +| first `get` of a contract | one compilation | +| first call per accepted version and method | one compilation (or a cache hit by schema object) | +| a failed check | one counted diagnostic walk (`Errors` builds the whole list on this rare path) | +| a first default principal key, a revocation key | one canonical walk | +| declaration | snapshot, closure check per method, fingerprint per version | +| binding | `checkAccepts` once, dispatch table built, nothing compiled | +| a state change | nothing in L1 | + +**Counters:** `chord.data.walks`, `chord.data.validations`, `chord.data.compilations` are telemetry counters +(`context.telemetry.metrics?.counter(name).add(1)`), always on. The function doing the work counts it; call sites never +count for others. Tests assert them per hop with the in-memory telemetry adapter. + +## Footguns + +- A provider must not mutate arguments or params it received (caller-owned, immutable by convention). +- `principalKey` must be the same function on every host; a different one splits identities silently. +- Params keys must escape the separator, so two different params never join to one key. +- Mappings are direct: when version 3 arrives, every still-accepted version's mappings are updated (compose step + functions: `(old) => v2to3(v1to2(old))`). + +## Tests (one group per mechanism, every case) + +- Contracts: every declaration check with its error path; D11 (kind and signature across variants, at runtime and + in the types); closed arguments (nested, raw `Type.Object`, records allowed, edits checked, state not checked); + schema vocabulary (each rejected feature); bytes (marker, bounds, check rebuilt after snapshot, no callback copied); + snapshots are independent of later changes to the app's schema objects; branding (spread copies refused). +- Fingerprints: stable across key order and annotations; sensitive to defaults, bounds, union order, params order, + `additionalProperties`; the same v1 from a separate declaration; local vs remote differ; a fixed known hash for one + fixed contract (protects the normal form). +- Evolution: each edit; kind changes refused (also remove then re-add); params and variant set fixed; new bases; + long chains (versions beyond 9, 99); history types. +- Accepts: every required-mapping rule above, at runtime and at the type level (`@ts-expect-error` for missing, + unnecessary and wrongly typed mappings); `"refuse"`; `call`; direct mapping over several versions; no mapping is + invoked by `checkAccepts`. +- Params: missing, extra and wrong fields with their paths; literal membership; constraints; the same key whatever + the property order; separator escaping (no two params join to one key); declaration refusals. +- Arguments: unknown fields (nested too); no copy (identity of the passed object); `local` and `remote`; bytes by + `instanceof` and length; old version checked with the old schema then mapped; current never mapped; `null` for + no-argument methods; unknown method; unaccepted version; `"refuse"`; lazy compilation (counted once per version and + method). +- Principals: default key, custom key, caching (no second walk), revocation keys. +- Errors: tagged union in types (`@ts-expect-error` for wrong pairings); JSON round trip; envelope checks (each + refusal); details kept by reference; the pinned protocol table; `FailureCause`, `broken`, `openFailure`. +- Cost guards: the counters per hop in the table above. +- GC: dropped contracts, bindings, dispatch tables, mappings, schemas and validators are collected after the last call + settles; params validators per contract; custom principal-key functions dropped while their principals live; nothing + retained by TypeBox's build accumulators. + +## Size + +Target: well below 1,500 source lines for `contracts/` and `data/` together, tests excluded. The type-level history +and mapping types are the heaviest part; keep the rest lean. diff --git a/packages/chord/docs/next/layer-2.md b/packages/chord/docs/next/layer-2.md new file mode 100644 index 000000000..e5dc5dee2 --- /dev/null +++ b/packages/chord/docs/next/layer-2.md @@ -0,0 +1,216 @@ +# L2: watchables and derives (spec, for the owner's approval) + +Status: **superseded (owner, 2026-10-22)**: the L2 code built from this spec was discarded (too much generic +machinery for callers that did not exist yet: nodes, sinks, deliveries, publications) and the build switched to +vertical slices (`slices.md`). Still valid: the section "Decided while specifying" below, and Chord's shared watch +handle (`src/watch.ts`). Everything else here is background only; do not implement from it. + +Original status: approved; open points decided (below). The single source for implementing L2; every rule comes from the decision log +(`packages/chord/docs/next/decisions.md`: "Watchable and one watch API", W1, W2, W3, W4, W12, W14, Q14, R23, R24, +"Changing contributions, `self.writable`", "Cost counters") unless marked **Decision needed**. Code style: the section +"Chord v2 Code Style" of the root `AGENTS.md`. + +## What L2 is + +Everything observable on one host, and how a change travels: a value changes inside a **publication** (one atomic step +that may change many sources), the derives over it are recomputed right after, and every consumer sees each +publication once, completely, in order. No objects, no storage, no links: L3 (store) publishes document commits +through it, L4 (objects) builds members from it, L6 (links) encodes what it flushes, L9 (registries) composes from it. + +Source: `packages/chord/src/next/watch/` (one directory; files by concept). Reused unchanged: Chord's shared +`src/watch.ts` (the watch-handle types **and implementation**, moved there from Durable's `CommittedWatch` by a pi change +first, owner 2026-10-22; Durable uses the same implementation), `src/delta/` (`Op`, `diffRevisions`), `src/context/`, +`src/json.ts`, and L1's `next/data` errors (`invalid/reentrant`). Nothing from `next-frozen/`. + +## 1. The two kinds + +```ts +interface Watchable { // documents, derives, reference members, constant(v), Durable views + readonly value: T; + watch(context: Context): Promise>; // listener (value, ops, context) +} +interface NativeWatchable { // writable(initial), composed registry values + readonly value: T; + watch(context: Context): Promise>; // listener (value, context) +} +``` + +- A `Watchable` carries JSON and changes with ops (Delta's `Op[]`): the ops turn the previous value into the next. + Values are immutable by convention: a new value shares unchanged parts with the old one; nobody mutates a value. +- A `NativeWatchable` carries any in-process value (functions, runtimes); a change is a new value by identity + (`!==`). Mutating a native value in place notifies nobody. It has no ops and can never be a contract state member or + go on the wire (Q14, W4). +- Both carry a runtime kind tag (`"json"` / `"native"`), so L4 can refuse a native one as state with one comparison + (no value walk). + +## 2. Publications (W1, W12) + +```ts +const publication = createPublication(); // one per host +publication.publish(() => { // one atomic step + notes.set(nextNotes, ops); // adopt: pointer swap + mark + index.set(nextIndex, indexOps); +}); // flush runs here, synchronously, before publish returns +``` + +- `createPublication()` returns the host's publication. A **source** (`jsonSource(initial, publication)`, + `writable(initial, publication)`) belongs to one publication. +- `publish(adopt)` runs `adopt` synchronously. Inside it, `source.set(value, ops)` (JSON) or `source.set(value)` + (native) swaps the value and **marks**: dependent derives become dirty, and the ops are appended by reference to each + watch's queue and each sink's batch. O(1) per dependent. Then, still synchronously and without any `await`, the + **flush**: + 1. recompute every dirty derive that has a consumer, in dependency order (each derive once per flush, after all its + sources), diffing its old and new value with `diffRevisions` into ops, and marking its own dependents; + 2. deliver each touched sink its batch for this publication (section 6); + 3. call the publication's flush listeners once (L6 uses this to send one message per link); + 4. schedule the queued watch frames (their listeners run later, asynchronously, in order). +- Every change goes through `publish`: L3's commit step (after the backend commit; S1), in-memory commits (no store + seq, the publication's own counter, W12), `writable.set` (which publishes itself), registry recomposition and + inbound wire batches. Each publication has a number (a counter per publication) for ordering and lazy checks. +- **Reentrancy:** during `adopt` and the flush, `publish`, `watch`, `attach`, a sink's `detach` and `writable.set` + throw `invalid/reentrant` (W1: no new transaction, watch or release inside the step). Stopping a watch handle is + allowed anywhere: it only detaches, and delivery tolerates removal during iteration. +- **Contexts** (context last, required, as in Durable): `createPublication(context)` keeps the host's context (it + counts lazy-read computes, since a getter has none); `publish(adopt, context)` and `writable.set(value, context)` take the context + that counts the flush's work and becomes the frames' context. `publication.number` is public (L3's in-memory commit + number, W12); `publication.onFlush(listener)` returns its removal. +- **Source ends:** a JSON source has `set(value, ops)`, `retire()`, `end()` and `end({ error })`, all only inside + `publish` (events in publication order), delivered in that publication's flush after its batch: watches retire, end + or fail; sinks get a retirement as `update(null, [["r", null]])` then `end()`, an end as `end()`, a failure as + `fail(error)`. A failed source's `value` throws its error, so derives over it fail with the same error; a retirement + is a change to `null` for derives over it; an end changes nothing for them. `watch` or `attach` on an ended source + returns a handle or sink ended the same way. A writable has only `set`. +- A source set outside `publish` throws (a programming error, `Error`, not a ChordError). + +## 3. Derives (W1, W4, W14, R24) + +```ts +const count = derive([notes], ([n]) => n.list.length, publication); +const visible = derive([notes, settings], ([n, s]) => ({ list: n.list.filter(...), theme: s.theme }), publication); +``` + +- `derive(sources, compute, publication)`: sources are `Watchable`s and `NativeWatchable`s (mixed) created by L2 on + the same publication (sources, derives, writables, constants; a foreign watchable such as a Durable view becomes an + L2 source first, L3d); `compute` is a synchronous pure function of their current values; the result is JSON and the + derive is a `Watchable`. +- **Attached only while consumed:** a derive with at least one consumer (a watch, a sink, or a consumed derive over + it) is attached to its sources and recomputed in the flush. A derive without consumers holds no subscription; reading + its `value` recomputes it if any source's publication number moved since its last compute (lazy). So an unused + derive never keeps work or memory alive, and nothing leaks when consumers go away. +- Ops: when a consumed derive recomputes, `diffRevisions(old, new)` gives its ops (identity short-circuit: unchanged + shared subtrees cost one `===`). A native source's change marks the derive dirty without ops. A recompute that + returns the identical value (`===`) publishes nothing. +- **Dependency order:** a derive's rank is one more than its highest-ranked source; the flush recomputes by rank. + Sources exist before the derive, so cycles cannot be built (cross-object cycles are rejected at activation, Q5, L4). +- **A throwing derive** fails only itself: its `value` throws the error, its watches end `{ reason: "failed", error }`, + its sinks get a failure, and derives over it fail with the same error. It stays failed (broken: no automatic + retry; L4 rebuilds it on reload). The publication and every other consumer continue. +- No `previous` (W14). The footgun and its remedy are documentation: a derive that builds new objects for unchanged + data costs O(value) per flush per consumer; `mapShared` (section 7) keeps unchanged elements identical. + +## 4. Constant and writable (R23, W4) + +- `constant(value)`: a `Watchable` that never changes; its watches never get a frame and end only by `stop` or + cancellation. No publication needed. Used for start data. +- `writable(initial, publication)`: a `NativeWatchable` with `set(value)`; `set` publishes by itself (one publication + per call); an identical value (`===`) publishes nothing. Used for registry contributions known only at runtime. + +## 5. Watch handles (W2, W3; Chord's shared implementation) + +L2 does not implement watch handles: it uses the one implementation in Chord's `src/watch.ts` (Durable's semantics, +moved there by the pi change). Sources create a handle per `watch(context)` and feed it: `advance` per publication +(during the flush, by reference), `retire`, `end`, `end({ error })`, cancellation from the context's signal. + +- `watch(context)` returns the handle at once (the Promise resolves immediately for local sources; L5 reference + members may wait). It holds the value current at acquisition and buffers every later publication. +- `start(listener)` installs the one listener; frames are delivered in order, each awaited, never inline; the listener + gets a context without the abort signal. +- **Overflow:** after 100 pending frames the backlog collapses: JSON to one frame `[["r", newest]]`, native to the + newest value. +- **Retirement** is an explicit source event (`source.retire()`, used by L3 for retired documents): the last frame is + `[["r", null]]` with value `null`, then the watch ends `retired`. Never inferred from a `null` value (W2a). +- **Ends:** `stopped`, `cancelled`, `retired`, `ended` (the source ended), `failed { error }` (the source failed, e.g. + a throwing derive), `listener_error { error }`; `stop` drops pending frames and never aborts or awaits a running call. + Termination clears every reference into the source (W3). +- `NativeWatchHandle`: the same lifecycle with listener `(value, context)`. +- **W2b** (routines delivering the acquisition value first) is L8's. + +## 6. Sinks (internal, for L6 and L9) + +A sink is a synchronous consumer inside the flush, used where a watch's asynchronous delivery is too late: the wire +encoder (L6: one part per hold, one message per link per publication) and registry composition (L9). + +```ts +const sink = attach(watchable, { update(value, ops) {...}, fail(error) {...}, end() {...} }); // returns detach() +``` + +- `update` is called once per publication that touched the source, during the flush, with that publication's ops + (several sets of one source within one publication arrive as one batch, in order). Native sources: `update(value)`. +- A sink counts as a consumer (derives over the source stay attached). `detach()` removes it; detaching inside the + flush throws `invalid/reentrant`. +- Not exported to apps; L4-L9 import it from `next/watch`. + +## 7. `mapShared` (W14, design §12) + +`mapShared(f)` returns a function over arrays that calls `f` only for elements it has not seen (by identity) and +returns the previous result for unchanged ones, so a derive over a list of 2,000 notes with one change builds one new +object and `diffRevisions` emits one op. The cache is a `WeakMap` from source element to result. + +## Cost per hop + +| Hop | Work | +| --- | --- | +| a publication | per set source: pointer swap, O(1) per dependent (mark, append ops by reference) | +| flush | each dirty consumed derive: one compute plus one `diffRevisions`; nothing for unconsumed derives | +| a watch frame | the value and ops by reference; no copy, no diff | +| a sink update | the ops by reference | +| an unconsumed derive read | one compute if a source moved, else nothing | +| constant | nothing | + +**Counters** (telemetry, the function doing the work counts it): `chord.watch.computes` (derive computes) and +`chord.watch.diffs` (`diffRevisions` calls). No walks or copies anywhere in L2. + +## Footguns + +- A derive that rebuilds unchanged data costs O(value) per flush per consumer (use `mapShared`; R24: changing one + item of a flat array still copies the array's slots). +- Mutating a value in place breaks every consumer silently (immutable by convention). +- Async work in a derive is impossible by design; it belongs in a routine (L8) writing an in-memory document. + +## Tests (one group per mechanism, every case) + +- Publication: several sources in one publication reach a watch as consecutive frames and a sink as one batch; + publication numbers increase; setting outside `publish` throws. +- Derives: dependency order (diamond: A→B, A→C, B+C→D computes D once, after B and C); identical result publishes + nothing; native source marks without ops; lazy read recomputes only after a source moved; attach on first consumer, + detach after the last (no subscription left; GC); a chain A→B→C where B throws (C fails, A's other consumers + continue); a failed derive stays failed. +- Two sources published alternately with a derive over both: the derive sees every combination exactly once per + publication. +- Reentrancy: `publish`, `watch`, `attach`, `writable.set`, `detach` inside a derive and inside a sink throw + `invalid/reentrant`; `stop` inside a sink or listener is allowed and delivery continues correctly. +- Watch handles (the handle itself is tested in Chord's shared tests): through L2 sources, a watch started after 150 + publications; retire; end and fail from the source; stop during a running call; cancellation by the context. +- Constant and writable: constant never delivers; writable identical set publishes nothing. +- `mapShared`: unchanged elements return the identical result; a changed element is recomputed. +- Sinks: one batch per publication; failure and end; detach. +- Cost guards: computes and diffs per hop (an unconsumed derive computes nothing on publish). +- GC: stopped handles hold no source; unconsumed derives hold no subscription and are collected with their sources; + `mapShared` caches drop with their elements; retained revisions after overflow are bounded (W1's GC count). + +## Size + +Target: about 500 source lines, tests excluded. + +## Decided while specifying (owner, 2026-10-22) + +- **Watchable state is JSON only.** No bytes in documents, derives, reference members or any watchable value; bytes + appear only in method arguments and results (passed whole, carried as raw byte segments by L6) and in chunked + transfers. Delta stays JSON-only. Supersedes W4's "JSON plus bytes" for watchables. +- **`mapShared` lives in `next/watch/`** for now; it moves into Delta when v2 goes upstream. +- **C10 rejected:** `constant` (JSON `Watchable`) and `writable` (`NativeWatchable`) stay two, because Q14 needs the + kind distinction. +- **One publication per host** (`createPublication()`, created by the host and passed to every source, derive and + writable on it; apps never see it), never a module-level singleton. +- **No CBOR:** the wire stays JSON plus raw byte segments (R1, Q4); revisit only if L6 benchmarks show encoding as a + bottleneck. diff --git a/packages/chord/docs/next/slice-1.md b/packages/chord/docs/next/slice-1.md new file mode 100644 index 000000000..e0f17f077 --- /dev/null +++ b/packages/chord/docs/next/slice-1.md @@ -0,0 +1,236 @@ +# Slice 1: one host, local objects (spec, for the owner's approval) + +Status: **approved (owner, 2026-10-22)**, after Astra's and Fable's spec reviews. The single source for building slice 1, +together with the feature map in `slices.md` (entries assigned to slice 1) and `design.md`, read in full (the most +relevant sections: §3 host, §5-§7, §8, §9, §11, §12, §16, §18); rules come from the decision log +(`packages/chord/docs/next/decisions.md`); code style: the section "Chord v2 Code Style" of the root `AGENTS.md`. Builds on +L1 (`src/next/contracts`, `src/next/data`) and Chord's shared watch handle (`src/watch.ts`). + +## What an app can do after this slice + +```ts +const Notes = local("notes.store", { + version: 1, + params: {}, + members: { + notes: obj({ list: arr(Note) }), + titles: arr(str()), + count: int(), + add: method(obj({ text: str() })), + saveDraft: method(obj({ conversation: int(), body: str() })), + }, +}); +const Draft = local("notes.draft", { + version: 1, + params: { conversation: int() }, + members: { summary: str() }, +}); + +const NotesDoc = defineState("notes", { schema: obj({ list: arr(Note) }), version: 1, initial: () => ({ list: [] }) }); +const Drafts = defineState("drafts", { schema: obj({ body: str() }), version: 1, initial: () => ({ body: "" }), keyed: true }); + +const facet = defineFacet((host) => { + host.serve(Notes, (_, self) => { // empty params: lives as long as the host + const notes = self.state(NotesDoc); + const drafts = self.state(Drafts); // keyed: one document per conversation (P3 pattern) + const title = mapShared((note: Note) => note.text.slice(0, 20)); + return { + notes, + titles: self.derive([notes], ([n]) => title(n.list)), + count: self.derive([notes], ([n]) => n.list.length), + add: async ({ text }, ctx) => { + await self.tx(async (tx) => { (await tx.draft(notes)).list.push({ text }); }, ctx); + return null; + }, + saveDraft: async ({ conversation, body }, ctx) => { + await self.tx(async (tx) => { (await tx.draft(drafts, String(conversation))).body = body; }, ctx); + return null; + }, + }; + }); + host.serve(Draft, ({ conversation }, self) => { // params: lives while referenced + const store = self.get(Notes, {}); // declared here, live before this object is + return { summary: self.derive([store.count], ([c]) => `conversation ${conversation}, ${c} notes`) }; + }); +}); + +const host = createHost({ onError: (error) => log(error) }); // no address: local types only +await host.load([{ name: "@app/notes", facet }], ctx); +const notes = await host.get(Notes, {}, ctx); // live and hydrated +await notes.add({ text: "milk" }, ctx); +notes.count.value; // 1 +const w = await notes.count.watch(ctx); +w.start(async (value) => console.log(value)); +notes.release(); // ends w; the object lives on (empty params) +await host.close(ctx); +``` + +## 1. Host and facets + +- `createHost({ telemetry?, onError? })`: a host without an address; it serves only `local` types in this slice. + `telemetry` is the adapter (NOOP by default); `onError` receives background errors (a throwing watch listener). +- `defineFacet

(setup)` returns a facet typed for the app's principal type `P` (base code binds it once; nothing is + checked at runtime). `setup(host, self)` runs synchronously inside `host.load` and only declares: `host.serve(Type, + factory)` and facet documents through `self.state(Def)` (C16, owner: built now). `host.serve` outside a setup + throws. +- **Facet documents (design §9, §16):** the facet is a pure object with the host's lifetime; its documents live as long + as the host. Objects the facet serves may read the facet's handles and draft them in their own `self.tx`, together + with their own documents, atomically (lock order: the object's own line, then the facet's line). The facet never + drafts a served object's documents; objects of other facets cannot use them. +- `host.load(facets, context)`: `facets` is `[{ name, facet }]` with npm-style plugin names; duplicate names or a type + served twice are refused. After every setup ran, every served `local` type with empty params is activated (Q-A, the + start barrier); `load` resolves when all are live. If one fails, `load` ends everything it activated and rejects; + the host is then empty. A `host.get` made before `load` resolves waits for it. +- `host.close(context)`: rejects new calls, ends every object (in reverse activation order) and resolves when all have + ended. + +## 2. Objects + +- **Identity:** one object per `(type, params key)` (L1's `checkParams` and `identityKey`; local has no principal). +- **Lifetime (Q-A):** a local type with empty params is created at `load` and lives until `close`; a type with params + is created by the first `get` and ends when its last reference is released. +- **Factory:** `(params, self) => implementation`, synchronous and declarative, run once per object. The + implementation has exactly the contract's members: a state member is a JSON `Watchable` of its value (a document + handle, a derive, a member of a reference), a method is `(args, ctx) => Promise`. L1's `Implementation` + types state members as `Watchable>` (owner, 2026-10-22). +- **Activation (design §6):** run the factory; acquire the references it declared with `self.get` (activating their + objects if needed) and wait until they are live; check the members (exactly the contract's, state members JSON + watchables, Q14); compute every derive's first value; then the object is live and calls proceed. A throw anywhere + in activation (factory, a dependency's activation, a member check, a first derive compute, `initial()` failing its + schema) rejects the `get` (or `load`) with that error, releases everything acquired and leaves no object. Failure + states, backoff and retries are slice 2. +- **Ending:** new calls on the object fail `released`; the context of each of its running calls is aborted; from then + on its `self.tx` throws `released` (a still-running method can no longer change anything); its derives are detached + from their sources (no source keeps a reference to an ended object's derive); its references are released; its + documents are dropped; every watch on its members ends `ended`. Ending does not wait for running calls (the drain + arrives with natives in slice 2, design §3). + +## 3. References + +- `host.get(Type, params, context)` returns a reference once its object is live; `self.get(Type, params)` declares one + in a factory: it is live by the time the object goes live, and released when the object ends. +- A reference is a plain object built from the contract's member list (no proxy): `ref.` is a `Watchable` + (`value`, `watch(ctx)`), `ref.(args, ctx)` calls the method, `ref.release()` releases it (idempotent). After + release, every use throws `released`. +- **Calls:** arguments and results pass by reference; arguments are checked at the provider with L1's `checkArgs` + against the current version (`bindProvider(contract, {})` at serve): unknown method `invalid/member`, bad arguments + `invalid/args`; results are not checked; no-argument methods take `null`. +- **Per-reference watches** (owner): watches obtained through a reference end (`ended`) when it is released, even + while other references keep the object alive. +- **Contract match** (owner): at `get`, the caller's contract must have the provider's version and fingerprint (the + same declaration, or an identical one), otherwise `contract_mismatch`. Older versions through `accepts` come with + slice 7. +- **Reserved member names** (owner): `then`, `release`, `state` and `retry` are refused as member names at + declaration (an L1 check in `remote`, `local` and `evolve`), because a reference has them (`release` now, `state` and + `retry` in slice 2) and `then` would make an awaited reference thenable. +- **Cycles (Q5, one host; owner):** at `self.get` during an activation, Chord follows the chain of objects waiting for + activation; reaching the object being activated fails that activation with `invalid/cycle`, naming the cycle + (`a → b → a`); nothing is kept. +- `ctx` in this slice is a plain `Context` (cancellation and telemetry); `ctx.get` and `ctx.principal` come with + slice 3. + +## 4. Documents in memory (§9, P2) + +- `defineState(name, { schema, version, initial, keyed? })`: the root is a JSON object. `checkpointWhen` is required + only for persisted documents (slice 6; owner, 2026-10-22). `migrate` comes with slices 6 and 7. +- `self.state(Def)` returns a handle bound to this object; every document is in memory, instance state of the object + (P2), gone when it ends. +- **Singleton handle:** `handle.value` is the current value, `initial()` until written or after a retire (Q11); the + handle is a `Watchable`. +- **Keyed handle:** `await handle.get(key)` gives the value or `undefined`; `handle.keys()` is an `AsyncIterable` of + the present keys (the shape persistence keeps); keys are strings. A keyed handle is not a `Watchable` (a live + collection is a singleton holding a list or map). +- `initial()` is checked against the schema once, the first time a definition's `initial()` is used (R8). + +## 5. Transactions (§9, R12, R13) + +- `self.tx(fn, ctx)`: an object's transactions run one at a time (its line). `fn` gets `tx`: `await tx.draft(handle, + key?)` returns the document's draft (Delta's `track`, `beginChange`; the same document gives the same draft within + one transaction); `tx.retire(handle, key?)` removes a document (a no-op when absent). `self.tx` returns what `fn` + returns. +- **Commit:** when `fn` resolves, if `ctx` is aborted the transaction is discarded; otherwise every draft is prepared + (Delta's `prepare`: ops recorded, only placed values checked), unchanged drafts commit nothing, and all changed + documents are adopted (pointer swap) in **one batch** (section 6). A throw discards everything. +- Drafts are revoked when the transaction settles (touching one throws, R12); a transaction returns plain values and + data is read from the handle after the commit. +- A transaction drafts only its own object's documents and its facet's (any other handle throws). Inside `fn`: no + second `self.tx`, no calls to other objects (Durable's rule; documented, not detected). +- **Shorthands (C14, owner: built now):** `handle.change(fn, ctx)` (keyed: `handle.change(key, fn, ctx)`) runs one + transaction that loads the document and calls the synchronous `fn(draft)`; `handle.retire(ctx)` (keyed: + `handle.retire(key, ctx)`) retires it. Both are thin wrappers over `self.tx`. + +## 6. Derives and the batch (W1, §11, §12) + +- `self.derive(sources, compute)`: sources are JSON `Watchable`s (this object's handles and derives, members of + its `self.get` references); only the declared sources are tracked. `compute` is a synchronous pure function of their + values; its result is JSON. +- **Pull model.** A commit only marks: every derive that depends on a changed document, directly or through other + derives, including derives of other objects on the host, is flagged dirty (O(dependents), flags only). A derive's + `value` recomputes when it is dirty, after reading its sources' values (which recompute first if they are dirty); + so each derive computes at most once per batch and always after its sources, diamonds included. +- **The batch** (one per host): a commit's adoptions happen inside it; after the last adoption, still synchronously, + every watched value that changed is read (watched derives recompute through the pull), its ops computed + (`diffRevisions(old, new)` for a derive, the commit's ops for a document), and its watches get one frame each. A + derive is recomputed in the batch only if something watched depends on it; otherwise it stays dirty until read. + Reading `.value` once does not make it watched. +- A derive that throws fails only itself: its `value` throws the error and its watches end `failed`; a derive over it + throws the same error when read. A derive that fails during activation fails the activation (section 2). A failed + derive stays failed until the object is rebuilt. +- **Reentrancy:** inside the batch (adoptions, derive computes, including lazy computes outside a commit), `self.tx`, + `watch`, `release`, `get`, `load` and `close` throw `invalid/reentrant`. One flag per host, cleared in a `finally`. +- `mapShared(f)`: maps an array calling `f` only for object elements not seen before (by identity), returning the + previous result for unchanged ones; primitive elements are mapped every time; the cache is a `WeakMap`. +- `constant(value)`: a JSON `Watchable` that never changes (start data, §18); usable as a state member and a source. +- State is JSON only (L2 decision): no bytes in documents or derives. + +## 7. Watching + +`watchable.watch(ctx)` returns Chord's shared `Watch` (`src/watch.ts`): the value at acquisition, then one frame per +batch that changed it, `(value, ops, ctx)`, in order, asynchronously, with its overflow, `stop` and end semantics. +Each frame is consistent with its commit; two different watches are not ordered relative to each other. A retired +singleton gives the frame `[["r", initial]]` (it changes back to `initial()`; a singleton always has a value). `ctx`'s +abort cancels the watch. A listener that throws ends its watch `listener_error` and is reported to `onError` (taken +from the carry-over). + +## Cost per hop + +| Hop | Work | +| --- | --- | +| a call | L1's three lookups and one compiled check; arguments by reference | +| a commit | Delta prepare per changed document, pointer swaps, dirty flags per dependent, then per watched changed value one read (derives recompute once) and one frame per watch, by reference | +| an unwatched derive | nothing until read | + +Counters (telemetry, where the work happens): L1's, plus `chord.derive.computes` and `chord.derive.diffs`. + +## Carry-over taken by this slice + +From `carry-over.md` (L4 section): watch listener errors reported to `onError`; native watchables refused as state +(by the member check: only JSON watchables exist in this slice); `defineFacet` typed by `P`. Left for +later: plugin error shape, old-version calls (7), `broken()` (2), cuts C5, C6, C7, C13 (2). + +## Tests + +- Host: load runs setups, activates empty-params types, rejects duplicates and double serves; a failing activation + makes `load` end what it activated; `get` during load waits; `serve` outside setup throws; close ends everything. +- Objects: one object per params key; params type created on first get and ended on last release (documents gone, + watches end `ended`, its derives detached from other objects' members); empty-params type lives across releases; + activation failures (factory, dependency, member check, first derive, `initial()`) reject the get and leave nothing; + a method running while its object ends cannot commit (`self.tx` throws `released`) and sees its `ctx` aborted. +- References: hydrated on get; members watchable; calls checked; released reference throws; `self.get` dependency + kept alive and released on end; per-reference watches; contract mismatch; reserved member names refused at + declaration; a two-object and a three-object cycle fail with `invalid/cycle` and leave nothing. +- Documents: singleton initial, retire back to initial; keyed get, keys, retire; `initial()` checked once. +- Transactions: serialized per object; same draft twice; unchanged commits nothing; throw and abort discard; drafts + revoked after settle; another object's handle refused; a served object's transaction over its own and its facet's + documents commits both in one batch; shorthands. +- Batch and derives: two documents in one transaction reach a derive over both in one compute and a watch in one + frame; a derive over another object's member updates in that object's commit's batch; diamond computes once; + unwatched derive stays lazy and a one-off read does not make it eager; a throwing derive fails only itself and its + dependents; reentrancy throws in commits and lazy computes and the flag clears after a throw; `mapShared` identity. +- Watching: frames in order, retire frame, cancellation, listener error reported. +- Cost guards per the table. GC: an ended object with its documents, derives and watches is collected; a released + reference and a stopped watch hold nothing; a detached derive is not held by its former sources. + +## Size + +About 1,100 source lines. diff --git a/packages/chord/docs/next/slice-2.md b/packages/chord/docs/next/slice-2.md new file mode 100644 index 000000000..003532ace --- /dev/null +++ b/packages/chord/docs/next/slice-2.md @@ -0,0 +1,174 @@ +# Slice 2: failures and natives (spec, for the owner's approval) + +Status: **approved (owner, 2026-10-22)**, after Astra's and Fable's spec reviews. The single source for building slice 2, together with the feature map in `slices.md` +(entries assigned to slice 2), `design.md` read in full (most relevant: §3 timeouts, §6 lifecycle and failures, §7 +references, §13 natives, §21 start barrier), the decision log, the carry-over (L4 section) and the root `AGENTS.md` +style section. Builds on slice 1 (`src/next/state`, `documents`, `host`, committed `d8f5e67e6`) and L1. + +## What an app can do after this slice + +```ts +host.serve(Inbox, (_, self) => { + const settings = self.get(Settings, {}); + const watcher = self.open( + async (context) => watchDir(settings.dir.value.path, context.abortSignal), // opened during activation + async (w, reason) => w.close(), // closed in reverse when the object ends + ); + return { count: …, rescan: async (ctx) => watcher.value.rescan(ctx) }; +}); + +host.serve(Remote, (_, self) => { + self.open(async (context) => { + const config = readConfig(); + if (!config) throw broken(new Error("no remote configured")); // retrying is pointless: broken + return connect(config, context.abortSignal); // a throw here is transient: retried with backoff + }, (c) => c.close()); + return { … }; +}); + +const host = createHost({ onError, closeTimeout: 30_000, readyTimeout: 60_000 }); +await host.load(facets, ctx); // resolves once every host-lifetime object is live or failed; failures reported +const inbox = await host.get(Inbox, {}, ctx); // waits through transient retries up to readyTimeout +await host.close(ctx); // closes natives (reason "stop"); rejects with every close error +``` + +## 1. Natives (§13) + +- `self.open(open, close, options?)` in a factory returns a handle (`Native`). Its `value` is readable from the + moment its own `open` resolved until its `close` starts (so a later opener and handlers during the drain can use + it); before that it throws `not_ready`, after that `released`. +- `open(context)` receives a context whose abort signal fires when the attempt is abandoned (the object ends, the + attempt times out). `close(value, reason, context)` with `reason: { kind: "end" } | { kind: "stop" } | { kind: + "reload" }` (`end`: last reference gone or a failed attempt; `stop`: `host.close`; `reload` is used from slice 7); + the close context's signal fires at the native's `closeTimeout`, so a slow close can stop its own work. `options.closeTimeout` + overrides the host default for this native. +- **Activation order:** factory; singleton documents hydrated; references live; natives opened **one after another in + declaration order**; members checked; derives computed; live. An opener may read documents, references and earlier + natives, not derives (they are computed after the opens). +- **Closing:** in reverse order. Each close is awaited up to its `closeTimeout`; one that takes longer is reported to + `onError` as a possible leak and the others continue. A throwing close does not stop the others: `host.close` collects + every close error and rejects with them (`AggregateError`); an object ending in the background reports each to + `onError`. +- **Abandoned opens:** a failed or abandoned attempt closes the natives already open (reason `end`). An `open` that + resolves after its attempt was abandoned has its value closed at once, exactly once (reason `end`); an abandoned + attempt never comes back to life. A later attempt opens every native anew. +- Facets cannot open natives (`FacetSelf` has no `open`). The two restart patterns of R16 (drain in the background, + swap through a registry) are documentation in the host README; there is nothing to build. + +## 2. Failures (§6) + +- Every activation failure becomes a `FailureCause` (L1: `{ kind: "transient" | "broken"; error: ChordError }`, + foreign errors wrapped by `asChordError`). The kind comes from where it failed, never from the error: + + | Where | Kind | + | --- | --- | + | factory throws; a singleton's `initial()` fails its schema; member check (incl. a native watchable as state, Q14); first derive compute; a dependency cycle (Q5) | broken | + | an `open` throws | transient, unless it threw `broken(error)` (L1) | + | the attempt did not become live within `readyTimeout` | transient: `unavailable` "did not become live within …" | + | a dependency failed (R18) | transient: `unavailable` "A: dependency B failed", with B's error as its `cause` | + + A keyed document's first `initial()` runs inside a later transaction; a failure there fails that transaction, not the + object. +- **Every attempt is bounded** by the host's `readyTimeout`: when it passes, the attempt is abandoned (openers' + contexts aborted, open natives closed `end`, a late open closed at once) and fails transient. A hanging `open` can + therefore never block an object or the start barrier. +- **What a failed object keeps:** only an R18 failure keeps the object's references (so the dependency's recovery can + retry it); every other failure releases them at once (a cycle, a factory, an open, a derive). A failed object stays + while anything references it (host-lifetime objects always); it ends with its last reference. +- **Retries.** One attempt per object at a time; a trigger arriving during an attempt is ignored. Transient: retried + with a **backoff per identity**, 1 s doubling to 60 s. The backoff lives in a host map `identity → { delay, timer }` + holding no object (the window is its timer); it is deleted on success, on a broken failure, or when a window has + passed with no object left to retry, and cleared by `host.close`; while its object stays failed the entry keeps the + delay for the next doubling. A get inside the + window waits until `until` before the first attempt; a failed object without references is not retried + (host-lifetime objects always are). Broken: no automatic retry and no entry; a new object after its end tries at + once. In this slice a broken host-lifetime object stays failed until the host is recreated (reload, slice 7, and + `retryOn`, slice 8, add triggers). +- **Dependency recovery (R18):** when a failed dependency goes live, its R18 dependents are retried at once (their + backoff timers cancelled). On every retry the new attempt binds its `self.get` references **before** the previous + attempt's are released, so a dependency held only by this object never ends between attempts. +- **Reporting:** every failed attempt is reported to `onError` (with the object's name and cause); gets waiting for it + are rejected in addition (section 3). +- **The start barrier (§21):** `load` resolves when every host-lifetime object's **first** attempt has an outcome (live + or failed); later retries never hold `load`. A facet setup that throws or a facet document's `initial()` failing its + schema still rejects `load` (code errors). + +## 3. Getting a failing object (§7) + +- `host.get(Type, params, context, { readyTimeout? })` resolves with a live reference. Each getter has its own deadline + (`readyTimeout`, host default, overridable per get) for its whole wait, including a wait for `load`; it is separate + from the attempt's own bound. A broken cause rejects the getter at once with its error; a transient one keeps it + waiting through retries until the object is live or its deadline passes, then rejects with the last cause's error + (or `unavailable` if no attempt has failed yet). A getter whose deadline passes or whose context is cancelled + rejects and releases only its own reference; it never cancels the attempt for other getters. + +## 4. Ending with natives (§3, §6) + +- When an object ends: its transactions are sealed at once (`self.tx` throws `released`); callers of its running calls + are rejected `released` at once and the handlers' contexts aborted (slice 1); Chord keeps each running **handler's** + own promise and waits for those to settle (the drain), bounded by `closeTimeout`; then it closes its natives in + reverse order (a handler still running after `closeTimeout` sees its natives close; that is reported); its documents + stay readable until its closes are done, then are dropped; then the rest as in slice 1. +- A new attempt or a new object for the same identity starts only after the previous instance's ending has finished; + gets arriving meanwhile wait. +- `host.close(context)`: marks every object ended at once (every call on the host fails `released`, every `self.tx` + throws), aborts pending opens (their late values are closed), joins endings already running in the background, then + drains and closes object by object in reverse live order (a dependent before its dependencies), with reason `stop`; + resolves when all are done, or rejects with the collected close errors. + +## 5. Timeouts (§3, cut C13) + +Host options with per-thing overrides: `closeTimeout` (default 30 s; per native), `readyTimeout` (default 60 s; per +get; also bounds each attempt). The call drain uses `closeTimeout` (no separate setting; the design's per-type drain +override is not built). Park deadline (slice 3), routine handover and `finish` retry (slice 5) and rollout install +(slice 7) come with their slices. + +## Cost per hop + +Commits and derives are unchanged. A call adds one set entry for its handler's promise (the drain needs it). +Activation adds one awaited open per native and one attempt timer; a failure adds one backoff entry; ending adds the +drain and the closes. + +## Carry-over taken by this slice + +From `carry-over.md` (L4): `broken()`/`openFailure()` used for opens; a native watchable as state fails `broken` +naming the member; the drain on ending; cuts C5, C6, C7, C13 decided here. + +## Tests + +- Natives: open order and access to earlier handles; `not_ready` before live; close in reverse with reasons `end` and + `stop`; a slow close reported after `closeTimeout` and the others still closed; throwing closes collected by + `host.close` (AggregateError) and reported to `onError` for a background end; a partly failed attempt closes what it + opened; the opener's context aborted when the object ends while opening; facets have no `open`. +- Failures: each row of the table gives the right kind; the cause's error is a ChordError; transient retries follow + the backoff (fake timers) and reset on success; concurrent gets share one attempt; a broken failure is not retried; + a failed object ends with its last reference and its backoff entry expires; a get right after a failure waits for + the scheduled retry. +- R18: a dependent of a failing dependency fails transient with the dependency's cause, keeps the reference and is + retried at once when the dependency goes live; releasing the dependent releases the dependency. +- Start barrier: load resolves with a failed host-lifetime object, reports it to `onError`, and the object goes live + on a later retry; a throwing facet setup still rejects load. +- Gets: broken rejects at once; transient waits and succeeds on retry; transient past `readyTimeout` rejects with the + cause; per-get `readyTimeout`; a cancelled caller releases only its own reference. +- Ending: the drain waits for a running handler (its own promise, not the rejected caller's) before its native + closes, bounded by `closeTimeout`; documents readable during closes; transactions sealed at once. +- Races: an open resolving after its attempt was abandoned is closed once; a get arriving while the previous instance + closes waits and gets a fresh attempt; simultaneous retry triggers give one attempt; two getters with different + deadlines; a dependency held only by a retrying dependent is not ended between attempts; `host.close` during a + pending open, during a background ending, and with several objects in dependency order; a cycle releases its + references. +- GC: a failed object (documents, dependencies, natives, callbacks) is collected after its last reference while its + backoff entry still exists; the entry expires on its own. +- Examples: `06-natives.ts`, `07-failures.ts` (numbering continues from slice 1). + +## Decided with the owner (2026-10-22) + +- `ref.state`, `ref.retry()` and `host.hold` move to slice 3 (no app-held reference can be failed before then). +- `host.get` on a failing object and the start barrier as in sections 2 and 3; C13 as in section 5; C5: no natives in + facets; openers take a `Context`; `close` gets a context aborting at `closeTimeout`. +- C7: `linger` is built in slice 3, with links. The plugin error shape is decided in slice 3, where errors first cross + a link. + +## Size + +About 700 source lines (both reviewers: the attempt states, backoff map, drain and timers). diff --git a/packages/chord/docs/next/slice-3a.md b/packages/chord/docs/next/slice-3a.md new file mode 100644 index 000000000..4605bb95c --- /dev/null +++ b/packages/chord/docs/next/slice-3a.md @@ -0,0 +1,296 @@ +# Slice 3a: remote objects over one link (spec, for the owner's approval) + +Status: **approved (owner, 2026-10-22)**, after Astra's and Fable's spec reviews. Guiding rule (owner): **simple, fast, +correct**; where a choice was open, this spec takes the simplest option that is correct and cheap per hop. + +Slice 3 of the plan is split: **3a** (this) gets two hosts talking over one link; **3b** makes the link survive drops +(re-holds, parking, `ref.state`, `host.hold`, `ref.retry()`, `linger`, `host.root`, `state: false`). Flow control, size +limits, compression, relays, routing by address and IPC/WebSocket stay in slice 4; trace carriers (R5) in slice 9. The +single source for building 3a, together with the feature map in `slices.md`, `design.md` read in full (most relevant: +§3 links and values, §4, §5, §7, §8, §22), the decision log (R1, R4, R6, R20, Q1, Q4, W1, D11), the carry-over (L5, +L6) and the root `AGENTS.md` style section. Builds on L1 and slices 1-2 (`da3a16e9f`). + +## What an app can do after this slice + +```ts +type P = { kind: "owner" } | { kind: "guest"; session: string } | { kind: "server" }; + +const Notes = remote("notes", { + version: 1, + params: { session: str() }, + variants: { + editor: { state: { notes: arr(Note), count: int() }, methods: { add: method(obj({ text: str() })) } }, + reader: { state: { notes: arr(PublicNote), count: int() } }, + }, +}); + +const facet = defineFacet

((host) => { + host.serve(Notes, ({ session }, principal, self) => { + const store = self.get(NoteStore, { session }); // a local shared store (slice 1) + const count = self.derive([store.notes], ([n]) => n.length); + if (principal.kind === "guest") { + return { variant: "reader", members: { notes: self.derive([store.notes], redact), count } }; + } + return { variant: "editor", members: { notes: store.notes, count, add: async (args, ctx) => store.add(args, ctx) } }; + }); +}); + +const server = createHost

({ principal: { kind: "server" }, onError }); +await server.load([{ name: "@app/notes", facet }], ctx); +server.accept(link.a, { principal: { kind: "guest", session: "S1" } }); // the entry point authenticated it + +const client = createHost

({ onError }); // a tab: no principal of its own +await client.load([], ctx); +client.connect(link.b, { principal: { kind: "server" } }); +const notes = await client.get(Notes, { session: "S1" }, ctx); +if (notes.variant === "reader") console.log(notes.notes.value); // narrowed by variant +``` + +## 1. Hosts and links + +- `createHost({ principal?, principalKey?, onError, … })`: a host with a `principal` acts as it for its own requests; + a host without one (a client) has no principal. `principalKey` defaults to L1's canonical JSON. +- **A link** is an ordered, bidirectional channel of `Uint8Array` messages: `send(message)`, `onMessage(listener)`, + `close()`, `onClose(listener)`. This slice ships `memoryLinkPair()`, which delivers each message one task later (as + every link kind will). `send` after close is a no-op. +- `host.accept(link, { principal })`: a link from a child. Every hold and call arriving on it acts as `principal`, + stamped by this host. `host.connect(link, { principal })`: this host's parent link (at most one; a second `connect` + throws). In 3a a host serves over `accept` links and requests over its `connect` link; the `connect` principal is + stored for 3b and slice 4 (nothing arrives for it yet). +- **Handshake:** each side sends `hello { protocol, incarnation }` first and nothing else until it received the + peer's. A message before `hello`, a second `hello`, or another `PROTOCOL_VERSION` (L1) closes the link. `incarnation` + is a uuidv7 minted per host; calls arriving on an `accept` link get the peer's as `ctx.incarnation` (R20). +- **Where a reference goes:** + + | Type | Served on this host | Not served here | + | --- | --- | --- | + | `local` | local (slices 1-2) | fails `unavailable` (local types never cross a link) | + | `remote` | local, as the requester's principal (table below) | over the `connect` link; without one, `unavailable` | + + Any object may reference remote types, local objects included (a client's local view may `self.get` a server type). + Remote objects live while referenced, also with empty params; `load` never activates them (Q-A). +- **Which principal a same-host remote reference uses:** + + | Requester | Principal | + | --- | --- | + | `host.get` | the host's `principal` (a host without one fails `forbidden`) | + | `ctx.get` in a handler | `ctx.principal` (a guest's call gets the guest's view); a local handler: the host's | + | `self.get` in a remote object's factory | that object's principal | + | `self.get` in a local object's factory or a facet | the host's | + +- **Closing:** a link closing (either side) ends what was bound over it: on the consumer, references through it end + (`released`, watches `ended`) and pending calls and gets reject `unavailable`; on the provider, its holds are + released. `host.close` closes its links first, then ends its objects (no `gone` to closed links). Surviving a drop + is slice 3b. + +## 2. Messages (R1, R4, Q1, Q4, §22) + +``` +| u32 header length, big-endian | header JSON (UTF-8) | part 0 | part 1 | … | +``` + +- **One format on every link.** The header is a JSON object, a tagged union over `kind`, defined as a TypeBox schema; + every received header is checked with the compiled schema (the wire is a trust boundary; skipping it on trusted IPC + links comes with IPC in slice 4). The header lists its parts: `parts: [{ json, bytes }]`, the JSON text's byte length and + the byte lengths of the segments that follow it (`bytes: []` for state parts and arguments without bytes). +- **Kinds,** consumer → provider: `hello`, `hold { hold, type, version, fingerprint, params }`, `release { hold }`, + `call { call, hold, method }` (1 part: the arguments), `cancel { call }`. Provider → consumer: `hello`, `bound { + hold, variant, failed? }` (1 part: the snapshot), `state { holds: [{ hold, failed? }] }` (1 part per listed hold), + `failed { hold, cause }`, `result { call }` (1 part), `error { call, error }`, `gone { hold, error }`. +- **Ids** are numbers chosen by the sender, unique per link and direction, never reused while in use (a counter). + Ids a receiver does not know are ignored (Q1: a `result` crossing a `cancel`, a `state` part crossing a `release`). + A `hold` or `call` reusing a live id is a protocol error. +- **One Delta encoder per state member** (owner, 2026-10-22, after the code review): the snapshot part is `{ [member]: + value }` (each member's encoder restarted); a `state` part is `{ [member]: WireOp[] }` with only the members that + changed (a derive's change is its `diffRevisions` ops; a retired singleton is `[["r", initial]]`). +- **A failed member** is listed in the header (`failed: [{ member, error }]`) of the `bound` or `state` that carries the + failure, never in the part; the consumer's replica of that member then fails (reads throw the error, watches end + `failed`), the others keep updating (W1). +- **Argument and result parts** are JSON text followed by raw byte segments: a `Uint8Array` leaf is written as + `{"$bytes": i}`, segment `i`'s length in the part's header entry. Decoding replaces each reference with a `subarray` + view of the received buffer (no copy). Example: `upload({ name: "chart.png", data: <1 MiB>, thumbs: [<2 KiB>, <512 B>] + })` travels as header `…"parts":[{"json":74,"bytes":[1048576,2048,512]}]`, then the JSON + `{"name":"chart.png","data":{"$bytes":0},"thumbs":[{"$bytes":1},{"$bytes":2}]}`, then the three raw segments. Encoding uses the + `JSON.stringify` replacer only for methods whose schema contains a `bytes()` leaf (decided once per method); others + are plain `JSON.stringify`. The key `$bytes` is reserved in those methods only (see below). State parts are plain JSON (state is JSON + only). +- **Protocol errors close the link:** a header failing the schema, a part that is not valid JSON or whose wire ops fail + Delta's checks, an id reused while live, traffic before or a second `hello`. These are structural checks of the wire, + never checks of application state (state is never validated). +- **Errors on the wire** are a tagged union: `{ kind: "chord", error }` (a `ChordError`'s JSON, L1) or `{ kind: + "plugin", code, message, details? }` (open point 3). Never a stack or a cause. + +## 2b. Codecs, the `Message` type and checks (owner, 2026-10-22, after the code review) + +Replaces the JSON-header framing of section 2 (`| u32 header length | header JSON | parts |`); the message kinds, ids, +races and rules of section 2 stay. + +- **`Message`** is a tagged union defined once as a TypeBox schema (`MESSAGE`; `type Message = Static`): `hello { protocol, incarnation }`, `hold { hold, type, version, fingerprint, params }`, `release { hold + }`, `call { call, hold, method, args? }`, `cancel { call }`, `bound { hold, variant, members: [{ member, value }], + failed: [{ member, error }] }`, `state { holds: [{ hold, members: [{ member, ops }], failed: [...] }] }`, `failed { + hold, cause }`, `result { call, value? }`, `error { call, error }`, `gone { hold, error }`. Values (`params`, `value`, + `ops`, arguments) are `unknown()` in the schema. `args` absent: a method without arguments; `value` absent: `null`. + A payload that may carry bytes is marked so by the protocol layer (decided once per method from its schema). +- **Names are indexes:** `member`, `method` and `variant` are indexes into the **hold's bound version's** names, + sorted per variant. Both sides hold that version's description (version and fingerprint matched at `hold`). An index + is valid only while its hold is bound to that version; a provider that stops accepting a version ends that version's + holds with `gone` (contract_mismatch) before its new code serves anything (slice 7 builds the reload side; carry-over). +- **`Codec { name, encode(message): Uint8Array, decode(bytes): Message }`**, chosen per link + (`host.accept(link, { principal, codec? }, context)`, `host.connect(...)`, default `binaryCodec`); both ends use the + same; a mismatch fails the first `hello` (a protocol error naming the expected codec). No negotiation. + - **`binaryCodec`:** a kind byte, then that kind's fields as unsigned varints (ids, indexes, counts, lengths), + length-prefixed UTF-8 strings, the incarnation as 16 raw bytes and the fingerprint as 8 raw bytes; values as varint + length plus JSON text; byte segments (payloads marked as carrying bytes) as a count, lengths, raw bytes, referenced + from the JSON by `{"$bytes": i}`. No outer length (links deliver whole messages). + - **`textCodec`:** one JSON tuple per message, Delta style (a short kind tag, positional fields, no field names): + `["h", protocol, incarnation]`, `["H", hold, type, version, fingerprint, params]`, `["R", hold]`, `["c", call, hold, + method, args?]`, `["x", call]`, `["b", hold, variant, [[member, value]…], [[member, error]…]?]`, `["s", [[hold, + [[member, ops]…], [[member, error]…]?]…]]`, `["f", hold, "t"|"b", error]`, `["r", call, value?]`, `["e", call, + error]`, `["g", hold, error]`; byte segments follow the tuple (their lengths in it). + - `describeMessage(message)` prints any message readably (the text tuple form) for logs and tests. +- **Who checks what** (layers 1-3 failing is a protocol error that closes the link): + + | Layer | Checks | + | --- | --- | + | 1. codec `decode` | lengths and varints fit, the buffer consumed exactly, a known kind, strict UTF-8, each JSON value parses, segment counts and references valid and only in payloads with bytes | + | 2. `MESSAGE` schema (protocol layer, after any codec) | the envelope: kinds, ids and indexes are integers, lists and error shapes; not the values | + | 3. the hold's bound version (protocol layer) | ids not reused while live (unknown ids ignored), indexes within the bound version's lists, order (`hello` first, `bound` before `state`) | + | 4. owners | `params`: `checkParams` (a broken `failed`, link stays); arguments: `checkArgs` (`error { call, invalid/args }`, link stays); ops: Delta's decoder and `applyImmutable` before anything is adopted (protocol error); errors: `chordErrorFromJson` (protocol error); snapshots and results: never validated (R9) | + +- **Memory link:** delivers with `queueMicrotask` (portable, asynchronous, no reentrancy); no Node-specific APIs. + +## 3. Remote objects (§4, §5, §6) + +- `host.serve(remoteContract, (params, principal, self) => ({ variant, members }))`: the factory returns one variant of + the contract with exactly that variant's members. L1's `Implementation` for a remote contract becomes the tagged + union `{ [V in Variants]: { variant: V; members: VariantImplementation } }[Variants]` (an L1 change); a local + contract's stays its members. An unknown variant, or missing or extra members of the chosen variant, fails the + activation broken (the member check, slice 1). +- Identity: `(type, params key, principal key)` (L1 `identityKey`). +- **Calls are checked against the binding's variant:** a call whose method is not in that variant fails + `invalid/member` before anything else (a reader cannot call `add` by forging a message); then L1's `checkArgs` checks + the arguments with the current version. +- **Contract match (R6):** a `hold` binds only when its version and fingerprint equal the provider's current ones + (`accepts` is slice 7); otherwise `failed { hold, cause: broken contract_mismatch }`. +- **A hold's activation** follows slice 2 on the provider: a broken failure is sent as `failed { hold, cause }` and is + final (the provider releases the hold); a transient one is sent as `failed { hold, cause }` with the hold kept + pending while the object retries; going live sends `bound`. The provider has no deadline of its own for a hold: the + consumer's getter deadline decides (it releases the hold when it gives up). A hold arriving before `load` resolved + waits for it; one arriving while the previous instance of its identity tears down waits for that (slice 2). + +## 4. References over a link (§7) + +- `host.get(Remote, params, ctx, { readyTimeout? })` sends `hold` and resolves with a live, hydrated reference on + `bound`. A `failed` with a broken cause rejects at once; a transient one is remembered while the get keeps waiting; + at its deadline (or cancellation) the get sends `release` and rejects with the last cause's error (or `unavailable` + if none arrived). `self.get(Remote)` follows slice 2 (R18): its object waits for `bound`, and a broken or transient + `failed` fails it as a failed dependency. +- `ref.variant` is the variant's name and the discriminant TypeScript narrows on; a local reference has no `variant`. +- **Replicas:** each state member of a remote reference is a watchable on the consumer, built from the snapshot and + then by Delta's `applyImmutable` of its ops (no validation, no copy). One `state` message is applied as **one batch** + of the consumer host, for all its holds, before any watch frame, so no consumer sees half a provider commit (W1). +- `ref.method(args, ctx)` (or `ref.method(ctx)` without arguments) sends `call` and resolves with its `result`, or + rejects with its `error`. A call before `bound` cannot be made (the reference does not exist yet); a forged one is + answered `error { call, not_ready }`. The caller's cancellation sends `cancel` and rejects the caller at once; the + provider aborts that call's context. +- **Release** (open point 1): `ref.release()` rejects the pending calls made through that reference with `released`, + sends `release`, and the provider aborts the handler contexts of that hold's calls and unholds (an object with params + ends with its last reference, local or remote). The same rule applies to local references (a change from slices + 1-2, where a release left its pending calls running). +- `gone { hold, error }` (the provider object ended): the reference ends (`released`, watches `ended`) and its pending + calls reject with the error; no per-call errors are sent. +- Two references to one object from one host are two holds, each with its own encoder (two tabs are two clients). + +## 5. The provider side: one message per link per commit (W1) + +- A hold attaches to its object's state members as a **synchronous subscriber**: `Source` gets a second set of + subscribers besides its watches, fed `(value, ops)` inside the batch's delivery; a hold counts as watching, so exposed + derives recompute in the batch. Taking the snapshot and attaching happen in one synchronous step, so the first + `state` part continues exactly from the snapshot. +- During a batch each touched hold collects its members' ops; at the **end of the batch** (a hook `Batch.commit` runs + after its delivery loop) each touched hold encodes its collected ops **once** into its part, and each link with + touched holds sends **one** `state` message. A link that closed meanwhile drops it. +- A hold whose object ends sends `gone`; a failed member is announced once in the next `state` (or the `bound`). + +## 6. `ctx` (§8) + +`ctx.principal` (the stamped caller; on a local handler the host's), `ctx.incarnation` (the calling host's over a link, +the own host's otherwise, R20), `ctx.get(Type, params)` (a reference released when the **handler** settles; a +`ctx.get` after that throws `released`), the abort signal and telemetry. + +## Races + +1. A `result` crossing a `cancel`; a `call` crossing `gone`; a `state` crossing a `release`; a `bound` crossing a + `release` (the consumer ignores a `bound` for a hold it released): ignored by id (Q1). +2. A commit between binding a hold and sending `bound`: the snapshot and the subscription are taken in one synchronous + step. +3. A forged call before `bound`: `error { call, not_ready }` (parking is slice 3b). +4. The provider object ends with calls pending: `gone`; the consumer rejects those calls itself. +5. The consumer releases with calls pending: rejected locally `released`; the provider aborts the hold's calls. +6. `release` while the provider is still activating for that hold: unhold (may end the object, slice 2); `bound` is + never sent. +7. A `hold` while the previous instance of its identity tears down: waits (slice 2's endings). +8. One object held over two links, one link closes: only that link's holds are released; the other keeps updating. +9. A link closing during a batch: its `state` message is dropped; holds and calls on it end (section 1). +10. A handshake that never completes: nothing else is accepted; a get over it waits up to its deadline. +11. A `ctx.get` finishing after its handler settled: the late reference is released at once. +12. A failed member and healthy members of several holds in one batch: each hold's part carries its healthy ops, its + header the failure. + +## Benchmarks (owner: light benchmarks for memory, timing and wire size) + +A script `packages/chord/test/next/bench/links.ts` (run by hand, not in CI) prints one table **per codec, side by +side** (owner: both codecs, memory and compute); results are recorded in `packages/chord/docs/next/benchmarks.md` for +later slices to compare against. Per case and codec: wire bytes per message; compute (encode and decode time per +message over many messages, and commit-to-frame time end to end); memory (heap allocated per message while encoding and +decoding 10,000 messages, measured with `--expose-gc`, and memory retained per hold). Payloads modelled on p2: + +| Case | Measured | +| --- | --- | +| hold of a notes list of 1,000 items; of a 500 KB document | snapshot encode/decode time, wire bytes | +| one appended item, one changed field, one removed item | commit-to-frame time on the consumer, wire bytes per message, encodes per hold | +| a streamed reply: 2,000 small text appends to one entry | time and bytes per append, total bytes | +| 1 and 10 holds of one object on one link | memory per hold (encoder and replica), messages per commit | +| a call with a small argument; with a 1 MiB `bytes()` argument | round trip, wire bytes (about 1 MiB, not 1.33) | + +## Carry-over taken by this slice + +L6: `PROTOCOL_VERSION` in the handshake; one message per link per commit; bytes as raw segments; a malformed error +envelope closes the link; the plugin error shape. L5: `ctx.get`, `ctx.principal`. Moved to 3b in the feature map: +`host.root`, C8 (`state: false`). + +## Tests + +- Messages: every kind round-trips; parts with byte segments; `$bytes` refused in methods with bytes and plain data elsewhere; each protocol error closes + the link; unknown ids ignored; id reuse refused. +- Handshake: traffic before `hello`, a second `hello`, protocol mismatch; incarnations stamped. +- Remote objects: variant per principal; identity per principal key; a call outside the binding's variant fails + `invalid/member` (forged at the wire, not just in types); arguments checked; contract mismatch; invalid variant or + members fail activation broken; remote objects with empty params are not activated by `load`. +- Principals: the four same-host cases of section 1. +- References: hydrated on get; broken `failed` rejects at once; transient `failed` keeps waiting and the deadline + rejects with the last cause and releases the hold; replicas apply one message as one batch (a consumer derive over two + members never sees half a commit); calls, results, cancel, errors of both kinds; release rejects pending calls (local + and remote); `gone`. +- One message per commit: three holds on one link, one message; a derive with its source; two links, two messages. +- Every race above. +- Cost guards: one encode per hold part; no validation or copy of state on either side. +- GC: a closed link and a released reference, **retained** by the test, hold no holds, encoders, replicas or objects. +- Examples: `08-remote.ts` (two hosts, variants per principal), `09-calls.ts` (calls, results, errors, cancellation, + release). + +## Decided with the owner (2026-10-22) + +Slice 3 is split into 3a and 3b; a remote factory returns `{ variant, members }`; `ref.variant` is the discriminant +(C15: no `inVariant`/`expectVariant`; `variant` reserved as a member name); headers are checked on every link for now. +Release rejects the calls made through the released reference, locally and over links (their handlers' contexts are +aborted). `$bytes` is reserved in the arguments and results of methods whose schema has a `bytes()` leaf (encoding +refuses it with `invalid/args`); elsewhere it is plain data, since only parts with byte segments resolve references +(owner, 2026-10-22, narrowing the first decision). Plugin errors: `new PluginError(code, message, details?)`; any other thrown value crosses +as `{ kind: "plugin", code: "internal", message }`; only code, message and details cross; same-host callers get the +thrown value; the consumer rethrows a `PluginError`. + +## Size + +About 1,500-2,000 source lines (both reviewers), including the L1 change and the batch's subscriber hook. diff --git a/packages/chord/docs/next/slice-3b.md b/packages/chord/docs/next/slice-3b.md new file mode 100644 index 000000000..f4f79d7c9 --- /dev/null +++ b/packages/chord/docs/next/slice-3b.md @@ -0,0 +1,197 @@ +# Slice 3b: surviving drops (spec, for the owner's approval) + +Status: **approved (owner, 2026-10-22)**, after Astra's and Fable's spec reviews. Guiding rule (owner): simple, fast, +correct. The single source for building 3b, together with the feature map in `slices.md` (entries assigned to 3b), +`design.md` read in full (most relevant: §3 links, recovery and timeouts, §4, §6, §7 references, §8, §19, §22), the +decision log, the carry-over (L4, L5, L6) and the root `AGENTS.md` style section. Builds on L1 and slices 1, 2 and 3a +(`76b32ed2d`). + +## What an app can do after this slice + +```ts +const notes = host.hold(Notes, { session: "S1" }, ctx); // a reference at once, not awaited +notes.state.value; // { kind: "loading" } +const w = await notes.state.watch(ctx); // the current state, then every change +w.start(async (state) => render(state)); // loading → live → loading (link dropped) → live … + +await notes.add({ text: "milk" }, ctx); // parks while loading, sent once live +if (notes.state.value.kind === "failed") notes.retry(); // a Retry button + +client.connect(newLink, { principal: { kind: "server" } }, ctx); // after a drop: every reference re-holds + +host.serve(Notes, factory, { linger: 5 * 60_000 }); // a just-released object stays warm for 5 minutes +``` + +## 1. Reference state (§7) + +- Every reference has `ref.state`, a `NativeWatchable` (in-process: it holds errors; open point 1): + + ```ts + type RefState = + | { kind: "loading" } + | { kind: "live" } + | { kind: "failed"; cause: FailureCause; retryAt: number | undefined } // undefined: broken, no automatic retry + | { kind: "released" }; + ``` + + `ref.state.watch(ctx)` delivers the current state, then every change. On release the state becomes `released` and + the watch ends `ended` (the end is the notification; a native watch drops undelivered frames when it ends). +- **One rule for both sides:** a reference is `loading` while its object is activating or retrying, while it waits for + `load` or a parent link, and while its link is down; `failed` while its object is failed and no attempt runs; `live` + when bound; `released` once released. `variant` is `undefined` before the first binding and keeps the last bound + variant while `loading` or `failed`. + + | State | Calls | State members | + | --- | --- | --- | + | `loading` | park (section 3) | readable: the last value; before the first binding a read throws `not_ready` and a `watch` waits for the first value | + | `live` | run | readable | + | `failed` | reject at once with the cause's error | reads throw the cause's error; existing watches stay open | + | `released` | throw `released` | reads throw `released`; watches end `ended` | + +- Member watches survive `loading` and `failed` (reference states): they get the next frame when the reference is live + again. They end only when the reference is released, the member itself fails (W1, `failed`), or the member is not in a + newly bound variant (`ended`). +- **Retry** (open point 2): `ref.retry()` asks the object for an attempt now, skipping the backoff window and renewing a + broken object's readiness (carry-over L5); one attempt runs at a time, so a retry during one is ignored. Local: the + object's `attempt()`. Remote: a new message `retry { hold }`; the provider does the same for the held object, for all + its holders. On `live` and `released` references `retry()` does nothing; without an open parent link the next + `connect` re-holds anyway. +- **Attempt notice** (open point 2): when a failed object starts a new attempt (backoff, retry, or a dependency going + live), its holds get `loading { hold }`, so remote references move `failed → loading` and park calls exactly like local + ones. `failed { hold, cause, retryIn? }` carries the remaining delay of the provider's window (`until - now`, from a + new `until` on slice 2's backoff entry, carry-over L4) as advice for `retryAt`; the attempt itself is announced by + `loading`. + +## 2. Acquiring: `hold` and `get` (§7) + +| Situation | `host.hold(Type, params, ctx, options?)` | `host.get(…)` | +| --- | --- | --- | +| invalid params, a local type not served here, no principal for a same-host remote type | throws at once (made before `load`: `failed` broken once `load` resolved, since the types are known only then) | rejects at once | +| before `load` resolved, or no parent link yet (remote) | `loading` | waits | +| the object activates | `loading` | waits | +| a transient failure | `failed` (with `retryAt`), then `loading` on the next attempt | waits; at its deadline it releases the reference and rejects with the last cause | +| a broken failure | `failed` | releases the reference and rejects at once with the cause | +| the link drops before the first binding | `loading`, re-held on the next `connect` | keeps waiting within its one deadline | +| live | `live` | resolves | +| the caller's context is aborted | (no effect on a `hold`) | releases the reference and rejects | + +A `hold` follows its object until released and never rejects. A `get` has one deadline (`readyTimeout`, per get) from +the moment it is called, across reconnects. + +## 3. Parking (§3, §7) + +- A call made on a `loading` reference is queued with its own deadline, measured from the moment it was called (a + reconnect never renews it). When the reference goes `live`, queued calls are **sent** in the order they were made, + before any later call (a new call cannot overtake queued ones); their completions may arrive in any order. When the + reference goes `failed` they reject with the cause; at its deadline a call rejects `unavailable/park_timeout`; the + caller's cancellation rejects it at once; release rejects them `released`. +- A queued call is resolved against the binding it is sent on: after a reconnect that bound another variant, a method + the new variant lacks rejects `invalid/member`; indexes are taken from the new binding. +- Calls already **sent** when the link drops fail `unavailable` (never resent, §7). +- **The park deadline is `readyTimeout`** (owner, 2026-10-22; C13's two host timeouts stay): a queued call waits at most + the reference's `readyTimeout` (host default 60 s, per `get`/`hold` override), then rejects `unavailable/park_timeout`. + +## 4. Link drops and re-holds (§3 recovery, §4, §22) + +- **One owner of remote objects across links:** the host keeps its remote references (their objects) independent of any + one link; each `connect` creates a link with fresh id counters (ids are link-local) and re-holds every non-released + remote reference on it; a closed link's tables are dropped, and nothing of an old link (a late callback) touches the + new one. +- **On a drop:** every non-released remote reference goes `loading` (also a `failed` one), sent calls fail + `unavailable`, new calls queue. `host.connect(link, …)` may be called again once the previous parent link has closed + (open point 6); while one is open it throws. +- **The re-hold:** a new `hold` (new id); on its `bound`, in **one batch** of this host and before any queued call is + sent: each member of the bound variant gets the snapshot value with one frame `[["r", value]]` and a fresh Delta + decoder (open point 4); members not in the bound variant end (`ended`, derives over them fail); new members get their + first value; failed members listed in `bound` fail (W1); `variant` is updated. A different variant is the normal result + of a changed grant (design §4: reconnect, authenticate anew, re-hold). Then queued calls are sent. +- **The provider side** is unchanged: a closed link releases its holds (objects with params end with their last + reference unless they linger, section 5); a reconnecting child is a new `accept` with new holds. +- **References a host's objects hold** (`self.get` of a remote type) go `loading` the same way; a **live** dependent + stays live (R18: a dependency leaving `live` does not change it) and its calls through them queue. An **activating** + dependent keeps waiting for the dependency; its own `readyTimeout` bounds it (slice 2), and R18 retries it on `bound`. +- Who reconnects is the app (its link adapter calls `connect` again); Chord only re-holds. + +## 5. `linger` (§3, §6, §19, D8, C7) + +`host.serve(Type, factory, { linger })` (milliseconds, default 0): a **live** object with params whose last reference +is released stays live for `linger`, keeping its own dependencies; a `get`, `hold` or arriving `hold` message within +that time reuses it; otherwise it ends. An object that is waiting, activating or failed when its last reference goes +ends at once (slice 2). A failed state member is not a failed object and does not end a lingering one. `host.close` +ends lingering objects at once. Host-lifetime objects (`local`, empty params) never end by references; a remote type +with empty params lingers like any other. + +## Races (listed before sizing) + +1. A sent call when the link drops: fails `unavailable`, never resent. +2. A call made while `loading`: queued with its deadline; sent once live, in order; `park_timeout`; cancellation; + release. +3. A reference released while `loading` (no link): nothing sent; not re-held; queued calls reject `released`. +4. `connect` while the previous parent link has not finished closing: throws. +5. A drop during a re-hold's handshake or before its `bound`: stays `loading`; the next `connect` re-holds again. +6. The re-hold finds the object failed: `failed` with `retryIn`; the provider's next attempt sends `loading`, then + `bound` when live. +7. A reconnect creates a fresh object (the old one ended without linger): new snapshot; the old in-memory documents are + gone (documented). +8. A get or an arriving `hold` within `linger` versus the linger timer: a reuse cancels the timer; the holder count + decides, so an expiry never ends an object that has a holder again; no grace period. +9. A failing object without references ends at once, never lingers; the last release during activation ends it. +10. `host.close` with queued calls and loading references: queued calls reject `released`, references go `released`. +11. `ref.retry()` during an attempt or a pending re-hold: ignored. +12. Shared remote retry: two consumers hold one failed object; one retries; both get `loading`, then `bound` or + `failed`. +13. A broken object held by another consumer: a `retry` renews it for everyone. +14. A reconnect binds another variant (a changed grant): replicas rebuilt in one batch; vanished members' watches end; + queued calls checked against the new variant. +15. A release or a call's deadline while a snapshot is being adopted or queued calls are being sent: the reference's + state decides at each step; nothing is sent for a released reference. +16. A member `watch` before the first binding: waits; rejects on `failed` (cause) or release. +17. A re-hold arrives while the previous instance (released by the drop, no linger) still tears down: waits for that + ending (slice 2), then a fresh object. +18. An activating dependent whose remote dependency's link drops: it fails transient at the drop (the dependency's + pending outcome rejects `unavailable`); R18 retries it when the dependency is bound again. +19. A drop while `failed`: `loading`, re-held on the next `connect`. + +## Cost per hop + +Nothing changes for live references (calls, commits, frames as 3a). A drop costs one `hold` and one snapshot per +reference on reconnect; a queued call holds its arguments until sent; `ref.state` is created lazily on first access. + +## Benchmarks + +Extend `test/next/bench/links.ts` (both codecs): reconnect with 1 and 100 references (time to all live, bytes), 1,000 +queued calls flushed on reconnect (time, order), and the live paths of 3a unchanged (regression rows). + +## Carry-over taken by this slice + +L4: `retryAt` (backoff `until`). L5: `ref.retry()` renewing a broken object's readiness; calls on a failed reference +reject with its cause; C8. L6: `linger`, `ref.state`, `ref.retry()`, `host.hold`. Moved out: a second `bound` as a reset +(slice 4 for `reset` after overflow, slice 7 for restarted objects); the park deadline's per-type override is not +built. + +## Tests + +- `ref.state`: every transition for local and remote references, watched (first value, each change, the end on + release); `variant` before, during and after bindings. +- `hold` and `get`: every row of the table in section 2. +- Parking: queue order, one deadline across reconnects, cancellation, release, failure, a variant change. +- Drops and re-holds: references go loading; sent calls fail; reconnect re-holds with fresh ids; one batch per bound + with `[["r", value]]` frames; another variant on reconnect; old-link callbacks never touch the new link. +- Retry: local and remote; shared across consumers; broken renewed; ignored during an attempt; `loading` notices. +- `linger`: reuse within the window, end after it, only live objects linger, failed members do not end it, + `host.close`, reconnect within linger keeps in-memory documents. +- Every race above, both codecs where a link is involved. +- GC: a released `hold` reference, a reference across a drop, a closed link's tables, a lingering object after expiry. +- Examples: `10-drops.ts` (drop, park, reconnect, re-hold, `ref.state`), `11-linger-and-retry.ts`. + +## Decided with the owner (2026-10-22) + +`ref.state` is a `NativeWatchable` (the `NativeWatchable` interface joins `Watchable` in `src/watch.ts`, +upstream first); remote retries use two new messages, `retry { hold }` and `loading { hold }`; parked calls wait at most +`readyTimeout` (no third host timeout); a re-hold replaces each member with `[["r", value]]`; C8 (`state: false`) is +cut; `connect` may be called again once the previous parent link has closed; `host.root` moves to slice 8. + +## Size + +About 1,100-1,400 source lines (reference state, acquisition, parking with its queue, re-holds with variant changes, +retry and loading notices, `linger`). diff --git a/packages/chord/docs/next/slice-4.md b/packages/chord/docs/next/slice-4.md new file mode 100644 index 000000000..f82525ff2 --- /dev/null +++ b/packages/chord/docs/next/slice-4.md @@ -0,0 +1,146 @@ +# Slice 4: the topology (draft: scope, structural changes, races) + +Status: **overview** (owner, 2026-10-23: slice 4 is split into 4a and 4b). The 4a spec proper is `slice-4a.md`, which +wins where the two differ; 4b is sketched at the end of this file. Reads with `design.md` (§3 hosts, links, routes, flow control; §4 principals and +revocation; §6 cycles; §7; §19 presence; §22 wire), the decision log (R1, R2, Q1-Q5, Q13, R20, "Flow control: an +ack-clocked window", "One pending queue for slow consumers of state", "Compression at the origin", "Wire protocol"), +the carry-over (L6, L7) and slices 3a/3b. Builds on `9066b343c`. + +## The split + +| Slice | Contents | +| --- | --- | +| **4a: links and relays** | IPC and WebSocket links; trusted links; parts as opaque bytes; flow control (acks, pending queue, call window); host kinds, addresses, `placement`; `route` (`authorize`, `resolve`, re-resolve with backoff); relayed holds and calls (ids mapped, principal and origin incarnation stamped, per-hop acks, `reset` and a second `bound`) | +| **4b: authority and housekeeping** | revocation (`revokedBy`, `host.revoke`, propagation to children); cycle checks across hosts (activation chain in hold headers); `host.references` (origin ids); `host.activity`; size limits per part (the admitting link's decoded limit stamped on relayed work); compression at the origin | + +4a is what the playground needs (server, workers over IPC, tabs over WebSocket). 4b adds header fields on top of 4a's. + +## 4a: structural changes the current code needs + +1. **A link carries both directions.** Slice 3 has `ProviderLink` (`accept`: serve only) and `ConsumerLink` (`connect`: + hold only). With routes, the server holds worker objects over a resolved link *and* serves the worker's holds of + server types over the same link; the worker serves the server's holds over its `connect` link *and* holds server + types over it. One per-link object with both sides (two id spaces: holds this side sent, holds it received; message + kinds are directional, so dispatch stays by kind). +2. **Parts are opaque bytes.** Today the codecs parse every value (`params`, `args`, `value`, `ops`, snapshot members) + at decode. A relay must forward them unparsed (R1), so values become byte parts that only the end that owns them + decodes (the consumer's replica, the provider's `checkArgs`). Exception: a `hold`'s params, which a relay parses to + compute placement and authorize (small; checked by `checkParams` anyway). Sizes then come for free (`byteLength`). +3. **Stamped header fields.** `hold` and `call` gain the origin's principal and incarnation (Q13), set by the relay + from the link the work came in on; honored only on a link from a parent; overwritten on every other link. 4b adds + the activation chain, the origin id and the decoded limit the same way. +4. **`ack`** (consumer to provider, after applying each `bound`/`state` message) and the call window in `hello` + (`calls: N`). +5. **Resets as `r` ops** (decided below): a `state` entry flagged `reset: true` carries `[["r", value]]` per member; + the consumer applies it like any ops (the decoder restarts at `r`). A second `bound` on a live hold stays refused + until slice 7. +6. **`loading` on a live hold**: a relay whose upstream link dropped tells its downstream holds; 3b's consumer moves + only `failed -> loading` on it today. +7. **The pending queue** (fold, leaf index, cap 10,000) per hold member on the provider side, and in Chord's shared + watch handle (pi first; carry-over L6). + +## 4a: races + +Routes and resolve: + +1. Two holds (or a hold and a call's re-hold) for one address while its `resolve` runs: one `resolve` per address, + shared; both holds go out on the link it returns. +2. `resolve` rejects or its signal aborts: the holds stay `loading`; re-resolve with the per-address backoff (1 s + doubling to 60 s) while holds for the address exist. +3. Every hold for an address is released while `resolve` runs: the resolve finishes (a worker start that produced a + link finishes even if its first requester left, §3); the link is kept for the next hold. Who closes an idle worker + link is the app's supervisor (`host.activity`, 4b). +4. `authorize` refuses: the hold fails broken `forbidden` before anything starts; a later hold with another principal + for the same address still resolves. +5. Placement: the caller computes `placement(params)` and routes by it; the provider computes it again and refuses an + object that is not its own (`invalid/placement`, broken). A placement rule changed by a reload between the two is + that refusal (rollout handling is slice 7). +6. A worker restarts (crash, the supervisor stopped it): its link drops; the new link's `hello` carries a new + incarnation; re-holds get fresh objects (in-memory documents gone, as 3b race 7). + +Relays (server between a tab and a worker): + +7. The worker link drops with relayed holds: the server sends `loading { hold }` downstream (tabs go `loading`, + values readable, new calls park on the tab), fails relayed calls in flight with `unavailable` (never resent), + re-resolves, re-holds upstream, and forwards the new `bound` downstream (accepted while `loading`, as 3b's + re-hold; another variant is allowed). Superseded by `slice-4a.md` section 4. +8. A tab call that crossed the `loading` notice reaches the server while the upstream link is down: answered + `unavailable` (as a call sent before a drop, §7), not parked at the relay. +9. The tab releases a hold while its upstream re-hold or `resolve` is pending: the relay releases upstream once the + upstream hold exists; nothing is sent downstream after the release (Q1 covers a `bound` crossing it). +10. The tab link closes: the relay releases every upstream hold it made for that tab and cancels its relayed calls. +11. Two tabs of one principal hold the same object: two upstream holds, each with its own encoder (R1, as today). +12. A worker sends `gone` for a relayed hold: forwarded downstream; the relay forgets the mapping. +13. A worker holds a server type (or another worker's): it goes over its `connect` link; the server serves it or + routes it on (worker to worker through the server, §3). + +Flow control: + +14. An `ack` crossing a `reset`, a `release` or a link close: acks count messages per link, not holds; an ack for a + message whose hold was released still frees its slot. +15. A `reset { hold }` reaching the provider while the hold is pending, failed or released: ignored unless bound + (Q1). +16. A relay with a full window towards a tab: it drops new parts for that tab's holds and marks them; when the tab + acks, it sends `reset { hold }` upstream for each marked hold and drops further parts for a hold until its new + `bound` arrives. +17. **Torn resets:** one commit touched holds A and B of a tab; both were dropped and reset; the two new `bound`s + arrive as two messages, so a derive over A and B could see A new and B old in between. Proposal: `reset` carries + a list of holds and the provider answers with one message holding all their snapshots. +18. The pending queue reaches its cap while the window is full: only "replace with the current value" stays; when a + slot frees, the provider sends a snapshot of the member (a second `bound` part or an `r` op; see open point 3). +19. Calls: more than N calls from one link are parked by the caller (one FIFO per link, across references); a + cancel frees the slot at the caller when sent and at the provider when received (per-direction order keeps the + counts equal). +20. **Call fairness at the relay:** each tab has N calls towards the server, but the server has one window of N + towards a worker for all tabs and its own calls. Proposal: the relay parks relayed calls per origin and sends + round-robin, so one flooding tab cannot occupy the worker's window; the relay holds at most tabs x N parked calls. + +Links and transports: + +21. IPC: `send` while the child is exiting, `disconnect` racing a message in flight: the link closes once; messages + after close are dropped; `process.send` errors close the link. +22. WebSocket: a frame above the transport's `maxPayload` closes the connection (code 1009); a client that stops + reading is bounded by the window (2 messages) plus the pending queue cap, not by the socket buffer. +23. Trusted links skip the message schema check (IPC, carry-over L6) but never grant stamping: stamped fields are + honored only from the link to the parent. + +## Decided (owner, 2026-10-23) + +1. **Trusted links are an explicit option:** `accept(link, { principal, trusted: true })`, and `resolve` may return + `{ link, principal, trusted }`; default untrusted. Trusted skips the message schema check; stamping is still + honored only from the parent link. +2. **Resets and overflow are `r` ops in an ordinary `state` message** (races 16-18): a relay's `reset { holds }` + (a list) is answered with one `state` message whose entries carry `[["r", value]]` for every member of each reset + hold that is still bound, all snapshots taken in one synchronous step, applied by the consumer in one batch (no + torn commit across the reset holds). An overflowed pending queue sends `[["r", value]]` the same way. Delta's + encoder and decoder restart at an `r`. A state entry gets the header flag `reset: true`, so a relay (which never + reads parts) knows when to stop dropping a reset hold's parts. A second `bound` stays only for slice 7 (a restarted + object may choose another variant). +3. **Call fairness: round-robin per origin at the relay** (race 20): each origin's calls keep their order; while the + worker's window is full, origins take turns, so one flooding tab cannot starve the others. +4. **The Durable conversation view:** known cost until Durable's windowed view (pi #11327, waiting for its reporter) + lands: a re-hold sends the whole active transcript. + +## 4b: scope and races (sketch) + +- **Revocation:** `createHost({ revokedBy })`; `host.revoke(r)` ends every object and closes every link whose + principal has `r` in `revokedBy` (L1's `revocationKeys`), then propagates `r` to children (a `revoke` message on + trusted child links). Races: a hold stamped with a revoked principal already on its way to a worker (it binds, then + its link closes: R10's window); revoke during an activation; revoke while a relayed call runs (its handler is + aborted; R11 jobs later). +- **Cycles across hosts (Q5):** a hold made during an activation carries the chain of activating instance ids + (bounded); the provider refuses a hold whose chain contains the instance being activated (`invalid/cycle`); chains + from untrusted links are ignored. Races: the chain crossing a relay (copied, never extended by the relay); a cycle + through a host that restarts. +- **`host.references(Type)`** (presence, §19): who holds objects of a type, with an origin id per client (stamped by + the relay, so two tabs are two clients); a link drop is a departure. +- **`host.activity`**: `{ references, calls, jobs, lingering }` as one readable for the app's keep-alive. +- **Size limits per part (R2, Q2)**: checked from the header before decoding; an oversized part fails only its hold or + call (`too_large`); the admitting link's decoded limit is stamped on relayed work and enforced where it is decoded. +- **Compression at the origin**: a part above a threshold is compressed once by its origin (codec field in the + header); relays forward it compressed; only the end that decodes inflates it, with the decoded limit. + +## Size + +Reviewers' estimate for slice 4 as a whole: 2,500-3,000 source lines; 4a carries most of it (the two-sided link, the +relay, transports, flow control). diff --git a/packages/chord/docs/next/slice-4a.md b/packages/chord/docs/next/slice-4a.md new file mode 100644 index 000000000..8bcf15562 --- /dev/null +++ b/packages/chord/docs/next/slice-4a.md @@ -0,0 +1,418 @@ +# Slice 4a: links and relays (spec, for the owner's approval) + +Status: **approved (owner, 2026-10-24)**, revised after Astra's and Fable's spec reviews (`/tmp/s4a-review-*.md`); +sections 4-6 and their races revised for the decision "Relay flow control per origin" (owner, 2026-10-24), which +replaces per-hop acks, `reset { holds }`, relay recovery and id renumbering. +Guiding rule (owner): simple, fast, correct. The single source for building 4a, together with the feature map in +`slices.md` (entries assigned to 4a), `design.md` read in full (most relevant: §3 hosts, links, routes, flow control, +recovery; §4; §7; §18; §22), the decision log (R1, Q1, Q4 and "Wire protocol", Q13, R20, "Flow control: an +ack-clocked window", "One pending queue for slow consumers of state", "Slice 4 split and first decisions", "Slice 4a +spec decisions", "Slice 4a review decisions", "Relay flow control per origin"), the carry-over (L6, L7, 4b), `slice-4.md` (the split and 4b) and the +root `AGENTS.md` style section. Builds on L1 and slices 1-3b (`9066b343c`). + +## What an app can do after this slice + +```ts +export const Server = hostKind("server"); +export const Session = hostKind("session", { id: str() }); // params like a contract's + +export const Notes = remote("notes", { + version: 1, + params: { session: str() }, + placement: ({ session }) => Session({ id: session }), // never part of the fingerprint + variants: { … }, +}); + +// server.ts +const server = createHost

({ address: Server(), principal: { kind: "server" }, onError }); +server.route(Session, { + authorize: (principal, address) => principal.kind === "owner" || (principal.kind === "guest" && principal.session === address.params.id), + resolve: async (address, context) => { + const child = fork("worker.ts", [], { serialization: "advanced" }); // no `signal`: see below + child.send({ id: address.params.id, init: initFor(address.params.id) }); // the app's own start message + return { link: ipcLink(child), principal: { kind: "worker", session: address.params.id }, trusted: true }; + }, +}); +await server.load(serverFacets, ctx); +wss.on("connection", (ws, req) => server.accept(webSocketLink(ws), { principal: authenticate(req) }, ctx)); + +// worker.ts: connect first (W6), then load; start data through the core facet (§18) +const { id, init } = await startMessage(process); +const worker = createHost

({ address: Session({ id }), principal: { kind: "worker", session: id }, onError }); +worker.connect(ipcLink(process), { principal: { kind: "server" }, trusted: true }, ctx); +await worker.load([{ name: "@app/core", facet: coreWorkerFacet(init) }, ...workerFacets], ctx); // serves SessionInfo + +// a tab +const tab = createHost

({ onError }); +tab.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }, ctx); +const notes = await tab.get(Notes, { session: "S1" }, ctx); // tab → server → route(Session) → worker +``` + +Do not pass `context.abortSignal` to `fork`: `host.close` aborts it, which kills the worker and makes Node emit an +`error` event on the child process that crashes an app without a handler. Chord closes the link at `host.close`; the +worker exits when its parent link closes. `resolve` returns the link before the worker has loaded (the worker connects first, W6): a `resolve` that waited for +the worker's `load` while the worker's facets hold server types would deadlock. + +## 1. Host kinds, addresses, placement (§3) + +- `hostKind(name, params?)` declares a kind of addressable host; `params` follow L1's params policy (a flat record of + required `str()`, `int()`, `bool()`, `oneOf(...)`), checked with L1's compiled params check. `Session({ id: "S1" })` is + an **address**: `{ kind, params, key }`, its key built like an identity key (`session|S1`). Two addresses are equal + when their keys are. No host-name strings in app code. +- `createHost({ address })`: an addressable host. Without an address a host is a client (§3 roles). A host without a + principal serves only local types: `serve` of a remote type makes `load` throw (owner, 2026-10-24; a remote object + is owned by the serving host's principal). +- `remote(name, { …, placement })` (an L1 change): `placement(params) => Address`, a typed function; not part of the + fingerprint (L1 normal form unchanged). `local` contracts have no placement. +- **The caller computes the address** and sends it in the `hold` (section 5). A host that relays never needs the + contract: it routes by the address alone. +- **Where a reference or an arriving hold goes** (first match): + + | Case | Goes | + | --- | --- | + | the type is served here and its placement (computed here from the params) is this host's address, or it has none | here | + | a reference made here (`host.get`/`hold`, `self.get`, `ctx.get`) to a served type whose address is another host's | on, as if not served here (next rows): worker S1 reaches `Notes { session: "S2" }` through the parent | + | an arriving hold for a served type whose params place it on another host | broken `invalid/placement` (misrouted; never forwarded); the address the hold names only routes it | + | the hold names an address of a kind this host has a route for | over that route's link (the address's params checked against the kind; `authorize` first) | + | otherwise (a remote type) | over the parent link; a hold that arrived *on* the parent link is never sent back to it: `unavailable` | + | no parent link | `unavailable` (as 3a) | + +- `placement` or `authorize` throwing fails the hold broken with that error; an address whose params fail the kind's + check is broken `invalid/placement`. + +## 2. Routes (§3) + +```ts +host.route(Session, { + authorize(principal: P, address: Address): boolean; + resolve(address: Address, context: Context): Promise<{ link: Link; principal: P; trusted?: boolean }>; +}); +``` + +- `route` is called by base code before `load`; one route per kind (a second throws). +- **`authorize`** runs for every hold routed over the route, with the hold's principal (section 4), before anything + starts. `false` fails the hold broken `forbidden`. +- **`resolve`** runs at most once at a time per address: the first hold starts it, later holds wait for the same + promise. The returned link becomes the address's link (section 3, role `route`). `context` is aborted only when the + host closes. A resolve whose holds were all released meanwhile finishes and its link is kept (§3). +- **Failure and re-resolve:** a rejected `resolve`, a returned link that is already closed, or the address's link + closing while holds over it exist, starts the per-address backoff (1 s doubling to 60 s; a backoff map keyed by + address, separate from objects' identity backoff). While waiting, the holds are `loading` (3b). Only a link that + completed its handshake resets the backoff. Without holds for the address, nothing is resolved again. +- A `resolve` that never settles is the app's (it bounds its own `fork`); getters still time out by their deadlines. +- Chord never closes a route link itself except at `host.close`; stopping idle workers is the app's (`host.activity`, + 4b). + +## 3. Links carry both directions (§3, §22) + +- **One link object per link** replaces 3a's `ProviderLink`/`ConsumerLink` pair: one channel (handshake, codec, + checks, heartbeat), a **provider side** (holds and calls it received; ids chosen by the peer), a **consumer side** + (holds and calls this host's own references sent; ids from its own counters). The provider side is one per + **origin** (section 4): the peer's own work, and each downstream link of a peer that relays. Message kinds are + directional, so each goes to its side. Locally served holds and relayed holds are two kinds of entries in a provider + side's table; a relayed hold is never hydrated into a `RemoteObject`. +- Every link has the peer's principal, `trusted`, and a role: + + | Role | Made by | A hold arriving on it acts as | This host holds over it | + | --- | --- | --- | --- | + | parent | `connect` | the hold's `principal`/`incarnation` fields when present, else the link's principal and peer incarnation | types routed to the parent (a worker's holds of server types, everything a client gets) | + | child | `accept` | the link's principal and peer incarnation (fields ignored) | nothing | + | route | `resolve` | the link's principal and peer incarnation (fields ignored) | the address's objects | + +- **Trusted links** (`trusted: true` on `accept`, `connect` and `resolve`'s result; default untrusted) skip only the + `MESSAGE` schema check (carry-over L6): framing, `checkParams`, `checkArgs` and Delta decoding still run. Trust never + grants the principal fields: they are read only on the parent link. +- `host.close` closes every link (children, routes, parent), then ends objects (3a order). +- **Remote objects** (3b's `RemoteObject`) belong to a **path**: the parent, or a route address. Each new link for a + path holds every remote object of that path again (3b's re-hold); a closed link drops its tables and the objects go + `loading`. + +## 4. Principals and relays (§4, §22, R1, Q13, R20) + +**Principals on holds.** A `hold` carries `principal?` and `incarnation?`. The receiver uses them only on its parent +link; everywhere else they are ignored and the link's principal and peer incarnation apply (a tab cannot claim to be +the owner). Calls carry no principal: a call runs as its hold's principal and with its hold's incarnation +(`ctx.principal`, `ctx.incarnation`), so stamping calls separately is not needed (refines R1/Q13 "hold and call"). + +**What a host sends** in those fields when it holds over a route link (towards a child that honours them): + +| The reference was made by | `principal` | `incarnation` | +| --- | --- | --- | +| `host.get` / `host.hold` | the host's own | this host's | +| `ctx.get` in a handler | the handler's `ctx.principal` (the caller's) | the handler's `ctx.incarnation` | +| `self.get` in a remote object | that object's principal | this host's | +| `self.get` in a local object or a facet | the host's own | this host's | +| a relayed hold (below) | the arriving hold's principal | the arriving hold's incarnation | + +So a guest's call on the server whose handler `ctx.get`s a worker type reaches the worker as the guest, and gets the +guest's variant. `RemoteObject` keeps its binding's caller (3b drops it today) and two references with different +principals are two remote objects. + +**Upward is a boundary.** A worker's hold of a server type goes over the worker's parent link; the server ignores +principal fields from a child, so it acts as the worker, whoever the worker's handler serves. A worker that acts for a +user towards the server passes the user as data and the server object decides. Documented. + +**Relays.** A host that receives a hold it cannot serve (section 1's table: a route or the parent) relays it: it holds +it again on that link (the **upstream** link) with the arriving hold's principal and incarnation, and forwards between +the two without parsing payloads (decision "Relay flow control per origin"). + +- **Origins.** The relay gives each downstream origin (a downstream link, and the origin its messages carry when that + link is itself a relay's) a number, one counter per host, never reused; 0 is a host's own work. Every message + upstream for that origin carries `origin` (section 5); the provider keys holds, calls, windows and acks by (link, + origin, id), and its answers carry the origin back. `origin` is honoured on every link (a worker may relay a + child's hold to its parent); principals and incarnations stay parent-only. +- **Ids are not renumbered:** an upstream hold has the downstream hold's id, under the origin's number; calls the + same. Two tabs (two holds) of one principal are two upstream holds with their own encoders (R1). +- **Nothing is parsed:** the relay routes by the hold's address and authorizes it; params, arguments, results, + snapshots and ops travel as opaque parts (section 5). +- **What the relay keeps:** per relayed hold, its upstream link and its original `hold` message (params copied once), + which routes `call`, `cancel`, `retry` and `release` upstream at once and holds again on the next link; per + downstream origin, the FIFO of the sender of each `bound`/`state` message in flight (section 6), and the set of + upstream links it relayed to. +- **Forwarding up:** `hold`, `call`, `cancel`, `retry`, `release` with the origin's number; a `hold` waiting for its + route is sent when the link exists; a `release` before that drops it. A call is forwarded at once; with no + upstream link it is answered `unavailable`. +- **Forwarding down:** a message from upstream with an origin goes to that origin's provider side, which forwards it + with its own origin (none for a tab), ids and parts unchanged: `bound`, `state`, `failed`, `loading`, `result`, + `error`, `gone`. One upstream message concerns one origin, so it is never split. **A link answers only for what was + relayed over it** (owner, 2026-10-24): every entry of a `bound` or `state`, every `failed`, `loading` and `gone` by + its hold, every `result` and `error` by its call is checked against the relay's table: the hold (the call's hold) + must be relayed over the link the answer came from; otherwise it is dropped (Q1: also an answer crossing a release). + A `bound` or `state` left with some entries is forwarded with those; left with none, it is acknowledged to that link + at once (its window would otherwise lose the slot). A message for an origin from a link the origin relayed nothing + to is ignored whole. Cost: one table lookup per entry, a copy of the entry list only when one is dropped. +- **Upstream drop:** the relay sends `loading { hold }` downstream for every relayed hold on the dropped link, fails + their relayed calls in flight with `unavailable` (never resent), re-resolves (section 2) and holds again upstream + from the kept message (same id, same origin); the new `bound` is forwarded. A `bound` on a hold the consumer has + bound already replaces its replicas in one batch, another variant allowed (3b's re-hold path, now also for live + holds); `loading` on a live hold moves it to `loading`, and on a hold still loading fails a dependent's wait at once + (R18, as a dropped link does). +- **Downstream drop:** the origin's provider side ends: its served holds are released, its relayed holds forgotten, + and every upstream link it relayed to gets one `closed { origin }`, which releases the origin's holds and aborts its + calls there. The origin's number is never used again. +- **`gone`** from upstream is forwarded; the relayed hold is forgotten. + +## 5. Messages (§22, 2b) + +Changes to slice 3a's `MESSAGE` and both codecs: + +- **Parts are opaque bytes:** `params`, `args`, `value` (results), `bound` member values and `state` member ops are the + UTF-8 JSON text as bytes (plus byte segments for payloads marked `bytes`). The binary codec already writes them + length-prefixed; decoding returns views of the received buffer. A part is parsed only where it is owned, right when + its message is handled (so a malformed part is still a protocol error that closes the link): the provider parses + `params` (`checkParams`) and `args` (`checkArgs`), the consumer parses results and snapshots and decodes ops. A + message naming an unknown id is ignored before its parts are parsed (Q1). The text codec, a debugging codec, + re-serializes parts at decode (accepted cost: one parse per part per relay hop). +- `hello { protocol, incarnation, calls }`: `calls` is the number of concurrent calls this side accepts (section 6). +- **`origin?`** on `hold`, `release`, `call`, `cancel`, `retry`, `ack` (towards the provider) and `bound`, `state`, + `failed`, `loading`, `result`, `error`, `gone` (back): the relay's number of the downstream origin (an integer from + 1; absent: the sender's own work). Binary codec: the kind byte's high bit, then the origin as a varint; text codec: + `["@", origin, tuple]`. The schema refuses an origin on `hello`, `ping` and `pong`. +- `hold { origin?, hold, type, version, fingerprint, params, address?, principal?, incarnation? }`: `address` is the + address the caller computed (absent for a type without placement); it routes the hold, the provider places it by + its params. +- **`bound { holds: [{ hold, variant, members, failed }] }`:** one message may bind several holds; the consumer applies + all of them in one batch. Used for first bindings and re-holds. +- `ack { origin? }`: consumer to provider, one per applied `bound` or `state` message of that origin. +- `closed { origin }`: relay to provider, the downstream origin ended: its holds are released and its calls aborted. +- `ping {}` and `pong {}`: the heartbeat (section 7). +- `PROTOCOL_VERSION` stays 1 (nothing is deployed; carry-over L6). + +## 6. Flow control (§3, decision log) + +**The window.** Per origin of a link, at most **K = 2** of the provider's own `bound` and `state` messages +unacknowledged; only these two kinds count (every other message is control and is never queued behind them). The +consumer sends `ack` (with the message's origin) after applying each (after its batch); an `ack` after the link or +the origin closed, or crossing nothing in flight, is ignored. The provider keeps per origin the sender of every +message in flight, in order: itself, or (at a relay) the upstream link it forwarded the message from. Acks arrive in +that order: one for its own message frees a slot, one for a relayed message is sent to that upstream link with the +origin's number there. So a tab served through a relay by W workers may have 2W messages in flight (plus the relay's +own 2), each worker's window bounding its own. + +**The provider while the window is full:** + +- each touched hold member keeps its ops in its **pending queue** (raw ops, never mutated: the same ops feed other + holds and watches; `r` drops everything before it; the leaf index merges a new append or primitive set into the last + pending op on the same path, any other op clears it; past 10,000 pending ops only "replace with the current value" + stays). The cap bounds entries, not payload bytes. When an `ack` frees a slot, one `state` message carries every + waiting hold's entry: its members' pending ops encoded once (or `[["r", value]]` after an overflow), all reflecting + the latest commit. +- a hold that binds while the window is full waits for a slot; its snapshot and subscription are taken in one + synchronous step when its `bound` is sent (3a race 2). +- a hold whose object leaves `live` while its `bound` waits goes back to pending (no `bound` is built for it; slice + 7's reload). + +**The pending queue** is shared Chord code (`src/delta/`); the shared watch handle uses it too, in the same pi change +(carry-over; pi first). + +**The relay** folds, drops and resets nothing: it forwards each upstream `bound`/`state` at once and returns its +`ack` when the downstream acknowledges it. A slow tab stalls only its origin at each worker, whose pending queues fold +its commits as they do for a direct consumer; other tabs and the relay's own holds on the same worker are other +origins with their own windows. A relay keeps no state per message beyond its senders FIFO. + +**Calls.** `hello` announces how many concurrent calls a side accepts per origin (N, default 256, the same on every +host); a side sends no call before the peer's `hello`. A host sends at most N of its own calls per link in flight; +further calls wait in the link's **call queue**, one FIFO, sent in order as replies free slots (it does not preempt +calls already running). A relay forwards calls at once: each downstream origin's consumer is bounded by its own N, +so the upstream sees at most N per origin. Rules: + +- a slot is freed once per call: by its `result` or `error`, by its `cancel` (when sent, at the caller; when received, + at the provider), or by its hold's `release`, `gone`, its origin's `closed` or the link closing; +- a call still queued when its hold rebinds (`loading`, then `bound`) is rejected `unavailable` (it may name a method + index of the old variant); +- a provider seeing more than N running calls of one origin closes the link (protocol error); a call naming a hold it + does not know is ignored before it is counted (Q1). + +## 7. Transports and heartbeat (§3) + +- **`ipcLink(port)`** over a Node IPC channel (`fork` with `serialization: "advanced"`, or `process` in the child), + typed structurally (`send`, `on("message")`, `on("disconnect")`, `disconnect()`), no Node import in Chord. A + `Uint8Array` crosses as one message. `send` returning `false` is backpressure, not failure; a thrown error, an error + callback or `disconnect` closes the link once; messages after close are dropped; listeners are removed on close. +- **`webSocketLink(socket)`** over a WebSocket (`ws` in Node, the browser's), typed structurally: binary frames, + `binaryType = "arraybuffer"`; messages sent while the socket is still connecting are buffered and sent on `open`; + a socket already closed gives a closed link; close on `close` and `error`. The transport's `maxPayload` stays its + hard cap (code 1009; size limits per part are 4b). +- Both deliver whole messages in order and close both ends, like `memoryLinkPair`. They live in `src/next/link/`. +- **Heartbeat on every link kind**, in the channel: each link keeps a `lastReceived` timestamp (updated on every + received message; sends do not count) and the host runs one interval timer for all its links (no timer per + message). A link silent for 15 s sends `ping` (once its handshake is done; `hello` comes first); the peer answers + `pong` at once, never queued behind the window. A link silent for 30 s, the handshake included, is closed (its + references go `loading`, 3b). Both numbers are host options; tests use fake timers. The heartbeat detects a silent + transport, not a peer that answers pings but withholds acks (bounded instead by the window and the pending cap). + +## Races + +Routes and resolve: + +1. Two holds for one address while its `resolve` runs: one `resolve`, shared. +2. `resolve` rejects or returns a closed link: holds stay `loading`; re-resolve after the address's backoff while holds + exist; only a link that completed its handshake resets the backoff. +3. Every hold for an address is released while `resolve` runs: it finishes; the link is kept. +4. `authorize` refuses or throws: the hold fails broken; other principals for the address still resolve. +5. Placement mismatch at the provider: broken `invalid/placement`. +6. A worker restarts: its link closes; the next link's `hello` has a new incarnation; re-holds get fresh objects. +7. `host.close` during `resolve`: its context is aborted; a link it still returns is closed at once. +8. A hold from the parent for a type this host does not serve and has no route for: `unavailable` (never back to the + parent). + +Principals and relays: + +9. A tab sends `principal: { kind: "owner" }` on its hold: ignored (child link); the hold acts as the tab's principal. +10. A guest's handler on the server `ctx.get`s a worker type: the worker sees the guest (principal and incarnation). +11. A worker's handler serving a guest gets a server type: the server sees the worker (upward boundary). +12. The upstream link drops with relayed holds: `loading` downstream, relayed calls in flight `unavailable`, + re-resolve, re-hold with the same id and origin, the new `bound` forwarded; acks for messages of the dropped link + still in flight downstream go nowhere. +13. A tab call that crossed the `loading` notice reaches the relay while upstream is down: `unavailable`. +14. A tab releases a hold whose upstream hold is not sent yet: dropped; already sent: released upstream; messages + crossing it are forwarded and ignored by the tab (Q1). +15. The tab link closes: one `closed { origin }` to each upstream link it relayed to (no `release` or `cancel`); the + worker releases the origin's holds and aborts its calls; its messages crossing the `closed` are dropped at the + relay (the origin is gone). +16. `gone` from upstream: forwarded, the relayed hold forgotten. +17. Holds of two tabs on one worker: two origins, so never one message; each tab's messages are atomic for it. + +Flow control: + +18. An `ack` crossing a `release` or a close, or crossing nothing in flight: acks count messages per origin; one + with nothing in flight is ignored. +19. Two workers' messages for one tab interleave at the relay: the senders FIFO returns each `ack` to its worker. +20. A first `bound` from a worker while the tab is slow: forwarded at once (the worker's window for the tab origin + has room for it, or the binding waits there). +21. A commit touches A and B of one tab on one worker while the tab is slow: one entry each in the worker's next + message for the tab origin, never split. +22. A worker answers for a hold (or a call) of the same origin relayed over another worker's link, or for one the tab + released: the entry, notice or reply is dropped; a `bound`/`state` left empty is acknowledged to it at once. A + link the origin relayed nothing to: its message is ignored whole. +23. The pending queue overflows while the window is full: the next message carries `[["r", value]]` for that member. +24. A failed member, a constant and an empty variant in a first binding: carried in the `bound` (values, failed list). +25. More than N calls: the excess waits in the caller's call queue; a relay forwards at once, so a worker counts at + most N per origin; one more is a protocol error. +26. A call queued across a rebind with another variant: rejected `unavailable`. + +Links and heartbeat: + +27. IPC: the child exits during `send`, or `disconnect` races a message in flight: the link closes once. +28. WebSocket: a frame above `maxPayload` closes the connection (1009): references go `loading`. +29. A WebSocket still connecting: `hello` and early messages are buffered until `open`. +30. A peer that never sends `hello`: closed after 30 s. A busy link sends no pings. Two sides pinging at once: both + answer. +31. A peer stalled by a synchronous handler for over 30 s is closed (documented: handlers must not block the event + loop that long). + +## Cost per hop + +| Hop | Work | +| --- | --- | +| relay, per message | decode the envelope, swap the origin (one shallow copy of the message), encode the envelope; parts copied once into the outgoing buffer (O(bytes)), never parsed; per `bound`/`state` one FIFO entry | +| relay, per hold | the address's params checked once (route admission, `authorize`) | +| provider, window free | as slice 3a: one encode per commit per touched hold member | +| provider, window full | per op: one path key and one map lookup (leaf index); one encode per waiting member per freed slot | +| provider, per origin | a window and pending state per origin with holds; none for a relay | +| consumer | as slice 3a, plus one `ack` per `bound`/`state` message | +| origin | two bytes per message on a relay's upstream links | +| heartbeat | one timestamp write per received message; one host-wide timer | +| trusted link | no `MESSAGE` schema check | + +Producer-side string diffs are not 4a's: a provider whose documents grow by plain string assignment pays Delta's +O(length) append detection per commit until it writes through `appendText` (pi #9265). + +## Benchmarks + +Extend `test/next/bench/` (by hand, recorded in `benchmarks.md`): commit-to-frame and bytes through a relay (tab ↔ +server ↔ worker over memory links, and over real IPC and a loopback WebSocket); hold-to-live through a relay including +`resolve` of an in-process worker; a consumer throttled to 64 KiB/s with stalls of 1, 10 and 60 s receiving a streamed +reply into a 500 KB view (bytes, catching up, sender heap; the producer's commit time recorded separately from the +encode); 10,000 calls from one tab with another tab's latency (p95) through the relay. + +## Carry-over taken by this slice + +L6: trusted links skip the message check; a second `bound` on a bound hold (now in 4a: upstream re-holds); +`PROTOCOL_VERSION` unchanged. L7: `placement` (`invalid/placement`, not in the fingerprint). Decided: C12 (no call +credits; the call window). The pending queue and the shared watch handle change (pi first). §18's core facet pattern +(start data from the start message), shown in example 13. Left for 4b: revocation, cross-host cycles, +`host.references`, `host.activity`, size limits, compression (carry-over section 4b). Left as it is: L6's +`map(bytes())` declaration bug. From the 4a reviews (owner, 2026-10-24): an activation's `readyTimeout` covers its wait for the +previous instance's teardown too (an attempt abandoned there has opened nothing). + +## Tests + +- Host kinds and placement: addresses and keys; `placement` excluded from the fingerprint; every row of section 1's + table; `invalid/placement`; throwing `placement` and `authorize`; never back to the parent. +- Routes: one `resolve` per address, rejection and backoff (fake timers), a closed returned link, re-resolve after a + drop while held and not without holds, `authorize` per principal, `host.close` aborting `resolve`. +- Two-sided links: the server holds a worker type and serves the worker's hold of a server type over one route link; + worker to worker through the server; a host that applies its own messages and forwards a tab origin's, each + acknowledged by its consumer. +- Principals: every row of section 4's table (guest `ctx.get`, remote `self.get`, local `self.get`, `host.get` + through a route), a forged `principal` from a tab ignored, the upward boundary, two origins with equal principal + keys but different incarnations. +- Relays: ids kept under the origin both ways, an origin per downstream link never reused, a worker's entries, + notices and replies for another worker's hold of the same origin dropped (a mixed message forwarded with its own + entries only, an emptied one acknowledged at once); no part parsed at a binary relay (a counter); upstream drop (`loading`, calls + `unavailable`, re-hold with the same id and origin, `bound` with another variant); downstream drop (`closed` per + worker); `gone`; mixed codecs (binary tab, text worker link). +- Flow control: K = 2 with a slow consumer; folding through the leaf index and the cap; a `bound` waiting for a slot, + and going back to pending when its object leaves `live`; a window per origin (a stalled origin folds, another + keeps receiving); `closed` per origin; through a relay: every message forwarded at once, each `ack` back to its + worker in order across two workers, a derive over two holds of a slow tab never torn. +- Calls: the window of N, the FIFO call queue, release and `gone` freeing slots once, a queued call across a + variant-changing rebind, N per origin at the provider (one more closes the link; an unknown hold ignored first), + calls in both directions with equal ids. +- Transports and heartbeat: IPC with a real `fork` (one small test) and a fake port (`send` returning false is not a + close); WebSocket over `ws` loopback and a fake socket (connecting buffer, already closed); silence before `hello`, + a busy link without pings, a half-open link closed at 30 s, simultaneous pings; timers and listeners released. +- Cost guards: no part parsed at a binary relay; one encode per waiting member per freed slot; no `MESSAGE` check on + trusted links; no timer per message. +- GC: a released relayed hold, a closed route link, a dropped downstream link, pending `resolve` waiters, the call + queue, the pending queue after a `closed`, received buffers not kept by parts after handling. +- Examples: `12-routes.ts` (a server routing to an in-process worker, a guest's `ctx.get` through it), `13-relay.ts` + (tab, server, worker over IPC and WebSocket; the worker's `SessionInfo` from its start message), `14-flow-control.ts` + (a slow tab: folding at the provider, and through a relay folding at the worker per origin while a fast tab keeps + receiving). + +## Size + +About 2,500-3,000 changed or new source lines (two-sided links, routes, principals, the relay with origins, opaque parts in both codecs, flow control with the pending queue and the call queue, two transports, the +heartbeat), plus the pi change for the pending queue and the watch handle. Tests and examples about as much again. diff --git a/packages/chord/docs/next/slice-4b.md b/packages/chord/docs/next/slice-4b.md new file mode 100644 index 000000000..51e267828 --- /dev/null +++ b/packages/chord/docs/next/slice-4b.md @@ -0,0 +1,208 @@ +# Slice 4b: authority and housekeeping (spec) + +Status: **revised (2026-10-24) for flow control per origin; the owner sent it to building directly**, with reviews of +the build by fresh reviewers. Guiding rule (owner): simple, fast, correct. The single source for building 4b, together +with the feature map in `slices.md` (entries assigned to 4b) and its "Carry-over" section for slice 4b, `design.md` read +in full (most relevant: §3 links, relays and keep-alive, §4 principals and revocation, §6 cycles, §19 presence, §22 +size limits and compression, §26 open points), the decision log (R2, R10, R11, Q2, Q5, R26, "Compression at the +origin", "Relay flow control per origin", "Slice 4b: `host.activity` is a watchable", "Simplification pass after 4a") +and the root `AGENTS.md` section "Chord v2 Code Style". Builds on slice 4a as simplified (`79a0206dc`). + +Points marked **proposal** are the builder's default; the owner decides them after the build (they are collected in +the last section). + +## What an app can do after this slice + +```ts +type Revocation = { grant: string } | { device: string } | { session: string }; +const server = createHost({ + address: Server(), principal: { kind: "server" }, onError, + revokedBy: (p) => p.kind === "guest" ? [{ grant: p.grant }, { session: p.session }] + : p.kind === "owner" && p.access === "device" ? [{ device: p.device }] : [], + compression: nodeCompression, // from `@earendil-works/chord/next/node` +}); +server.revoke({ grant: "g1" }, ctx); // ends g1's objects here and on every worker, closes g1's links + +host.serve(Notes, ({ session }, principal, self) => { + const viewers = self.derive([host.references(Notes)], ([refs]) => + refs.filter((r) => r.params.session === session).length); + … +}); + +const activity = await worker.activity.watch(ctx); // the app's supervisor decides + +server.accept(webSocketLink(ws), { principal, limits: { part: 16 * MiB, decoded: 64 * MiB } }, ctx); +``` + +## 1. Revocation (§4, R10, R11) + +- `createHost({ revokedBy })`: `(principal: P) => R[]`, `R` JSON; the same app code on every host. A principal's + **revocation keys** are the canonical JSON of each value `revokedBy` returns, computed once per principal object and + cached weakly (a `WeakMap` keyed by the principal object, with a GC test). L1's unused `revocationKeys` was cut + (owner, 2026-10-24): write the helper here, next to its user. +- `host.revoke(r, context)` on a host, synchronously: + 1. ends at once every object whose identity principal `p` has `r` in its keys (remote objects; local objects have no + principal), whatever references remain (R11): each holder gets `gone` with `forbidden` ("revoked"), running + calls' contexts are aborted (their callers are rejected), an activation in progress is abandoned. Teardown then + runs as for any ending (drain, closes with reason `end`). + 2. closes every **child** link (made by `accept`) whose principal has `r`; never the parent link, never the host's + own principal. Closing a tab's link at a relay sends `closed { origin }` upstream (4a), so that tab's holds on the + workers are released without a revocation message. + 3. sends `revoke { revocation: r }` on every **route** link (its children), which apply it with their own + `revokedBy`, and so on down. Nothing is replayed: a child connected later gets no past revocation (its objects + are new, and the revoked links are closed already). +- A `revoke` message is honoured only on the **parent** link (like the principal fields on holds); arriving on any + other link it is ignored. +- What `gone` means for consumers is 3a's rule (calls on the hold reject; a relay forwards it). Holds acting for a + revoked principal arrive at a worker as their own objects (identity principal = the hold's principal, 4a), so the + worker's own `revokedBy` ends them. +- Reconnecting never restores revoked authority: authentication is the app's; a changed grant is a new principal (its + key includes the grant revision, design §4). +- Jobs and revocation (R11) arrive with slice 5. + +## 2. Cycles across hosts (§6, Q5) + +- Every object instance gets an **instance id** (uuidv7) at its first attempt; a restarted object (a new instance) + gets a new one, so an old chain never matches it (owner, 2026-10-24, D4). +- A hold made during an activation (a `self.get` of a type that is not served here, routed over a link) carries + `chain: string[]`: the chain of the activating object (the chain of the hold that created it, if any) plus its own + instance id; at most 32 entries (a longer chain keeps its last 32: a longer cycle is not detected; documented). +- A provider refuses a hold whose chain contains the instance id of the object it would bind while that object is + activating: the hold fails broken `invalid/cycle` naming the cycle by labels where known. +- **Proposal:** a dependency failing with `invalid/cycle` fails the dependent **broken** with the same error (an + exception to R18's "dependency failures are transient"). Otherwise, A (server) → B (worker) → A would fail B + transient, B would retry with backoff, and every retry would hit the same cycle. +- `chain` is honoured on trusted links (any role: a cycle runs server → worker → server over route and parent links) + and dropped from holds arriving on untrusted links. Relays copy it upstream unchanged (they activate nothing). +- One-host cycles stay as built (slice 1: following waiting objects). + +## 3. Presence: `host.references(Type)` (§19) + +- `host.references(Type)` (on the host, and on the facet's host): a JSON `Watchable` of the references to objects of + that type on this host: `{ params, principal, origin }[]` (`principal` absent for a local type). `origin` is the + incarnation of the host where the reference was made: for a hold over a link, the hold's caller incarnation (the + tab's own, carried by a relay on route links, 4a); for a local reference, this host's incarnation. Two tabs are two + origins. +- It changes in the batch where a reference binds or ends; derives over it recompute there (W1). A link drop is a + departure (its holds end); smoothing is the UI's. Lingering objects have no references. +- Cost: one entry added or removed per reference, kept per type (a type nobody asks about costs a counter at most: + **proposal** build the list per type only while it is watched or read). + +## 4. Keep-alive counts: `host.activity` (§3, R26) + +- `host.activity`: a JSON `Watchable` of `{ references, calls, jobs, lingering }` (owner, 2026-10-24): holds arriving + over links and served here, method calls running on this host (local and remote), running jobs (0 until slice 5), + lingering objects. +- Counters are kept where the work happens; a change marks the watchable dirty, and one publication per microtask turn + carries every change of that turn (not one per call start and end); its value is built from the counters when + published or read. A slow watcher gets the newest counts (pending queue). + +## 5. Size limits per part (R2, Q2) + +- `accept(link, { …, limits })`, `connect(link, { …, limits })` and `resolve`'s result take + `limits: { part?: number; decoded?: number }` (bytes): `part` bounds a part's size on the wire, `decoded` its size + after inflating (section 6). Defaults (Q2): none on trusted links (IPC); on untrusted links 16 MiB and 64 MiB. +- Checked from the part's length before it is parsed or inflated (parts are opaque); the decoded bound is enforced + while inflating (section 6). +- An oversized part fails only its hold or call (`too_large/part`); the other parts of the message apply: an argument + part fails its call (`error` to the caller), a result fails its call, a snapshot fails its hold (no automatic + re-hold), a state part fails its hold. The link stays up. +- **The admitting link's limits travel with relayed work** (Q2): a relay checks `part` on the admitting link itself + (the bytes are there) and stamps `limit` (the admitting link's `decoded`) on the upstream `hold`; honoured only on + the parent link (like the principal fields); calls on the hold inherit it. The host that inflates enforces the + smaller of its own link's and the stamped one, so a worker never inflates a tab's argument beyond the tab link's + limit. +- The WebSocket's `maxPayload` stays the transport's hard cap (4a). + +## 6. Compression at the origin (#27: owner leaning (b), built as (b)) + +Problem: encoding happens in the synchronous flush (one message per commit, W1), so compression must be synchronous; +browsers have only the asynchronous `CompressionStream`/`DecompressionStream`. + +- **Who compresses:** a host with a synchronous compressor (`createHost({ compression })`) compresses each JSON part + above 16 KiB (deflate raw, level 1) when it encodes it; the part's envelope says so (a codec flag per part in both + codecs). Byte segments stay raw. Hosts without one (browsers) send every part uncompressed; their parts are + arguments and params, which are small. +- **Who inflates:** the end that owns the part, when its message is handled. Relays never inflate (parts are opaque). +- **The adapter**, portable: `compression?: { deflate(bytes): Uint8Array; inflate(bytes, limit): Uint8Array }` + (synchronous). Chord ships `nodeCompression` (Node's `zlib.deflateRawSync`/`inflateRawSync` with + `maxOutputLength`) in a Node-only subpath; `src/next` itself imports no Node module. Without the adapter, a host + inflates asynchronously with the web `DecompressionStream("deflate-raw")`, counting output bytes and aborting past + the limit. +- **Order with asynchronous inflate:** a received message whose parts need an asynchronous inflate is held, and every + later message of that link waits behind it, in order, until it is inflated and handled. The queue is bounded: a + provider has at most two `bound`/`state` messages in flight per origin (2W + 2 for a tab behind a relay with W + workers), at most N = 256 call results per origin, and the `ack` goes out only after the message was applied, so a + slow inflate is backpressure like a slow consumer (its commits fold at the provider). Control messages (`ping`, + `pong`) are answered when they are reached in order; the 30 s heartbeat leaves ample room. +- Threshold (16 KiB) and level (1) are protocol constants, not per link. +- `PROTOCOL_VERSION` stays 1 (nothing deployed). + +## Messages + +- `hold` gains `chain?: string[]` (trusted links only) and `limit?: number` (parent link only). +- `revoke { revocation }` (parent to child; honoured on the parent link only; no origin). +- A part's codec flag: in the binary codec a flag in the part's length prefix (builder's choice, documented); in the + text codec, which is for debugging but may sit on a relayed link, a compressed part travels as a tagged element + holding its base64 (**proposal**; the text codec itself never compresses). + +## Races + +1. `revoke` while an object of the principal activates: the attempt is abandoned, holders get `gone`. +2. `revoke` while a call of the principal runs: its context is aborted; the caller is rejected (`gone` reaches it). +3. A hold stamped with a revoked principal is already on its way to a worker: it binds, then the worker's `revoke` + (sent right after on the same link, per-direction order) ends it: R10's window, accepted. +4. Two tabs of one grant, one revoked: both principals have the key, so the shared objects end for both and both links + close. +5. `revoke` crossing a `resolve`: the new worker link gets no past `revoke`. +6. A `revoke` from a child link: ignored. +7. A cycle through a relay: the chain is copied upstream; detection at the provider. +8. A cycle through a restart: new instance ids, no false cycle. +9. `host.references` and a link drop in one batch with a commit: the derive sees the departure in that batch. +10. An oversized part inside a `state` message with other holds' entries: only its hold fails. +11. A compressed part whose inflated size exceeds the limit: its hold or call fails `too_large/part`; the link stays. +12. A stamped `limit` or `chain` from a child or untrusted link: ignored. +13. An asynchronous inflate in flight when the link closes: its message and everything queued behind it are dropped. +14. A slow asynchronous inflate at a tab: the worker's window for the tab's origin fills; its commits fold. + +## Cost per hop + +| Hop | Work | +| --- | --- | +| revoke | keys cached per principal; one lookup per object and link; O(objects + links) once | +| hold during activation | one chain array (at most 32) | +| presence | one entry added or removed per reference, in the batch | +| activity | counters; at most one publication per microtask turn with changes | +| size limits | one comparison per part before parsing | +| compression | one deflate per part above 16 KiB at its origin, one inflate at its owner; relays nothing | + +## Carry-over taken (from `slices.md`, section "Slice 4b") + +All of it: revocation, cross-host cycles with instance ids, `host.references` and `host.activity`, size limits and +compression, the Durable conversation view's size on re-hold (known cost until pi #11327; nothing to build), route +links' context (give `route` a `context` option, defaulting to the host's background context, used for the link's +work and telemetry), one N on every host (unchanged; note stays), `gone` becomes reachable (test it through +revocation; drop the hand-ended test only if the new one covers the same paths). + +## Tests + +Every race; revocation on one host, across a relay to a worker, two tabs of one grant; a child's `revoke` ignored; +cycles server → worker → server and through a relay, no false cycle after a restart, the broken dependent (proposal); +presence per origin with derives and link drops; activity counts per kind and one publication per turn; limits per +part in every message kind, the stamped decoded limit, a compressed bomb (both inflate paths); compression round +trips in both codecs, relays not inflating (a counter), the asynchronous inflate's ordering and its close race; cost +guards; GC (revoked objects, presence entries, revocation key cache, chains, the inflate queue). Examples: +`15-revocation.ts`, `16-presence-and-activity.ts`, `17-limits-and-compression.ts`. Benchmarks in `benchmarks.md`: a +500 KB snapshot and a 1 MiB result compressed and not (bytes, encode and inflate time, synchronous and asynchronous). + +## Size + +About 800-1,000 source lines. + +## Decisions for the owner after the build + +Collected here by the builder and reviewers; known now: + +1. A cycle error makes the dependent broken (section 2), an exception to R18. +2. Compression (b) as built: adapter shape, the asynchronous fallback, the text codec's form of a compressed part. +3. `host.references` built lazily per type, or always. diff --git a/packages/chord/docs/next/slice-5.md b/packages/chord/docs/next/slice-5.md new file mode 100644 index 000000000..97cdaa264 --- /dev/null +++ b/packages/chord/docs/next/slice-5.md @@ -0,0 +1,281 @@ +# Slice 5: routines and jobs (spec) + +Status: **reviewed, ready to build** (2026-10-24; reviews by Astra, Fable, Opus, Sol folded in). Guiding rule (owner): simple, fast, correct. The single source for building +slice 5, together with `design.md` §14 (routines) and §15 (jobs), read in full with the rest of `design.md`, the +feature map in `slices.md` (entries assigned to slice 5) and its "Carry-over" for slice 5, the decision log entries +"Routines for slice 5", "Routines and jobs open points" (with "Jobs keep accepts" and "Job crash handling"), D4, R11, +Q9, S8, W2b, P2, and the root `AGENTS.md` section "Chord v2 Code Style". Builds on slice 4b (`1c88b45d7`). + +Slice 5 builds routines completely and jobs **in memory**. Persistence of jobs is slice 6 (it needs the store); the +reload rules of both (handover, `accepts` in use, `blocked { version }`, native restarts) are slice 7 (they need +reloads). Their APIs are declared here so that later slices add behaviour, not shape. + +## What an app can do after this slice + +```ts +// routines +host.serve(ForwardStore, (_, self) => { + const forwards = self.state(Forwards); + const catalog = self.get(Catalog, {}); + self.on(catalog.sessions, async (sessions, ops, ctx) => { + for (const f of forwards.value.list) if (sessions[f.session]?.archived) await closeForward(f.id, ctx); + }); + self.onLive(async (ctx) => { + for (;;) { await sleep(HOUR, ctx); await dropExpired(ctx); } + }); + return { … }; +}); + +// jobs +const ShellJob = defineJob("chat.shell", { + version: 1, + args: obj({ command: str(), author: Author }), + result: obj({ exit: int(), output: str() }), + state: ShellState, + concurrency: 1, +}); +const shell = self.job(ShellJob, { run, finish }); +await shell.start({ command: "npm test", author }, ctx, { key: requestId }); +``` + +## 1. Routines (§14) + +- `self.on(source, fn)` and `self.onLive(fn)`, declared only while the factory runs (as `state`, `derive`, `get`, + `open`), in objects of both kinds and in facets (design §16: a facet holds documents, references, routines and + contributions). +- `on` over a JSON `Watchable` calls `fn(value, ops, ctx)`; over a `NativeWatchable` (a registry value, `ref.state`) + `fn(value, ctx)`. Overloads by the source's type. At runtime the source is resolved at declaration with + `resolveWatchable` (Chord's own sources, constants, reference members); anything else that is a `NativeWatchable` + (its `watch` yields a `NativeWatch`) is followed as native. There is no separate kind tag. +- Start: right after the object went live, in the same turn (owner); a method call admitted in that turn may run + before the first `on` call. Stop: when the object ends, their `ctx` is aborted with the object's end error; the + drain (§6) waits for routine calls like for method calls, bounded by `closeTimeout`. +- `on` is implemented over the shared `Watch` with a listener that **never throws** (a throwing listener would end the + watch): it runs the handler, catches its error, reports it (`routine_failed`) and schedules its own retry. Calls + never overlap; changes during a call fold into the next (the watch's pending queue: newest value, folded ops); a + running call is never aborted by a change. + - The first call gets the current value with `[["r", value]]` (W2b). + - After a call that threw, the next call (the retry after the backoff, 1 s doubling to 60 s, or a change during the + backoff, which calls at once) again gets `[["r", value]]`, never only the changes since: a handler that applies + ops incrementally would otherwise drift. A call that succeeds resets the backoff. + - The source ending (`ended`, `retired`: a document retired, a reference released) ends the routine without a + report; the source failing (`failed`: a derive's compute threw, a provider member failed) ends it with one + `routine_failed` report. (A failed provider member that recovers after a provider reload: slice 7 carry-over.) +- `onLive`: called once when the object is live. Returning ends it (until the next reload or activation, slice 7); + throwing is reported and restarts it after the same backoff (reset when a run lasted at least a minute); an abort + (the object ends) is no error and not reported. +- Each call's `ctx`: the object's principal (remote) or the host's (local), the host's incarnation, telemetry, and + `ctx.get` references released when the call settles (`CallContext`). +- Not counted in `host.activity`. +- Helpers, exported for apps: `sleep(ms, ctx)` (rejects with the abort reason when `ctx` is aborted; clears its timer + and listener; delays above the timer limit of 2^31-1 ms are chained) and `untilAborted(ctx)` (resolves when `ctx` + is aborted). Portable (timers and `AbortSignal` only). +- `HostError` gains `routine_failed { object, routine, error }`, `routine` being the kind and declaration position + (`"on #2"`, `"onLive #1"`). +- The reload handover per object is slice 7 (carry-over); in this slice a routine only starts with the object and + stops with its end. + +## 2. Jobs (§15), in memory + +### Declaration + +- `defineJob(name, { version, args, result, state?, concurrency?, accepts?, persist? })`: `args` closed (like method + arguments: checked at `start` with a compiled validator, `invalid/args`); `result` not validated (the provider's); + `state` a singleton document definition; `concurrency` a positive integer; `accepts` checked at declaration (shape, + older versions only) and used in slice 7. `persist` has its final shape now, `persist: { resume: "rerun" | + "abandon" }` (resume required, Durable's wording), and is refused in this slice (`not supported before slice 6`) + rather than silently kept in memory. +- `self.job(Job, { run, finish? })` in an object's factory: once per job name per object; returns `JobHandle`. +- The per-run state documents are Chord's: an internal keyed handle per job (named after the job, keyed by run id), + separate from the author's own `self.state` handles, so an author may also use the same definition for a document + of their own without "declared twice". + +### Run records and `running` + +- The job keeps its records on the owner (a map by run id, with the key index); records and state documents change + only inside transactions on the owner's line. Two hooks on `Transaction`: staged record changes (create, outcome, + delete, block, drop) adopted inside the commit step with the documents, and actions after the commit (start a + `run`, settle waiters). +- `running` is a `Source` of its own (not a `Derive`: keyed documents publish nothing), marked changed in the commit + step whenever a record or a run's state document changed, delivering the new list in that batch. Entries: + `{ id, version, key?, args, state, phase: "running" | "finishing" } | { id, version, phase: "blocked", reason }`. + An entry is a new object exactly when its record or its state document's value changed (cached per run by record + and state value identity, never by record alone); the list is copied on change (O(unsettled runs)). + +### Start + +- `start(args, ctx, { key? })`: one transaction on the owner's line; resolves with a `Run` after the commit. + `tx.start(job, args, { key? })` inside `self.tx` joins the transaction's other writes and returns a `Run` handle at + once (a handle, not data from a draft); its `outcome(ctx)` waits for the run and rejects `released` if the + transaction is discarded. So a caller can never miss a short run: the handle exists before the run can settle. + `job.outcome(id, ctx)` waits by id for an unsettled run only (an id of a settled or unknown run rejects `gone`; + documented). +- Order inside a transaction: check args; dedupe by key (an unsettled run with the key, including one started earlier + in the same transaction, returns its `Run`; the args are not compared: a key names one run); concurrency: running + plus finishing runs plus runs started earlier in the same transaction, **excluding the run whose `finish` this + transaction is** (so `finish` can chain the same job at `concurrency: 1`); at the limit `unavailable/busy`; then the + record and the state document (`initial()`, checked once per definition, R8/D3). +- `run` starts after the commit, outside the batch, after `self.tx` settled (an action after the commit). An `abort` + between the commit and the start pre-marks the run's signal, so `run` starts aborted. +- A `start` reaches only a live object (methods and routines run only then). New error reason `unavailable/busy` (the + error table; `PROTOCOL_VERSION` stays 1). Run ids: uuidv7. + +### Run + +- `run(args, ctx)` runs outside transactions. `JobCtx` = `Ctx` plus `jobId` and `state: { value; change(fn) }` (only + when the job has a `state`; typed so). +- `ctx.state.change(fn)` takes **no context**: it is a transaction on the owner's line drafting only the run's + document, under the owner's context without the run's abort signal, so changes made while `run` stops after an + abort still commit. It is accepted while the run's invocation is live (until `run` settles) and refused afterwards + with `released` (an invocation fence: it checks the run's invocation, not only the owner's liveness); it never + recreates a retired document. Changes admitted before `run` settled commit before the outcome is recorded (line + order), so `outcome.state` includes them. +- `run`'s abort signal fires on `abort(id)`, on the object's end and on `host.close`. A run that throws after its + signal fired ends `failed { aborted, reason: "abort" }` (not `threw`); a run that returns after an abort keeps its + result. +- `run` throwing (not aborted): `failed { threw, error }` and a `HostError` `job_failed { object, job, id, error }`. + +### Finish + +- When `run` settles: the outcome is recorded on the run (`phase: "finishing"`), and `finish(outcome, tx)` runs in + one transaction; the same commit deletes the record and the key and retires the state document. Without `finish`, + the commit only cleans up. +- `finish` throwing: nothing changes; retried with backoff (1 s doubling to 60 s), each failure reported + (`job_finish_failed { object, job, id, error }`); after five consecutive failures the run becomes + `blocked { reason: "finish_failed" }`: outcome, state and key kept, not counted for concurrency, retried on the + next reload (slice 7); a later `start` with its key returns the blocked run (dedupe). +- `Run.outcome(ctx)` resolves with the outcome when `finish` committed (also without `finish`); a blocked run's + waiters keep waiting (their own `ctx` ends the wait); a dropped run's waiters reject `aborted`; the object's end + rejects them with its error. + +### Abort, keep-alive, object end + +- `abort(id)`: fires the signal of a running run; drops a blocked run (owner: no `finish`; record, key and state + document go in one transaction; reported `job_dropped { object, job, id }`; waiters reject `aborted`); ignored for + finishing runs and unknown ids. +- Keep-alive: running and finishing runs are a **count** on the object, not `Holder`s (they are not references, so + they never appear in `host.references`). An object whose last reference goes while the count is above zero stays; + when the count reaches zero with no holders, it ends as on its last release (after its type's `linger`). Blocked runs + do not count: at that end they are dropped with a `job_dropped` report each. `host.activity.jobs` counts running + plus finishing runs. +- Object end (revocation R11, `host.close`, the last release): runs' signals fire; the drain waits for `run` + invocations like for calls (bounded by `closeTimeout`); no `finish`; outcomes, states and waiters end with the + object (waiters reject with the object's end error). + +## Messages + +None: jobs and routines are host-local; `running` is ordinary state. + +## Races + +1. Two `start`s with the same key in one turn: the line serializes them; the second sees the first's record and + returns its run. +2. A `start` at the concurrency limit while a run is finishing: finishing counts; `busy`. +3. `tx.start` in a transaction that is then discarded: no record, no run; the `Run` handle it returned rejects + `released` from `outcome(ctx)`. +4. `ctx.state.change` after `run` returned but before `finish` committed: refused (`released`); the outcome's state is + the state at `run`'s settle. A change made after `abort` while `run` is still stopping: commits (no run abort + signal on it). +5. `ctx.state.change` in flight (waiting for the line) when `run` settles: it was admitted while live, so it commits + before the finish transaction (line order); the outcome's state includes it. +6. `abort` while `finish` runs: ignored. +7. The object ends while `finish` waits for the line: the finish transaction is refused (`released`); nothing is + reported as a finish failure (it is the object's end). +8. `finish` throws five times: blocked; a new `start` with the same key returns the blocked run; a different key + starts a new run (blocked runs do not count for concurrency). +9. `on`'s source changes during the backoff after a throw: the call happens at once with the newest value. +10. A routine call throws while the object ends: the end wins; no restart, no report of the abort. +11. `sleep` with a huge delay and an abort: the chained timers are cleared. +12. A job's `run` calls `start` on the same job (recursion): allowed; a new run (it is a different key or none). +13. `finish` chains the same job at `concurrency: 1`: the finishing run is excluded from its own transaction's check. +14. Two `tx.start`s with the same key in one transaction: the second returns the first's `Run`. +15. A short run started with `tx.start` settles before the caller awaits `outcome`: the handle resolves anyway. +16. `abort` between the start's commit and `run`'s start: `run` starts with its signal aborted. +17. The last run settles on an object without holders: the object ends after its `linger`; a blocked run is dropped. + +## Cost + +| Work | Cost | +| --- | --- | +| `on` per change | the shared watch's frame; folded under a busy call | +| `start` | one transaction: an args check (compiled), a record and a document | +| `state.change` | one in-memory transaction on the owner's line (prepare, adopt, flush) | +| `running` per change | one entry replaced; the array copied (O(unsettled runs)) | +| `finish` | one transaction (finish's writes plus cleanup) | + +## Carry-over taken + +From `slices.md` "Slice 5": routine errors reported to `onError` with their own kind; W2b (`on` gets the acquisition +value first); C3 and C4 (decided); jobs and revocation (R11) and `host.activity.jobs`. + +## Carry-over left (to `slices.md` in the same change) + +- Slice 6: `persist: { resume }` built (record and state document in the owner's store; persisted only in `local` + objects with empty params, P3, enforced at declaration); outcomes and abort requests persisted (a recorded outcome + resumes only `finish`); the restart limit (five crash interruptions, persisted, clean closes not counted; whether a + state change resets the count; what a clean close records); deferred persistence for state documents (flushed before + the outcome is recorded); the `finish` failure count persisted; the job fingerprint stored with every record from + the start; the run's start trace carrier stored (§29). +- Slice 7: the routine handover per object on reload (bounded by `closeTimeout`, calls not waiting for it); job + `accepts` in use (args and result mapped forwards once per run; `blocked { version }` without an adapter or for a + newer version); when an older run's state document may migrate while its old code still writes it, and an older run + keeping its old handles valid until it settles; a native restart aborts running runs (`aborted`, reason `reload`) + and `finish` runs on the new code; a `finish` in flight across a reload; run records and state documents belong to + the object across attempts (documents belong to an attempt today); blocked runs retried on reload; a job declaration + removed while runs exist; a job fingerprint over args, result and state (a schema change without a version bump); an + `on` routine whose source failed and recovers after a provider reload. +- Slice 9: spans for routine calls and job runs. + +## Tests + +Every rule and race above, one `describe` per mechanism: + +- `on`: first call with `[["r", value]]`; folding; no overlap; a throw reported, retried with `[["r", value]]` after + the backoff, a change during the backoff calling at once, success resetting it; the watch survives a throwing + handler; source `ended`/`retired` (no report) and `failed` (one report); native sources (`ref.state`); in a facet. +- `onLive`: return; throw and backoff, reset after a long run; abort without report. +- Helpers: `sleep` abort, chaining above 2^31-1 ms (fake timers), no timer left; `untilAborted`. +- Routines with the drain and `host.close`; a routine call admitted in the first turn next to a method call. +- Jobs, start: args check; dedupe (across transactions and twice in one transaction; different args, same key); + concurrency and `busy` (running plus finishing; earlier starts in the same transaction; chaining the same job from + `finish` at `concurrency: 1`); `tx.start` atomic with other writes; a discarded `tx.start`'s handle rejecting; the + handle of a short run resolving; `outcome(id)` for unsettled, settled and unknown ids; `run` starting after the + commit outside the batch; an abort before `run` starts. +- Jobs, run: with and without state; `state.change` fence (after settle refused; in flight at settle committed before + the outcome; after abort while stopping committed; never recreating the retired document); throw after abort is + `aborted`; throw is `threw` with `job_failed`; result after abort kept. +- Jobs, finish: outcome kinds; record, key and state document gone in the finish commit; retries with backoff, + `job_finish_failed`, `blocked` after five; dedupe returns the blocked run; abort drops it (`job_dropped`, waiters + `aborted`). +- `running`: entries per phase; blocked entries without args or state; an entry replaced on a state-only change; no + walk of other runs; per-principal views (example 19). +- Keep-alive: an object without holders stays while runs run, ends after the last one (with `linger`), dropping + blocked runs; `host.activity.jobs`; runs not in `host.references`. +- Object end: revocation and `host.close` mid-run (signals, drain, no `finish`, waiters rejected). +- Cost guards: args compiled once per definition; `running` touches only changed runs. +- GC: an ended object releases runs, documents and waiters; routines release their watches; `sleep` leaves no timer; + dropped and finished runs leave nothing. + +Examples: `18-routines.ts` (`on` with a retry, `onLive` with `sleep`), `19-jobs.ts` (the shell job with a throttled +state document, flushed in a `finally`, changes awaited or caught, `OutputBuffer` with its required bounds, +per-principal views, abort and a chained job). + +## Size + +About 900-1,200 source lines. + +## Decided on the draft (owner, 2026-10-24) + +1. Routines start right after the object went live, in the same turn; a method call admitted in that turn may run + before the first `on` call. +2. `abort(id)` on a blocked run drops it: no `finish`, its record, key and state document go, reported + (`job_dropped { object, job, id }`), waiters reject with `aborted`. +3. `running` carries every run's `args`; views filter (no opt-in). + +Folded in from the spec reviews (`/tmp/s5-review-{astra,fable,opus,sol}.md`): routines in facets (design §16); `on` +over a never-throwing watch listener with its own backoff and a full value after a failure; no kind tag (resolution +at declaration); internal per-job state handles; records with transaction hooks and `running` as its own source; +`tx.start` returning a `Run` handle; dedupe and concurrency within one transaction and excluding the finishing run; +`run` started after the commit outside the batch, an early abort pre-marking it; `state.change` without a context; +throw after abort is `aborted`; `job_failed`; runs counted, not held, and the object ending when the last one settles; +`persist`'s final shape. diff --git a/packages/chord/docs/next/slice-6a.md b/packages/chord/docs/next/slice-6a.md new file mode 100644 index 000000000..417297c4a --- /dev/null +++ b/packages/chord/docs/next/slice-6a.md @@ -0,0 +1,676 @@ +# Slice 6a: the store (spec) + +Status: **built** (2026-10-27: four build-review rounds and six fix rounds; later rules are in the decision log's +2026-10-27 entries and `design.md`, which win over this spec). Was: **ready to build** (2026-10-27; the second spec review decided by its majority, section 15), written by the redesign session after +the first spec review (`/tmp/s6a-review-{astra,fable,opus,sol}.md`) and the owner's decisions of 2026-10-26 and +2026-10-27, including the general-model discussion (facets, attempts, the seal fence, keyed members). Guiding rules +(owner): simple, fast, correct; Chord and Durable aligned as much as possible. + +The single source for building 6a, together with: `design.md` read in full; the store book `store/book.md`; the +decision log `packages/chord/docs/next/decisions.md` (later entries win; superseded parts are marked), in particular "The +store", S1-S17, W1-W14, P2, P3, "Flat columns", "Query runner", "Index definitions are recorded", R8, R12-R15, D9, D10, +"Slice 6 first decisions", "One retirement mechanism", "Text is plain UTF-8, in code point order", and every entry of +2026-10-26 and 2026-10-27 from "The host's store and how it is opened" to "Slice 6a small items"; the feature map and +the carry-over of `slices.md` (slice 6a); the root `AGENTS.md` section "Chord v2 Code Style". Builds on slice 5 +(`4a5b45c45`), the adaptation to the shared `watch.ts` (`cc2d61f53`) and pi `8025c414d` (`runCounted`). + +6a builds the store and puts Chord's persisted data on it (tables, persisted documents, keyed state members), and +reworks three parts of the built slices that the store needs: facets only declare, objects own their documents, line +and job runs across attempts, and the seal fence. Durable on the store (`StoreStorage`), persisted jobs and +`durableView` are 6b. Sync is not a slice: the store keeps an optional outbox the sync layer will read. + +**Words.** An **object** is one identity (type, params, principal key); an **attempt** is one try at activating it +(factory, references, natives, members, derives, routines). A **seq** numbers a commit, one counter per store; +**`lastSeq`** is the seq of the store's last commit. A **write** (`StoreWrite`) is one change of the store (`put`, +`update`, `updateWhere`, `table`, `index`, `document.create`, …); a **commit** in the outbox is the writes of one seq; a +**batch** is the commits `from..to` that sync ships; an **archive** is a manifest, the writes from 0 to `lastSeq` and an +`end`. A **line** serializes one object's transactions; the **commit queue** serializes only the store's commit step. A +**pin** keeps one key of a keyed document loaded. + +## What an app can do after this slice + +```ts +// the start code of a session's worker +const db = await openNodeSqliteDatabase(join(dir, "session.sqlite")); // chord/sqlite/node +const store = await openStore(sqliteBackend(db), ctx, { outbox: false }); // omitted: keep the persisted setting +const worker = createHost

({ address: Session({ id }), principal, store, onError }); +… +await worker.close(ctx); +await store.close(); // the app owns the store + +// contracts +const NoteSchema = obj({ title: str(), body: str(), author: str() }); +export const NoteStore = local("notes.store", { + version: 1, params: {}, + members: { + notes: keyed(NoteSchema), // a keyed state member + create: method(obj({ title: str() }), str()), + rename: method(obj({ id: str(), title: str() })), + remove: method(obj({ id: str() })), + }, +}); +export const NoteView = remote("notes.view", { + version: 1, params: { session: str(), id: str() }, + placement: ({ session }) => Session({ id: session }), + variants: { + editor: { state: { note: NoteSchema }, methods: { rename: method(obj({ title: str() })) } }, + reader: { state: { note: obj({ title: str(), body: str() }) } }, + }, +}); + +// a plugin: the facet only declares +const Note = defineState("notes.note", { + schema: NoteSchema, version: 1, initial: () => ({ title: "", body: "", author: "" }), + keyed: true, persist: true, checkpointWhen: (_v, _ops, info) => info.deltasSinceBase >= 64, +}); +const Edits = defineTable("edits", { + version: 1, + columns: { id: str(), path: str(), at: int(), note: opt(str()), meta: json(EditMeta, { version: 1 }) }, + key: ["id"], + indexes: { by_path: ["path", "at"] }, +}); + +export default defineFacet((host) => { + host.serve(NoteStore, (_, self) => { // local, empty params (P3): host lifetime + const notes = self.state(Note); // persisted, keyed, evictable + const edits = self.table(Edits); + self.on(notes.at("pinned"), async ({ value, retired }, ctx) => { … }); // routines live in objects + return { + notes, + create: async ({ title }, ctx) => { + const id = uuidv7(); + await self.tx(async (tx) => { + tx.put(edits, { id: uuidv7(), path: `notes/${id}`, at: Date.now(), meta }); + (await tx.draft(notes, id)).title = title; + }, ctx); + return id; + }, + rename: ({ id, title }, ctx) => notes.change(id, (d) => { d.title = title; }, ctx), + remove: ({ id }, ctx) => notes.retire(id, ctx), + }; + }); + host.serve(NoteView, ({ id }, principal, self) => { // a view with params: only in-memory state of its own + const store = self.get(NoteStore, {}); + const note = store.notes.at(id); // pins key `id` while this view lives + if (principal.kind === "owner") { + return { variant: "editor", members: { note, rename: ({ title }, ctx) => store.rename({ id, title }, ctx) } }; + } + return { variant: "reader", members: { note: self.derive([note], ([n]) => ({ title: n.title, body: n.body })) } }; + }); +}); + +// a browser tab: an ordinary hold of the view +const view = await tab.get(NoteView, { session: "S1", id: "n-42" }, ctx); +const watch = await view.note.watch(ctx); +watch.start(async ({ value, retired }) => (retired ? showDeleted() : show(value))); + +// tables +await self.tx(async (tx) => { + const old = await tx.get(edits, { id }); + tx.update(edits, { id }, { note: "fix" }); // plain values + tx.updateWhere(edits, (e) => prefix(e.path, "a/"), (e) => ({ path: replacePrefix(e.path, "a/", "b/") })); +}, ctx); +const page = await edits.page(query(Edits).where((e) => eq(e.path, p)).orderBy((e) => e.at, "desc"), 20, ctx); + +// export and import +await pipeline(Readable.from(encodeJsonl(store.export(ctx))), createWriteStream(archivePath)); +const copy = await importStore(sqliteBackend(otherDb), decodeJsonl(createReadStream(archivePath)), ctx); +``` + +## 1. Changes to the built slices + +### 1.1 Facets only declare (decisions "Facet documents are removed", "Facets only declare") + +A facet's setup declares only `host.serve` and `host.references` (`self.contribute` comes with slice 8). Removed: +`FacetSelf` (the setup takes `host` alone until slice 8), facet documents (`FacetSelf.state`), facet routines +(`FacetSelf.on`/`onLive`), the facet's `Documents` and line, the lock order across two lines, `Documents.facet`, +`Transaction.#facetLine`/`close()`, the facet hydration and routine start in `host.ts#setUp`, the facet drain in +`#close`, `ChordHost.#facets` beyond the names, and their tests (`facet.test.ts`, the facet cases of +`routines.test.ts` and `documents`/`transaction` tests, `support.ts` helpers). A facet cannot fail after `load` (a +throwing setup still rejects `load`, slice 2). A transaction covers only its own object. + +### 1.2 Objects own documents, line and job runs (decisions "Documents and the line belong to the object", "Job runs belong to the object") + +- **Slots and handles.** The object (`ChordObject`) holds one slot per declared document name (trackers, the stored + version and fingerprint, pins) and the line. Each attempt's `self.state(def)` returns a thin handle bound to the + attempt's seal token and the definition's version; old handles keep their token and version (R15: old code touching + a slot migrated past its version throws, one integer comparison per read and draft). The same name with a higher + version migrates on access (in memory too); the same version with another fingerprint is `broken`; a flipped + `keyed` or `persist` is `broken`. "Declared twice" is checked per attempt. +- **Lifetime.** Slots live from the object's first activation to its end, across retries, failures and native + restarts. `initial()` is taken once per slot (again when the declaring definition changes). In-memory slots the new + factory no longer declares are dropped when the factory returns (never on a throwing factory); persisted ones drop + their trackers only. Persisted singletons load once per object, keyed data once per residency, again after a failed + load or a drop (section 15). In-memory state + survives a failure and retry; ending the object clears it. +- **Job runs.** Per job name the object holds the records, keys, active count, `running`, run state documents and + pending actions; the attempt holds the implementation (`run`, `finish`) and the running invocations (its drain, its + seal). A run's state changes and its outcome are fenced by the run's own invocation; `finish` runs on the current live + attempt (a finishing run waits for one); a run of a job the new code no longer declares stays `blocked`; a run + waiting for a live attempt keeps its object alive only while an attempt is live or activating. `Job.seal` fires + signals only; waiters reject at the object's end. +- **Derives and watches.** Derives detach with their attempt and the next attempt's derives attach to the same slots. + Routine watches end by their attempt's ctx cancellation, never by a slot drop. Local references re-bind every state + member by name when the object's new attempt goes live, as a remote reference re-holds and gets a fresh snapshot: + their member watches go on across attempts (documents and derives alike; one frame from the last value), and a + consumer's derive over a member recomputes from the new attempt's member. While the object is failed or activating + again, the members keep their last values and get no frames; the builder states this in one rule in `members.ts`. + Keys a reference holds on a keyed member (`at(key)`, `watch(key)`) follow the object's attempts the same way (fix + round 3). A failed derive heals on every later change of its sources and on a re-bind; failures reach watches as + `failed` calls (decision "Failures are watch calls", fix round 4). + +### 1.3 The seal fence (decision "The seal fence") + +- **Tokens.** Every attempt has a seal token; every job run invocation its own (for `ctx.state.change` and its + outcome). +- **At use:** after the seal, new work through the attempt's handles is refused with `released`: `self.tx`, `ctx.get` + (it checks the call's abort signal first, before anything is created), a job start, a routine start. The attempt's + references stay usable for its own natives' closes, then are released. +- **At commit:** a commit is checked when it enters the store's commit step (sealed: rejected `released`, nothing + written, not a storage failure) and at adopt (admitted while current, sealed during the SQL: durable, adopted into + the object's slots, its staged job changes adopted into the object, their invocations deferred to the next live + attempt; the handler's `self.tx` resolves with `fn`'s result, Durable's rule). In-memory commits are checked inside + the batch. The store gets the check through its interface: `store.commit(writes, { admit, adopt }, ctx)` (section 15). +- **The line:** at the seal, a transaction of the attempt that has not entered the step yet is aborted and the line + released; a step already started finishes before the attempt's teardown completes. +- **Cleanup still runs** after the seal: `ctx.get` references of stuck calls released at teardown, presence entries, + `host.activity` counts. The existing start guards stay (`ChordObject.#run`'s attempt identity, `Job.#invoke`, a + `ctx.get` resolving after its call settled). +- **Not a replacement:** a pure swap (slice 7) keeps the attempt. Natives' effects outside Chord are the plugin's; + their `close` is honoured. + +### 1.4 A live object can fail (decision "A live object can fail") + +A runtime failure of the owner (a `migrate` failing on access, a stored version newer than the code) seals the live +attempt and makes the object `failed` with a `broken` cause: calls reject at once with the cause (the triggering call +too), state members keep their last values (same-host reads return them) and get no frames after the commits already +admitted to the step (those adopt and publish once), natives close, slots stay; `onError` gets `object_failed { object, cause }` +once; recovery by reload or `ref.retry()` (a new attempt; only the failed load is redone). Live dependents stay live; +their calls into it fail. + +### 1.5 Smaller changes + +- `ctx.get` checks the call's abort signal first (an ended handler creates nothing). +- **Names** (decision "Restricted names instead of assigned table names"): object type names and host kind names use + `[a-z0-9@/._~-]+`, lowercase, checked at declaration (`contracts/declare.ts`, `host-kind.ts`); `hostKind("client")` + is refused. +- **Errors:** a new `HostError` kinds `storage_failed { error }` and `object_failed { object, cause }`; a new reason + `unavailable/storage_failed` in the protocol's reason list (the test pinning the list changes; `PROTOCOL_VERSION` + stays 1 until p3 ships). +- **Keyed handles** take a context: `get(key, ctx)`, `keys(ctx)` (code point order, in memory too), `watch(key, ctx)` + (decision "Slice 6a's three code questions"). +- **`retired` on the wire and in sources** (decision "One retirement mechanism"): `Source.advance` takes `retired`; + `Subscriber.next` gets it; `Singleton.remove` sends `{ value: initial(), ops: [["r", initial()]], retired: true }`; + a `state` entry lists retired members (`retired: [index]`), kept by the hold while its window is full (ORed until a + message carries it, as `#failed`), validated as member indexes; a derive over a retired source carries the flag on; + a `bound` snapshot carries no flag. + +## 2. The store + +- **Opening:** `openStore(backend, ctx, { outbox? })` with `sqliteBackend(db)` over any `SqliteDatabase` (Node through + `chord/sqlite/node`, Durable Objects through `chord/sqlite/cloudflare`, sqlite-wasm later). There is no hand-written + memory backend: tests use SQLite `:memory:` through `chord/sqlite/node`. The backend interface is internal; `src/next` + imports no Node module. No `openNodeStore` shortcut. +- `openStore` runs the store's own schema migrations (`applySqliteMigrations`), checks that the file uses the UTF-8 + encoding, and reads the metadata once: `lastSeq`, the outbox setting, the table definitions (version, state, columns + with `json(...)` versions and fingerprints, index definitions) and the document id counter (S10); in memory from then + on (one writer per store), persisted with every commit that changes them (S9). In-memory metadata changes only after + `COMMIT` returned. +- **Lifetime:** the app owns the store; the host never closes it (6b's Durable Session shares it). `store.close()` (no + context, as Durable's `Storage.close()`) waits for a running step, rejects queued commits, running exports and reads + with `released`, closes the backend; idempotent, also after `storage_failed`. A host without a store fails an object + declaring persisted data at activation (`broken`, "persist needs a store"). +- **`storage_failed` reaches the host** through an internal listener the store calls once; the host reports + `{ kind: "storage_failed", error }` once and closes itself. +- **The outbox** (decisions "The outbox is optional, internal and versioned", small items 3-4): opened with + `{ outbox: true }`, off with `{ outbox: false }`, an omitted option keeps the persisted setting (new stores: off). + Turning it on records "outbox from seq N+1"; turning it off drops its commits. A table `outbox(seq, format, writes)` + in the backend, written in the step's transaction; `store.outbox` is `undefined` without an outbox (typed). The store + persists a floor (N when turned on at N, advanced by `trim`): `store.outbox.read(afterSeq, limit, ctx)` returns + `{ seq, writes }[]` in the current `format` (older ones stepped forward, a newer one refused with the data intact) for + `floor <= afterSeq <= lastSeq` and rejects any other `afterSeq` (`invalid/args`); `store.outbox.trim(throughSeq, + ctx)` is idempotent, rejects beyond `lastSeq`, takes no seq and runs outside the commit queue (serialized by the + backend). No pluggable encoding. + +## 3. The commit step + +- **One commit queue per store** serializes the step only; each object keeps its own line for the whole transaction. +- **A step:** the seal-fence check; `BEGIN`; the writes (rows, revisions, the outbox commit when there is an outbox, + the new `lastSeq`); `COMMIT`; then the owner's `adopt(seq, ids)` in one `host.batch.commit(...)` (W1), so every source + the commit touched is marked in the step and the flush right after sends one message per link. The next step starts + only after `adopt` returned (S1). Document ids are assigned in the step (monotonic with seq) and passed to `adopt`. +- **Interface** (internal, Durable's shape): `store.commit(writes, adopt, ctx): Promise`. +- **seq:** one counter per store, strictly increasing; an empty commit takes none. +- **Errors** (decision "Store errors: Durable's rule"): any backend error in a step is `storage_failed`: the store + stops, every later commit and read rejects `unavailable/storage_failed`, the host closes. Plugin mistakes are refused + when a write is staged (`invalid/args`) and never reach the step. A seal-fenced commit is rejected `released` before + `BEGIN`. A backend error on a read is `storage_failed` unless the read's `ctx` was aborted (`aborted`); invalid + arguments (a foreign cursor, a bad key) are checked before the backend (`invalid/args`). An `adopt` that throws is + `storage_failed`. Accepted: integer overflow past 2^63 in a bulk `add` is fatal. +- **Counts** through `chord/sqlite`'s `runCounted` (pi `8025c414d`), only where a count matters: bulk writes, the + engine's conditional `UPDATE`, single `update`/`delete`. +- **Statements per commit:** a delta commit of one document is about three statements (revision, the record's + `UPDATE … WHERE id = ? AND version = ?` counted, `lastSeq`), plus one `INSERT` with an outbox, plus one `DELETE` on a + checkpoint (pruning). +- **In-memory commits** never enter the queue and take no seq: a transaction with only in-memory writes commits + synchronously in a batch of its own (no extra await, no promise; a flag set at the first persisted write, no scan). + +## 4. Transactions (decision "A Chord transaction enters the commit step as Durable's does") + +``` +take the object's line + -> run fn (only await tx.*) + -> abort or error: discard everything, reject + -> prepare every draft; checkpointWhen picks base or delta per persisted document + -> nothing persisted written: batch.commit(adopt) synchronously (W12) + -> otherwise: await store.commit(writes, adopt, ctx without its abort signal) + queue; seal check; BEGIN; revisions, rows, outbox, lastSeq; COMMIT + adopt(seq, ids) = host.batch.commit(() => { persisted slots; in-memory slots; staged changes }) +-> release the line; settle the staged changes; drafts revoked (R12) +``` + +- The line is held until the step returned; once admitted, a commit runs to its end (the caller's abort no longer + counts); the in-memory part is adopted inside the same step (P2). +- **Cold keyed loads** (decision "Cold keyed loads on the line"): a cold `get`, `watch` or pin of a keyed document is a + job on the object's line, as Durable's `snapshot`; cached reads take no line; `tx.draft` loads inside its + transaction; one load in flight per key, shared, each caller's `ctx` ending only its own wait; a load installs its + tracker only while the key is still absent. `tx.retire(handle, key)` of an unloaded keyed persisted document finds it + inside the transaction (one read; absent: a no-op, R13). +- **A migrated base** is written back by the next commit that changes the document; until then a load migrates again + (once per object). + +## 5. Tables + +- `defineTable(name, { version, columns, key, indexes? })` (later versions through `evolveTable`, section 9). `columns` is a flat record of L1's helpers as + a restricted subset: `str()`, `int()`, `bool()`, `num()`, `opt(...)` of those, and `json(schema, { version, + migrate? })`; defaults as `int({ default: 0 })`; no nesting outside `json(...)` (types and declaration). Keys, index + columns and `orderBy` columns are non-`opt` `str` or `int` columns. Every index ends with the primary key. +- `self.table(Def)` only in the factory of a `local` type with empty params (P3; type error and declaration error), + once per table name per object. +- **Names** (decision "Restricted names"): table names `[a-z0-9@/._~-]+`, lowercase; the SQL table name is + `"::"` (a host without an address uses `client`); `:` cannot occur in the parts, so + the rendering is injective. Index names `[a-z0-9_]+`, rendered `
:`. Column names + `[A-Za-z_][A-Za-z0-9_]*`, no two equal ignoring ASCII case, no `$`. +- **Layout** (S16): a real SQLite table per logical table (`STRICT`, `WITHOUT ROWID`, the declared key as primary + key), one column per declared column; a `json(...)` column is `TEXT` plus a hidden integer `$v`; a hidden + integer `$v` per row (eager steps applied). The store renders the DDL and records each definition as a `table` write + (columns with types, `opt`, defaults, `json(...)` versions and fingerprints, key, version, state). +- **Text** ("Text is plain UTF-8, in code point order"): `str()` is plain UTF-8 `TEXT` compared by code point; written + values and every operand go through `toWellFormed()` (check `isWellFormed()` first; copy only when needed); NUL fails + `invalid/args`; `compareText(a, b)` exported; a prefix's upper bound is the next code point after its last, carrying + past U+10FFFF (an all-U+10FFFF prefix has no upper bound; the empty prefix matches every value); `substr` offsets + count code points. +- **Values at write:** `int` a safe integer, `num` finite, `bool` stored 0/1 and read back as `boolean`, `str` as + above; a row has every non-`opt` column and no other field; one compiled closed check per table (a plugin's row is a + trust boundary); `json(...)` values are not walked (R9). +- **Nullable columns** (decision "Nullable columns and the `set` form"): an `opt(...)` column is an optional field, + absent when `NULL`, never an explicit `null`; SQL's `NULL` rules apply to every comparison. + +## 6. Queries and reads + +- `query(Table)` is immutable data: `.where((t) => …)` with `eq ne lt lte gt gte inArray isNull prefix and or not` over + scalar columns, `.orderBy((t) => t.col, "asc" | "desc")` (several, one direction per query in 6a); no joins, + grouping or projections (S12). The default order is the key ascending; the key is appended to every `orderBy` as + the tie-break. +- **SQL** is generated from the query data; values always bound, identifiers quoted; `limit` and cursor values bound, + `inArray` one JSON parameter through `json_each` (no 32,766-parameter limit; `inArray([])` matches nothing). Query + shapes are unbounded (any `and`/`or` tree), so the facade's prepared-statement cache is bounded (pi change, a fixed + LRU in `chord/sqlite/node`, section 15). +- **Outside a transaction** the handle reads committed state: `page(q, limit, ctx, { after? })` → `{ items, next? }`, + `rows(q, ctx)` (all pages), `get(key, ctx)`, `count(q, ctx)`. A key is an object with exactly the key columns + (`{ id }`, `{ path, entry }`); other or missing fields fail `invalid/args`. A query for another table is refused. +- **Cursors** are opaque strings bound to table, order columns and direction; decoding one is a trust boundary + (`invalid/args`). +- **Inside `self.tx`**: the table handle first (another object's is refused); reads async: `tx.page(table, q, limit, + { after? })` and `tx.rows(table, q)` read committed rows, stable for the transaction; `tx.get(table, key)` overlays + the transaction's own `put`/`update`/`delete` (one evaluator for `where` and `set`, also used for the overlay); + after an `updateWhere`/`deleteWhere` on a table, `tx.get` on it throws `invalid/read_after_bulk` (W10). + +## 7. Writes + +- Five writes (S17), synchronous and buffered like a draft's ops (checked and staged, no I/O, no line; a bad row + throws at its own call; the buffer is written in the step or dropped): `tx.put(table, row)` (an upsert: the whole row + replaced), `tx.update(table, key, set)`, `tx.delete(table, key)`, `tx.updateWhere(table, where, set)`, + `tx.deleteWhere(table, where)`. No write shorthands on the handle. An `update`/`delete` of an absent key changes + nothing. +- **`set`** is a plain object of values or a callback with typed column references: `{ note: "fix" }`, + `(e) => ({ hits: add(e.hits, 1), path: replacePrefix(e.path, from, to) })`; `undefined` clears an `opt` column. + Expressions over real columns only: `add sub max min concat replacePrefix coalesce when` (`when(cond, a, b)` with + `cond` a `where` expression over the row); integer arithmetic only; every right side is evaluated against the row + before the write (`{ a: e.b, b: e.a }` swaps); `replacePrefix` on a value without the prefix leaves it unchanged. +- **Writes in the outbox commit** carry what a replica needs, explicitly (decision "Writes carry their versions"): + `put` with the row, `v` and each `json(...)` column's version; `update` with its `set` and `count`; `delete` with the + key and `count`; `updateWhere`/`deleteWhere` with `where`, `set` and `count` (a replica affecting another count stops + applying, S17). +- `$v` is written only by eager migration steps (an internal write form) and by `put` (the table's version); `update`, + `updateWhere`, `deleteWhere` never touch it; a write of a `json(...)` column stores its current version (W11). + +## 8. The documents engine + +- One engine (S13). Address `(namespace, owner, kind, key)`; Chord's: namespace `chord`, owner + `:` (W7; `client` without an address), kind the definition's name, key the keyed document's + key or `""` for a singleton (so the partial unique index on the live address also refuses two live singletons). Owner, + kind, key and `meta` strings follow the `str()` rules. +- A record per incarnation (`id`, address, `history: "latest" | "rewindable"`, `createdAt` (a seq), `retiredAt?`, `meta`, + `revision`); revisions: a base (version, fingerprint, value) and deltas (ops). Chord uses `history: "latest"` (a new + base and a retirement prune older revisions). +- Reads: `find(address)`, `load(id)` (base plus deltas applied once: value, version, fingerprint, `deltasSinceBase`), + `list(query, limit, cursor)` by key. Reads at an older seq (`at`) and `history: "rewindable"` are 6b. +- Writes inside the step: `document.create`, `document.change`, `document.retire`, checked by the writes (one live per + address by a partial unique index; a delta only on its record's current version through the counted conditional + `UPDATE`; nothing after retirement); a failed check is `storage_failed` (memory and disk disagree). + +## 9. Migrations + +- **`json(schema, { version, migrate? })` columns** ("The `json(...)` column helper"): a read of an older value runs + `migrate` and checks the result once (R8); a read never writes back; without `migrate`, a throwing one or an invalid + result the read is refused and the owner fails (section 1.4); a newer stored version is refused the same way; at + activation, the same version with another fingerprint fails the object `broken` (D9). +- **Eager migrations: `evolveTable`** (decision "Second spec review: the majority"; all four reviewers accept it; the + derived adds and drops are withdrawn: a store at v1 cannot reconstruct v2's columns from v3's declaration): + + ```ts + const EditsV1 = defineTable("edits", { version: 1, columns: { id: str(), file: str(), at: int() }, key: ["id"] }); + const EditsV2 = evolveTable(EditsV1, (t) => + t.rename("file", "path").add("dir", str({ default: "" })).rows((row) => ({ dir: dirname(row.path) }))); + const EditsV3 = evolveTable(EditsV2, (t) => t.drop("dir").index("by_path", ["path", "at"])); + host.serve(FileHistoryStore, (_, self) => { const edits = self.table(EditsV3); … }); + ``` + + - **History.** `defineTable(name, { version, columns, key, indexes? })` is a base written in full (no `migrations` + option); `evolveTable(prev, (t) => t…)` appends `prev.version + 1` (contracts' `Increment`) and its edit returns + the editor it was given (contracts' rule, checked). Every version is materialized and fingerprinted at + declaration (`TABLE_FORM_VERSION`, its own form version: table fingerprints are persisted, not sent). The object + serves the newest. A new base (`defineTable(name, { version: 7, … })`) starts a new history; stores below it fail + `broken`. + - **Edits:** `rename(from, to)` (key, index and hidden `$v` references follow), `add(name, column)` (`opt(...)` + or with a `default`), `drop(name)` (its indexes are dropped first in the same commit; a key column cannot be + dropped), `index(name, columns)`, `dropIndex(name)`, `json(name, json(schema, { version, migrate? }))` (a + `json(...)` column's version change: no DDL, no data step, cells still migrate on access; every change of a table + definition is a table version), and at most one data step: `rows(fn)` or `update({ where?, set })`. No key or + column-type changes (`rebuild`, carry-over). The editor's types enforce the order: `rename`/`add`/`json` before the + data step, `drop`/`index`/`dropIndex` after it; the data step's row type is the columns at that point of the chain, + so a step may read a column its version drops and fill one its version adds. + - **Commits per version N:** (A) the edits before the data step as DDL plus `table { version: N, state: "migrating" }`, + one commit; the data step's chunks; (Z) the last chunk (or the `update`) with the edits after the step, their index + changes and `state: "ready"`, one commit. No data step: (A) and (Z) are one commit. Recorded writes: `alter + { table, version, edit }` per column edit (`rename | add | drop | json`), `index`/`dropIndex`, `table { version, + state, columns, key, fingerprint }`; the applier renders DDL from them. Resume at `version N, migrating`: (A) is + done; the step continues; then (Z); N+1 starts only after N is `ready`. + - **Data steps:** `rows` is synchronous and pure; it gets that version's typed row (`json(...)` cells migrated on + access to that version's column versions) and returns a patch of non-key columns (`{}` stamps `$v` only; a key or + unknown column fails `broken`); chunks of 100 in key order, paging forward by the last processed key (a crash + restarts the scan, skipping rows with `$v >= N`); the patch plus `$v = N` written through an internal write form. + `update` is one `updateWhere` with `and(where, lt($v, N))` stamping `$v = N`, `ready` in the same commit, so a + non-idempotent `set` never applies twice. A throwing `rows` fails the owning object `broken`, the table stays + `migrating`, committed chunks stay. Data functions are immutable once deployed (the guide). + - **Activation:** a recorded version newer than the code's newest, older than the code's base, or the same version + with another fingerprint (a history edited in place) fails `broken`; `self.table` of an older version than the + recorded one fails `broken`. A fresh table is created directly at the newest version (`ready`, `$v` = newest), with + its indexes, no replay. + - **Shared with contracts:** `hash64`, L1's `snapshotSchema` normal form per column, `Increment` and the "returns + the editor it was given" check move into one shared module; the table history is its own concept + (`store/table-history.ts`, `store/table-evolve.ts`), not contracts' `Draft`, `Editor` or `checkAccepts`. +- **Where and when:** at the owning object's activation, after its persisted documents loaded and before its + references are bound, on the store that owns the data (a replica never migrates). The attempt's `readyTimeout` + deadline is paused while eager migration work runs and resumed with its remaining budget (one pause scope over the + table checks, (A) and (Z) commits, data chunks and their waits in the commit queue; resumed in a `finally`); the attempt's abort is checked between chunks (a chunk admitted to the step completes); the migration + yields to the event loop between chunks; the start barrier waits for host-lifetime objects' migrations. Getters and + parked calls keep their own deadlines. +- Old code meeting a newer table version or a newer `json(...)` version fails the object `broken`, data intact. + +## 10. Persisted documents + +- `defineState(name, { …, persist: true, checkpointWhen })`: `checkpointWhen(value, ops, info)` required for persisted + documents (a type-level constraint), called once per commit per changed document; creation and version transitions + always store a base. The fingerprint is L1's structural fingerprint of the schema, computed once. +- Only in `local` types with empty params (P3). +- **Loading at activation:** once per object, right after the first factory, every persisted singleton and every + pinned `at(key)` with plain reads (the object holds its line; no read transaction): find, load, the fingerprint + check, `migrate` with its result checked once (R8); an absent document reads `initial()` (Q11). A storage error is + `storage_failed`; a fingerprint mismatch, a throwing `migrate` or an invalid result fail the object `broken`. +- **Keyed persisted documents** load on demand (section 4): `get(key, ctx)` (`undefined` when absent), `keys(ctx)` + (pages from the engine in code point order), `watch(key, ctx)` (`undefined` when absent), `tx.draft(handle, key)`. +- **Pins and eviction:** each loaded key counts its pins: an `at(key)` declared by the current attempt's factory, a + keyed member's `at(key)` of a consumer while it lives, each watch, each load in flight. A key with no pins not read or + written for 10 minutes leaves memory (one owner-level timer and idle structure, no timer per key; eviction runs on the + object's line). Persisted singletons stay in memory while the object lives (`handle.value` is a synchronous read). + +## 11. Keyed watches, `at(key)` and retirement + +- Every document, in memory or persisted, is watchable; a watch follows the address: + - a singleton's retirement is one call `{ value: initial(), ops: [["r", initial()]], retired: true }`; it reads + `initial()` meanwhile; a later write is an ordinary change; + - `await handle.watch(key, ctx)` is `undefined` when absent; a retirement is the same call and the watch goes on; a + later write delivers the new incarnation as `[["r", value]]` (a retire and recreate in one transaction: one call + with `retired: true` and the new value); + - watches end `ended` with their object. +- **`handle.at(key)`** in the factory: a view of the keyed handle's slot for that key (the same tracker, the same draft + in a transaction: never a second document), pinned while the declaring attempt lives, `initial()` while absent; a + state member, a derive source, an `on` source. + +## 12. Keyed state members (decision "Keyed state members, `local` contracts only") + +- `keyed(schema)` is a third member kind (`MemberKind` `"keyed"`), part of the fingerprint (normal form updated), only + in `local` contracts (`remote()` refuses it at the type level and at declaration). +- **Provider:** returns a keyed document handle for it (in memory or persisted), or re-exposes another reference's + keyed member; checked at activation as state members are. +- **Consumer on the same host:** `ref.member.at(key)` only in its factory (a `Watchable`: a pin on the provider's + per-key source while the consumer's attempt lives; loaded before the consumer goes live; `initial()` while absent; + the `retired` flag on retirement); `ref.member.get(key, ctx)` and `ref.member.watch(key, ctx)` anywhere (`watch` on an + absent key gives `undefined`). +- **Hydration:** `host.get` of a consumer waits for its declared `at(key)` sources like any state member's value; keys + read dynamically are not part of hydration. +- A key whose `migrate` fails fails its owning object `broken` (section 1.4), never the store. Versions of keyed members map per key in slice 7. +- Nothing on the wire changes: across links a view per entity is held as an ordinary remote object. + +## 13. Export and import (decisions "Export archives are operations", "Writes carry their versions; archives are lossless", small items 6-7) + +- `store.export(ctx)` yields `AsyncIterable`: the state after its `lastSeq` as writes, lossless: a + manifest `{ kind: "manifest", format: 1, lastSeq, schema }`; per table its `table` write (with `state`), its `put`s + (stored values and versions) and its `index` writes; the `document.create`s (stored version and fingerprint; for + `latest` the current value as the base); an `end` with the counts. Retired `latest` records, blobs, the outbox and + sync state are not in it; tables of plugins that are not loaded travel along. + + ``` + {"kind":"manifest","format":1,"lastSeq":4812,"schema":1} + {"kind":"table","table":"session:file-history.store:edits","version":4,"state":"ready","columns":{…},"key":["id"]} + {"kind":"put","table":"session:file-history.store:edits","row":{…},"v":4,"columns":{"meta":1}} + {"kind":"index","table":"session:file-history.store:edits","name":"by_path","columns":["path","at"]} + {"kind":"document.create","record":{…},"base":{…}} + {"kind":"end","writes":10061} + ``` + +- It takes the commit queue at its first `next()` and holds it until the iteration ends, throws, is closed + (`return()`) or `ctx` aborts (an abort listener releases it even while the consumer is suspended; a later `next()` + rejects `aborted`); it reads page by page with plain reads (no long read transaction). The guide: export to a local + file, then upload. +- `importStore(backend, parts, ctx, { outbox? })` is a trust boundary: every part checked against one compiled + `ArchivePart` schema, every name against the name rules, every row against its table's recorded columns, the order + above and the `end` counts; it refuses a non-empty backend (one with the store's tables) and parts after `end`; + applies everything in one backend transaction with the replica's applier (one code path for import, bootstrap and + batch apply) and returns the store at the manifest's `lastSeq`, its outbox (if asked) starting after it. +- The encoding is pluggable: the store yields and takes parts; it ships plain JSONL (`encodeJsonl(parts): + AsyncIterable`, `decodeJsonl(bytes: AsyncIterable)`). + +## 14. Benchmarks and tests + +- **Benchmarks** (`test/next/bench/store.ts`, results in `benchmarks.md`), the gate (decision "The commit-step + benchmark"): the store without outbox no slower than Durable's `SqliteStorage` per commit on (a) one document delta, + (b) one table `put` (against `SqliteStorage`'s entry append), (c) 1,000 commits back to back, with and without a + watched document; bursts (median and p95 over several runs), memory after GC, the same pragmas and file system. The + outbox's extra cost reported separately; also query pages, a 10k-row `updateWhere` (its step time: bulk writes block + every owner's commits), text per row, a migration chunk. A failed gate goes to the owner before the build continues. +- **Cost guards** (in-memory telemetry adapter): validations (`migrate` results), materializations (document loads), + store commits and statements per commit; none on the in-memory-only path; a cached keyed `get` touches no SQLite. +- **Tests,** one `describe` per mechanism: facets that only declare; slots, handles and job runs on the object across + retries, failures and native restarts (R15 through old handles); the seal fence (sealed handles, both commit checks, + a queued commit of a sealed attempt rejected, an admitted one adopted into the object, the line released, cleanup); + a live object failing; the commit step (order, adoption inside the step, errors, `storage_failed` once, reads after + the stop); transactions (the line across the step, no abort once admitted, the in-memory part in the same step, + in-memory-only commits without a step); cold keyed loads on the line (a cold `get` racing a draft of the same key, a + cancelled waiter); the outbox (on, off, omitted, switching, `read` gaps, `trim`, `format`); tables (declaration and + name checks, DDL, P3 errors, values, text, nullable columns); queries (every operator, `NULL` rules, order and + tie-breaks, cursors, keys as objects, reads inside and outside transactions, W10, bounded SQL texts); writes (all + five, `set` forms, the swap, upserts, counts, their outbox writes); migrations (`json(...)` on access, fingerprints, + eager steps, chunks, resume, indexes, the paused deadline with a fake clock, a timer firing during a long migration); + the documents engine (rules, ids, pruning, the singleton address); persisted documents (load once per object, + fingerprints, `migrate`, keyed `get`/`keys`/eviction and pins, checkpoints, the migrated base written back); + retirement (singleton, keyed, `at(key)`, `self.on`, derives carrying the flag, the wire with a full window); keyed + state members (pins, hydration, retirement, two consumers of one key); export and import (round trips with old + versions, mid-migration and uncheckpointed deltas, every refusal, the queue released on abort while suspended); GC + (evicted documents, unpinned keys, ended objects, stopped watches, export iterators closed early, abort listeners). + Examples: `20-tables.ts`, `21-persisted-documents.ts`, `22-migrations.ts`, `23-keyed-watches.ts`, + `24-keyed-members.ts`, `25-export-import.ts`. + +## 15. Rules from the second spec review (decision "Second spec review: the majority") + +The owner delegated the second review's findings to the reviewers' majority (`/tmp/s6a-review2-{astra,fable,opus,sol}.md`; +a point raised by one reviewer only takes that reviewer's recommendation). These rules complete the sections above; +where a section above says otherwise, this section wins. + +**Objects, attempts and the fence** +1. **Slot versions:** a slot keeps `valueVersion` (the version of its value in memory, the handle fence, R15) and + `storedVersion` (the version on disk, for write preparation), with their fingerprints, as Durable's + `LoadedDocument`. An old handle whose version is below the slot's fails that access, not the owner; a stored version + newer than the current definition fails the owner (1.4). 6a builds the version comparison and the in-memory + `migrate` (tested through a retry whose factory declares a newer definition); slice 7 wires it to reload. + `initial()` is retaken when the declaring definition's version or fingerprint changes, not on every attempt. +2. **The store's admission check:** `store.commit(writes, { admit, adopt }, ctx)` (internal): `admit()` is called at + the step's entry (false: the store rejects `released` and drops the entry, never writes it); `adopt(seq, ids)` + after `COMMIT`, `ids` mapping the commit's `document.create` writes by index to their ids. 6b's `StoreStorage` + passes `admit: () => true`. +3. **A transaction at the seal** is discarded: drafts aborted, every later `tx.*` throws `released`, a callback that + returns afterwards is rejected `released` without a commit, and the line is released. As Durable's `Transaction`, + it tracks its pending async `tx.*` operations and refuses a callback that settles with one still pending. +4. **Sealed handles** may still read synchronously (`.value`, cached keyed reads); anything taking the line (`change`, + `retire`, a cold `get`/`watch`/`keys`) is refused `released`. A live failure of an already sealed attempt is ignored. +5. **Jobs at the seal:** staged changes of an admitted commit are adopted into the object (section 1.3). A run cut off + by its attempt's seal ends `failed { aborted, reason: "restart" }` (design §15's reason, now also "the attempt + ended while the object continues"); its outcome transaction runs on the object's line fenced by the run's own token + (a transaction path owned by the object, not an attempt). A `finish` sealed mid-way is refused at the step and + retried on the next live attempt, not counted as one of the five failures. A run's state-change admission closes + when `run` returns; changes already admitted drain, then the outcome is recorded. +6. **Pins between attempts:** the object holds the factory's `at(key)` pins until the next factory returns (the moment + unused in-memory slots are dropped). Each attempt loads its declared singletons and pins not in memory (line jobs) + before it goes live. A consumer whose pin load fails the provider fails as a dependency failure (`transient`, R18). +7. **One host per store:** `createHost` refuses a store attached to an open host (`invalid/args`). + +**The commit step and errors** +8. **Serialization before `BEGIN`:** every write is serialized when it is staged (table writes) or prepared (documents, + before the queue); a value `JSON.stringify` cannot encode (a cycle, a `BigInt`) fails there (`invalid/args` or the + transaction), never in the step. +9. **Order inside one commit:** retirements before creations (the partial unique index on the live address), then + changes, rows, the outbox commit, `lastSeq`. +10. **Integers:** an `int` column has a SQL `CHECK` keeping it within `±(2^53 - 1)`; an expression result outside it + (`add` past the safe range) fails the step, which is `storage_failed` (accepted, as the 2^63 case was); the JS + evaluator (the `tx.get` overlay) checks the same range and refuses out-of-range results. `add`/`sub` take `int` + columns only; `max`/`min` take `int` or `num`. +11. **Reads after a failure:** as Durable's `GuardedStorage`, a read that settles after the store stopped rejects + `unavailable/storage_failed`, never returns a value. A shared cold load runs under the owner's context, not its + first waiter's. +12. **Open and import failures:** `openStore` and `importStore` close the backend they were given when they reject; + a refused import rolls back every table it created. An import whose input iterator stalls is cancelled by `ctx` + without holding the backend transaction past it. + +**Tables, queries, writes** +13. **Key columns are immutable** in `update`, `updateWhere` and migration patches (types and staging, `invalid/args`); + moving a row is `delete` plus `put`. +14. **Canonical writes:** both `set` forms compile at staging into one data-only expression vocabulary, with an + explicit `null` literal for a clear (`{ note: undefined }` would serialize to `{}`); the outbox and archives carry + that form, never callbacks or `undefined`. An empty `set` is `invalid/args`. +15. **Defaults:** a column with a `default` may be omitted in a `put` (SQL `DEFAULT`); reads always return it. +16. **Operands:** comparing an `opt` column with `undefined` is `invalid/args` (ask with `isNull`). +17. **Pages:** `limit` is a positive safe integer; `rows()` pages internally by 100; `next` comes from fetching + `limit + 1`; a cursor is a position in the order (one from another `where` with the same order is accepted); + duplicate `orderBy` columns are refused and the key tie-break is not repeated. +18. **The prepared-statement cache** of `chord/sqlite/node` is a fixed LRU (pi change, upstream first, before the + build): arbitrary query shapes cannot grow it without bound. + +**Documents and keyed members** +19. **Retiring a `latest` record** deletes its row and revisions (Chord never reads history; a key retired and + recreated often leaves no dead rows); `full` records keep them (6b). The record's counter is `revision` (not + "version") in the conditional `UPDATE`. +20. **Absent keys:** a keyed `get` of an absent key is not cached (a line job plus a read each time; accepted, stated in + the guide); `watch(key)` on an absent key returns `undefined`, and a later creation is seen through `at(key)` or a + new watch. +21. **Retired derives:** a derive over a retired source publishes `retired: true` even when its value is unchanged and + its ops are empty (`Derive.deliver`'s identity and empty-diff shortcuts must not swallow the flag). +22. **Keyed members:** a re-exposed keyed member resolves to the ultimate provider's source (no second tracker); pins + are released exactly once (failed consumer activation, retry, release, re-exposure); `ref.member.get/watch` while + the provider is loading or failed behave as calls (park while loading, reject with the cause when failed); keyed + members have no `keys` (a method lists keys); `keyed(...)` adds a node to the fingerprint without bumping + `NORMAL_FORM_VERSION`. +23. **Eviction's idle structure** is Durable's (`#touchDocument`'s granule rule: one move per tenth of the retention, + O(1) per touch); one owner-level timer. + +**Export and import** +24. **The manifest carries `nextDocumentId`** (the id high-water mark), restored atomically and checked against the + imported records; import keeps ids and `createdAt` seqs. +25. **Import validation** also covers record ids and seq ranges, one live record per address, versions, duplicate + definitions and agreement with the manifest; `json(...)` cells are checked as JSON text only (no plugin code runs). +26. **Export cancellation** releases the queue and its page buffers and metadata references (GC-tested with a retained, + aborted iterator). Code that commits while consuming its own export waits until `ctx` aborts (the guide; a test). + +**Benchmarks and cost** +27. A fourth gate case, (d) a commit with one in-memory document only (the store adds nothing to the W12 path); queue + wait reported separately from step time; late migration chunks measured as well as the first. + +**Carry-over (to slices.md):** slice 7: remote holds and remote objects leaving `live` (`Hold.moved`, +`RemoteObject.failed`; not reachable in 6a, P3), old table handles' version fence, the routine handover fenced per +routine invocation, holds re-bound by member name, keyed members' versions per key, `rebuild`. + +## Races + +1. Two objects commit at once: their steps queue; each adopts inside its own step, in seq order. +2. A transaction discarded after its callback while another owner's step runs: it never entered the queue. +3. The store fails during a step: that commit rejects; later commits and reads reject `unavailable/storage_failed`; + the host closes. +4. An object's attempt is sealed while its commit waits in the queue: rejected at the step's entry; one already in the + step lands and is adopted into the object's slots; the next attempt drafts from them. +5. A cold `get` and a transaction's draft of the same key: both on the line; the load installs only while absent. +6. A keyed document evicted while a `get` for it is in flight: eviction runs on the line and skips pinned keys. +7. A migration crashes in the middle: the next activation finds `version N, migrating` and continues. +8. An export holds the queue while commits and migration chunks wait; the export's `ctx` aborts while its consumer is + suspended: the queue is released. +9. A retire and a recreate of one key in one transaction: one call with `retired: true` and the new value. +10. Old code (after a rollback) meets a newer table, column or document version: the object fails `broken`, data + intact; a newer outbox `format`: `outbox.read` rejects, data intact. +11. The outbox is switched on for a store with data: the metadata says from which seq it starts. + +## Cost + +| Work | Cost | +| --- | --- | +| commit of one document delta | one step: about three statements, plus one with an outbox, plus one on a checkpoint | +| commit of in-memory documents only | as today: no step, no seq, no await | +| table put | two statements in the step (the row, `lastSeq`), plus its write in the outbox commit | +| `updateWhere` over N rows | one statement, one write; holds the queue for its duration | +| read of a page | one query | +| keyed `get` (cached) | no SQLite; one touch of the idle structure (Durable's granule rule); (cold) a line job and one or two reads | +| activation of an object with persisted singletons | plain reads, once per object | +| a `json(...)` column read at the current version | one `JSON.parse`; an older version adds `migrate` and one check | +| `rows` migration | about 15 µs per row, one commit per 100 rows | +| a keyed commit with no pins on the key | one `Map.get` | + +## Carry-over taken + +From `slices.md` "Slice 6a": facets only declare (1.1), documents, line and job runs on the object (1.2), keyed state +members (12), W12 and W7 (sections 3, 5, 8), retirement in Chord (1.5, 11). + +## Carry-over left + +- Slice 6b: persisted jobs, `StoreStorage` (Durable's record tables in the store: declared tables or internal), reads + at an older seq, `history: "rewindable"` export, `durableView`. +- Slice 7: in-memory `migrate` on reload; the routine handover fenced per routine invocation; holds unsubscribed at + seal and re-bound by member name; keyed members' versions per key; a native restart drains first (decided); `rebuild` + (key or column-type changes) if a case needs it. +- The sync layer: upload, acknowledgement and `outbox.trim`, epochs, pull-back, the split, snapshot reads for large + exports. +- Keyed members across links (Fable's design or selector holds) when a real case needs them. + +## Size + +About 3,000-3,500 source lines (the store core about 1,650, persisted documents about 400, tables and migrations +about 600, retirement and keyed watches about 200, keyed members about 300-350, the object/attempt/fence rework about +300), minus the facet code removed, plus the tests. diff --git a/packages/chord/docs/next/slice-6b.md b/packages/chord/docs/next/slice-6b.md new file mode 100644 index 000000000..9879fb104 --- /dev/null +++ b/packages/chord/docs/next/slice-6b.md @@ -0,0 +1,244 @@ +# Slice 6b: Durable and jobs on the store (spec) + +Status: **draft, open points marked "Decision needed"** (2026-10-27), written by the redesign session after reading +Durable's `Storage` interface (`packages/durable/src/types.ts`), its SQLite storage, schema and migrations, the storage +conformance suite and benchmark, `session/session.ts`, `session/transaction.ts`, `session/forks.ts`, and Chord v2's +store engine, applier, schema, commit step and jobs, in full. Durable's memory, JSONL and IndexedDB storages were not +read (they stay as they are, S15). + +The single source for building 6b, together with: `design.md` read in full; the decision log +(`packages/chord/docs/next/decisions.md`, later entries win), in particular "The store", S1, S2, S4, S10, S11, S13, S15, +W1, W12, W13, "Slice 6 first decisions", "Durable observers", "Durable's synchronous observers", "Text is plain UTF-8", +"Failures are watch calls", the "One vocabulary" entries, "The performance and memory bar"; `slice-6a.md` (what 6a +built); the feature map and the carry-over of `slices.md` (slice 6b); the root `AGENTS.md` "Chord v2 Code Style". + +6b puts Durable on the store (`StoreStorage`), follows Durable's views as Chord sources (`durableView`), persists jobs, +and extends the applier to every write kind. After 6b a worker has one store: Chord's objects and Durable's Session +commit through one queue, and a commit of either is one seq in one outbox. + +## What an app can do after this slice + +```ts +// the start code of a session's worker (p3 base code) +const store = await openStore(sqliteBackend(db), ctx); +const worker = createHost

({ address: Session({ id }), principal, store, onError }); + +// the core facet: the harness object opens Durable on the same store +host.serve(Harness, (_, self) => { + const durable = self.open(async (context) => + Durable.open(storeStorage(store, { owner: "session:harness" }), { …, paused: true }, context), + (h, _reason, context) => h.close(context)); + const view = self.follow(...) // Decision needed (D9): name and shape + return { … }; +}); + +// a persisted job: survives a crash +const ImportJob = defineJob("notes.import", { + version: 1, args: obj({ path: str() }), result: obj({ count: int() }), + state: ImportState, persist: { resume: "rerun" }, +}); +const imports = self.job(ImportJob, { run, finish }); +await imports.abort(id, ctx); // stored; Decision needed (D7): the result +``` + +## 1. `StoreStorage`: Durable's `Storage` on the store + +Durable's `Storage` (types.ts, "Atomic persistence boundary for Session records") implemented over a `ChordStore`, so +Durable's Session runs unchanged on it and passes Durable's storage conformance suite +(`createStorageConformance`), across reopen too. + +**Where it lives (D1, owner, 2026-10-27: "they are only used if durable is used"):** `StoreStorage` and +`durableView` belong to Durable: `packages/durable/src/next/`, private-only like `packages/chord/src/next`, importing +`@earendil-works/chord/next` (Chord's `./next` package export, private-only, returns for it). Durable already depends +on Chord and Chord imports nothing of Durable (S13); Durable's conformance suite runs in Durable's tests; the folder +goes upstream with Chord v2. + +### 1.1 Records + +Durable's four record tables (conversations, entries, tasks, submissions) and its ID namespace. + +> **Decision needed (D2): how Durable's records are stored.** +> (a) **Declared tables** in the store, written as ordinary `put` writes: `defineTable`s that `StoreStorage` declares +> for itself (SQL names `":

"`, e.g. `"session:harness:entries"`), each row the record's indexed columns +> (`id`, `conversation_id`, `kind`, `status`, …) plus the whole record in a `json(...)` column. The outbox, export, +> import and the applier handle them as they handle every table; Durable's layout changes later through +> `evolveTable` (Durable's own SQLite migrations, such as v2's `sleeping`, become table versions). Durable's +> secondary indexes map to declared indexes; the partial head index becomes an index on `(conversation_id, has_head, +> id)` (index columns are non-`opt`). Indexed strings that must stay lossless (`kind`, `requestId`, document kinds and +> keys) keep Durable's JSON encoding in their columns (decision "Text is plain UTF-8"). +> (b) **Internal tables** of `StoreStorage` with write kinds of their own (`durable.record`), which the outbox, +> export, import and the applier would each learn. More code, a second table mechanism. +> Proposal: (a). + +- **Writes:** every `StorageWrite` of a table record becomes a `put` of its row (Durable replaces complete records; + conversations and entries are immutable and written once). `record_ids` is a declared table too (`id`, + `record_type`), one `put` per new id (decision "Slice 6 first decisions" (3)). +- **Reads:** `conversation`, `entry` (with its `commitSeq`: a column), `findLatestHeadMarker`, `scanEntries` (both + orders, the fork chain walked as `SqliteStorage` does), `task`, `scanTasks`, `submission`, `scanSubmissions`, + `submissionByRequest`, `scanConversations`: queries over the declared tables, Durable's cursors (`{ after, order }`, + `scan.ts`) kept as they are (built from the last id, not from Chord's table cursors). +- **Validation before the step** (D4 below): what `SqliteStorage` checks inside its transaction + (`checkGlobalIds`, `prepareDocumentActions`, `checkDocumentActions`) runs before `store.commit`, with plain reads: + only Durable's Session writes these tables and it serializes its commits, so the reads see the state the step will + write on (the argument of S11). + +### 1.2 IDs + +> **Decision needed (D3): one id counter or two.** Durable mints ids before a commit (`mintId`), one namespace across +> conversations, entries, tasks, submissions and documents ("greater than every id minted since this storage was +> opened and every id stored, also across reopen", S10); its documents carry their id into the commit. Chord's store +> assigns its documents' ids inside the step from its own counter (`next_document_id`), and the engine's +> `chord_documents.id` is the primary key for both. Options: (a) **one counter per store**: `mintId` takes from the +> store's counter in memory (persisted as a high-water mark with every commit), Chord's step takes from it too; Chord's +> document ids are then no longer monotonic with seq (6a small item 11) and a Durable id minted but never committed +> is skipped; (b) **two counters**: Durable's from `record_ids` (`coalesce(max(id), 1) + 1` at open, decided), Chord's +> as today, and the engine's documents table keyed by `(namespace, id)` (Durable's document ids live in Durable's +> namespace). Proposal: (b): Durable keeps its id contract unchanged and Chord its step-assigned ids; the composite key +> costs nothing measurable and keeps the two adapters apart (S13: the engine is shared, the namespaces are not). + +### 1.3 Documents + +Durable's documents on the documents engine (S13): address `(namespace "durable", owner, kind, key?)`, owner the +scope as a string (`session`, `conversation:7`, `task:31`), kind and key JSON-encoded as Durable's indexed strings. +6a built `latest`; 6b adds what Durable needs: + +- **`rewindable` records** keep every revision and are not deleted at retirement: `retired_at` is set, revisions + stay (decision "Writes carry their versions" already defined the archive fields). +- **Reads at a point:** `findDocument(address, at)`, `document(id, at)` (the base at or below `at` plus the deltas up + to it, in one read transaction, S4), `scanDocuments(scope, at, kind?)`, with Durable's half-open lifetime + (`createdAt <= at < retiredAt`); a historical read of a `latest` (or session or task) record is a + `StorageRequestError` ("does not retain historical content"). +- **`document.copy`** (S11): `StoreStorage` reads the source at its point before the step and enters the step with a + plain create carrying the base; the conformance rules (source not changed in the copy batch, a matching record) are + checked there. +- **Stored version and `deltasSinceBase`** as `StoredDocument` reports them; Durable's records carry no fingerprint: + the engine's fingerprint column takes `""` for Durable records. +- **Export** carries every revision of a `rewindable` record, retired ones included, with its seq (format 1's + fields); import and the applier take them. + +### 1.4 Commits + +- `StoreStorage.commit(writes, adopt, ctx)`: validates (1.1), resolves copies (1.3), stages the writes (table `put`s, + document creates, changes and retirements) and calls `store.commit(staged, { admit: () => true, adopt: (seq) => + adopt(seq) }, ctx)`. The store's step runs `adopt` inside `host.batch.commit(...)` when the store has a host, so + Durable's observers mark Chord sources in the step (W1, "Slice 6 first decisions" (1)); `adopt` never throws + (Durable's rule). +- **Durable's seq is the store's seq** (gaps allowed: Chord's commits take seqs too). +- **Errors:** + +> **Decision needed (D4): what a refused Durable commit does to the store.** The store's rule: any backend error in a +> step is `storage_failed`, the store stops (Durable's own rule for its Session). Durable's conformance suite commits +> invalid batches on purpose ("ID 1 already belongs to conversation", "already has a current incarnation", "version +> transition requires a base", a mismatched copy) and expects the storage to go on. Proposal: those checks run before +> the step (1.1) and reject there, the store untouched; only a backend failure inside the step stops the store. In +> production the Session fails on any commit error either way (`GuardedStorage`), so the difference is only that the +> store, shared with Chord's objects, does not stop for a Durable bug caught by its checks. + +- `mintId`, `close(context)` (closes nothing of the store: the app owns it; later calls reject "closed"), reads after + a store failure reject as Durable's `GuardedStorage` expects. + +### 1.5 Statements per commit + +A target to measure, as 6a's: an entry append is the `record_ids` put, the entry put, `lastSeq` (plus the outbox); a +document delta is 6a's three statements. `SqliteStorage` does the same work today (`record_ids`, the row, metadata). + +## 2. `durableView`: Durable's views as Chord sources + +A Chord source over Durable's synchronous observers (`observeConversation`, `observeTaskGraph`, `observeDoc`; +decision "Durable's synchronous observers"): `advance(call, ctx)` records the call; on the store, `adopt` runs inside +the host's batch, so the source is marked in the step and its frame goes out in the flush with the rest of the commit +(one message per link per commit). On a Durable storage that is not the store (tests, other apps), calls arrive outside +any batch: the source opens one batch per Durable commit on the next microtask (carry-over). Durable's failures are +final: `end({ kind: "failed" })` ends the source's watches `failed`; `ended` ends them `ended`. A derive over two +Durable views of one commit sees both changes in one computation. + +> **Decision needed (D9): the API.** The carry-over names it `self.follow` (working name): `self.follow(observe)` in +> a factory, `observe` attaching an observer, returning a `Watchable` usable as a state member, a derive source and an +> `on` source, detached when the attempt ends. Options: (a) `self.follow((observer, ctx) => +> harness.observeConversation(id, observer, ctx, window))`; (b) a plain function `durableView(attach)` returning a +> native the object opens (`self.open`). Proposal: (a), named `self.follow`: the attempt owns the attachment as it +> owns a derive. + +## 3. Persisted jobs + +Design §15 "Persistence": `persist: { resume: "rerun" | "abandon" }`, only in `local` objects with empty params (P3, +at declaration); a run's record and state document in the owner's store; a recorded outcome resumes only `finish`; an +abort request is persisted; a restart limit; the job fingerprint stored with every record; the start trace carrier. + +- **Where records live:** + +> **Decision needed (D5).** (a) **Documents of the object:** each run's record is a keyed persisted document of the +> object (an internal definition per job, key the run id: `{ version, fingerprint, args, key?, phase, outcome?, +> abort?, restarts, trace? }`), its state document a second keyed document; both written in the transactions that +> already change them (start, state change, outcome, finish, block, drop), loaded at the object's first activation +> (all unsettled runs: `running` needs every one). (b) A table per job. Proposal: (a): the records are few, change +> in the same transactions as today, and reuse persisted documents (versions, checkpoints, export). + +- **Resume at activation:** a run recorded `finishing` resumes `finish` only; a run recorded `running` was cut off by + a crash: `rerun` invokes `run` again with the stored args and state, `abandon` records `failed { aborted, reason: + "restart" }` and finishes. +- **Restart limit** (aligned with Durable's `restarts`): + +> **Decision needed (D6).** Durable counts how many times in a row a crash cut a task off without the task changing +> its state since: open counts, a close does not, and at `restartLimit` the task ends visibly. Proposal for Chord: +> the same rule: at activation each run recorded `running` gets `restarts + 1` stored; a `ctx.state.change` resets it +> to 0 (progress); at 5 the run ends `failed { aborted, reason: "restart" }` with a `job_failed` report naming the +> limit (no `blocked { restart_limit }`: Durable dropped parking in #11139, and a blocked run would need a retry API); +> a clean close (`host.close`, a reload's native restart) records nothing: those runs end `aborted` (`stop`/`reload`) +> and do not resume. (Supersedes design §15's "then `blocked { restart_limit }`".) + +- **Abort** (owner, 2026-10-27: async and stored in 6b): + +> **Decision needed (D7): `job.abort(id, ctx)`'s result.** Cases: a running run (its signal fires, the abort is +> stored so a rerun after a crash does not run again but finishes `aborted`); a finishing run (its outcome is decided, +> nothing to do); a blocked run (dropped); an unknown or settled id. Durable's `abortTask` returns `"marked" | +> "terminal"` and throws for an unknown id. Proposal: `Promise<"marked" | "finishing" | "dropped">`, rejecting `gone` +> for an unknown or settled id (as `outcome(id)` does). + +- **Deferred state persistence:** + +> **Decision needed (D8).** Design §15: "a persisted state document may use the deferred persistence option (stored +> within about a second, flushed before the outcome is recorded)", as Durable's `CommitPersistence` `deferred`. +> Options: (a) per change: `ctx.state.change(fn, ctx, { persistence: { kind: "deferred", withinMs? } })`, as Durable's +> commit option; (b) per definition: `defineState(..., { persistDeferred: 1000 })`. Proposal: (a), Durable's shape: a +> deferred change is adopted and published at once, stored with the next immediate change of the document, before +> the outcome, at close, or after `withinMs` (default 1,000 ms); a crash loses it (acceptable for progress output). + +- **Fingerprint:** the job's fingerprint (args, result and state schemas) stored with every record; a record of the + same version with another fingerprint is `blocked { version }` (slice 7's family; the carry-over's "a job + fingerprint over args, result and state"). +- **Trace carrier:** the start's carrier (design §29) stored with the record; used in slice 9. + +## 4. The applier for batch apply + +6a's applier applies an archive's writes; 6b extends it to every `StoreWrite` kind (`alter`, `dropIndex`, `update`, +`updateWhere`, `delete`, `deleteWhere` with their recorded counts and versions, `document.change`, `document.retire`), +checked against the store's records (a trust boundary), one code path with import. A test replays a store's outbox +into an empty store and compares the two stores' exports. Applying a batch with a different count than recorded stops +applying (S17). + +## 5. Benchmarks and tests + +- **The gate** (decisions "The commit-step benchmark", "The performance and memory bar"): Durable's own storage + benchmark (`test/storage.bench.ts`: the write and read cases of `testing/storage-benchmark.ts`) on `StoreStorage` + against `SqliteStorage`, and Durable's `test/conversation-view-bench.ts` on both: each hot path within about 5%, + memory (allocation per op, retained after GC) no worse beyond noise; persisted job starts and state changes against + in-memory ones. +- **Tests:** Durable's storage conformance suite on `StoreStorage` (memory `:memory:` and across reopen); Durable's + Session and harness test files that take a storage, run on `StoreStorage`; durableView (one frame per commit, a + derive over two views, outside the store, failure and end); persisted jobs (rerun, abandon, a recorded outcome + resumes `finish`, restart counting and reset, the limit, abort stored, deferred state, fingerprints); the applier + (every kind, the outbox replay, refused counts); export and import with Durable records and `rewindable` history; + GC (observers detached with their attempt). +- **Examples:** `26-durable-on-the-store.ts`, `27-durable-views.ts`, `28-persisted-jobs.ts`. + +## Carry-over taken + +From `slices.md` "Slice 6b": persisted jobs (all of it), `StoreStorage` (ids, fork copies, `adopt` in the batch), +`durableView` (the batch on and off the store, `self.follow`), the applier for batch apply; `job.abort` async and +stored (decision of 2026-10-27). + +## Carry-over left + +Slice 7: a job fingerprint mismatch's `blocked { version }` resolution on reload; runs of jobs the code no longer +declares (with D5's records). Slice 9: the trace carrier's use. diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md new file mode 100644 index 000000000..ea6d73093 --- /dev/null +++ b/packages/chord/docs/next/slices.md @@ -0,0 +1,269 @@ +# Chord v2: the slice plan + +Status: approved (owner, 2026-10-22); kept current. `design.md` states the rules; this file says which slice builds +what, and (section "Carry-over") what a slice deferred to a later one. + +## Why slices + +Building bottom-up made each layer guess the needs of callers that did not exist yet; L2 grew generic machinery +(nodes, sinks, deliveries, publications) that nobody could follow, and was deleted. A **slice** is the thinnest +end-to-end path through every layer a feature needs, so every API has a real caller from the first line. Later slices +extend earlier code; reworking it when a real need appears is expected, guessing ahead is not. + +Per slice (owner, 2026-10-22): a one-page spec written by this session (`slice-N.md`, including the carry-over +entries it takes), the owner's approval, a build by a fresh Opus 5.5 subagent, reviews of the uncommitted code by this +session and by the reviewers Astra and Fable (`v2-review-astra`, `v2-review-fable`), fixes by the builder, the owner's +review of the uncommitted diff, commit. Deferred work goes into the section "Carry-over" below, under the slice +that must do it, in the same change (owner, 2026-10-22; moved here from `carry-over.md` on 2026-10-24). Every slice that touches a hot path (links, the store, relays) also adds a light benchmark script +(`packages/chord/test/next/bench/`, run by hand) for timing, memory and wire size of realistic payloads, with results +in `benchmarks.md` (owner, 2026-10-22). Every slice adds one or more runnable +examples (`packages/chord/test/next/examples/NN-topic.ts`, numbered across slices, in the format of Durable's +`test/examples/`), each on one topic: for the owner to run and dig around in, and as references for Chord's users +(owner, 2026-10-22). Implementers and reviewers read +the slice spec, the feature map below and the whole of `design.md` in full, and check the work against all three; a +slice spec's section citations are for orientation, not a reading limit (owner, 2026-10-22). + +## What exists + +- **L1** (`src/next/contracts`, `src/next/data`): contracts, params and identity, argument checks, principal keys, + errors. `index.ts` for apps, `internal.ts` for Chord's own code. +- **Slices 1, 2, 3a, 3b, 4a, 4b, 5, 6a** (committed; 5 `4a5b45c45`, 6a 2026-10-27): `src/next/documents`, `state`, `host`, `jobs`, `link`, + `remote`, `store`; examples 01-25; benchmarks in `benchmarks.md`. +- **Chord's shared code** (upstream pi): `src/watch.ts` (`Watch`, `NativeWatch`, `WatchHandle`, `WatchEnd`, + `NativeWatchable`), Delta (`track`, `diffRevisions`, `applyImmutable`, encoders, `PendingOps`, `appendText`), + context, JSON, output, SQLite drivers. + +## The slices + +Order (owner, 2026-10-22: routines and jobs before the store and reload). + +| # | Slice | In short | +| --- | --- | --- | +| 1 | **One host, local objects** | host, facets, local objects, references, in-memory documents, transactions, derives, watches | +| 2 | **Failures and natives** | `self.open`, activation failures and recovery, backoff, the drain, timeouts | +| 3a | **Remote objects over one link** | remote contracts, principals, variants, a memory link, the message format, holds, calls, one message per commit (no relay, no drops) | +| 3b | **Surviving drops** | re-holds, parking, `ref.state`, `host.hold`, `ref.retry()`, `linger` | +| 4a | **Links and relays** | host kinds, placement, routes, two-sided links, relays and principals, IPC and WebSocket, flow control, heartbeat | +| 4b | **Authority and housekeeping** | revocation, cycles across hosts, `host.references`, `host.activity`, size limits, compression | +| 5 | **Routines and jobs** | `self.routines`, `self.job` | +| 6a | **The store** | the store (backends, tables, queries, writes, commit queue, outbox, documents engine, migrations, export), persisted documents, `self.table`, keyed watches and retirement | +| 6b | **Durable and jobs on the store** | persisted jobs, Durable's `StoreStorage`, the Chord source over Durable's views (`durableView`) | +| 7 | **Reload and rollout** | plugin loading, staging, swaps and restarts, revisions, `accepts` in use | +| 8 | **Registries and the harness** | registries, `self.writable`, Durable's harness, `chord(fn, context)` | +| 9 | **Telemetry** | spans, events, metrics across hosts | + +The playground (`apps/chord-playground`) moves to the new code once slice 4 exists. Round-3 cuts (C3-C16) and +carry-over entries are decided in the slice that first needs them. + +## Feature map + +Every feature of `design.md`, by section, assigned to the slice that builds it. A slice's spec covers every entry +assigned to it; **implementers and reviewers check the slice spec, this map and the whole design** (owner, 2026-10-22). A feature missing here is a planning bug: add it before the slice that needs it. "Doc" means a documented +pattern, no code. Decided cuts are noted where they apply. + +| § | Feature | Slice | +| --- | --- | --- | +| 3 | `createHost` with `telemetry`, `onError`; host without address | 1 | +| 3 | `createHost` with `principal`, `principalKey` (3a), `address` (4a), `revokedBy` (4b), the store (6a) | 3a, 4a, 4b, 6a | +| 3 | `load([{ name, facet }])` with static facets; start barrier (host-lifetime objects live before `load` resolves) | 1 (failed plugins reported: 2) | +| 3 | `connect`, `accept`, memory link pair, incarnations in the handshake, protocol-version handshake | 3a | +| 3 | host kinds, addresses, `route` (`authorize`, `resolve`), relayed holds and calls, re-resolve after a drop | 4a | +| 3 | IPC and WebSocket links, trusted links, the ack-clocked window per origin (K unacked state messages, pending queues, call window N per origin), relays with flow control per origin (`origin`, `closed`, senders FIFO), heartbeat | 4a | +| 3 | timeouts: `closeTimeout`, call drain, `readyTimeout` (C13: two host timeouts) | 2 | +| 3 | provider-side `linger` (C7: kept) | 3b | +| 3 | park deadline (= `readyTimeout`, C13) | 3b | +| 3 | `host.activity` (keep-alive counts) | 4b | +| 3 | `host.root(fn, …)` for code outside objects | 8 | +| 4 | principal type `P`, links stamp principals, principal keys in identity | 3a | +| 4 | revocation (`revokedBy`, `host.revoke`, propagation to children) | 4b | +| 5 | `local` types: params, members, checked arguments (Q15), results unchecked | 1 | +| 5 | version and fingerprint match at acquisition; reserved member names (`then`, `release`, `state`, `retry`) | 1 | +| 5 | `remote` types, variants chosen by the factory (`{ variant, members }`), identity with principal key, calls checked against the binding's variant | 3a | +| 5 | `accepts` mappings at runtime (old consumers), `contract_mismatch`/`updating` | 7 | +| 6 | lifecycle, one object per identity, lifetime from kind and params (Q-A), factory, activation, ending | 1 | +| 6 | failure causes, backoff per identity, dependency recovery (R18) | 2 | +| 6 | `retryOn` | 8 (first independently living source: registries) | +| 6 | cycle rejection on one host (Q5) | 1 | +| 6 | cycle rejection across hosts (activation chain in hold headers) | 4b | +| 6 | shared store with per-principal views; worker-wide native owners; natives following a relocation | doc (1, 3, 2) | +| 7 | `self.get`, `host.get`, hydrated references, members as watchables, `release` | 1 | +| 7 | per-reference watches (released with their reference) | 1 | +| 7 | `ctx.get` (call-scoped references) | 3a | +| 7 | `ref.state` (tagged), `ref.retry()`, `host.hold`, calls parking, failed references | 3b | +| 7 | `ref.variant` as discriminant, `RefIn` (C15: no `inVariant`/`expectVariant`) | 3a | +| 7 | `state: false` (C8: cut) | - | +| 7 | optional dependencies as registries | doc (8) | +| 8 | `ctx` with telemetry and signal | 1 | +| 8 | `ctx.principal`, `ctx.incarnation` (R20), `ctx.get` (3a); relayed incarnations (Q13, 4a) | 3a, 4a | +| 8 | `ctx.trace` (C11) | 9 | +| 8 | a job's `ctx.jobId`, `ctx.state` | 5 | +| 9 | `defineState`, singleton and keyed handles, `self.state`, in-memory documents (P2) | 1 | +| 9 | `self.tx`, `tx.draft`, `tx.retire`, drafts revoked (R12), no existence check (R13), lines per object | 1 | +| 9 | `handle.change`/`handle.retire` shorthands (C14: kept) | 1 | +| 9 | `initial()` checked once per definition (R8, D3) | 1 | +| 9 | `persist`, `checkpointWhen` (required for persisted documents), persisted only in local empty-params objects (P3), eviction | 6a | +| 9 | keyed watches (`handle.watch(key, ctx)`), `handle.at(key)`, retirement as a flag (decision "One retirement mechanism") | 6a | +| 9 | versions, `migrate`, schema fingerprints on records | 6a (in-memory migrate on reload: 7) | +| 9 | ~~facet documents, a served object's transaction spanning its facet's documents~~ (removed: C16 accepted, owner, 2026-10-26) | 1, removed in 6a | +| 9 | rules across owners, reconciling routines | doc (5) | +| 10 | the store: backends, tables, query runner, writes, commit queue, outbox, migrations, export/import | 6a | +| 11 | `Watchable`, watches (Chord's `Watch`), the batch (W1), reentrancy | 1 | +| 11 | `NativeWatchable`, `self.writable` | 8 | +| 11 | Durable's views as sources (`durableView` over Durable's `observeXXX`) | 6b | +| 12 | `self.derive`, failures per member, lazy when unwatched, `mapShared` | 1 | +| 12 | container-width benchmark (R24) | 9 | +| 13 | `self.open`, close reasons, order, restart patterns (R16) | 2 (restarts on reload: 7) | +| 14 | `self.on` (W2b), `self.onLive`, `sleep`, `untilAborted` (C4 decided); the handover per object on reload | 5 (handover: 7) | +| 15 | `defineJob`, `self.job`, intent/run/finish, state documents, dedupe (Q9), `busy`, `running`, blocked `finish` (C3 decided); persistence, resume, restart limit; `accepts` in use | 5 (persistence: 6b; `accepts` and reload: 7) | +| 16 | `defineFacet

`, facet as declarations only (documents and routines removed in 6a) | 1, 6a | +| 17 | registries: `defineRegistry`, contributions, `compose`, versions (Q8), use-site guards | 8 | +| 18 | `constant(value)`, start data as a core object (`SessionInfo`) | 1 (core facet pattern: 4a) | +| 18 | platform bindings as core objects | doc (8) | +| 19 | `host.references` (presence) | 4b | +| 20 | flows kept by their owner, the `steps` helper, the Questions service | doc (5); deferred decisions #9, #11 | +| 21 | loading plugins (loader D), staging, atomic commit, pure swaps, native restarts, rollout, dev loop | 7 | +| 22 | message format, header schema, unknown ids, one message per commit per link, consumers apply ops, plugin errors | 3a | +| 22 | relays, size limits per part, compression at the origin, adapted bindings get diffs | 4a (relays), 4b (limits, compression), 7 | +| 22 | trace carriers (R5) | 9 | +| 23 | HTTP, channels, chunked transfers | after slice 9 (with the first real proof) | +| 24 | Durable integration: harness object, `chord(fn, context)` (Q6), model and env registries | 8 | +| 24 | `StoreStorage` | 6b | +| 27 | versions table: contracts (L1 done), registries 8, documents 6, jobs 5, link protocol 3a | as listed | +| 29 | telemetry spans, events, metrics, carriers | 9 (cost counters: every slice) | +| 30 | cost guards per hop | every slice | + +## Size + +About 9-10k source lines in total, as estimated for the layers (owner: fine, on the order of Durable). + +## Carry-over + +Obligations a slice deliberately left to a later one, under the slice that must do them, with where they came from. +Rules (owner, 2026-10-22): whoever defers something (this session, a subagent, a reviewer) adds it here in the same +change; a slice's spec includes every entry of its section, and its build and review check them off; a done entry is +deleted (the slice's spec and code are the record). Moved here from `carry-over.md` (owner, 2026-10-24). + +### Any slice (protocol and checks) + +- **Bump `PROTOCOL_VERSION`** for any message or reason change once p3 has shipped (slice 3b added `retry`, `loading` + and `failed`'s `retryIn`; slice 4a parts, the list `bound`, `hello`'s `calls`, `ack`, `ping`, `pong`, `closed`, + `origin` and the hold's `address`, `principal` and `incarnation`; all without a bump: nothing is deployed). (Owner, + slice 3b.) +- **The park deadline's per-type override** is not built: a parked call waits the reference's `readyTimeout`. Build + it only if a real case needs it. (Slice 3b spec.) +- Failed-check paths are TypeBox's instance paths (ambiguous for keys containing `/` or `~`); documented, not fixed. + (L1.) +- No CBOR; revisit only if benchmarks show encoding as a bottleneck (slice 3a's and 4a's do not). (L2 decisions.) +- **One N on every host:** a relay forwards calls at once, so a worker announcing a smaller N than the relay's could + see more running calls of one origin than it accepts (a protocol error). Every host announces 256; per-link N, if + ever needed, must bound the relay's forwarding too. (Slice 4a rework.) +- **The Durable conversation view's size on re-hold** is a known cost until pi #11327 (windowed view) lands. (Slice 4.) +- **`map(bytes())` is refused at declaration** ("unsupported keyword `~refine`"): TypeBox's `Record` copies the bytes + schema's check as an enumerable key, which the snapshot walk refuses. An L1 bug found in slice 3a, not fixed yet. + +### Slice 6b: Durable and jobs on the store (from slice 5 and L3a-d) + +- **Persisted jobs** (from slice 5): `persist: { resume: "rerun" | "abandon" }` built (record and state document in + the owner's store; only in `local` objects with empty params, P3, enforced at declaration); outcomes and abort + requests persisted (a recorded outcome resumes only `finish`); the restart limit (five crash interruptions, + persisted, clean closes not counted; whether a state change resets the count; what a clean close records); deferred + persistence for state documents (flushed before the outcome is recorded); the `finish` failure count persisted; the + job fingerprint stored with every record; the run's start trace carrier (§29). +- **`StoreStorage`**: Durable's `Storage` on the store, passing Durable's storage conformance suite (memory and across + reopen); ids from a `record_ids` table (S10, W13); fork copies resolved before the step (S11); Durable's `adopt` run + inside the host's batch. +- **`durableView`**: a Chord source over Durable's synchronous observers (`observeConversation`, `observeTaskGraph`, + `observeDoc`; decision "Durable's synchronous observers"), marked inside Chord's batch in the store and grouped into + one batch per Durable commit on the next microtask outside it; `self.follow` (working name) and the Durable harness + carried in a registry entry. The task graph's `blocked` refresh arrives outside any commit. +- **The applier for batch apply** (slice 6a builder, 2026-10-27): 6a's applier (`store/applier.ts`) applies the + writes an archive holds (`table`, `put`, `index`, `document.create`), checked against the store's records; the + replica's batch apply extends it to every `StoreWrite` kind (`alter`, `dropIndex`, `update`/`updateWhere` with their + recorded counts and `v`, `delete`/`deleteWhere`, `document.change`, `document.retire`), one code path with import. +- Not carry-over any more: Durable's synchronous view-change hook is the upstream "Durable observers" change + (publication at adoption, `observeXXX`), which lands before 6a. + +### Slice 7: reload and rollout + +- **A native restart drains first** (owner, 2026-10-26): park new calls, await running calls (bounded by + `closeTimeout`), then close the natives and restart; a call outliving the drain is cut off by the seal fence. Running + job runs on a native restart: decide with the runs now owned by the object. +- **From slice 6a's second spec review** (decision "Second spec review: the majority"): remote holds and remote objects + leaving `live` (`Hold.moved` for bound and binding holds, `RemoteObject.failed` for a live object; not reachable in + 6a, P3); old table handles' version fence (old code issuing SQL for a dropped column); the routine handover fenced + per routine invocation; holds unsubscribed at seal and re-bound by member name; keyed members' versions mapped per + key; in-memory `migrate` wired to reload (6a builds the slot version check and the migrate); `rebuild` (key and + column-type changes) if a case needs it. +- **A run whose job the new code no longer declares** (slice 6a builder, 2026-10-27): 6a keeps it in its phase, + waiting, not invoked (unreachable without reload); slice 7 decides `blocked` with a reason (design §15's + `blocked { version }` family). +- **A job's new version on a live object** (slice 6a build review, 2026-10-27): 6a keeps the first declaration's job + definition in the object's records and refuses a later declaration with another version, so a host-lifetime object + (`local`, empty params) stays broken until the host restarts; slice 7 maps the records to the new version (or blocks + the old runs) on reload. + +- **Routines on reload** (from slice 5): the handover per object, bounded by `closeTimeout`; calls do not wait for it. +- **Jobs on reload** (from slice 5): `accepts` in use (args and result mapped forwards once per run; `blocked + { version }` without an adapter or for a newer version); when an older run's state document may migrate while its + old code still writes it, and an older run keeping its old handles valid until it settles; a native restart aborts + running runs (`aborted`, reason `reload`) and `finish` runs on the new code; a `finish` in flight across a reload; + (run records and state documents belong to the object: built in 6a); + blocked runs retried on reload; a job declaration removed while runs exist; a job fingerprint over args, result and + state (a schema change without a version bump); an `on` routine whose source failed and recovers after a provider + reload. +- **Instance ids on restart** (from slice 4b): a retry keeps an object's instance id; a reload that restarts an object + must give the new instance a new id, so an old chain never matches it. +- **`host.references` and removed types** (from slice 4b): a type's presence list lives from its first + `host.references(Type)` for the host's life; a reload that removes the type drops it. + +- **Holds across reloads** (from slice 3a): a hold's indexes belong to its bound version. A reload that still accepts + that version keeps the hold: a pure object re-subscribes the hold through the version's mappings and sends + `[["r", value]]` for each member whose value changed; a restarted object sends a fresh snapshot for the same hold (a + second `bound` on a bound hold, which 4a's consumer applies as a re-binding). A served hold whose object leaves + `live` while its `bound` waits for a slot goes back to pending (cut from 4a, where nothing leaves `live`; owner, + 2026-10-24: slice 7 builds and tests it in `Hold.moved` and `ProviderSide.unbind`). A reload + that drops the version ends its holds with `gone` (`contract_mismatch`) before the new code serves anything. A + consumer whose own code moves to a new version re-holds (new hold id) and releases the old. +- **Old-version calls** (L1): `checkArgs` returns `{ kind: "method"; args; result? }` or `{ kind: "call"; args; call }`; + invoke the handler or the provider's `call`, and map the result back with `result` for an old-version caller. +- **Derives on reload:** derives have no names (owner, 2026-10-22); exposed derives are matched by their member name, + internal ones are rebuilt; add matching only if a real need appears. (W1.) +- **Retrying broken failures:** reload retries a broken object as `ref.retry()` does (`ChordObject.retry`: the + backoff skipped, an attempt now; holders are told `loading`). (Slices 2, 3b.) + +### Slice 8: registries and the harness + +- **`host.root(fn, context, { principal? })`**: a call scope for code outside objects, built with its first users + (Durable hooks, `chord(fn, context)`); decide what an aborted, uncooperative `fn` means for its references (match + the handler and `CallContext` rules). (Slice 3b, owner.) +- **`retryOn`** retries a broken object as `ref.retry()` does. (Slice 2.) +- **`host.activity` on the facet's host** (`FacetHost`), if the harness or a core facet needs it; today only + `references` is there. (Slice 4b.) +- Registries recompose inside the owner's flush, never inside another owner's commit step; changing contributions are + native watchables (`self.writable`). (W4.) + +### Slice 9: telemetry + +- Spans for routine calls and job runs (from slice 5). + +- Round-3 cut to decide: C11 (`ctx.trace`). + +### When v2 goes upstream + +- `mapShared` lives in `src/next/state/derive.ts` and moves into Delta. (L2 decisions.) +- `withLinkedAbortSignal` lives in `src/next/host/ctx.ts` and moves next to `withAbortSignal` in `src/context` (pi + first): a call's context linked to long-lived abort signals until the call settles, instead of `AbortSignal.any`, + which keeps cleanup entries per call while its sources live before Node 24.16.0 (nodejs/node#62367). (Slice 4a.) +- The `./next` package export returns when the new code has an entry point (removed with the frozen-kernel move). +- A `./next/node` export for `nodeCompression` (`src/next-node/`), with `./next`. (Slice 4b.) + +### The sync layer (not a slice; when Radius sync is built) + +- Upload, acknowledgement and `store.outbox.trim`, epochs, pull-back, the split (S5, S7); a store opened with an + outbox (decision "The outbox is optional, internal and versioned") starts its sync from an archive. +- Snapshot reads for large exports on a second SQLite connection (S4), if exports get large. + +### Outside the slices + +- The playground's runtime tests fail on a stale `node_modules` copy of chord 1.3.0; relink with + `npm install --ignore-scripts` when the playground is used again. (Frozen-kernel move.) diff --git a/packages/chord/docs/next/store/book.md b/packages/chord/docs/next/store/book.md new file mode 100644 index 000000000..4f8a53f69 --- /dev/null +++ b/packages/chord/docs/next/store/book.md @@ -0,0 +1,665 @@ +--- +title: One Store: how a p3 session keeps, exports and syncs everything it owns +cover: One / Store +kicker: A design book · Chord v2 and p3 · 2026-10-21 +subtitle: Why Durable's transcript, Chord's documents and every plugin's own tables live in one transactional store, and how that store moves a session to another machine without coming apart +stats: + - 17 | review items decided (S1–S17) + - 1 | commit queue per store, one seq for every owner + - ~150 B | outbox write for renaming 10,000 rows (instead of ~40 MB) + - 0 | lines of plugin code a replica needs +note: | + Written for the owner from the discussion of 2026-10-21 and the store review by Fable and Astra. Every decision is in + the decision log (`packages/chord/docs/next/decisions.md`, entries "The store", "S1" to "S17", "Context last"); the + review is summarized in `apps/p2/docs/p3-reviews/store-review-summary.md`, bulk operations and migrations in + `store-bulk-migrations.md`, the earlier discussion with measurements in `session-data-scenarios.md`. Numbers were + measured on slayer with Node 24.4.1 and `node:sqlite` unless marked as estimates. Nothing here is built yet. Notes + marked *review* are points the reviewers raised that the design answers. +colophon: One Store · first edition · 2026-10-21 · based on the redesign decision log and the store review +--- + +# Before we begin {.preface} + +Mario works in session S1 on his laptop. The agent edits files, and a plugin called **file-history** remembers every +edit: which file, which conversation entry made it, the content before and after. It keeps those rows in its own table, +because it needs queries: every version of `src/app.ts`, every file the last turn touched, every edit under `a/`. When +Mario renames the directory `a/` to `b/`, ten thousand rows change. + +In the evening he wants to continue S1 on his desktop. Another day he exports S1 to a file and imports it into a +colleague's p3. Each time the other machine must end up with the conversation (Durable), the plugins' documents (Chord) +and the file-history table, and all three must agree: a row that says "entry 812 made this edit" must find entry 812, +the same entry 812. + +Copying files does not do that, and neither does syncing each storage on its own. This book explains the problem, the +solutions that were tried and dropped, and the design that came out: one transactional store per session, under +Durable, Chord and every plugin. + +## Eight things to know + +:::findings +1. **A session is self-contained.** Everything it owns lives in its store and its directory; it can be exported from + the session alone, imported elsewhere, synced and continued on another machine (P1). +2. **One store holds everything.** Durable's records (through an adapter), Chord's persisted documents and plugin + tables commit into one store, through one commit queue, numbered by one seq. +3. **Order is guaranteed where it is observable.** Disk, outbox and export follow the seq; in-memory adoption runs + inside the commit step (S1), so no reader ever sees seq 41 without seq 40. +4. **The outbox records what changed, not code.** Rows, column updates, bulk operations and table changes, written in + the same transaction as the data. A replica applies them without any plugin code. +5. **Tables are real tables with a small query language.** Keys and indexed columns are typed; text is UTF-8 and + compares by code point on every backend. +6. **Bulk writes stay small.** `updateWhere` records one operation instead of one row per change, and every replica runs + the same operation. +7. **Migrations run only where the code is.** Changes inside `json(...)` columns migrate on access; key and index changes run eagerly on the + driver and ship as ordinary writes. +8. **Losing control never rolls back.** A device that learns another one took over turns its copy into a separate + session. +::: + +## Words used throughout + +A **session** is one agent workspace: its conversations, tasks and plugin data. An **owner** is the code that writes a +piece of data: Durable's Session, a Chord object, a plugin's object. The **store** is the transactional store of one +session; a **backend** is what it runs on (SQLite, including in the browser as WebAssembly; tests use SQLite `:memory:`). A **table** is a set of +rows owned by one object. A **commit** is one atomic write; the **commit step** is the short, serialized part of a commit +in which the store checks, writes and numbers it; the **seq** is that number. The **outbox** holds a record of every +commit, written in the same backend transaction. The **driver** is the machine that writes a session; a **replica** is +a copy on another machine that applies the driver's records. **Adoption** is an owner updating its in-memory state to a +committed write. + +# A Session Is More Than Its Transcript {#problem lede="What has to travel with a session, and why separate storages cannot keep it consistent."} + +A p3 session holds three kinds of data written by three kinds of owners. Durable writes the transcript: conversations, +entries, tasks and its own documents. Chord objects write persisted documents: settings, drafts, plugin state. And a +plugin may need data shaped for queries, like file-history's edits. + +The owner's rule for p3 (**P1**) is that a session is self-contained. Everything of it lives in its storage plus its +directory; the server only mirrors it (the session catalog) and keeps instance data (shares, devices, settings). That +makes three things possible: export a session from its files alone, import it into another p3, and sync it to Radius +so another device can continue it. + +Durable alone already supports that shape: one storage, every change in an atomic commit, numbered by a seq. Armin's +sync design builds on it: a decorator captures each committed batch into an outbox, batches go to Radius, a replica +replays them. It breaks as soon as a session has a second storage. + +:::steps | One edit, two storages, synced separately ++++ On the laptop +The edit tool records the edit in file-history's own `history.sqlite` (row: path `x.ts`, entry 812). Durable commits +the tool's result. Both exist and agree. ++++ The history arrives first +The desktop has received `history.sqlite` up to the edit, but Durable's log only up to the commit before. The row says +entry 812 made the edit; the desktop has no entry 812. ++++ Mario takes over now +The laptop is closed; the desktop becomes the driver and writes its next entry, which gets id 812: a different +message. The row now points to the wrong entry, for good. +::: + +Each storage is consistent on its own. What is missing is one order across them: which change of `history.sqlite` +happened before which commit of the transcript. Every solution below is judged by whether it provides that order, +lets a replica work without the plugin's code, and keeps plugins able to model what they need. + +# What We Tried and Dropped {#dropped lede="The solutions discussed on the way, and the reason each one failed."} + +The first answers tried to avoid giving plugins storage at all. The later ones gave them storage and tried to sync it. + +| Idea | Why it was dropped | +| --- | --- | +| Plugins' own databases hold only derived data (indexes, caches), rebuilt after an import | too limiting: some plugins need their own write model and queries (owner) | +| Export plugin directories as opaque files | needs a stopped session; a changing database cannot sync in small pieces | +| Plugin export and import hooks | every plugin would build its own incremental log for sync | +| Plugin record logs in the session log, the plugin's database rebuilt from them by an `apply` function | the plugin must keep `apply` deterministic and every record version forever | +| Several storages, each with an outbox inside its own transaction, ordered by one shared position counter | works (prototype), but many outboxes and a "safe cut" rule; SQLite's row-change records are per row (a 10k-row rename is 259–418 KB); schema changes are not captured | +| A SQLite store that records the plugin's SQL statements, checked by a hash of their effect | SQLite only; arbitrary SQL can give different results on replicas (`random()`, unordered `INSERT … SELECT`, SQLite versions) | +| Raw SQL reads with controlled writes | ties every backend to SQLite and needs a guard against SQL that writes | +| An IndexedDB-shaped API only (keys, indexes, ranges) | too weak for real queries (owner) | +| Chord's documents inside Durable's existing storage | covers Chord, not plugin tables | + +The store design kept something from almost each: one order for everything (from the shared counter), recorded +operations instead of rows where possible (from the statement idea, made safe because the operations are the store's +own), and a small query language (from the IndexedDB shape, but over real tables). + +Inside the store, more choices were dropped during the review: + +| Choice | Why dropped | +| --- | --- | +| all rows of all tables in one generic table `rows(tbl, key, v, row JSON)` | the owner wants real tables; with table definitions in the outbox a replica needs no plugin code anyway | +| index entries maintained by the store in a separate table | an extra row per index per write; SQLite could no longer filter natively | +| ~~code point order (SQLite's native order) with lone surrogates replaced~~ | rejected at first, then chosen (owner, 2026-10-26, "Text is plain UTF-8, in code point order") | +| text only for equality, order only over integers (Durable's own approach) | ordered text, ranges and prefixes must be possible (owner) | +| JSON-field operands in bulk updates | they lean on SQLite's JSON functions; added only when a real case needs them | + +# The Store {#store lede="Layers, tables owned by objects, and the one transaction API."} + +The store sits between the backends and everything that persists data in a session. + +:::flow wide | The layers. Durable keeps its existing storages and gets one more, the adapter; Chord and plugins use the store directly. +row "Owners" blue +box durable "Durable Session" "`StoreStorage` adapter" +box chord "Chord objects" "persisted documents" +box plugin "plugin objects" "`self.table(...)`" +row "Store" accent +box docs "documents engine" "records, revisions, versions" accent +box tables "tables" "query language, writes" accent +box commit "commit step" "seq, outbox, adoption" accent +row "Backends" muted +box sqlite "SQLite" "Node, Durable Objects, browser (WebAssembly)" +box memory "SQLite :memory:" "tests" +arrow durable -> docs +arrow chord -> docs +arrow plugin -> tables +arrow docs -> commit +arrow tables -> commit +arrow commit -> sqlite +note "Durable's memory, SQLite and JSONL storages stay (S15), with only the adopt argument added (S1); StoreStorage joins them." +::: + +**A table belongs to one object**, like a document: only that object reads and writes it; its address is (host kind, +object type, table name), never a host instance such as the session id (W7), so a params change, a split or an import +never orphans data; other plugins see the data only through the object's +state and methods. + +```ts +const Edits = defineTable("edits", { + version: 2, + columns: { id: str(), path: str(), entry: int(), at: int(), meta: json(EditMeta, { version: 1 }) }, // flat + key: ["id"], + indexes: { by_path: ["path", "at"], by_entry: ["entry"] }, +}); + +host.serve(FileHistoryStore, (_, self) => { // a local object with empty params + const edits = self.table(Edits); + return { + record: (edit, ctx) => self.tx((tx) => { tx.put(edits, edit); }, ctx), + versions: ({ path }, ctx) => + edits.page(query(Edits).where((e) => eq(e.path, path)).orderBy((e) => e.at, "desc"), 20, ctx), + }; +}); +``` + +- A table's `columns` are a flat record: each column is a scalar (`str`, `int`, `bool`, `num`) or `json(...)`; nesting + exists only inside `json(...)`. Every column is a real SQLite column. Scalars can be filtered and ordered; keys and + indexed columns must be `str` or `int`. A `json(...)` column holds JSON text: typed when read, never queryable or + indexable (the type checker refuses it). +- Tables and documents of an object are written in the same transaction API, `self.tx(fn, ctx)` (S14). The context + comes last in every signature (owner's rule). +- Index **definitions** are recorded like other table changes: when the owning object starts on the driver, the store + creates or drops the difference between declared and existing indexes, and every replica applies it. Index contents + never travel; every backend maintains them from the rows. So every copy of the store is the same file. + +# Writing It Down {#commit lede="One commit queue per store, one seq for every owner, and adoption inside the step."} + +Two different things are serialized, and they must not be confused. A **line** serializes a whole transaction +callback: read, decide, write. Chord has one line per object, Durable one per Session. The **commit queue** serializes +only the final step: checks, writes, seq, the outbox's commit. It is short and there is one per store. + +:::sequence animate | Two Chord objects and a Durable turn commit at the same time. Their callbacks never wait for each other; only the short commit steps queue. +actor a "object A" "its own line" +actor b "object B" "its own line" +actor d "Durable Session" "its own line" +actor q "commit queue" "one per store" accent +a -> a: draft notes … +b -> b: draft drafts … +d -> d: phase writes … +a -> q: commit +q => q: seq 41: write, outbox, adopt A +b -> q: commit +q => q: seq 42: write, outbox, adopt B +d -> q: commit +q => q: seq 43: write, outbox, adopt Durable +::: + +Durable's seq is the store's seq. Durable sees gaps (other owners' commits in between), and that is harmless: every use +of a seq in Durable compares (`created_at <= at`, `seq > base`); nothing computes `seq + 1` to find a neighbour (checked +in the storages and the Session). + +## Adoption inside the step (S1) + +After a commit is on disk, each owner swaps its in-memory state to the committed version. If that happens later, in +the owner's own continuation, the order can flip: + +``` +disk: 40 file-history, 41 Durable +memory: Durable adopts 41 first (one await fewer) + a routine on "entry 812 committed" reads file-history → still at 39 +``` + +So adoption runs inside the commit step, synchronously, before the next commit starts. Durable's storage interface +carries it as a new required argument: + +```ts +commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, context: Context): Promise; +``` + +Every storage calls `adopt(seq)` once its commit is durable and before its next commit. The Session catches its own +adoption errors; a rejection after adoption fails the Session without discarding the adopted changes. + +## Adopt and mark in the step, flush right after (W1) + +The step itself stays short: it writes, adopts (pointer swaps of every changed value, including Durable's views the +commit touched) and **marks** (dependent derives dirty, ops appended by reference to each touched hold's batch and each +watch's queue). Right after the step, synchronously and before any other code runs, the committing owner **flushes**: +it recomputes the dirty derives someone holds or watches, encodes one part per touched hold into one message per link, +and queues watch callbacks. So a source and the derives over it always travel in one message; a throwing derive fails +only its own member, never the commit. + +``` +step 40: write · adopt notes · mark count dirty, notes' ops into tab 1's batch +flush 40: count = 3 → its ops join the batch · ONE message to tab 1 { notes: +1, count: 3 } +step 41: write · adopt Durable's view · mark lastReply … +``` + +Watch listeners run afterwards, each in order; their order across owners may differ from the seq, but every listener +reads a complete prefix of the log. + +## When a commit counts as done (S3) + +The step ends only on the backend's terminal success: SQLite's `COMMIT` returned (with an asynchronous backend, the +transaction's final completion event, not its individual requests). Any backend error in the step stops the store +(`storage_failed`, Durable's rule): an ambiguous `COMMIT` cannot be told from a failed statement, so nothing more is +written or published. + +:::review | Review: request success is not commit completion +Astra's trace: IndexedDB reports the puts of seq 40 successful, the store publishes 40, the transaction aborts later +(quota), 50 commits on top: memory has 40, disk does not. The rule above is the answer. +::: + +## Ids + +Each owner's id counter starts, when the store opens, above that owner's highest stored id: one index lookup per table +(S10). Chord's document ids are assigned in the commit step; for Durable's `mintId` (6b), an id handed out to a +transaction that never committed may be handed out again after a restart. Nobody can +notice, because nothing leaves a transaction before it commits (S2), so Durable's `mintId` contract is narrowed to say +exactly that. + +# Reading {#reading lede="A small query language, one text order on every backend, and cursors."} + +Queries are immutable data built in TanStack DB's style; S12 starts without joins, grouping and projections: + +```ts +query(Edits).where((e) => eq(e.path, "src/app.ts")).orderBy((e) => e.at, "desc") +query(Edits).where((e) => and(gte(e.at, from), lt(e.at, to))) +query(Edits).where((e) => prefix(e.path, "a/")) +``` + +`where` takes `eq ne lt lte gt gte inArray isNull prefix and or not`. `limit` and the cursor are arguments of the +runner, not part of the query. `count(q)` is the one aggregate; others are computed in JavaScript over indexed scans +until a real case needs more. + +## One text order + +Text compares by Unicode code point on every backend (owner, 2026-10-26; this replaced the first design, UTF-16 BLOBs +in JavaScript order, S16 and W8). That is the order SQLite compares UTF-8 text in natively, and Durable Objects, +sqlite-wasm, Postgres (`COLLATE "C"`) and MySQL (`utf8mb4_0900_bin`) do too. Every `str()` column is plain UTF-8 +`TEXT`: strings go to the driver and come back as they are, readable in `sqlite3` and in any other tool. + +It differs from JavaScript's `<` and `sort()` in one range only: characters U+E000-U+FFFF (fullwidth forms such as +`~`, private use, U+FFFD) against characters above U+FFFF (emoji): + +``` +JS sort(): "测试😀" < "测试(一)" +store order: "测试(一)" < "测试😀" +``` + +The store exports `compareText(a, b)` for plugins that sort store text themselves. Page with the store's cursors and use `prefix()`; a hand-built bound such as +`p + "\uffff"` misses every `p` followed by an emoji. + +UTF-8 cannot hold a lone surrogate, so every `str()` value written, and every query operand, goes through +`toWellFormed()`: a lone surrogate becomes U+FFFD, as `TextEncoder` does. A string cut in the middle of an emoji by +`slice()` is stored with U+FFFD in its place; cut text with `[...s].slice()` to keep it whole. A `str()` value may not +contain NUL (`invalid/args`); content belongs in `json(...)` columns, which escape both. There is no Unicode +normalization: "é" written as one code point and "e" plus a combining accent are two keys. + +| 100k path-like keys | UTF-16 BLOB (the first design) | UTF-8 TEXT | +| --- | --- | --- | +| insert, one transaction | 125 ms | 113 ms | +| read | 106 ms | 32 ms | +| bytes of an 18-character ASCII path | 36 | 18 | + +## Running queries + +A query is immutable data; the table handle runs it, and the context comes last: + +```ts +const edits = self.table(Edits); +const byPath = query(Edits).where((e) => prefix(e.path, "src/")).orderBy((e) => e.path); + +const page = await edits.page(byPath, 100, ctx, { after: cursor }); // { rows, next? } +for await (const row of edits.rows(byPath, ctx)) { … } // all pages +const e1 = await edits.get({ id: "e1" }, ctx); // keys are objects of the key columns + +await self.tx(async (tx) => { + for await (const row of tx.rows(edits, byPath)) { // committed rows, stable during the tx + tx.update(edits, { id: row.id }, { dir: dirname(row.path) }); // writes are synchronous, buffered + } + const again = await tx.get(edits, { id: "e1" }); // sees this transaction's own writes +}, ctx); +``` + +Inside a transaction, `tx.page` and `tx.rows` return committed rows: a stable view, because only the owning object +writes its tables and its transactions run one at a time. They do not show the transaction's own writes; `tx.get` does. +The rule in one sentence: queries inside a transaction see the state before it; your own writes become visible after +the commit. After an `updateWhere` or `deleteWhere` on a table, `tx.get` on that table throws `read_after_bulk` +(W10): the store would need a second evaluator to know what a bulk operation did to one row. + +## Cursors and consistent reads + +Every index ends with the primary key, so order is total and cursors are stable. A cursor is opaque, carries its +direction and its last key as logical values, is bound to its query's order, and is refused by another query or the +other direction. + +Reads outside a transaction see committed state, one query at a time. A read that needs several queries, like a +document assembled from its base and later revisions outside its owner's line (Durable's reads, 6b), runs in one backend +read transaction (S4); Chord's own loads run on the object's line and need none. Two separately +awaited reads in object code are not a snapshot, and the guide says so. + +# Documents {#documents lede="One documents engine in the store, used by Durable's adapter and by Chord."} + +Durable and Chord both persist documents the same way: a record per incarnation, revisions made of a base and later +deltas, versions, `checkpointWhen`, one live document per address. The store implements it once (S13): + +```ts +type DocAddress = { namespace: string; owner: string; kind: string; key?: string }; + +interface Documents { // reads, committed state + find(address: DocAddress, at: At, context: Context): Promise; + load(id: number, at: At, context: Context): Promise; // base + deltas, applied once + list(query: DocListQuery, limit: number, cursor: Cursor | undefined, context: Context): Promise>; +} +interface DocumentWrites { // inside the commit step + create(record: NewDocRecord, base: BaseContent): number; + change(id: number, content: DocContent): void; + retire(id: number): void; +} +``` + +The `owner` string is opaque to the engine; each adapter fills it: + +| | namespace | owner | kind | key | +| --- | --- | --- | --- | --- | +| Durable session document | `pi-durable` | `session` | `pi.settings` | | +| Durable conversation document | `pi-durable` | `conversation:7` | `pi.provider` | | +| Durable task document | `pi-durable` | `task:31` | `pi.codemode` | | +| Chord persisted document | `chord` | `session:notes.store` | `notes` | `n-42` | + +What the owner string means (a conversation's documents fork and rewind with it, a task's are retired with the task) +stays in the adapter. The engine keeps nothing in memory; Durable's Session and Chord's objects keep their Delta +trackers, cache, pin and evict exactly as Durable does today. A fork's document copies are reconstructed by +`StoreStorage` before it enters the commit step (S11), so the step never replays revisions while every owner waits; +the outbox only ever carries the resulting documents. + +# Writing Many Rows {#bulk lede="Bulk operations recorded as operations, and migrations that run only where the code is."} + +A full-row put per changed row makes the rename of `a/` to `b/` over ten thousand 4 KB rows cost about 40 MB in the +outbox. The store's writes are therefore five: + +| Write | Records | +| --- | --- | +| `tx.put(table, row)` | the whole row | +| `tx.update(table, key, set)` | the changed columns of one row | +| `tx.delete(table, key)` | the key | +| `tx.updateWhere(table, where, set)` | one operation, plus the affected row count | +| `tx.deleteWhere(table, where)` | one operation, plus the affected row count | + +```ts +await self.tx((tx) => { + tx.updateWhere(edits, (e) => prefix(e.path, "a/"), (e) => ({ path: replacePrefix(e.path, "a/", "b/") })); +}, ctx); +``` + +```sql +-- on SQLite: one statement; path is UTF-8 TEXT, "a/" is 2 code points +UPDATE "session:file-history.store:edits" SET path = :to || substr(path, 3) WHERE path >= :lo AND path < :hi +``` + +``` +outbox, seq 57 (about 150 bytes): +{ kind: "updateWhere", table: "session:file-history.store:edits", where: { prefix: ["path", "a/"] }, + set: { path: { replacePrefix: ["a/", "b/"] } }, count: 10000 } +``` + +A replica runs the same operation, without plugin code, and compares the number of affected rows. A different number +means divergence: it stops applying that store and restores it from a snapshot. The `set` expressions touch only real +columns (plain values, or column references in a callback): `add`, `sub`, `max`, `min`, `concat`, `replacePrefix`, +`coalesce`, `when`. They are the store's +own, specified once and checked on every backend by one conformance suite, which is why there are no SQL subqueries, +joins, case folding or floats: each of those behaves differently somewhere. + +## Migrations (S17) + +Adding, renaming or dropping a column is a recorded table change (`alter`); a new column gets its default without a +bulk rewrite. A change of the shape **inside a `json(...)` column** (a field added, renamed, restructured) **migrates on +access** (W11): each `json(...)` column has its own version and a value-only migration, +`json(MetaV2, { version: 2, migrate: (old, from) => … })`, stored with its value; reading an older value migrates it, +writing the column stores the new shape and version. Nothing is rewritten in bulk and indexes are unaffected. The row's +`$v` counts only eager steps (below); `update`, `updateWhere` and `deleteWhere` never set it. Old code refuses a newer +table version or a newer JSON column version. + +A change to a key or an indexed column must be **eager**, because an index over a column filled only on access would +silently miss rows. It runs when the owning object starts on the driver, before the object goes live: + +```ts +const EditsV4 = evolveTable(EditsV3, (t) => + t.add("dir", str({ default: "" })).rows((row) => ({ dir: dirname(row.path) })).index("by_dir", ["dir"])); +const EditsV5 = evolveTable(EditsV4, (t) => + t.add("size", int({ default: 0 })).update({ where: (e) => lt(e.size, 0), set: { size: 0 } })); +``` + +Each version is its own materialized definition (as contracts' `evolve`), so a store at version 3 replays 4 and then 5, +each with exactly its own columns. + +:::steps | Migration 4 on the driver ++++ The table change +Commit 80 records `alter: add column dir` and marks table version 4 as `migrating`. Every replica adds the column +from the record. ++++ The scan +The store reads rows with `$v < 4` in key order, 100 at a time (about 1 ms per commit step), applies `rows` to each and writes the changed columns +plus `$v = 4` in the same write. One commit per chunk, so other owners' commits interleave and the commit step stays +short. (A full-row `put` also sets `$v`; partial and bulk writes never do.) ++++ Ready +The last chunk marks version 4 `ready`. The store creates the `byDir` index and records it, and `FileHistoryStore` +goes live. ++++ After a crash in the middle +The next start finds `migrating`, scans again from the first key and skips rows with `$v >= 4`. Each row is +transformed exactly once, even when the transform is not idempotent. A key change is never a `rows` step: it is a +recorded `rebuild` that every replica performs itself. +::: + +A replica, with or without the plugin, applies the recorded column changes and the column updates; it never runs a +migration itself. Chord documents keep their own per-document `migrate` on access, and +Durable versions its records itself. + +# The Easy Parts {#easy lede="Blobs and derived data: why they need none of the store's ordering machinery."} + +Two kinds of session data stay outside the store's tables, and both are easy for the same reason: nothing about them +ever changes in place. A blob is final once written; derived data has no data of its own. + +## Blobs + +Artifact versions, uploads and sent files are **blobs**: files in the session directory, named by the hash of their +content (`blobs/sha256/9f/9f3a…`), never changed. They are app layer, not store: any process may write one, because a +file named by its content cannot conflict with another writer. The order is the whole trick: the blob is written first, +then the row or document that names it. + +:::sequence animate | Saving artifact "chart" version 3. The blob exists before anything refers to it. +actor tool "artifact tool" +actor blobs "blobs/" muted +actor store "store" blue +tool -> blobs: write sha256:9f3a… (tmp, fsync, rename) +tool => store: seq 41: chart.versions[2] = { blob: "sha256:9f3a…" } +tool => store: seq 42: entry 812 "chart v3 saved" (Durable) +::: + +- **Export** copies `blobs/` together with the store snapshot. Every blob a record up to the export's `lastSeq` names was + written before that record, so it is in the directory. **Import** copies them back and checks each hash. +- **Sync:** before uploading a batch, the sync layer uploads every blob file that is present and not yet uploaded. No + reference tracking is needed: a blob is always written before the commit that names it, so a replica never sees a + hash whose file is missing. +- **Deleting:** no blob is deleted before its session is deleted. A blob that nothing references any more (a crash + between the write and the commit, a retired document) only costs disk; a garbage collector would have to know every + reference in every table. + +## Derived data + +A full-text index, a cache, a plugin's own search database: **derived data** can be recomputed from the store and the +blobs, so it is never exported or synced. Its owner keeps it current with a routine that compares it with the store. +The artifacts plugin's full-text index in its own SQLite file: + +```ts +host.serve(ArtifactStore, (_, self) => { + const artifacts = self.state(Artifact); // keyed documents in the store + const blobs = self.get(SessionBlobs, {}); + const index = self.open(async () => openIndex(join(dir, "derived/artifacts/index.sqlite")), (db) => db.close()); + + self.onLive(async (ctx) => { // the index repairs itself + for await (const name of artifacts.keys(ctx)) { + const a = await artifacts.get(name, ctx); + if (a && index.value.versions(name) !== a.versions.length) + index.value.put(name, a.versions.length, await blobs.read(a.versions.at(-1)!.blob, ctx)); + } + }); + … +}); +``` + +:::steps | One routine, three situations ++++ Normal save +`save` writes the blob, commits the document, then updates the index. The routine finds nothing to do. ++++ Crash after the commit +The process dies between the commit and the index update. The document says version 3, the index still has version 2. +On the next start the routine sees the difference, reads the blob of version 3 and indexes it. ++++ Import or sync +On the desktop `derived/artifacts/` does not exist. The routine finds every artifact missing and indexes them all from +the documents and blobs that came with the session. +::: + +This routine visits every artifact on each start: fine for hundreds, slow for millions. A large derived store keeps +the last seq it applied and reads only what changed after it. + +# Moving a Session {#sync lede="Outbox, replicas (no compaction yet), takeover, export and import."} + +With an outbox (optional: a store opened with `{ outbox: true }`), every commit's writes go into it, written in the +same backend transaction as its data, so a crash keeps both or neither. Sync is a layer above the store; it follows Armin's transport and fencing with the rules the review added. + +- **Uploading:** the driver may upload batches in parallel, resubmits each until Radius acknowledges it, and trims a + batch from its outbox only after its acknowledgement (S5). +- **Applying:** every batch carries `from` and `to`. A replica applies a batch only when its `from` equals the + replica's `lastSeq` (the seq of the last commit it applied), as a whole, in one backend transaction together with + its new `lastSeq`. A later batch + waits for a missing one. A crash leaves all of a batch or none of it. +- **No compaction at first (W9):** batches carry the ordered operations exactly as committed. "Keep the last write per + row" would lose rows built from `put` plus partial `update`s; a later compaction must merge those and treat bulk + operations as barriers. +- **After a power loss** the driver's local `lastSeq` may be behind what Radius has; it first pulls back its own uploaded + batches, then continues (S3). +- **Blobs** go up before the batch that may name them: every blob file present and not yet uploaded (chapter VIII). + +## Losing the driver role (S7) + +:::sequence animate | The laptop learns the desktop took over. Nothing is rolled back. +actor laptop "laptop" "driver, epoch 3" +actor radius "Radius" blue +actor desktop "desktop" +desktop -> radius: take over S1 +radius => desktop: epoch 4 +laptop -> laptop: commits 40, 41 locally (a routine, a late tool result) +laptop -> radius: upload 40–41 (epoch 3) +radius --> laptop: refused: epoch 4 +laptop => laptop: the copy becomes session S1′ (new id) +note laptop..desktop: S1 continues on the desktop; S1′ on the laptop; they never sync again +::: + +Rolling back is not possible: tools ran, files were written, natives and derived data followed the tail. The laptop's +copy becomes a separate session with a new id and keeps running; commits in flight land in it. The catalog shows both, +the split-off copy marked with its origin. + +## Export and import (S4) + +An export holds the commit queue while it reads, so the archive is the state after one seq, its `lastSeq`: Durable's records, Chord's +documents and every plugin table, including tables of plugins that are not installed, plus the session directory's +blobs (chapter VIII). The archive is written in the outbox's own operations (a manifest, `table`, `index`, `put` and `document.create` +records, an `end` record), as one batch from 0 to its `lastSeq`, so an import is applied by the same code as a +replica's batches. An import restores the archive into a fresh store in one transaction; the server then mirrors the +session into its catalog. The encoding is pluggable (the store yields and takes records); plain JSONL ships first. +Derived data (full-text indexes, caches) is never exported; it is rebuilt where it is needed. + +# Rules for Plugin Authors {#rules lede="What the store does not do for you."} + +**Nothing leaves a transaction before it commits (S2).** Ids are handed out inside the transaction that creates them. +Reference what is committed: the file-history plugin references the tool-call entry, which is committed before the tool +runs, never the tool-result entry, which does not exist yet. + +**Work that finishes out of order is yours (S8).** A routine that starts a slow job per change can let an old result +overwrite a newer one. The handler calls of one routine never overlap, so either chain through the routine or drop +stale results inside the destination's transaction: + +```ts +self.on(conversation.entries, async ({ value: entries }, ctx) => { + const run = await summarize.start({ upTo: entries.at(-1)!.id }, ctx); + await run.outcome(ctx); // the next handler waits for this run +}); +``` + +**Text compares by code point.** Do not sort keys yourself to page through a table; use the store's cursors, or +`compareText`. Normalize text yourself if two spellings must be one key. + +**Bulk changes are operations.** Prefer `updateWhere` and `deleteWhere` over loops of puts; a loop records every row. + +# What's Next {.epilogue lede="What is decided, and the order in which it gets built."} + +The direction and all seventeen review items are decided (decision log, 2026-10-21). The store is new code; the parts +outside it are small changes in pi: + +- **pi, one change:** `Storage.commit(writes, adopt, context)` with every storage calling `adopt` (S1), the + Session's handling of rejections after adoption, the narrowed `mintId` comment (S10), `traceIn(options, fn, + context)` (context last), Durable's `WatchHandle`/`WatchEnd` moved into Chord with generic end reasons and explicit + retirement (W2), and Durable's state getters and the `replicatedState` family removed (W14). +- **The store** in `packages/chord/src/next`: commit step and outbox, tables and the query language, the SQLite + backend with the text encoding, the documents engine, bulk operations and migrations, export and import; + about 1,650 lines by Fable's estimate, plus its tests. +- **`StoreStorage`**, passing Durable's existing storage conformance suite. +- **The sync layer** (upload, apply, pull-back, epochs, splitting a session) on top, when Radius sync is built. + +Open, to settle while building: how a split-off session gets its new id (store metadata, directory, catalog), and a Postgres backend once a serverless target needs one (W14). + +# Decisions {.appendix} + +| # | Decision | +| --- | --- | +| The store | one transactional store per session under Durable (adapter), Chord documents and plugin tables | +| S1 | adoption inside the commit step; `Storage.commit(writes, adopt, context)` | +| S2 | nothing leaves a transaction before it commits; no mechanism | +| S3 | a commit is done on the backend's terminal success; ambiguous outcomes stop the store; pull-back after restart | +| S4 | export holds the commit queue; multi-query reads in one read transaction; no global snapshots | +| S5 | parallel uploads, resubmit until acknowledged, trim after acknowledgement; apply whole batches in order | +| S6 | (superseded by W9) | +| S7 | losing the driver role splits the session; no rollback | +| S8 | out-of-order work is an app concern; routine handlers never overlap | +| S9 | `lastSeq` (the seq of the last commit) is persisted with every commit | +| S10 | id counters derived at open; `mintId` contract narrowed | +| S11 | fork copies reconstructed by `StoreStorage` before the commit step | +| S12 | the query language starts without joins, grouping and projections | +| S13 | one documents engine in the store, opaque owner | +| S14 | one transaction API, `self.tx(fn, ctx)` | +| S15 | Durable's existing storages are kept | +| S16, text | real tables; text as plain UTF-8 in code point order (replaced W8's UTF-16 BLOBs) | +| S17, W11 | five writes, bulk operations recorded with a row count; JSON columns migrate on access with their own version; eager steps by scan with row `v` | +| W1 | adopt and mark in the commit step, one synchronous flush right after (derives, wire, watches) | +| W6, W7 | workers connect before loading; store addresses carry the host kind, never an instance | +| W9, W10 | no compaction at first; no `tx.get` on a table after a bulk operation in the same transaction | +| W14 | no IndexedDB backend (SQLite in WebAssembly in the browser); no `replicatedState` | +| index definitions | recorded in the outbox and applied by every replica; contents never travel | +| blobs, derived data | blobs are app-layer files named by hash, written before what names them, uploaded before each batch, deleted only with the session; derived data is rebuilt, never shipped | + +# Glossary {.appendix} + +**Adoption:** an owner updating its in-memory state to a committed write. **Blob:** a file in the session directory +named by the hash of its content, never changed. **Derived data:** data recomputed from the store and blobs, never +shipped. **Backend:** what the store runs on (SQLite, +memory). **Write:** one change of the store (`put`, `update`, `updateWhere`, `table`, `document.create`, …), the same in the +outbox and in an archive. **Commit** (in the outbox): the writes of one seq. **Batch:** the commits from `from` to `to`, +uploaded and applied as a whole. **Archive:** a manifest, the writes from 0 to `lastSeq`, and an `end`. +**Commit step:** the short, serialized part of a commit in which the store checks, writes, numbers and records it, and +owners adopt. **Driver:** the machine that writes a session. **Line:** the serialization of one owner's transactions +(per Chord object, per Durable Session). **Outbox:** the record of every commit, written in the same transaction. +**Owner:** the code that writes a piece of data. **Replica:** a copy of a session that applies the driver's batches. +**Seq:** the number of a commit, one sequence per store. **Store:** the transactional store of one session. +**Table:** a set of rows owned by one object, with a flat record of typed columns (scalars or `json(...)`). diff --git a/packages/chord/docs/next/store/index.html b/packages/chord/docs/next/store/index.html new file mode 100644 index 000000000..067a77772 --- /dev/null +++ b/packages/chord/docs/next/store/index.html @@ -0,0 +1,861 @@ + + +One Store: how a p3 session keeps, exports and syncs everything it owns + + + + +

+
+
A design book · Chord v2 and p3 · 2026-10-21
+

One
Store

+

Why Durable's transcript, Chord's documents and every plugin's own tables live in one transactional store, and how that store moves a session to another machine without coming apart

+
17review items decided (S1–S17)
1commit queue per store, one seq for every owner
~150 Boutbox record for renaming 10,000 rows (instead of ~40 MB)
0lines of plugin code a replica needs
+

Written for the owner from the discussion of 2026-10-21 and the store review by Fable and Astra. Every decision is in +the decision log (packages/chord/docs/next/decisions.md, entries "The store", "S1" to "S17", "Context last"); the +review is summarized in apps/p2/docs/p3-reviews/store-review-summary.md, bulk operations and migrations in +store-bulk-migrations.md, the earlier discussion with measurements in session-data-scenarios.md. Numbers were +measured on slayer with Node 24.4.1 and node:sqlite unless marked as estimates. Nothing here is built yet. Notes +marked review are points the reviewers raised that the design answers.

+
+
+ + +
+

Before we begin

+

Mario works in session S1 on his laptop. The agent edits files, and a plugin called file-history remembers every +edit: which file, which conversation entry made it, the content before and after. It keeps those rows in its own table, +because it needs queries: every version of src/app.ts, every file the last turn touched, every edit under a/. When +Mario renames the directory a/ to b/, ten thousand rows change.

+

In the evening he wants to continue S1 on his desktop. Another day he exports S1 to a file and imports it into a +colleague's p3. Each time the other machine must end up with the conversation (Durable), the plugins' documents (Chord) +and the file-history table, and all three must agree: a row that says "entry 812 made this edit" must find entry 812, +the same entry 812.

+

Copying files does not do that, and neither does syncing each storage on its own. This book explains the problem, the +solutions that were tried and dropped, and the design that came out: one transactional store per session, under +Durable, Chord and every plugin.

+

Eight things to know

+
    +
  1. A session is self-contained. Everything it owns lives in its store and its directory; it can be exported from +the session alone, imported elsewhere, synced and continued on another machine (P1).
  2. +
  3. One store holds everything. Durable's records (through an adapter), Chord's persisted documents and plugin +tables commit into one store, through one commit queue, numbered by one seq.
  4. +
  5. Order is guaranteed where it is observable. Disk, outbox and export follow the seq; in-memory adoption runs +inside the commit step (S1), so no reader ever sees seq 41 without seq 40.
  6. +
  7. The outbox records what changed, not code. Rows, column updates, bulk operations and table changes, written in +the same transaction as the data. A replica applies them without any plugin code.
  8. +
  9. Tables are real tables with a small query language. Keys and indexed columns are typed; text is UTF-8 and +compares by code point on every backend.
  10. +
  11. Bulk writes stay small. updateWhere records one operation instead of one row per change, and every replica runs +the same operation.
  12. +
  13. Migrations run only where the code is. Changes inside json(...) columns migrate on access; key and index changes run eagerly on the +driver and ship as ordinary writes.
  14. +
  15. Losing control never rolls back. A device that learns another one took over turns its copy into a separate +session.
  16. +
+

Words used throughout

+

A session is one agent workspace: its conversations, tasks and plugin data. An owner is the code that writes a +piece of data: Durable's Session, a Chord object, a plugin's object. The store is the transactional store of one +session; a backend is what it runs on (SQLite, including in the browser as WebAssembly, or a memory backend for tests). A table is a set of +rows owned by one object. A commit is one atomic write; the commit step is the short, serialized part of a commit +in which the store checks, writes and numbers it; the seq is that number. The outbox holds a record of every +commit, written in the same backend transaction. The driver is the machine that writes a session; a replica is +a copy on another machine that applies the driver's records. Adoption is an owner updating its in-memory state to a +committed write.

+
+ +
I

A Session Is More Than Its Transcript

What has to travel with a session, and why separate storages cannot keep it consistent.

+

A p3 session holds three kinds of data written by three kinds of owners. Durable writes the transcript: conversations, +entries, tasks and its own documents. Chord objects write persisted documents: settings, drafts, plugin state. And a +plugin may need data shaped for queries, like file-history's edits.

+

The owner's rule for p3 (P1) is that a session is self-contained. Everything of it lives in its storage plus its +directory; the server only mirrors it (the session catalog) and keeps instance data (shares, devices, settings). That +makes three things possible: export a session from its files alone, import it into another p3, and sync it to Radius +so another device can continue it.

+

Durable alone already supports that shape: one storage, every change in an atomic commit, numbered by a seq. Armin's +sync design builds on it: a decorator captures each committed batch into an outbox, batches go to Radius, a replica +replays them. It breaks as soon as a session has a second storage.

+
One edit, two storages, synced separately

On the laptop

The edit tool records the edit in file-history's own history.sqlite (row: path x.ts, entry 812). Durable commits +the tool's result. Both exist and agree.

+

The history arrives first

The desktop has received history.sqlite up to the edit, but Durable's log only up to the commit before. The row says +entry 812 made the edit; the desktop has no entry 812.

+

Mario takes over now

The laptop is closed; the desktop becomes the driver and writes its next entry, which gets id 812: a different +message. The row now points to the wrong entry, for good.

+
+

Each storage is consistent on its own. What is missing is one order across them: which change of history.sqlite +happened before which commit of the transcript. Every solution below is judged by whether it provides that order, +lets a replica work without the plugin's code, and keeps plugins able to model what they need.

+
+ +
II

What We Tried and Dropped

The solutions discussed on the way, and the reason each one failed.

+

The first answers tried to avoid giving plugins storage at all. The later ones gave them storage and tried to sync it.

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IdeaWhy it was dropped
Plugins' own databases hold only derived data (indexes, caches), rebuilt after an importtoo limiting: some plugins need their own write model and queries (owner)
Export plugin directories as opaque filesneeds a stopped session; a changing database cannot sync in small pieces
Plugin export and import hooksevery plugin would build its own incremental log for sync
Plugin record logs in the session log, the plugin's database rebuilt from them by an apply functionthe plugin must keep apply deterministic and every record version forever
Several storages, each with an outbox inside its own transaction, ordered by one shared position counterworks (prototype), but many outboxes and a "safe cut" rule; SQLite's row-change records are per row (a 10k-row rename is 259–418 KB); schema changes are not captured
A SQLite store that records the plugin's SQL statements, checked by a hash of their effectSQLite only; arbitrary SQL can give different results on replicas (random(), unordered INSERT … SELECT, SQLite versions)
Raw SQL reads with controlled writesties every backend to SQLite and needs a guard against SQL that writes
An IndexedDB-shaped API only (keys, indexes, ranges)too weak for real queries (owner)
Chord's documents inside Durable's existing storagecovers Chord, not plugin tables
+

The store design kept something from almost each: one order for everything (from the shared counter), recorded +operations instead of rows where possible (from the statement idea, made safe because the operations are the store's +own), and a small query language (from the IndexedDB shape, but over real tables).

+

Inside the store, more choices were dropped during the review:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
ChoiceWhy dropped
all rows of all tables in one generic table rows(tbl, key, v, row JSON)the owner wants real tables; with table definitions in the outbox a replica needs no plugin code anyway
index entries maintained by the store in a separate tablean extra row per index per write; SQLite could no longer filter natively
code point order (SQLite's native order) with lone surrogates replacedlossy ("\uD800" and "\uFFFD" collide) and needs a comparator in JS
text only for equality, order only over integers (Durable's own approach)ordered text, ranges and prefixes must be possible (owner)
JSON-field operands in bulk updatesthey lean on SQLite's JSON functions; added only when a real case needs them
+ + +
III

The Store

Layers, tables owned by objects, and the one transaction API.

+

The store sits between the backends and everything that persists data in a session.

+
OWNERSSTOREBACKENDSDurable Session`StoreStorage` adapterChord objectspersisted documentsplugin objectsself.table(...)documents enginerecords, revisions, versionstablesquery language, writescommit stepseq, outbox, adoptionSQLiteNode, Durable Objects, browser(WebAssembly)memorytestsDurable's memory, SQLite and JSONL storages stay (S15), with only the adopt argument added (S1); StoreStorage joinsthem.
Figure 1. The layers. Durable keeps its existing storages and gets one more, the adapter; Chord and plugins use the store directly.
+

A table belongs to one object, like a document: only that object reads and writes it; its address is (host kind, +object type, table name), never a host instance such as the session id (W7), so a params change, a split or an import +never orphans data; other plugins see the data only through the object's +state and methods.

+
const Edits = defineTable("edits", {
+  version: 2,
+  columns: { id: str(), path: str(), entry: int(), at: int(), meta: json<EditMeta>() },   // flat
+  key: ["id"],
+  indexes: { byPath: ["path", "at"], byEntry: ["entry"] },
+});
+
+host.serve(FileHistoryStore, (_, self) => {          // a local object with empty params
+  const edits = self.table(Edits);
+  return {
+    record: (edit, ctx) => self.tx(async (tx) => { await tx.put(edits, edit); }, ctx),
+    versions: ({ path }, ctx) =>
+      edits.page(query(Edits).where((e) => eq(e.path, path)).orderBy((e) => e.at, "desc"), 20, ctx),
+  };
+});
+
    +
  • A table's columns are a flat record: each column is a scalar (str, int, bool, num) or json(...); nesting +exists only inside json(...). Every column is a real SQLite column. Scalars can be filtered and ordered; keys and +indexed columns must be str or int. A json(...) column holds JSON text: typed when read, never queryable or +indexable (the type checker refuses it).
  • +
  • Tables and documents of an object are written in the same transaction API, self.tx(fn, ctx) (S14). The context +comes last in every signature (owner's rule).
  • +
  • Index definitions are recorded like other table changes: when the owning object starts on the driver, the store +creates or drops the difference between declared and existing indexes, and every replica applies it. Index contents +never travel; every backend maintains them from the rows. So every copy of the store is the same file.
  • +
+
+ +
IV

Writing It Down

One commit queue per store, one seq for every owner, and adoption inside the step.

+

Two different things are serialized, and they must not be confused. A line serializes a whole transaction +callback: read, decide, write. Chord has one line per object, Durable one per Session. The commit queue serializes +only the final step: checks, writes, seq, outbox record. It is short and there is one per store.

+
object Aits own lineobject Bits own lineDurable Sessionits own linecommit queueone per storedraft notes …draft drafts …phase writes …commitseq 41: write, outbox, adopt Acommitseq 42: write, outbox, adopt Bcommitseq 43: write, outbox, adopt Durable
Figure 2. Two Chord objects and a Durable turn commit at the same time. Their callbacks never wait for each other; only the short commit steps queue.
+

Durable's seq is the store's seq. Durable sees gaps (other owners' commits in between), and that is harmless: every use +of a seq in Durable compares (created_at <= at, seq > base); nothing computes seq + 1 to find a neighbour (checked +in the storages and the Session).

+

Adoption inside the step (S1)

+

After a commit is on disk, each owner swaps its in-memory state to the committed version. If that happens later, in +the owner's own continuation, the order can flip:

+
disk:    40 file-history, 41 Durable
+memory:  Durable adopts 41 first (one await fewer)
+         a routine on "entry 812 committed" reads file-history → still at 39
+

So adoption runs inside the commit step, synchronously, before the next commit starts. Durable's storage interface +carries it as a new required argument:

+
commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, context: Context): Promise<Seq>;
+

Every storage calls adopt(seq) once its commit is durable and before its next commit. The Session catches its own +adoption errors; a rejection after adoption fails the Session without discarding the adopted changes.

+

Adopt and mark in the step, flush right after (W1)

+

The step itself stays short: it writes, adopts (pointer swaps of every changed value, including Durable's views the +commit touched) and marks (dependent derives dirty, ops appended by reference to each touched hold's batch and each +watch's queue). Right after the step, synchronously and before any other code runs, the committing owner flushes: +it recomputes the dirty derives someone holds or watches, encodes one part per touched hold into one message per link, +and queues watch callbacks. So a source and the derives over it always travel in one message; a throwing derive fails +only its own member, never the commit.

+
step 40:  write · adopt notes · mark count dirty, notes' ops into tab 1's batch
+flush 40: count = 3 → its ops join the batch · ONE message to tab 1 { notes: +1, count: 3 }
+step 41:  write · adopt Durable's view · mark lastReply …
+

Watch listeners run afterwards, each in order; their order across owners may differ from the seq, but every listener +reads a complete prefix of the log.

+

When a commit counts as done (S3)

+

The step ends only on the backend's terminal success: SQLite's COMMIT returned (with an asynchronous backend, the +transaction's final completion event, not its individual requests). The memory backend stages a +commit and swaps it in at the end. An ambiguous outcome, an error during COMMIT, stops the store +(storage_failed): nobody knows whether the commit landed, so nothing more is written or published.

+
Review: request success is not commit completion

Astra's trace: IndexedDB reports the puts of seq 40 successful, the store publishes 40, the transaction aborts later +(quota), 50 commits on top: memory has 40, disk does not. The rule above is the answer.

+
+

Ids

+

Each owner's id counter starts, when the store opens, above that owner's highest stored id: one index lookup per table +(S10). An id handed out to a transaction that never committed may be handed out again after a restart. Nobody can +notice, because nothing leaves a transaction before it commits (S2), so Durable's mintId contract is narrowed to say +exactly that.

+
+ +
V

Reading

A small query language, one text order on every backend, and cursors.

+

Queries are immutable data built in TanStack DB's style; S12 starts without joins, grouping and projections:

+
query(Edits).where((e) => eq(e.path, "src/app.ts")).orderBy((e) => e.at, "desc")
+query(Edits).where((e) => and(gte(e.at, from), lt(e.at, to)))
+query(Edits).where((e) => prefix(e.path, "a/"))
+

where takes eq ne lt lte gt gte inArray isNull prefix and or not. limit and the cursor are arguments of the +runner, not part of the query. count(q) is the one aggregate; others are computed in JavaScript over indexed scans +until a real case needs more.

+

One text order

+

Text compares by Unicode code point on every backend (owner, 2026-10-26; this replaced the first design, UTF-16 BLOBs +in JavaScript order, S16 and W8). That is the order SQLite compares UTF-8 text in natively, and Durable Objects, +sqlite-wasm, Postgres (COLLATE "C") and MySQL (utf8mb4_0900_bin) do too. Every str() column is plain UTF-8 +TEXT: strings go to the driver and come back as they are, readable in sqlite3 and in any other tool.

+

It differs from JavaScript's < and sort() in one range only: characters U+E000-U+FFFF (fullwidth forms such as +~, private use, U+FFFD) against characters above U+FFFF (emoji):

+
JS sort():    "测试😀" < "测试(一)"
+store order:  "测试(一)" < "测试😀"
+

The store exports compareText(a, b) for plugins that sort store text themselves; the memory backend uses it for +ordering, ranges, max and min. Page with the store's cursors and use prefix(); a hand-built bound such as +p + "\uffff" misses every p followed by an emoji.

+

UTF-8 cannot hold a lone surrogate, so every str() value written, and every query operand, goes through +toWellFormed(): a lone surrogate becomes U+FFFD, as TextEncoder does. A string cut in the middle of an emoji by +slice() is stored with U+FFFD in its place; cut text with [...s].slice() to keep it whole. A str() value may not +contain NUL (invalid/args); content belongs in json(...) columns, which escape both. There is no Unicode +normalization: "é" written as one code point and "e" plus a combining accent are two keys.

+ + + + + + + + + + + + + + + + + + + + + + + +
100k path-like keysUTF-16 BLOB (the first design)UTF-8 TEXT
insert, one transaction125 ms113 ms
read106 ms32 ms
bytes of an 18-character ASCII path3618
+

Running queries

+

A query is immutable data; the table handle runs it, and the context comes last:

+
const edits = self.table(Edits);
+const byPath = query(Edits).where((e) => prefix(e.path, "src/")).orderBy((e) => e.path);
+
+const page = await edits.page(byPath, 100, ctx, { after: cursor });   // { rows, next? }
+for await (const row of edits.rows(byPath, ctx)) { … }                 // all pages
+const e1 = await edits.get(["e1"], ctx);
+
+await self.tx(async (tx) => {
+  for await (const row of tx.rows(edits, byPath)) {                    // committed rows, stable during the tx
+    await tx.update(edits, [row.id], { dir: dirname(row.path) });
+  }
+  const again = await tx.get(edits, ["e1"]);                           // sees this transaction's own writes
+}, ctx);
+

Inside a transaction, tx.page and tx.rows return committed rows: a stable view, because only the owning object +writes its tables and its transactions run one at a time. They do not show the transaction's own writes; tx.get does. +The rule in one sentence: queries inside a transaction see the state before it; your own writes become visible after +the commit. After an updateWhere or deleteWhere on a table, tx.get on that table throws read_after_bulk +(W10): the store would need a second evaluator to know what a bulk operation did to one row.

+

Cursors and consistent reads

+

Every index ends with the primary key, so order is total and cursors are stable. A cursor is opaque, carries its +direction and its last key as logical values, is bound to its query's order, and is refused by another query or the +other direction.

+

Reads outside a transaction see committed state, one query at a time. A read that needs several queries, like a +document assembled from its base and later revisions, runs in one backend read transaction (S4). Two separately +awaited reads in object code are not a snapshot, and the guide says so.

+
+ +
VI

Documents

One documents engine in the store, used by Durable's adapter and by Chord.

+

Durable and Chord both persist documents the same way: a record per incarnation, revisions made of a base and later +deltas, versions, checkpointWhen, one live document per address. The store implements it once (S13):

+
type DocAddress = { namespace: string; owner: string; kind: string; key?: string };
+
+interface Documents {                                   // reads, committed state
+  find(address: DocAddress, at: At, context: Context): Promise<DocRecord | undefined>;
+  load(id: number, at: At, context: Context): Promise<StoredDoc | undefined>;   // base + deltas, applied once
+  list(query: DocListQuery, limit: number, cursor: Cursor | undefined, context: Context): Promise<Page<DocRecord>>;
+}
+interface DocumentWrites {                              // inside the commit step
+  create(record: NewDocRecord, base: BaseContent): number;
+  change(id: number, content: DocContent): void;
+  retire(id: number): void;
+}
+

The owner string is opaque to the engine; each adapter fills it:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
namespaceownerkindkey
Durable session documentpi-durablesessionpi.settings
Durable conversation documentpi-durableconversation:7pi.provider
Durable task documentpi-durabletask:31pi.codemode
Chord persisted documentchordsession/notes.storenotesn-42
+

What the owner string means (a conversation's documents fork and rewind with it, a task's are retired with the task) +stays in the adapter. The engine keeps nothing in memory; Durable's Session and Chord's objects keep their Delta +trackers, cache, pin and evict exactly as Durable does today. A fork's document copies are reconstructed by +StoreStorage before it enters the commit step (S11), so the step never replays revisions while every owner waits; +the outbox only ever carries the resulting documents.

+
+ +
VII

Writing Many Rows

Bulk operations recorded as operations, and migrations that run only where the code is.

+

A full-row put per changed row makes the rename of a/ to b/ over ten thousand 4 KB rows cost about 40 MB in the +outbox. The store's writes are therefore five:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
WriteRecords
tx.put(table, row)the whole row
tx.update(table, key, set)the changed columns of one row
tx.delete(table, key)the key
tx.updateWhere(table, where, set)one operation, plus the affected row count
tx.deleteWhere(table, where)one operation, plus the affected row count
+
await self.tx(async (tx) => {
+  await tx.updateWhere(Edits, (e) => prefix(e.path, "a/"), { path: replacePrefix("a/", "b/") });
+}, ctx);
+
-- on SQLite: one statement; path is UTF-8 TEXT, "a/" is 2 code points
+UPDATE "file-history/edits" SET path = :to || substr(path, 3) WHERE path >= :lo AND path < :hi
+
outbox, seq 57 (about 150 bytes):
+{ op: "update", table: "file-history/edits", where: { prefix: ["path", "a/"] },
+  set: { path: { replacePrefix: ["a/", "b/"] } }, count: 10000 }
+

A replica runs the same operation, without plugin code, and compares the number of affected rows. A different number +means divergence: it stops applying that store and restores it from a snapshot. The set expressions touch only real +columns: value, col, add, sub, max, min, concat, replacePrefix, coalesce, when. They are the store's +own, specified once and checked on every backend by one conformance suite, which is why there are no SQL subqueries, +joins, case folding or floats: each of those behaves differently somewhere.

+

Migrations (S17)

+

Adding, renaming or dropping a column is a recorded table change (alter); a new column gets its default without a +bulk rewrite. A change of the shape inside a json(...) column (a field added, renamed, restructured) migrates on +access (W11): each json(...) column has its own version and a value-only migration, +json(MetaV2, { version: 2, migrate: (old, from) => … }), stored with its value; reading an older value migrates it, +writing the column stores the new shape and version. Nothing is rewritten in bulk and indexes are unaffected. The row's +v counts only eager steps (below); update, updateWhere and deleteWhere never set it. Old code refuses a newer +row v or a newer JSON column version.

+

A change to a key or an indexed column must be eager, because an index over a column filled only on access would +silently miss rows. It runs when the owning object starts on the driver, before the object goes live:

+
migrations: {
+  4: { alter: { add: { dir: { type: "text" } } },
+       rows: (row) => ({ dir: dirname(row.path) }) },                         // scan + tx.update, in chunks
+  5: { alter: { add: { size: { type: "int", default: 0 } } },
+       update: { where: (e) => isNull(e.size), set: { size: value(0) } } },     // one updateWhere
+},
+
Migration 4 on the driver

The table change

Commit 80 records alter: add column dir and marks table version 4 as migrating. Every replica adds the column +from the record.

+

The scan

The store reads rows with v < 4 in key order, 500 at a time, applies rows to each and writes the changed columns +plus v = 4 in the same write. One commit per chunk, so other owners' commits interleave and the commit step stays +short. (A full-row put also sets v; partial and bulk writes never do.)

+

Ready

The last chunk marks version 4 ready. The store creates the byDir index and records it, and FileHistoryStore +goes live.

+

After a crash in the middle

The next start finds migrating, scans again from the first key and skips rows with v >= 4. Each row is +transformed exactly once, even when the transform is not idempotent. A key change is never a rows step: it is a +recorded rebuild that every replica performs itself.

+
+

A replica without the plugin applies the column change and the column updates; a replica with the plugin's code treats +the table as not ready while it is migrating. Chord documents keep their own per-document migrate on access, and +Durable versions its records itself.

+
+ +
VIII

The Easy Parts

Blobs and derived data: why they need none of the store's ordering machinery.

+

Two kinds of session data stay outside the store's tables, and both are easy for the same reason: nothing about them +ever changes in place. A blob is final once written; derived data has no data of its own.

+

Blobs

+

Artifact versions, uploads and sent files are blobs: files in the session directory, named by the hash of their +content (blobs/sha256/9f/9f3a…), never changed. They are app layer, not store: any process may write one, because a +file named by its content cannot conflict with another writer. The order is the whole trick: the blob is written first, +then the row or document that names it.

+
artifact toolblobs/storewrite sha256:9f3a… (tmp, fsync, rename)seq 41: chart.versions[2] = { blob: "sha256:9f3a…" }seq 42: entry 812 "chart v3 saved" (Durable)
Figure 3. Saving artifact "chart" version 3. The blob exists before anything refers to it.
+
    +
  • Export copies blobs/ together with the store snapshot. Every blob a record up to the export's head names was +written before that record, so it is in the directory. Import copies them back and checks each hash.
  • +
  • Sync: before uploading a batch, the sync layer uploads every blob file that is present and not yet uploaded. No +reference tracking is needed: a blob is always written before the commit that names it, so a replica never sees a +hash whose file is missing.
  • +
  • Deleting: no blob is deleted before its session is deleted. A blob that nothing references any more (a crash +between the write and the commit, a retired document) only costs disk; a garbage collector would have to know every +reference in every table.
  • +
+

Derived data

+

A full-text index, a cache, a plugin's own search database: derived data can be recomputed from the store and the +blobs, so it is never exported or synced. Its owner keeps it current with a routine that compares it with the store. +The artifacts plugin's full-text index in its own SQLite file:

+
host.serve(ArtifactStore, (_, self) => {
+  const artifacts = self.state(Artifact);                         // keyed documents in the store
+  const blobs = self.get(SessionBlobs, {});
+  const index = self.open(async () => openIndex(join(dir, "derived/artifacts/index.sqlite")), (db) => db.close());
+
+  self.routines.loop("catch-up", async (ctx) => {                 // the index repairs itself
+    for await (const name of artifacts.keys()) {
+      const a = await artifacts.get(name);
+      if (a && index.value.versions(name) !== a.versions.length)
+        index.value.put(name, a.versions.length, await blobs.read(a.versions.at(-1)!.blob, ctx));
+    }
+  });
+  …
+});
+
One routine, three situations

Normal save

save writes the blob, commits the document, then updates the index. The routine finds nothing to do.

+

Crash after the commit

The process dies between the commit and the index update. The document says version 3, the index still has version 2. +On the next start the routine sees the difference, reads the blob of version 3 and indexes it.

+

Import or sync

On the desktop derived/artifacts/ does not exist. The routine finds every artifact missing and indexes them all from +the documents and blobs that came with the session.

+
+

This routine visits every artifact on each start: fine for hundreds, slow for millions. A large derived store keeps +the last seq it applied and reads only what changed after it.

+
+ +
IX

Moving a Session

Outbox, replicas (no compaction yet), takeover, export and import.

+

Every commit's outbox record is written in the same backend transaction as its data, so a crash keeps both or +neither. Sync is a layer above the store; it follows Armin's transport and fencing with the rules the review added.

+
    +
  • Uploading: the driver may upload batches in parallel, resubmits each until Radius acknowledges it, and trims a +batch from its outbox only after its acknowledgement (S5).
  • +
  • Applying: every batch carries from and to. A replica applies a batch only when its from equals the +replica's applied head, as a whole, in one backend transaction together with the new applied head. A later batch +waits for a missing one. A crash leaves all of a batch or none of it.
  • +
  • No compaction at first (W9): batches carry the ordered operations exactly as committed. "Keep the last write per +row" would lose rows built from put plus partial updates; a later compaction must merge those and treat bulk +operations as barriers.
  • +
  • After a power loss the driver's local head may be behind what Radius has; it first pulls back its own uploaded +batches, then continues (S3).
  • +
  • Blobs go up before the batch that may name them: every blob file present and not yet uploaded (chapter VIII).
  • +
+

Losing the driver role (S7)

+
laptopdriver, epoch 3Radiusdesktoptake over S1epoch 4commits 40, 41 locally (a routine, a late tool result)upload 40–41 (epoch 3)refused: epoch 4the copy becomes session S1′ (new id)S1 continues on the desktop; S1′ on the laptop; they never sync again
Figure 4. The laptop learns the desktop took over. Nothing is rolled back.
+

Rolling back is not possible: tools ran, files were written, natives and derived data followed the tail. The laptop's +copy becomes a separate session with a new id and keeps running; commits in flight land in it. The catalog shows both, +the split-off copy marked with its origin.

+

Export and import (S4)

+

An export holds the commit queue while it reads, so the archive is the state at one head: Durable's records, Chord's +documents and every plugin table, including tables of plugins that are not installed, plus the session directory's +blobs (chapter VIII). An import restores the archive into a fresh store; the server then mirrors the session into its catalog. +Derived data (full-text indexes, caches) is never exported; it is rebuilt where it is needed.

+
+ +
X

Rules for Plugin Authors

What the store does not do for you.

+

Nothing leaves a transaction before it commits (S2). Ids are handed out inside the transaction that creates them. +Reference what is committed: the file-history plugin references the tool-call entry, which is committed before the tool +runs, never the tool-result entry, which does not exist yet.

+

Work that finishes out of order is yours (S8). A routine that starts a slow job per change can let an old result +overwrite a newer one. The handler calls of one routine never overlap, so either chain through the routine or drop +stale results inside the destination's transaction:

+
self.routines.on(conversation.entries, async (entries, ops, ctx) => {
+  const run = await summarize.start({ upTo: entries.at(-1)!.id }, ctx);
+  await run.outcome(ctx);                                  // the next handler waits for this run
+});
+

Text compares by code point. Do not sort keys yourself to page through a table; use the store's cursors, or +compareText. Normalize text yourself if two spellings must be one key.

+

Bulk changes are operations. Prefer updateWhere and deleteWhere over loops of puts; a loop records every row.

+
+ +
∴

What's Next

What is decided, and the order in which it gets built.

+

The direction and all seventeen review items are decided (decision log, 2026-10-21). The store is new code; the parts +outside it are small changes in pi:

+
    +
  • pi, one change: Storage.commit(writes, adopt, context) with every storage calling adopt (S1), the +Session's handling of rejections after adoption, the narrowed mintId comment (S10), traceIn(options, fn, context) (context last), Durable's WatchHandle/WatchEnd moved into Chord with generic end reasons and explicit +retirement (W2), and Durable's state getters and the replicatedState family removed (W14).
  • +
  • The store in packages/chord/src/next: commit step and outbox, tables and the query language, the SQLite and +memory backends with the text encoding, the documents engine, bulk operations and migrations, export and import; +about 1,650 lines by Fable's estimate, plus the conformance suite.
  • +
  • StoreStorage, passing Durable's existing storage conformance suite.
  • +
  • The sync layer (upload, apply, pull-back, epochs, splitting a session) on top, when Radius sync is built.
  • +
+

Open, to settle while building: how a split-off session gets its new id (store metadata, directory, catalog), the +chunk size of row migrations, and a Postgres backend once a serverless target needs one (W14).

+
+ +
A

Decisions

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#Decision
The storeone transactional store per session under Durable (adapter), Chord documents and plugin tables
S1adoption inside the commit step; Storage.commit(writes, adopt, context)
S2nothing leaves a transaction before it commits; no mechanism
S3a commit is done on the backend's terminal success; ambiguous outcomes stop the store; pull-back after restart
S4export holds the commit queue; multi-query reads in one read transaction; no global snapshots
S5parallel uploads, resubmit until acknowledged, trim after acknowledgement; apply whole batches in order
S6(superseded by W9)
S7losing the driver role splits the session; no rollback
S8out-of-order work is an app concern; routine handlers never overlap
S9the head is persisted with every commit
S10id counters derived at open; mintId contract narrowed
S11fork copies reconstructed by StoreStorage before the commit step
S12the query language starts without joins, grouping and projections
S13one documents engine in the store, opaque owner
S14one transaction API, self.tx(fn, ctx)
S15Durable's existing storages are kept
S16, textreal tables; text as plain UTF-8 in code point order (replaced W8's UTF-16 BLOBs)
S17, W11five writes, bulk operations recorded with a row count; JSON columns migrate on access with their own version; eager steps by scan with row v
W1adopt and mark in the commit step, one synchronous flush right after (derives, wire, watches)
W6, W7workers connect before loading; store addresses carry the host kind, never an instance
W9, W10no compaction at first; no tx.get on a table after a bulk operation in the same transaction
W14no IndexedDB backend (SQLite in WebAssembly in the browser); no replicatedState
index definitionsrecorded in the outbox and applied by every replica; contents never travel
blobs, derived datablobs are app-layer files named by hash, written before what names them, uploaded before each batch, deleted only with the session; derived data is rebuilt, never shipped
+
+ +
B

Glossary

+

Adoption: an owner updating its in-memory state to a committed write. Blob: a file in the session directory +named by the hash of its content, never changed. Derived data: data recomputed from the store and blobs, never +shipped. Backend: what the store runs on (SQLite, +memory). Batch: a range of outbox records from from to to, uploaded and applied as a whole. +Commit step: the short, serialized part of a commit in which the store checks, writes, numbers and records it, and +owners adopt. Driver: the machine that writes a session. Line: the serialization of one owner's transactions +(per Chord object, per Durable Session). Outbox: the record of every commit, written in the same transaction. +Owner: the code that writes a piece of data. Replica: a copy of a session that applies the driver's batches. +Seq: the number of a commit, one sequence per store. Store: the transactional store of one session. +Table: a set of rows owned by one object, with a flat record of typed columns (scalars or json(...)).

+
+ +

One Store · first edition · 2026-10-21 · based on the redesign decision log and the store review

+ + + diff --git a/packages/chord/docs/next/store/walkthrough.md b/packages/chord/docs/next/store/walkthrough.md new file mode 100644 index 000000000..c682fe3e3 --- /dev/null +++ b/packages/chord/docs/next/store/walkthrough.md @@ -0,0 +1,193 @@ +--- +title: One Store +kind: walkthrough +kicker: A walkthrough · Chord v2 and p3 · 2026-10-21 +subtitle: How one transactional store keeps a p3 session's transcript, documents and plugin tables consistent, taught one concept at a time +model: anthropic/claude-sonnet-5-5 +sources: + - packages/chord/docs/next/store/book.md + - packages/chord/docs/next/decisions.md + - apps/p2/docs/p3-reviews/store-review-summary.md + - apps/p2/docs/p3-reviews/store-bulk-migrations.md + - apps/p2/docs/p3-reviews/session-data-scenarios.md + - apps/p2/docs/p3-reviews/store-proposal.md + - packages/durable/src/storage/sqlite/storage.ts + - packages/durable/src/session/session.ts +truth: The decision log (redesign-decisions.md, entries "The store", "S1" to "S17", "Context last") wins over everything; the book (book.md) is written from it; store-proposal.md and session-data-scenarios.md are earlier drafts and the discussion, superseded where the log differs; the Durable sources show today's code, before the S1 change. +--- + +:::tutor +Audience: the owner of p3 and Chord, who decided this design in one long discussion and wants to understand it end to +end, or a developer joining the work who knows TypeScript, SQLite and pi-durable but not this store. Use the book's +running example throughout: Mario's session S1 on his laptop, the file-history plugin with its `edits` table, entry +812, the rename of `a/` to `b/` over 10,000 rows, continuing on the desktop. Read the matching book chapter before each +concept and check decisions in the log. Be concrete: code, short traces with seq numbers, small tables. Short messages; +one figure per concept at most. When the reader asks why something was not done another way, answer from the book's +"What We Tried and Dropped" chapter and the log. +::: + +:::intro +#### What this is about + +A p3 session must be self-contained: everything it owns lives in its own store and directory, so it can be exported, +imported into another p3, synced to Radius and continued on another device. Three kinds of owners write into a +session: Durable (the transcript), Chord objects (persisted documents) and plugins (their own tables). This walkthrough +explains the store they all share and why it is built that way. + +#### The example + +Mario works in session S1 on his laptop. A plugin called **file-history** records every file edit in its table +`edits`: path, the conversation entry that made it (say entry 812), the time. He renames the directory `a/` to `b/` +(10,000 rows change). In the evening he continues S1 on his desktop; another day he exports it and a colleague imports +it. Every concept follows this session through one mechanism. + +#### How it progresses + +Eleven concepts in four parts: the problem and what was dropped; how the store writes (commit queue, seq, adoption, +completion); how it reads and stores documents and tables (text order, the documents engine, bulk operations and +migrations; blobs and derived data); and how a session moves (sync, takeover, export) plus the rules left to plugin authors. +::: + +# The problem {#problem part="Why one store"} +Book "Before we begin" and chapter I. Must cover: P1 (a session is self-contained; the server only mirrors); three +owners (Durable, Chord objects, plugins); the trace where file-history's own `history.sqlite` and Durable's log sync +separately and entry 812 dangles or points to the wrong entry after takeover; what is missing is one order across +storages. A short trace or a steps figure helps. + +:::check +1. Both of S1's storages are consistent on their own. Why can the desktop still end up in a state the laptop never had? ++++ answers +1. Because nothing orders the changes of one storage relative to the other: the desktop can receive history.sqlite up to the edit but Durable's log only up to the commit before it (or the reverse), a combination that never existed on the laptop. +::: + +# What was dropped {#dropped} +Book chapter II; log entry "The store" (its list of rejected options) and S16. Must cover the main dropped options and +the reason for each: derived-only plugin data, opaque directories, export hooks, plugin record logs with apply, several +storages with a shared position counter, recorded SQLite statements, raw SQL reads, an IndexedDB-only API; inside the +store: generic JSON rows, store-maintained index entries, code point order with replaced surrogates, text equality only. +Point out what the final design kept from them (one order, recorded operations, a small query language). + +:::check +1. Why was "record the plugin's SQL statements and replay them on the replica" dropped, while bulk operations are now recorded as operations and replayed? ++++ answers +1. Arbitrary SQL can produce different results on a replica (random(), datetime('now'), unordered INSERT ... SELECT, other SQLite versions); the store's bulk operations use only the store's own expressions, specified once and identical on every backend, plus a row count check. +::: + +# The store and its tables {#store part="Writing"} +Book chapter III; log "The store", S14, S16. Must cover: layers (owners, store with documents engine, tables and commit +step, backends); a table belongs to one object (address host, object type, table name); `defineTable` with typed key +and indexed columns (`str` or `int`) in a flat `columns` record of real columns, nesting only inside `json()` columns (typed but not queryable); one transaction API +`self.tx(fn, ctx)`; index definitions are recorded in the outbox (contents never travel); Durable keeps its storages and gains `StoreStorage`; context last. + +:::check +1. In `edits`, which fields can a query filter on, and where does the rest of the row live? ++++ answers +1. Every scalar column (str, int, bool, num) is a real column and can be filtered; keys and indexes need str or int; json(...) columns hold JSON text, typed when read but not queryable or indexable. +::: + +# Lines, the commit queue and the seq {#commit} +Book chapter IV; log S1, S3, S9, S10. Must cover: lines (per Chord object, per Durable Session) serialize whole +transactions, the commit queue serializes only the short final step; one seq per store, Durable sees gaps and only +compares seqs; adoption inside the step with `Storage.commit(writes, adopt, context)`; why (the trace where Durable +adopts 41 before file-history adopts 40); listeners still run afterwards and read a complete prefix; a commit is done +only on the backend's terminal success; ambiguous outcomes stop the store; ids derived at open. The book's sequence +figure of two objects and a Durable turn helps. + +:::check +1. Object A commits seq 40 and Durable commits 41. Without S1, how could a routine see 41 but not 40? +2. Why is it harmless that Durable's seqs now have gaps? ++++ answers +1. Each owner adopted in its own continuation after the disk commit; Durable's continuation ran first, so a routine triggered by 41 read A's in-memory state still at 39. With S1 both adoptions run inside the commit step, in seq order. +2. Durable only compares seqs (created_at <= at, seq > base); it never computes seq + 1 to find a neighbouring commit. +::: + +# Reading and text order {#reading part="Reading and storing"} +Book chapter V; log S4, S12, S16. Must cover: the small query language (where operators, orderBy, limit, cursor; no +joins, grouping, projections at first); JavaScript and SQLite order strings differently (the ~ vs 😀 example); the +decision: JavaScript order everywhere, SQLite stores key and indexed text as UTF-16 BLOBs (big-endian), lossless, +measured costs; no normalization; cursors carry direction and logical values; multi-query reads in one read +transaction, no global snapshot. + +:::check +1. A plugin pages through `edits` by path with a cursor. Why would it break if SQLite stored paths as ordinary TEXT while the memory backend compared JS strings? +2. What does the UTF-16 BLOB encoding cost, according to the measurements? ++++ answers +1. SQLite would order by UTF-8 bytes and JS by UTF-16 code units; they disagree for characters between U+E000 and U+FFFF versus emoji, so a cursor taken from one order skips or repeats rows in the other (and tests on the memory backend would not show what SQLite does). +2. Mostly JS encoding and decoding plus twice the bytes for ASCII: about +40% on bulk inserts, +62% file size in the test; SQLite's own range scans are about 4% slower when values are not decoded from the BLOBs. +::: + +# The documents engine {#documents} +Book chapter VI; log S11, S13. Must cover: Durable and Chord persist documents the same way, so the store has one +engine; address (namespace, owner, kind, key) with an opaque owner string filled by each adapter (Durable: session, +conversation:7, task:31; Chord: host/object type); find, load, list outside the step, create, change, retire inside it; +load reconstructs base plus deltas once, the engine keeps nothing in memory; Durable's Session caches and evicts as +today; fork copies are reconstructed by StoreStorage before the commit step. + +:::check +1. Who decides that a conversation's documents fork and rewind with it: the documents engine or Durable's adapter? ++++ answers +1. Durable's adapter (StoreStorage): the engine only sees an opaque owner string such as "conversation:7"; the meaning of that owner stays with Durable. +::: + +# Bulk operations and migrations {#bulk} +Book chapter VII; log S17; store-bulk-migrations.md. Must cover: the five writes (put, update, delete, updateWhere, +deleteWhere); the rename of a/ to b/ as one recorded operation with a row count (about 150 bytes instead of about 40 +MB), compiled to one SQL UPDATE on the UTF-16 BLOB column; the replica runs the same operation and compares the count; +the set expressions and why they are limited (identical behaviour on every backend; no JSON operands yet); migrations: +changes inside json(...) columns on access via migrate(row, from), column changes as recorded alters, key and indexed columns eagerly by recorded table changes plus an update +step or a rows step (scan + tx.update in chunks, migrating then ready, resume after a crash). The book's migration steps +figure helps. + +:::check +1. Version 4 of `edits` adds an indexed column `dir` computed with dirname(path). Why can this not migrate on access, and how does it run instead? +2. A replica applies the rename and affects 9,998 rows instead of 10,000. What happens? ++++ answers +1. An index over a column filled only on access would silently miss unmigrated rows; so it runs eagerly on the driver before the object goes live: a recorded alter adds the column, then a rows step scans in key order and writes dir with tx.update, chunk by chunk, until the table is marked ready. +2. The counts differ, so the replica stops applying that store and restores it from a snapshot: divergence is detected, not silent. +::: + +# Blobs and derived data {#easy} +Book chapter VIII "The Easy Parts"; log "Blobs and derived data". Must cover: blobs are app-layer files named by the +hash of their content, written before the row or document that names them; export copies them, import checks hashes; +sync uploads every blob present and not yet uploaded before each batch (no reference tracking); no deletion before the +session is deleted; derived data is never shipped and is repaired by a catch-up routine (the artifacts full-text index +example: normal save, crash after commit, import). + +:::check +1. Why does the sync layer not need to know which rows reference which blobs? +2. The process dies between the artifact document's commit and the index update. What fixes the index? ++++ answers +1. A blob is always written before the commit that names it, so uploading every blob present before each batch guarantees that any hash a batch names has its file on Radius already. +2. The catch-up routine on the next start: it compares the index with the documents, finds version 3 missing and indexes it from the blob. +::: + +# Sync and takeover {#sync part="Moving a session"} +Book chapter IX; log S3, S5, S6, S7. Must cover: the outbox record is written in the same backend transaction; +parallel uploads, resubmit until acknowledged, trim after acknowledgement; a replica applies whole batches in order +(from equals its applied head), later batches wait; compaction is safe because batches apply whole; pull-back after a +power loss; losing the driver role splits the session (new id) without rollback, and why rollback is impossible. + +:::check +1. The laptop committed 40 and 41 locally after the desktop took over. What happens to those commits? ++++ answers +1. Nothing is rolled back: when Radius refuses the laptop's upload for an old epoch, the laptop's copy becomes a separate session with a new id, keeps 40 and 41, and never syncs with S1 again; S1 continues on the desktop. +::: + +# Export, import and the rules left to plugins {#export} +Book chapters IX ("Export and import") and X; log S2, S4, S8. Must cover: export holds the commit queue so the +archive is one head, includes tables of plugins that are not installed, never derived data; import restores into a +fresh store and the server mirrors the catalog; rules: nothing leaves a transaction before it commits (reference the +tool-call entry, not the tool-result entry); work that finishes out of order is the app's concern (routine handlers +never overlap: chain or drop stale results); text compares like JS strings; prefer updateWhere over loops of puts. + +:::check +1. The file-history plugin wants to store which entry made an edit. Which entry can it safely reference, and why? ++++ answers +1. The assistant entry with the tool call: it is committed before the tool runs. The tool-result entry does not exist yet while the tool runs, and its id is handed out only inside the harness's later transaction. +::: + +# What comes next {#next part="Where it stands"} +Book "What's Next"; log entries. Must cover: everything decided on 2026-10-21; the pi change (adopt argument, Session +rejection handling, mintId wording, traceIn context last); the store in packages/chord/src/next (about 1,650 lines +estimated); StoreStorage passing Durable's conformance suite; the sync layer later; open points (new id of a split +session, chunk size, whether IndexedDB is needed while sqlite-wasm covers the browser). diff --git a/packages/chord/src/next-node/compression.ts b/packages/chord/src/next-node/compression.ts new file mode 100644 index 000000000..d1b9e7006 --- /dev/null +++ b/packages/chord/src/next-node/compression.ts @@ -0,0 +1,24 @@ +// Compression for Chord v2 hosts on Node (§22, slice 4b): the synchronous compressor `createHost({ compression })` +// takes, over Node's zlib. Node-only, so it lives outside `src/next`, which imports no Node module. + +import { constants as buffer } from "node:buffer"; +import { deflateRawSync, inflateRawSync } from "node:zlib"; +import type { Compression } from "../next/link/index.ts"; + +/** + * Raw deflate at level 1 (`deflateRawSync`), and inflate bounded by `maxOutputLength` (`inflateRawSync`): past the + * limit `too_large`, data that is no raw deflate stream `corrupt`. + */ +export const nodeCompression: Compression = { + deflate: (bytes) => deflateRawSync(bytes, { level: 1 }), + inflate: (bytes, limit) => { + try { + // zlib refuses a bound of 0: inflate at most one byte, and one byte is past a limit of 0. + const inflated = inflateRawSync(bytes, { maxOutputLength: Math.max(1, Math.min(limit, buffer.MAX_LENGTH)) }); + return inflated.byteLength > limit ? { kind: "too_large" } : { kind: "bytes", bytes: inflated }; + } catch (error) { + if ((error as { readonly code?: unknown }).code === "ERR_BUFFER_TOO_LARGE") return { kind: "too_large" }; + return { kind: "corrupt", error }; + } + }, +}; diff --git a/packages/chord/src/next/contracts/README.md b/packages/chord/src/next/contracts/README.md new file mode 100644 index 000000000..a6e2a06ba --- /dev/null +++ b/packages/chord/src/next/contracts/README.md @@ -0,0 +1,80 @@ +# contracts + +Declarations of Chord object types: what an object's state and methods look like, version by version. + +## For apps (`index.ts`) + +```ts +import { arr, evolve, hostKind, type Implementation, method, obj, remote, str } from "./index.ts"; + +export const Session = hostKind("session", { id: str() }); // a kind of addressable host, params like a contract's +const Note = obj({ text: str() }); +export const NotesV1 = remote("notes", { + version: 1, + params: { session: str() }, + placement: ({ session }) => Session({ id: session }), // where an object lives; not in the fingerprint + variants: { + editor: { state: { notes: arr(Note) }, methods: { add: method(Note, str()), clear: method() } }, + reader: { state: { notes: arr(Note) } }, + }, +}); +export const Notes = evolve(NotesV1, (e) => e.args("add", obj({ text: str(), tag: str() }))); +type NotesProvider = Implementation; // { variant: "editor"; members: { notes, add(args, context), clear(context) } } | { variant: "reader"; ... } +``` + +- `remote(name, { version, params, placement?, variants })`: per-principal variants, each with `state` (schemas) and `methods` + (`method(args?, result?)`; an omitted side is `null`, and a method whose arguments are `null` omits them: + `clear(context)`). `local(name, { version, params, members })`: one flat map, + where a schema is state and `method(...)` a method. +- `placement(params) => Address` (remote only, optional): the address of the host that serves the object for these + params (`host/`'s routing table). The caller computes it and sends it with its hold, so a relay routes by the address + alone. It is not part of the normal form or the fingerprint and `evolve` carries it. +- `hostKind(name, params?)`: a kind of addressable host (`Server = hostKind("server")`, `Session = hostKind("session", + { id: str() })`); `params` follow the params policy below. Calling it makes an `Address { kind, params, key }`, its + key built like an identity key (`session|S1`); two addresses are equal when their keys are. The params are checked + where the address is routed or served (`invalid/placement`), not when it is made. No host-name strings in app code. +- Schema helpers: `str`, `num`, `int`, `bool`, `nul`, `lit`, `oneOf`, `union`, `obj`, `arr`, `map`, `opt`, `bytes`, + `unknown`, and `Static`. They return ordinary TypeBox schemas. +- `evolve(previous, (e) => e...)`: the next version from structured edits (`state`, `addState`, `removeState`, `args`, + `result`, `method`, `addMethod`, `removeMethod`); it must return the editor it was given. +- Types: `Contract`, `Implementation` (what a factory returns: for `local` a `Watchable` per state member and a + function per method, `(args, context)` or `(context)` without arguments; for `remote` one variant, `{ variant, + members }`; `X` is the context type the methods get), `ParamsOf`, `AcceptsTable` (the older versions a provider serves, with their mappings), + `HostKind`, `Address`, `KindShape` and `ParamsOfKind`. + +Rules: + +- Declarations are checked once and fail with an `Error` naming the path (`notes.editor.methods.add.args`). A contract + keeps its own copies of its schemas: changing a schema object after declaring it changes nothing. +- A member name has one kind (state or method) in every variant and for the whole history; a method has one + signature in every variant. State schemas may differ per variant. +- `then`, `release`, `variant`, `state` and `retry` are refused as member names: a reference has them (`then` would + make an awaited reference thenable). +- `$bytes` is reserved only in the values of a method whose argument or result schema has a `bytes()` leaf: there it + marks a byte segment on a link (an own key `$bytes` fails `invalid/args` when it crosses one); everywhere else it is + ordinary data. +- Every object inside a method's arguments is closed (`additionalProperties: false`); `obj()` always is, records + (`map()`) are allowed. +- Params are required `str()`, `int()`, `bool()` or `oneOf(...)` of strings. Params and the variant set never change by + evolution: changing them is a new history base, a new declaration with no accepted older versions. +- Mappings in an `AcceptsTable` are direct (old version to current); when a version is added, update every accepted + version's mappings. A mapping is required per edit after the accepted version, even for a no-op edit. +- Contracts are branded: a spread or hand-built copy is refused everywhere. + +## For Chord's layers (`internal.ts`) + +- `checkAccepts(contract, accepts)`: validates an accepts table without calling a mapping; returns the accepted + versions (version and fingerprint). +- `contractData(contract)`: the Chord-owned data behind a handle (its `placement`, its versions with their + `VariantData`, `MethodData` and schema snapshots, its `ParamsData`); a local contract's members are the variant + `LOCAL`. Type-level helpers: `ShapeOf`, `CurrentVariants`, `ContractShape`, `Params` and `Method`. +- `currentVariants(contract)`: the current version's variants (a local contract has the one variant `LOCAL`); + `takesNoArgs(method)`: whether its argument schema is `null`, so references and implementations omit the argument; + `VariantImplementation` is one variant's members as a type; `variantNames` and `variantOrder`: the names the wire's + indexes point into. +- A method's `args.bytes` and `result.bytes`: whether the schema has a `bytes()` leaf, decided once at declaration (a + link then encodes the method's values with byte segments). +- `kindParams(kind)`: a declared host kind's `ParamsData` (a route checks addresses with it). +- `NORMAL_FORM_VERSION`: changing the normal form changes it and the fingerprints; it is a link-protocol change. + Fingerprints are a portable hash, not a security boundary. +- Nothing in L1 is frozen: Chord-owned data is immutable by convention. diff --git a/packages/chord/src/next/contracts/accepts.ts b/packages/chord/src/next/contracts/accepts.ts new file mode 100644 index 000000000..1596f0132 --- /dev/null +++ b/packages/chord/src/next/contracts/accepts.ts @@ -0,0 +1,244 @@ +// Accepted versions: the older versions a provider still serves, with the direct mappings their edits require. + +import type { Context } from "../../context/index.ts"; +import { + type Contract, + type ContractData, + type ContractShape, + contractData, + type Kind, + LOCAL, + type ShapeOf, +} from "./contract.ts"; +import { checkKeys, entriesOf } from "./declare.ts"; +import type { + Change, + ChangesAfter, + CurrentStep, + MemberNames, + MethodAt, + OlderVersions, + StepAt, + VariantShape, + Variants, + VariantsWith, +} from "./history.ts"; +import type { Method, Static } from "./schema.ts"; + +/** Per accepted older version, exactly the mappings its later edits require (direct: old to current). */ +export type AcceptsTable = AcceptsOf>; + +type AcceptsOf = { + readonly [V in OlderVersions]?: VersionMappings< + S["kind"], + StepAt["variants"], + CurrentStep["variants"], + ChangesAfter + >; +}; + +type VersionMappings = Section< + "state", + K extends "local" ? StateMappings : RemoteStateMappings +> & + Section<"methods", MethodMappings>; + +type Local = typeof LOCAL; + +/** A section is required when it holds a required mapping; otherwise it may be absent or empty. */ +type Section = [keyof T] extends [never] + ? { readonly [K in Key]?: { readonly [member: string]: never } } + : { readonly [K in Key]: T }; + +/** `T` without its `never` properties: the members that need no mapping. */ +type OmitNever = { readonly [K in keyof T as [T[K]] extends [never] ? never : K]: T[K] }; + +type RemoteStateMappings = OmitNever<{ + [X in keyof Old & keyof Current]: [keyof StateMappings] extends [never] + ? never + : StateMappings; +}>; + +type StateMappings = OmitNever<{ + [S in keyof Old["state"]]: StateMapping; +}>; + +/** A changed member maps the current member back; a removed one maps the variant's whole current state, or refuses. */ +type StateMapping< + Old extends VariantShape, + Current extends VariantShape, + X, + S extends keyof Old["state"], + C, +> = S extends keyof Current["state"] + ? [X, S, "state"] extends C + ? (current: Static) => Static + : never + : ((state: { [M in keyof Current["state"]]: Static }) => Static) | "refuse"; + +type MethodMappings = OmitNever<{ + [M in MemberNames]: [RemovedFrom, M>] extends [never] + ? ChangedMethod< + Changed, M, "args">, + Changed, M, "result">, + MethodAt, + MethodAt + > + : Replacement>; +}>; + +/** The variants among `Xs` that no longer have method `M`. */ +type RemovedFrom = { + [X in Xs]: M extends keyof Current[X]["methods"] ? never : X; +}[Xs]; + +/** `true` when an edit changed `Side` of `M` in one of `Xs`. */ +type Changed = Xs extends unknown ? ([Xs, M, Side] extends C ? true : never) : never; + +/** Mappings for the changed sides of a method (`A`, `R` are `true` when changed), or a replacement. */ +type ChangedMethod = [A | R] extends [never] + ? never + : + | ({ readonly call?: never } & ([A] extends [never] + ? { readonly args?: never } + : { readonly args: (old: Static) => Static }) & + ([R] extends [never] + ? { readonly result?: never } + : { readonly result: (current: Static) => Static })) + | Replacement; + +/** A removed (or changed) method served by the provider's own function, or refused. */ +type Replacement = + | { + readonly call: (args: Static, context: Context) => Promise>; + readonly args?: never; + readonly result?: never; + } + | "refuse"; + +/** An accepted older version and its fingerprint. */ +export type AcceptedVersion = { readonly version: number; readonly fingerprint: string }; + +/** + * Validates an accepts table without calling any mapping: known older versions only, exactly the required mappings + * with the right shapes. Fails with an Error naming the path (`notes accepts[1].methods.add`). + */ +export function checkAccepts( + contract: C, + accepts: AcceptsTable, +): { readonly versions: readonly AcceptedVersion[] } { + const data = contractData(contract); + // Declaration policy: JavaScript callers reach every check below. + const versions = entriesOf(accepts, `${data.name} accepts`).map(([key, mappings]) => { + const path = `${data.name} accepts[${key}]`; + const index = data.versions.findIndex(({ version }) => String(version) === key); + if (index === -1 || index === data.versions.length - 1) throw new Error(`${path}: not an older version`); + checkMappings(mappings, needs(data, index), data.kind, path); + const { version, fingerprint } = data.versions[index]!; + return { version, fingerprint }; + }); + return { versions }; +} + +/** What an accepted version needs, by edit and never by schema comparison. */ +type Needs = { + /** Per variant, per state member. */ + readonly state: ReadonlyMap>; + readonly methods: ReadonlyMap; +}; + +type StateNeed = "changed" | "removed"; + +type MethodNeed = + | { readonly kind: "removed" } + | { readonly kind: "changed"; readonly args: boolean; readonly result: boolean }; + +function needs(data: ContractData, index: number): Needs { + const old = data.versions[index]!; + const current = data.versions.at(-1)!; + const changes = new Set(data.versions.slice(index + 1).flatMap((version) => version.changes.map(changeKey))); + const changed = (...change: Change) => changes.has(changeKey(change)); + const state = new Map>(); + const methods = new Map(); + for (const [variant, { state: oldState, methods: oldMethods }] of old.variants) { + const now = current.variants.get(variant)!; + const variantNeeds = new Map(); + for (const member of oldState.keys()) { + if (!now.state.has(member)) variantNeeds.set(member, "removed"); + else if (changed(variant, member, "state")) variantNeeds.set(member, "changed"); + } + if (variantNeeds.size > 0) state.set(variant, variantNeeds); + for (const member of oldMethods.keys()) { + const need = methods.get(member) ?? { removed: false, args: false, result: false }; + need.removed ||= !now.methods.has(member); + need.args ||= changed(variant, member, "args"); + need.result ||= changed(variant, member, "result"); + methods.set(member, need); + } + } + const methodNeeds = new Map(); + for (const [member, { removed, args, result }] of methods) { + if (removed) methodNeeds.set(member, { kind: "removed" }); + else if (args || result) methodNeeds.set(member, { kind: "changed", args, result }); + } + return { state, methods: methodNeeds }; +} + +function changeKey(change: Change): string { + return JSON.stringify(change); +} + +function checkMappings(mappings: unknown, needs: Needs, kind: Kind, path: string): void { + checkKeys(mappings, ["state", "methods"], path); + const { state, methods } = mappings as { readonly state?: unknown; readonly methods?: unknown }; + const checkVariant = (variantMappings: unknown, variantNeeds: ReadonlyMap, at: string) => + checkSection(variantMappings, variantNeeds, at, checkStateMapping); + // A local table is flat: its state section is the one variant's. + if (kind === "local") checkVariant(state, needs.state.get(LOCAL) ?? new Map(), `${path}.state`); + else checkSection(state, needs.state, `${path}.state`, checkVariant); + checkSection(methods, needs.methods, `${path}.methods`, checkMethodMapping); +} + +/** Exactly the needed keys: no missing and no unnecessary mapping; absent only when nothing is needed. */ +function checkSection( + section: unknown, + needs: ReadonlyMap, + path: string, + check: (mapping: unknown, need: N, path: string) => void, +): void { + const mappings = new Map(section === undefined ? [] : entriesOf(section, path)); + for (const key of mappings.keys()) if (!needs.has(key)) throw new Error(`${path}.${key}: unnecessary mapping`); + for (const [key, need] of needs) { + if (!mappings.has(key)) throw new Error(`${path}.${key}: missing mapping`); + check(mappings.get(key), need, `${path}.${key}`); + } +} + +function checkStateMapping(mapping: unknown, need: StateNeed, path: string): void { + if (typeof mapping === "function" || (need === "removed" && mapping === "refuse")) return; + throw new Error(`${path}: expected ${need === "removed" ? 'a function or "refuse"' : "a function"}`); +} + +function checkMethodMapping(mapping: unknown, need: MethodNeed, path: string): void { + if (mapping === "refuse") return; + checkKeys(mapping, ["args", "result", "call"], path); + const { args, result, call } = mapping as { + readonly args?: unknown; + readonly result?: unknown; + readonly call?: unknown; + }; + if (call !== undefined || need.kind === "removed") { + if (typeof call !== "function" || args !== undefined || result !== undefined) { + throw new Error(`${path}: expected { call } or "refuse"`); + } + return; + } + checkSide(args, need.args, `${path}.args`); + checkSide(result, need.result, `${path}.result`); +} + +function checkSide(mapping: unknown, needed: boolean, path: string): void { + if (!needed && mapping !== undefined) throw new Error(`${path}: unnecessary mapping`); + if (needed && mapping === undefined) throw new Error(`${path}: missing mapping`); + if (needed && typeof mapping !== "function") throw new Error(`${path}: expected a function`); +} diff --git a/packages/chord/src/next/contracts/contract.ts b/packages/chord/src/next/contracts/contract.ts new file mode 100644 index 000000000..cf113e96f --- /dev/null +++ b/packages/chord/src/next/contracts/contract.ts @@ -0,0 +1,178 @@ +// Contract handles: the branded, immutable value an app declares, its type-level shape, and the Chord-owned data +// behind it. + +import type { TNull } from "typebox"; +import type { Context } from "../../context/index.ts"; +import type { Watchable, WatchHandle } from "../../watch.ts"; +import type { Change, CurrentStep, History, VariantShape } from "./history.ts"; +import type { Address } from "./host-kind.ts"; +import type { Params, ParamsData } from "./params.ts"; +import type { Method, Static } from "./schema.ts"; +import type { Snapshot } from "./snapshot.ts"; + +export type Kind = "remote" | "local"; + +/** The variant name of a local contract's members. */ +export const LOCAL = "members"; + +/** Everything the types know about a contract: kind, name, params and the whole history. */ +export type ContractShape = { + readonly kind: Kind; + readonly name: string; + readonly params: Params; + readonly history: History; +}; + +/** A declared contract: a branded handle, immutable by convention. Spread or hand-built copies are refused. */ +export interface Contract { + readonly kind: S["kind"]; + readonly name: S["name"]; + readonly version: number; + readonly fingerprint: string; + /** Type only: absent at runtime. */ + readonly "~shape"?: S; +} + +export type ShapeOf = NonNullable; + +/** The variants of a contract's current version. */ +export type CurrentVariants = CurrentStep["history"]>["variants"]; + +/** The params object of a contract. */ +export type ParamsOf = { [F in keyof ShapeOf["params"]]: Static["params"][F]> }; + +/** + * What a factory returns: for `local`, a `Watchable` per state member and a function per method, `(args, context)`, + * or `(context)` when the method takes no arguments; for `remote`, one variant `{ variant, members }` with exactly + * that variant's members. `X` is the context the methods get. + */ +export type Implementation = ImplementationOf< + ShapeOf["kind"], + CurrentVariants, + X +>; + +type ImplementationOf< + K extends Kind, + V extends { readonly [variant: string]: VariantShape }, + X extends Context, +> = K extends "local" + ? VariantImplementation + : { [N in keyof V]: { readonly variant: N; readonly members: VariantImplementation } }[keyof V]; + +/** + * One variant's members: a `Watchable` per state member, a function per method and, in a local contract, a + * `KeyedWatchable` per keyed member. + */ +export type VariantImplementation = { + [M in keyof V["state"] | keyof V["methods"] | keyof NonNullable]: M extends keyof V["methods"] + ? MethodFunction + : M extends keyof V["state"] + ? Watchable> + : M extends keyof NonNullable + ? KeyedWatchable[M]>> + : never; +}; + +/** + * A keyed state member (slice 6a §12): documents by key, each watchable on its own; a keyed document handle + * provides it. `at(key)` is valid only in a factory (it pins the key while that attempt lives). + */ +export interface KeyedWatchable { + /** The document at `key`, `undefined` when absent. */ + get(key: string, context: Context): Promise; + /** A watch of the document at `key` that follows its address, `undefined` when absent. */ + watch(key: string, context: Context): Promise | undefined>; + /** In a factory only: the document at `key` as a watchable, `initial()` while absent. */ + at(key: string): Watchable; +} + +/** A method as a function: without the argument when its argument schema is `null` (`null` stays the wire form). */ +type MethodFunction = M["args"] extends TNull + ? (context: X) => Promise> + : (args: Static, context: X) => Promise>; + +/** The Chord-owned data behind a contract handle. */ +export type ContractData = { + readonly kind: Kind; + readonly name: string; + readonly params: ParamsData; + /** A remote contract's placement: the address of the host serving an object for its params. */ + readonly placement: Placement | undefined; + /** Every version of the history, oldest first; the last is current. */ + readonly versions: readonly VersionData[]; + /** The kind of every member name ever declared in the history. */ + readonly kinds: ReadonlyMap; +}; + +export type MemberKind = "state" | "method" | "keyed"; + +/** A placement as Chord calls it: with checked params. Plugin code: what it returns is checked where it is routed. */ +export type Placement = (params: ParamsOf) => Address; + +export type VersionData = { + readonly version: number; + readonly fingerprint: string; + readonly variants: ReadonlyMap; + /** The changes that produced this version; empty for a base. */ + readonly changes: readonly Change[]; +}; + +export type VariantData = { + readonly state: ReadonlyMap; + readonly methods: ReadonlyMap; + /** Keyed state members (slice 6a §12): only a local contract has any. */ + readonly keyed: ReadonlyMap; +}; + +export type MethodData = { readonly args: Snapshot; readonly result: Snapshot }; + +/** The data behind a contract handle; refuses spread and hand-built copies. */ +export function contractData(contract: Contract): ContractData { + const data = contracts.get(contract); + if (data === undefined) { + throw new Error(`${contract.name}: not a declared contract (spread and hand-built copies are refused)`); + } + return data; +} + +const contracts = new WeakMap(); + +/** Whether a method takes no arguments: its argument schema is `null`, so its functions omit the argument. */ +export function takesNoArgs(method: MethodData): boolean { + // A schema's normal form is an object of keywords. + const normal = method.args.normal as { readonly [keyword: string]: unknown }; + return normal.type === "null" && Object.keys(normal).length === 1; +} + +/** The variants of a contract's current version; a local contract has the one variant `LOCAL`. */ +export function currentVariants(contract: Contract): ReadonlyMap { + return contractData(contract).versions.at(-1)!.variants; +} + +/** A variant's names as links index them (2b): its state members and its methods, each sorted. */ +export type VariantNames = { readonly state: readonly string[]; readonly methods: readonly string[] }; + +/** The current version's variant names in the order links index them: sorted. */ +export function variantOrder(contract: Contract): readonly string[] { + return [...currentVariants(contract).keys()].sort(); +} + +/** A variant's state members and methods in the order links index them: each sorted. */ +export function variantNames(variant: VariantData): VariantNames { + return { state: [...variant.state.keys()].sort(), methods: [...variant.methods.keys()].sort() }; +} + +/** A new contract handle for `data`; the caller states the type-level shape. */ +export function createContract(data: ContractData): C { + const current = data.versions.at(-1)!; + const contract: Contract = { + kind: data.kind, + name: data.name, + version: current.version, + fingerprint: current.fingerprint, + }; + contracts.set(contract, data); + // The shape is type-level only; every caller states the shape it built. + return contract as C; +} diff --git a/packages/chord/src/next/contracts/declare.ts b/packages/chord/src/next/contracts/declare.ts new file mode 100644 index 000000000..2e3856c28 --- /dev/null +++ b/packages/chord/src/next/contracts/declare.ts @@ -0,0 +1,237 @@ +// Declarations: `remote(...)` and `local(...)`, the checks every declared version passes, and the reading of +// untyped declaration input. + +import { + type Contract, + createContract, + type Kind, + LOCAL, + type MemberKind, + type MethodData, + type Placement, + type VariantData, +} from "./contract.ts"; +import { materialize } from "./fingerprint.ts"; +import type { MemberNames, MethodAt, VariantShape } from "./history.ts"; +import type { Address } from "./host-kind.ts"; +import { checkName } from "./names.ts"; +import { type Params, type RequiredFields, readParams } from "./params.ts"; +import { KeyedMember, Method, type Static, type TSchema } from "./schema.ts"; +import { type Snapshot, snapshotArguments, snapshotSchema } from "./snapshot.ts"; + +type VariantDeclaration = { + readonly state?: { readonly [member: string]: TSchema }; + readonly methods?: { readonly [member: string]: Method }; +}; + +type VariantDeclarations = { readonly [variant: string]: VariantDeclaration }; + +type RemoteVariants = { + [V in keyof D]: { + state: D[V]["state"] extends object ? D[V]["state"] : Record; + methods: D[V]["methods"] extends object ? D[V]["methods"] : Record; + }; +}; + +type LocalVariants = { + [LOCAL]: { + state: { [K in keyof M as M[K] extends Method | KeyedMember ? never : K]: Exclude }; + methods: { [K in keyof M as M[K] extends Method ? K : never]: Extract }; + keyed: { [K in keyof M as M[K] extends KeyedMember ? K : never]: SchemaOf }; + }; +}; + +/** A local contract's member: a state schema, `method(...)` or `keyed(schema)`. */ +type LocalMember = TSchema | Method | KeyedMember; + +type SchemaOf = M extends KeyedMember ? S : never; + +/** D11: a member name has one kind in every variant, a method one signature; no keyed members. */ +type SameMembers> = { + [X in keyof D]: { + state?: { + // Keyed members are only in local contracts (slice 6a §12). + [M in keyof D[X]["state"]]: M extends MemberNames + ? never + : D[X]["state"][M] extends KeyedMember + ? never + : D[X]["state"][M]; + }; + methods?: { + [M in keyof D[X]["methods"]]: M extends MemberNames + ? never + : [MethodAt] extends [D[X]["methods"][M]] + ? D[X]["methods"][M] + : never; + }; + }; +}; + +/** Declares a remote contract: per-principal variants of state and methods. */ +export function remote< + const N extends string, + const P extends Params, + const V extends number, + const D extends VariantDeclarations, +>( + name: N, + declaration: { + readonly version: V; + readonly params: P & NoInfer>; + /** The address of the host serving an object for these params; never part of the fingerprint. */ + readonly placement?: (params: { [F in keyof P]: Static }) => Address; + readonly variants: D & NoInfer>; + }, +): Contract<{ + kind: "remote"; + name: N; + params: P; + history: [{ version: `${V}`; variants: RemoteVariants; changes: never }]; +}> { + // Declaration policy: JavaScript callers reach every check below. + checkName(name, "A contract name"); + checkKeys(declaration, ["version", "params", "placement", "variants"], name); + const variants = entriesOf(declaration.variants, `${name}.variants`); + if (variants.length === 0) throw new Error(`${name}.variants: at least one variant is required`); + const { placement } = declaration; + if (placement !== undefined && typeof placement !== "function") { + throw new Error(`${name}.placement: expected a function of the params`); + } + const read = variants.map(([variant, members]) => [variant, readVariant(members, `${name}.${variant}`)] as const); + // The placement gets the checked params of this contract. + return declare("remote", name, declaration, new Map(read), placement as Placement | undefined); +} + +/** + * Declares a local contract: one flat map of members, where a schema is state, `method(...)` a method and + * `keyed(schema)` a keyed state member. + */ +export function local< + const N extends string, + const P extends Params, + const V extends number, + const M extends { readonly [member: string]: LocalMember }, +>( + name: N, + declaration: { readonly version: V; readonly params: P & NoInfer>; readonly members: M }, +): Contract<{ + kind: "local"; + name: N; + params: P; + history: [{ version: `${V}`; variants: LocalVariants; changes: never }]; +}> { + // Declaration policy: JavaScript callers reach every check below. + checkName(name, "A contract name"); + checkKeys(declaration, ["version", "params", "members"], name); + const state = new Map(); + const methods = new Map(); + const keyed = new Map(); + for (const [member, value] of entriesOf(declaration.members, `${name}.${LOCAL}`)) { + const path = `${name}.${LOCAL}.${member}`; + if (value instanceof Method) methods.set(member, readMethod(value, path)); + else if (value instanceof KeyedMember) keyed.set(member, snapshotSchema(value.schema, path)); + else state.set(member, snapshotSchema(value, path)); + } + return declare("local", name, declaration, new Map([[LOCAL, { state, methods, keyed }]])); +} + +function readVariant(source: unknown, path: string): VariantData { + checkKeys(source, ["state", "methods"], path); + const { state = {}, methods = {} } = source as VariantDeclaration; + const stateEntries = entriesOf(state, `${path}.state`).map(([member, schema]) => { + if (schema instanceof Method) throw new Error(`${path}.state.${member}: a method belongs in methods`); + if (schema instanceof KeyedMember) { + throw new Error(`${path}.state.${member}: keyed members are only in local contracts`); + } + return [member, snapshotSchema(schema, `${path}.state.${member}`)] as const; + }); + const methodEntries = entriesOf(methods, `${path}.methods`).map(([member, method]) => { + if (!(method instanceof Method)) throw new Error(`${path}.methods.${member}: expected method(...)`); + return [member, readMethod(method, `${path}.methods.${member}`)] as const; + }); + return { state: new Map(stateEntries), methods: new Map(methodEntries), keyed: new Map() }; +} + +/** One method declaration as Chord-owned snapshots; its arguments must be closed. */ +export function readMethod(method: Method, path: string): MethodData { + return { + args: snapshotArguments(method.args, `${path}.args`), + result: snapshotSchema(method.result, `${path}.result`), + }; +} + +function declare( + kind: Kind, + name: string, + declaration: { readonly version: number; readonly params: unknown }, + variants: ReadonlyMap, + placement?: Placement, +): C { + const { version } = declaration; + if (!Number.isSafeInteger(version) || version < 1) throw new Error(`${name}.version: expected a positive integer`); + const kinds = new Map(); + checkMembers(variants, kinds, (variant, member, memberKind) => memberPath(name, kind, variant, member, memberKind)); + const params = readParams(entriesOf(declaration.params, `${name}.params`), `${name}.params`); + const versions = [materialize(kind, params, version, variants, [])]; + return createContract({ kind, name, params, placement, versions, kinds }); +} + +/** + * Checks the members of every variant: a name keeps one kind across variants and the whole history (`kinds`, updated + * here), a method one signature across variants. + */ +export function checkMembers( + variants: ReadonlyMap, + kinds: Map, + path: (variant: string, member: string, kind: MemberKind) => string, +): void { + const signatures = new Map(); + for (const [variant, { state, methods, keyed }] of variants) { + for (const member of state.keys()) recordKind(kinds, member, "state", path(variant, member, "state")); + for (const member of keyed.keys()) recordKind(kinds, member, "keyed", path(variant, member, "keyed")); + for (const [member, { args, result }] of methods) { + recordKind(kinds, member, "method", path(variant, member, "method")); + const signature = JSON.stringify([args.normal, result.normal]); + if ((signatures.get(member) ?? signature) !== signature) { + throw new Error(`${path(variant, member, "method")}: a method has the same signature in every variant`); + } + signatures.set(member, signature); + } + } +} + +function recordKind(kinds: Map, member: string, kind: MemberKind, path: string): void { + if (RESERVED_MEMBERS.has(member)) throw new Error(`${path}: "${member}" is reserved: a reference has it`); + const known = kinds.get(member); + if (known !== undefined && known !== kind) { + throw new Error(`${path}: "${member}" is a ${known} member; its kind never changes`); + } + kinds.set(member, kind); +} + +/** + * Names a reference has itself (`release`, `variant`, and `state` and `retry` with failures), and `then`, which would + * make an awaited reference thenable. + */ +const RESERVED_MEMBERS: ReadonlySet = new Set(["then", "release", "variant", "state", "retry"]); + +/** The path of a member in errors: `notes.editor.methods.add`, `info.members.clear`. */ +export function memberPath(base: string, kind: Kind, variant: string, member: string, memberKind: MemberKind): string { + if (kind === "local") return `${base}.${LOCAL}.${member}`; + return `${base}.${variant}.${memberKind === "state" ? "state" : "methods"}.${member}`; +} + +/** Refuses a non-object and unknown keys. */ +export function checkKeys(source: unknown, allowed: readonly string[], path: string): void { + for (const [key] of entriesOf(source, path)) { + if (!allowed.includes(key)) throw new Error(`${path}: unknown key "${key}"`); + } +} + +/** The entries of an object; anything else is refused. */ +export function entriesOf(source: unknown, path: string): [string, unknown][] { + if (typeof source !== "object" || source === null || Array.isArray(source)) { + throw new Error(`${path}: expected an object`); + } + return Object.entries(source); +} diff --git a/packages/chord/src/next/contracts/evolve.ts b/packages/chord/src/next/contracts/evolve.ts new file mode 100644 index 000000000..9e405d732 --- /dev/null +++ b/packages/chord/src/next/contracts/evolve.ts @@ -0,0 +1,310 @@ +// Evolution: `evolve(previous, (e) => e...)` appends the next version of a history with structured edits. + +import { + type Contract, + type ContractData, + type ContractShape, + type CurrentVariants, + contractData, + createContract, + type Kind, + LOCAL, + type MemberKind, + type MethodData, + type ShapeOf, +} from "./contract.ts"; +import { checkMembers, memberPath, readMethod } from "./declare.ts"; +import { materialize } from "./fingerprint.ts"; +import type { Change, CurrentStep, Increment, MemberNames, MethodAt, Variants, VariantsWith } from "./history.ts"; +import { Method, type TSchema } from "./schema.ts"; +import { type Snapshot, snapshotArguments, snapshotSchema } from "./snapshot.ts"; + +/** Method edits, the same for both kinds: each changes the method in every variant that has it. */ +interface MethodEditor { + /** Type only: the variants after the edits so far. */ + readonly "~variants"?: V; + /** Type only: the changes made by the edits so far. */ + readonly "~changes"?: C; + args, A extends TSchema>( + member: M, + args: A, + ): Editor< + K, + Put, "methods", M, Method["result"]>>, + C | [VariantsWith, M, "args"] + >; + result, R extends TSchema>( + member: M, + result: R, + ): Editor< + K, + Put, "methods", M, Method["args"], R>>, + C | [VariantsWith, M, "result"] + >; + method, A extends TSchema, R extends TSchema>( + member: M, + args: A, + result: R, + ): Editor, "methods", M, Method>, C | [VariantsWith, M, "args" | "result"]>; + removeMethod>(member: M): Editor, C>; +} + +/** Edits of a remote contract. */ +interface RemoteEditor extends MethodEditor<"remote", V, C> { + state( + variant: X, + member: M, + schema: S, + ): RemoteEditor, C | [X, M, "state"]>; + addState( + variant: X, + member: M, + schema: S, + ): RemoteEditor, C | [X, M, "state"]>; + removeState( + variant: X, + member: M, + ): RemoteEditor, C>; + addMethod( + variant: X, + member: M, + args: A, + result: R, + ): RemoteEditor>, C | [X, M, "args" | "result"]>; +} + +/** Edits of a local contract: like remote ones, without a variant. */ +interface LocalEditor extends MethodEditor<"local", V, C> { + state( + member: M, + schema: S, + ): LocalEditor, C | [Local, M, "state"]>; + addState( + member: M, + schema: S, + ): LocalEditor, C | [Local, M, "state"]>; + removeState(member: M): LocalEditor, C>; + addMethod( + member: M, + args: A, + result: R, + ): LocalEditor>, C | [Local, M, "args" | "result"]>; +} + +type Local = typeof LOCAL; + +type Editor = K extends "remote" + ? RemoteEditor + : LocalEditor; + +type EditorOf = Editor["kind"], CurrentVariants, never>; + +/** Variants `X` with member `M` of `Side` set to `T`. */ +type Put = { + [Y in keyof V]: Y extends X + ? { + [S in keyof V[Y]]: S extends Side ? { [K in keyof V[Y][S] | M]: K extends M ? T : V[Y][S][K] } : V[Y][S]; + } + : V[Y]; +}; + +/** Variants `X` without member `M` of `Side`. */ +type Drop = { + [Y in keyof V]: Y extends X ? { [S in keyof V[Y]]: S extends Side ? Omit : V[Y][S] } : V[Y]; +}; + +/** The sides of a variant that edits change; keyed members are carried over unchanged (their versions: slice 7). */ +type EditedSide = "state" | "methods"; + +type Evolved> = { + kind: S["kind"]; + name: S["name"]; + params: S["params"]; + history: [ + ...S["history"], + { + version: Increment["version"]>; + variants: NonNullable; + changes: NonNullable; + }, + ]; +}; + +/** + * Appends the next version (`previous.version + 1`) with the edits `edit` makes; `edit` must return the editor it was + * given. Params and the variant set never change: a change to either is a new history base. + */ +export function evolve["kind"], Variants, Change>>( + previous: C, + edit: (editor: EditorOf) => E, +): Contract, E>> { + const draft = new Draft(contractData(previous)); + const editor = draft.kind === "local" ? localEditor(draft) : draft; + // The editor interfaces track the edits in the types; at runtime every edit returns the same editor. + editWith(editor as unknown as EditorOf, edit, `${draft.path}: evolve`); + return createContract(draft.next()); +} + +/** + * Runs `edit` on `editor` (contracts' and tables' rule): an edit returning anything but the editor it was given throws + * (`what` names the call). + */ +export function editWith(editor: E, edit: (editor: E) => unknown, what: string): void { + if (edit(editor) !== editor) throw new Error(`${what} must return the editor it was given`); +} + +/** A variant under construction; keyed members are carried over unchanged (their versions are slice 7's). */ +type DraftVariant = { + readonly state: Map; + readonly methods: Map; + readonly keyed: ReadonlyMap; +}; + +/** The next version under construction, and the remote editor. Every edit is checked as it is made. */ +class Draft { + readonly kind: Kind; + readonly path: string; + /** What edits return: the draft itself, or the local editor wrapping it. */ + editor: object = this; + readonly #data: ContractData; + readonly #version: number; + readonly #variants: Map; + readonly #kinds: Map; + readonly #changes: Change[] = []; + + constructor(data: ContractData) { + const current = data.versions.at(-1)!; + this.kind = data.kind; + this.#data = data; + this.#version = current.version + 1; + this.path = `${data.name} v${this.#version}`; + this.#variants = new Map( + [...current.variants].map(([name, { state, methods, keyed }]) => [ + name, + { state: new Map(state), methods: new Map(methods), keyed }, + ]), + ); + this.#kinds = new Map(data.kinds); + } + + state(variant: string, member: string, schema: TSchema): object { + if (!this.#variant(variant).state.has(member)) { + throw new Error(`${this.#path(variant, member, "state")}: no such state member`); + } + return this.#putState(variant, member, schema); + } + + addState(variant: string, member: string, schema: TSchema): object { + this.#refuseDeclared(variant, member, "state"); + return this.#putState(variant, member, schema); + } + + removeState(variant: string, member: string): object { + if (!this.#variant(variant).state.delete(member)) { + throw new Error(`${this.#path(variant, member, "state")}: no such state member`); + } + return this.editor; + } + + args(member: string, args: TSchema): object { + return this.#changeMethod(member, args, undefined); + } + + result(member: string, result: TSchema): object { + return this.#changeMethod(member, undefined, result); + } + + method(member: string, args: TSchema, result: TSchema): object { + return this.#changeMethod(member, args, result); + } + + addMethod(variant: string, member: string, args: TSchema, result: TSchema): object { + this.#refuseDeclared(variant, member, "method"); + this.#variant(variant).methods.set( + member, + readMethod(new Method(args, result), this.#path(variant, member, "method")), + ); + this.#changes.push([variant, member, "args"], [variant, member, "result"]); + return this.#checked(); + } + + removeMethod(member: string): object { + for (const variant of this.#variantsWith(member)) this.#variant(variant).methods.delete(member); + return this.editor; + } + + /** The contract data with this version appended. */ + next(): ContractData { + const { kind, params, versions } = this.#data; + const version = materialize(kind, params, this.#version, this.#variants, this.#changes); + return { ...this.#data, versions: [...versions, version], kinds: this.#kinds }; + } + + #putState(variant: string, member: string, schema: TSchema): object { + this.#variant(variant).state.set(member, snapshotSchema(schema, this.#path(variant, member, "state"))); + this.#changes.push([variant, member, "state"]); + return this.#checked(); + } + + /** Changes the given sides of a method in every variant that has it. */ + #changeMethod(member: string, args: TSchema | undefined, result: TSchema | undefined): object { + const variants = this.#variantsWith(member); + const path = this.#path(variants[0]!, member, "method"); + const newArgs = args === undefined ? undefined : snapshotArguments(args, `${path}.args`); + const newResult = result === undefined ? undefined : snapshotSchema(result, `${path}.result`); + for (const variant of variants) { + const { methods } = this.#variant(variant); + const old = methods.get(member)!; + methods.set(member, { args: newArgs ?? old.args, result: newResult ?? old.result }); + if (newArgs !== undefined) this.#changes.push([variant, member, "args"]); + if (newResult !== undefined) this.#changes.push([variant, member, "result"]); + } + return this.#checked(); + } + + #variant(variant: string): DraftVariant { + const found = this.#variants.get(variant); + if (found === undefined) { + throw new Error(`${this.path}.${variant}: unknown variant (the variant set never changes)`); + } + return found; + } + + #variantsWith(member: string): string[] { + const variants = [...this.#variants].filter(([, { methods }]) => methods.has(member)).map(([variant]) => variant); + if (variants.length === 0) throw new Error(`${this.path}: no method "${member}"`); + return variants; + } + + #refuseDeclared(variant: string, member: string, kind: MemberKind): void { + const { state, methods } = this.#variant(variant); + if (state.has(member) || methods.has(member)) { + throw new Error(`${this.#path(variant, member, kind)}: already declared`); + } + } + + #path(variant: string, member: string, kind: MemberKind): string { + return memberPath(this.path, this.kind, variant, member, kind); + } + + /** A member's kind never changes, also not after a removal; a method keeps one signature across variants. */ + #checked(): object { + checkMembers(this.#variants, this.#kinds, (variant, member, kind) => this.#path(variant, member, kind)); + return this.editor; + } +} + +/** The local editor: the draft's edits with the variant filled in. */ +function localEditor(draft: Draft): object { + draft.editor = { + state: (member: string, schema: TSchema) => draft.state(LOCAL, member, schema), + addState: (member: string, schema: TSchema) => draft.addState(LOCAL, member, schema), + removeState: (member: string) => draft.removeState(LOCAL, member), + addMethod: (member: string, args: TSchema, result: TSchema) => draft.addMethod(LOCAL, member, args, result), + args: (member: string, args: TSchema) => draft.args(member, args), + result: (member: string, result: TSchema) => draft.result(member, result), + method: (member: string, args: TSchema, result: TSchema) => draft.method(member, args, result), + removeMethod: (member: string) => draft.removeMethod(member), + }; + return draft.editor; +} diff --git a/packages/chord/src/next/contracts/fingerprint.ts b/packages/chord/src/next/contracts/fingerprint.ts new file mode 100644 index 000000000..2cce1b7f0 --- /dev/null +++ b/packages/chord/src/next/contracts/fingerprint.ts @@ -0,0 +1,69 @@ +// Fingerprints: the normal form of a contract version and its portable 64-bit hash. Not a security boundary. + +import type { JsonValue } from "../../json.ts"; +import { type Kind, LOCAL, type VariantData, type VersionData } from "./contract.ts"; +import type { Change } from "./history.ts"; +import type { ParamsData } from "./params.ts"; + +/** Version of the normal form; changing the normal form changes it and is a link-protocol change. */ +export const NORMAL_FORM_VERSION = 2; + +/** + * Materializes a version with its fingerprint: `hash64` of `JSON.stringify({ normalFormVersion, value })` over the + * normal form `{ kind: "remote", params, variants }` or `{ kind: "local", params, members }`. + */ +export function materialize( + kind: Kind, + params: ParamsData, + version: number, + variants: ReadonlyMap, + changes: readonly Change[], +): VersionData { + const normal: JsonValue = + kind === "local" + ? { kind, params: params.normal, members: variantNormal(variants.get(LOCAL)!) } + : { + kind, + params: params.normal, + variants: sortedDict([...variants].map(([name, v]) => [name, variantNormal(v)])), + }; + return { version, fingerprint: fingerprintOf(normal), variants, changes }; +} + +/** `hash64` of `JSON.stringify({ normalFormVersion, value: normal })`: the fingerprint of a normal form. */ +export function fingerprintOf(normal: JsonValue): string { + return hash64(JSON.stringify({ normalFormVersion: NORMAL_FORM_VERSION, value: normal })); +} + +/** + * Members as a sorted dictionary of `{ state }`, `{ args, result }` or `{ keyed }` (slice 6a §12: a node of its own, + * so contracts without keyed members keep their fingerprints and `NORMAL_FORM_VERSION` stays). + */ +function variantNormal({ state, methods, keyed }: VariantData): JsonValue { + return sortedDict([ + ...[...state].map(([member, schema]) => [member, { state: schema.normal }] as const), + ...[...keyed].map(([member, schema]) => [member, { keyed: schema.normal }] as const), + ...[...methods].map( + ([member, { args, result }]) => [member, { args: args.normal, result: result.normal }] as const, + ), + ]); +} + +function sortedDict(entries: (readonly [string, JsonValue])[]): JsonValue { + return Object.fromEntries(entries.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))); +} + +/** 16 hex characters from two 32-bit FNV-1a lanes (different offset bases and primes) over the UTF-8 bytes. */ +export function hash64(text: string): string { + let low = 0x811c9dc5; + let high = 0xcbf29ce4; + for (const byte of new TextEncoder().encode(text)) { + low = Math.imul(low ^ byte, 0x01000193); + high = Math.imul(high ^ byte, 0x01000197); + } + return hex(high) + hex(low); +} + +function hex(lane: number): string { + return (lane >>> 0).toString(16).padStart(8, "0"); +} diff --git a/packages/chord/src/next/contracts/history.ts b/packages/chord/src/next/contracts/history.ts new file mode 100644 index 000000000..24c0d7f33 --- /dev/null +++ b/packages/chord/src/next/contracts/history.ts @@ -0,0 +1,66 @@ +// The type-level history of a contract: every version's members and the changes that produced it. + +import type { Method, TSchema } from "./schema.ts"; + +/** One variant's members at one version: state schemas and method signatures. */ +export type VariantShape = { + readonly state: { readonly [member: string]: TSchema }; + readonly methods: { readonly [member: string]: Method }; + /** A local contract's keyed state members (slice 6a §12). */ + readonly keyed?: { readonly [member: string]: TSchema }; +}; + +/** Variants by name. A local contract has the single variant `members`. */ +export type Variants = { readonly [variant: string]: VariantShape }; + +/** What one edit changed: one side of one member in one variant (a union of variants when it changed several). */ +export type Change = readonly [variant: string, member: string, side: "state" | "args" | "result"]; + +/** One version: its number as digits, its variants, and the changes that produced it (`never` for a base). */ +type Step = { readonly version: string; readonly variants: Variants; readonly changes: Change }; + +/** Every version from the base, oldest first. */ +export type History = readonly Step[]; + +export type CurrentStep = H extends readonly [...Step[], infer Last extends Step] ? Last : never; + +export type StepAt = Extract; + +/** Versions before the current one. */ +export type OlderVersions = H extends readonly [...infer Older extends Step[], Step] + ? Older[number]["version"] + : never; + +/** Every change made after version `V`. */ +export type ChangesAfter = H extends readonly [ + infer First extends Step, + ...infer Rest extends Step[], +] + ? First["version"] extends V + ? Rest[number]["changes"] + : ChangesAfter + : never; + +/** Names of the state members or methods in any variant. */ +export type MemberNames = { + [X in keyof V]: keyof V[X][Side]; +}[keyof V] & + string; + +/** The variants that have method `M`. */ +export type VariantsWith = { + [X in keyof V]: M extends keyof V[X]["methods"] ? X : never; +}[keyof V] & + string; + +/** Method `M` (one signature in every variant that has it). */ +export type MethodAt = { + [X in keyof V]: M extends keyof V[X]["methods"] ? V[X]["methods"][M] : never; +}[keyof V]; + +/** `N + 1` on decimal digits, so versions beyond 9 and 99 stay literal. */ +export type Increment = N extends `${infer Init}9` + ? `${Init extends "" ? "1" : Increment}0` + : { [D in keyof NextDigit]: N extends `${infer Init}${D}` ? `${Init}${NextDigit[D]}` : never }[keyof NextDigit]; + +type NextDigit = { "0": "1"; "1": "2"; "2": "3"; "3": "4"; "4": "5"; "5": "6"; "6": "7"; "7": "8"; "8": "9" }; diff --git a/packages/chord/src/next/contracts/host-kind.ts b/packages/chord/src/next/contracts/host-kind.ts new file mode 100644 index 000000000..3f0e61055 --- /dev/null +++ b/packages/chord/src/next/contracts/host-kind.ts @@ -0,0 +1,68 @@ +// Host kinds and addresses (§3): a kind of addressable host with params like a contract's, and the address of one +// host of a kind, which a remote contract's placement computes from its params. + +import { checkName } from "./names.ts"; +import { joinKey, type Params, type ParamsData, type RequiredFields, readParams } from "./params.ts"; +import type { Static } from "./schema.ts"; + +/** What an address's type needs of its kind: the name and the params, type only. */ +export type KindShape = { readonly kind: string; readonly "~params"?: Params }; + +/** One host of a kind: the kind's name, its params and its key (`session|S1`); two addresses are equal by key. */ +export type Address = { + readonly kind: K["kind"]; + readonly params: ParamsOfKind; + readonly key: string; +}; + +/** A kind of addressable host: called with its params, it makes an address. */ +export interface HostKind { + ( + ...params: keyof P extends never ? [params?: Record] : [params: { [F in keyof P]: Static }] + ): Address>; + readonly kind: N; + /** Type only: absent at runtime. */ + readonly "~params"?: P; +} + +/** The params object of a host kind. */ +export type ParamsOfKind = { + [F in keyof NonNullable]: Static[F]>; +}; + +/** + * Declares a host kind; `params` follow the params policy of contracts (a flat record of required `str()`, `int()`, + * `bool()` and `oneOf(...)`). Making an address computes its key only: its params are checked where it is routed. + */ +export function hostKind>( + name: N, + params?: P & NoInfer>, +): HostKind { + checkName(name, "A host kind's name"); + // `client` is the host kind of a host without an address in store names (owner strings, table names). + if (name === "client") throw new Error('A host kind\'s name "client" is reserved for hosts without an address'); + const data = readParams(Object.entries(params ?? {}), `${name}.params`); + const make = (values: { readonly [field: string]: unknown } = {}) => ({ + kind: name, + params: values, + key: addressKey(name, data, values), + }); + Object.defineProperty(make, "kind", { value: name, enumerable: true }); + kinds.set(make, data); + // The function and its `kind` are the declared shape; the params type is type-level only. + return make as unknown as HostKind; +} + +const kinds = new WeakMap(); + +/** The params of a declared host kind. */ +export function kindParams(kind: { readonly kind: string }): ParamsData { + const data = kinds.get(kind); + if (data === undefined) throw new Error(`${kind.kind}: not a declared host kind`); + return data; +} + +/** An address's key: the kind's name and each field value in declared order, joined (`joinKey`). */ +export function addressKey(kind: string, params: ParamsData, values: { readonly [field: string]: unknown }): string { + return joinKey([kind, ...params.fields.map((field) => String(values[field]))]); +} diff --git a/packages/chord/src/next/contracts/index.ts b/packages/chord/src/next/contracts/index.ts new file mode 100644 index 000000000..299bac9d3 --- /dev/null +++ b/packages/chord/src/next/contracts/index.ts @@ -0,0 +1,30 @@ +// Contracts for apps: declaring, evolving and implementing Chord object types. + +export type { AcceptsTable } from "./accepts.ts"; +export type { Contract, Implementation, KeyedWatchable, ParamsOf } from "./contract.ts"; +export { local, remote } from "./declare.ts"; +export { evolve } from "./evolve.ts"; +export { type Address, type HostKind, hostKind, type KindShape, type ParamsOfKind } from "./host-kind.ts"; +export { + arr, + type Bounds, + bool, + bytes, + type Defaulted, + int, + type KeyedMember, + keyed, + lit, + map, + method, + nul, + num, + obj, + oneOf, + opt, + type Static, + str, + type TSchema, + union, + unknown, +} from "./schema.ts"; diff --git a/packages/chord/src/next/contracts/internal.ts b/packages/chord/src/next/contracts/internal.ts new file mode 100644 index 000000000..9f43298fe --- /dev/null +++ b/packages/chord/src/next/contracts/internal.ts @@ -0,0 +1,25 @@ +// Contracts for Chord's layers: accepted versions, the data behind a contract handle, and the types L4 dispatches on. + +export { checkAccepts } from "./accepts.ts"; +export { + type ContractShape, + type CurrentVariants, + contractData, + currentVariants, + LOCAL, + type MethodData, + type ShapeOf, + takesNoArgs, + type VariantData, + type VariantImplementation, + type VariantNames, + variantNames, + variantOrder, +} from "./contract.ts"; +export { editWith } from "./evolve.ts"; +export { fingerprintOf, hash64, NORMAL_FORM_VERSION } from "./fingerprint.ts"; +export type { Increment } from "./history.ts"; +export { kindParams } from "./host-kind.ts"; +export { checkName } from "./names.ts"; +export type { Params, ParamsData } from "./params.ts"; +export { Method } from "./schema.ts"; diff --git a/packages/chord/src/next/contracts/names.ts b/packages/chord/src/next/contracts/names.ts new file mode 100644 index 000000000..e31294039 --- /dev/null +++ b/packages/chord/src/next/contracts/names.ts @@ -0,0 +1,14 @@ +// Names that become SQL names or stored identities in the store (decision "Restricted names"): object types, host +// kinds, tables, state definitions and jobs use npm package-name characters, lowercase, so `::` is injective and readable, and a document's or a job record's name is stored as declared. + +/** Lowercase npm package-name characters: `:` cannot occur, so names joined with it stay apart. */ +const NAME = /^[a-z0-9@/._~-]+$/; + +/** Throws unless `name` is a nonempty lowercase name of `[a-z0-9@/._~-]` (`what` names it in the error). */ +export function checkName(name: string, what: string): void { + // Declaration policy: JavaScript callers reach this check. + if (typeof name !== "string" || !NAME.test(name)) { + throw new Error(`${what} ${JSON.stringify(name)}: expected lowercase [a-z0-9@/._~-]`); + } +} diff --git a/packages/chord/src/next/contracts/params.ts b/packages/chord/src/next/contracts/params.ts new file mode 100644 index 000000000..565c88500 --- /dev/null +++ b/packages/chord/src/next/contracts/params.ts @@ -0,0 +1,77 @@ +// Params declarations: the policy for params fields and the closed object schema checked on every get. + +import type { TBoolean, TInteger, TLiteral, TString, TUnion } from "typebox"; +import type { JsonValue } from "../../json.ts"; +import type { TSchema } from "./schema.ts"; +import { snapshotSchema } from "./snapshot.ts"; + +/** A params field: a string, an integer, a boolean or a union of string literals; constraints allowed. */ +type ParamSchema = TString | TInteger | TBoolean | TUnion[]>; + +/** Params: a flat record of required fields, in identity order. */ +export type Params = { readonly [field: string]: ParamSchema }; + +/** Refuses optional params fields, which the field types alone admit. */ +export type RequiredFields

= { + [F in keyof P]: P[F] extends { readonly "~optional": true } ? never : P[F]; +}; + +export type ParamsData = { + /** Field names in identity order. */ + readonly fields: readonly string[]; + /** The closed object schema of all fields. */ + readonly schema: TSchema; + /** `[field, normal form]` pairs in identity order. */ + readonly normal: JsonValue; +}; + +/** Reads declared params fields: each a required str(), int(), bool() or oneOf(...) of strings, constraints allowed. */ +export function readParams(declared: readonly [string, unknown][], path: string): ParamsData { + const fields = declared.map(([field, schema]) => { + const fieldPath = `${path}.${field}`; + const snapshot = snapshotSchema(schema, fieldPath); + if (Object.hasOwn(schema as object, "~optional") || !isParamSchema(snapshot.normal)) { + throw new Error(`${fieldPath}: a param is a required str(), int(), bool() or oneOf(...) of strings`); + } + return [field, snapshot] as const; + }); + const schema = { + type: "object", + properties: Object.fromEntries(fields.map(([field, snapshot]) => [field, snapshot.schema])), + required: fields.map(([field]) => field), + additionalProperties: false, + }; + return { + fields: fields.map(([field]) => field), + schema, + normal: fields.map(([field, snapshot]) => [field, snapshot.normal]), + }; +} + +/** Joins key parts with `|`, each escaped (`\\` and `|`), so two different part lists never join to one key. */ +export function joinKey(parts: readonly string[]): string { + return parts.map((part) => part.replaceAll("\\", "\\\\").replaceAll("|", "\\|")).join("|"); +} + +type SchemaNode = { readonly [keyword: string]: JsonValue }; + +function isParamSchema(normal: JsonValue): boolean { + const node = normal as SchemaNode; + if (node.anyOf !== undefined) { + return Object.keys(node).length === 1 && Array.isArray(node.anyOf) && node.anyOf.every(isStringLiteral); + } + const allowed = PARAM_KEYWORDS.get(node.type); + return allowed !== undefined && Object.keys(node).every((key) => allowed.has(key)); +} + +/** Keywords a params field may use, by type. */ +const PARAM_KEYWORDS = new Map>([ + ["string", new Set(["type", "minLength", "maxLength", "pattern", "format"])], + ["integer", new Set(["type", "minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum", "multipleOf"])], + ["boolean", new Set(["type"])], +]); + +function isStringLiteral(branch: JsonValue): boolean { + const node = branch as SchemaNode; + return node.type === "string" && typeof node.const === "string" && Object.keys(node).length === 2; +} diff --git a/packages/chord/src/next/contracts/schema.ts b/packages/chord/src/next/contracts/schema.ts new file mode 100644 index 000000000..fc987dbb2 --- /dev/null +++ b/packages/chord/src/next/contracts/schema.ts @@ -0,0 +1,180 @@ +// Schema helpers: short constructors for TypeBox schemas, the bytes leaf, and method declarations. + +import Type, { + type TArray, + type TBoolean, + type TInteger, + type TLiteral, + type TNull, + type TNumber, + type TObject, + type TOptional, + type TProperties, + type TRecord, + type TSchema, + type TString, + type TUnion, + type TUnknown, + type TUnsafe, +} from "typebox"; + +export type { Static, TSchema } from "typebox"; + +/** Bounds of a string length, a number or an array length. */ +export type Bounds = { readonly min?: number; readonly max?: number }; + +/** A declared default value: a table column with one may be left out of a `put` (slice 6a §5). */ +export type Defaulted = { readonly default: T }; + +/** A string, with optional length bounds, pattern and default. */ +export function str( + options: Bounds & { readonly pattern?: string; readonly default: string }, +): TString & Defaulted; +export function str(options?: Bounds & { readonly pattern?: string }): TString; +export function str(options: Bounds & { readonly pattern?: string; readonly default?: string } = {}): TString { + const pattern = options.pattern === undefined ? {} : { pattern: options.pattern }; + return Type.String({ ...limits(options, "minLength", "maxLength"), ...pattern, ...defaultOf(options) }); +} + +/** A finite number within optional bounds, with an optional default. */ +export function num(options: Bounds & { readonly default: number }): TNumber & Defaulted; +export function num(options?: Bounds): TNumber; +export function num(options: Bounds & { readonly default?: number } = {}): TNumber { + return Type.Number({ ...limits(options, "minimum", "maximum"), ...defaultOf(options) }); +} + +/** An integer within optional bounds, with an optional default. */ +export function int(options: Bounds & { readonly default: number }): TInteger & Defaulted; +export function int(options?: Bounds): TInteger; +export function int(options: Bounds & { readonly default?: number } = {}): TInteger { + return Type.Integer({ ...limits(options, "minimum", "maximum"), ...defaultOf(options) }); +} + +/** A boolean, with an optional default. */ +export function bool(options: { readonly default: boolean }): TBoolean & Defaulted; +export function bool(options?: Record): TBoolean; +export function bool(options: { readonly default?: boolean } = {}): TBoolean { + return Type.Boolean(defaultOf(options)); +} + +/** The `default` keyword of `options`, left out when absent (snapshots refuse `undefined`). */ +function defaultOf(options: { readonly default?: string | number | boolean }): { default?: string | number | boolean } { + return options.default === undefined ? {} : { default: options.default }; +} + +/** `null`; also the omitted side of a method. */ +export function nul(): TNull { + return Type.Null(); +} + +/** Exactly the given value. */ +export function lit(value: V): TLiteral { + return Type.Literal(value); +} + +/** One of the given literals. */ +export function oneOf(...values: V): TUnion> { + // One literal schema per value, in order. + return Type.Union(values.map((value) => Type.Literal(value))) as TUnion>; +} + +type Literal = string | number | boolean; + +type Literals = V extends [infer First extends Literal, ...infer Rest extends Literal[]] + ? [TLiteral, ...Literals] + : []; + +/** Any of the given schemas. */ +export function union(schemas: [...S]): TUnion { + return Type.Union(schemas); +} + +/** A closed object: unknown fields are refused. */ +export function obj

(properties: P): TObject

{ + return Type.Object(properties, { additionalProperties: false }); +} + +/** An array, with optional length bounds. */ +export function arr(items: S, options: Bounds = {}): TArray { + return Type.Array(items, limits(options, "minItems", "maxItems")); +} + +/** A string-keyed record. */ +export function map(value: S): TRecord<"^.*$", S> { + return Type.Record(Type.String(), value); +} + +/** An optional object field. */ +export function opt(schema: S): TOptional { + return Type.Optional(schema) as TOptional; +} + +/** Any value. */ +export function unknown(): TUnknown { + return Type.Unknown(); +} + +/** + * A `Uint8Array` leaf, checked by `instanceof` and length, never enumerated. Its enumerable form is the data marker + * `{ type: "bytes", minLength?, maxLength? }`. + */ +export function bytes(options: Bounds = {}): TUnsafe { + return withBytesCheck({ type: "bytes", ...limits(options, "minLength", "maxLength") }); +} + +/** The data marker of a bytes schema. */ +export type BytesMarker = { readonly type: "bytes"; readonly minLength?: number; readonly maxLength?: number }; + +/** Attaches the TypeBox check of a bytes marker, non-enumerably so it is never copied as data. */ +export function withBytesCheck(marker: BytesMarker): TUnsafe { + const { minLength = 0, maxLength = Number.POSITIVE_INFINITY } = marker; + const refinement = { + check: (value: unknown) => value instanceof Uint8Array && value.length >= minLength && value.length <= maxLength, + error: () => "expected bytes of the declared length", + }; + Object.defineProperty(marker, "~refine", { value: [refinement] }); + // The marker object is the schema; TypeBox reads the attached check. + return marker as unknown as TUnsafe; +} + +/** TypeBox options for the given bounds; an absent bound is left out, since snapshots refuse `undefined`. */ +function limits(options: Bounds, min: string, max: string): { [keyword: string]: number } { + const result: { [keyword: string]: number } = {}; + if (options.min !== undefined) result[min] = options.min; + if (options.max !== undefined) result[max] = options.max; + return result; +} + +/** A method declaration: its argument and result schemas; an omitted side is `null` on the wire. */ +export class Method { + readonly args: A; + readonly result: R; + + constructor(args: A, result: R) { + this.args = args; + this.result = result; + } +} + +/** A keyed state member's declaration (slice 6a §12): documents of `schema` by string key; only in `local` contracts. */ +export class KeyedMember { + readonly schema: S; + + constructor(schema: S) { + this.schema = schema; + } +} + +/** Declares a keyed state member of a `local` contract: documents of `schema` by string key, each watchable. */ +export function keyed(schema: S): KeyedMember { + return new KeyedMember(schema); +} + +/** Declares a method: `method()`, `method(args)`, `method(args, result)` or `method(undefined, result)`. */ +export function method(): Method; +export function method(args: A): Method; +export function method(args: undefined, result: R): Method; +export function method(args: A, result: R): Method; +export function method(args?: TSchema, result?: TSchema): Method { + return new Method(args ?? nul(), result ?? nul()); +} diff --git a/packages/chord/src/next/contracts/snapshot.ts b/packages/chord/src/next/contracts/snapshot.ts new file mode 100644 index 000000000..a84ed79c0 --- /dev/null +++ b/packages/chord/src/next/contracts/snapshot.ts @@ -0,0 +1,206 @@ +// Schema snapshots: one schema-aware walk checks an app schema against Chord's vocabulary and produces a +// Chord-owned copy and its normal form (the fingerprint input). + +import { isJsonValue, type JsonValue } from "../../json.ts"; +import { type BytesMarker, type TSchema, withBytesCheck } from "./schema.ts"; + +/** + * A Chord-owned copy of an app schema, its normal form, and whether it has a `bytes()` leaf (values crossing a link + * then carry raw byte segments); immutable by convention. + */ +export type Snapshot = { readonly schema: TSchema; readonly normal: JsonValue; readonly bytes: boolean }; + +/** Snapshot of a state, result or params schema. Throws an Error naming `path` for anything outside the vocabulary. */ +export function snapshotSchema(source: unknown, path: string): Snapshot { + return walk(source, path, "open", new Set()); +} + +/** Snapshot of a method's arguments: every object schema inside must also be closed (records allowed). */ +export function snapshotArguments(source: unknown, path: string): Snapshot { + return walk(source, path, "closed", new Set()); +} + +/** Whether object schemas must set `additionalProperties: false`. */ +type Objects = "open" | "closed"; + +/** A copied keyword value, its normal form, and whether a schema inside it is a bytes leaf. */ +type Copy = { readonly schema: unknown; readonly normal: JsonValue; readonly bytes: boolean }; + +/** Every supported keyword, grouped by how its value is walked. */ +const KEYWORD_GROUPS = { + schema: "additionalProperties additionalItems", + schemas: "anyOf allOf prefixItems", + items: "items", + schemaMap: "properties patternProperties $defs", + type: "type", + required: "required", + enum: "enum", + data: + "const default $ref $id pattern format minLength maxLength minimum maximum exclusiveMinimum exclusiveMaximum " + + "multipleOf minItems maxItems uniqueItems minProperties maxProperties readOnly writeOnly", + annotation: "title description examples $comment deprecated", +}; + +type Keyword = keyof typeof KEYWORD_GROUPS; + +const KEYWORDS = new Map( + Object.entries(KEYWORD_GROUPS).flatMap(([keyword, names]) => + names.split(" ").map((name) => [name, keyword as Keyword]), + ), +); + +const TYPES = new Set(["null", "boolean", "number", "integer", "string", "array", "object", "bytes"]); + +/** TypeBox's executable features, attached non-enumerably. */ +const EXECUTABLE = ["~refine", "~codec", "~unsafe"]; + +const BYTES_EXECUTABLE = ["~refine", "~unsafe"]; + +const BYTES_KEYWORDS = new Set(["type", "minLength", "maxLength"]); + +function walk(source: unknown, path: string, objects: Objects, ancestors: Set): Snapshot { + if (typeof source === "boolean") return { schema: source, normal: source, bytes: false }; + if (!isPlainObject(source)) throw new Error(`${path}: expected a schema`); + if (ancestors.has(source)) throw new Error(`${path}: cyclic schema; use $defs and $ref for recursion`); + const isBytes = source.type === "bytes"; + // A bytes schema may carry its own check (rebuilt below); no other schema may carry an executable feature. + const allowed = isBytes ? BYTES_EXECUTABLE : []; + for (const key of EXECUTABLE) { + if (Object.hasOwn(source, key) && !allowed.includes(key)) { + throw new Error(`${path}: TypeBox refinements, codecs and unsafe types are not supported`); + } + } + ancestors.add(source); + const schema: [string, unknown][] = []; + const normal: [string, JsonValue][] = []; + let bytes = isBytes; + for (const [key, value] of ownEntries(source, path)) { + const keyword = KEYWORDS.get(key); + if (keyword === undefined || (isBytes && keyword !== "annotation" && !BYTES_KEYWORDS.has(key))) { + throw new Error(`${path}: unsupported keyword "${key}"`); + } + const copy = walkKeyword(keyword, value, `${path}.${key}`, objects, ancestors); + bytes ||= copy.bytes; + schema.push([key, copy.schema]); + // Annotations are dropped on schema nodes only; inside data they are ordinary keys. + if (keyword !== "annotation") normal.push([key, copy.normal]); + } + ancestors.delete(source); + const node = Object.fromEntries(schema); + if (objects === "closed" && isOpenObject(node)) { + throw new Error(`${path}: argument objects must be closed (additionalProperties: false)`); + } + if (isBytes) checkBytesBounds(node, path); + return { + schema: isBytes ? withBytesCheck(node as BytesMarker) : node, + normal: Object.fromEntries(normal), + bytes, + }; +} + +function walkKeyword(keyword: Keyword, value: unknown, path: string, objects: Objects, ancestors: Set): Copy { + switch (keyword) { + case "schema": + return walk(value, path, objects, ancestors); + case "schemas": + return walkList(value, path, objects, ancestors); + case "items": + return Array.isArray(value) + ? walkList(value, path, objects, ancestors) + : walk(value, path, objects, ancestors); + case "schemaMap": { + if (!isPlainObject(value)) throw new Error(`${path}: expected an object of schemas`); + const entries = ownEntries(value, path).map( + ([key, item]) => [key, walk(item, `${path}.${key}`, objects, ancestors)] as const, + ); + return { + schema: Object.fromEntries(entries.map(([key, item]) => [key, item.schema])), + normal: Object.fromEntries(entries.map(([key, item]) => [key, item.normal])), + bytes: entries.some(([, item]) => item.bytes), + }; + } + case "type": + if (typeof value !== "string" || !TYPES.has(value)) { + throw new Error(`${path}: unsupported type ${JSON.stringify(value)}`); + } + return { schema: value, normal: value, bytes: false }; + case "required": { + if (!Array.isArray(value) || !value.every((name) => typeof name === "string")) { + throw new Error(`${path}: expected an array of strings`); + } + const names = [...value].sort(); + return { schema: names, normal: names, bytes: false }; + } + case "enum": { + const values = data(value, path).normal; + if (!Array.isArray(values)) throw new Error(`${path}: expected an array`); + const sorted = [...values].sort((a, b) => compare(JSON.stringify(a), JSON.stringify(b))); + return { schema: sorted, normal: sorted, bytes: false }; + } + case "data": + case "annotation": + return data(value, path); + } +} + +function walkList(value: unknown, path: string, objects: Objects, ancestors: Set): Copy { + if (!Array.isArray(value)) throw new Error(`${path}: expected an array of schemas`); + const items = Array.from(value, (item, index) => walk(item, `${path}[${index}]`, objects, ancestors)); + return { + schema: items.map((item) => item.schema), + normal: items.map((item) => item.normal), + bytes: items.some((item) => item.bytes), + }; +} + +/** A JSON value as a copy with sorted keys; the same copy serves as snapshot and normal form. */ +function data(value: unknown, path: string): Copy { + if (!isJsonValue(value)) throw new Error(`${path}: expected JSON data (finite numbers, plain objects and arrays)`); + const copy = sortedCopy(value); + return { schema: copy, normal: copy, bytes: false }; +} + +function sortedCopy(value: JsonValue): JsonValue { + if (value === null || typeof value !== "object") return value; + if (Array.isArray(value)) return value.map(sortedCopy); + const keys = Object.keys(value).sort(); + return Object.fromEntries(keys.map((key) => [key, sortedCopy(value[key]!)])); +} + +/** Enumerable data entries sorted by key; symbol keys and accessors are refused. */ +function ownEntries(source: object, path: string): [string, unknown][] { + const entries: [string, unknown][] = []; + for (const key of Reflect.ownKeys(source)) { + if (typeof key === "symbol") throw new Error(`${path}: symbol keys are not supported`); + const descriptor = Object.getOwnPropertyDescriptor(source, key)!; + if (!("value" in descriptor)) throw new Error(`${path}.${key}: accessors are not supported`); + // Non-enumerable keys are TypeBox metadata (`~kind`, `~optional`), never schema data. + if (descriptor.enumerable) entries.push([key, descriptor.value]); + } + return entries.sort(([a], [b]) => compare(a, b)); +} + +function isPlainObject(value: unknown): value is Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +/** An object schema that admits unknown fields. A record (pattern properties only) is allowed. */ +function isOpenObject(node: Record): boolean { + const isRecord = "patternProperties" in node && !("properties" in node); + const isObject = node.type === "object" || "properties" in node; + return isObject && !isRecord && node.additionalProperties !== false; +} + +function checkBytesBounds(node: Record, path: string): void { + for (const bound of [node.minLength, node.maxLength]) { + if (bound !== undefined && !(Number.isSafeInteger(bound) && (bound as number) >= 0)) { + throw new Error(`${path}: bytes bounds must be non-negative integers`); + } + } +} + +function compare(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} diff --git a/packages/chord/src/next/data/README.md b/packages/chord/src/next/data/README.md new file mode 100644 index 000000000..1b8817496 --- /dev/null +++ b/packages/chord/src/next/data/README.md @@ -0,0 +1,60 @@ +# data + +Per-call data checks, principals and errors. Pure functions and weak caches; no I/O. + +## For apps (`index.ts`) + +```ts +import { broken, isChordError } from "./index.ts"; + +async function open(config: { path?: string }) { + if (config.path === undefined) throw broken(new Error("notes: no path configured")); // retrying will not help + // ... +} + +try { + await notes.add({ text: "hi" }); +} catch (error) { + if (isChordError(error, "invalid")) console.log(error.reason, error.message); // e.g. "args", "args/text: missing" +} +``` + +- `ChordError` with `code`, `reason`, `message`, `details?`; narrowing on `code` narrows `reason`. `isChordError(error, + code?)`, and the `ChordErrorCode`/`ChordErrorReason` types. +- `Principal` (plain JSON with a `kind`) and `PrincipalKeyFunction`. Apps declare principals as type aliases, not + interfaces (an interface does not fit `Principal`'s index signature). A custom principal key function must be the + same on every host; a different one silently splits identities. +- `broken(cause)`: thrown by an open to mark a bad configuration; `FailureCause` is what Chord reports. +- `new PluginError(code, message, details?)`: what a method throws to give its callers a code of its own. On one host + callers get the thrown value itself; over a link only `code`, `message` and JSON `details` cross and the consumer + rethrows a PluginError. Any other thrown value crosses as a PluginError with the code `internal` and its message. + +## For Chord's layers (`internal.ts`) + +- `checkParams(contract, raw, context)`: one compiled closed-object check of a get's params and their key; + `identityKey(contract, key, principalKey)` (no principal key for `local`); `checkAddress(kind, params, raw, + context)`: the same check of an address's params against its host kind's (`invalid/placement`), and its key. +- `bindProvider(contract, accepts)` once per registration, then `checkArgs(provider, method, raw, boundVersion, + context)` per call. It returns `{ kind: "method", args, result? }` (current-shape arguments, and the result mapping + back to the caller's version) or `{ kind: "call", args, call }` (old arguments for the provider's replacement). + `Value` is JSON plus bytes. +- `validatorFor(owner, context)`: the compiled validator of a Chord-owned schema, cached weakly by its owner; + `check(validator, value, reason, context)` runs it (counted) and throws `invalid/` naming the path (a job's + args, `jobs/`). +- `principalKey(principal, context, customKey?)`, `canonicalJson(value, context)`. +- `invalid`, `asChordError`, `chordErrorFromJson`, `PROTOCOL_VERSION`, `CHORD_ERROR_REASONS`; + `openFailure(thrown, label)`. + +Rules: + +- Params and arguments are the caller's own objects: never copied, frozen or defaulted. A provider must not mutate + them. Results and state are never checked. Nothing in L1 is frozen. +- A method without arguments (argument schema `null`) omits the argument in references and implementations; `null` + is its internal and wire form, which `checkArgs` checks. Unknown fields fail. +- A failed check is `invalid/args`, `invalid/params` or `invalid/placement` with `details.path`, a JSON Pointer from TypeBox's first error. + TypeBox does not escape `~` and `/`, so keys containing them make the path ambiguous. +- `checkArgs` refuses an unknown method with `invalid/member` and an unaccepted version or a refused method with + `contract_mismatch`. It never invokes a `call` replacement. +- Error codes, reasons and their pairings are link protocol: changing them bumps `PROTOCOL_VERSION`. +- Every function that works takes a required `context` (after the required arguments, before optional ones). Work is + counted in `chord.data.walks`, `chord.data.validations` and `chord.data.compilations` through its telemetry. diff --git a/packages/chord/src/next/data/args.ts b/packages/chord/src/next/data/args.ts new file mode 100644 index 000000000..88cb6f69f --- /dev/null +++ b/packages/chord/src/next/data/args.ts @@ -0,0 +1,142 @@ +// Arguments: a provider's binding (its dispatch table per accepted version) and the per-call argument check. + +import type { Validator } from "typebox/compile"; +import type { Context } from "../../context/index.ts"; +import { type AcceptedVersion, type AcceptsTable, checkAccepts } from "../contracts/accepts.ts"; +import { + type Contract, + type CurrentVariants, + contractData, + type MethodData, + type VersionData, +} from "../contracts/contract.ts"; +import type { MemberNames, MethodAt } from "../contracts/history.ts"; +import type { Static } from "../contracts/schema.ts"; +import type { Snapshot } from "../contracts/snapshot.ts"; +import { ChordError, invalid } from "./errors.ts"; +import { check, validatorFor } from "./validator.ts"; + +/** JSON plus byte leaves. Same-host values are plain data and immutable once handed over, by convention. */ +export type Value = null | boolean | number | string | Uint8Array | Value[] | { [key: string]: Value }; + +/** A provider's binding of a contract with the older versions it accepts. */ +export type Provider = { readonly contract: C; readonly accepted: readonly AcceptedVersion[] }; + +/** + * Checked arguments: a method's arguments in the current shape with the result mapping back to the caller's version, + * or the old arguments for the provider's own `call` replacement (L4 dispatches it). + */ +export type CheckedArgs = + | { readonly kind: "method"; readonly args: ArgsOf; readonly result?: (current: Value) => Value } + | { readonly kind: "call"; readonly args: Value; readonly call: (args: Value, context: Context) => Promise }; + +type ArgsOf = M extends MemberNames, "methods"> + ? Static, M>["args"]> + : Value; + +/** + * Binds a contract and the app's accepts table (validated once, kept as it is) into a provider. Builds the dispatch + * table: accepted version -> method -> schema and mapping; nothing is compiled yet. + */ +export function bindProvider(contract: C, accepts: AcceptsTable): Provider { + const { versions: accepted } = checkAccepts(contract, accepts); + const { versions } = contractData(contract); + const current = versions.at(-1)!; + // checkAccepts established this shape. + const mappings = accepts as CheckedAccepts; + const table: Dispatch = new Map([[current.version, dispatchEntries(current, undefined)]]); + for (const { version } of accepted) { + const data = versions.find((candidate) => candidate.version === version)!; + table.set(version, dispatchEntries(data, mappings[version]?.methods)); + } + const provider = { contract, accepted }; + dispatchTables.set(provider, table); + return provider; +} + +/** An accepts table after `checkAccepts`, as the binding reads it. */ +type CheckedAccepts = { + readonly [version: string]: { readonly methods?: { readonly [member: string]: CheckedMethodMapping } }; +}; + +type CheckedMethodMapping = + | "refuse" + | { readonly call: (args: Value, context: Context) => Promise } + | { readonly args?: (old: Value) => Value; readonly result?: (current: Value) => Value }; + +/** Accepted version -> method -> entry. */ +type Dispatch = Map>; + +type DispatchEntry = + | { readonly kind: "refuse" } + | { + readonly kind: "call"; + readonly args: Snapshot; + readonly call: (args: Value, context: Context) => Promise; + validator: Validator | undefined; + } + | { + readonly kind: "method"; + readonly args: Snapshot; + readonly mapArgs: ((old: Value) => Value) | undefined; + readonly mapResult: ((current: Value) => Value) | undefined; + validator: Validator | undefined; + }; + +const dispatchTables = new WeakMap, Dispatch>(); + +function dispatchEntries( + version: VersionData, + mappings: { readonly [member: string]: CheckedMethodMapping } | undefined, +): Map { + const entries = new Map(); + // A method has one signature in every variant that has it. + const methods = new Map([...version.variants.values()].flatMap(({ methods }) => [...methods])); + for (const [member, { args }] of methods) { + const mapping = mappings !== undefined && Object.hasOwn(mappings, member) ? mappings[member] : undefined; + entries.set(member, dispatchEntry(args, mapping)); + } + return entries; +} + +function dispatchEntry(args: Snapshot, mapping: CheckedMethodMapping | undefined): DispatchEntry { + if (mapping === "refuse") return { kind: "refuse" }; + if (mapping !== undefined && "call" in mapping) { + return { kind: "call", args, call: mapping.call, validator: undefined }; + } + return { kind: "method", args, mapArgs: mapping?.args, mapResult: mapping?.result, validator: undefined }; +} + +/** + * Checks a call's arguments against the provider's schema for the caller's bound version: three map lookups, one + * compiled check, at most one direct mapping old -> current (never for current-version calls). A method without + * arguments gets `null` (its internal and wire form). Unknown method: `invalid/member`; version not accepted or refused: `contract_mismatch`. + */ +export function checkArgs( + provider: Provider, + method: M, + raw: unknown, + boundVersion: number, + context: Context, +): CheckedArgs { + const name = provider.contract.name; + const methods = dispatchTables.get(provider)!.get(boundVersion); + if (methods === undefined) { + throw new ChordError({ code: "contract_mismatch", message: `${name} v${boundVersion} is not accepted` }); + } + const entry = methods.get(method); + if (entry === undefined) throw invalid("member", `${name} v${boundVersion} has no method "${method}"`); + if (entry.kind === "refuse") { + throw new ChordError({ code: "contract_mismatch", message: `${name} v${boundVersion} refuses "${method}"` }); + } + entry.validator ??= validatorFor(entry.args, context); + check(entry.validator, raw, "args", context); + // The check above establishes the argument type; mapping output is trusted. + const args = raw as Value; + if (entry.kind === "call") return { kind: "call", args, call: entry.call }; + return { + kind: "method", + args: (entry.mapArgs === undefined ? args : entry.mapArgs(args)) as ArgsOf, + result: entry.mapResult, + }; +} diff --git a/packages/chord/src/next/data/errors.ts b/packages/chord/src/next/data/errors.ts new file mode 100644 index 000000000..697f3b578 --- /dev/null +++ b/packages/chord/src/next/data/errors.ts @@ -0,0 +1,142 @@ +// ChordError: Chord's one error type, its codes and reasons (link protocol) and its wire form; and PluginError, what a +// plugin throws to give its callers a code of its own. + +import type { JsonValue } from "../../json.ts"; + +/** Version of the error table below on links; adding or moving a code or reason bumps it. */ +export const PROTOCOL_VERSION = 1; + +/** Every code with the reasons it may carry. `invalid` requires a reason; every other code may omit it. */ +export const CHORD_ERROR_REASONS = { + invalid: ["args", "params", "member", "placement", "reentrant", "read_after_bulk", "cycle"], + contract_mismatch: ["updating"], + forbidden: ["revoked"], + gone: [], + not_ready: [], + // No reason: released by its holder. `ended`: its object ended; `closed`: its host, store or native closed; + // `settled`: used after the call or run it belonged to settled; `version`: an old handle after a migration; + // `discarded`: a transaction cut at the seal. + released: ["ended", "closed", "settled", "version", "discarded"], + too_large: ["part"], + unavailable: ["locked", "park_timeout", "busy", "storage_failed"], + aborted: [], +} as const; + +export type ChordErrorCode = keyof typeof CHORD_ERROR_REASONS; + +export type ChordErrorReason = (typeof CHORD_ERROR_REASONS)[C][number]; + +/** The wire form of a ChordError and its constructor argument: one member per code, so wrong pairings fail to type. */ +type ChordErrorJson = { [C in ChordErrorCode]: ErrorFields }[ChordErrorCode]; + +type ErrorFields = { + readonly code: C; + readonly message: string; + readonly details?: JsonValue; +} & (C extends "invalid" + ? { readonly reason: ChordErrorReason } + : [ChordErrorReason] extends [never] + ? { readonly reason?: undefined } + : { readonly reason?: ChordErrorReason }); + +class ChordErrorClass extends Error { + readonly code: ChordErrorCode; + readonly reason: ChordErrorReason | undefined; + readonly details: JsonValue | undefined; + + constructor(init: ChordErrorJson, options?: ErrorOptions) { + super(init.message, options); + this.name = "ChordError"; + this.code = init.code; + this.reason = init.reason; + this.details = init.details; + } + + /** The wire form; stack and cause never cross a link. */ + toJSON(): ChordErrorJson { + const { code, reason, message, details } = this; + // The fields were copied from a ChordErrorJson, so the pairing holds. + return { + code, + ...(reason === undefined ? {} : { reason }), + message, + ...(details === undefined ? {} : { details }), + } as ChordErrorJson; + } +} + +/** A Chord failure. Narrowing on `code` narrows `reason`. */ +export type ChordError = ChordErrorClass & ChordErrorJson; + +/** + * Creates a ChordError: `new ChordError({ code: "invalid", reason: "args", message })`. A class cannot declare its + * instances as a tagged union; this constructor signature does. + */ +export const ChordError = ChordErrorClass as new (init: ChordErrorJson, options?: ErrorOptions) => ChordError; + +/** Whether `error` is a ChordError, optionally with the given code. */ +export function isChordError( + error: unknown, + code?: C, +): error is Extract { + return error instanceof ChordError && (code === undefined || error.code === code); +} + +/** + * An error a plugin throws for its callers: its own `code`, a message and JSON `details`. Only these cross a link + * (never a stack or a cause); a consumer rethrows it as a PluginError. + */ +export class PluginError extends Error { + readonly code: string; + readonly details: JsonValue | undefined; + + constructor(code: string, message: string, details?: JsonValue) { + super(message); + this.name = "PluginError"; + this.code = code; + this.details = details; + } +} + +/** An `invalid` error with the given reason. */ +export function invalid( + reason: ChordErrorReason<"invalid">, + message: string, + details?: JsonValue, +): Extract { + // The constructor's return type cannot narrow on its argument. + return new ChordError({ code: "invalid", reason, message, details }) as Extract; +} + +/** `error` itself when it is a ChordError, otherwise an `unavailable` error naming `label`, with `error` as cause. */ +export function asChordError(error: unknown, label: string): ChordError { + if (isChordError(error)) return error; + const message = error instanceof Error ? error.message : String(error); + return new ChordError({ code: "unavailable", message: `${label}: ${message}` }, { cause: error }); +} + +/** + * Rebuilds a ChordError from a link. Checks the envelope only: a known code, a reason allowed for it (required for + * `invalid`), a string message, no other keys. `details` is kept by reference. + */ +export function chordErrorFromJson(raw: unknown): ChordError { + // Trust boundary: `raw` is fresh JSON from the link decoder. + if (typeof raw !== "object" || raw === null || Array.isArray(raw)) throw envelopeError("not an object"); + const json = raw as Record; + for (const key of Object.keys(json)) if (!ENVELOPE_KEYS.has(key)) throw envelopeError(`unknown key "${key}"`); + const { code, reason, message } = json; + if (typeof code !== "string" || !Object.hasOwn(CHORD_ERROR_REASONS, code)) throw envelopeError("unknown code"); + const reasons: readonly string[] = CHORD_ERROR_REASONS[code as ChordErrorCode]; + if (reason === undefined ? code === "invalid" : !reasons.includes(reason as string)) { + throw envelopeError(`reason not allowed for ${code}`); + } + if (typeof message !== "string") throw envelopeError("message is not a string"); + // The checks above establish the pairing. + return new ChordError(json as ChordErrorJson); +} + +const ENVELOPE_KEYS = new Set(["code", "reason", "message", "details"]); + +function envelopeError(problem: string): Error { + return new Error(`Invalid ChordError JSON: ${problem}`); +} diff --git a/packages/chord/src/next/data/failure.ts b/packages/chord/src/next/data/failure.ts new file mode 100644 index 000000000..a76190118 --- /dev/null +++ b/packages/chord/src/next/data/failure.ts @@ -0,0 +1,26 @@ +// Failure causes of an open: transient (retry later) or broken (bad configuration). The failing stage decides. + +import { asChordError, type ChordError } from "./errors.ts"; + +/** Why an open failed; the kind comes from the failing stage, never from the error code. */ +export type FailureCause = + | { readonly kind: "transient"; readonly error: ChordError } + | { readonly kind: "broken"; readonly error: ChordError }; + +/** Marks an open's failure as broken (bad configuration): `throw broken(error)`. */ +export function broken(cause: unknown): Error { + return new BrokenOpen(cause); +} + +/** The cause of a value thrown by an open: broken when it came from `broken()`, transient otherwise. */ +export function openFailure(thrown: unknown, label: string): FailureCause { + if (thrown instanceof BrokenOpen) return { kind: "broken", error: asChordError(thrown.cause, label) }; + return { kind: "transient", error: asChordError(thrown, label) }; +} + +class BrokenOpen extends Error { + constructor(cause: unknown) { + super(cause instanceof Error ? cause.message : String(cause), { cause }); + this.name = "BrokenOpen"; + } +} diff --git a/packages/chord/src/next/data/index.ts b/packages/chord/src/next/data/index.ts new file mode 100644 index 000000000..3eb2015e8 --- /dev/null +++ b/packages/chord/src/next/data/index.ts @@ -0,0 +1,11 @@ +// Data for apps: Chord's errors and plugin errors, principals, and how a plugin's open reports a broken configuration. + +export { + ChordError, + type ChordErrorCode, + type ChordErrorReason, + isChordError, + PluginError, +} from "./errors.ts"; +export { broken, type FailureCause } from "./failure.ts"; +export type { Principal, PrincipalKeyFunction } from "./principal.ts"; diff --git a/packages/chord/src/next/data/internal.ts b/packages/chord/src/next/data/internal.ts new file mode 100644 index 000000000..d59881ebe --- /dev/null +++ b/packages/chord/src/next/data/internal.ts @@ -0,0 +1,14 @@ +// Data for Chord's layers: per-call checks of params and arguments, principal keys, and errors on the wire. + +export { bindProvider, type CheckedArgs, checkArgs, type Provider, type Value } from "./args.ts"; +export { + asChordError, + CHORD_ERROR_REASONS, + chordErrorFromJson, + invalid, + PROTOCOL_VERSION, +} from "./errors.ts"; +export { openFailure } from "./failure.ts"; +export { checkAddress, checkParams, identityKey } from "./params.ts"; +export { canonicalJson, principalKey } from "./principal.ts"; +export { check, validatorFor } from "./validator.ts"; diff --git a/packages/chord/src/next/data/params.ts b/packages/chord/src/next/data/params.ts new file mode 100644 index 000000000..17ce090e1 --- /dev/null +++ b/packages/chord/src/next/data/params.ts @@ -0,0 +1,43 @@ +// Params and identity: the checked params of a get, their key, and the in-process identity of an object. + +import type { Context } from "../../context/index.ts"; +import { type Contract, contractData, type ParamsOf, type ShapeOf } from "../contracts/contract.ts"; +import { type Address, addressKey } from "../contracts/host-kind.ts"; +import { joinKey, type ParamsData } from "../contracts/params.ts"; +import { check, validatorFor } from "./validator.ts"; + +/** + * Checks `raw` against the contract's params with one compiled closed-object check (`invalid/params` on failure) and + * returns the caller's own object with its key: the field values in declared order, each escaped, joined with `|`. + */ +export function checkParams( + contract: C, + raw: unknown, + context: Context, +): { readonly params: ParamsOf; readonly key: string } { + const { params } = contractData(contract); + check(validatorFor(params, context), raw, "params", context); + // The check above establishes the params type. + const checked = raw as ParamsOf & { readonly [field: string]: string | number | boolean }; + return { params: checked, key: joinKey(params.fields.map((field) => String(checked[field]))) }; +} + +/** The in-process identity of an object: `name|key|principalKey`, each part escaped; `local` has no principal. */ +export function identityKey( + contract: C, + key: string, + ...principalKey: ShapeOf["kind"] extends "local" ? [] : [principalKey: string] +): string { + return joinKey([contract.name, key, ...principalKey]); +} + +/** + * An address from a kind's name and params as a hold names it or a placement returned it: the params checked against + * the kind's with one compiled closed-object check (`invalid/placement` on failure), the key computed. + */ +export function checkAddress(kind: string, params: ParamsData, raw: unknown, context: Context): Address { + check(validatorFor(params, context), raw, "placement", context); + // The check above establishes a flat record of the kind's scalar fields. + const values = raw as { readonly [field: string]: string | number | boolean }; + return { kind, params: values, key: addressKey(kind, params, values) }; +} diff --git a/packages/chord/src/next/data/principal.ts b/packages/chord/src/next/data/principal.ts new file mode 100644 index 000000000..b9de9e5dc --- /dev/null +++ b/packages/chord/src/next/data/principal.ts @@ -0,0 +1,57 @@ +// Principals: the app's caller identities, and the keys Chord derives from them. + +import type { Context } from "../../context/index.ts"; +import type { JsonValue } from "../../json.ts"; + +/** An app's principal: plain JSON with a `kind`. Chord never persists it. */ +export type Principal = { readonly kind: string; readonly [field: string]: JsonValue }; + +/** An app's own principal key; it must be the same function on every host. */ +export type PrincipalKeyFunction

= (principal: P) => string; + +/** + * The key of a principal: its canonical JSON, or `customKey(principal)`. Cached per principal object; a custom key's + * cache is keyed by the function too, so a live principal never retains a dropped function. + */ +export function principalKey

( + principal: P, + context: Context, + customKey?: PrincipalKeyFunction

, +): string { + const cache = customKey === undefined ? defaultKeys : customKeyCache(customKey); + let key = cache.get(principal); + if (key === undefined) { + key = customKey === undefined ? canonicalJson(principal, context) : customKey(principal); + cache.set(principal, key); + } + return key; +} + +const defaultKeys = new WeakMap(); + +const customKeys = new WeakMap, WeakMap>(); + +function customKeyCache(customKey: PrincipalKeyFunction): WeakMap { + let cache = customKeys.get(customKey); + if (cache === undefined) { + cache = new WeakMap(); + customKeys.set(customKey, cache); + } + return cache; +} + +/** Canonical JSON: sorted object keys, `-0` written as `0`. Counts one walk. */ +export function canonicalJson(value: JsonValue, context: Context): string { + context.telemetry.metrics?.counter("chord.data.walks").add(1); + return canonical(value); +} + +function canonical(value: JsonValue): string { + // JSON.stringify writes -0 as 0. + if (value === null || typeof value !== "object") return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; + const fields = Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonical(value[key]!)}`); + return `{${fields.join(",")}}`; +} diff --git a/packages/chord/src/next/data/validator.ts b/packages/chord/src/next/data/validator.ts new file mode 100644 index 000000000..6c3369a6b --- /dev/null +++ b/packages/chord/src/next/data/validator.ts @@ -0,0 +1,72 @@ +// Compiled checks: TypeBox validators of Chord-owned schemas, and the `invalid` error of a failed check. + +import { Compile, type Validator } from "typebox/compile"; +import type { TLocalizedValidationError } from "typebox/error"; +import { ResetExternal, ResetFunctions } from "typebox/schema"; +import { Errors } from "typebox/value"; +import type { Context } from "../../context/index.ts"; +import type { TSchema } from "../contracts/schema.ts"; +import { type ChordError, invalid } from "./errors.ts"; + +/** A Chord-owned record holding a schema: a snapshot, or a contract's params. */ +type SchemaOwner = { readonly schema: TSchema }; + +/** The validator of an owner's schema, compiled on first use (counted) and cached weakly by the owner. */ +export function validatorFor(owner: SchemaOwner, context: Context): Validator { + let validator = validators.get(owner); + if (validator === undefined) { + context.telemetry.metrics?.counter("chord.data.compilations").add(1); + try { + validator = Compile(owner.schema); + } finally { + // TypeBox keeps its last build in module-level accumulators; reset them so no build state is retained. + ResetFunctions(); + ResetExternal(); + } + validators.set(owner, validator); + } + return validator; +} + +const validators = new WeakMap(); + +/** Checks `value` (counted); a failure throws `invalid/` with the first error's JSON Pointer path. */ +export function check( + validator: Validator, + value: unknown, + reason: "args" | "params" | "placement", + context: Context, +): void { + context.telemetry.metrics?.counter("chord.data.validations").add(1); + if (!validator.Check(value)) throw failedCheck(validator.Type(), value, reason, context); +} + +/** + * The error of a failed check: TypeBox's first error from its diagnostic walk (counted; it builds the whole list). + * TypeBox does not escape `~` and `/` in its instance paths, so keys containing them make the path ambiguous. + */ +function failedCheck( + schema: TSchema, + value: unknown, + reason: "args" | "params" | "placement", + context: Context, +): ChordError { + context.telemetry.metrics?.counter("chord.data.walks").add(1); + const [error] = Errors(schema, value); + const { path, problem } = describe(error); + return invalid(reason, `${reason}${path === "/" ? "" : path}: ${problem}`, { path }); +} + +/** + * The path and problem of a TypeBox error: TypeBox reports a missing field at its parent, and an unknown one as a + * false schema (`additionalProperties: false`) at its own path. Anything else keeps TypeBox's message. + */ +function describe(error: TLocalizedValidationError): { path: string; problem: string } { + if (error.keyword === "required") { + return { path: `${error.instancePath}/${error.params.requiredProperties[0]}`, problem: "missing" }; + } + if (error.keyword === "boolean" && error.schemaPath.endsWith("/additionalProperties")) { + return { path: error.instancePath, problem: "unknown field" }; + } + return { path: error.instancePath || "/", problem: error.message }; +} diff --git a/packages/chord/src/next/documents/README.md b/packages/chord/src/next/documents/README.md new file mode 100644 index 000000000..0c2748771 --- /dev/null +++ b/packages/chord/src/next/documents/README.md @@ -0,0 +1,91 @@ +# documents + +Documents: definitions, the handles an object's attempts get, the object's documents (in memory, or persisted in the +host's store), keyed watches and `at(key)`, retirement, and transactions over documents, tables and job runs. + +## For apps (`index.ts`) + +```ts +import { defineState } from "./index.ts"; + +const Notes = defineState("notes", { schema: obj({ list: arr(Note) }), version: 1, initial: () => ({ list: [] }) }); +const Drafts = defineState("drafts", { schema: obj({ body: str() }), version: 1, initial: () => ({ body: "" }), keyed: true }); +const Prefs = defineState("prefs", { + schema: obj({ theme: str() }), version: 2, initial: () => ({ theme: "light" }), + migrate: (old, from) => ({ theme: "light" }), // a stored or in-memory value of an older version + persist: true, // in the host's store (a local type with empty params) + checkpointWhen: (value, ops, { deltasSinceBase }) => deltasSinceBase >= 50, // store a new base, not a delta +}); + +const notes = self.state(Notes); // a singleton: a Watchable, `initial()` until written +const drafts = self.state(Drafts); // keyed: get(key, ctx), keys(ctx), watch(key, ctx), at(key) in the factory +const pinned = drafts.at("c1"); // a Watchable of one key (a state member, a derive or `on` source) +await self.tx(async (tx) => { + (await tx.draft(notes)).list.push({ text }); + (await tx.draft(drafts, "c1")).body = text; + tx.put(edits, { id, path }); // tables (`store/`) in the same commit +}, ctx); +await drafts.change("c1", (draft) => { draft.body = ""; }, ctx); // one-document shorthand +await notes.retire(ctx); // back to initial(), watches get `retired: true` +``` + +- `defineState(name, { schema, version, initial, keyed?, migrate?, persist?, checkpointWhen? })`: the root is a JSON + object. `initial()` is checked against the schema once, when the declaring definition is first used (retaken only + when the declaring version or fingerprint changes). `persist: true` requires `checkpointWhen` (types and + declaration) and a `local` type with empty params with a store on its host (P3). +- Singleton handle: `value`, `watch(ctx)`, `change(fn, ctx)`, `retire(ctx)`. A retirement reaches watches as + `initial()` with ops `[["r", initial]]` and `retired: true`; a retirement and a recreation in one transaction are one + root replacement, flagged. +- Keyed handle: `get(key, ctx)` (`undefined` when absent), `keys(ctx)` (code point order), `watch(key, ctx)` + (`undefined` when absent; it follows the address through retirements and recreations), `at(key)` (in the factory + only: `initial()` while absent, loaded before the object goes live, pinned until the next attempt's factory + returned), `change(key, fn, ctx)`, `retire(key, ctx)`. A persisted key is loaded on its first read on the object's + line (one load per key, shared by its readers), cached while read, pinned or watched, and evicted after ten + minutes idle; an absent key is not cached. +- `Tx`: `await tx.draft(handle, key?)` (the same draft for the same document within one transaction; a cold persisted + key loads first), `tx.retire(handle, key?)` (a no-op when absent), `tx.start(job, args, { key? })` (`jobs/`), and + the table writes and reads of `store/` (`put`, `update`, `delete`, `updateWhere`, `deleteWhere`, `get`, `page`, + `rows`). Every `tx.*` that returns a promise must be awaited before the callback returns. + +Rules: + +- Documents belong to their object, not to an attempt: a retry or a native restart keeps them, a new attempt's + `self.state` gets handles to the same documents (an old handle below the slot's version is refused + `released/version`). Persisted documents load once per object. An ending attempt's handles are sealed: they read + synchronously, anything taking the line is refused `released` (a cold read queued before the seal too); its open + transaction is discarded (drafts aborted, its callback's result rejected `released/discarded`). +- An object's transactions run one at a time on its line. A transaction covers only its own object. An in-memory-only + transaction commits synchronously in Chord's batch; one with a persisted document or a table write enters the + store's commit step, and once admitted it is adopted whatever happens to its attempt. +- When the callback resolves, an aborted `ctx` or an ending attempt discards the transaction; a throw discards it too. + Unchanged drafts commit nothing. Drafts are revoked once the transaction settles: return plain values, read data + from the handle after the commit. +- Inside the callback: only `await tx.*`; no second `self.tx`, no calls to other objects (not detected: it hangs). +- The shorthands `handle.change` and `handle.retire` run on the object's line from anyone holding the handle. That + only the object writes them is a convention, like the rule for `tx.draft`: keep handles inside the owner. +- A stored document of a newer version, a `migrate` that throws or returns a value outside the schema, or the same + version with another fingerprint fails the object `broken` (at activation, or while live for a keyed load); the data + stays. + +## For Chord's layers (`internal.ts`) + +- `Documents`: one object's documents and line: slots by name, `hydrate(declared, pins, ctx)` after each factory + (drops undeclared in-memory slots, takes the pins), `load(ctx)` (the persisted singletons and pinned keys, a line + job; `undefined` when nothing is persisted), `tx(fn, token, ctx)`, `fence(token)` (cuts the attempt's transaction at + its seal), `seal()` and `end()` at the object's end. +- `SealToken`: one per attempt; sealed when the attempt ends. Transactions and handles check it. +- `Singleton`, `Keyed`: the handles (`slot`, `token`, `version`); `KeyDeclarer` is what `at(key)` pins through. +- `KeyedSlot`, `KeySource`: a keyed slot's residents, per-key sources (the targets of `at(key)` and keyed members, + with their pins) and shared cold loads. +- `DocumentSlot`, `AnyDefinition`: a slot as transactions draft and adopt it. +- `Transaction`, `Staged`: Chord's own changes staged in a transaction (a job's run records), adopted with the + documents in the same commit, then told how the transaction ended once the line is free. + +Inside the directory: `definition.ts` (`defineState`, versions, fingerprints, `migrate`), `slot.ts` (singleton slots +and the slot contract), `keyed-slot.ts`, `address-source.ts` (the frames of one document address: changes, root +replacements, retirements), `persistence.ts` (a slot's stored side: load, find, keys, the writes of a commit), +`residency.ts` (keyed residents and their eviction: Durable's idle structure, one timer per object), `handles.ts`, +`fence.ts`, `transaction.ts`, `line.ts` (one holder at a time, in order) and `documents.ts`. + +Tests: `test/next/documents/` (one file per source). Examples: `02-documents.ts`, `21-persisted-documents.ts`, +`23-keyed-watches.ts`. diff --git a/packages/chord/src/next/documents/address-source.ts b/packages/chord/src/next/documents/address-source.ts new file mode 100644 index 000000000..dc0a2d416 --- /dev/null +++ b/packages/chord/src/next/documents/address-source.ts @@ -0,0 +1,56 @@ +// Watchable document addresses (slice 6a §11; decision "One retirement mechanism"): the frames of a source that +// follows one address, a singleton's slot or one key of a keyed slot. A change arrives with its ops, a write over +// another base as a root replacement, a retirement as `[["r", initial()]]` with the `retired` flag, and a document +// created after a retirement in the same batch as one root replacement with the flag. + +import type { Context } from "../../context/index.ts"; +import type { Op, Prepared } from "../../delta/index.ts"; +import { Source } from "../state/internal.ts"; +import type { JsonObject } from "./definition.ts"; +import type { Documents } from "./documents.ts"; + +const NO_OPS: readonly Op[] = []; + +/** A source following one document address; `value` is the document's, or `initial()` while it is absent. */ +export abstract class AddressSource extends Source { + /** The documents of the object whose document this is (a member source asks whose source it follows). */ + abstract readonly owner: Documents; + /** The ops of the current batch's frame, and whether the document was retired in it. */ + #ops = NO_OPS; + #retired = false; + + abstract override get value(): JsonObject; + + deliver(context: Context): void { + this.publish(this.value, this.#ops, this.#retired, context); + this.#ops = NO_OPS; + this.#retired = false; + } + + /** + * A commit wrote the document, which read `before`: its ops, or a root replacement when the draft started from + * another base or the document was retired earlier in the batch. + */ + written(before: JsonObject, prepared: Prepared): void { + // The ops are kept for a frame, which only a watched source gets: an unwatched one keeps none. + if (this.watched) this.#ops = prepared.base === before && !this.#retired ? prepared.ops : [["r", prepared.value]]; + this.changed(); + } + + /** The document was retired: it reads `initial` again, and the frame says so. */ + retired(initial: JsonObject): void { + // Reset by the frame, which only a watched source gets. + if (this.watched) { + this.#ops = [["r", initial]]; + this.#retired = true; + } + this.changed(); + this.retiredInBatch(); + } + + /** The value changed outside a commit (a migration, a load): a root replacement. */ + replaced(): void { + if (this.watched) this.#ops = [["r", this.value]]; + this.changed(); + } +} diff --git a/packages/chord/src/next/documents/definition.ts b/packages/chord/src/next/documents/definition.ts new file mode 100644 index 000000000..77906479b --- /dev/null +++ b/packages/chord/src/next/documents/definition.ts @@ -0,0 +1,157 @@ +// Document definitions: `defineState(name, ...)`, its fingerprint, the one check of a definition's `initial()` (R8), +// the checked `migrate` of an older value and, for a persisted document, its checkpoint rule (slice 6a §10). + +import type { Context } from "../../context/index.ts"; +import type { Op } from "../../delta/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { Static, TSchema } from "../contracts/index.ts"; +import { checkName, fingerprintOf } from "../contracts/internal.ts"; +import { snapshotSchema } from "../contracts/snapshot.ts"; +import { validatorFor } from "../data/internal.ts"; + +/** A document root: a JSON object. */ +export type JsonObject = { [key: string]: JsonValue }; + +/** What a persisted document's `checkpointWhen` learns of its stored revisions. */ +export type CheckpointInfo = { + /** The deltas stored after its newest base, without the change being decided. */ + readonly deltasSinceBase: number; +}; + +/** Whether a commit's change of a persisted document is stored as a whole new base (`true`) or as a delta. */ +export type CheckpointWhen = (value: T, ops: readonly Op[], info: CheckpointInfo) => boolean; + +/** What `defineState` takes for any document. */ +type StateOptions = { + readonly schema: S; + readonly version: number; + readonly initial: () => DocumentOf; + /** Maps a value of an older version `from` to this version's shape; pure. Required to read older values. */ + readonly migrate?: (old: JsonObject, from: number) => DocumentOf; + readonly keyed?: K; +}; + +/** What a persisted document adds (slice 6a §10): `checkpointWhen` is required. */ +type PersistOptions = { + /** Stored in the host's store; only in `local` types with empty params (P3). */ + readonly persist: true; + /** Called once per commit per changed document; creations and version changes always store a base. */ + readonly checkpointWhen: CheckpointWhen>; +}; + +/** + * Declares a document: a singleton per object, or with `keyed: true` many per object, each by a string key; with + * `persist: true` stored in the host's store. Its name is a restricted name (lowercase `[a-z0-9@/._~-]`). + */ +export function defineState( + name: string, + options: StateOptions & { readonly persist?: false; readonly checkpointWhen?: never }, +): StateDefinition, K, false>; +export function defineState( + name: string, + options: StateOptions & PersistOptions, +): StateDefinition, K, true>; +export function defineState( + name: string, + options: StateOptions & { + readonly persist?: boolean; + readonly checkpointWhen?: CheckpointWhen>; + }, +): StateDefinition, K, boolean> { + checkName(name, "A state name"); + const { schema, version, initial, migrate, keyed, persist, checkpointWhen } = options; + return new StateDefinition( + name, + schema, + version, + initial, + migrate, + keyed ?? false, + persist ?? false, + checkpointWhen, + ); +} + +/** The value of a document whose schema is `S`: its static type, which must be a JSON object. */ +type DocumentOf = Static extends JsonObject ? Static : never; + +const NEVER: CheckpointWhen = () => false; + +/** A declared document; `T` is its value, `K` whether it is keyed, `P` whether it is persisted. */ +export class StateDefinition { + readonly name: string; + readonly schema: TSchema; + readonly version: number; + readonly keyed: boolean; + readonly persist: boolean; + /** + * `hash64` of the schema's normal form (descriptions and titles ignored), computed once: the same version with + * another fingerprint is a schema changed without a version bump. + */ + readonly fingerprint: string; + /** Type only: whether the definition is keyed. */ + declare readonly "~keyed"?: K; + /** Type only: whether the definition is persisted. */ + declare readonly "~persist"?: P; + readonly #initial: () => T; + readonly #migrate: ((old: JsonObject, from: number) => T) | undefined; + /** Method-typed so that a definition of a narrower `T` still is a `StateDefinition`. */ + readonly #checkpointWhen: { rule(value: T, ops: readonly Op[], info: CheckpointInfo): boolean }["rule"]; + #checked = false; + + constructor( + name: string, + schema: TSchema, + version: number, + initial: () => T, + migrate: ((old: JsonObject, from: number) => T) | undefined, + keyed: boolean, + persist: boolean, + checkpointWhen: CheckpointWhen | undefined, + ) { + this.name = name; + this.schema = schema; + this.version = version; + this.#initial = initial; + this.#migrate = migrate; + this.keyed = keyed; + this.persist = persist; + this.#checkpointWhen = checkpointWhen ?? NEVER; + this.fingerprint = fingerprintOf(snapshotSchema(schema, name).normal); + } + + /** Whether a persisted document's change is stored as a new base (its `checkpointWhen`); never asked in memory. */ + checkpointWhen(value: T, ops: readonly Op[], info: CheckpointInfo): boolean { + return this.#checkpointWhen(value, ops, info); + } + + /** A fresh `initial()` value; the first is checked against the schema (counted), later ones are not (R8). */ + initial(context: Context): T { + const value = this.#initial(); + if (this.#checked) return value; + this.#check(value, "initial()", context); + this.#checked = true; + return value; + } + + /** + * `old`, a value of version `from`, migrated to this version and checked against the schema once, as it is + * produced (R8, counted). Throws without a `migrate` (the access is refused, the data kept) and when the result does + * not match. + */ + migrate(old: JsonObject, from: number, context: Context): T { + if (this.#migrate === undefined) { + throw new Error(`${this.name}: no migrate from version ${from} to ${this.version}`); + } + const value = this.#migrate(old, from); + this.#check(value, `migrate() from version ${from}`, context); + return value; + } + + #check(value: T, what: string, context: Context): void { + context.telemetry.metrics?.counter("chord.data.validations").add(1); + if (!validatorFor(this, context).Check(value)) { + throw new Error(`${this.name}: ${what} does not match the schema`); + } + } +} diff --git a/packages/chord/src/next/documents/documents.ts b/packages/chord/src/next/documents/documents.ts new file mode 100644 index 000000000..6ca0f5e82 --- /dev/null +++ b/packages/chord/src/next/documents/documents.ts @@ -0,0 +1,237 @@ +// An object's documents (§9, P2, slice 6a §1.2, §10): its slots across attempts, its line (its transactions and line +// jobs run one at a time), the seal fence over the transaction holding the line, the factory's pins between attempts, +// the activation's loads and the residency of keyed documents. + +import type { Context } from "../../context/index.ts"; +import { ChordError } from "../data/index.ts"; +import type { Batch } from "../state/internal.ts"; +import type { ChordStore } from "../store/store.ts"; +import type { JsonObject, StateDefinition } from "./definition.ts"; +import { SealToken } from "./fence.ts"; +import { type HandleOf, type KeyDeclarer, Keyed, Singleton } from "./handles.ts"; +import { KeyedSlot } from "./keyed-slot.ts"; +import { Line } from "./line.ts"; +import { Persistence } from "./persistence.ts"; +import { Residency } from "./residency.ts"; +import { type AnyDefinition, SingletonSlot } from "./slot.ts"; +import { Transaction } from "./transaction.ts"; + +/** Who owns an object's documents: the object's name, batch and context, its store address, its live failure. */ +export type DocumentsOwner = { + /** The owner's name in errors. */ + readonly label: string; + readonly batch: Batch; + /** The object's context: shared cold loads run under it (rule 11). */ + readonly context: Context; + /** The host's store and the object's owner string in it (`:`, W7), when it has one. */ + readonly store: { readonly store: ChordStore; readonly owner: string } | undefined; + /** A document the code cannot read fails the live object `broken` (slice 6a §1.4); ignored when it is not live. */ + fail(error: ChordError): void; +}; + +/** + * The documents of one object, from its first activation to its end: one slot per declared name, kept across + * attempts, and the line. Each attempt declares its handles under its own seal token; the object's own transactions + * (job outcomes) run under `own`, sealed only by the object's end. + */ +export class Documents { + /** The owner's name in errors. */ + readonly label: string; + readonly batch: Batch; + /** The object's context. */ + readonly context: Context; + readonly line = new Line(); + /** + * The idle persisted keyed documents and their eviction. A sweep's only rejection is `onLine`'s `released` once + * the object ended (a sweep itself drops trackers in memory and does not throw): nothing to report. + */ + readonly residency = new Residency((sweep) => void this.onLine(async () => sweep()).catch(() => {})); + readonly #owner: DocumentsOwner; + /** The token of the object's own transactions: sealed with the object (`seal`). */ + readonly own = new SealToken(); + readonly #slots = new Map(); + /** The transaction holding the line, until it settles or its token is sealed. */ + #holder: Transaction | undefined; + /** `sealed`: the object is ending; its documents stay readable, its transactions throw `released`. */ + #state: "open" | "sealed" | "ended" = "open"; + /** The pins the last factory that returned declared (`at(key)`): held until the next one returns (rule 6). */ + #pins: readonly (() => void)[] = []; + + constructor(owner: DocumentsOwner) { + this.label = owner.label; + this.batch = owner.batch; + this.context = owner.context; + this.#owner = owner; + } + + /** + * A handle to `definition`'s slot for an attempt sealed by `token`, whose `at(key)` pins `declarer` takes: the slot + * made by the first declaration of its name, checked against the values for later ones (`SlotDefinition.check`). + * "Declared twice" is the attempt's check. + */ + declare>( + definition: D, + token: SealToken, + declarer: KeyDeclarer, + ): HandleOf { + const slot = this.#slots.get(definition.name); + if (slot !== undefined) slot.definition.check(definition); + const persistence = slot === undefined && definition.persist ? this.#persistence(definition.name) : undefined; + // The definition's `keyed` decides the slot's class and the handle's; its documents are the definition's `T`. + if (definition.keyed) { + const keyed = slot ?? this.#add(new KeyedSlot(this, definition, persistence)); + return new Keyed(keyed as KeyedSlot, token, definition.version, declarer) as unknown as HandleOf; + } + const singleton = slot ?? this.#add(new SingletonSlot(this, definition, persistence)); + return new Singleton(singleton as SingletonSlot, token, definition.version) as unknown as HandleOf; + } + + /** + * Takes an attempt's declarations once its factory returned: its `at(key)` pins replace the previous factory's, the + * slots it no longer declares are dropped (their watches end; persisted ones load again when declared again), every + * declared one hydrated (`initial()`, migrations; a throw fails the activation). + */ + hydrate(declared: ReadonlyMap, pins: readonly (() => void)[], context: Context): void { + const previous = this.#pins; + this.#pins = pins; + for (const release of previous) release(); + // A dropped persisted slot goes whole, its loaded documents too: observably the same as keeping them, since + // every committed write is in the store and a later declaration reloads from there. + for (const [name, slot] of this.#slots) { + if (declared.has(name)) continue; + this.#slots.delete(name); + slot.drop(); + } + for (const [name, definition] of declared) this.#slots.get(name)!.hydrate(definition, context); + } + + /** + * Loads, in one line job, every persisted singleton not loaded yet and every pinned key not loaded since it was + * pinned (an attempt's activation, before it goes live, rule 6); `undefined` when nothing is persisted. A failure + * fails the activation. + */ + load(context: Context): Promise | undefined { + const persisted = [...this.#slots.values()].filter((slot) => slot.persisted !== undefined); + // In memory only: nothing to wait for (an activation without persisted documents takes no extra turn). + if (persisted.length === 0) return undefined; + return this.onLine(async () => { + for (const slot of persisted) await slot.loadDeclared(context); + }); + } + + /** + * Runs `job` on the line (a cold load, a page of keys, an eviction sweep); refused `released` once the object is + * ending or `token` (the handle's that asks, when the job is one handle's) is sealed, a job queued before either + * included (anything taking the line after the seal is refused, rule 4). + */ + async onLine(job: () => Promise, token?: SealToken): Promise { + if (this.#state !== "open" || token?.sealed) throw this.released(); + const release = await this.line.acquire(); + try { + if (this.#state !== "open" || token?.sealed) throw this.released(); + return await job(); + } finally { + release(); + } + } + + /** + * Runs `fn` as one transaction on the line under `token` and commits what it drafted (§9): a throw, an abort of + * `context`, the token's seal or the object's end discards everything. + */ + async tx(fn: (tx: Transaction) => R | Promise, token: SealToken, context: Context): Promise { + this.batch.assertOutside("tx"); + this.assertAdmitted(token); + const release = await this.line.acquire(); + const tx = new Transaction(this, token, release, context); + try { + this.assertAdmitted(token); + this.#holder = tx; + const result = await fn(tx); + // In-memory-only commits are synchronous (W12): no await unless the store's step is taken. + const step = tx.commit(); + if (step !== undefined) await step; + return result; + } catch (error) { + tx.discard(); + throw error; + } finally { + if (this.#holder === tx) this.#holder = undefined; + release(); + tx.settle(); + } + } + + /** + * Seals `token` (the seal fence): its transactions are refused from now on, and the one holding the line is + * discarded and the line released. + */ + fence(token: SealToken): void { + token.seal(); + if (this.#holder?.token === token) this.#cut(); + } + + /** Throws `released` once the object ended. */ + assertLive(): void { + if (this.#state === "ended") throw this.released(); + } + + /** Throws `released` once `token` is sealed or the object is ending: no new work takes the line. */ + assertAdmitted(token: SealToken): void { + if (token.sealed || this.#state !== "open") throw this.released(); + } + + /** The live object fails `broken` with `error` (a document the code cannot read); ignored when it is not live. */ + fail(error: ChordError): void { + this.#owner.fail(error); + } + + /** The object is ending: every transaction is refused, the one holding the line discarded; documents stay readable. */ + seal(): void { + if (this.#state !== "open") return; + this.#state = "sealed"; + this.own.seal(); + this.#cut(); + } + + /** + * Resolves once the line is free. After `seal` only an admitted step can hold it (every other holder was cut): + * the object's teardown waits here, so the step adopts into documents that still exist (slice 6a §1.3). + */ + async settled(): Promise { + const release = await this.line.acquire(); + release(); + } + + /** Drops every document and pin; watches end `ended`, and every later use throws `released`. */ + end(): void { + this.#state = "ended"; + this.residency.end(); + for (const slot of this.#slots.values()) slot.drop(); + this.#slots.clear(); + this.#pins = []; + } + + /** The `released` error of this object's documents. */ + released(): ChordError { + return new ChordError({ code: "released", reason: "ended", message: `${this.label}: ended` }); + } + + /** Discards the transaction holding the line and releases the line. */ + #cut(): void { + const holder = this.#holder; + this.#holder = undefined; + holder?.cut(); + } + + #add(slot: S): S { + this.#slots.set(slot.definition.current.name, slot); + return slot; + } + + /** The store side of persisted documents named `name`; the attempt checked the store first (P3). */ + #persistence(name: string): Persistence { + const { store, owner } = this.#owner.store!; + return new Persistence(store, owner, name); + } +} diff --git a/packages/chord/src/next/documents/fence.ts b/packages/chord/src/next/documents/fence.ts new file mode 100644 index 000000000..5f02888f4 --- /dev/null +++ b/packages/chord/src/next/documents/fence.ts @@ -0,0 +1,17 @@ +// The seal fence (slice 6a §1.3): the token an attempt, a job run's invocation or the object itself hands its +// transactions. Once sealed, new transactions under it are refused `released` and the open one is discarded. + +/** One seal token; sealed once, never unsealed. */ +export class SealToken { + #sealed = false; + + /** Whether work under this token is refused. */ + get sealed(): boolean { + return this.#sealed; + } + + /** Seals the token; `Documents.fence` also discards the transaction it holds open. */ + seal(): void { + this.#sealed = true; + } +} diff --git a/packages/chord/src/next/documents/handles.ts b/packages/chord/src/next/documents/handles.ts new file mode 100644 index 000000000..686d78e9a --- /dev/null +++ b/packages/chord/src/next/documents/handles.ts @@ -0,0 +1,133 @@ +// Document handles (slice 6a §1.2): what `self.state` returns, one per attempt and definition, a thin view of its +// object's slot bound to the attempt's seal token and the definition's version. + +import type { Context } from "../../context/index.ts"; +import type { Draft } from "../../delta/index.ts"; +import type { Watch, Watchable, WatchHandle } from "../../watch.ts"; +import type { JsonObject, StateDefinition } from "./definition.ts"; +import type { SealToken } from "./fence.ts"; +import type { KeyedSlot, KeySource } from "./keyed-slot.ts"; +import type { SingletonSlot } from "./slot.ts"; + +/** A singleton document: a watchable that always has a value (`initial()` until written, and after a retire). */ +export interface SingletonHandle extends Watchable { + /** One transaction that drafts this document and passes the draft to `fn`. */ + change(fn: (draft: Draft) => R, context: Context): Promise; + /** One transaction that retires this document: back to `initial()`. */ + retire(context: Context): Promise; +} + +/** + * Keyed documents: many per object, by string key. Not a watchable itself: each key is, through `watch(key)` and + * `at(key)`; a live collection is a singleton holding it. + */ +export interface KeyedHandle { + /** The document's value, `undefined` when absent (persisted and not in memory: loaded on the object's line). */ + get(key: string, context: Context): Promise; + /** The keys of the present documents, in code point order. */ + keys(context: Context): AsyncIterable; + /** + * A watch of the document at `key`, `undefined` when absent. It follows the address: a retirement is one frame with + * `retired: true` and `initial()`, a later creation a root replacement; it ends with the object. + */ + watch(key: string, context: Context): Promise | undefined>; + /** + * In the factory only: the document at `key` as a watchable (a state member, a derive source, an `on` source), + * `initial()` while absent, loaded before the object goes live and pinned while this attempt lives. + */ + at(key: string): Watchable; + /** One transaction that drafts the document at `key` and passes the draft to `fn`. */ + change(key: string, fn: (draft: Draft) => R, context: Context): Promise; + /** One transaction that retires the document at `key` (a no-op when absent). */ + retire(key: string, context: Context): Promise; +} + +/** The handle `self.state(definition)` returns. */ +export type HandleOf = D extends StateDefinition + ? K extends true + ? KeyedHandle + : SingletonHandle + : never; + +/** + * A singleton's handle. Reads go to the slot (allowed after the seal); `change` and `retire` take the line under the + * attempt's token. Exposed as a member or read by a derive, it resolves to the slot itself. + */ +export class Singleton implements SingletonHandle { + readonly slot: SingletonSlot; + readonly token: SealToken; + /** The version of the definition it was declared with: the slot refuses it once its values are newer (R15). */ + readonly version: number; + + constructor(slot: SingletonSlot, token: SealToken, version: number) { + this.slot = slot; + this.token = token; + this.version = version; + } + + get value(): JsonObject { + this.slot.definition.assertVersion(this.version); + return this.slot.value; + } + + async watch(context: Context): Promise> { + this.slot.definition.assertVersion(this.version); + // The slot's values are this handle's documents. + return this.slot.openWatch(context) as Watch; + } + + change(fn: (draft: Draft) => R, context: Context): Promise { + return this.slot.owner.tx(async (tx) => fn(await tx.draft(this)), this.token, context); + } + + retire(context: Context): Promise { + return this.slot.owner.tx((tx) => tx.retire(this), this.token, context); + } +} + +/** Who declares an attempt's `at(key)` pins: its factory, which alone may (the attempt). */ +export interface KeyDeclarer { + /** Pins `key` of `slot` for the attempt and returns its source; throws outside the factory. */ + at(slot: KeyedSlot, key: string): KeySource; +} + +/** A keyed handle: reads go to the slot; `change`, `retire` and cold reads take the line under the attempt's token. */ +export class Keyed implements KeyedHandle { + readonly slot: KeyedSlot; + readonly token: SealToken; + /** As `Singleton.version`. */ + readonly version: number; + readonly #declarer: KeyDeclarer; + + constructor(slot: KeyedSlot, token: SealToken, version: number, declarer: KeyDeclarer) { + this.slot = slot; + this.token = token; + this.version = version; + this.#declarer = declarer; + } + + get(key: string, context: Context): Promise { + return this.slot.get(key, this, context); + } + + keys(context: Context): AsyncIterable { + return this.slot.keys(this, context); + } + + watch(key: string, context: Context): Promise | undefined> { + return this.slot.watch(key, this, context); + } + + at(key: string): Watchable { + // The source's values are this slot's documents. + return this.#declarer.at(this.slot, key) as unknown as Watchable; + } + + change(key: string, fn: (draft: Draft) => R, context: Context): Promise { + return this.slot.owner.tx(async (tx) => fn(await tx.draft(this, key)), this.token, context); + } + + retire(key: string, context: Context): Promise { + return this.slot.owner.tx((tx) => tx.retire(this, key), this.token, context); + } +} diff --git a/packages/chord/src/next/documents/index.ts b/packages/chord/src/next/documents/index.ts new file mode 100644 index 000000000..e7568d8f0 --- /dev/null +++ b/packages/chord/src/next/documents/index.ts @@ -0,0 +1,5 @@ +// Documents for apps: definitions, handles and transactions. + +export { defineState, type JsonObject, type StateDefinition } from "./definition.ts"; +export type { HandleOf, KeyedHandle, SingletonHandle } from "./handles.ts"; +export type { Tx } from "./transaction.ts"; diff --git a/packages/chord/src/next/documents/internal.ts b/packages/chord/src/next/documents/internal.ts new file mode 100644 index 000000000..b2eee6b2b --- /dev/null +++ b/packages/chord/src/next/documents/internal.ts @@ -0,0 +1,9 @@ +// Documents for Chord's layers: an object's documents and slots, the handles' classes, the seal token, transactions. + +export { AddressSource } from "./address-source.ts"; +export { Documents, type DocumentsOwner } from "./documents.ts"; +export { SealToken } from "./fence.ts"; +export { type KeyDeclarer, Keyed, Singleton } from "./handles.ts"; +export { type KeyedSlot, KeySource } from "./keyed-slot.ts"; +export type { AnyDefinition, DocumentSlot } from "./slot.ts"; +export type { Staged, Transaction } from "./transaction.ts"; diff --git a/packages/chord/src/next/documents/keyed-slot.ts b/packages/chord/src/next/documents/keyed-slot.ts new file mode 100644 index 000000000..6575fd8ec --- /dev/null +++ b/packages/chord/src/next/documents/keyed-slot.ts @@ -0,0 +1,414 @@ +// Keyed slots (slice 6a §1.2, §4, §10, §11): the documents of one keyed name on one object, by key. In memory, every +// present document stays; persisted, a document is loaded on demand (a cold read is a job on the object's line, one +// load in flight per key, shared) and leaves memory when unpinned and idle for the retention (`residency.ts`). A key +// read through `at(key)` or a watch has its own source, which exists while the key is pinned. + +import { awaitWithContext, type Context } from "../../context/index.ts"; +import { type Prepared, type Tracker, track } from "../../delta/index.ts"; +import type { Watch } from "../../watch.ts"; +import { ChordError } from "../data/index.ts"; +import { asChordError } from "../data/internal.ts"; +import { compareText, storedText } from "../store/text.ts"; +import { AddressSource } from "./address-source.ts"; +import type { JsonObject } from "./definition.ts"; +import type { Documents } from "./documents.ts"; +import type { SealToken } from "./fence.ts"; +import type { Loaded, Persistence, Stored } from "./persistence.ts"; +import { Resident } from "./residency.ts"; +import { type AnyDefinition, type DocumentSlot, type PersistedSlot, SlotDefinition } from "./slot.ts"; + +/** Whom a read is for: a handle's seal token and the version it was declared with (R15). */ +export type Access = { readonly token: SealToken; readonly version: number }; + +/** + * A cold load in flight, shared by its waiters: their handles' tokens, and those already sealed when the load took + * the line (`undefined` until then), whose waiters it refuses (rule 4). + */ +type SharedLoad = { + readonly done: Promise; + readonly tokens: Set; + refused: ReadonlySet | undefined; +}; + +/** The keys `keys(ctx)` reads per page from the store. */ +const KEYS_PAGE = 100; + +/** A keyed slot: one resident per document in memory, one source per pinned key. */ +export class KeyedSlot implements DocumentSlot, PersistedSlot { + readonly owner: Documents; + readonly definition: SlotDefinition; + readonly persisted: PersistedSlot | undefined; + readonly #persistence: Persistence | undefined; + readonly #residents = new Map(); + /** The sources of pinned keys: a key is pinned while it has one. */ + readonly #sources = new Map(); + /** Cold loads in flight, by key: shared by every reader of the key. */ + readonly #loads = new Map(); + /** Names a refused key (the slot's name is the same at every version): made once, not per key check. */ + readonly #keyWhat: string; + + constructor(owner: Documents, definition: AnyDefinition, persistence: Persistence | undefined) { + this.owner = owner; + this.definition = new SlotDefinition(definition); + this.#keyWhat = `${definition.name}: key`; + this.#persistence = persistence; + this.persisted = persistence === undefined ? undefined : this; + } + + get persistence(): Persistence { + // Only reached through `persisted`, which a persisted slot alone has. + return this.#persistence!; + } + + /** + * The value at `key`, `undefined` when absent. A document in memory is read at once (also through a sealed + * handle); a cold persisted one is loaded on the line (refused `released` once the access's token is sealed), and + * `context` ends only this caller's wait. An absent key is not cached (rule 20). + */ + get(raw: string, access: Access, context: Context): Promise { + // Not `async`: `#get` is the one async layer; a refused key still reaches the caller as a rejection. + try { + return this.#get(this.key(raw), access, context); + } catch (error) { + return Promise.reject(error); + } + } + + /** `get` of a key that went through `key` already. */ + async #get(key: string, access: Access, context: Context): Promise { + this.owner.assertLive(); + this.definition.assertVersion(access.version); + const resident = this.#resident(key); + if (resident !== undefined) { + this.#touch(resident); + return resident.tracker.value; + } + if (this.#persistence === undefined) return undefined; + await this.#loadShared(key, access.token, context); + return this.#residents.get(key)?.tracker.value; + } + + /** The present keys in code point order: in memory, the residents; persisted, the store's, a page per line job. */ + async *keys(access: Access, context: Context): AsyncIterable { + this.owner.assertLive(); + this.definition.assertVersion(access.version); + const persistence = this.#persistence; + if (persistence === undefined) { + yield* [...this.#residents.keys()].sort(compareText); + return; + } + let after: string | undefined; + for (;;) { + const page = await this.owner.onLine(() => persistence.keys(after, KEYS_PAGE, context), access.token); + yield* page; + if (page.length < KEYS_PAGE) return; + after = page.at(-1); + } + } + + /** + * A watch of the document at `key`, `undefined` when absent (after a cold load); it follows the address (a + * retirement is a frame with the `retired` flag) and pins the key until it ends. + */ + async watch(raw: string, access: Access, context: Context): Promise | undefined> { + const key = this.key(raw); + if ((await this.#get(key, access, context)) === undefined) return undefined; + const release = this.pin(key, context); + // Retired between the load and now: absent. + if (!this.#residents.has(key)) { + release(); + return undefined; + } + let watch: Watch; + try { + // The source's values are this slot's documents. + watch = this.#sources.get(key)!.openWatch(context) as Watch; + } catch (error) { + release(); + throw error; + } + void watch.closed.then(release); + return watch; + } + + /** + * Pins `key`: its source (made by the first pin) and its document stay in memory until the returned release + * (idempotent) runs; a persisted key not in memory is loaded by the next activation load (`loadDeclared`). + */ + pin(raw: string, context: Context): () => void { + const key = this.key(raw); + let source = this.#sources.get(key); + if (source === undefined) { + const ready = this.#persistence === undefined || this.#residents.has(key); + source = new KeySource(this, key, this.definition.current.initial(context), ready); + this.#sources.set(key, source); + } + source.pins++; + const resident = this.#residents.get(key); + if (resident !== undefined) this.owner.residency.forget(resident); + const pinned = source; + let released = false; + return () => { + if (released) return; + released = true; + if (--pinned.pins > 0 || this.#sources.get(key) !== pinned) return; + this.#sources.delete(key); + const idle = this.#residents.get(key); + if (idle !== undefined) this.#touch(idle); + }; + } + + /** Loads pinned `key` on the line unless it was loaded since it was pinned (a consumer's `at(key)`, slice 6a §12). */ + async ensure(raw: string, context: Context): Promise { + const key = this.key(raw); + if (this.#sources.get(key)?.ready !== false) return; + await this.owner.onLine(() => this.load(key, context)); + } + + /** The source of pinned `key`. */ + source(key: string): KeySource { + return this.#sources.get(this.key(key))!; + } + + /** + * `key` as the slot and the store keep it: the `str()` text rules (slice 6a §8), a lone surrogate replaced by + * U+FFFD, NUL refused `invalid/args`. Plugin keys are a trust boundary; every entry point takes them through here, + * so memory and the store agree on one key. + */ + key(key: string): string { + return storedText(key, this.#keyWhat); + } + + /** Loads every pinned key not loaded since it was pinned (activation, holding the line). */ + async loadDeclared(context: Context): Promise { + for (const [key, source] of this.#sources) if (!source.ready) await this.load(key, context); + } + + /** + * Loads the document at `key` (the caller holds the line) and installs it while it is still absent from memory (a + * commit's `adopt` always wins). A stored version newer than the code's, another fingerprint or a failing `migrate` + * fails the owner (slice 6a §1.4) and rejects with the cause. + */ + async load(key: string, context: Context, token?: SealToken): Promise { + if (this.#residents.has(key)) return; + // A store error rejects as the store's (`storage_failed`, `aborted`). + const document = await this.persistence.read(key, context); + // The transaction was cut meanwhile: what it read may be stale already. + if (token?.sealed === true) return; + let loaded: Loaded | undefined; + try { + loaded = + document === undefined ? undefined : this.persistence.decode(document, this.definition.current, context); + } catch (error) { + // What the code cannot read (a failing `migrate`, whatever it throws) fails the owner. + const failure = asChordError(error, this.owner.label); + this.owner.fail(failure); + throw failure; + } + if (loaded !== undefined && !this.#residents.has(key)) { + const resident = new Resident(this, key, track(loaded.value), loaded.stored, this.definition.current.version); + this.#residents.set(key, resident); + this.#touch(resident); + } + const source = this.#sources.get(key); + if (source !== undefined) source.ready = true; + } + + cold(key: string): boolean { + return this.#persistence !== undefined && !this.#residents.has(key); + } + + stored(key: string): Stored | undefined { + return this.#residents.get(key)?.stored; + } + + /** + * Takes the attempt's declaration: a newer version migrates each document in memory on its next access (persisted + * ones store a base with their next change); the pinned keys are watched, so they migrate now, all or none, and + * take `initial()` again. + */ + hydrate(definition: AnyDefinition, context: Context): void { + const from = this.definition.hydrated; + this.definition.hydrate(definition); + if (from === undefined || from.version === definition.version) return; + const pinned = [...this.#sources.keys()].flatMap((key) => this.#residents.get(key) ?? []); + const migrated = pinned.map((resident) => [resident, this.#migrated(resident, context)] as const); + for (const [resident, tracker] of migrated) { + resident.tracker = tracker; + resident.version = definition.version; + } + this.owner.batch.commit(() => { + for (const source of this.#sources.values()) source.retake(definition.initial(context)); + }, context); + } + + current(key: string): Tracker | undefined { + return this.#resident(key)?.tracker; + } + + /** The value a draft of an absent document starts from: a fresh `initial()`. */ + initial(context: Context): JsonObject { + return this.definition.current.initial(context); + } + + adopt(key: string, tracker: Tracker, prepared: Prepared, stored: Stored | undefined): void { + const source = this.#sources.get(key); + const before = source?.peek(); + tracker.adopt(prepared); + let resident = this.#residents.get(key); + const { version } = this.definition.current; + if (resident === undefined) { + resident = new Resident(this, key, tracker, stored, version); + this.#residents.set(key, resident); + } else { + resident.tracker = tracker; + resident.stored = stored; + resident.version = version; + } + if (source === undefined) this.#touch(resident); + else source.written(before!, prepared); + } + + /** Retires the document at `key` (inside the batch): out of memory, its source reads `initial()` with the flag. */ + retire(key: string): void { + const resident = this.#residents.get(key); + if (resident !== undefined) { + this.#residents.delete(key); + this.owner.residency.forget(resident); + } + this.#sources.get(key)?.retire(); + } + + /** The sweep found `resident` idle past the retention: it leaves memory. */ + evict(resident: Resident): void { + if (this.#residents.get(resident.key) === resident) this.#residents.delete(resident.key); + } + + /** Its object ended or no attempt declares it any more: the documents go and the keys' watches end. */ + drop(): void { + for (const resident of this.#residents.values()) this.owner.residency.forget(resident); + this.#residents.clear(); + for (const source of this.#sources.values()) source.end(); + this.#sources.clear(); + } + + /** + * A cold persisted key's load, one in flight per key, on the line under the owner's context (rule 11). The load is + * every waiter's: when it takes the line, a waiter whose handle was sealed meanwhile (its attempt failed live, the + * object still open) is refused `released`, and with no other waiter the load does not run (rule 4). + */ + async #loadShared(key: string, token: SealToken, context: Context): Promise { + // Also when a load is in flight: a sealed handle's cold read is refused (rule 4). + this.owner.assertAdmitted(token); + let load = this.#loads.get(key); + if (load === undefined) load = this.#startLoad(key); + load.tokens.add(token); + await awaitWithContext(load.done, context); + if (load.refused?.has(token)) throw this.owner.released(); + } + + /** Starts the shared load of `key`; it leaves `#loads` when it settles. */ + #startLoad(key: string): SharedLoad { + const tokens = new Set(); + const load: SharedLoad = { + tokens, + refused: undefined, + // The job runs once the line is taken, after `load` is assigned. + done: this.owner.onLine(async () => { + const refused = new Set([...tokens].filter(({ sealed }) => sealed)); + load.refused = refused; + if (refused.size === tokens.size) throw this.owner.released(); + await this.load(key, this.owner.context); + }), + }; + this.#loads.set(key, load); + const settled = (): void => { + this.#loads.delete(key); + }; + load.done.then(settled, settled); + return load; + } + + /** The resident at `key`, migrated to the slot's version first when it is older (migration on access). */ + #resident(key: string): Resident | undefined { + const resident = this.#residents.get(key); + if (resident === undefined || resident.version === this.definition.current.version) return resident; + const tracker = this.#migrated(resident, this.owner.context); + resident.tracker = tracker; + resident.version = this.definition.current.version; + return resident; + } + + /** + * `resident`'s value migrated from its version to the slot's (checked, R8). A throwing `migrate` fails the live + * owner `broken` (slice 6a §1.4) and throws the failure (an activation fails with it). + */ + #migrated(resident: Resident, context: Context): Tracker { + try { + return track(this.definition.current.migrate(resident.tracker.value, resident.version, context)); + } catch (error) { + const failure = asChordError(error, this.owner.label); + this.owner.fail(failure); + throw failure; + } + } + + /** A use of `resident` while unpinned: persisted, it is idle from now (Durable's granule rule). */ + #touch(resident: Resident): void { + if (this.#persistence !== undefined && !this.#sources.has(resident.key)) this.owner.residency.touch(resident); + } +} + +/** + * One pinned key of a keyed slot as a source (`handle.at(key)`, a keyed member's `at(key)`, `watch(key)`): the + * document's value, `initial()` while it is absent; retirements arrive with the `retired` flag. + */ +export class KeySource extends AddressSource { + readonly owner: Documents; + readonly #slot: KeyedSlot; + readonly key: string; + /** Its pins: at 0 the slot forgets it. */ + pins = 0; + /** Whether the key was loaded since it was pinned (always, in memory): its value is known. */ + ready: boolean; + #initial: JsonObject; + + constructor(slot: KeyedSlot, key: string, initial: JsonObject, ready: boolean) { + super(slot.owner.batch); + this.owner = slot.owner; + this.#slot = slot; + this.key = key; + this.#initial = initial; + this.ready = ready; + } + + get value(): JsonObject { + this.#slot.owner.assertLive(); + if (!this.ready) { + const message = `${this.#slot.definition.current.name}(${this.key}): read before it was loaded`; + throw new ChordError({ code: "not_ready", message }); + } + return this.peek(); + } + + /** The value without the checks (inside its slot's commits). */ + peek(): JsonObject { + return this.#slot.current(this.key)?.value ?? this.#initial; + } + + /** The document was retired: `initial()` from now, with the flag. */ + retire(): void { + this.retired(this.#initial); + } + + /** A new definition took over (inside a batch): `initial()` again, the value as a root replacement. */ + retake(initial: JsonObject): void { + this.#initial = initial; + if (this.ready) this.replaced(); + } + + /** Its slot was dropped: the watches end. */ + end(): void { + this.endWatches(); + } +} diff --git a/packages/chord/src/next/documents/line.ts b/packages/chord/src/next/documents/line.ts new file mode 100644 index 000000000..3f5b62a89 --- /dev/null +++ b/packages/chord/src/next/documents/line.ts @@ -0,0 +1,15 @@ +// An object's line (§9): its transactions, cold loads, key pages and eviction sweeps run on it one at a time, in the +// order they asked. + +/** An object's line: `acquire()` resolves with the release once every earlier holder released; releasing twice is harmless. */ +export class Line { + #tail: Promise = Promise.resolve(); + + /** Resolves with this holder's release once every earlier holder released. */ + acquire(): Promise<() => void> { + const { promise, resolve } = Promise.withResolvers(); + const previous = this.#tail; + this.#tail = promise; + return previous.then(() => resolve); + } +} diff --git a/packages/chord/src/next/documents/persistence.ts b/packages/chord/src/next/documents/persistence.ts new file mode 100644 index 000000000..7b6082286 --- /dev/null +++ b/packages/chord/src/next/documents/persistence.ts @@ -0,0 +1,156 @@ +// Persisted documents' store side (slice 6a §8, §10): a slot's address in the documents engine, what is stored of each +// loaded document (Durable's `LoadedDocument` fields), its loads with the version and fingerprint checks and the +// checked `migrate`, and the writes one commit stages for a changed or retired document. + +import type { Context } from "../../context/index.ts"; +import type { Prepared } from "../../delta/index.ts"; +import { + changeDocument, + createDocument, + type DocumentAddress, + type DocumentContent, + findDocument, + listDocumentKeys, + loadDocument, + retireDocument, + type StoredDocument, +} from "../store/engine.ts"; +import type { StagedWrite } from "../store/step.ts"; +import type { ChordStore } from "../store/store.ts"; +import type { JsonObject } from "./definition.ts"; +import type { AnyDefinition } from "./slot.ts"; + +/** What is stored of a loaded document: its record's id and revision, its base's version, the deltas after it. */ +export type Stored = { + readonly id: number; + readonly revision: number; + /** The version of the stored base: below the definition's, the next change stores a base (rule 1). */ + readonly version: number; + readonly deltasSinceBase: number; +}; + +/** A loaded document: its value at the definition's version, and what is stored of it. */ +export type Loaded = { readonly value: JsonObject; readonly stored: Stored }; + +/** + * What one commit stores of a document, in the store writes' words: a retirement (nothing stored after it), a + * creation whose id the step assigns (`create`, its position among the commit's creations), or a change of the record + * `stored` was prepared on. + */ +export type StoredChange = + | { readonly kind: "retire" } + | { readonly kind: "create"; readonly version: number; readonly index: number } + | { readonly kind: "change"; readonly stored: Stored }; + +/** The documents of one persisted definition name on one object: `(chord, :, name, key)`. */ +export class Persistence { + readonly store: ChordStore; + readonly #address: Omit; + + /** `owner` is `:` (W7); `kind` the definition's name. */ + constructor(store: ChordStore, owner: string, kind: string) { + this.store = store; + this.#address = { namespace: "chord", owner, kind }; + } + + /** + * The stored document at `key`, `undefined` when absent: found and loaded with plain reads (a store error rejects + * as the store's). Counted (`chord.store.loads`). + */ + async read(key: string, context: Context): Promise { + const address = this.#at(key); + const document = await this.store.read(async (executor) => { + const record = await findDocument(executor, address); + return record === undefined ? undefined : loadDocument(executor, record); + }, context); + if (document !== undefined) context.telemetry.metrics?.counter("chord.store.loads").add(1); + return document; + } + + /** + * `document` as `definition` reads it: a stored version newer than the code's, or the same version with another + * fingerprint (D9), throws; an older one is migrated and the result checked once (R8). What throws here is the + * code's (a failing `migrate` too, whatever it throws), never the store's. + */ + decode(document: StoredDocument, definition: AnyDefinition, context: Context): Loaded { + const { record, deltasSinceBase } = document; + const { name, version, fingerprint } = definition; + if (record.version > version) { + throw new Error(`${name}: stored at version ${record.version}, newer than the code's ${version}`); + } + if (record.version === version && record.fingerprint !== fingerprint) { + throw new Error(`${name}: stored with another schema at version ${version} (a schema changed without a bump)`); + } + // The engine stores what commits gave it: JSON objects of the record's version. + const stored = document.value as JsonObject; + const value = record.version === version ? stored : definition.migrate(stored, record.version, context); + return { + value, + stored: { id: record.id, revision: record.revision, version: record.version, deltasSinceBase }, + }; + } + + /** What is stored of the live document at `key` without loading it (a retirement's find), `undefined` when absent. */ + async find(key: string, context: Context): Promise { + const record = await this.store.read((executor) => findDocument(executor, this.#at(key)), context); + if (record === undefined) return undefined; + // The change counter is all a retirement needs; `deltasSinceBase` is not read for one. + return { id: record.id, revision: record.revision, version: record.version, deltasSinceBase: 0 }; + } + + /** Up to `limit` keys of live documents after `after`, in code point order. */ + keys(after: string | undefined, limit: number, context: Context): Promise { + return this.store.read((executor) => listDocumentKeys(executor, this.#address, after, limit), context); + } + + /** + * Stages one commit's writes of the document at `key` into `writes` (rule 8: serialized here, before the queue): + * its retirement when `retired` is stored, then its creation, or the change of `stored` (a base when created, when + * the stored version is older, or when `checkpointWhen` asks; a delta otherwise). `creations` counts the commit's + * creations so far (the step's ids follow their order). Returns what the commit stores. + */ + stage( + key: string, + definition: AnyDefinition, + stored: Stored | undefined, + change: { readonly prepared: Prepared | undefined; readonly retires: boolean }, + writes: StagedWrite[], + creations: number, + ): StoredChange { + const { prepared, retires } = change; + if (retires && stored !== undefined) writes.push(retireDocument(stored.id, stored.revision)); + if (prepared === undefined) return { kind: "retire" }; + const { version, fingerprint } = definition; + if (retires || stored === undefined) { + const base = { version, fingerprint, value: prepared.value }; + writes.push(createDocument(this.#at(key), "latest", base, JSON.stringify(prepared.value))); + return { kind: "create", version, index: creations }; + } + const info = { deltasSinceBase: stored.deltasSinceBase }; + const base = stored.version !== version || definition.checkpointWhen(prepared.value, prepared.ops, info); + const content: DocumentContent = base + ? { kind: "base", version, fingerprint, value: prepared.value } + : { kind: "delta", version, ops: prepared.ops }; + const text = JSON.stringify(base ? prepared.value : prepared.ops); + writes.push(changeDocument(stored.id, stored.revision, content, text)); + const deltasSinceBase = base ? 0 : stored.deltasSinceBase + 1; + return { kind: "change", stored: { id: stored.id, revision: stored.revision + 1, version, deltasSinceBase } }; + } + + /** The address of `key`: a singleton's (`""`) has no key. */ + #at(key: string): DocumentAddress { + return key === "" ? this.#address : { ...this.#address, key }; + } +} + +/** What is stored after a commit's `change` was adopted with the commit's created ids. */ +export function storedAfter(change: StoredChange, ids: readonly number[]): Stored | undefined { + switch (change.kind) { + case "retire": + return undefined; + case "create": + return { id: ids[change.index]!, revision: 1, version: change.version, deltasSinceBase: 0 }; + case "change": + return change.stored; + } +} diff --git a/packages/chord/src/next/documents/residency.ts b/packages/chord/src/next/documents/residency.ts new file mode 100644 index 000000000..b8aad848c --- /dev/null +++ b/packages/chord/src/next/documents/residency.ts @@ -0,0 +1,110 @@ +// Residency of keyed documents (slice 6a §10, rule 23): which keyed documents are in memory, and the eviction of +// persisted ones nobody pinned and nobody read or wrote for the retention. One idle structure and one timer per +// object, Durable's (`Session.#touchDocument`): an insertion-ordered map whose oldest entry the timer waits for, a +// touch moving an entry to the back at most once per granule (O(1) per touch, no timer per key). + +import type { Tracker } from "../../delta/index.ts"; +import type { JsonObject } from "./definition.ts"; +import type { Stored } from "./persistence.ts"; + +/** One keyed document in memory: its slot, its tracker and, persisted, what is stored of it. */ +export class Resident { + readonly slot: { evict(resident: Resident): void }; + readonly key: string; + tracker: Tracker; + stored: Stored | undefined; + /** The definition version the tracker's value is at: below the slot's, it migrates on its next access. */ + version: number; + + constructor( + slot: { evict(resident: Resident): void }, + key: string, + tracker: Tracker, + stored: Stored | undefined, + version: number, + ) { + this.slot = slot; + this.key = key; + this.tracker = tracker; + this.stored = stored; + this.version = version; + } +} + +/** How long an unpinned persisted keyed document stays in memory after its last read or write (slice 6a §10). */ +export const RETENTION_MS = 10 * 60_000; + +/** A touch moves an entry at most once per granule: a tenth of the retention, up to 1 s (Durable's rule). */ +const GRANULE_MS = Math.min(1_000, RETENTION_MS / 10); + +/** The idle persisted keyed documents of one object, oldest first, and the one timer evicting them. */ +export class Residency { + /** Runs a sweep on the object's line. */ + readonly #onLine: (sweep: () => void) => void; + readonly #idle = new Map(); + /** The pending sweep: when it is due and its timer; stays set until its job runs on the line. */ + #sweep: { readonly at: number; timer: ReturnType | undefined } | undefined; + #ended = false; + + constructor(onLine: (sweep: () => void) => void) { + this.#onLine = onLine; + } + + /** A use of an unpinned `resident`: it becomes the newest idle one (once per granule), and a sweep is scheduled. */ + touch(resident: Resident): void { + const now = Date.now(); + const since = this.#idle.get(resident); + // A hot document moves at most once per granule: it may then leave up to a granule early (a reload). + if (since !== undefined && now - since < GRANULE_MS) return; + this.#idle.delete(resident); + this.#idle.set(resident, now); + if (this.#sweep === undefined) this.#schedule(); + } + + /** `resident` is pinned or left memory: it is no longer idle. */ + forget(resident: Resident): void { + this.#idle.delete(resident); + } + + /** The object ended: nothing is evicted any more, the timer is cleared. */ + end(): void { + this.#ended = true; + this.#idle.clear(); + clearTimeout(this.#sweep?.timer); + this.#sweep = undefined; + } + + /** Schedules the eviction of the oldest idle document when its retention expires. */ + #schedule(): void { + const oldest = this.#idle.values().next(); + if (this.#ended || oldest.done === true) return; + const at = oldest.value + RETENTION_MS; + const sweep: { readonly at: number; timer: ReturnType | undefined } = { at, timer: undefined }; + this.#sweep = sweep; + // A timer firing early sweeps nothing and schedules the next one. + sweep.timer = setTimeout( + () => + this.#onLine(() => { + if (this.#sweep === sweep) this.#sweep = undefined; + this.#evictExpired(); + }), + Math.max(0, at - Date.now()), + ); + } + + /** + * On the line: evicts the idle documents whose retention passed, oldest first, then schedules the next sweep. No + * load in flight needs a pin against it: cold loads run on the line too, so a sweep never runs during one, and a + * load installing an unpinned key touches it (it is then the newest idle one). + */ + #evictExpired(): void { + if (this.#ended) return; + const expired = Date.now() - RETENTION_MS; + for (const [resident, since] of this.#idle) { + if (since > expired) break; + this.#idle.delete(resident); + resident.slot.evict(resident); + } + if (this.#sweep === undefined) this.#schedule(); + } +} diff --git a/packages/chord/src/next/documents/slot.ts b/packages/chord/src/next/documents/slot.ts new file mode 100644 index 000000000..657e8d90d --- /dev/null +++ b/packages/chord/src/next/documents/slot.ts @@ -0,0 +1,239 @@ +// Document slots (slice 6a §1.2): one per declared document name on its object, across the object's attempts. A slot +// holds the documents' trackers, the version and definition its values are of, what is stored of persisted ones, and +// the watches; each attempt's handles are thin views of it. This file: what every slot is, and the singleton's slot +// (the keyed one is `keyed-slot.ts`). + +import type { Context } from "../../context/index.ts"; +import { type Prepared, type Tracker, track } from "../../delta/index.ts"; +import { ChordError } from "../data/index.ts"; +import { AddressSource } from "./address-source.ts"; +import type { JsonObject, StateDefinition } from "./definition.ts"; +import type { Documents } from "./documents.ts"; +import type { SealToken } from "./fence.ts"; +import type { Persistence, Stored } from "./persistence.ts"; + +/** A document definition, whatever its value and keyedness. */ +export type AnyDefinition = StateDefinition; + +/** + * What a transaction drafts and adopts: documents of one object by key (a singleton ignores the key). The slots of + * `self.state`, and Chord's own (a job's run state documents). + */ +export interface DocumentSlot { + readonly owner: Documents; + /** The tracker of the document at `key`, `undefined` while absent. */ + current(key: string): Tracker | undefined; + /** The value a draft of an absent document starts from. */ + initial(context: Context): JsonObject; + /** The persisted slot's store side; `undefined` for documents in memory only. */ + readonly persisted: PersistedSlot | undefined; + /** Adopts a commit of the document at `key` (inside the batch), with what is stored of it after the commit. */ + adopt(key: string, tracker: Tracker, prepared: Prepared, stored: Stored | undefined): void; + /** Retires the document at `key` (inside the batch). */ + retire(key: string): void; +} + +/** What a transaction needs of a persisted slot (slice 6a §4, §10). */ +export interface PersistedSlot { + readonly persistence: Persistence; + readonly definition: SlotDefinition; + /** Whether the document at `key` may be stored without being in memory (a persisted keyed document not loaded). */ + cold(key: string): boolean; + /** What is stored of the document at `key` as memory knows it; `undefined` when absent or cold. */ + stored(key: string): Stored | undefined; + /** + * Loads the document at `key` (its caller holds the line): installed when present and still absent from memory, + * and, for a transaction's load (`token`), while the token is not sealed (a cut transaction released the line, so + * a successor may have changed or retired the key meanwhile). + */ + load(key: string, context: Context, token?: SealToken): Promise; +} + +/** + * The definition a slot's values are of (the one that hydrated it last; the first declaration until then): the checks + * of a new attempt's declaration and the old handles' version fence (R15). + */ +export class SlotDefinition { + #current: AnyDefinition; + #hydrated = false; + + constructor(definition: AnyDefinition) { + this.#current = definition; + } + + /** The definition the values are of. */ + get current(): AnyDefinition { + return this.#current; + } + + /** The definition the values were hydrated with, `undefined` before the first hydration. */ + get hydrated(): AnyDefinition | undefined { + return this.#hydrated ? this.#current : undefined; + } + + /** + * Checks a declaration of this name by a new attempt: a flipped `keyed` or `persist`, a version older than the values (old code) + * or the same version with another fingerprint (a schema changed without a bump) throw (the attempt fails broken). + */ + check(definition: AnyDefinition): void { + const { name, version, keyed, persist, fingerprint } = this.#current; + if (definition.keyed !== keyed) { + throw new Error(`${name}: declared ${definition.keyed ? "keyed" : "singleton"}, its documents are not`); + } + if (definition.persist !== persist) { + throw new Error(`${name}: declared ${definition.persist ? "persisted" : "in memory"}, its documents are not`); + } + if (definition.version < version) { + throw new Error(`${name}: version ${definition.version} is older than its documents' (${version})`); + } + if (definition.version === version && definition.fingerprint !== fingerprint) { + throw new Error(`${name}: the schema changed without a version bump (version ${version})`); + } + } + + /** Records the hydration with `definition`. */ + hydrate(definition: AnyDefinition): void { + this.#current = definition; + this.#hydrated = true; + } + + /** + * Throws `released` for a handle of `version` once the values migrated past it (old code reads no new shape), and + * `not_ready` for a handle of a newer version until its activation migrated them (new code reads no old shape: a + * retry's factory). + */ + assertVersion(version: number): void { + const current = this.#current; + if (version === current.version) return; + const message = `${current.name}: a handle of version ${version}; the documents are at version ${current.version}`; + if (version < current.version) throw new ChordError({ code: "released", reason: "version", message }); + throw new ChordError({ code: "not_ready", message: `${message} until the activation migrates them` }); + } +} + +/** + * A singleton's slot: one tracker while written, `initial()` while absent; the source its handles and members watch. + * Persisted, it loads once per object (again after a failed load) and stays in memory while the object lives. + */ +export class SingletonSlot extends AddressSource implements DocumentSlot, PersistedSlot { + readonly owner: Documents; + readonly definition: SlotDefinition; + readonly persisted: PersistedSlot | undefined; + readonly #persistence: Persistence | undefined; + /** `initial()`, taken at hydration; `undefined` before the first. */ + #initial: JsonObject | undefined; + #tracker: Tracker | undefined; + /** What is stored of it (persisted); `undefined` while absent. */ + #stored: Stored | undefined; + /** In memory: always; persisted: once loaded. */ + #loaded: boolean; + + constructor(owner: Documents, definition: AnyDefinition, persistence: Persistence | undefined) { + super(owner.batch); + this.owner = owner; + this.definition = new SlotDefinition(definition); + this.#persistence = persistence; + this.persisted = persistence === undefined ? undefined : this; + this.#loaded = persistence === undefined; + } + + get persistence(): Persistence { + // Only reached through `persisted`, which a persisted slot alone has. + return this.#persistence!; + } + + get value(): JsonObject { + this.owner.assertLive(); + if (this.#initial === undefined || !this.#loaded) { + const message = `${this.definition.current.name}: read before activation`; + throw new ChordError({ code: "not_ready", message }); + } + return this.#tracker?.value ?? this.#initial; + } + + /** + * Takes the attempt's declaration `definition` (`check` accepted it): the first takes `initial()`; a newer version + * migrates the value in memory (a persisted one stores a base with its next change) and takes `initial()` again, + * and a value that changed reaches watches as a root replacement; the same version keeps everything. + */ + hydrate(definition: AnyDefinition, context: Context): void { + const from = this.definition.hydrated; + if (from !== undefined && from.version === definition.version) { + this.definition.hydrate(definition); + return; + } + const before = from === undefined || !this.#loaded ? undefined : this.value; + const tracker = + this.#tracker === undefined || from === undefined + ? this.#tracker + : track(definition.migrate(this.#tracker.value, from.version, context)); + this.#initial = definition.initial(context); + this.#tracker = tracker; + this.definition.hydrate(definition); + if (before !== undefined) this.batch.commit(() => this.replaced(), context); + } + + /** Loads the persisted document unless it is loaded already (the caller holds the line). */ + async load(_key: string, context: Context): Promise { + if (this.#loaded) return; + const document = await this.persistence.read("", context); + // Singletons load at activation, on the line: what the code cannot read fails the activation. + const loaded = + document === undefined ? undefined : this.persistence.decode(document, this.definition.current, context); + if (loaded !== undefined) { + this.#tracker = track(loaded.value); + this.#stored = loaded.stored; + } + this.#loaded = true; + } + + /** Loads the persisted document unless it is loaded already (activation, holding the line). */ + loadDeclared(context: Context): Promise { + return this.load("", context); + } + + /** A singleton has one document: its key is always `""`. */ + key(_key: string): string { + return ""; + } + + cold(_key: string): boolean { + return false; + } + + stored(_key: string): Stored | undefined { + return this.#stored; + } + + current(_key: string): Tracker | undefined { + return this.#tracker; + } + + /** The value a draft of the absent document starts from: the `initial()` taken at hydration. */ + initial(_context: Context): JsonObject { + return this.#initial!; + } + + adopt(_key: string, tracker: Tracker, prepared: Prepared, stored: Stored | undefined): void { + // Unchecked: a step admitted before the object ended adopts after its seal (slice 6a §1.3). + const before = this.#tracker?.value ?? this.#initial!; + tracker.adopt(prepared); + this.#tracker = tracker; + this.#stored = stored; + this.written(before, prepared); + } + + /** Retires the document: it reads `initial()` again, and the frame says it was retired. */ + retire(_key: string): void { + this.#tracker = undefined; + this.#stored = undefined; + this.retired(this.#initial!); + } + + /** Its object ended or no attempt declares it any more: the document goes and its watches end. */ + drop(): void { + this.#tracker = undefined; + this.#stored = undefined; + this.endWatches(); + } +} diff --git a/packages/chord/src/next/documents/transaction.ts b/packages/chord/src/next/documents/transaction.ts new file mode 100644 index 000000000..0dfb6c1bb --- /dev/null +++ b/packages/chord/src/next/documents/transaction.ts @@ -0,0 +1,510 @@ +// Transactions (§9, R12, R13): drafts of an object's documents, prepared and adopted in one batch, with Chord's own +// staged changes (job runs, §15), fenced by the token they run under; they run on the object's line (`line.ts`). + +import { type Context, withoutAbortSignal } from "../../context/index.ts"; +import { type Change, type Draft, type Prepared, type Tracker, track } from "../../delta/index.ts"; +import { ChordError } from "../data/index.ts"; +import type { JobHandle, Outcome, Run, StartOptions } from "../jobs/index.ts"; +import type { Job } from "../jobs/internal.ts"; +import type { Columns, PutRowOf, RowOf } from "../store/columns.ts"; +import type { Condition, SetOf } from "../store/expression.ts"; +import type { ColumnRefs, Query } from "../store/query.ts"; +import type { StagedWrite } from "../store/step.ts"; +import type { ChordStore } from "../store/store.ts"; +import type { KeyOf } from "../store/table.ts"; +import { TableChanges } from "../store/table-changes.ts"; +import { type Page, type PageOptions, ROWS_PAGE, type Table, TableHandle } from "../store/table-handle.ts"; +import type { JsonObject } from "./definition.ts"; +import type { Documents } from "./documents.ts"; +import type { SealToken } from "./fence.ts"; +import type { Keyed, KeyedHandle, Singleton, SingletonHandle } from "./handles.ts"; +import type { KeyedSlot } from "./keyed-slot.ts"; +import { type Stored, type StoredChange, storedAfter } from "./persistence.ts"; +import type { DocumentSlot, SingletonSlot } from "./slot.ts"; + +/** What `self.tx` gives its callback. Inside it: only `await tx.*`, no other `self.tx`, no calls to other objects. */ +export interface Tx { + /** The document's draft: the same one for the same document within one transaction. */ + draft(handle: SingletonHandle): Promise>; + draft(handle: KeyedHandle, key: string): Promise>; + /** Retires the document when the transaction commits (its address reads `initial()` again); a no-op when it is absent. */ + retire(handle: SingletonHandle): Promise; + retire(handle: KeyedHandle, key: string): Promise; + /** + * Starts a run of `job`, one of this object's jobs, with this transaction's other writes (§15): both commit or + * neither. Returns its `Run` at once; a discarded transaction's `Run` rejects `released` from `outcome`. A key of an + * unsettled run (one started earlier in this transaction too) returns that run. + */ + start(job: JobHandle, args: A, options?: StartOptions): Run>; + /** The row with `key` as this transaction leaves it (its own `put`, `update`, `delete` overlaid; W10). */ + get( + table: Table, + key: KeyOf, + ): Promise | undefined>; + /** A page of committed rows, stable for the transaction (without its own writes). */ + page( + table: Table, + q: Query, + limit: number, + options?: PageOptions, + ): Promise>>; + /** Every committed row of `q`, as `page`. */ + rows(table: Table, q: Query): AsyncIterable>; + /** Writes `row` when the transaction commits (an upsert: the whole row replaced). */ + put(table: Table, row: PutRowOf): void; + /** Changes the row with `key` when the transaction commits; an absent key changes nothing. */ + update( + table: Table, + key: KeyOf, + set: SetOf | ((row: ColumnRefs) => SetOf), + ): void; + /** Deletes the row with `key` when the transaction commits. */ + delete(table: Table, key: KeyOf): void; + /** Changes every row matching `where` when the transaction commits (one statement, its count recorded). */ + updateWhere( + table: Table, + where: (row: ColumnRefs) => Condition, + set: SetOf | ((row: ColumnRefs) => SetOf), + ): void; + /** Deletes every row matching `where` when the transaction commits. */ + deleteWhere( + table: Table, + where: (row: ColumnRefs) => Condition, + ): void; +} + +/** + * Chord's own changes staged in a transaction (a job's run records): adopted in its commit step with the documents, + * then told how the transaction ended, once the owner's line is free. + */ +export interface Staged { + /** Adopts the changes, inside the commit's batch. */ + adopt(): void; + /** The transaction committed: the actions after the commit (a run starts, waiters settle). */ + committed(): void; + /** The transaction was discarded: nothing was adopted. */ + discarded(): void; +} + +/** + * What one transaction does with one document: a draft, a retirement, or both (retired, then drafted anew); `found` + * is what is stored of a cold persisted document it retires without loading (R13). + */ +type Entry = { + readonly slot: DocumentSlot; + readonly key: string; + draft: { readonly tracker: Tracker; readonly change: Change } | undefined; + retire: boolean; + found: Stored | undefined; +}; + +/** + * One document change a commit adopts: a removal, a draft's prepared change, or both; and for a persisted document + * what the commit stores of it. + */ +type Adoption = { + readonly entry: Entry; + readonly prepared: Prepared | undefined; + readonly retires: boolean; + readonly stored: StoredChange | undefined; +}; + +/** The created documents' ids of a commit without store writes: none. */ +const NO_IDS: readonly number[] = []; + +/** + * One transaction of an object, under the seal token of whoever runs it (an attempt, a job run's invocation, the + * object). Commit prepares every draft (Delta records ops and checks only placed values) and adopts the changed + * documents and the staged changes in one batch; drafts are revoked once it settles. Cut at its token's seal, it + * discards everything, releases the line and refuses every later use. + */ +export class Transaction implements Tx { + readonly #documents: Documents; + readonly token: SealToken; + readonly #release: () => void; + readonly #context: Context; + readonly #entries = new Map>(); + /** Chord's own staged changes, by their owner (a job); made by the first. */ + #staged: Map | undefined; + /** + * The store writes, in staging order, and the store they commit to; made by the first (the flag that sends the + * commit through the store's step, W12). + */ + #writes: { readonly store: ChordStore; readonly staged: StagedWrite[] } | undefined; + /** The tables written, each with its writes and overlay. */ + #tables: Map | undefined; + /** `tx.*` reads underway: a callback settling with one is refused (rule 3, as Durable's). */ + #pending = 0; + #state: "open" | "committing" | "stepping" | "committed" | "discarded" | "cut" = "open"; + /** Rejected by `cut` while the commit waits in the store's queue: its caller is told at once (rule 3). */ + #queued: PromiseWithResolvers | undefined; + + constructor(documents: Documents, token: SealToken, release: () => void, context: Context) { + this.#documents = documents; + this.token = token; + this.#release = release; + this.#context = context; + } + + /** The object whose transaction this is. */ + get documents(): Documents { + return this.#documents; + } + + draft(handle: SingletonHandle): Promise>; + draft(handle: KeyedHandle, key: string): Promise>; + draft(handle: SingletonHandle | KeyedHandle, key = ""): Promise> { + // Not `async`: the draft's promise is returned as it is, without another promise around it (the in-memory path + // stays as cheap as before the store); a throw still reaches the caller as a rejection. + let slot: SingletonSlot | KeyedSlot; + let at: string; + // Keyed keys follow the text rules (`KeyedSlot.key`); a singleton's key is always "". + try { + slot = this.#slotOf(handle); + at = key === "" ? key : slot.key(key); + } catch (error) { + return Promise.reject(error); + } + // The entry is made for this handle's slot, whose documents are `T`. + return this.draftSlot(slot, at) as Promise>; + } + + /** + * The draft of the document at `key` of `slot` (Chord's own slots, a job's run states). A cold persisted document + * is loaded first, inside the transaction (it holds the line); the draft counts as pending until it settles (rule + * 3). A warm draft settles at once. + */ + async draftSlot(slot: DocumentSlot, key: string): Promise> { + const entry = this.#entry(slot, key); + const persisted = slot.persisted; + if (entry.draft !== undefined || entry.retire || persisted?.cold(key) !== true) return this.#begin(entry); + this.#pending++; + try { + await persisted.load(key, this.#context, this.token); + this.#assertOpen(); + return this.#begin(entry); + } finally { + this.#pending--; + } + } + + retire(handle: SingletonHandle): Promise; + retire(handle: KeyedHandle, key: string): Promise; + async retire(handle: SingletonHandle | KeyedHandle, key = ""): Promise { + const slot = this.#slotOf(handle); + const at = key === "" ? key : slot.key(key); + const entry = this.retireSlot(slot, at); + const persisted = slot.persisted; + // A cold persisted document is found, not loaded (one read; absent: a no-op, R13). + if (entry.found === undefined && persisted?.cold(at)) { + entry.found = await this.#track(persisted.persistence.find(at, this.#context)); + } + } + + /** Retires the document at `key` of `slot` when the transaction commits. */ + retireSlot(slot: DocumentSlot, key: string): Entry { + const entry = this.#entry(slot, key); + entry.draft?.change.abort(); + entry.draft = undefined; + entry.retire = true; + return entry; + } + + start(job: JobHandle, args: A, options: StartOptions = {}): Run> { + // Job handles are Chord's own: `self.job` made them. + return (job as unknown as Job).stage(this, args, this.#context, options) as Run>; + } + + get( + table: Table, + key: KeyOf, + ): Promise | undefined> { + const handle = this.#handle(table); + const changes = this.#tables?.get(handle); + const read = changes === undefined ? handle.get(key, this.#context) : changes.get(key, this.#context); + // The handle's rows are the table's, typed by `C`. + return this.#track(read) as Promise | undefined>; + } + + page( + table: Table, + q: Query, + limit: number, + options: PageOptions = {}, + ): Promise>> { + // As in `get`; the query's table is checked against the handle's. + const read = this.#handle(table).page(q as unknown as Query, limit, this.#context, options); + return this.#track(read) as Promise>>; + } + + async *rows( + table: Table, + q: Query, + ): AsyncIterable> { + let after: string | undefined; + do { + const page = await this.page(table, q, ROWS_PAGE, after === undefined ? {} : { after }); + yield* page.items; + after = page.next; + } while (after !== undefined); + } + + put(table: Table, row: PutRowOf): void { + this.#changes(table).put(row); + } + + update( + table: Table, + key: KeyOf, + set: SetOf | ((row: ColumnRefs) => SetOf), + ): void { + this.#changes(table).update(key, set); + } + + delete(table: Table, key: KeyOf): void { + this.#changes(table).delete(key); + } + + updateWhere( + table: Table, + where: (row: ColumnRefs) => Condition, + set: SetOf | ((row: ColumnRefs) => SetOf), + ): void { + this.#changes(table).updateWhere(where, set); + } + + deleteWhere( + table: Table, + where: (row: ColumnRefs) => Condition, + ): void { + this.#changes(table).deleteWhere(where); + } + + /** The changes `owner` staged in this transaction, made by `make` on its first use. */ + stage(owner: object, make: () => S): S { + this.#assertOpen(); + this.#staged ??= new Map(); + let staged = this.#staged.get(owner); + if (staged === undefined) { + staged = make(); + this.#staged.set(owner, staged); + } + // Each owner stages one kind of change, the one its `make` returns. + return staged as S; + } + + /** The changes `owner` staged in this transaction so far, if any. */ + staged(owner: object): S | undefined { + // As in `stage`: the owner's own kind. + return this.#staged?.get(owner) as S | undefined; + } + + /** + * Prepares every draft and adopts what changed, and the staged changes, in one batch; nothing when nothing changed. + * A document retired and drafted anew is removed, then adopted: its frame carries the retirement. Persisted + * documents stage their writes here (rule 8), which sends the commit through the store's step. + */ + commit(): void | Promise { + this.#assertOpen(); + if (this.#pending > 0) + throw new Error(`${this.#documents.label}: the callback settled before its pending tx operations`); + this.#state = "committing"; + this.#context.abortSignal?.throwIfAborted(); + const adoptions: Adoption[] = []; + let creations = 0; + for (const entries of this.#entries.values()) { + for (const entry of entries.values()) { + let prepared = entry.draft?.change.prepare(); + if (prepared !== undefined && prepared.ops.length === 0) { + prepared.abort(); + prepared = undefined; + } + const { slot, key } = entry; + const retires = entry.retire && (slot.current(key) !== undefined || entry.found !== undefined); + if (prepared === undefined && !retires) continue; + const persisted = slot.persisted; + let stored: StoredChange | undefined; + if (persisted !== undefined) { + const { persistence, definition } = persisted; + this.#writes ??= { store: persistence.store, staged: [] }; + const change = { prepared, retires }; + const was = persisted.stored(key) ?? entry.found; + stored = persistence.stage(key, definition.current, was, change, this.#writes.staged, creations); + if (stored.kind === "create") creations++; + } + adoptions.push({ entry, prepared, retires, stored }); + } + } + const writes = this.#writes; + if (writes === undefined) { + this.#adopt(adoptions, NO_IDS); + this.#state = "committed"; + return; + } + return this.#step(writes.store, writes.staged, adoptions); + } + + /** + * Commits through the store's step (§3): admitted while neither cut nor sealed (else `released`, nothing written), + * then the documents and staged changes adopted inside the step. Once queued, the caller's abort no longer counts; + * a cut while queued rejects `released` at once (the queue drops the entry at its admission, writing nothing). + */ + async #step(store: ChordStore, staged: readonly StagedWrite[], adoptions: readonly Adoption[]): Promise { + const fence = { + admit: () => { + if (this.#state !== "committing" || this.token.sealed) return false; + this.#state = "stepping"; + return true; + }, + adopt: (_seq: number, ids: readonly number[]) => this.#adopt(adoptions, ids), + }; + this.#queued = Promise.withResolvers(); + try { + // The race handles the commit's own later rejection too. + await Promise.race([store.commit(staged, fence, withoutAbortSignal(this.#context)), this.#queued.promise]); + } catch (error) { + if (this.#state === "cut") throw this.#cutError(); + throw error; + } finally { + this.#queued = undefined; + } + this.#state = "committed"; + } + + /** + * Adopts the changed documents (persisted ones with what is stored of them, `ids` being the commit's created + * documents' ids) and the staged changes in one batch; nothing when nothing changed. + */ + #adopt(adoptions: readonly Adoption[], ids: readonly number[]): void { + const staged = this.#staged; + if (adoptions.length === 0 && staged === undefined) return; + this.#documents.batch.commit(() => { + for (const { entry, prepared, retires, stored } of adoptions) { + if (retires) entry.slot.retire(entry.key); + if (prepared === undefined) continue; + const after = stored === undefined ? undefined : storedAfter(stored, ids); + entry.slot.adopt(entry.key, entry.draft!.tracker, prepared, after); + } + for (const each of staged?.values() ?? []) each.adopt(); + }, this.#context); + } + + /** Aborts every draft and prepared change. */ + discard(): void { + if (this.#state === "cut") return; + this.#state = "discarded"; + this.#abortDrafts(); + } + + /** + * Its token was sealed while it held the line: every draft is aborted, the line released, and every later use, + * its commit included, throws `released`. + */ + cut(): void { + // A commit admitted to the step runs to its end; one still queued is refused at admission. + if (this.#state !== "open" && this.#state !== "committing") return; + this.#state = "cut"; + this.#abortDrafts(); + this.#release(); + this.#queued?.reject(this.#cutError()); + } + + /** Tells the staged changes how the transaction ended; runs once the object's line is free. */ + settle(): void { + if (this.#staged === undefined) return; + const committed = this.#state === "committed"; + for (const staged of this.#staged.values()) { + if (committed) staged.committed(); + else staged.discarded(); + } + } + + #abortDrafts(): void { + for (const entries of this.#entries.values()) for (const entry of entries.values()) entry.draft?.change.abort(); + } + + /** The slot behind a handle of this object, after the handle's version fence (R15). */ + #slotOf(handle: SingletonHandle | KeyedHandle): SingletonSlot | KeyedSlot { + // Handles are Chord's own: `self.state` made them. + const { slot, version } = handle as Singleton | Keyed; + slot.definition.assertVersion(version); + return slot; + } + + #entry(slot: DocumentSlot, key: string): Entry { + this.#assertOpen(); + if (slot.owner !== this.#documents) { + throw new Error(`${this.#documents.label}: a transaction drafts only its own object's documents`); + } + let entries = this.#entries.get(slot); + if (entries === undefined) { + entries = new Map(); + this.#entries.set(slot, entries); + } + let entry = entries.get(key); + if (entry === undefined) { + entry = { slot, key, draft: undefined, retire: false, found: undefined }; + entries.set(key, entry); + } + return entry; + } + + #assertOpen(): void { + if (this.#state === "open") return; + if (this.#state === "cut") throw this.#cutError(); + throw new Error(`${this.#documents.label}: the transaction has settled`); + } + + #cutError(): ChordError { + return new ChordError({ + code: "released", + reason: "discarded", + message: `${this.#documents.label}: the transaction was discarded at the seal`, + }); + } + + /** The handle behind `table`, one of this object's tables (another object's is refused). */ + #handle(table: object): TableHandle { + this.#assertOpen(); + // Plugin code may pass any table: only this object's handles are taken. + const handle = table; + if (!(handle instanceof TableHandle) || handle.owner.scope !== this.#documents) { + throw new Error(`${this.#documents.label}: a transaction writes only its own object's tables`); + } + return handle; + } + + /** The writes to `table` in this transaction, made by the first. */ + #changes(table: object): TableChanges { + const handle = this.#handle(table); + this.#tables ??= new Map(); + let changes = this.#tables.get(handle); + if (changes === undefined) { + this.#writes ??= { store: handle.store, staged: [] }; + changes = new TableChanges(handle, this.#writes.staged); + this.#tables.set(handle, changes); + } + return changes; + } + + /** Counts `read` as pending until the operation's promise settles (rule 3). */ + async #track(read: Promise): Promise { + this.#pending++; + try { + return await read; + } finally { + this.#pending--; + } + } + + /** The entry's draft, begun on the current document (or a fresh `initial()`) by the first use. */ + #begin(entry: Entry): Draft { + if (entry.draft === undefined) { + const { slot, key } = entry; + const current = entry.retire ? undefined : slot.current(key); + const tracker = current ?? track(slot.initial(this.#context)); + entry.draft = { tracker, change: tracker.beginChange() }; + } + return entry.draft.change.state; + } +} diff --git a/packages/chord/src/next/host/README.md b/packages/chord/src/next/host/README.md new file mode 100644 index 000000000..2474ca16a --- /dev/null +++ b/packages/chord/src/next/host/README.md @@ -0,0 +1,479 @@ +# host + +Hosts, facets, objects, natives and references: hosts serving `local` and `remote` types, with failures and retries, +principals and variants, links to children, to a parent and over routes to addressable hosts (with relays between +them), the heartbeat, the `ctx` of method handlers, (slice 4b) revocation, cycles across hosts, presence, +keep-alive counts, size limits per part and compression at the origin, (slice 5) routines and the objects' jobs +(declared here, built in `jobs/`), and (slice 6a) the host's store: tables, persisted documents and eager table +migrations at activation (built in `store/` and `documents/`), and live objects that fail. + +## For apps (`index.ts`) + +```ts +import { broken } from "../data/index.ts"; +import { createHost, defineFacet, sleep } from "./index.ts"; + +const facet = defineFacet((host) => { // a facet declares only: types to serve + host.serve(Notes, (_, self) => { // empty params: activated at load, lives until close + const notes = self.state(NotesDoc); + const info = self.get(SessionInfo, {}); + const watcher = self.open( // a native: opened during activation, closed at the end + async (ctx) => watchDir(info.dataDir.value, ctx.abortSignal), + async (w, reason) => w.close(), + ); + self.on(info.dataDir, async ({ value: dir }, ctx) => watcher.value.follow(dir, ctx)); // first the current value; never a failure + self.onLive(async (ctx) => { // once live; its ctx is aborted when the object ends + for (;;) { await sleep(HOUR, ctx); await compact(ctx); } + }); + return { + notes, + count: self.derive([notes], ([n]) => n.list.length), + add: async ({ text }, ctx) => { + await self.tx(async (tx) => { (await tx.draft(notes)).list.push({ text }); }, ctx); + return null; + }, + rescan: async (ctx) => watcher.value.rescan(ctx), + }; + }); + host.serve(Remote, (_, self) => { + self.open(async (ctx) => { + const config = readConfig(); + if (!config) throw broken(new Error("no remote configured")); // retrying is pointless + return connect(config, ctx.abortSignal); // a throw here is retried with backoff + }, (c) => c.close()); + return {}; + }); + host.serve(Draft, ({ conversation }, self) => { // params: lives while referenced + const store = self.get(Notes, {}); // live before this object is + return { summary: self.derive([store.count], ([c]) => `${conversation}: ${c}`) }; + }); + host.serve(NotesView, (_, principal, self) => { // remote: one object per principal key, one variant + const store = self.get(Notes, {}); + if (principal.kind === "guest") return { variant: "reader", members: { count: store.count } }; + return { variant: "editor", members: { count: store.count, add: (args, ctx) => store.add(args, ctx) } }; + }); +}); + +const host = createHost({ + principal: { kind: "server" }, // what this host's own requests act as + onError: (error) => log(error), + closeTimeout: 30_000, + readyTimeout: 60_000, +}); +await host.load([{ name: "@app/notes", facet }], ctx); // every host-lifetime object live or failed (reported) +const notes = await host.get(Notes, {}, ctx); // live and hydrated; waits through transient retries +await notes.add({ text: "milk" }, ctx); +notes.release(); +const draft = host.hold(Draft, { conversation: "c1" }, ctx); // a reference at once: loading, then live +(await draft.state.watch(ctx)).start(async (call) => call.kind === "value" && render(call.value)); // loading, live, failed, ended +if (draft.state.value.kind === "failed") draft.retry(); // a Retry button +host.accept(link, { principal: { kind: "guest", session: "S1" } }, ctx); // a child's link, authenticated by its entry point +await host.close(ctx); // closes links, then natives with reason "close" +``` + +Addressable hosts and routes (slice 4a): a server routes holds of worker types to the worker of their address. + +```ts +export const Server = hostKind("server"); +export const Session = hostKind("session", { id: str() }); +export const Notes = remote("notes", { version: 1, params: { session: str() }, + placement: ({ session }) => Session({ id: session }), variants: { … } }); + +const server = createHost

({ address: Server(), principal: { kind: "server" }, onError }); +server.route(Session, { // before load, one route per kind + authorize: (principal, address) => principal.kind === "owner" || principal.session === address.params.id, + resolve: async (address, context) => { // once at a time per address; context aborted at close + // No `signal: context.abortSignal`: it aborts at host close and would kill the worker (an `error` on the child); + // the worker exits when its link closes. + const child = fork("worker.ts", [], { serialization: "advanced" }); + child.send({ id: address.params.id }); // the app's own start message + return { link: ipcLink(child), principal: { kind: "worker", session: address.params.id }, trusted: true }; + }, +}, ctx); // ctx: the parent of the route's work +await server.load(serverFacets, ctx); +wss.on("connection", (ws, req) => server.accept(webSocketLink(ws), { principal: authenticate(req) }, ctx)); + +// worker.ts: connect first (W6), then load +const worker = createHost

({ address: Session({ id }), principal: { kind: "worker", session: id }, onError }); +worker.connect(ipcLink(process), { principal: { kind: "server" }, trusted: true }, ctx); +await worker.load(workerFacets, ctx); + +// a tab: tab → server (relays, parses nothing) → route(Session) → worker +const notes = await tab.get(Notes, { session: "S1" }, ctx); +``` + +Authority and housekeeping (slice 4b): + +```ts +type Revocation = { grant: string } | { session: string }; +const server = createHost({ + address: Server(), principal: { kind: "server" }, onError, + revokedBy: (p) => (p.kind === "guest" ? [{ grant: p.grant }, { session: p.session }] : []), // the same on every host + compression: nodeCompression, // Node only: `src/next-node/compression.ts` +}); +server.revoke({ grant: "g1" }, ctx); // ends g1's objects here and on every worker, closes g1's links +host.serve(Notes, ({ session }, principal, self) => { // in a facet: `host.references` is the facet's host's too + const viewers = self.derive([host.references(Notes)], ([refs]) => refs.filter((r) => r.params.session === session).length); + … +}); +(await worker.activity.watch(ctx)).start(async (call) => call.kind === "value" && supervise(call.value)); // { references, calls, jobs, lingering } +server.accept(webSocketLink(ws), { principal, limits: { part: 16 * MiB, decoded: 64 * MiB } }, ctx); +``` + +- `createHost({ address?, principal?, principalKey?, telemetry?, onError, closeTimeout?, readyTimeout?, + heartbeat?, revokedBy?, compression? })`; `R` is the app's revocation type (JSON). `address` (`Session({ id: "S1" })`, see `contracts/`'s `hostKind`) makes the host addressable: it serves only + objects placed there; without one it is a client. A host with a `principal` acts as it for its own requests; a host + without one (a client) has none and serves only local types (`serve` of a remote type throws at `load`). + `principalKey` maps a principal + to its part of an identity (L1's canonical JSON by default; the same function on every host). `closeTimeout` (30 s) + bounds each native's close and the drain of an ending object's calls; `readyTimeout` (60 s) bounds every activation + attempt (its wait for the previous instance's teardown included) and is the default deadline of a `get`. `heartbeat: { ping?, close? }`: a link silent for `ping` ms (15 s) + sends a `ping`, one silent for `close` ms (30 s), its handshake included, is closed (its references go `loading`); + one interval timer per host (`min(ping, close) / 10`), running only while it has links. A handler that blocks the + event loop longer than `close` gets its links closed. +- `revokedBy(principal) => R[]`: what revokes a principal; its revocation keys are the canonical JSON of each value, + computed once per principal object (cached weakly). `host.revoke(r, ctx)`, synchronously: every remote object whose + principal has `r` among its keys ends at once, whatever references remain (R11): each holder gets `forbidden` with + reason `revoked` (a UI can say "access revoked"; a hold over a link sends `gone` with it), running calls are aborted + with it (callers rejected), an activation in progress is abandoned; teardown follows as for any end (drain, closes + with reason `end`). Holds of the principal that arrived before the revoke but still wait (for `load`, or relayed for + their route's link) end too (`gone`). + Then every route link is sent `revoke { revocation }`, which the child applies with its own `revokedBy` (and passes + down its own route links), and every child link whose principal it revokes closes (a relay sends + `closed { origin }` upstream for it, after the `revoke`). Never this host's own principal, never the parent link; a + `revoke` arriving on any link but the parent is ignored; nothing is replayed to links made later. Revocation is an + end, not a blocklist: a later hold for the principal (a new authentication) binds a new object. Local objects have + no principal and are never revoked. +- `host.references(Type)` (also on a facet's `host`): a JSON watchable `{ params, principal?, incarnation }[]`, one entry per + reference to an object of `Type` here: a binding of this host (`host.get`, `self.get`, `ctx.get`) or a hold over a + link served here. `principal` is the reference's (absent for a local type); `incarnation` its caller's (a + tab's own through a relay, this host's for its own references; a `ctx.get` reference the call's). Entries are + added when a reference binds and removed when it ends, in the batch where that happens (a link drop is a departure + derives see at once); lingering objects have none. The list is made for the first `references(Type)` (with the + references existing then) and kept; while watched, a batch's changes are folded into the published list once (one + splice op per removal, one for the additions: a revoke of many holds is one list and one frame), otherwise it is + built on its next read. A departure discovered while a commit's links send (a transport failing on send) lands in + the next batch, so one commit stays one message per link and origin. +- `host.activity`: a JSON watchable `{ references, calls, jobs, lingering }`: holds arriving over links and served + here, method calls running here (local and remote), job runs running or finishing here, objects lingering after their + last reference. Counted where the work happens; the changes of one microtask turn are one publication + (`[["r", counts]]`), a slow watcher gets the newest counts; nothing is scheduled while nobody watches or derives from + it. Chord counts; the app's supervisor decides. +- `compression`: a synchronous `Compression` (`{ deflate(bytes), inflate(bytes, limit) }`, see `link/`): this host + deflates each JSON part above 16 KiB (UTF-8) it originates, whatever the link (IPC included), only if it shrinks + (values, ops, results, arguments; never params, never arguments sent up its parent link) and inflates deflated parts + it owns in place. Child links refuse compressed parts (a protocol error: clients never send them, and a small part + could inflate to 64 MiB). Without one (a browser) it sends uncompressed and inflates asynchronously + (`DecompressionStream`), only parts it owns once their id and placement are known and their wire size passed, + within the smaller of its link's decoded limit and a hold's stamped one, holding later messages of the link in + order; acks go out after a message was applied, so a slow inflate is backpressure like a slow consumer. +- **Cycles across hosts (Q5):** every object instance gets an instance id (uuidv7) at its first attempt; a new object + for an identity is a new instance. A hold made during an activation carries the chain of instance ids waiting for it + (the activating object's chain, which it got from the hold that created it or from the object that declared it + here, plus its own id; the last 32); honoured from trusted links only, relays forward it as it is. A provider refuses + a hold whose chain names the instance it would bind while that activates: broken `invalid/cycle` naming the cycle by + label where known ("app.a(1) → … → app.a(1)"); a local `self.get` of an activation in the chain is refused the same + way. A dependency failing with `invalid/cycle` fails its dependent broken with that error (owner, an exception to + R18's transient dependency failures: a retry would meet the same cycle). Known limitation: a chain is fixed when its + object is created, so a cycle entering an object that is already activating (two activations started at the same + moment) is not detected: both fail `transient` at `readyTimeout` and retry with backoff. +- `onError(error: HostError)` gets everything that fails in the background, by `kind` (`failure` where the value is a + `FailureCause`, `error` otherwise; a position is `{ kind: "on" | "onLive" | "native", position }`, from 1): + `activation_failed` (`object`, `failure`: every failed attempt), `object_failed` (`object`, `failure`: a live object + failed, slice 6a), `storage_failed` (`error`: the host's store stopped; the host closes itself), `derive_failed` + (`object`, `member?`, `error`: a derive's compute threw, once per failure episode, watched or not; `member` when it is + an exposed state member), `close_failed` (`object`, `native` position, `error`: a close that threw while nobody + waited), `close_timeout` (`object`, `native`: a close still running at its `closeTimeout`, a possible leak), `drain_timeout` (`object`: calls still running at `closeTimeout`; their natives close anyway), + `listener_error` (a throwing watch listener), `delivery_failed` (a frame delivery or a link's flush that threw) and + `protocol_error` (`error`: what closed a link, see `link/`) and `resolve_failed` (`address`: its key, `error`: a + route's `resolve` rejected; the address waits its backoff), `routine_failed` (`object`, `routine`: its position, + `error`: a routine call threw, or its watch could not be taken; a source's failure calls never reach a routine), `job_failed` (`object`, `job`, + `id`, `error`: a run threw, not after an abort), `job_finish_failed` (the same: a `finish` threw; it is retried) and + `job_dropped` (`object`, `job`, `id`: a blocked run dropped without `finish`, by `abort` or its object's end). A + throwing `onError` is ignored. +- `host.load([{ name, facet }], ctx)`: once per host; runs every setup, then activates every type with empty params + and resolves when each has had its first outcome, live or failed (the start barrier; later retries never hold it). + A setup that throws rejects it with that error and closes the host as `close` does, its links included. Gets wait + for it. +- `host.hold(Type, params, ctx, { readyTimeout? })`: a reference at once (`Held`: its `variant` is `undefined` until + its first binding), which follows its object until released and never rejects: `loading` while `load` runs, while + the object activates or retries, while there is no parent link or it is down; `failed` while the object is failed + and no attempt runs; `live`. Throws at once for invalid params (`invalid/params`) and a local type not served here + (`unavailable`); before `load` resolved + these are decided once it did, and the reference fails `broken` with that error. A placement or `authorize` that + throws or refuses, or a placement whose params fail its kind's check (`invalid/placement`), fails the reference + `broken`. A local type never crosses a link. Where a reference made here (`host.get`/`hold`, `self.get`, `ctx.get`) + or a hold arriving on a link goes, first match (`Placement`; the address is the contract's `placement` of the params, + computed by the caller; a hold carries it to be routed, and a host serving the type computes it again): + + | Case | Goes | + | --- | --- | + | the type is served here and its address is this host's, or the host or the type has none | here | + | a reference made here to a served type whose address is another host's | on, as if not served here (rows below): worker S1 reaches `Notes { session: "S2" }` through the parent | + | an arriving hold for a served type whose params place it on another host | broken `invalid/placement` (misrouted; never forwarded) | + | the address is of a kind this host has a route for | over that route's link (the params checked against the kind, `authorize` first; `forbidden` when it refuses) | + | otherwise (a remote type) | over the parent link, open yet or not; a hold that arrived *on* the parent link is never sent back: `unavailable` | + | an arriving hold and no parent link | `unavailable` | + + A hold that arrives and goes on is relayed (`remote/`): this host holds it upstream as the arriving hold's caller + and forwards between the two without parsing params, arguments, results or state. +- `host.get(Type, params, ctx, { readyTimeout? })`: a hold, awaited until live. Each getter has one deadline for its + whole wait, `load` and reconnects included. A broken failure rejects it at once with its error; through transient + failures and drops it waits, and at its deadline rejects with the last failure's error (`unavailable` "did not + become live within …" when none failed yet). A getter whose deadline passes or whose context is cancelled releases + only its own reference. +- `host.serve(Type, factory, { linger? })` in a facet's setup: a live object with params whose last reference is + released stays live for `linger` ms (0 by default), with its documents and dependencies; a reference within that + time reuses it, otherwise it ends. An object waiting, activating or failed when its last reference goes ends at once; + a failed state member does not end a lingering object; `host.close` ends lingering objects at once. +- `limits: { part?, decoded? }` on `accept`, `connect` and a route's `resolve` result: per part of what arrives on the + link, its size on the wire and after inflating (none on trusted links, 16 MiB and 64 MiB otherwise, unless set). + Checked before a part is parsed or inflated; past a limit only its hold or call fails (`too_large/part`: an argument + its call, a result its call, a snapshot or a state part its hold, which the consumer releases at the provider; that + failure is final: the reference leaves its path, a reconnect holds nothing again, and only `ref.retry()` places it + anew and holds it with a new id), the link stays. A relay + checks `part` on the admitting link and stamps that link's decoded limit on the upstream hold (`limit`, honoured + only on a parent link); calls on the hold inherit it, and the host that inflates enforces the smaller of its own and + the stamped one while inflating. A relay does not check answers from upstream: the end that parses a part checks it + against its own link's limits. Limits count UTF-8 bytes; byte segments are no parts (the transport's cap bounds + them). +- `host.accept(link, { principal, trusted?, codec? }, ctx)`: a link from a child (`codec`: `binaryCodec` by default, + the same at both ends; `trusted` skips the message schema check, see `link/`). Every hold and call arriving on it + acts as `principal`, stamped by this host (principal fields on its holds are ignored); its calls get the child's + incarnation as `ctx.incarnation`. `ctx` is the parent of the work arriving on it (its cancellation is dropped: a + link lives until closed). Every link carries both directions: this host may hold types over it too. +- `host.connect(link, { principal, trusted?, codec? }, ctx)`: the parent link. It throws while the previous one is + open (also while it is still closing); once that closed (a drop), `connect` takes the next. Every remote reference + of the parent path that is not released is held on it again, with this link's own ids (see `remote/`). `principal` + is who the parent is; holds arriving on it may name whom they act for (`principal`, `incarnation`), which this host + honours. Connecting before `load` lets factories that reference remote types go live without waiting (W6). +- `host.route(Kind, { authorize, resolve }, ctx)`, before `load`, once per kind (`ctx`: the parent of its `resolve`s + and of the work arriving on its links, with their telemetry; aborted for them at host close): how this host reaches + hosts of `Kind`. + `authorize(principal, address)` runs for every hold routed over it with the hold's principal, before anything + starts (`false`: broken `forbidden`; a throw: broken with that error). `resolve(address, context)` returns `{ link, + principal, trusted?, codec? }`, the address's link (role `route`); it runs at most once at a time per address, later + holds wait for it, and `context` is aborted only when the host closes (a link it returns later is closed at once). A + `resolve` whose holds were all released meanwhile finishes and its link is kept. A rejection (`resolve_failed`), a + link closed already, or the address's link closing while holds over it exist starts the address's backoff (1 s + doubling to 60 s, separate from objects' identities); the holds stay `loading` and resolve again after it while they + exist. Only a link that completed its handshake resets the backoff. A `resolve` returns the link before the worker + loaded (the worker connects first): waiting for its `load` could deadlock. Chord never closes a route link except + at `host.close`. When this host holds over a route link it sends whom each hold acts for (the table in `remote/`): + a guest's handler that `ctx.get`s a worker type reaches the worker as the guest. +- `host.close(ctx)`: aborts every route's `resolve`, closes its links first, children, routes, then the parent (no + `gone` goes out), ends every remote reference (`released/closed`, parked calls rejected with it), then ends every + object at once (calls fail `released/ended`, `self.tx` throws; documents stay readable until it is done), aborts pending opens, waits for endings already running, then drains and + closes object by object, the last to go live first (a dependent before its dependencies), with reason `close`. + Every call returns the same closing. It rejects with an `AggregateError` of the closes that threw. It does not wait + for an open that ignores its abort: that value is closed when it arrives (a throw is `close_failed`). +- `defineFacet

(setup)`: `setup(host)` declares only, synchronously: `host.serve(Type, factory, options?)` and + `host.references(Type)`. Facets have no documents, routines, natives or jobs (slice 6a): state and work belong to + objects. A host for `P` takes only facets typed for exactly `P`. +- `createHost({ store?, ... })` (slice 6a): the host's store (`openStore` in `store/`), which the app opens, gives to + one open host at a time (`invalid/args` otherwise) and closes after it. A backend error stops the store: the host + reports `storage_failed` once and closes itself. A `local` type with empty params may then declare + `self.table(definition)` (once per table name) and persisted documents (`defineState(..., { persist: true, + checkpointWhen })`); the types refuse both elsewhere (P3), and so does the factory at runtime. Its activation + loads the persisted singletons and pinned keys, then creates or migrates its tables (the `readyTimeout` deadline + paused meanwhile, resumed with its remaining budget), then binds its references. A table or document the code + cannot read (a newer version, a history edited in place, a failed `rows` step or `migrate`) fails the object + `broken`; a live object whose read fails so fails while live (`object_failed`), keeping its documents for the + next attempt. +- A factory `(params, self) => implementation` declares with `self.state`, `self.derive`, `self.get`, `self.open`, + `self.on`, `self.onLive` and `self.job` (`jobs/`), and writes with `self.tx`. It runs once per attempt. The implementation has exactly the contract's members: state + members are JSON watchables (document handles, derives, reference members, constants), methods `(args, ctx) => + Promise`, or `(ctx) => Promise` for a method without arguments (the reference omits them too: + `notes.clear(ctx)`). A remote factory `(params, principal, self) => ({ variant, members })` gets the principal its + object is for and returns one variant with exactly that variant's members. +- **Routines** (§14): `self.on(source, fn)` and `self.onLive(fn)` start right after the object went live, in the same + turn (a method call admitted in that turn may run before the first `on` call), and stop when it ends: their ctx is + aborted with the end's error and the drain waits for their running calls (each admitted before its handler runs: a + handler may end its own object before its first await). Once the object ended no further call starts, and a routine + that ends its object as it starts (an `onLive` releasing the last reference) starts none after it. Each call gets its + own ctx (the object's principal for a remote type, the host's for a local one; the host's incarnation; `ctx.get` + released when the call settles). They are not counted in `host.activity`. + - `on` over a JSON watchable (one whose watch is Chord's shared `Watch`: a document, a derive, a reference member, a + constant, a Durable view) calls `fn(call, ctx)` with the value call `{ kind: "value", value, ops, retired }`; over a + native watchable (whose watch is the shared `NativeWatch`: `ref.state`, a registry value) `fn({ kind: "value", + value }, ctx)`, as the overloads type it. Failure calls never reach `fn`: the routine waits for the next value (the + failure is reported where it happens, a derive's as `derive_failed`). The first call + gets the current value as `[["r", value]]`, then one call per change; calls never overlap, every change made during + a call (the first one and retries included) folds into one next call (the newest value; for JSON its folded ops, + and `retired` when any change was a retirement), made one macrotask after the busy call so every value its watch + still buffers is taken first (a native watch replays them one per microtask), and none aborts it. A call that + throws is reported (`routine_failed`) and the backoff starts, also when the call changed the source (a handler + writing its own source and throwing never feeds itself): after 1 s, doubling to 60 s, it calls with the newest + value, or the last call's, as a whole value, `retired` again if the failed call had it; a change during the backoff + calls at once. Any call cancels a pending retry, and a call that succeeds resets the backoff. The source ending (a + document of an object that ended, a released reference) ends the routine without a report; a watch that cannot be + taken ends it with one report (`routine_failed`). A retirement (a Durable document's; + Chord's own documents retire this way from slice 6a) is a call with `retired: true`, and the routine goes on. + - `onLive(fn)` is called once: returning ends it, a throw is reported and restarts it after the same backoff (reset + after a run of at least a minute); an abort (the object ends) is no error. + - `sleep(ms, ctx)` resolves after `ms` and rejects with the abort reason once `ctx` is aborted, clearing its timer and + listener (delays above 2^31-1 ms are chained timers); `untilAborted(ctx)` resolves once `ctx` is aborted. Routines + and handlers wait only through calls that take their ctx. +- `ctx` (`Ctx

`) in a handler: `principal` (the caller as its hold's link stamped it, or as a parent link's hold + named it; on a local object's handler the host's), `incarnation` (the calling host's over a link, the origin's + through a relay, this host's otherwise, R20), `get(Type, params, { readyTimeout? })` (a reference as + `ctx.principal` sees it, with `ctx.incarnation`, also over a route; released when the handler settles; afterwards + `get` throws `released`), the abort signal (the caller cancelled or released, or the object ends) and telemetry. +- `self.open(open, close, { closeTimeout? })` returns a `Native`: its `value` is readable from the moment its open + resolved until its close starts; before, it throws `not_ready`, after, `released`. `open(ctx)` gets a context + aborted when the attempt is abandoned; `close(value, reason, ctx)` gets `{ kind: "end" | "close" | "reload" }` (end: + last reference gone or a failed attempt; close: `host.close`; reload: slice 7) and a context aborted at its + `closeTimeout`. Throw `broken(error)` from an open when retrying is pointless. +- A reference has the contract's members, `state`, `retry()` and `release()`. A remote reference has `variant`, its + discriminant (`RefIn` narrows the type), `undefined` before its first binding and the last bound variant while + loading or failed; at runtime it carries every member of every variant, and one outside the bound variant fails + `invalid/member`. +- `ref.state` (a `NativeWatchable`, made on first access): `{ kind: "loading" } | { kind: "live" } | { kind: + "failed"; cause; retryAt }` (`retryAt`: when the object's backoff window ends, `undefined` for a broken failure) `| + { kind: "ended"; error }` (for good: `released` without a reason when its holder released it, `released/ended` when + its object ended, `released/closed` at host close, `forbidden/revoked`). A watch gets the current state, then every + change; it ends `ended` when the reference ends. What a + reference does in each state: + + | State | Calls | State members | + | --- | --- | --- | + | `loading` | park: sent in the order made once live, each until the reference's `readyTimeout` (`unavailable/park_timeout`) | the last value; before the first binding a read throws `not_ready` and a `watch` waits for the first value | + | `live` | run | readable | + | `failed` | reject at once with the cause's error | a local object that went live once (it failed live, or a retry failed): the last values, a new `watch` opens at them, no frames until a new attempt is live; otherwise reads throw the cause's error and existing watches stay open | + | `ended` | throw its `error` | reads throw `released`; watches end `ended` | + + Calls made after the reference went live never overtake queued ones (a call a handler makes while they are sent + queues behind them). A parked call is checked against its deadline when it is about to be sent, and against the + variant it is sent on; it also leaves the queue when its caller cancels (rejected with the reason), the reference + fails (the cause's error) or is released (`released`). A `host.hold` released before `load` resolved makes nothing. +- A local reference follows its object's state members through member sources (`member-source.ts`, one per object and + member): when a new attempt goes live every member re-binds by name to it (watches get one frame from the last + value, other objects' derives over the member recompute, a failed one included), as a remote reference's re-hold + does. The keys a reference holds on a keyed member (`at(key)`, `watch(key)`) follow the object the same way + (`keyed-members.ts`, one source per member and key; the new attempt's member loads them before it goes live). A key + whose first load is still in flight when its object fails or ends is given up: the activation waiting on it fails + `transient` (R18) and retries on the next attempt; the late pin is released once. +- `ref.retry()`: a failed object (broken too) gets an attempt now, its backoff window skipped; every reference to it + goes `loading`. Ignored while an attempt runs, once live and once released; over a link it is a `retry` message. + +Rules: + +- One object per (type, params), per (type, params, principal key) for a remote type. A local type with empty params + lives from `load` to `close`, failed or not; every other type from its first reference (a get, a hold over a link) + to its last release, failed or not. `load` never activates remote types. +- **Principals of same-host references:** `host.get` acts as the host's principal; `ctx.get` as `ctx.principal`; + `self.get` in a remote object's factory as that object's principal, in a local object's factory as the host's. Over + a route link they travel with the hold; over the parent link the parent acts as this host (upward is a boundary: a + worker that acts for a user towards the server passes the user as data and the server object decides). +- **Activation**, one attempt: the factory; singleton documents hydrated; references bound and live; natives opened one + after another in declaration order (an opener may read documents, references and earlier natives, not derives); + members checked; derives computed; live. An attempt not live within `readyTimeout` is abandoned: its openers' + contexts are aborted, what it opened is closed (`end`), an open resolving later is closed at once. +- **Failures** become a `FailureCause` with a ChordError; the kind comes from where it failed: the factory, a + singleton's `initial()`, the member check (a native watchable as state included), a first derive compute or a + dependency cycle are `broken`; a throwing open is `transient` unless it threw `broken(error)`; the `readyTimeout` is + `transient`; a dependency that failed (R18) is `transient` with `unavailable` "A: dependency B failed" and B's error + as `cause`. A keyed document's `initial()` runs in a later transaction and fails only that transaction. A cycle + (`a(1) → b(1) → a(1)`) fails every object on it. +- **Retries:** one attempt per object at a time. A transient failure is retried after a backoff per identity: 1 s, + doubling to 60 s, reset by a success; a new object for an identity inside its window waits for the window's end. + A broken failure is retried only by `ref.retry()` (and reload, slice 7, `retryOn`, slice 8); a new object after its + end tries at once. Only a dependency failure, or a `readyTimeout` while waiting for dependencies, keeps the object's + references; when that dependency goes live (a remote one: bound again after a drop) the object is retried at once. A dependency failure is transient whatever the dependency's kind: a dependent of a broken dependency is + retried with its backoff for as long as it is held. A retry binds its references before the previous attempt's are + released, so a dependency held only by the retrying object never ends in between. +- **Ending:** transactions refused and the callers of running calls rejected `released` at once, their handlers' + contexts aborted, routines' and job runs' too (their waiters rejected); then the drain (the handlers', routine + calls' and runs' own promises, a run's `finish` included, up to `closeTimeout`), the natives closed in reverse + (each up to its `closeTimeout`; a throw never stops the others), the documents (readable until then) dropped, the + derives detached, the references released, every watch on its members ended (`ended`). A new instance or attempt + for the identity starts only after that; gets wait. +- Calls check the method against the bound variant first (`invalid/member`), then the arguments against the provider's + current version (`invalid/member`, `invalid/args`); on one host arguments and results pass by reference. +- **Release** rejects the calls made through the reference with `released` and aborts their handlers' contexts, on one + host as over a link. A get needs the provider's version and fingerprint (`contract_mismatch`); an unserved + type is `unavailable`, a closed host `released`. +- Watches taken through a reference end when it is released, even while others keep the object alive. +- A failure's error keeps its stack frames' receivers alive until its `stack` is read (V8), its failed attempt among + them (natives closed, documents dropped). An object keeps its last failure's error until it goes live or ends; an + app logs the errors `onError` gets and drops them. + +Patterns for natives (documentation, nothing to build). Restarting a native with work in flight (R16): a restart +(reload, slice 7) closes the old native, then opens the new one, one after the other. Two patterns keep in-flight work +alive: + +- **Drain in the background:** `close` detaches the old resource and returns at once; the resource disposes of itself + when its work has finished, or after the native's `closeTimeout` at most (code mode's sandbox). Chord then opens the + new one immediately; callers look the current resource up when they start (`watcher.value` at each use). +- **Swap through a registry** (slice 8): the resource is a registry contribution; readers always find the current one, + and a reader that must wait for one waits, cancellably, until one is present. + +A native that follows a relocation (design §6) is not restarted at all: the native is a stable manager opened once; a +routine on the session location (slice 5) switches the resource it manages (the watched directory) and bumps a +generation, so late events from the old location are dropped. + +Runnable examples, one topic each, in `test/next/examples/` (run from `packages/chord` with +`node --conditions=source --experimental-strip-types test/next/examples/`): `01-objects.ts` (contracts, facets, +references, calls, lifetimes), `02-documents.ts` (singleton and keyed documents, transactions, shorthands), +`03-derives.ts` (derives, `mapShared`, derives over other objects), `04-watches.ts` (frames, folding for a busy +listener, retire, cancellation, release), `05-errors.ts` (invalid arguments and params, unknown members, contract mismatch, released references, +cycles, throwing factories), `06-natives.ts` (opening in order, `not_ready`, close reasons, throwing and slow closes), +`07-failures.ts` (transient and broken failures, backoff, dependency recovery, `readyTimeout`, the start barrier), +`08-remote.ts` (two hosts, variants per principal over a shared store, replicas), `09-calls.ts` (calls over a link, +bytes, errors, `ctx.principal`, cancellation, release), `10-drops.ts` (a hold before any link, `ref.state`, parked +calls, a drop and the re-hold), `11-linger-and-retry.ts` (`linger`, failed references, `ref.retry()`), +`12-routes.ts` (addresses, placement, a route to an in-process worker, a guest's `ctx.get` through it), +`13-relay.ts` (tab, server and a forked worker over WebSocket and IPC, the worker's start data through its core +facet, §18), `14-flow-control.ts` (a slow tab: folding at the provider, and at the worker per origin through a +relay while a fast tab keeps receiving), `15-revocation.ts` (a grant revoked: its tabs' links closed, its objects on +the worker ended), `16-presence-and-activity.ts` (a viewer count from `host.references`, departures, the activity a +supervisor watches), `17-limits-and-compression.ts` (a 500 KB snapshot compressed or not, a part past a limit, a +compression bomb), `18-routines.ts` (a reconciling `on` with a retry, a periodic `onLive` with `sleep`) and +`19-jobs.ts` (a shell job with a throttled state document, per-principal views of `running`, an abort and a chained +job), and (slice 6a) `20-tables.ts` (tables, writes, pages, counts), `21-persisted-documents.ts` (documents across two +hosts on one store file), `22-migrations.ts` (`evolveTable` and an eager migration at activation), +`23-keyed-watches.ts` (`watch(key)`, `at(key)`, the `retired` flag), `24-keyed-members.ts` (a keyed member pinned per +key by its consumers) and `25-export-import.ts` (an archive in JSONL and an import). + +## For Chord's layers (`internal.ts`) + +For `remote/`: `ChordHost` (`place` decides where an arriving hold goes, here or relayed over a path; `serving` +finds the object a hold placed here binds to; `relays` and `relayOrigin`, the origins this host relays holds for by +their number; `pendingLoad`, `closing`, `linkOpened`, `unlink`), `AnyHost` (a host whatever its principal type: `ChordHost

` is invariant in `P`), +`Target` (what a binding is bound to, implemented by `ChordObject` and by `remote/`'s `RemoteObject`), +`ChordObject` (`state`, `retry`) and `Holder` (what holds an object: a binding, or a hold over a link: `moved()` when +the object's `state` changed, which a new holder also reads itself; `end(error)`), `Binding` (`moved`, `end`), +`CheckedParams`, `Caller` (principal and incarnation of a reference's maker or a hold's origin), `RefState` with `LOADING`, `LIVE`, `RELEASED`, and +`settlement`. Also on `ChordHost` for `remote/`: `revoke` (a parent's `revoke` message), `compression`, `count(kind, +delta)` (the keep-alive counts), `cycle(chain, id, closing)` (the `invalid/cycle` error naming a chain by labels), +`unplaced` (the path of remote references failed for a part past the limits) and `pathOf(Type, params, principal)` +(placing one anew on `retry`). + +Inside the directory: `ChordHost` (types, objects by identity, facets, the batch, the backoff map, the endings new +instances wait for, its links and the heartbeat timer, its routes and the parent path; `reference` checks a +reference's route, `bind` binds it to an object here or a remote object over a path, `acquire` gets one for a +caller), `Placement` (where a reference or an arriving hold goes: here, a route's path or the parent's; whether a hold +served here is placed here), `Route` (one per routed kind: per address its `Path`, at most one running +`resolve`, the backoff per address; an address is kept while it has holders, a link or a running `resolve`), +`ChordObject` (one per identity: its state across attempts, holders, retries, cycle checks with its instance id and +the chain it was created for, lingering, ending and revocation, and the count of its running and finishing job runs, +which keeps it like a reference: `countRuns`), `Activity` (`host.activity`: counters, one +publication per microtask turn), `References` (one type's presence list), `RevocationKeys` (each principal's keys, +in a `WeakMap`), +`Attempt` (one activation: it is the +factory's `self`, runs the steps with their failure kinds, takes the calls, starts its routines and jobs once live, +drains and closes), `Routines` (an owner's routines: declared, then started once for a `RoutineOwner`; each `on` +follows its source with a watch whose listener never throws and never waits: it keeps the newest frame, JSON ops +folded by Delta's `PendingOps`, and the routine makes one call at a time from it, the first call and retries +included; its state is tagged: starting, idle, calling, backoff with its timer, ended), +`sleep`, `NativeHandle` (one +native: opened once, closed once, a late value closed at once; `within` bounds a close or a drain), `Backoff` (the +windows per identity and when each ends, holding no object), `Binding` (a reference's link to its object or remote +object: its state, the object that declared it, its per-reference watches, calls, parked calls and the waits for a +state; released, it keeps only its name), `ParkedCalls` (a reference's queue of calls made while loading, a timer +each), `ReferenceState` (`ref.state`: its native watches, told of each change), `CallContext` (a handler's +`ctx`), `withLinkedAbortSignal` (a call's or a get's context aborted by the caller and by a long-lived signal, its +reference's or its attempt's, linked only until it settles: `AbortSignal.any` keeps cleanup entries per call while its +sources live before Node 24.16.0), `checkImplementation` (the variant and member check), `setUp` (runs one facet's setup) and +`resolveWatchable` (the member check, Q14), and `nextDelay` (the 1 s to 60 s doubling every backoff shares). Cycle checks follow the waits of activating objects with a visited set, +linear in the waits. diff --git a/packages/chord/src/next/host/activity.ts b/packages/chord/src/next/host/activity.ts new file mode 100644 index 000000000..badc5a609 --- /dev/null +++ b/packages/chord/src/next/host/activity.ts @@ -0,0 +1,56 @@ +// Keep-alive counts (§3, R26): `host.activity`, a JSON watchable of the work keeping a host busy, for the app's +// supervisor. Counters are kept where the work happens; their changes in one microtask turn are one publication. + +import { BACKGROUND_CONTEXT, type Context } from "../../context/index.ts"; +import type { JsonValue } from "../../json.ts"; +import { Source } from "../state/internal.ts"; + +/** What `host.activity` counts. */ +export type ActivityCounts = { + /** Holds arriving over links and served here. */ + readonly references: number; + /** Method calls running on this host, local and remote. */ + readonly calls: number; + /** Running jobs (slice 5). */ + readonly jobs: number; + /** Objects kept live by their type's `linger` after their last reference. */ + readonly lingering: number; +}; + +/** + * `host.activity`. A count marks it changed; while it is watched or derived from, the first change of a microtask turn + * schedules one publication, a batch that gives its watches the counts as they are then (`[["r", counts]]`: a slow + * watch gets the newest counts) and marks derives over it. Its value is built from the counters when read or + * published. + */ +export class Activity extends Source { + readonly #counts = { references: 0, calls: 0, jobs: 0, lingering: 0 }; + /** The counts as last read, until the next change. */ + #value: ActivityCounts | undefined; + #scheduled = false; + + get value(): JsonValue { + this.#value ??= { ...this.#counts }; + return this.#value; + } + + /** + * Adds `delta` to the count of `kind`; published with the turn's other changes. Nothing is scheduled while nobody + * watches or derives from it: a later watch or derive reads the counts as they are then. + */ + count(kind: keyof ActivityCounts, delta: number): void { + this.#counts[kind] += delta; + this.#value = undefined; + if (this.#scheduled || (!this.watched && this.dependents.size === 0)) return; + this.#scheduled = true; + queueMicrotask(() => { + this.#scheduled = false; + this.batch.within(() => this.changed(), BACKGROUND_CONTEXT); + }); + } + + deliver(context: Context): void { + const value = this.value; + this.publish(value, [["r", value]], false, context); + } +} diff --git a/packages/chord/src/next/host/attempt.ts b/packages/chord/src/next/host/attempt.ts new file mode 100644 index 000000000..aa9071dbd --- /dev/null +++ b/packages/chord/src/next/host/attempt.ts @@ -0,0 +1,620 @@ +// Attempts (§6, §13, §14, §15): one activation of an object, a fresh instance each time. It is the `self` its factory +// declares with (handles to the object's documents and jobs, under its seal token); it runs the activation steps, +// each failing with its own kind; it takes the live object's calls and starts its routines and jobs; and it ends with +// its seal (slice 6a §1.3), the drain and the closes of its natives. + +import { awaitWithContext, type Context, withAbortSignal } from "../../context/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { NativeWatchable, Watchable } from "../../watch.ts"; +import type { Contract, Implementation, ParamsOf } from "../contracts/index.ts"; +import { contractData, type ShapeOf } from "../contracts/internal.ts"; +import { ChordError, type FailureCause, type Principal } from "../data/index.ts"; +import { asChordError, checkArgs, checkParams, invalid, openFailure } from "../data/internal.ts"; +import type { HandleOf, JsonObject, StateDefinition, Tx } from "../documents/index.ts"; +import { + AddressSource, + type AnyDefinition, + type KeyDeclarer, + type KeyedSlot, + type KeySource, + SealToken, +} from "../documents/internal.ts"; +import type { JobDefinition, JobHandle, JobImplementation } from "../jobs/index.ts"; +import { type AnyJobImplementation, Job } from "../jobs/internal.ts"; +import { Derive, type DeriveSource } from "../state/internal.ts"; +import type { Columns } from "../store/columns.ts"; +import { tableSqlName } from "../store/sql.ts"; +import type { ChordStore } from "../store/store.ts"; +import type { TableDefinition } from "../store/table.ts"; +import { type Table, TableHandle } from "../store/table-handle.ts"; +import { prepareTables } from "../store/table-migration.ts"; +import { CallContext, type Caller, type Ctx, withLinkedAbortSignal } from "./ctx.ts"; +import { Deadline } from "./deadline.ts"; +import type { HostError } from "./host.ts"; +import { attachDerives, checkImplementation, type KeyedBinding, type Members } from "./members.ts"; +import { + type CloseFailed, + drained, + type Native, + type NativeCloseReason, + NativeHandle, + type NativeOptions, + type NativeSlot, +} from "./native.ts"; +import type { ChordObject } from "./object.ts"; +import type { Binding, CheckedParams, Reference, Target } from "./reference.ts"; +import { + type OnHandler, + type OnLiveHandler, + type OnNativeHandler, + type RoutineDeclarations, + Routines, +} from "./routines.ts"; + +/** + * What a factory declares with. Declarations (`state`, `derive`, `get`, `open`, `on`, `onLive`, `job`) are valid only + * while it runs. `P` is the app's principal type, which the ctx of routine calls and job runs carry. + */ +export interface Self

extends RoutineDeclarations

{ + /** This object's documents of `definition`, in memory for its life (across its attempts). */ + state>(definition: D): HandleOf; + /** A value computed from `sources` by the pure, synchronous `compute`. */ + derive[], T extends JsonValue>( + sources: S, + compute: (values: { readonly [I in keyof S]: S[I] extends Watchable ? V : never }) => T, + ): Watchable; + /** A reference to another object, live once this object is, released when it ends. */ + get(Type: C, params: ParamsOf): Reference; + /** + * A native, opened during activation after the references are live and the earlier natives are open, and closed + * in reverse when the attempt ends. `open` gets a context aborted when the attempt is abandoned; `close` one aborted + * at the close timeout. Throw `broken(error)` from `open` when retrying is pointless. + */ + open( + open: (context: Context) => Promise, + close: (value: T, reason: NativeCloseReason, context: Context) => void | Promise, + options?: NativeOptions, + ): Native; + /** Runs `fn` as one transaction of this object (§9). */ + tx(fn: (tx: Tx) => R | Promise, context: Context): Promise; + /** + * This object's job `definition` (§15), once per job name: `run` runs each started run outside transactions, + * `finish` records its outcome in one. Starts are taken once the object is live. + */ + job(definition: JobDefinition, implementation: JobImplementation): JobHandle; +} + +/** + * What the factory of a `local` type with empty params declares with: one object per host and type, so its data may + * persist (P3). `P` as for `Self`. + */ +export interface StoreSelf

extends Self

{ + /** This object's documents of `definition`: in memory for its life, or persisted in the host's store. */ + state>(definition: D): HandleOf; + /** This object's table `definition` in the host's store, once per table name. */ + table(definition: TableDefinition): Table; +} + +/** + * A factory: synchronous and declarative, run once per attempt. A remote one also gets the principal in its object's + * identity and returns one variant, `{ variant, members }`. Methods get a `Ctx` for the app's principal type `P`. + */ +export type Factory = ShapeOf["kind"] extends "local" + ? (params: ParamsOf, self: keyof ParamsOf extends never ? StoreSelf

: Self

) => Implementation> + : (params: ParamsOf, principal: P, self: Self

) => Implementation>; + +/** How an attempt's activation ended. `dependency_failed` (R18) keeps the attempt's references. */ +type Activation = + | { readonly kind: "live" } + | { readonly kind: "failed"; readonly cause: FailureCause } + | { readonly kind: "dependency_failed"; readonly cause: FailureCause } + | { readonly kind: "ended"; readonly error: ChordError }; + +/** The error of an object, or a getter, that waited `ms` without the object going live. */ +export function notLive(label: string, ms: number): ChordError { + return new ChordError({ code: "unavailable", message: `${label}: did not become live within ${ms} ms` }); +} + +/** + * One attempt of an object. Activation: factory, documents hydrated (the object's slots), references bound (before the + * previous attempt's are released) and live, natives opened one after another, members checked, derives computed; + * bounded by the host's `readyTimeout`. Ending: sealed at once (calls aborted, its transactions refused and the one + * holding the line discarded, its job invocations cut off), then the drain, the closes in reverse, derives detached; + * its object releases its references. Documents and job runs stay on the object. + */ +export class Attempt implements StoreSelf, KeyDeclarer { + readonly #object: ChordObject; + /** The object's context: the natives' closes run under it. */ + readonly #context: Context; + /** Aborted when the attempt is abandoned or ends: openers, activation's waits and calls run under its signal. */ + readonly #abort = new AbortController(); + /** The object's context under this attempt's signal. */ + readonly #opening: Context; + /** The seal token its handles and transactions carry (the seal fence). */ + readonly #token = new SealToken(); + /** The documents it declared, by name: "declared twice" is per attempt. */ + readonly #declared = new Map(); + /** Its factory's `at(key)` pins: the object holds them from its factory's return to the next one's (rule 6). */ + readonly #pins: (() => void)[] = []; + #declaring = false; + /** Set once it went live: its derives' failures are reported from then on (`derive_failed`). */ + #isLive = false; + /** The checked implementation and, for a remote object, its variant, once activation checked them. */ + #members: Members | undefined; + #variant: string | undefined; + readonly #derives: Derive[] = []; + readonly #routines = new Routines(); + readonly #jobs = new Map(); + /** The tables it declared, by name. */ + readonly #tables = new Map(); + /** References this attempt declared with `self.get` (params checked), bound during activation. */ + readonly #references: { readonly binding: Binding; readonly contract: Contract; readonly params: CheckedParams }[] = + []; + readonly #natives: NativeSlot[] = []; + /** + * Running calls' handlers, routine calls and job runs, each with its bookkeeping (call counts, `ctx.get` + * references): an ending attempt waits for them (the drain); the bookkeeping runs once, when its work settles or at + * teardown for work stuck past the drain (decision "The seal fence" (4)). One map and one reaction per piece of work: + * the call path costs no more than before the cleanups. + */ + readonly #running = new Map, (() => void) | undefined>(); + /** The previous attempt while it still holds its references (R18): released once this one has bound its own. */ + #previous: Attempt | undefined; + /** What activation returns once the attempt was abandoned: a timeout, a cycle through its object, or its end. */ + #abandoned: Exclude | undefined; + + constructor(object: ChordObject, previous: Attempt | undefined, context: Context) { + this.#object = object; + this.#previous = previous; + this.#context = context; + this.#opening = withAbortSignal(this.#abort.signal, context); + } + + state>(definition: D): HandleOf { + this.#assertDeclaring("state"); + const what = `state "${definition.name}"`; + if (definition.persist) this.#assertPersistable(what); + if (this.#declared.has(definition.name)) throw new Error(`${this.#object.label}: ${what} is declared twice`); + this.#declared.set(definition.name, definition); + return this.#object.documents.declare(definition, this.#token, this); + } + + /** `handle.at(key)`: pins `key` of `slot` while this attempt's factory's pins are held, and returns its source. */ + at(slot: KeyedSlot, key: string): KeySource { + this.#assertDeclaring("state(...).at"); + this.#pins.push(slot.pin(key, this.#opening)); + return slot.source(key); + } + + table(definition: TableDefinition): Table { + this.#assertDeclaring("table"); + const object = this.#object; + const { host, served, label } = object; + const store = this.#assertPersistable(`table "${definition.name}"`); + if (this.#tables.has(definition.name)) throw new Error(`${label}: table "${definition.name}" is declared twice`); + const sqlName = tableSqlName(host.address?.kind ?? "client", served.contract.name, definition.name); + const owner = { + label, + scope: object.documents, + assertLive: () => object.documents.assertLive(), + fail: (error: ChordError) => object.failLive(this, error), + }; + // The handle's rows are the definition's, typed by `C` and `K`. + const handle = new TableHandle(definition as unknown as TableDefinition, sqlName, store, owner); + this.#tables.set(definition.name, handle); + return handle as unknown as Table; + } + + derive[], T extends JsonValue>( + sources: S, + compute: (values: { readonly [I in keyof S]: S[I] extends Watchable ? V : never }) => T, + ): Watchable { + this.#assertDeclaring("derive"); + // The values are the sources' values, in order; the result is `T`. + const derive = new Derive(sources, compute as (values: readonly JsonValue[]) => T, this.#object.host.batch); + this.#derives.push(derive); + return derive as unknown as Watchable; + } + + get(Type: C, params: ParamsOf): Reference { + this.#assertDeclaring("get"); + const { host } = this.#object; + const checked = checkParams(Type, params, this.#opening); + const binding = host.reference(Type, this.#object, this.#caller(), {}); + this.#references.push({ binding, contract: Type, params: checked }); + // The binding was built from `Type`'s members. + return binding.reference as Reference; + } + + open( + open: (context: Context) => Promise, + close: (value: T, reason: NativeCloseReason, context: Context) => void | Promise, + options: NativeOptions = {}, + ): Native { + this.#assertDeclaring("open"); + const { host, label } = this.#object; + const closeTimeout = options.closeTimeout ?? host.timeouts.close; + const owner = { + object: label, + position: this.#natives.length + 1, + context: this.#context, + report: (error: HostError) => host.report(error), + }; + const native = new NativeHandle(open, close, closeTimeout, owner); + this.#natives.push(native); + return native; + } + + tx(fn: (tx: Tx) => R | Promise, context: Context): Promise { + return this.#object.documents.tx(fn, this.#token, context); + } + + on(source: Watchable, fn: OnHandler): void; + on(source: NativeWatchable, fn: OnNativeHandler): void; + on(source: Watchable | NativeWatchable, fn: (...args: never[]) => void | Promise): void { + this.#assertDeclaring("on"); + this.#routines.on(source, fn); + } + + onLive(fn: OnLiveHandler): void { + this.#assertDeclaring("onLive"); + this.#routines.onLive(fn); + } + + job(definition: JobDefinition, implementation: JobImplementation): JobHandle { + this.#assertDeclaring("job"); + const object = this.#object; + if (this.#jobs.has(definition.name)) { + throw new Error(`${object.label}: job "${definition.name}" is declared twice`); + } + const owner = { + host: object.host, + caller: this.#caller(), + context: this.#context, + signal: this.#abort.signal, + token: this.#token, + documents: object.documents, + admit: (running: Promise, cleanup?: () => void) => this.#admit(running, cleanup), + }; + // The job keeps its runs as JSON and calls the implementation with them; the handle types them again. + const records = object.jobRecords(definition as JobDefinition); + const job = new Job(records, implementation as unknown as AnyJobImplementation, owner); + this.#jobs.set(definition.name, job); + return job as unknown as JobHandle; + } + + /** + * Activates once `ending` (the previous instance's) has finished; `readyTimeout` bounds both (an attempt abandoned + * while it waits has opened nothing). + */ + async activate(ending: Promise | undefined): Promise { + const { ready } = this.#object.host.timeouts; + const deadline = new Deadline(ready, () => { + this.abandon({ kind: "failed", cause: { kind: "transient", error: notLive(this.#object.label, ready) } }); + }); + try { + if (ending !== undefined) { + try { + await awaitWithContext(ending, this.#opening); + } catch { + // An ending never rejects: only an abandonment (the deadline) ends this wait early. + return this.#abandoned!; + } + } + return await this.#activate(deadline); + } finally { + deadline.clear(); + } + } + + /** Abandons the activation with `outcome` (once): its waits stop and the openers' contexts are aborted. */ + abandon(outcome: Exclude): void { + if (this.#abandoned !== undefined) return; + this.#abandoned = outcome; + this.#abort.abort(outcome.kind === "ended" ? outcome.error : outcome.cause.error); + } + + /** The object went live: its jobs take starts and its routines start, in this turn; both stop when it is sealed. */ + live(): void { + this.#isLive = true; + for (const job of this.#jobs.values()) job.live(); + const { host, label } = this.#object; + this.#routines.start({ + label, + host, + caller: this.#caller(), + context: this.#opening, + admit: (running, cleanup) => this.#admit(running, cleanup), + }); + } + + /** The variant a remote object's factory chose; `undefined` for a local object. */ + get variant(): string | undefined { + return this.#variant; + } + + /** The state members, as Chord follows them. */ + get stateMembers(): ReadonlyMap { + return this.#members!.state; + } + + /** + * Calls a method for `caller`: a method outside the variant fails `invalid/member` first, then the arguments are + * checked against the current version (L1); the handler runs under a ctx aborted by the caller and by this attempt's + * end. Either rejects the caller at once; the handler runs on, its transactions refused, and the drain waits for it. + * Its `ctx.get` references are released when it settles. + */ + async call(method: string, args: unknown, context: Context, caller: Caller): Promise { + context.abortSignal?.throwIfAborted(); + const { host, served, label } = this.#object; + const handler = this.#members!.methods.get(method); + if (handler === undefined) throw invalid("member", `${label}${this.#inVariant()} has no method "${method}"`); + const checked = checkArgs(served.provider, method, args, served.contract.version, context); + const linked = withLinkedAbortSignal(this.#abort.signal, context); + const callContext = linked.context; + // A local object's handlers act as the host; a remote one's as their caller. + const principal = served.kind === "remote" ? caller.principal : host.principal; + const ctx = new CallContext(callContext, { principal, incarnation: caller.incarnation }, host); + // Admitted to the drain before the handler runs: the handler may end its own object before it returns. + const running = Promise.withResolvers(); + host.count("calls", 1); + this.#admit(running.promise, () => { + host.count("calls", -1); + linked.release(); + ctx.settle(); + }); + // Plugin code, not type-checked: a synchronous throw or a plain value becomes the promise. + (async () => handler(checked.args, ctx))().then(running.resolve, running.reject); + return awaitWithContext(running.promise, callContext); + } + + /** + * The keyed member `member`'s ultimate slot (slice 6a §12, rule 22: through re-exposures); an unknown one fails + * `invalid/member`. + */ + keyedMember(member: string): KeyedSlot { + const keyed = this.#members!.keyed.get(member); + if (keyed === undefined) throw invalid("member", `${this.#object.label} has no keyed member "${member}"`); + return keyed.kind === "own" ? keyed.slot : keyed.binding.keyedSlot(keyed.member); + } + + /** The keyed members as checked: what each is (`KeyedBinding`), for the keys its object holds across attempts. */ + get keyedMembers(): ReadonlyMap { + return this.#members!.keyed; + } + + /** + * Whether `source` (a state member) changes only through this object's own commits: one of this attempt's derives, + * or one of the object's documents. Another object's member source or document is foreign. + */ + owns(source: DeriveSource): boolean { + if (source instanceof Derive) return this.#derives.includes(source); + return source instanceof AddressSource && source.owner === this.#object.documents; + } + + /** The state member `member`, as Chord follows it; one outside the variant fails `invalid/member`. */ + member(member: string): DeriveSource { + const source = this.#members!.state.get(member); + if (source === undefined) { + throw invalid("member", `${this.#object.label}${this.#inVariant()} has no state member "${member}"`); + } + return source; + } + + /** + * Ends the attempt at once (the seal fence): calls, openers and routines aborted with `error` (their callers + * rejected with it), its transactions refused and the one holding the line discarded, its job invocations cut off; + * documents stay readable. + */ + seal(error: ChordError): void { + this.abandon({ kind: "ended", error }); + this.#object.documents.fence(this.#token); + for (const job of this.#jobs.values()) job.seal(error); + } + + /** + * Tears a sealed attempt down: waits for its running handlers, routine calls and runs up to the host's + * `closeTimeout` (the drain), then closes its natives in reverse with `reason`. Resolves with the closes that threw. + */ + async close(reason: NativeCloseReason): Promise { + const { host, label } = this.#object; + if ((await drained(this.#running.keys(), this.#running.size, host.timeouts.close)) === "timeout") { + host.report({ kind: "drain_timeout", object: label }); + } + // Work stuck past the drain keeps running, but no longer counts as a call or holds its `ctx.get` references. + for (const [running, cleanup] of this.#running) { + this.#running.delete(running); + cleanup?.(); + } + const failures: CloseFailed[] = []; + for (const native of [...this.#natives].reverse()) { + const failure = await native.close(reason); + if (failure !== undefined) failures.push(failure); + } + return failures; + } + + /** Detaches its derives (once their object no longer needs them): reads throw `released`, their watches end. */ + detach(): void { + const ended = this.#object.ended(); + for (const derive of this.#derives) derive.detach(ended); + } + + /** Releases its references, and the previous attempt's if this one never bound its own. */ + releaseReferences(): void { + this.#releasePrevious(); + for (const { binding } of this.#references) binding.release(); + } + + /** The activation steps; each catches its own failures with their kind (the table in slice 2's spec). */ + async #activate(deadline: Deadline): Promise { + let implementation: unknown; + let dependencies: Target[]; + try { + implementation = this.#declare(); + const { documents } = this.#object; + // The object holds the pins from now on. + documents.hydrate(this.#declared, this.#pins.splice(0), this.#opening); + const loading = documents.load(this.#opening); + if (loading !== undefined) await loading; + if (this.#tables.size > 0) await this.#prepareTables(deadline); + const { host } = this.#object; + dependencies = this.#references.map(({ binding, contract, params }) => + host.bind(binding, contract, params, this.#opening), + ); + for (const dependency of dependencies) this.#object.assertNoCycle(dependency, this.#opening); + } catch (error) { + return this.#broken(error); + } finally { + // Bound first: a dependency held only by the previous attempt never ends in between. + this.#releasePrevious(); + } + try { + await Promise.all(dependencies.map((dependency) => this.#object.waitFor(dependency, this.#opening))); + } catch (error) { + const abandoned = this.#abandoned; + // A timeout while waiting keeps the references (R18): a dependency that goes live later retries this object + // at once (a remote one whose link dropped and is held again). + if (abandoned?.kind === "failed" && abandoned.cause.kind === "transient") { + return { kind: "dependency_failed", cause: abandoned.cause }; + } + if (abandoned !== undefined) return abandoned; + // `waitFor` rejects with a ChordError naming the dependency, its error as the cause; or with a cycle's + // `invalid/cycle`, which fails this object broken too (proposal, slice 4b: an exception to R18, since every + // retry would meet the same cycle), keeping no references. + const failed = error as ChordError; + if (failed.code === "invalid" && failed.reason === "cycle") { + return { kind: "failed", cause: { kind: "broken", error: failed } }; + } + return { kind: "dependency_failed", cause: { kind: "transient", error: failed } }; + } + try { + // The keys the factory pinned on its dependencies' keyed members, loaded before this object goes live. + for (const { binding } of this.#references) { + const pinning = binding.pinKeys(this.#opening); + if (pinning !== undefined) await pinning; + } + } catch (error) { + // A pin load that failed its provider is a dependency failure (rule 6, R18). + const cause: FailureCause = { kind: "transient", error: asChordError(error, this.#object.label) }; + return this.#abandoned ?? { kind: "dependency_failed", cause }; + } + for (const native of this.#natives) { + // An attempt abandoned before this open (ended while it waited) closed its natives already: none opens late. + if (this.#abandoned !== undefined) return this.#abandoned; + try { + await awaitWithContext(native.open(this.#opening), this.#opening); + } catch (error) { + return this.#abandoned ?? { kind: "failed", cause: openFailure(error, this.#object.label) }; + } + } + try { + const checked = checkImplementation(this.#object.served.contract, implementation); + this.#members = checked.members; + this.#variant = checked.variant; + const { host, label } = this.#object; + attachDerives(this.#derives, this.#members.state, (member) => (error) => { + // During activation a throwing derive fails the attempt, which `activation_failed` reports alone. + if (!this.#isLive) return; + host.report({ kind: "derive_failed", object: label, ...(member === undefined ? {} : { member }), error }); + }); + for (const derive of this.#derives) void derive.value; + } catch (error) { + return this.#broken(error); + } + try { + // The keys references pin on the object's keyed members, loaded on this attempt's before it goes live. + const preparing = this.#object.prepareKeys(this.#members.keyed, this.#opening); + if (preparing !== undefined) await preparing; + } catch (error) { + const cause: FailureCause = { kind: "transient", error: asChordError(error, this.#object.label) }; + return this.#abandoned ?? { kind: "dependency_failed", cause }; + } + return this.#abandoned ?? { kind: "live" }; + } + + /** + * Brings the declared tables to their definitions before the references are bound (slice 6a §9): created fresh or + * migrated eagerly, with the `readyTimeout` deadline paused meanwhile (resumed with the budget it had left). + */ + async #prepareTables(deadline: Deadline): Promise { + const handles = [...this.#tables.values()]; + const tables = handles.map(({ definition, sql }) => ({ definition, table: sql.table })); + const fence = { admit: () => !this.#token.sealed && this.#abandoned === undefined, adopt: () => {} }; + deadline.pause(); + try { + await prepareTables(tables, handles[0]!.store, fence, this.#object.label, this.#opening); + } finally { + deadline.resume(); + } + } + + /** + * Runs the factory with declarations open; the member check establishes what it returned. A throwing factory + * releases its pins. + */ + #declare(): unknown { + this.#declaring = true; + try { + const { served, params, principal } = this.#object; + if (served.kind === "local") return served.factory(params, this); + // A remote object's identity has its principal. + return served.factory(params, principal!, this); + } catch (error) { + for (const release of this.#pins) release(); + throw error; + } finally { + this.#declaring = false; + } + } + + /** + * Persisted data needs a stable identity (P3: a `local` type with empty params, also checked by the types) and the + * host's store; returns the store. + */ + #assertPersistable(what: string): ChordStore { + const { host, served, label } = this.#object; + if (served.kind !== "local" || contractData(served.contract).params.fields.length > 0) { + throw new Error(`${label}: ${what}: persist needs a stable identity (a local type with empty params)`); + } + if (host.store === undefined) throw new Error(`${label}: ${what}: persist needs a store`); + return host.store; + } + + /** Adds `running` to the drain, with `cleanup` run once: when it settles, or at teardown if it never does. */ + #admit(running: Promise, cleanup?: () => void): void { + this.#running.set(running, cleanup); + // Out of the map already: the teardown ran the cleanup. + const settled = (): void => { + if (this.#running.delete(running)) cleanup?.(); + }; + running.then(settled, settled); + } + + /** Whom this object's own work acts for: a remote object as its principal, a local one as the host; from this host. */ + #caller(): Caller { + const { host, served, principal } = this.#object; + return { principal: served.kind === "remote" ? principal : host.principal, incarnation: host.incarnation }; + } + + /** ` (variant reader)` for a remote object, for errors. */ + #inVariant(): string { + return this.#variant === undefined ? "" : ` (variant ${this.#variant})`; + } + + #broken(error: unknown): Activation { + const cause: FailureCause = { kind: "broken", error: asChordError(error, this.#object.label) }; + return this.#abandoned ?? { kind: "failed", cause }; + } + + #releasePrevious(): void { + this.#previous?.releaseReferences(); + this.#previous = undefined; + } + + #assertDeclaring(declaration: string): void { + if (!this.#declaring) { + throw new Error(`${this.#object.label}: self.${declaration} is valid only in the factory`); + } + } +} diff --git a/packages/chord/src/next/host/backoff.ts b/packages/chord/src/next/host/backoff.ts new file mode 100644 index 000000000..bae0a91df --- /dev/null +++ b/packages/chord/src/next/host/backoff.ts @@ -0,0 +1,83 @@ +// Retry backoff per identity (§6): after a transient failure an identity waits 1 s, doubling up to 60 s. The entries +// hold no object, so a failed object is collected after its last reference while its window still runs. + +/** The delay after the first transient failure. */ +const FIRST_DELAY = 1_000; + +/** The longest delay. */ +const MAX_DELAY = 60_000; + +/** + * The delay after a failure: 1 s after the first (no `previous` delay), then the previous one doubled, up to 60 s. + * Identities, routines and job finishes back off alike. + */ +export function nextDelay(previous: number | undefined): number { + return previous === undefined ? FIRST_DELAY : Math.min(previous * 2, MAX_DELAY); +} + +/** + * One identity's backoff: the delay of its last window, when that window ends (`Date.now()` time), and the timer ending + * a window still running. + */ +type Entry = { readonly delay: number; readonly until: number; timer: ReturnType | undefined }; + +/** + * The host's backoff map. When a window ends, `due(identity)` runs: the host retries the identity's object, or + * deletes the entry when there is none. An entry is deleted on success, on a broken failure, and when its window has + * passed with nothing left to retry. + */ +export class Backoff { + readonly #entries = new Map(); + readonly #due: (identity: string) => void; + + constructor(due: (identity: string) => void) { + this.#due = due; + } + + /** Records a transient failure: a window of the previous delay doubled (1 s at first), then `due`. */ + failed(identity: string): void { + const previous = this.#entries.get(identity); + clearTimeout(previous?.timer); + const delay = nextDelay(previous?.delay); + const entry: Entry = { delay, until: Date.now() + delay, timer: undefined }; + entry.timer = setTimeout(() => { + entry.timer = undefined; + this.#due(identity); + }, delay); + this.#entries.set(identity, entry); + } + + /** Whether `identity` is inside a window: a new object for it waits for `due` before its first attempt. */ + waiting(identity: string): boolean { + return this.#entries.get(identity)?.timer !== undefined; + } + + /** When the window of `identity` ends, while one runs: a failed reference's `retryAt`. */ + until(identity: string): number | undefined { + const entry = this.#entries.get(identity); + return entry?.timer === undefined ? undefined : entry.until; + } + + /** Ends the window at once (its object is retried now, R18); the delay stays for the next failure. */ + skip(identity: string): void { + const entry = this.#entries.get(identity); + clearTimeout(entry?.timer); + if (entry !== undefined) entry.timer = undefined; + } + + /** Its object ended: an entry whose window has passed gets no outcome any more and goes. */ + ended(identity: string): void { + if (this.#entries.has(identity) && !this.waiting(identity)) this.#entries.delete(identity); + } + + /** Forgets `identity` (success, a broken failure, or a window that passed with nothing to retry). */ + delete(identity: string): void { + clearTimeout(this.#entries.get(identity)?.timer); + this.#entries.delete(identity); + } + + /** Forgets every identity (the host closes). */ + clear(): void { + for (const identity of [...this.#entries.keys()]) this.delete(identity); + } +} diff --git a/packages/chord/src/next/host/ctx.ts b/packages/chord/src/next/host/ctx.ts new file mode 100644 index 000000000..4eb44020b --- /dev/null +++ b/packages/chord/src/next/host/ctx.ts @@ -0,0 +1,132 @@ +// The `ctx` a method handler gets (§8): its call's context, who called, the calling host's incarnation, and +// call-scoped references; and the abort signal of a call's context, linked to long-lived sources until it settles. + +import { type Context, type ContextKey, withAbortSignal, withoutAbortSignal } from "../../context/index.ts"; +import type { Contract, ParamsOf } from "../contracts/index.ts"; +import { ChordError, type Principal } from "../data/index.ts"; +import type { AnyHost, GetOptions } from "./host.ts"; +import type { Binding, Reference } from "./reference.ts"; + +/** A handler's context. */ +export interface Ctx

extends Context { + /** The caller as its link stamped it; on a local object's handler the host's principal (none on a client). */ + readonly principal: P | undefined; + /** The calling host's incarnation over a link, this host's otherwise (R20). */ + readonly incarnation: string; + /** + * A reference as the caller (`principal`) sees it, released when the handler settles; afterwards `get` throws + * `released`. + */ + get(Type: C, params: ParamsOf, options?: GetOptions): Promise>; +} + +/** Who makes a call: the principal it acts as and its host's incarnation. */ +export type Caller = { readonly principal: Principal | undefined; readonly incarnation: string }; + +/** The ctx of one handler run: its references are released by `settle` when the handler settled. */ +export class CallContext implements Ctx { + readonly principal: Principal | undefined; + readonly incarnation: string; + readonly #context: Context; + readonly #host: AnyHost; + /** The references taken with `get`; `undefined` once the handler settled. */ + #references: Binding[] | undefined = []; + /** Aborted when the handler settles: a `get` still waiting gives up and releases its reference. By the first. */ + #settling: AbortController | undefined; + + constructor(context: Context, caller: Caller, host: AnyHost) { + this.#context = context; + this.principal = caller.principal; + this.incarnation = caller.incarnation; + this.#host = host; + } + + get abortSignal(): AbortSignal | undefined { + return this.#context.abortSignal; + } + + get telemetry(): Context["telemetry"] { + return this.#context.telemetry; + } + + value(key: ContextKey): T | undefined { + return this.#context.value(key); + } + + toString(): string { + return `${this.#context}.Ctx`; + } + + async get(Type: C, params: ParamsOf, options: GetOptions = {}): Promise> { + // An ended handler creates nothing: its call's abort signal is checked before anything else. + this.#context.abortSignal?.throwIfAborted(); + if (this.#references === undefined) throw settled(Type); + // Made for the caller: its principal and its origin's incarnation travel with the reference's holds. + const caller = { principal: this.principal, incarnation: this.incarnation }; + this.#settling ??= new AbortController(); + const scoped = withLinkedAbortSignal(this.#settling.signal, this.#context); + let binding: Binding; + try { + binding = await this.#host.acquire(Type, params, caller, scoped.context, options); + } catch (error) { + throw this.#references === undefined ? settled(Type) : error; + } finally { + scoped.release(); + } + // A get that finished in the turn its handler settled: nobody releases its reference later. + if (this.#references === undefined) { + binding.release(); + throw settled(Type); + } + this.#references.push(binding); + // The binding was built from `Type`'s members. + return binding.reference as Reference; + } + + /** The handler settled: its references are released, waiting gets give up, and `get` throws from now on. */ + settle(): void { + const references = this.#references ?? []; + this.#references = undefined; + this.#settling?.abort(); + for (const binding of references) binding.release(); + } +} + +function settled(Type: Contract): ChordError { + return new ChordError({ + code: "released", + reason: "settled", + message: `${Type.name}: ctx.get after its handler settled`, + }); +} + +/** + * A child of `context` aborted by `signal` and by the caller's signal (the caller's reason first) until `release()`, + * which the call runs when it settles. Not `AbortSignal.any`: before Node 24.16.0 it keeps cleanup entries per call for + * as long as its sources live (nodejs/node#62367), and a call's sources (its attempt, its reference) live long. + * Without a caller's signal the child is aborted by `signal` itself and `release` does nothing. + */ +export function withLinkedAbortSignal( + signal: AbortSignal, + context: Context, +): { readonly context: Context; release(): void } { + const caller = context.abortSignal; + if (caller === undefined) return { context: withAbortSignal(signal, context), release: NOTHING }; + const linked = new AbortController(); + const listeners = [caller, signal].map((source) => { + const forward = (): void => { + release(); + linked.abort(source.reason); + }; + return [source, forward] as const; + }); + const release = (): void => { + for (const [source, forward] of listeners) source.removeEventListener("abort", forward); + }; + const aborted = listeners.find(([source]) => source.aborted); + if (aborted !== undefined) linked.abort(aborted[0].reason); + else for (const [source, forward] of listeners) source.addEventListener("abort", forward, { once: true }); + return { context: withAbortSignal(linked.signal, withoutAbortSignal(context)), release }; +} + +const NOTHING = (): void => {}; diff --git a/packages/chord/src/next/host/deadline.ts b/packages/chord/src/next/host/deadline.ts new file mode 100644 index 000000000..4625b06b8 --- /dev/null +++ b/packages/chord/src/next/host/deadline.ts @@ -0,0 +1,43 @@ +// An activation's `readyTimeout` deadline (slice 6a §9): one timer that eager migration work pauses and resumes with +// the budget it had left, so a long migration does not fail its object while other activations keep their bounds. + +/** A deadline of `ms` calling `expired` once when reached; paused time does not count. */ +export class Deadline { + readonly #expired: () => void; + /** The budget left when the timer last started (or paused). */ + #remaining: number; + #started = 0; + #timer: ReturnType | undefined; + #cleared = false; + + constructor(ms: number, expired: () => void) { + this.#expired = expired; + this.#remaining = ms; + this.#start(); + } + + /** Stops the clock; `resume` restarts it with the budget left. */ + pause(): void { + if (this.#timer === undefined) return; + clearTimeout(this.#timer); + this.#timer = undefined; + this.#remaining = Math.max(0, this.#remaining - (Date.now() - this.#started)); + } + + /** Restarts a paused clock. */ + resume(): void { + if (this.#timer === undefined && !this.#cleared) this.#start(); + } + + /** Ends the deadline: `expired` is never called. */ + clear(): void { + clearTimeout(this.#timer); + this.#timer = undefined; + this.#cleared = true; + } + + #start(): void { + this.#started = Date.now(); + this.#timer = setTimeout(this.#expired, this.#remaining); + } +} diff --git a/packages/chord/src/next/host/facet.ts b/packages/chord/src/next/host/facet.ts new file mode 100644 index 000000000..0055285ff --- /dev/null +++ b/packages/chord/src/next/host/facet.ts @@ -0,0 +1,71 @@ +// Facets (§16): a plugin's setup for one host, typed by the app's principal type. A setup only declares: the types it +// serves and the host's presence lists; it has no documents, routines or natives, so a facet cannot fail after `load`. + +import type { Watchable } from "../../watch.ts"; +import type { Contract } from "../contracts/index.ts"; +import type { Principal } from "../data/index.ts"; +import type { Factory } from "./attempt.ts"; +import type { Presence } from "./presence.ts"; + +/** What a setup serves types with (valid only while the setup runs), and the host's presence lists (always). */ +export interface FacetHost

{ + /** Serves a type on this host with `factory`; a remote type's factory gets the principal its object is for. */ + serve(Type: C, factory: Factory, options?: ServeOptions): void; + /** `host.references(Type)`: the references to objects of `Type` on this host (§19), for factories to derive from. */ + references(Type: C): Watchable[]>; +} + +/** What `host.serve` takes besides the type and the factory. */ +export type ServeOptions = { + /** + * How long, in milliseconds, a live object with params stays live after its last reference is released (0 by + * default): a reference within that time reuses it, with its documents and dependencies. + */ + readonly linger?: number; +}; + +/** A type a setup served: its contract, its factory and its options. */ +export type ServedEntry = { + readonly contract: Contract; + readonly factory: ServedFactory; + readonly options: ServeOptions; +}; + +/** A factory as a setup served it, of either kind (the host dispatches on its contract's kind). */ +type ServedFactory = (params: never, principalOrSelf: never, self: never) => unknown; + +/** A plugin's facet for hosts of the app's principal type `P` (type only: nothing is checked at runtime). */ +export type Facet

= { + readonly setup: (host: FacetHost

) => void; + /** Type only: the principal type, invariant, so a host takes only facets typed for exactly its `P`. */ + readonly "~principal"?: (principal: P) => P; +}; + +/** Defines a facet; base code binds `P` once, next to its principal type. */ +export function defineFacet

(setup: (host: FacetHost

) => void): Facet

{ + return { setup }; +} + +/** Runs a facet's setup and returns the types it served, in order; `references` is the host's. */ +export function setUp

( + name: string, + facet: Facet

, + references: (Type: C) => Watchable[]>, +): readonly ServedEntry[] { + const served: ServedEntry[] = []; + let open = true; + const host: FacetHost

= { + serve: (Type, factory, options = {}) => { + if (!open) throw new Error(`${name}: host.serve is valid only in the facet's setup`); + // Served types are dispatched by name and kind; the factory matches its own contract. + served.push({ contract: Type, factory: factory as ServedFactory, options }); + }, + references, + }; + try { + facet.setup(host); + } finally { + open = false; + } + return served; +} diff --git a/packages/chord/src/next/host/host.ts b/packages/chord/src/next/host/host.ts new file mode 100644 index 000000000..8312e70bd --- /dev/null +++ b/packages/chord/src/next/host/host.ts @@ -0,0 +1,862 @@ +// Hosts (§3): `createHost`, loading facets and the start barrier, links to children, the parent (connected again after +// a drop) and routes, the origins it relays holds for, getting and holding references as a principal, the heartbeat +// of every link, revocation (§4), presence (§19) and keep-alive counts, closing; one batch per host, the backoff map +// and the endings new instances wait for. + +import type { TelemetryContext } from "@earendil-works/pi-telemetry"; +import { BACKGROUND_CONTEXT, type Context, withAbortSignal, withoutAbortSignal } from "../../context/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { Watchable } from "../../watch.ts"; +import type { Address, Contract, HostKind, ParamsOf } from "../contracts/index.ts"; +import { contractData, type Params } from "../contracts/internal.ts"; +import { ChordError, type FailureCause, type Principal, type PrincipalKeyFunction } from "../data/index.ts"; +import { + asChordError, + bindProvider, + canonicalJson, + checkParams, + identityKey, + invalid, + principalKey, +} from "../data/internal.ts"; +import { binaryCodec, type Codec, type Compression, type Link, type PartLimits } from "../link/index.ts"; +import { linkLimits } from "../link/internal.ts"; +import { + type HoldRequest, + HostLink, + type LinkRole, + Path, + type ProviderSide, + RemoteObject, +} from "../remote/internal.ts"; +import { Batch, type BatchError } from "../state/internal.ts"; +import type { ChordStore, Store } from "../store/store.ts"; +import { Activity, type ActivityCounts } from "./activity.ts"; +import { notLive } from "./attempt.ts"; +import { Backoff } from "./backoff.ts"; +import { type Caller, withLinkedAbortSignal } from "./ctx.ts"; +import { type Facet, type ServedEntry, setUp } from "./facet.ts"; +import type { CloseFailed } from "./native.ts"; +import { ChordObject, type Holder, type LocalContract, type RemoteContract, type Served } from "./object.ts"; +import { type Placed, Placement } from "./placement.ts"; +import { type Presence, References } from "./presence.ts"; +import { Binding, type CheckedParams, type Held, type Reference, type Target } from "./reference.ts"; +import { RevocationKeys, type RevokedBy } from "./revocation.ts"; +import { Route, type RouteOptions } from "./route.ts"; +import { uuidv7 } from "./uuid.ts"; + +/** What `createHost` takes; `R` is the app's revocation type (JSON). */ +export type HostOptions

= { + /** This host's address (`Session({ id })`): it serves only objects placed there. Without one, a client. */ + readonly address?: Address; + /** Who this host acts as for its own requests; a host without one (a client) has none. */ + readonly principal?: P; + /** The key of a principal in identities; L1's canonical JSON by default. The same function on every host. */ + readonly principalKey?: PrincipalKeyFunction

; + /** The telemetry adapter; NOOP by default. */ + readonly telemetry?: TelemetryContext; + /** Everything that fails in the background; every app handles it. */ + readonly onError: (error: HostError) => void; + /** How long a native's close, and an ending object's running calls, are waited for; 30 s by default. */ + readonly closeTimeout?: number; + /** How long an activation attempt, and a `get`, may take until the object is live; 60 s by default. */ + readonly readyTimeout?: number; + /** + * The heartbeat of every link: a link silent for `ping` ms (15 s by default) sends a `ping`; one silent for `close` + * ms (30 s by default), its handshake included, is closed. + */ + readonly heartbeat?: { readonly ping?: number; readonly close?: number }; + /** + * What revokes a principal (§4): `host.revoke(r)` ends the objects and closes the child links of every principal + * `p` whose `revokedBy(p)` holds `r` (compared as canonical JSON). The same function on every host. + */ + readonly revokedBy?: RevokedBy; + /** + * A synchronous compressor (`nodeCompression` for Node): JSON parts above 16 KiB this host originates are deflated + * once, and deflated parts it owns inflated in place. Without one (a browser), nothing is compressed and deflated + * parts are inflated asynchronously, later messages of their link waiting in order. + */ + readonly compression?: Compression; + /** + * The store persisted documents and tables live in (`openStore`); the app owns it and closes it after the host. + * One open host per store. A `storage_failed` of the store is reported once and closes the host. + */ + readonly store?: Store; +}; + +/** Where a reported part sits on its object: a routine (`on`, `onLive`) or a native, by kind, from 1 in declaration order. */ +export type Position = { readonly kind: "on" | "onLive" | "native"; readonly position: number }; + +/** + * What `onError` gets, by kind. `object` is the object's type, with its params key when it has params; `failure` is a + * `FailureCause`, `error` the plain error. + */ +export type HostError = + | BatchError + | { readonly kind: "activation_failed"; readonly object: string; readonly failure: FailureCause } + /** A live object failed at runtime (a `migrate` failing on access, a stored version newer than the code). */ + | { readonly kind: "object_failed"; readonly object: string; readonly failure: FailureCause } + /** The host's store stopped after a backend error: the host closes itself (once). */ + | { readonly kind: "storage_failed"; readonly error: unknown } + | { readonly kind: "close_failed"; readonly object: string; readonly native: Position; readonly error: unknown } + | { readonly kind: "close_timeout"; readonly object: string; readonly native: Position } + | { readonly kind: "drain_timeout"; readonly object: string } + | { readonly kind: "protocol_error"; readonly error: unknown } + | { readonly kind: "resolve_failed"; readonly address: string; readonly error: unknown } + /** + * A derive's compute threw (its owner's, once per failure episode: the first failure after a value or after its + * creation; a value ends the episode); `member` when it is an exposed state member. + */ + | { readonly kind: "derive_failed"; readonly object: string; readonly member?: string; readonly error: unknown } + /** A routine's call threw, or its source failed for good; `routine` is its kind and position. */ + | { readonly kind: "routine_failed"; readonly object: string; readonly routine: Position; readonly error: unknown } + /** A job's `run` threw (not after an abort): its outcome is `failed { threw }`. */ + | { + readonly kind: "job_failed"; + readonly object: string; + readonly job: string; + readonly id: string; + readonly error: unknown; + } + /** A job's `finish` threw: retried with backoff, blocked after five failures in a row. */ + | { + readonly kind: "job_finish_failed"; + readonly object: string; + readonly job: string; + readonly id: string; + readonly error: unknown; + } + /** A blocked run was dropped without `finish`: aborted, or its object ended. */ + | { readonly kind: "job_dropped"; readonly object: string; readonly job: string; readonly id: string }; + +/** + * What `accept` and `connect` take: the principal of the host at the other end, as this host stamps it; `trusted` skips + * the message check on the link (IPC to the host's own processes; never grants the principal fields); the codec both + * ends of the link use (`binaryCodec` by default); and the limits per part of what arrives on it (Q2: none on a + * trusted link, 16 MiB on the wire and 64 MiB decoded otherwise, unless set). + */ +export type LinkOptions

= { + readonly principal: P; + readonly trusted?: boolean; + readonly codec?: Codec; + readonly limits?: PartLimits; +}; + +/** What `host.get` and `host.hold` take besides the type, the params and the context. */ +export type GetOptions = { + /** + * A getter's deadline for its whole wait, and how long a call made while the reference is loading waits (the park + * deadline); the host's `readyTimeout` by default. + */ + readonly readyTimeout?: number; +}; + +/** + * A host whatever its principal type, as Chord's own parts use it: `ChordHost

` is invariant in `P` (`load` takes + * facets typed for exactly `P`). + */ +export type AnyHost = Omit; + +/** A host. `P` is the app's principal type, `R` its revocation type. */ +export interface Host

{ + /** + * Runs every facet's setup, then activates every served type with empty params and resolves when each has had its + * first outcome, live or failed (the start barrier; failures go to `onError`). A setup that throws rejects it and + * closes the host. Runs once. + */ + load(facets: readonly { readonly name: string; readonly facet: Facet

}[], context: Context): Promise; + /** + * A reference once its object is live; waits for `load` first. It acts as the host's principal and goes where its + * type's placement puts it: here, over a route, or over the parent link (`Placement`). A broken failure rejects at + * once; through transient ones it waits until its deadline, then rejects with the last failure's error. + */ + get(Type: C, params: ParamsOf, context: Context, options?: GetOptions): Promise>; + /** + * A reference at once, following its object until released: loading, live, failed (retried as its object is) and + * loading again while its link is down; it never rejects. Throws at once for invalid params and a local type not + * served here; before `load` resolved these are decided once it did, failing the reference broken. Its `variant` is `undefined` until its first binding. + */ + hold(Type: C, params: ParamsOf, context: Context, options?: GetOptions): Held; + /** + * A link from a child: this host serves its remote types over it, every hold and call arriving on it acting as + * `principal`. `context` is the parent of the work arriving on it (its cancellation is dropped). + */ + accept(link: Link, options: LinkOptions

, context: Context): void; + /** + * This host's parent link: remote types not served here and not routed are held over it, every such reference held + * again on a new one; holds arriving on it may name whom they act for. Throws while the previous one is open; once + * it closed (a drop), `connect` takes the next. `principal` is who the parent is. + */ + connect(link: Link, options: LinkOptions

, context: Context): void; + /** + * How this host reaches hosts of `Kind` (called before `load`, once per kind): `authorize` admits every hold routed + * over it, `resolve` makes the link of an address. Holds for an address wait while it resolves and again after its + * link drops (a backoff per address, 1 s doubling to 60 s), while they exist. `context` is the parent of the route's + * work (its `resolve`s, the holds and calls on its links), aborted for it when this host closes. + */ + route( + Kind: HostKind, + options: RouteOptions>, + context: Context, + ): void; + /** + * Revokes `revocation` (§4, R10, R11), synchronously: every remote object whose principal it revokes (`revokedBy`) + * ends at once, whatever references remain (holders get `forbidden`, running calls are aborted, an activation is + * abandoned); every child link whose principal it revokes closes (never the parent, never this host's own + * principal); and every route link is sent `revoke`, which the host there applies with its own `revokedBy`. Nothing + * is replayed to links made later. + */ + revoke(revocation: R, context: Context): void; + /** + * The references to objects of `Type` on this host (§19), one entry per reference: the object's params, the + * principal it acts as (remote types) and its origin, the incarnation of the host it was made for (two tabs are two + * origins). Changes in the batch where a reference binds or ends; a link drop is a departure. + */ + references(Type: C): Watchable[]>; + /** + * The work keeping this host busy, for the app's supervisor: holds arriving over links and served here, method + * calls running, running jobs and lingering objects; published at most once per microtask turn with changes. + */ + readonly activity: Watchable; + /** + * Closes its links, then ends every object at once, then drains and closes them one by one, the last to go live + * first (reason `close`). Rejects with an `AggregateError` of the closes that threw. + */ + close(context: Context): Promise; +} + +/** Creates a host. */ +export function createHost

( + options: HostOptions, +): Host { + return new ChordHost(options); +} + +/** No activation chain (Q5). */ +const NO_CHAIN: readonly string[] = []; + +/** The params of a type with empty params, as checked: nothing, and the empty key. */ +const EMPTY_PARAMS: CheckedParams = { params: {}, key: "" }; + +/** An npm package name, scoped or not. */ +const PLUGIN_NAME = /^(?:@[a-z0-9~-][a-z0-9._~-]*\/)?[a-z0-9~-][a-z0-9._~-]*$/; + +/** The host: its served types, its objects by identity, its batch, backoff and endings. */ +export class ChordHost

implements Host { + readonly batch: Batch; + readonly timeouts: { readonly close: number; readonly ready: number }; + readonly backoff = new Backoff((identity) => this.#due(identity)); + readonly address: Address | undefined; + readonly principal: P | undefined; + /** This run of the host, sent in every handshake (R20). */ + readonly incarnation = uuidv7(); + readonly compression: Compression | undefined; + /** The host's store, if it has one. */ + readonly store: ChordStore | undefined; + readonly #activity: Activity; + readonly #principalKey: PrincipalKeyFunction

| undefined; + readonly #revocationKeys: RevocationKeys; + /** The presence list of each type someone asked about (`references`), by type name. */ + readonly #references = new Map(); + readonly #onError: (error: HostError) => void; + readonly #types = new Map(); + /** By identity, in the order they went live (the others where they were created). */ + readonly #objects = new Map(); + /** Teardowns still running, by identity: the next instance of an identity waits for its previous ones. */ + readonly #endings = new Map>(); + readonly #loaded = Promise.withResolvers(); + /** Every open link: children, routes and the parent. */ + readonly #links = new Set(); + /** Where holds go that are not served here and not routed: the parent link, across reconnects. */ + readonly #parent = new Path(() => {}); + /** + * Remote references failed for a part past a link's limits: on no link's path (no reconnect holds them, no route + * resolves for them) until `ref.retry()` places them anew; ended at `close` like the others. + */ + readonly unplaced = new Path(() => {}); + readonly #routes = new Map(); + readonly #placement: Placement; + /** The downstream origins this host relays holds for, by their number on upstream links (never reused). */ + readonly relays = new Map(); + #nextOrigin = 1; + /** Aborted when the host closes: the context of every route's `resolve`. */ + readonly #routing = new AbortController(); + readonly #heartbeat: { readonly ping: number; readonly close: number }; + /** The one heartbeat timer, running while the host has a link. */ + #ticking: ReturnType | undefined; + #state: "new" | "loading" | "loaded" | "closed" = "new"; + /** The one closing, which every `close` returns. */ + #closing: Promise | undefined; + + constructor(options: HostOptions

) { + this.address = options.address; + this.principal = options.principal; + this.#principalKey = options.principalKey; + this.#onError = options.onError; + this.batch = new Batch((error) => this.report(error), options.telemetry?.metrics); + this.compression = options.compression; + this.#activity = new Activity(this.batch); + // `revokedBy` is called with this host's principals only, the `P`s its links stamp. + this.#revocationKeys = new RevocationKeys(options.revokedBy as RevokedBy | undefined); + this.timeouts = { close: options.closeTimeout ?? 30_000, ready: options.readyTimeout ?? 60_000 }; + this.#heartbeat = { ping: options.heartbeat?.ping ?? 15_000, close: options.heartbeat?.close ?? 30_000 }; + this.#placement = new Placement(this.address, this.#types, this.#routes, this.#parent); + // Chord's own: `openStore` made it. + this.store = options.store as ChordStore | undefined; + this.store?.attach((error) => this.#storageFailed(error)); + } + + async load(facets: readonly { readonly name: string; readonly facet: Facet

}[], context: Context): Promise { + this.batch.assertOutside("load"); + this.#assertOpen(); + if (this.#state !== "new") throw new Error("load runs once per host"); + this.#state = "loading"; + try { + this.#setUp(facets); + } catch (error) { + // The host closes as `close` would: its links too. Nothing was activated, so no close can fail. + this.#closing ??= this.#close(); + await this.#closing.catch(() => {}); + throw error; + } + const objects = [...this.#types.values()] + .filter((served) => served.kind === "local" && served.hostLifetime) + .map((served) => this.#object(served, EMPTY_PARAMS, undefined, context)); + await Promise.allSettled(objects.map(({ settled }) => settled)); + if (this.#state === "loading") this.#state = "loaded"; + this.#loaded.resolve(); + } + + async get( + Type: C, + params: ParamsOf, + context: Context, + options: GetOptions = {}, + ): Promise> { + const binding = await this.acquire(Type, params, this.#own(), context, options); + // The binding was built from `Type`'s members. + return binding.reference as Reference; + } + + hold(Type: C, params: ParamsOf, context: Context, options: GetOptions = {}): Held { + this.batch.assertOutside("hold"); + // The binding was built from `Type`'s members; it may be unbound, so `variant` may be undefined. + return this.#hold(Type, params, this.#own(), context, options).reference as Held; + } + + accept(link: Link, options: LinkOptions

, context: Context): void { + this.#assertOpen(); + this.#link(link, "child", options, context); + } + + connect(link: Link, options: LinkOptions

, context: Context): void { + this.#assertOpen(); + if (this.#parent.link !== undefined) throw new Error("connect: this host has an open parent link"); + const parent = this.#link(link, "parent", options, context); + if (parent.open) this.#parent.connected(parent); + } + + route( + Kind: HostKind, + options: RouteOptions>, + context: Context, + ): void { + this.#assertOpen(); + if (this.#state !== "new") throw new Error("route: routes are declared before load"); + if (this.#routes.has(Kind.kind)) throw new Error(`route: ${Kind.kind} is routed already`); + // The parent of the route's work: its `resolve`s and the holds and calls on its links, aborted when it closes. + const routing = withAbortSignal(this.#routing.signal, context); + // A route calls `authorize` with the principals of holds on this host, `P`s, and addresses of `Kind`. + const route = new Route( + Kind as unknown as HostKind, + options as unknown as RouteOptions, + this, + routing, + ); + this.#routes.set(Kind.kind, route); + } + + revoke(revocation: R, context: Context): void { + const key = canonicalJson(revocation, context); + const revoked = (principal: Principal): boolean => + !this.#isOwn(principal, context) && this.#revocationKeys.revokes(principal, key, context); + const error = (label: string): ChordError => + new ChordError({ code: "forbidden", reason: "revoked", message: `${label}: revoked` }); + for (const object of [...this.#objects.values()]) { + if (object.principal !== undefined && revoked(object.principal)) object.end(error(object.label)); + } + // Holds admitted before this revoke but still waiting for `load` would take revoked objects afterwards. + const waiting = error("a hold waiting for load"); + for (const link of this.#links) link.revokeWaiting(revoked, waiting); + // Down the route links before the child links close: a worker ends the principal's objects with the revocation's + // error before the closed origins release their holds. + for (const link of this.#links) if (link.role === "route") link.send({ kind: "revoke", revocation }); + for (const link of [...this.#links]) if (link.role === "child" && revoked(link.principal)) link.close(); + } + + get activity(): Watchable { + // Its value is an `ActivityCounts`. + return this.#activity as unknown as Watchable; + } + + references(Type: C): Watchable[]> { + let references = this.#references.get(Type.name); + if (references === undefined) { + references = new References(this.batch); + for (const object of this.#objects.values()) { + if (object.served.contract.name !== Type.name) continue; + for (const holder of object.holders) references.record(object, holder); + } + this.#references.set(Type.name, references); + } + // Entries of `Type`'s objects, acting as `P`s. + return references as unknown as Watchable[]>; + } + + async close(_context: Context): Promise { + this.batch.assertOutside("close"); + this.#closing ??= this.#close(); + return this.#closing; + } + + /** + * What a hold arriving before `load` resolved waits for; `undefined` once it resolved (or the host closed), so a + * later hold takes its reference at once, in the turn it arrived (R18's handover: a new hold before the old's release). + */ + get pendingLoad(): Promise | undefined { + return this.#state === "new" || this.#state === "loading" ? this.#loaded.promise : undefined; + } + + /** Whether the host closes: its objects end through `close`, not through its links. */ + get closing(): boolean { + return this.#state === "closed"; + } + + /** + * A reference to `Type` made by `caller` (its principal, its origin's incarnation), once it is live (`host.get`, + * `ctx.get`): a hold, waited for up to the getter's one deadline, across reconnects; a broken failure rejects at + * once, the deadline with the last failure's error. A getter that gives up releases its reference. + */ + async acquire( + Type: Contract, + params: unknown, + caller: Caller, + context: Context, + options: GetOptions, + ): Promise { + this.batch.assertOutside("get"); + const binding = this.#hold(Type, params, caller, context, options); + const timeout = options.readyTimeout ?? this.timeouts.ready; + const expired = new AbortController(); + const waiting = withLinkedAbortSignal(expired.signal, context); + const deadline = setTimeout(() => expired.abort(binding.lastFailure ?? notLive(binding.label, timeout)), timeout); + try { + await binding.live(waiting.context); + return binding; + } catch (error) { + binding.release(); + throw error; + } finally { + clearTimeout(deadline); + waiting.release(); + } + } + + /** Adds `delta` to the keep-alive count of `kind` (`host.activity`), where the work happens. */ + count(kind: keyof ActivityCounts, delta: number): void { + this.#activity.count(kind, delta); + } + + /** `holder` references `object`: listed in its type's presence, if someone asked for it. */ + referenced(object: ChordObject, holder: Holder, context: Context): void { + this.#references.get(object.served.contract.name)?.add(object, holder, context); + } + + /** `holders` of `object` ended: gone from its type's presence. */ + unreferenced(object: ChordObject, holders: Iterable, context: Context): void { + this.#references.get(object.served.contract.name)?.remove(holders, context); + } + + /** Reports a background failure to `onError`; a throwing `onError` has nowhere left to report to. */ + report(error: HostError): void { + try { + this.#onError(error); + } catch { + // Nothing to do: teardowns and attempts go on. + } + } + + /** + * An unbound reference to an object of `Type` made by `caller`, declared by `dependent` (`undefined` for + * `host.get`, `host.hold` and `ctx.get`). Throws where it cannot go (`#assertRoute`). + */ + reference(Type: Contract, dependent: ChordObject | undefined, caller: Caller, options: GetOptions): Binding { + this.#assertOpen(); + this.#assertRoute(Type); + return this.#binding(Type, dependent, caller, options); + } + + /** + * Binds `binding` to the object for (Type, params) as its principal where its placement puts it (`Placement`): the + * object here, created with its first attempt if there is none, or a remote object held over its path. Throws as + * `Placement.reference` does. + */ + bind(binding: Binding, Type: Contract, params: CheckedParams, context: Context): Target { + const { principal } = binding.caller; + const placed = this.#placement.reference(Type, params.params, principal, context); + let target: Target; + if (placed.kind === "path") { + const { caller, dependent } = binding; + const remote = new RemoteObject(this, Type, params, placed.address, caller, placed.path, dependent); + placed.path.attach(remote); + target = remote; + } else { + // Placed here: its type is served here. + const served = this.#types.get(Type.name)!; + target = this.#object(served, params, principal, context, NO_CHAIN, binding.dependent); + } + binding.bind(target); + return target; + } + + /** Where a hold arriving on `link` goes (`Placement.hold`); `released` once the host closed. */ + place(request: HoldRequest, link: HostLink, caller: Caller, context: Context): Placed { + this.#assertOpen(); + return this.#placement.hold(request, link, caller.principal, context); + } + + /** + * The object a hold placed here binds to, acting as `principal`: a remote type served here (otherwise + * `unavailable`) with the hold's version and fingerprint (`contract_mismatch`), valid params, and placed here, its + * placement computed from the params (`invalid/placement`); not an activation the hold's chain waits for + * (`invalid/cycle`, Q5). A new object keeps the hold's chain. + */ + serving(request: HoldRequest, params: unknown, principal: Principal, context: Context): ChordObject { + const { type, version, fingerprint, chain = NO_CHAIN } = request; + const served = this.#types.get(type); + if (served?.kind !== "remote") { + throw new ChordError({ code: "unavailable", message: `${type} is not served here` }); + } + this.#assertMatch(served, version, fingerprint); + const checked = checkParams(served.contract, params, context); + this.#placement.assertHere(served.contract, checked.params); + const object = this.#object(served, checked, principal, context, chain); + const activation = object.activationId; + if (activation !== undefined && chain.includes(activation)) throw this.cycle(chain, activation, [object.label]); + return object; + } + + /** + * The path a remote reference of `Type` with `params` acting as `principal` goes over now: `ref.retry()` places a + * reference failed for a part past the limits anew (a route may have forgotten its old path). Throws as + * `Placement.reference` does, and `unavailable` for a type served here now. + */ + pathOf(Type: Contract, params: ParamsOf, principal: Principal | undefined): Path { + // A retry has no context of its own: its admission checks count on none. + const placed = this.#placement.reference(Type, params, principal, BACKGROUND_CONTEXT); + if (placed.kind === "path") return placed.path; + throw new ChordError({ code: "unavailable", message: `${Type.name} is served here now` }); + } + + /** + * The `invalid/cycle` error of a cycle through `chain` (Q5): its entries from the activation `id` on, by label + * where the object is on this host ("…" for other hosts'), then `closing`. + */ + cycle(chain: readonly string[], id: string, closing: readonly string[]): ChordError { + const labels = new Map([...this.#objects.values()].map((each) => [each.activationId, each.label])); + const cycle = chain.slice(chain.indexOf(id)).map((entry) => labels.get(entry) ?? "…"); + return invalid("cycle", `dependency cycle: ${[...cycle, ...closing].join(" → ")}`); + } + + /** A new origin number for `side`, which relays holds upstream; registered in `relays` until it closes. */ + relayOrigin(side: ProviderSide): number { + const origin = this.#nextOrigin++; + this.relays.set(origin, side); + return origin; + } + + /** + * A route's link, its work under the route's `context`: registered with the host and its heartbeat; one closed + * already is returned closed. + */ + routeLink(link: Link, options: LinkOptions, context: Context): HostLink { + return this.#link(link, "route", options, context); + } + + /** A link completed its handshake: a route's resets its address's backoff. */ + linkOpened(link: HostLink): void { + if (link.role !== "route") return; + for (const route of this.#routes.values()) route.opened(link); + } + + /** + * Forgets a link once it closed: the origins relaying to it, then the parent's path or its route's address are + * told; the heartbeat may stop. + */ + unlink(link: HostLink): void { + this.#links.delete(link); + for (const relay of this.relays.values()) relay.unlinked(link); + if (this.#links.size === 0) { + clearInterval(this.#ticking); + this.#ticking = undefined; + } + if (link.role === "parent") this.#parent.disconnected(link); + if (link.role === "route") for (const route of this.#routes.values()) if (route.closed(link)) break; + } + + /** Moves `object` behind every object that went live before it (close ends in reverse); its backoff is over. */ + wentLive(object: ChordObject): void { + this.#objects.delete(object.identity); + this.#objects.set(object.identity, object); + this.backoff.delete(object.identity); + } + + /** Drops an ended object (once, before a next instance of its identity exists). */ + forget(object: ChordObject): void { + this.#objects.delete(object.identity); + this.backoff.ended(object.identity); + } + + /** Records a teardown for `identity`: its next attempt waits for it and for every earlier one still running. */ + ending(identity: string, teardown: Promise): void { + const previous = this.#endings.get(identity); + const ending = previous === undefined ? teardown : Promise.all([previous, teardown]).then(() => {}); + this.#endings.set(identity, ending); + void ending.then(() => { + if (this.#endings.get(identity) === ending) this.#endings.delete(identity); + }); + } + + /** What the next attempt for `identity` waits for: its previous instances' teardowns, if any still run. */ + endingOf(identity: string): Promise | undefined { + return this.#endings.get(identity); + } + + /** + * A reference to `Type` made by `caller` for `host.hold` and getters: params and where it goes checked at once + * (`invalid/params`, `#assertRoute`), then bound (a placement or route that fails fails the reference broken); + * before `load` resolved, both once it did (a host closed meanwhile releases it). + */ + #hold(Type: Contract, params: unknown, caller: Caller, context: Context, options: GetOptions): Binding { + this.#assertOpen(); + const checked = checkParams(Type, params, context); + if (this.pendingLoad === undefined) { + const binding = this.reference(Type, undefined, caller, options); + this.#bindOrFail(binding, Type, checked, context); + return binding; + } + const binding = this.#binding(Type, undefined, caller, options); + // Raced with the reference's end: a release before `load` resolved lets go of the binding, its contract and + // `context` at once (only the race's settled promise stays behind in `load`'s reactions). + void Promise.race([this.#loaded.promise, binding.ended()]).then(() => { + // Released while `load` ran: nothing is made for it. + if (binding.refState.kind === "ended") return; + if (this.#state === "closed") return binding.end(this.#closedError()); + this.#bindOrFail(binding, Type, checked, context); + }); + return binding; + } + + /** + * Binds a reference (`bind`); where it cannot go (`#assertRoute`, a placement or `authorize` that throws or refuses) + * fails it broken. + */ + #bindOrFail(binding: Binding, Type: Contract, checked: CheckedParams, context: Context): void { + try { + this.#assertRoute(Type); + this.bind(binding, Type, checked, context); + } catch (error) { + binding.refuse({ kind: "broken", error: asChordError(error, Type.name) }); + } + } + + /** A new reference's binding made by `caller`; its parked calls wait its `readyTimeout`. */ + #binding(Type: Contract, dependent: ChordObject | undefined, caller: Caller, options: GetOptions): Binding { + return new Binding(Type, this.batch, dependent, caller, options.readyTimeout ?? this.timeouts.ready); + } + + /** This host as the maker of its own references: its principal and incarnation. */ + #own(): Caller { + return { principal: this.principal, incarnation: this.incarnation }; + } + + /** Whether `principal` is this host's own (by principal key): revocation never touches it. */ + #isOwn(principal: Principal, context: Context): boolean { + const own = this.principal; + if (own === undefined) return false; + // Principals on this host are `P`s: its own, and the ones its links stamp. + return ( + principalKey(own, context, this.#principalKey) === principalKey(principal as P, context, this.#principalKey) + ); + } + + /** A new link of `role`, registered with the heartbeat while open. */ + #link(link: Link, role: LinkRole, options: LinkOptions, context: Context): HostLink { + const { principal, trusted = false, codec = binaryCodec } = options; + const limits = linkLimits(trusted, options.limits); + const made = new HostLink(this, link, { role, principal, trusted, codec, limits }, withoutAbortSignal(context)); + if (!made.open) return made; + this.#links.add(made); + if (this.#ticking === undefined) { + const { ping, close } = this.#heartbeat; + this.#ticking = setInterval(() => { + const now = Date.now(); + for (const each of [...this.#links]) each.tick(now, ping, close); + }, Math.min(ping, close) / 10); + } + return made; + } + + /** + * Throws where a reference to `Type` cannot go: a type served here needs the caller's version and fingerprint + * (`contract_mismatch`); a local type never crosses a link (`unavailable`). A remote type not served here goes on (a + * route or the parent link), open yet or not. + */ + #assertRoute(Type: Contract): void { + const served = this.#types.get(Type.name); + if (served === undefined) { + if (Type.kind === "local") { + throw new ChordError({ code: "unavailable", message: `${Type.name} is not served here` }); + } + return; + } + this.#assertMatch(served, Type.version, Type.fingerprint); + } + + /** + * The object for `params` (checked by L1) and, for a remote type, `principal` (its key is part of the identity), + * created if there is none; its first attempt starts at once, or when its identity's backoff window ends. A new one + * keeps the activation chain it is created for (Q5): `chain` (a hold's), or that of `dependent`'s activation when + * an activation declared it, so its own holds over links carry that chain on. + */ + #object( + served: Served, + params: CheckedParams, + principal: Principal | undefined, + context: Context, + chain: readonly string[] = NO_CHAIN, + dependent?: ChordObject, + ): ChordObject { + const { key, params: checked } = params; + const identity = + served.kind === "local" + ? identityKey(served.contract, key) + : // A host serving remote types has a principal, which its references act as unless a + // link stamped its own (only `P`s). + identityKey(served.contract, key, principalKey(principal as P, context, this.#principalKey)); + let object = this.#objects.get(identity); + if (object === undefined) { + // Shared by every getter: one getter's cancellation does not cancel it. + const owner = served.kind === "local" ? undefined : principal; + const created = dependent?.activationChain() ?? chain; + object = new ChordObject(this, served, checked, owner, key, identity, withoutAbortSignal(context), created); + this.#objects.set(identity, object); + if (!this.backoff.waiting(identity)) object.attempt(); + } + return object; + } + + /** + * Closes the links, then ends every object at once and tears them down one by one, the last to go live first, after + * the endings already running. + */ + async #close(): Promise { + this.#state = "closed"; + this.#routing.abort(this.#closedError()); + // Links first, children, routes, then the parent: no `gone` goes out for the objects ending below. + const links = [...this.#links]; + for (const role of ["child", "route", "parent"]) for (const link of links) if (link.role === role) link.close(); + // Remote references end now: their parked calls reject `released`, no later link holds them. + const closed = this.#closedError(); + const holders = [ + ...this.#parent.holders(), + ...this.unplaced.holders(), + ...[...this.#routes.values()].flatMap((route) => route.close()), + ]; + for (const holder of holders) if (holder.kind === "remote") holder.lost(closed); + this.backoff.clear(); + // Every object is sealed before any teardown releases a reference, so none ends in the background. + const teardowns = [...this.#objects.values()].reverse().map((object) => object.stop()); + await Promise.all(this.#endings.values()); + const failures: CloseFailed[] = []; + for (const tearDown of teardowns) failures.push(...(await tearDown())); + this.#types.clear(); + this.store?.detach(); + // Gets waiting for a load that will not come fail now. + this.#loaded.resolve(); + if (failures.length > 0) { + const message = `host.close: ${failures.length} native close(s) threw`; + throw new AggregateError( + failures.map(({ error }) => error), + message, + ); + } + } + + /** The store stopped after a backend error (it tells the host once): reported, and the host closes. */ + #storageFailed(error: unknown): void { + this.report({ kind: "storage_failed", error }); + this.#closing ??= this.#close(); + // The app's own `close` returns this closing and gets its failures. + this.#closing.catch(() => {}); + } + + /** A backoff window ended: the identity's object is retried; without one the entry goes. */ + #due(identity: string): void { + const object = this.#objects.get(identity); + if (object === undefined) this.backoff.delete(identity); + else object.attempt(); + } + + /** Runs every facet's setup, checking the plugin names (npm names, each once), and serves what they declared. */ + #setUp(facets: readonly { readonly name: string; readonly facet: Facet

}[]): void { + const names = new Set(); + for (const { name, facet } of facets) { + if (!PLUGIN_NAME.test(name)) throw new Error(`"${name}" is not a valid plugin name`); + if (names.has(name)) throw new Error(`plugin "${name}" is loaded twice`); + names.add(name); + for (const entry of setUp(name, facet, (Type) => this.references(Type))) this.#serve(entry); + } + } + + #serve({ contract, factory, options }: ServedEntry): void { + if (this.#types.has(contract.name)) throw new Error(`${contract.name} is served twice`); + // A client serves only local types (§3): a remote object is owned by a principal of the host serving it. + if (contract.kind === "remote" && this.principal === undefined) { + throw new Error(`${contract.name}: a host without a principal serves only local types`); + } + const common = { provider: bindProvider(contract, {}), linger: options.linger ?? 0 }; + // The contract's kind decides the factory's signature and the contract's shape. + const served: Served = + contract.kind === "local" + ? { + ...common, + kind: "local", + contract: contract as LocalContract, + factory: factory as Extract["factory"], + hostLifetime: contractData(contract).params.fields.length === 0, + } + : { + ...common, + kind: "remote", + contract: contract as RemoteContract, + factory: factory as Extract["factory"], + }; + this.#types.set(contract.name, served); + } + + #assertMatch(served: Served, version: number, fingerprint: string): void { + if (version !== served.contract.version || fingerprint !== served.contract.fingerprint) { + const message = `${served.contract.name} v${version} does not match the served v${served.contract.version}`; + throw new ChordError({ code: "contract_mismatch", message }); + } + } + + #assertOpen(): void { + if (this.#state === "closed") throw this.#closedError(); + } + + #closedError(): ChordError { + return new ChordError({ code: "released", reason: "closed", message: "the host is closed" }); + } +} diff --git a/packages/chord/src/next/host/index.ts b/packages/chord/src/next/host/index.ts new file mode 100644 index 000000000..6eb1a040c --- /dev/null +++ b/packages/chord/src/next/host/index.ts @@ -0,0 +1,24 @@ +// Hosts for apps: creating a host, facets, factories, natives, references and the ctx of method handlers, routines and +// their waiting helpers, revocation, presence and keep-alive counts. + +export type { ActivityCounts } from "./activity.ts"; +export type { Factory, Self, StoreSelf } from "./attempt.ts"; +export type { Ctx } from "./ctx.ts"; +export { defineFacet, type Facet, type FacetHost, type ServeOptions } from "./facet.ts"; +export { + createHost, + type GetOptions, + type Host, + type HostError, + type HostOptions, + type LinkOptions, + type Position, +} from "./host.ts"; +export type { Native, NativeCloseReason, NativeOptions } from "./native.ts"; +export type { Presence } from "./presence.ts"; +export type { RefState } from "./ref-state.ts"; +export type { Held, Reference, RefIn } from "./reference.ts"; +export type { RevokedBy } from "./revocation.ts"; +export type { RouteOptions } from "./route.ts"; +export type { OnHandler, OnLiveHandler, OnNativeHandler, RoutineDeclarations } from "./routines.ts"; +export { sleep, untilAborted } from "./sleep.ts"; diff --git a/packages/chord/src/next/host/internal.ts b/packages/chord/src/next/host/internal.ts new file mode 100644 index 000000000..a976a73a5 --- /dev/null +++ b/packages/chord/src/next/host/internal.ts @@ -0,0 +1,11 @@ +// Hosts for Chord's layers: what the remote sides of links use of a host, its objects and references; what jobs use of +// a handler's ctx, the backoff and ids. + +export { nextDelay } from "./backoff.ts"; +export { CallContext, type Caller } from "./ctx.ts"; +export type { AnyHost, ChordHost } from "./host.ts"; +export type { PinnedKey } from "./keyed-members.ts"; +export { type ChordObject, type Holder, settlement } from "./object.ts"; +export { LIVE, LOADING, type RefState } from "./ref-state.ts"; +export type { Binding, CheckedParams, Target } from "./reference.ts"; +export { uuidv7 } from "./uuid.ts"; diff --git a/packages/chord/src/next/host/keyed-members.ts b/packages/chord/src/next/host/keyed-members.ts new file mode 100644 index 000000000..e10166507 --- /dev/null +++ b/packages/chord/src/next/host/keyed-members.ts @@ -0,0 +1,268 @@ +// Keyed members across attempts (slice 6a §1.2, §1.4, §12): the keys of a local object's keyed members that references +// pin (`ref.member.at(key)`, `ref.member.watch(key)`), one member source per member and key, as state members have. +// While an attempt is live a key follows that attempt's keyed member (the object's own slot, or the provider's member +// it re-exposes, rule 22: one tracker at the end); while the object is failed it keeps its last value and gets no +// frames; when a new attempt goes live it follows that attempt's member of the same name, its key loaded before. + +import { awaitWithContext, type Context } from "../../context/index.ts"; +import type { ChordError } from "../data/index.ts"; +import type { KeyedSlot } from "../documents/internal.ts"; +import type { Batch, DeriveSource } from "../state/internal.ts"; +import { MemberSource } from "./member-source.ts"; +import type { KeyedBinding } from "./members.ts"; + +/** A key of a keyed member, pinned: its source, and the release (idempotent). */ +export type PinnedKey = { readonly source: DeriveSource; readonly release: () => void }; + +/** A pinned key on one attempt's keyed member; `foreign` when the member is another object's (re-exposed). */ +type Following = { readonly pin: PinnedKey; readonly foreign: boolean }; + +/** + * Where one pinned key stands: its first pin in flight (`loading`: `abandon` rejects its users' waits), following an + * attempt's member, frozen while its object is failed or activating again (both with what a new attempt prepared to + * follow, `next`), dropped (its last user left, or its object failed or ended while it loaded), or ended with its + * object (`error`, for a pin whose wait resolved just before). + */ +type EntryState = + | { readonly kind: "loading"; readonly abandon: (error: unknown) => void } + | { + readonly kind: "bound"; + readonly source: MemberSource; + readonly following: Following; + readonly next: Following | undefined; + } + | { readonly kind: "frozen"; readonly source: MemberSource; readonly next: Following | undefined } + | { readonly kind: "dropped" } + | { readonly kind: "ended"; readonly error: ChordError }; + +/** One pinned key of one keyed member: its users, the wait of its first pin, and where it stands. */ +type Entry = { + readonly member: string; + readonly key: string; + users: number; + readonly ready: Promise; + state: EntryState; +}; + +/** The pinned keys of one object's keyed members, across its attempts; every pin below them released exactly once. */ +export class KeyedMembers { + readonly #batch: Batch; + /** The object's context: a key's first load runs under it, shared by every user. */ + readonly #context: Context; + readonly #entries = new Map>(); + + constructor(batch: Batch, context: Context) { + this.#batch = batch; + this.#context = context; + } + + /** Whether no key is pinned. */ + get empty(): boolean { + return this.#entries.size === 0; + } + + /** + * Pins `key` (a stored key) of `member`, which the live attempt has as `keyed`: its member source once the key is + * loaded; `context` ends only this caller's wait. Rejects when the object fails or ends while the first pin loads. + */ + async pin(member: string, key: string, keyed: KeyedBinding, context: Context): Promise { + const entry = this.#entries.get(member)?.get(key) ?? this.#create(member, key, keyed); + entry.users++; + try { + await awaitWithContext(entry.ready, context); + } catch (error) { + this.#release(entry); + throw error; + } + const { state } = entry; + // Loaded, but its object ended before this wait resumed: nothing to pin (the entry is gone). + if (state.kind === "ended") throw state.error; + // Loaded, and its object neither failed nor ended since (a failure while loading rejects `ready` first). + const source = (state as Extract).source; + let released = false; + const release = (): void => { + if (released) return; + released = true; + this.#release(entry); + }; + return { source, release }; + } + + /** + * Loads every pinned key on `members` (a new attempt's, while it activates, before it goes live); `undefined` when + * no key is pinned. Rejects when a load fails; `rebind` or `discard` then takes what it pinned. + */ + prepare(members: ReadonlyMap, context: Context): Promise | undefined { + if (this.#entries.size === 0) return undefined; + return (async () => { + for (const byKey of this.#entries.values()) { + for (const entry of byKey.values()) { + const keyed = members.get(entry.member); + if (keyed === undefined || (entry.state.kind !== "bound" && entry.state.kind !== "frozen")) continue; + const next = { pin: await pinOn(keyed, entry.key, context), foreign: keyed.kind === "reexposed" }; + const state = entry.state; + // Dropped while it loaded: the new pin goes at once. + if (state.kind !== "bound" && state.kind !== "frozen") { + next.pin.release(); + continue; + } + state.next?.pin.release(); + entry.state = { ...state, next }; + } + } + })(); + } + + /** The new attempt went live, inside a batch: every prepared key follows its new member (one frame). */ + rebind(): void { + for (const byKey of this.#entries.values()) { + for (const entry of byKey.values()) { + const state = entry.state; + if ((state.kind !== "bound" && state.kind !== "frozen") || state.next === undefined) continue; + state.source.bind(state.next.pin.source); + if (state.kind === "bound") state.following.pin.release(); + entry.state = { kind: "bound", source: state.source, following: state.next, next: undefined }; + } + } + } + + /** The new attempt did not go live: what `prepare` pinned is released. */ + discard(): void { + for (const byKey of this.#entries.values()) { + for (const entry of byKey.values()) { + const state = entry.state; + if ((state.kind !== "bound" && state.kind !== "frozen") || state.next === undefined) continue; + state.next.pin.release(); + entry.state = { ...state, next: undefined }; + } + } + } + + /** + * The object failed live (`error`): the keys keep their values and release what they follow; `foreign` takes only + * the keys of re-exposed members (no admitted commit of this object changes them), `all` every key. A first pin + * still loading is given up at once (its users' waits reject with `error`; the pin, when it lands, is released). + */ + freeze(which: "foreign" | "all", error: unknown): void { + for (const byKey of this.#entries.values()) { + for (const entry of [...byKey.values()]) { + const state = entry.state; + if (state.kind === "loading") { + this.#abandon(entry, state, error); + continue; + } + if (state.kind !== "bound" || (which === "foreign" && !state.following.foreign)) continue; + state.source.freeze(); + state.following.pin.release(); + entry.state = { kind: "frozen", source: state.source, next: state.next }; + } + } + } + + /** The object ended, inside a batch: every key ends with `error` and releases what it pins. */ + end(error: ChordError): void { + for (const byKey of this.#entries.values()) { + for (const entry of byKey.values()) { + const state = entry.state; + if (state.kind === "loading") state.abandon(error); + else if (state.kind === "bound" || state.kind === "frozen") state.source.objectEnded(error); + this.#releaseAll(entry); + entry.state = { kind: "ended", error }; + } + } + this.#entries.clear(); + } + + /** A new entry, its first pin started under the object's context. */ + #create(member: string, key: string, keyed: KeyedBinding): Entry { + const ready = Promise.withResolvers(); + // Each user awaits `ready`; a rejection nobody waits for is not unhandled. + ready.promise.catch(() => {}); + const entry: Entry = { + member, + key, + users: 0, + ready: ready.promise, + state: { kind: "loading", abandon: ready.reject }, + }; + pinOn(keyed, key, this.#context).then( + (pin) => { + // Given up meanwhile (its object failed or ended, or every user left): the late pin goes, once. + if (entry.state.kind !== "loading") { + pin.release(); + return; + } + const source = new MemberSource(pin.source, this.#batch); + entry.state = { + kind: "bound", + source, + following: { pin, foreign: keyed.kind === "reexposed" }, + next: undefined, + }; + ready.resolve(); + }, + (error) => { + if (entry.state.kind === "loading") this.#abandon(entry, entry.state, error); + }, + ); + let byKey = this.#entries.get(member); + if (byKey === undefined) { + byKey = new Map(); + this.#entries.set(member, byKey); + } + byKey.set(key, entry); + return entry; + } + + /** Gives up `entry`'s first pin with `error`: dropped and forgotten, its users' waits rejected. */ + #abandon(entry: Entry, state: Extract, error: unknown): void { + entry.state = { kind: "dropped" }; + this.#forget(entry); + state.abandon(error); + } + + /** One user of `entry` left; the last drops it. */ + #release(entry: Entry): void { + if (--entry.users > 0) return; + this.#forget(entry); + this.#releaseAll(entry); + } + + /** Drops `entry`: what it follows and will follow is released; its source keeps its value and follows nothing. */ + #releaseAll(entry: Entry): void { + const state = entry.state; + entry.state = { kind: "dropped" }; + if (state.kind !== "bound" && state.kind !== "frozen") return; + state.source.freeze(); + if (state.kind === "bound") state.following.pin.release(); + state.next?.pin.release(); + } + + #forget(entry: Entry): void { + const byKey = this.#entries.get(entry.member); + if (byKey?.get(entry.key) !== entry) return; + byKey.delete(entry.key); + if (byKey.size === 0) this.#entries.delete(entry.member); + } +} + +/** + * Pins `key` on `keyed`: on the object's own slot it is pinned and loaded; on a re-exposed member it is pinned on the + * provider's member (whose own keys follow its attempts). + */ +function pinOn(keyed: KeyedBinding, key: string, context: Context): Promise { + if (keyed.kind === "reexposed") return keyed.binding.pinKey(keyed.member, key, context); + return pinned(keyed.slot, key, context); +} + +/** Pins `key` on `slot` and loads it when it is not in memory; a failed load releases the pin. */ +async function pinned(slot: KeyedSlot, key: string, context: Context): Promise { + const release = slot.pin(key, context); + try { + await slot.ensure(key, context); + } catch (error) { + release(); + throw error; + } + return { source: slot.source(key), release }; +} diff --git a/packages/chord/src/next/host/member-source.ts b/packages/chord/src/next/host/member-source.ts new file mode 100644 index 000000000..c5f8e6b68 --- /dev/null +++ b/packages/chord/src/next/host/member-source.ts @@ -0,0 +1,221 @@ +// Member sources (slice 6a §1.2, §1.4): a local object's state member as its references and other objects' derives +// follow it across attempts. While an attempt is live it passes that attempt's member on; while the object is failed +// or activating again it keeps the last value and gets no frames; when a new attempt goes live it re-binds to that +// attempt's member of the same name (one frame from the last value), as a remote reference's re-hold does. + +import type { Context } from "../../context/index.ts"; +import { diffRevisions, type Op } from "../../delta/index.ts"; +import type { JsonValue } from "../../json.ts"; +import { Watch, type WatchCall, type WatchEnd } from "../../watch.ts"; +import type { ChordError } from "../data/index.ts"; +import { + type Batch, + type Dependent, + type DeriveSource, + handedOut, + Source, + type Subscriber, + type WatchStart, +} from "../state/internal.ts"; + +/** + * Where a member source stands: following the live attempt's member, frozen at its last value, failed (its value + * threw when frozen, or a new attempt's variant lacks the member: a later attempt may bring it back; `last` is the + * last good value, `undefined` when there was none) or ended with its object. + */ +type MemberState = + | { readonly kind: "bound"; readonly source: DeriveSource } + | { readonly kind: "frozen"; readonly value: JsonValue } + | { readonly kind: "failed"; readonly error: unknown; readonly last: JsonValue | undefined } + | { readonly kind: "ended"; readonly error: ChordError }; + +const ENDED: WatchEnd = { kind: "ended" }; + +/** + * One state member of a local object across its attempts. Bound, it is a dependent of the attempt's member (its own + * dependents are marked with it). Its watches and subscribers (holds over a link, member sources re-exposing it) + * subscribe to the bound member itself, so the member's frames reach them as they are, with no extra hop, and a derive + * member stays lazy while nothing follows it; a freeze or a re-bind moves them. + */ +export class MemberSource extends Source implements Dependent { + #state: MemberState; + /** Its watches and subscribers, each subscribed to the bound member while bound; frozen, they get nothing. */ + readonly #passed = new Set(); + /** What its watches had at a re-bind (`undefined` for no value: the member had failed), until the re-bind's frame. */ + #framed: { readonly value: JsonValue | undefined } | undefined; + + constructor(source: DeriveSource, batch: Batch) { + super(batch); + this.#state = { kind: "bound", source }; + this.#hook(source); + } + + get value(): JsonValue { + const state = this.#state; + if (state.kind === "bound") return state.source.value; + if (state.kind === "frozen") return state.value; + throw state.error; + } + + /** What a new watch starts at: the bound member's start, the frozen value, or the failure with its last value. */ + start(): WatchStart { + const state = this.#state; + if (state.kind === "bound") { + return state.source instanceof Source ? state.source.start() : { kind: "value", value: state.source.value }; + } + if (state.kind === "frozen") return { kind: "value", value: state.value }; + if (state.kind === "failed") return { kind: "failed", error: state.error, last: state.last }; + throw state.error; + } + + /** + * A watch subscribed to the bound member itself (as `Source.openWatch` starts one: on a failed member at its last + * good value, the failure first). + */ + openWatch(context: Context): Watch { + this.batch.assertOutside("watch"); + const start = this.start(); + if (start.kind === "failed" && start.last === undefined) throw start.error; + const value = start.kind === "value" ? start.value : start.last!; + const watch: Watch = new Watch(value, () => this.unsubscribe(watch), this.batch.reportListener); + this.subscribe(watch); + if (context.abortSignal !== undefined) watch.observeAbort(context.abortSignal); + if (start.kind === "failed") watch.fail(handedOut(start.error), context); + return watch; + } + + /** Whether a watch or a subscriber follows it (through the bound member). */ + get watched(): boolean { + return this.#passed.size > 0; + } + + /** Feeds `subscriber` the bound member's frames (it subscribes there) and the re-binds' frames. */ + subscribe(subscriber: Subscriber): void { + this.#passed.add(subscriber); + const state = this.#state; + if (state.kind === "bound" && state.source instanceof Source) state.source.subscribe(subscriber); + } + + /** As `Source.unsubscribe`. */ + unsubscribe(subscriber: Subscriber): void { + this.#passed.delete(subscriber); + const state = this.#state; + if (state.kind === "bound" && state.source instanceof Source) state.source.unsubscribe(subscriber); + } + + /** The bound member changed: its dependents are marked (its frames reach the subscribers directly). */ + invalidate(): void { + for (const dependent of this.dependents) dependent.invalidate(); + } + + /** The bound member's document was retired: its dependents carry the flag on. */ + sourceRetired(visited: Set): void { + for (const dependent of this.dependents) dependent.sourceRetired(visited); + } + + /** Its object failed live and its admitted commits adopted: it keeps the member's value (or error) from now on. */ + freeze(): void { + const state = this.#state; + if (state.kind !== "bound") return; + this.#leave(); + const start = state.source instanceof Source ? state.source.start() : undefined; + if (start === undefined || start.kind === "value") { + this.#state = { kind: "frozen", value: start === undefined ? state.source.value : start.value }; + } else { + this.#state = { kind: "failed", error: start.error, last: start.last }; + } + } + + /** + * A new attempt went live: follows its member `source`, inside a batch. Watches get one frame from the last value + * (a full one after a failure); derives downstream are marked and recompute, a failed one included (self-healing). + */ + bind(source: DeriveSource): void { + if (this.#state.kind === "ended") return; + this.#leave(); + if (this.watched) this.#framed = { value: this.#shown() }; + this.#state = { kind: "bound", source }; + this.#hook(source); + // Marks the derives downstream: a failed one recomputes too (self-healing). + this.changed(); + } + + /** + * A new attempt's variant lacks the member, inside a batch: reads throw `error`; watches and subscribers get one + * failure call and stay open (a later attempt may bring the member back, as `[["r", value]]`). + */ + vanish(error: ChordError, context: Context): void { + if (this.#state.kind === "ended") return; + const last = this.#lastGood(); + this.#leave(); + this.#state = { kind: "failed", error, last }; + this.#call({ kind: "failed", error: handedOut(error) }, context); + this.changed(); + } + + /** + * Its object ended, inside a batch: reads throw `error`, watches end `ended`, and derives over it are marked (their + * next read throws it, no cached value stays). Its other subscribers end with their owners (a hold with its object). + */ + objectEnded(error: ChordError): void { + this.#leave(); + this.#state = { kind: "ended", error }; + this.#framed = undefined; + // A copy: an ending watch unsubscribes itself. + for (const subscriber of [...this.#passed]) subscriber.end(ENDED); + this.#passed.clear(); + for (const dependent of this.dependents) dependent.invalidate(); + } + + /** Delivers a re-bind's frame: the diff from the watches' value to the new member's. */ + deliver(context: Context): void { + const framed = this.#framed; + this.#framed = undefined; + if (framed === undefined) return; + let after: JsonValue; + try { + after = this.value; + } catch (error) { + if (framed.value !== undefined) this.#call({ kind: "failed", error: handedOut(error) }, context); + return; + } + const before = framed.value; + if (after === before) return; + const ops: readonly Op[] = before === undefined ? [["r", after]] : diffRevisions(before, after); + if (ops.length > 0) this.#call({ kind: "value", value: after, ops, retired: false }, context); + } + + /** A call of its own (a re-bind's frame, a vanished member) to its watches and subscribers. */ + #call(call: WatchCall, context: Context): void { + for (const subscriber of this.#passed) subscriber.advance(call, context); + } + + /** The value its watches have before a re-bind; `undefined` when they had a failure (their next value is whole). */ + #shown(): JsonValue | undefined { + const start = this.start(); + return start.kind === "value" ? start.value : undefined; + } + + /** Its last good value, which watches opened while it is failed start at; `undefined` when it never had one. */ + #lastGood(): JsonValue | undefined { + const start = this.start(); + return start.kind === "value" ? start.value : start.last; + } + + /** Starts reading `source`: its changes mark this source's dependents, its frames reach the subscribers. */ + #hook(source: DeriveSource): void { + if (!(source instanceof Source)) return; + source.dependents.add(this); + for (const subscriber of this.#passed) source.subscribe(subscriber); + } + + /** Stops following the bound source, when bound: no more marks or frames from it. */ + #leave(): void { + const state = this.#state; + if (state.kind !== "bound") return; + const { source } = state; + if (!(source instanceof Source)) return; + source.dependents.delete(this); + for (const subscriber of this.#passed) source.unsubscribe(subscriber); + } +} diff --git a/packages/chord/src/next/host/members.ts b/packages/chord/src/next/host/members.ts new file mode 100644 index 000000000..7eaf1fc73 --- /dev/null +++ b/packages/chord/src/next/host/members.ts @@ -0,0 +1,124 @@ +// The member check (Q14): an implementation is one variant with exactly that variant's members, methods being +// functions and state members JSON watchables; and the resolution of every derive's declared sources. +// +// The rule for watches across attempts (slice 6a §1.2, §1.4): references and other objects' derives follow a local +// object's state member through its member source (`member-source.ts`), and a key of a keyed member they hold (`at`, +// `watch(key)`) through a member source per key (`keyed-members.ts`); both live as long as the object. While the +// object is failed or activating again they keep their last values and get no frames; when a new attempt goes live +// they re-bind by name to that attempt's member (a pinned key loaded on it first), whether a document or a derive: one +// frame from the last value, and every derive downstream recovers, one that failed on the old member's error included +// (decision "self healing"). A re-exposed member is the re-exposing object's: it freezes with that object, its +// provider's source goes on. At a live failure, members following another object's sources freeze at once; the +// object's own documents and derives freeze once its admitted commits adopted, so until then an own derive over +// another object's member may still change. Holds over a link follow the live attempt's members and take a snapshot at +// each binding. + +import type { Context } from "../../context/index.ts"; +import type { Contract } from "../contracts/index.ts"; +import { currentVariants, LOCAL, takesNoArgs, type VariantData } from "../contracts/internal.ts"; +import type { KeyedSlot } from "../documents/internal.ts"; +import type { Derive, DeriveSource } from "../state/internal.ts"; +import { type Binding, resolveKeyed, resolveWatchable } from "./reference.ts"; + +/** A checked implementation, as Chord calls and follows it. */ +export type Members = { + /** The methods, each called with its arguments (`null` for none) and a context. */ + readonly methods: ReadonlyMap; + /** The state members, resolved to what Chord follows. */ + readonly state: ReadonlyMap; + /** The keyed members (slice 6a §12): the object's own slot, or the provider's member it re-exposes. */ + readonly keyed: ReadonlyMap; +}; + +/** + * What a keyed member is: a keyed document handle's slot (`own`), or another object's keyed member re-exposed through + * a reference (`reexposed`), which reads end at the ultimate provider's slot (rule 22) and pinned keys follow. + */ +export type KeyedBinding = + | { readonly kind: "own"; readonly slot: KeyedSlot } + | { readonly kind: "reexposed"; readonly binding: Binding; readonly member: string }; + +/** A method as Chord calls it. */ +type Method = (args: unknown, context: Context) => Promise; + +/** + * Checks what a factory returned (plugin code): a local contract's members, or a remote contract's `{ variant, + * members }` naming one of its current variants. Errors name the variant or member (the failure cause adds the + * object). + */ +export function checkImplementation( + contract: Contract, + implementation: unknown, +): { readonly variant: string | undefined; readonly members: Members } { + const variants = currentVariants(contract); + if (contract.kind === "local") { + // The member check establishes the shape. + return { variant: undefined, members: checkMembers(variants.get(LOCAL)!, implementation as Implementation) }; + } + // Plugin code: a remote factory returns `{ variant, members }`, which the checks below establish. + const { variant, members } = implementation as { readonly variant: string; readonly members: Implementation }; + const data = variants.get(variant); + if (data === undefined) throw new Error(`variant ${JSON.stringify(variant)}: not a variant of ${contract.name}`); + return { variant, members: checkMembers(data, members) }; +} + +type Implementation = { readonly [member: string]: unknown }; + +/** + * Checks members against a variant's (exactly its members, methods being functions, state members JSON watchables and + * keyed members keyed document handles); a method without arguments is called with the context alone. + */ +function checkMembers({ state, methods, keyed }: VariantData, implementation: Implementation): Members { + for (const member of Object.keys(implementation)) { + if (!state.has(member) && !methods.has(member) && !keyed.has(member)) throw new Error(`${member}: not a member`); + } + // Own properties only: an inherited `toString` is no implementation of a member named so. + const own = (member: string): unknown => + Object.hasOwn(implementation, member) ? implementation[member] : undefined; + const checked = { + methods: new Map(), + state: new Map(), + keyed: new Map(), + }; + for (const [member, data] of methods) { + const method = own(member); + if (typeof method !== "function") throw new Error(`${member}: a method is a function`); + const call: Method = takesNoArgs(data) + ? (_args, context) => method(context) + : (args, context) => method(args, context); + checked.methods.set(member, call); + } + for (const member of state.keys()) { + const source = resolveWatchable(own(member)); + if (source === undefined) throw new Error(`${member}: a state member is a JSON watchable`); + checked.state.set(member, source); + } + for (const member of keyed.keys()) { + const resolved = resolveKeyed(own(member)); + if (resolved === undefined) throw new Error(`${member}: a keyed member is a keyed document handle`); + checked.keyed.set(member, resolved); + } + return checked; +} + +/** + * Resolves every derive's sources; errors name a derive by the member exposing it, else by its position. `report` + * makes each derive's failure report, with the member exposing it when there is one (`derive_failed`). + */ +export function attachDerives( + derives: readonly Derive[], + state: ReadonlyMap, + report: (member: string | undefined) => (error: unknown) => void, +): void { + const exposed = new Map([...state].map(([member, source]) => [source, member])); + derives.forEach((derive, position) => { + const member = exposed.get(derive); + const name = member ?? `derive #${position + 1}`; + const resolve = (source: unknown, index: number): DeriveSource => { + const resolved = resolveWatchable(source); + if (resolved === undefined) throw new Error(`${name}: source ${index} is not a JSON watchable`); + return resolved; + }; + derive.attach(resolve, report(member)); + }); +} diff --git a/packages/chord/src/next/host/native.ts b/packages/chord/src/next/host/native.ts new file mode 100644 index 000000000..c0dc190f4 --- /dev/null +++ b/packages/chord/src/next/host/native.ts @@ -0,0 +1,151 @@ +// Natives (§13): the handle `self.open` returns, a value opened during activation and closed with a reason when its +// attempt ends; `within`, which bounds a close or a drain by a timeout; and the drain's admission of running work. + +import { type Context, withAbortSignal } from "../../context/index.ts"; +import { ChordError } from "../data/index.ts"; +import type { HostError } from "./host.ts"; + +/** Why a native closes: its object ended or its attempt failed (`end`), the host closes (`close`), a reload (slice 7). */ +export type NativeCloseReason = { readonly kind: "end" } | { readonly kind: "close" } | { readonly kind: "reload" }; + +/** A native's handle. */ +export interface Native { + /** The opened value: throws `not_ready` until its open resolved and `released` once its close started. */ + readonly value: T; +} + +/** What `self.open` takes besides the open and the close. */ +export type NativeOptions = { + /** How long its close may take; the host's `closeTimeout` by default. */ + readonly closeTimeout?: number; +}; + +/** A close that threw, as reported. */ +export type CloseFailed = Extract; + +/** + * Where a native belongs: its object's label and its position in declaration order (from 1) for errors and reports, + * the object's context (every close runs under it, never under the aborted opener's), and where it reports. + */ +type NativeOwner = { + readonly object: string; + readonly position: number; + readonly context: Context; + readonly report: (error: HostError) => void; +}; + +/** What an attempt does with each of its natives, whatever their value's type. */ +export interface NativeSlot { + open(context: Context): Promise; + close(reason: NativeCloseReason): Promise; +} + +/** Where a native stands. `abandoned`: closed while its open was pending, so the late value is closed at once. */ +type NativeState = + | { readonly kind: "unopened" } + | { readonly kind: "opening" } + | { readonly kind: "open"; readonly value: T } + | { readonly kind: "abandoned"; readonly reason: NativeCloseReason } + | { readonly kind: "closed" }; + +/** One native of one attempt: opened at most once, closed exactly once. */ +export class NativeHandle implements Native, NativeSlot { + readonly #open: (context: Context) => Promise; + readonly #close: (value: T, reason: NativeCloseReason, context: Context) => void | Promise; + readonly #closeTimeout: number; + readonly #owner: NativeOwner; + #state: NativeState = { kind: "unopened" }; + + constructor( + open: (context: Context) => Promise, + close: (value: T, reason: NativeCloseReason, context: Context) => void | Promise, + closeTimeout: number, + owner: NativeOwner, + ) { + this.#open = open; + this.#close = close; + this.#closeTimeout = closeTimeout; + this.#owner = owner; + } + + get value(): T { + const state = this.#state; + if (state.kind === "open") return state.value; + const name = this.#name(); + if (state.kind === "unopened" || state.kind === "opening") { + throw new ChordError({ code: "not_ready", message: `${name}: not open yet` }); + } + throw new ChordError({ code: "released", reason: "closed", message: `${name}: closed` }); + } + + /** Opens the value; one that arrives after its attempt was abandoned is closed at once, a failure reported. */ + async open(context: Context): Promise { + this.#state = { kind: "opening" }; + await this.#opened(await this.#open(context)); + } + + /** Closes the value if it is open (a pending open's value is closed when it arrives); resolves with a throw. */ + async close(reason: NativeCloseReason): Promise { + const state = this.#state; + this.#state = state.kind === "opening" ? { kind: "abandoned", reason } : { kind: "closed" }; + return state.kind === "open" ? this.#closeValue(state.value, reason) : undefined; + } + + /** Keeps the value an open resolved with, or closes it at once when the native was closed meanwhile. */ + async #opened(value: T): Promise { + const state = this.#state; + if (state.kind !== "abandoned") { + this.#state = { kind: "open", value }; + return; + } + this.#state = { kind: "closed" }; + const failure = await this.#closeValue(value, state.reason); + if (failure !== undefined) this.#owner.report(failure); + } + + /** + * Runs the close under the object's context, aborted at the close timeout, and waits for it up to then; longer is + * reported. + */ + async #closeValue(value: T, reason: NativeCloseReason): Promise { + const { object, position, context, report } = this.#owner; + const at = { object, native: { kind: "native", position } } as const; + const timeout = new AbortController(); + const closing = (async () => this.#close(value, reason, withAbortSignal(timeout.signal, context)))(); + try { + if ((await within(closing, this.#closeTimeout)) === "timeout") { + timeout.abort(new ChordError({ code: "aborted", message: `${this.#name()}: close timed out` })); + report({ kind: "close_timeout", ...at }); + } + return undefined; + } catch (error) { + return { kind: "close_failed", ...at, error }; + } + } + + #name(): string { + return `${this.#owner.object} native #${this.#owner.position}`; + } +} + +/** Whether `promise` settled within `ms` (it rejects when `promise` does first); `promise` itself runs on. */ +export async function within(promise: Promise, ms: number): Promise<"settled" | "timeout"> { + let timer: ReturnType | undefined; + const timeout = new Promise<"timeout">((resolve) => { + timer = setTimeout(resolve, ms, "timeout"); + }); + try { + return await Promise.race([promise.then(() => "settled" as const), timeout]); + } finally { + clearTimeout(timer); + } +} + +/** Whether the `size` pieces of work in `drain` settled within `ms` (at once when there is none). */ +export async function drained( + drain: Iterable>, + size: number, + ms: number, +): Promise<"settled" | "timeout"> { + return size === 0 ? "settled" : within(Promise.allSettled(drain), ms); +} diff --git a/packages/chord/src/next/host/object.ts b/packages/chord/src/next/host/object.ts new file mode 100644 index 000000000..ebdf37b4a --- /dev/null +++ b/packages/chord/src/next/host/object.ts @@ -0,0 +1,691 @@ +// Objects (§6): one per identity, from its first attempt to its end. Its state across attempts (its documents, line +// and job runs, its member sources, slice 6a §1.2), its holders, retries after failures (backoff, dependency recovery +// R18, `ref.retry()`), a live object's failure (slice 6a §1.4), the waits and cycle checks of activations (Q5, its +// instance id and the activation chain it was created for), lingering after its last reference (§3, D8), the job runs +// that keep it alive (§15), and its ending, revocation included (§4, R11). + +import { awaitWithContext, type Context } from "../../context/index.ts"; +import type { Contract, ParamsOf } from "../contracts/index.ts"; +import type { ContractShape } from "../contracts/internal.ts"; +import { ChordError, type FailureCause, isChordError, type Principal } from "../data/index.ts"; +import { invalid, type Provider } from "../data/internal.ts"; +import { Documents, type KeyedSlot } from "../documents/internal.ts"; +import type { JobDefinition } from "../jobs/index.ts"; +import { JobRecords } from "../jobs/internal.ts"; +import { CHAIN } from "../link/internal.ts"; +import type { DeriveSource } from "../state/internal.ts"; +import { Attempt, type Self } from "./attempt.ts"; +import type { Caller } from "./ctx.ts"; +import type { AnyHost } from "./host.ts"; +import { KeyedMembers, type PinnedKey } from "./keyed-members.ts"; +import { MemberSource } from "./member-source.ts"; +import type { KeyedBinding } from "./members.ts"; +import type { CloseFailed, NativeCloseReason } from "./native.ts"; +import { type Ended, type Failed, LIVE, LOADING, type RefState } from "./ref-state.ts"; +import type { Target } from "./reference.ts"; +import { uuidv7 } from "./uuid.ts"; + +/** A `local` contract, as a served type has it. */ +export type LocalContract = Contract; + +/** A `remote` contract, as a served type has it. */ +export type RemoteContract = Contract; + +/** + * A type served on a host, by kind: its contract, factory (a remote one gets the object's principal) and argument + * checks. Factories are plugin code: what they return is checked at activation. + */ +export type Served = + | (ServedType & { + readonly kind: "local"; + readonly factory: (params: ParamsOf, self: Self) => unknown; + /** A type with empty params: its object lives as long as the host (Q-A). */ + readonly hostLifetime: boolean; + }) + | (ServedType & { + readonly kind: "remote"; + readonly factory: (params: ParamsOf, principal: Principal, self: Self) => unknown; + }); + +type ServedType = { + readonly contract: C; + readonly provider: Provider; + /** How long a live object stays live after its last reference, in milliseconds (`serve`'s `linger`, 0 by default). */ + readonly linger: number; +}; + +/** What holds an object: a reference's binding, or a hold arriving over a link. */ +export interface Holder { + /** The object whose attempt declared the reference; retried when this object goes live (R18). */ + readonly dependent: ChordObject | undefined; + /** Whom the reference acts for, with its origin's incarnation (`host.references`). */ + readonly caller: Caller; + /** The object's state (`state`) changed: an attempt started after a failure, it went live, or it failed. */ + moved(): void; + /** The object ended with `error`. */ + end(error: ChordError): void; +} + +/** + * Where an object stands. `waiting`: for its identity's backoff window before its first attempt. `failed` has the + * state its references show, and keeps the failed attempt in `holding` after a dependency failure (R18): its + * references keep the dependencies. + */ +type ObjectState = + | { readonly kind: "waiting" } + | { readonly kind: "activating"; readonly attempt: Attempt } + | { readonly kind: "live"; readonly attempt: Attempt } + | { readonly kind: "failed"; readonly failure: Failed; readonly holding: Attempt | undefined } + | { readonly kind: "ended"; readonly ended: Ended }; + +/** + * What an ended object still has to do once its previous teardowns finished (`previous`): drain and close its attempt, + * or release a failed attempt's references; then drop its documents and job runs (`end` alone). + */ +type Teardown = + | { readonly kind: "none" } + | { readonly kind: "end"; readonly previous: Promise | undefined } + | { readonly kind: "close" | "release"; readonly attempt: Attempt; readonly previous: Promise | undefined }; + +const END: NativeCloseReason = { kind: "end" }; + +const CLOSE: NativeCloseReason = { kind: "close" }; + +/** + * One object. Each attempt is a fresh `Attempt`; the documents, the line and the job runs, the holders (and, after a + * dependency failure, the failed attempt's references) outlive it. A transient failure is retried after its backoff window, or at once when a + * dependency it failed on goes live or a reference asks (`retry`); a broken one stays failed until a reference asks. + * The object ends with its last reference, a live one after its type's `linger` (never, for a host-lifetime object), + * or when the host closes. + */ +export class ChordObject implements Target { + readonly host: AnyHost; + readonly served: Served; + readonly params: ParamsOf; + /** The principal in a remote object's identity, which its factory gets; `undefined` for a local object. */ + readonly principal: Principal | undefined; + readonly identity: string; + /** The object in errors, reports and cycles: its type, and its params key when it has params. */ + readonly label: string; + /** + * The activation chain it was created for (Q5): the instance ids of the activations waiting for it, from the hold + * that created it (a trusted link's) or the activation here that declared it; empty otherwise. + */ + readonly chain: readonly string[]; + /** Its documents and line, across its attempts. */ + readonly documents: Documents; + /** The context of every attempt: its creator's, without the creator's cancellation. */ + readonly #context: Context; + /** Its job runs by job name, across its attempts. */ + readonly #jobs = new Map(); + /** This instance's id (uuidv7), minted at its first attempt: a restarted object is a new instance (D4). */ + #instance: string | undefined; + #state: ObjectState = { kind: "waiting" }; + /** References and holds of this object. */ + readonly #holders = new Set(); + /** Objects this object's activation waits for; followed to find cycles (Q5). */ + readonly #waitingFor = new Set(); + /** The current attempt's outcome: resolves when it goes live, rejects with its error; renewed by every retry. */ + #outcome = settlement(); + /** The error of the last failed attempt, until one goes live. */ + #lastFailure: ChordError | undefined; + /** Ends the object when its type's `linger` passed after its last reference; cleared by a new one. */ + #lingering: ReturnType | undefined; + /** Its running and finishing job runs: they keep it alive like references (D4), without being references. */ + #runs = 0; + /** + * The attempt its member sources show: the live one, or the last one that went live while the object is failed or + * activating again (their last values); `undefined` before the first went live. + */ + #exposed: Attempt | undefined; + /** Its state members as references and other objects' derives follow them, by name; each made by its first use. */ + readonly #members = new Map(); + /** The keys of its keyed members that references hold, across attempts. */ + readonly #keys: KeyedMembers; + + constructor( + host: AnyHost, + served: Served, + params: ParamsOf, + principal: Principal | undefined, + key: string, + identity: string, + context: Context, + chain: readonly string[], + ) { + this.host = host; + this.served = served; + this.params = params; + this.principal = principal; + this.identity = identity; + this.label = key === "" ? served.contract.name : `${served.contract.name}(${key})`; + this.chain = chain; + this.#context = context; + this.#keys = new KeyedMembers(host.batch, context); + const { store } = host; + this.documents = new Documents({ + label: this.label, + batch: host.batch, + context, + store: + store === undefined + ? undefined + : { store, owner: `${host.address?.kind ?? "client"}:${served.contract.name}` }, + fail: (error) => { + if (this.#state.kind === "live") this.failLive(this.#state.attempt, error); + }, + }); + } + + /** The object's state as its references show it. */ + get state(): RefState { + const state = this.#state; + if (state.kind === "failed") return state.failure; + if (state.kind === "live") return LIVE; + return state.kind === "ended" ? state.ended : LOADING; + } + + /** Whether its state members have values: once live (a local object stays live until it ends). */ + get hydrated(): boolean { + return this.#state.kind === "live"; + } + + /** Whether its state members have values while it is not live: once an attempt went live, until it ends. */ + get lastValues(): boolean { + return this.#exposed !== undefined && this.#state.kind !== "ended"; + } + + /** The current attempt's outcome (the start barrier waits for the first). */ + get settled(): Promise { + return this.#outcome.promise; + } + + /** The error of the last failed attempt while none went live since. */ + get lastFailure(): ChordError | undefined { + return this.#lastFailure; + } + + /** Its instance id while an attempt activates; `undefined` otherwise (a hold's chain names only activations). */ + get activationId(): string | undefined { + return this.#state.kind === "activating" ? this.#instance : undefined; + } + + /** + * The chain a hold this object's activation makes over a link carries (Q5): its own chain and its instance id, the + * last 32; `undefined` when it is not activating. + */ + activationChain(): string[] | undefined { + const id = this.activationId; + if (id === undefined) return undefined; + const { chain } = this; + return [...(chain.length < CHAIN ? chain : chain.slice(chain.length - CHAIN + 1)), id]; + } + + /** Its holders, for a presence list made after they took it. */ + get holders(): ReadonlySet { + return this.#holders; + } + + /** The variant a live remote object's factory chose; `undefined` for a local object and before it is live. */ + get variant(): string | undefined { + return this.#state.kind === "live" ? this.#state.attempt.variant : undefined; + } + + /** The live object's state members, as Chord follows them. */ + get stateMembers(): ReadonlyMap { + return this.#live().stateMembers; + } + + /** + * Starts an attempt when waiting or failed; a trigger while an attempt runs (or once live) is ignored. After a + * failure the holders are told: their references are loading again. + */ + attempt(): void { + const state = this.#state; + if (state.kind !== "waiting" && state.kind !== "failed") return; + this.#instance ??= uuidv7(); + const attempt = new Attempt(this, state.kind === "failed" ? state.holding : undefined, this.#context); + this.#state = { kind: "activating", attempt }; + if (state.kind === "failed") { + this.#outcome = settlement(); + this.#notify(); + } + void this.#run(attempt); + } + + /** + * `ref.retry()` and a `retry` message: an attempt now when failed (a broken one too) or waiting, its backoff window + * skipped; ignored while an attempt runs and once live. + */ + retry(): void { + if (this.#state.kind !== "waiting" && this.#state.kind !== "failed") return; + this.host.backoff.skip(this.identity); + this.attempt(); + } + + /** Calls a method of the live object for `caller`. */ + async call(method: string, args: unknown, context: Context, caller: Caller): Promise { + return this.#live().call(method, args, context, caller); + } + + /** + * The state member `member` as references and other objects' derives follow it across attempts: its member source, + * with its last value while the object is not live (`lastValues`). + */ + member(member: string): DeriveSource { + // Made once per member and dropped at the end: a source found here is the one to follow. + const made = this.#members.get(member); + if (made !== undefined) return made; + const exposed = this.#exposed; + if (exposed === undefined || this.#state.kind === "ended") return this.#live().member(member); + return this.#memberSource(member, exposed); + } + + /** The keyed member `member` of the live object: its ultimate slot (slice 6a §12). */ + keyedMember(member: string): KeyedSlot { + return this.#live().keyedMember(member); + } + + /** Pins `key` of the live object's keyed member `member` across its attempts (`KeyedMembers.pin`). */ + async pinKey(member: string, key: string, context: Context): Promise { + const attempt = this.#live(); + const stored = attempt.keyedMember(member).key(key); + return this.#keys.pin(member, stored, attempt.keyedMembers.get(member)!, context); + } + + /** + * An activating attempt's keyed members are checked: the keys references pin are loaded on them before it goes + * live; `undefined` when none is held. + */ + prepareKeys(members: ReadonlyMap, context: Context): Promise | undefined { + return this.#keys.prepare(members, context); + } + + /** + * Counts `holder` as a reference to this object (the holder reads `state` itself), listed in its type's presence; a + * lingering object stays. + */ + hold(holder: Holder): void { + this.#holders.add(holder); + this.#stopLingering(); + // Last: a derive over the presence list may close a link synchronously, which may release this holder again. + this.host.referenced(this, holder, this.#context); + } + + /** `holder` was released: the object may end (`#idle`). */ + unhold(holder: Holder): void { + if (!this.#holders.delete(holder)) return; + this.#idle(); + // Last: a derive over the presence list may close a link synchronously, which releases other holders again. + this.host.unreferenced(this, [holder], this.#context); + } + + /** + * The records of job `definition`'s runs, made by its first declaration. A later declaration of another version is + * refused (versions of runs across attempts are slice 7's). + */ + jobRecords(definition: JobDefinition): JobRecords { + const records = this.#jobs.get(definition.name); + if (records === undefined) { + const owner = { + label: this.label, + host: this.host, + context: this.#context, + documents: this.documents, + countRuns: (delta: number) => this.countRuns(delta), + }; + const made = new JobRecords(definition, owner); + this.#jobs.set(definition.name, made); + return made; + } + if (records.definition.version !== definition.version) { + throw new Error( + `${this.label}: job "${definition.name}" is declared at version ${definition.version}, its runs are at ${records.definition.version}`, + ); + } + return records; + } + + /** + * A runtime failure of the live attempt `attempt` (a `migrate` failing on access, a stored version newer than the + * code): the attempt is sealed and the object `failed` with the broken `cause`; calls reject with it, holders are + * told, the state members keep their values once the admitted commits adopted, the natives close in the background + * (then its references are released), `onError` gets `object_failed` once. Documents and job runs stay. Recovery: + * `ref.retry()` (a new attempt, whose members the member sources follow once it is live). A failure of a sealed + * attempt is ignored. + */ + failLive(attempt: Attempt, error: ChordError): void { + const state = this.#state; + if (state.kind !== "live" || state.attempt !== attempt) return; + const cause: FailureCause = { kind: "broken", error }; + this.host.backoff.delete(this.identity); + const failure: Failed = { kind: "failed", cause, retryAt: undefined }; + this.#state = { kind: "failed", failure, holding: undefined }; + this.#lastFailure = error; + this.#outcome = settlement(); + this.#outcome.reject(error); + attempt.seal(error); + // Members following another object's sources freeze now: no commit of this object changes them. + this.#freeze(attempt, "foreign"); + // The commits admitted before the seal adopt and publish once (frames included); then the other members keep + // their values. A retry's factory and hydration wait for both (its ending), so no old commit lands in a new + // attempt. + const frozen = this.documents.settled().then(() => this.#freeze(attempt, "all")); + const closing = Promise.all([attempt.close(END), frozen]).then(([failures]) => { + attempt.detach(); + attempt.releaseReferences(); + return failures; + }); + this.host.ending(this.identity, this.#reported(closing)); + this.host.report({ kind: "object_failed", object: this.label, failure: cause }); + this.#notify(); + this.#idle(); + } + + /** + * Adds `delta` to its running and finishing job runs (and `host.activity.jobs`). A new run keeps a lingering object; + * the last one settling on an object without references ends it as its last release would (`#idle`). + */ + countRuns(delta: number): void { + this.#runs += delta; + this.host.count("jobs", delta); + // A run started while the object lingers keeps it, like a new reference. + if (delta > 0) this.#stopLingering(); + if (this.#runs === 0) this.#idle(); + } + + /** + * Throws `invalid/cycle` when waiting for `dependency` closes a chain of waits back to this object (Q5). Every other + * object on the chain fails broken with the same error, so no part of a cycle waits for a retry. Counts the objects + * it visits (`chord.cycle.visits`). Otherwise, a dependency whose activation is in this object's chain closes a + * cycle through other hosts (they fail through their dependency on this one). + */ + assertNoCycle(dependency: Target, context: Context): void { + const visited = new Set(); + const cycle = dependency.pathTo(this, visited); + context.telemetry.metrics?.counter("chord.cycle.visits").add(visited.size); + if (cycle === undefined) { + const across = dependency.activationId; + if (across === undefined || !this.chain.includes(across)) return; + throw this.host.cycle(this.chain, across, [this.label, dependency.label]); + } + const error = invalid("cycle", `dependency cycle: ${[this, ...cycle].map(({ label }) => label).join(" → ")}`); + for (const object of cycle) { + const state = object.#state; + if (object !== this && state.kind === "activating") { + state.attempt.abandon({ kind: "failed", cause: { kind: "broken", error } }); + } + } + throw error; + } + + /** + * Waits for `dependency`'s current attempt as part of this object's activation. Rejects at once when it is failed, + * and when its attempt fails, with `unavailable` naming it (R18), its error as the cause; a dependency that failed + * for a cycle rejects with that `invalid/cycle` itself. + */ + async waitFor(dependency: Target, context: Context): Promise { + this.#waitingFor.add(dependency); + try { + await awaitWithContext(dependency.settled, context); + } catch (error) { + if (isChordError(error, "invalid") && error.reason === "cycle") throw error; + const message = `${this.label}: dependency ${dependency.label} failed`; + throw new ChordError({ code: "unavailable", message }, { cause: error }); + } finally { + this.#waitingFor.delete(dependency); + } + } + + /** A dependency went live: retried at once (backoff window skipped) when it failed on a dependency (R18). */ + dependencyLive(): void { + if (this.#state.kind !== "failed" || this.#state.holding === undefined) return; + this.host.backoff.skip(this.identity); + this.attempt(); + } + + /** + * Ends the object (once): sealed now, its holders and running calls told `error` (`released` by default), torn down + * in the background with reason `end`, close failures reported. + */ + end(error: ChordError = this.ended()): void { + const teardown = this.#seal(error); + if (teardown.kind !== "none") this.host.ending(this.identity, this.#reported(this.#tearDown(teardown, END))); + } + + /** + * Ends the object for `host.close`: sealed now; returns its teardown (reason `close`), which the host runs once every + * object is sealed, so no reference it releases starts an ending in the background. + */ + stop(): () => Promise { + const teardown = this.#seal(this.ended()); + return () => this.#tearDown(teardown, CLOSE); + } + + /** The `released` error of this object. */ + ended(): ChordError { + return new ChordError({ code: "released", reason: "ended", message: `${this.label}: ended` }); + } + + async #run(attempt: Attempt): Promise { + const outcome = await attempt.activate(this.host.endingOf(this.identity)); + // The object may have ended after the activation's last step. + if (this.#state.kind !== "activating" || this.#state.attempt !== attempt) return; + switch (outcome.kind) { + case "live": + return this.#wentLive(attempt); + case "failed": + return this.#failed(attempt, outcome.cause, undefined); + case "dependency_failed": + return this.#failed(attempt, outcome.cause, attempt); + } + } + + #wentLive(attempt: Attempt): void { + this.#state = { kind: "live", attempt }; + this.#exposed = attempt; + if (this.#members.size > 0 || !this.#keys.empty) + this.host.batch.within(() => this.#rebind(attempt), this.#context); + this.#lastFailure = undefined; + this.host.wentLive(this); + this.#outcome.resolve(); + this.#notify(); + // R18: objects that failed because this one had failed are retried now. + for (const holder of [...this.#holders]) holder.dependent?.dependencyLive(); + attempt.live(); + } + + /** + * Records a failed attempt: reported, torn down in the background (reason `end`), its references released unless it + * is `holding` them (R18), its holders told. Transient: a backoff window, whose end is the failure's `retryAt`; + * broken: no automatic retry. + */ + #failed(attempt: Attempt, cause: FailureCause, holding: Attempt | undefined): void { + // What its activation loaded for the pinned keys goes; they stay frozen. + this.#keys.discard(); + if (cause.kind === "transient") this.host.backoff.failed(this.identity); + else this.host.backoff.delete(this.identity); + const failure: Failed = { kind: "failed", cause, retryAt: this.host.backoff.until(this.identity) }; + this.#state = { kind: "failed", failure, holding }; + this.#lastFailure = cause.error; + attempt.seal(this.ended()); + if (holding === undefined) attempt.releaseReferences(); + const closing = attempt.close(END).then((failures) => { + attempt.detach(); + return failures; + }); + this.host.ending(this.identity, this.#reported(closing)); + this.host.report({ kind: "activation_failed", object: this.label, failure: cause }); + this.#outcome.reject(cause.error); + this.#notify(); + // Job runs kept it while it activated; failed, they keep it no longer. + this.#idle(); + } + + /** The member source of `member`, made bound to `exposed`'s member by its first use. */ + #memberSource(member: string, exposed: Attempt): MemberSource { + let source = this.#members.get(member); + if (source === undefined) { + source = new MemberSource(exposed.member(member), this.host.batch); + this.#members.set(member, source); + } + return source; + } + + /** + * The attempt `attempt` failed live: its state members and pinned keys keep their values from now on (each made now + * when nothing used it yet, so reads while failed find it). `foreign`: those following another object's sources + * (at once); `all`: every one (once the admitted commits adopted). + */ + #freeze(attempt: Attempt, which: "foreign" | "all"): void { + if (this.#exposed !== attempt || this.#state.kind === "ended") return; + for (const [member, source] of attempt.stateMembers) { + if (which === "all" || !attempt.owns(source)) this.#memberSource(member, attempt).freeze(); + } + // A pin still loading is given up with the failure: a consumer activating on it fails transient (R18). + this.#keys.freeze(which, this.#lastFailure ?? this.ended()); + } + + /** + * A new attempt went live, inside a batch: every member source follows its member of the same name, every pinned key + * its keyed member's (loaded by its activation). + */ + #rebind(attempt: Attempt): void { + const members = attempt.stateMembers; + for (const [member, source] of this.#members) { + const found = members.get(member); + if (found !== undefined) { + source.bind(found); + continue; + } + const variant = attempt.variant === undefined ? "" : ` (variant ${attempt.variant})`; + source.vanish(invalid("member", `${this.label}${variant} has no state member "${member}"`), this.#context); + } + this.#keys.rebind(); + } + + /** Tells every holder that the state changed. */ + #notify(): void { + for (const holder of [...this.#holders]) holder.moved(); + } + + /** + * Marks the object ended: forgotten by the host, waits rejected, references to it ended with `error` (and gone from + * its type's presence), its job runs ended (signals fired, waiters rejected), its transactions refused, its attempt + * sealed. Returns what is left to tear down; it releases nothing itself. + */ + #seal(error: ChordError): Teardown { + const state = this.#state; + if (state.kind === "ended") return { kind: "none" }; + const previous = this.host.endingOf(this.identity); + this.#state = { kind: "ended", ended: { kind: "ended", error } }; + for (const records of this.#jobs.values()) records.end(error); + this.documents.seal(); + this.#stopLingering(); + this.host.forget(this); + this.#outcome.reject(error); + // In a batch: the derives over its members are marked, so none keeps a cached value. + this.host.batch.within(() => { + for (const source of this.#members.values()) source.objectEnded(error); + this.#keys.end(error); + }, this.#context); + this.#members.clear(); + this.#exposed = undefined; + const holders = [...this.#holders]; + this.#holders.clear(); + this.host.unreferenced(this, holders, this.#context); + for (const holder of holders) holder.end(error); + if (state.kind === "waiting") return { kind: "end", previous }; + if (state.kind === "failed") { + // The failed attempt is closed (or closing) since it failed; only the references it holds (R18) are left. + const { holding } = state; + return holding === undefined ? { kind: "end", previous } : { kind: "release", attempt: holding, previous }; + } + state.attempt.seal(error); + return { kind: "close", attempt: state.attempt, previous }; + } + + /** + * A remote object or a local one with params ends once it has no references and no running or finishing job runs: + * a live one once its type's `linger` passed without a new reference, any other at once. + */ + #idle(): void { + const kind = this.#state.kind; + // A run waiting for a live attempt keeps its object only while one is live or activating. + const running = this.#runs > 0 && (kind === "live" || kind === "activating"); + const kept = this.#holders.size > 0 || running || (this.served.kind === "local" && this.served.hostLifetime); + if (kept || this.#state.kind === "ended" || this.#lingering !== undefined) return; + if (this.#state.kind !== "live" || this.served.linger === 0) { + this.end(); + return; + } + // A new holder clears the timer (`hold`), so an expiry finds none. + this.#lingering = setTimeout(() => { + this.#stopLingering(); + this.end(); + }, this.served.linger); + this.host.count("lingering", 1); + } + + /** Its linger timer stops (a new holder, its expiry, its end): no longer counted as lingering. */ + #stopLingering(): void { + if (this.#lingering === undefined) return; + clearTimeout(this.#lingering); + this.#lingering = undefined; + this.host.count("lingering", -1); + } + + /** + * Once the previous teardowns finished (a failed attempt's closes may read the documents): drains and closes a + * sealed attempt, then releases its references (after the closes, which may use them), then waits for an admitted + * commit step to adopt and drops the documents and job runs. Resolves with the closes that threw. + */ + async #tearDown(teardown: Teardown, reason: NativeCloseReason): Promise { + if (teardown.kind === "none") return []; + await teardown.previous; + const failures = teardown.kind === "close" ? await teardown.attempt.close(reason) : []; + if (teardown.kind === "close") teardown.attempt.detach(); + if (teardown.kind !== "end") teardown.attempt.releaseReferences(); + await this.documents.settled(); + this.documents.end(); + for (const records of this.#jobs.values()) records.drop(); + this.#jobs.clear(); + return failures; + } + + /** Reports the close failures of a teardown nobody waits for. */ + async #reported(closing: Promise): Promise { + for (const failure of await closing) this.host.report(failure); + } + + /** + * The live attempt; `not_ready` before the object went live and while it activates again, the cause while failed, + * `released` once it ended. + */ + #live(): Attempt { + const state = this.#state; + if (state.kind === "live") return state.attempt; + if (state.kind === "ended") throw this.ended(); + if (state.kind === "failed") throw state.failure.cause.error; + throw new ChordError({ code: "not_ready", message: `${this.label}: not live yet` }); + } + + /** The chain of waits from this activation to `target` (see `Target`). */ + pathTo(target: ChordObject, visited: Set): ChordObject[] | undefined { + if (this === target) return [this]; + visited.add(this); + for (const next of this.#waitingFor) { + if (visited.has(next)) continue; + const rest = next.pathTo(target, visited); + if (rest !== undefined) return [this, ...rest]; + } + return undefined; + } +} + +/** A settlement nobody may wait for: its rejection is not unhandled. */ +export function settlement(): PromiseWithResolvers { + const settlement = Promise.withResolvers(); + settlement.promise.catch(() => {}); + return settlement; +} diff --git a/packages/chord/src/next/host/parked.ts b/packages/chord/src/next/host/parked.ts new file mode 100644 index 000000000..796ade209 --- /dev/null +++ b/packages/chord/src/next/host/parked.ts @@ -0,0 +1,98 @@ +// Parked calls (§3, §7): calls made on a loading reference, queued in the order they were made until it goes live, +// each with its own deadline from the moment it was called (the reference's `readyTimeout`, never renewed). + +import type { Context } from "../../context/index.ts"; +import { ChordError } from "../data/index.ts"; + +/** One queued call: what to send, how its caller is settled, its deadline (`Date.now()` time) and what ends its wait. */ +type ParkedCall = { + readonly method: string; + readonly args: unknown; + readonly context: Context; + readonly resolve: (value: unknown) => void; + readonly reject: (error: unknown) => void; + readonly due: number; + readonly deadline: ReturnType; + readonly cancel: (() => void) | undefined; +}; + +/** + * The queue of one reference's parked calls. A call leaves it when it is sent, at its deadline (`unavailable/ + * park_timeout`), when its caller cancels, or rejected with the queue (the reference failed or was released). + */ +export class ParkedCalls { + /** In the order the calls were made (a `Set` keeps insertion order and deletes in constant time). */ + readonly #calls = new Set(); + readonly #timeout: number; + readonly #label: string; + + constructor(timeout: number, label: string) { + this.#timeout = timeout; + this.#label = label; + } + + /** Whether calls are queued: a new call waits behind them, also while they are being sent. */ + get waiting(): boolean { + return this.#calls.size > 0; + } + + /** Queues a call; settles as the call it becomes once sent, or with why it left the queue. */ + park(method: string, args: unknown, context: Context): Promise { + const signal = context.abortSignal; + signal?.throwIfAborted(); + const { promise, resolve, reject } = Promise.withResolvers(); + const deadline = setTimeout(() => this.#expire(call), this.#timeout); + let cancel: (() => void) | undefined; + if (signal !== undefined) { + cancel = () => { + this.#take(call); + reject(signal.reason); + }; + signal.addEventListener("abort", cancel, { once: true }); + } + const due = Date.now() + this.#timeout; + const call: ParkedCall = { method, args, context, resolve, reject, due, deadline, cancel }; + this.#calls.add(call); + return promise; + } + + /** + * Sends the queued calls in order through `send`, each only while `live()` still holds: sending one runs code + * (a handler's synchronous part) that may release or drop the reference, and the rest then stays queued or is + * rejected by that; or that makes a new call, which is queued behind the rest and sent in this loop. A call whose + * deadline passed while earlier ones ran (their timers could not fire) is rejected instead of sent. + */ + flush(send: (method: string, args: unknown, context: Context) => Promise, live: () => boolean): void { + for (const call of this.#calls) { + if (!live()) return; + if (Date.now() >= call.due) { + this.#expire(call); + continue; + } + this.#take(call); + send(call.method, call.args, call.context).then(call.resolve, call.reject); + } + } + + /** Rejects every queued call with `error`. */ + reject(error: unknown): void { + for (const call of this.#calls) { + this.#take(call); + call.reject(error); + } + } + + /** `call`'s deadline passed: `unavailable/park_timeout`. */ + #expire(call: ParkedCall): void { + this.#take(call); + const message = `${this.#label}: a call waited ${this.#timeout} ms for its reference to go live`; + call.reject(new ChordError({ code: "unavailable", reason: "park_timeout", message })); + } + + /** Takes `call` out of the queue: its deadline and cancellation stop. */ + #take(call: ParkedCall): void { + this.#calls.delete(call); + clearTimeout(call.deadline); + if (call.cancel !== undefined) call.context.abortSignal?.removeEventListener("abort", call.cancel); + } +} diff --git a/packages/chord/src/next/host/placement.ts b/packages/chord/src/next/host/placement.ts new file mode 100644 index 000000000..67f4215ca --- /dev/null +++ b/packages/chord/src/next/host/placement.ts @@ -0,0 +1,112 @@ +// Placement (§3, section 1's table): where a reference made on a host, or a hold arriving on one of its links, goes: +// here, over the route of its placement's address, or over the parent link; and the check that an object served here +// is placed here. + +import type { Context } from "../../context/index.ts"; +import type { Address, Contract, ParamsOf } from "../contracts/index.ts"; +import { contractData } from "../contracts/internal.ts"; +import { ChordError, type Principal } from "../data/index.ts"; +import { invalid } from "../data/internal.ts"; +import type { WireAddress } from "../link/internal.ts"; +import type { HoldRequest, HostLink, Path } from "../remote/internal.ts"; +import type { Served } from "./object.ts"; +import type { Route } from "./route.ts"; + +/** Where something goes: here, or over a path (with the address it was placed at, if its type has placement). */ +export type Placed = + | { readonly kind: "here" } + | { readonly kind: "path"; readonly path: Path; readonly address: WireAddress | undefined }; + +/** A host's placement: its address, its served types, its routes and its parent path, as the host keeps them. */ +export class Placement { + readonly #address: Address | undefined; + readonly #types: ReadonlyMap; + readonly #routes: ReadonlyMap; + readonly #parent: Path; + + constructor( + address: Address | undefined, + types: ReadonlyMap, + routes: ReadonlyMap, + parent: Path, + ) { + this.#address = address; + this.#types = types; + this.#routes = routes; + this.#parent = parent; + } + + /** + * Where a reference to `Type` with `params` made as `principal` goes: here when its type is served here and placed + * here (or this host has no address, or the type no placement); otherwise over the route of its placement's kind, + * or over the parent link (placement decides where an object lives: a worker reaches another worker's object of a + * type it serves too). Throws `invalid/placement` for a placement whose params do not fit its kind, `forbidden` when + * the route refuses, and what a placement or `authorize` throws. + */ + reference(Type: Contract, params: ParamsOf, principal: Principal | undefined, context: Context): Placed { + const address = placementOf(Type, params); + if (this.#types.has(Type.name) && this.#placedHere(address)) return HERE; + return { kind: "path", path: this.#pathTo(address, principal, context) ?? this.#parent, address }; + } + + /** + * Where a hold arriving on `link` acting as `principal` goes: here when its type is served here; over the route of + * the address it names; else over the parent link, unless it came on the parent link or there is none + * (`unavailable`). Throws as `reference` does. + */ + hold(request: HoldRequest, link: HostLink, principal: Principal | undefined, context: Context): Placed { + if (this.#types.has(request.type)) return HERE; + const { address } = request; + const routed = this.#pathTo(address, principal, context); + if (routed !== undefined) return { kind: "path", path: routed, address }; + if (link.role === "parent" || this.#parent.link === undefined) { + throw new ChordError({ code: "unavailable", message: `${request.type} is not served here` }); + } + return { kind: "path", path: this.#parent, address }; + } + + /** + * Throws `invalid/placement` for a hold of `Type` with `params` (checked) served here but placed on another host + * (misrouted; never forwarded). The placement is computed here from the params: the address a hold names only + * routes it. + */ + assertHere(Type: Contract, params: ParamsOf): void { + const address = placementOf(Type, params); + if (this.#placedHere(address)) return; + throw invalid("placement", `${Type.name}: placed on ${address!.kind}, this host is ${this.#address!.key}`); + } + + /** + * The path to `address` when this host routes its kind (its params checked, `authorize` run for `principal`); + * `undefined` without an address or a route for its kind. + */ + #pathTo(address: WireAddress | undefined, principal: Principal | undefined, context: Context): Path | undefined { + if (address === undefined) return undefined; + return this.#routes.get(address.kind)?.path(address.params, principal, context); + } + + /** Whether an object placed at `address` lives here: this host has no address, the type no placement, or equal. */ + #placedHere(address: WireAddress | undefined): boolean { + const own = this.#address; + return own === undefined || address === undefined || sameAddress(own, address); + } +} + +const HERE: Placed = { kind: "here" }; + +/** The address `Type`'s placement computes for `params`; `undefined` for a type without placement. */ +function placementOf(Type: Contract, params: ParamsOf): WireAddress | undefined { + const placement = contractData(Type).placement; + if (placement === undefined) return undefined; + // Plugin code: what it returns is checked where it is routed or served. + const { kind, params: values } = placement(params) as { readonly kind: string; readonly params: unknown }; + return { kind, params: values }; +} + +/** Whether `address` names the host at `own`: the same kind and the same value for each of its params. */ +function sameAddress(own: Address, address: WireAddress): boolean { + if (own.kind !== address.kind || typeof address.params !== "object" || address.params === null) return false; + const values = address.params as { readonly [field: string]: unknown }; + const fields = Object.keys(own.params); + return Object.keys(values).length === fields.length && fields.every((field) => values[field] === own.params[field]); +} diff --git a/packages/chord/src/next/host/presence.ts b/packages/chord/src/next/host/presence.ts new file mode 100644 index 000000000..768153f56 --- /dev/null +++ b/packages/chord/src/next/host/presence.ts @@ -0,0 +1,132 @@ +// Presence (§19): `host.references(Type)`, a JSON watchable of the references to objects of one type on this host, one +// entry per reference with the object's params, the reference's principal (remote types) and its incarnation. + +import type { Context } from "../../context/index.ts"; +import type { Op } from "../../delta/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { Contract, ParamsOf } from "../contracts/index.ts"; +import type { Principal } from "../data/index.ts"; +import { Source } from "../state/internal.ts"; +import type { ChordObject, Holder } from "./object.ts"; + +/** + * One reference to an object of `C`: the object's params, the principal it acts as (for a remote type), and its + * incarnation: the host that made it as its caller sees it (a hold's caller over a link; two tabs are two + * incarnations), the word `ctx.incarnation` uses. + */ +export type Presence = { + readonly params: ParamsOf; + readonly principal?: P; + readonly incarnation: string; +}; + +/** + * The references of one type, made by the host for its first `host.references(Type)` and kept from then on: one entry + * per reference, added when it binds and removed when it ends, in the batch where that happens (derives over it + * recompute there). While watched, a batch's changes are folded into the published list once, with one splice op per + * removal and one for the additions; otherwise the list is built on its next read. + */ +export class References extends Source { + readonly #entries = new Map(); + /** The list as last published or read; `undefined` once it changed while nobody watched it. */ + #list: readonly Presence[] | undefined; + /** Entries added since `#list`, in order (only while it is defined). */ + #added: Presence[] = []; + /** Entries of `#list` removed since. */ + readonly #removed = new Set(); + /** The ops folding `#list` forward, for the current batch's frame. */ + #ops: Op[] = []; + + get value(): JsonValue { + this.#fold(); + // Entries are JSON: params, a principal and a string. + return this.#list as unknown as JsonValue; + } + + /** `holder` references `object`, added in the batch running now or the next one. */ + add(object: ChordObject, holder: Holder, context: Context): void { + this.batch.within(() => { + const entry = this.record(object, holder); + if (this.#follows()) this.#added.push(entry); + this.changed(); + }, context); + } + + /** `holders` of `object` ended, removed in one batch. */ + remove(holders: Iterable, context: Context): void { + this.batch.within(() => { + const follows = this.#follows(); + for (const holder of holders) { + const entry = this.#entries.get(holder); + if (entry === undefined) continue; + this.#entries.delete(holder); + if (!follows) continue; + const added = this.#added.indexOf(entry); + if (added === -1) this.#removed.add(entry); + else this.#added.splice(added, 1); + } + this.changed(); + }, context); + } + + /** Records `holder` without a batch (the list being made) and returns its entry. */ + record(object: ChordObject, holder: Holder): Presence { + const { principal, incarnation } = holder.caller; + const entry: Presence = { + params: object.params, + ...(object.served.kind === "remote" && principal !== undefined ? { principal } : {}), + incarnation, + }; + this.#entries.set(holder, entry); + return entry; + } + + deliver(context: Context): void { + const value = this.value; + const ops = this.#ops; + this.#ops = []; + // Listed once per change: the batch's first delivery carries all of them. + if (ops.length > 0) this.publish(value, ops, false, context); + } + + /** + * Whether a change is folded into the published list: while watched (a watch read the list first). Otherwise the + * list is dropped and built on its next read. + */ + #follows(): boolean { + if (this.watched && this.#list !== undefined) return true; + this.#list = undefined; + return false; + } + + /** + * Folds the changes since `#list` into a new list, once: a removal is one splice op at its index, the additions one + * splice op at the end. Builds the list from the entries when there is none. + */ + #fold(): void { + if (this.#list === undefined) { + this.#list = [...this.#entries.values()]; + this.#added = []; + this.#removed.clear(); + this.#ops = []; + return; + } + if (this.#added.length === 0 && this.#removed.size === 0) return; + let list = this.#list; + if (this.#removed.size > 0) { + const kept: Presence[] = []; + for (const entry of list) { + if (this.#removed.has(entry)) this.#ops.push(["p", [], kept.length, 1, []]); + else kept.push(entry); + } + this.#removed.clear(); + list = kept; + } + if (this.#added.length > 0) { + this.#ops.push(["p", [], list.length, 0, this.#added as unknown as JsonValue[]]); + list = [...list, ...this.#added]; + this.#added = []; + } + this.#list = list; + } +} diff --git a/packages/chord/src/next/host/ref-state.ts b/packages/chord/src/next/host/ref-state.ts new file mode 100644 index 000000000..75f9fa42e --- /dev/null +++ b/packages/chord/src/next/host/ref-state.ts @@ -0,0 +1,91 @@ +// The state of a reference (§7): loading, live, failed or ended; and `ref.state`, the native watchable that follows +// it. + +import type { Context } from "../../context/index.ts"; +import { NativeWatch, type NativeWatchable } from "../../watch.ts"; +import type { ChordError, FailureCause } from "../data/index.ts"; + +/** + * Where a reference stands. `loading`: its object activates or retries, it waits for `load` or a parent link, or its + * link is down (the reference's coarser view of its object's waiting and activating); `failed`: its object failed and + * no attempt runs (`retryAt` undefined: broken, no automatic retry); `live`: bound; `ended`: for good, with the error + * that says why (`released` without a reason: released by its holder; `released/ended`: its object ended; + * `released/closed`: its host closed; `forbidden/revoked`). + */ +export type RefState = + | { readonly kind: "loading" } + | { readonly kind: "live" } + | { readonly kind: "failed"; readonly cause: FailureCause; readonly retryAt: number | undefined } + | { readonly kind: "ended"; readonly error: ChordError }; + +/** A failed reference's state. */ +export type Failed = Extract; + +/** An ended reference's state. */ +export type Ended = Extract; + +/** The states without fields, one object each: a change of state is a new object (a native watch compares by identity). */ +export const LOADING: RefState = { kind: "loading" }; + +export const LIVE: RefState = { kind: "live" }; + +const IGNORE = (): void => {}; + +/** + * `ref.state`: its value is the reference's state, read through `read`; each watch gets every new state, in its own + * context, and ends `ended` when the reference is released (the end is the notification). + */ +export class ReferenceState implements NativeWatchable { + readonly #read: () => RefState; + /** Where a listener's throw goes (the host); dropped on release, so a released reference keeps no host. */ + #report: (error: unknown) => void; + /** Throws `invalid/reentrant` inside the host's batch (a watch taken in a derive compute); dropped on release. */ + #assertOutside: () => void; + /** The watches with the contexts they were taken in, which their frames carry. */ + readonly #watches = new Map, Context>(); + /** The state the watches have: a change is announced only when the state is another object. */ + #last: RefState; + + constructor(read: () => RefState, report: (error: unknown) => void, assertOutside: () => void) { + this.#read = read; + this.#report = report; + this.#assertOutside = assertOutside; + this.#last = read(); + } + + get value(): RefState { + return this.#read(); + } + + async watch(context: Context): Promise> { + this.#assertOutside(); + const state = this.#read(); + const watch: NativeWatch = new NativeWatch(state, () => this.#watches.delete(watch), this.#report); + // An ended reference has nothing more to announce. + if (state.kind === "ended") { + watch.end({ kind: "ended" }); + return watch; + } + this.#watches.set(watch, context); + if (context.abortSignal !== undefined) watch.observeAbort(context.abortSignal); + return watch; + } + + /** The reference's state may have changed: every watch gets it when it is another state. */ + changed(): void { + const state = this.#read(); + if (state === this.#last) return; + this.#last = state; + for (const [watch, context] of this.#watches) watch.advance(state, context); + } + + /** The reference ended (`ended`, the reference's state): every watch ends `ended`. */ + end(ended: Ended): void { + // An ended reference keeps nothing of its object: not the last failure's error, not the host. + this.#last = ended; + this.#report = IGNORE; + this.#assertOutside = IGNORE; + // Ending detaches, which deletes the current entry (safe while iterating) and runs no listener. + for (const watch of this.#watches.keys()) watch.end({ kind: "ended" }); + } +} diff --git a/packages/chord/src/next/host/reference.ts b/packages/chord/src/next/host/reference.ts new file mode 100644 index 000000000..142dae425 --- /dev/null +++ b/packages/chord/src/next/host/reference.ts @@ -0,0 +1,674 @@ +// References (§7): what `get` and `hold` return, a plain object built from the contract's members, and its binding to +// the object: counted as a reference, it follows the object's state, parks calls while loading, keeps the +// per-reference watches and is released once. + +import type { Context } from "../../context/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { NativeWatchable, Watch, Watchable } from "../../watch.ts"; +import type { Contract, Implementation, ParamsOf } from "../contracts/index.ts"; +import { + type CurrentVariants, + currentVariants, + type MethodData, + type ShapeOf, + takesNoArgs, + type VariantImplementation, +} from "../contracts/internal.ts"; +import { ChordError, type FailureCause } from "../data/index.ts"; +import { Keyed, type KeyedSlot, type SealToken, Singleton } from "../documents/internal.ts"; +import { type Batch, Constant, type DeriveSource, Source } from "../state/internal.ts"; +import { type Caller, withLinkedAbortSignal } from "./ctx.ts"; +import type { PinnedKey } from "./keyed-members.ts"; +import type { KeyedBinding } from "./members.ts"; +import type { ChordObject, Holder } from "./object.ts"; +import { ParkedCalls } from "./parked.ts"; +import { type Ended, type Failed, LOADING, ReferenceState, type RefState } from "./ref-state.ts"; + +/** + * A reference: a local contract's members, or for a remote contract the members of the variant its object chose, + * narrowed by `variant`; and its state, `retry()` and `release()`. Every use after release throws. + */ +export type Reference = ShapeOf["kind"] extends "local" + ? Implementation & Lifecycle + : InVariants; + +/** What `host.hold` returns: a reference whose `variant` is `undefined` until its first binding. */ +export type Held = ShapeOf["kind"] extends "local" + ? Implementation & Lifecycle + : InVariants; + +/** A remote reference narrowed to some of its variants. */ +export type RefIn = Extract, { readonly variant: V }>; + +/** One member of a remote reference's union per variant; `Unbound` is what `variant` may also be. */ +type InVariants = { + [V in keyof CurrentVariants]: { readonly variant: V | Unbound } & VariantImplementation[V]> & + Lifecycle; +}[keyof CurrentVariants]; + +/** What every reference has besides its members. */ +type Lifecycle = { + /** The reference's state: loading, live, failed or released; watches get every change. */ + readonly state: NativeWatchable; + /** Asks a failed object for an attempt now (the backoff skipped, a broken one renewed); otherwise nothing. */ + retry(): void; + release(): void; +}; + +/** + * What a binding is bound to: an object on this host (`ChordObject`), or an object held over a path, the parent link + * or a route address (`RemoteObject`). It tells its holders of each change of `state`. + */ +export interface Target { + /** The object in errors: its type, and its params key when it has params. */ + readonly label: string; + /** Its state as its references show it. */ + readonly state: RefState; + /** Whether its state members have values: live once, for a remote object bound once. */ + readonly hydrated: boolean; + /** The variant bound last; `undefined` for a local object and before the first binding. */ + readonly variant: string | undefined; + /** The error of its last failed attempt while none went live since. */ + readonly lastFailure: ChordError | undefined; + /** Its instance id while it activates on this host (Q5); `undefined` otherwise and over a link. */ + readonly activationId: string | undefined; + /** The current attempt's outcome, which a dependent's activation waits for (R18). */ + readonly settled: Promise; + /** Counts `binding` as a reference; the binding reads `state` itself. */ + hold(binding: Binding): void; + /** `binding` was released. */ + unhold(binding: Binding): void; + /** Calls a method of the live object for `caller`. */ + call(method: string, args: unknown, context: Context, caller: Caller): Promise; + /** + * The state member `member` as Chord follows it: a local object's member source (across its attempts), or the + * replica of the last binding over a link. + */ + member(member: string): DeriveSource; + /** + * Whether its members keep their last values while it is failed, for same-host reads and watches (a local object + * once an attempt went live, slice 6a §1.4); a remote object's failure is its reads' error. + */ + readonly lastValues: boolean; + /** The keyed member `member` of the live object: its ultimate slot (slice 6a §12; only local objects have any). */ + keyedMember(member: string): KeyedSlot; + /** + * Pins `key` of the keyed member `member` (a reference's `at(key)`, a key watch): its source across the object's + * attempts once loaded (`keyed-members.ts`); only a live local object takes it. + */ + pinKey(member: string, key: string, context: Context): Promise; + /** `ref.retry()`: an attempt now if failed; ignored otherwise. */ + retry(): void; + /** + * The chain of activation waits from this object to `target`, both included (Q5); `undefined` when there is none. + * `visited` keeps a search over shared dependencies linear in the waits. + */ + pathTo(target: ChordObject, visited: Set): ChordObject[] | undefined; +} + +/** A reference's params as L1 checked them, with their identity key. */ +export type CheckedParams = { readonly params: ParamsOf; readonly key: string }; + +/** + * Where a binding stands. `unbound`: waiting for `load` (a `host.hold`) or for its object's activation to bind it (a + * `self.get`); `refused`: a `host.hold` whose type turned out not to be gettable here once `load` resolved; a released + * binding keeps nothing but its name and its end (`ref.state`'s `ended`). `dependent` is the object whose attempt declared the reference with `self.get`, + * `undefined` for `host.get`, `host.hold` and `ctx.get`. + */ +type BindingState = + | { readonly kind: "unbound"; readonly batch: Batch; readonly dependent: ChordObject | undefined } + | { + readonly kind: "bound"; + readonly batch: Batch; + readonly dependent: ChordObject | undefined; + readonly target: Target; + } + | { readonly kind: "refused"; readonly batch: Batch; readonly failure: Failed } + | { readonly kind: "released"; readonly state: Ended }; + +/** What a wait for a state decides at each change. */ +type Decision = + | { readonly kind: "wait" } + | { readonly kind: "done" } + | { readonly kind: "failed"; readonly error: unknown }; + +const WAIT: Decision = { kind: "wait" }; + +const DONE: Decision = { kind: "done" }; + +const IGNORE = (): void => {}; + +/** A wait for a state: checked at every change, rejected with the reference's end. */ +type Waiter = { readonly check: () => void; readonly end: (error: ChordError) => void }; + +/** + * A reference's binding to its object: counted by the object as one of its references while bound, and told of each + * change of its state. Calls run while live, park while loading (sent in order once live) and reject while failed. + * Released, or ended with its object, it ends the watches taken through it and rejects the calls made through it. + */ +export class Binding implements Holder { + readonly reference: Reference; + /** Who calls through this reference: the principal it was got as, and this host's incarnation. */ + readonly caller: Caller; + /** The contract's name, for errors. */ + readonly #name: string; + /** How long a call made while loading waits: the reference's `readyTimeout`. */ + readonly #parkTimeout: number; + #state: BindingState; + readonly #watches = new Set>(); + /** + * Aborted when the reference ends: its calls' callers are rejected and their handlers' contexts aborted. Made by the + * first call and dropped at the end, so a released reference keeps no error (whose stack keeps its frames). + */ + #calls: AbortController | undefined; + /** Calls made while loading; made by the first. */ + #parked: ParkedCalls | undefined; + /** Waits for a state (getters, member watches before the first binding); each checks the state at every change. */ + readonly #waiters = new Set(); + /** `ref.state`, made on first access. */ + #refState: ReferenceState | undefined; + /** Resolved when the reference ends; made by the first `ended()`. */ + #ended: PromiseWithResolvers | undefined; + /** The keys its object's factory pinned with `ref.member.at(key)`: pinned once bound, released with it. */ + readonly #keys: KeyedMemberAt[] = []; + + constructor( + contract: Contract, + batch: Batch, + dependent: ChordObject | undefined, + caller: Caller, + parkTimeout: number, + ) { + this.#name = contract.name; + this.caller = caller; + this.#parkTimeout = parkTimeout; + this.#state = { kind: "unbound", batch, dependent }; + const reference: Record = { release: () => this.release(), retry: () => this.retry() }; + Object.defineProperty(reference, "state", { get: () => this.#watchableState(), enumerable: true }); + // Every member of every variant (one kind and signature per name, D11); the bound variant decides which work. + const methods = new Map(); + for (const variant of currentVariants(contract).values()) { + for (const member of variant.state.keys()) reference[member] = new ReferenceMember(this, member); + for (const member of variant.keyed.keys()) reference[member] = new ReferenceKeyed(this, member); + for (const [member, method] of variant.methods) methods.set(member, method); + } + for (const [member, method] of methods) { + // A method without arguments omits them; `null` is their internal and wire form. + reference[member] = takesNoArgs(method) + ? async (context: Context) => this.#call(member, null, context) + : async (args: unknown, context: Context) => this.#call(member, args, context); + } + if (contract.kind === "remote") { + Object.defineProperty(reference, "variant", { get: () => this.#variant(), enumerable: true }); + } + // Built from the contract's own member lists. + this.reference = reference as Reference; + } + + /** The object whose attempt declared this reference, until it is released. */ + get dependent(): ChordObject | undefined { + const state = this.#state; + return state.kind === "unbound" || state.kind === "bound" ? state.dependent : undefined; + } + + /** The reference's state. */ + get refState(): RefState { + const state = this.#state; + if (state.kind === "bound") return state.target.state; + if (state.kind === "refused") return state.failure; + return state.kind === "unbound" ? LOADING : state.state; + } + + /** The object in errors: its label once bound, else the contract's name. */ + get label(): string { + return this.#state.kind === "bound" ? this.#state.target.label : this.#name; + } + + /** The error of the object's last failed attempt while none went live since. */ + get lastFailure(): ChordError | undefined { + const state = this.#state; + if (state.kind === "refused") return state.failure.cause.error; + return state.kind === "bound" ? state.target.lastFailure : undefined; + } + + /** Binds to `target`, which counts this binding as a reference; the reference takes its state. */ + bind(target: Target): void { + const state = this.#state; + if (state.kind !== "unbound") return; + this.#state = { ...state, kind: "bound", target }; + target.hold(this); + this.moved(); + } + + /** A `host.hold` that cannot be bound once `load` resolved: failed with `cause`, which no retry changes. */ + refuse(cause: FailureCause): void { + const state = this.#state; + if (state.kind !== "unbound") return; + this.#state = { kind: "refused", batch: state.batch, failure: { kind: "failed", cause, retryAt: undefined } }; + this.moved(); + } + + /** + * The state may have changed: `ref.state`'s watches are told; live sends the parked calls in order, failed + * rejects them with the cause; waits for a state decide. + */ + moved(): void { + this.#refState?.changed(); + const state = this.refState; + if (state.kind === "live") { + this.#parked?.flush( + (method, args, context) => this.#send(method, args, context), + () => this.refState.kind === "live", + ); + } else if (state.kind === "failed") { + this.#parked?.reject(state.cause.error); + } + for (const waiter of [...this.#waiters]) waiter.check(); + } + + /** Resolves when the reference is live; rejects with a broken failure's error, or its end. Getters wait for it. */ + live(context: Context): Promise { + return this.#until((state) => { + if (state.kind === "live") return DONE; + const broken = state.kind === "failed" && state.cause.kind === "broken"; + return broken ? { kind: "failed", error: state.cause.error } : WAIT; + }, context); + } + + /** + * The state member behind `member`: its last value while loading, and while failed when its object keeps one (a + * local object that went live once, slice 6a §1.4); throws `released`, `not_ready` before the first binding, and + * the failure's error otherwise. + */ + member(member: string): DeriveSource { + const state = this.#state; + if (state.kind === "bound" && state.target.lastValues) return state.target.member(member); + return this.#target().member(member); + } + + /** The keyed member `member`'s slot (slice 6a §12), as `member` resolves a state member. */ + keyedSlot(member: string): KeyedSlot { + return this.#target().keyedMember(member); + } + + /** Pins `key` of the keyed member `member` on the bound object (`Target.pinKey`), as `keyedSlot` resolves. */ + pinKey(member: string, key: string, context: Context): Promise { + try { + return this.#target().pinKey(member, key, context); + } catch (error) { + return Promise.reject(error); + } + } + + /** The bound object unless it failed: throws `released`, the failure's error, `not_ready` before the binding. */ + #target(): Target { + const state = this.#state; + switch (state.kind) { + case "bound": { + const current = state.target.state; + if (current.kind === "failed") throw current.cause.error; + return state.target; + } + case "unbound": + throw new ChordError({ code: "not_ready", message: `${this.#name}: not live yet` }); + case "refused": + throw state.failure.cause.error; + case "released": + throw state.state.error; + } + } + + /** + * A watch of `member` that ends when this reference is released. Before the first binding it waits for it (and + * rejects when the reference fails or is released first); an object that keeps its last values (`lastValues`) + * gives a watch at once, which gets frames again when the object is live again. + */ + async watch(member: string, context: Context): Promise> { + // Checked here too: a constant's own watch has no batch to check. A released reference throws below. + if (this.#state.kind !== "released") this.#state.batch.assertOutside("watch"); + if (this.#state.kind !== "bound" || !this.#state.target.lastValues) await this.hydrated(context); + return this.follow(this.member(member).openWatch(context)); + } + + /** Resolves once the object's members have values; rejects when the reference fails or is released first. */ + async hydrated(context: Context): Promise { + if (this.#state.kind === "bound" && this.#state.target.hydrated) return; + await this.#until((state) => { + if (state.kind === "live") return DONE; + if (state.kind === "failed") return { kind: "failed", error: state.cause.error }; + return WAIT; + }, context); + } + + /** + * Ends `watch` when this reference is released; returns it. A watch opened while the reference was released (an + * async keyed watch racing `release`) is ended at once, its pin with it, and the caller gets `released`. + */ + follow>(watch: W): W { + if (this.#state.kind === "released") { + watch.end({ kind: "ended" }); + throw this.#state.state.error; + } + this.#watches.add(watch); + void watch.closed.then(() => this.#watches.delete(watch)); + return watch; + } + + /** + * `ref.member.at(key)`: only while its object's factory runs (the reference not yet bound); pinned once bound + * (`pinKeys`), released with the reference. + */ + at(member: string, key: string): KeyedMemberAt { + if (this.#state.kind !== "unbound" || this.#state.dependent === undefined) { + throw new Error(`${this.#name}: ${member}.at(key) is valid only in the factory`); + } + const at = new KeyedMemberAt(this, member, key); + this.#keys.push(at); + return at; + } + + /** + * Pins the keys of `at(key)` on the live object's keyed members and loads those not in memory, before the + * dependent goes live (slice 6a §12); `undefined` when there are none. A failed load rejects. + */ + pinKeys(context: Context): Promise | undefined { + if (this.#keys.length === 0) return undefined; + return (async () => { + for (const at of this.#keys) await at.pin(this, context); + })(); + } + + /** + * Resolves when the reference ends (released, or its object ended); never rejects. A wait that must not keep the + * reference races it (a `host.hold` waiting for `load`). + */ + ended(): Promise { + if (this.#state.kind === "released") return Promise.resolve(); + this.#ended ??= Promise.withResolvers(); + return this.#ended.promise; + } + + /** `ref.retry()`: the object is asked for an attempt now; it ignores this unless failed. */ + retry(): void { + if (this.#state.kind === "bound") this.#state.target.retry(); + } + + /** Releases the reference (idempotent); the object may end when it was its last. */ + release(): void { + const state = this.#state; + if (state.kind === "released") return; + state.batch.assertOutside("release"); + // First: an object over a link rejects the calls itself and sends no cancel for them. + if (state.kind === "bound") state.target.unhold(this); + this.end(this.#released()); + } + + /** + * Ends the reference without releasing it from its object (it ended): watches end, calls (running and parked) + * reject with `error`, `ref.state` is `ended` with it. + */ + end(error: ChordError): void { + // Kept by `ref.state`: its stack is made text now, so the error holds none of the frames' objects. + void error.stack; + const ended: Ended = { kind: "ended", error }; + this.#state = { kind: "released", state: ended }; + for (const at of this.#keys) at.release(); + this.#calls?.abort(error); + this.#calls = undefined; + this.#parked?.reject(error); + this.#parked = undefined; + for (const watch of this.#watches) watch.end({ kind: "ended" }); + this.#refState?.end(ended); + // Each removes itself; promise reactions run later. + for (const waiter of this.#waiters) waiter.end(error); + this.#ended?.resolve(); + this.#ended = undefined; + } + + #call(method: string, args: unknown, context: Context): Promise { + const state = this.refState; + switch (state.kind) { + case "live": + // Calls queued before it went live are still being sent: a new one never overtakes them. + if (this.#parked?.waiting) return this.#parked.park(method, args, context); + return this.#send(method, args, context); + case "loading": + this.#parked ??= new ParkedCalls(this.#parkTimeout, this.label); + return this.#parked.park(method, args, context); + case "failed": + throw state.cause.error; + case "ended": + throw state.error; + } + } + + /** Calls the live object; the call is aborted when the reference ends. */ + #send(method: string, args: unknown, context: Context): Promise { + // Only a bound reference is live. + const { target } = this.#state as Extract; + this.#calls ??= new AbortController(); + const linked = withLinkedAbortSignal(this.#calls.signal, context); + return target.call(method, args, linked.context, this.caller).finally(linked.release); + } + + /** + * Settles once `decide` does for the reference's state, checked now and at every change; rejects when the reference + * ends (with its error, `released` when it is released already) or `context` aborts. + */ + #until(decide: (state: RefState) => Decision, context: Context): Promise { + if (this.#state.kind === "released") return Promise.reject(this.#state.state.error); + const { promise, resolve, reject } = Promise.withResolvers(); + const signal = context.abortSignal; + const settle = (done: () => void): void => { + this.#waiters.delete(waiter); + signal?.removeEventListener("abort", abort); + done(); + }; + const waiter: Waiter = { + check: () => { + const decision = decide(this.refState); + if (decision.kind === "done") settle(resolve); + else if (decision.kind === "failed") settle(() => reject(decision.error)); + }, + end: (error) => settle(() => reject(error)), + }; + const abort = (): void => settle(() => reject(signal?.reason)); + this.#waiters.add(waiter); + signal?.addEventListener("abort", abort, { once: true }); + if (signal?.aborted) abort(); + else waiter.check(); + return promise; + } + + #watchableState(): ReferenceState { + if (this.#refState !== undefined) return this.#refState; + const state = this.#state; + // A released reference's state never changes: its watches end at once, with no listener to report. Bound, not a + // closure: `read` would share the closure's scope and keep the batch after release. + const [report, assertOutside] = + state.kind === "released" + ? [IGNORE, IGNORE] + : [state.batch.reportListener, state.batch.assertOutside.bind(state.batch, "watch")]; + this.#refState = new ReferenceState(() => this.refState, report, assertOutside); + return this.#refState; + } + + #variant(): string | undefined { + const state = this.#state; + if (state.kind === "released") throw state.state.error; + return state.kind === "bound" ? state.target.variant : undefined; + } + + /** The error of this reference's release by its holder (`released`, no reason). */ + #released(): ChordError { + return new ChordError({ code: "released", message: `${this.#name}: released` }); + } +} + +/** A state member of a reference: the provider's member, followed through this reference. */ +class ReferenceMember { + readonly #binding: Binding; + readonly #member: string; + + constructor(binding: Binding, member: string) { + this.#binding = binding; + this.#member = member; + } + + get value(): JsonValue { + return this.source.value; + } + + watch(context: Context): Promise> { + return this.#binding.watch(this.#member, context); + } + + /** The provider's member: a source or a constant. */ + get source(): DeriveSource { + return this.#binding.member(this.#member); + } +} + +/** A keyed member of a reference (slice 6a §12): reads by key behave as calls (they wait while loading). */ +class ReferenceKeyed { + readonly #binding: Binding; + readonly #member: string; + + constructor(binding: Binding, member: string) { + this.#binding = binding; + this.#member = member; + } + + /** The provider's slot: the ultimate provider's, through re-exposures (rule 22). */ + get slot(): KeyedSlot { + return this.#binding.keyedSlot(this.#member); + } + + /** This member as another object re-exposing it has it: its keys held through this reference. */ + get keyed(): KeyedBinding { + return { kind: "reexposed", binding: this.#binding, member: this.#member }; + } + + async get(key: string, context: Context): Promise { + await this.#binding.hydrated(context); + const slot = this.slot; + return slot.get(key, ownAccess(slot), context); + } + + /** + * A watch of the document at `key` across the object's attempts (it holds the key through the object), `undefined` + * when absent; it ends with this reference. + */ + async watch(key: string, context: Context): Promise | undefined> { + await this.#binding.hydrated(context); + const slot = this.slot; + if ((await slot.get(key, ownAccess(slot), context)) === undefined) return undefined; + const held = await this.#binding.pinKey(this.#member, key, context); + // Retired between the read and now: absent. + if (slot.current(slot.key(key)) === undefined) { + held.release(); + return undefined; + } + let watch: Watch; + try { + watch = held.source.openWatch(context); + } catch (error) { + held.release(); + throw error; + } + void watch.closed.then(held.release); + return this.#binding.follow(watch); + } + + at(key: string): Watchable { + // The pinned key's values are the member's documents. + return this.#binding.at(this.#member, key) as unknown as Watchable; + } +} + +/** A consumer reads its provider's keyed documents as the provider's own code may: current version, own token. */ +function ownAccess(slot: KeyedSlot): { readonly token: SealToken; readonly version: number } { + return { token: slot.owner.own, version: slot.definition.current.version }; +} + +/** Where a `ref.member.at(key)` stands: not held yet, held on the object's keyed member, or released for good. */ +type AtState = + | { readonly kind: "unpinned" } + | { readonly kind: "pinned"; readonly source: DeriveSource; readonly release: () => void } + | { readonly kind: "released" }; + +/** + * `ref.member.at(key)`: the key's source across the object's attempts once the reference is bound and the key held + * (`pin`), until the reference ends (`release`, exactly once): then it drops the source (a retained reference keeps no + * provider). + */ +class KeyedMemberAt { + readonly #binding: Binding; + readonly member: string; + readonly key: string; + #state: AtState = { kind: "unpinned" }; + + constructor(binding: Binding, member: string, key: string) { + this.#binding = binding; + this.member = member; + this.key = key; + } + + /** The key's source; `not_ready` before it is held, `released` after. */ + get source(): DeriveSource { + const state = this.#state; + if (state.kind === "pinned") return state.source; + const what = `${this.member}.at(${this.key})`; + if (state.kind === "released") throw new ChordError({ code: "released", message: `${what}: released` }); + throw new ChordError({ code: "not_ready", message: `${what}: not loaded yet` }); + } + + get value(): JsonValue { + return this.source.value; + } + + async watch(context: Context): Promise> { + return this.#binding.follow(this.source.openWatch(context)); + } + + /** Holds the key through `binding` (once), loaded; nothing once released (a hold that lands then is released). */ + async pin(binding: Binding, context: Context): Promise { + if (this.#state.kind !== "unpinned") return; + const held = await binding.pinKey(this.member, this.key, context); + if (this.#state.kind !== "unpinned") { + held.release(); + return; + } + this.#state = { kind: "pinned", source: held.source, release: held.release }; + } + + /** Releases the pin and drops the source (once). */ + release(): void { + const state = this.#state; + this.#state = { kind: "released" }; + if (state.kind === "pinned") state.release(); + } +} + +/** + * What a keyed member's value is (slice 6a §12): a keyed document handle's slot, or a reference's keyed member + * (re-exposed); `undefined` for anything else. + */ +export function resolveKeyed(value: unknown): KeyedBinding | undefined { + if (value instanceof Keyed) return { kind: "own", slot: value.slot }; + if (value instanceof ReferenceKeyed) return value.keyed; + return undefined; +} + +/** + * The watchable Chord follows behind a state member or a derive source (Q14): a source or a constant itself, a + * singleton handle's slot, a reference member's provider member, a keyed member's pinned key (a key source itself for + * `handle.at(key)`); `undefined` for anything else, a native watchable included. + */ +export function resolveWatchable(value: unknown): DeriveSource | undefined { + if (value instanceof Singleton) return value.slot; + if (value instanceof ReferenceMember) return value.source; + if (value instanceof KeyedMemberAt) return value.source; + if (value instanceof Source || value instanceof Constant) return value; + return undefined; +} diff --git a/packages/chord/src/next/host/revocation.ts b/packages/chord/src/next/host/revocation.ts new file mode 100644 index 000000000..6f77aace1 --- /dev/null +++ b/packages/chord/src/next/host/revocation.ts @@ -0,0 +1,35 @@ +// Revocation keys (§4, R10, R11): what `revokedBy` says revokes a principal, as canonical JSON, computed once per +// principal object. + +import type { Context } from "../../context/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { Principal } from "../data/index.ts"; +import { canonicalJson } from "../data/internal.ts"; + +/** What revokes a principal: the app's typed values, the same function on every host. */ +export type RevokedBy

= (principal: P) => readonly R[]; + +/** A host's revocation keys: each principal's, cached weakly by the principal object. */ +export class RevocationKeys { + readonly #revokedBy: RevokedBy | undefined; + readonly #keys = new WeakMap>(); + + constructor(revokedBy: RevokedBy | undefined) { + this.#revokedBy = revokedBy; + } + + /** + * Whether `key` (a revocation's canonical JSON) is one of `principal`'s revocation keys: the canonical JSON of each + * value `revokedBy` returns for it, computed on its first use. + */ + revokes(principal: Principal, key: string, context: Context): boolean { + const revokedBy = this.#revokedBy; + if (revokedBy === undefined) return false; + let keys = this.#keys.get(principal); + if (keys === undefined) { + keys = new Set(revokedBy(principal).map((revocation) => canonicalJson(revocation, context))); + this.#keys.set(principal, keys); + } + return keys.has(key); + } +} diff --git a/packages/chord/src/next/host/route.ts b/packages/chord/src/next/host/route.ts new file mode 100644 index 000000000..e4f24d337 --- /dev/null +++ b/packages/chord/src/next/host/route.ts @@ -0,0 +1,160 @@ +// Routes (§3): how a host reaches hosts of one kind. `authorize` admits each hold routed over the route; `resolve` +// makes the link of one address, at most once at a time, again after the link drops while holds exist, with a +// backoff per address. + +import type { Context } from "../../context/index.ts"; +import type { Address, HostKind, KindShape } from "../contracts/index.ts"; +import { kindParams, type ParamsData } from "../contracts/internal.ts"; +import { ChordError, type Principal } from "../data/index.ts"; +import { checkAddress } from "../data/internal.ts"; +import type { Codec, Link, PartLimits } from "../link/index.ts"; +import { type HostLink, Path, type Upstream } from "../remote/internal.ts"; +import { Backoff } from "./backoff.ts"; +import type { AnyHost } from "./host.ts"; + +/** What `host.route` takes for a kind. */ +export type RouteOptions

= { + /** Whether `principal` may reach `address`; runs for every hold routed over the route, before anything starts. */ + authorize(principal: P, address: Address): boolean; + /** + * The link to `address`: `principal` is who the host at the other end is, `trusted` skips the message check on it, + * `limits` bound its parts (as `accept`'s). `context` is the route's, aborted when this host closes. Runs at most + * once at a time per address. + */ + resolve( + address: Address, + context: Context, + ): Promise<{ + readonly link: Link; + readonly principal: P; + readonly trusted?: boolean; + readonly codec?: Codec; + readonly limits?: PartLimits; + }>; +}; + +/** One address of a route: its path (link and holders) and whether its `resolve` runs. */ +type Routed = { readonly address: Address; readonly path: Path; resolving: boolean }; + +/** + * A host's route for one kind. An address is kept while it has holders, a link or a running `resolve`: a resolve whose + * holds were all released meanwhile finishes, and its link is kept until it closes. + */ +export class Route { + readonly kind: string; + readonly #params: ParamsData; + readonly #options: RouteOptions; + readonly #host: AnyHost; + /** The route's context, aborted when the host closes: every `resolve` and its links' work run under it. */ + readonly #context: Context; + readonly #addresses = new Map(); + /** The addresses' links, which tell their address when they open or close. */ + readonly #links = new Map(); + /** Per address key, separate from objects' identities. */ + readonly #backoff = new Backoff((key) => this.#due(key)); + + constructor(kind: HostKind, options: RouteOptions, host: AnyHost, context: Context) { + this.kind = kind.kind; + this.#params = kindParams(kind); + this.#options = options; + this.#host = host; + this.#context = context; + } + + /** + * The path to the address with `params`, for a hold acting as `principal`: the params checked against the kind + * (`invalid/placement`), then `authorize` (`forbidden` when it refuses; a throw is the hold's failure). + */ + path(params: unknown, principal: Principal | undefined, context: Context): Path { + const address = checkAddress(this.kind, this.#params, params, context); + if (principal === undefined || !this.#options.authorize(principal, address)) { + throw new ChordError({ code: "forbidden", message: `${address.key}: not authorized` }); + } + let routed = this.#addresses.get(address.key); + if (routed === undefined) { + const made: Routed = { address, path: new Path(() => this.#check(made)), resolving: false }; + routed = made; + this.#addresses.set(address.key, made); + } + return routed.path; + } + + /** One of this route's links completed its handshake: its address's backoff is over. */ + opened(link: HostLink): void { + const routed = this.#links.get(link); + if (routed !== undefined) this.#backoff.delete(routed.address.key); + } + + /** One of this route's links closed: the address waits its backoff, then resolves again while it has holders. */ + closed(link: HostLink): boolean { + const routed = this.#links.get(link); + if (routed === undefined) return false; + this.#links.delete(link); + if (!this.#host.closing) this.#backoff.failed(routed.address.key); + routed.path.disconnected(link); + return true; + } + + /** The host closes: no address resolves again. Returns every holder of every path, for the host to end. */ + close(): Upstream[] { + this.#backoff.clear(); + return [...this.#addresses.values()].flatMap(({ path }) => path.holders()); + } + + /** + * An address's holders or link changed: it resolves when it has holders, no link, no `resolve` running and no + * backoff window; it is forgotten when it has neither holders, a link nor a running `resolve`. + */ + #check(routed: Routed): void { + const { path, address } = routed; + if (path.link !== undefined || routed.resolving || this.#host.closing) return; + if (!path.held) { + this.#addresses.delete(address.key); + return; + } + if (!this.#backoff.waiting(address.key)) void this.#resolve(routed); + } + + /** A backoff window ended. */ + #due(key: string): void { + const routed = this.#addresses.get(key); + if (routed === undefined) this.#backoff.delete(key); + else this.#check(routed); + } + + /** + * Runs `resolve` once for the address. A rejection, or a link closed already, starts the address's backoff; a link + * returned after the host closed is closed at once. + */ + async #resolve(routed: Routed): Promise { + const key = routed.address.key; + routed.resolving = true; + let resolved: Awaited["resolve"]>>; + try { + resolved = await this.#options.resolve(routed.address, this.#context); + } catch (error) { + routed.resolving = false; + if (this.#host.closing) return; + this.#host.report({ kind: "resolve_failed", address: key, error }); + this.#backoff.failed(key); + this.#check(routed); + return; + } + routed.resolving = false; + if (this.#host.closing) { + resolved.link.close(); + return; + } + const { link: made, ...options } = resolved; + const link = this.#host.routeLink(made, options, this.#context); + if (!link.open) { + this.#backoff.failed(key); + this.#check(routed); + return; + } + this.#links.set(link, routed); + // A peer that greeted first had its `hello` handled while the link was made, before it was registered here. + if (link.greeted) this.#backoff.delete(key); + routed.path.connected(link); + } +} diff --git a/packages/chord/src/next/host/routines.ts b/packages/chord/src/next/host/routines.ts new file mode 100644 index 000000000..824e0ee38 --- /dev/null +++ b/packages/chord/src/next/host/routines.ts @@ -0,0 +1,432 @@ +// Routines (§14): the standing behaviour an object or a facet declares, `self.on(source, fn)` and `self.onLive(fn)`. +// They start once their owner went live and stop when it ends (their ctx aborted, the drain waits for running calls); +// each call gets its own ctx; a call that throws is reported and called again after a backoff. + +import type { Context } from "../../context/index.ts"; +import { type Op, PendingOps } from "../../delta/index.ts"; +import type { JsonValue } from "../../json.ts"; +import { + NativeWatch, + type NativeWatchable, + type NativeWatchCall, + type Watchable, + type WatchCall, + type WatchEnd, +} from "../../watch.ts"; +import type { Principal } from "../data/index.ts"; +import { Source } from "../state/internal.ts"; +import { nextDelay } from "./backoff.ts"; +import { CallContext, type Caller, type Ctx } from "./ctx.ts"; +import type { AnyHost, Position } from "./host.ts"; +import { resolveWatchable } from "./reference.ts"; +import { sleep } from "./sleep.ts"; + +/** + * Where routines run: their owner's label (for reports), its host, whom their calls act for, and the context of their + * calls, aborted when the owner ends. `admit` adds a running call to the owner's drain. + */ +export type RoutineOwner = { + readonly label: string; + readonly host: AnyHost; + readonly caller: Caller; + readonly context: Context; + /** + * Adds a running call to the drain; `cleanup` (its bookkeeping) runs once, when it settles or when the attempt ends + * with it still running. + */ + admit(running: Promise, cleanup: () => void): void; +}; + +/** + * A routine's handler of a JSON source: the value call (the value, the ops since the previous call, whether the + * source's document was retired since then) and the call's ctx. Failures are not calls of a handler (design §11): the + * routine waits for the next value. + */ +export type OnHandler = (call: WatchValueCall, ctx: C) => void | Promise; + +/** A JSON watch's value call: what an `on` handler gets. */ +export type WatchValueCall = Extract, { readonly kind: "value" }>; + +/** A native watch's value call. */ +export type NativeWatchValueCall = Extract, { readonly kind: "value" }>; + +/** A routine's handler of a native source: the value call (the value) and the call's ctx. */ +export type OnNativeHandler = (call: NativeWatchValueCall, ctx: C) => void | Promise; + +/** An `onLive` routine: called once its owner is live. */ +export type OnLiveHandler = (ctx: C) => void | Promise; + +/** What the declarations of an owner (`on`, `onLive`) take; part of the factory's `self` and the facet's. */ +export interface RoutineDeclarations

{ + /** + * A routine over `source`: `fn` gets its current value first (ops `[["r", value]]`), then one call per change, + * never overlapping: changes during a call fold into one next call (the newest value, the ops since the previous + * call, `retired` when any of them was a retirement). After a call that threw, it is called again with the whole + * newest value (`retired` again if the call that threw had it) after a backoff (1 s doubling to 60 s), or at once on + * a change during the backoff. Ends with its source. A JSON watchable is one whose watch is Chord's shared `Watch` + * (every Chord and Durable source). + */ + on(source: Watchable, fn: OnHandler>): void; + /** + * A routine over a native source (a registry value, `ref.state`): one whose watch is Chord's shared `NativeWatch`. + * As above without ops: after a busy call, one call with the newest value. + */ + on(source: NativeWatchable, fn: OnNativeHandler>): void; + /** + * A routine called once its owner is live. Returning ends it; a throw is reported and restarts it after a backoff + * (1 s doubling to 60 s, reset after a run of a minute); the owner's end aborts its ctx. + */ + onLive(fn: OnLiveHandler>): void; +} + +/** A declared routine: its kind and position in reports (`{ kind: "on", position: 2 }`), its source and its handler. */ +type Declaration = + | { + readonly kind: "on"; + readonly at: Position; + readonly source: Watchable | NativeWatchable; + readonly fn: (...args: never[]) => void | Promise; + } + | { readonly kind: "onLive"; readonly at: Position; readonly fn: OnLiveHandler }; + +/** How one routine call ended. */ +type Called = { readonly kind: "done" } | { readonly kind: "failed"; readonly error: unknown }; + +const DONE: Called = { kind: "done" }; + +/** One run of an `onLive` that lasted this long resets its backoff. */ +const LONG_RUN = 60_000; + +/** An owner's routines: declared while its factory (or setup) runs, started once when it went live. */ +export class Routines { + readonly #declared: Declaration[] = []; + #ons = 0; + #lives = 0; + + /** Declares `on(source, fn)`; whether the source is JSON or native is decided by its watch (`OnRoutine`). */ + on(source: Watchable | NativeWatchable, fn: (...args: never[]) => void | Promise): void { + this.#declared.push({ kind: "on", at: { kind: "on", position: ++this.#ons }, source, fn }); + } + + /** Declares `onLive(fn)`. */ + onLive(fn: OnLiveHandler): void { + this.#declared.push({ kind: "onLive", at: { kind: "onLive", position: ++this.#lives }, fn }); + } + + /** + * Starts every routine for `owner`, which went live; they stop when its context is aborted. A routine whose + * synchronous start ends the owner (an `onLive` releasing its last reference) starts none of the later ones. + */ + start(owner: RoutineOwner): void { + for (const declaration of this.#declared) { + if (owner.context.abortSignal?.aborted) return; + if (declaration.kind === "on") void new OnRoutine(owner, declaration).start(); + else void live(owner, declaration); + } + } +} + +/** + * Runs `onLive`'s handler until it returns or its owner ends; a throw is reported and the handler restarts after the + * backoff. + */ +async function live(owner: RoutineOwner, { at, fn }: Extract): Promise { + let delay: number | undefined; + while (!owner.context.abortSignal?.aborted) { + const started = Date.now(); + const called = await call(owner, fn); + if (called.kind === "done" || owner.context.abortSignal?.aborted) return; + failed(owner, at, called.error); + delay = nextDelay(Date.now() - started >= LONG_RUN ? undefined : delay); + try { + await sleep(delay, owner.context); + } catch { + return; + } + } +} + +/** + * Runs one routine call under its own ctx (its `ctx.get` references released when it settles), admitted to the owner's + * drain before the handler runs: a handler may end its own owner before its first await. Resolves with how it ended; + * never rejects. + */ +function call(owner: RoutineOwner, invoke: (ctx: Ctx) => void | Promise): Promise { + const ctx = new CallContext(owner.context, owner.caller, owner.host); + const settled = Promise.withResolvers(); + owner.admit(settled.promise, () => ctx.settle()); + const finish = (called: Called): void => { + ctx.settle(); + settled.resolve(called); + }; + // Plugin code: a synchronous throw becomes the rejection. + (async () => invoke(ctx))().then( + () => finish(DONE), + (error: unknown) => finish({ kind: "failed", error }), + ); + return settled.promise; +} + +/** Reports a failed routine call or source. */ +function failed(owner: RoutineOwner, routine: Position, error: unknown): void { + owner.host.report({ kind: "routine_failed", object: owner.label, routine, error }); +} + +/** + * Where an `on` routine stands. `starting`: taking its watch; `idle`: waiting for a change; `calling`: a call runs + * (changes meanwhile wait, folded, for the next); `settling`: a call succeeded while changes arrived, and the next one + * waits for one macrotask, so every value its watch still buffers (a native watch replays them one per microtask) is + * taken first; `backoff`: the last call threw and its timer calls again, unless a change during the backoff calls + * first; `ended`: its watch ended (the owner ended, or the source ended or failed). + */ +type OnState = + | { readonly kind: "starting" | "idle" | "calling" | "ended" } + | { readonly kind: "settling" | "backoff"; readonly timer: ReturnType }; + +/** + * What the next call's ops are relative to: `none` before the first call and after a throw (the handler may not have + * applied what it got: the next call gets the whole value), `last` after a call that succeeded. + */ +type Basis = "none" | "last"; + +/** + * One `on` routine. Its watch's listener never throws (a throwing listener would end the watch) and never waits: it + * keeps the newest frame, folding JSON ops, and the routine makes one call at a time from it, the first call and the + * retries after a throw through the same pending frame. + */ +class OnRoutine { + readonly #owner: RoutineOwner; + readonly #declaration: Extract; + #state: OnState = { kind: "starting" }; + /** A native source's watch yields values without ops; decided by its watch's class. */ + #native = false; + /** The newest value no call got yet. */ + #pending: { readonly value: unknown } | undefined; + /** The ops of the frames since the last call (JSON sources), folded. */ + readonly #ops = new PendingOps(); + /** One of the frames since the last call was a retirement (JSON sources). */ + #retired = false; + #basis: Basis = "none"; + /** The value of the last call: a retry after a throw calls with it. */ + #last: unknown; + /** The last call reported a retirement: a retry after a throw reports it again. */ + #lastRetired = false; + #delay: number | undefined; + /** The source failed after its last value: no call until its next value (a retry waits too). */ + #failed = false; + + constructor(owner: RoutineOwner, declaration: Extract) { + this.#owner = owner; + this.#declaration = declaration; + } + + /** + * Takes the watch and calls with its value. On a failed Chord source it waits for its first good value instead (a + * source that never had one is followed until it has). A source that cannot be watched otherwise is reported; an + * owner or source that ended meanwhile ends the routine. Never rejects. + */ + async start(): Promise { + const { source, at } = this.#declaration; + let failing = this.#failing(); + while (failing?.kind === "no_value") { + if (!(await this.#firstValue(failing.source))) { + this.#state = { kind: "ended" }; + return; + } + // Asked again: a failure may have followed the first value before this resumed (one receive turn). + failing = this.#failing(); + } + let watch: Awaited>; + try { + watch = await source.watch(this.#owner.context); + } catch (error) { + this.#state = { kind: "ended" }; + if (!this.#aborted()) failed(this.#owner, at, error); + return; + } + this.#native = watch instanceof NativeWatch; + void watch.closed.then((end) => this.#end(end)); + try { + // Both listeners return at once. + watch.start((call: WatchCall | NativeWatchCall) => { + if (call.kind === "value") this.#frame(call); + else this.#failure(); + return Promise.resolve(); + }); + } catch { + // The watch ended while it was taken (the owner or the source ended): `closed` says how. + return; + } + // A watch opened on a failed source holds its last good value, a stale one: its first call is the failure. + if (failing?.kind === "failed") { + this.#state = { kind: "idle" }; + return; + } + this.#pending = { value: watch.value }; + this.#next(); + } + + /** + * Whether the source is a Chord source failed now (`failed`), or one with no value yet (`no_value`, with the source to + * follow); `undefined` otherwise (natives and Durable's views tell their failures through their watches). + */ + #failing(): { readonly kind: "failed" } | { readonly kind: "no_value"; readonly source: Source } | undefined { + let source: unknown; + try { + source = resolveWatchable(this.#declaration.source); + } catch { + // Not bound or ended: the watch says so. + return undefined; + } + if (!(source instanceof Source)) return undefined; + let start: ReturnType; + try { + start = source.start(); + } catch { + return undefined; + } + if (start.kind === "value") return undefined; + return start.last === undefined ? { kind: "no_value", source } : { kind: "failed" }; + } + + /** + * Follows `source`, failed with no value, until its first good value (`true`), or until it or the owner ends + * (`false`): a subscriber of its own, since a watch has nothing to start at. + */ + #firstValue(source: Source): Promise { + const signal = this.#owner.context.abortSignal; + return new Promise((resolve) => { + const done = (got: boolean): void => { + source.unsubscribe(subscriber); + signal?.removeEventListener("abort", aborted); + resolve(got); + }; + const subscriber = { + advance: (call: WatchCall) => { + if (call.kind === "value") done(true); + }, + end: () => done(false), + }; + const aborted = (): void => done(false); + signal?.addEventListener("abort", aborted, { once: true }); + if (signal?.aborted) aborted(); + else source.subscribe(subscriber); + }); + } + + /** + * The source failed: what was pending goes (a value a failure followed is stale), a pending retry or settle is + * dropped, and the next call waits for the next value, as a whole value; a running call runs on. + */ + #failure(): void { + const state = this.#state; + if (state.kind === "ended") return; + this.#failed = true; + this.#pending = undefined; + this.#ops.take(); + this.#basis = "none"; + if (state.kind === "backoff" || state.kind === "settling") { + clearTimeout(state.timer); + this.#state = { kind: "idle" }; + } + } + + /** + * A frame of the watch: kept as the newest; it is called at once when nothing runs or waits, during a backoff too + * (the backoff ends); while a call runs or settles it waits for the next call. + */ + #frame(call: WatchValueCall | NativeWatchValueCall): void { + const state = this.#state; + if (state.kind === "ended") return; + this.#failed = false; + this.#pending = { value: call.value }; + if (!this.#native) { + // A JSON watch's call, as its class decided. + const { ops, retired } = call as WatchValueCall; + this.#ops.push(ops); + if (retired) this.#retired = true; + } + if (state.kind === "backoff") clearTimeout(state.timer); + if (state.kind === "backoff" || state.kind === "idle") this.#next(); + } + + /** Makes the next call from the pending frame, if there is one and the owner still runs; otherwise idles. */ + #next(): void { + const pending = this.#pending; + if (pending === undefined || this.#aborted()) { + this.#state = { kind: "idle" }; + return; + } + this.#pending = undefined; + const { value } = pending; + this.#state = { kind: "calling" }; + this.#last = value; + void call(this.#owner, this.#invocation(value)).then((called) => this.#called(called)); + } + + /** + * The handler's call with `value`: a native one with the value alone, a JSON one with the folded ops or the whole + * value, and whether a retirement came since the last call. + */ + #invocation(value: unknown): (ctx: Ctx) => void | Promise { + const { fn } = this.#declaration; + // The overloads typed `fn` for its source's kind, which the watch decided. + if (this.#native) return (ctx) => (fn as OnNativeHandler)({ kind: "value", value }, ctx); + const taken = this.#ops.take(); + // A JSON source's values are JSON. + const ops: readonly Op[] = + this.#basis === "none" || taken.kind === "replace" ? [["r", value as JsonValue]] : taken.ops; + const retired = this.#retired; + this.#retired = false; + this.#lastRetired = retired; + return (ctx) => (fn as OnHandler)({ kind: "value", value, ops, retired }, ctx); + } + + /** + * A call settled. After a success the next frame's ops build on it, and changes that arrived during the call are + * called after one macrotask (`settling`). After a throw (reported) the backoff always starts, also when changes + * arrived during the call (a handler that changes its own source and throws would feed itself otherwise); its timer + * calls with the newest value, or the last call's, as a whole value. + */ + #called(called: Called): void { + if (this.#state.kind === "ended") return; + if (called.kind === "done") { + this.#basis = "last"; + this.#delay = undefined; + if (this.#pending === undefined) this.#state = { kind: "idle" }; + else this.#state = { kind: "settling", timer: setTimeout(() => this.#next(), 0) }; + return; + } + if (this.#aborted()) return; + failed(this.#owner, this.#declaration.at, called.error); + this.#basis = "none"; + // The handler may not have applied the retirement either. + if (this.#lastRetired) this.#retired = true; + this.#delay = nextDelay(this.#delay); + // The source failed meanwhile: the retry waits for its next value, never the stale last one. + if (this.#failed) { + this.#state = { kind: "idle" }; + return; + } + const timer = setTimeout(() => { + this.#pending ??= { value: this.#last }; + this.#next(); + }, this.#delay); + this.#state = { kind: "backoff", timer }; + } + + /** The watch ended: the owner ended (aborted), or the source ended (no report) or failed (reported once). */ + #end(end: WatchEnd): void { + const state = this.#state; + this.#state = { kind: "ended" }; + if (state.kind === "settling" || state.kind === "backoff") clearTimeout(state.timer); + this.#pending = undefined; + this.#ops.take(); + this.#retired = false; + if (end.kind === "failed" && !this.#aborted()) failed(this.#owner, this.#declaration.at, end.error); + } + + #aborted(): boolean { + return this.#owner.context.abortSignal?.aborted === true; + } +} diff --git a/packages/chord/src/next/host/sleep.ts b/packages/chord/src/next/host/sleep.ts new file mode 100644 index 000000000..56cf18407 --- /dev/null +++ b/packages/chord/src/next/host/sleep.ts @@ -0,0 +1,43 @@ +// Waiting helpers for routines, jobs and handlers (§14): `sleep(ms, ctx)` and `untilAborted(ctx)`, which end with +// their context, leaving no timer or listener behind. Portable: timers and `AbortSignal` only. + +import type { Context } from "../../context/index.ts"; + +/** The longest delay one timer takes (a longer one fires at once). */ +const MAX_TIMER = 2 ** 31 - 1; + +/** + * Resolves after `ms` milliseconds; rejects with the abort reason once `context` is aborted, clearing its timer and + * listener. Delays above the timer limit (2^31-1 ms) are chained timers. + */ +export function sleep(ms: number, context: Context): Promise { + const signal = context.abortSignal; + if (signal?.aborted) return Promise.reject(signal.reason); + const { promise, resolve, reject } = Promise.withResolvers(); + let timer: ReturnType | undefined; + const abort = (): void => { + clearTimeout(timer); + reject(signal?.reason); + }; + const wait = (left: number): void => { + timer = setTimeout( + () => { + if (left > MAX_TIMER) return wait(left - MAX_TIMER); + signal?.removeEventListener("abort", abort); + resolve(); + }, + Math.min(left, MAX_TIMER), + ); + }; + signal?.addEventListener("abort", abort, { once: true }); + wait(ms); + return promise; +} + +/** Resolves once `context` is aborted (never without an abort signal); leaves no listener behind. */ +export function untilAborted(context: Context): Promise { + const signal = context.abortSignal; + if (signal === undefined) return new Promise(() => {}); + if (signal.aborted) return Promise.resolve(); + return new Promise((resolve) => signal.addEventListener("abort", () => resolve(), { once: true })); +} diff --git a/packages/chord/src/next/host/uuid.ts b/packages/chord/src/next/host/uuid.ts new file mode 100644 index 000000000..1112f4bb9 --- /dev/null +++ b/packages/chord/src/next/host/uuid.ts @@ -0,0 +1,38 @@ +// UUIDv7 (§3, code copied from pi-ai's `uuidv7`): time-ordered ids, minted for host incarnations. + +const MAX_SEQUENCE = (1n << 41n) - 1n; + +/** The last timestamp used: a clock going backwards keeps the ids ordered. */ +let lastTimestamp = -1; + +/** A random start, then incremented per id, so ids within one millisecond stay ordered and unique. */ +let sequence: bigint | undefined; + +/** A time-ordered UUIDv7. */ +export function uuidv7(): string { + const timestamp = Math.max(Date.now(), lastTimestamp); + lastTimestamp = timestamp; + const bytes = new Uint8Array(16); + globalThis.crypto.getRandomValues(bytes); + if (sequence === undefined) { + sequence = + (BigInt(bytes[1]!) << 32n) | + (BigInt(bytes[2]!) << 24n) | + (BigInt(bytes[3]!) << 16n) | + (BigInt(bytes[4]!) << 8n) | + BigInt(bytes[5]!); + } else { + if (sequence === MAX_SEQUENCE) throw new RangeError("UUIDv7 generator sequence exhausted"); + sequence++; + } + const time = BigInt(timestamp); + for (let index = 5; index >= 0; index--) bytes[index] = Number(time >> BigInt((5 - index) * 8)) & 0xff; + bytes[6] = 0x70 | Number((sequence >> 37n) & 0x0fn); + bytes[7] = Number((sequence >> 29n) & 0xffn); + bytes[8] = 0x80 | Number((sequence >> 23n) & 0x3fn); + bytes[9] = Number((sequence >> 15n) & 0xffn); + bytes[10] = Number((sequence >> 7n) & 0xffn); + bytes[11] = Number((sequence & 0x7fn) << 1n) | (bytes[11]! & 0x01); + const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")); + return `${hex.slice(0, 4).join("")}-${hex.slice(4, 6).join("")}-${hex.slice(6, 8).join("")}-${hex.slice(8, 10).join("")}-${hex.slice(10).join("")}`; +} diff --git a/packages/chord/src/next/jobs/README.md b/packages/chord/src/next/jobs/README.md new file mode 100644 index 000000000..3fba3ca35 --- /dev/null +++ b/packages/chord/src/next/jobs/README.md @@ -0,0 +1,112 @@ +# jobs + +Jobs in memory (§15, slice 5): owned work a call starts that outlives the call, in Durable's sandwich: intent (a +transaction), run (any effect), finish (a transaction on the current code). Persistence is slice 6, reload rules slice +7. + +## For apps (`index.ts`) + +```ts +import { defineJob } from "./index.ts"; + +const ShellState = defineState("chat.shell.state", { schema: obj({ output: str() }), version: 1, initial: () => ({ output: "" }) }); +const ShellJob = defineJob("chat.shell", { + version: 1, + args: obj({ command: str(), author: Author }), // closed, JSON only: checked at every start + result: obj({ exit: int(), output: str() }), // not validated + state: ShellState, // optional: one document per run + concurrency: 1, // optional: running plus finishing runs per object +}); + +host.serve(ChatStore, (_, self) => { + const entries = self.state(Entries); + const shell = self.job(ShellJob, { + run: async ({ command }, ctx) => { // outside transactions; ctx.jobId, ctx.state + const out = new OutputBuffer({ maxBytes: 64 * 1024, maxLines: 2_000, retain: "tail" }); + const write = async () => { const text = out.snapshot().text; await ctx.state.change((d) => { d.output = text; }, ctx); return text.length; }; + const progress = new Progress(write, report, 100); // the author throttles state changes + try { return { exit: await runProcess(command, ctx.abortSignal, (chunk) => { out.push(chunk); progress.mark(); }), output: out.snapshot().text }; } + finally { await progress.stop(); await write(); } // the last output reaches outcome.state, after an abort too + }, + finish: async (outcome, tx) => { // one transaction with the cleanup: only `await tx.*` + (await tx.draft(entries)).list.push(entryOf(outcome)); + tx.start(notify, { text: outcome.args.command }); // chaining: atomic with this outcome + }, + }); + return { + shells: shell.running, // a JSON watchable: views filter it + runShell: async ({ command, author, requestId }, ctx) => + (await shell.start({ command, author }, ctx, { key: requestId })).id, + abortShell: async ({ id }) => { shell.abort(id); return null; }, + }; +}); +``` + +- `defineJob(name, { version, args, result, state?, concurrency?, accepts?, persist? })`: `args` closed (every object + `additionalProperties: false`) and without `bytes()` (`running` carries them); `state` a singleton definition; + `concurrency` a positive integer; `accepts` `{ [older version]: { args, result } }`, both functions, checked now and + used from slice 7 on; `persist: { resume: "rerun" | "abandon" }` is refused before slice 6. Declarations fail with + an `Error` naming the path (`chat.shell.accepts[2]: not an older version`). +- `self.job(Job, { run, finish? })` in an object's factory (not in a facet), once per job name: a `JobHandle`. + - `start(args, ctx, { key? })`: one transaction on the object's line; resolves with a `Run { id, outcome(ctx) }` + after the commit. Only a live object takes starts (`not_ready` before, `released` once it ended). + - `tx.start(job, args, { key? })` inside `self.tx` (and in `finish`): the same, with the transaction's other writes; + its `Run` at once. A discarded transaction's `Run` rejects `released`. + - In order: the args checked (`invalid/args`; one compiled check per definition); a key of an unsettled run (one + started earlier in the same transaction too, a blocked one too) returns that run, args not compared; at the + `concurrency` limit (running and finishing runs plus earlier starts in the transaction) `unavailable/busy`; then + the run (id uuidv7) and its state document (`initial()`). The run whose `finish` the transaction is counts for + neither: `finish` may start the same job again, with the same key too, and gets a new run. A start that only + dedupes commits nothing. + - `outcome(id, ctx)`: waits for an unsettled run; a settled or unknown id rejects `gone`. + - `abort(id)`: fires a running run's signal (before `run` started too: it starts aborted); drops a blocked run (no + `finish`; its record, key and state document go; `job_dropped`; its waiters reject `aborted`); ignored for + finishing runs and unknown ids. + - `running`: one entry per unsettled run in start order, `{ id, version, key?, args, state, phase: "running" | + "finishing" } | { id, version, phase: "blocked", reason: "finish_failed" }` (`state` only with a state + document). +- `run(args, ctx)`: `ctx` is a handler's `Ctx` (the object's principal for a remote type, the host's for a local one; + the host's incarnation; `ctx.get` released when `run` settles) plus `jobId` and, with a state document, `state: { + value, change(fn, ctx) }`. Its abort signal fires on `abort(id)`, on the object's end and at `host.close`. + `ctx.state.change(fn, ctx)`: a transaction drafting only the run's document, under `ctx` without its abort signal + (the run's own ctx is fine), accepted while `run` runs (also after an abort) and refused `released/settled` once it + settled; a change admitted + before `run` settled commits before the outcome is recorded. Every change is a commit (and a message while watched): + throttle and await or catch them. +- Outcomes: `{ id, args, state } & ({ kind: "done", result } | { kind: "failed", error })`, `error` being `{ kind: + "threw", error }` (`run` threw: reported `job_failed`; anything but a ChordError or PluginError becomes a + PluginError `internal` with its message) or `{ kind: "aborted", reason: "abort" }` (`run` threw after its signal + fired; a result returned after an abort is kept). +- `finish(outcome, tx)` runs when `run` settled (`phase: "finishing"`); its commit deletes the record and the key + and retires the state document; without `finish` the commit only cleans up. A throw changes nothing: retried after + 1 s doubling to 60 s, each failure reported `job_finish_failed`; after five in a row the run is `blocked`: outcome, + state and key kept, not counted for concurrency, its waiters waiting (their own ctx ends a wait). +- `run.outcome(ctx)` resolves once `finish` committed; the caller's ctx ends only its wait, which then keeps nothing + of it (no listener on its signal). A retained `Run` answers late waits from how the run settled; a rejection's error + keeps its stack's receivers (the transaction that ended the run) until its `stack` is read (V8), as every error. +- Keep-alive: running and finishing runs keep their object alive (a count, not references: never in + `host.references`); an object without references ends after its last run settles (after its `linger`), dropping + blocked runs (`job_dropped`). A run is counted in its start's commit step, before the commit publishes (a link + closing while it sends may release the last reference) and before any run of the same commit settles (a `finish` + chaining another job keeps the object). A run started while the object lingers keeps it. `host.activity.jobs` + counts them. +- The object's end (revocation, `host.close`, the last release): runs' signals fire with the end's error, the drain + waits for `run` and a `finish` running then (bounded by `closeTimeout`; admitted before `run` runs, which may end + its own object), nothing finishes after it (its commit is refused, not reported), waiters reject with the end's + error; blocked runs are dropped (`job_dropped`), at every end. `running` then reads `released` and keeps no run. + +## For Chord's layers (`internal.ts`) + +- `Job`: one job of one object (an `Attempt` makes it for `self.job`). `live()` once the object is live, `seal(error)` + when it ends, `end()` when its documents are dropped; `stage(tx, args, ctx, options)` is `start`'s and `tx.start`'s + work inside a transaction. Its records change only in staged changes (`documents/`'s `Staged`): `create`, + `outcome`, `delete`, `block`, `drop`, adopted in the commit step (a new run is counted there); the actions after the + commit start `run` (queued, after the transaction settled), settle waiters, uncount runs and report. A job stages + only when it records something (`Transaction.staged` reads what it staged so far). +- `JobOwner`: what a job needs of its object: label, host, caller, the object's context (without an abort signal), + documents, `admit` (the drain) and `countRuns` (keep-alive and `host.activity.jobs`). +- Inside: `JobRun` (one run: record, phase, abort signal, invocation fence closed as `run` settles, `running` entry, + the `Run` handle over its detachable waiters), the + run states (a keyed `DocumentHandle` by run id, apart from `self.state`'s, never drafted while absent) and `Running` + (`running`'s list, copied per change; while watched it records the ops of each change, a run's state ops re-pathed + under its entry, delivered as one frame per batch). diff --git a/packages/chord/src/next/jobs/definition.ts b/packages/chord/src/next/jobs/definition.ts new file mode 100644 index 000000000..c0dc86082 --- /dev/null +++ b/packages/chord/src/next/jobs/definition.ts @@ -0,0 +1,123 @@ +// Job definitions (§15): `defineJob(name, …)`, the checks of a declaration, and the types of what a run ends with. + +import { checkKeys, entriesOf } from "../contracts/declare.ts"; +import type { Static, TSchema } from "../contracts/index.ts"; +import { checkName } from "../contracts/internal.ts"; +import { type Snapshot, snapshotArguments, snapshotSchema } from "../contracts/snapshot.ts"; +import type { ChordError, PluginError } from "../data/index.ts"; +import type { JsonObject, StateDefinition } from "../documents/index.ts"; + +/** What `defineJob` takes besides the name. */ +export type JobOptions< + A extends TSchema, + R extends TSchema, + D extends StateDefinition | undefined, +> = { + /** The job's version; every change of its args, result or state bumps it. */ + readonly version: number; + /** The run's arguments: closed objects, JSON only (`running` carries them), checked at every start. */ + readonly args: A; + /** What `run` returns; not validated. */ + readonly result: R; + /** A singleton document definition: one document per run, `ctx.state` in `run`, `outcome.state` in `finish`. */ + readonly state?: D; + /** At most this many runs running or finishing per object; a start beyond it rejects `unavailable/busy`. */ + readonly concurrency?: number; + /** Per older version, how its args and result map forwards (used from slice 7 on; checked now). */ + readonly accepts?: JobAccepts; + /** How a persisted run resumes after a crash; refused before slice 6. */ + readonly persist?: { readonly resume: "rerun" | "abandon" }; +}; + +/** Per older version, its mappings forwards: its args to the current args, its result to the current result. */ +export type JobAccepts = { + readonly [version: number]: { readonly args: (old: never) => Static; readonly result: (old: never) => Static }; +}; + +/** The value of a job's state document: the definition's document, `undefined` without one. */ +type StateOf = D extends StateDefinition ? T : undefined; + +/** What a run ended with, as `finish` and `outcome` get it: its args and state, and its result or its error. */ +export type Outcome = { readonly id: string; readonly args: A; readonly state: S } & ( + | { readonly kind: "done"; readonly result: R } + | { readonly kind: "failed"; readonly error: JobError } +); + +/** + * Why a run failed: `run` threw (reported as `job_failed`; anything but a ChordError or PluginError becomes a + * PluginError `internal` with its message), or it was aborted (`abort`: `abort(id)`; `reload` and `restart` come with + * slices 7 and 6). + */ +export type JobError = + | { readonly kind: "threw"; readonly error: ChordError | PluginError } + | { readonly kind: "aborted"; readonly reason: "abort" | "reload" | "restart" }; + +/** Declares a job: owned work a call starts that outlives it (§15). */ +export function defineJob< + const A extends TSchema, + const R extends TSchema, + D extends StateDefinition | undefined = undefined, +>(name: string, options: JobOptions): JobDefinition, Static, StateOf> { + // Declaration policy: JavaScript callers reach every check below. + checkName(name, "A job name"); + checkKeys(options, ["version", "args", "result", "state", "concurrency", "accepts", "persist"], name); + const { version, state, concurrency, accepts, persist } = options; + if (!Number.isSafeInteger(version) || version < 1) throw new Error(`${name}.version: expected a positive integer`); + const args = snapshotArguments(options.args, `${name}.args`); + if (args.bytes) throw new Error(`${name}.args: job arguments are JSON (running carries them); no bytes()`); + const result = snapshotSchema(options.result, `${name}.result`); + if (state?.keyed) throw new Error(`${name}.state: expected a singleton document definition`); + if (concurrency !== undefined && !(Number.isSafeInteger(concurrency) && concurrency > 0)) { + throw new Error(`${name}.concurrency: expected a positive integer`); + } + if (accepts !== undefined) checkJobAccepts(accepts, version, `${name}.accepts`); + if (persist !== undefined) throw new Error(`${name}.persist: persisted jobs are not supported before slice 6`); + return new JobDefinition(name, version, args, result, state, concurrency, accepts ?? {}); +} + +/** Checks an accepts table's shape: older versions only, each with an `args` and a `result` function. */ +function checkJobAccepts(accepts: unknown, version: number, path: string): void { + for (const [key, adapter] of entriesOf(accepts, path)) { + const older = Number(key); + if (!Number.isSafeInteger(older) || older < 1 || older >= version) { + throw new Error(`${path}[${key}]: not an older version`); + } + checkKeys(adapter, ["args", "result"], `${path}[${key}]`); + const { args, result } = adapter as { readonly args?: unknown; readonly result?: unknown }; + if (typeof args !== "function") throw new Error(`${path}[${key}].args: expected a function`); + if (typeof result !== "function") throw new Error(`${path}[${key}].result: expected a function`); + } +} + +/** A declared job; `A`, `R` and `S` are its args, result and state document (`undefined` without one). */ +export class JobDefinition { + readonly name: string; + readonly version: number; + /** The args schema, Chord's copy; its compiled check is cached by this snapshot. */ + readonly args: Snapshot; + readonly result: Snapshot; + readonly state: StateDefinition | undefined; + readonly concurrency: number | undefined; + /** The older versions' mappings, checked at declaration; slice 7 maps older runs with them. */ + readonly accepts: JobAccepts; + /** Type only: the args, result and state types. */ + declare readonly "~types"?: { readonly args: A; readonly result: R; readonly state: S }; + + constructor( + name: string, + version: number, + args: Snapshot, + result: Snapshot, + state: StateDefinition | undefined, + concurrency: number | undefined, + accepts: JobAccepts, + ) { + this.name = name; + this.version = version; + this.args = args; + this.result = result; + this.state = state; + this.concurrency = concurrency; + this.accepts = accepts; + } +} diff --git a/packages/chord/src/next/jobs/index.ts b/packages/chord/src/next/jobs/index.ts new file mode 100644 index 000000000..5a8cd4739 --- /dev/null +++ b/packages/chord/src/next/jobs/index.ts @@ -0,0 +1,13 @@ +// Jobs for apps: declaring a job, its handle, its runs, their outcomes and `running`. + +export { + defineJob, + type JobAccepts, + type JobDefinition, + type JobError, + type JobOptions, + type Outcome, +} from "./definition.ts"; +export type { JobCtx, JobHandle, JobImplementation, RunState, StartOptions } from "./job.ts"; +export type { Run } from "./run.ts"; +export type { RunningEntry } from "./running.ts"; diff --git a/packages/chord/src/next/jobs/internal.ts b/packages/chord/src/next/jobs/internal.ts new file mode 100644 index 000000000..ffbd89a05 --- /dev/null +++ b/packages/chord/src/next/jobs/internal.ts @@ -0,0 +1,4 @@ +// Jobs for Chord's layers: a job's records on its object and the handle an attempt keeps, and what they need. + +export { type AnyJobImplementation, Job, type JobOwner } from "./job.ts"; +export { JobRecords, type RecordsOwner } from "./records.ts"; diff --git a/packages/chord/src/next/jobs/job.ts b/packages/chord/src/next/jobs/job.ts new file mode 100644 index 000000000..12e4b7403 --- /dev/null +++ b/packages/chord/src/next/jobs/job.ts @@ -0,0 +1,319 @@ +// Jobs (§15), in memory: the handle `self.job` returns, one per attempt, over the job's records on its object. It +// holds the implementation and the invocations: `run` runs outside transactions, `finish` in one with the cleanup, +// both on the attempt that is live; the attempt's seal cuts its invocations off (slice 6a §1.3). + +import { type Context, withAbortSignal, withoutAbortSignal } from "../../context/index.ts"; +import type { Draft } from "../../delta/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { Watchable } from "../../watch.ts"; +import { ChordError, isChordError, PluginError, type Principal } from "../data/index.ts"; +import type { JsonObject, Tx } from "../documents/index.ts"; +import type { Documents, SealToken, Transaction } from "../documents/internal.ts"; +import { type Ctx, sleep } from "../host/index.ts"; +import { type AnyHost, CallContext, type Caller, nextDelay } from "../host/internal.ts"; +import type { Outcome } from "./definition.ts"; +import type { JobRecords, Settled } from "./records.ts"; +import type { AnyOutcome, JobRun, Run } from "./run.ts"; +import type { RunningEntry } from "./running.ts"; + +/** A job's handle, `self.job(Job, { run, finish? })`; `A`, `R` and `S` are its args, result and state. */ +export interface JobHandle { + /** + * Starts a run in one transaction on the object's line and resolves with its `Run` after the commit: the args + * checked (`invalid/args`); a `key` of an unsettled run returns that run; at the `concurrency` limit (running plus + * finishing runs) it rejects `unavailable/busy`. `run` starts after the commit. + */ + start(args: A, context: Context, options?: StartOptions): Promise>>; + /** Waits for the outcome of the unsettled run `id`; a settled or unknown id rejects `gone`. */ + outcome(id: string, context: Context): Promise>; + /** + * Fires a running run's abort signal; drops a blocked run (no `finish`; its waiters reject `aborted`); ignored for + * finishing runs and unknown ids. + */ + abort(id: string): void; + /** The unsettled runs, one entry each in start order, tagged by phase (a JSON watchable, a state member). */ + readonly running: Watchable[]>; +} + +/** What `start` and `tx.start` take besides the args. */ +export type StartOptions = { + /** Names one run: a start with the key of an unsettled run returns that run (args not compared). */ + readonly key?: string; +}; + +/** What `self.job` takes: the run and the optional finish. */ +export type JobImplementation = { + /** The work, outside transactions; its ctx's abort signal fires on `abort(id)` and on its object's end. */ + readonly run: (args: A, ctx: JobCtx) => Promise; + /** Records the outcome in one transaction (only `await tx.*`); the run's record and state document go with it. */ + readonly finish?: (outcome: Outcome, tx: Tx) => void | Promise; +}; + +/** A run's ctx: a handler's ctx (the object's principal or the host's), its run's id and, with a state, the state. */ +export type JobCtx

= Ctx

& { readonly jobId: string } & ([S] extends [undefined] + ? unknown + : { readonly state: RunState }); + +/** A run's state document. */ +export interface RunState { + /** The document's value. */ + readonly value: S; + /** + * One transaction on the object's line drafting only this document, under `context` without its abort signal: + * admitted while `run` runs (also after an abort, with the run's own ctx), refused `released` once it settled. + */ + change(fn: (draft: Draft) => T, context: Context): Promise; +} + +/** Where a job's handle runs its work: its attempt, as the job sees it. */ +export type JobOwner = { + readonly host: AnyHost; + /** Whom runs act for: the object's principal (remote) or the host's (local), from this host. */ + readonly caller: Caller; + /** The object's context, without an abort signal: runs, state changes and finishes run under it. */ + readonly context: Context; + /** Aborted when the attempt is sealed: a finish's backoff waits under it. */ + readonly signal: AbortSignal; + /** The attempt's seal token: its starts and finishes are transactions under it. */ + readonly token: SealToken; + readonly documents: Documents; + /** + * Adds running work (an invocation and its finish, a deferred finish) to the attempt's drain; `cleanup` (its + * bookkeeping) runs once, when it settles or when the attempt ends with it still running. + */ + admit(running: Promise, cleanup?: () => void): void; +}; + +/** A job's implementation as Chord calls it, whatever the job's types. */ +export type AnyJobImplementation = { + readonly run: (args: JsonValue, ctx: JobContext) => Promise; + readonly finish?: (outcome: AnyOutcome, tx: Tx) => void | Promise; +}; + +/** Failed finishes in a row after which a run is blocked. */ +const FINISH_ATTEMPTS = 5; + +/** + * A job as one attempt declares it: its implementation and the invocations and finishes running on the attempt. Starts + * are taken while the attempt is live; the object's records keep the runs. + */ +export class Job implements JobHandle { + readonly #records: JobRecords; + readonly #implementation: AnyJobImplementation; + readonly #owner: JobOwner; + /** `declared` until the attempt is live (starts are refused), `sealed` once it is sealed. */ + #state: "declared" | "live" | "sealed" = "declared"; + /** Runs whose `run` runs on this attempt. */ + readonly #invocations = new Set(); + /** Runs whose finish runs on this attempt. */ + readonly #finishing = new Set(); + + constructor(records: JobRecords, implementation: AnyJobImplementation, owner: JobOwner) { + this.#records = records; + this.#implementation = implementation; + this.#owner = owner; + } + + get running(): Watchable[]> { + // Its value is the list of entries `JobRun.entryFor` builds. + return this.#records.running as unknown as Watchable[]>; + } + + async start(args: JsonValue, context: Context, options: StartOptions = {}): Promise> { + const { documents, token } = this.#owner; + return documents.tx((tx) => this.stage(tx, args, context, options), token, context); + } + + outcome(id: string, context: Context): Promise { + return this.#records.outcome(id, context); + } + + abort(id: string): void { + this.#records.abort(id); + } + + /** The attempt went live: starts are taken, and the records' waiting runs are invoked or finished here. */ + live(): void { + if (this.#state !== "declared") return; + this.#state = "live"; + this.#records.attach(this); + } + + /** + * Stages a run in `tx` (`start`, `tx.start`), in order: the args checked, dedupe by key (runs started earlier in + * `tx` included), the concurrency check, then the record and the state document, created in the commit. The run + * whose finish `tx` is counts for neither: `finish` may start the same job again, with the same key too. Returns + * the run's handle at once; a start that only dedupes stages nothing. + */ + stage(tx: Transaction, args: unknown, context: Context, options: StartOptions): Run { + // Declaration policy: a handle belongs to its object (types cannot say whose transaction this is). + if (tx.documents !== this.#owner.documents) { + throw new Error(`${this.#owner.documents.label}: a transaction starts only its own object's jobs`); + } + if (this.#state === "sealed") { + throw new ChordError({ + code: "released", + reason: "ended", + message: `${this.#records.name}: its attempt ended`, + }); + } + if (this.#state === "declared") { + throw new ChordError({ code: "not_ready", message: `${this.#records.name}: its object is not live yet` }); + } + return this.#records.stage(tx, args, context, options); + } + + /** + * The attempt is sealed with `error`: starts are refused, running invocations are cut off (their signals fire, they + * end `aborted` with reason `restart` and finish on the next live attempt), and the runs finishing here wait for it. + */ + seal(error: ChordError): void { + if (this.#state === "sealed") return; + this.#state = "sealed"; + this.#records.detach(this, this.#finishing); + this.#finishing.clear(); + for (const run of this.#invocations) this.#records.cut(run, error); + this.#invocations.clear(); + } + + /** + * Runs `run` for a committed run (with its signal aborted already after an early abort). The invocation and the + * run's finish are one piece of work in the drain, admitted before plugin code runs: `run` may end its own object + * before its first await. + */ + invoke(run: JobRun): void { + const { context, caller, host } = this.#owner; + const ctx = new JobContext(withAbortSignal(run.abort.signal, context), caller, host, run.id, this.#stateOf(run)); + const work = Promise.withResolvers(); + this.#owner.admit(work.promise, () => ctx.settle()); + this.#invocations.add(run); + run.invocation = "running"; + const invocation = (async () => { + try { + return await this.#implementation.run(run.args, ctx); + } finally { + // The fence closes as `run` settles: a change made later is refused. + run.invocation = "settled"; + } + })(); + invocation + .then( + (result) => this.#settled(run, ctx, { kind: "done", result }), + (error: unknown) => this.#settled(run, ctx, this.#failed(run, error)), + ) + .then(work.resolve); + } + + /** + * Runs the finish of `run`, whose outcome is recorded: `finish` with the cleanup in one transaction under the + * attempt's token, retried with backoff after a throw (reported), blocked after five in a row. The attempt's seal + * stops it without a report or a count; the next live attempt finishes the run. + */ + finish(run: JobRun): void { + const work = this.#finish(run); + this.#owner.admit(work); + } + + /** How a throwing `run` ended: `aborted` after its signal fired, otherwise `threw` (reported). */ + #failed(run: JobRun, error: unknown): Settled { + if (run.abort.signal.aborted) return { kind: "failed", error: { kind: "aborted", reason: "abort" } }; + this.#records.report({ kind: "job_failed", id: run.id, error }); + const thrown = isChordError(error) || error instanceof PluginError ? error : internal(error); + return { kind: "failed", error: { kind: "threw", error: thrown } }; + } + + /** + * `run` settled: its `ctx.get` references go; its outcome is recorded unless the attempt's seal cut it off (which + * recorded its own), then the run finishes here while this attempt is live. + */ + async #settled(run: JobRun, ctx: JobContext, ended: Settled): Promise { + ctx.settle(); + if (run.token.sealed) return; + this.#invocations.delete(run); + this.#records.settled(run, ended); + } + + async #finish(run: JobRun): Promise { + const { documents, token, context, signal } = this.#owner; + this.#finishing.add(run); + let delay: number | undefined; + try { + for (let failures = 1; ; failures++) { + try { + return await documents.tx( + async (tx) => { + const staging = this.#records.staging(tx); + staging.finishing = run; + await this.#implementation.finish?.(run.outcome!, tx); + staging.changes.push({ kind: "delete", run }); + }, + token, + context, + ); + } catch (error) { + if (this.#state === "sealed") return; + this.#records.report({ kind: "job_finish_failed", id: run.id, error }); + if (failures === FINISH_ATTEMPTS) return await this.#records.record(run, "block"); + delay = nextDelay(delay); + try { + await sleep(delay, withAbortSignal(signal, context)); + } catch { + return; + } + } + } + } finally { + this.#finishing.delete(run); + } + } + + /** `ctx.state` of `run`, when the job has a state: changes under the run's own token while `run` runs. */ + #stateOf(run: JobRun): RunState | undefined { + const records = this.#records; + const states = records.states; + if (states === undefined) return undefined; + const { documents } = this.#owner; + const refused = (): ChordError => + new ChordError({ + code: "released", + reason: "settled", + message: `${records.name}: run ${run.id} settled; its state is closed`, + }); + return { + get value(): JsonObject { + const value = records.stateOf(run.id); + if (value === undefined) throw refused(); + return value; + }, + change: (fn: (draft: Draft) => T, context: Context): Promise => { + if (run.invocation !== "running") return Promise.reject(refused()); + // Never the abort signal: a run records its progress also after an abort. + const unaborted = withoutAbortSignal(context); + return documents.tx(async (tx) => fn(await tx.draftSlot(states, run.id)), run.token, unaborted); + }, + }; + } +} + +/** A plugin's throw that is neither a ChordError nor a PluginError, as a run's outcome keeps it. */ +function internal(error: unknown): PluginError { + return new PluginError("internal", error instanceof Error ? error.message : String(error)); +} + +/** A run's ctx: the handler's ctx of its object, its id and, with a state, its state document. */ +export class JobContext extends CallContext { + readonly jobId: string; + declare readonly state: RunState | undefined; + + constructor( + context: Context, + caller: Caller, + host: AnyHost, + jobId: string, + state: RunState | undefined, + ) { + super(context, caller, host); + this.jobId = jobId; + if (state !== undefined) this.state = state; + } +} diff --git a/packages/chord/src/next/jobs/records.ts b/packages/chord/src/next/jobs/records.ts new file mode 100644 index 000000000..b1e8e306b --- /dev/null +++ b/packages/chord/src/next/jobs/records.ts @@ -0,0 +1,446 @@ +// A job's records on its object (§15, slice 6a §1.2): its runs by id and by key, the active count, the run state +// documents, `running` and the runs waiting for a live attempt, across the object's attempts. Records and state +// documents change only in transactions on the object's line (staged changes adopted in the commit step). + +import type { Context } from "../../context/index.ts"; +import { type Op, type Prepared, type Tracker, track } from "../../delta/index.ts"; +import type { JsonValue } from "../../json.ts"; +import { ChordError } from "../data/index.ts"; +import { check, validatorFor } from "../data/internal.ts"; +import type { JsonObject, StateDefinition } from "../documents/index.ts"; +import type { DocumentSlot, Documents, Staged, Transaction } from "../documents/internal.ts"; +import type { AnyHost } from "../host/internal.ts"; +import { uuidv7 } from "../host/internal.ts"; +import type { JobDefinition, JobError } from "./definition.ts"; +import type { Job, StartOptions } from "./job.ts"; +import { type AnyOutcome, JobRun, type Run } from "./run.ts"; +import { Running } from "./running.ts"; + +/** What a job's records need of their object. */ +export type RecordsOwner = { + readonly label: string; + readonly host: AnyHost; + /** The object's context, without an abort signal: outcomes are recorded under it. */ + readonly context: Context; + readonly documents: Documents; + /** Counts running and finishing runs: they keep the object alive (`host.activity.jobs`). */ + countRuns(delta: number): void; +}; + +/** How a run's invocation ended, before its outcome is recorded. */ +export type Settled = + | { readonly kind: "done"; readonly result: unknown } + | { readonly kind: "failed"; readonly error: JobError }; + +/** A change of the records, staged in a transaction and adopted in its commit step. */ +type RecordChange = + | { readonly kind: "create"; readonly run: JobRun; readonly state: JsonObject | undefined } + | { readonly kind: "outcome"; readonly run: JobRun; readonly outcome: AnyOutcome } + | { readonly kind: "delete" | "block" | "drop"; readonly run: JobRun }; + +/** One transaction's changes of one job: the runs it started (dedupe, concurrency) and whose finish it is. */ +export type Staging = Staged & { + readonly started: JobRun[]; + readonly changes: RecordChange[]; + finishing: JobRun | undefined; +}; + +const RESTART: Settled = { kind: "failed", error: { kind: "aborted", reason: "restart" } }; + +/** + * The records of one job name on one object. The live attempt's `Job` invokes and finishes its runs; between live + * attempts, committed runs not yet invoked and runs whose outcome is recorded wait for the next one. + */ +export class JobRecords { + readonly definition: JobDefinition; + readonly running: Running; + readonly #owner: RecordsOwner; + readonly #states: RunStates | undefined; + readonly #runs = new Map(); + readonly #keys = new Map(); + /** Running and finishing runs: concurrency and keep-alive count them. */ + #active = 0; + /** The live attempt's job; `undefined` between live attempts. */ + #job: Job | undefined; + /** Runs waiting for a live attempt: to be invoked (`invocation` pending) or finished (phase `finishing`). */ + readonly #waiting = new Set(); + /** The object's end error, once it ended. */ + #ended: ChordError | undefined; + + constructor(definition: JobDefinition, owner: RecordsOwner) { + this.definition = definition; + this.#owner = owner; + const { state } = definition; + this.#states = + state === undefined + ? undefined + : new RunStates(state, owner.documents, (id, ops) => this.#stateChanged(id, ops)); + this.running = new Running(owner.documents.batch, this.name); + } + + /** The job in errors: its object and its name. */ + get name(): string { + return `${this.#owner.label} job ${this.definition.name}`; + } + + /** Waits for the outcome of the unsettled run `id`; a settled or unknown id rejects `gone`. */ + outcome(id: string, context: Context): Promise { + const run = this.#runs.get(id); + if (run === undefined) { + return Promise.reject(new ChordError({ code: "gone", message: `${this.name}: no unsettled run ${id}` })); + } + return run.handle.outcome(context); + } + + /** Fires a running run's signal; drops a blocked run; ignored for finishing runs, unknown ids and after the end. */ + abort(id: string): void { + const run = this.#runs.get(id); + if (run === undefined || this.#ended !== undefined) return; + if (run.phase === "running") { + run.abort.abort(new ChordError({ code: "aborted", message: `${this.name}: run ${id} aborted` })); + } else if (run.phase === "blocked") { + void this.record(run, "drop"); + } + } + + /** See `Job.stage`: the checks and the staged creation, against the object's runs. */ + stage(tx: Transaction, args: unknown, context: Context, options: StartOptions): Run { + const definition = this.definition; + // Plugin code, not type-checked: the args are checked as a method's are (R7). + check(validatorFor(definition.args, context), args, "args", context); + const staged = tx.staged(this); + const { key } = options; + if (key !== undefined) { + const known = staged?.started.find((run) => run.key === key) ?? this.#keys.get(key); + if (known !== undefined && known !== staged?.finishing) return known.handle; + } + const { concurrency } = definition; + const active = this.#active + (staged?.started.length ?? 0) - (staged?.finishing === undefined ? 0 : 1); + if (concurrency !== undefined && active >= concurrency) { + const message = `${this.name}: at its concurrency limit (${concurrency})`; + throw new ChordError({ code: "unavailable", reason: "busy", message }); + } + const state = definition.state?.initial(context); + // Checked above: the args are JSON (the definition refuses bytes). + const run = new JobRun(uuidv7(), definition.version, key, args as JsonValue); + const staging = this.staging(tx); + staging.started.push(run); + staging.changes.push({ kind: "create", run, state }); + return run.handle; + } + + /** This job's changes in `tx`, made on the first. */ + staging(tx: Transaction): Staging { + return tx.stage(this, () => { + const changes: RecordChange[] = []; + return { + started: [], + changes, + finishing: undefined, + adopt: () => { + // A step admitted before the object ended adopts after it: the records are not changed any more. + if (this.#ended !== undefined) return; + for (const change of changes) this.#adopt(change); + }, + committed: () => { + const ended = this.#ended; + if (ended === undefined) { + for (const change of changes) this.#committed(change); + return; + } + // Its runs end as the object's other runs did. + for (const change of changes) if (change.kind === "create") change.run.reject(ended); + }, + discarded: () => { + for (const change of changes) if (change.kind === "create") change.run.reject(this.#discarded()); + }, + }; + }); + } + + /** `job`, of the attempt that went live, invokes and finishes the runs from now on; the waiting ones first. */ + attach(job: Job): void { + if (this.#ended !== undefined) return; + this.#job = job; + const waiting = [...this.#waiting]; + this.#waiting.clear(); + for (const run of waiting) this.#dispatch(run, job); + } + + /** + * `job`'s attempt was sealed: its running invocations are cut off (`cut`), the runs it was finishing wait for the + * next live attempt. + */ + detach(job: Job, finishing: Iterable): void { + if (this.#job === job) this.#job = undefined; + for (const run of finishing) this.#need(run); + } + + /** + * A run's invocation settled: its outcome is recorded (phase `finishing`) in a transaction of the object behind the + * state changes admitted before, so `outcome.state` has them; then it is finished on the live attempt. + */ + settled(run: JobRun, ended: Settled): void { + void this.#recordOutcome(run, ended); + } + + /** + * A running run whose attempt was sealed while the object continues: its signal fires with `error`, its invocation + * is fenced (later state changes refused, the open one discarded, its settling ignored), and it ends `failed { + * aborted, reason: "restart" }`, finished on the next live attempt. + */ + cut(run: JobRun, error: ChordError): void { + run.invocation = "settled"; + this.#owner.documents.fence(run.token); + run.abort.abort(error); + if (this.#ended === undefined) void this.#recordOutcome(run, RESTART); + } + + /** + * Records `kind` for `run` in a transaction of the object: `block` a finishing run, `drop` a blocked one (its + * record, key and state document go), if it still is one there. The object's end ends it. + */ + async record(run: JobRun, kind: "block" | "drop"): Promise { + const { documents, context } = this.#owner; + try { + await documents.tx( + (tx) => { + const current = this.#runs.get(run.id) === run; + if (current && run.phase === (kind === "block" ? "finishing" : "blocked")) { + this.staging(tx).changes.push({ kind, run }); + } + }, + documents.own, + context, + ); + } catch { + // The object ended: its runs went with it. + } + } + + /** The value of run `id`'s state document, `undefined` without one or once retired. */ + stateOf(id: string): JsonObject | undefined { + return this.#states?.value(id); + } + + /** The run state documents, for `ctx.state.change`; `undefined` without a state definition. */ + get states(): DocumentSlot | undefined { + return this.#states; + } + + /** + * The object ends: running runs' signals fire, every waiter rejects with `error`, blocked runs are dropped + * (reported); nothing finishes. The records stay readable until `drop`. + */ + end(error: ChordError): void { + if (this.#ended !== undefined) return; + this.#ended = error; + this.#job = undefined; + this.#waiting.clear(); + for (const run of this.#runs.values()) { + if (run.phase === "blocked") this.report({ kind: "job_dropped", id: run.id }); + run.abort.abort(error); + run.reject(error); + } + if (this.#active > 0) this.#owner.countRuns(-this.#active); + } + + /** Its object's documents are dropped: the runs, their state documents and `running`'s watches go. */ + drop(): void { + this.#runs.clear(); + this.#keys.clear(); + this.#states?.drop(); + this.running.end(); + } + + /** Reports a run's failure or drop to the host. */ + report( + error: + | { readonly kind: "job_failed" | "job_finish_failed"; readonly id: string; readonly error: unknown } + | { readonly kind: "job_dropped"; readonly id: string }, + ): void { + this.#owner.host.report({ ...error, object: this.#owner.label, job: this.definition.name }); + } + + /** Records the outcome of `run`, then has it finished on the live attempt. */ + async #recordOutcome(run: JobRun, ended: Settled): Promise { + const { documents, context } = this.#owner; + try { + await documents.tx( + (tx) => { + const outcome = { id: run.id, args: run.args, state: this.stateOf(run.id), ...ended }; + this.staging(tx).changes.push({ kind: "outcome", run, outcome }); + }, + documents.own, + context, + ); + } catch { + // The object ended: nothing finishes. + return; + } + this.#need(run); + } + + /** `run` needs a live attempt (to be invoked or finished): now, or once one attaches. */ + #need(run: JobRun): void { + if (this.#ended !== undefined || this.#runs.get(run.id) !== run) return; + const job = this.#job; + if (job === undefined) this.#waiting.add(run); + else this.#dispatch(run, job); + } + + /** Invokes or finishes `run` on `job`, as its record stands. */ + #dispatch(run: JobRun, job: Job): void { + if (this.#runs.get(run.id) !== run) return; + if (run.phase === "finishing") job.finish(run); + else if (run.phase === "running" && run.invocation === "pending") job.invoke(run); + } + + /** + * Adopts one record change, inside the commit step: the maps, the active count, the state document, `running`. A + * new run is counted for its object here, before the commit's publication (a link closing while it sends may + * release the object's last reference) and before any run of the same commit settles in `#committed`. + */ + #adopt(change: RecordChange): void { + const { run } = change; + switch (change.kind) { + case "create": + this.#runs.set(run.id, run); + if (run.key !== undefined) this.#keys.set(run.key, run); + this.#active++; + this.#owner.countRuns(1); + if (change.state !== undefined) this.#states?.create(run.id, change.state); + run.entry = run.entryFor(change.state); + this.running.add(run.entry); + return; + case "outcome": + run.outcome = change.outcome; + run.phase = "finishing"; + this.#replaceEntry(run); + return; + case "block": + run.phase = "blocked"; + this.#active--; + this.#replaceEntry(run); + return; + case "delete": + case "drop": + this.#runs.delete(run.id); + // The finishing run's key may name its successor already, started in the same commit. + if (run.key !== undefined && this.#keys.get(run.key) === run) this.#keys.delete(run.key); + if (change.kind === "delete") this.#active--; + this.#states?.retire(run.id); + this.running.remove(run.entry!); + run.entry = undefined; + return; + } + } + + /** The actions after a record change committed. */ + #committed(change: RecordChange): void { + const { run } = change; + switch (change.kind) { + case "create": + // After the commit, outside the batch, once the transaction settled; on the live attempt, or the next. + queueMicrotask(() => this.#need(run)); + return; + case "outcome": + return; + case "delete": + this.#owner.countRuns(-1); + run.resolve(run.outcome!); + return; + case "block": + this.#owner.countRuns(-1); + return; + case "drop": + this.report({ kind: "job_dropped", id: run.id }); + run.reject(new ChordError({ code: "aborted", message: `${this.name}: run ${run.id} dropped` })); + return; + } + } + + /** A run's state document changed by `ops`, inside the commit step: its entry is replaced. */ + #stateChanged(id: string, ops: readonly Op[]): void { + const run = this.#runs.get(id)!; + const before = run.entry!; + run.entry = run.entryFor(this.stateOf(id)); + this.running.stateChanged(before, run.entry, ops); + } + + /** A run's record changed, inside the commit step: its entry is replaced. */ + #replaceEntry(run: JobRun): void { + const before = run.entry!; + run.entry = run.entryFor(this.stateOf(run.id)); + this.running.replace(before, run.entry); + } + + #discarded(): ChordError { + return new ChordError({ + code: "released", + reason: "discarded", + message: `${this.name}: the start's transaction was discarded`, + }); + } +} + +/** + * A job's state documents, one per run by its id: Chord's own keyed slot, apart from the object's `self.state` slots. + * Created with the run and retired with its record; never drafted while absent, so a retired one is never recreated. + * Each commit tells the job, whose `running` carries the state. + */ +class RunStates implements DocumentSlot { + readonly owner: Documents; + /** In memory only (persisted jobs are slice 6b's). */ + readonly persisted = undefined; + readonly #definition: StateDefinition; + readonly #trackers = new Map>(); + readonly #changed: (id: string, ops: readonly Op[]) => void; + + constructor( + definition: StateDefinition, + owner: Documents, + changed: (id: string, ops: readonly Op[]) => void, + ) { + this.#definition = definition; + this.owner = owner; + this.#changed = changed; + } + + /** The value of run `id`'s document; `undefined` once retired. */ + value(id: string): JsonObject | undefined { + return this.#trackers.get(id)?.value; + } + + current(key: string): Tracker | undefined { + return this.#trackers.get(key); + } + + initial(_context: Context): JsonObject { + throw new ChordError({ + code: "released", + reason: "settled", + message: `${this.#definition.name}: the run's document is retired`, + }); + } + + adopt(key: string, tracker: Tracker, prepared: Prepared): void { + tracker.adopt(prepared); + this.#changed(key, prepared.ops); + } + + /** Retires run `key`'s document (its record's deletion). */ + retire(key: string): void { + this.#trackers.delete(key); + } + + /** Creates run `id`'s document with `value` (its start's commit). */ + create(id: string, value: JsonObject): void { + this.#trackers.set(id, track(value)); + } + + /** Drops every document (the object ended). */ + drop(): void { + this.#trackers.clear(); + } +} diff --git a/packages/chord/src/next/jobs/run.ts b/packages/chord/src/next/jobs/run.ts new file mode 100644 index 000000000..688e0efa1 --- /dev/null +++ b/packages/chord/src/next/jobs/run.ts @@ -0,0 +1,137 @@ +// One run of a job (§15): its record (id, version, key, args, phase, outcome), its abort signal, the fence of its +// invocation (its seal token), its `running` entry and the `Run` handle its callers wait on. + +import type { Context } from "../../context/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { ChordError } from "../data/index.ts"; +import { SealToken } from "../documents/internal.ts"; +import type { Outcome } from "./definition.ts"; + +/** A started run as its caller holds it: its id and a wait for its outcome. */ +export interface Run { + readonly id: string; + /** + * Resolves with the outcome once `finish` committed (also without a `finish`); a blocked run keeps waiting, a dropped + * one rejects `aborted`, the object's end rejects with its error; `context` ends only this wait, which then keeps + * nothing of it. A rejection's error keeps its stack's receivers (the transaction that ended the run) until its + * `stack` is read (V8). + */ + outcome(context: Context): Promise; +} + +/** A run's outcome, as Chord keeps it whatever the job's types. */ +export type AnyOutcome = Outcome; + +/** + * Where a run's record stands. `running` from its start's commit (its `run` starts after it), `finishing` once its + * outcome is recorded, `blocked` after five failed finishes. + */ +type Phase = "running" | "finishing" | "blocked"; + +/** Where a run's invocation stands: `ctx.state.change` is admitted only while it runs (the invocation fence). */ +type Invocation = "pending" | "running" | "settled"; + +/** How a run settled for its waiters. */ +type Answer = + | { readonly kind: "done"; readonly outcome: AnyOutcome } + | { readonly kind: "failed"; readonly error: ChordError }; + +/** One wait for a run's outcome, removed when its context aborts. */ +type Waiter = { readonly settle: (settled: Answer) => void }; + +/** One run: changed only by its job, inside its transactions' commit steps (records) and after them (actions). */ +export class JobRun { + readonly id: string; + readonly version: number; + readonly key: string | undefined; + readonly args: JsonValue; + /** `run`'s abort signal: `abort(id)`, the object's end and `host.close` fire it; an early abort pre-marks it. */ + readonly abort = new AbortController(); + readonly handle: Run; + /** The seal token of its invocation: sealed when the invocation is cut off by its attempt's seal (`ctx.state.change`). */ + readonly token = new SealToken(); + phase: Phase = "running"; + invocation: Invocation = "pending"; + /** What it ended with, once recorded (finishing and blocked). */ + outcome: AnyOutcome | undefined; + /** Its entry in `running`, while its record exists; replaced when its record or state document changes. */ + entry: JsonValue | undefined; + /** Its waiters: apart from the run, so a handle kept by a caller keeps only how the run settled. */ + readonly #waiters = new Waiters(); + + constructor(id: string, version: number, key: string | undefined, args: JsonValue) { + this.id = id; + this.version = version; + this.key = key; + this.args = args; + const waiters = this.#waiters; + this.handle = { id, outcome: (context) => waiters.wait(context) }; + } + + /** Its `running` entry for its current record and `state` (the job's state document, `undefined` without one). */ + entryFor(state: JsonValue | undefined): JsonValue { + const { id, version, key, args, phase } = this; + if (phase === "blocked") return { id, version, phase, reason: "finish_failed" }; + return { + id, + version, + ...(key === undefined ? {} : { key }), + args, + ...(state === undefined ? {} : { state }), + phase, + }; + } + + /** Its waiters get the outcome. */ + resolve(outcome: AnyOutcome): void { + this.#waiters.settle({ kind: "done", outcome }); + } + + /** Its waiters get `error`. */ + reject(error: ChordError): void { + this.#waiters.settle({ kind: "failed", error }); + } +} + +/** A run's waits for its outcome: each removed when its context aborts; late ones answered at once. */ +class Waiters { + /** How the run settled, once it did. */ + #settled: Answer | undefined; + readonly #waiting = new Set(); + + /** The run settled (once): every wait gets it. */ + settle(settled: Answer): void { + if (this.#settled !== undefined) return; + this.#settled = settled; + for (const waiter of this.#waiting) waiter.settle(settled); + this.#waiting.clear(); + } + + /** A wait for the outcome: at once once settled; otherwise until it settles or `context` aborts. */ + wait(context: Context): Promise { + const settled = this.#settled; + if (settled !== undefined) return answer(settled); + const signal = context.abortSignal; + if (signal?.aborted) return Promise.reject(signal.reason); + const { promise, resolve, reject } = Promise.withResolvers(); + const abort = (): void => { + this.#waiting.delete(waiter); + reject(signal?.reason); + }; + const waiter: Waiter = { + settle: (each) => { + signal?.removeEventListener("abort", abort); + if (each.kind === "done") resolve(each.outcome); + else reject(each.error); + }, + }; + this.#waiting.add(waiter); + signal?.addEventListener("abort", abort, { once: true }); + return promise; + } +} + +/** A settled run's answer to a wait. */ +function answer(settled: Answer): Promise { + return settled.kind === "done" ? Promise.resolve(settled.outcome) : Promise.reject(settled.error); +} diff --git a/packages/chord/src/next/jobs/running.ts b/packages/chord/src/next/jobs/running.ts new file mode 100644 index 000000000..3f24ccd24 --- /dev/null +++ b/packages/chord/src/next/jobs/running.ts @@ -0,0 +1,143 @@ +// A job's `running` (§15): a JSON watchable of its unsettled runs, one entry per run in start order, tagged by phase. +// It is a source of its own (keyed state documents publish nothing): the job tells it each change in the commit step, +// and a frame carries exactly the ops of those changes, a run's state ops re-pathed under its entry. + +import type { Context } from "../../context/index.ts"; +import type { NonEmptyPath, Op, Path } from "../../delta/index.ts"; +import type { JsonValue } from "../../json.ts"; +import { ChordError } from "../data/index.ts"; +import { type Batch, Source } from "../state/internal.ts"; + +/** + * One entry of `running`: a running or finishing run with its args (and state, when the job has a state document), or + * a blocked run, which carries neither (it may be of another version than the code). + */ +export type RunningEntry = + | ({ + readonly id: string; + readonly version: number; + readonly key?: string; + readonly args: A; + readonly phase: "running" | "finishing"; + } & ([S] extends [undefined] ? unknown : { readonly state: S })) + | { readonly id: string; readonly version: number; readonly phase: "blocked"; readonly reason: "finish_failed" }; + +const EMPTY: readonly JsonValue[] = []; + +/** + * The list behind `running`. Every change copies the list (O(unsettled runs)) and, while it is watched, records its + * ops; the batch delivers them as one frame. Unwatched, no op is built. + */ +export class Running extends Source { + /** The job in errors. */ + readonly #label: string; + /** `undefined` once its object ended: reads throw `released`, and no ended run's args or state are kept. */ + #list: readonly JsonValue[] | undefined = EMPTY; + /** The ops of the current batch's changes, kept only while watched. */ + #ops: Op[] = []; + /** Whether the current batch listed this source for a frame. */ + #listed = false; + + constructor(batch: Batch, label: string) { + super(batch); + this.#label = label; + } + + get value(): JsonValue { + // A list of JSON entries, shared and never mutated once published. + return this.#current() as JsonValue; + } + + /** Appends the entry of a new run. */ + add(entry: JsonValue): void { + const list = [...this.#current(), entry]; + this.#list = list; + if (this.#mark()) this.#ops.push(["p", [], list.length - 1, 0, [entry]]); + } + + /** Replaces a run's entry `before` by `after` (its record changed). */ + replace(before: JsonValue, after: JsonValue): void { + const index = this.#replaced(before, after); + if (this.#mark()) this.#ops.push(["s", [index], after]); + } + + /** Replaces a run's entry `before` by `after`, whose state document changed by `ops`. */ + stateChanged(before: JsonValue, after: JsonValue, ops: readonly Op[]): void { + const index = this.#replaced(before, after); + if (this.#mark()) for (const op of ops) this.#ops.push(under([index, "state"], op)); + } + + /** Removes a run's entry. */ + remove(entry: JsonValue): void { + const list = this.#current(); + const index = list.indexOf(entry); + this.#list = list.toSpliced(index, 1); + if (this.#mark()) this.#ops.push(["p", [], index, 1, []]); + } + + deliver(context: Context): void { + const ops = this.#ops; + this.#ops = []; + this.#listed = false; + this.publish(this.#current() as JsonValue, ops, false, context); + } + + /** Its object ended: every watch ends, and the list goes. */ + end(): void { + this.#list = undefined; + this.#ops = []; + this.endWatches(); + } + + #current(): readonly JsonValue[] { + if (this.#list === undefined) { + throw new ChordError({ code: "released", reason: "ended", message: `${this.#label}: ended` }); + } + return this.#list; + } + + /** Copies the list with `after` in the place of `before`; returns that place. */ + #replaced(before: JsonValue, after: JsonValue): number { + const list = this.#current(); + const index = list.indexOf(before); + this.#list = list.with(index, after); + return index; + } + + /** + * Records that the list changed, in the batch running now: derives over it are marked; while watched it is listed + * once per batch for a frame. Returns whether the change's ops are to be recorded (only while watched). + */ + #mark(): boolean { + if (!this.watched) { + this.changed(); + return false; + } + if (!this.#listed) { + this.#listed = true; + this.changed(); + } + return true; + } +} + +/** `op` of a value at `prefix` inside a larger one: a replacement becomes a set there. */ +function under(prefix: readonly [number, string], op: Op): Op { + const at = (path: Path): NonEmptyPath => [prefix[0], prefix[1], ...path]; + switch (op[0]) { + case "r": + return ["s", prefix, op[1]]; + case "s": + return ["s", at(op[1]), op[2]]; + case "d": + return ["d", at(op[1])]; + case "a": + return ["a", at(op[1]), op[2]]; + case "t": + return ["t", at(op[1]), op[2]]; + case "p": + return ["p", at(op[1]), op[2], op[3], op[4]]; + case "m": + return ["m", at(op[1]), op[2]]; + } +} diff --git a/packages/chord/src/next/link/README.md b/packages/chord/src/next/link/README.md new file mode 100644 index 000000000..381fea216 --- /dev/null +++ b/packages/chord/src/next/link/README.md @@ -0,0 +1,141 @@ +# link + +Links between hosts and the protocol on them (§3, §22, slices 3a 2b, 4a and 4b): one `Message` type checked after +every decode (skipped on trusted links), the codecs that turn it into bytes, opaque parts with their limits and +compression at the origin, the handshake, the heartbeat, in-order handling of messages waiting for an asynchronous +inflate, the IPC and WebSocket transports, values with raw byte segments, and errors on the wire. + +## For apps (`index.ts`) + +```ts +import { memoryLinkPair, textCodec } from "./index.ts"; + +const [up, down] = memoryLinkPair(); +server.accept(up, { principal: { kind: "guest", session: "S1" } }, ctx); // the entry point authenticated it +client.connect(down, { principal: { kind: "server" } }, ctx); // both ends: binaryCodec by default +// For a readable wire while debugging: { principal, codec: textCodec } on both ends. + +// A forked worker over IPC (the parent end; the child uses ipcLink(process)), trusted: no message schema check. +const child = fork("worker.ts", [], { serialization: "advanced" }); +server.accept(ipcLink(child), { principal: { kind: "worker" }, trusted: true }, ctx); +// A browser tab over a WebSocket (`ws` on the server, the browser's WebSocket in the tab). +wss.on("connection", (ws, req) => server.accept(webSocketLink(ws), { principal: authenticate(req) }, ctx)); +tab.connect(webSocketLink(new WebSocket(url)), { principal: { kind: "server" } }, ctx); +``` + +- `Link`: an ordered, bidirectional channel of `Uint8Array` messages: `send(message)` (a no-op once closed), + `onMessage(listener)` (messages that arrived earlier are delivered to it first; not kept on a closed link), + `close()` (both ends; undelivered messages are dropped), `onClose(listener)` (runs at once on a closed link). + Adapters only move whole messages, in order, and close both ends. +- `memoryLinkPair()`: two ends in one process; each message arrives a microtask later (`queueMicrotask`), in order. + Closing one end closes it at once and the other a microtask later. +- `ipcLink(port)`: over a Node IPC channel, a `ChildProcess` forked with `serialization: "advanced"` or `process` in + the child (`IpcPort`, typed structurally: Chord imports nothing from Node). Each message is one `Uint8Array`; other + messages on the channel (the app's own, such as a start message) are skipped, and messages that arrive before the + link's listener is set are queued, so a child makes its link before awaiting its start message. `send` returning + `false` is backpressure, not a failure; a thrown `send`, an error in its callback or `disconnect` closes the link once + and removes its listeners. +- `webSocketLink(socket)`: over a WebSocket, the browser's or `ws`'s (`WebSocketLike`, typed structurally): binary + frames, `binaryType = "arraybuffer"`; messages sent while it connects wait for `open`; a socket closing or closed + already gives a closed link; `close`, `error` and a text frame close the link once. The transport's `maxPayload` + is its hard cap (a larger frame closes the socket, code 1009); the limits per part are the host's (`host/`). +- `Compression` (`{ deflate(bytes), inflate(bytes, limit) }`, synchronous): what `createHost({ compression })` takes. + `deflate` is raw deflate at level 1; `inflate` gives a tagged `Inflated`: `{ kind: "bytes", bytes }` (at most + `limit` bytes; `Infinity`: none; 0 allowed), `{ kind: "too_large" }` when the output would be larger, `{ kind: + "corrupt", error }` for data that is no raw deflate stream. Chord ships `nodeCompression` + (zlib's `deflateRawSync`/`inflateRawSync` with `maxOutputLength`) in `src/next-node/compression.ts`, outside the + portable code. `PartLimits` (`{ part?, decoded? }`, bytes) is what a link's `limits` take. +- `Codec { name, encode(message), decode(bytes) }`, chosen per link (`codec` in `accept`/`connect` and a route's + `resolve` result, the same at both ends; no negotiation). Two links of one relay may use different codecs: + - `binaryCodec` (the default): a kind byte (its high bit set when an `origin` follows as a varint), then the kind's + fields: unsigned varints (ids, indexes, counts, + lengths), length-prefixed UTF-8 strings, the incarnation as 16 and the fingerprint as 8 raw bytes, parts as + length-prefixed bytes (decoded as views of the received message, never parsed), other JSON fields (a hold's + address and principal, errors) as length-prefixed JSON; a hold's optional fields behind one flags byte; a payload + marked as carrying bytes adds its segments (count, lengths, raw bytes); `failed`'s `retryIn` as one more than it + (0: absent). A part's length prefix is its length doubled, plus one when the part is deflated. A hold's `chain` + as a count and 16 raw bytes per instance id, its `limit` as a varint. No outer length. + - `textCodec`, for debugging: one JSON tuple per message with a short kind tag and positional fields (`["R", hold]`, + `["c", call, hold, method, args?]`, `["b", [[hold, variant, [[member, value]…], [[member, error]…]?]…]]`, `["s", + [[hold, [[member, ops]…], [[member, error]…]?]…]]`, `["f", hold, "t" | "b", error, retryIn?]`, `["T", hold]` for + `retry`, `["l", hold]` for `loading`, `["a"]` ack, `["C"]` closed, `["p"]` ping, `["P"]` pong, …; a message with an origin wrapped as + `["@", origin, tuple]`); parts are + spliced into the tuple as their JSON text and re-serialized at decode (one parse per part per hop, accepted for a + debugging codec); a payload marked as carrying bytes ends with its segments' lengths, the segments following the + tuple after a newline (JSON text has no raw one). Every part is tagged with its codec, so no JSON value is + reserved: `[0, json]`, or `[1, base64]` for a part its origin deflated (the text codec itself never compresses); + `["V", revocation]` is `revoke`. + - `describeMessage(message)`: the text tuple form of any message, for logs and tests. +- `accept`, `connect` and a route's `resolve` take `trusted` (default `false`): a trusted link skips the `MESSAGE` + schema check (framing, the codec's decode and the owners' checks still run). For links to the app's own processes; + trust never grants a hold's principal fields (they are read only on a parent link, `remote/`). + +## For Chord's layers (`internal.ts`) + +- `Message` (`message.ts`): the protocol's one tagged union, from the TypeBox schema `MESSAGE`: `hello { protocol, + incarnation, calls }` (`calls`: how many concurrent calls this side accepts per origin, its call window), + `hold { hold, type, version, fingerprint, params, address?, principal?, incarnation? }` (`address`: `{ kind, params }` + the caller's placement computed, it routes the hold; the provider places it by its params; absent for a type without placement; `principal` and `incarnation`: whom it acts for, read only + on a parent link; `chain?`: the instance ids of the activations waiting for the hold, at most 32, read only on a + trusted link; `limit?`: the decoded limit per part of the link a relay admitted it on, read only on a parent link), `release`, `call { call, hold, method, args?, segments? }`, `abort`, `bound { holds: [{ hold, + variant, members: [{ member, value }], failed }] }` (one message may bind several holds: first bindings, re-holds), + `state { holds: [{ hold, members: [{ member, ops }], failed }] }`, `ack {}` (one per applied `bound` or `state` of + its origin), `closed { origin }` (a relay's downstream origin ended: its holds and calls go), `ping {}`, `pong {}`, + `revoke { revocation }` (from the parent, applied with the child's own `revokedBy`; no origin), `failed { hold, cause, + retryIn? }` (`retryIn`: the milliseconds left of the provider's backoff window, absent for a broken failure), + `loading { hold }` (the hold's object, or a relay's upstream, starts over), `retry { hold }` (the consumer asks for an + attempt now), `result { call, value?, segments? }`, `error`, `gone`. `error`, `gone` and a `failed` cause carry the + wire error format (`WireError`: a ChordError's JSON, or a plugin's code and message). `args` absent is a method without arguments, + `value` absent is `null`; `segments` present marks a payload with byte segments (its method's schema has a `bytes()` + leaf, `Snapshot.bytes`). `member`, `method` and `variant` are indexes into the hold's bound version's names, each + variant's sorted (`variantNames`, `variantOrder` in contracts). Every kind but `hello`, `revoke`, `ping` and `pong` may carry + `origin` (an integer from 1; absent: the sender's own work): a relay's number for one of its downstream origins, + keying holds, calls, windows and acks with the ids (`OriginMessage`; decision "Relay flow control per origin"). + `PROTOCOL_VERSION` is 1. +- `Part = string | Uint8Array | DeflatedPart` (`params`, `args`, `value`, `bound` member values, `state` member ops, + `part.ts`): a value's JSON text at its origin (and after a text decode), its UTF-8 bytes once a binary codec + received it, or those bytes deflated by an origin that compresses. A part is opaque on the way and opened and parsed + only where it is owned, while its message is handled (a malformed part is still a protocol error that closes the + link): the provider opens `params` and `args`, the consumer results, snapshots and ops. A relay forwards parts + unparsed and still deflated; a message naming an unknown id is ignored before its parts are parsed (Q1). +- Parts and their limits (`part.ts`): `originPart(json, compression, ctx)` (a JSON part at its origin, deflated above + 16 KiB of UTF-8, `COMPRESS_ABOVE`, when the host compresses and only if it got smaller, else the bytes encoded for + the attempt; counted `chord.link.deflates`), + `oversized(part, limit)` (the `too_large/part` error of a part past a wire limit: one comparison), + `openPart(part, limits, compression, ctx)` (the JSON its owner parses: the wire size checked, then the decoded size, + a deflated part inflated within it; counted `chord.link.inflates`), `inflateAsync(part, limit, signal, ctx)` (the + `DecompressionStream` fallback, counting output bytes and stopping past the limit; it leaves its tagged `Inflated` + on the part for `openPart`; `signal`, the link's close, cancels it at once), `linkLimits(trusted, limits?)` (none on + trusted links, 16 MiB on the wire and 64 MiB decoded otherwise). Sizes are UTF-8 bytes (a text part's counted + without encoding it); byte segments are not parts and no part limit bounds them (the transport's cap does). +- `Channel`: a host's end of a link. It sends `hello { protocol, incarnation, calls }` first and queues its own messages + until the peer's arrived. Every received message is decoded by the codec and, unless the link is trusted, checked + with `checkMessage` (the compiled `MESSAGE` schema, counted in `chord.link.checks`); a first message that is no + `hello` of this codec ("the first message is no binary message"), another protocol version, a second `hello`, a + failed check, or a throw while its `ChannelHandler` handles a message, is a protocol error: reported + (`protocol_error`), and the link closes. `peer` is the peer's incarnation, `peerCalls` its call window. A handler + that returns a promise (a message waiting for an asynchronous inflate) holds every later message, `ping` included, + until it settles; they are then handled in order, and dropped when the link closes meanwhile; a rejection is a + protocol error. +- The heartbeat, in `Channel`: every received message stamps `lastReceived` (sends do not count); the host's one + interval timer calls `tick(now, ping, close)` on every link: a link silent for `close` ms (30 s by default), the + handshake included, closes; an open one silent for `ping` ms (15 s) sends one `ping`. A `ping` is answered with + `pong` at once, never queued behind the window; neither reaches the handler. It detects a silent transport, not a + peer that answers pings but withholds acks (the window and the pending cap bound that). +- Who checks what (layers 1-3 failing close the link): + 1. the codec's `decode`: lengths and varints fit, the bytes consumed exactly, a known kind and arity, strict UTF-8, + each JSON value parses, segment references valid and only in payloads marked as carrying bytes; + 2. `checkMessage`: the envelope (kinds, ids and indexes, lists, error shapes), not the parts; skipped on a trusted + link; + 3. the remote side (`remote/`): ids not reused while live (unknown ids ignored), indexes within the bound version, + `bound` before `state`, at most N concurrent calls; + 4. owners, where a part is owned: `checkParams`, `checkArgs`, a route's address check, Delta's decoder and + `applyImmutable` before adoption, `chordErrorFromJson`; snapshots and results are never validated. +- Parts (`values.ts`): `jsonPart(value)` (a value's JSON text), `payloadPart(value, bytes)` (a payload's JSON, with + segments and `{"$bytes": i}` references when marked: an own key `$bytes` there fails `invalid/args`; elsewhere it is + plain data), `parsePart(part)` and `parsePayload(part, segments)` (strict UTF-8 JSON; references read back as views + of the message, no copy). +- Errors: `wireError(thrown)` is `{ kind: "chord", error }` for a ChordError, `{ kind: "plugin", code, message, + details? }` for a PluginError, and `{ kind: "plugin", code: "internal", message }` for anything else; never a stack + or a cause. `errorFromWire` rebuilds a ChordError (L1's envelope check; a malformed one throws, a protocol error) or + a PluginError. Failure causes and `gone` carry ChordError JSON directly. diff --git a/packages/chord/src/next/link/binary.ts b/packages/chord/src/next/link/binary.ts new file mode 100644 index 000000000..2786e0601 --- /dev/null +++ b/packages/chord/src/next/link/binary.ts @@ -0,0 +1,565 @@ +// The binary codec (2b): a kind byte (its high bit set when an origin follows as a varint), then the kind's fields as +// unsigned varints, length-prefixed UTF-8 strings, raw incarnation (16 bytes) and fingerprint (8 bytes), parts as +// length-prefixed bytes whose prefix also says whether the part is deflated (decoded as views of the message, never +// parsed or inflated here), and byte segments only beside payloads marked as carrying bytes. No outer length: links +// deliver whole messages. + +import { + type BoundEntry, + type Codec, + type FailedMember, + type Message, + NO_RETIRED, + type StateEntry, +} from "./message.ts"; +import { DeflatedPart, type Part } from "./part.ts"; +import { parseJson, readUtf8 } from "./values.ts"; + +const UTF8 = new TextEncoder(); + +/** Kinds by their byte. */ +const KINDS = [ + "hello", + "hold", + "release", + "call", + "abort", + "bound", + "state", + "failed", + "result", + "error", + "gone", + "retry", + "loading", + "ack", + "closed", + "ping", + "pong", + "revoke", +] as const; + +const KIND_BYTES = new Map(KINDS.map((kind, byte) => [kind, byte])); + +/** The kind byte's flag for an origin. */ +const ORIGIN = 0x80; + +/** A payload's flags: present, and carrying byte segments. */ +const PRESENT = 1; + +const SEGMENTS = 2; + +/** A hold's optional fields, one bit each. */ +const ADDRESS = 1; + +const PRINCIPAL = 2; + +const INCARNATION = 4; + +const CHAIN = 8; + +const LIMIT = 16; + +/** A part's length prefix is its length doubled, plus one when the part is deflated. */ +const DEFLATED = 1; + +/** Messages as bytes; the default codec of every link. */ +export const binaryCodec: Codec = { + name: "binary", + encode(message) { + const writer = new Writer(); + const origin = "origin" in message ? message.origin : undefined; + if (origin === undefined) { + writer.byte(KIND_BYTES.get(message.kind)!); + } else { + writer.byte(KIND_BYTES.get(message.kind)! | ORIGIN); + writer.varint(origin); + } + encodeFields(message, writer); + return writer.finish(); + }, + decode(bytes) { + const reader = new Reader(bytes); + const byte = reader.byte(); + const origin = byte & ORIGIN ? reader.varint() : undefined; + const kind = KINDS[byte & ~ORIGIN]; + if (kind === undefined) throw new Error("binary: an unknown kind"); + const message = decodeFields(kind, reader); + reader.end(); + // A fresh object; the message schema refuses an origin on a kind without one. + if (origin !== undefined) (message as { origin?: number }).origin = origin; + return message; + }, +}; + +function encodeFields(message: Message, writer: Writer): void { + switch (message.kind) { + case "hello": + writer.varint(message.protocol); + writer.raw(hex(message.incarnation.replaceAll("-", ""), 16)); + writer.varint(message.calls); + return; + case "hold": { + writer.varint(message.hold); + writer.string(message.type); + writer.varint(message.version); + writer.raw(hex(message.fingerprint, 8)); + writer.part(message.params); + const { address, principal, incarnation, chain, limit } = message; + writer.byte( + (address === undefined ? 0 : ADDRESS) | + (principal === undefined ? 0 : PRINCIPAL) | + (incarnation === undefined ? 0 : INCARNATION) | + (chain === undefined ? 0 : CHAIN) | + (limit === undefined ? 0 : LIMIT), + ); + if (address !== undefined) { + writer.string(address.kind); + writer.json(address.params); + } + if (principal !== undefined) writer.json(principal); + if (incarnation !== undefined) writer.raw(hex(incarnation.replaceAll("-", ""), 16)); + if (chain !== undefined) { + writer.varint(chain.length); + for (const instance of chain) writer.raw(hex(instance.replaceAll("-", ""), 16)); + } + if (limit !== undefined) writer.varint(limit); + return; + } + case "release": + case "retry": + case "loading": + writer.varint(message.hold); + return; + case "call": + writer.varint(message.call); + writer.varint(message.hold); + writer.varint(message.method); + writer.payload(message.args, message.segments); + return; + case "abort": + writer.varint(message.call); + return; + case "bound": + writer.varint(message.holds.length); + for (const { hold, variant, members, failed } of message.holds) { + writer.varint(hold); + writer.varint(variant); + writer.varint(members.length); + for (const { member, value } of members) { + writer.varint(member); + writer.part(value); + } + writer.failed(failed); + } + return; + case "state": + writer.varint(message.holds.length); + for (const { hold, members, failed, retired } of message.holds) { + writer.varint(hold); + writer.varint(members.length); + for (const { member, ops } of members) { + writer.varint(member); + writer.part(ops); + } + writer.failedAndRetired(failed, retired); + } + return; + case "failed": + writer.varint(message.hold); + writer.byte(message.cause.kind === "transient" ? 0 : 1); + writer.json(message.cause.error); + // 0 when absent, else one more than the delay. + writer.varint(message.retryIn === undefined ? 0 : message.retryIn + 1); + return; + case "result": + writer.varint(message.call); + writer.payload(message.value, message.segments); + return; + case "error": + writer.varint(message.call); + writer.json(message.error); + return; + case "gone": + writer.varint(message.hold); + writer.json(message.error); + return; + case "revoke": + writer.json(message.revocation); + return; + case "ack": + case "closed": + case "ping": + case "pong": + return; + } +} + +/** The kind's fields, read structurally; the envelope's types are checked by the message schema afterwards. */ +function decodeFields(kind: (typeof KINDS)[number], reader: Reader): Message { + // The message schema establishes the types of what is read here (JSON fields are checked by it). + switch (kind) { + case "hello": + return { kind, protocol: reader.varint(), incarnation: uuid(reader.raw(16)), calls: reader.varint() }; + case "hold": { + const hold = reader.varint(); + const type = reader.string(); + const version = reader.varint(); + const fingerprint = hexText(reader.raw(8)); + const params = reader.part(); + const flags = reader.byte(); + if (flags > (ADDRESS | PRINCIPAL | INCARNATION | CHAIN | LIMIT)) throw new Error("binary: unknown hold flags"); + const address = flags & ADDRESS ? { address: { kind: reader.string(), params: reader.json() } } : {}; + const principal = flags & PRINCIPAL ? { principal: reader.json() as { kind: string } } : {}; + const incarnation = flags & INCARNATION ? { incarnation: uuid(reader.raw(16)) } : {}; + const chain = flags & CHAIN ? { chain: reader.list(() => uuid(reader.raw(16))) } : {}; + const limit = flags & LIMIT ? { limit: reader.varint() } : {}; + return { + kind, + hold, + type, + version, + fingerprint, + params, + ...address, + ...principal, + ...incarnation, + ...chain, + ...limit, + }; + } + case "release": + case "retry": + case "loading": + return { kind, hold: reader.varint() }; + case "call": { + const [call, hold, method] = [reader.varint(), reader.varint(), reader.varint()]; + return { kind, call, hold, method, ...reader.payload("args") }; + } + case "abort": + return { kind, call: reader.varint() }; + case "bound": { + const holds = reader.list((): BoundEntry => { + const [hold, variant] = [reader.varint(), reader.varint()]; + const members = reader.list(() => ({ member: reader.varint(), value: reader.part() })); + return { hold, variant, members, failed: reader.failed() }; + }); + return { kind, holds }; + } + case "state": { + const holds = reader.list((): StateEntry => { + const hold = reader.varint(); + const members = reader.list(() => ({ member: reader.varint(), ops: reader.part() })); + // The failure count times two, plus one when retirements follow. + const flags = reader.varint(); + const failed = reader.failedEntries(Math.floor(flags / 2)); + const retired = flags % 2 === 0 ? NO_RETIRED : reader.list(() => reader.varint()); + return { hold, members, failed, retired }; + }); + return { kind, holds }; + } + case "failed": { + const hold = reader.varint(); + const cause = reader.byte(); + if (cause > 1) throw new Error("binary: an unknown failure kind"); + // The schema check after decoding establishes the wire error. + const error = reader.json() as Extract["error"]; + const retryIn = reader.varint(); + return { + kind, + hold, + cause: { kind: cause === 0 ? "transient" : "broken", error }, + ...(retryIn === 0 ? {} : { retryIn: retryIn - 1 }), + }; + } + case "result": + return { kind, call: reader.varint(), ...reader.payload("value") }; + case "error": + return { kind, call: reader.varint(), error: reader.json() as Extract["error"] }; + case "gone": + return { kind, hold: reader.varint(), error: reader.json() as Extract["error"] }; + case "revoke": + return { kind, revocation: reader.json() }; + case "ack": + case "ping": + case "pong": + return { kind }; + case "closed": + // Its origin is the kind byte's; 0 without one, which the message schema refuses. + return { kind, origin: 0 }; + } +} + +/** Up to this many UTF-16 units a string takes at most 126 bytes, so its length is one varint byte. */ +const SHORT = 42; + +/** From this many UTF-16 units a string is encoded in place (below it, encoding on its own and copying is cheaper). */ +const LONG = 4096; + +/** The fixed length prefix of a long string: five varint bytes, up to 32 GiB (16 GiB for a part). */ +const LENGTH_SLOT = 5; + +/** The room left after a large chunk for the fields that follow it. */ +const SPARE = 4096; + +/** Writes a message into one growing buffer, right-sized at the end. */ +class Writer { + /** Large enough for most small messages (a state change, a call), small enough to zero cheaply. */ + #buffer = new Uint8Array(256); + #length = 0; + + byte(byte: number): void { + this.#reserve(1); + this.#buffer[this.#length++] = byte; + } + + /** An unsigned LEB128 varint of a safe integer. */ + varint(value: number): void { + this.#reserve(8); + let rest = value; + while (rest >= 0x80) { + this.#buffer[this.#length++] = (rest % 0x80) | 0x80; + rest = Math.floor(rest / 0x80); + } + this.#buffer[this.#length++] = rest; + } + + raw(bytes: Uint8Array): void { + this.#reserve(bytes.byteLength); + this.#buffer.set(bytes, this.#length); + this.#length += bytes.byteLength; + } + + /** A length-prefixed UTF-8 string (`#text`). */ + string(text: string): void { + this.#text(text, 1); + } + + json(value: unknown): void { + this.string(JSON.stringify(value) ?? "null"); + } + + /** + * A part: its text encoded in place, or its bytes (a part received) copied as they are, behind its length doubled, + * plus one when deflated. + */ + part(part: Part): void { + if (typeof part === "string") { + this.#text(part, 2); + return; + } + const bytes = part instanceof DeflatedPart ? part.bytes : part; + this.varint(bytes.byteLength * 2 + (part instanceof DeflatedPart ? DEFLATED : 0)); + this.raw(bytes); + } + + /** + * Text behind its UTF-8 length times `scale` (2 for a part's prefix), encoded straight into the buffer where the + * prefix is known in advance: a short text (its prefix fits one byte: at most 126 bytes, 42 UTF-16 units, divided + * by `scale`) and a long one (a snapshot's JSON, behind a fixed five-byte prefix, a padded varint, which readers + * accept). Others are encoded on their own and copied in. No text is encoded twice, and the largest value of a + * message is never copied. + */ + #text(text: string, scale: 1 | 2): void { + if (text.length <= SHORT / scale) { + this.#reserve(1 + text.length * 3); + const { written } = UTF8.encodeInto(text, this.#buffer.subarray(this.#length + 1)); + this.#buffer[this.#length] = written * scale; + this.#length += 1 + written; + return; + } + if (text.length < LONG) { + const bytes = UTF8.encode(text); + this.varint(bytes.byteLength * scale); + this.raw(bytes); + return; + } + const slot = this.#length; + // At least one byte per UTF-16 unit; more room is made when the text is not all ASCII. + this.#reserve(LENGTH_SLOT + text.length); + this.#length += LENGTH_SLOT; + let read = 0; + while (read < text.length) { + const rest = read === 0 ? text : text.slice(read); + const done = UTF8.encodeInto(rest, this.#buffer.subarray(this.#length)); + read += done.read; + this.#length += done.written; + if (read < text.length) this.#reserve(Math.max((text.length - read) * 2, 4)); + } + let length = (this.#length - slot - LENGTH_SLOT) * scale; + for (let index = 0; index < LENGTH_SLOT; index++) { + this.#buffer[slot + index] = (length % 0x80) | (index < LENGTH_SLOT - 1 ? 0x80 : 0); + length = Math.floor(length / 0x80); + } + } + + /** A payload: its flags, then its part and, when marked, its segments (count, lengths, raw bytes). */ + payload(part: Part | undefined, segments: readonly Uint8Array[] | undefined): void { + if (part === undefined) { + this.varint(0); + return; + } + this.varint(segments === undefined ? PRESENT : PRESENT | SEGMENTS); + this.part(part); + if (segments === undefined) return; + this.varint(segments.length); + for (const segment of segments) this.varint(segment.byteLength); + for (const segment of segments) this.raw(segment); + } + + failed(failed: readonly FailedMember[]): void { + this.varint(failed.length); + this.#failedEntries(failed); + } + + /** + * A state entry's failures and retirements: the failure count times two, plus one when retirements follow, so an + * entry without retirements costs no byte more than its failures. + */ + failedAndRetired(failed: readonly FailedMember[], retired: readonly number[]): void { + this.varint(failed.length * 2 + (retired.length > 0 ? 1 : 0)); + this.#failedEntries(failed); + if (retired.length === 0) return; + this.varint(retired.length); + for (const member of retired) this.varint(member); + } + + #failedEntries(failed: readonly FailedMember[]): void { + for (const { member, error } of failed) { + this.varint(member); + this.json(error); + } + } + + /** + * The message: a view of the buffer, or a right-sized copy when the spare room is meaningful (at least 64 bytes and + * more than an eighth of the message), so a kept message keeps at most that much beyond itself. A large message + * ends within its 4 KiB of room and is not copied. + */ + finish(): Uint8Array { + const message = this.#buffer.subarray(0, this.#length); + const spare = this.#buffer.byteLength - this.#length; + return spare >= 64 && spare * 8 > this.#length ? message.slice() : message; + } + + /** Grows the buffer to fit `bytes` more: doubled, or for a large chunk to its end plus room for the small fields after it. */ + #reserve(bytes: number): void { + if (this.#length + bytes <= this.#buffer.byteLength) return; + const grown = new Uint8Array(Math.max(this.#buffer.byteLength * 2, this.#length + bytes + SPARE)); + grown.set(this.#buffer.subarray(0, this.#length)); + this.#buffer = grown; + } +} + +/** Reads a message's bytes; every read is bounds-checked, segments are views of the message. */ +class Reader { + readonly #bytes: Uint8Array; + #offset = 0; + + constructor(bytes: Uint8Array) { + this.#bytes = bytes; + } + + byte(): number { + const byte = this.#bytes[this.#offset]; + if (byte === undefined) throw new Error("binary: the message ends early"); + this.#offset++; + return byte; + } + + /** An unsigned LEB128 varint of at most 8 bytes, at most a safe integer. */ + varint(): number { + let value = 0; + let scale = 1; + for (let read = 1; ; read++) { + if (read > 8) throw new Error("binary: a varint does not fit"); + const byte = this.byte(); + value += (byte & 0x7f) * scale; + if (value > Number.MAX_SAFE_INTEGER) throw new Error("binary: a varint does not fit"); + if (byte < 0x80) return value; + scale *= 0x80; + } + } + + raw(length: number): Uint8Array { + if (length > this.#bytes.byteLength - this.#offset) throw new Error("binary: the message ends early"); + const bytes = this.#bytes.subarray(this.#offset, this.#offset + length); + this.#offset += length; + return bytes; + } + + string(): string { + return readUtf8(this.raw(this.varint())); + } + + json(): unknown { + return parseJson(this.raw(this.varint())); + } + + /** A part as a view of the message, unparsed and not inflated. */ + part(): Uint8Array | DeflatedPart { + const prefix = this.varint(); + const bytes = this.raw(Math.floor(prefix / 2)); + return prefix % 2 === DEFLATED ? new DeflatedPart(bytes) : bytes; + } + + /** A list: its count (each element takes at least a byte, so it cannot pass the message's end), then its elements. */ + list(read: () => T): T[] { + const count = this.varint(); + if (count > this.#bytes.byteLength - this.#offset) throw new Error("binary: a count does not fit"); + return Array.from({ length: count }, read); + } + + /** A payload part under `key`, absent when not present, with its segments when marked as carrying them. */ + payload(key: K): { [P in K]?: Part } & { segments?: Uint8Array[] } { + const flags = this.varint(); + if (flags === 0) return {}; + if (flags !== PRESENT && flags !== (PRESENT | SEGMENTS)) throw new Error("binary: unknown payload flags"); + const part = this.part(); + if (flags === PRESENT) return { [key]: part } as { [P in K]?: Part }; + const lengths = this.list(() => this.varint()); + const segments = lengths.map((length) => this.raw(length)); + return { [key]: part, segments } as { [P in K]?: Part } & { segments: Uint8Array[] }; + } + + failed(): FailedMember[] { + return this.failedEntries(this.varint()); + } + + /** `count` failed members, as `Writer.failed` writes them after their count. */ + failedEntries(count: number): FailedMember[] { + if (count > this.#bytes.byteLength - this.#offset) throw new Error("binary: a count does not fit"); + // The message schema checks each error's shape. + return Array.from({ length: count }, () => ({ + member: this.varint(), + error: this.json() as Extract["error"], + })); + } + + /** The message is consumed exactly. */ + end(): void { + if (this.#offset !== this.#bytes.byteLength) throw new Error("binary: bytes after the message"); + } +} + +/** Lowercase hex of 8 and 16 bytes: a fingerprint, an incarnation without its dashes. */ +const HEX = new Map([ + [8, /^[0-9a-f]{16}$/], + [16, /^[0-9a-f]{32}$/], +]); + +/** `length` bytes from lowercase hex (an incarnation without dashes, a fingerprint). */ +function hex(text: string, length: number): Uint8Array { + if (!HEX.get(length)!.test(text)) throw new Error(`binary: not ${length} bytes of hex`); + return Uint8Array.from({ length }, (_, index) => Number.parseInt(text.slice(index * 2, index * 2 + 2), 16)); +} + +function hexText(bytes: Uint8Array): string { + return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +/** 16 bytes as a UUID's text. */ +function uuid(bytes: Uint8Array): string { + const text = hexText(bytes); + return `${text.slice(0, 8)}-${text.slice(8, 12)}-${text.slice(12, 16)}-${text.slice(16, 20)}-${text.slice(20)}`; +} diff --git a/packages/chord/src/next/link/channel.ts b/packages/chord/src/next/link/channel.ts new file mode 100644 index 000000000..14a43c216 --- /dev/null +++ b/packages/chord/src/next/link/channel.ts @@ -0,0 +1,243 @@ +// A host's end of a link (§3, §22, 2b): the handshake, the link's codec, the checks of every message (skipped on a +// trusted link), the heartbeat, and messages handled in order while one waits for an asynchronous inflate; a protocol +// error closes the link. + +import type { Context } from "../../context/index.ts"; +import { PROTOCOL_VERSION } from "../data/internal.ts"; +import type { Link } from "./link.ts"; +import { type Codec, checkMessage, type Message } from "./message.ts"; + +/** What the owner of a channel handles. */ +export interface ChannelHandler { + /** The peer's `hello` arrived: the link is open. */ + opened(): void; + /** + * A checked message after the handshake (never `hello`, `ping` or `pong`). A throw is a protocol error. A promise + * (the message waits for an asynchronous inflate) holds every later message until it settles; a rejection is a + * protocol error too. + */ + message(message: Message): void | Promise; + /** The link closed, from either end; called once. */ + closed(): void; +} + +/** What a channel is made with besides its link and handler. */ +type ChannelOptions = { + readonly codec: Codec; + /** This host's incarnation, sent in `hello`. */ + readonly incarnation: string; + /** How many concurrent calls this side accepts, sent in `hello`. */ + readonly calls: number; + /** Skips the `MESSAGE` check (framing, the codec's decode and the owners' checks still run). */ + readonly trusted: boolean; + /** Where protocol errors go. */ + readonly report: (error: unknown) => void; +}; + +/** + * Where a channel stands: `greeting` until the peer's `hello`, with what this end sent meanwhile; `open` with the + * peer's incarnation and call window. + */ +type ChannelState = + | { readonly kind: "greeting"; readonly queued: Uint8Array[] } + | { readonly kind: "open"; readonly peer: string; readonly calls: number } + | { readonly kind: "closed" }; + +/** + * One end of a link. It sends `hello { protocol, incarnation, calls }` first and nothing else until the peer's arrived + * (its own messages wait). Every message is decoded by the link's codec and, unless the link is trusted, checked + * against the message schema; a first message that is no `hello` of this codec, a second `hello`, another protocol + * version, or a message failing a check closes the link and is reported. The heartbeat: every received message + * stamps `lastReceived`; the host's timer calls `tick`, which pings a silent open link and closes a link silent for + * too long (the handshake included); a `ping` is answered with `pong` at once. A message the handler handles + * asynchronously holds the later ones, `ping` included, until it settles; they are handled in order then, and dropped + * when the link closes meanwhile. + */ +export class Channel { + readonly #link: Link; + readonly #codec: Codec; + readonly #handler: ChannelHandler; + readonly #report: (error: unknown) => void; + readonly #trusted: boolean; + readonly #context: Context; + #state: ChannelState = { kind: "greeting", queued: [] }; + /** When the last message arrived (`Date.now()`), or the channel was made. */ + #lastReceived = Date.now(); + /** Whether a `ping` went out since the last received message. */ + #pinged = false; + /** + * Messages received while one is handled asynchronously, in order, and the index of the next to handle; `undefined` + * while none waits. + */ + #held: { queue: Uint8Array[]; next: number } | undefined; + /** Whether a message is being handled asynchronously: the held ones wait. */ + #waiting = false; + readonly #hello: Uint8Array; + + constructor(link: Link, handler: ChannelHandler, options: ChannelOptions, context: Context) { + this.#link = link; + this.#codec = options.codec; + this.#handler = handler; + this.#report = options.report; + this.#trusted = options.trusted; + this.#context = context; + this.#hello = this.#codec.encode({ + kind: "hello", + protocol: PROTOCOL_VERSION, + incarnation: options.incarnation, + calls: options.calls, + }); + } + + /** + * Sends `hello` and starts listening: messages that arrived already are handled now, and a link closed already + * calls `closed` now; so the handler is complete before it starts. + */ + start(): void { + this.#link.send(this.#hello); + this.#link.onMessage((bytes) => this.#receive(bytes)); + this.#link.onClose(() => this.#closed()); + } + + /** The peer's incarnation, from its `hello`; empty before it arrived (nothing else can arrive before it). */ + get peer(): string { + return this.#state.kind === "open" ? this.#state.peer : ""; + } + + /** How many concurrent calls the peer accepts, from its `hello`; 0 before it arrived. */ + get peerCalls(): number { + return this.#state.kind === "open" ? this.#state.calls : 0; + } + + get opened(): boolean { + return this.#state.kind === "open"; + } + + get closed(): boolean { + return this.#state.kind === "closed"; + } + + /** Sends one message; before the handshake it waits, after the close it is dropped. */ + send(message: Message): void { + const state = this.#state; + if (state.kind === "closed") return; + const bytes = this.#codec.encode(message); + if (state.kind === "greeting") state.queued.push(bytes); + else this.#link.send(bytes); + } + + close(): void { + this.#link.close(); + } + + /** + * The heartbeat, at `now`: a link silent for `close` ms (the handshake included) closes; an open one silent for + * `ping` ms sends one `ping`. + */ + tick(now: number, ping: number, close: number): void { + const silent = now - this.#lastReceived; + if (silent >= close) this.close(); + else if (silent >= ping && !this.#pinged && this.#state.kind === "open") { + this.#pinged = true; + this.#link.send(this.#codec.encode({ kind: "ping" })); + } + } + + #receive(bytes: Uint8Array): void { + if (this.#state.kind === "closed") return; + this.#lastReceived = Date.now(); + this.#pinged = false; + if (this.#held === undefined) this.#handle(bytes); + else this.#held.queue.push(bytes); + } + + /** Handles one received message; one the handler handles asynchronously holds the next until it settles. */ + #handle(bytes: Uint8Array): void { + const state = this.#state; + try { + if (state.kind === "greeting") { + this.#greet(bytes, state.queued); + return; + } + const message = this.#decode(bytes); + if (message.kind === "hello") throw new Error("handshake: a second hello"); + // Control of the link itself, never queued behind the window. + if (message.kind === "ping") this.#link.send(this.#codec.encode({ kind: "pong" })); + else if (message.kind !== "pong") { + const handling = this.#handler.message(message); + if (handling !== undefined) this.#hold(handling); + } + } catch (error) { + this.#failed(error); + } + } + + /** Holds later messages until `handling` settles, then handles them in order (each may hold again). */ + #hold(handling: Promise): void { + this.#held ??= { queue: [], next: 0 }; + this.#waiting = true; + handling.then( + () => { + this.#waiting = false; + this.#release(); + }, + (error: unknown) => this.#failed(error), + ); + } + + /** + * The held message was handled: the ones behind it are handled in order, until one holds again (the handled ones are + * dropped from the queue then). + */ + #release(): void { + const held = this.#held; + if (held === undefined) return; + while (held.next < held.queue.length) { + if (this.#state.kind !== "open") return; + this.#handle(held.queue[held.next++]!); + if (this.#waiting) { + held.queue.splice(0, held.next); + held.next = 0; + return; + } + } + this.#held = undefined; + } + + /** A protocol error: reported, and the link closes. */ + #failed(error: unknown): void { + if (this.#state.kind === "closed") return; + this.#report(error); + this.close(); + } + + /** A message from its bytes, checked against the message schema unless the link is trusted. */ + #decode(bytes: Uint8Array): Message { + const message = this.#codec.decode(bytes); + return this.#trusted ? message : checkMessage(message, this.#context); + } + + /** The first message: the peer's `hello` in this link's codec and protocol version. */ + #greet(bytes: Uint8Array, queued: readonly Uint8Array[]): void { + let message: Message; + try { + message = this.#decode(bytes); + } catch (error) { + throw new Error(`handshake: the first message is no ${this.#codec.name} message`, { cause: error }); + } + if (message.kind !== "hello") throw new Error(`handshake: ${message.kind} before hello`); + if (message.protocol !== PROTOCOL_VERSION) { + throw new Error(`handshake: protocol ${message.protocol}, this host speaks ${PROTOCOL_VERSION}`); + } + this.#state = { kind: "open", peer: message.incarnation, calls: message.calls }; + for (const bytes of queued) this.#link.send(bytes); + this.#handler.opened(); + } + + #closed(): void { + if (this.#state.kind === "closed") return; + this.#state = { kind: "closed" }; + this.#held = undefined; + this.#handler.closed(); + } +} diff --git a/packages/chord/src/next/link/errors.ts b/packages/chord/src/next/link/errors.ts new file mode 100644 index 000000000..cb650549e --- /dev/null +++ b/packages/chord/src/next/link/errors.ts @@ -0,0 +1,23 @@ +// Errors on the wire: a ChordError's JSON or a plugin's code, message and details; never a stack or a cause. + +import type { JsonValue } from "../../json.ts"; +import { type ChordError, isChordError, PluginError } from "../data/index.ts"; +import { chordErrorFromJson } from "../data/internal.ts"; +import type { WireError } from "./message.ts"; + +/** What crosses for a thrown value: a ChordError, a PluginError, or anything else as the plugin code `internal`. */ +export function wireError(error: unknown): WireError { + if (isChordError(error)) return { kind: "chord", error: error.toJSON() }; + if (error instanceof PluginError) { + const { code, message, details } = error; + return { kind: "plugin", code, message, ...(details === undefined ? {} : { details }) }; + } + return { kind: "plugin", code: "internal", message: error instanceof Error ? error.message : String(error) }; +} + +/** The error a consumer rethrows. A malformed ChordError envelope throws (a protocol error). */ +export function errorFromWire(error: WireError): ChordError | PluginError { + if (error.kind === "chord") return chordErrorFromJson(error.error); + // Fresh JSON from the link: details are JSON. + return new PluginError(error.code, error.message, error.details as JsonValue | undefined); +} diff --git a/packages/chord/src/next/link/index.ts b/packages/chord/src/next/link/index.ts new file mode 100644 index 000000000..e2dac1d96 --- /dev/null +++ b/packages/chord/src/next/link/index.ts @@ -0,0 +1,10 @@ +// Links for apps: the link interface adapters implement, the in-memory pair, IPC and WebSocket links, the codecs a link +// may use, its limits per part and the compressor a host may bring. + +export { binaryCodec } from "./binary.ts"; +export { type IpcPort, ipcLink } from "./ipc.ts"; +export { type Link, memoryLinkPair } from "./link.ts"; +export type { Codec, Message } from "./message.ts"; +export type { Compression, Inflated, PartLimits } from "./part.ts"; +export { describeMessage, textCodec } from "./text.ts"; +export { type WebSocketLike, webSocketLink } from "./websocket.ts"; diff --git a/packages/chord/src/next/link/internal.ts b/packages/chord/src/next/link/internal.ts new file mode 100644 index 000000000..e3026f2a8 --- /dev/null +++ b/packages/chord/src/next/link/internal.ts @@ -0,0 +1,35 @@ +// Links for Chord's layers: a host's end of a link (handshake, codec, checks, heartbeat and in-order handling), the +// message type, parts with their limits and compression, and errors in messages. + +export { binaryCodec } from "./binary.ts"; +export { Channel, type ChannelHandler } from "./channel.ts"; +export { errorFromWire, wireError } from "./errors.ts"; +export type { Link } from "./link.ts"; +export { + type BoundEntry, + CHAIN, + type Codec, + type ConsumerMessage, + checkMessage, + type FailedMember, + type Message, + NO_RETIRED, + type OriginMessage, + type ProviderMessage, + type StateEntry, + type WireAddress, +} from "./message.ts"; +export { + type Compression, + DeflatedPart, + inflateAsync, + type Limits, + linkLimits, + openPart, + originPart, + oversized, + type Part, + type PartLimits, +} from "./part.ts"; +export { describeMessage, textCodec } from "./text.ts"; +export { jsonPart, parsePart, parsePayload, payloadPart } from "./values.ts"; diff --git a/packages/chord/src/next/link/ipc.ts b/packages/chord/src/next/link/ipc.ts new file mode 100644 index 000000000..1c5f1137b --- /dev/null +++ b/packages/chord/src/next/link/ipc.ts @@ -0,0 +1,78 @@ +// IPC links (§3): a link over a Node IPC channel, a forked child (`fork` with `serialization: "advanced"`) or +// `process` in the child, typed structurally so Chord imports nothing from Node. + +import { type Link, LinkEnd } from "./link.ts"; + +/** + * The part of a Node IPC channel a link uses: a `ChildProcess` or the child's `process`. `send` returning `false` is + * backpressure (the message is queued by Node), not a failure. + */ +export interface IpcPort { + send(message: Uint8Array, callback: (error: Error | null) => void): boolean; + on(event: "message", listener: (message: unknown) => void): unknown; + on(event: "disconnect", listener: () => void): unknown; + removeListener(event: "message", listener: (message: unknown) => void): unknown; + removeListener(event: "disconnect", listener: () => void): unknown; + disconnect(): void; + readonly connected: boolean; +} + +/** + * A link over `port`: each message one `Uint8Array` (V8 serialization keeps it one). It listens from the moment it is + * made and queues messages until its listener is set, so a child makes it before awaiting its start message; messages + * that are no `Uint8Array` (the app's own, such as that start message) are not the link's and are skipped. A thrown + * `send`, an error in its callback or `disconnect` closes the link once; its listeners are removed then. + */ +export function ipcLink(port: IpcPort): Link { + return new IpcLink(port); +} + +class IpcLink extends LinkEnd { + /** The port while the link is open. */ + #port: IpcPort | undefined; + readonly #onMessage = (message: unknown): void => { + if (message instanceof Uint8Array) this.deliver(message); + }; + readonly #onDisconnect = (): void => this.#end(); + readonly #onSent = (error: Error | null): void => { + if (error !== null) this.#end(); + }; + + constructor(port: IpcPort) { + super(); + this.#port = port; + if (!port.connected) { + this.#end(); + return; + } + port.on("message", this.#onMessage); + port.on("disconnect", this.#onDisconnect); + } + + send(message: Uint8Array): void { + const port = this.#port; + if (port === undefined) return; + try { + port.send(message, this.#onSent); + } catch { + this.#end(); + } + } + + close(): void { + const port = this.#port; + if (port === undefined) return; + this.#end(); + // Disconnecting a channel that is gone already would emit an error. + if (port.connected) port.disconnect(); + } + + /** Detaches from the port and closes this end. */ + #end(): void { + const port = this.#port; + this.#port = undefined; + port?.removeListener("message", this.#onMessage); + port?.removeListener("disconnect", this.#onDisconnect); + this.ended(); + } +} diff --git a/packages/chord/src/next/link/link.ts b/packages/chord/src/next/link/link.ts new file mode 100644 index 000000000..0678cdca1 --- /dev/null +++ b/packages/chord/src/next/link/link.ts @@ -0,0 +1,106 @@ +// Links (§3): an ordered, bidirectional channel of messages between two hosts, and the in-memory pair. + +/** An ordered, bidirectional channel of `Uint8Array` messages between two hosts. Adapters only move bytes. */ +export interface Link { + /** Sends one message; a no-op once closed. */ + send(message: Uint8Array): void; + /** + * Sets the one listener for incoming messages; messages that arrived earlier are delivered to it first. On a closed + * link it is not kept. + */ + onMessage(listener: (message: Uint8Array) => void): void; + /** Closes both ends; messages not delivered yet are dropped. */ + close(): void; + /** Sets the one listener for the close of either end; it runs at once when the link is closed already. */ + onClose(listener: () => void): void; +} + +/** Two ends of one link: what one sends, the other receives asynchronously (a microtask later), in order. */ +export function memoryLinkPair(): [Link, Link] { + const a = new MemoryLink(); + const b = new MemoryLink(); + a.peer = b; + b.peer = a; + return [a, b]; +} + +/** + * What every link end shares: the one message listener (messages that arrived before it are queued for it), the one + * close listener, and closing once. Subclasses move the bytes: `deliver` what arrived, `ended` when the transport + * closed, and implement `send` and `close`. + */ +export abstract class LinkEnd implements Link { + #closed = false; + #listener: ((message: Uint8Array) => void) | undefined; + /** Messages that arrived before the listener was set. */ + #queued: Uint8Array[] = []; + #closeListener: (() => void) | undefined; + + abstract send(message: Uint8Array): void; + + abstract close(): void; + + get closed(): boolean { + return this.#closed; + } + + onMessage(listener: (message: Uint8Array) => void): void { + // A closed end delivers nothing: keeping the listener would keep its owner. + if (this.#closed) return; + this.#listener = listener; + // A listener may close the link: what is still queued then is dropped. + for (const message of this.#queued.splice(0)) { + if (this.#closed) return; + listener(message); + } + } + + onClose(listener: () => void): void { + if (this.#closed) listener(); + else this.#closeListener = listener; + } + + /** A message arrived; dropped once closed. */ + protected deliver(message: Uint8Array): void { + if (this.#closed) return; + if (this.#listener === undefined) this.#queued.push(message); + else this.#listener(message); + } + + /** Closes this end (once): nothing is delivered any more, and nothing the listeners reach is kept. */ + protected ended(): void { + if (this.#closed) return; + this.#closed = true; + const listener = this.#closeListener; + this.#listener = undefined; + this.#closeListener = undefined; + this.#queued = []; + listener?.(); + } +} + +/** + * One end of a memory link. Closing it closes this end at once and the peer a microtask later, as a transport would. + * Portable: `queueMicrotask` only. + */ +class MemoryLink extends LinkEnd { + peer: MemoryLink | undefined; + + send(message: Uint8Array): void { + if (this.closed) return; + const peer = this.peer!; + queueMicrotask(() => peer.deliver(message)); + } + + close(): void { + if (this.closed) return; + const peer = this.peer!; + this.#end(); + queueMicrotask(() => peer.#end()); + } + + #end(): void { + this.peer = undefined; + this.ended(); + } +} diff --git a/packages/chord/src/next/link/message.ts b/packages/chord/src/next/link/message.ts new file mode 100644 index 000000000..a8c022571 --- /dev/null +++ b/packages/chord/src/next/link/message.ts @@ -0,0 +1,182 @@ +// Messages (§22, R1, R4, 2b): the one `Message` type every link carries, defined as a TypeBox schema and checked after +// every decode, and the codecs that turn it into bytes. + +import Type from "typebox"; +import type { Context } from "../../context/index.ts"; +import { arr, int, lit, obj, opt, type Static, str, union, unknown } from "../contracts/index.ts"; +import { validatorFor } from "../data/internal.ts"; +import type { Part } from "./part.ts"; + +/** A link-local id, chosen by its origin: a counter, unique per link, direction and origin while in use. */ +const id = int({ min: 0 }); + +/** + * Whose work a message is (decision "Relay flow control per origin"): a relay's number for one downstream link (and + * the origin that link's messages carried), never reused on a link; absent for the sender's own work. Holds, calls, + * windows and acks are keyed by (origin, id), and replies carry the origin of what they answer. + */ +const origin = int({ min: 1 }); + +/** A name as an index into the hold's bound version (each variant's names sorted). */ +const index = int({ min: 0 }); + +/** An error that may come from plugin code: a ChordError's JSON (checked by L1), or a plugin's code and message. */ +const wireError = union([ + obj({ kind: lit("chord"), error: unknown() }), + obj({ kind: lit("plugin"), code: str(), message: str(), details: opt(unknown()) }), +]); + +/** The members of a hold that failed with this message (W1). */ +const failed = arr(obj({ member: index, error: wireError })); + +/** A part's envelope: the codec built it, so any value passes (its content is the owner's to parse). */ +const part = Type.Unsafe({}); + +/** + * The byte segments of a payload part whose method's schema has a `bytes()` leaf, referenced from its JSON by + * `{"$bytes": i}`; absent for every other payload. Built by the codec, like the part. + */ +const segments = opt(Type.Unsafe({})); + +/** An address as a hold names it: its host kind and params, checked against the kind where it is routed. */ +const address = obj({ kind: str(), params: unknown() }); + +/** A principal on a hold: JSON with a `kind`, honoured only on a parent link. */ +const principal = Type.Object({ kind: str() }); + +/** A uuidv7 as text. */ +const incarnation = str({ pattern: "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" }); + +/** The longest activation chain a hold carries (Q5): a longer one keeps its last entries. */ +export const CHAIN = 32; + +/** The protocol's messages, one tagged union. Parts are opaque here (the owners parse them). */ +const MESSAGE = union([ + // `calls`: how many concurrent calls this side accepts (the call window). + obj({ kind: lit("hello"), protocol: int({ min: 0 }), incarnation, calls: int({ min: 1 }) }), + obj({ + kind: lit("hold"), + origin: opt(origin), + hold: id, + type: str(), + version: int({ min: 1 }), + fingerprint: str({ pattern: "^[0-9a-f]{16}$" }), + params: part, + // The address the caller computed (absent for a type without placement). + address: opt(address), + // Whom the hold acts for and the origin's incarnation; read only on a parent link. + principal: opt(principal), + incarnation: opt(incarnation), + // The instance ids of the activations waiting for this hold (Q5); read only on a trusted link. + chain: opt(Type.Array(incarnation, { maxItems: CHAIN })), + // The decoded limit per part of the link the hold was admitted on (Q2); read only on a parent link. + limit: opt(int({ min: 0 })), + }), + obj({ kind: lit("release"), origin: opt(origin), hold: id }), + obj({ kind: lit("call"), origin: opt(origin), call: id, hold: id, method: index, args: opt(part), segments }), + obj({ kind: lit("abort"), origin: opt(origin), call: id }), + // Asks the provider for an attempt of a failed hold's object now (`ref.retry()`). + obj({ kind: lit("retry"), origin: opt(origin), hold: id }), + // Binds several holds at once (first bindings, re-holds), applied in one batch. + obj({ + kind: lit("bound"), + origin: opt(origin), + holds: arr(obj({ hold: id, variant: index, members: arr(obj({ member: index, value: part })), failed })), + }), + obj({ + kind: lit("state"), + origin: opt(origin), + // `retired`: the members whose document was retired since the hold's last message (decision "One retirement + // mechanism"); a `bound` snapshot carries no flag. + holds: arr(obj({ hold: id, members: arr(obj({ member: index, ops: part })), failed, retired: arr(index) })), + }), + // One per applied `bound` or `state` message of its origin (the window). + obj({ kind: lit("ack"), origin: opt(origin) }), + // A relay's downstream link closed: the origin's holds are released and its calls aborted. + obj({ kind: lit("closed"), origin }), + // A revocation from the parent, applied with this host's own `revokedBy` (no origin). + obj({ kind: lit("revoke"), revocation: unknown() }), + obj({ kind: lit("ping") }), + obj({ kind: lit("pong") }), + obj({ + kind: lit("failed"), + origin: opt(origin), + hold: id, + // A failure cause's error in the wire error format (a ChordError, checked by L1's envelope check, or a plugin's). + cause: obj({ kind: union([lit("transient"), lit("broken")]), error: wireError }), + // The rest of the object's backoff window in milliseconds: when it retries by itself (absent: broken). + retryIn: opt(int({ min: 0 })), + }), + // The failed object of a hold started a new attempt. + obj({ kind: lit("loading"), origin: opt(origin), hold: id }), + obj({ kind: lit("result"), origin: opt(origin), call: id, value: opt(part), segments }), + obj({ kind: lit("error"), origin: opt(origin), call: id, error: wireError }), + obj({ kind: lit("gone"), origin: opt(origin), hold: id, error: wireError }), +]); + +/** + * A protocol message. `args` absent: a method without arguments; `value` absent: `null`. A `state` entry's `retired` + * list is read-only (the schema's type is mutable): entries without retirements share `NO_RETIRED`. + */ +export type Message = + | Exclude + | (Omit & { readonly holds: readonly StateEntry[] }); + +type WireMessage = Static; + +type WireState = Extract; + +/** An error as it crosses a link. */ +export type WireError = Static; + +/** A message that may carry an origin (every kind but `hello`, `revoke`, `ping` and `pong`). */ +export type OriginMessage = Exclude; + +/** A message for a provider side: the direction from the consumer (`closed` is the link's). */ +export type ProviderMessage = Extract< + Message, + { readonly kind: "hold" | "release" | "call" | "abort" | "retry" | "ack" } +>; + +/** A message for a consumer side: the direction from the provider. */ +export type ConsumerMessage = Extract< + Message, + { readonly kind: "bound" | "state" | "failed" | "loading" | "result" | "error" | "gone" } +>; + +/** One hold's entry of a `bound` message. */ +export type BoundEntry = Extract["holds"][number]; + +/** One hold's entry of a `state` message. */ +export type StateEntry = Omit & { readonly retired: readonly number[] }; + +/** The `retired` list of a state entry without retirements, shared (no allocation per message). */ +export const NO_RETIRED: readonly number[] = []; + +/** An address as a `hold` carries it. */ +export type WireAddress = Static; + +/** A member of a hold that failed, by index, as `bound` and `state` list it. */ +export type FailedMember = Static[number]; + +/** How a link turns messages into bytes and back; both ends of a link use the same one. */ +export interface Codec { + readonly name: string; + encode(message: Message): Uint8Array; + /** Throws on bytes that are no message of this codec (a protocol error); the envelope is checked afterwards. */ + decode(bytes: Uint8Array): Message; +} + +const MESSAGE_SCHEMA = { schema: MESSAGE }; + +/** + * Checks a decoded message's envelope with the compiled schema (the wire is a trust boundary, after any codec): kinds, + * ids and indexes, lists and error shapes; not the parts. Throws on a mismatch (a protocol error). Counted in + * `chord.link.checks`. + */ +export function checkMessage(message: unknown, context: Context): Message { + context.telemetry.metrics?.counter("chord.link.checks").add(1); + if (!validatorFor(MESSAGE_SCHEMA, context).Check(message)) throw new Error("message: fails the message schema"); + // The check above establishes the type. + return message as Message; +} diff --git a/packages/chord/src/next/link/part.ts b/packages/chord/src/next/link/part.ts new file mode 100644 index 000000000..d0daeb45a --- /dev/null +++ b/packages/chord/src/next/link/part.ts @@ -0,0 +1,208 @@ +// Parts on links (R1, R2, Q2, §22): a value as it crosses a link (JSON text at its origin, its bytes once received, or +// deflated by its origin), the size limits per part, compression at the origin, and opening a part where it is owned: +// its size checked before anything is parsed or inflated, a deflated part inflated within the decoded limit. + +import type { Context } from "../../context/index.ts"; +import { ChordError } from "../data/index.ts"; + +/** + * A part (R1): a value as JSON text at its origin (and after a text decode), its UTF-8 bytes once a binary codec + * received it (or at an origin that encoded it already), or those bytes deflated by an origin that compresses. Opaque + * on the way: only the end that owns it opens and parses it, while handling its message (a malformed part is a protocol + * error then), so a relay forwards it unparsed and still deflated. + */ +export type Part = string | Uint8Array | DeflatedPart; + +/** What inflating a deflated part gave: its JSON bytes, more than the decoded limit, or data that is no raw deflate. */ +export type Inflated = + | { readonly kind: "bytes"; readonly bytes: Uint8Array } + | { readonly kind: "too_large" } + | { readonly kind: "corrupt"; readonly error: unknown }; + +/** + * A JSON part deflated (raw deflate) by its origin. A host without a synchronous compressor inflates it before its + * owner handles the message (`inflateAsync`) and keeps the outcome here for the owner to open. + */ +export class DeflatedPart { + readonly bytes: Uint8Array; + /** The asynchronous inflate's outcome; `undefined` until one ran. */ + inflated: Inflated | undefined; + + constructor(bytes: Uint8Array) { + this.bytes = bytes; + } +} + +/** + * A synchronous compressor (`createHost({ compression })`; `nodeCompression` for Node). `deflate` is raw deflate at + * level 1. `inflate` gives at most `limit` bytes (`Infinity`: no limit; 0 allowed), `too_large` when the output would + * be larger, `corrupt` for data that is no raw deflate stream. + */ +export interface Compression { + deflate(bytes: Uint8Array): Uint8Array; + inflate(bytes: Uint8Array, limit: number): Inflated; +} + +/** A link's limits per part as `accept`, `connect` and `resolve` take them, in bytes. */ +export type PartLimits = { + /** A part's size on the wire. */ + readonly part?: number; + /** A part's size after inflating it. */ + readonly decoded?: number; +}; + +/** A link's limits in force; `Infinity` for none. */ +export type Limits = { readonly part: number; readonly decoded: number }; + +const MiB = 1024 * 1024; + +/** The limits of untrusted links unless set (Q2); trusted links (IPC) have none. */ +const UNTRUSTED: Limits = { part: 16 * MiB, decoded: 64 * MiB }; + +const NONE: Limits = { part: Number.POSITIVE_INFINITY, decoded: Number.POSITIVE_INFINITY }; + +/** The limits of a link: the ones set, the defaults of its trust for the others. */ +export function linkLimits(trusted: boolean, limits: PartLimits = {}): Limits { + const defaults = trusted ? NONE : UNTRUSTED; + return { part: limits.part ?? defaults.part, decoded: limits.decoded ?? defaults.decoded }; +} + +/** JSON parts above this many UTF-8 bytes are compressed at their origin (a protocol constant). */ +export const COMPRESS_ABOVE = 16 * 1024; + +const UTF8 = new TextEncoder(); + +/** + * A JSON part at its origin: deflated when the host compresses, its UTF-8 bytes are above `COMPRESS_ABOVE` and + * deflating made it smaller; otherwise the text, or its bytes once they were encoded (never encoded twice). Counts one + * deflate (`chord.link.deflates`). + */ +export function originPart(json: string, compression: Compression | undefined, context: Context): Part { + // A UTF-16 unit is at most three UTF-8 bytes: a text this short is within the threshold without encoding it. + if (compression === undefined || json.length * 3 <= COMPRESS_ABOVE) return json; + const bytes = UTF8.encode(json); + if (bytes.byteLength <= COMPRESS_ABOVE) return bytes; + context.telemetry.metrics?.counter("chord.link.deflates").add(1); + const deflated = compression.deflate(bytes); + return deflated.byteLength < bytes.byteLength ? new DeflatedPart(deflated) : bytes; +} + +/** + * The `too_large/part` error of a part whose size on the wire (its UTF-8 bytes; a deflated part's deflated bytes) is + * above `limit`; `undefined` within it. Nothing is parsed or inflated. + */ +export function oversized(part: Part, limit: number): ChordError | undefined { + if (limit === Number.POSITIVE_INFINITY) return undefined; + const size = sizeOf(part); + return size > limit ? tooLarge(size, limit, "wire") : undefined; +} + +/** + * The JSON of a part its owner is about to parse: its size checked against `limits.part` first, then its decoded size + * against `limits.decoded`, a deflated part inflated within it (synchronously with `compression`, or by the + * asynchronous inflate that ran ahead of its owner). Throws `too_large/part` past a limit; throws a plain error for a + * deflated part nothing inflated or that is no raw deflate stream (a protocol error). Counts one synchronous inflate + * (`chord.link.inflates`). + */ +export function openPart( + part: Part, + limits: Limits, + compression: Compression | undefined, + context: Context, +): string | Uint8Array { + const wire = oversized(part, limits.part); + if (wire !== undefined) throw wire; + if (!(part instanceof DeflatedPart)) { + // Not deflated: its decoded size is its size. + const size = limits.decoded === Number.POSITIVE_INFINITY ? 0 : sizeOf(part); + if (size > limits.decoded) throw tooLarge(size, limits.decoded, "decoded"); + return part; + } + let inflated = part.inflated; + if (inflated === undefined) { + if (compression === undefined) throw new Error("part: a deflated part this host did not inflate"); + context.telemetry.metrics?.counter("chord.link.inflates").add(1); + inflated = compression.inflate(part.bytes, limits.decoded); + } + switch (inflated.kind) { + case "bytes": + return inflated.bytes; + case "too_large": + throw tooLarge(undefined, limits.decoded, "decoded"); + case "corrupt": + throw new Error("part: no raw deflate stream", { cause: inflated.error }); + } +} + +/** + * Inflates `part` with the web `DecompressionStream` (a host without a synchronous compressor), counting output bytes + * and stopping past `limit`; the outcome is kept on the part. `signal` (the link's close) cancels the stream at once, + * leaving no outcome. Counts one inflate (`chord.link.inflates`). + */ +export async function inflateAsync( + part: DeflatedPart, + limit: number, + signal: AbortSignal, + context: Context, +): Promise { + context.telemetry.metrics?.counter("chord.link.inflates").add(1); + const reader = new Blob([part.bytes as Uint8Array]) + .stream() + .pipeThrough(new DecompressionStream("deflate-raw")) + .getReader(); + // A pending read settles once cancelled: nothing of the stream outlives the link. + const cancel = (): void => void reader.cancel().catch(() => {}); + signal.addEventListener("abort", cancel, { once: true }); + const chunks: Uint8Array[] = []; + let length = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done || signal.aborted) break; + length += value.byteLength; + if (length > limit) { + cancel(); + part.inflated = { kind: "too_large" }; + return; + } + chunks.push(value); + } + } catch (error) { + if (!signal.aborted) part.inflated = { kind: "corrupt", error }; + return; + } finally { + signal.removeEventListener("abort", cancel); + } + if (signal.aborted) return; + const bytes = new Uint8Array(length); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + part.inflated = { kind: "bytes", bytes }; +} + +/** A part's size on the wire: its UTF-8 bytes (counted for text, without encoding it). */ +function sizeOf(part: Part): number { + if (typeof part !== "string") return part instanceof DeflatedPart ? part.bytes.byteLength : part.byteLength; + let size = part.length; + for (let index = 0; index < part.length; index++) { + const unit = part.charCodeAt(index); + if (unit < 0x80) continue; + if (unit < 0x800) size += 1; + else if (unit >= 0xd800 && unit < 0xdc00 && (part.charCodeAt(index + 1) & 0xfc00) === 0xdc00) { + // A surrogate pair: four bytes for two units. + size += 2; + index++; + } else size += 2; + } + return size; +} + +/** The error of a part past a limit (`size` unknown while inflating). */ +function tooLarge(size: number | undefined, limit: number, measure: "wire" | "decoded"): ChordError { + const what = size === undefined ? "a part" : `a part of ${size} bytes`; + const message = `${what} is over the ${measure} limit of ${limit} bytes`; + return new ChordError({ code: "too_large", reason: "part", message }); +} diff --git a/packages/chord/src/next/link/text.ts b/packages/chord/src/next/link/text.ts new file mode 100644 index 000000000..c9b7b8e0d --- /dev/null +++ b/packages/chord/src/next/link/text.ts @@ -0,0 +1,356 @@ +// The text codec (2b): one JSON tuple per message, Delta style (a short kind tag, positional fields, no field names), +// and `describeMessage`, which prints any message in that form. A part's JSON text is spliced into the tuple as it is; +// decoding parses the tuple and writes each part back as text (one parse per part per hop: a debugging codec). Byte +// segments of a payload marked as carrying bytes follow the tuple after a newline (JSON text never contains a raw +// one), their lengths the tuple's last field. A message with an origin is wrapped: `["@", origin, tuple]`. Every part +// is tagged with its codec, so no JSON value is reserved: `[0, json]`, or `[1, base64]` for a part its origin deflated +// (the text codec itself never compresses). + +import { + type BoundEntry, + type Codec, + type FailedMember, + type Message, + NO_RETIRED, + type StateEntry, +} from "./message.ts"; +import type { Part } from "./part.ts"; +import { parseJson, partText, textPart } from "./values.ts"; + +const UTF8 = new TextEncoder(); + +/** Messages as JSON tuples: readable on the wire, for debugging links. */ +export const textCodec: Codec = { + name: "text", + encode(message) { + const json = UTF8.encode(tuple(message)); + const segments = segmentsOf(message); + if (segments === undefined) return json; + let length = json.byteLength + 1; + for (const segment of segments) length += segment.byteLength; + const bytes = new Uint8Array(length); + bytes.set(json, 0); + bytes[json.byteLength] = NEWLINE; + let offset = json.byteLength + 1; + for (const segment of segments) { + bytes.set(segment, offset); + offset += segment.byteLength; + } + return bytes; + }, + decode(bytes) { + const newline = bytes.indexOf(NEWLINE); + const json = newline === -1 ? bytes : bytes.subarray(0, newline); + const rest = newline === -1 ? bytes.subarray(bytes.byteLength) : bytes.subarray(newline + 1); + const fields = parseJson(json); + if (!Array.isArray(fields)) throw new Error("text: a message is a tuple"); + return fromTuple(fields, rest); + }, +}; + +/** A message in the text codec's tuple form, for logs and tests (byte segments show as `{"$bytes": i}`). */ +export function describeMessage(message: Message): string { + return tuple(message); +} + +const NEWLINE = 0x0a; + +/** The segments following a payload's tuple; `undefined` without a payload marked as carrying bytes. */ +function segmentsOf(message: Message): readonly Uint8Array[] | undefined { + return message.kind === "call" || message.kind === "result" ? message.segments : undefined; +} + +/** The tuple's JSON text, parts spliced in; wrapped with its origin when it has one. */ +function tuple(message: Message): string { + const origin = "origin" in message ? message.origin : undefined; + return origin === undefined ? ownTuple(message) : `["@",${origin},${ownTuple(message)}]`; +} + +/** The tuple of a message's own fields. */ +function ownTuple(message: Message): string { + switch (message.kind) { + case "hello": + return json(["h", message.protocol, message.incarnation, message.calls]); + case "hold": { + const { address, principal, incarnation, chain, limit } = message; + const head = json(["H", message.hold, message.type, message.version, message.fingerprint]).slice(0, -1); + // Optional trailing fields: an absent one in the middle is null, absent ones at the end are left out. + const optional = [ + address === undefined ? null : [address.kind, address.params], + principal ?? null, + incarnation ?? null, + chain ?? null, + limit ?? null, + ]; + while (optional.length > 0 && optional.at(-1) === null) optional.pop(); + const rest = optional.map((field) => `,${json(field)}`).join(""); + return `${head},${partText(message.params)}${rest}]`; + } + case "release": + return json(["R", message.hold]); + case "call": + return withPayload(json(["c", message.call, message.hold, message.method]), message.args, message.segments); + case "abort": + return json(["x", message.call]); + case "bound": + return `["b",[${message.holds.map(boundEntry).join(",")}]]`; + case "state": + return `["s",[${message.holds.map(stateEntry).join(",")}]]`; + case "failed": { + const kind = message.cause.kind === "transient" ? "t" : "b"; + const retryIn = message.retryIn === undefined ? [] : [message.retryIn]; + return json(["f", message.hold, kind, message.cause.error, ...retryIn]); + } + case "result": + return withPayload(json(["r", message.call]), message.value, message.segments); + case "error": + return json(["e", message.call, message.error]); + case "gone": + return json(["g", message.hold, message.error]); + case "revoke": + return json(["V", message.revocation]); + case "retry": + return json(["T", message.hold]); + case "loading": + return json(["l", message.hold]); + case "ack": + return json(["a"]); + case "closed": + return json(["C"]); + case "ping": + return json(["p"]); + case "pong": + return json(["P"]); + } +} + +function json(value: unknown): string { + return JSON.stringify(value); +} + +/** `[hold, variant, [[member, value]…], failed?]`. */ +function boundEntry({ hold, variant, members, failed }: BoundEntry): string { + const pairs = members.map(({ member, value }) => `[${member},${partText(value)}]`).join(","); + return `[${hold},${variant},[${pairs}]${failedField(failed)}]`; +} + +/** `[hold, [[member, ops]…], failed?, retired?]`; `failed` is `[]` when only `retired` follows. */ +function stateEntry({ hold, members, failed, retired }: StateEntry): string { + const pairs = members.map(({ member, ops }) => `[${member},${partText(ops)}]`).join(","); + if (retired.length === 0) return `[${hold},[${pairs}]${failedField(failed)}]`; + return `[${hold},[${pairs}],${json(failed.map(({ member, error }) => [member, error]))},${json(retired)}]`; +} + +/** The failures as an entry's optional last field: left out when there are none. */ +function failedField(failed: readonly FailedMember[]): string { + return failed.length === 0 ? "" : `,${json(failed.map(({ member, error }) => [member, error]))}`; +} + +/** + * A tuple (its closed JSON `head`) ending in a payload part; one marked as carrying bytes adds the segments' lengths + * as the last field. + */ +function withPayload(head: string, part: Part | undefined, segments: readonly Uint8Array[] | undefined): string { + if (part === undefined) return head; + const prefix = `${head.slice(0, -1)},${partText(part)}`; + if (segments === undefined) return `${prefix}]`; + return `${prefix},${json(segments.map((segment) => segment.byteLength))}]`; +} + +/** + * A message from its tuple: the tag and arity checked, lists of pairs checked as lists, parts written back as text; + * the field types are the message schema's to check. `rest` holds the segments, which only a payload marked as + * carrying bytes may have. + */ +function fromTuple(fields: unknown[], rest: Uint8Array): Message { + if (fields[0] !== "@") return fromFields(fields, rest); + const inner = fields.length === 3 ? list(fields[2]) : []; + if (inner[0] === "@" || inner.length === 0) throw new Error("text: an origin wraps one message"); + const message = fromFields(inner, rest); + // A fresh object; the message schema checks the origin and refuses one on a kind without it. + (message as { origin?: unknown }).origin = fields[1]; + return message; +} + +/** A message from its own fields' tuple (see `fromTuple`). */ +function fromFields(fields: unknown[], rest: Uint8Array): Message { + const [tag, ...f] = fields; + const arity = (...allowed: number[]): void => { + if (!allowed.includes(f.length)) throw new Error(`text: a "${String(tag)}" message with ${f.length} fields`); + }; + if (tag !== "c" && tag !== "r" && rest.byteLength > 0) { + throw new Error("text: byte segments without a payload that carries them"); + } + // The message schema establishes the types of the fields taken here. + const field = (index: number): T => f[index] as T; + switch (tag) { + case "h": + arity(3); + return { kind: "hello", protocol: field(0), incarnation: field(1), calls: field(2) }; + case "H": { + arity(5, 6, 7, 8, 9, 10); + const [address, principal, incarnation, chain, limit] = [5, 6, 7, 8, 9].map((index) => f[index] ?? null); + let wireAddress = {}; + if (address !== null) { + const pair = list(address); + if (pair.length !== 2) throw new Error("text: an address is a pair"); + wireAddress = { address: { kind: pair[0], params: pair[1] } }; + } + return { + kind: "hold", + hold: field(0), + type: field(1), + version: field(2), + fingerprint: field(3), + params: textPart(f[4]), + ...wireAddress, + ...(principal === null ? {} : { principal: principal as { kind: string } }), + ...(incarnation === null ? {} : { incarnation: incarnation as string }), + ...(chain === null ? {} : { chain: chain as string[] }), + ...(limit === null ? {} : { limit: limit as number }), + }; + } + case "R": + arity(1); + return { kind: "release", hold: field(0) }; + case "c": + arity(3, 4, 5); + return { + kind: "call", + call: field(0), + hold: field(1), + method: field(2), + ...payload("args", f.slice(3), rest), + }; + case "x": + arity(1); + return { kind: "abort", call: field(0) }; + case "b": + arity(1); + return { + kind: "bound", + holds: list(f[0]).map((entry): BoundEntry => { + const fields = list(entry); + if (fields.length !== 3 && fields.length !== 4) { + throw new Error(`text: a bound entry with ${fields.length} fields`); + } + const [hold, variant, members, failed] = fields; + return { + hold: hold as number, + variant: variant as number, + members: pairs(members).map(([member, value]) => ({ + member: member as number, + value: textPart(value), + })), + failed: failures(failed), + }; + }), + }; + case "s": + arity(1); + return { + kind: "state", + holds: list(f[0]).map((entry): StateEntry => { + const fields = list(entry); + if (fields.length < 2 || fields.length > 4) { + throw new Error(`text: a state entry with ${fields.length} fields`); + } + const [hold, members, failed, retired] = fields; + return { + hold: hold as number, + members: pairs(members).map(([member, ops]) => ({ member: member as number, ops: textPart(ops) })), + failed: failures(failed), + // The message schema checks each index. + retired: retired === undefined ? NO_RETIRED : (list(retired) as number[]), + }; + }), + }; + case "f": { + arity(3, 4); + if (f[1] !== "t" && f[1] !== "b") throw new Error("text: an unknown failure kind"); + // The schema check after decoding establishes the wire error. + const error = f[2] as Extract["error"]; + const cause = { kind: f[1] === "t" ? "transient" : "broken", error } as const; + return { kind: "failed", hold: field(0), cause, ...(f.length === 4 ? { retryIn: field(3) } : {}) }; + } + case "r": + arity(1, 2, 3); + return { kind: "result", call: field(0), ...payload("value", f.slice(1), rest) }; + case "e": + arity(2); + return { kind: "error", call: field(0), error: field(1) }; + case "g": + arity(2); + return { kind: "gone", hold: field(0), error: field(1) }; + case "V": + arity(1); + return { kind: "revoke", revocation: f[0] }; + case "T": + arity(1); + return { kind: "retry", hold: field(0) }; + case "l": + arity(1); + return { kind: "loading", hold: field(0) }; + case "a": + arity(0); + return { kind: "ack" }; + case "C": + arity(0); + // Its origin is the wrapper's; 0 without one, which the message schema refuses. + return { kind: "closed", origin: 0 }; + case "p": + arity(0); + return { kind: "ping" }; + case "P": + arity(0); + return { kind: "pong" }; + default: + throw new Error("text: an unknown kind"); + } +} + +/** + * A payload part from its fields: none, the value, or the value and its segments' lengths (marked as carrying bytes, + * the segments views of `rest`). + */ +function payload( + key: K, + fields: unknown[], + rest: Uint8Array, +): { [P in K]?: Part } & { segments?: Uint8Array[] } { + if (fields.length < 2) { + if (rest.byteLength > 0) throw new Error("text: byte segments without a payload that carries them"); + return (fields.length === 0 ? {} : { [key]: textPart(fields[0]) }) as { [P in K]?: Part }; + } + const lengths = list(fields[1]); + let offset = 0; + const segments = lengths.map((length) => { + if (!Number.isSafeInteger(length) || (length as number) < 0) throw new Error("text: a segment length"); + const segment = rest.subarray(offset, offset + (length as number)); + offset += length as number; + return segment; + }); + if (offset !== rest.byteLength) throw new Error("text: the segments do not fill the message"); + return { [key]: textPart(fields[0]), segments } as { [P in K]?: Part } & { segments: Uint8Array[] }; +} + +function list(value: unknown): unknown[] { + if (!Array.isArray(value)) throw new Error("text: expected a list"); + return value; +} + +/** A list of `[index, value]` pairs. */ +function pairs(value: unknown): [unknown, unknown][] { + return list(value).map((pair) => { + const entry = list(pair); + if (entry.length !== 2) throw new Error("text: expected a pair"); + return entry as [unknown, unknown]; + }); +} + +/** The optional failures field; absent means none. */ +function failures(value: unknown): FailedMember[] { + if (value === undefined) return []; + // The message schema checks each error's shape. + return pairs(value).map(([member, error]) => ({ member, error }) as FailedMember); +} diff --git a/packages/chord/src/next/link/values.ts b/packages/chord/src/next/link/values.ts new file mode 100644 index 000000000..858866766 --- /dev/null +++ b/packages/chord/src/next/link/values.ts @@ -0,0 +1,126 @@ +// Values in messages (§3 values, §22): JSON text, and for payloads marked as carrying bytes raw byte segments, a +// `Uint8Array` leaf written as `{"$bytes": i}` and read back as a view of the received message (no copy, never +// base64); a part as the text codec splices it. Shared by the codecs. + +import { invalid } from "../data/internal.ts"; +import { DeflatedPart, type Part } from "./part.ts"; + +/** Strict, and a leading U+FEFF is data, not a byte order mark to strip. */ +const STRICT_UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + +/** A JSON part at its origin: the value's JSON text (state and params are JSON only). */ +export function jsonPart(value: unknown): string { + return JSON.stringify(value) ?? "null"; +} + +/** + * A payload part at its origin (arguments, a result): its JSON and, when `bytes` (its method's schema has a `bytes()` + * leaf), its byte segments: each `Uint8Array` leaf becomes a segment, and an own key `$bytes`, reserved there, fails + * `invalid/args`; otherwise plain JSON, where `$bytes` is ordinary data. `undefined` (a handler returning nothing) is + * `null`. + */ +export function payloadPart( + value: unknown, + bytes: boolean, +): { readonly json: string; readonly segments: Uint8Array[] | undefined } { + if (!bytes) return { json: jsonPart(value), segments: undefined }; + const segments: Uint8Array[] = []; + const json = JSON.stringify(value, (_key, leaf: unknown) => { + if (leaf instanceof Uint8Array) { + segments.push(leaf); + return { $bytes: segments.length - 1 }; + } + // The replacer does not see the markers it returns, only their fields. + if (typeof leaf === "object" && leaf !== null && Object.hasOwn(leaf, "$bytes")) { + throw invalid("args", '"$bytes" is reserved in values crossing a link'); + } + return leaf; + }); + return { json: json ?? "null", segments }; +} + +/** + * Parses a part's JSON where it is owned, once opened (`openPart`). Throws on a part that is no strict UTF-8 JSON (a + * protocol error). + */ +export function parsePart(json: string | Uint8Array): unknown { + return typeof json === "string" ? JSON.parse(json) : parseJson(json); +} + +/** + * Parses a payload part's JSON where it is owned, once opened, its `{"$bytes": i}` references resolved to views of + * `segments` when it has them. Throws on a malformed part (a protocol error). + */ +export function parsePayload(json: string | Uint8Array, segments: readonly Uint8Array[] | undefined): unknown { + const value = parsePart(json); + return segments === undefined ? value : resolveSegments(value, segments); +} + +/** + * A part as the text codec splices it into its tuple, tagged so that no JSON value is reserved: `[0, json]` for a plain + * part, `[1, base64]` for a deflated one. Throws on invalid UTF-8. + */ +export function partText(part: Part): string { + if (part instanceof DeflatedPart) return `[1,"${base64(part.bytes)}"]`; + return `[0,${typeof part === "string" ? part : readUtf8(part)}]`; +} + +/** A part from its place in a text tuple (`partText` reversed). Throws on anything else (a protocol error). */ +export function textPart(value: unknown): Part { + // Trust boundary: fresh JSON from the text decoder. + if (!Array.isArray(value) || value.length !== 2) throw new Error("text: a part is no [codec, payload] pair"); + const [codec, payload] = value as [unknown, unknown]; + if (codec === 0) return JSON.stringify(payload); + if (codec === 1 && typeof payload === "string") return new DeflatedPart(fromBase64(payload)); + throw new Error("text: a part of an unknown codec"); +} + +/** Strict UTF-8 JSON. Throws on anything else (a protocol error). */ +export function parseJson(bytes: Uint8Array): unknown { + return JSON.parse(STRICT_UTF8.decode(bytes)); +} + +/** Strict UTF-8 text. Throws on invalid UTF-8 (a protocol error). */ +export function readUtf8(bytes: Uint8Array): string { + return STRICT_UTF8.decode(bytes); +} + +/** + * Replaces every `{"$bytes": i}` in a freshly decoded payload with segment `i`, in place. Throws on a malformed + * reference (a protocol error). + */ +export function resolveSegments(value: unknown, segments: readonly Uint8Array[]): unknown { + if (typeof value !== "object" || value === null) return value; + // Trust boundary: fresh JSON from the link, owned by this decode. + const node = value as Record; + if (!Array.isArray(node) && Object.hasOwn(node, "$bytes")) { + const index = node.$bytes; + if (Object.keys(node).length !== 1 || typeof index !== "number" || segments[index] === undefined) { + throw new Error("payload: a malformed byte segment reference"); + } + return segments[index]; + } + for (const key of Object.keys(node)) { + const child = node[key]; + const resolved = resolveSegments(child, segments); + // Defined, not assigned: a key `__proto__` is an own property of parsed JSON. + if (resolved !== child) + Object.defineProperty(node, key, { value: resolved, writable: true, enumerable: true, configurable: true }); + } + return node; +} + +/** Bytes as base64 (the text codec's deflated parts; portable: `btoa` over a binary string, in chunks). */ +function base64(bytes: Uint8Array): string { + let binary = ""; + for (let offset = 0; offset < bytes.byteLength; offset += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000)); + } + return btoa(binary); +} + +/** Bytes from base64. Throws on text that is no base64 (a protocol error). */ +function fromBase64(text: string): Uint8Array { + const binary = atob(text); + return Uint8Array.from(binary, (char) => char.charCodeAt(0)); +} diff --git a/packages/chord/src/next/link/websocket.ts b/packages/chord/src/next/link/websocket.ts new file mode 100644 index 000000000..52fe1b2f5 --- /dev/null +++ b/packages/chord/src/next/link/websocket.ts @@ -0,0 +1,106 @@ +// WebSocket links (§3): a link over a WebSocket, the browser's or `ws` in Node, typed structurally; binary frames, +// messages sent while it still connects buffered until it opens. + +import { type Link, LinkEnd } from "./link.ts"; + +/** The part of a WebSocket a link uses (the browser's `WebSocket`, `ws`'s `WebSocket`). */ +export interface WebSocketLike { + /** 0 connecting, 1 open, 2 closing, 3 closed. */ + readonly readyState: number; + binaryType: string; + /** Typed as the browser's `WebSocket` takes bytes: on an `ArrayBuffer`, never shared memory. */ + send(data: Uint8Array): void; + close(): void; + addEventListener(type: "open" | "close" | "error", listener: () => void): void; + addEventListener(type: "message", listener: (event: { readonly data: unknown }) => void): void; + removeEventListener(type: "open" | "close" | "error", listener: () => void): void; + removeEventListener(type: "message", listener: (event: { readonly data: unknown }) => void): void; +} + +const CONNECTING = 0; + +const OPEN = 1; + +/** + * A link over `socket`, one binary frame per message (`binaryType = "arraybuffer"`). Messages sent while it connects + * wait for `open`; a socket closing or closed already gives a closed link; `close` and `error` close the link once, and + * so does a text frame (no message of this protocol). A frame above the transport's `maxPayload` closes the socket + * (code 1009), which closes the link. + */ +export function webSocketLink(socket: WebSocketLike): Link { + return new WebSocketLink(socket); +} + +class WebSocketLink extends LinkEnd { + /** The socket while the link is open. */ + #socket: WebSocketLike | undefined; + /** Messages sent while the socket connects; `undefined` once it is open. */ + #connecting: Uint8Array[] | undefined; + readonly #onOpen = (): void => { + const queued = this.#connecting ?? []; + this.#connecting = undefined; + for (const message of queued) this.send(message); + }; + readonly #onMessage = (event: { readonly data: unknown }): void => { + if (event.data instanceof ArrayBuffer) this.deliver(new Uint8Array(event.data)); + else this.close(); + }; + readonly #onError = (): void => this.#end(); + + constructor(socket: WebSocketLike) { + super(); + if (socket.readyState !== CONNECTING && socket.readyState !== OPEN) { + this.ended(); + return; + } + this.#socket = socket; + socket.binaryType = "arraybuffer"; + if (socket.readyState === CONNECTING) { + this.#connecting = []; + socket.addEventListener("open", this.#onOpen); + } + socket.addEventListener("message", this.#onMessage); + // `error` and `close` stay handled until the socket's own close, after this end closed: `ws` emits an `error` + // for a socket closed while it connects, and an unhandled one would end the process. + const onClose = (): void => { + socket.removeEventListener("close", onClose); + socket.removeEventListener("error", this.#onError); + this.#end(); + }; + socket.addEventListener("close", onClose); + socket.addEventListener("error", this.#onError); + } + + send(message: Uint8Array): void { + const socket = this.#socket; + if (socket === undefined) return; + if (this.#connecting !== undefined) { + this.#connecting.push(message); + return; + } + try { + // A link's messages are codec output or received frames, both on plain `ArrayBuffer`s. + socket.send(message as Uint8Array); + } catch { + this.close(); + } + } + + close(): void { + const socket = this.#socket; + if (socket === undefined) return; + this.#end(); + socket.close(); + } + + /** Stops listening for frames, drops what waits to be sent and closes this end. */ + #end(): void { + const socket = this.#socket; + if (socket === undefined) return; + this.#socket = undefined; + this.#connecting = undefined; + socket.removeEventListener("open", this.#onOpen); + socket.removeEventListener("message", this.#onMessage); + this.ended(); + } +} diff --git a/packages/chord/src/next/remote/README.md b/packages/chord/src/next/remote/README.md new file mode 100644 index 000000000..cd27471a5 --- /dev/null +++ b/packages/chord/src/next/remote/README.md @@ -0,0 +1,172 @@ +# remote + +Remote objects over links (§3 hosts, links, routes, flow control and recovery, §4, §7, §22, W1; decision "Relay flow +control per origin"): both sides of every link of a host, the paths its upstream holds take across links, relayed +holds, the window of `bound` and `state` messages per origin, the call window, and (slice 4b) parts made and opened +within each link's limits, compression at the origin, activation chains and revocation from the parent. Apps use them through `host.accept`, `host.connect`, +`host.route` and references; nothing here is exported for apps. + +## For Chord's layers (`internal.ts`) + +- `HostLink` (one per link: `accept`, `connect` or a route's `resolve`): one `Channel` (handshake, codec, checks, + heartbeat), a role (`parent`, `child`, `route`), the peer's principal as this host stamps it, a provider side per + **origin** and one consumer side. Message kinds are directional, so each goes to its side: `hold`, `release`, + `call`, `abort`, `retry` and `ack` to the provider side of their `origin` (0 when absent; a `hold` makes it, other + messages for an origin without one are ignored), `closed { origin }` ends that provider side; `bound`, `state`, + `failed`, `loading`, `result`, `error` and `gone` to the consumer side; `revoke` to the host, from a parent link only + (`host.revoke` with this host's own `revokedBy`; anywhere else it is ignored). `origin` is honoured on every link. + Closing it, from either end, ends both sides: the provider sides release the peer's holds and abort their calls, the + consumer side's calls reject and its path tells the holders. It makes the parts this host originates (`part`: + deflated above 16 KiB with the host's compressor) and opens the parts it owns (`openPart`: within the limits its + owner gives, inflated in place with the host's compressor). On a host without a compressor the owner decides first + what it owns (`inflates`: a call's arguments on a bound hold served here, the values of its own known holds' `bound` + and `state`, the result of a call still waiting; never a part it relays, never one for an unknown id, Q1, never one + past the wire limit), then inflates those asynchronously one after another within their decoded limit (`inflate`) + and handles the message again; the channel holds the link's later messages until then, and the link's close cancels + the inflate at once. +- **Whom a hold acts for** (`ProviderSide`): on a parent link, the hold's `principal` and `incarnation` fields when + present, else the link's principal and the peer's incarnation; on child and route links always the link's (fields + ignored: a tab cannot claim to be the owner). Calls carry no principal: a call runs as its hold's caller + (`ctx.principal`, `ctx.incarnation`). Trust never grants the fields. + + What a host sends in those fields when it holds over a route link (`actingFor`, for `RemoteObject` and + `RelayedHold`); on a parent link it sends none (the parent ignores them from a child: upward is a boundary, a worker + that acts for a user towards the server passes the user as data). A relay never passes on the fields it received + (`HoldRequest` has none): it decides whom the hold acts for once, when it arrives: + + | The reference was made by | `principal` | `incarnation` | + | --- | --- | --- | + | `host.get` / `host.hold` | the host's own | this host's | + | `ctx.get` in a handler | the handler's `ctx.principal` (the caller's) | the handler's `ctx.incarnation` | + | `self.get` in a remote object | that object's principal | this host's | + | `self.get` in a local object or a facet | the host's own | this host's | + | a relayed hold | the arriving hold's principal | the arriving hold's incarnation | + +- **Origins.** An origin is whose work a message is: 0 for the sender's own, or a relay's number for one of its + downstream origins (a downstream link, and the origin its messages carry), one counter per host (`host.relays`), + never reused. Messages carry it as `origin` (absent for 0) and answers carry it back. +- `ProviderSide` (one origin of a link: the holds and calls the peer sent for it, by their ids): a message naming an id + it does not know is ignored (Q1), one reusing a live id is a protocol error, and so are more running calls than this + side announced in `hello` (N = 256 per origin; a call on an unknown hold is ignored before it is counted). Its table + has two kinds of entries: `Hold` (served here) and `RelayedHold` (relayed, never hydrated). A call on a pending hold + is answered `not_ready`; an aborted, released or gone call gets no reply. While the host closes, its holds are only + dropped (the host ends the objects itself). It sends with its origin. A hold's `chain` is kept only from a trusted + link and its stamped `limit` only from a parent link; its decoded limit is the smaller of the link's and the stamped + one, which a relayed hold carries upstream as `limit`. A call on a relayed hold whose arguments are past the link's + wire limit, or on a served hold whose arguments are past its limits, is answered `too_large/part` and never + admitted. Compressed params (on any link) and compressed arguments from a child are protocol errors: params are never + compressed, and clients never send compressed parts. Holds of a principal being revoked that still wait for `load` + end with `gone` (`revokeWaiting`). +- **The window** (K = 2, `ProviderSide`): at most two of this host's own `bound` and `state` messages unacknowledged + per origin; every other message is control and never waits. The consumer sends `ack` with the origin after applying + each. The side keeps the sender of every message in flight in order (`local`, or the upstream link a relayed one came + from); each `ack` answers the oldest: its own frees a slot, a relayed one goes to that upstream link as `ack` with + the origin's number there. One with nothing in flight is ignored. + - At the end of a batch, with a slot free and nothing waiting, one `state` message carries an entry per hold the + batch touched. Otherwise each touched hold's ops join its members' pending queues (`PendingOps` from `delta/`: raw + ops, folded by the leaf index, past 10,000 only "replace with the current value") and the hold waits. + - A hold that goes live binds now when a slot is free and nothing waits, else waits for a slot; its snapshot and + subscription are taken in one synchronous step when its `bound` is sent (race 2). + - When an `ack` frees a slot: waiting state goes out as one `state` message (each waiting member's pending ops encoded + once, or `[["r", value]]` after an overflow), waiting bindings as one `bound` with every waiting hold; when both + wait they take turns, state first, so neither a hold that keeps changing nor a stream of new holds starves the + other. + - A live object stays live until it ends, so a hold waiting for its slot never goes back to pending; slice 7's + reload adds that. +- **At a relay** nothing is folded, dropped or reset: each upstream `bound`/`state` for an origin is forwarded at once + and acknowledged upstream when the downstream acknowledges it. A slow tab fills only its own origin's window at each + worker, whose pending queues fold its commits; other tabs are other origins. A tab served by W workers may have 2W + messages in flight, plus its relay's own 2. + + Trace (tab D through relay R, worker U, a derive over A and B on the tab): U's window for D's origin is full; a + commit touching A and B joins U's pending queues. D acks one: R returns the `ack` to U, whose freed slot carries one + `state` with A's and B's entries: the derive sees both at once, never A new with B old. +- `Hold` (one per `hold` served here, a `Holder` of its object): takes its object once `load` resolved; the hold's + `params` part is parsed then (malformed: a protocol error). A refused type, a mismatch, invalid params, a wrong + placement or a snapshot that throws is a broken `failed` that ends it (the provider forgets the id). While pending + it follows its object's `state`: each failure, transient or broken, is sent as `failed` with `retryIn`, the rest of + the object's backoff window (absent when broken), the current one at once when the object is failed already; the + next attempt is announced with `loading`; the hold stays, so a `retry` (from any holder) renews even a broken object + for every holder. Live, it binds through the window (`boundEntry`: the variant and each member's value as a part, + names as indexes into the bound version). Its members' frames arrive as `Subscriber` calls inside the batch; at its + end the hold's entry lists the members that changed with their ops, each encoded with that member's own Delta + encoder (`entry`), or queues them (`queue`, later `pendingEntry`). Each entry counts one `chord.link.encodes`. A + call's method is an index into the bound variant's methods; one outside them is a protocol error, its arguments are + parsed when it arrives. A member that fails is listed in the header of the message that carries the failure, once + per failure episode; while the window is full a member keeps one pending failure at most (a later one replaces it, + dropping the member's pending ops; a value after it replaces it), and a retirement of a failed member waits for its + next value (never listed beside its failure). Its + object's end sends `gone` with its error (`forbidden/revoked` for a revocation; its running calls aborted with it). A + released pending hold stops its wait for `load` at once. Params past the link's wire limit, or its decoded one, fail + the hold broken `too_large/part`; a cycle its chain closes, broken `invalid/cycle` (`host.serving`). It counts as one + of the host's `references` (`host.activity`) while it keeps its object. +- `RelayedHold` (a hold this host neither serves nor parses, `host.place` decided it): id `h` of its downstream + origin, held with the same id on its path's current link under that origin's number, as the arriving hold's caller, + from its kept `hold` message (params copied once out of the received message, so no view keeps the buffer). `call`, + `abort`, `retry` and `release` go upstream at once; a call with no upstream link is answered `unavailable`. What + comes back with the origin reaches its downstream `ProviderSide` (`forward`), which sends it whole with its own + origin, ids and parts untouched. A link answers only for what was relayed over it: an entry, notice or reply whose + hold (or call) is not relayed over the link it came from is dropped (Q1), a `bound`/`state` left empty is + acknowledged to that link at once, and a link the origin relayed nothing to is ignored whole. Answers travel + unchecked and unparsed: the end that parses a part checks it against its own link's limits. A revocation ends a + relayed hold of the principal still waiting for its path's link (`revoked`: `gone` downstream), since the link it + would be held on next gets no past `revoke`. Two downstream holds are two upstream holds with their own encoders + (R1). + - Upstream drop (its path tells it): relayed calls in flight `unavailable` (never resent), `loading` downstream; the + path's next link holds it again with the same id and origin, and its `bound` is forwarded. + - Downstream drop: the origin's provider side forgets its relayed holds and sends one `closed { origin }` to each + upstream link it relayed to (no `release` or `abort`). + - `gone` from upstream is forwarded and the relayed hold forgotten. +- `ConsumerSide` (the holds and calls this host's own references sent on a link, by its own counters from 0): + `attach(holder)` holds a `RemoteObject` with a new id. A message with an origin goes to that origin's provider side + (`host.relays`; an unknown origin is ignored). A `bound` or `state` message is prepared for every entry first (one + that does not fit, an index outside the bound version or a member twice included, closes the link before anything + changed), then adopted in one batch of this host, then one `ack` is sent. A malformed reply rejects its call + `unavailable` before the link closes. When it closes, calls in flight and waiting reject `unavailable` (never + resent) and its tables are dropped: nothing of a closed link reaches the next. +- `CallWindow` (one per consumer side): at most the peer's N calls (from its `hello`; none before it) in flight; further + calls wait in one FIFO queue, sent in order as replies free slots; it does not preempt running calls. A relay + forwards its downstream calls at once, so only this host's own calls wait here. A slot is freed once per call: its + `result` or `error`, its `abort` (when sent), its hold's `release` or `gone`, or the link's close. A call still + waiting when its hold gets `loading` is rejected `unavailable` (it may name a method of the old variant). +- `Path` (the parent, or one route address): its current link and its holders (`RemoteObject`s and `RelayedHold`s), + held on that link and again on every next one; when the link drops, every holder is told (`dropped`). `changed` + tells its owner (a route) when it needs a link or lost its last holder. +- `RemoteObject` (one per remote reference, owned by its host across the links of its path): its `state` (`loading` + until a link's hold is bound and again on every drop or `loading`, `failed` from `failed` with `retryAt = now + + retryIn`, `live` on `bound`, `released`), `settled` (pending while loading, resolved while live, rejected while + failed and by a drop or a relay's `loading` while loading: a dependent's activation waits for it and fails at once + then, R18, and `bound` retries the dependent), `lastFailure`, `variant` (set in `bound`'s batch, before derives recompute) and + `member` (the last binding's, kept while loading or failed; outside the variant `invalid/member`), `call` (only while + live; outside the variant `invalid/member`), `retry` (sends `retry` while failed on an open link; after a part past + the limits, placed anew with `host.pathOf` and held with a new id, now or on that path's next link). Its holds made + while the object that declared it activates carry that object's chain (`activationChain`). A snapshot or state part + past the link's limits fails only its hold (`tooLarge`): released at the provider, the reference broken + `too_large/part` and moved to the host's `unplaced` path, so no reconnect holds it and no route resolves for it, + until `retry`; the message's other entries apply. Its call arguments go up a parent link + uncompressed (the parent may have accepted it as a child). It keeps its + binding's caller: two references with different principals are two remote objects. Its holds carry the params as a + part and the address its placement computed. A `bound` (a first binding, a re-hold; also on a live hold) + replaces the replicas in the message's batch: members it had get the snapshot as one frame `[["r", value]]` and a + fresh decoder, members outside a new variant vanish (reads throw `invalid/member`, watches get one `failed` call and + stay open, derives over them fail; a later binding with the member brings it back as `[["r", value]]`, as on the + provider's host), new members are made; then the reference goes live (its parked calls are sent). Its release sends + `release` on the current link and rejects its calls `released`; `gone` and the host's close end the reference with + their error. +- `Replica`: a state member on the consumer, a `Source` with its own Delta decoder: built from the snapshot, then + `decode(wire)` (the new value by Delta's `applyImmutable`, structure shared, nothing validated) and `adopt(value, + ops)` inside the batch, `fail(error)` (a failure call: the watches stay open; the provider's next value for it is + `[["r", value]]`, which `decode` alone accepts while failed); inside a re-hold's batch `reset(snapshot)` and + `vanish(error)`; `end(error)` outside a batch when its reference ends. + +Rules: + +- One encoder and one decoder per member of a hold: two references from one host are two holds, and so are two + downstream holds a relay forwards. +- A host serves over all its links (a server holds a worker's types over a route link and serves the worker's holds of + server types on the same link) and holds over the parent link and its route links. Who reconnects a parent link is + the app (its adapter calls `connect` again); route links are resolved again by their route. Chord only re-holds. + Calls on a loading reference park in its `Binding` (`host/`), so links see only calls on live holds. +- Costs per hop: a relay decodes the envelope, swaps the origin (one shallow copy) and encodes the envelope, copying + parts once (never parsing them), one FIFO entry per `bound`/`state`; a provider encodes once per commit per touched + member, or once per waiting member per freed slot; a consumer adds one `ack` per `bound`/`state` message. diff --git a/packages/chord/src/next/remote/calls.ts b/packages/chord/src/next/remote/calls.ts new file mode 100644 index 000000000..dc415da51 --- /dev/null +++ b/packages/chord/src/next/remote/calls.ts @@ -0,0 +1,70 @@ +// The call window (§3, decision log "Flow control: an ack-clocked window"): at most the peer's N calls in flight on a +// link from this host; further calls wait in one FIFO queue, sent in order as replies free slots. + +/** A call waiting for a slot: its hold (a rebind or release takes it out) and what sends it. */ +export type QueuedCall = { readonly hold: number; readonly send: () => void }; + +/** + * The calls of one link's consumer side. `limit` is the peer's N from its `hello` (0 before it, so every call waits). + * A relay forwards its downstream calls at once, so only this host's own calls are counted here. + */ +export class CallWindow { + #limit = 0; + #inFlight = 0; + readonly #waiting = new Set(); + + /** Sends `call` now when a slot is free and nothing waits, otherwise queues it behind the earlier calls. */ + submit(call: C): void { + if (this.#waiting.size === 0 && this.#inFlight < this.#limit) { + this.#inFlight++; + call.send(); + return; + } + this.#waiting.add(call); + } + + /** The peer's window arrived with its `hello`: waiting calls go out. */ + opened(limit: number): void { + this.#limit = limit; + this.#drain(); + } + + /** A call in flight ended (its reply, its abort sent, its hold released or gone): its slot is free. */ + freed(): void { + this.#inFlight--; + this.#drain(); + } + + /** Takes a waiting call out (its caller cancelled); nothing happens for one that is not waiting. */ + remove(call: C): void { + this.#waiting.delete(call); + } + + /** Takes every waiting call of `hold` out and returns them (its hold rebinds or was released). */ + removeHold(hold: number): C[] { + const removed: C[] = []; + for (const call of this.#waiting) { + if (call.hold !== hold) continue; + this.#waiting.delete(call); + removed.push(call); + } + return removed; + } + + /** Takes every waiting call out and returns them (the link closed). */ + clear(): C[] { + const removed = [...this.#waiting]; + this.#waiting.clear(); + return removed; + } + + /** Sends waiting calls in order while slots are free. */ + #drain(): void { + for (const call of this.#waiting) { + if (this.#inFlight >= this.#limit) return; + this.#waiting.delete(call); + this.#inFlight++; + call.send(); + } + } +} diff --git a/packages/chord/src/next/remote/consumer.ts b/packages/chord/src/next/remote/consumer.ts new file mode 100644 index 000000000..8d654e291 --- /dev/null +++ b/packages/chord/src/next/remote/consumer.ts @@ -0,0 +1,355 @@ +// The consumer side of a link (§7, §22, W1): the holds this host's own remote references sent on it, by this side's +// ids, the calls through them within the peer's call window, and each `bound` and `state` message applied to their +// replicas as one batch, then acknowledged. Messages with an origin answer holds this host relays: they go to that +// origin's provider side. On a host without a compressor, the deflated parts of this host's own holds and calls are +// inflated asynchronously first, and the message is handled once they are. + +import type { Context } from "../../context/index.ts"; +import { type MethodData, takesNoArgs } from "../contracts/internal.ts"; +import { ChordError, isChordError } from "../data/index.ts"; +import { asChordError } from "../data/internal.ts"; +import { + type BoundEntry, + type ConsumerMessage, + type DeflatedPart, + errorFromWire, + type Message, + type Part, + parsePayload, + payloadPart, + type StateEntry, +} from "../link/internal.ts"; +import { CallWindow } from "./calls.ts"; +import { actingFor } from "./hold.ts"; +import type { HostLink } from "./host-link.ts"; +import type { RemoteObject } from "./remote-object.ts"; + +/** A reply to a call. */ +type Reply = Extract; + +/** + * A call through this side: waiting for a slot of the call window (no id yet), then in flight under its id until its + * reply, its abort, its hold's release or `gone`, or the link's close. + */ +type OutgoingCall = { + readonly hold: number; + /** The method's index in the bound variant and its arguments' part. */ + readonly method: number; + readonly args: Part | undefined; + readonly segments: readonly Uint8Array[] | undefined; + readonly resolve: (value: unknown) => void; + readonly reject: (error: unknown) => void; + id: number | undefined; + send(): void; +}; + +/** + * The consumer side of one link. Ids are its own counters, from 0; a reply naming an id this side gave up is ignored + * (Q1). When the link closes, every call rejects `unavailable` (never resent) and its path tells every holder. + */ +export class ConsumerSide { + readonly #link: HostLink; + readonly #entries = new Map(); + readonly #calls = new Map(); + readonly #window = new CallWindow(); + #nextHold = 0; + #nextCall = 0; + + constructor(link: HostLink) { + this.#link = link; + } + + /** + * Holds `holder` on this link and returns its new id: its first hold, or a re-hold on a new link; `chain`: the + * activation chain it is held for, if any (Q5). + */ + attach(holder: RemoteObject, chain: string[] | undefined): number { + const id = this.#nextHold++; + this.#entries.set(id, holder); + const acting = actingFor(this.#link, holder.caller); + this.#link.send({ + kind: "hold", + hold: id, + ...holder.request, + ...acting, + ...(chain === undefined ? {} : { chain }), + }); + return id; + } + + /** + * Calls method `method` (an index into the bound variant) on hold `hold` with `args`: left out for a method without + * arguments, with byte segments when its argument schema has a `bytes()` leaf, deflated above the threshold except on + * the parent link. The result is parsed here. The caller's cancellation rejects it at once and sends `abort` when it + * is in flight. + */ + call(hold: number, method: number, data: MethodData, args: unknown, context: Context): Promise { + const signal = context.abortSignal; + signal?.throwIfAborted(); + const payload = takesNoArgs(data) ? undefined : payloadPart(args, data.args.bytes); + // Up a parent link they travel uncompressed: the parent may have accepted this host as a child, and child links + // refuse compressed parts (§22). + const link = this.#link; + const part = + payload === undefined ? undefined : link.role === "parent" ? payload.json : link.part(payload.json, context); + const { promise, resolve, reject } = Promise.withResolvers(); + const cancel = (): void => { + this.#cancel(call); + reject(signal!.reason); + }; + const done = (): void => signal?.removeEventListener("abort", cancel); + const call: OutgoingCall = { + hold, + method, + args: part, + segments: payload?.segments, + resolve: (value) => { + done(); + resolve(value); + }, + reject: (error) => { + done(); + reject(error); + }, + id: undefined, + send: () => this.#send(call), + }; + signal?.addEventListener("abort", cancel, { once: true }); + this.#window.submit(call); + return promise; + } + + /** Hold `hold` was released: the provider is told, its calls fail with `error`. */ + release(hold: number, error: ChordError): void { + this.#link.send({ kind: "release", hold }); + this.forget(hold, error); + } + + /** `ref.retry()` on a failed hold: the provider is asked for an attempt now. */ + retry(hold: number): void { + this.#link.send({ kind: "retry", hold }); + } + + /** Drops hold `hold` from the tables; its calls, in flight and waiting, fail with `error` and free their slots. */ + forget(hold: number, error: ChordError): void { + this.#entries.delete(hold); + this.#failCalls(hold, error); + } + + /** The peer's `hello` arrived: calls go out within its window. */ + opened(calls: number): void { + this.#window.opened(calls); + } + + /** + * Handles one message from the provider: this host's own, or (with an origin) one for a relayed hold. A `bound`, + * `state` or `result` with deflated parts to inflate asynchronously returns that promise and is handled after it. + */ + message(message: ConsumerMessage): void | Promise { + if (message.origin !== undefined) { + this.#link.host.relays.get(message.origin)?.forward(message, this.#link); + return; + } + switch (message.kind) { + case "bound": + case "state": { + const deflated = this.#deflated(message.holds); + if (deflated !== undefined) return this.#link.inflate(deflated, () => this.message(message)); + if (message.kind === "bound") this.#bound(message.holds); + else this.#state(message.holds); + break; + } + case "failed": + this.#entries.get(message.hold)?.failed(message); + break; + case "loading": + this.#loading(message.hold); + break; + case "result": + case "error": + return this.#reply(message); + case "gone": + this.#gone(message); + break; + } + } + + /** The link closed: calls in flight and waiting reject `unavailable` (never resent, §7); the tables are dropped. */ + closed(): void { + const error = new ChordError({ code: "unavailable", message: "the link closed" }); + const calls = [...this.#calls.values(), ...this.#window.clear()]; + this.#calls.clear(); + this.#entries.clear(); + for (const call of calls) call.reject(error); + } + + /** Sends a call that got its slot, with the next id. */ + #send(call: OutgoingCall): void { + const id = this.#nextCall++; + call.id = id; + this.#calls.set(id, call); + const { hold, method, args, segments } = call; + this.#link.send({ + kind: "call", + call: id, + hold, + method, + ...(args === undefined ? {} : { args }), + ...(segments === undefined ? {} : { segments }), + }); + } + + /** + * Cancels `call`: a waiting one leaves the window and nothing is sent; one in flight sends `abort` and frees its + * slot. Settling the caller is the caller's. + */ + #cancel(call: OutgoingCall): void { + if (call.id === undefined) { + this.#window.remove(call); + return; + } + if (!this.#calls.delete(call.id)) return; + this.#link.send({ kind: "abort", call: call.id }); + this.#window.freed(); + } + + /** + * The deflated parts of this side's own holds in `entries` that wait for an asynchronous inflate, each with the + * link's decoded limit; `undefined` when none does (nothing allocated). Entries of holds this side gave up are not + * inflated (Q1). + */ + #deflated(entries: readonly (BoundEntry | StateEntry)[]): [DeflatedPart, number][] | undefined { + const { limits } = this.#link; + let deflated: [DeflatedPart, number][] | undefined; + for (const entry of entries) { + if (!this.#entries.has(entry.hold)) continue; + for (const member of entry.members) { + const part = "value" in member ? member.value : member.ops; + if (!this.#link.inflates(part, limits)) continue; + deflated ??= []; + deflated.push([part, limits.decoded]); + } + } + return deflated; + } + + /** + * One `bound` message: every entry prepared first (one that does not fit closes the link before anything changed) + * and applied in one batch; then the references go live, and the message is acknowledged. An entry with a part past + * the link's limits fails only its hold (`too_large/part`); the others apply. + */ + #bound(holds: readonly BoundEntry[]): void { + const adoptions: (() => void)[] = []; + const bound: RemoteObject[] = []; + let oversized: [RemoteObject, ChordError][] | undefined; + for (const entry of holds) { + const holder = this.#entries.get(entry.hold); + if (holder === undefined) continue; + try { + adoptions.push(holder.prepareBound(entry, this.#link)); + } catch (error) { + if (!isChordError(error, "too_large")) throw error; + oversized ??= []; + oversized.push([holder, error]); + continue; + } + bound.push(holder); + } + this.#adopt(adoptions); + // After the batch: the references go live with every replica of the message in place. + for (const holder of bound) holder.wentLive(); + for (const [holder, error] of oversized ?? []) holder.tooLarge(error); + this.#link.send({ kind: "ack" }); + } + + /** One `state` message, as `#bound` does it. */ + #state(holds: readonly StateEntry[]): void { + const adoptions: (() => void)[] = []; + let oversized: [RemoteObject, ChordError][] | undefined; + for (const entry of holds) { + const holder = this.#entries.get(entry.hold); + if (holder === undefined) continue; + try { + adoptions.push(holder.prepare(entry, this.#link)); + } catch (error) { + if (!isChordError(error, "too_large")) throw error; + oversized ??= []; + oversized.push([holder, error]); + } + } + this.#adopt(adoptions); + for (const [holder, error] of oversized ?? []) holder.tooLarge(error); + this.#link.send({ kind: "ack" }); + } + + /** Applies prepared entries in one batch of this host, so no watch or derive sees half a provider commit (W1). */ + #adopt(adoptions: readonly (() => void)[]): void { + if (adoptions.length === 0) return; + this.#link.host.batch.commit(() => { + for (const adopt of adoptions) adopt(); + }, this.#link.context); + } + + /** + * `loading`: the hold's object, or its relay's upstream, starts over; its calls still waiting for a slot reject + * `unavailable` (they may name a method of the old variant). + */ + #loading(hold: number): void { + const holder = this.#entries.get(hold); + if (holder === undefined) return; + const error = new ChordError({ code: "unavailable", message: `hold ${hold}: rebinding` }); + for (const call of this.#window.removeHold(hold)) call.reject(error); + holder.loading(); + } + + /** + * `gone`: one for a hold this side gave up is ignored (Q1); the holder ends, which forgets its hold here and fails + * its calls. + */ + #gone(message: Extract): void { + const holder = this.#entries.get(message.hold); + if (holder === undefined) return; + holder.lost(asChordError(errorFromWire(message.error), `hold ${message.hold}`)); + } + + /** + * A reply: its call settles with the parsed result or the error. A result past the link's limits rejects the call + * `too_large/part`. A malformed part rejects the call `unavailable` and throws (a protocol error: the link closes). + * A result to inflate asynchronously is inflated first (for a call still waiting), then handled again. + */ + #reply(reply: Reply): void | Promise { + const call = this.#calls.get(reply.call); + if (call === undefined) return; + const link = this.#link; + if (reply.kind === "result" && link.inflates(reply.value, link.limits)) { + return link.inflate([[reply.value, link.limits.decoded]], () => this.#reply(reply)); + } + this.#calls.delete(reply.call); + this.#window.freed(); + try { + if (reply.kind === "error") call.reject(errorFromWire(reply.error)); + else if (reply.value === undefined) call.resolve(null); + else call.resolve(parsePayload(link.openPart(reply.value, link.limits, link.context), reply.segments)); + } catch (error) { + if (isChordError(error, "too_large")) { + call.reject(error); + return; + } + call.reject(new ChordError({ code: "unavailable", message: `call ${reply.call}: a malformed reply` })); + throw error; + } + } + + /** Fails `hold`'s calls in flight and waiting with `error`; each in-flight one frees its slot. */ + #failCalls(hold: number, error: ChordError): void { + // Waiting ones first: a freed slot must not send one of them. + const failed = this.#window.removeHold(hold); + for (const [id, call] of this.#calls) { + if (call.hold !== hold) continue; + this.#calls.delete(id); + this.#window.freed(); + failed.push(call); + } + for (const call of failed) call.reject(error); + } +} diff --git a/packages/chord/src/next/remote/hold.ts b/packages/chord/src/next/remote/hold.ts new file mode 100644 index 000000000..a87c4aaa8 --- /dev/null +++ b/packages/chord/src/next/remote/hold.ts @@ -0,0 +1,434 @@ +// A hold arriving on a link and served here (§7, §22, W1): where it goes once `load` resolved, the object it keeps, +// the notices of its object's failures and new attempts, the snapshot and subscription taken in one synchronous step +// when a slot of the window is free, and the ops of its members per batch (sent at once, or pending while the window +// is full), each member with its own Delta encoder. + +import type { Context } from "../../context/index.ts"; +import { type Encoder, encoder, type Op, PendingOps } from "../../delta/index.ts"; +import type { JsonValue } from "../../json.ts"; +import { + currentVariants, + type VariantData, + type VariantNames, + variantNames, + variantOrder, +} from "../contracts/internal.ts"; +import { type ChordError, isChordError } from "../data/index.ts"; +import { asChordError } from "../data/internal.ts"; +import type { Caller, ChordObject, Holder } from "../host/internal.ts"; +import { + type BoundEntry, + type FailedMember, + jsonPart, + type Limits, + type Message, + NO_RETIRED, + oversized, + parsePart, + type StateEntry, + wireError, +} from "../link/internal.ts"; +import { Source, type Subscriber } from "../state/internal.ts"; +import type { HostLink } from "./host-link.ts"; +import type { ProviderSide } from "./provider.ts"; +import { RelayedHold } from "./relay.ts"; + +/** A `hold` message. */ +type HoldMessage = Extract; + +/** + * What a `hold` message asks for. Whom it acts for is not part of it: a host takes that from the message once, as its + * link allows (`ProviderSide`), and names it again only on a route link (`actingFor`). Its `chain` and `limit` are the + * ones this host honours, sent on as they are when it relays the hold. + */ +export type HoldRequest = Omit; + +/** + * A state member a bound hold follows: its index in the bound version, its source and subscription (which feeds this + * record), its own encoder (R1, path interning) and the ops waiting for a slot of the window. + */ +type Followed = { + readonly index: number; + readonly source: Source; + readonly subscriber: Subscriber; + readonly encoder: Encoder; + readonly pending: PendingOps; +}; + +/** + * Where a hold stands: `pending` until its object is live (waiting for `load`, the activation and its retries; + * `toldFailed` once the consumer was told of a failure, so the next attempt is announced with `loading`); `binding` + * while its object is live and its `bound` waits for a slot; `bound` with the object's variant and the members it + * follows. + */ +type HoldState = + | { readonly kind: "pending"; readonly toldFailed: boolean } + | { readonly kind: "binding" } + | { + readonly kind: "bound"; + readonly variant: VariantData; + /** The bound version's names of the variant, which the wire's indexes point into. */ + readonly names: VariantNames; + /** The members it follows, in the order they were subscribed. */ + readonly followed: Followed[]; + } + | { readonly kind: "released" }; + +/** A bound hold's state. */ +type Bound = Extract; + +const PENDING: HoldState = { kind: "pending", toldFailed: false }; + +const BINDING: HoldState = { kind: "binding" }; + +/** + * One hold served here: a reference to its object acting as its caller (the hold's principal). While pending, each + * failure of its object is sent as `failed` (with the rest of its backoff window) and the next attempt as `loading`; a + * hold that cannot be taken (a refused type, a mismatch, invalid params or placement) is a broken `failed` that ends + * it. Once live it binds when its link's window has a slot, then sends a part per batch that changed its members + * (or keeps their ops pending while the window is full), and `gone` when its object ends. + */ +export class Hold implements Holder { + readonly kind = "served"; + readonly id: number; + /** A hold is declared by no object on this host. */ + readonly dependent = undefined; + /** Whom the hold acts for: its calls run as it, with its origin's incarnation. */ + readonly caller: Caller; + /** The limits per part of its params and its calls' arguments: its link's, the decoded one smaller when stamped (Q2). */ + readonly limits: Limits; + readonly #provider: ProviderSide; + #state: HoldState = PENDING; + /** The held object, from its binding until the hold is released. */ + #object: ChordObject | undefined; + /** + * Resolved on release: a pending hold stops waiting for `load`. A plain resolution, not an abort: an aborted wait + * rejects with an error whose stack keeps this hold, and the waited promise keeps that rejection. + */ + readonly #released = Promise.withResolvers(); + /** The members the current batch changed, with their ops, in order. */ + #changes: (readonly [Followed, readonly Op[]])[] = []; + /** The members that failed since the last message that carried this hold. */ + #failed: FailedMember[] = []; + /** The members whose document was retired since the last message that carried this hold (ORed while it waits). */ + #retired: readonly number[] = NO_RETIRED; + #touched = false; + /** Whether the hold counts as one of the host's references (`host.activity`): while it keeps its object. */ + #counted = false; + + constructor(id: number, provider: ProviderSide, caller: Caller, limits: Limits) { + this.id = id; + this.#provider = provider; + this.caller = caller; + this.limits = limits; + } + + /** Whether the hold is live: calls on it reach its object. */ + get bound(): boolean { + return this.#state.kind === "bound"; + } + + /** Whether the hold waits for `load`: admitted, nothing taken yet (revocation ends it, §4). */ + get waiting(): boolean { + return this.#state.kind === "pending" && this.#object === undefined; + } + + /** + * Admits the hold once `load` resolved: its params within the link's part limit (`too_large/part`), then relayed + * (it is replaced by a relayed hold), or served here, its params opened and parsed (a malformed part is a protocol + * error), its object taken and followed (`moved`). A hold that cannot go anywhere, a type that does not match, + * invalid params, a wrong placement or a cycle (Q5) is a broken `failed`, which ends it. + */ + async start(request: HoldRequest): Promise { + const { link } = this.#provider; + // Waits only for a `load` still running (it never rejects): once loaded, the reference is taken in the turn the + // hold arrived. + const pendingLoad = link.host.pendingLoad; + if (pendingLoad !== undefined) await Promise.race([pendingLoad, this.#released.promise]); + // Released meanwhile (the wait may have settled first): nothing is taken. + if (this.#state.kind !== "pending") return; + let object: ChordObject; + try { + const oversize = oversized(request.params, link.limits.part); + if (oversize !== undefined) throw oversize; + const placed = link.host.place(request, link, this.caller, link.context); + if (placed.kind === "path") { + this.#state = { kind: "released" }; + new RelayedHold(this.id, this.#provider, request, this.caller, placed.path).start(); + return; + } + let params: unknown; + try { + params = parsePart(link.openPart(request.params, this.limits, link.context)); + } catch (error) { + // Past its limits: the hold's \`too_large/part\`. No JSON: a protocol error, reported, and the link closes. + if (isChordError(error, "too_large")) throw error; + link.host.report({ kind: "protocol_error", error }); + link.close(); + return; + } + object = link.host.serving(request, params, this.caller.principal!, link.context); + } catch (error) { + return this.#refuse(asChordError(error, request.type)); + } + // Counted before the object lists it in its presence: a derive over that may close this link synchronously. + this.#object = object; + this.#counted = true; + link.host.count("references", 1); + object.hold(this); + this.moved(); + } + + /** + * The object's state changed, or the hold took it: a pending hold binds once it is live (when a slot is free), + * sends each failure, and announces the attempt after a failure with `loading`. A live object stays live until it + * ends, so a hold waiting for its slot or bound has nothing to do (slice 7's reload changes that). + */ + moved(): void { + const current = this.#state; + if (current.kind !== "pending") return; + const state = this.#object!.state; + if (state.kind === "live") { + this.#state = BINDING; + this.#provider.bind(this, this.#provider.link.context); + } else if (state.kind === "failed") { + const { cause, retryAt } = state; + const retryIn = retryAt === undefined ? {} : { retryIn: Math.max(0, retryAt - Date.now()) }; + const failed = { kind: cause.kind, error: wireError(cause.error) }; + this.#provider.send({ kind: "failed", hold: this.id, cause: failed, ...retryIn }); + this.#state = { kind: "pending", toldFailed: true }; + } else if (state.kind === "loading" && current.toldFailed) { + this.#provider.send({ kind: "loading", hold: this.id }); + this.#state = PENDING; + } + } + + /** A `retry` message: the object is asked for an attempt now (it ignores this unless failed). */ + retry(): void { + this.#object?.retry(); + } + + /** Calls a method of the held object as the hold's caller. */ + call(method: string, args: unknown, context: Context): Promise { + return this.#object!.call(method, args, context, this.caller); + } + + /** + * The method at `index` of the bound variant and whether its result carries byte segments (its schema has a + * `bytes()` leaf). Throws on an index outside the variant (a protocol error); a hold that is not bound has none. + */ + method(index: number): { readonly name: string; readonly resultBytes: boolean } { + const state = this.#state; + const name = state.kind === "bound" ? state.names.methods[index] : undefined; + if (name === undefined) throw new Error(`hold ${this.id}: no method ${index}`); + // The bound state holds the variant whose names these are. + const { variant } = state as Bound; + return { name, resultBytes: variant.methods.get(name)!.result.bytes }; + } + + /** + * The hold's binding, sent when a slot is free. The snapshot of every member, as a value or failed, and the + * subscription are taken in this synchronous step (race 2), so each member's next ops continue exactly from it. + * Names travel as indexes into the bound version (2b). Counts one encode (`chord.link.encodes`). + */ + boundEntry(context: Context): BoundEntry { + const object = this.#object!; + const { contract } = object.served; + // A live remote object has its variant. + const variant = object.variant!; + const state = this.#bindTo(object); + const members: BoundEntry["members"][number][] = []; + const failed: FailedMember[] = []; + for (const [member, source] of object.stateMembers) { + const index = state.names.state.indexOf(member); + let value: JsonValue; + try { + value = source.value; + } catch (error) { + // A failed member (a derive whose compute throws) is followed: it recovers with a root replacement. + failed.push({ member: index, error: wireError(error) }); + if (source instanceof Source) this.#follow(state, index, source); + continue; + } + try { + members.push({ member: index, value: this.#provider.link.part(jsonPart(value), context) }); + } catch (error) { + // A value past the link's limits: failed for this hold, which takes no more of it. + failed.push({ member: index, error: wireError(error) }); + continue; + } + // A constant never changes: nothing to follow. + if (source instanceof Source) this.#follow(state, index, source); + } + this.#changes = []; + this.#failed = []; + this.#retired = NO_RETIRED; + context.telemetry.metrics?.counter("chord.link.encodes").add(1); + return { hold: this.id, variant: variantOrder(contract).indexOf(variant), members, failed }; + } + + /** + * This batch's entry of the `state` message: the members it changed with their ops, each encoded once with its + * member's encoder, and the members that failed; resets both. Counts one encode per hold (`chord.link.encodes`). + */ + entry(context: Context): StateEntry { + const { link } = this.#provider; + const members = this.#take().map(([{ index, encoder }, ops]) => ({ + member: index, + ops: link.part(jsonPart(encoder.encode(ops)), context), + })); + return this.#entry(members, context); + } + + /** The window is full: this batch's ops join the members' pending queues (raw, never mutated). */ + queue(): void { + for (const [{ pending }, ops] of this.#take()) pending.push(ops); + } + + /** + * The entry of a freed slot's `state` message: each member's pending ops encoded once, or `[["r", value]]` after an + * overflow, all reflecting the latest commit; and the members that failed meanwhile. Counts one encode per hold. + */ + pendingEntry(context: Context): StateEntry { + const { link } = this.#provider; + const members: StateEntry["members"][number][] = []; + // Only a bound hold has pending ops. + for (const { index, source, encoder, pending } of (this.#state as Bound).followed) { + if (pending.size === 0) continue; + const taken = pending.take(); + const ops = taken.kind === "replace" ? [["r", source.value] as const] : taken.ops; + if (ops.length > 0) members.push({ member: index, ops: link.part(jsonPart(encoder.encode(ops)), context) }); + } + return this.#entry(members, context); + } + + /** The consumer released the hold, or its link closed: its waits stop and its object loses this reference. */ + release(): void { + this.drop(); + this.#object?.unhold(this); + this.#object = undefined; + } + + /** Ends the hold without releasing its object (the host closes and ends its objects itself). */ + drop(): void { + if (this.#state.kind === "bound") { + for (const { source, subscriber } of this.#state.followed) source.unsubscribe(subscriber); + } + this.#state = { kind: "released" }; + this.#released.resolve(); + if (this.#counted) this.#provider.link.host.count("references", -1); + this.#counted = false; + } + + /** + * The object ended while held (revocation; a reload, slice 7), or the hold was revoked while it waited for `load`: + * the hold sends `gone` with `error`. + */ + end(error: ChordError): void { + if (this.#state.kind === "released") return; + this.#provider.forget(this, error); + this.drop(); + this.#object = undefined; + this.#provider.send({ kind: "gone", hold: this.id, error: wireError(error) }); + } + + /** The binding: its variant and names, with no member followed yet. */ + #bindTo(object: ChordObject): Bound { + const data = currentVariants(object.served.contract).get(object.variant!)!; + const state: Bound = { kind: "bound", variant: data, names: variantNames(data), followed: [] }; + this.#state = state; + return state; + } + + /** + * Follows member `index` from its snapshot on (a failed one too: it may recover in this hold): its own encoder and + * pending queue, and a subscription feeding its calls into this hold's batch. Failures fold per member: a failure + * drops what was pending for it and replaces a failure not sent yet (one at most per member, however long the + * consumer stalls); the value after it (a root replacement) replaces the failure. A retirement waits for a value. + */ + #follow(state: Bound, index: number, source: Source): void { + const followed: Followed = { + index, + source, + subscriber: { + advance: (call) => { + this.#touch(); + if (call.kind === "failed") { + followed.pending.take(); + const failure = { member: index, error: wireError(call.error) }; + const at = this.#failed.findIndex(({ member }) => member === index); + // A new list when one is replaced (rare): a list handed to an entry is never written to. + if (at === -1) this.#failed.push(failure); + else this.#failed = this.#failed.map((pending, i) => (i === at ? failure : pending)); + return; + } + // Rare: a failure of this member still waits to be sent. + if (this.#failed.length > 0) this.#failed = this.#failed.filter(({ member }) => member !== index); + const { ops, retired } = call; + // A retired derive's frame may carry no ops (rule 21): only the flag travels. + if (ops.length > 0) this.#changes.push([followed, ops]); + // A new list per retirement (rare): the shared empty one is never written to. + if (retired && !this.#retired.includes(index)) this.#retired = [...this.#retired, index]; + }, + // The member's source ended with its object, which ends this hold. + end: () => {}, + }, + encoder: encoder(), + pending: new PendingOps(), + }; + source.subscribe(followed.subscriber); + state.followed.push(followed); + } + + /** Takes the current batch's changes; the hold is untouched again. */ + #take(): (readonly [Followed, readonly Op[]])[] { + const changes = this.#changes; + this.#changes = []; + this.#touched = false; + return changes; + } + + /** A state entry with `members` and the failures and retirements since the last one; counts one encode. */ + #entry(members: StateEntry["members"], context: Context): StateEntry { + const failed = this.#failed; + let retired = this.#retired; + this.#failed = []; + this.#retired = NO_RETIRED; + if (retired.length > 0 && failed.length > 0) { + // A failed member's retirement travels with its next value, never with its failure. + const kept = retired.filter((index) => failed.some(({ member }) => member === index)); + if (kept.length > 0) { + this.#retired = kept; + retired = retired.filter((index) => !kept.includes(index)); + } + } + context.telemetry.metrics?.counter("chord.link.encodes").add(1); + return { hold: this.id, members, failed, retired }; + } + + #touch(): void { + if (this.#touched) return; + this.#touched = true; + this.#provider.touched(this); + } + + /** Refuses the hold: a broken `failed`, final; the provider forgets it. */ + #refuse(error: ChordError): void { + this.#provider.forget(this); + this.#provider.send({ kind: "failed", hold: this.id, cause: { kind: "broken", error: wireError(error) } }); + this.release(); + } +} + +/** A `hold` message's fields naming whom it acts for. */ +type ActingFor = Pick; + +const NOBODY: ActingFor = {}; + +/** + * The fields of a `hold` message naming whom it acts for (`caller`): sent only on a route link, whose child honours + * them (§4); none elsewhere. + */ +export function actingFor(link: HostLink, { principal, incarnation }: Caller): ActingFor { + return link.role === "route" && principal !== undefined ? { principal, incarnation } : NOBODY; +} diff --git a/packages/chord/src/next/remote/host-link.ts b/packages/chord/src/next/remote/host-link.ts new file mode 100644 index 000000000..f0da9e79f --- /dev/null +++ b/packages/chord/src/next/remote/host-link.ts @@ -0,0 +1,219 @@ +// One link of a host (§3, §22): its channel, its role, the peer's principal, its limits per part, and both sides: a +// provider side per origin (holds and calls the peer sent, its own or a relay's for one of its downstream links) and +// the consumer side (holds and calls this host sent). Message kinds are directional, so each goes to its side; a +// revocation is honoured from the parent only. Parts are made and opened here, with the host's compressor, or inflated +// asynchronously for their owner when the host has none. + +import type { Context } from "../../context/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { ChordError, Principal } from "../data/index.ts"; +import type { AnyHost } from "../host/internal.ts"; +import { + Channel, + type ChannelHandler, + type Codec, + DeflatedPart, + inflateAsync, + type Limits, + type Link, + type Message, + openPart, + originPart, + type Part, +} from "../link/internal.ts"; +import { ConsumerSide } from "./consumer.ts"; +import { ProviderSide } from "./provider.ts"; + +/** + * How a link was made: `parent` by `connect` (holds arriving on it may name whom they act for), `child` by `accept`, + * `route` by a route's `resolve`. + */ +export type LinkRole = "parent" | "child" | "route"; + +/** What a link is made with. */ +type HostLinkOptions = { + readonly role: LinkRole; + /** The peer, as this host stamps it. */ + readonly principal: Principal; + /** Skips the message schema check (framing and the owners' checks still run). */ + readonly trusted: boolean; + readonly codec: Codec; + readonly limits: Limits; +}; + +/** How many concurrent calls a host accepts on each of its links, announced in `hello`. */ +const CALLS = 256; + +/** + * A link of a host, both directions over one channel. Closing it, from either end, ends both sides: the provider sides + * release the peer's holds, the consumer side's holds go loading until their path's next link. + */ +export class HostLink implements ChannelHandler { + readonly host: AnyHost; + readonly role: LinkRole; + readonly principal: Principal; + /** Whether the link is trusted: activation chains on holds arriving on it are honoured (Q5). */ + readonly trusted: boolean; + /** The limits per part of what arrives on this link (Q2). */ + readonly limits: Limits; + /** The parent of the work arriving on this link. */ + readonly context: Context; + readonly consumer: ConsumerSide; + /** + * The provider side of each origin the peer sent holds for, made by its first; `origin` is honoured on every link + * (decision "Relay flow control per origin"). + */ + readonly #providers = new Map(); + readonly #channel: Channel; + /** Aborted when the link closes: stops its asynchronous inflates at once. */ + readonly #closing = new AbortController(); + + constructor(host: AnyHost, link: Link, options: HostLinkOptions, context: Context) { + this.host = host; + this.role = options.role; + this.principal = options.principal; + this.trusted = options.trusted; + this.limits = options.limits; + this.context = context; + this.consumer = new ConsumerSide(this); + const { codec, trusted } = options; + const report = (error: unknown): void => host.report({ kind: "protocol_error", error }); + const channel = { codec, incarnation: host.incarnation, calls: CALLS, trusted, report }; + this.#channel = new Channel(link, this, channel, context); + // Last: a link closed already calls `closed` at once. + this.#channel.start(); + } + + /** The peer's incarnation, from its `hello`. */ + get peer(): string { + return this.#channel.peer; + } + + /** Whether the peer's `hello` arrived (it may have arrived while the link was made). */ + get greeted(): boolean { + return this.#channel.opened; + } + + /** Whether the link is open; one closed already when made is not kept. */ + get open(): boolean { + return !this.#channel.closed; + } + + send(message: Message): void { + this.#channel.send(message); + } + + close(): void { + this.#channel.close(); + } + + /** The heartbeat (see `Channel.tick`). */ + tick(now: number, ping: number, close: number): void { + this.#channel.tick(now, ping, close); + } + + opened(): void { + this.consumer.opened(this.#channel.peerCalls); + this.host.linkOpened(this); + } + + /** A JSON part this host originates on this link: deflated above the threshold when the host compresses. */ + part(json: string, context: Context): Part { + return originPart(json, this.host.compression, context); + } + + /** The JSON of a part arriving on this link, where it is owned, within `limits`. Throws as `openPart` does. */ + openPart(part: Part, limits: Limits, context: Context): string | Uint8Array { + return openPart(part, limits, this.host.compression, context); + } + + /** + * Whether `part` waits for an asynchronous inflate before its owner can open it: deflated, not inflated yet, on a + * host without a synchronous compressor, and within the wire limit (past it, opening refuses it uninflated). + */ + inflates(part: Part | undefined, limits: Limits): part is DeflatedPart { + return ( + part instanceof DeflatedPart && + part.inflated === undefined && + this.host.compression === undefined && + part.bytes.byteLength <= limits.part + ); + } + + /** + * Inflates `parts` (deflated parts an owner admitted, each with its decoded limit) one after another with the web + * `DecompressionStream`, then lets the owner handle its message again (`handle`); the channel holds the link's later + * messages until then. A link closed meanwhile stops the inflate at once and drops the message (race 13). + */ + async inflate(parts: readonly (readonly [DeflatedPart, number])[], handle: () => void): Promise { + const { signal } = this.#closing; + for (const [part, limit] of parts) { + await inflateAsync(part, limit, signal, this.context); + if (signal.aborted) return; + } + handle(); + } + + /** + * A message from the peer, to its side. One whose owner first waits for an asynchronous inflate returns its promise: + * the channel holds later messages until it settled. + */ + message(message: Message): void | Promise { + switch (message.kind) { + case "hold": + this.#hold(message); + break; + case "release": + case "call": + case "abort": + case "retry": + case "ack": + // An origin this link has no holds for: its ids are unknown (Q1). + return this.#providers.get(message.origin ?? 0)?.message(message); + case "closed": + this.#providers.get(message.origin)?.closed(); + this.#providers.delete(message.origin); + break; + case "bound": + case "state": + case "failed": + case "loading": + case "result": + case "error": + case "gone": + return this.consumer.message(message); + case "revoke": + // Only the parent revokes; a child or a route cannot (race 6). + // JSON from the parent, the app's revocation type there as here (the same app code). + if (this.role === "parent") this.host.revoke(message.revocation as JsonValue, this.context); + break; + default: + // `hello`, `ping` and `pong` are the channel's. + break; + } + } + + /** Ends the holds on this link still waiting for `load` that act for a principal `revoked` names (§4). */ + revokeWaiting(revoked: (principal: Principal) => boolean, error: ChordError): void { + for (const provider of this.#providers.values()) provider.revokeWaiting(revoked, error); + } + + closed(): void { + this.#closing.abort(); + for (const provider of this.#providers.values()) provider.closed(); + this.#providers.clear(); + this.consumer.closed(); + this.host.unlink(this); + } + + /** A `hold` goes to its origin's provider side, made for the origin's first. */ + #hold(message: Extract): void { + const origin = message.origin ?? 0; + let provider = this.#providers.get(origin); + if (provider === undefined) { + provider = new ProviderSide(this, origin, CALLS); + this.#providers.set(origin, provider); + } + provider.message(message); + } +} diff --git a/packages/chord/src/next/remote/internal.ts b/packages/chord/src/next/remote/internal.ts new file mode 100644 index 000000000..92ae4ad8c --- /dev/null +++ b/packages/chord/src/next/remote/internal.ts @@ -0,0 +1,8 @@ +// Remote objects for Chord's layers: a host's links (both sides of each), the paths its upstream holds take, and the +// objects of its remote references. + +export type { HoldRequest } from "./hold.ts"; +export { HostLink, type LinkRole } from "./host-link.ts"; +export { Path, type Upstream } from "./path.ts"; +export type { ProviderSide } from "./provider.ts"; +export { RemoteObject } from "./remote-object.ts"; diff --git a/packages/chord/src/next/remote/path.ts b/packages/chord/src/next/remote/path.ts new file mode 100644 index 000000000..b1050ec7d --- /dev/null +++ b/packages/chord/src/next/remote/path.ts @@ -0,0 +1,68 @@ +// Paths (§3 recovery): where this host's upstream holds go, the parent link or a route address's link, across links. +// Each new link of a path holds every holder of the path again; when it drops, every holder is told. + +import { ChordError } from "../data/index.ts"; +import type { HostLink } from "./host-link.ts"; +import type { RelayedHold } from "./relay.ts"; +import type { RemoteObject } from "./remote-object.ts"; + +/** What a path holds on its links: one of this host's remote references, or a hold it relays. */ +export type Upstream = RemoteObject | RelayedHold; + +/** + * The parent link or one route address: its current link and its holders, held on that link and again on every next + * one. `changed` runs when it gets a holder without a link, loses its link or its last holder (a route decides whether + * to resolve). + */ +export class Path { + readonly #holders = new Set(); + readonly #changed: () => void; + #link: HostLink | undefined; + + constructor(changed: () => void) { + this.#changed = changed; + } + + /** The current link while one is open. */ + get link(): HostLink | undefined { + return this.#link; + } + + /** Whether anything is held over this path. */ + get held(): boolean { + return this.#holders.size > 0; + } + + /** Holds `holder` over this path: on the current link now, and on every next one. */ + attach(holder: Upstream): void { + this.#holders.add(holder); + if (this.#link === undefined) this.#changed(); + else holder.attach(this.#link); + } + + /** `holder` was released or ended; no later link holds it. */ + detach(holder: Upstream): void { + this.#holders.delete(holder); + if (this.#holders.size === 0) this.#changed(); + } + + /** `link` is the path's link now: every holder is held on it. */ + connected(link: HostLink): void { + this.#link = link; + for (const holder of this.#holders) holder.attach(link); + } + + /** `link` closed (its sides ended their calls already): every holder goes loading until the next link. */ + disconnected(link: HostLink): void { + if (this.#link !== link) return; + this.#link = undefined; + const error = new ChordError({ code: "unavailable", message: "the link closed" }); + for (const holder of this.#holders) holder.dropped(error); + this.#changed(); + } + + /** Every holder, for the host's close. */ + holders(): Upstream[] { + return [...this.#holders]; + } +} diff --git a/packages/chord/src/next/remote/provider.ts b/packages/chord/src/next/remote/provider.ts new file mode 100644 index 000000000..4325f7c12 --- /dev/null +++ b/packages/chord/src/next/remote/provider.ts @@ -0,0 +1,510 @@ +// The provider side of one origin on a link (§3, §22, W1; decision "Relay flow control per origin"): the holds the +// peer sent for that origin, served here or relayed, by their ids; the calls on them, at most this side's N at once; +// and the window: at most K unacknowledged `bound` and `state` messages of this host's own, with what waits meanwhile, +// and the sender of every message in flight, so each `ack` reaches the host that sent its message. + +import { type Context, withAbortSignal } from "../../context/index.ts"; +import { ChordError, isChordError, type Principal } from "../data/index.ts"; +import type { Caller } from "../host/internal.ts"; +import { + type BoundEntry, + type ConsumerMessage, + DeflatedPart, + type Message, + type OriginMessage, + oversized, + type ProviderMessage, + parsePayload, + payloadPart, + type StateEntry, + wireError, +} from "../link/internal.ts"; +import { Hold } from "./hold.ts"; +import type { HostLink } from "./host-link.ts"; +import type { RelayedHold } from "./relay.ts"; + +/** + * A call being handled: on a hold served here, with whether its result carries byte segments and the abort of its + * handler's context; or relayed, its reply forwarded from upstream. + */ +type RunningCall = + | { readonly kind: "served"; readonly hold: Hold; readonly bytes: boolean; readonly abort: AbortController } + | { readonly kind: "relayed"; readonly hold: RelayedHold }; + +/** Who sent a `bound` or `state` message in flight: this host, or the upstream link a relay forwarded it from. */ +type Sender = "local" | HostLink; + +/** A relaying origin: its number on upstream links, and the open upstream links it relayed holds to. */ +type Relay = { readonly origin: number; readonly links: Set }; + +/** At most this many of this host's own `bound` and `state` messages unacknowledged per origin. */ +const WINDOW = 2; + +/** + * The provider side of one origin on a link. Ids are the origin's; a message naming an id this side does not know is + * ignored (Q1), one reusing a live id is a protocol error, and so are more concurrent calls than this side announced. + * This host's own `bound` and `state` messages go through the window: locally served holds wait while it is full + * (their ops in their members' pending queues, their bindings for the next slot). A relay forwards upstream messages + * at once (each upstream host keeps its own window for the origin it sees), so a slow consumer stalls only its origin. + */ +export class ProviderSide { + readonly link: HostLink; + /** Whose work this side serves: 0 for the peer's own, a relay's number for one of its downstream links. */ + readonly origin: number; + readonly #calls = new Map(); + readonly #entries = new Map(); + readonly #limit: number; + /** The sender of each `bound` and `state` message in flight, in order: the consumer acknowledges in order. */ + readonly #sent: Sender[] = []; + /** This host's own messages in flight. */ + #inFlight = 0; + /** The holds the current batch touched, in order; its flush sends or queues them. */ + #touched: Hold[] = []; + /** Holds whose ops wait in their pending queues for a slot. */ + readonly #waiting = new Set(); + /** Holds waiting for a slot to send their first binding. */ + readonly #binding = new Set(); + /** The waiting bindings take the next slot: the last one went to waiting state while they waited. */ + #bindingTurn = false; + /** Set once a hold of this origin is relayed. */ + #relay: Relay | undefined; + + constructor(link: HostLink, origin: number, limit: number) { + this.link = link; + this.origin = origin; + this.#limit = limit; + } + + /** Handles one message of this origin; a call whose arguments wait for an asynchronous inflate returns its promise. */ + message(message: ProviderMessage): void | Promise { + switch (message.kind) { + case "hold": + this.#hold(message); + break; + case "release": + this.#release(message.hold); + break; + case "call": + return this.#call(message); + case "abort": + this.#abort(message.call); + break; + case "retry": + this.#entries.get(message.hold)?.retry(); + break; + case "ack": + this.#ack(); + break; + } + } + + /** + * The link closed, or the origin did (`closed`): every hold is released (relayed ones by one `closed` to each + * upstream link) and every call aborted. + */ + closed(): void { + const closing = this.link.host.closing; + const entries = [...this.#entries.values()]; + this.#entries.clear(); + const closed = new ChordError({ code: "unavailable", message: "the link closed" }); + for (const call of this.#calls.values()) if (call.kind === "served") call.abort.abort(closed); + this.#calls.clear(); + // While the host closes, it ends its objects itself. + for (const entry of entries) { + if (entry.kind === "served" && !closing) entry.release(); + else entry.drop(); + } + // A batch flushing later finds nothing of this side. + this.#touched = []; + const relay = this.#relay; + if (relay === undefined) return; + this.link.host.relays.delete(relay.origin); + for (const link of relay.links) link.send({ kind: "closed", origin: relay.origin }); + } + + /** Sends `message` on the link with this origin. */ + send(message: OriginMessage): void { + this.link.send(this.origin === 0 ? message : { ...message, origin: this.origin }); + } + + /** + * Ends with `error` every hold of this origin that acts for a principal `revoked` names and still waits: for `load` + * (served here), or for its route's link (relayed). + */ + revokeWaiting(revoked: (principal: Principal) => boolean, error: ChordError): void { + for (const hold of [...this.#entries.values()]) { + // A hold arriving on a link acts for a principal: the link's, or one its parent named. + if (!hold.waiting || !revoked(hold.caller.principal!)) continue; + if (hold.kind === "served") hold.end(error); + else hold.revoked(error); + } + } + + /** A pending hold turned out to be relayed: it takes the hold's place in the table. */ + relayed(hold: RelayedHold): void { + this.#entries.set(hold.id, hold); + } + + /** This origin's number on upstream links (one per host, never reused), with `link` as one it relays to. */ + relayTo(link: HostLink): number { + let relay = this.#relay; + if (relay === undefined) { + relay = { origin: this.link.host.relayOrigin(this), links: new Set() }; + this.#relay = relay; + } + relay.links.add(link); + return relay.origin; + } + + /** + * A message from upstream link `from` for this origin's relayed holds, forwarded with this origin: `bound` and + * `state` join the window's senders (their `ack` goes back to `from`), a reply ends its call, `gone` forgets its + * hold. A link answers only for what was relayed over it: an entry, a notice or a reply naming a hold (or a call) + * this origin did not relay over `from` is dropped (Q1); a `bound` or `state` left without entries is acknowledged + * to `from` at once. Parts travel unchecked and unparsed: the end that parses a part checks it against its own + * link's limits. + */ + forward(message: ConsumerMessage, from: HostLink): void { + const relay = this.#relay; + if (relay === undefined || !relay.links.has(from)) return; + const { origin: _, ...forwarded } = message; + switch (forwarded.kind) { + case "bound": + case "state": { + const holds = this.#relayedOver(forwarded.holds, from); + if (holds.length === 0) { + from.send({ kind: "ack", origin: relay.origin }); + return; + } + this.#sent.push(from); + // The entries of the message's own kind. + this.send(holds === forwarded.holds ? forwarded : ({ ...forwarded, holds } as typeof forwarded)); + return; + } + case "result": + case "error": { + const call = this.#calls.get(forwarded.call); + if (call?.kind === "relayed" && call.hold.heldOn(from)) this.reply(forwarded.call, forwarded); + return; + } + case "gone": { + const hold = this.#relayedHold(forwarded.hold, from); + if (hold === undefined) return; + hold.lost(); + this.send(forwarded); + return; + } + default: + if (this.#relayedHold(forwarded.hold, from) !== undefined) this.send(forwarded); + } + } + + /** An upstream link closed: nothing more is relayed to it, nor accepted from it. */ + unlinked(link: HostLink): void { + this.#relay?.links.delete(link); + } + + /** + * The upstream link of relayed hold `hold` dropped with `error`: its calls in flight fail, the hold goes loading + * downstream, and its path's next link holds it again. + */ + upstreamDropped(hold: RelayedHold, error: ChordError): void { + for (const [id, call] of this.#calls) { + if (call.hold === hold) this.reply(id, { kind: "error", call: id, error: wireError(error) }); + } + this.send({ kind: "loading", hold: hold.id }); + } + + /** + * Forgets a hold (released, refused or gone) and every call on it; local calls are aborted with `error` (its + * object's end for a gone hold, `released` otherwise). + */ + forget(hold: Hold | RelayedHold, error?: ChordError): void { + if (this.#entries.get(hold.id) === hold) this.#entries.delete(hold.id); + const reason = error ?? new ChordError({ code: "released", message: `hold ${hold.id}: released` }); + for (const [id, call] of this.#calls) { + if (call.hold !== hold) continue; + this.#calls.delete(id); + if (call.kind === "served") call.abort.abort(reason); + } + if (hold.kind === "served") this.unbind(hold); + } + + /** A call ended with `reply` (a result or an error): its slot is free; nothing is sent for one already gone. */ + reply(id: number, reply: Extract): void { + if (!this.#calls.delete(id)) return; + this.send(reply); + } + + /** A hold's object is live: its `bound` goes now when a slot is free, else with the next free slot's bindings. */ + bind(hold: Hold, context: Context): void { + if (this.#free) { + this.#sendOwn({ kind: "bound", holds: [hold.boundEntry(context)] }); + } else { + this.#binding.add(hold); + } + } + + /** A hold stopped waiting for a slot: it was forgotten. */ + unbind(hold: Hold): void { + this.#waiting.delete(hold); + this.#binding.delete(hold); + } + + /** A hold's members changed in the current batch; its flush sends or queues it. */ + touched(hold: Hold): void { + if (this.#touched.length === 0) this.link.host.batch.flushing((context) => this.#flush(context)); + this.#touched.push(hold); + } + + /** + * `entries` whose holds this origin relayed over `from`, in one pass: the same array when all are, else a copy made + * at the first one that is not. + */ + #relayedOver(entries: readonly E[], from: HostLink): readonly E[] { + let kept: E[] | undefined; + for (let i = 0; i < entries.length; i++) { + const entry = entries[i]!; + const relayed = this.#relayedHold(entry.hold, from) !== undefined; + if (kept !== undefined) { + if (relayed) kept.push(entry); + } else if (!relayed) { + kept = entries.slice(0, i); + } + } + return kept ?? entries; + } + + /** The hold `id` of this origin when it is relayed over `from`. */ + #relayedHold(id: number, from: HostLink): RelayedHold | undefined { + const hold = this.#entries.get(id); + return hold?.kind === "relayed" && hold.heldOn(from) ? hold : undefined; + } + + /** + * A hold message: a protocol error for a live id or compressed params (params are never compressed), otherwise + * admitted once `load` resolved (`Hold.start`). Its activation chain is kept only from a trusted link (Q5); its + * decoded limit is the link's, or the smaller one a parent stamped (Q2), and travels with it when it is relayed. + */ + #hold(message: Extract): void { + if (this.#entries.has(message.hold)) throw new Error(`hold ${message.hold}: the id is in use`); + if (message.params instanceof DeflatedPart) throw new Error(`hold ${message.hold}: compressed params`); + // Whom it acts for is decided here once (`#caller`), never passed on: a relay names it only on a route link. + const { + kind: _kind, + origin: _origin, + hold: id, + principal: _principal, + incarnation: _incarnation, + chain, + limit: stamped, + ...fields + } = message; + const { link } = this; + const limit = Math.min(link.limits.decoded, (link.role === "parent" ? stamped : undefined) ?? Infinity); + const request = { + ...fields, + ...(chain !== undefined && link.trusted ? { chain } : {}), + ...(limit === Infinity ? {} : { limit }), + }; + const limits = limit === link.limits.decoded ? link.limits : { part: link.limits.part, decoded: limit }; + const hold = new Hold(id, this, this.#caller(message), limits); + this.#entries.set(id, hold); + void hold.start(request); + } + + /** + * Whom a hold acts for: on a parent link the hold's own principal and incarnation when it names them, everywhere + * else the link's principal and the peer's incarnation (a child cannot claim to be someone else). + */ + #caller(message: Extract): Caller { + const { link } = this; + if (link.role !== "parent") return { principal: link.principal, incarnation: link.peer }; + // The message schema checked a principal's shape on an untrusted link; a trusted parent is believed. + const principal = (message.principal as Caller["principal"]) ?? link.principal; + return { principal, incarnation: message.incarnation ?? link.peer }; + } + + #release(id: number): void { + const hold = this.#entries.get(id); + if (hold === undefined) return; + this.forget(hold); + hold.release(); + } + + /** + * A call: one on a hold this side does not know is ignored (Q1), more than this side's N at once and compressed + * arguments from a child (clients never compress) are protocol errors. On a relayed hold it is forwarded, its + * arguments checked against this link's part limit first (the bytes are here). On a served hold that is not bound + * yet it is refused `not_ready`; its method is an index into the bound variant (one outside it is a protocol error) + * and its arguments are opened and parsed here within the hold's limits (past one the call fails `too_large/part`; a + * malformed part is a protocol error), absent ones `null`. Arguments this host must inflate asynchronously are + * inflated first, then the call is handled again. + */ + #call(message: Extract): void | Promise { + if (this.#calls.has(message.call)) throw new Error(`call ${message.call}: the id is in use`); + const hold = this.#entries.get(message.hold); + if (hold === undefined) return; + if (this.#calls.size >= this.#limit) throw new Error(`call ${message.call}: more than ${this.#limit} calls`); + const { args: part, segments } = message; + if (part instanceof DeflatedPart && this.link.role === "child") { + throw new Error(`call ${message.call}: compressed arguments from a child`); + } + if (hold.kind === "relayed") { + const error = part === undefined ? undefined : oversized(part, this.link.limits.part); + if (error !== undefined) { + this.#refuse(message.call, error); + return; + } + this.#calls.set(message.call, { kind: "relayed", hold }); + hold.call(message); + return; + } + if (!hold.bound) { + this.#refuse(message.call, new ChordError({ code: "not_ready", message: `hold ${hold.id}: not bound yet` })); + return; + } + const method = hold.method(message.method); + if (this.link.inflates(part, hold.limits)) { + return this.link.inflate([[part, hold.limits.decoded]], () => this.#call(message)); + } + let json: string | Uint8Array | undefined; + try { + json = part === undefined ? undefined : this.link.openPart(part, hold.limits, this.link.context); + } catch (error) { + if (!isChordError(error, "too_large")) throw error; + this.#refuse(message.call, error); + return; + } + const args = json === undefined ? null : parsePayload(json, segments); + const call = { kind: "served", hold, bytes: method.resultBytes, abort: new AbortController() } as const; + this.#calls.set(message.call, call); + void this.#run(message.call, call, method.name, args); + } + + /** + * Runs a call and sends its result (absent when `null`) or error, unless it was aborted, released or gone + * meanwhile. A result that cannot be encoded is sent as its error. + */ + async #run( + id: number, + call: Extract, + method: string, + args: unknown, + ): Promise { + let reply: Extract; + try { + // A link's context has no abort signal, so this is the call's own signal: no `AbortSignal.any` per call. + const result = await call.hold.call(method, args, withAbortSignal(call.abort.signal, this.link.context)); + // Results are not validated: they are the provider's. + reply = + result === null || result === undefined + ? { kind: "result", call: id } + : this.#result(id, result, call.bytes, this.link.context); + } catch (error) { + reply = { kind: "error", call: id, error: wireError(error) }; + } + this.reply(id, reply); + } + + /** Answers call `id` with `error` before it was admitted: it took no slot. */ + #refuse(id: number, error: ChordError): void { + this.send({ kind: "error", call: id, error: wireError(error) }); + } + + #abort(id: number): void { + const call = this.#calls.get(id); + if (call === undefined) return; + this.#calls.delete(id); + if (call.kind === "relayed") call.hold.abort(id); + else call.abort.abort(new ChordError({ code: "aborted", message: `call ${id}: aborted by its caller` })); + } + + /** + * An `ack` answers the oldest message in flight: this host's own frees a slot, a relayed one is acknowledged to its + * upstream link (closed meanwhile: nothing to do). An ack crossing nothing in flight is ignored. + */ + #ack(): void { + const sender = this.#sent.shift(); + if (sender === undefined) return; + if (sender !== "local") { + // A relayed message implies the relay's number. + sender.send({ kind: "ack", origin: this.#relay!.origin }); + return; + } + this.#inFlight--; + this.#slotFreed(); + } + + /** + * The end of a batch: while a slot is free and nothing waits, one `state` message with an entry per touched hold; + * otherwise each touched hold's ops join its pending queues until a slot frees. + */ + #flush(context: Context): void { + const holds = this.#touched; + // The side closed during the batch: nothing is sent. + if (holds.length === 0) return; + this.#touched = []; + if (this.#free) { + this.#sendOwn({ kind: "state", holds: holds.map((hold) => hold.entry(context)) }); + return; + } + // Only a bound hold is touched, and a bound hold never waits for its binding. + for (const hold of holds) { + hold.queue(); + this.#waiting.add(hold); + } + } + + /** Whether this host's next message goes out now: a slot is free and nothing waits for one. */ + get #free(): boolean { + return this.#binding.size === 0 && this.#waiting.size === 0 && this.#inFlight < WINDOW; + } + + /** + * A slot may be free: while there is room, the waiting state goes out as one message and the waiting bindings as + * one `bound`, taking turns when both wait (state first), so neither a hold that keeps changing nor a stream of new + * holds starves the other. + */ + #slotFreed(): void { + const context = this.link.context; + while (this.#inFlight < WINDOW) { + if (this.#binding.size > 0 && (this.#waiting.size === 0 || this.#bindingTurn)) { + const holds = [...this.#binding]; + this.#binding.clear(); + this.#bindingTurn = false; + // Every snapshot in one synchronous step: the binding shows all of them at one moment. + this.#sendOwn({ kind: "bound", holds: holds.map((hold) => hold.boundEntry(context)) }); + } else if (this.#waiting.size > 0) { + const holds = [...this.#waiting]; + this.#waiting.clear(); + this.#bindingTurn = this.#binding.size > 0; + this.#sendOwn({ kind: "state", holds: holds.map((hold) => hold.pendingEntry(context)) }); + } else { + return; + } + } + } + + /** + * A call's result message, its part deflated above the threshold when this host compresses; with byte segments when + * its method's result schema has a `bytes()` leaf. + */ + #result( + id: number, + result: unknown, + bytes: boolean, + context: Context, + ): Extract { + const { json, segments } = payloadPart(result, bytes); + const value = this.link.part(json, context); + return { kind: "result", call: id, value, ...(segments === undefined ? {} : { segments }) }; + } + + /** Sends one of this host's own `bound` or `state` messages, taking a slot. */ + #sendOwn(message: Extract): void { + this.#inFlight++; + this.#sent.push("local"); + this.send(message); + } +} diff --git a/packages/chord/src/next/remote/relay.ts b/packages/chord/src/next/remote/relay.ts new file mode 100644 index 000000000..b8857992c --- /dev/null +++ b/packages/chord/src/next/remote/relay.ts @@ -0,0 +1,130 @@ +// Relayed holds (§4, §22, R1; decision "Relay flow control per origin"): a hold that arrived on one link (downstream) +// and that this host neither serves nor parses, held again on its path's link (upstream) with the same id under its +// downstream origin's number there; messages are forwarded between the two with the origin rewritten and parts +// untouched. + +import { ChordError } from "../data/index.ts"; +import type { Caller } from "../host/internal.ts"; +import { type Message, wireError } from "../link/internal.ts"; +import { actingFor, type HoldRequest } from "./hold.ts"; +import type { HostLink } from "./host-link.ts"; +import type { Path } from "./path.ts"; +import type { ProviderSide } from "./provider.ts"; + +/** + * One relayed hold: `id` on its downstream origin, and the same id upstream on its path's current link (none while + * the path has no link). It keeps its `hold` message to hold again on the next link. Calls, `abort`, `retry` and + * `release` go upstream at once; what comes back is forwarded by its downstream origin (`ProviderSide.forward`). Two + * downstream holds are two upstream holds, each with its own encoders (R1). + */ +export class RelayedHold { + readonly kind = "relayed"; + readonly id: number; + readonly #downstream: ProviderSide; + /** + * What the upstream hold asks for, with the chain and decoded limit this host honoured; its params owned, not a view + * of the message they came in. + */ + readonly #request: HoldRequest; + /** Whom the hold acts for, sent on route links. */ + readonly caller: Caller; + readonly #path: Path; + /** The upstream link while the path has one, and the downstream origin's number on it. */ + #upstream: { readonly link: HostLink; readonly origin: number } | undefined; + + constructor(id: number, downstream: ProviderSide, request: HoldRequest, caller: Caller, path: Path) { + this.id = id; + this.#downstream = downstream; + const { params } = request; + // Params are never deflated: the provider side refused deflated ones. + const owned = typeof params === "string" ? params : (params as Uint8Array).slice(); + this.#request = { ...request, params: owned }; + this.caller = caller; + this.#path = path; + } + + /** Takes the pending hold's place downstream and holds upstream (now, or when the path has a link). */ + start(): void { + this.#downstream.relayed(this); + this.#path.attach(this); + } + + /** Whether it is held on `link` now: only that link answers for it. */ + heldOn(link: HostLink): boolean { + return this.#upstream?.link === link; + } + + /** Holds on `link`, the path's link now: its kept `hold` message, with whom it acts for on a route link. */ + attach(link: HostLink): void { + const origin = this.#downstream.relayTo(link); + this.#upstream = { link, origin }; + link.send({ kind: "hold", origin, hold: this.id, ...this.#request, ...actingFor(link, this.caller) }); + } + + /** The upstream link dropped: downstream its calls fail and the hold goes loading until the path's next link. */ + dropped(error: ChordError): void { + const upstream = this.#upstream; + this.#upstream = undefined; + if (upstream !== undefined) this.#downstream.upstreamDropped(this, error); + } + + /** + * Whether it waits for its path's link (a route still resolving, or a dropped link's next): a revocation ends it then, + * since the link it is held on next gets no past `revoke` (§4). + */ + get waiting(): boolean { + return this.#upstream === undefined; + } + + /** Revoked while it waited for its path's link: forgotten, and `gone` with `error` downstream. */ + revoked(error: ChordError): void { + this.drop(); + this.#downstream.forget(this); + this.#downstream.send({ kind: "gone", hold: this.id, error: wireError(error) }); + } + + /** `gone` from upstream: the hold is forgotten (the downstream origin forwards the message). */ + lost(): void { + this.drop(); + this.#downstream.forget(this); + } + + /** + * A call from downstream (registered there under its id), forwarded upstream at once; its reply comes back by the + * origin. Without an upstream link it is answered `unavailable` at once (it crossed the `loading` notice). + */ + call(message: Extract): void { + const upstream = this.#upstream; + if (upstream !== undefined) { + upstream.link.send({ ...message, origin: upstream.origin }); + return; + } + const error = new ChordError({ code: "unavailable", message: `hold ${this.id}: its upstream link is down` }); + this.#downstream.reply(message.call, { kind: "error", call: message.call, error: wireError(error) }); + } + + /** A downstream `abort` of call `id`, forwarded. */ + abort(id: number): void { + const upstream = this.#upstream; + upstream?.link.send({ kind: "abort", origin: upstream.origin, call: id }); + } + + /** A downstream `retry`, forwarded. */ + retry(): void { + const upstream = this.#upstream; + upstream?.link.send({ kind: "retry", origin: upstream.origin, hold: this.id }); + } + + /** Released downstream: released upstream (its calls with it) and no later link holds it. */ + release(): void { + const upstream = this.#upstream; + this.drop(); + upstream?.link.send({ kind: "release", origin: upstream.origin, hold: this.id }); + } + + /** No later link holds it (gone, or its downstream origin closed, which `closed` tells the upstream links). */ + drop(): void { + this.#upstream = undefined; + this.#path.detach(this); + } +} diff --git a/packages/chord/src/next/remote/remote-object.ts b/packages/chord/src/next/remote/remote-object.ts new file mode 100644 index 000000000..450c051e1 --- /dev/null +++ b/packages/chord/src/next/remote/remote-object.ts @@ -0,0 +1,428 @@ +// An object held over a link (§7): what one remote reference is bound to on the consumer, across the links of its path +// (the parent, or a route address). Its state, the replicas of its last bound variant (kept while its link is down), +// its hold on the path's current link and calls through it. + +import type { Context } from "../../context/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { Contract, ParamsOf } from "../contracts/index.ts"; +import { + currentVariants, + type MethodData, + type VariantNames, + variantNames, + variantOrder, +} from "../contracts/internal.ts"; +import { ChordError } from "../data/index.ts"; +import { asChordError, invalid } from "../data/internal.ts"; +import { + type AnyHost, + type Binding, + type Caller, + type CheckedParams, + type ChordObject, + LIVE, + LOADING, + type PinnedKey, + type RefState, + settlement, + type Target, +} from "../host/internal.ts"; +import { + type BoundEntry, + errorFromWire, + jsonPart, + type Message, + parsePart, + type StateEntry, + type WireAddress, +} from "../link/internal.ts"; +import type { DeriveSource } from "../state/internal.ts"; +import type { ConsumerSide } from "./consumer.ts"; +import type { HoldRequest } from "./hold.ts"; +import type { HostLink } from "./host-link.ts"; +import type { Path } from "./path.ts"; +import { type MemberSnapshot, Replica } from "./replica.ts"; + +/** The last binding: its variant, methods, the bound version's names (the wire's indexes) and a replica per member. */ +type Bound = { + readonly variant: string; + readonly methods: ReadonlyMap; + readonly names: VariantNames; + readonly replicas: ReadonlyMap; +}; + +/** + * One remote reference's object, owned by its host across the links of its path. It is `loading` until a link's hold + * of it is bound, and again whenever that link drops or its relay's upstream starts over; each link holds it with its + * own id. Two references are two holds, each member of each with its own Delta decoder. + */ +export class RemoteObject implements Target { + readonly kind = "remote"; + /** The object in errors: its type, and its params key when it has params. */ + readonly label: string; + readonly contract: Contract; + readonly #host: AnyHost; + /** What its holds ask for: the params as a part (checked by L1 when the reference was made), the address. */ + readonly request: HoldRequest; + /** Whom its reference acts for: named on route links (`actingFor`). */ + readonly caller: Caller; + /** Its params, for placing it anew (`retry` after a part past the limits). */ + readonly #params: ParamsOf; + /** Its path: the parent's or a route address's; the host's `unplaced` one after a part past the limits. */ + #path: Path; + /** The object whose activation declared the reference (`self.get`): its holds carry its chain while it activates. */ + #dependent: ChordObject | undefined; + #state: RefState = LOADING; + /** The last binding, kept while loading or failed: members keep their last values and watches. */ + #bound: Bound | undefined; + /** The hold on the path's link while one is open: that link's consumer side and this hold's id on it. */ + #hold: { readonly side: ConsumerSide; readonly id: number } | undefined; + #binding: Binding | undefined; + /** Pending while loading, resolved while live, rejected while failed: a dependent's activation waits for it (R18). */ + #settled = settlement(); + #lastFailure: ChordError | undefined; + + constructor( + host: AnyHost, + contract: Contract, + { params, key }: CheckedParams, + address: WireAddress | undefined, + caller: Caller, + path: Path, + dependent: ChordObject | undefined, + ) { + this.#host = host; + this.contract = contract; + this.label = key === "" ? contract.name : `${contract.name}(${key})`; + const { name: type, version, fingerprint } = contract; + this.request = { + type, + version, + fingerprint, + params: jsonPart(params), + ...(address === undefined ? {} : { address }), + }; + this.caller = caller; + this.#params = params; + this.#path = path; + this.#dependent = dependent; + } + + get state(): RefState { + return this.#state; + } + + /** Pending while loading; resolves when bound, rejects with a failure or the reference's end. */ + get settled(): Promise { + return this.#settled.promise; + } + + /** The error of the last `failed`, until `bound`. */ + get lastFailure(): ChordError | undefined { + return this.#lastFailure; + } + + /** The variant last bound; `undefined` before the first binding. */ + get variant(): string | undefined { + return this.#bound?.variant; + } + + /** Whether its state members have values: bound once. */ + get hydrated(): boolean { + return this.#bound !== undefined; + } + + /** Its one reference bound to it. */ + hold(binding: Binding): void { + this.#binding = binding; + } + + /** The reference was released: the current link's provider is told and the reference's sent calls rejected. */ + unhold(_binding: Binding): void { + const error = new ChordError({ code: "released", message: `${this.label}: released` }); + this.#hold?.side.release(this.#hold.id, error); + this.#end(error); + } + + /** + * Holds on `link`, the path's link now (a first hold, or a re-hold on a new link), with the activation chain of its + * dependent while that activates (Q5). + */ + attach(link: HostLink): void { + const chain = this.#dependent?.activationChain(); + this.#hold = { side: link.consumer, id: link.consumer.attach(this, chain) }; + } + + /** The link dropped with `error`: loading until a new link's hold is bound (`#startOver`). */ + dropped(error: ChordError): void { + this.#hold = undefined; + this.#startOver(error); + } + + /** + * Calls a method of the bound variant over the link; one outside it fails `invalid/member`. Only a live object is + * called: bound on an open link. + */ + async call(method: string, args: unknown, context: Context): Promise { + const bound = this.#bound!; + const hold = this.#hold!; + const data = bound.methods.get(method); + if (data === undefined) { + throw invalid("member", `${this.label} (variant ${bound.variant}) has no method "${method}"`); + } + // Arguments are checked by the provider. + return hold.side.call(hold.id, bound.names.methods.indexOf(method), data, args, context); + } + + /** + * The replica of `member`; `not_ready` before the first binding, `invalid/member` for a member no binding had (one + * a later variant lacks reads its `invalid/member` failure). + */ + member(member: string): DeriveSource { + const bound = this.#bound; + if (bound === undefined) throw new ChordError({ code: "not_ready", message: `${this.label}: not bound yet` }); + const replica = bound.replicas.get(member); + if (replica === undefined) { + throw invalid("member", `${this.label} (variant ${bound.variant}) has no state member "${member}"`); + } + return replica; + } + + /** A failed remote object's members are not read: its failure is the reads' error (see `Target.lastValues`). */ + get lastValues(): false { + return false; + } + + /** Keyed members are only in local contracts (`keyedMember`). */ + async pinKey(member: string): Promise { + return this.keyedMember(member); + } + + /** Keyed members are only in local contracts, whose objects are never held over a link (slice 6a §12). */ + keyedMember(member: string): never { + throw invalid("member", `${this.label} has no keyed member "${member}": keyed members are local`); + } + + /** An object over a link activates on another host: it is in no chain here (Q5). */ + get activationId(): undefined { + return undefined; + } + + /** An object over a link waits for nothing on this host: no chain of waits (its holds carry the chain, Q5). */ + pathTo(): undefined { + return undefined; + } + + /** + * `ref.retry()`: a failed object's provider is asked for an attempt; one that failed for a part past the link's + * limits is placed anew and held with a new id, now or on its path's next link (a placement that throws fails it + * broken again); ignored otherwise and without a link. + */ + retry(): void { + if (this.#state.kind !== "failed") return; + if (this.#path !== this.#host.unplaced) { + this.#hold?.side.retry(this.#hold.id); + return; + } + let path: Path; + try { + path = this.#host.pathOf(this.contract, this.#params, this.caller.principal); + } catch (error) { + this.#fail(asChordError(error, this.contract.name)); + return; + } + this.#path.detach(this); + this.#path = path; + this.#move(LOADING); + path.attach(this); + } + + /** + * A part of its snapshot or of a state change was past the link's limits (`too_large/part`): the hold is released + * at the provider, and the reference fails broken with `error`, final: no link holds it again (a reconnect + * included) until `retry`. + */ + tooLarge(error: ChordError): void { + const hold = this.#hold; + if (hold === undefined) return; + this.#hold = undefined; + hold.side.release(hold.id, error); + this.#path.detach(this); + this.#path = this.#host.unplaced; + this.#path.attach(this); + this.#fail(error); + } + + /** Fails broken with `error`: no automatic retry. */ + #fail(error: ChordError): void { + this.#lastFailure = error; + this.#move({ kind: "failed", cause: { kind: "broken", error }, retryAt: undefined }); + } + + /** + * Prepares one entry of a `bound` message (a first binding, a re-hold; a live hold is bound again): the + * variant and every state member once, as a value or as failed. Returns what applies it inside the batch of the + * whole message: a member it had gets the snapshot as one frame `[["r", value]]` and a fresh decoder, a member + * outside the new variant fails (its replica stays, for a later binding that has it again), a new one gets its first + * value. Throws on a snapshot that + * does not fit (a protocol error), and `too_large/part` for a value past the link's limits, before anything changed. + * `wentLive` follows the batch. + */ + prepareBound(entry: BoundEntry, link: HostLink): () => void { + const variant = variantOrder(this.contract)[entry.variant]; + if (variant === undefined) throw new Error(`hold ${entry.hold}: no variant ${entry.variant}`); + const data = currentVariants(this.contract).get(variant)!; + const names = variantNames(data); + const members = new Map(); + const add = (index: number, state: MemberSnapshot): void => { + const member = names.state[index]; + if (member === undefined || members.has(member)) + throw new Error(`hold ${entry.hold}: member ${index} in bound`); + members.set(member, state); + }; + // Fresh JSON from the link: state is JSON and never validated (§30). + for (const { member, value } of entry.members) { + add(member, { kind: "value", value: parsePart(link.openPart(value, link.limits, link.context)) as JsonValue }); + } + for (const { member, error } of entry.failed) add(member, { kind: "failed", error: errorFromWire(error) }); + if (members.size !== names.state.length) throw new Error(`hold ${entry.hold}: the snapshot misses a member`); + return () => { + const previous = this.#bound?.replicas; + const replicas = new Map(); + // First: a derive recomputing in this batch reads the new variant through its reference. + this.#bound = { variant, methods: data.methods, names, replicas }; + for (const [member, state] of members) { + const replica = previous?.get(member); + if (replica === undefined) { + replicas.set(member, new Replica(this.#host.batch, state)); + } else { + replica.reset(state); + replicas.set(member, replica); + } + } + for (const [member, replica] of previous ?? []) { + if (members.has(member)) continue; + replica.vanish(invalid("member", `${this.label} (variant ${variant}) has no state member "${member}"`)); + replicas.set(member, replica); + } + }; + } + + /** After a `bound` message's batch: the reference goes live (its parked calls are sent) and R18 retries follow. */ + wentLive(): void { + this.#lastFailure = undefined; + this.#move(LIVE); + // R18: a dependent that failed while waiting for this object is retried now. + this.#binding?.dependent?.dependencyLive(); + } + + /** + * `failed`: the provider's object failed and no attempt runs; `retryIn` is the rest of its backoff window. Ignored + * once live (only a pending hold is told). + */ + failed(message: Extract): void { + if (this.#state.kind === "live") return; + // A plugin's error a provider failed with is this object's failure, as a ChordError. + const error = asChordError(errorFromWire(message.cause.error), this.label); + this.#lastFailure = error; + const retryAt = message.retryIn === undefined ? undefined : Date.now() + message.retryIn; + this.#move({ kind: "failed", cause: { kind: message.cause.kind, error }, retryAt }); + } + + /** + * `loading`: the provider's object started a new attempt after a failure, or its relay's upstream link dropped + * (`#startOver`). + */ + loading(): void { + this.#startOver( + new ChordError({ code: "unavailable", message: `${this.label}: its relay's upstream link dropped` }), + ); + } + + /** + * Decodes and applies one entry of a state message (its members by index, each with its ops), without changing + * anything yet: returns what adopts it into the replicas, run inside the batch that applies the whole message. + * Throws on an entry that does not fit (a protocol error), and `too_large/part` for ops past the link's limits. + */ + prepare({ hold: id, members, failed, retired }: StateEntry, link: HostLink): () => void { + const bound = this.#bound; + if (this.#state.kind !== "live" || bound === undefined) throw new Error(`hold ${id}: state before bound`); + const replica = (index: number): Replica => { + const name = bound.names.state[index]; + const found = name === undefined ? undefined : bound.replicas.get(name); + if (found === undefined) throw new Error(`hold ${id}: no state member ${index}`); + return found; + }; + // Made by the first retirement (rare): no allocation per message otherwise. + let retirements: Set | undefined; + for (const member of retired) { + const target = replica(member); + retirements ??= new Set(); + if (retirements.has(target)) throw new Error(`hold ${id}: member ${member} retired twice in one state`); + retirements.add(target); + } + const changed = new Set(); + const updates = members.map(({ member, ops }) => { + const target = replica(member); + if (changed.has(target)) throw new Error(`hold ${id}: member ${member} twice in one state`); + changed.add(target); + return { target, ...target.decode(parsePart(link.openPart(ops, link.limits, link.context))) }; + }); + const failures = failed.map(({ member, error }) => ({ target: replica(member), error: errorFromWire(error) })); + return () => { + for (const { target, value, ops } of updates) target.adopt(value, ops, retirements?.has(target) === true); + for (const { target, error } of failures) target.fail(error); + // A retirement without ops (a derive whose value stayed, rule 21): the flag alone, after the failures (a + // failed member keeps it for its next value). + if (retirements !== undefined) { + for (const target of retirements) if (!changed.has(target)) target.retireAlone(); + } + }; + } + + /** `gone`, or the host closes: the reference ends, its calls reject with `error`. */ + lost(error: ChordError): void { + const binding = this.#binding; + this.#hold?.side.forget(this.#hold.id, error); + this.#end(error); + binding?.end(error); + } + + /** Ends the object: detached from its path, waits rejected, replicas ended; keeps nothing of its reference. */ + #end(error: ChordError): void { + const bound = this.#bound; + this.#state = { kind: "ended", error }; + this.#binding = undefined; + this.#dependent = undefined; + this.#hold = undefined; + this.#bound = undefined; + this.#settled.reject(error); + this.#path.detach(this); + for (const replica of bound?.replicas.values() ?? []) replica.end(error); + } + + /** + * Loading again until the next binding; the members keep their last values. A dependent's activation waiting for + * this hold while it was loading already fails at once with `error` (transient) and is retried when a new hold is + * bound (R18). + */ + #startOver(error: ChordError): void { + if (this.#state.kind === "ended") return; + if (this.#state === LOADING) { + this.#settled.reject(error); + this.#settled = settlement(); + } else { + this.#move(LOADING); + } + } + + /** Moves to `state` and tells the reference; `settled` follows (a new wait each time it is loading again). */ + #move(state: RefState): void { + const previous = this.#state; + this.#state = state; + if (state.kind === "loading" && previous.kind !== "loading") this.#settled = settlement(); + if (state.kind === "live") this.#settled.resolve(); + if (state.kind === "failed") this.#settled.reject(state.cause.error); + this.#binding?.moved(); + } +} diff --git a/packages/chord/src/next/remote/replica.ts b/packages/chord/src/next/remote/replica.ts new file mode 100644 index 000000000..e1c8ce7a3 --- /dev/null +++ b/packages/chord/src/next/remote/replica.ts @@ -0,0 +1,141 @@ +// Replicas (§7, §22): a state member of a remote reference on the consumer, built from the snapshot and then changed +// by the provider's ops, decoded with the member's own Delta decoder and applied without validation or copy +// (`applyImmutable` shares structure); replaced by the snapshot of each re-hold. + +import type { Context } from "../../context/index.ts"; +import { applyImmutable, decoder, type Op } from "../../delta/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { ChordError } from "../data/index.ts"; +import { type Batch, handedOut, Source, type WatchStart } from "../state/internal.ts"; + +/** + * Where a replica stands: a value, a failed member (reads throw; the provider's member failed, or a newly bound variant + * lacks it, and either may come back; `last` is its last good value, `undefined` when it had none), or ended with its + * reference. + */ +type ReplicaState = + | { readonly kind: "value"; readonly value: JsonValue } + | { readonly kind: "failed"; readonly error: unknown; readonly last: JsonValue | undefined } + | { readonly kind: "ended"; readonly error: ChordError }; + +/** A member as a snapshot has it: a value, or failed. */ +export type MemberSnapshot = + | { readonly kind: "value"; readonly value: JsonValue } + | { readonly kind: "failed"; readonly error: unknown }; + +const NO_OPS: readonly Op[] = []; + +/** One state member of a hold, as the consumer reads and watches it; changed only inside the consumer's batch. */ +export class Replica extends Source { + #state: ReplicaState; + /** The member's decoder, started with the snapshot as the provider's encoder was; a new one per snapshot. */ + #decoder = decoder(); + /** The ops of the current batch's frame, and whether the provider's document was retired in it. */ + #ops = NO_OPS; + #retired = false; + /** A retirement that reached it while failed: its next value carries the flag. */ + #retiredWhileFailed = false; + + constructor(batch: Batch, state: MemberSnapshot) { + super(batch); + this.#state = state.kind === "value" ? state : { kind: "failed", error: handedOut(state.error), last: undefined }; + } + + get value(): JsonValue { + const state = this.#state; + if (state.kind === "value") return state.value; + throw state.error; + } + + /** Its value, or while failed its failure and last good value. */ + start(): WatchStart { + const state = this.#state; + if (state.kind === "failed") return { kind: "failed", error: state.error, last: state.last }; + return { kind: "value", value: this.value }; + } + + /** + * The value `wire` (this member's wire ops from one state part) makes, and its ops; changes nothing yet. Throws on + * ops that do not fit (a protocol error): an ended member's, and a failed member's other than its recovery, which + * the provider sends as `[["r", value]]`. + */ + decode(wire: unknown): { readonly value: JsonValue; readonly ops: readonly Op[] } { + // Trust boundary: fresh JSON from the link; Delta's decoder checks each op. + if (!Array.isArray(wire)) throw new Error("part: a member's changes are a list of ops"); + const ops = this.#decoder.decode(wire); + const state = this.#state; + if (state.kind === "ended") throw state.error; + if (state.kind === "value") return { value: applyImmutable(state.value, ops), ops }; + if (ops[0]?.[0] !== "r") throw new Error("part: a failed member recovers with a root replacement"); + return { value: applyImmutable(null, ops), ops }; + } + + /** + * Adopts the member's new value, the ops that made it and whether the provider's document was retired, in the + * current batch (one adoption per member and batch). + */ + adopt(value: JsonValue, ops: readonly Op[], retired: boolean): void { + const flagged = retired || this.#retiredWhileFailed; + this.#state = { kind: "value", value }; + this.#ops = ops; + this.#retired = flagged; + this.#retiredWhileFailed = false; + this.changed(); + if (flagged) this.retiredInBatch(); + } + + /** + * The provider's document was retired with no ops for this member (a derive whose value stayed, rule 21), in the + * current batch: the flag alone; while failed it waits for the next value (a failed member's value is never read). + */ + retireAlone(): void { + const state = this.#state; + if (state.kind === "value") this.adopt(state.value, NO_OPS, true); + else if (state.kind === "failed") this.#retiredWhileFailed = true; + } + + /** + * A re-hold's snapshot of the member, in the batch that applies it, with a fresh decoder (the provider's encoder + * starts anew with it): a value is one frame `[["r", value]]`, a failed member fails. + */ + reset(state: MemberSnapshot): void { + this.#decoder = decoder(); + if (state.kind === "failed") this.fail(state.error); + else this.adopt(state.value, [["r", state.value]], false); + } + + /** + * The provider's member failed, in the batch that applies it: reads throw `error`, watches get one failure call and + * stay open, derives over it fail; its recovery is `[["r", value]]`. + */ + fail(error: unknown): void { + const state = this.#state; + const last = state.kind === "value" ? state.value : state.kind === "failed" ? state.last : undefined; + this.#state = { kind: "failed", error: handedOut(error), last }; + // Failed already: its watches and derives have the failure (one call per failure episode). + if (state.kind !== "failed") this.changed(); + } + + deliver(context: Context): void { + const state = this.#state; + if (state.kind === "failed") this.failWatches(state.error, context); + else if (state.kind === "value") this.publish(state.value, this.#ops, this.#retired, context); + else this.endWatches(); + this.#ops = NO_OPS; + this.#retired = false; + } + + /** + * The member is not in a newly bound variant, in the batch that binds it: as a failure (`fail`) with `error`; a later + * binding of a variant that has it brings it back (`reset`), as on the provider's host. + */ + vanish(error: ChordError): void { + this.fail(error); + } + + /** Its reference ended (outside a batch): reads throw `error`, watches end `ended`. */ + end(error: ChordError): void { + this.#state = { kind: "ended", error }; + this.endWatches(); + } +} diff --git a/packages/chord/src/next/state/README.md b/packages/chord/src/next/state/README.md new file mode 100644 index 000000000..b386d167b --- /dev/null +++ b/packages/chord/src/next/state/README.md @@ -0,0 +1,70 @@ +# state + +Watchables, derives and the batch: how values change, are computed from each other and reach their watches. + +## For apps (`index.ts`) + +```ts +import { constant, mapShared } from "./index.ts"; + +const title = mapShared((note: Note) => note.text.slice(0, 20)); // called once per new note object +const titles = self.derive([notes], ([n]) => title(n.list)); +const info = constant({ dataDir: "/data" }); // a watchable that never changes +const watch = await titles.watch(ctx); +watch.start(async (call, ctx) => { + if (call.kind === "failed") return showError(call.error); // the watch stays open + render(call.value); +}); +``` + +- `Watchable` (from `src/watch.ts`): `value` and `watch(ctx)`, which returns Chord's `WatchHandle`: the value at + acquisition, then one call `(call, ctx)` per batch that changed it, in order, while the listener keeps up; batches + that arrive while it is busy fold into its next call (the newest value, the ops since its previous call, folded by + Delta's `PendingOps`). A call is `{ kind: "value", value, ops, retired }` or `{ kind: "failed", error }` (the source + cannot compute its value now: the watch stays open, `watch.value` keeps the last value, the next value arrives as + `[["r", value]]`). Values are JSON. A watch ends only for good (its owner ended, `stop()`, an aborted context, a + throwing listener). +- `constant(value)`: a watchable that never changes; a state member or a derive source. +- `mapShared(f)`: maps an array, calling `f` once per object element (by identity) and reusing its result while the + element stays the same object, so a derive over a list keeps unchanged parts identical and its diff cheap. + +Rules: + +- `self.derive(sources, compute)`: a derive has no name; errors name it by the member that exposes it, or by its + position (`derive #2`). Its `compute` is synchronous and pure; it reads only its declared sources. A derive that + throws fails only itself and the derives over it: their reads throw, their watches get one `failed` call per failure + episode (a recompute that throws again is quiet) and stay open, and its owner reports `derive_failed` once per + episode (not while its attempt activates: `activation_failed` covers that). It recomputes on every later change of + its sources (and on a re-bind of a member it reads) and heals as soon as it computes a value: its watches get it as + `[["r", value]]`, with `retired` when a source's document was retired during the episode. +- A watch opened on a failed source starts at its last good value and gets the failure as its first call, then the + heal; a source that never had a value (a remote member failed since its binding) rejects `watch` with its error. An + `on` routine over a failed source waits for its next good value; a value queued behind its busy call is dropped + when a failure follows it. +- An unwatched derive computes only when read. Reading `.value` once does not make it watched. +- Inside a commit's adoptions and in any derive compute, `self.tx`, `watch`, `release`, `get`, `load` and `close` + throw `invalid/reentrant`. A raw `constant(...)` belongs to no host, so its own `watch` is not checked; through a + reference it is. +- A frame delivery that throws (a derive whose value is no JSON) is reported to `onError` as `delivery_failed`; the + commit's other frames are still delivered. A throwing watch listener is reported as `listener_error`. + +## For Chord's layers (`internal.ts`) + +- `Batch`, one per host, reporting `BatchError`s (`listener_error`, `delivery_failed`; watches get `reportListener`): + `commit(adopt, ctx)` runs a commit's adoptions, then delivers one frame per watched source they changed, then runs + the flushes listed with `flushing(flush)` during it (a link sends one state message per batch, W1); + `within(adopt, ctx)` adopts a change inside the commit running now, or as a commit of its own, or, while a commit's + flushes send, as the next commit (presence and keep-alive counts: a departure found while sending lands there); + `run(work)` sets the reentrancy flag (nested); `assertOutside(operation)`; `count(counter)` counts derive work on + the host's metrics (`chord.derive.computes`, `chord.derive.diffs`). +- `Source`: a changing value on a host (a singleton document, a derive, a replica) with its watches, subscribers and + dependents (`Dependent`: derives, and member sources passing changes on to theirs). Subclasses call `changed()` when + a commit changes them and implement `deliver(ctx)` (`publish` a value, `failWatches` a recoverable failure, + `endWatches` for good). `openWatch(ctx)` is `watch` without the Promise. `subscribe(subscriber)`: a `Subscriber` (a + hold over a link, a member source) takes the watch's own sink shape, `advance(call, ctx)` synchronously inside each + batch that changes the source (a value or a failure call) and `end(end)` when the source ends; a subscriber counts + as watching, so a derive it follows recomputes in the batch. +- `Derive`: declared with its raw sources; `attach(resolve, report)` resolves them at activation (and drops the raw + list; `report` gets a failure of its compute, once per episode), `detach(error)` when its object ends (later reads + throw `error` for good), `invalidate()` from its sources (a failed derive too: it recomputes on its next read). +- `Constant`: the class behind `constant()`. `DeriveSource` is what a derive reads: a `Source` or a `Constant`. diff --git a/packages/chord/src/next/state/batch.ts b/packages/chord/src/next/state/batch.ts new file mode 100644 index 000000000..d58c5c347 --- /dev/null +++ b/packages/chord/src/next/state/batch.ts @@ -0,0 +1,118 @@ +// The batch (W1), one per host: a commit adopts inside it, then every watched value that changed gets one frame, then +// the flushes it collected run (a link sends one message); Chord calls that would reenter it are refused while it runs. + +import type { TelemetryMetrics } from "@earendil-works/pi-telemetry"; +import type { Context } from "../../context/index.ts"; +import { invalid } from "../data/internal.ts"; +import type { Source } from "./watchable.ts"; + +/** What a batch reports: a watch listener that threw, or a frame delivery that threw (a derive's value is no JSON). */ +export type BatchError = + | { readonly kind: "listener_error"; readonly error: unknown } + | { readonly kind: "delivery_failed"; readonly error: unknown }; + +/** A host's batch: the reentrancy flag, the watched sources a commit changed, and where background errors go. */ +export class Batch { + /** Reports a throwing watch listener's error (to the host's `onError`); what watches get as their `report`. */ + readonly reportListener: (error: unknown) => void; + readonly #report: (error: BatchError) => void; + readonly #metrics: TelemetryMetrics | undefined; + /** Set while adopting, delivering or computing a derive; one flag per host. */ + #running = false; + /** The watched sources the current commit changed, each listed once by its `changed`. */ + #changed: Source[] = []; + /** What runs once the current commit's frames were delivered, each listed once by its `flushing`. */ + #flushes: ((context: Context) => void)[] = []; + /** Set while the current commit's flushes run (its links send). */ + #sending = false; + /** Changes `within` deferred while the flushes sent, for the next commit. */ + #later: (() => void)[] = []; + + constructor(report: (error: BatchError) => void, metrics: TelemetryMetrics | undefined) { + this.#report = report; + this.reportListener = (error) => report({ kind: "listener_error", error }); + this.#metrics = metrics; + } + + /** Throws `invalid/reentrant` inside the batch: `operation` may not run during adoptions or derive computes. */ + assertOutside(operation: string): void { + if (this.#running) throw invalid("reentrant", `${operation} inside a commit or a derive compute`); + } + + /** Runs `work` with reentrant calls refused; nests (a derive compute inside a commit). */ + run(work: () => T): T { + const outer = this.#running; + this.#running = true; + try { + return work(); + } finally { + this.#running = outer; + } + } + + /** + * Runs one commit's adoptions, then, still synchronously, delivers a frame for every watched value they changed + * (watched derives recompute through their reads), then runs the flushes the deliveries listed. A delivery or flush + * that throws is reported; the others still run. What `within` deferred while the flushes sent runs as the next + * commit, right after. + */ + commit(adopt: () => void, context: Context): void { + this.run(() => { + try { + adopt(); + for (const source of this.#changed) { + try { + source.deliver(context); + } catch (error) { + this.#report({ kind: "delivery_failed", error }); + } + } + this.#sending = true; + for (const flush of this.#flushes) { + try { + flush(context); + } catch (error) { + this.#report({ kind: "delivery_failed", error }); + } + } + } finally { + this.#sending = false; + this.#changed = []; + this.#flushes = []; + } + }); + const later = this.#later; + if (later.length === 0) return; + this.#later = []; + this.commit(() => { + for (const each of later) each(); + }, context); + } + + /** + * Runs `adopt` inside the commit running now, which delivers what it changed; outside one, as a commit of its own; + * while a commit's flushes send, as the next commit (a link closing while it sends ends its holds: their departure + * lands in the next batch, so one commit stays one message per link and origin). For changes that happen wherever a + * reference binds or ends (`host.references`, `host.activity`). + */ + within(adopt: () => void, context: Context): void { + if (this.#sending) this.#later.push(adopt); + else if (this.#running) adopt(); + else this.commit(adopt, context); + } + + /** Lists a watched source the current commit changed. */ + changed(source: Source): void { + this.#changed.push(source); + } + + /** Lists `flush` to run at the end of the current commit, after its deliveries (a link sending what they fed it). */ + flushing(flush: (context: Context) => void): void { + this.#flushes.push(flush); + } + + /** Counts one unit of derive work on the host's metrics (a lazy read has no context to count through). */ + count(counter: "chord.derive.computes" | "chord.derive.diffs"): void { + this.#metrics?.counter(counter).add(1); + } +} diff --git a/packages/chord/src/next/state/derive.ts b/packages/chord/src/next/state/derive.ts new file mode 100644 index 000000000..6bb86f617 --- /dev/null +++ b/packages/chord/src/next/state/derive.ts @@ -0,0 +1,191 @@ +// Derives (§12): values computed from watchables, pulled when read and marked dirty by commits; and `mapShared`, which +// keeps unchanged elements identical so a derive's diff stays cheap. + +import type { Context } from "../../context/index.ts"; +import { diffRevisions, type Op } from "../../delta/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { Batch } from "./batch.ts"; +import { type Constant, type Dependent, handedOut, Source, type WatchStart } from "./watchable.ts"; + +/** What a derive reads: a source it depends on, or a constant. */ +export type DeriveSource = Source | Constant; + +/** + * Where a derive stands. `dirty`: a source changed since the last compute (or there was none). A failed derive (its + * compute threw) throws its error until a source it reads changes or is re-bound, then recomputes (self-healing, design + * §11); a detached one (its object ended) throws its object's error for good. + */ +type DeriveState = + | { readonly kind: "dirty" } + | { readonly kind: "clean"; readonly value: JsonValue } + | { readonly kind: "failed"; readonly error: unknown } + | { readonly kind: "detached"; readonly error: unknown }; + +const DIRTY: DeriveState = { kind: "dirty" }; + +const NO_OPS: readonly Op[] = []; + +/** What a failed derive's watches had: no value, so its next value frame replaces the value whole. */ +const NO_VALUE: unique symbol = Symbol("no value"); + +/** + * A derive: `compute` over its sources' values, recomputed on read when dirty, so each compute runs at most once per + * batch and after its sources. A commit only marks it; the batch recomputes it only when something watched depends on + * it. + */ +export class Derive extends Source implements Dependent { + readonly #compute: (values: readonly JsonValue[]) => JsonValue; + /** The sources as declared, until `attach` resolves them. */ + #declared: readonly unknown[] | undefined; + #sources: readonly DeriveSource[] = []; + #state: DeriveState = { kind: "dirty" }; + /** The value this derive's watches have, kept from its marking to its frame in the batch. */ + #framed: JsonValue | typeof NO_VALUE | undefined; + /** Whether a source's document was retired in the current batch while this derive is watched (rule 21). */ + #retired = false; + /** Reports a failure of its compute to its owner (`attach`); once per failure episode, which a value ends. */ + #report: ((error: unknown) => void) | undefined; + #failing = false; + /** Its last good value, which a watch opened while it is failed starts at (`NO_VALUE` before the first). */ + #good: JsonValue | typeof NO_VALUE = NO_VALUE; + + constructor(sources: readonly unknown[], compute: (values: readonly JsonValue[]) => JsonValue, batch: Batch) { + super(batch); + this.#declared = sources; + this.#compute = compute; + } + + get value(): JsonValue { + const state = this.#state; + if (state.kind === "clean") return state.value; + if (state.kind === "dirty") return this.#recompute(); + throw state.error; + } + + /** Its value, or while failed its failure and last good value (a detached derive throws). */ + start(): WatchStart { + try { + return { kind: "value", value: this.value }; + } catch (error) { + if (this.#state.kind !== "failed") throw error; + return { kind: "failed", error, last: this.#good === NO_VALUE ? undefined : this.#good }; + } + } + + /** + * Starts reading its sources, as `resolve` maps the declared ones (at activation); the first read computes. + * `report` gets a throw of its compute, once per failure episode (a value ends it); a failing source's error is its + * source's to report. + */ + attach(resolve: (source: unknown, index: number) => DeriveSource, report: (error: unknown) => void): void { + this.#report = report; + this.#sources = this.#declared!.map(resolve); + this.#declared = undefined; + for (const source of this.#sources) if (source instanceof Source) source.dependents.add(this); + } + + /** Its object ended: no source keeps it, its watches end `ended`, reads throw `error` (the object's). */ + detach(error: Error): void { + for (const source of this.#sources) if (source instanceof Source) source.dependents.delete(this); + this.#sources = []; + this.#declared = undefined; + this.#state = { kind: "detached", error }; + this.#framed = undefined; + this.endWatches(); + } + + /** + * A source changed: mark this derive and its dependents dirty, flags only; a failed one too, which recomputes on its + * next read and heals when its sources give a value it can compute. An already dirty derive has dirty dependents + * (every read recomputes its sources first), so marking stops there. + */ + invalidate(): void { + const state = this.#state; + if (state.kind === "dirty" || state.kind === "detached") return; + if (this.watched) this.#framed = state.kind === "clean" ? state.value : NO_VALUE; + this.#state = DIRTY; + this.changed(); + } + + /** + * A source's document was retired in this batch (`Source.retiredInBatch`): a watched derive's frame carries the + * flag, and the derives reading this one are told too. + */ + sourceRetired(visited: Set): void { + // Once per derive per retirement: a diamond of derives is walked in its size, not once per path. + if (visited.has(this)) return; + visited.add(this); + if (this.watched) this.#retired = true; + for (const derive of this.dependents) derive.sourceRetired(visited); + } + + /** + * Delivers the frame: the diff from the watches' value, a retirement even without ops (rule 21). A compute that + * throws gives one failure call when the watches had a value (one per failure episode), the watches stay open; a + * retirement waits for the next value. + */ + deliver(context: Context): void { + const before = this.#framed!; + const retired = this.#retired; + this.#framed = undefined; + let after: JsonValue; + try { + after = this.value; + } catch (error) { + // Still failed since the last call: nothing new for the watches. + if (before !== NO_VALUE) this.failWatches(error, context); + return; + } + this.#retired = false; + let ops: readonly Op[] = NO_OPS; + if (after !== before) { + this.batch.count("chord.derive.diffs"); + ops = before === NO_VALUE ? [["r", after]] : diffRevisions(before, after); + } + if (ops.length > 0 || retired) this.publish(after, ops, retired, context); + } + + #recompute(): JsonValue { + return this.batch.run(() => { + try { + this.batch.count("chord.derive.computes"); + // A failing source throws its own error here, so a derive over a failed derive fails with the same error. + const values = this.#sources.map((source) => source.value); + let value: JsonValue; + try { + value = this.#compute(values); + } catch (error) { + if (!this.#failing) this.#report?.(handedOut(error)); + this.#failing = true; + throw error; + } + this.#failing = false; + this.#state = { kind: "clean", value }; + this.#good = value; + return value; + } catch (error) { + this.#state = { kind: "failed", error }; + throw error; + } + }); + } +} + +/** + * Maps an array calling `f` only for object elements not seen before (by identity), returning the previous result for + * the others; primitive elements are mapped every time. The cache is weak: dropped elements take their results along. + */ +export function mapShared(f: (element: E) => R): (elements: readonly E[]) => R[] { + const results = new WeakMap(); + return (elements) => + elements.map((element) => { + // The decided rule: identity caching for objects only. + if (typeof element !== "object" || element === null) return f(element); + let result = results.get(element); + if (result === undefined) { + result = f(element); + results.set(element, result); + } + return result; + }); +} diff --git a/packages/chord/src/next/state/index.ts b/packages/chord/src/next/state/index.ts new file mode 100644 index 000000000..8fbd9ad1e --- /dev/null +++ b/packages/chord/src/next/state/index.ts @@ -0,0 +1,5 @@ +// State for apps: watchables, constants, and the derive helper `mapShared`. + +export type { Watchable } from "../../watch.ts"; +export { mapShared } from "./derive.ts"; +export { constant } from "./watchable.ts"; diff --git a/packages/chord/src/next/state/internal.ts b/packages/chord/src/next/state/internal.ts new file mode 100644 index 000000000..5ecd3641f --- /dev/null +++ b/packages/chord/src/next/state/internal.ts @@ -0,0 +1,5 @@ +// State for Chord's layers: the batch, sources, derives and constants behind the app's watchables. + +export { Batch, type BatchError } from "./batch.ts"; +export { Derive, type DeriveSource } from "./derive.ts"; +export { Constant, type Dependent, handedOut, Source, type Subscriber, type WatchStart } from "./watchable.ts"; diff --git a/packages/chord/src/next/state/watchable.ts b/packages/chord/src/next/state/watchable.ts new file mode 100644 index 000000000..8bdbac622 --- /dev/null +++ b/packages/chord/src/next/state/watchable.ts @@ -0,0 +1,190 @@ +// Chord's JSON watchables: sources (documents and derives, with their watches and dependent derives) and constants. + +import type { Context } from "../../context/index.ts"; +import type { Op } from "../../delta/index.ts"; +import type { JsonValue } from "../../json.ts"; +import { Watch, type Watchable, type WatchCall, type WatchEnd } from "../../watch.ts"; +import type { Batch } from "./batch.ts"; + +/** + * A synchronous subscriber of a source (a hold over a link, a member source): fed the source's call inside the batch + * that made it, before the batch ends, so everything one commit changed can leave in one message. The watch's own sink + * shape (`Watch.advance`, `Watch.end`). + */ +export interface Subscriber { + /** + * The source's call of this batch, in its batch's `context`: a value (the ops from the previous one, and whether its + * document was retired in it, or for a derive one of its sources' documents), or a failure it may recover from; the + * next value after a failure is `[["r", value]]`. + */ + advance(call: WatchCall, context: Context): void; + /** The source ended for good with `end`; nothing follows, and the subscriber is dropped. */ + end(end: WatchEnd): void; +} + +/** + * What a new watch of a source starts at (design §11): its value, or, while it is failed, the failure and its last good + * value (`undefined` when it never had one: a watch cannot start then, there is nothing to start at). + */ +export type WatchStart = + | { readonly kind: "value"; readonly value: JsonValue } + | { readonly kind: "failed"; readonly error: unknown; readonly last: JsonValue | undefined }; + +/** + * An error Chord hands to apps (a failure call, a report), its stack made text first: until then V8 keeps the frames' + * receivers, which would keep the host behind the throw alive for as long as an app keeps the error. + */ +export function handedOut(error: E): E { + if (error instanceof Error) void error.stack; + return error; +} + +/** + * What reads a source and is marked by its changes: a derive, or a member source forwarding to derives (`Source. + * dependents`). + */ +export interface Dependent { + /** The source changed in the current batch: flags only, no compute. */ + invalidate(): void; + /** A source's document was retired in the current batch (rule 21); `visited` keeps a diamond walked once. */ + sourceRetired(visited: Set): void; +} + +const ENDED: WatchEnd = { kind: "ended" }; + +/** + * A changing value on a host (a singleton document, a derive, a replica): its watches and subscribers get one frame + * per batch that changed it, and the derives reading it are marked dirty when it changes. + */ +export abstract class Source implements Watchable { + /** Derives reading this source (and member sources forwarding to some); each removes itself when it stops. */ + readonly dependents = new Set(); + protected readonly batch: Batch; + readonly #watches = new Set>(); + readonly #subscribers = new Set(); + + constructor(batch: Batch) { + this.batch = batch; + } + + abstract get value(): JsonValue; + + /** Whether a watch or a subscriber follows this source. */ + get watched(): boolean { + return this.#watches.size > 0 || this.#subscribers.size > 0; + } + + async watch(context: Context): Promise> { + return this.openWatch(context); + } + + /** + * `watch` without the Promise, for references, which must register the watch before anything else runs. On a failed + * source it opens at the last good value and its first call is the failure; with no value to start at it throws the + * failure. + */ + openWatch(context: Context): Watch { + this.batch.assertOutside("watch"); + const start = this.start(); + if (start.kind === "failed" && start.last === undefined) throw start.error; + const value = start.kind === "value" ? start.value : start.last!; + const watch: Watch = new Watch(value, () => this.#watches.delete(watch), this.batch.reportListener); + this.#watches.add(watch); + if (context.abortSignal !== undefined) watch.observeAbort(context.abortSignal); + if (start.kind === "failed") watch.fail(start.error, context); + return watch; + } + + /** What a new watch starts at; a source that can fail and recover tells its failure (`WatchStart`). */ + start(): WatchStart { + return { kind: "value", value: this.value }; + } + + /** Feeds `subscriber` every frame from now on, inside each batch; the caller took the current value first. */ + subscribe(subscriber: Subscriber): void { + this.#subscribers.add(subscriber); + } + + unsubscribe(subscriber: Subscriber): void { + this.#subscribers.delete(subscriber); + } + + /** Delivers this batch's frame to its watches; called once per batch that listed it with `changed`. */ + abstract deliver(context: Context): void; + + /** Records a change in the current batch: listed for a frame when watched, and every dependent marked dirty. */ + protected changed(): void { + if (this.watched) this.batch.changed(this); + for (const derive of this.dependents) derive.invalidate(); + } + + /** + * Records a retirement of this source's document in the current batch: every derive reading it, directly or + * through other derives, carries the flag on in its frame (rule 21). Called after `changed`. + */ + protected retiredInBatch(): void { + // Allocated only for a retirement, which is rare. + const visited = new Set(); + for (const derive of this.dependents) derive.sourceRetired(visited); + } + + /** Queues one frame on every watch and feeds it to every subscriber. */ + protected publish(value: JsonValue, ops: readonly Op[], retired: boolean, context: Context): void { + this.forward({ kind: "value", value, ops, retired }, context); + } + + /** Queues `call` on every watch and feeds it to every subscriber, as it is (a member source passes one on). */ + protected forward(call: WatchCall, context: Context): void { + for (const watch of this.#watches) watch.advance(call, context); + for (const subscriber of this.#subscribers) subscriber.advance(call, context); + } + + /** + * Gives every watch and subscriber one failure call (design §11): the source cannot compute its value now and may + * recover; they stay open, and its next value is `[["r", value]]`. + */ + protected failWatches(error: unknown, context: Context): void { + this.forward({ kind: "failed", error: handedOut(error) }, context); + } + + /** Ends every watch and subscriber for good (its owner ended): `ended`; nothing follows. */ + protected endWatches(): void { + // Ending detaches, which deletes the current entry (safe while iterating) and runs no listener. + for (const watch of this.#watches) watch.end(ENDED); + if (this.#subscribers.size === 0) return; + // A copy: a subscriber may unsubscribe as it ends. + const subscribers = [...this.#subscribers]; + this.#subscribers.clear(); + for (const subscriber of subscribers) subscriber.end(ENDED); + } +} + +/** A watchable that never changes (start data, §18); usable as a state member and as a derive source. */ +export function constant(value: T): Watchable { + // The constant holds `value` as it was given. + return new Constant(value) as unknown as Watchable; +} + +/** The watchable behind `constant(value)`: its watches never get a frame and end only by stop or cancellation. */ +export class Constant implements Watchable { + readonly value: JsonValue; + + constructor(value: JsonValue) { + this.value = value; + } + + async watch(context: Context): Promise> { + return this.openWatch(context); + } + + /** As `Source.openWatch`; nothing is delivered, so nothing is reported. */ + openWatch(context: Context): Watch { + const watch = new Watch( + this.value, + () => {}, + () => {}, + ); + if (context.abortSignal !== undefined) watch.observeAbort(context.abortSignal); + return watch; + } +} diff --git a/packages/chord/src/next/store/README.md b/packages/chord/src/next/store/README.md new file mode 100644 index 000000000..933fb6df6 --- /dev/null +++ b/packages/chord/src/next/store/README.md @@ -0,0 +1,67 @@ +# store + +The store (slice 6a): one per host, opened by the app over a backend (SQLite: `chord/sqlite/node`, Durable Objects' +storage), holding the host's tables, persisted documents and outbox; one commit queue serializing only the commit +step; eager table migrations; export and import. + +## For apps (`index.ts`) + +```ts +import { defineTable, evolveTable, importStore, json, openStore, query, sqliteBackend } from "./index.ts"; + +const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), ctx, { outbox: true }); +const host = createHost({ store, onError }); // one open host per store; the app closes the store after it + +const EditsV1 = defineTable("edits", { + version: 1, + columns: { id: str(), path: str(), at: int(), done: bool({ default: false }), meta: opt(json(Meta, { version: 1 })) }, + key: ["id"], + indexes: { by_path: ["path", "at"] }, +}); +const EditsV2 = evolveTable(EditsV1, (t) => t.add("dir", str({ default: "" })).rows((row) => ({ dir: dirname(row.path) }))); + +host.serve(History, (_, self) => { // a local type with empty params (P3) + const edits = self.table(EditsV2); // created or migrated at activation + const q = query(EditsV2).where((e) => prefix(e.path, "src/")).orderBy((e) => e.at, "desc"); + return { + add: async (row, ctx) => self.tx(async (tx) => tx.put(edits, row), ctx), + list: async ({ after }, ctx) => edits.page(q, 50, ctx, { after }), + }; +}); + +for await (const part of store.export(ctx)) write(part); // or encodeJsonl(store.export(ctx)) +const copy = await importStore(sqliteBackend(db), decodeJsonl(bytes), ctx); +``` + +- `openStore(backend, ctx, { outbox? })`: the store's own schema migrations, a UTF-8 check, the metadata read once; + `outbox` turns the outbox on (from the current `lastSeq`) or off; omitted, the stored setting stays. A failure + closes the backend. `store.lastSeq`, `store.outbox` (`read(afterSeq, limit, ctx)` from its floor, `trim(throughSeq, ctx)`), + `store.close(ctx)` (idempotent; waits for a running step; `ctx` ends only the caller's wait). +- `defineTable(name, { version, columns, key, indexes? })`: flat columns of `str int num bool`, `opt(...)` of them, + `json(schema, { version, migrate? })` (one JSON cell, migrated on access); defaults; a key and index columns of + non-`opt` `str`/`int`. `evolveTable(prev, (t) => t…)` appends the next version: `rename`, `add`, `json` before an + optional data step (`rows(fn)` or `update({ where?, set })`), `drop`, `index`, `dropIndex` after it. +- Queries: `query(Table).where((t) => cond).orderBy((t) => col, dir)` with `eq ne lt lte gt gte inArray isNull prefix + and or not` (SQL's `NULL` rules); pages `{ items, next? }` with opaque cursors, `rows()` paging by 100, `count`. +- Writes in `self.tx`: `put` (an upsert), `update(key, set)`, `delete(key)`, `updateWhere(where, set)`, + `deleteWhere(where)`; a `set` is values or expressions (`add sub max min concat replacePrefix coalesce when`); key + columns never change. `tx.get` sees the transaction's own single-key writes, not after a bulk one (W10). +- Text is UTF-8 compared by code points (`compareText`); NUL is refused, lone surrogates replaced. +- `store.export(ctx)` holds the commit queue from its first `next()` until it ends, is closed or `ctx` aborts; parts: + the manifest, per table its `table` write, `put`s and `index` writes, the live documents, `end`. `importStore` + checks every part (a trust boundary) and applies the archive in one backend transaction; JSONL via `encodeJsonl` and + `decodeJsonl`. + +## For Chord's layers + +`ChordStore` (`store.ts`: `commit(staged, fence, ctx)`, `read(fn, ctx)` guarded as Durable's `GuardedStorage`, +`take()` for an export, the catalog), the commit `Step` and `StagedWrite`s in their `WriteOrder` (`step.ts`), the documents engine +(`engine.ts`), the catalog (`catalog.ts`), the write vocabulary (`writes.ts`, `table-writes.ts`), rows and columns +(`rows.ts`, `columns.ts`, `json-column.ts`), the compiled where/set vocabulary (`expression.ts`) and its JS evaluator (`evaluate.ts`), +SQL (`sql.ts`), the table handle and a transaction's table changes (`table-handle.ts`, `table-changes.ts`), table +histories and migrations (`table.ts`, `table-evolve.ts`, `table-history.ts`, `table-migration.ts`), the outbox +(`outbox.ts`), archives (`archive.ts`, `export.ts`, `import.ts`, `applier.ts`, `jsonl.ts`), the store's SQL schema +(`schema.ts`) and backends (`backend.ts`). + +Tests: `test/next/store/` (one file per source); the benchmarks in `test/next/bench/store.ts` (results in +`docs/next/benchmarks.md`). Examples: `20-tables.ts`, `22-migrations.ts`, `25-export-import.ts`. diff --git a/packages/chord/src/next/store/applier.ts b/packages/chord/src/next/store/applier.ts new file mode 100644 index 000000000..f6e55e214 --- /dev/null +++ b/packages/chord/src/next/store/applier.ts @@ -0,0 +1,163 @@ +// The applier (slice 6a §13): recorded writes applied as they were recorded, inside a backend transaction, checked +// against the store's own records (a trust boundary: archives, later a replica's batches). One code path for import, +// bootstrap and, from slice 6b, batch apply; 6a applies the kinds an archive holds. + +import type { SqliteValue } from "../../sqlite/index.ts"; +import { checkName } from "../contracts/internal.ts"; +import { invalid } from "../data/internal.ts"; +import type { ArchivePart } from "./archive.ts"; +import type { TableRecord } from "./catalog.ts"; +import { COLUMN_NAME, type ColumnRecord } from "./columns.ts"; +import { insertDocument } from "./engine.ts"; +import { checkScalar, ROW_VERSION, toSqlite, versionColumn } from "./rows.ts"; +import { recordedPut } from "./sql.ts"; +import type { Step } from "./step.ts"; +import { INDEX_NAME } from "./table.ts"; +import { applyIndex, applyTable, type TableWrite } from "./table-writes.ts"; + +/** A recorded write the applier takes: the writes of an archive. */ +export type AppliedWrite = Exclude; + +/** Applies `write` in `step`; a write the records refuse throws `invalid/args` (`what` names its place). */ +export async function applyWrite(step: Step, write: AppliedWrite, what: string): Promise { + switch (write.kind) { + case "table": + checkTable(write, what); + await applyTable(step, write); + return; + case "index": + checkIndex(recordOf(step, write.table, what), write, what); + await applyIndex(step, write); + return; + case "put": { + const record = recordOf(step, write.table, what); + const { sql, physical } = recordedPut(write.table, record.columns); + await step.run(sql, ...putParams(record, write, physical, what)); + return; + } + case "document.create": { + let text: string; + try { + text = JSON.stringify(write.base.value); + } catch (error) { + throw invalid("args", `${what}: the base is not JSON (${String(error)})`); + } + await insertDocument(step, write.record, text); + return; + } + } +} + +/** A recorded table's SQL name (three names joined by `:`), columns, key and json versions, by the name rules. */ +function checkTable(write: Extract, what: string): void { + const parts = write.table.split(":"); + if (parts.length !== 3) throw invalid("args", `${what}: table ${JSON.stringify(write.table)}: not kind:type:table`); + try { + for (const part of parts) checkName(part, `${what}: table name part`); + } catch (error) { + // `checkName` throws plain Errors. + throw invalid("args", (error as Error).message); + } + const folded = new Set(); + for (const [name, column] of Object.entries(write.columns)) { + const where = `${what}: column ${JSON.stringify(name)}`; + if (!COLUMN_NAME.test(name) || folded.has(name.toLowerCase())) + throw invalid("args", `${where}: not a column name`); + folded.add(name.toLowerCase()); + checkColumn(column, where); + } + if (write.key.length === 0) throw invalid("args", `${what}: the key has no column`); + checkOrderColumns(write.columns, write.key, `${what}: the key`); +} + +function checkColumn(column: ColumnRecord, what: string): void { + const json = column.type === "json"; + if (json !== (column.version !== undefined) || json !== (column.fingerprint !== undefined)) { + throw invalid("args", `${what}: a json(...) column has a version and a fingerprint, no other column has`); + } + if (column.default === undefined) return; + if (json || column.optional) throw invalid("args", `${what}: only a non-opt scalar column has a default`); + checkScalar(column.type, column.default, `${what}: the default`); +} + +/** `names` are distinct, recorded, non-`opt` `str` or `int` columns. */ +function checkOrderColumns( + columns: { readonly [name: string]: ColumnRecord }, + names: readonly string[], + what: string, +): void { + if (new Set(names).size !== names.length) throw invalid("args", `${what}: a column occurs twice`); + for (const name of names) { + const column = Object.hasOwn(columns, name) ? columns[name] : undefined; + if (column === undefined || column.optional || (column.type !== "str" && column.type !== "int")) { + throw invalid("args", `${what}: ${JSON.stringify(name)} is not a non-opt str or int column`); + } + } +} + +function checkIndex(record: TableRecord, write: Extract, what: string): void { + if (!INDEX_NAME.test(write.name) || Object.hasOwn(record.indexes, write.name)) { + throw invalid("args", `${what}: index ${JSON.stringify(write.name)}: not a new index name`); + } + checkOrderColumns(record.columns, write.columns, `${what}: index ${write.name}`); +} + +/** The record of `table` in `step`; a write for a table not recorded is refused. */ +function recordOf(step: Step, table: string, what: string): TableRecord { + const record = step.table(table); + if (record === undefined) throw invalid("args", `${what}: table ${JSON.stringify(table)} is not recorded`); + return record; +} + +/** + * The parameters of a `put` in `physical` order, checked against the recorded columns: every non-`opt` column, no + * other field, values of their types, `json(...)` cells as JSON (no plugin code runs) with a version up to the + * column's, and `version` up to the table's. + */ +function putParams( + record: TableRecord, + write: Extract, + physical: readonly string[], + what: string, +): SqliteValue[] { + const { row, columns: versions, version } = write; + for (const field of Object.keys(row)) { + if (!Object.hasOwn(record.columns, field)) + throw invalid("args", `${what}: unknown column ${JSON.stringify(field)}`); + } + for (const field of Object.keys(versions)) { + if (record.columns[field]?.type !== "json" || !Object.hasOwn(row, field)) { + throw invalid( + "args", + `${what}: a version for ${JSON.stringify(field)}, which is no json(...) cell of the row`, + ); + } + } + if (version > record.version) { + throw invalid("args", `${what}: version ${version} is newer than the table's version ${record.version}`); + } + const cells = new Map(); + for (const [name, column] of Object.entries(record.columns)) { + const where = `${what}: column ${JSON.stringify(name)}`; + const value = row[name]; + // A row has no explicit `null`: an absent field is `NULL`; a `json(...)` cell's `null` is the JSON value. + if (column.type === "json" ? !Object.hasOwn(row, name) : value === undefined) { + if (!column.optional) throw invalid("args", `${where} is missing`); + cells.set(name, null); + continue; + } + if (column.type !== "json") { + cells.set(name, toSqlite(checkScalar(column.type, value, where))); + continue; + } + const version = versions[name]; + if (version === undefined || version > column.version!) throw invalid("args", `${where}: no readable version`); + try { + cells.set(name, JSON.stringify(value)); + } catch (error) { + throw invalid("args", `${where}: not JSON (${String(error)})`); + } + cells.set(versionColumn(name), version); + } + return physical.map((name) => (name === ROW_VERSION ? version : (cells.get(name) ?? null))); +} diff --git a/packages/chord/src/next/store/archive.ts b/packages/chord/src/next/store/archive.ts new file mode 100644 index 000000000..963cfae48 --- /dev/null +++ b/packages/chord/src/next/store/archive.ts @@ -0,0 +1,106 @@ +// Archives (slice 6a §13; decisions "Export archives are operations", "Writes carry their versions; archives are +// lossless"): the parts an export yields and an import takes, their order, and the one compiled schema every +// imported part is checked against first. + +import Type, { type TInteger } from "typebox"; +import type { Context } from "../../context/index.ts"; +import { arr, bool, int, lit, num, obj, opt, str, union, unknown } from "../contracts/index.ts"; +import { invalid, validatorFor } from "../data/internal.ts"; +import type { DocumentWrite } from "./engine.ts"; +import type { TableWrite } from "./table-writes.ts"; + +/** The archive format `export` writes; `import` takes this one only. */ +export const ARCHIVE_FORMAT = 1; + +/** The first part: the format, the store's schema version, its `lastSeq` and the next document id (rule 24). */ +export type ArchiveManifest = { + readonly kind: "manifest"; + readonly format: number; + readonly schema: number; + readonly lastSeq: number; + readonly nextDocumentId: number; +}; + +/** + * One part of an archive, in this order: the manifest; per table (by SQL name) its `table` write, its `put`s and its + * `index` writes; the live documents' `document.create`s (by id, the current value as the base); `end` with the + * count of the parts between. + */ +export type ArchivePart = + | ArchiveManifest + | Extract + | Extract + | { readonly kind: "end"; readonly writes: number }; + +/** An integer from `min` up to 2^53 - 1: seqs, ids, versions and counts stay exact (a seq after 2^53 would repeat). */ +function safe(min: number): TInteger { + return int({ min, max: Number.MAX_SAFE_INTEGER }); +} + +const version = safe(1); + +const column = obj({ + type: union([lit("str"), lit("int"), lit("num"), lit("bool"), lit("json")]), + optional: opt(lit(true)), + default: opt(union([str(), num(), bool()])), + version: opt(version), + fingerprint: opt(str()), +}); + +const PART = union([ + obj({ + kind: lit("manifest"), + format: safe(1), + schema: safe(1), + lastSeq: safe(0), + nextDocumentId: safe(1), + }), + obj({ + kind: lit("table"), + table: str(), + version, + state: union([lit("ready"), lit("migrating")]), + columns: Type.Record(Type.String(), column), + key: arr(str()), + fingerprint: str(), + }), + obj({ + kind: lit("put"), + table: str(), + row: Type.Record(Type.String(), unknown()), + version, + columns: Type.Record(Type.String(), version), + }), + obj({ kind: lit("index"), table: str(), name: str(), columns: arr(str()) }), + obj({ + kind: lit("document.create"), + record: obj({ + id: safe(1), + namespace: str(), + owner: str(), + kind: str(), + // Absent for a singleton. + key: opt(str()), + history: union([lit("latest"), lit("rewindable")]), + createdAt: safe(1), + meta: opt(str()), + revision: safe(1), + version, + fingerprint: str(), + }), + base: obj({ version, fingerprint: str(), value: unknown() }), + }), + obj({ kind: lit("end"), writes: safe(0) }), +]); + +const PART_SCHEMA = { schema: PART }; + +/** Checks one imported part against the archive schema (a trust boundary); `invalid/args` naming its position. */ +export function checkPart(part: unknown, position: number, context: Context): ArchivePart { + context.telemetry.metrics?.counter("chord.data.validations").add(1); + if (!validatorFor(PART_SCHEMA, context).Check(part)) { + throw invalid("args", `import: part ${position}: not an archive part`); + } + // The check above establishes the type. + return part as ArchivePart; +} diff --git a/packages/chord/src/next/store/backend.ts b/packages/chord/src/next/store/backend.ts new file mode 100644 index 000000000..3d1a52a84 --- /dev/null +++ b/packages/chord/src/next/store/backend.ts @@ -0,0 +1,19 @@ +// Store backends (slice 6a §2): what a store runs on. One kind today, any `SqliteDatabase` (Node through +// `chord/sqlite/node`, Durable Objects through `chord/sqlite/cloudflare`); its interface is internal to the store. + +import type { SqliteDatabase } from "../../sqlite/index.ts"; + +/** A backend for `openStore` and `importStore`: opaque to apps, owned by the store once given to it. */ +export class Backend { + /** The database the store runs its SQL on. */ + readonly db: SqliteDatabase; + + constructor(db: SqliteDatabase) { + this.db = db; + } +} + +/** A backend over a SQLite database (`openNodeSqliteDatabase(path)`, a Durable Object's storage, `:memory:` in tests). */ +export function sqliteBackend(db: SqliteDatabase): Backend { + return new Backend(db); +} diff --git a/packages/chord/src/next/store/catalog.ts b/packages/chord/src/next/store/catalog.ts new file mode 100644 index 000000000..ae2fc9f6c --- /dev/null +++ b/packages/chord/src/next/store/catalog.ts @@ -0,0 +1,37 @@ +// The catalog (slice 6a §2, §5): the table definitions the store records (version, state, columns, key, fingerprint, +// indexes), read once at open, changed only by `table`, `index`, `dropIndex` and `alter` writes inside a step, and in +// memory again only after `COMMIT` returned. + +import type { SqliteExecutor } from "../../sqlite/index.ts"; +import type { ColumnRecord } from "./columns.ts"; +import { TABLES } from "./schema.ts"; +import type { Step } from "./step.ts"; + +/** A table as the store records it, by its SQL name. */ +export type TableRecord = { + readonly table: string; + readonly version: number; + /** `migrating` while an eager migration of `version` runs (slice 6a §9). */ + readonly state: "ready" | "migrating"; + readonly columns: { readonly [name: string]: ColumnRecord }; + readonly key: readonly string[]; + readonly fingerprint: string; + /** Index definitions by name: their declared columns (the key is appended when rendered). */ + readonly indexes: { readonly [name: string]: readonly string[] }; +}; + +/** Every recorded table, by SQL name. */ +export async function readCatalog(executor: SqliteExecutor): Promise> { + const rows = await executor.all<{ readonly name: string; readonly definition: string }>( + `SELECT name, definition FROM ${TABLES.tables}`, + ); + return new Map(rows.map(({ name, definition }) => [name, JSON.parse(definition) as TableRecord])); +} + +/** Writes the table records `step` changed, once each, inside its backend transaction. */ +export async function writeCatalog(step: Step): Promise { + for (const [name, record] of step.tables) { + const definition = JSON.stringify(record); + await step.run(`INSERT OR REPLACE INTO ${TABLES.tables} (name, definition) VALUES (?, ?)`, name, definition); + } +} diff --git a/packages/chord/src/next/store/columns.ts b/packages/chord/src/next/store/columns.ts new file mode 100644 index 000000000..8d4118262 --- /dev/null +++ b/packages/chord/src/next/store/columns.ts @@ -0,0 +1,153 @@ +// Table columns (decisions "Flat columns", "Nullable columns and the `set` form", "Restricted names"): the restricted +// subset of L1's helpers a table's `columns` take (`str int num bool`, `opt(...)` of them, `json(...)`, defaults), +// their types for rows, keys and column references, and the column descriptors and records the store keeps. + +import type { TBoolean, TInteger, TNumber, TOptional, TString } from "typebox"; +import type { Static, TSchema } from "../contracts/index.ts"; +import { JsonColumn, type JsonColumnSchema } from "./json-column.ts"; +import { storedText } from "./text.ts"; + +/** A schema a table column takes. */ +export type ColumnSchema = + | ScalarSchema + | JsonColumnSchema + | TOptional>; + +type ScalarSchema = TString | TInteger | TNumber | TBoolean; + +/** A table's `columns`: a flat record of column schemas. */ +export type Columns = { readonly [name: string]: ColumnSchema }; + +/** A column's type in the store. */ +export type ColumnType = "str" | "int" | "num" | "bool" | "json"; + +/** The store type of column schema `S`. */ +export type ColumnTypeOf = S extends TOptional + ? ColumnTypeOf + : S extends TString + ? "str" + : S extends TInteger + ? "int" + : S extends TNumber + ? "num" + : S extends TBoolean + ? "bool" + : "json"; + +/** The value type of column schema `S` (without `undefined` for an `opt(...)` one). */ +export type ValueOf = S extends TOptional ? ValueOf : S extends TSchema ? Static : never; + +/** The names of the columns that are not `opt(...)`. */ +type RequiredNames = { [K in keyof C]: C[K] extends TOptional ? never : K }[keyof C]; + +/** The names of the columns with a declared default. */ +type DefaultedNames = { + [K in keyof C]: C[K] extends { readonly default: unknown } ? K : never; +}[keyof C]; + +/** A row as reads return it: every non-`opt` column, the `opt(...)` ones when not `NULL`. */ +export type RowOf = { [K in RequiredNames]: ValueOf } & { + [K in Exclude>]?: ValueOf; +}; + +/** A row as `put` takes it: as read, the columns with a default may be left out (rule 15). */ +export type PutRowOf = { [K in Exclude, DefaultedNames>]: ValueOf } & { + [K in Exclude, DefaultedNames>>]?: ValueOf; +}; + +/** The columns a key, an index or an `orderBy` may use: non-`opt` `str` or `int`. */ +export type OrderColumn = { + [K in keyof C]: C[K] extends TString | TInteger ? (C[K] extends TOptional ? never : K) : never; +}[keyof C] & + string; + +/** A column of the store: its name, type, nullability and default; a `json(...)` one with its definition. */ +export type Column = + | { + readonly name: string; + readonly type: "str" | "int" | "num" | "bool"; + readonly optional: boolean; + readonly default: string | number | boolean | undefined; + } + | { readonly name: string; readonly type: "json"; readonly optional: boolean; readonly json: JsonColumn }; + +/** A column as the `table` write records it (slice 6a §5): what a replica needs without the plugin's code. */ +export type ColumnRecord = { + readonly type: ColumnType; + readonly optional?: true; + readonly default?: string | number | boolean; + /** A `json(...)` column's version and schema fingerprint. */ + readonly version?: number; + readonly fingerprint?: string; +}; + +/** The column name rule (decision "Restricted names"). */ +export const COLUMN_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/; + +/** + * The columns of `columns` in declaration order, checked (declaration policy: JavaScript callers reach it): names + * `[A-Za-z_][A-Za-z0-9_]*` without two equal ignoring ASCII case, only the helpers' types, no bounds or patterns (a + * bulk `set` could not keep them), a default of the column's type and not on an `opt(...)` column. + */ +export function describeColumns(columns: Columns, table: string): Column[] { + const seen = new Set(); + return Object.entries(columns).map(([name, schema]) => { + checkColumnName(name, table); + const folded = name.toLowerCase(); + if (seen.has(folded)) throw new Error(`${table}: column "${name}" differs from another only in case`); + seen.add(folded); + return describeColumn(name, schema, `${table}: column "${name}"`); + }); +} + +/** Throws unless `name` follows the column name rule (declaration policy); `what` names the table or the edit. */ +export function checkColumnName(name: string, what: string): void { + if (!COLUMN_NAME.test(name)) throw new Error(`${what}: column "${name}": expected [A-Za-z_][A-Za-z0-9_]*`); +} + +const SCALAR_TYPES: { readonly [type: string]: "str" | "int" | "num" | "bool" } = { + string: "str", + integer: "int", + number: "num", + boolean: "bool", +}; + +function describeColumn(name: string, schema: ColumnSchema, what: string): Column { + const optional = Object.hasOwn(schema, "~optional"); + // Declaration policy: a JavaScript caller may pass any schema, so its fields are read as unknown data. + const fields = schema as unknown as { readonly [key: string]: unknown }; + const json = fields.json; + if (json instanceof JsonColumn) return { name, type: "json", optional, json }; + const type = SCALAR_TYPES[String(fields.type)]; + if (type === undefined) throw new Error(`${what}: expected str(), int(), num(), bool(), opt(...) or json(...)`); + for (const key of Object.keys(schema)) { + if (key !== "type" && key !== "default") throw new Error(`${what}: "${key}" is not supported in a table column`); + } + const value = fields.default; + if (value !== undefined) { + if (optional) throw new Error(`${what}: an opt(...) column has no default`); + checkDefault(type, value, what); + } + return { name, type, optional, default: value as string | number | boolean | undefined }; +} + +function checkDefault(type: "str" | "int" | "num" | "bool", value: unknown, what: string): void { + const valid = + type === "str" + ? typeof value === "string" && storedText(value, what) === value + : type === "int" + ? Number.isSafeInteger(value) + : type === "num" + ? typeof value === "number" && Number.isFinite(value) + : typeof value === "boolean"; + if (!valid) throw new Error(`${what}: the default ${JSON.stringify(value)} is not a ${type} value`); +} + +/** The record of `column` in the `table` write. */ +export function columnRecord(column: Column): ColumnRecord { + const optional = column.optional ? { optional: true as const } : {}; + if (column.type === "json") { + return { type: "json", ...optional, version: column.json.version, fingerprint: column.json.fingerprint }; + } + return { type: column.type, ...optional, ...(column.default === undefined ? {} : { default: column.default }) }; +} diff --git a/packages/chord/src/next/store/engine.ts b/packages/chord/src/next/store/engine.ts new file mode 100644 index 000000000..b5fbc8f80 --- /dev/null +++ b/packages/chord/src/next/store/engine.ts @@ -0,0 +1,295 @@ +// The documents engine (S13, slice 6a §8): one record per incarnation of an address, revisions made of a base and +// later deltas; its writes inside the commit step, checked by the writes themselves (the partial unique index on the +// live address, the counted conditional `UPDATE` on the record's revision), and its plain reads. + +import { apply, type Op } from "../../delta/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { SqliteExecutor } from "../../sqlite/index.ts"; +import { TABLES } from "./schema.ts"; +import type { StagedWrite, Step } from "./step.ts"; + +/** + * A document's address: the adapter's namespace, the owner string, the definition's name and, for a keyed document, its + * key; a singleton has none (as Durable's addresses). The column keeps `""` for it, which a keyed document's empty key + * shares (its definition says which it is): a record read back gives no key for either. + */ +export type DocumentAddress = { + readonly namespace: string; + readonly owner: string; + readonly kind: string; + readonly key?: string; +}; + +/** A stored record: one incarnation of an address. Chord keeps only the latest revision (`history: "latest"`). */ +export type DocumentRecord = DocumentAddress & { + readonly id: number; + readonly history: "latest" | "rewindable"; + /** The seq that created it. */ + readonly createdAt: number; + readonly meta?: string; + /** Counts its changes: a change names the revision it was prepared on. */ + readonly revision: number; + /** The version and schema fingerprint of its current base. */ + readonly version: number; + readonly fingerprint: string; +}; + +/** A base: a whole value at a version. */ +export type DocumentBase = { readonly version: number; readonly fingerprint: string; readonly value: JsonValue }; + +/** A change's content: a new base, or a delta on the current base's version. */ +export type DocumentContent = + | ({ readonly kind: "base" } & DocumentBase) + | { readonly kind: "delta"; readonly version: number; readonly ops: readonly Op[] }; + +/** The engine's writes as the outbox and archives record them. */ +export type DocumentWrite = + | { readonly kind: "document.create"; readonly record: DocumentRecord; readonly base: DocumentBase } + | { + readonly kind: "document.change"; + readonly id: number; + readonly revision: number; + readonly content: DocumentContent; + } + | { readonly kind: "document.retire"; readonly id: number; readonly revision: number }; + +/** A loaded document: its record and its value (base plus deltas applied once). */ +export type StoredDocument = { + readonly record: DocumentRecord; + readonly value: JsonValue; + /** The deltas stored after its base (what `checkpointWhen` sees). */ + readonly deltasSinceBase: number; +}; + +/** + * Stages the creation of a document at `address` with `base`; its id comes from the step. `text` is the base's value + * serialized before the queue (rule 8). + */ +export function createDocument( + address: DocumentAddress, + history: "latest" | "rewindable", + base: DocumentBase, + text: string, +): StagedWrite { + return { + order: "create", + apply: async (step) => { + // The next id is stored too: it stays in the safe range, or ids would stop counting exactly and repeat. + if (step.nextDocumentId >= Number.MAX_SAFE_INTEGER) { + throw new Error("store: no document id past Number.MAX_SAFE_INTEGER"); + } + const id = step.nextDocumentId++; + step.created.push(id); + const { version, fingerprint } = base; + const record: DocumentRecord = { + id, + ...address, + history, + createdAt: step.seq, + revision: 1, + version, + fingerprint, + }; + await insertDocument(step, record, text); + return { kind: "document.create", record, base }; + }, + }; +} + +/** + * Inserts `record` as it is (id, `createdAt`, revision kept: a creation, or an import) with its base, `text` being + * the base's value serialized. + */ +export async function insertDocument(step: Step, record: DocumentRecord, text: string): Promise { + const { id, namespace, owner, kind, key, history, createdAt, meta, revision, version, fingerprint } = record; + await step.run( + `INSERT INTO ${TABLES.documents} + (id, namespace, owner, kind, key, history, created_at, meta, revision, version, fingerprint) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + id, + namespace, + owner, + kind, + key ?? "", + history, + createdAt, + meta ?? null, + revision, + version, + fingerprint, + ); + await insertRevision(step, id, "base", version, text); +} + +/** + * Stages a change of document `id` prepared on `revision`: a delta on the current base's version, or a new base + * (which prunes the older revisions of a `latest` record). `text` is the content serialized before the queue. + */ +export function changeDocument(id: number, revision: number, content: DocumentContent, text: string): StagedWrite { + const write: DocumentWrite = { kind: "document.change", id, revision, content }; + return { + order: "write", + apply: async (step) => { + // As ids: a revision past the safe range would stop counting exactly. + if (revision >= Number.MAX_SAFE_INTEGER) throw new Error(`document ${id}: no revision past the safe range`); + await insertRevision(step, id, content.kind, content.version, text); + const changed = + content.kind === "delta" + ? await step.runCounted( + `UPDATE ${TABLES.documents} SET revision = revision + 1 + WHERE id = ? AND revision = ? AND version = ? AND retired_at IS NULL`, + id, + revision, + content.version, + ) + : await step.runCounted( + `UPDATE ${TABLES.documents} SET revision = revision + 1, version = ?, fingerprint = ? + WHERE id = ? AND revision = ? AND retired_at IS NULL`, + content.version, + content.fingerprint, + id, + revision, + ); + // Memory and disk disagree: the step fails, which stops the store (Durable's rule). + if (changed !== 1) throw new Error(`document ${id}: not live at revision ${revision}`); + if (content.kind === "base") { + await step.run(`DELETE FROM ${TABLES.revisions} WHERE document_id = ? AND seq < ?`, id, step.seq); + } + return write; + }, + }; +} + +/** Stages the retirement of `latest` document `id` at `revision`: its record and revisions go (rule 19). */ +export function retireDocument(id: number, revision: number): StagedWrite { + const write: DocumentWrite = { kind: "document.retire", id, revision }; + return { + order: "retire", + apply: async (step) => { + const deleted = await step.runCounted( + `DELETE FROM ${TABLES.documents} WHERE id = ? AND revision = ? AND retired_at IS NULL`, + id, + revision, + ); + if (deleted !== 1) throw new Error(`document ${id}: not live at revision ${revision}`); + await step.run(`DELETE FROM ${TABLES.revisions} WHERE document_id = ?`, id); + return write; + }, + }; +} + +type RecordRow = { + readonly id: number; + readonly namespace: string; + readonly owner: string; + readonly kind: string; + readonly key: string; + readonly history: "latest" | "rewindable"; + readonly created_at: number; + readonly meta: string | null; + readonly revision: number; + readonly version: number; + readonly fingerprint: string; +}; + +const RECORD_COLUMNS = "id, namespace, owner, kind, key, history, created_at, meta, revision, version, fingerprint"; + +/** The live record at `address`, if any. */ +export async function findDocument( + executor: SqliteExecutor, + address: DocumentAddress, +): Promise { + const row = await executor.get( + `SELECT ${RECORD_COLUMNS} FROM ${TABLES.documents} + WHERE namespace = ? AND owner = ? AND kind = ? AND key = ? AND retired_at IS NULL`, + address.namespace, + address.owner, + address.kind, + address.key ?? "", + ); + return row === undefined ? undefined : recordOf(row); +} + +type RevisionRow = { + readonly seq: number; + readonly kind: "base" | "delta"; + readonly version: number; + readonly content: string; +}; + +/** Loads the live document `record` found: its latest base and the deltas after it, applied once. */ +export async function loadDocument(executor: SqliteExecutor, record: DocumentRecord): Promise { + const revisions = await executor.all( + `SELECT seq, kind, version, content FROM ${TABLES.revisions} WHERE document_id = ? AND seq >= ( + SELECT max(seq) FROM ${TABLES.revisions} WHERE document_id = ? AND kind = 'base') ORDER BY seq`, + record.id, + record.id, + ); + const [base, ...deltas] = revisions; + if (base === undefined || base.kind !== "base") throw new Error(`document ${record.id}: no base`); + let value = JSON.parse(base.content) as JsonValue; + for (const delta of deltas) { + if (delta.version !== base.version) throw new Error(`document ${record.id}: a delta crosses a version`); + value = apply(value, JSON.parse(delta.content) as Op[]); + } + return { record, value, deltasSinceBase: deltas.length }; +} + +/** Up to `limit` keys of live documents of `kind` under `owner` after `after` (from the first), in code point order. */ +export async function listDocumentKeys( + executor: SqliteExecutor, + address: Omit, + after: string | undefined, + limit: number, +): Promise { + // The first page has no lower bound: `""` is a key too. + const from = after === undefined ? "" : " AND key > ?"; + const rows = await executor.all<{ readonly key: string }>( + `SELECT key FROM ${TABLES.documents} WHERE namespace = ? AND owner = ? AND kind = ?${from} + AND retired_at IS NULL ORDER BY key LIMIT ?`, + address.namespace, + address.owner, + address.kind, + ...(after === undefined ? [] : [after]), + limit, + ); + return rows.map(({ key }) => key); +} + +/** Up to `limit` live records with ids above `after`, by id (an export's pages). */ +export async function listDocuments(executor: SqliteExecutor, after: number, limit: number): Promise { + const rows = await executor.all( + `SELECT ${RECORD_COLUMNS} FROM ${TABLES.documents} WHERE id > ? AND retired_at IS NULL ORDER BY id LIMIT ?`, + after, + limit, + ); + return rows.map(recordOf); +} + +function recordOf(row: RecordRow): DocumentRecord { + const { id, namespace, owner, kind, key, history, created_at, meta, revision, version, fingerprint } = row; + return { + id, + namespace, + owner, + kind, + ...(key === "" ? {} : { key }), + history, + createdAt: created_at, + ...(meta === null ? {} : { meta }), + revision, + version, + fingerprint, + }; +} + +function insertRevision(step: Step, id: number, kind: "base" | "delta", version: number, text: string): Promise { + return step.run( + `INSERT INTO ${TABLES.revisions} (document_id, seq, kind, version, content) VALUES (?, ?, ?, ?, ?)`, + id, + step.seq, + kind, + version, + text, + ); +} diff --git a/packages/chord/src/next/store/evaluate.ts b/packages/chord/src/next/store/evaluate.ts new file mode 100644 index 000000000..0699c83db --- /dev/null +++ b/packages/chord/src/next/store/evaluate.ts @@ -0,0 +1,131 @@ +// The JavaScript evaluator of the compiled table vocabulary (slice 6a §6; rule 10): the `tx.get` overlay matches +// `where` nodes and applies `set` nodes with SQL's three-valued `NULL` rules and code point text order, as the SQL +// that `sql.ts` renders from the same nodes does in SQLite. + +import type { JsonValue } from "../../json.ts"; +import { invalid } from "../data/internal.ts"; +import type { Comparison, ExprNode, Operation, SetNode, WhereNode } from "./expression.ts"; +import type { Scalar } from "./rows.ts"; +import { compareText } from "./text.ts"; + +/** A row as the evaluator reads it: stored cells, an absent column being `NULL`. */ +export type EvaluatedRow = { readonly [column: string]: Scalar | JsonValue | undefined }; + +/** SQL's three-valued truth: `null` is unknown (a comparison with `NULL`). */ +type Truth = boolean | null; + +/** Whether `row` matches `node` under SQL's rules: unknown (`NULL`) does not match. */ +export function matches(node: WhereNode, row: EvaluatedRow): boolean { + return truth(node, row) === true; +} + +function truth(node: WhereNode, row: EvaluatedRow): Truth { + if ("and" in node) + return node.and.reduce((all, each) => (all === false ? false : both(all, truth(each, row))), true); + if ("or" in node) + return node.or.reduce((any, each) => (any === true ? true : either(any, truth(each, row))), false); + if ("not" in node) { + const inner = truth(node.not, row); + return inner === null ? null : !inner; + } + if ("isNull" in node) return row[node.isNull] === undefined; + // The row is checked against the table the condition was compiled for: its columns hold scalars. + if ("inArray" in node) { + const [column, values] = node.inArray; + // SQL: `x IN ()` is false, for a `NULL` x too. + if (values.length === 0) return false; + const value = row[column] as Scalar | undefined; + return value === undefined ? null : values.some((each) => compareScalar(value, each) === 0); + } + if ("prefix" in node) { + const value = row[node.prefix[0]] as string | undefined; + return value === undefined ? null : value.startsWith(node.prefix[1]); + } + // The remaining nodes are the comparisons, one entry each; `Object.entries` loses that. + const [op, [column, operand]] = Object.entries(node)[0] as [Comparison, readonly [string, Scalar]]; + const value = row[column] as Scalar | undefined; + if (value === undefined) return null; + const order = compareScalar(value, operand); + switch (op) { + case "eq": + return order === 0; + case "ne": + return order !== 0; + case "lt": + return order < 0; + case "lte": + return order <= 0; + case "gt": + return order > 0; + case "gte": + return order >= 0; + } +} + +function both(a: Truth, b: Truth): Truth { + if (a === false || b === false) return false; + return a === null || b === null ? null : true; +} + +function either(a: Truth, b: Truth): Truth { + if (a === true || b === true) return true; + return a === null || b === null ? null : false; +} + +/** Orders two cells of one column as SQLite does: text by code point, booleans as 0 and 1, numbers numerically. */ +export function compareScalar(a: Scalar, b: Scalar): number { + if (typeof a === "string") return compareText(a, b as string); + return Number(a) - Number(b); +} + +const SAFE = Number.MAX_SAFE_INTEGER; + +/** + * `row` after `set`, every right side evaluated against `row` (simultaneous: `{ a: b, b: a }` swaps); an integer + * result outside ±(2^53 - 1) throws `invalid/args` (rule 10), as the column's `CHECK` fails the step. + */ +export function applySet(set: SetNode, row: EvaluatedRow, table: string): EvaluatedRow { + const next: { [column: string]: Scalar | JsonValue | undefined } = { ...row }; + for (const [column, node] of Object.entries(set)) { + const value = evaluate(node, row, table); + if (value === undefined) delete next[column]; + else next[column] = value; + } + return next; +} + +function evaluate(node: ExprNode, row: EvaluatedRow, table: string): Scalar | JsonValue | undefined { + if (node === null) return undefined; + if (typeof node !== "object") return node; + if ("col" in node) return row[node.col]; + if ("json" in node) return node.json; + if ("when" in node) { + const [condition, a, b] = node.when; + return evaluate(matches(condition, row) ? a : b, row, table); + } + if ("coalesce" in node) return evaluate(node.coalesce[0], row, table) ?? evaluate(node.coalesce[1], row, table); + if ("replacePrefix" in node) { + const [subject, from, to] = node.replacePrefix; + const value = evaluate(subject, row, table) as string | undefined; + return value === undefined || !value.startsWith(from) ? value : to + value.slice(from.length); + } + // The remaining nodes are the arithmetic and text operations, one entry each, over scalar operands (compiled so). + const [op, [x, y]] = Object.entries(node)[0] as [Operation, readonly [ExprNode, ExprNode]]; + const a = evaluate(x, row, table) as Scalar | undefined; + const b = evaluate(y, row, table) as Scalar | undefined; + if (a === undefined || b === undefined) return undefined; + switch (op) { + case "concat": + return (a as string) + (b as string); + case "max": + return compareScalar(a, b) >= 0 ? a : b; + case "min": + return compareScalar(a, b) <= 0 ? a : b; + default: { + const result = op === "add" ? (a as number) + (b as number) : (a as number) - (b as number); + if (result > SAFE || result < -SAFE) + throw invalid("args", `${table}: ${op} gives ${result}, outside ±(2^53 - 1)`); + return result; + } + } +} diff --git a/packages/chord/src/next/store/export.ts b/packages/chord/src/next/store/export.ts new file mode 100644 index 000000000..cdadc0424 --- /dev/null +++ b/packages/chord/src/next/store/export.ts @@ -0,0 +1,207 @@ +// `store.export(ctx)` (slice 6a §13, rule 26): the store's state after its `lastSeq` as archive parts, read page by +// page with plain reads while the export holds the commit queue (no commit can interleave, no long read transaction). +// The queue is taken at the first `next()` and given back when the iteration ends, throws, is closed or `ctx` aborts; +// an abort while the consumer is suspended gives it back at once and drops the pages and records the export held, and +// an abort while it waits for the queue ends the wait. Closing the store ends it too (`released`). + +import { awaitWithContext, type Context } from "../../context/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { SqliteValue } from "../../sqlite/index.ts"; +import { ChordError } from "../data/index.ts"; +import { ARCHIVE_FORMAT, type ArchivePart } from "./archive.ts"; +import type { TableRecord } from "./catalog.ts"; +import { listDocuments, loadDocument } from "./engine.ts"; +import { ROW_VERSION, type Scalar, versionColumn } from "./rows.ts"; +import { STORE_SCHEMA } from "./schema.ts"; +import { quote, recordedPut } from "./sql.ts"; +import type { ChordStore } from "./store.ts"; + +/** The rows and documents an export reads per page. */ +export const EXPORT_PAGE = 100; + +/** + * An export of `store` under `context`: an async iterator of its parts. One pass; after an abort `next()` rejects + * `aborted`, after the store closed `released`. + */ +export class StoreExport implements AsyncIterableIterator { + readonly #context: Context; + /** The parts, until the export ends; dropped then, with the pages they hold. */ + #parts: AsyncGenerator | undefined; + #state: "open" | "aborted" | "closed" | "ended" = "open"; + /** Gives the commit queue back while the export holds it. */ + #release: (() => void) | undefined; + readonly #abort = (): void => this.#end("aborted"); + + constructor(store: ChordStore, context: Context) { + this.#context = context; + const hold = (release: () => void): void => this.#hold(release); + this.#parts = parts(store, context, hold, () => this.#end("closed")); + } + + [Symbol.asyncIterator](): this { + return this; + } + + async next(): Promise> { + if (this.#context.abortSignal?.aborted) this.#end("aborted"); + this.#assertOpen(); + if (this.#parts === undefined) return { done: true, value: undefined }; + const pending = this.#parts.next(); + let result: IteratorResult; + try { + // Until the export holds the queue only `context` ends the wait; from then on its abort listener does. + result = await (this.#release === undefined ? awaitWithContext(pending, this.#context) : pending); + } catch (error) { + if (this.#context.abortSignal?.aborted) this.#end("aborted"); + this.#assertOpen(); + this.#end("ended"); + throw error; + } + // Ended meanwhile (an abort, the store's close): no part after that, even one that needed no read. + this.#assertOpen(); + if (result.done) this.#end("ended"); + return result; + } + + async return(): Promise> { + this.#end("ended"); + return { done: true, value: undefined }; + } + + /** The parts took the queue: the export holds it until it ends, and an abort ends it; ended already, it gives it back. */ + #hold(release: () => void): void { + if (this.#state !== "open") { + release(); + return; + } + this.#release = release; + this.#context.abortSignal?.addEventListener("abort", this.#abort, { once: true }); + } + + /** Ends the export once: the queue given back, the parts closed (once they reach a `yield`) and dropped. */ + #end(state: "aborted" | "closed" | "ended"): void { + if (this.#state !== "open") return; + this.#state = state; + this.#release?.(); + this.#release = undefined; + this.#context.abortSignal?.removeEventListener("abort", this.#abort); + void this.#parts?.return(undefined).catch(() => {}); + this.#parts = undefined; + } + + /** Throws `aborted` after an abort, `released` after the store closed. */ + #assertOpen(): void { + if (this.#state === "aborted") throw aborted(this.#context); + if (this.#state === "closed") + throw new ChordError({ code: "released", reason: "closed", message: "store: closed" }); + } +} + +/** The error of a use of an aborted export: `aborted`, the signal's reason as the cause. */ +function aborted(context: Context): ChordError { + return new ChordError({ code: "aborted", message: "store: export aborted" }, { cause: context.abortSignal?.reason }); +} + +/** + * The parts in archive order, after taking the queue (handed to `hold`; the store's close calls `end`); the queue is + * given back when they end. + */ +async function* parts( + store: ChordStore, + context: Context, + hold: (release: () => void) => void, + end: () => void, +): AsyncGenerator { + const release = await store.take(end); + hold(release); + try { + if (context.abortSignal?.aborted) throw aborted(context); + const { lastSeq, nextDocumentId } = store; + yield { kind: "manifest", format: ARCHIVE_FORMAT, schema: STORE_SCHEMA, lastSeq, nextDocumentId }; + let writes = 0; + const tables = [...store.tables.values()].sort((a, b) => (a.table < b.table ? -1 : 1)); + for (const record of tables) { + const { table, version, state, columns, key, fingerprint } = record; + yield { kind: "table", table, version, state, columns, key, fingerprint }; + writes++; + for await (const put of puts(store, record, context)) { + yield put; + writes++; + } + for (const [name, on] of Object.entries(record.indexes)) { + yield { kind: "index", table, name, columns: on }; + writes++; + } + } + for await (const create of documents(store, context)) { + yield create; + writes++; + } + yield { kind: "end", writes }; + } finally { + release(); + } +} + +/** The `put`s of a recorded table in key order, a page per read: the stored values and their versions. */ +async function* puts( + store: ChordStore, + record: TableRecord, + context: Context, +): AsyncGenerator> { + const { table, columns, key } = record; + const select = recordedPut(table, columns) + .physical.map((name) => quote(name)) + .join(", "); + const order = key.map(quote).join(", "); + let after: SqliteValue[] | undefined; + do { + const page = after; + const from = page === undefined ? "" : ` WHERE (${order}) > (${page.map(() => "?").join(", ")})`; + const sql = `SELECT ${select} FROM ${quote(table)}${from} ORDER BY ${order} LIMIT ?`; + const rows = await store.read( + (executor) => executor.all<{ readonly [column: string]: SqliteValue }>(sql, ...(page ?? []), EXPORT_PAGE), + context, + ); + for (const stored of rows) yield putOf(record, stored); + after = rows.length < EXPORT_PAGE ? undefined : key.map((name) => rows.at(-1)![name]!); + } while (after !== undefined); +} + +/** A stored row as its `put`: `NULL` cells absent, booleans from 0 and 1, `json(...)` cells parsed with versions. */ +function putOf( + record: TableRecord, + stored: { readonly [column: string]: SqliteValue }, +): Extract { + const row: { [column: string]: Scalar | JsonValue } = {}; + const versions: { [column: string]: number } = {}; + for (const [name, column] of Object.entries(record.columns)) { + const value = stored[name]; + if (value === null || value === undefined) continue; + // The recorded columns' types say what SQLite stored (STRICT tables): text for `json(...)` with its version. + if (column.type === "bool") row[name] = value === 1; + else if (column.type === "json") { + row[name] = JSON.parse(value as string) as JsonValue; + versions[name] = stored[versionColumn(name)] as number; + } else row[name] = value as Scalar; + } + return { kind: "put", table: record.table, row, version: stored[ROW_VERSION] as number, columns: versions }; +} + +/** The live documents by id as their creations: the record and the current value as the base. */ +async function* documents( + store: ChordStore, + context: Context, +): AsyncGenerator> { + let after = 0; + for (;;) { + const records = await store.read((executor) => listDocuments(executor, after, EXPORT_PAGE), context); + for (const record of records) { + const { value } = await store.read((executor) => loadDocument(executor, record), context); + const { version, fingerprint } = record; + yield { kind: "document.create", record, base: { version, fingerprint, value } }; + } + if (records.length < EXPORT_PAGE) return; + after = records.at(-1)!.id; + } +} diff --git a/packages/chord/src/next/store/expression.ts b/packages/chord/src/next/store/expression.ts new file mode 100644 index 000000000..f62f47cf4 --- /dev/null +++ b/packages/chord/src/next/store/expression.ts @@ -0,0 +1,356 @@ +// The table expression language (slice 6a §6, §7; rules 10, 13, 14, 16; decision "Nullable columns and the `set` +// form"): the builders plugins write `where` and `set` with, and their compilation at staging into one data-only +// vocabulary (what the outbox and archives carry; `evaluate.ts` runs it in JavaScript, `sql.ts` renders it). + +import type { TOptional } from "typebox"; +import type { JsonValue } from "../../json.ts"; +import type { TSchema } from "../contracts/index.ts"; +import { invalid } from "../data/internal.ts"; +import type { Column, Columns, ColumnType, ColumnTypeOf, ValueOf } from "./columns.ts"; +import { checkScalar, encodeCell, type Scalar } from "./rows.ts"; +import type { TableDefinition } from "./table.ts"; + +/** A compiled condition: column comparisons with checked operands, combined by `and`, `or`, `not`. */ +export type WhereNode = + | { readonly eq: readonly [string, Scalar] } + | { readonly ne: readonly [string, Scalar] } + | { readonly lt: readonly [string, Scalar] } + | { readonly lte: readonly [string, Scalar] } + | { readonly gt: readonly [string, Scalar] } + | { readonly gte: readonly [string, Scalar] } + | { readonly inArray: readonly [string, readonly Scalar[]] } + | { readonly isNull: string } + | { readonly prefix: readonly [string, string] } + | { readonly and: readonly WhereNode[] } + | { readonly or: readonly WhereNode[] } + | { readonly not: WhereNode }; + +/** A compiled value expression: a literal (`null` clears), a column, a `json(...)` cell, or an operation. */ +export type ExprNode = + | Scalar + | null + | { readonly col: string } + | { readonly json: JsonValue } + | { readonly add: readonly [ExprNode, ExprNode] } + | { readonly sub: readonly [ExprNode, ExprNode] } + | { readonly max: readonly [ExprNode, ExprNode] } + | { readonly min: readonly [ExprNode, ExprNode] } + | { readonly concat: readonly [ExprNode, ExprNode] } + | { readonly coalesce: readonly [ExprNode, ExprNode] } + | { readonly replacePrefix: readonly [ExprNode, string, string] } + | { readonly when: readonly [WhereNode, ExprNode, ExprNode] }; + +/** A compiled `set`: the new value of each changed column, every right side evaluated against the row before. */ +export type SetNode = { readonly [column: string]: ExprNode }; + +/** The comparison operators of a `WhereNode`. */ +export type Comparison = "eq" | "ne" | "lt" | "lte" | "gt" | "gte"; + +/** The operations of an `ExprNode`. */ +export type Operation = "add" | "sub" | "max" | "min" | "concat" | "coalesce" | "replacePrefix" | "when"; + +/** A value expression as a plugin builds it: `T` is its value, `K` its column type. Compiled when staged. */ +export class Expression { + readonly op: Operation | "col"; + readonly args: readonly unknown[]; + /** Type only. */ + declare readonly "~value"?: T; + declare readonly "~type"?: K; + + constructor(op: Operation | "col", args: readonly unknown[]) { + this.op = op; + this.args = args; + } +} + +/** A reference to a column of the row, as `(t) => …` callbacks get them. */ +export class ColumnRef extends Expression { + readonly column: string; + + constructor(column: string) { + super("col", [column]); + this.column = column; + } +} + +/** A condition as a plugin builds it; compiled when staged or run. */ +export class Condition { + readonly op: Comparison | "inArray" | "isNull" | "prefix" | "and" | "or" | "not"; + readonly args: readonly unknown[]; + + constructor(op: Condition["op"], args: readonly unknown[]) { + this.op = op; + this.args = args; + } +} + +/** + * A `set` of a table with columns `C` and key `K`: new values of non-key columns, each a value or an expression of + * the column's type; `undefined` clears an `opt(...)` column. + */ +export type SetOf = { + readonly [P in Exclude]?: C[P] extends TOptional + ? ValueOf | Expression, ColumnTypeOf> | undefined + : ValueOf | Expression, ColumnTypeOf>; +}; + +/** A scalar column type. */ +type ScalarType = Exclude; + +/** An operand of type `T` and column type `K`: an expression or a plain value. */ +type Operand = Expression | T; + +/** The column equals `value` (`NULL` never matches). */ +export function eq(column: ColumnRef, value: T): Condition { + return new Condition("eq", [column, value]); +} + +/** The column differs from `value` (`NULL` never matches). */ +export function ne(column: ColumnRef, value: T): Condition { + return new Condition("ne", [column, value]); +} + +/** The column is below `value`. */ +export function lt(column: ColumnRef, value: T): Condition { + return new Condition("lt", [column, value]); +} + +/** The column is at most `value`. */ +export function lte(column: ColumnRef, value: T): Condition { + return new Condition("lte", [column, value]); +} + +/** The column is above `value`. */ +export function gt(column: ColumnRef, value: T): Condition { + return new Condition("gt", [column, value]); +} + +/** The column is at least `value`. */ +export function gte(column: ColumnRef, value: T): Condition { + return new Condition("gte", [column, value]); +} + +/** The column equals one of `values` (`[]` matches nothing). */ +export function inArray(column: ColumnRef, values: readonly T[]): Condition { + return new Condition("inArray", [column, values]); +} + +/** The column is `NULL` (an `opt(...)` column without a value). */ +export function isNull(column: ColumnRef): Condition { + return new Condition("isNull", [column]); +} + +/** The text column starts with `prefix`, compared by code point (`""` matches every value). */ +export function prefix(column: ColumnRef, prefix: string): Condition { + return new Condition("prefix", [column, prefix]); +} + +/** Every condition holds (`and()` always does). */ +export function and(...conditions: Condition[]): Condition { + return new Condition("and", conditions); +} + +/** Some condition holds (`or()` never does). */ +export function or(...conditions: Condition[]): Condition { + return new Condition("or", conditions); +} + +/** The condition does not hold (`NULL` stays `NULL`: it never matches). */ +export function not(condition: Condition): Condition { + return new Condition("not", [condition]); +} + +/** `a + b` over integers; a result outside ±(2^53 - 1) fails (rule 10). */ +export function add(a: Operand, b: Operand): Expression { + return new Expression("add", [a, b]); +} + +/** `a - b` over integers; a result outside ±(2^53 - 1) fails (rule 10). */ +export function sub(a: Operand, b: Operand): Expression { + return new Expression("sub", [a, b]); +} + +/** The larger of `a` and b` (`int` or `num`). */ +export function max(a: Operand, b: Operand): Expression { + return new Expression("max", [a, b]); +} + +/** The smaller of `a` and `b` (`int` or `num`). */ +export function min(a: Operand, b: Operand): Expression { + return new Expression("min", [a, b]); +} + +/** `a` followed by `b`. */ +export function concat(a: Operand, b: Operand): Expression { + return new Expression("concat", [a, b]); +} + +/** `value` with its prefix `from` replaced by `to`; a value without the prefix stays as it is. */ +export function replacePrefix(value: Operand, from: string, to: string): Expression { + return new Expression("replacePrefix", [value, from, to]); +} + +/** `a` unless it is `NULL`, else `b`. */ +export function coalesce(a: Expression, b: Operand): Expression { + return new Expression("coalesce", [a, b]); +} + +/** `a` where `condition` holds for the row, else `b`. */ +export function when( + condition: Condition, + a: Operand, + b: Operand, +): Expression { + return new Expression("when", [condition, a, b]); +} + +/** One `ColumnRef` per column of `definition`, for its `(t) => …` callbacks. */ +export function columnRefs(columns: readonly Column[]): { readonly [column: string]: ColumnRef } { + return Object.fromEntries(columns.map(({ name }) => [name, new ColumnRef(name)])); +} + +/** + * Compiles a plugin's condition over `table`'s columns (a trust boundary: plugin values): known scalar columns, + * operands of their type made storable (`invalid/args` otherwise, `undefined` included: ask with `isNull`, rule 16). + */ +export function compileWhere(table: TableDefinition, condition: unknown, what: string): WhereNode { + if (!(condition instanceof Condition)) throw invalid("args", `${table.name}: ${what}: expected a condition`); + const { op, args } = condition; + switch (op) { + case "and": + case "or": { + const nodes = args.map((each) => compileWhere(table, each, what)); + return op === "and" ? { and: nodes } : { or: nodes }; + } + case "not": + return { not: compileWhere(table, args[0], what) }; + case "isNull": + return { isNull: scalarColumn(table, args[0], what).name }; + case "prefix": { + const column = scalarColumn(table, args[0], what); + if (column.type !== "str") throw invalid("args", `${table.name}: ${what}: prefix needs a str() column`); + return { prefix: [column.name, operand(column, args[1], what) as string] }; + } + case "inArray": { + const column = scalarColumn(table, args[0], what); + const values = args[1]; + if (!Array.isArray(values)) throw invalid("args", `${table.name}: ${what}: inArray needs an array`); + return { inArray: [column.name, values.map((value) => operand(column, value, what))] }; + } + default: { + const column = scalarColumn(table, args[0], what); + // One of the six comparisons: the node's shape is `{ [op]: [column, value] }`. + return { [op]: [column.name, operand(column, args[1], what)] as const } as WhereNode; + } + } +} + +/** + * Compiles a plugin's `set` (plain values or a callback's result) for `table` (a trust boundary): known non-key + * columns (rule 13), at least one (rule 14), `undefined` clearing an `opt(...)` column as the explicit `null` (a + * `null` value is a `json(...)` cell's JSON `null`, refused for other columns), an + * expression whose result may be `NULL` only for an `opt(...)` column; integer arithmetic only on `int` columns. + */ +export function compileSet(table: TableDefinition, set: unknown, what: string): SetNode { + if (typeof set !== "object" || set === null || Array.isArray(set)) { + throw invalid("args", `${table.name}: ${what}: expected an object of column values`); + } + const entries = Object.entries(set); + if (entries.length === 0) throw invalid("args", `${table.name}: ${what}: the set is empty`); + const node: { [column: string]: ExprNode } = {}; + for (const [name, value] of entries) { + const column = table.byName.get(name); + if (column === undefined) throw invalid("args", `${table.name}: ${what}: no column "${name}"`); + if (table.key.includes(name)) { + throw invalid("args", `${table.name}: ${what}: key column "${name}" cannot change (delete and put)`); + } + const where = `${what}: column "${name}"`; + if (value === undefined) { + if (!column.optional) throw invalid("args", `${table.name}: ${where} is not opt(...) and cannot be cleared`); + node[name] = null; + } else if (value === null && column.type !== "json") { + // A row never holds an explicit `null` (decision "Nullable columns and the set form"); a JSON `null` is a value. + throw invalid("args", `${table.name}: ${where}: null is no ${column.type} value (undefined clears)`); + } else if (value instanceof Expression) { + if (column.type === "json") + throw invalid("args", `${table.name}: ${where}: json(...) columns take plain values`); + const compiled = compileExpr(table, value, column.type, where); + if (compiled.nullable && !column.optional) { + throw invalid("args", `${table.name}: ${where}: the expression may be NULL; the column is not opt(...)`); + } + node[name] = compiled.node; + } else if (column.type === "json") { + node[name] = { json: encodeCell(column, value, `${table.name}: ${where}`).stored }; + } else node[name] = checkScalar(column.type, value, `${table.name}: ${where}`); + } + return node; +} + +/** A compiled expression and whether its result may be `NULL`. */ +type Compiled = { readonly node: ExprNode; readonly nullable: boolean }; + +function compileExpr(table: TableDefinition, value: unknown, type: ScalarType, what: string): Compiled { + if (!(value instanceof Expression)) { + if (value === undefined || value === null) return { node: null, nullable: true }; + return { node: checkScalar(type, value, `${table.name}: ${what}`), nullable: false }; + } + const { op, args } = value; + const refuse = (message: string): never => { + throw invalid("args", `${table.name}: ${what}: ${message}`); + }; + switch (op) { + case "col": { + const column = table.byName.get(args[0] as string) ?? refuse(`no column "${String(args[0])}"`); + if (column.type !== type) refuse(`column "${column.name}" is ${column.type}, not ${type}`); + return { node: { col: column.name }, nullable: column.optional }; + } + case "add": + case "sub": + case "max": + case "min": + case "concat": { + const types: readonly ScalarType[] = + op === "concat" ? ["str"] : op === "add" || op === "sub" ? ["int"] : ["int", "num"]; + if (!types.includes(type)) refuse(`${op} gives ${types.join(" or ")}, the column is ${type}`); + const [a, b] = [compileExpr(table, args[0], type, what), compileExpr(table, args[1], type, what)]; + // The operation's shape is `{ [op]: [a, b] }`. + return { node: { [op]: [a.node, b.node] } as ExprNode, nullable: a.nullable || b.nullable }; + } + case "coalesce": { + const [a, b] = [compileExpr(table, args[0], type, what), compileExpr(table, args[1], type, what)]; + return { node: { coalesce: [a.node, b.node] }, nullable: a.nullable && b.nullable }; + } + case "replacePrefix": { + if (type !== "str") refuse(`replacePrefix gives str, the column is ${type}`); + const subject = compileExpr(table, args[0], type, what); + const [from, to] = [args[1], args[2]].map( + (each) => checkScalar("str", each, `${table.name}: ${what}`) as string, + ); + return { node: { replacePrefix: [subject.node, from!, to!] }, nullable: subject.nullable }; + } + case "when": { + const condition = compileWhere(table, args[0], what); + const [a, b] = [compileExpr(table, args[1], type, what), compileExpr(table, args[2], type, what)]; + return { node: { when: [condition, a.node, b.node] }, nullable: a.nullable || b.nullable }; + } + } +} + +/** The column a condition compares: a reference to a known scalar column. */ +function scalarColumn(table: TableDefinition, ref: unknown, what: string): Column { + if (!(ref instanceof ColumnRef)) throw invalid("args", `${table.name}: ${what}: expected a column reference`); + const column = table.byName.get(ref.column); + if (column === undefined) throw invalid("args", `${table.name}: ${what}: no column "${ref.column}"`); + if (column.type === "json") + throw invalid("args", `${table.name}: ${what}: json(...) column "${ref.column}" is not queryable`); + return column; +} + +/** An operand compared with `column`, made storable; `undefined` and `null` are refused (rule 16). */ +function operand(column: Column, value: unknown, what: string): Scalar { + if (value === undefined || value === null) { + throw invalid("args", `${what}: column "${column.name}" compared with ${String(value)}; ask with isNull`); + } + // `scalarColumn` refused json columns. + return checkScalar(column.type as ScalarType, value, `${what}: column "${column.name}"`); +} diff --git a/packages/chord/src/next/store/import.ts b/packages/chord/src/next/store/import.ts new file mode 100644 index 000000000..c53ca4e77 --- /dev/null +++ b/packages/chord/src/next/store/import.ts @@ -0,0 +1,236 @@ +// `importStore(backend, parts, ctx, { outbox? })` (slice 6a §13, rules 12, 24, 25): a store made from an archive, a +// trust boundary. Every part is checked against the archive schema, its place in the archive's order and its content +// (names, rows against their table's recorded columns, ids and seqs, one live record per address, versions, +// duplicates, the `end` count, the manifest); everything is applied in one backend transaction with the applier, so a +// refused import leaves the backend as it was. An input that stalls is cancelled by `ctx`. + +import { awaitWithContext, type Context } from "../../context/index.ts"; +import type { SqliteDatabase, SqliteExecutor } from "../../sqlite/index.ts"; +import { ChordError } from "../data/index.ts"; +import { invalid } from "../data/internal.ts"; +import { applyWrite } from "./applier.ts"; +import { ARCHIVE_FORMAT, type ArchiveManifest, type ArchivePart, checkPart } from "./archive.ts"; +import type { Backend } from "./backend.ts"; +import { readCatalog, writeCatalog } from "./catalog.ts"; +import { applyStoreSchema, type MetaRow, STORE_SCHEMA, TABLES } from "./schema.ts"; +import { Step } from "./step.ts"; +import { ChordStore, type Store, type StoreOptions } from "./store.ts"; +import { storedText } from "./text.ts"; + +/** + * A store holding the archive `parts` on `backend`, which must not hold a store yet: at the manifest's `lastSeq`, its + * outbox (when `options.outbox`) starting after it. A refused archive, a stalled input cancelled by `context` or a + * backend error rejects; the backend is closed then (rule 12). + */ +export async function importStore( + backend: Backend, + parts: AsyncIterable | Iterable, + context: Context, + options: StoreOptions = {}, +): Promise { + const { db } = backend; + try { + context.abortSignal?.throwIfAborted(); + const existing = await db.get(`SELECT name FROM sqlite_master WHERE name = ?`, TABLES.meta); + if (existing !== undefined) throw invalid("args", "import: the backend holds a store already"); + const meta = await db.transaction(async (executor) => { + const meta = await applyStoreSchema(inTransaction(executor)); + const archive = new ArchiveImport(executor, context); + await each(parts, context, (part) => archive.take(part)); + return archive.finish(meta, options.outbox === true); + }); + return new ChordStore(db, meta, await readCatalog(db)); + } catch (error) { + try { + await db.close(); + } catch { + // The import's own failure is the one to report. + } + throw error; + } +} + +/** The store's schema migrations run inside the import's backend transaction (they open none of their own). */ +function inTransaction(executor: SqliteExecutor): SqliteDatabase { + return { + exec: (sql) => executor.exec(sql), + run: (sql, ...params) => executor.run(sql, ...params), + runCounted: (sql, ...params) => executor.runCounted(sql, ...params), + get: (sql, ...params) => executor.get(sql, ...params), + all: (sql, ...params) => executor.all(sql, ...params), + transaction: (callback) => callback(executor), + close: () => Promise.reject(new Error("import: the backend closes after the import")), + }; +} + +/** A place in the archive's order: before the manifest, in a table group's puts or indexes, the documents, the end. */ +type Place = "manifest" | "puts" | "indexes" | "documents" | "ended"; + +/** + * One archive being imported: the place reached in its order (the manifest, then table groups of a `table` write, its + * `put`s and its `index` writes, then the documents, then `end`), what was seen (tables, ids, live addresses, the + * write count) and the step its writes are applied in. + */ +class ArchiveImport { + readonly #executor: SqliteExecutor; + readonly #context: Context; + #position = 0; + #manifest: ArchiveManifest | undefined; + #step: Step | undefined; + /** Where the next part may go. */ + #place: Place = "manifest"; + /** The table whose group is open. */ + #table: string | undefined; + readonly #tables = new Set(); + readonly #ids = new Set(); + readonly #addresses = new Set(); + #writes = 0; + + constructor(executor: SqliteExecutor, context: Context) { + this.#executor = executor; + this.#context = context; + } + + /** Checks the next part and applies it. */ + async take(value: unknown): Promise { + const position = this.#position++; + const part = checkPart(value, position, this.#context); + const what = `import: part ${position} (${part.kind})`; + if (this.#place === "ended") throw invalid("args", `${what}: after the end`); + if (part.kind === "manifest" || this.#step === undefined) { + this.#start(part, what); + return; + } + if (part.kind === "end") { + if (part.writes !== this.#writes) { + throw invalid("args", `${what}: counts ${part.writes} writes, the archive has ${this.#writes}`); + } + this.#place = "ended"; + return; + } + this.#place = this.#placeOf(part, what); + this.#writes++; + await applyWrite(this.#step, part, what); + } + + /** The stored metadata once the archive ended: its `lastSeq`, next document id and the outbox setting. */ + async finish(meta: MetaRow, outbox: boolean): Promise { + if (this.#place !== "ended" || this.#step === undefined || this.#manifest === undefined) { + throw invalid("args", "import: the archive has no end part"); + } + const { lastSeq, nextDocumentId } = this.#manifest; + await writeCatalog(this.#step); + await this.#executor.run( + `UPDATE ${TABLES.meta} SET last_seq = ?, next_document_id = ?, outbox = ?, outbox_floor = ? WHERE singleton = 1`, + lastSeq, + nextDocumentId, + outbox ? 1 : 0, + lastSeq, + ); + return { + ...meta, + last_seq: lastSeq, + next_document_id: nextDocumentId, + outbox: outbox ? 1 : 0, + outbox_floor: lastSeq, + }; + } + + /** The manifest, first: a format and schema this code reads. */ + #start(part: ArchivePart, what: string): void { + if (part.kind !== "manifest" || this.#step !== undefined) + throw invalid("args", `${what}: the manifest comes first, once`); + if (part.format !== ARCHIVE_FORMAT) + throw invalid("args", `${what}: format ${part.format} is not ${ARCHIVE_FORMAT}`); + if (part.schema > STORE_SCHEMA) + throw invalid("args", `${what}: schema ${part.schema} is newer than ${STORE_SCHEMA}`); + this.#manifest = part; + this.#step = new Step(this.#executor, part.lastSeq, part.nextDocumentId, new Map()); + } + + /** Where `part` puts the archive, refused when out of order, a duplicate, or outside the manifest's ranges. */ + #placeOf(part: Exclude, what: string): Place { + const place = this.#place; + switch (part.kind) { + case "table": + if (place === "documents") throw invalid("args", `${what}: tables come before the documents`); + if (this.#tables.has(part.table)) throw invalid("args", `${what}: table ${part.table} occurs twice`); + this.#tables.add(part.table); + this.#table = part.table; + return "puts"; + case "put": + if (place !== "puts" || part.table !== this.#table) { + throw invalid("args", `${what}: a put follows its table's table write, before its indexes`); + } + return "puts"; + case "index": + if ((place !== "puts" && place !== "indexes") || part.table !== this.#table) { + throw invalid("args", `${what}: an index follows its table's table write and puts`); + } + return "indexes"; + case "document.create": + this.#checkDocument(part, what); + return "documents"; + } + } + + /** + * A record within the manifest's ids and seqs, unique, alone at its address, its base at its version; its address + * and meta well-formed text without NUL (SQLite would cut a NUL off, slice 6a §8). + */ + #checkDocument(part: Extract, what: string): void { + const { record, base } = part; + const { lastSeq, nextDocumentId } = this.#manifest!; + if (record.id >= nextDocumentId) throw invalid("args", `${what}: id ${record.id} is not below ${nextDocumentId}`); + if (this.#ids.has(record.id)) throw invalid("args", `${what}: id ${record.id} occurs twice`); + if (record.createdAt > lastSeq) throw invalid("args", `${what}: created after the manifest's lastSeq`); + if (base.version !== record.version || base.fingerprint !== record.fingerprint) { + throw invalid("args", `${what}: the base is not at the record's version and fingerprint`); + } + const { namespace, owner, kind, meta } = record; + // A singleton's record has no key; its column keeps `""`. + const key = record.key ?? ""; + for (const text of [namespace, owner, kind, key]) { + if (storedText(text, what) !== text) throw invalid("args", `${what}: an address is not well-formed text`); + } + if (meta !== undefined && storedText(meta, what) !== meta) { + throw invalid("args", `${what}: the meta is not well-formed text`); + } + const address = JSON.stringify([namespace, owner, kind, key]); + if (this.#addresses.has(address)) throw invalid("args", `${what}: a second live record at one address`); + this.#ids.add(record.id); + this.#addresses.add(address); + } +} + +/** Runs `take` on each part in turn; a wait for the next part ends when `context` aborts (the input is closed). */ +async function each( + parts: AsyncIterable | Iterable, + context: Context, + take: (part: unknown) => Promise, +): Promise { + const iterator = Symbol.asyncIterator in parts ? parts[Symbol.asyncIterator]() : parts[Symbol.iterator](); + let done = false; + try { + for (;;) { + let result: IteratorResult; + try { + result = await awaitWithContext(Promise.resolve(iterator.next()), context); + } catch (error) { + if (!context.abortSignal?.aborted) throw error; + throw new ChordError( + { code: "aborted", message: "import: aborted" }, + { cause: context.abortSignal.reason }, + ); + } + if (result.done) { + done = true; + return; + } + await take(result.value); + } + } finally { + // A stalled input is not awaited: the backend transaction ends now. + if (!done) void Promise.resolve(iterator.return?.()).catch(() => {}); + } +} diff --git a/packages/chord/src/next/store/index.ts b/packages/chord/src/next/store/index.ts new file mode 100644 index 000000000..57ee09a63 --- /dev/null +++ b/packages/chord/src/next/store/index.ts @@ -0,0 +1,49 @@ +// The store for apps: opening it over a backend, tables with their queries, writes and migrations, export and +// import of archives (with their JSONL encoding), the outbox, text order. + +export type { ArchiveManifest, ArchivePart } from "./archive.ts"; +export { type Backend, sqliteBackend } from "./backend.ts"; +export type { Columns, PutRowOf, RowOf } from "./columns.ts"; +export { + add, + and, + type ColumnRef, + type Condition, + coalesce, + concat, + type Expression, + eq, + gt, + gte, + inArray, + isNull, + lt, + lte, + max, + min, + ne, + not, + or, + prefix, + replacePrefix, + type SetOf, + sub, + when, +} from "./expression.ts"; +export { importStore } from "./import.ts"; +export { type JsonColumnOptions, type JsonColumnSchema, json } from "./json-column.ts"; +export { decodeJsonl, encodeJsonl } from "./jsonl.ts"; +export type { Outbox, OutboxCommit } from "./outbox.ts"; +export { type ColumnRefs, type Query, query } from "./query.ts"; +export { openStore, type Store, type StoreOptions } from "./store.ts"; +export { defineTable, type KeyOf, type TableDefinition, type TableOptions } from "./table.ts"; +export { + type AddedColumn, + evolveTable, + type PatchOf, + type TableEditor, + type TableTailEditor, +} from "./table-evolve.ts"; +export type { Page, PageOptions, Table, TableOf } from "./table-handle.ts"; +export { compareText } from "./text.ts"; +export type { StoreWrite } from "./writes.ts"; diff --git a/packages/chord/src/next/store/json-column.ts b/packages/chord/src/next/store/json-column.ts new file mode 100644 index 000000000..8e8a0a1c7 --- /dev/null +++ b/packages/chord/src/next/store/json-column.ts @@ -0,0 +1,78 @@ +// `json(schema, { version, migrate? })` table columns (decision "The `json(...)` column helper", W11): one JSON `TEXT` +// cell with its own version, typed when read, migrated on access (the result checked once, R8), never walked on write +// (R9). + +import Type, { type TUnsafe } from "typebox"; +import type { Context } from "../../context/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { Static, TSchema } from "../contracts/index.ts"; +import { fingerprintOf } from "../contracts/internal.ts"; +import { snapshotSchema } from "../contracts/snapshot.ts"; +import { ChordError } from "../data/index.ts"; +import { validatorFor } from "../data/internal.ts"; + +/** What `json(...)` takes besides the schema. */ +export type JsonColumnOptions = { + /** The version of the cells' shape; a cell stores the version it was written at. */ + readonly version: number; + /** Maps a cell of an older version `from` to this version's shape; pure. Without it older cells are refused. */ + readonly migrate?: (old: JsonValue, from: number) => T; +}; + +/** A `json(...)` column as `columns` takes it: a schema of the cells' type carrying the column. */ +export type JsonColumnSchema = TUnsafe & { readonly json: JsonColumn }; + +/** A column holding one JSON value of `schema` per row, at `version`; wrap it in `opt(...)` to make it nullable. */ +export function json(schema: S, options: JsonColumnOptions>): JsonColumnSchema> { + const column = new JsonColumn(schema, options.version, options.migrate); + // A TypeBox schema (so `opt(...)` and `Static` take it) whose enumerable `json` field `opt`'s clone keeps. + return Type.Unsafe>({ json: column }) as JsonColumnSchema>; +} + +/** A `json(...)` column's definition: its schema snapshot, version and fingerprint, and the read of a stored cell. */ +export class JsonColumn { + readonly schema: TSchema; + readonly version: number; + /** The fingerprint of the schema's normal form, recorded with the table: the same version with another is `broken`. */ + readonly fingerprint: string; + readonly #migrate: ((old: JsonValue, from: number) => unknown) | undefined; + + constructor(schema: TSchema, version: number, migrate: ((old: JsonValue, from: number) => unknown) | undefined) { + // Declaration policy: JavaScript callers reach this check. + if (!Number.isSafeInteger(version) || version < 1) + throw new Error(`json(...): version ${version} is not a positive integer`); + const snapshot = snapshotSchema(schema, "json(...)"); + this.schema = snapshot.schema; + this.version = version; + this.fingerprint = fingerprintOf(snapshot.normal); + this.#migrate = migrate; + } + + /** + * The value of a stored cell written at `version`: parsed; an older one migrated and checked once (R8, counted). A + * newer version, a missing or throwing `migrate` and an invalid result throw `unavailable` (`what` names the cell); + * the caller fails the owner (slice 6a §1.4), the data stays. + */ + read(text: string, version: number, what: string, context: Context): unknown { + const value = JSON.parse(text) as JsonValue; + if (version === this.version) return value; + if (version > this.version) throw refused(`${what}: stored at version ${version}, newer than ${this.version}`); + if (this.#migrate === undefined) throw refused(`${what}: no migrate from version ${version} to ${this.version}`); + let migrated: unknown; + try { + migrated = this.#migrate(value, version); + } catch (error) { + throw refused(`${what}: migrate from version ${version} threw`, error); + } + context.telemetry.metrics?.counter("chord.data.validations").add(1); + if (!validatorFor(this, context).Check(migrated)) { + throw refused(`${what}: migrate from version ${version} does not match the schema`); + } + return migrated; + } +} + +/** The error of a cell the code cannot read: the owner fails `broken` with it. */ +function refused(message: string, cause?: unknown): ChordError { + return new ChordError({ code: "unavailable", message }, cause === undefined ? undefined : { cause }); +} diff --git a/packages/chord/src/next/store/jsonl.ts b/packages/chord/src/next/store/jsonl.ts new file mode 100644 index 000000000..302f43dce --- /dev/null +++ b/packages/chord/src/next/store/jsonl.ts @@ -0,0 +1,60 @@ +// Plain JSONL for archives (slice 6a §13): one part per line, UTF-8. The store yields and takes parts; this is the +// encoding it ships (others, compressed ones, come later without touching the applier). + +import { invalid } from "../data/internal.ts"; +import type { ArchivePart } from "./archive.ts"; + +/** The bytes a chunk of encoded lines collects before it is yielded. */ +const CHUNK = 64 * 1024; + +/** `parts` as JSONL bytes, in chunks of whole lines. */ +export async function* encodeJsonl(parts: AsyncIterable): AsyncIterable { + const encoder = new TextEncoder(); + let lines = ""; + for await (const part of parts) { + lines += `${JSON.stringify(part)}\n`; + if (lines.length >= CHUNK) { + yield encoder.encode(lines); + lines = ""; + } + } + if (lines.length > 0) yield encoder.encode(lines); +} + +/** + * The values of JSONL `bytes`, one per nonempty line, for `importStore` to check (a trust boundary: bytes that are + * no UTF-8 and a line that is no JSON are `invalid/args`). + */ +export async function* decodeJsonl(bytes: AsyncIterable): AsyncIterable { + const decoder = new TextDecoder("utf-8", { fatal: true }); + let rest = ""; + let line = 0; + const parse = (text: string): unknown => { + try { + return JSON.parse(text); + } catch { + throw invalid("args", `jsonl: line ${line} is not JSON`); + } + }; + const decode = (chunk?: Uint8Array): string => { + try { + return chunk === undefined ? decoder.decode() : decoder.decode(chunk, { stream: true }); + } catch { + throw invalid("args", `jsonl: line ${line + 1} is not UTF-8`); + } + }; + for await (const chunk of bytes) { + rest += decode(chunk); + let start = 0; + for (let end = rest.indexOf("\n"); end >= 0; end = rest.indexOf("\n", start)) { + const text = rest.slice(start, end); + start = end + 1; + line++; + if (text.trim().length > 0) yield parse(text); + } + rest = rest.slice(start); + } + rest += decode(); + line++; + if (rest.trim().length > 0) yield parse(rest); +} diff --git a/packages/chord/src/next/store/outbox.ts b/packages/chord/src/next/store/outbox.ts new file mode 100644 index 000000000..642feba73 --- /dev/null +++ b/packages/chord/src/next/store/outbox.ts @@ -0,0 +1,100 @@ +// The outbox (slice 6a §2; decisions "The outbox is optional, internal and versioned", small items 3-4): the store's +// commits as the sync layer reads them, one `{ seq, writes }` per commit from its floor on, in the current format. + +import type { Context } from "../../context/index.ts"; +import { ChordError } from "../data/index.ts"; +import { invalid } from "../data/internal.ts"; +import { TABLES } from "./schema.ts"; +import type { ChordStore } from "./store.ts"; +import type { StoreWrite } from "./writes.ts"; + +/** The write vocabulary's version an outbox commit is stored in; a newer one is refused, data intact. */ +export const OUTBOX_FORMAT = 1; + +/** One commit as the outbox gives it. */ +export type OutboxCommit = { readonly seq: number; readonly writes: readonly StoreWrite[] }; + +const NOTHING = (): void => {}; + +/** A store's outbox: its commits after the floor (the seq it was turned on at, advanced by `trim`). */ +export class Outbox { + readonly #store: ChordStore; + #floor: number; + /** + * The outbox's reads and trims, one at a time: a read checks the floor and reads with no trim running (a trim's + * commit may have deleted rows before its floor is set), a trim waits for the reads in flight. + */ + #queue: Promise = Promise.resolve(); + + constructor(store: ChordStore, floor: number) { + this.#store = store; + this.#floor = floor; + } + + /** The seq the outbox starts after: `read` takes any `afterSeq` from it to `lastSeq`. */ + get floor(): number { + return this.#floor; + } + + /** Up to `limit` commits after `afterSeq`, in seq order; `afterSeq` outside floor..lastSeq is `invalid/args`. */ + async read(afterSeq: number, limit: number, context: Context): Promise { + if (!Number.isSafeInteger(afterSeq)) throw invalid("args", `outbox: read after ${afterSeq}, not an integer`); + if (!Number.isSafeInteger(limit) || limit < 1) throw invalid("args", "outbox: limit is a positive integer"); + return this.#queued(() => this.#read(afterSeq, limit, context)); + } + + /** + * Drops the commits through `throughSeq` and advances the floor; idempotent; beyond `lastSeq` is `invalid/args`. + * Trims run one at a time, so the stored floor only rises and equals the one in memory. + */ + trim(throughSeq: number, context: Context): Promise { + try { + context.abortSignal?.throwIfAborted(); + if (!Number.isSafeInteger(throughSeq) || throughSeq > this.#store.lastSeq) { + throw invalid("args", `outbox: trim through ${throughSeq}, beyond ${this.#store.lastSeq}`); + } + } catch (error) { + return Promise.reject(error); + } + return this.#queued(() => this.#trim(throughSeq)); + } + + /** Runs `operation` once the outbox operations before it settled. */ + #queued(operation: () => Promise): Promise { + const run = this.#queue.then(operation); + this.#queue = run.then(NOTHING, NOTHING); + return run; + } + + /** `read`, its turn come: the floor as no trim moves it. */ + async #read(afterSeq: number, limit: number, context: Context): Promise { + if (afterSeq < this.#floor || afterSeq > this.#store.lastSeq) { + throw invalid("args", `outbox: read after ${afterSeq}, outside ${this.#floor}..${this.#store.lastSeq}`); + } + const rows = await this.#store.read( + (executor) => + executor.all<{ readonly seq: number; readonly format: number; readonly writes: string }>( + `SELECT seq, format, writes FROM ${TABLES.outbox} WHERE seq > ? ORDER BY seq LIMIT ?`, + afterSeq, + limit, + ), + context, + ); + return rows.map(({ seq, format, writes }) => { + if (format !== OUTBOX_FORMAT) { + const message = `outbox: commit ${seq} is in format ${format}, newer than ${OUTBOX_FORMAT}`; + throw new ChordError({ code: "unavailable", message }); + } + return { seq, writes: JSON.parse(writes) as StoreWrite[] }; + }); + } + + async #trim(throughSeq: number): Promise { + if (throughSeq <= this.#floor) return; + await this.#store.maintain(async (executor) => { + await executor.run(`DELETE FROM ${TABLES.outbox} WHERE seq <= ?`, throughSeq); + await executor.run(`UPDATE ${TABLES.meta} SET outbox_floor = ? WHERE singleton = 1`, throughSeq); + }); + this.#floor = throughSeq; + } +} diff --git a/packages/chord/src/next/store/query.ts b/packages/chord/src/next/store/query.ts new file mode 100644 index 000000000..e4e9c7f81 --- /dev/null +++ b/packages/chord/src/next/store/query.ts @@ -0,0 +1,118 @@ +// Queries (decision "Query runner"; slice 6a §6, rule 17): `query(Table).where(…).orderBy(…)` as immutable data +// (the condition compiled when given, the order with the key appended as the tie-break), and the opaque cursors that +// continue a page, bound to the table, the order columns and the direction. + +import type { SqliteValue } from "../../sqlite/index.ts"; +import { invalid } from "../data/internal.ts"; +import type { Columns, ColumnTypeOf, ValueOf } from "./columns.ts"; +import { type ColumnRef, type Condition, compileWhere, type WhereNode } from "./expression.ts"; +import { checkScalar } from "./rows.ts"; +import type { Order } from "./sql.ts"; +import type { TableDefinition } from "./table.ts"; + +/** The column references a table's callbacks get: `e.path`, `e.at`. */ +export type ColumnRefs = { + readonly [K in keyof C & string]: ColumnRef, ColumnTypeOf>; +}; + +/** A query of `table`: its rows, in key order. */ +export function query(table: TableDefinition): Query { + return new Query(table, undefined, [], undefined); +} + +/** + * An immutable query of one table: an optional condition (several `where`s are `and`ed) and an order (`orderBy` + * columns, one direction, the key appended unless already there). + */ +export class Query { + readonly table: TableDefinition; + readonly condition: WhereNode | undefined; + readonly #orderBy: readonly string[]; + readonly #direction: "asc" | "desc" | undefined; + + constructor( + table: TableDefinition, + condition: WhereNode | undefined, + orderBy: readonly string[], + direction: "asc" | "desc" | undefined, + ) { + this.table = table; + this.condition = condition; + this.#orderBy = orderBy; + this.#direction = direction; + } + + /** This query with `fn`'s condition too. */ + where(fn: (row: ColumnRefs) => Condition): Query { + // The refs are the table's columns, typed by `C`. + const compiled = compileWhere(this.table, fn(this.table.refs as ColumnRefs), "where"); + const condition = this.condition === undefined ? compiled : { and: [this.condition, compiled] }; + return new Query(this.table, condition, this.#orderBy, this.#direction); + } + + /** + * This query ordered by `fn`'s column next (a non-`opt` `str` or `int` one), in `direction`: one direction per + * query, no column twice (`invalid/args`). + */ + orderBy( + fn: (row: ColumnRefs) => ColumnRef, + direction: "asc" | "desc" = "asc", + ): Query { + const table = this.table; + // The refs are `C`'s columns; what the plugin's callback returns is checked below. + const ref = fn(table.refs as ColumnRefs) as ColumnRef | undefined; + const column = table.byName.get(ref?.column ?? ""); + if (column === undefined || column.optional || (column.type !== "str" && column.type !== "int")) { + throw invalid("args", `${table.name}: orderBy: expected a non-opt str() or int() column`); + } + if (this.#orderBy.includes(column.name)) throw invalid("args", `${table.name}: orderBy: "${column.name}" twice`); + if (this.#direction !== undefined && this.#direction !== direction) { + throw invalid("args", `${table.name}: orderBy: one direction per query`); + } + return new Query(table, this.condition, [...this.#orderBy, column.name], direction); + } + + /** The order of its rows: the `orderBy` columns, then the key columns not among them. */ + get order(): Order { + const columns = [...this.#orderBy, ...this.table.key.filter((column) => !this.#orderBy.includes(column))]; + return { columns, direction: this.#direction ?? "asc" }; + } +} + +/** The cursor of the position after a row with the order values `values` in the table named `table` (its SQL name). */ +export function encodeCursor(table: string, order: Order, values: readonly SqliteValue[]): string { + const json = JSON.stringify({ t: table, o: order.columns, d: order.direction, v: values }); + let binary = ""; + for (const byte of new TextEncoder().encode(json)) binary += String.fromCharCode(byte); + return btoa(binary).replaceAll("+", "-").replaceAll("/", "_").replace(/=+$/, ""); +} + +/** + * The order values of `cursor` (a trust boundary: an app hands it back): one made for `table`'s `order` (any + * condition) in the table named `sqlName` (another object's table of the same definition is another table), each value + * of its column's type; `invalid/args` otherwise. + */ +export function decodeCursor(table: TableDefinition, sqlName: string, order: Order, cursor: unknown): SqliteValue[] { + const refuse = (): never => { + throw invalid("args", `${table.name}: the cursor is not one of this table's order`); + }; + if (typeof cursor !== "string") return refuse(); + let decoded: unknown; + try { + const binary = atob(cursor.replaceAll("-", "+").replaceAll("_", "/")); + decoded = JSON.parse( + new TextDecoder("utf-8", { fatal: true }).decode(Uint8Array.from(binary, (c) => c.charCodeAt(0))), + ); + } catch { + return refuse(); + } + // Wire input: every field is checked below. + const { t, o, d, v } = (decoded ?? {}) as { t?: unknown; o?: unknown; d?: unknown; v?: unknown }; + if (t !== sqlName || d !== order.direction || JSON.stringify(o) !== JSON.stringify(order.columns)) refuse(); + if (!Array.isArray(v) || v.length !== order.columns.length) return refuse(); + return order.columns.map((name, index) => { + const column = table.byName.get(name)!; + // Order columns are non-opt `str` or `int` (checked when the query was built). + return checkScalar(column.type as "str" | "int", v[index], `${table.name}: cursor`) as string | number; + }); +} diff --git a/packages/chord/src/next/store/rows.ts b/packages/chord/src/next/store/rows.ts new file mode 100644 index 000000000..da1152f0d --- /dev/null +++ b/packages/chord/src/next/store/rows.ts @@ -0,0 +1,213 @@ +// Rows (slice 6a §5 "Values at write", §9): one compiled codec per table version between a plugin's rows and keys +// (a trust boundary, checked closed) and what the store binds and reads; `json(...)` cells serialized at staging +// (rule 8), never walked (R9), and migrated on access when read. + +import type { Context } from "../../context/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { SqliteValue } from "../../sqlite/index.ts"; +import { invalid } from "../data/internal.ts"; +import type { Column, ColumnType } from "./columns.ts"; +import { storedText } from "./text.ts"; + +/** A scalar cell as the store keeps it in rows, writes and operands: a well-formed string, a number or a boolean. */ +export type Scalar = string | number | boolean; + +/** A row as the outbox records it: the stored values, `opt(...)` columns absent when `NULL`. */ +export type StoredRow = { readonly [column: string]: Scalar | JsonValue }; + +/** A `put` ready for the step: its parameters in `physical` order and its record. */ +export type EncodedRow = { + readonly params: readonly SqliteValue[]; + readonly row: StoredRow; + /** Each written `json(...)` column's version. */ + readonly columns: { readonly [column: string]: number }; + /** The row's key identity, as `EncodedKey.id`. */ + readonly id: string; +}; + +/** A key ready for the store: its parameters in key order, its record and its identity within one table. */ +export type EncodedKey = { + readonly params: readonly SqliteValue[]; + readonly key: { readonly [column: string]: Scalar }; + /** `JSON.stringify` of the parameters: equal keys, equal ids. */ + readonly id: string; +}; + +/** The physical name of a `json(...)` column's version column, and of the row's eager-step version. */ +export function versionColumn(column: string): string { + return `${column}$v`; +} + +/** The row's eager-step version column (W11): set by `put` and eager migration steps only. */ +export const ROW_VERSION = "$v"; + +/** The physical column order of a stored row: the declared columns, each `json(...)` column's version, then `$v`. */ +export function physicalColumns(columns: readonly { readonly name: string; readonly type: ColumnType }[]): string[] { + return [ + ...columns.map(({ name }) => name), + ...columns.filter(({ type }) => type === "json").map(({ name }) => versionColumn(name)), + ROW_VERSION, + ]; +} + +/** A scalar as SQLite binds it: booleans as 0 and 1. */ +export function toSqlite(scalar: Scalar): string | number { + return scalar === true ? 1 : scalar === false ? 0 : scalar; +} + +/** + * A table version's codec: `put` rows and keys checked and encoded, stored rows decoded. `physical` is the column + * order of an encoded row (`physicalColumns`). + */ +export class RowCodec { + readonly physical: readonly string[]; + readonly #columns: readonly Column[]; + readonly #key: readonly Column[]; + readonly #json: readonly Extract[]; + readonly #names: ReadonlySet; + readonly #keyNames: ReadonlySet; + /** The positions of the key columns in `physical`. */ + readonly #keyPositions: readonly number[]; + readonly #table: string; + + constructor(columns: readonly Column[], key: readonly string[], table: string) { + this.#columns = columns; + this.#key = key.map((name) => columns.find((column) => column.name === name)!); + this.#json = columns.filter( + (column): column is Extract => column.type === "json", + ); + this.#names = new Set(columns.map(({ name }) => name)); + this.#keyNames = new Set(key); + this.#keyPositions = key.map((name) => columns.findIndex((column) => column.name === name)); + this.#table = table; + this.physical = physicalColumns(columns); + } + + /** + * Checks and encodes a `put` row of table version `version` (a trust boundary): a plain object with every + * non-`opt` column (a defaulted one filled in) and no other field; `invalid/args` otherwise. + */ + put(row: unknown, version: number): EncodedRow { + const what = `${this.#table}: put`; + const fields = this.#closed(row, this.#names, what); + const params: SqliteValue[] = []; + const stored: { [column: string]: Scalar | JsonValue } = {}; + const versions: { [column: string]: number } = {}; + for (const column of this.#columns) { + let value = fields[column.name]; + if (value === undefined && column.type !== "json") value = column.default; + if (value === undefined) { + if (!column.optional) throw invalid("args", `${what}: column "${column.name}" is missing`); + params.push(null); + continue; + } + const cell = encodeCell(column, value, `${what}: column "${column.name}"`); + params.push(cell.param); + stored[column.name] = cell.stored; + } + for (const column of this.#json) { + const present = Object.hasOwn(stored, column.name); + if (present) versions[column.name] = column.json.version; + params.push(present ? column.json.version : null); + } + params.push(version); + const id = JSON.stringify(this.#keyPositions.map((position) => params[position])); + return { params, row: stored, columns: versions, id }; + } + + /** Checks and encodes a key (a trust boundary): an object with exactly the key columns; `invalid/args` otherwise. */ + key(key: unknown, what: string): EncodedKey { + const fields = this.#closed(key, this.#keyNames, `${this.#table}: ${what}: the key`); + const params: SqliteValue[] = []; + const stored: { [column: string]: Scalar } = {}; + for (const column of this.#key) { + const value = fields[column.name]; + const where = `${this.#table}: ${what}: key column "${column.name}"`; + if (value === undefined) throw invalid("args", `${where} is missing`); + const cell = encodeScalar(column, value, where); + params.push(cell); + stored[column.name] = cell; + } + return { params, key: stored, id: JSON.stringify(params) }; + } + + /** + * A stored row as reads return it: booleans from 0 and 1, `opt(...)` columns absent when `NULL`, `json(...)` cells + * parsed and migrated on access (a cell the code cannot read throws, see `JsonColumn.read`). + */ + decode(stored: { readonly [column: string]: SqliteValue }, context: Context): { [column: string]: unknown } { + const row: { [column: string]: unknown } = {}; + for (const column of this.#columns) { + const value = stored[column.name]; + if (value === null || value === undefined) continue; + if (column.type === "bool") row[column.name] = value === 1; + else if (column.type === "json") { + const version = stored[versionColumn(column.name)] as number; + row[column.name] = column.json.read(value as string, version, `${this.#table}.${column.name}`, context); + } else row[column.name] = value; + } + return row; + } + + /** `value`'s own fields when it is a plain object with only fields of `names`; `invalid/args` otherwise. */ + #closed(value: unknown, names: ReadonlySet, what: string): { readonly [field: string]: unknown } { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw invalid("args", `${what}: expected an object`); + } + for (const field of Object.keys(value)) { + if (!names.has(field)) throw invalid("args", `${what}: unknown column "${field}"`); + } + // Checked above: a plain object with known fields. + return value as { readonly [field: string]: unknown }; + } +} + +/** A checked cell: what is bound and what is recorded. */ +type Cell = { readonly param: SqliteValue; readonly stored: Scalar | JsonValue }; + +/** Checks and encodes a value of `column`; a `json(...)` value is serialized, not walked (R9). */ +export function encodeCell(column: Column, value: unknown, what: string): Cell { + if (column.type !== "json") { + const stored = checkScalar(column.type, value, what); + return { param: toSqlite(stored), stored }; + } + let text: string | undefined; + try { + text = JSON.stringify(value); + } catch (error) { + throw invalid("args", `${what}: not JSON (${error instanceof Error ? error.message : String(error)})`); + } + if (text === undefined) throw invalid("args", `${what}: not JSON`); + // `JSON.stringify` accepted it; the recorded value is the one serialized. It is kept, not copied or parsed back: + // same-host values are immutable by convention, so the outbox serializing it again in the step records this text + // (a plugin mutating a value after `tx.put` breaks that convention, as for any other value it hands over). + return { param: text, stored: value as JsonValue }; +} + +/** Checks a scalar value of `column` and returns what is bound (booleans as 0 and 1). */ +function encodeScalar(column: Column, value: unknown, what: string): string | number { + if (column.type === "json") throw invalid("args", `${what}: a json(...) column has no scalar value`); + return toSqlite(checkScalar(column.type, value, what)); +} + +/** + * A value of a scalar column type as the store keeps it (a trust boundary): `str` well-formed without NUL, `int` a + * safe integer, `num` finite, `bool` a boolean; `invalid/args` otherwise. + */ +export function checkScalar(type: "str" | "int" | "num" | "bool", value: unknown, what: string): Scalar { + switch (type) { + case "str": + if (typeof value === "string") return storedText(value, what); + break; + case "int": + if (Number.isSafeInteger(value)) return value as number; + break; + case "num": + if (typeof value === "number" && Number.isFinite(value)) return value; + break; + case "bool": + if (typeof value === "boolean") return value; + break; + } + throw invalid("args", `${what}: ${JSON.stringify(value) ?? String(value)} is not a ${type} value`); +} diff --git a/packages/chord/src/next/store/schema.ts b/packages/chord/src/next/store/schema.ts new file mode 100644 index 000000000..c528360ce --- /dev/null +++ b/packages/chord/src/next/store/schema.ts @@ -0,0 +1,89 @@ +// The store's own SQL schema (slice 6a §2): its metadata, the recorded table definitions, the documents engine's +// records and revisions, and the outbox; versioned by `applySqliteMigrations`. Declared tables are named +// `"::

"`, so no name here can meet one of them. + +import { applySqliteMigrations, type SqliteDatabase, type SqliteMigration } from "../../sqlite/index.ts"; + +/** The store's tables. */ +export const TABLES = { + meta: "chord_meta", + tables: "chord_tables", + documents: "chord_documents", + revisions: "chord_revisions", + outbox: "chord_outbox", +} as const; + +/** + * The schema history; never change an entry, append one. `chord_meta` is one row: `last_seq`, the next document id + * (S10), and the outbox setting with its floor (the seq it starts after, advanced by `trim`). A document record keeps + * its current base's version and fingerprint and a `revision` counter for the conditional `UPDATE`; the partial + * unique index refuses two live records at one address. + */ +const MIGRATIONS: readonly SqliteMigration[] = [ + { + version: 1, + statements: [ + `CREATE TABLE ${TABLES.meta} ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + last_seq INTEGER NOT NULL, + next_document_id INTEGER NOT NULL, + outbox INTEGER NOT NULL CHECK (outbox IN (0, 1)), + outbox_floor INTEGER NOT NULL + ) STRICT`, + `INSERT INTO ${TABLES.meta} (singleton, last_seq, next_document_id, outbox, outbox_floor) VALUES (1, 0, 1, 0, 0)`, + `CREATE TABLE ${TABLES.tables} (name TEXT PRIMARY KEY, definition TEXT NOT NULL) STRICT, WITHOUT ROWID`, + `CREATE TABLE ${TABLES.documents} ( + id INTEGER PRIMARY KEY, + namespace TEXT NOT NULL, + owner TEXT NOT NULL, + kind TEXT NOT NULL, + key TEXT NOT NULL, + history TEXT NOT NULL CHECK (history IN ('latest', 'rewindable')), + created_at INTEGER NOT NULL, + retired_at INTEGER, + meta TEXT, + revision INTEGER NOT NULL, + version INTEGER NOT NULL, + fingerprint TEXT NOT NULL + ) STRICT`, + `CREATE UNIQUE INDEX chord_documents_live ON ${TABLES.documents} (namespace, owner, kind, key) + WHERE retired_at IS NULL`, + `CREATE TABLE ${TABLES.revisions} ( + document_id INTEGER NOT NULL, + seq INTEGER NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('base', 'delta')), + version INTEGER NOT NULL, + content TEXT NOT NULL, + PRIMARY KEY (document_id, seq) + ) STRICT, WITHOUT ROWID`, + `CREATE TABLE ${TABLES.outbox} (seq INTEGER PRIMARY KEY, format INTEGER NOT NULL, writes TEXT NOT NULL) STRICT`, + ], + }, +]; + +/** The store's schema version: the newest entry of its history (archives carry it). */ +export const STORE_SCHEMA = MIGRATIONS.at(-1)!.version; + +/** The store's metadata row, as `openStore` reads it once. */ +export type MetaRow = { + readonly last_seq: number; + readonly next_document_id: number; + readonly outbox: number; + readonly outbox_floor: number; +}; + +/** + * Brings `db` to the store's newest schema (one transaction) and checks that it stores text as UTF-8 (the store + * compares text by its bytes); returns the metadata row. + */ +export async function applyStoreSchema(db: SqliteDatabase): Promise { + const encoding = await db.get<{ readonly encoding: string }>("PRAGMA encoding"); + if (encoding?.encoding !== "UTF-8") + throw new Error(`store: the database encoding is ${encoding?.encoding}, not UTF-8`); + await applySqliteMigrations(db, MIGRATIONS, { table: "chord_store_schema" }); + const meta = await db.get( + `SELECT last_seq, next_document_id, outbox, outbox_floor FROM ${TABLES.meta} WHERE singleton = 1`, + ); + if (meta === undefined) throw new Error("store: the metadata row is missing"); + return meta; +} diff --git a/packages/chord/src/next/store/sql.ts b/packages/chord/src/next/store/sql.ts new file mode 100644 index 000000000..95529eb55 --- /dev/null +++ b/packages/chord/src/next/store/sql.ts @@ -0,0 +1,266 @@ +// SQL of declared tables (slice 6a §5, §6, §7): the DDL a table definition renders, and the statements of its reads +// and writes, generated from the compiled vocabulary with every value bound and every identifier quoted. Names were +// checked at declaration (and at import), so no `"` occurs in them and quoting is always safe. + +import type { SqliteValue } from "../../sqlite/index.ts"; +import type { Column, ColumnRecord } from "./columns.ts"; +import type { ExprNode, SetNode, WhereNode } from "./expression.ts"; +import { physicalColumns, ROW_VERSION, toSqlite, versionColumn } from "./rows.ts"; +import { prefixUpperBound } from "./text.ts"; + +/** A statement and its parameters. */ +export type Statement = { readonly sql: string; readonly params: SqliteValue[] }; + +/** An order of rows: the order columns (the key appended as the tie-break) and one direction. */ +export type Order = { readonly columns: readonly string[]; readonly direction: "asc" | "desc" }; + +/** A quoted SQL identifier. */ +export function quote(name: string): string { + return `"${name}"`; +} + +/** The SQL name of an object's table: `::
`, injective since no part contains `:`. */ +export function tableSqlName(hostKind: string, objectType: string, table: string): string { + return `${hostKind}:${objectType}:${table}`; +} + +/** + * `CREATE TABLE` for `columns` (as recorded) and `key`: `STRICT`, `WITHOUT ROWID`, the key as primary key, an `int` + * column checked within ±(2^53 - 1) (rule 10), a `bool` one within 0 and 1, a `json(...)` one as `TEXT` with its + * version column, and the row's `$v`. + */ +export function createTableSql( + table: string, + columns: { readonly [name: string]: ColumnRecord }, + key: readonly string[], +): string { + const definitions = Object.entries(columns).flatMap(([name, column]) => columnSql(name, column)); + definitions.push(`${quote(ROW_VERSION)} INTEGER NOT NULL`); + definitions.push(`PRIMARY KEY (${key.map(quote).join(", ")})`); + return `CREATE TABLE ${quote(table)} (${definitions.join(", ")}) STRICT, WITHOUT ROWID`; +} + +const SAFE = Number.MAX_SAFE_INTEGER; + +/** The definitions of one column: itself, and a `json(...)` column's version column. */ +export function columnSql(name: string, column: ColumnRecord): string[] { + const nullability = column.optional ? "" : " NOT NULL"; + const defaulted = column.default === undefined ? "" : ` DEFAULT ${literal(column.default)}`; + const id = quote(name); + switch (column.type) { + case "str": + return [`${id} TEXT${nullability}${defaulted}`]; + case "int": + return [`${id} INTEGER${nullability}${defaulted} CHECK (${id} BETWEEN ${-SAFE} AND ${SAFE})`]; + case "num": + return [`${id} REAL${nullability}${defaulted}`]; + case "bool": + return [`${id} INTEGER${nullability}${defaulted} CHECK (${id} IN (0, 1))`]; + case "json": + return [`${id} TEXT${nullability}`, `${quote(versionColumn(name))} INTEGER${nullability}`]; + } +} + +/** A default as a SQL literal: declaration data (checked), which DDL cannot bind. */ +function literal(value: string | number | boolean): string { + if (typeof value === "string") return `'${value.replaceAll("'", "''")}'`; + return typeof value === "boolean" ? (value ? "1" : "0") : String(value); +} + +/** + * The physical columns of a recorded table in the order `RowCodec.physical` uses (`physicalColumns`), and its + * whole-row `INSERT OR REPLACE` over them. + */ +export function recordedPut( + table: string, + columns: { readonly [name: string]: ColumnRecord }, +): { readonly physical: readonly string[]; readonly sql: string } { + const physical = physicalColumns(Object.entries(columns).map(([name, { type }]) => ({ name, type }))); + return { physical, sql: putSql(table, physical) }; +} + +/** `INSERT OR REPLACE` of a whole row over `physical` (`put` is an upsert). */ +function putSql(table: string, physical: readonly string[]): string { + return `INSERT OR REPLACE INTO ${quote(table)} (${physical.map(quote).join(", ")}) VALUES (${physical.map(() => "?").join(", ")})`; +} + +/** `CREATE INDEX` named `
:` over `columns`, the key appended (every index ends with it). */ +export function createIndexSql( + table: string, + index: string, + columns: readonly string[], + key: readonly string[], +): string { + const all = [...columns, ...key.filter((column) => !columns.includes(column))]; + return `CREATE INDEX ${quote(`${table}:${index}`)} ON ${quote(table)} (${all.map(quote).join(", ")})`; +} + +/** `DROP INDEX` of `
:`. */ +export function dropIndexSql(table: string, index: string): string { + return `DROP INDEX ${quote(`${table}:${index}`)}`; +} + +/** + * The statements of one table at one version, rendered once per table handle: the fixed ones as texts, the ones + * over compiled conditions and `set`s on demand. + */ +export class TableSql { + readonly table: string; + /** The columns a read selects: the declared ones and each `json(...)` column's version. */ + readonly #select: string; + readonly #byKey: string; + /** `INSERT OR REPLACE` of a whole row, in `RowCodec.physical`'s order (`put` is an upsert). */ + readonly put: string; + readonly get: string; + readonly delete: string; + + constructor(table: string, columns: readonly Column[], physical: readonly string[], key: readonly string[]) { + this.table = table; + const id = quote(table); + this.#select = [ + ...columns.map(({ name }) => quote(name)), + ...columns.filter(({ type }) => type === "json").map(({ name }) => quote(versionColumn(name))), + ].join(", "); + this.#byKey = key.map((column) => `${quote(column)} = ?`).join(" AND "); + this.put = putSql(table, physical); + this.get = `SELECT ${this.#select} FROM ${id} WHERE ${this.#byKey}`; + this.delete = `DELETE FROM ${id} WHERE ${this.#byKey}`; + } + + /** `UPDATE` of one row by key; `versions` gives each `json(...)` column's version to stamp. */ + update(set: SetNode, key: readonly SqliteValue[], versions: ReadonlyMap): Statement { + const params: SqliteValue[] = []; + const assignments = setSql(set, versions, params); + params.push(...key); + return { sql: `UPDATE ${quote(this.table)} SET ${assignments} WHERE ${this.#byKey}`, params }; + } + + /** `UPDATE` of every row matching `where`. */ + updateWhere(where: WhereNode, set: SetNode, versions: ReadonlyMap): Statement { + const params: SqliteValue[] = []; + const assignments = setSql(set, versions, params); + const condition = whereSql(where, params); + return { sql: `UPDATE ${quote(this.table)} SET ${assignments} WHERE ${condition}`, params }; + } + + /** `DELETE` of every row matching `where`. */ + deleteWhere(where: WhereNode): Statement { + const params: SqliteValue[] = []; + return { sql: `DELETE FROM ${quote(this.table)} WHERE ${whereSql(where, params)}`, params }; + } + + /** + * A page: rows matching `where` in `order`, after the order values `after` (row-value comparison; the key in the + * order makes it total), at most `limit` (bound). + */ + select( + where: WhereNode | undefined, + order: Order, + after: readonly SqliteValue[] | undefined, + limit: number, + ): Statement { + const params: SqliteValue[] = []; + const conditions: string[] = []; + if (where !== undefined) conditions.push(whereSql(where, params)); + if (after !== undefined) { + const columns = order.columns.map(quote).join(", "); + conditions.push(`(${columns}) ${order.direction === "asc" ? ">" : "<"} (${after.map(() => "?").join(", ")})`); + params.push(...after); + } + const filter = conditions.length === 0 ? "" : ` WHERE ${conditions.join(" AND ")}`; + const direction = order.direction === "asc" ? "ASC" : "DESC"; + const sorting = order.columns.map((column) => `${quote(column)} ${direction}`).join(", "); + params.push(limit); + return { sql: `SELECT ${this.#select} FROM ${quote(this.table)}${filter} ORDER BY ${sorting} LIMIT ?`, params }; + } + + /** `count(*)` of the rows matching `where`. */ + count(where: WhereNode | undefined): Statement { + const params: SqliteValue[] = []; + const filter = where === undefined ? "" : ` WHERE ${whereSql(where, params)}`; + return { sql: `SELECT count(*) AS count FROM ${quote(this.table)}${filter}`, params }; + } +} + +/** The assignments of `set`; a written `json(...)` column also stamps its version (a clear stamps `NULL`). */ +function setSql(set: SetNode, versions: ReadonlyMap, params: SqliteValue[]): string { + const assignments: string[] = []; + for (const [column, node] of Object.entries(set)) { + assignments.push(`${quote(column)} = ${exprSql(node, params)}`); + const version = versions.get(column); + if (version !== undefined) { + assignments.push(`${quote(versionColumn(column))} = ?`); + params.push(node === null ? null : version); + } + } + return assignments.join(", "); +} + +function exprSql(node: ExprNode, params: SqliteValue[]): string { + if (node === null) return "NULL"; + if (typeof node !== "object") { + params.push(toSqlite(node)); + return "?"; + } + if ("col" in node) return quote(node.col); + if ("json" in node) { + // Serialized once at staging already (rule 8): this cannot throw. + params.push(JSON.stringify(node.json)); + return "?"; + } + if ("when" in node) { + const [condition, a, b] = node.when; + return `(CASE WHEN ${whereSql(condition, params)} THEN ${exprSql(a, params)} ELSE ${exprSql(b, params)} END)`; + } + if ("replacePrefix" in node) { + const [subject, from, to] = node.replacePrefix; + const length = [...from].length; + // SQLite's `substr` counts characters of UTF-8 text: code points, as the prefix's length does. + // Parameters in the order their `?` occur in the text. + const value = exprSql(subject, params); + params.push(length, from, to); + const rest = exprSql(subject, params); + params.push(length + 1); + const unchanged = exprSql(subject, params); + return `(CASE WHEN substr(${value}, 1, ?) = ? THEN ? || substr(${rest}, ?) ELSE ${unchanged} END)`; + } + if ("coalesce" in node) + return `coalesce(${exprSql(node.coalesce[0], params)}, ${exprSql(node.coalesce[1], params)})`; + if ("add" in node) return `(${exprSql(node.add[0], params)} + ${exprSql(node.add[1], params)})`; + if ("sub" in node) return `(${exprSql(node.sub[0], params)} - ${exprSql(node.sub[1], params)})`; + if ("max" in node) return `max(${exprSql(node.max[0], params)}, ${exprSql(node.max[1], params)})`; + if ("min" in node) return `min(${exprSql(node.min[0], params)}, ${exprSql(node.min[1], params)})`; + return `(${exprSql(node.concat[0], params)} || ${exprSql(node.concat[1], params)})`; +} + +const COMPARISONS = { eq: "=", ne: "<>", lt: "<", lte: "<=", gt: ">", gte: ">=" } as const; + +/** A condition with its operands bound; SQL's `NULL` rules are the vocabulary's. */ +export function whereSql(node: WhereNode, params: SqliteValue[]): string { + if ("and" in node) + return node.and.length === 0 ? "1" : `(${node.and.map((each) => whereSql(each, params)).join(" AND ")})`; + if ("or" in node) + return node.or.length === 0 ? "0" : `(${node.or.map((each) => whereSql(each, params)).join(" OR ")})`; + if ("not" in node) return `(NOT ${whereSql(node.not, params)})`; + if ("isNull" in node) return `${quote(node.isNull)} IS NULL`; + if ("inArray" in node) { + const [column, values] = node.inArray; + params.push(JSON.stringify(values.map(toSqlite))); + return `${quote(column)} IN (SELECT value FROM json_each(?))`; + } + if ("prefix" in node) { + const [column, prefix] = node.prefix; + const upper = prefixUpperBound(prefix); + params.push(prefix); + if (upper === undefined) return `${quote(column)} >= ?`; + params.push(upper); + return `(${quote(column)} >= ? AND ${quote(column)} < ?)`; + } + // The remaining nodes are the comparisons, one entry each; `Object.entries` loses that. + const [op, [column, value]] = Object.entries(node)[0] as [ + keyof typeof COMPARISONS, + readonly [string, string | number | boolean], + ]; + params.push(toSqlite(value)); + return `${quote(column)} ${COMPARISONS[op]} ?`; +} diff --git a/packages/chord/src/next/store/step.ts b/packages/chord/src/next/store/step.ts new file mode 100644 index 000000000..8a428774a --- /dev/null +++ b/packages/chord/src/next/store/step.ts @@ -0,0 +1,69 @@ +// The commit step's working state (slice 6a §3) and the shape every write takes into it: a write staged before the +// queue (serialized, rule 8), applied inside the step in rule 9's order. + +import type { SqliteExecutor, SqliteValue } from "../../sqlite/index.ts"; +import type { TableRecord } from "./catalog.ts"; +import type { StoreWrite } from "./writes.ts"; + +/** Where a staged write goes in its commit (rule 9): retirements, creations, then everything else in order. */ +export type WriteOrder = "retire" | "create" | "write"; + +/** A write staged for the commit step: its order and what it does there, returning the write as recorded. */ +export type StagedWrite = { + readonly order: WriteOrder; + /** Applies the write inside the step's backend transaction; resolves with the write as the outbox records it. */ + apply(step: Step): Promise; +}; + +/** + * One commit step: its backend transaction, its seq, the document ids it assigns (S10, monotonic with seq), the table + * records it changed (the store's catalog takes them after `COMMIT`) and the statements it ran + * (`chord.store.statements`). + */ +export class Step { + readonly executor: SqliteExecutor; + readonly seq: number; + /** The next document id; the store adopts it after `COMMIT`. */ + nextDocumentId: number; + /** The ids of the commit's created documents, in the order of their writes. */ + readonly created: number[] = []; + /** The table records this step changed, by SQL name. */ + readonly tables = new Map(); + readonly #catalog: ReadonlyMap; + statements = 0; + + constructor( + executor: SqliteExecutor, + seq: number, + nextDocumentId: number, + catalog: ReadonlyMap, + ) { + this.executor = executor; + this.seq = seq; + this.nextDocumentId = nextDocumentId; + this.#catalog = catalog; + } + + /** The record of table `name` as this step leaves it so far. */ + table(name: string): TableRecord | undefined { + return this.tables.get(name) ?? this.#catalog.get(name); + } + + /** Runs one statement. */ + run(sql: string, ...params: SqliteValue[]): Promise { + this.statements++; + return this.executor.run(sql, ...params); + } + + /** Runs one statement and resolves with the rows it changed. */ + runCounted(sql: string, ...params: SqliteValue[]): Promise { + this.statements++; + return this.executor.runCounted(sql, ...params); + } + + /** Runs one query. */ + all(sql: string, ...params: SqliteValue[]): Promise { + this.statements++; + return this.executor.all(sql, ...params); + } +} diff --git a/packages/chord/src/next/store/store.ts b/packages/chord/src/next/store/store.ts new file mode 100644 index 000000000..f24f2d51e --- /dev/null +++ b/packages/chord/src/next/store/store.ts @@ -0,0 +1,347 @@ +// The store (slice 6a §2, §3): opened over a backend, one commit queue that serializes only the commit step (the seal +// fence's admission check, `BEGIN`, the writes in rule 9's order, the outbox commit, `lastSeq`, `COMMIT`, then the +// owner's adoption), reads guarded as Durable's `GuardedStorage` guards them, and the stop after a backend error. + +import { awaitWithContext, type Context } from "../../context/index.ts"; +import type { SqliteDatabase, SqliteExecutor } from "../../sqlite/index.ts"; +import { ChordError } from "../data/index.ts"; +import type { ArchivePart } from "./archive.ts"; +import type { Backend } from "./backend.ts"; +import { readCatalog, type TableRecord, writeCatalog } from "./catalog.ts"; +import { StoreExport } from "./export.ts"; +import { OUTBOX_FORMAT, Outbox } from "./outbox.ts"; +import { applyStoreSchema, type MetaRow, TABLES } from "./schema.ts"; +import { type StagedWrite, Step, type WriteOrder } from "./step.ts"; +import type { StoreWrite } from "./writes.ts"; + +/** A store as apps hold it: opened by `openStore`, owned and closed by the app. */ +export interface Store { + /** The seq of the store's last commit (0 for a new store). */ + readonly lastSeq: number; + /** The outbox the sync layer reads; `undefined` without one. */ + readonly outbox: Outbox | undefined; + /** + * The store's state after its `lastSeq` as archive parts (slice 6a §13), holding the commit queue from the first + * `next()` until the iteration ends, throws, is closed or `context` aborts. + */ + export(context: Context): AsyncIterableIterator; + /** + * Closes the store: waits for a running commit step, rejects queued commits and reads `released`, closes the + * backend. Idempotent, also after `storage_failed`; `context` ends only this caller's wait (the close goes on). + */ + close(context: Context): Promise; +} + +/** What `openStore` takes besides the backend. */ +export type StoreOptions = { + /** Turns the outbox on (`true`) or off (`false`); omitted, the persisted setting stays (a new store: off). */ + readonly outbox?: boolean; +}; + +/** + * Opens a store over `backend`: the store's schema migrations, the UTF-8 check, the metadata read once, the outbox + * switched when asked. A failure closes the backend and rejects (rule 12). + */ +export async function openStore(backend: Backend, context: Context, options: StoreOptions = {}): Promise { + const { db } = backend; + try { + context.abortSignal?.throwIfAborted(); + const meta = await switchOutbox(db, await applyStoreSchema(db), options.outbox); + return new ChordStore(db, meta, await readCatalog(db)); + } catch (error) { + try { + await db.close(); + } catch { + // The open's own failure is the one to report. + } + throw error; + } +} + +/** Turns the outbox on (from the current `lastSeq`) or off (its commits dropped) when `outbox` asks for the other. */ +async function switchOutbox(db: SqliteDatabase, meta: MetaRow, outbox: boolean | undefined): Promise { + if (outbox === undefined || outbox === (meta.outbox === 1)) return meta; + const floor = outbox ? meta.last_seq : 0; + await db.transaction(async (executor) => { + await executor.run( + `UPDATE ${TABLES.meta} SET outbox = ?, outbox_floor = ? WHERE singleton = 1`, + outbox ? 1 : 0, + floor, + ); + if (!outbox) await executor.run(`DELETE FROM ${TABLES.outbox}`); + }); + return { ...meta, outbox: outbox ? 1 : 0, outbox_floor: floor }; +} + +/** The seal fence's two checks of a commit (§1.3), as the commit step calls them. */ +export type CommitFence = { + /** At the step's entry: `false` rejects the commit `released` and writes nothing. */ + admit(): boolean; + /** After `COMMIT`: adopts the commit, `ids` being its created documents' ids in the order of their writes. */ + adopt(seq: number, ids: readonly number[]): void; +}; + +/** Where a store stands: open, stopped by a backend error, or closed. */ +type StoreState = + | { readonly kind: "open" } + | { readonly kind: "failed"; readonly error: unknown } + | { readonly kind: "closed" }; + +const ORDER: readonly WriteOrder[] = ["retire", "create", "write"]; + +const NOTHING = (): void => {}; + +/** The store behind `Store`, with what Chord's layers use: commits, guarded reads, the host's failure listener. */ +export class ChordStore implements Store { + readonly #db: SqliteDatabase; + #lastSeq: number; + #nextDocumentId: number; + #outbox: Outbox | undefined; + #state: StoreState = { kind: "open" }; + /** The commit queue's tail: each step starts after the one before it settled. */ + #tail: Promise = Promise.resolve(); + /** Reads underway, which `close` waits for. */ + readonly #underway = new Set>(); + /** The open host's listener for `storage_failed`, told once; one host per store (rule 7). */ + #listener: ((error: unknown) => void) | undefined; + #closing: Promise | undefined; + /** The recorded tables, by SQL name; changed only after a step's `COMMIT`. */ + readonly #catalog: Map; + /** The ends of the exports holding the queue or waiting for it, which closing the store calls. */ + readonly #held = new Set<() => void>(); + + constructor(db: SqliteDatabase, meta: MetaRow, catalog: Map) { + this.#db = db; + this.#catalog = catalog; + this.#lastSeq = meta.last_seq; + this.#nextDocumentId = meta.next_document_id; + this.#outbox = meta.outbox === 1 ? new Outbox(this, meta.outbox_floor) : undefined; + } + + get lastSeq(): number { + return this.#lastSeq; + } + + get outbox(): Outbox | undefined { + return this.#outbox; + } + + /** The id the next created document gets. */ + get nextDocumentId(): number { + return this.#nextDocumentId; + } + + /** Every recorded table, by SQL name, as of the last commit. */ + get tables(): ReadonlyMap { + return this.#catalog; + } + + export(context: Context): AsyncIterableIterator { + this.assertUsable(); + return new StoreExport(this, context); + } + + /** + * Takes the commit queue (an export): resolves once every earlier commit settled, with the release that lets the + * later ones run (idempotent). Closing the store calls `end` (from the call on, also while waiting, and at once for a + * store closed before the call took the queue), which must release. + */ + take(end: () => void): Promise<() => void> { + const held = Promise.withResolvers(); + const release = (): void => { + this.#held.delete(end); + held.resolve(); + }; + this.#held.add(end); + const taken = this.#tail.then(() => { + // A store closed meanwhile (also before the export's first `next()`) holds nothing: the export ends. + if (this.#state.kind === "closed") { + release(); + end(); + } + return release; + }); + this.#tail = taken.then(() => held.promise); + return taken; + } + + /** The recorded table `name` (its SQL name), as of the last commit. */ + table(name: string): TableRecord | undefined { + return this.#catalog.get(name); + } + + /** Attaches the open host's `storage_failed` listener; a store attached to an open host is refused (rule 7). */ + attach(listener: (error: unknown) => void): void { + if (this.#listener !== undefined) { + throw new ChordError({ code: "invalid", reason: "args", message: "store: already attached to an open host" }); + } + this.#listener = listener; + } + + /** The host closed: the store may serve another one. */ + detach(): void { + this.#listener = undefined; + } + + /** + * Commits `writes` as one seq when the commit queue reaches it: `fence.admit()` at the step's entry, the writes in + * rule 9's order (retirements, creations, the rest), the outbox commit, `lastSeq`, `COMMIT`, then `fence.adopt` + * before the next step starts. Resolves with the seq. A backend error or a throwing `adopt` stops the store + * (`storage_failed`); a refused admission rejects `released`. `context` counts the step (no abort once queued). + */ + commit(writes: readonly StagedWrite[], fence: CommitFence, context: Context): Promise { + try { + this.assertUsable(); + } catch (error) { + return Promise.reject(error); + } + const step = this.#tail.then(() => this.#step(writes, fence, context)); + this.#tail = step.then(NOTHING, NOTHING); + return step; + } + + /** + * Runs `read` on the backend under the guard (rule 11, Durable's `GuardedStorage`): refused once the store stopped + * or closed; a backend error fails the store unless `context` was aborted (then `aborted`); a read settling after + * the store stopped rejects `unavailable/storage_failed`. + */ + async read(read: (executor: SqliteExecutor) => Promise, context: Context): Promise { + this.assertUsable(); + const call = (async () => read(this.#db))(); + this.#underway.add(call); + let result: T; + try { + result = await call; + } catch (error) { + this.assertUsable(); + const signal = context.abortSignal; + if (signal?.aborted) + throw new ChordError({ code: "aborted", message: "store: read aborted" }, { cause: signal.reason }); + throw this.#fail(error); + } finally { + this.#underway.delete(call); + } + this.assertUsable(); + return result; + } + + /** + * Runs `write` on the backend outside the commit queue (the outbox's `trim`), in a backend transaction, guarded as + * a read without a context: any error stops the store. + */ + async maintain(write: (executor: SqliteExecutor) => Promise): Promise { + this.assertUsable(); + const call = this.#db.transaction(write); + this.#underway.add(call); + try { + await call; + } catch (error) { + this.assertUsable(); + throw this.#fail(error); + } finally { + this.#underway.delete(call); + } + } + + close(context: Context): Promise { + this.#closing ??= this.#close(); + return awaitWithContext(this.#closing, context); + } + + async #close(): Promise { + this.#state = { kind: "closed" }; + for (const end of [...this.#held]) end(); + await this.#tail; + await Promise.allSettled([...this.#underway]); + await this.#db.close(); + } + + async #step(writes: readonly StagedWrite[], fence: CommitFence, context: Context): Promise { + this.assertUsable(); + if (!fence.admit()) { + throw new ChordError({ code: "released", reason: "discarded", message: "store: the commit's attempt ended" }); + } + if (writes.length === 0) { + // An empty commit takes no seq; a throwing `adopt` stops the store as for any commit. + try { + fence.adopt(this.#lastSeq, []); + } catch (error) { + throw this.#fail(error); + } + return this.#lastSeq; + } + const seq = this.#lastSeq + 1; + // Past the safe range seqs stop counting exactly (a later one would repeat): the store cannot number further. + if (seq > Number.MAX_SAFE_INTEGER) throw this.#fail(new Error("store: no seq past Number.MAX_SAFE_INTEGER")); + const outbox = this.#outbox !== undefined; + let step: Step; + try { + step = await this.#db.transaction(async (executor) => { + const step = new Step(executor, seq, this.#nextDocumentId, this.#catalog); + const recorded: StoreWrite[] | undefined = outbox ? [] : undefined; + for (const order of ORDER) { + for (const write of writes) { + if (write.order !== order) continue; + const done = await write.apply(step); + recorded?.push(done); + } + } + await writeCatalog(step); + if (recorded !== undefined) { + await step.run( + `INSERT INTO ${TABLES.outbox} (seq, format, writes) VALUES (?, ?, ?)`, + seq, + OUTBOX_FORMAT, + JSON.stringify(recorded), + ); + } + await step.run( + `UPDATE ${TABLES.meta} SET last_seq = ?, next_document_id = ? WHERE singleton = 1`, + seq, + step.nextDocumentId, + ); + return step; + }); + } catch (error) { + throw this.#fail(error); + } + // `COMMIT` returned: the commit is durable, and memory follows. + this.#lastSeq = seq; + this.#nextDocumentId = step.nextDocumentId; + for (const [name, record] of step.tables) this.#catalog.set(name, record); + const metrics = context.telemetry.metrics; + metrics?.counter("chord.store.commits").add(1); + metrics?.counter("chord.store.statements").add(step.statements); + try { + fence.adopt(seq, step.created); + } catch (error) { + throw this.#fail(error); + } + return seq; + } + + /** Throws `unavailable/storage_failed` once the store stopped, `released` once it closed. */ + assertUsable(): void { + const state = this.#state; + if (state.kind === "failed") throw storageFailed(state.error); + if (state.kind === "closed") + throw new ChordError({ code: "released", reason: "closed", message: "store: closed" }); + } + + /** Stops the store after `error` (once; its host is told once); returns the error the caller rejects with. */ + #fail(error: unknown): ChordError { + if (this.#state.kind === "open") { + this.#state = { kind: "failed", error }; + this.#listener?.(error); + } + return storageFailed(error); + } +} + +/** The `unavailable/storage_failed` error of a stopped store, its backend error as the cause. */ +function storageFailed(error: unknown): ChordError { + return new ChordError( + { code: "unavailable", reason: "storage_failed", message: "store: storage failed" }, + { cause: error }, + ); +} diff --git a/packages/chord/src/next/store/table-changes.ts b/packages/chord/src/next/store/table-changes.ts new file mode 100644 index 000000000..faa66cd32 --- /dev/null +++ b/packages/chord/src/next/store/table-changes.ts @@ -0,0 +1,147 @@ +// A transaction's writes to one table (slice 6a §6, §7; W10): checked and staged synchronously like a draft's ops (no +// I/O, a bad row throws at its own call), kept in staging order for the commit step, and overlaid on committed rows +// for `tx.get` until a bulk write makes that impossible (`invalid/read_after_bulk`). + +import type { Context } from "../../context/index.ts"; +import { invalid } from "../data/internal.ts"; +import { applySet, type EvaluatedRow } from "./evaluate.ts"; +import { compileSet, compileWhere, type SetNode } from "./expression.ts"; +import type { StagedWrite } from "./step.ts"; +import type { TableHandle } from "./table-handle.ts"; +import { + jsonVersions, + recordedVersions, + stageDelete, + stageDeleteWhere, + stagePut, + stageUpdate, + stageUpdateWhere, +} from "./table-writes.ts"; + +/** One staged write of one key, as the overlay replays it. */ +type KeyWrite = + | { readonly kind: "put"; readonly row: EvaluatedRow } + | { readonly kind: "update"; readonly set: SetNode } + | { readonly kind: "delete" }; + +/** + * The writes of one transaction to one table: each appended to the transaction's staged list (`staged`, one order + * across tables) and, for the single-key ones, to its key's overlay. + */ +export class TableChanges { + readonly handle: TableHandle; + readonly #staged: StagedWrite[]; + readonly #byKey = new Map(); + /** An `updateWhere` or `deleteWhere` was staged: `get` is refused from then on (W10). */ + #bulk = false; + + constructor(handle: TableHandle, staged: StagedWrite[]) { + this.handle = handle; + this.#staged = staged; + } + + /** Stages a `put` of `row` (an upsert). */ + put(row: unknown): void { + const { definition, sql } = this.handle; + const encoded = definition.rows.put(row, definition.version); + this.#staged.push(stagePut(sql.put, sql.table, encoded, definition.version)); + this.#write(encoded.id, { kind: "put", row: encoded.row }); + } + + /** Stages an `update` of the row with `key` (`set`: values, or a callback of column references). */ + update(key: unknown, set: unknown): void { + const { definition, sql } = this.handle; + const encoded = definition.rows.key(key, "update"); + const compiled = this.#set(set, "update"); + const versions = jsonVersions(this.handle.definition, compiled); + this.#staged.push( + stageUpdate( + sql.update(compiled, encoded.params, versions), + sql.table, + encoded.key, + compiled, + recordedVersions(compiled, versions), + ), + ); + this.#write(encoded.id, { kind: "update", set: compiled }); + } + + /** Stages a `delete` of the row with `key`. */ + delete(key: unknown): void { + const { definition, sql } = this.handle; + const encoded = definition.rows.key(key, "delete"); + this.#staged.push(stageDelete({ sql: sql.delete, params: [...encoded.params] }, sql.table, encoded.key)); + this.#write(encoded.id, { kind: "delete" }); + } + + /** Stages an `updateWhere`: every row matching `where` gets `set`. */ + updateWhere(where: unknown, set: unknown): void { + const { definition, sql } = this.handle; + const condition = compileWhere( + definition, + typeof where === "function" ? where(definition.refs) : where, + "updateWhere", + ); + const compiled = this.#set(set, "updateWhere"); + const versions = jsonVersions(this.handle.definition, compiled); + const statement = sql.updateWhere(condition, compiled, versions); + this.#staged.push( + stageUpdateWhere(statement, sql.table, condition, compiled, recordedVersions(compiled, versions)), + ); + this.#bulk = true; + } + + /** Stages a `deleteWhere`: every row matching `where` goes. */ + deleteWhere(where: unknown): void { + const { definition, sql } = this.handle; + const condition = compileWhere( + definition, + typeof where === "function" ? where(definition.refs) : where, + "deleteWhere", + ); + this.#staged.push(stageDeleteWhere(sql.deleteWhere(condition), sql.table, condition)); + this.#bulk = true; + } + + /** + * The row with `key` as this transaction leaves it: the committed row with this transaction's `put`, `update` and + * `delete` of the key replayed; refused after a bulk write (W10). + */ + async get(key: unknown, context: Context): Promise { + const { definition } = this.handle; + if (this.#bulk) { + throw invalid( + "read_after_bulk", + `${definition.name}: tx.get after an updateWhere or deleteWhere on the table`, + ); + } + const encoded = definition.rows.key(key, "get"); + const writes = this.#byKey.get(encoded.id) ?? []; + let start = writes.length; + while (start > 0 && writes[start - 1]!.kind === "update") start--; + const from = writes[start - 1]; + // A committed row is decoded by the table's columns: scalars and JSON, what the evaluator takes. + let row: EvaluatedRow | undefined = + from === undefined + ? ((await this.handle.get(encoded.key, context)) as EvaluatedRow | undefined) + : from.kind === "put" + ? from.row + : undefined; + for (const write of writes.slice(start)) { + // An `update` of an absent row changes nothing. + if (row !== undefined && write.kind === "update") row = applySet(write.set, row, definition.name); + } + return row; + } + + #set(set: unknown, what: string): SetNode { + const { definition } = this.handle; + return compileSet(definition, typeof set === "function" ? set(definition.refs) : set, what); + } + + #write(id: string, write: KeyWrite): void { + const writes = this.#byKey.get(id); + if (writes === undefined) this.#byKey.set(id, [write]); + else writes.push(write); + } +} diff --git a/packages/chord/src/next/store/table-evolve.ts b/packages/chord/src/next/store/table-evolve.ts new file mode 100644 index 000000000..3e31ff82e --- /dev/null +++ b/packages/chord/src/next/store/table-evolve.ts @@ -0,0 +1,306 @@ +// `evolveTable(previous, (t) => t…)` (slice 6a §9; decision "Second spec review: the majority"): the next version of a +// table from the edits of one migration. The editor's types order the edits (`rename`, `add` and `json` before the +// data step, `drop`, `index` and `dropIndex` after it) and track the columns and key through the chain; at runtime one +// editor records the edits, checks each as it is made, and materializes the new version at the end. + +import type { TOptional } from "typebox"; +import { opt, type TSchema } from "../contracts/index.ts"; +import { editWith, type Increment } from "../contracts/internal.ts"; +import { + type ColumnSchema, + type Columns, + type ColumnTypeOf, + checkColumnName, + columnRecord, + describeColumns, + type OrderColumn, + type RowOf, + type ValueOf, +} from "./columns.ts"; +import { type Condition, compileSet, compileWhere, type SetOf } from "./expression.ts"; +import type { ColumnRefs } from "./query.ts"; +import { INDEX_NAME, orderColumns, TableDefinition, type TableOptions } from "./table.ts"; +import type { DataStep, TableEdit } from "./table-history.ts"; + +/** The edits after the data step: `drop`, `index` and `dropIndex`; `C` and `K` are the columns and key so far. */ +export interface TableTailEditor { + /** Drops a non-key column; its indexes are dropped first, in the same commit. */ + drop>(name: N): TableTailEditor, K>; + /** Adds an index (`[a-z0-9_]+`) over non-`opt` `str` or `int` columns; it ends with the key. */ + index(name: string, columns: readonly OrderColumn[]): TableTailEditor; + /** Drops an index. */ + dropIndex(name: string): TableTailEditor; + /** Type only: the columns and key so far. */ + readonly "~columns"?: C; + readonly "~key"?: K; +} + +/** The editor `evolveTable`'s edit gets: every edit; the column edits only before the data step. */ +export interface TableEditor extends TableTailEditor { + /** Renames a column; the key, the indexes and a `json(...)` column's version column follow. */ + rename( + from: F, + to: T, + ): TableEditor, RenamedKey>; + /** Adds a column: `opt(...)`, or a scalar one with a `default` (existing rows get it). */ + add(name: N, column: S): TableEditor, K>; + /** A `json(...)` column's new version (its nullability kept): no DDL, no data step, cells migrate on access. */ + json, S extends ColumnSchema>(name: N, column: S): TableEditor, K>; + /** + * The data step: `patch` gets each row as the columns stand here and returns new values of non-key columns (`{}` + * stamps the row only); synchronous and pure, run in chunks of 100 in key order. + */ + rows(patch: (row: RowOf) => PatchOf): TableTailEditor; + /** The data step as one `updateWhere` over the rows not migrated yet. */ + update(step: { + readonly where?: (row: ColumnRefs) => Condition; + readonly set: SetOf | ((row: ColumnRefs) => SetOf); + }): TableTailEditor; +} + +/** A column `add` takes: `opt(...)` or one with a `default`. */ +export type AddedColumn = ColumnSchema & (TOptional | { readonly default: unknown }); + +/** A `rows` patch: new values of non-key columns; `undefined` clears an `opt(...)` one. */ +export type PatchOf = { + readonly [P in Exclude]?: ValueOf; +}; + +type Renamed = { + readonly [P in keyof C as P extends F ? T : P]: C[P]; +}; + +type RenamedKey = { + readonly [I in keyof K]: K[I] extends F ? T : K[I]; +}; + +type With = { + readonly [P in keyof C | N]: P extends N ? S : C[P & keyof C]; +}; + +type Without = { readonly [P in Exclude]: C[P] }; + +type JsonNames = { [P in keyof C]: ColumnTypeOf extends "json" ? P : never }[keyof C] & string; + +type WithJson = With< + C, + N, + C[N] extends TOptional ? TOptional : S +>; + +/** `V + 1` for a literal version (contracts' `Increment`); `number` stays `number`. */ +type Next = number extends V + ? number + : `${Increment<`${V}`>}` extends `${infer N extends number}` + ? N + : number; + +/** + * Appends the next version (`previous.version + 1`) with the edits `edit` makes; `edit` must return the editor it was + * given. Every version is materialized and fingerprinted here; the object serves the newest, and its activation + * migrates a store at an older version through each one. + */ +export function evolveTable< + C extends Columns, + K extends readonly string[], + V extends number, + E extends { readonly "~columns"?: Columns; readonly "~key"?: readonly string[] }, +>( + previous: TableDefinition, + edit: (t: TableEditor) => E, +): TableDefinition, NonNullable, Next> { + // The runtime editor works on untyped definitions; the types live in the editor interfaces. + const editor = new Editor(previous as unknown as TableDefinition); + // The editor interfaces track the edits in the types; at runtime every edit returns the same editor. + editWith(editor as unknown as TableEditor, edit, `${previous.name}: evolveTable`); + return editor.finish() as unknown as TableDefinition, NonNullable, Next>; +} + +/** + * The editor at runtime: the columns, key and indexes as the edits leave them, the edits before and after the data + * step, and the step. Each edit is checked as it is made (declaration policy: JavaScript callers reach the checks). + */ +class Editor { + readonly #previous: TableDefinition; + readonly #what: string; + #schemas: { [name: string]: ColumnSchema }; + #key: string[]; + readonly #indexes: Map; + /** `before` until the data step or the first `drop`, `index` or `dropIndex`. */ + #phase: "before" | "after" = "before"; + readonly #before: TableEdit[] = []; + #step: DataStep | undefined; + readonly #after: TableEdit[] = []; + + constructor(previous: TableDefinition) { + this.#previous = previous; + this.#what = `${previous.name} version ${previous.version + 1}`; + this.#schemas = { ...previous.schemas }; + this.#key = [...previous.key]; + this.#indexes = new Map(previous.indexes); + } + + rename(from: string, to: string): this { + this.#assertBefore("rename"); + this.#assertColumn(from, "rename"); + // Checked here, not by the final definition: a later edit may rename it again, but this one runs as DDL. + checkColumnName(to, `${this.#what}: rename`); + this.#assertNewName(to, "rename", from); + const rename = (column: string): string => (column === from ? to : column); + this.#schemas = Object.fromEntries(Object.entries(this.#schemas).map(([name, schema]) => [rename(name), schema])); + this.#key = this.#key.map(rename); + for (const [name, columns] of this.#indexes) this.#indexes.set(name, columns.map(rename)); + this.#before.push({ kind: "rename", from, to }); + return this; + } + + add(name: string, column: ColumnSchema): this { + this.#assertBefore("add"); + this.#assertNewName(name, "add"); + const [described] = describeColumns({ [name]: column }, `${this.#what}: add`); + if (!described!.optional && (described!.type === "json" || described!.default === undefined)) { + throw new Error(`${this.#what}: add: column "${name}" needs opt(...) or a default (existing rows get it)`); + } + this.#schemas[name] = column; + this.#before.push({ kind: "add", name, column: columnRecord(described!) }); + return this; + } + + json(name: string, column: ColumnSchema): this { + this.#assertBefore("json"); + const existing = describeColumns({ [name]: this.#assertColumn(name, "json") }, this.#what)[0]!; + if (existing.type !== "json") throw new Error(`${this.#what}: json: column "${name}" is not a json(...) column`); + if (Object.hasOwn(column, "~optional")) { + throw new Error(`${this.#what}: json: pass json(...) itself; the column keeps its nullability`); + } + const schema = existing.optional ? opt(column) : column; + const described = describeColumns({ [name]: schema }, `${this.#what}: json`)[0]!; + if (described.type !== "json") throw new Error(`${this.#what}: json: expected a json(...) column`); + const [before, after] = [existing.json, described.json]; + // D9 across table versions: a cell's version says which schema it was written with. + if (after.version < before.version) { + throw new Error(`${this.#what}: json: column "${name}" goes back from version ${before.version}`); + } + if (after.version === before.version && after.fingerprint !== before.fingerprint) { + throw new Error( + `${this.#what}: json: column "${name}" changes its schema at version ${before.version} (raise its version)`, + ); + } + this.#schemas[name] = schema; + this.#before.push({ kind: "json", name, column: columnRecord(described) }); + return this; + } + + rows(patch: (row: { readonly [column: string]: unknown }) => unknown): this { + this.#assertBefore("rows"); + if (typeof patch !== "function") throw new Error(`${this.#what}: rows: expected a function`); + this.#step = { kind: "rows", table: this.#current(), patch }; + this.#phase = "after"; + return this; + } + + update(step: { readonly where?: unknown; readonly set: unknown }): this { + this.#assertBefore("update"); + const table = this.#current(); + const what = `version ${table.version} update`; + const where = typeof step.where === "function" ? step.where(table.refs) : step.where; + const set = typeof step.set === "function" ? step.set(table.refs) : step.set; + this.#step = { + kind: "update", + table, + where: where === undefined ? undefined : compileWhere(table, where, what), + set: compileSet(table, set, what), + }; + this.#phase = "after"; + return this; + } + + drop(name: string): this { + this.#phase = "after"; + this.#assertColumn(name, "drop"); + if (this.#key.includes(name)) throw new Error(`${this.#what}: drop: "${name}" is a key column`); + for (const [index, columns] of this.#indexes) { + if (!columns.includes(name)) continue; + this.#indexes.delete(index); + this.#after.push({ kind: "dropIndex", name: index }); + } + const { [name]: _dropped, ...schemas } = this.#schemas; + this.#schemas = schemas; + this.#after.push({ kind: "drop", name }); + return this; + } + + index(name: string, columns: readonly string[]): this { + this.#phase = "after"; + const what = `${this.#what}: index "${name}"`; + if (!INDEX_NAME.test(name)) throw new Error(`${what}: expected [a-z0-9_]+`); + if (this.#indexes.has(name)) throw new Error(`${what} exists`); + // Checked against the columns at this edit: the migration creates the index here, whatever later edits do. + const byName = new Map(describeColumns(this.#schemas, this.#what).map((column) => [column.name, column])); + const checked = orderColumns(byName, columns, what); + this.#indexes.set(name, checked); + this.#after.push({ kind: "index", name, columns: checked }); + return this; + } + + dropIndex(name: string): this { + this.#phase = "after"; + if (!this.#indexes.delete(name)) throw new Error(`${this.#what}: dropIndex: no index "${name}"`); + this.#after.push({ kind: "dropIndex", name }); + return this; + } + + /** The new version, materialized and fingerprinted, linked to the previous one and the migration from it. */ + finish(): TableDefinition { + const migration = { before: this.#before, step: this.#step, after: this.#after }; + return new TableDefinition(this.#previous.name, this.#options(), { previous: this.#previous, migration }); + } + + /** The table as the edits so far leave it, at the new version: what a data step reads and writes. */ + #current(): TableDefinition { + return new TableDefinition(this.#previous.name, this.#options()); + } + + #options(): TableOptions { + const version = this.#previous.version + 1; + const options = { + version, + columns: { ...this.#schemas }, + key: [...this.#key], + indexes: Object.fromEntries(this.#indexes), + }; + // The key and indexes are names the edits kept; the constructor checks them as it checks a base's. + return options as unknown as TableOptions; + } + + #assertBefore(edit: string): void { + if (this.#phase === "after") { + throw new Error(`${this.#what}: ${edit} comes before the data step and before drop, index and dropIndex`); + } + } + + /** + * `name` names no column yet, also ignoring ASCII case (SQLite's column names do): checked per edit, since a later + * `drop` cannot take back an `add` the migration runs first. A rename may change only the case of `from`. + */ + #assertNewName(name: string, edit: string, from?: string): void { + const folded = foldAscii(name); + for (const existing of Object.keys(this.#schemas)) { + if (existing !== from && foldAscii(existing) === folded) { + throw new Error( + `${this.#what}: ${edit}: column "${existing}" exists (names differing only in case collide)`, + ); + } + } + } + + #assertColumn(name: string, edit: string): ColumnSchema { + if (!Object.hasOwn(this.#schemas, name)) throw new Error(`${this.#what}: ${edit}: no column "${name}"`); + return this.#schemas[name]!; + } +} + +/** `name` with ASCII letters lowercased and nothing else changed, as SQLite folds identifiers. */ +function foldAscii(name: string): string { + return name.replace(/[A-Z]+/g, (letters) => letters.toLowerCase()); +} diff --git a/packages/chord/src/next/store/table-handle.ts b/packages/chord/src/next/store/table-handle.ts new file mode 100644 index 000000000..65e9dcde1 --- /dev/null +++ b/packages/chord/src/next/store/table-handle.ts @@ -0,0 +1,149 @@ +// Table handles (slice 6a §6; decision "Query runner"): what `self.table(Def)` returns, one per attempt and table: reads +// of committed rows (`page`, `rows`, `get`, `count`) through the store's guarded reads, and what a transaction's +// writes to the table are staged with. + +import type { Context } from "../../context/index.ts"; +import type { SqliteValue } from "../../sqlite/index.ts"; +import { ChordError } from "../data/index.ts"; +import { invalid } from "../data/internal.ts"; +import type { Columns, RowOf } from "./columns.ts"; +import { decodeCursor, encodeCursor, type Query } from "./query.ts"; +import { TableSql } from "./sql.ts"; +import type { ChordStore } from "./store.ts"; +import type { KeyOf, TableDefinition } from "./table.ts"; + +/** One page of rows (`items`, as Durable's pages) and, when more follow, the cursor to continue after it. */ +export type Page = { readonly items: T[]; readonly next?: string }; + +/** What `page` takes besides the query, the limit and the context. */ +export type PageOptions = { + /** A cursor from an earlier page of the same order. */ + readonly after?: string; +}; + +/** A table of this object: reads of committed state, the context last. Writes go through `self.tx`. */ +export interface Table { + /** Up to `limit` rows of `q` (after `options.after`), and the cursor of the next page when there is one. */ + page(q: Query, limit: number, context: Context, options?: PageOptions): Promise>>; + /** Every row of `q`, read a page of 100 at a time. */ + rows(q: Query, context: Context): AsyncIterable>; + /** The row with `key`, `undefined` when absent. */ + get(key: KeyOf, context: Context): Promise | undefined>; + /** The number of rows of `q`. */ + count(q: Query, context: Context): Promise; +} + +/** The handle type of table definition `D`: what `self.table(D)` returns. */ +export type TableOf = D extends TableDefinition ? Table : never; + +/** Who owns a table handle: its object (whose transactions alone write it), liveness check and live failure. */ +export type TableOwner = { + readonly label: string; + /** The object's documents: a transaction of another object refuses the handle. */ + readonly scope: object; + /** Throws `released` once the object ended. */ + assertLive(): void; + /** A row the code cannot read (a `json(...)` cell, slice 6a §9) fails the owner `broken` (§1.4). */ + fail(error: ChordError): void; +}; + +/** The rows `rows()` reads per page (rule 17). */ +export const ROWS_PAGE = 100; + +/** A table handle: the definition the attempt declared, the table's SQL name and statements, the store. */ +export class TableHandle implements Table { + readonly definition: TableDefinition; + readonly sql: TableSql; + readonly store: ChordStore; + readonly owner: TableOwner; + + constructor(definition: TableDefinition, sqlName: string, store: ChordStore, owner: TableOwner) { + this.definition = definition; + this.sql = new TableSql(sqlName, definition.columns, definition.rows.physical, definition.key); + this.store = store; + this.owner = owner; + } + + async page(q: Query, limit: number, context: Context, options: PageOptions = {}): Promise>> { + this.owner.assertLive(); + this.#assertQuery(q); + if (!Number.isSafeInteger(limit) || limit < 1) + throw invalid("args", `${this.definition.name}: limit is a positive integer`); + const order = q.order; + const after = + options.after === undefined ? undefined : decodeCursor(this.definition, this.sql.table, order, options.after); + const statement = this.sql.select(q.condition, order, after, limit + 1); + const stored = await this.store.read( + (executor) => executor.all<{ [column: string]: SqliteValue }>(statement.sql, ...statement.params), + context, + ); + const more = stored.length > limit; + if (more) stored.length = limit; + const items = stored.map((row) => this.decode(row, context)); + if (!more) return { items }; + const last = stored[limit - 1]!; + return { + items, + next: encodeCursor( + this.sql.table, + order, + order.columns.map((column) => last[column]!), + ), + }; + } + + async *rows(q: Query, context: Context): AsyncIterable> { + let after: string | undefined; + do { + const page = await this.page(q, ROWS_PAGE, context, after === undefined ? {} : { after }); + yield* page.items; + after = page.next; + } while (after !== undefined); + } + + async get(key: unknown, context: Context): Promise | undefined> { + this.owner.assertLive(); + const encoded = this.definition.rows.key(key, "get"); + const stored = await this.store.read( + (executor) => executor.get<{ [column: string]: SqliteValue }>(this.sql.get, ...encoded.params), + context, + ); + return stored === undefined ? undefined : this.decode(stored, context); + } + + async count(q: Query, context: Context): Promise { + this.owner.assertLive(); + this.#assertQuery(q); + const statement = this.sql.count(q.condition); + const row = await this.store.read( + (executor) => executor.get<{ readonly count: number }>(statement.sql, ...statement.params), + context, + ); + return row?.count ?? 0; + } + + /** A stored row as reads return it; one the code cannot read fails the owner and throws. */ + decode(stored: { readonly [column: string]: SqliteValue }, context: Context): RowOf { + try { + // The codec decodes by the definition's columns: a row of them. + return this.definition.rows.decode(stored, context) as RowOf; + } catch (error) { + const failure = + error instanceof ChordError + ? error + : new ChordError({ code: "unavailable", message: String(error) }, { cause: error }); + this.owner.fail(failure); + throw failure; + } + } + + /** A query of another table is refused (`invalid/args`). */ + #assertQuery(q: Query): void { + if (q.table !== this.definition) { + throw invalid( + "args", + `${this.owner.label}: a query of table "${q.table.name}" run on "${this.definition.name}"`, + ); + } + } +} diff --git a/packages/chord/src/next/store/table-history.ts b/packages/chord/src/next/store/table-history.ts new file mode 100644 index 000000000..639cbedd2 --- /dev/null +++ b/packages/chord/src/next/store/table-history.ts @@ -0,0 +1,98 @@ +// A table's history (slice 6a §9; decision "Second spec review: the majority"): the versions from a base +// `defineTable` through each `evolveTable`, the migration each version recorded, and what an activation does with the +// version the store recorded: create the table, use it, migrate it from there, or refuse it. + +import type { TableRecord } from "./catalog.ts"; +import type { ColumnRecord } from "./columns.ts"; +import type { SetNode, WhereNode } from "./expression.ts"; +import type { TableDefinition } from "./table.ts"; + +/** One edit of a migration as its writes record it (`alter`, `index`, `dropIndex`); the table is named when applied. */ +export type TableEdit = + | { readonly kind: "rename"; readonly from: string; readonly to: string } + | { readonly kind: "add"; readonly name: string; readonly column: ColumnRecord } + | { readonly kind: "drop"; readonly name: string } + | { readonly kind: "json"; readonly name: string; readonly column: ColumnRecord } + | { readonly kind: "index"; readonly name: string; readonly columns: readonly string[] } + | { readonly kind: "dropIndex"; readonly name: string }; + +/** The column edits an `alter` write records. */ +export type AlterEdit = Extract; + +/** + * A version's data step over `table`, the columns between the edits before it and after it: `rows` patches each row + * (synchronous and pure), `update` is one `updateWhere`. + */ +export type DataStep = + | { + readonly kind: "rows"; + readonly table: TableDefinition; + readonly patch: (row: { readonly [column: string]: unknown }) => unknown; + } + | { + readonly kind: "update"; + readonly table: TableDefinition; + readonly where: WhereNode | undefined; + readonly set: SetNode; + }; + +/** What one `evolveTable` changed: the edits before the data step, the step, the edits after it. */ +export type Migration = { + readonly before: readonly TableEdit[]; + readonly step: DataStep | undefined; + readonly after: readonly TableEdit[]; +}; + +/** + * What an activation does with a declared table: create it at the newest version, use it as recorded, or migrate it + * through `versions` (oldest first), the first one from its start (`begin`) or, recorded `migrating`, after its (A) + * commit (`resume`). + */ +export type TablePlan = + | { readonly kind: "create" } + | { readonly kind: "ready" } + | { + readonly kind: "migrate"; + readonly versions: readonly TableDefinition[]; + readonly first: "begin" | "resume"; + }; + +/** + * The plan for `definition` (the code's newest version) against `recorded`. A recorded version newer than the newest, + * older than the history's base, or the same version with another fingerprint (a history edited in place) throws: + * the object fails `broken`, the data stays. + */ +export function planTable(definition: TableDefinition, recorded: TableRecord | undefined, what: string): TablePlan { + if (recorded === undefined) return { kind: "create" }; + const { version, state, fingerprint } = recorded; + if (version > definition.version) { + throw new Error(`${what}: the store has version ${version}, newer than the code's ${definition.version}`); + } + const versions = historyOf(definition); + const at = versions.find((each) => each.version === version); + if (at === undefined) { + throw new Error( + `${what}: the store has version ${version}, older than the history's base ${versions[0]!.version}`, + ); + } + if (at.fingerprint !== fingerprint) { + throw new Error(`${what}: version ${version} is recorded with another definition (a history edited in place)`); + } + if (state === "ready" && version === definition.version) return { kind: "ready" }; + if (state === "migrating" && at.evolution?.migration.step === undefined) { + throw new Error(`${what}: version ${version} is recorded migrating, but has no data step to resume`); + } + if (state === "migrating") { + return { kind: "migrate", versions: versions.filter((each) => each.version >= version), first: "resume" }; + } + return { kind: "migrate", versions: versions.filter((each) => each.version > version), first: "begin" }; +} + +/** Every version of `definition`'s history, the base first. */ +function historyOf(definition: TableDefinition): TableDefinition[] { + const versions: TableDefinition[] = []; + for (let at: TableDefinition | undefined = definition; at !== undefined; at = at.evolution?.previous) { + versions.unshift(at); + } + return versions; +} diff --git a/packages/chord/src/next/store/table-migration.ts b/packages/chord/src/next/store/table-migration.ts new file mode 100644 index 000000000..8fef8fa9a --- /dev/null +++ b/packages/chord/src/next/store/table-migration.ts @@ -0,0 +1,245 @@ +// Declared tables brought to their version at activation (slice 6a §9): a table the store does not have is created at +// the newest version with its indexes (no replay); an older one is migrated eagerly, one version at a time, in its +// commits. Per version N: (A) the edits before the data step and `table { version: N, state: "migrating" }`; the data +// step's chunks; (Z) the last chunk (or the `update`) with the edits after the step and `state: "ready"`. Without a +// data step (A) and (Z) are one commit. A version recorded `migrating` resumes after (A), skipping the rows already +// stamped `$v >= N`. Its record holds the columns during the step but version N's fingerprint, not the fingerprint of +// those columns: the fingerprint names the version being reached, which resume, export and import check. + +import { type Context, withoutAbortSignal } from "../../context/index.ts"; +import type { SqliteValue } from "../../sqlite/index.ts"; +import type { Column } from "./columns.ts"; +import type { SetNode, WhereNode } from "./expression.ts"; +import { encodeCell, ROW_VERSION, type Scalar } from "./rows.ts"; +import { type Order, TableSql } from "./sql.ts"; +import type { StagedWrite } from "./step.ts"; +import type { ChordStore, CommitFence } from "./store.ts"; +import type { TableDefinition } from "./table.ts"; +import { type DataStep, planTable, type TablePlan } from "./table-history.ts"; +import { + jsonVersions, + recordedVersions, + stageEdit, + stageIndex, + stageTable, + stageUpdate, + stageUpdateWhere, +} from "./table-writes.ts"; + +/** A declared table: its newest definition and its SQL name. */ +export type DeclaredTable = { readonly definition: TableDefinition; readonly table: string }; + +/** Where a migration runs: the store, the table's SQL name and the commits' fence. */ +type MigrationTarget = { readonly store: ChordStore; readonly table: string; readonly fence: CommitFence }; + +/** + * Brings `tables` to their definitions: the ones the store does not have created in one commit, then each older one + * migrated. A table the code cannot use (see `planTable`) throws before anything is written; `what` names the owner. + */ +export async function prepareTables( + tables: readonly DeclaredTable[], + store: ChordStore, + fence: CommitFence, + what: string, + context: Context, +): Promise { + const created: StagedWrite[] = []; + const migrations: { readonly table: string; readonly plan: Extract }[] = []; + for (const { definition, table } of tables) { + const plan = planTable(definition, store.table(table), `${what}: table "${definition.name}"`); + if (plan.kind === "create") created.push(...creation(definition, table)); + else if (plan.kind === "migrate") migrations.push({ table, plan }); + } + if (created.length > 0) await commit(created, { store, fence }, context); + for (const { table, plan } of migrations) { + await migrateTable(plan, { store, table, fence }, context); + } +} + +/** The writes creating `definition` fresh as `table`: at its version, `ready`, with its indexes. */ +function creation(definition: TableDefinition, table: string): StagedWrite[] { + const { version, record: columns, key, fingerprint } = definition; + return [ + stageTable({ kind: "table", table, version, state: "ready", columns, key, fingerprint }), + ...[...definition.indexes].map(([name, columns]) => stageIndex({ kind: "index", table, name, columns })), + ]; +} + +/** + * Migrates the table through the plan's versions (oldest first, each with its migration), the first resumed after its + * (A) commit when the plan says so. `context`'s abort is checked between chunks (a chunk admitted to the step completes); a data + * step that throws, or returns a patch the version cannot take, rejects: the table stays `migrating` and the committed + * chunks stay. + */ +async function migrateTable( + plan: Extract, + target: MigrationTarget, + context: Context, +): Promise { + for (const [index, version] of plan.versions.entries()) { + await migrateVersion(version, index === 0 ? plan.first : "begin", target, context); + } +} + +/** The size of a `rows` chunk: one commit each. */ +export const MIGRATION_CHUNK = 100; + +/** Migrates to `definition`'s version from its (A) commit (`begin`) or after it (`resume`). */ +async function migrateVersion( + definition: TableDefinition, + from: "begin" | "resume", + target: MigrationTarget, + context: Context, +): Promise { + const { before, step, after } = definition.evolution!.migration; + const { table } = target; + const edits = (list: typeof before): StagedWrite[] => list.map((edit) => stageEdit(table, definition.version, edit)); + const ready = [...edits(after), recordOf(definition, definition, "ready", table)]; + if (step === undefined) return commit([...edits(before), ...ready], target, context); + if (from === "begin") + await commit([...edits(before), recordOf(step.table, definition, "migrating", table)], target, context); + if (step.kind === "update") return commit([updateStep(step, definition.version, table), ...ready], target, context); + await rowsStep(step, definition.version, ready, target, context); +} + +/** The `table` write of `table`'s columns and key, with `definition`'s version and fingerprint, in `state`. */ +function recordOf( + columns: TableDefinition, + definition: TableDefinition, + state: "ready" | "migrating", + table: string, +): StagedWrite { + const { version, fingerprint } = definition; + return stageTable({ kind: "table", table, version, state, columns: columns.record, key: columns.key, fingerprint }); +} + +/** The `update` step: one `updateWhere` over the rows below version `v`, stamping it, so it never applies twice. */ +function updateStep(step: Extract, v: number, table: string): StagedWrite { + const sql = sqlOf(step.table, table); + const below: WhereNode = { lt: [ROW_VERSION, v] }; + const where: WhereNode = step.where === undefined ? below : { and: [step.where, below] }; + const versions = jsonVersions(step.table, step.set); + const statement = sql.updateWhere(where, { ...step.set, [ROW_VERSION]: v }, versions); + return stageUpdateWhere(statement, table, where, step.set, recordedVersions(step.set, versions), v); +} + +/** + * The `rows` step: chunks of `MIGRATION_CHUNK` rows below version `v` in key order, paging forward by the last key; + * each patched row stamped `v`; the last chunk commits with `ready`. + */ +async function rowsStep( + step: Extract, + v: number, + ready: readonly StagedWrite[], + target: MigrationTarget, + context: Context, +): Promise { + const { table: columns } = step; + const sql = sqlOf(columns, target.table); + const order: Order = { columns: columns.key, direction: "asc" }; + const below: WhereNode = { lt: [ROW_VERSION, v] }; + let after: SqliteValue[] | undefined; + // The fallback yield between chunks (`macrotask`), made by the first yield that needs it, closed at the end. + let channel: InstanceType | undefined; + try { + for (;;) { + const statement = sql.select(below, order, after, MIGRATION_CHUNK + 1); + const stored = await target.store.read( + (executor) => executor.all<{ [column: string]: SqliteValue }>(statement.sql, ...statement.params), + context, + ); + const last = stored.length <= MIGRATION_CHUNK; + if (!last) stored.length = MIGRATION_CHUNK; + const writes = stored.map((row) => patchRow(step, row, v, sql, target.table, context)); + if (last) return await commit([...writes, ...ready], target, context); + await commit(writes, target, context); + after = columns.key.map((name) => stored[MIGRATION_CHUNK - 1]![name]!); + await macrotask(() => { + channel ??= new MessageChannel(); + return channel; + }); + // An abandoned activation stops here. + context.abortSignal?.throwIfAborted(); + } + } finally { + channel?.port1.close(); + } +} + +/** + * Resolves after one macrotask (other commits and timers run between chunks) without the 1 ms clamp of + * `setTimeout(0)`: `setImmediate` where the runtime has it (Node), else a message through the channel `channel()` + * gives (browsers, Workers). In Node a message costs 0.27 ms per yield against `setImmediate`'s 13 µs (the 10k-row + * migration bench: 100 ms against 70 ms). + */ +function macrotask(channel: () => InstanceType): Promise { + return new Promise((resolve) => { + // A runtime feature check: Workers and browsers have no `setImmediate`. + if ("setImmediate" in globalThis) { + setImmediate(resolve); + return; + } + const { port1, port2 } = channel(); + port1.addEventListener("message", () => resolve(), { once: true }); + // A port listened to with `addEventListener` delivers only once started (browsers, Workers). + port1.start(); + port2.postMessage(null); + }); +} + +/** + * The write of one row's patch: `patch` run on the decoded row, its result checked (an object of non-key columns of + * the version, values of their types) and stamped `v` through the internal form of `update`. + */ +function patchRow( + step: Extract, + stored: { readonly [column: string]: SqliteValue }, + v: number, + sql: TableSql, + table: string, + context: Context, +): StagedWrite { + const { table: definition } = step; + const patch = step.patch(definition.rows.decode(stored, context)); + const what = `${definition.name} version ${v}: rows`; + if (typeof patch !== "object" || patch === null || Array.isArray(patch)) { + throw new Error(`${what}: expected an object of column values`); + } + const set: { [column: string]: SetNode[string] } = {}; + for (const [name, value] of Object.entries(patch)) { + const column = definition.byName.get(name); + if (column === undefined) throw new Error(`${what}: no column "${name}"`); + if (definition.key.includes(name)) throw new Error(`${what}: key column "${name}" cannot change`); + set[name] = cellOf(column, value, `${what}: column "${name}"`); + } + const key: { [column: string]: Scalar } = {}; + // Key columns are non-opt `str` or `int`: stored scalars. + for (const name of definition.key) key[name] = stored[name] as Scalar; + const versions = jsonVersions(definition, set); + const params = definition.key.map((name) => stored[name]!); + const statement = sql.update({ ...set, [ROW_VERSION]: v }, params, versions); + return stageUpdate(statement, table, key, set, recordedVersions(set, versions), v); +} + +/** + * A patch value as a `set` node: `undefined` clears an `opt(...)` column, `json(...)` cells as JSON literals (a JSON + * `null` too); a scalar `null` is refused by `encodeCell`. + */ +function cellOf(column: Column, value: unknown, what: string): SetNode[string] { + if (value === undefined) { + if (!column.optional) throw new Error(`${what} is not opt(...) and cannot be cleared`); + return null; + } + const cell = encodeCell(column, value, what); + // `encodeCell` returns a scalar for a scalar column. + return column.type === "json" ? { json: cell.stored } : (cell.stored as Scalar); +} + +function sqlOf(definition: TableDefinition, table: string): TableSql { + return new TableSql(table, definition.columns, definition.rows.physical, definition.key); +} + +/** Commits `writes` without the activation's abort: a commit admitted to the step completes (the fence refuses one). */ +async function commit(writes: StagedWrite[], target: Omit, context: Context): Promise { + await target.store.commit(writes, target.fence, withoutAbortSignal(context)); +} diff --git a/packages/chord/src/next/store/table-writes.ts b/packages/chord/src/next/store/table-writes.ts new file mode 100644 index 000000000..101c7f8b3 --- /dev/null +++ b/packages/chord/src/next/store/table-writes.ts @@ -0,0 +1,260 @@ +// Table writes (slice 6a §5, §7, §9; S17; decisions "Writes carry their versions", "Index definitions are recorded"): +// the recorded form of every table write, the staged writes a transaction or an activation hands the commit step +// (their SQL rendered at staging, rule 8), and the appliers of the schema writes, which render DDL from the record. + +import type { TableRecord } from "./catalog.ts"; +import type { ColumnRecord } from "./columns.ts"; +import type { SetNode, WhereNode } from "./expression.ts"; +import type { EncodedRow, Scalar, StoredRow } from "./rows.ts"; +import { versionColumn } from "./rows.ts"; +import { columnSql, createIndexSql, createTableSql, dropIndexSql, quote, type Statement } from "./sql.ts"; +import type { StagedWrite, Step } from "./step.ts"; +import type { TableDefinition } from "./table.ts"; +import type { AlterEdit, TableEdit } from "./table-history.ts"; + +/** A key as writes record it: the key columns' stored values. */ +export type StoredKey = { readonly [column: string]: Scalar }; + +/** `json(...)` columns' versions written by a write, by column. */ +export type ColumnVersions = { readonly [column: string]: number }; + +/** The table writes as the outbox and archives record them. */ +export type TableWrite = + | { + readonly kind: "table"; + readonly table: string; + readonly version: number; + readonly state: "ready" | "migrating"; + readonly columns: { readonly [name: string]: ColumnRecord }; + readonly key: readonly string[]; + readonly fingerprint: string; + } + | { readonly kind: "index"; readonly table: string; readonly name: string; readonly columns: readonly string[] } + | { readonly kind: "dropIndex"; readonly table: string; readonly name: string } + /** A column edit of the migration to `version`; DDL rendered from it. */ + | { readonly kind: "alter"; readonly table: string; readonly version: number; readonly edit: AlterEdit } + | { + readonly kind: "put"; + readonly table: string; + readonly row: StoredRow; + /** The row's eager-step version (the table's version at the `put`). */ + readonly version: number; + readonly columns: ColumnVersions; + } + | { + readonly kind: "update"; + readonly table: string; + readonly key: StoredKey; + readonly set: SetNode; + readonly columns: ColumnVersions; + readonly count: number; + /** A migration's data step: the version it stamps as the row's `$v` (a plugin's `update` never does). */ + readonly version?: number; + } + | { readonly kind: "delete"; readonly table: string; readonly key: StoredKey; readonly count: number } + | { + readonly kind: "updateWhere"; + readonly table: string; + readonly where: WhereNode; + readonly set: SetNode; + readonly columns: ColumnVersions; + readonly count: number; + /** A migration's `update` step: the version it stamps as the rows' `$v`. */ + readonly version?: number; + } + | { readonly kind: "deleteWhere"; readonly table: string; readonly where: WhereNode; readonly count: number }; + +/** The versions of the `json(...)` columns of `definition` that `set` writes, which its statement stamps. */ +export function jsonVersions(definition: TableDefinition, set: SetNode): ReadonlyMap { + const versions = new Map(); + for (const name of Object.keys(set)) { + const column = definition.byName.get(name)!; + if (column.type === "json") versions.set(name, column.json.version); + } + return versions; +} + +/** The `columns` a write records: the versions of the `json(...)` columns it sets to a value (a clear has none). */ +export function recordedVersions(set: SetNode, versions: ReadonlyMap): ColumnVersions { + const columns: { [column: string]: number } = {}; + for (const [column, version] of versions) if (set[column] !== null) columns[column] = version; + return columns; +} + +/** The `table` write. */ +export type TableDefinitionWrite = Extract; + +/** Stages a `table` write: the table created when the store does not have it yet, its record replaced otherwise. */ +export function stageTable(write: TableDefinitionWrite): StagedWrite { + return { order: "write", apply: (step) => applyTable(step, write) }; +} + +/** Stages an `index` write. */ +export function stageIndex(write: Extract): StagedWrite { + return { order: "write", apply: (step) => applyIndex(step, write) }; +} + +/** Stages one edit of the migration of `table` to `version` as its `alter`, `index` or `dropIndex` write. */ +export function stageEdit(table: string, version: number, edit: TableEdit): StagedWrite { + switch (edit.kind) { + case "index": + return stageIndex({ kind: "index", table, name: edit.name, columns: edit.columns }); + case "dropIndex": { + const write = { kind: "dropIndex", table, name: edit.name } as const; + return { order: "write", apply: (step) => applyDropIndex(step, write) }; + } + default: { + const write = { kind: "alter", table, version, edit } as const; + return { order: "write", apply: (step) => applyAlter(step, write) }; + } + } +} + +/** Stages a `put` of an encoded row with its `INSERT OR REPLACE` (an upsert: the whole row replaced). */ +export function stagePut(sql: string, table: string, row: EncodedRow, version: number): StagedWrite { + const write: TableWrite = { kind: "put", table, row: row.row, version, columns: row.columns }; + return { + order: "write", + apply: async (step) => { + await step.run(sql, ...row.params); + return write; + }, + }; +} + +/** + * Stages an `update` of one row by key; its count (0 for an absent key) is recorded, and `version` when the statement stamps + * the row's `$v` (a migration's `rows` step). + */ +export function stageUpdate( + statement: Statement, + table: string, + key: StoredKey, + set: SetNode, + columns: ColumnVersions, + version?: number, +): StagedWrite { + const stamped = version === undefined ? {} : { version }; + return counted(statement, (count) => ({ kind: "update", table, key, set, columns, count, ...stamped })); +} + +/** Stages a `delete` of one row by key; its count is recorded. */ +export function stageDelete(statement: Statement, table: string, key: StoredKey): StagedWrite { + return counted(statement, (count) => ({ kind: "delete", table, key, count })); +} + +/** + * Stages an `updateWhere`: one statement, its count recorded (a replica affecting another count stops, S17), and `version` + * when it stamps the rows' `$v` (a migration's `update` step). + */ +export function stageUpdateWhere( + statement: Statement, + table: string, + where: WhereNode, + set: SetNode, + columns: ColumnVersions, + version?: number, +): StagedWrite { + const stamped = version === undefined ? {} : { version }; + return counted(statement, (count) => ({ kind: "updateWhere", table, where, set, columns, count, ...stamped })); +} + +/** Stages a `deleteWhere`: one statement, its count recorded. */ +export function stageDeleteWhere(statement: Statement, table: string, where: WhereNode): StagedWrite { + return counted(statement, (count) => ({ kind: "deleteWhere", table, where, count })); +} + +function counted(statement: Statement, record: (count: number) => TableWrite): StagedWrite { + return { + order: "write", + apply: async (step) => record(await step.runCounted(statement.sql, ...statement.params)), + }; +} + +/** Applies a `table` write: `CREATE TABLE` when the store does not have the table yet, then its record. */ +export async function applyTable(step: Step, write: TableDefinitionWrite): Promise { + const existing = step.table(write.table); + if (existing === undefined) await step.run(createTableSql(write.table, write.columns, write.key)); + const { table, version, state, columns, key, fingerprint } = write; + step.tables.set(table, { table, version, state, columns, key, fingerprint, indexes: existing?.indexes ?? {} }); + return write; +} + +/** Applies an `index` write: `CREATE INDEX` and the record's index. */ +export async function applyIndex( + step: Step, + write: Extract, +): Promise { + const existing = recorded(step, write.table); + await step.run(createIndexSql(write.table, write.name, write.columns, existing.key)); + step.tables.set(write.table, { ...existing, indexes: { ...existing.indexes, [write.name]: write.columns } }); + return write; +} + +/** Applies a `dropIndex` write: `DROP INDEX` and the record without it. */ +export async function applyDropIndex( + step: Step, + write: Extract, +): Promise { + const existing = recorded(step, write.table); + await step.run(dropIndexSql(write.table, write.name)); + const { [write.name]: _dropped, ...indexes } = existing.indexes; + step.tables.set(write.table, { ...existing, indexes }); + return write; +} + +/** Applies an `alter` write: its DDL (a `json(...)` column's version column follows it) and the record's change. */ +export async function applyAlter( + step: Step, + write: Extract, +): Promise { + const existing = recorded(step, write.table); + const { edit } = write; + const id = quote(write.table); + switch (edit.kind) { + case "rename": { + const json = existing.columns[edit.from]?.type === "json"; + await step.run(`ALTER TABLE ${id} RENAME COLUMN ${quote(edit.from)} TO ${quote(edit.to)}`); + if (json) { + const [from, to] = [versionColumn(edit.from), versionColumn(edit.to)]; + await step.run(`ALTER TABLE ${id} RENAME COLUMN ${quote(from)} TO ${quote(to)}`); + } + const rename = (column: string): string => (column === edit.from ? edit.to : column); + const columns = Object.entries(existing.columns).map(([name, column]) => [rename(name), column] as const); + const indexes = Object.entries(existing.indexes).map(([name, on]) => [name, on.map(rename)] as const); + step.tables.set(write.table, { + ...existing, + columns: Object.fromEntries(columns), + key: existing.key.map(rename), + indexes: Object.fromEntries(indexes), + }); + break; + } + case "add": + for (const definition of columnSql(edit.name, edit.column)) { + await step.run(`ALTER TABLE ${id} ADD COLUMN ${definition}`); + } + step.tables.set(write.table, { ...existing, columns: { ...existing.columns, [edit.name]: edit.column } }); + break; + case "drop": { + const json = existing.columns[edit.name]?.type === "json"; + await step.run(`ALTER TABLE ${id} DROP COLUMN ${quote(edit.name)}`); + if (json) await step.run(`ALTER TABLE ${id} DROP COLUMN ${quote(versionColumn(edit.name))}`); + const { [edit.name]: _dropped, ...columns } = existing.columns; + step.tables.set(write.table, { ...existing, columns }); + break; + } + case "json": + step.tables.set(write.table, { ...existing, columns: { ...existing.columns, [edit.name]: edit.column } }); + break; + } + return write; +} + +/** The record of `table` in `step`; a write for a table the store does not have fails the step. */ +function recorded(step: Step, table: string): TableRecord { + const record = step.table(table); + // Memory and disk disagree: the step fails, which stops the store. + if (record === undefined) throw new Error(`table ${table}: not in the store`); + return record; +} diff --git a/packages/chord/src/next/store/table.ts b/packages/chord/src/next/store/table.ts new file mode 100644 index 000000000..d86d2fa16 --- /dev/null +++ b/packages/chord/src/next/store/table.ts @@ -0,0 +1,157 @@ +// Table definitions (slice 6a §5, §9; decisions "Flat columns", "Restricted names", "Index definitions are recorded"): +// `defineTable(name, { version, columns, key, indexes? })`, checked and materialized once at declaration: its columns, +// its key, its indexes, its fingerprint and the compiled row codec its writes and reads use. A version `evolveTable` +// appends links to the one before it and to the migration between them. + +import { checkName, hash64 } from "../contracts/internal.ts"; +import { + type Column, + type ColumnRecord, + type Columns, + columnRecord, + describeColumns, + type OrderColumn, +} from "./columns.ts"; +import { type ColumnRef, columnRefs } from "./expression.ts"; +import { RowCodec } from "./rows.ts"; +import type { Migration } from "./table-history.ts"; + +/** What `defineTable` takes. */ +export type TableOptions[], V extends number = number> = { + /** The table's version: a positive integer, raised by every change (`evolveTable`). */ + readonly version: V; + readonly columns: C; + /** The primary key's columns, non-`opt` `str` or `int` ones. */ + readonly key: K; + /** Secondary indexes by name (`[a-z0-9_]+`); each ends with the primary key. */ + readonly indexes?: { readonly [name: string]: readonly OrderColumn[] }; +}; + +/** The key of a row of a table with columns `C` and key `K`: an object with exactly the key columns. */ +export type KeyOf = { + readonly [P in K[number]]: P extends keyof C + ? C[P] extends { readonly "~kind": "Integer" } + ? number + : string + : never; +}; + +/** + * Declares a table: a base version written in full, the first of its history. Later versions come from + * `evolveTable`; a new base starts a new history. + */ +export function defineTable[], const V extends number>( + name: string, + options: TableOptions, +): TableDefinition { + return new TableDefinition(name, options); +} + +/** The table form's version: the fingerprints of table definitions are persisted, so their input is versioned. */ +export const TABLE_FORM_VERSION = 1; + +/** The index name rule. */ +export const INDEX_NAME = /^[a-z0-9_]+$/; + +/** How a version came about: the version before it and the migration from there, for one `evolveTable`. */ +export type Evolution = { readonly previous: TableDefinition; readonly migration: Migration }; + +/** + * A declared table at one version: its columns in declaration order, key, indexes (declared columns; the key is + * appended when they are rendered), the fingerprint of all of it and the row codec. `C` and `K` type its rows and keys, + * `V` its version. + */ +export class TableDefinition< + C extends Columns = Columns, + K extends readonly string[] = readonly string[], + V extends number = number, +> { + readonly name: string; + readonly version: V; + /** The column schemas as declared, which `evolveTable` edits into the next version's. */ + readonly schemas: Columns; + readonly columns: readonly Column[]; + readonly byName: ReadonlyMap; + readonly key: readonly string[]; + readonly indexes: ReadonlyMap; + /** `hash64` of the columns, key and indexes in `TABLE_FORM_VERSION`'s form. */ + readonly fingerprint: string; + /** The columns as the `table` write records them. */ + readonly record: { readonly [name: string]: ColumnRecord }; + readonly rows: RowCodec; + /** One reference per column, for the `(t) => …` callbacks of queries and writes. */ + readonly refs: { readonly [column: string]: ColumnRef }; + /** The version before this one and the migration from it; `undefined` for a base. */ + readonly evolution: Evolution | undefined; + /** Type only: the columns and key. */ + declare readonly "~columns"?: C; + declare readonly "~key"?: K; + + constructor(name: string, options: TableOptions[], V>, evolution?: Evolution) { + checkName(name, "table"); + const { version, columns, key, indexes = {} } = options; + // Declaration policy: JavaScript callers reach these checks. + if (!Number.isSafeInteger(version) || version < 1) + throw new Error(`${name}: version ${version} is not a positive integer`); + this.name = name; + this.version = version; + this.schemas = columns; + this.columns = describeColumns(columns, name); + this.byName = new Map(this.columns.map((column) => [column.name, column])); + this.key = orderColumns(this.byName, key, `${name}: the key`); + if (this.key.length === 0) throw new Error(`${name}: the key has no column`); + this.indexes = new Map( + Object.entries(indexes).map(([index, columns]) => { + if (!INDEX_NAME.test(index)) throw new Error(`${name}: index "${index}": expected [a-z0-9_]+`); + return [index, orderColumns(this.byName, columns, `${name}: index "${index}"`)] as const; + }), + ); + this.record = Object.fromEntries(this.columns.map((column) => [column.name, columnRecord(column)])); + this.fingerprint = tableFingerprint(this.record, this.key, this.indexes); + this.rows = new RowCodec(this.columns, this.key, name); + this.refs = columnRefs(this.columns); + this.evolution = evolution; + } + + /** The column `name`; an unknown one throws (`what` names the use). */ + column(name: string, what: string): Column { + const column = this.byName.get(name); + if (column === undefined) throw new Error(`${this.name}: ${what}: no column "${name}"`); + return column; + } +} + +/** + * Checks `names` as key, index or order columns of a table whose columns are `byName`: existing, non-`opt` `str` or + * `int`, no column twice (declaration policy: JavaScript callers reach it); `what` names the use in errors. + */ +export function orderColumns( + byName: ReadonlyMap, + names: readonly string[], + what: string, +): readonly string[] { + if (!Array.isArray(names)) throw new Error(`${what}: expected an array of column names`); + if (new Set(names).size !== names.length) throw new Error(`${what}: a column occurs twice`); + for (const name of names) { + const column = byName.get(name); + if (column === undefined) throw new Error(`${what}: no column "${name}"`); + if (column.optional || (column.type !== "str" && column.type !== "int")) { + throw new Error(`${what}: column "${name}" is not a non-opt str() or int() column`); + } + } + return [...names]; +} + +/** The fingerprint of a table's columns, key and indexes; column and index names sorted (their order is no change). */ +export function tableFingerprint( + columns: { readonly [name: string]: ColumnRecord }, + key: readonly string[], + indexes: ReadonlyMap, +): string { + const value = { columns: sorted(Object.entries(columns)), key, indexes: sorted([...indexes]) }; + return hash64(JSON.stringify({ tableFormVersion: TABLE_FORM_VERSION, value })); +} + +function sorted(entries: [string, T][]): { [name: string]: T } { + return Object.fromEntries(entries.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))); +} diff --git a/packages/chord/src/next/store/text.ts b/packages/chord/src/next/store/text.ts new file mode 100644 index 000000000..e526e27b9 --- /dev/null +++ b/packages/chord/src/next/store/text.ts @@ -0,0 +1,51 @@ +// Text in the store (decision "Text is plain UTF-8, in code point order"): Chord compares strings by code point, as +// SQLite's BINARY collation compares their UTF-8 bytes, never by JavaScript's UTF-16 code units; every stored string +// is well-formed and free of NUL, and a prefix becomes a range by its upper bound. + +import { invalid } from "../data/internal.ts"; + +/** Compares `a` and `b` by code point: negative, zero or positive. */ +export function compareText(a: string, b: string): number { + const length = Math.min(a.length, b.length); + for (let i = 0; i < length; i++) { + const x = a.charCodeAt(i); + const y = b.charCodeAt(i); + if (x !== y) return codePointRank(x) - codePointRank(y); + } + return a.length - b.length; +} + +/** + * A UTF-16 code unit's rank in code point order: a surrogate starts a code point above U+FFFF, so it ranks above every + * unit from U+E000 on. + */ +function codePointRank(unit: number): number { + if (unit < 0xd800) return unit; + return unit < 0xe000 ? unit + 0x2000 : unit - 0x800; +} + +/** + * `text` as the store keeps it: a lone surrogate replaced by U+FFFD (copied only when one occurs), NUL refused + * `invalid/args` (`what` names the value). Plugin strings are a trust boundary. + */ +export function storedText(text: string, what: string): string { + if (text.includes("\0")) throw invalid("args", `${what}: text may not contain NUL`); + return text.isWellFormed() ? text : text.toWellFormed(); +} + +/** + * The smallest string above every string starting with `prefix`, in code point order: the next code point after its + * last, carrying past U+10FFFF; `undefined` when there is none (`""` and all-U+10FFFF prefixes). `prefix` is + * well-formed. + */ +export function prefixUpperBound(prefix: string): string | undefined { + const points = [...prefix]; + while (points.length > 0) { + const last = points.pop()!.codePointAt(0)!; + if (last === 0x10ffff) continue; + // The surrogates are no code points of well-formed text: after U+D7FF comes U+E000. + const next = last === 0xd7ff ? 0xe000 : last + 1; + return points.join("") + String.fromCodePoint(next); + } + return undefined; +} diff --git a/packages/chord/src/next/store/writes.ts b/packages/chord/src/next/store/writes.ts new file mode 100644 index 000000000..8b10b8797 --- /dev/null +++ b/packages/chord/src/next/store/writes.ts @@ -0,0 +1,8 @@ +// The store's write vocabulary (`StoreWrite`, slice 6a §7, §8; decision "Writes carry their versions"): one change of +// the store each, as the outbox and archives record it. + +import type { DocumentWrite } from "./engine.ts"; +import type { TableWrite } from "./table-writes.ts"; + +/** One change of the store, tagged by `kind`. */ +export type StoreWrite = DocumentWrite | TableWrite; diff --git a/packages/chord/test/next-node/compression.test.ts b/packages/chord/test/next-node/compression.test.ts new file mode 100644 index 000000000..00f50adf4 --- /dev/null +++ b/packages/chord/test/next-node/compression.test.ts @@ -0,0 +1,39 @@ +// Tests of nodeCompression: raw deflate and the bounded inflate with its tagged outcomes, a limit of 0 included. + +import { describe, expect, it } from "vitest"; +import type { Inflated } from "../../src/next/link/part.ts"; +import { nodeCompression } from "../../src/next-node/compression.ts"; + +const UTF8 = new TextEncoder(); + +const bytesOf = (inflated: Inflated) => (inflated.kind === "bytes" ? new Uint8Array(inflated.bytes) : inflated); + +describe("nodeCompression", () => { + it("deflates raw at level 1 and inflates back", () => { + const bytes = UTF8.encode(JSON.stringify({ text: "lorem ipsum ".repeat(5000) })); + const deflated = nodeCompression.deflate(bytes); + expect(deflated.byteLength).toBeLessThan(bytes.byteLength / 10); + expect(bytesOf(nodeCompression.inflate(deflated, bytes.byteLength))).toEqual(bytes); + expect(bytesOf(nodeCompression.inflate(deflated, Number.POSITIVE_INFINITY))).toEqual(bytes); + }); + + it("gives too_large past the limit, without inflating the rest (a bomb)", () => { + const bomb = nodeCompression.deflate(new Uint8Array(256 * 1024 * 1024)); + expect(nodeCompression.inflate(bomb, 1024 * 1024)).toEqual({ kind: "too_large" }); + }); + + it("takes a limit of 0: an empty stream inflates, anything else is too_large (never zlib's range error)", () => { + expect(bytesOf(nodeCompression.inflate(nodeCompression.deflate(new Uint8Array(0)), 0))).toEqual( + new Uint8Array(0), + ); + expect(nodeCompression.inflate(nodeCompression.deflate(new Uint8Array(1)), 0)).toEqual({ kind: "too_large" }); + expect(nodeCompression.inflate(nodeCompression.deflate(new Uint8Array(100)), 0)).toEqual({ kind: "too_large" }); + expect(bytesOf(nodeCompression.inflate(nodeCompression.deflate(new Uint8Array(1)), 1))).toEqual( + new Uint8Array(1), + ); + }); + + it("gives corrupt for data that is no raw deflate stream", () => { + expect(nodeCompression.inflate(new Uint8Array([255, 1, 2, 3]), 100)).toMatchObject({ kind: "corrupt" }); + }); +}); diff --git a/packages/chord/test/next/bench/compression.ts b/packages/chord/test/next/bench/compression.ts new file mode 100644 index 000000000..7b74ad8f8 --- /dev/null +++ b/packages/chord/test/next/bench/compression.ts @@ -0,0 +1,155 @@ +// Light benchmark of compression at the origin (slice 4b): a 500 KB snapshot and a 1 MiB call result, uncompressed and +// deflated (level 1) by the server; their bytes on the wire, the deflate at the origin, the inflate at the owner +// (synchronous `nodeCompression`, asynchronous `DecompressionStream`), and end to end over a memory link. +// Run by hand from packages/chord (not in CI): +// node --conditions=source --experimental-strip-types --expose-gc test/next/bench/compression.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { type Compression, type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; +import { DeflatedPart, inflateAsync } from "../../../src/next/link/internal.ts"; +import { nodeCompression } from "../../../src/next-node/compression.ts"; + +const context = BACKGROUND_CONTEXT; + +type P = { kind: "owner" } | { kind: "server" }; + +const Entry = obj({ id: int(), text: str(), author: str(), at: int() }); + +const Store = local("bench.store", { version: 1, params: {}, members: { doc: obj({ entries: arr(Entry) }) } }); + +/** A tab's view of the 500 KB document, and a method returning 1 MiB. */ +const View = remote("bench.view", { + version: 1, + params: {}, + variants: { owner: { state: { doc: obj({ entries: arr(Entry) }) }, methods: { export: method(undefined, str()) } } }, +}); + +/** Words that make text compress like prose and code (about 3x at level 1), not like a repeated string. */ +const WORDS = "the a model tool call file read write test error result agent session note code run build".split(" "); + +/** About `length` characters of words, from a fixed pseudo-random sequence starting at `seed`. */ +function prose(seed: number, length: number): string { + let state = seed * 7919 + 1; + const words: string[] = []; + for (let total = 0; total < length; total += words.at(-1)!.length + 1) { + state = (state * 1103515245 + 12345) % 2147483648; + const word = WORDS[state % WORDS.length]!; + words.push(state % 5 === 0 ? `${word}${state % 1000}` : word); + } + return words.join(" ").slice(0, length); +} + +/** The 500 KB document: 500 entries of 960 characters, as in the slice 3a benchmark, with prose-like text. */ +const entries = Array.from({ length: 500 }, (_, id) => ({ + id, + text: prose(id, 960), + author: "mario", + at: 1_700_000_000 + id, +})); + +/** The 1 MiB result. */ +const RESULT = prose(1, 1024 * 1024); + +const Doc = defineState("doc", { schema: obj({ entries: arr(Entry) }), version: 1, initial: () => ({ entries }) }); + +const facet = defineFacet

((host) => { + host.serve(Store, (_, self) => ({ doc: self.state(Doc) })); + host.serve(View, (_, _principal, self) => ({ + variant: "owner", + members: { doc: self.get(Store, {}).doc, export: async () => RESULT }, + })); +}); + +/** A link end counting the bytes its host receives. */ +function counted(link: Link, bytes: { received: number }): Link { + return { + send: (message) => link.send(message), + onMessage: (listener) => + link.onMessage((message) => { + bytes.received += message.byteLength; + listener(message); + }), + close: () => link.close(), + onClose: (listener) => link.onClose(listener), + }; +} + +/** The mean milliseconds of `runs` runs of `work`, the fastest of three such loops. */ +async function mean(runs: number, work: () => unknown): Promise { + let best = Number.POSITIVE_INFINITY; + for (let loop = 0; loop < 3; loop++) { + const started = performance.now(); + for (let run = 0; run < runs; run++) await work(); + best = Math.min(best, (performance.now() - started) / runs); + } + return best; +} + +const UTF8 = new TextEncoder(); + +const kib = (bytes: number) => `${(bytes / 1024).toFixed(1)} KiB`; + +const ms = (value: number) => `${value.toFixed(3)} ms`; + +console.log( + "| Payload | JSON | deflated (level 1) | deflate at the origin | inflate, sync (`nodeCompression`) | inflate, async (`DecompressionStream`) |", +); +console.log("| --- | --- | --- | --- | --- | --- |"); +for (const [name, json] of [ + ["500 KB snapshot", JSON.stringify({ entries })], + ["1 MiB result", JSON.stringify(RESULT)], +] as const) { + const bytes = UTF8.encode(json); + const deflated = nodeCompression.deflate(bytes); + const deflate = await mean(20, () => nodeCompression.deflate(bytes)); + const inflate = await mean(20, () => nodeCompression.inflate(deflated, Number.POSITIVE_INFINITY)); + const inflateAsyncMs = await mean(20, () => + inflateAsync(new DeflatedPart(deflated), Number.POSITIVE_INFINITY, new AbortController().signal, context), + ); + console.log( + `| ${name} | ${kib(bytes.byteLength)} | ${kib(deflated.byteLength)} | ${ms(deflate)} | ${ms(inflate)} | ${ms(inflateAsyncMs)} |`, + ); +} + +/** A server compressing with `server` and a client with `client` (none: it inflates asynchronously). */ +async function linked(server: Compression | undefined, client: Compression | undefined) { + const serverHost = createHost

({ principal: { kind: "server" }, compression: server, onError: console.error }); + await serverHost.load([{ name: "@bench/app", facet }], context); + const clientHost = createHost

({ compression: client, onError: console.error }); + await clientHost.load([], context); + const [a, b] = memoryLinkPair(); + const bytes = { received: 0 }; + serverHost.accept(a, { principal: { kind: "owner" } }, context); + clientHost.connect(counted(b, bytes), { principal: { kind: "server" } }, context); + return { server: serverHost, client: clientHost, bytes }; +} + +console.log(); +console.log( + "| End to end over a memory link (binary codec) | snapshot: bytes received | hold to live (mean of 20) | 1 MiB result: bytes received | call to result (mean of 20) |", +); +console.log("| --- | --- | --- | --- | --- |"); +for (const [label, server, client] of [ + ["uncompressed", undefined, undefined], + ["deflated, inflated synchronously (Node client)", nodeCompression, nodeCompression], + ["deflated, inflated asynchronously (browser client)", nodeCompression, undefined], +] as const) { + const hosts = await linked(server, client); + // Warm up: the store and the view activate once. + (await hosts.client.get(View, {}, context)).release(); + const before = hosts.bytes.received; + const view = await hosts.client.get(View, {}, context); + const snapshotBytes = hosts.bytes.received - before; + view.release(); + const hold = await mean(20, async () => (await hosts.client.get(View, {}, context)).release()); + const held = await hosts.client.get(View, {}, context); + const beforeCall = hosts.bytes.received; + await held.export(context); + const resultBytes = hosts.bytes.received - beforeCall; + const call = await mean(20, () => held.export(context)); + console.log(`| ${label} | ${kib(snapshotBytes)} | ${ms(hold)} | ${kib(resultBytes)} | ${ms(call)} |`); + await hosts.client.close(context); + await hosts.server.close(context); +} diff --git a/packages/chord/test/next/bench/derives.ts b/packages/chord/test/next/bench/derives.ts new file mode 100644 index 000000000..7c2785173 --- /dev/null +++ b/packages/chord/test/next/bench/derives.ts @@ -0,0 +1,73 @@ +// Benchmark of a retirement through derives (slice 6a rule 21): a singleton under a diamond of 40 derives (13 layers of +// two derives joined by a third, 2^13 paths from the top to the bottom), its bottom watched; each step retires the +// document and creates it again in one transaction, so every step carries the `retired` flag through the diamond. +// Median and p95 per step over several bursts. The flag reaches each derive once per retirement, not once per path. +// Run by hand from packages/chord (not in CI): +// node --conditions=source --experimental-strip-types --expose-gc test/next/bench/derives.ts +import { performance } from "node:perf_hooks"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Self } from "../../../src/next/host/index.ts"; +import type { Watchable } from "../../../src/watch.ts"; + +const STEPS = 1_000; +const RUNS = 7; +const LAYERS = 13; + +const List = defineState("bench.list", { + schema: obj({ items: arr(str()) }), + version: 1, + initial: () => ({ items: [] }), +}); +const Diamond = local("bench.diamond", { version: 1, params: {}, members: { bottom: int(), touch: method() } }); + +/** The diamond over `list`: its bottom derive. */ +function diamond(self: Self, list: SingletonHandle<{ items: string[] }>): Watchable { + let previous: Watchable = self.derive([list], ([value]) => value.items.length); + for (let layer = 0; layer < LAYERS; layer++) { + const left = self.derive([previous], ([n]) => n + 1); + const right = self.derive([previous], ([n]) => n + 2); + previous = self.derive([left, right], ([a, b]) => (a + b) % 1_000); + } + return previous; +} + +/** Per-step times (ms) of `RUNS` bursts of `STEPS` retire-and-recreate transactions, after one warm-up burst. */ +async function bursts(): Promise<{ readonly median: number; readonly p95: number }> { + const times: number[] = []; + for (let run = -1; run < RUNS; run++) { + let list: SingletonHandle<{ items: string[] }> | undefined; + let self: Self | undefined; + const host = createHost({ onError: () => {} }); + const facet = defineFacet((h) => + h.serve(Diamond, (_, declared) => { + self = declared; + list = declared.state(List); + return { bottom: diamond(declared, list), touch: async () => null }; + }), + ); + await host.load([{ name: "@bench/derives", facet }], context); + const reference = await host.get(Diamond, {}, context); + const watch = await reference.bottom.watch(context); + watch.start(async () => {}); + for (let step = 0; step < STEPS; step++) { + const before = performance.now(); + await self!.tx(async (tx) => { + await tx.retire(list!); + (await tx.draft(list!)).items.push(`${step}`); + }, context); + if (run >= 0) times.push(performance.now() - before); + } + await watch.stop(); + reference.release(); + await host.close(context); + } + times.sort((a, b) => a - b); + return { median: times[Math.floor(times.length / 2)]!, p95: times[Math.floor(times.length * 0.95)]! }; +} + +const { median, p95 } = await bursts(); +console.log( + `retire + recreate under a ${1 + 3 * LAYERS}-derive diamond: ${(median * 1000).toFixed(1)} µs median, ${(p95 * 1000).toFixed(1)} µs p95`, +); diff --git a/packages/chord/test/next/bench/links.ts b/packages/chord/test/next/bench/links.ts new file mode 100644 index 000000000..3bbbf6d31 --- /dev/null +++ b/packages/chord/test/next/bench/links.ts @@ -0,0 +1,469 @@ +// Light benchmarks of remote objects over one memory link (slices 3a, 3b), per codec side by side: wire bytes per +// message; compute (encode and decode time per message, commit-to-frame end to end); memory (heap allocated per message +// while encoding and decoding, retained per hold); drops (3b: reconnects re-holding 1 and 100 references, 1,000 calls +// parked while the link is down). Payloads are modelled on p2: a notes list, a 500 KB conversation document, a reply +// streamed token by token, calls with and without bytes. +// Run by hand from packages/chord (not in CI): +// node --conditions=source --experimental-strip-types --expose-gc test/next/bench/links.ts +import { GCProfiler } from "node:v8"; +import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; +import { BACKGROUND_CONTEXT, type Context, withTelemetry } from "../../../src/context/index.ts"; +import { arr, bytes, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Host, type RefState } from "../../../src/next/host/index.ts"; +import { type Codec, type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; +import { binaryCodec, type Message, textCodec } from "../../../src/next/link/internal.ts"; +import type { NativeWatchable } from "../../../src/watch.ts"; + +type P = { kind: "owner" } | { kind: "server" }; + +const Entry = obj({ id: int(), text: str(), author: str(), at: int() }); +const Shape = obj({ entries: arr(Entry) }); + +/** The shared store: one document of entries, and the writes the benchmark makes. */ +const Store = local("bench.store", { + version: 1, + params: { name: str() }, + members: { + doc: Shape, + append: method(Entry), + edit: method(obj({ index: int(), text: str() })), + remove: method(int()), + stream: method(str()), + }, +}); + +/** The remote view a client holds: the store's document as it is. */ +const View = remote("bench.view", { + version: 1, + params: { name: str() }, + variants: { owner: { state: { doc: Shape } } }, +}); + +/** A file service: a call with a small argument, and one with bytes. */ +const Files = remote("bench.files", { + version: 1, + params: {}, + variants: { + owner: { methods: { ping: method(obj({ n: int() }), int()), put: method(obj({ data: bytes() }), int()) } }, + }, +}); + +const Doc = defineState("doc", { schema: Shape, version: 1, initial: () => ({ entries: [] }) }); + +/** The `n` of every `ping` in the order the provider ran them. */ +const pings: number[] = []; + +const facet = defineFacet

((host) => { + host.serve(Store, (_, self) => { + const doc = self.state(Doc); + const change = ( + fn: (draft: { entries: { id: number; text: string; author: string; at: number }[] }) => void, + ctx: Context, + ) => doc.change((draft) => void fn(draft), ctx).then(() => null); + return { + doc, + append: (entry, ctx) => change((d) => void d.entries.push(entry), ctx), + edit: ({ index, text }, ctx) => + change((d) => { + d.entries[index]!.text = text; + }, ctx), + remove: (index, ctx) => change((d) => void d.entries.splice(index, 1), ctx), + stream: (token, ctx) => + change((d) => { + d.entries[d.entries.length - 1]!.text += token; + }, ctx), + }; + }); + host.serve(View, ({ name }, _principal, self) => ({ + variant: "owner", + members: { doc: self.get(Store, { name }).doc }, + })); + host.serve(Files, () => ({ + variant: "owner", + members: { + ping: async ({ n }) => { + pings.push(n); + return n + 1; + }, + put: async ({ data }) => data.byteLength, + }, + })); +}); + +/** A link end counting what it sends and keeping the last message. */ +function counted(link: Link): Link & { bytes: number; messages: number; last: Uint8Array | undefined } { + const counting = { + bytes: 0, + messages: 0, + last: undefined as Uint8Array | undefined, + send(message: Uint8Array) { + counting.bytes += message.byteLength; + counting.messages++; + counting.last = message; + link.send(message); + }, + onMessage: (listener: (message: Uint8Array) => void) => link.onMessage(listener), + close: () => link.close(), + onClose: (listener: () => void) => link.onClose(listener), + }; + return counting; +} + +const telemetry = new InMemoryTelemetryContext(); +const span = await telemetry.startSpan({ name: "bench" }, (s) => s); +const counting = withTelemetry(span, BACKGROUND_CONTEXT); + +/** The encodes counted so far (the in-memory adapter records each add). */ +const encodes = (): number => + telemetry + .getMetrics() + .filter(({ name }) => name === "chord.link.encodes") + .reduce((sum, { value }) => sum + value, 0); + +type Entry = { id: number; text: string; author: string; at: number }; + +function entry(id: number, length: number): Entry { + return { id, text: "x".repeat(length), author: "mario", at: 1_700_000_000_000 + id }; +} + +const notes = (): Entry[] => Array.from({ length: 1_000 }, (_, id) => entry(id, 80)); + +const document = (): Entry[] => Array.from({ length: 500 }, (_, id) => entry(id, 960)); + +/** + * A server with `entries` in store `name`, a client connected over a counted link in `codec`. `metered: false` leaves + * telemetry out (the in-memory adapter keeps every record, which a memory measurement would count). + */ +async function setup(name: string, entries: Entry[], codec: Codec, metered = true) { + const context = metered ? counting : BACKGROUND_CONTEXT; + const server = createHost

({ + principal: { kind: "server" }, + telemetry: metered ? telemetry : undefined, + onError: console.error, + }); + await server.load([{ name: "@bench/app", facet }], context); + const store = await server.get(Store, { name }, context); + for (const item of entries) await store.append(item, context); + const [a, b] = memoryLinkPair(); + const down = counted(a); + const up = counted(b); + server.accept(down, { principal: { kind: "owner" }, codec }, context); + const client = createHost

({ onError: console.error }); + client.connect(up, { principal: { kind: "server" }, codec }, context); + await client.load([], context); + const close = async () => { + await client.close(context); + await server.close(context); + }; + return { server, client, store, down, up, close, context }; +} + +const gc = (globalThis as { gc?: () => void }).gc!; + +/** Heap and `ArrayBuffer` bytes: the message bytes themselves live outside the heap. */ +type Memory = { readonly heap: number; readonly arrayBuffers: number }; + +/** Waits until queued deliveries ran and collects garbage, repeatedly, then returns the memory in use. */ +async function settledMemory(): Promise { + for (let i = 0; i < 5; i++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + const { heapUsed, arrayBuffers } = process.memoryUsage(); + return { heap: heapUsed, arrayBuffers }; +} + +/** Heap bytes `work` allocated: the growth plus what the collections it caused freed. */ +function allocated(work: () => void): number { + gc(); + const before = process.memoryUsage().heapUsed; + const profiler = new GCProfiler(); + profiler.start(); + work(); + const after = process.memoryUsage().heapUsed; + let freed = 0; + for (const { beforeGC, afterGC } of profiler.stop().statistics) { + freed += beforeGC.heapStatistics.usedHeapSize - afterGC.heapStatistics.usedHeapSize; + } + return after - before + freed; +} + +/** + * `ArrayBuffer` bytes one encoded message keeps alive (its backing store, which may be larger than the message), over + * `runs` messages kept at once. Freed buffers are not reported reliably during a loop, so this measures what is kept. + */ +async function retainedBuffers(codec: Codec, message: Message, runs: number): Promise { + const before = (await settledMemory()).arrayBuffers; + const kept = Array.from({ length: runs }, () => codec.encode(message)); + const after = (await settledMemory()).arrayBuffers; + return kept.length === runs ? (after - before) / runs : 0; +} + +/** + * One message through a codec, `runs` times: wire bytes, encode and decode time per message, and heap allocated per + * message while encoding and decoding. + */ +function codecRun(codec: Codec, message: Message, runs: number) { + const bytes = codec.encode(message); + for (let i = 0; i < 100; i++) codec.decode(codec.encode(message)); + // The fastest of five loops: one loop of a microsecond-sized message is at the mercy of the collector and the JIT. + let encode = Number.POSITIVE_INFINITY; + let decode = Number.POSITIVE_INFINITY; + for (let repeat = 0; repeat < 5; repeat++) { + const t0 = performance.now(); + for (let i = 0; i < runs; i++) codec.encode(message); + const t1 = performance.now(); + for (let i = 0; i < runs; i++) codec.decode(bytes); + const t2 = performance.now(); + encode = Math.min(encode, (t1 - t0) / runs); + decode = Math.min(decode, (t2 - t1) / runs); + } + const heap = allocated(() => { + for (let i = 0; i < runs; i++) codec.decode(codec.encode(message)); + }); + return { bytes: bytes.byteLength, encode, decode, heap: heap / runs }; +} + +const CODECS = [binaryCodec, textCodec]; + +/** Results by row label, one value per codec. */ +const table = new Map(); + +const record = (label: string, codec: Codec, value: string): void => { + const row = table.get(label) ?? []; + row[CODECS.indexOf(codec)] = value; + table.set(label, row); +}; + +const ms = (value: number) => `${value.toFixed(3)} ms`; +const us = (value: number) => `${(value * 1000).toFixed(1)} µs`; +const kb = (value: number) => `${(value / 1024).toFixed(1)} KiB`; +const b = (value: number) => (value >= 10_240 ? kb(value) : `${value.toFixed(0)} B`); + +/** Records a captured message's codec run under `label`. */ +async function codecRows(label: string, codec: Codec, message: Message, runs: number): Promise { + const run = codecRun(codec, message, runs); + record(`${label}: wire bytes`, codec, b(run.bytes)); + record(`${label}: encode / decode`, codec, `${us(run.encode)} / ${us(run.decode)}`); + record(`${label}: heap per encode + decode`, codec, b(run.heap)); + record(`${label}: array buffer kept per encoded message`, codec, b(await retainedBuffers(codec, message, 50))); +} + +/** A snapshot over a link: hold to live, then the captured `bound` through the codec alone. */ +async function snapshot(label: string, codec: Codec, entries: Entry[]): Promise { + const { client, down, close } = await setup(label, entries, codec); + let start = performance.now(); + // The first hold also activates the view; it stays held, so later holds measure the snapshot path alone. + await client.get(View, { name: label }, counting); + record(`${label}: first hold to live (activation, one sample)`, codec, ms(performance.now() - start)); + const runs = 100; + start = performance.now(); + for (let i = 0; i < runs; i++) (await client.get(View, { name: label }, counting)).release(); + record(`${label}: hold to live (two hops, mean of ${runs})`, codec, ms((performance.now() - start) / runs)); + await codecRows(`${label}: snapshot`, codec, codec.decode(down.last!), 1_000); + await close(); +} + +/** One commit per run: commit-to-frame, bytes and messages per commit, encodes per hold; then its message alone. */ +async function commits( + label: string, + codec: Codec, + write: (store: Awaited>["store"], i: number) => Promise, +): Promise { + const { client, store, down, close } = await setup(label, notes(), codec); + const view = await client.get(View, { name: label }, counting); + const watch = await view.doc.watch(counting); + let frame: () => void = () => {}; + watch.start(async () => frame()); + const runs = 1_000; + const [messages0, encodes0] = [down.messages, encodes()]; + let total = 0; + for (let i = 0; i < runs; i++) { + const arrived = new Promise((resolve) => { + frame = resolve; + }); + const start = performance.now(); + await write(store, i); + await arrived; + total += performance.now() - start; + } + record(`${label}: commit to frame`, codec, ms(total / runs)); + record( + `${label}: messages per commit, encodes per hold`, + codec, + `${(down.messages - messages0) / runs}, ${(encodes() - encodes0) / runs}`, + ); + await codecRows(`${label}: state message`, codec, codec.decode(down.last!), 10_000); + await close(); +} + +/** A streamed reply: 2,000 small text appends to one entry. */ +async function stream(codec: Codec): Promise { + const { client, store, down, close } = await setup("stream", [entry(0, 0)], codec); + const view = await client.get(View, { name: "stream" }, counting); + const watch = await view.doc.watch(counting); + let frames = 0; + const done = Promise.withResolvers(); + watch.start(async () => { + if (++frames === 2_000) done.resolve(); + }); + const bytes0 = down.bytes; + const start = performance.now(); + for (let i = 0; i < 2_000; i++) await store.stream(" tok", counting); + await done.promise; + record("streamed reply, 2,000 appends: time per append", codec, ms((performance.now() - start) / 2_000)); + record( + "streamed reply: bytes per append, total", + codec, + `${((down.bytes - bytes0) / 2_000).toFixed(0)} B, ${kb(down.bytes - bytes0)}`, + ); + await codecRows("streamed token", codec, codec.decode(down.last!), 10_000); + await close(); +} + +/** One trial: the memory retained per hold and the messages of one commit, with `holds` holds on one link. */ +async function holdTrial( + codec: Codec, + holds: number, +): Promise<{ readonly perHold: Memory; readonly messages: number }> { + const name = `holds ${holds}`; + const { client, store, down, close } = await setup(name, notes(), codec, false); + const before = await settledMemory(); + const views = []; + for (let i = 0; i < holds; i++) views.push(await client.get(View, { name }, BACKGROUND_CONTEXT)); + await store.edit({ index: 1, text: "warm" }, BACKGROUND_CONTEXT); + const after = await settledMemory(); + const messages0 = down.messages; + await store.edit({ index: 2, text: "changed" }, BACKGROUND_CONTEXT); + await new Promise((resolve) => setTimeout(resolve, 0)); + const messages = down.messages - messages0; + await close(); + const perHold = { + heap: (after.heap - before.heap) / holds, + arrayBuffers: (after.arrayBuffers - before.arrayBuffers) / holds, + }; + return { perHold, messages }; +} + +/** Calls: a small argument, and a 1 MiB `bytes()` argument; round trips, then the call and result messages alone. */ +async function calls(codec: Codec): Promise { + const { client, down, up, close } = await setup("calls", [], codec); + const files = await client.get(Files, {}, counting); + let start = performance.now(); + for (let i = 0; i < 1_000; i++) await files.ping({ n: i }, counting); + record("call, small argument: round trip", codec, ms((performance.now() - start) / 1_000)); + await codecRows("call, small argument: call message", codec, codec.decode(up.last!), 10_000); + await codecRows("call, small argument: result message", codec, codec.decode(down.last!), 10_000); + const data = new Uint8Array(1024 * 1024); + start = performance.now(); + for (let i = 0; i < 50; i++) await files.put({ data }, counting); + record("call, 1 MiB bytes argument: round trip", codec, ms((performance.now() - start) / 50)); + await codecRows("call, 1 MiB bytes argument: call message", codec, codec.decode(up.last!), 1_000); + await close(); +} + +/** + * Drops the client's parent link (the server's end closes it) and waits until the client saw the close; then connects + * a new counted link and returns it with the time it was connected. + */ +async function relink(server: Host

, client: Host

, down: Link, codec: Codec) { + down.close(); + await new Promise((resolve) => setTimeout(resolve, 0)); + const [a, b] = memoryLinkPair(); + const next = { down: counted(a), up: counted(b) }; + const start = performance.now(); + server.accept(next.down, { principal: { kind: "owner" }, codec }, BACKGROUND_CONTEXT); + client.connect(next.up, { principal: { kind: "server" }, codec }, BACKGROUND_CONTEXT); + return { ...next, start }; +} + +/** Resolves when `state` is live. */ +async function live(state: NativeWatchable): Promise { + if (state.value.kind === "live") return; + const watch = await state.watch(BACKGROUND_CONTEXT); + const done = Promise.withResolvers(); + watch.start(async (call) => { + if (call.kind === "value" && call.value.kind === "live") done.resolve(); + }); + await done.promise; + await watch.stop(); +} + +/** + * Reconnects with `references` held references of the 1,000-item view (one provider object, a hold each): time from + * `connect` to every reference live again, and the bytes of the re-holds both ways; the mean of `runs` reconnects. + */ +async function reconnects(codec: Codec, references: number, runs: number): Promise { + const name = `reconnect ${references}`; + const { server, client, down, close } = await setup(name, notes(), codec, false); + const views = []; + for (let i = 0; i < references; i++) views.push(await client.get(View, { name }, BACKGROUND_CONTEXT)); + let link = down; + let time = 0; + let bytes = { down: 0, up: 0 }; + for (let run = 0; run < runs; run++) { + const next = await relink(server, client, link, codec); + await Promise.all(views.map((view) => live(view.state))); + time += performance.now() - next.start; + bytes = { down: next.down.bytes, up: next.up.bytes }; + link = next.down; + } + const label = `reconnect, ${references} reference(s) of 1,000 items`; + record(`${label}: time to all live (mean of ${runs})`, codec, ms(time / runs)); + record(`${label}: bytes down / up`, codec, `${b(bytes.down)} / ${b(bytes.up)}`); + await close(); +} + +/** 1,000 calls made while the link is down, flushed on reconnect: time from `connect` to the last result, and order. */ +async function parkedCalls(codec: Codec): Promise { + const { server, client, down, close } = await setup("parked", [], codec, false); + const files = await client.get(Files, {}, BACKGROUND_CONTEXT); + let link = down; + let time = 0; + let ordered = true; + const runs = 10; + for (let run = 0; run < runs; run++) { + pings.length = 0; + const next = await relink(server, client, link, codec); + // The link is connected but not yet live: every call below parks until the re-hold is bound. + const results = await Promise.all(Array.from({ length: 1_000 }, (_, n) => files.ping({ n }, BACKGROUND_CONTEXT))); + time += performance.now() - next.start; + ordered &&= pings.every((n, index) => n === index) && results.every((result, index) => result === index + 1); + link = next.down; + } + record("1,000 calls parked while down, flushed on reconnect: time to the last result", codec, ms(time / runs)); + record("1,000 parked calls: run in the order they were made", codec, ordered ? "yes" : "no"); + await close(); +} + +for (const codec of CODECS) { + // Warm-up: the first run pays for compilation. + await snapshot("warm-up", codec, notes()); + await snapshot("notes list of 1,000 items", codec, notes()); + await snapshot("500 KB document", codec, document()); + await commits("one appended item", codec, (store, i) => store.append(entry(10_000 + i, 80), counting)); + await commits("one changed field", codec, (store, i) => store.edit({ index: 500, text: `edit ${i}` }, counting)); + await commits("one removed item", codec, (store) => store.remove(0, counting)); + await stream(codec); + for (const holds of [1, 10]) { + // The median of seven trials: a single hold is small next to the collector's noise. + const trials: Awaited>[] = []; + for (let i = 0; i < 7; i++) trials.push(await holdTrial(codec, holds)); + const median = (of: (memory: Memory) => number) => + trials.map(({ perHold }) => of(perHold)).sort((a, b) => a - b)[3]!; + const retained = `${kb(median(({ heap }) => heap))} / ${b(median(({ arrayBuffers }) => arrayBuffers))}`; + record(`${holds} hold(s) of 1,000 items: heap / array buffers retained per hold (both ends)`, codec, retained); + record(`${holds} hold(s): messages per commit`, codec, `${trials[0]!.messages}`); + } + await calls(codec); + await reconnects(codec, 1, 20); + await reconnects(codec, 100, 5); + await parkedCalls(codec); +} + +const rows = [...table].filter(([label]) => !label.startsWith("warm-up")); +const width = Math.max(...rows.map(([label]) => label.length)); +console.log(`| ${"Case".padEnd(width)} | binary | text |`); +console.log(`| ${"-".repeat(width)} | --- | --- |`); +for (const [label, [binary, text]] of rows) console.log(`| ${label.padEnd(width)} | ${binary} | ${text} |`); diff --git a/packages/chord/test/next/bench/relay.ts b/packages/chord/test/next/bench/relay.ts new file mode 100644 index 000000000..4f2c382b0 --- /dev/null +++ b/packages/chord/test/next/bench/relay.ts @@ -0,0 +1,619 @@ +// Light benchmarks of links and relays (slice 4a): commit-to-frame and wire bytes through a relay (tab ↔ server ↔ +// worker) over memory links, a real IPC link to a forked worker and a loopback WebSocket to the tab; hold-to-live +// through a relay including the `resolve` of a worker; a tab throttled to 64 KiB/s with stalls of 1, 10 and 60 s +// receiving a streamed reply into a 500 KB view, directly and through a relay (the worker folds either way: its window +// is per origin), in virtual time; 10,000 calls from one tab with another tab's latency through the relay. +// Run by hand from packages/chord (not in CI): +// node --conditions=source --experimental-strip-types --expose-gc --no-flush-bytecode test/next/bench/relay.ts +// (`--no-flush-bytecode`: V8 otherwise drops the bytecode of functions idle during a stall, which reads as freed heap.) +// The same file is the forked worker when its first argument is `worker`. +import { type ChildProcess, fork } from "node:child_process"; +import type { AddressInfo } from "node:net"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; +import { WebSocketServer } from "ws"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import { type IpcPort, ipcLink, type Link, memoryLinkPair, webSocketLink } from "../../../src/next/link/index.ts"; + +type P = { kind: "owner" } | { kind: "server" } | { kind: "worker"; session: string }; + +const Server = hostKind("server"); +const Session = hostKind("session", { id: str() }); + +const Entry = obj({ id: int(), text: str(), author: str(), at: int() }); +const Shape = obj({ entries: arr(Entry) }); + +type Entry = { id: number; text: string; author: string; at: number }; + +/** A worker's own store: one document of entries, and the write the benchmark makes (a streamed token). */ +const Store = local("bench.store", { + version: 1, + params: {}, + members: { doc: Shape, stream: method(str()) }, +}); + +/** What a tab holds: its session's store as it is, placed on the session's worker, and a small call. */ +const View = remote("bench.view", { + version: 1, + params: { session: str() }, + placement: ({ session }) => Session({ id: session }), + variants: { owner: { state: { doc: Shape }, methods: { stream: method(str()), ping: method(int(), int()) } } }, +}); + +/** The start data a worker gets in its start message: its session and the payload its store begins with. */ +type Start = { readonly id: string; readonly entries: "notes" | "document" }; + +function entry(id: number, length: number): Entry { + return { id, text: "x".repeat(length), author: "mario", at: 1_700_000_000_000 + id }; +} + +/** The notes list (1,000 entries of 80 characters, 134 KiB) or a 500 KB conversation document (500 of 960). */ +function entriesOf(kind: Start["entries"]): Entry[] { + return kind === "notes" + ? Array.from({ length: 1_000 }, (_, id) => entry(id, 80)) + : Array.from({ length: 500 }, (_, id) => entry(id, 960)); +} + +/** A worker's facet: its store starts with the start message's entries (the §18 core facet pattern). */ +function workerFacet(start: Start) { + const Doc = defineState("doc", { + schema: Shape, + version: 1, + initial: () => ({ entries: entriesOf(start.entries) }), + }); + return defineFacet

((host) => { + host.serve(Store, (_, self) => { + const doc = self.state(Doc); + return { + doc, + stream: (token, ctx) => + doc + .change((draft) => { + draft.entries[draft.entries.length - 1]!.text += token; + }, ctx) + .then(() => null), + }; + }); + host.serve(View, (_params, _principal, self) => { + const store = self.get(Store, {}); + return { + variant: "owner", + members: { doc: store.doc, stream: (token, ctx) => store.stream(token, ctx), ping: async (n) => n + 1 }, + }; + }); + }); +} + +/** A worker host for `start`, not loaded yet. */ +function workerHost(start: Start): Host

{ + return createHost

({ + address: Session({ id: start.id }), + principal: { kind: "worker", session: start.id }, + onError: console.error, + }); +} + +/** A worker on `link`, its parent link: connect first, then load (W6). */ +function startWorker(start: Start, link: Link): Host

{ + const worker = workerHost(start); + worker.connect(link, { principal: { kind: "server" }, trusted: true }, context); + void worker.load([{ name: "@bench/worker", facet: workerFacet(start) }], context); + return worker; +} + +/** A link end counting the bytes it sends and receives. */ +type Counted = Link & { readonly sent: number; readonly received: number }; + +function counted(link: Link): Counted { + let sent = 0; + let received = 0; + return { + get sent() { + return sent; + }, + get received() { + return received; + }, + send(message) { + sent += message.byteLength; + link.send(message); + }, + onMessage: (listener) => + link.onMessage((message) => { + received += message.byteLength; + listener(message); + }), + close: () => link.close(), + onClose: (listener) => link.onClose(listener), + }; +} + +/** A link end timing how long its host handles each received message. */ +type Timed = Link & { longest: number }; + +function timed(link: Link): Timed { + const timing: Timed = { + longest: 0, + send: (message) => link.send(message), + onMessage: (listener) => + link.onMessage((message) => { + const start = performance.now(); + listener(message); + timing.longest = Math.max(timing.longest, performance.now() - start); + }), + close: () => link.close(), + onClose: (listener) => link.onClose(listener), + }; + return timing; +} + +/** How a server reaches its workers: in this process, or forked over IPC. */ +type WorkerKind = "memory" | "ipc"; + +/** The children forked in this run, ended with it. */ +const children: ChildProcess[] = []; + +/** A server routing `Session` to workers of `kind` whose stores start with `entries`. */ +async function server(kind: WorkerKind, entries: Start["entries"]) { + const host = createHost

({ address: Server(), principal: { kind: "server" }, onError: console.error }); + /** Each worker link's server end, counted; the in-process workers and their ends, timed. */ + const workerLinks: Counted[] = []; + const workers: { readonly host: Host

; readonly link: Timed }[] = []; + host.route( + Session, + { + authorize: () => true, + resolve: async (address) => { + const start = { id: address.params.id, entries }; + const principal = { kind: "worker", session: start.id } as const; + if (kind === "memory") { + const [a, b] = memoryLinkPair(); + const end = timed(b); + workers.push({ host: startWorker(start, end), link: end }); + const link = counted(a); + workerLinks.push(link); + return { link, principal, trusted: true }; + } + const child = fork(fileURLToPath(import.meta.url), ["worker"], { + serialization: "advanced", + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + }); + children.push(child); + const link = counted(ipcLink(child)); + // The app's own start message; the link skips it. + child.send(start); + workerLinks.push(link); + return { link, principal, trusted: true }; + }, + }, + context, + ); + await host.load([], context); + return { host, workerLinks, workers }; +} + +/** A tab: a client host connected to `link`. */ +async function tab(link: Link): Promise> { + const host = createHost

({ onError: console.error }); + host.connect(link, { principal: { kind: "server" } }, context); + await host.load([], context); + return host; +} + +/** A tab on `on` over a memory link, the server's end counted. */ +async function memoryTab(on: Host

): Promise<{ readonly tab: Host

; readonly link: Counted }> { + const [a, b] = memoryLinkPair(); + const link = counted(a); + on.accept(link, { principal: { kind: "owner" } }, context); + return { tab: await tab(b), link }; +} + +/** A tab on `on` over a loopback WebSocket (`ws` serving, Node's own `WebSocket` in the tab), the server's end counted. */ +async function webSocketTab(on: Host

) { + const wss = new WebSocketServer({ host: "127.0.0.1", port: 0 }); + await new Promise((resolve) => wss.once("listening", resolve)); + const accepted = Promise.withResolvers(); + wss.on("connection", (socket) => { + const link = counted(webSocketLink(socket)); + on.accept(link, { principal: { kind: "owner" } }, context); + accepted.resolve(link); + }); + const { port } = wss.address() as AddressInfo; + const host = await tab(webSocketLink(new WebSocket(`ws://127.0.0.1:${port}`))); + const close = () => new Promise((resolve) => wss.close(resolve)); + return { tab: host, link: await accepted.promise, close }; +} + +/** A worker with `entries` served directly to a tab over one memory link (no relay), the worker's end counted. */ +async function direct(entries: Start["entries"], wrap: (link: Link) => Link = (link) => link) { + const worker = workerHost({ id: "D", entries }); + await worker.load([{ name: "@bench/worker", facet: workerFacet({ id: "D", entries }) }], context); + const [a, b] = memoryLinkPair(); + const link = counted(a); + const end = timed(link); + worker.accept(end, { principal: { kind: "owner" } }, context); + return { worker, link, end, tab: await tab(wrap(b)) }; +} + +const ms = (value: number) => `${value.toFixed(3)} ms`; +const kb = (value: number) => `${(value / 1024).toFixed(1)} KiB`; +const b = (value: number) => (value >= 10_240 ? kb(value) : `${value.toFixed(0)} B`); +const s = (value: number) => `${value.toFixed(2)} s`; +const gc = (globalThis as { gc?: () => void }).gc!; + +/** Prints a markdown table. */ +function print(title: string, head: readonly string[], rows: readonly (readonly string[])[]): void { + console.log(`\n${title}\n`); + console.log(`| ${head.join(" | ")} |`); + console.log(`| ${head.map(() => "---").join(" | ")} |`); + for (const row of rows) console.log(`| ${row.join(" | ")} |`); +} + +/** + * Commits as a tab sees them: the tab calls `stream` on its view 1,000 times, each one commit on the worker; the time + * from the call to the tab's watch frame (the call's hops down, the frame's up), and the bytes per commit on the tab + * link (sent to the tab) and the worker link (received from the worker). + */ +async function commits(holder: Host

, session: string, tabLink: Counted, workerLinks: readonly Counted[]) { + const view = await holder.get(View, { session }, context); + // Made by the route's `resolve` for the first hold (none without a relay). + const workerLink = workerLinks[0]; + const watch = await view.doc.watch(context); + let frame: () => void = () => {}; + watch.start(async () => frame()); + // Warm-up. + for (let i = 0; i < 100; i++) await view.stream(" w", context); + const runs = 1_000; + const tab0 = tabLink.sent; + const worker0 = workerLink?.received ?? 0; + let total = 0; + for (let i = 0; i < runs; i++) { + const arrived = new Promise((resolve) => { + frame = resolve; + }); + const start = performance.now(); + await Promise.all([view.stream(" tok", context), arrived]); + total += performance.now() - start; + } + await watch.stop(); + view.release(); + const workerBytes = workerLink === undefined ? "-" : b((workerLink.received - worker0) / runs); + return [ms(total / runs), `${b((tabLink.sent - tab0) / runs)} / ${workerBytes}`]; +} + +/** Commit-to-frame through a relay over each transport, and directly (one memory link, no relay) as the baseline. */ +async function transports(): Promise { + const rows: string[][] = []; + { + const { host, workerLinks } = await server("memory", "notes"); + const { tab: t, link } = await memoryTab(host); + rows.push(["memory tab, in-process worker (memory)", ...(await commits(t, "C", link, workerLinks))]); + await t.close(context); + await host.close(context); + } + { + const { host, workerLinks } = await server("ipc", "notes"); + const { tab: t, link } = await memoryTab(host); + const start = performance.now(); + (await t.get(View, { session: "C" }, context)).release(); + const first = performance.now() - start; + rows.push(["memory tab, forked worker (IPC)", ...(await commits(t, "C", link, workerLinks))]); + rows.push([" first hold to live, the fork included (one sample)", ms(first), "-"]); + await t.close(context); + await host.close(context); + } + { + const { host, workerLinks } = await server("ipc", "notes"); + const { tab: t, link, close } = await webSocketTab(host); + rows.push(["loopback WebSocket tab, forked worker (IPC)", ...(await commits(t, "C", link, workerLinks))]); + await t.close(context); + await host.close(context); + await close(); + } + { + const { worker, link, tab: t } = await direct("notes"); + rows.push(["no relay: memory tab on the worker (baseline)", ...(await commits(t, "D", link, []))]); + await t.close(context); + await worker.close(context); + } + print( + "Commits through a relay (1,000 streamed tokens into the 1,000-item notes list view)", + ["Topology", "call to frame", "bytes per commit: to the tab / from the worker"], + rows, + ); +} + +/** + * Hold to live through a relay: a first hold of each of 20 sessions (the route's `resolve` starts an in-process worker, + * which loads and activates the view: one hop more than a later hold), then 100 holds of a resolved session, each + * released; and the same 100 holds directly on a worker. + */ +async function holds(): Promise { + const rows: string[][] = []; + const { host } = await server("memory", "notes"); + const { tab: t } = await memoryTab(host); + let start = performance.now(); + for (let i = 0; i < 20; i++) await t.get(View, { session: `H${i}` }, context); + rows.push([ + "first hold, `resolve` of an in-process worker included (mean of 20)", + ms((performance.now() - start) / 20), + ]); + start = performance.now(); + for (let i = 0; i < 100; i++) (await t.get(View, { session: "H0" }, context)).release(); + rows.push([ + "hold of a resolved session through the relay (four hops, mean of 100)", + ms((performance.now() - start) / 100), + ]); + await t.close(context); + await host.close(context); + const { worker, tab: d } = await direct("notes"); + await d.get(View, { session: "D" }, context); + start = performance.now(); + for (let i = 0; i < 100; i++) (await d.get(View, { session: "D" }, context)).release(); + rows.push(["no relay: hold on the worker (two hops, mean of 100, baseline)", ms((performance.now() - start) / 100)]); + await d.close(context); + await worker.close(context); + print("Hold to live, the 1,000-item notes list view (133.7 KiB snapshot)", ["Case", "time"], rows); +} + +/** + * A tab's end of a link that delivers what arrives at `rate` bytes per virtual second, whole messages in order, and + * nothing while stalled; virtual time advances by `step`. Idle time banks no budget. + */ +class Throttled implements Link { + stalled = false; + delivered = 0; + readonly #link: Link; + readonly #rate: number; + readonly #queue: Uint8Array[] = []; + #budget = 0; + #listener: (message: Uint8Array) => void = () => {}; + + constructor(link: Link, rate: number) { + this.#link = link; + this.#rate = rate; + link.onMessage((message) => this.#queue.push(message)); + } + + send(message: Uint8Array): void { + this.#link.send(message); + } + + onMessage(listener: (message: Uint8Array) => void): void { + this.#listener = listener; + } + + close(): void { + this.#link.close(); + } + + onClose(listener: () => void): void { + this.#link.onClose(listener); + } + + /** `milliseconds` of virtual time pass. */ + step(milliseconds: number): void { + if (this.stalled) return; + const budget = (this.#rate * milliseconds) / 1000; + this.#budget += budget; + while (this.#queue.length > 0 && this.#queue[0]!.byteLength <= this.#budget) { + const message = this.#queue.shift()!; + this.#budget -= message.byteLength; + this.delivered += message.byteLength; + this.#listener(message); + } + if (this.#queue.length === 0) this.#budget = Math.min(this.#budget, budget); + } +} + +/** The heap in use once queued work ran and garbage was collected, repeatedly. */ +async function settledHeap(): Promise { + for (let i = 0; i < 5; i++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + return process.memoryUsage().heapUsed; +} + +/** Lets every host handle what the last step delivered (memory links deliver in microtasks). */ +const settle = () => new Promise((resolve) => setImmediate(resolve)); + +const STEP = 10; + +/** Tokens per virtual second: a fast model's streamed reply. */ +const TOKENS = 100; + +/** + * A pause of the throttled tab: a stall (it receives nothing and so acknowledges nothing), or the same time without one + * (the control: what the streaming alone costs). + */ +type Pause = { readonly kind: "stall" | "control"; readonly seconds: number }; + +/** + * A tab throttled to 64 KiB/s holding the 500 KB document view of a worker that streams a reply at 100 tokens/s, in + * virtual time (10 ms steps): 2 s of streaming, the pause, 10 s more of streaming, then until the tab caught up. + * `relayed`: through a server (it forwards at once, the worker folds the tab origin's ops); `direct`: on the worker (it + * folds the ops in its pending queues). + */ +async function throttled(topology: "direct" | "relayed", pause: Pause): Promise { + const relayed = topology === "relayed"; + let throttle: Throttled | undefined; + const throttling = (link: Link) => { + throttle = new Throttled(link, 64 * 1024); + return throttle; + }; + let t: Host

; + let producer: () => { readonly host: Host

; readonly end: Timed }; + let close: () => Promise; + if (relayed) { + const { host, workers } = await server("memory", "document"); + const [a, b] = memoryLinkPair(); + host.accept(a, { principal: { kind: "owner" } }, context); + t = await tab(throttling(b)); + producer = () => ({ host: workers[0]!.host, end: workers[0]!.link }); + close = async () => { + await t.close(context); + await host.close(context); + }; + } else { + const made = await direct("document", throttling); + t = made.tab; + producer = () => ({ host: made.worker, end: made.end }); + close = async () => { + await t.close(context); + await made.worker.close(context); + }; + } + // The first snapshot crosses the throttle in virtual time too. + const getting = t.get(View, { session: relayed ? "T" : "D" }, context); + let bound = false; + void getting.then(() => { + bound = true; + }); + while (!bound) { + throttle!.step(STEP); + await settle(); + } + const view = await getting; + const { host: worker, end: providerEnd } = producer(); + const store = await worker.get(Store, {}, context); + const last = (value: { readonly entries: readonly Entry[] }) => value.entries.at(-1)!.text.length; + const caughtUp = () => last(view.doc.value) === last(store.doc.value); + let clock = 0; + const commitTimes = { outside: 0, outsideCount: 0, during: 0, duringCount: 0 }; + /** One virtual step: a token when streaming (its commit timed), delivery, the hosts' turn. */ + const step = async (streaming: boolean, phase: "outside" | "during") => { + clock += STEP; + if (streaming && clock % (1000 / TOKENS) === 0) { + const start = performance.now(); + await store.stream(" tok", context); + commitTimes[phase] += performance.now() - start; + commitTimes[phase === "outside" ? "outsideCount" : "duringCount"]++; + } + throttle!.step(STEP); + await settle(); + }; + for (let i = 0; i < 2000 / STEP; i++) await step(true, "outside"); + while (!caughtUp()) await step(false, "outside"); + const heap0 = await settledHeap(); + throttle!.stalled = pause.kind === "stall"; + for (let i = 0; i < (pause.seconds * 1000) / STEP; i++) await step(true, "during"); + const heap = (await settledHeap()) - heap0; + throttle!.stalled = false; + const stallEnd = clock; + const delivered0 = throttle!.delivered; + providerEnd.longest = 0; + let recovered: number | undefined; + for (let i = 0; recovered === undefined && i < 120_000 / STEP; i++) { + await step(clock - stallEnd < 10_000, "outside"); + if (caughtUp()) recovered = clock - stallEnd; + } + const row = [ + pause.kind === "stall" + ? `${topology}, ${pause.seconds} s stall` + : `${topology}, ${pause.seconds} s without a stall (control)`, + recovered === undefined ? "> 120 s" : s(recovered / 1000), + b(throttle!.delivered - delivered0), + kb(heap), + `${ms(commitTimes.outside / commitTimes.outsideCount)} / ${ms(commitTimes.during / commitTimes.duringCount)}`, + ms(providerEnd.longest), + ]; + store.release(); + view.release(); + await close(); + return row; +} + +async function slowTabs(): Promise { + const rows: string[][] = []; + for (const topology of ["direct", "relayed"] as const) { + for (const seconds of [1, 10, 60]) rows.push(await throttled(topology, { kind: "stall", seconds })); + rows.push(await throttled(topology, { kind: "control", seconds: 60 })); + } + print( + "A tab throttled to 64 KiB/s, a reply streamed at 100 tokens/s into the 500 KB view (virtual time)", + [ + "Case", + "caught up after the stall", + "bytes to the tab until then", + "heap growth during the stall", + "commit at the worker, outside / during the pause", + "longest message handling at the provider after the stall", + ], + rows, + ); +} + +/** 10,000 calls from tab A through the relay at once, and tab B's sequential calls meanwhile; B alone as the baseline. */ +async function calls(): Promise { + const { host } = await server("memory", "notes"); + const a = await memoryTab(host); + const bTab = await memoryTab(host); + const viewA = await a.tab.get(View, { session: "K" }, context); + const viewB = await bTab.tab.get(View, { session: "K" }, context); + const latencies = async (until: () => boolean, max: number) => { + const times: number[] = []; + while (!until() && times.length < max) { + const start = performance.now(); + await viewB.ping(times.length, context); + times.push(performance.now() - start); + } + return times.sort((x, y) => x - y); + }; + const percentile = (times: readonly number[], p: number) => + times[Math.min(times.length - 1, Math.floor(times.length * p))]!; + for (let i = 0; i < 1_000; i++) await viewB.ping(i, context); + const alone = await latencies(() => false, 1_000); + let done = false; + const start = performance.now(); + const burst = Promise.all(Array.from({ length: 10_000 }, (_, i) => viewA.ping(i, context))).then(() => { + done = true; + }); + const during = await latencies(() => done, 100_000); + await burst; + const total = performance.now() - start; + print( + "Calls through the relay (in-process worker, memory links, N = 256 per link)", + ["Case", "p50", "p95", "max", "calls"], + [ + ["tab B alone", ms(percentile(alone, 0.5)), ms(percentile(alone, 0.95)), ms(alone.at(-1)!), `${alone.length}`], + [ + "tab B while tab A has 10,000 calls in flight or queued", + ms(percentile(during, 0.5)), + ms(percentile(during, 0.95)), + ms(during.at(-1)!), + `${during.length}`, + ], + ["tab A: 10,000 calls at once, until the last result", ms(total), "-", "-", "10000"], + ], + ); + await a.tab.close(context); + await bTab.tab.close(context); + await host.close(context); +} + +// The forked worker: its link first (it queues what arrives before the start message), then the start message. +if (process.argv[2] === "worker") { + // A forked child's `process` is an IPC port. + const link = ipcLink(process as unknown as IpcPort); + const start = await new Promise((resolve) => { + const listener = (message: unknown) => { + if (message instanceof Uint8Array) return; + process.off("message", listener); + // The app's own message, sent by `resolve` above. + resolve(message as Start); + }; + process.on("message", listener); + }); + startWorker(start, link); + link.onClose(() => process.exit(0)); +} else { + await transports(); + await holds(); + await slowTabs(); + await calls(); + for (const child of children) child.kill(); +} diff --git a/packages/chord/test/next/bench/store.ts b/packages/chord/test/next/bench/store.ts new file mode 100644 index 000000000..ce2923de9 --- /dev/null +++ b/packages/chord/test/next/bench/store.ts @@ -0,0 +1,450 @@ +// Benchmarks of the store (slice 6a §14, decision "The commit-step benchmark"): the commit step against Durable's +// `SqliteStorage` on the same SQLite file settings, (a) one document delta per commit, (b) one table put per commit, +// (c) 1,000 whole transactions back to back (a host's `change` against a Durable Session's `commit`), with and without +// a watched document, and (d) transactions of one in-memory document only (no store); as bursts (median and p95 per +// commit over several runs) and the heap after GC; the outbox's extra cost apart. Then (e) the other costs: queue wait +// against step time, query pages, a 10k-row `updateWhere`'s step, long text per row, and migration chunks (the first +// and the late ones). +// Run by hand from packages/chord (not in CI): +// node --conditions=source --experimental-strip-types --expose-gc test/next/bench/store.ts +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { performance } from "node:perf_hooks"; +import { defineDoc } from "../../../../durable/src/documents.ts"; +import { createSession } from "../../../../durable/src/session/session.ts"; +import { SqliteStorage } from "../../../../durable/src/storage/sqlite/storage.ts"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { sqliteBackend } from "../../../src/next/store/backend.ts"; +import { changeDocument, createDocument } from "../../../src/next/store/engine.ts"; +import { lt } from "../../../src/next/store/expression.ts"; +import { query } from "../../../src/next/store/query.ts"; +import { TableSql, tableSqlName } from "../../../src/next/store/sql.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import { type ChordStore, openStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import { evolveTable } from "../../../src/next/store/table-evolve.ts"; +import { TableHandle } from "../../../src/next/store/table-handle.ts"; +import { prepareTables } from "../../../src/next/store/table-migration.ts"; +import { stagePut, stageTable } from "../../../src/next/store/table-writes.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const COMMITS = 1_000; +const RUNS = 7; + +/** One commit as a benchmark makes it, given its index. */ +type Commit = (index: number) => Promise; + +/** Per-commit times (ms) of `RUNS` bursts of `COMMITS` commits each, after one warm-up burst. */ +async function bursts(setup: () => Promise<{ readonly commit: Commit; readonly close: () => Promise }>) { + const times: number[] = []; + const perRun: number[] = []; + for (let run = -1; run < RUNS; run++) { + const { commit, close } = await setup(); + const started = performance.now(); + for (let index = 0; index < COMMITS; index++) { + const before = performance.now(); + await commit(index); + if (run >= 0) times.push(performance.now() - before); + } + if (run >= 0) perRun.push((performance.now() - started) / COMMITS); + await close(); + } + times.sort((a, b) => a - b); + perRun.sort((a, b) => a - b); + return { + median: times[Math.floor(times.length / 2)]!, + p95: times[Math.floor(times.length * 0.95)]!, + burst: perRun[Math.floor(perRun.length / 2)]!, + }; +} + +/** A fresh SQLite file in its own directory. */ +async function file(): Promise<{ readonly path: string; readonly remove: () => Promise }> { + const directory = await mkdtemp(join(tmpdir(), "chord-store-bench-")); + return { path: join(directory, "bench.sqlite"), remove: () => rm(directory, { recursive: true, force: true }) }; +} + +const VALUE = { title: "a note", body: "x".repeat(200), n: 0 }; + +/** Chord's store: one document created, then one delta per commit. */ +function chordDeltas(outbox: boolean) { + return async () => { + const { path, remove } = await file(); + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context, { + outbox, + })) as ChordStore; + const address = { namespace: "chord", owner: "session:bench.store", kind: "note", key: "" }; + const base = { version: 1, fingerprint: "0123456789abcdef", value: VALUE }; + let id = 0; + const fence = { + admit: () => true, + adopt: (_seq: number, ids: readonly number[]) => { + id = ids[0] ?? id; + }, + }; + await store.commit([createDocument(address, "latest", base, JSON.stringify(VALUE))], fence, context); + return { + commit: (index: number) => { + const ops = [["s", ["n"], index + 1]] as const; + return store.commit( + [changeDocument(id, index + 1, { kind: "delta", version: 1, ops }, JSON.stringify(ops))], + fence, + context, + ); + }, + close: async () => { + await store.close(context); + await remove(); + }, + }; + }; +} + +/** Durable's `SqliteStorage`: one session document created, then one delta per commit. */ +async function durableDeltas() { + const { path, remove } = await file(); + const storage = await SqliteStorage.open(await openNodeSqliteDatabase(path)); + const id = await storage.mintId(); + const record = { id, kind: "note", scope: { kind: "session" } } as never; + await storage.commit( + [{ type: "document.create", record, content: { kind: "base", version: 1, value: VALUE } }], + () => {}, + context, + ); + return { + commit: (index: number) => + storage.commit( + [ + { type: "document.change", id, content: { kind: "delta", version: 1, ops: [["s", ["n"], index + 1]] } }, + ] as never, + () => {}, + context, + ), + close: async () => { + await storage.close(context); + await remove(); + }, + }; +} + +const EDIT_COLUMNS = { id: str(), path: str(), at: int(), body: str() }; +const Edits = defineTable("edits", { version: 1, columns: EDIT_COLUMNS, key: ["id"] }); +/** `Edits` with an index, for pages in a non-key order. */ +const IndexedEdits = defineTable("edits", { + version: 1, + columns: EDIT_COLUMNS, + key: ["id"], + indexes: { by_at: ["at"] }, +}); + +/** Chord's store: one table created, then one `put` per commit (the row checked and encoded in the commit). */ +function chordPuts(outbox: boolean) { + return async () => { + const { path, remove } = await file(); + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context, { + outbox, + })) as ChordStore; + const table = tableSqlName("session", "bench.store", Edits.name); + const sql = new TableSql(table, Edits.columns, Edits.rows.physical, Edits.key); + const fence = { admit: () => true, adopt: () => {} }; + const { version, record: columns, key, fingerprint } = Edits; + await store.commit( + [stageTable({ kind: "table", table, version, state: "ready", columns, key, fingerprint })], + fence, + context, + ); + return { + commit: (index: number) => { + const row = Edits.rows.put( + { id: `e${index}`, path: `src/file${index % 50}.ts`, at: index, body: VALUE.body }, + version, + ); + return store.commit([stagePut(sql.put, table, row, version)], fence, context); + }, + close: async () => { + await store.close(context); + await remove(); + }, + }; + }; +} + +/** Durable's `SqliteStorage`: one conversation created, then one entry appended per commit. */ +async function durableEntries() { + const { path, remove } = await file(); + const storage = await SqliteStorage.open(await openNodeSqliteDatabase(path)); + await storage.commit([{ type: "conversation", value: { id: 1 } }] as never, () => {}, context); + return { + commit: async (index: number) => { + const id = await storage.mintId(); + const data = { path: `src/file${index % 50}.ts`, at: index, body: VALUE.body }; + return storage.commit( + [{ type: "entry", value: { id, conversationId: 1, kind: "edit", data } }] as never, + () => {}, + context, + ); + }, + close: async () => { + await storage.close(context); + await remove(); + }, + }; +} + +/** The heap after GC, in KiB. */ +const NoteSchema = obj({ title: str(), body: str(), n: int() }); + +const InMemoryNote = defineState("bench.note", { schema: NoteSchema, version: 1, initial: () => ({ ...VALUE }) }); + +const PersistedNote = defineState("bench.note", { + schema: NoteSchema, + version: 1, + initial: () => ({ ...VALUE }), + persist: true, + checkpointWhen: () => false, +}); + +const Counter = local("bench.counter", { version: 1, params: {}, members: { touch: method() } }); + +/** Whole Chord transactions on a host: one `change` of one document per commit, persisted or in memory, watched or not. */ +function chordTransactions(options: { + readonly persist: boolean; + readonly watched: boolean; + readonly outbox: boolean; +}) { + return async () => { + const { path, remove } = await file(); + const store = options.persist + ? await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context, { outbox: options.outbox }) + : undefined; + let note: SingletonHandle | undefined; + const host = createHost({ store, onError: () => {} }); + const facet = defineFacet((h) => + h.serve(Counter, (_, self) => { + note = self.state(options.persist ? PersistedNote : InMemoryNote); + return { touch: async () => null }; + }), + ); + await host.load([{ name: "@bench/store", facet }], context); + const reference = await host.get(Counter, {}, context); + await note!.change((draft) => { + draft.n = -1; + }, context); + const watch = options.watched ? await note!.watch(context) : undefined; + watch?.start(async () => {}); + return { + commit: (index: number) => + note!.change((draft) => { + draft.n = index + 1; + }, context), + close: async () => { + await watch?.stop(); + reference.release(); + await host.close(context); + await store?.close(context); + await remove(); + }, + }; + }; +} + +const DurableNote = defineDoc({ + kind: "bench.note", + version: 1, + scope: "session", + initial: () => ({ ...VALUE }), + checkpointWhen: () => false, +}); + +/** Whole Durable transactions on a Session over `SqliteStorage`: one document change per commit, watched or not. */ +function durableTransactions(watched: boolean) { + return async () => { + const { path, remove } = await file(); + const storage = await SqliteStorage.open(await openNodeSqliteDatabase(path)); + const session = createSession(storage); + await session.commit(async (tx) => { + (await tx.doc(DurableNote)).n = -1; + }, context); + const watch = watched ? await session.watchDoc(DurableNote, context) : undefined; + watch?.start(async () => {}); + return { + commit: (index: number) => + session.commit(async (tx) => { + (await tx.doc(DurableNote)).n = index + 1; + }, context), + close: async () => { + await watch?.stop(); + await session.close(context); + await remove(); + }, + }; + }; +} + +function heap(): number { + globalThis.gc?.(); + return Math.round(process.memoryUsage().heapUsed / 1024); +} + +const format = ({ median, p95, burst }: { median: number; p95: number; burst: number }) => + `median ${(median * 1000).toFixed(1)} µs, p95 ${(p95 * 1000).toFixed(1)} µs, burst ${(burst * 1000).toFixed(1)} µs/commit`; + +const before = heap(); +const durable = await bursts(durableDeltas); +const chord = await bursts(chordDeltas(false)); +const outbox = await bursts(chordDeltas(true)); +console.log(`(a, c) one document delta per commit, ${RUNS} bursts of ${COMMITS}:`); +console.log(` Durable SqliteStorage: ${format(durable)}`); +console.log(` Chord store: ${format(chord)}`); +console.log(` Chord store + outbox: ${format(outbox)}`); +const durablePuts = await bursts(durableEntries); +const chordTable = await bursts(chordPuts(false)); +const tableOutbox = await bursts(chordPuts(true)); +console.log(`(b) one table put (Durable: one entry append) per commit, ${RUNS} bursts of ${COMMITS}:`); +console.log(` Durable SqliteStorage: ${format(durablePuts)}`); +console.log(` Chord store: ${format(chordTable)}`); +console.log(` Chord store + outbox: ${format(tableOutbox)}`); +for (const watched of [false, true]) { + const durableTx = await bursts(durableTransactions(watched)); + const chordTx = await bursts(chordTransactions({ persist: true, watched, outbox: false })); + const chordTxOutbox = await bursts(chordTransactions({ persist: true, watched, outbox: true })); + console.log(`(c) whole transactions, ${watched ? "a watched document" : "no watch"}, ${RUNS} bursts of ${COMMITS}:`); + console.log(` Durable Session: ${format(durableTx)}`); + console.log(` Chord host: ${format(chordTx)}`); + console.log(` Chord host + outbox: ${format(chordTxOutbox)}`); +} +const memory = await bursts(chordTransactions({ persist: false, watched: false, outbox: false })); +const memoryWatched = await bursts(chordTransactions({ persist: false, watched: true, outbox: false })); +console.log(`(d) one in-memory document per transaction (no store), ${RUNS} bursts of ${COMMITS}:`); +console.log(` Chord host: ${format(memory)}`); +console.log(` Chord host, watched: ${format(memoryWatched)}`); +console.log(`heap after GC: ${heap() - before} KiB above the start`); + +/** A file store with `count` rows of `Edits` at version 1 (one commit), as a table handle. */ +async function filledEdits(count: number, body = VALUE.body, definition: typeof Edits = Edits) { + const { path, remove } = await file(); + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; + const table = tableSqlName("session", "bench.store", Edits.name); + const fence = { admit: () => true, adopt: () => {} }; + await prepareTables([{ definition, table }], store, fence, "bench", context); + const handle = new TableHandle(definition, table, store, { + label: "bench", + scope: {}, + assertLive: () => {}, + fail: () => {}, + }); + const staged: StagedWrite[] = []; + const changes = new TableChanges(handle, staged); + for (let i = 0; i < count; i++) + changes.put({ id: `e${String(i).padStart(6, "0")}`, path: `src/f${i % 50}.ts`, at: i, body }); + await store.commit(staged, fence, context); + return { + store, + table, + handle, + fence, + close: async () => { + await store.close(context); + await remove(); + }, + }; +} + +/** The median of `values`. */ +function median(values: number[]): number { + values.sort((a, b) => a - b); + return values[Math.floor(values.length / 2)]!; +} + +console.log("(e) other costs:"); +{ + // 100 commits submitted at once: the time each waits in the queue, apart from its own step. + const { store, table, fence, close } = await filledEdits(0); + const sql = new TableSql(table, Edits.columns, Edits.rows.physical, Edits.key); + const waits: number[] = []; + const steps: number[] = []; + await Promise.all( + Array.from({ length: 100 }, async (_, index) => { + const row = Edits.rows.put({ id: `q${index}`, path: "p", at: index, body: VALUE.body }, 1); + const put = stagePut(sql.put, table, row, 1); + const submitted = performance.now(); + let started = 0; + const timed: StagedWrite = { + order: put.order, + apply: (step) => { + started = performance.now(); + return put.apply(step); + }, + }; + await store.commit([timed], fence, context); + waits.push(started - submitted); + steps.push(performance.now() - started); + }), + ); + console.log( + ` queue: 100 commits at once, median wait ${(median(waits) * 1000).toFixed(0)} µs, step ${(median(steps) * 1000).toFixed(1)} µs`, + ); + await close(); +} +{ + const { handle, close } = await filledEdits(10_000, VALUE.body, IndexedEdits); + const q = query(IndexedEdits).orderBy((e) => e.at, "desc"); + let after: string | undefined; + const started = performance.now(); + for (let page = 0; page < 100; page++) { + const result = await handle.page(q, 100, context, after === undefined ? {} : { after }); + after = result.next; + } + console.log( + ` query pages: 100 rows in an index's order, ${((performance.now() - started) * 10).toFixed(0)} µs per page`, + ); + await close(); +} +{ + const { store, handle, fence, close } = await filledEdits(10_000); + const staged: StagedWrite[] = []; + new TableChanges(handle, staged).updateWhere((e: { at: never }) => lt(e.at, 10_000 as never), { path: "moved" }); + const started = performance.now(); + await store.commit(staged, fence, context); + console.log( + ` updateWhere over 10,000 rows: one step of ${(performance.now() - started).toFixed(1)} ms (every owner's commits wait)`, + ); + await close(); +} +for (const length of [10, 10_000]) { + const body = "ü".repeat(length); + const started = performance.now(); + const { close } = await filledEdits(2_000, body); + console.log( + ` text: 2,000 puts with ${length}-character cells, ${(((performance.now() - started) / 2_000) * 1000).toFixed(1)} µs per row`, + ); + await close(); +} +{ + const { store, table, fence, close } = await filledEdits(10_000); + const EditsV2 = evolveTable(Edits, (t) => + t.add("dir", str({ default: "" })).rows((row) => ({ dir: row.path.slice(0, 3) })), + ); + const chunks: number[] = []; + const commit = store.commit.bind(store); + store.commit = async (writes, commitFence, commitContext) => { + const started = performance.now(); + const seq = await commit(writes, commitFence, commitContext); + chunks.push(performance.now() - started); + return seq; + }; + const started = performance.now(); + await prepareTables([{ definition: EditsV2, table }], store, fence, "bench", context); + const total = performance.now() - started; + const rows = chunks.slice(1, -1); + console.log( + ` migration (rows) of 10,000 rows: ${total.toFixed(0)} ms, ${((total / 10_000) * 1000).toFixed(1)} µs per row; ` + + `chunk commits: first ${rows[0]!.toFixed(2)} ms, last ${rows.at(-1)!.toFixed(2)} ms, median ${median(rows).toFixed(2)} ms`, + ); + await close(); +} diff --git a/packages/chord/test/next/contracts/accepts.test.ts b/packages/chord/test/next/contracts/accepts.test.ts new file mode 100644 index 000000000..83cbfcca9 --- /dev/null +++ b/packages/chord/test/next/contracts/accepts.test.ts @@ -0,0 +1,247 @@ +import { describe, expect, it, vi } from "vitest"; +import { + type AcceptsTable, + arr, + evolve, + int, + local, + method, + nul, + obj, + remote, + str, +} from "../../../src/next/contracts/index.ts"; +import { checkAccepts } from "../../../src/next/contracts/internal.ts"; + +const Note = obj({ text: str() }); +const NoteV2 = obj({ text: str(), tag: str() }); + +const NotesV1 = remote("notes", { + version: 1, + params: { session: str() }, + variants: { + editor: { state: { notes: arr(Note), count: int() }, methods: { add: method(Note, str()), clear: method() } }, + reader: { state: { notes: arr(Note) }, methods: { add: method(Note, str()) } }, + }, +}); + +/** Calls `checkAccepts` through an untyped door, as a JavaScript caller would. */ +const checkUntyped = checkAccepts as (contract: unknown, accepts: unknown) => unknown; + +const toV1 = (note: { text: string; tag: string }) => ({ text: note.text }); + +describe("state mappings", () => { + const v2 = evolve(NotesV1, (e) => e.state("editor", "notes", arr(NoteV2))); + + it("a changed member maps the current member to the old one", () => { + const accepts: AcceptsTable = { 1: { state: { editor: { notes: (current) => current.map(toV1) } } } }; + expect(checkAccepts(v2, accepts)).toEqual({ versions: [{ version: 1, fingerprint: NotesV1.fingerprint }] }); + // @ts-expect-error: the changed member needs a mapping + expect(() => checkAccepts(v2, { 1: { state: { editor: {} } } })).toThrow( + "notes accepts[1].state.editor.notes: missing mapping", + ); + // @ts-expect-error: "refuse" is only for removed members + expect(() => checkAccepts(v2, { 1: { state: { editor: { notes: "refuse" } } } })).toThrow( + "notes accepts[1].state.editor.notes: expected a function", + ); + // @ts-expect-error: the mapping returns the old shape, not the current one + checkAccepts(v2, { 1: { state: { editor: { notes: (current) => current.map((note) => note.tag) } } } }); + }); + + it("a removed member maps the variant's whole current state, or refuses", () => { + const v3 = evolve(v2, (e) => e.removeState("editor", "count")); + const accepts: AcceptsTable = { + 1: { state: { editor: { notes: (current) => current.map(toV1), count: (state) => state.notes.length } } }, + 2: { state: { editor: { count: "refuse" } } }, + }; + expect(checkAccepts(v3, accepts).versions.map(({ version }) => version)).toEqual([1, 2]); + expect(() => checkUntyped(v3, { 2: { state: { editor: { count: 1 } } } })).toThrow( + 'notes accepts[2].state.editor.count: expected a function or "refuse"', + ); + }); + + it("an absent section is allowed only when nothing is needed there", () => { + // @ts-expect-error: the state section is needed + expect(() => checkAccepts(v2, { 1: {} })).toThrow("notes accepts[1].state.editor: missing mapping"); + const v3 = evolve(v2, (e) => e.addState("reader", "count", int())); + expect(() => checkAccepts(v3, { 2: {} })).not.toThrow(); + expect(() => checkAccepts(v3, { 2: { state: {}, methods: {} } })).not.toThrow(); + }); + + it("unnecessary mappings are refused", () => { + expect(() => + // @ts-expect-error: reader.notes did not change + checkAccepts(v2, { 1: { state: { editor: { notes: (c) => c.map(toV1) }, reader: { notes: (c) => c } } } }), + ).toThrow("notes accepts[1].state.reader: unnecessary mapping"); + const v3 = evolve(v2, (e) => e.addState("reader", "count", int())); + // @ts-expect-error: added members need nothing + expect(() => checkAccepts(v3, { 2: { state: { reader: { count: () => 0 } } } })).toThrow( + "notes accepts[2].state.reader: unnecessary mapping", + ); + }); + + it("is checked by edit, not by schema: a no-op edit and a reverted change still need mappings", () => { + const noop = evolve(NotesV1, (e) => e.state("reader", "notes", arr(Note))); + // @ts-expect-error: the edit needs an (identity) mapping + expect(() => checkAccepts(noop, { 1: {} })).toThrow("notes accepts[1].state.reader: missing mapping"); + expect(() => checkAccepts(noop, { 1: { state: { reader: { notes: (current) => current } } } })).not.toThrow(); + const reverted = evolve(v2, (e) => e.state("editor", "notes", arr(Note))); + expect(() => checkAccepts(reverted, { 1: { state: { editor: { notes: (current) => current } } } })).not.toThrow(); + const readded = evolve(NotesV1, (e) => e.removeState("editor", "count").addState("editor", "count", int())); + expect(() => checkAccepts(readded, { 1: { state: { editor: { count: (current) => current } } } })).not.toThrow(); + }); +}); + +describe("method mappings", () => { + it("changed args map old to current; a changed result maps current to old; both need both", () => { + const args = evolve(NotesV1, (e) => e.args("add", NoteV2)); + expect(() => + checkAccepts(args, { 1: { methods: { add: { args: (old) => ({ ...old, tag: "" }) } } } }), + ).not.toThrow(); + expect(() => + // @ts-expect-error: only args changed + checkAccepts(args, { 1: { methods: { add: { args: (old) => ({ ...old, tag: "" }), result: (r) => r } } } }), + ).toThrow("notes accepts[1].methods.add.result: unnecessary mapping"); + // @ts-expect-error: args must produce the current shape + checkAccepts(args, { 1: { methods: { add: { args: (old) => old } } } }); + + const result = evolve(NotesV1, (e) => e.result("add", int())); + expect(() => + checkAccepts(result, { 1: { methods: { add: { result: (current) => String(current) } } } }), + ).not.toThrow(); + // @ts-expect-error: the result mapping is missing + expect(() => checkAccepts(result, { 1: { methods: { add: {} } } })).toThrow( + "notes accepts[1].methods.add.result: missing mapping", + ); + + const both = evolve(NotesV1, (e) => e.method("add", NoteV2, int())); + const accepts: AcceptsTable = { + 1: { methods: { add: { args: (old) => ({ ...old, tag: "" }), result: (current) => String(current) } } }, + }; + expect(() => checkAccepts(both, accepts)).not.toThrow(); + // @ts-expect-error: both sides changed + expect(() => checkAccepts(both, { 1: { methods: { add: { args: (old) => ({ ...old, tag: "" }) } } } })).toThrow( + "notes accepts[1].methods.add.result: missing mapping", + ); + }); + + it("a removed method takes a call replacement or refuses", () => { + const v2 = evolve(NotesV1, (e) => e.removeMethod("clear")); + const call = async (args: null) => args; + expect(() => checkAccepts(v2, { 1: { methods: { clear: { call } } } })).not.toThrow(); + expect(() => checkAccepts(v2, { 1: { methods: { clear: "refuse" } } })).not.toThrow(); + // @ts-expect-error: a removed method has no args mapping + expect(() => checkAccepts(v2, { 1: { methods: { clear: { args: (old) => old } } } })).toThrow( + 'notes accepts[1].methods.clear: expected { call } or "refuse"', + ); + // @ts-expect-error: the removed method needs a replacement or a refusal + expect(() => checkAccepts(v2, { 1: {} })).toThrow("notes accepts[1].methods.clear: missing mapping"); + }); + + it("a method removed from one variant that had it counts as removed", () => { + const v2 = evolve(NotesV1, (e) => e.removeMethod("add").addMethod("editor", "add", Note, str())); + // @ts-expect-error: reader lost `add` + expect(() => checkAccepts(v2, { 1: { methods: { add: { args: (old) => old, result: (r) => r } } } })).toThrow( + 'notes accepts[1].methods.add: expected { call } or "refuse"', + ); + expect(() => checkAccepts(v2, { 1: { methods: { add: "refuse" } } })).not.toThrow(); + }); + + it("call and refuse are also allowed for changed methods", () => { + const v2 = evolve(NotesV1, (e) => e.args("add", NoteV2)); + expect(() => checkAccepts(v2, { 1: { methods: { add: { call: async (old) => old.text } } } })).not.toThrow(); + expect(() => checkAccepts(v2, { 1: { methods: { add: "refuse" } } })).not.toThrow(); + expect(() => + // @ts-expect-error: call cannot be combined with args + checkAccepts(v2, { 1: { methods: { add: { call: async () => "", args: (old) => ({ ...old, tag: "" }) } } } }), + ).toThrow('notes accepts[1].methods.add: expected { call } or "refuse"'); + // @ts-expect-error: call returns the old result type + checkAccepts(v2, { 1: { methods: { add: { call: async () => 1 } } } }); + }); + + it("added methods and methods added to a new variant need nothing", () => { + const v2 = evolve(NotesV1, (e) => e.addMethod("reader", "clear", nul(), nul())); + expect(checkAccepts(v2, { 1: {} }).versions).toHaveLength(1); + // @ts-expect-error: unnecessary mapping + expect(() => checkAccepts(v2, { 1: { methods: { clear: "refuse" } } })).toThrow( + "notes accepts[1].methods.clear: unnecessary mapping", + ); + }); + + it("refuses wrong shapes", () => { + const v2 = evolve(NotesV1, (e) => e.args("add", NoteV2)); + expect(() => checkUntyped(v2, { 1: { methods: { add: { args: 1 } } } })).toThrow( + "notes accepts[1].methods.add.args: expected a function", + ); + expect(() => checkUntyped(v2, { 1: { methods: { add: { args: () => 0, extra: 1 } } } })).toThrow( + 'notes accepts[1].methods.add: unknown key "extra"', + ); + expect(() => checkUntyped(v2, { 1: { methods: { add: { args: () => 0 } } }, other: 1 })).toThrow( + "notes accepts[other]: not an older version", + ); + expect(() => checkUntyped(v2, { 1: { methods: { add: { args: () => 0 } }, extra: {} } })).toThrow( + 'notes accepts[1]: unknown key "extra"', + ); + expect(() => checkUntyped(v2, null)).toThrow("notes accepts: expected an object"); + }); +}); + +describe("versions", () => { + it("refuses unknown versions and the current version", () => { + const v2 = evolve(NotesV1, (e) => e); + // @ts-expect-error: version 3 does not exist + expect(() => checkAccepts(v2, { 3: {} })).toThrow("notes accepts[3]: not an older version"); + // @ts-expect-error: version 2 is current + expect(() => checkAccepts(v2, { 2: {} })).toThrow("notes accepts[2]: not an older version"); + expect(checkAccepts(v2, {})).toEqual({ versions: [] }); + }); + + it("maps directly over several versions: every edit after the accepted version counts", () => { + const v2 = evolve(NotesV1, (e) => e.args("add", NoteV2)); + const v3 = evolve(v2, (e) => e.result("add", int()).removeState("editor", "count")); + const accepts: AcceptsTable = { + 1: { + state: { editor: { count: "refuse" } }, + methods: { add: { args: (old) => ({ ...old, tag: "" }), result: (current) => String(current) } }, + }, + 2: { + state: { editor: { count: (state) => state.notes.length } }, + methods: { add: { result: (r) => `${r}` } }, + }, + }; + expect(checkAccepts(v3, accepts).versions).toEqual([ + { version: 1, fingerprint: NotesV1.fingerprint }, + { version: 2, fingerprint: v2.fingerprint }, + ]); + const fromV1 = { + state: { editor: { count: "refuse" } }, + methods: { add: { result: (r: number) => `${r}` } }, + } as const; + // @ts-expect-error: from v1, args changed too + expect(() => checkAccepts(v3, { 1: fromV1 })).toThrow("notes accepts[1].methods.add.args: missing mapping"); + }); + + it("never calls a mapping", () => { + const v2 = evolve(NotesV1, (e) => e.args("add", NoteV2).removeMethod("clear").state("editor", "count", str())); + const mappings = { args: vi.fn(), call: vi.fn(), count: vi.fn() }; + checkUntyped(v2, { + 1: { + state: { editor: { count: mappings.count } }, + methods: { add: { args: mappings.args }, clear: { call: mappings.call } }, + }, + }); + expect(Object.values(mappings).map((mapping) => mapping.mock.calls.length)).toEqual([0, 0, 0]); + }); + + it("local tables are flat: state without a variant level", () => { + const v1 = local("info", { version: 1, params: {}, members: { title: str(), save: method(str()) } }); + const v2 = evolve(v1, (e) => e.state("title", int()).args("save", int())); + const accepts: AcceptsTable = { + 1: { state: { title: (current) => String(current) }, methods: { save: { args: (old) => old.length } } }, + }; + expect(() => checkAccepts(v2, accepts)).not.toThrow(); + expect(() => + checkUntyped(v2, { 1: { state: { members: { title: String } }, methods: { save: "refuse" } } }), + ).toThrow("info accepts[1].state.members: unnecessary mapping"); + }); +}); diff --git a/packages/chord/test/next/contracts/contract.test.ts b/packages/chord/test/next/contracts/contract.test.ts new file mode 100644 index 000000000..c9a6f12ab --- /dev/null +++ b/packages/chord/test/next/contracts/contract.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, expectTypeOf, it } from "vitest"; +import { BACKGROUND_CONTEXT, type Context } from "../../../src/context/index.ts"; +import { type Contract, contractData } from "../../../src/next/contracts/contract.ts"; +import { + arr, + bytes, + evolve, + type Implementation, + int, + local, + method, + nul, + obj, + type ParamsOf, + remote, + str, + union, +} from "../../../src/next/contracts/index.ts"; +import { checkAccepts, currentVariants, LOCAL, takesNoArgs } from "../../../src/next/contracts/internal.ts"; +import { bindProvider, checkParams } from "../../../src/next/data/internal.ts"; +import type { Watchable } from "../../../src/next/state/index.ts"; +import { expectCollected } from "../support.ts"; + +const Notes = remote("notes", { + version: 1, + params: { session: str(), page: int() }, + variants: { + editor: { state: { notes: arr(str()) }, methods: { add: method(obj({ text: str() }), str()), clear: method() } }, + reader: { state: { notes: arr(str()) } }, + }, +}); + +describe("contract handles", () => { + it("carry kind, name, version and fingerprint only", () => { + expect(Object.keys(Notes)).toEqual(["kind", "name", "version", "fingerprint"]); + }); + + it("are branded: spread and hand-built copies are refused", () => { + const copy = { ...Notes }; + const refusal = "notes: not a declared contract (spread and hand-built copies are refused)"; + expect(() => contractData(copy)).toThrow(refusal); + expect(() => evolve(copy, (e) => e)).toThrow(refusal); + expect(() => checkAccepts(copy, {})).toThrow(refusal); + expect(() => bindProvider(copy, {})).toThrow(refusal); + expect(() => checkParams(copy, { session: "s", page: 1 }, BACKGROUND_CONTEXT)).toThrow(refusal); + }); + + it("are collected when dropped, with their data", async () => { + const refs = (() => { + const contract = local("gc", { version: 1, params: {}, members: { s: str() } }); + return [new WeakRef(contract), new WeakRef(contractData(contract))]; + })(); + await expectCollected(...refs); + }); +}); + +/** The members of one variant of a remote implementation. */ +type Members = Extract, { readonly variant: V }>["members"]; + +describe("derived types", () => { + it("Implementation: one variant with its members, state watchables and methods taking args and a context", () => { + expectTypeOf["variant"]>().toEqualTypeOf<"editor" | "reader">(); + type Editor = Members; + expectTypeOf().toEqualTypeOf>(); + expectTypeOf().toEqualTypeOf<(args: { text: string }, context: Context) => Promise>(); + // A method without arguments omits them. + expectTypeOf().toEqualTypeOf<(context: Context) => Promise>(); + expectTypeOf>().toEqualTypeOf<"notes">(); + }); + + it("Implementation of a local contract is flat", () => { + const Info = local("info", { + version: 1, + params: {}, + members: { title: str(), blob: bytes(), save: method(str()) }, + }); + type Info = Implementation; + expectTypeOf().toEqualTypeOf<"title" | "blob" | "save">(); + expectTypeOf().toEqualTypeOf<(args: string, context: Context) => Promise>(); + const Counts = local("counts", { + version: 1, + params: {}, + members: { reset: method(), total: method(undefined, int()), maybe: method(union([nul(), str()])) }, + }); + type Counts = Implementation; + expectTypeOf().toEqualTypeOf<(context: Context) => Promise>(); + expectTypeOf().toEqualTypeOf<(context: Context) => Promise>(); + // Only a `null` argument schema is omitted; a union with null is an argument. + expectTypeOf().toEqualTypeOf<(args: string | null, context: Context) => Promise>(); + }); + + it("takesNoArgs: exactly the methods whose argument schema is null", () => { + const Counts = local("counts", { + version: 1, + params: {}, + members: { + reset: method(), + total: method(undefined, int()), + set: method(int()), + maybe: method(union([nul(), str()])), + }, + }); + const methods = [...currentVariants(Counts).get(LOCAL)!.methods].map(([name, data]) => [name, takesNoArgs(data)]); + expect(methods).toEqual([ + ["reset", true], + ["total", true], + ["set", false], + ["maybe", false], + ]); + }); + + it("ParamsOf: the params object", () => { + expectTypeOf>().toEqualTypeOf<{ readonly session: string; readonly page: number }>(); + }); +}); diff --git a/packages/chord/test/next/contracts/declare.test.ts b/packages/chord/test/next/contracts/declare.test.ts new file mode 100644 index 000000000..016fb2f90 --- /dev/null +++ b/packages/chord/test/next/contracts/declare.test.ts @@ -0,0 +1,228 @@ +import Type from "typebox"; +import { describe, expect, it } from "vitest"; +import { arr, evolve, int, keyed, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { currentVariants, LOCAL } from "../../../src/next/contracts/internal.ts"; + +const Note = obj({ text: str() }); + +/** Declares through an untyped door, as a JavaScript caller would. */ +const untyped = { + remote: remote as (name: unknown, declaration: unknown) => unknown, + local: local as (name: unknown, declaration: unknown) => unknown, +}; + +describe("declaration checks", () => { + it("accept a remote and a local declaration", () => { + const Notes = remote("notes", { + version: 1, + params: { session: str() }, + variants: { + editor: { state: { notes: arr(Note) }, methods: { add: method(Note, str()), clear: method() } }, + reader: { state: { notes: arr(Note) } }, + }, + }); + expect(Notes).toMatchObject({ kind: "remote", name: "notes", version: 1 }); + expect(Notes.fingerprint).toMatch(/^[0-9a-f]{16}$/); + const Info = local("app.session-info", { + version: 3, + params: {}, + members: { init: obj({ title: str() }), clear: method() }, + }); + expect(Info).toMatchObject({ kind: "local", name: "app.session-info", version: 3 }); + }); + + it.each([ + ["empty name", "", { version: 1, params: {}, variants: { a: {} } }, 'A contract name "": expected lowercase'], + [ + "name not a string", + 1, + { version: 1, params: {}, variants: { a: {} } }, + "A contract name 1: expected lowercase", + ], + ["an uppercase name", "Notes", { version: 1, params: {}, variants: { a: {} } }, "expected lowercase"], + ["a name with a colon", "a:b", { version: 1, params: {}, variants: { a: {} } }, "expected lowercase"], + ["version zero", "n", { version: 0, params: {}, variants: { a: {} } }, "n.version: expected a positive integer"], + [ + "fractional version", + "n", + { version: 1.5, params: {}, variants: { a: {} } }, + "n.version: expected a positive integer", + ], + [ + "unsafe version", + "n", + { version: 2 ** 53, params: {}, variants: { a: {} } }, + "n.version: expected a positive integer", + ], + [ + "unknown declaration key", + "n", + { version: 1, params: {}, variants: { a: {} }, place: "x" }, + 'n: unknown key "place"', + ], + [ + "a placement that is no function", + "n", + { version: 1, params: {}, variants: { a: {} }, placement: "x" }, + "n.placement: expected a function of the params", + ], + ["no variants", "n", { version: 1, params: {}, variants: {} }, "n.variants: at least one variant is required"], + ["missing params", "n", { version: 1, variants: { a: {} } }, "n.params: expected an object"], + [ + "unknown variant key", + "n", + { version: 1, params: {}, variants: { a: { members: {} } } }, + 'n.a: unknown key "members"', + ], + [ + "a schema under methods", + "n", + { version: 1, params: {}, variants: { a: { methods: { m: str() } } } }, + "n.a.methods.m: expected method(...)", + ], + [ + "a method under state", + "n", + { version: 1, params: {}, variants: { a: { state: { s: method() } } } }, + "n.a.state.s: a method belongs in methods", + ], + [ + "bad state schema", + "n", + { version: 1, params: {}, variants: { a: { state: { s: { not: {} } } } } }, + 'n.a.state.s: unsupported keyword "not"', + ], + [ + "open arguments", + "n", + { version: 1, params: {}, variants: { a: { methods: { m: method(Type.Object({})) } } } }, + "n.a.methods.m.args: argument objects must be closed", + ], + [ + "bad result", + "n", + { version: 1, params: {}, variants: { a: { methods: { m: method(undefined, { not: {} }) } } } }, + 'n.a.methods.m.result: unsupported keyword "not"', + ], + ])("remote refuses %s", (_case, name, declaration, message) => { + expect(() => untyped.remote(name, declaration)).toThrow(message); + }); + + it.each([ + ["unknown key", { version: 1, params: {}, members: {}, variants: {} }, 'i: unknown key "variants"'], + ["a placement", { version: 1, params: {}, members: {}, placement: () => null }, 'i: unknown key "placement"'], + [ + "bad member schema", + { version: 1, params: {}, members: { s: { not: {} } } }, + 'i.members.s: unsupported keyword "not"', + ], + [ + "open arguments", + { version: 1, params: {}, members: { m: method(Type.Object({})) } }, + "i.members.m.args: argument objects must be closed", + ], + ])("local refuses %s", (_case, declaration, message) => { + expect(() => untyped.local("i", declaration)).toThrow(message); + }); + + it("does not check state for closed objects", () => { + expect(() => local("i", { version: 1, params: {}, members: { s: Type.Object({ a: str() }) } })).not.toThrow(); + }); +}); + +describe("D11: members across variants", () => { + it("refuses a name that is state in one variant and a method in another", () => { + expect(() => + remote("n", { + version: 1, + params: {}, + // @ts-expect-error: `x` is a method in variant b + variants: { a: { state: { x: str() } }, b: { methods: { x: method() } } }, + }), + ).toThrow('n.b.methods.x: "x" is a state member; its kind never changes'); + }); + + it("refuses one name as state and method within a variant", () => { + expect(() => + remote("n", { + version: 1, + params: {}, + // @ts-expect-error: `x` is both state and a method + variants: { a: { state: { x: str() }, methods: { x: method() } } }, + }), + ).toThrow('n.a.methods.x: "x" is a state member'); + }); + + it("refuses a method with different signatures in two variants", () => { + expect(() => + remote("n", { + version: 1, + params: {}, + // @ts-expect-error: `m` takes a string in a and an integer in b + variants: { a: { methods: { m: method(str()) } }, b: { methods: { m: method(int()) } } }, + }), + ).toThrow("n.b.methods.m: a method has the same signature in every variant"); + expect(() => + remote("n", { + version: 1, + params: {}, + variants: { + a: { methods: { m: method(str(), str()) } }, + b: { methods: { m: method(str({ max: 3 }), str()) } }, + }, + }), + ).toThrow("n.b.methods.m: a method has the same signature in every variant"); + }); + + it("allows state schemas to differ per variant (redaction)", () => { + expect(() => + remote("n", { + version: 1, + params: {}, + variants: { + editor: { state: { notes: arr(obj({ text: str(), author: str() })) }, methods: { m: method(str()) } }, + reader: { state: { notes: arr(obj({ text: str() })) }, methods: { m: method(str()) } }, + }, + }), + ).not.toThrow(); + }); +}); + +describe("reserved member names", () => { + it.each(["then", "release", "variant", "state", "retry"])("refuses %s in remote, local and evolve", (name) => { + expect(() => remote("n", { version: 1, params: {}, variants: { v: { state: { [name]: str() } } } })).toThrow( + `n.v.state.${name}: "${name}" is reserved: a reference has it`, + ); + expect(() => local("n", { version: 1, params: {}, members: { [name]: method() } })).toThrow( + `n.members.${name}: "${name}" is reserved: a reference has it`, + ); + const base = local("n", { version: 1, params: {}, members: {} }); + expect(() => evolve(base, (e) => e.addState(name, str()))).toThrow( + `n v2.members.${name}: "${name}" is reserved: a reference has it`, + ); + }); +}); + +describe("keyed members (slice 6a §12)", () => { + it("declare a keyed member in a local contract: its own kind and a node of its own in the fingerprint", () => { + const Keyed = local("app.store", { version: 1, params: {}, members: { notes: keyed(Note), add: method() } }); + const State = local("app.store", { version: 1, params: {}, members: { notes: Note, add: method() } }); + const variant = currentVariants(Keyed).get(LOCAL)!; + expect([[...variant.keyed.keys()], [...variant.state.keys()]]).toEqual([["notes"], []]); + expect(Keyed.fingerprint).not.toBe(State.fingerprint); + // Evolving the contract carries its keyed members over unchanged (their versions are slice 7's). + const next = evolve(Keyed, (e) => e.addMethod("clear", str(), str())); + expect(currentVariants(next).get(LOCAL)!.keyed.has("notes")).toBe(true); + }); + + it("refuse a keyed member in a remote contract, at the type level and at declaration", () => { + expect(() => + remote("app.notes", { + version: 1, + params: {}, + // @ts-expect-error keyed members are only in local contracts + variants: { editor: { state: { notes: keyed(Note) } } }, + }), + ).toThrow("app.notes.editor.state.notes: keyed members are only in local contracts"); + }); +}); diff --git a/packages/chord/test/next/contracts/evolve.test.ts b/packages/chord/test/next/contracts/evolve.test.ts new file mode 100644 index 000000000..abc2263e3 --- /dev/null +++ b/packages/chord/test/next/contracts/evolve.test.ts @@ -0,0 +1,217 @@ +import Type from "typebox"; +import { describe, expect, expectTypeOf, it } from "vitest"; +import type { Context } from "../../../src/context/index.ts"; +import { type Contract, contractData, type ShapeOf } from "../../../src/next/contracts/contract.ts"; +import type { CurrentStep } from "../../../src/next/contracts/history.ts"; +import { + arr, + bool, + evolve, + type Implementation, + int, + local, + method, + nul, + obj, + remote, + type Static, + str, +} from "../../../src/next/contracts/index.ts"; + +const Note = obj({ text: str() }); + +const NotesV1 = remote("notes", { + version: 1, + params: { session: str() }, + variants: { + editor: { state: { notes: arr(Note), count: int() }, methods: { add: method(Note, str()), clear: method() } }, + reader: { state: { notes: arr(Note) }, methods: { add: method(Note, str()) } }, + }, +}); + +const InfoV1 = local("info", { version: 1, params: {}, members: { title: str(), clear: method() } }); + +/** The current members of a contract's variant, as Chord holds them. */ +function current(contract: Contract, variant: string) { + const data = contractData(contract); + const members = data.versions.at(-1)!.variants.get(variant)!; + return { state: [...members.state.keys()], methods: [...members.methods.keys()], data }; +} + +/** The current version of a contract's type-level history. */ +type VersionOf = CurrentStep["history"]>["version"]; + +/** The members of one variant of a remote implementation. */ +type Members = Extract, { readonly variant: V }>["members"]; + +describe("edits", () => { + it("state changes one state member of one variant", () => { + const v2 = evolve(NotesV1, (e) => e.state("editor", "count", str())); + expect(v2.version).toBe(2); + expectTypeOf["count"]["value"]>().toEqualTypeOf(); + expectTypeOf["notes"]["value"]>().toEqualTypeOf<{ text: string }[]>(); + // @ts-expect-error: `missing` is not a state member of editor + expect(() => evolve(NotesV1, (e) => e.state("editor", "missing", str()))).toThrow( + "notes v2.editor.state.missing: no such state member", + ); + }); + + it("addState adds a state member", () => { + const v2 = evolve(NotesV1, (e) => e.addState("reader", "count", int())); + expect(current(v2, "reader").state).toEqual(["notes", "count"]); + expectTypeOf["count"]["value"]>().toEqualTypeOf(); + expect(() => evolve(NotesV1, (e) => e.addState("reader", "notes", int()))).toThrow( + "notes v2.reader.state.notes: already declared", + ); + }); + + it("removeState removes a state member", () => { + const v2 = evolve(NotesV1, (e) => e.removeState("editor", "count")); + expect(current(v2, "editor").state).toEqual(["notes"]); + expectTypeOf>().toEqualTypeOf<"notes" | "add" | "clear">(); + }); + + it("args and result change one side of a method in every variant that has it", () => { + const v2 = evolve(NotesV1, (e) => e.args("add", obj({ text: str(), tag: str() })).result("add", int())); + const { data } = current(v2, "editor"); + const editor = data.versions[1]!.variants.get("editor")!.methods.get("add")!; + const reader = data.versions[1]!.variants.get("reader")!.methods.get("add")!; + expect(editor).toEqual(reader); + expect(data.versions[1]!.changes).toEqual([ + ["editor", "add", "args"], + ["reader", "add", "args"], + ["editor", "add", "result"], + ["reader", "add", "result"], + ]); + type Add = Members["add"]; + expectTypeOf[0]>().toEqualTypeOf<{ text: string; tag: string }>(); + expectTypeOf>>().toEqualTypeOf(); + }); + + it("method replaces both sides", () => { + const v2 = evolve(NotesV1, (e) => e.method("clear", obj({ all: bool() }), bool())); + type Clear = Members["clear"]; + expectTypeOf[0]>().toEqualTypeOf<{ all: boolean }>(); + expectTypeOf>>().toEqualTypeOf(); + // @ts-expect-error: no method `missing` + expect(() => evolve(NotesV1, (e) => e.method("missing", nul(), nul()))).toThrow('notes v2: no method "missing"'); + }); + + it("addMethod adds a method; to another variant only with the same signature", () => { + const v2 = evolve(NotesV1, (e) => e.addMethod("reader", "clear", nul(), nul())); + expect(current(v2, "reader").methods).toEqual(["add", "clear"]); + expect(() => evolve(NotesV1, (e) => e.addMethod("reader", "clear", str(), nul()))).toThrow( + "notes v2.reader.methods.clear: a method has the same signature in every variant", + ); + expect(() => evolve(NotesV1, (e) => e.addMethod("reader", "add", nul(), nul()))).toThrow( + "notes v2.reader.methods.add: already declared", + ); + }); + + it("removeMethod removes a method from every variant", () => { + const v2 = evolve(NotesV1, (e) => e.removeMethod("add")); + expect(current(v2, "editor").methods).toEqual(["clear"]); + expect(current(v2, "reader").methods).toEqual([]); + expectTypeOf>().toEqualTypeOf<"notes">(); + }); + + it("local edits take no variant", () => { + const v2 = evolve(InfoV1, (e) => + e + .state("title", int()) + .addState("body", str()) + .addMethod("save", obj({ force: bool() }), nul()) + .removeMethod("clear"), + ); + expect(current(v2, "members")).toMatchObject({ state: ["title", "body"], methods: ["save"] }); + expectTypeOf>().toEqualTypeOf<"title" | "body" | "save">(); + const v3 = evolve(v2, (e) => e.removeState("body").args("save", nul())); + // Arguments changed to `null`: the method omits them now. + expectTypeOf["save"]>>().toEqualTypeOf<[context: Context]>(); + }); + + it("checks every edit: closed arguments and the vocabulary", () => { + expect(() => evolve(NotesV1, (e) => e.args("add", Type.Object({ text: str() })))).toThrow( + "notes v2.editor.methods.add.args: argument objects must be closed", + ); + expect(() => evolve(InfoV1, (e) => e.addMethod("m", Type.Object({}), nul()))).toThrow( + "info v2.members.m.args: argument objects must be closed", + ); + expect(() => evolve(NotesV1, (e) => e.state("editor", "count", { not: {} } as never))).toThrow( + 'notes v2.editor.state.count: unsupported keyword "not"', + ); + expect(() => evolve(NotesV1, (e) => e.state("editor", "count", Type.Object({})))).not.toThrow(); + }); +}); + +describe("kinds and fixed parts", () => { + it("a member's kind never changes, also not after a removal and re-add", () => { + expect(() => + evolve(NotesV1, (e) => e.removeState("editor", "count").addMethod("editor", "count", nul(), nul())), + ).toThrow('notes v2.editor.methods.count: "count" is a state member; its kind never changes'); + const v2 = evolve(NotesV1, (e) => e.removeMethod("clear")); + const v3 = evolve(v2, (e) => e.state("editor", "count", str())); + expect(() => evolve(v3, (e) => e.addState("reader", "clear", str()))).toThrow( + 'notes v4.reader.state.clear: "clear" is a method member; its kind never changes', + ); + }); + + it("the variant set never changes", () => { + // @ts-expect-error: no variant `admin` + expect(() => evolve(NotesV1, (e) => e.addState("admin", "x", str()))).toThrow( + "notes v2.admin: unknown variant (the variant set never changes)", + ); + }); + + it("a new base is a full declaration with an explicit version and no older versions", () => { + const base = remote("notes", { version: 5, params: { session: str(), tenant: str() }, variants: { editor: {} } }); + expect(contractData(base).versions.map(({ version }) => version)).toEqual([5]); + expect(evolve(base, (e) => e).version).toBe(6); + }); + + it("evolve must return the editor it was given", () => { + const other = evolve(NotesV1, (e) => e); + expect(other.version).toBe(2); + let foreign: unknown; + evolve(InfoV1, (e) => { + foreign = e; + return e; + }); + expect(() => evolve(NotesV1, () => foreign as never)).toThrow( + "notes v2: evolve must return the editor it was given", + ); + }); + + it("leaves the previous contract untouched", () => { + evolve(NotesV1, (e) => e.removeState("editor", "count").removeMethod("add")); + expect(current(NotesV1, "editor")).toMatchObject({ state: ["notes", "count"], methods: ["add", "clear"] }); + }); +}); + +describe("long histories", () => { + it("count versions beyond 9 and 99, at runtime and in the types", () => { + let contract: Contract = InfoV1; + for (let step = 0; step < 120; step++) contract = evolve(contract, (e) => e); + expect(contract.version).toBe(121); + expect(contractData(contract).versions).toHaveLength(121); + + const v9 = local("i", { version: 9, params: {}, members: { s: str() } }); + const v10 = evolve(v9, (e) => e); + const v99 = local("i", { version: 99, params: {}, members: { s: str() } }); + const v100 = evolve(v99, (e) => e); + const v101 = evolve(v100, (e) => e.state("s", int())); + expectTypeOf>().toEqualTypeOf<"10">(); + expectTypeOf>().toEqualTypeOf<"100">(); + expectTypeOf>().toEqualTypeOf<"101">(); + }); + + it("the types track every step with its changes", () => { + const v2 = evolve(NotesV1, (e) => e.state("editor", "count", str()).addMethod("reader", "clear", nul(), nul())); + type History = ShapeOf["history"]; + expectTypeOf().toEqualTypeOf<2>(); + expectTypeOf().toEqualTypeOf< + ["editor", "count", "state"] | ["reader", "clear", "args" | "result"] + >(); + expectTypeOf>().toEqualTypeOf(); + }); +}); diff --git a/packages/chord/test/next/contracts/fingerprint.test.ts b/packages/chord/test/next/contracts/fingerprint.test.ts new file mode 100644 index 000000000..d28e360fb --- /dev/null +++ b/packages/chord/test/next/contracts/fingerprint.test.ts @@ -0,0 +1,80 @@ +import Type, { type TSchema } from "typebox"; +import { describe, expect, it } from "vitest"; +import { evolve, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { NORMAL_FORM_VERSION } from "../../../src/next/contracts/internal.ts"; +import type { Params } from "../../../src/next/contracts/params.ts"; + +/** Fingerprint of a one-variant remote contract with state `s` and params `params`. */ +function fingerprintOf(state: TSchema, params: Params = {}): string { + return remote("n", { version: 1, params, variants: { a: { state: { s: state } } } }).fingerprint; +} + +describe("fingerprints", () => { + it("pin the normal form: a fixed contract has a fixed hash", () => { + expect(NORMAL_FORM_VERSION).toBe(2); + const Notes = remote("notes", { + version: 1, + params: { session: str() }, + variants: { + editor: { state: { notes: obj({ text: str() }) }, methods: { add: method(obj({ text: str() }), str()) } }, + reader: { state: { notes: obj({ text: str() }) } }, + }, + }); + expect(Notes.fingerprint).toBe("b3eda86070a7e319"); + }); + + it("are stable across key order and annotations", () => { + const plain = { + type: "object", + properties: { a: { type: "string" }, b: { type: "integer" } }, + required: ["a", "b"], + }; + const reordered = { + required: ["b", "a"], + title: "T", + properties: { + b: { type: "integer", description: "d" }, + a: { $comment: "c", type: "string", examples: ["x"] }, + }, + type: "object", + deprecated: true, + }; + expect(fingerprintOf(reordered)).toBe(fingerprintOf(plain)); + }); + + it.each([ + ["defaults", Type.String(), Type.String({ default: "x" })], + ["bounds", Type.String(), Type.String({ maxLength: 3 })], + ["formats", Type.String(), Type.String({ format: "email" })], + ["union order", Type.Union([Type.String(), Type.Integer()]), Type.Union([Type.Integer(), Type.String()])], + ["additionalProperties", Type.Object({}), Type.Object({}, { additionalProperties: false })], + ["readOnly", Type.String(), Type.String({ readOnly: true })], + ])("are sensitive to %s", (_case, a, b) => { + expect(fingerprintOf(a)).not.toBe(fingerprintOf(b)); + }); + + it("are sensitive to params order", () => { + expect(fingerprintOf(str(), { a: str(), b: int() })).not.toBe(fingerprintOf(str(), { b: int(), a: str() })); + }); + + it("give the same v1 from a separate declaration, and evolve keeps v1's", () => { + const declare = () => remote("n", { version: 1, params: {}, variants: { a: { state: { s: str() } } } }); + const v1 = declare(); + const v2 = evolve(v1, (e) => e.state("a", "s", int())); + expect(declare().fingerprint).toBe(v1.fingerprint); + expect(v2.fingerprint).not.toBe(v1.fingerprint); + expect(evolve(declare(), (e) => e.state("a", "s", int())).fingerprint).toBe(v2.fingerprint); + }); + + it("differ between local and remote", () => { + const asRemote = remote("n", { version: 1, params: {}, variants: { members: { state: { s: str() } } } }); + const asLocal = local("n", { version: 1, params: {}, members: { s: str() } }); + expect(asLocal.fingerprint).not.toBe(asRemote.fingerprint); + }); + + it("do not depend on the name or version", () => { + const a = remote("a", { version: 1, params: {}, variants: { v: {} } }); + const b = remote("b", { version: 7, params: {}, variants: { v: {} } }); + expect(a.fingerprint).toBe(b.fingerprint); + }); +}); diff --git a/packages/chord/test/next/contracts/history.test.ts b/packages/chord/test/next/contracts/history.test.ts new file mode 100644 index 000000000..9d7ab7ce2 --- /dev/null +++ b/packages/chord/test/next/contracts/history.test.ts @@ -0,0 +1,37 @@ +import { describe, expectTypeOf, it } from "vitest"; +import type { + ChangesAfter, + CurrentStep, + Increment, + OlderVersions, + StepAt, +} from "../../../src/next/contracts/history.ts"; + +type Empty = { readonly a: { readonly state: Record; readonly methods: Record } }; +type History = [ + { version: "9"; variants: Empty; changes: never }, + { version: "10"; variants: Empty; changes: ["a", "x", "state"] }, + { version: "11"; variants: Empty; changes: ["a", "m", "args"] }, +]; + +describe("type-level history", () => { + it("increments decimal versions", () => { + expectTypeOf>().toEqualTypeOf<"2">(); + expectTypeOf>().toEqualTypeOf<"10">(); + expectTypeOf>().toEqualTypeOf<"20">(); + expectTypeOf>().toEqualTypeOf<"100">(); + expectTypeOf>().toEqualTypeOf<"1100">(); + }); + + it("finds the current step, older versions and steps by version", () => { + expectTypeOf["version"]>().toEqualTypeOf<"11">(); + expectTypeOf>().toEqualTypeOf<"9" | "10">(); + expectTypeOf["changes"]>().toEqualTypeOf<["a", "x", "state"]>(); + }); + + it("collects every change after a version", () => { + expectTypeOf>().toEqualTypeOf<["a", "x", "state"] | ["a", "m", "args"]>(); + expectTypeOf>().toEqualTypeOf<["a", "m", "args"]>(); + expectTypeOf>().toEqualTypeOf(); + }); +}); diff --git a/packages/chord/test/next/contracts/host-kind.test.ts b/packages/chord/test/next/contracts/host-kind.test.ts new file mode 100644 index 000000000..3414ffd97 --- /dev/null +++ b/packages/chord/test/next/contracts/host-kind.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { evolve, hostKind, int, oneOf, opt, remote, str } from "../../../src/next/contracts/index.ts"; +import { contractData, kindParams } from "../../../src/next/contracts/internal.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { checkAddress } from "../../../src/next/data/internal.ts"; +import { expectCollected } from "../support.ts"; + +const Server = hostKind("server"); +const Session = hostKind("session", { id: str() }); +const Shard = hostKind("shard", { region: oneOf("eu", "us"), n: int({ min: 0 }) }); + +describe("host kinds", () => { + it("make addresses: the kind, the params and a key built like an identity key", () => { + expect(Server()).toEqual({ kind: "server", params: {}, key: "server" }); + expect(Session({ id: "S1" })).toEqual({ kind: "session", params: { id: "S1" }, key: "session|S1" }); + expect(Shard({ region: "eu", n: 3 }).key).toBe("shard|eu|3"); + // Escaped like identity keys: two different param lists never make one key. + expect(Session({ id: "a|b" }).key).not.toBe(Shard({ region: "a" as "eu", n: "b" as unknown as number }).key); + expect(Session({ id: "a|b" }).key).toBe("session|a\\|b"); + }); + + it("follow the params policy of contracts at declaration", () => { + // @ts-expect-error: params fields are required + expect(() => hostKind("k", { id: opt(str()) })).toThrow("k.params.id: a param is"); + expect(() => hostKind("")).toThrow(`A host kind's name "": expected lowercase [a-z0-9@/._~-]`); + expect(() => hostKind("Session")).toThrow("expected lowercase"); + expect(() => hostKind("a:b")).toThrow("expected lowercase"); + expect(() => hostKind("client")).toThrow('"client" is reserved for hosts without an address'); + expect(hostKind("@pi/session.v2_~-").kind).toBe("@pi/session.v2_~-"); + }); + + it("check an address's params where it is routed, with the kind's compiled check: invalid/placement", () => { + const params = kindParams(Session); + expect(checkAddress("session", params, { id: "S1" }, context).key).toBe("session|S1"); + for (const bad of [{ id: 1 }, {}, { id: "S1", extra: 1 }, null]) { + let thrown: unknown; + try { + checkAddress("session", params, bad, context); + } catch (error) { + thrown = error; + } + expect(isChordError(thrown, "invalid") && thrown.reason).toBe("placement"); + } + }); + + it("keep their params data only while the kind is referenced", async () => { + const ref = (() => { + const Temp = hostKind("temp", { id: str() }); + kindParams(Temp); + return new WeakRef(Temp); + })(); + await expectCollected(ref); + }); +}); + +describe("placement", () => { + const variants = { v: { state: { n: int() } } }; + + it("is not part of the fingerprint and is carried by evolve", () => { + const placement = ({ session }: { session: string }) => Session({ id: session }); + const placed = remote("app.placed", { version: 1, params: { session: str() }, placement, variants }); + const unplaced = remote("app.placed", { version: 1, params: { session: str() }, variants }); + expect(placed.fingerprint).toBe(unplaced.fingerprint); + expect(contractData(placed).placement).toBe(placement); + expect(contractData(unplaced).placement).toBeUndefined(); + const next = evolve(placed, (v) => v.addState("v", "m", int())); + expect(contractData(next).placement).toBe(placement); + }); + + it("is typed by the contract's params", () => { + remote("app.typed", { + version: 1, + params: { session: str() }, + // @ts-expect-error: the params have no `id` + placement: ({ id }) => Session({ id }), + variants, + }); + }); +}); diff --git a/packages/chord/test/next/contracts/params.test.ts b/packages/chord/test/next/contracts/params.test.ts new file mode 100644 index 000000000..159ff72ee --- /dev/null +++ b/packages/chord/test/next/contracts/params.test.ts @@ -0,0 +1,41 @@ +import Type from "typebox"; +import { describe, expect, it } from "vitest"; +import { arr, bool, bytes, int, num, obj, oneOf, opt, remote, str } from "../../../src/next/contracts/index.ts"; + +const variants = { a: { state: { s: str() } } }; + +describe("params policy", () => { + it("accepts required str, int, bool and oneOf of strings, with constraints", () => { + expect(() => + remote("n", { + version: 1, + params: { + a: str({ min: 1, max: 9, pattern: "^x" }), + b: int({ min: 0 }), + c: bool(), + d: oneOf("chat", "agent"), + }, + variants, + }), + ).not.toThrow(); + }); + + it("refuses optional fields, in the types and at declaration", () => { + // @ts-expect-error: params fields are required + expect(() => remote("n", { version: 1, params: { a: opt(str()) }, variants })).toThrow( + "n.params.a: a param is a required str(), int(), bool() or oneOf(...) of strings", + ); + }); + + it.each([ + ["a number", num()], + ["nested objects", obj({ a: str() })], + ["arrays", arr(str())], + ["bytes", bytes()], + ["numeric literals", oneOf(1, 2)], + ["defaults", Type.String({ default: "x" })], + ])("refuses %s, in the types and at declaration", (_case, schema) => { + // @ts-expect-error: not a params field schema + expect(() => remote("n", { version: 1, params: { p: schema }, variants })).toThrow("n.params.p: a param is"); + }); +}); diff --git a/packages/chord/test/next/contracts/schema.test.ts b/packages/chord/test/next/contracts/schema.test.ts new file mode 100644 index 000000000..f509d0262 --- /dev/null +++ b/packages/chord/test/next/contracts/schema.test.ts @@ -0,0 +1,99 @@ +import { Compile } from "typebox/compile"; +import { describe, expect, expectTypeOf, it } from "vitest"; +import { + arr, + bool, + bytes, + int, + lit, + map, + method, + nul, + num, + obj, + oneOf, + opt, + type Static, + str, + union, + unknown, +} from "../../../src/next/contracts/index.ts"; +import { Method } from "../../../src/next/contracts/internal.ts"; +import type { Equal } from "../support.ts"; + +describe("schema helpers", () => { + it("produce ordinary TypeBox schemas with bounds under their JSON Schema names", () => { + expect(str({ min: 1, max: 3, pattern: "^a" })).toEqual({ + type: "string", + minLength: 1, + maxLength: 3, + pattern: "^a", + }); + expect(num({ min: 0 })).toEqual({ type: "number", minimum: 0 }); + expect(int({ max: 9 })).toEqual({ type: "integer", maximum: 9 }); + expect(arr(bool(), { min: 1, max: 2 })).toEqual({ + type: "array", + items: { type: "boolean" }, + minItems: 1, + maxItems: 2, + }); + expect(nul()).toEqual({ type: "null" }); + expect(lit(3)).toEqual({ type: "number", const: 3 }); + expect(oneOf("a", "b")).toEqual({ anyOf: [lit("a"), lit("b")] }); + expect(union([str(), nul()])).toEqual({ anyOf: [str(), nul()] }); + expect(map(int())).toEqual({ type: "object", patternProperties: { "^.*$": int() } }); + expect(unknown()).toEqual({}); + }); + + it("leave absent bounds out instead of writing undefined", () => { + expect(Object.keys(str())).toEqual(["type"]); + expect(Object.keys(bytes())).toEqual(["type"]); + }); + + it("obj() is always closed and opt() makes a field optional", () => { + const schema = obj({ a: str(), b: opt(int()) }); + expect(schema).toEqual({ + type: "object", + required: ["a"], + properties: { a: str(), b: int() }, + additionalProperties: false, + }); + expect(Compile(schema).Check({ a: "x" })).toBe(true); + expect(Compile(schema).Check({ a: "x", c: 1 })).toBe(false); + }); + + it("Static and TypeBox's compiler work unchanged", () => { + const schema = obj({ kind: oneOf("chat", "agent"), n: opt(int()), tags: arr(str()), data: bytes() }); + type Expected = { kind: "chat" | "agent"; n?: number; tags: string[]; data: Uint8Array }; + expectTypeOf, Expected>>().toEqualTypeOf(); + expect(Compile(schema).Check({ kind: "chat", tags: [], data: new Uint8Array(1) })).toBe(true); + }); +}); + +describe("bytes", () => { + it("is the data marker with the check attached non-enumerably", () => { + const schema = bytes({ min: 1, max: 2 }); + expect(JSON.parse(JSON.stringify(schema))).toEqual({ type: "bytes", minLength: 1, maxLength: 2 }); + expect(Object.keys(schema)).toEqual(["type", "minLength", "maxLength"]); + expect(Object.getOwnPropertyDescriptor(schema, "~refine")?.enumerable).toBe(false); + }); + + it("checks instanceof Uint8Array and the length bounds", () => { + const validator = Compile(bytes({ min: 1, max: 2 })); + expect(validator.Check(new Uint8Array(1))).toBe(true); + expect(validator.Check(new Uint8Array(2))).toBe(true); + expect(validator.Check(new Uint8Array(0))).toBe(false); + expect(validator.Check(new Uint8Array(3))).toBe(false); + expect(validator.Check([1])).toBe(false); + expect(validator.Check("ab")).toBe(false); + }); +}); + +describe("method", () => { + it("an omitted side is null", () => { + expect(method()).toEqual(new Method(nul(), nul())); + expect(method(str())).toEqual(new Method(str(), nul())); + expect(method(undefined, int())).toEqual(new Method(nul(), int())); + expectTypeOf(method(undefined, int())).toEqualTypeOf, ReturnType>>(); + }); +}); diff --git a/packages/chord/test/next/contracts/snapshot.test.ts b/packages/chord/test/next/contracts/snapshot.test.ts new file mode 100644 index 000000000..d90e5a284 --- /dev/null +++ b/packages/chord/test/next/contracts/snapshot.test.ts @@ -0,0 +1,190 @@ +import Type from "typebox"; +import { Compile } from "typebox/compile"; +import { describe, expect, it } from "vitest"; +import { arr, bytes, int, map, obj, opt, str } from "../../../src/next/contracts/schema.ts"; +import { snapshotArguments, snapshotSchema } from "../../../src/next/contracts/snapshot.ts"; + +describe("vocabulary", () => { + it("accepts primitives, arrays, tuples, objects, records, unions, intersections, $defs/$ref, constraints", () => { + const recursive = Type.Cyclic( + { Node: Type.Object({ value: Type.Number(), next: Type.Union([Type.Null(), Type.Ref("Node")]) }) }, + "Node", + ); + const schemas = [ + Type.Tuple([Type.String(), Type.Integer()]), + Type.Intersect([Type.Object({ a: Type.String() }), Type.Object({ b: Type.String() })]), + Type.Enum(["a", "b"]), + recursive, + str({ min: 1, pattern: "^x" }), + Type.String({ format: "email", default: "a@b.c", readOnly: true, title: "t", description: "d" }), + Type.Number({ multipleOf: 2, exclusiveMinimum: 0 }), + Type.Array(Type.String(), { uniqueItems: true }), + Type.Object({}, { minProperties: 1, maxProperties: 2, $comment: "c", deprecated: true, examples: [{}] }), + map(int()), + bytes({ max: 4 }), + ]; + for (const schema of schemas) expect(() => snapshotSchema(schema, "s")).not.toThrow(); + }); + + it.each([ + ["conditional", { if: { type: "string" }, else: { minLength: 1 } }, 'unsupported keyword "else"'], + ["dependent", { type: "object", dependentSchemas: { a: {} } }, 'unsupported keyword "dependentSchemas"'], + [ + "dependentRequired", + { type: "object", dependentRequired: { a: ["b"] } }, + 'unsupported keyword "dependentRequired"', + ], + ["not", { not: { type: "string" } }, 'unsupported keyword "not"'], + ["contains", { type: "array", contains: { type: "string" } }, 'unsupported keyword "contains"'], + ["propertyNames", { type: "object", propertyNames: { pattern: "^a" } }, 'unsupported keyword "propertyNames"'], + ["unevaluatedProperties", { unevaluatedProperties: false }, 'unsupported keyword "unevaluatedProperties"'], + ["unevaluatedItems", { unevaluatedItems: false }, 'unsupported keyword "unevaluatedItems"'], + ["contentSchema", { type: "string", contentSchema: {} }, 'unsupported keyword "contentSchema"'], + ["definitions", { definitions: {} }, 'unsupported keyword "definitions"'], + ["oneOf", { oneOf: [{ type: "string" }] }, 'unsupported keyword "oneOf"'], + ["unknown keyword", { type: "string", flavour: "x" }, 'unsupported keyword "flavour"'], + ["non-JSON type", Type.BigInt(), 'unsupported type "bigint"'], + ["undefined", Type.Undefined(), 'unsupported type "undefined"'], + ["function", Type.Function([], Type.Null()), "s: unsupported"], + ["Unsafe", Type.Unsafe({ type: "string" }), "unsafe types are not supported"], + ["refinement", Type.Refine(Type.String(), () => true), "refinements"], + [ + "codec", + Type.Codec(Type.String()) + .Decode((v) => v) + .Encode((v) => v), + "codecs", + ], + ["non-JSON data", { type: "number", default: Number.NaN }, "s.default: expected JSON data"], + ["undefined data", { type: "number", maximum: undefined }, "s.maximum: expected JSON data"], + ["foreign prototype", new (class Schema {})(), "s: expected a schema"], + ["array as schema", [], "s: expected a schema"], + ])("refuses %s", (_name, schema, message) => { + expect(() => snapshotSchema(schema, "s")).toThrow(message); + }); + + it("refuses symbol keys, accessors and cycles, naming the path", () => { + expect(() => snapshotSchema({ type: "string", [Symbol("x")]: 1 }, "s")).toThrow( + "s: symbol keys are not supported", + ); + const accessor = Object.defineProperty({ type: "object" }, "properties", { get: () => ({}), enumerable: true }); + expect(() => snapshotSchema(accessor, "s")).toThrow("s.properties: accessors are not supported"); + const cyclic: { type: string; items?: object } = { type: "array" }; + cyclic.items = cyclic; + expect(() => snapshotSchema(cyclic, "s")).toThrow("s.items: cyclic schema"); + expect(() => snapshotSchema({ type: "object", properties: { a: { type: 1 } } }, "s")).toThrow( + "s.properties.a.type: unsupported type 1", + ); + }); + + it("allows a schema object shared by two places (not a cycle)", () => { + const shared = str(); + expect(() => snapshotSchema(obj({ a: shared, b: shared }), "s")).not.toThrow(); + }); +}); + +describe("closed arguments", () => { + it("requires every object, nested ones included, to be closed", () => { + expect(() => snapshotArguments(obj({ a: obj({ b: str() }) }), "m.args")).not.toThrow(); + expect(() => snapshotArguments(obj({ a: Type.Object({ b: str() }) }), "m.args")).toThrow( + "m.args.properties.a: argument objects must be closed (additionalProperties: false)", + ); + expect(() => snapshotArguments(Type.Object({}), "m.args")).toThrow("m.args: argument objects must be closed"); + expect(() => snapshotArguments(obj({ list: arr(Type.Object({})) }), "m.args")).toThrow( + "m.args.properties.list.items: argument objects must be closed", + ); + expect(() => snapshotArguments({ type: "object" }, "m.args")).toThrow("argument objects must be closed"); + }); + + it("accepts a raw closed Type.Object and records", () => { + expect(() => snapshotArguments(Type.Object({ a: str() }, { additionalProperties: false }), "m")).not.toThrow(); + expect(() => snapshotArguments(obj({ tags: map(str()) }), "m")).not.toThrow(); + }); + + it("checks objects inside $defs, unions and tuples", () => { + const open = Type.Object({ a: str() }); + expect(() => snapshotArguments(Type.Union([open, str()]), "m")).toThrow("m.anyOf[0]: argument objects"); + expect(() => snapshotArguments(Type.Tuple([open]), "m")).toThrow("m.items[0]: argument objects"); + expect(() => snapshotArguments({ $defs: { A: open }, $ref: "A" }, "m")).toThrow("m.$defs.A: argument objects"); + }); + + it("does not apply to state or results", () => { + expect(() => snapshotSchema(Type.Object({ a: str() }), "state")).not.toThrow(); + }); +}); + +describe("snapshots", () => { + it("are copies, independent of later changes to the app's schema objects", () => { + const app = obj({ text: str(), tags: arr(str()) }); + const { schema } = snapshotSchema(app, "s"); + (app.properties.text as { minLength?: number }).minLength = 5; + (app.required as string[]).push("x"); + expect(schema).not.toBe(app); + expect(Compile(schema).Check({ text: "", tags: [] })).toBe(true); + }); + + it("copy data values (defaults, consts)", () => { + const fallback = { a: [1] }; + const { schema } = snapshotSchema(Type.Object({}, { default: fallback, additionalProperties: false }), "s"); + fallback.a.push(2); + const copied = (schema as { default: { a: number[] } }).default; + expect(copied).toEqual({ a: [1] }); + }); + + it("rebuild the bytes check from the marker; no app callback is copied", () => { + const app = bytes({ max: 2 }); + const appCheck = Object.getOwnPropertyDescriptor(app, "~refine")!.value; + const { schema } = snapshotSchema(app, "s"); + expect(Object.getOwnPropertyDescriptor(schema, "~refine")!.value).not.toBe(appCheck); + expect(Compile(schema).Check(new Uint8Array(2))).toBe(true); + expect(Compile(schema).Check(new Uint8Array(3))).toBe(false); + // A hand-written marker works too: the check comes from the marker, not from the app. + const marker = snapshotSchema({ type: "bytes", minLength: 1 }, "s").schema; + expect(Compile(marker).Check(new Uint8Array(0))).toBe(false); + expect(Compile(marker).Check(new Uint8Array(1))).toBe(true); + }); + + it("refuse a bytes marker with other keywords or bad bounds", () => { + expect(() => snapshotSchema({ type: "bytes", items: {} }, "s")).toThrow('s: unsupported keyword "items"'); + expect(() => snapshotSchema({ type: "bytes", maxLength: -1 }, "s")).toThrow( + "s: bytes bounds must be non-negative integers", + ); + }); + + it("normal form: sorted keys, sorted required and enum, annotations dropped on schema nodes only", () => { + const { normal } = snapshotSchema( + { + title: "T", + type: "object", + required: ["b", "a"], + properties: { b: { enum: ["y", "x"] }, a: { type: "string", default: { title: "kept", z: 1, a: 2 } } }, + additionalProperties: false, + }, + "s", + ); + expect(JSON.stringify(normal)).toBe( + JSON.stringify({ + additionalProperties: false, + properties: { a: { default: { a: 2, title: "kept", z: 1 }, type: "string" }, b: { enum: ["x", "y"] } }, + required: ["a", "b"], + type: "object", + }), + ); + }); + + it("normal form keeps union and tuple order", () => { + const ab = snapshotSchema(Type.Union([str(), int()]), "s").normal; + const ba = snapshotSchema(Type.Union([int(), str()]), "s").normal; + expect(ab).not.toEqual(ba); + expect(snapshotSchema(opt(str()), "s").normal).toEqual({ type: "string" }); + }); +}); + +describe("bytes", () => { + it("mark a snapshot with a bytes leaf anywhere inside", () => { + expect(snapshotSchema(str(), "s").bytes).toBe(false); + expect(snapshotSchema(obj({ a: arr(obj({ b: bytes() })) }), "s").bytes).toBe(true); + expect(snapshotSchema(Type.Union([str(), bytes()]), "s").bytes).toBe(true); + expect(snapshotSchema(Type.Object({ a: str() }, { default: { type: "bytes" } }), "s").bytes).toBe(false); + }); +}); diff --git a/packages/chord/test/next/data/args.test.ts b/packages/chord/test/next/data/args.test.ts new file mode 100644 index 000000000..90dc8cae6 --- /dev/null +++ b/packages/chord/test/next/data/args.test.ts @@ -0,0 +1,223 @@ +import { describe, expect, expectTypeOf, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT, type Context } from "../../../src/context/index.ts"; +import { + type AcceptsTable, + arr, + bytes, + evolve, + int, + local, + method, + nul, + obj, + remote, + str, +} from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { bindProvider, type CheckedArgs, checkArgs, type Value } from "../../../src/next/data/internal.ts"; +import { counted, expectCollected } from "../support.ts"; + +const Note = obj({ text: str() }); +const NoteV2 = obj({ text: str(), tag: str() }); + +const NotesV1 = remote("notes", { + version: 1, + params: { session: str() }, + variants: { + editor: { + state: { notes: arr(Note) }, + methods: { add: method(Note, str()), clear: method(), attach: method(obj({ data: bytes({ max: 4 }) })) }, + }, + reader: { state: { notes: arr(Note) } }, + }, +}); +const NotesV2 = evolve(NotesV1, (e) => e.args("add", NoteV2).result("add", int())); +const NotesV3 = evolve(NotesV2, (e) => e.removeMethod("attach")); + +const toV2 = vi.fn((old: { text: string }) => ({ ...old, tag: "v1" })); +const resultToOld = (current: number) => String(current); +const attach = async () => null; +const accepts: AcceptsTable = { + 1: { methods: { add: { args: toV2, result: resultToOld }, attach: { call: attach } } }, + 2: { methods: { attach: "refuse" } }, +}; +const provider = bindProvider(NotesV3, accepts); + +/** The error a refused check throws, as JSON. */ +function refusal(run: () => unknown): unknown { + try { + run(); + } catch (error) { + if (isChordError(error)) return error.toJSON(); + throw error; + } + throw new Error("expected a refusal"); +} + +describe("bindProvider", () => { + it("checks the accepts table once and lists the accepted versions", () => { + expect(provider.contract).toBe(NotesV3); + expect(provider.accepted).toEqual([ + { version: 1, fingerprint: NotesV1.fingerprint }, + { version: 2, fingerprint: NotesV2.fingerprint }, + ]); + expect(() => bindProvider(NotesV3, { 1: {} } as never)).toThrow("notes accepts[1].methods.add: missing mapping"); + }); + + it("compiles nothing and calls no mapping", async () => { + const counts = await counted(() => bindProvider(NotesV3, accepts)); + expect(counts).toEqual({ walks: 0, validations: 0, compilations: 0 }); + expect(toV2).not.toHaveBeenCalled(); + }); +}); + +describe("checkArgs", () => { + it("returns current-version arguments as passed: no copy, never mapped", () => { + const raw = { text: "t", tag: "x" }; + const checked = checkArgs(provider, "add", raw, 3, BACKGROUND_CONTEXT); + expect(checked).toEqual({ kind: "method", args: raw, result: undefined }); + expect(checked.args).toBe(raw); + expect(toV2).not.toHaveBeenCalled(); + expectTypeOf(checked).toEqualTypeOf>(); + if (checked.kind === "method") expectTypeOf(checked.args).toEqualTypeOf<{ text: string; tag: string }>(); + }); + + it("checks an old version against the old schema, then maps directly to the current shape", () => { + const checked = checkArgs(provider, "add", { text: "t" }, 1, BACKGROUND_CONTEXT); + expect(checked).toEqual({ kind: "method", args: { text: "t", tag: "v1" }, result: resultToOld }); + expect(refusal(() => checkArgs(provider, "add", { text: "t", tag: "x" }, 1, BACKGROUND_CONTEXT))).toMatchObject({ + code: "invalid", + reason: "args", + details: { path: "/tag" }, + }); + // Version 2 already has the current args: checked with that schema, not mapped. + expect(checkArgs(provider, "add", { text: "t", tag: "x" }, 2, BACKGROUND_CONTEXT)).toEqual({ + kind: "method", + args: { text: "t", tag: "x" }, + result: undefined, + }); + }); + + it("returns the old arguments with the call replacement; never invokes it", () => { + const raw = { data: new Uint8Array(2) }; + expect(checkArgs(provider, "attach", raw, 1, BACKGROUND_CONTEXT)).toEqual({ + kind: "call", + args: raw, + call: attach, + }); + }); + + it("refuses unknown fields, nested ones too, and checks bytes by instanceof and length", () => { + expect( + refusal(() => checkArgs(provider, "add", { text: "t", tag: "x", extra: 1 }, 3, BACKGROUND_CONTEXT)), + ).toMatchObject({ + details: { path: "/extra" }, + }); + expect(refusal(() => checkArgs(provider, "attach", { data: [1, 2] }, 1, BACKGROUND_CONTEXT))).toMatchObject({ + details: { path: "/data" }, + }); + expect( + refusal(() => checkArgs(provider, "attach", { data: new Uint8Array(5) }, 1, BACKGROUND_CONTEXT)), + ).toMatchObject({ + details: { path: "/data" }, + }); + const Nested = local("nested", { version: 1, params: {}, members: { m: method(obj({ a: obj({ b: str() }) })) } }); + const nested = bindProvider(Nested, {}); + expect(refusal(() => checkArgs(nested, "m", { a: { b: "x", c: 1 } }, 1, BACKGROUND_CONTEXT))).toMatchObject({ + details: { path: "/a/c" }, + }); + }); + + it("takes null for no-argument methods", () => { + expect(checkArgs(provider, "clear", null, 3, BACKGROUND_CONTEXT)).toEqual({ + kind: "method", + args: null, + result: undefined, + }); + expect(refusal(() => checkArgs(provider, "clear", {}, 3, BACKGROUND_CONTEXT))).toMatchObject({ + code: "invalid", + reason: "args", + details: { path: "/" }, + }); + }); + + it("checks local callers too", () => { + const Info = local("info", { version: 1, params: {}, members: { save: method(obj({ title: str() }), nul()) } }); + const local1 = bindProvider(Info, {}); + expect(checkArgs(local1, "save", { title: "t" }, 1, BACKGROUND_CONTEXT).args).toEqual({ title: "t" }); + expect(refusal(() => checkArgs(local1, "save", { title: 1 }, 1, BACKGROUND_CONTEXT))).toMatchObject({ + message: "args/title: must be string", + }); + }); + + it("refuses an unknown method, an unaccepted version and a refused method", () => { + expect(refusal(() => checkArgs(provider, "missing", null, 3, BACKGROUND_CONTEXT))).toEqual({ + code: "invalid", + reason: "member", + message: 'notes v3 has no method "missing"', + }); + expect(refusal(() => checkArgs(provider, "attach", null, 3, BACKGROUND_CONTEXT))).toMatchObject({ + code: "invalid", + reason: "member", + }); + expect(refusal(() => checkArgs(provider, "add", { text: "t" }, 4, BACKGROUND_CONTEXT))).toEqual({ + code: "contract_mismatch", + message: "notes v4 is not accepted", + }); + expect( + refusal(() => checkArgs(bindProvider(NotesV3, {}), "add", { text: "t" }, 1, BACKGROUND_CONTEXT)), + ).toMatchObject({ + code: "contract_mismatch", + }); + expect(refusal(() => checkArgs(provider, "attach", { data: new Uint8Array() }, 2, BACKGROUND_CONTEXT))).toEqual({ + code: "contract_mismatch", + message: 'notes v2 refuses "attach"', + }); + }); + + it("compiles lazily, once per accepted version and method; equal schema objects share a validator", async () => { + const v1 = local("lazy", { version: 1, params: {}, members: { m: method(obj({ a: str() })), clear: method() } }); + const v2 = evolve(v1, (e) => e.args("m", obj({ a: str(), b: str() }))); + const fresh = bindProvider(v2, { 1: { methods: { m: { args: (old) => ({ ...old, b: "" }) } } } }); + const calls = [ + (context: Context) => checkArgs(fresh, "clear", null, 2, context), + (context: Context) => checkArgs(fresh, "clear", null, 2, context), + // Version 1 shares the unchanged `clear` schema object with version 2. + (context: Context) => checkArgs(fresh, "clear", null, 1, context), + (context: Context) => checkArgs(fresh, "m", { a: "x" }, 1, context), + (context: Context) => checkArgs(fresh, "m", { a: "x" }, 1, context), + (context: Context) => checkArgs(fresh, "m", { a: "x", b: "y" }, 2, context), + ]; + const compilations = []; + for (const call of calls) compilations.push((await counted(call)).compilations); + expect(compilations).toEqual([1, 0, 0, 1, 0, 1]); + }); + + it("counts one validation per call and no walk", async () => { + const counts = await counted((context) => checkArgs(provider, "add", { text: "t" }, 1, context)); + expect(counts).toEqual({ walks: 0, validations: 1, compilations: 0 }); + }); + + it("types a call replacement's arguments as values", () => { + expectTypeOf, { kind: "call" }>["call"]>().toEqualTypeOf< + (args: Value, context: Context) => Promise + >(); + }); +}); + +describe("memory", () => { + it("a dropped binding releases its dispatch table, mappings, schemas and validators after the last call", async () => { + const refs = await (async () => { + const contract = evolve( + local("gc", { version: 1, params: {}, members: { m: method(obj({ a: str() })) } }), + (e) => e.args("m", obj({ a: str(), b: str() })), + ); + const mapping = (old: { a: string }) => ({ ...old, b: "" }); + const bound = bindProvider(contract, { 1: { methods: { m: { args: mapping } } } }); + const pending = Promise.resolve(checkArgs(bound, "m", { a: "x" }, 1, BACKGROUND_CONTEXT)); + await pending; + return [new WeakRef(contract), new WeakRef(bound), new WeakRef(mapping)]; + })(); + await expectCollected(...refs); + }); +}); diff --git a/packages/chord/test/next/data/errors.test.ts b/packages/chord/test/next/data/errors.test.ts new file mode 100644 index 000000000..65dee086a --- /dev/null +++ b/packages/chord/test/next/data/errors.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, expectTypeOf, it } from "vitest"; +import { ChordError, type ChordErrorReason, isChordError, PluginError } from "../../../src/next/data/index.ts"; +import { + asChordError, + CHORD_ERROR_REASONS, + chordErrorFromJson, + invalid, + PROTOCOL_VERSION, +} from "../../../src/next/data/internal.ts"; + +describe("ChordError", () => { + it("is a tagged union: a wrong pairing is a type error", () => { + new ChordError({ code: "invalid", reason: "args", message: "m" }); + new ChordError({ code: "unavailable", message: "m" }); + new ChordError({ code: "unavailable", reason: "locked", message: "m" }); + // @ts-expect-error: invalid requires a reason + new ChordError({ code: "invalid", message: "m" }); + // @ts-expect-error: gone carries no reason + new ChordError({ code: "gone", reason: "args", message: "m" }); + // @ts-expect-error: part belongs to too_large + new ChordError({ code: "unavailable", reason: "part", message: "m" }); + }); + + it("narrowing on code narrows reason", () => { + const error = new ChordError({ code: "too_large", reason: "part", message: "m" }) as ChordError; + if (error.code === "invalid") expectTypeOf(error.reason).toEqualTypeOf>(); + if (error.code === "unavailable") + expectTypeOf(error.reason).toEqualTypeOf<"locked" | "park_timeout" | "busy" | "storage_failed" | undefined>(); + if (error.code === "gone") expectTypeOf(error.reason).toEqualTypeOf(); + expect(error).toBeInstanceOf(Error); + expect(error.name).toBe("ChordError"); + }); + + it("round-trips through JSON without stack or cause", () => { + const details = { path: "/text" }; + const error = new ChordError( + { code: "invalid", reason: "args", message: "args/text: bad", details }, + { cause: new Error("x") }, + ); + const json = JSON.parse(JSON.stringify(error)); + expect(json).toEqual({ code: "invalid", reason: "args", message: "args/text: bad", details }); + const back = chordErrorFromJson(json); + expect(back).toBeInstanceOf(ChordError); + expect(back.toJSON()).toEqual(json); + expect(new ChordError({ code: "gone", message: "m" }).toJSON()).toEqual({ code: "gone", message: "m" }); + }); + + it("keeps details by reference", () => { + const details = { large: [1, 2, 3] }; + expect(chordErrorFromJson({ code: "gone", message: "m", details }).details).toBe(details); + }); + + it.each([ + ["not an object", "x", "not an object"], + ["null", null, "not an object"], + ["an array", [], "not an object"], + ["unknown key", { code: "gone", message: "m", stack: "s" }, 'unknown key "stack"'], + ["unknown code", { code: "teapot", message: "m" }, "unknown code"], + ["inherited name as code", { code: "toString", message: "m" }, "unknown code"], + ["code not a string", { code: 1, message: "m" }, "unknown code"], + ["invalid without reason", { code: "invalid", message: "m" }, "reason not allowed for invalid"], + ["reason not allowed", { code: "gone", reason: "args", message: "m" }, "reason not allowed for gone"], + [ + "reason of another code", + { code: "unavailable", reason: "part", message: "m" }, + "reason not allowed for unavailable", + ], + ["message not a string", { code: "gone", message: 1 }, "message is not a string"], + ])("chordErrorFromJson refuses %s", (_case, raw, problem) => { + expect(() => chordErrorFromJson(raw)).toThrow(`Invalid ChordError JSON: ${problem}`); + }); +}); + +describe("protocol table", () => { + it("is pinned together with PROTOCOL_VERSION", () => { + const pinned = { + version: 1, + reasons: { + invalid: ["args", "params", "member", "placement", "reentrant", "read_after_bulk", "cycle"], + contract_mismatch: ["updating"], + forbidden: ["revoked"], + gone: [], + not_ready: [], + released: ["ended", "closed", "settled", "version", "discarded"], + too_large: ["part"], + unavailable: ["locked", "park_timeout", "busy", "storage_failed"], + aborted: [], + }, + }; + const changed = + PROTOCOL_VERSION !== pinned.version || JSON.stringify(CHORD_ERROR_REASONS) !== JSON.stringify(pinned.reasons); + expect(changed, "the error table changed: bump PROTOCOL_VERSION and update this pin").toBe(false); + }); +}); + +describe("helpers", () => { + it("isChordError checks the class and optionally the code", () => { + const error = invalid("member", "no method"); + expect(isChordError(error)).toBe(true); + expect(isChordError(error, "invalid")).toBe(true); + expect(isChordError(error, "gone")).toBe(false); + expect(isChordError(new Error("x"))).toBe(false); + expect(isChordError({ code: "invalid", reason: "member", message: "m" })).toBe(false); + }); + + it("invalid builds an invalid error with its reason and details", () => { + const error = invalid("params", "params/a: missing", { path: "/a" }); + expect(error.toJSON()).toEqual({ + code: "invalid", + reason: "params", + message: "params/a: missing", + details: { path: "/a" }, + }); + expectTypeOf(error.code).toEqualTypeOf<"invalid">(); + }); + + it("asChordError keeps a ChordError and wraps anything else as unavailable", () => { + const chord = invalid("cycle", "c"); + expect(asChordError(chord, "open")).toBe(chord); + const foreign = new TypeError("boom"); + const wrapped = asChordError(foreign, "open notes"); + expect(wrapped.toJSON()).toEqual({ code: "unavailable", message: "open notes: boom" }); + expect(wrapped.cause).toBe(foreign); + expect(asChordError("text", "open").message).toBe("open: text"); + }); +}); + +describe("PluginError", () => { + it("carries a plugin's code, message and JSON details", () => { + const error = new PluginError("quota", "too many", { limit: 3 }); + expect(error).toBeInstanceOf(Error); + expect(error).toMatchObject({ name: "PluginError", code: "quota", message: "too many", details: { limit: 3 } }); + expect(new PluginError("c", "m").details).toBeUndefined(); + }); +}); diff --git a/packages/chord/test/next/data/failure.test.ts b/packages/chord/test/next/data/failure.test.ts new file mode 100644 index 000000000..a5ddbacf2 --- /dev/null +++ b/packages/chord/test/next/data/failure.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { broken } from "../../../src/next/data/index.ts"; +import { invalid, openFailure } from "../../../src/next/data/internal.ts"; + +describe("failure causes", () => { + it("a value thrown by an open is transient unless marked broken", () => { + const error = new Error("network down"); + const transient = openFailure(error, "open notes"); + expect(transient.kind).toBe("transient"); + expect(transient.error.toJSON()).toEqual({ code: "unavailable", message: "open notes: network down" }); + expect(transient.error.cause).toBe(error); + }); + + it("broken() marks an open's failure as broken; its message is the cause's", () => { + const marked = broken(new Error("bad config")); + expect(marked.message).toBe("bad config"); + const cause = openFailure(marked, "open notes"); + expect(cause.kind).toBe("broken"); + expect(cause.error.message).toBe("open notes: bad config"); + }); + + it("the kind comes from the stage, never from the error code", () => { + const chord = invalid("placement", "wrong host"); + expect(openFailure(chord, "open")).toEqual({ kind: "transient", error: chord }); + expect(openFailure(broken(chord), "open")).toEqual({ kind: "broken", error: chord }); + }); +}); diff --git a/packages/chord/test/next/data/params.test.ts b/packages/chord/test/next/data/params.test.ts new file mode 100644 index 000000000..55b9a8ebc --- /dev/null +++ b/packages/chord/test/next/data/params.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, expectTypeOf, it } from "vitest"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { contractData } from "../../../src/next/contracts/contract.ts"; +import { bool, int, local, oneOf, remote, str } from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { checkParams, identityKey } from "../../../src/next/data/internal.ts"; +import { validatorFor } from "../../../src/next/data/validator.ts"; +import { counted, expectCollected } from "../support.ts"; + +const Session = remote("session", { + version: 1, + params: { id: str({ min: 1 }), mode: oneOf("chat", "agent"), page: int({ min: 0 }), open: bool() }, + variants: { owner: {} }, +}); + +const valid = { id: "S1", mode: "chat", page: 2, open: true } as const; + +/** The `invalid/params` error of a refused check. */ +function refusal(raw: unknown): unknown { + try { + checkParams(Session, raw, BACKGROUND_CONTEXT); + } catch (error) { + if (isChordError(error, "invalid")) + return { reason: error.reason, message: error.message, details: error.details }; + throw error; + } + throw new Error("expected a refusal"); +} + +describe("checkParams", () => { + it("returns the caller's own object and the key of the field values in declared order", () => { + const { params, key } = checkParams(Session, valid, BACKGROUND_CONTEXT); + expect(params).toBe(valid); + expect(key).toBe("S1|chat|2|true"); + expectTypeOf(params).toEqualTypeOf<{ + readonly id: string; + readonly mode: "chat" | "agent"; + readonly page: number; + readonly open: boolean; + }>(); + }); + + it("gives the same key whatever the property order", () => { + expect(checkParams(Session, { open: true, page: 2, mode: "chat", id: "S1" }, BACKGROUND_CONTEXT).key).toBe( + "S1|chat|2|true", + ); + }); + + it("refuses missing, extra and wrong fields at their paths", () => { + const { id: _, ...missing } = valid; + expect(refusal(missing)).toEqual({ + reason: "params", + message: expect.stringMatching(/^params\/id: /), + details: { path: "/id" }, + }); + expect(refusal({ ...valid, extra: 1 })).toMatchObject({ + message: expect.stringMatching(/^params\/extra: /), + details: { path: "/extra" }, + }); + expect(refusal({ ...valid, page: "2" })).toMatchObject({ + message: "params/page: must be integer", + details: { path: "/page" }, + }); + expect(refusal("S1")).toMatchObject({ message: "params: must be object", details: { path: "/" } }); + }); + + it("enforces literal membership and constraints", () => { + expect(refusal({ ...valid, mode: "other" })).toMatchObject({ details: { path: "/mode" } }); + expect(refusal({ ...valid, id: "" })).toMatchObject({ details: { path: "/id" } }); + expect(refusal({ ...valid, page: -1 })).toMatchObject({ details: { path: "/page" } }); + expect(refusal({ ...valid, page: 1.5 })).toMatchObject({ details: { path: "/page" } }); + }); + + it("escapes the separator: no two params join to one key", () => { + const Pair = local("pair", { version: 1, params: { a: str(), b: str() }, members: {} }); + const keys = [ + { a: "x|y", b: "z" }, + { a: "x", b: "y|z" }, + { a: "x\\", b: "|z" }, + { a: "x\\|", b: "z" }, + ].map((raw) => checkParams(Pair, raw, BACKGROUND_CONTEXT).key); + expect(new Set(keys).size).toBe(keys.length); + expect(keys[0]).toBe("x\\|y|z"); + }); + + it("counts one compilation per contract, then one validation per get; a failure adds one walk", async () => { + const Fresh = local("fresh", { version: 1, params: { id: str() }, members: {} }); + expect(await counted((context) => checkParams(Fresh, { id: "a" }, context))).toEqual({ + walks: 0, + validations: 1, + compilations: 1, + }); + expect(await counted((context) => checkParams(Fresh, { id: "b" }, context))).toEqual({ + walks: 0, + validations: 1, + compilations: 0, + }); + const failed = await counted((context) => { + expect(() => checkParams(Fresh, { id: 1 }, context)).toThrow(); + }); + expect(failed).toEqual({ walks: 1, validations: 1, compilations: 0 }); + }); + + it("does not retain a dropped contract's params validator", async () => { + const refs = (() => { + const contract = local("gc", { version: 1, params: { id: str() }, members: {} }); + checkParams(contract, { id: "a" }, BACKGROUND_CONTEXT); + const validator = validatorFor(contractData(contract).params, BACKGROUND_CONTEXT); + return [new WeakRef(contract), new WeakRef(validator)]; + })(); + await expectCollected(...refs); + }); +}); + +describe("identityKey", () => { + it("joins name, key and principal key, each escaped", () => { + expect(identityKey(Session, "S1|chat|2|true", '{"kind":"user"}')).toBe( + 'session|S1\\|chat\\|2\\|true|{"kind":"user"}', + ); + }); + + it("local objects take no principal key, by type", () => { + const Info = local("info", { version: 1, params: {}, members: {} }); + expect(identityKey(Info, "")).toBe("info|"); + // @ts-expect-error: local has no principal + identityKey(Info, "", "p"); + // @ts-expect-error: remote needs a principal key + identityKey(Session, "k"); + }); +}); diff --git a/packages/chord/test/next/data/principal.test.ts b/packages/chord/test/next/data/principal.test.ts new file mode 100644 index 000000000..adca8418f --- /dev/null +++ b/packages/chord/test/next/data/principal.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { canonicalJson, principalKey } from "../../../src/next/data/internal.ts"; +import { counted, expectCollected } from "../support.ts"; + +describe("canonicalJson", () => { + it("sorts object keys and writes -0 as 0", () => { + expect(canonicalJson({ b: 1, a: [{ d: -0, c: null }] }, BACKGROUND_CONTEXT)).toBe( + '{"a":[{"c":null,"d":0}],"b":1}', + ); + expect(canonicalJson("x", BACKGROUND_CONTEXT)).toBe('"x"'); + }); + + it("counts one walk", async () => { + expect(await counted((context) => canonicalJson({ a: [1, 2] }, context))).toEqual({ + walks: 1, + validations: 0, + compilations: 0, + }); + }); +}); + +describe("principalKey", () => { + it("defaults to the canonical JSON, whatever the property order", () => { + expect(principalKey({ kind: "user", id: "u1" }, BACKGROUND_CONTEXT)).toBe('{"id":"u1","kind":"user"}'); + expect(principalKey({ id: "u1", kind: "user" }, BACKGROUND_CONTEXT)).toBe( + principalKey({ kind: "user", id: "u1" }, BACKGROUND_CONTEXT), + ); + }); + + it("uses a custom key function when given", () => { + const byId = (principal: { kind: string; id: string }) => `${principal.kind}:${principal.id}`; + expect(principalKey({ kind: "user", id: "u1" }, BACKGROUND_CONTEXT, byId)).toBe("user:u1"); + }); + + it("caches per principal object: no second walk", async () => { + const principal = { kind: "user", id: "u1" }; + const first = await counted((context) => principalKey(principal, context)); + const second = await counted((context) => principalKey(principal, context)); + expect([first.walks, second.walks]).toEqual([1, 0]); + }); + + it("caches custom keys per function", () => { + const principal = { kind: "user", id: "u1" }; + let calls = 0; + const custom = () => `key${++calls}`; + expect([ + principalKey(principal, BACKGROUND_CONTEXT, custom), + principalKey(principal, BACKGROUND_CONTEXT, custom), + ]).toEqual(["key1", "key1"]); + expect(principalKey(principal, BACKGROUND_CONTEXT, () => "other")).toBe("other"); + expect(principalKey(principal, BACKGROUND_CONTEXT)).toBe('{"id":"u1","kind":"user"}'); + }); + + it("never retains a dropped custom key function while its principal lives", async () => { + const principal = { kind: "user", id: "u1" }; + const ref = (() => { + const custom = () => "k"; + principalKey(principal, BACKGROUND_CONTEXT, custom); + return new WeakRef(custom); + })(); + await expectCollected(ref); + expect(principalKey(principal, BACKGROUND_CONTEXT)).toBeTypeOf("string"); + }); + + it("does not retain dropped principals", async () => { + const ref = (() => { + const principal = { kind: "user", id: "gone" }; + principalKey(principal, BACKGROUND_CONTEXT); + principalKey(principal, BACKGROUND_CONTEXT, () => "k"); + return new WeakRef(principal); + })(); + await expectCollected(ref); + }); +}); diff --git a/packages/chord/test/next/data/validator.test.ts b/packages/chord/test/next/data/validator.test.ts new file mode 100644 index 000000000..b78b55c58 --- /dev/null +++ b/packages/chord/test/next/data/validator.test.ts @@ -0,0 +1,106 @@ +import { GetExternal, GetFunctions } from "typebox/schema"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, bytes, int, map, obj, str } from "../../../src/next/contracts/schema.ts"; +import { snapshotArguments } from "../../../src/next/contracts/snapshot.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { check, validatorFor } from "../../../src/next/data/validator.ts"; +import { counted, expectCollected } from "../support.ts"; + +const args = snapshotArguments( + obj({ text: str(), items: arr(obj({ n: int() })), tags: map(str()), blob: bytes({ max: 2 }) }), + "m.args", +); + +const valid = { text: "t", items: [{ n: 1 }], tags: {}, blob: new Uint8Array(1) }; + +/** The `{ message, details }` of the `invalid/args` error a failed check throws. */ +function failure(value: unknown): unknown { + try { + check(validatorFor(args, BACKGROUND_CONTEXT), value, "args", BACKGROUND_CONTEXT); + } catch (error) { + if (isChordError(error, "invalid")) + return { reason: error.reason, message: error.message, details: error.details }; + throw error; + } + return "passed"; +} + +describe("failed checks", () => { + it("pass a valid value", () => { + expect(failure(valid)).toBe("passed"); + }); + + it("report TypeBox's first error with its JSON Pointer path", () => { + expect(failure({ ...valid, text: 1 })).toEqual({ + reason: "args", + message: "args/text: must be string", + details: { path: "/text" }, + }); + expect(failure({ ...valid, items: [{ n: "x" }] })).toMatchObject({ details: { path: "/items/0/n" } }); + expect(failure(null)).toEqual({ reason: "args", message: "args: must be object", details: { path: "/" } }); + }); + + it("report a missing field at its own path (TypeBox reports it at the parent)", () => { + const { text: _, ...missing } = valid; + expect(failure(missing)).toMatchObject({ + message: "args/text: missing", + details: { path: "/text" }, + }); + expect(failure({ ...valid, items: [{}] })).toMatchObject({ details: { path: "/items/0/n" } }); + }); + + it("report an unknown field at its own path, nested too", () => { + expect(failure({ ...valid, extra: 1 })).toEqual({ + reason: "args", + message: "args/extra: unknown field", + details: { path: "/extra" }, + }); + expect(failure({ ...valid, items: [{ n: 1, m: 2 }] })).toMatchObject({ details: { path: "/items/0/m" } }); + }); + + it("check bytes by instanceof and length", () => { + expect(failure({ ...valid, blob: [1] })).toMatchObject({ + message: "args/blob: expected bytes of the declared length", + }); + expect(failure({ ...valid, blob: new Uint8Array(3) })).toMatchObject({ details: { path: "/blob" } }); + }); + + it("leave keys containing `/` or `~` unescaped (TypeBox's instance path, ambiguous)", () => { + expect(failure({ ...valid, tags: { "a/b": 1 } })).toMatchObject({ details: { path: "/tags/a/b" } }); + }); +}); + +describe("validators", () => { + it("compile once per schema owner (counted), then validate (counted); a failure adds one walk", async () => { + const owner = snapshotArguments(obj({ a: str() }), "m"); + const first = await counted((context) => check(validatorFor(owner, context), { a: "x" }, "args", context)); + const second = await counted((context) => check(validatorFor(owner, context), { a: "y" }, "args", context)); + const failed = await counted((context) => { + expect(() => check(validatorFor(owner, context), {}, "args", context)).toThrow(); + }); + expect([first, second, failed]).toEqual([ + { walks: 0, validations: 1, compilations: 1 }, + { walks: 0, validations: 1, compilations: 0 }, + { walks: 1, validations: 1, compilations: 0 }, + ]); + }); + + it("leave nothing in TypeBox's build accumulators", () => { + validatorFor(snapshotArguments(obj({ blob: bytes(), n: int() }), "m"), BACKGROUND_CONTEXT); + expect(GetExternal().variables).toEqual([]); + expect(GetFunctions()).toEqual([]); + }); + + it("are collected with their schema owner; TypeBox retains no build state", async () => { + const refs = (() => { + const owner = snapshotArguments(obj({ blob: bytes() }), "m"); + const validator = validatorFor(owner, BACKGROUND_CONTEXT); + const blob = (owner.schema as { properties: { blob: object } }).properties.blob; + // TypeBox's external variables would hold the bytes check if the build state were retained. + const [check] = Object.getOwnPropertyDescriptor(blob, "~refine")!.value as object[]; + return [new WeakRef(owner), new WeakRef(validator), new WeakRef(check!)]; + })(); + await expectCollected(...refs); + }); +}); diff --git a/packages/chord/test/next/documents/address-source.test.ts b/packages/chord/test/next/documents/address-source.test.ts new file mode 100644 index 000000000..ea621c072 --- /dev/null +++ b/packages/chord/test/next/documents/address-source.test.ts @@ -0,0 +1,114 @@ +// Tests of address sources (slice 6a §11; decision "One retirement mechanism"): the frame a watched document address +// gets for a change, a write over another base, a retirement, a recreation after it in the same batch, and a +// replacement outside a commit. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import type { Op, Prepared } from "../../../src/delta/index.ts"; +import { AddressSource } from "../../../src/next/documents/address-source.ts"; +import type { Documents } from "../../../src/next/documents/documents.ts"; +import type { JsonObject } from "../../../src/next/documents/index.ts"; +import { Batch } from "../../../src/next/state/internal.ts"; +import { changesOf, delivered, expectCollected } from "../support.ts"; + +/** An address whose value the test sets. */ +class Address extends AddressSource { + // No object's documents: the test reads frames only. + readonly owner = {} as Documents; + current: JsonObject = { n: 0 }; + + get value(): JsonObject { + return this.current; + } +} + +/** A prepared write of `value` over `base` with `ops`. */ +function prepared(base: JsonObject, value: JsonObject, ops: readonly Op[]): Prepared { + return { base, value, ops, baseRevision: 0, abort: () => {} }; +} + +/** A watched address on its own batch, and the changes its watch got. */ +async function watched() { + const batch = new Batch(() => {}, undefined); + const address = new Address(batch); + const changes = changesOf(await address.watch(context)); + return { batch, address, changes }; +} + +describe("address frames", () => { + it("a write over the read value carries its ops; over another base, a root replacement", async () => { + const { batch, address, changes } = await watched(); + const first = address.current; + batch.commit(() => { + address.current = { n: 1 }; + address.written(first, prepared(first, address.current, [["s", ["n"], 1]])); + }, context); + await delivered(); + const second = address.current; + batch.commit(() => { + address.current = { n: 2 }; + address.written(second, prepared({ n: 7 }, address.current, [["s", ["n"], 2]])); + }, context); + await delivered(); + expect(changes).toEqual([ + { kind: "value", value: { n: 1 }, ops: [["s", ["n"], 1]], retired: false }, + { kind: "value", value: { n: 2 }, ops: [["r", { n: 2 }]], retired: false }, + ]); + }); + + it("a retirement reads initial() with the flag; a recreation in the same batch is one root replacement, flagged", async () => { + const { batch, address, changes } = await watched(); + batch.commit(() => { + address.current = { n: 0 }; + address.retired(address.current); + }, context); + await delivered(); + batch.commit(() => { + const initial = address.current; + address.retired(initial); + address.current = { n: 5 }; + address.written(initial, prepared(initial, address.current, [["s", ["n"], 5]])); + }, context); + await delivered(); + const recreated = address.current; + batch.commit(() => { + address.current = { n: 6 }; + address.written(recreated, prepared(recreated, address.current, [["s", ["n"], 6]])); + }, context); + await delivered(); + expect(changes).toEqual([ + { kind: "value", value: { n: 0 }, ops: [["r", { n: 0 }]], retired: true }, + { kind: "value", value: { n: 5 }, ops: [["r", { n: 5 }]], retired: true }, + // The flag was reset by the frame that carried it. + { kind: "value", value: { n: 6 }, ops: [["s", ["n"], 6]], retired: false }, + ]); + }); + + it("a replacement outside a commit (a load, a migration) is a root replacement", async () => { + const { batch, address, changes } = await watched(); + batch.commit(() => { + address.current = { n: 3 }; + address.replaced(); + }, context); + await delivered(); + expect(changes).toEqual([{ kind: "value", value: { n: 3 }, ops: [["r", { n: 3 }]], retired: false }]); + }); +}); + +describe("unwatched addresses", () => { + it("keep no ops of a write: no frame takes them", async () => { + const batch = new Batch(() => {}, undefined); + const address = new Address(batch); + let ops: Op[] | undefined = [["s", ["n"], 1]]; + const collected = new WeakRef(ops); + const base = address.current; + batch.commit(() => { + address.current = { n: 1 }; + address.written(base, prepared(base, address.current, ops!)); + }, context); + ops = undefined; + await expectCollected(collected); + // The address itself is still here. + expect(address.value).toEqual({ n: 1 }); + }); +}); diff --git a/packages/chord/test/next/documents/definition.test.ts b/packages/chord/test/next/documents/definition.test.ts new file mode 100644 index 000000000..9efbb1224 --- /dev/null +++ b/packages/chord/test/next/documents/definition.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, expectTypeOf, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, int, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState, type StateDefinition } from "../../../src/next/documents/index.ts"; +import { metered } from "../support.ts"; + +describe("defineState", () => { + it("types the value and whether it is keyed", () => { + const Notes = defineState("notes", { + schema: obj({ list: arr(str()) }), + version: 1, + initial: () => ({ list: [] }), + }); + const Drafts = defineState("drafts", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: 0 }), + keyed: true, + }); + expectTypeOf(Notes).toEqualTypeOf>(); + expectTypeOf(Drafts).toEqualTypeOf>(); + expect([Notes.keyed, Drafts.keyed]).toEqual([false, true]); + }); + + it('takes a restricted name only: lowercase [a-z0-9@/._~-] (decision "Restricted names")', () => { + const options = { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }; + for (const name of ["@app/notes.v_1~x-y", "n"]) expect(defineState(name, options).name).toBe(name); + for (const name of ["", "Notes", "app:notes", "app notes", "a\0b", "a\uD800", "é"]) { + expect(() => defineState(name, options)).toThrow( + `A state name ${JSON.stringify(name)}: expected lowercase [a-z0-9@/._~-]`, + ); + } + // Plugin code, not type-checked. + expect(() => defineState(42 as never, options)).toThrow("A state name 42"); + }); +}); + +describe("initial() (R8)", () => { + it("is checked against the schema once, the first time it is used", async () => { + const telemetry = await metered(); + const initial = vi.fn(() => ({ n: 0 })); + const Doc = defineState("doc", { schema: obj({ n: int() }), version: 1, initial }); + expect(Doc.initial(telemetry.context)).toEqual({ n: 0 }); + expect(Doc.initial(telemetry.context)).toEqual({ n: 0 }); + expect(initial).toHaveBeenCalledTimes(2); + expect(telemetry.counts()).toEqual({ "chord.data.compilations": 1, "chord.data.validations": 1 }); + }); + + it("refuses a value that does not match, every time until one does", () => { + let value: unknown = { n: "x" }; + const Doc = defineState("doc", { schema: obj({ n: int() }), version: 1, initial: () => value as { n: number } }); + expect(() => Doc.initial(context)).toThrow("doc: initial() does not match the schema"); + expect(() => Doc.initial(context)).toThrow("doc: initial() does not match the schema"); + value = { n: 1 }; + expect(Doc.initial(context)).toEqual({ n: 1 }); + }); +}); + +describe("fingerprints", () => { + it("are computed once from the schema's structure: descriptions ignored, a structural change seen", () => { + const make = (schema: Parameters[1]["schema"]) => + defineState("doc", { schema, version: 1, initial: () => ({ n: 0 }) as never }).fingerprint; + const plain = make(obj({ n: int() })); + expect(plain).toMatch(/^[0-9a-f]{16}$/); + expect(make(obj({ n: { ...int(), description: "a count" } }))).toBe(plain); + expect(make(obj({ n: str() }))).not.toBe(plain); + }); +}); + +describe("migrate (R8)", () => { + it("maps an older value and checks the result against the schema each time it produces one", async () => { + const telemetry = await metered(); + const Doc = defineState("doc", { + schema: obj({ n: int() }), + version: 2, + initial: () => ({ n: 0 }), + migrate: (old, from) => ({ n: (old.count as number) * from }), + }); + expect(Doc.migrate({ count: 3 }, 1, telemetry.context)).toEqual({ n: 3 }); + expect(() => Doc.migrate({ count: "x" }, 1, telemetry.context)).toThrow( + "doc: migrate() from version 1 does not match the schema", + ); + expect(telemetry.counts()["chord.data.validations"]).toBe(2); + }); + + it("refuses an older value without a migrate", () => { + const Doc = defineState("doc", { schema: obj({ n: int() }), version: 2, initial: () => ({ n: 0 }) }); + expect(() => Doc.migrate({ n: 1 }, 1, context)).toThrow("doc: no migrate from version 1 to 2"); + }); +}); diff --git a/packages/chord/test/next/documents/documents.test.ts b/packages/chord/test/next/documents/documents.test.ts new file mode 100644 index 000000000..acf9252da --- /dev/null +++ b/packages/chord/test/next/documents/documents.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState, type KeyedHandle, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { delivered, framesOf, loaded, rejection } from "../support.ts"; + +const List = defineState("list", { schema: obj({ items: arr(str()) }), version: 1, initial: () => ({ items: [] }) }); +const Drafts = defineState("drafts", { + schema: obj({ body: str() }), + version: 1, + initial: () => ({ body: "" }), + keyed: true, +}); +const Owner = local("app.owner", { + version: 1, + params: { id: int() }, + members: { list: obj({ items: arr(str()) }), touch: method() }, +}); + +/** Serves `Owner` and hands out its handles. */ +async function owner() { + const handles: { list?: SingletonHandle<{ items: string[] }>; drafts?: KeyedHandle<{ body: string }> } = {}; + const { host } = await loaded((h) => + h.serve(Owner, (_, self) => { + handles.list = self.state(List); + handles.drafts = self.state(Drafts); + return { list: handles.list, touch: async () => null }; + }), + ); + const reference = await host.get(Owner, { id: 1 }, context); + return { host, reference, list: handles.list!, drafts: handles.drafts! }; +} + +/** The keys a keyed handle lists. */ +async function keysOf(handle: KeyedHandle<{ body: string }>): Promise { + const keys: string[] = []; + for await (const key of handle.keys(context)) keys.push(key); + return keys; +} + +describe("singleton documents", () => { + it("read initial() until written and again after a retire, whose frame replaces the root", async () => { + const { list } = await owner(); + const initial = list.value; + expect(initial).toEqual({ items: [] }); + const frames = framesOf(await list.watch(context)); + await list.change((draft) => draft.items.push("a"), context); + expect(list.value).toEqual({ items: ["a"] }); + await list.retire(context); + expect(list.value).toBe(initial); + await list.retire(context); + await delivered(); + expect(frames).toEqual([ + [{ items: ["a"] }, [["p", ["items"], 0, 0, ["a"]]]], + [initial, [["r", initial]]], + ]); + }); + + it("throw not_ready when read in the factory", async () => { + const { host } = await loaded((h) => + h.serve(Owner, (_, self) => { + const list = self.state(List); + void list.value; + return { list, touch: async () => null }; + }), + ); + await expect(host.get(Owner, { id: 1 }, context)).rejects.toThrow("list: read before activation"); + }); +}); + +describe("keyed documents", () => { + it("are absent until written; get, keys and retire by key", async () => { + const { drafts } = await owner(); + expect(await drafts.get("a", context)).toBeUndefined(); + await drafts.change( + "a", + (draft) => { + draft.body = "x"; + }, + context, + ); + await drafts.change( + "b", + (draft) => { + draft.body = "y"; + }, + context, + ); + await drafts.change("c", () => {}, context); + expect(await drafts.get("a", context)).toEqual({ body: "x" }); + expect(await keysOf(drafts)).toEqual(["a", "b"]); + await drafts.retire("a", context); + await drafts.retire("missing", context); + expect(await drafts.get("a", context)).toBeUndefined(); + expect(await keysOf(drafts)).toEqual(["b"]); + }); +}); + +describe("when the owner ends", () => { + it("documents are dropped: every use throws released and watches end", async () => { + const { reference, list, drafts } = await owner(); + const watch = await list.watch(context); + reference.release(); + expect(await watch.closed).toEqual({ kind: "ended" }); + const released = { code: "released", reason: "ended", message: "app.owner(1): ended" }; + expect(() => list.value).toThrow("app.owner(1): ended"); + expect(await rejection(drafts.get("a", context))).toEqual(released); + expect(await rejection(keysOf(drafts))).toEqual(released); + expect(await rejection(list.change(() => {}, context))).toEqual(released); + }); +}); diff --git a/packages/chord/test/next/documents/fence.test.ts b/packages/chord/test/next/documents/fence.test.ts new file mode 100644 index 000000000..3bb649e38 --- /dev/null +++ b/packages/chord/test/next/documents/fence.test.ts @@ -0,0 +1,14 @@ +// Seal tokens (slice 6a §3, decision "The seal fence"): open until sealed, sealed for good. + +import { describe, expect, it } from "vitest"; +import { SealToken } from "../../../src/next/documents/internal.ts"; + +describe("seal tokens", () => { + it("start open and stay sealed once sealed", () => { + const token = new SealToken(); + expect(token.sealed).toBe(false); + token.seal(); + token.seal(); + expect(token.sealed).toBe(true); + }); +}); diff --git a/packages/chord/test/next/documents/handles.test.ts b/packages/chord/test/next/documents/handles.test.ts new file mode 100644 index 000000000..a574cb51a --- /dev/null +++ b/packages/chord/test/next/documents/handles.test.ts @@ -0,0 +1,75 @@ +// Document handles (slice 6a §4): keyed handles' reads, watches and `at(key)`, and sealed handles refusing every use. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState, type KeyedHandle, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { instancesOf, rejection, restartable } from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +const Body = defineState("body", { schema: obj({ text: str() }), version: 1, initial: () => ({ text: "" }) }); +const Notes = defineState("notes", { + schema: obj({ text: str() }), + version: 1, + initial: () => ({ text: "" }), + keyed: true, +}); + +/** The keys a keyed handle lists. */ +async function keysOf(handle: KeyedHandle<{ text: string }>): Promise { + const keys: string[] = []; + for await (const key of handle.keys(context)) keys.push(key); + return keys; +} + +describe("keyed handles", () => { + it("list keys in code point order, not UTF-16 order", async () => { + let notes: KeyedHandle<{ text: string }> | undefined; + await restartable(objects, (_, self) => { + notes = self.state(Notes); + }); + // U+FF5E sorts before U+1F600 by code point; as UTF-16 its unit 0xFF5E sorts after the surrogate 0xD83D. + for (const key of ["\u{1F600}", "\uFF5E", "b", "a"]) + await notes!.change( + key, + (d) => { + d.text = key; + }, + context, + ); + expect(await keysOf(notes!)).toEqual(["a", "b", "\uFF5E", "\u{1F600}"]); + expect(await notes!.get("\uFF5E", context)).toEqual({ text: "\uFF5E" }); + }); +}); + +describe("sealed handles", () => { + it("still read synchronously and from memory; changes and retirements are refused released", async () => { + let body: SingletonHandle<{ text: string }> | undefined; + let notes: KeyedHandle<{ text: string }> | undefined; + const { fail } = await restartable(objects, (_, self) => { + body = self.state(Body); + notes = self.state(Notes); + }); + await body!.change((d) => { + d.text = "x"; + }, context); + await notes!.change( + "k", + (d) => { + d.text = "y"; + }, + context, + ); + fail(); + expect(body!.value).toEqual({ text: "x" }); + expect(await notes!.get("k", context)).toEqual({ text: "y" }); + expect(await keysOf(notes!)).toEqual(["k"]); + const released = { code: "released", reason: "ended", message: "app.restartable(1): ended" }; + expect(await rejection(body!.change(() => {}, context))).toEqual(released); + expect(await rejection(body!.retire(context))).toEqual(released); + expect(await rejection(notes!.change("k", () => {}, context))).toEqual(released); + expect(await rejection(notes!.retire("k", context))).toEqual(released); + }); +}); diff --git a/packages/chord/test/next/documents/keyed-slot.test.ts b/packages/chord/test/next/documents/keyed-slot.test.ts new file mode 100644 index 000000000..674ff1e85 --- /dev/null +++ b/packages/chord/test/next/documents/keyed-slot.test.ts @@ -0,0 +1,704 @@ +// Keyed slots (slice 6a §4, §10, §11): keyed reads with a context, per-key watches and `at(key)` sources with their +// pins, persisted keyed documents loaded on demand on the object's line (one load per key, shared), and their +// eviction. + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { keyed, local, obj, str } from "../../../src/next/contracts/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import { defineState, type KeyedHandle, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { RETENTION_MS } from "../../../src/next/documents/residency.ts"; +import type { StoreSelf } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import type { Watchable, WatchHandle } from "../../../src/watch.ts"; +import { + changesOf, + delivered, + expectCollected, + failLive, + fakeTimers, + flushed, + holdNextCommit, + instancesOf, + loaded, + memoryStore, + rejection, + restartable, + storeRestartable, +} from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +type Note = { title: string }; + +const Memo = defineState("notes.memo", { + schema: obj({ title: str() }), + version: 1, + initial: () => ({ title: "" }), + keyed: true, +}); + +const Note = defineState("notes.note", { + schema: obj({ title: str() }), + version: 1, + initial: () => ({ title: "" }), + keyed: true, + persist: true, + checkpointWhen: () => false, +}); + +const stores: ChordStore[] = []; + +afterEach(async () => { + for (const store of stores.splice(0)) await store.close(context); +}); + +/** A fresh `:memory:` store, closed after the test. */ +async function freshStore(): Promise { + const store = await memoryStore({ outbox: true }); + stores.push(store); + return store; +} + +/** + * `StoreRestartable` over `store` declaring `Note` (and what `declare` adds) per attempt; `notes()` is the latest + * attempt's handle. + */ +async function persisted( + store: ChordStore, + declare: (attempt: number, self: StoreSelf, notes: KeyedHandle) => void = () => {}, +): Promise> & { readonly notes: () => KeyedHandle }> { + let notes: KeyedHandle | undefined; + const restartable = await storeRestartable( + objects, + (attempt, self) => { + notes = self.state(Note); + declare(attempt, self, notes); + }, + { store }, + ); + return { ...restartable, notes: () => notes! }; +} + +/** Writes `title` at `key` in `store`'s `Note` documents through a host of its own. */ +async function seed(store: ChordStore, entries: Record): Promise { + const { host, notes } = await persisted(store); + for (const [key, title] of Object.entries(entries)) { + await notes().change( + key, + (draft) => { + draft.title = title; + }, + context, + ); + } + await host.close(context); +} + +describe("keyed reads in memory", () => { + it("get and keys take a context; keys come in code point order", async () => { + let memos: KeyedHandle | undefined; + const { host } = await restartable(objects, (_, self) => { + memos = self.state(Memo); + }); + for (const key of ["b", "\u{1F600}", "a", "\uFFFF"]) + await memos!.change( + key, + (d) => { + d.title = key; + }, + context, + ); + expect(await memos!.get("a", context)).toEqual({ title: "a" }); + expect(await memos!.get("z", context)).toBeUndefined(); + const keys: string[] = []; + for await (const key of memos!.keys(context)) keys.push(key); + expect(keys).toEqual(["a", "b", "\uFFFF", "\u{1F600}"]); + await host.close(context); + }); +}); + +describe("migration on access", () => { + it("a newer version migrates each in-memory key when it is read; a key whose migrate fails fails the owner then", async () => { + const migrated: string[] = []; + const MemoV2 = defineState("notes.memo", { + schema: obj({ title: str() }), + version: 2, + initial: () => ({ title: "" }), + keyed: true, + migrate: (old) => { + const title = old.title as string; + if (title === "bad") throw new Error("cannot migrate"); + migrated.push(title); + return { title: `${title}!` }; + }, + }); + const memos: KeyedHandle[] = []; + const { attempts, restart, errors, reference } = await restartable(objects, (attempt, self) => { + memos.push(self.state(attempt === 1 ? Memo : MemoV2)); + }); + for (const key of ["a", "b", "c"]) { + await memos[0]!.change( + key, + (draft) => { + draft.title = key === "c" ? "bad" : key; + }, + context, + ); + } + await restart(); + // The activation migrated nothing: no key was read. + expect(migrated).toEqual([]); + expect(await memos[1]!.get("a", context)).toEqual({ title: "a!" }); + expect(migrated).toEqual(["a"]); + await attempts[1]!.tx(async (tx) => { + expect(await tx.draft(memos[1]!, "b")).toEqual({ title: "b!" }); + }, context); + expect(migrated).toEqual(["a", "b"]); + expect(await rejection(memos[1]!.get("c", context))).toMatchObject({ + message: expect.stringContaining("cannot migrate"), + }); + expect(errors.map(({ kind }) => kind)).toEqual(["object_failed", "object_failed"]); + expect(reference.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + }); +}); + +describe("keys and the text rules", () => { + /** Every key of `handle`, in order. */ + async function keysOf(handle: KeyedHandle): Promise { + const keys: string[] = []; + for await (const key of handle.keys(context)) keys.push(key); + return keys; + } + + const title = (value: string) => (draft: Note) => { + draft.title = value; + }; + + it("a lone surrogate in a key is U+FFFD at every entry point, in memory and in the store; NUL is refused", async () => { + const store = await freshStore(); + const { host, notes, attempts } = await persisted(store); + await notes().change("x\uD800", title("one"), context); + expect(await keysOf(notes())).toEqual(["x\uFFFD"]); + // One key: the change through either spelling reaches the same document and record. + await notes().change("x\uFFFD", title("two"), context); + await attempts[0]!.tx(async (tx) => { + (await tx.draft(notes(), "x\uD800")).title += "!"; + }, context); + expect(await notes().get("x\uD800", context)).toEqual({ title: "two!" }); + expect((await notes().watch("x\uD800", context))?.value).toEqual({ title: "two!" }); + for (const use of [ + () => notes().change("a\0b", title("nul"), context), + () => notes().get("a\0b", context), + () => notes().watch("a\0b", context), + () => notes().retire("a\0b", context), + ]) { + expect(await rejection(use())).toMatchObject({ code: "invalid", reason: "args" }); + } + await notes().retire("x\uD800", context); + expect(await keysOf(notes())).toEqual([]); + await host.close(context); + // The store has the key as memory had it. + const again = await persisted(store); + expect(await again.notes().get("x\uFFFD", context)).toBeUndefined(); + await again.host.close(context); + }); + + it("in memory too: one key per spelling, NUL refused", async () => { + let memos: KeyedHandle | undefined; + const { host } = await restartable(objects, (_, self) => { + memos = self.state(Memo); + }); + await memos!.change("x\uDC00", title("one"), context); + expect(await memos!.get("x\uFFFD", context)).toEqual({ title: "one" }); + expect(await keysOf(memos!)).toEqual(["x\uFFFD"]); + expect(await rejection(memos!.change("\0", title("nul"), context))).toMatchObject({ code: "invalid" }); + await host.close(context); + }); + + it("the empty key is a key: listed first, also on a later page", async () => { + const store = await freshStore(); + const entries: Record = { "": "empty" }; + for (let i = 0; i < 120; i++) entries[`k${String(i).padStart(3, "0")}`] = "x"; + await seed(store, entries); + const { host, notes } = await persisted(store); + const keys = await keysOf(notes()); + expect(keys.length).toBe(121); + expect(keys[0]).toBe(""); + expect(await notes().get("", context)).toEqual({ title: "empty" }); + await host.close(context); + }); +}); + +describe("keyed watches", () => { + it("watch(key) is undefined when absent; it follows the address through a retirement and a recreation", async () => { + let memos: KeyedHandle | undefined; + const { host, attempts } = await restartable(objects, (_, self) => { + memos = self.state(Memo); + }); + expect(await memos!.watch("a", context)).toBeUndefined(); + await memos!.change( + "a", + (d) => { + d.title = "x"; + }, + context, + ); + const watch = (await memos!.watch("a", context))!; + expect(watch.value).toEqual({ title: "x" }); + const changes = changesOf(watch); + await memos!.change( + "a", + (d) => { + d.title = "y"; + }, + context, + ); + await delivered(); + await memos!.retire("a", context); + await delivered(); + await memos!.change( + "a", + (d) => { + d.title = "z"; + }, + context, + ); + await delivered(); + await attempts[0]!.tx(async (tx) => { + await tx.retire(memos!, "a"); + (await tx.draft(memos!, "a")).title = "w"; + }, context); + await delivered(); + expect(changes).toEqual([ + { kind: "value", value: { title: "y" }, ops: [["s", ["title"], "y"]], retired: false }, + { kind: "value", value: { title: "" }, ops: [["r", { title: "" }]], retired: true }, + { kind: "value", value: { title: "z" }, ops: [["r", { title: "z" }]], retired: false }, + { kind: "value", value: { title: "w" }, ops: [["r", { title: "w" }]], retired: true }, + ]); + await host.close(context); + expect(await watch.closed).toEqual({ kind: "ended" }); + }); + + it("a stopped watch unpins its key: its source is collected", async () => { + let memos: KeyedHandle | undefined; + const { host } = await restartable(objects, (_, self) => { + memos = self.state(Memo); + }); + await memos!.change( + "a", + (d) => { + d.title = "x"; + }, + context, + ); + let watch: WatchHandle | undefined = (await memos!.watch("a", context))!; + // The watch's source: the slot's per-key source while pinned. + const slot = (memos as unknown as { slot: { source(key: string): object } }).slot; + const source = new WeakRef(slot.source("a")); + await watch.stop(); + watch = undefined; + await flushed(); + await expectCollected(source); + await host.close(context); + }); +}); + +describe("at(key)", () => { + it("is initial() while absent, a state member, derive and on source; retirement arrives with the flag", async () => { + let memos: KeyedHandle | undefined; + let pinned: Watchable | undefined; + let upper: Watchable | undefined; + const seen: { title: string; retired: boolean }[] = []; + const { host } = await restartable(objects, (_, self) => { + memos = self.state(Memo); + pinned = memos.at("a"); + upper = self.derive([pinned], ([note]) => note.title.toUpperCase()); + self.on(pinned, async ({ value, retired }) => void seen.push({ title: value.title, retired })); + }); + expect(pinned!.value).toEqual({ title: "" }); + const changes = changesOf(await upper!.watch(context)); + await memos!.change( + "a", + (d) => { + d.title = "x"; + }, + context, + ); + await delivered(); + expect(pinned!.value).toEqual({ title: "x" }); + await memos!.retire("a", context); + await delivered(); + await flushed(); + expect(changes).toEqual([ + { kind: "value", value: "X", ops: [["r", "X"]], retired: false }, + { kind: "value", value: "", ops: [["r", ""]], retired: true }, + ]); + expect(seen).toEqual([ + { title: "", retired: false }, + { title: "x", retired: false }, + { title: "", retired: true }, + ]); + await host.close(context); + }); + + it("only in the factory", async () => { + let memos: KeyedHandle | undefined; + const { host } = await restartable(objects, (_, self) => { + memos = self.state(Memo); + }); + expect(() => memos!.at("a")).toThrow("valid only in the factory"); + await host.close(context); + }); + + it("a persisted key is loaded before the object goes live and stays pinned until the next factory returns", async () => { + const store = await freshStore(); + await seed(store, { a: "x", b: "y" }); + let pinned: Watchable | undefined; + const { host, restart, notes } = await persisted(store, (attempt, _, notes) => { + pinned = notes.at(attempt === 1 ? "a" : "b"); + }); + expect(pinned!.value).toEqual({ title: "x" }); + const slot = (notes() as unknown as { slot: { source(key: string): object | undefined } }).slot; + const first = new WeakRef(slot.source("a")!); + await restart(); + expect(pinned!.value).toEqual({ title: "y" }); + expect(slot.source("a")).toBeUndefined(); + await expectCollected(first); + await host.close(context); + }); +}); + +describe("cold keyed loads on the line", () => { + it("load a cold key once; a cached get touches no SQLite; an absent key is not cached", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const { host, notes } = await persisted(store); + const read = vi.spyOn(store, "read"); + expect(await notes().get("a", context)).toEqual({ title: "x" }); + expect(read).toHaveBeenCalledTimes(1); + expect(await notes().get("a", context)).toEqual({ title: "x" }); + expect(read).toHaveBeenCalledTimes(1); + expect(await notes().get("z", context)).toBeUndefined(); + expect(await notes().get("z", context)).toBeUndefined(); + expect(read).toHaveBeenCalledTimes(3); + await host.close(context); + }); + + it("two readers share one load; a cancelled waiter rejects alone", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const { host, notes, attempts } = await persisted(store); + const read = vi.spyOn(store, "read"); + // Hold the line so that the load waits behind it. + const { promise: holding, resolve: release } = Promise.withResolvers(); + const tx = attempts[0]!.tx(() => holding, context); + const cancel = new AbortController(); + const first = notes().get("a", { ...context, abortSignal: cancel.signal }); + const second = notes().get("a", context); + cancel.abort(new Error("gone")); + await expect(first).rejects.toThrow("gone"); + release(); + await tx; + expect(await second).toEqual({ title: "x" }); + expect(read).toHaveBeenCalledTimes(1); + await host.close(context); + }); + + it("a cold get racing a draft of the same key: the load installs only while the key is absent", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const { host, notes, attempts } = await persisted(store); + let get: Promise | undefined; + await attempts[0]!.tx(async (tx) => { + get = notes().get("a", context); + const draft = await tx.draft(notes(), "a"); + expect(draft.title).toBe("x"); + draft.title = "y"; + }, context); + expect(await get).toEqual({ title: "y" }); + expect(await notes().get("a", context)).toEqual({ title: "y" }); + await host.close(context); + }); + + it("keys come from the store in code point order, a page per line job", async () => { + const store = await freshStore(); + const entries: Record = {}; + for (let i = 0; i < 150; i++) entries[`k${String(i).padStart(3, "0")}`] = "x"; + entries["\u{1F600}"] = "x"; + entries["\uFFFF"] = "x"; + await seed(store, entries); + const { host, notes } = await persisted(store); + const keys: string[] = []; + for await (const key of notes().keys(context)) keys.push(key); + expect(keys).toEqual([...Object.keys(entries).filter((key) => key.startsWith("k")), "\uFFFF", "\u{1F600}"]); + await host.close(context); + }); + + it("tx.retire of a cold key finds it without loading it; an absent key retires nothing", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const { host, notes } = await persisted(store); + const seq = store.lastSeq; + await notes().retire("z", context); + expect(store.lastSeq).toBe(seq); + await notes().retire("a", context); + expect(store.lastSeq).toBe(seq + 1); + expect(await notes().get("a", context)).toBeUndefined(); + await host.close(context); + }); + + it("a sealed handle reads cached documents; a cold read is refused released", async () => { + const store = await freshStore(); + await seed(store, { a: "x", b: "y" }); + const { host, notes, restart } = await persisted(store); + const old = notes(); + await old.get("a", context); + await restart(); + expect(await old.get("a", context)).toEqual({ title: "x" }); + expect(await rejection(old.get("b", context))).toMatchObject({ code: "released" }); + await host.close(context); + }); + + it("a sealed handle joining a cold load in flight is refused released (rule 4)", async () => { + const store = await freshStore(); + await seed(store, { b: "y" }); + const { host, notes, restart } = await persisted(store); + const gate = holdNextRead(store); + const old = notes(); + const first = old.get("b", context); + await gate.reached; + const restarting = restart(); + await delivered(); + expect(await rejection(old.get("b", context))).toMatchObject({ code: "released" }); + gate.open(); + expect(await first).toEqual({ title: "y" }); + await restarting; + await host.close(context); + }); + + it("cold reads queued on the line before the object's end are refused released when they run after it", async () => { + const store = await freshStore(); + await seed(store, { a: "x", b: "y", c: "z" }); + const { host, notes } = await persisted(store); + const gate = holdNextRead(store); + const first = notes().get("a", context); + await gate.reached; + const get = rejection(notes().get("b", context)); + const watch = rejection(notes().watch("c", context)); + const closing = host.close(context); + gate.open(); + expect(await get).toMatchObject({ code: "released" }); + expect(await watch).toMatchObject({ code: "released" }); + // The load holding the line before the end finishes. + expect(await first).toEqual({ title: "x" }); + await closing; + }); + + it("cold reads queued behind an admitted step are refused released when their attempt failed live meanwhile", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const P = defineState("notes.p", { + schema: obj({ title: str() }), + version: 1, + initial: () => ({ title: "" }), + persist: true, + checkpointWhen: () => false, + }); + let p: SingletonHandle | undefined; + const { host, notes, attempts } = await persisted(store, (_, self) => { + p = self.state(P); + }); + const held = holdNextCommit(store); + const change = p!.change((draft) => { + draft.title = "p"; + }, context); + await held.entered; + const get = rejection(notes().get("a", context)); + const watch = rejection(notes().watch("a", context)); + const keys = rejection( + (async () => { + for await (const _ of notes().keys(context)); + })(), + ); + // The attempt is sealed; the object stays open. + failLive(objects, attempts[0]!, new ChordError({ code: "unavailable", message: "boom" })); + held.release(); + await change; + expect(await get).toMatchObject({ code: "released" }); + expect(await watch).toMatchObject({ code: "released" }); + expect(await keys).toMatchObject({ code: "released" }); + await host.close(context); + }); + + it("a still-valid waiter sharing that load gets the document: only the sealed handle's read is refused", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const Notes = local("app.store", { version: 1, params: {}, members: { notes: keyed(obj({ title: str() })) } }); + let self: StoreSelf | undefined; + let notes: KeyedHandle | undefined; + const { host } = await loaded( + (h) => + h.serve(Notes, (_, s) => { + self = s; + notes = s.state(Note); + return { notes }; + }), + { store }, + ); + const ref = await host.get(Notes, {}, context); + const held = holdNextCommit(store); + const change = notes!.change( + "b", + (draft) => { + draft.title = "y"; + }, + context, + ); + await held.entered; + const sealed = rejection(notes!.get("a", context)); + // A consumer reads as the provider's own code may (the object's own token), joining the same load. + const shared = ref.notes.get("a", context); + await flushed(); + failLive(objects, self!, new ChordError({ code: "unavailable", message: "boom" })); + held.release(); + await change; + expect(await sealed).toMatchObject({ code: "released" }); + expect(await shared).toEqual({ title: "x" }); + await host.close(context); + }); + + it("a load cut by the seal never installs: a successor's retirement stands", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const { host, notes, attempts, restart } = await persisted(store); + const gate = holdNextRead(store); + const old = notes(); + const cut = attempts[0]!.tx(async (tx) => { + (await tx.draft(old, "a")).title = "z"; + }, context); + await gate.reached; + // The seal cuts the transaction and releases the line; the successor retires the key. + await restart(); + await notes().retire("a", context); + gate.open(); + expect(await rejection(cut)).toMatchObject({ code: "released" }); + expect(await notes().get("a", context)).toBeUndefined(); + await host.close(context); + }); + + it("a plugin migrate throwing a ChordError fails its owner like any other error", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const V2 = defineState("notes.note", { + schema: obj({ title: str() }), + version: 2, + initial: () => ({ title: "" }), + migrate: () => { + throw new ChordError({ code: "unavailable", message: "plugin says no" }); + }, + keyed: true, + persist: true, + checkpointWhen: () => false, + }); + let notes: KeyedHandle | undefined; + const { host, errors, reference } = await storeRestartable( + objects, + (_, self) => { + notes = self.state(V2); + }, + { store }, + ); + expect(await rejection(notes!.get("a", context))).toMatchObject({ message: "plugin says no" }); + expect(errors.map(({ kind }) => kind)).toEqual(["object_failed"]); + expect(reference.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + await host.close(context); + }); + + it("a key whose migrate fails fails its owner broken; the get rejects with the cause", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const V2 = defineState("notes.note", { + schema: obj({ title: str() }), + version: 2, + initial: () => ({ title: "" }), + migrate: () => { + throw new Error("cannot migrate"); + }, + keyed: true, + persist: true, + checkpointWhen: () => false, + }); + let notes: KeyedHandle | undefined; + const { host, errors, reference } = await storeRestartable( + objects, + (_, self) => { + notes = self.state(V2); + }, + { store }, + ); + expect(await rejection(notes!.get("a", context))).toMatchObject({ + code: "unavailable", + message: expect.stringContaining("cannot migrate"), + }); + expect(errors.map(({ kind }) => kind)).toEqual(["object_failed"]); + expect(reference.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + await host.close(context); + }); +}); + +describe("eviction", () => { + fakeTimers(); + + it("an unpinned key idle for the retention leaves memory; a pinned one stays", async () => { + const store = await freshStore(); + await seed(store, { a: "x", b: "y" }); + const { host, notes } = await persisted(store); + const read = vi.spyOn(store, "read"); + await notes().get("a", context); + const watch = (await notes().watch("b", context))!; + expect(read).toHaveBeenCalledTimes(2); + vi.advanceTimersByTime(RETENTION_MS + 1_000); + await flushed(); + await notes().get("a", context); + await notes().get("b", context); + expect(read).toHaveBeenCalledTimes(3); + await watch.stop(); + await flushed(); + vi.advanceTimersByTime(RETENTION_MS + 1_000); + await flushed(); + await notes().get("b", context); + expect(read).toHaveBeenCalledTimes(4); + await host.close(context); + }); + + it("an evicted document is collected", async () => { + const store = await freshStore(); + await seed(store, { a: "x" }); + const { host, notes } = await persisted(store); + let value: Note | undefined = await notes().get("a", context); + const collected = new WeakRef(value!); + value = undefined; + vi.advanceTimersByTime(RETENTION_MS + 1_000); + await flushed(); + await expectCollected(collected); + expect(await notes().get("a", context)).toEqual({ title: "x" }); + await host.close(context); + }); +}); + +/** Holds `store`'s next read after the backend returned its result, until `open()`. */ +function holdNextRead(store: ChordStore): { readonly reached: Promise; open(): void } { + const reached = Promise.withResolvers(); + const gate = Promise.withResolvers(); + const read = store.read.bind(store); + store.read = async (fn, ctx) => { + store.read = read; + const result = await read(fn, ctx); + reached.resolve(); + await gate.promise; + return result; + }; + return { reached: reached.promise, open: () => gate.resolve() }; +} diff --git a/packages/chord/test/next/documents/line.test.ts b/packages/chord/test/next/documents/line.test.ts new file mode 100644 index 000000000..b473cc378 --- /dev/null +++ b/packages/chord/test/next/documents/line.test.ts @@ -0,0 +1,45 @@ +// The line (§9): holders run one at a time in the order they asked; a release given twice changes nothing. + +import { describe, expect, it } from "vitest"; +import { Line } from "../../../src/next/documents/line.ts"; +import { flushed } from "../support.ts"; + +describe("the line", () => { + it("hands the line to each holder in the order they asked, once the previous one released", async () => { + const line = new Line(); + const order: string[] = []; + const first = await line.acquire(); + const second = line.acquire().then((release) => { + order.push("second"); + return release; + }); + const third = line.acquire().then((release) => { + order.push("third"); + return release; + }); + await flushed(); + expect(order).toEqual([]); + first(); + (await second)(); + await third; + expect(order).toEqual(["second", "third"]); + }); + + it("ignores a second release of the same holder", async () => { + const line = new Line(); + const first = await line.acquire(); + const second = line.acquire(); + let thirdHeld = false; + const third = line.acquire().then(() => { + thirdHeld = true; + }); + first(); + first(); + await second; + await flushed(); + expect(thirdHeld).toBe(false); + (await second)(); + await third; + expect(thirdHeld).toBe(true); + }); +}); diff --git a/packages/chord/test/next/documents/persistence.test.ts b/packages/chord/test/next/documents/persistence.test.ts new file mode 100644 index 000000000..6b97b83a2 --- /dev/null +++ b/packages/chord/test/next/documents/persistence.test.ts @@ -0,0 +1,292 @@ +// Persisted documents' store side (slice 6a §8, §10): loads with the version and fingerprint checks and `migrate`, +// finds, keys, and the writes one commit stages (creation, base or delta by `checkpointWhen`, retirement). + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { track } from "../../../src/delta/index.ts"; +import { int, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState, type JsonObject } from "../../../src/next/documents/index.ts"; +import { Persistence, type Stored, storedAfter } from "../../../src/next/documents/persistence.ts"; +import type { AnyDefinition } from "../../../src/next/documents/slot.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { memoryStore, metered } from "../support.ts"; + +const context = BACKGROUND_CONTEXT; + +const Note = defineState("notes.note", { + schema: obj({ title: str(), n: int() }), + version: 1, + initial: () => ({ title: "", n: 0 }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 2, +}); + +let store: ChordStore; +let notes: Persistence; + +beforeEach(async () => { + store = await memoryStore({ outbox: true }); + notes = new Persistence(store, "session:app.store", "notes.note"); +}); + +afterEach(async () => { + await store.close(context); +}); + +/** Stages the change of `before` to `after` at `key` and commits it; resolves with what is stored afterwards. */ +async function change( + key: string, + before: JsonObject | undefined, + after: (draft: JsonObject) => void, + stored: Stored | undefined, + definition: AnyDefinition = Note, +): Promise { + const tracker = track(before ?? definition.initial(context)); + const draft = tracker.beginChange(); + after(draft.state); + const prepared = draft.prepare(); + const writes: StagedWrite[] = []; + const result = notes.stage(key, definition, stored, { prepared, retires: false }, writes, 0); + let ids: readonly number[] = []; + await store.commit( + writes, + { + admit: () => true, + adopt: (_seq, created) => { + ids = created; + }, + }, + context, + ); + tracker.adopt(prepared); + return storedAfter(result, ids); +} + +/** The document at `key` read from the store and decoded by `definition`, `undefined` when absent. */ +async function load(key: string, definition: AnyDefinition, at: typeof context) { + const document = await notes.read(key, at); + return document === undefined ? undefined : notes.decode(document, definition, at); +} + +/** The kinds of the outbox's writes, per commit. */ +async function outboxKinds(): Promise { + const commits = await store.outbox!.read(0, 100, context); + return commits.map(({ writes }) => + writes.map((write) => (write.kind === "document.change" ? `change:${write.content.kind}` : write.kind)), + ); +} + +describe("loads", () => { + it("load an absent document as undefined and a stored one with its record", async () => { + expect(await load("a", Note, context)).toBeUndefined(); + const stored = await change( + "a", + undefined, + (d) => { + d.title = "x"; + }, + undefined, + ); + const { counts, context: counting } = await metered(); + expect(await load("a", Note, counting)).toEqual({ + value: { title: "x", n: 0 }, + stored: { id: stored!.id, revision: 1, version: 1, deltasSinceBase: 0 }, + }); + expect(counts()["chord.store.loads"]).toBe(1); + }); + + it("migrate an older stored version and check the result once", async () => { + const V1 = defineState("notes.note", { + schema: obj({ title: str() }), + version: 1, + initial: () => ({ title: "" }), + persist: true, + checkpointWhen: () => false, + }); + await change( + "a", + undefined, + (d) => { + d.title = "x"; + }, + undefined, + V1, + ); + const V2 = defineState("notes.note", { + schema: obj({ title: str(), n: int() }), + version: 2, + initial: () => ({ title: "", n: 0 }), + migrate: (old) => ({ title: String(old.title), n: 7 }), + persist: true, + checkpointWhen: () => false, + }); + const { counts, context: counting } = await metered(); + expect(await load("a", V2, counting)).toMatchObject({ + value: { title: "x", n: 7 }, + stored: { version: 1 }, + }); + expect(counts()["chord.data.validations"]).toBe(1); + const Bad = defineState("notes.note", { + schema: obj({ title: str(), n: int() }), + version: 2, + initial: () => ({ title: "", n: 0 }), + // A migrate returning a wrong shape: the load is refused. + migrate: (old) => ({ title: String(old.title) }) as { title: string; n: number }, + persist: true, + checkpointWhen: () => false, + }); + await expect(load("a", Bad, context)).rejects.toThrow("migrate() from version 1 does not match"); + }); + + it("refuse a stored version newer than the code's and the same version with another fingerprint (D9)", async () => { + await change( + "a", + undefined, + (d) => { + d.title = "x"; + }, + undefined, + ); + const Old = defineState("notes.note", { + schema: obj({ title: str(), n: int() }), + version: 0, + initial: () => ({ title: "", n: 0 }), + persist: true, + checkpointWhen: () => false, + }); + await expect(load("a", Old, context)).rejects.toThrow("stored at version 1, newer than the code's 0"); + const Edited = defineState("notes.note", { + schema: obj({ title: str(), n: int(), extra: str() }), + version: 1, + initial: () => ({ title: "", n: 0, extra: "" }), + persist: true, + checkpointWhen: () => false, + }); + await expect(load("a", Edited, context)).rejects.toThrow("stored with another schema at version 1"); + }); + + it("find a live record without loading it, and list keys in code point order by pages", async () => { + expect(await notes.find("a", context)).toBeUndefined(); + for (const key of ["b", "\u{1F600}", "a", "\uFFFF"]) + await change( + key, + undefined, + (d) => { + d.n = 1; + }, + undefined, + ); + expect(await notes.find("a", context)).toMatchObject({ revision: 1, version: 1 }); + expect(await notes.keys(undefined, 10, context)).toEqual(["a", "b", "\uFFFF", "\u{1F600}"]); + expect(await notes.keys("b", 1, context)).toEqual(["\uFFFF"]); + }); +}); + +describe("staged writes", () => { + it("store a creation as a base, changes as deltas until checkpointWhen asks for a base", async () => { + let stored = await change( + "a", + undefined, + (d) => { + d.n = 1; + }, + undefined, + ); + expect(stored).toMatchObject({ revision: 1, deltasSinceBase: 0 }); + let value = (await load("a", Note, context))!.value; + for (const n of [2, 3, 4]) { + stored = await change( + "a", + value, + (d) => { + d.n = n; + }, + stored, + ); + value = (await load("a", Note, context))!.value; + } + expect(stored).toMatchObject({ revision: 4, deltasSinceBase: 0 }); + expect(value).toEqual({ title: "", n: 4 }); + expect(await outboxKinds()).toEqual([["document.create"], ["change:delta"], ["change:delta"], ["change:base"]]); + expect((await load("a", Note, context))!.stored.deltasSinceBase).toBe(0); + }); + + it("store a base when the stored version is older than the definition's (the migrated base written back)", async () => { + const stored = await change( + "a", + undefined, + (d) => { + d.n = 1; + }, + undefined, + ); + const V2 = defineState("notes.note", { + schema: obj({ title: str(), n: int() }), + version: 2, + initial: () => ({ title: "", n: 0 }), + migrate: (old) => old as { title: string; n: number }, + persist: true, + checkpointWhen: () => false, + }); + const after = await change( + "a", + { title: "", n: 1 }, + (d) => { + d.n = 2; + }, + stored, + V2, + ); + expect(after).toMatchObject({ version: 2, revision: 2, deltasSinceBase: 0 }); + expect((await outboxKinds()).at(-1)).toEqual(["change:base"]); + expect(await load("a", V2, context)).toMatchObject({ value: { n: 2 }, stored: { version: 2 } }); + }); + + it("stage a retirement before a recreation, and number creations by their order in the commit", async () => { + const stored = await change( + "a", + undefined, + (d) => { + d.n = 1; + }, + undefined, + ); + const writes: StagedWrite[] = []; + const fresh = (n: number) => { + const tracker = track(Note.initial(context)); + const draft = tracker.beginChange(); + draft.state.n = n; + return draft.prepare(); + }; + const first = notes.stage("b", Note, undefined, { prepared: fresh(2), retires: false }, writes, 0); + const again = notes.stage("a", Note, stored, { prepared: fresh(3), retires: true }, writes, 1); + const gone = notes.stage("c", Note, undefined, { prepared: undefined, retires: true }, writes, 2); + expect([first, again, gone]).toEqual([ + { kind: "create", version: 1, index: 0 }, + { kind: "create", version: 1, index: 1 }, + { kind: "retire" }, + ]); + let ids: readonly number[] = []; + await store.commit( + writes, + { + admit: () => true, + adopt: (_seq, created) => { + ids = created; + }, + }, + context, + ); + expect(ids).toEqual([stored!.id + 1, stored!.id + 2]); + expect(storedAfter(again, ids)).toEqual({ id: stored!.id + 2, revision: 1, version: 1, deltasSinceBase: 0 }); + expect((await outboxKinds()).at(-1)).toEqual(["document.retire", "document.create", "document.create"]); + }); + + it("serialize at staging: a value JSON cannot encode fails there, never in the step (rule 8)", () => { + // Drafts refuse such values when placed; a prepared change made by hand shows where the text is made. + const value = { title: "", n: 1n } as unknown as JsonObject; + const prepared = { base: value, value, ops: [], baseRevision: 0, abort: () => {} }; + expect(() => notes.stage("a", Note, undefined, { prepared, retires: false }, [], 0)).toThrow(TypeError); + }); +}); diff --git a/packages/chord/test/next/documents/residency.test.ts b/packages/chord/test/next/documents/residency.test.ts new file mode 100644 index 000000000..1451cee02 --- /dev/null +++ b/packages/chord/test/next/documents/residency.test.ts @@ -0,0 +1,77 @@ +// Residency (slice 6a §10, rule 23): Durable's idle structure, one timer per object, the granule rule, eviction on +// the line oldest first. + +import { describe, expect, it, vi } from "vitest"; +import { track } from "../../../src/delta/index.ts"; +import { RETENTION_MS, Residency, Resident } from "../../../src/next/documents/residency.ts"; +import { fakeTimers, flushed } from "../support.ts"; + +/** A residency whose sweeps run at once (no line), and the keys it evicted, in order. */ +function idle() { + const evicted: string[] = []; + const slot = { evict: (resident: Resident) => void evicted.push(resident.key) }; + + const residency = new Residency((sweep) => { + sweep(); + }); + const resident = (key: string) => new Resident(slot, key, track({}), undefined, 1); + return { residency, resident, evicted }; +} + +describe("the idle structure", () => { + fakeTimers(); + + it("evict idle documents oldest first once their retention passed, with one timer", async () => { + const { residency, resident, evicted } = idle(); + const a = resident("a"); + const b = resident("b"); + residency.touch(a); + vi.advanceTimersByTime(5_000); + residency.touch(b); + expect(vi.getTimerCount()).toBe(1); + vi.advanceTimersByTime(RETENTION_MS - 5_000); + await flushed(); + expect(evicted).toEqual(["a"]); + expect(vi.getTimerCount()).toBe(1); + vi.advanceTimersByTime(5_000); + expect(evicted).toEqual(["a", "b"]); + expect(vi.getTimerCount()).toBe(0); + vi.advanceTimersByTime(RETENTION_MS); + expect(evicted).toEqual(["a", "b"]); + }); + + it("move a touched document to the back at most once per granule", () => { + const { residency, resident, evicted } = idle(); + const a = resident("a"); + const b = resident("b"); + residency.touch(a); + residency.touch(b); + vi.advanceTimersByTime(500); + residency.touch(a); + vi.advanceTimersByTime(RETENTION_MS - 500); + // Within its granule the touch did not move `a`: it leaves up to a granule early. + expect(evicted).toEqual(["a", "b"]); + residency.touch(a); + residency.touch(b); + vi.advanceTimersByTime(1_000); + residency.touch(a); + vi.advanceTimersByTime(RETENTION_MS - 1_000); + expect(evicted).toEqual(["a", "b", "b"]); + vi.advanceTimersByTime(1_000); + expect(evicted).toEqual(["a", "b", "b", "a"]); + }); + + it("forget pinned documents; after end nothing is evicted and no timer is left", () => { + const { residency, resident, evicted } = idle(); + const a = resident("a"); + residency.touch(a); + residency.forget(a); + vi.advanceTimersByTime(RETENTION_MS); + expect(evicted).toEqual([]); + residency.touch(resident("b")); + residency.end(); + expect(vi.getTimerCount()).toBe(0); + vi.advanceTimersByTime(RETENTION_MS); + expect(evicted).toEqual([]); + }); +}); diff --git a/packages/chord/test/next/documents/slot.test.ts b/packages/chord/test/next/documents/slot.test.ts new file mode 100644 index 000000000..7141fc6ab --- /dev/null +++ b/packages/chord/test/next/documents/slot.test.ts @@ -0,0 +1,375 @@ +// Singleton slots (slice 6a §4, §10, §11): retirement, slots kept across attempts, persisted singletons loaded once. + +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, int, obj, str } from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { + defineState, + type JsonObject, + type KeyedHandle, + type SingletonHandle, +} from "../../../src/next/documents/index.ts"; +import type { HostError, StoreSelf } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { + changesOf, + delivered, + instancesOf, + loaded, + memoryStore, + rejection, + restartable, + StoreRestartable, + storeRestartable, + valuesOf, +} from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +const ListV1 = defineState("list", { schema: obj({ items: arr(str()) }), version: 1, initial: () => ({ items: [] }) }); + +describe("retirement", () => { + it("reaches watches as initial() with retired: true", async () => { + let list: SingletonHandle<{ items: string[] }> | undefined; + await restartable(objects, (_, self) => { + list = self.state(ListV1); + }); + const initial = list!.value; + const changes = changesOf(await list!.watch(context)); + await list!.change((draft) => void draft.items.push("a"), context); + await list!.retire(context); + await delivered(); + expect(changes).toEqual([ + { kind: "value", value: { items: ["a"] }, ops: [["p", ["items"], 0, 0, ["a"]]], retired: false }, + { kind: "value", value: initial, ops: [["r", initial]], retired: true }, + ]); + }); + + it("a retire and a recreate of a present document in one transaction: one frame, retired, the new value", async () => { + let list: SingletonHandle<{ items: string[] }> | undefined; + const { attempts } = await restartable(objects, (_, self) => { + list = self.state(ListV1); + }); + await list!.change((draft) => void draft.items.push("a"), context); + const changes = changesOf(await list!.watch(context)); + await attempts[0]!.tx(async (tx) => { + await tx.retire(list!); + (await tx.draft(list!)).items.push("b"); + }, context); + await delivered(); + expect(changes).toEqual([ + { kind: "value", value: { items: ["b"] }, ops: [["r", { items: ["b"] }]], retired: true }, + ]); + }); +}); + +describe("across attempts", () => { + it("keep in-memory documents across a failure and retry; initial() is taken once", async () => { + let initials = 0; + const List = defineState("list", { + schema: obj({ items: arr(str()) }), + version: 1, + initial: () => { + initials++; + return { items: [] }; + }, + }); + const handles: SingletonHandle<{ items: string[] }>[] = []; + const { restart } = await restartable(objects, (_, self) => { + handles.push(self.state(List)); + }); + const changes = changesOf(await handles[0]!.watch(context)); + await handles[0]!.change((draft) => void draft.items.push("a"), context); + await restart(); + expect(handles[1]!.value).toEqual({ items: ["a"] }); + await handles[1]!.change((draft) => void draft.items.push("b"), context); + await delivered(); + // The watch taken through the first attempt's handle follows the slot across attempts. + expect(valuesOf(changes)).toEqual([{ items: ["a"] }, { items: ["a", "b"] }]); + expect(initials).toBe(1); + }); + + it("drop the in-memory documents a later factory no longer declares: their watches end", async () => { + let list: SingletonHandle<{ items: string[] }> | undefined; + const { restart } = await restartable(objects, (attempt, self) => { + if (attempt === 1) list = self.state(ListV1); + }); + await list!.change((draft) => void draft.items.push("a"), context); + const watch = await list!.watch(context); + await restart(); + expect(await watch.closed).toEqual({ kind: "ended" }); + }); + + it("keep them across a throwing factory: only a factory that returns drops what it does not declare", async () => { + const handles: SingletonHandle<{ items: string[] }>[] = []; + const { fail, retry, errors } = await restartable(objects, (attempt, self) => { + if (attempt === 2) throw new Error("factory threw"); + if (attempt !== 2) handles.push(self.state(ListV1)); + }); + await handles[0]!.change((draft) => void draft.items.push("a"), context); + fail(); + await rejection(retry()); + expect(failure(errors)).toBe("app.restartable(1): factory threw"); + await retry(); + expect(handles[1]!.value).toEqual({ items: ["a"] }); + }); + + it("migrate in memory when a later attempt declares a newer version; the old handles are refused", async () => { + let initials = 0; + const ListV2 = defineState("list", { + schema: obj({ entries: arr(obj({ text: str() })) }), + version: 2, + initial: () => { + initials++; + return { entries: [] }; + }, + migrate: (old) => ({ entries: (old.items as string[]).map((text) => ({ text })) }), + }); + const Notes1 = defineState("notes", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: 0 }), + keyed: true, + }); + const Notes2 = defineState("notes", { + schema: obj({ n: int(), m: int() }), + version: 2, + initial: () => ({ n: 0, m: 0 }), + keyed: true, + migrate: (old) => ({ n: old.n as number, m: 1 }), + }); + const lists: SingletonHandle[] = []; + const notes: KeyedHandle[] = []; + const inFactory: unknown[] = []; + const { restart } = await restartable(objects, (attempt, self) => { + lists.push(self.state(attempt === 1 ? ListV1 : ListV2) as unknown as SingletonHandle); + notes.push(self.state(attempt === 1 ? Notes1 : Notes2) as unknown as KeyedHandle); + // The new code's handle in its own factory: the values are still at version 1 until the activation. + if (attempt === 2) inFactory.push(rejectionOf(() => lists[1]!.value)); + }); + await lists[0]!.change((draft) => void (draft.items as string[]).push("a"), context); + await notes[0]!.change( + "k", + (draft) => { + draft.n = 5; + }, + context, + ); + const changes = changesOf(await lists[0]!.watch(context)); + await restart(); + expect(lists[1]!.value).toEqual({ entries: [{ text: "a" }] }); + expect(await notes[1]!.get("k", context)).toEqual({ n: 5, m: 1 }); + const stale = "list: a handle of version 1; the documents are at version 2"; + expect(() => lists[0]!.value).toThrow(stale); + expect(await rejection(notes[0]!.get("k", context))).toMatchObject({ code: "released" }); + await delivered(); + expect(changes).toEqual([ + { + kind: "value", + value: { entries: [{ text: "a" }] }, + ops: [["r", { entries: [{ text: "a" }] }]], + retired: false, + }, + ]); + expect(initials).toBe(1); + expect(inFactory).toEqual([expect.objectContaining({ code: "not_ready" })]); + }); + + it("fail the attempt broken for a flipped keyed, an older version or a schema changed without a bump", async () => { + const KeyedList = defineState("list", { + schema: obj({ items: arr(str()) }), + version: 1, + initial: () => ({ items: [] }), + keyed: true, + }); + const Changed = defineState("list", { + schema: obj({ items: arr(int()) }), + version: 1, + initial: () => ({ items: [] }), + }); + const Newer = defineState("list", { + schema: obj({ items: arr(str()) }), + version: 2, + initial: () => ({ items: [] }), + migrate: (old) => old as { items: string[] }, + }); + const cases = [ + [ListV1, KeyedList, "list: declared keyed, its documents are not"], + [ListV1, Changed, "list: the schema changed without a version bump (version 1)"], + [Newer, ListV1, "list: version 1 is older than its documents' (2)"], + ] as const; + for (const [first, later, message] of cases) { + const { fail, retry, errors } = await restartable(objects, (attempt, self) => { + self.state(attempt === 1 ? first : later); + }); + fail(); + await rejection(retry()); + expect(errors.map(({ kind }) => kind)).toEqual(["object_failed", "activation_failed"]); + expect(failure(errors)).toBe(`app.restartable(1): ${message}`); + } + }); +}); + +describe("persisted singletons", () => { + const Prefs = defineState("prefs", { + schema: obj({ theme: str(), n: int() }), + version: 1, + initial: () => ({ theme: "light", n: 0 }), + persist: true, + checkpointWhen: () => false, + }); + + /** A store host declaring `definition` per attempt; `prefs()` is the latest attempt's handle. */ + async function prefsIn(store: ChordStore, definition: typeof Prefs = Prefs) { + let prefs: SingletonHandle<{ theme: string; n: number }> | undefined; + const restartable = await storeRestartable( + objects, + (_, self) => { + prefs = self.state(definition); + }, + { store }, + ); + return { ...restartable, prefs: () => prefs! }; + } + + /** The kinds of the outbox's writes, per commit. */ + async function outboxKinds(store: ChordStore): Promise { + const commits = await store.outbox!.read(0, 100, context); + return commits.map(({ writes }) => + writes.map((write) => (write.kind === "document.change" ? `change:${write.content.kind}` : write.kind)), + ); + } + + it("load once per object at activation (not again on a retry), and survive the host", async () => { + const store = await memoryStore({ outbox: true }); + const first = await prefsIn(store); + expect(first.prefs().value).toEqual({ theme: "light", n: 0 }); + await first.prefs().change((draft) => { + draft.theme = "dark"; + }, context); + await first.prefs().change((draft) => { + draft.n = 1; + }, context); + await first.host.close(context); + const read = vi.spyOn(store, "read"); + const second = await prefsIn(store); + expect(second.prefs().value).toEqual({ theme: "dark", n: 1 }); + expect(read).toHaveBeenCalledTimes(1); + await second.restart(); + expect(second.prefs().value).toEqual({ theme: "dark", n: 1 }); + expect(read).toHaveBeenCalledTimes(1); + await second.prefs().retire(context); + expect(second.prefs().value).toEqual({ theme: "light", n: 0 }); + expect(await outboxKinds(store)).toEqual([["document.create"], ["change:delta"], ["document.retire"]]); + await second.host.close(context); + await store.close(context); + }); + + it("migrate a stored older version at load; the next change writes the migrated base back", async () => { + const store = await memoryStore({ outbox: true }); + const old = await prefsIn(store); + await old.prefs().change((draft) => { + draft.n = 1; + }, context); + await old.host.close(context); + const V2 = defineState("prefs", { + schema: obj({ theme: str(), n: int() }), + version: 2, + initial: () => ({ theme: "light", n: 0 }), + migrate: (value) => ({ theme: "migrated", n: Number(value.n) }), + persist: true, + checkpointWhen: () => false, + }); + const { host, prefs } = await prefsIn(store, V2 as unknown as typeof Prefs); + expect(prefs().value).toEqual({ theme: "migrated", n: 1 }); + await prefs().change((draft) => { + draft.n = 2; + }, context); + expect((await outboxKinds(store)).at(-1)).toEqual(["change:base"]); + await host.close(context); + await store.close(context); + }); + + it("fail the activation broken for a newer stored version or another fingerprint at the same version (D9)", async () => { + const store = await memoryStore(); + const V2 = defineState("prefs", { + schema: obj({ theme: str(), n: int() }), + version: 2, + initial: () => ({ theme: "light", n: 0 }), + persist: true, + checkpointWhen: () => false, + }); + const Edited = defineState("prefs", { + schema: obj({ theme: str() }), + version: 2, + initial: () => ({ theme: "light" }), + persist: true, + checkpointWhen: () => false, + }); + const writer = await prefsIn(store, V2 as unknown as typeof Prefs); + await writer.prefs().change((draft) => { + draft.n = 1; + }, context); + await writer.host.close(context); + for (const [definition, message] of [ + [Prefs, "prefs: stored at version 2, newer than the code's 1"], + [Edited, "prefs: stored with another schema at version 2 (a schema changed without a bump)"], + ] as const) { + const { host, errors } = await loaded( + (h) => + h.serve(StoreRestartable, (_, self) => { + self.state(definition); + return { touch: async () => null }; + }), + { store }, + ); + await rejection(host.get(StoreRestartable, {}, context)); + expect(failure(errors)).toBe(`app.store: ${message}`); + await host.close(context); + } + await store.close(context); + }); + + it("refuse persisting without a stable identity or a store, and a flipped persist", async () => { + const InMemory = defineState("prefs", { + schema: obj({ theme: str(), n: int() }), + version: 1, + initial: () => ({ theme: "light", n: 0 }), + }); + const keyed = await restartable(objects, (_, self) => { + // The types refuse it; a plugin may still try (P3). + (self as StoreSelf).state(Prefs); + }).catch((error: unknown) => error); + expect(String(keyed)).toContain("persist needs a stable identity"); + const storeless = await storeRestartable(objects, (_, self) => void self.state(Prefs)).catch((error) => error); + expect(String(storeless)).toContain("persist needs a store"); + const store = await memoryStore(); + const { fail, retry, errors } = await storeRestartable( + objects, + (attempt, self) => void self.state(attempt === 1 ? Prefs : InMemory), + { store }, + ); + fail(); + await rejection(retry()); + expect(failure(errors)).toBe("app.store: prefs: declared in memory, its documents are not"); + await store.close(context); + }); +}); + +/** The message of the last reported activation failure's error. */ +function failure(errors: readonly HostError[]): string | undefined { + const last = errors.findLast((error) => error.kind === "activation_failed"); + return last?.kind === "activation_failed" ? last.failure.error.message : undefined; +} + +/** What `read` throws, as ChordError JSON when it is one. */ +function rejectionOf(read: () => unknown): unknown { + try { + read(); + } catch (error) { + return isChordError(error) ? error.toJSON() : error; + } + return undefined; +} diff --git a/packages/chord/test/next/documents/transaction.test.ts b/packages/chord/test/next/documents/transaction.test.ts new file mode 100644 index 000000000..ff75ea375 --- /dev/null +++ b/packages/chord/test/next/documents/transaction.test.ts @@ -0,0 +1,731 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import type { Draft } from "../../../src/delta/index.ts"; +import { int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState, type KeyedHandle, type SingletonHandle, type Tx } from "../../../src/next/documents/index.ts"; +import type { Staged, Transaction } from "../../../src/next/documents/internal.ts"; +import type { Self } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { query } from "../../../src/next/store/query.ts"; +import type { StagedWrite as StoreStaged } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import type { TableOf } from "../../../src/next/store/table-handle.ts"; +import { + delivered, + framesOf, + instancesOf, + loaded, + memoryStore, + metered, + rejection, + restartable, + storeRestartable, +} from "../support.ts"; + +const instances = instancesOf(ChordObject, "attempt"); + +const A = defineState("a", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); +const B = defineState("b", { schema: obj({ s: str() }), version: 1, initial: () => ({ s: "" }) }); +const Owner = local("app.owner", { version: 1, params: { id: int() }, members: { sum: str() } }); + +type Handles = { + readonly self: Self; + readonly a: SingletonHandle<{ n: number }>; + readonly b: SingletonHandle<{ s: string }>; +}; + +/** Objects with two documents and a derive over them; returns each object's handles. */ +async function objects(telemetry?: Awaited>["telemetry"]) { + const handles = new Map(); + const { host } = await loaded( + (h) => { + h.serve(Owner, ({ id }, self) => { + const a = self.state(A); + const b = self.state(B); + handles.set(id, { self, a, b }); + return { sum: self.derive([a, b], ([x, y]) => `${x.n}${y.s}`) }; + }); + }, + { telemetry }, + ); + const reference = await host.get(Owner, { id: 1 }, context); + await host.get(Owner, { id: 2 }, context); + return { reference, one: handles.get(1)!, two: handles.get(2)! }; +} + +describe("transactions", () => { + it("run one at a time per object and return what their callback returns", async () => { + const { one } = await objects(); + const order: string[] = []; + const gate = Promise.withResolvers(); + const first = one.self.tx(async (tx) => { + order.push("first"); + await gate.promise; + (await tx.draft(one.a)).n = 1; + order.push("first done"); + return "one"; + }, context); + const second = one.self.tx(async (tx) => { + order.push(`second sees ${one.a.value.n}`); + (await tx.draft(one.a)).n++; + return "two"; + }, context); + await delivered(); + gate.resolve(); + expect(await Promise.all([first, second])).toEqual(["one", "two"]); + expect(order).toEqual(["first", "first done", "second sees 1"]); + expect(one.a.value).toEqual({ n: 2 }); + }); + + it("give the same draft for the same document", async () => { + const { one } = await objects(); + await one.self.tx(async (tx) => { + expect(await tx.draft(one.a)).toBe(await tx.draft(one.a)); + }, context); + }); + + it("settle a warm draft at once: a continuation the callback did not await runs before the commit (rule 3)", async () => { + const { one } = await objects(); + await one.self.tx((tx) => { + void tx.draft(one.a).then((draft) => { + draft.n = 7; + }); + }, context); + expect(one.a.value).toEqual({ n: 7 }); + }); + + it("reject a draft of a handle a transaction cannot use instead of throwing", async () => { + const { one, two } = await objects(); + await one.self.tx(async (tx) => { + const draft = tx.draft(two.a); + expect(draft).toBeInstanceOf(Promise); + await expect(draft).rejects.toThrow("drafts only its own object's documents"); + }, context); + }); + + it("commit nothing when nothing changed", async () => { + const telemetry = await metered(); + const { one, reference } = await objects(telemetry.telemetry); + const frames = framesOf(await reference.sum.watch(context)); + const value = one.a.value; + const computes = telemetry.counts()["chord.derive.computes"]; + await one.self.tx(async (tx) => { + void (await tx.draft(one.a)).n; + }, context); + await delivered(); + expect(one.a.value).toBe(value); + expect(frames).toEqual([]); + expect(telemetry.counts()["chord.derive.computes"]).toBe(computes); + }); + + it("discard everything on a throw and on an aborted context", async () => { + const { one } = await objects(); + const change = async (tx: Tx) => { + (await tx.draft(one.a)).n = 5; + }; + await expect( + one.self.tx(async (tx) => { + await change(tx); + throw new Error("no"); + }, context), + ).rejects.toThrow("no"); + const { context: cancelled, cancel } = withCancel(context); + await expect( + one.self.tx(async (tx) => { + await change(tx); + cancel(new Error("cancelled")); + }, cancelled), + ).rejects.toThrow("cancelled"); + expect(one.a.value).toEqual({ n: 0 }); + }); + + it("revoke their drafts when they settle", async () => { + const { one } = await objects(); + let kept: Draft<{ n: number }> | undefined; + await one.self.tx(async (tx) => { + kept = await tx.draft(one.a); + kept.n = 1; + }, context); + expect(() => kept!.n).toThrow(); + await expect(one.self.tx(async (tx) => tx, context).then((tx) => tx.draft(one.a))).rejects.toThrow( + "app.owner(1): the transaction has settled", + ); + }); + + it("refuse another object's documents", async () => { + const { one, two } = await objects(); + expect(await rejection(one.self.tx((tx) => tx.draft(two.a), context))).toEqual( + new Error("app.owner(1): a transaction drafts only its own object's documents"), + ); + }); + + it("commit the object's documents in one batch: one compute, one frame", async () => { + const telemetry = await metered(); + const { one, reference } = await objects(telemetry.telemetry); + const frames = framesOf(await reference.sum.watch(context)); + const computes = telemetry.counts()["chord.derive.computes"]!; + await one.self.tx(async (tx) => { + (await tx.draft(one.a)).n = 1; + (await tx.draft(one.b)).s = "b"; + }, context); + await delivered(); + expect(frames).toEqual([["1b", [["r", "1b"]]]]); + expect(telemetry.counts()["chord.derive.computes"]).toBe(computes + 1); + }); + + it("commit their other changes when they retire an absent document (R13)", async () => { + const { one } = await objects(); + await one.self.tx(async (tx) => { + await tx.retire(one.b); + (await tx.draft(one.a)).n = 7; + }, context); + expect([one.a.value, one.b.value]).toEqual([{ n: 7 }, { s: "" }]); + }); + + it("draft a retired document anew: watches get a root replacement", async () => { + const { one } = await objects(); + await one.a.change((draft) => { + draft.n = 3; + }, context); + const frames = framesOf(await one.a.watch(context)); + await one.self.tx(async (tx) => { + await tx.retire(one.a); + (await tx.draft(one.a)).n += 1; + }, context); + await delivered(); + expect(frames).toEqual([[{ n: 1 }, [["r", { n: 1 }]]]]); + }); +}); + +describe("the in-memory change's cost", () => { + it("settles an in-memory change within five microtask turns (a guard: an extra await per tx operation shows here)", async () => { + const { one } = await objects(); + for (let run = 0; run < 3; run++) { + let turns = 0; + let running = true; + const change = one.a.change((draft) => { + draft.n++; + }, context); + void change.then(() => { + running = false; + }); + while (running) { + turns++; + await null; + } + await change; + expect(turns).toBeLessThanOrEqual(5); + } + expect(one.a.value.n).toBe(3); + }); +}); + +describe("staged changes", () => { + /** A staged change recording what happens to it, and the document value its adoption sees. */ + function recorder(events: string[], a: SingletonHandle<{ n: number }>): Staged { + return { + adopt: () => void events.push(`adopt sees ${a.value.n}`), + committed: () => void events.push("committed"), + discarded: () => void events.push("discarded"), + }; + } + + it("adopt in the commit step with the documents, then hear of the commit once the line is free", async () => { + const { one } = await objects(); + const events: string[] = []; + const owner = {}; + await one.self.tx(async (tx) => { + (await tx.draft(one.a)).n = 7; + const transaction = tx as Transaction; + const staged = transaction.stage(owner, () => recorder(events, one.a)); + // One per owner and transaction. + expect(transaction.stage(owner, () => recorder([], one.a))).toBe(staged); + events.push("callback done"); + }, context); + expect(events).toEqual(["callback done", "adopt sees 7", "committed"]); + }); + + it("adopt without any document change, and hear of a discard instead when the transaction throws", async () => { + const { one } = await objects(); + const events: string[] = []; + await one.self.tx((tx) => void (tx as Transaction).stage({}, () => recorder(events, one.a)), context); + await rejection( + one.self.tx((tx) => { + (tx as Transaction).stage({}, () => recorder(events, one.a)); + throw new Error("no"); + }, context), + ); + expect(events).toEqual(["adopt sees 0", "committed", "discarded"]); + }); +}); + +describe("the seal fence", () => { + const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + + /** A restartable object with `N`; its handles by attempt. */ + async function sealable() { + const handles: SingletonHandle<{ n: number }>[] = []; + const object = await restartable(instances, (_, self) => { + handles.push(self.state(N)); + }); + return { ...object, handles }; + } + + it("discards the transaction holding the line at the seal: the line is free, its later uses are refused", async () => { + const { attempts, handles, restart } = await sealable(); + const gate = Promise.withResolvers(); + let held: Tx | undefined; + const first = attempts[0]!.tx(async (tx) => { + (await tx.draft(handles[0]!)).n = 1; + held = tx; + await gate.promise; + return "late"; + }, context); + await delivered(); + await restart(); + // The next attempt's transaction runs while the cut one's callback still waits. + await handles[1]!.change((draft) => { + draft.n = 2; + }, context); + gate.resolve(); + expect(await rejection(first)).toMatchObject({ code: "released" }); + expect(await rejection(held!.draft(handles[1]!))).toMatchObject({ + code: "released", + message: "app.restartable(1): the transaction was discarded at the seal", + }); + expect(handles[1]!.value).toEqual({ n: 2 }); + }); + + it("refuses a sealed attempt's transactions waiting for the line and every new one; its handles still read", async () => { + const { attempts, handles, fail } = await sealable(); + const gate = Promise.withResolvers(); + const first = attempts[0]!.tx(() => gate.promise, context); + const queued = handles[0]!.change((draft) => { + draft.n = 1; + }, context); + await delivered(); + fail(); + expect(await rejection(queued)).toMatchObject({ code: "released" }); + expect(await rejection(handles[0]!.change(() => {}, context))).toMatchObject({ code: "released" }); + expect(handles[0]!.value).toEqual({ n: 0 }); + gate.resolve(); + expect(await rejection(first)).toMatchObject({ code: "released" }); + }); +}); + +describe("tables in a transaction", () => { + const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + const Edits = defineTable("edits", { version: 1, columns: { id: str(), at: int() }, key: ["id"] }); + type EditsTable = TableOf; + + /** A store object with `N` and `Edits`, on a store with an outbox; its handles by attempt. */ + async function stored(telemetry?: Awaited>["telemetry"]) { + const store: ChordStore = await memoryStore({ outbox: true }); + const handles: { n: SingletonHandle<{ n: number }>; edits: EditsTable }[] = []; + const object = await storeRestartable( + instances, + (_, self) => { + handles.push({ n: self.state(N), edits: self.table(Edits) }); + }, + { store, telemetry }, + ); + return { ...object, store, handles, self: object.attempts[0]!, ...handles[0]! }; + } + + it("commit rows and documents in one step: the documents adopt inside it, one seq", async () => { + const { self, n, edits, store, host } = await stored(); + const watch = await n.watch(context); + const seen: number[] = []; + watch.start(async () => { + seen.push(store.lastSeq); + }); + await delivered(); + const before = store.lastSeq; + await self.tx(async (tx) => { + tx.put(edits, { id: "e1", at: 1 }); + (await tx.draft(n)).n = 1; + }, context); + await delivered(); + expect(store.lastSeq).toBe(before + 1); + expect(seen.at(-1)).toBe(before + 1); + expect(n.value).toEqual({ n: 1 }); + expect(await edits.get({ id: "e1" }, context)).toEqual({ id: "e1", at: 1 }); + watch.stop(); + await host.close(context); + await store.close(context); + }); + + it("in-memory-only commits take no step and no seq (W12)", async () => { + const { telemetry, context: counting, counts } = await metered(); + const { self, n, store, host } = await stored(telemetry); + const commits = counts()["chord.store.commits"] ?? 0; + const seq = store.lastSeq; + await self.tx(async (tx) => { + (await tx.draft(n)).n = 5; + }, counting); + expect(store.lastSeq).toBe(seq); + expect(counts()["chord.store.commits"] ?? 0).toBe(commits); + await host.close(context); + await store.close(context); + }); + + it("read committed rows with tx.page and tx.rows, overlay own writes with tx.get", async () => { + const { self, edits, store, host } = await stored(); + await self.tx((tx) => tx.put(edits, { id: "e1", at: 1 }), context); + await self.tx(async (tx) => { + tx.put(edits, { id: "e2", at: 2 }); + expect((await tx.page(edits, query(Edits), 10)).items.map((row) => row.id)).toEqual(["e1"]); + const all: string[] = []; + for await (const row of tx.rows(edits, query(Edits))) all.push(row.id); + expect(all).toEqual(["e1"]); + expect(await tx.get(edits, { id: "e2" })).toEqual({ id: "e2", at: 2 }); + }, context); + expect(await edits.count(query(Edits), context)).toBe(2); + await host.close(context); + await store.close(context); + }); + + it("write nothing on a throw or an abort, and refuse a callback that settles with a pending read (rule 3)", async () => { + const { self, edits, store, host } = await stored(); + const seq = store.lastSeq; + await expect( + self.tx((tx) => { + tx.put(edits, { id: "e1", at: 1 }); + throw new Error("no"); + }, context), + ).rejects.toThrow("no"); + const { context: cancelled, cancel } = withCancel(context); + const aborted = self.tx(async (tx) => { + tx.put(edits, { id: "e1", at: 1 }); + cancel(); + }, cancelled); + await expect(aborted).rejects.toThrow(); + await expect( + self.tx((tx) => { + void tx.get(edits, { id: "e1" }); + tx.put(edits, { id: "e1", at: 1 }); + }, context), + ).rejects.toThrow("settled before its pending tx operations"); + expect(store.lastSeq).toBe(seq); + expect(await edits.count(query(Edits), context)).toBe(0); + await host.close(context); + await store.close(context); + }); + + it("refuse another object's table", async () => { + const { self, store, host } = await stored(); + const other = await stored(); + await expect(self.tx((tx) => tx.put(other.edits, { id: "e1", at: 1 }), context)).rejects.toThrow( + "writes only its own object's tables", + ); + await host.close(context); + await other.host.close(context); + await store.close(context); + await other.store.close(context); + }); + + it("a commit queued behind another owner's step is refused at the seal; an admitted one is adopted", async () => { + const { attempts, handles, store, fail, retry, host } = await stored(); + // Hold the commit queue with a step that waits. + const gate = Promise.withResolvers(); + const holding = store.commit( + [ + { + order: "write", + apply: async () => { + await gate.promise; + return { kind: "dropIndex", table: "x", name: "y" }; + }, + }, + ], + { admit: () => true, adopt: () => {} }, + context, + ); + const queued = attempts[0]!.tx(async (tx) => { + tx.put(handles[0]!.edits, { id: "e1", at: 1 }); + (await tx.draft(handles[0]!.n)).n = 1; + }, context); + await delivered(); + fail(); + gate.resolve(); + await holding.catch(() => {}); + expect(await rejection(queued)).toMatchObject({ code: "released" }); + await retry(); + expect(handles[1]!.n.value).toEqual({ n: 0 }); + expect(await handles[1]!.edits.get({ id: "e1" }, context)).toBeUndefined(); + // Admitted, then sealed while in the step: durable, adopted into the object's slots, `self.tx` resolves. + const commit = store.commit.bind(store); + store.commit = (writes, fence, ctx) => + commit(writes, { admit: () => sealedAfter(fence.admit()), adopt: fence.adopt }, ctx); + const sealedAfter = (admitted: boolean): boolean => { + if (admitted) fail(); + return admitted; + }; + const admitted = await attempts[1]!.tx(async (tx) => { + tx.put(handles[1]!.edits, { id: "e2", at: 2 }); + (await tx.draft(handles[1]!.n)).n = 2; + return "done"; + }, context); + store.commit = commit; + expect(admitted).toBe("done"); + await retry(); + expect(handles[2]!.n.value).toEqual({ n: 2 }); + expect(await handles[2]!.edits.get({ id: "e2" }, context)).toEqual({ id: "e2", at: 2 }); + await host.close(context); + await store.close(context); + }); +}); + +describe("persisted documents in a transaction", () => { + const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + const P = defineState("p", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: 0 }), + persist: true, + checkpointWhen: () => false, + }); + const K = defineState("k", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: 0 }), + keyed: true, + persist: true, + checkpointWhen: () => false, + }); + + /** A store object with in-memory `N`, persisted `P` and keyed persisted `K`, on a store with an outbox. */ + async function stored(telemetry?: Awaited>["telemetry"], on?: ChordStore) { + const store: ChordStore = on ?? (await memoryStore({ outbox: true })); + const handles: { + n: SingletonHandle<{ n: number }>; + p: SingletonHandle<{ n: number }>; + k: KeyedHandle<{ n: number }>; + }[] = []; + const object = await storeRestartable( + instances, + (_, self) => { + handles.push({ n: self.state(N), p: self.state(P), k: self.state(K) }); + }, + { store, telemetry }, + ); + return { ...object, store, self: object.attempts[0]!, ...handles[0]! }; + } + + it("commit persisted and in-memory documents in one step; a delta commit takes three statements", async () => { + const { telemetry, context: counting, counts } = await metered(); + const { self, n, p, store, host } = await stored(telemetry); + await self.tx(async (tx) => { + (await tx.draft(n)).n = 1; + (await tx.draft(p)).n = 1; + }, counting); + expect(store.lastSeq).toBe(1); + const before = { ...counts() }; + await self.tx(async (tx) => { + (await tx.draft(p)).n = 2; + }, counting); + expect(counts()["chord.store.commits"]! - before["chord.store.commits"]!).toBe(1); + // The revision, the record's conditional UPDATE, lastSeq; plus the outbox commit. + expect(counts()["chord.store.statements"]! - before["chord.store.statements"]!).toBe(4); + expect([n.value, p.value]).toEqual([{ n: 1 }, { n: 2 }]); + await host.close(context); + await store.close(context); + }); + + it("assign the created documents' ids by their order in the commit; retire and recreate in one commit", async () => { + const { self, k, store, host } = await stored(); + await self.tx(async (tx) => { + (await tx.draft(k, "b")).n = 1; + (await tx.draft(k, "a")).n = 2; + }, context); + await self.tx(async (tx) => { + await tx.retire(k, "b"); + (await tx.draft(k, "b")).n = 3; + }, context); + const commits = await store.outbox!.read(0, 10, context); + const created = commits.flatMap(({ writes }) => + writes.flatMap((write) => (write.kind === "document.create" ? [[write.record.key, write.record.id]] : [])), + ); + expect(created).toEqual([ + ["b", 1], + ["a", 2], + ["b", 3], + ]); + expect(commits[1]!.writes.map(({ kind }) => kind)).toEqual(["document.retire", "document.create"]); + // What is stored of `b` is the new record's: the next change is a delta on it. + await k.change( + "b", + (draft) => { + draft.n = 4; + }, + context, + ); + expect((await store.outbox!.read(2, 10, context))[0]!.writes).toMatchObject([{ kind: "document.change", id: 3 }]); + await host.close(context); + await store.close(context); + }); + + it("draft a cold keyed document by loading it inside the transaction; refuse a callback with a pending load", async () => { + const first = await stored(); + const { store } = first; + await first.k.change( + "a", + (draft) => { + draft.n = 1; + }, + context, + ); + await first.host.close(context); + const { self, k, host } = await stored(undefined, store); + await self.tx(async (tx) => { + (await tx.draft(k, "a")).n += 1; + }, context); + expect(await k.get("a", context)).toEqual({ n: 2 }); + await expect( + self.tx(async (tx) => { + // Not awaited: the draft is still loading when the callback returns (it rejects later, settled). + tx.draft(k, "b").catch(() => {}); + }, context), + ).rejects.toThrow("the callback settled before its pending tx operations"); + await host.close(context); + await store.close(context); + }); + + it("settle a warm keyed draft at once: a continuation the callback did not await runs before the commit", async () => { + const { self, k, host, store } = await stored(); + await self.tx(async (tx) => { + (await tx.draft(k, "a")).n = 1; + }, context); + await self.tx((tx) => { + void tx.draft(k, "a").then((draft) => { + draft.n = 7; + }); + }, context); + expect(await k.get("a", context)).toEqual({ n: 7 }); + await host.close(context); + await store.close(context); + }); + + it("a commit cut at the seal while it waits in the store's queue rejects released at once, writing nothing", async () => { + const { p, store, host } = await stored(); + // A suspended export holds the commit queue. + const parts = store.export(context); + await parts.next(); + const change = rejection( + p.change((draft) => { + draft.n = 5; + }, context), + ); + await delivered(); + await host.close(context); + expect(await change).toMatchObject({ code: "released" }); + await parts.return!(); + expect(store.lastSeq).toBe(0); + const next = await stored(undefined, store); + expect(next.p.value).toEqual({ n: 0 }); + await next.p.change((draft) => { + draft.n = 6; + }, context); + expect(store.lastSeq).toBe(1); + await next.host.close(context); + await store.close(context); + }); + + it("a host closing while an admitted commit is in the SQL waits for its adoption; the store stays usable", async () => { + const { p, store, host } = await stored(); + // The next commit's step waits inside the backend transaction, after its admission. + const gate = Promise.withResolvers(); + const inStep = Promise.withResolvers(); + const commit = store.commit.bind(store); + store.commit = (writes, fence, ctx) => { + const held: StoreStaged = { + order: "write", + apply: async () => { + inStep.resolve(); + await gate.promise; + return { kind: "dropIndex", table: "x", name: "y" }; + }, + }; + return commit([...writes, held], fence, ctx); + }; + // Not from a method handler: the drain does not wait for it. + const change = p.change((draft) => { + draft.n = 5; + }, context); + await inStep.promise; + let closed = false; + const closing = host.close(context).then(() => { + closed = true; + }); + await delivered(); + expect(closed).toBe(false); + gate.resolve(); + await change; + await closing; + store.commit = commit; + expect(store.lastSeq).toBe(1); + const next = await stored(undefined, store); + expect(next.p.value).toEqual({ n: 5 }); + await next.p.change((draft) => { + draft.n = 6; + }, context); + expect(store.lastSeq).toBe(2); + await next.host.close(context); + await store.close(context); + }); + + it("a handler's commit held in the SQL past closeTimeout still adopts before the object ends; the store stays usable", async () => { + const Writer = local("app.writer", { version: 1, params: {}, members: { write: method() } }); + const store = await memoryStore({ outbox: true }); + const serve = (h: Parameters[0]>[0]) => + h.serve(Writer, (_, self) => { + const p = self.state(P); + return { + write: async (ctx) => { + await p.change((draft) => { + draft.n++; + }, ctx); + return null; + }, + }; + }); + const { host, errors } = await loaded(serve, { store, closeTimeout: 20 }); + const gate = Promise.withResolvers(); + const inStep = Promise.withResolvers(); + const commit = store.commit.bind(store); + store.commit = (writes, fence, ctx) => { + const held: StoreStaged = { + order: "write", + apply: async () => { + inStep.resolve(); + await gate.promise; + return { kind: "dropIndex", table: "x", name: "y" }; + }, + }; + return commit([...writes, held], fence, ctx); + }; + const call = rejection((await host.get(Writer, {}, context)).write(context)); + await inStep.promise; + let closed = false; + const closing = host.close(context).then(() => { + closed = true; + }); + await new Promise((resolve) => setTimeout(resolve, 60)); + // The drain gave up on the handler; the object's teardown still waits for the admitted step. + expect(errors.map(({ kind }) => kind)).toEqual(["drain_timeout"]); + expect(closed).toBe(false); + gate.resolve(); + await closing; + // The stuck call was released at the end; its commit, admitted before, is durable and was adopted. + expect(await call).toMatchObject({ code: "released" }); + store.commit = commit; + expect(store.lastSeq).toBe(1); + const next = await loaded(serve, { store }); + await (await next.host.get(Writer, {}, context)).write(context); + expect(store.lastSeq).toBe(2); + expect(next.errors).toEqual([]); + await next.host.close(context); + await store.close(context); + }); +}); diff --git a/packages/chord/test/next/examples/01-objects.ts b/packages/chord/test/next/examples/01-objects.ts new file mode 100644 index 000000000..57a691e58 --- /dev/null +++ b/packages/chord/test/next/examples/01-objects.ts @@ -0,0 +1,68 @@ +// Objects on one host: a contract, a facet serving it, references, calls, and how long objects live. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/01-objects.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; + +// Every call takes a context, used for cancellation; BACKGROUND_CONTEXT is never cancelled. +const context = BACKGROUND_CONTEXT; + +// A contract declares what an object type offers: state members (schemas) and methods (arguments, result). +// `params` select the instance: one object per params value. +const Greeter = local("example.greeter", { + version: 1, + params: { language: str() }, + members: { language: str(), greet: method(obj({ name: str() }), str()) }, +}); +// Empty params: one object for the whole host. +const Clock = local("example.clock", { version: 1, params: {}, members: { started: int() } }); + +// A facet is a plugin's setup for one host. It serves types with factories; a factory only declares and returns the +// implementation: a watchable per state member, an async function per method. +const facet = defineFacet((host) => { + host.serve(Clock, () => { + console.log("clock: created at load"); + return { started: constant(Date.now()) }; + }); + host.serve(Greeter, ({ language }) => { + console.log(`greeter ${language}: created by its first get`); + return { + language: constant(language), + greet: async ({ name }) => (language === "de" ? `Hallo ${name}` : `Hello ${name}`), + }; + }); +}); + +// onError receives background errors, such as a watch listener that throws. +const host = createHost({ onError: (error) => console.error("background error:", error) }); +// load runs every setup and activates the types with empty params before it resolves. +await host.load([{ name: "@example/greeter", facet }], context); + +// A reference is live once its get resolves: its state members have values, its methods can be called. +const german = await host.get(Greeter, { language: "de" }, context); +console.log("language:", german.language.value); +console.log("greet:", await german.greet({ name: "Mario" }, context)); + +// Arguments are checked against the contract at the object. +try { + // @ts-expect-error: `name` must be a string + await german.greet({ name: 42 }, context); +} catch (error) { + if (isChordError(error, "invalid")) console.log("refused:", error.reason, error.message); +} + +// The same params share one object; releasing the last reference ends it. +const again = await host.get(Greeter, { language: "de" }, context); +german.release(); +again.release(); +try { + german.language.value; +} catch (error) { + if (isChordError(error, "released")) console.log("after release:", error.message); +} + +// close ends every object. +await host.close(context); diff --git a/packages/chord/test/next/examples/02-documents.ts b/packages/chord/test/next/examples/02-documents.ts new file mode 100644 index 000000000..1387cf0dc --- /dev/null +++ b/packages/chord/test/next/examples/02-documents.ts @@ -0,0 +1,79 @@ +// Documents and transactions: an object's in-memory JSON state, written atomically. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/02-documents.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; + +const context = BACKGROUND_CONTEXT; + +// A singleton document: one per object, `initial()` until written. +const Todos = defineState("todos", { + schema: obj({ items: arr(obj({ id: int(), text: str() })), next: int() }), + version: 1, + initial: () => ({ items: [], next: 1 }), +}); +// A keyed document: many per object, one per string key, absent until written. +const Notes = defineState("notes", { + schema: obj({ body: str() }), + version: 1, + initial: () => ({ body: "" }), + keyed: true, +}); + +const Store = local("example.todos", { + version: 1, + params: {}, + members: { + todos: obj({ items: arr(obj({ id: int(), text: str() })), next: int() }), + add: method(obj({ text: str(), note: opt(str()) }), int()), + note: method(obj({ id: int() }), str()), + clear: method(), + }, +}); + +const facet = defineFacet((host) => { + host.serve(Store, (_, self) => { + const todos = self.state(Todos); + const notes = self.state(Notes); + return { + // A singleton handle is a watchable, so it can be a state member as it is. + todos, + add: async ({ text, note }, ctx) => { + // One transaction over two documents: both change together, or neither does. + const id = await self.tx(async (tx) => { + const draft = await tx.draft(todos); + const id = draft.next++; + draft.items.push({ id, text }); + if (note !== undefined) (await tx.draft(notes, String(id))).body = note; + // Return plain values: drafts are revoked once the transaction settles. + return id; + }, ctx); + return id; + }, + // Read keyed documents outside transactions; an absent one is `undefined`. + note: async ({ id }, ctx) => (await notes.get(String(id), ctx))?.body ?? "(none)", + clear: async (ctx) => { + // Shorthands for one document: retire resets a singleton to `initial()`. + await todos.retire(ctx); + for await (const key of notes.keys(ctx)) await notes.retire(key, ctx); + return null; + }, + }; + }); +}); + +const host = createHost({ onError: (error) => console.error("background error:", error) }); +await host.load([{ name: "@example/todos", facet }], context); +const store = await host.get(Store, {}, context); + +const first = await store.add({ text: "buy milk", note: "the oat one" }, context); +await store.add({ text: "call Anna" }, context); +console.log("todos:", store.todos.value.items); +console.log("note of", first, "->", await store.note({ id: first }, context)); + +await store.clear(context); +console.log("after clear:", store.todos.value, await store.note({ id: first }, context)); + +await host.close(context); diff --git a/packages/chord/test/next/examples/03-derives.ts b/packages/chord/test/next/examples/03-derives.ts new file mode 100644 index 000000000..3d756253c --- /dev/null +++ b/packages/chord/test/next/examples/03-derives.ts @@ -0,0 +1,66 @@ +// Derives: values computed from documents and from other objects' members, recomputed only when needed. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/03-derives.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { mapShared } from "../../../src/next/state/index.ts"; + +const context = BACKGROUND_CONTEXT; + +const NotesDoc = defineState("notes", { + schema: obj({ list: arr(obj({ text: str() })) }), + version: 1, + initial: () => ({ list: [] }), +}); + +const Notes = local("example.notes", { + version: 1, + params: {}, + members: { titles: arr(str()), count: int(), add: method(obj({ text: str() })) }, +}); +const Badge = local("example.badge", { version: 1, params: { user: str() }, members: { label: str() } }); + +const facet = defineFacet((host) => { + host.serve(Notes, (_, self) => { + const notes = self.state(NotesDoc); + // mapShared calls its function once per note object: unchanged notes keep the same title, so the derive's + // diff after an append is one insert, not a comparison of every title. + const title = mapShared((note: { text: string }) => { + console.log(" title computed for:", note.text); + return note.text.slice(0, 12); + }); + return { + // A derive declares its sources; its compute is synchronous and pure. + titles: self.derive([notes], ([n]) => title(n.list)), + count: self.derive([notes], ([n]) => n.list.length), + add: async ({ text }, ctx) => { + await notes.change((draft) => { + draft.list.push({ text }); + }, ctx); + return null; + }, + }; + }); + // A derive over another object's member, through a reference declared with self.get. + host.serve(Badge, ({ user }, self) => { + const notes = self.get(Notes, {}); + return { label: self.derive([notes.count], ([count]) => `${user}: ${count} notes`) }; + }); +}); + +const host = createHost({ onError: (error) => console.error("background error:", error) }); +await host.load([{ name: "@example/notes", facet }], context); +const notes = await host.get(Notes, {}, context); +const badge = await host.get(Badge, { user: "mario" }, context); + +await notes.add({ text: "buy milk and bread" }, context); +await notes.add({ text: "call Anna" }, context); +// Nobody watches `titles`, so the commits only marked it; reading computes it, each title once. +console.log("titles:", notes.titles.value); +await notes.add({ text: "water plants" }, context); +console.log("titles:", notes.titles.value); +console.log("badge:", badge.label.value); + +await host.close(context); diff --git a/packages/chord/test/next/examples/04-watches.ts b/packages/chord/test/next/examples/04-watches.ts new file mode 100644 index 000000000..f6f81972c --- /dev/null +++ b/packages/chord/test/next/examples/04-watches.ts @@ -0,0 +1,101 @@ +// Watches: follow a state member from its current value through the commits that change it; a busy listener gets +// what changed meanwhile folded into one call; a derive that throws gives a `failed` call, the watch stays open, and +// its next value arrives as a root replacement (it heals on a later change). +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/04-watches.ts +import { BACKGROUND_CONTEXT, withCancel } from "../../../src/context/index.ts"; +import { int, local, method, obj } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; + +const context = BACKGROUND_CONTEXT; + +const Count = defineState("count", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); +const Counter = local("example.counter", { + version: 1, + params: {}, + members: { count: obj({ n: int() }), double: int(), half: int(), bump: method(), reset: method() }, +}); + +const facet = defineFacet((host) => { + host.serve(Counter, (_, self) => { + const count = self.state(Count); + return { + count, + double: self.derive([count], ([c]) => c.n * 2), + half: self.derive([count], ([c]) => { + if (c.n % 2 === 1) throw new Error(`${c.n} has no half`); + return c.n / 2; + }), + bump: async (ctx) => { + await count.change((draft) => { + draft.n++; + }, ctx); + return null; + }, + reset: async (ctx) => { + await count.retire(ctx); + return null; + }, + }; + }); +}); + +// A derive whose compute throws is reported once per failure episode (`derive_failed`). +const host = createHost({ + onError: (error) => { + if (error.kind === "derive_failed") console.log(`reported: ${error.object}.${error.member}: ${error.error}`); + else console.error("background error:", error); + }, +}); +await host.load([{ name: "@example/counter", facet }], context); +const counter = await host.get(Counter, {}, context); + +// A watch holds the value at acquisition; its listener then gets the changes, asynchronously and in order, never +// overlapping itself. A listener that keeps up gets one call per commit: the new value and the ops that turn the +// previous value into it. +const watch = await counter.count.watch(context); +console.log("acquired:", watch.value); +watch.start(async (call) => { + if (call.kind === "value") console.log("count frame:", call.value, JSON.stringify(call.ops)); +}); + +// A derive's frame carries the diff of its old and new value. +const { context: cancellable, cancel } = withCancel(context); +const doubled = await counter.double.watch(cancellable); +doubled.start(async (call) => { + if (call.kind === "value") console.log("double frame:", call.value, JSON.stringify(call.ops)); +}); + +// A failing derive: one `failed` call per failure, the watch stays open; the next value replaces the root. +const half = await counter.half.watch(context); +half.start(async (call) => { + if (call.kind === "failed") console.log("half failed:", (call.error as Error).message); + else console.log("half frame:", call.value, JSON.stringify(call.ops)); +}); + +// A slow listener: commits made while it is busy fold into its next call, with the newest value and the ops since +// its previous call (an `s` on one path replaces the one before it). +const slow = await counter.count.watch(context); +slow.start(async (call) => { + if (call.kind !== "value") return; + console.log("slow frame:", call.value, JSON.stringify(call.ops)); + await new Promise((resolve) => setTimeout(resolve, 20)); +}); + +await counter.bump(context); +await new Promise((resolve) => setTimeout(resolve, 0)); +await counter.bump(context); +await counter.bump(context); +await new Promise((resolve) => setTimeout(resolve, 50)); +// Retiring the singleton resets it to initial(): watches get a root replacement. +await counter.reset(context); +await new Promise((resolve) => setTimeout(resolve, 50)); + +// Cancelling the watch's context ends it; stop() ends it too. Releasing the reference ends the watches taken through it. +cancel(); +console.log("double watch:", await doubled.closed); +counter.release(); +console.log("count watch:", await watch.closed); + +await host.close(context); diff --git a/packages/chord/test/next/examples/05-errors.ts b/packages/chord/test/next/examples/05-errors.ts new file mode 100644 index 000000000..7ea09797d --- /dev/null +++ b/packages/chord/test/next/examples/05-errors.ts @@ -0,0 +1,93 @@ +// Errors: what Chord refuses and how a caller tells the cases apart (`ChordError` with `code` and `reason`). +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/05-errors.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { createHost, defineFacet, type HostError } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; + +const context = BACKGROUND_CONTEXT; + +/** Prints how `work` failed: a ChordError's code, reason and message, or a plain error's message. */ +async function attempt(label: string, work: () => unknown): Promise { + try { + await work(); + console.log(`${label}: ok`); + } catch (error) { + if (isChordError(error)) + console.log(`${label}: ${error.code}${error.reason ? `/${error.reason}` : ""} - ${error.message}`); + else console.log(`${label}: ${error instanceof Error ? error.message : String(error)}`); + } +} + +const Echo = local("example.echo", { + version: 1, + params: { id: int() }, + members: { id: int(), echo: method(obj({ text: str() }), str()) }, +}); +// Two types that get each other while they activate: a cycle. +const Ping = local("example.ping", { version: 1, params: {}, members: { n: int() } }); +const Pong = local("example.pong", { version: 1, params: {}, members: { n: int() } }); +const Broken = local("example.broken", { version: 1, params: { id: int() }, members: { n: int() } }); +const Sloppy = local("example.sloppy", { version: 1, params: { id: int() }, members: { n: int() } }); + +const facet = defineFacet((host) => { + host.serve(Echo, ({ id }) => ({ id: constant(id), echo: async ({ text }) => text })); + host.serve(Broken, () => { + throw new Error("broken: missing configuration"); + }); + // Plugins are bundled without type checks; Chord checks the implementation's members when the object activates. + host.serve(Sloppy, () => ({ n: constant(1), extra: constant(2) }) as never); +}); +// Ping and Pong are served by a second plugin: types with empty params activate at load. +const cycle = defineFacet((host) => { + host.serve(Ping, (_, self) => ({ n: self.get(Pong, {}).n })); + host.serve(Pong, (_, self) => ({ n: self.get(Ping, {}).n })); +}); + +// Failed activations are reported to onError; 07-failures.ts shows the other reports. +const onError = (error: HostError): void => { + if (error.kind === "activation_failed") console.log(`reported: ${error.object} failed ${error.failure.kind}`); + else console.log("reported:", error.kind); +}; + +// A dependency cycle fails every object on it broken, with invalid/cycle naming the chain. Load still resolves: the +// start barrier waits for each object's first outcome, live or failed. +const host = createHost({ onError }); +await host.load( + [ + { name: "@example/errors", facet }, + { name: "@example/cycle", facet: cycle }, + ], + context, +); +await attempt("get in a cycle", () => host.get(Ping, {}, context)); +const echo = await host.get(Echo, { id: 1 }, context); + +// Arguments are checked against the contract at the object: wrong types and unknown fields are invalid/args. +// @ts-expect-error: text must be a string +await attempt("wrong argument", () => echo.echo({ text: 1 }, context)); +// @ts-expect-error: an unknown field +await attempt("unknown field", () => echo.echo({ text: "hi", extra: true }, context)); + +// Params select the object and are checked on every get: invalid/params. +// @ts-expect-error: id must be an integer +await attempt("bad params", () => host.get(Echo, { id: "one" }, context)); + +// An implementation with a member its contract does not declare fails activation. +await attempt("unknown member", () => host.get(Sloppy, { id: 1 }, context)); + +// A get needs the provider's version and fingerprint; another declaration of the type does not fit. +const EchoV2 = local("example.echo", { version: 2, params: { id: int() }, members: { id: int() } }); +await attempt("contract mismatch", () => host.get(EchoV2, { id: 1 }, context)); + +// A factory that throws fails broken: the get rejects at once with its error, and no object is left behind. +await attempt("throwing factory", () => host.get(Broken, { id: 1 }, context)); + +// After release, every use of a reference throws released. +echo.release(); +await attempt("released reference", () => echo.echo({ text: "hi" }, context)); + +await host.close(context); +await attempt("get after close", () => host.get(Echo, { id: 1 }, context)); diff --git a/packages/chord/test/next/examples/06-natives.ts b/packages/chord/test/next/examples/06-natives.ts new file mode 100644 index 000000000..8e74f06d6 --- /dev/null +++ b/packages/chord/test/next/examples/06-natives.ts @@ -0,0 +1,96 @@ +// Natives: values an object opens during activation and closes when it ends (a file watcher, a connection, a timer). +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/06-natives.ts +import { setTimeout as sleep } from "node:timers/promises"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { int, local, method, str } from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; + +const context = BACKGROUND_CONTEXT; + +const Clock = local("example.clock", { + version: 1, + params: { zone: str() }, + members: { zone: str(), now: method(undefined, str()) }, +}); +const Leaky = local("example.leaky", { version: 1, params: {}, members: { n: int() } }); + +const facet = defineFacet((host) => { + host.serve(Clock, ({ zone }, self) => { + // `self.open(open, close)` declares a native; it opens after the references are live, in declaration order. + const config = self.open( + async () => { + console.log(`${zone}: open config`); + return { prefix: `[${zone}]` }; + }, + (_config, reason) => console.log(`${zone}: close config (${reason.kind})`), + ); + // A later opener reads an earlier native's value. Its context is aborted if the object ends while it opens. + const ticker = self.open( + async () => { + console.log(`${zone}: open ticker, config ${config.value.prefix}`); + const ticker = { ticks: 0, timer: setInterval(() => ticker.ticks++, 10) }; + return ticker; + }, + (ticker, reason) => { + clearInterval(ticker.timer); + console.log(`${zone}: close ticker (${reason.kind})`); + }, + ); + // The factory only declares: a value is there once its open resolved. + try { + config.value; + } catch (error) { + if (isChordError(error, "not_ready")) console.log(`${zone}: in the factory, ${error.message}`); + } + return { + zone: constant(zone), + now: async () => `${config.value.prefix} ${ticker.value.ticks > 0 ? "ticking" : "starting"}`, + }; + }); + // Closes that throw or hang: they never stop the other closes. + host.serve(Leaky, (_, self) => { + self.open( + async () => "socket", + () => { + throw new Error("socket close failed"); + }, + ); + self.open( + async () => "worker", + () => new Promise(() => {}), + { closeTimeout: 50 }, + ); + return { n: constant(1) }; + }); +}); + +// onError gets what fails in the background: here a close past its closeTimeout. +const host = createHost({ + onError: (error) => console.log("reported:", error.kind, "object" in error ? error.object : ""), +}); +await host.load([{ name: "@example/natives", facet }], context); + +const vienna = await host.get(Clock, { zone: "vienna" }, context); +console.log("now:", await vienna.now(context)); +await sleep(30); +console.log("now:", await vienna.now(context)); + +// The last reference gone, the object ends: natives close in reverse order, with reason `end`. +vienna.release(); +await sleep(0); + +// host.close ends every object at once and closes their natives with reason `close`; it rejects with every close +// that threw. +await host.get(Clock, { zone: "tokyo" }, context); +try { + await host.close(context); +} catch (error) { + if (error instanceof AggregateError) + console.log( + "host.close:", + error.errors.map((e: Error) => e.message), + ); +} diff --git a/packages/chord/test/next/examples/07-failures.ts b/packages/chord/test/next/examples/07-failures.ts new file mode 100644 index 000000000..2d8bb9697 --- /dev/null +++ b/packages/chord/test/next/examples/07-failures.ts @@ -0,0 +1,96 @@ +// Failures: how an activation fails (transient or broken), how Chord retries it, and what getters see. +// Run from packages/chord (takes about 3 s: the retries wait 1 s, then 2 s): +// node --conditions=source --experimental-strip-types test/next/examples/07-failures.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { int, local, str } from "../../../src/next/contracts/index.ts"; +import { broken, isChordError } from "../../../src/next/data/index.ts"; +import { createHost, defineFacet, type HostError } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; + +const context = BACKGROUND_CONTEXT; +const start = Date.now(); +const at = (): string => `${((Date.now() - start) / 1000).toFixed(1)} s`; + +// A connection to a remote service, which is down for the first two attempts. +const Remote = local("example.remote", { version: 1, params: {}, members: { status: str() } }); +// An inbox per user, which needs the connection. +const Inbox = local("example.inbox", { version: 1, params: { user: str() }, members: { status: str() } }); +// A mirror whose configuration is missing: retrying cannot help. +const Mirror = local("example.mirror", { version: 1, params: { id: int() }, members: { status: str() } }); +// A service whose open never finishes. +const Stuck = local("example.stuck", { version: 1, params: { id: int() }, members: { status: str() } }); + +let attempts = 0; +const facet = defineFacet((host) => { + host.serve(Remote, (_, self) => { + const attempt = ++attempts; + self.open( + async () => { + console.log(`${at()}: remote, attempt ${attempt}`); + // A throwing open is transient: Chord retries it after 1 s, then 2 s, 4 s, ... up to 60 s. + if (attempt <= 2) throw new Error("connection refused"); + return { connected: true }; + }, + () => {}, + ); + return { status: constant("connected") }; + }); + host.serve(Inbox, ({ user }, self) => { + // A dependency that fails fails this object transient too; it is retried the moment the dependency is live. + const remote = self.get(Remote, {}); + console.log(`${at()}: inbox ${user}, attempt`); + return { status: self.derive([remote.status], ([status]) => `${user}: ${status}`) }; + }); + host.serve(Mirror, (_, self) => { + self.open( + async () => { + // broken(error) says retrying is pointless: no retry, and getters are rejected at once. + throw broken(new Error("no mirror configured")); + }, + () => {}, + ); + return { status: constant("mirroring") }; + }); + host.serve(Stuck, (_, self) => { + self.open( + () => new Promise(() => {}), + () => {}, + ); + return { status: constant("never") }; + }); +}); + +// Every failed attempt is reported, with its object and cause. +const onError = (error: HostError): void => { + if (error.kind === "activation_failed") { + console.log(`${at()}: reported ${error.object}, ${error.failure.kind}: ${error.failure.error.message}`); + } +}; + +/** Prints how a get ended. */ +async function attempt(label: string, work: () => Promise<{ status: { value: string } }>): Promise { + try { + const reference = await work(); + console.log(`${at()}: ${label}: ${reference.status.value}`); + } catch (error) { + if (isChordError(error)) console.log(`${at()}: ${label}: ${error.code} - ${error.message}`); + } +} + +// Every attempt must be live within readyTimeout (here 500 ms); getters wait up to it too, unless they say otherwise. +const host = createHost({ onError, readyTimeout: 500 }); +// The start barrier: load resolves once the remote (empty params: it lives as long as the host) had its first outcome. +await host.load([{ name: "@example/failures", facet }], context); +console.log(`${at()}: loaded`); + +// A broken failure rejects the get at once. +await attempt("mirror", () => host.get(Mirror, { id: 1 }, context)); + +// An attempt not live within readyTimeout is abandoned and fails transient; this get waits longer (700 ms) and then +// rejects with the last failure's error. +await attempt("stuck", () => host.get(Stuck, { id: 1 }, context, { readyTimeout: 700 })); + +// Transient failures keep a get waiting through the retries, up to its deadline: the inbox goes live with the remote. +await attempt("inbox", () => host.get(Inbox, { user: "mario" }, context, { readyTimeout: 10_000 })); + +await host.close(context); diff --git a/packages/chord/test/next/examples/08-remote.ts b/packages/chord/test/next/examples/08-remote.ts new file mode 100644 index 000000000..c88d81e15 --- /dev/null +++ b/packages/chord/test/next/examples/08-remote.ts @@ -0,0 +1,111 @@ +// Remote objects over one link: a server and a client host, a remote contract with two variants, a factory choosing +// the variant per principal over a shared local store, and the client's replicas following the server's commits. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/08-remote.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { mapShared } from "../../../src/next/state/index.ts"; + +const context = BACKGROUND_CONTEXT; + +// The app's principals: who calls. Links stamp them; a caller never sets its own. +type P = { kind: "owner" } | { kind: "guest"; session: string } | { kind: "server" }; + +const Note = obj({ text: str(), author: str() }); +const PublicNote = obj({ text: str() }); + +// A remote contract: one object per (type, params, principal key), in the variant its factory chooses. +const Notes = remote("example.notes", { + version: 1, + params: { session: str() }, + variants: { + editor: { state: { notes: arr(Note), count: int() }, methods: { add: method(obj({ text: str() })) } }, + reader: { state: { notes: arr(PublicNote), count: int() } }, + }, +}); + +// The shared store: the notes of a session once, behind per-principal views (design §6). Local types never cross a link. +const NoteStore = local("example.note-store", { + version: 1, + params: { session: str() }, + members: { notes: arr(Note), add: method(Note) }, +}); + +const NotesDoc = defineState("notes", { schema: obj({ list: arr(Note) }), version: 1, initial: () => ({ list: [] }) }); + +// A redaction computed once per store element, so a reader's derive keeps unchanged notes identical. +const redact = mapShared((note: { text: string; author: string }) => ({ text: note.text })); + +const facet = defineFacet

((host) => { + host.serve(NoteStore, (_, self) => { + const doc = self.state(NotesDoc); + return { + notes: self.derive([doc], ([d]) => d.list), + add: async (note, ctx) => { + await doc.change((draft) => void draft.list.push(note), ctx); + return null; + }, + }; + }); + // A remote factory gets the principal its object is for and returns one variant with exactly its members. + host.serve(Notes, ({ session }, principal, self) => { + const store = self.get(NoteStore, { session }); + const count = self.derive([store.notes], ([n]) => n.length); + if (principal.kind === "guest") { + return { variant: "reader", members: { notes: self.derive([store.notes], ([n]) => redact(n)), count } }; + } + return { + variant: "editor", + members: { + notes: store.notes, + count, + add: async ({ text }, ctx) => store.add({ text, author: principal.kind }, ctx), + }, + }; + }); +}); + +// 1. The server: it acts as `server` for its own requests. +const server = createHost

({ principal: { kind: "server" }, onError: (error) => console.error("server:", error) }); +await server.load([{ name: "@example/notes", facet }], context); + +// 2. A client (a tab): no principal of its own. The entry point authenticated it as a guest of S1 and stamps that on +// the link; the client connects before it loads (its plugins may reference server types). +const [up, down] = memoryLinkPair(); +server.accept(up, { principal: { kind: "guest", session: "S1" } }, context); +const client = createHost

({ onError: (error) => console.error("client:", error) }); +client.connect(down, { principal: { kind: "server" } }, context); +await client.load([], context); + +// 3. The server writes as an editor; the guest reads the redacted view over the link. +const own = await server.get(Notes, { session: "S1" }, context); +if (own.variant === "editor") await own.add({ text: "hello" }, context); +const notes = await client.get(Notes, { session: "S1" }, context); +console.log("guest variant:", notes.variant); +console.log("guest notes:", JSON.stringify(notes.notes.value), "count:", notes.count.value); + +// 4. A commit on the server reaches the client as one state message: both members change in the same batch. +const watch = await notes.notes.watch(context); +watch.start(async (call) => { + if (call.kind === "value") console.log("frame:", JSON.stringify(call.value), "ops:", JSON.stringify(call.ops)); +}); +if (own.variant === "editor") await own.add({ text: "world" }, context); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("count after the commit:", notes.count.value); + +// 5. TypeScript narrows on `variant`: a reader has no `add` (a forged call would fail `invalid/member` at the server). +if (notes.variant === "reader") { + // @ts-expect-error: `add` is not a member of the reader variant + void notes.add; +} + +// 6. Closing the server closes its links first: the client's reference goes loading and keeps its last values until +// a new link holds it again (10-drops.ts). Closing the client releases it and ends its watches. +await server.close(context); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("after the server closed:", notes.state.value, "count:", notes.count.value); +await client.close(context); +console.log("after the client closed:", notes.state.value, await watch.closed); diff --git a/packages/chord/test/next/examples/09-calls.ts b/packages/chord/test/next/examples/09-calls.ts new file mode 100644 index 000000000..01dfbbe14 --- /dev/null +++ b/packages/chord/test/next/examples/09-calls.ts @@ -0,0 +1,107 @@ +// Calls over a link: results, bytes as raw segments, ChordErrors and plugin errors, the caller's principal and +// incarnation in `ctx`, cancellation, and release rejecting the calls made through a reference. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/09-calls.ts +import { BACKGROUND_CONTEXT, withCancel } from "../../../src/context/index.ts"; +import { bytes, int, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { isChordError, PluginError } from "../../../src/next/data/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; + +const context = BACKGROUND_CONTEXT; + +type P = { kind: "owner" } | { kind: "server" }; + +const Files = remote("example.files", { + version: 1, + params: {}, + variants: { + owner: { + methods: { + whoami: method(undefined, str()), + // A `bytes()` leaf: the argument and result travel as raw segments, never base64. + upload: method(obj({ name: str(), data: bytes() }), int()), + download: method(obj({ name: str() }), bytes()), + quota: method(), + slow: method(), + }, + }, + }, +}); + +const stored = new Map(); + +const facet = defineFacet

((host) => { + host.serve(Files, () => ({ + variant: "owner", + members: { + // ctx.principal is the caller as its link stamped it; ctx.incarnation the calling host's run (R20). + whoami: async (ctx) => `${ctx.principal?.kind} from ${ctx.incarnation.slice(0, 8)}…`, + upload: async ({ name, data }) => { + stored.set(name, data); + return data.byteLength; + }, + download: async ({ name }) => stored.get(name) ?? new Uint8Array(), + // A PluginError crosses with its code, message and details; any other throw as code `internal`. + quota: async () => { + throw new PluginError("quota", "the store is full", { limit: 1 }); + }, + // A handler's context is aborted when its caller cancels or releases the reference. + slow: (ctx) => + new Promise((resolve) => + ctx.abortSignal?.addEventListener("abort", () => { + console.log("server: slow aborted:", (ctx.abortSignal?.reason as Error).message); + resolve(null); + }), + ), + }, + })); +}); + +const server = createHost

({ principal: { kind: "server" }, onError: (error) => console.error("server:", error) }); +await server.load([{ name: "@example/files", facet }], context); +const [up, down] = memoryLinkPair(); +server.accept(up, { principal: { kind: "owner" } }, context); +const client = createHost

({ onError: (error) => console.error("client:", error) }); +client.connect(down, { principal: { kind: "server" } }, context); +await client.load([], context); + +const files = await client.get(Files, {}, context); + +// 1. A call resolves with its result; a method without arguments takes only the context. +console.log("whoami:", await files.whoami(context)); + +// 2. Bytes: 1 MiB goes out as one raw segment next to the JSON part. +const data = new Uint8Array(1024 * 1024).fill(7); +console.log("uploaded bytes:", await files.upload({ name: "chart.png", data }, context)); +const back = await files.download({ name: "chart.png" }, context); +console.log("downloaded:", back.byteLength, "bytes, a", back.constructor.name); + +// 3. Errors: arguments are checked at the provider (a ChordError); a plugin's own error keeps its code. +try { + // @ts-expect-error: `name` must be a string + await files.upload({ name: 1, data }, context); +} catch (error) { + if (isChordError(error, "invalid")) console.log("refused:", error.reason, error.message); +} +try { + await files.quota(context); +} catch (error) { + if (error instanceof PluginError) console.log("plugin error:", error.code, error.message, error.details); +} + +// 4. Cancellation: the caller is rejected at once, the provider aborts the handler. +const { context: cancellable, cancel } = withCancel(context); +const slow = files.slow(cancellable); +setTimeout(() => cancel(new Error("the user gave up")), 5); +await slow.catch((error: unknown) => console.log("client: slow rejected:", (error as Error).message)); + +// 5. Release rejects the calls made through the reference; the provider aborts their handlers. +const pending = files.slow(context); +await new Promise((resolve) => setTimeout(resolve, 5)); +files.release(); +await pending.catch((error: unknown) => console.log("client: pending call:", (error as Error).message)); +await new Promise((resolve) => setTimeout(resolve, 5)); + +await client.close(context); +await server.close(context); diff --git a/packages/chord/test/next/examples/10-drops.ts b/packages/chord/test/next/examples/10-drops.ts new file mode 100644 index 000000000..701961c45 --- /dev/null +++ b/packages/chord/test/next/examples/10-drops.ts @@ -0,0 +1,107 @@ +// Surviving drops: a reference held before any link exists, `ref.state` as it follows the link, calls parked while +// loading and sent in order once live, a call in flight when the link drops, and the re-hold after `connect` again. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/10-drops.ts +import { BACKGROUND_CONTEXT, type Context } from "../../../src/context/index.ts"; +import { arr, int, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import { type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; + +const context = BACKGROUND_CONTEXT; + +type P = { kind: "owner" } | { kind: "server" }; + +const Notes = remote("example.notes", { + version: 1, + params: { session: str() }, + variants: { + editor: { + state: { notes: arr(str()) }, + methods: { add: method(obj({ text: str() }), int()), slow: method() }, + }, + }, +}); + +const NotesDoc = defineState("notes", { schema: obj({ list: arr(str()) }), version: 1, initial: () => ({ list: [] }) }); + +const facet = defineFacet

((host) => { + // Lingers for a minute after its last reference: a client that reconnects within it finds its notes again. + host.serve( + Notes, + (_, _principal, self) => { + const doc = self.state(NotesDoc); + return { + variant: "editor", + members: { + notes: self.derive([doc], ([d]) => d.list), + add: async ({ text }, ctx) => { + await doc.change((draft) => void draft.list.push(text), ctx); + return doc.value.list.length; + }, + slow: (ctx) => new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", () => resolve(null))), + }, + }; + }, + { linger: 60_000 }, + ); +}); + +const server = createHost

({ principal: { kind: "server" }, onError: (error) => console.error("server:", error) }); +await server.load([{ name: "@example/notes", facet }], context); +const client = createHost

({ onError: (error) => console.error("client:", error) }); +await client.load([], context); + +/** A new link between the hosts: the server stamps the client as its owner. Returns the server's end. */ +function connect(server: Host

, client: Host

, context: Context): Link { + const [up, down] = memoryLinkPair(); + server.accept(up, { principal: { kind: "owner" } }, context); + client.connect(down, { principal: { kind: "server" } }, context); + return up; +} + +const settle = () => new Promise((resolve) => setTimeout(resolve, 10)); + +// 1. A hold before any link: a reference at once, `loading`, its variant unknown. Calls on it park. +const notes = client.hold(Notes, { session: "S1" }, context); +console.log("held without a link:", notes.state.value, "variant:", notes.variant); +const watch = await notes.state.watch(context); +watch.start(async (call) => { + if (call.kind === "value") console.log(" ref.state →", call.value.kind); +}); +const parked = notes.add({ text: "written while loading" }, context); + +// 2. Connect: the reference is held over the link, goes live, and the parked call is sent. +let link = connect(server, client, context); +console.log("parked call's result once live:", await parked); +console.log("notes:", notes.notes.value, "variant:", notes.variant); + +// 3. The link drops (the server's end closes it). A call in flight fails `unavailable` and is never resent; the +// reference goes loading and keeps its last values. +const inFlight = notes.slow(context).catch((error: Error) => `rejected: ${error.message}`); +await settle(); +link.close(); +console.log("in-flight call:", await inFlight); +await settle(); +console.log("after the drop:", notes.state.value, "last notes:", notes.notes.value); + +// 4. Calls made while the link is down park; they wait at most the reference's readyTimeout (60 s here). +const first = notes.add({ text: "first while down" }, context); +const second = notes.add({ text: "second while down" }, context); + +// 5. The app connects again (its link adapter would): every reference is held again with fresh ids, its members get +// the new snapshot as one frame `[["r", value]]`, then the parked calls are sent in the order they were made. +const frames = await notes.notes.watch(context); +frames.start(async (call) => { + if (call.kind === "value") console.log(" notes frame:", JSON.stringify(call.ops)); +}); +link = connect(server, client, context); +console.log("parked results, in order:", await first, await second); +await settle(); +console.log("notes after the re-hold:", notes.notes.value); + +// 6. Release: `ref.state` is `ended` (its error says the holder released it) and its watch ends. +notes.release(); +console.log("watch ended:", await watch.closed, "state:", notes.state.value); +await client.close(context); +await server.close(context); diff --git a/packages/chord/test/next/examples/11-linger-and-retry.ts b/packages/chord/test/next/examples/11-linger-and-retry.ts new file mode 100644 index 000000000..391d55f64 --- /dev/null +++ b/packages/chord/test/next/examples/11-linger-and-retry.ts @@ -0,0 +1,109 @@ +// `linger` and `ref.retry()`: an object that stays warm after its last reference, failed references with `retryAt`, +// a Retry button that skips the backoff, and a broken object renewed by a retry. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/11-linger-and-retry.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { broken } from "../../../src/next/data/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type RefState } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; + +const context = BACKGROUND_CONTEXT; + +const Draft = local("example.draft", { + version: 1, + params: { conversation: str() }, + members: { text: str(), type: method(obj({ text: str() })) }, +}); + +const Mirror = local("example.mirror", { version: 1, params: { id: int() }, members: { status: str() } }); + +const DraftDoc = defineState("draft", { schema: obj({ text: str() }), version: 1, initial: () => ({ text: "" }) }); + +// What the mirror's open finds: nothing configured, then a server that is down, then one that answers. +const remote = { configured: false, up: false }; + +const facet = defineFacet((host) => { + // 1. A draft lingers 200 ms after its last reference: a tab that closes and reopens quickly finds its text. + host.serve( + Draft, + ({ conversation }, self) => { + console.log(` (draft ${conversation} activated)`); + const doc = self.state(DraftDoc); + return { + text: self.derive([doc], ([d]) => d.text), + type: async ({ text }, ctx) => { + await doc.change((draft) => { + draft.text = text; + }, ctx); + return null; + }, + }; + }, + { linger: 200 }, + ); + host.serve(Mirror, (_, self) => { + self.open( + async () => { + // `broken`: retrying cannot help until someone changes something (no automatic retry). + if (!remote.configured) throw broken(new Error("no mirror configured")); + // A plain throw is transient: retried with backoff (1 s, doubling). + if (!remote.up) throw new Error("mirror unreachable"); + return null; + }, + () => {}, + ); + return { status: constant("mirroring") }; + }); +}); + +const host = createHost({ onError: (error) => console.log(` onError: ${error.kind}`) }); +await host.load([{ name: "@example/app", facet }], context); +const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** A state for the log: a failure with its kind, when it retries by itself, and its error. */ +function describe(state: RefState): string { + if (state.kind !== "failed") return state.kind; + const retry = state.retryAt === undefined ? "no automatic retry" : `retries in ${state.retryAt - Date.now()} ms`; + return `failed, ${state.cause.kind}, ${retry}: ${state.cause.error.message}`; +} + +// 2. Linger: release, get again within 200 ms, the same object and its in-memory document. +let draft = await host.get(Draft, { conversation: "c1" }, context); +await draft.type({ text: "hello" }, context); +draft.release(); +await wait(100); +draft = await host.get(Draft, { conversation: "c1" }, context); +console.log("within linger:", JSON.stringify(draft.text.value)); +draft.release(); +await wait(300); +draft = await host.get(Draft, { conversation: "c1" }, context); +console.log("after linger:", JSON.stringify(draft.text.value)); +draft.release(); + +// 3. A held reference follows its object: broken first (no retryAt), so nothing happens by itself. +const mirror = host.hold(Mirror, { id: 1 }, context); +const watch = await mirror.state.watch(context); +watch.start(async (call) => { + if (call.kind === "value") console.log(` ref.state → ${describe(call.value)}`); +}); +await wait(10); + +// 4. The user fixes the configuration and presses Retry: the broken object is renewed and tries again. The server is +// still down: a transient failure, retried automatically at retryAt. +remote.configured = true; +mirror.retry(); +await wait(10); + +// 5. Retry again (a Retry button): the backoff window is skipped; this time it goes live. +remote.up = true; +mirror.retry(); +await wait(10); +console.log("status:", mirror.status.value); + +// 6. A retry while live does nothing. +mirror.retry(); +await wait(10); +mirror.release(); +await host.close(context); diff --git a/packages/chord/test/next/examples/12-routes.ts b/packages/chord/test/next/examples/12-routes.ts new file mode 100644 index 000000000..7074bd979 --- /dev/null +++ b/packages/chord/test/next/examples/12-routes.ts @@ -0,0 +1,160 @@ +// Routes: host kinds and addresses, a contract's placement, a server routing a session's objects to an in-process +// worker (one `resolve` per address, `authorize` per principal), a guest's `ctx.get` on the server reaching the worker +// as the guest, and a worker that drops and is resolved again. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/12-routes.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import { type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; + +const context = BACKGROUND_CONTEXT; + +type P = + | { kind: "owner" } + | { kind: "guest"; session: string } + | { kind: "server" } + | { kind: "worker"; session: string }; + +// Kinds of addressable hosts; `Session({ id: "S1" })` is an address, its key `session|S1`. +const Server = hostKind("server"); +const Session = hostKind("session", { id: str() }); + +// Notes live on their session's worker: the caller computes the address and sends it with its hold. +const Notes = remote("example.notes", { + version: 1, + params: { session: str() }, + placement: ({ session }) => Session({ id: session }), + variants: { + editor: { + state: { notes: arr(str()) }, + methods: { add: method(obj({ text: str() }), int()), whoami: method(undefined, str()) }, + }, + reader: { state: { notes: arr(str()) }, methods: { whoami: method(undefined, str()) } }, + }, +}); + +// The lobby lives on the server (no placement): its handler gets notes for whoever calls it. +const Lobby = remote("example.lobby", { + version: 1, + params: {}, + variants: { visitor: { methods: { peek: method(obj({ session: str() }), str()) } } }, +}); + +// The worker's own store of its session's notes, shared by every principal's `Notes` object. +const NoteStore = local("example.note-store", { + version: 1, + params: {}, + members: { list: arr(str()), add: method(obj({ text: str() }), int()) }, +}); + +const NotesDoc = defineState("notes", { schema: obj({ list: arr(str()) }), version: 1, initial: () => ({ list: [] }) }); + +/** The worker's facet: an owner edits, anyone else reads; `whoami` shows whom a call acts for. */ +const workerFacet = defineFacet

((host) => { + host.serve(NoteStore, (_, self) => { + const doc = self.state(NotesDoc); + return { + list: self.derive([doc], ([d]) => d.list), + add: async ({ text }, ctx) => { + await doc.change((draft) => void draft.list.push(text), ctx); + return doc.value.list.length; + }, + }; + }); + host.serve(Notes, (_, principal, self) => { + const store = self.get(NoteStore, {}); + const whoami = async (ctx: { readonly principal: P | undefined }) => JSON.stringify(ctx.principal); + if (principal.kind !== "owner") return { variant: "reader", members: { notes: store.list, whoami } }; + return { variant: "editor", members: { notes: store.list, add: (args, ctx) => store.add(args, ctx), whoami } }; + }); +}); + +const serverFacet = defineFacet

((host) => { + host.serve(Lobby, () => ({ + variant: "visitor", + members: { + // `ctx.get` acts as the caller: over the route, the worker sees the guest, not the server. + peek: async ({ session }, ctx) => { + const notes = await ctx.get(Notes, { session }); + return `${notes.variant} variant, ${await notes.whoami(ctx)}`; + }, + }, + })); +}); + +// 1. The server: an address, and a route for sessions. `resolve` starts an in-process worker over a memory link and +// returns the link at once (the worker connects first, then loads, W6). `authorize` admits owners everywhere and a +// guest only into its own session. +const server = createHost

({ address: Server(), principal: { kind: "server" }, onError: console.error }); +let resolves = 0; +let workerLink: Link | undefined; +server.route( + Session, + { + authorize: (principal, address) => + principal.kind === "owner" || (principal.kind === "guest" && principal.session === address.params.id), + resolve: async (address) => { + console.log(`server: resolving ${address.key} (resolve #${++resolves})`); + const [up, down] = memoryLinkPair(); + const worker = createHost

({ + address, + principal: { kind: "worker", session: address.params.id }, + onError: console.error, + }); + worker.connect(down, { principal: { kind: "server" }, trusted: true }, context); + void worker.load([{ name: "@example/worker", facet: workerFacet }], context); + workerLink = up; + return { link: up, principal: { kind: "worker", session: address.params.id }, trusted: true }; + }, + }, + context, +); +await server.load([{ name: "@example/server", facet: serverFacet }], context); + +/** A tab on the server, stamped as `principal` by the server's entry point. */ +async function tabAs(principal: P): Promise> { + const [up, down] = memoryLinkPair(); + server.accept(up, { principal }, context); + const tab = createHost

({ onError: console.error }); + tab.connect(down, { principal: { kind: "server" } }, context); + await tab.load([], context); + return tab; +} + +// 2. The owner's tab gets S1's notes: tab → server (relays) → route → worker. A second get shares the worker. +const owner = await tabAs({ kind: "owner" }); +const notes = await owner.get(Notes, { session: "S1" }, context); +if (notes.variant === "editor") await notes.add({ text: "milk" }, context); +const again = await owner.get(Notes, { session: "S1" }, context); +console.log("owner:", notes.variant, again.notes.value, "resolves so far:", resolves); + +// 3. A guest of S1 reads S1 and is refused S2 by `authorize` (no worker is started for it). +const guest = await tabAs({ kind: "guest", session: "S1" }); +const read = await guest.get(Notes, { session: "S1" }, context); +console.log("guest:", read.variant, read.notes.value, "acts as", await read.whoami(context)); +await guest.get(Notes, { session: "S2" }, context).catch((error) => { + console.log("guest, S2:", isChordError(error) ? error.code : error, "resolves so far:", resolves); +}); + +// 4. The guest calls the lobby on the server; its handler's `ctx.get` goes over the route as the guest. +const lobby = await guest.get(Lobby, {}, context); +console.log("guest via the lobby:", await lobby.peek({ session: "S1" }, context)); + +// 5. The worker drops: references go loading, the route resolves the address again after its backoff (1 s), and +// the new worker (a fresh run, empty notes here) binds them again. +const states: string[] = []; +const watch = await notes.state.watch(context); +watch.start(async (call) => { + if (call.kind === "value") states.push(call.value.kind); +}); +await new Promise((resolve) => setTimeout(resolve, 0)); +workerLink!.close(); +while (states.length < 2 || states.at(-1) !== "live") await new Promise((resolve) => setTimeout(resolve, 50)); +console.log("after the drop:", states.join(" → "), "notes:", notes.notes.value, "resolves:", resolves); + +await owner.close(context); +await guest.close(context); +await server.close(context); diff --git a/packages/chord/test/next/examples/13-relay.ts b/packages/chord/test/next/examples/13-relay.ts new file mode 100644 index 000000000..6203d3f52 --- /dev/null +++ b/packages/chord/test/next/examples/13-relay.ts @@ -0,0 +1,146 @@ +// A relay over real transports: a tab on a loopback WebSocket, a server that relays, and a worker per session forked +// over IPC. The worker gets its start data from its start message and serves it as `SessionInfo` through its core +// facet (§18); the server routes by address and parses no part. +// Run from packages/chord (the same file is the worker when its first argument is `worker`): +// node --conditions=source --experimental-strip-types test/next/examples/13-relay.ts +import { fork } from "node:child_process"; +import type { AddressInfo } from "node:net"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; +import { WebSocketServer } from "ws"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { type IpcPort, ipcLink, webSocketLink } from "../../../src/next/link/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; + +const context = BACKGROUND_CONTEXT; + +type P = { kind: "owner" } | { kind: "server" } | { kind: "worker"; session: string }; + +const Server = hostKind("server"); +const Session = hostKind("session", { id: str() }); + +const Init = obj({ id: str(), title: str(), pid: int() }); + +/** Start data: a local object with empty params (host lifetime) and constant state, served by the core facet. */ +const SessionInfo = local("example.session-info", { version: 1, params: {}, members: { init: Init } }); + +const Notes = remote("example.notes", { + version: 1, + params: { session: str() }, + placement: ({ session }) => Session({ id: session }), + variants: { + owner: { state: { info: Init, notes: arr(str()) }, methods: { add: method(obj({ text: str() }), int()) } }, + }, +}); + +/** What the server sends a worker it forked: the app's own message, not the link's. */ +type Start = { readonly id: string; readonly title: string }; + +/** The worker's core facet, made from its start message (base code, loaded under a fixed name). */ +const coreWorkerFacet = (init: { id: string; title: string; pid: number }) => + defineFacet

((host) => { + host.serve(SessionInfo, () => ({ init: constant(init) })); + }); + +const NotesDoc = defineState("notes", { schema: obj({ list: arr(str()) }), version: 1, initial: () => ({ list: [] }) }); + +/** A plugin's facet on the worker: its notes, and the start data read through a reference. */ +const notesFacet = defineFacet

((host) => { + host.serve(Notes, (_, _principal, self) => { + const doc = self.state(NotesDoc); + const info = self.get(SessionInfo, {}); + return { + variant: "owner", + members: { + info: info.init, + notes: self.derive([doc], ([d]) => d.list), + add: async ({ text }, ctx) => { + await doc.change((draft) => void draft.list.push(text), ctx); + return doc.value.list.length; + }, + }, + }; + }); +}); + +if (process.argv[2] === "worker") { + // The worker. 1. Its link first: it queues the link's messages until the host listens, and skips the app's own. + const link = ipcLink(process as unknown as IpcPort); + const start = await new Promise((resolve) => { + const listener = (message: unknown) => { + if (message instanceof Uint8Array) return; + process.off("message", listener); + // The start message the server's `resolve` sent: the app's own shape. + resolve(message as Start); + }; + process.on("message", listener); + }); + // 2. Its host at its address: connect first (W6), then load the core facet with the start data, then the plugins. + const worker = createHost

({ + address: Session({ id: start.id }), + principal: { kind: "worker", session: start.id }, + onError: console.error, + }); + worker.connect(link, { principal: { kind: "server" }, trusted: true }, context); + await worker.load( + [ + { name: "@example/core", facet: coreWorkerFacet({ ...start, pid: process.pid }) }, + { name: "@example/notes", facet: notesFacet }, + ], + context, + ); + // The server closing the link ends the worker. + link.onClose(() => process.exit(0)); +} else { + // The server. 3. A route forks a worker per session, sends it its start message and returns the IPC link at once. + const server = createHost

({ address: Server(), principal: { kind: "server" }, onError: console.error }); + server.route( + Session, + { + authorize: (principal) => principal.kind === "owner", + // No `signal` for the fork: the context is aborted when the host closes, which would kill the worker (and emit + // `error` on the child); a link Chord closes, at `host.close` or because it arrived after it, ends the worker. + resolve: async (address) => { + const child = fork(fileURLToPath(import.meta.url), ["worker"], { + serialization: "advanced", + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + }); + child.send({ id: address.params.id, title: `Session ${address.params.id}` } satisfies Start); + console.log(`server: forked pid ${child.pid} for ${address.key}`); + return { link: ipcLink(child), principal: { kind: "worker", session: address.params.id }, trusted: true }; + }, + }, + context, + ); + await server.load([], context); + + // 4. Tabs reach the server over a WebSocket; its entry point stamps each as the owner here. + const wss = new WebSocketServer({ host: "127.0.0.1", port: 0 }); + await new Promise((resolve) => wss.once("listening", resolve)); + wss.on("connection", (socket) => server.accept(webSocketLink(socket), { principal: { kind: "owner" } }, context)); + const { port } = wss.address() as AddressInfo; + + // 5. A tab: Node's own WebSocket, as a browser's. Its holds go tab → server (relay) → worker. + const tab = createHost

({ onError: console.error }); + tab.connect(webSocketLink(new WebSocket(`ws://127.0.0.1:${port}`)), { principal: { kind: "server" } }, context); + await tab.load([], context); + const s1 = await tab.get(Notes, { session: "S1" }, context); + console.log("tab: S1 info", s1.info.value, "server pid", process.pid); + await s1.add({ text: "milk" }, context); + await s1.add({ text: "bread" }, context); + await new Promise((resolve) => setTimeout(resolve, 10)); + console.log("tab: S1 notes", s1.notes.value); + + // 6. Another session is another worker process. + const s2 = await tab.get(Notes, { session: "S2" }, context); + console.log("tab: S2 info", s2.info.value, "notes", s2.notes.value); + + // 7. Closing the server closes its route links; each worker exits when its link closes. + await tab.close(context); + await server.close(context); + await new Promise((resolve) => wss.close(resolve)); + console.log("closed"); +} diff --git a/packages/chord/test/next/examples/14-flow-control.ts b/packages/chord/test/next/examples/14-flow-control.ts new file mode 100644 index 000000000..4eba6e1ee --- /dev/null +++ b/packages/chord/test/next/examples/14-flow-control.ts @@ -0,0 +1,208 @@ +// Flow control: a slow tab. At most two of a provider's own `bound`/`state` messages are unacknowledged per origin; +// while the window is full the provider folds the ops of every commit into its pending queues and sends them as one +// message when an `ack` frees a slot. A relay folds and drops nothing: it forwards at once under the tab's origin and +// returns each `ack` to the worker that sent the message, so a slow tab folds at the worker, alone. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/14-flow-control.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { hostKind, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import { binaryCodec, describeMessage, type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; + +const context = BACKGROUND_CONTEXT; + +type P = { kind: "owner" } | { kind: "server" } | { kind: "worker"; session: string }; + +const Server = hostKind("server"); +const Session = hostKind("session", { id: str() }); + +/** The worker's own document of a streamed reply, and its one write. */ +const Reply = local("example.reply", { + version: 1, + params: {}, + members: { reply: obj({ text: str() }), append: method(str()) }, +}); + +/** What a tab holds: the reply, on its session's worker. */ +const View = remote("example.view", { + version: 1, + params: { session: str() }, + placement: ({ session }) => Session({ id: session }), + variants: { owner: { state: { reply: obj({ text: str() }) } } }, +}); + +const ReplyDoc = defineState("reply", { schema: obj({ text: str() }), version: 1, initial: () => ({ text: "" }) }); + +const workerFacet = defineFacet

((host) => { + host.serve(Reply, (_, self) => { + const doc = self.state(ReplyDoc); + return { + reply: doc, + append: async (token, ctx) => { + await doc.change((draft) => { + draft.text += token; + }, ctx); + return null; + }, + }; + }); + host.serve(View, (_params, _principal, self) => ({ + variant: "owner", + members: { reply: self.get(Reply, {}).reply }, + })); +}); + +/** A worker for `id`, loaded. */ +async function worker(id: string): Promise> { + const host = createHost

({ + address: Session({ id }), + principal: { kind: "worker", session: id }, + onError: console.error, + }); + await host.load([{ name: "@example/worker", facet: workerFacet }], context); + return host; +} + +/** A link end that logs, as `label`, every `bound`, `state` and `ack` its host receives. */ +function tap(link: Link, label: string): Link { + return { + send: (message) => link.send(message), + onMessage: (listener) => + link.onMessage((message) => { + const decoded = binaryCodec.decode(message); + const shown = decoded.kind === "ack" || decoded.kind === "bound" || decoded.kind === "state"; + if (shown) console.log(` ${label} ←`, describeMessage(decoded).slice(0, 110)); + listener(message); + }), + close: () => link.close(), + onClose: (listener) => link.onClose(listener), + }; +} + +/** A slow tab's end of its link: while paused, what arrives waits (so the tab acknowledges nothing). */ +class Pausable implements Link { + paused = false; + readonly #link: Link; + readonly #waiting: Uint8Array[] = []; + #listener: (message: Uint8Array) => void = () => {}; + + constructor(link: Link) { + this.#link = link; + link.onMessage((message) => { + if (this.paused) this.#waiting.push(message); + else this.#listener(message); + }); + } + + resume(): void { + this.paused = false; + for (const message of this.#waiting.splice(0)) this.#listener(message); + } + + send(message: Uint8Array): void { + this.#link.send(message); + } + + onMessage(listener: (message: Uint8Array) => void): void { + this.#listener = listener; + } + + close(): void { + this.#link.close(); + } + + onClose(listener: () => void): void { + this.#link.onClose(listener); + } +} + +/** A tab connected over `link`, loaded. */ +async function tab(link: Link): Promise> { + const host = createHost

({ onError: console.error }); + host.connect(link, { principal: { kind: "server" } }, context); + await host.load([], context); + return host; +} + +const settle = () => new Promise((resolve) => setTimeout(resolve, 10)); + +// 1. Folding at the provider: a tab directly on a worker. Paused, it acknowledges nothing; the worker sends two +// `state` messages (the window), then folds 1,000 appends into its pending queue. +console.log("a slow tab on the provider:"); +const w1 = await worker("S1"); +const [up, down] = memoryLinkPair(); +w1.accept(up, { principal: { kind: "owner" } }, context); +const slow = new Pausable(down); +const t1 = await tab(tap(slow, "tab")); +const view1 = await t1.get(View, { session: "S1" }, context); +const reply1 = await w1.get(Reply, {}, context); +slow.paused = true; +for (let i = 0; i < 1_000; i++) await reply1.append(" tok", context); +await settle(); +console.log( + " paused: the tab shows", + view1.reply.value.text.length, + "characters, the worker has", + reply1.reply.value.text.length, +); + +// 2. Resumed: the tab applies and acknowledges the two, and the freed slot carries one `state` with one folded append. +slow.resume(); +await settle(); +console.log(" resumed: the tab shows", view1.reply.value.text.length, "characters"); + +// 3. The same slow tab through a server: its holds reach the worker with the tab's ids under the tab link's origin +// (`["@",1,…]`), the worker's window for that origin fills and it folds there; a second, fast tab is another origin and +// receives every commit. +console.log("a slow tab through a relay:"); +const server = createHost

({ address: Server(), principal: { kind: "server" }, onError: console.error }); +let w2: Host

| undefined; +server.route( + Session, + { + authorize: () => true, + resolve: async (address) => { + const [a, b] = memoryLinkPair(); + w2 = await worker(address.params.id); + w2.connect(tap(b, "worker"), { principal: { kind: "server" }, trusted: true }, context); + return { link: a, principal: { kind: "worker", session: address.params.id }, trusted: true }; + }, + }, + context, +); +await server.load([], context); +const [up2, down2] = memoryLinkPair(); +server.accept(up2, { principal: { kind: "owner" } }, context); +const slow2 = new Pausable(down2); +const t2 = await tab(tap(slow2, "tab")); +const view2 = await t2.get(View, { session: "S2" }, context); +const [up3, down3] = memoryLinkPair(); +server.accept(up3, { principal: { kind: "owner" } }, context); +const t3 = await tab(down3); +const view3 = await t3.get(View, { session: "S2" }, context); +let fast = 0; +const watch = await view3.reply.watch(context); +watch.start(async () => { + fast++; +}); +const reply2 = await w2!.get(Reply, {}, context); +slow2.paused = true; +for (let i = 0; i < 5; i++) await reply2.append(` t${i}`, context); +await settle(); +console.log(" paused: the slow tab shows", view2.reply.value, "the worker has", reply2.reply.value); +console.log(" the fast tab saw", fast, "changes:", view3.reply.value); + +// 4. Resumed: the slow tab acknowledges the two; the relay returns each `ack` to the worker with the origin, and the +// worker's freed slot carries one `state` with the three folded appends. +slow2.resume(); +await settle(); +console.log(" resumed: the slow tab shows", view2.reply.value); + +await t1.close(context); +await w1.close(context); +await watch.stop(); +await t2.close(context); +await t3.close(context); +await server.close(context); +await w2!.close(context); diff --git a/packages/chord/test/next/examples/15-revocation.ts b/packages/chord/test/next/examples/15-revocation.ts new file mode 100644 index 000000000..1a004ffa6 --- /dev/null +++ b/packages/chord/test/next/examples/15-revocation.ts @@ -0,0 +1,142 @@ +// Revocation: the app says which revocations apply to a principal (`revokedBy`, the same function on every host); +// `host.revoke(r)` ends that principal's objects at once, whatever references remain, closes its child links (tabs) and +// sends `revoke` down every route link, where each worker applies it with its own `revokedBy`. Nothing is replayed. +// What ends gets the error `forbidden` with reason `revoked` (a UI can say "access revoked"): holders get it in `gone`, +// running handlers in their aborted context. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/15-revocation.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, hostKind, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import type { ChordError } from "../../../src/next/data/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; + +const context = BACKGROUND_CONTEXT; + +/** The app's principals: a guest holds a grant to one session; the server and its workers act as themselves. */ +type P = + | { kind: "owner" } + | { kind: "guest"; grant: string; session: string } + | { kind: "server" } + | { kind: "worker"; session: string }; + +/** What the app revokes: a grant, or everything of a session. */ +type Revocation = { grant: string } | { session: string }; + +/** The same function on every host: a guest is revoked by its grant and by its session. */ +const revokedBy = (p: P): Revocation[] => (p.kind === "guest" ? [{ grant: p.grant }, { session: p.session }] : []); + +const Server = hostKind("server"); +const Session = hostKind("session", { id: str() }); + +const Notes = remote("example.notes", { + version: 1, + params: { session: str() }, + placement: ({ session }) => Session({ id: session }), + variants: { + all: { state: { notes: arr(str()) }, methods: { add: method(obj({ text: str() })), follow: method() } }, + }, +}); + +const NotesDoc = defineState("notes", { schema: obj({ list: arr(str()) }), version: 1, initial: () => ({ list: [] }) }); + +const notesFacet = defineFacet

((host) => { + host.serve(Notes, (_, principal, self) => { + const doc = self.state(NotesDoc); + console.log(` worker: Notes for ${principal.kind}${principal.kind === "guest" ? ` ${principal.grant}` : ""}`); + return { + variant: "all", + members: { + notes: self.derive([doc], ([d]) => d.list), + add: async ({ text }, ctx) => { + await doc.change((draft) => void draft.list.push(text), ctx); + return null; + }, + // A long-running call: it ends when its context is aborted, and says why. + follow: async (ctx) => { + await new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", resolve)); + const { code, reason } = ctx.abortSignal!.reason as ChordError; + console.log(` worker: ${principal.kind}'s follow aborted: ${code}/${reason}`); + return null; + }, + }, + }; + }); +}); + +// 1. A server with `revokedBy`, routing sessions to in-process workers (trusted links) with the same `revokedBy`. +const server = createHost({ + address: Server(), + principal: { kind: "server" }, + revokedBy, + onError: console.error, +}); +const workers: Host[] = []; +server.route( + Session, + { + authorize: (p, address) => p.kind === "owner" || (p.kind === "guest" && p.session === address.params.id), + resolve: async (address) => { + const [up, down] = memoryLinkPair(); + const worker = createHost({ + address, + principal: { kind: "worker", session: address.params.id }, + revokedBy, + onError: console.error, + }); + worker.connect(up, { principal: { kind: "server" }, trusted: true }, context); + void worker.load([{ name: "@example/notes", facet: notesFacet }], context); + workers.push(worker); + return { link: down, principal: { kind: "worker", session: address.params.id }, trusted: true }; + }, + }, + context, +); +await server.load([], context); + +/** A tab whose link the server's entry point stamped with `principal`. */ +async function tab(principal: P) { + const host = createHost

({ onError: console.error }); + await host.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal }, context); + host.connect(b, { principal: { kind: "server" } }, context); + return host; +} + +// 2. Two tabs of guest grant g1 (one person, two windows) and the owner's tab, all on session S1. +const [one, two, owner] = await Promise.all([ + tab({ kind: "guest", grant: "g1", session: "S1" }), + tab({ kind: "guest", grant: "g1", session: "S1" }), + tab({ kind: "owner" }), +]); +const guestNotes = await one.get(Notes, { session: "S1" }, context); +await two.get(Notes, { session: "S1" }, context); +const ownerNotes = await owner.get(Notes, { session: "S1" }, context); +await guestNotes.add({ text: "from the guest" }, context); +const following = guestNotes.follow(context).catch((error: ChordError) => error.code); +// The call reaches the worker and runs there. +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("worker S1 references:", workers[0]!.references(Notes).value.length); + +// 3. Revoke the grant: the worker gets `revoke { grant: "g1" }` and ends the guest's objects there (their running +// calls aborted with forbidden/revoked), and both guest tabs' links close. +server.revoke({ grant: "g1" }, context); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("guest tab:", guestNotes.state.value.kind, "(its link closed; a new one needs a new authentication)"); +console.log("guest's follow call:", await following); +// The owner's Notes is its own object (identities include the principal), untouched. +console.log("owner tab:", ownerNotes.state.value.kind, ownerNotes.notes.value); +console.log( + "worker S1 references:", + workers[0]!.references(Notes).value.map(({ principal }) => principal?.kind), +); + +// 4. A worker resolved later gets no past revocation: its objects are new, and the revoked links are closed already. +const s2 = await owner.get(Notes, { session: "S2" }, context); +console.log("owner on S2:", s2.state.value.kind); + +await Promise.all([one, two, owner].map((host) => host.close(context))); +await server.close(context); +await Promise.all(workers.map((worker) => worker.close(context))); diff --git a/packages/chord/test/next/examples/16-presence-and-activity.ts b/packages/chord/test/next/examples/16-presence-and-activity.ts new file mode 100644 index 000000000..9091557e0 --- /dev/null +++ b/packages/chord/test/next/examples/16-presence-and-activity.ts @@ -0,0 +1,89 @@ +// Presence and keep-alive counts. `host.references(Type)` is a JSON watchable of the references to objects of a type +// on this host, one entry per reference with the object's params, its principal and its incarnation (two tabs are two +// origins); a derive over it is a viewer count, and a link drop is a departure in the same batch. `host.activity` counts +// what keeps a host busy (holds over links, running calls, jobs, lingering objects), published at most once per +// microtask turn: the app's supervisor decides when an idle worker stops. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/16-presence-and-activity.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { int, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; + +const context = BACKGROUND_CONTEXT; + +type P = { kind: "user"; name: string } | { kind: "server" }; + +/** A shared document view: who else is looking (`viewers`), and a slow method. */ +const Doc = remote("example.doc", { + version: 1, + params: { id: str() }, + variants: { v: { state: { viewers: int() }, methods: { think: method(undefined, int()) } } }, +}); + +// 1. The server serves `Doc`; its factory derives the viewer count from `host.references(Doc)` (on the facet's host). +const server = createHost

({ principal: { kind: "server" }, onError: console.error }); +const facet = defineFacet

((host) => { + host.serve( + Doc, + ({ id }, _principal, self) => ({ + variant: "v", + members: { + viewers: self.derive([host.references(Doc)], ([refs]) => refs.filter((ref) => ref.params.id === id).length), + think: async (ctx) => { + await new Promise((resolve) => setTimeout(resolve, 20)); + return ctx.principal?.kind === "user" ? 42 : 0; + }, + }, + }), + { linger: 50 }, + ); +}); +await server.load([{ name: "@example/doc", facet }], context); + +// 2. The supervisor's view: every publication of the counts. +const activity = await server.activity.watch(context); +activity.start(async (call) => { + if (call.kind === "value") console.log(" activity:", JSON.stringify(call.value)); +}); + +/** A tab of `name` on a new link; returns the tab and the server's end of its link. */ +async function tab(name: string) { + const host = createHost

({ onError: console.error }); + await host.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "user", name } }, context); + host.connect(b, { principal: { kind: "server" } }, context); + return { host, link: a }; +} + +// 3. Ada opens the document in two tabs, Bob in one: three references, three incarnations, two objects (one per principal); +// the viewer count is over every object of the document. +const [ada1, ada2, bob] = await Promise.all([tab("ada"), tab("ada"), tab("bob")]); +const doc = await ada1.host.get(Doc, { id: "d1" }, context); +await ada2.host.get(Doc, { id: "d1" }, context); +await bob.host.get(Doc, { id: "d1" }, context); +await new Promise((resolve) => setTimeout(resolve, 5)); +console.log("viewers seen by ada:", doc.viewers.value); +const present = server + .references(Doc) + .value.map( + ({ principal, incarnation }) => `${principal?.kind === "user" ? principal.name : "?"} @ ${incarnation.slice(-4)}`, + ); +console.log("references:", present); + +// 4. A call running counts as activity until it settles. +await doc.think(context); + +// 5. Bob's link drops: a departure, at once. Bob's own `Doc` object (identities include the principal) lingers. +bob.link.close(); +await new Promise((resolve) => setTimeout(resolve, 5)); +console.log("viewers after bob's link dropped:", doc.viewers.value); + +// 6. Ada closes both tabs: the object lingers (50 ms) with no references, then ends; then everything is zero. +await ada1.host.close(context); +await ada2.host.close(context); +await new Promise((resolve) => setTimeout(resolve, 100)); +console.log("references at the end:", server.references(Doc).value.length); +await activity.stop(); +await server.close(context); diff --git a/packages/chord/test/next/examples/17-limits-and-compression.ts b/packages/chord/test/next/examples/17-limits-and-compression.ts new file mode 100644 index 000000000..970dce626 --- /dev/null +++ b/packages/chord/test/next/examples/17-limits-and-compression.ts @@ -0,0 +1,189 @@ +// Size limits per part and compression at the origin. A host with a synchronous compressor (`nodeCompression` on +// Node) deflates each JSON part above 16 KiB once, when it encodes it; relays forward it as it is; the end that owns it +// inflates it, within its link's decoded limit. A browser has no synchronous deflate: it sends uncompressed and +// inflates with `DecompressionStream`, later messages of its link waiting in order. Only a part that shrinks is sent +// deflated, and never hold params. Clients never send compressed parts: a host sends its call arguments up its parent +// link uncompressed, and a link the server accepted (a child) refuses compressed parts as a protocol error (a small +// part could inflate to 64 MiB there). A part past a limit fails only its own hold or call (`too_large/part`); the +// link stays. Limits and the threshold count UTF-8 bytes. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/17-limits-and-compression.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import { type Compression, type Link, memoryLinkPair, type PartLimits } from "../../../src/next/link/index.ts"; +import { nodeCompression } from "../../../src/next-node/compression.ts"; + +const context = BACKGROUND_CONTEXT; + +type P = { kind: "user" } | { kind: "server" }; + +const Entry = obj({ id: str(), text: str() }); + +/** The shared store of a conversation: its entries, once, and the method that appends to them. */ +const Store = local("example.store", { + version: 1, + params: { id: str() }, + members: { entries: arr(Entry), append: method(Entry) }, +}); + +/** What a tab holds: the conversation's entries, and appending. */ +const Conversation = remote("example.conversation", { + version: 1, + params: { id: str() }, + variants: { v: { state: { entries: arr(Entry) }, methods: { append: method(Entry) } } }, +}); + +const Entries = defineState("entries", { + schema: obj({ list: arr(Entry) }), + version: 1, + initial: () => ({ list: [] }), +}); + +const facet = defineFacet

((host) => { + host.serve(Store, (_, self) => { + const doc = self.state(Entries); + return { + entries: self.derive([doc], ([d]) => d.list), + append: async (entry, ctx) => { + await doc.change((draft) => void draft.list.push(entry), ctx); + return null; + }, + }; + }); + host.serve(Conversation, ({ id }, _principal, self) => { + const store = self.get(Store, { id }); + return { variant: "v", members: { entries: store.entries, append: (entry, ctx) => store.append(entry, ctx) } }; + }); +}); + +/** A link end counting the bytes its host receives. */ +function counted(link: Link, bytes: { received: number }): Link { + return { + send: (message) => link.send(message), + onMessage: (listener) => + link.onMessage((message) => { + bytes.received += message.byteLength; + listener(message); + }), + close: () => link.close(), + onClose: (listener) => link.onClose(listener), + }; +} + +/** + * A server compressing with `compression` and a tab without a compressor (a browser) on a link the server accepts with + * `limits`; the tab's own link to the server has `tabLimits`. + */ +async function setUp(compression: Compression | undefined, limits?: PartLimits, tabLimits?: PartLimits) { + const server = createHost

({ principal: { kind: "server" }, compression, onError: console.error }); + await server.load([{ name: "@example/conversation", facet }], context); + const tab = createHost

({ onError: console.error }); + await tab.load([], context); + const [a, b] = memoryLinkPair(); + const bytes = { received: 0 }; + server.accept(a, { principal: { kind: "user" }, limits }, context); + tab.connect(counted(b, bytes), { principal: { kind: "server" }, limits: tabLimits }, context); + return { server, tab, bytes }; +} + +/** Words for entries that compress like prose (about 3x), not like a repeated string. */ +const WORDS = "the a model tool call file read write test error result agent session note code run build".split(" "); + +/** Entry `i`'s text: about 1 KB of words picked by a fixed pseudo-random sequence. */ +function text(i: number): string { + let seed = i * 7919 + 1; + const words: string[] = []; + for (let length = 0; length < 1000; length += words.at(-1)!.length + 1) { + seed = (seed * 1103515245 + 12345) % 2147483648; + const word = WORDS[seed % WORDS.length]!; + words.push(seed % 5 === 0 ? `${word}${seed % 1000}` : word); + } + return words.join(" "); +} + +/** Fills the server's conversation `id` with 500 entries of about 1 KB (its store kept by this reference). */ +async function fill(server: Host

, id: string): Promise { + const conversation = await server.get(Store, { id }, context); + for (let i = 0; i < 500; i++) await conversation.append({ id: `e${i}`, text: text(i) }, context); +} + +// 1. A 500 KB snapshot to a browser tab, without and with compression at the server. +for (const compression of [undefined, nodeCompression]) { + const { server, tab, bytes } = await setUp(compression); + await fill(server, "c1"); + const started = performance.now(); + const conversation = await tab.get(Conversation, { id: "c1" }, context); + const ms = (performance.now() - started).toFixed(1); + const label = compression === undefined ? "uncompressed" : "deflated at the server, inflated asynchronously"; + console.log( + `snapshot ${label}: ${conversation.entries.value.length} entries, ${(bytes.received / 1024).toFixed(1)} KiB received, ${ms} ms`, + ); + await tab.close(context); + await server.close(context); +} + +// 2. Limits per part: the server admits at most 64 KiB per part from this tab. A larger argument fails its call only. +{ + const { server, tab } = await setUp(nodeCompression, { part: 64 * 1024 }); + const conversation = await tab.get(Conversation, { id: "c2" }, context); + try { + await conversation.append({ id: "big", text: "x".repeat(100_000) }, context); + } catch (error) { + console.log( + "a 100 KB argument:", + isChordError(error) ? `${error.code}/${error.reason}: ${error.message}` : error, + ); + } + await conversation.append({ id: "small", text: "fits" }, context); + console.log( + "the link stays: entries", + conversation.entries.value.map(({ id }) => id), + ); + await tab.close(context); + await server.close(context); +} + +// 3. A compression bomb: 50 KB deflated that would inflate to 6 MB, past the tab's decoded limit of 1 MiB. The tab +// stops inflating at the limit; only that hold fails. +{ + const { server, tab } = await setUp(nodeCompression, undefined, { decoded: 1024 * 1024 }); + const bomb = await server.get(Store, { id: "bomb" }, context); + for (let i = 0; i < 6; i++) await bomb.append({ id: `z${i}`, text: "0".repeat(1_000_000) }, context); + await fill(server, "fine"); + try { + await tab.get(Conversation, { id: "bomb" }, context); + } catch (error) { + console.log("the bomb's hold:", isChordError(error) ? `${error.code}/${error.reason}` : error); + } + const fine = await tab.get(Conversation, { id: "fine" }, context); + console.log("another hold on the same link:", fine.entries.value.length, "entries"); + await tab.close(context); + await server.close(context); +} + +// 4. A client with a compressor still sends its arguments up uncompressed: the server, which accepted its link as a +// child, would refuse a compressed part. Its 100 KB argument arrives as JSON, within the server's default limits. +{ + const server = createHost

({ + principal: { kind: "server" }, + compression: nodeCompression, + onError: console.error, + }); + await server.load([{ name: "@example/conversation", facet }], context); + const client = createHost

({ compression: nodeCompression, onError: console.error }); + await client.load([], context); + const [a, b] = memoryLinkPair(); + const bytes = { received: 0 }; + server.accept(counted(a, bytes), { principal: { kind: "user" } }, context); + client.connect(b, { principal: { kind: "server" } }, context); + const conversation = await client.get(Conversation, { id: "c4" }, context); + await conversation.append({ id: "big", text: text(0).repeat(100) }, context); + console.log( + `a compressing client's 100 KB argument: ${(bytes.received / 1024).toFixed(1)} KiB received by the server`, + ); + await client.close(context); + await server.close(context); +} diff --git a/packages/chord/test/next/examples/18-routines.ts b/packages/chord/test/next/examples/18-routines.ts new file mode 100644 index 000000000..8ff6b8d65 --- /dev/null +++ b/packages/chord/test/next/examples/18-routines.ts @@ -0,0 +1,115 @@ +// Routines: an object's standing behaviour, declared in its factory. `self.on(source, fn)` reconciles from a value: the +// first call gets the current value (`[["r", value]]`), then one call per change, never overlapping; a call that +// throws is reported (`routine_failed`) and called again with the whole current value after a backoff (1 s doubling to +// 60 s), or at once when the source changes. `self.onLive(fn)` runs once the object is live: a loop with `sleep` is a +// periodic task; its ctx is aborted when the object ends, which `sleep` turns into a rejection, so nothing is left +// running. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/18-routines.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, bool, local, map, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, sleep } from "../../../src/next/host/index.ts"; + +const context = BACKGROUND_CONTEXT; + +/** The catalog of sessions: another owner's state, which the forwards reconcile from. */ +const Catalog = local("example.catalog", { + version: 1, + params: {}, + members: { sessions: map(obj({ archived: bool() })), archive: method(obj({ session: str() })) }, +}); + +/** Port forwards per session; a forward of an archived session must be closed (an invariant across owners, §9). */ +const ForwardStore = local("example.forwards", { + version: 1, + params: {}, + members: { open: arr(str()), cleanups: arr(str()) }, +}); + +const Sessions = defineState("example.sessions", { + schema: obj({ sessions: map(obj({ archived: bool() })) }), + version: 1, + initial: () => ({ sessions: { s1: { archived: false }, s2: { archived: false } } }), +}); + +const Forwards = defineState("example.forward-list", { + schema: obj({ open: arr(str()), cleanups: arr(str()) }), + version: 1, + initial: () => ({ open: ["s1", "s2"], cleanups: [] }), +}); + +const started = Date.now(); +const at = () => `${Math.round((Date.now() - started) / 100) / 10}s`; + +// 1. The host reports background failures: a routine call that threw arrives as `routine_failed`. +const host = createHost({ + onError: (error) => { + if (error.kind === "routine_failed") { + const { kind, position } = error.routine; + console.log(` [${at()}] ${error.object} ${kind} #${position} failed: ${(error.error as Error).message}`); + } else console.error(error); + }, +}); + +let flaky = true; +const facet = defineFacet((h) => { + h.serve(Catalog, (_, self) => { + const doc = self.state(Sessions); + return { + sessions: self.derive([doc], ([d]) => d.sessions), + archive: async ({ session }, ctx) => { + await doc.change((d) => { + d.sessions[session] = { archived: true }; + }, ctx); + return null; + }, + }; + }); + h.serve(ForwardStore, (_, self) => { + const forwards = self.state(Forwards); + const catalog = self.get(Catalog, {}); + // 2. Reconcile: close every forward of an archived session. Idempotent, from the current value; the first close + // fails once (a flaky remote), and the retry comes with the whole value. + self.on(catalog.sessions, async ({ value: sessions, ops }, ctx) => { + console.log(` [${at()}] on: ${JSON.stringify(ops)}`); + for (const session of forwards.value.open) { + if (!sessions[session]?.archived) continue; + if (flaky) { + flaky = false; + throw new Error(`closing the forward of ${session} timed out`); + } + await forwards.change((d) => { + d.open = d.open.filter((open) => open !== session); + }, ctx); + console.log(` [${at()}] closed the forward of ${session}`); + } + }); + // 3. A periodic task: a loop with `sleep`, ended by the object's end (the abort rejects `sleep`). + self.onLive(async (ctx) => { + for (;;) { + await sleep(400, ctx); + await forwards.change((d) => void d.cleanups.push(at()), ctx); + } + }); + return { + open: self.derive([forwards], ([d]) => d.open), + cleanups: self.derive([forwards], ([d]) => d.cleanups), + }; + }); +}); +await host.load([{ name: "@example/forwards", facet }], context); +const store = await host.get(ForwardStore, {}, context); +const catalog = await host.get(Catalog, {}, context); +await sleep(50, context); + +// 4. Archive s2: the routine's call throws once, is retried after 1 s with the whole value, and closes the forward. +console.log("archiving s2"); +await catalog.archive({ session: "s2" }, context); +await sleep(1_300, context); +console.log("open forwards:", store.open.value); +console.log("periodic cleanups so far:", store.cleanups.value); + +// 5. Closing the host aborts the routines: the periodic loop ends in its `sleep`, nothing is reported. +await host.close(context); +console.log("closed; nothing left running"); diff --git a/packages/chord/test/next/examples/19-jobs.ts b/packages/chord/test/next/examples/19-jobs.ts new file mode 100644 index 000000000..f52ee317a --- /dev/null +++ b/packages/chord/test/next/examples/19-jobs.ts @@ -0,0 +1,211 @@ +// Jobs: owned work a call starts that outlives the call, in Durable's sandwich: intent (`start`, one transaction), run +// (any effect, outside transactions) and finish (one transaction on the current code, with the cleanup). Here a shell +// job of a shared chat store: its state document carries the output, throttled by the author and flushed in a +// `finally`, so the last output reaches `outcome.state` after an abort too; `running` is ordinary state, which +// per-principal views filter (a guest sees no output); `finish` records an entry and chains a second job. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/19-jobs.ts +import { BACKGROUND_CONTEXT, type Context } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, remote, type Static, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, sleep } from "../../../src/next/host/index.ts"; +import { defineJob } from "../../../src/next/jobs/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { mapShared } from "../../../src/next/state/index.ts"; +import { OutputBuffer, Progress } from "../../../src/output.ts"; + +const context = BACKGROUND_CONTEXT; + +type P = { kind: "owner" } | { kind: "guest" } | { kind: "server" }; + +// 1. The job: closed args (checked at every start), a result, a state document per run, one run at a time. +const ShellState = defineState("example.shell.state", { + schema: obj({ output: str() }), + version: 1, + initial: () => ({ output: "" }), +}); +const ShellJob = defineJob("example.shell", { + version: 1, + args: obj({ command: str(), lines: int() }), + result: obj({ exit: int(), output: str() }), + state: ShellState, + concurrency: 1, +}); +/** A second job `finish` chains: it announces a finished command. */ +const NotifyJob = defineJob("example.notify", { version: 1, args: obj({ text: str() }), result: str() }); + +const Entries = defineState("example.entries", { + schema: obj({ list: arr(str()) }), + version: 1, + initial: () => ({ list: [] }), +}); + +/** A shell as its owner sees it; a guest's view has no output. */ +const Shell = obj({ id: str(), phase: str(), command: str(), output: str() }); +const GuestShell = obj({ id: str(), phase: str(), command: str() }); + +/** The shared store: the entries and the shells once, whoever views them (§6). */ +const ChatStore = local("example.chat-store", { + version: 1, + params: {}, + members: { + entries: arr(str()), + shells: arr(Shell), + runShell: method(obj({ command: str(), lines: int(), requestId: str() }), str()), + abortShell: method(obj({ id: str() })), + }, +}); + +/** Per-principal views: the owner runs shells and sees their output, a guest sees which shells run. */ +const Chat = remote("example.chat", { + version: 1, + params: {}, + variants: { + owner: { + state: { entries: arr(str()), shells: arr(Shell) }, + methods: { + runShell: method(obj({ command: str(), lines: int(), requestId: str() }), str()), + abortShell: method(obj({ id: str() })), + }, + }, + guest: { state: { entries: arr(str()), shells: arr(GuestShell) } }, + }, +}); + +/** A fake process: one line every 30 ms; `sleep` rejects when the run is aborted. */ +async function runProcess(lines: number, ctx: Context, onLine: (line: string) => void): Promise { + for (let n = 1; n <= lines; n++) { + await sleep(30, ctx); + onLine(`line ${n}\n`); + } + return 0; +} + +const facet = defineFacet

((host) => { + host.serve(ChatStore, (_, self) => { + const entries = self.state(Entries); + const notify = self.job(NotifyJob, { + run: async ({ text }) => `notified: ${text}`, + finish: async (outcome, tx) => { + if (outcome.kind === "done") (await tx.draft(entries)).list.push(outcome.result); + }, + }); + const shell = self.job(ShellJob, { + run: async ({ lines }, ctx) => { + // The output's bounds are required: a tail of at most 2,000 lines and 64 KiB. + const out = new OutputBuffer({ maxBytes: 64 * 1024, maxLines: 2_000, retain: "tail" }); + // Every state change is a commit and, while watched, a message: the author throttles them (50 ms here). + const write = async () => { + const text = out.snapshot().text; + await ctx.state.change((draft) => { + draft.output = text; + }, ctx); + return text.length; + }; + const progress = new Progress(write, (error) => console.error("progress failed:", error), 50); + try { + const exit = await runProcess(lines, ctx, (line) => { + out.push(line); + progress.mark(); + }); + return { exit, output: out.snapshot().text }; + } finally { + // The last output reaches `outcome.state`, also after an abort (changes are taken until run settles). + await progress.stop(); + await write(); + } + }, + finish: async (outcome, tx) => { + const { command } = outcome.args; + const lines = outcome.state.output.split("\n").length - 1; + const text = + outcome.kind === "done" + ? `${command}: exit ${outcome.result.exit}, ${lines} lines` + : `${command}: interrupted (${outcome.error.kind}), ${lines} lines kept`; + (await tx.draft(entries)).list.push(text); + // Chaining: the next job starts atomically with this outcome (both commit or neither). + tx.start(notify, { text: command }); + }, + }); + // What the views read: one object per run, kept while the run's entry is the same object (`mapShared`). + const toShell = mapShared( + (entry: (typeof shell.running.value)[number]): Static => + entry.phase === "blocked" + ? { id: entry.id, phase: entry.phase, command: "?", output: "" } + : { id: entry.id, phase: entry.phase, command: entry.args.command, output: entry.state.output }, + ); + return { + entries: self.derive([entries], ([d]) => d.list), + shells: self.derive([shell.running], ([running]) => toShell(running)), + runShell: async ({ command, lines, requestId }, ctx) => + (await shell.start({ command, lines }, ctx, { key: requestId })).id, + abortShell: async ({ id }) => { + shell.abort(id); + return null; + }, + }; + }); + host.serve(Chat, (_, principal, self) => { + const store = self.get(ChatStore, {}); + if (principal.kind !== "owner") { + // A guest sees no output: the view drops it (computed once per variant in a real store, §12). + const strip = mapShared( + (s: Static): Static => ({ id: s.id, phase: s.phase, command: s.command }), + ); + return { + variant: "guest", + members: { entries: store.entries, shells: self.derive([store.shells], ([list]) => strip(list)) }, + }; + } + return { + variant: "owner", + members: { + entries: store.entries, + shells: store.shells, + runShell: (args, ctx) => store.runShell(args, ctx), + abortShell: (args, ctx) => store.abortShell(args, ctx), + }, + }; + }); +}); + +// 2. A server and two tabs, the owner's and a guest's. +const server = createHost

({ principal: { kind: "server" }, onError: (error) => console.error(error) }); +await server.load([{ name: "@example/chat", facet }], context); +async function tab(principal: P) { + const host = createHost

({ onError: (error) => console.error(error) }); + await host.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal }, context); + host.connect(b, { principal: { kind: "server" } }, context); + return host.get(Chat, {}, context); +} +const owner = await tab({ kind: "owner" }); +const guest = await tab({ kind: "guest" }); +if (owner.variant !== "owner") throw new Error("expected the owner's variant"); + +// 3. The owner runs a command; its request id dedupes a resent request (the same run comes back). +const id = await owner.runShell({ command: "npm test", lines: 5, requestId: "r1" }, context); +const again = await owner.runShell({ command: "npm test", lines: 5, requestId: "r1" }, context); +console.log("a resent request returns the same run:", again === id); +await sleep(100, context); +console.log("owner sees:", owner.shells.value); +console.log("guest sees:", guest.shells.value); + +// 4. One run at a time: a second command is refused while the first runs. +try { + await owner.runShell({ command: "npm run build", lines: 1, requestId: "r2" }, context); +} catch (error) { + console.log("second run refused:", (error as Error).message); +} +await sleep(300, context); +console.log("entries after the first run:", owner.entries.value); + +// 5. Abort is a request: the run stops at its next `sleep`, its last output is flushed, `finish` records it. +const long = await owner.runShell({ command: "tail -f log", lines: 100, requestId: "r3" }, context); +await sleep(100, context); +await owner.abortShell({ id: long }, context); +await sleep(100, context); +console.log("entries after the abort:", owner.entries.value); +console.log("running now:", owner.shells.value); +await server.close(context); diff --git a/packages/chord/test/next/examples/20-tables.ts b/packages/chord/test/next/examples/20-tables.ts new file mode 100644 index 000000000..5b626fe3a --- /dev/null +++ b/packages/chord/test/next/examples/20-tables.ts @@ -0,0 +1,101 @@ +// Tables: an object's rows in the host's store, declared with `defineTable`, written in transactions (with the +// object's documents, if it has any), read by key, by page with cursors, and counted. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/20-tables.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, bool, int, local, method, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { add, defineTable, eq, openStore, prefix, query, sqliteBackend } from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const context = BACKGROUND_CONTEXT; + +// Flat columns (str, int, num, bool, opt(...), json(...)), defaults, a key of non-opt str or int columns, and +// indexes, which end with the key. +const Edits = defineTable("edits", { + version: 1, + columns: { + id: str(), + path: str(), + at: int(), + hits: int({ default: 0 }), + done: bool({ default: false }), + note: opt(str()), + }, + key: ["id"], + indexes: { by_path: ["path", "at"] }, +}); + +const History = local("example.history", { + version: 1, + params: {}, + members: { + record: method(obj({ id: str(), path: str(), at: int() })), + hit: method(str()), + finish: method(str()), + hits: method(str(), int()), + open: method(obj({ after: opt(str()) }), obj({ ids: arr(str()), next: opt(str()) })), + finished: method(obj({}), int()), + }, +}); + +const facet = defineFacet((host) => { + // Tables need a stable identity: a local type with empty params (one object per host). + host.serve(History, (_, self) => { + const edits = self.table(Edits); + // A query of the table: a condition and an order (the key is the tie-break); immutable, built once. + const open = query(Edits) + .where((e) => eq(e.done, false)) + .orderBy((e) => e.at, "desc"); + return { + record: async (row, ctx) => { + // Writes are staged synchronously (a bad row throws here) and commit together in one step. + await self.tx(async (tx) => tx.put(edits, row), ctx); + return null; + }, + hit: async (id, ctx) => { + // A set is values or an expression over the row's columns, computed by the store. + await self.tx(async (tx) => tx.update(edits, { id }, (e) => ({ hits: add(e.hits, 1) })), ctx); + return null; + }, + finish: async (path, ctx) => { + // One statement for every matching row; its count is recorded with the write. + await self.tx(async (tx) => tx.updateWhere(edits, (e) => prefix(e.path, path), { done: true }), ctx); + return null; + }, + hits: async (id, ctx) => (await edits.get({ id }, ctx))?.hits ?? 0, + open: async ({ after }, ctx) => { + const page = await edits.page(open, 1, ctx, after === undefined ? {} : { after }); + return { ids: page.items.map((row) => row.id), ...(page.next === undefined ? {} : { next: page.next }) }; + }, + finished: async (_, ctx) => + edits.count( + query(Edits).where((e) => eq(e.done, true)), + ctx, + ), + }; + }); +}); + +// A store over SQLite (`:memory:` here, a file in an app): the app opens it, gives it to one host, and closes it. +const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context); +const host = createHost({ store, onError: (error) => console.error("background error:", error) }); +await host.load([{ name: "@example/history", facet }], context); +const history = await host.get(History, {}, context); + +for (let i = 0; i < 5; i++) { + await history.record({ id: `e${i}`, path: `src/${i % 2 === 0 ? "a" : "b"}.ts`, at: i }, context); +} +await history.hit("e1", context); +await history.hit("e1", context); +await history.finish("src/a", context); + +console.log("hits of e1:", await history.hits("e1", context)); +// Pages continue from an opaque cursor. +const first = await history.open({}, context); +console.log("open, first page:", first.ids); +console.log("open, next page:", (await history.open({ after: first.next }, context)).ids); +console.log("finished:", await history.finished({}, context)); + +await host.close(context); +await store.close(context); diff --git a/packages/chord/test/next/examples/21-persisted-documents.ts b/packages/chord/test/next/examples/21-persisted-documents.ts new file mode 100644 index 000000000..e9771455b --- /dev/null +++ b/packages/chord/test/next/examples/21-persisted-documents.ts @@ -0,0 +1,91 @@ +// Persisted documents: `persist: true` keeps an object's documents in the host's store, so a new host on the same +// store finds them again; `checkpointWhen` decides when a change is stored as a new base instead of a delta. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/21-persisted-documents.ts +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Reference } from "../../../src/next/host/index.ts"; +import { openStore, sqliteBackend } from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const context = BACKGROUND_CONTEXT; + +// A persisted singleton: loaded once when its object activates; every 20th stored change is a new base. +const Tags = defineState("tags", { + schema: obj({ tags: arr(str()) }), + version: 1, + initial: () => ({ tags: [] }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 20, +}); +// A persisted keyed document: each key loaded on its first read, then cached until idle for ten minutes. +const Notes = defineState("notes", { + schema: obj({ body: str() }), + version: 1, + initial: () => ({ body: "" }), + keyed: true, + persist: true, + checkpointWhen: () => false, +}); + +const Notebook = local("example.notebook", { + version: 1, + params: {}, + members: { + tags: obj({ tags: arr(str()) }), + write: method(obj({ id: str(), body: str(), tag: str() })), + read: method(str(), str()), + }, +}); + +const facet = defineFacet((host) => { + // Persisted documents need a stable identity, as tables do: a local type with empty params. + host.serve(Notebook, (_, self) => { + const tags = self.state(Tags); + const notes = self.state(Notes); + return { + tags, + write: async ({ id, body, tag }, ctx) => { + // Both documents commit in one store step, or neither does. + await self.tx(async (tx) => { + (await tx.draft(notes, id)).body = body; + const draft = await tx.draft(tags); + if (!draft.tags.includes(tag)) draft.tags.push(tag); + }, ctx); + return null; + }, + read: async (id, ctx) => (await notes.get(id, ctx))?.body ?? "(none)", + }; + }); +}); + +/** A host on the store at `path`, run until `work` is done with the notebook. */ +async function session(path: string, work: (notebook: Reference) => Promise) { + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context); + const host = createHost({ store, onError: (error) => console.error("background error:", error) }); + await host.load([{ name: "@example/notebook", facet }], context); + await work(await host.get(Notebook, {}, context)); + await host.close(context); + await store.close(context); +} + +const directory = await mkdtemp(join(tmpdir(), "chord-example-")); +const path = join(directory, "notebook.db"); +try { + await session(path, async (notebook) => { + await notebook.write({ id: "groceries", body: "oat milk", tag: "home" }, context); + await notebook.write({ id: "standup", body: "demo the store", tag: "work" }, context); + console.log("first host, tags:", notebook.tags.value.tags); + }); + // A new host on the same file: the documents come back from the store. + await session(path, async (notebook) => { + console.log("second host, tags:", notebook.tags.value.tags); + console.log("second host, standup:", await notebook.read("standup", context)); + }); +} finally { + await rm(directory, { recursive: true, force: true }); +} diff --git a/packages/chord/test/next/examples/22-migrations.ts b/packages/chord/test/next/examples/22-migrations.ts new file mode 100644 index 000000000..f9932070d --- /dev/null +++ b/packages/chord/test/next/examples/22-migrations.ts @@ -0,0 +1,100 @@ +// Table migrations: `evolveTable` appends a version with its edits (rename, add, json before an optional data step; +// drop, index, dropIndex after it). An object whose store holds an older version migrates it when it activates, one +// version at a time, in chunks of 100 rows; a fresh store gets the newest version directly. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/22-migrations.ts +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet, type Reference } from "../../../src/next/host/index.ts"; +import { + defineTable, + evolveTable, + openStore, + query, + sqliteBackend, + type TableDefinition, +} from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const context = BACKGROUND_CONTEXT; + +// Version 1, the base, written in full. +const EditsV1 = defineTable("edits", { version: 1, columns: { id: str(), file: str(), at: int() }, key: ["id"] }); +// Version 2: `file` becomes `path`, a `dir` column is added and filled from each row (the data step). +const EditsV2 = evolveTable(EditsV1, (t) => + t + .rename("file", "path") + .add("dir", str({ default: "" })) + .rows((row) => ({ dir: row.path.slice(0, row.path.lastIndexOf("/")) })), +); +// Version 3: `dir` is dropped again, an index is added. Data functions stay as deployed; history only grows. +const EditsV3 = evolveTable(EditsV2, (t) => t.drop("dir").index("by_path", ["path", "at"])); + +const History = local("example.history", { + version: 1, + params: {}, + members: { record: method(obj({ id: str(), at: int() })), dump: method(obj({}), arr(str())) }, +}); + +/** The facet of one deployment: its newest table version, and how it writes a row. */ +function facetOf(definition: TableDefinition, row: (id: string, at: number) => Record) { + return defineFacet((host) => { + host.serve(History, (_, self) => { + const edits = self.table(definition); + return { + record: async ({ id, at }, ctx) => { + // A deployment's rows match its own version (an example shortcut for two versions in one file). + await self.tx(async (tx) => tx.put(edits, row(id, at) as never), ctx); + return null; + }, + dump: async (_, ctx) => { + const lines: string[] = []; + for await (const row of edits.rows(query(definition), ctx)) lines.push(JSON.stringify(row)); + return lines; + }, + }; + }); + }); +} + +/** A host on the store at `path` serving `facet`, run until `work` is done. */ +async function deployment( + path: string, + facet: ReturnType, + work: (history: Reference) => Promise, +) { + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context); + const host = createHost({ store, onError: (error) => console.error("background error:", error) }); + // Objects with empty params activate at load: the migration runs before `load` resolves. + await host.load([{ name: "@example/history", facet }], context); + await work(await host.get(History, {}, context)); + await host.close(context); + await store.close(context); +} + +const directory = await mkdtemp(join(tmpdir(), "chord-example-")); +const path = join(directory, "history.db"); +try { + await deployment( + path, + facetOf(EditsV1, (id, at) => ({ id, file: `src/${id}.ts`, at })), + async (history) => { + for (let i = 0; i < 3; i++) await history.record({ id: `e${i}`, at: i }, context); + console.log("version 1:", await history.dump({}, context)); + }, + ); + // The next deployment declares version 3: the store goes through 2 (rename, add, rows) and 3 (drop, index). + await deployment( + path, + facetOf(EditsV3, (id, at) => ({ id, path: `lib/${id}.ts`, at })), + async (history) => { + await history.record({ id: "e9", at: 9 }, context); + console.log("version 3:", await history.dump({}, context)); + }, + ); +} finally { + await rm(directory, { recursive: true, force: true }); +} diff --git a/packages/chord/test/next/examples/23-keyed-watches.ts b/packages/chord/test/next/examples/23-keyed-watches.ts new file mode 100644 index 000000000..0a13e62c1 --- /dev/null +++ b/packages/chord/test/next/examples/23-keyed-watches.ts @@ -0,0 +1,84 @@ +// Keyed watches and `at(key)`: a keyed document is watched per key; `handle.at(key)` in the factory makes one key a +// watchable (a state member here) that reads `initial()` while absent. A retirement arrives as `initial()` with +// `retired: true`, so a watcher can tell "deleted" from "changed back". +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/23-keyed-watches.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState, type KeyedHandle } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; + +const context = BACKGROUND_CONTEXT; + +type Note = { body: string }; + +const Notes = defineState("notes", { + schema: obj({ body: str() }), + version: 1, + initial: (): Note => ({ body: "" }), + keyed: true, +}); + +const Notebook = local("example.notebook", { + version: 1, + params: {}, + members: { + // The note "pinned" as a state member: `initial()` while absent. + pinned: obj({ body: str() }), + write: method(obj({ id: str(), body: str() })), + remove: method(str()), + }, +}); + +// The example's own window into the object: its keyed handle, to watch other keys from outside. +let notes: KeyedHandle | undefined; + +const facet = defineFacet((host) => { + host.serve(Notebook, (_, self) => { + notes = self.state(Notes); + const handle = notes; + return { + // Pinned while this attempt lives, and loaded before the object goes live. + pinned: handle.at("pinned"), + write: async ({ id, body }, ctx) => { + await handle.change( + id, + (draft) => { + draft.body = body; + }, + ctx, + ); + return null; + }, + remove: async (id, ctx) => { + await handle.retire(id, ctx); + return null; + }, + }; + }); +}); + +const host = createHost({ onError: (error) => console.error("background error:", error) }); +await host.load([{ name: "@example/notebook", facet }], context); +const notebook = await host.get(Notebook, {}, context); + +// A watch of the state member: every value call carries the value, its ops and the retired flag. +const pinned = await notebook.pinned.watch(context); +pinned.start(async (call) => { + if (call.kind === "value") console.log("pinned:", call.value, call.retired ? "(retired)" : ""); +}); + +await notebook.write({ id: "pinned", body: "buy milk" }, context); +// A watch of one key; an absent key gives `undefined` (watch it once it exists, or pin it with `at`). +console.log("watch of an absent key:", await notes!.watch("later", context)); +await notebook.write({ id: "later", body: "call Anna" }, context); +const later = (await notes!.watch("later", context))!; +later.start(async (call) => { + if (call.kind === "value") console.log("later:", call.value, call.retired ? "(retired)" : ""); +}); +await notebook.remove("later", context); +await notebook.remove("pinned", context); + +// Frames are delivered after the commit, asynchronously: let them arrive before closing. +await new Promise((resolve) => setTimeout(resolve, 10)); +await host.close(context); diff --git a/packages/chord/test/next/examples/24-keyed-members.ts b/packages/chord/test/next/examples/24-keyed-members.ts new file mode 100644 index 000000000..37f13a08d --- /dev/null +++ b/packages/chord/test/next/examples/24-keyed-members.ts @@ -0,0 +1,64 @@ +// Keyed state members: a `local` contract exposes a keyed document set as one member, `keyed(schema)`. A consumer on +// the same host pins one key with `ref.member.at(key)` in its factory (a watchable, part of its own hydration), or +// reads and watches any key with `ref.member.get/watch(key, ctx)`. One source per key: consumers share it. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/24-keyed-members.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { keyed, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; + +const context = BACKGROUND_CONTEXT; + +const Profile = obj({ name: str(), status: str() }); + +const Profiles = defineState("profiles", { + schema: Profile, + version: 1, + initial: () => ({ name: "", status: "offline" }), + keyed: true, +}); + +// The directory: every user's profile, one keyed member. +const Directory = local("example.directory", { + version: 1, + params: {}, + members: { profiles: keyed(Profile), set: method(obj({ user: str(), name: str(), status: str() })) }, +}); + +// A badge per user: its own object, showing one profile. +const Badge = local("example.badge", { version: 1, params: { user: str() }, members: { label: str() } }); + +const facet = defineFacet((host) => { + host.serve(Directory, (_, self) => { + const profiles = self.state(Profiles); + return { + // The keyed handle is the member as it is. + profiles, + set: async ({ user, name, status }, ctx) => { + await profiles.change(user, (draft) => Object.assign(draft, { name, status }), ctx); + return null; + }, + }; + }); + host.serve(Badge, ({ user }, self) => { + // Pinned for this attempt; the badge goes live once the profile is loaded. + const profile = self.get(Directory, {}).profiles.at(user); + return { label: self.derive([profile], ([p]) => `${p.name || user} (${p.status})`) }; + }); +}); + +const host = createHost({ onError: (error) => console.error("background error:", error) }); +await host.load([{ name: "@example/directory", facet }], context); +const directory = await host.get(Directory, {}, context); +await directory.set({ user: "ana", name: "Ana", status: "online" }, context); + +const badge = await host.get(Badge, { user: "ana" }, context); +console.log("badge:", badge.label.value); +await directory.set({ user: "ana", name: "Ana", status: "away" }, context); +console.log("badge after a change:", badge.label.value); +// Any key, read where it is needed. +console.log("ref.profiles.get(bo):", await directory.profiles.get("bo", context)); + +badge.release(); +await host.close(context); diff --git a/packages/chord/test/next/examples/25-export-import.ts b/packages/chord/test/next/examples/25-export-import.ts new file mode 100644 index 000000000..f8dd8a812 --- /dev/null +++ b/packages/chord/test/next/examples/25-export-import.ts @@ -0,0 +1,102 @@ +// Export and import: `store.export(ctx)` yields the store's state as archive parts (lossless: tables with their rows +// and versions, the live documents), holding the commit queue while it runs; `encodeJsonl` turns them into JSONL. +// `importStore` checks an archive part by part and makes a new store from it in one backend transaction. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/25-export-import.ts +import { createReadStream, createWriteStream } from "node:fs"; +import { mkdtemp, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pipeline } from "node:stream/promises"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Reference } from "../../../src/next/host/index.ts"; +import { + decodeJsonl, + defineTable, + encodeJsonl, + importStore, + openStore, + type Store, + sqliteBackend, +} from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const context = BACKGROUND_CONTEXT; + +const Visits = defineTable("visits", { version: 1, columns: { page: str(), count: int() }, key: ["page"] }); +const Title = defineState("title", { + schema: obj({ text: str() }), + version: 1, + initial: () => ({ text: "untitled" }), + persist: true, + checkpointWhen: () => false, +}); + +const Site = local("example.site", { + version: 1, + params: {}, + members: { + title: obj({ text: str() }), + visit: method(str(), int()), + rename: method(str()), + }, +}); + +const facet = defineFacet((host) => { + host.serve(Site, (_, self) => { + const visits = self.table(Visits); + const title = self.state(Title); + return { + title, + visit: async (page, ctx) => + self.tx(async (tx) => { + const count = ((await tx.get(visits, { page }))?.count ?? 0) + 1; + tx.put(visits, { page, count }); + return count; + }, ctx), + rename: async (text, ctx) => { + await title.change((draft) => { + draft.text = text; + }, ctx); + return null; + }, + }; + }); +}); + +/** A host on `store` serving the site, run until `work` is done. */ +async function serve(store: Store, work: (site: Reference) => Promise) { + const host = createHost({ store, onError: (error) => console.error("background error:", error) }); + await host.load([{ name: "@example/site", facet }], context); + await work(await host.get(Site, {}, context)); + await host.close(context); +} + +const directory = await mkdtemp(join(tmpdir(), "chord-example-")); +try { + const original = await openStore(sqliteBackend(await openNodeSqliteDatabase(join(directory, "a.db"))), context); + await serve(original, async (site) => { + await site.rename("Chord", context); + await site.visit("/", context); + await site.visit("/", context); + await site.visit("/docs", context); + }); + // Export to a local file (then upload it, if needed): the queue is held only while the parts are read. + const archive = join(directory, "archive.jsonl"); + await pipeline(encodeJsonl(original.export(context)), createWriteStream(archive)); + console.log("archive:", (await stat(archive)).size, "bytes at seq", original.lastSeq); + await original.close(context); + + // Import into an empty backend: every part is checked; a refused archive leaves the backend empty. + const parts = decodeJsonl(createReadStream(archive)); + const copy = await importStore(sqliteBackend(await openNodeSqliteDatabase(join(directory, "b.db"))), parts, context); + console.log("imported at seq", copy.lastSeq); + await serve(copy, async (site) => { + console.log("title:", site.title.value.text, "- next visit of /:", await site.visit("/", context)); + }); + await copy.close(context); +} finally { + await rm(directory, { recursive: true, force: true }); +} diff --git a/packages/chord/test/next/host/activity.test.ts b/packages/chord/test/next/host/activity.test.ts new file mode 100644 index 000000000..405089d6d --- /dev/null +++ b/packages/chord/test/next/host/activity.test.ts @@ -0,0 +1,146 @@ +// Tests of \`host.activity\` (§3, R26): the counts per kind, one publication per microtask turn with changes, a slow +// watcher, derives over it, nothing scheduled while nobody follows it, lingering. + +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { bool, int, local, method, remote } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { fakeTimers, flushed, framesOf, linked, loaded, Notes, serveNotes, transported } from "../support.ts"; + +const Kept = remote("app.kept", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + +describe("host.activity", () => { + it("counts holds arriving over links and served here, not this host's own references", async () => { + const { server, client } = await linked((h) => void serveNotes(h)); + expect(server.activity.value).toEqual({ references: 0, calls: 0, jobs: 0, lingering: 0 }); + const notes = await client.get(Notes, { session: "S1" }, context); + await server.get(Notes, { session: "S2" }, context); + expect(server.activity.value.references).toBe(1); + notes.release(); + await transported(); + expect(server.activity.value.references).toBe(0); + }); + + it("counts method calls running on this host, local and remote, until they settle", async () => { + const { server, client } = await linked((h) => void serveNotes(h)); + const remoteRef = await client.get(Notes, { session: "S1" }, context); + const localRef = await server.get(Notes, { session: "S1" }, context); + const calls = [remoteRef.wait(context), localRef.wait(context)]; + await transported(); + expect(server.activity.value.calls).toBe(2); + remoteRef.release(); + localRef.release(); + await Promise.allSettled(calls); + await transported(); + expect(server.activity.value.calls).toBe(0); + }); + + it("publishes once per microtask turn with changes; a slow watcher gets the newest counts", async () => { + const Counter = local("app.counter", { version: 1, params: {}, members: { bump: method() } }); + const { host } = await loaded((h) => h.serve(Counter, () => ({ bump: async () => null }))); + const counter = await host.get(Counter, {}, context); + const watch = await host.activity.watch(context); + const frames = framesOf(watch); + // Ten calls started in one turn: ten changes, one publication; they settle in a later turn: one more. + const bumps = Array.from({ length: 10 }, () => counter.bump(context)); + await Promise.all(bumps); + await flushed(); + const counts = (calls: number) => ({ references: 0, calls, jobs: 0, lingering: 0 }); + expect(frames).toEqual([ + [counts(10), [["r", counts(10)]]], + [counts(0), [["r", counts(0)]]], + ]); + }); + + it("gives a slow watcher the newest counts once it is free: its pending frames fold", async () => { + const Counter = local("app.counter", { version: 1, params: {}, members: { bump: method() } }); + const { host } = await loaded((h) => h.serve(Counter, () => ({ bump: async () => null }))); + const counter = await host.get(Counter, {}, context); + const watch = await host.activity.watch(context); + const gate = Promise.withResolvers(); + const seen: unknown[] = []; + watch.start(async (call) => { + if (call.kind !== "value") return; + seen.push(call.value); + if (seen.length === 1) await gate.promise; + }); + // Three more publications while the watcher is busy with the first. + for (let i = 0; i < 2; i++) { + await counter.bump(context); + await flushed(); + } + gate.resolve(); + await flushed(); + const counts = (calls: number) => ({ references: 0, calls, jobs: 0, lingering: 0 }); + expect(seen).toEqual([counts(1), counts(0)]); + }); + + it("schedules nothing while nobody watches or derives from it; a later watch reads the counts then", async () => { + const Counter = local("app.counter", { version: 1, params: {}, members: { bump: method(), wait: method() } }); + const gate = Promise.withResolvers(); + const { host } = await loaded((h) => + h.serve(Counter, () => ({ bump: async () => null, wait: () => gate.promise })), + ); + const counter = await host.get(Counter, {}, context); + const scheduled = vi.spyOn(globalThis, "queueMicrotask"); + try { + await counter.bump(context); + const waiting = counter.wait(context); + await flushed(); + expect(scheduled).not.toHaveBeenCalled(); + const watch = await host.activity.watch(context); + expect(watch.value).toMatchObject({ calls: 1 }); + gate.resolve(null); + await waiting; + expect(scheduled).toHaveBeenCalledTimes(1); + } finally { + scheduled.mockRestore(); + } + }); + + it("feeds derives (base code holding the host): they recompute in the publishing batch", async () => { + const Busy = local("app.busy", { version: 1, params: {}, members: { busy: bool(), work: method() } }); + const host = createHost({ onError: () => {} }); + const facet = defineFacet((h) => + h.serve(Busy, (_, self) => ({ + busy: self.derive([host.activity], ([{ calls }]) => calls > 0), + work: async () => null, + })), + ); + await host.load([{ name: "@test/core", facet }], context); + const busy = await host.get(Busy, {}, context); + const values = framesOf(await busy.busy.watch(context)); + await busy.work(context); + await flushed(); + expect(values.map(([value]) => value)).toEqual([true, false]); + expect(busy.busy.value).toBe(false); + }); + + describe("lingering", () => { + fakeTimers(); + + it("counts live objects kept by their type's linger after their last reference, until they end", async () => { + const { server, client } = await linked((h) => + h.serve(Kept, () => ({ variant: "v", members: { n: constant(1) } }), { linger: 1000 }), + ); + const kept = await client.get(Kept, {}, context); + kept.release(); + await transported(); + expect(server.activity.value).toMatchObject({ references: 0, lingering: 1 }); + // A new reference within the linger takes it back. + const again = await client.get(Kept, {}, context); + expect(server.activity.value).toMatchObject({ references: 1, lingering: 0 }); + again.release(); + await transported(); + expect(server.activity.value.lingering).toBe(1); + await vi.advanceTimersByTimeAsync(1000); + expect(server.activity.value.lingering).toBe(0); + }); + }); + + it("counts no jobs before slice 5", async () => { + const { host } = await loaded(() => {}); + expect(host.activity.value.jobs).toBe(0); + }); +}); diff --git a/packages/chord/test/next/host/attempt.test.ts b/packages/chord/test/next/host/attempt.test.ts new file mode 100644 index 000000000..73983ae0a --- /dev/null +++ b/packages/chord/test/next/host/attempt.test.ts @@ -0,0 +1,848 @@ +import { createHook } from "node:async_hooks"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, opt, remote, str } from "../../../src/next/contracts/index.ts"; +import { broken, isChordError } from "../../../src/next/data/index.ts"; +import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { + createHost, + defineFacet, + type HostError, + type Reference, + type Self, + type StoreSelf, +} from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import { evolveTable } from "../../../src/next/store/table-evolve.ts"; +import { TableHandle } from "../../../src/next/store/table-handle.ts"; +import { prepareTables } from "../../../src/next/store/table-migration.ts"; +import { failures, fakeTimers, flushed, loaded, memoryStore, metered, rejection, tableOwner } from "../support.ts"; + +const List = defineState("list", { schema: obj({ items: arr(str()) }), version: 1, initial: () => ({ items: [] }) }); +const Page = local("app.page", { version: 1, params: { id: int() }, members: { label: str(), slow: method() } }); + +describe("failure kinds", () => { + const Broken = local("app.broken", { version: 1, params: { id: int() }, members: { label: str() } }); + const Bad = defineState("bad", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: "x" }) as never }); + + it.each<[string, (self: Self) => unknown, string]>([ + [ + "a throwing factory", + () => { + throw new Error("factory threw"); + }, + "app.broken(1): factory threw", + ], + ["a missing member", () => ({}), "app.broken(1): label: a state member is a JSON watchable"], + ["an unknown member", () => ({ label: constant(""), extra: 1 }), "app.broken(1): extra: not a member"], + [ + "a native watchable as state (Q14)", + () => ({ label: { value: "x", watch: async () => ({}) } }), + "app.broken(1): label: a state member is a JSON watchable", + ], + [ + "a throwing first derive", + (self) => ({ + label: self.derive([], () => { + throw new Error("derive threw"); + }), + }), + "app.broken(1): derive threw", + ], + [ + "an initial() that fails its schema", + (self) => ({ label: self.derive([self.state(Bad)], () => "") }), + "app.broken(1): bad: initial() does not match the schema", + ], + [ + "an open that throws broken(error)", + (self) => { + self.open( + async () => { + throw broken(new Error("no remote configured")); + }, + () => {}, + ); + return { label: constant("") }; + }, + "app.broken(1): no remote configured", + ], + ])( + "%s fails broken: reported, the get rejects at once with a ChordError, nothing is left", + async (_, factory, message) => { + const served = vi.fn((_: { id: number }, self: Self) => factory(self) as never); + const { host, errors } = await loaded((h) => h.serve(Broken, served)); + const error = await host.get(Broken, { id: 1 }, context).catch((error: unknown) => error); + expect(isChordError(error) && error.message).toBe(message); + await expect(host.get(Broken, { id: 1 }, context)).rejects.toThrow(message); + expect(served).toHaveBeenCalledTimes(2); + expect(failures(errors)).toEqual(["app.broken(1): broken", "app.broken(1): broken"]); + // One report per failure: a derive throwing while its attempt activates reports no derive_failed. + expect(errors.map(({ kind }) => kind)).toEqual(["activation_failed", "activation_failed"]); + }, + ); + + it("an open that throws fails transient: reported, the get waits through retries", async () => { + const { host, errors } = await loaded((h) => + h.serve(Broken, (_, self) => { + self.open( + async () => { + throw new Error("offline"); + }, + () => {}, + ); + return { label: constant("") }; + }), + ); + expect(await rejection(host.get(Broken, { id: 1 }, context, { readyTimeout: 10 }))).toEqual({ + code: "unavailable", + message: "app.broken(1): offline", + }); + expect(errors).toMatchObject([ + { + kind: "activation_failed", + object: "app.broken(1)", + failure: { kind: "transient", error: { code: "unavailable" } }, + }, + ]); + await host.close(context); + }); + + describe("an attempt not live within readyTimeout", () => { + fakeTimers(); + + it("is abandoned and fails transient: the opener's context aborted, what opened closed", async () => { + let opener: Context | undefined; + const closes: string[] = []; + const { host, errors } = await loaded( + (h) => + h.serve(Broken, (_, self) => { + self.open( + async () => "a", + (value, reason) => void closes.push(`${value} ${reason.kind}`), + ); + self.open( + async (ctx) => { + opener = ctx; + return new Promise(() => {}); + }, + () => {}, + ); + return { label: constant("") }; + }), + { readyTimeout: 100 }, + ); + const got = rejection(host.get(Broken, { id: 1 }, context, { readyTimeout: 150 })); + await vi.advanceTimersByTimeAsync(100); + expect([opener?.abortSignal?.aborted, closes]).toEqual([true, ["a end"]]); + const notLive = { code: "unavailable", message: "app.broken(1): did not become live within 100 ms" }; + expect(errors).toMatchObject([{ kind: "activation_failed", failure: { kind: "transient", error: notLive } }]); + await vi.advanceTimersByTimeAsync(50); + expect(await got).toEqual(notLive); + }); + }); + + describe("an attempt waiting for its previous instance's teardown", () => { + fakeTimers(); + + it("is bounded by readyTimeout too: abandoned transient without running its factory, retried after it", async () => { + const closing = Promise.withResolvers(); + let factories = 0; + const { host, errors } = await loaded( + (h) => + h.serve(Page, (_, self) => { + factories++; + self.open( + async () => "native", + () => closing.promise, + ); + return { label: constant("p"), slow: async () => null }; + }), + { readyTimeout: 100, closeTimeout: 10_000 }, + ); + (await host.get(Page, { id: 1 }, context)).release(); + const next = host.hold(Page, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(100); + expect(factories).toBe(1); + expect(errors).toMatchObject([ + { + kind: "activation_failed", + failure: { kind: "transient", error: { message: "app.page(1): did not become live within 100 ms" } }, + }, + ]); + closing.resolve(); + await vi.advanceTimersByTimeAsync(1_000); + expect([factories, next.state.value]).toEqual([2, { kind: "live" }]); + }); + }); + + it("a keyed document's initial() failing its schema fails that transaction, not the object", async () => { + const Keys = defineState("keys", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: "x" }) as never, + keyed: true, + }); + const { host, errors } = await loaded((h) => + h.serve(Page, (_, self) => { + const keys = self.state(Keys); + return { + label: constant("live"), + slow: async (ctx) => { + await keys.change("a", () => {}, ctx); + return null; + }, + }; + }), + ); + const page = await host.get(Page, { id: 1 }, context); + await expect(page.slow(context)).rejects.toThrow("keys: initial() does not match the schema"); + expect([page.label.value, errors]).toEqual(["live", []]); + }); + + it("releases the references it acquired", async () => { + const pages = vi.fn((_: { id: number }) => ({ label: constant("p"), slow: async () => null })); + const { host } = await loaded((h) => { + h.serve(Broken, (_, self) => { + const page = self.get(Page, { id: 7 }); + const label = self.derive([page.label], (): string => { + throw new Error("late"); + }); + return { label }; + }); + h.serve(Page, pages); + }); + await expect(host.get(Broken, { id: 1 }, context)).rejects.toThrow("late"); + expect(pages).toHaveBeenCalledTimes(1); + await host.get(Page, { id: 7 }, context); + expect(pages).toHaveBeenCalledTimes(2); + }); +}); + +describe("ending", () => { + it("rejects their callers at once; the handler runs on with its context aborted and cannot commit", async () => { + const started = Promise.withResolvers(); + const proceed = Promise.withResolvers(); + const finished = Promise.withResolvers<{ aborted?: boolean; error?: unknown }>(); + const { host } = await loaded((h) => + h.serve(Page, (_, self) => { + const list = self.state(List); + return { + label: constant(""), + slow: async (ctx) => { + started.resolve(); + await proceed.promise; + const tx = self.tx(async (tx) => (await tx.draft(list)).items.push("x"), context); + finished.resolve({ aborted: ctx.abortSignal?.aborted, error: await rejection(tx) }); + return null; + }, + }; + }), + ); + const page = await host.get(Page, { id: 1 }, context); + const call = page.slow(context); + await started.promise; + page.release(); + const released = { code: "released", reason: "ended", message: "app.page(1): ended" }; + expect(await rejection(call)).toEqual(released); + proceed.resolve(); + expect(await finished.promise).toEqual({ aborted: true, error: released }); + }); + + it("rejects a call whose caller's context is already aborted, before the handler runs", async () => { + const slow = vi.fn(async () => null); + const { host } = await loaded((h) => h.serve(Page, () => ({ label: constant(""), slow }))); + const page = await host.get(Page, { id: 1 }, context); + const { context: cancelled, cancel } = withCancel(context); + cancel(new Error("gone")); + await expect(page.slow(cancelled)).rejects.toThrow("gone"); + expect(slow).not.toHaveBeenCalled(); + }); + + it("waits for the running handlers' own promises (the drain) before the natives close", async () => { + const events: string[] = []; + const started = Promise.withResolvers(); + const proceed = Promise.withResolvers(); + const { host } = await loaded((h) => + h.serve(Page, (_, self) => { + self.open( + async () => "native", + (value, reason) => void events.push(`close ${value} ${reason.kind}`), + ); + return { + label: constant(""), + slow: async () => { + started.resolve(); + await proceed.promise; + events.push("handler settled"); + return null; + }, + }; + }), + ); + const page = await host.get(Page, { id: 1 }, context); + const call = page.slow(context); + await started.promise; + page.release(); + await expect(call).rejects.toThrow("app.page(1): ended"); + await flushed(); + expect(events).toEqual([]); + proceed.resolve(); + await flushed(); + expect(events).toEqual(["handler settled", "close native end"]); + }); + + it("drains a handler that releases its object's last reference before it returns its promise", async () => { + const events: string[] = []; + const proceed = Promise.withResolvers(); + let page: Reference | undefined; + const { host } = await loaded((h) => + h.serve(Page, (_, self) => { + self.open( + async () => null, + () => void events.push("closed"), + ); + return { + label: constant(""), + // Not async: the release runs before the handler has a promise to return. + slow: () => { + page!.release(); + return proceed.promise.then(() => { + events.push("handler settled"); + return null; + }); + }, + }; + }), + ); + page = await host.get(Page, { id: 1 }, context); + const call = page.slow(context); + await expect(call).rejects.toThrow("app.page(1): ended"); + await flushed(); + expect(events).toEqual([]); + proceed.resolve(); + await flushed(); + expect(events).toEqual(["handler settled", "closed"]); + }); + + it("refuses transactions at once; documents stay readable until the closes are done", async () => { + let list: SingletonHandle<{ items: string[] }> | undefined; + const seen = Promise.withResolvers(); + const { host } = await loaded((h) => + h.serve(Page, (_, self) => { + const handle = self.state(List); + list = handle; + self.open( + async () => null, + async () => seen.resolve([handle.value, await rejection(handle.change(() => {}, context))]), + ); + return { label: constant(""), slow: async () => null }; + }), + ); + (await host.get(Page, { id: 1 }, context)).release(); + expect(await seen.promise).toEqual([ + { items: [] }, + { code: "released", reason: "ended", message: "app.page(1): ended" }, + ]); + await flushed(); + expect(() => list!.value).toThrow("app.page(1): ended"); + }); + + describe("the drain", () => { + fakeTimers(); + + it("is bounded by closeTimeout: a handler still running then sees its natives close, which is reported", async () => { + const closed = vi.fn(); + const { host, errors } = await loaded( + (h) => + h.serve(Page, (_, self) => { + self.open(async () => null, closed); + return { label: constant(""), slow: () => new Promise(() => {}) }; + }), + { closeTimeout: 1000 }, + ); + const page = await host.get(Page, { id: 1 }, context); + void page.slow(context).catch(() => {}); + page.release(); + await vi.advanceTimersByTimeAsync(999); + expect(closed).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + expect(closed).toHaveBeenCalledTimes(1); + expect(errors).toEqual([{ kind: "drain_timeout", object: "app.page(1)" }]); + }); + + it("a handler stuck past it no longer counts as a call nor holds its ctx.get references", async () => { + const Item = local("app.item", { version: 1, params: { id: int() }, members: { label: str() } }); + const got = Promise.withResolvers(); + const { host } = await loaded( + (h) => { + h.serve(Item, () => ({ label: constant("item") })); + h.serve(Page, () => ({ + label: constant(""), + slow: async (ctx) => { + await ctx.get(Item, { id: 7 }); + got.resolve(); + return new Promise(() => {}); + }, + })); + }, + { closeTimeout: 1000 }, + ); + const items = host.references(Item); + const page = await host.get(Page, { id: 1 }, context); + void page.slow(context).catch(() => {}); + await got.promise; + expect(host.activity.value.calls).toBe(1); + expect(items.value).toHaveLength(1); + page.release(); + await vi.advanceTimersByTimeAsync(1000); + expect(host.activity.value.calls).toBe(0); + expect(items.value).toEqual([]); + }); + }); +}); + +describe("declarations", () => { + it("are valid only while the factory runs", async () => { + let later: Self | undefined; + const { host } = await loaded((h) => + h.serve(Page, (_, self) => { + later = self; + return { label: constant(""), slow: async () => null }; + }), + ); + await host.get(Page, { id: 1 }, context); + expect(() => later!.state(List)).toThrow("app.page(1): self.state is valid only in the factory"); + expect(() => later!.derive([], () => 1)).toThrow("self.derive is valid only in the factory"); + expect(() => later!.get(Page, { id: 2 })).toThrow("self.get is valid only in the factory"); + expect(() => + later!.open( + async () => null, + () => {}, + ), + ).toThrow("self.open is valid only in the factory"); + }); + + it("refuse a document definition declared twice", async () => { + const Twice = local("app.twice", { version: 1, params: { n: int() }, members: {} }); + const { host } = await loaded((h) => + h.serve(Twice, (_, self) => { + [self.state(List), self.state(List)]; + return {}; + }), + ); + await expect(host.get(Twice, { n: 2 }, context)).rejects.toThrow('app.twice(2): state "list" is declared twice'); + }); +}); + +describe("calls", () => { + it("check arguments with L1 and pass arguments and results by reference", async () => { + const seen: unknown[] = []; + const result = { items: [] }; + const Echo = local("app.echo", { + version: 1, + params: {}, + members: { echo: method(obj({ text: str() }), obj({ items: arr(str()) })), none: method() }, + }); + const { host } = await loaded((h) => + h.serve(Echo, () => ({ + echo: async (args) => { + seen.push(args); + return result; + }, + // A method without arguments gets only the context. + none: async (...args) => { + seen.push(args.length); + return null; + }, + })), + ); + const echo = await host.get(Echo, {}, context); + const args = { text: "t" }; + expect(await echo.echo(args, context)).toBe(result); + expect(await echo.none(context)).toBeNull(); + expect(seen).toEqual([args, 1]); + expect(seen[0]).toBe(args); + // @ts-expect-error: an unknown field + expect(await rejection(echo.echo({ text: "t", extra: 1 }, context))).toMatchObject({ + code: "invalid", + reason: "args", + }); + }); + + it("take a plain value or a synchronous throw from a handler that is no async function", async () => { + const Plain = local("app.plain", { + version: 1, + params: {}, + members: { value: method(undefined, int()), fail: method() }, + }); + const { host } = await loaded((h) => + // Plugin code is bundled without type checks. + h.serve(Plain, () => ({ + value: (() => 7) as never, + fail: (() => { + throw new Error("sync throw"); + }) as never, + })), + ); + const plain = await host.get(Plain, {}, context); + expect(await plain.value(context)).toBe(7); + await expect(plain.fail(context)).rejects.toThrow("sync throw"); + }); + + it("cost one validation per call and one compilation per method", async () => { + const telemetry = await metered(); + // A fresh declaration: compiled checks are cached per contract. + const Fresh = local("app.store", { version: 1, params: {}, members: { add: method(obj({ text: str() })) } }); + const { host } = await loaded((h) => h.serve(Fresh, () => ({ add: async () => null }))); + const store = await host.get(Fresh, {}, context); + await store.add({ text: "a" }, telemetry.context); + await store.add({ text: "b" }, telemetry.context); + expect(telemetry.counts()).toEqual({ "chord.data.compilations": 1, "chord.data.validations": 2 }); + }); +}); + +describe("the call path's cost", () => { + it("makes at most 19 promises per local call (a guard: the drain and the cleanups share one reaction)", async () => { + const Touch = local("app.touch", { version: 1, params: {}, members: { touch: method() } }); + const { host } = await loaded((h) => h.serve(Touch, () => ({ touch: async () => null }))); + const ref = await host.get(Touch, {}, context); + await ref.touch(context); + let promises = 0; + // Every `then` makes a promise: one more reaction per call shows here (HEAD before the cleanups made 20). + const hook = createHook({ + init: (_id, type) => { + if (type === "PROMISE") promises++; + }, + }); + hook.enable(); + try { + for (let call = 0; call < 100; call++) await ref.touch(context); + } finally { + hook.disable(); + } + expect(promises / 100).toBeLessThanOrEqual(19); + await host.close(context); + }); +}); + +describe("remote variants", () => { + const Two = remote("app.two", { + version: 1, + params: {}, + variants: { full: { state: { n: int() }, methods: { bump: method() } }, view: { state: { n: int() } } }, + }); + const owner = { kind: "owner" } as const; + + /** A host acting as `owner` whose factory for `Two` returns `implementation`. */ + async function two(implementation: unknown) { + const errors: HostError[] = []; + const host = createHost({ principal: owner, onError: (error) => errors.push(error) }); + const facet = defineFacet((h) => h.serve(Two, () => implementation as never)); + await host.load([{ name: "@test/app", facet }], context); + return { host, errors }; + } + + it("the factory chooses one; the reference narrows on its variant", async () => { + const { host } = await two({ variant: "view", members: { n: constant(1) } }); + const ref = await host.get(Two, {}, context); + expect(ref.variant).toBe("view"); + if (ref.variant === "view") expect(ref.n.value).toBe(1); + }); + + it.each([ + ["an unknown variant", { variant: "other", members: {} }, 'variant "other": not a variant of app.two'], + ["a missing member", { variant: "full", members: { n: constant(1) } }, "bump: a method is a function"], + [ + "an extra member", + { variant: "view", members: { n: constant(1), bump: async () => null } }, + "bump: not a member", + ], + ])("%s fails the activation broken", async (_, implementation, message) => { + const { host, errors } = await two(implementation); + expect(await rejection(host.get(Two, {}, context))).toEqual({ + code: "unavailable", + message: `app.two: ${message}`, + }); + expect(failures(errors)).toEqual(["app.two: broken"]); + }); + + it("a member outside the bound variant fails invalid/member", async () => { + const { host } = await two({ variant: "view", members: { n: constant(1) } }); + const ref = (await host.get(Two, {}, context)) as unknown as { bump(ctx: Context): Promise }; + expect(await rejection(ref.bump(context))).toEqual({ + code: "invalid", + reason: "member", + message: 'app.two (variant view) has no method "bump"', + }); + }); +}); + +describe("tables", () => { + const Edits = defineTable("edits", { + version: 1, + columns: { id: str(), at: int() }, + key: ["id"], + indexes: { by_at: ["at"] }, + }); + const Store = local("app.store", { version: 1, params: {}, members: { n: int() } }); + + it("a fresh table is created at activation at its version, with its indexes, in one commit", async () => { + const store = await memoryStore({ outbox: true }); + const { host } = await loaded( + (h) => + h.serve(Store, (_, self) => { + self.table(Edits); + return { n: constant(1) }; + }), + { + store, + }, + ); + await host.get(Store, {}, context); + expect(store.table("client:app.store:edits")).toMatchObject({ + version: 1, + state: "ready", + indexes: { by_at: ["at"] }, + }); + expect((await store.outbox!.read(0, 10, context)).map(({ writes }) => writes.map(({ kind }) => kind))).toEqual([ + ["table", "index"], + ]); + await host.close(context); + const again = await loaded( + (h) => + h.serve(Store, (_, self) => { + self.table(Edits); + return { n: constant(1) }; + }), + { + store, + }, + ); + await again.host.get(Store, {}, context); + expect(store.lastSeq).toBe(1); + await again.host.close(context); + await store.close(context); + }); + + it.each<[string, (self: StoreSelf) => void, string]>([ + [ + "declared twice", + (self) => { + self.table(Edits); + self.table(Edits); + }, + 'table "edits" is declared twice', + ], + [ + "another version recorded", + (self) => self.table(defineTable("edits", { version: 2, columns: { id: str() }, key: ["id"] })), + "older than the history's base 2", + ], + [ + "another definition at the same version", + (self) => self.table(defineTable("edits", { version: 1, columns: { id: str(), at: int() }, key: ["id"] })), + "recorded with another definition", + ], + ])("fails the activation broken: %s", async (_, declare, message) => { + const store = await memoryStore(); + const first = await loaded( + (h) => + h.serve(Store, (_, self) => { + self.table(Edits); + return { n: constant(1) }; + }), + { + store, + }, + ); + await first.host.get(Store, {}, context); + await first.host.close(context); + const { host } = await loaded( + (h) => + h.serve(Store, (_, self) => { + declare(self); + return { n: constant(1) }; + }), + { store }, + ); + expect(await rejection(host.get(Store, {}, context))).toMatchObject({ + message: expect.stringContaining(message), + }); + await host.close(context); + await store.close(context); + }); + + it("needs a local type with empty params (P3) and a host with a store", async () => { + const { host } = await loaded((h) => { + h.serve(Page, (_, self) => { + // @ts-expect-error: only a local type with empty params has tables (P3) + const declare = () => self.table(Edits); + void declare; + (self as unknown as StoreSelf).table(Edits); + return { label: constant(""), slow: async () => null }; + }); + h.serve(Store, (_, self) => { + self.table(Edits); + return { n: constant(1) }; + }); + }); + expect(await rejection(host.get(Page, { id: 1 }, context))).toMatchObject({ + message: expect.stringContaining("persist needs a stable identity"), + }); + expect(await rejection(host.get(Store, {}, context))).toMatchObject({ + message: expect.stringContaining("persist needs a store"), + }); + await host.close(context); + }); +}); + +describe("eager migrations at activation (slice 6a §9)", () => { + fakeTimers(); + // The clock is faked, but the zero-delay timers a migration yields with between chunks run every real turn. + let pumping = true; + beforeEach(() => { + pumping = true; + const pump = (): void => { + if (!pumping) return; + vi.advanceTimersByTime(0); + setImmediate(pump); + }; + setImmediate(pump); + }); + afterEach(() => { + pumping = false; + }); + const TABLE = "client:app.store:edits"; + const OPEN = { admit: () => true, adopt: () => {} }; + const EditsV1 = defineTable("edits", { version: 1, columns: { id: str(), at: int() }, key: ["id"] }); + const Store = local("app.store", { version: 1, params: {}, members: { n: int() } }); + const Slow = local("app.slow", { version: 1, params: {}, members: { n: int() } }); + + /** A store whose edits table is at version 1 with `count` rows. */ + async function seeded(count: number): Promise { + const store = await memoryStore(); + await prepareTables([{ definition: EditsV1, table: TABLE }], store, OPEN, "app.store", context); + const staged: StagedWrite[] = []; + const changes = new TableChanges(new TableHandle(EditsV1, TABLE, store, tableOwner()), staged); + for (let i = 0; i < count; i++) changes.put({ id: `e${String(i).padStart(3, "0")}`, at: i }); + await store.commit(staged, OPEN, context); + return store; + } + + /** Waits (real turns of the event loop, the clock faked) until `done`. */ + async function until(done: () => boolean): Promise { + for (let turn = 0; turn < 1_000 && !done(); turn++) await flushed(); + expect(done()).toBe(true); + } + + /** Version 2 adding `note`, whose `rows` runs `each` per row first. */ + function evolved(each: (id: string) => void) { + return evolveTable(EditsV1, (t) => + t.add("note", opt(str())).rows((row) => { + each(row.id); + return { note: `#${row.at}` }; + }), + ); + } + + /** A native whose open never succeeds: it rejects when its context is aborted (the deadline). */ + function hanging(ctx: Context): Promise { + return new Promise((_, reject) => ctx.abortSignal!.addEventListener("abort", () => reject(new Error("aborted")))); + } + + it("pause the readyTimeout deadline and resume it with the budget left", async () => { + const store = await seeded(250); + // Each row takes 10 ms of the fake clock: 2.5 s of migration against a 1 s deadline. + const Edits = evolved(() => vi.advanceTimersByTime(10)); + const loading = loaded( + (h) => + h.serve(Store, (_, self) => { + self.table(Edits); + // The factory takes 600 ms of the budget; the native after the migration never opens. + vi.advanceTimersByTime(600); + self.open(hanging, () => {}); + return { n: constant(1) }; + }), + { store, readyTimeout: 1_000 }, + ); + await until(() => store.table(TABLE)?.version === 2 && store.table(TABLE)?.state === "ready"); + expect(Date.now()).toBe(3_100); + await vi.advanceTimersByTimeAsync(399); + await vi.advanceTimersByTimeAsync(1); + const { host, errors } = await loading; + expect(Date.now()).toBe(3_500); + expect(errors).toMatchObject([ + { + kind: "activation_failed", + failure: { kind: "transient", error: { message: "app.store: did not become live within 1000 ms" } }, + }, + ]); + await host.close(context); + await store.close(context); + }); + + it("a timer firing during a long migration: another activation's deadline keeps its bound", async () => { + const store = await seeded(250); + const Edits = evolved(() => vi.advanceTimersByTime(10)); + // Both types have empty params: the host activates both at load, the start barrier waiting for both. + const { host, errors } = await loaded( + (h) => { + h.serve(Slow, (_, self) => { + self.open(hanging, () => {}); + return { n: constant(1) }; + }); + h.serve(Store, (_, self) => { + self.table(Edits); + return { n: constant(1) }; + }); + }, + { store, readyTimeout: 1_000 }, + ); + expect(Date.now()).toBe(2_500); + expect(errors).toMatchObject([{ kind: "activation_failed", object: "app.slow", failure: { kind: "transient" } }]); + expect((await host.get(Store, {}, context)).n.value).toBe(1); + await host.close(context); + await store.close(context); + }); + + it("a throwing rows fails the object broken, the table migrating; a retry resumes after the committed chunks", async () => { + const store = await seeded(250); + let crash = true; + const patched: string[] = []; + const Edits = evolved((id) => { + if (crash && id === "e150") throw new Error("crash"); + patched.push(id); + }); + const { host, errors } = await loaded( + (h) => + h.serve(Store, (_, self) => { + self.table(Edits); + return { n: constant(1) }; + }), + { store }, + ); + const held = host.hold(Store, {}, context); + await until(() => held.state.value.kind === "failed"); + expect(held.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "broken", error: { message: expect.stringContaining("crash") } }, + }); + expect(errors).toMatchObject([{ kind: "activation_failed", failure: { kind: "broken" } }]); + expect(store.table(TABLE)).toMatchObject({ version: 2, state: "migrating" }); + crash = false; + patched.length = 0; + held.retry(); + await until(() => held.state.value.kind === "live"); + expect(patched).toHaveLength(150); + expect(patched[0]).toBe("e100"); + expect(store.table(TABLE)).toMatchObject({ version: 2, state: "ready" }); + held.release(); + await host.close(context); + await store.close(context); + }); +}); diff --git a/packages/chord/test/next/host/backoff.test.ts b/packages/chord/test/next/host/backoff.test.ts new file mode 100644 index 000000000..f53bdd426 --- /dev/null +++ b/packages/chord/test/next/host/backoff.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it, vi } from "vitest"; +import { Backoff } from "../../../src/next/host/backoff.ts"; +import { fakeTimers } from "../support.ts"; + +/** The length of the window `failed` opens for `identity`, measured until `due` runs. */ +function window(backoff: Backoff, due: ReturnType, identity: string): number { + const calls = due.mock.calls.length; + const start = Date.now(); + backoff.failed(identity); + while (due.mock.calls.length === calls) vi.advanceTimersByTime(100); + return Date.now() - start; +} + +describe("backoff", () => { + fakeTimers(); + + it("opens a window of 1 s after the first failure, doubling up to 60 s, and calls due at its end", () => { + const due = vi.fn(); + const backoff = new Backoff(due); + const windows = Array.from({ length: 9 }, () => window(backoff, due, "a")); + expect(windows).toEqual([1000, 2000, 4000, 8000, 16000, 32000, 60000, 60000, 60000]); + expect(due.mock.calls.every(([identity]) => identity === "a")).toBe(true); + }); + + it("tells when a running window ends: a failed reference's retryAt", () => { + const backoff = new Backoff(() => {}); + vi.advanceTimersByTime(5_000); + backoff.failed("a"); + expect([backoff.until("a"), backoff.until("b")]).toEqual([6_000, undefined]); + vi.advanceTimersByTime(1_000); + expect(backoff.until("a")).toBeUndefined(); + backoff.failed("a"); + expect(backoff.until("a")).toBe(8_000); + backoff.skip("a"); + expect(backoff.until("a")).toBeUndefined(); + }); + + it("is waiting only inside a window", () => { + const backoff = new Backoff(() => {}); + backoff.failed("a"); + expect([backoff.waiting("a"), backoff.waiting("b")]).toEqual([true, false]); + vi.advanceTimersByTime(1000); + expect(backoff.waiting("a")).toBe(false); + }); + + it("skip ends a window without due and keeps its delay; delete starts over", () => { + const due = vi.fn(); + const backoff = new Backoff(due); + backoff.failed("a"); + backoff.skip("a"); + vi.advanceTimersByTime(5000); + expect([backoff.waiting("a"), due.mock.calls.length]).toEqual([false, 0]); + expect(window(backoff, due, "a")).toBe(2000); + backoff.delete("a"); + expect(window(backoff, due, "a")).toBe(1000); + }); + + it("ended keeps a running window and drops a passed one", () => { + const due = vi.fn(); + const backoff = new Backoff(due); + backoff.failed("a"); + backoff.ended("a"); + expect(backoff.waiting("a")).toBe(true); + vi.advanceTimersByTime(1000); + backoff.ended("a"); + expect(window(backoff, due, "a")).toBe(1000); + }); + + it("a failure inside a running window replaces it: one due, after the doubled delay", () => { + const due = vi.fn(); + const backoff = new Backoff(due); + backoff.failed("a"); + backoff.failed("a"); + vi.advanceTimersByTime(1999); + expect([due.mock.calls.length, vi.getTimerCount()]).toEqual([0, 1]); + vi.advanceTimersByTime(1); + expect(due).toHaveBeenCalledTimes(1); + }); + + it("clear forgets every identity with its timer", () => { + const backoff = new Backoff(() => {}); + backoff.failed("a"); + backoff.failed("b"); + backoff.clear(); + expect([backoff.waiting("a"), backoff.waiting("b"), vi.getTimerCount()]).toEqual([false, false, 0]); + }); +}); diff --git a/packages/chord/test/next/host/ctx.test.ts b/packages/chord/test/next/host/ctx.test.ts new file mode 100644 index 000000000..0d4cc480b --- /dev/null +++ b/packages/chord/test/next/host/ctx.test.ts @@ -0,0 +1,173 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withAbortSignal } from "../../../src/context/index.ts"; +import { int, local, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { withLinkedAbortSignal } from "../../../src/next/host/ctx.ts"; +import type { ChordHost } from "../../../src/next/host/host.ts"; +import type { Ctx, Host, Reference } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { flushed, loaded, rawParent, rejection, type TestPrincipal } from "../support.ts"; + +const Item = local("app.item", { version: 1, params: { id: int() }, members: { id: int() } }); +const Desk = local("app.desk", { + version: 1, + params: {}, + members: { use: method(undefined, int()), leak: method(), late: method(), who: method(undefined, str()) }, +}); + +/** A desk whose handlers use `ctx.get`; what they got and their contexts are recorded. */ +async function desk() { + const seen: { ctx?: Ctx; item?: Reference; late?: Promise; factories: number } = { + factories: 0, + }; + const { host } = await loaded((h) => { + h.serve(Item, ({ id }) => { + seen.factories++; + return { id: constant(id) }; + }); + h.serve(Desk, () => ({ + use: async (ctx) => { + const item = await ctx.get(Item, { id: 1 }); + return item.id.value; + }, + leak: async (ctx) => { + seen.ctx = ctx; + seen.item = await ctx.get(Item, { id: 2 }); + return null; + }, + late: async (ctx) => { + seen.late = ctx.get(Item, { id: 3 }).catch((error: unknown) => error); + return null; + }, + who: async (ctx) => `${ctx.principal?.kind ?? "none"} ${ctx.incarnation}`, + })); + }); + return { host, seen, desk: await host.get(Desk, {}, context) }; +} + +describe("ctx", () => { + it("ctx.get references are released when the handler settles", async () => { + const { desk: ref, seen } = await desk(); + expect(await ref.use(context)).toBe(1); + await ref.leak(context); + expect(() => seen.item!.id.value).toThrow("app.item: released"); + // The object ended with its last reference: a new get creates it anew. + await ref.use(context); + expect(seen.factories).toBe(3); + }); + + it("ctx.get after the handler settled throws released", async () => { + const { desk: ref, seen } = await desk(); + await ref.leak(context); + expect(await rejection(seen.ctx!.get(Item, { id: 1 }))).toEqual({ + code: "released", + reason: "settled", + message: "app.item: ctx.get after its handler settled", + }); + }); + + it("ctx.get of a handler whose call was aborted checks the signal first: nothing is created", async () => { + let factories = 0; + let got: Promise | undefined; + const gate = Promise.withResolvers(); + const { host } = await loaded((h) => { + h.serve(Item, ({ id }) => { + factories++; + return { id: constant(id) }; + }); + h.serve(Desk, () => ({ + use: async (ctx) => { + await gate.promise; + got = ctx.get(Item, { id: 1 }).catch((error: unknown) => error); + return 0; + }, + leak: async () => null, + late: async () => null, + who: async () => "", + })); + }); + const ref = await host.get(Desk, {}, context); + const caller = new AbortController(); + const call = ref.use(withAbortSignal(caller.signal, context)).catch(() => {}); + await flushed(); + caller.abort(new Error("caller gave up")); + gate.resolve(); + await call; + await flushed(); + expect(await got).toEqual(new Error("caller gave up")); + expect(factories).toBe(0); + }); + + it("a ctx.get finishing after its handler settled is released at once (race 11)", async () => { + const { desk: ref, seen } = await desk(); + await ref.late(context); + expect(await seen.late).toMatchObject({ + code: "released", + reason: "settled", + message: "app.item: ctx.get after its handler settled", + }); + await flushed(); + // The late reference's object ended with it. + expect(seen.factories).toBe(1); + }); + + it("a ctx.get still waiting when its handler settles gives up: its reference is released at once", async () => { + const Far = remote("app.far", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const Starter = local("app.starter", { version: 1, params: {}, members: { start: method() } }); + let pending: Promise | undefined; + const { host } = await loaded((h) => + h.serve(Starter, () => ({ + start: async (ctx) => { + pending = ctx.get(Far, {}).catch((error: unknown) => error); + return null; + }, + })), + ); + // A host of any principal type takes a parent link. + const parent = await rawParent(host as Host); + await (await host.get(Starter, {}, context)).start(context); + const { hold } = await parent.next("hold"); + expect(await parent.next("release")).toMatchObject({ hold }); + expect(await pending).toMatchObject({ + code: "released", + reason: "settled", + message: "app.far: ctx.get after its handler settled", + }); + }); + + it("a local handler's ctx.principal is the host's, its incarnation this host's", async () => { + const { desk: ref, host } = await desk(); + expect(await ref.who(context)).toBe(`none ${(host as ChordHost).incarnation}`); + }); +}); + +describe("linked abort signals", () => { + it("abort with the caller's signal or the call's, the caller's reason first", () => { + for (const first of ["caller", "call"] as const) { + const [caller, call] = [new AbortController(), new AbortController()]; + const { context: linked } = withLinkedAbortSignal(call.signal, withAbortSignal(caller.signal, context)); + (first === "caller" ? caller : call).abort(first); + expect(linked.abortSignal?.reason).toBe(first); + } + const [caller, call] = [new AbortController(), new AbortController()]; + call.abort("call"); + caller.abort("caller"); + const { context: linked } = withLinkedAbortSignal(call.signal, withAbortSignal(caller.signal, context)); + expect(linked.abortSignal?.reason).toBe("caller"); + }); + + it("detach from both sources on release: a later abort reaches nothing", () => { + const [caller, call] = [new AbortController(), new AbortController()]; + const linked = withLinkedAbortSignal(call.signal, withAbortSignal(caller.signal, context)); + linked.release(); + caller.abort("late"); + call.abort("late"); + expect(linked.context.abortSignal?.aborted).toBe(false); + }); + + it("is the call's own signal without a caller's", () => { + const call = new AbortController(); + const linked = withLinkedAbortSignal(call.signal, context); + expect(linked.context.abortSignal).toBe(call.signal); + linked.release(); + }); +}); diff --git a/packages/chord/test/next/host/deadline.test.ts b/packages/chord/test/next/host/deadline.test.ts new file mode 100644 index 000000000..23b6e5093 --- /dev/null +++ b/packages/chord/test/next/host/deadline.test.ts @@ -0,0 +1,42 @@ +// Tests of an activation's deadline (slice 6a §9): it expires once after its budget; paused time does not count; a +// cleared deadline never expires. + +import { describe, expect, it, vi } from "vitest"; +import { Deadline } from "../../../src/next/host/deadline.ts"; +import { fakeTimers } from "../support.ts"; + +describe("deadline", () => { + fakeTimers(); + + it("expires once after its budget", () => { + const expired = vi.fn(); + new Deadline(100, expired); + vi.advanceTimersByTime(99); + expect(expired).not.toHaveBeenCalled(); + vi.advanceTimersByTime(1_000); + expect(expired).toHaveBeenCalledTimes(1); + }); + + it("paused time does not count: resumed with the budget it had left", () => { + const expired = vi.fn(); + const deadline = new Deadline(100, expired); + vi.advanceTimersByTime(60); + deadline.pause(); + vi.advanceTimersByTime(1_000); + deadline.resume(); + vi.advanceTimersByTime(39); + expect(expired).not.toHaveBeenCalled(); + vi.advanceTimersByTime(1); + expect(expired).toHaveBeenCalledTimes(1); + }); + + it("a cleared deadline never expires, also when resumed", () => { + const expired = vi.fn(); + const deadline = new Deadline(100, expired); + deadline.pause(); + deadline.clear(); + deadline.resume(); + vi.advanceTimersByTime(1_000); + expect(expired).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/chord/test/next/host/facet.test.ts b/packages/chord/test/next/host/facet.test.ts new file mode 100644 index 000000000..54fe1113d --- /dev/null +++ b/packages/chord/test/next/host/facet.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, expectTypeOf, it } from "vitest"; +import { int, local } from "../../../src/next/contracts/index.ts"; +import { defineFacet, type Facet, type FacetHost, type Host } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { loaded } from "../support.ts"; + +const Info = local("app.info", { version: 1, params: {}, members: { n: int() } }); + +describe("facets", () => { + it("declare only while their setup runs", async () => { + let later: FacetHost | undefined; + await loaded((host) => { + later = host; + }); + expect(() => later!.serve(Info, () => ({ n: constant(1) }))).toThrow( + "@test/app: host.serve is valid only in the facet's setup", + ); + }); + + it("only declare: a setup takes the host alone, which has no documents, routines or natives", () => { + expectTypeOf["setup"]>>().toEqualTypeOf<[FacetHost<{ kind: "owner" }>]>(); + expectTypeOf().not.toHaveProperty("state"); + expectTypeOf().not.toHaveProperty("on"); + expectTypeOf().not.toHaveProperty("onLive"); + }); + + it("are typed by exactly the app's principal type", () => { + type App = { kind: "owner" } | { kind: "guest"; session: string }; + const facet = defineFacet(() => {}); + expectTypeOf(facet).toEqualTypeOf>(); + type Load = Host["load"]; + expectTypeOf[0]>().toEqualTypeOf< + readonly { readonly name: string; readonly facet: Facet }[] + >(); + expectTypeOf>().toExtend>(); + expectTypeOf>().not.toExtend>(); + expectTypeOf>().not.toExtend>(); + }); +}); diff --git a/packages/chord/test/next/host/host.test.ts b/packages/chord/test/next/host/host.test.ts new file mode 100644 index 000000000..96d2482c4 --- /dev/null +++ b/packages/chord/test/next/host/host.test.ts @@ -0,0 +1,944 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { int, local, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { broken } from "../../../src/next/data/index.ts"; +import { + type Ctx, + createHost, + defineFacet, + type FacetHost, + type HostError, + type Self, +} from "../../../src/next/host/index.ts"; +import type { ChordHost } from "../../../src/next/host/internal.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { jsonPart } from "../../../src/next/link/internal.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { sqliteBackend } from "../../../src/next/store/backend.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import { type ChordStore, openStore } from "../../../src/next/store/store.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { + failures, + fakeTimers, + flushed, + forEachCodec, + linked, + loaded, + Notes, + PlacedNotes, + partValue, + RawPeer, + rejection, + Session, + serveNotes, + type TestPrincipal, + testCodec, + transported, +} from "../support.ts"; + +const Info = local("app.info", { version: 1, params: {}, members: { title: str() } }); +const Counter = local("app.counter", { version: 1, params: {}, members: { count: int(), bump: method() } }); +const Page = local("app.page", { version: 1, params: { id: int() }, members: { id: int() } }); +const Svc = local("app.svc", { version: 1, params: { id: int() }, members: { n: int() } }); +const Core = local("app.core", { version: 1, params: {}, members: { n: int() } }); + +const noErrors = { onError: () => {} }; + +/** Serves `Svc`: one native whose open waits for `gate` and whose close records its value and reason in `closes`. */ +function serveGated(h: FacetHost, gate: Promise, closes: string[]): void { + h.serve(Svc, ({ id }, self) => { + self.open( + async () => { + await gate; + return `svc ${id}`; + }, + (value, reason) => void closes.push(`${value} ${reason.kind}`), + ); + return { n: constant(id) }; + }); +} + +describe("load", () => { + it("runs every setup and activates the types with empty params before it resolves", async () => { + const factories = { info: vi.fn(() => ({ title: constant("t") })), page: vi.fn() }; + const facet = defineFacet((host) => { + host.serve(Info, factories.info); + host.serve(Page, factories.page); + }); + const host = createHost(noErrors); + await host.load([{ name: "@app/info", facet }], context); + expect(factories.info).toHaveBeenCalledTimes(1); + expect(factories.page).not.toHaveBeenCalled(); + await host.close(context); + }); + + it("refuses invalid and duplicate plugin names, double serves and a second load", async () => { + const empty = defineFacet(() => {}); + const info = defineFacet((host) => host.serve(Info, () => ({ title: constant("t") }))); + await expect(createHost(noErrors).load([{ name: "Bad Name", facet: empty }], context)).rejects.toThrow( + '"Bad Name" is not a valid plugin name', + ); + const twice = [ + { name: "a", facet: empty }, + { name: "a", facet: empty }, + ]; + await expect(createHost(noErrors).load(twice, context)).rejects.toThrow('plugin "a" is loaded twice'); + const served = [ + { name: "a", facet: info }, + { name: "b", facet: info }, + ]; + await expect(createHost(noErrors).load(served, context)).rejects.toThrow("app.info is served twice"); + const host = createHost(noErrors); + await host.load([], context); + await expect(host.load([], context)).rejects.toThrow("load runs once per host"); + }); + + it("makes a get before it resolves wait for it", async () => { + const host = createHost(noErrors); + const got = host.get(Info, {}, context); + await host.load( + [{ name: "a", facet: defineFacet((h) => h.serve(Info, () => ({ title: constant("t") }))) }], + context, + ); + expect((await got).title.value).toBe("t"); + }); +}); + +describe("the start barrier", () => { + fakeTimers(); + + it("resolves once every host-lifetime object had its first outcome; a failed one is reported and retried", async () => { + let attempts = 0; + const { host, errors } = await loaded((h) => { + h.serve(Info, () => ({ title: constant("t") })); + h.serve(Counter, (_, self) => { + const attempt = ++attempts; + self.open( + async () => { + if (attempt === 1) throw new Error("offline"); + return null; + }, + () => {}, + ); + return { count: constant(attempt), bump: async () => null }; + }); + }); + expect(failures(errors)).toEqual(["app.counter: transient"]); + const counter = host.get(Counter, {}, context); + await vi.advanceTimersByTimeAsync(1000); + expect((await counter).count.value).toBe(2); + }); + + it("is not held by a hanging open: its attempt is abandoned at readyTimeout", async () => { + const host = createHost({ ...noErrors, readyTimeout: 100 }); + const facet = defineFacet((h) => + h.serve(Info, (_, self) => { + self.open( + () => new Promise(() => {}), + () => {}, + ); + return { title: constant("t") }; + }), + ); + let done = false; + void host.load([{ name: "a", facet }], context).then(() => { + done = true; + }); + await vi.advanceTimersByTimeAsync(99); + expect(done).toBe(false); + await vi.advanceTimersByTimeAsync(1); + expect(done).toBe(true); + }); +}); + +describe("get", () => { + it("refuses an unserved type and a contract mismatch", async () => { + const { host } = await loaded((h) => h.serve(Info, () => ({ title: constant("t") }))); + expect(await rejection(host.get(Page, { id: 1 }, context))).toEqual({ + code: "unavailable", + message: "app.page is not served here", + }); + const InfoV2 = local("app.info", { version: 2, params: {}, members: { title: str() } }); + const Changed = local("app.info", { version: 1, params: {}, members: { title: int() } }); + expect(await rejection(host.get(InfoV2, {}, context))).toMatchObject({ code: "contract_mismatch" }); + expect(await rejection(host.get(Changed, {}, context))).toMatchObject({ code: "contract_mismatch" }); + const Same = local("app.info", { version: 1, params: {}, members: { title: str() } }); + expect((await host.get(Same, {}, context)).title.value).toBe("t"); + }); + + it("checks params with L1 (invalid/params)", async () => { + const { host } = await loaded((h) => h.serve(Page, ({ id }) => ({ id: constant(id) }))); + // @ts-expect-error: a string is not an int + expect(await rejection(host.get(Page, { id: "x" }, context))).toMatchObject({ + code: "invalid", + reason: "params", + }); + }); +}); + +describe("deadlines of gets", () => { + fakeTimers(); + + it("reject with unavailable while no attempt has failed (the last failure: see attempt.test.ts)", async () => { + const gate = Promise.withResolvers(); + const { host } = await loaded((h) => serveGated(h, gate.promise, [])); + const waiting = rejection(host.get(Svc, { id: 1 }, context, { readyTimeout: 500 })); + await vi.advanceTimersByTimeAsync(500); + expect(await waiting).toEqual({ code: "unavailable", message: "app.svc(1): did not become live within 500 ms" }); + }); + + it("include the wait for load", async () => { + const host = createHost(noErrors); + const waiting = rejection(host.get(Info, {}, context, { readyTimeout: 100 })); + await vi.advanceTimersByTimeAsync(100); + expect(await waiting).toEqual({ code: "unavailable", message: "app.info: did not become live within 100 ms" }); + }); + + it("are per getter: the first to pass releases only its own reference, the attempt goes on for the other", async () => { + const gate = Promise.withResolvers(); + const closes: string[] = []; + const { host } = await loaded((h) => serveGated(h, gate.promise, closes), { readyTimeout: 10_000 }); + const early = rejection(host.get(Svc, { id: 1 }, context, { readyTimeout: 100 })); + const late = host.get(Svc, { id: 1 }, context, { readyTimeout: 1000 }); + await vi.advanceTimersByTimeAsync(100); + expect(await early).toMatchObject({ code: "unavailable" }); + gate.resolve(); + expect((await late).n.value).toBe(1); + expect(closes).toEqual([]); + }); +}); + +describe("close", () => { + it("ends every object, the last to go live first, and rejects new calls", async () => { + const Outer = local("app.outer", { version: 1, params: {}, members: { title: str() } }); + const { host } = await loaded((h) => { + h.serve(Info, () => ({ title: constant("info") })); + h.serve(Outer, (_, self) => ({ title: self.get(Info, {}).title })); + }); + const ends: string[] = []; + const outer = await host.get(Outer, {}, context); + const info = await host.get(Info, {}, context); + void (await outer.title.watch(context)).closed.then(() => ends.push("outer")); + void (await info.title.watch(context)).closed.then(() => ends.push("info")); + await host.close(context); + expect(ends).toEqual(["outer", "info"]); + expect(await rejection(host.get(Info, {}, context))).toEqual({ + code: "released", + reason: "closed", + message: "the host is closed", + }); + expect(() => info.title.value).toThrow("app.info: ended"); + }); + + it("fails a get waiting for load", async () => { + const host = createHost(noErrors); + const got = host.get(Info, {}, context); + await host.close(context); + expect(await rejection(got)).toEqual({ code: "released", reason: "closed", message: "the host is closed" }); + }); + + it("ends every object at once: calls fail released before any close ran", async () => { + const closing = Promise.withResolvers(); + const { host } = await loaded((h) => + h.serve(Counter, (_, self) => { + self.open( + async () => null, + () => closing.promise, + ); + return { count: constant(1), bump: async () => null }; + }), + ); + const counter = await host.get(Counter, {}, context); + const closed = host.close(context); + expect(await rejection(counter.bump(context))).toEqual({ + code: "released", + reason: "ended", + message: "app.counter: ended", + }); + closing.resolve(); + await closed; + }); + + it("aborts a pending open and does not wait for it; its late value is closed with close", async () => { + const gate = Promise.withResolvers(); + const closes: string[] = []; + const { host } = await loaded((h) => serveGated(h, gate.promise, closes)); + const got = rejection(host.get(Svc, { id: 1 }, context)); + await flushed(); + await host.close(context); + expect(await got).toEqual({ code: "released", reason: "ended", message: "app.svc(1): ended" }); + gate.resolve(); + await flushed(); + expect(closes).toEqual(["svc 1 close"]); + }); + + it("joins an ending running in the background", async () => { + const closing = Promise.withResolvers(); + const { host } = await loaded((h) => + h.serve(Svc, ({ id }, self) => { + self.open( + async () => null, + () => closing.promise, + ); + return { n: constant(id) }; + }), + ); + (await host.get(Svc, { id: 1 }, context)).release(); + let closed = false; + const close = host.close(context).then(() => { + closed = true; + }); + await flushed(); + expect(closed).toBe(false); + closing.resolve(); + await close; + }); + + it("closes object by object, a dependent before its dependencies", async () => { + const View = local("app.view", { version: 1, params: { id: int() }, members: { n: int() } }); + const events: string[] = []; + /** A native whose close records when it starts and, a turn later, when it ends. */ + const open = (self: Self, name: string): void => { + self.open( + async () => null, + async () => { + events.push(`${name} closing`); + await flushed(); + events.push(`${name} closed`); + }, + ); + }; + const { host } = await loaded((h) => { + h.serve(Svc, ({ id }, self) => { + open(self, `svc ${id}`); + return { n: constant(id) }; + }); + h.serve(View, ({ id }, self) => { + open(self, `view ${id}`); + return { n: self.get(Svc, { id }).n }; + }); + }); + await host.get(View, { id: 1 }, context); + await host.get(View, { id: 2 }, context); + await host.close(context); + expect(events).toEqual([ + "view 2 closing", + "view 2 closed", + "svc 2 closing", + "svc 2 closed", + "view 1 closing", + "view 1 closed", + "svc 1 closing", + "svc 1 closed", + ]); + }); +}); + +describe("close, its order and its races", () => { + it("returns one closing to every caller: a second close waits for the first", async () => { + const closing = Promise.withResolvers(); + const { host } = await loaded((h) => + h.serve(Counter, (_, self) => { + self.open( + async () => null, + () => closing.promise, + ); + return { count: constant(1), bump: async () => null }; + }), + ); + const first = host.close(context); + let second = false; + void host.close(context).then(() => { + second = true; + }); + await flushed(); + expect(second).toBe(false); + closing.resolve(); + await first; + await flushed(); + expect(second).toBe(true); + }); + + it("tears down what a failed dependent holds with close, and waits for it", async () => { + const Pair = local("app.pair", { version: 1, params: { id: int() }, members: { n: int() } }); + const closing = Promise.withResolvers(); + const closes: string[] = []; + const { host } = await loaded((h) => { + h.serve(Svc, ({ id }, self) => { + self.open( + async () => null, + async (_value, reason) => { + closes.push(`svc ${reason.kind}`); + await closing.promise; + }, + ); + return { n: constant(id) }; + }); + h.serve(Core, (_, self) => { + self.open( + async () => { + throw new Error("offline"); + }, + () => {}, + ); + return { n: constant(0) }; + }); + h.serve(Pair, ({ id }, self) => ({ + n: self.derive([self.get(Svc, { id }).n, self.get(Core, {}).n], ([n]) => n), + })); + }); + // The pair fails on the core (R18) and keeps its references: svc(1), live before the pair, held by it alone. + const svc = await host.get(Svc, { id: 1 }, context); + void host.get(Pair, { id: 1 }, context).catch(() => {}); + await flushed(); + svc.release(); + let closed = false; + const close = host.close(context).then(() => { + closed = true; + }); + await flushed(); + expect([closes, closed]).toEqual([["svc close"], false]); + closing.resolve(); + await close; + }); + + it("goes on when onError throws: failures are still classified, teardowns still finish", async () => { + const host = createHost({ + onError: () => { + throw new Error("onError threw"); + }, + }); + const facet = defineFacet((h) => + h.serve(Svc, ({ id }, self) => { + self.open( + async () => { + if (id === 2) throw broken(new Error("no config")); + return null; + }, + () => { + throw new Error("close threw"); + }, + ); + return { n: constant(id) }; + }), + ); + await host.load([{ name: "a", facet }], context); + // The background end reports its throwing close; the next instance waits for that teardown. + (await host.get(Svc, { id: 1 }, context)).release(); + expect((await host.get(Svc, { id: 1 }, context)).n.value).toBe(1); + expect(await rejection(host.get(Svc, { id: 2 }, context))).toEqual({ + code: "unavailable", + message: "app.svc(2): no config", + }); + }); + + describe("with a backoff window running", () => { + fakeTimers(); + + it("ends an object waiting for its window and clears every timer", async () => { + const { host } = await loaded((h) => + h.serve(Svc, (_, self) => { + self.open( + async () => { + throw new Error("offline"); + }, + () => {}, + ); + return { n: constant(0) }; + }), + ); + const first = rejection(host.get(Svc, { id: 1 }, context, { readyTimeout: 100 })); + await vi.advanceTimersByTimeAsync(100); + await first; + const waiting = rejection(host.get(Svc, { id: 1 }, context)); + await flushed(); + await host.close(context); + expect(await waiting).toEqual({ code: "released", reason: "ended", message: "app.svc(1): ended" }); + expect(vi.getTimerCount()).toBe(0); + }); + }); +}); + +describe("getters", () => { + it("one cancelled while another waits for the same object: the other gets it, the object lives", async () => { + const { context: cancellable, cancel } = withCancel(context); + const factory = vi.fn((_: { id: number }) => { + cancel(new Error("cancelled")); + return { id: constant(1) }; + }); + const { host } = await loaded((h) => h.serve(Page, factory)); + const [cancelled, kept] = await Promise.allSettled([ + host.get(Page, { id: 1 }, cancellable), + host.get(Page, { id: 1 }, context), + ]); + expect(cancelled).toMatchObject({ status: "rejected", reason: new Error("cancelled") }); + expect(kept.status === "fulfilled" && kept.value.id.value).toBe(1); + await host.get(Page, { id: 1 }, context); + expect(factory).toHaveBeenCalledTimes(1); + }); +}); + +forEachCodec(() => { + describe("principals and where a reference goes", () => { + const Who = remote("app.who", { version: 1, params: { id: int() }, variants: { v: { state: { who: str() } } } }); + const Outer = remote("app.outer", { version: 1, params: {}, variants: { v: { state: { who: str() } } } }); + const Inner = local("app.inner", { version: 1, params: { id: int() }, members: { who: str() } }); + const Desk = local("app.desk", { version: 1, params: {}, members: { who: method(undefined, str()) } }); + const Gate = remote("app.gate", { + version: 1, + params: {}, + variants: { v: { methods: { who: method(undefined, str()) } } }, + }); + type P = TestPrincipal; + + /** Serves every type above: `Who` shows the principal its object is for. */ + function serveAll(h: FacetHost

, factories: string[]) { + h.serve(Who, ({ id }, principal) => { + factories.push(`${id} ${principal.kind}`); + return { variant: "v", members: { who: constant(principal.kind) } }; + }); + h.serve(Outer, (_, _principal, self) => ({ variant: "v", members: { who: self.get(Who, { id: 3 }).who } })); + h.serve(Inner, ({ id }, self) => ({ who: self.get(Who, { id }).who })); + const who = async (ctx: Ctx

) => { + const ref = await ctx.get(Who, { id: 2 }); + return ref.who.value; + }; + h.serve(Desk, () => ({ who })); + h.serve(Gate, () => ({ variant: "v", members: { who } })); + } + + it("host.get acts as the host's principal; a host without one (a client) serves no remote type", async () => { + const factories: string[] = []; + const { server } = await linked((h) => serveAll(h, factories)); + expect((await server.get(Who, { id: 1 }, context)).who.value).toBe("server"); + const client = createHost

({ onError: () => {} }); + const loading = client.load([{ name: "@test/app", facet: defineFacet

((h) => serveAll(h, [])) }], context); + await expect(loading).rejects.toThrow("app.who: a host without a principal serves only local types"); + }); + + it("ctx.get acts as ctx.principal: a guest's call gets the guest's view; a local handler's acts as the host", async () => { + const { server, client } = await linked((h) => serveAll(h, []), { child: { kind: "guest", session: "S1" } }); + expect(await (await client.get(Gate, {}, context)).who(context)).toBe("guest"); + expect(await (await server.get(Desk, {}, context)).who(context)).toBe("server"); + }); + + it("self.get acts as a remote object's principal, a local object's as the host's", async () => { + const { server, client } = await linked((h) => serveAll(h, []), { child: { kind: "guest", session: "S1" } }); + expect((await client.get(Outer, {}, context)).who.value).toBe("guest"); + expect((await server.get(Inner, { id: 4 }, context)).who.value).toBe("server"); + }); + + it("identity includes the principal key: equal keys share one object", async () => { + const factories: string[] = []; + const host = createHost

({ + principal: { kind: "guest", session: "S1" }, + principalKey: (principal) => principal.kind, + onError: () => {}, + }); + await host.load([{ name: "@test/app", facet: defineFacet

((h) => serveAll(h, factories)) }], context); + const [a, b] = memoryLinkPair(); + host.accept(a, { principal: { kind: "guest", session: "S2" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + await host.get(Who, { id: 1 }, context); + peer.send({ + kind: "hold", + hold: 0, + type: Who.name, + version: Who.version, + fingerprint: Who.fingerprint, + params: jsonPart({ id: 1 }), + }); + await peer.next("bound"); + expect(factories).toEqual(["1 guest"]); + const [c, d] = memoryLinkPair(); + host.accept(c, { principal: { kind: "owner" }, codec: testCodec() }, context); + const owner = new RawPeer(d); + owner.send({ + kind: "hold", + hold: 0, + type: Who.name, + version: Who.version, + fingerprint: Who.fingerprint, + params: jsonPart({ id: 1 }), + }); + await owner.next("bound"); + expect(factories).toEqual(["1 guest", "1 owner"]); + }); + + it("remote objects with empty params are not activated by load; they live while referenced", async () => { + const factories: string[] = []; + const Empty = remote("app.empty", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const { server } = await linked((h) => + h.serve(Empty, (_, principal) => { + factories.push(principal.kind); + return { variant: "v", members: { n: constant(1) } }; + }), + ); + expect(factories).toEqual([]); + (await server.get(Empty, {}, context)).release(); + await flushed(); + await server.get(Empty, {}, context); + expect(factories).toEqual(["server", "server"]); + }); + + it("a local type never crosses a link; a remote type without a parent link waits for one", async () => { + const { client } = await linked(() => {}); + expect(await rejection(client.get(Inner, { id: 1 }, context))).toEqual({ + code: "unavailable", + message: "app.inner is not served here", + }); + const alone = createHost

({ onError: () => {} }); + await alone.load([], context); + expect(alone.hold(Who, { id: 1 }, context).state.value).toEqual({ kind: "loading" }); + expect(await rejection(alone.get(Who, { id: 1 }, context, { readyTimeout: 10 }))).toEqual({ + code: "unavailable", + message: "app.who(1): did not become live within 10 ms", + }); + }); + + it("connect takes one open parent link", async () => { + const { client } = await linked(() => {}); + expect(() => + client.connect(memoryLinkPair()[0], { principal: { kind: "server" }, codec: testCodec() }, context), + ).toThrow("connect: this host has an open parent link"); + }); + + it("close closes the links first: the consumer sees the link drop, no gone", async () => { + const { server, client } = await linked((h) => serveAll(h, [])); + const who = await client.get(Who, { id: 1 }, context); + const watch = await who.who.watch(context); + await server.close(context); + await transported(); + expect(who.state.value).toEqual({ kind: "loading" }); + expect(who.who.value).toBe("owner"); + expect(await Promise.race([watch.closed, transported().then(() => "open")])).toBe("open"); + }); + }); +}); + +describe("a load whose setup throws", () => { + it("closes the host as close does, links included, and rejects with the setup's error", async () => { + const host = createHost({ principal: { kind: "server" }, onError: () => {} }); + const [a, b] = memoryLinkPair(); + const [c, d] = memoryLinkPair(); + host.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + host.connect(c, { principal: { kind: "server" }, codec: testCodec() }, context); + const child = new RawPeer(b); + const parent = new RawPeer(d); + const facet = defineFacet(() => { + throw new Error("setup threw"); + }); + await expect(host.load([{ name: "@test/app", facet }], context)).rejects.toThrow("setup threw"); + await transported(); + expect([child.closed, parent.closed]).toEqual([true, true]); + expect(await rejection(host.get(Page, { id: 1 }, context))).toEqual({ + code: "released", + reason: "closed", + message: "the host is closed", + }); + }); +}); + +describe("acquiring: hold and get", () => { + fakeTimers(); + + /** Serves `Svc`, whose open throws `offline` (broken when `failing.broken`) while `failing.left` is above 0. */ + function serveFlaky(h: FacetHost, failing: { left: number; broken: boolean }): void { + h.serve(Svc, ({ id }, self) => { + self.open( + async () => { + if (failing.left-- <= 0) return; + throw failing.broken ? broken(new Error("offline")) : new Error("offline"); + }, + () => {}, + ); + return { n: constant(id) }; + }); + } + + const offline = { code: "unavailable", message: "app.svc(1): offline" }; + + it("invalid params and a local type not served here: at once", async () => { + const host = createHost({ ...noErrors, principal: { kind: "server" } }); + await host.load([{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context); + const params = { code: "invalid", reason: "params" }; + expect(() => host.hold(Notes, { session: 1 } as never, context)).toThrow("params"); + expect(await rejection(host.get(Notes, { session: 1 } as never, context))).toMatchObject(params); + const unserved = { code: "unavailable", message: "app.info is not served here" }; + expect(() => host.hold(Info, {}, context)).toThrow(unserved.message); + expect(await rejection(host.get(Info, {}, context))).toEqual(unserved); + }); + + it("before load resolved: hold is loading, get waits; once it resolved both bind, or an unserved type fails", async () => { + const host = createHost(noErrors); + const held = host.hold(Info, {}, context); + const unserved = host.hold(Page, { id: 1 }, context); + const got = host.get(Info, {}, context); + const refused = rejection(host.get(Page, { id: 1 }, context)); + expect([held.state.value, unserved.state.value]).toEqual([{ kind: "loading" }, { kind: "loading" }]); + await host.load( + [{ name: "@test/app", facet: defineFacet((h) => h.serve(Info, () => ({ title: constant("t") }))) }], + context, + ); + await flushed(); + expect(held.state.value).toEqual({ kind: "live" }); + expect((await got).title.value).toBe("t"); + const error = { code: "unavailable", message: "app.page is not served here" }; + expect(unserved.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "broken", error }, + retryAt: undefined, + }); + expect(await refused).toEqual(error); + }); + + it("a hold released before load resolved makes nothing once it did", async () => { + const factory = vi.fn(({ id }: { id: number }) => ({ id: constant(id) })); + const host = createHost(noErrors); + host.hold(Page, { id: 1 }, context).release(); + await host.load([{ name: "@test/app", facet: defineFacet((h) => h.serve(Page, factory)) }], context); + await flushed(); + expect(factory).not.toHaveBeenCalled(); + }); + + it("through transient failures: hold is failed with retryAt, then loading; get waits, then rejects with the last cause", async () => { + const { host } = await loaded((h) => serveFlaky(h, { left: 5, broken: false })); + const held = host.hold(Svc, { id: 1 }, context); + const got = rejection(host.get(Svc, { id: 1 }, context, { readyTimeout: 1_500 })); + await flushed(); + expect(held.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "transient", error: offline }, + retryAt: 1_000, + }); + await vi.advanceTimersByTimeAsync(1_000); + await flushed(); + expect(held.state.value).toMatchObject({ kind: "failed", retryAt: 3_000 }); + await vi.advanceTimersByTimeAsync(500); + expect(await got).toEqual(offline); + // The getter released only its own reference: the hold follows the object's retries. + await vi.advanceTimersByTimeAsync(1_500); + expect(held.state.value).toMatchObject({ kind: "failed", retryAt: 7_000 }); + }); + + it("a broken failure: hold is failed without retryAt; get releases its reference and rejects at once", async () => { + const { host } = await loaded((h) => serveFlaky(h, { left: 1, broken: true })); + const held = host.hold(Svc, { id: 1 }, context); + expect(await rejection(host.get(Svc, { id: 1 }, context))).toEqual(offline); + expect(held.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" }, retryAt: undefined }); + }); + + it("a cancelled context: no effect on a hold; a get releases its reference and rejects", async () => { + const gate = Promise.withResolvers(); + const closes: string[] = []; + const { host } = await loaded((h) => serveGated(h, gate.promise, closes)); + const { context: cancellable, cancel } = withCancel(context); + const held = host.hold(Svc, { id: 1 }, cancellable); + const got = host.get(Svc, { id: 2 }, cancellable); + cancel(new Error("user cancelled")); + await expect(got).rejects.toThrow("user cancelled"); + gate.resolve(); + await flushed(); + expect(held.state.value).toEqual({ kind: "live" }); + // The getter's object ended with its only reference: its late value was closed. + expect(closes).toEqual(["svc 2 end"]); + }); + + it("host.close: queued calls reject released, references go released, their watches end (race 10)", async () => { + const gate = Promise.withResolvers(); + const { host } = await loaded((h) => serveGated(h, gate.promise, [])); + const local = host.hold(Svc, { id: 1 }, context); + const remoteHost = createHost(noErrors); + await remoteHost.load([], context); + const remote = remoteHost.hold(Notes, { session: "S1" }, context); + const call = remote.wait(context); + const watches = [await local.state.watch(context), await remote.state.watch(context)]; + await Promise.all([host.close(context), remoteHost.close(context)]); + expect(await rejection(call)).toEqual({ code: "released", reason: "closed", message: "the host is closed" }); + expect([local.state.value, remote.state.value]).toMatchObject([{ kind: "ended" }, { kind: "ended" }]); + expect(await Promise.all(watches.map(({ closed }) => closed))).toEqual([{ kind: "ended" }, { kind: "ended" }]); + }); +}); + +forEachCodec(() => { + describe("connect after a drop", () => { + it("throws while the previous parent link is open or still closing, and takes the next once it closed (race 4)", async () => { + const { client, links } = await linked((h) => void serveNotes(h)); + const notes = await client.get(Notes, { session: "S1" }, context); + const next = (): void => + client.connect(memoryLinkPair()[0], { principal: { kind: "server" }, codec: testCodec() }, context); + // The server's end closes now, the client's a microtask later. + links[0].close(); + expect(next).toThrow("connect: this host has an open parent link"); + await transported(); + expect(notes.state.value).toEqual({ kind: "loading" }); + expect(next).not.toThrow(); + }); + }); +}); + +forEachCodec(() => { + describe("where a reference or an arriving hold goes (section 1's table)", () => { + /** A worker for session S1 serving `PlacedNotes`, a raw peer as its child, and one as its parent when `parent`. */ + async function worker(options: { readonly parent?: boolean } = {}) { + const errors: HostError[] = []; + const host = createHost({ + address: Session({ id: "S1" }), + principal: { kind: "worker", session: "S1" }, + onError: (error) => errors.push(error), + }); + let parent: RawPeer | undefined; + if (options.parent === true) { + const [a, b] = memoryLinkPair(); + host.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + parent = new RawPeer(b); + } + await host.load( + [{ name: "@test/w", facet: defineFacet((h) => serveNotes(h, PlacedNotes)) }], + context, + ); + const [a, b] = memoryLinkPair(); + host.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + return { host, child: new RawPeer(b), parent, errors }; + } + + /** Sends a hold of `PlacedNotes` (or `type`) for `session`, naming `address` when given. */ + function holdOf(peer: RawPeer, session: string, address?: string, type: string = PlacedNotes.name) { + peer.send({ + kind: "hold", + hold: 0, + type, + version: PlacedNotes.version, + fingerprint: PlacedNotes.fingerprint, + params: jsonPart({ session }), + ...(address === undefined ? {} : { address: { kind: "session", params: { id: address } } }), + }); + } + + const misplaced = { + code: "invalid", + reason: "placement", + message: "app.placed-notes: placed on session, this host is session|S1", + }; + + it("binds a hold of a type served here that names this host's address", async () => { + const { child } = await worker(); + holdOf(child, "S1", "S1"); + expect((await child.next("bound")).holds[0]).toMatchObject({ hold: 0 }); + }); + + it("refuses a hold served here whose params place it elsewhere, whatever address it names: invalid/placement (race 5)", async () => { + const { child } = await worker(); + holdOf(child, "S2", "S1"); + expect(await child.next("failed")).toMatchObject({ + hold: 0, + cause: { kind: "broken", error: { kind: "chord", error: misplaced } }, + }); + holdOf(child, "S2"); + expect(await child.next("failed")).toMatchObject({ + hold: 0, + cause: { kind: "broken", error: { kind: "chord", error: misplaced } }, + }); + }); + + it("places a hold served here by its params, not by the address it names (that only routes it)", async () => { + const { child } = await worker(); + holdOf(child, "S1", "S2"); + expect((await child.next("bound")).holds[0]).toMatchObject({ hold: 0 }); + }); + + it("sends a reference made here whose placement is another host over the parent link, though it serves the type", async () => { + const { host, parent } = await worker({ parent: true }); + const elsewhere = host.hold(PlacedNotes, { session: "S2" }, context); + const hold = await parent!.next("hold"); + expect(hold).toMatchObject({ type: PlacedNotes.name, address: { kind: "session", params: { id: "S2" } } }); + expect(elsewhere.state.value).toEqual({ kind: "loading" }); + expect((await host.get(PlacedNotes, { session: "S1" }, context)).variant).toBe("editor"); + }); + + it("sends a hold it cannot serve or route over the parent link, unparsed", async () => { + const { child, parent } = await worker({ parent: true }); + holdOf(child, "S9", undefined, "app.elsewhere"); + const relayed = await parent!.next("hold"); + expect(relayed).toMatchObject({ type: "app.elsewhere" }); + expect(partValue(relayed.params)).toEqual({ session: "S9" }); + }); + + it("never sends a hold from the parent back to it: unavailable (race 8)", async () => { + const { parent } = await worker({ parent: true }); + holdOf(parent!, "S9", undefined, "app.elsewhere"); + expect(await parent!.next("failed")).toMatchObject({ + hold: 0, + cause: { + kind: "broken", + error: { kind: "chord", error: { code: "unavailable", message: "app.elsewhere is not served here" } }, + }, + }); + expect(parent!.kinds()).toEqual(["hello"]); + }); + + it("refuses a hold it cannot serve or route without a parent link: unavailable", async () => { + const { child } = await worker(); + holdOf(child, "S9", undefined, "app.elsewhere"); + expect(await child.next("failed")).toMatchObject({ + cause: { kind: "broken", error: { kind: "chord", error: { code: "unavailable" } } }, + }); + }); + + it("fails a reference broken with what its placement throws", async () => { + const Throwing = remote("app.throwing", { + version: 1, + params: {}, + placement: () => { + throw new Error("placement threw"); + }, + variants: { v: { state: { n: int() } } }, + }); + const client = createHost(noErrors); + await client.load([], context); + expect(await rejection(client.get(Throwing, {}, context))).toMatchObject({ + message: "app.throwing: placement threw", + }); + expect(client.hold(Throwing, {}, context).state.value).toMatchObject({ kind: "failed" }); + }); + }); +}); + +describe("the host's store", () => { + /** A store on `:memory:`, as Chord's layers see it. */ + async function memoryStore(): Promise { + return (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context)) as ChordStore; + } + + it("belongs to one open host at a time (rule 7)", async () => { + const store = await memoryStore(); + const first = createHost({ store, onError: () => {} }); + expect(() => createHost({ store, onError: () => {} })).toThrow("store: already attached to an open host"); + await first.close(context); + await createHost({ store, onError: () => {} }).close(context); + await store.close(context); + }); + + it("reports storage_failed once and closes the host; the app still closes the store", async () => { + const store = await memoryStore(); + const errors: HostError[] = []; + const host = createHost({ store, onError: (error) => errors.push(error) }); + await host.load([], context); + const broken: StagedWrite = { + order: "write", + apply: async (step) => { + await step.run("INSERT INTO no_such_table VALUES (1)"); + return { kind: "document.retire", id: 0, revision: 0 }; + }, + }; + const fence = { admit: () => true, adopt: () => {} }; + await expect(store.commit([broken], fence, context)).rejects.toMatchObject({ reason: "storage_failed" }); + await expect(store.commit([broken], fence, context)).rejects.toMatchObject({ reason: "storage_failed" }); + // The host closed itself; the app's own close returns that closing. + expect((host as ChordHost).closing).toBe(true); + await host.close(context); + expect(errors.map(({ kind }) => kind)).toEqual(["storage_failed"]); + await store.close(context); + }); +}); diff --git a/packages/chord/test/next/host/keyed-members.test.ts b/packages/chord/test/next/host/keyed-members.test.ts new file mode 100644 index 000000000..d2570dd56 --- /dev/null +++ b/packages/chord/test/next/host/keyed-members.test.ts @@ -0,0 +1,452 @@ +// Tests of keyed members across attempts (slice 6a §1.2, §1.4, §12): the keys references hold on an object's keyed +// members, frozen while it is failed, re-bound by name (loaded first) when its next attempt goes live, released once. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withAbortSignal } from "../../../src/context/index.ts"; +import { int, keyed, local, obj, str } from "../../../src/next/contracts/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import { defineState, type KeyedHandle } from "../../../src/next/documents/index.ts"; +import type { Self } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { + changesOf, + delivered, + expectCollected, + failLive, + instancesOf, + loaded, + memoryStore, + valuesOf, +} from "../support.ts"; + +const attempted = instancesOf(ChordObject, "attempt"); + +type N = { n: number }; +const Schema = obj({ n: int() }); +const Notes = defineState("notes", { schema: Schema, version: 1, initial: () => ({ n: 0 }), keyed: true }); +const Own = defineState("own", { schema: Schema, version: 1, initial: () => ({ n: 0 }) }); +const Provider = local("app.provider", { version: 1, params: { id: str() }, members: { notes: keyed(Schema) } }); +const Exposer = local("app.exposer", { version: 1, params: {}, members: { notes: keyed(Schema) } }); +/** A viewer with params: it ends with its last reference (an object with empty params lives as long as its host). */ +const Viewer = local("app.viewer", { version: 1, params: { id: int() }, members: { value: Schema, double: int() } }); + +/** Providers `a` and `b`, an exposer re-exposing provider `which()`'s notes, and a viewer of its key `k`. */ +async function served(which: () => string) { + const providers: Record> = {}; + const exposers: Self[] = []; + const { host } = await loaded((h) => { + h.serve(Provider, ({ id }, self) => { + const notes = self.state(Notes); + providers[id] = notes; + return { notes }; + }); + h.serve(Exposer, (_, self) => { + exposers.push(self); + self.state(Own); + return { notes: self.get(Provider, { id: which() }).notes }; + }); + h.serve(Viewer, (_, self) => { + const value = self.get(Exposer, {}).notes.at("k"); + return { value, double: self.derive([value], ([v]) => v.n * 2) }; + }); + }); + const a = await host.get(Provider, { id: "a" }, context); + const b = await host.get(Provider, { id: "b" }, context); + const set = (id: string, n: number) => + providers[id]!.change( + "k", + (draft) => { + draft.n = n; + }, + context, + ); + await set("a", 1); + await set("b", 2); + const exposer = await host.get(Exposer, {}, context); + const viewer = await host.get(Viewer, { id: 1 }, context); + const fail = (): void => + failLive(attempted, exposers.at(-1)!, new ChordError({ code: "unavailable", message: "boom" })); + return { host, a, b, exposer, viewer, set, fail, providers }; +} + +describe("a failed object's held keys", () => { + it("keep their last values and get no frames (at, watch, a derive over at); the provider goes on", async () => { + const { host, a, exposer, viewer, set, fail } = await served(() => "a"); + const values = changesOf(await viewer.value.watch(context)); + const keyWatch = (await exposer.notes.watch("k", context))!; + const keyed = changesOf(keyWatch); + const direct = changesOf((await a.notes.watch("k", context))!); + fail(); + await set("a", 3); + await delivered(); + expect(exposer.state.value.kind).toBe("failed"); + expect([viewer.value.value, viewer.double.value, keyWatch.value]).toEqual([{ n: 1 }, 2, { n: 1 }]); + expect([values, keyed]).toEqual([[], []]); + expect(valuesOf(direct)).toEqual([{ n: 3 }]); + await host.close(context); + }); +}); + +describe("re-binding when a new attempt goes live", () => { + it("follows the new attempt's member of the same name (one frame); the old provider's pin is released", async () => { + let which = "a"; + const { host, exposer, viewer, set, fail, providers } = await served(() => which); + const values = changesOf(await viewer.value.watch(context)); + const doubles = changesOf(await viewer.double.watch(context)); + const keyWatch = (await exposer.notes.watch("k", context))!; + const keyed = changesOf(keyWatch); + const pins = (id: string) => + (providers[id] as unknown as { slot: { source(key: string): { pins: number } | undefined } }).slot.source("k") + ?.pins; + expect(pins("a")).toBe(1); + fail(); + which = "b"; + exposer.retry(); + (await host.get(Exposer, {}, context)).release(); + await delivered(); + expect([viewer.value.value, viewer.double.value, keyWatch.value]).toEqual([{ n: 2 }, 4, { n: 2 }]); + expect([pins("a"), pins("b")]).toEqual([undefined, 1]); + await set("b", 4); + await set("a", 5); + await delivered(); + expect(valuesOf(values)).toEqual([{ n: 2 }, { n: 4 }]); + expect(valuesOf(keyed)).toEqual([{ n: 2 }, { n: 4 }]); + expect(valuesOf(doubles)).toEqual([4, 8]); + await host.close(context); + }); + + it("loads a cold persisted key of the new member before the new attempt goes live", async () => { + const Stored = defineState("notes", { + schema: Schema, + version: 1, + initial: () => ({ n: 0 }), + keyed: true, + persist: true, + checkpointWhen: () => false, + }); + const Durable = local("app.durable", { version: 1, params: {}, members: { notes: keyed(Schema) } }); + const store = await memoryStore(); + let memos: KeyedHandle | undefined; + const serveDurable = (h: Parameters[0]>[0]) => + h.serve(Durable, (_, self) => { + memos = self.state(Stored); + return { notes: memos }; + }); + const seed = await loaded(serveDurable, { store }); + await seed.host.get(Durable, {}, context); + await memos!.change( + "k", + (draft) => { + draft.n = 10; + }, + context, + ); + await seed.host.close(context); + // A new host: the key is in the store only. + let runs = 0; + const exposers: Self[] = []; + const { host } = await loaded( + (h) => { + h.serve(Provider, (_, self) => ({ notes: self.state(Notes) })); + serveDurable(h); + h.serve(Exposer, (_, self) => { + exposers.push(self); + self.state(Own); + return { notes: ++runs === 1 ? self.get(Provider, { id: "a" }).notes : self.get(Durable, {}).notes }; + }); + h.serve(Viewer, (_, self) => { + const value = self.get(Exposer, {}).notes.at("k"); + return { value, double: self.derive([value], ([v]) => v.n * 2) }; + }); + }, + { store }, + ); + const exposer = await host.get(Exposer, {}, context); + const viewer = await host.get(Viewer, { id: 1 }, context); + failLive(attempted, exposers[0]!, new ChordError({ code: "unavailable", message: "boom" })); + exposer.retry(); + (await host.get(Exposer, {}, context)).release(); + expect([viewer.value.value, viewer.double.value]).toEqual([{ n: 10 }, 20]); + await host.close(context); + await store.close(context); + }); +}); + +/** A provider with a persisted keyed state (empty params: a stable identity), whose keys a new host loads cold. */ +const ColdProvider = local("app.cold-provider", { version: 1, params: {}, members: { notes: keyed(Schema) } }); +/** A persisted keyed state whose keys a new host loads from the store. */ +const Stored = defineState("notes", { + schema: Schema, + version: 1, + initial: () => ({ n: 0 }), + keyed: true, + persist: true, + checkpointWhen: () => false, +}); + +/** The pins on key `k` of a provider's notes (`undefined`: none). */ +function pinsOf(notes: KeyedHandle): number | undefined { + // The handle's slot keeps the key's source while it is pinned. + return (notes as unknown as { slot: { source(key: string): { pins: number } | undefined } }).slot.source("k")?.pins; +} + +/** + * Provider `a` (its key `k` in the store only, its loads held at `gate` once `hold()` is called) and `b` (in memory), + * an exposer re-exposing `which()`'s notes, and a viewer of key `k`. + */ +async function cold(which: () => string) { + const store = await memoryStore(); + let seeded: KeyedHandle | undefined; + const seed = await loaded( + (h) => + h.serve(ColdProvider, (_, self) => { + seeded = self.state(Stored); + return { notes: seeded }; + }), + { + store, + }, + ); + await seed.host.get(ColdProvider, {}, context); + await seeded!.change( + "k", + (draft) => { + draft.n = 1; + }, + context, + ); + await seed.host.close(context); + // A new host: the key is in the store only. + const providers: Record> = {}; + const exposers: Self[] = []; + const { host, errors } = await loaded( + (h) => { + h.serve(ColdProvider, (_, self) => { + providers.a = self.state(Stored); + return { notes: providers.a }; + }); + h.serve(Provider, ({ id }, self) => { + providers[id] = self.state(Notes); + return { notes: providers[id] }; + }); + h.serve(Exposer, (_, self) => { + exposers.push(self); + self.state(Own); + const notes = which() === "a" ? self.get(ColdProvider, {}).notes : self.get(Provider, { id: "b" }).notes; + return { notes }; + }); + h.serve(Viewer, (_, self) => { + const value = self.get(Exposer, {}).notes.at("k"); + return { value, double: self.derive([value], ([v]) => v.n * 2) }; + }); + }, + { store }, + ); + await host.get(ColdProvider, {}, context); + await host.get(Provider, { id: "b" }, context); + await providers.b!.change( + "k", + (draft) => { + draft.n = 2; + }, + context, + ); + const exposer = await host.get(Exposer, {}, context); + const gate = Promise.withResolvers(); + const entered = Promise.withResolvers(); + const hold = (): void => { + // The slot's persistence reads cold keys from the store; the test holds the read open. + const persistence = (providers.a as unknown as { slot: { persistence: { read(...args: unknown[]): unknown } } }) + .slot.persistence; + const read = persistence.read.bind(persistence); + persistence.read = async (...args: unknown[]) => { + const result = await read(...args); + entered.resolve(); + await gate.promise; + return result; + }; + }; + const fail = (): void => + failLive(attempted, exposers.at(-1)!, new ChordError({ code: "unavailable", message: "boom" })); + const close = async (): Promise => { + await host.close(context); + await store.close(context); + }; + return { host, errors, exposer, providers, hold, gate, entered, fail, close }; +} + +describe("a key whose first pin still loads", () => { + it("the object fails and re-binds meanwhile: the consumer retries on the new member, the late pin is released", async () => { + let which = "a"; + const { host, errors, exposer, providers, hold, gate, entered, fail, close } = await cold(() => which); + hold(); + const loading = host.get(Viewer, { id: 1 }, context); + await entered.promise; + fail(); + which = "b"; + exposer.retry(); + (await host.get(Exposer, {}, context)).release(); + gate.resolve(); + const viewer = await loading; + await delivered(); + expect([viewer.value.value, viewer.double.value]).toEqual([{ n: 2 }, 4]); + expect([pinsOf(providers.a!), pinsOf(providers.b!)]).toEqual([undefined, 1]); + expect(errors.filter((error) => error.kind === "activation_failed").length).toBe(1); + const values = changesOf(await viewer.value.watch(context)); + await providers.a!.change( + "k", + (draft) => { + draft.n = 3; + }, + context, + ); + await providers.b!.change( + "k", + (draft) => { + draft.n = 4; + }, + context, + ); + await delivered(); + expect(valuesOf(values)).toEqual([{ n: 4 }]); + await close(); + }); + + it("the object fails and stays failed: the consumer's activation fails transient, never live on it", async () => { + const { host, errors, exposer, providers, hold, gate, entered, fail, close } = await cold(() => "a"); + hold(); + const abort = new AbortController(); + const loading = host.get(Viewer, { id: 1 }, withAbortSignal(abort.signal, context)); + await entered.promise; + fail(); + gate.resolve(); + await delivered(); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(exposer.state.value.kind).toBe("failed"); + const failure = errors.find((error) => error.kind === "activation_failed"); + expect(failure).toMatchObject({ failure: { kind: "transient" } }); + abort.abort(); + await expect(loading).rejects.toThrow(); + expect(pinsOf(providers.a!)).toBeUndefined(); + await close(); + }); + + it("the object ends meanwhile: the consumer's wait rejects, the late pin is released", async () => { + const { host, providers, hold, gate, entered, close } = await cold(() => "a"); + hold(); + const loading = host.get(Viewer, { id: 1 }, context); + await entered.promise; + const closing = host.close(context); + gate.resolve(); + await expect(loading).rejects.toThrow(); + await closing; + expect(pinsOf(providers.a!)).toBeUndefined(); + await close(); + }); +}); + +describe("a keyed watch whose object closes while it is set up", () => { + it("rejects with the object's end error, whichever microtask the close lands in", async () => { + for (let gap = 0; gap < 40; gap++) { + let notes: KeyedHandle | undefined; + const { host } = await loaded((h) => + h.serve(Exposer, (_, self) => { + notes = self.state(Notes); + return { notes }; + }), + ); + const exposer = await host.get(Exposer, {}, context); + await notes!.change( + "a", + (draft) => { + draft.n = 1; + }, + context, + ); + const watching = exposer.notes.watch("a", context).then( + () => "watch", + (error: unknown) => `${(error as ChordError).code}/${(error as ChordError).reason}`, + ); + for (let turn = 0; turn < gap; turn++) await Promise.resolve(); + await host.close(context); + expect(["watch", "released/ended"]).toContain(await watching); + } + }); +}); + +describe("chains and releases", () => { + it("a key re-exposed through two objects freezes with the middle one and follows its next attempt", async () => { + const Outer = local("app.outer", { version: 1, params: {}, members: { notes: keyed(Schema) } }); + const OuterViewer = local("app.outer-viewer", { version: 1, params: {}, members: { value: Schema } }); + let which = "a"; + const middles: Self[] = []; + const providers: Record> = {}; + const { host } = await loaded((h) => { + h.serve(Provider, ({ id }, self) => { + const notes = self.state(Notes); + providers[id] = notes; + return { notes }; + }); + h.serve(Exposer, (_, self) => { + middles.push(self); + self.state(Own); + return { notes: self.get(Provider, { id: which }).notes }; + }); + h.serve(Outer, (_, self) => ({ notes: self.get(Exposer, {}).notes })); + h.serve(OuterViewer, (_, self) => ({ value: self.get(Outer, {}).notes.at("k") })); + }); + for (const id of ["a", "b"]) { + await host.get(Provider, { id }, context); + await providers[id]!.change( + "k", + (draft) => { + draft.n = id === "a" ? 1 : 2; + }, + context, + ); + } + const middle = await host.get(Exposer, {}, context); + const viewer = await host.get(OuterViewer, {}, context); + const values = changesOf(await viewer.value.watch(context)); + failLive(attempted, middles[0]!, new ChordError({ code: "unavailable", message: "boom" })); + await providers.a!.change( + "k", + (draft) => { + draft.n = 3; + }, + context, + ); + await delivered(); + expect([viewer.value.value, values]).toEqual([{ n: 1 }, []]); + which = "b"; + middle.retry(); + (await host.get(Exposer, {}, context)).release(); + await delivered(); + expect(viewer.value.value).toEqual({ n: 2 }); + expect(valuesOf(values)).toEqual([{ n: 2 }]); + await host.close(context); + }); + + it("a consumer released while the object re-activates leaves no hold behind; nothing is kept for it", async () => { + let which = "a"; + const refs = await (async () => { + const { host, exposer, viewer, fail, providers } = await served(() => which); + const pins = (id: string) => + (providers[id] as unknown as { slot: { source(key: string): { pins: number } | undefined } }).slot.source( + "k", + )?.pins; + fail(); + which = "b"; + exposer.retry(); + viewer.release(); + (await host.get(Exposer, {}, context)).release(); + // The viewer's teardown releases its references. + await new Promise((resolve) => setTimeout(resolve, 10)); + expect([pins("a"), pins("b")]).toEqual([undefined, undefined]); + const kept = [new WeakRef(viewer), new WeakRef(providers.a!)]; + await host.close(context); + return kept; + })(); + await expectCollected(...refs); + }); +}); diff --git a/packages/chord/test/next/host/member-source.test.ts b/packages/chord/test/next/host/member-source.test.ts new file mode 100644 index 000000000..77939396b --- /dev/null +++ b/packages/chord/test/next/host/member-source.test.ts @@ -0,0 +1,507 @@ +// Tests of member sources (slice 6a §1.2, §1.4): a local object's state members as references and other objects' +// derives follow them across attempts: passed on while live, frozen while failed, re-bound by name when live again. + +import { describe, expect, it } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, int, keyed, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Self, type StoreSelf } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { resolveWatchable } from "../../../src/next/host/reference.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import type { Source, Subscriber } from "../../../src/next/state/internal.ts"; +import { + changesOf, + delivered, + expectCollected, + failLive, + holdNextCommit, + instancesOf, + linked, + loaded, + memoryStore, + metered, + transported, + valuesOf, +} from "../support.ts"; + +const attempted = instancesOf(ChordObject, "attempt"); + +const List = defineState("list", { schema: obj({ items: arr(str()) }), version: 1, initial: () => ({ items: [] }) }); +const Store = local("app.store", { + version: 1, + params: {}, + members: { list: obj({ items: arr(str()) }), count: int(), add: method(obj({ text: str() })) }, +}); +const Viewer = local("app.count-viewer", { version: 1, params: {}, members: { tens: int() } }); + +/** Serves `Store` (a list, its count, `add`) and `Viewer` (a derive over the store's count); `attempts` are the store's. */ +async function served(options: Parameters[1] = {}) { + const attempts: Self[] = []; + const lists: SingletonHandle<{ items: string[] }>[] = []; + const { host, errors } = await loaded((h) => { + h.serve(Store, (_, self) => { + attempts.push(self); + const list = self.state(List); + lists.push(list); + return { + list, + count: self.derive([list], ([l]) => l.items.length), + add: async ({ text }: { text: string }, ctx: Context) => { + await list.change((draft) => void draft.items.push(text), ctx); + return null; + }, + }; + }); + h.serve(Viewer, (_, self) => ({ tens: self.derive([self.get(Store, {}).count], ([n]) => n * 10) })); + }, options); + const store = await host.get(Store, {}, context); + const fail = (): void => + failLive(attempted, attempts.at(-1)!, new ChordError({ code: "unavailable", message: "boom" })); + return { host, errors, store, attempts, lists, fail }; +} + +describe("following the live attempt", () => { + it("passes the member's frames on as they are, and keeps a derive member lazy while nothing watches it", async () => { + const { telemetry, counts } = await metered(); + const { host, store, lists: handles } = await served({ telemetry }); + expect(store.count.value).toBe(0); + const computes = (): number => counts()["chord.derive.computes"] ?? 0; + const before = computes(); + expect(before).toBeGreaterThan(0); + await store.add({ text: "a" }, context); + await store.add({ text: "b" }, context); + // Unwatched: marked only, computed by the next read. + expect(computes()).toBe(before); + expect(store.count.value).toBe(2); + const watch = await store.count.watch(context); + const counted = changesOf(watch); + const lists = changesOf(await store.list.watch(context)); + const own = changesOf(await handles[0]!.watch(context)); + await store.add({ text: "c" }, context); + await delivered(); + expect(counted).toEqual([{ kind: "value", value: 3, ops: [["r", 3]], retired: false }]); + // The slot's own change, not a copy. + expect(lists).toHaveLength(1); + expect(lists[0]).toBe(own[0]); + await watch.stop(); + await delivered(); + const stopped = computes(); + await store.add({ text: "d" }, context); + expect(computes()).toBe(stopped); + await host.close(context); + }); +}); + +describe("a failed object's members", () => { + it("keep their last values and get no frames; a new watch opens at the last value (slice 6a §1.4)", async () => { + const { host, store, lists, fail } = await served(); + await store.add({ text: "a" }, context); + const counted = changesOf(await store.count.watch(context)); + fail(); + await delivered(); + expect(store.state.value.kind).toBe("failed"); + const late = await store.count.watch(context); + expect(late.value).toBe(1); + const lately = changesOf(late); + const listed = changesOf(await store.list.watch(context)); + // The failed attempt's handle is sealed; the slot changes only through a new attempt. + expect(lists[0]!.value).toEqual({ items: ["a"] }); + store.retry(); + await (await host.get(Store, {}, context)).add({ text: "b" }, context); + await delivered(); + expect(valuesOf(counted)).toEqual([2]); + expect(valuesOf(listed)).toEqual([{ items: ["a", "b"] }]); + expect(valuesOf(lately)).toEqual([2]); + await host.close(context); + }); + + it("re-exposed from another object freeze with the re-exposing object; the provider's source goes on", async () => { + const Note = obj({ title: str() }); + const Memo = defineState("memo", { schema: Note, version: 1, initial: () => ({ title: "" }), keyed: true }); + const Own = defineState("own", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + const Provider = local("app.provider", { + version: 1, + params: {}, + members: { n: int(), notes: keyed(Note), set: method(int()) }, + }); + const Exposer = local("app.exposer", { version: 1, params: {}, members: { n: int(), note: Note } }); + let exposer: Self | undefined; + const { host } = await loaded((h) => { + h.serve(Provider, (_, self) => { + const memos = self.state(Memo); + const n = self.derive([memos.at("a")], ([note]) => note.title.length); + return { + n, + notes: memos, + set: async (length, ctx) => { + await memos.change( + "a", + (draft) => { + draft.title = "x".repeat(length); + }, + ctx, + ); + return null; + }, + }; + }); + h.serve(Exposer, (_, self) => { + exposer = self; + self.state(Own); + const provider = self.get(Provider, {}); + return { n: provider.n, note: provider.notes.at("a") }; + }); + }); + const provider = await host.get(Provider, {}, context); + const exposed = await host.get(Exposer, {}, context); + await provider.set(1, context); + const exposedN = changesOf(await exposed.n.watch(context)); + const exposedNote = changesOf(await exposed.note.watch(context)); + const providerN = changesOf(await provider.n.watch(context)); + failLive(attempted, exposer!, new ChordError({ code: "unavailable", message: "boom" })); + await delivered(); + await provider.set(7, context); + await delivered(); + expect(exposed.state.value.kind).toBe("failed"); + expect([exposed.n.value, exposed.note.value]).toEqual([1, { title: "x" }]); + expect([exposedN, exposedNote]).toEqual([[], []]); + expect(provider.n.value).toBe(7); + expect(valuesOf(providerN)).toEqual([7]); + await host.close(context); + }); +}); + +describe("the window of a live failure", () => { + it("members following another object's sources freeze at once, while an admitted commit of the object runs", async () => { + const Counter = defineState("counter", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + const Persisted = defineState("own", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: 0 }), + persist: true, + checkpointWhen: () => false, + }); + const Provider = local("app.provider", { version: 1, params: {}, members: { n: int(), set: method(int()) } }); + const Exposer = local("app.exposer", { version: 1, params: {}, members: { n: int(), own: obj({ n: int() }) } }); + const store = await memoryStore(); + let exposer: StoreSelf | undefined; + let own: SingletonHandle<{ n: number }> | undefined; + const { host } = await loaded( + (h) => { + h.serve(Provider, (_, self) => { + const doc = self.state(Counter); + return { + n: self.derive([doc], ([d]) => d.n), + set: async (n, ctx) => { + await doc.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }); + h.serve(Exposer, (_, self) => { + exposer = self; + own = self.state(Persisted); + return { n: self.get(Provider, {}).n, own }; + }); + }, + { store }, + ); + const provider = await host.get(Provider, {}, context); + const exposed = await host.get(Exposer, {}, context); + await provider.set(1, context); + const frames = changesOf(await exposed.n.watch(context)); + const owned = changesOf(await exposed.own.watch(context)); + const held = holdNextCommit(store); + const change = own!.change((draft) => { + draft.n = 5; + }, context); + await held.entered; + failLive(attempted, exposer!, new ChordError({ code: "unavailable", message: "boom" })); + await provider.set(9, context); + await delivered(); + expect([exposed.n.value, frames]).toEqual([1, []]); + held.release(); + await change; + await delivered(); + // The object's own admitted commit still publishes once. + expect(valuesOf(owned)).toEqual([{ n: 5 }]); + await host.close(context); + await store.close(context); + }); +}); + +describe("re-binding when a new attempt goes live", () => { + it("recovers a consumer's derive that failed on the provider's earlier error, and the derives downstream", async () => { + const Counted = local("app.counted", { version: 1, params: {}, members: { count: int(), add: method(str()) } }); + const Tens = local("app.tens", { version: 1, params: {}, members: { tens: int(), hundreds: int() } }); + const selves: Self[] = []; + const { host } = await loaded((h) => { + h.serve(Counted, (_, self) => { + selves.push(self); + const first = selves.length === 1; + const list = self.state(List); + return { + count: self.derive([list], ([l]) => { + if (first && l.items.length === 1) throw new Error("count broke"); + return l.items.length; + }), + add: async (text, ctx) => { + await list.change((draft) => void draft.items.push(text), ctx); + return null; + }, + }; + }); + h.serve(Tens, (_, self) => { + const tens = self.derive([self.get(Counted, {}).count], ([n]) => n * 10); + return { tens, hundreds: self.derive([tens], ([n]) => n * 10) }; + }); + }); + const counted = await host.get(Counted, {}, context); + const tens = await host.get(Tens, {}, context); + await counted.add("a", context); + expect(() => tens.tens.value).toThrow("count broke"); + expect(() => tens.hundreds.value).toThrow("count broke"); + failLive(attempted, selves[0]!, new ChordError({ code: "unavailable", message: "boom" })); + counted.retry(); + (await host.get(Counted, {}, context)).release(); + expect([tens.tens.value, tens.hundreds.value]).toEqual([10, 100]); + const frames = changesOf(await tens.hundreds.watch(context)); + await counted.add("b", context); + await delivered(); + expect(valuesOf(frames)).toEqual([200]); + await host.close(context); + }); + + it("re-binds a consumer's derive over a retried provider's derive member, and its watches get frames again", async () => { + const { host, store, fail } = await served(); + const viewer = await host.get(Viewer, {}, context); + const tens = changesOf(await viewer.tens.watch(context)); + const counted = changesOf(await store.count.watch(context)); + await store.add({ text: "a" }, context); + await delivered(); + fail(); + await delivered(); + expect([store.count.value, viewer.tens.value]).toEqual([1, 10]); + store.retry(); + await (await host.get(Store, {}, context)).add({ text: "b" }, context); + await delivered(); + expect(viewer.state.value.kind).toBe("live"); + expect([store.count.value, viewer.tens.value]).toEqual([2, 20]); + expect(valuesOf(tens)).toEqual([10, 20]); + expect(valuesOf(counted)).toEqual([1, 2]); + await host.close(context); + }); + + it("sends one frame from the last value when the new attempt's member differs", async () => { + const Shown = local("app.shown", { version: 1, params: {}, members: { label: str() } }); + const labels = ["old", "new"]; + let attempt = 0; + const selves: Self[] = []; + const { host } = await loaded((h) => + h.serve(Shown, (_, self) => { + selves.push(self); + self.state(List); + const label = labels[attempt++]!; + return { label: self.derive([], () => label) }; + }), + ); + const shown = await host.get(Shown, {}, context); + const frames = changesOf(await shown.label.watch(context)); + failLive(attempted, selves[0]!, new ChordError({ code: "unavailable", message: "boom" })); + shown.retry(); + (await host.get(Shown, {}, context)).release(); + await delivered(); + expect(shown.label.value).toBe("new"); + expect(frames).toEqual([{ kind: "value", value: "new", ops: [["r", "new"]], retired: false }]); + await host.close(context); + }); +}); + +describe("subscribers", () => { + it("a hold over a link on a remote object re-exposing a local member gets frames again after the local retry", async () => { + const Count = local("app.count", { version: 1, params: {}, members: { n: obj({ n: int() }), add: method() } }); + const View = remote("app.view", { version: 1, params: {}, variants: { v: { state: { n: obj({ n: int() }) } } } }); + const Counter = defineState("counter", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + const selves: Self[] = []; + const { server, client } = await linked((h) => { + h.serve(Count, (_, self) => { + selves.push(self); + const counter = self.state(Counter); + return { + n: self.derive([counter], ([c]) => ({ n: c.n * 10 })), + add: async (ctx: Context) => { + await counter.change((draft) => void draft.n++, ctx); + return null; + }, + }; + }); + h.serve(View, (_, _principal, self) => ({ variant: "v", members: { n: self.get(Count, {}).n } })); + }); + const count = await server.get(Count, {}, context); + const view = await client.get(View, {}, context); + if (view.variant !== "v") throw new Error("expected variant v"); + const frames = changesOf(await view.n.watch(context)); + await count.add(context); + await transported(); + failLive(attempted, selves[0]!, new ChordError({ code: "unavailable", message: "boom" })); + count.retry(); + await (await server.get(Count, {}, context)).add(context); + await transported(); + expect(view.n.value).toEqual({ n: 20 }); + expect(valuesOf(frames)).toEqual([{ n: 10 }, { n: 20 }]); + await client.close(context); + await server.close(context); + }); +}); + +describe("subscribers, directly", () => { + it("are fed the member's calls: a failure, then its recovery at the re-bind as a root replacement", async () => { + const Counted = local("app.counted", { version: 1, params: {}, members: { count: int(), add: method(str()) } }); + const selves: Self[] = []; + const { host } = await loaded((h) => + h.serve(Counted, (_, self) => { + selves.push(self); + const first = selves.length === 1; + const list = self.state(List); + return { + count: self.derive([list], ([l]) => { + if (first && l.items.length === 2) throw new Error("count broke"); + return l.items.length; + }), + add: async (text, ctx) => { + await list.change((draft) => void draft.items.push(text), ctx); + return null; + }, + }; + }), + ); + const counted = await host.get(Counted, {}, context); + const source = resolveWatchable(counted.count) as Source; + const events: unknown[] = []; + const subscriber: Subscriber = { + advance: (call) => + void events.push(call.kind === "value" ? [call.value, call.ops] : (call.error as Error).message), + end: (end) => void events.push(end.kind), + }; + expect(source.value).toBe(0); + source.subscribe(subscriber); + await counted.add("a", context); + await counted.add("b", context); + failLive(attempted, selves[0]!, new ChordError({ code: "unavailable", message: "boom" })); + counted.retry(); + await (await host.get(Counted, {}, context)).add("c", context); + expect(counted.count.value).toBe(3); + expect(events).toEqual([[1, [["r", 1]]], "count broke", [2, [["r", 2]]], [3, [["r", 3]]]]); + await host.close(context); + }); +}); + +describe("a member the new attempt's variant lacks", () => { + it("fails: its watch gets one failure call and stays open, reads throw invalid/member; a later attempt brings it back", async () => { + const Two = remote("app.two", { + version: 1, + params: {}, + variants: { full: { state: { n: int() } }, view: { state: { m: int() } } }, + }); + const owner = { kind: "owner" } as const; + const host = createHost({ principal: owner, onError: () => {} }); + const selves: Self[] = []; + const facet = defineFacet((h) => + h.serve(Two, (_, _principal, self) => { + selves.push(self); + self.state(List); + return selves.length === 2 + ? { variant: "view", members: { m: constant(2) } } + : { variant: "full", members: { n: constant(selves.length) } }; + }), + ); + await host.load([{ name: "@test/app", facet }], context); + const two = await host.get(Two, {}, context); + if (two.variant !== "full") throw new Error("expected the full variant"); + const n = two.n; + const calls = changesOf(await n.watch(context)); + const boom = new ChordError({ code: "unavailable", message: "boom" }); + failLive(attempted, selves[0]!, boom); + two.retry(); + (await host.get(Two, {}, context)).release(); + await delivered(); + expect(calls).toEqual([ + { kind: "failed", error: expect.objectContaining({ code: "invalid", reason: "member" }) }, + ]); + expect(() => n.value).toThrow('app.two (variant view) has no state member "n"'); + // The same as over a link: the member comes back with the next attempt that has it. + failLive(attempted, selves[1]!, boom); + two.retry(); + (await host.get(Two, {}, context)).release(); + await delivered(); + expect(n.value).toBe(3); + expect(calls.at(-1)).toEqual({ kind: "value", value: 3, ops: [["r", 3]], retired: false }); + await host.close(context); + }); +}); + +describe("an ended object's members", () => { + it("mark the derives over them: a consumer's derive keeps no cached value, its read throws released", async () => { + const Paged = local("app.paged", { version: 1, params: { id: int() }, members: { n: int() } }); + const Reader = local("app.reader", { version: 1, params: {}, members: { twice: int() } }); + const { host } = await loaded((h) => { + h.serve(Paged, () => ({ n: constant(4) })); + h.serve(Reader, (_, self) => ({ twice: self.derive([self.get(Paged, { id: 1 }).n], ([n]) => n * 2) })); + }); + const reader = await host.get(Reader, {}, context); + expect(reader.twice.value).toBe(8); + for (const object of attempted) { + const paged = object.deref(); + if (paged?.label === "app.paged(1)") paged.end(); + } + expect(() => reader.twice.value).toThrow(expect.objectContaining({ code: "released" })); + await host.close(context); + }); +}); + +describe("member sources and memory", () => { + it("a failed attempt is collected after a retry while the object, its member source and a retained watch live", async () => { + const attempts: WeakRef[] = []; + const { host } = await loaded((h) => + h.serve(Store, (_, self) => { + attempts.push(new WeakRef(self)); + const list = self.state(List); + return { + list, + count: self.derive([list], ([l]) => l.items.length), + add: async ({ text }, ctx) => { + await list.change((draft) => void draft.items.push(text), ctx); + return null; + }, + }; + }), + ); + const store = await host.get(Store, {}, context); + const frames = changesOf(await store.count.watch(context)); + failLive(attempted, attempts[0]!.deref()!, new ChordError({ code: "unavailable", message: "boom" })); + store.retry(); + (await host.get(Store, {}, context)).release(); + await store.add({ text: "a" }, context); + await delivered(); + expect(valuesOf(frames)).toEqual([1]); + await expectCollected(attempts[0]!); + expect(attempts[1]!.deref()).toBeDefined(); + await host.close(context); + }); + + it("an ended object's member sources are collected with it", async () => { + const refs = await (async () => { + const { host, store } = await served(); + const source = resolveWatchable(store.count); + expect(source?.value).toBe(0); + const ref = new WeakRef(source!); + await host.close(context); + return [ref]; + })(); + await expectCollected(...refs); + }); +}); diff --git a/packages/chord/test/next/host/members.test.ts b/packages/chord/test/next/host/members.test.ts new file mode 100644 index 000000000..2efc060e0 --- /dev/null +++ b/packages/chord/test/next/host/members.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method } from "../../../src/next/contracts/index.ts"; +import { loaded } from "../support.ts"; + +describe("the member check (Q14)", () => { + it("refuses an inherited property for a member", async () => { + const Named = local("app.named", { version: 1, params: { id: int() }, members: { toString: method() } }); + const { host } = await loaded((h) => h.serve(Named, () => ({}) as never)); + await expect(host.get(Named, { id: 1 }, context)).rejects.toThrow( + "app.named(1): toString: a method is a function", + ); + }); + + it("names a derive in errors by its member when exposed, else by its position", async () => { + const Named = local("app.named", { version: 1, params: { n: int() }, members: { label: int() } }); + const bad = { value: 1 }; + const { host } = await loaded((h) => + h.serve(Named, ({ n }, self) => { + const inner = self.derive(n === 1 ? [] : [bad as never], () => 1); + return { label: self.derive(n === 1 ? [inner, bad as never] : [inner], () => 2) }; + }), + ); + await expect(host.get(Named, { n: 1 }, context)).rejects.toThrow( + "app.named(1): label: source 1 is not a JSON watchable", + ); + await expect(host.get(Named, { n: 2 }, context)).rejects.toThrow( + "app.named(2): derive #1: source 0 is not a JSON watchable", + ); + }); +}); diff --git a/packages/chord/test/next/host/native.test.ts b/packages/chord/test/next/host/native.test.ts new file mode 100644 index 000000000..d2ad9ae97 --- /dev/null +++ b/packages/chord/test/next/host/native.test.ts @@ -0,0 +1,244 @@ +import { describe, expect, expectTypeOf, it, vi } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { int, local } from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import type { FacetHost, Native } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { failures, fakeTimers, flushed, loaded, rejection } from "../support.ts"; + +const Res = local("app.res", { version: 1, params: { id: int() }, members: { n: int() } }); + +describe("opening", () => { + it("opens one after another in declaration order; a later opener reads an earlier handle", async () => { + const events: string[] = []; + const first = Promise.withResolvers(); + const { host } = await loaded((h) => + h.serve(Res, (_, self) => { + const a = self.open( + async () => { + events.push("open a"); + return first.promise; + }, + () => {}, + ); + self.open( + async () => { + events.push(`open b with ${a.value}`); + return "b"; + }, + () => {}, + ); + return { n: constant(1) }; + }), + ); + const got = host.get(Res, { id: 1 }, context); + await flushed(); + expect(events).toEqual(["open a"]); + first.resolve("a"); + await got; + expect(events).toEqual(["open a", "open b with a"]); + }); + + it("opens nothing for an object ended before its first open started", async () => { + const opens = vi.fn(async () => null); + const { host } = await loaded((h) => + h.serve(Res, (_, self) => { + self.open(opens, () => {}); + return { n: constant(1) }; + }), + ); + host.hold(Res, { id: 1 }, context).release(); + await flushed(); + expect(opens).not.toHaveBeenCalled(); + }); + + it("throws not_ready before its open resolved and released once its close started", async () => { + const reads: unknown[] = []; + const read = (native: Native): void => { + try { + reads.push(native.value); + } catch (error) { + reads.push(isChordError(error) ? error.toJSON() : error); + } + }; + const { host } = await loaded((h) => + h.serve(Res, (_, self) => { + const a: Native = self.open( + async () => { + read(b); + return "a"; + }, + () => read(a), + ); + const b: Native = self.open( + async () => "b", + () => read(a), + ); + read(a); + return { n: constant(1) }; + }), + ); + (await host.get(Res, { id: 1 }, context)).release(); + await flushed(); + expect(reads).toEqual([ + // In the factory, and b in a's opener. + { code: "not_ready", message: "app.res(1) native #1: not open yet" }, + { code: "not_ready", message: "app.res(1) native #2: not open yet" }, + // a in b's close (b closes first), then in a's own close. + "a", + { code: "released", reason: "closed", message: "app.res(1) native #1: closed" }, + ]); + }); + + it("a partly failed attempt closes what it opened, with reason end", async () => { + const closes: string[] = []; + const { host, errors } = await loaded((h) => + h.serve(Res, (_, self) => { + self.open( + async () => "a", + (value, reason) => void closes.push(`${value} ${reason.kind}`), + ); + self.open( + async () => { + throw new Error("offline"); + }, + () => void closes.push("b"), + ); + return { n: constant(1) }; + }), + ); + expect(await rejection(host.get(Res, { id: 1 }, context, { readyTimeout: 10 }))).toEqual({ + code: "unavailable", + message: "app.res(1): offline", + }); + expect(closes).toEqual(["a end"]); + expect(failures(errors)).toEqual(["app.res(1): transient"]); + await host.close(context); + }); + + it.each(["end", "close"])( + "aborts the opener's context when the object ends while opening; a late value is closed once (%s)", + async (reason) => { + const opener = Promise.withResolvers(); + const late = Promise.withResolvers(); + const closes: unknown[] = []; + const { host } = await loaded((h) => + h.serve(Res, (_, self) => { + self.open( + async (ctx) => { + opener.resolve(ctx); + return late.promise; + }, + // The close runs under the object's context, not under the opener's aborted one. + (value, reason, ctx) => void closes.push([value, reason.kind, ctx.abortSignal?.aborted]), + ); + return { n: constant(1) }; + }), + ); + const { context: cancellable, cancel } = withCancel(context); + const got = host.get(Res, { id: 1 }, cancellable); + const openContext = await opener.promise; + if (reason === "end") cancel(new Error("left")); + else await host.close(context); + await expect(got).rejects.toThrow(reason === "end" ? "left" : "app.res(1): ended"); + expect(openContext.abortSignal?.aborted).toBe(true); + late.resolve("value"); + await flushed(); + await host.close(context); + expect(closes).toEqual([["value", reason, false]]); + }, + ); +}); + +describe("closing", () => { + it("closes in reverse, with reason end when the object ends and close when the host closes", async () => { + const closes: string[] = []; + const { host } = await loaded((h) => + h.serve(Res, ({ id }, self) => { + for (const name of ["a", "b"]) { + self.open( + async () => `${id}${name}`, + (value, reason) => void closes.push(`${value} ${reason.kind}`), + ); + } + return { n: constant(id) }; + }), + ); + const one = await host.get(Res, { id: 1 }, context); + await host.get(Res, { id: 2 }, context); + one.release(); + await flushed(); + await host.close(context); + expect(closes).toEqual(["1b end", "1a end", "2b close", "2a close"]); + }); + + it("collects every throwing close in host.close; an object ending in the background reports each", async () => { + const thrown = (name: string) => new Error(`close ${name} threw`); + const { host, errors } = await loaded((h) => + h.serve(Res, ({ id }, self) => { + for (const name of ["a", "b"]) { + self.open( + async () => name, + () => { + throw thrown(`${id}${name}`); + }, + ); + } + return { n: constant(id) }; + }), + ); + const one = await host.get(Res, { id: 1 }, context); + await host.get(Res, { id: 2 }, context); + one.release(); + await flushed(); + expect(errors).toEqual([ + { kind: "close_failed", object: "app.res(1)", native: { kind: "native", position: 2 }, error: thrown("1b") }, + { kind: "close_failed", object: "app.res(1)", native: { kind: "native", position: 1 }, error: thrown("1a") }, + ]); + const closed = await host.close(context).catch((error: unknown) => error); + expect(closed).toBeInstanceOf(AggregateError); + expect((closed as AggregateError).errors).toEqual([thrown("2b"), thrown("2a")]); + }); + + describe("timeouts", () => { + fakeTimers(); + + it("report a close still running after its closeTimeout, abort its context then, and close the others", async () => { + const events: string[] = []; + let slow: Context | undefined; + const { host, errors } = await loaded( + (h) => + h.serve(Res, (_, self) => { + self.open( + async () => "a", + () => void events.push("close a"), + ); + self.open( + async () => "b", + (_value, _reason, ctx) => { + slow = ctx; + return new Promise(() => {}); + }, + { closeTimeout: 500 }, + ); + return { n: constant(1) }; + }), + { closeTimeout: 10_000 }, + ); + (await host.get(Res, { id: 1 }, context)).release(); + await vi.advanceTimersByTimeAsync(499); + expect([events, slow?.abortSignal?.aborted]).toEqual([[], false]); + await vi.advanceTimersByTimeAsync(1); + expect([events, slow?.abortSignal?.aborted]).toEqual([["close a"], true]); + expect(errors).toEqual([ + { kind: "close_timeout", object: "app.res(1)", native: { kind: "native", position: 2 } }, + ]); + }); + }); +}); + +describe("facets", () => { + it("cannot open natives", () => { + expectTypeOf().not.toHaveProperty("open"); + }); +}); diff --git a/packages/chord/test/next/host/object.test.ts b/packages/chord/test/next/host/object.test.ts new file mode 100644 index 000000000..2df9253e8 --- /dev/null +++ b/packages/chord/test/next/host/object.test.ts @@ -0,0 +1,1047 @@ +import { describe, expect, it, vi } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { broken, ChordError } from "../../../src/next/data/index.ts"; +import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import type { Self } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { jsonPart, type Message } from "../../../src/next/link/internal.ts"; +import { constant, type Watchable } from "../../../src/next/state/index.ts"; +import { + changesOf, + delivered, + expectCollected, + failLive, + failures, + fakeTimers, + flushed, + holdNextCommit, + instancesOf, + loaded, + memoryStore, + metered, + RawPeer, + rawWorkers, + rejection, + relayed, + restartable, + Session, + StoreRestartable, + storeRestartable, + testCodec, + transported, + valuesOf, +} from "../support.ts"; + +const List = defineState("list", { schema: obj({ items: arr(str()) }), version: 1, initial: () => ({ items: [] }) }); +const COUNTER = { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) } as const; +const Store = local("app.store", { + version: 1, + params: {}, + members: { count: int(), add: method(obj({ text: str() })), clear: method() }, +}); +const Page = local("app.page", { version: 1, params: { id: int() }, members: { label: str(), slow: method() } }); + +/** The store: a list document, its count, and methods to add and clear. */ +function serveStore(self: Self) { + const list = self.state(List); + return { + count: self.derive([list], ([l]) => l.items.length), + add: async ({ text }: { text: string }, ctx: Context) => { + await list.change((draft) => draft.items.push(text), ctx); + return null; + }, + clear: async (ctx: Context) => { + await list.retire(ctx); + return null; + }, + }; +} + +const Svc = local("app.svc", { version: 1, params: { id: int() }, members: { n: int() } }); +const Core = local("app.core", { version: 1, params: {}, members: { n: int() } }); + +/** + * A factory with one native whose open throws "offline" on the attempts `fails` picks (counted from 1), and the times + * (fake, from 0) of its attempts. `gate`, when given, holds every open that does not fail until it resolves. + */ +function flaky(fails: (attempt: number) => boolean, gate?: Promise) { + const times: number[] = []; + const factory = (_: unknown, self: Self) => { + times.push(Date.now()); + const attempt = times.length; + self.open( + async () => { + if (fails(attempt)) throw new Error("offline"); + await gate; + return null; + }, + () => {}, + ); + return { n: constant(attempt) }; + }; + return { times, factory }; +} + +describe("identity and lifetime", () => { + it("one object per params key; a params type lives from its first get to its last release", async () => { + const factory = vi.fn(({ id }: { id: number }, self: Self) => { + const store = self.get(Store, {}); + return { + label: self.derive([store.count], ([count]) => `${id}: ${count}`), + slow: async () => null, + }; + }); + const { host } = await loaded((h) => { + h.serve(Store, (_, self) => serveStore(self)); + h.serve(Page, factory); + }); + const [a, b, c] = await Promise.all([ + host.get(Page, { id: 1 }, context), + host.get(Page, { id: 1 }, context), + host.get(Page, { id: 2 }, context), + ]); + expect(factory).toHaveBeenCalledTimes(2); + const watch = await a.label.watch(context); + a.release(); + expect(b.label.value).toBe("1: 0"); + b.release(); + expect(await watch.closed).toEqual({ kind: "ended" }); + const again = await host.get(Page, { id: 1 }, context); + expect(factory).toHaveBeenCalledTimes(3); + expect([again.label.value, c.label.value]).toEqual(["1: 0", "2: 0"]); + }); + + it("keeps a type with empty params across releases", async () => { + const factory = vi.fn((_: object, self: Self) => serveStore(self)); + const { host } = await loaded((h) => h.serve(Store, factory)); + const first = await host.get(Store, {}, context); + await first.add({ text: "milk" }, context); + first.release(); + const second = await host.get(Store, {}, context); + expect(second.count.value).toBe(1); + expect(factory).toHaveBeenCalledTimes(1); + }); + + it("drops an ended object's documents and detaches its derives from other objects' members", async () => { + const telemetry = await metered(); + const { host } = await loaded( + (h) => { + h.serve(Store, (_, self) => serveStore(self)); + h.serve(Page, ({ id }, self) => ({ + label: self.derive([self.get(Store, {}).count], ([count]) => `${id}: ${count}`), + slow: async () => null, + })); + }, + { telemetry: telemetry.telemetry }, + ); + const store = await host.get(Store, {}, context); + const page = await host.get(Page, { id: 1 }, context); + const watch = await page.label.watch(context); + page.release(); + expect(await watch.closed).toEqual({ kind: "ended" }); + const before = telemetry.counts()["chord.derive.computes"]; + await store.add({ text: "x" }, context); + expect(telemetry.counts()["chord.derive.computes"]).toBe(before); + }); +}); + +describe("retries", () => { + fakeTimers(); + + it("retry a transient failure after 1 s, doubling up to 60 s, and start over after a success", async () => { + const { times, factory } = flaky((attempt) => attempt <= 8 || attempt === 10); + const { host } = await loaded((h) => h.serve(Svc, factory)); + const got = host.get(Svc, { id: 1 }, context, { readyTimeout: 1_000_000 }); + await vi.advanceTimersByTimeAsync(183_000); + (await got).release(); + expect(times).toEqual([0, 1000, 3000, 7000, 15000, 31000, 63000, 123000, 183000]); + // The success deleted the backoff: a new object tries at once and waits 1 s after its failure. + const again = host.get(Svc, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(1000); + expect((await again).n.value).toBe(11); + expect(times.slice(9)).toEqual([183000, 184000]); + }); + + it("share one attempt among concurrent gets", async () => { + const { times, factory } = flaky(() => false); + const { host } = await loaded((h) => h.serve(Svc, factory)); + const refs = await Promise.all([1, 2, 3].map(() => host.get(Svc, { id: 1 }, context))); + expect([times, refs.map(({ n }) => n.value)]).toEqual([[0], [1, 1, 1]]); + }); + + it("never retry a broken failure; a new object after its end tries at once", async () => { + const bug = vi.fn(() => { + throw new Error("bug"); + }); + const { host } = await loaded((h) => { + h.serve(Svc, bug); + h.serve(Core, bug); + }); + expect(await rejection(host.get(Svc, { id: 1 }, context))).toEqual({ + code: "unavailable", + message: "app.svc(1): bug", + }); + await vi.advanceTimersByTimeAsync(120_000); + expect(bug).toHaveBeenCalledTimes(2); + expect(await rejection(host.get(Core, {}, context))).toEqual({ code: "unavailable", message: "app.core: bug" }); + await expect(host.get(Svc, { id: 1 }, context)).rejects.toThrow("bug"); + expect(bug).toHaveBeenCalledTimes(3); + }); + + it("end a failed object with its last reference; its backoff entry expires on its own", async () => { + const { times, factory } = flaky(() => true); + const { host } = await loaded((h) => h.serve(Svc, factory)); + const first = rejection(host.get(Svc, { id: 1 }, context, { readyTimeout: 500 })); + await vi.advanceTimersByTimeAsync(500); + expect(await first).toEqual({ code: "unavailable", message: "app.svc(1): offline" }); + await vi.advanceTimersByTimeAsync(4500); + expect([times, vi.getTimerCount()]).toEqual([[0], 0]); + // The entry expired: a new object tries at once and its window starts over at 1 s. + const second = rejection(host.get(Svc, { id: 1 }, context, { readyTimeout: 1500 })); + await vi.advanceTimersByTimeAsync(1500); + await second; + expect(times).toEqual([0, 5000, 6000]); + }); + + it("make a get right after a failure wait for the scheduled retry", async () => { + const { times, factory } = flaky((attempt) => attempt === 1); + const { host } = await loaded((h) => h.serve(Svc, factory)); + const first = rejection(host.get(Svc, { id: 1 }, context, { readyTimeout: 500 })); + await vi.advanceTimersByTimeAsync(500); + await first; + const second = host.get(Svc, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(499); + expect(times).toEqual([0]); + await vi.advanceTimersByTimeAsync(1); + expect((await second).n.value).toBe(2); + expect(times).toEqual([0, 1000]); + }); + + it("start a new instance only after the previous one closed", async () => { + const closing = Promise.withResolvers(); + const factory = vi.fn((_: { id: number }, self: Self) => { + self.open( + async () => null, + () => closing.promise, + ); + return { n: constant(factory.mock.calls.length) }; + }); + const { host } = await loaded((h) => h.serve(Svc, factory)); + (await host.get(Svc, { id: 1 }, context)).release(); + const again = host.get(Svc, { id: 1 }, context); + await flushed(); + expect(factory).toHaveBeenCalledTimes(1); + closing.resolve(); + expect((await again).n.value).toBe(2); + }); +}); + +describe("dependency recovery (R18)", () => { + fakeTimers(); + const View = local("app.view", { version: 1, params: { id: int() }, members: { n: int() } }); + + /** A view over `Type`'s `n`, and the times of its attempts. */ + function view(Type: typeof Svc | typeof Core) { + const times: number[] = []; + const factory = ({ id }: { id: number }, self: Self) => { + times.push(Date.now()); + return { n: Type === Core ? self.get(Core, {}).n : self.get(Svc, { id }).n }; + }; + return { times, factory }; + } + + it("fails a dependent transient with the dependency's cause and retries it at once when it goes live", async () => { + const core = flaky((attempt) => attempt === 1); + const viewer = view(Core); + const { host, errors } = await loaded((h) => { + h.serve(Core, core.factory); + h.serve(View, viewer.factory); + }); + await vi.advanceTimersByTimeAsync(500); + const got = host.get(View, { id: 1 }, context); + await flushed(); + const failed = errors.find((error) => error.kind === "activation_failed" && error.object === "app.view(1)"); + expect(failed).toMatchObject({ + failure: { + kind: "transient", + error: { code: "unavailable", message: "app.view(1): dependency app.core failed" }, + }, + }); + expect(failed?.kind === "activation_failed" && failed.failure.error.cause).toMatchObject({ + message: "app.core: offline", + }); + // The core's retry at 1 s succeeds; the view is retried then, not at the end of its own window (1.5 s). + await vi.advanceTimersByTimeAsync(500); + expect((await got).n.value).toBe(2); + await vi.advanceTimersByTimeAsync(5000); + expect(viewer.times).toEqual([500, 1000]); + }); + + it("keeps the dependency while failed; releasing the dependent releases it", async () => { + const svc = flaky(() => true); + const viewer = view(Svc); + const { host } = await loaded((h) => { + h.serve(Svc, svc.factory); + h.serve(View, viewer.factory); + }); + const got = rejection(host.get(View, { id: 1 }, context, { readyTimeout: 1500 })); + await vi.advanceTimersByTimeAsync(1500); + expect(await got).toEqual({ code: "unavailable", message: "app.view(1): dependency app.svc(1) failed" }); + await vi.advanceTimersByTimeAsync(10_000); + // Held by the failed view, the dependency was retried at 1 s; released with it at 1.5 s, it is not any more. + expect(svc.times).toEqual([0, 1000]); + }); + + it("binds a retry's references before releasing the previous attempt's", async () => { + const svc = flaky((attempt) => attempt === 1); + const viewer = view(Svc); + const { host } = await loaded((h) => { + h.serve(Svc, svc.factory); + h.serve(View, viewer.factory); + }); + const got = host.get(View, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(1000); + // Held only by the view: released first, the live dependency would have ended and started anew. + expect((await got).n.value).toBe(2); + expect([svc.times, viewer.times]).toEqual([ + [0, 1000], + [0, 1000], + ]); + }); + + it("ignores a trigger while an attempt runs: simultaneous triggers give one attempt", async () => { + const gate = Promise.withResolvers(); + const svc = flaky((attempt) => attempt === 1, gate.promise); + const viewer = view(Svc); + const { host } = await loaded((h) => { + h.serve(Svc, svc.factory); + h.serve(View, viewer.factory); + }); + const got = host.get(View, { id: 1 }, context); + // At 1 s both windows end: the dependency's retry waits at its gate, the view's waits for it. + await vi.advanceTimersByTimeAsync(1000); + gate.resolve(); + expect((await got).n.value).toBe(2); + expect(viewer.times).toEqual([0, 1000]); + }); + + it("retries a dependent of a broken dependency with its backoff, for as long as it is held", async () => { + const viewer = view(Core); + const { host } = await loaded((h) => { + h.serve(Core, () => { + throw new Error("bug"); + }); + h.serve(View, viewer.factory); + }); + const got = rejection(host.get(View, { id: 1 }, context, { readyTimeout: 16_000 })); + await vi.advanceTimersByTimeAsync(16_000); + expect(await got).toEqual({ code: "unavailable", message: "app.view(1): dependency app.core failed" }); + expect(viewer.times).toEqual([0, 1000, 3000, 7000, 15000]); + }); + + it("fails every object of a cycle broken; none is retried", async () => { + const Ping = local("app.ping", { version: 1, params: {}, members: { n: int() } }); + const Pong = local("app.pong", { version: 1, params: {}, members: { n: int() } }); + const factories = { ping: vi.fn(), pong: vi.fn() }; + const { host, errors } = await loaded((h) => { + h.serve(Ping, (_, self) => { + factories.ping(); + return { n: self.get(Pong, {}).n }; + }); + h.serve(Pong, (_, self) => { + factories.pong(); + return { n: self.get(Ping, {}).n }; + }); + }); + expect(failures(errors).sort()).toEqual(["app.ping: broken", "app.pong: broken"]); + await vi.advanceTimersByTimeAsync(120_000); + expect([factories.ping.mock.calls.length, factories.pong.mock.calls.length]).toEqual([1, 1]); + expect(await rejection(host.get(Pong, {}, context))).toEqual({ + code: "invalid", + reason: "cycle", + message: "dependency cycle: app.ping → app.pong → app.ping", + }); + }); +}); + +describe("cycle checks", () => { + it("stay linear on shared dependencies (a layered graph, every object depending on the whole next layer)", async () => { + const Layer = local("app.layer", { + version: 1, + params: { layer: int(), index: int() }, + members: { depth: int() }, + }); + const layers = 28; + const { host } = await loaded((h) => + h.serve(Layer, ({ layer }, self) => { + if (layer === layers) return { depth: constant(0) }; + const next = [0, 1].map((index) => self.get(Layer, { layer: layer + 1, index }).depth); + return { depth: self.derive(next, ([a]) => (a as number) + 1) }; + }), + ); + const telemetry = await metered(); + const top = await host.get(Layer, { layer: 0, index: 0 }, telemetry.context); + expect(top.depth.value).toBe(layers); + // 55 objects, 2 waits each: a search per wait visits at most every object once (an exponential one 2^28 chains). + expect(telemetry.counts()["chord.cycle.visits"]).toBeLessThanOrEqual(2 * 55 * 55); + }); +}); + +describe("GC", () => { + it("collects an ended object with its documents, derives and watches", async () => { + const refs: WeakRef[] = []; + const { host } = await loaded((h) => { + h.serve(Store, (_, self) => serveStore(self)); + h.serve(Page, (_, self) => { + const list = self.state(List); + const label = self.derive([list, self.get(Store, {}).count], ([l, c]) => `${l.items.length}/${c}`); + refs.push(new WeakRef(self), new WeakRef(list), new WeakRef(label)); + return { label, slow: async () => null }; + }); + }); + const store = await host.get(Store, {}, context); + await (async () => { + const page = await host.get(Page, { id: 1 }, context); + const watch = await page.label.watch(context); + refs.push(new WeakRef(watch), new WeakRef(page)); + page.release(); + })(); + await store.add({ text: "x" }, context); + await delivered(); + await expectCollected(...refs); + }); + + it("keeps nothing in a released reference or a stopped watch", async () => { + const { host } = await loaded((h) => h.serve(Store, (_, self) => serveStore(self))); + const store = await host.get(Store, {}, context); + const held = await host.get(Store, {}, context); + const refs = await (async () => { + const watch = await store.count.watch(context); + await watch.stop(); + const reference = await host.get(Store, {}, context); + const watched: Watchable = reference.count; + await watched.watch(context); + reference.release(); + return [new WeakRef(watch), new WeakRef(reference)]; + })(); + await held.add({ text: "x" }, context); + await expectCollected(...refs); + }); +}); + +describe("GC of released references", () => { + it("keeps neither the declaring object nor the host in a released self.get reference", async () => { + const { reference, refs } = await (async () => { + const refs: WeakRef[] = []; + let declared: unknown; + const { host } = await loaded((h) => { + h.serve(Store, (_, self) => serveStore(self)); + h.serve(Page, (_, self) => { + declared = self.get(Store, {}); + refs.push(new WeakRef(self)); + return { label: constant(""), slow: async () => null }; + }); + }); + (await host.get(Page, { id: 1 }, context)).release(); + await flushed(); + refs.push(new WeakRef(host)); + return { reference: declared, refs }; + })(); + await expectCollected(...refs); + expect(reference).toBeDefined(); + }); +}); + +describe("GC of failed objects", () => { + fakeTimers(); + + it("collects one after its last reference while its backoff entry exists; the entry expires on its own", async () => { + const refs: WeakRef[] = []; + const { host, errors } = await loaded((h) => { + h.serve(Store, (_, self) => serveStore(self)); + h.serve(Page, (_, self) => { + const list = self.state(List); + const label = self.derive([list, self.get(Store, {}).count], ([l, c]) => `${l.items.length}/${c}`); + const native = self.open( + async () => ({ buffer: new Uint8Array(16) }), + () => {}, + ); + self.open( + async () => { + throw new Error("offline"); + }, + () => {}, + ); + const slow = async () => null; + refs.push(new WeakRef(self), new WeakRef(list), new WeakRef(label), new WeakRef(native), new WeakRef(slow)); + return { label, slow }; + }); + }); + const got = rejection(host.get(Page, { id: 1 }, context, { readyTimeout: 100 })); + await vi.advanceTimersByTimeAsync(100); + await got; + expect(vi.getTimerCount()).toBe(1); + // The reports' errors keep their stack frames' receivers until formatted (V8): an app logs and drops them. + errors.length = 0; + await expectCollected(...refs); + await vi.advanceTimersByTimeAsync(900); + expect(vi.getTimerCount()).toBe(0); + }); +}); + +describe("ref.retry()", () => { + fakeTimers(); + + it("skips a transient failure's backoff window: an attempt now", async () => { + const { times, factory } = flaky((attempt) => attempt === 1); + const { host } = await loaded((h) => h.serve(Svc, factory)); + const svc = host.hold(Svc, { id: 1 }, context); + await flushed(); + expect(svc.state.value.kind).toBe("failed"); + vi.advanceTimersByTime(200); + svc.retry(); + await flushed(); + expect([times, svc.state.value, svc.n.value]).toEqual([[0, 200], { kind: "live" }, 2]); + }); + + it("starts an object waiting for its identity's backoff window at once", async () => { + const { times, factory } = flaky((attempt) => attempt === 1); + const { host } = await loaded((h) => h.serve(Svc, factory)); + const first = host.hold(Svc, { id: 1 }, context); + await flushed(); + // The failed object ends with its reference; its identity's window (1 s) still runs. + first.release(); + vi.advanceTimersByTime(100); + const second = host.hold(Svc, { id: 1 }, context); + await flushed(); + expect([times, second.state.value]).toEqual([[0], { kind: "loading" }]); + second.retry(); + await flushed(); + expect([times, second.state.value]).toEqual([[0, 100], { kind: "live" }]); + }); + + it("is ignored while an attempt runs and once live (race 11)", async () => { + const gate = Promise.withResolvers(); + const { times, factory } = flaky((attempt) => attempt === 1, gate.promise); + const { host } = await loaded((h) => h.serve(Svc, factory)); + const svc = host.hold(Svc, { id: 1 }, context); + await flushed(); + svc.retry(); + svc.retry(); + await flushed(); + expect(times).toEqual([0, 0]); + gate.resolve(); + await flushed(); + svc.retry(); + await vi.advanceTimersByTimeAsync(5_000); + expect([times, svc.state.value]).toEqual([[0, 0], { kind: "live" }]); + }); + + it("renews a broken object for every reference (race 13)", async () => { + let fixed = false; + const factory = vi.fn((_: { id: number }, self: Self) => { + self.open( + async () => { + if (!fixed) throw broken(new Error("no configuration")); + }, + () => {}, + ); + return { n: constant(1) }; + }); + const { host } = await loaded((h) => h.serve(Svc, factory)); + const one = host.hold(Svc, { id: 1 }, context); + const two = host.hold(Svc, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(120_000); + expect([one.state.value.kind, two.state.value.kind, factory.mock.calls.length]).toEqual(["failed", "failed", 1]); + fixed = true; + one.retry(); + expect(two.state.value).toEqual({ kind: "loading" }); + await flushed(); + expect([one.state.value, two.state.value]).toEqual([{ kind: "live" }, { kind: "live" }]); + }); +}); + +describe("linger", () => { + fakeTimers(); + const Tally = local("app.tally", { version: 1, params: { id: int() }, members: { n: int(), bump: method() } }); + const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + + /** Serves `Tally` with `linger`: a document, a derive that fails at 2, a native whose closes `closes` records. */ + async function tallies(linger: number, open: () => Promise = async () => {}) { + const closes: string[] = []; + const factory = vi.fn(({ id }: { id: number }, self: Self) => { + const doc = self.state(N); + self.open(open, (_, reason) => void closes.push(`${id} ${reason.kind}`)); + return { + n: self.derive([doc], ([{ n }]) => { + if (n === 2) throw new Error("two"); + return n; + }), + bump: async (ctx: Context) => { + await doc.change((draft) => { + draft.n++; + }, ctx); + return null; + }, + }; + }); + const { host } = await loaded((h) => h.serve(Tally, factory, { linger })); + return { host, factory, closes }; + } + + it("keeps a live object live for linger after its last release: a get within it reuses it, with its documents", async () => { + const { host, factory, closes } = await tallies(500); + const first = await host.get(Tally, { id: 1 }, context); + await first.bump(context); + first.release(); + await vi.advanceTimersByTimeAsync(499); + const second = await host.get(Tally, { id: 1 }, context); + expect([second.n.value, factory.mock.calls.length, closes]).toEqual([1, 1, []]); + second.release(); + await vi.advanceTimersByTimeAsync(500); + expect(closes).toEqual(["1 end"]); + expect((await host.get(Tally, { id: 1 }, context)).n.value).toBe(0); + expect(factory).toHaveBeenCalledTimes(2); + }); + + it("a reuse cancels the expiry: the holder count decides, a later release starts a new window (race 8)", async () => { + const { host, closes } = await tallies(500); + (await host.get(Tally, { id: 1 }, context)).release(); + await vi.advanceTimersByTimeAsync(499); + const held = await host.get(Tally, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(5_000); + held.release(); + await vi.advanceTimersByTimeAsync(499); + expect(closes).toEqual([]); + await vi.advanceTimersByTimeAsync(1); + expect(closes).toEqual(["1 end"]); + }); + + it("only a live object lingers: an activating or failed one ends with its last reference (race 9)", async () => { + const gate = Promise.withResolvers(); + const activating = await tallies(500, () => gate.promise); + const opening = activating.host.hold(Tally, { id: 1 }, context); + await flushed(); + opening.release(); + gate.resolve(); + await flushed(); + // Its open's late value was closed at once. + expect(activating.closes).toEqual(["1 end"]); + const failing = await tallies(500, async () => { + throw broken(new Error("no configuration")); + }); + const failed = failing.host.hold(Tally, { id: 1 }, context); + await flushed(); + failed.release(); + failing.host.hold(Tally, { id: 1 }, context); + await flushed(); + expect(failing.factory).toHaveBeenCalledTimes(2); + }); + + it("a failed state member does not end a lingering object", async () => { + const { host, factory, closes } = await tallies(500); + const tally = await host.get(Tally, { id: 1 }, context); + await tally.bump(context); + await tally.bump(context); + expect(() => tally.n.value).toThrow("two"); + tally.release(); + await vi.advanceTimersByTimeAsync(400); + await host.get(Tally, { id: 1 }, context); + expect([factory.mock.calls.length, closes]).toEqual([1, []]); + }); + + it("host.close ends a lingering object at once", async () => { + const { host, closes } = await tallies(60_000); + (await host.get(Tally, { id: 1 }, context)).release(); + await host.close(context); + expect([closes, vi.getTimerCount()]).toEqual([["1 close"], 0]); + }); + + it("a lingering object is collected after its expiry", async () => { + const refs: WeakRef[] = []; + const { host } = await loaded((h) => + h.serve( + Svc, + ({ id }, self) => { + refs.push(new WeakRef(self)); + return { n: constant(id) }; + }, + { linger: 100 }, + ), + ); + (await host.get(Svc, { id: 1 }, context)).release(); + await vi.advanceTimersByTimeAsync(100); + vi.useRealTimers(); + await expectCollected(...refs); + }); +}); + +/** Every object attempted in this file, weakly: the objects of the cycles among them. */ +const attempted = instancesOf(ChordObject, "attempt"); + +/** On the server (no placement): its activation holds `Far` on the session's worker. */ +const Near = remote("app.near", { version: 1, params: { session: str() }, variants: { v: { state: { n: int() } } } }); + +/** On the session's worker: its activation holds `Near` back on the server, or, with `local`, `Middle`. */ +const Far = remote("app.far", { + version: 1, + params: { session: str() }, + placement: ({ session }) => Session({ id: session }), + variants: { v: { state: { n: int() } } }, +}); + +/** On the server, held by `Far`: its activation holds `Hub` on its own host (a cycle closed on one host). */ +const Middle = remote("app.middle", { + version: 1, + params: { session: str() }, + variants: { v: { state: { n: int() } } }, +}); + +/** On the server, a local type (no principal in its identity): its activation holds `Far` as the host. */ +const Hub = local("app.hub", { version: 1, params: { session: str() }, members: { n: int() } }); + +/** `Loop({ here, there })` lives on `here`'s worker and holds `Loop({ here: there, there: here })`. */ +const Loop = remote("app.loop", { + version: 1, + params: { here: str(), there: str() }, + placement: ({ here }) => Session({ id: here }), + variants: { v: { state: { n: int() } } }, +}); + +/** A tab, a server and its workers over trusted route links, serving the cycle types (`Far` holds `back`). */ +async function cyclic(back: "near" | "middle" = "near", trusted = true) { + return relayed({ + trusted, + server: (h) => { + h.serve(Near, ({ session }, _principal, self) => ({ + variant: "v", + members: { n: self.get(Far, { session }).n }, + })); + h.serve(Middle, ({ session }, _principal, self) => ({ + variant: "v", + members: { n: self.get(Hub, { session }).n }, + })); + h.serve(Hub, ({ session }, self) => ({ n: self.get(Far, { session }).n })); + }, + worker: (h) => { + h.serve(Far, ({ session }, _principal, self) => ({ + variant: "v", + members: { n: back === "near" ? self.get(Near, { session }).n : self.get(Middle, { session }).n }, + })); + h.serve(Loop, ({ here, there }, _principal, self) => ({ + variant: "v", + members: { n: self.get(Loop, { here: there, there: here }).n }, + })); + }, + }); +} + +describe("cycles across hosts (Q5)", () => { + it("refuse a hold whose chain names the activation it would bind: server → worker → server", async () => { + const { tab, errors } = await cyclic(); + const error = await rejection(tab.get(Near, { session: "S1" }, context)); + // The worker's hold back acts as the worker (upward boundary): it meets the cycle one turn later, at that + // principal's `Near`. + const message = "dependency cycle: app.near(S1) → … → app.near(S1)"; + expect(error).toEqual({ code: "invalid", reason: "cycle", message }); + // Proposal: every object on the cycle fails broken with the same error, none transient (nothing retries it). + const failed = new Set(failures([...errors.server, ...errors.worker])); + expect([...failed].sort()).toEqual(["app.far(S1): broken", "app.near(S1): broken"]); + }); + + it("find a cycle closed on one host through another: server → worker → server, then a local get", async () => { + const { server } = await cyclic("middle"); + const error = await rejection(server.get(Hub, { session: "S1" }, context)); + expect(error).toEqual({ + code: "invalid", + reason: "cycle", + message: "dependency cycle: app.hub(S1) → … → app.middle(S1) → app.hub(S1)", + }); + }); + + it("carry the chain through a relay (race 7): worker → server → worker → server → worker", async () => { + const { tab, errors } = await cyclic(); + const error = await rejection(tab.get(Loop, { here: "S1", there: "S2" }, context)); + expect(error).toMatchObject({ code: "invalid", reason: "cycle" }); + expect([...new Set(failures(errors.worker))].sort()).toEqual([ + "app.loop(S1|S2): broken", + "app.loop(S2|S1): broken", + ]); + }); + + it("drop the chain from holds arriving on untrusted links, and send none for holds made outside an activation", async () => { + const workers: RawPeer[] = []; + const relay = await relayed({ resolve: rawWorkers(workers) }); + const [a, b] = memoryLinkPair(); + relay.server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const tab = new RawPeer(b); + const forged = "0190f5a2-7c3e-7d4a-9b1c-2d3e4f5a6b7c"; + tab.send({ ...farHold(0), chain: [forged] }); + await transported(); + await transported(); + const held = await workers[0]!.next("hold"); + expect(held.chain).toBeUndefined(); + relay.tab.hold(Far, { session: "S1" }, context); + expect((await workers[0]!.next("hold")).chain).toBeUndefined(); + }); + + it("never match a restarted object: its new instance has a new id (race 8)", async () => { + // The raw worker holds `Near` on the server itself (as the worker), whose activation holds `Far` back with its + // chain. The raw worker never binds `Far`. + const { worker } = await nearBehindRawWorker(); + const chainOf = async () => (await worker.next("hold")).chain!; + worker.send(nearHold(0)); + const [old] = await chainOf(); + // Released: that instance ends. The next hold makes a new instance with a new id. + worker.send({ kind: "release", hold: 0 }); + worker.send(nearHold(1)); + const [renewed] = await chainOf(); + expect(renewed).not.toBe(old); + // A hold naming the old instance binds to the new one (it is pending there); one naming the new one is a cycle. + worker.send({ ...nearHold(2), chain: [old!] }); + worker.send({ ...nearHold(3), chain: [renewed!] }); + const failed = await worker.next("failed"); + expect(failed).toMatchObject({ + hold: 3, + cause: { kind: "broken", error: { kind: "chord", error: { reason: "cycle" } } }, + }); + await transported(); + expect(worker.received.filter((message) => message.kind === "failed")).toEqual([]); + }); + + it("keep the last 32 entries of a longer chain: the oldest is dropped (a longer cycle goes undetected)", async () => { + const { worker } = await nearBehindRawWorker(); + const forged = Array.from( + { length: 32 }, + (_, index) => `0190f5a2-7c3e-7d4a-9b1c-${String(index).padStart(12, "0")}`, + ); + worker.send({ ...nearHold(0), chain: forged }); + const { chain } = await worker.next("hold"); + expect(chain).toHaveLength(32); + expect(chain!.slice(0, 31)).toEqual(forged.slice(1)); + expect(forged).not.toContain(chain![31]); + }); + + it("miss a cycle entering an activation that started already (documented limitation): both wait, transient", async () => { + const { relay, worker } = await nearBehindRawWorker(); + // The server's `Near` activates for the tab and holds `Far` with its chain. + const getting = rejection(relay.tab.get(Near, { session: "S1" }, context, { readyTimeout: 100 })); + const { chain } = await worker.next("hold"); + // The worker's own activation, started at the same moment, holds that `Near` back with a chain of its own. + worker.send({ ...nearHold(0), chain: ["0190f5a2-7c3e-7d4a-9b1c-000000000001"] }); + await transported(); + expect(chain).toHaveLength(1); + expect(worker.received.filter((message) => message.kind === "failed")).toEqual([]); + expect(await getting).toMatchObject({ code: "unavailable" }); + }); + + it("keep no object, and so no chain, once released (GC)", async () => { + const refs = await (async () => { + const made = attempted.length; + const { tab, server } = await cyclic(); + await rejection(tab.get(Near, { session: "S1" }, context)); + await transported(); + const refs = attempted.slice(made); + expect(refs.length).toBeGreaterThanOrEqual(4); + await tab.close(context); + await server.close(context); + return refs; + })(); + await expectCollected(...refs); + }); +}); + +/** + * A server serving `Near` (its activation holds `Far`) and a raw worker for S1 over a trusted route link, made by a + * first hold of `Far` and with that hold taken. + */ +async function nearBehindRawWorker() { + const workers: RawPeer[] = []; + const relay = await relayed({ + resolve: rawWorkers(workers, { trusted: true }), + server: (h) => + h.serve(Near, ({ session }, _principal, self) => ({ + variant: "v", + members: { n: self.get(Far, { session }).n }, + })), + }); + relay.tab.hold(Far, { session: "S1" }, context); + const worker = await (async () => { + for (let tries = 0; workers[0] === undefined && tries < 10; tries++) await transported(); + return workers[0]!; + })(); + await worker.next("hold"); + return { relay, worker }; +} + +/** A hold of `Far` for S1, as a tab sends it. */ +function farHold(hold: number): Extract { + const { name: type, version, fingerprint } = Far; + const address = { kind: "session", params: { id: "S1" } }; + return { kind: "hold", hold, type, version, fingerprint, params: jsonPart({ session: "S1" }), address }; +} + +/** A hold of `Near` for S1, as a worker sends it to the server. */ +function nearHold(hold: number): Extract { + const { name: type, version, fingerprint } = Near; + return { kind: "hold", hold, type, version, fingerprint, params: jsonPart({ session: "S1" }) }; +} + +describe("a live object failing (slice 6a §1.4)", () => { + it("fails broken: calls reject with the cause, one object_failed report, natives close, then references go", async () => { + const events: string[] = []; + let list: Watchable | undefined; + const { reference, fail, errors } = await restartable( + attempted, + (_, self) => { + list = self.state(List); + const core = self.get(Core, {}); + self.open( + async () => null, + () => { + events.push(`close sees core ${core.n.value}`); + }, + ); + }, + { serve: (h) => h.serve(Core, () => ({ n: constant(1) })) }, + ); + const states: string[] = []; + (await reference.state.watch(context)).start(async (call) => { + if (call.kind === "value") states.push(call.value.kind); + }); + fail(); + const touched = rejection(reference.touch(context)); + expect(await touched).toMatchObject({ code: "unavailable", message: "boom" }); + expect(errors).toEqual([expect.objectContaining({ kind: "object_failed", object: "app.restartable(1)" })]); + // A failure of the sealed attempt is ignored. + fail(); + await flushed(); + expect(events).toEqual(["close sees core 1"]); + expect(errors.length).toBe(1); + expect(list!.value).toEqual({ items: [] }); + await delivered(); + expect(states).toEqual(["failed"]); + }); + + it("keeps its state members' last values for same-host reads; a live dependent's derive over them stays live", async () => { + const Viewer = local("app.count-viewer", { version: 1, params: {}, members: { tens: int() } }); + let store: Self | undefined; + const { host, errors } = await loaded((h) => { + h.serve(Store, (_, self) => { + store = self; + return serveStore(self); + }); + h.serve(Viewer, (_, self) => ({ tens: self.derive([self.get(Store, {}).count], ([n]) => n * 10) })); + }); + const ref = await host.get(Store, {}, context); + await ref.add({ text: "a" }, context); + const viewer = await host.get(Viewer, {}, context); + const tens = changesOf(await viewer.tens.watch(context)); + // A change committed in the turn of the failure: the frozen member computes it. + await ref.add({ text: "b" }, context); + failLive(attempted, store!, new ChordError({ code: "unavailable", message: "boom" })); + expect(ref.count.value).toBe(2); + expect(await rejection(ref.add({ text: "c" }, context))).toMatchObject({ message: "boom" }); + expect(viewer.state.value).toEqual({ kind: "live" }); + expect(viewer.tens.value).toBe(20); + await delivered(); + expect(valuesOf(tens)).toEqual([20]); + expect(errors.map(({ kind }) => kind)).toEqual(["object_failed"]); + // A retry takes the new attempt's members. + ref.retry(); + (await host.get(Store, {}, context)).release(); + expect(ref.count.value).toBe(2); + await host.close(context); + }); + + it("lets a commit admitted before the failure adopt and publish once, derives and dependents included", async () => { + const Counter = defineState("counter", { ...COUNTER, persist: true, checkpointWhen: () => false }); + const Doubled = local("app.doubled", { version: 1, params: {}, members: { double: int() } }); + const Viewer = local("app.double-viewer", { version: 1, params: {}, members: { tens: int() } }); + const store = await memoryStore(); + let self: Self | undefined; + let counter: SingletonHandle<{ n: number }> | undefined; + const { host } = await loaded( + (h) => { + h.serve(Doubled, (_, s) => { + self = s; + counter = s.state(Counter); + return { double: s.derive([counter], ([c]) => c.n * 2) }; + }); + h.serve(Viewer, (_, s) => ({ tens: s.derive([s.get(Doubled, {}).double], ([n]) => n * 10) })); + }, + { store }, + ); + const ref = await host.get(Doubled, {}, context); + const viewer = await host.get(Viewer, {}, context); + const doubles = changesOf(await ref.double.watch(context)); + const tens = changesOf(await viewer.tens.watch(context)); + const held = holdNextCommit(store); + const change = counter!.change((draft) => { + draft.n = 1; + }, context); + await held.entered; + failLive(attempted, self!, new ChordError({ code: "unavailable", message: "boom" })); + held.release(); + await change; + await delivered(); + expect(ref.state.value.kind).toBe("failed"); + expect([counter!.value, ref.double.value, viewer.tens.value]).toEqual([{ n: 1 }, 2, 20]); + expect(valuesOf(doubles)).toEqual([2]); + expect(valuesOf(tens)).toEqual([20]); + await host.close(context); + await store.close(context); + }); + + it("starts a retry's factory only once a commit admitted before the failure adopted", async () => { + const V1 = defineState("counter", { ...COUNTER, persist: true, checkpointWhen: () => false }); + const V2 = defineState("counter", { + schema: obj({ m: int() }), + version: 2, + initial: () => ({ m: 0 }), + migrate: (old) => ({ m: (old as { n: number }).n }), + persist: true, + checkpointWhen: () => false, + }); + const store = await memoryStore(); + const counters: SingletonHandle<{ n: number } | { m: number }>[] = []; + const { host, reference, attempts } = await storeRestartable( + attempted, + (attempt, self) => void counters.push(self.state(attempt === 1 ? V1 : V2)), + { store, closeTimeout: 5 }, + ); + const v1 = counters[0] as SingletonHandle<{ n: number }>; + await v1.change((draft) => { + draft.n = 1; + }, context); + const held = holdNextCommit(store); + const change = v1.change((draft) => { + draft.n = 2; + }, context); + await held.entered; + failLive(attempted, attempts[0]!, new ChordError({ code: "unavailable", message: "boom" })); + // Past the drain: only the admitted step holds the retry. + await new Promise((resolve) => setTimeout(resolve, 20)); + reference.retry(); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(counters).toHaveLength(1); + held.release(); + await change; + (await host.get(StoreRestartable, {}, context)).release(); + expect(counters[1]!.value).toEqual({ m: 2 }); + await host.close(context); + await store.close(context); + }); + + it("is recovered by ref.retry(): a new attempt, the documents as they were", async () => { + const lists: SingletonHandle<{ items: string[] }>[] = []; + const { restart } = await restartable(attempted, (_, self) => { + lists.push(self.state(List)); + }); + await lists[0]!.change((draft) => void draft.items.push("a"), context); + await restart(); + expect(lists[1]!.value).toEqual({ items: ["a"] }); + }); +}); diff --git a/packages/chord/test/next/host/parked.test.ts b/packages/chord/test/next/host/parked.test.ts new file mode 100644 index 000000000..207da3019 --- /dev/null +++ b/packages/chord/test/next/host/parked.test.ts @@ -0,0 +1,213 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { int, local, method, str } from "../../../src/next/contracts/index.ts"; +import { createHost, type Held } from "../../../src/next/host/index.ts"; +import { + EDITOR, + fakeTimers, + flushed, + forEachCodec, + loaded, + Notes, + partValue, + rawParent, + rawProvider, + rejection, + type TestPrincipal, + transported, + wireNames, +} from "../support.ts"; + +const Gated = local("app.gated", { version: 1, params: { id: int() }, members: { log: method(str(), int()) } }); + +/** + * Serves `Gated`, whose open waits for `gate`; `log` records its argument and returns how many it recorded. In its + * synchronous part a `log` of "release" releases `self.ref`, one of "call" makes the call `log("from a handler")` on + * it, and one of "block" takes 30 ms (the clock moves while no timer can fire). + */ +async function gated() { + const gate = Promise.withResolvers(); + const log: string[] = []; + const self: { ref?: Held } = {}; + const { host } = await loaded((h) => + h.serve(Gated, (_, attempt) => { + attempt.open( + () => gate.promise, + () => {}, + ); + return { + log: async (text) => { + log.push(text); + if (text === "release") self.ref?.release(); + if (text === "call") void self.ref?.log("from a handler", context); + if (text === "block") vi.setSystemTime(Date.now() + 30); + return log.length; + }, + }; + }), + ); + return { host, gate, log, self }; +} + +describe("calls on a loading local reference", () => { + fakeTimers(); + + it("are queued and sent in the order they were made once it is live, before later calls", async () => { + const { host, gate, log } = await gated(); + const ref = host.hold(Gated, { id: 1 }, context); + const first = ref.log("a", context); + const second = ref.log("b", context); + await flushed(); + expect(log).toEqual([]); + gate.resolve(); + await flushed(); + const third = ref.log("c", context); + expect(await Promise.all([first, second, third])).toEqual([1, 2, 3]); + expect(log).toEqual(["a", "b", "c"]); + }); + + it("keep their order when a handler calls the reference while they are sent: its call queues behind them", async () => { + const { host, gate, log, self } = await gated(); + const ref = host.hold(Gated, { id: 1 }, context); + self.ref = ref; + const calls = [ref.log("call", context), ref.log("b", context)]; + gate.resolve(); + await Promise.all(calls); + await flushed(); + expect(log).toEqual(["call", "b", "from a handler"]); + }); + + it("are checked against their deadline when sent: one that passed while earlier ones ran is not sent", async () => { + const { host, gate, log } = await gated(); + const ref = host.hold(Gated, { id: 1 }, context, { readyTimeout: 10 }); + const first = ref.log("block", context); + const second = rejection(ref.log("b", context)); + gate.resolve(); + expect(await first).toBe(1); + expect(await second).toMatchObject({ code: "unavailable", reason: "park_timeout" }); + expect(log).toEqual(["block"]); + }); + + it("reject unavailable/park_timeout at the reference's readyTimeout, measured from each call", async () => { + const { host, gate, log } = await gated(); + const ref = host.hold(Gated, { id: 1 }, context, { readyTimeout: 100 }); + const first = rejection(ref.log("a", context)); + await vi.advanceTimersByTimeAsync(60); + const second = ref.log("b", context); + await vi.advanceTimersByTimeAsync(40); + expect(await first).toEqual({ + code: "unavailable", + reason: "park_timeout", + message: "app.gated(1): a call waited 100 ms for its reference to go live", + }); + gate.resolve(); + expect(await second).toBe(1); + expect(log).toEqual(["b"]); + }); + + it("reject at once when their caller cancels; the others stay queued", async () => { + const { host, gate, log } = await gated(); + const ref = host.hold(Gated, { id: 1 }, context); + const { context: cancellable, cancel } = withCancel(context); + const cancelled = ref.log("a", cancellable); + const kept = ref.log("b", context); + cancel(new Error("user cancelled")); + await expect(cancelled).rejects.toThrow("user cancelled"); + gate.resolve(); + expect(await kept).toBe(1); + expect(log).toEqual(["b"]); + }); + + it("reject released when the reference is released; nothing runs", async () => { + const { host, gate, log } = await gated(); + const ref = host.hold(Gated, { id: 1 }, context); + const call = ref.log("a", context); + ref.release(); + expect(await rejection(call)).toEqual({ code: "released", message: "app.gated: released" }); + gate.resolve(); + await flushed(); + expect(log).toEqual([]); + }); + + it("reject with the cause when the object fails", async () => { + const { host, gate } = await gated(); + const ref = host.hold(Gated, { id: 1 }, context); + const call = ref.log("a", context); + gate.reject(new Error("offline")); + expect(await rejection(call)).toEqual({ code: "unavailable", message: "app.gated(1): offline" }); + }); + + it("stop being sent when one's handler releases the reference: the rest reject released (race 15)", async () => { + const { host, gate, log, self } = await gated(); + const ref = host.hold(Gated, { id: 1 }, context); + self.ref = ref; + const first = ref.log("release", context); + const second = ref.log("b", context); + gate.resolve(); + expect(await rejection(second)).toEqual({ code: "released", message: "app.gated: released" }); + await first.catch(() => {}); + expect(log).toEqual(["release"]); + }); +}); + +forEachCodec(() => { + describe("calls on a loading remote reference", () => { + fakeTimers(); + + it("keep one deadline across reconnects; nothing is sent for them on either link", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context, { readyTimeout: 1_000 }); + const call = rejection(notes.wait(context)); + await vi.advanceTimersByTimeAsync(600); + peer.close(); + await transported(); + const next = await rawParent(client); + await next.next("hold"); + await vi.advanceTimersByTimeAsync(400); + expect(await call).toMatchObject({ code: "unavailable", reason: "park_timeout" }); + expect([peer.kinds(), next.kinds()]).toEqual([["hold"], []]); + }); + + it("are resolved against the binding they are sent on: indexes from the new variant, missing methods invalid", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + // Typed for the editor: before its first binding the reference may become either variant. + const editor = notes as Extract, { add: unknown }>; + const add = rejection(editor.add({ text: "a" }, context)); + const wait = editor.wait(context); + const { hold } = await peer.next("hold"); + const READER = wireNames(Notes, "reader"); + const members = [ + { member: READER.state.notes!, value: "[]" }, + { member: READER.state.count!, value: "0" }, + ]; + peer.send({ kind: "bound", holds: [{ hold, variant: READER.variant, members, failed: [] }] }); + expect(await add).toEqual({ + code: "invalid", + reason: "member", + message: 'app.notes(S1) (variant reader) has no method "add"', + }); + const sent = await peer.next("call"); + expect(sent.method).toBe(READER.methods.wait); + expect(sent.method).not.toBe(EDITOR.methods.wait); + peer.send({ kind: "result", call: sent.call }); + expect(await wait).toBe(null); + }); + + it("on a reference released while loading without a link: nothing is sent, the next connect holds nothing (race 3)", async () => { + const client = createHost({ onError: () => {} }); + await client.load([], context); + const notes = client.hold(Notes, { session: "S1" }, context); + const other = client.hold(Notes, { session: "S2" }, context); + const call = notes.wait(context); + notes.release(); + expect(await rejection(call)).toEqual({ code: "released", message: "app.notes: released" }); + const peer = await rawParent(client); + const { params } = await peer.next("hold"); + expect(partValue(params)).toEqual({ session: "S2" }); + await transported(); + expect(peer.kinds()).toEqual([]); + expect(other.state.value).toEqual({ kind: "loading" }); + }); + }); +}); diff --git a/packages/chord/test/next/host/presence.test.ts b/packages/chord/test/next/host/presence.test.ts new file mode 100644 index 000000000..6260f9360 --- /dev/null +++ b/packages/chord/test/next/host/presence.test.ts @@ -0,0 +1,328 @@ +// Tests of presence (§19): \`host.references(Type)\`, its entries and incarnations, its batches (a link drop as a departure, +// one discovered while a commit sends lands in the next batch), derives over it, its cost and GC. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, remote, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; +import { jsonPart } from "../../../src/next/link/internal.ts"; +import { Hold } from "../../../src/next/remote/hold.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { + addNote, + delivered, + expectCollected, + framesOf, + incarnationOf, + instancesOf, + linked, + NoteStore, + Notes, + PlacedNotes, + RawPeer, + relayed, + serveNotes, + type TestPrincipal, + testCodec, + transported, + wireNames, +} from "../support.ts"; + +/** Every hold started in this file, weakly. */ +const holds = instancesOf(Hold, "start"); + +describe("host.references", () => { + it("lists every reference with the object's params, its principal and its incarnation", async () => { + const { server, client } = await linked((h) => void serveNotes(h), { child: { kind: "guest", session: "S1" } }); + const notes = server.references(Notes); + expect(notes.value).toEqual([]); + await client.get(Notes, { session: "S1" }, context); + const own = await server.get(Notes, { session: "S2" }, context); + expect(notes.value).toEqual([ + { params: { session: "S1" }, principal: { kind: "guest", session: "S1" }, incarnation: incarnationOf(client) }, + { params: { session: "S2" }, principal: { kind: "server" }, incarnation: incarnationOf(server) }, + ]); + // A local type's entries have no principal: the store is referenced by both views. + expect(server.references(NoteStore).value).toEqual([ + { params: { session: "S1" }, incarnation: incarnationOf(server) }, + { params: { session: "S2" }, incarnation: incarnationOf(server) }, + ]); + own.release(); + expect(notes.value).toHaveLength(1); + }); + + it("is made for the first ask with the references that exist then, and kept", async () => { + const { server, client } = await linked((h) => void serveNotes(h)); + await client.get(Notes, { session: "S1" }, context); + const notes = server.references(Notes); + expect(notes.value).toHaveLength(1); + expect(server.references(Notes)).toBe(notes); + }); + + it("changes in the batch where a reference binds or ends: a watch gets one splice per change", async () => { + const { server, client } = await linked((h) => void serveNotes(h)); + const watch = await server.references(Notes).watch(context); + const frames = framesOf(watch); + const one = await client.get(Notes, { session: "S1" }, context); + await client.get(Notes, { session: "S2" }, context); + one.release(); + await transported(); + await delivered(); + const incarnation = incarnationOf(client); + const entry = (session: string) => ({ params: { session }, principal: { kind: "owner" }, incarnation }); + expect(frames.map(([, ops]) => ops)).toEqual([ + [["p", [], 0, 0, [entry("S1")]]], + [["p", [], 1, 0, [entry("S2")]]], + [["p", [], 0, 1, []]], + ]); + expect(frames.at(-1)![0]).toEqual([entry("S2")]); + }); + + it("feeds derives in factories: viewers per session, a link drop a departure in its batch (race 9)", async () => { + const Viewers = local("app.viewers", { version: 1, params: { session: str() }, members: { count: int() } }); + const Shown = remote("app.shown", { + version: 1, + params: { session: str() }, + variants: { v: { state: { here: int() } } }, + }); + const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + await server.load( + [ + { + name: "@test/app", + facet: defineFacet((host) => { + host.serve(Shown, () => ({ variant: "v", members: { here: constant(1) } })); + host.serve(Viewers, ({ session }, self) => ({ + count: self.derive( + [host.references(Shown)], + ([refs]) => refs.filter((ref) => ref.params.session === session).length, + ), + })); + }), + }, + ], + context, + ); + const viewers = await server.get(Viewers, { session: "S1" }, context); + const counts = framesOf(await viewers.count.watch(context)); + const tabs = await Promise.all( + [0, 1].map(async () => { + const tab = createHost({ onError: () => {} }); + await tab.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" } }, context); + tab.connect(b, { principal: { kind: "server" } }, context); + return { tab, link: a }; + }), + ); + await Promise.all(tabs.map(({ tab }) => tab.get(Shown, { session: "S1" }, context))); + await tabs[0]!.tab.get(Shown, { session: "S2" }, context); + expect(viewers.count.value).toBe(2); + tabs[1]!.link.close(); + // The departure is in the batch of the link's close: the derive has it at once. + expect(viewers.count.value).toBe(1); + await delivered(); + expect(counts.at(-1)![0]).toBe(1); + }); + + it("puts a departure discovered while a commit sends into the next batch: one state message per commit (race 9)", async () => { + const Viewed = remote("app.viewed", { + version: 1, + params: { session: str() }, + variants: { v: { state: { count: int(), viewers: int() } } }, + }); + const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const facet = defineFacet((host) => { + serveNotes(host); + host.serve(Viewed, ({ session }, _principal, self) => { + const store = self.get(NoteStore, { session }); + const count = self.derive([store.list], ([list]) => list.length); + const viewers = self.derive([host.references(Viewed)], ([refs]) => refs.length); + return { variant: "v", members: { count, viewers } }; + }); + }); + await server.load([{ name: "@test/app", facet }], context); + const names = wireNames(Viewed, "v").state as { count: number; viewers: number }; + /** A raw tab; its server end closes while it sends, once `drop.armed` (a transport failing on send). */ + const tab = (drop?: { armed: boolean }) => { + const [a, b] = memoryLinkPair(); + const end = drop === undefined ? a : closingOnSend(a, drop); + server.accept(end, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const { name: type, version, fingerprint } = Viewed; + peer.send({ kind: "hold", hold: 0, type, version, fingerprint, params: jsonPart({ session: "S1" }) }); + return peer; + }; + const drop = { armed: false }; + // The dropped tab binds first, so its link sends (and fails) before the kept one's in the commit's flushes. + const dropped = tab(drop); + await dropped.next("bound"); + const kept = tab(); + await kept.next("bound"); + await transported(); + kept.received.length = 0; + drop.armed = true; + await addNote(server, "S1", "x"); + await transported(); + const states = kept.received.filter((message) => message.kind === "state"); + const changed = states.map(({ holds }) => holds.flatMap(({ members }) => members.map(({ member }) => member))); + // The commit's message carries the commit (the count); the departure is the next batch's (the viewers). + expect(changed).toEqual([[names.count], [names.viewers]]); + expect(dropped.closed).toBe(true); + }); + + it("completes a hold's bookkeeping before its presence entry publishes: a link closing then leaks nothing", async () => { + const Viewed = remote("app.viewed", { + version: 1, + params: {}, + variants: { v: { state: { viewers: int() } } }, + }); + const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const facet = defineFacet((host) => { + host.serve( + Viewed, + (_params, _principal, self) => ({ + variant: "v", + members: { viewers: self.derive([host.references(Viewed)], ([refs]) => refs.length) }, + }), + { linger: 60_000 }, + ); + }); + await server.load([{ name: "@test/app", facet }], context); + const [a, b] = memoryLinkPair(); + const drop = { armed: false }; + server.accept(closingOnSend(a, drop), { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const { name: type, version, fingerprint } = Viewed; + peer.send({ kind: "hold", hold: 0, type, version, fingerprint, params: "{}" }); + await peer.next("bound"); + peer.send({ kind: "ack" }); + await transported(); + // The second hold's presence entry changes the first's state: sending it closes the link. + drop.armed = true; + peer.send({ kind: "hold", hold: 1, type, version, fingerprint, params: "{}" }); + await transported(); + expect(peer.closed).toBe(true); + expect(server.references(Viewed).value).toEqual([]); + expect(server.activity.value).toMatchObject({ references: 0, lingering: 1 }); + await server.close(context); + }); + + it("completes a release's bookkeeping before its departure publishes: a link closing then leaves no lingering", async () => { + const Viewed = remote("app.viewed", { version: 1, params: {}, variants: { v: { state: { viewers: int() } } } }); + const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const facet = defineFacet((host) => { + host.serve( + Viewed, + (_params, _principal, self) => ({ + variant: "v", + members: { viewers: self.derive([host.references(Viewed)], ([refs]) => refs.length) }, + }), + { linger: 50 }, + ); + }); + await server.load([{ name: "@test/app", facet }], context); + const [a, b] = memoryLinkPair(); + const drop = { armed: false }; + server.accept(closingOnSend(a, drop), { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const { name: type, version, fingerprint } = Viewed; + for (const hold of [0, 1]) { + peer.send({ kind: "hold", hold, type, version, fingerprint, params: "{}" }); + await peer.next("bound"); + peer.send({ kind: "ack" }); + } + await transported(); + // The first release changes the second hold's state: sending it closes the link, which releases the second. + drop.armed = true; + peer.send({ kind: "release", hold: 0 }); + await transported(); + expect(peer.closed).toBe(true); + expect(server.activity.value).toMatchObject({ references: 0, lingering: 1 }); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(server.references(Viewed).value).toEqual([]); + expect(server.activity.value).toMatchObject({ references: 0, lingering: 0 }); + await server.close(context); + }); + + it("removes a bulk departure with one list and one splice op per entry, in one frame (a revoke of many holds)", async () => { + const server = createHost({ + principal: { kind: "server" }, + revokedBy: (principal) => (principal.kind === "guest" ? [{ session: principal.session }] : []), + onError: () => {}, + }); + await server.load([{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "guest", session: "S1" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const { name: type, version, fingerprint } = Notes; + for (let hold = 0; hold < 4; hold++) { + peer.send({ kind: "hold", hold, type, version, fingerprint, params: jsonPart({ session: "S1" }) }); + } + await transported(); + const references = server.references(Notes); + const frames = framesOf(await references.watch(context)); + expect(references.value).toHaveLength(4); + server.revoke({ session: "S1" }, context); + await delivered(); + expect(frames.map(([, ops]) => ops)).toEqual([Array(4).fill(["p", [], 0, 1, []])]); + expect(frames[0]![0]).toEqual([]); + }); + + it("counts two tabs through a relay as two incarnations on the worker", async () => { + const relay = await relayed(); + const second = createHost({ onError: () => {} }); + await second.load([], context); + const [a, b] = memoryLinkPair(); + relay.server.accept(a, { principal: { kind: "owner" } }, context); + second.connect(b, { principal: { kind: "server" } }, context); + await relay.tab.get(PlacedNotes, { session: "S1" }, context); + await second.get(PlacedNotes, { session: "S1" }, context); + const worker = relay.workers.get("session|S1")!; + const incarnations = worker.references(PlacedNotes).value.map(({ incarnation }) => incarnation); + expect(incarnations).toEqual([incarnationOf(relay.tab), incarnationOf(second)]); + }); + + it("has no entries for lingering objects", async () => { + const Kept = remote("app.kept", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const { server, client } = await linked((h) => + h.serve(Kept, () => ({ variant: "v", members: { n: constant(1) } }), { linger: 60_000 }), + ); + const kept = await client.get(Kept, {}, context); + const references = server.references(Kept); + expect(references.value).toHaveLength(1); + kept.release(); + await transported(); + expect(references.value).toEqual([]); + expect(server.activity.value.lingering).toBe(1); + await server.close(context); + }); + + it("keeps no released reference (GC)", async () => { + const { server, refs } = await (async () => { + const { server, client } = await linked((h) => void serveNotes(h)); + const references = server.references(Notes); + const held = holds.length; + const notes = await client.get(Notes, { session: "S1" }, context); + expect(references.value).toHaveLength(1); + const refs = holds.slice(held); + notes.release(); + await transported(); + expect(references.value).toEqual([]); + return { server, refs }; + })(); + await expectCollected(...refs); + expect(server.references(Notes).value).toEqual([]); + }); +}); + +/** `link`, closing itself on a send once `drop.armed`, as a transport whose send fails. */ +function closingOnSend(link: Link, drop: { readonly armed: boolean }): Link { + return { + send: (message) => (drop.armed ? link.close() : link.send(message)), + onMessage: (listener) => link.onMessage(listener), + close: () => link.close(), + onClose: (listener) => link.onClose(listener), + }; +} diff --git a/packages/chord/test/next/host/ref-state.test.ts b/packages/chord/test/next/host/ref-state.test.ts new file mode 100644 index 000000000..d7ab08c47 --- /dev/null +++ b/packages/chord/test/next/host/ref-state.test.ts @@ -0,0 +1,189 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { int, local, method } from "../../../src/next/contracts/index.ts"; +import { broken } from "../../../src/next/data/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { + bindHold, + bindNext, + fakeTimers, + flushed, + forEachCodec, + loaded, + Notes, + rawParent, + rawProvider, + rejection, + statesOf, + transported, +} from "../support.ts"; + +const Flaky = local("app.flaky", { + version: 1, + params: { id: int() }, + members: { n: int(), get: method(undefined, int()) }, +}); + +/** Serves `Flaky`, whose open throws while `failing.left` is above 0 (broken while `failing.broken`). */ +async function flaky(failing: { left: number; broken: boolean }) { + return loaded((h) => + h.serve(Flaky, ({ id }, self) => { + self.open( + async () => { + if (failing.left-- <= 0) return; + if (failing.broken) throw broken(new Error("no configuration")); + throw new Error("offline"); + }, + () => {}, + ); + return { n: constant(id), get: async () => id }; + }), + ); +} + +const offline = { code: "unavailable", message: "app.flaky(1): offline" }; + +describe("ref.state of a local reference", () => { + fakeTimers(); + + it("is watched from its current state: failed with retryAt, loading on the next attempt, live; release ends it", async () => { + const { host } = await flaky({ left: 1, broken: false }); + const ref = host.hold(Flaky, { id: 1 }, context); + expect(ref.state.value).toEqual({ kind: "loading" }); + const watch = await ref.state.watch(context); + expect(watch.value).toEqual({ kind: "loading" }); + const states = statesOf(watch); + await flushed(); + expect(ref.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "transient", error: offline }, + retryAt: 1_000, + }); + await vi.advanceTimersByTimeAsync(1_000); + await flushed(); + expect(ref.state.value).toEqual({ kind: "live" }); + ref.release(); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(states).toEqual(["failed: app.flaky(1): offline", "loading", "live"]); + expect(ref.state.value).toMatchObject({ kind: "ended", error: { code: "released" } }); + }); + + it("reads and calls follow the state: not_ready before live, the cause's error while failed, released after", async () => { + const { host } = await flaky({ left: 1, broken: true }); + const ref = host.hold(Flaky, { id: 1 }, context); + expect(() => ref.n.value).toThrow("app.flaky(1): not live yet"); + await flushed(); + const cause = { code: "unavailable", message: "app.flaky(1): no configuration" }; + expect(ref.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "broken", error: cause }, + retryAt: undefined, + }); + expect(() => ref.n.value).toThrow("no configuration"); + expect(await rejection(ref.get(context))).toEqual(cause); + ref.release(); + expect(() => ref.n.value).toThrow("app.flaky: released"); + expect(await rejection(ref.get(context))).toEqual({ code: "released", message: "app.flaky: released" }); + }); + + it("retry renews a broken object: loading, then live", async () => { + const { host } = await flaky({ left: 1, broken: true }); + const ref = host.hold(Flaky, { id: 1 }, context); + await flushed(); + expect(ref.state.value.kind).toBe("failed"); + ref.retry(); + expect(ref.state.value).toEqual({ kind: "loading" }); + await flushed(); + expect(ref.state.value).toEqual({ kind: "live" }); + expect(await ref.get(context)).toBe(1); + }); + + it("is made on first access, once; a watch taken after release has ended", async () => { + const { host } = await flaky({ left: 0, broken: false }); + const ref = await host.get(Flaky, { id: 1 }, context); + expect(ref.state).toBe(ref.state); + expect(ref.state.value).toEqual({ kind: "live" }); + ref.release(); + const late = await ref.state.watch(context); + expect(late.value).toMatchObject({ kind: "ended", error: { code: "released" } }); + expect(await late.closed).toEqual({ kind: "ended" }); + }); + + it("a watch ends cancelled with its context", async () => { + const { host } = await flaky({ left: 0, broken: false }); + const ref = host.hold(Flaky, { id: 1 }, context); + const { context: cancellable, cancel } = withCancel(context); + const watch = await ref.state.watch(cancellable); + cancel(); + expect(await watch.closed).toEqual({ kind: "aborted" }); + }); +}); + +forEachCodec(() => { + describe("ref.state of a remote reference", () => { + fakeTimers(); + + it("loading until bound, loading while its link is down, failed with retryAt from retryIn, live again", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + const states = statesOf(await notes.state.watch(context)); + await bindNext(peer); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + peer.close(); + await transported(); + expect(notes.state.value).toEqual({ kind: "loading" }); + const next = await rawParent(client); + const { hold } = await next.next("hold"); + const error = { code: "unavailable", message: "offline" } as const; + next.send({ + kind: "failed", + hold, + cause: { kind: "transient", error: { kind: "chord", error } }, + retryIn: 500, + }); + await transported(); + expect(notes.state.value).toMatchObject({ kind: "failed", cause: { kind: "transient", error }, retryAt: 500 }); + next.send({ kind: "loading", hold }); + await transported(); + expect(notes.state.value).toEqual({ kind: "loading" }); + bindHold(next, hold); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + expect(states).toEqual(["live", "loading", "failed: offline", "loading", "live"]); + }); + + it("a failed with retryIn 0 retries now: retryAt is the current time", async () => { + const { client, peer } = await rawProvider(); + vi.setSystemTime(2_000); + const notes = client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + const error = { code: "unavailable", message: "offline" } as const; + peer.send({ kind: "failed", hold, cause: { kind: "transient", error: { kind: "chord", error } }, retryIn: 0 }); + await transported(); + expect(notes.state.value).toMatchObject({ kind: "failed", retryAt: 2_000 }); + }); + + it("variant: undefined before the first binding, the last bound one while loading or failed", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + expect(notes.variant).toBeUndefined(); + await bindNext(peer, [{ text: "a" }]); + await transported(); + expect(notes.variant).toBe("editor"); + peer.close(); + await transported(); + expect([notes.variant, notes.count.value]).toEqual(["editor", 1]); + const next = await rawParent(client); + const { hold } = await next.next("hold"); + const error = { code: "unavailable", message: "offline" } as const; + next.send({ kind: "failed", hold, cause: { kind: "broken", error: { kind: "chord", error } } }); + await transported(); + expect(notes.state.value).toMatchObject({ kind: "failed", retryAt: undefined }); + expect(notes.variant).toBe("editor"); + expect(() => notes.count.value).toThrow("offline"); + notes.release(); + expect(() => notes.variant).toThrow("app.notes: released"); + }); + }); +}); diff --git a/packages/chord/test/next/host/reference.test.ts b/packages/chord/test/next/host/reference.test.ts new file mode 100644 index 000000000..adeeb7288 --- /dev/null +++ b/packages/chord/test/next/host/reference.test.ts @@ -0,0 +1,656 @@ +import { describe, expect, expectTypeOf, it, vi } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context, withAbortSignal } from "../../../src/context/index.ts"; +import { int, keyed, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { broken, type ChordError } from "../../../src/next/data/index.ts"; +import { defineState, type KeyedHandle } from "../../../src/next/documents/index.ts"; +import { + createHost, + defineFacet, + type Held, + type Reference, + type RefIn, + type RefState, + type Self, +} from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { RemoteObject } from "../../../src/next/remote/remote-object.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import type { Store } from "../../../src/next/store/index.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import type { NativeWatchable, Watchable } from "../../../src/watch.ts"; +import { + changesOf, + delivered, + expectCollected, + flushed, + framesOf, + instancesOf, + loaded, + memoryStore, + Notes, + rejection, + settledHeap, +} from "../support.ts"; + +/** Every remote object bound while this file runs, weakly. */ +const remoteObjects = instancesOf(RemoteObject, "hold"); +const chordObjects = instancesOf(ChordObject, "attempt"); + +const Doc = defineState("doc", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); +const Counter = local("app.counter", { version: 1, params: {}, members: { doc: obj({ n: int() }), bump: method() } }); +const Viewer = local("app.viewer", { version: 1, params: { id: int() }, members: { n: int() } }); + +/** A counter: one document, exposed, and a method that increments it. */ +function serveCounter(self: Self) { + const doc = self.state(Doc); + return { + doc, + bump: async (ctx: typeof context) => { + await doc.change((draft) => { + draft.n++; + }, ctx); + return null; + }, + }; +} + +describe("references", () => { + it("are hydrated when got; their state members are watchables", async () => { + const { host } = await loaded((h) => h.serve(Counter, (_, self) => serveCounter(self))); + const counter = await host.get(Counter, {}, context); + expect(counter.doc.value).toEqual({ n: 0 }); + const frames = framesOf(await counter.doc.watch(context)); + await counter.bump(context); + await delivered(); + expect(frames).toEqual([[{ n: 1 }, [["s", ["n"], 1]]]]); + expect(counter.doc.value).toEqual({ n: 1 }); + }); + + it("throw released on every use after release, which is idempotent", async () => { + const { host } = await loaded((h) => h.serve(Counter, (_, self) => serveCounter(self))); + const counter = await host.get(Counter, {}, context); + counter.release(); + counter.release(); + const released = { code: "released", message: "app.counter: released" }; + expect(() => counter.doc.value).toThrow("app.counter: released"); + expect(await rejection(counter.doc.watch(context))).toEqual(released); + expect(await rejection(counter.bump(context))).toEqual(released); + }); + + it("end their watches when released, while other references keep the object", async () => { + const { host } = await loaded((h) => h.serve(Counter, (_, self) => serveCounter(self))); + const first = await host.get(Counter, {}, context); + const second = await host.get(Counter, {}, context); + const one = await first.doc.watch(context); + const two = await second.doc.watch(context); + first.release(); + expect(await one.closed).toEqual({ kind: "ended" }); + const frames = framesOf(two); + await second.bump(context); + await delivered(); + expect(frames.map(([value]) => value)).toEqual([{ n: 1 }]); + }); + + it("declared with self.get keep their object alive and are released when the declaring object ends", async () => { + const Inner = local("app.inner", { version: 1, params: { id: int() }, members: { n: int() } }); + const inner = vi.fn(() => ({ n: constant(1) })); + const { host } = await loaded((h) => { + h.serve(Inner, inner); + h.serve(Viewer, ({ id }, self) => ({ n: self.get(Inner, { id }).n })); + }); + const viewer = await host.get(Viewer, { id: 1 }, context); + const direct = await host.get(Inner, { id: 1 }, context); + direct.release(); + expect(viewer.n.value).toBe(1); + expect(inner).toHaveBeenCalledTimes(1); + viewer.release(); + // The viewer's teardown releases its reference in the background. + await flushed(); + await host.get(Inner, { id: 1 }, context); + expect(inner).toHaveBeenCalledTimes(2); + }); +}); + +describe("cycles (Q5)", () => { + const A = local("a", { version: 1, params: { n: int() }, members: { v: str() } }); + const B = local("b", { version: 1, params: { n: int() }, members: { v: str() } }); + const C = local("c", { version: 1, params: { n: int() }, members: { v: str() } }); + + it("fail a two-object cycle with invalid/cycle and keep nothing", async () => { + const factories = { a: vi.fn(), b: vi.fn() }; + const { host } = await loaded((h) => { + h.serve(A, ({ n }, self) => { + factories.a(); + return { v: self.get(B, { n }).v }; + }); + h.serve(B, ({ n }, self) => { + factories.b(); + return { v: self.get(A, { n }).v }; + }); + }); + expect(await rejection(host.get(A, { n: 1 }, context))).toEqual({ + code: "invalid", + reason: "cycle", + message: "dependency cycle: a(1) → b(1) → a(1)", + }); + await expect(host.get(A, { n: 1 }, context)).rejects.toThrow("dependency cycle"); + expect([factories.a.mock.calls.length, factories.b.mock.calls.length]).toEqual([2, 2]); + }); + + it("fail a three-object cycle", async () => { + const { host } = await loaded((h) => { + h.serve(A, ({ n }, self) => ({ v: self.get(B, { n }).v })); + h.serve(B, ({ n }, self) => ({ v: self.get(C, { n }).v })); + h.serve(C, ({ n }, self) => ({ v: self.get(A, { n }).v })); + }); + expect(await rejection(host.get(A, { n: 1 }, context))).toMatchObject({ + message: "dependency cycle: a(1) → b(1) → c(1) → a(1)", + }); + }); + + it("fail an object that gets itself", async () => { + const { host } = await loaded((h) => h.serve(A, ({ n }, self) => ({ v: self.get(A, { n }).v }))); + expect(await rejection(host.get(A, { n: 1 }, context))).toMatchObject({ + message: "dependency cycle: a(1) → a(1)", + }); + }); +}); + +describe("GC", () => { + // AbortSignal.any keeps cleanup entries per call while its sources live before Node 24.16.0 (nodejs/node#62367): a + // call's signal is linked to its reference and its attempt only until it settles. + it.each([ + ["without a caller's signal", () => context], + ["with a long-lived caller's signal", () => withAbortSignal(new AbortController().signal, context)], + ])("10,000 calls on one reference retain flat memory, %s", async (_case, caller) => { + const { host } = await loaded((h) => h.serve(Counter, (_, self) => serveCounter(self))); + const counter = await host.get(Counter, {}, context); + const calls = caller(); + const run = async () => { + for (let i = 0; i < 10_000; i++) await counter.bump(calls); + }; + await run(); + const before = await settledHeap(); + await run(); + // The leak retained about 350 B per call; a document commit itself retains nothing. + expect((await settledHeap()) - before).toBeLessThan(500_000); + }); + + it("a released reference keeps neither its contract nor its object", async () => { + const objects: WeakRef[] = []; + const { host } = await loaded((h) => + h.serve(Viewer, ({ id }, self) => { + objects.push(new WeakRef(self)); + return { n: constant(id) }; + }), + ); + const { reference, contract } = await (async () => { + // The caller's own declaration, equal to the served one. + const Same = local("app.viewer", { version: 1, params: { id: int() }, members: { n: int() } }); + const reference = await host.get(Same, { id: 1 }, context); + await reference.n.watch(context); + reference.release(); + return { reference, contract: new WeakRef(Same) }; + })(); + await expectCollected(contract, ...objects); + expect(() => reference.n.value).toThrow("app.viewer: released"); + }); + + it("a released reference whose state was read keeps no host", async () => { + const { reference, host } = await (async () => { + const { host } = await loaded((h) => h.serve(Viewer, ({ id }) => ({ n: constant(id) }))); + const reference = await host.get(Viewer, { id: 1 }, context); + expect(reference.state.value).toEqual({ kind: "live" }); + await (await reference.state.watch(context)).stop(); + reference.release(); + return { reference, host: new WeakRef(host) }; + })(); + await expectCollected(host); + expect(reference.state.value).toMatchObject({ kind: "ended", error: { code: "released" } }); + }); + + it("a released reference whose state was read after a failure keeps neither the failure's error nor the host", async () => { + const { reference, refs } = await (async () => { + const { host } = await loaded((h) => + h.serve(Viewer, (_, self) => { + self.open( + async () => { + throw broken(new Error("no configuration")); + }, + () => {}, + ); + return { n: constant(1) }; + }), + ); + const reference = host.hold(Viewer, { id: 1 }, context); + await flushed(); + const state = reference.state.value; + if (state.kind !== "failed") throw new Error("expected a failure"); + const refs = [new WeakRef(state.cause.error), new WeakRef(host)]; + reference.release(); + return { reference, refs }; + })(); + await expectCollected(...refs); + expect(reference.state.value).toMatchObject({ kind: "ended", error: { code: "released" } }); + }); + + it("a hold released before load keeps nothing: the reference and its contract go while the host waits", async () => { + const host = createHost({ onError: () => {} }); + const refs = (() => { + const Same = local("app.viewer", { version: 1, params: { id: int() }, members: { n: int() } }); + const reference = host.hold(Same, { id: 1 }, context); + reference.release(); + return [new WeakRef(reference), new WeakRef(Same)]; + })(); + await expectCollected(...refs); + expect(host).toBeDefined(); + }); + + it("a released hold keeps neither its object, nor its state's watches, nor a remote object without a link", async () => { + const objects: WeakRef[] = []; + const { host } = await loaded((h) => + h.serve(Viewer, ({ id }, self) => { + objects.push(new WeakRef(self)); + self.open( + () => new Promise(() => {}), + () => {}, + ); + return { n: constant(id) }; + }), + ); + const client = createHost({ onError: () => {} }); + await client.load([], context); + const { refs, held } = await (async () => { + const made = remoteObjects.length; + const local = host.hold(Viewer, { id: 1 }, context); + const remote = client.hold(Notes, { session: "S1" }, context); + const watches = [await local.state.watch(context), await remote.state.watch(context)]; + // A member watch waiting for the first value, and a parked call: both rejected by the release. + const waits = [local.n.watch(context).catch(() => {}), remote.wait(context).catch(() => {})]; + local.release(); + remote.release(); + await Promise.all(waits); + const refs = [...watches.map((watch) => new WeakRef(watch)), ...remoteObjects.slice(made)]; + return { refs, held: [local, remote] }; + })(); + await expectCollected(...objects, ...refs); + expect(held).toHaveLength(2); + }); +}); + +describe("release", () => { + it("rejects the calls made through the reference with released and aborts their handlers' contexts", async () => { + const Slow = local("app.slow", { version: 1, params: {}, members: { wait: method() } }); + const aborted: unknown[] = []; + const { host } = await loaded((h) => + h.serve(Slow, () => ({ + wait: (ctx) => + new Promise((resolve) => + ctx.abortSignal?.addEventListener("abort", () => { + aborted.push(ctx.abortSignal?.reason); + resolve(null); + }), + ), + })), + ); + const first = await host.get(Slow, {}, context); + const second = await host.get(Slow, {}, context); + const pending = first.wait(context); + const other = second.wait(context); + first.release(); + expect(await rejection(pending)).toEqual({ code: "released", message: "app.slow: released" }); + expect(aborted).toHaveLength(1); + second.release(); + await other.catch(() => {}); + expect(aborted).toHaveLength(2); + }); + + it("reads, watches and calls after the object ended carry the end's reason; the holder's own release has none", async () => { + const Ended = local("app.ended", { version: 1, params: {}, members: { n: int(), m: method() } }); + const { host } = await loaded((h) => h.serve(Ended, () => ({ n: constant(1), m: async () => null }))); + const ended = await host.get(Ended, {}, context); + const released = await host.get(Ended, {}, context); + released.release(); + await host.close(context); + const reasonOf = (error: unknown) => { + const { code, reason } = error as ChordError; + return `${code}/${reason}`; + }; + const thrown = (read: () => unknown) => { + try { + read(); + return "no throw"; + } catch (error) { + return reasonOf(error); + } + }; + expect([ + thrown(() => ended.n.value), + reasonOf(await ended.n.watch(context).catch((error: unknown) => error)), + reasonOf(await ended.m(context).catch((error: unknown) => error)), + ]).toEqual(["released/ended", "released/ended", "released/ended"]); + expect(thrown(() => released.n.value)).toBe("released/undefined"); + }); +}); + +describe("remote references", () => { + it("have every member of every variant; variant is undefined before live and released after release", async () => { + const Two = remote("app.two", { + version: 1, + params: {}, + variants: { full: { state: { n: int() }, methods: { bump: method() } }, view: { state: { m: int() } } }, + }); + const Holder = local("app.holder", { version: 1, params: {}, members: { n: int() } }); + const seen: unknown[] = []; + const owner = { kind: "owner" } as const; + const host = createHost({ principal: owner, onError: () => {} }); + const facet = defineFacet((h) => { + h.serve(Two, () => ({ variant: "view", members: { m: constant(1) } })); + h.serve(Holder, (_, self) => { + const two = self.get(Two, {}); + seen.push(Object.keys(two).sort()); + seen.push(two.variant); + return { n: constant(1) }; + }); + }); + await host.load([{ name: "@test/app", facet }], context); + expect(seen).toEqual([["bump", "m", "n", "release", "retry", "state", "variant"], undefined]); + const two = await host.get(Two, {}, context); + expect(() => (two as unknown as { n: { value: number } }).n.value).toThrow( + 'app.two (variant view) has no state member "n"', + ); + two.release(); + expect(() => two.variant).toThrow("app.two: released"); + }); +}); + +describe("remote reference types", () => { + it("narrow on variant; RefIn picks variants", () => { + const Two = remote("app.two", { + version: 1, + params: {}, + variants: { full: { state: { n: int() }, methods: { bump: method() } }, view: { state: { m: int() } } }, + }); + type Ref = Reference; + expectTypeOf().toEqualTypeOf<"full" | "view">(); + expectTypeOf["m"]["value"]>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf< + "variant" | "n" | "bump" | "release" | "retry" | "state" + >(); + expectTypeOf>().not.toHaveProperty("variant"); + }); + + it("a held reference's variant may be undefined (before its first binding) and still narrows", () => { + const Two = remote("app.two", { + version: 1, + params: {}, + variants: { full: { state: { n: int() }, methods: { bump: method() } }, view: { state: { m: int() } } }, + }); + expectTypeOf["variant"]>().toEqualTypeOf<"full" | "view" | undefined>(); + const narrow = (held: Held) => (held.variant === "full" ? held.bump : undefined); + expectTypeOf(narrow).returns.toEqualTypeOf<((ctx: Context) => Promise) | undefined>(); + expectTypeOf>().toEqualTypeOf>(); + expectTypeOf["state"]>().toEqualTypeOf>(); + expectTypeOf["retry"]>().toEqualTypeOf<() => void>(); + }); +}); + +describe("keyed state members (slice 6a §12)", () => { + const NoteSchema = obj({ title: str() }); + const Memo = defineState("memo", { + schema: NoteSchema, + version: 1, + initial: () => ({ title: "" }), + keyed: true, + }); + const NoteStore = local("app.note-store", { + version: 1, + params: {}, + members: { notes: keyed(NoteSchema), write: method(obj({ id: str(), title: str() })), remove: method(str()) }, + }); + const Relay = local("app.relay", { version: 1, params: {}, members: { notes: keyed(NoteSchema) } }); + /** A view of note `id`; views of one note with another `copy` are other objects. */ + const NoteView = local("app.note-view", { + version: 1, + params: { id: str(), copy: int() }, + members: { note: NoteSchema }, + }); + + /** Serves the store, a relay re-exposing its keyed member, and views pinning one key each; `memos()` is the store's. */ + async function notes(options: { readonly viewOf?: "store" | "relay" } = {}) { + let memos: KeyedHandle<{ title: string }> | undefined; + const pinned: Watchable<{ title: string }>[] = []; + const refs: object[] = []; + const { host, errors } = await loaded((h) => { + h.serve(NoteStore, (_, self) => { + memos = self.state(Memo); + return { + notes: memos, + write: async ({ id, title }, ctx) => { + await memos!.change( + id, + (draft) => { + draft.title = title; + }, + ctx, + ); + return null; + }, + remove: async (id, ctx) => { + await memos!.retire(id, ctx); + return null; + }, + }; + }); + h.serve(Relay, (_, self) => ({ notes: self.get(NoteStore, {}).notes })); + h.serve(NoteView, ({ id }, self) => { + const source = options.viewOf === "relay" ? self.get(Relay, {}) : self.get(NoteStore, {}); + refs.push(source); + const note = source.notes.at(id); + pinned.push(note); + return { note }; + }); + }); + const store = await host.get(NoteStore, {}, context); + return { host, errors, store, memos: () => memos!, pinned, refs }; + } + + it("a consumer's at(key) is the provider's per-key source, loaded before it goes live; retirement carries the flag", async () => { + const { host, store } = await notes(); + await store.write({ id: "a", title: "x" }, context); + const view = await host.get(NoteView, { id: "a", copy: 1 }, context); + expect(view.note.value).toEqual({ title: "x" }); + const changes = changesOf(await view.note.watch(context)); + await store.write({ id: "a", title: "y" }, context); + await delivered(); + await store.remove("a", context); + await delivered(); + expect(changes).toEqual([ + { kind: "value", value: { title: "y" }, ops: [["s", ["title"], "y"]], retired: false }, + { kind: "value", value: { title: "" }, ops: [["r", { title: "" }]], retired: true }, + ]); + view.release(); + await host.close(context); + }); + + it("two consumers of one key share one source and one pin; each hold is released once, with its reference", async () => { + const { host, store, memos, pinned } = await notes(); + await store.write({ id: "a", title: "x" }, context); + const first = await host.get(NoteView, { id: "a", copy: 1 }, context); + const second = await host.get(NoteView, { id: "a", copy: 2 }, context); + const slot = (memos() as unknown as { slot: { source(key: string): { pins: number } | undefined } }).slot; + const [one, two] = pinned.map((at) => (at as unknown as { source: object }).source); + // The store object's source of the key, which follows its attempts. + expect(one).toBe(two); + const shared = slot.source("a")!; + expect(shared.pins).toBe(1); + first.release(); + await flushed(); + expect(slot.source("a")).toBe(shared); + second.release(); + await flushed(); + expect(slot.source("a")).toBeUndefined(); + await host.close(context); + }); + + it("a re-exposed keyed member resolves to the ultimate provider's source (no second tracker)", async () => { + const { host, store, memos } = await notes({ viewOf: "relay" }); + await store.write({ id: "a", title: "x" }, context); + const view = await host.get(NoteView, { id: "a", copy: 1 }, context); + expect(view.note.value).toEqual({ title: "x" }); + const slot = (memos() as unknown as { slot: { source(key: string): { pins: number } | undefined } }).slot; + // One pin on the ultimate slot, through the relay's key and the store's. + expect(slot.source("a")!.pins).toBe(1); + await store.write({ id: "a", title: "y" }, context); + expect(view.note.value).toEqual({ title: "y" }); + await host.close(context); + }); + + it("ref.member.get and watch work anywhere; at(key) only in a factory", async () => { + const { host, store } = await notes(); + expect(await store.notes.get("a", context)).toBeUndefined(); + expect(await store.notes.watch("a", context)).toBeUndefined(); + await store.write({ id: "a", title: "x" }, context); + expect(await store.notes.get("a", context)).toEqual({ title: "x" }); + const watch = (await store.notes.watch("a", context))!; + expect(watch.value).toEqual({ title: "x" }); + expect(() => store.notes.at("a")).toThrow("valid only in the factory"); + store.release(); + expect(await watch.closed).toEqual({ kind: "ended" }); + await host.close(context); + }); + + it("a watch(key) settling after its reference was released is ended at once with its pin; it rejects released", async () => { + const { host, store, memos } = await notes(); + await store.write({ id: "a", title: "x" }, context); + const slot = (memos() as unknown as { slot: { source(key: string): object | undefined } }).slot; + const watching = store.notes.watch("a", context); + // The slot's watch is underway: the release lands before the reference follows it. + await Promise.resolve(); + await Promise.resolve(); + store.release(); + expect(await rejection(watching)).toMatchObject({ code: "released" }); + await flushed(); + expect(slot.source("a")).toBeUndefined(); + await host.close(context); + }); + + it("a retained released factory reference keeps no provider: its at(key) drops the source", async () => { + const from = chordObjects.length; + const { host, store, refs, pinned } = await notes(); + await store.write({ id: "a", title: "x" }, context); + const view = await host.get(NoteView, { id: "a", copy: 1 }, context); + view.release(); + store.release(); + await host.close(context); + // The factory's reference and its at(key) are still held here (and by the host's factory). + expect(refs).toHaveLength(1); + await expectCollected(...chordObjects.slice(from)); + expect(() => (pinned[0] as unknown as { value: unknown }).value).toThrow("released"); + }); + + /** Serves `NoteStore` over `definition` (persisted) on `store`, and views pinning one key each. */ + async function persistedNotes(store: Store, definition: ReturnType) { + return loaded( + (h) => { + h.serve(NoteStore, (_, self) => { + const memos = self.state(definition); + return { + notes: memos, + write: async ({ id, title }, ctx) => { + await memos.change( + id, + (draft) => { + draft.title = title; + }, + ctx, + ); + return null; + }, + remove: async () => null, + }; + }); + h.serve(NoteView, ({ id }, self) => ({ note: self.get(NoteStore, {}).notes.at(id) })); + }, + { store }, + ); + } + + /** A persisted keyed `memo` at `version`, whose `migrate` (from older versions) throws. */ + function persistedMemo(version: number) { + return defineState("memo", { + schema: NoteSchema, + version, + initial: () => ({ title: "" }), + migrate: () => { + throw new Error("cannot migrate"); + }, + keyed: true, + persist: true, + checkpointWhen: () => false, + }); + } + + /** A store holding memo `a` = "x" at version 1, written by a host of its own. */ + async function seeded(): Promise { + const store = await memoryStore(); + const { host } = await persistedNotes(store, persistedMemo(1)); + await (await host.get(NoteStore, {}, context)).write({ id: "a", title: "x" }, context); + await host.close(context); + return store; + } + + it("a persisted key a consumer pins is loaded from the store before the consumer goes live", async () => { + const store = await seeded(); + const { host } = await persistedNotes(store, persistedMemo(1)); + const view = await host.get(NoteView, { id: "a", copy: 1 }, context); + expect(view.note.value).toEqual({ title: "x" }); + await host.close(context); + await store.close(context); + }); + + it("a pin load that fails its provider fails the consumer as a dependency failure (transient)", async () => { + const store = await seeded(); + const { host, errors } = await persistedNotes(store, persistedMemo(2)); + const provider = await host.get(NoteStore, {}, context); + const view = host.hold(NoteView, { id: "a", copy: 1 }, context); + await vi.waitFor(() => expect(view.state.value.kind).toBe("failed")); + expect(view.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "transient", error: { message: expect.stringContaining("cannot migrate") } }, + }); + expect(provider.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + expect(errors.map(({ kind }) => kind)).toContain("object_failed"); + view.release(); + await host.close(context); + await store.close(context); + }); + + it("a failed provider's ref.member.get rejects with the failure's cause", async () => { + const store = await seeded(); + const { host } = await persistedNotes(store, persistedMemo(2)); + const provider = await host.get(NoteStore, {}, context); + expect(await rejection(provider.notes.get("a", context))).toMatchObject({ + message: expect.stringContaining("cannot migrate"), + }); + expect(await rejection(provider.notes.get("b", context))).toMatchObject({ + message: expect.stringContaining("cannot migrate"), + }); + await host.close(context); + await store.close(context); + }); + + it("a provider returning something else for a keyed member fails broken", async () => { + const { host } = await loaded((h) => { + // Plugin code returning a wrong value for the keyed member. + h.serve( + NoteStore, + () => ({ notes: constant({}), write: async () => null, remove: async () => null }) as never, + ); + }); + expect(await rejection(host.get(NoteStore, {}, context))).toMatchObject({ + message: expect.stringContaining("notes: a keyed member is a keyed document handle"), + }); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/revocation.test.ts b/packages/chord/test/next/host/revocation.test.ts new file mode 100644 index 000000000..a80fba200 --- /dev/null +++ b/packages/chord/test/next/host/revocation.test.ts @@ -0,0 +1,430 @@ +// Tests of revocation (§4, R10, R11): revocation keys, \`host.revoke\` on one host (objects, running calls, activations, +// holds still waiting for \`load\`), child links and \`revoke\` messages, GC, and revocation across a relay. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, remote } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet, type FacetHost, type Host } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { jsonPart } from "../../../src/next/link/internal.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { + expectCollected, + forEachCodec, + instancesOf, + Notes, + PlacedNotes, + RawPeer, + rawParent, + rawWorkers, + relayed, + Session, + serveNotes, + type TestPrincipal, + testCodec, + transported, + wireNames, +} from "../support.ts"; + +/** Every object attempted in this file, weakly. */ +const objects = instancesOf(ChordObject, "attempt"); + +/** The test app's revocations: a guest's session, or every owner. */ +type Revocation = { session: string } | { owners: true }; + +const revokedBy = (principal: TestPrincipal): Revocation[] => + principal.kind === "guest" + ? [{ session: principal.session }] + : principal.kind === "owner" + ? [{ owners: true }] + : principal.kind === "worker" + ? [{ session: principal.session }] + : []; + +const S1: TestPrincipal = { kind: "guest", session: "S1" }; + +/** A host-lifetime type whose activation waits for an open: `load` waits for it (the start barrier). */ +const Barrier = local("app.barrier", { version: 1, params: {}, members: { n: int() } }); + +/** A guest's variant: its `wait` method's index. */ +const { wait } = wireNames(Notes, "reader").methods as { wait: number }; + +/** A worker of session S1 serving `Notes` (as `serveNotes`), its parent a raw peer whose holds name their principal. */ +async function worker(setup: (host: FacetHost) => void = (h) => void serveNotes(h)) { + const host = createHost({ + principal: { kind: "worker", session: "S1" }, + revokedBy, + onError: () => {}, + }); + await host.load([{ name: "@test/worker", facet: defineFacet(setup) }], context); + const parent = await rawParent(host); + return { host, parent }; +} + +/** Sends a hold of `Notes` for S1 with id `hold`, acting as `principal` (honoured: the peer is the parent). */ +function holdAs(peer: RawPeer, hold: number, principal: TestPrincipal, Type: typeof Notes = Notes): void { + peer.send({ + kind: "hold", + hold, + type: Type.name, + version: Type.version, + fingerprint: Type.fingerprint, + params: jsonPart({ session: "S1" }), + principal, + }); +} + +describe("revocation keys", () => { + it("are computed once per principal object, compared as canonical JSON", async () => { + let calls = 0; + const host = createHost({ + principal: { kind: "server" }, + revokedBy: () => { + calls++; + return [{ b: 2, a: 1 }]; + }, + onError: () => {}, + }); + await host.load([{ name: "@test/app", facet: defineFacet((h) => serveNotes(h)) }], context); + const peer = await rawParent(host); + holdAs(peer, 0, S1); + holdAs(peer, 1, { kind: "guest", session: "S2" }); + await peer.next("bound"); + await peer.next("bound"); + host.revoke({ a: 1, b: 2 }, context); + host.revoke({ a: 1, b: 3 }, context); + await transported(); + expect(calls).toBe(2); + expect((await peer.next("gone")).hold).toBe(0); + expect((await peer.next("gone")).hold).toBe(1); + }); + + it("keep no principal alive (a WeakMap keyed by the principal the host cached them for)", async () => { + const cached: WeakRef[] = []; + const host = createHost({ + principal: { kind: "worker", session: "S1" }, + revokedBy: (principal) => { + // The principal decoded on this host, the cache's key: not the sender's object. + cached.push(new WeakRef(principal)); + return revokedBy(principal); + }, + onError: () => {}, + }); + await host.load([{ name: "@test/worker", facet: defineFacet((h) => serveNotes(h)) }], context); + const parent = await rawParent(host); + holdAs(parent, 0, { kind: "guest", session: "S9" }); + await parent.next("bound"); + host.revoke({ session: "S1" }, context); + parent.send({ kind: "release", hold: 0 }); + await transported(); + expect(cached.length).toBeGreaterThan(0); + await expectCollected(...cached); + // The cache's owner is still alive and working. + holdAs(parent, 1, S1); + await parent.next("bound"); + host.revoke({ session: "S1" }, context); + expect((await parent.next("gone")).hold).toBe(1); + await host.close(context); + }); +}); + +forEachCodec(() => { + describe("revoke on one host", () => { + it("ends the principal's objects at once, whatever references remain: each hold gets gone forbidden", async () => { + const { host, parent } = await worker(); + holdAs(parent, 0, S1); + holdAs(parent, 1, S1); + holdAs(parent, 2, { kind: "owner" }); + for (let i = 0; i < 3; i++) await parent.next("bound"); + host.revoke({ session: "S1" }, context); + await transported(); + const gone = [await parent.next("gone"), await parent.next("gone")]; + expect(gone.map(({ hold }) => hold).sort()).toEqual([0, 1]); + expect(gone[0]!.error).toEqual({ + kind: "chord", + error: { code: "forbidden", reason: "revoked", message: "app.notes(S1): revoked" }, + }); + // The owner's hold, the parent link and the local store stay. + expect(parent.closed).toBe(false); + expect(parent.kinds()).toEqual([]); + holdAs(parent, 3, S1); + expect((await parent.next("bound")).holds[0]!.hold).toBe(3); + }); + + it("aborts running calls (race 2): the handler's context, no result; a call crossing the gone is ignored", async () => { + let probe: ReturnType | undefined; + const { host, parent } = await worker((h) => { + probe = serveNotes(h); + }); + holdAs(parent, 0, S1); + await parent.next("bound"); + parent.send({ kind: "call", call: 0, hold: 0, method: wait }); + await transported(); + host.revoke({ session: "S1" }, context); + expect(probe!.waits[0]!.abortSignal?.reason).toMatchObject({ code: "forbidden" }); + expect((await parent.next("gone")).hold).toBe(0); + await transported(); + expect(parent.kinds()).toEqual([]); + parent.send({ kind: "call", call: 1, hold: 0, method: wait }); + await transported(); + expect(parent.kinds()).toEqual([]); + }); + + it("abandons an activation in progress (race 1): its open's context is aborted, its holders get gone", async () => { + const Slow = remote("app.slow", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const aborted: unknown[] = []; + const { host, parent } = await worker((h) => + h.serve(Slow, (_, _principal, self) => { + self.open( + (opening) => + new Promise((_, reject) => + opening.abortSignal?.addEventListener("abort", () => { + aborted.push(opening.abortSignal?.reason); + reject(opening.abortSignal?.reason); + }), + ), + () => {}, + ); + return { variant: "v", members: { n: constant(1) } }; + }), + ); + parent.send({ + kind: "hold", + hold: 0, + type: Slow.name, + version: Slow.version, + fingerprint: Slow.fingerprint, + params: "{}", + principal: S1, + }); + await transported(); + host.revoke({ session: "S1" }, context); + expect((await parent.next("gone")).error).toMatchObject({ kind: "chord", error: { code: "forbidden" } }); + expect(aborted).toMatchObject([{ code: "forbidden" }]); + }); + + it("never revokes the host's own principal", async () => { + const { host } = await worker(); + const own = await host.get(Notes, { session: "S1" }, context); + host.revoke({ session: "S1" }, context); + expect(own.state.value.kind).toBe("live"); + await host.close(context); + }); + + it("ends a hold the parent sent before its revoke while it still waits for load (race 3): gone, never bound", async () => { + for (const delayed of [false, true]) { + // A worker connects before it loads (W6); a load can wait for an activation (the start barrier). + const opened = Promise.withResolvers(); + const host = createHost({ + principal: { kind: "worker", session: "S1" }, + revokedBy, + onError: () => {}, + }); + const parent = await rawParent(host); + const facet = defineFacet((h) => { + serveNotes(h); + if (delayed) { + h.serve(Barrier, (_params, self) => { + self.open( + () => opened.promise, + () => {}, + ); + return { n: constant(0) }; + }); + } + }); + const loading = host.load([{ name: "@test/worker", facet }], context); + holdAs(parent, 0, S1); + parent.send({ kind: "revoke", revocation: { session: "S1" } }); + await transported(); + opened.resolve(); + await loading; + await transported(); + expect(await parent.next("gone")).toMatchObject({ + hold: 0, + error: { kind: "chord", error: { code: "forbidden", reason: "revoked" } }, + }); + expect(parent.kinds()).toEqual([]); + expect(host.references(Notes).value).toEqual([]); + await host.close(context); + } + }); + + it("ends a relayed hold of the principal still waiting for its route's link: never held there", async () => { + const host = createHost({ + principal: { kind: "worker", session: "S0" }, + revokedBy, + onError: () => {}, + }); + const resolving = Promise.withResolvers(); + const routed: RawPeer[] = []; + host.route( + Session, + { + authorize: () => true, + resolve: async () => { + await resolving.promise; + const [a, b] = memoryLinkPair(); + routed.push(new RawPeer(b)); + return { link: a, principal: { kind: "worker", session: "S1" }, codec: testCodec() }; + }, + }, + context, + ); + await host.load([], context); + const parent = await rawParent(host); + const { name: type, version, fingerprint } = PlacedNotes; + const address = { kind: "session", params: { id: "S1" } }; + parent.send({ + kind: "hold", + hold: 0, + type, + version, + fingerprint, + params: jsonPart({ session: "S1" }), + address, + principal: S1, + }); + await transported(); + parent.send({ kind: "revoke", revocation: { session: "S1" } }); + expect(await parent.next("gone")).toMatchObject({ + hold: 0, + error: { kind: "chord", error: { code: "forbidden", reason: "revoked" } }, + }); + resolving.resolve(); + await transported(); + await transported(); + expect(routed.flatMap((peer) => peer.received.filter(({ kind }) => kind === "hold"))).toEqual([]); + await host.close(context); + }); + + it("binds a hold made for a revoked principal, then the parent's revoke after it ends it (race 3, R10's window)", async () => { + const { host, parent } = await worker(); + holdAs(parent, 0, S1); + await parent.next("bound"); + // The parent's revoke follows the hold on the same link, so it reaches the worker after it. + parent.send({ kind: "revoke", revocation: { session: "S1" } }); + expect((await parent.next("gone")).error).toMatchObject({ kind: "chord", error: { code: "forbidden" } }); + await host.close(context); + }); + }); + + describe("links and revoke messages", () => { + it("closes the child links whose principal it revokes, never the parent link", async () => { + const host = createHost({ + principal: { kind: "server" }, + revokedBy, + onError: () => {}, + }); + await host.load([], context); + const peers = [S1, { kind: "guest", session: "S2" } as const, S1].map((principal) => { + const [a, b] = memoryLinkPair(); + host.accept(a, { principal, codec: testCodec() }, context); + return new RawPeer(b); + }); + const parent = await rawParent(host); + host.revoke({ session: "S1" }, context); + await transported(); + // Two tabs of one grant: both links close (race 4). + expect(peers.map(({ closed }) => closed)).toEqual([true, false, true]); + expect(parent.closed).toBe(false); + // A revocation is no message for a child, nor for the parent. + expect(peers[1]!.kinds()).toEqual(["hello"]); + expect(parent.kinds()).toEqual([]); + }); + + it("ignores a revoke from a child link (race 6)", async () => { + const host = createHost({ + principal: { kind: "server" }, + revokedBy, + onError: () => {}, + }); + await host.load([{ name: "@test/app", facet: defineFacet((h) => serveNotes(h)) }], context); + const [a, b] = memoryLinkPair(); + host.accept(a, { principal: S1, codec: testCodec() }, context); + const child = new RawPeer(b); + await child.next("hello"); + holdAs(child, 0, S1); + await child.next("bound"); + child.send({ kind: "revoke", revocation: { session: "S1" } }); + await transported(); + expect(child.closed).toBe(false); + expect(child.kinds()).toEqual([]); + }); + + it("sends revoke on every route link, and on none made later (race 5: nothing is replayed)", async () => { + const workers: RawPeer[] = []; + const relay = await relayed({ resolve: rawWorkers(workers) }); + const server = relay.server as unknown as Host; + relay.tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + server.revoke({ session: "S7" }, context); + await transported(); + expect(workers[0]!.received.filter(({ kind }) => kind === "revoke")).toEqual([ + { kind: "revoke", revocation: { session: "S7" } }, + ]); + relay.tab.hold(PlacedNotes, { session: "S2" }, context); + await transported(); + expect(workers[1]!.received.some(({ kind }) => kind === "revoke")).toBe(false); + }); + }); +}); + +describe("GC of revoked objects", () => { + it("keeps no revoked object, however many references its holders still have", async () => { + const { host, refs, parent } = await (async () => { + const { host, parent } = await worker(); + const made = objects.length; + holdAs(parent, 0, S1); + await parent.next("bound"); + const refs = objects.slice(made); + expect(refs.length).toBeGreaterThan(0); + host.revoke({ session: "S1" }, context); + await parent.next("gone"); + return { host, refs, parent }; + })(); + await expectCollected(...refs.filter((ref: WeakRef) => ref.deref()?.served.kind === "remote")); + expect(parent.closed).toBe(false); + await host.close(context); + }); +}); + +describe("revocation across a relay", () => { + it("closes a revoked tab's link at the server and ends its objects on the worker", async () => { + const relay = await relayed({ tab: S1, revokedBy }); + const notes = relay.tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + await transported(); + expect(notes.state.value.kind).toBe("live"); + const worker = relay.workers.get("session|S1")!; + const present = worker.references(PlacedNotes); + expect(present.value).toHaveLength(1); + const server = relay.server as unknown as Host; + server.revoke({ session: "S1" }, context); + await transported(); + await transported(); + // The tab's link closed: its reference waits for a new link, its holds on the worker are released. + expect(notes.state.value.kind).toBe("loading"); + expect(present.value).toEqual([]); + }); + + it("ends the worker's objects of the principal through the revoke message alone; the relay forwards their gone", async () => { + // The server's own revokedBy names no guest: its tab link stays, only the worker applies the revocation. + const relay = await relayed({ tab: S1, revokedBy, serverRevokedBy: () => [] }); + const notes = relay.tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + await transported(); + expect(notes.state.value.kind).toBe("live"); + const server = relay.server as unknown as Host; + server.revoke({ session: "S1" }, context); + await transported(); + await transported(); + expect(notes.state.value.kind).toBe("ended"); + expect(relay.workers.get("session|S1")!.references(PlacedNotes).value).toEqual([]); + expect(relay.errors.tab).toEqual([]); + // The tab's link stayed open: a new hold binds. + const again = await relay.tab.get(PlacedNotes, { session: "S1" }, context); + expect(again.state.value.kind).toBe("live"); + }); +}); diff --git a/packages/chord/test/next/host/route.test.ts b/packages/chord/test/next/host/route.test.ts new file mode 100644 index 000000000..dc4485504 --- /dev/null +++ b/packages/chord/test/next/host/route.test.ts @@ -0,0 +1,368 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, createContextKey, withContextValue } from "../../../src/context/index.ts"; +import { hostKind, int, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { + addNote, + expectCollected, + fakeTimers, + flushed, + forEachCodec, + PlacedNotes, + rejection, + relayed, + Session, + serveNotes, + type TestPrincipal, + transported, +} from "../support.ts"; + +/** Advances fake time by `ms`, then lets links and promises settle. */ +async function elapse(ms: number): Promise { + await vi.advanceTimersByTimeAsync(ms); + await transported(); +} + +forEachCodec(() => { + describe("route admission", () => { + it("runs one resolve per address, shared by the holds that wait for it (race 1)", async () => { + const { tab, resolves, workers } = await relayed(); + const [one, two] = await Promise.all([ + tab.get(PlacedNotes, { session: "S1" }, context), + tab.get(PlacedNotes, { session: "S1" }, context), + ]); + expect(resolves).toEqual(["session|S1"]); + await addNote(workers.get("session|S1")!, "S1", "a"); + await transported(); + expect(one.notes.value).toEqual([{ text: "a" }]); + expect(two.notes.value).toEqual([{ text: "a" }]); + await tab.get(PlacedNotes, { session: "S2" }, context); + expect(resolves).toEqual(["session|S1", "session|S2"]); + }); + + it("authorizes every hold with its principal: a refusal is broken forbidden, other principals still resolve (race 4)", async () => { + const authorize = (principal: TestPrincipal, address: { readonly key: string }) => + principal.kind === "owner" || + (principal.kind === "guest" && address.key === `session|${principal.session}`); + const guest = await relayed({ tab: { kind: "guest", session: "S1" }, authorize }); + expect(await rejection(guest.tab.get(PlacedNotes, { session: "S2" }, context))).toEqual({ + code: "forbidden", + message: "session|S2: not authorized", + }); + expect(guest.resolves).toEqual([]); + expect((await guest.tab.get(PlacedNotes, { session: "S1" }, context)).variant).toBe("reader"); + const owner = await relayed({ authorize }); + expect((await owner.tab.get(PlacedNotes, { session: "S2" }, context)).variant).toBe("editor"); + }); + + it("fails a hold broken with what authorize throws", async () => { + const { tab } = await relayed({ + authorize: () => { + throw new Error("authorize threw"); + }, + }); + expect(await rejection(tab.get(PlacedNotes, { session: "S1" }, context))).toMatchObject({ + message: "app.placed-notes: authorize threw", + }); + }); + + it("fails a hold broken invalid/placement for an address whose params do not fit the kind", async () => { + const Misplaced = remote("app.misplaced", { + version: 1, + params: { n: int() }, + // An address of the right kind with params of the wrong type, as a buggy placement could compute. + placement: ({ n }) => ({ kind: "session", params: { id: n }, key: "" }) as never, + variants: { v: { state: { n: int() } } }, + }); + const { tab, resolves } = await relayed(); + expect(await rejection(tab.get(Misplaced, { n: 1 }, context))).toMatchObject({ + code: "invalid", + reason: "placement", + }); + expect(resolves).toEqual([]); + }); + }); + + describe("resolve failures and re-resolves", () => { + fakeTimers(); + + it("a rejected resolve leaves the holds loading, is reported, and resolves again after the address's backoff (race 2)", async () => { + let failures = 2; + const { tab, resolves, errors } = await relayed({ + resolve: async (_address, worker) => { + if (failures-- > 0) throw new Error("no worker yet"); + return worker(); + }, + }); + const notes = tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + expect(notes.state.value).toEqual({ kind: "loading" }); + expect(errors.server).toMatchObject([{ kind: "resolve_failed", address: "session|S1" }]); + await elapse(999); + expect(errors.server).toHaveLength(1); + await elapse(1); + // The second failure doubles the window. + expect(errors.server).toHaveLength(2); + await elapse(1999); + expect(notes.state.value).toEqual({ kind: "loading" }); + await elapse(1); + expect(resolves).toEqual(["session|S1"]); + expect(notes.state.value).toEqual({ kind: "live" }); + }); + + it("a returned link that is closed already starts the backoff", async () => { + let closed = true; + const { tab, resolves } = await relayed({ + resolve: async (address, worker) => { + if (!closed) return worker(); + closed = false; + const [a, b] = memoryLinkPair(); + b.close(); + await flushed(); + return { link: a, principal: { kind: "worker", session: address.params.id } }; + }, + }); + const notes = tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + expect(notes.state.value).toEqual({ kind: "loading" }); + await elapse(1000); + expect(resolves).toEqual(["session|S1"]); + expect(notes.state.value).toEqual({ kind: "live" }); + }); + + it("a dropped link resolves again while held: references go loading, then live on a fresh worker (race 6)", async () => { + const { tab, resolves, routeLinks, workers } = await relayed(); + const notes = await tab.get(PlacedNotes, { session: "S1" }, context); + const first = workers.get("session|S1")!; + routeLinks[0]!.close(); + await transported(); + expect(notes.state.value).toEqual({ kind: "loading" }); + await elapse(1000); + expect(resolves).toEqual(["session|S1", "session|S1"]); + expect(workers.get("session|S1")).not.toBe(first); + expect(notes.state.value).toEqual({ kind: "live" }); + await addNote(workers.get("session|S1")!, "S1", "fresh"); + await transported(); + expect(notes.notes.value).toEqual([{ text: "fresh" }]); + }); + + it("only a link that completed its handshake resets the backoff", async () => { + let silent = 0; + const { tab, resolves, routeLinks } = await relayed({ + resolve: async (address, worker) => { + if (silent-- <= 0) return worker(); + // A link whose other end never greets and closes. + const [a, b] = memoryLinkPair(); + setTimeout(() => b.close(), 10); + return { link: a, principal: { kind: "worker", session: address.params.id } }; + }, + }); + const notes = await tab.get(PlacedNotes, { session: "S1" }, context); + silent = 1; + routeLinks[0]!.close(); + await elapse(1000); + // The silent link closed before its handshake: the window doubles. + await elapse(10); + await elapse(1999); + expect(resolves).toEqual(["session|S1"]); + await elapse(1); + expect(resolves).toEqual(["session|S1", "session|S1"]); + expect(notes.state.value).toEqual({ kind: "live" }); + // That link greeted: the next drop starts again at 1 s. + routeLinks.at(-1)!.close(); + await elapse(1000); + expect(resolves).toHaveLength(3); + }); + + it("a link whose peer never greets closes after 30 s of silence; the address backs off, then resolves again while held", async () => { + // The heartbeat's timer too, before the hosts start it. + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"], now: 0 }); + let silent = true; + const { tab, resolves } = await relayed({ + resolve: async (address, worker) => { + if (!silent) return worker(); + silent = false; + // Its other end is never read: no hello, no close. + const [a] = memoryLinkPair(); + return { link: a, principal: { kind: "worker", session: address.params.id } }; + }, + }); + const notes = tab.hold(PlacedNotes, { session: "S1" }, context); + await elapse(29_000); + expect(resolves).toEqual([]); + expect(notes.state.value).toEqual({ kind: "loading" }); + // The heartbeat checks every 1.5 s: the silent link closes within it, then its address waits 1 s. + await elapse(2_500); + await elapse(1_000); + expect(resolves).toEqual(["session|S1"]); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + }); + + it("an address without holds is not resolved again after its link drops", async () => { + const { tab, resolves, routeLinks } = await relayed(); + const notes = await tab.get(PlacedNotes, { session: "S1" }, context); + notes.release(); + await transported(); + routeLinks[0]!.close(); + await elapse(60_000); + expect(resolves).toEqual(["session|S1"]); + }); + + it("a resolve whose holds were all released meanwhile finishes and its link is kept (race 3)", async () => { + let finish: () => void = () => {}; + const { tab, resolves, routeLinks, server } = await relayed({ + resolve: async (_address, worker) => { + await new Promise((resolve) => { + finish = resolve; + }); + return worker(); + }, + }); + const notes = tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + notes.release(); + await transported(); + finish(); + await transported(); + expect(resolves).toEqual(["session|S1"]); + let closed = false; + routeLinks[0]!.onClose(() => { + closed = true; + }); + await elapse(60_000); + expect(closed).toBe(false); + // The kept link serves the next hold without a resolve. + await tab.get(PlacedNotes, { session: "S1" }, context); + expect(resolves).toEqual(["session|S1"]); + await server.close(context); + expect(closed).toBe(true); + }); + }); + + describe("host.close during a resolve (race 7)", () => { + it("aborts the resolve's context and closes a link returned afterwards", async () => { + let routing: AbortSignal | undefined; + let finish: () => void = () => {}; + const { tab, server, routeLinks } = await relayed({ + resolve: async (_address, worker, context) => { + routing = context.abortSignal; + await new Promise((resolve) => { + finish = resolve; + }); + return worker(); + }, + }); + tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + expect(routing?.aborted).toBe(false); + await server.close(context); + expect(routing?.aborted).toBe(true); + finish(); + await transported(); + let closed = false; + routeLinks[0]!.onClose(() => { + closed = true; + }); + expect(closed).toBe(true); + }); + }); + + describe("declaring routes", () => { + it("is done before load, once per kind", async () => { + const host = createHost({ onError: () => {} }); + const options = { authorize: () => true, resolve: () => new Promise(() => {}) }; + host.route(Session, options, context); + expect(() => host.route(Session, options, context)).toThrow("route: session is routed already"); + await host.load([], context); + expect(() => host.route(hostKind("other"), options, context)).toThrow( + "route: routes are declared before load", + ); + }); + }); + + describe("a route's context", () => { + it("is the parent of its resolves and of the work arriving on its links", async () => { + const App = createContextKey("app"); + const Who = remote("app.who", { + version: 1, + params: {}, + variants: { v: { methods: { app: method(undefined, str()) } } }, + }); + const resolved: (string | undefined)[] = []; + let worker: Host | undefined; + const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + server.route( + Session, + { + authorize: () => true, + resolve: async (address, routing) => { + resolved.push(routing.value(App)); + const [a, b] = memoryLinkPair(); + worker = createHost({ + address, + principal: { kind: "worker", session: "S1" }, + onError: () => {}, + }); + worker.connect(b, { principal: { kind: "server" } }, context); + await worker.load( + [ + { + name: "@test/worker", + facet: defineFacet((h) => void serveNotes(h, PlacedNotes)), + }, + ], + context, + ); + return { link: a, principal: { kind: "worker", session: "S1" } }; + }, + }, + withContextValue(App, "the app's", context), + ); + const facet = defineFacet((h) => + h.serve(Who, () => ({ variant: "v", members: { app: async (ctx) => ctx.value(App) ?? "none" } })), + ); + await server.load([{ name: "@test/server", facet }], context); + server.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + await transported(); + // The worker's call reaches the server over the route link: its handler runs under the route's context. + const who = await worker!.get(Who, {}, context); + expect(await who.app(context)).toBe("the app's"); + expect(resolved).toEqual(["the app's"]); + }); + }); + + describe("GC", () => { + it("a closed route link and its released references keep no route state", async () => { + const { refs, tab } = await (async () => { + const { tab, routeLinks, workers } = await relayed(); + const notes = await tab.get(PlacedNotes, { session: "S1" }, context); + const refs = [new WeakRef(notes), new WeakRef(workers.get("session|S1")!), new WeakRef(routeLinks[0]!)]; + notes.release(); + await transported(); + routeLinks[0]!.close(); + workers.clear(); + routeLinks.length = 0; + await transported(); + return { refs, tab }; + })(); + await expectCollected(...refs); + expect(tab).toBeDefined(); + }); + + it("holds released while their resolve never settles are not kept", async () => { + const { refs, server } = await (async () => { + const { tab, server } = await relayed({ resolve: () => new Promise(() => {}) }); + const notes = tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + notes.release(); + await transported(); + return { refs: [new WeakRef(notes)], server }; + })(); + await expectCollected(...refs); + expect(server).toBeDefined(); + }); + }); +}); diff --git a/packages/chord/test/next/host/routines.test.ts b/packages/chord/test/next/host/routines.test.ts new file mode 100644 index 000000000..e3bb69a6e --- /dev/null +++ b/packages/chord/test/next/host/routines.test.ts @@ -0,0 +1,1148 @@ +// Tests of routines (§14): `on` (the first call with the current value, folding, no overlap, retries after a throw +// with the whole value, retirements, the source ending or failing, native sources, facets), `onLive` (returning, +// retries, the backoff reset after a long run, aborts), the drain and `host.close`, and what routines leave behind. + +import { describe, expect, expectTypeOf, it, vi } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import type { Op } from "../../../src/delta/index.ts"; +import { int, local, method, obj, remote } from "../../../src/next/contracts/index.ts"; +import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { + type Ctx, + createHost, + defineFacet, + type HostError, + type RefState, + type Self, + sleep, + untilAborted, +} from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { jsonPart } from "../../../src/next/link/internal.ts"; +import { + NativeWatch, + type NativeWatchable, + type NativeWatchCall, + Watch, + type Watchable, + type WatchCall, +} from "../../../src/watch.ts"; +import { + EDITOR, + expectCollected, + fakeTimers, + flushed, + linked, + loaded, + Notes, + rawParent, + serveNotes, + type TestPrincipal, + transported, +} from "../support.ts"; + +/** What an `on` handler gets: a watch's value call. */ +type WatchValueCall = Extract, { kind: "value" }>; +type NativeWatchValueCall = Extract, { kind: "value" }>; + +const Doc = defineState("doc", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + +/** A host-lifetime object with a counter document and `bump`; its factory declares routines with `routines`. */ +const Store = local("app.store", { version: 1, params: {}, members: { n: int(), bump: method() } }); + +/** The same with params: it lives while referenced. */ +const Item = local("app.item", { version: 1, params: { id: int() }, members: { n: int(), bump: method() } }); + +/** Loads a host serving `Store`; `declare` declares its routines over its document. */ +async function store( + declare: (doc: SingletonHandle<{ n: number }>, self: Self) => void, + options: { readonly closeTimeout?: number } = {}, +) { + const loadedHost = await loaded( + (h) => + h.serve(Store, (_, self) => { + const doc = self.state(Doc); + declare(doc, self); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }), + options, + ); + const ref = await loadedHost.host.get(Store, {}, context); + return { ...loadedHost, ref }; +} + +/** Counts one up in one transaction. */ +async function bump(doc: SingletonHandle<{ n: number }>, ctx: Context): Promise { + await doc.change((d) => { + d.n++; + }, ctx); + return null; +} + +/** Waits one macrotask and a little: a call after a busy one waits for one (`settling`). */ +function settled(): Promise { + return new Promise((resolve) => setTimeout(resolve, 5)); +} + +/** The routine reports among `errors`, as `kind #position: message`. */ +function reports(errors: readonly HostError[]): string[] { + return errors.flatMap((error) => + error.kind === "routine_failed" + ? [`${error.routine.kind} #${error.routine.position}: ${(error.error as Error).message}`] + : [], + ); +} + +/** A JSON watchable (Chord's shared `Watch`) the test advances by hand, as a Durable document would. */ +function jsonSource(initial: { n: number }) { + let value = initial; + const watches = new Set>(); + const source: Watchable<{ n: number }> = { + get value() { + return value; + }, + async watch(ctx) { + const watch: Watch<{ n: number }> = new Watch( + value, + () => watches.delete(watch), + () => {}, + ); + watches.add(watch); + if (ctx.abortSignal !== undefined) watch.observeAbort(ctx.abortSignal); + return watch; + }, + }; + const advance = (call: WatchCall<{ n: number }>): void => { + if (call.kind === "value") value = call.value; + for (const watch of watches) watch.advance(call, context); + }; + return { source, advance }; +} + +/** A native watchable the test sets and ends by hand. */ +function writable(initial: T) { + let value = initial; + const watches = new Set>(); + const source: NativeWatchable & { set(next: T): void; end(): void } = { + get value() { + return value; + }, + async watch(ctx) { + const watch: NativeWatch = new NativeWatch( + value, + () => watches.delete(watch), + () => {}, + ); + watches.add(watch); + if (ctx.abortSignal !== undefined) watch.observeAbort(ctx.abortSignal); + return watch; + }, + set(next) { + value = next; + for (const watch of watches) watch.advance(next, context); + }, + end() { + for (const watch of watches) watch.end({ kind: "ended" }); + }, + }; + return { source, watches }; +} + +describe("on", () => { + it("calls first with the current value as [['r', value]], then once per change with its ops", async () => { + const calls: [unknown, readonly Op[]][] = []; + const { ref } = await store((doc, self) => { + self.on(doc, async ({ value, ops }) => { + calls.push([value, ops]); + }); + }); + await flushed(); + await ref.bump(context); + await flushed(); + expect(calls).toEqual([ + [{ n: 0 }, [["r", { n: 0 }]]], + [{ n: 1 }, [["s", ["n"], 1]]], + ]); + }); + + it("never overlaps calls: changes during a call fold into the next, which is not aborted by them", async () => { + const gate = Promise.withResolvers(); + const calls: [unknown, readonly Op[]][] = []; + let running = 0; + let overlapped = false; + let aborted = false; + const { ref } = await store((doc, self) => { + self.on(doc, async ({ value, ops }, ctx) => { + overlapped ||= running > 0; + running++; + calls.push([value, ops]); + if (calls.length === 2) await gate.promise; + aborted ||= ctx.abortSignal?.aborted === true; + running--; + }); + }); + await flushed(); + await ref.bump(context); + await flushed(); + await ref.bump(context); + await ref.bump(context); + gate.resolve(); + await settled(); + expect(calls.map(([value]) => value)).toEqual([{ n: 0 }, { n: 1 }, { n: 3 }]); + expect(calls[2]![1]).toEqual([["s", ["n"], 3]]); + expect({ overlapped, aborted }).toEqual({ overlapped: false, aborted: false }); + }); + + describe("after a throw", () => { + fakeTimers(); + + it("reports, keeps the watch, and calls again with [['r', value]] after 1 s doubling; success resets", async () => { + const calls: [unknown, readonly Op[], number][] = []; + let failing = 3; + const { ref, errors } = await store((doc, self) => { + self.on(doc, async ({ value, ops }) => { + calls.push([value, ops, Date.now()]); + if (failing-- > 0) throw new Error(`fail ${calls.length}`); + }); + }); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(1_000); + await vi.advanceTimersByTimeAsync(2_000); + await vi.advanceTimersByTimeAsync(4_000); + expect(calls.map(([, , at]) => at)).toEqual([0, 1_000, 3_000, 7_000]); + expect(calls.every(([, ops]) => ops[0]![0] === "r")).toBe(true); + expect(reports(errors)).toEqual(["on #1: fail 1", "on #1: fail 2", "on #1: fail 3"]); + // Succeeded: the next change gets its ops, and a later throw backs off from 1 s again. + await ref.bump(context); + await vi.advanceTimersByTimeAsync(0); + expect(calls.at(-1)!.slice(0, 2)).toEqual([{ n: 1 }, [["s", ["n"], 1]]]); + failing = 1; + await ref.bump(context); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(1_000); + expect(calls.map(([, , at]) => at).slice(-2)).toEqual([7_000, 8_000]); + expect(vi.getTimerCount()).toBe(0); + }); + + it("retries a change folded behind the throwing call after the backoff, once: no third call, no timer", async () => { + const gate = Promise.withResolvers(); + const calls: [unknown, readonly Op[], number][] = []; + const { ref } = await store((doc, self) => { + self.on(doc, async ({ value, ops }) => { + calls.push([value, ops, Date.now()]); + if (calls.length === 1) { + await gate.promise; + throw new Error("first"); + } + }); + }); + await vi.advanceTimersByTimeAsync(0); + await ref.bump(context); + gate.resolve(); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(5_000); + expect(calls).toEqual([ + [{ n: 0 }, [["r", { n: 0 }]], 0], + [{ n: 1 }, [["r", { n: 1 }]], 1_000], + ]); + expect(vi.getTimerCount()).toBe(0); + }); + + it("makes one retry per throw; a change during a failing retry waits for its backoff; no timer left", async () => { + const calls: [unknown, number][] = []; + const { host, errors } = await loaded((h) => + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + self.on(doc, async ({ value }) => { + calls.push([value, Date.now()]); + // The retry at 1 s changes the source, then throws too. + if (calls.length === 2) await bump(doc, context); + if (calls.length <= 2) throw new Error(`fail ${calls.length}`); + }); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(1_000); + await vi.advanceTimersByTimeAsync(10_000); + expect(calls).toEqual([ + [{ n: 0 }, 0], + [{ n: 0 }, 1_000], + [{ n: 1 }, 3_000], + ]); + expect(reports(errors)).toEqual(["on #1: fail 1", "on #1: fail 2"]); + expect(vi.getTimerCount()).toBe(0); + item.release(); + await vi.advanceTimersByTimeAsync(0); + expect(vi.getTimerCount()).toBe(0); + }); + + it("backs off after a throw even when the failed call changed its own source: no burst", async () => { + const calls: number[] = []; + const { errors } = await store((doc, self) => { + self.on(doc, async (_change, ctx) => { + calls.push(Date.now()); + if (calls.length > 100) return; + // A reconciler that records its attempt in its own source, then fails. + await bump(doc, ctx); + throw new Error("downstream unavailable"); + }); + }); + await vi.advanceTimersByTimeAsync(0); + expect(calls).toEqual([0]); + await vi.advanceTimersByTimeAsync(1_000); + await vi.advanceTimersByTimeAsync(2_000); + expect(calls).toEqual([0, 1_000, 3_000]); + expect(reports(errors)).toHaveLength(3); + }); + + it("retries with the value that arrived during the failed call, as a whole value", async () => { + const calls: [unknown, readonly Op[], number][] = []; + await store((doc, self) => { + self.on(doc, async ({ value, ops }, ctx) => { + calls.push([value, ops, Date.now()]); + if (calls.length > 1) return; + await bump(doc, ctx); + throw new Error("once"); + }); + }); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(1_000); + expect(calls).toEqual([ + [{ n: 0 }, [["r", { n: 0 }]], 0], + [{ n: 1 }, [["r", { n: 1 }]], 1_000], + ]); + }); + + it("folds changes made during a slow retry into one call with the newest value", async () => { + const gate = Promise.withResolvers(); + const calls: [unknown, readonly Op[]][] = []; + const { ref } = await store((doc, self) => { + self.on(doc, async ({ value, ops }) => { + calls.push([value, ops]); + if (calls.length === 1) throw new Error("first"); + if (calls.length === 2) await gate.promise; + }); + }); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(1_000); + for (let i = 0; i < 3; i++) await ref.bump(context); + gate.resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(calls).toEqual([ + [{ n: 0 }, [["r", { n: 0 }]]], + [{ n: 0 }, [["r", { n: 0 }]]], + [{ n: 3 }, [["s", ["n"], 3]]], + ]); + }); + + it("waits for the next good value when its source fails during the backoff: the retry is dropped", async () => { + const calls: unknown[] = []; + let set: ((n: number) => Promise) | undefined; + const { errors } = await store((doc, self) => { + const failing = self.derive([doc], ([d]) => { + if (d.n < 0) throw new Error("compute"); + return d.n; + }); + set = (n) => + doc.change((d) => { + d.n = n; + }, context); + self.on(failing, async ({ value }) => { + calls.push(value); + if (value === 1) throw new Error("handler threw"); + }); + }); + await vi.advanceTimersByTimeAsync(0); + await set!(1); + await vi.advanceTimersByTimeAsync(0); + await set!(-1); + await vi.advanceTimersByTimeAsync(2_000); + expect(calls).toEqual([0, 1]); + expect(vi.getTimerCount()).toBe(0); + await set!(3); + await vi.advanceTimersByTimeAsync(0); + expect(calls).toEqual([0, 1, 3]); + expect(reports(errors)).toEqual(["on #1: handler threw"]); + }); + + it("clears its retry timer when its object ends during the backoff", async () => { + const { host } = await loaded((h) => + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + self.on(doc, async () => { + throw new Error("always"); + }); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(0); + expect(vi.getTimerCount()).toBe(1); + item.release(); + await vi.advanceTimersByTimeAsync(0); + expect(vi.getTimerCount()).toBe(0); + }); + + it("reports a retirement again on the retry after the call that got it threw", async () => { + const { source, advance } = jsonSource({ n: 0 }); + const calls: WatchValueCall<{ n: number }>[] = []; + await store((_, self) => { + self.on(source, async (call) => { + calls.push(call); + if (calls.length === 2) throw new Error("retirement not handled"); + }); + }); + await vi.advanceTimersByTimeAsync(0); + advance({ kind: "value", value: { n: 0 }, ops: [], retired: true }); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(1_000); + await vi.advanceTimersByTimeAsync(10_000); + expect(calls.slice(1)).toEqual([ + { kind: "value", value: { n: 0 }, ops: [], retired: true }, + { kind: "value", value: { n: 0 }, ops: [["r", { n: 0 }]], retired: true }, + ]); + }); + + it("calls at once with the newest whole value when the source changes during the backoff", async () => { + const calls: [unknown, readonly Op[], number][] = []; + const { ref } = await store((doc, self) => { + self.on(doc, async ({ value, ops }) => { + calls.push([value, ops, Date.now()]); + if (calls.length === 1) throw new Error("first"); + }); + }); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(500); + await ref.bump(context); + await vi.advanceTimersByTimeAsync(0); + await ref.bump(context); + await vi.advanceTimersByTimeAsync(0); + expect(calls).toEqual([ + [{ n: 0 }, [["r", { n: 0 }]], 0], + [{ n: 1 }, [["r", { n: 1 }]], 500], + [{ n: 2 }, [["s", ["n"], 2]], 500], + ]); + // The backoff timer is gone: nothing calls again. + await vi.advanceTimersByTimeAsync(10_000); + expect(calls).toHaveLength(3); + }); + }); + + it("folds every change made during the first call into one next call: [0, 3], not [0, 1, 3]", async () => { + const gate = Promise.withResolvers(); + const calls: [unknown, readonly Op[]][] = []; + const { ref } = await store((doc, self) => { + self.on(doc, async ({ value, ops }) => { + calls.push([value, ops]); + if (calls.length === 1) await gate.promise; + }); + }); + await flushed(); + for (let i = 0; i < 3; i++) await ref.bump(context); + await flushed(); + gate.resolve(); + await settled(); + expect(calls).toEqual([ + [{ n: 0 }, [["r", { n: 0 }]]], + [{ n: 3 }, [["s", ["n"], 3]]], + ]); + }); + + it("follows an app's JSON watchable whose watch is Chord's shared Watch with ops, as its type says", async () => { + const { source, advance } = jsonSource({ n: 0 }); + const calls: [unknown, readonly Op[]][] = []; + await store((_, self) => { + self.on(source, async ({ value, ops }) => void calls.push([value, ops])); + }); + await flushed(); + advance({ kind: "value", value: { n: 1 }, ops: [["s", ["n"], 1]], retired: false }); + await flushed(); + expect(calls).toEqual([ + [{ n: 0 }, [["r", { n: 0 }]]], + [{ n: 1 }, [["s", ["n"], 1]]], + ]); + }); + + it("reports a retirement as retired, folded with the changes around it into one call, and goes on", async () => { + const gate = Promise.withResolvers(); + const { source, advance } = jsonSource({ n: 0 }); + const calls: WatchValueCall<{ n: number }>[] = []; + await store((_, self) => { + self.on(source, async (call) => { + calls.push(call); + if (calls.length === 1) await gate.promise; + }); + }); + await flushed(); + // A Durable document retired (the last value, no ops) and created again, while the first call runs. + advance({ kind: "value", value: { n: 0 }, ops: [], retired: true }); + advance({ kind: "value", value: { n: 5 }, ops: [["r", { n: 5 }]], retired: false }); + gate.resolve(); + await settled(); + advance({ kind: "value", value: { n: 6 }, ops: [["s", ["n"], 6]], retired: false }); + await flushed(); + expect(calls).toEqual([ + { kind: "value", value: { n: 0 }, ops: [["r", { n: 0 }]], retired: false }, + { kind: "value", value: { n: 5 }, ops: [["r", { n: 5 }]], retired: true }, + { kind: "value", value: { n: 6 }, ops: [["s", ["n"], 6]], retired: false }, + ]); + }); + + it("folds a native source to its newest value after a busy call", async () => { + const gate = Promise.withResolvers(); + const { source } = writable("a"); + const calls: unknown[] = []; + await store((_, self) => { + self.on(source, async ({ value }: { value: string }) => { + calls.push(value); + if (calls.length === 1) await gate.promise; + }); + }); + await flushed(); + source.set("b"); + source.set("c"); + await flushed(); + gate.resolve(); + await settled(); + expect(calls).toEqual(["a", "c"]); + }); + + it("takes every value a busy call left buffered before calling again: [0, 20], also when it set them itself", async () => { + const { source } = writable(0); + const calls: unknown[] = []; + await store((_, self) => { + self.on(source, async ({ value }: { value: number }) => { + calls.push(value); + if (value === 0) for (let n = 1; n <= 20; n++) source.set(n); + }); + }); + await settled(); + expect(calls).toEqual([0, 20]); + const gate = Promise.withResolvers(); + const later = writable(0); + const seen: unknown[] = []; + await store((_, self) => { + self.on(later.source, async ({ value }: { value: number }) => { + seen.push(value); + if (value === 0) await gate.promise; + }); + }); + await flushed(); + for (let n = 1; n <= 20; n++) later.source.set(n); + // No flush in between: the watch still holds the values when the call ends. + gate.resolve(); + await settled(); + expect(seen).toEqual([0, 20]); + }); + + it("reports a source whose watch cannot be taken; a source that ended meanwhile ends it quietly", async () => { + const failing: NativeWatchable = { + value: 0, + watch: async () => { + throw new Error("no watch"); + }, + }; + const ended: NativeWatchable = { + value: 0, + watch: async () => { + const watch = new NativeWatch( + 0, + () => {}, + () => {}, + ); + watch.end({ kind: "ended" }); + return watch; + }, + }; + const calls: unknown[] = []; + const { errors } = await store((_, self) => { + self.on(failing, async ({ value }) => void calls.push(value)); + self.on(ended, async ({ value }) => void calls.push(value)); + }); + await flushed(); + expect(calls).toEqual([]); + expect(reports(errors)).toEqual(["on #1: no watch"]); + }); + + it("ends without a report when its source ends", async () => { + const Other = local("app.other", { version: 1, params: { id: int() }, members: { n: int() } }); + let otherDoc: SingletonHandle<{ n: number }> | undefined; + const calls: unknown[] = []; + const { host, errors } = await loaded((h) => { + h.serve(Other, (_, self) => { + otherDoc = self.state(Doc); + return { n: self.derive([otherDoc], ([d]) => d.n) }; + }); + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + // Another object's document: it ends with that object. + self.on(otherDoc!, async ({ value }) => void calls.push(value)); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }); + }); + const other = await host.get(Other, { id: 1 }, context); + await host.get(Item, { id: 1 }, context); + await flushed(); + other.release(); + await flushed(); + expect(calls).toEqual([{ n: 0 }]); + expect(errors).toEqual([]); + }); + + it("is not called for its source's failures: it waits for the next value (a derive healing)", async () => { + const calls: unknown[] = []; + const { ref, errors } = await store((doc, self) => { + const failing = self.derive([doc], ([d]) => { + if (d.n > 0 && d.n < 3) throw new Error("compute"); + return d.n; + }); + self.on(failing, async ({ value, ops }) => void calls.push([value, ops])); + }); + await flushed(); + await ref.bump(context); + await ref.bump(context); + await flushed(); + expect(calls).toEqual([[0, [["r", 0]]]]); + await ref.bump(context); + await flushed(); + expect(calls).toEqual([ + [0, [["r", 0]]], + [3, [["r", 3]]], + ]); + // The derive's owner reports the failure (once for the episode); the routine reports nothing. + expect(errors).toEqual([expect.objectContaining({ kind: "derive_failed", error: new Error("compute") })]); + }); + + it("drops a value queued behind a busy call when a failure follows it; the next call is the healed value", async () => { + const calls: unknown[] = []; + const gate = Promise.withResolvers(); + const entered = Promise.withResolvers(); + let set: ((n: number) => Promise) | undefined; + const { errors } = await store((doc, self) => { + const failing = self.derive([doc], ([d]) => { + if (d.n < 0) throw new Error("compute"); + return d.n; + }); + set = (n) => + doc.change((d) => { + d.n = n; + }, context); + self.on(failing, async ({ value }) => { + calls.push(value); + if (value === 1) { + entered.resolve(); + await gate.promise; + } + }); + }); + await flushed(); + await set!(1); + await entered.promise; + // Queued behind the busy call, then replaced by the failure. + await set!(2); + await set!(-1); + gate.resolve(); + await settled(); + expect(calls).toEqual([0, 1]); + await set!(3); + await settled(); + expect(calls).toEqual([0, 1, 3]); + expect(reports(errors)).toEqual([]); + }); + + it("starts on a source failed now: no call and no report until its first good value (a reference member)", async () => { + const Failing = local("app.failing", { version: 1, params: {}, members: { t: int(), set: method(int()) } }); + const Viewer = local("app.on-viewer", { version: 1, params: { id: int() }, members: { m: method() } }); + const calls: unknown[] = []; + const { host, errors } = await loaded((h) => { + h.serve(Failing, (_, self) => { + const doc = self.state(Doc); + return { + t: self.derive([doc], ([d]) => { + if (d.n === 1) throw new Error("t threw"); + return d.n; + }), + set: async (n, ctx) => { + await doc.change((d) => { + d.n = n; + }, ctx); + return null; + }, + }; + }); + h.serve(Viewer, (_, self) => { + self.on(self.get(Failing, {}).t, async ({ value, ops }) => void calls.push([value, ops])); + return { m: async () => null }; + }); + }); + const failing = await host.get(Failing, {}, context); + await failing.set(1, context); + const viewer = await host.get(Viewer, { id: 1 }, context); + await settled(); + expect(calls).toEqual([]); + await failing.set(2, context); + await settled(); + expect(calls).toEqual([[2, [["r", 2]]]]); + expect(errors.map(({ kind }) => kind)).toEqual(["derive_failed"]); + viewer.release(); + await host.close(context); + }); + + it("follows a reference member: the provider's value first, then its changes", async () => { + const calls: [unknown, readonly Op[]][] = []; + let bumper: (() => Promise) | undefined; + const { host } = await loaded((h) => { + h.serve(Store, (_, self) => { + const doc = self.state(Doc); + bumper = () => bump(doc, context); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }); + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + const store = self.get(Store, {}); + self.on(store.n, async ({ value, ops }) => void calls.push([value, ops])); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }); + }); + await host.get(Item, { id: 1 }, context); + await flushed(); + await bumper!(); + await flushed(); + expect(calls).toEqual([ + [0, [["r", 0]]], + [1, [["r", 1]]], + ]); + }); + + it("converges when it writes its own source: an unchanged draft commits nothing", async () => { + let calls = 0; + await store((doc, self) => { + self.on(doc, async ({ value }, ctx) => { + calls++; + await doc.change((d) => { + d.n = Math.min(value.n + 1, 3); + }, ctx); + }); + }); + await new Promise((resolve) => setTimeout(resolve, 20)); + // 0 → 1 → 2 → 3, then the write of 3 changes nothing and nothing calls again. + expect(calls).toBe(4); + }); + + it("follows a native source: one call per new value, without ops; ends with it", async () => { + const { source, watches } = writable("a"); + const calls: unknown[] = []; + const { errors } = await store((_, self) => { + self.on(source, async (call) => void calls.push(call)); + }); + await flushed(); + source.set("b"); + source.set("c"); + await settled(); + expect(calls).toEqual([ + { kind: "value", value: "a" }, + { kind: "value", value: "b" }, + { kind: "value", value: "c" }, + ]); + source.end(); + expect(watches.size).toBe(0); + expect(errors).toEqual([]); + }); + + it("on a remote member with no value yet, a first value followed at once by a failure calls nothing", async () => { + const Viewer = local("app.viewer", { version: 1, params: { id: int() }, members: {} }); + const calls: unknown[] = []; + const client = createHost({ onError: () => {} }); + const facet = defineFacet((h) => + h.serve(Viewer, (_, self) => { + self.on(self.get(Notes, { session: "S1" }).count, async ({ value }) => void calls.push(value)); + return {}; + }), + ); + await client.load([{ name: "@test/client", facet }], context); + const peer = await rawParent(client); + const getting = client.get(Viewer, { id: 1 }, context); + const { hold } = await peer.next("hold"); + const member = EDITOR.state.count!; + const failed = [{ member, error: { kind: "plugin", code: "internal", message: "count threw" } } as const]; + const notes = { member: EDITOR.state.notes!, value: jsonPart([]) }; + peer.send({ kind: "bound", holds: [{ hold, variant: EDITOR.variant, members: [notes], failed }] }); + await getting; + await transported(); + // Two state messages in one receive turn: the first value, then a failure again. + const value = (n: number) => [{ member, ops: jsonPart([["r", n]]) }]; + peer.send({ kind: "state", holds: [{ hold, members: value(5), failed: [], retired: [] }] }); + peer.send({ kind: "state", holds: [{ hold, members: [], failed, retired: [] }] }); + await transported(); + await flushed(); + expect(calls).toEqual([]); + peer.send({ kind: "state", holds: [{ hold, members: value(6), failed: [], retired: [] }] }); + await transported(); + await flushed(); + expect(calls).toEqual([6]); + await client.close(context); + }); + + it("on a remote member failed since its binding (no value yet), waits for its first value", async () => { + const Halves = remote("app.halves", { + version: 1, + params: {}, + variants: { v: { state: { half: int() }, methods: { set: method(int()) } } }, + }); + const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const serve = defineFacet((h) => + h.serve(Halves, (_, _principal, self) => { + const doc = self.state(Doc); + return { + variant: "v", + members: { + half: self.derive([doc], ([{ n }]) => { + if (n === 2) throw new Error("half threw"); + return n / 2; + }), + set: async (n, ctx) => { + await doc.change((d) => { + d.n = n; + }, ctx); + return null; + }, + }, + }; + }), + ); + await server.load([{ name: "@test/app", facet: serve }], context); + const halves = await server.get(Halves, {}, context); + if (halves.variant !== "v") throw new Error("expected variant v"); + await halves.set(2, context); + const Viewer = local("app.viewer", { version: 1, params: { id: int() }, members: {} }); + const calls: unknown[] = []; + const errors: HostError[] = []; + const client = createHost({ onError: (error) => errors.push(error) }); + const facet = defineFacet((h) => + h.serve(Viewer, (_, self) => { + self.on(self.get(Halves, {}).half, async ({ value, ops }) => void calls.push([value, ops])); + return {}; + }), + ); + await client.load([{ name: "@test/client", facet }], context); + const [a, b] = memoryLinkPair(); + // The server's own principal: the client holds the object `halves` refers to. + server.accept(a, { principal: { kind: "server" } }, context); + client.connect(b, { principal: { kind: "server" } }, context); + await client.get(Viewer, { id: 1 }, context); + await transported(); + await flushed(); + expect([calls, errors]).toEqual([[], []]); + await halves.set(4, context); + await transported(); + await flushed(); + expect(calls).toEqual([[2, [["r", 2]]]]); + await client.close(context); + await server.close(context); + }); + + it("follows ref.state: a reference over a link going loading and live again", async () => { + const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + await server.load([{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context); + const Viewer = local("app.viewer", { version: 1, params: { id: int() }, members: {} }); + const states: string[] = []; + const client = createHost({ onError: () => {} }); + const facet = defineFacet((h) => + h.serve(Viewer, (_, self) => { + const notes = self.get(Notes, { session: "S1" }); + self.on(notes.state, async ({ value: state }: { value: RefState }) => void states.push(state.kind)); + return {}; + }), + ); + await client.load([{ name: "@test/client", facet }], context); + const connect = () => { + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" } }, context); + client.connect(b, { principal: { kind: "server" } }, context); + return a; + }; + const link = connect(); + await client.get(Viewer, { id: 1 }, context); + await flushed(); + link.close(); + await transported(); + connect(); + await transported(); + await flushed(); + expect(states).toEqual(["live", "loading", "live"]); + await client.close(context); + await server.close(context); + }); + + it("releases its watch when its object ends; a call that throws as it ends is not reported", async () => { + const { source, watches } = writable(0); + const { host, errors } = await loaded((h) => + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + self.on(source, async (_change, ctx) => { + await untilAborted(ctx); + throw new Error("stopping"); + }); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + await flushed(); + expect(watches.size).toBe(1); + item.release(); + await flushed(); + expect(watches.size).toBe(0); + expect(errors).toEqual([]); + }); +}); + +describe("onLive", () => { + fakeTimers(); + + it("is called once when its object is live; returning ends it", async () => { + let calls = 0; + await store((_, self) => { + self.onLive(async () => { + calls++; + }); + }); + await vi.advanceTimersByTimeAsync(10_000); + expect(calls).toBe(1); + }); + + it("restarts after a throw with the backoff, reset after a run of at least a minute", async () => { + const starts: number[] = []; + const { errors } = await store((_, self) => { + self.onLive(async (ctx) => { + starts.push(Date.now()); + if (starts.length === 3) await sleep(60_000, ctx); + throw new Error(`run ${starts.length}`); + }); + }); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(1_000); + await vi.advanceTimersByTimeAsync(2_000 + 60_000); + await vi.advanceTimersByTimeAsync(1_000); + expect(starts).toEqual([0, 1_000, 3_000, 64_000]); + expect(reports(errors)).toEqual(["onLive #1: run 1", "onLive #1: run 2", "onLive #1: run 3", "onLive #1: run 4"]); + }); + + it("is aborted by its object's end: no report, no restart", async () => { + let calls = 0; + const { host, errors } = await loaded((h) => + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + self.onLive(async (ctx) => { + calls++; + await sleep(1_000_000, ctx); + }); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(0); + item.release(); + await vi.advanceTimersByTimeAsync(10_000); + expect({ calls, errors, timers: vi.getTimerCount() }).toEqual({ calls: 1, errors: [], timers: 0 }); + }); +}); + +describe("routines and the end", () => { + it("start in the turn their object went live, next to a method call admitted in that turn", async () => { + const order: string[] = []; + const { host } = await loaded((h) => + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + self.on(doc, async ({ value }) => void order.push(`on ${value.n}`)); + self.onLive(async () => void order.push("onLive")); + return { + n: self.derive([doc], ([d]) => d.n), + bump: async (ctx) => { + order.push("bump"); + return bump(doc, ctx); + }, + }; + }), + ); + const item = host.hold(Item, { id: 1 }, context); + const bumped = item.bump(context); + // The parked call runs as the object goes live, then `onLive` in that turn, the first `on` call after its watch; + // the call's commit may land while `on 0` runs, and a call after a busy one waits for the settling turn. + await item.state.watch(context).then((watch) => watch.stop()); + await bumped; + await settled(); + expect(order).toEqual(["bump", "onLive", "on 0", "on 1"]); + }); + + it("never start a routine call once the object ended in its first turn: no call, no report, no rejection", async () => { + const calls: string[] = []; + let held: { release(): void } | undefined; + const { host, errors } = await loaded((h) => + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + self.on(doc, async () => void calls.push("on")); + self.onLive(async () => { + calls.push("onLive #1"); + held!.release(); + }); + self.onLive(async () => void calls.push("onLive #2")); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }), + ); + held = host.hold(Item, { id: 1 }, context); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect({ calls, errors }).toEqual({ calls: ["onLive #1"], errors: [] }); + }); + + it("are in the drain before their handler runs: a handler releasing the last reference keeps natives open", async () => { + const gate = Promise.withResolvers(); + const events: string[] = []; + let held: { release(): void } | undefined; + const { host } = await loaded((h) => + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + self.open( + async () => null, + () => void events.push("native closed"), + ); + self.on(doc, async ({ value }) => { + if (value.n === 0) return; + // Ends its own object before its first await. + held!.release(); + await gate.promise; + events.push("handler done"); + }); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + held = item; + await flushed(); + // The call may see its object end: its own routine ends it. + await item.bump(context).catch(() => {}); + await flushed(); + expect(events).toEqual([]); + gate.resolve(); + await flushed(); + expect(events).toEqual(["handler done", "native closed"]); + }); + + it("make no call queued behind a busy one once the object ended", async () => { + const gate = Promise.withResolvers(); + const calls: unknown[] = []; + const { host } = await loaded((h) => + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + self.on(doc, async ({ value }) => { + calls.push(value); + await gate.promise; + }); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + await flushed(); + await item.bump(context); + await flushed(); + item.release(); + gate.resolve(); + await flushed(); + expect(calls).toEqual([{ n: 0 }]); + }); + + it("release their calls' ctx.get references when each call settles", async () => { + const Other = local("app.other", { version: 1, params: { id: int() }, members: { n: int() } }); + const { host } = await loaded((h) => { + h.serve(Other, (_, self) => ({ n: self.derive([self.state(Doc)], ([d]) => d.n) })); + h.serve(Store, (_, self) => { + const doc = self.state(Doc); + self.onLive(async (ctx) => { + await ctx.get(Other, { id: 1 }); + }); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }); + }); + await flushed(); + expect(host.references(Other).value).toEqual([]); + }); + + it("act as a remote object's principal", async () => { + const Watcher = remote("app.watcher", { version: 1, params: {}, variants: { v: { state: {} } } }); + const seen: unknown[] = []; + const { client } = await linked( + (h) => + h.serve(Watcher, (_, _principal, self) => { + self.onLive(async (ctx) => void seen.push(ctx.principal)); + return { variant: "v", members: {} }; + }), + { child: { kind: "guest", session: "S1" } }, + ); + await client.get(Watcher, {}, context); + await flushed(); + expect(seen).toEqual([{ kind: "guest", session: "S1" }]); + }); + + it("type their handler by the source's kind: ops and retired for a JSON watchable, the value alone for a native one", () => { + const declare = (self: Self, doc: Watchable<{ n: number }>, state: NativeWatchable) => { + self.on(doc, async (call, ctx) => { + expectTypeOf(call).toEqualTypeOf>(); + expectTypeOf(call.ops).toEqualTypeOf(); + expectTypeOf(call.retired).toEqualTypeOf(); + expectTypeOf(ctx).toEqualTypeOf>(); + }); + self.on(state, async (call, ctx) => { + expectTypeOf(call).toEqualTypeOf>(); + expectTypeOf(ctx).toEqualTypeOf>(); + }); + }; + void declare; + }); + + it("are aborted by host.close, which drains their running calls (bounded by closeTimeout)", async () => { + const ended: string[] = []; + const { host, errors } = await loaded( + (h) => { + h.serve(Store, (_, self) => { + const doc = self.state(Doc); + self.onLive(async (ctx) => { + await new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", resolve)); + await new Promise((resolve) => setTimeout(resolve, 10)); + ended.push("object"); + }); + self.onLive(async () => { + // Never stops: the drain gives up at closeTimeout. + await new Promise(() => {}); + }); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }); + }, + { closeTimeout: 50 }, + ); + await flushed(); + await host.close(context); + expect(ended).toEqual(["object"]); + expect(errors).toEqual([{ kind: "drain_timeout", object: "app.store" }]); + }); + + it("leave nothing of an ended object: its routines' watches and contexts are collected", async () => { + const { source, watches } = writable(0); + let refs: WeakRef[] = []; + const { host } = await loaded((h) => + h.serve(Item, (_, self) => { + const doc = self.state(Doc); + self.on(source, async (_change, ctx) => void refs.push(new WeakRef(ctx))); + self.on(doc, async (_change, ctx) => void refs.push(new WeakRef(ctx))); + return { n: self.derive([doc], ([d]) => d.n), bump: async (ctx) => bump(doc, ctx) }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + await flushed(); + item.release(); + await flushed(); + expect({ watches: watches.size, calls: refs.length }).toEqual({ watches: 0, calls: 2 }); + await expectCollected(...refs); + refs = []; + }); +}); diff --git a/packages/chord/test/next/host/sleep.test.ts b/packages/chord/test/next/host/sleep.test.ts new file mode 100644 index 000000000..93c5fb9ab --- /dev/null +++ b/packages/chord/test/next/host/sleep.test.ts @@ -0,0 +1,74 @@ +// Tests of the waiting helpers (§14): `sleep` resolving, rejecting with the abort reason, chaining delays above the +// timer limit and leaving no timer behind; `untilAborted`. + +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT, withCancel } from "../../../src/context/index.ts"; +import { sleep, untilAborted } from "../../../src/next/host/index.ts"; +import { fakeTimers } from "../support.ts"; + +describe("sleep", () => { + fakeTimers(); + + it("resolves after its delay", async () => { + let done = false; + void sleep(1_000, BACKGROUND_CONTEXT).then(() => { + done = true; + }); + await vi.advanceTimersByTimeAsync(999); + expect(done).toBe(false); + await vi.advanceTimersByTimeAsync(1); + expect(done).toBe(true); + expect(vi.getTimerCount()).toBe(0); + }); + + it("rejects with the abort reason and clears its timer", async () => { + const { context, cancel } = withCancel(BACKGROUND_CONTEXT); + const waiting = sleep(60_000, context); + const reason = new Error("stop"); + cancel(reason); + await expect(waiting).rejects.toBe(reason); + expect(vi.getTimerCount()).toBe(0); + }); + + it("rejects at once when its context is aborted already", async () => { + const { context, cancel } = withCancel(BACKGROUND_CONTEXT); + cancel(new Error("before")); + await expect(sleep(10, context)).rejects.toThrow("before"); + expect(vi.getTimerCount()).toBe(0); + }); + + it("chains timers for delays above 2^31-1 ms; an abort clears the chained timer", async () => { + const limit = 2 ** 31 - 1; + let done = false; + void sleep(limit + 5_000, BACKGROUND_CONTEXT).then(() => { + done = true; + }); + await vi.advanceTimersByTimeAsync(limit); + expect(done).toBe(false); + expect(vi.getTimerCount()).toBe(1); + await vi.advanceTimersByTimeAsync(5_000); + expect(done).toBe(true); + const { context, cancel } = withCancel(BACKGROUND_CONTEXT); + const long = sleep(3 * limit, context); + await vi.advanceTimersByTimeAsync(limit + 1); + cancel(new Error("late")); + await expect(long).rejects.toThrow("late"); + expect(vi.getTimerCount()).toBe(0); + }); +}); + +describe("untilAborted", () => { + it("resolves once its context is aborted, at once when it is already", async () => { + const { context, cancel } = withCancel(BACKGROUND_CONTEXT); + let done = false; + const waiting = untilAborted(context).then(() => { + done = true; + }); + await Promise.resolve(); + expect(done).toBe(false); + cancel(); + await waiting; + expect(done).toBe(true); + await untilAborted(context); + }); +}); diff --git a/packages/chord/test/next/jobs/definition.test.ts b/packages/chord/test/next/jobs/definition.test.ts new file mode 100644 index 000000000..4f16502ac --- /dev/null +++ b/packages/chord/test/next/jobs/definition.test.ts @@ -0,0 +1,106 @@ +// Tests of job definitions (§15): what `defineJob` checks at declaration (names, versions, closed JSON args, a +// singleton state, concurrency, the shape of `accepts`, `persist` refused before slice 6) and the types it gives. + +import { describe, expect, expectTypeOf, it } from "vitest"; +import { bytes, int, map, obj, str, unknown } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { defineJob, type JobDefinition } from "../../../src/next/jobs/index.ts"; + +const Progress = defineState("app.progress", { + schema: obj({ output: str() }), + version: 1, + initial: () => ({ output: "" }), +}); + +const Keyed = defineState("app.keyed", { + schema: obj({ output: str() }), + version: 1, + initial: () => ({ output: "" }), + keyed: true, +}); + +const args = obj({ command: str() }); + +describe("defineJob", () => { + it("keeps its name, version, Chord's copies of its schemas, its state and concurrency", () => { + const Shell = defineJob("app.shell", { version: 2, args, result: int(), state: Progress, concurrency: 1 }); + expect(Shell).toMatchObject({ name: "app.shell", version: 2, state: Progress, concurrency: 1, accepts: {} }); + expect(Shell.args.schema).not.toBe(args); + expectTypeOf(Shell).toEqualTypeOf>(); + const Plain = defineJob("app.plain", { version: 1, args, result: str() }); + expectTypeOf(Plain).toEqualTypeOf>(); + }); + + it('takes a restricted name only: lowercase [a-z0-9@/._~-] (decision "Restricted names")', () => { + for (const name of ["@app/jobs.run_1~x-y", "a"]) + expect(defineJob(name, { version: 1, args, result: int() }).name).toBe(name); + for (const name of ["", "App.run", "app:run", "app run", "a\0b", "a\uD800", "é"]) { + expect(() => defineJob(name, { version: 1, args, result: int() })).toThrow( + `A job name ${JSON.stringify(name)}: expected lowercase [a-z0-9@/._~-]`, + ); + } + // Plugin code, not type-checked. + expect(() => defineJob(42 as never, { version: 1, args, result: int() })).toThrow("A job name 42"); + }); + + it("refuses a bad name, version, unknown option or concurrency", () => { + expect(() => defineJob("", { version: 1, args, result: int() })).toThrow("A job name"); + expect(() => defineJob("j", { version: 0, args, result: int() })).toThrow( + "j.version: expected a positive integer", + ); + // Plugin code, not type-checked. + const unknownKey = { version: 1, args, result: int(), retries: 3 } as never; + expect(() => defineJob("j", unknownKey)).toThrow('j: unknown key "retries"'); + for (const concurrency of [0, 1.5, -1]) { + expect(() => defineJob("j", { version: 1, args, result: int(), concurrency })).toThrow( + "j.concurrency: expected a positive integer", + ); + } + }); + + it("requires closed JSON args, as a method's: open objects and bytes are refused; records are fine", () => { + const open = { type: "object", properties: { a: str() } } as never; + expect(() => defineJob("j", { version: 1, args: open, result: int() })).toThrow( + "argument objects must be closed", + ); + expect(() => defineJob("j", { version: 1, args: obj({ data: bytes() }), result: int() })).toThrow( + "j.args: job arguments are JSON", + ); + expect(() => defineJob("j", { version: 1, args: obj({ tags: map(str()) }), result: unknown() })).not.toThrow(); + }); + + it("takes a singleton document definition as its state", () => { + expect(() => defineJob("j", { version: 1, args, result: int(), state: Keyed as never })).toThrow( + "j.state: expected a singleton document definition", + ); + }); + + it("checks accepts at declaration: older versions only, each with an args and a result function", () => { + const adapter = { args: (old: { cmd: string }) => ({ command: old.cmd }), result: (old: number) => old }; + expect(() => + defineJob("j", { version: 3, args, result: int(), accepts: { 1: adapter, 2: adapter } }), + ).not.toThrow(); + expect(() => defineJob("j", { version: 2, args, result: int(), accepts: { 2: adapter } })).toThrow( + "j.accepts[2]: not an older version", + ); + expect(() => defineJob("j", { version: 2, args, result: int(), accepts: { 0: adapter } })).toThrow( + "j.accepts[0]: not an older version", + ); + const noResult = { 1: { args: adapter.args } } as never; + expect(() => defineJob("j", { version: 2, args, result: int(), accepts: noResult })).toThrow( + "j.accepts[1].result: expected a function", + ); + const extra = { 1: { ...adapter, state: () => null } } as never; + expect(() => defineJob("j", { version: 2, args, result: int(), accepts: extra })).toThrow( + 'j.accepts[1]: unknown key "state"', + ); + }); + + it("refuses persist before slice 6, in its final shape", () => { + expect(() => defineJob("j", { version: 1, args, result: int(), persist: { resume: "rerun" } })).toThrow( + "j.persist: persisted jobs are not supported before slice 6", + ); + // @ts-expect-error: resume is required + expect(() => defineJob("j", { version: 1, args, result: int(), persist: {} })).toThrow(); + }); +}); diff --git a/packages/chord/test/next/jobs/job.test.ts b/packages/chord/test/next/jobs/job.test.ts new file mode 100644 index 000000000..a868d0f04 --- /dev/null +++ b/packages/chord/test/next/jobs/job.test.ts @@ -0,0 +1,1378 @@ +// Tests of jobs in memory (§15): starts (args, dedupe, concurrency, `tx.start`, `outcome`, when `run` starts), runs +// (the ctx, the state document and its fence, throws and aborts), finishes (cleanup, retries, blocked, dropped), +// `running`, keep-alive, the object's end, cost guards and GC. + +import { describe, expect, expectTypeOf, it, vi } from "vitest"; +import { + type Context, + BACKGROUND_CONTEXT as context, + withAbortSignal, + withCancel, +} from "../../../src/context/index.ts"; +import type { Op } from "../../../src/delta/index.ts"; +import { arr, int, local, obj, remote, str, unknown } from "../../../src/next/contracts/index.ts"; +import { ChordError, PluginError, type Principal } from "../../../src/next/data/index.ts"; +import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type HostError, type Self, untilAborted } from "../../../src/next/host/index.ts"; +import { + defineJob, + type JobCtx, + type JobDefinition, + type JobHandle, + type JobImplementation, + type Outcome, + type Run, +} from "../../../src/next/jobs/index.ts"; +import { type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; +import { Batch } from "../../../src/next/state/internal.ts"; +import { + delivered, + expectCollected, + fakeTimers, + flushed, + framesOf, + loaded, + metered, + rejection, + type TestPrincipal, + transported, +} from "../support.ts"; + +const Log = defineState("app.log", { schema: obj({ list: arr(str()) }), version: 1, initial: () => ({ list: [] }) }); + +const Progress = defineState("app.progress", { + schema: obj({ output: str() }), + version: 1, + initial: () => ({ output: "" }), +}); + +/** A job without a state document. */ +const Echo = defineJob("app.echo", { version: 1, args: obj({ text: str() }), result: str() }); + +/** A job with a state document and one run at a time. */ +const Shell = defineJob("app.shell", { + version: 1, + args: obj({ command: str() }), + result: obj({ exit: int() }), + state: Progress, + concurrency: 1, +}); + +/** The host-lifetime object of most tests: its log document, and the jobs the test declares. */ +const Store = local("app.store", { version: 1, params: {}, members: { log: obj({ list: arr(str()) }) } }); + +/** The same with params: it lives while referenced or running jobs. */ +const Item = local("app.item", { version: 1, params: { id: int() }, members: { log: obj({ list: arr(str()) }) } }); + +type Declared = { + readonly self: Self; + readonly log: SingletonHandle<{ list: string[] }>; + readonly job: JobHandle; + readonly errors: HostError[]; +}; + +/** Loads a host serving `Store` with the job `definition`; `implement` gets the object's `self` and log. */ +async function declared( + definition: JobDefinition, + implement: (self: Self, log: SingletonHandle<{ list: string[] }>) => JobImplementation, + options: Parameters[1] = {}, +): Promise & { readonly host: Awaited>["host"] }> { + let found: Omit, "errors"> | undefined; + const { host, errors } = await loaded( + (h) => + h.serve(Store, (_, self) => { + const log = self.state(Log); + found = { self, log, job: self.job(definition, implement(self, log)) }; + return { log }; + }), + options, + ); + return { ...found!, errors, host }; +} + +/** Appends `entry` to the log in `tx`. */ +async function append( + tx: Parameters[0]>[0], + log: SingletonHandle<{ list: string[] }>, + entry: string, +) { + (await tx.draft(log)).list.push(entry); +} + +/** A `run` that waits for `gate`, then returns `text`. */ +function gated(gate: Promise) { + return async ({ text }: { text: string }) => { + await gate; + return text; + }; +} + +/** The reported job errors, as `kind id: message`. */ +function jobReports(errors: readonly HostError[]): string[] { + return errors.flatMap((error) => { + if (error.kind === "job_failed" || error.kind === "job_finish_failed") { + return [`${error.kind}: ${(error.error as Error).message}`]; + } + return error.kind === "job_dropped" ? [`${error.kind} ${error.job}`] : []; + }); +} + +describe("start", () => { + it("checks the args against the definition (invalid/args) before anything is recorded", async () => { + const { job } = await declared(Echo, () => ({ run: async ({ text }) => text })); + // Plugin code passing wrong args (not type-checked). + const wrong = { text: 1 } as unknown as { text: string }; + expect(await rejection(job.start(wrong, context))).toMatchObject({ code: "invalid", reason: "args" }); + const extra = { text: "a", more: true } as unknown as { text: string }; + expect(await rejection(job.start(extra, context))).toMatchObject({ code: "invalid", reason: "args" }); + expect(job.running.value).toEqual([]); + }); + + it("compiles the args check once per definition (cost guard)", async () => { + const { telemetry, context: counting, counts } = await metered(); + const { job } = await declared(Echo, () => ({ run: async ({ text }) => text }), { telemetry }); + const before = counts()["chord.data.compilations"] ?? 0; + for (const text of ["a", "b", "c"]) await (await job.start({ text }, counting)).outcome(context); + expect((counts()["chord.data.compilations"] ?? 0) - before).toBeLessThanOrEqual(1); + expect(counts()["chord.data.validations"]).toBeGreaterThanOrEqual(3); + }); + + it("returns the unsettled run with the same key, across transactions, whatever its args", async () => { + const gate = Promise.withResolvers(); + const { job } = await declared(Echo, () => ({ run: gated(gate.promise) })); + // Two starts in one turn: the line serializes them, the second sees the first's record. + const [first, again] = await Promise.all([ + job.start({ text: "a" }, context, { key: "k" }), + job.start({ text: "other" }, context, { key: "k" }), + ]); + expect(again).toBe(first); + expect(await job.start({ text: "third" }, context, { key: "k" })).toBe(first); + expect(job.running.value).toHaveLength(1); + gate.resolve(); + expect(await first.outcome(context)).toMatchObject({ kind: "done", result: "a" }); + // Settled: the key names nothing any more (Q9). + const next = await job.start({ text: "b" }, context, { key: "k" }); + expect(next.id).not.toBe(first.id); + }); + + it("commits nothing for a start that only dedupes", async () => { + const gate = Promise.withResolvers(); + const { job } = await declared(Echo, () => ({ run: gated(gate.promise) })); + const run = await job.start({ text: "a" }, context, { key: "k" }); + const commits = vi.spyOn(Batch.prototype, "commit"); + try { + expect(await job.start({ text: "a" }, context, { key: "k" })).toBe(run); + expect(commits).not.toHaveBeenCalled(); + } finally { + commits.mockRestore(); + gate.resolve(); + } + }); + + it("returns the same run for a key started twice in one transaction", async () => { + const { job, self } = await declared(Echo, () => ({ run: async ({ text }) => text })); + const [one, two] = await self.tx(async (tx) => { + return [tx.start(job, { text: "a" }, { key: "k" }), tx.start(job, { text: "b" }, { key: "k" })]; + }, context); + expect(two).toBe(one); + expect((await one!.outcome(context)).args).toEqual({ text: "a" }); + }); + + describe("at the concurrency limit", () => { + fakeTimers(); + + it("rejects unavailable/busy: running and finishing runs count", async () => { + const runGate = Promise.withResolvers(); + let finishes = 0; + const { job } = await declared(Shell, () => ({ + run: async () => { + await runGate.promise; + return { exit: 0 }; + }, + finish: async () => { + // The first finish throws: the run stays finishing during the backoff, the line free. + if (finishes++ === 0) throw new Error("once"); + }, + })); + const run = await job.start({ command: "a" }, context); + const busy = { code: "unavailable", reason: "busy" }; + expect(await rejection(job.start({ command: "b" }, context))).toMatchObject(busy); + runGate.resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(job.running.value.map((entry) => entry.phase)).toEqual(["finishing"]); + expect(await rejection(job.start({ command: "b" }, context))).toMatchObject(busy); + await vi.advanceTimersByTimeAsync(1_000); + await run.outcome(context); + await (await job.start({ command: "c" }, context)).outcome(context); + }); + }); + + it("counts runs started earlier in the same transaction", async () => { + const { job, self } = await declared(Shell, () => ({ run: async () => ({ exit: 0 }) })); + const error = await rejection( + self.tx(async (tx) => { + tx.start(job, { command: "a" }); + tx.start(job, { command: "b" }); + }, context), + ); + expect(error).toMatchObject({ code: "unavailable", reason: "busy" }); + expect(job.running.value).toEqual([]); + }); + + it("lets finish chain the same job at concurrency 1: its own run is not counted", async () => { + const done: string[] = []; + const { job } = await declared(Shell, (_, log) => ({ + run: async ({ command }) => ({ exit: command.length }), + finish: async (outcome, tx) => { + done.push(outcome.args.command); + await append(tx, log, outcome.args.command); + if (outcome.args.command.length < 3) tx.start(job, { command: `${outcome.args.command}x` }); + }, + })); + await (await job.start({ command: "a" }, context)).outcome(context); + await flushed(); + await flushed(); + expect(done).toEqual(["a", "ax", "axx"]); + }); + + it("lets finish start the same job again with the same key: a new run, not the finishing one", async () => { + const gate = Promise.withResolvers(); + let successor: Run | undefined; + const { job } = await declared(Echo, () => ({ + run: async ({ text }) => { + if (text === "again") await gate.promise; + return text; + }, + finish: async (outcome, tx) => { + if (outcome.args.text === "first") successor = tx.start(job, { text: "again" }, { key: "k" }); + }, + })); + const first = await job.start({ text: "first" }, context, { key: "k" }); + expect(await first.outcome(context)).toMatchObject({ kind: "done", result: "first" }); + expect(successor!.id).not.toBe(first.id); + expect(job.running.value).toMatchObject([{ id: successor!.id, key: "k", phase: "running" }]); + // The key names the successor now. + expect(await job.start({ text: "other" }, context, { key: "k" })).toBe(successor); + gate.resolve(); + expect(await successor!.outcome(context)).toMatchObject({ kind: "done", result: "again" }); + }); + + it("tx.start commits with the transaction's other writes, both or neither", async () => { + const { job, self, log } = await declared(Echo, () => ({ run: async ({ text }) => text })); + const run = await self.tx(async (tx) => { + await append(tx, log, "requested"); + return tx.start(job, { text: "a" }); + }, context); + expect(log.value.list).toEqual(["requested"]); + expect((await run.outcome(context)).kind).toBe("done"); + let discarded: Run | undefined; + const error = await rejection( + self.tx(async (tx) => { + await append(tx, log, "lost"); + discarded = tx.start(job, { text: "b" }); + throw new Error("changed my mind"); + }, context), + ); + expect(error).toMatchObject({ message: "changed my mind" }); + expect(log.value.list).toEqual(["requested"]); + expect(job.running.value).toEqual([]); + expect(await rejection(discarded!.outcome(context))).toMatchObject({ code: "released" }); + expect(await rejection(job.outcome(discarded!.id, context))).toMatchObject({ code: "gone" }); + }); + + it("gives a short run's handle its outcome however late the caller waits", async () => { + const { job } = await declared(Echo, () => ({ run: async ({ text }) => text })); + const run = await job.start({ text: "quick" }, context); + await flushed(); + expect(job.running.value).toEqual([]); + expect(await run.outcome(context)).toEqual({ + id: run.id, + args: { text: "quick" }, + kind: "done", + result: "quick", + }); + }); + + it("waits by id for unsettled runs only: settled and unknown ids reject gone", async () => { + const gate = Promise.withResolvers(); + const { job } = await declared(Echo, () => ({ run: gated(gate.promise) })); + const run = await job.start({ text: "a" }, context); + const waiting = job.outcome(run.id, context); + gate.resolve(); + expect((await waiting).kind).toBe("done"); + expect(await rejection(job.outcome(run.id, context))).toMatchObject({ code: "gone" }); + expect(await rejection(job.outcome("nope", context))).toMatchObject({ code: "gone" }); + }); + + it("starts run after the commit, outside the batch, once the transaction settled", async () => { + const seen: string[] = []; + const { job, self, log } = await declared(Echo, (s, l) => ({ + run: async ({ text }) => { + seen.push(`run sees ${l.value.list.join(",")}`); + // Outside the batch and the line: a transaction of its own works. + await s.tx(async (tx) => append(tx, l, `from run ${text}`), context); + return text; + }, + })); + const run = await self.tx(async (tx) => { + const started = tx.start(job, { text: "a" }); + await append(tx, log, "written with the start"); + seen.push("transaction done"); + return started; + }, context); + await run.outcome(context); + expect(seen).toEqual(["transaction done", "run sees written with the start"]); + expect(log.value.list).toEqual(["written with the start", "from run a"]); + }); + + it("starts run with its signal aborted after an abort between the commit and its start", async () => { + let signalled: boolean | undefined; + const { job } = await declared(Echo, () => ({ + run: async ({ text }, ctx) => { + signalled = ctx.abortSignal?.aborted; + return text; + }, + })); + const watch = await job.running.watch(context); + // The watch's frame of the start's commit arrives before `run` starts. + watch.start(async (call) => { + if (call.kind !== "value") return; + for (const entry of call.value) job.abort(entry.id); + }); + const run = await job.start({ text: "a" }, context); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: "a" }); + expect(signalled).toBe(true); + await watch.stop(); + }); + + it("creates nothing for an aborted caller; a caller aborted after the commit does not cancel the run", async () => { + const gate = Promise.withResolvers(); + let signalled: boolean | undefined; + const { job } = await declared(Echo, () => ({ + run: async ({ text }, ctx) => { + await gate.promise; + signalled = ctx.abortSignal?.aborted; + return text; + }, + })); + const early = withCancel(context); + early.cancel(new Error("gone already")); + expect(await rejection(job.start({ text: "a" }, early.context))).toMatchObject({ message: "gone already" }); + expect(job.running.value).toEqual([]); + const caller = withCancel(context); + const run = await job.start({ text: "b" }, caller.context); + caller.cancel(new Error("caller left")); + gate.resolve(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: "b" }); + expect(signalled).toBe(false); + }); + + it("rolls back the transaction's other writes when the state's initial() fails, naming the definition", async () => { + const Bad = defineState("app.bad-state", { + schema: obj({ output: str() }), + version: 1, + // Plugin code: a wrong initial value. + initial: () => ({ output: 1 }) as unknown as { output: string }, + }); + const BadJob = defineJob("app.bad", { version: 1, args: obj({ text: str() }), result: str(), state: Bad }); + const { job, self, log } = await declared(BadJob, () => ({ run: async ({ text }) => text })); + const error = await rejection( + self.tx(async (tx) => { + await append(tx, log, "lost"); + tx.start(job, { text: "a" }); + }, context), + ); + expect(error).toMatchObject({ message: "app.bad-state: initial() does not match the schema" }); + expect({ log: log.value.list, running: job.running.value }).toEqual({ log: [], running: [] }); + }); + + it("is refused before its object is live and once it ended", async () => { + let early: Promise | undefined; + const { host } = await loaded((h) => + h.serve(Item, (_, self) => { + const log = self.state(Log); + const job = self.job(Echo, { run: async ({ text }) => text }); + self.open( + async () => { + early = job.start({ text: "a" }, context); + return null; + }, + () => {}, + ); + return { log }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + expect(await rejection(early!)).toMatchObject({ code: "not_ready" }); + item.release(); + }); +}); + +describe("self.job", () => { + it("is declared once per job name, only while the factory runs", async () => { + let later: Self | undefined; + const { host, errors } = await loaded((h) => + h.serve(Item, (_, self) => { + later = self; + self.job(Echo, { run: async ({ text }) => text }); + self.job(Echo, { run: async ({ text }) => text }); + return { log: self.state(Log) }; + }), + ); + expect(await rejection(host.get(Item, { id: 1 }, context))).toMatchObject({ + message: expect.stringContaining('job "app.echo" is declared twice'), + }); + expect(errors.map((error) => error.kind)).toEqual(["activation_failed"]); + expect(() => later!.job(Shell, { run: async () => ({ exit: 0 }) })).toThrow( + "app.item(1): self.job is valid only in the factory", + ); + }); +}); + +describe("run", () => { + it("gets a ctx with the run's id, the host's principal for a local object, and no state without one", async () => { + let seen: Record | undefined; + const host = createHost({ principal: { kind: "server" }, onError: () => {} }); + let job: JobHandle<{ text: string }, string, undefined> | undefined; + const facet = defineFacet((h) => + h.serve(Store, (_, self) => { + job = self.job(Echo, { + run: async ({ text }, ctx) => { + seen = { jobId: ctx.jobId, principal: ctx.principal, hasState: "state" in ctx }; + expectTypeOf(ctx).toEqualTypeOf>(); + return text; + }, + }); + return { log: self.state(Log) }; + }), + ); + await host.load([{ name: "@test/app", facet }], context); + const run = await job!.start({ text: "a" }, context); + await run.outcome(context); + expect(seen).toEqual({ jobId: run.id, principal: { kind: "server" }, hasState: false }); + await host.close(context); + }); + + it("reads and changes its state document; `running` carries it", async () => { + const gate = Promise.withResolvers(); + const values: string[] = []; + const { job } = await declared(Shell, () => ({ + run: async (_, ctx) => { + values.push(ctx.state.value.output); + await ctx.state.change((d) => { + d.output = "half"; + }, ctx); + values.push(ctx.state.value.output); + await gate.promise; + return { exit: 0 }; + }, + })); + const run = await job.start({ command: "a" }, context); + await flushed(); + expect(job.running.value).toEqual([ + { id: run.id, version: 1, args: { command: "a" }, state: { output: "half" }, phase: "running" }, + ]); + gate.resolve(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", state: { output: "half" } }); + expect(values).toEqual(["", "half"]); + }); + + it("keeps its state documents apart from the object's own documents of the same definition", async () => { + const gate = Promise.withResolvers(); + const Many = defineJob("app.many", { version: 1, args: obj({ n: int() }), result: int(), state: Progress }); + let mine: SingletonHandle<{ output: string }> | undefined; + let job: JobHandle<{ n: number }, number, { output: string }> | undefined; + await loaded((h) => + h.serve(Store, (_, self) => { + mine = self.state(Progress); + job = self.job(Many, { + run: async ({ n }, ctx) => { + await ctx.state.change((d) => { + d.output = `run ${n}`; + }, ctx); + await gate.promise; + return n; + }, + }); + return { log: self.state(Log) }; + }), + ); + const runs = await Promise.all([job!.start({ n: 1 }, context), job!.start({ n: 2 }, context)]); + await flushed(); + expect(job!.running.value.map((entry) => entry.phase !== "blocked" && entry.state.output)).toEqual([ + "run 1", + "run 2", + ]); + expect(mine!.value).toEqual({ output: "" }); + gate.resolve(); + expect((await Promise.all(runs.map((run) => run.outcome(context)))).map((o) => o.state.output)).toEqual([ + "run 1", + "run 2", + ]); + }); + + it("refuses state changes once run settled (released), and never recreates the retired document", async () => { + let captured: JobCtx | undefined; + let inFinish: unknown; + const { job } = await declared(Shell, () => ({ + run: async (_, ctx) => { + captured = ctx; + return { exit: 0 }; + }, + // After run returned, before finish committed: refused; the outcome's state is the state at run's settle. + finish: async (outcome) => { + inFinish = [outcome.state, await rejection(captured!.state.change(() => {}, context))]; + }, + })); + await (await job.start({ command: "a" }, context)).outcome(context); + expect(inFinish).toEqual([{ output: "" }, expect.objectContaining({ code: "released" })]); + const late = captured!.state.change((d) => { + d.output = "late"; + }, context); + expect(await rejection(late)).toMatchObject({ code: "released" }); + expect(() => captured!.state.value).toThrow(ChordError); + expect(job.running.value).toEqual([]); + }); + + it("commits a change still waiting for the line when run settles, before the outcome is recorded", async () => { + const go = Promise.withResolvers(); + const line = Promise.withResolvers(); + const entered = Promise.withResolvers(); + let returned = false; + const { job, self } = await declared(Shell, () => ({ + run: async (_, ctx) => { + await go.promise; + // Admitted while run runs; it waits behind the transaction holding the line. + void ctx.state.change((d) => { + d.output = "last words"; + }, ctx); + returned = true; + return { exit: 0 }; + }, + })); + const run = await job.start({ command: "a" }, context); + const holding = self.tx(async () => { + entered.resolve(); + await line.promise; + }, context); + await entered.promise; + go.resolve(); + await flushed(); + // run settled with its change still waiting: nothing committed, the outcome not recorded yet. + expect(returned).toBe(true); + expect(job.running.value).toMatchObject([{ phase: "running", state: { output: "" } }]); + line.resolve(); + await holding; + expect(await run.outcome(context)).toMatchObject({ kind: "done", state: { output: "last words" } }); + }); + + it("commits changes made after an abort while run stops; a throw then ends aborted, not threw", async () => { + const { job, errors } = await declared(Shell, () => ({ + run: async (_, ctx) => { + await untilAborted(ctx); + await ctx.state.change((d) => { + d.output = "stopped cleanly"; + }, ctx); + throw new Error("interrupted"); + }, + })); + const run = await job.start({ command: "a" }, context); + await flushed(); + job.abort(run.id); + expect(await run.outcome(context)).toEqual({ + id: run.id, + args: { command: "a" }, + state: { output: "stopped cleanly" }, + kind: "failed", + error: { kind: "aborted", reason: "abort" }, + }); + expect(jobReports(errors)).toEqual([]); + }); + + it("keeps a result returned after an abort", async () => { + const { job } = await declared(Echo, () => ({ + run: async ({ text }, ctx) => { + await untilAborted(ctx); + return text; + }, + })); + const run = await job.start({ text: "kept" }, context); + await flushed(); + job.abort(run.id); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: "kept" }); + }); + + it("ends threw when run throws, reported as job_failed; a plain error becomes PluginError internal", async () => { + const plugin = new PluginError("quota", "too many"); + const { job, errors } = await declared(Echo, () => ({ + run: async ({ text }) => { + if (text === "plugin") throw plugin; + throw new Error("plain"); + }, + })); + const first = await (await job.start({ text: "plugin" }, context)).outcome(context); + expect(first).toMatchObject({ kind: "failed", error: { kind: "threw", error: plugin } }); + const second = await (await job.start({ text: "plain" }, context)).outcome(context); + expect(second.kind === "failed" && second.error).toMatchObject({ kind: "threw" }); + const thrown = second.kind === "failed" && second.error.kind === "threw" ? second.error.error : undefined; + expect(thrown).toBeInstanceOf(PluginError); + expect(thrown).toMatchObject({ code: "internal", message: "plain" }); + expect(jobReports(errors)).toEqual(["job_failed: too many", "job_failed: plain"]); + expect(errors[0]).toMatchObject({ kind: "job_failed", object: "app.store", job: "app.echo", id: first.id }); + }); + + it("releases its ctx.get references when it settles", async () => { + const Other = local("app.other", { + version: 1, + params: { id: int() }, + members: { log: obj({ list: arr(str()) }) }, + }); + let job: JobHandle<{ text: string }, string, undefined> | undefined; + const { host } = await loaded((h) => { + h.serve(Other, (_, self) => ({ log: self.state(Log) })); + h.serve(Store, (_, self) => { + job = self.job(Echo, { + run: async ({ text }, ctx) => { + await ctx.get(Other, { id: 1 }); + expect(host.references(Other).value).toHaveLength(1); + return text; + }, + }); + return { log: self.state(Log) }; + }); + }); + await (await job!.start({ text: "a" }, context)).outcome(context); + expect(host.references(Other).value).toHaveLength(0); + }); + + it("that ignores its signal at the object's end is drained up to closeTimeout; its later changes are refused", async () => { + const gate = Promise.withResolvers(); + const late = Promise.withResolvers(); + const { job, host, errors } = await declared( + Shell, + () => ({ + run: async (_, ctx) => { + await gate.promise; + late.resolve( + await rejection( + ctx.state.change((d) => { + d.output = "too late"; + }, ctx), + ), + ); + return { exit: 0 }; + }, + }), + { closeTimeout: 20 }, + ); + await job.start({ command: "a" }, context); + await flushed(); + await host.close(context); + expect(errors).toEqual([{ kind: "drain_timeout", object: "app.store" }]); + gate.resolve(); + expect(await late.promise).toMatchObject({ code: "released" }); + }); + + it("may start the same job again from run (recursion): a new run", async () => { + const ids: string[] = []; + const { job } = await declared(Echo, () => ({ + run: async ({ text }, ctx) => { + ids.push(ctx.jobId); + if (text.length < 3) await job.start({ text: `${text}+` }, ctx); + return text; + }, + })); + await (await job.start({ text: "a" }, context)).outcome(context); + await flushed(); + await flushed(); + expect(new Set(ids).size).toBe(3); + }); +}); + +describe("finish", () => { + it("gets each outcome kind; the record, key and state document go in its commit, atomically with its writes", async () => { + const outcomes: string[] = []; + const { job, log, self } = await declared(Shell, (_, l) => ({ + run: async ({ command }, ctx) => { + await ctx.state.change((d) => { + d.output = `out ${command}`; + }, ctx); + if (command === "fail") throw new Error("no"); + return { exit: 0 }; + }, + finish: async (outcome, tx) => { + outcomes.push(`${outcome.kind} ${outcome.state.output}`); + await append(tx, l, outcome.args.command); + }, + })); + // The log entry and the removed entry arrive in one commit: whenever the log's watch sees an entry, the run is gone. + const seen: [number, number][] = []; + (await log.watch(context)).start(async (call) => { + if (call.kind === "value") seen.push([call.value.list.length, job.running.value.length]); + }); + void self; + await (await job.start({ command: "ok" }, context, { key: "k" })).outcome(context); + await (await job.start({ command: "fail" }, context, { key: "k" })).outcome(context); + await delivered(); + expect(outcomes).toEqual(["done out ok", "failed out fail"]); + expect(seen).toEqual([ + [1, 0], + [2, 0], + ]); + expect(job.running.value).toEqual([]); + }); + + it("only cleans up without a finish", async () => { + const { job } = await declared(Shell, () => ({ run: async () => ({ exit: 3 }) })); + const run = await job.start({ command: "a" }, context, { key: "k" }); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { exit: 3 } }); + expect(job.running.value).toEqual([]); + }); + + describe("throwing", () => { + fakeTimers(); + + it("is retried after 1 s doubling, each failure reported; a later success finishes the run", async () => { + const at: number[] = []; + const { job, errors } = await declared(Echo, () => ({ + run: async ({ text }) => text, + finish: async () => { + at.push(Date.now()); + if (at.length < 4) throw new Error(`finish ${at.length}`); + }, + })); + const run = await job.start({ text: "a" }, context); + await vi.advanceTimersByTimeAsync(10_000); + expect(at).toEqual([0, 1_000, 3_000, 7_000]); + expect(jobReports(errors)).toEqual([ + "job_finish_failed: finish 1", + "job_finish_failed: finish 2", + "job_finish_failed: finish 3", + ]); + expect((await run.outcome(context)).kind).toBe("done"); + }); + + it("blocks the run after five failures in a row: outcome, state and key kept, not counted; abort drops it", async () => { + const { job, errors } = await declared(Shell, () => ({ + run: async (_, ctx) => { + await ctx.state.change((d) => { + d.output = "kept"; + }, ctx); + return { exit: 0 }; + }, + finish: async () => { + throw new Error("always"); + }, + })); + const run = await job.start({ command: "a" }, context, { key: "k" }); + const { context: waiting, cancel } = withCancel(context); + const waiter = rejection(run.outcome(waiting)); + const dropped = rejection(run.outcome(context)); + await vi.advanceTimersByTimeAsync(60_000); + expect(jobReports(errors)).toHaveLength(5); + expect(job.running.value).toEqual([{ id: run.id, version: 1, phase: "blocked", reason: "finish_failed" }]); + // Its waiters keep waiting; their own context ends a wait. + cancel(new Error("gave up")); + expect(await waiter).toMatchObject({ message: "gave up" }); + // Dedupe returns it; a different key starts a new run (blocked runs do not count for concurrency). + expect(await job.start({ command: "b" }, context, { key: "k" })).toBe(run); + const other = await job.start({ command: "c" }, context, { key: "other" }); + await vi.advanceTimersByTimeAsync(0); + expect(job.running.value.map((entry) => entry.phase)).toEqual(["blocked", "finishing"]); + job.abort(run.id); + await vi.advanceTimersByTimeAsync(0); + expect(await dropped).toMatchObject({ code: "aborted" }); + expect(job.running.value.map((entry) => entry.id)).toEqual([other.id]); + expect(errors.filter((error) => error.kind === "job_dropped")).toEqual([ + { kind: "job_dropped", object: "app.store", job: "app.shell", id: run.id }, + ]); + }); + }); + + describe("backing off", () => { + fakeTimers(); + + it("leaves no timer behind when its object ends during the backoff", async () => { + const { job, host, errors } = await declared(Echo, () => ({ + run: async ({ text }) => text, + finish: async () => { + throw new Error("stuck"); + }, + })); + const run = rejection((await job.start({ text: "a" }, context)).outcome(context)); + await vi.advanceTimersByTimeAsync(0); + expect(errors).toHaveLength(1); + expect(vi.getTimerCount()).toBe(1); + await host.close(context); + expect(vi.getTimerCount()).toBe(0); + await vi.advanceTimersByTimeAsync(60_000); + expect(errors).toHaveLength(1); + expect(await run).toMatchObject({ code: "released" }); + }); + }); + + it("ignores an abort while it runs", async () => { + const gate = Promise.withResolvers(); + const { job } = await declared(Echo, () => ({ + run: async ({ text }) => text, + finish: async () => gate.promise, + })); + const run = await job.start({ text: "a" }, context); + await flushed(); + job.abort(run.id); + gate.resolve(); + expect((await run.outcome(context)).kind).toBe("done"); + }); + + it("does not run when its object ends while the outcome and finish wait for the line: no call, no report", async () => { + const go = Promise.withResolvers(); + const line = Promise.withResolvers(); + const entered = Promise.withResolvers(); + const finished: string[] = []; + const { job, self, host, errors } = await declared(Echo, () => ({ + run: async ({ text }) => { + await go.promise; + return text; + }, + finish: async ({ args }) => void finished.push(args.text), + })); + const run = await job.start({ text: "a" }, context); + const waiting = rejection(run.outcome(context)); + const holding = self.tx(async () => { + entered.resolve(); + await line.promise; + }, context); + await entered.promise; + // run settles while the line is held: its outcome and finish queue behind it. + go.resolve(); + await flushed(); + const closing = host.close(context); + line.resolve(); + await holding.catch(() => {}); + await closing; + expect({ finished, errors }).toEqual({ finished: [], errors: [] }); + expect(await waiting).toMatchObject({ code: "released" }); + }); + + it("is drained at the object's end when it runs: its commit is refused, nothing is reported", async () => { + const gate = Promise.withResolvers(); + const entered = Promise.withResolvers(); + const events: string[] = []; + const { job, host, errors } = await declared(Echo, () => ({ + run: async ({ text }) => text, + finish: async () => { + entered.resolve(); + await gate.promise; + events.push("finish returned"); + }, + })); + await job.start({ text: "a" }, context); + await entered.promise; + const closing = host.close(context).then(() => void events.push("closed")); + await flushed(); + expect(events).toEqual([]); + gate.resolve(); + await closing; + expect(events).toEqual(["finish returned", "closed"]); + expect(errors).toEqual([]); + }); +}); + +describe("running", () => { + it("has one entry per unsettled run, in start order, tagged by phase; a key when given", async () => { + const gates = [Promise.withResolvers(), Promise.withResolvers()]; + const finishing = Promise.withResolvers(); + const { job } = await declared(Echo, () => ({ + run: async ({ text }) => { + await gates[Number(text)]!.promise; + return text; + }, + finish: async () => finishing.promise, + })); + const zero = await job.start({ text: "0" }, context, { key: "z" }); + const one = await job.start({ text: "1" }, context); + gates[0]!.resolve(); + await flushed(); + expect(job.running.value).toEqual([ + { id: zero.id, version: 1, key: "z", args: { text: "0" }, phase: "finishing" }, + { id: one.id, version: 1, args: { text: "1" }, phase: "running" }, + ]); + finishing.resolve(); + gates[1]!.resolve(); + await one.outcome(context); + expect(job.running.value).toEqual([]); + }); + + it("replaces only the entry whose state changed, with ops under its entry (cost guard: no walk of other runs)", async () => { + const gate = Promise.withResolvers(); + const contexts = new Map>(); + const Many = defineJob("app.many", { version: 1, args: obj({ n: int() }), result: int(), state: Progress }); + const { job } = await declared(Many, () => ({ + run: async ({ n }, ctx) => { + contexts.set(String(n), ctx); + await gate.promise; + return n; + }, + })); + for (let n = 0; n < 100; n++) await job.start({ n }, context); + await flushed(); + const before = job.running.value; + const frames = framesOf(await job.running.watch(context)); + await contexts.get("42")!.state.change((d) => { + d.output = "x"; + }, context); + await delivered(); + const after = job.running.value; + expect(after).not.toBe(before); + expect(after.filter((entry, index) => entry !== before[index])).toHaveLength(1); + expect(after[42]).toMatchObject({ state: { output: "x" } }); + expect(frames.map(([, ops]) => ops)).toEqual([[["a", [42, "state", "output"], "x"]] satisfies Op[]]); + gate.resolve(); + }); + + it("does no validation, walk or compilation for a state change (cost guard)", async () => { + const { telemetry, context: counting, counts } = await metered(); + const gate = Promise.withResolvers(); + const contexts: JobCtx[] = []; + const { job } = await declared( + Shell, + () => ({ + run: async (_, ctx) => { + contexts.push(ctx); + await gate.promise; + return { exit: 0 }; + }, + }), + { telemetry, context: counting }, + ); + await job.start({ command: "a" }, counting); + await flushed(); + const frames = framesOf(await job.running.watch(context)); + const data = () => Object.entries(counts()).filter(([name]) => name.startsWith("chord.data.")); + const before = data(); + await contexts[0]!.state.change((d) => { + d.output = "x".repeat(10_000); + }, context); + await delivered(); + expect(data()).toEqual(before); + expect(frames).toHaveLength(1); + gate.resolve(); + }); + + it("frames record changes as splices and sets of the list", async () => { + const gate = Promise.withResolvers(); + const { job } = await declared(Echo, () => ({ run: gated(gate.promise) })); + const frames = framesOf(await job.running.watch(context)); + const run = await job.start({ text: "a" }, context); + await delivered(); + gate.resolve(); + await run.outcome(context); + await delivered(); + const entry = { id: run.id, version: 1, args: { text: "a" }, phase: "running" }; + expect(frames.map(([, ops]) => ops)).toEqual([ + [["p", [], 0, 0, [entry]]], + [["s", [0], { ...entry, phase: "finishing" }]], + [["p", [], 0, 1, []]], + ]); + }); +}); + +describe("keep-alive", () => { + it("keeps an object without holders while runs run, and ends it after the last one and its linger", async () => { + const gate = Promise.withResolvers(); + let job: JobHandle<{ text: string }, string, undefined> | undefined; + let ended = false; + const { host } = await loaded((h) => + h.serve( + Item, + (_, self) => { + job = self.job(Echo, { run: gated(gate.promise) }); + self.onLive(async (ctx) => { + await untilAborted(ctx); + ended = true; + }); + return { log: self.state(Log) }; + }, + { linger: 20 }, + ), + ); + const item = await host.get(Item, { id: 1 }, context); + await job!.start({ text: "a" }, context); + expect(host.activity.value.jobs).toBe(1); + expect(host.references(Item).value).toHaveLength(1); + item.release(); + await new Promise((resolve) => setTimeout(resolve, 40)); + expect(ended).toBe(false); + expect(host.references(Item).value).toHaveLength(0); + expect(host.activity.value).toMatchObject({ jobs: 1, lingering: 0 }); + gate.resolve(); + await flushed(); + expect(host.activity.value).toMatchObject({ jobs: 0, lingering: 1 }); + await new Promise((resolve) => setTimeout(resolve, 40)); + expect(ended).toBe(true); + expect(host.activity.value).toMatchObject({ jobs: 0, lingering: 0 }); + }); + + it("keeps an object whose last run's finish chains another job: the chained run runs, then the object ends", async () => { + const A = defineJob("app.a", { version: 1, args: obj({ text: str() }), result: str() }); + const events: string[] = []; + let a: JobHandle<{ text: string }, string, undefined> | undefined; + let chained: Run | undefined; + const { host } = await loaded((h) => + h.serve(Item, (_, self) => { + self.open( + async () => null, + () => void events.push("object ended"), + ); + const b = self.job(Echo, { + run: async ({ text }) => { + await new Promise((resolve) => setTimeout(resolve, 10)); + events.push(`b ran ${text}`); + return text; + }, + }); + a = self.job(A, { + run: async ({ text }) => text, + finish: async (_outcome, tx) => { + chained = tx.start(b, { text: "chained" }); + }, + }); + return { log: self.state(Log) }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + await a!.start({ text: "x" }, context); + item.release(); + await new Promise((resolve) => setTimeout(resolve, 40)); + expect(await chained!.outcome(context)).toMatchObject({ kind: "done", result: "chained" }); + expect(events).toEqual(["b ran chained", "object ended"]); + expect(host.activity.value.jobs).toBe(0); + }); + + it("keeps an object whose last reference goes while its start's commit publishes (a link closing as it sends)", async () => { + const View = remote("app.view", { + version: 1, + params: {}, + variants: { v: { state: { running: arr(unknown()) } } }, + }); + let job: JobHandle<{ text: string }, string, undefined> | undefined; + let ran = false; + const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const facet = defineFacet((h) => + h.serve(View, (_, _principal, self) => { + job = self.job(Echo, { + run: async ({ text }) => { + ran = true; + return text; + }, + }); + // The running list as a state member, so the start's commit sends a state message. + return { variant: "v", members: { running: job.running as never } }; + }), + ); + await server.load([{ name: "@test/app", facet }], context); + const client = createHost({ onError: () => {} }); + await client.load([], context); + const [a, b] = memoryLinkPair(); + let closeOnSend = false; + // The server's end of the link closes while it sends: the client's hold, the last reference, goes at once. + const failing: Link = { + send: (message) => { + if (!closeOnSend) return a.send(message); + closeOnSend = false; + a.close(); + }, + onMessage: (listener) => a.onMessage(listener), + close: () => a.close(), + onClose: (listener) => a.onClose(listener), + }; + server.accept(failing, { principal: { kind: "owner" } }, context); + client.connect(b, { principal: { kind: "server" } }, context); + await client.get(View, {}, context); + await transported(); + closeOnSend = true; + const run = await job!.start({ text: "a" }, context); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: "a" }); + expect(ran).toBe(true); + await client.close(context); + await server.close(context); + }); + + it("drops blocked runs, reported, when the object ends after its last run", async () => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "Date"], now: 0 }); + try { + let job: JobHandle<{ text: string }, string, undefined> | undefined; + const errors: HostError[] = []; + const host = createHost({ onError: (error) => errors.push(error) }); + const facet = defineFacet((h) => + h.serve(Item, (_, self) => { + job = self.job(Echo, { + run: async ({ text }) => text, + finish: async () => { + throw new Error("stuck"); + }, + }); + return { log: self.state(Log) }; + }), + ); + await host.load([{ name: "@test/app", facet }], context); + const item = await host.get(Item, { id: 1 }, context); + const run = await job!.start({ text: "a" }, context); + item.release(); + await vi.advanceTimersByTimeAsync(60_000); + expect(errors.map((error) => error.kind)).toEqual([...Array(5).fill("job_finish_failed"), "job_dropped"]); + expect(await rejection(run.outcome(context))).toMatchObject({ code: "released" }); + expect(host.activity.value.jobs).toBe(0); + } finally { + vi.useRealTimers(); + } + }); + + it("keeps a lingering object when a run starts during its linger", async () => { + const gate = Promise.withResolvers(); + let job: JobHandle<{ text: string }, string, undefined> | undefined; + const { host } = await loaded((h) => + h.serve( + Item, + (_, self) => { + job = self.job(Echo, { run: gated(gate.promise) }); + return { log: self.state(Log) }; + }, + { linger: 20 }, + ), + ); + (await host.get(Item, { id: 1 }, context)).release(); + expect(host.activity.value.lingering).toBe(1); + const run = await job!.start({ text: "a" }, context); + expect(host.activity.value).toMatchObject({ jobs: 1, lingering: 0 }); + await new Promise((resolve) => setTimeout(resolve, 40)); + gate.resolve(); + expect((await run.outcome(context)).kind).toBe("done"); + }); +}); + +describe("object end", () => { + type Revocation = { session: string }; + const Jobs = remote("app.jobs", { version: 1, params: {}, variants: { v: { state: {} } } }); + + it("revocation (R11): runs' signals fire with the end's error, the drain waits for them, no finish", async () => { + let job: JobHandle<{ text: string }, string, undefined> | undefined; + const finished: string[] = []; + let reason: unknown; + const server = createHost({ + principal: { kind: "server" }, + revokedBy: (p) => (p.kind === "guest" ? [{ session: p.session }] : []), + onError: () => {}, + }); + const facet = defineFacet((h) => + h.serve(Jobs, (_, _principal, self) => { + job = self.job(Echo, { + run: async ({ text }, ctx) => { + await untilAborted(ctx); + reason = ctx.abortSignal?.reason; + await new Promise((resolve) => setTimeout(resolve, 10)); + finished.push("run stopped"); + return text; + }, + finish: async () => void finished.push("finish"), + }); + return { variant: "v", members: {} }; + }), + ); + await server.load([{ name: "@test/app", facet }], context); + const client = createHost({ onError: () => {} }); + await client.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "guest", session: "S1" } }, context); + client.connect(b, { principal: { kind: "server" } }, context); + await client.get(Jobs, {}, context); + const run = await job!.start({ text: "a" }, context); + await flushed(); + server.revoke({ session: "S1" }, context); + await flushed(); + expect(reason).toMatchObject({ code: "forbidden", reason: "revoked" }); + expect(await rejection(run.outcome(context))).toMatchObject({ code: "forbidden", reason: "revoked" }); + await new Promise((resolve) => setTimeout(resolve, 30)); + await transported(); + expect(finished).toEqual(["run stopped"]); + expect(server.activity.value.jobs).toBe(0); + await client.close(context); + await server.close(context); + }); + + it("drains a run that ends its own object before its first await: its natives stay open until it stops", async () => { + let job: JobHandle<{ text: string }, string, undefined> | undefined; + const gate = Promise.withResolvers(); + const events: string[] = []; + const server = createHost({ + principal: { kind: "server" }, + revokedBy: (p) => (p.kind === "guest" ? [{ session: p.session }] : []), + onError: () => {}, + }); + const facet = defineFacet((h) => + h.serve(Jobs, (_, _principal, self) => { + self.open( + async () => null, + () => void events.push("native closed"), + ); + job = self.job(Echo, { + run: async ({ text }) => { + server.revoke({ session: "S1" }, context); + await gate.promise; + events.push("run stopped"); + return text; + }, + }); + return { variant: "v", members: {} }; + }), + ); + await server.load([{ name: "@test/app", facet }], context); + const client = createHost({ onError: () => {} }); + await client.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "guest", session: "S1" } }, context); + client.connect(b, { principal: { kind: "server" } }, context); + await client.get(Jobs, {}, context); + const run = rejection((await job!.start({ text: "a" }, context)).outcome(context)); + await flushed(); + expect(events).toEqual([]); + gate.resolve(); + await flushed(); + expect(events).toEqual(["run stopped", "native closed"]); + expect(await run).toMatchObject({ code: "forbidden", reason: "revoked" }); + await client.close(context); + await server.close(context); + }); + + it("host.close: signals fire, close waits for the run (drain), no finish, waiters rejected released", async () => { + const finished: string[] = []; + const { job, host } = await declared(Echo, () => ({ + run: async ({ text }, ctx) => { + await untilAborted(ctx); + await new Promise((resolve) => setTimeout(resolve, 10)); + finished.push("run stopped"); + return text; + }, + finish: async () => void finished.push("finish"), + })); + const run = await job.start({ text: "a" }, context); + await flushed(); + const waiting = rejection(run.outcome(context)); + await host.close(context); + expect(finished).toEqual(["run stopped"]); + expect(await waiting).toMatchObject({ code: "released" }); + }); +}); + +describe("GC", () => { + it("finished and dropped runs leave nothing on their object", async () => { + const refs: WeakRef[] = []; + const { job } = await declared(Echo, () => ({ + run: async (args, ctx) => { + refs.push(new WeakRef(args), new WeakRef(ctx)); + return args.text; + }, + })); + await (await job.start({ text: "a" }, context)).outcome(context); + await flushed(); + await expectCollected(...refs); + }); + + describe("of dropped runs", () => { + fakeTimers(); + + it("a dropped blocked run leaves nothing on its object", async () => { + const refs: WeakRef[] = []; + const { job, errors } = await declared(Echo, () => ({ + run: async (args, ctx) => { + refs.push(new WeakRef(args), new WeakRef(ctx)); + return args.text; + }, + finish: async () => { + throw new Error("stuck"); + }, + })); + const run = await job.start({ text: "a" }, context); + await vi.advanceTimersByTimeAsync(60_000); + expect(job.running.value.map((entry) => entry.phase)).toEqual(["blocked"]); + const dropped = run.outcome(context).then( + () => new Error("expected a rejection"), + (error: Error) => error, + ); + job.abort(run.id); + await vi.advanceTimersByTimeAsync(0); + expect(await dropped).toMatchObject({ code: "aborted" }); + expect(job.running.value).toEqual([]); + // An error keeps its stack frames' receivers (the dropping transaction) until its stack is read (V8). + void (await dropped).stack; + for (const error of errors) if (error.kind === "job_finish_failed") void (error.error as Error).stack; + await expectCollected(...refs); + }); + }); + + it("keeps nothing of a cancelled wait while its run still runs", async () => { + const gate = Promise.withResolvers(); + const { job } = await declared(Echo, () => ({ run: gated(gate.promise) })); + const run = await job.start({ text: "a" }, context); + /** One wait, cancelled; returns its signal, weakly. */ + const cancelled = async (): Promise> => { + const controller = new AbortController(); + const waiting = rejection(run.outcome(withAbortSignal(controller.signal, context))); + controller.abort("gave up"); + expect(await waiting).toBe("gave up"); + return new WeakRef(controller.signal); + }; + const refs: WeakRef[] = []; + for (let i = 0; i < 5; i++) refs.push(await cancelled()); + await expectCollected(...refs); + gate.resolve(); + expect((await run.outcome(context)).kind).toBe("done"); + }); + + it("keeps no ended run's args or state in a job handle kept after its object ended", async () => { + const refs: WeakRef[] = []; + const { job, host } = await declared(Shell, () => ({ + run: async (args, ctx) => { + refs.push(new WeakRef(args), new WeakRef(ctx.state.value)); + await untilAborted(ctx); + return { exit: 1 }; + }, + })); + await job.start({ command: "a" }, context); + await flushed(); + await host.close(context); + await expectCollected(...refs); + expect(() => job.running.value).toThrow(ChordError); + }); + + it("an ended object releases its runs, their documents and waiters", async () => { + const refs: WeakRef[] = []; + let job: JobHandle<{ command: string }, { exit: number }, { output: string }> | undefined; + const { host } = await loaded((h) => + h.serve(Item, (_, self) => { + job = self.job(Shell, { + run: async (args, ctx) => { + refs.push(new WeakRef(args), new WeakRef(ctx), new WeakRef(ctx.state.value)); + await untilAborted(ctx); + return { exit: 1 }; + }, + }); + return { log: self.state(Log) }; + }), + ); + const item = await host.get(Item, { id: 1 }, context); + await job!.start({ command: "a" }, context); + await flushed(); + refs.push(new WeakRef(job!)); + job = undefined; + await host.close(context); + void item; + await expectCollected(...refs); + }); +}); + +// The type of `outcome` follows the definition. +expectTypeOf["args"]>().toEqualTypeOf<{ + command: string; +}>(); +void (null as unknown as Context); diff --git a/packages/chord/test/next/jobs/records.test.ts b/packages/chord/test/next/jobs/records.test.ts new file mode 100644 index 000000000..736791615 --- /dev/null +++ b/packages/chord/test/next/jobs/records.test.ts @@ -0,0 +1,205 @@ +// Tests of a job's records on its object (slice 6a §1.2, §15 rule 5): runs across attempts, cut off by an attempt's +// seal, finished on the next live attempt, and the keep-alive of runs waiting for one. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import { untilAborted } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { defineJob, type JobHandle, type Outcome, type RunningEntry } from "../../../src/next/jobs/index.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import { delivered, flushed, instancesOf, memoryStore, rejection, restartable, storeRestartable } from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +const Progress = defineState("app.progress", { + schema: obj({ output: str() }), + version: 1, + initial: () => ({ output: "" }), +}); + +const Work = defineJob("app.work", { version: 1, args: obj({ text: str() }), result: str(), state: Progress }); + +type WorkHandle = JobHandle<{ text: string }, string, { output: string }>; + +/** What a test's attempts do with a run: `run` waits for its abort (or `release`), `finish` records by attempt. */ +async function working( + finish: (attempt: number, outcome: Outcome<{ text: string }, string, { output: string }>) => void | Promise, +) { + const jobs: WorkHandle[] = []; + const object = await restartable( + objects, + (attempt, self) => { + jobs.push( + self.job(Work, { + run: async ({ text }, ctx) => { + await ctx.state.change((draft) => { + draft.output = `${text} started`; + }, ctx); + await untilAborted(ctx); + return text; + }, + finish: (outcome) => finish(attempt, outcome), + }), + ); + }, + { closeTimeout: 50 }, + ); + return { ...object, jobs }; +} + +describe("runs across attempts", () => { + it("a run cut off by its attempt's seal ends aborted (restart) and is finished on the next live attempt", async () => { + const finished: string[] = []; + const { jobs, fail, retry } = await working((attempt, outcome) => { + finished.push(`${attempt}: ${outcome.kind} ${JSON.stringify(outcome)}`); + }); + const run = await jobs[0]!.start({ text: "a" }, context); + await flushed(); + fail(); + await flushed(); + // No live attempt: the run waits, finishing. + expect(finished).toEqual([]); + expect((jobs[0]!.running.value as readonly RunningEntry[]).map(({ phase }) => phase)).toEqual([ + "finishing", + ]); + await retry(); + const outcome = await run.outcome(context); + expect(outcome).toEqual({ + id: run.id, + args: { text: "a" }, + state: { output: "a started" }, + kind: "failed", + error: { kind: "aborted", reason: "restart" }, + }); + expect(finished).toEqual([`2: failed ${JSON.stringify(outcome)}`]); + expect(jobs[1]!.running.value).toEqual([]); + }); + + it("refuses the cut run's changes made while it stops; its settling records nothing more", async () => { + const late = Promise.withResolvers(); + const jobs: WorkHandle[] = []; + const { restart } = await restartable(objects, (_, self) => { + const job = self.job(Work, { + run: async (_args, ctx) => { + await untilAborted(ctx); + // Accepted after an abort (§15), refused once the attempt's seal cut the run off. + await ctx.state + .change((draft) => { + draft.output = "late"; + }, context) + .then(late.resolve, late.resolve); + return "done"; + }, + }); + jobs.push(job); + }); + const run = await jobs[0]!.start({ text: "a" }, context); + await flushed(); + await restart(); + const outcome = await run.outcome(context); + expect(await late.promise).toMatchObject({ code: "released" }); + expect(outcome).toMatchObject({ kind: "failed", error: { reason: "restart" }, state: { output: "" } }); + await flushed(); + expect(jobs[1]!.running.value).toEqual([]); + }); + + it("a finish sealed mid-way is not counted as a failure and runs again on the next live attempt", async () => { + const gate = Promise.withResolvers(); + const attempts: number[] = []; + const { jobs, restart, errors } = await working(async (attempt) => { + attempts.push(attempt); + if (attempt === 1) await gate.promise; + }); + const run = await jobs[0]!.start({ text: "a" }, context); + await flushed(); + jobs[0]!.abort(run.id); + await flushed(); + expect(attempts).toEqual([1]); + await restart(); + gate.resolve(); + // `run` returns its result after the abort: a success. + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: "a" }); + expect(attempts).toEqual([1, 2]); + // No job_finish_failed: the stuck finish only outlived the drain. + expect(errors.map(({ kind }) => kind)).toEqual(["object_failed", "drain_timeout"]); + }); +}); + +describe("keep-alive", () => { + it("a run waiting for a live attempt keeps a failed object no longer: it ends with its last reference", async () => { + const { jobs, fail, reference } = await working(() => {}); + const run = await jobs[0]!.start({ text: "a" }, context); + await flushed(); + fail(); + reference.release(); + await delivered(); + expect(await rejection(run.outcome(context))).toMatchObject({ code: "released" }); + }); +}); + +describe("the object's end during an admitted commit", () => { + const Saved = defineState("app.saved", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: 0 }), + persist: true, + checkpointWhen: () => false, + }); + + it("a run started in a commit held in the SQL while the host closes ends with the object; no run is left counted", async () => { + const store = await memoryStore(); + const declared: { saved: SingletonHandle<{ n: number }>; job: WorkHandle }[] = []; + const { host, attempts } = await storeRestartable( + objects, + (_, self) => { + const job = self.job(Work, { + run: async (_args, ctx) => { + await untilAborted(ctx); + return ""; + }, + finish: () => {}, + }); + declared.push({ saved: self.state(Saved), job }); + }, + { store }, + ); + const { saved, job } = declared[0]!; + // The next commit's step waits inside the backend transaction, after its admission. + const gate = Promise.withResolvers(); + const inStep = Promise.withResolvers(); + const commit = store.commit.bind(store); + store.commit = (writes, fence, ctx) => { + const held: StagedWrite = { + order: "write", + apply: async () => { + inStep.resolve(); + await gate.promise; + return { kind: "dropIndex", table: "x", name: "y" }; + }, + }; + return commit([...writes, held], fence, ctx); + }; + const started = attempts[0]!.tx(async (tx) => { + (await tx.draft(saved)).n = 1; + return tx.start(job, { text: "a" }); + }, context); + await inStep.promise; + let closed = false; + const closing = host.close(context).then(() => { + closed = true; + }); + await delivered(); + // The object's teardown waits for the admitted step, so the run's record is adopted before the end. + expect(closed).toBe(false); + gate.resolve(); + const run = await started; + await closing; + store.commit = commit; + expect(await rejection(run.outcome(context))).toMatchObject({ code: "released" }); + expect(host.activity.value.jobs).toBe(0); + expect(store.lastSeq).toBe(1); + await store.close(context); + }); +}); diff --git a/packages/chord/test/next/jobs/run.test.ts b/packages/chord/test/next/jobs/run.test.ts new file mode 100644 index 000000000..0899a33f7 --- /dev/null +++ b/packages/chord/test/next/jobs/run.test.ts @@ -0,0 +1,49 @@ +// Tests of one run's record (§15): its `running` entry per phase, and its handle's wait for the outcome. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import { JobRun } from "../../../src/next/jobs/run.ts"; +import { rejection } from "../support.ts"; + +describe("a run", () => { + it("has an entry per phase: args and state while running or finishing, neither when blocked", () => { + const run = new JobRun("r1", 2, "k", { command: "ls" }); + expect(run.entryFor({ output: "" })).toEqual({ + id: "r1", + version: 2, + key: "k", + args: { command: "ls" }, + state: { output: "" }, + phase: "running", + }); + run.phase = "finishing"; + expect(new JobRun("r2", 1, undefined, { a: 1 }).entryFor(undefined)).toEqual({ + id: "r2", + version: 1, + args: { a: 1 }, + phase: "running", + }); + run.phase = "blocked"; + expect(run.entryFor({ output: "x" })).toEqual({ + id: "r1", + version: 2, + phase: "blocked", + reason: "finish_failed", + }); + }); + + it("gives every waiter the outcome, late ones too; a waiter's context ends only its wait", async () => { + const run = new JobRun("r1", 1, undefined, {}); + const { context: waiting, cancel } = withCancel(context); + const cancelled = rejection(run.handle.outcome(waiting)); + cancel(new Error("left")); + expect(await cancelled).toMatchObject({ message: "left" }); + const outcome = { id: "r1", args: {}, state: undefined, kind: "done", result: 1 } as const; + run.resolve(outcome); + expect(await run.handle.outcome(context)).toBe(outcome); + const rejected = new JobRun("r2", 1, undefined, {}); + rejected.reject(new ChordError({ code: "aborted", message: "dropped" })); + expect(await rejection(rejected.handle.outcome(context))).toMatchObject({ code: "aborted" }); + }); +}); diff --git a/packages/chord/test/next/jobs/running.test.ts b/packages/chord/test/next/jobs/running.test.ts new file mode 100644 index 000000000..6f153ec1f --- /dev/null +++ b/packages/chord/test/next/jobs/running.test.ts @@ -0,0 +1,124 @@ +// Tests of `running`'s list (§15): each change copies the list, keeps the other entries identical and gives the ops +// of exactly that change (a state change re-pathed under its entry); one frame per batch; derives over it are marked. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { applyImmutable, type Op } from "../../../src/delta/index.ts"; +import type { JsonValue } from "../../../src/json.ts"; +import { Running } from "../../../src/next/jobs/running.ts"; +import { Batch, Derive } from "../../../src/next/state/internal.ts"; +import { delivered, framesOf } from "../support.ts"; + +function running() { + const batch = new Batch(() => {}, undefined); + return { batch, list: new Running(batch, "app.store job app.echo") }; +} + +describe("running's list", () => { + it("copies the list per change, keeps other entries identical, and frames apply to the previous value", async () => { + const { batch, list } = running(); + const a = { id: "a", phase: "running", state: { output: "" } }; + const b = { id: "b", phase: "running", state: { output: "" } }; + const watch = await list.watch(context); + const frames = framesOf(watch); + let previous = list.value; + batch.commit(() => { + list.add(a); + list.add(b); + }, context); + await delivered(); + const added = list.value as JsonValue[]; + expect(added).toEqual([a, b]); + expect(added).not.toBe(previous); + const b2 = { ...b, state: { output: "x" } }; + batch.commit(() => list.stateChanged(b, b2, [["a", ["output"], "x"]]), context); + await delivered(); + const a2 = { ...a, phase: "finishing" }; + batch.commit(() => list.replace(a, a2), context); + await delivered(); + batch.commit(() => list.remove(a2), context); + await delivered(); + expect(list.value).toEqual([b2]); + expect((list.value as JsonValue[])[0]).toBe(b2); + expect(frames.map(([, ops]) => ops)).toEqual([ + [ + ["p", [], 0, 0, [a]], + ["p", [], 1, 0, [b]], + ], + [["a", [1, "state", "output"], "x"]], + [["s", [0], a2]], + [["p", [], 0, 1, []]], + ] satisfies Op[][]); + // Every frame turns the previous value into the next. + previous = []; + for (const [value, ops] of frames) { + expect(applyImmutable(previous, ops)).toEqual(value); + previous = value; + } + }); + + it("re-paths every kind of state op under its entry; a replacement becomes a set", async () => { + const { batch, list } = running(); + const entry = { id: "a", state: { list: [1], text: "", n: 0 } }; + batch.commit(() => list.add(entry), context); + const frames = framesOf(await list.watch(context)); + const after = { id: "a", state: { list: [2], text: "t", n: 1 } }; + const ops: Op[] = [ + ["r", { list: [], text: "", n: 0 }], + ["s", ["n"], 1], + ["p", ["list"], 0, 0, [2]], + ["a", ["text"], "t"], + ["d", ["gone"]], + ["t", ["text"], 1], + ["m", ["list"], [0]], + ]; + batch.commit(() => list.stateChanged(entry, after, ops), context); + await delivered(); + expect(frames[0]![1]).toEqual([ + ["s", [0, "state"], { list: [], text: "", n: 0 }], + ["s", [0, "state", "n"], 1], + ["p", [0, "state", "list"], 0, 0, [2]], + ["a", [0, "state", "text"], "t"], + ["d", [0, "state", "gone"]], + ["t", [0, "state", "text"], 1], + ["m", [0, "state", "list"], [0]], + ]); + }); + + it("builds no op for a state change while unwatched (cost guard: the ops are not even walked)", () => { + const { batch, list } = running(); + const entry = { id: "a", state: { output: "" } }; + batch.commit(() => list.add(entry), context); + const unwalked = { + [Symbol.iterator]: () => { + throw new Error("walked"); + }, + } as unknown as Op[]; + const after = { id: "a", state: { output: "x" } }; + batch.commit(() => list.stateChanged(entry, after, unwalked), context); + expect(list.value).toEqual([after]); + }); + + it("keeps nothing once ended: reads throw released", async () => { + const { batch, list } = running(); + batch.commit(() => list.add({ id: "a" }), context); + const watch = await list.watch(context); + list.end(); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(() => list.value).toThrow("app.store job app.echo: ended"); + }); + + it("records no ops while unwatched, and marks derives over it at every change", () => { + const { batch, list } = running(); + const count = new Derive([list], ([value]) => (value as JsonValue[]).length, batch); + count.attach( + (source) => source as Running, + () => {}, + ); + expect(count.value).toBe(0); + batch.commit(() => list.add({ id: "a" }), context); + expect(count.value).toBe(1); + batch.commit(() => list.add({ id: "b" }), context); + expect(count.value).toBe(2); + }); +}); diff --git a/packages/chord/test/next/link/channel.test.ts b/packages/chord/test/next/link/channel.test.ts new file mode 100644 index 000000000..f5abbe8eb --- /dev/null +++ b/packages/chord/test/next/link/channel.test.ts @@ -0,0 +1,342 @@ +// Tests of a host's end of a link: the handshake, checks, the heartbeat, trusted links, and in-order handling while a +// message waits for an asynchronous inflate. + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, remote } from "../../../src/next/contracts/index.ts"; +import { PROTOCOL_VERSION } from "../../../src/next/data/internal.ts"; +import { createHost, type HostError } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { binaryCodec, Channel, type Message, textCodec } from "../../../src/next/link/internal.ts"; +import { + bindNext, + expectCollected, + flushed, + forEachCodec, + linked, + metered, + Notes, + RAW_PEER, + RawPeer, + rawLinked, + rawProvider, + testCodec, + transported, +} from "../support.ts"; + +const Thing = remote("app.thing", { version: 1, params: {}, variants: { only: { state: { n: int() } } } }); + +const hold = { + kind: "hold", + hold: 0, + type: "app.thing", + version: 1, + fingerprint: "0000000000000000", + params: "{}", +} as const; + +const problems = (errors: readonly HostError[]) => + errors.flatMap((error) => (error.kind === "protocol_error" ? [(error.error as Error).message] : [])); + +forEachCodec(() => + describe("the handshake", () => { + it("sends hello with the protocol version and the host's incarnation first", async () => { + const { peer } = await rawLinked(() => {}); + const hello = await peer.next("hello"); + expect(hello.protocol).toBe(PROTOCOL_VERSION); + expect(hello.incarnation).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/); + }); + + it("closes the link on traffic before hello", async () => { + const { peer, errors } = await rawLinked(() => {}, { hello: false }); + peer.send(hold); + await transported(); + expect(peer.closed).toBe(true); + expect(problems(errors)).toEqual(["handshake: hold before hello"]); + }); + + it("closes the link on a second hello", async () => { + const { peer, errors } = await rawLinked(() => {}); + peer.send({ kind: "hello", protocol: PROTOCOL_VERSION, incarnation: RAW_PEER, calls: 256 }); + await transported(); + expect(peer.closed).toBe(true); + expect(problems(errors)).toEqual(["handshake: a second hello"]); + }); + + it("closes the link on another protocol version", async () => { + const { peer, errors } = await rawLinked(() => {}, { hello: false }); + peer.send({ kind: "hello", protocol: PROTOCOL_VERSION + 1, incarnation: RAW_PEER, calls: 256 }); + await transported(); + expect(peer.closed).toBe(true); + expect(problems(errors)).toEqual([ + `handshake: protocol ${PROTOCOL_VERSION + 1}, this host speaks ${PROTOCOL_VERSION}`, + ]); + }); + + it("closes the link on a message failing its checks", async () => { + const { peer, errors } = await rawLinked(() => {}); + peer.sendBytes(testCodec().encode({ kind: "error", call: 0, error: { kind: "nope" } } as never)); + await transported(); + expect(peer.closed).toBe(true); + expect(problems(errors)).toEqual(["message: fails the message schema"]); + }); + + it("queues a consumer's messages until the peer's hello; a get over a link that never greets waits for its deadline", async () => { + const client = createHost({ onError: () => {} }); + await client.load([], context); + const [a, b] = memoryLinkPair(); + client.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + const peer = new RawPeer(b, { hello: false }); + const got = client.get(Thing, {}, context, { readyTimeout: 20 }); + await flushed(); + expect(peer.kinds()).toEqual(["hello"]); + await expect(got).rejects.toThrow("app.thing: did not become live within 20 ms"); + peer.send({ kind: "hello", protocol: PROTOCOL_VERSION, incarnation: RAW_PEER, calls: 256 }); + await transported(); + expect(peer.kinds()).toEqual(["hello", "hold", "release"]); + }); + }), +); + +describe("a codec mismatch", () => { + it("fails the first hello as a protocol error naming the expected codec", async () => { + const errors: HostError[] = []; + const server = createHost({ onError: (error) => errors.push(error) }); + await server.load([], context); + const client = createHost({ onError: () => {} }); + await client.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: binaryCodec }, context); + client.connect(b, { principal: { kind: "server" }, codec: textCodec }, context); + await transported(); + expect(problems(errors)).toEqual(["handshake: the first message is no binary message"]); + }); +}); + +describe("the heartbeat", () => { + beforeEach(() => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"], now: 0 }); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + /** Advances fake time by `ms` in heartbeat ticks, letting links deliver between them. */ + async function elapse(ms: number): Promise { + for (let step = 0; step < ms; step += 1500) { + await vi.advanceTimersByTimeAsync(Math.min(1500, ms - step)); + await transported(); + } + } + + it("closes a link whose peer never sends hello after 30 s (race 30)", async () => { + const { peer } = await rawLinked(() => {}, { hello: false }); + await elapse(29_000); + expect(peer.closed).toBe(false); + await elapse(1_500); + expect(peer.closed).toBe(true); + }); + + it("pings a silent link after 15 s and closes it after 30 s without an answer: references go loading", async () => { + const { client, peer } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + await bindNext(peer); + const notes = await got; + await elapse(13_500); + expect(peer.kinds()).not.toContain("ping"); + await elapse(3_000); + expect(peer.kinds().filter((kind) => kind === "ping")).toEqual(["ping"]); + await elapse(15_000); + expect(peer.closed).toBe(true); + expect(notes.state.value).toEqual({ kind: "loading" }); + }); + + it("sends no pings on a busy link", async () => { + const { peer } = await rawLinked(() => {}); + for (let second = 0; second < 60; second += 5) { + peer.send({ kind: "release", hold: 99 }); + await elapse(5_000); + } + expect(peer.kinds()).toEqual(["hello"]); + expect(peer.closed).toBe(false); + }); + + it("answers each ping with a pong at once; two sides pinging at once both answer", async () => { + const { client, server } = await linked(() => {}); + await elapse(60_000); + // Both sides pinged at 15 s and were answered: neither closed the link. + const notes = client.hold(Notes, { session: "S1" }, context); + await transported(); + expect(notes.state.value).toMatchObject({ kind: "failed" }); + expect(server).toBeDefined(); + const { peer } = await rawLinked(() => {}); + peer.send({ kind: "ping" }); + expect(await peer.next("pong")).toEqual({ kind: "pong" }); + }); + + it("runs one timer per host, only while it has links: none per message", async () => { + const host = createHost({ onError: () => {} }); + await host.load([], context); + expect(vi.getTimerCount()).toBe(0); + const [a, b] = memoryLinkPair(); + host.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const [c, d] = memoryLinkPair(); + host.accept(c, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peers = [new RawPeer(b), new RawPeer(d)]; + for (let round = 0; round < 50; round++) peers[0]!.send({ kind: "release", hold: round }); + await transported(); + expect(vi.getTimerCount()).toBe(1); + for (const peer of peers) peer.close(); + await transported(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("takes its numbers from the host's options", async () => { + const host = createHost({ onError: () => {}, heartbeat: { ping: 1_000, close: 2_000 } }); + await host.load([], context); + const [a, b] = memoryLinkPair(); + host.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + await vi.advanceTimersByTimeAsync(1_100); + await transported(); + expect(peer.kinds()).toEqual(["hello", "ping"]); + await vi.advanceTimersByTimeAsync(1_000); + await transported(); + expect(peer.closed).toBe(true); + }); +}); + +describe("trusted links", () => { + it("skip the message schema check: no chord.link.checks, the codec and the owners' checks still run", async () => { + const run = async (trusted: boolean) => { + const { context: counting, counts } = await metered(); + const host = createHost({ onError: () => {} }); + await host.load([], context); + const [a, b] = memoryLinkPair(); + host.accept(a, { principal: { kind: "owner" }, trusted, codec: testCodec() }, counting); + const peer = new RawPeer(b); + peer.send({ kind: "release", hold: 1 }); + peer.sendBytes(new Uint8Array([0xff])); + await transported(); + return { checks: counts()["chord.link.checks"] ?? 0, closed: peer.closed }; + }; + expect(await run(false)).toEqual({ checks: 2, closed: true }); + expect(await run(true)).toEqual({ checks: 0, closed: true }); + }); +}); + +describe("in-order handling", () => { + /** A channel whose handler handles `release` of hold 0 asynchronously (until `finish`), every other message at once. */ + async function holding() { + const [a, b] = memoryLinkPair(); + const handled: string[] = []; + const reports: unknown[] = []; + let finish: { resolve(): void; reject(error: Error): void } | undefined; + let closed = false; + const handler = { + opened: () => {}, + closed: () => { + closed = true; + }, + message: (message: Message): void | Promise => { + const name = message.kind === "release" ? `release ${message.hold}` : message.kind; + if (message.kind !== "release" || message.hold !== 0) { + handled.push(name); + return; + } + const { promise, resolve, reject } = Promise.withResolvers(); + finish = { resolve, reject }; + return promise.then(() => void handled.push(name)); + }, + }; + const options = { + codec: testCodec(), + incarnation: RAW_PEER, + calls: 1, + trusted: true, + report: (e: unknown) => reports.push(e), + }; + new Channel(a, handler, options, context).start(); + const peer = new RawPeer(b); + await peer.next("hello"); + return { peer, handled, reports, finish: () => finish!, isClosed: () => closed }; + } + + forEachCodec(() => { + it("holds every later message, a ping included, until the asynchronous one was handled, then handles them in order", async () => { + const { peer, handled, finish } = await holding(); + peer.send({ kind: "release", hold: 0 }); + peer.send({ kind: "release", hold: 1 }); + peer.send({ kind: "ping" }); + peer.send({ kind: "release", hold: 2 }); + await transported(); + expect(handled).toEqual([]); + expect(peer.kinds()).toEqual([]); + finish().resolve(); + await transported(); + expect(handled).toEqual(["release 0", "release 1", "release 2"]); + expect(peer.kinds()).toEqual(["pong"]); + }); + + it("keeps the rest behind a held message that holds again, in order, a long queue included", async () => { + const { peer, handled, finish } = await holding(); + const release = (hold: number) => testCodec().encode({ kind: "release", hold }); + peer.sendBytes(release(0)); + peer.sendBytes(release(1)); + peer.sendBytes(release(0)); + // Beyond what a spread into function arguments takes: the queue is read by index. + const many = release(2); + for (let i = 0; i < 70_000; i++) peer.sendBytes(many); + await transported(); + expect(handled).toEqual([]); + finish().resolve(); + await transported(); + expect(handled).toEqual(["release 0", "release 1"]); + finish().resolve(); + await transported(); + expect(handled.length).toBe(70_003); + expect(handled.slice(0, 4)).toEqual(["release 0", "release 1", "release 0", "release 2"]); + }); + + it("drops the held message's successors when the link closes meanwhile (race 13)", async () => { + const { peer, handled, finish, isClosed } = await holding(); + peer.send({ kind: "release", hold: 0 }); + peer.send({ kind: "release", hold: 1 }); + await transported(); + peer.close(); + await transported(); + expect(isClosed()).toBe(true); + finish().resolve(); + await transported(); + expect(handled).toEqual(["release 0"]); + }); + + it("keeps no held message once the link closed (GC)", async () => { + const { peer, finish, refs } = await (async () => { + const { peer, finish } = await holding(); + peer.send({ kind: "release", hold: 0 }); + const held = testCodec().encode({ kind: "release", hold: 1 }); + peer.sendBytes(held); + await transported(); + peer.close(); + await transported(); + return { peer, finish, refs: [new WeakRef(held)] }; + })(); + await expectCollected(...refs); + finish().resolve(); + expect(peer.closed).toBe(true); + }); + + it("treats a rejected asynchronous handling as a protocol error: reported, the link closes", async () => { + const { peer, handled, finish, reports } = await holding(); + peer.send({ kind: "release", hold: 0 }); + peer.send({ kind: "release", hold: 1 }); + await transported(); + finish().reject(new Error("corrupt")); + await transported(); + expect(peer.closed).toBe(true); + expect(reports).toEqual([new Error("corrupt")]); + expect(handled).toEqual([]); + }); + }); +}); diff --git a/packages/chord/test/next/link/errors.test.ts b/packages/chord/test/next/link/errors.test.ts new file mode 100644 index 000000000..2303a00b9 --- /dev/null +++ b/packages/chord/test/next/link/errors.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; +import { ChordError, PluginError } from "../../../src/next/data/index.ts"; +import { errorFromWire, wireError } from "../../../src/next/link/internal.ts"; + +describe("errors on the wire", () => { + it("carry a ChordError as its JSON, without stack or cause", () => { + const error = new ChordError({ code: "invalid", reason: "args", message: "bad" }, { cause: new Error("x") }); + const wire = wireError(error); + expect(wire).toEqual({ kind: "chord", error: { code: "invalid", reason: "args", message: "bad" } }); + const back = errorFromWire(JSON.parse(JSON.stringify(wire))); + expect(back).toBeInstanceOf(ChordError); + expect(back).toMatchObject({ code: "invalid", reason: "args", message: "bad" }); + }); + + it("carry a PluginError's code, message and details only; the consumer rethrows a PluginError", () => { + const error = new PluginError("quota", "too many notes", { limit: 3 }); + const wire = wireError(error); + expect(wire).toEqual({ kind: "plugin", code: "quota", message: "too many notes", details: { limit: 3 } }); + const back = errorFromWire(wire); + expect(back).toBeInstanceOf(PluginError); + expect(back).toMatchObject({ code: "quota", message: "too many notes", details: { limit: 3 } }); + expect(wireError(new PluginError("c", "m"))).toEqual({ kind: "plugin", code: "c", message: "m" }); + }); + + it("carry anything else as the plugin code internal with its message", () => { + expect(wireError(new TypeError("boom"))).toEqual({ kind: "plugin", code: "internal", message: "boom" }); + expect(wireError("plain")).toEqual({ kind: "plugin", code: "internal", message: "plain" }); + }); + + it("throw on a malformed ChordError envelope (a protocol error)", () => { + expect(() => errorFromWire({ kind: "chord", error: { code: "nope", message: "m" } })).toThrow( + "Invalid ChordError JSON", + ); + }); +}); diff --git a/packages/chord/test/next/link/fixtures/ipc-echo.ts b/packages/chord/test/next/link/fixtures/ipc-echo.ts new file mode 100644 index 000000000..6764f8992 --- /dev/null +++ b/packages/chord/test/next/link/fixtures/ipc-echo.ts @@ -0,0 +1,10 @@ +// A forked child for the IPC link test: echoes every message of its parent link, and exits when the link closes. +import process from "node:process"; +import { ipcLink } from "../../../../src/next/link/index.ts"; + +const link = ipcLink(process as unknown as Parameters[0]); +link.onMessage((message) => { + if (message[0] === 0xff) process.exit(0); + link.send(message); +}); +link.onClose(() => process.exit(0)); diff --git a/packages/chord/test/next/link/ipc.test.ts b/packages/chord/test/next/link/ipc.test.ts new file mode 100644 index 000000000..da9e87d21 --- /dev/null +++ b/packages/chord/test/next/link/ipc.test.ts @@ -0,0 +1,143 @@ +import { fork } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { type IpcPort, ipcLink } from "../../../src/next/link/index.ts"; +import { flushed } from "../support.ts"; + +/** A port playing a Node IPC channel by hand: what was sent, its listeners, and what the test makes happen. */ +class FakePort implements IpcPort { + connected = true; + readonly sent: Uint8Array[] = []; + /** What `send` returns, or throws when an error. */ + next: boolean | Error = true; + callbacks: ((error: Error | null) => void)[] = []; + disconnects = 0; + readonly #message = new Set<(message: unknown) => void>(); + readonly #disconnect = new Set<() => void>(); + + send(message: Uint8Array, callback: (error: Error | null) => void): boolean { + if (this.next instanceof Error) throw this.next; + this.sent.push(message); + this.callbacks.push(callback); + return this.next; + } + + on(event: "message" | "disconnect", listener: (message: unknown) => void): this { + if (event === "message") this.#message.add(listener); + else this.#disconnect.add(listener as () => void); + return this; + } + + removeListener(event: "message" | "disconnect", listener: (message: unknown) => void): this { + if (event === "message") this.#message.delete(listener); + else this.#disconnect.delete(listener as () => void); + return this; + } + + disconnect(): void { + this.disconnects++; + this.connected = false; + for (const listener of [...this.#disconnect]) listener(); + } + + receive(message: unknown): void { + for (const listener of [...this.#message]) listener(message); + } + + get listeners(): number { + return this.#message.size + this.#disconnect.size; + } +} + +describe("IPC links over a fake port", () => { + it("deliver Uint8Array messages in order, queued until the listener is set; other messages are not the link's", () => { + const port = new FakePort(); + const link = ipcLink(port); + port.receive(new Uint8Array([1])); + port.receive({ start: "the app's own" }); + port.receive(new Uint8Array([2])); + const received: number[] = []; + link.onMessage((message) => received.push(message[0]!)); + port.receive(new Uint8Array([3])); + expect(received).toEqual([1, 2, 3]); + }); + + it("treat send returning false as backpressure, not a close", () => { + const port = new FakePort(); + const link = ipcLink(port); + let closed = 0; + link.onClose(() => closed++); + port.next = false; + link.send(new Uint8Array([1])); + link.send(new Uint8Array([2])); + expect(port.sent).toHaveLength(2); + expect(closed).toBe(0); + }); + + it.each([ + [ + "a thrown send", + (port: FakePort) => { + port.next = new Error("channel closed"); + }, + ], + ["an error in the send callback", (port: FakePort) => port.callbacks[0]!(new Error("EPIPE"))], + ["a disconnect", (port: FakePort) => port.disconnect()], + ])("close once on %s; listeners removed, later messages dropped (race 27)", (_, fail) => { + const port = new FakePort(); + const link = ipcLink(port); + let closed = 0; + const received: Uint8Array[] = []; + link.onClose(() => closed++); + link.onMessage((message) => received.push(message)); + link.send(new Uint8Array([1])); + fail(port); + link.send(new Uint8Array([2])); + port.disconnect(); + port.receive(new Uint8Array([3])); + expect(closed).toBe(1); + expect(received).toEqual([]); + expect(port.listeners).toBe(0); + }); + + it("disconnect the port on close, once; a port not connected gives a closed link", () => { + const port = new FakePort(); + const link = ipcLink(port); + link.close(); + link.close(); + expect(port.disconnects).toBe(1); + let closed = false; + ipcLink(port).onClose(() => { + closed = true; + }); + expect(closed).toBe(true); + }); +}); + +describe("IPC links to a forked child", () => { + it("carry Uint8Array messages both ways and close when the child exits mid-send (race 27)", async () => { + const child = fork(fileURLToPath(new URL("./fixtures/ipc-echo.ts", import.meta.url)), [], { + serialization: "advanced", + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + }); + const link = ipcLink(child); + const echoed = new Promise((resolve) => link.onMessage(resolve)); + let closes = 0; + const closed = new Promise((resolve) => + link.onClose(() => { + closes++; + resolve(); + }), + ); + link.send(new Uint8Array([1, 2, 3])); + const message = await echoed; + expect(message).toBeInstanceOf(Uint8Array); + expect([...message]).toEqual([1, 2, 3]); + // The child exits on this one; the sends after it race the exit. + link.send(new Uint8Array([0xff])); + for (let send = 0; send < 20; send++) link.send(new Uint8Array(1024)); + await closed; + await flushed(); + expect(closes).toBe(1); + }, 20_000); +}); diff --git a/packages/chord/test/next/link/link.test.ts b/packages/chord/test/next/link/link.test.ts new file mode 100644 index 000000000..0f720f196 --- /dev/null +++ b/packages/chord/test/next/link/link.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { createHost } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { expectCollected, flushed } from "../support.ts"; + +const bytes = (...values: number[]) => new Uint8Array(values); + +describe("memory links", () => { + it("deliver in order, one task later, and queue what arrives before the listener", async () => { + const [a, b] = memoryLinkPair(); + const got: number[] = []; + a.send(bytes(1)); + a.send(bytes(2)); + await flushed(); + b.onMessage((message) => got.push(message[0]!)); + expect(got).toEqual([1, 2]); + a.send(bytes(3)); + expect(got).toEqual([1, 2]); + await flushed(); + expect(got).toEqual([1, 2, 3]); + }); + + it("close this end at once and the peer one task later; send after close is a no-op", async () => { + const [a, b] = memoryLinkPair(); + const closes: string[] = []; + const got: number[] = []; + a.onClose(() => closes.push("a")); + b.onClose(() => closes.push("b")); + b.onMessage((message) => got.push(message[0]!)); + a.close(); + expect(closes).toEqual(["a"]); + a.send(bytes(1)); + await flushed(); + expect(closes).toEqual(["a", "b"]); + b.send(bytes(2)); + await flushed(); + expect(got).toEqual([]); + const late: string[] = []; + a.onClose(() => late.push("late")); + expect(late).toEqual(["late"]); + }); + + it("drop messages still in flight to a closed end", async () => { + const [a, b] = memoryLinkPair(); + const got: number[] = []; + a.onMessage((message) => got.push(message[0]!)); + b.send(bytes(1)); + a.close(); + await flushed(); + expect(got).toEqual([]); + }); + + it("stop delivering queued messages once a listener closed the link", async () => { + const [a, b] = memoryLinkPair(); + const got: number[] = []; + a.send(bytes(1)); + a.send(bytes(2)); + await flushed(); + b.onMessage((message) => { + got.push(message[0]!); + b.close(); + }); + expect(got).toEqual([1]); + }); + + it("keep no listener once closed", async () => { + const [a, b] = memoryLinkPair(); + const refs = (() => { + const listener = { onMessage: () => {} }; + a.onMessage(listener.onMessage); + a.onClose(listener.onMessage); + return [new WeakRef(listener.onMessage)]; + })(); + a.close(); + await flushed(); + await expectCollected(...refs); + expect(b).toBeDefined(); + }); +}); + +describe("a closed memory link", () => { + it("keeps no listener set after its close: accepting it keeps no host", async () => { + const [a, b] = memoryLinkPair(); + a.close(); + await flushed(); + const refs = await (async () => { + const host = createHost({ onError: () => {} }); + await host.load([], BACKGROUND_CONTEXT); + host.accept(a, { principal: { kind: "owner" } }, BACKGROUND_CONTEXT); + return [new WeakRef(host)]; + })(); + await expectCollected(...refs); + expect([a, b]).toBeDefined(); + }); +}); diff --git a/packages/chord/test/next/link/message.test.ts b/packages/chord/test/next/link/message.test.ts new file mode 100644 index 000000000..6adf76b4d --- /dev/null +++ b/packages/chord/test/next/link/message.test.ts @@ -0,0 +1,540 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { + binaryCodec, + checkMessage, + DeflatedPart, + describeMessage, + jsonPart, + type Message, + parsePart, + parsePayload, + payloadPart, + textCodec, +} from "../../../src/next/link/internal.ts"; +import type { Part } from "../../../src/next/link/part.ts"; +import { forEachCodec, testCodec } from "../support.ts"; + +/** A part as JSON text: a received part's bytes decoded (a deflated part stays as it is). */ +function partText(part: Part): Part { + return part instanceof Uint8Array ? new TextDecoder().decode(part) : part; +} + +const incarnation = "0190f5a2-7c3e-7d4a-9b1c-2d3e4f5a6b7c"; +const error = { kind: "chord", error: { code: "released", message: "gone" } } as const; + +/** One message of every kind, with the optional fields both present and absent; parts are JSON text. */ +const EVERY_KIND: Message[] = [ + { kind: "hello", protocol: 1, incarnation, calls: 256 }, + { kind: "hold", hold: 300, type: "app.notes", version: 2, fingerprint: "0123456789abcdef", params: '{"id":1}' }, + { + kind: "hold", + hold: 1, + type: "app.notes", + version: 1, + fingerprint: "0123456789abcdef", + params: '{"session":"S1"}', + address: { kind: "session", params: { id: "S1" } }, + principal: { kind: "guest" }, + incarnation, + }, + { + kind: "hold", + hold: 2, + type: "app.notes", + version: 1, + fingerprint: "0123456789abcdef", + params: "{}", + incarnation, + }, + { kind: "release", hold: 0 }, + { kind: "call", call: 3, hold: 0, method: 1, args: '{"text":"x"}' }, + { kind: "call", call: 4, hold: 0, method: 0 }, + { kind: "abort", call: 3 }, + { + kind: "bound", + holds: [{ hold: 0, variant: 1, members: [{ member: 0, value: '[1,"a"]' }], failed: [{ member: 1, error }] }], + }, + { + kind: "bound", + holds: [ + { hold: 0, variant: 0, members: [], failed: [] }, + { hold: 5, variant: 2, members: [{ member: 1, value: "null" }], failed: [] }, + ], + }, + { + kind: "state", + holds: [ + { hold: 0, members: [{ member: 2, ops: '[["s",["n"],1]]' }], failed: [], retired: [] }, + { hold: 1, members: [], failed: [{ member: 0, error }], retired: [] }, + // Retired members (decision "One retirement mechanism"), alone and with a failure. + { hold: 2, members: [{ member: 1, ops: '[["r",{}]]' }], failed: [], retired: [1, 3] }, + { hold: 3, members: [], failed: [{ member: 0, error }], retired: [2] }, + ], + }, + { + kind: "failed", + hold: 0, + cause: { kind: "transient", error: { kind: "chord", error: { code: "unavailable", message: "m" } } }, + }, + { + kind: "failed", + hold: 0, + cause: { kind: "broken", error: { kind: "chord", error: { code: "unavailable", message: "m" } } }, + }, + { + kind: "failed", + hold: 0, + cause: { kind: "transient", error: { kind: "chord", error: { code: "unavailable", message: "m" } } }, + retryIn: 0, + }, + // The wire error format in `failed` and `gone`: a plugin's error travels too. + { kind: "failed", hold: 0, cause: { kind: "broken", error: { kind: "plugin", code: "config", message: "m" } } }, + { kind: "gone", hold: 1, error: { kind: "plugin", code: "config", message: "m", details: { at: 1 } } }, + { + kind: "failed", + hold: 9, + cause: { kind: "transient", error: { kind: "chord", error: { code: "unavailable", message: "m" } } }, + retryIn: 60_000, + }, + { kind: "retry", hold: 300 }, + { kind: "loading", hold: 300 }, + { kind: "result", call: 3, value: '{"n":1}' }, + { kind: "result", call: 3 }, + { kind: "error", call: 3, error: { kind: "plugin", code: "c", message: "m", details: [1] } }, + { kind: "gone", hold: 0, error: { kind: "chord", error: { code: "released", message: "m" } } }, + { kind: "ack" }, + { kind: "ack", origin: 1 }, + { kind: "closed", origin: 300 }, + { kind: "hold", origin: 2, hold: 0, type: "t", version: 1, fingerprint: "0123456789abcdef", params: "{}" }, + { kind: "release", origin: 2, hold: 3 }, + { kind: "call", origin: 2, call: 1, hold: 3, method: 0, args: "[1]" }, + { kind: "abort", origin: 2, call: 1 }, + { kind: "retry", origin: 2, hold: 3 }, + { kind: "bound", origin: 2, holds: [{ hold: 3, variant: 0, members: [], failed: [] }] }, + { kind: "state", origin: 2, holds: [] }, + { + kind: "failed", + origin: 2, + hold: 3, + cause: { kind: "broken", error: { kind: "chord", error: { code: "unavailable", message: "m" } } }, + }, + { kind: "loading", origin: 2, hold: 3 }, + { kind: "result", origin: 2, call: 1 }, + { kind: "error", origin: 2, call: 1, error: { kind: "plugin", code: "c", message: "m" } }, + { kind: "gone", origin: 2, hold: 3, error: { kind: "chord", error: { code: "released", message: "m" } } }, + { + kind: "hold", + hold: 3, + type: "app.notes", + version: 1, + fingerprint: "0123456789abcdef", + params: "{}", + chain: [incarnation, incarnation], + limit: 65_536, + }, + { kind: "hold", hold: 4, type: "t", version: 1, fingerprint: "0123456789abcdef", params: "{}", limit: 0 }, + { kind: "revoke", revocation: { grant: "g1" } }, + { kind: "ping" }, + { kind: "pong" }, +]; + +/** A decoded message with every part as its text, to compare with one built from text parts. */ +function plain(message: Message): Message { + switch (message.kind) { + case "hold": + return { ...message, params: partText(message.params) }; + case "call": + return message.args === undefined ? message : { ...message, args: partText(message.args) }; + case "result": + return message.value === undefined ? message : { ...message, value: partText(message.value) }; + case "bound": + return { + ...message, + holds: message.holds.map((entry) => ({ + ...entry, + members: entry.members.map(({ member, value }) => ({ member, value: partText(value) })), + })), + }; + case "state": + return { + ...message, + holds: message.holds.map((entry) => ({ + ...entry, + members: entry.members.map(({ member, ops }) => ({ member, ops: partText(ops) })), + })), + }; + default: + return message; + } +} + +forEachCodec(() => { + describe("messages", () => { + it.each(EVERY_KIND)("round-trip %j and pass the message schema", (message) => { + const codec = testCodec(); + const decoded = checkMessage(codec.decode(codec.encode(message)), context); + expect(plain(decoded)).toEqual(message); + }); + + it("round-trip deflated parts in every part's place, still deflated (a relay forwards them so)", () => { + const codec = testCodec(); + const deflated = () => new DeflatedPart(new Uint8Array([0, 1, 2, 255])); + const messages: Message[] = [ + { kind: "hold", hold: 0, type: "t", version: 1, fingerprint: "0123456789abcdef", params: deflated() }, + { kind: "call", call: 0, hold: 0, method: 0, args: deflated() }, + { kind: "result", call: 0, value: deflated(), segments: [new Uint8Array([7])] }, + { + kind: "bound", + holds: [{ hold: 0, variant: 0, members: [{ member: 0, value: deflated() }], failed: [] }], + }, + { kind: "state", holds: [{ hold: 0, members: [{ member: 0, ops: deflated() }], failed: [], retired: [] }] }, + ]; + for (const message of messages) { + const decoded = checkMessage(codec.decode(codec.encode(message)), context); + expect(decoded).toEqual(message); + } + }); + + it("round-trip parts received as bytes (a relay forwards them as they came)", () => { + const codec = testCodec(); + const message: Message = { kind: "result", call: 0, value: new TextEncoder().encode('{"n":1}') }; + expect(plain(codec.decode(codec.encode(message)))).toEqual({ kind: "result", call: 0, value: '{"n":1}' }); + }); + + it("carry byte segments beside a payload, as views of the message", () => { + const codec = testCodec(); + const data = new Uint8Array(1024).fill(7); + const thumbs = [new Uint8Array([1, 2]), new Uint8Array([3])]; + const { json, segments } = payloadPart({ name: "a", data, thumbs }, true); + const message: Message = { kind: "call", call: 0, hold: 0, method: 0, args: json, segments }; + const bytes = codec.encode(message); + const decoded = codec.decode(bytes) as Extract; + expect(decoded.segments).toHaveLength(3); + expect(decoded.segments![0]!.buffer).toBe(bytes.buffer); + const args = parsePayload(decoded.args as Uint8Array, decoded.segments) as { + data: Uint8Array; + thumbs: Uint8Array[]; + }; + expect(args.data).toEqual(data); + expect(args.data.buffer).toBe(bytes.buffer); + expect(args.thumbs).toEqual(thumbs); + // The segments travel raw: the message is barely larger than the bytes it carries. + expect(bytes.byteLength).toBeLessThan(1024 + 3 + 100); + }); + + it("refuse an empty message, an unknown kind and trailing bytes", () => { + const codec = testCodec(); + const release = codec.encode({ kind: "release", hold: 1 }); + const trailing = new Uint8Array([...release, ...(codec === binaryCodec ? [0] : [0x20, 0x31])]); + expect(() => codec.decode(new Uint8Array())).toThrow(); + expect(() => + codec.decode(codec === binaryCodec ? new Uint8Array([99]) : new TextEncoder().encode('["?"]')), + ).toThrow("unknown kind"); + expect(() => codec.decode(trailing)).toThrow(); + }); + + it("refuse an origin on a kind without one, an origin 0, and a closed without one (the message schema)", () => { + const codec = testCodec(); + const check = (message: unknown) => () => + checkMessage(codec.decode(codec.encode(message as Message)), context); + expect(check({ kind: "ping", origin: 1 })).toThrow("message schema"); + expect(check({ kind: "ack", origin: 0 })).toThrow("message schema"); + const closed = codec === binaryCodec ? new Uint8Array([14]) : new TextEncoder().encode('["C"]'); + expect(() => checkMessage(codec.decode(closed), context)).toThrow("message schema"); + }); + + it("keep a leading U+FEFF in strings and parts (it is data, not a byte order mark)", () => { + const codec = testCodec(); + const message: Message = { + kind: "hold", + hold: 0, + type: "\uFEFFthing", + version: 1, + fingerprint: "0123456789abcdef", + params: jsonPart({ "\uFEFFkey": "\uFEFFvalue" }), + }; + const decoded = codec.decode(codec.encode(message)) as Extract; + expect(plain(decoded)).toEqual(message); + expect(parsePart(decoded.params as string | Uint8Array)).toEqual({ "\uFEFFkey": "\uFEFFvalue" }); + const result: Message = { kind: "result", call: 0, value: jsonPart("\uFEFF") }; + expect(plain(codec.decode(codec.encode(result)))).toEqual(result); + }); + + it("leave the envelope to the message schema: a misshapen error passes the codec and fails the schema", () => { + const codec = testCodec(); + const decoded = codec.decode(codec.encode({ kind: "error", call: 0, error: { kind: "nope" } } as never)); + expect(() => checkMessage(decoded, context)).toThrow("message: fails the message schema"); + expect(() => checkMessage({ kind: "release", hold: -1 }, context)).toThrow( + "message: fails the message schema", + ); + }); + }); +}); + +describe("the binary codec", () => { + it("writes ids as varints, the incarnation as 16 and the fingerprint as 8 raw bytes, no outer length", () => { + expect(binaryCodec.encode({ kind: "hello", protocol: 1, incarnation, calls: 1 })).toHaveLength(1 + 1 + 16 + 1); + expect([...binaryCodec.encode({ kind: "release", hold: 300 })]).toEqual([2, 0xac, 0x02]); + const hold = binaryCodec.encode({ + kind: "hold", + hold: 0, + type: "t", + version: 1, + fingerprint: "0123456789abcdef", + params: "{}", + }); + // kind, id, type, version, fingerprint, params, the optional fields' flags + expect(hold).toHaveLength(1 + 1 + 2 + 1 + 8 + 3 + 1); + expect([...binaryCodec.encode({ kind: "ack" })]).toEqual([13]); + // An origin: the kind byte's high bit, then the origin as a varint. + expect([...binaryCodec.encode({ kind: "ack", origin: 300 })]).toEqual([13 | 0x80, 0xac, 0x02]); + expect([...binaryCodec.encode({ kind: "closed", origin: 1 })]).toEqual([14 | 0x80, 1]); + }); + + it("costs a state entry without retirements no byte: the failure count is doubled, plus one when they follow", () => { + const state = (retired: number[]): Message => ({ + kind: "state", + holds: [{ hold: 0, members: [], failed: [], retired }], + }); + // kind, entry count, hold, member count, failures (0 doubled) + expect([...binaryCodec.encode(state([]))]).toEqual([6, 1, 0, 0, 0]); + // the low bit set, then the retired count and member + expect([...binaryCodec.encode(state([3]))]).toEqual([6, 1, 0, 0, 1, 1, 3]); + const first = binaryCodec.decode(binaryCodec.encode(state([]))); + const second = binaryCodec.decode(binaryCodec.encode(state([]))); + const retired = (message: Message) => (message as Extract).holds[0]!.retired; + // No allocation per message: decoded entries without retirements share one empty list. + expect(retired(first)).toBe(retired(second)); + }); + + it("prefixes a part with its length doubled, plus one when it is deflated", () => { + const plain = binaryCodec.encode({ kind: "result", call: 0, value: "12" }); + const deflated = binaryCodec.encode({ kind: "result", call: 0, value: new DeflatedPart(new Uint8Array([7, 8])) }); + // kind, call, payload flags, prefix, bytes + expect([...plain]).toEqual([8, 0, 1, 4, 0x31, 0x32]); + expect([...deflated]).toEqual([8, 0, 1, 5, 7, 8]); + }); + + it("decodes parts as views of the message, never parsing them: a malformed part decodes", () => { + const part = new TextEncoder().encode("{not json"); + const bytes = binaryCodec.encode({ kind: "result", call: 0, value: part }); + const decoded = binaryCodec.decode(bytes) as Extract; + const value = decoded.value as Uint8Array; + expect(value.buffer).toBe(bytes.buffer); + expect(() => parsePart(value)).toThrow(); + }); + + it("refuses varints past a safe integer and lengths past the message", () => { + expect(() => binaryCodec.decode(new Uint8Array([2, ...new Array(8).fill(0xff), 0x7f]))).toThrow( + "a varint does not fit", + ); + expect(() => binaryCodec.decode(new Uint8Array([1, 0, 9, 0x61]))).toThrow("the message ends early"); + }); + + it("refuses a varint of more than 8 bytes, which would read as no number", () => { + const state = new Uint8Array([6, ...new Array(150).fill(0x80), 0]); + expect(() => binaryCodec.decode(state)).toThrow("a varint does not fit"); + }); + + it("refuses unknown hold flags", () => { + const hold = binaryCodec.encode({ + kind: "hold", + hold: 0, + type: "t", + version: 1, + fingerprint: "0123456789abcdef", + params: "{}", + }); + const forged = hold.slice(); + forged[forged.length - 1] = 32; + expect(() => binaryCodec.decode(forged)).toThrow("unknown hold flags"); + }); + + it("returns a right-sized message: a large one keeps no larger backing store", () => { + const value = jsonPart("x".repeat(500_000)); + // Small fields after the large value: they must not double the buffer the message keeps. + const members = [ + { member: 0, value }, + { member: 1, value: "1" }, + ]; + const bytes = binaryCodec.encode({ kind: "bound", holds: [{ hold: 0, variant: 0, members, failed: [] }] }); + expect(bytes.buffer.byteLength - bytes.byteLength).toBeLessThanOrEqual(4096); + const small = binaryCodec.encode({ kind: "release", hold: 1 }); + expect(small.buffer.byteLength).toBe(small.byteLength); + // A message of about two thirds of the starting buffer keeps no spare room either. + const medium = binaryCodec.encode({ kind: "result", call: 0, value: jsonPart("x".repeat(160)) }); + expect(medium.buffer.byteLength).toBe(medium.byteLength); + }); + + it("refuses invalid UTF-8 in strings", () => { + expect(() => binaryCodec.decode(new Uint8Array([1, 0, 1, 0xff]))).toThrow(); + }); + + it("writes retry and loading as their kind and hold, and failed's retryIn as one more than it (0: absent)", () => { + expect([...binaryCodec.encode({ kind: "retry", hold: 300 })]).toEqual([11, 0xac, 0x02]); + expect([...binaryCodec.encode({ kind: "loading", hold: 1 })]).toEqual([12, 1]); + const cause = { kind: "broken", error: { kind: "chord", error: { code: "released", message: "" } } } as const; + const absent = binaryCodec.encode({ kind: "failed", hold: 0, cause }); + const present = binaryCodec.encode({ kind: "failed", hold: 0, cause, retryIn: 0 }); + expect(absent.at(-1)).toBe(0); + expect(present.at(-1)).toBe(1); + }); +}); + +describe("the text codec", () => { + const text = (message: Message) => new TextDecoder().decode(textCodec.encode(message)); + + it("writes one JSON tuple per message with short tags and positional fields, parts spliced in", () => { + expect(text({ kind: "release", hold: 3 })).toBe('["R",3]'); + expect(text({ kind: "call", call: 1, hold: 0, method: 2 })).toBe('["c",1,0,2]'); + const members = [{ member: 0, value: "5" }]; + expect(text({ kind: "bound", holds: [{ hold: 0, variant: 1, members, failed: [] }] })).toBe( + '["b",[[0,1,[[0,[0,5]]]]]]', + ); + expect(text({ kind: "retry", hold: 2 })).toBe('["T",2]'); + expect(text({ kind: "loading", hold: 2 })).toBe('["l",2]'); + const cause = { + kind: "transient", + error: { kind: "chord", error: { code: "unavailable", message: "m" } }, + } as const; + expect(text({ kind: "failed", hold: 2, cause })).toBe( + '["f",2,"t",{"kind":"chord","error":{"code":"unavailable","message":"m"}}]', + ); + expect(text({ kind: "failed", hold: 2, cause, retryIn: 750 })).toBe( + '["f",2,"t",{"kind":"chord","error":{"code":"unavailable","message":"m"}},750]', + ); + expect(text({ kind: "ack" })).toBe('["a"]'); + // An origin wraps the message's own tuple. + expect(text({ kind: "release", origin: 4, hold: 3 })).toBe('["@",4,["R",3]]'); + expect(text({ kind: "closed", origin: 4 })).toBe('["@",4,["C"]]'); + }); + + it("tags every part with its codec, [0, json] or [1, base64] for a deflated one, so no value is reserved", () => { + const message: Message = { kind: "result", call: 1, value: new DeflatedPart(new Uint8Array([1, 2, 3])) }; + expect(text(message)).toBe('["r",1,[1,"AQID"]]'); + expect(textCodec.decode(textCodec.encode(message))).toEqual(message); + // A value of any shape is data, the one-key `$deflated` object the first builder reserved included. + const shaped: Message = { kind: "result", call: 1, value: '{"$deflated":"AQID"}' }; + expect(text(shaped)).toBe('["r",1,[0,{"$deflated":"AQID"}]]'); + expect(textCodec.decode(textCodec.encode(shaped))).toEqual(shaped); + expect(text({ kind: "revoke", revocation: { grant: "g" } })).toBe('["V",{"grant":"g"}]'); + }); + + it("refuses a part that is no [codec, payload] pair, or of an unknown codec", () => { + const decode = (tuple: string) => textCodec.decode(new TextEncoder().encode(tuple)); + expect(() => decode('["r",1,{"n":1}]')).toThrow("no [codec, payload] pair"); + expect(() => decode('["r",1,[2,"AQID"]]')).toThrow("unknown codec"); + expect(() => decode('["r",1,[1,5]]')).toThrow("unknown codec"); + }); + + it("carries a binary link's deflated part over a text hop and back unchanged (a relay between codecs)", () => { + const part = new DeflatedPart(new Uint8Array([9, 8, 7])); + const message: Message = { + kind: "state", + holds: [{ hold: 0, members: [{ member: 0, ops: part }], failed: [], retired: [] }], + }; + const overText = textCodec.decode(textCodec.encode(binaryCodec.decode(binaryCodec.encode(message)))); + expect(binaryCodec.decode(binaryCodec.encode(overText))).toEqual(message); + }); + + it("leaves out a hold's absent optional fields at the end, null in the middle", () => { + const hold = { + kind: "hold", + hold: 0, + type: "t", + version: 1, + fingerprint: "0123456789abcdef", + params: "{}", + } as const; + expect(text(hold)).toBe('["H",0,"t",1,"0123456789abcdef",[0,{}]]'); + expect(text({ ...hold, incarnation })).toBe(`["H",0,"t",1,"0123456789abcdef",[0,{}],null,null,"${incarnation}"]`); + expect(text({ ...hold, limit: 5 })).toBe('["H",0,"t",1,"0123456789abcdef",[0,{}],null,null,null,null,5]'); + }); + + it("puts byte segments after the tuple and a newline, their lengths the tuple's last field", () => { + const bytes = textCodec.encode({ + kind: "result", + call: 0, + value: '{"$bytes":0}', + segments: [new Uint8Array([1, 2])], + }); + expect([...bytes]).toEqual([...new TextEncoder().encode('["r",0,[0,{"$bytes":0}],[2]]\n'), 1, 2]); + }); + + it("refuses a tuple of the wrong arity and segments without a payload carrying them", () => { + expect(() => textCodec.decode(new TextEncoder().encode('["R",1,2]'))).toThrow('a "R" message with 2 fields'); + expect(() => textCodec.decode(new TextEncoder().encode('["s",[[0,[],[],[],9]]]'))).toThrow( + "a state entry with 5 fields", + ); + expect(() => textCodec.decode(new TextEncoder().encode('["s",[[0]]]'))).toThrow("a state entry with 1 fields"); + expect(() => textCodec.decode(new Uint8Array([...new TextEncoder().encode('["R",1]\n'), 1]))).toThrow( + "byte segments without a payload", + ); + expect(() => textCodec.decode(new TextEncoder().encode('["@",1,["@",2,["a"]]]'))).toThrow("wraps one message"); + expect(() => textCodec.decode(new TextEncoder().encode('["@",1]'))).toThrow("wraps one message"); + }); +}); + +describe("describeMessage", () => { + it("prints a message in the text tuple form", () => { + expect(describeMessage({ kind: "abort", call: 7 })).toBe('["x",7]'); + expect(describeMessage({ kind: "result", call: 0, value: '{"$bytes":0}', segments: [new Uint8Array(3)] })).toBe( + '["r",0,[0,{"$bytes":0}],[3]]', + ); + }); +}); + +describe("a codec mismatch", () => { + it("decodes the other codec's hello as no message of its own", () => { + const hello: Message = { kind: "hello", protocol: 1, incarnation, calls: 256 }; + expect(() => binaryCodec.decode(textCodec.encode(hello))).toThrow(); + expect(() => textCodec.decode(binaryCodec.encode(hello))).toThrow(); + }); +}); + +describe("the binary codec's strings", () => { + it("survive the buffer growing around them, at every offset near the initial size", () => { + for (let length = 0; length < 400; length++) { + // A long value of three-byte characters (filling what it reserved), then a short one that lands at every offset around the buffer's first growth. + const members = [ + { member: 0, value: jsonPart("€".repeat(length)) }, + { member: 1, value: "1" }, + ]; + const message: Message = { kind: "bound", holds: [{ hold: 1, variant: 0, members, failed: [] }] }; + expect(plain(binaryCodec.decode(binaryCodec.encode(message)))).toEqual(message); + } + }); +}); + +describe("the binary codec's long strings", () => { + const texts = [ + ["ASCII", "x".repeat(10_000)], + ["three-byte characters", "€".repeat(10_000)], + ["surrogate pairs", "😀".repeat(5_000)], + ["mixed, at the in-place threshold", `${"a".repeat(4_093)}😀€`], + ["a leading U+FEFF", `\uFEFF${"y".repeat(5_000)}`], + ]; + + it.each(texts)("round-trip %s, encoded in place behind a padded length", (_, text) => { + const message: Message = { kind: "result", call: 0, value: jsonPart(text) }; + const bytes = binaryCodec.encode(message); + expect(plain(binaryCodec.decode(bytes))).toEqual(message); + expect(bytes.buffer.byteLength - bytes.byteLength).toBeLessThanOrEqual(4096); + }); +}); + +describe("the binary codec's short strings", () => { + it.each([0, 1, 41, 42, 43])("round-trip at %i three-byte characters, around the one-byte length", (length) => { + const type = "€".repeat(length); + const message: Message = { + kind: "hold", + hold: 0, + type, + version: 1, + fingerprint: "0123456789abcdef", + params: jsonPart(type), + }; + expect(plain(binaryCodec.decode(binaryCodec.encode(message)))).toEqual(message); + }); +}); diff --git a/packages/chord/test/next/link/part.test.ts b/packages/chord/test/next/link/part.test.ts new file mode 100644 index 000000000..83d76f54e --- /dev/null +++ b/packages/chord/test/next/link/part.test.ts @@ -0,0 +1,196 @@ +// Tests of parts on links: the limits per part (in UTF-8 bytes), compression at the origin, opening a part where it is +// owned, and the asynchronous inflate with its limit, its corrupt outcome and its stop when the link closes. + +import { describe, expect, it } from "vitest"; +import { + COMPRESS_ABOVE, + type Compression, + DeflatedPart, + inflateAsync, + linkLimits, + openPart, + originPart, + oversized, +} from "../../../src/next/link/part.ts"; +import { nodeCompression } from "../../../src/next-node/compression.ts"; +import { metered } from "../support.ts"; + +const UTF8 = new TextEncoder(); + +const NONE = { part: Number.POSITIVE_INFINITY, decoded: Number.POSITIVE_INFINITY }; + +/** A link that never closes. */ +const NEVER = new AbortController().signal; + +/** A JSON text of `length` characters that deflates well. */ +const json = (length: number) => JSON.stringify("x".repeat(length - 2)); + +/** What `fn` threw. */ +function thrown(fn: () => unknown): unknown { + try { + fn(); + } catch (error) { + return error; + } + throw new Error("expected a throw"); +} + +describe("limits per part", () => { + it("default to none on trusted links and to 16 MiB on the wire, 64 MiB decoded on untrusted ones", () => { + expect(linkLimits(true)).toEqual(NONE); + expect(linkLimits(false)).toEqual({ part: 16 * 1024 * 1024, decoded: 64 * 1024 * 1024 }); + expect(linkLimits(false, { part: 10 })).toEqual({ part: 10, decoded: 64 * 1024 * 1024 }); + expect(linkLimits(true, { decoded: 5 })).toEqual({ part: Number.POSITIVE_INFINITY, decoded: 5 }); + }); + + it("check a part's size on the wire with one comparison, nothing parsed", () => { + expect(oversized("[1,2]", 5)).toBeUndefined(); + expect(oversized("[1,2,3]", 5)?.toJSON()).toEqual({ + code: "too_large", + reason: "part", + message: "a part of 7 bytes is over the wire limit of 5 bytes", + }); + expect(oversized(new Uint8Array(6), 5)?.code).toBe("too_large"); + expect(oversized(new DeflatedPart(new Uint8Array(5)), 5)).toBeUndefined(); + }); + + it("count text in UTF-8 bytes, not UTF-16 units (a text link bounds what a binary link does)", () => { + const text = JSON.stringify("界".repeat(6000)); + expect(UTF8.encode(text).byteLength).toBe(18_002); + expect(oversized(text, 18_002)).toBeUndefined(); + expect(oversized(text, 18_001)?.message).toBe("a part of 18002 bytes is over the wire limit of 18001 bytes"); + // Two-byte characters and surrogate pairs (four bytes for two units). + expect(oversized('"é😀"', 8)).toBeUndefined(); + expect(oversized('"é😀"', 7)?.code).toBe("too_large"); + }); +}); + +describe("compression at the origin", () => { + it("deflates a JSON part above 16 KiB when the host compresses, and counts it", async () => { + const { context, counts } = await metered(); + const small = json(COMPRESS_ABOVE / 3); + expect(originPart(small, nodeCompression, context)).toBe(small); + const large = json(COMPRESS_ABOVE + 1); + const part = originPart(large, nodeCompression, context); + expect(part).toBeInstanceOf(DeflatedPart); + expect((part as DeflatedPart).bytes.byteLength).toBeLessThan(200); + expect(counts()["chord.link.deflates"]).toBe(1); + }); + + it("measures the threshold in UTF-8 bytes: a text of few units but many bytes is compressed", async () => { + const { context } = await metered(); + const text = JSON.stringify("界".repeat(6000)); + expect(text.length).toBeLessThan(COMPRESS_ABOVE); + expect(originPart(text, nodeCompression, context)).toBeInstanceOf(DeflatedPart); + // Within the threshold in bytes: its bytes, encoded once (the codec copies them). + const within = json(COMPRESS_ABOVE); + expect(originPart(within, nodeCompression, context)).toEqual(UTF8.encode(within)); + }); + + it("sends a part as it is without a compressor, and its bytes when deflating does not make it smaller", async () => { + const { context } = await metered(); + const large = json(COMPRESS_ABOVE * 2); + expect(originPart(large, undefined, context)).toBe(large); + const useless: Compression = { + deflate: (bytes) => bytes, + inflate: (bytes) => ({ kind: "bytes", bytes }), + }; + expect(originPart(large, useless, context)).toEqual(UTF8.encode(large)); + }); +}); + +describe("opening a part", () => { + it("returns a plain part within its limits; past either it throws too_large/part", async () => { + const { context } = await metered(); + expect(openPart("[1]", NONE, undefined, context)).toBe("[1]"); + expect(() => openPart("[1,2]", { part: 4, decoded: 100 }, undefined, context)).toThrow( + "a part of 5 bytes is over the wire limit of 4 bytes", + ); + // Not deflated: its decoded size is its size. + expect(() => openPart(UTF8.encode("[1,2]"), { part: 100, decoded: 4 }, undefined, context)).toThrow( + "a part of 5 bytes is over the decoded limit of 4 bytes", + ); + }); + + it("inflates a deflated part synchronously within the decoded limit, and counts it", async () => { + const { context, counts } = await metered(); + const text = json(100_000); + const part = new DeflatedPart(nodeCompression.deflate(UTF8.encode(text))); + const opened = openPart(part, { part: 1000, decoded: 100_000 }, nodeCompression, context); + expect(new TextDecoder().decode(opened as Uint8Array)).toBe(text); + expect(counts()["chord.link.inflates"]).toBe(1); + }); + + it("refuses a compression bomb: a small part inflating past the decoded limit", async () => { + const { context } = await metered(); + const bomb = new DeflatedPart(nodeCompression.deflate(new Uint8Array(10 * 1024 * 1024))); + expect(bomb.bytes.byteLength).toBeLessThan(50_000); + const error = thrown(() => openPart(bomb, { part: 1_000_000, decoded: 1024 * 1024 }, nodeCompression, context)); + expect(error).toMatchObject({ code: "too_large", reason: "part" }); + }); + + it("checks the wire size before inflating anything", async () => { + const { context, counts } = await metered(); + const part = new DeflatedPart(nodeCompression.deflate(UTF8.encode(json(100_000)))); + const error = thrown(() => openPart(part, { part: 10, decoded: 1_000_000 }, nodeCompression, context)); + expect(error).toMatchObject({ code: "too_large", message: expect.stringContaining("wire limit") }); + expect(counts()["chord.link.inflates"]).toBeUndefined(); + }); + + it("opens what an asynchronous inflate left on the part", async () => { + const { context } = await metered(); + const part = new DeflatedPart(nodeCompression.deflate(UTF8.encode(json(1000)))); + await inflateAsync(part, 2000, NEVER, context); + expect(openPart(part, { part: 1000, decoded: 2000 }, undefined, context)).toHaveLength(1000); + const bomb = new DeflatedPart(nodeCompression.deflate(UTF8.encode(json(1000)))); + await inflateAsync(bomb, 999, NEVER, context); + expect(() => openPart(bomb, { part: 1000, decoded: 999 }, undefined, context)).toThrow("decoded limit of 999"); + }); + + it("throws a plain error for a deflated part nothing inflated, or no raw deflate stream (a protocol error)", async () => { + const { context } = await metered(); + const part = new DeflatedPart(new Uint8Array([255, 255, 255])); + expect(() => openPart(part, NONE, undefined, context)).toThrow("a deflated part this host did not inflate"); + expect(() => openPart(part, NONE, nodeCompression, context)).toThrow("no raw deflate stream"); + expect(thrown(() => openPart(part, NONE, nodeCompression, context))).not.toMatchObject({ code: "too_large" }); + }); +}); + +describe("the asynchronous inflate", () => { + it("inflates with DecompressionStream, counting output bytes", async () => { + const { context, counts } = await metered(); + const text = json(200_000); + const part = new DeflatedPart(nodeCompression.deflate(UTF8.encode(text))); + await inflateAsync(part, Number.POSITIVE_INFINITY, NEVER, context); + expect(part.inflated?.kind).toBe("bytes"); + expect(new TextDecoder().decode(openPart(part, NONE, undefined, context) as Uint8Array)).toBe(text); + expect(counts()["chord.link.inflates"]).toBe(1); + }); + + it("stops past the limit: a bomb leaves too_large, never the whole output; a limit of 0 too", async () => { + const { context } = await metered(); + const bomb = new DeflatedPart(nodeCompression.deflate(new Uint8Array(64 * 1024 * 1024))); + await inflateAsync(bomb, 1024 * 1024, NEVER, context); + expect(bomb.inflated).toEqual({ kind: "too_large" }); + const one = new DeflatedPart(nodeCompression.deflate(new Uint8Array(1))); + await inflateAsync(one, 0, NEVER, context); + expect(one.inflated).toEqual({ kind: "too_large" }); + }); + + it("leaves corrupt for data that is no raw deflate stream", async () => { + const { context } = await metered(); + const part = new DeflatedPart(new Uint8Array([255, 255, 255])); + await inflateAsync(part, 100, NEVER, context); + expect(part.inflated?.kind).toBe("corrupt"); + }); + + it("stops at once when its link closes (the signal), leaving no outcome", async () => { + const { context } = await metered(); + const closing = new AbortController(); + const part = new DeflatedPart(nodeCompression.deflate(new Uint8Array(64 * 1024 * 1024))); + const inflating = inflateAsync(part, Number.POSITIVE_INFINITY, closing.signal, context); + closing.abort(); + await inflating; + expect(part.inflated).toBeUndefined(); + }); +}); diff --git a/packages/chord/test/next/link/values.test.ts b/packages/chord/test/next/link/values.test.ts new file mode 100644 index 000000000..84656b516 --- /dev/null +++ b/packages/chord/test/next/link/values.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { jsonPart, parsePart, parsePayload, payloadPart, resolveSegments } from "../../../src/next/link/values.ts"; + +describe("parts", () => { + it("are JSON text at their origin, parsed from text or from UTF-8 bytes where they are owned", () => { + expect(jsonPart({ a: [1, "x"] })).toBe('{"a":[1,"x"]}'); + expect(jsonPart(undefined)).toBe("null"); + expect(parsePart('{"a":1}')).toEqual({ a: 1 }); + expect(parsePart(new TextEncoder().encode('{"a":"\u00e9"}'))).toEqual({ a: "\u00e9" }); + }); + + it("that are no strict UTF-8 JSON throw (a protocol error where they are parsed)", () => { + expect(() => parsePart("{")).toThrow(); + expect(() => parsePart(new Uint8Array([0x22, 0xff, 0x22]))).toThrow(); + }); +}); + +describe("payloads", () => { + it("marked as carrying bytes write each byte leaf as a segment referenced by {$bytes: i}", () => { + const data = new Uint8Array(1024).fill(7); + const thumbs = [new Uint8Array([1, 2]), new Uint8Array([3])]; + const { json, segments } = payloadPart({ name: "chart.png", data, thumbs }, true); + expect(json).toBe('{"name":"chart.png","data":{"$bytes":0},"thumbs":[{"$bytes":1},{"$bytes":2}]}'); + expect(segments).toEqual([data, ...thumbs]); + expect(segments![0]).toBe(data); + const read = parsePayload(json, segments) as { data: Uint8Array; thumbs: Uint8Array[] }; + expect(read.data).toBe(data); + expect(read.thumbs).toEqual(thumbs); + }); + + it("marked as carrying bytes refuse the reserved key $bytes with invalid/args", () => { + let thrown: unknown; + try { + payloadPart({ meta: { $bytes: 0 }, data: new Uint8Array(1) }, true); + } catch (error) { + thrown = error; + } + expect(isChordError(thrown, "invalid") && thrown.reason).toBe("args"); + }); + + it("not marked are plain JSON, $bytes included; undefined is null", () => { + expect(payloadPart({ $bytes: 0 }, false)).toEqual({ json: '{"$bytes":0}', segments: undefined }); + expect(payloadPart(undefined, false).json).toBe("null"); + expect(parsePayload('{"$bytes":0}', undefined)).toEqual({ $bytes: 0 }); + }); + + it.each([ + ["an index past the segments", { $bytes: 1 }], + ["a non-integer index", { $bytes: "0" }], + ["other keys next to $bytes", { $bytes: 0, x: 1 }], + ])("refuse a malformed reference: %s", (_, value) => { + expect(() => resolveSegments({ a: [value] }, [new Uint8Array(1)])).toThrow( + "payload: a malformed byte segment reference", + ); + }); + + it("define a resolved key __proto__ as an own property", () => { + const parsed = JSON.parse('{"__proto__":{"$bytes":0}}'); + const resolved = resolveSegments(parsed, [new Uint8Array(1)]) as object; + expect(Object.getPrototypeOf(resolved)).toBe(Object.prototype); + expect(Object.getOwnPropertyDescriptor(resolved, "__proto__")?.value).toEqual(new Uint8Array(1)); + }); +}); diff --git a/packages/chord/test/next/link/websocket.test.ts b/packages/chord/test/next/link/websocket.test.ts new file mode 100644 index 000000000..821c1468e --- /dev/null +++ b/packages/chord/test/next/link/websocket.test.ts @@ -0,0 +1,159 @@ +import type { AddressInfo } from "node:net"; +import { describe, expect, it } from "vitest"; +import { WebSocket, WebSocketServer } from "ws"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { type WebSocketLike, webSocketLink } from "../../../src/next/link/index.ts"; +import { Notes, serveNotes, type TestPrincipal, transported } from "../support.ts"; + +/** A socket playing a WebSocket by hand. */ +class FakeSocket implements WebSocketLike { + readyState: number; + binaryType = "blob"; + readonly sent: Uint8Array[] = []; + closes = 0; + readonly #listeners = new Map void>>(); + + constructor(readyState: number) { + this.readyState = readyState; + } + + send(data: Uint8Array): void { + this.sent.push(data); + } + + close(): void { + this.closes++; + this.readyState = 3; + } + + addEventListener(type: string, listener: (event: { readonly data: unknown }) => void): void { + const listeners = this.#listeners.get(type) ?? new Set(); + listeners.add(listener); + this.#listeners.set(type, listeners); + } + + removeEventListener(type: string, listener: (event: { readonly data: unknown }) => void): void { + this.#listeners.get(type)?.delete(listener); + } + + emit(type: string, data?: unknown): void { + if (type === "open") this.readyState = 1; + for (const listener of [...(this.#listeners.get(type) ?? [])]) listener({ data }); + } + + get listeners(): number { + return [...this.#listeners.values()].reduce((count, listeners) => count + listeners.size, 0); + } +} + +describe("WebSocket links over a fake socket", () => { + it("buffer messages sent while connecting and send them on open, as binary frames (race 29)", () => { + const socket = new FakeSocket(0); + const link = webSocketLink(socket); + expect(socket.binaryType).toBe("arraybuffer"); + link.send(new Uint8Array([1])); + link.send(new Uint8Array([2])); + expect(socket.sent).toEqual([]); + socket.emit("open"); + expect(socket.sent.map((message) => message[0])).toEqual([1, 2]); + const received: number[] = []; + link.onMessage((message) => received.push(message[0]!)); + socket.emit("message", new Uint8Array([7]).buffer); + expect(received).toEqual([7]); + }); + + it("give a closed link for a socket closing or closed already", () => { + for (const state of [2, 3]) { + let closed = false; + webSocketLink(new FakeSocket(state)).onClose(() => { + closed = true; + }); + expect(closed).toBe(true); + } + }); + + it.each([ + ["close", (socket: FakeSocket) => socket.emit("close")], + ["error", (socket: FakeSocket) => socket.emit("error")], + ["a text frame", (socket: FakeSocket) => socket.emit("message", "text")], + ])("close once on %s, listeners removed", (_, fail) => { + const socket = new FakeSocket(1); + const link = webSocketLink(socket); + let closes = 0; + link.onClose(() => closes++); + fail(socket); + socket.emit("close"); + link.send(new Uint8Array([1])); + expect(closes).toBe(1); + expect(socket.sent).toEqual([]); + expect(socket.listeners).toBe(0); + }); +}); + +/** A `ws` server on a free loopback port whose connections a server host accepts, and a URL to it. */ +async function loopback(options: { readonly maxPayload?: number } = {}) { + const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + await server.load([{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context); + const wss = new WebSocketServer({ port: 0, host: "127.0.0.1", maxPayload: options.maxPayload }); + await new Promise((resolve) => wss.once("listening", resolve)); + wss.on("connection", (socket) => server.accept(webSocketLink(socket), { principal: { kind: "owner" } }, context)); + const url = `ws://127.0.0.1:${(wss.address() as AddressInfo).port}`; + return { server, wss, url }; +} + +describe("WebSocket links over a loopback ws server", () => { + it("survive closing a socket that still connects: ws's late error is handled", async () => { + const errors: unknown[] = []; + const record = (error: unknown) => errors.push(error); + process.on("uncaughtException", record); + try { + const socket = new WebSocket("ws://127.0.0.1:1"); + let closed = false; + const link = webSocketLink(socket); + link.onClose(() => { + closed = true; + }); + link.close(); + expect(closed).toBe(true); + await new Promise((resolve) => socket.once("close", () => resolve())); + expect(errors).toEqual([]); + expect(socket.listenerCount("error")).toBe(0); + } finally { + process.off("uncaughtException", record); + } + }); + + it("carry a live reference both ways; the client's hello waits for open", async () => { + const { server, wss, url } = await loopback(); + const tab = createHost({ onError: () => {} }); + await tab.load([], context); + tab.connect(webSocketLink(new WebSocket(url)), { principal: { kind: "server" } }, context); + const notes = await tab.get(Notes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("expected the editor"); + expect(await notes.add({ text: "over ws" }, context)).toBe(1); + await transported(); + expect(notes.notes.value).toEqual([{ text: "over ws" }]); + await tab.close(context); + await server.close(context); + await new Promise((resolve) => wss.close(resolve)); + }); + + it("close the connection on a frame above maxPayload (1009): references go loading (race 28)", async () => { + const { server, wss, url } = await loopback({ maxPayload: 4096 }); + const tab = createHost({ onError: () => {} }); + await tab.load([], context); + const socket = new WebSocket(url); + const code = new Promise((resolve) => socket.once("close", resolve)); + tab.connect(webSocketLink(socket), { principal: { kind: "server" } }, context); + const notes = await tab.get(Notes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("expected the editor"); + const call = notes.echo({ data: new Uint8Array(10_000) }, context).catch((error: unknown) => error); + expect(await code).toBe(1009); + expect(await call).toMatchObject({ code: "unavailable" }); + expect(notes.state.value).toEqual({ kind: "loading" }); + await tab.close(context); + await server.close(context); + await new Promise((resolve) => wss.close(resolve)); + }); +}); diff --git a/packages/chord/test/next/remote/calls.test.ts b/packages/chord/test/next/remote/calls.test.ts new file mode 100644 index 000000000..79f3981da --- /dev/null +++ b/packages/chord/test/next/remote/calls.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import { CallWindow, type QueuedCall } from "../../../src/next/remote/calls.ts"; +import { expectCollected } from "../support.ts"; + +/** A call of `hold` that records its name in `sent` when it goes out. */ +function callOf(name: string, sent: string[], hold = 0): QueuedCall { + return { hold, send: () => void sent.push(name) }; +} + +describe("the call window", () => { + it("sends nothing before the peer's hello, then up to its N; the rest waits", () => { + const sent: string[] = []; + const window = new CallWindow(); + for (const name of ["a", "b", "c"]) window.submit(callOf(name, sent)); + expect(sent).toEqual([]); + window.opened(2); + expect(sent).toEqual(["a", "b"]); + window.freed(); + expect(sent).toEqual(["a", "b", "c"]); + }); + + it("sends waiting calls in order as slots free", () => { + const sent: string[] = []; + const window = new CallWindow(); + window.opened(1); + for (const name of ["a", "b", "c", "d"]) window.submit(callOf(name, sent)); + for (let slot = 0; slot < 3; slot++) window.freed(); + expect(sent).toEqual(["a", "b", "c", "d"]); + }); + + it("never preempts: a call that waits does not overtake while every slot is taken", () => { + const sent: string[] = []; + const window = new CallWindow(); + window.opened(1); + window.submit(callOf("long", sent)); + window.submit(callOf("short", sent)); + expect(sent).toEqual(["long"]); + }); + + it("takes out a cancelled waiting call, and every waiting call of a hold that rebinds", () => { + const sent: string[] = []; + const window = new CallWindow(); + window.opened(1); + window.submit(callOf("first", sent)); + const cancelled = callOf("cancelled", sent); + window.submit(cancelled); + const rebinding = [callOf("hold 1 a", sent, 1), callOf("hold 1 b", sent, 1)]; + for (const call of rebinding) window.submit(call); + window.submit(callOf("last", sent)); + window.remove(cancelled); + // A second remove of the same call changes nothing. + window.remove(cancelled); + expect(window.removeHold(1)).toEqual(rebinding); + window.freed(); + expect(sent).toEqual(["first", "last"]); + expect(window.clear()).toEqual([]); + }); + + it("keeps no call that went out or was taken", async () => { + const window = new CallWindow(); + window.opened(1); + const refs = (() => { + const sent = callOf("a", []); + const next = callOf("b", []); + const waiting = callOf("c", []); + for (const call of [sent, next, waiting]) window.submit(call); + window.freed(); + window.remove(waiting); + return [new WeakRef(sent), new WeakRef(waiting)]; + })(); + await expectCollected(...refs); + expect(window).toBeDefined(); + }); +}); diff --git a/packages/chord/test/next/remote/consumer.test.ts b/packages/chord/test/next/remote/consumer.test.ts new file mode 100644 index 000000000..32e4fb839 --- /dev/null +++ b/packages/chord/test/next/remote/consumer.test.ts @@ -0,0 +1,718 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { int, local, map, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { PluginError } from "../../../src/next/data/index.ts"; +import { createHost, defineFacet, type HostError } from "../../../src/next/host/index.ts"; +import { resolveWatchable } from "../../../src/next/host/reference.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { DeflatedPart, jsonPart, type Part, wireError } from "../../../src/next/link/internal.ts"; +import { RemoteObject } from "../../../src/next/remote/remote-object.ts"; +import { nodeCompression } from "../../../src/next-node/compression.ts"; +import { + addNote, + bindNext, + EDITOR, + expectCollected, + forEachCodec, + framesOf, + instancesOf, + linked, + metered, + Notes, + RawPeer, + rawParent, + rawProvider, + rejection, + serveNotes, + testCodec, + transported, +} from "../support.ts"; + +/** Every remote object bound while this file runs, weakly. */ +const remoteObjects = instancesOf(RemoteObject, "hold"); + +forEachCodec(() => { + describe("references over a link", () => { + it("are live and hydrated when got, with the variant of the link's principal", async () => { + const { client, server } = await linked((h) => void serveNotes(h), { + child: { kind: "guest", session: "S1" }, + }); + await addNote(server, "S1", "a"); + const notes = await client.get(Notes, { session: "S1" }, context); + expect(notes.variant).toBe("reader"); + expect(notes.notes.value).toEqual([{ text: "a" }]); + expect(notes.count.value).toBe(1); + const frames = framesOf(await notes.notes.watch(context)); + await addNote(server, "S1", "b"); + await transported(); + expect(frames).toEqual([[[{ text: "a" }, { text: "b" }], [["p", [], 1, 0, [{ text: "b" }]]]]]); + expect(notes.count.value).toBe(2); + }); + + it("apply one state message as one batch: a derive over two members never sees half a commit", async () => { + const View = local("app.view", { version: 1, params: {}, members: { pair: str() } }); + const { server } = await linked((h) => void serveNotes(h)); + const seen: string[] = []; + const client = createHost({ onError: () => {} }); + // Connected first (W6): the view's factory references a server type. + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + client.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); + await client.load( + [ + { + name: "@test/view", + facet: defineFacet((h) => + h.serve(View, (_, self) => { + const notes = self.get(Notes, { session: "S1" }); + return { + pair: self.derive([notes.notes, notes.count], ([list, count]) => { + seen.push(`${list.length}/${count}`); + return `${list.length}/${count}`; + }), + }; + }), + ), + }, + ], + context, + ); + const view = await client.get(View, {}, context); + const frames = framesOf(await view.pair.watch(context)); + await addNote(server, "S1", "a"); + await addNote(server, "S1", "b"); + await transported(); + expect(frames.map(([value]) => value)).toEqual(["1/1", "2/2"]); + expect(seen.every((pair) => pair.split("/")[0] === pair.split("/")[1])).toBe(true); + }); + + it("call methods: results, ChordErrors, PluginErrors and other throws", async () => { + const { client } = await linked((h) => void serveNotes(h)); + const notes = await client.get(Notes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("expected the editor"); + expect(await notes.add({ text: "x" }, context)).toBe(1); + const chord = await rejection(notes.fail({ how: "chord" }, context)); + expect(chord).toEqual({ code: "forbidden", message: "not yours" }); + const plugin = await notes.fail({ how: "plugin" }, context).catch((error: unknown) => error); + expect(plugin).toBeInstanceOf(PluginError); + expect(plugin).toMatchObject({ code: "quota", message: "too many", details: { limit: 1 } }); + expect(await notes.fail({ how: "plain" }, context).catch((error: unknown) => error)).toMatchObject({ + name: "PluginError", + code: "internal", + message: "plain", + }); + expect(await rejection(notes.add({ text: 1 } as never, context))).toMatchObject({ + code: "invalid", + reason: "args", + }); + }); + + it("carry bytes as raw segments both ways", async () => { + const { client } = await linked((h) => void serveNotes(h)); + const notes = await client.get(Notes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("expected the editor"); + const data = new Uint8Array([1, 2, 3]); + const echoed = await notes.echo({ data }, context); + expect(echoed).toBeInstanceOf(Uint8Array); + expect([...echoed]).toEqual([1, 2, 3]); + }); + + it("cancel a call: the caller is rejected at once, the provider's handler aborted", async () => { + let probe: ReturnType | undefined; + const { client } = await linked((h) => { + probe = serveNotes(h); + }); + const notes = await client.get(Notes, { session: "S1" }, context); + const { context: cancellable, cancel } = withCancel(context); + const call = notes.wait(cancellable); + await transported(); + cancel(new Error("user cancelled")); + await expect(call).rejects.toThrow("user cancelled"); + await transported(); + expect(probe!.waits[0]!.abortSignal?.aborted).toBe(true); + }); + + it("release: pending calls reject released, the provider aborts their handlers and lets the object go", async () => { + let probe: ReturnType | undefined; + const { client } = await linked((h) => { + probe = serveNotes(h); + }); + const notes = await client.get(Notes, { session: "S1" }, context); + const watch = await notes.notes.watch(context); + const call = notes.wait(context); + await transported(); + notes.release(); + expect(await rejection(call)).toEqual({ code: "released", message: "app.notes(S1): released" }); + expect(await watch.closed).toEqual({ kind: "ended" }); + await transported(); + expect(probe!.waits[0]!.abortSignal?.aborted).toBe(true); + // The object ended with its last reference: a new get runs the factory again. + await client.get(Notes, { session: "S1" }, context); + expect(probe!.factories).toEqual(["S1 owner", "S1 owner"]); + }); + + it("a closing link rejects sent calls unavailable; references go loading, gets keep waiting", async () => { + const { client, peer } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + await bindNext(peer); + const notes = await got; + const call = notes.wait(context); + const pending = client.get(Notes, { session: "S2" }, context, { readyTimeout: 50 }); + await peer.next("call"); + peer.close(); + expect(await rejection(call)).toEqual({ code: "unavailable", message: "the link closed" }); + expect(notes.state.value).toEqual({ kind: "loading" }); + expect(await rejection(pending)).toEqual({ + code: "unavailable", + message: "app.notes(S2): did not become live within 50 ms", + }); + }); + }); + + describe("races at the consumer", () => { + it("ignores a bound and a state for a hold it released, and a result crossing its cancel", async () => { + const { client, peer, errors } = await rawProvider(); + const late = client.get(Notes, { session: "S1" }, context, { readyTimeout: 10 }); + const { hold } = await peer.next("hold"); + await expect(late).rejects.toThrow("did not become live"); + const members = [ + { member: EDITOR.state.notes!, value: "[]" }, + { member: EDITOR.state.count!, value: "0" }, + ]; + peer.send({ kind: "bound", holds: [{ hold, variant: EDITOR.variant, members, failed: [] }] }); + peer.send({ + kind: "state", + holds: [ + { hold, members: [{ member: EDITOR.state.count!, ops: jsonPart([["r", 1]]) }], failed: [], retired: [] }, + ], + }); + const got = client.get(Notes, { session: "S1" }, context); + await bindNext(peer); + const notes = await got; + const { context: cancellable, cancel } = withCancel(context); + const call = notes.wait(cancellable); + const { call: id } = await peer.next("call"); + cancel(new Error("cancelled")); + await expect(call).rejects.toThrow("cancelled"); + await peer.next("abort"); + peer.send({ kind: "result", call: id }); + await transported(); + expect(peer.closed).toBe(false); + expect(errors).toEqual([]); + }); + }); + + describe("protocol errors at the consumer", () => { + it.each([ + ["wire ops failing Delta's checks", [{ member: EDITOR.state.count!, ops: '[["x"]]' }]], + ["a member index outside the variant", [{ member: 9, ops: '[["r",1]]' }]], + [ + "a member twice", + [ + { member: EDITOR.state.count!, ops: '[["r",1]]' }, + { member: EDITOR.state.count!, ops: '[["r",2]]' }, + ], + ], + ["a member's changes that are no list", [{ member: EDITOR.state.count!, ops: "1" }]], + ["ops that are no JSON", [{ member: EDITOR.state.count!, ops: "[" }]], + ["ops that do not apply to the member", [{ member: EDITOR.state.count!, ops: '[["s",["x"],1]]' }]], + ])("close the link: %s", async (_, members) => { + const { client, peer, errors } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + const hold = await bindNext(peer); + const notes = await got; + peer.send({ kind: "state", holds: [{ hold, members, failed: [], retired: [] }] }); + await transported(); + expect(peer.closed).toBe(true); + expect(errors.map(({ kind }) => kind)).toEqual(["protocol_error"]); + // Nothing of the message was applied; the reference waits for the next link. + expect(notes.state.value).toEqual({ kind: "loading" }); + expect(notes.count.value).toBe(0); + }); + + it.each([ + ["a result that is no JSON", { kind: "result", call: 0, value: "{" }], + ["a $bytes reference without its segment", { kind: "result", call: 0, value: '{"$bytes":0}', segments: [] }], + ["a malformed ChordError envelope", { kind: "error", call: 0, error: { kind: "chord", error: { code: 1 } } }], + ] as const)("reject the call and close the link on a malformed reply: %s", async (_, reply) => { + const { client, peer, errors } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + await bindNext(peer); + const notes = await got; + const call = notes.wait(context); + await peer.next("call"); + peer.send(reply); + // The text codec cannot even decode a result that is no JSON: the link closes and the call with it. + expect(await rejection(call)).toMatchObject({ code: "unavailable" }); + expect(peer.closed).toBe(true); + expect(errors.map(({ kind }) => kind)).toEqual(["protocol_error"]); + }); + + it("ignore a gone for a hold it does not know before reading its error (Q1)", async () => { + const { client, peer, errors } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + await bindNext(peer); + await got; + peer.send({ kind: "gone", hold: 9, error: { kind: "chord", error: { code: "nope" } } }); + await transported(); + expect(peer.closed).toBe(false); + expect(errors).toEqual([]); + }); + + it("close the link on a malformed error envelope", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + peer.send({ + kind: "failed", + hold, + cause: { kind: "broken", error: { kind: "chord", error: { code: "nope", message: "m" } } }, + }); + await transported(); + expect(peer.closed).toBe(true); + expect(notes.state.value).toEqual({ kind: "loading" }); + }); + }); + + describe("the wire format of a hold", () => { + it("is the snapshot by member, then each commit's changed members with their ops", async () => { + const { server } = await linked((h) => void serveNotes(h)); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + peer.send({ + kind: "hold", + hold: 7, + type: Notes.name, + version: Notes.version, + fingerprint: Notes.fingerprint, + params: jsonPart({ session: "S1" }), + }); + const [bound] = (await peer.next("bound")).holds; + expect(bound!.variant).toBe(EDITOR.variant); + expect(EDITOR.named(bound!.members)).toEqual({ notes: [], count: 0 }); + await addNote(server, "S1", "a"); + const state = await peer.next("state"); + expect(state.holds).toMatchObject([{ hold: 7, failed: [] }]); + expect(EDITOR.named(state.holds[0]!.members)).toEqual({ + notes: [["p", [], 0, 0, [{ text: "a" }]]], + count: [["r", 1]], + }); + }); + }); + + describe("cost guards at the consumer", () => { + it("applies state without validating, walking or copying it: unchanged parts stay the same objects", async () => { + const { context: counting, counts } = await metered(); + const { server } = await linked((h) => void serveNotes(h)); + const client = createHost({ onError: () => {} }); + await client.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + client.connect(b, { principal: { kind: "server" }, codec: testCodec() }, counting); + await addNote(server, "S1", "a"); + const notes = await client.get(Notes, { session: "S1" }, context); + const first = notes.notes.value[0]; + await addNote(server, "S1", "b"); + await transported(); + expect(notes.notes.value).toEqual([{ text: "a" }, { text: "b" }]); + expect(notes.notes.value[0]).toBe(first); + expect(counts()["chord.data.validations"]).toBeUndefined(); + expect(counts()["chord.data.walks"]).toBeUndefined(); + }); + }); + + describe("GC at the consumer", () => { + it("a closed link retained by the test, with its host, keeps no hold, replica or released reference", async () => { + const { client, link, refs } = await (async () => { + const { client, peer } = await rawProvider(); + const made = remoteObjects.length; + const got = client.get(Notes, { session: "S1" }, context); + await bindNext(peer); + const notes = await got; + const refs: WeakRef[] = [new WeakRef(resolveWatchable(notes.notes)!), new WeakRef(notes)]; + refs.push(...remoteObjects.slice(made)); + expect(refs).toHaveLength(3); + peer.close(); + await transported(); + // Across the drop the host keeps the reference's object for the next link; released, nothing stays. + notes.release(); + return { client, link: peer, refs }; + })(); + await expectCollected(...refs); + expect([client, link]).toBeDefined(); + }); + }); + + describe("one state message over several holds", () => { + const Pair = local("app.pair", { version: 1, params: {}, members: { pair: str() } }); + + /** A client whose `Pair` derives over the counts of two holds (S1, S2), both bound by a raw provider. */ + async function pair() { + const seen: string[] = []; + const errors: HostError[] = []; + const client = createHost({ onError: (error) => errors.push(error) }); + const [a, b] = memoryLinkPair(); + client.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const facet = defineFacet((h) => + h.serve(Pair, (_, self) => { + const one = self.get(Notes, { session: "S1" }); + const two = self.get(Notes, { session: "S2" }); + return { + pair: self.derive([one.count, two.count], ([x, y]) => { + seen.push(`${x}/${y}`); + return `${x}/${y}`; + }), + }; + }), + ); + const loading = client.load([{ name: "@test/pair", facet }], context); + await bindNext(peer); + await bindNext(peer); + await loading; + const view = await client.get(Pair, {}, context); + const frames = framesOf(await view.pair.watch(context)); + return { client, peer, seen, frames, errors, view }; + } + + it("is applied as one batch: a derive over both holds never sees half of it, a failed member included", async () => { + const { peer, seen, frames, view } = await pair(); + const failed = [{ member: EDITOR.state.notes!, error: wireError(new PluginError("derive", "notes threw")) }]; + const count = [{ member: EDITOR.state.count!, ops: '[["r",1]]' }]; + peer.send({ + kind: "state", + holds: [ + { hold: 0, members: count, failed: [], retired: [] }, + { hold: 1, members: count, failed, retired: [] }, + ], + }); + await transported(); + expect(frames.map(([value]) => value)).toEqual(["1/1"]); + expect(seen).toEqual(["0/0", "1/1"]); + expect(view.pair.value).toBe("1/1"); + }); + + it("with a malformed later entry applies nothing and closes the link", async () => { + const { peer, seen, frames, errors } = await pair(); + peer.send({ + kind: "state", + holds: [ + { hold: 0, members: [{ member: EDITOR.state.count!, ops: '[["r",1]]' }], failed: [], retired: [] }, + { hold: 1, members: [{ member: EDITOR.state.count!, ops: '[["x"]]' }], failed: [], retired: [] }, + ], + }); + await transported(); + expect(frames).toEqual([]); + expect(seen).toEqual(["0/0"]); + expect(peer.closed).toBe(true); + expect(errors.map(({ kind }) => kind)).toEqual(["protocol_error"]); + }); + }); + + describe("$bytes outside methods with bytes", () => { + it("is plain data: {$bytes: 0} round-trips unchanged through a method without bytes", async () => { + const Echo = remote("app.echo", { + version: 1, + params: {}, + variants: { v: { methods: { echo: method(map(int()), map(int())) } } }, + }); + const { client } = await linked((h) => + h.serve(Echo, () => ({ variant: "v", members: { echo: async (args) => args } })), + ); + const echo = await client.get(Echo, {}, context); + expect(await echo.echo({ $bytes: 0 }, context)).toEqual({ $bytes: 0 }); + }); + }); +}); + +forEachCodec(() => { + describe("the call window at the consumer", () => { + /** A client connected to a raw provider announcing one concurrent call, and two bound references. */ + async function narrow() { + const client = createHost({ onError: () => {} }); + await client.load([], context); + const [a, b] = memoryLinkPair(); + client.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + const peer = new RawPeer(b, { calls: 1 }); + const one = client.get(Notes, { session: "S1" }, context); + const first = await bindNext(peer); + const two = client.get(Notes, { session: "S2" }, context); + const second = await bindNext(peer); + return { peer, one: await one, two: await two, holds: [first, second] as const }; + } + + /** The calls the provider received, by hold. */ + const calls = (peer: RawPeer) => + peer.received.flatMap((message) => (message.kind === "call" ? [message.hold] : [])); + + it("frees a slot once when a hold with a call in flight and one waiting is released", async () => { + const { peer, one, two, holds } = await narrow(); + const sent = one.wait(context); + const waiting = one.wait(context); + const other = [two.wait(context), two.wait(context)]; + await transported(); + expect(calls(peer)).toEqual([holds[0]]); + one.release(); + expect(await rejection(sent)).toMatchObject({ code: "released" }); + expect(await rejection(waiting)).toMatchObject({ code: "released" }); + await transported(); + expect(calls(peer)).toEqual([holds[0], holds[1]]); + expect(other).toHaveLength(2); + }); + + it("frees a slot once on gone", async () => { + const { peer, one, two, holds } = await narrow(); + const sent = one.wait(context); + const other = [two.wait(context), two.wait(context)]; + await transported(); + peer.send({ + kind: "gone", + hold: holds[0], + error: { kind: "chord", error: { code: "unavailable", message: "revoked" } }, + }); + expect(await rejection(sent)).toMatchObject({ message: "revoked" }); + await transported(); + expect(calls(peer)).toEqual([holds[0], holds[1]]); + expect(other).toHaveLength(2); + }); + + it("rejects a call still waiting when its hold rebinds: unavailable (race 26)", async () => { + const { peer, one, holds } = await narrow(); + const sent = one.wait(context); + const waiting = one.wait(context); + await transported(); + peer.send({ kind: "loading", hold: holds[0] }); + expect(await rejection(waiting)).toEqual({ code: "unavailable", message: `hold ${holds[0]}: rebinding` }); + const { call } = await peer.next("call"); + peer.send({ kind: "result", call }); + expect(await sent).toBe(null); + }); + }); +}); + +forEachCodec(() => { + describe("limits per part at the consumer", () => { + /** `[member, value]` of `Notes`'s editor variant: `notes` and `count`, the notes' JSON given as text. */ + const members = (notes: string): { member: number; value: Part }[] => [ + { member: EDITOR.state.notes!, value: notes }, + { member: EDITOR.state.count!, value: "1" }, + ]; + + it("rejects a call whose result is past the link's limits with too_large/part; the link stays", async () => { + const { client, peer } = await rawProvider({ limits: { part: 50 } }); + const notes = client.hold(Notes, { session: "S1" }, context); + await bindNext(peer); + await transported(); + if (notes.variant !== "editor") throw new Error("bound as the editor"); + const added = notes.add({ text: "x" }, context); + const { call } = await peer.next("call"); + peer.send({ kind: "result", call, value: jsonPart("y".repeat(60)) }); + expect(await rejection(added)).toMatchObject({ code: "too_large", reason: "part" }); + const again = notes.add({ text: "x" }, context); + peer.send({ kind: "result", call: (await peer.next("call")).call, value: "2" }); + expect(await again).toBe(2); + }); + + it("fails a hold whose snapshot is past the limits broken, releases it, and binds the message's others", async () => { + const { client, peer } = await rawProvider({ limits: { part: 50 } }); + const big = client.hold(Notes, { session: "S1" }, context); + const small = client.hold(Notes, { session: "S2" }, context); + const [one, two] = [await peer.next("hold"), await peer.next("hold")]; + const holds = [ + { + hold: one.hold, + variant: EDITOR.variant, + members: members(jsonPart([{ text: "z".repeat(60) }])), + failed: [], + }, + { hold: two.hold, variant: EDITOR.variant, members: members('[{"text":"a"}]'), failed: [] }, + ]; + peer.send({ kind: "bound", holds }); + expect(await peer.next("release")).toEqual({ kind: "release", hold: one.hold }); + expect(await peer.next("ack")).toEqual({ kind: "ack" }); + expect(big.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "broken", error: { code: "too_large" } }, + }); + expect(small.notes.value).toEqual([{ text: "a" }]); + // No automatic re-hold; a retry holds again. + await transported(); + expect(peer.kinds()).toEqual([]); + big.retry(); + expect(big.state.value.kind).toBe("loading"); + const again = await peer.next("hold"); + expect(again.hold).toBeGreaterThan(two.hold); + }); + + it("acknowledges a message waiting for its asynchronous inflate only once applied, in order (race 14)", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + const list = [{ text: "q".repeat(30_000) }]; + const value = new DeflatedPart(nodeCompression.deflate(new TextEncoder().encode(JSON.stringify(list)))); + const bound = { hold, variant: EDITOR.variant, members: [...members("[]")], failed: [] }; + bound.members[0] = { member: EDITOR.state.notes!, value }; + peer.send({ kind: "bound", holds: [bound] }); + const ops = jsonPart([["r", 2]]); + peer.send({ + kind: "state", + holds: [{ hold, members: [{ member: EDITOR.state.count!, ops }], failed: [], retired: [] }], + }); + await peer.next("ack"); + expect(notes.notes.value).toEqual(list); + await peer.next("ack"); + expect(notes.count.value).toBe(2); + }); + + it("drops a message whose asynchronous inflate is in flight when the link closes, and those behind it (race 13)", async () => { + const { client, peer, errors } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + const list = [{ text: "q".repeat(30_000) }]; + const value = new DeflatedPart(nodeCompression.deflate(new TextEncoder().encode(JSON.stringify(list)))); + const bound = { hold, variant: EDITOR.variant, members: [...members("[]")], failed: [] }; + bound.members[0] = { member: EDITOR.state.notes!, value }; + peer.send({ kind: "bound", holds: [bound] }); + peer.close(); + for (let i = 0; i < 20; i++) await new Promise((resolve) => setTimeout(resolve, 1)); + expect(notes.state.value.kind).toBe("loading"); + expect(errors).toEqual([]); + }); + + it("inflates nothing it gave up (Q1): a corrupt deflated result of an unknown call, an entry of a released hold (both paths)", async () => { + for (const compression of [nodeCompression, undefined]) { + const { client, peer, errors } = await rawProvider({ compression }); + const kept = client.hold(Notes, { session: "S1" }, context); + const released = client.hold(Notes, { session: "S2" }, context); + const [a, b] = [await bindNext(peer), await bindNext(peer)]; + await transported(); + released.release(); + await peer.next("release"); + const corrupt = () => new DeflatedPart(new Uint8Array([255, 255, 255])); + peer.send({ kind: "result", call: 999, value: corrupt() }); + const ops = jsonPart([["r", 4]]); + peer.send({ + kind: "state", + holds: [ + { hold: b, members: [{ member: EDITOR.state.count!, ops: corrupt() }], failed: [], retired: [] }, + { hold: a, members: [{ member: EDITOR.state.count!, ops }], failed: [], retired: [] }, + ], + }); + await peer.next("ack"); + expect(kept.count.value).toBe(4); + expect(peer.closed).toBe(false); + expect(errors).toEqual([]); + } + }); + + it("checks a deflated part's wire size before inflating it: corrupt data past the limit only fails its hold", async () => { + for (const compression of [nodeCompression, undefined]) { + const { client, peer, errors } = await rawProvider({ compression, limits: { part: 10 } }); + const notes = client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + const value = new DeflatedPart(new Uint8Array(20).fill(255)); + peer.send({ + kind: "bound", + holds: [ + { + hold, + variant: EDITOR.variant, + members: [ + { member: EDITOR.state.notes!, value }, + { member: EDITOR.state.count!, value: "1" }, + ], + failed: [], + }, + ], + }); + expect(await peer.next("release")).toEqual({ kind: "release", hold }); + expect(notes.state.value).toMatchObject({ kind: "failed", cause: { error: { code: "too_large" } } }); + expect(errors).toEqual([]); + } + }); + + it("keeps a hold failed for a part past the limits off the next link until retry, offline retry included", async () => { + const { client, peer } = await rawProvider({ limits: { part: 50 } }); + const notes = client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + peer.send({ + kind: "bound", + holds: [ + { hold, variant: EDITOR.variant, members: members(jsonPart([{ text: "z".repeat(60) }])), failed: [] }, + ], + }); + await peer.next("release"); + peer.close(); + await transported(); + const next = await rawParent(client, { part: 50 }); + await transported(); + // A reconnect holds nothing again: the failure is final until retry. + expect(next.kinds()).toEqual([]); + expect(notes.state.value.kind).toBe("failed"); + next.close(); + await transported(); + // A retry while offline holds it on the next link. + notes.retry(); + expect(notes.state.value.kind).toBe("loading"); + const third = await rawParent(client); + expect((await third.next("hold")).type).toBe(Notes.name); + }); + + it("stops an asynchronous inflate at once when the link closes: nothing of the link stays (GC)", async () => { + // Decompression that never yields: only the close ends it. + const streams: WeakRef[] = []; + class Stalled { + readonly writable: WritableStream; + readonly readable: ReadableStream; + constructor() { + const stream = new TransformStream({ transform() {} }); + [this.writable, this.readable] = [stream.writable, stream.readable]; + streams.push(new WeakRef(this)); + } + } + vi.stubGlobal("DecompressionStream", Stalled); + try { + const refs = await (async () => { + const { client, peer } = await rawProvider(); + client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + const value = new DeflatedPart(nodeCompression.deflate(new TextEncoder().encode("[]"))); + const entry = { hold, variant: EDITOR.variant, members: members("[]"), failed: [] }; + entry.members[0] = { member: EDITOR.state.notes!, value }; + peer.send({ kind: "bound", holds: [entry] }); + await transported(); + expect(streams).toHaveLength(1); + peer.close(); + await transported(); + await client.close(context); + return [new WeakRef(client), ...streams]; + })(); + await expectCollected(...refs); + } finally { + vi.unstubAllGlobals(); + } + }); + + it("fails only the hold of a state part past the limits (race 10); the others of the message apply", async () => { + const { client, peer } = await rawProvider({ limits: { part: 50 } }); + const one = client.hold(Notes, { session: "S1" }, context); + const two = client.hold(Notes, { session: "S2" }, context); + const [a, b] = [await bindNext(peer), await bindNext(peer)]; + await transported(); + const ops = (count: number, pad: number) => jsonPart([["r", count], ...Array(pad).fill(["r", count])]); + peer.send({ + kind: "state", + holds: [ + { hold: a, members: [{ member: EDITOR.state.count!, ops: ops(5, 10) }], failed: [], retired: [] }, + { hold: b, members: [{ member: EDITOR.state.count!, ops: ops(7, 0) }], failed: [], retired: [] }, + ], + }); + expect(await peer.next("release")).toEqual({ kind: "release", hold: a }); + await transported(); + expect(one.state.value).toMatchObject({ kind: "failed", cause: { error: { code: "too_large" } } }); + expect(two.count.value).toBe(7); + expect(peer.closed).toBe(false); + }); + }); +}); diff --git a/packages/chord/test/next/remote/hold.test.ts b/packages/chord/test/next/remote/hold.test.ts new file mode 100644 index 000000000..ab1c07b8a --- /dev/null +++ b/packages/chord/test/next/remote/hold.test.ts @@ -0,0 +1,694 @@ +import { describe, expect, it, vi } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import type * as Delta from "../../../src/delta/index.ts"; +import { decoder } from "../../../src/delta/index.ts"; +import { bool, int, local, method, obj, remote } from "../../../src/next/contracts/index.ts"; +import { broken } from "../../../src/next/data/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Host, type Self } from "../../../src/next/host/index.ts"; +import type { ChordHost } from "../../../src/next/host/internal.ts"; +import { type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; +import { jsonPart } from "../../../src/next/link/internal.ts"; +import { Hold } from "../../../src/next/remote/hold.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { + changesOf, + expectCollected, + fakeTimers, + forEachCodec, + instancesOf, + linked, + RawPeer, + rawLinked, + type TestPrincipal, + testCodec, + transported, + wireNames, +} from "../support.ts"; + +/** Every Delta encoder made while this file runs, weakly: the holds' member encoders among them. */ +const encoders = vi.hoisted(() => [] as WeakRef[]); + +vi.mock("../../../src/delta/index.ts", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + encoder: () => { + const made = original.encoder(); + encoders.push(new WeakRef(made)); + return made; + }, + }; +}); + +/** Every hold started while this file runs, weakly. */ +const holds = instancesOf(Hold, "start"); + +const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + +/** `half` throws once `n` is 2: a member failing after the object is live (W1). */ +const Halves = remote("app.halves", { + version: 1, + params: {}, + variants: { v: { state: { n: obj({ n: int() }), half: int() }, methods: { set: method(int()), wait: method() } } }, +}); + +/** The wire's indexes of `Halves`. */ +const HALVES = wireNames(Halves, "v"); + +/** A server serving `Halves` and a raw peer; `set` writes `n` through hold 0, the object of the link's principal. */ +async function halves() { + const linked = await rawLinked((h) => + h.serve(Halves, (_, _principal, self) => { + const n = self.state(N); + return { + variant: "v", + members: { + n, + half: self.derive([n], ([{ n }]) => { + if (n === 2) throw new Error("half threw"); + return n / 2; + }), + set: async (value, ctx) => { + await n.change((draft) => { + draft.n = value; + }, ctx); + return null; + }, + wait: (ctx) => new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", () => resolve(null))), + }, + }; + }), + ); + let calls = 0; + const set = async (value: number) => { + const call = calls++; + linked.peer.send({ kind: "call", call, hold: 0, method: HALVES.methods.set!, args: jsonPart(value) }); + expect((await linked.peer.next("result")).call).toBe(call); + }; + return { ...linked, set }; +} + +function hold(peer: RawPeer, id: number, Type: { name: string; version: number; fingerprint: string }) { + const { name: type, version, fingerprint } = Type; + peer.send({ kind: "hold", hold: id, type, version, fingerprint, params: "{}" }); +} + +/** The provider object behind a hold, reached through the host's own lookup (to end it by hand). */ +function providerObject(server: unknown) { + const host = server as ChordHost; + const { name: type, version, fingerprint } = Halves; + return host.serving({ type, version, fingerprint, params: "{}" }, {}, { kind: "owner" }, context); +} + +/** A remote type whose activation never completes: its open waits for its abort. */ +const Pending = remote("app.pending", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + +function servePending(h: Parameters[0]>[0]) { + h.serve(Pending, (_, _principal, self) => { + self.open( + (ctx) => new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", resolve)), + () => {}, + ); + return { variant: "v", members: { n: constant(1) } }; + }); +} + +forEachCodec(() => { + describe("the parts of a hold", () => { + it("are its members, each with its own encoder: the snapshot by member, then the changed members' ops", async () => { + const { peer, set } = await halves(); + hold(peer, 0, Halves); + expect(HALVES.named((await peer.next("bound")).holds[0]!.members)).toEqual({ n: { n: 0 }, half: 0 }); + await set(1); + const first = HALVES.named((await peer.next("state")).holds[0]!.members); + expect(first).toEqual({ n: [["s", ["n"], 1]], half: [["r", 0.5]] }); + await set(4); + // The member's encoder interns a path on its second use. + const n = decoder(); + n.decode(first.n as unknown[]); + const second = HALVES.named((await peer.next("state")).holds[0]!.members); + expect(n.decode(second.n as unknown[])).toEqual([["s", ["n"], 4]]); + }); + }); + + describe("failed members", () => { + it("are listed in the header of the state that carries the failure, never in its part; they recover in the same hold", async () => { + const { peer, set } = await halves(); + hold(peer, 0, Halves); + hold(peer, 1, Halves); + await peer.next("bound"); + await peer.next("bound"); + await set(2); + const state = await peer.next("state"); + const failed = [ + { member: HALVES.state.half!, error: { kind: "plugin", code: "internal", message: "half threw" } }, + ]; + // Race 12: each hold's entry carries its healthy members' ops and the failure. + expect(state.holds.map(({ hold, failed }) => ({ hold, failed }))).toEqual([ + { hold: 0, failed }, + { hold: 1, failed }, + ]); + for (const entry of state.holds) expect(HALVES.named(entry.members)).toEqual({ n: [["s", ["n"], 2]] }); + await set(3); + const next = await peer.next("state"); + expect(next.holds.map(({ hold, failed }) => ({ hold, failed }))).toEqual([ + { hold: 0, failed: [] }, + { hold: 1, failed: [] }, + ]); + // The derive healed: its next value replaces the root. + expect(HALVES.named(next.holds[0]!.members).half).toEqual([["r", 1.5]]); + }); + + it("reach a client's watch over the link as one failure call, then the healed value: the watch stays open", async () => { + const { server, client } = await linked((h) => + h.serve(Halves, (_, _principal, self) => { + const n = self.state(N); + return { + variant: "v", + members: { + n, + half: self.derive([n], ([{ n }]) => { + if (n === 2) throw new Error("half threw"); + return n / 2; + }), + set: async (value, ctx) => { + await n.change((draft) => { + draft.n = value; + }, ctx); + return null; + }, + wait: async () => null, + }, + }; + }), + ); + const halves = await client.get(Halves, {}, context); + if (halves.variant !== "v") throw new Error("expected variant v"); + const calls = changesOf(await halves.half.watch(context)); + await halves.set(2, context); + await transported(); + expect(calls).toEqual([{ kind: "failed", error: expect.objectContaining({ message: "half threw" }) }]); + await halves.set(6, context); + await transported(); + expect(halves.half.value).toBe(3); + expect(calls.at(-1)).toEqual({ kind: "value", value: 3, ops: [["r", 3]], retired: false }); + await client.close(context); + await server.close(context); + }); + + it("are listed in bound for a hold made after the failure", async () => { + const { peer, set } = await halves(); + hold(peer, 0, Halves); + await peer.next("bound"); + await set(2); + await peer.next("state"); + hold(peer, 1, Halves); + const [bound] = (await peer.next("bound")).holds; + expect(bound!.failed).toEqual([ + { member: HALVES.state.half!, error: { kind: "plugin", code: "internal", message: "half threw" } }, + ]); + expect(HALVES.named(bound!.members)).toEqual({ n: { n: 2 } }); + // Followed though it failed at the snapshot: it recovers in this hold. + await set(4); + const next = await peer.next("state"); + expect(HALVES.named(next.holds.find(({ hold }) => hold === 1)!.members).half).toEqual([["r", 2]]); + }); + }); + + describe("transient failures", () => { + // Timers stand still: the failed object's backoff window does not end while the test runs. + fakeTimers(); + + it("a new hold of an object failed transiently is sent the failure at once", async () => { + const Flaky = remote("app.flaky", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const { peer } = await rawLinked((h) => + h.serve(Flaky, (_, _principal, self) => { + self.open( + async () => { + throw new Error("offline"); + }, + () => {}, + ); + return { variant: "v", members: { n: constant(1) } }; + }), + ); + hold(peer, 0, Flaky); + const cause = { + kind: "transient", + error: { kind: "chord", error: { code: "unavailable", message: "app.flaky: offline" } }, + }; + expect(await peer.next("failed")).toMatchObject({ hold: 0, cause }); + hold(peer, 1, Flaky); + // The object stays failed for its backoff window (1 s); the new hold does not wait for the next attempt. + expect(await peer.next("failed")).toMatchObject({ hold: 1, cause }); + }); + }); + + // A hold whose object ends sends `gone` and nothing for its pending calls (race 4): through revocation, the path + // that ends a held object (`host/revocation.test.ts`, "aborts running calls"). + + describe("GC", () => { + it("a closed link retained by the test keeps no hold, encoder or object", async () => { + const { server, refs, link } = await (async () => { + const { server } = await halves(); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const [held, made] = [holds.length, encoders.length]; + hold(peer, 0, Halves); + await peer.next("bound"); + const refs: WeakRef[] = [new WeakRef(providerObject(server)), ...holds.slice(held)]; + refs.push(...encoders.slice(made)); + expect(refs).toHaveLength(4); + peer.close(); + await transported(); + return { server, refs, link: a }; + })(); + await expectCollected(...refs); + expect([server, link]).toBeDefined(); + }); + + it("a released pending hold is kept neither by its object, still activating for another hold, nor by its waits", async () => { + const { server, peer, refs } = await (async () => { + const { server, peer } = await rawLinked((h) => servePending(h)); + hold(peer, 0, Pending); + await transported(); + const held = holds.length; + hold(peer, 1, Pending); + await transported(); + const refs = holds.slice(held); + expect(refs).toHaveLength(1); + peer.send({ kind: "release", hold: 1 }); + await transported(); + return { server, peer, refs }; + })(); + await expectCollected(...refs); + expect([server, peer]).toBeDefined(); + }); + + it("a pending activation on a closed link keeps no hold", async () => { + const { server, link, refs } = await (async () => { + const { server } = await rawLinked((h) => servePending(h)); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const held = holds.length; + hold(peer, 0, Pending); + await transported(); + const refs = holds.slice(held); + peer.close(); + await transported(); + return { server, link: a, refs }; + })(); + await expectCollected(...refs); + expect([server, link]).toBeDefined(); + }); + + it("a hold released while waiting for load is not kept by the wait", async () => { + const Slow = local("app.slow", { version: 1, params: {}, members: { n: int() } }); + const { server, refs } = await (async () => { + const server = createHost({ + principal: { kind: "server" }, + readyTimeout: 2_000, + onError: () => {}, + }); + const facet = defineFacet((h) => { + servePending(h); + // A host-lifetime object whose open never resolves: `load` waits for it (until its readyTimeout). + h.serve(Slow, (_, self) => { + self.open( + () => new Promise(() => {}), + () => {}, + ); + return { n: constant(1) }; + }); + }); + void server.load([{ name: "@test/app", facet }], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const held = holds.length; + hold(peer, 0, Pending); + await transported(); + peer.send({ kind: "release", hold: 0 }); + await transported(); + return { server, refs: holds.slice(held) }; + })(); + expect(refs).toHaveLength(1); + await expectCollected(...refs); + expect(server).toBeDefined(); + }); + }); +}); + +/** A remote type whose open throws `offline` (broken when `failing.broken`) while `failing.left` is above 0. */ +const Flaky = remote("app.flaky", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + +function serveFlaky(h: Parameters[0]>[0], failing: { left: number; broken: boolean }) { + h.serve(Flaky, (_, _principal, self) => { + self.open( + async () => { + if (failing.left-- <= 0) return; + throw failing.broken ? broken(new Error("offline")) : new Error("offline"); + }, + () => {}, + ); + return { variant: "v", members: { n: constant(1) } }; + }); +} + +/** A second raw peer on `server`, stamped with the same principal as `rawLinked`'s: it holds the same objects. */ +function secondPeer(server: Host): RawPeer { + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + return new RawPeer(b); +} + +const offline = { code: "unavailable", message: "app.flaky: offline" }; + +forEachCodec(() => { + describe("notices of a failed object", () => { + fakeTimers(); + + it("its holds get failed with the rest of its window, loading for each attempt, then bound (race 6)", async () => { + const { peer } = await rawLinked((h) => serveFlaky(h, { left: 2, broken: false })); + hold(peer, 0, Flaky); + expect(await peer.next("failed")).toEqual({ + kind: "failed", + hold: 0, + cause: { kind: "transient", error: { kind: "chord", error: offline } }, + retryIn: 1_000, + }); + await vi.advanceTimersByTimeAsync(600); + hold(peer, 1, Flaky); + expect(await peer.next("failed")).toMatchObject({ hold: 1, retryIn: 400 }); + await vi.advanceTimersByTimeAsync(400); + await transported(); + expect(peer.kinds()).toEqual(["hello", "loading", "loading", "failed", "failed"]); + expect((await peer.next("failed")).retryIn).toBe(2_000); + await vi.advanceTimersByTimeAsync(2_000); + await peer.next("bound"); + await peer.next("bound"); + expect(peer.kinds()).toEqual(["hello", "loading", "loading", "failed", "loading", "loading"]); + }); + + it("a retry from one consumer renews a broken object for every holder: loading, then bound (races 12, 13)", async () => { + const failing = { left: 1, broken: true }; + const { server, peer } = await rawLinked((h) => serveFlaky(h, failing)); + const other = secondPeer(server); + hold(peer, 0, Flaky); + hold(other, 0, Flaky); + const broken = { + kind: "failed", + hold: 0, + cause: { kind: "broken", error: { kind: "chord", error: offline } }, + }; + expect([await peer.next("failed"), await other.next("failed")]).toEqual([broken, broken]); + peer.send({ kind: "retry", hold: 0 }); + for (const consumer of [peer, other]) { + expect(await consumer.next("loading")).toEqual({ kind: "loading", hold: 0 }); + expect((await consumer.next("bound")).holds[0]).toMatchObject({ hold: 0 }); + } + }); + + it("a retry skips a transient failure's window for every holder (race 12)", async () => { + const { server, peer } = await rawLinked((h) => serveFlaky(h, { left: 1, broken: false })); + const other = secondPeer(server); + hold(peer, 0, Flaky); + await peer.next("failed"); + hold(other, 0, Flaky); + await other.next("failed"); + other.send({ kind: "retry", hold: 0 }); + for (const consumer of [peer, other]) { + await consumer.next("loading"); + await consumer.next("bound"); + } + }); + + it("a retry of a live or an unknown hold does nothing", async () => { + const { peer } = await rawLinked((h) => serveFlaky(h, { left: 0, broken: false })); + hold(peer, 0, Flaky); + await peer.next("bound"); + peer.send({ kind: "retry", hold: 0 }); + peer.send({ kind: "retry", hold: 9 }); + await transported(); + expect([peer.kinds(), peer.closed]).toEqual([["hello"], false]); + }); + }); + + describe("a reconnecting consumer", () => { + const Tally = remote("app.tally", { + version: 1, + params: {}, + variants: { v: { state: { n: obj({ n: int() }) }, methods: { bump: method() } } }, + }); + const Fresh = remote("app.fresh", { + version: 1, + params: {}, + variants: { v: { state: { n: obj({ n: int() }) }, methods: { bump: method() } } }, + }); + + /** A server serving `Tally` (lingering a minute) and `Fresh` (not lingering, its native's close waits for `closing`). */ + async function served(closing: Promise) { + const serve = (self: Self) => { + const doc = self.state(N); + return { + variant: "v" as const, + members: { + n: doc, + bump: async (ctx: Context) => { + await doc.change((draft) => { + draft.n++; + }, ctx); + return null; + }, + }, + }; + }; + return linked((h) => { + h.serve(Tally, (_, _principal, self) => serve(self), { linger: 60_000 }); + h.serve(Fresh, (_, _principal, self) => { + self.open( + async () => null, + () => closing, + ); + return serve(self); + }); + }); + } + + /** Drops the client's link and connects it again. */ + async function reconnect(server: Host, client: Host, link: Link) { + link.close(); + await transported(); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + client.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); + await transported(); + } + + it("finds a lingering object as it left it: its in-memory documents kept; one without linger is new (race 7)", async () => { + const closing = Promise.resolve(); + const { server, client, links } = await served(closing); + const tally = await client.get(Tally, {}, context); + const fresh = await client.get(Fresh, {}, context); + await tally.bump(context); + await fresh.bump(context); + await reconnect(server, client, links[0]); + expect([tally.state.value, fresh.state.value]).toEqual([{ kind: "live" }, { kind: "live" }]); + expect([tally.n.value, fresh.n.value]).toEqual([{ n: 1 }, { n: 0 }]); + }); + + it("re-holds an object whose previous instance still tears down: bound once that ended (race 17)", async () => { + const closing = Promise.withResolvers(); + const { server, client, links } = await served(closing.promise); + const fresh = await client.get(Fresh, {}, context); + await fresh.bump(context); + await reconnect(server, client, links[0]); + expect([fresh.state.value, fresh.n.value]).toEqual([{ kind: "loading" }, { n: 1 }]); + closing.resolve(); + await transported(); + expect([fresh.state.value, fresh.n.value]).toEqual([{ kind: "live" }, { n: 0 }]); + }); + }); +}); + +/** + * A remote type whose document can be retired; `big` derives a flag that a retirement of a small value leaves as is, + * `t` is `n` and throws while it is negative. + */ +const Retiring = remote("app.retiring", { + version: 1, + params: {}, + variants: { + v: { + state: { doc: obj({ n: int() }), big: bool(), t: int() }, + methods: { set: method(int()), retire: method() }, + }, + }, +}); + +const RETIRING = wireNames(Retiring, "v"); + +/** Every member of `Retiring` retired with its document, sorted. */ +const retiredAll = [RETIRING.state.doc!, RETIRING.state.big!, RETIRING.state.t!].sort(); + +/** A server serving `Retiring` to a raw peer that acks only when told; `call` calls a method of hold 0. */ +async function retiring() { + const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + await server.load( + [ + { + name: "@test/app", + facet: defineFacet((h) => + h.serve(Retiring, (_, _principal, self) => { + const doc = self.state(N); + return { + variant: "v", + members: { + doc, + big: self.derive([doc], ([{ n }]) => n > 10), + t: self.derive([doc], ([{ n }]) => { + if (n < 0) throw new Error(`failed ${n}`); + return n; + }), + set: async (n, ctx) => { + await doc.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + retire: async (ctx) => { + await doc.retire(ctx); + return null; + }, + }, + }; + }), + ), + }, + ], + context, + ); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b, { ack: false }); + await transported(); + let calls = 0; + const call = async (method: "set" | "retire", args?: number) => { + const id = calls++; + const payload = args === undefined ? {} : { args: jsonPart(args) }; + peer.send({ kind: "call", call: id, hold: 0, method: RETIRING.methods[method]!, ...payload }); + expect((await peer.next("result")).call).toBe(id); + }; + const { name: type, version, fingerprint } = Retiring; + peer.send({ kind: "hold", hold: 0, type, version, fingerprint, params: "{}" }); + await peer.next("bound"); + peer.ack(); + return { peer, call }; +} + +forEachCodec(() => { + describe('retired members (decision "One retirement mechanism")', () => { + it("are listed in the state that carries the retirement, a derive whose value stayed without ops", async () => { + const { peer, call } = await retiring(); + await call("set", 1); + const set = await peer.next("state"); + expect(set.holds[0]!.retired).toEqual([]); + peer.ack(); + await call("retire"); + const retired = (await peer.next("state")).holds[0]!; + expect(RETIRING.named(retired.members)).toEqual({ doc: [["r", { n: 0 }]], t: [["r", 0]] }); + expect(retired.retired.toSorted()).toEqual(retiredAll); + }); + + it("are kept while the window is full and sent once with the folded ops", async () => { + const { peer, call } = await retiring(); + await call("set", 1); + await call("set", 2); + await peer.next("state"); + await peer.next("state"); + // The window is full: the retirement and the next change wait, folded into one entry. + await call("retire"); + await call("set", 4); + await transported(); + expect(peer.kinds().filter((kind) => kind === "state")).toEqual([]); + peer.ack(); + const pending = (await peer.next("state")).holds[0]!; + expect(RETIRING.named(pending.members)).toEqual({ + doc: [ + ["r", { n: 0 }], + ["s", ["n"], 4], + ], + t: [["r", 4]], + }); + expect(pending.retired.toSorted()).toEqual(retiredAll); + peer.ack(); + peer.ack(); + await call("set", 5); + expect((await peer.next("state")).holds[0]!.retired).toEqual([]); + }); + + it("of a member failed again before its entry leaves wait for its next value, never with its failure", async () => { + const { peer, call } = await retiring(); + await call("set", 1); + await call("set", 2); + await peer.next("state"); + await peer.next("state"); + // The window is full: `t` fails, heals with the retirement (n is 0 again), and fails again. + await call("set", -1); + await call("retire"); + await call("set", -3); + peer.ack(); + const failed = (await peer.next("state")).holds[0]!; + expect(failed.failed).toEqual([ + { member: RETIRING.state.t!, error: { kind: "plugin", code: "internal", message: "failed -3" } }, + ]); + expect(Object.keys(RETIRING.named(failed.members))).toEqual(["doc"]); + expect(failed.retired.toSorted()).toEqual([RETIRING.state.doc!, RETIRING.state.big!].sort()); + peer.ack(); + peer.ack(); + await call("set", 5); + const healed = (await peer.next("state")).holds[0]!; + expect(RETIRING.named(healed.members).t).toEqual([["r", 5]]); + expect(healed.retired).toEqual([RETIRING.state.t!]); + }); + }); + + describe("failures while the window is full", () => { + it("fold per member: 1000 failing commits leave one failure entry, the member's ops dropped; bounded", async () => { + const { peer, call } = await retiring(); + await call("set", 1); + await call("set", 2); + await peer.next("state"); + await peer.next("state"); + // An earlier episode, healed: its failure leaves with the heal. + await call("set", -1); + await call("set", 3); + for (let n = 1; n <= 1000; n++) await call("set", -1000 - n); + peer.ack(); + const pending = (await peer.next("state")).holds[0]!; + // One failure call per episode: the error of the commit that started it. + expect(pending.failed).toEqual([ + { member: RETIRING.state.t!, error: { kind: "plugin", code: "internal", message: "failed -1001" } }, + ]); + expect(Object.keys(RETIRING.named(pending.members))).not.toContain("t"); + expect(peer.codec.encode({ kind: "state", holds: [pending] }).byteLength).toBeLessThan(200); + }); + + it("are replaced by a value that follows them: the value as a root replacement, no failure", async () => { + const { peer, call } = await retiring(); + await call("set", 1); + await call("set", 2); + await peer.next("state"); + await peer.next("state"); + await call("set", -1); + await call("set", 3); + peer.ack(); + const pending = (await peer.next("state")).holds[0]!; + expect(pending.failed).toEqual([]); + expect(RETIRING.named(pending.members).t).toEqual([["r", 3]]); + }); + }); +}); diff --git a/packages/chord/test/next/remote/host-link.test.ts b/packages/chord/test/next/remote/host-link.test.ts new file mode 100644 index 000000000..7eb0a8551 --- /dev/null +++ b/packages/chord/test/next/remote/host-link.test.ts @@ -0,0 +1,429 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet, type FacetHost, type HostError } from "../../../src/next/host/index.ts"; +import { type Compression, memoryLinkPair, type PartLimits } from "../../../src/next/link/index.ts"; +import { jsonPart } from "../../../src/next/link/internal.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { nodeCompression } from "../../../src/next-node/compression.ts"; +import { + addNote, + countInflated, + delivered, + EDITOR, + forEachCodec, + framesOf, + incarnationOf, + Notes, + PlacedNotes, + partValue, + RAW_PEER, + RawPeer, + rawWorkers, + rejection, + relayed, + Session, + serveNotes, + type TestPrincipal, + testCodec, + transported, +} from "../support.ts"; + +/** A server type that tells who holds it, and echoes as whom it is called. */ +const ServerWho = remote("app.server-who", { + version: 1, + params: {}, + variants: { v: { state: { who: str() }, methods: { echo: method(str(), str()) } } }, +}); + +/** A worker type that asks the server, from a handler (`ask`) and from its factory (`who`). */ +const Asker = remote("app.asker", { + version: 1, + params: { session: str() }, + placement: ({ session }) => Session({ id: session }), + variants: { v: { state: { who: str() }, methods: { ask: method(str(), str()) } } }, +}); + +/** A worker type that reads another session's notes through the server. */ +const Neighbour = remote("app.neighbour", { + version: 1, + params: { session: str(), other: str() }, + placement: ({ session }) => Session({ id: session }), + variants: { v: { state: { count: int() } } }, +}); + +function serveWho(h: FacetHost): void { + h.serve(ServerWho, (_, principal) => ({ + variant: "v", + members: { who: constant(principal.kind), echo: async (text, ctx) => `${ctx.principal?.kind}:${text}` }, + })); +} + +function serveAsker(h: FacetHost): void { + h.serve(Asker, (_, _principal, self) => { + const server = self.get(ServerWho, {}); + return { + variant: "v", + members: { + who: server.who, + ask: async (text, ctx) => (await ctx.get(ServerWho, {})).echo(text, ctx), + }, + }; + }); + h.serve(Neighbour, ({ other }, _principal, self) => { + const notes = self.get(PlacedNotes, { session: other }); + return { variant: "v", members: { count: notes.count } }; + }); +} + +forEachCodec(() => { + describe("two-sided links", () => { + it("carry the server's holds of a worker type and the worker's holds of a server type over one route link", async () => { + const { tab, routeLinks } = await relayed({ server: serveWho, worker: serveAsker }); + const asker = await tab.get(Asker, { session: "S1" }, context); + // The worker's factory held `ServerWho` over its parent link, the route link the server made. + expect(asker.who.value).toBe("worker"); + expect(routeLinks).toHaveLength(1); + }); + + it("carry calls both ways with equal ids: each side's ids are its own", async () => { + const { tab } = await relayed({ server: serveWho, worker: serveAsker }); + const asker = await tab.get(Asker, { session: "S1" }, context); + // Server to worker: call 0 on the route link; inside it, worker to server: call 0 the other way. + expect(await asker.ask("x", context)).toBe("worker:x"); + expect(await Promise.all([asker.ask("a", context), asker.ask("b", context)])).toEqual([ + "worker:a", + "worker:b", + ]); + }); + + it("let a worker reach another worker through the server", async () => { + const { tab, workers, resolves } = await relayed({ worker: serveAsker }); + const neighbour = await tab.get(Neighbour, { session: "S1", other: "S2" }, context); + expect(resolves).toEqual(["session|S1", "session|S2"]); + await addNote(workers.get("session|S2")!, "S2", "a"); + await transported(); + expect(neighbour.count.value).toBe(1); + }); + + it("apply this host's own messages and forward the tab origin's, each acknowledged by its consumer", async () => { + const peers: RawPeer[] = []; + const { tab, server } = await relayed({ resolve: rawWorkers(peers) }); + const own = server.get(PlacedNotes, { session: "S1" }, context); + await transported(); + const [worker] = peers; + const first = await worker!.next("hold"); + bindPlaced(worker!, first); + const relayedHold = tab.get(PlacedNotes, { session: "S1" }, context); + const second = await worker!.next("hold"); + expect([first.origin, second.origin]).toEqual([undefined, 1]); + bindPlaced(worker!, second); + const [serverNotes, tabs] = [await own, await relayedHold]; + await transported(); + worker!.received.length = 0; + const ops = jsonPart([["r", 5]]); + for (const { hold, origin } of [first, second]) { + const from = origin === undefined ? {} : { origin }; + const members = [{ member: EDITOR.state.count!, ops }]; + worker!.send({ kind: "state", ...from, holds: [{ hold, members, failed: [], retired: [] }] }); + } + await transported(); + expect(serverNotes.count.value).toBe(5); + expect(tabs.count.value).toBe(5); + expect(worker!.received).toEqual([{ kind: "ack" }, { kind: "ack", origin: 1 }]); + }); + }); + + describe("principals on holds (section 4's table)", () => { + /** A server serving a guest-facing gateway and a local view, both reaching `PlacedNotes` on a raw worker. */ + async function principals(tab: TestPrincipal = { kind: "guest", session: "S1" }) { + const peers: RawPeer[] = []; + const Gateway = remote("app.gateway", { + version: 1, + params: {}, + variants: { v: { state: { n: int() }, methods: { open: method(str(), int()) } } }, + }); + // With params: not activated by `load`, which would wait for the raw worker. + const View = local("app.view", { version: 1, params: { id: int() }, members: { n: int() } }); + const relay = await relayed({ + tab, + resolve: rawWorkers(peers), + server: (h) => { + h.serve(Gateway, (_, _principal, self) => { + // `self.get` in a remote object: the object's principal, this host's incarnation. + const own = self.get(PlacedNotes, { session: "own" }); + return { + variant: "v", + members: { + n: own.count, + // `ctx.get` in a handler: the caller's principal and incarnation. + open: async (session, ctx) => (await ctx.get(PlacedNotes, { session })).count.value, + }, + }; + }); + h.serve(View, (_, self) => ({ n: self.get(PlacedNotes, { session: "view" }).count })); + }, + }); + return { ...relay, peers, Gateway, View }; + } + + /** The next hold the raw worker of `peers[index]` receives, bound at once, with whom it acts for. */ + async function nextHold(peers: RawPeer[], index = 0) { + for (let tries = 0; peers[index] === undefined && tries < 10; tries++) await transported(); + const hold = await peers[index]!.next("hold"); + bindPlaced(peers[index]!, hold); + const { session } = partValue(hold.params) as { session: string }; + return { session, principal: hold.principal, incarnation: hold.incarnation }; + } + + it("a relayed hold acts as the link's principal: a forged principal from a tab is ignored (race 9)", async () => { + const { server, peers } = await principals(); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "guest", session: "S1" }, codec: testCodec() }, context); + const tab = new RawPeer(b); + tab.send({ + kind: "hold", + hold: 0, + type: PlacedNotes.name, + version: PlacedNotes.version, + fingerprint: PlacedNotes.fingerprint, + params: jsonPart({ session: "S1" }), + address: { kind: "session", params: { id: "S1" } }, + principal: { kind: "owner" }, + }); + expect(await nextHold(peers)).toEqual({ + session: "S1", + principal: { kind: "guest", session: "S1" }, + incarnation: RAW_PEER, + }); + }); + + it("a hold relayed over a parent path names nobody: a child's principal and incarnation stay behind", async () => { + const worker = createHost({ principal: { kind: "worker", session: "S1" }, onError: () => {} }); + await worker.load([], context); + const [up, parentEnd] = memoryLinkPair(); + worker.connect(up, { principal: { kind: "server" }, codec: testCodec() }, context); + const parent = new RawPeer(parentEnd); + const [a, b] = memoryLinkPair(); + worker.accept(a, { principal: { kind: "guest", session: "S1" }, codec: testCodec() }, context); + const child = new RawPeer(b); + child.send({ + kind: "hold", + hold: 0, + type: PlacedNotes.name, + version: PlacedNotes.version, + fingerprint: PlacedNotes.fingerprint, + params: jsonPart({ session: "S1" }), + address: { kind: "session", params: { id: "S1" } }, + principal: { kind: "owner" }, + incarnation: RAW_PEER, + }); + const relayedHold = await parent.next("hold"); + expect(relayedHold).toMatchObject({ hold: 0, origin: 1, type: PlacedNotes.name }); + expect("principal" in relayedHold).toBe(false); + expect("incarnation" in relayedHold).toBe(false); + await worker.close(context); + }); + + it("host.get through a route acts as the host: its principal and incarnation", async () => { + const { server, peers } = await principals(); + void server.get(PlacedNotes, { session: "S1" }, context); + expect(await nextHold(peers)).toEqual({ + session: "S1", + principal: { kind: "server" }, + incarnation: incarnationOf(server), + }); + }); + + it("ctx.get in a guest's call acts as the guest, with the tab's incarnation (race 10)", async () => { + const { tab, peers, Gateway } = await principals(); + const got = tab.get(Gateway, {}, context); + expect(await nextHold(peers)).toMatchObject({ session: "own" }); + const opened = (await got).open("S1", context); + // Another address: another worker. + expect(await nextHold(peers, 1)).toEqual({ + session: "S1", + principal: { kind: "guest", session: "S1" }, + incarnation: incarnationOf(tab), + }); + expect(await opened).toBe(0); + }); + + it("self.get in a remote object acts as that object's principal, with this host's incarnation", async () => { + const { tab, server, peers, Gateway } = await principals(); + tab.hold(Gateway, {}, context); + expect(await nextHold(peers)).toEqual({ + session: "own", + principal: { kind: "guest", session: "S1" }, + incarnation: incarnationOf(server), + }); + }); + + it("self.get in a local object acts as the host", async () => { + const { server, peers, View } = await principals(); + void server.get(View, { id: 1 }, context); + expect(await nextHold(peers)).toEqual({ + session: "view", + principal: { kind: "server" }, + incarnation: incarnationOf(server), + }); + }); + + it("two origins with equal principal keys but other incarnations are two upstream holds", async () => { + const { tab, server, peers } = await principals({ kind: "owner" }); + const other = createHost({ onError: () => {} }); + await other.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + other.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); + void tab.get(PlacedNotes, { session: "S1" }, context); + const first = await nextHold(peers); + void other.get(PlacedNotes, { session: "S1" }, context); + const second = await nextHold(peers); + expect([first, second]).toEqual([ + { session: "S1", principal: { kind: "owner" }, incarnation: incarnationOf(tab) }, + { session: "S1", principal: { kind: "owner" }, incarnation: incarnationOf(other) }, + ]); + }); + + it("upward is a boundary: a worker's hold of a server type acts as the worker, whoever it serves (race 11)", async () => { + const { tab } = await relayed({ tab: { kind: "guest", session: "S1" }, server: serveWho, worker: serveAsker }); + const asker = await tab.get(Asker, { session: "S1" }, context); + expect(await asker.ask("x", context)).toBe("worker:x"); + }); + }); + + describe("trusted links", () => { + it("work as untrusted ones do", async () => { + const { tab } = await relayed({ trusted: true }); + expect((await tab.get(PlacedNotes, { session: "S1" }, context)).variant).toBe("editor"); + }); + }); +}); + +/** Binds a hold a raw worker received (its id and origin): `PlacedNotes`'s editor variant, no notes. */ +function bindPlaced(peer: RawPeer, { hold, origin }: { readonly hold: number; readonly origin?: number }): void { + const members = [ + { member: EDITOR.state.notes!, value: "[]" }, + { member: EDITOR.state.count!, value: "0" }, + ]; + const from = origin === undefined ? {} : { origin }; + peer.send({ kind: "bound", ...from, holds: [{ hold, variant: EDITOR.variant, members, failed: [] }] }); +} + +/** A compressor that counts what it does (`nodeCompression` underneath). */ +function counting(): Compression & { deflates: number; inflates: number } { + const counts = { + deflates: 0, + inflates: 0, + deflate: (bytes: Uint8Array) => { + counts.deflates++; + return nodeCompression.deflate(bytes); + }, + inflate: (bytes: Uint8Array, limit: number) => { + counts.inflates++; + return nodeCompression.inflate(bytes, limit); + }, + }; + return counts; +} + +/** A note text large enough that a notes list with it is deflated at its origin. */ +const LARGE = "lorem ipsum ".repeat(2000); + +/** + * A server serving `Notes` with `compression`, and a client with its own `compression` (none: it inflates + * asynchronously), its parent link's limits `limits`. + */ +async function compressing(options: { + readonly server?: Compression; + readonly client?: Compression; + readonly limits?: PartLimits; +}) { + const errors: HostError[] = []; + const server = createHost({ + principal: { kind: "server" }, + compression: options.server, + onError: (error) => errors.push(error), + }); + await server.load([{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context); + const client = createHost({ compression: options.client, onError: (error) => errors.push(error) }); + await client.load([], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + client.connect(b, { principal: { kind: "server" }, codec: testCodec(), limits: options.limits }, context); + return { server, client, errors }; +} + +forEachCodec(() => { + describe("parts: compression at the origin and limits", () => { + it("deflate JSON parts above 16 KiB at their origin and inflate them where owned, synchronously; never up a parent link's arguments", async () => { + const [server, client] = [counting(), counting()]; + const hosts = await compressing({ server, client }); + await addNote(hosts.server, "S1", LARGE); + const notes = await hosts.client.get(Notes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("an owner's notes are the editor's"); + expect(notes.notes.value).toEqual([{ text: LARGE }]); + expect([server.deflates, client.inflates]).toEqual([1, 1]); + // A large argument goes up the client's parent link uncompressed: child links refuse compressed parts. + await notes.add({ text: LARGE }, context); + expect([client.deflates, server.inflates]).toEqual([0, 0]); + await transported(); + expect(notes.notes.value).toEqual([{ text: LARGE }, { text: LARGE }]); + }); + + it("inflate asynchronously without a compressor: later messages of the link wait, in order", async () => { + const hosts = await compressing({ server: nodeCompression }); + await addNote(hosts.server, "S1", LARGE); + const notes = await hosts.client.get(Notes, { session: "S1" }, context); + expect(notes.notes.value).toEqual([{ text: LARGE }]); + const counts = framesOf(await notes.count.watch(context)); + // Each commit sends the whole changed list (a derive's diff appends one note): large ones deflated. + for (let i = 0; i < 5; i++) await addNote(hosts.server, "S1", i % 2 === 0 ? LARGE : "small"); + for (let i = 0; i < 100 && notes.count.value < 6; i++) await new Promise((resolve) => setTimeout(resolve, 5)); + expect(notes.count.value).toBe(6); + expect(notes.notes.value.map(({ text }) => text.length)).toEqual( + [LARGE, LARGE, "small", LARGE, "small", LARGE].map((text) => text.length), + ); + await delivered(); + const seen = counts.map(([count]) => count); + expect(seen).toEqual([...seen].sort()); + }); + + it("fail only the hold of a compressed bomb, on either inflate path (race 11): the link stays", async () => { + for (const client of [nodeCompression, undefined]) { + const hosts = await compressing({ server: nodeCompression, client, limits: { decoded: 100_000 } }); + await addNote(hosts.server, "S1", "x".repeat(200_000)); + await addNote(hosts.server, "S2", "small"); + const error = await rejection(hosts.client.get(Notes, { session: "S1" }, context)); + expect(error).toMatchObject({ code: "too_large", reason: "part" }); + const other = await hosts.client.get(Notes, { session: "S2" }, context); + expect(other.notes.value).toEqual([{ text: "small" }]); + } + }); + + it("cost nothing for uncompressed messages at a host without a compressor: no inflate, no stream", async () => { + const inflated = countInflated(); + try { + const hosts = await compressing({ server: nodeCompression }); + const notes = await hosts.client.get(Notes, { session: "S1" }, context); + for (let i = 0; i < 5; i++) await addNote(hosts.server, "S1", "small"); + await transported(); + expect(notes.count.value).toBe(5); + expect(inflated.bytes).toBe(0); + expect(hosts.errors).toEqual([]); + } finally { + inflated.restore(); + } + }); + + it("send every part uncompressed from a host without a compressor", async () => { + const server = counting(); + const hosts = await compressing({ server }); + const notes = await hosts.client.get(Notes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("an owner's notes are the editor's"); + await notes.add({ text: LARGE }, context); + expect(server.inflates).toBe(0); + }); + }); +}); diff --git a/packages/chord/test/next/remote/path.test.ts b/packages/chord/test/next/remote/path.test.ts new file mode 100644 index 000000000..c3efb197d --- /dev/null +++ b/packages/chord/test/next/remote/path.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; +import type { HostLink } from "../../../src/next/remote/host-link.ts"; +import { Path, type Upstream } from "../../../src/next/remote/path.ts"; + +/** A holder recording where it was held and what it was told, as `name@link` and `name dropped`. */ +function holder(name: string, log: string[]): Upstream { + // The path calls only `attach` and `dropped`. + return { + attach: (link: { readonly name: string }) => void log.push(`${name}@${link.name}`), + dropped: () => void log.push(`${name} dropped`), + } as unknown as Upstream; +} + +/** A link known by its name; the path only passes it on. */ +function linkOf(name: string): HostLink { + return { name } as unknown as HostLink; +} + +describe("a path", () => { + it("asks for a link when it gets a holder without one; holds every holder on each link it gets", () => { + const log: string[] = []; + let changes = 0; + const path = new Path(() => changes++); + path.attach(holder("a", log)); + path.attach(holder("b", log)); + expect(changes).toBe(2); + expect(log).toEqual([]); + const first = linkOf("1"); + path.connected(first); + expect(log).toEqual(["a@1", "b@1"]); + path.attach(holder("c", log)); + expect(log.at(-1)).toBe("c@1"); + expect(changes).toBe(2); + }); + + it("tells every holder when its link drops, then asks for the next; a stale link's drop is ignored", () => { + const log: string[] = []; + let changes = 0; + const path = new Path(() => changes++); + const [first, second] = [linkOf("1"), linkOf("2")]; + path.connected(first); + path.attach(holder("a", log)); + path.disconnected(first); + expect(log).toEqual(["a@1", "a dropped"]); + expect(changes).toBe(1); + expect(path.link).toBeUndefined(); + path.connected(second); + path.disconnected(first); + expect(log).toEqual(["a@1", "a dropped", "a@2"]); + expect(changes).toBe(1); + expect(path.link).toBe(second); + }); + + it("reports its last holder's detach: no later link holds it", () => { + const log: string[] = []; + let changes = 0; + const path = new Path(() => changes++); + const [a, b] = [holder("a", log), holder("b", log)]; + path.connected(linkOf("1")); + path.attach(a); + path.attach(b); + path.detach(a); + expect(changes).toBe(0); + path.detach(b); + expect(changes).toBe(1); + expect(path.held).toBe(false); + path.connected(linkOf("2")); + expect(log).toEqual(["a@1", "b@1"]); + }); +}); diff --git a/packages/chord/test/next/remote/provider.test.ts b/packages/chord/test/next/remote/provider.test.ts new file mode 100644 index 000000000..b1be51b9e --- /dev/null +++ b/packages/chord/test/next/remote/provider.test.ts @@ -0,0 +1,974 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { PendingOps } from "../../../src/delta/index.ts"; +import { type Contract, int, local, map, method, obj, remote } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type FacetHost } from "../../../src/next/host/index.ts"; +import { type Compression, memoryLinkPair, type PartLimits } from "../../../src/next/link/index.ts"; +import { DeflatedPart, jsonPart, type Message } from "../../../src/next/link/internal.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { nodeCompression } from "../../../src/next-node/compression.ts"; +import { + addNote, + countInflated, + EDITOR, + expectCollected, + forEachCodec, + metered, + Notes, + type NotesProbe, + partValue, + RAW_PEER, + RawPeer, + rawLinked, + serveNotes, + type TestPrincipal, + testCodec, + transported, + wireNames, +} from "../support.ts"; + +/** Sends a hold of `Type` (its own version and fingerprint unless overridden), of `origin` when given. */ +function hold( + peer: RawPeer, + id: number, + Type: Contract, + params: unknown, + override: Partial = {}, + origin?: number, +) { + const { name: type, version, fingerprint } = { ...Type, ...override }; + const from = origin === undefined ? {} : { origin }; + peer.send({ kind: "hold", ...from, hold: id, type, version, fingerprint, params: jsonPart(params) }); +} + +/** Sends a call of the method at index `method` of the bound variant, with `args` (none when left out). */ +function call(peer: RawPeer, id: number, holdId: number, method: number, args?: unknown, origin?: number) { + const from = origin === undefined ? {} : { origin }; + const payload = args === undefined ? {} : { args: jsonPart(args) }; + peer.send({ kind: "call", ...from, call: id, hold: holdId, method, ...payload }); +} + +/** A remote type kept per id. */ +const Kept = remote("app.kept", { version: 1, params: { id: int() }, variants: { v: { state: { n: int() } } } }); + +/** The wire's indexes of `Notes`'s reader variant. */ +const READER = wireNames(Notes, "reader"); + +const { add, fail, wait } = EDITOR.methods as { add: number; fail: number; wait: number }; + +/** A server serving `Notes` and a raw consumer peer stamped `child`. */ +async function notesServer(child: TestPrincipal = { kind: "owner" }) { + let probe: NotesProbe | undefined; + const linked = await rawLinked( + (h) => { + probe = serveNotes(h); + }, + { child }, + ); + return { ...linked, probe: probe! }; +} + +forEachCodec(() => { + describe("holds arriving on a link", () => { + it("bind as the link's principal: bound with the variant and the snapshot", async () => { + const { peer, server } = await notesServer({ kind: "guest", session: "S1" }); + await addNote(server, "S1", "a"); + hold(peer, 0, Notes, { session: "S1" }); + const [bound] = (await peer.next("bound")).holds; + expect(bound).toMatchObject({ hold: 0, variant: READER.variant, failed: [] }); + expect(READER.named(bound!.members)).toEqual({ notes: [{ text: "a" }], count: 1 }); + }); + + it.each([ + [ + "a type not served here", + { type: "app.nope" }, + { code: "unavailable", message: "app.nope is not served here" }, + ], + [ + "a local type", + { type: "app.note-store" }, + { code: "unavailable", message: "app.note-store is not served here" }, + ], + [ + "another version", + { version: 2 }, + { code: "contract_mismatch", message: "app.notes v2 does not match the served v1" }, + ], + [ + "another fingerprint", + { fingerprint: "0000000000000000" }, + { code: "contract_mismatch", message: "app.notes v1 does not match the served v1" }, + ], + ])("refuse %s with a broken failed", async (_, override, error) => { + const { peer } = await notesServer(); + const { type, ...rest } = override as { type?: string }; + hold(peer, 0, { ...Notes, name: type ?? Notes.name } as Contract, { session: "S1" }, rest); + expect(await peer.next("failed")).toMatchObject({ + hold: 0, + cause: { kind: "broken", error: { kind: "chord", error } }, + }); + }); + + it("refuse invalid params with a broken failed", async () => { + const { peer } = await notesServer(); + hold(peer, 0, Notes, { session: 1 }); + expect(await peer.next("failed")).toMatchObject({ + cause: { kind: "broken", error: { kind: "chord", error: { code: "invalid", reason: "params" } } }, + }); + }); + + it("arriving before load resolved wait for it", async () => { + const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + hold(peer, 0, Notes, { session: "S1" }); + await transported(); + expect(peer.kinds()).toEqual(["hello"]); + await server.load( + [{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], + context, + ); + expect((await peer.next("bound")).holds[0]).toMatchObject({ hold: 0, variant: EDITOR.variant }); + }); + }); + + describe("calls arriving on a link", () => { + it("run as the link's principal with the child's incarnation; results and errors go back", async () => { + const { peer, probe } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + call(peer, 0, 0, add, { text: "x" }); + expect(partValue((await peer.next("result")).value!)).toBe(1); + call(peer, 1, 0, fail, { how: "plugin" }); + expect((await peer.next("error")).error).toEqual({ + kind: "plugin", + code: "quota", + message: "too many", + details: { limit: 1 }, + }); + call(peer, 2, 0, wait); + await transported(); + expect(probe.waits[0]).toMatchObject({ principal: { kind: "owner" }, incarnation: RAW_PEER }); + }); + + it("outside the binding's variant cannot be named: an index past its methods, forged at the wire, closes the link", async () => { + const { peer, errors } = await notesServer({ kind: "guest", session: "S1" }); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + // The reader variant has one method, `wait`; the editor's `add` has no index in it. + call(peer, 0, 0, Object.keys(READER.methods).length, { text: "x" }); + await transported(); + expect(peer.closed).toBe(true); + expect(errors).toMatchObject([{ kind: "protocol_error", error: { message: "hold 0: no method 1" } }]); + }); + + it("check their arguments", async () => { + const { peer } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + call(peer, 0, 0, add, { text: "x", extra: 1 }); + expect((await peer.next("error")).error).toMatchObject({ + kind: "chord", + error: { code: "invalid", reason: "args" }, + }); + }); + + it("before bound are refused not_ready (race 3)", async () => { + const Slow = remote("app.slow", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const { peer } = await rawLinked((h) => + h.serve(Slow, (_, _principal, self) => { + self.open( + () => new Promise(() => {}), + () => {}, + ); + return { variant: "v", members: { n: constant(1) } }; + }), + ); + hold(peer, 0, Slow, {}); + call(peer, 0, 0, 0); + expect((await peer.next("error")).error).toMatchObject({ kind: "chord", error: { code: "not_ready" } }); + }); + + it("cancelled are aborted and get no reply; a result crossing the cancel is never sent (race 1)", async () => { + const { peer, probe } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + call(peer, 0, 0, wait); + await transported(); + peer.send({ kind: "abort", call: 0 }); + await transported(); + expect(probe.waits[0]!.abortSignal?.aborted).toBe(true); + expect(peer.kinds()).toEqual(["hello"]); + }); + }); + + describe("ids", () => { + it("a link ignores ids it does not know and messages for the other direction", async () => { + const { peer, errors } = await notesServer(); + peer.send({ kind: "release", hold: 9 }); + peer.send({ kind: "abort", call: 9 }); + call(peer, 0, 9, add, { text: "x" }); + peer.send({ kind: "gone", hold: 9, error: { kind: "chord", error: { code: "released", message: "m" } } }); + await transported(); + expect(peer.closed).toBe(false); + expect(peer.kinds()).toEqual(["hello"]); + expect(errors).toEqual([]); + }); + + it("a hold reusing a live id closes the link", async () => { + const { peer, errors } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + hold(peer, 0, Notes, { session: "S1" }); + await transported(); + expect(peer.closed).toBe(true); + expect(errors).toMatchObject([{ kind: "protocol_error", error: { message: "hold 0: the id is in use" } }]); + }); + + it("a call reusing a live id closes the link", async () => { + const { peer } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + call(peer, 0, 0, wait); + call(peer, 0, 0, wait); + await transported(); + expect(peer.closed).toBe(true); + }); + + it("a message the codec cannot decode closes the link", async () => { + const { peer } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + peer.sendBytes(new Uint8Array([0xff, 0xff])); + await transported(); + expect(peer.closed).toBe(true); + }); + }); + + describe("one message per link per commit (W1)", () => { + it("three holds on one link: one state message, one part per hold, one encode each", async () => { + const { peer, server } = await notesServer(); + for (const id of [0, 1, 2]) hold(peer, id, Notes, { session: "S1" }); + for (const _ of [0, 1, 2]) await peer.next("bound"); + const { context: counting, counts } = await metered(); + const store = await server.get(Notes, { session: "S1" }, context); + if (store.variant === "editor") await store.add({ text: "a" }, counting); + await transported(); + const states = peer.received.filter((message) => message.kind === "state"); + expect(states).toHaveLength(1); + expect(states[0]).toMatchObject({ holds: [{ hold: 0 }, { hold: 1 }, { hold: 2 }] }); + expect(counts()["chord.link.encodes"]).toBe(3); + // No validation or walk of state on the provider: only the argument checks of `add` and the store's `add`. + expect(counts()["chord.data.validations"]).toBe(2); + expect(counts()["chord.data.walks"]).toBeUndefined(); + }); + + it("a derive travels with its source in the same part", async () => { + const { peer, server } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + await addNote(server, "S1", "a"); + const state = await peer.next("state"); + expect(Object.keys(EDITOR.named(state.holds[0]!.members)).sort()).toEqual(["count", "notes"]); + }); + + it("two links: one message each; when one closes, the other keeps updating (race 8)", async () => { + const { peer, server } = await notesServer(); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const other = new RawPeer(b); + hold(peer, 0, Notes, { session: "S1" }); + hold(other, 0, Notes, { session: "S1" }); + await peer.next("bound"); + await other.next("bound"); + await addNote(server, "S1", "a"); + await peer.next("state"); + await other.next("state"); + peer.close(); + await transported(); + await addNote(server, "S1", "b"); + const next = await other.next("state"); + expect(Object.keys(EDITOR.named(next.holds[0]!.members)).sort()).toEqual(["count", "notes"]); + }); + }); + + describe("races at the provider", () => { + /** Serves `Kept`, recording each factory run and each native close. */ + function serveKept(h: Parameters[0]>[0], events: string[]) { + h.serve(Kept, ({ id }, _principal, self) => { + events.push(`factory ${id}`); + self.open( + async () => null, + (_value, reason) => void events.push(`close ${reason.kind}`), + ); + return { variant: "v", members: { n: constant(id) } }; + }); + } + + it("a new hold and the old one's release in one turn hand the object over without ending it (R18)", async () => { + const events: string[] = []; + const { peer } = await rawLinked((h) => serveKept(h, events)); + hold(peer, 0, Kept, { id: 1 }); + await peer.next("bound"); + hold(peer, 1, Kept, { id: 1 }); + peer.send({ kind: "release", hold: 0 }); + expect((await peer.next("bound")).holds[0]).toMatchObject({ hold: 1 }); + expect(events).toEqual(["factory 1"]); + }); + + it("a hold released right after it was sent keeps no object", async () => { + const events: string[] = []; + const { peer } = await rawLinked((h) => serveKept(h, events)); + hold(peer, 0, Kept, { id: 1 }); + await Promise.resolve(); + peer.send({ kind: "release", hold: 0 }); + await transported(); + // The object ended: a new hold makes a new one. + hold(peer, 1, Kept, { id: 1 }); + await peer.next("bound"); + expect(events.filter((event) => event.startsWith("factory"))).toEqual(["factory 1", "factory 1"]); + }); + + it("a hold released while it waits for load takes nothing once load resolves", async () => { + const events: string[] = []; + const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + await transported(); + hold(peer, 0, Kept, { id: 1 }); + await Promise.resolve(); + peer.send({ kind: "release", hold: 0 }); + await transported(); + await server.load( + [{ name: "@test/app", facet: defineFacet((h) => serveKept(h, events)) }], + context, + ); + await transported(); + expect(events).toEqual([]); + }); + + it("a release while activating unholds; bound is never sent (race 6)", async () => { + const Slow = remote("app.slow", { + version: 1, + params: { id: int() }, + variants: { v: { state: { n: int() } } }, + }); + const aborted: string[] = []; + const { peer } = await rawLinked((h) => + h.serve(Slow, ({ id }, _principal, self) => { + self.open( + (ctx) => + new Promise((resolve) => + ctx.abortSignal?.addEventListener("abort", () => resolve(aborted.push(`${id}`))), + ), + () => {}, + ); + return { variant: "v", members: { n: constant(id) } }; + }), + ); + hold(peer, 0, Slow, { id: 1 }); + await transported(); + peer.send({ kind: "release", hold: 0 }); + await transported(); + expect(aborted).toEqual(["1"]); + expect(peer.kinds()).toEqual(["hello"]); + }); + + it("a hold while the previous instance of its identity tears down waits for it (race 7)", async () => { + const Closing = remote("app.closing", { + version: 1, + params: { id: int() }, + variants: { v: { state: { n: int() } } }, + }); + const events: string[] = []; + let finish: () => void = () => {}; + const { peer } = await rawLinked((h) => + h.serve(Closing, ({ id }, _principal, self) => { + self.open( + async () => void events.push("open"), + () => + new Promise((resolve) => { + events.push("closing"); + finish = () => { + events.push("closed"); + resolve(); + }; + }), + ); + return { variant: "v", members: { n: constant(id) } }; + }), + ); + hold(peer, 0, Closing, { id: 1 }); + await peer.next("bound"); + peer.send({ kind: "release", hold: 0 }); + hold(peer, 1, Closing, { id: 1 }); + await transported(); + expect(events).toEqual(["open", "closing"]); + finish(); + await peer.next("bound"); + expect(events).toEqual(["open", "closing", "closed", "open"]); + }); + + it("a commit right after the binding continues exactly from the snapshot (race 2)", async () => { + const { peer, server } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + const adding = addNote(server, "S1", "a"); + const [bound] = (await peer.next("bound")).holds; + await adding; + await transported(); + const snapshot = EDITOR.named(bound!.members) as { notes: unknown[] }; + const states = peer.received.filter((message) => message.kind === "state"); + const added = states.flatMap((message) => (EDITOR.named(message.holds[0]!.members).notes ?? []) as unknown[]); + expect(snapshot.notes.length + added.length).toBe(1); + }); + + it("closing a link during a batch drops its state message and releases its holds (race 9)", async () => { + const Counter = remote("app.counter", { + version: 1, + params: { id: int() }, + variants: { v: { state: { n: int(), seen: int() }, methods: { bump: method() } } }, + }); + const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + // Called inside the batch, from a derive the hold follows. + let during: () => void = () => {}; + const ends: string[] = []; + const { server, errors } = await rawLinked((h) => + h.serve(Counter, ({ id }, _principal, self) => { + const n = self.state(N); + self.open( + async () => null, + () => void ends.push(`counter ${id} ended`), + ); + return { + variant: "v", + members: { + n: self.derive([n], ([{ n }]) => n), + seen: self.derive([n], ([{ n }]) => { + if (n > 0) during(); + return n; + }), + bump: async (ctx) => { + await n.change((draft) => void draft.n++, ctx); + return null; + }, + }, + }; + }), + ); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + hold(peer, 0, Counter, { id: 1 }); + await peer.next("bound"); + during = () => a.close(); + call(peer, 0, 0, 0); + await transported(); + expect(peer.kinds()).toEqual(["hello"]); + expect(ends).toEqual(["counter 1 ended"]); + expect(errors).toEqual([]); + }); + }); + + describe("local types over a link", () => { + it("never cross one", async () => { + const Store = local("app.store", { version: 1, params: {}, members: { n: int() } }); + const { peer } = await rawLinked((h) => h.serve(Store, () => ({ n: constant(1) }))); + hold(peer, 0, Store, {}); + expect(await peer.next("failed")).toMatchObject({ + cause: { kind: "broken", error: { kind: "chord", error: { code: "unavailable" } } }, + }); + }); + }); +}); + +/** A store whose `fill` sets `cells[0..n)` to a value in one commit: `n` leaf ops on distinct paths. */ +const Grid = remote("app.grid", { + version: 1, + params: {}, + variants: { + v: { + state: { grid: obj({ cells: map(int()) }), one: int() }, + methods: { fill: method(obj({ n: int(), value: int() })) }, + }, + }, +}); + +const GridDoc = defineState("grid", { schema: obj({ cells: map(int()) }), version: 1, initial: () => ({ cells: {} }) }); + +function serveGrid(h: FacetHost): void { + h.serve(Grid, (_, _principal, self) => { + const grid = self.state(GridDoc); + return { + variant: "v", + members: { + grid, + one: constant(1), + fill: async ({ n, value }, ctx) => { + await grid.change((draft) => { + for (let cell = 0; cell < n; cell++) draft.cells[`c${cell}`] = value; + }, ctx); + return null; + }, + }, + }; + }); +} + +/** A server serving `Notes` and `Grid` to a raw peer that acks only when told, the link's context counting. */ +async function slowConsumer() { + const counting = await metered(); + const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + await server.load( + [ + { + name: "@test/app", + facet: defineFacet((h) => { + serveNotes(h); + serveGrid(h); + }), + }, + ], + context, + ); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, counting.context); + const peer = new RawPeer(b, { ack: false }); + await transported(); + return { server, peer, counts: counting.counts }; +} + +/** The `state` messages received and not taken. */ +function statesOf(peer: RawPeer): Extract[] { + return peer.received.filter((message) => message.kind === "state"); +} + +/** The `bound` and `state` messages received and not taken, by kind. */ +function windowed(peer: RawPeer): string[] { + return peer.kinds().filter((kind) => kind === "bound" || kind === "state"); +} + +const GRID = wireNames(Grid, "v"); + +forEachCodec(() => { + describe("the window at the provider", () => { + it("holds at most two unacknowledged bound and state messages; what waits goes out folded, encoded once", async () => { + const { server, peer, counts } = await slowConsumer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + for (const text of ["a", "b", "c", "d"]) await addNote(server, "S1", text); + await transported(); + expect(windowed(peer)).toEqual(["state"]); + const before = counts()["chord.link.encodes"] ?? 0; + peer.ack(); + await transported(); + const [, pending] = peer.received.filter(({ kind }) => kind === "state") as Extract< + Message, + { kind: "state" } + >[]; + // Three commits waited: one message, one entry, `count`'s three replacements folded into the last. + expect(EDITOR.named(pending!.holds[0]!.members).count).toEqual([["r", 4]]); + expect(counts()["chord.link.encodes"]! - before).toBe(1); + expect(windowed(peer)).toEqual(["state", "state"]); + await addNote(server, "S1", "e"); + await transported(); + expect(windowed(peer)).toEqual(["state", "state"]); + }); + + it("ignores acks that cross nothing: they never open the window past two (race 18)", async () => { + const { server, peer } = await slowConsumer(); + for (const _ of [0, 1, 2]) peer.ack(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + for (const text of ["a", "b", "c"]) await addNote(server, "S1", text); + await transported(); + expect(windowed(peer)).toEqual(["state"]); + }); + + it("lets a hold that binds while the window is full wait: the waiting state first, its bound at the next slot", async () => { + const { server, peer } = await slowConsumer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + await addNote(server, "S1", "a"); + await addNote(server, "S1", "b"); + hold(peer, 1, Notes, { session: "S1" }); + await transported(); + expect(windowed(peer)).toEqual(["state"]); + peer.ack(); + await transported(); + expect(windowed(peer)).toEqual(["state", "state"]); + await addNote(server, "S1", "c"); + peer.ack(); + await transported(); + // Hold 1's snapshot is taken now, with every commit so far. + const bound = peer.received.findLast(({ kind }) => kind === "bound") as Extract; + expect(bound.holds.map(({ hold }) => hold)).toEqual([1]); + expect(EDITOR.named(bound.holds[0]!.members).count).toBe(3); + }); + + it("overflows past 10,000 pending ops to a replacement with the current value (race 23)", async () => { + const { peer } = await slowConsumer(); + hold(peer, 0, Grid, {}); + await peer.next("bound"); + const fill = (id: number, value: number) => + peer.send({ + kind: "call", + call: id, + hold: 0, + method: GRID.methods.fill!, + args: jsonPart({ n: 6_000, value }), + }); + fill(0, 1); + fill(1, 2); + fill(2, 3); + for (const _ of [0, 1, 2]) await peer.next("result"); + // The first fill took the second slot; the next two overflowed the queue. + peer.ack(); + await transported(); + const states = peer.received.filter(({ kind }) => kind === "state") as Extract[]; + const last = GRID.named(states.at(-1)!.holds[0]!.members).grid as [string, { cells: object }][]; + expect(last[0]![0]).toBe("r"); + expect(Object.keys(last[0]![1].cells)).toHaveLength(6_000); + }); + }); + + describe("origins at the provider", () => { + it("key holds and calls by (origin, id): the same ids on two origins are two holds; answers carry the origin", async () => { + const { peer } = await slowConsumer(); + hold(peer, 0, Notes, { session: "S1" }); + hold(peer, 0, Notes, { session: "S2" }, {}, 3); + const [own, relayed] = [await peer.next("bound"), await peer.next("bound")]; + expect(own.origin).toBeUndefined(); + expect(relayed.origin).toBe(3); + call(peer, 0, 0, add, { text: "a" }); + call(peer, 0, 0, add, { text: "b" }, 3); + const results = [await peer.next("result"), await peer.next("result")]; + expect(results.map(({ origin, call }) => ({ origin, call }))).toEqual([ + { origin: undefined, call: 0 }, + { origin: 3, call: 0 }, + ]); + }); + + it("keep a window per origin: a stalled origin folds its commits, another keeps receiving each", async () => { + const { server, peer } = await slowConsumer(); + hold(peer, 0, Notes, { session: "S1" }); + hold(peer, 0, Notes, { session: "S1" }, {}, 1); + await peer.next("bound"); + await peer.next("bound"); + for (const text of ["a", "b", "c", "d"]) { + await addNote(server, "S1", text); + // Only origin 0 acknowledges. + peer.ack(); + } + await transported(); + const states = statesOf(peer); + expect(states.filter(({ origin }) => origin === undefined)).toHaveLength(4); + // Origin 1: its bound and one state fill its window; three commits wait, folded into its next state. + expect(states.filter(({ origin }) => origin === 1)).toHaveLength(1); + peer.ack(1); + peer.ack(1); + await transported(); + const folded = statesOf(peer) + .filter(({ origin }) => origin === 1) + .at(-1)!; + expect(EDITOR.named(folded.holds[0]!.members).count).toEqual([["r", 4]]); + }); + + it("count calls per origin against N: 256 on each of two origins run, one more on one closes the link", async () => { + const { peer, errors } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + hold(peer, 0, Notes, { session: "S1" }, {}, 1); + await peer.next("bound"); + await peer.next("bound"); + for (let id = 0; id < 256; id++) { + call(peer, id, 0, wait); + call(peer, id, 0, wait, undefined, 1); + } + await transported(); + expect(peer.closed).toBe(false); + call(peer, 256, 0, wait, undefined, 1); + await transported(); + expect(peer.closed).toBe(true); + expect(errors).toMatchObject([ + { kind: "protocol_error", error: { message: "call 256: more than 256 calls" } }, + ]); + }); + + it("ignore a call on a hold they do not know before counting it (Q1)", async () => { + const { peer, errors } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + for (let id = 0; id < 256; id++) call(peer, id, 0, wait); + call(peer, 256, 9, wait); + call(peer, 257, 0, wait, undefined, 4); + await transported(); + expect(peer.closed).toBe(false); + expect(errors).toEqual([]); + }); + + it("closed releases the origin's holds and aborts its calls; another origin keeps its own", async () => { + const { peer, probe, server } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + hold(peer, 0, Notes, { session: "S2" }, {}, 2); + await peer.next("bound"); + await peer.next("bound"); + call(peer, 0, 0, wait, undefined, 2); + await transported(); + const [waiting] = probe.waits; + peer.send({ kind: "closed", origin: 2 }); + await transported(); + expect(waiting!.abortSignal?.aborted).toBe(true); + // Its ids are free: nothing answers them, a new hold of origin 2 starts over. + call(peer, 1, 0, add, { text: "x" }, 2); + await addNote(server, "S1", "a"); + await transported(); + expect(peer.kinds().filter((kind) => kind === "result" || kind === "error")).toEqual([]); + expect(statesOf(peer).map(({ origin }) => origin)).toEqual([undefined]); + }); + + it("carry constants and failed members, and an empty variant's binding, in a first bound (race 24)", async () => { + const Empty = remote("app.empty", { version: 1, params: {}, variants: { v: {} } }); + const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + const { peer } = await rawLinked((h) => { + h.serve(Empty, () => ({ variant: "v", members: {} })); + h.serve(Halves, (_, _principal, self) => { + const n = self.state(N); + return { + variant: "v", + members: { + one: constant(1), + half: self.derive([n], ([{ n }]) => { + if (n === 2) throw new Error("half threw"); + return n / 2; + }), + set: async (value, ctx) => { + await n.change((draft) => { + draft.n = value; + }, ctx); + return null; + }, + }, + }; + }); + }); + hold(peer, 0, Halves, {}); + await peer.next("bound"); + call(peer, 0, 0, HALVES.methods.set!, 2); + await peer.next("result"); + hold(peer, 1, Empty, {}); + hold(peer, 2, Halves, {}); + const bounds = [await peer.next("bound"), await peer.next("bound")]; + const [empty, halves] = [1, 2].map((id) => bounds.find(({ holds }) => holds[0]!.hold === id)!); + expect(empty.holds).toEqual([{ hold: 1, variant: 0, members: [], failed: [] }]); + expect(halves.holds).toEqual([ + { + hold: 2, + variant: 0, + members: [{ member: HALVES.state.one!, value: expect.anything() }], + failed: [ + { member: HALVES.state.half!, error: { kind: "plugin", code: "internal", message: "half threw" } }, + ], + }, + ]); + expect(HALVES.named(halves.holds[0]!.members)).toEqual({ one: 1 }); + }); + }); + + describe("GC of the window", () => { + it("a closed origin drops its pending queue's ops: nothing keeps them", async () => { + const push = PendingOps.prototype.push; + const pushed: WeakRef[] = []; + // Patched by hand: a spy would keep every argument in its call log. + PendingOps.prototype.push = function (this: PendingOps, ops) { + pushed.push(new WeakRef(ops)); + push.call(this, ops); + }; + try { + const { server, peer } = await slowConsumer(); + hold(peer, 0, Notes, { session: "S1" }, {}, 1); + await peer.next("bound"); + for (const text of ["a", "b", "c"]) await addNote(server, "S1", text); + expect(pushed.length).toBeGreaterThan(0); + peer.send({ kind: "closed", origin: 1 }); + await transported(); + await expectCollected(...pushed); + expect(peer.closed).toBe(false); + } finally { + PendingOps.prototype.push = push; + } + }); + }); + + describe("the call limit", () => { + it("closes a link whose peer sends more calls than this side announced", async () => { + const { peer, errors } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + for (let id = 0; id <= 256; id++) call(peer, id, 0, wait); + await transported(); + expect(peer.closed).toBe(true); + expect(errors).toMatchObject([ + { kind: "protocol_error", error: { message: "call 256: more than 256 calls" } }, + ]); + }); + + it("frees a slot per reply, cancel and release", async () => { + const { peer, errors } = await notesServer(); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + for (let id = 0; id < 256; id++) call(peer, id, 0, wait); + peer.send({ kind: "abort", call: 0 }); + call(peer, 256, 0, wait); + peer.send({ kind: "release", hold: 0 }); + hold(peer, 1, Notes, { session: "S1" }); + await peer.next("bound"); + for (let id = 300; id < 556; id++) call(peer, id, 1, wait); + await transported(); + expect(peer.closed).toBe(false); + expect(errors).toEqual([]); + }); + }); +}); + +/** `Halves`: a constant, and a member that fails once `set(2)`. */ +const Halves = remote("app.halves", { + version: 1, + params: {}, + variants: { v: { state: { one: int(), half: int() }, methods: { set: method(int()) } } }, +}); + +const HALVES = wireNames(Halves, "v"); + +forEachCodec(() => { + describe("limits per part at the provider", () => { + /** + * A server serving `Notes` behind a raw consumer whose link has `limits`, compressing with `compression`; the raw + * peer is its child, or its parent (only a parent may send compressed arguments). + */ + async function limited(limits: PartLimits, compression?: Compression, role?: "child" | "parent") { + return rawLinked((h) => void serveNotes(h), { limits, compression, role }); + } + + /** Deflated JSON of `{ text }` with `length` characters. */ + const deflatedText = (length: number) => + new DeflatedPart(nodeCompression.deflate(UTF8.encode(JSON.stringify({ text: "x".repeat(length) })))); + + it("fails a call whose argument part is past the wire limit with too_large/part; the link stays", async () => { + const { peer } = await limited({ part: 100 }); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + call(peer, 0, 0, add, { text: "x".repeat(200) }); + expect(await peer.next("error")).toMatchObject({ + call: 0, + error: { kind: "chord", error: { code: "too_large", reason: "part" } }, + }); + call(peer, 1, 0, add, { text: "fits" }); + expect(partValue((await peer.next("result")).value!)).toBe(1); + expect(peer.closed).toBe(false); + }); + + it("fails a hold whose params are past the limit broken too_large/part", async () => { + const { peer } = await limited({ part: 30 }); + hold(peer, 0, Notes, { session: "s".repeat(40) }); + expect(await peer.next("failed")).toMatchObject({ + hold: 0, + cause: { kind: "broken", error: { kind: "chord", error: { code: "too_large", reason: "part" } } }, + }); + }); + + it("fails a call whose deflated argument inflates past the decoded limit, on both inflate paths", async () => { + for (const compression of [nodeCompression, undefined]) { + const { peer } = await limited({ decoded: 10_000 }, compression, "parent"); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + peer.send({ kind: "call", call: 0, hold: 0, method: add, args: deflatedText(50_000) }); + expect(await peer.next("error")).toMatchObject({ error: { error: { code: "too_large" } } }); + call(peer, 1, 0, add, { text: "fits" }); + expect((await peer.next("result")).call).toBe(1); + } + }); + + it("inflates a deflated argument within the hold's stamped decoded limit, a bomb included (both paths)", async () => { + const output = countInflated(); + try { + for (const compression of [nodeCompression, undefined]) { + // The stamped limit is smaller than the link's own: it bounds the inflate. + const { peer } = await limited({}, compression, "parent"); + peer.send({ ...notesHold(0), limit: 1000 }); + await peer.next("bound"); + const bomb = new DeflatedPart( + nodeCompression.deflate(UTF8.encode(JSON.stringify("x".repeat(64 << 20)))), + ); + peer.send({ kind: "call", call: 0, hold: 0, method: add, args: bomb }); + expect(await peer.next("error")).toMatchObject({ error: { error: { code: "too_large" } } }); + peer.send({ kind: "call", call: 1, hold: 0, method: add, args: deflatedText(500) }); + expect((await peer.next("result")).call).toBe(1); + } + // The asynchronous inflate stopped within a chunk of the stamped limit, never near the bomb's 64 MiB. + expect(output.bytes).toBeLessThan(1024 * 1024); + } finally { + output.restore(); + } + }); + + it("checks a deflated argument's wire size before inflating it (both paths)", async () => { + for (const compression of [nodeCompression, undefined]) { + const { peer, errors } = await limited({ part: 18 }, compression, "parent"); + hold(peer, 0, Notes, { session: "S1" }); + await peer.next("bound"); + // Corrupt data past the wire limit: refused by its size, never inflated (no protocol error). + peer.send({ kind: "call", call: 0, hold: 0, method: add, args: new DeflatedPart(new Uint8Array(20)) }); + expect(await peer.next("error")).toMatchObject({ error: { error: { code: "too_large" } } }); + expect(peer.closed).toBe(false); + expect(errors).toEqual([]); + } + }); + + it("inflates nothing for a call it ignores: an unknown hold (Q1), a refused one (both paths)", async () => { + for (const compression of [nodeCompression, undefined]) { + const { peer, errors } = await limited({}, compression, "parent"); + const corrupt = () => new DeflatedPart(new Uint8Array([255, 255, 255])); + peer.send({ kind: "call", call: 0, hold: 9, method: add, args: corrupt() }); + peer.send({ ...notesHold(0), type: "app.kept" }); + await peer.next("failed"); + peer.send({ kind: "call", call: 1, hold: 0, method: 0, args: corrupt() }); + await transported(); + expect(peer.closed).toBe(false); + expect(errors).toEqual([]); + } + }); + + it("refuses compressed arguments from a child, and compressed params on any link: protocol errors", async () => { + const child = await limited({}); + hold(child.peer, 0, Notes, { session: "S1" }); + await child.peer.next("bound"); + child.peer.send({ kind: "call", call: 0, hold: 0, method: add, args: deflatedText(10) }); + await transported(); + expect(child.peer.closed).toBe(true); + expect(child.errors).toMatchObject([{ kind: "protocol_error" }]); + const parent = await limited({}, undefined, "parent"); + parent.peer.send({ ...notesHold(0), params: new DeflatedPart(nodeCompression.deflate(UTF8.encode("{}"))) }); + await transported(); + expect(parent.peer.closed).toBe(true); + expect(parent.errors).toMatchObject([{ kind: "protocol_error" }]); + }); + + it("honours a stamped decoded limit only on a parent link: a child's is ignored (race 12)", async () => { + const { peer } = await limited({}); + peer.send({ ...notesHold(0), limit: 10 }); + await peer.next("bound"); + call(peer, 0, 0, add, { text: "longer than ten bytes" }); + expect((await peer.next("result")).call).toBe(0); + }); + }); +}); + +/** A hold of `Notes` for S1 as a message. */ +function notesHold(id: number): Extract { + const { name: type, version, fingerprint } = Notes; + return { kind: "hold", hold: id, type, version, fingerprint, params: jsonPart({ session: "S1" }) }; +} + +const UTF8 = new TextEncoder(); diff --git a/packages/chord/test/next/remote/relay.test.ts b/packages/chord/test/next/remote/relay.test.ts new file mode 100644 index 000000000..78fbddb0c --- /dev/null +++ b/packages/chord/test/next/remote/relay.test.ts @@ -0,0 +1,732 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { local, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { type Compression, type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; +import { binaryCodec, jsonPart, type Message, textCodec } from "../../../src/next/link/internal.ts"; +import type * as Values from "../../../src/next/link/values.ts"; +import { RelayedHold } from "../../../src/next/remote/relay.ts"; +import { nodeCompression } from "../../../src/next-node/compression.ts"; +import { + addNote, + countInflated, + EDITOR, + expectCollected, + fakeTimers, + forEachCodec, + instancesOf, + PlacedNotes, + RawPeer, + rawWorkers, + rejection, + relayed, + type TestPrincipal, + testCodec, + transported, + wireNames, +} from "../support.ts"; + +/** How often a part was parsed in this file, by anyone. */ +const parses = vi.hoisted(() => ({ count: 0 })); + +vi.mock("../../../src/next/link/values.ts", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + parsePart: (part: string | Uint8Array) => { + parses.count++; + return original.parsePart(part); + }, + parsePayload: (part: string | Uint8Array, segments: readonly Uint8Array[] | undefined) => { + parses.count++; + return original.parsePayload(part, segments); + }, + }; +}); + +/** Every relayed hold started in this file, weakly. */ +const relayedHolds = instancesOf(RelayedHold, "start"); + +const READER = wireNames(PlacedNotes, "reader"); + +const { wait, add } = EDITOR.methods as { wait: number; add: number }; + +/** A server routing to raw workers, and a raw tab on a link it accepted as `principal`. */ +async function rawRelay(options: { readonly principal?: TestPrincipal; readonly calls?: number } = {}) { + const workers: RawPeer[] = []; + const relay = await relayed({ resolve: rawWorkers(workers, { calls: options.calls }) }); + const tab = rawTab(relay.server, options.principal); + return { ...relay, workers, raw: tab }; +} + +/** A raw tab on a new link the server accepts as `principal` (owner by default). */ +function rawTab(server: Awaited>["server"], principal?: TestPrincipal): RawPeer { + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: principal ?? { kind: "owner" }, codec: testCodec() }, context); + return new RawPeer(b); +} + +/** Sends a hold of `PlacedNotes` for `session` with id `hold`, naming its address. */ +function holdPlaced(peer: RawPeer, hold: number, session = "S1"): void { + peer.send({ + kind: "hold", + hold, + type: PlacedNotes.name, + version: PlacedNotes.version, + fingerprint: PlacedNotes.fingerprint, + params: jsonPart({ session }), + address: { kind: "session", params: { id: session } }, + }); +} + +/** The raw worker at `index` once it exists (a resolve runs a turn after the hold arrived). */ +async function workerAt(workers: RawPeer[], index = 0): Promise { + for (let tries = 0; workers[index] === undefined && tries < 10; tries++) await transported(); + return workers[index]!; +} + +/** A hold as a raw worker received it: its id and its origin, which every answer names. */ +type Held = { readonly hold: number; readonly origin?: number }; + +/** The origin field of an answer to `held`. */ +function originOf({ origin }: Held): { origin?: number } { + return origin === undefined ? {} : { origin }; +} + +/** Binds `held` on a raw worker with `PlacedNotes`'s editor (or reader) variant and `count`. */ +function bindPlaced(peer: RawPeer, held: Held, count = 0, variant = EDITOR): void { + const members = [ + { member: variant.state.notes!, value: "[]" }, + { member: variant.state.count!, value: jsonPart(count) }, + ]; + peer.send({ + kind: "bound", + ...originOf(held), + holds: [{ hold: held.hold, variant: variant.variant, members, failed: [] }], + }); +} + +/** A `state` of `count` for `holds` (of one origin) on a raw worker. */ +function countState(holds: readonly Held[], count: number): Extract { + const ops = jsonPart([["r", count]]); + return { + kind: "state", + ...originOf(holds[0]!), + holds: holds.map(({ hold }) => ({ + hold, + members: [{ member: EDITOR.state.count!, ops }], + failed: [], + retired: [], + })), + }; +} + +forEachCodec(() => { + describe("ids and origins", () => { + it("keep the downstream ids upstream, under the downstream link's origin; answers come back by it", async () => { + const { raw, workers } = await rawRelay(); + holdPlaced(raw, 7); + const worker = await workerAt(workers); + const held = await worker.next("hold"); + expect(held).toMatchObject({ hold: 7, origin: 1 }); + bindPlaced(worker, held, 3); + const bound = await raw.next("bound"); + expect(bound.origin).toBeUndefined(); + expect(bound.holds[0]).toMatchObject({ hold: 7, variant: EDITOR.variant }); + raw.send({ kind: "call", call: 5, hold: 7, method: add, args: jsonPart({ text: "x" }) }); + // Calls are forwarded at once, with the same id. + expect(await worker.next("call")).toMatchObject({ origin: 1, call: 5, hold: 7, method: add }); + worker.send({ kind: "result", origin: 1, call: 5, value: "1" }); + const result = await raw.next("result"); + expect(result).toMatchObject({ kind: "result", call: 5 }); + expect(result.origin).toBeUndefined(); + raw.send({ kind: "retry", hold: 7 }); + expect(await worker.next("retry")).toEqual({ kind: "retry", origin: 1, hold: 7 }); + }); + + it("give each downstream link its own origin, never reused", async () => { + const { raw, server, workers } = await rawRelay(); + holdPlaced(raw, 0); + const worker = await workerAt(workers); + const one = await worker.next("hold"); + const other = rawTab(server); + holdPlaced(other, 0); + const two = await worker.next("hold"); + expect([one.origin, two.origin]).toEqual([1, 2]); + other.close(); + await transported(); + holdPlaced(rawTab(server), 0); + expect((await worker.next("hold")).origin).toBe(3); + }); + + it("drop entries, notices and replies for holds of the origin relayed over another link", async () => { + const { raw, workers } = await rawRelay(); + holdPlaced(raw, 0, "S1"); + holdPlaced(raw, 1, "S2"); + const one = await workerAt(workers, 0); + const x = await one.next("hold"); + const two = await workerAt(workers, 1); + const y = await two.next("hold"); + expect([x.origin, y.origin]).toEqual([1, 1]); + bindPlaced(one, x); + await raw.next("bound"); + raw.send({ kind: "call", call: 0, hold: 0, method: wait }); + await one.next("call"); + // Worker two speaks for worker one's hold of the same tab origin. + two.send(countState([x], 9)); + two.send({ kind: "loading", origin: 1, hold: 0 }); + two.send({ kind: "result", origin: 1, call: 0 }); + two.send({ + kind: "gone", + origin: 1, + hold: 0, + error: { kind: "chord", error: { code: "unavailable", message: "forged" } }, + }); + bindPlaced(two, x); + await transported(); + expect(raw.kinds()).toEqual(["hello"]); + // Its windowed messages, left empty, are acknowledged to it at once. + expect(two.kinds().filter((kind) => kind === "ack")).toEqual(["ack", "ack"]); + // A message with its own hold and the other's: only its own entry is forwarded. + bindPlaced(two, y); + await raw.next("bound"); + two.send({ kind: "state", origin: 1, holds: [...countState([x], 9).holds, ...countState([y], 3).holds] }); + const state = await raw.next("state"); + expect(state.holds.map(({ hold }) => hold)).toEqual([1]); + one.send({ kind: "result", origin: 1, call: 0 }); + expect(await raw.next("result")).toMatchObject({ call: 0 }); + }); + + it("ignore answers for an origin from a link it relayed nothing to", async () => { + const { raw, server, workers } = await rawRelay(); + holdPlaced(raw, 0); + const worker = await workerAt(workers); + const held = await worker.next("hold"); + // Another tab speaks for the first tab's origin on its own link. + const other = rawTab(server); + await transported(); + bindPlaced(other, held); + await transported(); + expect(raw.kinds()).toEqual(["hello"]); + bindPlaced(worker, held); + await raw.next("bound"); + }); + }); + + describe("parts at a relay", () => { + it("are never parsed: params, snapshots, ops, arguments and results travel as they came", async () => { + const { raw, workers } = await rawRelay(); + await transported(); + const before = parses.count; + holdPlaced(raw, 0); + const worker = await workerAt(workers); + const held = await worker.next("hold"); + bindPlaced(worker, held, 1); + await raw.next("bound"); + worker.send(countState([held], 2)); + await raw.next("state"); + raw.send({ kind: "call", call: 0, hold: 0, method: add, args: jsonPart({ text: "x" }) }); + await worker.next("call"); + worker.send({ kind: "result", origin: held.origin, call: 0, value: "2" }); + await raw.next("result"); + expect(parses.count).toBe(before); + }); + }); + + describe("the window through a relay", () => { + it("forwards every bound and state at once and relays each ack to its worker", async () => { + const workers: RawPeer[] = []; + const relay = await relayed({ resolve: rawWorkers(workers, { ack: false }) }); + const [a, b] = memoryLinkPair(); + relay.server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const tab = new RawPeer(b, { ack: false }); + holdPlaced(tab, 0); + const worker = await workerAt(workers); + const held = await worker.next("hold"); + bindPlaced(worker, held); + for (const count of [1, 2, 3]) worker.send(countState([held], count)); + await transported(); + // The relay holds nothing back: the worker's window for this origin is the limit. + expect(tab.kinds()).toEqual(["hello", "bound", "state", "state", "state"]); + expect(worker.kinds().filter((kind) => kind === "ack")).toEqual([]); + tab.ack(); + tab.ack(); + await transported(); + expect(worker.received.filter(({ kind }) => kind === "ack")).toEqual([ + { kind: "ack", origin: held.origin }, + { kind: "ack", origin: held.origin }, + ]); + }); + + it("acknowledges to the right worker in order: two workers sharing one tab", async () => { + const workers: RawPeer[] = []; + const relay = await relayed({ resolve: rawWorkers(workers, { ack: false }) }); + const [a, b] = memoryLinkPair(); + relay.server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const tab = new RawPeer(b, { ack: false }); + holdPlaced(tab, 0, "S1"); + const one = await workerAt(workers, 0); + const x = await one.next("hold"); + holdPlaced(tab, 1, "S2"); + const two = await workerAt(workers, 1); + const y = await two.next("hold"); + bindPlaced(one, x); + await transported(); + bindPlaced(two, y); + await transported(); + tab.ack(); + await transported(); + expect(one.kinds().filter((kind) => kind === "ack")).toEqual(["ack"]); + expect(two.kinds().filter((kind) => kind === "ack")).toEqual([]); + tab.ack(); + await transported(); + expect(two.kinds().filter((kind) => kind === "ack")).toEqual(["ack"]); + }); + + it("never tears a commit: a slow tab's states fold at the worker, a derive over two holds sees both at one moment (race 21)", async () => { + const Pair = local("app.pair", { version: 1, params: {}, members: { pair: str() } }); + const relay = await relayed(); + const seen: string[] = []; + const tab = createHost({ onError: () => {} }); + const [a, b] = memoryLinkPair(); + relay.server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const slow = pausable(b); + tab.connect(slow.link, { principal: { kind: "server" }, codec: testCodec() }, context); + await tab.load( + [ + { + name: "@test/pair", + facet: defineFacet((h) => + h.serve(Pair, (_, self) => { + const one = self.get(PlacedNotes, { session: "S1" }); + const two = self.get(PlacedNotes, { session: "S1" }); + return { + pair: self.derive([one.count, two.count], ([x, y]) => { + seen.push(`${x}/${y}`); + return `${x}/${y}`; + }), + }; + }), + ), + }, + ], + context, + ); + const view = await tab.get(Pair, {}, context); + const watch = await view.pair.watch(context); + watch.start(async () => {}); + const worker = relay.workers.get("session|S1")!; + slow.pause(); + slow.kinds.length = 0; + for (const text of ["a", "b", "c", "d", "e"]) await addNote(worker, "S1", text); + await transported(); + // Two states in the tab origin's window at the worker, the other commits folded there. + expect(slow.kinds).toEqual(["state", "state"]); + slow.resume(); + await transported(); + await transported(); + expect(view.pair.value).toBe("5/5"); + expect(slow.kinds).toEqual(["state", "state", "state"]); + expect(seen.every((pair) => pair.split("/")[0] === pair.split("/")[1])).toBe(true); + expect(relay.errors.server).toEqual([]); + }); + }); + + describe("an upstream drop", () => { + fakeTimers(); + + it("sends loading downstream, fails relayed calls unavailable, holds again with the same id and forwards the new bound, another variant allowed (race 12)", async () => { + const { tab, workers, errors } = await rawRelay(); + const got = tab.get(PlacedNotes, { session: "S1" }, context); + const first = await workerAt(workers); + const held = await first.next("hold"); + bindPlaced(first, held, 1); + const notes = await got; + expect(notes.variant).toBe("editor"); + const call = notes.wait(context); + await first.next("call"); + first.close(); + expect(await rejection(call)).toMatchObject({ code: "unavailable" }); + await transported(); + expect(notes.state.value).toEqual({ kind: "loading" }); + await vi.advanceTimersByTimeAsync(1000); + const second = await workerAt(workers, 1); + const again = await second.next("hold"); + expect(again).toMatchObject({ hold: held.hold, origin: held.origin }); + bindPlaced(second, again, 4, READER); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + expect(notes.variant).toBe("reader"); + expect(notes.count.value).toBe(4); + expect(errors.tab).toEqual([]); + }); + + it("answers a call that crossed the loading notice unavailable (race 13)", async () => { + const { raw, workers } = await rawRelay(); + holdPlaced(raw, 0); + const worker = await workerAt(workers); + bindPlaced(worker, await worker.next("hold")); + await raw.next("bound"); + worker.close(); + await transported(); + raw.send({ kind: "call", call: 0, hold: 0, method: wait }); + expect(await raw.next("loading")).toEqual({ kind: "loading", hold: 0 }); + expect((await raw.next("error")).error).toMatchObject({ error: { code: "unavailable" } }); + }); + }); + + describe("a downstream release", () => { + it("before the upstream hold was sent drops it; after, releases it upstream; messages crossing it are dropped (race 14)", async () => { + let finish: () => void = () => {}; + const workers: RawPeer[] = []; + const make = rawWorkers(workers); + const { server } = await relayed({ + resolve: async (address) => { + await new Promise((resolve) => { + finish = resolve; + }); + return make(address); + }, + }); + const raw = rawTab(server); + holdPlaced(raw, 0); + await transported(); + raw.send({ kind: "release", hold: 0 }); + holdPlaced(raw, 1); + await transported(); + finish(); + const worker = await workerAt(workers); + const sent = await worker.next("hold"); + expect(sent.hold).toBe(1); + await transported(); + expect(worker.kinds()).toEqual(["hello"]); + raw.send({ kind: "release", hold: 1 }); + expect(await worker.next("release")).toEqual({ kind: "release", origin: sent.origin, hold: 1 }); + bindPlaced(worker, sent); + worker.send(countState([sent], 1)); + await transported(); + // The relay no longer relays the hold: dropped there (Q1), each message acknowledged to the worker at once. + expect(raw.kinds()).toEqual(["hello"]); + expect(worker.received.filter(({ kind }) => kind === "ack")).toHaveLength(2); + }); + }); + + describe("a downstream drop", () => { + it("closes its origin upstream: one closed per worker, no release or cancel (race 15)", async () => { + const { raw, workers } = await rawRelay(); + holdPlaced(raw, 0, "S1"); + holdPlaced(raw, 1, "S2"); + const one = await workerAt(workers, 0); + const held = await one.next("hold"); + const two = await workerAt(workers, 1); + await two.next("hold"); + bindPlaced(one, held); + await raw.next("bound"); + raw.send({ kind: "call", call: 0, hold: 0, method: wait }); + await one.next("call"); + raw.close(); + await transported(); + for (const worker of [one, two]) { + const ends = worker.received.filter(({ kind }) => ["release", "abort", "closed"].includes(kind)); + expect(ends).toEqual([{ kind: "closed", origin: held.origin }]); + } + }); + }); + + describe("gone from upstream", () => { + it("is forwarded and the relayed hold forgotten (race 16)", async () => { + const { tab, workers } = await rawRelay(); + const got = tab.get(PlacedNotes, { session: "S1" }, context); + const worker = await workerAt(workers); + const held = await worker.next("hold"); + bindPlaced(worker, held); + const notes = await got; + const watch = await notes.count.watch(context); + worker.send({ + kind: "gone", + ...originOf(held), + hold: held.hold, + error: { kind: "chord", error: { code: "unavailable", message: "revoked" } }, + }); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(() => notes.count.value).toThrow("revoked"); + // A state crossing the gone reaches the tab, which ignores it; nothing closes. + worker.send(countState([held], 9)); + await transported(); + expect(worker.closed).toBe(false); + }); + }); + + describe("GC at a relay", () => { + it("a released relayed hold and a closed downstream link keep no relay state", async () => { + const { refs, server } = await (async () => { + const { raw, workers, server } = await rawRelay(); + const made = relayedHolds.length; + holdPlaced(raw, 0); + holdPlaced(raw, 1); + const worker = await workerAt(workers); + const one = await worker.next("hold"); + bindPlaced(worker, one); + await raw.next("bound"); + raw.send({ kind: "call", call: 0, hold: 0, method: wait }); + await worker.next("call"); + raw.send({ kind: "release", hold: 0 }); + raw.close(); + await transported(); + return { refs: relayedHolds.slice(made), server }; + })(); + expect(refs).toHaveLength(2); + await expectCollected(...refs); + expect(server).toBeDefined(); + }); + }); +}); + +describe("received buffers at a relay", () => { + it("are not kept by the parts of a relayed hold after its message was handled", async () => { + const workers: RawPeer[] = []; + const { server } = await relayed({ resolve: rawWorkers(workers) }); + const [a, b] = memoryLinkPair(); + const buffers: WeakRef[] = []; + const listening: Link = { + send: (message) => a.send(message), + close: () => a.close(), + onClose: (listener) => a.onClose(listener), + onMessage: (listener) => + a.onMessage((message) => { + buffers.push(new WeakRef(message.buffer)); + listener(message); + }), + }; + server.accept(listening, { principal: { kind: "owner" }, codec: binaryCodec }, context); + const tab = (() => { + const peer = new RawPeer(b, { codec: binaryCodec }); + holdPlaced(peer, 0, "S1"); + return peer; + })(); + const worker = await workerAt(workers); + bindPlaced(worker, await worker.next("hold")); + await tab.next("bound"); + expect(buffers.length).toBeGreaterThan(1); + await expectCollected(...buffers); + expect(relayedHolds.at(-1)?.deref()).toBeDefined(); + }); +}); + +describe("mixed codecs", () => { + it("relay between a binary tab link and a text route link", async () => { + const { tab, workers } = await relayed({ codecs: { tab: binaryCodec, route: textCodec } }); + const notes = await tab.get(PlacedNotes, { session: "S1" }, context); + await addNote(workers.get("session|S1")!, "S1", "é"); + await transported(); + expect(notes.notes.value).toEqual([{ text: "é" }]); + if (notes.variant !== "editor") throw new Error("expected the editor"); + expect(await notes.add({ text: "b" }, context)).toBe(2); + const { context: cancellable, cancel } = withCancel(context); + const waiting = notes.wait(cancellable); + cancel(new Error("stop")); + await expect(waiting).rejects.toThrow("stop"); + }); +}); + +/** + * A link end whose incoming messages can be held back (a slow tab): paused, they wait; resumed, they arrive in order. + * `kinds` lists what arrived, decoded with the test codec. + */ +function pausable(link: Link): { readonly link: Link; readonly kinds: string[]; pause(): void; resume(): void } { + let paused = false; + const held: Uint8Array[] = []; + const kinds: string[] = []; + let listener: (message: Uint8Array) => void = () => {}; + link.onMessage((message) => { + kinds.push(testCodec().decode(message).kind); + if (paused) held.push(message); + else listener(message); + }); + return { + kinds, + link: { + send: (message) => link.send(message), + close: () => link.close(), + onClose: (close) => link.onClose(close), + onMessage: (next) => { + listener = next; + }, + }, + pause: () => { + paused = true; + }, + resume: () => { + paused = false; + for (const message of held.splice(0)) listener(message); + }, + }; +} + +/** A compressor that counts its inflates (`nodeCompression` underneath). */ +function countingInflates(): Compression & { inflates: number } { + const counts = { + inflates: 0, + deflate: (bytes: Uint8Array) => nodeCompression.deflate(bytes), + inflate: (bytes: Uint8Array, limit: number) => { + counts.inflates++; + return nodeCompression.inflate(bytes, limit); + }, + }; + return counts; +} + +forEachCodec(() => { + describe("limits and compression through a relay", () => { + it("check an argument's size on the admitting link at the relay: too_large/part, nothing forwarded", async () => { + const workers: RawPeer[] = []; + const relay = await relayed({ resolve: rawWorkers(workers) }); + const [a, b] = memoryLinkPair(); + relay.server.accept(a, { principal: { kind: "owner" }, codec: testCodec(), limits: { part: 100 } }, context); + const raw = new RawPeer(b); + holdPlaced(raw, 0); + const worker = await workerAt(workers); + bindPlaced(worker, await worker.next("hold")); + await raw.next("bound"); + raw.send({ kind: "call", call: 0, hold: 0, method: add, args: jsonPart({ text: "x".repeat(200) }) }); + expect(await raw.next("error")).toMatchObject({ + call: 0, + error: { error: { code: "too_large", reason: "part" } }, + }); + await transported(); + expect(worker.kinds()).not.toContain("call"); + }); + + it("stamp the admitting link's decoded limit on the upstream hold; a tab's own is ignored (race 12)", async () => { + const workers: RawPeer[] = []; + const relay = await relayed({ resolve: rawWorkers(workers) }); + const [a, b] = memoryLinkPair(); + relay.server.accept( + a, + { principal: { kind: "owner" }, codec: testCodec(), limits: { decoded: 4096 } }, + context, + ); + const raw = new RawPeer(b); + raw.send({ + kind: "hold", + hold: 0, + type: PlacedNotes.name, + version: PlacedNotes.version, + fingerprint: PlacedNotes.fingerprint, + params: jsonPart({ session: "S1" }), + address: { kind: "session", params: { id: "S1" } }, + limit: 1_000_000_000, + }); + const worker = await workerAt(workers); + expect((await worker.next("hold")).limit).toBe(4096); + }); + + it("enforce the stamped limit where the part is decoded: the worker refuses the tab's large argument", async () => { + const relay = await relayed({ trusted: true, limits: { decoded: 1000 } }); + const notes = await relay.tab.get(PlacedNotes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("bound as the editor"); + // Within the tab link's wire limit, past its decoded limit, which the trusted worker link does not have. + expect(await rejection(notes.add({ text: "x".repeat(2000) }, context))).toMatchObject({ + code: "too_large", + reason: "part", + }); + expect(await notes.add({ text: "small" }, context)).toBe(1); + }); + + it("forward deflated parts as they came: the relay inflates nothing (a counter)", async () => { + const [server, tab] = [countingInflates(), countingInflates()]; + const relay = await relayed({ compression: { server, worker: nodeCompression, tab } }); + const notes = await relay.tab.get(PlacedNotes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("bound as the editor"); + const text = "lorem ipsum ".repeat(3000); + await notes.add({ text }, context); + await transported(); + expect(notes.notes.value).toEqual([{ text }]); + // The worker's state deflated once at its origin, inflated by its owner, the tab. + expect([server.inflates, tab.inflates]).toEqual([0, 1]); + }); + + it("forward deflated parts as they came at a relay without a compressor: nothing inflated there", async () => { + const inflated = countInflated(); + try { + const tab = countingInflates(); + const relay = await relayed({ compression: { worker: nodeCompression, tab } }); + const notes = await relay.tab.get(PlacedNotes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("bound as the editor"); + await notes.add({ text: "lorem ipsum ".repeat(3000) }, context); + await transported(); + expect(tab.inflates).toBe(1); + expect(inflated.bytes).toBe(0); + } finally { + inflated.restore(); + } + }); + + it("forward answers from upstream unchecked: the end that parses a part checks it against its own link", async () => { + const workers: RawPeer[] = []; + const relay = await relayed({ resolve: rawWorkers(workers, { limits: { part: 50 } }) }); + const getting = relay.tab.get(PlacedNotes, { session: "S1" }, context); + const worker = await workerAt(workers); + const held = await worker.next("hold"); + bindPlaced(worker, held); + const notes = await getting; + if (notes.variant !== "editor") throw new Error("bound as the editor"); + // A state part and a result past the route link's wire limit reach the tab, whose link has none that small. + const ops = jsonPart([["r", 1234567890], ...Array(10).fill(["r", 7])]); + worker.send({ + kind: "state", + ...originOf(held), + holds: [{ hold: held.hold, members: [{ member: EDITOR.state.count!, ops }], failed: [], retired: [] }], + }); + const added = notes.add({ text: "x" }, context); + const call = await worker.next("call"); + worker.send({ kind: "result", ...originOf(held), call: call.call, value: jsonPart("y".repeat(60)) }); + expect(await added).toBe("y".repeat(60)); + await transported(); + expect(notes.count.value).toBe(7); + expect(notes.state.value).toEqual({ kind: "live" }); + }); + + it.each([0, 1200])( + "detach a reference failed too_large from its route; retry %i ms after the drop places it anew, a new hold", + async (wait) => { + const workers: RawPeer[] = []; + const relay = await relayed({ resolve: rawWorkers(workers, { limits: { part: 50 } }) }); + const notes = relay.server.hold(PlacedNotes, { session: "S1" }, context); + const first = await workerAt(workers); + const held = await first.next("hold"); + const members = [ + { member: EDITOR.state.notes!, value: jsonPart([{ text: "z".repeat(60) }]) }, + { member: EDITOR.state.count!, value: jsonPart(1) }, + ]; + first.send({ kind: "bound", holds: [{ hold: held.hold, variant: EDITOR.variant, members, failed: [] }] }); + expect(await first.next("release")).toMatchObject({ hold: held.hold }); + expect(notes.state.value).toMatchObject({ kind: "failed", cause: { error: { code: "too_large" } } }); + // The worker's link drops: nothing holds the address, so nothing resolves it again for this reference. + first.close(); + await new Promise((resolve) => setTimeout(resolve, wait)); + await transported(); + expect(workers).toHaveLength(1); + // The route forgot the address: the retry places the reference anew, resolves it and holds it. + notes.retry(); + expect(notes.state.value.kind).toBe("loading"); + for (let tries = 0; workers.length < 2 && tries < 30; tries++) await new Promise((r) => setTimeout(r, 100)); + const second = await workerAt(workers, 1); + bindPlaced(second, await second.next("hold"), 2); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + expect(notes.count.value).toBe(2); + }, + ); + + it("let a tab without a compressor inflate the worker's deflated parts asynchronously, in order", async () => { + const relay = await relayed({ compression: { worker: nodeCompression, server: nodeCompression } }); + const notes = await relay.tab.get(PlacedNotes, { session: "S1" }, context); + if (notes.variant !== "editor") throw new Error("bound as the editor"); + const text = "lorem ipsum ".repeat(3000); + for (let i = 0; i < 3; i++) await notes.add({ text }, context); + for (let i = 0; i < 100 && notes.count.value < 3; i++) await new Promise((resolve) => setTimeout(resolve, 5)); + expect(notes.notes.value).toEqual([{ text }, { text }, { text }]); + }); + }); +}); diff --git a/packages/chord/test/next/remote/remote-object.test.ts b/packages/chord/test/next/remote/remote-object.test.ts new file mode 100644 index 000000000..e5e3d69b7 --- /dev/null +++ b/packages/chord/test/next/remote/remote-object.test.ts @@ -0,0 +1,768 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { PluginError } from "../../../src/next/data/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { + createHost, + defineFacet, + type Host, + type HostError, + type Reference, + type RefIn, +} from "../../../src/next/host/index.ts"; +import { resolveWatchable } from "../../../src/next/host/reference.ts"; +import { type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; +import { jsonPart, wireError } from "../../../src/next/link/internal.ts"; +import { RemoteObject } from "../../../src/next/remote/remote-object.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { + bindHold, + bindNext, + changesOf, + delivered, + EDITOR, + expectCollected, + failures, + fakeTimers, + forEachCodec, + framesOf, + instancesOf, + linked, + Notes, + partValue, + RawPeer, + rawParent, + rawProvider, + rejection, + serveNotes, + type TestPrincipal, + testCodec, + transported, + wireNames, +} from "../support.ts"; + +/** A counter document of the remote types defined in tests. */ +const Counter = defineState("counter", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + +forEachCodec(() => { + describe("a hold over the parent link", () => { + it("reject a broken failure at once", async () => { + const Broken = remote("app.broken", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const { client } = await linked((h) => + h.serve(Broken, () => { + throw new Error("factory threw"); + }), + ); + expect(await rejection(client.get(Broken, {}, context, { readyTimeout: 5_000 }))).toEqual({ + code: "unavailable", + message: "app.broken: factory threw", + }); + }); + + it("gone: the reference ends, its pending calls reject with the error", async () => { + const { client, peer } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + const hold = await bindNext(peer); + const notes = await got; + const watch = await notes.notes.watch(context); + const call = notes.wait(context); + await peer.next("call"); + peer.send({ + kind: "gone", + hold, + error: { kind: "chord", error: { code: "released", reason: "ended", message: "app.notes(S1): ended" } }, + }); + expect(await rejection(call)).toEqual({ code: "released", reason: "ended", message: "app.notes(S1): ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(() => notes.count.value).toThrow("app.notes(S1): ended"); + }); + + it("two references from one host are two holds", async () => { + const { client, peer } = await rawProvider(); + const first = client.get(Notes, { session: "S1" }, context); + const second = client.get(Notes, { session: "S1" }, context); + expect(await bindNext(peer)).toBe(0); + expect(await bindNext(peer)).toBe(1); + await Promise.all([first, second]); + }); + + it("a member failure fails only that replica: reads throw it, its watch gets a failure call and stays open; the others update", async () => { + const { client, peer } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + const hold = await bindNext(peer); + const notes = await got; + const count = changesOf(await notes.count.watch(context)); + const list = framesOf(await notes.notes.watch(context)); + const failed = [{ member: EDITOR.state.count!, error: wireError(new PluginError("derive", "count threw")) }]; + const members = [{ member: EDITOR.state.notes!, ops: jsonPart([["p", [], 0, 0, [{ text: "a" }]]]) }]; + peer.send({ kind: "state", holds: [{ hold, members, failed, retired: [] }] }); + await delivered(); + expect(count).toEqual([{ kind: "failed", error: expect.objectContaining({ code: "derive" }) }]); + expect(list).toEqual([[[{ text: "a" }], [["p", [], 0, 0, [{ text: "a" }]]]]]); + expect(() => notes.count.value).toThrow("count threw"); + // The provider's derive healed: its next value is a root replacement, in the same hold. + const healed = [{ member: EDITOR.state.count!, ops: jsonPart([["r", 1]]) }]; + peer.send({ kind: "state", holds: [{ hold, members: healed, failed: [], retired: [] }] }); + await delivered(); + expect(notes.count.value).toBe(1); + expect(count.at(-1)).toEqual({ kind: "value", value: 1, ops: [["r", 1]], retired: false }); + }); + }); + + describe("retired members", () => { + it("reach the replicas' watches with the flag: with the member's ops, or alone", async () => { + const { client, peer } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + const hold = await bindNext(peer, [{ text: "a" }]); + const notes = await got; + const list = changesOf(await notes.notes.watch(context)); + const count = changesOf(await notes.count.watch(context)); + const members = [{ member: EDITOR.state.notes!, ops: jsonPart([["r", []]]) }]; + const retired = [EDITOR.state.notes!, EDITOR.state.count!]; + peer.send({ kind: "state", holds: [{ hold, members, failed: [], retired }] }); + await delivered(); + expect(list).toEqual([{ kind: "value", value: [], ops: [["r", []]], retired: true }]); + expect(count).toEqual([{ kind: "value", value: 1, ops: [], retired: true }]); + }); + + it("of a failed member wait for its next value: never read from the failed replica; the link stays open", async () => { + const { client, peer, errors } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + const hold = await bindNext(peer, [{ text: "a" }]); + const notes = await got; + const count = changesOf(await notes.count.watch(context)); + const member = EDITOR.state.count!; + const failed = [{ member, error: wireError(new PluginError("derive", "count threw")) }]; + peer.send({ kind: "state", holds: [{ hold, members: [], failed, retired: [] }] }); + // A retirement alone (no ops) of the failed member, then its healed value. + peer.send({ kind: "state", holds: [{ hold, members: [], failed: [], retired: [member] }] }); + await delivered(); + const value = [{ member, ops: jsonPart([["r", 5]]) }]; + peer.send({ kind: "state", holds: [{ hold, members: value, failed: [], retired: [] }] }); + await transported(); + expect([peer.closed, notes.state.value, errors]).toEqual([false, { kind: "live" }, []]); + expect(count).toEqual([ + { kind: "failed", error: expect.objectContaining({ message: "count threw" }) }, + { kind: "value", value: 5, ops: [["r", 5]], retired: true }, + ]); + }); + + it.each([ + ["names a member outside the variant", [7]], + ["names a member twice", [EDITOR.state.count!, EDITOR.state.count!]], + ])("that %s closes the link", async (_, retired) => { + const { client, peer, errors } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context); + const hold = await bindNext(peer); + await got; + peer.send({ kind: "state", holds: [{ hold, members: [], failed: [], retired }] }); + await transported(); + expect(peer.closed).toBe(true); + expect(errors.map(({ kind }) => kind)).toEqual(["protocol_error"]); + }); + }); + + describe("a snapshot", () => { + it.each([ + ["misses a member", [{ member: EDITOR.state.notes!, value: "[]" }], EDITOR.variant], + [ + "names a member twice", + [ + { member: EDITOR.state.notes!, value: "[]" }, + { member: EDITOR.state.notes!, value: "[]" }, + { member: EDITOR.state.count!, value: "0" }, + ], + EDITOR.variant, + ], + ["names a member outside the variant", [{ member: 7, value: "0" }], EDITOR.variant], + ["names a variant outside the version", [], 5], + ])("that %s closes the link; the reference stays loading", async (_, members, variant) => { + const { client, peer, errors } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + peer.send({ kind: "bound", holds: [{ hold, variant, members, failed: [] }] }); + await transported(); + expect(peer.closed).toBe(true); + expect(errors.map(({ kind }) => kind)).toEqual(["protocol_error"]); + expect(notes.state.value).toEqual({ kind: "loading" }); + expect(notes.variant).toBeUndefined(); + }); + }); + + describe("transient failures over a link", () => { + fakeTimers(); + + it("are waited through; the deadline rejects with the last cause and releases the hold", async () => { + const Flaky = remote("app.flaky", { + version: 1, + params: { id: int() }, + variants: { v: { state: { n: int() } } }, + }); + const opens: string[] = []; + const { client } = await linked((h) => + h.serve(Flaky, ({ id }, _principal, self) => { + self.open( + async (ctx) => { + opens.push(`open ${opens.length + 1}`); + if (opens.length === 1) throw new Error("offline"); + await new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", resolve)); + opens.push("aborted"); + }, + () => {}, + ); + return { variant: "v", members: { n: constant(id) } }; + }), + ); + const got = rejection(client.get(Flaky, { id: 1 }, context, { readyTimeout: 1_500 })); + await transported(); + await vi.advanceTimersByTimeAsync(1_000); + await transported(); + expect(opens).toEqual(["open 1", "open 2"]); + await vi.advanceTimersByTimeAsync(500); + expect(await got).toEqual({ code: "unavailable", message: "app.flaky(1): offline" }); + await transported(); + // The release ended the object: its pending attempt was abandoned. + expect(opens).toEqual(["open 1", "open 2", "aborted"]); + }); + }); + + /** Every remote object bound while this file runs, weakly. */ + const remoteObjects = instancesOf(RemoteObject, "hold"); + + describe("GC of a released remote reference", () => { + it("a released reference retained by the test keeps no hold, replica or object", async () => { + const { server, client } = await linked((h) => void serveNotes(h)); + const { notes, refs } = await (async () => { + const made = remoteObjects.length; + const notes = await client.get(Notes, { session: "S1" }, context); + const refs: WeakRef[] = [new WeakRef(resolveWatchable(notes.notes)!), ...remoteObjects.slice(made)]; + expect(refs).toHaveLength(2); + notes.release(); + await transported(); + return { notes, refs }; + })(); + await expectCollected(...refs); + expect([notes, server, client]).toBeDefined(); + }); + + it("a failure call's error kept by the app keeps neither the replica nor the remote object", async () => { + const Halves = remote("app.halves", { + version: 1, + params: {}, + variants: { v: { state: { half: int() }, methods: { set: method(int()) } } }, + }); + const { server, client } = await linked((h) => + h.serve(Halves, (_, _principal, self) => { + const doc = self.state(Counter); + return { + variant: "v", + members: { + half: self.derive([doc], ([{ n }]) => { + // A fresh error: its stack's frames would keep what threw it. + if (n === 2) throw new Error("half threw"); + return n / 2; + }), + set: async (value, ctx) => { + await doc.change((draft) => { + draft.n = value; + }, ctx); + return null; + }, + }, + }; + }), + ); + const kept: unknown[] = []; + const refs = await (async () => { + const made = remoteObjects.length; + const halves = await client.get(Halves, {}, context); + if (halves.variant !== "v") throw new Error("expected variant v"); + const refs: WeakRef[] = [new WeakRef(resolveWatchable(halves.half)!), ...remoteObjects.slice(made)]; + (await halves.half.watch(context)).start(async (call) => { + if (call.kind === "failed") kept.push(call.error); + }); + await halves.set(2, context); + await transported(); + halves.release(); + await transported(); + return refs; + })(); + expect(kept).toHaveLength(1); + await expectCollected(...refs); + expect([server, client]).toBeDefined(); + }); + }); + + describe("self.get of a type over the link (R18)", () => { + it("a failed hold fails the dependent transiently; bound retries it at once with a new hold", async () => { + const View = local("app.view", { version: 1, params: {}, members: { count: int() } }); + const errors: HostError[] = []; + const client = createHost({ onError: (error) => errors.push(error) }); + const [a, b] = memoryLinkPair(); + client.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const facet = defineFacet((h) => + h.serve(View, (_, self) => ({ count: self.get(Notes, { session: "S1" }).count })), + ); + const loading = client.load([{ name: "@test/view", facet }], context); + const { hold } = await peer.next("hold"); + peer.send({ + kind: "failed", + hold, + cause: { kind: "transient", error: { kind: "chord", error: { code: "unavailable", message: "warming" } } }, + }); + await loading; + expect(errors).toMatchObject([ + { + kind: "activation_failed", + object: "app.view", + failure: { kind: "transient", error: { code: "unavailable" } }, + }, + ]); + const members = [ + { member: EDITOR.state.notes!, value: "[]" }, + { member: EDITOR.state.count!, value: "0" }, + ]; + peer.send({ kind: "bound", holds: [{ hold, variant: EDITOR.variant, members, failed: [] }] }); + // The retry binds its own hold before the failed attempt's is released. + const retried = await bindNext(peer, [{ text: "a" }]); + expect(retried).toBe(1); + expect(await peer.next("release")).toMatchObject({ hold }); + const view = await client.get(View, {}, context); + expect(view.count.value).toBe(1); + }); + }); + + describe("self.get through a relay whose upstream dropped (R18)", () => { + it("a loading notice while loading fails the dependent at once, transient; bound retries it", async () => { + const View = local("app.view", { version: 1, params: {}, members: { count: int() } }); + const errors: HostError[] = []; + const client = createHost({ onError: (error) => errors.push(error) }); + const [a, b] = memoryLinkPair(); + client.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const facet = defineFacet((h) => + h.serve(View, (_, self) => ({ count: self.get(Notes, { session: "S1" }).count })), + ); + const loading = client.load([{ name: "@test/view", facet }], context); + const { hold } = await peer.next("hold"); + // The relay's upstream link dropped before the hold was bound: no wait for the readyTimeout. + peer.send({ kind: "loading", hold }); + await loading; + expect(errors).toMatchObject([ + { + kind: "activation_failed", + object: "app.view", + failure: { kind: "transient", error: { message: expect.stringContaining("app.notes") } }, + }, + ]); + bindHold(peer, hold); + const retried = await bindNext(peer, [{ text: "a" }]); + expect(retried).toBe(1); + const view = await client.get(View, {}, context); + expect(view.count.value).toBe(1); + }); + }); + + describe("a dependent of an object failed transiently over the link", () => { + // Timers stand still: neither the backoff window nor a readyTimeout ends while the test runs. + fakeTimers(); + + it("fails at once with the real cause when its load holds it (no wait for a timeout)", async () => { + const Flaky = remote("app.flaky", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const View = local("app.view", { version: 1, params: {}, members: { n: int() } }); + const { server, client: first } = await linked((h) => + h.serve(Flaky, (_, _principal, self) => { + self.open( + async () => { + throw new Error("offline"); + }, + () => {}, + ); + return { variant: "v", members: { n: constant(1) } }; + }), + ); + // The first client's get makes the object fail; it stays failed for its backoff window. + const failing = first.get(Flaky, {}, context).catch(() => {}); + await transported(); + const errors: HostError[] = []; + const second = createHost({ onError: (error) => errors.push(error) }); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + second.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); + const facet = defineFacet((h) => h.serve(View, (_, self) => ({ n: self.get(Flaky, {}).n }))); + await second.load([{ name: "@test/view", facet }], context); + expect(errors).toMatchObject([ + { + kind: "activation_failed", + object: "app.view", + failure: { + kind: "transient", + error: { message: "app.view: dependency app.flaky failed", cause: { message: "app.flaky: offline" } }, + }, + }, + ]); + await first.close(context); + await failing; + }); + }); +}); + +/** A remote type whose variants share `y`: `a` has `x` and a method, `b` has `z`. */ +const Shifting = remote("app.shifting", { + version: 1, + params: {}, + variants: { + a: { state: { x: int(), y: int() }, methods: { m: method() } }, + b: { state: { y: int(), z: int() } }, + }, +}); + +/** A client whose `View` derives `x`+`y` of a `Shifting` reference, connected to a raw provider; `View` is bound. */ +async function shifting() { + const View = local("app.view", { version: 1, params: {}, members: { sum: int(), label: str() } }); + const errors: HostError[] = []; + const client = createHost({ onError: (error) => errors.push(error) }); + const peer = await rawParent(client); + let held: Reference | undefined; + const facet = defineFacet((h) => + h.serve(View, (_, self) => { + const ref = self.get(Shifting, {}); + held = ref; + const { x, y } = ref as RefIn; + return { + sum: self.derive([x, y], ([x, y]) => x + y), + // Reads the reference's variant when `y` changes. + label: self.derive([y], ([y]) => `${ref.variant}:${y}`), + }; + }), + ); + const loading = client.load([{ name: "@test/view", facet }], context); + const A = wireNames(Shifting, "a"); + const { hold } = await peer.next("hold"); + const members = [ + { member: A.state.x!, value: "1" }, + { member: A.state.y!, value: "2" }, + ]; + peer.send({ kind: "bound", holds: [{ hold, variant: A.variant, members, failed: [] }] }); + await loading; + const view = await client.get(View, {}, context); + return { client, peer, view, ref: held!, errors }; +} + +forEachCodec(() => { + describe("re-holds after a drop", () => { + it("hold every reference not released again on the next link, with that link's fresh ids, in order", async () => { + const { client, peer } = await rawProvider(); + const refs = ["S1", "S2", "S3"].map((session) => client.hold(Notes, { session }, context)); + for (let i = 0; i < 3; i++) await bindNext(peer); + refs[1]!.release(); + peer.close(); + await transported(); + const next = await rawParent(client); + const holds = [await next.next("hold"), await next.next("hold")]; + expect(holds.map(({ hold, params }) => [hold, partValue(params)])).toEqual([ + [0, { session: "S1" }], + [1, { session: "S3" }], + ]); + }); + + it("apply bound in one batch: each member gets one frame [['r', value]] and a fresh decoder", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + await bindNext(peer, [{ text: "a" }]); + await transported(); + const frames = framesOf(await notes.notes.watch(context)); + const counts = framesOf(await notes.count.watch(context)); + peer.close(); + await transported(); + const next = await rawParent(client); + const hold = await bindNext(next, [{ text: "a" }, { text: "b" }]); + await delivered(); + const list = [{ text: "a" }, { text: "b" }]; + expect(frames).toEqual([[list, [["r", list]]]]); + expect(counts).toEqual([[2, [["r", 2]]]]); + // The fresh decoder: the provider's new encoder starts over, interning paths anew. + next.send({ + kind: "state", + holds: [{ hold, members: [{ member: EDITOR.state.count!, ops: '[["r",3]]' }], failed: [], retired: [] }], + }); + await delivered(); + expect(counts.at(-1)).toEqual([3, [["r", 3]]]); + }); + + it("a derive over two members of a re-held reference sees the snapshot whole: one batch", async () => { + const { client, peer, view } = await shifting(); + const sums = framesOf(await view.sum.watch(context)); + peer.close(); + await transported(); + const next = await rawParent(client); + const { hold } = await next.next("hold"); + const A = wireNames(Shifting, "a"); + const members = [ + { member: A.state.x!, value: "10" }, + { member: A.state.y!, value: "20" }, + ]; + next.send({ kind: "bound", holds: [{ hold, variant: A.variant, members, failed: [] }] }); + await transported(); + expect(sums.map(([sum]) => sum)).toEqual([30]); + }); + + it("another variant: vanished members fail (watches stay open), derives over them fail, new members get values, parked calls are checked (race 14)", async () => { + const { client, peer, view, ref } = await shifting(); + expect(view.sum.value).toBe(3); + const a = ref as RefIn; + const x = changesOf(await a.x.watch(context)); + const y = framesOf(await a.y.watch(context)); + const sum = changesOf(await view.sum.watch(context)); + const labels = framesOf(await view.label.watch(context)); + peer.close(); + await transported(); + const parked = rejection(a.m(context)); + const next = await rawParent(client); + const { hold } = await next.next("hold"); + const B = wireNames(Shifting, "b"); + const members = [ + { member: B.state.y!, value: "20" }, + { member: B.state.z!, value: "30" }, + ]; + next.send({ kind: "bound", holds: [{ hold, variant: B.variant, members, failed: [] }] }); + await transported(); + expect(ref.variant).toBe("b"); + const vanished = { kind: "failed", error: expect.objectContaining({ code: "invalid", reason: "member" }) }; + expect(x).toEqual([vanished]); + expect(sum).toEqual([vanished]); + expect(y).toEqual([[20, [["r", 20]]]]); + // The variant changed in the same batch: the derive recomputed with the new one. + expect(labels.map(([label]) => label)).toEqual(["b:20"]); + expect((ref as RefIn).z.value).toBe(30); + expect(() => a.x.value).toThrow('app.shifting (variant b) has no state member "x"'); + expect(await parked).toEqual({ + code: "invalid", + reason: "member", + message: 'app.shifting (variant b) has no method "m"', + }); + // A later binding of the variant that has `x` brings it back in the same watches; the derive heals. + next.close(); + await transported(); + const third = await rawParent(client); + const again = (await third.next("hold")).hold; + const A = wireNames(Shifting, "a"); + const back = [ + { member: A.state.x!, value: "5" }, + { member: A.state.y!, value: "6" }, + ]; + third.send({ kind: "bound", holds: [{ hold: again, variant: A.variant, members: back, failed: [] }] }); + await transported(); + expect(x.at(-1)).toEqual({ kind: "value", value: 5, ops: [["r", 5]], retired: false }); + expect(view.sum.value).toBe(11); + expect(sum.at(-1)).toEqual({ kind: "value", value: 11, ops: [["r", 11]], retired: false }); + }); + + it("calls sent before a drop fail unavailable, never resent; an old link's leftovers never reach the next (race 1)", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + await bindNext(peer); + const { context: cancellable, cancel } = withCancel(context); + const sent = notes.wait(cancellable); + await peer.next("call"); + peer.close(); + expect(await rejection(sent)).toEqual({ code: "unavailable", message: "the link closed" }); + const next = await rawParent(client); + const hold = await bindNext(next); + await transported(); + cancel(new Error("late")); + notes.release(); + await transported(); + expect(next.received).toEqual([{ kind: "ack" }, { kind: "release", hold }]); + }); + + it("a drop before bound, or during the next link's handshake, leaves it loading; the next connect holds again (race 5)", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + await peer.next("hold"); + peer.close(); + await transported(); + const [a, b] = memoryLinkPair(); + client.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + // The peer never answers the handshake; the hold waits in the channel, then the link drops. + b.close(); + await transported(); + expect(notes.state.value).toEqual({ kind: "loading" }); + await bindNext(await rawParent(client)); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + }); + + it("a re-hold refused after a provider upgrade stays failed; the next connect holds it again", async () => { + const NotesV2 = remote("app.notes", { + version: 2, + params: { session: str() }, + variants: { editor: { state: { count: int() } } }, + }); + const { server, client, links } = await linked((h) => void serveNotes(h)); + const upgraded = createHost({ principal: { kind: "server" }, onError: () => {} }); + const facet = defineFacet((h) => + h.serve(NotesV2, () => ({ variant: "editor", members: { count: constant(0) } })), + ); + await upgraded.load([{ name: "@test/app", facet }], context); + const notes = await client.get(Notes, { session: "S1" }, context); + /** Drops the client's link and connects it to `to`. */ + const relink = async (to: Host, from: Link) => { + from.close(); + await transported(); + const [a, b] = memoryLinkPair(); + to.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + client.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); + await transported(); + return a; + }; + const toUpgraded = await relink(upgraded, links[0]); + expect(notes.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "broken", error: { code: "contract_mismatch" } }, + retryAt: undefined, + }); + notes.retry(); + await transported(); + expect(notes.state.value.kind).toBe("failed"); + await relink(server, toUpgraded); + expect(notes.state.value).toEqual({ kind: "live" }); + }); + + it("a drop while failed: loading, held again on the next link (race 19)", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + const error = { code: "unavailable", message: "no configuration" } as const; + peer.send({ kind: "failed", hold, cause: { kind: "broken", error: { kind: "chord", error } } }); + await transported(); + expect(notes.state.value.kind).toBe("failed"); + peer.close(); + await transported(); + expect(notes.state.value).toEqual({ kind: "loading" }); + await bindNext(await rawParent(client)); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + }); + + it("a member watch before the first binding waits for it, and rejects when it fails or is released (race 16)", async () => { + const { client, peer } = await rawProvider(); + const bound = client.hold(Notes, { session: "S1" }, context); + const failing = client.hold(Notes, { session: "S2" }, context); + const released = client.hold(Notes, { session: "S3" }, context); + const watches = [bound.count.watch(context), failing.count.watch(context), released.count.watch(context)]; + await bindNext(peer, [{ text: "a" }]); + const { hold } = await peer.next("hold"); + const error = { code: "unavailable", message: "offline" } as const; + peer.send({ + kind: "failed", + hold, + cause: { kind: "transient", error: { kind: "chord", error } }, + retryIn: 1_000, + }); + released.release(); + expect((await watches[0]!).value).toBe(1); + expect(await rejection(watches[1]!)).toEqual(error); + expect(await rejection(watches[2]!)).toEqual({ code: "released", message: "app.notes: released" }); + }); + }); + + describe("a get across a drop", () => { + fakeTimers(); + + it("keeps waiting within its one deadline, from the moment it was called", async () => { + const { client, peer } = await rawProvider(); + const got = client.get(Notes, { session: "S1" }, context, { readyTimeout: 1_000 }); + const late = rejection(client.get(Notes, { session: "S2" }, context, { readyTimeout: 1_000 })); + await peer.next("hold"); + await peer.next("hold"); + await vi.advanceTimersByTimeAsync(400); + peer.close(); + await transported(); + const next = await rawParent(client); + await bindNext(next, [{ text: "a" }]); + expect((await got).count.value).toBe(1); + await vi.advanceTimersByTimeAsync(600); + expect(await late).toEqual({ + code: "unavailable", + message: "app.notes(S2): did not become live within 1000 ms", + }); + }); + }); + + describe("an activating dependent whose remote dependency's link drops (race 18)", () => { + fakeTimers(); + + const View = local("app.view", { version: 1, params: {}, members: { count: int() } }); + const facet = defineFacet((h) => + h.serve(View, (_, self) => ({ count: self.get(Notes, { session: "S1" }).count })), + ); + + /** Binds the kept hold (0) on `next`; the retry's own hold (1) is bound and the kept one released (R18). */ + async function retried(client: Host, next: RawPeer): Promise { + expect(await bindNext(next, [{ text: "a" }])).toBe(0); + // Retried at once, long before its backoff window (1 s) ends: its new attempt holds anew. + expect(await bindNext(next, [{ text: "a" }])).toBe(1); + expect(await next.next("release")).toEqual({ kind: "release", hold: 0 }); + expect((await client.get(View, {}, context, { readyTimeout: 1 })).count.value).toBe(1); + } + + it("fails transiently at once with the drop, keeps the reference, and is retried at once when it is bound", async () => { + const errors: HostError[] = []; + const client = createHost({ onError: (error) => errors.push(error) }); + const peer = await rawParent(client); + const loading = client.load([{ name: "@test/view", facet }], context); + await peer.next("hold"); + peer.close(); + await loading; + expect(errors).toMatchObject([ + { + kind: "activation_failed", + failure: { + kind: "transient", + error: { + message: "app.view: dependency app.notes(S1) failed", + cause: { message: "the link closed" }, + }, + }, + }, + ]); + await retried(client, await rawParent(client)); + }); + + it("without any link fails at its readyTimeout, keeps the reference, and is retried when the first link binds it", async () => { + const errors: HostError[] = []; + const client = createHost({ readyTimeout: 1_000, onError: (error) => errors.push(error) }); + const loading = client.load([{ name: "@test/view", facet }], context); + await vi.advanceTimersByTimeAsync(1_000); + await loading; + expect(failures(errors)).toEqual(["app.view: transient"]); + await retried(client, await rawParent(client)); + }); + }); + + describe("GC across a drop", () => { + it("a reference that lived through a drop keeps nothing of the closed link", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + await bindNext(peer); + const closed = await (async () => { + const [a, b] = memoryLinkPair(); + peer.close(); + await transported(); + client.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + const second = new RawPeer(b); + await second.next("hello"); + await bindNext(second); + second.close(); + await transported(); + return [new WeakRef(a), new WeakRef(second)]; + })(); + await bindNext(await rawParent(client)); + await expectCollected(...closed); + expect(notes.state.value).toEqual({ kind: "live" }); + }); + }); +}); diff --git a/packages/chord/test/next/remote/replica.test.ts b/packages/chord/test/next/remote/replica.test.ts new file mode 100644 index 000000000..e42bb477b --- /dev/null +++ b/packages/chord/test/next/remote/replica.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import { Replica } from "../../../src/next/remote/replica.ts"; +import { Batch, Derive } from "../../../src/next/state/internal.ts"; +import { changesOf, delivered, framesOf } from "../support.ts"; + +/** A replica of `{ n }` on a fresh batch, with a derive over it attached. */ +function replica() { + const batch = new Batch(() => {}, undefined); + const source = new Replica(batch, { kind: "value", value: { n: 1 } }); + const double = new Derive([source], ([value]) => (value as { n: number }).n * 2, batch); + double.attach( + (declared) => declared as Replica, + () => {}, + ); + return { batch, source, double }; +} + +describe("replicas", () => { + it("adopt a value and its ops inside a batch: one frame, derives recomputed", async () => { + const { batch, source, double } = replica(); + const frames = framesOf(source.openWatch(context)); + const doubles = framesOf(double.openWatch(context)); + batch.commit(() => source.adopt({ n: 2 }, [["s", ["n"], 2]], false), context); + await delivered(); + expect(frames).toEqual([[{ n: 2 }, [["s", ["n"], 2]]]]); + expect(doubles).toEqual([[4, [["r", 4]]]]); + }); + + it("fail: reads throw the error, watches and derives over it get one failure call and recover with a root replacement", async () => { + const { batch, source, double } = replica(); + const calls = changesOf(source.openWatch(context)); + const derived = changesOf(double.openWatch(context)); + const error = new Error("member failed"); + batch.commit(() => source.fail(error), context); + await delivered(); + expect(calls).toEqual([{ kind: "failed", error }]); + expect(derived).toEqual([{ kind: "failed", error }]); + expect(() => source.value).toThrow("member failed"); + // A non-root op cannot apply to a failed member; its recovery is a root replacement. + expect(() => source.decode([["s", ["n"], 2]])).toThrow("root replacement"); + const { value, ops } = source.decode([["r", { n: 3 }]]); + batch.commit(() => source.adopt(value, ops, false), context); + await delivered(); + expect(calls.at(-1)).toEqual({ kind: "value", value: { n: 3 }, ops: [["r", { n: 3 }]], retired: false }); + expect(derived.at(-1)).toEqual({ kind: "value", value: 6, ops: [["r", 6]], retired: false }); + }); + + it("fail once per episode: a second failure while failed gives no call; a retirement meanwhile waits for the value", async () => { + const { batch, source } = replica(); + const calls = changesOf(source.openWatch(context)); + const first = new Error("first"); + batch.commit(() => source.fail(first), context); + batch.commit(() => source.fail(new Error("second")), context); + await delivered(); + // A retirement alone while failed: the value is not read, the flag waits. + batch.commit(() => source.retireAlone(), context); + const { value, ops } = source.decode([["r", { n: 5 }]]); + batch.commit(() => source.adopt(value, ops, false), context); + await delivered(); + expect(calls).toEqual([ + { kind: "failed", error: first }, + { kind: "value", value: { n: 5 }, ops: [["r", { n: 5 }]], retired: true }, + ]); + }); + + it("open a new watch while failed at the last value, the failure first; with no value yet, reject it", async () => { + const { batch, source } = replica(); + const error = new Error("member failed"); + batch.commit(() => source.fail(error), context); + const watch = source.openWatch(context); + expect(watch.value).toEqual({ n: 1 }); + const calls = changesOf(watch); + await delivered(); + expect(calls).toEqual([{ kind: "failed", error }]); + // Failed since its binding: there is no value to start at. + const unborn = new Replica(batch, { kind: "failed", error }); + expect(() => unborn.openWatch(context)).toThrow(error); + }); + + it("end with their hold: reads throw its error, watches end", async () => { + const { source } = replica(); + const watch = source.openWatch(context); + source.end(new ChordError({ code: "released", message: "app.notes: ended" })); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(() => source.value).toThrow("app.notes: ended"); + }); +}); diff --git a/packages/chord/test/next/state/batch.test.ts b/packages/chord/test/next/state/batch.test.ts new file mode 100644 index 000000000..c1fd58801 --- /dev/null +++ b/packages/chord/test/next/state/batch.test.ts @@ -0,0 +1,154 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method, obj } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import type { Host, Reference } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { Batch, type BatchError } from "../../../src/next/state/internal.ts"; +import { delivered, framesOf, loaded, rejection } from "../support.ts"; + +const Num = defineState("num", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); +const Probe = local("app.probe", { + version: 1, + params: {}, + members: { probe: int(), fixed: int(), set: method(int()) }, +}); + +/** Every Chord call a derive compute may not make, each refused or allowed. */ +type Attempts = { [operation: string]: unknown }; + +/** + * A host whose `probe` derive runs `attempt` on every compute with n > 0. `attempt` gets the host and a reference to + * the probe itself; what each operation threw is recorded. + */ +async function probe(attempt: (host: Host, self: Reference) => Attempts) { + const seen: Attempts[] = []; + const holder: { host?: Host; reference?: Reference } = {}; + const { host } = await loaded((h) => + h.serve(Probe, (_, self) => { + const num = self.state(Num); + return { + probe: self.derive([num], ([v]) => { + if (v.n > 0) seen.push(attempt(holder.host!, holder.reference!)); + if (v.n > 1) throw new Error("compute threw"); + return v.n; + }), + fixed: constant(1), + set: async (n, ctx) => { + await num.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }), + ); + holder.host = host; + holder.reference = await host.get(Probe, {}, context); + return { host, reference: holder.reference, seen }; +} + +/** What a synchronous throw or a rejection carries, as JSON for ChordErrors. */ +function outcome(run: () => unknown): unknown { + try { + const result = run(); + return result instanceof Promise ? rejection(result) : result; + } catch (error) { + return rejection(Promise.reject(error)); + } +} + +const reentrant = { code: "invalid", reason: "reentrant" }; + +describe("reentrancy", () => { + const attempts = (host: Host, self: Reference): Attempts => ({ + watch: outcome(() => self.probe.watch(context)), + "watch a constant member": outcome(() => self.fixed.watch(context)), + "watch ref.state": outcome(() => self.state.watch(context)), + release: outcome(() => self.release()), + get: outcome(() => host.get(Probe, {}, context)), + load: outcome(() => host.load([], context)), + close: outcome(() => host.close(context)), + set: outcome(() => self.set(5, context)), + }); + + it("refuses watch (ref.state too), release, get, load, close and a transaction in a commit's derive computes", async () => { + const { reference, seen } = await probe(attempts); + await reference.probe.watch(context); + await reference.set(1, context); + for (const result of await Promise.all(Object.values(seen[0]!))) expect(result).toMatchObject(reentrant); + }); + + it("refuses them in a lazy compute outside a commit", async () => { + const { reference, seen } = await probe(attempts); + await reference.set(1, context); + expect(seen).toEqual([]); + expect(reference.probe.value).toBe(1); + for (const result of await Promise.all(Object.values(seen[0]!))) expect(result).toMatchObject(reentrant); + }); + + it("clears the flag after a throwing compute", async () => { + const { host, reference } = await probe(() => ({})); + await reference.set(2, context); + expect(() => reference.probe.value).toThrow("compute threw"); + expect((await host.get(Probe, {}, context)).probe).toBeDefined(); + await expect(reference.set(3, context)).resolves.toBeNull(); + }); +}); + +describe("delivery", () => { + it("reports a delivery that throws and still delivers the later frames of the commit", async () => { + const Other = defineState("other", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + const Two = local("app.two", { + version: 1, + params: {}, + members: { bad: obj({ a: int() }), other: obj({ n: int() }), set: method(int()) }, + }); + const { host, errors } = await loaded((h) => + h.serve(Two, (_, self) => { + const num = self.state(Num); + const other = self.state(Other); + return { + // `undefined` is no JSON: diffing it throws. + bad: self.derive([num], ([v]) => (v.n === 0 ? { a: 0 } : ({ a: undefined } as never))), + other, + set: async (n, ctx) => { + // `num` is adopted first, so the derive's delivery comes before `other`'s. + await self.tx(async (tx) => { + (await tx.draft(num)).n = n; + (await tx.draft(other)).n = n; + }, ctx); + return null; + }, + }; + }), + ); + const two = await host.get(Two, {}, context); + await two.bad.watch(context); + const frames = framesOf(await two.other.watch(context)); + await two.set(1, context); + await delivered(); + expect(errors).toMatchObject([{ kind: "delivery_failed" }]); + expect(frames.map(([value]) => value)).toEqual([{ n: 1 }]); + }); +}); + +describe("flushes", () => { + it("run once per listing at the end of the commit, after its deliveries; a throwing one is reported", () => { + const reported: BatchError[] = []; + const batch = new Batch((error) => reported.push(error), undefined); + const order: string[] = []; + batch.commit(() => { + batch.flushing(() => order.push("first")); + batch.flushing(() => { + throw new Error("flush threw"); + }); + batch.flushing(() => order.push("third")); + order.push("adopted"); + }, context); + expect(order).toEqual(["adopted", "first", "third"]); + expect(reported).toMatchObject([{ kind: "delivery_failed", error: { message: "flush threw" } }]); + batch.commit(() => {}, context); + expect(order).toEqual(["adopted", "first", "third"]); + }); +}); diff --git a/packages/chord/test/next/state/derive.test.ts b/packages/chord/test/next/state/derive.test.ts new file mode 100644 index 000000000..a6434688f --- /dev/null +++ b/packages/chord/test/next/state/derive.test.ts @@ -0,0 +1,484 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; +import type { Self } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { mapShared } from "../../../src/next/state/index.ts"; +import { Derive } from "../../../src/next/state/internal.ts"; +import type { Watchable } from "../../../src/watch.ts"; +import { + changesOf, + delivered, + expectCollected, + framesOf, + instancesOf, + loaded, + metered, + restartable, +} from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +const Num = defineState("num", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); +const Store = local("app.store", { + version: 1, + params: {}, + members: { n: int(), double: int(), set: method(int()) }, +}); + +/** A store with a number, its double, and a setter. */ +function serveStore(self: Self) { + const num = self.state(Num); + return { + n: self.derive([num], ([v]) => v.n), + double: self.derive([num], ([v]) => v.n * 2), + set: async (n: number, ctx: typeof context) => { + await num.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; +} + +/** Counts derive computes on a host's adapter. */ +async function computes() { + const telemetry = await metered(); + return { telemetry: telemetry.telemetry, count: () => telemetry.counts()["chord.derive.computes"] ?? 0 }; +} + +describe("derives across objects", () => { + it("update in the batch of the commit that changed their source's object", async () => { + const Viewer = local("app.viewer", { version: 1, params: {}, members: { label: str() } }); + const { host } = await loaded((h) => { + h.serve(Store, (_, self) => serveStore(self)); + h.serve(Viewer, (_, self) => ({ + label: self.derive([self.get(Store, {}).double], ([d]) => `double ${d}`), + })); + }); + const store = await host.get(Store, {}, context); + const viewer = await host.get(Viewer, {}, context); + const frames = framesOf(await viewer.label.watch(context)); + await store.set(2, context); + await delivered(); + expect(frames).toEqual([["double 4", [["r", "double 4"]]]]); + }); +}); + +describe("the pull model", () => { + it("computes a diamond's derives once each per batch", async () => { + const { telemetry, count } = await computes(); + const Diamond = local("app.diamond", { version: 1, params: {}, members: { d: int(), set: method(int()) } }); + const { host } = await loaded( + (h) => + h.serve(Diamond, (_, self) => { + const num = self.state(Num); + const b = self.derive([num], ([v]) => v.n + 1); + const c = self.derive([num], ([v]) => v.n + 2); + return { + d: self.derive([b, c], ([x, y]) => x + y), + set: async (n, ctx) => { + await num.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }), + { telemetry }, + ); + const diamond = await host.get(Diamond, {}, context); + const frames = framesOf(await diamond.d.watch(context)); + const before = count(); + await diamond.set(1, context); + expect(count() - before).toBe(3); + await delivered(); + expect(frames.map(([value]) => value)).toEqual([5]); + }); + + it("leaves unwatched derives dirty until read; a one-off read does not make them eager", async () => { + const { telemetry, count } = await computes(); + const { host } = await loaded((h) => h.serve(Store, (_, self) => serveStore(self)), { telemetry }); + const store = await host.get(Store, {}, context); + const before = count(); + await store.set(1, context); + expect(count()).toBe(before); + expect(store.double.value).toBe(2); + expect(store.double.value).toBe(2); + expect(count()).toBe(before + 1); + await store.set(2, context); + expect(count()).toBe(before + 1); + }); + + it("diffs a watched derive's old and new value (counted), and sends no frame when they are equal", async () => { + const telemetry = await metered(); + const Parity = local("app.parity", { + version: 1, + params: {}, + members: { even: obj({ even: int() }), set: method(int()) }, + }); + const { host } = await loaded( + (h) => + h.serve(Parity, (_, self) => { + const num = self.state(Num); + return { + even: self.derive([num], ([v]) => ({ even: v.n % 2 })), + set: async (n, ctx) => { + await num.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }), + { telemetry: telemetry.telemetry }, + ); + const parity = await host.get(Parity, {}, context); + const frames = framesOf(await parity.even.watch(context)); + await parity.set(2, context); + await parity.set(3, context); + await delivered(); + expect(frames).toEqual([[{ even: 1 }, [["s", ["even"], 1]]]]); + expect(telemetry.counts()["chord.derive.diffs"]).toBe(2); + }); +}); + +describe("failures", () => { + it("give the throwing derive's watches and those of the derives over it one failure call; they heal on a later change", async () => { + const { failing, errors } = await failingDerives(); + const [t, u, v] = await Promise.all([ + failing.t.watch(context), + failing.u.watch(context), + failing.v.watch(context), + ]); + const [ts, us] = [changesOf(t), changesOf(u)]; + const frames = framesOf(v); + await failing.set(1, context); + await delivered(); + expect(ts).toEqual([{ kind: "failed", error: THREW }]); + expect(us).toEqual([{ kind: "failed", error: THREW }]); + expect(() => failing.t.value).toThrow(THREW); + expect(() => failing.u.value).toThrow(THREW); + expect(failing.v.value).toBe(1); + await failing.set(2, context); + await delivered(); + // Open all along; the next value replaces the root. + expect(ts.at(-1)).toEqual({ kind: "value", value: 2, ops: [["r", 2]], retired: false }); + expect(us.at(-1)).toEqual({ kind: "value", value: 3, ops: [["r", 3]], retired: false }); + expect(frames.map(([value]) => value)).toEqual([1, 2]); + // The throwing derive's owner reports it, once; the derive over it (its source failed) does not. + expect(errors).toEqual([{ kind: "derive_failed", object: "app.failing", member: "t", error: THREW }]); + }); + + it("give one failure call per episode: a recompute that throws again is quiet; a heal ends the episode", async () => { + const { failing } = await failingDerives(); + const [t, u] = [changesOf(await failing.t.watch(context)), changesOf(await failing.u.watch(context))]; + for (const n of [1, -1, -2, 3, -3]) { + await failing.set(n, context); + await delivered(); + } + const failed = { kind: "failed", error: THREW }; + expect(t).toEqual([failed, { kind: "value", value: 3, ops: [["r", 3]], retired: false }, failed]); + expect(u).toEqual([failed, { kind: "value", value: 4, ops: [["r", 4]], retired: false }, failed]); + }); + + it("open a new watch on a failed derive at its last good value, the failure its first call, then the heal", async () => { + const { failing } = await failingDerives(); + await failing.set(5, context); + // Lazy: the last good value is the last one computed. + expect(failing.u.value).toBe(6); + await failing.set(1, context); + const [t, u] = [await failing.t.watch(context), await failing.u.watch(context)]; + expect([t.value, u.value]).toEqual([5, 6]); + const [ts, us] = [changesOf(t), changesOf(u)]; + await delivered(); + await failing.set(-1, context); + await failing.set(7, context); + await delivered(); + const failed = { kind: "failed", error: THREW }; + expect(ts).toEqual([failed, { kind: "value", value: 7, ops: [["r", 7]], retired: false }]); + expect(us).toEqual([failed, { kind: "value", value: 8, ops: [["r", 8]], retired: false }]); + }); + + it("report once per failure episode, also when nothing watches (a lazy derive fails when read)", async () => { + const { failing, errors } = await failingDerives(); + await failing.set(1, context); + expect(() => failing.t.value).toThrow(THREW); + expect(() => failing.t.value).toThrow(THREW); + await failing.set(-1, context); + expect(() => failing.t.value).toThrow(THREW); + expect(errors).toHaveLength(1); + await failing.set(2, context); + expect(failing.t.value).toBe(2); + await failing.set(1, context); + expect(() => failing.t.value).toThrow(THREW); + expect(errors).toHaveLength(2); + }); + + it("fold a failure and a value for a busy listener: the value replaces the failure, as a root replacement", async () => { + const { failing } = await failingDerives(); + const watch = await failing.t.watch(context); + const calls: unknown[] = []; + const gate = Promise.withResolvers(); + watch.start(async (call) => { + calls.push(call); + if (calls.length === 1) await gate.promise; + }); + await failing.set(5, context); + await delivered(); + await failing.set(1, context); + await failing.set(6, context); + gate.resolve(); + await delivered(); + expect(calls).toEqual([ + { kind: "value", value: 5, ops: [["r", 5]], retired: false }, + { kind: "value", value: 6, ops: [["r", 6]], retired: false }, + ]); + }); +}); + +const THREW = new Error("t threw"); + +/** `app.failing`: `t` throws while `n` is 1 or negative, `u` is `t + 1`, `v` is `n`; the host's reports collected. */ +async function failingDerives() { + const Failing = local("app.failing", { + version: 1, + params: {}, + members: { t: int(), u: int(), v: int(), set: method(int()) }, + }); + const { host, errors } = await loaded((h) => + h.serve(Failing, (_, self) => { + const num = self.state(Num); + const t = self.derive([num], ([v]) => { + if (v.n === 1 || v.n < 0) throw THREW; + return v.n; + }); + return { + t, + u: self.derive([t], ([x]) => x + 1), + v: self.derive([num], ([x]) => x.n), + set: async (n, ctx) => { + await num.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }), + ); + const failing = await host.get(Failing, {}, context); + return { host, failing, errors }; +} + +describe("GC", () => { + it("a detached derive is not held by its former sources", async () => { + const Viewer = local("app.viewer", { version: 1, params: { id: int() }, members: { label: int() } }); + const refs: WeakRef[] = []; + const { host } = await loaded((h) => { + h.serve(Store, (_, self) => serveStore(self)); + h.serve(Viewer, (_, self) => { + const label = self.derive([self.get(Store, {}).n], ([n]) => n); + refs.push(new WeakRef(label)); + return { label }; + }); + }); + const store = await host.get(Store, {}, context); + (await host.get(Viewer, { id: 1 }, context)).release(); + await store.set(1, context); + await expectCollected(...refs); + }); + + it("a watch open on a failed derive keeps nothing more: it ends with its reference, and a retained one holds no source", async () => { + const Paged = local("app.paged", { + version: 1, + params: { id: int() }, + members: { t: int(), set: method(int()) }, + }); + const refs: WeakRef[] = []; + const { host } = await loaded((h) => + h.serve(Paged, (_, self) => { + const num = self.state(Num); + const t = self.derive([num], ([v]) => { + if (v.n === 1) throw THREW; + return v.n; + }); + refs.push(new WeakRef(t), new WeakRef(num)); + return { + t, + set: async (n, ctx) => { + await num.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }), + ); + const paged = await host.get(Paged, { id: 1 }, context); + const watch = await paged.t.watch(context); + const calls = changesOf(watch); + await paged.set(1, context); + await delivered(); + expect(calls).toEqual([{ kind: "failed", error: THREW }]); + paged.release(); + expect(await watch.closed).toEqual({ kind: "ended" }); + // `watch` stays referenced here. + await expectCollected(...refs); + void watch; + await host.close(context); + }); +}); + +describe("GC of handed-out errors", () => { + it("a failure call's fresh error kept by its listener keeps nothing of its host once the host closes", async () => { + const Fresh = local("app.fresh", { version: 1, params: {}, members: { t: int(), set: method(int()) } }); + const refs: WeakRef[] = []; + const kept: unknown[] = []; + await (async () => { + const { host } = await loaded((h) => + h.serve(Fresh, (_, self) => { + const num = self.state(Num); + const t = self.derive([num], ([v]) => { + // A fresh error: its stack's frames hold the derive, the document and the attempt. + if (v.n === 1) throw new Error("fresh"); + return v.n; + }); + refs.push(new WeakRef(t), new WeakRef(num), new WeakRef(self)); + return { + t, + set: async (n, ctx) => { + await num.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }), + ); + refs.push(new WeakRef(host)); + const fresh = await host.get(Fresh, {}, context); + (await fresh.t.watch(context)).start(async (call) => { + if (call.kind === "failed") kept.push(call.error); + }); + await fresh.set(1, context); + await delivered(); + await host.close(context); + })(); + expect(kept).toHaveLength(1); + await expectCollected(...refs); + }); +}); + +describe("mapShared", () => { + it("maps an object element once while it stays the same object; primitives every time", () => { + const f = vi.fn((x: { n: number } | number) => (typeof x === "number" ? x : x.n)); + const map = mapShared(f); + const a = { n: 1 }; + const b = { n: 2 }; + expect(map([a, b, 3])).toEqual([1, 2, 3]); + expect(map([a, { n: 2 }, 3])).toEqual([1, 2, 3]); + expect(f.mock.calls.map(([x]) => x)).toEqual([a, b, 3, { n: 2 }, 3]); + }); + + it("keeps no element alive in its cache", async () => { + const map = mapShared((note: { text: string }) => ({ text: note.text })); + const refs = (() => { + const note = { text: "t" }; + const [result] = map([note]); + return [new WeakRef(note), new WeakRef(result!)]; + })(); + await expectCollected(...refs); + expect(map([{ text: "u" }])).toEqual([{ text: "u" }]); + }); + + it("returns the previous result object for an unchanged element", () => { + const strip = mapShared((note: { text: string; at: number }) => ({ text: note.text })); + const note = { text: "t", at: 1 }; + expect(strip([note])[0]).toBe(strip([note, { text: "u", at: 2 }])[0]); + }); +}); + +const ListV1 = defineState("list", { schema: obj({ items: arr(str()) }), version: 1, initial: () => ({ items: [] }) }); + +describe("the retired flag", () => { + it("a derive over a retired source publishes retired: true although its value and ops are unchanged (rule 21)", async () => { + let list: SingletonHandle<{ items: string[] }> | undefined; + let many: Watchable | undefined; + let twice: Watchable | undefined; + await restartable(objects, (_, self) => { + list = self.state(ListV1); + many = self.derive([list], ([value]) => value.items.length > 1); + twice = self.derive([many], ([value]) => !value); + }); + await list!.change((draft) => void draft.items.push("a"), context); + const manys = changesOf(await many!.watch(context)); + const twices = changesOf(await twice!.watch(context)); + await list!.retire(context); + await delivered(); + expect(manys).toEqual([{ kind: "value", value: false, ops: [], retired: true }]); + expect(twices).toEqual([{ kind: "value", value: true, ops: [], retired: true }]); + await list!.change((draft) => void draft.items.push("a", "b"), context); + await delivered(); + expect(manys.at(-1)).toEqual({ kind: "value", value: true, ops: [["r", true]], retired: false }); + }); + + it("a retirement while the derive is failed travels with its next good value", async () => { + const Gate = defineState("gate", { schema: obj({ on: int() }), version: 1, initial: () => ({ on: 0 }) }); + let num: SingletonHandle<{ n: number }> | undefined; + let gate: SingletonHandle<{ on: number }> | undefined; + let n: Watchable | undefined; + await restartable(objects, (_, self) => { + num = self.state(Num); + gate = self.state(Gate); + n = self.derive([num, gate], ([v, g]) => { + if (g.on === 1 && v.n === 0) throw THREW; + return v.n; + }); + }); + await num!.change((draft) => { + draft.n = 1; + }, context); + await gate!.change((draft) => { + draft.on = 1; + }, context); + const calls = changesOf(await n!.watch(context)); + // The retirement makes `n` 0 again, which throws. + await num!.retire(context); + await num!.change((draft) => { + draft.n = 2; + }, context); + await delivered(); + expect(calls).toEqual([ + { kind: "failed", error: THREW }, + { kind: "value", value: 2, ops: [["r", 2]], retired: true }, + ]); + }); + + it("a retirement visits each derive of a diamond once, not once per path (40 derives)", async () => { + let list: SingletonHandle<{ items: string[] }> | undefined; + let bottom: Watchable | undefined; + await restartable(objects, (_, self) => { + list = self.state(ListV1); + let previous: Watchable = self.derive([list], ([value]) => value.items.length); + // 13 layers of two derives joined by a third: 2^13 paths from the top to the bottom. + for (let layer = 0; layer < 13; layer++) { + const left = self.derive([previous], ([n]) => n + 1); + const right = self.derive([previous], ([n]) => n + 2); + previous = self.derive([left, right], ([a, b]) => a + b); + } + bottom = previous; + }); + await list!.change((draft) => void draft.items.push("a"), context); + const frames = changesOf(await bottom!.watch(context)); + const visits = vi.spyOn(Derive.prototype, "sourceRetired"); + try { + await list!.retire(context); + await delivered(); + expect(visits.mock.calls.length).toBeLessThan(100); + } finally { + visits.mockRestore(); + } + expect(frames.at(-1)).toMatchObject({ retired: true }); + }); +}); diff --git a/packages/chord/test/next/state/watchable.test.ts b/packages/chord/test/next/state/watchable.test.ts new file mode 100644 index 000000000..f59f113c1 --- /dev/null +++ b/packages/chord/test/next/state/watchable.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { int, local, method, obj } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { resolveWatchable } from "../../../src/next/host/reference.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import type { Source, Subscriber } from "../../../src/next/state/internal.ts"; +import { delivered, expectCollected, framesOf, loaded } from "../support.ts"; + +const Num = defineState("num", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); +const Counter = local("app.counter", { + version: 1, + params: {}, + members: { num: obj({ n: int() }), set: method(int()) }, +}); + +/** A counter exposing its document, with background errors collected. */ +async function counter() { + const { host, errors } = await loaded((h) => + h.serve(Counter, (_, self) => { + const num = self.state(Num); + return { + num, + set: async (n, ctx) => { + await num.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }), + ); + return { counter: await host.get(Counter, {}, context), errors }; +} + +describe("watches", () => { + it("start from the value at acquisition; a listener that keeps up gets each commit", async () => { + const { counter: c } = await counter(); + const watch = await c.num.watch(context); + expect(watch.value).toEqual({ n: 0 }); + const frames = framesOf(watch); + await c.set(1, context); + await delivered(); + await c.set(2, context); + await delivered(); + expect(frames).toEqual([ + [{ n: 1 }, [["s", ["n"], 1]]], + [{ n: 2 }, [["s", ["n"], 2]]], + ]); + }); + + it("give a busy listener one call with the newest value and the folded ops", async () => { + const { counter: c } = await counter(); + const watch = await c.num.watch(context); + const frames: [unknown, unknown][] = []; + let release: () => void = () => {}; + watch.start(async (call) => { + if (call.kind !== "value") return; + frames.push([call.value, call.ops]); + // The first call stays busy until the test lets it return. + if (frames.length === 1) + await new Promise((resolve) => { + release = resolve; + }); + }); + await c.set(1, context); + await delivered(); + await c.set(2, context); + await c.set(3, context); + release(); + await delivered(); + expect(frames).toEqual([ + [{ n: 1 }, [["s", ["n"], 1]]], + // Two commits while busy: one call, the second `s` on the same path replaced the first. + [{ n: 3 }, [["s", ["n"], 3]]], + ]); + }); + + it("end cancelled when their context aborts", async () => { + const { counter: c } = await counter(); + const { context: cancellable, cancel } = withCancel(context); + const watch = await c.num.watch(cancellable); + cancel(); + expect(await watch.closed).toEqual({ kind: "aborted" }); + }); + + it("end with listener_error when the listener throws, reported to onError", async () => { + const { counter: c, errors } = await counter(); + const watch = await c.num.watch(context); + const error = new Error("listener threw"); + watch.start(async () => { + throw error; + }); + await c.set(1, context); + expect(await watch.closed).toEqual({ kind: "listener_error", error }); + expect(errors).toEqual([{ kind: "listener_error", error }]); + }); +}); + +describe("constant", () => { + it("never changes; its watches end only by stop or cancellation and hold nothing after", async () => { + const start = constant({ id: "S1" }); + expect(start.value).toEqual({ id: "S1" }); + const refs = await (async () => { + const watch = await start.watch(context); + expect(watch.value).toBe(start.value); + expect(await watch.stop()).toEqual({ kind: "stopped" }); + return [new WeakRef(watch)]; + })(); + await expectCollected(...refs); + }); +}); + +describe("subscribers", () => { + it("get each batch's frame synchronously, inside the commit, and count as watching", async () => { + const { counter: c } = await counter(); + const source = resolveWatchable(c.num) as Source; + const frames: unknown[] = []; + const subscriber: Subscriber = { + advance: (call) => void (call.kind === "value" && frames.push([call.value, call.ops])), + end: () => {}, + }; + source.subscribe(subscriber); + expect(source.watched).toBe(true); + const setting = c.set(1, context); + await setting; + expect(frames).toEqual([[{ n: 1 }, [["s", ["n"], 1]]]]); + source.unsubscribe(subscriber); + expect(source.watched).toBe(false); + await c.set(2, context); + expect(frames).toHaveLength(1); + }); +}); diff --git a/packages/chord/test/next/store/applier.test.ts b/packages/chord/test/next/store/applier.test.ts new file mode 100644 index 000000000..00d8e61f5 --- /dev/null +++ b/packages/chord/test/next/store/applier.test.ts @@ -0,0 +1,114 @@ +// Tests of the applier (slice 6a §13): recorded writes applied as recorded, checked against the store's own records: +// table names and columns by the name rules, indexes, rows against their recorded columns (NULL cells, defaults kept +// as stored, booleans, json(...) cells as JSON text) and documents with their ids kept. + +import { describe, expect, it } from "vitest"; +import { type AppliedWrite, applyWrite } from "../../../src/next/store/applier.ts"; +import { Step } from "../../../src/next/store/step.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { rejection } from "../support.ts"; + +const TABLE = "client:app.store:edits"; + +const table: AppliedWrite = { + kind: "table", + table: TABLE, + version: 2, + state: "ready", + columns: { + id: { type: "str" }, + done: { type: "bool", default: false }, + note: { type: "str", optional: true }, + meta: { type: "json", optional: true, version: 1, fingerprint: "f" }, + }, + key: ["id"], + fingerprint: "t", +}; + +/** Applies `writes` in one transaction of a fresh database; resolves with the rows of the table. */ +async function applied(...writes: AppliedWrite[]): Promise { + const db = await openNodeSqliteDatabase(":memory:"); + try { + return await db.transaction(async (executor) => { + const step = new Step(executor, 1, 1, new Map()); + for (const write of writes) await applyWrite(step, write, "test"); + return executor.all(`SELECT * FROM "${TABLE}"`); + }); + } finally { + await db.close(); + } +} + +describe("applied writes", () => { + it("a table and its rows as recorded: NULL for absent opt cells, booleans as 0 and 1, json(...) as text", async () => { + const rows = await applied(table, { + kind: "put", + table: TABLE, + row: { id: "a", done: true, meta: { x: [1] } }, + version: 1, + columns: { meta: 1 }, + }); + expect(rows).toEqual([{ id: "a", done: 1, note: null, meta: '{"x":[1]}', meta$v: 1, $v: 1 }]); + }); + + it.each<[string, AppliedWrite[], string]>([ + [ + "a put for a table not recorded", + [{ kind: "put", table: TABLE, row: {}, version: 1, columns: {} }], + "is not recorded", + ], + [ + "a key over an opt column", + [{ ...table, key: ["note"] } as AppliedWrite], + '"note" is not a non-opt str or int column', + ], + [ + "a json column without a fingerprint", + [{ ...table, columns: { id: { type: "str" }, meta: { type: "json", version: 1 } } } as AppliedWrite], + "a json(...) column has a version and a fingerprint", + ], + [ + "a default of another type", + [{ ...table, columns: { id: { type: "str" }, n: { type: "int", default: "x" } } } as AppliedWrite], + "is not a int value", + ], + [ + "an index over an unknown column", + [table, { kind: "index", table: TABLE, name: "by_x", columns: ["x"] }], + '"x" is not a non-opt str or int column', + ], + [ + "an index twice", + [ + table, + { kind: "index", table: TABLE, name: "by_id", columns: ["id"] }, + { kind: "index", table: TABLE, name: "by_id", columns: ["id"] }, + ], + "not a new index name", + ], + [ + "a version for a cell the row lacks", + [table, { kind: "put", table: TABLE, row: { id: "a", done: false }, version: 1, columns: { meta: 1 } }], + "no json(...) cell of the row", + ], + [ + "a json cell that is no JSON", + [ + table, + { + kind: "put", + table: TABLE, + row: { id: "a", done: false, meta: 1n as never }, + version: 1, + columns: { meta: 1 }, + }, + ], + "not JSON", + ], + ])("refuses %s", async (_, writes, message) => { + expect(await rejection(applied(...writes))).toMatchObject({ + code: "invalid", + message: expect.stringContaining(message), + }); + }); +}); diff --git a/packages/chord/test/next/store/archive.test.ts b/packages/chord/test/next/store/archive.test.ts new file mode 100644 index 000000000..3df415558 --- /dev/null +++ b/packages/chord/test/next/store/archive.test.ts @@ -0,0 +1,42 @@ +// Tests of the archive part schema (slice 6a §13, rule 25): every part an export yields passes; a part with an +// unknown kind, a missing or extra field or a value of another type is refused, naming its position. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { checkPart } from "../../../src/next/store/archive.ts"; +import { archiveStore, exported } from "../support.ts"; + +describe("the archive part schema", () => { + it("takes every part an export yields", async () => { + const store = await archiveStore(); + const parts = await exported(store); + for (const [position, part] of parts.entries()) expect(checkPart(part, position, context)).toBe(part); + await store.close(context); + }); + + it.each<[string, unknown]>([ + ["an unknown kind", { kind: "blob" }], + ["a missing field", { kind: "end" }], + ["an extra field", { kind: "end", writes: 0, more: 1 }], + ["a negative count", { kind: "end", writes: -1 }], + [ + "a column of an unknown type", + { + kind: "table", + table: "a:b:c", + version: 1, + state: "ready", + columns: { a: { type: "date" } }, + key: ["a"], + fingerprint: "f", + }, + ], + ["a put at version 0", { kind: "put", table: "a:b:c", row: {}, version: 0, columns: {} }], + [ + "a record without an id", + { kind: "document.create", record: { namespace: "n" }, base: { version: 1, fingerprint: "f", value: {} } }, + ], + ])("refuses %s", (_, part) => { + expect(() => checkPart(part, 4, context)).toThrow("import: part 4: not an archive part"); + }); +}); diff --git a/packages/chord/test/next/store/catalog.test.ts b/packages/chord/test/next/store/catalog.test.ts new file mode 100644 index 000000000..451c45cb2 --- /dev/null +++ b/packages/chord/test/next/store/catalog.test.ts @@ -0,0 +1,65 @@ +// The catalog (slice 6a §8): the store's records of its tables, read once at open, changed after COMMIT returned. + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, str } from "../../../src/next/contracts/index.ts"; +import { sqliteBackend } from "../../../src/next/store/backend.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import { type ChordStore, openStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { stageTable } from "../../../src/next/store/table-writes.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { memoryStore, tableIn } from "../support.ts"; + +const Edits = defineTable("edits", { + version: 1, + columns: { id: str(), at: int() }, + key: ["id"], + indexes: { by_at: ["at"] }, +}); +const TABLE = "session:app.store:edits"; + +describe("the catalog", () => { + it("is read once at open: a reopened store knows its tables", async () => { + const directory = await mkdtemp(join(tmpdir(), "chord-catalog-")); + const path = join(directory, "store.sqlite"); + const first = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; + await tableIn(first, Edits); + await first.close(context); + const second = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; + expect(second.table(TABLE)).toEqual({ + table: TABLE, + version: 1, + state: "ready", + columns: Edits.record, + key: ["id"], + fingerprint: Edits.fingerprint, + indexes: { by_at: ["at"] }, + }); + await second.close(context); + await rm(directory, { recursive: true, force: true }); + }); + + it("changes in memory only after COMMIT returned", async () => { + const store = await memoryStore(); + const { version, record: columns, key, fingerprint } = Edits; + const failing: StagedWrite = { + order: "write", + apply: (step) => + step + .run("INSERT INTO no_such_table VALUES (1)") + .then(() => ({ kind: "dropIndex", table: TABLE, name: "x" })), + }; + const table = stageTable({ kind: "table", table: TABLE, version, state: "ready", columns, key, fingerprint }); + await expect( + store.commit([table, failing], { admit: () => true, adopt: () => {} }, context), + ).rejects.toMatchObject({ + reason: "storage_failed", + }); + expect(store.table(TABLE)).toBeUndefined(); + await store.close(context); + }); +}); diff --git a/packages/chord/test/next/store/columns.test.ts b/packages/chord/test/next/store/columns.test.ts new file mode 100644 index 000000000..48e11981c --- /dev/null +++ b/packages/chord/test/next/store/columns.test.ts @@ -0,0 +1,73 @@ +// Table columns (decisions "Flat columns", "Nullable columns and the `set` form"): descriptors and row types. + +import { describe, expect, it } from "vitest"; +import { arr, bool, int, num, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { + type Columns, + columnRecord, + describeColumns, + type PutRowOf, + type RowOf, +} from "../../../src/next/store/columns.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import type { Equal } from "../support.ts"; + +describe("column descriptors", () => { + it("describes the helpers in declaration order", () => { + const columns = describeColumns( + { + id: str(), + n: int({ default: 0 }), + x: num(), + b: bool(), + note: opt(str()), + meta: opt(json(obj({}), { version: 2 })), + }, + "t", + ); + expect(columns.map(({ name, type, optional }) => [name, type, optional])).toEqual([ + ["id", "str", false], + ["n", "int", false], + ["x", "num", false], + ["b", "bool", false], + ["note", "str", true], + ["meta", "json", true], + ]); + expect(columns.map(columnRecord)).toEqual([ + { type: "str" }, + { type: "int", default: 0 }, + { type: "num" }, + { type: "bool" }, + { type: "str", optional: true }, + { type: "json", optional: true, version: 2, fingerprint: expect.any(String) }, + ]); + }); + + it("checks names: the pattern, no two equal ignoring case, no $", () => { + expect(() => describeColumns({ "a-b": str() }, "t")).toThrow("[A-Za-z_][A-Za-z0-9_]*"); + expect(() => describeColumns({ a$v: str() }, "t")).toThrow('column "a$v"'); + expect(() => describeColumns({ path: str(), Path: str() }, "t")).toThrow("only in case"); + expect(describeColumns({ camelCase: str(), _x1: int() }, "t")).toHaveLength(2); + }); + + it("refuses other schemas, bounds, patterns, opt defaults and defaults of another type", () => { + const columns = (value: unknown) => ({ c: value }) as unknown as Columns; + expect(() => describeColumns(columns(arr(str())), "t")).toThrow("expected str()"); + expect(() => describeColumns(columns(obj({})), "t")).toThrow("expected str()"); + expect(() => describeColumns(columns(str({ max: 3 })), "t")).toThrow('"maxLength" is not supported'); + expect(() => describeColumns(columns(str({ pattern: "a" })), "t")).toThrow('"pattern" is not supported'); + expect(() => describeColumns(columns(opt(int({ default: 1 }))), "t")).toThrow("no default"); + expect(() => describeColumns(columns(int({ default: 1.5 })), "t")).toThrow("not a int value"); + expect(() => describeColumns(columns(str({ default: "a\0" })), "t")).toThrow("NUL"); + }); +}); + +describe("row types", () => { + it("types rows with optional opt(...) fields and puts with optional defaulted fields", () => { + const shape = { id: str(), hits: int({ default: 0 }), note: opt(str()) }; + type S = typeof shape; + const row: Equal, { id: string; hits: number } & { note?: string }> = true; + const put: Equal, { id: string } & { hits?: number; note?: string }> = true; + expect([row, put]).toEqual([true, true]); + }); +}); diff --git a/packages/chord/test/next/store/engine.test.ts b/packages/chord/test/next/store/engine.test.ts new file mode 100644 index 000000000..78a41dab0 --- /dev/null +++ b/packages/chord/test/next/store/engine.test.ts @@ -0,0 +1,142 @@ +// The engine (slice 6a §10, §11): persisted documents as bases and deltas, their keys in code point order, retirement. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import type { Op } from "../../../src/delta/index.ts"; +import type { ArchivePart } from "../../../src/next/store/archive.ts"; +import { sqliteBackend } from "../../../src/next/store/backend.ts"; +import { + changeDocument, + createDocument, + type DocumentAddress, + findDocument, + listDocumentKeys, + loadDocument, + retireDocument, +} from "../../../src/next/store/engine.ts"; +import { importStore } from "../../../src/next/store/import.ts"; +import { TABLES } from "../../../src/next/store/schema.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import { type ChordStore, openStore } from "../../../src/next/store/store.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { exported, rejection } from "../support.ts"; + +const OWNER = { namespace: "chord", owner: "session:app.store", kind: "notes" }; + +/** A store on `:memory:` (or `on`) and a commit helper resolving with the created ids. */ +async function engine(on?: ChordStore) { + const store = + on ?? ((await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context)) as ChordStore); + const commit = async (...writes: StagedWrite[]): Promise => { + let created: readonly number[] = []; + await store.commit( + writes, + { + admit: () => true, + adopt: (_seq, ids) => { + created = ids; + }, + }, + context, + ); + return created; + }; + const at = (key: string): DocumentAddress => ({ ...OWNER, key }); + const base = (value: object, version = 1) => ({ + version, + fingerprint: `f${version}`.padEnd(16, "0"), + value: value as never, + }); + const create = (key: string, value: object) => createDocument(at(key), "latest", base(value), JSON.stringify(value)); + const delta = (id: number, revision: number, ops: Op[]) => + changeDocument(id, revision, { kind: "delta", version: 1, ops }, JSON.stringify(ops)); + const read = (fn: Parameters[0]) => store.read(fn, context) as Promise; + return { store, commit, at, base, create, delta, read }; +} + +/** `store`'s archive with `edit` applied to its parts, imported into a fresh `:memory:` store. */ +async function reimported(store: ChordStore, edit: (parts: ArchivePart[]) => unknown[]): Promise { + const parts = edit(await exported(store)); + return (await importStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), parts, context)) as ChordStore; +} + +const LIMIT = Number.MAX_SAFE_INTEGER; + +describe("documents", () => { + it("load as their base with the deltas after it applied; a new base prunes the older revisions", async () => { + const { commit, at, base, create, delta, read } = await engine(); + const [id] = await commit(create("a", { n: 0 })); + await commit(delta(id!, 1, [["s", ["n"], 1]])); + await commit(delta(id!, 2, [["s", ["n"], 2]])); + const record = (await read>>((e) => findDocument(e, at("a"))))!; + expect(record).toMatchObject({ id, revision: 3, version: 1, history: "latest", createdAt: 1 }); + expect(await read((e) => loadDocument(e, record))).toMatchObject({ value: { n: 2 }, deltasSinceBase: 2 }); + const next = base({ m: 9 }, 2); + await commit(changeDocument(id!, 3, { kind: "base", ...next }, JSON.stringify(next.value))); + const rebased = (await read>>((e) => findDocument(e, at("a"))))!; + expect(rebased).toMatchObject({ revision: 4, version: 2, fingerprint: next.fingerprint }); + expect(await read((e) => loadDocument(e, rebased))).toMatchObject({ value: { m: 9 }, deltasSinceBase: 0 }); + expect(await read((e) => e.all(`SELECT seq FROM ${TABLES.revisions}`))).toEqual([{ seq: 4 }]); + }); + + it("list live keys in code point order, paging after a key", async () => { + const { commit, create, read } = await engine(); + await commit(create("\u{1F600}", {}), create("\uFF5E", {}), create("b", {}), create("a", {})); + expect(await read((e) => listDocumentKeys(e, OWNER, undefined, 10))).toEqual(["a", "b", "\uFF5E", "\u{1F600}"]); + expect(await read((e) => listDocumentKeys(e, OWNER, "b", 1))).toEqual(["\uFF5E"]); + }); + + it("retire a latest record with its revisions (rule 19); the address takes a new incarnation", async () => { + const { commit, at, create, read } = await engine(); + const [id] = await commit(create("a", { n: 0 })); + const [next] = await commit(retireDocument(id!, 1), create("a", { n: 1 })); + expect(next).toBe(2); + expect(await read((e) => e.all(`SELECT id FROM ${TABLES.documents}`))).toEqual([{ id: 2 }]); + expect(await read((e) => e.all(`SELECT document_id FROM ${TABLES.revisions}`))).toEqual([{ document_id: 2 }]); + expect(await read((e) => findDocument(e, at("a")))).toMatchObject({ id: 2 }); + }); + + it("refuse two live records at one address and a delta on another version: storage_failed", async () => { + const first = await engine(); + await first.commit(first.create("a", {})); + expect(await rejection(first.commit(first.create("a", {})))).toMatchObject({ reason: "storage_failed" }); + const second = await engine(); + const [id] = await second.commit(second.create("a", {})); + const crossing = changeDocument(id!, 1, { kind: "delta", version: 2, ops: [] }, "[]"); + expect(await rejection(second.commit(crossing))).toMatchObject({ reason: "storage_failed" }); + }); +}); + +describe("counters at the safe-integer limit", () => { + it("allocate document ids up to the safe range after an import near it; past it a commit fails storage_failed", async () => { + const { store: source } = await engine(); + const near = (next: number) => + reimported(source, ([manifest, ...rest]) => [{ ...manifest, nextDocumentId: next }, ...rest]); + const { store, commit, create } = await engine(await near(LIMIT - 1)); + expect(await commit(create("a", { n: 0 }))).toEqual([LIMIT - 1]); + expect(store.nextDocumentId).toBe(LIMIT); + expect(await rejection(commit(create("b", { n: 0 })))).toMatchObject({ reason: "storage_failed" }); + // Two creations in one commit where only one id is left: nothing of it is written. + const both = await engine(await near(LIMIT - 1)); + expect(await rejection(both.commit(both.create("a", { n: 0 }), both.create("b", { n: 0 })))).toMatchObject({ + reason: "storage_failed", + }); + expect([both.store.lastSeq, both.store.nextDocumentId]).toEqual([0, LIMIT - 1]); + }); + + it("count revisions up to the safe range after an import near it; a change past it fails storage_failed", async () => { + const { store: source, commit: seed, create } = await engine(); + await seed(create("a", { n: 0 })); + const store = await reimported(source, (parts) => + parts.map((part) => + part.kind === "document.create" ? { ...part, record: { ...part.record, revision: LIMIT - 1 } } : part, + ), + ); + const { commit, delta, read, at } = await engine(store); + const record = (await read>>((e) => findDocument(e, at("a"))))!; + await commit(delta(record.id, LIMIT - 1, [["s", ["n"], 1]])); + expect(await rejection(commit(delta(record.id, LIMIT, [["s", ["n"], 2]])))).toMatchObject({ + reason: "storage_failed", + }); + }); +}); diff --git a/packages/chord/test/next/store/evaluate.test.ts b/packages/chord/test/next/store/evaluate.test.ts new file mode 100644 index 000000000..9f94e2b94 --- /dev/null +++ b/packages/chord/test/next/store/evaluate.test.ts @@ -0,0 +1,211 @@ +// Tests of the JavaScript evaluator of the table vocabulary, and that it agrees with SQLite on the same nodes. +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { bool, int, num, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { applySet, type EvaluatedRow, matches } from "../../../src/next/store/evaluate.ts"; +import { + add, + and, + coalesce, + compileSet, + compileWhere, + concat, + eq, + gt, + gte, + inArray, + isNull, + lt, + lte, + max, + min, + ne, + not, + or, + prefix, + replacePrefix, + sub, + type WhereNode, + when, +} from "../../../src/next/store/expression.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import type { ColumnRefs } from "../../../src/next/store/query.ts"; +import { createTableSql, TableSql, whereSql } from "../../../src/next/store/sql.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import type { SqliteValue } from "../../../src/sqlite/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const T = defineTable("t", { + version: 1, + columns: { + id: str(), + path: str(), + n: int(), + x: num(), + b: bool(), + note: opt(str()), + count: opt(int()), + meta: opt(json(obj({ a: int() }), { version: 1 })), + }, + key: ["id"], +}); +type C = typeof T extends { "~columns"?: infer X } ? NonNullable : never; +const refs = T.refs as ColumnRefs; +const args = expect.objectContaining({ code: "invalid", reason: "args" }); +describe("the evaluator", () => { + const row: EvaluatedRow = { id: "r", path: "a/x", n: 2, x: 1, b: true, count: 3 }; + + it("follows SQL's NULL rules: a comparison with NULL never matches, not and ne included", () => { + const where = (condition: unknown) => compileWhere(T, condition, "where"); + expect(matches(where(eq(refs.note, "a")), row)).toBe(false); + expect(matches(where(ne(refs.note, "a")), row)).toBe(false); + expect(matches(where(not(eq(refs.note, "a"))), row)).toBe(false); + expect(matches(where(or(isNull(refs.note), ne(refs.note, "a"))), row)).toBe(true); + expect(matches(where(and()), row)).toBe(true); + expect(matches(where(or()), row)).toBe(false); + expect(matches(where(inArray(refs.n, [])), row)).toBe(false); + }); + + it("evaluates every right side against the row before (a swap)", () => { + const Swap = defineTable("s", { version: 1, columns: { id: str(), a: str(), b: str() }, key: ["id"] }); + const set = compileSet( + Swap, + { a: (Swap.refs as { b: unknown }).b, b: (Swap.refs as { a: unknown }).a }, + "update", + ); + expect(applySet(set, { id: "1", a: "x", b: "y" }, "s")).toEqual({ id: "1", a: "y", b: "x" }); + }); + + it("leaves a value without the prefix unchanged and refuses integers outside the safe range (rule 10)", () => { + expect(applySet(compileSet(T, { path: replacePrefix(refs.path, "b/", "c/") }, "u"), row, "t").path).toBe("a/x"); + expect(applySet(compileSet(T, { path: replacePrefix(refs.path, "a/", "") }, "u"), row, "t").path).toBe("x"); + const big = { ...row, n: Number.MAX_SAFE_INTEGER }; + expect(() => applySet(compileSet(T, { n: add(refs.n, 1) }, "u"), big, "t")).toThrow(args); + }); +}); + +describe("one semantics: the evaluator agrees with SQLite", () => { + const ROWS: EvaluatedRow[] = [ + { id: "1", path: "a/x", n: 1, x: 0.5, b: true, note: "z", count: 1 }, + { id: "2", path: "a/\u{1F600}", n: 2, x: 2, b: false }, + { id: "3", path: "a\uFFFF", n: -3, x: -1, b: true, note: "" }, + { id: "4", path: "b", n: 0, x: 0, b: false, count: 0 }, + { id: "5", path: "", n: 9, x: 9.5, b: true, note: "a/" }, + { id: "6", path: "\u{10FFFF}", n: 5, x: 1, b: false, count: -2 }, + ]; + const CONDITIONS = [ + eq(refs.note, "z"), + ne(refs.note, "z"), + not(ne(refs.count, 0)), + lt(refs.path, "a\uFFFF"), + gte(refs.path, "a/\u{1F600}"), + prefix(refs.path, "a"), + prefix(refs.path, ""), + prefix(refs.path, "\u{10FFFF}"), + inArray(refs.path, ["b", "", "x"]), + inArray(refs.b, [true]), + or(isNull(refs.count), gt(refs.x, 1)), + and(not(isNull(refs.note)), lte(refs.n, 1)), + not(or(eq(refs.b, true), lt(refs.count, 0))), + ]; + + async function table() { + const db = await openNodeSqliteDatabase(":memory:"); + await db.run(createTableSql("t", T.record, T.key)); + const sql = new TableSql("t", T.columns, T.rows.physical, T.key); + for (const row of ROWS) await db.run(sql.put, ...T.rows.put(row, 1).params); + return { db, sql }; + } + + it("selects the rows the evaluator matches", async () => { + const { db } = await table(); + for (const condition of CONDITIONS) { + const node = compileWhere(T, condition, "where"); + const params: SqliteValue[] = []; + const text = whereSql(node, params); + const rows = await db.all<{ id: string }>(`SELECT id FROM t WHERE ${text} ORDER BY id`, ...params); + const expected = ROWS.filter((row) => matches(node, row)).map(({ id }) => id); + expect([JSON.stringify(node), rows.map(({ id }) => id)]).toEqual([JSON.stringify(node), expected]); + } + await db.close(); + }); + + it("agrees on every operator's NULL truth under not, and, or and when", async () => { + const PREDICATES = [ + eq(refs.note, "z"), + ne(refs.note, "z"), + lt(refs.count, 1), + lte(refs.count, 0), + gt(refs.note, "a"), + gte(refs.count, 0), + inArray(refs.note, ["z", ""]), + inArray(refs.note, []), + prefix(refs.note, "a"), + prefix(refs.note, ""), + isNull(refs.count), + ]; + const WRAPPED = PREDICATES.flatMap((p) => [ + p, + not(p), + and(p, eq(refs.b, true)), + or(p, eq(refs.b, true)), + not(and(p, eq(refs.b, true))), + not(or(p, eq(refs.b, false))), + ]); + const { db } = await table(); + for (const condition of WRAPPED) { + const node = compileWhere(T, condition, "where"); + const params: SqliteValue[] = []; + const text = whereSql(node, params); + const rows = await db.all<{ id: string }>(`SELECT id FROM t WHERE ${text} ORDER BY id`, ...params); + const expected = ROWS.filter((row) => matches(node, row)).map(({ id }) => id); + expect([JSON.stringify(node), rows.map(({ id }) => id)]).toEqual([JSON.stringify(node), expected]); + } + await db.close(); + for (const condition of WRAPPED) { + const { db, sql } = await table(); + const node = compileSet(T, { n: when(condition, 1, 2) }, "updateWhere"); + const statement = sql.updateWhere({ and: [] } as WhereNode, node, new Map()); + await db.run(statement.sql, ...statement.params); + const stored = await db.all<{ [column: string]: SqliteValue }>(`SELECT * FROM t ORDER BY id`); + const read = stored.map((row) => T.rows.decode(row, context)); + expect([JSON.stringify(node), read]).toEqual([ + JSON.stringify(node), + ROWS.map((row) => applySet(node, row, "t")), + ]); + await db.close(); + } + }); + + it("binds a long inArray list as one parameter (beyond SQLite's 32,766 variables)", async () => { + const { db } = await table(); + const values = Array.from({ length: 40_000 }, (_, i) => `v${i}`); + values.push("b"); + const node = compileWhere(T, inArray(refs.path, values), "where"); + const params: SqliteValue[] = []; + const text = whereSql(node, params); + expect(params).toHaveLength(1); + const rows = await db.all<{ id: string }>(`SELECT id FROM t WHERE ${text}`, ...params); + expect(rows.map(({ id }) => id)).toEqual(["4"]); + await db.close(); + }); + + it("updates rows as the evaluator does", async () => { + const SETS = [ + { n: add(refs.n, 10), count: sub(coalesce(refs.count, 5), 1) }, + { path: replacePrefix(refs.path, "a/", "\u{1F600}/"), note: concat(refs.path, "!") }, + { x: min(refs.x, 1), n: max(refs.n, 1), note: when(prefix(refs.path, "a"), refs.path, refs.note) }, + { note: refs.path, path: coalesce(refs.note, "none"), count: undefined }, + ]; + for (const set of SETS) { + const { db, sql } = await table(); + const node = compileSet(T, set, "updateWhere"); + const statement = sql.updateWhere({ and: [] } as WhereNode, node, new Map()); + await db.run(statement.sql, ...statement.params); + const stored = await db.all<{ [column: string]: SqliteValue }>(`SELECT * FROM t ORDER BY id`); + const read = stored.map((row) => T.rows.decode(row, context)); + expect(read).toEqual(ROWS.map((row) => applySet(node, row, "t"))); + await db.close(); + } + }); +}); diff --git a/packages/chord/test/next/store/export.test.ts b/packages/chord/test/next/store/export.test.ts new file mode 100644 index 000000000..e0e743023 --- /dev/null +++ b/packages/chord/test/next/store/export.test.ts @@ -0,0 +1,230 @@ +// Tests of `store.export(ctx)` (slice 6a §13, rule 26): the parts in archive order, lossless; the commit queue held +// from the first `next()` until the export ends, is closed or aborts (also while the consumer is suspended); GC of +// closed and aborted iterators and their abort listeners. + +import { getEventListeners } from "node:events"; +import { afterEach, describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { int, str } from "../../../src/next/contracts/index.ts"; +import { ARCHIVE_FORMAT } from "../../../src/next/store/archive.ts"; +import { EXPORT_PAGE } from "../../../src/next/store/export.ts"; +import { STORE_SCHEMA } from "../../../src/next/store/schema.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import { + archiveStore, + expectCollected, + exported, + flushed, + memoryStore, + rejection, + settledHeap, + tableIn, +} from "../support.ts"; + +const OPEN = { admit: () => true, adopt: () => {} }; + +let store: ChordStore; +afterEach(async () => { + await store?.close(context); +}); + +describe("parts", () => { + it("come in archive order: manifest, per table its table write, puts and indexes, the live documents, end", async () => { + store = await archiveStore(); + const parts = await exported(store); + expect(parts[0]).toEqual({ + kind: "manifest", + format: ARCHIVE_FORMAT, + schema: STORE_SCHEMA, + lastSeq: store.lastSeq, + nextDocumentId: store.nextDocumentId, + }); + const kinds = parts.map((part) => + part.kind === "put" || part.kind === "table" || part.kind === "index" + ? `${part.kind} ${part.table.split(":")[2]}` + : part.kind, + ); + const runs: [string, number][] = []; + for (const kind of kinds) { + const last = runs.at(-1); + if (last?.[0] === kind) last[1]++; + else runs.push([kind, 1]); + } + expect(runs).toEqual([ + ["manifest", 1], + ["table edits", 1], + ["put edits", 150], + ["index edits", 1], + ["table notes", 1], + ["put notes", 150], + ["document.create", 3], + ["end", 1], + ]); + expect(parts.at(-1)).toEqual({ kind: "end", writes: parts.length - 2 }); + }); + + it("are lossless: stored values and versions, NULL cells absent, a migrating table's state, deltas folded into the base", async () => { + store = await archiveStore(); + const parts = await exported(store); + const edits = parts.filter((part) => part.kind === "put" && part.table.endsWith(":edits")); + expect(edits.slice(0, 2)).toEqual([ + { + kind: "put", + table: "client:app.store:edits", + row: { id: "e000", path: "p/0", at: 0, note: "n0", meta: { a: 0 }, done: false }, + version: 1, + columns: { meta: 1 }, + }, + { + kind: "put", + table: "client:app.store:edits", + row: { id: "e001", path: "p/1", at: 1, done: false }, + version: 1, + columns: {}, + }, + ]); + expect(parts.find((part) => part.kind === "table" && part.table.endsWith(":notes"))).toMatchObject({ + version: 2, + state: "migrating", + }); + const notes = parts.filter((part) => part.kind === "put" && part.table.endsWith(":notes")); + expect(notes.map((part) => (part.kind === "put" ? part.version : 0)).filter((v) => v === 2)).toHaveLength(100); + const documents = parts.filter((part) => part.kind === "document.create"); + expect( + documents.map((part) => + part.kind === "document.create" ? [part.record.key, part.record.revision, part.base.value] : [], + ), + ).toEqual([ + // A singleton's record has no key. + [undefined, 1, { title: "" }], + ["a", 3, { title: "a3" }], + ["b", 2, { title: "b2" }], + ]); + }); +}); + +describe("the commit queue", () => { + it("is held from the first next() until the export ends: a commit waits", async () => { + store = await archiveStore(); + const iterator = store.export(context); + await iterator.next(); + let committed = false; + const commit = store.commit([], OPEN, context).then(() => { + committed = true; + }); + await flushed(); + expect(committed).toBe(false); + while (!(await iterator.next()).done) {} + await commit; + expect(committed).toBe(true); + }); + + it("is given back by return(); the closed iterator keeps nothing", async () => { + store = await archiveStore(); + let iterator: AsyncIterableIterator | undefined = store.export(context); + await iterator.next(); + await iterator.next(); + await iterator.return!(); + await store.commit([], OPEN, context); + const collected = new WeakRef(iterator); + iterator = undefined; + await expectCollected(collected); + }); + + it("is given back at once when ctx aborts while the consumer is suspended; a later next() rejects aborted", async () => { + store = await archiveStore(); + const { context: cancellable, cancel } = withCancel(context); + const iterator = store.export(cancellable); + await iterator.next(); + const signal = cancellable.abortSignal!; + expect(getEventListeners(signal, "abort")).toHaveLength(1); + cancel(new Error("stop")); + await store.commit([], OPEN, context); + expect(getEventListeners(signal, "abort")).toHaveLength(0); + expect(await rejection(iterator.next())).toMatchObject({ code: "aborted" }); + }); + + it("a retained aborted iterator keeps no page", async () => { + store = await memoryStore(); + const Big = defineTable("big", { version: 1, columns: { id: int(), text: str() }, key: ["id"] }); + const staged: StagedWrite[] = []; + const changes = new TableChanges(await tableIn(store, Big), staged); + for (let i = 0; i < EXPORT_PAGE; i++) changes.put({ id: i, text: `${i}`.padEnd(8_000, "x") }); + await store.commit(staged, OPEN, context); + const { context: cancellable, cancel } = withCancel(context); + const before = await settledHeap(); + const iterator = store.export(cancellable); + // The manifest, the table write, and the first put of a page of 100 rows of 8 KB each. + for (let i = 0; i < 3; i++) await iterator.next(); + cancel(new Error("stop")); + const after = await settledHeap(); + expect(after - before).toBeLessThan(200 * 1024); + expect(await rejection(iterator.next())).toMatchObject({ code: "aborted" }); + }); + + it("code that commits while consuming its own export waits until ctx aborts", async () => { + store = await archiveStore(); + const { context: cancellable, cancel } = withCancel(context); + const iterator = store.export(cancellable); + await iterator.next(); + const commit = store.commit([], OPEN, context); + let committed = false; + void commit.then(() => { + committed = true; + }); + await flushed(); + expect(committed).toBe(false); + cancel(new Error("stop")); + await commit; + expect(committed).toBe(true); + }); + + it("closing the store ends a held export: its next() rejects released, also for a part that needs no read", async () => { + store = await archiveStore(); + const iterator = store.export(context); + await iterator.next(); + await store.close(context); + // The table write would need no read. + expect(await rejection(iterator.next())).toMatchObject({ code: "released" }); + expect(await rejection(iterator.next())).toMatchObject({ code: "released" }); + }); + + it("an abort while next() waits for the queue rejects at once; no abort listener is left", async () => { + store = await archiveStore(); + const first = store.export(context); + await first.next(); + const { context: cancellable, cancel } = withCancel(context); + const second = store.export(cancellable); + const waiting = rejection(second.next()); + await flushed(); + cancel(new Error("stop")); + expect(await waiting).toMatchObject({ code: "aborted" }); + expect(getEventListeners(cancellable.abortSignal!, "abort")).toHaveLength(0); + // The first export still holds the queue; once it ends, the aborted one gives the queue back at once. + await first.return!(); + await store.commit([], OPEN, context); + expect(await rejection(second.next())).toMatchObject({ code: "aborted" }); + }); + + it("closing the store before an export's first next() rejects it released: not even a manifest", async () => { + store = await archiveStore(); + const iterator = store.export(context); + await store.close(context); + expect(await rejection(iterator.next())).toMatchObject({ code: "released" }); + expect(await rejection(iterator.next())).toMatchObject({ code: "released" }); + }); + + it("closing the store while an export waits for the queue rejects its next() released", async () => { + store = await archiveStore(); + const first = store.export(context); + await first.next(); + const second = store.export(context); + const waiting = rejection(second.next()); + await flushed(); + await store.close(context); + expect(await waiting).toMatchObject({ code: "released" }); + }); +}); diff --git a/packages/chord/test/next/store/expression.test.ts b/packages/chord/test/next/store/expression.test.ts new file mode 100644 index 000000000..c39c844ec --- /dev/null +++ b/packages/chord/test/next/store/expression.test.ts @@ -0,0 +1,152 @@ +// Tests of the table expression language: the builders and their compilation into the data vocabulary. +import { describe, expect, it } from "vitest"; +import { bool, int, nul, num, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { + add, + and, + coalesce, + compileSet, + compileWhere, + concat, + eq, + gt, + gte, + inArray, + isNull, + lt, + lte, + max, + ne, + not, + or, + prefix, + replacePrefix, + sub, + when, +} from "../../../src/next/store/expression.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import type { ColumnRefs } from "../../../src/next/store/query.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; + +const T = defineTable("t", { + version: 1, + columns: { + id: str(), + path: str(), + n: int(), + x: num(), + b: bool(), + note: opt(str()), + count: opt(int()), + meta: opt(json(obj({ a: int() }), { version: 1 })), + }, + key: ["id"], +}); +type C = typeof T extends { "~columns"?: infer X } ? NonNullable : never; +const refs = T.refs as ColumnRefs; +const args = expect.objectContaining({ code: "invalid", reason: "args" }); +describe("compiling conditions", () => { + it("compiles every operator into data", () => { + const where = and( + eq(refs.path, "a"), + ne(refs.n, 1), + lt(refs.n, 2), + lte(refs.x, 2.5), + gt(refs.n, 0), + gte(refs.path, "a"), + inArray(refs.n, [1, 2]), + isNull(refs.note), + prefix(refs.path, "a/"), + or(not(eq(refs.b, true))), + ); + expect(compileWhere(T, where, "where")).toEqual({ + and: [ + { eq: ["path", "a"] }, + { ne: ["n", 1] }, + { lt: ["n", 2] }, + { lte: ["x", 2.5] }, + { gt: ["n", 0] }, + { gte: ["path", "a"] }, + { inArray: ["n", [1, 2]] }, + { isNull: "note" }, + { prefix: ["path", "a/"] }, + { or: [{ not: { eq: ["b", true] } }] }, + ], + }); + }); + + it("refuses undefined and null operands (rule 16), wrong types, json and unknown columns", () => { + expect(() => compileWhere(T, eq(refs.note, undefined as never), "where")).toThrow("ask with isNull"); + expect(() => compileWhere(T, eq(refs.note, null as never), "where")).toThrow(args); + expect(() => compileWhere(T, eq(refs.n, "1" as never), "where")).toThrow(args); + expect(() => compileWhere(T, eq(refs.n, 1.5), "where")).toThrow(args); + expect(() => compileWhere(T, eq(refs.meta as never, 1), "where")).toThrow("not queryable"); + expect(() => compileWhere(T, eq({ column: "nope" } as never, 1), "where")).toThrow("expected a column reference"); + expect(() => compileWhere(T, "x", "where")).toThrow("expected a condition"); + expect(() => compileWhere(T, prefix(refs.n as never, "a"), "where")).toThrow("str() column"); + }); + + it("makes text operands storable", () => { + expect(compileWhere(T, eq(refs.path, "a\uD800"), "where")).toEqual({ eq: ["path", "a\uFFFD"] }); + expect(() => compileWhere(T, prefix(refs.path, "a\0"), "where")).toThrow("NUL"); + }); +}); + +describe("compiling sets", () => { + it("compiles plain values and expressions into one vocabulary, a clear as null (rule 14)", () => { + expect(compileSet(T, { path: "p", note: undefined, b: true, meta: { a: 1 } }, "update")).toEqual({ + path: "p", + note: null, + b: true, + meta: { json: { a: 1 } }, + }); + const set = compileSet( + T, + { + n: add(refs.n, 1), + count: sub(coalesce(refs.count, 0), refs.n), + x: max(refs.x, 1.5), + path: replacePrefix(concat(refs.path, "/"), "a/", "b/"), + note: when(gt(refs.n, 1), refs.path, refs.note), + }, + "update", + ); + expect(JSON.parse(JSON.stringify(set))).toEqual({ + n: { add: [{ col: "n" }, 1] }, + count: { sub: [{ coalesce: [{ col: "count" }, 0] }, { col: "n" }] }, + x: { max: [{ col: "x" }, 1.5] }, + path: { replacePrefix: [{ concat: [{ col: "path" }, "/"] }, "a/", "b/"] }, + note: { when: [{ gt: ["n", 1] }, { col: "path" }, { col: "note" }] }, + }); + }); + + it("refuses an empty set, key columns (rule 13), unknown columns and clears of non-opt columns", () => { + expect(() => compileSet(T, {}, "update")).toThrow("empty"); + expect(() => compileSet(T, { id: "x" }, "update")).toThrow("cannot change"); + expect(() => compileSet(T, { nope: 1 }, "update")).toThrow('no column "nope"'); + expect(() => compileSet(T, { path: undefined }, "update")).toThrow("cannot be cleared"); + }); + + it("undefined clears an opt column; null is a json(...) cell's JSON null and refused for scalar columns", () => { + expect(compileSet(T, { note: undefined, meta: undefined }, "update")).toEqual({ note: null, meta: null }); + const Nullable = defineTable("n", { + version: 1, + columns: { id: str(), data: json(nul(), { version: 1 }) }, + key: ["id"], + }); + expect(compileSet(Nullable, { data: null }, "update")).toEqual({ data: { json: null } }); + expect(() => compileSet(T, { note: null as never }, "update")).toThrow("null is no str value"); + }); + + it("refuses integer arithmetic outside int columns (rule 10), type mismatches and nullable results into non-opt columns", () => { + expect(() => compileSet(T, { x: add(refs.n, 1) }, "update")).toThrow("add gives int"); + expect(() => compileSet(T, { path: max(refs.n, 1) as never }, "update")).toThrow("gives int or num"); + expect(() => compileSet(T, { path: refs.n as never }, "update")).toThrow('column "n" is int, not str'); + expect(() => compileSet(T, { n: add(refs.count, 1) }, "update")).toThrow("may be NULL"); + expect(() => compileSet(T, { path: refs.note }, "update")).toThrow("may be NULL"); + expect(compileSet(T, { path: coalesce(refs.note, "d") }, "update")).toEqual({ + path: { coalesce: [{ col: "note" }, "d"] }, + }); + expect(() => compileSet(T, { meta: refs.meta as never }, "update")).toThrow("plain values"); + }); +}); diff --git a/packages/chord/test/next/store/import.test.ts b/packages/chord/test/next/store/import.test.ts new file mode 100644 index 000000000..d440a901e --- /dev/null +++ b/packages/chord/test/next/store/import.test.ts @@ -0,0 +1,436 @@ +// Tests of `importStore` (slice 6a §13, rules 12, 24, 25): round trips (old row versions, a migrating table, +// uncheckpointed deltas), the store it returns, every refusal of the archive's order and content, the rollback, and an +// input that stalls cancelled by `ctx`. + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import type { JsonValue } from "../../../src/json.ts"; +import { nul, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import type { ArchivePart } from "../../../src/next/store/archive.ts"; +import { sqliteBackend } from "../../../src/next/store/backend.ts"; +import { findDocument, loadDocument } from "../../../src/next/store/engine.ts"; +import { importStore } from "../../../src/next/store/import.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import { evolveTable } from "../../../src/next/store/table-evolve.ts"; +import { TableHandle } from "../../../src/next/store/table-handle.ts"; +import { prepareTables } from "../../../src/next/store/table-migration.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { ArchiveTables, archiveStore, exported, memoryStore, rejection, tableIn, tableOwner } from "../support.ts"; + +const stores: ChordStore[] = []; +afterEach(async () => { + for (const store of stores.splice(0)) await store.close(context); +}); + +/** `parts` imported into a fresh `:memory:` backend. */ +async function imported(parts: Iterable | AsyncIterable, outbox?: boolean): Promise { + const backend = sqliteBackend(await openNodeSqliteDatabase(":memory:")); + const store = (await importStore(backend, parts, context, { outbox })) as ChordStore; + stores.push(store); + return store; +} + +/** The archive of `archiveStore`. */ +async function archive(): Promise { + const source = await archiveStore(); + stores.push(source); + return exported(source); +} + +describe("round trips", () => { + it("an imported archive exports the same parts: lossless, at the manifest's lastSeq and next document id", async () => { + const parts = await archive(); + const store = await imported(parts); + expect(await exported(store)).toEqual(parts); + const manifest = parts[0] as Extract; + expect([store.lastSeq, store.nextDocumentId]).toEqual([manifest.lastSeq, manifest.nextDocumentId]); + }); + + it("the store keeps its ids, seqs and folded documents; the outbox, when asked, starts after lastSeq", async () => { + const parts = await archive(); + const store = await imported(parts, true); + expect(await store.outbox!.read(store.lastSeq, 10, context)).toEqual([]); + expect(await rejection(store.outbox!.read(0, 10, context))).toMatchObject({ code: "invalid" }); + const record = await store.read( + (executor) => + findDocument(executor, { namespace: "chord", owner: "client:app.store", kind: "memo", key: "a" }), + context, + ); + const original = parts.find((part) => part.kind === "document.create" && part.record.key === "a"); + expect(record).toEqual((original as Extract).record); + expect((await store.read((executor) => loadDocument(executor, record!), context)).value).toEqual({ title: "a3" }); + }); + + it("a migrating table resumes on the imported store; rows written at version 1 read as the table's", async () => { + const store = await imported(await archive()); + const { EditsV2, NotesV2 } = ArchiveTables; + const notes = "client:app.store:notes"; + const fence = { admit: () => true, adopt: () => {} }; + await expect( + prepareTables([{ definition: NotesV2, table: notes }], store, fence, "app", context), + ).rejects.toThrow("crash"); + const edits = new TableHandle(EditsV2, "client:app.store:edits", store, tableOwner()); + expect(await edits.get({ id: "e003" }, context)).toEqual({ + id: "e003", + path: "p/3", + at: 3, + meta: { a: 3 }, + done: false, + }); + }); +}); + +describe("json(...) cells holding null", () => { + it("a JSON null round-trips, required and optional, put and updated; an absent optional cell stays absent", async () => { + const source = await memoryStore(); + stores.push(source); + const Cells = defineTable("cells", { + version: 1, + columns: { id: str(), data: json(nul(), { version: 1 }), more: opt(json(nul(), { version: 1 })) }, + key: ["id"], + }); + const handle = await tableIn(source, Cells); + const staged: StagedWrite[] = []; + const changes = new TableChanges(handle, staged); + changes.put({ id: "a", data: null, more: null }); + changes.put({ id: "b", data: null }); + changes.put({ id: "c", data: null, more: null }); + changes.update({ id: "c" }, { more: undefined }); + await source.commit(staged, { admit: () => true, adopt: () => {} }, context); + const rows = [ + { id: "a", data: null, more: null }, + { id: "b", data: null }, + { id: "c", data: null }, + ]; + for (const row of rows) expect(await handle.get({ id: row.id }, context)).toEqual(row); + const parts = await exported(source); + const store = await imported(parts); + expect(await exported(store)).toEqual(parts); + const copy = new TableHandle(Cells, handle.sql.table, store, tableOwner()); + for (const row of rows) expect(await copy.get({ id: row.id }, context)).toEqual(row); + }); + + it("a JSON null at an older cell version round-trips with its version and migrates on access in the copy", async () => { + const source = await memoryStore(); + stores.push(source); + const V1 = defineTable("cells", { + version: 1, + columns: { id: str(), data: json(nul(), { version: 1 }) }, + key: ["id"], + }); + const old = await tableIn(source, V1); + const staged: StagedWrite[] = []; + new TableChanges(old, staged).put({ id: "a", data: null }); + await source.commit(staged, { admit: () => true, adopt: () => {} }, context); + const V2 = evolveTable(V1, (t) => + t.json("data", json(obj({ was: nul() }), { version: 2, migrate: (value) => ({ was: value as null }) })), + ); + const handle = await tableIn(source, V2); + expect(await handle.get({ id: "a" }, context)).toEqual({ id: "a", data: { was: null } }); + const parts = await exported(source); + expect(parts.find((part) => (part as ArchivePart).kind === "put")).toMatchObject({ + row: { id: "a", data: null }, + columns: { data: 1 }, + }); + const store = await imported(parts); + expect(await exported(store)).toEqual(parts); + const copy = new TableHandle(V2, handle.sql.table, store, tableOwner()); + expect(await copy.get({ id: "a" }, context)).toEqual({ id: "a", data: { was: null } }); + }); +}); + +/** The archive with `edit` applied to a copy of its parts. */ +async function edited(edit: (parts: ArchivePart[]) => unknown[]): Promise { + return edit(structuredClone(await archive())); +} + +/** The index of the first part of `kind` (and table suffix). */ +function first(parts: readonly ArchivePart[], kind: ArchivePart["kind"], table?: string): number { + return parts.findIndex( + (part) => part.kind === kind && (table === undefined || ("table" in part && part.table.endsWith(table))), + ); +} + +interface Mutable { + [field: string]: JsonValue | Mutable; +} + +/** Part `index` as a writable object. */ +function at(parts: unknown[], index: number): Mutable { + return parts[index] as Mutable; +} + +describe("refusals", () => { + it.each<[string, (parts: ArchivePart[]) => unknown[], string]>([ + [ + "a part outside the schema", + (parts) => [...parts.slice(0, 1), { kind: "put" }, ...parts.slice(1)], + "not an archive part", + ], + ["no manifest first", (parts) => parts.slice(1), "the manifest comes first"], + ["a second manifest", (parts) => [parts[0], ...parts], "the manifest comes first, once"], + [ + "another format", + (parts) => { + at(parts, 0).format = 2; + return parts; + }, + "format 2 is not 1", + ], + [ + "a newer store schema", + (parts) => { + at(parts, 0).schema = 9; + return parts; + }, + "schema 9 is newer", + ], + [ + "a lastSeq of 2^53 (a later seq would repeat)", + (parts) => { + at(parts, 0).lastSeq = 2 ** 53; + return parts; + }, + "part 0: not an archive part", + ], + [ + "a document revision beyond 2^53 - 1", + (parts) => { + const record = at(parts, first(parts, "document.create")).record as Mutable; + record.revision = 2 ** 53; + return parts; + }, + "not an archive part", + ], + ["a part after the end", (parts) => [...parts, { kind: "end", writes: 0 }], "after the end"], + ["no end", (parts) => parts.slice(0, -1), "has no end part"], + [ + "a wrong end count", + (parts) => { + at(parts, parts.length - 1).writes = 1; + return parts; + }, + "counts 1 writes", + ], + [ + "a put before its table", + (parts) => { + const put = first(parts, "put", ":notes"); + return [...parts.slice(0, first(parts, "table")), parts[put], ...parts.slice(first(parts, "table"))]; + }, + "a put follows its table's table write", + ], + [ + "a put after its table's indexes", + (parts) => { + const index = first(parts, "index"); + return [...parts.slice(0, index + 1), parts[index - 1], ...parts.slice(index + 1)]; + }, + "a put follows its table's table write, before its indexes", + ], + [ + "a table twice", + (parts) => { + const table = first(parts, "table", ":notes"); + return [...parts.slice(0, table), parts[first(parts, "table")], ...parts.slice(table)]; + }, + "occurs twice", + ], + [ + "a table after the documents", + (parts) => { + const table = first(parts, "table", ":notes"); + const documents = first(parts, "document.create"); + return [ + ...parts.slice(0, table), + ...parts.slice(documents, -1), + ...parts.slice(table, documents), + parts.at(-1), + ]; + }, + "tables come before the documents", + ], + [ + "a table name that is not kind:type:table", + (parts) => { + at(parts, first(parts, "table")).table = "edits"; + return parts; + }, + "not kind:type:table", + ], + [ + "a column name breaking the rules", + (parts) => { + const table = at(parts, first(parts, "table")); + table.columns = { ...(table.columns as Mutable), "bad name": { type: "str", optional: true } }; + return parts; + }, + "not a column name", + ], + [ + "a row with an unknown column", + (parts) => { + (at(parts, first(parts, "put")).row as Mutable).extra = 1; + return parts; + }, + 'unknown column "extra"', + ], + [ + "a row without a non-opt column", + (parts) => { + delete (at(parts, first(parts, "put")).row as Mutable).path; + return parts; + }, + 'column "path" is missing', + ], + [ + "a value of another type", + (parts) => { + (at(parts, first(parts, "put")).row as Mutable).at = "x"; + return parts; + }, + "is not a int value", + ], + [ + "a json(...) cell newer than its column", + (parts) => { + (at(parts, first(parts, "put")).columns as Mutable).meta = 5; + return parts; + }, + "no readable version", + ], + [ + "a row newer than its table", + (parts) => { + at(parts, first(parts, "put")).version = 7; + return parts; + }, + "version 7 is newer", + ], + [ + "an id not below the manifest's next id", + (parts) => { + (at(parts, first(parts, "document.create")).record as Mutable).id = 10_000; + return parts; + }, + "is not below", + ], + [ + "an id twice", + (parts) => { + const document = first(parts, "document.create"); + (at(parts, document + 1).record as Mutable).id = (at(parts, document).record as Mutable).id!; + return parts; + }, + "occurs twice", + ], + [ + "a record created after lastSeq", + (parts) => { + (at(parts, first(parts, "document.create")).record as Mutable).createdAt = 10_000; + return parts; + }, + "created after the manifest's lastSeq", + ], + [ + "a base at another version", + (parts) => { + (at(parts, first(parts, "document.create")).base as Mutable).version = 2; + return parts; + }, + "the base is not at the record's version", + ], + [ + "two live records at one address", + (parts) => { + const document = first(parts, "document.create"); + (at(parts, document + 1).record as Mutable).key = (at(parts, document).record as Mutable).key!; + return parts; + }, + "a second live record at one address", + ], + [ + "a record's meta with a NUL", + (parts) => { + (at(parts, first(parts, "document.create")).record as Mutable).meta = "a\u0000b"; + return parts; + }, + "text may not contain NUL", + ], + [ + "a record's meta with a lone surrogate", + (parts) => { + (at(parts, first(parts, "document.create")).record as Mutable).meta = "a\uD800"; + return parts; + }, + "the meta is not well-formed text", + ], + ])("%s", async (_, edit, message) => { + const parts = await edited(edit); + expect(await rejection(imported(parts))).toMatchObject({ + code: "invalid", + reason: "args", + message: expect.stringContaining(message), + }); + }); + + it("a non-empty backend is refused; a refused import closes its backend and leaves it as it was", async () => { + const directory = await mkdtemp(join(tmpdir(), "chord-import-")); + try { + const path = join(directory, "store.db"); + const parts = await archive(); + const broken = [...parts.slice(0, -1), { kind: "end", writes: 0 }]; + expect( + await rejection(importStore(sqliteBackend(await openNodeSqliteDatabase(path)), broken, context)), + ).toMatchObject({ + code: "invalid", + }); + const db = await openNodeSqliteDatabase(path); + expect(await db.all("SELECT name FROM sqlite_master")).toEqual([]); + await db.close(); + const store = await importStore(sqliteBackend(await openNodeSqliteDatabase(path)), parts, context); + await store.close(context); + expect( + await rejection(importStore(sqliteBackend(await openNodeSqliteDatabase(path)), parts, context)), + ).toMatchObject({ + message: "import: the backend holds a store already", + }); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + + it("an input that stalls is cancelled by ctx: the import rejects aborted and the input is closed", async () => { + const parts = await archive(); + let closed = false; + const stalled: AsyncIterable = { + [Symbol.asyncIterator]: () => { + let index = 0; + return { + next: () => + index < 3 + ? Promise.resolve({ done: false, value: parts[index++] }) + : new Promise>(() => {}), + return: async () => { + closed = true; + return { done: true, value: undefined }; + }, + }; + }, + }; + const { context: cancellable, cancel } = withCancel(context); + const backend = sqliteBackend(await openNodeSqliteDatabase(":memory:")); + const importing = importStore(backend, stalled, cancellable); + await new Promise((resolve) => setTimeout(resolve, 10)); + cancel(new Error("stop")); + expect(await rejection(importing)).toMatchObject({ code: "aborted" }); + expect(closed).toBe(true); + }); +}); diff --git a/packages/chord/test/next/store/json-column.test.ts b/packages/chord/test/next/store/json-column.test.ts new file mode 100644 index 000000000..339aaf1f3 --- /dev/null +++ b/packages/chord/test/next/store/json-column.test.ts @@ -0,0 +1,58 @@ +// `json(...)` columns (rule 8): their declaration, and a cell read and migrated on access. + +import { describe, expect, it } from "vitest"; +import { int, obj, str } from "../../../src/next/contracts/index.ts"; +import { JsonColumn, json } from "../../../src/next/store/json-column.ts"; +import { counted } from "../support.ts"; + +const MetaV1 = obj({ tag: str() }); +const MetaV2 = obj({ tags: str(), n: int() }); + +describe("json(...)", () => { + it("is a schema carrying its column: version and the fingerprint of the schema's normal form", () => { + const column = (json(MetaV1, { version: 1 }) as unknown as { json: JsonColumn }).json; + expect(column).toBeInstanceOf(JsonColumn); + expect(column.version).toBe(1); + const again = (json(obj({ tag: str({}) }), { version: 1 }) as unknown as { json: JsonColumn }).json; + const other = (json(MetaV2, { version: 1 }) as unknown as { json: JsonColumn }).json; + expect(again.fingerprint).toBe(column.fingerprint); + expect(other.fingerprint).not.toBe(column.fingerprint); + }); + + it("refuses a version that is not a positive integer", () => { + expect(() => json(MetaV1, { version: 0 })).toThrow("version 0"); + }); +}); + +describe("reading a cell", () => { + const v2 = new JsonColumn(MetaV2, 2, (old, from) => ({ tags: (old as { tag: string }).tag, n: from })); + + it("parses a cell of the current version with no check", async () => { + const counts = await counted((context) => { + expect(v2.read('{"tags":"a","n":1}', 2, "t.meta", context)).toEqual({ tags: "a", n: 1 }); + }); + expect(counts.validations).toBe(0); + }); + + it("migrates an older cell and checks the result once (R8)", async () => { + const counts = await counted((context) => { + expect(v2.read('{"tag":"x"}', 1, "t.meta", context)).toEqual({ tags: "x", n: 1 }); + }); + expect(counts.validations).toBe(1); + }); + + it("refuses a newer cell, a missing or throwing migrate and an invalid result", async () => { + await counted((context) => { + expect(() => v2.read("{}", 3, "t.meta", context)).toThrow("newer than 2"); + expect(() => new JsonColumn(MetaV2, 2, undefined).read("{}", 1, "t.meta", context)).toThrow("no migrate"); + const throwing = new JsonColumn(MetaV2, 2, () => { + throw new Error("no"); + }); + expect(() => throwing.read("{}", 1, "t.meta", context)).toThrow("threw"); + const invalid = new JsonColumn(MetaV2, 2, () => ({ tags: 1, n: 1 }) as never); + expect(() => invalid.read("{}", 1, "t.meta", context)).toThrow( + expect.objectContaining({ code: "unavailable" }), + ); + }); + }); +}); diff --git a/packages/chord/test/next/store/jsonl.test.ts b/packages/chord/test/next/store/jsonl.test.ts new file mode 100644 index 000000000..3d36e2321 --- /dev/null +++ b/packages/chord/test/next/store/jsonl.test.ts @@ -0,0 +1,56 @@ +// Tests of the JSONL archive encoding (slice 6a §13): parts as lines, chunks split anywhere (inside a line, inside a +// UTF-8 sequence), blank lines, and the refusal of a line that is no JSON. + +import { describe, expect, it } from "vitest"; +import type { ArchivePart } from "../../../src/next/store/archive.ts"; +import { decodeJsonl, encodeJsonl } from "../../../src/next/store/jsonl.ts"; +import { rejection } from "../support.ts"; + +async function* from(values: readonly T[]): AsyncIterable { + yield* values; +} + +async function collect(values: AsyncIterable): Promise { + const all: T[] = []; + for await (const value of values) all.push(value); + return all; +} + +const PARTS: ArchivePart[] = [ + { kind: "manifest", format: 1, schema: 1, lastSeq: 3, nextDocumentId: 2 }, + { kind: "put", table: "client:app.store:edits", row: { id: "ü€😀", n: 1 }, version: 1, columns: {} }, + { kind: "end", writes: 1 }, +]; + +describe("jsonl", () => { + it("encodes one part per line and decodes it back, from chunks split anywhere", async () => { + const bytes = Buffer.concat(await collect(encodeJsonl(from(PARTS)))); + expect(bytes.toString("utf8").split("\n")).toHaveLength(PARTS.length + 1); + for (const size of [1, 2, 3, 7, bytes.length]) { + const chunks: Uint8Array[] = []; + for (let at = 0; at < bytes.length; at += size) chunks.push(bytes.subarray(at, at + size)); + expect(await collect(decodeJsonl(from(chunks)))).toEqual(PARTS); + } + }); + + it("skips blank lines and takes a last line without a newline", async () => { + const text = `\n${JSON.stringify(PARTS[0])}\n\n${JSON.stringify(PARTS[2])}`; + expect(await collect(decodeJsonl(from([new TextEncoder().encode(text)])))).toEqual([PARTS[0], PARTS[2]]); + }); + + it("refuses a line that is no JSON, naming it", async () => { + const text = `${JSON.stringify(PARTS[0])}\n{nope\n`; + expect(await rejection(collect(decodeJsonl(from([new TextEncoder().encode(text)]))))).toMatchObject({ + code: "invalid", + reason: "args", + message: "jsonl: line 2 is not JSON", + }); + }); + + it("refuses bytes that are no UTF-8", async () => { + expect(await rejection(collect(decodeJsonl(from([new Uint8Array([0x7b, 0xff, 0x0a])]))))).toMatchObject({ + code: "invalid", + message: "jsonl: line 1 is not UTF-8", + }); + }); +}); diff --git a/packages/chord/test/next/store/outbox.test.ts b/packages/chord/test/next/store/outbox.test.ts new file mode 100644 index 000000000..2a720c080 --- /dev/null +++ b/packages/chord/test/next/store/outbox.test.ts @@ -0,0 +1,165 @@ +// Tests of the outbox (slice 6a §2; decisions "The outbox is optional, internal and versioned", small items 3-4): on, +// off and kept across opens, its commits in their recorded form, `trim` and its floor, and the format check. + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { sqliteBackend } from "../../../src/next/store/backend.ts"; +import { changeDocument, createDocument } from "../../../src/next/store/engine.ts"; +import { OUTBOX_FORMAT } from "../../../src/next/store/outbox.ts"; +import { TABLES } from "../../../src/next/store/schema.ts"; +import { type ChordStore, openStore } from "../../../src/next/store/store.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { memoryStore, rejection } from "../support.ts"; + +const ADDRESS = { namespace: "chord", owner: "session:app.store", kind: "notes", key: "" }; +const BASE = { version: 1, fingerprint: "0000000000000000", value: { n: 0 } }; +const OPEN = { admit: () => true, adopt: () => {} }; + +/** Creates the singleton at `ADDRESS` (with `key`) in one commit. */ +async function created(store: ChordStore, key = ""): Promise { + await store.commit([createDocument({ ...ADDRESS, key }, "latest", BASE, JSON.stringify(BASE.value))], OPEN, context); +} + +let directory: string | undefined; + +afterEach(async () => { + if (directory !== undefined) await rm(directory, { recursive: true, force: true }); + directory = undefined; +}); + +/** A store on a fresh file, reopened by `open`. */ +async function onFile() { + directory = await mkdtemp(join(tmpdir(), "chord-outbox-")); + const path = join(directory, "store.sqlite"); + return async (outbox?: boolean) => + (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context, { outbox })) as ChordStore; +} + +describe("the outbox", () => { + it("is off for a new store, on from the seq it is turned on at, kept when omitted, dropped when turned off", async () => { + const open = await onFile(); + let store = await open(); + await created(store); + await store.close(context); + store = await open(true); + expect(store.outbox?.floor).toBe(1); + const id = 1; + const ops = [["s", ["n"], 2]] as const; + await store.commit( + [changeDocument(id, 1, { kind: "delta", version: 1, ops }, JSON.stringify(ops))], + OPEN, + context, + ); + await store.close(context); + store = await open(); + expect(await store.outbox!.read(1, 10, context)).toEqual([ + { + seq: 2, + writes: [{ kind: "document.change", id, revision: 1, content: { kind: "delta", version: 1, ops } }], + }, + ]); + expect(await rejection(store.outbox!.read(0, 10, context))).toMatchObject({ code: "invalid", reason: "args" }); + expect(await rejection(store.outbox!.read(3, 10, context))).toMatchObject({ code: "invalid", reason: "args" }); + await store.close(context); + store = await open(false); + expect(store.outbox).toBeUndefined(); + await store.close(context); + }); + + it("records creations with their ids and seqs; trim is idempotent and refuses beyond lastSeq", async () => { + const store = await memoryStore({ outbox: true }); + await created(store); + const [commit] = await store.outbox!.read(0, 10, context); + expect(commit!.writes[0]).toMatchObject({ + kind: "document.create", + record: { id: 1, createdAt: 1, ...ADDRESS }, + base: BASE, + }); + await store.outbox!.trim(1, context); + await store.outbox!.trim(1, context); + expect(store.outbox!.floor).toBe(1); + expect(await store.outbox!.read(1, 10, context)).toEqual([]); + expect(await rejection(store.outbox!.trim(2, context))).toMatchObject({ code: "invalid", reason: "args" }); + }); + + it("overlapping trims run one at a time: the stored floor never moves back and equals the one in memory", async () => { + const open = await onFile(); + let store = await open(true); + for (let i = 0; i < 11; i++) await created(store, `k${i}`); + await Promise.all([store.outbox!.trim(9, context), store.outbox!.trim(3, context)]); + expect(store.outbox!.floor).toBe(9); + await store.close(context); + store = await open(); + expect(store.outbox!.floor).toBe(9); + expect(await rejection(store.outbox!.read(3, 10, context))).toMatchObject({ code: "invalid", reason: "args" }); + expect((await store.outbox!.read(9, 10, context)).map(({ seq }) => seq)).toEqual([10, 11]); + await store.close(context); + }); + + it("a read waits for a trim whose commit has not returned: the trimmed floor refuses it, never an empty page", async () => { + const db = await openNodeSqliteDatabase(":memory:"); + const transaction = db.transaction.bind(db); + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + const store = (await openStore(sqliteBackend(db), context, { outbox: true })) as ChordStore; + await created(store); + // The trim's DELETE committed; its transaction has not returned (the floor in memory is still 0). + db.transaction = async (fn) => { + const result = await transaction(fn); + entered.resolve(); + await gate.promise; + return result; + }; + const trim = store.outbox!.trim(1, context); + await entered.promise; + db.transaction = transaction; + const read = rejection(store.outbox!.read(0, 100, context)); + gate.resolve(); + await trim; + expect(await read).toMatchObject({ code: "invalid", reason: "args" }); + await store.close(context); + }); + + it("a trim waits for a read in flight: the read gets the commits it was admitted for", async () => { + const db = await openNodeSqliteDatabase(":memory:"); + const store = (await openStore(sqliteBackend(db), context, { outbox: true })) as ChordStore; + await created(store); + const all = db.all.bind(db); + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + db.all = (async (sql: string, ...params: never[]) => { + entered.resolve(); + await gate.promise; + return all(sql, ...params); + }) as typeof db.all; + const read = store.outbox!.read(0, 100, context); + await entered.promise; + db.all = all; + const trim = store.outbox!.trim(1, context); + gate.resolve(); + expect((await read).map(({ seq }) => seq)).toEqual([1]); + await trim; + expect(store.outbox!.floor).toBe(1); + await store.close(context); + }); + + it("a commit stored in a newer format is refused unavailable; the data stays", async () => { + const open = await onFile(); + let store = await open(true); + await created(store); + await store.close(context); + const db = await openNodeSqliteDatabase(join(directory!, "store.sqlite")); + await db.run(`UPDATE ${TABLES.outbox} SET format = ? WHERE seq = 1`, OUTBOX_FORMAT + 1); + await db.close(); + store = await open(); + expect(await rejection(store.outbox!.read(0, 10, context))).toMatchObject({ + code: "unavailable", + message: expect.stringContaining(`format ${OUTBOX_FORMAT + 1}`), + }); + expect(await rejection(store.outbox!.read(0, 10, context))).toMatchObject({ code: "unavailable" }); + await store.close(context); + }); +}); diff --git a/packages/chord/test/next/store/query.test.ts b/packages/chord/test/next/store/query.test.ts new file mode 100644 index 000000000..9dfb93473 --- /dev/null +++ b/packages/chord/test/next/store/query.test.ts @@ -0,0 +1,86 @@ +// Queries (slice 6a §7): `where`, `orderBy` and limits over a table, and cursors paging through them. + +import { describe, expect, it } from "vitest"; +import { int, opt, str } from "../../../src/next/contracts/index.ts"; +import { eq, gt } from "../../../src/next/store/expression.ts"; +import { decodeCursor, encodeCursor, query } from "../../../src/next/store/query.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; + +const T = defineTable("edits", { + version: 1, + columns: { id: str(), entry: int(), at: int(), note: opt(str()) }, + key: ["id", "entry"], +}); +const args = expect.objectContaining({ code: "invalid", reason: "args" }); + +describe("queries", () => { + it("are immutable; several wheres are and-ed", () => { + const base = query(T); + const one = base.where((e) => eq(e.id, "a")); + const two = one.where((e) => gt(e.at, 1)); + expect(base.condition).toBeUndefined(); + expect(one.condition).toEqual({ eq: ["id", "a"] }); + expect(two.condition).toEqual({ and: [{ eq: ["id", "a"] }, { gt: ["at", 1] }] }); + }); + + it("order by the key ascending by default and append the key as the tie-break, not repeated", () => { + expect(query(T).order).toEqual({ columns: ["id", "entry"], direction: "asc" }); + expect(query(T).orderBy((e) => e.at, "desc").order).toEqual({ + columns: ["at", "id", "entry"], + direction: "desc", + }); + expect(query(T).orderBy((e) => e.entry).order).toEqual({ columns: ["entry", "id"], direction: "asc" }); + }); + + it("refuse duplicate orderBy columns, two directions and opt columns (rule 17)", () => { + expect(() => + query(T) + .orderBy((e) => e.at) + .orderBy((e) => e.at), + ).toThrow("twice"); + expect(() => + query(T) + .orderBy((e) => e.at) + .orderBy((e) => e.id, "desc"), + ).toThrow("one direction"); + expect(() => query(T).orderBy((e) => e.note as never)).toThrow(args); + }); +}); + +describe("cursors", () => { + const order = query(T).orderBy((e) => e.at, "desc").order; + + it("round-trip the order values, also across conditions with the same order", () => { + const cursor = encodeCursor("client:app:edits", order, [5, "a\u{1F600}", 2]); + expect(cursor).toMatch(/^[A-Za-z0-9_-]+$/); + expect(decodeCursor(T, "client:app:edits", order, cursor)).toEqual([5, "a\u{1F600}", 2]); + const other = query(T) + .where((e) => eq(e.id, "x")) + .orderBy((e) => e.at, "desc").order; + expect(decodeCursor(T, "client:app:edits", other, cursor)).toEqual([5, "a\u{1F600}", 2]); + }); + + it("are a trust boundary: another table, order or direction, garbage and bad values are invalid/args", () => { + const cursor = encodeCursor("client:app:edits", order, [5, "a", 2]); + expect(() => decodeCursor(T, "client:app:edits", query(T).order, cursor)).toThrow(args); + expect(() => decodeCursor(T, "client:app:edits", query(T).orderBy((e) => e.at).order, cursor)).toThrow(args); + expect(() => + decodeCursor( + defineTable("other", { version: 1, columns: { id: str(), entry: int(), at: int() }, key: ["id", "entry"] }), + "client:app:other", + order, + cursor, + ), + ).toThrow(args); + // The same definition in another object's table. + expect(() => decodeCursor(T, "client:other:edits", order, cursor)).toThrow(args); + expect(() => decodeCursor(T, "client:app:edits", order, "%%%")).toThrow(args); + expect(() => decodeCursor(T, "client:app:edits", order, 42)).toThrow(args); + expect(() => + decodeCursor(T, "client:app:edits", order, encodeCursor("client:app:edits", order, ["x", "a", 2])), + ).toThrow(args); + expect(() => + decodeCursor(T, "client:app:edits", order, encodeCursor("client:app:edits", order, [5, "a"])), + ).toThrow(args); + }); +}); diff --git a/packages/chord/test/next/store/rows.test.ts b/packages/chord/test/next/store/rows.test.ts new file mode 100644 index 000000000..9f784ad90 --- /dev/null +++ b/packages/chord/test/next/store/rows.test.ts @@ -0,0 +1,77 @@ +// Rows (slice 6a §5, §9): `put` rows and keys checked and encoded, stored rows decoded. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { bool, int, num, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; + +const T = defineTable("t", { + version: 3, + columns: { + id: str(), + n: int({ default: 7 }), + x: num(), + b: bool(), + note: opt(str()), + meta: json(obj({ a: int() }), { version: 2 }), + }, + key: ["id"], +}); +const args = expect.objectContaining({ code: "invalid", reason: "args" }); + +describe("put rows", () => { + it("encodes a row in physical order, defaults filled, booleans as 0 and 1, json serialized with its version", () => { + const encoded = T.rows.put({ id: "a", x: 1.5, b: true, meta: { a: 1 } }, 3); + expect(encoded.params).toEqual(["a", 7, 1.5, 1, null, '{"a":1}', 2, 3]); + expect(encoded.row).toEqual({ id: "a", n: 7, x: 1.5, b: true, meta: { a: 1 } }); + expect(encoded.columns).toEqual({ meta: 2 }); + expect(encoded.id).toBe(T.rows.key({ id: "a" }, "get").id); + }); + + it("is a closed check: missing and unknown columns, wrong types, unsafe integers, non-finite numbers, NUL", () => { + const row = { id: "a", x: 1, b: false, meta: {} }; + expect(() => T.rows.put({ ...row, id: undefined }, 3)).toThrow(args); + expect(() => T.rows.put({ ...row, extra: 1 }, 3)).toThrow('unknown column "extra"'); + expect(() => T.rows.put({ ...row, n: 2 ** 53 }, 3)).toThrow(args); + expect(() => T.rows.put({ ...row, n: 1.5 }, 3)).toThrow(args); + expect(() => T.rows.put({ ...row, x: Number.POSITIVE_INFINITY }, 3)).toThrow(args); + expect(() => T.rows.put({ ...row, b: 1 }, 3)).toThrow(args); + expect(() => T.rows.put({ ...row, id: "a\0" }, 3)).toThrow("NUL"); + expect(() => T.rows.put({ ...row, note: null }, 3)).toThrow(args); + expect(() => T.rows.put([], 3)).toThrow("expected an object"); + }); + + it("serializes json cells at staging: a cycle or a BigInt fails there (rule 8)", () => { + const cycle: { self?: unknown } = {}; + cycle.self = cycle; + expect(() => T.rows.put({ id: "a", x: 1, b: false, meta: cycle }, 3)).toThrow(args); + expect(() => T.rows.put({ id: "a", x: 1, b: false, meta: { a: 1n } }, 3)).toThrow(args); + }); + + it("replaces lone surrogates in text", () => { + expect(T.rows.put({ id: "a\uD800", x: 1, b: false, meta: {} }, 3).row.id).toBe("a\uFFFD"); + }); +}); + +describe("keys", () => { + it("takes exactly the key columns", () => { + expect(T.rows.key({ id: "a" }, "get").params).toEqual(["a"]); + expect(() => T.rows.key({}, "get")).toThrow("is missing"); + expect(() => T.rows.key({ id: "a", n: 1 }, "get")).toThrow('unknown column "n"'); + expect(() => T.rows.key({ id: 1 }, "get")).toThrow(args); + }); +}); + +describe("decoding", () => { + it("reads booleans, leaves NULL opt columns absent and parses json cells", () => { + const row = T.rows.decode({ id: "a", n: 1, x: 2, b: 0, note: null, meta: '{"a":1}', meta$v: 2 }, context); + expect(row).toEqual({ id: "a", n: 1, x: 2, b: false, meta: { a: 1 } }); + }); + + it("refuses a json cell of a newer version", () => { + expect(() => T.rows.decode({ id: "a", n: 1, x: 2, b: 0, note: null, meta: "{}", meta$v: 3 }, context)).toThrow( + "newer than 2", + ); + }); +}); diff --git a/packages/chord/test/next/store/sql.test.ts b/packages/chord/test/next/store/sql.test.ts new file mode 100644 index 000000000..e8adaa0eb --- /dev/null +++ b/packages/chord/test/next/store/sql.test.ts @@ -0,0 +1,106 @@ +// SQL text (slice 6a §8): quoted names, the DDL of tables and indexes, and the statements of a table. + +import { describe, expect, it } from "vitest"; +import { bool, int, num, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import { + createIndexSql, + createTableSql, + dropIndexSql, + quote, + TableSql, + tableSqlName, +} from "../../../src/next/store/sql.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const T = defineTable("edits", { + version: 1, + columns: { + id: str(), + at: int({ default: 0 }), + x: num(), + b: bool(), + note: opt(str()), + meta: json(obj({}), { version: 2 }), + }, + key: ["id"], +}); + +describe("names", () => { + it("renders ::

, quoted", () => { + expect(tableSqlName("session", "@pi/file-history.store", "edits")).toBe("session:@pi/file-history.store:edits"); + expect(quote("session:a:b")).toBe('"session:a:b"'); + }); +}); + +describe("DDL", () => { + it("renders a STRICT WITHOUT ROWID table with checks, defaults, hidden version columns and the key", () => { + expect(createTableSql("s:o:edits", T.record, T.key)).toBe( + 'CREATE TABLE "s:o:edits" ("id" TEXT NOT NULL, "at" INTEGER NOT NULL DEFAULT 0 CHECK ("at" BETWEEN -9007199254740991 AND 9007199254740991), ' + + '"x" REAL NOT NULL, "b" INTEGER NOT NULL CHECK ("b" IN (0, 1)), "note" TEXT, "meta" TEXT NOT NULL, "meta$v" INTEGER NOT NULL, ' + + '"$v" INTEGER NOT NULL, PRIMARY KEY ("id")) STRICT, WITHOUT ROWID', + ); + expect(createIndexSql("s:o:edits", "by_at", ["at"], ["id"])).toBe( + 'CREATE INDEX "s:o:edits:by_at" ON "s:o:edits" ("at", "id")', + ); + expect(createIndexSql("s:o:edits", "by_id", ["id", "at"], ["id"])).toBe( + 'CREATE INDEX "s:o:edits:by_id" ON "s:o:edits" ("id", "at")', + ); + expect(dropIndexSql("s:o:edits", "by_at")).toBe('DROP INDEX "s:o:edits:by_at"'); + }); + + it("keeps int columns within ±(2^53 - 1) and bool columns within 0 and 1 (rule 10)", async () => { + const db = await openNodeSqliteDatabase(":memory:"); + await db.run(createTableSql("t", T.record, T.key)); + const sql = new TableSql("t", T.columns, T.rows.physical, T.key); + const params = T.rows.put({ id: "a", x: 1, b: true, meta: {} }, 1).params; + await db.run(sql.put, ...params); + await expect(db.run(`UPDATE t SET at = ?`, 2 ** 53)).rejects.toThrow("CHECK"); + await expect(db.run(`UPDATE t SET b = 2`)).rejects.toThrow("CHECK"); + await expect(db.run(`UPDATE t SET at = 'x'`)).rejects.toThrow(); + await db.run(`INSERT INTO t (id, x, b, meta, meta$v, "$v") VALUES ('d', 1, 0, '{}', 2, 1)`); + expect(await db.get(`SELECT at FROM t WHERE id = 'd'`)).toEqual({ at: 0 }); + await db.close(); + }); +}); + +describe("statements", () => { + const sql = new TableSql("t", T.columns, T.rows.physical, T.key); + + it("renders the fixed statements once", () => { + expect(sql.put).toBe( + 'INSERT OR REPLACE INTO "t" ("id", "at", "x", "b", "note", "meta", "meta$v", "$v") VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + ); + expect(sql.get).toBe('SELECT "id", "at", "x", "b", "note", "meta", "meta$v" FROM "t" WHERE "id" = ?'); + expect(sql.delete).toBe('DELETE FROM "t" WHERE "id" = ?'); + }); + + it("stamps a json column's version with its value, NULL with a clear", () => { + const versions = new Map([["meta", 2]]); + expect(sql.update({ meta: { json: { a: 1 } } }, ["k"], versions)).toEqual({ + sql: 'UPDATE "t" SET "meta" = ?, "meta$v" = ? WHERE "id" = ?', + params: ['{"a":1}', 2, "k"], + }); + expect(sql.update({ meta: null }, ["k"], versions).params).toEqual([null, "k"]); + }); + + it("binds every value: one text per query shape, whatever the values (rule 18)", () => { + const order = { columns: ["at", "id"], direction: "desc" as const }; + const a = sql.select({ inArray: ["id", ["a", "b"]] }, order, [1, "x"], 10); + const b = sql.select({ inArray: ["id", Array.from({ length: 40_000 }, (_, i) => `k${i}`)] }, order, [2, "y"], 99); + expect(a.sql).toBe(b.sql); + expect(a.sql).toBe( + 'SELECT "id", "at", "x", "b", "note", "meta", "meta$v" FROM "t" WHERE "id" IN (SELECT value FROM json_each(?)) AND ("at", "id") < (?, ?) ORDER BY "at" DESC, "id" DESC LIMIT ?', + ); + expect(a.params).toEqual(['["a","b"]', 1, "x", 10]); + expect(sql.count({ prefix: ["id", "\u{10FFFF}"] })).toEqual({ + sql: 'SELECT count(*) AS count FROM "t" WHERE "id" >= ?', + params: ["\u{10FFFF}"], + }); + expect(sql.deleteWhere({ prefix: ["id", "a"] })).toEqual({ + sql: 'DELETE FROM "t" WHERE ("id" >= ? AND "id" < ?)', + params: ["a", "b"], + }); + }); +}); diff --git a/packages/chord/test/next/store/store.test.ts b/packages/chord/test/next/store/store.test.ts new file mode 100644 index 000000000..03f1c0087 --- /dev/null +++ b/packages/chord/test/next/store/store.test.ts @@ -0,0 +1,250 @@ +// The store (slice 6a §2, §3): opening, the commit step, storage failures, one host per store (rule 7), close. + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { sqliteBackend } from "../../../src/next/store/backend.ts"; +import { changeDocument, createDocument, retireDocument } from "../../../src/next/store/engine.ts"; +import { importStore } from "../../../src/next/store/import.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import { type ChordStore, openStore, type Store } from "../../../src/next/store/store.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { exported, metered, rejection } from "../support.ts"; + +const ADDRESS = { namespace: "chord", owner: "session:app.store", kind: "notes", key: "" }; +const BASE = { version: 1, fingerprint: "0000000000000000", value: { n: 0 } }; + +/** A store on a fresh `:memory:` database, as Chord's layers see it. */ +async function memoryStore(options?: { readonly outbox?: boolean }): Promise { + return (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context, options)) as ChordStore; +} + +/** A fence that admits and records what it adopted. */ +function recording(adopted: [number, readonly number[]][] = []) { + return { + adopted, + fence: { admit: () => true, adopt: (seq: number, ids: readonly number[]) => void adopted.push([seq, ids]) }, + }; +} + +/** Creates the singleton at `ADDRESS` and resolves with its id. */ +async function created(store: ChordStore): Promise { + const { adopted, fence } = recording(); + await store.commit([createDocument(ADDRESS, "latest", BASE, JSON.stringify(BASE.value))], fence, context); + return adopted[0]![1][0]!; +} + +/** A staged write whose step fails in the backend. */ +const BROKEN: StagedWrite = { + order: "write", + apply: (step) => + step.run("INSERT INTO no_such_table VALUES (1)").then(() => ({ kind: "document.retire", id: 0, revision: 0 })), +}; + +let directory: string | undefined; + +afterEach(async () => { + if (directory !== undefined) await rm(directory, { recursive: true, force: true }); + directory = undefined; +}); + +describe("opening", () => { + it("creates a new store at seq 0 without an outbox, and reopens a file at its last seq and ids", async () => { + directory = await mkdtemp(join(tmpdir(), "chord-store-")); + const path = join(directory, "store.sqlite"); + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; + expect([store.lastSeq, store.outbox]).toEqual([0, undefined]); + expect(await created(store)).toBe(1); + await store.close(context); + const reopened = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; + expect(reopened.lastSeq).toBe(1); + const { adopted, fence } = recording(); + await reopened.commit([createDocument({ ...ADDRESS, key: "b" }, "latest", BASE, "{}")], fence, context); + expect(adopted).toEqual([[2, [2]]]); + await reopened.close(context); + }); + + it("refuses a database that is not UTF-8, closing the backend (rule 12)", async () => { + const db = await openNodeSqliteDatabase(":memory:"); + await db.exec('PRAGMA encoding = "UTF-16le"'); + await expect(openStore(sqliteBackend(db), context)).rejects.toThrow("not UTF-8"); + await expect(db.get("SELECT 1")).rejects.toThrow(); + }); +}); + +describe("the commit step", () => { + it("assigns one seq per commit, applies retirements before creations, and adopts before the next step", async () => { + const store = await memoryStore(); + const id = await created(store); + const events: string[] = []; + const fence = (name: string) => ({ + admit: () => true, + adopt: (seq: number, ids: readonly number[]) => void events.push(`${name} ${seq} ${ids}`), + }); + const first = store.commit( + [createDocument(ADDRESS, "latest", BASE, "{}"), retireDocument(id, 1)], + fence("recreate"), + context, + ); + const second = store.commit( + [changeDocument(2, 1, { kind: "delta", version: 1, ops: [["s", ["n"], 1]] }, '[["s",["n"],1]]')], + fence("change"), + context, + ); + expect(await Promise.all([first, second])).toEqual([2, 3]); + expect(events).toEqual(["recreate 2 2", "change 3 "]); + expect(store.lastSeq).toBe(3); + }); + + it("rejects a commit its fence refuses at the entry released, writing nothing; an empty commit takes no seq", async () => { + const store = await memoryStore(); + const refused = store.commit( + [createDocument(ADDRESS, "latest", BASE, "{}")], + { admit: () => false, adopt: () => {} }, + context, + ); + expect(await rejection(refused)).toMatchObject({ code: "released" }); + const { adopted, fence } = recording(); + expect(await store.commit([], fence, context)).toBe(0); + expect(adopted).toEqual([[0, []]]); + expect(await created(store)).toBe(1); + }); + + it("numbers up to Number.MAX_SAFE_INTEGER after an import near it, then fails storage_failed: no seq repeats", async () => { + const [manifest, ...rest] = await exported(await memoryStore()); + const parts = [{ ...manifest!, lastSeq: Number.MAX_SAFE_INTEGER - 1 }, ...rest]; + const store = (await importStore( + sqliteBackend(await openNodeSqliteDatabase(":memory:")), + parts, + context, + )) as ChordStore; + expect(await created(store)).toBe(1); + expect(store.lastSeq).toBe(Number.MAX_SAFE_INTEGER); + const refused = store.commit( + [createDocument({ ...ADDRESS, key: "b" }, "latest", BASE, "{}")], + recording().fence, + context, + ); + expect(await rejection(refused)).toMatchObject({ code: "unavailable", reason: "storage_failed" }); + expect(store.lastSeq).toBe(Number.MAX_SAFE_INTEGER); + expect(await rejection(store.read(async () => null, context))).toMatchObject({ reason: "storage_failed" }); + }); + + it("counts one commit and three statements for a delta, four with an outbox (cost guard)", async () => { + for (const [outbox, statements] of [ + [false, 3], + [true, 4], + ] as const) { + const store = await memoryStore({ outbox }); + const id = await created(store); + const telemetry = await metered(); + const ops = [["s", ["n"], 1]] as const; + await store.commit( + [changeDocument(id, 1, { kind: "delta", version: 1, ops }, JSON.stringify(ops))], + recording().fence, + telemetry.context, + ); + expect(telemetry.counts()).toEqual({ "chord.store.commits": 1, "chord.store.statements": statements }); + } + }); +}); + +describe("storage failures", () => { + it("stop the store: the commit rejects storage_failed, later commits and reads too; the host is told once", async () => { + const store = await memoryStore(); + const told: unknown[] = []; + store.attach((error) => told.push(error)); + expect(await rejection(store.commit([BROKEN], recording().fence, context))).toMatchObject({ + code: "unavailable", + reason: "storage_failed", + }); + expect(await rejection(store.commit([BROKEN], recording().fence, context))).toMatchObject({ + reason: "storage_failed", + }); + expect(await rejection(store.read(async () => 1, context))).toMatchObject({ reason: "storage_failed" }); + expect(told.length).toBe(1); + expect(store.lastSeq).toBe(0); + await store.close(context); + await store.close(context); + }); + + it("include an adopt that throws and a conditional update that finds another revision", async () => { + const store = await memoryStore(); + const id = await created(store); + const throwing = { + admit: () => true, + adopt: () => { + throw new Error("adopt threw"); + }, + }; + expect(await rejection(store.commit([retireDocument(id, 1)], throwing, context))).toMatchObject({ + reason: "storage_failed", + }); + // An empty commit's adopt too. + const empty = await memoryStore(); + expect(await rejection(empty.commit([], throwing, context))).toMatchObject({ reason: "storage_failed" }); + expect(await rejection(empty.read(async () => 1, context))).toMatchObject({ reason: "storage_failed" }); + const other = await memoryStore(); + const otherId = await created(other); + const stale = changeDocument(otherId, 7, { kind: "delta", version: 1, ops: [] }, "[]"); + expect(await rejection(other.commit([stale], recording().fence, context))).toMatchObject({ + reason: "storage_failed", + }); + expect(other.lastSeq).toBe(1); + }); + + it("spare a read whose context was aborted (aborted), and reject a read that settles after the stop", async () => { + const store = await memoryStore(); + const cancel = withCancel(context); + cancel.cancel(); + const failing = store.read(() => Promise.reject(new Error("backend")), cancel.context); + expect(await rejection(failing)).toMatchObject({ code: "aborted" }); + const gate = Promise.withResolvers(); + const late = store.read(() => gate.promise, context); + await rejection(store.commit([BROKEN], recording().fence, context)); + gate.resolve(1); + expect(await rejection(late)).toMatchObject({ reason: "storage_failed" }); + }); +}); + +describe("one host per store (rule 7)", () => { + it("refuses a second listener until the first detaches", async () => { + const store = await memoryStore(); + store.attach(() => {}); + expect(() => store.attach(() => {})).toThrow("already attached"); + store.detach(); + store.attach(() => {}); + }); +}); + +describe("close", () => { + it("waits for the running step, rejects queued commits and later reads released, and closes the backend", async () => { + const store = await memoryStore(); + const gate = Promise.withResolvers(); + const entered = Promise.withResolvers(); + const create = createDocument(ADDRESS, "latest", BASE, "{}"); + const held: StagedWrite = { + order: "create", + apply: async (step) => { + entered.resolve(); + await gate.promise; + return create.apply(step); + }, + }; + const running = store.commit([held], recording().fence, context); + await entered.promise; + const queued = store.commit( + [createDocument({ ...ADDRESS, key: "q" }, "latest", BASE, "{}")], + recording().fence, + context, + ); + const closed = (store as Store).close(context); + gate.resolve(); + expect(await running).toBe(1); + expect(await rejection(queued)).toMatchObject({ code: "released" }); + await closed; + expect(await rejection(store.read(async () => 1, context))).toMatchObject({ code: "released" }); + }); +}); diff --git a/packages/chord/test/next/store/table-changes.test.ts b/packages/chord/test/next/store/table-changes.test.ts new file mode 100644 index 000000000..2f51e3e37 --- /dev/null +++ b/packages/chord/test/next/store/table-changes.test.ts @@ -0,0 +1,84 @@ +// A transaction's table writes (slice 6a §6, §7): staging, and the `tx.get` overlay over committed rows. + +import { afterEach, describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, opt, str } from "../../../src/next/contracts/index.ts"; +import { add, eq, prefix } from "../../../src/next/store/expression.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import { memoryStore, rejection, tableIn } from "../support.ts"; + +const Edits = defineTable("edits", { + version: 1, + columns: { id: str(), path: str(), hits: int({ default: 0 }), note: opt(str()) }, + key: ["id"], +}); +const FENCE = { admit: () => true, adopt: () => {} }; + +let store: ChordStore; +afterEach(async () => { + await store?.close(context); +}); + +/** Changes to a table holding `e1` (committed); the staged list they append to. */ +async function seeded(): Promise<{ readonly changes: TableChanges; readonly staged: StagedWrite[] }> { + store = await memoryStore(); + const edits = await tableIn(store, Edits); + const seed: StagedWrite[] = []; + new TableChanges(edits, seed).put({ id: "e1", path: "a/1", hits: 1 }); + await store.commit(seed, FENCE, context); + const staged: StagedWrite[] = []; + return { changes: new TableChanges(edits, staged), staged }; +} + +describe("staging", () => { + it("checks at the call, appends in order, and touches no backend before the commit", async () => { + const { changes, staged } = await seeded(); + changes.put({ id: "e2", path: "a/2" }); + expect(() => changes.put({ id: "e3" })).toThrow('column "path" is missing'); + expect(() => changes.update({ id: "e1" }, { id: "x" })).toThrow("cannot change"); + changes.update({ id: "e1" }, { note: "n" }); + changes.delete({ id: "e9" }); + expect(staged).toHaveLength(3); + expect( + await store.read((executor) => executor.get(`SELECT count(*) AS n FROM "session:app.store:edits"`), context), + ).toEqual({ n: 1 }); + }); +}); + +describe("the tx.get overlay", () => { + it("replays the transaction's own put, update and delete over the committed row", async () => { + const { changes } = await seeded(); + changes.update({ id: "e1" }, (e: { hits: never }) => ({ hits: add(e.hits, 2) })); + expect(await changes.get({ id: "e1" }, context)).toEqual({ id: "e1", path: "a/1", hits: 3 }); + changes.update({ id: "e1" }, { note: "n" }); + expect(await changes.get({ id: "e1" }, context)).toEqual({ id: "e1", path: "a/1", hits: 3, note: "n" }); + changes.delete({ id: "e1" }); + expect(await changes.get({ id: "e1" }, context)).toBeUndefined(); + changes.put({ id: "e1", path: "b" }); + changes.update({ id: "e1" }, (e: { hits: never }) => ({ hits: add(e.hits, 1) })); + expect(await changes.get({ id: "e1" }, context)).toEqual({ id: "e1", path: "b", hits: 1 }); + }); + + it("an update of an absent key changes nothing", async () => { + const { changes } = await seeded(); + changes.update({ id: "e9" }, { note: "n" }); + expect(await changes.get({ id: "e9" }, context)).toBeUndefined(); + }); + + it("is refused on a table after an updateWhere or deleteWhere (W10); other tables are unaffected", async () => { + const { changes } = await seeded(); + expect(await changes.get({ id: "e1" }, context)).toBeDefined(); + changes.updateWhere((e: { path: never }) => prefix(e.path, "a/"), { note: "bulk" }); + expect(await rejection(changes.get({ id: "e1" }, context))).toMatchObject({ + code: "invalid", + reason: "read_after_bulk", + }); + const other = new TableChanges(changes.handle, []); + other.deleteWhere((e: { id: never }) => eq(e.id, "x")); + expect(await rejection(other.get({ id: "e1" }, context))).toMatchObject({ reason: "read_after_bulk" }); + expect(await new TableChanges(changes.handle, []).get({ id: "e1" }, context)).toBeDefined(); + }); +}); diff --git a/packages/chord/test/next/store/table-evolve.test.ts b/packages/chord/test/next/store/table-evolve.test.ts new file mode 100644 index 000000000..8118de4bd --- /dev/null +++ b/packages/chord/test/next/store/table-evolve.test.ts @@ -0,0 +1,225 @@ +// Tests of `evolveTable` (slice 6a §9): versions appended and materialized at declaration, the edits recorded before +// and after the data step, their checks, the editor rule shared with contracts, and the types along the chain. + +import { describe, expect, expectTypeOf, it } from "vitest"; +import { int, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import type { RowOf } from "../../../src/next/store/columns.ts"; +import { add, eq } from "../../../src/next/store/expression.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import { defineTable, type TableDefinition } from "../../../src/next/store/table.ts"; +import { evolveTable } from "../../../src/next/store/table-evolve.ts"; + +const EditsV1 = defineTable("edits", { + version: 1, + columns: { id: str(), file: str(), at: int(), meta: opt(json(obj({ a: int() }), { version: 1 })) }, + key: ["id"], + indexes: { by_file: ["file", "at"] }, +}); + +describe("versions", () => { + it("each evolveTable appends version + 1, materialized and fingerprinted, linked to the one before", () => { + const V2 = evolveTable(EditsV1, (t) => t.add("dir", str({ default: "" }))); + const V3 = evolveTable(V2, (t) => t.index("by_dir", ["dir"])); + expect([V2.version, V3.version]).toEqual([2, 3]); + expect(V3.evolution?.previous).toBe(V2); + expect(V2.evolution?.previous).toBe(EditsV1); + expect(new Set([EditsV1.fingerprint, V2.fingerprint, V3.fingerprint]).size).toBe(3); + expect(V3.indexes).toEqual( + new Map([ + ["by_file", ["file", "at"]], + ["by_dir", ["dir"]], + ]), + ); + expectTypeOf(V3.version).toEqualTypeOf<3>(); + }); + + it("the edit must return the editor it was given (contracts' rule)", () => { + expect(() => evolveTable(EditsV1, () => ({}) as never)).toThrow( + "edits: evolveTable must return the editor it was given", + ); + }); +}); + +describe("edits", () => { + it("are recorded before and after the data step; the step reads the columns as they stand there", () => { + const V2 = evolveTable(EditsV1, (t) => + t + .rename("file", "path") + .add("dir", str({ default: "" })) + .rows((row) => ({ dir: row.path })) + .drop("at") + .index("by_dir", ["dir"]), + ); + const migration = V2.evolution!.migration; + expect(migration.before).toEqual([ + { kind: "rename", from: "file", to: "path" }, + { kind: "add", name: "dir", column: { type: "str", default: "" } }, + ]); + expect(migration.after).toEqual([ + { kind: "dropIndex", name: "by_file" }, + { kind: "drop", name: "at" }, + { kind: "index", name: "by_dir", columns: ["dir"] }, + ]); + const step = migration.step!; + expect(step.kind).toBe("rows"); + expect(step.table.columns.map(({ name }) => name)).toEqual(["id", "path", "at", "meta", "dir"]); + expect(V2.columns.map(({ name }) => name)).toEqual(["id", "path", "meta", "dir"]); + expect(V2.indexes).toEqual(new Map([["by_dir", ["dir"]]])); + }); + + it("a rename carries the key and the indexes", () => { + const V2 = evolveTable(EditsV1, (t) => t.rename("id", "uid").rename("file", "path")); + expect(V2.key).toEqual(["uid"]); + expect(V2.indexes).toEqual(new Map([["by_file", ["path", "at"]]])); + expectTypeOf ? C : never>>().toEqualTypeOf< + { uid: string; path: string; at: number } & { meta?: { a: number } } + >(); + }); + + it("an update step is compiled at declaration against the columns there", () => { + const V2 = evolveTable(EditsV1, (t) => + t.update({ where: (e) => eq(e.file, "x"), set: (e) => ({ at: add(e.at, 1) }) }), + ); + expect(V2.evolution!.migration.step).toMatchObject({ + kind: "update", + where: { eq: ["file", "x"] }, + set: { at: { add: [{ col: "at" }, 1] } }, + }); + expect(() => + evolveTable(EditsV1, (t) => + // @ts-expect-error: key columns cannot change (rule 13) + t.update({ set: { id: "x" } }), + ), + ).toThrow('key column "id" cannot change'); + }); + + it("a json edit changes the column's version and keeps its nullability", () => { + const V2 = evolveTable(EditsV1, (t) => + t.json( + "meta", + json(obj({ a: int(), b: int() }), { version: 2, migrate: (old) => ({ ...(old as { a: number }), b: 0 }) }), + ), + ); + expect(V2.byName.get("meta")).toMatchObject({ type: "json", optional: true }); + expect(V2.evolution!.migration.before).toEqual([ + { kind: "json", name: "meta", column: expect.objectContaining({ type: "json", optional: true, version: 2 }) }, + ]); + expectTypeOf ? C : never>>().toEqualTypeOf< + { id: string; file: string; at: number } & { meta?: { a: number; b: number } } + >(); + }); +}); + +describe("checks", () => { + it("a rename may change only the case of its own column", () => { + expect(evolveTable(EditsV1, (t) => t.rename("at", "AT")).byName.has("AT")).toBe(true); + }); + + it.each<[string, () => unknown, string]>([ + ["rename to an existing column", () => evolveTable(EditsV1, (t) => t.rename("file", "at")), 'column "at" exists'], + [ + "rename an unknown column", + () => evolveTable(EditsV1, (t) => t.rename("nope" as "file", "x")), + 'no column "nope"', + ], + [ + "add a column without opt(...) or a default", + // @ts-expect-error: an added column is opt(...) or has a default + () => evolveTable(EditsV1, (t) => t.add("dir", str())), + "needs opt(...) or a default", + ], + [ + "drop a key column", + // @ts-expect-error: a key column cannot be dropped + () => evolveTable(EditsV1, (t) => t.drop("id")), + '"id" is a key column', + ], + [ + "rename after a drop", + // @ts-expect-error: rename comes before drop, index and dropIndex + () => evolveTable(EditsV1, (t) => t.drop("at").rename("file", "path")), + "rename comes before the data step", + ], + [ + "a second data step", + // @ts-expect-error: at most one data step + () => evolveTable(EditsV1, (t) => t.rows(() => ({})).rows(() => ({}))), + "rows comes before the data step", + ], + ["an existing index", () => evolveTable(EditsV1, (t) => t.index("by_file", ["at"])), 'index "by_file" exists'], + ["dropIndex of an unknown index", () => evolveTable(EditsV1, (t) => t.dropIndex("nope")), 'no index "nope"'], + [ + "an index over an opt(...) column", + // @ts-expect-error: index columns are non-opt str() or int() columns + () => evolveTable(EditsV1, (t) => t.index("by_meta", ["meta"])), + "is not a non-opt str() or int() column", + ], + [ + "a json edit of a scalar column", + // @ts-expect-error: only json(...) columns take a json edit + () => evolveTable(EditsV1, (t) => t.json("file", json(obj({}), { version: 2 }))), + "is not a json(...) column", + ], + [ + "a json edit changing the schema at the cell's version (D9)", + () => evolveTable(EditsV1, (t) => t.json("meta", json(obj({ b: str() }), { version: 1 }))), + "changes its schema at version 1", + ], + [ + "a json edit going back a version", + () => + evolveTable( + evolveTable(EditsV1, (t) => t.json("meta", json(obj({ a: int() }), { version: 2 }))), + (t) => t.json("meta", json(obj({ a: int() }), { version: 1 })), + ), + "goes back from version 2", + ], + [ + "an add colliding by case with a column a later drop removes", + () => evolveTable(EditsV1, (t) => t.add("File", str({ default: "" })).drop("file")), + 'column "file" exists', + ], + [ + "a rename colliding by case with another column", + () => evolveTable(EditsV1, (t) => t.rename("at", "FILE")), + 'column "file" exists', + ], + [ + "a name folding to a column's only beyond ASCII (SQLite folds ASCII): the name rule, not a collision", + () => { + const Kinds = defineTable("kinds", { version: 1, columns: { id: str(), kind: str() }, key: ["id"] }); + return evolveTable(Kinds, (t) => t.add("\u212Aind", opt(str()))); + }, + "expected [A-Za-z_][A-Za-z0-9_]*", + ], + [ + "an index on a column missing at that edit, though a later dropIndex removes the index", + // @ts-expect-error: index columns are the table's columns at that edit + () => evolveTable(EditsV1, (t) => t.index("bad", ["missing"]).dropIndex("bad")), + 'index "bad": no column "missing"', + ], + [ + "an index on a column dropped before it", + // @ts-expect-error: the dropped column is gone at this edit + () => evolveTable(EditsV1, (t) => t.drop("at").index("by_at", ["at"])), + 'index "by_at": no column "at"', + ], + [ + "a rename to a name breaking the column name rule, though a later rename fixes it", + () => evolveTable(EditsV1, (t) => t.rename("at", "bad\u0000name").rename("bad\u0000name", "when")), + 'rename: column "bad\u0000name": expected [A-Za-z_][A-Za-z0-9_]*', + ], + [ + "a rename to a name breaking the column name rule, though a later drop removes it", + () => evolveTable(EditsV1, (t) => t.rename("at", "1st").drop("1st")), + 'rename: column "1st": expected', + ], + [ + "an index name outside [a-z0-9_]+", + () => evolveTable(EditsV1, (t) => t.index("By", ["file"])), + "expected [a-z0-9_]+", + ], + ])("refuses %s at declaration", (_, declare, message) => { + expect(declare).toThrow(message); + }); +}); diff --git a/packages/chord/test/next/store/table-handle.test.ts b/packages/chord/test/next/store/table-handle.test.ts new file mode 100644 index 000000000..7a51ef960 --- /dev/null +++ b/packages/chord/test/next/store/table-handle.test.ts @@ -0,0 +1,136 @@ +// Table handles (slice 6a §7): reads outside a transaction, and cells the code cannot read. + +import { afterEach, describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { bool, int, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { eq, gte, isNull, prefix } from "../../../src/next/store/expression.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import { query } from "../../../src/next/store/query.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import type { TableHandle } from "../../../src/next/store/table-handle.ts"; +import { memoryStore, rejection, tableIn, tableOwner } from "../support.ts"; + +const Edits = defineTable("edits", { + version: 1, + columns: { + id: str(), + path: str(), + at: int(), + done: bool({ default: false }), + note: opt(str()), + meta: json(obj({ a: int() }), { version: 1 }), + }, + key: ["id"], + indexes: { by_path: ["path", "at"] }, +}); + +let store: ChordStore; +afterEach(async () => { + await store?.close(context); +}); + +/** A table with `count` rows e0..e(count-1): path `p/`, at `i`. */ +async function filled(count: number, owner = tableOwner()): Promise { + store = await memoryStore(); + const edits = await tableIn(store, Edits, owner); + const staged: StagedWrite[] = []; + const changes = new TableChanges(edits, staged); + for (let i = 0; i < count; i++) + changes.put({ id: `e${String(i).padStart(3, "0")}`, path: `p/${i % 3}`, at: i, meta: { a: i } }); + await store.commit(staged, { admit: () => true, adopt: () => {} }, context); + return edits; +} + +describe("reads outside a transaction", () => { + it("get takes a key object and reads committed rows as stored", async () => { + const edits = await filled(3); + expect(await edits.get({ id: "e001" }, context)).toEqual({ + id: "e001", + path: "p/1", + at: 1, + done: false, + meta: { a: 1 }, + }); + expect(await edits.get({ id: "nope" }, context)).toBeUndefined(); + expect(await rejection(edits.get({ id: "e001", path: "x" }, context))).toMatchObject({ + code: "invalid", + reason: "args", + }); + }); + + it("page fetches limit + 1 for next, and the cursor continues in order with the key tie-break", async () => { + const edits = await filled(10); + const q = query(Edits) + .where((e) => prefix(e.path, "p/")) + .orderBy((e) => e.path, "desc"); + const seen: string[] = []; + let after: string | undefined; + do { + const page = await edits.page(q, 3, context, after === undefined ? {} : { after }); + expect(page.items.length).toBeLessThanOrEqual(3); + seen.push(...page.items.map((row) => `${row.path}:${row.id}`)); + after = page.next; + } while (after !== undefined); + const all = (await edits.page(q, 100, context)).items.map((row) => `${row.path}:${row.id}`); + expect(seen).toEqual(all); + expect(all.slice(0, 4)).toEqual(["p/2:e008", "p/2:e005", "p/2:e002", "p/1:e007"]); + expect((await edits.page(q, 4, context)).next).toBeDefined(); + expect((await edits.page(query(Edits), 10, context)).next).toBeUndefined(); + }); + + it("rows reads every page; count counts", async () => { + const edits = await filled(250); + const ids: string[] = []; + for await (const row of edits.rows( + query(Edits).where((e) => gte(e.at, 10)), + context, + )) + ids.push(row.id as string); + expect(ids).toHaveLength(240); + expect(ids[0]).toBe("e010"); + expect( + await edits.count( + query(Edits).where((e) => eq(e.path, "p/0")), + context, + ), + ).toBe(84); + expect( + await edits.count( + query(Edits).where((e) => isNull(e.note)), + context, + ), + ).toBe(250); + }); + + it("refuses a query of another table, a bad limit and a foreign cursor before the backend", async () => { + const edits = await filled(1); + const Other = defineTable("other", { version: 1, columns: { id: str() }, key: ["id"] }); + expect(await rejection(edits.page(query(Other) as never, 1, context))).toMatchObject({ reason: "args" }); + expect(await rejection(edits.page(query(Edits), 0, context))).toMatchObject({ reason: "args" }); + expect(await rejection(edits.page(query(Edits), 1.5, context))).toMatchObject({ reason: "args" }); + expect(await rejection(edits.page(query(Edits), 1, context, { after: "x" }))).toMatchObject({ reason: "args" }); + }); + + it("reads go through the store's guard: refused once the store closed", async () => { + const edits = await filled(1); + await store.close(context); + expect(await rejection(edits.get({ id: "e000" }, context))).toMatchObject({ code: "released" }); + }); +}); + +describe("unreadable cells", () => { + it("a json cell of a newer version fails the owner and the read, the data intact", async () => { + const owner = tableOwner(); + const edits = await filled(1, owner); + await store.read((executor) => executor.run(`UPDATE "session:app.store:edits" SET "meta$v" = 2`), context); + const error = await rejection(edits.get({ id: "e000" }, context)); + expect(error).toMatchObject({ code: "unavailable" }); + expect(owner.failures).toHaveLength(1); + expect( + await store.read((executor) => executor.get(`SELECT meta FROM "session:app.store:edits"`), context), + ).toEqual({ meta: '{"a":0}' }); + }); +}); diff --git a/packages/chord/test/next/store/table-history.test.ts b/packages/chord/test/next/store/table-history.test.ts new file mode 100644 index 000000000..957e679f9 --- /dev/null +++ b/packages/chord/test/next/store/table-history.test.ts @@ -0,0 +1,73 @@ +// Tests of a table's history (slice 6a §9): what an activation does with the recorded version: create, use, migrate +// from there (resuming a `migrating` one), or refuse a version the code cannot reach. + +import { describe, expect, it } from "vitest"; +import { int, obj, str } from "../../../src/next/contracts/index.ts"; +import type { TableRecord } from "../../../src/next/store/catalog.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import { defineTable, type TableDefinition } from "../../../src/next/store/table.ts"; +import { evolveTable } from "../../../src/next/store/table-evolve.ts"; +import { planTable } from "../../../src/next/store/table-history.ts"; + +const V1 = defineTable("edits", { version: 1, columns: { id: str(), at: int() }, key: ["id"] }); +const V2 = evolveTable(V1, (t) => t.add("dir", str({ default: "" })).rows(() => ({}))); +const V3 = evolveTable(V2, (t) => t.index("by_at", ["at"])); + +/** The record of `definition` in `state`. */ +function recorded(definition: TableDefinition, state: "ready" | "migrating" = "ready"): TableRecord { + const { version, record: columns, key, fingerprint } = definition; + return { table: "t", version, state, columns, key, fingerprint, indexes: {} }; +} + +describe("plans", () => { + it("create a table the store does not have; use one recorded at the newest version", () => { + expect(planTable(V3, undefined, "app.store")).toEqual({ kind: "create" }); + expect(planTable(V3, recorded(V3), "app.store")).toEqual({ kind: "ready" }); + }); + + it("migrate through every later version, oldest first; resume a version recorded migrating", () => { + expect(planTable(V3, recorded(V1), "app.store")).toEqual({ kind: "migrate", versions: [V2, V3], first: "begin" }); + expect(planTable(V3, recorded(V2, "migrating"), "app.store")).toEqual({ + kind: "migrate", + versions: [V2, V3], + first: "resume", + }); + }); +}); + +describe("refusals", () => { + it.each<[string, TableDefinition, TableRecord, string]>([ + ["a newer version (old code)", V2, recorded(V3), "version 3, newer than the code's 2"], + [ + "a version older than a new base", + defineTable("edits", { version: 7, columns: { id: str() }, key: ["id"] }), + recorded(V3), + "older than the history's base 7", + ], + [ + "a history edited in place", + evolveTable(V1, (t) => t.add("dir", str({ default: "x" }))), + recorded(V2), + "recorded with another definition", + ], + [ + "a json(...) column with another schema at the same version (D9)", + defineTable("notes", { + version: 1, + columns: { id: str(), meta: json(obj({ b: int() }), { version: 1 }) }, + key: ["id"], + }), + recorded( + defineTable("notes", { + version: 1, + columns: { id: str(), meta: json(obj({ a: int() }), { version: 1 }) }, + key: ["id"], + }), + ), + "recorded with another definition", + ], + ["a migrating version without a data step", V3, recorded(V3, "migrating"), "has no data step to resume"], + ])("%s", (_, definition, record, message) => { + expect(() => planTable(definition, record, "app.store")).toThrow(message); + }); +}); diff --git a/packages/chord/test/next/store/table-migration.test.ts b/packages/chord/test/next/store/table-migration.test.ts new file mode 100644 index 000000000..c55ae4b96 --- /dev/null +++ b/packages/chord/test/next/store/table-migration.test.ts @@ -0,0 +1,329 @@ +// Tests of tables brought to their version at activation (slice 6a §9): fresh tables created at the newest version, +// eager migrations one version at a time in their commits ((A), chunks of 100, (Z)), `rows` and `update` steps, +// index edits, resumption at `migrating`, and the refusals that leave the data as it was. + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, nul, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { lt } from "../../../src/next/store/expression.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import { query } from "../../../src/next/store/query.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { defineTable, type TableDefinition } from "../../../src/next/store/table.ts"; +import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import { evolveTable } from "../../../src/next/store/table-evolve.ts"; +import { TableHandle } from "../../../src/next/store/table-handle.ts"; +import { MIGRATION_CHUNK, prepareTables } from "../../../src/next/store/table-migration.ts"; +import { memoryStore, rejection, tableOwner } from "../support.ts"; + +const TABLE = "client:app.store:edits"; +const OPEN = { admit: () => true, adopt: () => {} }; + +const EditsV1 = defineTable("edits", { + version: 1, + columns: { id: str(), file: str(), at: int() }, + key: ["id"], + indexes: { by_file: ["file"] }, +}); +const EditsV2 = evolveTable(EditsV1, (t) => + t + .rename("file", "path") + .add("dir", str({ default: "" })) + .rows((row) => ({ dir: row.path.slice(0, row.path.lastIndexOf("/")) })), +); +const EditsV3 = evolveTable(EditsV2, (t) => t.drop("dir").index("by_path", ["path", "at"])); + +let store: ChordStore; +afterEach(async () => { + await store?.close(context); +}); + +/** Prepares `definition` as the edits table of `store`; a handle to it. */ +async function prepared(definition: TableDefinition): Promise { + await prepareTables([{ definition, table: TABLE }], store, OPEN, "app.store", context); + return new TableHandle(definition, TABLE, store, tableOwner()); +} + +/** A store whose edits table is at version 1 with `count` rows e000.. (file `d/f`, at `i`). */ +async function filled(count: number): Promise { + store = await memoryStore({ outbox: true }); + const edits = await prepared(EditsV1); + const staged: StagedWrite[] = []; + const changes = new TableChanges(edits, staged); + for (let i = 0; i < count; i++) changes.put({ id: id(i), file: `d${i % 3}/f${i}`, at: i }); + await store.commit(staged, OPEN, context); +} + +function id(i: number): string { + return `e${String(i).padStart(3, "0")}`; +} + +/** The kinds of the writes of every commit after `seq`, each kind with its count. */ +async function commitsAfter(seq: number): Promise { + const commits = await store.outbox!.read(seq, 100, context); + return commits.map(({ writes }) => { + const counts = new Map(); + for (const { kind } of writes) counts.set(kind, (counts.get(kind) ?? 0) + 1); + return [...counts].map(([kind, count]) => (count === 1 ? kind : `${kind} x${count}`)); + }); +} + +/** Every row of `handle`, in key order. */ +async function all(handle: TableHandle, definition: TableDefinition): Promise { + const rows: unknown[] = []; + for await (const row of handle.rows(query(definition), context)) rows.push(row); + return rows; +} + +describe("fresh tables", () => { + it("are created at the newest version, ready, with their indexes, in one commit (no replay)", async () => { + store = await memoryStore({ outbox: true }); + await prepared(EditsV3); + expect(store.table(TABLE)).toMatchObject({ + version: 3, + state: "ready", + fingerprint: EditsV3.fingerprint, + indexes: { by_file: ["path"], by_path: ["path", "at"] }, + }); + expect(await commitsAfter(0)).toEqual([["table", "index x2"]]); + }); + + it("a table at the newest version, ready, is used as recorded: nothing is written", async () => { + await filled(1); + const seq = store.lastSeq; + await prepared(EditsV1); + expect(store.lastSeq).toBe(seq); + }); +}); + +describe("eager migrations", () => { + it("one version at a time: (A), chunks of 100 in key order, (Z) with the later edits and ready", async () => { + await filled(250); + const seq = store.lastSeq; + const edits = await prepared(EditsV3); + expect(await commitsAfter(seq)).toEqual([ + ["alter x2", "table"], + ["update x100"], + ["update x100"], + ["update x50", "table"], + ["alter", "index", "table"], + ]); + expect(store.table(TABLE)).toMatchObject({ + version: 3, + state: "ready", + columns: { id: { type: "str" }, path: { type: "str" }, at: { type: "int" } }, + indexes: { by_file: ["path"], by_path: ["path", "at"] }, + }); + const rows = await all(edits, EditsV3); + expect(rows).toHaveLength(250); + expect(rows[4]).toEqual({ id: "e004", path: "d1/f4", at: 4 }); + }); + + it("the data step reads the columns as they stand at its version and fills the one it adds", async () => { + await filled(3); + const edits = await prepared(EditsV2); + expect(await all(edits, EditsV2)).toEqual([ + { id: "e000", path: "d0/f0", at: 0, dir: "d0" }, + { id: "e001", path: "d1/f1", at: 1, dir: "d1" }, + { id: "e002", path: "d2/f2", at: 2, dir: "d2" }, + ]); + }); + + it("the migration's rows carry the internal form: the patch and the version stamped as $v", async () => { + await filled(1); + const seq = store.lastSeq; + await prepared(EditsV2); + const [, chunk] = await store.outbox!.read(seq, 10, context); + expect(chunk!.writes).toEqual([ + { kind: "update", table: TABLE, key: { id: "e000" }, set: { dir: "d0" }, columns: {}, count: 1, version: 2 }, + expect.objectContaining({ kind: "table", version: 2, state: "ready" }), + ]); + }); + + it("an empty table still runs (A) and (Z); a version without a data step is one commit", async () => { + await filled(0); + const seq = store.lastSeq; + await prepared(EditsV3); + expect(await commitsAfter(seq)).toEqual([["alter x2", "table"], ["table"], ["alter", "index", "table"]]); + }); + + it("a dropped column's indexes are dropped first, in the same commit", async () => { + await filled(2); + const Dropped = evolveTable(EditsV1, (t) => t.drop("file")); + const seq = store.lastSeq; + const edits = await prepared(Dropped); + const [commit] = await store.outbox!.read(seq, 10, context); + expect(commit!.writes.slice(0, 2)).toEqual([ + { kind: "dropIndex", table: TABLE, name: "by_file" }, + { kind: "alter", table: TABLE, version: 2, edit: { kind: "drop", name: "file" } }, + ]); + expect(store.table(TABLE)).toMatchObject({ columns: { id: { type: "str" }, at: { type: "int" } }, indexes: {} }); + expect(await all(edits, Dropped)).toEqual([ + { id: "e000", at: 0 }, + { id: "e001", at: 1 }, + ]); + }); + + it("an update step is one updateWhere over the rows below its version, stamping it, with ready", async () => { + await filled(5); + const Bumped = evolveTable(EditsV1, (t) => t.update({ where: (e) => lt(e.at, 3), set: { file: "old" } })); + const seq = store.lastSeq; + const edits = await prepared(Bumped); + const commits = await store.outbox!.read(seq, 10, context); + expect(commits.map(({ writes }) => writes.map(({ kind }) => kind))).toEqual([ + ["table"], + ["updateWhere", "table"], + ]); + expect(commits[1]!.writes[0]).toMatchObject({ kind: "updateWhere", count: 3, version: 2, set: { file: "old" } }); + expect((await all(edits, Bumped)).map((row) => (row as { file: string }).file)).toEqual([ + "old", + "old", + "old", + "d0/f3", + "d1/f4", + ]); + }); + + it("a json(...) version change records the column's version: no DDL, cells migrate on access", async () => { + const Notes1 = defineTable("edits", { + version: 1, + columns: { id: str(), meta: opt(json(obj({ a: int() }), { version: 1 })) }, + key: ["id"], + }); + const Notes2 = evolveTable(Notes1, (t) => + t.json( + "meta", + json(obj({ a: int(), b: int() }), { version: 2, migrate: (old) => ({ ...(old as { a: number }), b: 0 }) }), + ), + ); + store = await memoryStore({ outbox: true }); + const v1 = await prepared(Notes1); + const staged: StagedWrite[] = []; + new TableChanges(v1, staged).put({ id: "a", meta: { a: 1 } }); + await store.commit(staged, OPEN, context); + const seq = store.lastSeq; + const v2 = await prepared(Notes2); + const [commit] = await store.outbox!.read(seq, 10, context); + expect(commit!.writes).toEqual([ + { + kind: "alter", + table: TABLE, + version: 2, + edit: { kind: "json", name: "meta", column: expect.objectContaining({ type: "json", version: 2 }) }, + }, + expect.objectContaining({ kind: "table", version: 2, state: "ready" }), + ]); + expect(await v2.get({ id: "a" }, context)).toEqual({ id: "a", meta: { a: 1, b: 0 } }); + }); +}); + +describe("the chunk yield", () => { + it("yields between chunks without Node's setImmediate (Workers, browsers), through a message, not a timer", async () => { + await filled(3 * MIGRATION_CHUNK); + const saved = globalThis.setImmediate; + // Runtimes without it: Cloudflare Workers, browsers. + Reflect.deleteProperty(globalThis, "setImmediate"); + const timers = vi.spyOn(globalThis, "setTimeout"); + try { + await prepared(EditsV2); + expect(timers).not.toHaveBeenCalled(); + } finally { + timers.mockRestore(); + globalThis.setImmediate = saved; + } + expect(store.table(TABLE)).toMatchObject({ version: 2, state: "ready" }); + }); + + it("yields between chunks without a timer (no 1 ms clamp), while timers due run in between (a latency guard)", async () => { + await filled(10 * MIGRATION_CHUNK); + let fired = false; + // Due during the migration: it fires between chunks (a microtask-only loop would starve it). + setTimeout(() => { + fired = true; + }, 0); + const timers = vi.spyOn(globalThis, "setTimeout"); + try { + await prepared(EditsV2); + expect(timers).not.toHaveBeenCalled(); + } finally { + timers.mockRestore(); + } + expect(fired).toBe(true); + expect(store.table(TABLE)).toMatchObject({ version: 2, state: "ready" }); + }); +}); + +describe("patches", () => { + it("a rows patch writes a JSON null into a json(...) cell and clears an opt column with undefined", async () => { + const Cells1 = defineTable("edits", { + version: 1, + columns: { id: str(), data: json(nul(), { version: 1 }), note: opt(str()) }, + key: ["id"], + }); + const Cells2 = evolveTable(Cells1, (t) => + t.add("more", opt(json(nul(), { version: 1 }))).rows(() => ({ more: null, note: undefined })), + ); + store = await memoryStore(); + const v1 = await prepared(Cells1); + const staged: StagedWrite[] = []; + new TableChanges(v1, staged).put({ id: "a", data: null, note: "n" }); + await store.commit(staged, OPEN, context); + const v2 = await prepared(Cells2); + expect(await v2.get({ id: "a" }, context)).toEqual({ id: "a", data: null, more: null }); + }); +}); + +describe("resumption", () => { + it("a data step that throws leaves the table migrating with its committed chunks; the next run continues", async () => { + await filled(250); + let fail = true; + const Failing = evolveTable(EditsV1, (t) => + t.rows((row) => { + if (fail && row.id === id(150)) throw new Error("crash"); + return { file: `${row.file}!` }; + }), + ); + expect(await rejection(prepared(Failing))).toMatchObject({ message: "crash" }); + expect(store.table(TABLE)).toMatchObject({ version: 2, state: "migrating" }); + const seq = store.lastSeq; + fail = false; + const edits = await prepared(Failing); + expect(await commitsAfter(seq)).toEqual([["update x100"], ["update x50", "table"]]); + const files = (await all(edits, Failing)).map((row) => (row as { file: string }).file); + // Every row patched once: the first chunk's rows were skipped on the second run. + expect(files.every((file) => file.endsWith("!") && !file.endsWith("!!"))).toBe(true); + expect(store.table(TABLE)).toMatchObject({ version: 2, state: "ready" }); + }); + + it("a refused commit (the activation abandoned) stops the migration between chunks", async () => { + await filled(MIGRATION_CHUNK * 2 + 1); + let admitted = 2; + const fence = { admit: () => admitted-- > 0, adopt: () => {} }; + const migration = prepareTables([{ definition: EditsV2, table: TABLE }], store, fence, "app.store", context); + expect(await rejection(migration)).toMatchObject({ code: "released" }); + expect(store.table(TABLE)).toMatchObject({ version: 2, state: "migrating" }); + }); +}); + +describe("refusals", () => { + it.each<[string, () => Promise, string]>([ + ["a newer version recorded", async () => prepared(EditsV1), "newer than the code's 1"], + [ + "a version below the history's base", + async () => prepared(defineTable("edits", { version: 7, columns: { id: str() }, key: ["id"] })), + "older than the history's base 7", + ], + [ + "a history edited in place", + async () => prepared(evolveTable(EditsV1, (t) => t.rename("file", "name"))), + "recorded with another definition", + ], + ])("%s (the store at version 2) throws before anything is written", async (_, prepare, message) => { + await filled(1); + await prepared(EditsV2); + const seq = store.lastSeq; + expect(await rejection(prepare())).toMatchObject({ message: expect.stringContaining(message) }); + expect(store.lastSeq).toBe(seq); + }); +}); diff --git a/packages/chord/test/next/store/table-writes.test.ts b/packages/chord/test/next/store/table-writes.test.ts new file mode 100644 index 000000000..d1bd06c0a --- /dev/null +++ b/packages/chord/test/next/store/table-writes.test.ts @@ -0,0 +1,233 @@ +// Table writes (slice 6a §6, §9): schema writes, row writes as the outbox records them, and alter writes. + +import { afterEach, describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { add, eq, prefix, replacePrefix } from "../../../src/next/store/expression.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import { query } from "../../../src/next/store/query.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import type { TableEdit } from "../../../src/next/store/table-history.ts"; +import { stageEdit, stageIndex } from "../../../src/next/store/table-writes.ts"; +import { memoryStore, metered, tableIn } from "../support.ts"; + +const Edits = defineTable("edits", { + version: 2, + columns: { + id: str(), + path: str(), + hits: int({ default: 0 }), + note: opt(str()), + meta: opt(json(obj({}), { version: 3 })), + }, + key: ["id"], + indexes: { by_path: ["path"] }, +}); +const TABLE = "session:app.store:edits"; +const FENCE = { admit: () => true, adopt: () => {} }; + +let store: ChordStore; +afterEach(async () => { + await store?.close(context); +}); + +async function commitWith(fill: (changes: TableChanges) => void): Promise { + const staged: StagedWrite[] = []; + fill(new TableChanges(handle, staged)); + await store.commit(staged, FENCE, context); +} + +let handle: Awaited>; + +async function opened(): Promise { + store = await memoryStore({ outbox: true }); + handle = await tableIn(store, Edits); +} + +describe("schema writes", () => { + it("record the table and its indexes; the table write carries version, state, columns, key, fingerprint", async () => { + await opened(); + const [commit] = await store.outbox!.read(0, 1, context); + expect(commit!.writes).toEqual([ + { + kind: "table", + table: TABLE, + version: 2, + state: "ready", + columns: Edits.record, + key: ["id"], + fingerprint: Edits.fingerprint, + }, + { kind: "index", table: TABLE, name: "by_path", columns: ["path"] }, + ]); + expect(store.table(TABLE)).toMatchObject({ version: 2, state: "ready", indexes: { by_path: ["path"] } }); + const indexes = await store.read( + (executor) => executor.all(`SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = ?`, TABLE), + context, + ); + expect(indexes).toContainEqual({ name: `${TABLE}:by_path` }); + }); + + it("an index write for a table the store does not have fails the step (storage_failed)", async () => { + await opened(); + const write = stageIndex({ kind: "index", table: "session:app.store:nope", name: "x", columns: ["id"] }); + await expect(store.commit([write], FENCE, context)).rejects.toMatchObject({ reason: "storage_failed" }); + }); +}); + +describe("row writes in the outbox", () => { + it("carry what a replica needs: put with v and json versions, update and delete with counts, the canonical set", async () => { + await opened(); + await commitWith((changes) => { + changes.put({ id: "a", path: "x/1", meta: {} }); + changes.put({ id: "b", path: "x/2" }); + changes.update({ id: "a" }, (e: { hits: never }) => ({ hits: add(e.hits, 1), note: undefined, meta: {} })); + changes.update({ id: "zz" }, { note: "n" }); + changes.delete({ id: "b" }); + changes.delete({ id: "b" }); + }); + const [, commit] = await store.outbox!.read(0, 10, context); + expect(commit!.writes).toEqual([ + { + kind: "put", + table: TABLE, + row: { id: "a", path: "x/1", hits: 0, meta: {} }, + version: 2, + columns: { meta: 3 }, + }, + { kind: "put", table: TABLE, row: { id: "b", path: "x/2", hits: 0 }, version: 2, columns: {} }, + { + kind: "update", + table: TABLE, + key: { id: "a" }, + set: { hits: { add: [{ col: "hits" }, 1] }, note: null, meta: { json: {} } }, + columns: { meta: 3 }, + count: 1, + }, + { kind: "update", table: TABLE, key: { id: "zz" }, set: { note: "n" }, columns: {}, count: 0 }, + { kind: "delete", table: TABLE, key: { id: "b" }, count: 1 }, + { kind: "delete", table: TABLE, key: { id: "b" }, count: 0 }, + ]); + }); + + it("a put is an upsert: the whole row replaced", async () => { + await opened(); + await commitWith((changes) => changes.put({ id: "a", path: "x", hits: 5, note: "n" })); + await commitWith((changes) => changes.put({ id: "a", path: "y" })); + expect(await handle.get({ id: "a" }, context)).toEqual({ id: "a", path: "y", hits: 0 }); + }); + + it("updateWhere and deleteWhere are one statement each, recorded with where, set and count", async () => { + await opened(); + await commitWith((changes) => { + for (let i = 0; i < 5; i++) changes.put({ id: `e${i}`, path: i < 3 ? `a/${i}` : `b/${i}` }); + }); + const { counts, context: counting } = await metered(); + const staged: StagedWrite[] = []; + const changes = new TableChanges(handle, staged); + changes.updateWhere( + (e: { path: never }) => prefix(e.path, "a/"), + (e: { path: never }) => ({ path: replacePrefix(e.path, "a/", "c/") }), + ); + changes.deleteWhere((e: { path: never }) => eq(e.path, "b/4")); + await store.commit(staged, FENCE, counting); + // The two bulk writes, the outbox commit, lastSeq. + expect(counts()["chord.store.statements"]).toBe(4); + const [, , commit] = await store.outbox!.read(0, 10, context); + expect(commit!.writes).toEqual([ + { + kind: "updateWhere", + table: TABLE, + where: { prefix: ["path", "a/"] }, + set: { path: { replacePrefix: [{ col: "path" }, "a/", "c/"] } }, + columns: {}, + count: 3, + }, + { kind: "deleteWhere", table: TABLE, where: { eq: ["path", "b/4"] }, count: 1 }, + ]); + expect((await handle.page(query(Edits), 10, context)).items.map((r) => r.path)).toEqual([ + "c/0", + "c/1", + "c/2", + "b/3", + ]); + }); +}); + +describe("alter writes", () => { + /** Applies `edits` to the edits table as the migration to version 3 does, one commit. */ + async function altered(...edits: TableEdit[]): Promise { + await store.commit( + edits.map((edit) => stageEdit(TABLE, 3, edit)), + FENCE, + context, + ); + } + + /** The physical columns of the edits table. */ + async function physical(): Promise { + const rows = await store.read( + (executor) => executor.all<{ readonly name: string }>(`SELECT name FROM pragma_table_info('${TABLE}')`), + context, + ); + return rows.map(({ name }) => name); + } + + it("rename a column: a json(...) one's version column, the key and the indexes follow", async () => { + await opened(); + await commitWith((changes) => changes.put({ id: "a", path: "p" })); + await altered({ kind: "rename", from: "meta", to: "info" }, { kind: "rename", from: "path", to: "file" }); + expect(await physical()).toEqual(["id", "file", "hits", "note", "info", "info$v", "$v"]); + expect(store.table(TABLE)).toMatchObject({ indexes: { by_path: ["file"] } }); + expect(Object.keys(store.table(TABLE)!.columns)).toEqual(["id", "file", "hits", "note", "info"]); + }); + + it("add a column: existing rows get its default; an opt(...) json(...) one adds its version column", async () => { + await opened(); + await commitWith((changes) => changes.put({ id: "a", path: "p" })); + await altered( + { kind: "add", name: "level", column: { type: "int", default: 3 } }, + { kind: "add", name: "extra", column: { type: "json", optional: true, version: 1, fingerprint: "f" } }, + ); + expect(await physical()).toEqual([ + "id", + "path", + "hits", + "note", + "meta", + "meta$v", + "$v", + "level", + "extra", + "extra$v", + ]); + const rows = await store.read((executor) => executor.all(`SELECT level FROM "${TABLE}"`), context); + expect(rows).toEqual([{ level: 3 }]); + expect(store.table(TABLE)!.columns.level).toEqual({ type: "int", default: 3 }); + }); + + it("drop a column (with its version column); a json edit changes the record only", async () => { + await opened(); + await commitWith((changes) => changes.put({ id: "a", path: "p" })); + const column = { type: "json", optional: true, version: 4, fingerprint: "g" } as const; + await altered({ kind: "drop", name: "note" }, { kind: "json", name: "meta", column }); + expect(await physical()).toEqual(["id", "path", "hits", "meta", "meta$v", "$v"]); + expect(store.table(TABLE)!.columns.meta).toEqual(column); + await altered({ kind: "drop", name: "meta" }); + expect(await physical()).toEqual(["id", "path", "hits", "$v"]); + }); + + it("are recorded as alter writes with the migration's version; index edits as index and dropIndex", async () => { + await opened(); + const seq = store.lastSeq; + await altered({ kind: "dropIndex", name: "by_path" }, { kind: "drop", name: "path" }); + const [commit] = await store.outbox!.read(seq, 1, context); + expect(commit!.writes).toEqual([ + { kind: "dropIndex", table: TABLE, name: "by_path" }, + { kind: "alter", table: TABLE, version: 3, edit: { kind: "drop", name: "path" } }, + ]); + }); +}); diff --git a/packages/chord/test/next/store/table.test.ts b/packages/chord/test/next/store/table.test.ts new file mode 100644 index 000000000..70acab99d --- /dev/null +++ b/packages/chord/test/next/store/table.test.ts @@ -0,0 +1,66 @@ +// `defineTable` (slice 6a §5): what a declaration materializes, the checks it makes, and its fingerprint. + +import { describe, expect, it } from "vitest"; +import { int, obj, opt, str } from "../../../src/next/contracts/index.ts"; +import { json } from "../../../src/next/store/json-column.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; + +const columns = { id: str(), path: str(), at: int(), note: opt(str()), meta: json(obj({}), { version: 1 }) }; + +describe("defineTable", () => { + it("materializes columns, key, indexes and the record", () => { + const Edits = defineTable("edits", { version: 2, columns, key: ["id"], indexes: { by_path: ["path", "at"] } }); + expect(Edits.name).toBe("edits"); + expect(Edits.version).toBe(2); + expect(Edits.key).toEqual(["id"]); + expect([...Edits.indexes]).toEqual([["by_path", ["path", "at"]]]); + expect(Object.keys(Edits.record)).toEqual(["id", "path", "at", "note", "meta"]); + expect(Edits.rows.physical).toEqual(["id", "path", "at", "note", "meta", "meta$v", "$v"]); + }); + + it("checks the table name, the version, key, index names and index columns", () => { + expect(() => defineTable("Edits", { version: 1, columns, key: ["id"] })).toThrow("expected lowercase"); + expect(() => defineTable("a:b", { version: 1, columns, key: ["id"] })).toThrow("expected lowercase"); + expect(defineTable("@pi/file-history.edits", { version: 1, columns, key: ["id"] }).name).toBe( + "@pi/file-history.edits", + ); + expect(() => defineTable("edits", { version: 0, columns, key: ["id"] })).toThrow("positive integer"); + expect(() => defineTable("edits", { version: 1, columns, key: [] })).toThrow("no column"); + expect(() => defineTable("edits", { version: 1, columns, key: ["note" as "id"] })).toThrow( + "non-opt str() or int()", + ); + expect(() => defineTable("edits", { version: 1, columns, key: ["meta" as "id"] })).toThrow( + "non-opt str() or int()", + ); + expect(() => defineTable("edits", { version: 1, columns, key: ["id", "id"] })).toThrow("occurs twice"); + expect(() => defineTable("edits", { version: 1, columns, key: ["nope" as "id"] })).toThrow('no column "nope"'); + expect(() => defineTable("edits", { version: 1, columns, key: ["id"], indexes: { ByPath: ["path"] } })).toThrow( + "[a-z0-9_]+", + ); + expect(() => + defineTable("edits", { version: 1, columns, key: ["id"], indexes: { by: ["note" as "path"] } }), + ).toThrow("non-opt"); + }); + + it("fingerprints columns, key and indexes, not their declaration order", () => { + const a = defineTable("edits", { version: 1, columns, key: ["id"], indexes: { x: ["path"], y: ["at"] } }); + const reordered = { meta: columns.meta, note: columns.note, at: columns.at, path: columns.path, id: columns.id }; + const b = defineTable("edits", { + version: 1, + columns: reordered, + key: ["id"], + indexes: { y: ["at"], x: ["path"] }, + }); + expect(b.fingerprint).toBe(a.fingerprint); + expect(defineTable("edits", { version: 1, columns, key: ["id"] }).fingerprint).not.toBe(a.fingerprint); + expect( + defineTable("edits", { version: 1, columns, key: ["id", "at"], indexes: { x: ["path"], y: ["at"] } }) + .fingerprint, + ).not.toBe(a.fingerprint); + const otherMeta = { ...columns, meta: json(obj({ a: int() }), { version: 1 }) }; + expect( + defineTable("edits", { version: 1, columns: otherMeta, key: ["id"], indexes: { x: ["path"], y: ["at"] } }) + .fingerprint, + ).not.toBe(a.fingerprint); + }); +}); diff --git a/packages/chord/test/next/store/text.test.ts b/packages/chord/test/next/store/text.test.ts new file mode 100644 index 000000000..f6db55f3f --- /dev/null +++ b/packages/chord/test/next/store/text.test.ts @@ -0,0 +1,62 @@ +// Text rules (slice 6a §5): code point order, stored text (lone surrogates replaced, NUL refused), prefix bounds. + +import { describe, expect, it } from "vitest"; +import { compareText, prefixUpperBound, storedText } from "../../../src/next/store/text.ts"; + +describe("compareText", () => { + it("orders by code point: a supplementary character after every BMP character, U+E000 to U+FFFF included", () => { + const sorted = ["\u{10000}", "\uFFFF", "\uE000", "\uD7FF", "z", "a", "", "ab"].sort(compareText); + expect(sorted).toEqual(["", "a", "ab", "z", "\uD7FF", "\uE000", "\uFFFF", "\u{10000}"]); + expect(compareText("a", "a")).toBe(0); + expect(compareText("\u{1F600}", "\u{1F601}")).toBeLessThan(0); + }); + + it("agrees with UTF-8 byte order", () => { + const encode = (text: string) => [...new TextEncoder().encode(text)]; + const texts = ["\u{1F600}", "\uFF5E", "\u00E9", "e", "\u{10FFFF}", "\uE000", "aa", "a"]; + const byBytes = [...texts].sort((a, b) => { + const [x, y] = [encode(a), encode(b)]; + for (let i = 0; i < Math.min(x.length, y.length); i++) if (x[i] !== y[i]) return x[i]! - y[i]!; + return x.length - y.length; + }); + expect([...texts].sort(compareText)).toEqual(byBytes); + }); +}); + +describe("storedText", () => { + it("keeps well-formed text as it is and replaces a lone surrogate with U+FFFD", () => { + const text = "a\u{1F600}b"; + expect(storedText(text, "v")).toBe(text); + expect(storedText("a\uD800b", "v")).toBe("a\uFFFDb"); + }); + + it("refuses NUL with invalid/args", () => { + expect(() => storedText("a\0b", "edits.path")).toThrow( + expect.objectContaining({ code: "invalid", reason: "args" }), + ); + }); +}); + +describe("prefixUpperBound", () => { + it("is the next code point after the last one", () => { + expect(prefixUpperBound("a/")).toBe("a0"); + expect(prefixUpperBound("a\u{1F600}")).toBe("a\u{1F601}"); + expect(prefixUpperBound("\uFFFF")).toBe("\u{10000}"); + }); + + it("skips the surrogates and carries past U+10FFFF", () => { + expect(prefixUpperBound("\uD7FF")).toBe("\uE000"); + expect(prefixUpperBound("a\u{10FFFF}")).toBe("b"); + expect(prefixUpperBound("\u{10FFFF}\u{10FFFF}")).toBeUndefined(); + expect(prefixUpperBound("")).toBeUndefined(); + }); + + it("bounds exactly the strings starting with the prefix, in code point order", () => { + const texts = ["a", "a/", "a/x", "a/\u{1F600}", "a/\uFFFF", "a0", "a.", "b"]; + for (const prefix of ["a/", "a", "a/\uFFFF"]) { + const upper = prefixUpperBound(prefix)!; + const inRange = texts.filter((t) => compareText(t, prefix) >= 0 && compareText(t, upper) < 0); + expect(inRange).toEqual(texts.filter((t) => t.startsWith(prefix))); + } + }); +}); diff --git a/packages/chord/test/next/support.ts b/packages/chord/test/next/support.ts new file mode 100644 index 000000000..7ad2b9cd7 --- /dev/null +++ b/packages/chord/test/next/support.ts @@ -0,0 +1,973 @@ +// Test support: cost counters through the in-memory telemetry adapter, GC checks, and hosts, links, watches and +// rejections for the slice tests. + +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; +import { afterEach, beforeAll, beforeEach, describe, expect, vi } from "vitest"; +import { BACKGROUND_CONTEXT, type Context, withTelemetry } from "../../src/context/index.ts"; +import type { Op } from "../../src/delta/index.ts"; +import type { JsonValue } from "../../src/json.ts"; +import { + type Address, + arr, + bool, + bytes, + type Contract, + hostKind, + int, + local, + method, + obj, + opt, + type ParamsOf, + remote, + str, +} from "../../src/next/contracts/index.ts"; +import { currentVariants, variantNames, variantOrder } from "../../src/next/contracts/internal.ts"; +import { ChordError, isChordError, PluginError, type Principal } from "../../src/next/data/index.ts"; +import { PROTOCOL_VERSION } from "../../src/next/data/internal.ts"; +import { defineState } from "../../src/next/documents/index.ts"; +import type { Attempt } from "../../src/next/host/attempt.ts"; +import { + type Ctx, + createHost, + defineFacet, + type FacetHost, + type Host, + type HostError, + type RefState, + type Self, + type StoreSelf, +} from "../../src/next/host/index.ts"; +import type { ChordObject } from "../../src/next/host/object.ts"; +import { type Compression, type Link, memoryLinkPair, type PartLimits } from "../../src/next/link/index.ts"; +import { + binaryCodec, + type Codec, + checkMessage, + jsonPart, + type Message, + openPart, + type Part, + parsePart, + textCodec, +} from "../../src/next/link/internal.ts"; +import type { ArchivePart } from "../../src/next/store/archive.ts"; +import { sqliteBackend } from "../../src/next/store/backend.ts"; +import { changeDocument, createDocument, retireDocument } from "../../src/next/store/engine.ts"; +import { json } from "../../src/next/store/json-column.ts"; +import type { StagedWrite } from "../../src/next/store/step.ts"; +import { type ChordStore, openStore, type Store } from "../../src/next/store/store.ts"; +import { defineTable, type TableDefinition } from "../../src/next/store/table.ts"; +import { TableChanges } from "../../src/next/store/table-changes.ts"; +import { evolveTable } from "../../src/next/store/table-evolve.ts"; +import { TableHandle, type TableOwner } from "../../src/next/store/table-handle.ts"; +import { prepareTables } from "../../src/next/store/table-migration.ts"; +import { nodeCompression } from "../../src/next-node/compression.ts"; +import { openNodeSqliteDatabase } from "../../src/sqlite/node.ts"; +import type { NativeWatchHandle, WatchCall, WatchHandle } from "../../src/watch.ts"; + +/** Counts of the `chord.data.*` counters. */ +export type Counts = { walks: number; validations: number; compilations: number }; + +/** Runs `work` with a counting context and returns the counters it added. */ +export async function counted(work: (context: Context) => void): Promise { + const telemetry = new InMemoryTelemetryContext(); + await telemetry.startSpan({ name: "test" }, (span) => work(withTelemetry(span, BACKGROUND_CONTEXT))); + const counts: Counts = { walks: 0, validations: 0, compilations: 0 }; + for (const { name, value } of telemetry.getMetrics()) { + const counter = name.replace("chord.data.", "") as keyof Counts; + counts[counter] += value; + } + return counts; +} + +setFlagsFromString("--expose-gc"); +const gc = runInNewContext("gc") as () => void; + +/** Expects every referent to be collected once nothing else holds it. */ +export async function expectCollected(...refs: WeakRef[]): Promise { + for (let round = 0; round < 10 && refs.some((ref) => ref.deref() !== undefined); round++) { + await new Promise((resolve) => setImmediate(resolve)); + gc(); + } + expect(refs.map((ref) => ref.deref() === undefined)).toEqual(refs.map(() => true)); +} + +/** The heap in use once pending work ran and garbage was collected, repeatedly. */ +export async function settledHeap(): Promise { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setImmediate(resolve)); + gc(); + } + return process.memoryUsage().heapUsed; +} + +/** `true` when `A` and `B` are the same type; for type-level tests where `expectTypeOf` cannot compare (bytes). */ +export type Equal = (() => T extends A ? 1 : 2) extends () => T extends B ? 1 : 2 ? true : false; + +/** A telemetry adapter, a context counting through it, and the counters recorded so far, by name. */ +export async function metered(): Promise<{ + readonly telemetry: InMemoryTelemetryContext; + readonly context: Context; + counts(): Record; +}> { + const telemetry = new InMemoryTelemetryContext(); + // A settled span still records through its adapter's metrics. + const span = await telemetry.startSpan({ name: "test" }, (span) => span); + const counts = (): Record => { + const result: Record = {}; + for (const { name, value } of telemetry.getMetrics()) result[name] = (result[name] ?? 0) + value; + return result; + }; + return { telemetry, context: withTelemetry(span, BACKGROUND_CONTEXT), counts }; +} + +/** Waits until queued watch frames have been delivered. */ +export function delivered(): Promise { + return new Promise((resolve) => setTimeout(resolve, 0)); +} + +/** Starts `watch` and collects its frames as `[value, ops]`. */ +export function framesOf(watch: WatchHandle): [T, readonly Op[]][] { + const frames: [T, readonly Op[]][] = []; + watch.start(async (call) => { + if (call.kind === "value") frames.push([call.value, call.ops]); + }); + return frames; +} + +/** Starts `watch` and collects every call it gets: values (the `retired` flag included) and failures. */ +export function changesOf(watch: WatchHandle): WatchCall[] { + const calls: WatchCall[] = []; + watch.start(async (call) => { + calls.push(call); + }); + return calls; +} + +/** The values of `calls` (a failure as `{ failed: message }`), for short expectations. */ +export function valuesOf(calls: readonly WatchCall[]): (T | { readonly failed: string })[] { + return calls.map((call) => (call.kind === "value" ? call.value : { failed: (call.error as Error).message })); +} + +/** + * Fails the live attempt `self` as a runtime failure of its owner does (slice 6a §1.4); `objects` is the test file's + * `instancesOf(ChordObject, "attempt")`, whose objects other than `self`'s ignore it. + */ +export function failLive(objects: readonly WeakRef[], self: Self, error: ChordError): void { + // The factory's `self` is its attempt. + for (const object of objects) object.deref()?.failLive(self as unknown as Attempt, error); +} + +/** A local type whose objects' attempts a test ends by hand (`restartable`); it ends with its last reference. */ +export const Restartable = local("app.restartable", { + version: 1, + params: { id: int() }, + members: { touch: method() }, +}); + +/** A local type with empty params whose attempts a test ends by hand (`storeRestartable`): it may persist (P3). */ +export const StoreRestartable = local("app.store", { version: 1, params: {}, members: { touch: method() } }); + +/** + * Serves `Restartable` with `factory` per attempt (its number from 1) and gets object 1 live. `fail()` fails the live + * attempt as a runtime failure of its owner does (slice 6a §1.4); `retry()` asks for the next attempt and resolves + * once it is live; `restart()` does both. `objects` is the test file's `instancesOf(ChordObject, "attempt")`; `serve` serves other + * types on the same host. + */ +export function restartable( + objects: readonly WeakRef[], + factory: (attempt: number, self: Self) => void, + options: Parameters[1] & { readonly serve?: (host: FacetHost) => void } = {}, +) { + return restartableOf(Restartable, { id: 1 }, objects, factory, options); +} + +/** As `restartable`, for `StoreRestartable`: its factories may declare tables and persisted documents. */ +export function storeRestartable( + objects: readonly WeakRef[], + factory: (attempt: number, self: StoreSelf) => void, + options: Parameters[1] & { readonly serve?: (host: FacetHost) => void } = {}, +) { + return restartableOf(StoreRestartable, {}, objects, factory as (attempt: number, self: Self) => void, options); +} + +async function restartableOf( + Type: C, + params: ParamsOf, + objects: readonly WeakRef[], + factory: (attempt: number, self: Self) => void, + options: Parameters[1] & { readonly serve?: (host: FacetHost) => void }, +) { + const attempts: Self[] = []; + let started = 0; + const { host, errors } = await loaded((h) => { + options.serve?.(h); + // Both types have the one method `touch`; the factory's `self` is the attempt either way. + h.serve(Type as typeof Restartable, (_, self) => { + factory(++started, self); + attempts.push(self); + return { touch: async () => null }; + }); + }, options); + const get = () => host.get(Type as typeof Restartable, params as ParamsOf, BACKGROUND_CONTEXT); + const reference = await get(); + const fail = (): void => { + failLive(objects, attempts.at(-1)!, new ChordError({ code: "unavailable", message: "boom" })); + }; + const retry = async (): Promise => { + reference.retry(); + (await get()).release(); + }; + const restart = async (): Promise => { + fail(); + await retry(); + }; + return { host, errors, reference, attempts, fail, retry, restart }; +} + +/** A host with background errors collected, loaded with one facet (`@test/app`) whose setup is `setup`. */ +export async function loaded( + setup: (host: FacetHost) => void, + options: { + readonly telemetry?: InMemoryTelemetryContext; + readonly context?: Context; + readonly closeTimeout?: number; + readonly readyTimeout?: number; + readonly store?: Store; + } = {}, +): Promise<{ readonly host: Host; readonly errors: HostError[] }> { + const errors: HostError[] = []; + const { telemetry, closeTimeout, readyTimeout, store } = options; + const host = createHost({ telemetry, closeTimeout, readyTimeout, store, onError: (error) => errors.push(error) }); + await host.load([{ name: "@test/app", facet: defineFacet(setup) }], options.context ?? BACKGROUND_CONTEXT); + return { host, errors }; +} + +/** A store on a fresh SQLite `:memory:` database, as Chord's layers see it. */ +export async function memoryStore(options?: { readonly outbox?: boolean }): Promise { + const db = await openNodeSqliteDatabase(":memory:"); + return (await openStore(sqliteBackend(db), BACKGROUND_CONTEXT, options)) as ChordStore; +} + +/** + * Holds `store`'s next commit step inside its backend transaction, after its admission: `entered` resolves there, + * `release()` lets it finish; later commits run as usual. + */ +export function holdNextCommit(store: ChordStore): { readonly entered: Promise; release(): void } { + const gate = Promise.withResolvers(); + const entered = Promise.withResolvers(); + const commit = store.commit.bind(store); + store.commit = (writes, fence, ctx) => { + store.commit = commit; + const held: StagedWrite = { + order: "write", + apply: async () => { + entered.resolve(); + await gate.promise; + return { kind: "dropIndex", table: "x", name: "y" }; + }, + }; + return commit([...writes, held], fence, ctx); + }; + return { entered: entered.promise, release: () => gate.resolve() }; +} + +/** The owner of a test's table handles: its failures collected. */ +export function tableOwner(failures: ChordError[] = []): TableOwner & { readonly failures: ChordError[] } { + return { label: "app.store", scope: {}, assertLive: () => {}, fail: (error) => void failures.push(error), failures }; +} + +/** Creates `definition` (and its indexes) in `store` as `session:app.store:` and returns a handle to it. */ +export async function tableIn( + store: ChordStore, + definition: TableDefinition, + owner = tableOwner(), +): Promise { + const table = `session:app.store:${definition.name}`; + const fence = { admit: () => true, adopt: () => {} }; + await prepareTables([{ definition, table }], store, fence, "app.store", BACKGROUND_CONTEXT); + return new TableHandle(definition, table, store, owner); +} + +/** The error `promise` rejects with, as ChordError JSON when it is one. */ +export async function rejection(promise: Promise): Promise { + try { + await promise; + } catch (error) { + return isChordError(error) ? error.toJSON() : error; + } + throw new Error("expected a rejection"); +} + +/** Fakes `setTimeout` and `Date` (from 0) in the calling `describe`; `setImmediate` stays real for `flushed()`. */ +export function fakeTimers(): void { + beforeEach(() => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "Date"], now: 0 }); + }); + afterEach(() => { + vi.useRealTimers(); + }); +} + +/** Waits until every pending promise callback has run. */ +export function flushed(): Promise { + return new Promise((resolve) => setImmediate(resolve)); +} + +/** The kinds and objects of the reported `activation_failed` errors, with their causes' kinds. */ +export function failures(errors: readonly HostError[]): string[] { + return errors.flatMap((error) => + error.kind === "activation_failed" ? [`${error.object}: ${error.failure.kind}`] : [], + ); +} + +/** The principals of the remote tests. */ +export type TestPrincipal = + | { kind: "owner" } + | { kind: "guest"; session: string } + | { kind: "server" } + | { kind: "worker"; session: string }; + +/** + * A server (principal `server`, loaded with one facet whose setup is `setup`) and a client connected to it, the + * client's link stamped `child`; background errors collected per host. + */ +export async function linked( + setup: (host: FacetHost) => void, + options: { + readonly child?: TestPrincipal; + readonly context?: Context; + readonly telemetry?: InMemoryTelemetryContext; + readonly readyTimeout?: number; + } = {}, +): Promise<{ + readonly server: Host; + readonly client: Host; + readonly errors: { readonly server: HostError[]; readonly client: HostError[] }; + readonly links: readonly [Link, Link]; +}> { + const context = options.context ?? BACKGROUND_CONTEXT; + const errors = { server: [] as HostError[], client: [] as HostError[] }; + const { telemetry, readyTimeout } = options; + const server = createHost({ + principal: { kind: "server" }, + telemetry, + readyTimeout, + onError: (error) => errors.server.push(error), + }); + await server.load([{ name: "@test/app", facet: defineFacet(setup) }], context); + const client = createHost({ telemetry, readyTimeout, onError: (error) => errors.client.push(error) }); + await client.load([], context); + const links = memoryLinkPair(); + server.accept(links[0], { principal: options.child ?? { kind: "owner" }, codec: testCodec() }, context); + client.connect(links[1], { principal: { kind: "server" }, codec: testCodec() }, context); + return { server, client, errors, links }; +} + +/** The incarnation a raw peer greets with. */ +export const RAW_PEER = "00000000-0000-7000-8000-000000000000"; + +/** + * A peer speaking the wire by hand, in the current test codec, at the other end of `link`: it sends `hello` (unless + * `hello: false`, announcing `calls`, default 256), records what arrives, acks every `bound` and `state` with its origin + * (unless `ack: false`: then `ack()` sends one), and sends what a test forges. + */ +export class RawPeer { + readonly received: Message[] = []; + closed = false; + readonly codec: Codec; + readonly #link: Link; + #waiters: (() => void)[] = []; + + constructor( + link: Link, + options: { + readonly hello?: boolean; + readonly calls?: number; + readonly ack?: boolean; + readonly codec?: Codec; + } = {}, + ) { + this.#link = link; + this.codec = options.codec ?? testCodec(); + link.onMessage((bytes) => { + const message = checkMessage(this.codec.decode(bytes), BACKGROUND_CONTEXT); + this.received.push(message); + if (options.ack !== false && (message.kind === "bound" || message.kind === "state")) this.ack(message.origin); + for (const wake of this.#waiters.splice(0)) wake(); + }); + link.onClose(() => { + this.closed = true; + for (const wake of this.#waiters.splice(0)) wake(); + }); + if (options.hello !== false) { + this.send({ kind: "hello", protocol: PROTOCOL_VERSION, incarnation: RAW_PEER, calls: options.calls ?? 256 }); + } + } + + send(message: Message): void { + this.#link.send(this.codec.encode(message)); + } + + /** Acknowledges one `bound` or `state` message of `origin` (none: the peer's own). */ + ack(origin?: number): void { + this.send(origin === undefined ? { kind: "ack" } : { kind: "ack", origin }); + } + + close(): void { + this.#link.close(); + } + + /** Sends a message as raw bytes. */ + sendBytes(message: Uint8Array): void { + this.#link.send(message); + } + + /** The first received message of `kind` not taken yet, waiting for it; throws if the link closes first. */ + async next(kind: K): Promise> { + for (;;) { + const index = this.received.findIndex((message) => message.kind === kind); + if (index >= 0) return this.received.splice(index, 1)[0] as Extract; + if (this.closed) throw new Error(`link closed before ${kind}`); + await new Promise((resolve) => this.#waiters.push(resolve)); + } + } + + /** The kinds received and not taken, in order. */ + kinds(): string[] { + return this.received.map(({ kind }) => kind); + } +} + +/** A part's value as its owner reads it: opened (inflated when deflated) without limits, and parsed. */ +export function partValue(part: Part): unknown { + const unlimited = { part: Number.POSITIVE_INFINITY, decoded: Number.POSITIVE_INFINITY }; + return parsePart(openPart(part, unlimited, nodeCompression, BACKGROUND_CONTEXT)); +} + +/** The codecs every protocol test runs with. */ +export const CODECS: readonly Codec[] = [binaryCodec, textCodec]; + +let currentCodec: Codec = binaryCodec; + +/** The codec of the protocol test running now (`forEachCodec`); links in tests use it at both ends. */ +export function testCodec(): Codec { + return currentCodec; +} + +/** Runs `suite` once per codec, as `binary codec` and `text codec`. */ +export function forEachCodec(suite: () => void): void { + for (const codec of CODECS) { + describe(`${codec.name} codec`, () => { + beforeAll(() => { + currentCodec = codec; + }); + suite(); + }); + } +} + +/** The wire's indexes of a remote contract's variant (2b): the variant's, and its state members' and methods' by name. */ +export function wireNames(Type: Contract, variant: string) { + const names = variantNames(currentVariants(Type).get(variant)!); + const indexes = (list: readonly string[]) => Object.fromEntries(list.map((name, index) => [name, index])); + return { + variant: variantOrder(Type).indexOf(variant), + state: indexes(names.state), + methods: indexes(names.methods), + /** A message's members by index as an object by name, their parts parsed. */ + named: (members: readonly { readonly member: number; readonly ops?: Part; readonly value?: Part }[]) => + Object.fromEntries(members.map(({ member, ops, value }) => [names.state[member], partValue((ops ?? value)!)])), + }; +} + +/** A server host as in `linked` and a raw peer on a link it accepted as `child`, with `limits`. */ +export async function rawLinked( + setup: (host: FacetHost) => void, + options: { + readonly child?: Principal; + readonly hello?: boolean; + readonly readyTimeout?: number; + readonly limits?: PartLimits; + readonly compression?: Compression; + /** What the raw peer is to the host: a child (`accept`, the default) or its parent (`connect`). */ + readonly role?: "child" | "parent"; + } = {}, +): Promise<{ readonly server: Host; readonly peer: RawPeer; readonly errors: HostError[] }> { + const errors: HostError[] = []; + const server = createHost({ + principal: { kind: "server" }, + readyTimeout: options.readyTimeout, + compression: options.compression, + onError: (error) => errors.push(error), + }); + await server.load([{ name: "@test/app", facet: defineFacet(setup) }], BACKGROUND_CONTEXT); + const [a, b] = memoryLinkPair(); + const child = (options.child ?? { kind: "owner" }) as TestPrincipal; + const link = { principal: child, codec: testCodec(), limits: options.limits }; + if (options.role === "parent") server.connect(a, link, BACKGROUND_CONTEXT); + else server.accept(a, link, BACKGROUND_CONTEXT); + return { server, peer: new RawPeer(b, { hello: options.hello }), errors }; +} + +/** Waits until messages and closes in flight on memory links (one task per hop) have arrived. */ +export async function transported(): Promise { + for (let hop = 0; hop < 6; hop++) await flushed(); +} + +/** The variants of the notes types: an editor variant for owners, a reader one for guests. */ +const NOTES_VARIANTS = { + editor: { + state: { notes: arr(obj({ text: str() })), count: int() }, + methods: { + add: method(obj({ text: str() }), int()), + fail: method(obj({ how: str() })), + wait: method(), + echo: method(obj({ data: bytes() }), bytes()), + }, + }, + reader: { state: { notes: arr(obj({ text: str() })), count: int() }, methods: { wait: method() } }, +}; + +/** The remote type of the link tests: notes per session, an editor variant for owners, a reader one for guests. */ +export const Notes = remote("app.notes", { version: 1, params: { session: str() }, variants: NOTES_VARIANTS }); + +/** The kind of the relay tests' workers: one per session. */ +export const Session = hostKind("session", { id: str() }); + +/** `Notes` placed on the session's worker: the relay tests' type. */ +export const PlacedNotes = remote("app.placed-notes", { + version: 1, + params: { session: str() }, + placement: ({ session }) => Session({ id: session }), + variants: NOTES_VARIANTS, +}); + +const NotesDoc = defineState("notes", { + schema: obj({ list: arr(obj({ text: str() })) }), + version: 1, + initial: () => ({ list: [] }), +}); + +/** The shared store behind `Notes` (§6): the notes of a session, once, whoever views them. */ +export const NoteStore = local("app.note-store", { + version: 1, + params: { session: str() }, + members: { list: arr(obj({ text: str() })), add: method(obj({ text: str() }), int()) }, +}); + +/** What `serveNotes` records: the handlers' contexts of `wait` calls, and every factory run of `Notes`. */ +export type NotesProbe = { readonly waits: Ctx[]; readonly factories: string[] }; + +/** + * Serves `NoteStore` and `Notes` over it: `notes` and `count` are derives over the store's list; guests get the reader + * variant. `wait` resolves when its context is aborted; `fail` throws a ChordError (`chord`), a PluginError (`plugin`) + * or a plain Error. + */ +export function serveNotes( + host: FacetHost, + Type: typeof Notes | typeof PlacedNotes = Notes, +): NotesProbe { + const probe: NotesProbe = { waits: [], factories: [] }; + host.serve(NoteStore, (_, self) => { + const doc = self.state(NotesDoc); + return { + list: self.derive([doc], ([d]) => d.list), + add: async ({ text }, ctx) => { + await doc.change((draft) => void draft.list.push({ text }), ctx); + return doc.value.list.length; + }, + }; + }); + // The two types differ in their name and placement only. + host.serve(Type as typeof Notes, ({ session }, principal, self) => { + probe.factories.push(`${session} ${principal.kind}`); + const store = self.get(NoteStore, { session }); + const notes = self.derive([store.list], ([list]) => list); + const count = self.derive([store.list], ([list]) => list.length); + const wait = async (ctx: Ctx) => { + probe.waits.push(ctx); + await new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", resolve)); + return null; + }; + if (principal.kind === "guest") return { variant: "reader", members: { notes, count, wait } }; + return { + variant: "editor", + members: { + notes, + count, + wait, + add: async (args, ctx) => store.add(args, ctx), + fail: async ({ how }) => { + if (how === "chord") throw new ChordError({ code: "forbidden", message: "not yours" }); + if (how === "plugin") throw new PluginError("quota", "too many", { limit: 1 }); + throw new Error("plain"); + }, + echo: async ({ data }) => data, + }, + }; + }); + return probe; +} + +/** Adds a note to the session's store from the server itself; the reference it takes keeps the store. */ +export async function addNote(server: Host, session: string, text: string): Promise { + const store = await server.get(NoteStore, { session }, BACKGROUND_CONTEXT); + await store.add({ text }, BACKGROUND_CONTEXT); +} + +/** A client connected to a raw peer playing the provider (the link's limits `limits`), after the handshake. */ +export async function rawProvider( + options: { readonly readyTimeout?: number; readonly limits?: PartLimits; readonly compression?: Compression } = {}, +) { + const errors: HostError[] = []; + const client = createHost({ + readyTimeout: options.readyTimeout, + compression: options.compression, + onError: (error) => errors.push(error), + }); + await client.load([], BACKGROUND_CONTEXT); + const peer = await rawParent(client, options.limits); + return { client, peer, errors }; +} + +/** + * Connects `host` to a new raw peer playing its parent (a first link, or the next after a drop), the link's limits + * `limits`, after the handshake. + */ +export async function rawParent(host: Host, limits?: PartLimits): Promise { + const [a, b] = memoryLinkPair(); + host.connect(a, { principal: { kind: "server" }, codec: testCodec(), limits }, BACKGROUND_CONTEXT); + const peer = new RawPeer(b); + await peer.next("hello"); + return peer; +} + +/** Starts a `ref.state` watch and collects the states it delivers, by kind (failed ones with their error's message). */ +export function statesOf(watch: NativeWatchHandle): string[] { + const states: string[] = []; + watch.start(async (call) => { + if (call.kind !== "value") return; + const state = call.value; + states.push(state.kind === "failed" ? `failed: ${state.cause.error.message}` : state.kind); + }); + return states; +} + +/** The wire's indexes of `Notes`'s editor variant. */ +export const EDITOR = wireNames(Notes, "editor"); + +/** Answers the next hold with `bound`: the editor variant with `notes` and `count`. */ +export async function bindNext(peer: RawPeer, notes: { text: string }[] = []) { + const { hold } = await peer.next("hold"); + bindHold(peer, hold, notes); + return hold; +} + +/** Sends `bound` for `hold`: the editor variant with `notes` and `count`. */ +export function bindHold(peer: RawPeer, hold: number, notes: { text: string }[] = []): void { + const members = [ + { member: EDITOR.state.notes!, value: jsonPart(notes) }, + { member: EDITOR.state.count!, value: jsonPart(notes.length) }, + ]; + peer.send({ kind: "bound", holds: [{ hold, variant: EDITOR.variant, members, failed: [] }] }); +} + +/** + * Records weakly every instance of a class that calls `method` from now on, for GC tests of objects a test cannot + * reach (holds, remote objects). Patches the prototype for the rest of the test file. + */ +export function instancesOf( + Class: { readonly prototype: T }, + method: keyof T & string, +): WeakRef[] { + const instances: WeakRef[] = []; + const prototype = Class.prototype as Record unknown>; + const original = prototype[method]!; + prototype[method] = function (this: T, ...args: unknown[]) { + instances.push(new WeakRef(this)); + return original.apply(this, args); + }; + return instances; +} + +/** What a route's `resolve` returns in the relay tests. */ +export type Resolved = { + readonly link: Link; + readonly principal: TestPrincipal; + readonly trusted?: boolean; + readonly codec?: Codec; + readonly limits?: PartLimits; +}; + +/** What `relayed` builds: the hosts, the links the server holds, the resolves it ran and the errors per host. */ +export type Relayed = { + readonly tab: Host; + readonly server: Host; + /** The worker of each resolved session address, the latest one per key. */ + readonly workers: Map>; + /** The server's end of each resolved route link, in order. */ + readonly routeLinks: Link[]; + /** The server's end of the tab's link. */ + readonly tabLink: Link; + /** The address keys `resolve` ran for, in order. */ + readonly resolves: string[]; + readonly errors: { readonly tab: HostError[]; readonly server: HostError[]; readonly worker: HostError[] }; +}; + +/** + * Three hosts: a tab connected to a server, and a server routing `Session` to an in-process worker per address (made + * by each `resolve`, connected before it loads, W6) that serves `PlacedNotes` and `Notes`'s store. The tab acts as + * `tab` (owner by default); `authorize` admits everyone by default; route links are trusted when `trusted`. + * `server` and `worker` add setups of their own; `resolve` wraps the default one (`worker()` makes the worker). + */ +export async function relayed( + options: { + readonly tab?: TestPrincipal; + readonly authorize?: (principal: TestPrincipal, address: { readonly key: string }) => boolean; + readonly trusted?: boolean; + readonly codecs?: { readonly tab?: Codec; readonly route?: Codec }; + readonly server?: (host: FacetHost) => void; + readonly worker?: (host: FacetHost) => void; + readonly resolve?: ( + address: Address, + worker: () => Resolved, + context: Context, + ) => Promise; + readonly context?: Context; + /** The server's and the workers' `revokedBy`. */ + readonly revokedBy?: (principal: TestPrincipal) => readonly JsonValue[]; + /** The server's `revokedBy` when it differs from the workers'. */ + readonly serverRevokedBy?: (principal: TestPrincipal) => readonly JsonValue[]; + /** The compressor of each host: tab, server, workers. */ + readonly compression?: { + readonly tab?: Compression; + readonly server?: Compression; + readonly worker?: Compression; + }; + /** The limits of the tab's link at the server. */ + readonly limits?: PartLimits; + } = {}, +): Promise { + const context = options.context ?? BACKGROUND_CONTEXT; + const errors = { tab: [] as HostError[], server: [] as HostError[], worker: [] as HostError[] }; + const workers = new Map>(); + const routeLinks: Link[] = []; + const resolves: string[] = []; + const { revokedBy, compression = {} } = options; + const server = createHost({ + principal: { kind: "server" }, + revokedBy: options.serverRevokedBy ?? revokedBy, + compression: compression.server, + onError: (error) => errors.server.push(error), + }); + const codec = options.codecs?.route ?? testCodec(); + const worker = (address: Address): Resolved => { + resolves.push(address.key); + const [a, b] = memoryLinkPair(); + const principal = { kind: "worker", session: address.params.id } as const; + const host = createHost({ + address, + principal, + revokedBy, + compression: compression.worker, + onError: (error) => errors.worker.push(error), + }); + host.connect(b, { principal: { kind: "server" }, trusted: options.trusted, codec }, context); + const facet = defineFacet((h) => { + serveNotes(h, PlacedNotes); + options.worker?.(h); + }); + void host.load([{ name: "@test/worker", facet }], context); + workers.set(address.key, host); + routeLinks.push(a); + return { link: a, principal, trusted: options.trusted, codec }; + }; + const resolve = options.resolve ?? (async (_address, made) => made()); + server.route( + Session, + { + authorize: options.authorize ?? (() => true), + resolve: (address, routing) => resolve(address, () => worker(address), routing), + }, + context, + ); + await server.load( + [{ name: "@test/server", facet: defineFacet((h) => options.server?.(h)) }], + context, + ); + const tab = createHost({ compression: compression.tab, onError: (error) => errors.tab.push(error) }); + await tab.load([], context); + const [a, b] = memoryLinkPair(); + const tabCodec = options.codecs?.tab ?? testCodec(); + const accepted = { principal: options.tab ?? { kind: "owner" }, codec: tabCodec, limits: options.limits } as const; + server.accept(a, accepted, context); + tab.connect(b, { principal: { kind: "server" }, codec: tabCodec }, context); + return { tab, server, workers, routeLinks, tabLink: a, resolves, errors }; +} + +/** + * A `resolve` for `relayed` whose workers are raw peers playing the provider, collected in `peers` in order; their + * links trusted when `trusted`. + */ +export function rawWorkers( + peers: RawPeer[], + options: { + readonly calls?: number; + readonly ack?: boolean; + readonly trusted?: boolean; + readonly limits?: PartLimits; + } = {}, +): (address: Address) => Promise { + return async (address) => { + const [a, b] = memoryLinkPair(); + peers.push(new RawPeer(b, options)); + const principal = { kind: "worker", session: address.params.id } as const; + return { link: a, principal, codec: testCodec(), trusted: options.trusted, limits: options.limits }; + }; +} + +/** A host's incarnation, which its handshakes send. */ +export function incarnationOf(host: Host): string { + // Every host is a `ChordHost`. + return (host as unknown as { readonly incarnation: string }).incarnation; +} + +/** Counts the bytes every `DecompressionStream` puts out while installed; `restore` puts the real one back. */ +export function countInflated(): { readonly bytes: number; restore(): void } { + const Real = globalThis.DecompressionStream; + const counter = { bytes: 0, restore: () => vi.unstubAllGlobals() }; + class Counted { + readonly writable: WritableStream; + readonly readable: ReadableStream; + constructor(format: CompressionFormat) { + const real = new Real(format); + this.writable = real.writable; + this.readable = real.readable.pipeThrough( + new TransformStream({ + transform(chunk, controller) { + counter.bytes += chunk.byteLength; + controller.enqueue(chunk); + }, + }), + ); + } + } + vi.stubGlobal("DecompressionStream", Counted); + return counter; +} + +/** The tables of `archiveStore`: edits (v1 with an index; v2 adds `done`) and notes, whose v2 migration crashed. */ +export const ArchiveTables = (() => { + const EditsV1 = defineTable("edits", { + version: 1, + columns: { + id: str(), + path: str(), + at: int(), + note: opt(str()), + meta: opt(json(obj({ a: int() }), { version: 1 })), + }, + key: ["id"], + indexes: { by_path: ["path", "at"] }, + }); + const EditsV2 = evolveTable(EditsV1, (t) => t.add("done", bool({ default: false }))); + const NotesV1 = defineTable("notes", { version: 1, columns: { id: int(), text: str() }, key: ["id"] }); + const NotesV2 = evolveTable(NotesV1, (t) => + t.rows((row) => { + if (row.id === 120) throw new Error("crash"); + return { text: `${row.text}!` }; + }), + ); + return { EditsV1, EditsV2, NotesV1, NotesV2 }; +})(); + +/** + * A store holding a bit of everything an archive carries: `client:app.store:edits` at version 2 with 150 rows (two + * export pages; some written at version 1, `NULL` and `json(...)` cells, an index), `client:app.store:notes` left + * `migrating` at version 2 after one committed chunk, and documents: a singleton, keyed ones with deltas after their + * base, and a retired one. + */ +export async function archiveStore(options?: { readonly outbox?: boolean }): Promise { + const { EditsV1, EditsV2, NotesV1, NotesV2 } = ArchiveTables; + const store = await memoryStore(options); + const fence = { admit: () => true, adopt: () => {} }; + const edits = await tableAt(store, "client:app.store:edits", EditsV1); + const notes = await tableAt(store, "client:app.store:notes", NotesV1); + const staged: StagedWrite[] = []; + const editChanges = new TableChanges(edits, staged); + for (let i = 0; i < 150; i++) { + editChanges.put({ + id: `e${String(i).padStart(3, "0")}`, + path: `p/${i % 4}`, + at: i, + ...(i % 2 === 0 ? { note: `n${i}` } : {}), + ...(i % 3 === 0 ? { meta: { a: i } } : {}), + }); + } + const noteChanges = new TableChanges(notes, staged); + for (let i = 0; i < 150; i++) noteChanges.put({ id: i, text: `t${i}` }); + await store.commit(staged, fence, BACKGROUND_CONTEXT); + await prepareTables( + [{ definition: EditsV2, table: "client:app.store:edits" }], + store, + fence, + "app", + BACKGROUND_CONTEXT, + ); + await prepareTables( + [{ definition: NotesV2, table: "client:app.store:notes" }], + store, + fence, + "app", + BACKGROUND_CONTEXT, + ).catch(() => {}); + const address = { namespace: "chord", owner: "client:app.store", kind: "memo" }; + const base = (value: JsonValue) => ({ version: 1, fingerprint: "f1", value }); + let ids: readonly number[] = []; + const created = { + admit: () => true, + adopt: (_: number, created: readonly number[]) => { + ids = created; + }, + }; + await store.commit( + ["", "a", "b", "gone"].map((key) => + createDocument({ ...address, key }, "latest", base({ title: key }), JSON.stringify({ title: key })), + ), + created, + BACKGROUND_CONTEXT, + ); + const [, a, b, gone] = ids; + const delta = (title: string) => ({ kind: "delta", version: 1, ops: [["s", ["title"], title]] }) as const; + await store.commit( + [changeDocument(a!, 1, delta("a2"), JSON.stringify(delta("a2").ops)), retireDocument(gone!, 1)], + fence, + BACKGROUND_CONTEXT, + ); + await store.commit([changeDocument(a!, 2, delta("a3"), JSON.stringify(delta("a3").ops))], fence, BACKGROUND_CONTEXT); + await store.commit([changeDocument(b!, 1, delta("b2"), JSON.stringify(delta("b2").ops))], fence, BACKGROUND_CONTEXT); + return store; +} + +/** `definition` prepared as `table` in `store`, and a handle to it. */ +async function tableAt(store: ChordStore, table: string, definition: TableDefinition): Promise { + await prepareTables( + [{ definition, table }], + store, + { admit: () => true, adopt: () => {} }, + "app", + BACKGROUND_CONTEXT, + ); + return new TableHandle(definition, table, store, tableOwner()); +} + +/** Every part of `store`'s export. */ +export async function exported(store: Store): Promise { + const parts: ArchivePart[] = []; + for await (const part of store.export(BACKGROUND_CONTEXT)) parts.push(part); + return parts; +} From bdb68f750ed7724a7718237895b98604f7f2c114 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 10:39:09 +0000 Subject: [PATCH 059/271] docs(chord): Chord v2 and its docs live upstream; slice 6b D1 revised --- packages/chord/docs/next/decisions.md | 13 ++++++++++--- packages/chord/docs/next/slice-6b.md | 7 +++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index c87c860ca..e833cf02a 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1622,10 +1622,17 @@ validation and copies the v2 kernel added around Delta. throwing Chord's codes (6b), one retirement value ("One retirement mechanism" (2) stands). Added (owner, 2026-10-27, "lgtm"): Durable's `SessionEnd` and the `ReportFrom` argument of `runtime.report()` are tagged by `kind` too (neither is stored). +- **Chord v2 and its docs live in pi upstream (owner, 2026-10-27: "we can put chord v2 in upstream", "the docs can + all be public, and in upstream"):** `packages/chord/src/next`, `src/next-node`, `test/next`, `test/next-node` and + `docs/next` moved to pi `main` (`93029a0f1`), not exported until v2 has an entry point (experimental, noted in the + changelog); the decision log moved from `apps/p2/docs/redesign-decisions.md` to `packages/chord/docs/next/decisions.md`; + the root `AGENTS.md` carries the "Chord v2 Code Style" section. From now on every slice and every design doc change + goes to pi first (worktree of `upstream/main`, check, tests, push, CI, merge into `private`); only `apps/` stays + private. - **Slice 6b D1: `StoreStorage` and `durableView` live in Durable (owner, 2026-10-27: "they are only used if durable - is used"):** in `packages/durable/src/next/`, private-only like `packages/chord/src/next`, importing - `@earendil-works/chord/next` (Chord's `./next` export returns, private-only). No package cycle: Durable already - depends on Chord, Chord imports nothing of Durable (S13). The folder goes upstream with Chord v2. + is used"):** in Durable's own `src` (an ordinary pi module, now that Chord v2 is upstream), importing + `@earendil-works/chord/next` (Chord's `./next` export comes with them). No package cycle: Durable already depends on + Chord, Chord imports nothing of Durable (S13). Durable's conformance suite runs in Durable's tests and CI. - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the diff --git a/packages/chord/docs/next/slice-6b.md b/packages/chord/docs/next/slice-6b.md index 9879fb104..4a5255b80 100644 --- a/packages/chord/docs/next/slice-6b.md +++ b/packages/chord/docs/next/slice-6b.md @@ -48,10 +48,9 @@ Durable's Session runs unchanged on it and passes Durable's storage conformance (`createStorageConformance`), across reopen too. **Where it lives (D1, owner, 2026-10-27: "they are only used if durable is used"):** `StoreStorage` and -`durableView` belong to Durable: `packages/durable/src/next/`, private-only like `packages/chord/src/next`, importing -`@earendil-works/chord/next` (Chord's `./next` package export, private-only, returns for it). Durable already depends -on Chord and Chord imports nothing of Durable (S13); Durable's conformance suite runs in Durable's tests; the folder -goes upstream with Chord v2. +`durableView` belong to Durable, in its own `src` (Chord v2 is in pi upstream since `93029a0f1`), importing +`@earendil-works/chord/next` (Chord's `./next` package export comes with them). Durable already depends on Chord and +Chord imports nothing of Durable (S13); Durable's conformance suite runs in Durable's tests and CI. ### 1.1 Records From 7dff3effa3caf46a73dcfad15b04bd2eab34a2d3 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 11:09:06 +0000 Subject: [PATCH 060/271] docs(chord): slice 6b D2, Durable's records as declared tables --- packages/chord/docs/next/decisions.md | 6 ++++++ packages/chord/docs/next/slice-6b.md | 19 +++++++------------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index e833cf02a..ca666124d 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1633,6 +1633,12 @@ validation and copies the v2 kernel added around Delta. is used"):** in Durable's own `src` (an ordinary pi module, now that Chord v2 is upstream), importing `@earendil-works/chord/next` (Chord's `./next` export comes with them). No package cycle: Durable already depends on Chord, Chord imports nothing of Durable (S13). Durable's conformance suite runs in Durable's tests and CI. +- **Slice 6b D2: Durable's records are ordinary declared tables of the store (owner, 2026-10-27: "a, that's the whole + purpose of this slice"):** `StoreStorage` declares conversations, entries, tasks, submissions and `record_ids` as + `defineTable`s, writes them with `put` (Durable replaces complete records), and reads them with the store's queries; + the outbox, export, import, the applier and later `evolveTable` versions handle them as every table. The partial + head index becomes an index on `(conversation_id, has_head, id)`; indexed strings that must stay lossless keep + Durable's JSON encoding. - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the diff --git a/packages/chord/docs/next/slice-6b.md b/packages/chord/docs/next/slice-6b.md index 4a5255b80..257637ab4 100644 --- a/packages/chord/docs/next/slice-6b.md +++ b/packages/chord/docs/next/slice-6b.md @@ -56,18 +56,13 @@ Chord imports nothing of Durable (S13); Durable's conformance suite runs in Dura Durable's four record tables (conversations, entries, tasks, submissions) and its ID namespace. -> **Decision needed (D2): how Durable's records are stored.** -> (a) **Declared tables** in the store, written as ordinary `put` writes: `defineTable`s that `StoreStorage` declares -> for itself (SQL names `":
"`, e.g. `"session:harness:entries"`), each row the record's indexed columns -> (`id`, `conversation_id`, `kind`, `status`, …) plus the whole record in a `json(...)` column. The outbox, export, -> import and the applier handle them as they handle every table; Durable's layout changes later through -> `evolveTable` (Durable's own SQLite migrations, such as v2's `sleeping`, become table versions). Durable's -> secondary indexes map to declared indexes; the partial head index becomes an index on `(conversation_id, has_head, -> id)` (index columns are non-`opt`). Indexed strings that must stay lossless (`kind`, `requestId`, document kinds and -> keys) keep Durable's JSON encoding in their columns (decision "Text is plain UTF-8"). -> (b) **Internal tables** of `StoreStorage` with write kinds of their own (`durable.record`), which the outbox, -> export, import and the applier would each learn. More code, a second table mechanism. -> Proposal: (a). +**Layout (D2, owner, 2026-10-27: "the whole purpose of this slice"):** declared tables of the store, written as +ordinary `put` writes: `defineTable`s that `StoreStorage` declares for itself (SQL names `":
"`, e.g. +`"session:harness:entries"`), each row the record's indexed columns (`id`, `conversation_id`, `kind`, `status`, …) plus +the whole record in a `json(...)` column. The outbox, export, import and the applier handle them as every table; +Durable's layout changes later through `evolveTable`. The partial head index becomes an index on `(conversation_id, +has_head, id)` (index columns are non-`opt`); indexed strings that must stay lossless (`kind`, `requestId`, document +kinds and keys) keep Durable's JSON encoding in their columns. - **Writes:** every `StorageWrite` of a table record becomes a `put` of its row (Durable replaces complete records; conversations and entries are immutable and written once). `record_ids` is a declared table too (`id`, From cc12dad236ba91eed30b8e4f240c36ec375fe848 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 11:12:48 +0000 Subject: [PATCH 061/271] docs(chord): slice 6b D3, one id counter per store --- packages/chord/docs/next/decisions.md | 9 +++++++++ packages/chord/docs/next/slice-6b.md | 17 ++++++----------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index ca666124d..cf76ce60f 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1639,6 +1639,15 @@ validation and copies the v2 kernel added around Delta. the outbox, export, import, the applier and later `evolveTable` versions handle them as every table. The partial head index becomes an index on `(conversation_id, has_head, id)`; indexed strings that must stay lossless keep Durable's JSON encoding. +- **Slice 6b D3: one id counter per store (owner, 2026-10-27: "a"):** the store's counter (`next_document_id`, renamed + `next_id`; the archive manifest's `nextDocumentId` becomes `nextId`) numbers every id in the store: Durable's + `mintId` (conversations, entries, tasks, submissions, documents) takes from it in memory, explicit ids in Durable's + writes raise it (an entry committed with id 100: the next mint is 101), and Chord's commit step takes its documents' + ids from it; it is persisted as a high-water mark with every commit. `record_ids` stays for Durable's type check of + an id ("already belongs to entry"); supersedes "Slice 6 first decisions" (3)'s `coalesce(max(id), 1) + 1` and 6a's + "document ids monotonic with seq". Durable's own ids still rise in its creation order (its Session line serializes + its commits and mints inside them), which its scans, transcripts and windows rely on; Chord's ids are gaps between + them, as the five record types already leave. A minted id never committed may come again after a reopen (S10). - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the diff --git a/packages/chord/docs/next/slice-6b.md b/packages/chord/docs/next/slice-6b.md index 257637ab4..32e066ff4 100644 --- a/packages/chord/docs/next/slice-6b.md +++ b/packages/chord/docs/next/slice-6b.md @@ -78,17 +78,12 @@ kinds and keys) keep Durable's JSON encoding in their columns. ### 1.2 IDs -> **Decision needed (D3): one id counter or two.** Durable mints ids before a commit (`mintId`), one namespace across -> conversations, entries, tasks, submissions and documents ("greater than every id minted since this storage was -> opened and every id stored, also across reopen", S10); its documents carry their id into the commit. Chord's store -> assigns its documents' ids inside the step from its own counter (`next_document_id`), and the engine's -> `chord_documents.id` is the primary key for both. Options: (a) **one counter per store**: `mintId` takes from the -> store's counter in memory (persisted as a high-water mark with every commit), Chord's step takes from it too; Chord's -> document ids are then no longer monotonic with seq (6a small item 11) and a Durable id minted but never committed -> is skipped; (b) **two counters**: Durable's from `record_ids` (`coalesce(max(id), 1) + 1` at open, decided), Chord's -> as today, and the engine's documents table keyed by `(namespace, id)` (Durable's document ids live in Durable's -> namespace). Proposal: (b): Durable keeps its id contract unchanged and Chord its step-assigned ids; the composite key -> costs nothing measurable and keeps the two adapters apart (S13: the engine is shared, the namespaces are not). +**One counter per store (D3, owner, 2026-10-27):** the store's counter (`next_document_id` renamed `next_id`, the +manifest's `nextDocumentId` renamed `nextId`) numbers every id: `StoreStorage.mintId()` takes from it in memory, +explicit ids in Durable's writes raise it, Chord's commit step takes its documents' ids from it, and every commit +persists it as a high-water mark. `record_ids` stays for Durable's "ID n already belongs to entry" check. Durable's ids +rise in its creation order (its scans and windows rely on it); Chord's ids are gaps between them. Spec reviewers check +that nothing relies on Chord's document ids rising with seq. ### 1.3 Documents From a9181321a0c96a03082966d3bd5d8ddf18314483 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 11:18:41 +0000 Subject: [PATCH 062/271] docs(chord): slice 6b D4, the store has Durable's Storage semantics --- packages/chord/docs/next/decisions.md | 11 +++++++++++ packages/chord/docs/next/design.md | 19 ++++++++++++------- packages/chord/docs/next/slice-6b.md | 20 +++++++++----------- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index cf76ce60f..2e82dc59c 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1648,6 +1648,17 @@ validation and copies the v2 kernel added around Delta. "document ids monotonic with seq". Durable's own ids still rise in its creation order (its Session line serializes its commits and mints inside them), which its scans, transcripts and windows rely on; Chord's ids are gaps between them, as the five record types already leave. A minted id never committed may come again after a reopen (S10). +- **Slice 6b D4: the store has Durable's Storage semantics (owner, 2026-10-27: "agreed"; supersedes in part "Store + errors: Durable's rule" and slice-6a rule 10's "an expression result outside it fails the step, which is + `storage_failed`"):** Durable has two levels: its `Storage` rolls a failed commit back and stays usable (the + conformance suite tests it), its Session fails on any commit error (`GuardedStorage`). "Store errors: Durable's + rule" had put the Session's level into the store. Now: a step that fails before `COMMIT` rolls back and rejects that + commit only (the step's code runs in the transaction callback, so an error there is known to come before `COMMIT`); + the store stops only when the outcome is unknown (`COMMIT` or `ROLLBACK` failed), when `adopt` threw, or on a read's + backend error. Owners react as Durable's Session: Durable's Session fails (unchanged); a Chord object whose commit the + step refused fails live (`object_failed`, `broken`: memory and disk disagree); an `int` `CHECK` failure (a plugin's + value out of range) fails only that transaction (`invalid/args`). `StoreStorage` runs Durable's commit checks in the + step, where `SqliteStorage` runs them: no special case. Built in 6b (the store's step and object failure). - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index af87a6489..cef6bc3f1 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -894,9 +894,9 @@ host.serve(DraftStore, (_, self) => { // local, empty params: may storage batch through the store's commit step, `await store.commit(writes, adopt, ctx)` without the caller's abort signal (deltas, or a base when `checkpointWhen` says so; the line held until the step returned, as Durable's `#runCommit`, owner, 2026-10-26); adopt by pointer swap and mark dependents in the step, persisted and in-memory - documents and staged changes in one `host.batch.commit` (any backend error in the step is `storage_failed`, - Durable's rule, owner, 2026-10-26: plugin mistakes are refused when a write is staged; a seal-fenced commit is - rejected `released` before `BEGIN`); then the owner's synchronous flush recomputes dirty derives, sends one message + documents and staged changes in one `host.batch.commit` (errors as Durable's Storage, owner, 2026-10-27: a step that + fails before `COMMIT` rolls back and rejects that commit only, nothing adopted, the store goes on; plugin mistakes are + refused when a write is staged; a seal-fenced commit is rejected `released` before `BEGIN`); then the owner's synchronous flush recomputes dirty derives, sends one message per link and queues watch callbacks (W1); drafts then throw if touched. In-memory-only commits take no store seq and never enter the commit queue (W12). No revalidation of the whole document, no copy, no freeze. @@ -1077,10 +1077,15 @@ host.serve(DraftStore, (_, self) => { // local, empty params: may runtime has it, otherwise a `MessageChannel` message (2026-10-27, slice 6a build reviews). - **Eviction as in Durable,** for keyed persisted documents only (persisted singletons and `at(key)` documents stay in memory while their object lives: `handle.value` is a synchronous read; owner, 2026-10-26): one not read, written or watched for the retention - period (default 10 minutes) leaves memory and reloads on the next access. A storage error ends the store and closes - the host; `onError` receives `{ kind: "storage_failed" }` and the app decides (section 3). Any backend error in a - commit step, or on a read whose `ctx` was not aborted, is a storage error (Durable's rule); later calls reject - `unavailable/storage_failed`. + period (default 10 minutes) leaves memory and reloads on the next access. **Store errors (owner, 2026-10-27; supersedes in part "Store errors: Durable's rule"):** the store has Durable's + `Storage` semantics, its owners react as Durable's Session does. A step that fails before `COMMIT` rolls back: that + commit rejects, nothing is adopted, the store goes on. The store stops (`storage_failed`: later calls reject + `unavailable/storage_failed`, the host closes, `onError` receives `{ kind: "storage_failed" }`, the app decides, + section 3) only when the outcome is unknown (`COMMIT` or `ROLLBACK` failed) or `adopt` threw, and on a backend + error of a read whose `ctx` was not aborted. A Chord object whose commit the step refused (memory and disk disagree: + the counted conditional `UPDATE`, the live-address index) fails live (`object_failed`, `broken`); an `int` result out + of range (`CHECK`) fails only that transaction (`invalid/args`). Durable's Session fails on any commit error + (`GuardedStorage`), as today. --- diff --git a/packages/chord/docs/next/slice-6b.md b/packages/chord/docs/next/slice-6b.md index 32e066ff4..aa874e71f 100644 --- a/packages/chord/docs/next/slice-6b.md +++ b/packages/chord/docs/next/slice-6b.md @@ -71,10 +71,8 @@ kinds and keys) keep Durable's JSON encoding in their columns. orders, the fork chain walked as `SqliteStorage` does), `task`, `scanTasks`, `submission`, `scanSubmissions`, `submissionByRequest`, `scanConversations`: queries over the declared tables, Durable's cursors (`{ after, order }`, `scan.ts`) kept as they are (built from the last id, not from Chord's table cursors). -- **Validation before the step** (D4 below): what `SqliteStorage` checks inside its transaction - (`checkGlobalIds`, `prepareDocumentActions`, `checkDocumentActions`) runs before `store.commit`, with plain reads: - only Durable's Session writes these tables and it serializes its commits, so the reads see the state the step will - write on (the argument of S11). +- **Validation in the step** (D4): what `SqliteStorage` checks inside its transaction (`checkGlobalIds`, + `prepareDocumentActions`, `checkDocumentActions`) runs inside the store's step, which rolls back a refused commit. ### 1.2 IDs @@ -115,13 +113,13 @@ scope as a string (`session`, `conversation:7`, `task:31`), kind and key JSON-en - **Durable's seq is the store's seq** (gaps allowed: Chord's commits take seqs too). - **Errors:** -> **Decision needed (D4): what a refused Durable commit does to the store.** The store's rule: any backend error in a -> step is `storage_failed`, the store stops (Durable's own rule for its Session). Durable's conformance suite commits -> invalid batches on purpose ("ID 1 already belongs to conversation", "already has a current incarnation", "version -> transition requires a base", a mismatched copy) and expects the storage to go on. Proposal: those checks run before -> the step (1.1) and reject there, the store untouched; only a backend failure inside the step stops the store. In -> production the Session fails on any commit error either way (`GuardedStorage`), so the difference is only that the -> store, shared with Chord's objects, does not stop for a Durable bug caught by its checks. +**Errors (D4, owner, 2026-10-27):** the store takes Durable's `Storage` semantics (design §10 "Store errors"): a +step that fails before `COMMIT` rolls back and rejects that commit only; the store stops only on an unknown outcome +(`COMMIT` or `ROLLBACK` failed), a throwing `adopt`, or a read's backend error. Owners react as Durable's Session: +Durable's Session fails (`GuardedStorage`, unchanged); a Chord object whose commit the step refused fails live +(`object_failed`, `broken`); an `int` `CHECK` failure fails only its transaction (`invalid/args`). `StoreStorage` runs +Durable's commit checks in the step, as `SqliteStorage` does, so the conformance suite's refused commits leave the +store usable. 6b changes 6a's step and object failure accordingly. - `mintId`, `close(context)` (closes nothing of the store: the app owns it; later calls reject "closed"), reads after a store failure reject as Durable's `GuardedStorage` expects. From 4ba356228c7fea1524413b1d56f9a31351329575 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 11:18:50 +0000 Subject: [PATCH 063/271] docs(chord): mark Store errors superseded in part --- packages/chord/docs/next/decisions.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 2e82dc59c..0e929cda3 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1775,7 +1775,8 @@ validation and copies the v2 kernel added around Delta. writes no billed rows, so making every `run` count would nearly double a small write on Durable Objects for callers that need no count (Durable's `SqliteStorage`): hence a separate method (option 2, owner). Name `runCounted` (owner). pi `8025c414d` (upstream first), with a real Durable Object test in Durable's workerd suite. -- **Store errors: Durable's rule (owner, 2026-10-26; slice 6a spec review blocker 3):** any backend error in a commit +- **Store errors: Durable's rule (owner, 2026-10-26; slice 6a spec review blocker 3; **superseded in part** by "Slice 6b + D4: the store has Durable's Storage semantics": a step failing before `COMMIT` rejects only its commit):** any backend error in a commit step is `storage_failed` (the store stops, every later commit and read rejects `unavailable/storage_failed`, a new reason in the protocol list; the host closes; `onError` once), as Durable's `GuardedStorage`: the SQLite facade cannot tell a failed statement from a failed `COMMIT`, and what reaches the step is a real storage problem or memory and disk From 66330961bdc289f4cce4ecb0ee03dc0b2c905868 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 11:41:48 +0000 Subject: [PATCH 064/271] docs(chord): slice 6b D5; Harness.open in the examples --- packages/chord/docs/next/decisions.md | 10 ++++++++++ packages/chord/docs/next/design.md | 4 ++-- packages/chord/docs/next/slice-6b.md | 22 ++++++++++++---------- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 0e929cda3..c72a6d3b0 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1659,6 +1659,16 @@ validation and copies the v2 kernel added around Delta. step refused fails live (`object_failed`, `broken`: memory and disk disagree); an `int` `CHECK` failure (a plugin's value out of range) fails only that transaction (`invalid/args`). `StoreStorage` runs Durable's commit checks in the step, where `SqliteStorage` runs them: no special case. Built in 6b (the store's step and object failure). +- **Slice 6b D5: persisted run records are documents of their object; one harness per store (owner, 2026-10-27: + "sure"):** each unsettled run's record is a keyed persisted document of its object in the engine's namespace + `chord.jobs` (owner the object's owner string, kind the job name, key the run id); Chord sets that namespace, a plugin + never does, so nothing collides; the state document stays an ordinary document of the run; `finish` deletes the + record and retires the state (Q9), so only unsettled runs are stored and in memory; activation loads them through the + engine's key listing and a load per run (no new engine read). Durable's tables are named under the type name + `pi.durable` (`"session:pi.durable:entries"`; type names are unique per app). One `StoreStorage` per store: a second + one on the same store is refused (`invalid/args`), as one host per store (rule 7). Rejected on the way: a table per + job, a store-wide runs table, `$`-marked internal names (owner: "if our abstractions can't cleanly support this + without $ magic, it's a bad abstraction"). - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index cef6bc3f1..0e9f439f4 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -1923,12 +1923,12 @@ loop. Chord has no stream type (it would need its own rules for drops, cancellat access and execution environments from registry entries, looked up at each use: ```ts - const harness = self.open(async () => Durable.open(storage, { + const harness = self.open(async (context) => Harness.open(new StoreStorage(store), { // the host's store (slice 6b) models: () => host.registry(HarnessModels).value, env: (target) => host.registry(HarnessEnvironments).value(target.cwd), settings: () => currentSettings(), registry: durableRegistry(host.registry(HarnessExtensions)), // a RegistryReader over the composed NativeWatchable - }), (h) => h.close()); + }, context), (h, _reason, context) => h.close(context)); ``` Durable's `models` and `settings` accept functions (pi `7ec1f648e`). Durable opens with scheduling paused; base code diff --git a/packages/chord/docs/next/slice-6b.md b/packages/chord/docs/next/slice-6b.md index aa874e71f..4a87d1f19 100644 --- a/packages/chord/docs/next/slice-6b.md +++ b/packages/chord/docs/next/slice-6b.md @@ -25,8 +25,8 @@ const worker = createHost

({ address: Session({ id }), principal, store, onErr // the core facet: the harness object opens Durable on the same store host.serve(Harness, (_, self) => { - const durable = self.open(async (context) => - Durable.open(storeStorage(store, { owner: "session:harness" }), { …, paused: true }, context), + const harness = self.open(async (context) => + Harness.open(new StoreStorage(store), { models, registry, … }, context), // one StoreStorage per store (h, _reason, context) => h.close(context)); const view = self.follow(...) // Decision needed (D9): name and shape return { … }; @@ -57,8 +57,9 @@ Chord imports nothing of Durable (S13); Durable's conformance suite runs in Dura Durable's four record tables (conversations, entries, tasks, submissions) and its ID namespace. **Layout (D2, owner, 2026-10-27: "the whole purpose of this slice"):** declared tables of the store, written as -ordinary `put` writes: `defineTable`s that `StoreStorage` declares for itself (SQL names `":

"`, e.g. -`"session:harness:entries"`), each row the record's indexed columns (`id`, `conversation_id`, `kind`, `status`, …) plus +ordinary `put` writes: `defineTable`s that `StoreStorage` declares for itself under the type name `pi.durable` (SQL names +`":pi.durable:
"`, e.g. `"session:pi.durable:entries"`; type names are unique per app, so no plugin +table can meet them), each row the record's indexed columns (`id`, `conversation_id`, `kind`, `status`, …) plus the whole record in a `json(...)` column. The outbox, export, import and the applier handle them as every table; Durable's layout changes later through `evolveTable`. The partial head index becomes an index on `(conversation_id, has_head, id)` (index columns are non-`opt`); indexed strings that must stay lossless (`kind`, `requestId`, document @@ -154,12 +155,13 @@ abort request is persisted; a restart limit; the job fingerprint stored with eve - **Where records live:** -> **Decision needed (D5).** (a) **Documents of the object:** each run's record is a keyed persisted document of the -> object (an internal definition per job, key the run id: `{ version, fingerprint, args, key?, phase, outcome?, -> abort?, restarts, trace? }`), its state document a second keyed document; both written in the transactions that -> already change them (start, state change, outcome, finish, block, drop), loaded at the object's first activation -> (all unsettled runs: `running` needs every one). (b) A table per job. Proposal: (a): the records are few, change -> in the same transactions as today, and reuse persisted documents (versions, checkpoints, export). +**Where records live (D5, owner, 2026-10-27):** each unsettled run's record is a keyed persisted document of its +object in the engine's namespace `chord.jobs` (owner the object's owner string, kind the job name, key the run id: +`{ version, fingerprint, args, key?, phase, outcome?, abort?, restarts, trace? }`); Chord sets that namespace, a plugin +never does (its documents are always `chord`), so nothing can collide. Its state document stays an ordinary document +of the run. Both change in the transactions that change them today (start, state change, outcome, finish, block, +drop); `finish` deletes the record and retires the state in its commit (Q9), so only unsettled runs are stored. At the +object's first activation the records are loaded through the engine's key listing and a load per run (no new read). - **Resume at activation:** a run recorded `finishing` resumes `finish` only; a run recorded `running` was cut off by a crash: `rerun` invokes `run` again with the stored args and state, `abandon` records `failed { aborted, reason: From cf4920c8275cba5567f74dda9574bfd9edce6b93 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 11:43:04 +0000 Subject: [PATCH 065/271] docs(chord): slice 6b D6, persisted jobs restart as Durable's tasks --- packages/chord/docs/next/decisions.md | 6 ++++++ packages/chord/docs/next/design.md | 10 +++++++--- packages/chord/docs/next/slice-6b.md | 15 ++++++++------- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index c72a6d3b0..8e4fa1f44 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1669,6 +1669,12 @@ validation and copies the v2 kernel added around Delta. one on the same store is refused (`invalid/args`), as one host per store (rule 7). Rejected on the way: a table per job, a store-wide runs table, `$`-marked internal names (owner: "if our abstractions can't cleanly support this without $ magic, it's a bad abstraction"). +- **Slice 6b D6: persisted jobs restart as Durable's tasks (owner, 2026-10-27: "yes, same rule for chord"):** at + activation a run recorded `running` gets `restarts + 1` (stored) and resumes per its `resume`; a clean close writes + its stopped runs back so the next activation counts nothing (as Durable's `writeAtClose`); a `ctx.state.change` + resets the count; at five in a row the run ends `failed { kind: "restart_limit", restarts }` (a new `JobError`), + `finish` runs, `job_interrupted` is reported. Supersedes design §15's `blocked { restart_limit }` (no parking, as + Durable since #11139). - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index 0e9f439f4..13ab30dc0 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -1402,7 +1402,8 @@ host.serve(ChatStore, ({ conversation }, self) => { // local: no princi ); type JobError = | { readonly kind: "threw"; readonly error: ChordError | PluginError } // run threw (reported) - | { readonly kind: "aborted"; readonly reason: "abort" | "reload" | "restart" }; + | { readonly kind: "aborted"; readonly reason: "abort" | "reload" | "restart" } + | { readonly kind: "restart_limit"; readonly restarts: number }; // persisted runs (slice 6b) // abort(id); a native restart on reload (slice 7); "restart": a persisted run abandoned after a crash (slice 6b), // or a run cut off by its attempt's seal while the object continues (slice 6a) ``` @@ -1441,8 +1442,11 @@ host.serve(ChatStore, ({ conversation }, self) => { // local: no princi `run`, args mapped forwards, state migrated; only work since the last state change repeats, so it must be idempotent) or abandon (`finish` gets `failed { aborted, reason: "restart" }`); wording aligned with Durable (replay-safe, abandon on restart). A recorded outcome resumes only `finish`, never `run`; an abort request is - persisted. A restart limit as Durable's: five crash interruptions, counted and persisted, clean closes not counted; - then `blocked { restart_limit }`, cleared by `abort` or a reload. A persisted state document may use the deferred + persisted. A restart limit as Durable's (owner, 2026-10-27, slice 6b D6): at activation a run recorded `running` was + cut off by a crash: `restarts + 1`, stored, then it resumes (rerun or abandon); a clean close writes its stopped runs + back so the next activation counts nothing; a `ctx.state.change` (progress) resets the count; at five in a row the + run ends with `failed { kind: "restart_limit", restarts }`, `finish` runs, `job_interrupted` is reported (no + parking, as Durable since #11139). A persisted state document may use the deferred persistence option (stored within about a second, flushed before the outcome is recorded). - **Chaining:** `finish` may `tx.start` the next job, atomically with the outcome (separate steps, branches). - **Not jobs:** request/response work (a method), standing behaviour (a routine), input into running work (a flow, diff --git a/packages/chord/docs/next/slice-6b.md b/packages/chord/docs/next/slice-6b.md index 4a87d1f19..8d4e7bb1f 100644 --- a/packages/chord/docs/next/slice-6b.md +++ b/packages/chord/docs/next/slice-6b.md @@ -168,13 +168,14 @@ object's first activation the records are loaded through the engine's key listin "restart" }` and finishes. - **Restart limit** (aligned with Durable's `restarts`): -> **Decision needed (D6).** Durable counts how many times in a row a crash cut a task off without the task changing -> its state since: open counts, a close does not, and at `restartLimit` the task ends visibly. Proposal for Chord: -> the same rule: at activation each run recorded `running` gets `restarts + 1` stored; a `ctx.state.change` resets it -> to 0 (progress); at 5 the run ends `failed { aborted, reason: "restart" }` with a `job_failed` report naming the -> limit (no `blocked { restart_limit }`: Durable dropped parking in #11139, and a blocked run would need a retry API); -> a clean close (`host.close`, a reload's native restart) records nothing: those runs end `aborted` (`stop`/`reload`) -> and do not resume. (Supersedes design §15's "then `blocked { restart_limit }`".) +**The same rule as Durable (D6, owner, 2026-10-27: "yes, same rule for chord"):** at activation a run recorded +`running` was cut off by a crash: its `restarts` rises by one (stored), then it resumes (`rerun` invokes `run` again, +`abandon` finishes with `failed { aborted, reason: "restart" }`); a clean close (`host.close`, a native restart on +reload) writes its stopped runs back so the next activation counts nothing (Durable's `writeAtClose`); a +`ctx.state.change` resets the count (progress); at five in a row the run ends with a new `JobError` `{ kind: +"restart_limit", restarts }`, its `finish` runs, and `job_interrupted` is reported (Durable's `restart_limit`, +`task_interrupted`). No `blocked { restart_limit }`: supersedes design §15's parking, as Durable dropped parking in +#11139. - **Abort** (owner, 2026-10-27: async and stored in 6b): From 97622a8f1e48a97e93e33cd73d0f5625dd53e3b7 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 11:44:13 +0000 Subject: [PATCH 066/271] docs(chord): slice 6b D7, job.abort as Durable's abortTask --- packages/chord/docs/next/decisions.md | 4 ++++ packages/chord/docs/next/design.md | 2 +- packages/chord/docs/next/slice-6b.md | 11 ++++++----- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 8e4fa1f44..e1807c69a 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1675,6 +1675,10 @@ validation and copies the v2 kernel added around Delta. resets the count; at five in a row the run ends `failed { kind: "restart_limit", restarts }` (a new `JobError`), `finish` runs, `job_interrupted` is reported. Supersedes design §15's `blocked { restart_limit }` (no parking, as Durable since #11139). +- **Slice 6b D7: `job.abort(id, ctx): Promise<"marked" | "terminal">` (owner, 2026-10-27: "yes"):** Durable's words + and behaviour (`abortTask`): a running run's abort is stored and its signal fired, resolving `"marked"` once `run` + settled; a finishing run is only marked; a blocked run is dropped (`"marked"`, as Durable's orphan); no unsettled run + with that id resolves `"terminal"` (Chord keeps no settled runs, Q9). Supersedes "job.abort stays until 6b". - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index 13ab30dc0..d98f43e86 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -1371,7 +1371,7 @@ host.serve(ChatStore, ({ conversation }, self) => { // local: no princi slice 6. `self.job(Job, { run, finish? })` is declared in an object's factory (not in a facet) and returns the job handle: `start(args, ctx, { key? })` (a `Run { id, outcome(ctx) }` after the commit), `tx.start(job, args, { key? })` inside `self.tx` (a `Run` handle at once, whose `outcome` rejects if the transaction is discarded), `outcome(id, ctx)` - (unsettled runs only), `abort(id)` (async and stored with persisted jobs, slice 6b) and `running`. + (unsettled runs only), `abort(id, ctx)` (`"marked" | "terminal"`, as Durable's `abortTask`: stored, resolves once `run` settled; slice 6b) and `running`. - **1. Intent:** one transaction on the owner's line. Arguments are checked against `args` (closed schema, like method arguments). Dedupe first: a run with the same `key` that is not settled is returned as it is (the run whose `finish` this transaction is never counts, so `finish` can start the same job again with the same key). Then the diff --git a/packages/chord/docs/next/slice-6b.md b/packages/chord/docs/next/slice-6b.md index 8d4e7bb1f..1800397d1 100644 --- a/packages/chord/docs/next/slice-6b.md +++ b/packages/chord/docs/next/slice-6b.md @@ -179,11 +179,12 @@ reload) writes its stopped runs back so the next activation counts nothing (Dura - **Abort** (owner, 2026-10-27: async and stored in 6b): -> **Decision needed (D7): `job.abort(id, ctx)`'s result.** Cases: a running run (its signal fires, the abort is -> stored so a rerun after a crash does not run again but finishes `aborted`); a finishing run (its outcome is decided, -> nothing to do); a blocked run (dropped); an unknown or settled id. Durable's `abortTask` returns `"marked" | -> "terminal"` and throws for an unknown id. Proposal: `Promise<"marked" | "finishing" | "dropped">`, rejecting `gone` -> for an unknown or settled id (as `outcome(id)` does). +**`job.abort(id, ctx): Promise<"marked" | "terminal">` (D7, owner, 2026-10-27), Durable's words and behaviour:** +a running run gets its abort stored (a rerun after a crash then finishes `aborted` instead of running again) and its +signal fired, and `abort` resolves `"marked"` once `run` settled (as `abortTask` joins the run); a finishing run is only +marked (`"marked"`, as Durable's `completing`); a blocked run is dropped at once (`"marked"`, as Durable's orphan); an id +with no unsettled run resolves `"terminal"` (Chord keeps no settled runs, Q9, so a settled and an unknown id are the +same; Durable throws for an unknown one). - **Deferred state persistence:** From 47dba56f92c64fd41ee6784e896fcf9247890352 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 11:45:47 +0000 Subject: [PATCH 067/271] docs(chord): slice 6b D8, deferred run state per change --- packages/chord/docs/next/decisions.md | 4 ++++ packages/chord/docs/next/design.md | 5 +++-- packages/chord/docs/next/slice-6b.md | 11 +++++------ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index e1807c69a..db1ef860f 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1679,6 +1679,10 @@ validation and copies the v2 kernel added around Delta. and behaviour (`abortTask`): a running run's abort is stored and its signal fired, resolving `"marked"` once `run` settled; a finishing run is only marked; a blocked run is dropped (`"marked"`, as Durable's orphan); no unsettled run with that id resolves `"terminal"` (Chord keeps no settled runs, Q9). Supersedes "job.abort stays until 6b". +- **Slice 6b D8: deferred persistence of a run's state, per change (owner, 2026-10-27: "agree"):** `ctx.state.change(fn, + ctx, { persistence: { kind: "deferred", withinMs? } })`, Durable's `CommitPersistence` shape (default 1,000 ms): + adopted and published at once, stored with the next immediate change, before the outcome is recorded, at close, or + after `withinMs`; a crash loses what is not stored. Only run state documents for now. - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index d98f43e86..5e646fb5b 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -1446,8 +1446,9 @@ host.serve(ChatStore, ({ conversation }, self) => { // local: no princi cut off by a crash: `restarts + 1`, stored, then it resumes (rerun or abandon); a clean close writes its stopped runs back so the next activation counts nothing; a `ctx.state.change` (progress) resets the count; at five in a row the run ends with `failed { kind: "restart_limit", restarts }`, `finish` runs, `job_interrupted` is reported (no - parking, as Durable since #11139). A persisted state document may use the deferred - persistence option (stored within about a second, flushed before the outcome is recorded). + parking, as Durable since #11139). A persisted state document may defer storing a change + (`ctx.state.change(fn, ctx, { persistence: { kind: "deferred", withinMs? } })`, Durable's shape, default 1,000 ms; + stored before the outcome is recorded; slice 6b D8). - **Chaining:** `finish` may `tx.start` the next job, atomically with the outcome (separate steps, branches). - **Not jobs:** request/response work (a method), standing behaviour (a routine), input into running work (a flow, §20: the owner's document plus methods), multi-step durable work with sleeps, waits or ownership trees on a host diff --git a/packages/chord/docs/next/slice-6b.md b/packages/chord/docs/next/slice-6b.md index 1800397d1..b9d6e902e 100644 --- a/packages/chord/docs/next/slice-6b.md +++ b/packages/chord/docs/next/slice-6b.md @@ -188,12 +188,11 @@ same; Durable throws for an unknown one). - **Deferred state persistence:** -> **Decision needed (D8).** Design §15: "a persisted state document may use the deferred persistence option (stored -> within about a second, flushed before the outcome is recorded)", as Durable's `CommitPersistence` `deferred`. -> Options: (a) per change: `ctx.state.change(fn, ctx, { persistence: { kind: "deferred", withinMs? } })`, as Durable's -> commit option; (b) per definition: `defineState(..., { persistDeferred: 1000 })`. Proposal: (a), Durable's shape: a -> deferred change is adopted and published at once, stored with the next immediate change of the document, before -> the outcome, at close, or after `withinMs` (default 1,000 ms); a crash loses it (acceptable for progress output). +**Per change, Durable's shape (D8, owner, 2026-10-27):** `ctx.state.change(fn, ctx, { persistence: { kind: +"deferred", withinMs? } })` (default 1,000 ms, as Durable's `CommitPersistence`): adopted and published at once, +stored with the next immediate change of the document, before the outcome is recorded (so `outcome.state` has it), at +close, or after `withinMs`; a crash loses what is not stored. Only run state documents; other persisted documents get +it when a real case needs it. - **Fingerprint:** the job's fingerprint (args, result and state schemas) stored with every record; a record of the same version with another fingerprint is `blocked { version }` (slice 7's family; the carry-over's "a job From 87eff524f96b69c919482ac46b26bd9333f26e22 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 11:45:03 +0000 Subject: [PATCH 068/271] feat(durable): afterTool runs on every tool task result --- packages/durable/CHANGELOG.md | 1 + packages/durable/README.md | 4 +- packages/durable/docs/spec.md | 43 +- packages/durable/src/harness/tool.ts | 163 ++++- packages/durable/src/harness/types.ts | 13 +- .../durable/test/harness-tool-hooks.test.ts | 626 ++++++++++++++++++ packages/durable/test/harness-tools.test.ts | 4 +- 7 files changed, 817 insertions(+), 37 deletions(-) create mode 100644 packages/durable/test/harness-tool-hooks.test.ts diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index cf45dd551..441b8644c 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -92,6 +92,7 @@ - `FileChange` (`FileSystem.watch()` in `@earendil-works/pi-durable/env`) is tagged by `kind`: `{ kind: "paths", paths }`, `{ kind: "overflow" }`, or `{ kind: "failed", error }` (was `{ paths }`, `{ overflow: true }`, `{ error }`), and `FileWatcher.close(context)` is `stop(context)`. Before: `if ("overflow" in change) rescan(); else if ("paths" in change) reload(change.paths); await watcher.close(ctx);`. After: `if (change.kind === "overflow") rescan(); else if (change.kind === "paths") reload(change.paths); await watcher.stop(ctx);`. - `SessionEnd` (`Session.closed`, `Harness.closed`) is tagged by `kind`: `{ kind: "closed" }` or `{ kind: "failed", error }` (was `reason`). Before: `harness.closed.then((end) => { if (end.reason === "failed") reopen(end.error); })`. After: `harness.closed.then((end) => { if (end.kind === "failed") reopen(end.error); })`. - `ReportFrom`, the second argument of a task's `runtime.report(error, from)`, is tagged by `kind`: `{ kind: "extension", extension? }` or `{ kind: "host", callback }` (was `type`). Before: `runtime.report(error, { type: "host", callback: "env" })`. After: `runtime.report(error, { kind: "host", callback: "env" })`. +- `afterTool` runs on every result a tool task writes, not only on an executed call's: also on the error results the Harness writes itself, for a tool that does not resolve (`tool_unavailable`), invalid arguments (`invalid_arguments`, with `call.arguments` as the model sent them), a block (`blocked`), an interruption (`interrupted`), the restart limit (`restart_limit`), and an abort (`aborted`, `abandoned`); the diagnostic code tells them apart. Such a result is bounded with the tool's output limits after the hooks, never keeps a `structuredOutput`, and applies `control` only when the call completes; a nested call a hook turns into a success gets its bounded output as value unless the tool declares a schema. Once a tool task was cut off more than `settings.restartLimit` times its hooks no longer run, nor in the last attempt the scheduler gives its abort handler, and an abort's hooks delay `Conversation.abort()`. A non-replay-safe tool interrupted that often now ends with `restart_limit` too. Hooks that assumed the tool ran (cast `call.arguments`, read `structuredOutput`) check `result.diagnostics` first ([#11043](https://github.com/earendil-works/pi/issues/11043)) ### Added diff --git a/packages/durable/README.md b/packages/durable/README.md index 9b2882e22..996f484cf 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -292,7 +292,7 @@ const lines = defineTool({ }); ``` -A missing, undeclared, or invalid `structuredOutput` turns a nested call's result into an error result with an `invalid_structured_output` diagnostic; for a call the model made, which never sees it, it is dropped and reported through `onReport`. An error result may omit it, and programs then find its text in a `tool_error` diagnostic; error results the Harness writes itself (blocked, invalid, interrupted, aborted) carry none, and their `diagnostics` say what went wrong. An `afterTool` hook that redacts a schema tool's `output` must redact its `structuredOutput` too. The built-in `bash` returns `{ output, truncated, fullOutputPath?, exitCode }`, where `output` is the retained tail the model sees, read with `api.retainedOutput()`, and answers a nonzero exit with an error result that still carries it. +A missing, undeclared, or invalid `structuredOutput` turns a nested call's result into an error result with an `invalid_structured_output` diagnostic; for a call the model made, which never sees it, it is dropped and reported through `onReport`. An error result may omit it, and programs then find its text in a `tool_error` diagnostic; error results the Harness writes itself (blocked, invalid, interrupted, aborted) carry none, even when an `afterTool` hook returns one, and their `diagnostics` say what went wrong; one a hook turns into a success gives a schema-less tool's program its output. An `afterTool` hook that redacts a schema tool's `output` must redact its `structuredOutput` too. The built-in `bash` returns `{ output, truncated, fullOutputPath?, exitCode }`, where `output` is the retained tail the model sees, read with `api.retainedOutput()`, and answers a nonzero exit with an error result that still carries it. A later extension's tool with the same name replaces an earlier one where both are selected, and `wrapTool()` decorates whichever tool won: @@ -630,7 +630,7 @@ const Guard = defineExtension({ ``` - **Generation:** `beforeRequest` (replace the messages of one request), `afterResponse`, `onYield` (continue the run with another user message), and `afterTools` (runs once a round's tools are done). -- **Tools:** `beforeTool` (block or rewrite arguments) and `afterTool` (replace the result). +- **Tools:** `beforeTool` (block or rewrite arguments) and `afterTool` (replace the result). `afterTool` sees every result a tool task writes, also the error results the Harness writes itself (unavailable, invalid, blocked, interrupted, restart limit, aborted), told apart by their diagnostic code; past `settings.restartLimit` it no longer runs. On an abort it runs inside the abort handler, so a slow hook delays `abort()`. To limit a hook to some conversations, select its extension only there, for example with `configure({ extensions: { add: [Guard] } })`. diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index 70a185a86..7fe639106 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -3217,7 +3217,10 @@ interface ToolHooks { api: HookApi, context: Context, ): HookResult<{ readonly arguments?: JsonObject; readonly block?: string }>; - /** After execution, before the result is committed; replaces the result. */ + /** + * Before the result is committed; replaces the result. Runs on every result the tool task writes, also the error + * results the Harness writes itself (section 8.4), except past the restart limit. + */ afterTool( call: ToolHookCall, result: ToolExecutionResult, @@ -3503,7 +3506,10 @@ what programs get. An error result the Harness writes itself (unavailable, invalid arguments, blocked, interrupted, aborted, abandoned, ended at the restart limit, or a scheduler fault) has no -`structuredOutput`; its diagnostics say what went wrong. A thrown error +`structuredOutput`; its diagnostics say what went wrong. When an `afterTool` +hook turns such a result into a success, programs get its bounded output for a +schema-less tool, or one that did not resolve, and no value for a schema tool; +the hook's own `structuredOutput` is dropped. A thrown error becomes a `tool_error` diagnostic with its message. A nested call whose result breaks this gets an error result with an `invalid_structured_output` diagnostic, without the broken value: a schema tool's then has no `structuredOutput`, a schema-less tool's its output; @@ -4679,12 +4685,15 @@ Below, `{ entryId }` is a model-issued call's result. same handler executes the tool, runs the `afterTool` chain, and commits the result; it builds `api.env` with `HarnessOptions.env` right before executing (section 2.2). A tool that does not resolve, invalid arguments, or a block - commits the corresponding error result instead, without intent. + runs the `afterTool` chain on the corresponding error result and commits it + instead, without intent. - `execute` is reached only by recovery. When both the stored and the current policy, resolved as in `call`, are `safe`, it executes again with the stored arguments, without `beforeTool`, and settles like `call`. Otherwise it commits an `interrupted` error result from the slot's durable partial output, details, and - diagnostics, and ends `failed` with `{ entryId }`. + diagnostics, after the `afterTool` chain, and ends `failed` with `{ entryId }`. + Either way, a task cut off `settings.restartLimit` times ends with a + `restart_limit` result instead (section 5.4), also after the chain. - The result commit bounds the content, appends the diagnostics block (section 7.3), appends one `pi.tool-result` entry with `model: [{ role: "toolResult", toolCallId, toolName, content, details, isError, durationMs, timestamp }]` and @@ -4704,7 +4713,25 @@ Below, `{ entryId }` is a model-issued call's result. `completing` with the result entry already written (section 5.5); the generation resumes only when the tool task is terminal. - The abort handler commits an `aborted` error result from the slot's durable - partial output, details, and diagnostics and ends `aborted` with `{ entryId }`. + partial output, details, and diagnostics, after the `afterTool` chain, and ends + `aborted` with `{ entryId }`. The scheduler faults a task whose abort handler + was cut off `restartLimit` times, without a result (section 5.4), so the last + attempt it runs, at `restartLimit - 1` restarts, skips the chain. +- Every result the task writes passes the `afterTool` chain: the slot is read + before the result commit, since hooks run outside commits, and does not change + while the tool does not run. A result the Harness wrote itself is then bounded + with the tool's output limits (the defaults when it did not resolve), gets a + truncation diagnostic when that dropped content (a slot's truncation + diagnostic stays only while the hooks keep its output), and loses any + `structuredOutput`, except that a nested call the chain turns into a success + gets its bounded output as value, as a schema-less tool's success does + (unless the tool declares a schema, section 7.3); `control` applies only when + the task completes. Once the + task's restarts exceed `settings.restartLimit`, the chain no longer runs, + since it may be what keeps cutting the task off. Results that no tool task + writes never pass it: the generation's `aborted` results for calls a + sequential round never started (section 8.5), a nested call's fallback result + (section 7.3), and a missing result synthesized in context (section 2.1). ### 8.5 Tool rounds @@ -5916,6 +5943,12 @@ These are contracts, not invitations to add defensive machinery: background conversation-owned tasks awaited with `waitForTask()`; a non-background one would be aborted by any cascade that reaches its conversation. +- **Hooks in abort handlers:** a tool task's abort handler runs the `afterTool` + chain on its `aborted` result, and abort invocations cannot be aborted. A + slow hook delays the result, and with it `Conversation.abort()` and + `waitForIdle()`; a hanging one holds them until the host restarts, and the + handler's last attempt before the restart limit runs without hooks (section + 8.4). - **Hook memo names:** hooks share the asking task's memo namespace with the task and other hooks. Prefix memo names. - **Long transactions:** an async commit callback holds the Session mutation diff --git a/packages/durable/src/harness/tool.ts b/packages/durable/src/harness/tool.ts index 946294524..02179a817 100644 --- a/packages/durable/src/harness/tool.ts +++ b/packages/durable/src/harness/tool.ts @@ -42,6 +42,7 @@ import { type NestedToolSummary, PROGRESS_COMMIT, removeNestedSlots, + salvageable, type ToolSlot, toolSlot, } from "./live.ts"; @@ -141,12 +142,11 @@ export const ToolTask = defineTask error, context); + return reject(runtime, task.input, call, undefined, error, context); } const prepared = prepare(tool, call.arguments as JsonObject); const checked = "error" in prepared ? prepared : validate(tool, call, prepared.args); - if ("error" in checked) - return settle(runtime, task.input, call, COMPLETED, () => invalid(checked.error), context); + if ("error" in checked) return reject(runtime, task.input, call, tool, invalid(checked.error), context); let args = checked.args; let block: string | undefined; await runtime.hooks.each("beforeTool", async (hook) => { @@ -162,11 +162,10 @@ export const ToolTask = defineTask blocked, context); + return reject(runtime, task.input, call, tool, blocked, context); } const validated = validate(tool, call, args); - if ("error" in validated) - return settle(runtime, task.input, call, COMPLETED, () => invalid(validated.error), context); + if ("error" in validated) return reject(runtime, task.input, call, tool, invalid(validated.error), context); // Strict JSON: repair may set optional properties to undefined, which the slot and checkpoint cannot hold. const final = copyJson(validated.args, { omitUndefinedProperties: true }) as JsonObject; await runtime.commit(async (tx) => { @@ -185,9 +184,10 @@ export const ToolTask = defineTask { const { arguments: args, replay } = task.state.checkpoint; const call = await readCall(runtime, task.input, context); + // Also before an interruption result: its `afterTool` hooks may be what cut the task off. + if (cutOffTooOften(task, runtime)) return endCutOff(runtime, task, call, context); const tool = await resolveTool(runtime, task.input, call.name, context); if (replay === "safe" && tool?.replay === "safe") { - if (cutOffTooOften(task, runtime)) return endCutOff(runtime, task, call, context); // The rerun reports from scratch; clear what the interrupted attempt published. await runtime.commit(async (tx) => { const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId); @@ -199,7 +199,8 @@ export const ToolTask = defineTask fromSlot(slot, "interrupted", message), context); + // Not cut off too often (checked above), so the hooks run. + await settleFromSlot(runtime, task.input, call, { tool }, ending, "interrupted", message, context); }, }, abort: async (task, runtime, context) => { @@ -211,8 +212,13 @@ export const ToolTask = defineTask fromSlot(slot, code, message), context); + // The scheduler runs an abort handler only while its restarts are below the limit, and faults the task without a + // result when the last attempt is cut off. That attempt skips the hooks, which may be what cuts it off. + const hooks = + (task.restarts ?? 0) < runtime.settings.restartLimit - 1 + ? { tool: await toolForLimits(runtime, task.input, call.name, context) } + : undefined; + await settleFromSlot(runtime, task.input, call, hooks, { status: "aborted" }, code, message, context); }, }); @@ -228,7 +234,68 @@ async function endCutOff( ): Promise { const message = `Tool ${call.name} was interrupted ${cutOffCount(task)}; it is not run again`; const ending = { status: "failed", message, reason: "restart_limit" } as const; - await settle(runtime, task.input, call, ending, (slot) => fromSlot(slot, "restart_limit", message), context); + // Past the limit the hooks may be what keeps cutting the task off (`salvageable`). + const hooks = salvageable(task, runtime) + ? { tool: await toolForLimits(runtime, task.input, call.name, context) } + : undefined; + await settleFromSlot(runtime, task.input, call, hooks, ending, "restart_limit", message, context); +} + +/** + * Settle a result the Harness decided in the `call` phase, without running the tool: the tool did not resolve, the + * arguments are invalid, or `beforeTool` blocked the call. `afterTool` applies as to an executed call's result. + */ +async function reject( + runtime: Runtime, + input: ToolTaskInput, + call: ToolHookCall, + tool: ToolRegistration | undefined, + result: ToolExecutionResult, + context: Context, +): Promise { + const settled = await afterHarnessResult(runtime, input, call, tool, result, undefined, context); + await settle(runtime, input, call, COMPLETED, settled, context); +} + +/** + * Settle an error result built from the call's slot (`fromSlot`): an interruption, the restart limit, or an abort. + * The tool is not running, so the slot read before the commit is still current when it commits. With `hooks`, the + * result passes `afterTool` and is bounded with the limits of `hooks.tool` (the defaults when it is `undefined`); + * without, it is settled as built, when the hooks may be what cuts the task off. + */ +async function settleFromSlot( + runtime: Runtime, + input: ToolTaskInput, + call: ToolHookCall, + hooks: { readonly tool: ToolRegistration | undefined } | undefined, + ending: Ending, + code: string, + message: string, + context: Context, +): Promise { + const live = await runtime.snapshot(LiveDoc, runtime.conversationId, context); + const { result, truncation } = fromSlot( + live === undefined ? undefined : toolSlot(live, runtime.taskId), + code, + message, + ); + const settled = + hooks !== undefined + ? await afterHarnessResult(runtime, input, call, hooks.tool, result, truncation, context) + : truncation === undefined + ? result + : { ...result, diagnostics: [...(result.diagnostics ?? []), truncation] }; + await settle(runtime, input, call, ending, settled, context); +} + +/** The tool whose output limits bound a hooks' result: the called tool, or none when it, or the agent, fails to resolve. */ +async function toolForLimits( + runtime: Runtime, + input: ToolTaskInput, + name: string, + context: Context, +): Promise { + return resolveTool(runtime, input, name, context).catch(() => undefined); } /** The called tool: among the tools the model is offered for a model-issued call, else among those tools may call. */ @@ -242,6 +309,15 @@ async function resolveTool( return (input.kind === "model" ? agent.tools : agent.callable).find((tool) => tool.name === name); } +/** The tool's output limits, the defaults filled in; all defaults when the tool did not resolve. */ +function outputLimits(tool: ToolRegistration | undefined): OutputLimits { + return { + maxBytes: tool?.outputLimits?.maxBytes ?? DEFAULT_MAX_BYTES, + maxLines: tool?.outputLimits?.maxLines ?? DEFAULT_MAX_LINES, + retain: tool?.outputLimits?.retain ?? "head", + }; +} + /** The call: from the input for a nested call, which hooks see with its parent, else from the assistant entry. */ async function readCall(runtime: Runtime, input: ToolTaskInput, context: Context): Promise { if (input.kind === "nested") { @@ -310,11 +386,7 @@ async function run( args: JsonObject, context: Context, ): Promise { - const limits: OutputLimits = { - maxBytes: tool.outputLimits?.maxBytes ?? DEFAULT_MAX_BYTES, - maxLines: tool.outputLimits?.maxLines ?? DEFAULT_MAX_LINES, - retain: tool.outputLimits?.retain ?? "head", - }; + const limits = outputLimits(tool); const reported: Reported = { output: new OutputBuffer(limits), limits, diagnostics: [], details: undefined }; const streams = input.kind === "model" || input.progress !== false; const progress = publishProgress(runtime, reported, streams, context); @@ -473,7 +545,7 @@ async function run( const pending = await progress.stop(); try { const settled = await finalResult(runtime, input, call, tool, result, reported, context); - await settle(runtime, input, call, ending, () => settled, context, durationMs); + await settle(runtime, input, call, ending, settled, context, durationMs); } catch (error) { for (const waiter of pending) waiter.reject(error); throw error; @@ -630,6 +702,41 @@ async function finalResult( return { ...final, output: bounded.content, diagnostics: [...(final.diagnostics ?? []), ...harness] }; } +/** + * A result the Harness wrote itself, without executing the tool, after the `afterTool` chain, normalized like an + * executed call's: the output bounded with the tool's limits (the defaults when it did not resolve), a truncation + * diagnostic when that dropped something, and the hooks' `structuredOutput` dropped, since the tool never produced + * one. A nested call that the hooks turned into a success gets the bounded output as its value, as a schema-less + * tool's success does, unless the tool declares a schema. `usage` is kept and recorded; `control` applies only when + * the task completes. `truncation` describes what the result's own output lost before (a slot's retained output); it + * is kept only while the hooks keep that output. + */ +async function afterHarnessResult( + runtime: Runtime, + input: ToolTaskInput, + call: ToolHookCall, + tool: ToolRegistration | undefined, + result: ToolExecutionResult, + truncation: ToolDiagnostic | undefined, + context: Context, +): Promise { + let final = result; + await runtime.hooks.each("afterTool", async (hook) => { + final = (await hook(call, final, runtime, context)) ?? final; + }); + const harness: ToolDiagnostic[] = []; + if (truncation !== undefined && final.output === result.output) harness.push(truncation); + const limits = outputLimits(tool); + const bounded = boundContent(final.output ?? [], limits); + if (bounded.droppedBytes > 0) harness.push(truncated(bounded, limits.retain)); + const { structuredOutput: _dropped, ...rest } = final; + const value = + input.kind === "nested" && final.isError !== true && tool?.structuredOutputSchema === undefined + ? { structuredOutput: outputValue(bounded.content) as JsonValue } + : {}; + return { ...rest, ...value, output: bounded.content, diagnostics: [...(rest.diagnostics ?? []), ...harness] }; +} + const structuredValidators = new WeakMap>(); /** Why a result's `structuredOutput` breaks the tool's contract, or `undefined` when it keeps it. */ @@ -659,15 +766,14 @@ function structuredOutputError(tool: ToolRegistration, result: ToolExecutionResu * Commit the tool's terminal state and mark its slot done. A model-issued call appends its result entry and ends with * the entry ID; a nested call writes its result to the caller's `NestedCallDoc`, records its usage, and ends with a * small receipt. Nested calls this call left running are aborted first, so they report into their slots before the - * slots below the call leave `pi.live`. `build` receives the slot so interruption and abort can report the durable - * partial output. + * slots below the call leave `pi.live`. `control` applies only when the task completes. */ async function settle( runtime: Runtime, input: ToolTaskInput, call: ToolHookCall, ending: Ending, - build: (slot: Readonly | undefined) => ToolExecutionResult, + result: ToolExecutionResult, context: Context, durationMs?: number, ): Promise { @@ -684,7 +790,6 @@ async function settle( const conversationId = runtime.conversationId; const live = await tx.doc(LiveDoc, conversationId); const slot = toolSlot(live, runtime.taskId); - const result = build(slot); let settled: ToolTaskResult; if (input.kind === "nested") { const nested = nestedResult(runtime.taskId, result, durationMs); @@ -854,22 +959,24 @@ type Ending = const COMPLETED: Ending = { status: "completed" }; -/** An error result from the slot's durable partial output, details, and diagnostics. */ +/** + * An error result from the slot's durable partial output, details, and diagnostics, ending with `code`; and the + * truncation diagnostic of that output, when its bounds dropped some, which the caller adds after `afterTool`. + */ function fromSlot( slot: Readonly | undefined, code: string, message: string, -): ToolExecutionResult { - const diagnostics = [...(slot?.diagnostics ?? [])]; +): { readonly result: ToolExecutionResult; readonly truncation?: ToolDiagnostic } { const droppedBytes = slot?.droppedBytes ?? 0; - if (droppedBytes > 0) diagnostics.push(truncated({ droppedBytes, droppedLines: slot?.droppedLines ?? 0 })); - diagnostics.push(toolDiagnostic(code, message)); - return { + const result: ToolExecutionResult = { output: slot?.output === undefined || slot.output === "" ? [] : [{ type: "text", text: slot.output }], isError: true, ...(slot?.details === undefined ? {} : { details: slot.details }), - diagnostics, + diagnostics: [...(slot?.diagnostics ?? []), toolDiagnostic(code, message)], }; + if (droppedBytes === 0) return { result }; + return { result, truncation: truncated({ droppedBytes, droppedLines: slot?.droppedLines ?? 0 }) }; } /** An error result the Harness writes itself: no content and one `error` diagnostic with `code`. */ diff --git a/packages/durable/src/harness/types.ts b/packages/durable/src/harness/types.ts index d787dd3cd..272af94f4 100644 --- a/packages/durable/src/harness/types.ts +++ b/packages/durable/src/harness/types.ts @@ -917,7 +917,18 @@ export interface ToolHooks { api: HookApi, context: Context, ): HookResult<{ readonly arguments?: JsonObject; readonly block?: string }>; - /** After execution, before the result is committed; replaces the result. */ + /** + * Before the result is committed; replaces the result. Runs on every result the tool task writes: an executed + * call's, and the error results the Harness writes itself, told apart by their diagnostic code: a tool that does not + * resolve (`tool_unavailable`), invalid arguments (`invalid_arguments`, with `call.arguments` as the model sent + * them), a block (`blocked`), and, rebuilt from the call's partial output, an interruption (`interrupted`), the + * restart limit (`restart_limit`), and an abort (`aborted`, `abandoned`). Once the task was cut off more than + * `settings.restartLimit` times the hooks no longer run, since they may be what cuts it off, nor in the last attempt + * the scheduler gives an abort handler (restarts at the limit minus one). A result's `structuredOutput` is dropped + * when the Harness wrote it (a nested call a hook turns into a success gets the bounded output as a schema-less tool's + * does), and its `control` applies only to a call that completes (not to an interrupted, cut-off, or aborted one). + * An abort's hooks run inside the abort handler, which cannot be aborted: a slow hook delays `Conversation.abort()`. + */ afterTool( call: ToolHookCall, result: ToolExecutionResult, diff --git a/packages/durable/test/harness-tool-hooks.test.ts b/packages/durable/test/harness-tool-hooks.test.ts new file mode 100644 index 000000000..aeecb2e1f --- /dev/null +++ b/packages/durable/test/harness-tool-hooks.test.ts @@ -0,0 +1,626 @@ +// `afterTool` runs on every result a tool task writes, also those the Harness decides without executing the tool +// (#11043). +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { JsonValue } from "@earendil-works/chord"; +import { + type AssistantMessage, + fauxAssistantMessage, + fauxText, + fauxToolCall, + type ToolResultMessage, + Type, +} from "@earendil-works/pi-ai"; +import { + defineTool, + type EntryRecord, + type Harness, + LiveDoc, + MemoryStorage, + type NestedToolExecutionResult, + ROOT_CONVERSATION_ID, + type Storage, + type TaskId, + type ToolExecutionResult, + type ToolHookCall, + type ToolRegistration, + ToolResultEntry, + ToolTask, +} from "@earendil-works/pi-durable"; +import { afterEach, describe, expect, it } from "vitest"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { allEntries, type ChatSetup, chatSetup, openChat, waitFor } from "./chat-support.ts"; +import { addHooks, addTool } from "./harness-support.ts"; +import { context } from "./session-support.ts"; +import { aborted, deferred } from "./task-support.ts"; + +const directories = new Set(); +/** Storages of killed Harnesses, never closed by them: closed after each test. */ +const killed = new Set(); + +afterEach(async () => { + for (const storage of killed) await storage.close(context).catch(() => {}); + killed.clear(); + for (const directory of directories) await rm(directory, { recursive: true, force: true }); + directories.clear(); +}); + +async function sqlitePath(): Promise { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-tool-hooks-")); + directories.add(directory); + return join(directory, "session.sqlite"); +} + +/** Open a chat over the SQLite file at `path`; `kill()` simulates a crash: every later storage call hangs. */ +async function open(path: string, setup: ChatSetup) { + const inner = await openNodeSqliteStorage(path); + let dead = false; + const storage = new Proxy(inner, { + get(target, property) { + const value = Reflect.get(target, property, target); + if (typeof value !== "function") return value; + return (...args: unknown[]) => (dead ? new Promise(() => {}) : value.apply(target, args)); + }, + }); + const opened = await openChat(storage, setup); + opened.harness.resume(); + const kill = (): void => { + dead = true; + killed.add(inner); + }; + return { ...opened, kill }; +} + +/** Let deferred progress commits (`persistWithinMs: 0`) reach Storage before a crash. */ +function stored(): Promise { + return new Promise((resolve) => setTimeout(resolve, 50)); +} + +const Parameters = Type.Object({ text: Type.Optional(Type.String()) }); + +function tool( + name: string, + execute: ToolRegistration["execute"], + extra: Partial> = {}, +): ToolRegistration { + return defineTool({ name, description: name, parameters: Parameters, execute, ...extra }); +} + +function calls(...list: readonly (readonly [string, Record, string])[]): AssistantMessage { + return fauxAssistantMessage( + list.map(([name, args, id]) => fauxToolCall(name, args, { id })), + { stopReason: "toolUse" }, + ); +} + +const DONE = fauxAssistantMessage([fauxText("done")]); + +function results(entries: readonly EntryRecord[]): Map { + const messages = entries + .filter((entry) => ToolResultEntry.is(entry)) + .map((entry) => entry.model![0] as ToolResultMessage); + return new Map(messages.map((message) => [message.toolCallId, message])); +} + +function text(message: ToolResultMessage | undefined): string { + return (message?.content ?? []).map((item) => (item.type === "text" ? item.text : "")).join("|"); +} + +function codes(result: { readonly diagnostics?: readonly { readonly code?: string }[] }): (string | undefined)[] { + return (result.diagnostics ?? []).map((diagnostic) => diagnostic.code); +} + +async function diagnosticsOf(entries: readonly EntryRecord[], callId: string): Promise<(string | undefined)[]> { + const entry = entries.find( + (each) => ToolResultEntry.is(each) && (each.model![0] as ToolResultMessage).toolCallId === callId, + ); + return codes((entry?.data ?? {}) as { diagnostics?: { code?: string }[] }); +} + +type Seen = { readonly call: ToolHookCall; readonly result: ToolExecutionResult }; + +async function toolTaskId(harness: Harness, index = 0): Promise { + let id: TaskId | undefined; + await waitFor(async () => { + id = (await harness.snapshot(LiveDoc, ROOT_CONVERSATION_ID, context))?.tools?.[index]?.taskId; + return id !== undefined; + }); + return id!; +} + +describe("afterTool on results the call phase decides", () => { + // #11043 + it("sees and replaces unavailable, invalid, and blocked results, with the raw arguments", async () => { + const setup = chatSetup(); + let ran = 0; + const echo = tool("echo", async () => { + ran++; + return { output: [{ type: "text", text: "ran" }] }; + }); + addTool(setup.registry, echo); + addTool( + setup.registry, + tool("gone", async () => ({ output: [] })), + ); + const seen: Seen[] = []; + addHooks(setup.registry, ToolTask, { + beforeTool: (call) => { + if (call.id === "block") return { block: "not today" }; + if (call.id === "rewrite") return { arguments: { text: { not: "a string" } } }; + return undefined; + }, + afterTool: (call, result) => { + seen.push({ call, result }); + return { ...result, output: [{ type: "text", text: `hooked ${call.id}` }], details: { by: "hook" } }; + }, + }); + let root!: Awaited>["root"]; + setup.faux.setResponses([ + async () => { + // The tool is offered but no longer resolves when its task runs. + await root.configure({ tools: [echo] }, context); + return calls( + ["gone", {}, "unavailable"], + ["echo", { text: { not: "a string" } }, "invalid"], + ["echo", { text: "x" }, "rewrite"], + ["echo", {}, "block"], + ); + }, + DONE, + ]); + const opened = await openChat(new MemoryStorage(), setup); + root = opened.root; + const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(settled.status).toBe("done"); + expect(ran).toBe(0); + const byId = new Map(seen.map((each) => [each.call.id, each])); + expect([...byId.keys()].sort()).toEqual(["block", "invalid", "rewrite", "unavailable"]); + expect(codes(byId.get("unavailable")!.result)).toEqual(["tool_unavailable"]); + expect(codes(byId.get("invalid")!.result)).toEqual(["invalid_arguments"]); + expect(codes(byId.get("rewrite")!.result)).toEqual(["invalid_arguments"]); + expect(codes(byId.get("block")!.result)).toEqual(["blocked"]); + // Hooks see the call as the model made it, also when it is invalid or `beforeTool` rewrote it. + expect(byId.get("invalid")!.call.arguments).toEqual({ text: { not: "a string" } }); + expect(byId.get("rewrite")!.call.arguments).toEqual({ text: "x" }); + const entries = await allEntries(root); + const messages = results(entries); + for (const id of ["unavailable", "invalid", "rewrite", "block"]) { + const message = messages.get(id)!; + expect(message).toMatchObject({ isError: true, details: { by: "hook" } }); + expect(text(message).startsWith(`hooked ${id}|`)).toBe(true); + } + await opened.harness.close(context); + }); + + // #11043 + it("bounds a replacement output with the tool's limits, or the defaults for an unresolved tool", async () => { + const setup = chatSetup(); + addTool( + setup.registry, + tool("small", async () => ({ output: [] }), { outputLimits: { maxBytes: 10 } }), + ); + addHooks(setup.registry, ToolTask, { + beforeTool: () => ({ block: "no" }), + afterTool: (call, result) => ({ + ...result, + output: [{ type: "text", text: call.id === "c2" ? "x".repeat(100) : "y".repeat(60 * 1024) }], + }), + }); + let root!: Awaited>["root"]; + setup.faux.setResponses([ + async () => { + await root.configure({ tools: [] }, context); + return calls(["small", {}, "c1"]); + }, + DONE, + ]); + const opened = await openChat(new MemoryStorage(), setup); + root = opened.root; + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await root.configure({ tools: null }, context); + setup.faux.setResponses([calls(["small", {}, "c2"]), DONE]); + await (await root.submit({ type: "input", content: "again" }, context)).wait(context); + const entries = await allEntries(root); + const messages = results(entries); + // Unresolved: the default 50 KB. + const unresolved = messages.get("c1")!.content[0]; + expect(unresolved?.type === "text" && unresolved.text.length).toBe(50 * 1024); + expect(await diagnosticsOf(entries, "c1")).toEqual(["tool_unavailable", "truncated"]); + const bounded = messages.get("c2")!.content[0]; + expect(bounded?.type === "text" && bounded.text).toBe("x".repeat(10)); + expect(await diagnosticsOf(entries, "c2")).toEqual(["blocked", "truncated"]); + await opened.harness.close(context); + }); + + // #11043 + it("lets a hook's terminate on a blocked call count toward the round", async () => { + const setup = chatSetup(); + addTool( + setup.registry, + tool("echo", async () => ({ output: [], control: { terminate: true } })), + ); + addHooks(setup.registry, ToolTask, { + beforeTool: (call) => (call.id === "c2" ? { block: "no" } : undefined), + afterTool: (_call, result) => ({ ...result, control: { terminate: true } }), + }); + setup.faux.setResponses([calls(["echo", {}, "c1"], ["echo", {}, "c2"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(settled.status).toBe("done"); + // The round terminated: no second request. + expect(setup.faux.state.callCount).toBe(1); + await harness.close(context); + }); + + // #11043 + it("runs on nested invalid and blocked calls; the caller gets diagnostics and details, not output", async () => { + const setup = chatSetup(); + addTool( + setup.registry, + tool("echo", async () => ({ output: [] })), + ); + const nested: NestedToolExecutionResult[] = []; + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + nested.push(await api.executeTool("echo", { text: { not: "a string" } }, callContext)); + nested.push(await api.executeTool("echo", { text: "secret" }, callContext)); + nested.push(await api.executeTool("missing", {}, callContext)); + return {}; + }), + ); + const seen: ToolHookCall[] = []; + addHooks(setup.registry, ToolTask, { + beforeTool: (call) => (call.arguments.text === "secret" ? { block: "no secrets" } : undefined), + afterTool: (call, result) => { + if (call.parent === undefined) return undefined; + seen.push(call); + return { + ...result, + output: [{ type: "text", text: "dropped" }], + structuredOutput: { dropped: true }, + details: { hooked: call.id }, + diagnostics: [...(result.diagnostics ?? []), { severity: "info", code: "hooked", message: "seen" }], + }; + }, + }); + setup.faux.setResponses([calls(["batch", {}, "c1"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(seen.map((call) => call.id)).toEqual(["c1/1", "c1/2", "c1/3"]); + expect(seen.every((call) => call.parent?.callId === "c1")).toBe(true); + expect(nested.map((result) => [result.isError, codes(result)])).toEqual([ + [true, ["invalid_arguments", "hooked"]], + [true, ["blocked", "hooked"]], + [true, ["tool_unavailable", "hooked"]], + ]); + expect(nested.map((result) => result.details)).toEqual([ + { hooked: "c1/1" }, + { hooked: "c1/2" }, + { hooked: "c1/3" }, + ]); + // A Harness error result has no structured output, even when a hook supplies one. + expect(nested[1]!.structuredOutput).toBeUndefined(); + expect(nested[2]!.structuredOutput).toBeUndefined(); + await harness.close(context); + }); + + // #11043 + it("gives a nested call a hook turns into a success its output as value, unless the tool has a schema", async () => { + const setup = chatSetup(); + addTool( + setup.registry, + tool("plain", async () => ({ output: [] })), + ); + addTool( + setup.registry, + tool("typed", async () => ({ output: [], structuredOutput: { ok: true } }), { + structuredOutputSchema: Type.Object({ ok: Type.Boolean() }), + }), + ); + const nested: NestedToolExecutionResult[] = []; + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + nested.push(await api.executeTool("plain", {}, callContext)); + nested.push(await api.executeTool("typed", {}, callContext)); + return {}; + }), + ); + addHooks(setup.registry, ToolTask, { + beforeTool: (call) => (call.parent === undefined ? undefined : { block: "no" }), + afterTool: (call) => + call.parent === undefined + ? undefined + : { output: [{ type: "text", text: `fine ${call.name}` }], structuredOutput: { ok: false } }, + }); + setup.faux.setResponses([calls(["batch", {}, "c1"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(nested.map((result) => [result.isError, result.structuredOutput])).toEqual([ + [false, "fine plain"], + [false, undefined], + ]); + await harness.close(context); + }); +}); + +describe("afterTool on results rebuilt from the slot", () => { + /** A tool that writes output, then blocks until its invocation is cancelled. */ + function blocking(name: string, extra: Partial> = {}) { + const started = deferred(); + const state = { runs: 0 }; + const registration = tool( + name, + async (_args, api, callContext) => { + state.runs++; + api.output(`${name} ${"o".repeat(40)}\n`); + await api.details({ run: state.runs }, callContext); + started.resolve(); + return aborted(callContext.abortSignal!); + }, + extra, + ); + return { registration, started, state }; + } + + // #11043 + it("runs on an interrupted call; a replaced output loses the slot's truncation diagnostic", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.progress = { outputIntervalMs: 0, persistWithinMs: 0 }; + const limits = { outputLimits: { maxBytes: 10, retain: "tail" } } as const; + const work = blocking("work", limits); + const keep = blocking("keep", limits); + addTool(setup.registry, work.registration); + addTool(setup.registry, keep.registration); + const seen: Seen[] = []; + addHooks(setup.registry, ToolTask, { + afterTool: (call, result) => { + seen.push({ call, result }); + return call.name === "work" ? { ...result, output: [{ type: "text", text: "redacted" }] } : undefined; + }, + }); + setup.faux.setResponses([calls(["work", {}, "c1"], ["keep", {}, "c2"]), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await work.started.promise; + await keep.started.promise; + await stored(); + opened.kill(); + + opened = await open(path, setup); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + expect(seen.map((each) => [each.call.id, codes(each.result)]).sort()).toEqual([ + ["c1", ["interrupted"]], + ["c2", ["interrupted"]], + ]); + const entries = await allEntries(opened.root); + const messages = results(entries); + expect(text(messages.get("c1")).startsWith("redacted|")).toBe(true); + expect(messages.get("c1")).toMatchObject({ isError: true, details: { run: 1 } }); + expect(await diagnosticsOf(entries, "c1")).toEqual(["interrupted"]); + // The kept output keeps its truncation diagnostic, after the hooks' result. + expect(await diagnosticsOf(entries, "c2")).toEqual(["interrupted", "truncated"]); + await opened.harness.close(context); + }); + + // #11043 + it("runs on an abort result, but not on the aborted results of calls that never started", async () => { + const setup = chatSetup(); + setup.settings.toolExecution = "sequential"; + const first = blocking("first"); + addTool(setup.registry, first.registration); + addTool( + setup.registry, + tool("second", async () => ({ output: [] })), + ); + const seen: Seen[] = []; + addHooks(setup.registry, ToolTask, { + afterTool: (call, result) => { + seen.push({ call, result }); + return { ...result, details: { hooked: true } }; + }, + }); + setup.faux.setResponses([calls(["first", {}, "c1"], ["second", {}, "c2"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + await first.started.promise; + await root.abort(context); + expect((await submission.wait(context)).status).toBe("unanswered"); + expect(seen.map((each) => [each.call.id, codes(each.result)])).toEqual([["c1", ["aborted"]]]); + const messages = results(await allEntries(root)); + expect(messages.get("c1")).toMatchObject({ isError: true, details: { hooked: true } }); + expect(messages.get("c2")?.details).toBeUndefined(); + await harness.close(context); + }); + + // #11043 + it("reports a throwing hook on an abort result and settles the result as built", async () => { + const setup = chatSetup(); + const work = blocking("work"); + addTool(setup.registry, work.registration); + addHooks(setup.registry, ToolTask, { + afterTool: () => { + throw new Error("hook failed"); + }, + }); + setup.faux.setResponses([calls(["work", {}, "c1"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + await work.started.promise; + await root.abort(context); + expect((await submission.wait(context)).status).toBe("unanswered"); + const entries = await allEntries(root); + expect(text(results(entries).get("c1")).startsWith(`work ${"o".repeat(40)}`)).toBe(true); + expect(await diagnosticsOf(entries, "c1")).toEqual(["aborted"]); + expect(setup.reports.map((error) => (error as Error).message)).toContain("hook failed"); + await harness.close(context); + }); + + // #11043 + it("settles an abort without the hooks in the abort handler's last attempt", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.restartLimit = 2; + setup.settings.progress = { outputIntervalMs: 0, persistWithinMs: 0 }; + const work = blocking("work"); + addTool(setup.registry, work.registration); + const seen: Seen[] = []; + const hanging = deferred(); + addHooks(setup.registry, ToolTask, { + afterTool: async (call, result) => { + seen.push({ call, result }); + hanging.resolve(); + // Never returns: the host dies inside the hook. + return new Promise(() => {}); + }, + }); + setup.faux.setResponses([calls(["work", {}, "c1"]), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await work.started.promise; + const taskId = await toolTaskId(opened.harness); + await stored(); + void opened.root.abort(context).catch(() => {}); + await hanging.promise; + opened.kill(); + + // One restart, at the limit minus one: the attempt the scheduler would fault when cut off runs no hooks. + opened = await open(path, setup); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("unanswered"); + expect(seen.map((each) => codes(each.result))).toEqual([["aborted"]]); + const entries = await allEntries(opened.root); + expect(text(results(entries).get("c1")).startsWith(`work ${"o".repeat(40)}`)).toBe(true); + expect(await diagnosticsOf(entries, "c1")).toEqual(["aborted"]); + expect((await opened.harness.getTask(taskId, context))?.state).toMatchObject({ + status: "terminal", + outcome: { status: "aborted" }, + }); + await opened.harness.close(context); + }); + + // #11043 + it("sees the held progress of an aborted call, and the result stores it", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + // Progress commits are held in memory until the result commit stores them. + setup.settings.progress = { outputIntervalMs: 0, persistWithinMs: 60_000 }; + const work = blocking("work"); + addTool(setup.registry, work.registration); + const seen: Seen[] = []; + addHooks(setup.registry, ToolTask, { + afterTool: (call, result) => { + seen.push({ call, result }); + return undefined; + }, + }); + setup.faux.setResponses([calls(["work", {}, "c1"]), DONE]); + let opened = await open(path, setup); + const submission = await opened.root.submit({ type: "input", content: "go" }, context); + await work.started.promise; + await opened.root.abort(context); + expect((await submission.wait(context)).status).toBe("unanswered"); + expect(seen).toHaveLength(1); + expect(seen[0]!.result).toMatchObject({ details: { run: 1 } }); + expect(text({ content: seen[0]!.result.output } as ToolResultMessage)).toBe(`work ${"o".repeat(40)}\n`); + await opened.harness.close(context); + + opened = await open(path, setup); + const message = results(await allEntries(opened.root)).get("c1"); + expect(message).toMatchObject({ isError: true, details: { run: 1 } }); + expect(text(message).startsWith(`work ${"o".repeat(40)}`)).toBe(true); + await opened.harness.close(context); + }); + + // #11043 + it("runs on an abandoned nested call's result", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const gated = deferred(); + let gate = true; + addTool( + setup.registry, + tool("gated", async () => ({ output: [] })), + ); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + await api.executeTool("gated", {}, callContext); + return {}; + }), + ); + const seen: Seen[] = []; + addHooks(setup.registry, ToolTask, { + // Holds the nested call in its `call` phase, before intent, until the Harness closes. + beforeTool: async (call, _api, hookContext) => { + if (call.name !== "gated" || !gate) return undefined; + gated.resolve(); + return aborted(hookContext.abortSignal!); + }, + afterTool: (call, result) => { + if (call.parent === undefined) return undefined; + seen.push({ call, result }); + return { ...result, details: { hooked: true } }; + }, + }); + setup.faux.setResponses([calls(["batch", {}, "c1"]), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await gated.promise; + await opened.harness.close(context); + gate = false; + + opened = await open(path, setup); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + expect(seen.map((each) => [each.call.id, codes(each.result)])).toEqual([["c1/1", ["abandoned"]]]); + await opened.harness.close(context); + }); + + // #11043 + it("runs on a restart_limit result at the limit, and not once the hooks themselves were cut off", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.restartLimit = 1; + setup.settings.progress = { outputIntervalMs: 0, persistWithinMs: 0 }; + const work = blocking("work"); + addTool(setup.registry, work.registration); + const seen: Seen[] = []; + const hanging = deferred(); + addHooks(setup.registry, ToolTask, { + afterTool: async (call, result) => { + seen.push({ call, result }); + hanging.resolve(); + // Never returns: the host dies inside the hook. + return new Promise(() => {}); + }, + }); + setup.faux.setResponses([calls(["work", {}, "c1"]), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await work.started.promise; + const taskId = await toolTaskId(opened.harness); + await stored(); + opened.kill(); + + // Cut off once, at the limit: the task ends itself, and the hooks run on its result. + opened = await open(path, setup); + await hanging.promise; + expect(seen.map((each) => codes(each.result))).toEqual([["restart_limit"]]); + opened.kill(); + + // Past the limit: ends without the hooks. + opened = await open(path, setup); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + expect(seen).toHaveLength(1); + expect(work.state.runs).toBe(1); + const entries = await allEntries(opened.root); + expect(await diagnosticsOf(entries, "c1")).toEqual(["restart_limit"]); + expect((await opened.harness.getTask(taskId, context))?.state).toMatchObject({ + status: "terminal", + outcome: { status: "failed", error: { detail: { reason: "restart_limit" } } }, + }); + await opened.harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-tools.test.ts b/packages/durable/test/harness-tools.test.ts index 29f6fd3f7..613975d52 100644 --- a/packages/durable/test/harness-tools.test.ts +++ b/packages/durable/test/harness-tools.test.ts @@ -1168,7 +1168,9 @@ describe("tool progress and lifetime", () => { }), ); addHooks(setup.registry, ToolTask, { - afterTool: async (_call, _result, _api, callContext) => { + afterTool: async (_call, result, _api, callContext) => { + // The abort handler's result runs the hooks too (#11043); only the executed call's waits. + if (result.diagnostics?.some((diagnostic) => diagnostic.code === "aborted")) return undefined; inAfterTool.resolve(); await aborted(callContext.abortSignal!); return undefined; From 1c6928b3099a3c3f7d290dd07beda121df22e2a7 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 11:45:19 +0000 Subject: [PATCH 069/271] feat(durable): calls to tools not offered run as tool tasks --- packages/durable/CHANGELOG.md | 1 + packages/durable/docs/spec.md | 48 +++-- packages/durable/src/harness/generation.ts | 64 ++++--- packages/durable/src/harness/live.ts | 7 +- packages/durable/src/harness/tool.ts | 40 +++- .../durable/test/harness-generation.test.ts | 2 +- .../durable/test/harness-live-deltas.test.ts | 7 +- .../durable/test/harness-nested-tools.test.ts | 60 +++++- .../durable/test/harness-tool-hooks.test.ts | 178 ++++++++++++++++++ packages/durable/test/harness-tools.test.ts | 30 ++- .../durable/test/harness-view-run.test.ts | 15 +- 11 files changed, 370 insertions(+), 82 deletions(-) diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 441b8644c..3ccc3662e 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -93,6 +93,7 @@ - `SessionEnd` (`Session.closed`, `Harness.closed`) is tagged by `kind`: `{ kind: "closed" }` or `{ kind: "failed", error }` (was `reason`). Before: `harness.closed.then((end) => { if (end.reason === "failed") reopen(end.error); })`. After: `harness.closed.then((end) => { if (end.kind === "failed") reopen(end.error); })`. - `ReportFrom`, the second argument of a task's `runtime.report(error, from)`, is tagged by `kind`: `{ kind: "extension", extension? }` or `{ kind: "host", callback }` (was `type`). Before: `runtime.report(error, { type: "host", callback: "env" })`. After: `runtime.report(error, { kind: "host", callback: "env" })`. - `afterTool` runs on every result a tool task writes, not only on an executed call's: also on the error results the Harness writes itself, for a tool that does not resolve (`tool_unavailable`), invalid arguments (`invalid_arguments`, with `call.arguments` as the model sent them), a block (`blocked`), an interruption (`interrupted`), the restart limit (`restart_limit`), and an abort (`aborted`, `abandoned`); the diagnostic code tells them apart. Such a result is bounded with the tool's output limits after the hooks, never keeps a `structuredOutput`, and applies `control` only when the call completes; a nested call a hook turns into a success gets its bounded output as value unless the tool declares a schema. Once a tool task was cut off more than `settings.restartLimit` times its hooks no longer run, nor in the last attempt the scheduler gives its abort handler, and an abort's hooks delay `Conversation.abort()`. A non-replay-safe tool interrupted that often now ends with `restart_limit` too. Hooks that assumed the tool ran (cast `call.arguments`, read `structuredOutput`) check `result.diagnostics` first ([#11043](https://github.com/earendil-works/pi/issues/11043)) +- A call to a tool its request did not offer gets a `pi.tool` task like any call, flagged `unoffered: true` in its input and its `pi.live.tools` slot, instead of a result the generation wrote without one: the task resolves nothing (a deferred tool the agent has still never runs), settles `tool_unavailable`, and runs `afterTool`, whose `control` then counts toward the round (one such call no longer prevents `terminate`). Its slot starts `pending` with a `taskId` instead of `done`, and its result entry lands in a later commit than the tool-calling answer. Controls are applied in call order. `pi.tool` is version 3 and `pi.generation` version 2 (a sequential round keeps such calls pending until their turn): an older Harness leaves both blocked (`task_too_old`) instead of resolving the flagged name ([#11043](https://github.com/earendil-works/pi/issues/11043)) ### Added diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index 7fe639106..35330bce3 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -3140,7 +3140,7 @@ await conversation.configure({ extensions: { add: [Venv] } }, context); |---|---|---| | generation `prepare` | agent (tools, sections, model, thinking level), stream options, compaction thresholds for that model | once per request; model, stream options, and offered tools are fixed in the request checkpoint (section 8.3) | | every task phase | extension selection for hooks; `runtime.agent()` | at most once per phase handler, at its first hook dispatch or `agent()` call, from that phase's snapshot and the committed `pi.agent`, off the Session line, with the invocation's context; fixed for the rest of the phase | -| tool task | the implementation of an accepted call | at `call` and `execute`: the name among the phase agent's `tools`, the current selection after the tools filter; no such tool, for example one removed after preparation: `tool_unavailable`. Replay policy on reopen uses the same lookup (section 7.3) | +| tool task | the implementation of an accepted call | at `call` and `execute`: the name among the phase agent's `tools`, the current selection after the tools filter; no such tool, for example one removed after preparation, or a call flagged `unoffered` (section 8.3): `tool_unavailable`. Replay policy on reopen uses the same lookup (section 7.3) | | generation, when a tool round starts | tool execution mode | once per round (section 8.3) | | generation, compaction | retry policy | at each attempt's classification | | generation, on a context overflow | compaction policy for the request's model | at classification | @@ -3703,7 +3703,10 @@ A tool call is accepted only if its request offered the tool. Generation checks this against the tool set replayed from the committed model context through the request's `cutoff`; `beforeRequest` replacements do not change it. It answers a call to a tool it did not offer with the `tool_unavailable` result described in -section 2.2, without a tool task. An agent change after preparation does not +section 2.2, through a tool task whose input is flagged `unoffered`: the task +resolves nothing, so even a tool the agent has (a deferred one not loaded yet) +never runs, and `afterTool` and `control` apply to the result as to any call's. +An agent change after preparation does not affect this check (section 2.2). The tool task resolves the called name among the `tools` of the phase's agent resolution, the tools filter applied. It uses that implementation @@ -4256,7 +4259,7 @@ is exported as an `Entry` token. | `pi.user` | `[UserMessage]`, timestamp from the Harness clock at admission or placement | submissions, `onYield` continuations | | `pi.assistant` | `[AssistantMessage]` with any stop reason | generation | | `pi.system` | `[SystemMessage]` with `content: ""` (section 7.4) | generation preparation | -| `pi.tool-result` | `[ToolResultMessage]` | tool tasks; generation for calls to tools its request did not offer | +| `pi.tool-result` | `[ToolResultMessage]` | tool tasks; generation for calls an aborted round never started | | `pi.reset` | absent, or `[UserMessage]` with the handoff text; always `head: "self"` | `reset()`, generation `tools` phase for `handoff` | | `pi.compaction` | `[UserMessage]` with the wrapped summary; `head` is the first kept entry | compaction tasks (section 8.7) | @@ -4308,11 +4311,10 @@ type LiveState = { tools?: { callId: string; name: string; - /** - * Absent for a call not started yet (sequential round, `pending`) and for a call its request did not offer, - * which starts `done` with the `entry` generation wrote. - */ + /** Absent for a call not started yet (sequential round, `pending`). */ taskId?: TaskId; + /** A call to a tool its request did not offer: its task settles `tool_unavailable` without running anything. */ + unoffered?: true; status: "pending" | "running" | "done"; /** Retained running output and what the bounds dropped. */ output?: string; @@ -4396,8 +4398,8 @@ its 64 KiB scan, the commit writes the retained window as one set. Either way each commit writes at most one window, and the throttle bounds the rate. Generation creates `tools` with one `pending` slot per call in the commit that -appends the tool-calling answer; a call its request did not offer starts `done` -with its result entry. A tool task sets its slot `running` in its intent commit, +appends the tool-calling answer; a call its request did not offer is flagged +`unoffered`. A tool task sets its slot `running` in its intent commit, publishes throttled output and details into it with deferred commits (section 4.1), and in its terminal commit sets `done` and `entry` and removes `output`, `droppedBytes`, `droppedLines`, @@ -4506,7 +4508,11 @@ type GenerationCheckpoint = type GenerationResult = { entryId: EntryId }; ``` -`pi.generation` is version 1 and starts at `{ phase: "prepare", attempt: 1 }`. +`pi.generation` is version 2 and starts at `{ phase: "prepare", attempt: 1 }`; +`migrate` keeps a version 1 record. Version 2 keeps calls to tools not offered +in a sequential round's `pending`, flagged `unoffered` in their slots (section +8.5), which a version 1 Harness would start as tool tasks without the flag, so +it leaves version 2 blocked (`task_too_old`). The run's inputs live in `pi.live.run`, not in the task input. - `prepare` runs section 7.4 with the phase's agent resolution and resolves @@ -4630,11 +4636,12 @@ A tool round starts in the commit that appends the tool-calling answer: 1. The offered tools are replayed with pi-ai `getCurrentTools()` from the committed model context through `cutoff`: `request` already holds it, and - `poll` derives it again. A call to a tool not offered gets its - `tool_unavailable` result entry here, without a task. -2. Every other call gets a `pi.tool` task owned by the generation, with input - `{ assistant, callId }`. The round is sequential when the settings' - `toolExecution`, read as the round starts, is `sequential` or any called + `poll` derives it again. +2. Every call gets a `pi.tool` task owned by the generation, in call order, with + input `{ assistant, callId }`, plus `unoffered: true` (also on its slot) for a + call to a tool not offered, which settles `tool_unavailable` without resolving + the name (section 8.4). The round is sequential when the settings' + `toolExecution`, read as the round starts, is `sequential` or any offered called tool, resolved from the phase's agent as the tool task resolves it (section 7.3), has `executionMode: "sequential"`: then only the first call gets its task now and the rest wait in `pending`. Otherwise every call gets its task at @@ -4664,9 +4671,11 @@ type ToolTaskCheckpoint = type ToolTaskResult = { kind: "model"; entryId: EntryId; control?: ToolControl } | { kind: "nested" }; ``` -`pi.tool` is version 2 and starts at `{ phase: "call" }`; `migrate` gives a -version 1 input `kind: "model"`, and readers treat a terminal result without -`kind` as a model-issued call's. A model-issued call's input stays small +`pi.tool` is version 3 and starts at `{ phase: "call" }`; `migrate` gives a +version 1 input `kind: "model"` and keeps a version 2 input (version 3 added +`unoffered`), and readers treat a terminal result without `kind` as a +model-issued call's. A call flagged `unoffered` resolves no tool: `call` settles +it `tool_unavailable` and the abort handler does too. A model-issued call's input stays small because the terminal record keeps it; the call is read from the assistant entry. A nested call (section 7.3) carries its call, since no entry holds it, and settles differently: its result commit records the result's `usage` @@ -4767,7 +4776,8 @@ and then commits once: It completes with `{ entryId }` of the tool-calling answer. Its abort handler runs only after its tool tasks are terminal (section 5.5). For every call still -in `pending`, which never started, it appends an `aborted` error result; a +in `pending`, which never started, it appends an `aborted` error result +(`tool_unavailable` for an `unoffered` one), without `afterTool`; a started call whose task faulted or was orphaned keeps its `done` slot without an entry (section 8.2). It then settles the inputs `unanswered` with `aborted`, removes `run`, `generation`, and `tools`, and ends diff --git a/packages/durable/src/harness/generation.ts b/packages/durable/src/harness/generation.ts index 462f73e23..56d130082 100644 --- a/packages/durable/src/harness/generation.ts +++ b/packages/durable/src/harness/generation.ts @@ -143,8 +143,15 @@ const DEFAULT_POLL_AFTER_MS = 5000; */ export const GenerationTask = defineTask({ name: "pi.generation", - version: 1, + version: 2, initial: () => ({ phase: "prepare", attempt: 1 }), + // Version 2 leaves calls to tools not offered in a sequential round's `pending`, flagged `unoffered` in their slots. + // A version 1 Harness would start their tool tasks unflagged, resolving the name, so it leaves version 2 blocked + // instead. A version 1 checkpoint has no such calls and is kept as it is. + migrate: (input, checkpoint) => ({ + input: input as GenerationInput, + checkpoint: checkpoint as GenerationCheckpoint, + }), phases: { /** * Render the system prompt and tool loadout and append the positional `pi.system` entries they need, then move to @@ -307,8 +314,8 @@ export const GenerationTask = defineTask => { const live = await tx.doc(LiveDoc, runtime.conversationId); - const taskId = await createToolTask(tx, runtime, assistant, next); const slot = live.tools?.find((slot) => slot.callId === next && slot.taskId === undefined); + const taskId = await createToolTask(tx, runtime, assistant, next, slot?.unoffered); if (slot !== undefined) slot.taskId = taskId; const checkpoint: GenerationCheckpoint = { phase: "tools", @@ -325,7 +332,8 @@ export const GenerationTask = defineTask slot.callId === call.id)?.unoffered === true; + const result = unoffered + ? harnessError("tool_unavailable", `Tool ${call.name} is not available`) + : harnessError("aborted", `Tool ${call.name} was aborted`); await appendToolResult(tx, conversationId, call, result, { timestamp: runtime.now() }); } endRun(tx, live, runtime.taskId, { status: "unanswered", reason: "aborted" }); @@ -355,11 +367,17 @@ async function readCalls( return callIds.flatMap((id) => calls.filter((call) => call.id === id).slice(0, 1)); } -/** A tool task for call `callId`, owned by the generation. */ -function createToolTask(tx: Tx, runtime: Runtime, assistant: EntryId, callId: string): Promise> { +/** A tool task for call `callId`, owned by the generation; `unoffered` for a call to a tool its request did not offer. */ +function createToolTask( + tx: Tx, + runtime: Runtime, + assistant: EntryId, + callId: string, + unoffered: true | undefined, +): Promise> { return tx.createTask( ToolTask, - { kind: "model", assistant, callId }, + { kind: "model", assistant, callId, ...(unoffered === undefined ? {} : { unoffered }) }, { ownership: { kind: "task", taskId: runtime.taskId } }, ); } @@ -776,9 +794,10 @@ async function answer(runtime: Runtime, message: AssistantMessage, context: Cont } /** - * Append the tool-calling answer and start its tool round in one commit (spec §8.3). A call to a tool the request did - * not offer gets its `tool_unavailable` result here; every other call gets a tool task owned by the generation, only the - * first one now when the round is sequential. The generation then waits for them in its `tools` phase, keeping the run. + * Append the tool-calling answer and start its tool round in one commit (spec §8.3). Every call gets a tool task owned by + * the generation, in call order, only the first one now when the round is sequential; a call to a tool the request did + * not offer is flagged, and its task settles `tool_unavailable`. The generation then waits for them in its `tools` + * phase, keeping the run. */ async function startToolRound( runtime: Runtime, @@ -806,20 +825,15 @@ async function startToolRound( const tools: TaskId[] = []; const pending: string[] = []; for (const call of calls) { - if (!offered.has(call.name)) { - const unavailable = harnessError("tool_unavailable", `Tool ${call.name} is not available`); - const result = await appendToolResult(tx, conversationId, call, unavailable, { timestamp: runtime.now() }); - slots.push({ callId: call.id, name: call.name, status: "done", entry: result.id }); - continue; - } + const unoffered = offered.has(call.name) ? {} : ({ unoffered: true } as const); if (sequential && tools.length > 0) { pending.push(call.id); - slots.push({ callId: call.id, name: call.name, status: "pending" }); + slots.push({ callId: call.id, name: call.name, status: "pending", ...unoffered }); continue; } - const taskId = await createToolTask(tx, runtime, entry.id, call.id); + const taskId = await createToolTask(tx, runtime, entry.id, call.id, unoffered.unoffered); tools.push(taskId); - slots.push({ callId: call.id, name: call.name, taskId, status: "pending" }); + slots.push({ callId: call.id, name: call.name, taskId, status: "pending", ...unoffered }); } delete live.generation; live.tools = slots; @@ -853,13 +867,13 @@ async function finishToolRound( const slots = (await runtime.snapshot(LiveDoc, conversationId, context))?.tools ?? []; const results = slots.flatMap((slot) => (slot.entry === undefined ? [] : [slot.entry])); await runtime.hooks.each("afterTools", (hook) => hook(assistant, results, runtime, context)); - // Every call of the round, including those answered without a task, must ask to terminate. - const terminate = - slots.length > 0 && - slots.every((slot) => slot.taskId !== undefined && controls.get(slot.taskId)?.terminate === true); - const added = [...controls.values()].flatMap((control) => control?.addTools ?? []); + // In call order, the slots' order; a slot without a task (faulted or orphaned before it got one) has no control. + const ordered = slots.map((slot) => (slot.taskId === undefined ? undefined : controls.get(slot.taskId))); + // Every call of the round must ask to terminate. + const terminate = slots.length > 0 && ordered.every((control) => control?.terminate === true); + const added = ordered.flatMap((control) => control?.addTools ?? []); // The last handoff in call order wins. - const handoff = [...controls.values()].findLast((control) => control?.handoff !== undefined)?.handoff; + const handoff = ordered.findLast((control) => control?.handoff !== undefined)?.handoff; await runtime.commit(async (tx): Promise => { const boundary = await prepareBoundary(tx, conversationId, runtime.settings); if (added.length > 0) await addTools(tx, conversationId, added); diff --git a/packages/durable/src/harness/live.ts b/packages/durable/src/harness/live.ts index 837225a04..a6669e3cb 100644 --- a/packages/durable/src/harness/live.ts +++ b/packages/durable/src/harness/live.ts @@ -33,11 +33,10 @@ type SlotProgress = { /** Presentation of one tool call of the current round. */ export type ToolSlot = SlotProgress & { - /** - * Absent for a call not started yet (sequential round) and for a call its request did not offer, which starts `done` - * with the `entry` generation wrote. - */ + /** Absent for a call not started yet (sequential round). */ taskId?: TaskId; + /** A call to a tool its request did not offer: its task settles `tool_unavailable` without running anything. */ + unoffered?: true; /** Result entry once done; absent when the tool task faulted or was orphaned. */ entry?: EntryId; }; diff --git a/packages/durable/src/harness/tool.ts b/packages/durable/src/harness/tool.ts index 02179a817..bdc806aa9 100644 --- a/packages/durable/src/harness/tool.ts +++ b/packages/durable/src/harness/tool.ts @@ -64,7 +64,13 @@ import { optionalExtension } from "./util.ts"; * whose call the input carries because no entry holds it. */ export type ToolTaskInput = - | { kind: "model"; assistant: EntryId; callId: string } + | { + kind: "model"; + assistant: EntryId; + callId: string; + /** A call to a tool its request did not offer: settles `tool_unavailable` without resolving the name. */ + unoffered?: true; + } | { kind: "nested"; parent: TaskId; @@ -128,18 +134,21 @@ type Content = (TextContent | ImageContent)[]; */ export const ToolTask = defineTask({ name: "pi.tool", - version: 2, + version: 3, initial: () => ({ phase: "call" }), - // Version 1 had only model-issued calls. - migrate: (input, checkpoint) => ({ - input: { kind: "model", ...(input as { assistant: EntryId; callId: string }) }, + // Version 1 had only model-issued calls, without `kind`; version 3 added `unoffered`, absent in version 2 inputs. + migrate: (input, checkpoint, fromVersion) => ({ + input: + fromVersion === 1 + ? { kind: "model", ...(input as { assistant: EntryId; callId: string }) } + : (input as ToolTaskInput), checkpoint: checkpoint as ToolTaskCheckpoint, }), phases: { call: async (task, runtime, context) => { const call = await readCall(runtime, task.input, context); if (cutOffTooOften(task, runtime)) return endCutOff(runtime, task, call, context); - const tool = await resolveTool(runtime, task.input, call.name, context); + const tool = isUnoffered(task.input) ? undefined : await resolveTool(runtime, task.input, call.name, context); if (tool === undefined) { const error = harnessError("tool_unavailable", `Tool ${call.name} is not available`); return reject(runtime, task.input, call, undefined, error, context); @@ -205,9 +214,11 @@ export const ToolTask = defineTask { const call = await readCall(runtime, task.input, context); - // Abandoned after a restart (`TaskOptions.abandonOnRestart`): say whether the call may have run. - const [code, message] = - task.abortReason !== "restart" + // Abandoned after a restart (`TaskOptions.abandonOnRestart`): say whether the call may have run. An unoffered call + // could not run in any case, and its result says so. + const [code, message] = isUnoffered(task.input) + ? ["tool_unavailable", `Tool ${call.name} is not available`] + : task.abortReason !== "restart" ? ["aborted", `Tool ${call.name} was aborted`] : task.state.checkpoint.phase === "execute" ? ["interrupted", `Tool ${call.name} was interrupted by a restart and may have partially run`] @@ -288,16 +299,25 @@ async function settleFromSlot( await settle(runtime, input, call, ending, settled, context); } -/** The tool whose output limits bound a hooks' result: the called tool, or none when it, or the agent, fails to resolve. */ +/** + * The tool whose output limits bound a hooks' result: the called tool, or none when it, or the agent, fails to resolve, + * and for an unoffered call, which never resolves its name. + */ async function toolForLimits( runtime: Runtime, input: ToolTaskInput, name: string, context: Context, ): Promise { + if (isUnoffered(input)) return undefined; return resolveTool(runtime, input, name, context).catch(() => undefined); } +/** A model-issued call to a tool its request did not offer (spec §8.3). */ +function isUnoffered(input: ToolTaskInput): boolean { + return input.kind === "model" && input.unoffered === true; +} + /** The called tool: among the tools the model is offered for a model-issued call, else among those tools may call. */ async function resolveTool( runtime: Runtime, diff --git a/packages/durable/test/harness-generation.test.ts b/packages/durable/test/harness-generation.test.ts index fa6b0d21e..4fd7697f0 100644 --- a/packages/durable/test/harness-generation.test.ts +++ b/packages/durable/test/harness-generation.test.ts @@ -713,7 +713,7 @@ describe("generation", () => { entry: entry.id, }); const taskId = await tx.createTask( - { definition: { ...GenerationTask.definition, version: 2 } }, + { definition: { ...GenerationTask.definition, version: GenerationTask.definition.version + 1 } }, {}, { ownership: { kind: "conversation" }, conversationId: root.id }, ); diff --git a/packages/durable/test/harness-live-deltas.test.ts b/packages/durable/test/harness-live-deltas.test.ts index 0802a324e..737b577a8 100644 --- a/packages/durable/test/harness-live-deltas.test.ts +++ b/packages/durable/test/harness-live-deltas.test.ts @@ -347,7 +347,7 @@ describe("pi.live deltas", () => { await harness.close(context); }); - it("starts calls the request did not offer as done and marks a faulted tool's slot done without an entry", async () => { + it("starts calls the request did not offer as flagged tasks and marks a faulted tool's slot done without an entry", async () => { const setup = chatSetup(); addTool( setup.registry, @@ -376,14 +376,13 @@ describe("pi.live deltas", () => { end: () => {}, }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); - const entries = await root.entries({}, 10, undefined, context); - const ghostResult = entries.items.find((entry) => entry.kind === "pi.tool-result")!.id; const handover = commits.find((ops) => ops.some((op) => op[0] === "s" && op[1][0] === "tools"))!; + // #11043: an unoffered call gets a task too, flagged on its slot. expect(handover).toContainEqual([ "s", ["tools"], [ - { callId: "g", name: "ghost", status: "done", entry: ghostResult }, + { callId: "g", name: "ghost", taskId: expect.any(Number), status: "pending", unoffered: true }, { callId: "b", name: "bad", taskId: expect.any(Number), status: "pending" }, ], ]); diff --git a/packages/durable/test/harness-nested-tools.test.ts b/packages/durable/test/harness-nested-tools.test.ts index de61c3ec4..e1442b25f 100644 --- a/packages/durable/test/harness-nested-tools.test.ts +++ b/packages/durable/test/harness-nested-tools.test.ts @@ -1233,14 +1233,34 @@ describe("nested tool calls", () => { checkpoint: { phase: "call" }, }); }); + + // #11043 + it("keeps version 2 tool task input as it is", () => { + const migrate = ToolTask.definition.migrate!; + const model = { kind: "model", assistant: 3, callId: "c1" }; + const nested = { + kind: "nested", + parent: 7, + parentCallId: "c1", + key: "k", + call: { type: "toolCall", id: "c1/k", name: "echo", arguments: {} }, + }; + expect(migrate(model, { phase: "call" }, 2).input).toEqual(model); + expect(migrate(nested, { phase: "call" }, 2).input).toEqual(nested); + }); }); -describe("tool task version 1 records", () => { +describe("tool task version 1 and 2 records", () => { /** - * Store a live tool task for call `c1` of an assistant entry, close, and rewrite its record as version 1 stored it: - * `{ assistant, callId }` input at version 1, with `checkpoint`. Returns the storage path and the task ID. + * Store a live tool task for call `c1` of an assistant entry, close, and rewrite its record as `version` stored it, + * with `checkpoint`: version 1 had `{ assistant, callId }` input, version 2 added `kind`. Returns the storage path + * and the task ID. */ - async function storeVersion1(setup: ChatSetup, checkpoint: JsonValue): Promise<{ path: string; id: TaskId }> { + async function storeVersion( + setup: ChatSetup, + checkpoint: JsonValue, + version: 1 | 2 = 1, + ): Promise<{ path: string; id: TaskId }> { const directory = await mkdtemp(join(tmpdir(), "pi-durable-tool-v1-")); directories.add(directory); const path = join(directory, "session.sqlite"); @@ -1257,10 +1277,11 @@ describe("tool task version 1 records", () => { await harness.close(context); const storage = await openNodeSqliteStorage(path); const record = (await storage.task(id, context))!; - const { kind: _kind, ...input } = record.input as Extract; + const { kind: _kind, ...version1 } = record.input as Extract; + const input = version === 1 ? version1 : record.input; const state = { ...record.state, checkpoint } as typeof record.state; await storage.commit( - [{ type: "task", value: { ...record, version: 1, input, state } as typeof record }], + [{ type: "task", value: { ...record, version, input, state } as typeof record }], () => {}, context, ); @@ -1272,12 +1293,12 @@ describe("tool task version 1 records", () => { const setup = chatSetup(); const echo = echoTool(); addTool(setup.registry, echo.registration); - const { path, id } = await storeVersion1(setup, { phase: "call" }); + const { path, id } = await storeVersion(setup, { phase: "call" }); const { harness, root } = await openChat(await openNodeSqliteStorage(path), setup); expect((await harness.getTask(id, context))?.version).toBe(1); harness.resume(); const settled = await harness.waitForTask(id, context); - expect(settled.version).toBe(2); + expect(settled.version).toBe(3); expect(settled.input).toMatchObject({ kind: "model", callId: "c1" }); expect(settled.state.outcome).toMatchObject({ status: "completed", result: { kind: "model" } }); expect(echo.state.runs).toBe(1); @@ -1291,11 +1312,11 @@ describe("tool task version 1 records", () => { const echo = echoTool(); addTool(setup.registry, echo.registration); const intent = { phase: "execute", arguments: { text: "old" }, replay: "unsafe" }; - const { path, id } = await storeVersion1(setup, intent); + const { path, id } = await storeVersion(setup, intent); const { harness, root } = await openChat(await openNodeSqliteStorage(path), setup); harness.resume(); const settled = await harness.waitForTask(id, context); - expect(settled.version).toBe(2); + expect(settled.version).toBe(3); expect(settled.state.outcome).toMatchObject({ status: "failed", result: { kind: "model" } }); expect(echo.state.runs).toBe(0); const [result] = results(await allEntries(root)); @@ -1303,4 +1324,23 @@ describe("tool task version 1 records", () => { expect(text(result)).toContain("Tool echo was interrupted"); await harness.close(context); }); + + // #11043 + it("reserves a stored version 2 task under version 3 and runs it as a model-issued call", async () => { + const setup = chatSetup(); + const echo = echoTool(); + addTool(setup.registry, echo.registration); + const { path, id } = await storeVersion(setup, { phase: "call" }, 2); + const { harness, root } = await openChat(await openNodeSqliteStorage(path), setup); + expect((await harness.getTask(id, context))?.version).toBe(2); + harness.resume(); + const settled = await harness.waitForTask(id, context); + expect(settled.version).toBe(3); + expect(settled.input).toEqual({ kind: "model", assistant: expect.any(Number), callId: "c1" }); + expect(settled.state.outcome).toMatchObject({ status: "completed", result: { kind: "model" } }); + expect(echo.state.runs).toBe(1); + const [result] = results(await allEntries(root)); + expect([result!.toolCallId, text(result)]).toEqual(["c1", "echo old"]); + await harness.close(context); + }); }); diff --git a/packages/durable/test/harness-tool-hooks.test.ts b/packages/durable/test/harness-tool-hooks.test.ts index aeecb2e1f..1847c768b 100644 --- a/packages/durable/test/harness-tool-hooks.test.ts +++ b/packages/durable/test/harness-tool-hooks.test.ts @@ -19,6 +19,7 @@ import { LiveDoc, MemoryStorage, type NestedToolExecutionResult, + ResetEntry, ROOT_CONVERSATION_ID, type Storage, type TaskId, @@ -624,3 +625,180 @@ describe("afterTool on results rebuilt from the slot", () => { await opened.harness.close(context); }); }); + +describe("calls to tools the request did not offer", () => { + // #11043 + it("run afterTool inside their own tool task", async () => { + const setup = chatSetup(); + const seen: { readonly id: string; readonly taskId: TaskId }[] = []; + addHooks(setup.registry, ToolTask, { + afterTool: (call, result, api) => { + seen.push({ id: call.id, taskId: api.taskId }); + return { ...result, output: [{ type: "text", text: "try another tool" }] }; + }, + }); + setup.faux.setResponses([calls(["ghost", {}, "c1"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(seen.map((each) => each.id)).toEqual(["c1"]); + expect(await harness.getTask(seen[0]!.taskId, context)).toMatchObject({ + kind: "pi.tool", + input: { kind: "model", callId: "c1", unoffered: true }, + }); + const entries = await allEntries(root); + expect(text(results(entries).get("c1")).startsWith("try another tool|")).toBe(true); + expect(await diagnosticsOf(entries, "c1")).toEqual(["tool_unavailable"]); + await harness.close(context); + }); + + // #11043 + it("count toward a round's terminate", async () => { + const setup = chatSetup(); + addTool( + setup.registry, + tool("echo", async () => ({ output: [] })), + ); + addHooks(setup.registry, ToolTask, { + afterTool: (_call, result) => ({ ...result, control: { terminate: true } }), + }); + setup.faux.setResponses([calls(["echo", {}, "c1"], ["ghost", {}, "c2"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(settled.status).toBe("done"); + expect(setup.faux.state.callCount).toBe(1); + await harness.close(context); + }); + + // #11043 + it("take their turn in a sequential round, and the last handoff in call order wins", async () => { + const setup = chatSetup(); + setup.settings.toolExecution = "sequential"; + const order: string[] = []; + for (const name of ["a", "b"]) { + addTool( + setup.registry, + tool(name, async () => { + order.push(name); + return { output: [] }; + }), + ); + } + addHooks(setup.registry, ToolTask, { + afterTool: (call, result) => { + order.push(`hook ${call.id}`); + return call.id === "c1" ? undefined : { ...result, control: { handoff: `from ${call.id}` } }; + }, + }); + setup.faux.setResponses([calls(["a", {}, "c1"], ["ghost", {}, "c2"], ["b", {}, "c3"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(order).toEqual(["a", "hook c1", "hook c2", "b", "hook c3"]); + const resets = (await allEntries(root)).filter((entry) => ResetEntry.is(entry)); + expect(resets.map((entry) => (entry.model?.[0] as { content: string }).content)).toEqual(["from c3"]); + await harness.close(context); + }); + + // #11043 + it("each get a tool task in a parallel round of only such calls", async () => { + const setup = chatSetup(); + const seen: string[] = []; + addHooks(setup.registry, ToolTask, { + afterTool: (call) => { + seen.push(call.id); + return undefined; + }, + }); + setup.faux.setResponses([calls(["ghost", {}, "c1"], ["spirit", {}, "c2"], ["ghost", {}, "c3"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(settled.status).toBe("done"); + expect(seen.sort()).toEqual(["c1", "c2", "c3"]); + const entries = await allEntries(root); + for (const id of ["c1", "c2", "c3"]) expect(await diagnosticsOf(entries, id)).toEqual(["tool_unavailable"]); + // The round's results go to the next request. + expect(setup.faux.state.callCount).toBe(2); + await harness.close(context); + }); + + // #11043 + it("keep their flag in a version 2 generation's sequential round, also one migrated from version 1", async () => { + for (const version of [2, 1]) { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.toolExecution = "sequential"; + const started = deferred(); + let runs = 0; + const a = tool( + "a", + async (_args, _api, callContext) => { + runs++; + if (runs > 1) return { output: [] }; + started.resolve(); + return aborted(callContext.abortSignal!); + }, + { replay: "safe" }, + ); + addTool(setup.registry, a); + addTool( + setup.registry, + tool("ghost", async () => ({ output: [{ type: "text", text: "ran" }] })), + ); + setup.faux.setResponses([calls(["a", {}, "c1"], ["ghost", {}, "c2"]), DONE]); + let opened = await open(path, setup); + let root = opened.root; + // `ghost` is registered, but not offered to this request. + await root.configure({ tools: [a] }, context); + const id = (await root.submit({ type: "input", content: "go" }, context)).id; + await started.promise; + const live = (await opened.harness.snapshot(LiveDoc, ROOT_CONVERSATION_ID, context))!; + const generation = live.run!.taskId; + expect(live.tools?.map((slot) => [slot.callId, slot.unoffered])).toEqual([ + ["c1", undefined], + ["c2", true], + ]); + expect((await opened.harness.getTask(generation, context))?.version).toBe(2); + // The tools are offered when the round goes on: only the flag keeps `ghost` from running. + await root.configure({ tools: null }, context); + await opened.harness.close(context); + if (version === 1) { + const storage = await openNodeSqliteStorage(path); + const record = (await storage.task(generation, context))!; + await storage.commit([{ type: "task", value: { ...record, version: 1 } }], () => {}, context); + await storage.close(context); + } + + opened = await open(path, setup); + root = opened.root; + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + expect((await opened.harness.getTask(generation, context))?.version).toBe(2); + const entries = await allEntries(root); + expect(await diagnosticsOf(entries, "c2")).toEqual(["tool_unavailable"]); + expect(text(results(entries).get("c2"))).not.toContain("ran"); + await opened.harness.close(context); + } + }); + + // #11043 + it("settle unavailable, not aborted, when their sequential round is aborted before they start", async () => { + const setup = chatSetup(); + setup.settings.toolExecution = "sequential"; + const started = deferred(); + addTool( + setup.registry, + tool("wait", async (_args, _api, callContext) => { + started.resolve(); + return aborted(callContext.abortSignal!); + }), + ); + setup.faux.setResponses([calls(["wait", {}, "c1"], ["ghost", {}, "c2"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + await started.promise; + await root.abort(context); + expect((await submission.wait(context)).status).toBe("unanswered"); + const entries = await allEntries(root); + expect(await diagnosticsOf(entries, "c1")).toEqual(["aborted"]); + expect(await diagnosticsOf(entries, "c2")).toEqual(["tool_unavailable"]); + await harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-tools.test.ts b/packages/durable/test/harness-tools.test.ts index 613975d52..6849861b8 100644 --- a/packages/durable/test/harness-tools.test.ts +++ b/packages/durable/test/harness-tools.test.ts @@ -138,7 +138,7 @@ describe("tool round", () => { await harness.close(context); }); - it("answers calls to tools the request did not offer without a task", async () => { + it("answers calls to tools the request did not offer through a flagged tool task", async () => { const setup = chatSetup(); addTool( setup.registry, @@ -157,8 +157,34 @@ describe("tool round", () => { expect(ghostEntry?.data).toEqual({ diagnostics: [{ severity: "error", code: "tool_unavailable", message: "Tool ghost is not available" }], }); + // #11043: the unoffered call has its own task, so `afterTool` and `control` apply to it as to any call. const tasks = await harness.commit((tx) => tx.scanTasks({ conversationId: root.id }, 20), context); - expect(tasks.items.filter((task) => task.kind === "pi.tool")).toHaveLength(1); + const inputs = tasks.items.filter((task) => task.kind === "pi.tool").map((task) => task.input); + expect(inputs).toHaveLength(2); + expect(inputs).toContainEqual(expect.objectContaining({ callId: "c1", unoffered: true })); + expect(inputs).toContainEqual(expect.not.objectContaining({ unoffered: true })); + await harness.close(context); + }); + + // #11043 + it("never runs a tool the request did not offer, even when the agent has it", async () => { + const setup = chatSetup(); + let ran = 0; + addTool( + setup.registry, + tool( + "hidden", + async () => { + ran++; + return { output: [] }; + }, + { deferred: true }, + ), + ); + const { harness, entries, status } = await run(setup, [calls(["hidden", {}, "c1"]), DONE]); + expect(status).toBe("done"); + expect(ran).toBe(0); + expect(resultText(results(entries)[0]!)).toBe("\n[error] Tool hidden is not available\n"); await harness.close(context); }); diff --git a/packages/durable/test/harness-view-run.test.ts b/packages/durable/test/harness-view-run.test.ts index 62c70b850..a4445ea91 100644 --- a/packages/durable/test/harness-view-run.test.ts +++ b/packages/durable/test/harness-view-run.test.ts @@ -327,7 +327,7 @@ describe("a run as the conversation view shows it", () => { await harness.close(context); }); - it("ends a call that never runs with its result at once, and a tool aborted with its generation", async () => { + it("ends a call that never runs through its own task, and a tool aborted with its generation", async () => { const setup = chatSetup(); addTool( setup.registry, @@ -352,13 +352,14 @@ describe("a run as the conversation view shows it", () => { await submission.wait(context); history.detach(); const states = history.live(); - // The call not offered is listed done, with its result entry, in the commit of its calling answer. + // #11043: the call not offered is listed pending with its task in the commit of its calling answer, and its task + // ends it done with its result entry. const listed = slotsOf(states, "c1").findIndex((slot) => slot !== undefined); - const ghost = slotsOf(states, "c1")[listed]!; - expect(ghost.status).toBe("done"); - const listing = history.views[listed]!.entries; - expect(listing.map((entry) => entry.kind).slice(-2)).toEqual(["pi.assistant", "pi.tool-result"]); - expect(ghost.entry).toBe(listing.at(-1)!.id); + expect(slotsOf(states, "c1")[listed]).toMatchObject({ status: "pending", unoffered: true }); + expect(slotsOf(states, "c1")[listed]!.taskId).toBeDefined(); + expect(history.views[listed]!.entries.at(-1)!.kind).toBe("pi.assistant"); + const ended = slotsOf(states, "c1").findIndex((slot) => slot?.status === "done"); + expect(slotsOf(states, "c1")[ended]!.entry).toBe(history.views[ended]!.entries.at(-1)!.id); // The aborted tool ends with its result entry, then the run ends without an answer. const done = slotsOf(states, "c2").findIndex((slot) => slot?.status === "done"); expect(slotsOf(states, "c2")[done]!.entry).toBe(history.views[done]!.entries.at(-1)!.id); From fb524dd2ea4dbb8f09ef9f8f730606643fe0b8fe Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 11:45:29 +0000 Subject: [PATCH 070/271] feat(durable): continue submissions --- packages/durable/CHANGELOG.md | 3 + packages/durable/README.md | 16 +- packages/durable/docs/spec.md | 89 +++- packages/durable/src/harness/generation.ts | 21 +- packages/durable/src/harness/live.ts | 7 +- packages/durable/src/harness/submissions.ts | 19 +- packages/durable/src/harness/telemetry.ts | 4 +- packages/durable/src/harness/types.ts | 42 +- packages/durable/src/session/transaction.ts | 6 +- .../src/testing/storage-conformance.ts | 48 ++ packages/durable/src/types.ts | 33 +- .../durable/test/harness-continue.test.ts | 422 ++++++++++++++++++ .../durable/test/harness-submissions.test.ts | 2 +- 13 files changed, 675 insertions(+), 37 deletions(-) create mode 100644 packages/durable/test/harness-continue.test.ts diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 3ccc3662e..32fbab885 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -94,6 +94,8 @@ - `ReportFrom`, the second argument of a task's `runtime.report(error, from)`, is tagged by `kind`: `{ kind: "extension", extension? }` or `{ kind: "host", callback }` (was `type`). Before: `runtime.report(error, { type: "host", callback: "env" })`. After: `runtime.report(error, { kind: "host", callback: "env" })`. - `afterTool` runs on every result a tool task writes, not only on an executed call's: also on the error results the Harness writes itself, for a tool that does not resolve (`tool_unavailable`), invalid arguments (`invalid_arguments`, with `call.arguments` as the model sent them), a block (`blocked`), an interruption (`interrupted`), the restart limit (`restart_limit`), and an abort (`aborted`, `abandoned`); the diagnostic code tells them apart. Such a result is bounded with the tool's output limits after the hooks, never keeps a `structuredOutput`, and applies `control` only when the call completes; a nested call a hook turns into a success gets its bounded output as value unless the tool declares a schema. Once a tool task was cut off more than `settings.restartLimit` times its hooks no longer run, nor in the last attempt the scheduler gives its abort handler, and an abort's hooks delay `Conversation.abort()`. A non-replay-safe tool interrupted that often now ends with `restart_limit` too. Hooks that assumed the tool ran (cast `call.arguments`, read `structuredOutput`) check `result.diagnostics` first ([#11043](https://github.com/earendil-works/pi/issues/11043)) - A call to a tool its request did not offer gets a `pi.tool` task like any call, flagged `unoffered: true` in its input and its `pi.live.tools` slot, instead of a result the generation wrote without one: the task resolves nothing (a deferred tool the agent has still never runs), settles `tool_unavailable`, and runs `afterTool`, whose `control` then counts toward the round (one such call no longer prevents `terminate`). Its slot starts `pending` with a `taskId` instead of `done`, and its result entry lands in a later commit than the tool-calling answer. Controls are applied in call order. `pi.tool` is version 3 and `pi.generation` version 2 (a sequential round keeps such calls pending until their turn): an older Harness leaves both blocked (`task_too_old`) instead of resolving the flagged name ([#11043](https://github.com/earendil-works/pi/issues/11043)) +- `SubmissionRecord` gains a `type: "continue"` variant (`placed` without `entry`, `done` with `answer`, `unanswered`), so narrowing a record by `status` alone no longer guarantees an `entry`: check `type === "input"` too. A generation whose context does not end in a user message or tool results, system messages aside, now ends its run `unanswered` with the new `RunFailureReason` `nothing_to_answer` without a request; an empty conversation no longer faults its generation. `ConversationHandle.submit()` takes any `SubmissionDraft` (was `InputSubmissionDraft`), as `Conversation.submit()` does ([#11043](https://github.com/earendil-works/pi/issues/11043)) +- `RunFailureReason` (`pi.live.runFailure.reason`) gains `nothing_to_answer`, also an `unanswered` submission's `reason`: exhaustive switches over it need a case ([#11043](https://github.com/earendil-works/pi/issues/11043)) ### Added @@ -104,6 +106,7 @@ - `RunFailureReason` gains `interrupted` (the host stopped while the model was answering, with no retry left) and `restart_limit` (the run's task was cut off `settings.restartLimit` times in a row) ([#11139](https://github.com/earendil-works/pi/issues/11139)). - `onReport` gets `task_interrupted` (`reason` `restart_limit` or `interrupted`, `restarts`, and the task) when crashes end a built-in task: a tool call, a compaction, or a generation at the restart limit, or a run that ends `interrupted`. The telemetry report attributes gain the type `task_interrupted` and the reason `interrupted` ([#11139](https://github.com/earendil-works/pi/issues/11139)). - `HarnessOptions.readOnly` opens a Harness as a viewer of a session another process drives: open leaves running tasks as committed instead of recovering them, every commit with writes rejects with `ReadOnlySession` before it reaches Storage, and `resume()` and every call that would run tasks reject with it. +- Continue submissions: `submit({ type: "continue" })` starts a run on the context as it is, without a user message, for example after a run that ended without an answer (`pi.live.runFailure`). Only an idle conversation admits one (a busy one throws `ConversationBusy`; it never queues); items a failed run left queued are placed first and join the run. A context that ends in an answer, also a text answer cut off at its output limit, has nothing to continue (`nothing_to_answer`); a cut-off answer with tool calls is continued, as its calls end in error results. Tasks and tools submit one through `ConversationHandle`. The `pi.durable.submit` span starts a root trace for it, as for an input ([#11043](https://github.com/earendil-works/pi/issues/11043)) ### Changed diff --git a/packages/durable/README.md b/packages/durable/README.md index 996f484cf..ae3aae44a 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -540,9 +540,21 @@ await root.submit({ type: "write", entry: { kind: "app.note", data: "user opened - `await submission.abort(context)` withdraws a queued submission. - The [settings](#settings) `steeringMode: "all"` and `followUpMode: "all"` place every queued item at once instead of one per turn. -If a run fails, queued items stay in the inbox until the next input or queued write places them, oldest first. +If a run fails, queued items stay in the inbox until the next input, queued write, or continue places them, oldest first. -When a run ends without an answer, `docs["pi.live"].runFailure` says why until the next run starts: `model_error` and `no_model` (with the error text or the missing model in `detail`), `aborted`, `reset`, `interrupted` (the host stopped while the model answered, with no retry left), or `faulted`, `orphaned`, and `restart_limit` (the run's task could not go on). The run's inputs are in it too, settled `unanswered`; the settlement's reason is the same, except that an orphaned run's inputs carry why it was orphaned (`missing_task`, `task_too_old`, `migration_failed`) and `no_model` has no `detail` there (spec §8.2). +When a run ends without an answer, `docs["pi.live"].runFailure` says why until the next run starts: `model_error` and `no_model` (with the error text or the missing model in `detail`), `aborted`, `reset`, `interrupted` (the host stopped while the model answered, with no retry left), `faulted`, `orphaned`, and `restart_limit` (the run's task could not go on), or `nothing_to_answer` (a continue, below, found nothing to answer). The run's inputs are in it too, settled `unanswered`; the settlement's reason is the same, except that an orphaned run's inputs carry why it was orphaned (`missing_task`, `task_too_old`, `migration_failed`) and `no_model` has no `detail` there (spec §8.2). + +To resume such a run without typing a message, submit a continue. It starts a run on the context as it is, so the model answers the user message or tool results the stopped run left: + +```typescript +const live = await harness.snapshot(LiveDoc, root.id, context); +if (live?.run === undefined && live?.runFailure !== undefined) { + const resumed = await root.submit({ type: "continue" }, context); // throws ConversationBusy while busy + await resumed.wait(context); // done with the answer, or unanswered like any run +} +``` + +A continue is never queued: it throws `ConversationBusy` while a run is working, since whether there is anything to continue depends on how that run ends. Items a failed run left queued are placed first and join the run. A context with nothing to answer (empty, or ending in an answer, also a text answer cut off at its output limit) ends the run without a request, `unanswered` with `nothing_to_answer`. A cut-off answer with tool calls can be continued: its calls never ran, so the context ends in their error results. Tools can continue the conversations they own through `api.conversation(id)`, as they submit to them. An input is placed as a `pi.user` entry. To show it as something else, give it an `entry` with your kind and data; the model still sees `content` as the user message: diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index 35330bce3..2d07abfb1 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -241,6 +241,32 @@ type SubmissionRecord = readonly reason: string; readonly detail?: JsonValue; } + )) + /** A run without a user message (section 6): no entry, never queued. */ + | (SubmissionRecordBase & { + readonly type: "continue"; + } & ( + | { + readonly status: "placed"; + readonly entry?: never; + readonly answer?: never; + readonly reason?: never; + readonly detail?: never; + } + | { + readonly status: "done"; + readonly entry?: never; + readonly answer: EntryId; + readonly reason?: never; + readonly detail?: never; + } + | { + readonly status: "unanswered"; + readonly entry?: never; + readonly answer?: never; + readonly reason: string; + readonly detail?: JsonValue; + } )); type SubmissionSettlement = @@ -349,6 +375,13 @@ type SubmissionDraft = { readonly content?: never; readonly whenBusy?: never; } + /** Start a run on the context as it is, without a user message; idle only (section 6). */ + | { + readonly type: "continue"; + readonly content?: never; + readonly entry?: never; + readonly whenBusy?: never; + } ); type InputSubmissionDraft = Extract; @@ -1410,7 +1443,7 @@ interface Tx { ): Promise>; /** Raw submission record; no admission rules (busy check, inbox, placement). Hosts use `Conversation.submit()`. */ createSubmission(create: SubmissionCreate): Promise; - /** Settle a queued or placed submission; only a placed input can be answered. A settled one stays unchanged. */ + /** Settle a queued or placed submission; only a placed input or continue can be answered. A settled one stays unchanged. */ settleSubmission(id: SubmissionId, settlement: SubmissionSettlement): void; /** Newest visible entry of the conversation that carries a `head`. */ latestHeadMarker(conversationId: ConversationId): Promise<(EntryRecord & { readonly head: EntryId }) | undefined>; @@ -2795,12 +2828,14 @@ Admission and terminal transitions: | action | submission state | other writes | |---|---|---| | input, idle with empty inbox | `placed`, with user entry | create run and generation | +| continue, idle | `placed`, no entry | place queued items (final boundary); create run and generation with them and itself | +| continue, busy | none | reject with `ConversationBusy` | | display-only write, no queued reset | `done`, with entry | append entry; no run | | write, idle with empty inbox | `done`, with entry | append entry; no run | | input or other write, busy or non-empty inbox | `queued` | append inbox item | | boundary places user item | `placed`, with entry | add ID to current or successor run | | boundary places write | `done`, with entry | append entry | -| run answers | input `done`, with required answer entry | remove `run`, or hand it to a successor | +| run answers | input or continue `done`, with required answer entry | remove `run`, or hand it to a successor | | run fails or its task aborts | input `unanswered`, with reason | remove `run`; inbox per `afterFailedRun` below | | withdraw queued item | `unanswered`, reason `aborted` | remove inbox item | | stale head write | `unanswered`, reason `stale` | remove inbox item | @@ -2810,8 +2845,26 @@ for the other submission type rejects. A busy input with `whenBusy: "reject"` writes no record and reports `ConversationBusy`. An idle input queues with mode `steer` when `whenBusy` is `steer` and `followUp` otherwise. +A continue starts a run on the context as it is, without a user message, for +example after a run that ended without an answer. It is admitted only while the +conversation is idle and never queues: whether a run can continue depends on how +the current one ends, so a busy conversation rejects it with `ConversationBusy` +before any write. Its record is `placed` without an entry; an idle conversation's +queued items are placed first by a final boundary in the same commit, and its +selected user items join the run, ahead of the continue. Whether the context has +anything to answer is checked when the run's generation prepares (section 8.3), +not at admission: a context that does not end in a user message or tool results, +system messages aside, ends the run without a request, `unanswered` with +`nothing_to_answer`. An answer cut off at its output limit (`length`) is an +answer, so a text answer is not continued; one with tool calls is, because its +calls never ran and the context ends in their synthesized `missing_result` +error results (section 2.1). A host offers continue while the conversation is +idle and `pi.live.runFailure` is set (section 8.2); a stale failure (one a later +run started in its commit kept) settles `nothing_to_answer`. Each continue starts +a run with its own turn budget. + `settings.afterFailedRun` decides what a run that fails with `model_error`, -`no_model`, `interrupted`, or `restart_limit` does with the inputs queued behind it, in the commit that settles its +`no_model`, `interrupted`, `restart_limit`, or `nothing_to_answer` does with the inputs queued behind it, in the commit that settles its own inputs: `keep` (the default) leaves them queued; `continue` applies the final boundary, so the next run starts with them, as after an answer; `withdraw` settles them `unanswered` with reason `run_failed`, keeping queued @@ -3316,7 +3369,8 @@ type ToolExecutionResult = { interface ConversationHandle { readonly id: ConversationId; - submit(submission: InputSubmissionDraft, context: Context): Promise; + /** Any submission, as `Conversation.submit()` takes: a task may also continue a conversation it owns. */ + submit(submission: SubmissionDraft, context: Context): Promise; abort(context: Context): Promise; waitForIdle(context: Context): Promise; } @@ -4347,7 +4401,16 @@ type RunFailure = { taskId: TaskId; /** The run's inputs, settled `unanswered`. */ inputs: SubmissionId[]; - reason: "model_error" | "no_model" | "aborted" | "reset" | "faulted" | "orphaned" | "interrupted" | "restart_limit"; + reason: + | "model_error" + | "no_model" + | "aborted" + | "reset" + | "faulted" + | "orphaned" + | "interrupted" + | "restart_limit" + | "nothing_to_answer"; /** * `model_error`: the error text; `no_model`: which model is missing; `faulted`: the error message; `orphaned`: why * the task could not run (`missing_task`, `task_too_old`, `migration_failed`). @@ -4440,6 +4503,7 @@ that settles its inputs `unanswered`: | `orphaned` | its task was aborted while no definition could take it; `detail` is the reason | the reason | | `interrupted` | the host stopped while the model was answering, with no retry left (section 8.3); `detail` names the attempt | `interrupted` with the text | | `restart_limit` | its generation was cut off `restartLimit` times in a row (section 5.4), also while an interrupted request was charged; `detail` says in which phase | `restart_limit` with the text | +| `nothing_to_answer` | its generation found a context that does not end in a user message or tool results, system messages aside, such as a continue after an answer (section 8.3); no `detail` | `nothing_to_answer` | The next commit that starts a run removes it. A run started in the same commit, the queue's next run under `afterFailedRun: "continue"` (section 6) or after a @@ -4519,7 +4583,13 @@ The run's inputs live in `pi.live.run`, not in the task input. the settings once. When the agent has no model or `models.getModel()` does not know it, the task fails with `no_model`. When it resumes with `overflow` and its compaction did not complete with an `entryId`, the run fails with - `model_error` and the overflow text as detail. Otherwise one commit appends + `model_error` and the overflow text as detail. When the committed model + context does not end in a user message or tool results, system messages + aside (it is empty, or ends in an answer, as for a continue after one), the + run fails with `nothing_to_answer` before any section is rendered or request + sent. Only roles count: a `length` answer with only text ends the context, + while one with tool calls is followed by the error results synthesized for + its calls (section 2.1), so a continue answers those. Otherwise one commit appends the planned `pi.system` entries and moves to `request` with the new tail as `cutoff`, the agent's model and thinking level, and the settings' stream options. These stay fixed for this request attempt; a retry prepares again. `compacted` carries over to @@ -4653,9 +4723,10 @@ A tool round starts in the commit that appends the tool-calling answer: 4. `pi.live.tools` receives the round's slots (section 8.2), and `generation` is removed. -Input submissions settle `unanswered` with one of these reasons: `no_model`, -`model_error` (detail: provider error text), `aborted`, `faulted` (detail: error -message), or an orphaning blocked reason (section 5.4). Fault and orphan +Input and continue submissions settle `unanswered` with one of these reasons: +`no_model`, `model_error` (detail: provider error text), `nothing_to_answer`, +`aborted`, `faulted` (detail: error message), or an orphaning blocked reason +(section 5.4). Fault and orphan settlement convert a committed partial into an aborted `pi.assistant` entry, like the abort handler. diff --git a/packages/durable/src/harness/generation.ts b/packages/durable/src/harness/generation.ts index 56d130082..3302b467d 100644 --- a/packages/durable/src/harness/generation.ts +++ b/packages/durable/src/harness/generation.ts @@ -183,6 +183,11 @@ export const GenerationTask = defineTask message.role !== "system"); + if (last?.role !== "user" && last?.role !== "toolResult") return failNothingToAnswer(runtime, context); const shown = replaySections(view.messages); let env: ExecutionEnv | undefined; try { @@ -218,9 +223,9 @@ export const GenerationTask = defineTask { - let cutoff = (await tx.scanEntries({ conversationId }, 1)).items[0]?.id; + // The context ends in a message the model answers, so the conversation has an entry. + let cutoff = (await tx.scanEntries({ conversationId }, 1)).items[0]!.id; for (const entry of entries) cutoff = (await tx.appendEntry(SystemEntry, conversationId, entry)).id; - if (cutoff === undefined) throw new Error(`Conversation ${conversationId} has no entries to send`); // Checked in this commit, so a compaction admitted during preparation counts. if (threshold === "background" && (await tx.doc(LiveDoc, conversationId)).compactions === undefined) { await createCompaction(tx, conversationId, { reason: "threshold" }); @@ -552,6 +557,18 @@ async function failNoModel(runtime: Runtime, ref: ModelRef | undefined, context: }, context); } +/** End the run without a request: its context has nothing the model would answer. */ +async function failNothingToAnswer(runtime: Runtime, context: Context): Promise { + const message = "The conversation has nothing to answer"; + await runtime.commit(async (tx) => { + const failure = await prepareFailure(tx, runtime); + const live = await tx.doc(LiveDoc, runtime.conversationId); + await endFailedRun(tx, runtime, live, failure, { status: "unanswered", reason: "nothing_to_answer" }); + const error = { message, detail: { reason: "nothing_to_answer" } }; + return { status: "terminal", outcome: { status: "failed", error } }; + }, context); +} + /** What a failed run's end needs from the table, read before the commit's first table write. */ type Failure = { readonly policy: AfterFailedRun; readonly boundary?: Boundary }; diff --git a/packages/durable/src/harness/live.ts b/packages/durable/src/harness/live.ts index a6669e3cb..5fb3cfb46 100644 --- a/packages/durable/src/harness/live.ts +++ b/packages/durable/src/harness/live.ts @@ -100,7 +100,9 @@ export type CompactionFailure = { * - `orphaned`: the run's task was aborted while no registered definition could take it; * - `interrupted`: the host stopped while the model was answering, more often than `settings.retry` retries; * - `restart_limit`: the run's generation was cut off `settings.restartLimit` times in a row; a request or poll it was - * cut off during counts as a failed attempt first, so this ends one whose charge itself keeps failing. + * cut off during counts as a failed attempt first, so this ends one whose charge itself keeps failing; + * - `nothing_to_answer`: the context did not end in a user message or tool results (empty, or ending in an answer), so + * no request was sent, as for a continue after an answer. */ export type RunFailureReason = | "model_error" @@ -110,7 +112,8 @@ export type RunFailureReason = | "faulted" | "orphaned" | "interrupted" - | "restart_limit"; + | "restart_limit" + | "nothing_to_answer"; /** The newest run that ended without an answer (spec §8.2). */ export type RunFailure = { diff --git a/packages/durable/src/harness/submissions.ts b/packages/durable/src/harness/submissions.ts index 419c6786d..7de0054b8 100644 --- a/packages/durable/src/harness/submissions.ts +++ b/packages/durable/src/harness/submissions.ts @@ -67,7 +67,9 @@ export class Submissions { /** Admit a submission in one commit; see `admitSubmission()`. */ async submit(conversationId: ConversationId, draft: SubmissionDraft, context: Context): Promise { this.#resume(); - const parent = draft.type === "input" ? this.#telemetry : context.telemetry; + // Inputs and continues start runs: root traces linked to their callers. + const startsRun = draft.type !== "write"; + const parent = startsRun ? this.#telemetry : context.telemetry; return createTypedSpanStarter(parent, [DURABLE_TELEMETRY_SCHEMA])( "pi.durable.submit", { @@ -75,7 +77,7 @@ export class Submissions { "conversation.id": conversationId, }, async (span) => { - if (draft.type === "input") span.addLink({ context: context.telemetry }); + if (startsRun) span.addLink({ context: context.telemetry }); try { const { id, status } = await this.#session.commitWith( (tx) => admitSubmission(tx, conversationId, draft, this.#now(), this.#queueModes(), span), @@ -187,7 +189,8 @@ function checkInputEntry(entry: InputEntry): void { * known request ID returns its existing submission without writing. A display-only write (only `kind` and `data`) is * appended and settled `done` at once, busy or not, as it changes nothing the model sees; only behind a queued reset it * queues, so it lands after the reset instead of before it in the context the reset ends. A busy conversation queues anything else in - * `pi.inbox`, or rejects `whenBusy: "reject"` input with `ConversationBusy`. An idle conversation with queued items + * `pi.inbox`, or rejects `whenBusy: "reject"` input and any continue with `ConversationBusy`. An idle continue places + * queued items and starts a run with them and itself, placing no entry of its own. An idle conversation with queued items * queues it behind them and runs a final boundary. Otherwise idle input places its entry (`pi.user`, or the input's * own) and starts a run, and an idle write appends its entry and settles `done`, or `stale` when its head reaches * before the active range. @@ -232,6 +235,16 @@ export async function admitSubmission( if (run?.trace !== undefined) span.addLink({ context: span.propagation.extract(run.trace) }); } if (busy && draft.type === "input" && draft.whenBusy === "reject") throw new ConversationBusy(conversationId); + if (draft.type === "continue") { + // Never queued: whether a run can continue depends on how the current one ends. Rejected before any write. + if (busy) throw new ConversationBusy(conversationId); + const boundary = await prepareBoundary(tx, conversationId, queueModes); + const { id } = await tx.createSubmission({ ...fields, type: "continue", status: "placed" }); + // Items an earlier failed run left queued (`afterFailedRun: "keep"`) are placed first and join the run. + const { users } = await applyBoundary(tx, boundary, "final", now); + await startRun(tx, conversationId, live, [...users, id]); + return { id, status: "placed" }; + } // A boundary reads the table, so it is prepared before the first table write; a busy one needs none. const boundary = busy ? undefined : await prepareBoundary(tx, conversationId, queueModes); if (boundary === undefined || boundary.inbox.items.length > 0) { diff --git a/packages/durable/src/harness/telemetry.ts b/packages/durable/src/harness/telemetry.ts index 1cf66ef73..0c5f60ec6 100644 --- a/packages/durable/src/harness/telemetry.ts +++ b/packages/durable/src/harness/telemetry.ts @@ -52,13 +52,13 @@ export const DURABLE_TELEMETRY_SCHEMA = defineTelemetrySchema({ spans: { "pi.durable.submit": { description: - "Public submission admission and its commit. Inputs start root traces linked to their callers; writes are children. Internal admissions emit phase events only.", + "Public submission admission and its commit. Inputs and continues start root traces linked to their callers; writes are children. Internal admissions emit phase events only.", parents: { kind: "any" }, startAttributes: { "submission.type": { type: "string", required: true, - values: ["input", "write"], + values: ["input", "write", "continue"], description: "Submission type.", }, "conversation.id": { type: "number", required: true, description: "Conversation ID.", cardinality: "high" }, diff --git a/packages/durable/src/harness/types.ts b/packages/durable/src/harness/types.ts index 272af94f4..353231dac 100644 --- a/packages/durable/src/harness/types.ts +++ b/packages/durable/src/harness/types.ts @@ -63,7 +63,11 @@ export type UserInput = UserMessage["content"]; */ export type InputEntry = { readonly kind: string; readonly data?: JsonValue }; -/** Host submission: user input that may start a run, or a passive entry write. */ +/** + * Host submission: user input that may start a run, a passive entry write, or a continue: a run without a new user + * message, for an idle conversation whose context the model still owes an answer (it ends in a user message or tool + * results, for example after a run that ended without an answer). + */ export type SubmissionDraft = { readonly requestId?: string; } & ( @@ -80,6 +84,19 @@ export type SubmissionDraft = { readonly content?: never; readonly whenBusy?: never; } + | { + /** + * Start a run on the context as it is, without a user message. Only when idle: a busy conversation rejects it with + * `ConversationBusy`, and it never queues. Items kept in `pi.inbox` are placed first and join the run. A context + * with nothing to answer (empty, or ending in an answer, also a text answer cut off at its output limit) ends the + * run without a request, `unanswered` with `nothing_to_answer`. A cut-off answer with tool calls is continued: + * its calls never ran, and the context ends in their error results (`missing_result`). + */ + readonly type: "continue"; + readonly content?: never; + readonly entry?: never; + readonly whenBusy?: never; + } ); export type InputSubmissionDraft = Extract; @@ -127,12 +144,13 @@ export type ConversationAbortOptions = { export type HooksOf = K extends Task ? H : never; /** - * Invocation-bound conversation operations for tasks and tools. Rejects after the invocation ends; passive entries are - * written with ordinary transaction writes instead. + * Invocation-bound conversation operations for tasks and tools. Rejects after the invocation ends. `submit` takes any + * submission, as `Conversation.submit()` does: a subagent's conversation is an ordinary one, so a tool may also continue + * it. */ export interface ConversationHandle { readonly id: ConversationId; - submit(submission: InputSubmissionDraft, context: Context): Promise; + submit(submission: SubmissionDraft, context: Context): Promise; /** `Conversation.abort()`: withdraw queued inputs, abort the ordinary ownership scope, and wait until it is idle. */ abort(context: Context, options?: ConversationAbortOptions): Promise; /** Resolve when the conversation's ordinary ownership scope has no live non-background task. */ @@ -578,9 +596,10 @@ export type HarnessSettings = { */ readonly documentRetentionMs?: number; /** - * What a run that fails (`model_error`, `no_model`, `interrupted`, `restart_limit`) does with the inputs queued behind it: `keep` leaves them - * queued until the next input or queued write, `continue` starts the next run with them, as an answer would, and - * `withdraw` settles them `unanswered` with reason `run_failed`. Default `keep`. + * What a run that fails (`model_error`, `no_model`, `interrupted`, `restart_limit`, `nothing_to_answer`) does with the + * inputs queued behind it: `keep` leaves them queued until the next input or queued write, `continue` starts the + * next run with them, as an answer would, and `withdraw` settles them `unanswered` with reason `run_failed`. Default + * `keep`. */ readonly afterFailedRun?: AfterFailedRun; /** @@ -752,10 +771,11 @@ export interface Conversation { configure(change: AgentChange, context: Context): Promise; /** - * Durably admit user input or a passive entry write. A busy conversation, or one with queued items, queues input and - * writes that affect model context in `pi.inbox`; `whenBusy: "reject"` rejects input with `ConversationBusy` instead - * and writes nothing. A write of only `kind` and `data` (no `model`, `head`, or `edits`) is appended at once, - * even mid-run, as it changes nothing the model sees. + * Durably admit user input, a passive entry write, or a continue. A busy conversation, or one with queued items, + * queues input and writes that affect model context in `pi.inbox`; `whenBusy: "reject"` rejects input with + * `ConversationBusy` instead and writes nothing. A write of only `kind` and `data` (no `model`, `head`, or `edits`) + * is appended at once, even mid-run, as it changes nothing the model sees. A continue starts a run on the context as + * it is, after placing queued items; a busy conversation rejects it with `ConversationBusy`, as it never queues. */ submit(submission: SubmissionDraft, context: Context): Promise; /** diff --git a/packages/durable/src/session/transaction.ts b/packages/durable/src/session/transaction.ts index cd59bba27..79f91ada8 100644 --- a/packages/durable/src/session/transaction.ts +++ b/packages/durable/src/session/transaction.ts @@ -64,8 +64,8 @@ type AnyTaskRecord = TaskRecord; type SubmissionChange = SubmissionSettlement | { readonly status: "placed"; readonly entry: EntryId }; /** - * Complete record after applying one change. Placement turns a queued input `placed` and a queued write `done`; only a - * placed input can be answered. A settled record stays. + * Complete record after applying one change. Placement turns a queued input `placed` and a queued write `done`; a + * continue is never queued, so never placed. Only a placed input or continue can be answered. A settled record stays. */ function applySubmissionChange(current: SubmissionRecord, change: SubmissionChange): SubmissionRecord { if (current.status === "done" || current.status === "unanswered") return current; @@ -75,7 +75,7 @@ function applySubmissionChange(current: SubmissionRecord, change: SubmissionChan return { ...current, status, entry: change.entry } as SubmissionRecord; } if (change.status === "done" && current.status !== "placed") { - throw new Error(`Submission ${current.id} is not a placed input`); + throw new Error(`Submission ${current.id} is not a placed input or continue`); } // Queued and placed records carry no answer, reason, or detail; an unanswered input keeps its entry. return { ...current, ...change } as SubmissionRecord; diff --git a/packages/durable/src/testing/storage-conformance.ts b/packages/durable/src/testing/storage-conformance.ts index 58dcd599b..2797c0462 100644 --- a/packages/durable/src/testing/storage-conformance.ts +++ b/packages/durable/src/testing/storage-conformance.ts @@ -908,6 +908,54 @@ export function createStorageConformance(options: StorageConformanceOptions): re expect(await storage.submissionByRequest(rootId, "passive-failed", context)).toEqual(unanswered); }), + createCase(options, "stores continue submissions, which have no entry and are never queued", async (storage) => { + const rootId = await createRoot(storage); + const doneId = await storage.mintId(); + const failedId = await storage.mintId(); + const placedDone: SubmissionRecord = { + id: doneId, + conversationId: rootId, + requestId: "continue-done", + type: "continue", + status: "placed", + }; + const placedFailed: SubmissionRecord = { + id: failedId, + conversationId: rootId, + requestId: "continue-failed", + type: "continue", + status: "placed", + }; + await storage.commit( + [ + { type: "submission", value: placedDone }, + { type: "submission", value: placedFailed }, + ], + adopt, + context, + ); + expect(await storage.submission(doneId, context)).toEqual(placedDone); + expect((await storage.scanSubmissions({ status: "placed" }, 10, undefined, context)).items).toEqual([ + placedDone, + placedFailed, + ]); + + const done: SubmissionRecord = { ...placedDone, status: "done", answer: await storage.mintId() }; + const unanswered: SubmissionRecord = { ...placedFailed, status: "unanswered", reason: "nothing_to_answer" }; + await storage.commit( + [ + { type: "submission", value: done }, + { type: "submission", value: unanswered }, + ], + adopt, + context, + ); + expect(await storage.submission(doneId, context)).toEqual(done); + expect(await storage.submissionByRequest(rootId, "continue-done", context)).toEqual(done); + expect(await storage.submission(failedId, context)).toEqual(unanswered); + expect(await storage.submissionByRequest(rootId, "continue-failed", context)).toEqual(unanswered); + }), + createCase(options, "reconstructs rewindable documents and preserves half-open incarnations", async (storage) => { const rootId = await createRoot(storage); const firstId = await storage.mintId(); diff --git a/packages/durable/src/types.ts b/packages/durable/src/types.ts index 3e20ff6ac..3ea1f1ca4 100644 --- a/packages/durable/src/types.ts +++ b/packages/durable/src/types.ts @@ -491,7 +491,7 @@ type SubmissionRecordBase = { readonly requestId?: string; }; -/** Durable lifecycle of one admitted user input or passive entry write. */ +/** Durable lifecycle of one admitted user input, passive entry write, or continue. */ export type SubmissionRecord = | (SubmissionRecordBase & { readonly type: "input"; @@ -556,6 +556,35 @@ export type SubmissionRecord = readonly reason: string; readonly detail?: JsonValue; } + )) + | (SubmissionRecordBase & { + /** A run started without a user message; it places no entry and is never queued. */ + readonly type: "continue"; + } & ( + | { + /** Owned by an active run. */ + readonly status: "placed"; + readonly entry?: never; + readonly answer?: never; + readonly reason?: never; + readonly detail?: never; + } + | { + /** The run answered. */ + readonly status: "done"; + readonly entry?: never; + readonly answer: EntryId; + readonly reason?: never; + readonly detail?: never; + } + | { + /** The run ended without an answer. */ + readonly status: "unanswered"; + readonly entry?: never; + readonly answer?: never; + readonly reason: string; + readonly detail?: JsonValue; + } )); /** Terminal status staged for a submission; identity, type, and entry come from its current record. */ @@ -992,7 +1021,7 @@ export interface Tx { */ createSubmission(create: SubmissionCreate): Promise; /** - * Settle a queued or placed submission; only a placed input can be answered, and a settled submission stays + * Settle a queued or placed submission; only a placed input or continue can be answered, and a settled submission stays * unchanged. Resolved against this transaction's latest record of the submission, so it works after table writes. * Run tasks settle the inputs they answer. */ diff --git a/packages/durable/test/harness-continue.test.ts b/packages/durable/test/harness-continue.test.ts new file mode 100644 index 000000000..8b609b8eb --- /dev/null +++ b/packages/durable/test/harness-continue.test.ts @@ -0,0 +1,422 @@ +// `submit({ type: "continue" })`: a run without a new user message (#11043). +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { withTelemetry } from "@earendil-works/chord/context"; +import { + type AssistantMessage, + type FauxResponseStep, + fauxAssistantMessage, + fauxText, + fauxToolCall, + type Message, + Type, +} from "@earendil-works/pi-ai"; +import { + AssistantEntry, + type Conversation, + ConversationBusy, + configure, + defineTool, + Harness, + InboxDoc, + LiveDoc, + MemoryStorage, + type SettledSubmissionRecord, + type Storage, + UserEntry, +} from "@earendil-works/pi-durable"; +import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; +import { afterEach, describe, expect, it } from "vitest"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { allEntries, type ChatSetup, chatSetup, openChat, textOf, unanswered, waitFor } from "./chat-support.ts"; +import { addTool } from "./harness-support.ts"; +import { context } from "./session-support.ts"; +import { aborted, deferred } from "./task-support.ts"; + +const directories = new Set(); +/** Storages of killed Harnesses, never closed by them: closed after each test. */ +const killed = new Set(); + +afterEach(async () => { + for (const storage of killed) await storage.close(context).catch(() => {}); + killed.clear(); + for (const directory of directories) await rm(directory, { recursive: true, force: true }); + directories.clear(); +}); + +async function sqlitePath(): Promise { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-continue-")); + directories.add(directory); + return join(directory, "session.sqlite"); +} + +/** Open a chat over the SQLite file at `path` and start scheduling; `kill()` simulates a crash. */ +async function open(path: string, setup: ChatSetup) { + const inner = await openNodeSqliteStorage(path); + let dead = false; + const storage = new Proxy(inner, { + get(target, property) { + const value = Reflect.get(target, property, target); + if (typeof value !== "function") return value; + return (...args: unknown[]) => (dead ? new Promise(() => {}) : value.apply(target, args)); + }, + }); + const opened = await openChat(storage, setup); + opened.harness.resume(); + const kill = (): void => { + dead = true; + killed.add(inner); + }; + return { ...opened, kill }; +} + +const failure = (): AssistantMessage => + fauxAssistantMessage([], { stopReason: "error", errorMessage: "Invalid request" }); + +/** A response step that records the roles of the context it was asked with, then answers `text`. */ +function answering(text: string, seen: Message["role"][][]): FauxResponseStep { + return (request) => { + seen.push(request.messages.map((message) => message.role)); + return fauxAssistantMessage([fauxText(text)]); + }; +} + +async function answerText(root: Conversation, settled: SettledSubmissionRecord): Promise { + if (settled.status !== "done" || settled.type === "write") throw new Error(`Unexpected ${settled.status}`); + const answer = await root.commit((tx) => tx.entry(AssistantEntry, settled.answer), context); + return textOf(answer?.model?.[0]); +} + +async function userEntries(root: Conversation): Promise { + return (await allEntries(root)).filter((entry) => UserEntry.is(entry)).length; +} + +describe("continue", () => { + // #11043 + it("answers a run that failed, without a new user message", async () => { + const setup = chatSetup(); + const seen: Message["role"][][] = []; + setup.faux.setResponses([failure, answering("recovered", seen)]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const first = await root.submit({ type: "input", content: "Analyze this" }, context); + expect(await first.wait(context)).toMatchObject({ status: "unanswered", reason: "model_error" }); + expect((await harness.snapshot(LiveDoc, root.id, context))?.runFailure?.reason).toBe("model_error"); + + const resumed = await root.submit({ type: "continue" }, context); + expect((await resumed.status(context)).status).toBe("placed"); + const settled = await resumed.wait(context); + expect(settled.type).toBe("continue"); + expect(await answerText(root, settled)).toBe("recovered"); + expect(seen.at(-1)?.at(-1)).toBe("user"); + // The failed input stays unanswered; no user entry was added. + expect(await first.status(context)).toMatchObject({ status: "unanswered" }); + expect(await userEntries(root)).toBe(1); + expect((await harness.snapshot(LiveDoc, root.id, context))?.runFailure).toBeUndefined(); + await harness.close(context); + }); + + // #11043 + it("answers the tool results of a round aborted midway", async () => { + const setup = chatSetup(); + const started = deferred(); + addTool( + setup.registry, + defineTool({ + name: "wait", + description: "Waits until aborted", + parameters: Type.Object({}), + execute: async (_args, _api, callContext) => { + started.resolve(); + return aborted(callContext.abortSignal!); + }, + }), + ); + const seen: Message["role"][][] = []; + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("wait", {}, { id: "c1" })], { stopReason: "toolUse" }), + answering("after the abort", seen), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const first = await root.submit({ type: "input", content: "go" }, context); + await started.promise; + await root.abort(context); + expect(await first.wait(context)).toMatchObject({ status: "unanswered", reason: "aborted" }); + const settled = await (await root.submit({ type: "continue" }, context)).wait(context); + expect(await answerText(root, settled)).toBe("after the abort"); + expect(seen.at(-1)?.at(-1)).toBe("toolResult"); + await harness.close(context); + }); + + // #11043 + it("skips system messages when deciding there is something to answer", async () => { + const setup = chatSetup(); + const seen: Message["role"][][] = []; + setup.faux.setResponses([fauxAssistantMessage("first answer"), failure, answering("resumed", seen)]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await root.configure({ instructions: "Be thorough." }, context); + await (await root.submit({ type: "input", content: "one" }, context)).wait(context); + // Changed instructions append a system delta after the next user message, before its request. + await root.configure({ instructions: "Be brief." }, context); + const second = await root.submit({ type: "input", content: "two" }, context); + expect(await second.wait(context)).toMatchObject({ status: "unanswered", reason: "model_error" }); + const roles = (await root.context(context)).messages.map((message) => message.role); + expect(roles.at(-1)).toBe("system"); + const settled = await (await root.submit({ type: "continue" }, context)).wait(context); + expect(await answerText(root, settled)).toBe("resumed"); + await harness.close(context); + }); + + // #11043 + it("is rejected while busy, after request ID deduplication", async () => { + const setup = chatSetup(); + const busy = unanswered(); + setup.faux.setResponses([failure, busy.step]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const resumed = await root.submit({ type: "continue", requestId: "resume-1" }, context); + await busy.reached; + await expect(root.submit({ type: "continue" }, context)).rejects.toBeInstanceOf(ConversationBusy); + // The same request ID returns the existing submission instead of rejecting. + expect((await root.submit({ type: "continue", requestId: "resume-1" }, context)).id).toBe(resumed.id); + await harness.close(context); + }); + + // #11043 + it("ends without a request when there is nothing to answer", async () => { + const setup = chatSetup(); + setup.faux.setResponses([fauxAssistantMessage("answered")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const empty = await (await root.submit({ type: "continue" }, context)).wait(context); + expect(empty).toMatchObject({ type: "continue", status: "unanswered", reason: "nothing_to_answer" }); + await (await root.submit({ type: "input", content: "hi" }, context)).wait(context); + const calls = setup.faux.state.callCount; + // A context ending in an answer has nothing to continue. + const after = await (await root.submit({ type: "continue" }, context)).wait(context); + expect(after).toMatchObject({ status: "unanswered", reason: "nothing_to_answer" }); + expect(setup.faux.state.callCount).toBe(calls); + expect((await harness.snapshot(LiveDoc, root.id, context))?.runFailure?.reason).toBe("nothing_to_answer"); + await harness.close(context); + }); + + // #11043 + it("continues an answer with tool calls cut off at its output limit, which ends in their missing results", async () => { + const setup = chatSetup(); + const seen: Message["role"][][] = []; + setup.faux.setResponses([ + fauxAssistantMessage([fauxText("calling"), fauxToolCall("echo", {}, { id: "c1" })], { stopReason: "length" }), + answering("resumed", seen), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + // The run answered: a cut-off answer is an answer, and its calls never ran. + expect((await (await root.submit({ type: "input", content: "go" }, context)).wait(context)).status).toBe("done"); + const settled = await (await root.submit({ type: "continue" }, context)).wait(context); + expect(await answerText(root, settled)).toBe("resumed"); + expect(seen.at(-1)?.slice(-2)).toEqual(["assistant", "toolResult"]); + await harness.close(context); + }); + + // #11043 + it("does not continue a text answer cut off at its output limit", async () => { + const setup = chatSetup(); + setup.faux.setResponses([fauxAssistantMessage([fauxText("cut")], { stopReason: "length" })]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "write a lot" }, context)).wait(context); + const settled = await (await root.submit({ type: "continue" }, context)).wait(context); + expect(settled).toMatchObject({ status: "unanswered", reason: "nothing_to_answer" }); + expect(setup.faux.state.callCount).toBe(1); + await harness.close(context); + }); + + // #11043 + it("answers a run interrupted by a host that died while the model answered", async () => { + const path = await sqlitePath(); + const first = chatSetup({ tokensPerSecond: 20, tokenSize: { min: 1, max: 1 } }); + first.faux.setResponses([fauxAssistantMessage("z".repeat(400))]); + first.settings.progress = { persistWithinMs: 0 }; + let opened = await open(path, first); + const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; + await waitFor( + async () => + (await opened.harness.snapshot(LiveDoc, opened.root.id, context))?.generation?.message !== undefined, + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + opened.kill(); + + const setup = chatSetup(); + setup.settings.retry = { enabled: false }; + const seen: Message["role"][][] = []; + setup.faux.setResponses([answering("after the crash", seen)]); + opened = await open(path, setup); + const interrupted = await (await opened.harness.submission(id, context))!.wait(context); + expect(interrupted).toMatchObject({ status: "unanswered", reason: "interrupted" }); + const settled = await (await opened.root.submit({ type: "continue" }, context)).wait(context); + expect(await answerText(opened.root, settled)).toBe("after the crash"); + // The aborted partial is not model context: the request ends in the user message. + expect(seen.at(-1)?.at(-1)).toBe("user"); + await opened.harness.close(context); + }); + + // #11043 + it("places a reset a failed run kept, and finds nothing to answer in the context it starts", async () => { + const setup = chatSetup(); + const gate = deferred(); + const reached = deferred(); + setup.faux.setResponses([ + async () => { + reached.resolve(); + await gate.promise; + return failure(); + }, + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const first = await root.submit({ type: "input", content: "first" }, context); + await reached.promise; + await root.reset(undefined, context); + gate.resolve(); + expect(await first.wait(context)).toMatchObject({ status: "unanswered", reason: "model_error" }); + expect((await harness.snapshot(InboxDoc, root.id, context))?.items).toHaveLength(1); + + const settled = await (await root.submit({ type: "continue" }, context)).wait(context); + expect(settled).toMatchObject({ status: "unanswered", reason: "nothing_to_answer" }); + expect((await harness.snapshot(InboxDoc, root.id, context))?.items ?? []).toEqual([]); + expect((await root.context(context)).messages.filter((message) => message.role !== "system")).toEqual([]); + expect(setup.faux.state.callCount).toBe(1); + await harness.close(context); + }); + + // #11043 + it("starts the inputs queued behind a continue with nothing to answer under afterFailedRun continue", async () => { + const setup = chatSetup(); + setup.settings.afterFailedRun = "continue"; + setup.faux.setResponses([fauxAssistantMessage("answered"), fauxAssistantMessage("second answered")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "one" }, context)).wait(context); + const resumed = await root.submit({ type: "continue" }, context); + const queued = await root.submit({ type: "input", content: "two", whenBusy: "followUp" }, context); + expect((await queued.status(context)).status).toBe("queued"); + expect(await resumed.wait(context)).toMatchObject({ status: "unanswered", reason: "nothing_to_answer" }); + const settled = await queued.wait(context); + expect(await answerText(root, settled)).toBe("second answered"); + await harness.close(context); + }); + + // #11043 + it("is submitted by a tool through the handle of a conversation it owns, and rejected there while busy", async () => { + const setup = chatSetup(); + const busy = deferred(); + const release = deferred(); + const outcome: { busy?: unknown; first?: string; resumed?: string } = {}; + addTool( + setup.registry, + defineTool({ + name: "delegate", + description: "Delegates", + parameters: Type.Object({}), + execute: async (_args, api, callContext) => { + const child = await api.commit(async (tx) => { + const created = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); + await configure(tx, created.id, { model: { provider: "faux", modelId: "faux-1" } }); + return created.id; + }, callContext); + const handle = (await api.conversation(child, callContext))!; + const first = await (await handle.submit({ type: "input", content: "work" }, callContext)).wait( + callContext, + ); + outcome.first = first.status; + const resumed = await handle.submit({ type: "continue" }, callContext); + await busy.promise; + outcome.busy = await handle.submit({ type: "continue" }, callContext).catch((error: unknown) => error); + release.resolve(); + const settled = await resumed.wait(callContext); + outcome.resumed = settled.status; + return { output: [{ type: "text", text: settled.status }] }; + }, + }), + ); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("delegate", {}, { id: "c1" })], { stopReason: "toolUse" }), + failure, + async () => { + busy.resolve(); + await release.promise; + return fauxAssistantMessage("child recovered"); + }, + fauxAssistantMessage("done"), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(outcome).toMatchObject({ first: "unanswered", resumed: "done" }); + expect(outcome.busy).toBeInstanceOf(ConversationBusy); + await harness.close(context); + }); + + // #11043 + it("starts a root trace linked to its caller, as an input does", async () => { + const setup = chatSetup(); + const telemetry = new InMemoryTelemetryContext(); + setup.faux.setResponses([failure, fauxAssistantMessage("recovered")]); + const harness = await Harness.open( + new MemoryStorage(), + { models: setup.models, registry: setup.registry, settings: setup.settings, telemetry }, + context, + ); + const root = await harness.root(context, { agent: { model: { provider: "faux", modelId: "faux-1" } } }); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const id = await telemetry.startSpan({ name: "caller" }, async (caller) => { + const callContext = withTelemetry(caller, context); + const resumed = await root.submit({ type: "continue" }, callContext); + await resumed.wait(callContext); + return resumed.id; + }); + await harness.waitForIdle(context); + await waitFor(() => telemetry.getSpans().every((span) => span.settled)); + const spans = telemetry.getSpans(); + const caller = spans.find((span) => span.name === "caller")!; + const submit = spans.find( + (span) => span.name === "pi.durable.submit" && span.attributes["submission.id"] === id, + )!; + expect(submit).toMatchObject({ parentId: null, attributes: { result: "placed" } }); + expect(submit.links).toEqual([{ context: caller.context, attributes: {} }]); + expect((await (await harness.submission(id, context))!.status(context)).trace).toEqual(submit.context); + expect( + spans.some( + (span) => + span.name === "pi.durable.phase" && + span.attributes.phase === "prepare" && + JSON.stringify(span.parentContext) === JSON.stringify(submit.context), + ), + ).toBe(true); + await harness.close(context); + }); + + // #11043 + it("places inputs a failed run left queued first, and they join its run", async () => { + const setup = chatSetup(); + const gate = deferred(); + const reached = deferred(); + setup.faux.setResponses([ + async () => { + reached.resolve(); + await gate.promise; + return failure(); + }, + fauxAssistantMessage("both handled"), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await root.submit({ type: "input", content: "first" }, context); + await reached.promise; + const queued = await root.submit({ type: "input", content: "second", whenBusy: "followUp" }, context); + gate.resolve(); + await harness.waitForIdle(context); + expect((await queued.status(context)).status).toBe("queued"); + + const resumed = await root.submit({ type: "continue" }, context); + const live = await harness.snapshot(LiveDoc, root.id, context); + expect(live?.run?.inputs).toEqual([queued.id, resumed.id]); + const [byQueued, byContinue] = [await queued.wait(context), await resumed.wait(context)]; + expect(await answerText(root, byContinue)).toBe("both handled"); + expect(byQueued).toMatchObject({ status: "done", answer: (byContinue as { answer: number }).answer }); + await harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-submissions.test.ts b/packages/durable/test/harness-submissions.test.ts index bae654985..08630183a 100644 --- a/packages/durable/test/harness-submissions.test.ts +++ b/packages/durable/test/harness-submissions.test.ts @@ -64,7 +64,7 @@ describe("submissions", () => { const submission = await root.submit({ type: "input", content: "hi" }, context); await busy.reached; const record = await submission.status(context); - if (record.status !== "placed") throw new Error(`Unexpected ${record.status}`); + if (record.type !== "input" || record.status !== "placed") throw new Error(`Unexpected ${record.status}`); const entry = await root.commit((tx) => tx.entry(UserEntry, record.entry), context); expect(entry?.model).toEqual([{ role: "user", content: "hi", timestamp: 42 }]); const live = await harness.snapshot(LiveDoc, root.id, context); From f6a9db1d8355d53a0b1e90c6af3fab1364ee8b7d Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 11:45:40 +0000 Subject: [PATCH 071/271] feat(durable): per-conversation turn limit closes #11043 --- packages/durable/CHANGELOG.md | 4 +- packages/durable/README.md | 7 +- packages/durable/docs/spec.md | 43 ++- packages/durable/src/harness/agent.ts | 6 + packages/durable/src/harness/generation.ts | 47 ++- packages/durable/src/harness/live.ts | 23 +- packages/durable/src/harness/types.ts | 21 +- .../durable/test/harness-compaction.test.ts | 4 + .../test/harness-generation-recovery.test.ts | 2 +- .../durable/test/harness-generation.test.ts | 2 +- .../durable/test/harness-live-deltas.test.ts | 9 +- .../durable/test/harness-ownership.test.ts | 2 +- .../test/harness-scheduler-retention.test.ts | 6 +- .../durable/test/harness-turn-limit.test.ts | 357 ++++++++++++++++++ .../durable/test/harness-view-run.test.ts | 2 +- 15 files changed, 493 insertions(+), 42 deletions(-) create mode 100644 packages/durable/test/harness-turn-limit.test.ts diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 32fbab885..e63340b27 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -95,7 +95,8 @@ - `afterTool` runs on every result a tool task writes, not only on an executed call's: also on the error results the Harness writes itself, for a tool that does not resolve (`tool_unavailable`), invalid arguments (`invalid_arguments`, with `call.arguments` as the model sent them), a block (`blocked`), an interruption (`interrupted`), the restart limit (`restart_limit`), and an abort (`aborted`, `abandoned`); the diagnostic code tells them apart. Such a result is bounded with the tool's output limits after the hooks, never keeps a `structuredOutput`, and applies `control` only when the call completes; a nested call a hook turns into a success gets its bounded output as value unless the tool declares a schema. Once a tool task was cut off more than `settings.restartLimit` times its hooks no longer run, nor in the last attempt the scheduler gives its abort handler, and an abort's hooks delay `Conversation.abort()`. A non-replay-safe tool interrupted that often now ends with `restart_limit` too. Hooks that assumed the tool ran (cast `call.arguments`, read `structuredOutput`) check `result.diagnostics` first ([#11043](https://github.com/earendil-works/pi/issues/11043)) - A call to a tool its request did not offer gets a `pi.tool` task like any call, flagged `unoffered: true` in its input and its `pi.live.tools` slot, instead of a result the generation wrote without one: the task resolves nothing (a deferred tool the agent has still never runs), settles `tool_unavailable`, and runs `afterTool`, whose `control` then counts toward the round (one such call no longer prevents `terminate`). Its slot starts `pending` with a `taskId` instead of `done`, and its result entry lands in a later commit than the tool-calling answer. Controls are applied in call order. `pi.tool` is version 3 and `pi.generation` version 2 (a sequential round keeps such calls pending until their turn): an older Harness leaves both blocked (`task_too_old`) instead of resolving the flagged name ([#11043](https://github.com/earendil-works/pi/issues/11043)) - `SubmissionRecord` gains a `type: "continue"` variant (`placed` without `entry`, `done` with `answer`, `unanswered`), so narrowing a record by `status` alone no longer guarantees an `entry`: check `type === "input"` too. A generation whose context does not end in a user message or tool results, system messages aside, now ends its run `unanswered` with the new `RunFailureReason` `nothing_to_answer` without a request; an empty conversation no longer faults its generation. `ConversationHandle.submit()` takes any `SubmissionDraft` (was `InputSubmissionDraft`), as `Conversation.submit()` does ([#11043](https://github.com/earendil-works/pi/issues/11043)) -- `RunFailureReason` (`pi.live.runFailure.reason`) gains `nothing_to_answer`, also an `unanswered` submission's `reason`: exhaustive switches over it need a case ([#11043](https://github.com/earendil-works/pi/issues/11043)) +- `RunFailureReason` (`pi.live.runFailure.reason`) gains `nothing_to_answer` and `turn_limit`, also an `unanswered` submission's `reason`: exhaustive switches over it need cases ([#11043](https://github.com/earendil-works/pi/issues/11043)) +- `pi.live.run` has a required `turns` field, the run's tool rounds so far; code that writes `run` itself sets it (`turns: 0`). The unreleased `pi.live` version 2 `migrate` adds `turns: 0` to a stored version 1 run; a database written by an unreleased build at version 2 without `run.turns` is not supported (its run counts `NaN` turns) ([#11043](https://github.com/earendil-works/pi/issues/11043)) ### Added @@ -107,6 +108,7 @@ - `onReport` gets `task_interrupted` (`reason` `restart_limit` or `interrupted`, `restarts`, and the task) when crashes end a built-in task: a tool call, a compaction, or a generation at the restart limit, or a run that ends `interrupted`. The telemetry report attributes gain the type `task_interrupted` and the reason `interrupted` ([#11139](https://github.com/earendil-works/pi/issues/11139)). - `HarnessOptions.readOnly` opens a Harness as a viewer of a session another process drives: open leaves running tasks as committed instead of recovering them, every commit with writes rejects with `ReadOnlySession` before it reaches Storage, and `resume()` and every call that would run tasks reject with it. - Continue submissions: `submit({ type: "continue" })` starts a run on the context as it is, without a user message, for example after a run that ended without an answer (`pi.live.runFailure`). Only an idle conversation admits one (a busy one throws `ConversationBusy`; it never queues); items a failed run left queued are placed first and join the run. A context that ends in an answer, also a text answer cut off at its output limit, has nothing to continue (`nothing_to_answer`); a cut-off answer with tool calls is continued, as its calls end in error results. Tasks and tools submit one through `ConversationHandle`. The `pi.durable.submit` span starts a root trace for it, as for an input ([#11043](https://github.com/earendil-works/pi/issues/11043)) +- Per-conversation turn limits: `AgentState.maxTurns` (`configure({ maxTurns })`, `null` clears; a `RangeError` for anything but an integer 1 or more) bounds the tool rounds one run takes. The round that reaches it runs its tools, then the run ends `unanswered` with the new `RunFailureReason` `turn_limit`, like a failed run, so `settings.afterFailedRun` decides the queued inputs and a continue resumes it on the round's results. The count is `pi.live.run.turns`; every run starts at 0, and an `onYield` continuation starts it over. Task-owned children and forks inherit the limit; `Agent.maxTurns` exposes it ([#11043](https://github.com/earendil-works/pi/issues/11043)) ### Changed diff --git a/packages/durable/README.md b/packages/durable/README.md index ae3aae44a..a60ee6b17 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -217,7 +217,7 @@ The default `setTimer` declines the timers that only free memory in Workers, so **Reports.** `onReport(report)` gets one `Report`: the `error` and what happened in `report.kind`: `storage_failed` for the error that failed the Harness (a Storage error, often a platform reset); `task_faulted`, `task_interrupted` (crashes ended it, with `report.reason` and `report.restarts`), `migration_failed`, `task_outdated` (`reason` `missing_task`, `task_too_old`, or `migration_failed`, the words a blocked task shows), or `task_error` for a task, with `report.task` (its `id`, `kind`, and `conversationId`) and the extension that installed it in `report.extension`; `extension_error` for hooks, sections, wrappers, and tools, with the extension's name; `host_error` for a host callback, named in `report.callback`; and `listener_error`. A host can feed `task_faulted` reports of one kind to its own breaker. -**Queued inputs after a failed run.** A run that fails with a model error, a missing model, or crashes (`interrupted`, `restart_limit`) leaves the inputs queued behind it waiting until the next input arrives. `settings.afterFailedRun` changes that: `continue` starts the next run with them, as after an answer, and `withdraw` settles them `unanswered` with reason `run_failed`. +**Queued inputs after a failed run.** A run that fails with a model error, a missing model, nothing to answer, its turn limit, or crashes (`interrupted`, `restart_limit`) leaves the inputs queued behind it waiting until the next input arrives. `settings.afterFailedRun` changes that: `continue` starts the next run with them, as after an answer, and `withdraw` settles them `unanswered` with reason `run_failed`. **Blocked tasks.** Tasks blocked on a missing or unmigratable definition show their reason in the task graph too, and a registry change that unblocks them makes `onWake` report the Session active, so a host need not poll them. @@ -391,6 +391,7 @@ await root.configure( modelTools: [readTool], // of those, offer the model only read; tools may still call bash instructions: "Only read; never edit files.", cwd: "/work/repo", + maxTurns: 50, // tool rounds one run takes at most; unset: no limit }, context, ); @@ -400,6 +401,8 @@ const agent = await root.agent(context); // resolved: model, extensions, tools, Extensions and tools are passed as objects and stored by name, so a stored name outlives its code: after an extension is uninstalled, conversations that select it just stop getting it until it is installed again. `createConversation()`, `fork()`, and `root()` take the same change as `agent`. A task-owned conversation, such as a subagent's, starts as a copy of its owner's conversation's agent. A fork starts with the agent its parent had at the fork entry. The model, prompt, and offered tools of a request are fixed when it is prepared; a change applies from the next request. Tool calls and hooks use the agent as their task phase resolves it, and the environment is built from the current `cwd` at each use, so a `cwd` or extension change can reach calls the model already made. +**Turn limits.** `maxTurns` bounds the tool rounds one run takes: the round that reaches it still runs its tools, then the run ends `unanswered` with `turn_limit`, like a failed run (`runFailure`, and `settings.afterFailedRun` decides the inputs queued behind it). An answer always ends the run, so it is never cut off. The count is `docs["pi.live"].run.turns`; it survives restarts, and every run starts at 0, also one a continue starts (see [Busy Conversations](#busy-conversations)), so a host or tool that continues automatically gets a fresh budget each time and must bound that itself. An `onYield` continuation also starts the count over: an extension that keeps a run going bounds its own loop, reading `pi.live.run.turns` and `pi.agent.maxTurns` with `api.snapshot`. `configure()` throws a `RangeError` for anything but an integer 1 or more (or `null`); a change applies at the end of the current round, and children and forks inherit it like the rest of the agent. + ## Settings Run policy shared by every conversation is passed as `settings`. It is read at every use and never stored, so getters make it live, for example backed by a settings file: @@ -542,7 +545,7 @@ await root.submit({ type: "write", entry: { kind: "app.note", data: "user opened If a run fails, queued items stay in the inbox until the next input, queued write, or continue places them, oldest first. -When a run ends without an answer, `docs["pi.live"].runFailure` says why until the next run starts: `model_error` and `no_model` (with the error text or the missing model in `detail`), `aborted`, `reset`, `interrupted` (the host stopped while the model answered, with no retry left), `faulted`, `orphaned`, and `restart_limit` (the run's task could not go on), or `nothing_to_answer` (a continue, below, found nothing to answer). The run's inputs are in it too, settled `unanswered`; the settlement's reason is the same, except that an orphaned run's inputs carry why it was orphaned (`missing_task`, `task_too_old`, `migration_failed`) and `no_model` has no `detail` there (spec §8.2). +When a run ends without an answer, `docs["pi.live"].runFailure` says why until the next run starts: `model_error` and `no_model` (with the error text or the missing model in `detail`), `aborted`, `reset`, `interrupted` (the host stopped while the model answered, with no retry left), `faulted`, `orphaned`, and `restart_limit` (the run's task could not go on), `turn_limit` (a tool round reached the conversation's `maxTurns`), or `nothing_to_answer` (a continue, below, found nothing to answer). The run's inputs are in it too, settled `unanswered`; the settlement's reason is the same, except that an orphaned run's inputs carry why it was orphaned (`missing_task`, `task_too_old`, `migration_failed`) and `no_model` has no `detail` there (spec §8.2). To resume such a run without typing a message, submit a continue. It starts a run on the context as it is, so the model answers the user message or tool results the stopped run left: diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index 2d07abfb1..284009b86 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -548,6 +548,8 @@ type AgentState = { instructions?: string; /** Directory within the environment's file system, passed to `HarnessOptions.env`. */ cwd?: string; + /** Tool rounds one run takes at most (section 8.5); an integer, `1` or more; unset, no limit. */ + maxTurns?: number; }; /** Built-in rewindable agent document; see below. */ @@ -568,6 +570,8 @@ type AgentChange = { | null; readonly instructions?: string | null; readonly cwd?: string | null; + /** `configure()` throws a `RangeError` unless it is an integer `1` or more, or `null`. */ + readonly maxTurns?: number | null; }; /** One change in any commit. A given field replaces the stored one; `null` clears it; `undefined` changes nothing. */ @@ -588,6 +592,7 @@ type Agent = { readonly sections: readonly PromptSection[]; readonly instructions?: string; readonly cwd?: string; + readonly maxTurns?: number; }; /** Entry whose `data` has type `D`; `never` means the kind carries no data. */ @@ -2837,6 +2842,7 @@ Admission and terminal transitions: | boundary places write | `done`, with entry | append entry | | run answers | input or continue `done`, with required answer entry | remove `run`, or hand it to a successor | | run fails or its task aborts | input `unanswered`, with reason | remove `run`; inbox per `afterFailedRun` below | +| tool round reaches `maxTurns` | input `unanswered`, reason `turn_limit` | as a failed run | | withdraw queued item | `unanswered`, reason `aborted` | remove inbox item | | stale head write | `unanswered`, reason `stale` | remove inbox item | @@ -2864,7 +2870,8 @@ run started in its commit kept) settles `nothing_to_answer`. Each continue start a run with its own turn budget. `settings.afterFailedRun` decides what a run that fails with `model_error`, -`no_model`, `interrupted`, `restart_limit`, or `nothing_to_answer` does with the inputs queued behind it, in the commit that settles its +`no_model`, `interrupted`, `restart_limit`, `nothing_to_answer`, or +`turn_limit` does with the inputs queued behind it, in the commit that settles its own inputs: `keep` (the default) leaves them queued; `continue` applies the final boundary, so the next run starts with them, as after an answer; `withdraw` settles them `unanswered` with reason `run_failed`, keeping queued @@ -4326,8 +4333,11 @@ messages out of later requests, so no separate usage or notice kind exists. ```ts type LiveState = { - /** Run control (section 6); present exactly while the conversation is busy. */ - run?: { taskId: TaskId; inputs: SubmissionId[] }; + /** + * Run control (section 6); present exactly while the conversation is busy. `turns` counts the run's tool rounds, + * which `AgentState.maxTurns` bounds (section 8.5); an `onYield` continuation starts it over (section 8.3). + */ + run?: { taskId: TaskId; inputs: SubmissionId[]; turns: number }; /** Presentation of the current generation attempt. */ generation?: { attempt: number; @@ -4410,10 +4420,12 @@ type RunFailure = { | "orphaned" | "interrupted" | "restart_limit" - | "nothing_to_answer"; + | "nothing_to_answer" + | "turn_limit"; /** * `model_error`: the error text; `no_model`: which model is missing; `faulted`: the error message; `orphaned`: why - * the task could not run (`missing_task`, `task_too_old`, `migration_failed`). + * the task could not run (`missing_task`, `task_too_old`, `migration_failed`); `interrupted` and `restart_limit`: + * what was cut off, and how often; `turn_limit`: the tool rounds the run took, `"1 turn"` or `"N turns"`. */ detail?: string; }; @@ -4444,7 +4456,7 @@ type CompactionStatus = { | version | `2` | | scope/history/fork | conversation, `latest`, `initial` | | `initial()` | `{}` | -| `migrate()` | from version 1: `reason` of each `compactions` status and of `compactionFailure` renamed `trigger` (section 3.6) | +| `migrate()` | from version 1: `reason` of each `compactions` status and of `compactionFailure` renamed `trigger`, and `run.turns` set to `0` (section 3.6) | | checkpoint | complete base whenever nothing runs: `generation` absent and no `running` tool or nested slot | | view mount | `docs["pi.live"]` | | created | with every Harness conversation (section 2.2) | @@ -4504,6 +4516,7 @@ that settles its inputs `unanswered`: | `interrupted` | the host stopped while the model was answering, with no retry left (section 8.3); `detail` names the attempt | `interrupted` with the text | | `restart_limit` | its generation was cut off `restartLimit` times in a row (section 5.4), also while an interrupted request was charged; `detail` says in which phase | `restart_limit` with the text | | `nothing_to_answer` | its generation found a context that does not end in a user message or tool results, system messages aside, such as a continue after an answer (section 8.3); no `detail` | `nothing_to_answer` | +| `turn_limit` | a tool round reached the conversation's `maxTurns` and ran its tools (section 8.5); `detail` is the run's count, `"1 turn"` or `"N turns"` | `turn_limit` with the count | The next commit that starts a run removes it. A run started in the same commit, the queue's next run under `afterFailedRun: "continue"` (section 6) or after a @@ -4664,7 +4677,11 @@ The run's inputs live in `pi.live.run`, not in the task input. `{ continue }` wins. The commit appends the answer and applies the final boundary (section 6). With a continuation and no selected user item or reset, it appends a `pi.user` entry with the continuation content, creates a - successor generation, and hands it `pi.live.run`, keeping the inputs open. + successor generation, and hands it `pi.live.run`, keeping the inputs open and + setting its `turns` to `0`: each stretch a continuation starts gets the whole + turn budget, and bounding `onYield` loops is the extension's job (it reads + `pi.live.run.turns` and `pi.agent.maxTurns` with `api.snapshot`). The chain runs + whatever the count. Otherwise it settles the run's inputs `done`, removes `run` and `generation`, and starts a successor run for the selected user items, if any; a dropped continuation is not retried. Both complete with @@ -4720,12 +4737,12 @@ A tool round starts in the commit that appends the tool-calling answer: `pending` shrinks. 3. The generation commits `waiting` on its tool tasks with `allSettled` and the `tools` checkpoint (section 8.5); `pi.live.run` stays with it. -4. `pi.live.tools` receives the round's slots (section 8.2), and `generation` is - removed. +4. `pi.live.tools` receives the round's slots (section 8.2), `generation` is + removed, and `pi.live.run.turns` grows by one. Input and continue submissions settle `unanswered` with one of these reasons: `no_model`, `model_error` (detail: provider error text), `nothing_to_answer`, -`aborted`, `faulted` (detail: error message), or an orphaning blocked reason +`turn_limit` (detail: the turn count, section 8.5), `aborted`, `faulted` (detail: error message), or an orphaning blocked reason (section 5.4). Fault and orphan settlement convert a committed partial into an aborted `pi.assistant` entry, like the abort handler. @@ -4837,6 +4854,12 @@ and then commits once: - When every result of the round requests `terminate`, it settles the run's inputs `done` with the tool-calling answer, removes `run` and `tools`, and applies the final boundary. +- Otherwise, when the run's `turns`, counted in the commit that started the + round, reached the phase agent's `maxTurns`, the run ends as a failed run + (section 6): its inputs settle `unanswered` with `turn_limit` (detail: the + count), `runFailure` records it, `afterFailedRun` decides the queued items, and + the generation still completes with `{ entryId }`. A continue resumes it on the + round's results (section 6). - Otherwise it removes `tools` and applies the `postTools` boundary. When that boundary placed a queued reset, which wins over a handoff written before it, the run's inputs settle `unanswered` with `reset` and selected user items diff --git a/packages/durable/src/harness/agent.ts b/packages/durable/src/harness/agent.ts index d996bec6f..df15a9f13 100644 --- a/packages/durable/src/harness/agent.ts +++ b/packages/durable/src/harness/agent.ts @@ -158,6 +158,11 @@ function applyChange(state: Draft, change: AgentChange): void { ); set("instructions", change.instructions); set("cwd", change.cwd); + const maxTurns = change.maxTurns; + if (maxTurns !== undefined && maxTurns !== null && !(Number.isSafeInteger(maxTurns) && maxTurns >= 1)) { + throw new RangeError(`maxTurns must be an integer 1 or more, got ${String(maxTurns)}`); + } + set("maxTurns", maxTurns); } function isList(value: readonly T[] | object): value is readonly T[] { @@ -281,6 +286,7 @@ export function resolveAgent( sections: agentSections, ...(instructions === undefined ? {} : { instructions }), ...(state?.cwd === undefined ? {} : { cwd: state.cwd }), + ...(state?.maxTurns === undefined ? {} : { maxTurns: state.maxTurns }), }; return agent; } diff --git a/packages/durable/src/harness/generation.ts b/packages/durable/src/harness/generation.ts index 3302b467d..2b756bd37 100644 --- a/packages/durable/src/harness/generation.ts +++ b/packages/durable/src/harness/generation.ts @@ -29,7 +29,7 @@ import type { } from "../types.ts"; import { addTools, sectionExtension } from "./agent.ts"; import { compactionPolicy, createCompaction, estimateContext, selectCut } from "./compaction.ts"; -import { applyBoundary, type Boundary, prepareBoundary, withdrawQueuedInputs } from "./inbox.ts"; +import { applyBoundary, type Boundary, hasQueuedReset, prepareBoundary, withdrawQueuedInputs } from "./inbox.ts"; import { cutOffCount, cutOffTooOften, @@ -796,10 +796,13 @@ async function answer(runtime: Runtime, message: AssistantMessage, context: Cont const live = await tx.doc(LiveDoc, conversationId); const entry = await appendAssistant(tx, conversationId, message); const result: Next = { status: "terminal", outcome: { status: "completed", result: { entryId: entry.id } } }; - const { users, reset } = await applyBoundary(tx, boundary, "final", runtime.now()); - if (continuation !== undefined && users.length === 0 && !reset) { - const user = { role: "user", content: continuation, timestamp: runtime.now() } as const; + const continues = takesContinuation(continuation, boundary) ? continuation : undefined; + const { users } = await applyBoundary(tx, boundary, "final", runtime.now()); + if (continues !== undefined) { + const user = { role: "user", content: continues, timestamp: runtime.now() } as const; await tx.appendEntry(UserEntry, conversationId, { model: [user] }); + // The continuation starts the run's turn count over: bounding `onYield` loops is the extension's job. + if (live.run?.taskId === runtime.taskId) live.run.turns = 0; handOver(live, runtime.taskId, await createGeneration(tx, conversationId)); delete live.generation; return result; @@ -810,6 +813,20 @@ async function answer(runtime: Runtime, message: AssistantMessage, context: Cont }, context); } +/** + * Whether an answer's `onYield` continuation is taken (spec §8.3), decided from the inbox before the final boundary + * places anything: only when it holds no user item, of which the boundary would select one, and no reset, which would + * otherwise run first. A continuation starts the turn count over. + */ +function takesContinuation(continuation: UserInput | undefined, boundary: Boundary): continuation is UserInput { + const items = boundary.inbox.items; + return ( + continuation !== undefined && + !items.some((item) => item.mode === "steer" || item.mode === "followUp") && + !hasQueuedReset(items) + ); +} + /** * Append the tool-calling answer and start its tool round in one commit (spec §8.3). Every call gets a tool task owned by * the generation, in call order, only the first one now when the round is sequential; a call to a tool the request did @@ -854,6 +871,8 @@ async function startToolRound( } delete live.generation; live.tools = slots; + // A tool round is one turn of the run (`AgentState.maxTurns`). + if (live.run?.taskId === runtime.taskId) live.run.turns += 1; const checkpoint = { phase: "tools", assistant: entry.id, tools, pending } as const; return { status: "waiting", checkpoint, on: tools, policy: "allSettled" }; }, context); @@ -861,8 +880,9 @@ async function startToolRound( /** * The round's tools are terminal: apply their controls, append a `handoff`'s reset, and then end the run at the final - * boundary (`terminate`), end it at a queued reset, or hand it to the next generation at the `postTools` boundary, in - * the handoff's new context when there is one (spec §8.5). + * boundary (`terminate`), end it like a failure at the conversation's turn limit (`turn_limit`), end it at a queued + * reset, or hand it to the next generation at the `postTools` boundary, in the handoff's new context when there is one + * (spec §8.5). */ async function finishToolRound( runtime: Runtime, @@ -891,6 +911,8 @@ async function finishToolRound( const added = ordered.flatMap((control) => control?.addTools ?? []); // The last handoff in call order wins. const handoff = ordered.findLast((control) => control?.handoff !== undefined)?.handoff; + // Read from the phase's agent, as every agent field; a `configure()` during the round applies at its end. + const maxTurns = (await runtime.agent(context)).maxTurns; await runtime.commit(async (tx): Promise => { const boundary = await prepareBoundary(tx, conversationId, runtime.settings); if (added.length > 0) await addTools(tx, conversationId, added); @@ -902,10 +924,21 @@ async function finishToolRound( const entry = await tx.appendEntry(ResetEntry, conversationId, { head: "self", model: [message] }); boundary.head = entry.id; } + const turns = live.run?.taskId === runtime.taskId ? live.run.turns : 0; if (terminate) { const { users } = await applyBoundary(tx, boundary, "final", now); endRun(tx, live, runtime.taskId, { status: "done", answer: assistant }); if (users.length > 0) await startRun(tx, conversationId, live, users); + } else if (maxTurns !== undefined && turns >= maxTurns) { + // The turn limit ends the run like a failure: `afterFailedRun` decides the queued items, and a continue resumes + // it on the round's results. The table was read for the boundary already, before any write. + const policy = runtime.settings.afterFailedRun; + const failure = { policy, ...(policy === "continue" ? { boundary } : {}) }; + await endFailedRun(tx, runtime, live, failure, { + status: "unanswered", + reason: "turn_limit", + detail: turns === 1 ? "1 turn" : `${turns} turns`, + }); } else { const { users, reset } = await applyBoundary(tx, boundary, "postTools", now); if (reset) { @@ -947,7 +980,7 @@ export async function startRun( inputs: SubmissionId[], options?: { readonly keepFailure?: boolean }, ): Promise { - live.run = { taskId: await createGeneration(tx, conversationId, { submissionIds: inputs }), inputs }; + live.run = { taskId: await createGeneration(tx, conversationId, { submissionIds: inputs }), inputs, turns: 0 }; if (options?.keepFailure !== true) delete live.runFailure; } diff --git a/packages/durable/src/harness/live.ts b/packages/durable/src/harness/live.ts index 5fb3cfb46..a0fcccd3b 100644 --- a/packages/durable/src/harness/live.ts +++ b/packages/durable/src/harness/live.ts @@ -102,7 +102,8 @@ export type CompactionFailure = { * - `restart_limit`: the run's generation was cut off `settings.restartLimit` times in a row; a request or poll it was * cut off during counts as a failed attempt first, so this ends one whose charge itself keeps failing; * - `nothing_to_answer`: the context did not end in a user message or tool results (empty, or ending in an answer), so - * no request was sent, as for a continue after an answer. + * no request was sent, as for a continue after an answer; + * - `turn_limit`: a tool round reached the conversation's `maxTurns`; a continue resumes on its results. */ export type RunFailureReason = | "model_error" @@ -113,7 +114,8 @@ export type RunFailureReason = | "orphaned" | "interrupted" | "restart_limit" - | "nothing_to_answer"; + | "nothing_to_answer" + | "turn_limit"; /** The newest run that ended without an answer (spec §8.2). */ export type RunFailure = { @@ -125,15 +127,18 @@ export type RunFailure = { /** * `model_error`: the error text; `no_model`: which model is missing; `faulted`: the error message; `orphaned`: why * the task could not run (`missing_task`, `task_too_old`, `migration_failed`); `interrupted` and `restart_limit`: what - * was cut off, and how often. + * was cut off, and how often; `turn_limit`: the tool rounds the run took, `"1 turn"` or `"N turns"`. */ detail?: string; }; /** Built-in live conversation state: run control and presentation of the current generation and tool round. */ export type LiveState = { - /** Run control: the task that settles the run's inputs, and those inputs; present exactly while busy. */ - run?: { taskId: TaskId; inputs: SubmissionId[] }; + /** + * Run control: the task that settles the run's inputs, those inputs, and the run's tool rounds so far, which + * `AgentState.maxTurns` bounds (an `onYield` continuation starts the count over); present exactly while busy. + */ + run?: { taskId: TaskId; inputs: SubmissionId[]; turns: number }; /** Presentation of the current generation attempt. */ generation?: { attempt: number; @@ -173,12 +178,14 @@ export const LiveDoc = defineDoc({ history: "latest", fork: "initial", initial: () => ({}), - // Version 1 named a compaction's trigger `reason`, in `compactions` and `compactionFailure`. + // Version 1 named a compaction's trigger `reason`, in `compactions` and `compactionFailure`, and had no `run.turns`: + // a run in flight across the upgrade counts its turns from 0. migrate: (value) => { - const live = value as { compactions?: JsonObject[]; compactionFailure?: JsonObject }; - const { compactions, compactionFailure } = live; + const live = value as { run?: JsonObject; compactions?: JsonObject[]; compactionFailure?: JsonObject }; + const { run, compactions, compactionFailure } = live; return { ...value, + ...(run === undefined ? {} : { run: { ...run, turns: 0 } }), ...(compactions === undefined ? {} : { compactions: compactions.map(renameReason) }), ...(compactionFailure === undefined ? {} : { compactionFailure: renameReason(compactionFailure) }), } as LiveState; diff --git a/packages/durable/src/harness/types.ts b/packages/durable/src/harness/types.ts index 353231dac..742edb085 100644 --- a/packages/durable/src/harness/types.ts +++ b/packages/durable/src/harness/types.ts @@ -455,6 +455,11 @@ export type AgentState = { instructions?: string; /** Directory within the environment's file system, passed to `HarnessOptions.env`. */ cwd?: string; + /** + * Tool rounds one run takes at most (`pi.live.run.turns`): the round that reaches it runs its tools, then the run ends + * `unanswered` with `turn_limit`. An integer, `1` or more; unset, no limit. Task-owned children and forks inherit it. + */ + maxTurns?: number; }; /** A change to `pi.agent`: a given field replaces the stored one, `null` clears it, `undefined` changes nothing. */ @@ -473,6 +478,8 @@ export type AgentChange = { | null; readonly instructions?: string | null; readonly cwd?: string | null; + /** `configure()` throws a `RangeError` unless it is an integer `1` or more (or `null`). */ + readonly maxTurns?: number | null; }; /** A conversation's agent resolved against a registry snapshot and the settings. */ @@ -490,6 +497,11 @@ export type Agent = { readonly sections: readonly PromptSection[]; readonly instructions?: string; readonly cwd?: string; + /** + * Tool rounds one run takes at most (`pi.live.run.turns`): the round that reaches it runs its tools, then the run ends + * `unanswered` with `turn_limit`. An integer, `1` or more; unset, no limit. Task-owned children and forks inherit it. + */ + readonly maxTurns?: number; }; /** @@ -596,8 +608,8 @@ export type HarnessSettings = { */ readonly documentRetentionMs?: number; /** - * What a run that fails (`model_error`, `no_model`, `interrupted`, `restart_limit`, `nothing_to_answer`) does with the - * inputs queued behind it: `keep` leaves them queued until the next input or queued write, `continue` starts the + * What a run that fails (`model_error`, `no_model`, `interrupted`, `restart_limit`, `nothing_to_answer`, + * `turn_limit`) does with the inputs queued behind it: `keep` leaves them queued until the next input or queued write, `continue` starts the * next run with them, as an answer would, and `withdraw` settles them `unanswered` with reason `run_failed`. Default * `keep`. */ @@ -613,7 +625,10 @@ export type HarnessSettings = { readonly restartLimit?: number; }; -/** What a failed run does with the inputs queued behind it (`HarnessSettings.afterFailedRun`). */ +/** + * What a failed run does with the inputs queued behind it (`HarnessSettings.afterFailedRun`): a run that ends with + * `model_error`, `no_model`, `interrupted`, `restart_limit`, `nothing_to_answer`, or `turn_limit`. + */ export type AfterFailedRun = "keep" | "continue" | "withdraw"; /** Resolved settings: every field over its built-in default, object fields merged. */ diff --git a/packages/durable/test/harness-compaction.test.ts b/packages/durable/test/harness-compaction.test.ts index afe74558a..a473b825b 100644 --- a/packages/durable/test/harness-compaction.test.ts +++ b/packages/durable/test/harness-compaction.test.ts @@ -29,6 +29,7 @@ import { type LiveState, MemoryStorage, ProviderDoc, + type SubmissionId, type TaskId, UsageDoc, } from "@earendil-works/pi-durable"; @@ -2473,6 +2474,7 @@ describe("pi.live version 2", () => { ); await session.commit(async (tx) => { const live = await tx.doc(LiveV1, conversationId); + live.run = { taskId: 9, inputs: [4] }; live.compactions = [{ taskId: 7, reason: "threshold", blocking: true, attempt: 2 }]; live.compactionFailure = { taskId: 5, reason: "manual", error: "no model" }; }, context); @@ -2486,6 +2488,8 @@ describe("pi.live version 2", () => { return (await storage.document(record!.id, "current", context))!; }; const migrated: LiveState = { + // A run in flight across the upgrade counts its turns from 0 (#11043). + run: { taskId: 9 as TaskId, inputs: [4 as SubmissionId], turns: 0 }, compactions: [{ taskId: 7 as TaskId, trigger: "threshold", blocking: true, attempt: 2 }], compactionFailure: { taskId: 5 as TaskId, trigger: "manual", error: "no model" }, }; diff --git a/packages/durable/test/harness-generation-recovery.test.ts b/packages/durable/test/harness-generation-recovery.test.ts index 6e044a9fb..52c0f62be 100644 --- a/packages/durable/test/harness-generation-recovery.test.ts +++ b/packages/durable/test/harness-generation-recovery.test.ts @@ -197,7 +197,7 @@ describe("generation recovery", () => { await new Promise((resolve) => setTimeout(resolve, 20)); expect(setup.faux.state.callCount).toBe(1); expect(await opened.harness.snapshot(LiveDoc, opened.root.id, context)).toEqual({ - run: { taskId, inputs: [id] }, + run: { taskId, inputs: [id], turns: 0 }, generation: { attempt: 1, retry: { at: 61_000, error: "503 Service Unavailable" } }, }); now = 61_000; diff --git a/packages/durable/test/harness-generation.test.ts b/packages/durable/test/harness-generation.test.ts index 4fd7697f0..46a3fe58f 100644 --- a/packages/durable/test/harness-generation.test.ts +++ b/packages/durable/test/harness-generation.test.ts @@ -717,7 +717,7 @@ describe("generation", () => { {}, { ownership: { kind: "conversation" }, conversationId: root.id }, ); - (await tx.doc(LiveDoc, root.id)).run = { taskId, inputs: [submission.id] }; + (await tx.doc(LiveDoc, root.id)).run = { taskId, inputs: [submission.id], turns: 0 }; return { taskId, submissionId: submission.id }; }, context); harness.resume(); diff --git a/packages/durable/test/harness-live-deltas.test.ts b/packages/durable/test/harness-live-deltas.test.ts index 737b577a8..696820e7c 100644 --- a/packages/durable/test/harness-live-deltas.test.ts +++ b/packages/durable/test/harness-live-deltas.test.ts @@ -131,13 +131,14 @@ describe("pi.live deltas", () => { const result = entries.items.find((entry) => entry.kind === "pi.tool-result")!.id; expect(commits).toEqual([ // submission - [["s", ["run"], { taskId: firstGeneration, inputs: [expect.any(Number)] }]], + [["s", ["run"], { taskId: firstGeneration, inputs: [expect.any(Number)], turns: 0 }]], // request [["s", ["generation"], { attempt: 1 }]], - // the generation starts its tool round and keeps the run + // the generation starts its tool round and keeps the run, counting the round as one turn (#11043) expect.arrayContaining([ ["d", ["generation"]], ["s", ["tools"], [{ callId: "c1", name: "noop", taskId: tool, status: "pending" }]], + ["s", ["run", "turns"], 1], ]), // intent [["s", ["tools", 0, "status"], "running"]], @@ -158,7 +159,7 @@ describe("pi.live deltas", () => { ["d", ["generation"]], ]), ]); - expect(commits[2]).toHaveLength(2); + expect(commits[2]).toHaveLength(3); expect(commits[4]).toHaveLength(2); expect(commits[5]).toHaveLength(2); await harness.close(context); @@ -456,7 +457,7 @@ describe("pi.live deltas", () => { const base = (value: LiveState) => LiveDoc.definition.checkpointWhen!(value, [], { deltasSinceBase: 1000 } as never); const slot = (status: "pending" | "running" | "done") => ({ callId: "c", name: "n", status }); - const run = { taskId: 1 as never, inputs: [] }; + const run = { taskId: 1 as never, inputs: [], turns: 0 }; expect(base({})).toBe(true); expect(base({ run, generation: { attempt: 1 } })).toBe(false); expect(base({ run, tools: [slot("pending"), slot("done")] })).toBe(true); diff --git a/packages/durable/test/harness-ownership.test.ts b/packages/durable/test/harness-ownership.test.ts index e7e796133..86bd715b2 100644 --- a/packages/durable/test/harness-ownership.test.ts +++ b/packages/durable/test/harness-ownership.test.ts @@ -174,7 +174,7 @@ async function waitUntil(check: () => Promise): Promise { /** Mark a conversation busy with `task` standing in for its run, so submissions queue. */ async function busy(conversation: Conversation, task: TaskId): Promise { await conversation.commit(async (tx) => { - (await tx.doc(LiveDoc, conversation.id)).run = { taskId: task, inputs: [] }; + (await tx.doc(LiveDoc, conversation.id)).run = { taskId: task, inputs: [], turns: 0 }; }, context); } diff --git a/packages/durable/test/harness-scheduler-retention.test.ts b/packages/durable/test/harness-scheduler-retention.test.ts index 8b12c4cb0..918ffaa24 100644 --- a/packages/durable/test/harness-scheduler-retention.test.ts +++ b/packages/durable/test/harness-scheduler-retention.test.ts @@ -267,7 +267,7 @@ describe("scheduler retention", () => { const c = (await harness.conversation(tree.c, context))!; // Busy, so the input queues instead of starting a run. await c.commit(async (tx) => { - (await tx.doc(LiveDoc, tree.c)).run = { taskId: tree.keep, inputs: [] }; + (await tx.doc(LiveDoc, tree.c)).run = { taskId: tree.keep, inputs: [], turns: 0 }; }, context); const queued = (await c.submit({ type: "input", content: "later" }, context)).id; await harness.abortTask(tree.p, context); @@ -452,7 +452,7 @@ describe("scheduler retention", () => { await harness.waitForTask(tree.b, context); const c = (await harness.conversation(tree.c, context))!; await c.commit(async (tx) => { - (await tx.doc(LiveDoc, tree.c)).run = { taskId: tree.keep, inputs: [] }; + (await tx.doc(LiveDoc, tree.c)).run = { taskId: tree.keep, inputs: [], turns: 0 }; }, context); // Queued while nothing cancels: no withdrawal scan. const queued = (await c.submit({ type: "input", content: "later" }, context)).id; @@ -529,7 +529,7 @@ describe("scheduler retention", () => { context, ); await root.commit(async (tx) => { - (await tx.doc(LiveDoc, root.id)).run = { taskId: busy, inputs: [] }; + (await tx.doc(LiveDoc, root.id)).run = { taskId: busy, inputs: [], turns: 0 }; }, context); const before = scans; await root.submit({ type: "input", content: "queued" }, context); diff --git a/packages/durable/test/harness-turn-limit.test.ts b/packages/durable/test/harness-turn-limit.test.ts new file mode 100644 index 000000000..e5d819468 --- /dev/null +++ b/packages/durable/test/harness-turn-limit.test.ts @@ -0,0 +1,357 @@ +// Per-conversation turn limits (`AgentState.maxTurns`) and continuing a run that reached one (#11043). +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { + type AfterFailedRun, + type Conversation, + defineTool, + GenerationTask, + LiveDoc, + MemoryStorage, + ResetEntry, + type Storage, + type ToolRegistration, + ToolResultEntry, + ToolTask, +} from "@earendil-works/pi-durable"; +import { afterEach, describe, expect, it } from "vitest"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { allEntries, type ChatSetup, chatSetup, openChat, textOf } from "./chat-support.ts"; +import { addHooks, addTool } from "./harness-support.ts"; +import { context } from "./session-support.ts"; +import { aborted, deferred } from "./task-support.ts"; + +const directories = new Set(); +const killed = new Set(); + +afterEach(async () => { + for (const storage of killed) await storage.close(context).catch(() => {}); + killed.clear(); + for (const directory of directories) await rm(directory, { recursive: true, force: true }); + directories.clear(); +}); + +const Parameters = Type.Object({}); + +function tool( + name: string, + execute: ToolRegistration["execute"], +): ToolRegistration { + return defineTool({ name, description: name, parameters: Parameters, execute }); +} + +/** A tool-calling answer with one `noop` call. */ +function round(id: string): AssistantMessage { + return fauxAssistantMessage([fauxToolCall("noop", {}, { id })], { stopReason: "toolUse" }); +} + +const DONE = fauxAssistantMessage([fauxText("done")]); + +/** A chat with a `noop` tool and the given turn limit. */ +async function limited(maxTurns: number | undefined, configure?: (setup: ChatSetup) => void) { + const setup = chatSetup(); + addTool( + setup.registry, + tool("noop", async () => ({ output: [] })), + ); + configure?.(setup); + const opened = await openChat(new MemoryStorage(), setup); + if (maxTurns !== undefined) await opened.root.configure({ maxTurns }, context); + return { setup, ...opened }; +} + +async function lastKind(root: Conversation): Promise { + return (await allEntries(root)).at(-1)?.kind; +} + +describe("turn limit", () => { + // #11043 + it("ends a run after the tool round that reaches the limit, like a failure", async () => { + const { setup, harness, root } = await limited(1); + setup.faux.setResponses([round("c1"), DONE]); + const submission = await root.submit({ type: "input", content: "go" }, context); + expect(await submission.wait(context)).toMatchObject({ + status: "unanswered", + reason: "turn_limit", + detail: "1 turn", + }); + expect(setup.faux.state.callCount).toBe(1); + // The transcript ends with the round's results; the generation completed, so nothing is reported. + expect(await lastKind(root)).toBe(ToolResultEntry.kind); + expect((await harness.snapshot(LiveDoc, root.id, context))?.runFailure).toMatchObject({ + reason: "turn_limit", + inputs: [submission.id], + }); + const generations = await harness.commit((tx) => tx.scanTasks({ conversationId: root.id }, 20), context); + const generation = generations.items.find((task) => task.kind === GenerationTask.definition.name); + expect(generation?.state).toMatchObject({ status: "terminal", outcome: { status: "completed" } }); + expect(setup.reports).toEqual([]); + await harness.close(context); + }); + + // #11043 + it("counts tool rounds only: a run answers below its limit, and an answer at the limit settles done", async () => { + const { setup, harness, root } = await limited(2); + setup.faux.setResponses([round("c1"), DONE, round("c2"), round("c3"), DONE]); + expect((await (await root.submit({ type: "input", content: "one" }, context)).wait(context)).status).toBe("done"); + // A new run starts with a fresh budget. + const second = await (await root.submit({ type: "input", content: "two" }, context)).wait(context); + expect(second).toMatchObject({ status: "unanswered", reason: "turn_limit", detail: "2 turns" }); + await harness.close(context); + }); + + // #11043 + it("runs onYield at the limit, and a continuation starts the count over", async () => { + let yields = 0; + const { setup, harness, root } = await limited(2, (chat) => + addHooks(chat.registry, GenerationTask, { + onYield: () => (yields++ === 0 ? { continue: "keep going" } : undefined), + }), + ); + // Round (1 turn), answer: the continuation resets the count; then two rounds reach the limit. + setup.faux.setResponses([round("c1"), DONE, round("c2"), round("c3"), DONE]); + const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(settled).toMatchObject({ status: "unanswered", reason: "turn_limit" }); + expect(yields).toBe(1); + expect(setup.faux.state.callCount).toBe(4); + await harness.close(context); + }); + + // #11043 + it("settles a steer that joined the run at an earlier round with turn_limit too", async () => { + const gate = deferred(); + const started = deferred(); + const setup = chatSetup(); + let calls = 0; + addTool( + setup.registry, + tool("noop", async () => { + if (calls++ === 0) { + started.resolve(); + await gate.promise; + } + return { output: [] }; + }), + ); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await root.configure({ maxTurns: 2 }, context); + setup.faux.setResponses([round("c1"), round("c2"), DONE]); + const first = await root.submit({ type: "input", content: "go" }, context); + await started.promise; + const steer = await root.submit({ type: "input", content: "also this", whenBusy: "steer" }, context); + gate.resolve(); + for (const submission of [first, steer]) { + expect(await submission.wait(context)).toMatchObject({ + status: "unanswered", + reason: "turn_limit", + detail: "2 turns", + }); + } + expect((await harness.snapshot(LiveDoc, root.id, context))?.runFailure?.inputs).toEqual([first.id, steer.id]); + expect(setup.faux.state.callCount).toBe(2); + await harness.close(context); + }); + + // #11043 + it("leaves items queued during the last round to afterFailedRun", async () => { + for (const policy of ["keep", "continue", "withdraw"] as const satisfies readonly AfterFailedRun[]) { + const gate = deferred(); + const started = deferred(); + const setup = chatSetup(); + setup.settings.afterFailedRun = policy; + addTool( + setup.registry, + tool("noop", async () => { + started.resolve(); + await gate.promise; + return { output: [] }; + }), + ); + setup.faux.setResponses([round("c1"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await root.configure({ maxTurns: 1 }, context); + const first = await root.submit({ type: "input", content: "go" }, context); + await started.promise; + const steer = await root.submit({ type: "input", content: "also", whenBusy: "followUp" }, context); + gate.resolve(); + expect((await first.wait(context)).status).toBe("unanswered"); + await harness.waitForIdle(context); + const queued = await steer.status(context); + if (policy === "keep") expect(queued.status).toBe("queued"); + if (policy === "continue") expect(queued.status).toBe("done"); + if (policy === "withdraw") expect(queued).toMatchObject({ status: "unanswered", reason: "run_failed" }); + await harness.close(context); + } + }); + + // #11043 + it("applies a changed limit at the end of the round; null removes it", async () => { + const setup = chatSetup(); + let root!: Conversation; + let rounds = 0; + addTool( + setup.registry, + tool("noop", async () => { + rounds++; + // Lowered during the second round: that round ends the run. + if (rounds === 2) await root.configure({ maxTurns: 1 }, context); + return { output: [] }; + }), + ); + setup.faux.setResponses([round("c1"), round("c2"), round("c3"), DONE]); + const opened = await openChat(new MemoryStorage(), setup); + root = opened.root; + await root.configure({ maxTurns: 5 }, context); + const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(settled).toMatchObject({ status: "unanswered", reason: "turn_limit", detail: "2 turns" }); + await root.configure({ maxTurns: null }, context); + expect((await root.agent(context)).maxTurns).toBeUndefined(); + setup.faux.setResponses([round("c4"), round("c5"), DONE]); + expect((await (await root.submit({ type: "input", content: "again" }, context)).wait(context)).status).toBe( + "done", + ); + await opened.harness.close(context); + }); + + // #11043 + it("rejects a limit that is not an integer 1 or more", async () => { + const { harness, root } = await limited(undefined); + for (const value of [0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY]) { + await expect(root.configure({ maxTurns: value }, context)).rejects.toBeInstanceOf(RangeError); + } + await root.configure({ maxTurns: 3 }, context); + expect((await root.agent(context)).maxTurns).toBe(3); + await harness.close(context); + }); + + // #11043 + it("lets terminate settle done at the limit, and writes a handoff's reset before ending", async () => { + const { setup, harness, root } = await limited(1, (chat) => + addHooks(chat.registry, ToolTask, { + afterTool: (call, result) => + call.id === "t" + ? { ...result, control: { terminate: true } } + : { ...result, control: { handoff: "fresh" } }, + }), + ); + setup.faux.setResponses([round("t"), round("h"), DONE]); + expect((await (await root.submit({ type: "input", content: "one" }, context)).wait(context)).status).toBe("done"); + const handedOff = await (await root.submit({ type: "input", content: "two" }, context)).wait(context); + expect(handedOff).toMatchObject({ status: "unanswered", reason: "turn_limit" }); + const resets = (await allEntries(root)).filter((entry) => ResetEntry.is(entry)); + expect(resets.map((entry) => (entry.model?.[0] as { content: string }).content)).toEqual(["fresh"]); + await harness.close(context); + }); + + // #11043 + it("is resumed by a continue, with a fresh budget, on the round's results", async () => { + const { setup, harness, root } = await limited(1); + let continued: string | undefined; + setup.faux.setResponses([ + round("c1"), + (request) => { + continued = request.messages.at(-1)?.role; + return fauxAssistantMessage([fauxText("finished")]); + }, + ]); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const resumed = await root.submit({ type: "continue" }, context); + expect((await harness.snapshot(LiveDoc, root.id, context))?.run?.turns).toBe(0); + const settled = await resumed.wait(context); + if (settled.status !== "done" || settled.type === "write") throw new Error(`Unexpected ${settled.status}`); + const answer = (await allEntries(root)).find((entry) => entry.id === settled.answer); + expect(textOf(answer?.model?.[0])).toBe("finished"); + expect(continued).toBe("toolResult"); + await harness.close(context); + }); + + // #11043 + it("is inherited by forks and children, which may set their own", async () => { + const setup = chatSetup(); + let child: number | undefined; + addTool( + setup.registry, + tool("spawn", async (_args, api, callContext) => { + child = await api.commit( + async (tx) => (await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } })).id, + callContext, + ); + return { output: [] }; + }), + ); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("spawn", {}, { id: "c1" })], { stopReason: "toolUse" }), + DONE, + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await root.configure({ maxTurns: 4 }, context); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const spawned = await harness.conversation(child as never, context); + expect((await spawned!.agent(context)).maxTurns).toBe(4); + const at = (await allEntries(root)).at(-1)!.id; + const fork = await root.fork(at, { ownership: { kind: "ownerless" } }, context); + expect((await fork.agent(context)).maxTurns).toBe(4); + const own = await harness.createConversation( + { ownership: { kind: "ownerless" }, agent: { maxTurns: 2 } }, + context, + ); + expect((await own.agent(context)).maxTurns).toBe(2); + await harness.close(context); + }); + + // #11043 + it("keeps the stored count across a restart", async () => { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-turns-")); + directories.add(directory); + const path = join(directory, "session.sqlite"); + const setup = chatSetup(); + setup.settings.progress = { outputIntervalMs: 0, persistWithinMs: 0 }; + const started = deferred(); + let runs = 0; + addTool( + setup.registry, + defineTool({ + name: "noop", + description: "noop", + parameters: Parameters, + replay: "safe", + execute: async (_args, _api, callContext) => { + runs++; + if (runs === 2) { + started.resolve(); + return aborted(callContext.abortSignal!); + } + return { output: [] }; + }, + }), + ); + setup.faux.setResponses([round("c1"), round("c2"), DONE]); + const inner = await openNodeSqliteStorage(path); + let dead = false; + const storage = new Proxy(inner, { + get(target, property) { + const value = Reflect.get(target, property, target); + if (typeof value !== "function") return value; + return (...args: unknown[]) => (dead ? new Promise(() => {}) : value.apply(target, args)); + }, + }); + const first = await openChat(storage, setup); + await first.root.configure({ maxTurns: 2 }, context); + const id = (await first.root.submit({ type: "input", content: "go" }, context)).id; + await started.promise; + expect((await first.harness.snapshot(LiveDoc, first.root.id, context))?.run?.turns).toBe(2); + dead = true; + killed.add(inner); + + // The replay-safe call runs again after reopen; the stored count still ends the run at this round. + runs = 0; + const second = await openChat(await openNodeSqliteStorage(path), setup); + second.harness.resume(); + expect((await second.harness.snapshot(LiveDoc, second.root.id, context))?.run?.turns).toBe(2); + const settled = await (await second.harness.submission(id, context))!.wait(context); + expect(settled).toMatchObject({ status: "unanswered", reason: "turn_limit", detail: "2 turns" }); + await second.harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-view-run.test.ts b/packages/durable/test/harness-view-run.test.ts index a4445ea91..a87fc009c 100644 --- a/packages/durable/test/harness-view-run.test.ts +++ b/packages/durable/test/harness-view-run.test.ts @@ -103,7 +103,7 @@ describe("a run as the conversation view shows it", () => { expect(partialsIn(states).map(textOf)).toEqual(partials.map(textOf)); // The run shows from the input's commit until the commit that appends the answer and clears the partial. const started = states.findIndex((state) => state.run !== undefined); - expect(states[started]!.run).toEqual({ taskId: expect.any(Number), inputs: [submission.id] }); + expect(states[started]!.run).toEqual({ taskId: expect.any(Number), inputs: [submission.id], turns: 0 }); expect(history.views[started]!.entries.at(-1)!.kind).toBe("pi.user"); const ended = states.findIndex((state, index) => index > started && state.run === undefined); expect(states[ended]!.generation).toBeUndefined(); From e4c58271b82c83a43e1ab91e4ea3bfff542d5bba Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 11:50:25 +0000 Subject: [PATCH 072/271] docs(chord): slice 6b D9, self.follow; spec decided --- packages/chord/docs/next/decisions.md | 8 ++++ packages/chord/docs/next/slice-6b.md | 55 ++++++++++++++++----------- packages/chord/docs/next/slices.md | 2 +- 3 files changed, 42 insertions(+), 23 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index db1ef860f..97ac5e874 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1683,6 +1683,14 @@ validation and copies the v2 kernel added around Delta. ctx, { persistence: { kind: "deferred", withinMs? } })`, Durable's `CommitPersistence` shape (default 1,000 ms): adopted and published at once, stored with the next immediate change, before the outcome is recorded, at close, or after `withinMs`; a crash loses what is not stored. Only run state documents for now. +- **Slice 6b D9: `self.follow(attach)` follows Durable's views (owner, 2026-10-27: "sounds good"):** a generic Chord + primitive, no `durableView` module in Durable: `attach(observer, ctx)` returns `Promise<{ value, detach }>` + (Durable's `Observation`; its observers have Chord's watch shapes since pi `5f312dac4`). Chord makes the observer, + attaches during activation (early calls held, `value` the start value), detaches at the attempt's end; calls inside a + batch mark the source in the step, calls outside one are grouped into one batch on the next microtask; a rejected or + `undefined` attach fails the activation as a failed dependency; Durable's `failed` end ends the watches `failed`. + Returns a `Watchable` (state member, derive source, `on` source). Supersedes the carry-over's `durableView` and D1's + `durableView` part. - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the diff --git a/packages/chord/docs/next/slice-6b.md b/packages/chord/docs/next/slice-6b.md index b9d6e902e..c3423c403 100644 --- a/packages/chord/docs/next/slice-6b.md +++ b/packages/chord/docs/next/slice-6b.md @@ -1,6 +1,6 @@ # Slice 6b: Durable and jobs on the store (spec) -Status: **draft, open points marked "Decision needed"** (2026-10-27), written by the redesign session after reading +Status: **decided, ready for the spec reviews** (2026-10-27; D1-D9 decided by the owner), written by the redesign session after reading Durable's `Storage` interface (`packages/durable/src/types.ts`), its SQLite storage, schema and migrations, the storage conformance suite and benchmark, `session/session.ts`, `session/transaction.ts`, `session/forks.ts`, and Chord v2's store engine, applier, schema, commit step and jobs, in full. Durable's memory, JSONL and IndexedDB storages were not @@ -12,7 +12,7 @@ W1, W12, W13, "Slice 6 first decisions", "Durable observers", "Durable's synchro "Failures are watch calls", the "One vocabulary" entries, "The performance and memory bar"; `slice-6a.md` (what 6a built); the feature map and the carry-over of `slices.md` (slice 6b); the root `AGENTS.md` "Chord v2 Code Style". -6b puts Durable on the store (`StoreStorage`), follows Durable's views as Chord sources (`durableView`), persists jobs, +6b puts Durable on the store (`StoreStorage`), follows Durable's views as Chord sources (`self.follow`), persists jobs, and extends the applier to every write kind. After 6b a worker has one store: Chord's objects and Durable's Session commit through one queue, and a commit of either is one seq in one outbox. @@ -28,7 +28,7 @@ host.serve(Harness, (_, self) => { const harness = self.open(async (context) => Harness.open(new StoreStorage(store), { models, registry, … }, context), // one StoreStorage per store (h, _reason, context) => h.close(context)); - const view = self.follow(...) // Decision needed (D9): name and shape + const view = self.follow((observer, ctx) => harness.value.observeConversation(id, observer, ctx)); // section 2 return { … }; }); @@ -47,8 +47,8 @@ Durable's `Storage` (types.ts, "Atomic persistence boundary for Session records" Durable's Session runs unchanged on it and passes Durable's storage conformance suite (`createStorageConformance`), across reopen too. -**Where it lives (D1, owner, 2026-10-27: "they are only used if durable is used"):** `StoreStorage` and -`durableView` belong to Durable, in its own `src` (Chord v2 is in pi upstream since `93029a0f1`), importing +**Where it lives (D1, owner, 2026-10-27: "they are only used if durable is used"):** `StoreStorage` belongs to +Durable (D9 makes `durableView` a generic Chord primitive, `self.follow`), in its own `src` (Chord v2 is in pi upstream since `93029a0f1`), importing `@earendil-works/chord/next` (Chord's `./next` package export comes with them). Durable already depends on Chord and Chord imports nothing of Durable (S13); Durable's conformance suite runs in Durable's tests and CI. @@ -130,22 +130,33 @@ store usable. 6b changes 6a's step and object failure accordingly. A target to measure, as 6a's: an entry append is the `record_ids` put, the entry put, `lastSeq` (plus the outbox); a document delta is 6a's three statements. `SqliteStorage` does the same work today (`record_ids`, the row, metadata). -## 2. `durableView`: Durable's views as Chord sources +## 2. `self.follow`: Durable's views as Chord sources (D9, owner, 2026-10-27) -A Chord source over Durable's synchronous observers (`observeConversation`, `observeTaskGraph`, `observeDoc`; -decision "Durable's synchronous observers"): `advance(call, ctx)` records the call; on the store, `adopt` runs inside -the host's batch, so the source is marked in the step and its frame goes out in the flush with the rest of the commit -(one message per link per commit). On a Durable storage that is not the store (tests, other apps), calls arrive outside -any batch: the source opens one batch per Durable commit on the next microtask (carry-over). Durable's failures are -final: `end({ kind: "failed" })` ends the source's watches `failed`; `ended` ends them `ended`. A derive over two -Durable views of one commit sees both changes in one computation. +Durable's synchronous observers (`observeConversation`, `observeTaskGraph`, `observeDoc`; decision "Durable's +synchronous observers") have Chord's watch shapes (`advance(call, ctx)` with a `WatchCall`, `end(end)` with a +`WatchEnd`, pi `5f312dac4`), so Chord follows them with one generic primitive and Durable needs no adapter: -> **Decision needed (D9): the API.** The carry-over names it `self.follow` (working name): `self.follow(observe)` in -> a factory, `observe` attaching an observer, returning a `Watchable` usable as a state member, a derive source and an -> `on` source, detached when the attempt ends. Options: (a) `self.follow((observer, ctx) => -> harness.observeConversation(id, observer, ctx, window))`; (b) a plain function `durableView(attach)` returning a -> native the object opens (`self.open`). Proposal: (a), named `self.follow`: the attempt owns the attachment as it -> owns a derive. +```ts +host.serve(ChatView, ({ conversation }, self) => { + const harness = self.get(Harness, {}); + const view = self.follow((observer, ctx) => + harness.value.observeConversation(conversation, observer, ctx, { min: 50 })); + return { view, title: self.derive([view], ([v]) => v.title) }; +}); +``` + +- `self.follow(attach)` returns a `Watchable`: a state member, a derive source, an `on` source. `attach(observer, + ctx)` returns `Promise<{ value, detach }>` (Durable's `Observation`). +- **Attaching** is that one call during activation: Chord makes the observer, calls `attach`, takes `value` as the + source's value (calls arriving before the promise resolves are held and applied after it), and the object goes live. + The attempt's end calls `detach()`. An attach that rejects, or resolves `undefined` (`observeDoc` of an absent + address), fails the activation like a failed dependency. +- **Calls:** `advance` records the call. Inside a batch (on the store, Durable's `adopt` runs in the host's batch) the + source is marked in the step and its frame goes out in that flush; outside a batch (a Durable storage that is not + the store) the source collects the calls and opens one batch on the next microtask. A derive over two views of one + Durable commit sees both changes in one computation. +- **End:** Durable's failures are final: `end({ kind: "failed" })` ends the watches `failed`, `ended` ends them + `ended`. ## 3. Persisted jobs @@ -215,8 +226,8 @@ applying (S17). memory (allocation per op, retained after GC) no worse beyond noise; persisted job starts and state changes against in-memory ones. - **Tests:** Durable's storage conformance suite on `StoreStorage` (memory `:memory:` and across reopen); Durable's - Session and harness test files that take a storage, run on `StoreStorage`; durableView (one frame per commit, a - derive over two views, outside the store, failure and end); persisted jobs (rerun, abandon, a recorded outcome + Session and harness test files that take a storage, run on `StoreStorage`; `self.follow` (one frame per commit, a + derive over two views, outside the store, early calls held, failure and end, an absent `observeDoc`); persisted jobs (rerun, abandon, a recorded outcome resumes `finish`, restart counting and reset, the limit, abort stored, deferred state, fingerprints); the applier (every kind, the outbox replay, refused counts); export and import with Durable records and `rewindable` history; GC (observers detached with their attempt). @@ -225,7 +236,7 @@ applying (S17). ## Carry-over taken From `slices.md` "Slice 6b": persisted jobs (all of it), `StoreStorage` (ids, fork copies, `adopt` in the batch), -`durableView` (the batch on and off the store, `self.follow`), the applier for batch apply; `job.abort` async and +the Chord source over Durable's views (`self.follow`, the batch on and off the store), the applier for batch apply; `job.abort` async and stored (decision of 2026-10-27). ## Carry-over left diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index ea6d73093..a638c6db4 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -109,7 +109,7 @@ pattern, no code. Decided cuts are noted where they apply. | 10 | the store: backends, tables, query runner, writes, commit queue, outbox, migrations, export/import | 6a | | 11 | `Watchable`, watches (Chord's `Watch`), the batch (W1), reentrancy | 1 | | 11 | `NativeWatchable`, `self.writable` | 8 | -| 11 | Durable's views as sources (`durableView` over Durable's `observeXXX`) | 6b | +| 11 | Durable's views as sources (`self.follow` over Durable's `observeXXX`) | 6b | | 12 | `self.derive`, failures per member, lazy when unwatched, `mapShared` | 1 | | 12 | container-width benchmark (R24) | 9 | | 13 | `self.open`, close reasons, order, restart patterns (R16) | 2 (restarts on reload: 7) | From bb6ae4b8ef0901ea8ecaaa6f142b5b30dd8c690c Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 11:50:41 +0000 Subject: [PATCH 073/271] docs(chord): self.follow in design and slices --- packages/chord/docs/next/design.md | 2 +- packages/chord/docs/next/slices.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index 5e646fb5b..2468b9ec3 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -1920,7 +1920,7 @@ loop. Chord has no stream type (it would need its own rules for drops, cancellat - Durable keeps importing today's Chord's `context`, `delta`, `json`, `output` and `sqlite`; `next/` imports the same modules, so `Context`, `Draft` and `JsonValue` are one type. Durable's conversation view, task graph and documents are `Watchable`s through Durable's watches (Durable's `WatchHandle` now lives in Chord); Durable adopts and dispatches - them synchronously inside the storage's commit (S1, W1; with `StoreStorage` that is the store's commit step), so a derive over a Durable view travels with it. This needs Durable to publish a commit when it adopts it (owner, 2026-10-26; a pi change, upstream first; explainer `durable-observers/`): inside `adopt`, its trackers, telemetry events and internal observers (view mounts, task graph, scheduler, submissions) run, and the host `subscribeCommits` listeners are dispatched in one microtask queued before the observers, so they run before every wait the commit resolves and every abort it queues, but after the storage's synchronous step, never inside Chord's batch; host listeners and watch listeners have no order between them (owner, 2026-10-26). Durable's public primitive for adapters is a synchronous observer, `observeConversation`, `observeTaskGraph`, `observeDoc` (its `advance({ value, ops, retired }, ctx)` runs inside `adopt`: record only, no awaits, no Session calls, no throws); `watchXXX` is built on it for app code. Chord's `durableView` (slice 6b) is a source over it. Otherwise a derive over two Durable views computes between their frames of one commit. With `StoreStorage`, Durable's `adopt` runs inside the host's batch; the store knows nothing of Durable (no special cases) and needs only its step and flush to stay synchronous. Durable's + them synchronously inside the storage's commit (S1, W1; with `StoreStorage` that is the store's commit step), so a derive over a Durable view travels with it. This needs Durable to publish a commit when it adopts it (owner, 2026-10-26; a pi change, upstream first; explainer `durable-observers/`): inside `adopt`, its trackers, telemetry events and internal observers (view mounts, task graph, scheduler, submissions) run, and the host `observeCommits` observers are dispatched in one microtask queued before the observers, so they run before every wait the commit resolves and every abort it queues, but after the storage's synchronous step, never inside Chord's batch; host listeners and watch listeners have no order between them (owner, 2026-10-26). Durable's public primitive for adapters is a synchronous observer, `observeConversation`, `observeTaskGraph`, `observeDoc` (its `advance(call, ctx)` gets a `WatchCall` and runs inside `adopt`: record only, no awaits, no Session calls, no throws); `watchXXX` is built on it for app code. Chord's `self.follow(attach)` (slice 6b D9) is a source over it. Otherwise a derive over two Durable views computes between their frames of one commit. With `StoreStorage`, Durable's `adopt` runs inside the host's batch; the store knows nothing of Durable (no special cases) and needs only its step and flush to stay synchronous. Durable's state getters (`documentState`, `viewState`, `taskGraph()`) and the `replicatedState` family are removed (W14). Only plain data crosses between Durable and v2 objects. - **The harness is an object in the core facet** (base code): a `Harness` `local` object with empty params, so host lifetime (it runs for the worker's whole life, diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index a638c6db4..4c64947d1 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -47,7 +47,7 @@ Order (owner, 2026-10-22: routines and jobs before the store and reload). | 4b | **Authority and housekeeping** | revocation, cycles across hosts, `host.references`, `host.activity`, size limits, compression | | 5 | **Routines and jobs** | `self.routines`, `self.job` | | 6a | **The store** | the store (backends, tables, queries, writes, commit queue, outbox, documents engine, migrations, export), persisted documents, `self.table`, keyed watches and retirement | -| 6b | **Durable and jobs on the store** | persisted jobs, Durable's `StoreStorage`, the Chord source over Durable's views (`durableView`) | +| 6b | **Durable and jobs on the store** | persisted jobs, Durable's `StoreStorage`, the Chord source over Durable's views (`self.follow`) | | 7 | **Reload and rollout** | plugin loading, staging, swaps and restarts, revisions, `accepts` in use | | 8 | **Registries and the harness** | registries, `self.writable`, Durable's harness, `chord(fn, context)` | | 9 | **Telemetry** | spans, events, metrics across hosts | @@ -172,7 +172,7 @@ deleted (the slice's spec and code are the record). Moved here from `carry-over. - **`StoreStorage`**: Durable's `Storage` on the store, passing Durable's storage conformance suite (memory and across reopen); ids from a `record_ids` table (S10, W13); fork copies resolved before the step (S11); Durable's `adopt` run inside the host's batch. -- **`durableView`**: a Chord source over Durable's synchronous observers (`observeConversation`, `observeTaskGraph`, +- **`self.follow`** (was `durableView`, decision "Slice 6b D9"): a Chord source over Durable's synchronous observers (`observeConversation`, `observeTaskGraph`, `observeDoc`; decision "Durable's synchronous observers"), marked inside Chord's batch in the store and grouped into one batch per Durable commit on the next microtask outside it; `self.follow` (working name) and the Durable harness carried in a registry entry. The task graph's `blocked` refresh arrives outside any commit. From c3d8ce44d3231c7e46071d2d859f41ca7d39d21b Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 11:56:31 +0000 Subject: [PATCH 074/271] docs(chord): slice 8 is registries and host.root; the harness object is p3's --- packages/chord/docs/next/decisions.md | 6 ++++++ packages/chord/docs/next/slices.md | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 97ac5e874..7dbb94b49 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1691,6 +1691,12 @@ validation and copies the v2 kernel added around Delta. `undefined` attach fails the activation as a failed dependency; Durable's `failed` end ends the watches `failed`. Returns a `Watchable` (state member, derive source, `on` source). Supersedes the carry-over's `durableView` and D1's `durableView` part. +- **Slice 8 is "Registries and `host.root`"; the harness object is p3's (owner, 2026-10-27: "we want to get chord done + this week. p3 needs design and is next week. just need all chord primitives in"):** Chord builds the primitives the + harness needs (registries, `self.writable`, `retryOn`, a `RegistryReader` for Durable, `host.root`, `chord(fn, + context)`); the harness object (`Harness.open` over `StoreStorage`, `resume()` after the start barrier, the models, + environments and extensions registries) is p3's base code, shown in Chord only as an example or test. Plan: Chord v2 + (6b, 7, 8, 9) this week, p3's design next week. - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index 4c64947d1..826f223c0 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -49,7 +49,7 @@ Order (owner, 2026-10-22: routines and jobs before the store and reload). | 6a | **The store** | the store (backends, tables, queries, writes, commit queue, outbox, documents engine, migrations, export), persisted documents, `self.table`, keyed watches and retirement | | 6b | **Durable and jobs on the store** | persisted jobs, Durable's `StoreStorage`, the Chord source over Durable's views (`self.follow`) | | 7 | **Reload and rollout** | plugin loading, staging, swaps and restarts, revisions, `accepts` in use | -| 8 | **Registries and the harness** | registries, `self.writable`, Durable's harness, `chord(fn, context)` | +| 8 | **Registries and `host.root`** | registries, `self.writable`, `retryOn`, a `RegistryReader` for Durable, `host.root` and `chord(fn, context)` (the harness object is p3's base code) | | 9 | **Telemetry** | spans, events, metrics across hosts | The playground (`apps/chord-playground`) moves to the new code once slice 4 exists. Round-3 cuts (C3-C16) and @@ -126,7 +126,7 @@ pattern, no code. Decided cuts are noted where they apply. | 22 | relays, size limits per part, compression at the origin, adapted bindings get diffs | 4a (relays), 4b (limits, compression), 7 | | 22 | trace carriers (R5) | 9 | | 23 | HTTP, channels, chunked transfers | after slice 9 (with the first real proof) | -| 24 | Durable integration: harness object, `chord(fn, context)` (Q6), model and env registries | 8 | +| 24 | Durable integration: `chord(fn, context)` (Q6), a `RegistryReader` over composed registries; the harness object and its model and env registries are p3's (base code, owner, 2026-10-27) | 8 (Chord parts), p3 | | 24 | `StoreStorage` | 6b | | 27 | versions table: contracts (L1 done), registries 8, documents 6, jobs 5, link protocol 3a | as listed | | 29 | telemetry spans, events, metrics, carriers | 9 (cost counters: every slice) | @@ -231,7 +231,7 @@ deleted (the slice's spec and code are the record). Moved here from `carry-over. - **Retrying broken failures:** reload retries a broken object as `ref.retry()` does (`ChordObject.retry`: the backoff skipped, an attempt now; holders are told `loading`). (Slices 2, 3b.) -### Slice 8: registries and the harness +### Slice 8: registries and `host.root` - **`host.root(fn, context, { principal? })`**: a call scope for code outside objects, built with its first users (Durable hooks, `chord(fn, context)`); decide what an aborted, uncooperative `fn` means for its references (match From 6a282511362bb61097a7d07e56f4864e7536c933 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 13:08:18 +0000 Subject: [PATCH 075/271] feat(durable): reject input on idle conversations with whenIdle An input with whenIdle: "reject" is refused with the new ConversationIdle when the conversation has no run, before any write, so it never starts a run. With whenBusy: "steer" it steers the current run or is refused. Request-ID deduplication runs first. --- packages/durable/CHANGELOG.md | 1 + packages/durable/README.md | 2 + packages/durable/docs/spec.md | 13 +- packages/durable/src/errors.ts | 11 ++ packages/durable/src/harness/submissions.ts | 12 +- packages/durable/src/harness/types.ts | 10 +- packages/durable/src/index.ts | 9 +- .../test/harness-submit-in-commit.test.ts | 122 ++++++++++++++++++ 8 files changed, 171 insertions(+), 9 deletions(-) create mode 100644 packages/durable/test/harness-submit-in-commit.test.ts diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index e63340b27..f5faab47b 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -109,6 +109,7 @@ - `HarnessOptions.readOnly` opens a Harness as a viewer of a session another process drives: open leaves running tasks as committed instead of recovering them, every commit with writes rejects with `ReadOnlySession` before it reaches Storage, and `resume()` and every call that would run tasks reject with it. - Continue submissions: `submit({ type: "continue" })` starts a run on the context as it is, without a user message, for example after a run that ended without an answer (`pi.live.runFailure`). Only an idle conversation admits one (a busy one throws `ConversationBusy`; it never queues); items a failed run left queued are placed first and join the run. A context that ends in an answer, also a text answer cut off at its output limit, has nothing to continue (`nothing_to_answer`); a cut-off answer with tool calls is continued, as its calls end in error results. Tasks and tools submit one through `ConversationHandle`. The `pi.durable.submit` span starts a root trace for it, as for an input ([#11043](https://github.com/earendil-works/pi/issues/11043)) - Per-conversation turn limits: `AgentState.maxTurns` (`configure({ maxTurns })`, `null` clears; a `RangeError` for anything but an integer 1 or more) bounds the tool rounds one run takes. The round that reaches it runs its tools, then the run ends `unanswered` with the new `RunFailureReason` `turn_limit`, like a failed run, so `settings.afterFailedRun` decides the queued inputs and a continue resumes it on the round's results. The count is `pi.live.run.turns`; every run starts at 0, and an `onYield` continuation starts it over. Task-owned children and forks inherit the limit; `Agent.maxTurns` exposes it ([#11043](https://github.com/earendil-works/pi/issues/11043)) +- `whenIdle: "reject"` on input submissions: an idle conversation rejects the input with the new `ConversationIdle` and writes nothing, so it never starts a run. With `whenBusy: "steer"` it steers the current run or is refused, in one admission instead of a host's check and a separate submit that can race the run's end. A known request ID still returns its submission ([#11149](https://github.com/earendil-works/pi/issues/11149)) ### Changed diff --git a/packages/durable/README.md b/packages/durable/README.md index a60ee6b17..283ccafbb 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -534,11 +534,13 @@ A conversation is busy while a run is working on an input. Submitting to a busy await root.submit({ type: "input", content: "Also run the tests" }, context); // follow-up (default) await root.submit({ type: "input", content: "Use pnpm, not npm", whenBusy: "steer" }, context); await root.submit({ type: "input", content: "Only if idle", whenBusy: "reject" }, context); // throws ConversationBusy +await root.submit({ type: "input", content: "Only if busy", whenBusy: "steer", whenIdle: "reject" }, context); // throws ConversationIdle when idle await root.submit({ type: "write", entry: { kind: "app.note", data: "user opened a file" } }, context); ``` - **Steers** are placed after the current tool round and join the running work. - **Follow-ups** are placed when the run answers, and start the next run. +- **`whenIdle: "reject"`** refuses an input when no run is working, so it never starts a run of its own. With `whenBusy: "steer"`, it steers the current run or throws `ConversationIdle`. - **Writes** append an entry without asking the model anything. A write of only `kind` and `data` changes nothing the model sees, so it is appended at once, even mid-run, and is never queued: a chat message shows in the transcript as soon as it is sent. It can therefore appear before items submitted earlier that are still queued, except behind a queued reset, where it waits so the reset does not hide it. A write with `model`, `head`, or `edits` waits for the next boundary like a steer. - `await submission.abort(context)` withdraws a queued submission. - The [settings](#settings) `steeringMode: "all"` and `followUpMode: "all"` place every queued item at once instead of one per turn. diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index 284009b86..dbe7cb96a 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -367,6 +367,8 @@ type SubmissionDraft = { readonly type: "input"; readonly content: UserInput; readonly whenBusy?: "steer" | "followUp" | "reject"; + /** `reject`: an idle conversation rejects the input with `ConversationIdle` (section 6). */ + readonly whenIdle?: "reject"; readonly entry?: never; } | { @@ -374,6 +376,7 @@ type SubmissionDraft = { readonly entry: EntryDraft; readonly content?: never; readonly whenBusy?: never; + readonly whenIdle?: never; } /** Start a run on the context as it is, without a user message; idle only (section 6). */ | { @@ -381,6 +384,7 @@ type SubmissionDraft = { readonly content?: never; readonly entry?: never; readonly whenBusy?: never; + readonly whenIdle?: never; } ); @@ -2835,6 +2839,7 @@ Admission and terminal transitions: | input, idle with empty inbox | `placed`, with user entry | create run and generation | | continue, idle | `placed`, no entry | place queued items (final boundary); create run and generation with them and itself | | continue, busy | none | reject with `ConversationBusy` | +| input with `whenIdle: "reject"`, idle | none | reject with `ConversationIdle` | | display-only write, no queued reset | `done`, with entry | append entry; no run | | write, idle with empty inbox | `done`, with entry | append entry; no run | | input or other write, busy or non-empty inbox | `queued` | append inbox item | @@ -2848,8 +2853,12 @@ Admission and terminal transitions: `requestId` deduplicates within one conversation before any write; reusing one for the other submission type rejects. A busy input with `whenBusy: "reject"` -writes no record and reports `ConversationBusy`. An idle input queues with mode -`steer` when `whenBusy` is `steer` and `followUp` otherwise. +writes no record and reports `ConversationBusy`. An idle input with +`whenIdle: "reject"` writes no record and reports `ConversationIdle`, also when +the inbox holds queued items, so the input never starts a run: with +`whenBusy: "steer"` it steers the current run or is refused. Idle means +`run === undefined`, the inverse of `busy` above. An idle input queues with +mode `steer` when `whenBusy` is `steer` and `followUp` otherwise. A continue starts a run on the context as it is, without a user message, for example after a run that ended without an answer. It is admitted only while the diff --git a/packages/durable/src/errors.ts b/packages/durable/src/errors.ts index e2a36dfc1..796ae7e1f 100644 --- a/packages/durable/src/errors.ts +++ b/packages/durable/src/errors.ts @@ -66,3 +66,14 @@ export class ConversationBusy extends Error { this.conversationId = conversationId; } } + +/** An input with `whenIdle: "reject"` reached an idle conversation and was not admitted. */ +export class ConversationIdle extends Error { + readonly conversationId: ConversationId; + + constructor(conversationId: ConversationId) { + super(`Conversation ${conversationId} is idle`); + this.name = "ConversationIdle"; + this.conversationId = conversationId; + } +} diff --git a/packages/durable/src/harness/submissions.ts b/packages/durable/src/harness/submissions.ts index 7de0054b8..76379d1b2 100644 --- a/packages/durable/src/harness/submissions.ts +++ b/packages/durable/src/harness/submissions.ts @@ -6,7 +6,7 @@ import { type TelemetryContext, type TelemetrySpan, } from "@earendil-works/pi-telemetry"; -import { ConversationBusy } from "../errors.ts"; +import { ConversationBusy, ConversationIdle } from "../errors.ts"; import type { ObservedCommit, SessionImpl } from "../session/session.ts"; import type { ConversationId, JsonObject, Storage, SubmissionId, SubmissionRecord, Tx } from "../types.ts"; import { startRun } from "./generation.ts"; @@ -188,10 +188,11 @@ function checkInputEntry(entry: InputEntry): void { * Admit a submission inside a commit (spec §6); `Conversation.submit()` and conversation-owned compactions share it. A * known request ID returns its existing submission without writing. A display-only write (only `kind` and `data`) is * appended and settled `done` at once, busy or not, as it changes nothing the model sees; only behind a queued reset it - * queues, so it lands after the reset instead of before it in the context the reset ends. A busy conversation queues anything else in - * `pi.inbox`, or rejects `whenBusy: "reject"` input and any continue with `ConversationBusy`. An idle continue places - * queued items and starts a run with them and itself, placing no entry of its own. An idle conversation with queued items - * queues it behind them and runs a final boundary. Otherwise idle input places its entry (`pi.user`, or the input's + * queues, so it lands after the reset instead of before it in the context the reset ends. A busy conversation queues + * anything else in `pi.inbox`, or rejects `whenBusy: "reject"` input and any continue with `ConversationBusy`; an idle + * one rejects `whenIdle: "reject"` input with `ConversationIdle`. An idle continue places queued items and starts a run + * with them and itself, placing no entry of its own. An idle conversation with queued items queues it behind them and + * runs a final boundary. Otherwise idle input places its entry (`pi.user`, or the input's * own) and starts a run, and an idle write appends its entry and settles `done`, or `stale` when its head reaches * before the active range. */ @@ -235,6 +236,7 @@ export async function admitSubmission( if (run?.trace !== undefined) span.addLink({ context: span.propagation.extract(run.trace) }); } if (busy && draft.type === "input" && draft.whenBusy === "reject") throw new ConversationBusy(conversationId); + if (!busy && draft.type === "input" && draft.whenIdle === "reject") throw new ConversationIdle(conversationId); if (draft.type === "continue") { // Never queued: whether a run can continue depends on how the current one ends. Rejected before any write. if (busy) throw new ConversationBusy(conversationId); diff --git a/packages/durable/src/harness/types.ts b/packages/durable/src/harness/types.ts index 742edb085..8ec9b6c8b 100644 --- a/packages/durable/src/harness/types.ts +++ b/packages/durable/src/harness/types.ts @@ -75,6 +75,11 @@ export type SubmissionDraft = { readonly type: "input"; readonly content: UserInput; readonly whenBusy?: "steer" | "followUp" | "reject"; + /** + * `reject`: an idle conversation (no run) rejects the input with `ConversationIdle` and writes nothing, so the input + * never starts a run. With `whenBusy: "steer"` it steers the current run or is refused. + */ + readonly whenIdle?: "reject"; /** Place the input as this entry; default `pi.user` without data. The kind must not start with `pi.`. */ readonly entry?: InputEntry; } @@ -83,6 +88,7 @@ export type SubmissionDraft = { readonly entry: EntryDraft; readonly content?: never; readonly whenBusy?: never; + readonly whenIdle?: never; } | { /** @@ -96,6 +102,7 @@ export type SubmissionDraft = { readonly content?: never; readonly entry?: never; readonly whenBusy?: never; + readonly whenIdle?: never; } ); @@ -788,7 +795,8 @@ export interface Conversation { /** * Durably admit user input, a passive entry write, or a continue. A busy conversation, or one with queued items, * queues input and writes that affect model context in `pi.inbox`; `whenBusy: "reject"` rejects input with - * `ConversationBusy` instead and writes nothing. A write of only `kind` and `data` (no `model`, `head`, or `edits`) + * `ConversationBusy` instead and writes nothing. An idle conversation rejects `whenIdle: "reject"` input with + * `ConversationIdle` and writes nothing. A write of only `kind` and `data` (no `model`, `head`, or `edits`) * is appended at once, even mid-run, as it changes nothing the model sees. A continue starts a run on the context as * it is, after placing queued items; a busy conversation rejects it with `ConversationBusy`, as it never queues. */ diff --git a/packages/durable/src/index.ts b/packages/durable/src/index.ts index be0ea2317..a08c8ceb5 100644 --- a/packages/durable/src/index.ts +++ b/packages/durable/src/index.ts @@ -9,7 +9,14 @@ export { ToolResultEntry, UserEntry, } from "./entries.ts"; -export { ConversationBusy, ReadAfterWrite, ReadOnlySession, SessionFailed, StorageRequestError } from "./errors.ts"; +export { + ConversationBusy, + ConversationIdle, + ReadAfterWrite, + ReadOnlySession, + SessionFailed, + StorageRequestError, +} from "./errors.ts"; export { AgentDoc, configure, diff --git a/packages/durable/test/harness-submit-in-commit.test.ts b/packages/durable/test/harness-submit-in-commit.test.ts new file mode 100644 index 000000000..8cd6997da --- /dev/null +++ b/packages/durable/test/harness-submit-in-commit.test.ts @@ -0,0 +1,122 @@ +// Admission for hosts that must not start runs: `whenIdle: "reject"` (#11149). +import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { type Conversation, ConversationIdle, defineTool, InboxDoc, LiveDoc } from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import { chatSetup, openChat, unanswered } from "./chat-support.ts"; +import { addTool } from "./harness-support.ts"; +import { ControlledStorage, context } from "./session-support.ts"; +import { deferred } from "./task-support.ts"; + +const MODEL = { provider: "faux", modelId: "faux-1" } as const; + +const failure = (): AssistantMessage => + fauxAssistantMessage([], { stopReason: "error", errorMessage: "Invalid request" }); + +/** Open a chat whose run hangs until aborted, and wait until it works on its first input. */ +async function openBusy(storage = new ControlledStorage()) { + const setup = chatSetup(); + const busy = unanswered(); + setup.faux.setResponses([busy.step]); + const opened = await openChat(storage, setup); + const first = await opened.root.submit({ type: "input", content: "work" }, context); + await busy.reached; + return { ...opened, storage, first }; +} + +describe("whenIdle reject", () => { + // #11149 + it("rejects input to an idle conversation without writing", async () => { + const storage = new ControlledStorage(); + const { harness, root } = await openChat(storage, chatSetup()); + const commits = storage.commits.length; + const rejected = root.submit({ type: "input", content: "hi", whenBusy: "steer", whenIdle: "reject" }, context); + await expect(rejected).rejects.toBeInstanceOf(ConversationIdle); + await expect(rejected).rejects.toMatchObject({ conversationId: root.id }); + expect(storage.commits.length).toBe(commits); + expect((await harness.snapshot(LiveDoc, root.id, context))?.run).toBeUndefined(); + await harness.close(context); + }); + + // #11149 + it("rejects input to an idle conversation that keeps queued items after a failed run", async () => { + const setup = chatSetup(); + const failing = deferred(); + const requested = deferred(); + setup.faux.setResponses([ + async () => { + requested.resolve(); + await failing.promise; + return failure(); + }, + ]); + const { harness, root } = await openChat(new ControlledStorage(), setup); + const first = await root.submit({ type: "input", content: "first" }, context); + await requested.promise; + const queued = await root.submit({ type: "input", content: "queued" }, context); + failing.resolve(); + expect(await first.wait(context)).toMatchObject({ status: "unanswered", reason: "model_error" }); + await harness.waitForIdle(context); + const rejected = root.submit({ type: "input", content: "steer", whenBusy: "steer", whenIdle: "reject" }, context); + await expect(rejected).rejects.toBeInstanceOf(ConversationIdle); + expect(await queued.status(context)).toMatchObject({ status: "queued" }); + expect((await harness.snapshot(LiveDoc, root.id, context))?.run).toBeUndefined(); + await harness.close(context); + }); + + // #11149 + it("steers a working run", async () => { + const { harness, root, first } = await openBusy(); + const steer = await root.submit( + { type: "input", content: "also this", whenBusy: "steer", whenIdle: "reject" }, + context, + ); + expect(await steer.status(context)).toMatchObject({ type: "input", status: "queued" }); + const inbox = await harness.snapshot(InboxDoc, root.id, context); + expect(inbox?.items).toEqual([{ id: steer.id, mode: "steer", content: "also this" }]); + expect((await harness.snapshot(LiveDoc, root.id, context))?.run?.inputs).toEqual([first.id]); + await harness.close(context); + }); + + // #11149 + it("returns the submission of a known request ID, also once idle", async () => { + const setup = chatSetup(); + setup.faux.setResponses([fauxAssistantMessage([fauxText("done")])]); + const { harness, root } = await openChat(new ControlledStorage(), setup); + const first = await root.submit({ type: "input", content: "work", requestId: "r1" }, context); + await first.wait(context); + const draft = { type: "input", content: "work", requestId: "r1", whenBusy: "steer", whenIdle: "reject" } as const; + expect((await root.submit(draft, context)).id).toBe(first.id); + await harness.close(context); + }); + + // #11149 + it("rejects through a tool's conversation handle", async () => { + const setup = chatSetup(); + let other: Conversation | undefined; + let rejection: unknown; + addTool( + setup.registry, + defineTool({ + name: "nudge", + description: "Steers another conversation", + parameters: Type.Object({}), + execute: async (_args, api, callContext) => { + const handle = (await api.conversation(other!.id, callContext))!; + const draft = { type: "input", content: "nudge", whenBusy: "steer", whenIdle: "reject" } as const; + rejection = await handle.submit(draft, callContext).catch((error: unknown) => error); + return { output: [{ type: "text", text: "nudged" }] }; + }, + }), + ); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("nudge", {}, { id: "c1" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("ok")]), + ]); + const { harness, root } = await openChat(new ControlledStorage(), setup); + other = await harness.createConversation({ ownership: { kind: "ownerless" }, agent: { model: MODEL } }, context); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(rejection).toBeInstanceOf(ConversationIdle); + expect((await harness.snapshot(LiveDoc, other.id, context))?.run).toBeUndefined(); + await harness.close(context); + }); +}); From cc9c523107887455ec97734592cf0e08d46be511 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 13:08:38 +0000 Subject: [PATCH 076/271] feat(durable): admit submissions inside the caller's commit with submit(tx, draft) Conversation.submit(tx, draft) and ConversationHandle.submit(tx, draft) run the same admission inside the caller's commit and return the submission ID, so a host's own checks and writes and the admission are stored together or not at all. Admission must come before the commit's first table write, once per commit, and throws ReadAfterWrite otherwise on every path. A first argument that is not a draft is taken as a transaction; one of another Session, or a wrapped one, is refused. The handle form also rejects once its invocation is aborted. The pi.durable.submit span covers the admission only. closes #11149 --- packages/durable/CHANGELOG.md | 1 + packages/durable/README.md | 15 +- packages/durable/docs/spec.md | 36 ++ packages/durable/src/errors.ts | 10 +- packages/durable/src/harness/harness.ts | 40 +- packages/durable/src/harness/submissions.ts | 61 +- packages/durable/src/harness/telemetry.ts | 2 +- packages/durable/src/harness/types.ts | 18 +- packages/durable/src/session/session.ts | 5 + packages/durable/src/session/transaction.ts | 20 +- .../test/harness-submit-in-commit.test.ts | 534 +++++++++++++++++- 11 files changed, 700 insertions(+), 42 deletions(-) diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index f5faab47b..6511187f0 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -110,6 +110,7 @@ - Continue submissions: `submit({ type: "continue" })` starts a run on the context as it is, without a user message, for example after a run that ended without an answer (`pi.live.runFailure`). Only an idle conversation admits one (a busy one throws `ConversationBusy`; it never queues); items a failed run left queued are placed first and join the run. A context that ends in an answer, also a text answer cut off at its output limit, has nothing to continue (`nothing_to_answer`); a cut-off answer with tool calls is continued, as its calls end in error results. Tasks and tools submit one through `ConversationHandle`. The `pi.durable.submit` span starts a root trace for it, as for an input ([#11043](https://github.com/earendil-works/pi/issues/11043)) - Per-conversation turn limits: `AgentState.maxTurns` (`configure({ maxTurns })`, `null` clears; a `RangeError` for anything but an integer 1 or more) bounds the tool rounds one run takes. The round that reaches it runs its tools, then the run ends `unanswered` with the new `RunFailureReason` `turn_limit`, like a failed run, so `settings.afterFailedRun` decides the queued inputs and a continue resumes it on the round's results. The count is `pi.live.run.turns`; every run starts at 0, and an `onYield` continuation starts it over. Task-owned children and forks inherit the limit; `Agent.maxTurns` exposes it ([#11043](https://github.com/earendil-works/pi/issues/11043)) - `whenIdle: "reject"` on input submissions: an idle conversation rejects the input with the new `ConversationIdle` and writes nothing, so it never starts a run. With `whenBusy: "steer"` it steers the current run or is refused, in one admission instead of a host's check and a separate submit that can race the run's end. A known request ID still returns its submission ([#11149](https://github.com/earendil-works/pi/issues/11149)) +- `Conversation.submit(tx, draft)` admits a submission inside the caller's commit (`Harness.commit()`, `Conversation.commit()`) with the same rules as `submit(draft, context)`, and returns its ID: a host checks its own documents and `pi.live`, admits, and writes its own state in one commit, which stores all of it or none. `ConversationHandle.submit(tx, draft)` does the same inside a task's or tool's own commit. The Harness supplies the clock, queue modes, the `pi.durable.submit` span (covering the admission only; its `result` is provisional until the commit is stored), and the request for progress. It must come before the commit's first table write, once per commit, and throws `ReadAfterWrite` otherwise on every path ([#11149](https://github.com/earendil-works/pi/issues/11149)) ### Changed diff --git a/packages/durable/README.md b/packages/durable/README.md index 283ccafbb..3403a4997 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -81,7 +81,7 @@ Every async call takes a Chord `Context`, which carries cancellation and its cur `HarnessOptions.telemetry` accepts a host-owned `TelemetryContext` from `@earendil-works/pi-telemetry`, with optional `propagation` and `metrics` capabilities; omitted, it is a no-op. `DURABLE_TELEMETRY_SCHEMA`, exported from the package root, documents its typed spans, events, and metrics. Attributes contain IDs and control metadata, never messages, arguments, or results; IDs point to the transcript. -- `pi.durable.submit` covers the public submit call and its commit. Each input starts a new root trace with a link to its caller's current span; a write is a child of its caller. Duplicate `requestId` preserves the original submission and carrier. Internal admissions, such as compaction writes, emit events on their phase span without a submit span. +- `pi.durable.submit` covers the public submit call and its commit; for `submit(tx, draft)` it covers the admission only, its `result` is provisional until your commit is stored, and the commit's events go to your commit's context. Each input starts a new root trace with a link to its caller's current span; a write is a child of its caller. Duplicate `requestId` preserves the original submission and carrier. Internal admissions, such as compaction writes, emit events on their phase span without a submit span. - `pi.durable.phase` covers one run or abort phase of every task, including its decision commit, never the next phase, a durable wait, or a restart. Its attributes identify the task, phase, attempt when present, `resumed`, and outcome. Tool phases also identify the tool and call. Nested tasks and background compactions inherit the creating phase; model requests receive that phase context. Hooks add no spans. - Failed phase outcomes set error status with a fixed category (`task_faulted`, `task_failed`, or `task_orphaned`) and the outcome's diagnostic message, truncated to 500 characters (the reason for an orphaned task). Diagnostics can contain caller content; no error detail or exception name is recorded. An `aborted` outcome keeps status `ok`, including interruption by close or cancellation. - Every `onReport` report also emits a `pi.durable.report` event on its current span. Without a span, the NOOP span drops the event; no root report span is created. Only `type` (the report's `kind`), optional fixed `reason`, `extension`, and `task.id` are recorded; no error payload is included. `pi.durable.reports` counts all reports by `type` only, including those without a span. Clock failures report without a phase context. The `onReport` callback is unchanged. @@ -541,6 +541,19 @@ await root.submit({ type: "write", entry: { kind: "app.note", data: "user opened - **Steers** are placed after the current tool round and join the running work. - **Follow-ups** are placed when the run answers, and start the next run. - **`whenIdle: "reject"`** refuses an input when no run is working, so it never starts a run of its own. With `whenBusy: "steer"`, it steers the current run or throws `ConversationIdle`. +- **`submit(tx, draft)`** admits inside your own commit, with the same rules, and returns the submission's ID. Your checks, the admission, and your writes are stored together or not at all: + + ```typescript + const id = await harness.commit(async (tx) => { + const chat = await tx.doc(MyChat, root.id); // your own conversation document + if (chat.muted.includes(userId)) throw new Error("Muted"); + const id = await root.submit(tx, { type: "input", content: text, whenBusy: "steer", whenIdle: "reject" }); + chat.sent.push({ id, userId }); // your own state, same commit + return id; + }, context); + ``` + + Call it before the commit's own table writes (`appendEntry()`, `createTask()`, another admission), once per commit: otherwise it throws `ReadAfterWrite`, on every path. Documents may be read and written before and after it. A rejection (`ConversationBusy`, `ConversationIdle`) throws before any write, so you may catch it. Until your commit is stored the admission is provisional: if your callback throws later, no submission exists. Tasks and tools do the same in their own commits through `api.conversation(id)`; entries the admission places carry the task's `byTaskId`. - **Writes** append an entry without asking the model anything. A write of only `kind` and `data` changes nothing the model sees, so it is appended at once, even mid-run, and is never queued: a chat message shows in the transcript as soon as it is sent. It can therefore appear before items submitted earlier that are still queued, except behind a queued reset, where it waits so the reset does not hide it. A write with `model`, `head`, or `edits` waits for the next boundary like a steer. - `await submission.abort(context)` withdraws a queued submission. - The [settings](#settings) `steeringMode: "all"` and `followUpMode: "all"` place every queued item at once instead of one per turn. diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index dbe7cb96a..3ef2b232f 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -709,6 +709,8 @@ type HooksOf = K extends Task interface Conversation { readonly id: ConversationId; submit(submission: SubmissionDraft, context: Context): Promise; + /** Admit inside the caller's commit, with the same rules; stored with that commit (section 6). */ + submit(tx: Tx, submission: SubmissionDraft): Promise; /** Resolved with the current registry snapshot and settings. */ agent(context: Context): Promise; /** `configure()` in its own commit. */ @@ -1921,6 +1923,10 @@ Table rules: - Any table read after the first table write throws `ReadAfterWrite`. - Document access and read-your-writes remain available after table writes. - Creation methods return their created ID/record; callers do not read it back. +- `Conversation.submit(tx, draft)` and `ConversationHandle.submit(tx, draft)` + come before the commit's first table write, another admission's included; + otherwise they throw `ReadAfterWrite`, whether or not the admission would read + a table (section 6). Storage ownership: @@ -2860,6 +2866,25 @@ the inbox holds queued items, so the input never starts a run: with `run === undefined`, the inverse of `busy` above. An idle input queues with mode `steer` when `whenBusy` is `steer` and `followUp` otherwise. +`Conversation.submit(tx, draft)` runs the same admission inside the caller's +commit instead of one of its own and returns the submission's ID; a task or tool +does so through `ConversationHandle.submit(tx, draft)` in its own commit. The +Harness supplies the clock, queue modes, the `pi.durable.submit` span (same +parent, links, and trace carrier as `submit(draft, context)`), and the request +for progress a started run needs. The admission's writes, a started run +included, are stored with the caller's other writes in that commit, or not at +all: a host can check its own documents and `pi.live`, admit, and record the +submission in one commit. A rejection throws before any write. Admission must +precede the commit's first table write, another admission's included: it throws +`ReadAfterWrite` otherwise, on every path, also where it would read no table (a +busy input without request ID or trace). Entries it appends inside a task's or +tool's commit carry that task's `byTaskId`, as the commit's own appends do. The +span covers the admission only: it ends before the caller's commit is stored, +its `result` is provisional (a later throw in the callback or a storage failure +leaves no submission), and the commit's events (`pi.durable.entry`, +`pi.durable.submission`) go to the caller's commit context. A first argument +that is not a draft is taken as a transaction; one of another Session throws. + A continue starts a run on the context as it is, without a user message, for example after a run that ended without an answer. It is admitted only while the conversation is idle and never queues: whether a run can continue depends on how @@ -3387,6 +3412,11 @@ interface ConversationHandle { readonly id: ConversationId; /** Any submission, as `Conversation.submit()` takes: a task may also continue a conversation it owns. */ submit(submission: SubmissionDraft, context: Context): Promise; + /** + * Inside the invocation's commit (`runtime.commit()`, a tool's `api.commit()`), as `Conversation.submit(tx, ...)`; + * appended entries carry the task's `byTaskId`. Rejects once the invocation is aborted or ended. + */ + submit(tx: Tx, submission: SubmissionDraft): Promise; abort(context: Context): Promise; waitForIdle(context: Context): Promise; } @@ -6007,6 +6037,12 @@ These are contracts, not invitations to add defensive machinery: unsupported. - **Read after write:** read every required table row before the first table write. Document drafts remain usable afterward; table reads do not. +- **Admission inside a commit:** `submit(tx, draft)` throws `ReadAfterWrite` + after the commit's first table write: one admission per commit, before the + commit's own `appendEntry()`, `createTask()`, or `createSubmission()`. The + admission is provisional until the commit is stored: a later throw in the + callback, or a storage failure, discards it with the rest of the commit, + though its `pi.durable.submit` span already ended with a `result`. - **Writing to a busy conversation:** only the Harness appends to a conversation with an active run. Raw entries appended by `Harness.commit()` or a custom task while a generation prepares its request can misplace its system prompt entries; diff --git a/packages/durable/src/errors.ts b/packages/durable/src/errors.ts index 796ae7e1f..622b8def6 100644 --- a/packages/durable/src/errors.ts +++ b/packages/durable/src/errors.ts @@ -13,10 +13,14 @@ export function errorMessage(error: unknown): string { } } -/** A transaction read a table after its first table write. Read every required row before writing. */ +/** + * A transaction read a table after its first table write, or admitted a submission (`submit(tx, draft)`) after it. Read + * every required row, and admit, before writing. + */ export class ReadAfterWrite extends Error { - constructor(method: string) { - super(`Tx.${method}() cannot read tables after the first table write`); + /** `operation` names the call, such as `Tx.task()`. */ + constructor(operation: string) { + super(`${operation} cannot read tables after the first table write`); this.name = "ReadAfterWrite"; } } diff --git a/packages/durable/src/harness/harness.ts b/packages/durable/src/harness/harness.ts index a82e40fe7..055c05bd7 100644 --- a/packages/durable/src/harness/harness.ts +++ b/packages/durable/src/harness/harness.ts @@ -123,8 +123,12 @@ class ConversationImpl implements Conversation { return this.#host.harness.commitWith((tx) => configure(tx, this.id, change), context); } - submit(submission: SubmissionDraft, context: Context): Promise { - return this.#host.submissions.submit(this.id, submission, context); + submit(submission: SubmissionDraft, context: Context): Promise; + submit(tx: Tx, submission: SubmissionDraft): Promise; + submit(first: SubmissionDraft | Tx, second: Context | SubmissionDraft): Promise { + // The overloads pair a draft with a context, and anything else (a transaction) with a draft. + if ("type" in first) return this.#host.submissions.submit(this.id, first, second as Context); + return this.#host.submissions.admit(first, this.id, second as SubmissionDraft); } async compact(instructions: string | undefined, context: Context): Promise> { @@ -536,18 +540,30 @@ function boundConversation( return operation(bind(context)); }; }; + function submit(draft: SubmissionDraft, context: Context): Promise; + function submit(tx: Tx, draft: SubmissionDraft): Promise; + async function submit( + first: SubmissionDraft | Tx, + second: Context | SubmissionDraft, + ): Promise { + binding.check(); + // The overloads pair a draft with a context, and anything else (a transaction) with a draft. + if (!("type" in first)) { + // The caller's commit carries its own context, so the invocation's abort is checked here. + binding.signal.throwIfAborted(); + return submissions.admit(first, id, second as SubmissionDraft); + } + const submission = await submissions.submit(id, first, bind(second as Context)); + return { + id: submission.id, + status: bound((callContext) => submission.status(callContext)), + wait: bound((callContext) => submission.wait(callContext)), + abort: bound((callContext) => submission.abort(callContext)), + }; + } return { id, - submit: async (draft, context) => { - binding.check(); - const submission = await submissions.submit(id, draft, bind(context)); - return { - id: submission.id, - status: bound((callContext) => submission.status(callContext)), - wait: bound((callContext) => submission.wait(callContext)), - abort: bound((callContext) => submission.abort(callContext)), - }; - }, + submit, abort: async (context, options) => { binding.check(); return tasks.abortConversation(id, options?.background === true, bind(context)); diff --git a/packages/durable/src/harness/submissions.ts b/packages/durable/src/harness/submissions.ts index 76379d1b2..df0363e67 100644 --- a/packages/durable/src/harness/submissions.ts +++ b/packages/durable/src/harness/submissions.ts @@ -6,7 +6,7 @@ import { type TelemetryContext, type TelemetrySpan, } from "@earendil-works/pi-telemetry"; -import { ConversationBusy, ConversationIdle } from "../errors.ts"; +import { ConversationBusy, ConversationIdle, ReadAfterWrite } from "../errors.ts"; import type { ObservedCommit, SessionImpl } from "../session/session.ts"; import type { ConversationId, JsonObject, Storage, SubmissionId, SubmissionRecord, Tx } from "../types.ts"; import { startRun } from "./generation.ts"; @@ -66,6 +66,36 @@ export class Submissions { /** Admit a submission in one commit; see `admitSubmission()`. */ async submit(conversationId: ConversationId, draft: SubmissionDraft, context: Context): Promise { + const { id } = await this.#traced(conversationId, draft, context, (span) => + this.#session.commitWith( + (tx) => admitSubmission(tx, conversationId, draft, this.#now(), this.#queueModes(), span), + withTelemetry(span, context), + ), + ); + return new SubmissionHandle(id, this); + } + + /** + * Admit a submission inside the caller's commit `tx`, with the rules of `submit()`. The admission is stored or lost + * with that commit. It must come before the commit's first table write, another admission's included. + */ + async admit(tx: Tx, conversationId: ConversationId, draft: SubmissionDraft): Promise { + if (!this.#session.ownsTransaction(tx)) throw new Error("The transaction belongs to another Session"); + // Enforced on every path: admission reads tables only on some (request IDs, an idle boundary, a steer's trace). + if (tx.hasTableWrite) throw new ReadAfterWrite("submit(tx, draft)"); + const admission = await this.#traced(conversationId, draft, tx.context, (span) => + admitSubmission(tx, conversationId, draft, this.#now(), this.#queueModes(), span), + ); + return admission.id; + } + + /** Run one admission in its `pi.durable.submit` span, asking for progress first, as the run it starts needs it. */ + #traced( + conversationId: ConversationId, + draft: SubmissionDraft, + context: Context, + admit: (span: TelemetrySpan) => Promise, + ): Promise { this.#resume(); // Inputs and continues start runs: root traces linked to their callers. const startsRun = draft.type !== "write"; @@ -79,12 +109,9 @@ export class Submissions { async (span) => { if (startsRun) span.addLink({ context: context.telemetry }); try { - const { id, status } = await this.#session.commitWith( - (tx) => admitSubmission(tx, conversationId, draft, this.#now(), this.#queueModes(), span), - withTelemetry(span, context), - ); - span.setAttributes({ "submission.id": id, result: status }); - return new SubmissionHandle(id, this); + const admission = await admit(span); + span.setAttributes({ "submission.id": admission.id, result: admission.status }); + return admission; } catch (error) { // Error text may contain caller content. Record only the status. span.setStatus({ status: "error" }); @@ -185,16 +212,16 @@ function checkInputEntry(entry: InputEntry): void { } /** - * Admit a submission inside a commit (spec §6); `Conversation.submit()` and conversation-owned compactions share it. A - * known request ID returns its existing submission without writing. A display-only write (only `kind` and `data`) is - * appended and settled `done` at once, busy or not, as it changes nothing the model sees; only behind a queued reset it - * queues, so it lands after the reset instead of before it in the context the reset ends. A busy conversation queues - * anything else in `pi.inbox`, or rejects `whenBusy: "reject"` input and any continue with `ConversationBusy`; an idle - * one rejects `whenIdle: "reject"` input with `ConversationIdle`. An idle continue places queued items and starts a run - * with them and itself, placing no entry of its own. An idle conversation with queued items queues it behind them and - * runs a final boundary. Otherwise idle input places its entry (`pi.user`, or the input's - * own) and starts a run, and an idle write appends its entry and settles `done`, or `stale` when its head reaches - * before the active range. + * Admit a submission inside a commit (spec §6); both `Conversation.submit()` forms and conversation-owned compactions + * share it. A known request ID returns its existing submission without writing. A display-only write (only `kind` and + * `data`) is appended and settled `done` at once, busy or not, as it changes nothing the model sees; only behind a + * queued reset it queues, so it lands after the reset instead of before it in the context the reset ends. A busy + * conversation queues anything else in `pi.inbox`, or rejects `whenBusy: "reject"` input and any continue with + * `ConversationBusy`; an idle one rejects `whenIdle: "reject"` input with `ConversationIdle`. An idle continue places + * queued items and starts a run with them and itself, placing no entry of its own. An idle conversation with queued + * items queues it behind them and runs a final boundary. Otherwise idle input places its entry (`pi.user`, or the + * input's own) and starts a run, and an idle write appends its entry and settles `done`, or `stale` when its head + * reaches before the active range. */ export async function admitSubmission( tx: Tx, diff --git a/packages/durable/src/harness/telemetry.ts b/packages/durable/src/harness/telemetry.ts index 0c5f60ec6..baebaae0d 100644 --- a/packages/durable/src/harness/telemetry.ts +++ b/packages/durable/src/harness/telemetry.ts @@ -52,7 +52,7 @@ export const DURABLE_TELEMETRY_SCHEMA = defineTelemetrySchema({ spans: { "pi.durable.submit": { description: - "Public submission admission and its commit. Inputs and continues start root traces linked to their callers; writes are children. Internal admissions emit phase events only.", + "Public submission admission: with its commit for submit(draft, context); the admission only for submit(tx, draft), whose result is provisional (a later throw or storage failure of the caller's commit leaves no submission) and whose commit events go to the caller's commit context. Inputs and continues start root traces linked to their callers; writes are children. Internal admissions emit phase events only.", parents: { kind: "any" }, startAttributes: { "submission.type": { diff --git a/packages/durable/src/harness/types.ts b/packages/durable/src/harness/types.ts index 8ec9b6c8b..3306ffe09 100644 --- a/packages/durable/src/harness/types.ts +++ b/packages/durable/src/harness/types.ts @@ -152,12 +152,15 @@ export type HooksOf = K extends Task ? /** * Invocation-bound conversation operations for tasks and tools. Rejects after the invocation ends. `submit` takes any - * submission, as `Conversation.submit()` does: a subagent's conversation is an ordinary one, so a tool may also continue - * it. + * submission, in both forms of `Conversation.submit()`: a subagent's conversation is an ordinary one, so a tool may also + * continue it, and a task may admit a submission in the same commit as its own writes (`runtime.commit()`); entries + * the admission appends there carry the task's `byTaskId`. The transaction form also rejects once the invocation is + * aborted. */ export interface ConversationHandle { readonly id: ConversationId; submit(submission: SubmissionDraft, context: Context): Promise; + submit(tx: Tx, submission: SubmissionDraft): Promise; /** `Conversation.abort()`: withdraw queued inputs, abort the ordinary ownership scope, and wait until it is idle. */ abort(context: Context, options?: ConversationAbortOptions): Promise; /** Resolve when the conversation's ordinary ownership scope has no live non-background task. */ @@ -801,6 +804,17 @@ export interface Conversation { * it is, after placing queued items; a busy conversation rejects it with `ConversationBusy`, as it never queues. */ submit(submission: SubmissionDraft, context: Context): Promise; + /** + * Admit a submission inside the caller's commit `tx` (`Harness.commit()`, `Conversation.commit()`, a task's + * `runtime.commit()`), with the same rules, and return its ID: the admission is stored with that commit, or not at all + * when the callback throws later or the commit fails. Call it before the commit's first table write (`appendEntry()`, + * `createTask()`, `createSubmission()`, another admission): otherwise it throws `ReadAfterWrite`, on every path. + * Documents may be read and written before and after it. A rejection (`ConversationBusy`, `ConversationIdle`) throws + * before any write, so the callback may catch it. Do not call `Submission` or other Harness reads inside the callback: + * they wait for the commit holding the line. Its `pi.durable.submit` span covers the admission only, so its `result` + * is provisional. Throws for a transaction of another Session. + */ + submit(tx: Tx, submission: SubmissionDraft): Promise; /** * Admit a write of a `pi.reset` entry that starts a new context, carrying `handoff` as a user message when given. * Resolves after admission; while busy, it is placed at the next boundary. diff --git a/packages/durable/src/session/session.ts b/packages/durable/src/session/session.ts index 36fd435d0..5f5df8778 100644 --- a/packages/durable/src/session/session.ts +++ b/packages/durable/src/session/session.ts @@ -344,6 +344,11 @@ export class SessionImpl implements Session { return ms; } + /** Internal: whether `tx` is the transaction of one of this Session's commits. */ + ownsTransaction(tx: Tx): tx is Transaction { + return Transaction.ownedBy(tx, this.#host); + } + /** Internal: run a read-only job on the mutation line so multi-read derivations observe one committed state. */ readOnLine(job: () => Promise): Promise { try { diff --git a/packages/durable/src/session/transaction.ts b/packages/durable/src/session/transaction.ts index 79f91ada8..857728fba 100644 --- a/packages/durable/src/session/transaction.ts +++ b/packages/durable/src/session/transaction.ts @@ -254,6 +254,24 @@ export class Transaction implements Tx { this.#scope = scope; } + /** Internal: the context of the commit this transaction belongs to. */ + get context(): Context { + return this.#context; + } + + /** Internal: whether a table write was staged, after which table reads throw `ReadAfterWrite`. */ + get hasTableWrite(): boolean { + return this.#hasTableWrite; + } + + /** + * Internal: whether `tx` is a transaction of a commit of the Session behind `host`. A brand check, so a proxy or + * another wrapper of a transaction is not one. + */ + static ownedBy(tx: Tx, host: TransactionHost): tx is Transaction { + return #host in tx && tx.#host === host; + } + // ─── Table reads ──────────────────────────────────────────────────────── conversation(id: ConversationId): Promise { @@ -1063,7 +1081,7 @@ export class Transaction implements Tx { #read(method: string, read: () => Promise): Promise { try { this.#assertOpen(); - if (this.#hasTableWrite) throw new ReadAfterWrite(method); + if (this.#hasTableWrite) throw new ReadAfterWrite(`Tx.${method}()`); return this.#track(read()); } catch (error) { return Promise.reject(error); diff --git a/packages/durable/test/harness-submit-in-commit.test.ts b/packages/durable/test/harness-submit-in-commit.test.ts index 8cd6997da..3d8ad2986 100644 --- a/packages/durable/test/harness-submit-in-commit.test.ts +++ b/packages/durable/test/harness-submit-in-commit.test.ts @@ -1,9 +1,35 @@ -// Admission for hosts that must not start runs: `whenIdle: "reject"` (#11149). -import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; -import { type Conversation, ConversationIdle, defineTool, InboxDoc, LiveDoc } from "@earendil-works/pi-durable"; +// Admission for hosts that must not start runs: `whenIdle: "reject"`, and `submit(tx, draft)` inside the caller's +// commit (#11149). +import { withTelemetry } from "@earendil-works/chord/context"; +import { + type AssistantMessage, + type FauxResponseStep, + fauxAssistantMessage, + fauxText, + fauxToolCall, + Type, +} from "@earendil-works/pi-ai"; +import { + type Conversation, + ConversationBusy, + type ConversationHandle, + ConversationIdle, + defineDoc, + defineTask, + defineTool, + Harness, + InboxDoc, + LiveDoc, + MemoryStorage, + ReadAfterWrite, + type SubmissionId, + type Tx, + UserEntry, +} from "@earendil-works/pi-durable"; +import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; import { describe, expect, it } from "vitest"; -import { chatSetup, openChat, unanswered } from "./chat-support.ts"; -import { addTool } from "./harness-support.ts"; +import { type ChatSetup, chatSetup, openChat, unanswered, waitFor } from "./chat-support.ts"; +import { addTask, addTool } from "./harness-support.ts"; import { ControlledStorage, context } from "./session-support.ts"; import { deferred } from "./task-support.ts"; @@ -12,6 +38,17 @@ const MODEL = { provider: "faux", modelId: "faux-1" } as const; const failure = (): AssistantMessage => fauxAssistantMessage([], { stopReason: "error", errorMessage: "Invalid request" }); +/** A host's own conversation document. */ +const Sent = defineDoc<{ ids: number[] }>({ + checkpointWhen: () => false, + kind: "test.sent", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ ids: [] }), +}); + /** Open a chat whose run hangs until aborted, and wait until it works on its first input. */ async function openBusy(storage = new ControlledStorage()) { const setup = chatSetup(); @@ -23,6 +60,48 @@ async function openBusy(storage = new ControlledStorage()) { return { ...opened, storage, first }; } +/** Open a chat whose Harness records telemetry. */ +async function openTraced(setup: ChatSetup, telemetry: InMemoryTelemetryContext) { + const harness = await Harness.open( + new MemoryStorage(), + { models: setup.models, registry: setup.registry, settings: setup.settings, telemetry }, + context, + ); + const root = await harness.root(context, { agent: { model: MODEL } }); + return { harness, root }; +} + +/** Install tool `hold`, which reports its call and waits for `release`; its first call answers `tool`. */ +function addHold(setup: ChatSetup) { + const reached = deferred(); + const release = deferred(); + addTool( + setup.registry, + defineTool({ + name: "hold", + description: "Holds until released", + parameters: Type.Object({}), + execute: async () => { + reached.resolve(); + await release.promise; + return { output: [] }; + }, + }), + ); + const call = fauxAssistantMessage([fauxToolCall("hold", {}, { id: "c1" })], { stopReason: "toolUse" }); + return { reached, release, call }; +} + +/** Settled status of submission `id`, polled without asking for progress (`Submission.wait()` would). */ +async function settledStatus(harness: Harness, id: SubmissionId) { + const submission = (await harness.submission(id, context))!; + await waitFor(async () => { + const { status } = await submission.status(context); + return status === "done" || status === "unanswered"; + }); + return submission.status(context); +} + describe("whenIdle reject", () => { // #11149 it("rejects input to an idle conversation without writing", async () => { @@ -86,6 +165,8 @@ describe("whenIdle reject", () => { await first.wait(context); const draft = { type: "input", content: "work", requestId: "r1", whenBusy: "steer", whenIdle: "reject" } as const; expect((await root.submit(draft, context)).id).toBe(first.id); + // Deduplication runs before the idle rejection in the caller's commit too. + expect(await harness.commit((tx) => root.submit(tx, draft), context)).toBe(first.id); await harness.close(context); }); @@ -120,3 +201,446 @@ describe("whenIdle reject", () => { await harness.close(context); }); }); + +describe("submit inside the caller's commit", () => { + // #11149 + it("admits with the host's own check and writes in one commit", async () => { + const { harness, root, storage, first } = await openBusy(); + const commits = storage.commits.length; + const id = await harness.commit(async (tx) => { + if ((await tx.doc(LiveDoc, root.id)).run === undefined) throw new Error("not working"); + // The host's own document write may come before the admission. + const sent = await tx.doc(Sent, root.id); + const admitted = await root.submit(tx, { type: "input", content: "steer", whenBusy: "steer" }); + sent.ids.push(admitted); + // The host's own table write comes after it. + await tx.appendEntry(root.id, { kind: "app.sent", data: { submission: admitted } }); + return admitted; + }, context); + expect(storage.commits.length).toBe(commits + 1); + const writes = storage.commits.at(-1)!; + expect(writes.some((write) => write.type === "submission" && write.value.id === id)).toBe(true); + expect(writes.some((write) => write.type === "entry" && write.value.kind === "app.sent")).toBe(true); + expect((await harness.snapshot(Sent, root.id, context))?.ids).toEqual([id]); + const inbox = await harness.snapshot(InboxDoc, root.id, context); + expect(inbox?.items).toEqual([{ id, mode: "steer", content: "steer" }]); + expect((await harness.snapshot(LiveDoc, root.id, context))?.run?.inputs).toEqual([first.id]); + await harness.close(context); + }); + + // #11149 + it("is rolled back with the commit when the callback throws later", async () => { + const storage = new ControlledStorage(); + const { harness, root } = await openChat(storage, chatSetup()); + const commits = storage.commits.length; + const failed = harness.commit(async (tx) => { + await root.submit(tx, { type: "input", content: "hi", requestId: "r1" }); + throw new Error("host check failed"); + }, context); + await expect(failed).rejects.toThrow("host check failed"); + expect(storage.commits.length).toBe(commits); + expect((await harness.snapshot(LiveDoc, root.id, context))?.run).toBeUndefined(); + expect(await harness.commit((tx) => tx.submissionByRequest(root.id, "r1"), context)).toBeUndefined(); + await harness.close(context); + }); + + // #11149 + it("leaves no submission and no run when Storage fails the commit, also after reopening", async () => { + const storage = new ControlledStorage(); + const setup = chatSetup(); + const { harness, root } = await openChat(storage, setup); + const failure = new Error("disk full"); + storage.failNextCommit(failure); + const failed = harness.commit(async (tx) => { + await root.submit(tx, { type: "input", content: "hi", requestId: "r1" }); + await tx.appendEntry(root.id, { kind: "app.sent" }); + }, context); + await expect(failed).rejects.toBe(failure); + await harness.closed; + const reopened = await openChat(storage.reopen(), setup); + expect(reopened.root.id).toBe(root.id); + const found = await reopened.harness.commit(async (tx) => { + const submission = await tx.submissionByRequest(root.id, "r1"); + const tasks = await tx.scanTasks({ conversationId: root.id }, 10); + const entries = await tx.scanEntries({ conversationId: root.id }, 10); + return { submission, tasks: tasks.items, entries: entries.items }; + }, context); + expect(found).toEqual({ submission: undefined, tasks: [], entries: [] }); + expect((await reopened.harness.snapshot(LiveDoc, root.id, context))?.run).toBeUndefined(); + await reopened.harness.close(context); + }); + + // #11149 + it("starts the run of an idle input once the commit is stored", async () => { + const setup = chatSetup(); + setup.faux.setResponses([fauxAssistantMessage([fauxText("answer")])]); + // No `resume()`: the admission asks for progress itself. + const { harness, root } = await openChat(new ControlledStorage(), setup); + const id = await harness.commit((tx) => root.submit(tx, { type: "input", content: "hi" }), context); + expect(await settledStatus(harness, id)).toMatchObject({ type: "input", status: "done" }); + await harness.close(context); + }); + + // #11149 + it("queues a model-visible write while busy and places it at the next boundary", async () => { + const setup = chatSetup(); + const hold = addHold(setup); + setup.faux.setResponses([hold.call, fauxAssistantMessage([fauxText("answer")])]); + const { harness, root } = await openChat(new ControlledStorage(), setup); + const first = await root.submit({ type: "input", content: "work" }, context); + await hold.reached.promise; + const note = { role: "user", content: "a note the model sees", timestamp: 1 } as const; + const id = await harness.commit( + (tx) => root.submit(tx, { type: "write", entry: { kind: "test.note", model: [note] } }), + context, + ); + expect((await harness.snapshot(InboxDoc, root.id, context))?.items.map((item) => item.id)).toEqual([id]); + hold.release.resolve(); + expect(await first.wait(context)).toMatchObject({ status: "done" }); + expect(await settledStatus(harness, id)).toMatchObject({ + type: "write", + status: "done", + entry: expect.any(Number), + }); + await harness.close(context); + }); + + // #11149 + it("deduplicates request IDs", async () => { + const { harness, root, storage } = await openBusy(); + const draft = { type: "input", content: "steer", whenBusy: "steer", requestId: "r1" } as const; + const first = await harness.commit((tx) => root.submit(tx, draft), context); + const commits = storage.commits.length; + const again = await harness.commit((tx) => root.submit(tx, draft), context); + expect(again).toBe(first); + expect(storage.commits.length).toBe(commits); + await harness.close(context); + }); + + // #11149 + it("rejects before any write, so the callback may catch the rejection", async () => { + const storage = new ControlledStorage(); + const { harness, root } = await openChat(storage, chatSetup()); + const commits = storage.commits.length; + const outcome = await harness.commit(async (tx) => { + try { + await root.submit(tx, { type: "input", content: "hi", whenBusy: "steer", whenIdle: "reject" }); + return "admitted"; + } catch (error) { + return error instanceof ConversationIdle ? "idle" : "other"; + } + }, context); + expect(outcome).toBe("idle"); + expect(storage.commits.length).toBe(commits); + expect((await harness.snapshot(LiveDoc, root.id, context))?.run).toBeUndefined(); + await harness.close(context); + }); + + // #11149 + it("throws ReadAfterWrite after a table write of the commit, on every path", async () => { + const { harness, root } = await openBusy(); + // A busy input without request ID or trace reads no table, and is still refused. + const afterAppend = harness.commit(async (tx) => { + await tx.appendEntry(root.id, { kind: "app.note" }); + await root.submit(tx, { type: "input", content: "steer", whenBusy: "steer" }); + }, context); + await expect(afterAppend).rejects.toBeInstanceOf(ReadAfterWrite); + await expect(afterAppend).rejects.toThrow("submit(tx, draft) cannot read tables after the first table write"); + const second = harness.commit(async (tx) => { + await root.submit(tx, { type: "input", content: "one", whenBusy: "steer" }); + await root.submit(tx, { type: "input", content: "two", whenBusy: "steer" }); + }, context); + await expect(second).rejects.toBeInstanceOf(ReadAfterWrite); + expect((await harness.snapshot(InboxDoc, root.id, context))?.items).toEqual([]); + + const idle = await openChat(new ControlledStorage(), chatSetup()); + const late = idle.harness.commit(async (tx) => { + await tx.appendEntry(idle.root.id, { kind: "app.note" }); + await idle.root.submit(tx, { type: "input", content: "hi" }); + }, context); + await expect(late).rejects.toBeInstanceOf(ReadAfterWrite); + await idle.harness.close(context); + await harness.close(context); + }); + + // #11149 + it("continues a failed run, and must precede the commit's table writes", async () => { + const setup = chatSetup(); + setup.faux.setResponses([failure, fauxAssistantMessage([fauxText("recovered")])]); + const { harness, root } = await openChat(new ControlledStorage(), setup); + const first = await root.submit({ type: "input", content: "Analyze this" }, context); + expect(await first.wait(context)).toMatchObject({ status: "unanswered", reason: "model_error" }); + + const late = harness.commit(async (tx) => { + await tx.appendEntry(root.id, { kind: "app.note" }); + await root.submit(tx, { type: "continue" }); + }, context); + await expect(late).rejects.toBeInstanceOf(ReadAfterWrite); + + const id = await harness.commit(async (tx) => { + const continued = await root.submit(tx, { type: "continue" }); + await tx.appendEntry(root.id, { kind: "app.note" }); + return continued; + }, context); + expect(await settledStatus(harness, id)).toMatchObject({ type: "continue", status: "done" }); + await harness.close(context); + }); + + // #11149 + it("rejects a busy continue and steers with whenIdle inside the commit", async () => { + const { harness, root } = await openBusy(); + const busy = harness.commit((tx) => root.submit(tx, { type: "continue" }), context); + await expect(busy).rejects.toBeInstanceOf(ConversationBusy); + const steer = await harness.commit( + (tx) => root.submit(tx, { type: "input", content: "steer", whenBusy: "steer", whenIdle: "reject" }), + context, + ); + expect((await harness.snapshot(InboxDoc, root.id, context))?.items.map((item) => item.id)).toEqual([steer]); + await harness.close(context); + }); + + // #11149 + it("throws for a transaction of another Session, or a wrapped one", async () => { + const a = await openChat(new ControlledStorage(), chatSetup()); + const b = await openChat(new ControlledStorage(), chatSetup()); + const draft = { type: "input", content: "hi" } as const; + const foreign = b.harness.commit((tx) => a.root.submit(tx, draft), context); + await expect(foreign).rejects.toThrow("The transaction belongs to another Session"); + const proxied = a.harness.commit((tx) => a.root.submit(new Proxy(tx, {}), draft), context); + await expect(proxied).rejects.toThrow("The transaction belongs to another Session"); + const plain = a.harness.commit((tx) => a.root.submit({ doc: tx.doc.bind(tx) } as unknown as Tx, draft), context); + await expect(plain).rejects.toThrow("The transaction belongs to another Session"); + expect((await a.harness.snapshot(LiveDoc, a.root.id, context))?.run).toBeUndefined(); + await a.harness.close(context); + await b.harness.close(context); + }); + + // #11149 + it("records the trace carrier and links a steer to its caller and the running trace", async () => { + const setup = chatSetup(); + const telemetry = new InMemoryTelemetryContext(); + const busy = unanswered(); + setup.faux.setResponses([busy.step]); + const { harness, root } = await openTraced(setup, telemetry); + await root.submit({ type: "input", content: "work" }, context); + await busy.reached; + const run = (await harness.inspect(context)).tasks.find((task) => task.record.kind === "pi.generation")!.record; + const id = await telemetry.startSpan({ name: "caller" }, (caller) => + harness.commit( + (tx) => root.submit(tx, { type: "input", content: "steer", whenBusy: "steer" }), + withTelemetry(caller, context), + ), + ); + const spans = telemetry.getSpans(); + const caller = spans.find((span) => span.name === "caller")!; + const submit = spans.find( + (span) => span.name === "pi.durable.submit" && span.attributes["submission.id"] === id, + )!; + expect(submit).toMatchObject({ parentId: null, attributes: { result: "queued" } }); + expect(submit.links).toEqual([ + { context: caller.context, attributes: {} }, + { context: run.trace, attributes: {} }, + ]); + expect((await (await harness.submission(id, context))!.status(context)).trace).toEqual(submit.context); + await harness.close(context); + }); + + // #11149 + it("starts an idle input's run in the admission's trace", async () => { + const setup = chatSetup(); + const telemetry = new InMemoryTelemetryContext(); + const busy = unanswered(); + setup.faux.setResponses([busy.step]); + const { harness, root } = await openTraced(setup, telemetry); + const id = await telemetry.startSpan({ name: "caller" }, (caller) => + harness.commit((tx) => root.submit(tx, { type: "input", content: "hi" }), withTelemetry(caller, context)), + ); + await busy.reached; + const submit = telemetry + .getSpans() + .find((span) => span.name === "pi.durable.submit" && span.attributes["submission.id"] === id)!; + const record = await (await harness.submission(id, context))!.status(context); + expect(record.trace).toEqual(submit.context); + const run = (await harness.inspect(context)).tasks.find((task) => task.record.kind === "pi.generation")!.record; + expect(run.trace).toEqual(record.trace); + await harness.close(context); + }); + + // #11149 + it("makes a write a child of its caller and leaves the commit's events to the caller's commit", async () => { + const setup = chatSetup(); + const telemetry = new InMemoryTelemetryContext(); + const { harness, root } = await openTraced(setup, telemetry); + const id = await telemetry.startSpan({ name: "caller" }, (caller) => + harness.commit( + (tx) => root.submit(tx, { type: "write", entry: { kind: "app.note" } }), + withTelemetry(caller, context), + ), + ); + const spans = telemetry.getSpans(); + const caller = spans.find((span) => span.name === "caller")!; + const submit = spans.find((span) => span.name === "pi.durable.submit")!; + expect(submit).toMatchObject({ + parentContext: caller.context, + settled: true, + attributes: { "submission.id": id, result: "done" }, + events: [], + }); + expect(caller.events.map((event) => event.name)).toEqual(["pi.durable.entry", "pi.durable.submission"]); + await harness.close(context); + }); + + // #11149 + it("ends its span before the commit, and a later throw leaves no submission", async () => { + const setup = chatSetup(); + const telemetry = new InMemoryTelemetryContext(); + const { harness, root } = await openTraced(setup, telemetry); + await telemetry.startSpan({ name: "caller" }, async (caller) => { + const failed = harness.commit( + async (tx) => { + await root.submit(tx, { type: "write", entry: { kind: "app.note" } }); + throw new Error("later"); + }, + withTelemetry(caller, context), + ); + await expect(failed).rejects.toThrow("later"); + }); + const spans = telemetry.getSpans(); + const submit = spans.find((span) => span.name === "pi.durable.submit")!; + // The span reported an admission the commit then discarded. + expect(submit).toMatchObject({ settled: true, attributes: { result: "done" } }); + const id = submit.attributes["submission.id"] as SubmissionId; + expect(await harness.submission(id, context)).toBeUndefined(); + expect(spans.find((span) => span.name === "caller")!.events).toEqual([]); + await harness.close(context); + }); + + // #11149 + it("admits atomically from a tool's commit through its conversation handle", async () => { + const setup = chatSetup(); + let other: Conversation | undefined; + let toolTask: number | undefined; + addTool( + setup.registry, + defineTool({ + name: "delegate", + description: "Delegates to another conversation", + parameters: Type.Object({}), + execute: async (_args, api, callContext) => { + toolTask = api.taskId; + const handle = (await api.conversation(other!.id, callContext))!; + const id = await api.commit(async (tx) => { + const admitted = await handle.submit(tx, { type: "input", content: "from the tool" }); + await tx.appendEntry(other!.id, { kind: "app.delegated", data: { submission: admitted } }); + return admitted; + }, callContext); + return { output: [{ type: "text", text: `submitted ${id}` }] }; + }, + }), + ); + const reply: FauxResponseStep = (request) => { + const last = request.messages.at(-1); + if (last?.role === "user" && last.content === "delegate") { + return fauxAssistantMessage([fauxToolCall("delegate", {}, { id: "c1" })], { stopReason: "toolUse" }); + } + return fauxAssistantMessage([fauxText("ok")]); + }; + setup.faux.setResponses([reply, reply, reply]); + const { harness, root } = await openChat(new ControlledStorage(), setup); + other = await harness.createConversation({ ownership: { kind: "ownerless" }, agent: { model: MODEL } }, context); + const run = await root.submit({ type: "input", content: "delegate" }, context); + expect(await run.wait(context)).toMatchObject({ status: "done" }); + await harness.waitForIdle(context); + const entries = [...(await other.entries({}, 100, undefined, context)).items].reverse(); + const delegated = entries.find((entry) => entry.kind === "app.delegated"); + const submission = (delegated?.data as { submission: SubmissionId }).submission; + const settled = await settledStatus(harness, submission); + if (settled.type !== "input" || settled.status !== "done") throw new Error(`Unexpected ${settled.status}`); + // The placed input carries the tool task's byTaskId, as the commit's own appends do. + const user = await other.commit((tx) => tx.entry(UserEntry, settled.entry), context); + expect(user?.byTaskId).toBe(toolTask); + await harness.close(context); + }); + + // #11149 + it("admits inside a task's terminal commit", async () => { + const setup = chatSetup(); + setup.faux.setResponses([fauxAssistantMessage([fauxText("answer")])]); + const Admitting = defineTask({ + name: "test.admitting", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (_task, runtime, taskContext) => { + const handle = (await runtime.conversation(runtime.conversationId, taskContext))!; + await runtime.commit(async (tx) => { + const id = await handle.submit(tx, { type: "input", content: "from the task" }); + return { status: "terminal", outcome: { status: "completed", result: id } }; + }, taskContext); + }, + }, + abort: (_task, runtime, taskContext) => + runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), taskContext), + }); + addTask(setup.registry, Admitting); + const { harness, root } = await openChat(new ControlledStorage(), setup); + const task = await root.commit( + (tx) => tx.createTask(Admitting, null, { ownership: { kind: "conversation" } }), + context, + ); + harness.resume(); + const settled = await harness.waitForTask(task, context); + if (settled.state.outcome.status !== "completed") throw new Error(settled.state.outcome.status); + const status = await settledStatus(harness, settled.state.outcome.result); + if (status.type !== "input" || status.status !== "done") throw new Error(`Unexpected ${status.status}`); + const user = await root.commit((tx) => tx.entry(UserEntry, status.entry), context); + expect(user?.byTaskId).toBe(task); + await harness.close(context); + }); + + // #11149 + it("rejects through a handle once its invocation is aborted, and once it ended", async () => { + const setup = chatSetup(); + let other: Conversation | undefined; + let handle: ConversationHandle | undefined; + const started = deferred(); + const abortSeen = deferred(); + const release = deferred(); + addTool( + setup.registry, + defineTool({ + name: "hold", + description: "Holds until released, also when aborted", + parameters: Type.Object({}), + execute: async (_args, api, callContext) => { + handle = (await api.conversation(other!.id, callContext))!; + callContext.abortSignal!.addEventListener("abort", () => abortSeen.resolve(), { once: true }); + started.resolve(); + await release.promise; + return { output: [] }; + }, + }), + ); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("hold", {}, { id: "c1" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("ok")]), + ]); + const { harness, root } = await openChat(new ControlledStorage(), setup); + other = await harness.createConversation({ ownership: { kind: "ownerless" }, agent: { model: MODEL } }, context); + const run = await root.submit({ type: "input", content: "go" }, context); + await started.promise; + const tool = (await harness.inspect(context)).tasks.find((task) => task.record.kind === "pi.tool")!.record; + const aborting = harness.abortTask(tool.id, context); + await abortSeen.promise; + const draft = { type: "input", content: "late" } as const; + await expect(handle!.submit(draft, context)).rejects.toThrow(); + await expect(harness.commit((tx) => handle!.submit(tx, draft), context)).rejects.toThrow(); + expect((await harness.snapshot(LiveDoc, other.id, context))?.run).toBeUndefined(); + release.resolve(); + await aborting; + await run.wait(context); + await expect(harness.commit((tx) => handle!.submit(tx, draft), context)).rejects.toThrow(); + expect((await harness.snapshot(LiveDoc, other.id, context))?.run).toBeUndefined(); + await harness.close(context); + }); +}); From 65fba65b2ea26c58916b4fde606d09234d64fa5e Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 14:03:00 +0000 Subject: [PATCH 077/271] feat(durable): entry reads for hooks and tasks HookApi.entry() and entries(), and TaskRuntime.entries(): a hook or task reads the conversation's raw history, which compaction never changes, so a range bounded by entry IDs reads the same before and after a compaction and in a rerun. --- packages/durable/CHANGELOG.md | 1 + packages/durable/docs/spec.md | 27 +++++- packages/durable/src/harness/harness.ts | 11 +-- packages/durable/src/harness/scheduler.ts | 13 ++- packages/durable/src/harness/types.ts | 18 +++- packages/durable/src/harness/util.ts | 15 ++- packages/durable/src/types.ts | 11 +++ .../durable/test/harness-hook-reads.test.ts | 94 +++++++++++++++++++ 8 files changed, 178 insertions(+), 12 deletions(-) create mode 100644 packages/durable/test/harness-hook-reads.test.ts diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 6511187f0..f76b9543f 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -111,6 +111,7 @@ - Per-conversation turn limits: `AgentState.maxTurns` (`configure({ maxTurns })`, `null` clears; a `RangeError` for anything but an integer 1 or more) bounds the tool rounds one run takes. The round that reaches it runs its tools, then the run ends `unanswered` with the new `RunFailureReason` `turn_limit`, like a failed run, so `settings.afterFailedRun` decides the queued inputs and a continue resumes it on the round's results. The count is `pi.live.run.turns`; every run starts at 0, and an `onYield` continuation starts it over. Task-owned children and forks inherit the limit; `Agent.maxTurns` exposes it ([#11043](https://github.com/earendil-works/pi/issues/11043)) - `whenIdle: "reject"` on input submissions: an idle conversation rejects the input with the new `ConversationIdle` and writes nothing, so it never starts a run. With `whenBusy: "steer"` it steers the current run or is refused, in one admission instead of a host's check and a separate submit that can race the run's end. A known request ID still returns its submission ([#11149](https://github.com/earendil-works/pi/issues/11149)) - `Conversation.submit(tx, draft)` admits a submission inside the caller's commit (`Harness.commit()`, `Conversation.commit()`) with the same rules as `submit(draft, context)`, and returns its ID: a host checks its own documents and `pi.live`, admits, and writes its own state in one commit, which stores all of it or none. `ConversationHandle.submit(tx, draft)` does the same inside a task's or tool's own commit. The Harness supplies the clock, queue modes, the `pi.durable.submit` span (covering the admission only; its `result` is provisional until the commit is stored), and the request for progress. It must come before the commit's first table write, once per commit, and throws `ReadAfterWrite` otherwise on every path ([#11149](https://github.com/earendil-works/pi/issues/11149)) +- `HookApi.entry()` and `entries()`, and `TaskRuntime.entries()`: a hook or task reads the conversation's raw history, which compaction never changes, so a range bounded by entry IDs reads the same before and after a compaction and on a rerun ([#11141](https://github.com/earendil-works/pi/issues/11141)) ### Changed diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index 3ef2b232f..e795d5556 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -2144,6 +2144,13 @@ interface TaskRuntime extends DocumentObserver, Docum /** Committed entry visible from the task's conversation. */ entry(id: EntryId, context: Context): Promise; entry(token: Entry, id: EntryId, context: Context): Promise | undefined>; + /** Committed raw history of the task's conversation, fork-aware, as `Conversation.entries()`. */ + entries( + query: Omit, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise>; /** Committed raw active transcript and model context, optionally cut off at `at`. */ context(conversationId: ConversationId, context: Context, options?: { readonly at?: EntryId }): Promise; /** The Harness clock. */ @@ -3279,11 +3286,20 @@ type HookResult = T | undefined | Promise; interface HookApi extends DocumentReader { readonly taskId: TaskId; readonly conversationId: ConversationId; - /** `HarnessOptions.models`. */ /** The current `HarnessOptions.models`, read at each access. */ readonly models: Models; memo(name: string, context: Context): Promise; memo(name: string, candidate: T, context: Context): Promise; + /** As `TaskRuntime.entry()` (section 5.1). */ + entry(id: EntryId, context: Context): Promise; + entry(token: Entry, id: EntryId, context: Context): Promise | undefined>; + /** As `TaskRuntime.entries()` (section 5.1). */ + entries( + query: Omit, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise>; } interface GenerationHooks { @@ -3355,6 +3371,15 @@ after a first decision, does so inside its own `invoke`. Hooks receive the task' runtime as their `HookApi`; hook memos and the task's own memos share one namespace, so hook authors prefix their memo names. +Hooks read committed state as it is when they run, off the line, so a hook a +crash reruns may see more. Entries never change and compaction never removes +them (section 8.7): a range bounded by entry IDs reads the same before and +after a compaction and in a rerun. `HookApi` has no `context()`: the active +transcript changes in the middle of a run, when a compaction summary or a +handoff's reset lands, so a hook asking what a run did reads its history +instead. A range also holds entries of others, such as display-only writes, +compaction summaries, and entries of other tasks; hooks filter by `kind`. + Selection decides which behavior a conversation has; the extension's own document holds that behavior's state: diff --git a/packages/durable/src/harness/harness.ts b/packages/durable/src/harness/harness.ts index 055c05bd7..906fb3539 100644 --- a/packages/durable/src/harness/harness.ts +++ b/packages/durable/src/harness/harness.ts @@ -74,7 +74,7 @@ import type { Wake, } from "./types.ts"; import { addUsageState, UsageDoc, type UsageState } from "./usage.ts"; -import { scanAll } from "./util.ts"; +import { conversationEntries, scanAll } from "./util.ts"; import { type ConversationView, ConversationViews, type ConversationWindow } from "./view.ts"; const SCAN_PAGE_SIZE = 256; @@ -160,13 +160,8 @@ class ConversationImpl implements Conversation { cursor: Cursor | undefined, context: Context, ): Promise> { - const bounded: EntryQuery = { - conversationId: this.id, - ...(query.minEntryId === undefined ? {} : { minEntryId: query.minEntryId }), - ...(query.maxEntryId === undefined ? {} : { maxEntryId: query.maxEntryId }), - ...(query.order === undefined ? {} : { order: query.order }), - }; - return this.#host.harness.readOnLine(() => this.#host.storage.scanEntries(bounded, limit, cursor, context)); + const scan = conversationEntries(this.id, query); + return this.#host.harness.readOnLine(() => this.#host.storage.scanEntries(scan, limit, cursor, context)); } fork(at: EntryId, options: ConversationCreateOptions, context: Context): Promise { diff --git a/packages/durable/src/harness/scheduler.ts b/packages/durable/src/harness/scheduler.ts index 1fc981be6..de26aceb3 100644 --- a/packages/durable/src/harness/scheduler.ts +++ b/packages/durable/src/harness/scheduler.ts @@ -52,7 +52,7 @@ import type { TaskInspection, Wake, } from "./types.ts"; -import { closedError, optionalExtension, scanAll, Waiters } from "./util.ts"; +import { closedError, conversationEntries, optionalExtension, scanAll, Waiters } from "./util.ts"; type AnyTaskRecord = TaskRecord; /** A record that can still run code: pending, running, waiting, or sleeping. */ @@ -2257,6 +2257,17 @@ export class TaskScheduler { return token === undefined || entry?.kind === token.kind ? entry : undefined; }); }) as ErasedRuntime["entry"], + entries: (query, limit, cursor, context) => + this.#read(invocation, () => + this.#session.readOnLine(() => + this.#storage.scanEntries( + conversationEntries(invocation.conversationId, query), + limit, + cursor, + context, + ), + ), + ), context: (conversationId, context, options) => this.#read(invocation, async () => { const { view, range } = await readContextFrom( diff --git a/packages/durable/src/harness/types.ts b/packages/durable/src/harness/types.ts index 3306ffe09..99a024c6a 100644 --- a/packages/durable/src/harness/types.ts +++ b/packages/durable/src/harness/types.ts @@ -25,6 +25,7 @@ import type { Cursor, DocumentObserver, DocumentReader, + Entry, EntryDraft, EntryId, EntryQuery, @@ -44,6 +45,7 @@ import type { TaskRecord, TaskState, Tx, + TypedEntry, } from "../types.ts"; import type { TaskGraph, TaskGraphWatch } from "./task-graph.ts"; import type { UsageState } from "./usage.ts"; @@ -933,7 +935,10 @@ export interface Harness extends Session { observeTaskGraph(observer: Observer, context: Context): Promise>; } -/** What a hook may use: committed reads and the asking task's memos, which hooks and the task share. */ +/** + * What a hook may use: committed reads and the asking task's memos, which hooks and the task share. Hooks run off the + * Session line, so reads see what is committed when they run; a hook rerun after a crash may see more. + */ export interface HookApi extends DocumentReader { readonly taskId: TaskId; readonly conversationId: ConversationId; @@ -941,6 +946,17 @@ export interface HookApi extends DocumentReader { readonly models: Models; memo(name: string, context: Context): Promise; memo(name: string, candidate: T, context: Context): Promise; + /** Committed entry visible from the task's conversation, as `TaskRuntime.entry()`. */ + entry(id: EntryId, context: Context): Promise; + /** Undefined when the entry is absent, not visible, or has another kind. */ + entry(token: Entry, id: EntryId, context: Context): Promise | undefined>; + /** Committed raw history of the task's conversation, as `TaskRuntime.entries()`. */ + entries( + query: Omit, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise>; } export type HookResult = T | undefined | Promise; diff --git a/packages/durable/src/harness/util.ts b/packages/durable/src/harness/util.ts index cf2222a5a..f3c8ea374 100644 --- a/packages/durable/src/harness/util.ts +++ b/packages/durable/src/harness/util.ts @@ -1,6 +1,6 @@ import type { Context } from "@earendil-works/chord"; import { SessionFailed } from "../errors.ts"; -import type { Cursor, Page } from "../types.ts"; +import type { ConversationId, Cursor, EntryQuery, Page } from "../types.ts"; /** Pending waits by key. Each settles once: through `resolve`, `rejectAll`, or cancellation of its context. */ export class Waiters { @@ -64,3 +64,16 @@ export function closedError(session: { readonly failure: { readonly error: unkno export function optionalExtension(extension: string | undefined): { readonly extension?: string } { return extension === undefined ? {} : { extension }; } + +/** The scan of conversation `conversationId`'s history that `query` asks for, without fields an untyped caller added. */ +export function conversationEntries( + conversationId: ConversationId, + query: Omit, +): EntryQuery { + return { + conversationId, + ...(query.minEntryId === undefined ? {} : { minEntryId: query.minEntryId }), + ...(query.maxEntryId === undefined ? {} : { maxEntryId: query.maxEntryId }), + ...(query.order === undefined ? {} : { order: query.order }), + }; +} diff --git a/packages/durable/src/types.ts b/packages/durable/src/types.ts index 3ea1f1ca4..4f17a3711 100644 --- a/packages/durable/src/types.ts +++ b/packages/durable/src/types.ts @@ -324,6 +324,17 @@ export interface TaskRuntime extends DocumentObserver entry(id: EntryId, context: Context): Promise; /** Undefined when the entry is absent, not visible, or has another kind. */ entry(token: Entry, id: EntryId, context: Context): Promise | undefined>; + /** + * Committed raw history of the task's conversation, fork-aware, as `Conversation.entries()`: every entry, also those a + * compaction summary or reset left behind. Entries never change, so a range bounded by entry IDs reads the same + * before and after a compaction, and in a rerun after a crash. + */ + entries( + query: Omit, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise>; /** Committed raw active transcript and model context, optionally cut off at the visible entry `at`. */ context(conversationId: ConversationId, context: Context, options?: { readonly at?: EntryId }): Promise; /** The Harness clock. */ diff --git a/packages/durable/test/harness-hook-reads.test.ts b/packages/durable/test/harness-hook-reads.test.ts new file mode 100644 index 000000000..2681d244d --- /dev/null +++ b/packages/durable/test/harness-hook-reads.test.ts @@ -0,0 +1,94 @@ +import { fauxAssistantMessage } from "@earendil-works/pi-ai"; +import { + defineExtension, + defineTask, + type EntryRecord, + GenerationTask, + hook, + MemoryStorage, + UserEntry, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import { allEntries, chatSetup, openChat } from "./chat-support.ts"; +import { context } from "./session-support.ts"; +import { completed } from "./task-support.ts"; + +describe("entry reads for hooks and tasks", () => { + // #11141 + it("reads raw history and single entries from a hook", async () => { + const setup = chatSetup(); + const seen: { kinds: string[]; user?: string }[] = []; + setup.registry.install( + defineExtension({ + name: "reader", + hooks: [ + hook(GenerationTask, { + onYield: async (_answer, api, ctx) => { + const page = await api.entries({ order: "ascending" }, 10, undefined, ctx); + const first = page.items[0]!; + const user = await api.entry(UserEntry, first.id, ctx); + seen.push({ + kinds: page.items.map((entry) => entry.kind), + ...(user === undefined ? {} : { user: user.kind }), + }); + // Another kind: undefined. + expect(await api.entry(GenerationTask as never, first.id, ctx)).toBeUndefined(); + return undefined; + }, + }), + ], + }), + ); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + // The answer is not committed yet when `onYield` asks. + expect(seen).toEqual([{ kinds: ["pi.user"], user: "pi.user" }]); + await harness.close(context); + }); + + // #11141 + it("pages a task's conversation history by entry ID range", async () => { + const setup = chatSetup(); + const pages: (readonly EntryRecord[])[] = []; + const Reader = defineTask({ + name: "test.reader", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (_task, runtime, ctx) => { + const all = await runtime.entries({ order: "ascending" }, 1, undefined, ctx); + pages.push(all.items); + const rest = await runtime.entries({ order: "ascending" }, 1, all.next, ctx); + pages.push(rest.items); + const bounded = await runtime.entries( + { minEntryId: rest.items[0]!.id, maxEntryId: rest.items[0]!.id }, + 10, + undefined, + ctx, + ); + pages.push(bounded.items); + await runtime.commit(() => completed(null), ctx); + }, + }, + abort: async () => {}, + }); + setup.registry.install(defineExtension({ name: "reader", tasks: [Reader] })); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const id = await root.commit( + (tx) => tx.createTask(Reader, null, { ownership: { kind: "conversation" } }), + context, + ); + await harness.waitForTask(id, context); + const entries = await allEntries(root); + expect(entries.map((entry) => entry.kind)).toEqual(["pi.user", "pi.assistant"]); + expect(pages.map((page) => page.map((entry) => entry.id))).toEqual([ + [entries[0]!.id], + [entries[1]!.id], + [entries[1]!.id], + ]); + await harness.close(context); + }); +}); From 7e15f960190966815fcc23eac24e088cd811685d Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 14:03:25 +0000 Subject: [PATCH 078/271] feat(durable): run-end tasks with the afterRun hook GenerationHooks.afterRun asks, after a run answered, for durable tasks (runEndTask(task, input)). The answer commits and the run's inputs settle done first; the generation then creates the tasks as its children in a finish phase and waits for them in drain, keeping pi.live.run (run.afterRun) and the conversation busy, so idle waits include them, Stop aborts them and a crash resumes them. Queued input is placed when they are terminal. pi.live.run.first and runFailure.first name where a run's history starts, also for a continue after a failure. closes #11141 --- packages/durable/CHANGELOG.md | 1 + packages/durable/README.md | 41 +- packages/durable/docs/spec.md | 228 +++- packages/durable/src/harness/define.ts | 20 +- packages/durable/src/harness/generation.ts | 202 ++- packages/durable/src/harness/inbox.ts | 16 +- packages/durable/src/harness/live.ts | 44 +- packages/durable/src/harness/scheduler.ts | 38 +- packages/durable/src/harness/submissions.ts | 13 +- packages/durable/src/harness/types.ts | 29 +- packages/durable/src/index.ts | 5 +- .../test/examples/32-tests-after-edits.ts | 140 ++ .../durable/test/harness-after-run.test.ts | 1204 +++++++++++++++++ .../test/harness-generation-recovery.test.ts | 2 +- .../durable/test/harness-generation.test.ts | 18 +- .../test/harness-input-entries.test.ts | 1 + .../durable/test/harness-live-deltas.test.ts | 8 +- .../test/harness-restart-failure.test.ts | 12 +- .../durable/test/harness-run-failure.test.ts | 8 +- .../durable/test/harness-structured.test.ts | 3 +- .../durable/test/harness-view-run.test.ts | 7 +- 21 files changed, 1954 insertions(+), 86 deletions(-) create mode 100644 packages/durable/test/examples/32-tests-after-edits.ts create mode 100644 packages/durable/test/harness-after-run.test.ts diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index f76b9543f..98f05d090 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -112,6 +112,7 @@ - `whenIdle: "reject"` on input submissions: an idle conversation rejects the input with the new `ConversationIdle` and writes nothing, so it never starts a run. With `whenBusy: "steer"` it steers the current run or is refused, in one admission instead of a host's check and a separate submit that can race the run's end. A known request ID still returns its submission ([#11149](https://github.com/earendil-works/pi/issues/11149)) - `Conversation.submit(tx, draft)` admits a submission inside the caller's commit (`Harness.commit()`, `Conversation.commit()`) with the same rules as `submit(draft, context)`, and returns its ID: a host checks its own documents and `pi.live`, admits, and writes its own state in one commit, which stores all of it or none. `ConversationHandle.submit(tx, draft)` does the same inside a task's or tool's own commit. The Harness supplies the clock, queue modes, the `pi.durable.submit` span (covering the admission only; its `result` is provisional until the commit is stored), and the request for progress. It must come before the commit's first table write, once per commit, and throws `ReadAfterWrite` otherwise on every path ([#11149](https://github.com/earendil-works/pi/issues/11149)) - `HookApi.entry()` and `entries()`, and `TaskRuntime.entries()`: a hook or task reads the conversation's raw history, which compaction never changes, so a range bounded by entry IDs reads the same before and after a compaction and on a rerun ([#11141](https://github.com/earendil-works/pi/issues/11141)) +- Run-end tasks: a `GenerationHooks.afterRun(run, api)` hook returns `{ tasks }`, built with the typed `runEndTask(task, input)`, after a run answered (a final answer or a `terminate` round; not when an `onYield` continuation is taken, not after a failure or an abort). The answer is committed and the run's inputs settle `done` first, so `Submission.wait()` resolves at the answer; the generation then creates the tasks as its children and waits for them, keeping `pi.live.run` (with `run.afterRun`) and the conversation busy, so idle waits include them, Stop aborts them, and a crash resumes them. Input queued meanwhile, steers included, is placed when they are terminal and starts the next run; a task admits a model-visible write or a fix-run follow-up in its own commit with `submit(tx, draft)`. A crash before the tasks exist asks the hooks again; a broken request (an uninstalled or built-in task, a non-JSON input, a throwing `initial()`) is reported and skipped. `run` names the run's entries: `first` (`pi.live.run.first`, carried into `runFailure.first` so a continue covers a failed run's history), `answer`, and `last` ([#11141](https://github.com/earendil-works/pi/issues/11141)) ### Changed diff --git a/packages/durable/README.md b/packages/durable/README.md index 3403a4997..703731bf9 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -540,7 +540,7 @@ await root.submit({ type: "write", entry: { kind: "app.note", data: "user opened - **Steers** are placed after the current tool round and join the running work. - **Follow-ups** are placed when the run answers, and start the next run. -- **`whenIdle: "reject"`** refuses an input when no run is working, so it never starts a run of its own. With `whenBusy: "steer"`, it steers the current run or throws `ConversationIdle`. +- **`whenIdle: "reject"`** refuses an input when no run is working, so it never starts a run of its own. With `whenBusy: "steer"`, it steers the current run or throws `ConversationIdle`. A run that answered and waits for its [after-run tasks](#work-at-the-end-of-a-run) is still working, so such an input queues and starts the next run. - **`submit(tx, draft)`** admits inside your own commit, with the same rules, and returns the submission's ID. Your checks, the admission, and your writes are stored together or not at all: ```typescript @@ -659,11 +659,47 @@ const Guard = defineExtension({ }); ``` -- **Generation:** `beforeRequest` (replace the messages of one request), `afterResponse`, `onYield` (continue the run with another user message), and `afterTools` (runs once a round's tools are done). +- **Generation:** `beforeRequest` (replace the messages of one request), `afterResponse`, `onYield` (continue the run with another user message), `afterTools` (runs once a round's tools are done), and `afterRun` (tasks to run before the run ends, below). - **Tools:** `beforeTool` (block or rewrite arguments) and `afterTool` (replace the result). `afterTool` sees every result a tool task writes, also the error results the Harness writes itself (unavailable, invalid, blocked, interrupted, restart limit, aborted), told apart by their diagnostic code; past `settings.restartLimit` it no longer runs. On an abort it runs inside the abort handler, so a slow hook delays `abort()`. To limit a hook to some conversations, select its extension only there, for example with `configure({ extensions: { add: [Guard] } })`. +Hooks read committed documents (`api.snapshot()`) and the conversation's raw history (`api.entry()`, `api.entries()`), which compaction never changes. + +### Work at the end of a run + +An `afterRun` hook starts durable work after a run answered, without a tool call: for example the project's tests after a run that edited files. It returns tasks, built with `runEndTask()`, and the generation creates them as its children and waits for them before the run ends: + +```typescript +import { defineExtension, GenerationTask, hook, runEndTask, ToolResultEntry } from "@earendil-works/pi-durable"; + +const TestsAfterEdits = defineExtension({ + name: "tests-after-edits", + tasks: [RunTests], // a task of yours, defineTask(): runs the tests, then commits its result + hooks: [ + hook(GenerationTask, { + afterRun: async (run, api, context) => { + // The run's own entries, also those a compaction summarized. + const range = { minEntryId: run.first, maxEntryId: run.last, order: "ascending" } as const; + const { items } = await api.entries(range, 1000, undefined, context); // page with `next` for longer runs + const edited = items.some((entry) => { + const message = entry.model?.[0]; + return ToolResultEntry.is(entry) && message?.role === "toolResult" && message.toolName === "edit"; + }); + return edited ? { tasks: [runEndTask(RunTests, { command: "npm test" })] } : undefined; + }, + }), + ], +}); +``` + +- **When:** after a final answer or a `terminate` round, not while an `onYield` continuation keeps the run going, never after a failure or an abort. +- **The answer first:** the answer is committed and the run's inputs settle `done` before the hooks are asked, so `Submission.wait()` resolves at the answer. +- **Busy until done:** `pi.live.run` stays, with `run.afterRun` listing the tasks, so the conversation is busy and `waitForIdle()` includes them; Stop aborts them; a crash resumes them. A crash before they exist asks the hooks again, never twice once they do. +- **Input meanwhile:** steers and follow-ups queue and start the next run once the tasks are done; `whenBusy: "reject"` and a continue throw `ConversationBusy`. +- **Results for the model:** a task gets its conversation's handle before its commit, `const handle = (await runtime.conversation(runtime.conversationId, context))!`, and admits a write or a follow-up (say "the tests failed: ...") inside it with `handle.submit(tx, draft)`, before its own table writes. Both are placed when the run ends: a write before any queued user input, a follow-up in queue order with the rest. A follow-up starts a fix run, which asks `afterRun` again with a fresh turn budget, so cap such loops in your extension. +- **Broken requests** (an uninstalled or built-in task, a non-JSON input, a throwing `initial()`) are reported through `onReport` against the hook's extension and skipped; the run still ends. + ## More Conversations and Forks ```typescript @@ -917,6 +953,7 @@ node --conditions=source --experimental-strip-types test/examples/13-chat.ts | [29-tool-override](test/examples/29-tool-override.ts) | A same-name bash for some conversations, and a wrapper that times whichever bash won | | [30-reload-and-restart](test/examples/30-reload-and-restart.ts) | Reloading an extension mid-call, and stored choices surviving a restart | | [31-transcript](test/examples/31-transcript.ts) | A client's transcript over the windowed view: live appends, a stall with a gap, loading it and earlier pages, and resuming with `from` | +| [32-tests-after-edits](test/examples/32-tests-after-edits.ts) | An `afterRun` hook that runs the tests after every run that edited files, a fix run when they fail, and input waiting for them | | [00](test/examples/00-conversation.ts)–[12](test/examples/12-recovery.ts) | The layers underneath: sessions, documents, forks, watches, the Harness, agent configuration, reload, extension state, tasks, recovery | Examples that call OpenAI need `OPENAI_API_KEY`; most use the faux provider otherwise. diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index e795d5556..72d630c1b 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -628,6 +628,10 @@ type SettledSubmissionRecord = SubmissionRecord & { interface Submission { readonly id: SubmissionId; status(context: Context): Promise; + /** + * Resolves once the input is answered or unanswered: at the answer, also while the run still waits for its after-run + * tasks (section 8.3), so the conversation may still be busy. + */ wait(context: Context): Promise; abort(context: Context): Promise<"aborted" | "already_placed" | "settled">; } @@ -2488,7 +2492,7 @@ task for it), the task ends visibly instead of running again: |---|---| | tool call (`pi.tool`) | ends with an error result, diagnostic code `restart_limit` ("Tool X was interrupted 3 times in a row; it is not run again"), built from its slot like an interrupted call's; its task fails with detail `{ reason: "restart_limit" }` | | compaction | fails with that error and detail, recorded in `pi.live.compactionFailure` | -| generation | ends its run `unanswered` with `pi.live.runFailure` reason `restart_limit`, failing with that detail | +| generation | ends its run `unanswered` with `pi.live.runFailure` reason `restart_limit`, failing with that detail; a run that answered already, in `finish` or `drain`, ends without a failure and the generation completes (section 8.3) | | a kind of an extension | the scheduler ends it `faulted` with detail `{ reason: "restart_limit" }`, reported as `task_faulted` with reason `restart_limit` | A built-in task that ends itself this way, and a generation whose run ends @@ -2625,7 +2629,9 @@ The **ordinary owned work** of a task `T` is every task that ordinary traversal reaches below `T`: the tasks `T` owns, the tasks in the conversations `T` owns, and, transitively, the same for each of those that is not background. A background task and everything below it are excluded. A conversation-owned task -may be background; a child task may not. +may be background; a child task may not. The tasks a generation creates for its +`afterRun` hooks (section 8.3) are such children, so Stop and idle waits include +them, and a cascade from the generation aborts them first. Two rules follow from the tree. @@ -2843,7 +2849,11 @@ inputs, including while its tool round runs, then the next generation. Tool tasks never own the run; the current round's tool tasks are listed in `pi.live.tools`. The ID list is mutable state because a boundary adds placed steering inputs to an active run; every terminal path -settles exactly the listed inputs. +settles exactly the listed inputs. A run that answered while `afterRun` hooks +are selected (section 8.3) settles its inputs `done` at the answer and keeps +`run`, with an empty ID list and `run.afterRun` set, until the tasks the hooks +asked for are terminal: the conversation stays busy, and `Submission.wait()` +has already resolved. Admission and terminal transitions: @@ -2858,7 +2868,7 @@ Admission and terminal transitions: | input or other write, busy or non-empty inbox | `queued` | append inbox item | | boundary places user item | `placed`, with entry | add ID to current or successor run | | boundary places write | `done`, with entry | append entry | -| run answers | input or continue `done`, with required answer entry | remove `run`, or hand it to a successor | +| run answers | input or continue `done`, with required answer entry | remove `run`, or hand it to a successor; with `afterRun` hooks selected, keep `run` until the tasks they ask for are terminal | | run fails or its task aborts | input `unanswered`, with reason | remove `run`; inbox per `afterFailedRun` below | | tool round reaches `maxTurns` | input `unanswered`, reason `turn_limit` | as a failed run | | withdraw queued item | `unanswered`, reason `aborted` | remove inbox item | @@ -2868,9 +2878,11 @@ Admission and terminal transitions: for the other submission type rejects. A busy input with `whenBusy: "reject"` writes no record and reports `ConversationBusy`. An idle input with `whenIdle: "reject"` writes no record and reports `ConversationIdle`, also when -the inbox holds queued items, so the input never starts a run: with -`whenBusy: "steer"` it steers the current run or is refused. Idle means -`run === undefined`, the inverse of `busy` above. An idle input queues with +the inbox holds queued items, so the input never starts a run of its own: with +`whenBusy: "steer"` it steers the current run or is refused, except as below. Idle means +`run === undefined`, the inverse of `busy` above. A run that answered and waits +for its after-run tasks is busy, so such an input queues there; it starts the +next run when that run ends, as every item queued behind a final answer does. An idle input queues with mode `steer` when `whenBusy` is `steer` and `followUp` otherwise. `Conversation.submit(tx, draft)` runs the same admission inside the caller's @@ -2985,8 +2997,15 @@ except for an `onYield` continuation (section 8.3), which applies only when the boundary selected no user item and no reset. Only successful run ends apply the final boundary: an answer or `terminate`. +With `afterRun` hooks selected, it applies when the run ends after its +after-run tasks, not with the answer, so whether items queued before the answer +land right after it depends on the conversation's extension selection. Items +queued meanwhile wait as well, steers included: there is no next model request +in the run for them to join. A run whose after-run phase was cut off +`settings.restartLimit` times ends without this boundary, as a failed run does. Failure, a run task's abort handler, fault, and orphan settle the run's inputs -`unanswered` and leave the inbox alone (`Conversation.abort()` separately withdraws queued user items); the queued items stay visible in +`unanswered`, unless the run answered already and waits for its after-run tasks +(its inputs are `done`), and leave the inbox alone (`Conversation.abort()` separately withdraws queued user items); the queued items stay visible in `pi.inbox` until the next input or queued write runs a boundary, or their withdrawal. Selected and stale items are removed positionally while retained item order is @@ -3274,6 +3293,7 @@ consuming commit may rerun them. Abort errors always propagate. | `beforeTool` | argument replacement chain; first block wins | block tool with error text | | `afterTool` | result replacement chain | report, continue | | `afterTools` | all observers | report, continue | +| `afterRun` | all observers; requested tasks concatenated in extension order | report, continue | | `beforeCompact` | first decision wins | report, continue | Hooks use task memos for durable first-writer-wins decisions. There is no public @@ -3315,8 +3335,25 @@ interface GenerationHooks { onYield(answer: AssistantMessage, api: HookApi, context: Context): HookResult<{ readonly continue: UserInput }>; /** After every tool of the round is terminal; `results` are the round's result entries in call order. */ afterTools(assistant: EntryId, results: readonly EntryId[], api: HookApi, context: Context): void | Promise; + /** After a run answered: the tasks to run before it ends (section 8.3). */ + afterRun(run: RunSummary, api: HookApi, context: Context): HookResult<{ readonly tasks: readonly RunEndTask[] }>; } +/** How a run ended, for `afterRun`. */ +type RunSummary = { + /** A final answer, or a tool round whose every call asked to `terminate`. */ + readonly reason: "answer" | "terminate"; + /** The run's first entry (`pi.live.run.first`, section 8.2), or the answer for a run without one. */ + readonly first: EntryId; + /** The answer, or the terminating round's tool-calling answer. */ + readonly answer: EntryId; + /** The run's last entry: the answer; for a `terminate` round its newest result, or the reset of its handoff. */ + readonly last: EntryId; +}; + +/** Built by `runEndTask(task, input)`, which types `input` by the task. */ +type RunEndTask = { readonly task: AnyTask; readonly input: JsonValue }; + /** `parent` is set for a nested call (section 7.3). */ type ToolHookCall = ToolCall & { readonly parent?: { readonly taskId: TaskId; readonly callId: string } }; @@ -3373,10 +3410,11 @@ namespace, so hook authors prefix their memo names. Hooks read committed state as it is when they run, off the line, so a hook a crash reruns may see more. Entries never change and compaction never removes -them (section 8.7): a range bounded by entry IDs reads the same before and -after a compaction and in a rerun. `HookApi` has no `context()`: the active -transcript changes in the middle of a run, when a compaction summary or a -handoff's reset lands, so a hook asking what a run did reads its history +them (section 8.7): a range bounded by entry IDs, such as an `afterRun` hook's +`entries({ minEntryId: run.first, maxEntryId: run.last })`, reads the same +before and after a compaction and in a rerun. `HookApi` has no `context()`: +the active transcript changes in the middle of a run, when a compaction summary +or a handoff's reset lands, so a hook asking what a run did reads its history instead. A range also holds entries of others, such as display-only writes, compaction summaries, and entries of other tasks; hooks filter by `kind`. @@ -3405,6 +3443,53 @@ await root.commit(async (tx) => { }, context); ``` +An `afterRun` hook starts durable work at the end of a run, without a tool +call, and the run waits for it (section 8.3). Here the project's tests run after +every run that edited files, and a failure starts a fix run: + +```ts +const RunTests = defineTask<{ command: string }, { phase: "run" }, string>({ + name: "app.run-tests", version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (task, runtime, context) => { + // The app's helper over `ExecutionEnv.exec()`: { passed: boolean; output: string }. A rerun after a crash runs it again. + const result = await runTests(await runtime.env(context), task.input.command, context); + const handle = (await runtime.conversation(runtime.conversationId, context))!; + await runtime.commit(async (tx) => { + // Admitted in this commit, before its table writes; placed when the run ends, in queue order with other input. + if (!result.passed) { + await handle.submit(tx, { type: "input", content: `Tests failed:\n${result.output}`, whenBusy: "followUp" }); + } + await tx.appendEntry(runtime.conversationId, { kind: "app.tests", data: { passed: result.passed } }); + return { status: "terminal", outcome: { status: "completed", result: result.output } }; + }, context); + }, + }, + abort: async (_task, runtime, context) => + runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), context), +}); +export const TestsAfterEdits = defineExtension({ + name: "tests-after-edits", + tasks: [RunTests], + hooks: [hook(GenerationTask, { + afterRun: async (run, api, context) => { + const range = { minEntryId: run.first, maxEntryId: run.last, order: "ascending" } as const; + const { items } = await api.entries(range, 1000, undefined, context); + const edited = items.some((entry) => { + const message = entry.model?.[0]; + return ToolResultEntry.is(entry) && message?.role === "toolResult" && ["edit", "write"].includes(message.toolName); + }); + return edited ? { tasks: [runEndTask(RunTests, { command: "npm test" })] } : undefined; + }, + })], +}); +``` + +A fix run starts another run, which asks `afterRun` again, with a fresh turn +budget (`AgentState.maxTurns`); the extension caps such a loop itself, for +example with a counter in its own document. + ### 7.3 Tools ```ts @@ -4401,7 +4486,20 @@ type LiveState = { * Run control (section 6); present exactly while the conversation is busy. `turns` counts the run's tool rounds, * which `AgentState.maxTurns` bounds (section 8.5); an `onYield` continuation starts it over (section 8.3). */ - run?: { taskId: TaskId; inputs: SubmissionId[]; turns: number }; + run?: { + taskId: TaskId; + /** Placed inputs, settled when the run ends; `[]` once it answered and waits for its after-run tasks. */ + inputs: SubmissionId[]; + turns: number; + /** + * The run's first entry, where `afterRun` hooks start reading (section 8.3): the first one the boundary that + * started it placed, a write's or an input's; a continue's `runFailure.first`; otherwise the run's first + * tool-calling answer. A run started before it was recorded may lack it and gets its answer instead. + */ + first?: EntryId; + /** The run answered and waits for the tasks its `afterRun` hooks asked for, listed once created (section 8.3). */ + afterRun?: { tasks?: TaskId[] }; + }; /** Presentation of the current generation attempt. */ generation?: { attempt: number; @@ -4492,6 +4590,8 @@ type RunFailure = { * what was cut off, and how often; `turn_limit`: the tool rounds the run took, `"1 turn"` or `"N turns"`. */ detail?: string; + /** The failed run's `run.first`, which a continue takes over. */ + first?: EntryId; }; type CompactionFailure = { @@ -4520,7 +4620,7 @@ type CompactionStatus = { | version | `2` | | scope/history/fork | conversation, `latest`, `initial` | | `initial()` | `{}` | -| `migrate()` | from version 1: `reason` of each `compactions` status and of `compactionFailure` renamed `trigger`, and `run.turns` set to `0` (section 3.6) | +| `migrate()` | from version 1: `reason` of each `compactions` status and of `compactionFailure` renamed `trigger`, and `run.turns` set to `0` (section 3.6); `run.first`, `run.afterRun`, and `runFailure.first` are optional and need no step | | checkpoint | complete base whenever nothing runs: `generation` absent and no `running` tool or nested slot | | view mount | `docs["pi.live"]` | | created | with every Harness conversation (section 2.2) | @@ -4564,7 +4664,10 @@ that outcome holds as `completing` (section 5.5), or the scheduler's cleanup It stays small: the summary text lives in the task's placement, never here. Partials are normalized to strict JSON before assignment. Every terminal path of a run task removes `run`, `generation`, and `tools` in the commit that settles the run's -inputs. +inputs. A run that answered while `afterRun` hooks are selected is the one +exception: the answer's commit settles its inputs and removes `generation` and +`tools`, and `run` stays, with `run.afterRun` set, until the run ends after its +after-run tasks (section 8.3). A run that ends without an answer records why in `runFailure`, in the commit that settles its inputs `unanswered`: @@ -4586,7 +4689,10 @@ The next commit that starts a run removes it. A run started in the same commit, the queue's next run under `afterFailedRun: "continue"` (section 6) or after a reset, keeps it, so the failure is shown until a later run starts. A run that ends with an answer, including a round whose tools all ask to terminate, leaves -it alone. Inputs settled without a run (withdrawn, stale, aborted while +it alone; so does one that answered and then ends without its after-run tasks +(aborted, orphaned, or cut off): it answered. A run that fails copies its +`first` into `runFailure.first`, so a continue reads the failed run's history +too. Inputs settled without a run (withdrawn, stale, aborted while queued, `run_failed`) are not run failures; their submission records say why. `tx.settleSubmission()` stages each input's new status and resolves the transaction's latest candidate submission record, falling back to committed state, during assembly, like task-document validation (section 3.3), @@ -4645,6 +4751,17 @@ type GenerationCheckpoint = tools: TaskId[]; /** Calls of a sequential round not started yet, in call order. */ pending: string[]; + } + | { + /** The run answered; its `afterRun` hooks are asked for tasks, again after a crash. */ + phase: "finish"; + run: RunSummary; + } + | { + /** Waiting on the tasks the hooks asked for, which the generation owns; `waiting` holds this checkpoint. */ + phase: "drain"; + answer: EntryId; + tasks: TaskId[]; }; type GenerationResult = { entryId: EntryId }; ``` @@ -4653,7 +4770,8 @@ type GenerationResult = { entryId: EntryId }; `migrate` keeps a version 1 record. Version 2 keeps calls to tools not offered in a sequential round's `pending`, flagged `unoffered` in their slots (section 8.5), which a version 1 Harness would start as tool tasks without the flag, so -it leaves version 2 blocked (`task_too_old`). +it leaves version 2 blocked (`task_too_old`); the `finish` and `drain` phases +below are version 2 as well. The run's inputs live in `pi.live.run`, not in the task input. - `prepare` runs section 7.4 with the phase's agent resolution and resolves @@ -4749,7 +4867,15 @@ The run's inputs live in `pi.live.run`, not in the task input. Otherwise it settles the run's inputs `done`, removes `run` and `generation`, and starts a successor run for the selected user items, if any; a dropped continuation is not retried. Both complete with - `{ entryId }`. + `{ entryId }`. Whether a continuation applies is decided from the inbox + before the boundary places anything: it applies when the inbox holds no + steer, no follow-up, and no reset. + When the phase's agent selects an `afterRun` hook and no continuation + applies, the commit instead appends the answer, settles the run's inputs + `done`, keeps `run` with `inputs: []` and `afterRun: {}`, removes + `generation`, applies no boundary, and moves to `finish` with + `{ reason: "answer", first, answer, last }`: `first` is `run.first`, or the + answer when the run has none, and `answer` and `last` are the answer. - `toolUse` with at least one tool call: the commit appends the assistant entry and starts the tool round described below, moving to `waiting` in the `tools` phase. A `toolUse` message without calls is classified like `stop`. @@ -4779,9 +4905,34 @@ The run's inputs live in `pi.live.run`, not in the task input. it sleeps again. A still deferred result sleeps until a strictly later `pollAt`; any other result is classified as above. A model gone after reopen fails `no_model` once the poll is due. +- `finish` asks the `afterRun` hooks with its `run` (section 7.2) and checks + each requested task off the line: its kind must be an installed task that is + not built-in (`pi.*`), its input JSON, and the installed definition's + `initial()` must return a JSON checkpoint of one of its phases. A request + that fails a check is skipped and reported as an `extension_error` of the + hook that asked for it; the run never fails for one. The task is + created from the installed definition, not the hook's token, with the + checkpoint `initial()` returned, so `initial()` runs once. One commit creates + the tasks as children of the generation, lists them in `run.afterRun.tasks`, + and commits `waiting` on them with `allSettled` at `{ phase: "drain", answer, + tasks }`. Without tasks it ends the run at once, as `drain` does. A crash in + `finish` runs it again and asks the hooks again; nothing was created. Once the + tasks exist, the hooks are not asked again for this run, and a crash leaves + the wait and the tasks to resume. +- `drain` runs once every after-run task is terminal, whatever its outcome. One + commit applies the final boundary (section 6), removes `run`, starts a + successor run for the selected user items, and completes with + `{ entryId: answer }`. +- `finish` and `drain` check the restart count like every phase (section 5.4). + At the limit the run ends `completed` with the answer, without its tasks, + without a `runFailure`, and without the final boundary, so queued items stay + queued as after a failed run; the scheduler reports it `task_interrupted`. - The abort handler calls `models.cancelDeferred()` in `poll` when the model is known (a failure is reported), converts a committed partial, settles the inputs `unanswered` with - `aborted`, removes `run` and `generation`, and ends `aborted`. + `aborted`, removes `run` and `generation`, and ends `aborted`. In `finish` and + `drain` the run answered already: the handler runs after the after-run tasks + are terminal (section 5.5), removes `run`, and records no `runFailure`; the + inputs stay `done`. A tool round starts in the commit that appends the tool-calling answer: @@ -4802,7 +4953,8 @@ A tool round starts in the commit that appends the tool-calling answer: 3. The generation commits `waiting` on its tool tasks with `allSettled` and the `tools` checkpoint (section 8.5); `pi.live.run` stays with it. 4. `pi.live.tools` receives the round's slots (section 8.2), `generation` is - removed, and `pi.live.run.turns` grows by one. + removed, and `pi.live.run.turns` grows by one; a run without `first` gets the + tool-calling answer as its first entry. Input and continue submissions settle `unanswered` with one of these reasons: `no_model`, `model_error` (detail: provider error text), `nothing_to_answer`, @@ -4917,7 +5069,11 @@ and then commits once: text, and a queued head write targeting an entry before it is stale. - When every result of the round requests `terminate`, it settles the run's inputs `done` with the tool-calling answer, removes `run` and `tools`, and - applies the final boundary. + applies the final boundary. When the phase's agent selects an `afterRun` + hook, it keeps `run` instead and moves to `finish` (section 8.3) with + `{ reason: "terminate", first, answer, last }`: `answer` is the tool-calling + answer and `last` the newest of the round's result entries and the handoff's + reset; the final boundary waits for the run's end. - Otherwise, when the run's `turns`, counted in the commit that started the round, reached the phase agent's `maxTurns`, the run ends as a failed run (section 6): its inputs settle `unanswered` with `turn_limit` (detail: the @@ -5016,9 +5172,10 @@ ways to start one; the task is the same, only ownership and placement differ: Only a blocking compaction runs while the conversation's run waits for it. The others never take run control: the conversation keeps working while they -summarize, and their summary is placed like any passive write. Compactions do -not coordinate with each other; several may run at once, and section 6 decides -between their summaries. +summarize, and their summary is placed like any passive write: while a run +that answered waits for its after-run tasks (section 8.3), at the boundary that +ends it. Compactions do not coordinate with each other; several may run at +once, and section 6 decides between their summaries. **Range selection** is a pure function of a `ContextView`, whose `contributions` give each active entry's model messages after every edit in the @@ -5412,8 +5569,10 @@ exposed through their own Chord objects, not mounted. Everything a renderer draws is state in this value: the transcript in `entries`, the run and its presentation in `pi.live` (`run`, `generation`, `tools`, `nestedTools`, `compactions`, `compactionFailure`, `runFailure`, section 8.2), the queue in -`pi.inbox`, the agent in `pi.agent`, spend in `pi.usage`. A renderer renders -the newest value; it never assembles state from a stream of events. +`pi.inbox`, the agent in `pi.agent`, spend in `pi.usage`. A run that answered +and waits for its after-run tasks shows as `run` with `run.afterRun` and no +`generation`. A renderer renders the newest value; it never assembles state +from a stream of events. **The window.** Each watch and observer has its own window. It starts with the newest `min` entries and the ID of the entry before them in `before`. With @@ -6123,6 +6282,23 @@ These are contracts, not invitations to add defensive machinery: `waitForIdle()`; a hanging one holds them until the host restarts, and the handler's last attempt before the restart limit runs without hooks (section 8.4). +- **Busy after the answer:** with `afterRun` hooks selected, `Submission.wait()` + resolves at the answer while the conversation stays busy until the after-run + tasks end: `whenBusy: "reject"` and a continue throw `ConversationBusy`, and a + steer, also with `whenIdle: "reject"`, queues and starts the next run. +- **After-run tasks waiting on their own run:** an after-run task that awaits + `waitForIdle()` of its own conversation, or the settlement of a follow-up it + submitted, waits for the run that waits for it, and never resolves. Submit and + let the run's end place it. +- **Missing after-run tasks:** an after-run task whose definition is uninstalled + stays blocked, and so does its run: the conversation is busy until Stop + orphans the task. +- **Fix loops:** a follow-up an after-run task submits starts a run with a fresh + turn budget, which asks `afterRun` again; `maxTurns` does not bound the loop, + the extension does, for example with a counter in its own document. +- **Submissions from after-run tasks:** admit them in the task's own commit with + `submit(tx, draft)`, before its table writes; one submitted outside a commit + should carry a stable `requestId`, as a rerun after a crash submits it again. - **Hook memo names:** hooks share the asking task's memo namespace with the task and other hooks. Prefix memo names. - **Long transactions:** an async commit callback holds the Session mutation diff --git a/packages/durable/src/harness/define.ts b/packages/durable/src/harness/define.ts index b9bbd913f..dd4bd2e8d 100644 --- a/packages/durable/src/harness/define.ts +++ b/packages/durable/src/harness/define.ts @@ -1,6 +1,16 @@ import type { JsonValue } from "@earendil-works/chord"; import type { TSchema } from "@earendil-works/pi-ai"; -import type { AnyTask, Extension, HookRegistration, HooksOf, PromptSection, ToolRegistration, Wrap } from "./types.ts"; +import type { Task } from "../types.ts"; +import type { + AnyTask, + Extension, + HookRegistration, + HooksOf, + PromptSection, + RunEndTask, + ToolRegistration, + Wrap, +} from "./types.ts"; /** Identity function that types an extension. */ export function defineExtension( @@ -30,6 +40,14 @@ export function hook(task: K, handlers: Partial>): return { task: task.definition.name, handlers }; } +/** A task an `afterRun` hook asks for, its input typed by the task (spec §8.3). */ +export function runEndTask( + task: Task, + input: I, +): RunEndTask { + return { task, input }; +} + /** Wrap the tool named like `tool` wherever the wrapping extension is selected. */ export function wrapTool(tool: Tool, wrapper: (tool: Tool) => Tool): Wrap { return { tool: tool.name, wrap: wrapper }; diff --git a/packages/durable/src/harness/generation.ts b/packages/durable/src/harness/generation.ts index 2b756bd37..b86af2ed0 100644 --- a/packages/durable/src/harness/generation.ts +++ b/packages/durable/src/harness/generation.ts @@ -1,4 +1,4 @@ -import { type Context, copyJson, type Draft } from "@earendil-works/chord"; +import { type Context, copyJson, type Draft, type JsonValue } from "@earendil-works/chord"; import type { Api, AssistantMessage, @@ -21,13 +21,14 @@ import type { EntryId, NextTaskState, SubmissionId, + Task, TaskId, TaskOutcome, TaskRuntime, Tx, TypedEntry, } from "../types.ts"; -import { addTools, sectionExtension } from "./agent.ts"; +import { addTools, agentHooks, sectionExtension } from "./agent.ts"; import { compactionPolicy, createCompaction, estimateContext, selectCut } from "./compaction.ts"; import { applyBoundary, type Boundary, hasQueuedReset, prepareBoundary, withdrawQueuedInputs } from "./inbox.ts"; import { @@ -44,6 +45,7 @@ import { import { PartialChanges, type PartialMessage } from "./partial-changes.ts"; import { planSystemEntries, renderSections, replaySections } from "./prompt.ts"; import { ensureProviderSessionId } from "./provider.ts"; +import { endedByCrashes } from "./scheduler.ts"; import { appendToolResult, harnessError, ToolTask, type ToolTaskResult } from "./tool.ts"; import type { AfterFailedRun, @@ -54,6 +56,8 @@ import type { GenerationHooks, ModelRef, PromptInput, + RunEndTask, + RunSummary, ToolControl, UserInput, } from "./types.ts"; @@ -115,6 +119,20 @@ export type GenerationCheckpoint = tools: TaskId[]; /** Calls of a sequential round not started yet, in call order. */ pending: string[]; + } + | { + /** + * The run answered: its answer is committed and its inputs settled, and it keeps `pi.live.run` while `afterRun` + * hooks are asked for tasks (spec §8.3). A crash here asks them again: nothing was created yet. + */ + phase: "finish"; + run: RunSummary; + } + | { + /** Waiting on the tasks `afterRun` hooks asked for, which the generation owns; then the run ends. */ + phase: "drain"; + answer: EntryId; + tasks: TaskId[]; }; export type GenerationResult = { entryId: EntryId }; @@ -331,6 +349,36 @@ export const GenerationTask = defineTask { + const { run } = task.state.checkpoint; + if (cutOffTooOften(task, runtime)) return endAnsweredCutOff(runtime, task, run.answer, context); + const prepared: PreparedTask[] = []; + await runtime.hooks.each("afterRun", async (hook) => { + const asked = await hook(run, runtime, context); + if (asked !== undefined) prepareRunEndTasks(runtime, asked.tasks, prepared); + }); + if (prepared.length === 0) return endAnsweredRun(runtime, run.answer, context); + await runtime.commit(async (tx): Promise => { + const tasks: TaskId[] = []; + for (const { task, input } of prepared) { + tasks.push(await tx.createTask(task, input, { ownership: { kind: "task", taskId: runtime.taskId } })); + } + const live = await tx.doc(LiveDoc, runtime.conversationId); + if (live.run?.taskId === runtime.taskId && live.run.afterRun !== undefined) live.run.afterRun.tasks = tasks; + const checkpoint = { phase: "drain", answer: run.answer, tasks } as const; + return { status: "waiting", checkpoint, on: tasks, policy: "allSettled" }; + }, context); + }, + /** The after-run tasks are terminal, whatever their outcomes: end the run. */ + drain: async (task, runtime, context) => { + const { answer } = task.state.checkpoint; + if (cutOffTooOften(task, runtime)) return endAnsweredCutOff(runtime, task, answer, context); + return endAnsweredRun(runtime, answer, context); + }, }, abort: async (task, runtime, context) => { const checkpoint = task.state.checkpoint; @@ -354,6 +402,7 @@ export const GenerationTask = defineTask 0) await startRun(tx, conversationId, live, users, { keepFailure: true }); + const { users, first } = await applyBoundary(tx, failure.boundary, "final", runtime.now()); + if (users.length > 0) await startRun(tx, conversationId, live, { inputs: users, first }, { keepFailure: true }); } /** @@ -780,8 +829,9 @@ async function classify( /** * A final answer; the final boundary places queued items (spec §6). The first `onYield` continuation appends a user - * message and hands the run to a successor generation, but only when the boundary selected no user item and no reset. - * Otherwise the run's inputs settle `done`, and selected user items start the next run. + * message and hands the run to a successor generation, but only when the inbox holds no user item and no reset. + * Otherwise the run's inputs settle `done`; with `afterRun` hooks selected the run waits for the tasks they ask for, + * in `finish` and `drain`, before the final boundary; without, selected user items start the next run now. */ async function answer(runtime: Runtime, message: AssistantMessage, context: Context): Promise { let continuation: UserInput | undefined; @@ -789,6 +839,7 @@ async function answer(runtime: Runtime, message: AssistantMessage, context: Cont if (continuation !== undefined) return; continuation = (await hook(message, runtime, context))?.continue; }); + const afterRun = await asksAfterRun(runtime, context); const conversationId = runtime.conversationId; await runtime.commit(async (tx): Promise => { // Queue modes are read on the Session line, when the boundary is decided. @@ -797,7 +848,11 @@ async function answer(runtime: Runtime, message: AssistantMessage, context: Cont const entry = await appendAssistant(tx, conversationId, message); const result: Next = { status: "terminal", outcome: { status: "completed", result: { entryId: entry.id } } }; const continues = takesContinuation(continuation, boundary) ? continuation : undefined; - const { users } = await applyBoundary(tx, boundary, "final", runtime.now()); + if (continues === undefined && afterRun) { + const held = holdRun(tx, live, runtime, { reason: "answer", answer: entry.id, last: entry.id }); + if (held !== undefined) return held; + } + const { users, first } = await applyBoundary(tx, boundary, "final", runtime.now()); if (continues !== undefined) { const user = { role: "user", content: continues, timestamp: runtime.now() } as const; await tx.appendEntry(UserEntry, conversationId, { model: [user] }); @@ -808,11 +863,107 @@ async function answer(runtime: Runtime, message: AssistantMessage, context: Cont return result; } endRun(tx, live, runtime.taskId, { status: "done", answer: entry.id }); - if (users.length > 0) await startRun(tx, conversationId, live, users); + if (users.length > 0) await startRun(tx, conversationId, live, { inputs: users, first }); return result; }, context); } +/** Whether the phase's agent selects an `afterRun` hook, so a run that answers waits for its tasks (spec §8.3). */ +async function asksAfterRun(runtime: Runtime, context: Context): Promise { + const hooks = agentHooks(await runtime.agent(context), GenerationTask.definition.name); + return hooks.some(({ handlers }) => typeof (handlers as Partial).afterRun === "function"); +} + +/** + * Commit A of a run that answered with `afterRun` hooks selected (spec §8.3): settle the run's inputs `done` with the + * answer and keep `pi.live.run` for the after-run tasks, then ask the hooks in `finish`. The final boundary waits for + * the run's end, so queued items stay queued. + */ +function holdRun(tx: Tx, live: Draft, runtime: Runtime, end: Omit): Next | undefined { + const run = live.run; + if (run?.taskId !== runtime.taskId) return undefined; + for (const id of run.inputs) tx.settleSubmission(id, { status: "done", answer: end.answer }); + run.inputs = []; + run.afterRun = {}; + delete live.generation; + delete live.tools; + delete live.nestedTools; + // A run started before `first` was recorded has its answer as its first entry. + const summary: RunSummary = { ...end, first: run.first ?? end.answer }; + return { status: "running", checkpoint: { phase: "finish", run: summary } }; +} + +/** A task an `afterRun` hook asked for, checked and ready to create. */ +type PreparedTask = { readonly task: Task; readonly input: JsonValue }; + +/** + * Check the tasks one `afterRun` hook asked for, off the line, and add those that pass to `prepared`, so a broken request + * is skipped instead of failing the creating commit: its kind must be an installed task that is not built-in, its input + * JSON, and its definition's `initial()` must return a JSON checkpoint of one of its phases. The task is created from + * the installed definition, as every task runs under it, with that checkpoint, so `initial()` runs once. Then throws + * the rejections, which `hooks.each` reports against the hook's extension. + */ +function prepareRunEndTasks(runtime: Runtime, requests: readonly RunEndTask[], prepared: PreparedTask[]): void { + const rejected: unknown[] = []; + for (const request of requests) { + try { + const name = request.task.definition.name; + if (name.startsWith("pi.")) throw new Error(`An afterRun hook cannot start the built-in task ${name}`); + const installed = runtime.registry.task(name); + if (installed === undefined) throw new Error(`Task ${name} is not installed`); + const input = copyJson(request.input, { omitUndefinedProperties: true }); + // The registry erases definitions; an installed one is a task definition of its kind. + const definition = installed.definition as PreparedTask["task"]["definition"]; + const checkpoint = copyJson(definition.initial(input), { omitUndefinedProperties: true }) as { phase: string }; + if (!Object.hasOwn(definition.phases, checkpoint.phase)) { + throw new Error(`Task ${name} starts at the unknown phase ${String(checkpoint.phase)}`); + } + prepared.push({ task: { definition: { ...definition, initial: () => checkpoint } }, input }); + } catch (error) { + rejected.push(error); + } + } + if (rejected.length === 1) throw rejected[0]; + if (rejected.length > 1) + throw new AggregateError(rejected, `${rejected.length} afterRun task requests were rejected`); +} + +/** + * Commit B of a run that answered (spec §8.3): its after-run tasks are terminal, or there were none. Place what the + * final boundary selects, end the run, and start the next one for selected user items. + */ +async function endAnsweredRun(runtime: Runtime, answer: EntryId, context: Context): Promise { + const conversationId = runtime.conversationId; + await runtime.commit(async (tx): Promise => { + const boundary = await prepareBoundary(tx, conversationId, runtime.settings); + const live = await tx.doc(LiveDoc, conversationId); + const { users, first } = await applyBoundary(tx, boundary, "final", runtime.now()); + endRun(tx, live, runtime.taskId, { status: "done", answer }); + if (users.length > 0) await startRun(tx, conversationId, live, { inputs: users, first }); + return { status: "terminal", outcome: { status: "completed", result: { entryId: answer } } }; + }, context); +} + +/** + * End a run that answered once its `finish` or `drain` phase was cut off `settings.restartLimit` times (spec §5.4): + * the run answered, so it ends `completed`, without a failure and without the tasks it would still have waited for, + * marked so the scheduler reports it `task_interrupted`. Queued items stay queued, as after a failed run. + */ +async function endAnsweredCutOff( + runtime: Runtime, + task: { readonly restarts?: number; readonly state: { readonly checkpoint: GenerationCheckpoint } }, + answer: EntryId, + context: Context, +): Promise { + const text = `The generation was interrupted ${cutOffCount(task)} in its ${task.state.checkpoint.phase} phase`; + await runtime.commit(async (tx): Promise => { + const live = await tx.doc(LiveDoc, runtime.conversationId); + endRun(tx, live, runtime.taskId, { status: "done", answer }); + const completed: Next = { status: "terminal", outcome: { status: "completed", result: { entryId: answer } } }; + return endedByCrashes(completed, text); + }, context); +} + /** * Whether an answer's `onYield` continuation is taken (spec §8.3), decided from the inbox before the final boundary * places anything: only when it holds no user item, of which the boundary would select one, and no reset, which would @@ -871,8 +1022,12 @@ async function startToolRound( } delete live.generation; live.tools = slots; - // A tool round is one turn of the run (`AgentState.maxTurns`). - if (live.run?.taskId === runtime.taskId) live.run.turns += 1; + if (live.run?.taskId === runtime.taskId) { + // A tool round is one turn of the run (`AgentState.maxTurns`). + live.run.turns += 1; + // A run its boundary started without placing anything, as a continue after an answer, begins here. + live.run.first ??= entry.id; + } const checkpoint = { phase: "tools", assistant: entry.id, tools, pending } as const; return { status: "waiting", checkpoint, on: tools, policy: "allSettled" }; }, context); @@ -913,22 +1068,28 @@ async function finishToolRound( const handoff = ordered.findLast((control) => control?.handoff !== undefined)?.handoff; // Read from the phase's agent, as every agent field; a `configure()` during the round applies at its end. const maxTurns = (await runtime.agent(context)).maxTurns; + const afterRun = terminate && (await asksAfterRun(runtime, context)); await runtime.commit(async (tx): Promise => { const boundary = await prepareBoundary(tx, conversationId, runtime.settings); if (added.length > 0) await addTools(tx, conversationId, added); const live = await tx.doc(LiveDoc, conversationId); const now = runtime.now(); + // The run's last entry so far: the newest result, or the assistant when no result has an entry. + let last = results.reduce((newest, id) => (id > newest ? id : newest), assistant); if (handoff !== undefined) { // The handoff's reset precedes every queued item, so queued writes and selected steers land in its context. const message = { role: "user", content: handoff, timestamp: now } as const; const entry = await tx.appendEntry(ResetEntry, conversationId, { head: "self", model: [message] }); boundary.head = entry.id; + last = entry.id; } const turns = live.run?.taskId === runtime.taskId ? live.run.turns : 0; + const held = afterRun ? holdRun(tx, live, runtime, { reason: "terminate", answer: assistant, last }) : undefined; + if (held !== undefined) return held; if (terminate) { - const { users } = await applyBoundary(tx, boundary, "final", now); + const { users, first } = await applyBoundary(tx, boundary, "final", now); endRun(tx, live, runtime.taskId, { status: "done", answer: assistant }); - if (users.length > 0) await startRun(tx, conversationId, live, users); + if (users.length > 0) await startRun(tx, conversationId, live, { inputs: users, first }); } else if (maxTurns !== undefined && turns >= maxTurns) { // The turn limit ends the run like a failure: `afterFailedRun` decides the queued items, and a continue resumes // it on the round's results. The table was read for the boundary already, before any write. @@ -940,11 +1101,13 @@ async function finishToolRound( detail: turns === 1 ? "1 turn" : `${turns} turns`, }); } else { - const { users, reset } = await applyBoundary(tx, boundary, "postTools", now); + const { users, first, reset } = await applyBoundary(tx, boundary, "postTools", now); if (reset) { // The queued reset cut the run's context before an answer. endRun(tx, live, runtime.taskId, { status: "unanswered", reason: "reset" }); - if (users.length > 0) await startRun(tx, conversationId, live, users, { keepFailure: true }); + if (users.length > 0) { + await startRun(tx, conversationId, live, { inputs: users, first }, { keepFailure: true }); + } } else { delete live.tools; delete live.nestedTools; @@ -970,17 +1133,20 @@ async function appendAssistant( } /** - * Start a run for `inputs`, placed input submissions: a new generation takes `pi.live.run`. Removes `runFailure`, unless - * `keepFailure`: a run started in the commit that ended the failed one keeps its failure. + * Start a run for `inputs`, placed input submissions, whose first entry is `first` (`LiveRun.first`): a new generation + * takes `pi.live.run`. Removes `runFailure`, unless `keepFailure`: a run started in the commit that ended the failed one + * keeps its failure. */ export async function startRun( tx: Tx, conversationId: ConversationId, live: Draft, - inputs: SubmissionId[], + run: { readonly inputs: SubmissionId[]; readonly first: EntryId | undefined }, options?: { readonly keepFailure?: boolean }, ): Promise { - live.run = { taskId: await createGeneration(tx, conversationId, { submissionIds: inputs }), inputs, turns: 0 }; + const { inputs, first } = run; + const taskId = await createGeneration(tx, conversationId, { submissionIds: inputs }); + live.run = { taskId, inputs, turns: 0, ...(first === undefined ? {} : { first }) }; if (options?.keepFailure !== true) delete live.runFailure; } diff --git a/packages/durable/src/harness/inbox.ts b/packages/durable/src/harness/inbox.ts index ced697767..3469e8e79 100644 --- a/packages/durable/src/harness/inbox.ts +++ b/packages/durable/src/harness/inbox.ts @@ -41,8 +41,15 @@ export type Boundary = { head: EntryId | undefined; }; -/** Selected user items, in ID order, and whether a `head: "self"` write (a reset) was placed. */ -export type BoundaryResult = { readonly users: SubmissionId[]; readonly reset: boolean }; +/** + * Selected user items, in ID order, the first entry the boundary placed, a write's or a user item's (a run it starts + * begins there, `LiveRun.first`), and whether a `head: "self"` write (a reset) was placed. + */ +export type BoundaryResult = { + readonly users: SubmissionId[]; + readonly first: EntryId | undefined; + readonly reset: boolean; +}; /** * Read what a boundary needs. Table reads must precede the commit's first table write, so callers prepare the @@ -86,6 +93,7 @@ export async function applyBoundary( users.sort((a, b) => a - b); // `appendEntry()` copies the drafts' values; the items are removed only afterwards. + let first: EntryId | undefined; for (const index of writes) { const item = items[index] as Draft>; const draft = item.entry as unknown as EntryDraft; @@ -96,6 +104,7 @@ export async function applyBoundary( const entry = await tx.appendEntry(conversationId, draft); if (draft.head !== undefined) boundary.head = draft.head === "self" ? entry.id : draft.head; tx.placeSubmission(item.id, entry.id); + first ??= entry.id; } const placed: SubmissionId[] = []; for (const index of users) { @@ -103,10 +112,11 @@ export async function applyBoundary( const entry = await appendInput(tx, conversationId, item.content as UserInput, item.entry as InputEntry, now); tx.placeSubmission(item.id, entry.id); placed.push(item.id); + first ??= entry.id; } const removed = [...writes, ...users].sort((a, b) => b - a); for (const index of removed) items.splice(index, 1); - return { users: placed, reset }; + return { users: placed, first, reset }; } /** Append the entry an input is placed as: `entry`'s kind and data, or `pi.user`, with the user message of `content`. */ diff --git a/packages/durable/src/harness/live.ts b/packages/durable/src/harness/live.ts index a0fcccd3b..d9696f46d 100644 --- a/packages/durable/src/harness/live.ts +++ b/packages/durable/src/harness/live.ts @@ -130,15 +130,35 @@ export type RunFailure = { * was cut off, and how often; `turn_limit`: the tool rounds the run took, `"1 turn"` or `"N turns"`. */ detail?: string; + /** The failed run's `run.first`, so a continue's `afterRun` hooks see its history too. */ + first?: EntryId; +}; + +/** Run control (spec §8.2). */ +export type LiveRun = { + /** The task that settles the run's inputs: its generation, handed from one to the next. */ + taskId: TaskId; + /** The run's placed inputs, settled when it ends; `[]` once it answered and runs only its `afterRun` tasks. */ + inputs: SubmissionId[]; + /** Tool rounds so far, which `AgentState.maxTurns` bounds; an `onYield` continuation starts the count over. */ + turns: number; + /** + * The run's first entry: the first one the boundary that started it placed, a continue's the failed run's it + * continues; otherwise its first tool-calling answer. A run started before it was recorded may lack it; `afterRun` + * hooks then get the answer as the first entry. + */ + first?: EntryId; + /** + * Present once the run answered and its generation waits for the tasks `afterRun` hooks asked for, listed once + * created: queued items stay queued until they are terminal, and the run then ends without a failure. + */ + afterRun?: { tasks?: TaskId[] }; }; /** Built-in live conversation state: run control and presentation of the current generation and tool round. */ export type LiveState = { - /** - * Run control: the task that settles the run's inputs, those inputs, and the run's tool rounds so far, which - * `AgentState.maxTurns` bounds (an `onYield` continuation starts the count over); present exactly while busy. - */ - run?: { taskId: TaskId; inputs: SubmissionId[]; turns: number }; + /** Run control; present exactly while busy, also while a run that answered runs its `afterRun` tasks. */ + run?: LiveRun; /** Presentation of the current generation attempt. */ generation?: { attempt: number; @@ -263,14 +283,16 @@ export type RunEnd = /** * End the run owned by `taskId`: settle each of its inputs, remove `run`, and, without an answer, record - * `runFailure`. Always removes `generation`, `tools`, and `nestedTools`, whose presentation belongs to the ending run. + * `runFailure`. A run that answered already, and runs only its `afterRun` tasks, records no failure, however it ends. + * Always removes `generation`, `tools`, and `nestedTools`, whose presentation belongs to the ending run. */ export function endRun(tx: Tx, live: Draft, taskId: TaskId, end: RunEnd): void { if (live.run?.taskId === taskId) { const inputs = [...live.run.inputs]; + const { first } = live.run; if (end.status === "done") { for (const id of inputs) tx.settleSubmission(id, { status: "done", answer: end.answer }); - } else { + } else if (live.run.afterRun === undefined) { const { reason, detail } = end; const settlement = end.settlement ?? { status: "unanswered", @@ -278,7 +300,13 @@ export function endRun(tx: Tx, live: Draft, taskId: TaskId, end: RunE ...(detail === undefined ? {} : { detail }), }; for (const id of inputs) tx.settleSubmission(id, settlement); - live.runFailure = { taskId, inputs, reason, ...(detail === undefined ? {} : { detail }) }; + live.runFailure = { + taskId, + inputs, + reason, + ...(detail === undefined ? {} : { detail }), + ...(first === undefined ? {} : { first }), + }; } delete live.run; } diff --git a/packages/durable/src/harness/scheduler.ts b/packages/durable/src/harness/scheduler.ts index de26aceb3..fc3c2107c 100644 --- a/packages/durable/src/harness/scheduler.ts +++ b/packages/durable/src/harness/scheduler.ts @@ -2114,7 +2114,13 @@ export class TaskScheduler { }, commit: async (change, context, options) => { let checkpoint: string | undefined; - let interrupted: { readonly error: TaskOutcomeError; readonly restarts: number } | undefined; + let interrupted: + | { + readonly message: string; + readonly reason: "restart_limit" | "interrupted"; + readonly restarts: number; + } + | undefined; await this.#gated( invocation, async (tx, current) => { @@ -2124,9 +2130,16 @@ export class TaskScheduler { checkpoint = next.checkpoint.phase; } // A task that ends for crashes says so (`task_interrupted`), so a host keeps a crash-loop signal. - if (next?.status === "terminal" && next.outcome.status === "failed") { - if (interruptionOf(next.outcome.error) !== undefined) { - interrupted = { error: next.outcome.error, restarts: current.restarts ?? 0 }; + if (next?.status === "terminal") { + const restarts = current.restarts ?? 0; + const reason = next.outcome.status === "failed" ? interruptionOf(next.outcome.error) : undefined; + if (reason !== undefined && next.outcome.status === "failed") { + interrupted = { message: next.outcome.error.message, reason, restarts }; + } else { + // A task that ended for crashes with another outcome says so itself, as a generation whose run + // answered ends `completed` without its after-run tasks. + const message = endedByCrashesStates.get(next); + if (message !== undefined) interrupted = { message, reason: "restart_limit", restarts }; } } }, @@ -2135,11 +2148,11 @@ export class TaskScheduler { ); if (interrupted !== undefined) { this.#report( - new Error(interrupted.error.message), + new Error(interrupted.message), { kind: "task_interrupted", ...this.#aboutTask(invocation), - reason: interruptionOf(interrupted.error)!, + reason: interrupted.reason, restarts: interrupted.restarts, }, invocation.context, @@ -2407,6 +2420,19 @@ function recordTask(record: Pick(); + +/** `state`, marked as ending its task for crashes (`task_interrupted` with `message`), whatever its outcome. */ +export function endedByCrashes(state: T, message: string): T { + endedByCrashesStates.set(state, message); + return state; +} + function interruptionOf(error: TaskOutcomeError): "restart_limit" | "interrupted" | undefined { const detail = error.detail; if (typeof detail !== "object" || detail === null || !("reason" in detail)) return undefined; diff --git a/packages/durable/src/harness/submissions.ts b/packages/durable/src/harness/submissions.ts index df0363e67..66f116af3 100644 --- a/packages/durable/src/harness/submissions.ts +++ b/packages/durable/src/harness/submissions.ts @@ -269,9 +269,10 @@ export async function admitSubmission( if (busy) throw new ConversationBusy(conversationId); const boundary = await prepareBoundary(tx, conversationId, queueModes); const { id } = await tx.createSubmission({ ...fields, type: "continue", status: "placed" }); - // Items an earlier failed run left queued (`afterFailedRun: "keep"`) are placed first and join the run. - const { users } = await applyBoundary(tx, boundary, "final", now); - await startRun(tx, conversationId, live, [...users, id]); + // Items an earlier failed run left queued (`afterFailedRun: "keep"`) are placed first and join the run. Its history + // starts with the failed run's, so `afterRun` hooks see what that run did. + const { users, first } = await applyBoundary(tx, boundary, "final", now); + await startRun(tx, conversationId, live, { inputs: [...users, id], first: live.runFailure?.first ?? first }); return { id, status: "placed" }; } // A boundary reads the table, so it is prepared before the first table write; a busy one needs none. @@ -295,8 +296,8 @@ export async function admitSubmission( items.push({ id, mode, content, ...entry }); } if (boundary === undefined) return { id, status: "queued" }; - const { users } = await applyBoundary(tx, boundary, "final", now); - if (users.length > 0) await startRun(tx, conversationId, live, users); + const { users, first } = await applyBoundary(tx, boundary, "final", now); + if (users.length > 0) await startRun(tx, conversationId, live, { inputs: users, first }); return { id, status: draft.type === "write" ? "done" : users.at(-1) === id ? "placed" : "queued" }; } if (draft.type === "write") { @@ -314,6 +315,6 @@ export async function admitSubmission( const entry = await appendInput(tx, conversationId, draft.content, draft.entry, now); const input = { ...fields, type: "input", status: "placed", entry: entry.id } as const; const { id } = await tx.createSubmission(input); - await startRun(tx, conversationId, live, [id]); + await startRun(tx, conversationId, live, { inputs: [id], first: entry.id }); return { id, status: "placed" }; } diff --git a/packages/durable/src/harness/types.ts b/packages/durable/src/harness/types.ts index 99a024c6a..d7f4a05ec 100644 --- a/packages/durable/src/harness/types.ts +++ b/packages/durable/src/harness/types.ts @@ -79,7 +79,8 @@ export type SubmissionDraft = { readonly whenBusy?: "steer" | "followUp" | "reject"; /** * `reject`: an idle conversation (no run) rejects the input with `ConversationIdle` and writes nothing, so the input - * never starts a run. With `whenBusy: "steer"` it steers the current run or is refused. + * never starts a run of its own. With `whenBusy: "steer"` it steers the current run or is refused; a run that + * answered and waits for its `afterRun` tasks is still busy, and such an input queues and starts the next run. */ readonly whenIdle?: "reject"; /** Place the input as this entry; default `pi.user` without data. The kind must not start with `pi.`. */ @@ -975,8 +976,34 @@ export interface GenerationHooks { onYield(answer: AssistantMessage, api: HookApi, context: Context): HookResult<{ readonly continue: UserInput }>; /** After every tool of the round is terminal; `results` are the round's result entries in call order. */ afterTools(assistant: EntryId, results: readonly EntryId[], api: HookApi, context: Context): void | Promise; + /** + * After a run answered, its answer committed and its inputs settled `done`: a final answer, or a `terminate` round; + * not when an `onYield` continuation is taken, and never after a failure or an abort. The requested tasks of every + * hook are created together as children of the generation, which keeps the run, and the conversation busy, until + * they are terminal; then queued items are placed and the run ends (spec §8.3). A crash before they are created asks + * again; once created, never again for this run. + */ + afterRun(run: RunSummary, api: HookApi, context: Context): HookResult<{ readonly tasks: readonly RunEndTask[] }>; } +/** How a run ended, for `afterRun` hooks. Read the run's history with `api.entries({ minEntryId: first, maxEntryId: last })`. */ +export type RunSummary = { + /** A final answer, or a tool round whose every call asked to `terminate`. */ + readonly reason: "answer" | "terminate"; + /** + * The run's first entry (`LiveRun.first`): the first one the boundary that started it placed, for a continue the failed + * run's it continues, otherwise its first tool-calling answer; the answer itself for a run that has none of these. + */ + readonly first: EntryId; + /** The answer, or the terminating round's tool-calling answer, whose results follow it. */ + readonly answer: EntryId; + /** The run's last entry: the answer; for a `terminate` round its newest result, or the reset of its handoff. */ + readonly last: EntryId; +}; + +/** A task an `afterRun` hook asks for, built by `runEndTask()`. */ +export type RunEndTask = { readonly task: AnyTask; readonly input: JsonValue }; + /** A call as tool hooks see it; `parent` is set for a nested call, one a running tool made through `executeTool()`. */ export type ToolHookCall = ToolCall & { readonly parent?: { readonly taskId: TaskId; readonly callId: string }; diff --git a/packages/durable/src/index.ts b/packages/durable/src/index.ts index a08c8ceb5..4b4671129 100644 --- a/packages/durable/src/index.ts +++ b/packages/durable/src/index.ts @@ -31,7 +31,7 @@ export { CompactionTask, type SummaryRequest, } from "./harness/compaction.ts"; -export { defineExtension, defineTool, hook, section, wrapSection, wrapTool } from "./harness/define.ts"; +export { defineExtension, defineTool, hook, runEndTask, section, wrapSection, wrapTool } from "./harness/define.ts"; export { type GenerationCheckpoint, type GenerationInput, @@ -44,6 +44,7 @@ export { type CompactionFailure, type CompactionStatus, LiveDoc, + type LiveRun, type LiveState, type NestedToolSlot, type NestedToolSummary, @@ -112,6 +113,8 @@ export type { Registry, RegistryReader, RegistrySnapshot, + RunEndTask, + RunSummary, Settings, SettledSubmissionRecord, SettledTask, diff --git a/packages/durable/test/examples/32-tests-after-edits.ts b/packages/durable/test/examples/32-tests-after-edits.ts new file mode 100644 index 000000000..32292c521 --- /dev/null +++ b/packages/durable/test/examples/32-tests-after-edits.ts @@ -0,0 +1,140 @@ +// Run the project's tests at the end of every run that edited files, with an `afterRun` hook: the run waits for the +// tests, a failure starts a fix run, and input sent meanwhile waits for them too. +// Run from packages/durable: +// node --conditions=source --experimental-strip-types test/examples/32-tests-after-edits.ts +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { Type } from "@earendil-works/pi-ai"; +import { createModels } from "@earendil-works/pi-ai/models"; +import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { + createRegistry, + defineExtension, + defineTask, + defineTool, + GenerationTask, + Harness, + hook, + LiveDoc, + MemoryStorage, + runEndTask, + ToolResultEntry, +} from "../../src/index.ts"; + +const context = BACKGROUND_CONTEXT; + +// ─── Product code: the project, an edit tool, and the test runner ───────────── + +// The project under test, in memory: the tests pass once `add` adds. +const files = new Map([["math.ts", "export const add = (a, b) => a - b;"]]); +const runTests = () => { + const passed = files.get("math.ts")!.includes("a + b"); + return { passed, output: passed ? "1 passed" : "add(1, 2): expected 3, got -1" }; +}; + +const edit = defineTool({ + name: "edit", + description: "Replace a file's content", + parameters: Type.Object({ path: Type.String(), content: Type.String() }), + execute: async (args) => { + files.set(args.path, args.content); + return { output: [{ type: "text", text: `wrote ${args.path}` }] }; + }, +}); + +// A durable task: it runs once the run answered, and the run ends after it. A crash resumes it. +const RunTests = defineTask<{ attempt: number }, { phase: "run" }, boolean>({ + name: "app.run-tests", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (task, runtime, ctx) => { + const result = runTests(); + console.log(` [tests, attempt ${task.input.attempt}] ${result.output}`); + // The handle is fetched before the commit: inside it, runtime reads would wait for the commit itself. + const conversation = (await runtime.conversation(runtime.conversationId, ctx))!; + await runtime.commit(async (tx) => { + // Admitted with this commit, before its own table writes: placed when the run ends. + if (!result.passed && task.input.attempt < 3) { + await conversation.submit(tx, { + type: "input", + content: `The tests failed: ${result.output}. Fix it.`, + whenBusy: "followUp", + requestId: `fix-${task.input.attempt}`, + }); + } + await tx.appendEntry(runtime.conversationId, { kind: "app.tests", data: result }); + return { status: "terminal", outcome: { status: "completed", result: result.passed } }; + }, ctx); + }, + }, + abort: async (_task, runtime, ctx) => + runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }) as const, ctx), +}); + +let attempts = 0; +const TestsAfterEdits = defineExtension({ + name: "tests-after-edits", + tools: [edit], + tasks: [RunTests], + hooks: [ + hook(GenerationTask, { + afterRun: async (run, api, ctx) => { + // The run's own entries, from its first input to its answer; a compaction never removes them. + const range = { minEntryId: run.first, maxEntryId: run.last, order: "ascending" } as const; + const { items } = await api.entries(range, 1000, undefined, ctx); + const edited = items.some((entry) => { + const message = entry.model?.[0]; + return ToolResultEntry.is(entry) && message?.role === "toolResult" && message.toolName === "edit"; + }); + // Each fix run asks again with a fresh turn budget: the extension caps the loop itself. + return edited ? { tasks: [runEndTask(RunTests, { attempt: ++attempts })] } : undefined; + }, + }), + ], +}); + +// ─── Host setup ───────────────────────────────────────────────────────────── + +const faux = fauxProvider(); +const models = createModels(); +models.setProvider(faux.provider); +faux.setResponses([ + // The first run edits, wrongly. + fauxAssistantMessage(fauxToolCall("edit", { path: "math.ts", content: "export const add = (a, b) => a - b + 0;" }), { + stopReason: "toolUse", + }), + fauxAssistantMessage("I cleaned up add."), + // The user's question, queued first, runs first; it edits nothing, so no tests run after it. + fauxAssistantMessage("Not yet: add subtracts. The fix is queued next."), + // The fix run the failed tests started. + fauxAssistantMessage(fauxToolCall("edit", { path: "math.ts", content: "export const add = (a, b) => a + b;" }), { + stopReason: "toolUse", + }), + fauxAssistantMessage("Fixed: add adds now."), +]); + +const registry = createRegistry(); +registry.install(TestsAfterEdits); +const harness = await Harness.open(new MemoryStorage(), { models, registry }, context); +const root = await harness.root(context, { agent: { model: { provider: "faux", modelId: "faux-1" } } }); + +const submission = await root.submit({ type: "input", content: "Clean up math.ts." }, context); +// Resolves with the answer; the conversation stays busy while the tests run. +console.log("answered:", (await submission.wait(context)).status); +const live = await harness.snapshot(LiveDoc, root.id, context); +console.log("busy with after-run tasks:", live?.run?.afterRun !== undefined); +// Sent while the tests run: it waits for them. Follow-ups run in queue order, so it runs before the fix run. +await root.submit({ type: "input", content: "Are the tests green now?" }, context); + +await root.waitForIdle(context); +const page = await root.entries({ order: "ascending" }, 100, undefined, context); +for (const entry of page.items) { + const message = entry.model?.[0]; + if (entry.kind === "app.tests") console.log("tests:", (entry.data as { output: string }).output); + else if (message?.role === "user") console.log(">", message.content); + else if (message?.role === "assistant") { + const text = message.content.flatMap((part) => (part.type === "text" ? [part.text] : [])).join(""); + if (text !== "") console.log("agent:", text); + } +} +await harness.close(context); diff --git a/packages/durable/test/harness-after-run.test.ts b/packages/durable/test/harness-after-run.test.ts new file mode 100644 index 000000000..373922122 --- /dev/null +++ b/packages/durable/test/harness-after-run.test.ts @@ -0,0 +1,1204 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Draft, JsonValue } from "@earendil-works/chord"; +import { fauxAssistantMessage, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { + CompactionTask, + ConversationBusy, + type ConversationHandle, + defineExtension, + defineTask, + defineTool, + type EntryRecord, + type GenerationHooks, + GenerationTask, + hook, + InboxDoc, + LiveDoc, + type LiveState, + MemoryStorage, + type RunEndTask, + type RunSummary, + runEndTask, + type Storage, + type TaskId, + type Tx, +} from "@earendil-works/pi-durable"; +import { afterEach, describe, expect, it } from "vitest"; +import { endRun } from "../src/harness/live.ts"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { allEntries, type ChatSetup, chatSetup, openChat, textOf } from "./chat-support.ts"; +import { context } from "./session-support.ts"; +import { aborted, completed, type Deferred, deferred, eventually, settled } from "./task-support.ts"; + +const directories = new Set(); +/** Storages of killed Harnesses, never closed by them: closed after each test. */ +const killed = new Set(); + +afterEach(async () => { + for (const storage of killed) await storage.close(context).catch(() => {}); + killed.clear(); + for (const directory of directories) await rm(directory, { recursive: true, force: true }); + directories.clear(); +}); + +async function sqlitePath(): Promise { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-after-run-")); + directories.add(directory); + return join(directory, "session.sqlite"); +} + +/** Open a chat over SQLite at `path`; `kill()` simulates a crash: every later storage call hangs, so nothing more lands. */ +async function openKillable(path: string, setup: ChatSetup) { + const inner = await openNodeSqliteStorage(path); + let dead = false; + const storage = new Proxy(inner, { + get(target, property) { + const value = Reflect.get(target, property, target); + if (typeof value !== "function") return value; + return (...args: unknown[]) => (dead ? new Promise(() => {}) : value.apply(target, args)); + }, + }); + const opened = await openChat(storage, setup); + const kill = (): void => { + dead = true; + killed.add(inner); + }; + return { ...opened, kill }; +} + +/** What the after-run task and hook of a test did, and how they behave. */ +type Probe = { + /** Labels of the after-run task's runs, in order, reruns included. */ + runs: string[]; + /** Summaries the `afterRun` hook was asked with. */ + asked: RunSummary[]; + aborted: number; + /** Holds every run of the after-run task until resolved. */ + gate?: Deferred; + /** Holds the hook until resolved. */ + hookGate?: Deferred; + /** Admitted in the after-run task's commit, before its own entry. */ + admit?: (handle: ConversationHandle, tx: Tx) => Promise; +}; + +function probe(): Probe { + return { runs: [], asked: [], aborted: 0 }; +} + +/** The after-run task: records its run, waits for the gate, then appends a `test.tests` entry and completes. */ +function testsTask(state: Probe) { + return defineTask<{ label: string }, { phase: "run" }, string>({ + name: "test.run-tests", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (task, runtime, ctx) => { + state.runs.push(task.input.label); + if (state.gate !== undefined) await Promise.race([state.gate.promise, aborted(runtime.signal)]); + const handle = + state.admit === undefined ? undefined : await runtime.conversation(runtime.conversationId, ctx); + await runtime.commit(async (tx) => { + if (state.admit !== undefined) await state.admit(handle!, tx); + await tx.appendEntry(runtime.conversationId, { kind: "test.tests", data: task.input.label }); + return completed(task.input.label); + }, ctx); + }, + }, + abort: async (_task, runtime, ctx) => { + state.aborted++; + await runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }) as const, ctx); + }, + }); +} + +/** + * Install the after-run extension: its task and an `afterRun` hook, by default asking for one task per run. Returns + * the task. + */ +function install(setup: ChatSetup, state: Probe, afterRun?: GenerationHooks["afterRun"]) { + const Tests = testsTask(state); + const handler: GenerationHooks["afterRun"] = + afterRun ?? + (async (run) => { + state.asked.push(run); + if (state.hookGate !== undefined) await state.hookGate.promise; + return { tasks: [runEndTask(Tests, { label: `run ${state.asked.length}` })] }; + }); + setup.registry.install( + defineExtension({ name: "after-run", tasks: [Tests], hooks: [hook(GenerationTask, { afterRun: handler })] }), + ); + return Tests; +} + +const Parameters = Type.Object({}); +const tools = defineExtension({ + name: "tools", + tools: [ + defineTool({ + name: "edit", + description: "Edit a file", + parameters: Parameters, + execute: async () => ({ output: [{ type: "text", text: "edited" }] }), + }), + defineTool({ + name: "finish", + description: "End the run", + parameters: Parameters, + execute: async () => ({ output: [{ type: "text", text: "finished" }], control: { terminate: true } }), + }), + defineTool({ + name: "handoff", + description: "End the run in a new context", + parameters: Parameters, + execute: async () => ({ + output: [{ type: "text", text: "handed off" }], + control: { terminate: true, handoff: "fresh start" }, + }), + }), + ], +}); + +function call(name: string, id: string) { + return fauxAssistantMessage([fauxToolCall(name, {}, { id })], { stopReason: "toolUse" }); +} + +function kindOf(entries: readonly EntryRecord[], kind: string): EntryRecord[] { + return entries.filter((entry) => entry.kind === kind); +} + +/** Texts of the user messages of the conversation's entries, in order. */ +function userTexts(entries: readonly EntryRecord[]): string[] { + return entries.flatMap((entry) => + (entry.model ?? []).flatMap((message) => (message.role === "user" ? [textOf(message) ?? ""] : [])), + ); +} + +describe("afterRun", () => { + // #11141 + it("waits for the tasks an answer's hooks ask for: inputs done at the answer, the run busy until they end", async () => { + const setup = chatSetup(); + const state = probe(); + state.gate = deferred(); + install(setup, state); + setup.registry.install(tools); + setup.faux.setResponses([call("edit", "a"), fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + // The input settles with the answer, before the task finishes. + expect((await submission.wait(context)).status).toBe("done"); + await eventually(() => state.runs.length === 1); + const live = (await harness.snapshot(LiveDoc, root.id, context))!; + expect(live.run?.inputs).toEqual([]); + expect(live.run?.afterRun?.tasks).toHaveLength(1); + const child = await harness.getTask(live.run!.afterRun!.tasks![0]!, context); + expect(child?.owner).toBe(live.run!.taskId); + expect(child?.background).toBe(false); + const idle = root.waitForIdle(context); + expect(await settled(idle)).toBe(false); + + const entries = await allEntries(root); + const answer = kindOf(entries, "pi.assistant").at(-1)!; + expect(state.asked).toEqual([ + { reason: "answer", first: kindOf(entries, "pi.user")[0]!.id, answer: answer.id, last: answer.id }, + ]); + state.gate.resolve(); + await idle; + expect((await harness.snapshot(LiveDoc, root.id, context))?.run).toBeUndefined(); + expect((await harness.snapshot(LiveDoc, root.id, context))?.runFailure).toBeUndefined(); + expect((await allEntries(root)).at(-1)?.kind).toBe("test.tests"); + const generation = await harness.getTask(live.run!.taskId, context); + expect(generation?.state).toMatchObject({ + status: "terminal", + outcome: { status: "completed", result: { entryId: answer.id } }, + }); + await harness.close(context); + }); + + // #11141 + it("asks after a terminate round, the run's last entry its newest result or its handoff's reset", async () => { + for (const [tool, lastKind] of [ + ["finish", "pi.tool-result"], + ["handoff", "pi.reset"], + ] as const) { + const setup = chatSetup(); + const state = probe(); + install(setup, state); + setup.registry.install(tools); + setup.faux.setResponses([call(tool, "a")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await root.waitForIdle(context); + const entries = await allEntries(root); + const assistant = kindOf(entries, "pi.assistant").at(-1)!; + expect(state.asked).toEqual([ + { + reason: "terminate", + first: kindOf(entries, "pi.user")[0]!.id, + answer: assistant.id, + last: kindOf(entries, lastKind).at(-1)!.id, + }, + ]); + expect(state.runs).toEqual(["run 1"]); + expect((await harness.snapshot(LiveDoc, root.id, context))?.run).toBeUndefined(); + await harness.close(context); + } + }); + + // #11141 + it("ends the run in one commit without an afterRun hook, and after the hook when it asks for nothing", async () => { + const plain = chatSetup(); + const state = probe(); + plain.registry.install(defineExtension({ name: "tasks-only", tasks: [testsTask(state)] })); + plain.faux.setResponses([fauxAssistantMessage("done")]); + const opened = await openChat(new MemoryStorage(), plain); + await (await opened.root.submit({ type: "input", content: "go" }, context)).wait(context); + // No hook: the answer's commit ended the run. + expect(await opened.harness.snapshot(LiveDoc, opened.root.id, context)).toEqual({}); + await opened.harness.close(context); + + const setup = chatSetup(); + install(setup, state, async (run) => { + state.asked.push(run); + return undefined; + }); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await root.waitForIdle(context); + expect(state.asked).toHaveLength(1); + expect(state.runs).toEqual([]); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + await harness.close(context); + }); + + // #11141 + it("queues input during the tasks and places it after them: writes first, then steers and follow-ups", async () => { + const setup = chatSetup(); + const state = probe(); + state.gate = deferred(); + const Tests = testsTask(state); + setup.registry.install( + defineExtension({ + name: "after-run", + tasks: [Tests], + hooks: [ + hook(GenerationTask, { + afterRun: async (run) => { + state.asked.push(run); + // Only the first run gets a task. + return state.asked.length === 1 ? { tasks: [runEndTask(Tests, { label: "run 1" })] } : undefined; + }, + }), + ], + }), + ); + // The task admits its result for the model in its own commit. + state.admit = (handle, tx) => + handle.submit(tx, { + type: "write", + entry: { kind: "test.result", model: [{ role: "user", content: "tests passed", timestamp: 1 }] }, + }); + setup.faux.setResponses([ + fauxAssistantMessage("first"), + fauxAssistantMessage("second"), + fauxAssistantMessage("third"), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.runs.length === 1); + + const steer = await root.submit({ type: "input", content: "steer", whenBusy: "steer" }, context); + const followUp = await root.submit({ type: "input", content: "later" }, context); + await expect(root.submit({ type: "input", content: "x", whenBusy: "reject" }, context)).rejects.toBeInstanceOf( + ConversationBusy, + ); + await expect(root.submit({ type: "continue" }, context)).rejects.toBeInstanceOf(ConversationBusy); + // Busy, so not idle: it queues and starts the next run after the tasks. + const idleSteer = await root.submit( + { type: "input", content: "steer 2", whenBusy: "steer", whenIdle: "reject" }, + context, + ); + const note = await root.submit({ type: "write", entry: { kind: "test.note" } }, context); + expect((await note.status(context)).status).toBe("done"); + expect((await steer.status(context)).status).toBe("queued"); + expect(await harness.snapshot(InboxDoc, root.id, context)).toMatchObject({ items: { length: 3 } }); + + state.gate.resolve(); + await root.waitForIdle(context); + for (const submission of [steer, followUp, idleSteer]) { + expect((await submission.wait(context)).status).toBe("done"); + } + const entries = await allEntries(root); + const kinds = entries.filter((entry) => entry.kind !== "pi.system").map((entry) => entry.kind); + expect(kinds).toEqual([ + "pi.user", + "pi.assistant", + "test.note", + "test.tests", + "test.result", + "pi.user", + "pi.user", + "pi.assistant", + "pi.user", + "pi.assistant", + ]); + expect(userTexts(entries)).toEqual(["go", "tests passed", "steer", "later", "steer 2"]); + await harness.close(context); + }); + + // #11141 + it("places a reset queued during the tasks after them", async () => { + const setup = chatSetup(); + const state = probe(); + state.gate = deferred(); + install(setup, state); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.runs.length === 1); + await root.reset(undefined, context); + expect(kindOf(await allEntries(root), "pi.reset")).toEqual([]); + state.gate.resolve(); + await root.waitForIdle(context); + const kinds = (await allEntries(root)).map((entry) => entry.kind); + expect(kinds.slice(-2)).toEqual(["test.tests", "pi.reset"]); + await harness.close(context); + }); + + // #11141 + it("asks once a run really ends: not while onYield continues it, and when a queued steer drops the continuation", async () => { + const setup = chatSetup(); + const state = probe(); + install(setup, state); + let yields = 0; + setup.registry.install( + defineExtension({ + name: "again", + hooks: [hook(GenerationTask, { onYield: async () => (++yields <= 2 ? { continue: "again" } : undefined) })], + }), + ); + setup.faux.setResponses([fauxAssistantMessage("1"), fauxAssistantMessage("2"), fauxAssistantMessage("3")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await root.waitForIdle(context); + const entries = await allEntries(root); + expect(state.asked).toEqual([ + { + reason: "answer", + first: kindOf(entries, "pi.user")[0]!.id, + answer: kindOf(entries, "pi.assistant").at(-1)!.id, + last: kindOf(entries, "pi.assistant").at(-1)!.id, + }, + ]); + await harness.close(context); + + // A steer queued while the answer's hooks run: the continuation is dropped, and the run ends after its task. + const dropped = chatSetup(); + const second = probe(); + install(dropped, second); + let root2: Awaited>["root"] | undefined; + dropped.registry.install( + defineExtension({ + name: "steering", + hooks: [ + hook(GenerationTask, { + onYield: async () => { + if (second.asked.length > 0) return undefined; + await root2!.submit({ type: "input", content: "steer", whenBusy: "steer" }, context); + return { continue: "again" }; + }, + }), + ], + }), + ); + dropped.faux.setResponses([fauxAssistantMessage("1"), fauxAssistantMessage("2")]); + const opened = await openChat(new MemoryStorage(), dropped); + root2 = opened.root; + await (await opened.root.submit({ type: "input", content: "go" }, context)).wait(context); + await opened.root.waitForIdle(context); + expect(second.asked).toHaveLength(2); + expect(second.runs).toEqual(["run 1", "run 2"]); + expect(userTexts(await allEntries(opened.root))).toEqual(["go", "steer"]); + await opened.harness.close(context); + }); + + // #11141 + it("asks the hooks again after a crash before the tasks exist, and never once they do", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const state = probe(); + state.hookGate = deferred(); + install(setup, state); + setup.faux.setResponses([fauxAssistantMessage("done")]); + let opened = await openKillable(path, setup); + await (await opened.root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.asked.length === 1); + opened.kill(); // a crash while the hook runs, in `finish` + state.hookGate = undefined; + state.gate = deferred(); + opened = await openKillable(path, setup); + opened.harness.resume(); + await eventually(() => state.runs.length === 1); + expect(state.asked).toHaveLength(2); + opened.kill(); // a crash while the task runs, in `drain` + state.gate = undefined; + opened = await openKillable(path, setup); + await opened.root.waitForIdle(context); + // The task ran again; the hook was not asked again, and only one task exists. + expect(state.asked).toHaveLength(2); + expect(state.runs).toEqual(["run 2", "run 2"]); + expect(kindOf(await allEntries(opened.root), "test.tests")).toHaveLength(1); + const tasks = (await opened.harness.inspect(context)).tasks; + expect(tasks).toEqual([]); + await opened.harness.close(context); + + // An orderly close in `drain` resumes the same way. + const closing = await sqlitePath(); + const other = probe(); + other.gate = deferred(); + const setup2 = chatSetup(); + install(setup2, other); + setup2.faux.setResponses([fauxAssistantMessage("done")]); + let reopened = await openKillable(closing, setup2); + await (await reopened.root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => other.runs.length === 1); + await reopened.harness.close(context); + other.gate = undefined; + reopened = await openKillable(closing, setup2); + await reopened.root.waitForIdle(context); + expect(other.asked).toHaveLength(1); + expect(other.runs).toEqual(["run 1", "run 1"]); + await reopened.harness.close(context); + }); + + // #11141 + it("ends an answered run without a failure when Stop comes while the hooks run or the tasks run", async () => { + for (const stage of ["finish", "drain"] as const) { + const setup = chatSetup(); + const state = probe(); + const Tests = testsTask(state); + state.gate = deferred(); + setup.registry.install( + defineExtension({ + name: "after-run", + tasks: [Tests], + hooks: [ + hook(GenerationTask, { + afterRun: async (run, _api, ctx) => { + state.asked.push(run); + if (stage === "finish") await aborted(ctx.abortSignal!); + return { tasks: [runEndTask(Tests, { label: "run 1" })] }; + }, + }), + ], + }), + ); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + await submission.wait(context); + await eventually(() => (stage === "finish" ? state.asked.length === 1 : state.runs.length === 1)); + await root.abort(context); + const live = await harness.snapshot(LiveDoc, root.id, context); + expect(live?.run).toBeUndefined(); + expect(live?.runFailure).toBeUndefined(); + expect((await submission.status(context)).status).toBe("done"); + expect(state.aborted).toBe(stage === "drain" ? 1 : 0); + expect(state.runs).toEqual(stage === "drain" ? ["run 1"] : []); + await harness.close(context); + } + }); + + // #11141 + it("ends the run normally when one of its tasks is aborted on its own", async () => { + const setup = chatSetup(); + const state = probe(); + state.gate = deferred(); + install(setup, state); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.runs.length === 1); + const live = (await harness.snapshot(LiveDoc, root.id, context))!; + await harness.abortTask(live.run!.afterRun!.tasks![0]!, context); + await root.waitForIdle(context); + expect(state.aborted).toBe(1); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + expect((await harness.getTask(live.run!.taskId, context))?.state).toMatchObject({ + status: "terminal", + outcome: { status: "completed" }, + }); + await harness.close(context); + }); + + // #11141 + it("reports broken requests and hooks and creates the rest", async () => { + const setup = chatSetup(); + const state = probe(); + const Tests = testsTask(state); + const Throwing = defineTask({ + name: "test.throwing", + version: 1, + initial: () => { + throw new Error("initial failed"); + }, + phases: { run: async () => {} }, + abort: async () => {}, + }); + // Its `initial()` starts at a phase the definition does not have. + const BadStart = defineTask({ + name: "test.bad-start", + version: 1, + initial: () => ({ phase: "nowhere" }) as unknown as { phase: "run" }, + phases: { run: async () => {} }, + abort: async () => {}, + }); + const requests: RunEndTask[] = [ + { task: { definition: { ...Tests.definition, name: "test.unknown" } }, input: null }, + { task: GenerationTask, input: {} }, + { task: Tests, input: { label: () => "not JSON" } as unknown as JsonValue }, + runEndTask(Throwing, null), + runEndTask(BadStart, null), + runEndTask(Tests, { label: "ok" }), + ]; + setup.registry.install( + defineExtension({ + name: "after-run", + tasks: [Tests, Throwing, BadStart], + hooks: [hook(GenerationTask, { afterRun: async () => ({ tasks: requests }) })], + }), + ); + setup.registry.install( + defineExtension({ + name: "broken", + hooks: [ + hook(GenerationTask, { + afterRun: async () => { + throw new Error("hook failed"); + }, + }), + ], + }), + ); + setup.registry.install( + defineExtension({ + name: "not-a-list", + hooks: [hook(GenerationTask, { afterRun: async () => ({ tasks: "x" as unknown as RunEndTask[] }) })], + }), + ); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await root.waitForIdle(context); + expect(state.runs).toEqual(["ok"]); + // One report per hook, against the hook's extension; the requests of one hook come together. + expect(setup.infos.filter((info) => info.kind === "extension_error")).toEqual([ + expect.objectContaining({ extension: "after-run" }), + expect.objectContaining({ extension: "broken" }), + expect.objectContaining({ extension: "not-a-list" }), + ]); + const rejected = setup.reports[0] as AggregateError; + expect(rejected.errors.map((error: Error) => error.message)).toEqual([ + "Task test.unknown is not installed", + "An afterRun hook cannot start the built-in task pi.generation", + "Value contains a non-JSON function; expected strict JSON", + "initial failed", + "Task test.bad-start starts at the unknown phase nowhere", + ]); + expect((setup.reports[1] as Error).message).toBe("hook failed"); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + await harness.close(context); + }); + + // #11141 + it("creates a requested task from the installed definition, calling its initial() once, off the line", async () => { + const setup = chatSetup(); + const calls = { installed: 0, token: 0 }; + const define = (version: number, count: () => void) => + defineTask({ + name: "test.versioned", + version, + initial: () => { + count(); + return { phase: "run" }; + }, + phases: { run: async (_task, runtime, ctx) => runtime.commit(() => completed(null), ctx) }, + abort: async () => {}, + }); + const Installed = define(2, () => calls.installed++); + // The hook's extension holds an older copy of the task. + const Stale = define(1, () => calls.token++); + setup.registry.install(defineExtension({ name: "tasks", tasks: [Installed] })); + setup.registry.install( + defineExtension({ + name: "asker", + hooks: [hook(GenerationTask, { afterRun: async () => ({ tasks: [runEndTask(Stale, null)] }) })], + }), + ); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await root.waitForIdle(context); + const created = await harness.commit( + (tx) => tx.scanTasks({ conversationId: root.id, kind: "test.versioned" }, 10), + context, + ); + expect(created.items).toHaveLength(1); + expect(created.items[0]).toMatchObject({ version: 2, state: { status: "terminal" } }); + expect(calls).toEqual({ installed: 1, token: 0 }); + await harness.close(context); + }); + + // #11141 + it("lets a hook read the run's history, tool results included", async () => { + const setup = chatSetup(); + const kinds: string[] = []; + setup.registry.install(tools); + setup.registry.install( + defineExtension({ + name: "reader", + hooks: [ + hook(GenerationTask, { + afterRun: async (run, api, ctx) => { + const page = await api.entries( + { minEntryId: run.first, maxEntryId: run.last, order: "ascending" }, + 100, + undefined, + ctx, + ); + kinds.push(...page.items.map((entry) => entry.kind)); + expect((await api.entry(run.answer, ctx))?.kind).toBe("pi.assistant"); + return undefined; + }, + }), + ], + }), + ); + setup.faux.setResponses([call("edit", "a"), fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await root.waitForIdle(context); + expect(kinds.filter((kind) => kind !== "pi.system")).toEqual([ + "pi.user", + "pi.assistant", + "pi.tool-result", + "pi.assistant", + ]); + await harness.close(context); + }); + + // #11141 + it("lets a hook read the run's history after a compaction summarized the start of the run", async () => { + const setup = chatSetup(); + setup.settings.compaction = { enabled: false, reserveTokens: 1000, keepRecentTokens: 1, backgroundTokens: 0 }; + const gate = deferred(); + const started = deferred(); + const runs: string[][] = []; + setup.registry.install( + defineExtension({ + name: "slow", + tools: [ + defineTool({ + name: "edit", + description: "Edit a file", + parameters: Parameters, + execute: async () => { + started.resolve(); + await gate.promise; + return { output: [{ type: "text", text: "edited" }] }; + }, + }), + ], + hooks: [ + hook(CompactionTask, { beforeCompact: async () => ({ summary: "earlier work" }) }), + hook(GenerationTask, { + afterRun: async (run, api, ctx) => { + const page = await api.entries( + { minEntryId: run.first, maxEntryId: run.last, order: "ascending" }, + 100, + undefined, + ctx, + ); + runs.push(page.items.map((entry) => entry.kind)); + return undefined; + }, + }), + ], + }), + ); + setup.faux.setResponses([call("edit", "a"), fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + await started.promise; + // Its summary is placed at the round's post-tools boundary, in the middle of the run. + const compaction = await root.compact(undefined, context); + await harness.waitForTask(compaction, context); + gate.resolve(); + await submission.wait(context); + await root.waitForIdle(context); + const active = (await root.context(context)).entries.map((entry) => entry.kind); + // The active context lost the run's input to the summary; the run's history did not. + expect(active).not.toContain("pi.user"); + expect(runs).toHaveLength(1); + expect(runs[0]!.filter((kind) => kind !== "pi.system")).toEqual([ + "pi.user", + "pi.assistant", + "pi.tool-result", + "pi.compaction", + "pi.assistant", + ]); + await harness.close(context); + }); + + // #11141 + it("ends an answered run completed, without a failure, once finish was cut off restartLimit times", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.restartLimit = 1; + const state = probe(); + state.hookGate = deferred(); + install(setup, state); + setup.faux.setResponses([fauxAssistantMessage("done")]); + let opened = await openKillable(path, setup); + await (await opened.root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.asked.length === 1); + await opened.root.submit({ type: "input", content: "later" }, context); + opened.kill(); + opened = await openKillable(path, setup); + await opened.root.waitForIdle(context); + expect(state.asked).toHaveLength(1); + const live = await opened.harness.snapshot(LiveDoc, opened.root.id, context); + expect(live?.run).toBeUndefined(); + expect(live?.runFailure).toBeUndefined(); + // Queued items stay, as after a failed run. + expect((await opened.harness.snapshot(InboxDoc, opened.root.id, context))?.items).toHaveLength(1); + expect(setup.infos).toContainEqual( + expect.objectContaining({ kind: "task_interrupted", reason: "restart_limit" }), + ); + await opened.harness.close(context); + }); + + // #11141 + it("keeps the run busy while an after-run task cannot run, until Stop orphans it", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const state = probe(); + state.gate = deferred(); + install(setup, state); + setup.faux.setResponses([fauxAssistantMessage("done")]); + let opened = await openKillable(path, setup); + await (await opened.root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.runs.length === 1); + opened.kill(); + // Reopened without the extension: its task is blocked. + const bare = chatSetup(); + opened = await openKillable(path, bare); + opened.harness.resume(); + const live = (await opened.harness.snapshot(LiveDoc, opened.root.id, context))!; + expect(live.run?.afterRun?.tasks).toHaveLength(1); + await eventually(async () => + (await opened.harness.inspect(context)).tasks.some((task) => task.state.kind === "blocked"), + ); + expect(await settled(opened.root.waitForIdle(context))).toBe(false); + await opened.root.abort(context); + const after = await opened.harness.snapshot(LiveDoc, opened.root.id, context); + expect(after?.run).toBeUndefined(); + expect(after?.runFailure).toBeUndefined(); + expect((await opened.harness.getTask(live.run!.afterRun!.tasks![0]!, context))?.state).toMatchObject({ + status: "terminal", + outcome: { status: "orphaned" }, + }); + await opened.harness.close(context); + }); + + // #11141 + it("starts a fix run with a follow-up a task admits, after the tasks", async () => { + const setup = chatSetup(); + const state = probe(); + const Tests = testsTask(state); + setup.registry.install( + defineExtension({ + name: "after-run", + tasks: [Tests], + hooks: [ + hook(GenerationTask, { + afterRun: async (run) => { + state.asked.push(run); + return state.asked.length === 1 ? { tasks: [runEndTask(Tests, { label: "run 1" })] } : undefined; + }, + }), + ], + }), + ); + state.admit = (handle, tx) => + handle.submit(tx, { type: "input", content: "fix the tests", requestId: "fix-1", whenBusy: "followUp" }); + setup.faux.setResponses([fauxAssistantMessage("done"), fauxAssistantMessage("fixed")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await root.waitForIdle(context); + const entries = await allEntries(root); + expect(entries.filter((entry) => entry.kind !== "pi.system").map((entry) => entry.kind)).toEqual([ + "pi.user", + "pi.assistant", + "test.tests", + "pi.user", + "pi.assistant", + ]); + expect(userTexts(entries)).toEqual(["go", "fix the tests"]); + expect(state.asked).toHaveLength(2); + // The fix run's history starts with its own input. + expect(state.asked[1]!.first).toBe(kindOf(entries, "pi.user")[1]!.id); + await harness.close(context); + }); + + // #11141 + it("gives a continue after a failed run the failed run's history", async () => { + const setup = chatSetup(); + const state = probe(); + install(setup, state); + setup.registry.install(tools); + setup.faux.setResponses([call("edit", "a"), fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await root.configure({ maxTurns: 1 }, context); + const failed = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(failed.status).toBe("unanswered"); + const user = kindOf(await allEntries(root), "pi.user")[0]!; + expect((await harness.snapshot(LiveDoc, root.id, context))?.runFailure).toMatchObject({ + reason: "turn_limit", + first: user.id, + }); + expect(state.asked).toEqual([]); + await harness.commit((tx) => root.submit(tx, { type: "continue" }), context); + await root.waitForIdle(context); + const answer = kindOf(await allEntries(root), "pi.assistant").at(-1)!; + expect(state.asked).toEqual([{ reason: "answer", first: user.id, answer: answer.id, last: answer.id }]); + await harness.close(context); + }); + + // #11141 + it("ends the run without asking when the hooks are deselected before finish runs", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const state = probe(); + state.hookGate = deferred(); + install(setup, state); + setup.faux.setResponses([fauxAssistantMessage("done")]); + let opened = await openKillable(path, setup); + await (await opened.root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.asked.length === 1); + opened.kill(); + setup.registry.uninstall(defineExtension({ name: "after-run" })); + opened = await openKillable(path, setup); + await opened.root.waitForIdle(context); + expect(state.asked).toHaveLength(1); + expect(state.runs).toEqual([]); + expect(await opened.harness.snapshot(LiveDoc, opened.root.id, context)).toEqual({}); + await opened.harness.close(context); + }); + + // #11141 + it("starts a fork of a conversation running its after-run tasks idle", async () => { + const setup = chatSetup(); + const state = probe(); + state.gate = deferred(); + install(setup, state); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.runs.length === 1); + await root.submit({ type: "input", content: "later" }, context); + const answer = kindOf(await allEntries(root), "pi.assistant").at(-1)!; + const fork = await root.fork(answer.id, { ownership: { kind: "ownerless" } }, context); + expect((await harness.snapshot(LiveDoc, fork.id, context))?.run).toBeUndefined(); + expect((await harness.snapshot(InboxDoc, fork.id, context))?.items ?? []).toEqual([]); + state.gate.resolve(); + await root.waitForIdle(context); + await harness.close(context); + }); + + // #11141 + it("gives a run without a recorded first entry its answer as first", async () => { + const setup = chatSetup(); + const state = probe(); + install(setup, state); + let root: Awaited>["root"] | undefined; + setup.registry.install( + defineExtension({ + name: "legacy", + hooks: [ + hook(GenerationTask, { + // As a run started before `first` existed. + onYield: async () => { + await root!.commit(async (tx) => { + delete (await tx.doc(LiveDoc, root!.id)).run!.first; + }, context); + return undefined; + }, + }), + ], + }), + ); + setup.faux.setResponses([fauxAssistantMessage("done")]); + const opened = await openChat(new MemoryStorage(), setup); + root = opened.root; + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await root.waitForIdle(context); + const answer = kindOf(await allEntries(root), "pi.assistant").at(-1)!; + expect(state.asked).toEqual([{ reason: "answer", first: answer.id, answer: answer.id, last: answer.id }]); + await opened.harness.close(context); + }); + + // #11141 + it("ends the run normally when its after-run task is cut off restartLimit times", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.settings.restartLimit = 1; + const state = probe(); + state.gate = deferred(); + install(setup, state); + setup.faux.setResponses([fauxAssistantMessage("done")]); + let opened = await openKillable(path, setup); + await (await opened.root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.runs.length === 1); + const generation = (await opened.harness.snapshot(LiveDoc, opened.root.id, context))!.run!.taskId; + // The task is cut off, and the waiting generation is not: the scheduler faults the task, and the run ends. + opened.kill(); + state.gate = undefined; + opened = await openKillable(path, setup); + await opened.root.waitForIdle(context); + expect((await opened.harness.getTask(generation, context))?.state).toMatchObject({ + status: "terminal", + outcome: { status: "completed" }, + }); + expect((await opened.harness.snapshot(LiveDoc, opened.root.id, context))?.runFailure).toBeUndefined(); + expect(state.runs).toEqual(["run 1"]); + expect(setup.infos).toContainEqual(expect.objectContaining({ kind: "task_faulted", reason: "restart_limit" })); + await opened.harness.close(context); + }); + + // #11141 + it("reports no task_interrupted for a task that ends normally after the limit was lowered", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const state = { started: 0 }; + const Worker = defineTask({ + name: "test.worker", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (_task, runtime, ctx) => { + state.started++; + if (state.started === 1) return new Promise(() => {}); // the run a crash cuts off + // The host lowers the limit while the rerun runs. + setup.settings.restartLimit = 1; + await runtime.commit(() => completed(null), ctx); + }, + }, + abort: async () => {}, + }); + setup.registry.install(defineExtension({ name: "worker", tasks: [Worker] })); + let opened = await openKillable(path, setup); + const id = await opened.root.commit( + (tx) => tx.createTask(Worker, null, { ownership: { kind: "conversation" } }), + context, + ); + opened.harness.resume(); + await eventually(() => state.started === 1); + opened.kill(); + opened = await openKillable(path, setup); + expect((await opened.harness.waitForTask(id, context)).state.outcome.status).toBe("completed"); + expect(setup.infos.filter((info) => info.kind === "task_interrupted")).toEqual([]); + setup.settings.restartLimit = undefined; + await opened.harness.close(context); + }); + + // #11141 + it("reports no task_interrupted for a generation that ends normally after the limit was lowered in finish", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const state = probe(); + state.hookGate = deferred(); + install(setup, state); + setup.faux.setResponses([fauxAssistantMessage("done")]); + let opened = await openKillable(path, setup); + await (await opened.root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.asked.length === 1); + opened.kill(); // `finish` cut off once + const rerun = deferred(); + state.hookGate = rerun; + opened = await openKillable(path, setup); + opened.harness.resume(); + await eventually(() => state.asked.length === 2); + // The host lowers the limit while the hook runs; the phase had started under the old one and ends normally. + setup.settings.restartLimit = 1; + rerun.resolve(); + await opened.root.waitForIdle(context); + expect(state.runs).toEqual(["run 2"]); + expect(setup.infos.filter((info) => info.kind === "task_interrupted")).toEqual([]); + await opened.harness.close(context); + }); + + // #11141 + it("asks after a terminate round at the turn limit: terminate wins", async () => { + const setup = chatSetup(); + const state = probe(); + install(setup, state); + setup.registry.install(tools); + setup.faux.setResponses([call("finish", "a")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await root.configure({ maxTurns: 1 }, context); + expect((await (await root.submit({ type: "input", content: "go" }, context)).wait(context)).status).toBe("done"); + await root.waitForIdle(context); + expect(state.asked).toMatchObject([{ reason: "terminate" }]); + expect(state.runs).toEqual(["run 1"]); + expect((await harness.snapshot(LiveDoc, root.id, context))?.runFailure).toBeUndefined(); + await harness.close(context); + }); + + // #11141 + it("starts a continue's history at its first tool round when nothing failed and nothing was placed", async () => { + const setup = chatSetup(); + const kinds: string[][] = []; + setup.registry.install(tools); + setup.registry.install( + defineExtension({ + name: "reader", + hooks: [ + hook(GenerationTask, { + afterRun: async (run, api, ctx) => { + const range = { minEntryId: run.first, maxEntryId: run.last, order: "ascending" } as const; + kinds.push((await api.entries(range, 100, undefined, ctx)).items.map((entry) => entry.kind)); + return undefined; + }, + }), + ], + }), + ); + setup.faux.setResponses([call("edit", "a"), fauxAssistantMessage("done")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + // A host writes the user message itself, then continues. + await root.submit( + { type: "write", entry: { kind: "app.prompt", model: [{ role: "user", content: "go", timestamp: 1 }] } }, + context, + ); + await (await root.submit({ type: "continue" }, context)).wait(context); + await root.waitForIdle(context); + expect(kinds).toEqual([["pi.assistant", "pi.tool-result", "pi.assistant"]]); + await harness.close(context); + }); + + // #11141 + it("leaves a summary queued during the tasks stale behind a terminate round's handoff", async () => { + const setup = chatSetup(); + setup.settings.compaction = { enabled: false, reserveTokens: 1000, keepRecentTokens: 1, backgroundTokens: 0 }; + const state = probe(); + install(setup, state); + const gate = deferred(); + const started = deferred(); + setup.registry.install( + defineExtension({ + name: "slow-handoff", + tools: [ + defineTool({ + name: "handoff", + description: "End the run in a new context", + parameters: Parameters, + execute: async () => { + started.resolve(); + await gate.promise; + return { + output: [{ type: "text", text: "handed off" }], + control: { terminate: true, handoff: "fresh start" }, + }; + }, + }), + ], + hooks: [hook(CompactionTask, { beforeCompact: async () => ({ summary: "earlier work" }) })], + }), + ); + setup.faux.setResponses([call("handoff", "a")]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + await started.promise; + const compaction = await root.compact(undefined, context); + await harness.waitForTask(compaction, context); + gate.resolve(); + await submission.wait(context); + await root.waitForIdle(context); + const entries = await allEntries(root); + // The handoff's reset landed with the answer; the summary, cutting before it, settled stale at the run's end. + expect(kindOf(entries, "pi.compaction")).toEqual([]); + expect(entries.map((entry) => entry.kind).slice(-2)).toEqual(["pi.reset", "test.tests"]); + expect(state.asked[0]).toMatchObject({ reason: "terminate", last: kindOf(entries, "pi.reset")[0]!.id }); + await harness.close(context); + }); + + // #11141 + it("stores the answer's held progress with commit A, and asks again after an orderly close in finish", async () => { + const path = await sqlitePath(); + const setup = chatSetup({ tokensPerSecond: 200, tokenSize: { min: 1, max: 1 } }); + setup.settings.progress = { persistWithinMs: Number.POSITIVE_INFINITY }; + const state = probe(); + const Tests = testsTask(state); + let hookGate: Deferred | undefined = deferred(); + setup.registry.install( + defineExtension({ + name: "after-run", + tasks: [Tests], + hooks: [ + hook(GenerationTask, { + afterRun: async (run, _api, ctx) => { + state.asked.push(run); + if (hookGate !== undefined) await Promise.race([hookGate.promise, aborted(ctx.abortSignal!)]); + return { tasks: [runEndTask(Tests, { label: "run" })] }; + }, + }), + ], + }), + ); + setup.faux.setResponses([fauxAssistantMessage("x".repeat(40))]); + let opened = await openKillable(path, setup); + await (await opened.root.submit({ type: "input", content: "go" }, context)).wait(context); + await eventually(() => state.asked.length === 1); + opened.kill(); + opened = await openKillable(path, setup); + const stored = (await opened.harness.snapshot(LiveDoc, opened.root.id, context))!; + expect(stored.generation).toBeUndefined(); + expect(stored.run?.afterRun).toEqual({}); + expect(textOf(kindOf(await allEntries(opened.root), "pi.assistant")[0]!.model![0])).toBe("x".repeat(40)); + opened.harness.resume(); + await eventually(() => state.asked.length === 2); + await opened.harness.close(context); + hookGate = undefined; + opened = await openKillable(path, setup); + await opened.root.waitForIdle(context); + expect(state.asked).toHaveLength(3); + expect(state.runs).toEqual(["run"]); + await opened.harness.close(context); + }); +}); + +describe("endRun", () => { + const tx = { settleSubmission: () => {} } as unknown as Tx; + + // #11141 + it("leaves a run it does not own alone, and ends an answered run without a failure", () => { + const live = { + run: { taskId: 1 as TaskId, inputs: [], turns: 0, afterRun: { tasks: [5 as TaskId] } }, + } as unknown as Draft; + endRun(tx, live, 2 as TaskId, { status: "unanswered", reason: "faulted" }); + expect(live.run?.afterRun).toEqual({ tasks: [5] }); + endRun(tx, live, 1 as TaskId, { status: "unanswered", reason: "aborted" }); + expect(live.run).toBeUndefined(); + expect(live.runFailure).toBeUndefined(); + }); + + // #11141 + it("copies a failed run's first entry into runFailure", () => { + const live = { run: { taskId: 1 as TaskId, inputs: [], turns: 0, first: 7 } } as unknown as Draft; + endRun(tx, live, 1 as TaskId, { status: "unanswered", reason: "model_error" }); + expect(live.runFailure).toMatchObject({ reason: "model_error", first: 7 }); + }); +}); diff --git a/packages/durable/test/harness-generation-recovery.test.ts b/packages/durable/test/harness-generation-recovery.test.ts index 52c0f62be..d88c34ea7 100644 --- a/packages/durable/test/harness-generation-recovery.test.ts +++ b/packages/durable/test/harness-generation-recovery.test.ts @@ -197,7 +197,7 @@ describe("generation recovery", () => { await new Promise((resolve) => setTimeout(resolve, 20)); expect(setup.faux.state.callCount).toBe(1); expect(await opened.harness.snapshot(LiveDoc, opened.root.id, context)).toEqual({ - run: { taskId, inputs: [id], turns: 0 }, + run: { taskId, inputs: [id], turns: 0, first: expect.any(Number) }, generation: { attempt: 1, retry: { at: 61_000, error: "503 Service Unavailable" } }, }); now = 61_000; diff --git a/packages/durable/test/harness-generation.test.ts b/packages/durable/test/harness-generation.test.ts index 46a3fe58f..a835aea34 100644 --- a/packages/durable/test/harness-generation.test.ts +++ b/packages/durable/test/harness-generation.test.ts @@ -156,7 +156,9 @@ describe("generation", () => { // The earlier settlement stays; the run's own settlement leaves it unchanged. expect(await submission.status(context)).toMatchObject({ status: "unanswered", reason: "withdrawn" }); // A run ends without an answer even when its inputs were settled already. - expect(await live(harness, root)).toEqual({ runFailure: { taskId, inputs: [submission.id], reason: "aborted" } }); + expect(await live(harness, root)).toEqual({ + runFailure: { taskId, inputs: [submission.id], reason: "aborted", first: expect.any(Number) }, + }); await harness.close(context); }); @@ -186,6 +188,7 @@ describe("generation", () => { inputs: [expect.any(Number)], reason: "no_model", detail: "Model faux/missing is not available", + first: expect.any(Number), }, }); expect((await live(harness, plain))?.runFailure).toMatchObject({ @@ -315,7 +318,9 @@ describe("generation", () => { const converted = entries[1]!.model![0] as AssistantMessage; expect(converted.stopReason).toBe("aborted"); expect(textOf(converted)!.startsWith(textOf(partial as Message)!)).toBe(true); - expect(await live(harness, root)).toEqual({ runFailure: { taskId, inputs: [submission.id], reason: "aborted" } }); + expect(await live(harness, root)).toEqual({ + runFailure: { taskId, inputs: [submission.id], reason: "aborted", first: expect.any(Number) }, + }); expect((await harness.waitForTask(taskId, context)).state.outcome).toEqual({ status: "aborted" }); await harness.close(context); }); @@ -332,7 +337,9 @@ describe("generation", () => { await harness.abortTask(taskId, context); expect(await submission.wait(context)).toMatchObject({ status: "unanswered", reason: "aborted" }); expect(setup.faux.state.cancelledDeferred).toHaveLength(1); - expect(await live(harness, root)).toEqual({ runFailure: { taskId, inputs: [submission.id], reason: "aborted" } }); + expect(await live(harness, root)).toEqual({ + runFailure: { taskId, inputs: [submission.id], reason: "aborted", first: expect.any(Number) }, + }); await harness.close(context); }); @@ -356,7 +363,9 @@ describe("generation", () => { await harness.abortTask(taskId, context); expect(await submission.wait(context)).toMatchObject({ status: "unanswered", reason: "aborted" }); expect(setup.reports).toContainEqual(new Error("cancel failed")); - expect(await live(harness, root)).toEqual({ runFailure: { taskId, inputs: [submission.id], reason: "aborted" } }); + expect(await live(harness, root)).toEqual({ + runFailure: { taskId, inputs: [submission.id], reason: "aborted", first: expect.any(Number) }, + }); await harness.close(context); }); @@ -691,6 +700,7 @@ describe("generation", () => { inputs: [submission.id], reason: "faulted", detail: expect.stringContaining("non-JSON function"), + first: expect.any(Number), }, }); const tasks = await harness.commit((tx) => tx.scanTasks({ conversationId: root.id }, 10), context); diff --git a/packages/durable/test/harness-input-entries.test.ts b/packages/durable/test/harness-input-entries.test.ts index 44d68a9ac..f0c44b9fb 100644 --- a/packages/durable/test/harness-input-entries.test.ts +++ b/packages/durable/test/harness-input-entries.test.ts @@ -377,6 +377,7 @@ describe("data-only writes", () => { inputs: [input.id], reason: "model_error", detail: "invalid request", + first: expect.any(Number), }, }); expect(setup.faux.state.callCount).toBe(1); diff --git a/packages/durable/test/harness-live-deltas.test.ts b/packages/durable/test/harness-live-deltas.test.ts index 696820e7c..518399392 100644 --- a/packages/durable/test/harness-live-deltas.test.ts +++ b/packages/durable/test/harness-live-deltas.test.ts @@ -131,7 +131,13 @@ describe("pi.live deltas", () => { const result = entries.items.find((entry) => entry.kind === "pi.tool-result")!.id; expect(commits).toEqual([ // submission - [["s", ["run"], { taskId: firstGeneration, inputs: [expect.any(Number)], turns: 0 }]], + [ + [ + "s", + ["run"], + { taskId: firstGeneration, inputs: [expect.any(Number)], turns: 0, first: expect.any(Number) }, + ], + ], // request [["s", ["generation"], { attempt: 1 }]], // the generation starts its tool round and keeps the run, counting the round as one turn (#11043) diff --git a/packages/durable/test/harness-restart-failure.test.ts b/packages/durable/test/harness-restart-failure.test.ts index f0552a953..356f2318e 100644 --- a/packages/durable/test/harness-restart-failure.test.ts +++ b/packages/durable/test/harness-restart-failure.test.ts @@ -220,7 +220,7 @@ describe("a generation interrupted while the model answers (#11139)", () => { detail: text, }); expect(await live(opened.harness)).toEqual({ - runFailure: { taskId, inputs: [id], reason: "interrupted", detail: text }, + runFailure: { taskId, inputs: [id], reason: "interrupted", detail: text, first: expect.any(Number) }, }); expect((await opened.harness.waitForTask(taskId, context)).state.outcome).toEqual({ status: "failed", @@ -351,7 +351,7 @@ describe("a generation interrupted while the model answers (#11139)", () => { detail: text, }); expect(await live(opened.harness)).toEqual({ - runFailure: { taskId, inputs: [id], reason: "restart_limit", detail: text }, + runFailure: { taskId, inputs: [id], reason: "restart_limit", detail: text, first: expect.any(Number) }, }); await waitFor(() => setup.infos.length > 0); expect(setup.infos).toContainEqual( @@ -563,7 +563,7 @@ describe("crash edges (#11139)", () => { detail: text, }); expect(await live(opened.harness)).toEqual({ - runFailure: { taskId, inputs: [id], reason: "interrupted", detail: text }, + runFailure: { taskId, inputs: [id], reason: "interrupted", detail: text, first: expect.any(Number) }, }); expect(setup.faux.state.cancelledDeferred).toHaveLength(1); await opened.harness.close(context); @@ -634,7 +634,7 @@ describe("crash edges (#11139)", () => { detail: text, }); expect(await live(opened.harness)).toEqual({ - runFailure: { taskId, inputs: [id], reason: "restart_limit", detail: text }, + runFailure: { taskId, inputs: [id], reason: "restart_limit", detail: text, first: expect.any(Number) }, }); const entries = await allEntries(opened.root); expect(entries.map((entry) => entry.kind)).toEqual(["pi.user", "pi.assistant"]); @@ -693,7 +693,9 @@ describe("crash edges (#11139)", () => { } const detail = `Abort handler of task ${taskId} (pi.generation) was cut off 2 times`; expect(await settled(opened.harness, id)).toMatchObject({ status: "unanswered", reason: "aborted", detail }); - expect(await live(opened.harness)).toEqual({ runFailure: { taskId, inputs: [id], reason: "aborted", detail } }); + expect(await live(opened.harness)).toEqual({ + runFailure: { taskId, inputs: [id], reason: "aborted", detail, first: expect.any(Number) }, + }); expect((await opened.harness.waitForTask(taskId, context)).state.outcome).toMatchObject({ status: "faulted", error: { detail: { reason: "restart_limit" } }, diff --git a/packages/durable/test/harness-run-failure.test.ts b/packages/durable/test/harness-run-failure.test.ts index 4c21cc970..ded8c8ebc 100644 --- a/packages/durable/test/harness-run-failure.test.ts +++ b/packages/durable/test/harness-run-failure.test.ts @@ -49,6 +49,7 @@ describe("pi.live.runFailure", () => { inputs: [submission.id], reason: "model_error", detail: "Invalid request", + first: expect.any(Number), } satisfies RunFailure); const next = await root.submit({ type: "input", content: "again" }, context); // The commit that starts the next run removes it. @@ -85,7 +86,12 @@ describe("pi.live.runFailure", () => { const taskId = (await live(harness, root)).run!.taskId; await root.abort(context); expect(await submission.wait(context)).toMatchObject({ status: "unanswered", reason: "aborted" }); - expect((await live(harness, root)).runFailure).toEqual({ taskId, inputs: [submission.id], reason: "aborted" }); + expect((await live(harness, root)).runFailure).toEqual({ + taskId, + inputs: [submission.id], + reason: "aborted", + first: expect.any(Number), + }); await harness.close(context); }); diff --git a/packages/durable/test/harness-structured.test.ts b/packages/durable/test/harness-structured.test.ts index 87fa7c72d..ac7cc9b55 100644 --- a/packages/durable/test/harness-structured.test.ts +++ b/packages/durable/test/harness-structured.test.ts @@ -1253,7 +1253,7 @@ describe("tool rounds", () => { const results = (await allEntries(root)).filter((entry) => ToolResultEntry.is(entry)); expect(results.map((entry) => (entry.model![0] as ToolResultMessage).toolCallId)).toEqual(["c1", "c2"]); expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({ - runFailure: { taskId: generation, inputs: [submission.id], reason: "aborted" }, + runFailure: { taskId: generation, inputs: [submission.id], reason: "aborted", first: expect.any(Number) }, }); expect((await harness.waitForTask(generation, context)).state.outcome.status).toBe("aborted"); await harness.close(context); @@ -1407,6 +1407,7 @@ describe("tool rounds", () => { inputs: [submission.id], reason: "faulted", detail: "Value contains a non-JSON function; expected strict JSON", + first: expect.any(Number), }, }); expect((await allEntries(reopened.root)).map((entry) => entry.kind)).toEqual(["pi.user", "pi.assistant"]); diff --git a/packages/durable/test/harness-view-run.test.ts b/packages/durable/test/harness-view-run.test.ts index a87fc009c..61431cd13 100644 --- a/packages/durable/test/harness-view-run.test.ts +++ b/packages/durable/test/harness-view-run.test.ts @@ -103,7 +103,12 @@ describe("a run as the conversation view shows it", () => { expect(partialsIn(states).map(textOf)).toEqual(partials.map(textOf)); // The run shows from the input's commit until the commit that appends the answer and clears the partial. const started = states.findIndex((state) => state.run !== undefined); - expect(states[started]!.run).toEqual({ taskId: expect.any(Number), inputs: [submission.id], turns: 0 }); + expect(states[started]!.run).toEqual({ + taskId: expect.any(Number), + inputs: [submission.id], + turns: 0, + first: expect.any(Number), + }); expect(history.views[started]!.entries.at(-1)!.kind).toBe("pi.user"); const ended = states.findIndex((state, index) => index > started && state.run === undefined); expect(states[ended]!.generation).toBeUndefined(); From 3ac64a074d6be9ead2ab4c6fc774edb6e64207d9 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 14:03:36 +0000 Subject: [PATCH 079/271] docs(durable): examples for turn limits and continue, and admission inside a commit --- packages/durable/README.md | 2 + .../examples/33-turn-limit-and-continue.ts | 73 +++++++++++++ .../test/examples/34-submit-in-commit.ts | 103 ++++++++++++++++++ 3 files changed, 178 insertions(+) create mode 100644 packages/durable/test/examples/33-turn-limit-and-continue.ts create mode 100644 packages/durable/test/examples/34-submit-in-commit.ts diff --git a/packages/durable/README.md b/packages/durable/README.md index 703731bf9..bdf59282e 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -954,6 +954,8 @@ node --conditions=source --experimental-strip-types test/examples/13-chat.ts | [30-reload-and-restart](test/examples/30-reload-and-restart.ts) | Reloading an extension mid-call, and stored choices surviving a restart | | [31-transcript](test/examples/31-transcript.ts) | A client's transcript over the windowed view: live appends, a stall with a gap, loading it and earlier pages, and resuming with `from` | | [32-tests-after-edits](test/examples/32-tests-after-edits.ts) | An `afterRun` hook that runs the tests after every run that edited files, a fix run when they fail, and input waiting for them | +| [33-turn-limit-and-continue](test/examples/33-turn-limit-and-continue.ts) | A per-conversation turn limit, `runFailure`, and continuing without a user message | +| [34-submit-in-commit](test/examples/34-submit-in-commit.ts) | Admitting a visitor's steer inside the host's own commit with `submit(tx, draft)` and `whenIdle: "reject"` | | [00](test/examples/00-conversation.ts)–[12](test/examples/12-recovery.ts) | The layers underneath: sessions, documents, forks, watches, the Harness, agent configuration, reload, extension state, tasks, recovery | Examples that call OpenAI need `OPENAI_API_KEY`; most use the faux provider otherwise. diff --git a/packages/durable/test/examples/33-turn-limit-and-continue.ts b/packages/durable/test/examples/33-turn-limit-and-continue.ts new file mode 100644 index 000000000..cac9d315c --- /dev/null +++ b/packages/durable/test/examples/33-turn-limit-and-continue.ts @@ -0,0 +1,73 @@ +// A per-conversation turn limit stops a long run after a number of tool rounds; a continue resumes it without a user +// message, with a fresh budget. A host learns that a run stopped from `pi.live.runFailure`. +// Run from packages/durable: +// node --conditions=source --experimental-strip-types test/examples/33-turn-limit-and-continue.ts +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { Type } from "@earendil-works/pi-ai"; +import { createModels } from "@earendil-works/pi-ai/models"; +import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { + ConversationBusy, + createRegistry, + defineExtension, + defineTool, + Harness, + LiveDoc, + MemoryStorage, +} from "../../src/index.ts"; + +const context = BACKGROUND_CONTEXT; + +const step = defineTool({ + name: "step", + description: "Do one step of the work", + parameters: Type.Object({ n: Type.Number() }), + execute: async (args) => ({ output: [{ type: "text", text: `step ${args.n} done` }] }), +}); +const call = (n: number) => fauxAssistantMessage(fauxToolCall("step", { n }), { stopReason: "toolUse" }); + +const faux = fauxProvider(); +const models = createModels(); +models.setProvider(faux.provider); +// Four steps, then the answer. Two tool rounds per run are allowed. +faux.setResponses([call(1), call(2), call(3), call(4), fauxAssistantMessage("All four steps are done.")]); + +const registry = createRegistry(); +registry.install(defineExtension({ name: "steps", tools: [step] })); +const harness = await Harness.open(new MemoryStorage(), { models, registry }, context); +// Stored on the conversation, like its model: children and forks inherit it. +const root = await harness.root(context, { agent: { model: { provider: "faux", modelId: "faux-1" }, maxTurns: 2 } }); + +const first = await (await root.submit({ type: "input", content: "Do the four steps." }, context)).wait(context); +const failure = (await harness.snapshot(LiveDoc, root.id, context))?.runFailure; +console.log("first run:", first.status, first.reason, `(${failure?.detail})`); + +// An idle conversation runs a continue on the context as it is, the round's results last, with a fresh budget. +const resumed = await root.submit({ type: "continue" }, context); +// It never queues: while that run works, another continue is refused. +try { + await root.submit({ type: "continue" }, context); +} catch (error) { + if (!(error instanceof ConversationBusy)) throw error; + console.log("second continue while busy: ConversationBusy"); +} +const second = await resumed.wait(context); +console.log("first continue:", second.status, second.status === "done" ? "" : second.reason); +const third = await (await root.submit({ type: "continue" }, context)).wait(context); +const live = await harness.snapshot(LiveDoc, root.id, context); +console.log("second continue:", third.status, live?.runFailure === undefined ? "(runFailure cleared)" : ""); + +const page = await root.entries({ order: "ascending" }, 100, undefined, context); +for (const entry of page.items) { + const message = entry.model?.[0]; + if (message?.role === "user") console.log(">", message.content); + if (message?.role === "toolResult") + console.log(" tool:", message.content[0]?.type === "text" ? message.content[0].text : ""); + if (message?.role === "assistant" && message.stopReason === "stop") { + console.log("agent:", message.content.flatMap((part) => (part.type === "text" ? [part.text] : [])).join("")); + } +} +// A continue after an answer has nothing to answer: it ends without a model request. +const after = await (await root.submit({ type: "continue" }, context)).wait(context); +console.log("continue after the answer:", after.status, after.reason); +await harness.close(context); diff --git a/packages/durable/test/examples/34-submit-in-commit.ts b/packages/durable/test/examples/34-submit-in-commit.ts new file mode 100644 index 000000000..291305cc6 --- /dev/null +++ b/packages/durable/test/examples/34-submit-in-commit.ts @@ -0,0 +1,103 @@ +// A host lets people message an agent while it works: a message must join the current run as a steer and never start a +// run of its own. `submit(tx, draft)` admits it inside the host's own commit, together with the host's check and its own +// state; `whenIdle: "reject"` refuses it when no run is working. +// Run from packages/durable: +// node --conditions=source --experimental-strip-types test/examples/34-submit-in-commit.ts +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { Type } from "@earendil-works/pi-ai"; +import { createModels } from "@earendil-works/pi-ai/models"; +import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { + ConversationIdle, + createRegistry, + defineDoc, + defineExtension, + defineTool, + Harness, + MemoryStorage, +} from "../../src/index.ts"; + +const context = BACKGROUND_CONTEXT; + +// The host's own state: who may message this conversation, and what they sent. +const Visitors = defineDoc<{ muted: string[]; sent: { from: string; text: string }[] }>({ + kind: "app.visitors", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ muted: [], sent: [] }), + checkpointWhen: () => false, +}); + +// A tool that waits until released, so messages arrive while the run works. +let release!: () => void; +const working = new Promise((resolve) => { + release = resolve; +}); +const build = defineTool({ + name: "build", + description: "Build the project", + parameters: Type.Object({}), + execute: async () => { + await working; + return { output: [{ type: "text", text: "built" }] }; + }, +}); + +const faux = fauxProvider(); +const models = createModels(); +models.setProvider(faux.provider); +faux.setResponses([ + fauxAssistantMessage(fauxToolCall("build", {}), { stopReason: "toolUse" }), + fauxAssistantMessage("Built, and I added the test Ana asked for."), +]); +const registry = createRegistry(); +registry.install(defineExtension({ name: "build", tools: [build] })); +const harness = await Harness.open(new MemoryStorage(), { models, registry }, context); +const root = await harness.root(context, { agent: { model: { provider: "faux", modelId: "faux-1" } } }); +await root.commit(async (tx) => { + (await tx.doc(Visitors, root.id)).muted.push("bob"); +}, context); + +/** A visitor's message: checked, admitted as a steer of the current run, and recorded, in one commit or not at all. */ +async function message(from: string, text: string, requestId: string): Promise { + try { + return await harness.commit(async (tx) => { + const visitors = await tx.doc(Visitors, root.id); + if (visitors.muted.includes(from)) return `${from} is muted`; + // Before the commit's own table writes; document writes may come before or after. + const id = await root.submit(tx, { + type: "input", + content: `${from}: ${text}`, + whenBusy: "steer", + whenIdle: "reject", + requestId, + }); + visitors.sent.push({ from, text }); + return `${from}'s message is submission ${id}`; + }, context); + } catch (error) { + if (error instanceof ConversationIdle) return `${from}: the agent isn't working now`; + throw error; + } +} + +console.log(await message("ana", "Add a test for the build.", "ana-1")); // idle: refused, nothing written +const run = await root.submit({ type: "input", content: "Build the project." }, context); +console.log(await message("ana", "Add a test for the build.", "ana-2")); // busy: joins the run as a steer +console.log(await message("bob", "Delete everything.", "bob-1")); // the host's own check refuses it +release(); +await run.wait(context); +await root.waitForIdle(context); + +const page = await root.entries({ order: "ascending" }, 100, undefined, context); +for (const entry of page.items) { + const message = entry.model?.[0]; + if (message?.role === "user") console.log(">", message.content); + if (message?.role === "assistant" && message.stopReason === "stop") { + console.log("agent:", message.content.flatMap((part) => (part.type === "text" ? [part.text] : [])).join("")); + } +} +console.log("recorded:", (await harness.snapshot(Visitors, root.id, context))?.sent); +await harness.close(context); From 2eb6a8d15ff038d0ac9565f760720e5bec9b4a3e Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 14:18:12 +0000 Subject: [PATCH 080/271] docs(chord): slice 6b R1, retirement per record by history --- packages/chord/docs/next/decisions.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 7dbb94b49..3969a2f5e 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1697,6 +1697,14 @@ validation and copies the v2 kernel added around Delta. context)`); the harness object (`Harness.open` over `StoreStorage`, `resume()` after the start barrier, the models, environments and extensions registries) is p3's base code, shown in Chord only as an example or test. Plan: Chord v2 (6b, 7, 8, 9) this week, p3's design next week. +- **Slice 6b R1: retirement per record by `history` (owner, 2026-10-27: "sgtm"; spec review, all four reviewers + found that Durable keeps retired records):** the engine's `history` gets three values, set by the adapter at + creation and applied at retirement: `none` (Chord's documents and job records: older revisions dropped at a new base, + row and revisions deleted at retirement, 6a's behaviour, until now called `latest`), `latest` (Durable's session, task + and latest conversation documents: older revisions dropped, the row kept with `retired_at` at retirement, so reads at + a point find it), `rewindable` (everything kept). `retireDocument` reads the record's `history`; no flag, archives + and replicas carry it. Rejected: keeping every retired record (one rule, but a dead row per finished job run, two + with its state, and per retired keyed document, that Chord never reads). - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the From 018fdcf43e73bb63b1971c3cd21241285a82036e Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 14:27:53 +0000 Subject: [PATCH 081/271] fix(durable): pi.live version 3 gives a stored run without turns a count of 0 A database an earlier unreleased build wrote at pi.live version 2 has a run without run.turns; a run in flight across the upgrade then counted NaN turns and failed its next tool round with "Value contains a non-finite number and is not strict JSON". Version 3's migrate adds turns: 0 wherever it is missing. --- packages/durable/CHANGELOG.md | 2 +- packages/durable/docs/spec.md | 4 +- packages/durable/src/harness/live.ts | 9 +++-- .../durable/test/harness-compaction.test.ts | 38 ++++++++++++++++++- 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 98f05d090..46658afba 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -96,7 +96,7 @@ - A call to a tool its request did not offer gets a `pi.tool` task like any call, flagged `unoffered: true` in its input and its `pi.live.tools` slot, instead of a result the generation wrote without one: the task resolves nothing (a deferred tool the agent has still never runs), settles `tool_unavailable`, and runs `afterTool`, whose `control` then counts toward the round (one such call no longer prevents `terminate`). Its slot starts `pending` with a `taskId` instead of `done`, and its result entry lands in a later commit than the tool-calling answer. Controls are applied in call order. `pi.tool` is version 3 and `pi.generation` version 2 (a sequential round keeps such calls pending until their turn): an older Harness leaves both blocked (`task_too_old`) instead of resolving the flagged name ([#11043](https://github.com/earendil-works/pi/issues/11043)) - `SubmissionRecord` gains a `type: "continue"` variant (`placed` without `entry`, `done` with `answer`, `unanswered`), so narrowing a record by `status` alone no longer guarantees an `entry`: check `type === "input"` too. A generation whose context does not end in a user message or tool results, system messages aside, now ends its run `unanswered` with the new `RunFailureReason` `nothing_to_answer` without a request; an empty conversation no longer faults its generation. `ConversationHandle.submit()` takes any `SubmissionDraft` (was `InputSubmissionDraft`), as `Conversation.submit()` does ([#11043](https://github.com/earendil-works/pi/issues/11043)) - `RunFailureReason` (`pi.live.runFailure.reason`) gains `nothing_to_answer` and `turn_limit`, also an `unanswered` submission's `reason`: exhaustive switches over it need cases ([#11043](https://github.com/earendil-works/pi/issues/11043)) -- `pi.live.run` has a required `turns` field, the run's tool rounds so far; code that writes `run` itself sets it (`turns: 0`). The unreleased `pi.live` version 2 `migrate` adds `turns: 0` to a stored version 1 run; a database written by an unreleased build at version 2 without `run.turns` is not supported (its run counts `NaN` turns) ([#11043](https://github.com/earendil-works/pi/issues/11043)) +- `pi.live.run` has a required `turns` field, the run's tool rounds so far; code that writes `run` itself sets it (`turns: 0`). `pi.live` is version 3: its `migrate` adds `turns: 0` to a stored run without it, version 1 or a version 2 an earlier unreleased build wrote, whose run in flight otherwise failed its next tool round with "Value contains a non-finite number" ([#11043](https://github.com/earendil-works/pi/issues/11043)) ### Added diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index 72d630c1b..4f3795e33 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -4617,10 +4617,10 @@ type CompactionStatus = { | field | value | |---|---| | kind | `pi.live` | -| version | `2` | +| version | `3` | | scope/history/fork | conversation, `latest`, `initial` | | `initial()` | `{}` | -| `migrate()` | from version 1: `reason` of each `compactions` status and of `compactionFailure` renamed `trigger`, and `run.turns` set to `0` (section 3.6); `run.first`, `run.afterRun`, and `runFailure.first` are optional and need no step | +| `migrate()` | from version 1: `reason` of each `compactions` status and of `compactionFailure` renamed `trigger`; from versions 1 and 2: `run.turns` set to `0` where it is missing (section 3.6); `run.first`, `run.afterRun`, and `runFailure.first` are optional and need no step | | checkpoint | complete base whenever nothing runs: `generation` absent and no `running` tool or nested slot | | view mount | `docs["pi.live"]` | | created | with every Harness conversation (section 2.2) | diff --git a/packages/durable/src/harness/live.ts b/packages/durable/src/harness/live.ts index d9696f46d..9888a402e 100644 --- a/packages/durable/src/harness/live.ts +++ b/packages/durable/src/harness/live.ts @@ -193,19 +193,20 @@ export type LiveState = { export const LiveDoc = defineDoc({ kind: "pi.live", - version: 2, + version: 3, scope: "conversation", history: "latest", fork: "initial", initial: () => ({}), - // Version 1 named a compaction's trigger `reason`, in `compactions` and `compactionFailure`, and had no `run.turns`: - // a run in flight across the upgrade counts its turns from 0. + // Version 1 named a compaction's trigger `reason`, in `compactions` and `compactionFailure`. Versions 1 and 2 (as + // builds before `run.turns` stored it) may lack `run.turns`: a run in flight across the upgrade counts its turns + // from 0. Version 2 already names the trigger `trigger`, so renaming leaves it alone. migrate: (value) => { const live = value as { run?: JsonObject; compactions?: JsonObject[]; compactionFailure?: JsonObject }; const { run, compactions, compactionFailure } = live; return { ...value, - ...(run === undefined ? {} : { run: { ...run, turns: 0 } }), + ...(run === undefined ? {} : { run: { ...run, turns: typeof run.turns === "number" ? run.turns : 0 } }), ...(compactions === undefined ? {} : { compactions: compactions.map(renameReason) }), ...(compactionFailure === undefined ? {} : { compactionFailure: renameReason(compactionFailure) }), } as LiveState; diff --git a/packages/durable/test/harness-compaction.test.ts b/packages/durable/test/harness-compaction.test.ts index a473b825b..b60eb2dae 100644 --- a/packages/durable/test/harness-compaction.test.ts +++ b/packages/durable/test/harness-compaction.test.ts @@ -2454,7 +2454,7 @@ describe("per-model compaction overrides", () => { }); }); -describe("pi.live version 2", () => { +describe("pi.live migrations", () => { it("reads a version 1 value with each compaction's `reason` as `trigger`, and stores it migrated on its next write", async () => { const storage = new MemoryStorage(); const session = createSession(storage); @@ -2505,9 +2505,43 @@ describe("pi.live version 2", () => { (await tx.doc(LiveDoc, conversationId)).compactions![0]!.attempt = 3; }, context); expect(await stored()).toMatchObject({ - version: 2, + version: 3, value: { compactions: [{ trigger: "threshold", attempt: 3 }], compactionFailure: { trigger: "manual" } }, }); await session.close(context); }); + + // #11043: an unreleased build stored version 2 without `run.turns`; a run in flight then counted NaN turns. + it("gives a version 2 run without turns a count of 0", async () => { + const storage = new MemoryStorage(); + const session = createSession(storage); + const LiveV2 = defineDoc({ + kind: "pi.live", + version: 2, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({}), + checkpointWhen: () => false, + }); + const conversationId = await session.commit( + async (tx) => (await tx.createConversation({ ownership: { kind: "ownerless" } })).id, + context, + ); + await session.commit(async (tx) => { + const live = await tx.doc(LiveV2, conversationId); + live.run = { taskId: 9, inputs: [4] }; + live.compactions = [{ taskId: 7, trigger: "threshold", blocking: true, attempt: 2 }]; + }, context); + expect(await session.snapshot(LiveDoc, conversationId, context)).toEqual({ + run: { taskId: 9, inputs: [4], turns: 0 }, + compactions: [{ taskId: 7, trigger: "threshold", blocking: true, attempt: 2 }], + }); + // A version 2 run that has a count keeps it. + await session.commit(async (tx) => { + (await tx.doc(LiveV2, conversationId)).run = { taskId: 9, inputs: [4], turns: 3 }; + }, context); + expect((await session.snapshot(LiveDoc, conversationId, context))?.run?.turns).toBe(3); + await session.close(context); + }); }); From 84576c70c89de5872cf09c443633bee0165d4df6 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 14:33:53 +0000 Subject: [PATCH 082/271] docs(chord): slice 6b R2, no host kind in store names --- packages/chord/docs/next/decisions.md | 6 ++++++ packages/chord/docs/next/design.md | 10 +++++----- packages/chord/docs/next/slice-6b.md | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 3969a2f5e..07ea60b21 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1705,6 +1705,12 @@ validation and copies the v2 kernel added around Delta. a point find it), `rewindable` (everything kept). `retireDocument` reads the record's `history`; no flag, archives and replicas carry it. Rejected: keeping every retired record (one rule, but a dead row per finished job run, two with its state, and per retired keyed document, that Chord never reads). +- **Slice 6b R2: no host kind in store names (owner, 2026-10-27: "what use is host in table names anyways? seems + stupid. no host"; supersedes the host-kind part of W7, "Restricted names" and "Slice 6a's three code questions" (3)):** + tables are `":
"` (`"notes.store:edits"`, `"pi.durable:entries"`), document owners + `""`; a store belongs to one host (rule 7), so the host kind told nothing, and `StoreStorage` and stores + used before or without a host get the same names. W7's rule stays: no host instance in a name. Changed in 6a's code + in 6b; the applier checks two parts. - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index 2468b9ec3..7ea0ced16 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -945,8 +945,8 @@ host.serve(DraftStore, (_, self) => { // local, empty params: may reload) and survive reloads and native restarts of a live object; long-lived objects creating many keyed in-memory documents retire them themselves. Persisted documents stay until retired. - **Persisted data only in `local` objects with empty params (P3, owner, 2026-10-21):** persisted documents and tables - live in `local` objects with empty params (one per host and type); their address is (host kind, object type, name, - key), never a host instance such as the session id (W7), + live in `local` objects with empty params (one per host and type); their address is (object type, name, key), never a + host instance such as the session id (W7; no host kind either since slice 6b R2: a store belongs to one host), so a params change never orphans data. Per-entity data is keyed documents or table rows there, with an app-chosen key; objects with params keep only in-memory state and read persisted data by key. Renaming a persisted store's type or definition, or changing how keys are formed, is an explicit one-off migration in the store. @@ -1000,7 +1000,7 @@ host.serve(DraftStore, (_, self) => { // local, empty params: may access, `checkpointWhen` (required for persisted documents, R14), retire, and the documents engine's invariants (create before change, no change after retire, one live incarnation per address, `deltasSinceBase`). - **Where persisted documents live:** rows of the store's documents engine (S13) with address `(namespace: "chord", - owner: ":", kind: , key)` (key `""` for a singleton), never a host instance (W7); ids from the store's + owner: "", kind: , key)` (key `""` for a singleton), never a host instance (W7); ids from the store's counter (S10). Chord ships no storage of its own: no `DocumentStorage` interface, no Chord SQLite or memory document storage and no Chord storage conformance suite (the store's conformance suite covers the engine); the frozen `next/` storages are not ported. The store is passed to the session's host by base code (owner, 2026-10-26): @@ -1069,8 +1069,8 @@ host.serve(DraftStore, (_, self) => { // local, empty params: may materialized and fingerprinted at declaration, as contracts' `evolve`; the store replays the recorded version's successors in order (DDL and `migrating`, the data step, `ready`); key columns are immutable in `set`. - **Eager migrations and addresses (owner, 2026-10-26):** the activation attempt's `readyTimeout` deadline does not - run while an eager migration step runs (getters still time out by their own); a host without an address uses `client` - as its host kind in store addresses. + run while an eager migration step runs (getters still time out by their own). Store names carry no host kind (slice + 6b R2, owner, 2026-10-27): tables are `":
"`, document owners `""`. - **Eager `rows` migrations** commit in chunks of 100 rows (owner, 2026-10-26; a fixed constant: about 1 ms of commit step per chunk, the total time is the same for larger chunks, and every other owner's commit waits for the step). Between chunks the migration yields one macrotask without `setTimeout`'s 1 ms clamp: `setImmediate` where the diff --git a/packages/chord/docs/next/slice-6b.md b/packages/chord/docs/next/slice-6b.md index c3423c403..c52ddfd46 100644 --- a/packages/chord/docs/next/slice-6b.md +++ b/packages/chord/docs/next/slice-6b.md @@ -58,7 +58,7 @@ Durable's four record tables (conversations, entries, tasks, submissions) and it **Layout (D2, owner, 2026-10-27: "the whole purpose of this slice"):** declared tables of the store, written as ordinary `put` writes: `defineTable`s that `StoreStorage` declares for itself under the type name `pi.durable` (SQL names -`":pi.durable:
"`, e.g. `"session:pi.durable:entries"`; type names are unique per app, so no plugin +`"pi.durable:
"`, e.g. `"pi.durable:entries"` (R2: no host kind in store names); type names are unique per app, so no plugin table can meet them), each row the record's indexed columns (`id`, `conversation_id`, `kind`, `status`, …) plus the whole record in a `json(...)` column. The outbox, export, import and the applier handle them as every table; Durable's layout changes later through `evolveTable`. The partial head index becomes an index on `(conversation_id, From 74bbffadf364b305e48630de66c2f92c8ffca06f Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 14:50:33 +0000 Subject: [PATCH 083/271] docs(chord): slice 6b owner names and claims, no pi.durable --- packages/chord/docs/next/decisions.md | 12 ++++++++++++ packages/chord/docs/next/slice-6b.md | 8 ++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 07ea60b21..30f8cef0a 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1711,6 +1711,18 @@ validation and copies the v2 kernel added around Delta. `""`; a store belongs to one host (rule 7), so the host kind told nothing, and `StoreStorage` and stores used before or without a host get the same names. W7's rule stays: no host instance in a name. Changed in 6a's code in 6b; the applier checks two parts. +- **Slice 6b: owner names and claims, no `pi.durable` (owner, 2026-10-27: "seems reasonable"; supersedes D2's and D5's + `pi.durable` type name and D5's "one `StoreStorage` per store" rule):** every table and document in the store belongs + to an owner name, claimed once per store. A `local` object with empty params (P3) claims its type when it first + declares persisted data (`self.table`, persisted `self.state`, persisted `self.job`); `StoreStorage.open(store, owner, + ctx)` claims `owner`; a second claim of a name is refused (`invalid/args`): a second harness, or a plugin type that + meets the harness's name. `StoreStorage` declares its own tables (`entries`, `tasks`, …) through the store, the path + `self.table` takes, named `":
"`, and writes its documents under `owner` in the engine's namespace + `durable`. In p3 the harness object passes its own type (`StoreStorage.open(store, Harness.name, ctx)`; base code gives + the store to the core facet); Durable's conformance suite passes a test name. The owner name plays the role of + `SqliteStorage`'s `prefix`, checked by the store. Objects never see the store: their API writes to the host's store + under their type; a host without a store fails an object declaring persisted data (`broken`, 6a). Rejected: an owner + handle (`self.storage`), an indirection that only added protection against trusted code. - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the diff --git a/packages/chord/docs/next/slice-6b.md b/packages/chord/docs/next/slice-6b.md index c52ddfd46..02f685ac0 100644 --- a/packages/chord/docs/next/slice-6b.md +++ b/packages/chord/docs/next/slice-6b.md @@ -26,7 +26,7 @@ const worker = createHost

({ address: Session({ id }), principal, store, onErr // the core facet: the harness object opens Durable on the same store host.serve(Harness, (_, self) => { const harness = self.open(async (context) => - Harness.open(new StoreStorage(store), { models, registry, … }, context), // one StoreStorage per store + Harness.open(await StoreStorage.open(store, Harness.name, context), { models, registry, … }, context), (h, _reason, context) => h.close(context)); const view = self.follow((observer, ctx) => harness.value.observeConversation(id, observer, ctx)); // section 2 return { … }; @@ -57,9 +57,9 @@ Chord imports nothing of Durable (S13); Durable's conformance suite runs in Dura Durable's four record tables (conversations, entries, tasks, submissions) and its ID namespace. **Layout (D2, owner, 2026-10-27: "the whole purpose of this slice"):** declared tables of the store, written as -ordinary `put` writes: `defineTable`s that `StoreStorage` declares for itself under the type name `pi.durable` (SQL names -`"pi.durable:

"`, e.g. `"pi.durable:entries"` (R2: no host kind in store names); type names are unique per app, so no plugin -table can meet them), each row the record's indexed columns (`id`, `conversation_id`, `kind`, `status`, …) plus +ordinary `put` writes: `defineTable`s that `StoreStorage` declares for itself under its owner name (`StoreStorage.open(store, owner, ctx)`, +the name claimed once per store; SQL names `":
"`, e.g. `"p3.harness:entries"`; decision "owner names +and claims"), each row the record's indexed columns (`id`, `conversation_id`, `kind`, `status`, …) plus the whole record in a `json(...)` column. The outbox, export, import and the applier handle them as every table; Durable's layout changes later through `evolveTable`. The partial head index becomes an index on `(conversation_id, has_head, id)` (index columns are non-`opt`); indexed strings that must stay lossless (`kind`, `requestId`, document From 8c38be894da19f41ed1c76b27cd5737fea0eba80 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 14:51:05 +0000 Subject: [PATCH 084/271] docs(chord): slice 6b R3, a seal is a clean stop for persisted runs --- packages/chord/docs/next/decisions.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 30f8cef0a..16ec213fc 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1723,6 +1723,10 @@ validation and copies the v2 kernel added around Delta. `SqliteStorage`'s `prefix`, checked by the store. Objects never see the store: their API writes to the host's store under their type; a host without a store fails an object declaring persisted data (`broken`, 6a). Rejected: an owner handle (`self.storage`), an indirection that only added protection against trusted code. +- **Slice 6b R3: a seal is a clean stop for persisted runs (owner, 2026-10-27: "a"):** a persisted run cut off by + its attempt's seal while the object continues (a live failure, then a retry) is written back stopped, and the next + live attempt resumes it per its `resume` (rerun or abandon) without counting a restart, as after a clean close (D6). + In-memory runs keep 6a's rule (`failed { aborted, reason: "restart" }`, finished on the next live attempt). - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the From a2c5657c86d38e5b330e72f27b2f0134c1e0b322 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 14:54:03 +0000 Subject: [PATCH 085/271] docs(chord): slice 6b R4, a follow source's end --- packages/chord/docs/next/decisions.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 16ec213fc..f8b3782ca 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1727,6 +1727,10 @@ validation and copies the v2 kernel added around Delta. its attempt's seal while the object continues (a live failure, then a retry) is written back stopped, and the next live attempt resumes it per its `resume` (rerun or abandon) without counting a restart, as after a clean close (D6). In-memory runs keep 6a's rule (`failed { aborted, reason: "restart" }`, finished on the next live attempt). +- **Slice 6b R4: a follow source's end (owner, 2026-10-27: "yes"):** Durable ends an observer `failed` (its Session + failed: the object that opened Durable fails `transient`, its retry reopens its natives, Durable included, and attaches + again, as Durable's own recovery is to reopen from Storage) or `ended` (the observed owner ended normally, a task + document's task: final, the member keeps its last value, its watches end `ended`). - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the From 7f3cfb060d3fcbf0ec72874d813dce89ddb5d468 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 14:55:45 +0000 Subject: [PATCH 086/271] docs(chord): slice 6b spec revised after the first spec review --- packages/chord/docs/next/slice-6b.md | 443 ++++++++++++++++----------- 1 file changed, 257 insertions(+), 186 deletions(-) diff --git a/packages/chord/docs/next/slice-6b.md b/packages/chord/docs/next/slice-6b.md index 02f685ac0..52d9cd21e 100644 --- a/packages/chord/docs/next/slice-6b.md +++ b/packages/chord/docs/next/slice-6b.md @@ -1,245 +1,316 @@ # Slice 6b: Durable and jobs on the store (spec) -Status: **decided, ready for the spec reviews** (2026-10-27; D1-D9 decided by the owner), written by the redesign session after reading -Durable's `Storage` interface (`packages/durable/src/types.ts`), its SQLite storage, schema and migrations, the storage -conformance suite and benchmark, `session/session.ts`, `session/transaction.ts`, `session/forks.ts`, and Chord v2's -store engine, applier, schema, commit step and jobs, in full. Durable's memory, JSONL and IndexedDB storages were not -read (they stay as they are, S15). +Status: **revised after the first spec review** (2026-10-27): D1-D9 and R1-R4 decided by the owner (decision log, +entries "Slice 6b …" and "owner names and claims"); the four reviews (`/tmp/s6b-review-{astra,fable,opus,sol}.md`) +kept every decision and found the gaps this revision closes. Written by the redesign session after reading, in full, +Durable's `types.ts`, its SQLite storage, schema and migrations, the storage conformance suite and benchmark, +`session/session.ts`, `session/transaction.ts`, `session/forks.ts`, the scheduler's restarts, `writeAtClose` and `abort`, +and Chord v2's store, engine, applier, commit step and jobs. Durable's memory, JSONL and IndexedDB storages stay as they +are (S15). -The single source for building 6b, together with: `design.md` read in full; the decision log -(`packages/chord/docs/next/decisions.md`, later entries win), in particular "The store", S1, S2, S4, S10, S11, S13, S15, -W1, W12, W13, "Slice 6 first decisions", "Durable observers", "Durable's synchronous observers", "Text is plain UTF-8", -"Failures are watch calls", the "One vocabulary" entries, "The performance and memory bar"; `slice-6a.md` (what 6a -built); the feature map and the carry-over of `slices.md` (slice 6b); the root `AGENTS.md` "Chord v2 Code Style". +The single source for building 6b, together with `design.md` read in full; the decision log `decisions.md` (later +entries win), in particular "The store", S1, S2, S4, S10, S11, S13, S15, W1, W12, "Slice 6 first decisions", "Durable's +synchronous observers", "Text is plain UTF-8", "Failures are watch calls", the "One vocabulary" entries, "The performance +and memory bar", and every "Slice 6b" entry; `slice-6a.md` (what 6a built); the feature map and the carry-over of +`slices.md` (6b); the root `AGENTS.md` "Chord v2 Code Style". 6b puts Durable on the store (`StoreStorage`), follows Durable's views as Chord sources (`self.follow`), persists jobs, -and extends the applier to every write kind. After 6b a worker has one store: Chord's objects and Durable's Session -commit through one queue, and a commit of either is one seq in one outbox. +and extends the applier to every write kind. A worker then has one store: Chord's objects and Durable's Session commit +through one queue, and a commit of either is one seq in one outbox. ## What an app can do after this slice ```ts -// the start code of a session's worker (p3 base code) +// p3 worker base code const store = await openStore(sqliteBackend(db), ctx); -const worker = createHost

({ address: Session({ id }), principal, store, onError }); +const host = createHost

({ address: Session({ id }), principal, store, onError }); +await host.load([{ name: "@p3/core", facet: coreWorkerFacet({ init, store }) }, ...plugins], ctx); -// the core facet: the harness object opens Durable on the same store -host.serve(Harness, (_, self) => { +// the core facet: the harness object opens Durable on the same store, its data under the harness's own type +host.serve(Harness, (_, self) => { // local, empty params (P3) const harness = self.open(async (context) => Harness.open(await StoreStorage.open(store, Harness.name, context), { models, registry, … }, context), (h, _reason, context) => h.close(context)); - const view = self.follow((observer, ctx) => harness.value.observeConversation(id, observer, ctx)); // section 2 - return { … }; + const view = self.follow((observer, ctx) => harness.value.observeConversation(rootId, observer, ctx, { min: 50 })); + return { view }; }); -// a persisted job: survives a crash +// a persisted job: survives a crash, a clean close and its object's retry const ImportJob = defineJob("notes.import", { version: 1, args: obj({ path: str() }), result: obj({ count: int() }), - state: ImportState, persist: { resume: "rerun" }, + state: ImportState /* persist: true, checkpointWhen */, persist: { resume: "rerun" }, }); const imports = self.job(ImportJob, { run, finish }); -await imports.abort(id, ctx); // stored; Decision needed (D7): the result +await imports.abort(id, ctx); // "marked" | "terminal", stored +await ctx.state.change((d) => { d.output = text; }, ctx, { persistence: { kind: "deferred" } }); ``` -## 1. `StoreStorage`: Durable's `Storage` on the store +## 1. Changes to 6a's store -Durable's `Storage` (types.ts, "Atomic persistence boundary for Session records") implemented over a `ChordStore`, so -Durable's Session runs unchanged on it and passes Durable's storage conformance suite -(`createStorageConformance`), across reopen too. +Every item below changes 6a's code; nothing is deployed, so the schema and the archive format change in place. -**Where it lives (D1, owner, 2026-10-27: "they are only used if durable is used"):** `StoreStorage` belongs to -Durable (D9 makes `durableView` a generic Chord primitive, `self.follow`), in its own `src` (Chord v2 is in pi upstream since `93029a0f1`), importing -`@earendil-works/chord/next` (Chord's `./next` package export comes with them). Durable already depends on Chord and -Chord imports nothing of Durable (S13); Durable's conformance suite runs in Durable's tests and CI. +### 1.1 Names and claims (decisions "R2", "owner names and claims") -### 1.1 Records +- Tables are `":

"` (`"notes.store:edits"`), document owners `""`; no host kind. The applier and + import check two name parts. +- Every owner name is claimed once per store (in memory, for the store's lifetime): an object claims its type when it + first declares persisted data; `StoreStorage.open` claims its name; a second claim is refused `invalid/args`; a + claim is released by its owner's end (`StoreStorage.close`, the object's end), so a new Session can reopen over the + same store. -Durable's four record tables (conversations, entries, tasks, submissions) and its ID namespace. +### 1.2 One id allocator (D3) -**Layout (D2, owner, 2026-10-27: "the whole purpose of this slice"):** declared tables of the store, written as -ordinary `put` writes: `defineTable`s that `StoreStorage` declares for itself under its owner name (`StoreStorage.open(store, owner, ctx)`, -the name claimed once per store; SQL names `":
"`, e.g. `"p3.harness:entries"`; decision "owner names -and claims"), each row the record's indexed columns (`id`, `conversation_id`, `kind`, `status`, …) plus -the whole record in a `json(...)` column. The outbox, export, import and the applier handle them as every table; -Durable's layout changes later through `evolveTable`. The partial head index becomes an index on `(conversation_id, -has_head, id)` (index columns are non-`opt`); indexed strings that must stay lossless (`kind`, `requestId`, document -kinds and keys) keep Durable's JSON encoding in their columns. +- The store owns one synchronous allocator: `mintId()` takes the next id; the commit step takes its documents' ids from + the same allocator when a creation applies (no copy in the `Step`, no overwrite after `COMMIT`); explicit ids in + staged writes raise it at staging (`max`); a refused step leaves its ids unused (gaps, S10). Each commit persists + the allocator's current value. +- It starts at 2 (id 1 is Durable's root conversation). It saturates at `Number.MAX_SAFE_INTEGER`: an explicit write of + that id is accepted, the persisted value stays `MAX_SAFE_INTEGER`, every later `mintId` rejects "ID space is + exhausted", and a Chord creation at the saturated allocator refuses its own commit (1.4). +- Outbox commits carry the allocator's value after them (`{ seq, nextId, writes }`); the archive manifest's + `nextDocumentId` is `nextId`; import and batch apply restore it, so a replica mints above every id it holds. -- **Writes:** every `StorageWrite` of a table record becomes a `put` of its row (Durable replaces complete records; - conversations and entries are immutable and written once). `record_ids` is a declared table too (`id`, - `record_type`), one `put` per new id (decision "Slice 6 first decisions" (3)). -- **Reads:** `conversation`, `entry` (with its `commitSeq`: a column), `findLatestHeadMarker`, `scanEntries` (both - orders, the fork chain walked as `SqliteStorage` does), `task`, `scanTasks`, `submission`, `scanSubmissions`, - `submissionByRequest`, `scanConversations`: queries over the declared tables, Durable's cursors (`{ after, order }`, - `scan.ts`) kept as they are (built from the last id, not from Chord's table cursors). -- **Validation in the step** (D4): what `SqliteStorage` checks inside its transaction (`checkGlobalIds`, - `prepareDocumentActions`, `checkDocumentActions`) runs inside the store's step, which rolls back a refused commit. +### 1.3 The commit step -### 1.2 IDs +- **A staged write returns the writes it recorded:** `StagedWrite.apply(step): Promise`. + `StoreStorage` stages one composite write per Durable commit (1.3 of section 2). +- **Order:** within the step, writes apply in the order staged, except that a staged write that frees a live address + (a retirement of another incarnation) is applied before the creation at that address (rule 9's reason, the partial + unique index); Chord's own staging keeps 6a's order. The outbox records the writes in applied order; the applier + replays them in that order (no re-sort). +- **Stamps:** a staged write can take values only the step knows (`step.seq`): Durable's `entries.commit_seq`. The + outbox records the resolved values. +- **Adopt in the owner's batch:** the host gives the store its batch when it attaches (`attach({ failed, batch })`); + an adapter's fence runs its `adopt` in `batch.within(…)` (joins a running batch, opens one otherwise); Chord's own + transactions keep their `batch.commit`. Without a host, `adopt` runs plain. -**One counter per store (D3, owner, 2026-10-27):** the store's counter (`next_document_id` renamed `next_id`, the -manifest's `nextDocumentId` renamed `nextId`) numbers every id: `StoreStorage.mintId()` takes from it in memory, -explicit ids in Durable's writes raise it, Chord's commit step takes its documents' ids from it, and every commit -persists it as a high-water mark. `record_ids` stays for Durable's "ID n already belongs to entry" check. Durable's ids -rise in its creation order (its scans and windows rely on it); Chord's ids are gaps between them. Spec reviewers check -that nothing relies on Chord's document ids rising with seq. +### 1.4 Errors: Durable's Storage semantics (D4) -### 1.3 Documents +- The step records the error its own code threw inside the transaction callback. A rejection of the backend + transaction that is that very error is a **refusal**: the facade rolled back (a failed `ROLLBACK` reaches us as + another error, an `AggregateError` on Node), the commit rejects, nothing is adopted, the allocator keeps its raised + value, the store goes on. Any other rejection (`BEGIN`, `COMMIT`, `ROLLBACK`, a throwing `adopt`) is an unknown + outcome: the store stops (`storage_failed`). The Cloudflare facade is checked to keep the same identity; a test + injects failures at `BEGIN`, in the body, at `COMMIT` and at `ROLLBACK` on both drivers' facades. +- **The refusal's error reaches the owner unchanged** (Durable's suite matches its messages); the store classifies it + for Chord's owners: an `int` `CHECK` failure (SQLite's extended code `SQLITE_CONSTRAINT_CHECK`) is `invalid/args` and + fails only that transaction; every other refusal of a Chord commit (the engine's counted `UPDATE`/`DELETE`, the + live-address index, a primary key, the saturated allocator, a backend error in a statement) fails the object live + (`object_failed`, `broken`: memory and disk disagree). +- **6a's owners:** a refused `self.tx` rejects with the error above and the object fails as above; a refused + migration commit (a chunk, (A) or (Z)) fails the activation `broken`, the table stays `migrating`, committed chunks + stay; a refused job start rejects the start; a refused outcome or finish commit fails the object (records and + memory disagree); a refused commit that carried held changes (D8) fails the object. +- **Reads:** a read's callback returns data, never throws for a request: an adapter's request checks (an unknown + conversation, a cursor, history a document does not keep) happen after the guarded read returned, from tagged + results, so only a backend error fails the store. The store offers a read transaction for reads of several queries + that must see one state (S4). -Durable's documents on the documents engine (S13): address `(namespace "durable", owner, kind, key?)`, owner the -scope as a string (`session`, `conversation:7`, `task:31`), kind and key JSON-encoded as Durable's indexed strings. -6a built `latest`; 6b adds what Durable needs: +### 1.5 The documents engine -- **`rewindable` records** keep every revision and are not deleted at retirement: `retired_at` is set, revisions - stay (decision "Writes carry their versions" already defined the archive fields). -- **Reads at a point:** `findDocument(address, at)`, `document(id, at)` (the base at or below `at` plus the deltas up - to it, in one read transaction, S4), `scanDocuments(scope, at, kind?)`, with Durable's half-open lifetime - (`createdAt <= at < retiredAt`); a historical read of a `latest` (or session or task) record is a - `StorageRequestError` ("does not retain historical content"). -- **`document.copy`** (S11): `StoreStorage` reads the source at its point before the step and enters the step with a - plain create carrying the base; the conformance rules (source not changed in the copy batch, a matching record) are - checked there. -- **Stored version and `deltasSinceBase`** as `StoredDocument` reports them; Durable's records carry no fingerprint: - the engine's fingerprint column takes `""` for Durable records. -- **Export** carries every revision of a `rewindable` record, retired ones included, with its seq (format 1's - fields); import and the applier take them. +- **Addresses:** `(owner, namespace, kind, key?)`: `owner` the claimed name; `namespace` the adapter's partition + (Chord: `chord` for documents, `chord.jobs` and `chord.job-states` for persisted runs; Durable: its scope, + `session`, `conversation:`, `task:`); `kind` and `key` as the adapter writes them (Durable JSON-encodes both, + as its indexed strings). One live record per address (the partial unique index). +- **`history` per record (R1):** `none` (Chord: older revisions dropped at a new base; row and revisions deleted at + retirement), `latest` (Durable's session, task and latest conversation documents: older revisions dropped; the row + kept with `retired_at` at retirement), `rewindable` (nothing dropped; the row kept). `retireDocument` reads the record + and applies its rule. +- **Explicit ids:** a creation may carry its id (Durable mints ahead); otherwise the allocator gives one. +- **Changes without a known revision:** an adapter whose writes carry no revision (Durable) stages a change that reads + the record in the step and uses its revision; the counted conditional `UPDATE` stays the guard. +- **`meta`:** Durable's `fork` policy and `history` details live in the record's `meta`. +- **Reads at a point:** `find(address, at)`, `load(id, at)` (the base at or below `at` plus the deltas up to it, its + version the base's), `list(owner, namespace, at, kind?, limit, after?)` by id, with the half-open lifetime + (`createdAt <= at < retiredAt`); records carry `retiredAt`. New indexes (store schema version 2): + `(owner, namespace, kind, key, created_at)`, `(owner, namespace, id)`, revisions `(document_id, kind, seq)`; their + cost on Chord's commit gate is measured. -### 1.4 Commits +### 1.6 The archive and batch apply -- `StoreStorage.commit(writes, adopt, ctx)`: validates (1.1), resolves copies (1.3), stages the writes (table `put`s, - document creates, changes and retirements) and calls `store.commit(staged, { admit: () => true, adopt: (seq) => - adopt(seq) }, ctx)`. The store's step runs `adopt` inside `host.batch.commit(...)` when the store has a host, so - Durable's observers mark Chord sources in the step (W1, "Slice 6 first decisions" (1)); `adopt` never throws - (Durable's rule). -- **Durable's seq is the store's seq** (gaps allowed: Chord's commits take seqs too). -- **Errors:** +- **Format 1 gains** (nothing deployed): `document.create` carries `history`, `retiredAt?` and `meta`; a + `document.revision { id, seq, kind, version, content }` part per kept revision, in seq order after its create; for + `none` and `latest` records only the current base and its deltas are exported (as 6a), for `rewindable` every + revision, retired records included. Import checks: revisions in seq order, a base first, `seq <= lastSeq`, lifetimes + at one address never overlapping (a second live record is still refused), versions per revision; the `end` count + includes revisions. +- **Batch apply** (`applyBatch(store, commits, ctx)`, internal): only on a store with no attached host (no owner's + memory to update); `commits[0].seq` follows `lastSeq` (S5); one backend transaction for the batch, every write applied + with its commit's seq, `lastSeq` and `nextId` set from the last commit; the applier handles every `StoreWrite` kind + (`alter`, `dropIndex`, `update`/`updateWhere`/`delete`/`deleteWhere` with their recorded counts and versions, + `document.change`, `document.retire`, `document.revision`), checked against the store's records (a trust boundary); + a count that differs from the recorded one stops applying and marks the store diverged (S17): every later + `applyBatch` rejects. A test replays a store's outbox into an empty store and compares the exports, manifest + included. -**Errors (D4, owner, 2026-10-27):** the store takes Durable's `Storage` semantics (design §10 "Store errors"): a -step that fails before `COMMIT` rolls back and rejects that commit only; the store stops only on an unknown outcome -(`COMMIT` or `ROLLBACK` failed), a throwing `adopt`, or a read's backend error. Owners react as Durable's Session: -Durable's Session fails (`GuardedStorage`, unchanged); a Chord object whose commit the step refused fails live -(`object_failed`, `broken`); an `int` `CHECK` failure fails only its transaction (`invalid/args`). `StoreStorage` runs -Durable's commit checks in the step, as `SqliteStorage` does, so the conformance suite's refused commits leave the -store usable. 6b changes 6a's step and object failure accordingly. +## 2. `StoreStorage`: Durable's `Storage` on the store (D1, D2) -- `mintId`, `close(context)` (closes nothing of the store: the app owns it; later calls reject "closed"), reads after - a store failure reject as Durable's `GuardedStorage` expects. +In Durable's `src` (an ordinary pi module), importing `@earendil-works/chord/next` (Chord's `./next` export comes +with it: the store and the adapter surface below, marked internal for adapters). It passes Durable's storage +conformance suite on SQLite `:memory:` and across reopen (a wrapper that closes and reopens the store file after +every commit, Durable's reopen harness), and Durable's Session and harness test files run on it. -### 1.5 Statements per commit +### 2.1 Opening and the adapter surface -A target to measure, as 6a's: an entry append is the `record_ids` put, the entry put, `lastSeq` (plus the outbox); a -document delta is 6a's three statements. `SqliteStorage` does the same work today (`record_ids`, the row, metadata). +`StoreStorage.open(store, owner, ctx)`: claims `owner` (1.1); declares its tables through the store's table +preparation (the path `self.table` takes: created at the newest version or migrated); raises the allocator to +`max(record_ids.id) + 1` (one index read; covers stores built by import or batch apply); attaches to the store's host +batch if there is one. Chord exports for it: `StoreOwnerTables` (declare and prepare tables under an owner), +staged table writes and engine writes, the step reader, the read transaction, `mintId`, the commit with a fence. Its +`close(context)` releases the claim and refuses every later call ("StoreStorage is closed"); the store stays open (the +app owns it). -## 2. `self.follow`: Durable's views as Chord sources (D9, owner, 2026-10-27) +### 2.2 Tables -Durable's synchronous observers (`observeConversation`, `observeTaskGraph`, `observeDoc`; decision "Durable's -synchronous observers") have Chord's watch shapes (`advance(call, ctx)` with a `WatchCall`, `end(end)` with a -`WatchEnd`, pi `5f312dac4`), so Chord follows them with one generic primitive and Durable needs no adapter: +Every row carries the record's indexed columns and the whole record as `json(unknown(), { version: 1 })`; indexed +strings that must stay lossless (`kind`, `request_id`) hold Durable's JSON encoding; index columns are non-`opt`, so +absent values take sentinels that no encoded value can be (`0` for an absent owner id, ids start at 2; `""` for an +absent request id, JSON-encoded strings start with `"`). + +| Table | Key | Columns | Indexes | +| --- | --- | --- | --- | +| `record_ids` | `id` | `record_type str` | | +| `conversations` | `id` | `owner_conversation_id int`, `owner_task_id int`, `record` | `(owner_conversation_id, id)`, `(owner_task_id, id)` | +| `entries` | `id` | `conversation_id int`, `has_head int` (0/1), `commit_seq int` (stamped), `record` | `(conversation_id, id)`, `(conversation_id, has_head, id)` | +| `tasks` | `id` | `conversation_id int`, `kind str`, `status str`, `abort_requested int`, `background int`, `record` | `(status, id)`, `(conversation_id, id)`, `(kind, id)`, `(abort_requested, id)`, `(background, id)` | +| `submissions` | `id` | `conversation_id int`, `request_id str`, `status str`, `record` | `(conversation_id, request_id)`, `(conversation_id, id)`, `(status, id)` | + +### 2.3 Commits + +`commit(writes, adopt, ctx)` stages one composite write and calls the store's commit with `{ admit: () => true, +adopt: (seq) => adopt(seq) }` (adopted in the host's batch, 1.3). Inside the step the composite write does what +`SqliteStorage` does inside its transaction, with the step's reader: `checkGlobalIds` (an id already claimed by +another record type, a conversation, entry or document written twice: Durable's messages), the per-document actions +of `prepareDocumentActions`/`checkDocumentActions` (one content command per document, a delta on its version, one live +incarnation per address: "already has a current incarnation"), then the table `put`s (`record_ids` only for ids not yet +claimed by the same type), then per document: its creation (explicit id) or content, then its retirement; a +create-plus-retire is an empty lifetime (`createdAt = retiredAt = seq`). Copies (`document.copy`) are resolved before +the step (S11): the source read at its point, checked against the copy's record ("changed in the copy batch", +kind, key, history, fork), entered as a creation carrying the base. A refused check rejects with Durable's message (a +refusal, 1.4): the store goes on, Durable's Session fails (`GuardedStorage`). + +### 2.4 Reads + +`conversation`, `entry` (both overloads, `commitSeq` from its column), `findLatestHeadMarker`, `scanEntries` (both +orders, the fork chain walked as `SqliteStorage` does), `task`, `scanTasks`, `submission`, `scanSubmissions`, +`submissionByRequest`, `scanConversations`: queries over the tables, Durable's cursors (`scan.ts`) as they are; +`findDocument`, `document` (one read transaction), `scanDocuments`: the engine's reads at a point (1.5). Request errors +(`StorageRequestError`) are thrown after the guarded read (1.4). + +### 2.5 Cost + +Statements per commit, as the gate measures them: an entry append, a `record_ids` check read, the `record_ids` put, +the entry put, the meta update (4; `SqliteStorage` 5); a document delta, the record read, the revision insert, the +counted update, the meta update (about 4-5; `SqliteStorage` about 7). Per `put` one compiled row check and the JSON +encoding of indexed strings. + +## 3. `self.follow`: Durable's views as Chord sources (D9, R4) + +`self.follow(attach)` in a factory returns a `Watchable` (a state member, a derive source, an `on` source); +`attach(observer, ctx)` returns `Promise<{ value, detach } | undefined>` (Durable's `Observation`; `undefined` for an +absent `observeDoc` address). In 6b it is used inside the object that opened Durable (a native's value); reaching +Durable from other objects is slice 8's (the harness in a registry entry, carry-over). + +- **Activation:** after the natives opened, before the members are checked and derives compute: Chord makes the + observer and calls `attach`; calls arriving before it resolves are queued and applied in order on `value`; a rejected + or `undefined` attach fails the activation like a failed dependency (`transient`); a failed activation detaches + what attached. The attempt's end detaches. +- **Calls:** a call inside a running batch (Durable's `adopt` on the store) applies the queued calls first, then marks + the source in that batch; a call outside any batch (Durable's deferred commits, `#hold`, and the task graph's + `blocked` refresh, even on the store; any Durable storage that is not the store) is queued and one batch on the next + microtask applies everything queued. A derive over two views of one commit sees both changes in one computation. +- **Ends (R4):** `failed` fails the object `transient` (its retry reopens Durable and attaches again); `ended` is final: + a pending retirement call reaches the source's watches first (Durable's `endAfterDelivery`), then they end `ended`; + the member keeps its last value; a new `watch` on an ended follow source opens at that value and ends `ended` at once. + +## 4. Persisted jobs (D5-D8, R3) + +`persist: { resume: "rerun" | "abandon" }` only in `local` types with empty params (P3, at `self.job`); a persisted job +needs a state definition with `persist: true` and `checkpointWhen` if it has state, and a JSON-only result (no `bytes()`, +refused at declaration). + +### 4.1 Records + +Each unsettled run is a keyed persisted document in namespace `chord.jobs` (kind the job name, key the run id, +`history: none`); its state is a keyed persisted document in namespace `chord.job-states` (same kind and key), so two +jobs sharing a state definition never meet. The record: ```ts -host.serve(ChatView, ({ conversation }, self) => { - const harness = self.get(Harness, {}); - const view = self.follow((observer, ctx) => - harness.value.observeConversation(conversation, observer, ctx, { min: 50 })); - return { view, title: self.derive([view], ([v]) => v.title) }; -}); +{ version, fingerprint, args, key?, trace?, + phase: "pending" | "running" | "stopped" | "finishing" | "blocked", + restarts, finishFailures, abort?: true, blocked?: "finish_failed" | "version", + outcome?: { kind: "done", result } | { kind: "failed", error: StoredJobError } } +// StoredJobError: { kind: "threw", error: WireError } | { kind: "aborted", reason } | { kind: "restart_limit", restarts } ``` -- `self.follow(attach)` returns a `Watchable`: a state member, a derive source, an `on` source. `attach(observer, - ctx)` returns `Promise<{ value, detach }>` (Durable's `Observation`). -- **Attaching** is that one call during activation: Chord makes the observer, calls `attach`, takes `value` as the - source's value (calls arriving before the promise resolves are held and applied after it), and the object goes live. - The attempt's end calls `detach()`. An attach that rejects, or resolves `undefined` (`observeDoc` of an absent - address), fails the activation like a failed dependency. -- **Calls:** `advance` records the call. Inside a batch (on the store, Durable's `adopt` runs in the host's batch) the - source is marked in the step and its frame goes out in that flush; outside a batch (a Durable storage that is not - the store) the source collects the calls and opens one batch on the next microtask. A derive over two views of one - Durable commit sees both changes in one computation. -- **End:** Durable's failures are final: `end({ kind: "failed" })` ends the watches `failed`, `ended` ends them - `ended`. +Errors are stored in the wire error format (a `PluginError`'s code, message and details survive); a stored outcome is +rebuilt as the in-memory one. -## 3. Persisted jobs +### 4.2 Transitions -Design §15 "Persistence": `persist: { resume: "rerun" | "abandon" }`, only in `local` objects with empty params (P3, -at declaration); a run's record and state document in the owner's store; a recorded outcome resumes only `finish`; an -abort request is persisted; a restart limit; the job fingerprint stored with every record; the start trace carrier. +| Event | Writes (one commit unless noted) | After | +| --- | --- | --- | +| start | record `pending`, state `initial()` | invoked on the live attempt | +| reservation (before `run`) | record `running` | `run` invoked (a crash after it counts) | +| `ctx.state.change` | state; the record's `restarts: 0` only when it was not 0 | | +| abort (D7) | record `abort: true` (on the line); the signal fires after the commit, outside the batch; the line is released, then `abort` joins `run` | `"marked"` | +| `run` settles | held state flushed (D8), record `finishing` with the outcome | `finish` on the live attempt | +| `finish` fails | record `finishFailures + 1`; at 5 `blocked: "finish_failed"` | retried after the backoff, a blocked run on reload | +| `finish` succeeds | record deleted, state retired (Q9), with what `finish` wrote | the run settles | +| clean stop (host close, a native restart, R3's seal) | the stopped runs written `stopped`, held state flushed, by the close writer | | +| activation (once per object, not per retry) | each `running` record: `restarts + 1` (or, at 5, `failed { restart_limit }` and `finishing`, `job_interrupted` reported); `stopped` and `pending`: back to `pending` | maps, keys, active count, `running` rebuilt before anything runs; `rerun` invokes, `abandon` records `failed { aborted, restart }`; an abort-marked run finishes `aborted` without `run`; `finishing` resumes `finish`; `blocked` stays | -- **Where records live:** +- **The close writer:** at the object's end, after the drain and before the documents drop, one commit past the seal + (Durable's `writeAtClose`): the runs stopped by the end written `stopped`, held state flushed; a run whose `run` + outlived the drain is fenced and written `stopped` too; blocked persisted runs keep their records (dropped only by + `abort`). In-memory runs keep 6a's rules. +- **Pinning:** records and state documents of unsettled runs stay in memory (never evicted): `ctx.state.value` and + `running` read them synchronously. +- **Fingerprint:** the job's fingerprint (args, result, state) is stored; a record of the job's version with another + fingerprint is `blocked: "version"` (resolved by slice 7). -**Where records live (D5, owner, 2026-10-27):** each unsettled run's record is a keyed persisted document of its -object in the engine's namespace `chord.jobs` (owner the object's owner string, kind the job name, key the run id: -`{ version, fingerprint, args, key?, phase, outcome?, abort?, restarts, trace? }`); Chord sets that namespace, a plugin -never does (its documents are always `chord`), so nothing can collide. Its state document stays an ordinary document -of the run. Both change in the transactions that change them today (start, state change, outcome, finish, block, -drop); `finish` deletes the record and retires the state in its commit (Q9), so only unsettled runs are stored. At the -object's first activation the records are loaded through the engine's key listing and a load per run (no new read). +### 4.3 Deferred state (D8) -- **Resume at activation:** a run recorded `finishing` resumes `finish` only; a run recorded `running` was cut off by - a crash: `rerun` invokes `run` again with the stored args and state, `abandon` records `failed { aborted, reason: - "restart" }` and finishes. -- **Restart limit** (aligned with Durable's `restarts`): - -**The same rule as Durable (D6, owner, 2026-10-27: "yes, same rule for chord"):** at activation a run recorded -`running` was cut off by a crash: its `restarts` rises by one (stored), then it resumes (`rerun` invokes `run` again, -`abandon` finishes with `failed { aborted, reason: "restart" }`); a clean close (`host.close`, a native restart on -reload) writes its stopped runs back so the next activation counts nothing (Durable's `writeAtClose`); a -`ctx.state.change` resets the count (progress); at five in a row the run ends with a new `JobError` `{ kind: -"restart_limit", restarts }`, its `finish` runs, and `job_interrupted` is reported (Durable's `restart_limit`, -`task_interrupted`). No `blocked { restart_limit }`: supersedes design §15's parking, as Durable dropped parking in -#11139. - -- **Abort** (owner, 2026-10-27: async and stored in 6b): - -**`job.abort(id, ctx): Promise<"marked" | "terminal">` (D7, owner, 2026-10-27), Durable's words and behaviour:** -a running run gets its abort stored (a rerun after a crash then finishes `aborted` instead of running again) and its -signal fired, and `abort` resolves `"marked"` once `run` settled (as `abortTask` joins the run); a finishing run is only -marked (`"marked"`, as Durable's `completing`); a blocked run is dropped at once (`"marked"`, as Durable's orphan); an id -with no unsettled run resolves `"terminal"` (Chord keeps no settled runs, Q9, so a settled and an unknown id are the -same; Durable throws for an unknown one). - -- **Deferred state persistence:** - -**Per change, Durable's shape (D8, owner, 2026-10-27):** `ctx.state.change(fn, ctx, { persistence: { kind: -"deferred", withinMs? } })` (default 1,000 ms, as Durable's `CommitPersistence`): adopted and published at once, -stored with the next immediate change of the document, before the outcome is recorded (so `outcome.state` has it), at -close, or after `withinMs`; a crash loses what is not stored. Only run state documents; other persisted documents get -it when a real case needs it. - -- **Fingerprint:** the job's fingerprint (args, result and state schemas) stored with every record; a record of the - same version with another fingerprint is `blocked { version }` (slice 7's family; the carry-over's "a job - fingerprint over args, result and state"). -- **Trace carrier:** the start's carrier (design §29) stored with the record; used in slice 9. - -## 4. The applier for batch apply - -6a's applier applies an archive's writes; 6b extends it to every `StoreWrite` kind (`alter`, `dropIndex`, `update`, -`updateWhere`, `delete`, `deleteWhere` with their recorded counts and versions, `document.change`, `document.retire`), -checked against the store's records (a trust boundary), one code path with import. A test replays a store's outbox -into an empty store and compares the two stores' exports. Applying a batch with a different count than recorded stops -applying (S17). +As Durable's holds: the slot adopts and publishes the change at once (`valueVersion` ahead of what is stored), keeps +the held ops in a `PendingOps` (a base flag when `checkpointWhen` selected one), and the run's documents stay pinned; +the next immediate change of the state document stores the held ops and its own as one delta on the stored revision +(or a base); one foreground timer per object for the earliest deadline (`withinMs`, default 1,000 ms, `Infinity` waits +for the others; a negative or non-number `withinMs` is `invalid/args`); the outcome's commit and the close writer +store what is held; a held change of a fenced invocation is still stored; a crash loses it. A deferred change that +would reset `restarts` is stored at once (the reset must be durable). On an in-memory job the option is accepted and +does nothing. ## 5. Benchmarks and tests -- **The gate** (decisions "The commit-step benchmark", "The performance and memory bar"): Durable's own storage - benchmark (`test/storage.bench.ts`: the write and read cases of `testing/storage-benchmark.ts`) on `StoreStorage` - against `SqliteStorage`, and Durable's `test/conversation-view-bench.ts` on both: each hot path within about 5%, - memory (allocation per op, retained after GC) no worse beyond noise; persisted job starts and state changes against - in-memory ones. -- **Tests:** Durable's storage conformance suite on `StoreStorage` (memory `:memory:` and across reopen); Durable's - Session and harness test files that take a storage, run on `StoreStorage`; `self.follow` (one frame per commit, a - derive over two views, outside the store, early calls held, failure and end, an absent `observeDoc`); persisted jobs (rerun, abandon, a recorded outcome - resumes `finish`, restart counting and reset, the limit, abort stored, deferred state, fingerprints); the applier - (every kind, the outbox replay, refused counts); export and import with Durable records and `rewindable` history; - GC (observers detached with their attempt). -- **Examples:** `26-durable-on-the-store.ts`, `27-durable-views.ts`, `28-persisted-jobs.ts`. +- **The gate** ("The performance and memory bar"): Durable's storage benchmark (`test/storage.bench.ts`, every read and + write case, 1k and 10k) and `test/conversation-view-bench.ts` on `StoreStorage` against `SqliteStorage`, outbox off + (its cost reported separately): each hot path within about 5%, allocation per op and retained memory no worse beyond + noise; Chord's 6a commit gate again with the new indexes and the allocator; persisted job start and state change + reported against in-memory ones (a disk commit is not held to the in-memory number). +- **Tests:** the conformance suite (both fixtures); the D4 matrix (`BEGIN`, body, `COMMIT`, `ROLLBACK` failures; a + refused Durable commit keeps the store; a refused Chord commit fails its object; a `CHECK` failure only its + transaction; a refused migration chunk; a request error inside a multi-query read keeps the store); the allocator (a + mint during a running Chord step, saturation across reopen, a refused step's gap, replay restoring `nextId`); + create-plus-retire and change-plus-retire through the outbox and the applier; reads at a point on retired `latest` + and `rewindable` records; export and import of rewindable history and mixed Chord and Durable data; `applyBatch` + (every kind, the replay equality, a diverged count); `self.follow` (one frame per commit, a derive over two views, + calls outside a batch on the store, early calls, an absent `observeDoc`, `failed` and `ended`, a retirement before + the end); persisted jobs (every transition of 4.2, a crash between reservation and `run`, a crash during the close + writer, a seal and retry, restart counting and its reset, the limit, the finish failure count across a crash, abort + joined and stored, abort racing an outcome, deferred state merged, flushed at the outcome and at close, pinned past + the retention); GC (closed `StoreStorage`, ended follow sources, cancelled abort waiters, deferred timers, settled and + dropped runs). +- **Examples:** `26-durable-on-the-store.ts` (a worker with Durable's Session and a Chord store object on one store, + exporting the archive), `27-durable-views.ts`, `28-persisted-jobs.ts`. ## Carry-over taken -From `slices.md` "Slice 6b": persisted jobs (all of it), `StoreStorage` (ids, fork copies, `adopt` in the batch), -the Chord source over Durable's views (`self.follow`, the batch on and off the store), the applier for batch apply; `job.abort` async and -stored (decision of 2026-10-27). +From `slices.md` "Slice 6b": persisted jobs (all of it, the finish failure count included), `StoreStorage` (ids, fork +copies, `adopt` in the batch), the Chord source over Durable's views (`self.follow`), the applier for batch apply; +`job.abort` async and stored. ## Carry-over left -Slice 7: a job fingerprint mismatch's `blocked { version }` resolution on reload; runs of jobs the code no longer -declares (with D5's records). Slice 9: the trace carrier's use. +Slice 7: `blocked: "version"` resolved on reload; runs of jobs the code no longer declares; blocked runs retried on +reload. Slice 8: Durable's views from other objects (the harness in a registry entry). Slice 9: the trace carrier's +use. From b2fcfee0835b7b09a2d34611442608b3925ac279 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 15:09:35 +0000 Subject: [PATCH 087/271] docs(chord): slice 6b ready to build; claims per table and namespace --- packages/chord/docs/next/decisions.md | 9 +++- packages/chord/docs/next/design.md | 10 ++-- packages/chord/docs/next/slice-6b.md | 70 ++++++++++++++++----------- packages/chord/docs/next/slices.md | 32 +++++------- 4 files changed, 70 insertions(+), 51 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index f8b3782ca..7ef784b6e 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1717,8 +1717,8 @@ validation and copies the v2 kernel added around Delta. declares persisted data (`self.table`, persisted `self.state`, persisted `self.job`); `StoreStorage.open(store, owner, ctx)` claims `owner`; a second claim of a name is refused (`invalid/args`): a second harness, or a plugin type that meets the harness's name. `StoreStorage` declares its own tables (`entries`, `tasks`, …) through the store, the path - `self.table` takes, named `":
"`, and writes its documents under `owner` in the engine's namespace - `durable`. In p3 the harness object passes its own type (`StoreStorage.open(store, Harness.name, ctx)`; base code gives + `self.table` takes, named `":
"`, and writes its documents under `owner` in the engine's namespaces + (Durable's scopes; corrected 2026-10-27, it was "`durable`"). In p3 the harness object passes its own type (`StoreStorage.open(store, Harness.name, ctx)`; base code gives the store to the core facet); Durable's conformance suite passes a test name. The owner name plays the role of `SqliteStorage`'s `prefix`, checked by the store. Objects never see the store: their API writes to the host's store under their type; a host without a store fails an object declaring persisted data (`broken`, 6a). Rejected: an owner @@ -1731,6 +1731,11 @@ validation and copies the v2 kernel added around Delta. failed: the object that opened Durable fails `transient`, its retry reopens its natives, Durable included, and attaches again, as Durable's own recovery is to reopen from Storage) or `ended` (the observed owner ended normally, a task document's task: final, the member keeps its last value, its watches end `ended`). +- **Slice 6b: claims per table and per document namespace (owner, 2026-10-27: "agree"; refines "owner names and + claims"; Fable's second review N2):** what can collide is claimed once per store: a table by its SQL name, documents by + `(owner, namespace)`; so the object that opens Durable can also declare its own persisted data under the same owner + name, and two harnesses can share a store under two owner names (owner: "seems like they could"); a second + `StoreStorage` on the same owner is refused by its tables. "One harness per store" is no rule, only how p3 uses it. - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index 7ea0ced16..cd55b3bfb 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -215,7 +215,7 @@ and closed by the app with `store.close(ctx)` after `host.close`; the host never background, as a tagged `HostError`: `listener_error` (a watch listener threw), `delivery_failed` (a frame delivery threw), `activation_failed { object, failure }`, `close_failed { object, native, error }`, `close_timeout { object, native }`, `drain_timeout { object }`, `protocol_error { error }`, `resolve_failed { address, error }`; later routine -errors (slice 5), `derive_failed { object, member?, error }` (a derive's compute threw, once per failure episode, slice 6a), `object_failed { object, failure }` (a live object failing, slice 6a; `failure` where the value is a `FailureCause`, `error` otherwise, 2026-10-27) and `storage_failed { error }` +errors (slice 5), `derive_failed { object, member?, error }` (a derive's compute threw, once per failure episode, slice 6a), `object_failed { object, failure }` (a live object failing, slice 6a, `broken`; `transient` when a `self.follow` source failed, slice 6b R4; `failure` where the value is a `FailureCause`, `error` otherwise, 2026-10-27), `job_interrupted { object, job, id, restarts }` (a persisted run ended at the restart limit, slice 6b D6) and `storage_failed { error }` (slice 6a), which every app must handle: Chord never ends the process, the app exits with its own code for its supervisor or reopens. @@ -1024,8 +1024,12 @@ host.serve(DraftStore, (_, self) => { // local, empty params: may - **Statements per commit (owner, 2026-10-26):** the store keeps `lastSeq` (the seq of its last commit) in memory (read at open, persisted with each commit, S9) and folds the documents engine's checks into the writes (unique keys, a partial unique index on the live address, conditional `UPDATE … WHERE id = ? AND version = ?`): a delta commit is about three - statements (revision, the record's counted conditional `UPDATE`, `lastSeq`), plus one `INSERT` with an outbox. Durable's ids in the store: `StoreStorage` keeps a `record_ids` table (one - namespace across Durable's five tables), its counter at open `coalesce(max(id), 1) + 1`. + statements (revision, the record's counted conditional `UPDATE`, `lastSeq`), plus one `INSERT` with an outbox. Durable's ids in the store (slice 6b D3): one id allocator per store + for every id (Durable's `mintId` and Chord's document ids; it starts at 2, saturates at `Number.MAX_SAFE_INTEGER`, each + commit persists it, outbox commits and archives carry it as `nextId`); `StoreStorage`'s `record_ids` table only checks + an id's record type. Every table and document namespace in the store is claimed once (slice 6b "owner names and + claims"): a table by its SQL name `":
"`, documents by `(owner, namespace)`; an object claims under its + type, `StoreStorage.open(store, owner, ctx)` under `owner` (two harnesses on one store under two names are fine). - **Tables in a transaction (owner, 2026-10-26):** the table handle first, keys as objects with exactly the key columns, reads async, writes synchronous and buffered like a draft's ops (checked and staged, no I/O; written in the commit step or dropped on failure); no write shorthands on the handle: diff --git a/packages/chord/docs/next/slice-6b.md b/packages/chord/docs/next/slice-6b.md index 52d9cd21e..32e1eb69b 100644 --- a/packages/chord/docs/next/slice-6b.md +++ b/packages/chord/docs/next/slice-6b.md @@ -1,8 +1,8 @@ # Slice 6b: Durable and jobs on the store (spec) -Status: **revised after the first spec review** (2026-10-27): D1-D9 and R1-R4 decided by the owner (decision log, -entries "Slice 6b …" and "owner names and claims"); the four reviews (`/tmp/s6b-review-{astra,fable,opus,sol}.md`) -kept every decision and found the gaps this revision closes. Written by the redesign session after reading, in full, +Status: **ready to build** (2026-10-27): D1-D9, R1-R4 and the claims decided by the owner (decision log, entries +"Slice 6b …" and "owner names and claims"); two spec reviews (`/tmp/s6b-review{,2}-{astra,fable,opus,sol}.md`) +kept every decision; this version closes their findings. Written by the redesign session after reading, in full, Durable's `types.ts`, its SQLite storage, schema and migrations, the storage conformance suite and benchmark, `session/session.ts`, `session/transaction.ts`, `session/forks.ts`, the scheduler's restarts, `writeAtClose` and `abort`, and Chord v2's store, engine, applier, commit step and jobs. Durable's memory, JSONL and IndexedDB storages stay as they @@ -53,10 +53,14 @@ Every item below changes 6a's code; nothing is deployed, so the schema and the a - Tables are `":
"` (`"notes.store:edits"`), document owners `""`; no host kind. The applier and import check two name parts. -- Every owner name is claimed once per store (in memory, for the store's lifetime): an object claims its type when it - first declares persisted data; `StoreStorage.open` claims its name; a second claim is refused `invalid/args`; a - claim is released by its owner's end (`StoreStorage.close`, the object's end), so a new Session can reopen over the - same store. +- **Claims** (owner, 2026-10-27, decision "owner names and claims", refined): what can collide is claimed once per + store, in memory: a table by its SQL name, documents by `(owner, namespace)`. An object claims its tables and + `(type, "chord")` (plus `chord.jobs` and `chord.job-states` with persisted jobs) when it declares them; + `StoreStorage.open` claims its five tables and its document namespaces (Durable's scopes contain a `:`, Chord's + namespaces never do). A second claim is refused `invalid/args`; a claim is released by its owner's end + (`StoreStorage.close` after its calls underway drained, the object's end), so a new Session can reopen over the same + store. So the object that opens Durable can declare its own persisted data under the same owner name, and two + harnesses can share a store under two owner names. ### 1.2 One id allocator (D3) @@ -67,8 +71,10 @@ Every item below changes 6a's code; nothing is deployed, so the schema and the a - It starts at 2 (id 1 is Durable's root conversation). It saturates at `Number.MAX_SAFE_INTEGER`: an explicit write of that id is accepted, the persisted value stays `MAX_SAFE_INTEGER`, every later `mintId` rejects "ID space is exhausted", and a Chord creation at the saturated allocator refuses its own commit (1.4). -- Outbox commits carry the allocator's value after them (`{ seq, nextId, writes }`); the archive manifest's - `nextDocumentId` is `nextId`; import and batch apply restore it, so a replica mints above every id it holds. +- Each commit takes the allocator's value once, after its writes applied, for the outbox commit (`{ seq, nextId, + writes }`) and the metadata row (mints meanwhile only raise it further); an export reads it once with `lastSeq`; the + archive manifest's `nextDocumentId` is `nextId`; import and batch apply restore it, so a replica mints above every id + it holds. ### 1.3 The commit step @@ -93,13 +99,14 @@ Every item below changes 6a's code; nothing is deployed, so the schema and the a outcome: the store stops (`storage_failed`). The Cloudflare facade is checked to keep the same identity; a test injects failures at `BEGIN`, in the body, at `COMMIT` and at `ROLLBACK` on both drivers' facades. - **The refusal's error reaches the owner unchanged** (Durable's suite matches its messages); the store classifies it - for Chord's owners: an `int` `CHECK` failure (SQLite's extended code `SQLITE_CONSTRAINT_CHECK`) is `invalid/args` and + for Chord's owners: an `int` `CHECK` failure (SQLite's extended code `SQLITE_CONSTRAINT_CHECK` on Node; the error + message on the Durable Object facade, which has no extended codes) is `invalid/args` and fails only that transaction; every other refusal of a Chord commit (the engine's counted `UPDATE`/`DELETE`, the live-address index, a primary key, the saturated allocator, a backend error in a statement) fails the object live (`object_failed`, `broken`: memory and disk disagree). - **6a's owners:** a refused `self.tx` rejects with the error above and the object fails as above; a refused - migration commit (a chunk, (A) or (Z)) fails the activation `broken`, the table stays `migrating`, committed chunks - stay; a refused job start rejects the start; a refused outcome or finish commit fails the object (records and + migration commit (a chunk, (A) or (Z)) is rolled back whole, fails the activation `broken`, the table stays + `migrating`, earlier committed chunks stay; a refused job start rejects the start; a refused outcome or finish commit fails the object (records and memory disagree); a refused commit that carried held changes (D8) fails the object. - **Reads:** a read's callback returns data, never throws for a request: an adapter's request checks (an unknown conversation, a cursor, history a document does not keep) happen after the guarded read returned, from tagged @@ -108,9 +115,9 @@ Every item below changes 6a's code; nothing is deployed, so the schema and the a ### 1.5 The documents engine -- **Addresses:** `(owner, namespace, kind, key?)`: `owner` the claimed name; `namespace` the adapter's partition - (Chord: `chord` for documents, `chord.jobs` and `chord.job-states` for persisted runs; Durable: its scope, - `session`, `conversation:`, `task:`); `kind` and `key` as the adapter writes them (Durable JSON-encodes both, +- **Addresses:** `(owner, namespace, kind, key?)`: `owner` the owner name; `namespace` the adapter's partition, claimed + with the owner (Chord: `chord` for documents, `chord.jobs` and `chord.job-states` for persisted runs; Durable: the + document's scope, `session`, `conversation:`, `task:`, which Durable's reads by scope need); `kind` and `key` as the adapter writes them (Durable JSON-encodes both, as its indexed strings). One live record per address (the partial unique index). - **`history` per record (R1):** `none` (Chord: older revisions dropped at a new base; row and revisions deleted at retirement), `latest` (Durable's session, task and latest conversation documents: older revisions dropped; the row @@ -130,12 +137,13 @@ Every item below changes 6a's code; nothing is deployed, so the schema and the a - **Format 1 gains** (nothing deployed): `document.create` carries `history`, `retiredAt?` and `meta`; a `document.revision { id, seq, kind, version, content }` part per kept revision, in seq order after its create; for - `none` and `latest` records only the current base and its deltas are exported (as 6a), for `rewindable` every - revision, retired records included. Import checks: revisions in seq order, a base first, `seq <= lastSeq`, lifetimes + live `none` and `latest` records only the current base and its deltas are exported (as 6a); a retired `latest` record + is exported as its record alone (no base: its content was dropped at retirement); for `rewindable` every revision, + retired records included. Import checks: revisions in seq order, a base first, `seq <= lastSeq`, lifetimes at one address never overlapping (a second live record is still refused), versions per revision; the `end` count includes revisions. -- **Batch apply** (`applyBatch(store, commits, ctx)`, internal): only on a store with no attached host (no owner's - memory to update); `commits[0].seq` follows `lastSeq` (S5); one backend transaction for the batch, every write applied +- **Batch apply** (`applyBatch(store, commits, ctx)`, internal): refused while any owner holds a claim on the store + (an object or a `StoreStorage`: its memory would go stale) and run through the store's commit queue; `commits[0].seq` follows `lastSeq` (S5); one backend transaction for the batch, every write applied with its commit's seq, `lastSeq` and `nextId` set from the last commit; the applier handles every `StoreWrite` kind (`alter`, `dropIndex`, `update`/`updateWhere`/`delete`/`deleteWhere` with their recorded counts and versions, `document.change`, `document.retire`, `document.revision`), checked against the store's records (a trust boundary); @@ -154,10 +162,11 @@ every commit, Durable's reopen harness), and Durable's Session and harness test `StoreStorage.open(store, owner, ctx)`: claims `owner` (1.1); declares its tables through the store's table preparation (the path `self.table` takes: created at the newest version or migrated); raises the allocator to -`max(record_ids.id) + 1` (one index read; covers stores built by import or batch apply); attaches to the store's host +`max(record_ids.id) + 1`, saturating at `Number.MAX_SAFE_INTEGER` (one index read; covers stores built by import or +batch apply); attaches to the store's host batch if there is one. Chord exports for it: `StoreOwnerTables` (declare and prepare tables under an owner), staged table writes and engine writes, the step reader, the read transaction, `mintId`, the commit with a fence. Its -`close(context)` releases the claim and refuses every later call ("StoreStorage is closed"); the store stays open (the +`close(context)` refuses new calls ("StoreStorage is closed"), waits for its calls underway, then releases its claims; the store stays open (the app owns it). ### 2.2 Tables @@ -183,8 +192,11 @@ adopt: (seq) => adopt(seq) }` (adopted in the host's batch, 1.3). Inside the ste another record type, a conversation, entry or document written twice: Durable's messages), the per-document actions of `prepareDocumentActions`/`checkDocumentActions` (one content command per document, a delta on its version, one live incarnation per address: "already has a current incarnation"), then the table `put`s (`record_ids` only for ids not yet -claimed by the same type), then per document: its creation (explicit id) or content, then its retirement; a -create-plus-retire is an empty lifetime (`createdAt = retiredAt = seq`). Copies (`document.copy`) are resolved before +claimed by the same type), then the documents in two passes: first every document that already exists (its content, +then its retirement), which frees the addresses a creation of this commit takes, then the creations and copies (each +with its retirement, a create-plus-retire being an empty lifetime, `createdAt = retiredAt = seq`). Durable's +"reconstructs rewindable documents" case (`create secondId` at address A, `retire firstId` at A, `change firstId`) +needs this order. Copies (`document.copy`) are resolved before the step (S11): the source read at its point, checked against the copy's record ("changed in the copy batch", kind, key, history, fork), entered as a creation carrying the base. A refused check rejects with Durable's message (a refusal, 1.4): the store goes on, Durable's Session fails (`GuardedStorage`). @@ -219,7 +231,8 @@ Durable from other objects is slice 8's (the harness in a registry entry, carry- the source in that batch; a call outside any batch (Durable's deferred commits, `#hold`, and the task graph's `blocked` refresh, even on the store; any Durable storage that is not the store) is queued and one batch on the next microtask applies everything queued. A derive over two views of one commit sees both changes in one computation. -- **Ends (R4):** `failed` fails the object `transient` (its retry reopens Durable and attaches again); `ended` is final: +- **Ends (R4):** `failed` fails the live object `transient` (reported once as `object_failed` with that failure, its + identity's backoff, several sources ending at once fail it once; its retry reopens Durable and attaches again); `ended` is final: a pending retirement call reaches the source's watches first (Durable's `endAfterDelivery`), then they end `ended`; the member keeps its last value; a new `watch` on an ended follow source opens at that value and ends `ended` at once. @@ -258,16 +271,19 @@ rebuilt as the in-memory one. | `finish` fails | record `finishFailures + 1`; at 5 `blocked: "finish_failed"` | retried after the backoff, a blocked run on reload | | `finish` succeeds | record deleted, state retired (Q9), with what `finish` wrote | the run settles | | clean stop (host close, a native restart, R3's seal) | the stopped runs written `stopped`, held state flushed, by the close writer | | -| activation (once per object, not per retry) | each `running` record: `restarts + 1` (or, at 5, `failed { restart_limit }` and `finishing`, `job_interrupted` reported); `stopped` and `pending`: back to `pending` | maps, keys, active count, `running` rebuilt before anything runs; `rerun` invokes, `abandon` records `failed { aborted, restart }`; an abort-marked run finishes `aborted` without `run`; `finishing` resumes `finish`; `blocked` stays | +| activation (records loaded once per object; each live attempt resumes what waits) | each `running` record: `restarts + 1` (or, at 5, `failed { restart_limit }` and `finishing`, `job_interrupted` reported); `stopped` and `pending`: back to `pending` | maps, keys, active count, `running` rebuilt before anything runs; `rerun` invokes, `abandon` records `failed { aborted, restart }`; an abort-marked run finishes `aborted` without `run`; `finishing` resumes `finish`; `blocked` stays; a record of another job version, or of the same version with another fingerprint, is `blocked: "version"` | +- **A retry after a seal (R3):** the runs the seal stopped are resumed by the next live attempt only once their old + invocation settled (bounded by `closeTimeout`, as the drain; an invocation outliving it is fenced, its run resumes + then), so a `run` that ignores its signal never runs twice at once. - **The close writer:** at the object's end, after the drain and before the documents drop, one commit past the seal (Durable's `writeAtClose`): the runs stopped by the end written `stopped`, held state flushed; a run whose `run` outlived the drain is fenced and written `stopped` too; blocked persisted runs keep their records (dropped only by `abort`). In-memory runs keep 6a's rules. - **Pinning:** records and state documents of unsettled runs stay in memory (never evicted): `ctx.state.value` and `running` read them synchronously. -- **Fingerprint:** the job's fingerprint (args, result, state) is stored; a record of the job's version with another - fingerprint is `blocked: "version"` (resolved by slice 7). +- **Fingerprint:** the job's fingerprint (args, result, state) is stored; a mismatch is `blocked: "version"` (resolved + by slice 7). A run ending at the limit reports `job_interrupted { object, job, id, restarts }` (a new `HostError`). ### 4.3 Deferred state (D8) diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index 826f223c0..d0498da76 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -163,25 +163,16 @@ deleted (the slice's spec and code are the record). Moved here from `carry-over. ### Slice 6b: Durable and jobs on the store (from slice 5 and L3a-d) -- **Persisted jobs** (from slice 5): `persist: { resume: "rerun" | "abandon" }` built (record and state document in - the owner's store; only in `local` objects with empty params, P3, enforced at declaration); outcomes and abort - requests persisted (a recorded outcome resumes only `finish`); the restart limit (five crash interruptions, - persisted, clean closes not counted; whether a state change resets the count; what a clean close records); deferred - persistence for state documents (flushed before the outcome is recorded); the `finish` failure count persisted; the - job fingerprint stored with every record; the run's start trace carrier (§29). -- **`StoreStorage`**: Durable's `Storage` on the store, passing Durable's storage conformance suite (memory and across - reopen); ids from a `record_ids` table (S10, W13); fork copies resolved before the step (S11); Durable's `adopt` run - inside the host's batch. -- **`self.follow`** (was `durableView`, decision "Slice 6b D9"): a Chord source over Durable's synchronous observers (`observeConversation`, `observeTaskGraph`, - `observeDoc`; decision "Durable's synchronous observers"), marked inside Chord's batch in the store and grouped into - one batch per Durable commit on the next microtask outside it; `self.follow` (working name) and the Durable harness - carried in a registry entry. The task graph's `blocked` refresh arrives outside any commit. -- **The applier for batch apply** (slice 6a builder, 2026-10-27): 6a's applier (`store/applier.ts`) applies the - writes an archive holds (`table`, `put`, `index`, `document.create`), checked against the store's records; the - replica's batch apply extends it to every `StoreWrite` kind (`alter`, `dropIndex`, `update`/`updateWhere` with their - recorded counts and `v`, `delete`/`deleteWhere`, `document.change`, `document.retire`), one code path with import. -- Not carry-over any more: Durable's synchronous view-change hook is the upstream "Durable observers" change - (publication at adoption, `observeXXX`), which lands before 6a. +Specified in `slice-6b.md` (decided 2026-10-27, D1-D9, R1-R4, "owner names and claims"); each entry below is taken there. + +- **Persisted jobs** (from slice 5): `persist: { resume }`, records and state documents in the owner's store (D5), the + restart limit (D6), `abort` stored (D7), deferred state (D8), the seal as a clean stop (R3), the `finish` failure + count persisted, the job fingerprint and the start trace carrier stored. +- **`StoreStorage`** (D1-D4, R1, R2, claims): Durable's `Storage` on the store, passing its conformance suite; one id + allocator (D3); Durable's checks in the step; fork copies resolved before the step (S11); `adopt` in the host's batch. +- **`self.follow`** (D9, R4): a Chord source over Durable's synchronous observers. Reaching Durable from other objects + (the harness in a registry entry) moves to slice 8. +- **The applier for batch apply**: every `StoreWrite` kind, `applyBatch`, one code path with import. ### Slice 7: reload and rollout @@ -233,6 +224,9 @@ deleted (the slice's spec and code are the record). Moved here from `carry-over. ### Slice 8: registries and `host.root` +- **Durable's views from other objects** (from slice 6b): the harness carried in a registry entry, so an object other + than the one that opened Durable can `self.follow` its views. + - **`host.root(fn, context, { principal? })`**: a call scope for code outside objects, built with its first users (Durable hooks, `chord(fn, context)`); decide what an aborted, uncooperative `fn` means for its references (match the handler and `CallContext` rules). (Slice 3b, owner.) From ad702637fdf6fbd3f965c5dba0b14bae100b74eb Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 15:31:08 +0000 Subject: [PATCH 088/271] docs(chord): slice 8 spec draft --- packages/chord/docs/next/slice-8.md | 147 ++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 packages/chord/docs/next/slice-8.md diff --git a/packages/chord/docs/next/slice-8.md b/packages/chord/docs/next/slice-8.md new file mode 100644 index 000000000..3bc1511b5 --- /dev/null +++ b/packages/chord/docs/next/slice-8.md @@ -0,0 +1,147 @@ +# Slice 8: registries and `host.root` (spec) + +Status: **draft, open points marked "Decision needed"** (2026-10-27), written by the redesign session. Built in +parallel with 6b (mostly new files). The single source for building 8, together with `design.md` (§7, §8, §11, §16, +§17, §18, §24 read in full), the decision log `decisions.md` (R21, R23, R25, Q8, D1 of "Review round 1", "Registries +work like Durable's", "Changing contributions, `self.writable`", W4, "Failures are watch calls", "Slice 8 is +'Registries and `host.root`'"), the feature map and carry-over of `slices.md` (slice 8), and the root `AGENTS.md` +"Chord v2 Code Style". + +8 builds the extension point between plugins (registries), in-process values that change (`self.writable`), wake +signals for failed objects (`retryOn`), and the call scope for code outside objects (`host.root`). The harness object +is p3's base code; Chord shows it only in an example. + +## What an app can do after this slice + +```ts +// the defining plugin (e.g. http): the token, exported for other plugins +export const Http = defineRegistry("chord.http", { + version: 2, + accepts: { 1: (old: HttpContributionV1): HttpContribution => upgrade(old) }, + compose: (table, { report }) => buildRouter(table, report), // pure; always a value; conflicts reported +}); +export default defineFacet((host, self) => { + self.declare(Http); // the one authoritative declaration (Q8) + host.serve(HttpServer, (_, self) => { + const router = self.registry(Http); // a NativeWatchable of the composed value + self.on(router, ({ value }, ctx) => …); + return { … }; + }); +}); + +// any plugin: a facet or a host-lifetime object contributes one value per registry +export default defineFacet((host, self) => { + self.contribute(Http, { routes: { "GET /notes/export": exportNotes } }); +}); +host.serve(McpStore, (_, self) => { // local, empty params (R21) + const ext = self.writable(mcpExtension([])); // an in-process value that changes + self.contribute(HarnessExtensions, ext); + self.on(mcp.tools, ({ value: tools }) => ext.set(mcpExtension(tools))); + self.retryOn(self.registry(HarnessModels)); // a model contribution wakes a failed attempt + return {}; +}); + +// code outside objects (a Durable tool, a CLI): a call scope +await host.root(async (ctx) => (await ctx.get(Notes, { session })).add(note, ctx), context); +``` + +## 1. Registries + +- **Token:** `defineRegistry(name, { version, accepts?, compose })` (names as types, `[a-z0-9@/._~-]`): the + registry's identity (name, `C` the contribution type, `V` the composed value type) and the defining plugin's + declaration in one object; every plugin's bundled copy carries the version it was built against. + +> **Decision needed (D1): how the defining plugin declares.** Q8: one authoritative declaration, from the defining +> plugin's facet; other plugins' tokens only identify the registry. Options: (a) `self.declare(Http)` in the defining +> facet: the token carries `compose` and `accepts`, the host uses those of the declaring facet's copy only; (b) the +> token is name and types only (`defineRegistry("chord.http")`), and the facet declares `self.declare(Http, { +> version, compose, accepts })`. Proposal: (a), the shape design §17 shows: one object, and a token used for +> contributing never has its `compose` called. + +- **Contributors** (R21): facets (by plugin name) and host-lifetime objects (`local`, empty params: by plugin name and + type), at most one value per registry each; `self.contribute(R, value | Watchable | NativeWatchable)` in a facet setup + or such an object's factory (a type error elsewhere; a declaration error at runtime). A facet's contribution is in the + table while its plugin is loaded; an object's from `live` until it ends, fails or restarts. +- **The table:** an immutable list of `{ contributor, plugin, value }`, sorted by plugin name, then type (never load + order); a contribution carries its token's version: an older one is adapted once with the declaration's `accepts` + when the table is built, an older one without an adapter or a newer one is left out and reported (`HostError` + `contribution_refused { registry, plugin, version }`). +- **`compose(table, { report })`:** pure, runs whenever the table changes (a contributor goes live or ends, a + contribution that is a watchable changes), in the owner's flush, never inside another owner's commit step (W4); + `report(conflict)` goes to `onError` as `registry_conflict { registry, message, plugins }`. A throwing `compose` + makes the composed value fail: its watches get a `failed` call, `.value` throws the error, the next good compose heals + it ("Failures are watch calls"). +- **Without a declaration** (the defining plugin is not loaded): the composed value is failed with + `unavailable/undeclared` (readers see the error as from a failed reference); contributions wait in the table; a + declaration heals it. +- **Reading:** `self.registry(R)` in any object's factory returns a `NativeWatchable` (a derive source, an `on` + source, a `retryOn` source); methods and routines read `.value` at use; `host.registry(R)` for code outside objects. +- **Helpers:** `keyed(table, entriesOf)` (a map, a duplicate key reported, the first by plugin name wins) and + `ordered(table, itemsOf, orderOf?)`. +- **Use sites guard contributions** (Q8): Chord's helpers keep each entry's plugin name; the guide shows catching a + contribution's throws and rejections, reporting the plugin, and continuing. +- **Registries carry natives** (design §17): a contribution may be any in-process value; an entry lives exactly as long + as its contributor. + +## 2. `self.writable(initial)` + +A `NativeWatchable` with `set(value)` (identity compared: `set` of the same value notifies nobody; a new value is a +new frame), owned by the attempt (dropped with it); never a state member (a type error and the member check refuse +it); usable as a contribution, a derive source and an `on` source. `set` inside a batch marks it there; outside, it +opens its own batch. + +## 3. `self.retryOn(...sources)` + +In any object's factory, anywhere: Chord installs a watch on each source before natives open, so a failed open still +has them; a call of any source while the object is failed (`transient` or `broken`) retries it as `ref.retry()` does +(backoff skipped, an attempt now, holders told `loading`); ignored while live or activating. Sources must live +independently of the attempt: a registry's composed value, another object's `writable` reached through a registry, +`constant`; a reference's member is a type error (it would fail with the dependency, R18 handles that). + +> **Decision needed (D2): which calls retry.** A registry value's `failed` call (compose threw) is no reason to retry +> the dependent. Proposal: only `value` calls retry; `failed` calls are ignored. + +## 4. `host.root(fn, context, options?)` + +> **Decision needed (D3): the signature and its guarantees** (design §26 #25). Proposal: `host.root(fn: (ctx: Ctx) +> => Promise, context: Context, options?: { principal?: P }): Promise`: `fn` gets a call context like a +> handler's: the host's principal (or `options.principal`, a trusted same-host caller's choice, as base code acting for +> an authenticated user), this host's incarnation, `context`'s telemetry and abort signal linked until `fn` settles +> (`withLinkedAbortSignal`), and `ctx.get`. When `fn` settles, every reference it took is released and a later +> `ctx.get` throws `released`; an aborted `fn` that does not stop keeps running, but its references are already +> released (as a handler after its caller's cancellation). `host.root` before `load` resolved parks its `ctx.get`s +> like any reference; after `host.close` it rejects `released`. No span of its own until slice 9. + +- `chord(fn, context)` is p3's helper over `host.root` (design §24); Chord ships none. + +## 5. Durable on registries (examples, p3's pattern) + +- Durable's `HarnessOptions.registry` takes a `RegistryReader` (`snapshot()`, `subscribe(listener)`); Durable gets + `registryReader(watchable)` (in its `src`, as `StoreStorage`): `snapshot()` reads the watchable's value, + `subscribe` watches it. `models` and `settings` already accept functions. +- **Durable's views from other objects** (carry-over from 6b): the object that opens Durable contributes its harness to + a registry (`HarnessRegistry`, p3's); another object reads it with `self.registry(...)` and follows a view: + `self.follow((observer, ctx) => self.registry(HarnessRegistry).value.observeConversation(id, observer, ctx))`; a + harness that ends ends the follow (R4), a new one is followed on the next attempt (`retryOn` the registry). +- Example `29-registries.ts` (a router registry with two contributing plugins, a conflict reported, a `writable` + contribution, a missing declaration healing); example `30-host-root.ts`; example `31-harness-registry.ts` (a + Durable harness on the store, its extensions from a registry, its views followed by a second object). + +## 6. Carry-over + +From `slices.md` "Slice 8": `host.root` (D3); `retryOn` (section 3); registries recomposing in the owner's flush (W4); +`host.activity` on the facet's host: not built (nothing needs it; dropped from the carry-over); Durable's views from +other objects (section 5). Left to slice 7: registries on reload (a reload replaces a contribution and a declaration +atomically; staging composes nothing, decision D1 of "Review round 1"); `accepts` of contracts and jobs. + +## 7. Tests and cost + +- One `describe` per mechanism: tokens and declarations (two declarations refused, a missing one healing), contributors + (R21's type and runtime refusals, facet and object lifetimes, one value each), the table (order by plugin, versions + adapted, refused versions reported), `compose` (in the flush, after a commit that changed a watchable contribution, + conflicts reported, a throw as a failure call and its heal), reading (`self.registry` in derives and `on`, `.value` at + use), `writable` (identity, batching, as a contribution), `retryOn` (a failed object retried by a contribution, value + calls only, sources installed before natives), `host.root` (principal, release at settle, abort, before load, after + close), `registryReader` with a Durable harness; GC for every table, writable and root scope. +- Cost guards: one compose per flush per changed table; no compose for an unchanged table; a `writable.set` of the same + value costs one comparison. From 1e1716e69a355f2ea7e7ed31d26e49437755e51e Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 17:08:18 +0000 Subject: [PATCH 089/271] docs(chord): slice 8 D1, registries served by their provider --- packages/chord/docs/next/decisions.md | 6 +++++ packages/chord/docs/next/slice-8.md | 34 +++++++++++---------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 7ef784b6e..825343ec1 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1736,6 +1736,12 @@ validation and copies the v2 kernel added around Delta. `(owner, namespace)`; so the object that opens Durable can also declare its own persisted data under the same owner name, and two harnesses can share a store under two owner names (owner: "seems like they could"); a second `StoreStorage` on the same owner is refused by its tables. "One harness per store" is no rule, only how p3 uses it. +- **Slice 8 D1: a registry's token is shared, its provider serves it (owner, 2026-10-27: "b", "yes"):** the token + `defineRegistry(name, { version })` carries identity only (bundled by every contributor, as a contract); the + defining plugin's facet provides it with `host.serve(R, { compose, accepts? })`, as it serves object types. One rule + for versioned things: a shared definition (object types, registries) keeps its adapters with the provider; a + definition of one owner (documents, tables, `json(...)` columns, jobs) keeps them in itself. Refines Q8's "one + authoritative declaration" (the serve is it) and design §17's example. - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the diff --git a/packages/chord/docs/next/slice-8.md b/packages/chord/docs/next/slice-8.md index 3bc1511b5..a30aaa885 100644 --- a/packages/chord/docs/next/slice-8.md +++ b/packages/chord/docs/next/slice-8.md @@ -15,13 +15,12 @@ is p3's base code; Chord shows it only in an example. ```ts // the defining plugin (e.g. http): the token, exported for other plugins -export const Http = defineRegistry("chord.http", { - version: 2, - accepts: { 1: (old: HttpContributionV1): HttpContribution => upgrade(old) }, - compose: (table, { report }) => buildRouter(table, report), // pure; always a value; conflicts reported -}); +export const Http = defineRegistry("chord.http", { version: 2 }); // shared: identity only export default defineFacet((host, self) => { - self.declare(Http); // the one authoritative declaration (Q8) + host.serve(Http, { // the one provider (Q8), as for object types + compose: (table, { report }) => buildRouter(table, report), // pure; always a value; conflicts reported + accepts: { 1: (old: HttpContributionV1): HttpContribution => upgrade(old) }, + }); host.serve(HttpServer, (_, self) => { const router = self.registry(Http); // a NativeWatchable of the composed value self.on(router, ({ value }, ctx) => …); @@ -47,16 +46,11 @@ await host.root(async (ctx) => (await ctx.get(Notes, { session })).add(note, ctx ## 1. Registries -- **Token:** `defineRegistry(name, { version, accepts?, compose })` (names as types, `[a-z0-9@/._~-]`): the - registry's identity (name, `C` the contribution type, `V` the composed value type) and the defining plugin's - declaration in one object; every plugin's bundled copy carries the version it was built against. - -> **Decision needed (D1): how the defining plugin declares.** Q8: one authoritative declaration, from the defining -> plugin's facet; other plugins' tokens only identify the registry. Options: (a) `self.declare(Http)` in the defining -> facet: the token carries `compose` and `accepts`, the host uses those of the declaring facet's copy only; (b) the -> token is name and types only (`defineRegistry("chord.http")`), and the facet declares `self.declare(Http, { -> version, compose, accepts })`. Proposal: (a), the shape design §17 shows: one object, and a token used for -> contributing never has its `compose` called. +- **Token and provider (D1, owner, 2026-10-27):** the token `defineRegistry(name, { version })` is shared, as a + contract: its name (as types, `[a-z0-9@/._~-]`), the contribution type `C`, the composed value type `V`, and the + version the bundling plugin was built against. The defining plugin's facet provides it with `host.serve(R, { + compose, accepts? })`, as it serves object types (adapters with the provider, never in the shared token, as + `checkAccepts` for contracts); a second serve on a host is refused; a reload replaces it (slice 7). - **Contributors** (R21): facets (by plugin name) and host-lifetime objects (`local`, empty params: by plugin name and type), at most one value per registry each; `self.contribute(R, value | Watchable | NativeWatchable)` in a facet setup @@ -71,9 +65,9 @@ await host.root(async (ctx) => (await ctx.get(Notes, { session })).add(note, ctx `report(conflict)` goes to `onError` as `registry_conflict { registry, message, plugins }`. A throwing `compose` makes the composed value fail: its watches get a `failed` call, `.value` throws the error, the next good compose heals it ("Failures are watch calls"). -- **Without a declaration** (the defining plugin is not loaded): the composed value is failed with - `unavailable/undeclared` (readers see the error as from a failed reference); contributions wait in the table; a - declaration heals it. +- **Without a provider** (the defining plugin is not loaded): the composed value is failed with + `unavailable/undeclared` (readers see the error as from a failed reference); contributions wait in the table; its + serve heals it. - **Reading:** `self.registry(R)` in any object's factory returns a `NativeWatchable` (a derive source, an `on` source, a `retryOn` source); methods and routines read `.value` at use; `host.registry(R)` for code outside objects. - **Helpers:** `keyed(table, entriesOf)` (a map, a duplicate key reported, the first by plugin name wins) and @@ -136,7 +130,7 @@ atomically; staging composes nothing, decision D1 of "Review round 1"); `accepts ## 7. Tests and cost -- One `describe` per mechanism: tokens and declarations (two declarations refused, a missing one healing), contributors +- One `describe` per mechanism: tokens and providers (two serves refused, a missing one healing), contributors (R21's type and runtime refusals, facet and object lifetimes, one value each), the table (order by plugin, versions adapted, refused versions reported), `compose` (in the flush, after a commit that changed a watchable contribution, conflicts reported, a throw as a failure call and its heal), reading (`self.registry` in derives and `on`, `.value` at From 9cef0db2d61262395e5b9b9a33126930c20cb4cb Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 17:15:50 +0000 Subject: [PATCH 090/271] docs(chord): slice 8 registry failures --- packages/chord/docs/next/decisions.md | 5 +++++ packages/chord/docs/next/slice-8.md | 10 +++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 825343ec1..3d00c73e7 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1742,6 +1742,11 @@ validation and copies the v2 kernel added around Delta. for versioned things: a shared definition (object types, registries) keeps its adapters with the provider; a definition of one owner (documents, tables, `json(...)` columns, jobs) keeps them in itself. Refines Q8's "one authoritative declaration" (the serve is it) and design §17's example. +- **Slice 8: registry failures (owner, 2026-10-27: "ok", "agree"):** contributions never fail a registry: a + watchable contribution that fails is left out of the table and reported (`contribution_failed`), back once healed; + a throwing `compose` (the owner's bug) is reported (`registry_failed`), its watches get a `failed` call, and `.value` + keeps the last good composition (before any, it throws). Derives keep `.value` throwing while failed: state must match + its sources, an extension point must stay available. - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the diff --git a/packages/chord/docs/next/slice-8.md b/packages/chord/docs/next/slice-8.md index a30aaa885..f18d456ac 100644 --- a/packages/chord/docs/next/slice-8.md +++ b/packages/chord/docs/next/slice-8.md @@ -62,9 +62,13 @@ await host.root(async (ctx) => (await ctx.get(Notes, { session })).add(note, ctx `contribution_refused { registry, plugin, version }`). - **`compose(table, { report })`:** pure, runs whenever the table changes (a contributor goes live or ends, a contribution that is a watchable changes), in the owner's flush, never inside another owner's commit step (W4); - `report(conflict)` goes to `onError` as `registry_conflict { registry, message, plugins }`. A throwing `compose` - makes the composed value fail: its watches get a `failed` call, `.value` throws the error, the next good compose heals - it ("Failures are watch calls"). + `report(conflict)` goes to `onError` as `registry_conflict { registry, message, plugins }`. +- **Failures (owner, 2026-10-27):** contributions never fail the registry. A contribution that is a watchable and fails + is left out of the table and reported (`contribution_failed { registry, plugin, error }`), the table recomposes without + it, and it is back once it heals. A `compose` that throws (the owner's bug) is reported (`registry_failed { registry, + error }`), the composed value's watches get a `failed` call, and `.value` keeps the last good composition (an extension + point stays available; derives keep their own rule, `.value` throwing while failed); the next good composition + replaces it. Before any good composition `.value` throws the error. - **Without a provider** (the defining plugin is not loaded): the composed value is failed with `unavailable/undeclared` (readers see the error as from a failed reference); contributions wait in the table; its serve heals it. From 07a76f1ef0bfa30b4f4fd2cdfc8d4ff355d54ac9 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 17:20:43 +0000 Subject: [PATCH 091/271] docs(chord): slice 8 D2, D3; spec decided --- packages/chord/docs/next/decisions.md | 6 ++++++ packages/chord/docs/next/design.md | 9 ++++----- packages/chord/docs/next/slice-8.md | 23 ++++++++++++----------- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 3d00c73e7..b7f2464ed 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1747,6 +1747,12 @@ validation and copies the v2 kernel added around Delta. a throwing `compose` (the owner's bug) is reported (`registry_failed`), its watches get a `failed` call, and `.value` keeps the last good composition (before any, it throws). Derives keep `.value` throwing while failed: state must match its sources, an extension point must stay available. +- **Slice 8 D2 and D3 (owner, 2026-10-27: "yes"):** D2: a `retryOn` source retries a failed object only on a `value` + call; a `failed` call does nothing. D3: `host.root(fn, context, { principal? })` gives `fn` a handler's call + context (the host's principal or the given one, this host's incarnation, `context`'s telemetry and signal linked + until `fn` settles, `ctx.get`); at settle or abort the context is sealed (`ctx.get` rejects `released`, its references + released, their calls reject), so an uncooperative `fn` fails at its next use of Chord; before `load` its gets park; + after `host.close` it rejects `released`. `chord(fn, context)` is p3's helper. Closes design §26 #25. - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index cd55b3bfb..de9324a66 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -1494,13 +1494,13 @@ export default defineFacet((host, self) => { // `self` (contributions) a ## 17. Registries ```ts -export const Http = defineRegistry("chord.http", { - version: 2, +export const Http = defineRegistry("chord.http", { version: 2 }); // shared: identity only +host.serve(Http, { // the defining plugin's facet: the one provider (slice 8 D1) + compose: (table, { report }) => buildRouter(table, report), // pure; always delivers a value; reports conflicts accepts: { 1: (old: HttpContributionV1): HttpContribution => upgrade(old) }, - compose: (table, { report }) => buildRouter(table, report), // pure; always delivers a value; reports conflicts }); self.contribute(Http, { routes: { … }, wraps: [ … ] }); -const router = host.registry(Http).value; // live +const router = self.registry(Http).value; // live (host.registry(Http) outside objects) ``` - Each contributor (a facet or an object) contributes **at most one value per registry**; the table is an immutable, @@ -2019,7 +2019,6 @@ this document): | 14 | project plugins | 21 | own pass over `plugin-loading.md` | | 16 | a Durable Object profile | 24 | when a DO target becomes concrete | | 24 | the rollout install timeout (60 s) and a per-type drain override: host options or fixed (the routine handover uses `closeTimeout`; a failing job `finish` backs off 1-60 s and blocks after five failures, decided) | 3 | with slice 7 | -| 25 | `host.root` signature (`root(fn, context, { principal? })` proposed) and what an aborted, uncooperative `fn` means for its references (match the handler and `CallContext` rules) | 3, 24 | slice 8 | | 26 | how a migration commit reaches consumers (`[["r", value]]` per changed member, or a fresh `bound`; adapted by `accepts` for older consumers) | 21, 22 | slice 7 | | 28 | Astra's circuit breaker (a hard transport-buffer cap plus a stall deadline); the heartbeat is built, the cap was never decided | 3 | if a real case needs it | | 29 | round-3 cut C11 (`ctx.trace`); C3 and C4 are decided (§14, §15) | 8 | slice 9 | diff --git a/packages/chord/docs/next/slice-8.md b/packages/chord/docs/next/slice-8.md index f18d456ac..303fe2410 100644 --- a/packages/chord/docs/next/slice-8.md +++ b/packages/chord/docs/next/slice-8.md @@ -1,6 +1,6 @@ # Slice 8: registries and `host.root` (spec) -Status: **draft, open points marked "Decision needed"** (2026-10-27), written by the redesign session. Built in +Status: **decided, ready for the spec reviews** (2026-10-27; D1-D3 and the registry failures decided by the owner), written by the redesign session. Built in parallel with 6b (mostly new files). The single source for building 8, together with `design.md` (§7, §8, §11, §16, §17, §18, §24 read in full), the decision log `decisions.md` (R21, R23, R25, Q8, D1 of "Review round 1", "Registries work like Durable's", "Changing contributions, `self.writable`", W4, "Failures are watch calls", "Slice 8 is @@ -96,19 +96,20 @@ has them; a call of any source while the object is failed (`transient` or `broke independently of the attempt: a registry's composed value, another object's `writable` reached through a registry, `constant`; a reference's member is a type error (it would fail with the dependency, R18 handles that). -> **Decision needed (D2): which calls retry.** A registry value's `failed` call (compose threw) is no reason to retry -> the dependent. Proposal: only `value` calls retry; `failed` calls are ignored. +**Which calls retry (D2, owner, 2026-10-27):** only `value` calls: a new value of a source wakes a failed object; a +`failed` call (a throwing `compose`) does nothing, its next good composition retries. ## 4. `host.root(fn, context, options?)` -> **Decision needed (D3): the signature and its guarantees** (design §26 #25). Proposal: `host.root(fn: (ctx: Ctx) -> => Promise, context: Context, options?: { principal?: P }): Promise`: `fn` gets a call context like a -> handler's: the host's principal (or `options.principal`, a trusted same-host caller's choice, as base code acting for -> an authenticated user), this host's incarnation, `context`'s telemetry and abort signal linked until `fn` settles -> (`withLinkedAbortSignal`), and `ctx.get`. When `fn` settles, every reference it took is released and a later -> `ctx.get` throws `released`; an aborted `fn` that does not stop keeps running, but its references are already -> released (as a handler after its caller's cancellation). `host.root` before `load` resolved parks its `ctx.get`s -> like any reference; after `host.close` it rejects `released`. No span of its own until slice 9. +**Signature and guarantees (D3, owner, 2026-10-27; design §26 #25):** `host.root(fn: (ctx: Ctx) => Promise, +context: Context, options?: { principal?: P }): Promise`. `fn` gets a call context like a handler's: the host's +principal (or `options.principal`, a trusted same-host caller's choice, as base code acting for an authenticated +user), this host's incarnation, `context`'s telemetry and abort signal linked until `fn` settles +(`withLinkedAbortSignal`), and `ctx.get`. When `fn` settles or `context` aborts, the call context is sealed: +`ctx.abortSignal` aborted, `ctx.get` rejects `released`, every reference it took released (their calls reject +`released`, their watches end), so an uncooperative `fn` fails at its next use of Chord, as a handler after its caller +cancelled; work outside Chord runs on until then. `ctx.get` before `load` resolved parks like any reference; +`host.root` after `host.close` rejects `released`. No span of its own until slice 9. - `chord(fn, context)` is p3's helper over `host.root` (design §24); Chord ships none. From cf9042b48a1eca0b881077bc7232f9a98a2cfb21 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 17:35:55 +0000 Subject: [PATCH 092/271] docs(chord): slice 8 S1, a Session close fails Durable's observers --- packages/chord/docs/next/decisions.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index b7f2464ed..8cc77d3f3 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1753,6 +1753,11 @@ validation and copies the v2 kernel added around Delta. until `fn` settles, `ctx.get`); at settle or abort the context is sealed (`ctx.get` rejects `released`, its references released, their calls reject), so an uncooperative `fn` fails at its next use of Chord; before `load` its gets park; after `host.close` it rejects `released`. `chord(fn, context)` is p3's helper. Closes design §26 #25. +- **Slice 8 S1: a Session close fails Durable's observers (owner, 2026-10-27: "a"; spec review, all four reviewers):** + Durable ends an observer `failed` (error "session closed") when its Session closes, and `ended` only when the + observed owner ended (a task document's task); refines R4 (Durable change, upstream). So a follower of a harness + reached through a registry fails `transient` when that harness closes, and its retry follows the new one. A + `retryOn` wake that arrives while the object activates is kept and retries it at once if that attempt fails. - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the From df9275d4347f187610804c71d52e7c46bfd14848 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 17:39:33 +0000 Subject: [PATCH 093/271] docs(chord): slice 8 S2, registry readers of an older version --- packages/chord/docs/next/decisions.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 8cc77d3f3..f200cdace 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1758,6 +1758,11 @@ validation and copies the v2 kernel added around Delta. observed owner ended (a task document's task); refines R4 (Durable change, upstream). So a follower of a harness reached through a registry fails `transient` when that harness closes, and its retry follows the new one. A `retryOn` wake that arrives while the object activates is kept and retries it at once if that attempt fails. +- **Slice 8 S2: registry readers of an older version (owner, 2026-10-27: "a"):** as contract state, the provider maps + back: `host.serve(R, { compose, accepts: { [old]: { contribution?: (old) => current, value?: (current) => old } } })`; + a reader bundling an older token gets the composed value mapped to its version (computed once per composition and + version, only while such a reader exists); without a `value` mapping, or for a reader newer than the provider, its + value is failed `contract_mismatch` (reported). - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the From b0eb8ace190350da5466b2f86702d462ffc99597 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 17:43:57 +0000 Subject: [PATCH 094/271] docs(chord): slice 6b entry_heads table --- packages/chord/docs/next/decisions.md | 3 ++- packages/chord/docs/next/slice-6b.md | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index f200cdace..5440aa7bf 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1637,7 +1637,8 @@ validation and copies the v2 kernel added around Delta. purpose of this slice"):** `StoreStorage` declares conversations, entries, tasks, submissions and `record_ids` as `defineTable`s, writes them with `put` (Durable replaces complete records), and reads them with the store's queries; the outbox, export, import, the applier and later `evolveTable` versions handle them as every table. The partial - head index becomes an index on `(conversation_id, has_head, id)`; indexed strings that must stay lossless keep + head index becomes a table `entry_heads` (one row per entry with a head; build, 2026-10-27, faster than an index on + `(conversation_id, has_head, id)`); indexed strings that must stay lossless keep Durable's JSON encoding. - **Slice 6b D3: one id counter per store (owner, 2026-10-27: "a"):** the store's counter (`next_document_id`, renamed `next_id`; the archive manifest's `nextDocumentId` becomes `nextId`) numbers every id in the store: Durable's diff --git a/packages/chord/docs/next/slice-6b.md b/packages/chord/docs/next/slice-6b.md index 32e1eb69b..837a4ceff 100644 --- a/packages/chord/docs/next/slice-6b.md +++ b/packages/chord/docs/next/slice-6b.md @@ -174,13 +174,16 @@ app owns it). Every row carries the record's indexed columns and the whole record as `json(unknown(), { version: 1 })`; indexed strings that must stay lossless (`kind`, `request_id`) hold Durable's JSON encoding; index columns are non-`opt`, so absent values take sentinels that no encoded value can be (`0` for an absent owner id, ids start at 2; `""` for an -absent request id, JSON-encoded strings start with `"`). +absent request id, JSON-encoded strings start with `"`). SQLite's partial head index is the table `entry_heads` +(builder, 2026-10-27: an index on `(conversation_id, has_head, id)` cost about 5.5 µs per commit; the table is written +only for entries with a head and is faster on both paths). | Table | Key | Columns | Indexes | | --- | --- | --- | --- | | `record_ids` | `id` | `record_type str` | | | `conversations` | `id` | `owner_conversation_id int`, `owner_task_id int`, `record` | `(owner_conversation_id, id)`, `(owner_task_id, id)` | -| `entries` | `id` | `conversation_id int`, `has_head int` (0/1), `commit_seq int` (stamped), `record` | `(conversation_id, id)`, `(conversation_id, has_head, id)` | +| `entries` | `id` | `conversation_id int`, `commit_seq int` (stamped), `record` | `(conversation_id, id)` | +| `entry_heads` | `id` | `conversation_id int` (one row per entry with a `head`, written with it) | `(conversation_id, id)` | | `tasks` | `id` | `conversation_id int`, `kind str`, `status str`, `abort_requested int`, `background int`, `record` | `(status, id)`, `(conversation_id, id)`, `(kind, id)`, `(abort_requested, id)`, `(background, id)` | | `submissions` | `id` | `conversation_id int`, `request_id str`, `status str`, `record` | `(conversation_id, request_id)`, `(conversation_id, id)`, `(status, id)` | From 677e8067b334692439d29dc3d73bf533793c63df Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 17:46:55 +0000 Subject: [PATCH 095/271] docs(chord): slice 6b revision part and examples as built --- packages/chord/docs/next/slice-6b.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/chord/docs/next/slice-6b.md b/packages/chord/docs/next/slice-6b.md index 837a4ceff..78f3a71b3 100644 --- a/packages/chord/docs/next/slice-6b.md +++ b/packages/chord/docs/next/slice-6b.md @@ -136,7 +136,8 @@ Every item below changes 6a's code; nothing is deployed, so the schema and the a ### 1.6 The archive and batch apply - **Format 1 gains** (nothing deployed): `document.create` carries `history`, `retiredAt?` and `meta`; a - `document.revision { id, seq, kind, version, content }` part per kept revision, in seq order after its create; for + `document.revision { id, seq, content }` (`content` a `DocumentContent`: `{ kind: "base", version, value }` or `{ kind: + "delta", version, ops }`) part per kept revision, in seq order after its create; for live `none` and `latest` records only the current base and its deltas are exported (as 6a); a retired `latest` record is exported as its record alone (no base: its content was dropped at retirement); for `rewindable` every revision, retired records included. Import checks: revisions in seq order, a base first, `seq <= lastSeq`, lifetimes @@ -319,8 +320,9 @@ does nothing. joined and stored, abort racing an outcome, deferred state merged, flushed at the outcome and at close, pinned past the retention); GC (closed `StoreStorage`, ended follow sources, cancelled abort waiters, deferred timers, settled and dropped runs). -- **Examples:** `26-durable-on-the-store.ts` (a worker with Durable's Session and a Chord store object on one store, - exporting the archive), `27-durable-views.ts`, `28-persisted-jobs.ts`. +- **Examples:** Durable's `test/examples/35-durable-on-the-store.ts` (a worker with Durable's Session and a Chord store + object on one store, exporting the archive) and `36-durable-views.ts` (Chord's tests import nothing of Durable); + Chord's `28-persisted-jobs.ts`. ## Carry-over taken From 376ccf6e89609ce934bb1c64841a2cdf9a691291 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 17:48:03 +0000 Subject: [PATCH 096/271] docs(chord): slice 8 revised: self.registry is a dependency; S1-S3; review gaps --- packages/chord/docs/next/slice-8.md | 93 ++++++++++++++++++++--------- 1 file changed, 66 insertions(+), 27 deletions(-) diff --git a/packages/chord/docs/next/slice-8.md b/packages/chord/docs/next/slice-8.md index 303fe2410..881fbca47 100644 --- a/packages/chord/docs/next/slice-8.md +++ b/packages/chord/docs/next/slice-8.md @@ -1,6 +1,8 @@ # Slice 8: registries and `host.root` (spec) -Status: **decided, ready for the spec reviews** (2026-10-27; D1-D3 and the registry failures decided by the owner), written by the redesign session. Built in +Status: **revised after the spec review** (2026-10-27; D1-D3, the registry failures, S1-S3 decided by the owner; the +four reviews `/tmp/s8-review-{astra,fable,opus,sol}.md` and a check against the design's rules found `self.registry` +wrongly specified as a plain read: it is a declared dependency, as `self.get`), written by the redesign session. Built in parallel with 6b (mostly new files). The single source for building 8, together with `design.md` (§7, §8, §11, §16, §17, §18, §24 read in full), the decision log `decisions.md` (R21, R23, R25, Q8, D1 of "Review round 1", "Registries work like Durable's", "Changing contributions, `self.writable`", W4, "Failures are watch calls", "Slice 8 is @@ -60,20 +62,37 @@ await host.root(async (ctx) => (await ctx.get(Notes, { session })).add(note, ctx order); a contribution carries its token's version: an older one is adapted once with the declaration's `accepts` when the table is built, an older one without an adapter or a newer one is left out and reported (`HostError` `contribution_refused { registry, plugin, version }`). -- **`compose(table, { report })`:** pure, runs whenever the table changes (a contributor goes live or ends, a - contribution that is a watchable changes), in the owner's flush, never inside another owner's commit step (W4); - `report(conflict)` goes to `onError` as `registry_conflict { registry, message, plugins }`. +- **Contributions:** a plain value, or a Chord watchable (a `writable`, a derive, a document member, another + registry's value), followed as a dependent of the table (its change recomposes); a foreign watchable (a Durable watch) + is refused at `contribute` (declaration policy). The table's entry is `{ plugin, type?, version, value }` (`type` for + an object contributor; served types carry their plugin's name). +- **`compose(table, { report })`:** pure, lazy as a derive: the registry's value is marked when its table changes (a + contributor goes live or ends, a watchable contribution changes) and composed when read or, while watched, in the + owner's flush (W4: never by another owner's adoption itself; the flush after a commit step is where it runs). The first + composition happens at the serve: the facets' setups of one `load` serve and contribute, then every served registry + composes once in one batch at the end of the setups (an empty table composes too), so a served registry always has + a value before any object activates. `report(conflict)` goes to `onError` as `registry_conflict { registry, message, + plugins }`, once per conflict episode. - **Failures (owner, 2026-10-27):** contributions never fail the registry. A contribution that is a watchable and fails is left out of the table and reported (`contribution_failed { registry, plugin, error }`), the table recomposes without it, and it is back once it heals. A `compose` that throws (the owner's bug) is reported (`registry_failed { registry, error }`), the composed value's watches get a `failed` call, and `.value` keeps the last good composition (an extension point stays available; derives keep their own rule, `.value` throwing while failed); the next good composition replaces it. Before any good composition `.value` throws the error. -- **Without a provider** (the defining plugin is not loaded): the composed value is failed with - `unavailable/undeclared` (readers see the error as from a failed reference); contributions wait in the table; its - serve heals it. -- **Reading:** `self.registry(R)` in any object's factory returns a `NativeWatchable` (a derive source, an `on` - source, a `retryOn` source); methods and routines read `.value` at use; `host.registry(R)` for code outside objects. +- **Without a provider** (the defining plugin is not loaded): the registry has no value; contributions wait in the + table; its serve composes it. +- **Reading is a dependency (S3, owner, 2026-10-27: "isn't self.registry meant to wait like self.get?"):** + `self.registry(R)` in a factory declares the registry as a dependency of the attempt, as `self.get`: activation waits + until it has a value (bounded by `readyTimeout`); without one in time the object fails `transient` with + `unavailable/undeclared` as the dependency's cause and is retried the moment the registry gets its first value + (R18). It returns a `RegistryValue` (a native source: a derive source, an `on` source, a `retryOn` source; never a + state member); methods and routines read `.value` at use. Once the object is live, a later failure of the registry + (a throwing `compose`) changes nothing for it: `.value` keeps the last good composition. Code outside objects: + `await host.registry(R, ctx)` waits the same way. +- **Older readers (S2, owner, 2026-10-27):** a reader whose bundled token has an older version than the provider's gets + the composed value mapped back with `accepts[old].value` (computed once per composition and version, while such a + reader exists); without that mapping, or for a reader newer than the provider, its dependency fails + `contract_mismatch` (reported). `accepts: { [old]: { contribution?: (old) => current, value?: (current) => old } }`. - **Helpers:** `keyed(table, entriesOf)` (a map, a duplicate key reported, the first by plugin name wins) and `ordered(table, itemsOf, orderOf?)`. - **Use sites guard contributions** (Q8): Chord's helpers keep each entry's plugin name; the guide shows catching a @@ -81,20 +100,24 @@ await host.root(async (ctx) => (await ctx.get(Notes, { session })).add(note, ctx - **Registries carry natives** (design §17): a contribution may be any in-process value; an entry lives exactly as long as its contributor. -## 2. `self.writable(initial)` +## 2. `self.writable(initial)` and native sources A `NativeWatchable` with `set(value)` (identity compared: `set` of the same value notifies nobody; a new value is a -new frame), owned by the attempt (dropped with it); never a state member (a type error and the member check refuse -it); usable as a contribution, a derive source and an `on` source. `set` inside a batch marks it there; outside, it -opens its own batch. +new frame), owned by the attempt (dropped with it; `set` after the attempt's seal throws `released`); never a state +member; usable as a contribution, a derive source and an `on` source. `set` inside a batch marks it there; outside, it +opens its own batch. Both `writable` and `RegistryValue` are native sources: one base (`NativeSource`) the batch marks +and derives depend on; `self.derive` takes JSON and native sources (its type and the source resolver widen), while the +member check still refuses a native as a state member. ## 3. `self.retryOn(...sources)` -In any object's factory, anywhere: Chord installs a watch on each source before natives open, so a failed open still -has them; a call of any source while the object is failed (`transient` or `broken`) retries it as `ref.retry()` does -(backoff skipped, an attempt now, holders told `loading`); ignored while live or activating. Sources must live -independently of the attempt: a registry's composed value, another object's `writable` reached through a registry, -`constant`; a reference's member is a type error (it would fail with the dependency, R18 handles that). +In any object's factory, anywhere: Chord follows each source with watches owned by the object (under its context, +replaced by each attempt's declaration, stopped at the object's end; GC-tested), installed right after the factory +returns, before natives open, so a failed open still has them; a `value` call of a source while the object is failed +(`transient` or `broken`) retries it as `ref.retry()` does (backoff skipped, an attempt now, holders told `loading`); +a call while it activates is remembered and retries it at once if that attempt fails (S1); ignored while live. Sources +are those that live independently of the attempt: a `RegistryValue`, `constant`; anything else (a reference's member, +a `writable` of the same attempt) is refused at runtime (declaration policy; TypeScript cannot tell them apart). **Which calls retry (D2, owner, 2026-10-27):** only `value` calls: a new value of a source wakes a failed object; a `failed` call (a throwing `compose`) does nothing, its next good composition retries. @@ -109,19 +132,31 @@ user), this host's incarnation, `context`'s telemetry and abort signal linked un `ctx.abortSignal` aborted, `ctx.get` rejects `released`, every reference it took released (their calls reject `released`, their watches end), so an uncooperative `fn` fails at its next use of Chord, as a handler after its caller cancelled; work outside Chord runs on until then. `ctx.get` before `load` resolved parks like any reference; -`host.root` after `host.close` rejects `released`. No span of its own until slice 9. +`host.close` seals every open root scope (the host keeps them), so an `fn` waiting in `sleep(ms, ctx)` ends; `host.root` +after `host.close` rejects `released`. No span of its own until slice 9. - `chord(fn, context)` is p3's helper over `host.root` (design §24); Chord ships none. ## 5. Durable on registries (examples, p3's pattern) -- Durable's `HarnessOptions.registry` takes a `RegistryReader` (`snapshot()`, `subscribe(listener)`); Durable gets - `registryReader(watchable)` (in its `src`, as `StoreStorage`): `snapshot()` reads the watchable's value, - `subscribe` watches it. `models` and `settings` already accept functions. +- Durable's `HarnessOptions.registry` takes a `RegistryReader` (`snapshot()`, `subscribe(listener)`, the listener + called synchronously after every publication). A `RegistryValue` offers a synchronous observer of its compositions + (`observe(observer)`, as Durable's `observeXXX`: `advance` inside the flush that composed, record only); Durable's + `registryReader(value)` (in its `src`) reads `.value` in `snapshot()` and forwards `observe` to `subscribe`. The + harness object declares the extensions registry with `self.registry(...)`, so it activates (and opens Durable) only + once the registry has a value. p3's `compose` builds Durable's snapshot without throwing (a duplicate task name is + reported and the later extension left out). - **Durable's views from other objects** (carry-over from 6b): the object that opens Durable contributes its harness to - a registry (`HarnessRegistry`, p3's); another object reads it with `self.registry(...)` and follows a view: - `self.follow((observer, ctx) => self.registry(HarnessRegistry).value.observeConversation(id, observer, ctx))`; a - harness that ends ends the follow (R4), a new one is followed on the next attempt (`retryOn` the registry). + a registry (`HarnessRegistry`, p3's); another object declares it in its factory and follows a view through its value: + + ```ts + const harness = self.registry(HarnessRegistry); // a dependency: activation waits for it + const view = self.follow((observer, ctx) => harness.value.observeConversation(id, observer, ctx)); + ``` + + When that harness closes, Durable ends the observer `failed` (S1, a Durable change in this slice: a Session close + ends observers `failed`, `ended` only when the observed owner ended), so the follower fails `transient` and its retry + follows the new harness. - Example `29-registries.ts` (a router registry with two contributing plugins, a conflict reported, a `writable` contribution, a missing declaration healing); example `30-host-root.ts`; example `31-harness-registry.ts` (a Durable harness on the store, its extensions from a registry, its views followed by a second object). @@ -130,8 +165,12 @@ cancelled; work outside Chord runs on until then. `ctx.get` before `load` resolv From `slices.md` "Slice 8": `host.root` (D3); `retryOn` (section 3); registries recomposing in the owner's flush (W4); `host.activity` on the facet's host: not built (nothing needs it; dropped from the carry-over); Durable's views from -other objects (section 5). Left to slice 7: registries on reload (a reload replaces a contribution and a declaration -atomically; staging composes nothing, decision D1 of "Review round 1"); `accepts` of contracts and jobs. +other objects (section 5). Left to slice 7: registries on reload (a reload replaces a contribution and a provider +atomically; staging composes nothing, decision D1 of "Review round 1"); `accepts` of contracts and jobs; a `writable` +across a pure swap; older registry readers across a reload of the provider. + +**New names** (design §3, §5, the pinned reason list): the reason `unavailable/undeclared`; `HostError` kinds +`registry_conflict`, `registry_failed`, `contribution_failed`, `contribution_refused`, each reported once per episode. ## 7. Tests and cost From 5f75cd77f814e8448c9ac9f0e75b90b2937bdd01 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 17:48:15 +0000 Subject: [PATCH 097/271] docs(chord): slice 8 S3, self.registry is a dependency --- packages/chord/docs/next/decisions.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 5440aa7bf..9151ef419 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1764,6 +1764,11 @@ validation and copies the v2 kernel added around Delta. a reader bundling an older token gets the composed value mapped to its version (computed once per composition and version, only while such a reader exists); without a `value` mapping, or for a reader newer than the provider, its value is failed `contract_mismatch` (reported). +- **Slice 8 S3: `self.registry` is a dependency, as `self.get` (owner, 2026-10-27: "isn't self.registry meant to wait + like self.get?"; my spec had it as a plain read, a mistake against design §6, §7 and §17):** activation waits until + the registry has a value (served and composed once; an empty table composes too, at the serve), bounded by + `readyTimeout`; without one in time the object fails `transient` (`unavailable/undeclared`) and is retried when the + registry gets its first value (R18); once live, a later failure of the registry changes nothing (last good value). - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the From 02a2b2532d8f6ea7b26da578a6844a1fcfd50e03 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 17:49:48 +0000 Subject: [PATCH 098/271] docs(chord): slice 8 accepts adapters guarded; example shape --- packages/chord/docs/next/slice-8.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/chord/docs/next/slice-8.md b/packages/chord/docs/next/slice-8.md index 881fbca47..bc99fa89b 100644 --- a/packages/chord/docs/next/slice-8.md +++ b/packages/chord/docs/next/slice-8.md @@ -21,7 +21,7 @@ export const Http = defineRegistry("chord.http", { ver export default defineFacet((host, self) => { host.serve(Http, { // the one provider (Q8), as for object types compose: (table, { report }) => buildRouter(table, report), // pure; always a value; conflicts reported - accepts: { 1: (old: HttpContributionV1): HttpContribution => upgrade(old) }, + accepts: { 1: { contribution: (old: HttpContributionV1) => upgrade(old), value: (router) => routerV1(router) } }, }); host.serve(HttpServer, (_, self) => { const router = self.registry(Http); // a NativeWatchable of the composed value @@ -93,6 +93,10 @@ await host.root(async (ctx) => (await ctx.get(Notes, { session })).add(note, ctx the composed value mapped back with `accepts[old].value` (computed once per composition and version, while such a reader exists); without that mapping, or for a reader newer than the provider, its dependency fails `contract_mismatch` (reported). `accepts: { [old]: { contribution?: (old) => current, value?: (current) => old } }`. + Adapters are the provider's code, guarded as `compose`: a `contribution` adapter that throws leaves that contribution + out (reported `contribution_failed`); a `value` adapter that throws fails only that version's value, by the rules of a + throwing `compose` (its readers keep the last good mapped value; before any, a reader of that version waiting at + activation fails `transient`, retried when a mapped value arrives); current readers are never affected. - **Helpers:** `keyed(table, entriesOf)` (a map, a duplicate key reported, the first by plugin name wins) and `ordered(table, itemsOf, orderOf?)`. - **Use sites guard contributions** (Q8): Chord's helpers keep each entry's plugin name; the guide shows catching a From eb9d4434627ea56697bc1b47e94cdef42fdf3817 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 17:51:28 +0000 Subject: [PATCH 099/271] =?UTF-8?q?docs(chord):=20slice=208=20review=202?= =?UTF-8?q?=20fixes;=20design=20=C2=A717=20and=20slices=20folded?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/chord/docs/next/design.md | 8 +++++++- packages/chord/docs/next/slice-8.md | 23 ++++++++++++++--------- packages/chord/docs/next/slices.md | 12 +++--------- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index de9324a66..a474aa829 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -1497,7 +1497,7 @@ export default defineFacet((host, self) => { // `self` (contributions) a export const Http = defineRegistry("chord.http", { version: 2 }); // shared: identity only host.serve(Http, { // the defining plugin's facet: the one provider (slice 8 D1) compose: (table, { report }) => buildRouter(table, report), // pure; always delivers a value; reports conflicts - accepts: { 1: (old: HttpContributionV1): HttpContribution => upgrade(old) }, + accepts: { 1: { contribution: (old: HttpContributionV1) => upgrade(old), value: (router) => routerV1(router) } }, }); self.contribute(Http, { routes: { … }, wraps: [ … ] }); const router = self.registry(Http).value; // live (host.registry(Http) outside objects) @@ -1535,6 +1535,12 @@ const router = self.registry(Http).value; // live (host.registry(Http) outsi use site catches synchronous throws and rejections when it calls an entry, reports naming the contributor (the composed table keeps each entry's plugin name), and continues with the other entries. Durable already does this for extensions (`extension_error`; a throwing tool becomes an error result); `compose` reports through `report`. +- **Slice 8's rules** (decisions "Slice 8 D1" to "Slice 8 S3", `slice-8.md`): the token is shared, the provider serves it + (`host.serve(R, { compose, accepts })`); `self.registry(R)` in a factory is a declared dependency (activation waits + for a value; with none after `load`, the reader fails `transient` and is retried when one arrives); contributions + never fail a registry (a failed one is left out and reported); a throwing `compose` keeps the last good composition; + older readers get the value mapped back (`accepts[old].value`); `retryOn` retries on value calls only; `host.root(fn, + context, { principal? })` seals its context at settle, abort and `host.close`. - **Registries may carry natives.** An entry lives exactly as long as its contributor; consumers look up at each use or follow the composed value and keep nothing from an entry beyond a use. - **Contributions that change (owner, 2026-10-19; W4):** registries are host-local, so contributions are usually diff --git a/packages/chord/docs/next/slice-8.md b/packages/chord/docs/next/slice-8.md index bc99fa89b..090ee3bbf 100644 --- a/packages/chord/docs/next/slice-8.md +++ b/packages/chord/docs/next/slice-8.md @@ -62,8 +62,8 @@ await host.root(async (ctx) => (await ctx.get(Notes, { session })).add(note, ctx order); a contribution carries its token's version: an older one is adapted once with the declaration's `accepts` when the table is built, an older one without an adapter or a newer one is left out and reported (`HostError` `contribution_refused { registry, plugin, version }`). -- **Contributions:** a plain value, or a Chord watchable (a `writable`, a derive, a document member, another - registry's value), followed as a dependent of the table (its change recomposes); a foreign watchable (a Durable watch) +- **Contributions:** a plain value, or a Chord watchable (a `writable`, a derive, a document member; another + registry's value too, a cycle of registries reading each other refused when the second one serves or contributes), followed as a dependent of the table (its change recomposes); a foreign watchable (a Durable watch) is refused at `contribute` (declaration policy). The table's entry is `{ plugin, type?, version, value }` (`type` for an object contributor; served types carry their plugin's name). - **`compose(table, { report })`:** pure, lazy as a derive: the registry's value is marked when its table changes (a @@ -83,9 +83,11 @@ await host.root(async (ctx) => (await ctx.get(Notes, { session })).add(note, ctx table; its serve composes it. - **Reading is a dependency (S3, owner, 2026-10-27: "isn't self.registry meant to wait like self.get?"):** `self.registry(R)` in a factory declares the registry as a dependency of the attempt, as `self.get`: activation waits - until it has a value (bounded by `readyTimeout`); without one in time the object fails `transient` with - `unavailable/undeclared` as the dependency's cause and is retried the moment the registry gets its first value - (R18). It returns a `RegistryValue` (a native source: a derive source, an `on` source, a `retryOn` source; never a + until it has a value; once `load`'s setups ran, a registry with no value cannot get one before a reload (a provider + comes only with a plugin), so a reader then fails at once `transient` with `unavailable/undeclared` as the + dependency's cause (as `self.get` of a type not served here fails at once, and the start barrier waits for no + contribution) and is retried the moment the registry gets its first value (R18); a first `compose` that throws fails + its readers the same way. It returns a `RegistryValue` (a native source: a derive source, an `on` source, a `retryOn` source; never a state member); methods and routines read `.value` at use. Once the object is live, a later failure of the registry (a throwing `compose`) changes nothing for it: `.value` keeps the last good composition. Code outside objects: `await host.registry(R, ctx)` waits the same way. @@ -145,7 +147,8 @@ after `host.close` rejects `released`. No span of its own until slice 9. - Durable's `HarnessOptions.registry` takes a `RegistryReader` (`snapshot()`, `subscribe(listener)`, the listener called synchronously after every publication). A `RegistryValue` offers a synchronous observer of its compositions - (`observe(observer)`, as Durable's `observeXXX`: `advance` inside the flush that composed, record only); Durable's + (`observe(observer, ctx)`, as Durable's `observeXXX`: `advance` inside the flush that composed, record only; an + attached observer counts as watching, so the lazy compose runs for it); Durable's `registryReader(value)` (in its `src`) reads `.value` in `snapshot()` and forwards `observe` to `subscribe`. The harness object declares the extensions registry with `self.registry(...)`, so it activates (and opens Durable) only once the registry has a value. p3's `compose` builds Durable's snapshot without throwing (a duplicate task name is @@ -155,15 +158,17 @@ after `host.close` rejects `released`. No span of its own until slice 9. ```ts const harness = self.registry(HarnessRegistry); // a dependency: activation waits for it + self.retryOn(harness); // the new harness's contribution wakes the failed follower const view = self.follow((observer, ctx) => harness.value.observeConversation(id, observer, ctx)); ``` When that harness closes, Durable ends the observer `failed` (S1, a Durable change in this slice: a Session close ends observers `failed`, `ended` only when the observed owner ended), so the follower fails `transient` and its retry follows the new harness. -- Example `29-registries.ts` (a router registry with two contributing plugins, a conflict reported, a `writable` - contribution, a missing declaration healing); example `30-host-root.ts`; example `31-harness-registry.ts` (a - Durable harness on the store, its extensions from a registry, its views followed by a second object). +- Examples: Chord's `29-registries.ts` (a router registry with two contributing plugins, a conflict reported, a + `writable` contribution, a missing provider healing) and `30-host-root.ts`; Durable's `test/examples/37-harness- + registry.ts` (a Durable harness on the store, its extensions from a registry, its views followed by a second object) + and the `registryReader` tests in Durable's tree. ## 6. Carry-over diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index d0498da76..c2498ecba 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -49,7 +49,7 @@ Order (owner, 2026-10-22: routines and jobs before the store and reload). | 6a | **The store** | the store (backends, tables, queries, writes, commit queue, outbox, documents engine, migrations, export), persisted documents, `self.table`, keyed watches and retirement | | 6b | **Durable and jobs on the store** | persisted jobs, Durable's `StoreStorage`, the Chord source over Durable's views (`self.follow`) | | 7 | **Reload and rollout** | plugin loading, staging, swaps and restarts, revisions, `accepts` in use | -| 8 | **Registries and `host.root`** | registries, `self.writable`, `retryOn`, a `RegistryReader` for Durable, `host.root` and `chord(fn, context)` (the harness object is p3's base code) | +| 8 | **Registries and `host.root`** | registries, `self.writable`, `retryOn`, a `RegistryReader` for Durable, `host.root` (`chord(fn, context)` and the harness object are p3's base code) | | 9 | **Telemetry** | spans, events, metrics across hosts | The playground (`apps/chord-playground`) moves to the new code once slice 4 exists. Round-3 cuts (C3-C16) and @@ -227,14 +227,8 @@ Specified in `slice-6b.md` (decided 2026-10-27, D1-D9, R1-R4, "owner names and c - **Durable's views from other objects** (from slice 6b): the harness carried in a registry entry, so an object other than the one that opened Durable can `self.follow` its views. -- **`host.root(fn, context, { principal? })`**: a call scope for code outside objects, built with its first users - (Durable hooks, `chord(fn, context)`); decide what an aborted, uncooperative `fn` means for its references (match - the handler and `CallContext` rules). (Slice 3b, owner.) -- **`retryOn`** retries a broken object as `ref.retry()` does. (Slice 2.) -- **`host.activity` on the facet's host** (`FacetHost`), if the harness or a core facet needs it; today only - `references` is there. (Slice 4b.) -- Registries recompose inside the owner's flush, never inside another owner's commit step; changing contributions are - native watchables (`self.writable`). (W4.) +- Specified in `slice-8.md` (decided 2026-10-27): `host.root` (D3), `retryOn` (D2), registries recomposing in the + owner's flush (W4); `host.activity` on the facet's host dropped (nothing needs it). ### Slice 9: telemetry From 82a0d646e5399d236713c421cff5679f049b7ef8 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 17:52:54 +0000 Subject: [PATCH 100/271] =?UTF-8?q?docs(chord):=20slice=208=20review=202?= =?UTF-8?q?=20leftovers;=20design=20=C2=A724=20harness=20example?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/chord/docs/next/design.md | 11 +++++++---- packages/chord/docs/next/slice-8.md | 9 ++++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index a474aa829..1bc182b61 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -1938,11 +1938,14 @@ loop. Chord has no stream type (it would need its own rules for drops, cancellat access and execution environments from registry entries, looked up at each use: ```ts - const harness = self.open(async (context) => Harness.open(new StoreStorage(store), { // the host's store (slice 6b) - models: () => host.registry(HarnessModels).value, - env: (target) => host.registry(HarnessEnvironments).value(target.cwd), + const models = self.registry(HarnessModels); // declared dependencies (slice 8 S3): activation waits + const envs = self.registry(HarnessEnvironments); + const extensions = self.registry(HarnessExtensions); + const harness = self.open(async (context) => Harness.open(await StoreStorage.open(store, Harness.name, context), { + models: () => models.value, // read at use + env: (target) => envs.value(target.cwd), settings: () => currentSettings(), - registry: durableRegistry(host.registry(HarnessExtensions)), // a RegistryReader over the composed NativeWatchable + registry: registryReader(extensions, context), // Durable's RegistryReader over the composed value (slice 8) }, context), (h, _reason, context) => h.close(context)); ``` diff --git a/packages/chord/docs/next/slice-8.md b/packages/chord/docs/next/slice-8.md index 090ee3bbf..e28bcb7ee 100644 --- a/packages/chord/docs/next/slice-8.md +++ b/packages/chord/docs/next/slice-8.md @@ -99,7 +99,8 @@ await host.root(async (ctx) => (await ctx.get(Notes, { session })).add(note, ctx out (reported `contribution_failed`); a `value` adapter that throws fails only that version's value, by the rules of a throwing `compose` (its readers keep the last good mapped value; before any, a reader of that version waiting at activation fails `transient`, retried when a mapped value arrives); current readers are never affected. -- **Helpers:** `keyed(table, entriesOf)` (a map, a duplicate key reported, the first by plugin name wins) and +- **Helpers:** `keyed(table, entriesOf, report)` (a map, a duplicate key reported through `report`, the first by plugin + name wins) and `ordered(table, itemsOf, orderOf?)`. - **Use sites guard contributions** (Q8): Chord's helpers keep each entry's plugin name; the guide shows catching a contribution's throws and rejections, reporting the plugin, and continuing. @@ -121,7 +122,7 @@ In any object's factory, anywhere: Chord follows each source with watches owned replaced by each attempt's declaration, stopped at the object's end; GC-tested), installed right after the factory returns, before natives open, so a failed open still has them; a `value` call of a source while the object is failed (`transient` or `broken`) retries it as `ref.retry()` does (backoff skipped, an attempt now, holders told `loading`); -a call while it activates is remembered and retries it at once if that attempt fails (S1); ignored while live. Sources +a call while it activates is remembered and retries it at once if that attempt fails (decision S1); ignored while live. Sources are those that live independently of the attempt: a `RegistryValue`, `constant`; anything else (a reference's member, a `writable` of the same attempt) is refused at runtime (declaration policy; TypeScript cannot tell them apart). @@ -168,7 +169,9 @@ after `host.close` rejects `released`. No span of its own until slice 9. - Examples: Chord's `29-registries.ts` (a router registry with two contributing plugins, a conflict reported, a `writable` contribution, a missing provider healing) and `30-host-root.ts`; Durable's `test/examples/37-harness- registry.ts` (a Durable harness on the store, its extensions from a registry, its views followed by a second object) - and the `registryReader` tests in Durable's tree. + and the `registryReader` tests in Durable's tree. Durable's `subscribe` listeners then run inside Chord's flush: + they only wake Durable's scheduler (no Chord call; a reentrant one would throw `invalid/reentrant`), which the + `registryReader` test pins. ## 6. Carry-over From 568f6c47eb4b5c8b7e5e5de552ade143ab5baaf6 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 17:53:56 +0000 Subject: [PATCH 101/271] docs(chord): slice 8 ready to build --- packages/chord/docs/next/slice-8.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/chord/docs/next/slice-8.md b/packages/chord/docs/next/slice-8.md index e28bcb7ee..939ecb881 100644 --- a/packages/chord/docs/next/slice-8.md +++ b/packages/chord/docs/next/slice-8.md @@ -1,13 +1,8 @@ # Slice 8: registries and `host.root` (spec) -Status: **revised after the spec review** (2026-10-27; D1-D3, the registry failures, S1-S3 decided by the owner; the -four reviews `/tmp/s8-review-{astra,fable,opus,sol}.md` and a check against the design's rules found `self.registry` -wrongly specified as a plain read: it is a declared dependency, as `self.get`), written by the redesign session. Built in -parallel with 6b (mostly new files). The single source for building 8, together with `design.md` (§7, §8, §11, §16, -§17, §18, §24 read in full), the decision log `decisions.md` (R21, R23, R25, Q8, D1 of "Review round 1", "Registries -work like Durable's", "Changing contributions, `self.writable`", W4, "Failures are watch calls", "Slice 8 is -'Registries and `host.root`'"), the feature map and carry-over of `slices.md` (slice 8), and the root `AGENTS.md` -"Chord v2 Code Style". +Status: **ready to build** (2026-10-27; D1-D3, the registry failures and S1-S3 decided by the owner; two spec reviews, +`/tmp/s8-review{,2}-{astra,fable,opus,sol}.md`, found no remaining blocker; S3 refined after the second: with no value +after `load`, a reader fails at once, as `self.get` of a type not served here) 8 builds the extension point between plugins (registries), in-process values that change (`self.writable`), wake signals for failed objects (`retryOn`), and the call scope for code outside objects (`host.root`). The harness object From dc35ae2d92eb2c220d94fe8749028f559f480829 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 17:56:37 +0000 Subject: [PATCH 102/271] docs(chord): slice 8: ref.state is a native source too --- packages/chord/docs/next/slice-8.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/chord/docs/next/slice-8.md b/packages/chord/docs/next/slice-8.md index 939ecb881..2bd988d7a 100644 --- a/packages/chord/docs/next/slice-8.md +++ b/packages/chord/docs/next/slice-8.md @@ -107,8 +107,8 @@ await host.root(async (ctx) => (await ctx.get(Notes, { session })).add(note, ctx A `NativeWatchable` with `set(value)` (identity compared: `set` of the same value notifies nobody; a new value is a new frame), owned by the attempt (dropped with it; `set` after the attempt's seal throws `released`); never a state member; usable as a contribution, a derive source and an `on` source. `set` inside a batch marks it there; outside, it -opens its own batch. Both `writable` and `RegistryValue` are native sources: one base (`NativeSource`) the batch marks -and derives depend on; `self.derive` takes JSON and native sources (its type and the source resolver widen), while the +opens its own batch. `writable`, `RegistryValue` and `ref.state` (a `NativeWatchable` since slice 3b, until now not derivable: design audit, +2026-10-27) are native sources: one base (`NativeSource`) the batch marks and derives depend on; `self.derive` takes JSON and native sources (its type and the source resolver widen), while the member check still refuses a native as a state member. ## 3. `self.retryOn(...sources)` From 9ae3d8814c0cef5e0f3c1ea6ab8d850bfa088332 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 17:58:27 +0000 Subject: [PATCH 103/271] =?UTF-8?q?docs(chord):=20design=20=C2=A76=20seal?= =?UTF-8?q?=20fence:=20staged=20job=20changes=20adopted?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/chord/docs/next/design.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index 1bc182b61..4f920dcee 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -586,8 +586,8 @@ host.serve(Notes, ({ session }, principal, self) => - **The seal fence** (owner, 2026-10-26): every attempt has a seal token (a job run invocation its own, for its state changes and outcome). After the seal, new work through the attempt's handles is refused at use (`self.tx`, `ctx.get`, a job or routine start); commits are checked when they enter the store's commit step (sealed: rejected `released`, - nothing written) and at adopt (sealed during the SQL: durable, adopted into the object's documents, staged job - changes abandoned); at the seal a transaction not yet in the step is aborted and the line released, a started step + nothing written) and at adopt (sealed during the SQL: durable, adopted into the object's documents, its staged job + changes adopted into the object too: decision "Second spec review: the majority" (3)); at the seal a transaction not yet in the step is aborted and the line released, a started step finishes before teardown completes; cleanup (references, presence, activity counts) still runs. A pure swap is not a replacement. - **Ending** (slice 1, 2): new calls are rejected; every caller of a running call gets `released` at once; the From 3bae85274a9ea64d035a8a35753c93031489b659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Hou=C5=A1ka?= Date: Wed, 28 Oct 2026 18:44:07 +0100 Subject: [PATCH 104/271] fix(tui,coding-agent): handle selection page keys in all scrolling lists tui.select.pageUp/pageDown were configurable but ignored by SelectList, SettingsList, editor autocomplete, and most coding-agent selectors. In fullscreen mode, the default PageUp/PageDown also scrolled the transcript instead of reaching the focused list. - add getSelectPageIndex() and handle page keys in SelectList, SettingsList, editor autocomplete, and the model, scoped models, auth, user message, settings submenu, llama search, and /mcp selectors - add Component.capturesViewportInput: a focused component that sets it receives fullscreen viewport keys instead of the transcript, as a focused overlay already does - let page keys move an open autocomplete menu before app actions, so a remapped Ctrl+V pages the menu instead of pasting an image closes #7629 Co-authored-by: muyiyr <940955635@qq.com> --- packages/coding-agent/CHANGELOG.md | 1 + packages/coding-agent/docs/keybindings.md | 2 +- .../examples/extensions/preset.ts | 2 + .../coding-agent/examples/extensions/tools.ts | 2 + .../coding-agent/src/extensions/llama/ui.ts | 39 ++++++--- .../coding-agent/src/extensions/mcp/ui.ts | 59 ++++++++------ .../interactive/components/custom-editor.ts | 10 +++ .../interactive/components/model-selector.ts | 16 +++- .../interactive/components/oauth-selector.ts | 16 +++- .../components/scoped-models-selector.ts | 9 +++ .../components/session-selector.ts | 4 + .../components/settings-submenu.ts | 2 + .../interactive/components/tree-selector.ts | 4 + .../components/user-message-selector.ts | 17 +++- .../custom-editor-history-keybindings.test.ts | 34 -------- .../test/custom-editor-keybindings.test.ts | 79 +++++++++++++++++++ .../test/mcp-manager-view.test.ts | 21 +++++ .../coding-agent/test/model-selector.test.ts | 24 ++++++ .../test/settings-selector.test.ts | 20 +++++ packages/tui/CHANGELOG.md | 3 + packages/tui/README.md | 2 + packages/tui/src/components/editor.ts | 12 ++- packages/tui/src/components/select-list.ts | 10 ++- packages/tui/src/components/settings-list.ts | 7 +- packages/tui/src/index.ts | 1 + packages/tui/src/keybindings.ts | 19 +++++ packages/tui/src/tui-alt-screen.ts | 6 +- packages/tui/src/tui.ts | 6 ++ packages/tui/test/editor.test.ts | 30 +++++++ packages/tui/test/select-list.test.ts | 15 ++++ packages/tui/test/settings-list.test.ts | 22 ++++++ packages/tui/test/tui-alt-screen.test.ts | 79 +++++++++++++++++++ 32 files changed, 492 insertions(+), 81 deletions(-) delete mode 100644 packages/coding-agent/test/custom-editor-history-keybindings.test.ts create mode 100644 packages/coding-agent/test/custom-editor-keybindings.test.ts diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index ae86551c1..1ed5d448d 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -25,6 +25,7 @@ - Fixed aborting a bash command on macOS sometimes running the next command of a list such as `long_step; next_step`. pi now stops the process group before killing it ([#11252](https://github.com/earendil-works/pi/issues/11252), [#11289](https://github.com/earendil-works/pi/pull/11289)) - Fixed streamed tool calls re-running their call renderer on every argument delta; the interactive mode now renders the latest arguments once per frame, so a large streamed `write` no longer re-highlights its whole content per delta ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed the login dialog opening sign-in URLs that are not web URLs, and printing control characters from sign-in, device-code, and info URLs, which could end a terminal hyperlink and send escape sequences to the terminal +- Fixed `tui.select.pageUp` and `tui.select.pageDown` being ignored in slash command and file autocomplete, `/settings`, and the model, scoped models, login, fork, `/mcp`, and llama.cpp selectors. In fullscreen mode, PageUp/PageDown now move an open list or autocomplete menu instead of scrolling the transcript, and a page key bound to Ctrl+V pages autocomplete instead of pasting an image ([#7629](https://github.com/earendil-works/pi/issues/7629), based on [#7680](https://github.com/earendil-works/pi/pull/7680) by [@muyiyr](https://github.com/muyiyr)) ## [1.4.0] - 2026-10-21 diff --git a/packages/coding-agent/docs/keybindings.md b/packages/coding-agent/docs/keybindings.md index 77838b629..34e4d68b4 100644 --- a/packages/coding-agent/docs/keybindings.md +++ b/packages/coding-agent/docs/keybindings.md @@ -98,7 +98,7 @@ The dedicated history actions browse prompt history regardless of cursor positio #### Fullscreen -In fullscreen mode, these actions control the transcript and take precedence over editor actions using the same key. +In fullscreen mode, these actions control the transcript and take precedence over editor actions using the same key. While an overlay or a scrollable list such as autocomplete, `/settings`, or the model selector has focus, the scrolling and prompt-jump actions go to it instead of the transcript, so `tui.select.pageUp` and `tui.select.pageDown` move the list selection. Transcript search actions keep working. | Keybinding id | Default | Description | |---|---|---| diff --git a/packages/coding-agent/examples/extensions/preset.ts b/packages/coding-agent/examples/extensions/preset.ts index 39f55642c..3b5f98764 100644 --- a/packages/coding-agent/examples/extensions/preset.ts +++ b/packages/coding-agent/examples/extensions/preset.ts @@ -252,6 +252,8 @@ export default function presetExtension(pi: ExtensionAPI) { container.addChild(new DynamicBorder((str) => theme.fg("accent", str))); return { + // Let PageUp/PageDown page the list instead of scrolling the fullscreen transcript. + capturesViewportInput: selectList.capturesViewportInput, render(width: number) { return container.render(width); }, diff --git a/packages/coding-agent/examples/extensions/tools.ts b/packages/coding-agent/examples/extensions/tools.ts index 8db496a51..7f65b1796 100644 --- a/packages/coding-agent/examples/extensions/tools.ts +++ b/packages/coding-agent/examples/extensions/tools.ts @@ -117,6 +117,8 @@ export default function toolsExtension(pi: ExtensionAPI) { container.addChild(settingsList); const component = { + // Let PageUp/PageDown page the list instead of scrolling the fullscreen transcript. + capturesViewportInput: settingsList.capturesViewportInput, render(width: number) { return container.render(width); }, diff --git a/packages/coding-agent/src/extensions/llama/ui.ts b/packages/coding-agent/src/extensions/llama/ui.ts index 694921bde..25f64c07f 100644 --- a/packages/coding-agent/src/extensions/llama/ui.ts +++ b/packages/coding-agent/src/extensions/llama/ui.ts @@ -3,6 +3,7 @@ import { Container, type Focusable, fuzzyFilter, + getSelectPageIndex, Input, type SelectItem, SelectList, @@ -21,9 +22,12 @@ import type { LlamaModelInfo, LlamaProgress } from "./client.ts"; import type { HuggingFaceModel } from "./huggingface.ts"; const DOWNLOAD_VALUE = "\0download"; +const SEARCH_MAX_VISIBLE = 10; export type LlamaManagerAction = { type: "model"; model: LlamaModelInfo } | { type: "download" } | { type: "close" }; +type InputHandler = Pick; + interface ProgressState extends LlamaProgress { title: string; model: string; @@ -94,6 +98,7 @@ function compactCount(value: number): string { } class HuggingFaceSearch extends Container implements Focusable { + readonly capturesViewportInput: boolean = true; private readonly tui: TUI; private readonly theme: Theme; private readonly keybindings: KeybindingsManager; @@ -145,12 +150,14 @@ class HuggingFaceSearch extends Container implements Focusable { private updateResults(): void { this.resultsContainer.clear(); - const maxVisible = 10; const start = Math.max( 0, - Math.min(this.selectedIndex - Math.floor(maxVisible / 2), this.filteredResults.length - maxVisible), + Math.min( + this.selectedIndex - Math.floor(SEARCH_MAX_VISIBLE / 2), + this.filteredResults.length - SEARCH_MAX_VISIBLE, + ), ); - const end = Math.min(start + maxVisible, this.filteredResults.length); + const end = Math.min(start + SEARCH_MAX_VISIBLE, this.filteredResults.length); for (let index = start; index < end; index++) { const model = this.filteredResults[index]; if (!model) continue; @@ -255,6 +262,20 @@ class HuggingFaceSearch extends Container implements Focusable { } return; } + const pageIndex = getSelectPageIndex( + this.keybindings, + data, + this.selectedIndex, + this.filteredResults.length, + SEARCH_MAX_VISIBLE, + ); + if (pageIndex !== undefined) { + if (this.filteredResults.length > 0) { + this.selectedIndex = pageIndex; + this.updateResults(); + } + return; + } if (this.keybindings.matches(data, "tui.select.confirm")) { const exact = /^[^/\s]+\/[^:\s]+(?::[^\s:]+)?$/u.test(this.query) ? this.query : undefined; const selected = exact ?? this.filteredResults[this.selectedIndex]?.id; @@ -279,7 +300,7 @@ class LlamaView implements LlamaUi, Focusable { private readonly keybindings: KeybindingsManager; private readonly searchCache = new Map(); private content: Container; - private inputHandler: { handleInput?(data: string): void } | undefined; + private inputHandler: InputHandler | undefined; private inputTarget: Focusable | undefined; private progressPromise: Promise | undefined; private progressResolver: (() => void) | undefined; @@ -302,11 +323,11 @@ class LlamaView implements LlamaUi, Focusable { if (this.inputTarget) this.inputTarget.focused = value; } - private setContent( - content: Container, - inputHandler?: { handleInput?(data: string): void }, - inputTarget?: Focusable, - ): void { + get capturesViewportInput(): boolean { + return this.inputHandler?.capturesViewportInput === true; + } + + private setContent(content: Container, inputHandler?: InputHandler, inputTarget?: Focusable): void { if (this.inputTarget) this.inputTarget.focused = false; this.progressPromise = undefined; this.progressResolver = undefined; diff --git a/packages/coding-agent/src/extensions/mcp/ui.ts b/packages/coding-agent/src/extensions/mcp/ui.ts index 771e76b61..38bdfcdae 100644 --- a/packages/coding-agent/src/extensions/mcp/ui.ts +++ b/packages/coding-agent/src/extensions/mcp/ui.ts @@ -70,12 +70,14 @@ function frame(theme: Theme, title: string, body: Component[], footer?: string): const MAX_VISIBLE_ITEMS = 12; +type InputHandler = Pick; + export class McpManagerView implements McpUi, Component, Focusable { private readonly tui: TUI; private readonly theme: Theme; private readonly keybindings: KeybindingsManager; private content: Container; - private inputHandler: ((data: string) => void) | undefined; + private inputHandler: InputHandler | undefined; private inputTarget: Focusable | undefined; private _focused = false; @@ -95,7 +97,11 @@ export class McpManagerView implements McpUi, Component, Focusable { if (this.inputTarget) this.inputTarget.focused = value; } - private setContent(content: Container, inputHandler?: (data: string) => void, inputTarget?: Focusable): void { + get capturesViewportInput(): boolean { + return this.inputHandler?.capturesViewportInput === true; + } + + private setContent(content: Container, inputHandler?: InputHandler, inputTarget?: Focusable): void { if (this.inputTarget) this.inputTarget.focused = false; this.content = content; this.inputHandler = inputHandler; @@ -125,12 +131,11 @@ export class McpManagerView implements McpUi, Component, Focusable { const footer = `${keyHint("tui.select.confirm", menu.confirmLabel)} • ${keyHint("tui.select.cancel", menu.cancelLabel)}`; if (menu.items.length === 0) { body.push(new Text(this.theme.fg("muted", menu.empty ?? "Nothing to show."), 1, 0)); - this.setContent( - frame(this.theme, menu.title, body, keyHint("tui.select.cancel", menu.cancelLabel)), - (data) => { + this.setContent(frame(this.theme, menu.title, body, keyHint("tui.select.cancel", menu.cancelLabel)), { + handleInput: (data) => { if (this.keybindings.matches(data, "tui.select.cancel")) finish(undefined); }, - ); + }); return; } const list = new SelectList( @@ -147,7 +152,7 @@ export class McpManagerView implements McpUi, Component, Focusable { list.onSelect = (item) => finish(item.value); list.onCancel = () => finish(undefined); body.push(list); - this.setContent(frame(this.theme, menu.title, body, footer), (data) => list.handleInput(data)); + this.setContent(frame(this.theme, menu.title, body, footer), list); }; render(); unsubscribe = subscribe?.(() => { @@ -162,8 +167,10 @@ export class McpManagerView implements McpUi, Component, Focusable { this.setContent(frame(this.theme, title, body)); return; } - this.setContent(frame(this.theme, title, body, keyHint("tui.select.cancel", "cancel")), (data) => { - if (this.keybindings.matches(data, "tui.select.cancel")) onCancel(); + this.setContent(frame(this.theme, title, body, keyHint("tui.select.cancel", "cancel")), { + handleInput: (data) => { + if (this.keybindings.matches(data, "tui.select.cancel")) onCancel(); + }, }); } @@ -203,21 +210,23 @@ export class McpManagerView implements McpUi, Component, Focusable { body, `${keyHint("tui.select.confirm", "submit")} • ${keyHint("tui.select.cancel", "cancel")}`, ), - (data) => { - if (this.keybindings.matches(data, "tui.select.confirm")) { - const value = input.getValue().trim(); - if (value) finish(value); - return; - } - if (this.keybindings.matches(data, "tui.select.cancel")) { - finish(undefined); - return; - } - if (this.keybindings.matches(data, "app.message.copy")) { - void link.copy(); - return; - } - input.handleInput(data); + { + handleInput: (data) => { + if (this.keybindings.matches(data, "tui.select.confirm")) { + const value = input.getValue().trim(); + if (value) finish(value); + return; + } + if (this.keybindings.matches(data, "tui.select.cancel")) { + finish(undefined); + return; + } + if (this.keybindings.matches(data, "app.message.copy")) { + void link.copy(); + return; + } + input.handleInput(data); + }, }, input, ); @@ -225,7 +234,7 @@ export class McpManagerView implements McpUi, Component, Focusable { } handleInput(data: string): void { - this.inputHandler?.(data); + this.inputHandler?.handleInput?.(data); this.tui.requestRender(); } diff --git a/packages/coding-agent/src/modes/interactive/components/custom-editor.ts b/packages/coding-agent/src/modes/interactive/components/custom-editor.ts index 9d345b4e9..1f0c53958 100644 --- a/packages/coding-agent/src/modes/interactive/components/custom-editor.ts +++ b/packages/coding-agent/src/modes/interactive/components/custom-editor.ts @@ -91,6 +91,16 @@ export class CustomEditor extends Editor { return; } + // Selection page keys move an open autocomplete menu before app actions run. + // This lets users bind Ctrl+V even though it pastes images by default. + if ( + this.isShowingAutocomplete() && + (this.keybindings.matches(data, "tui.select.pageUp") || this.keybindings.matches(data, "tui.select.pageDown")) + ) { + super.handleInput(data); + return; + } + // Check for clipboard paste keybinding if (this.keybindings.matches(data, "app.clipboard.pasteImage")) { this.onPasteImage?.(); diff --git a/packages/coding-agent/src/modes/interactive/components/model-selector.ts b/packages/coding-agent/src/modes/interactive/components/model-selector.ts index d603b8bef..f33eecf27 100644 --- a/packages/coding-agent/src/modes/interactive/components/model-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/model-selector.ts @@ -4,6 +4,7 @@ import { type Focusable, fuzzyFilter, getKeybindings, + getSelectPageIndex, Input, Spacer, Text, @@ -34,10 +35,13 @@ interface DefaultModelReference { type ModelScope = "all" | "scoped"; +const MAX_VISIBLE = 10; + /** * Component that renders a model selector with search */ export class ModelSelectorComponent extends Container implements Focusable { + readonly capturesViewportInput: boolean = true; private searchInput: Input; // Focusable implementation - propagate to searchInput for IME cursor positioning @@ -305,12 +309,11 @@ export class ModelSelectorComponent extends Container implements Focusable { private updateList(): void { this.listContainer.clear(); - const maxVisible = 10; const startIndex = Math.max( 0, - Math.min(this.selectedIndex - Math.floor(maxVisible / 2), this.filteredModels.length - maxVisible), + Math.min(this.selectedIndex - Math.floor(MAX_VISIBLE / 2), this.filteredModels.length - MAX_VISIBLE), ); - const endIndex = Math.min(startIndex + maxVisible, this.filteredModels.length); + const endIndex = Math.min(startIndex + MAX_VISIBLE, this.filteredModels.length); // Show visible slice of filtered models for (let i = startIndex; i < endIndex; i++) { @@ -363,6 +366,7 @@ export class ModelSelectorComponent extends Container implements Focusable { handleInput(keyData: string): void { const kb = getKeybindings(); + const pageIndex = getSelectPageIndex(kb, keyData, this.selectedIndex, this.filteredModels.length, MAX_VISIBLE); if (kb.matches(keyData, "tui.input.tab")) { if (this.scopedModelItems.length > 0) { const nextScope: ModelScope = this.scope === "all" ? "scoped" : "all"; @@ -385,6 +389,12 @@ export class ModelSelectorComponent extends Container implements Focusable { this.selectedIndex = this.selectedIndex === this.filteredModels.length - 1 ? 0 : this.selectedIndex + 1; this.updateList(); } + // Page up/down + else if (pageIndex !== undefined) { + if (this.filteredModels.length === 0) return; + this.selectedIndex = pageIndex; + this.updateList(); + } // Enter else if (kb.matches(keyData, "tui.select.confirm")) { const selectedModel = this.filteredModels[this.selectedIndex]; diff --git a/packages/coding-agent/src/modes/interactive/components/oauth-selector.ts b/packages/coding-agent/src/modes/interactive/components/oauth-selector.ts index c03f74836..e3f70985b 100644 --- a/packages/coding-agent/src/modes/interactive/components/oauth-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/oauth-selector.ts @@ -4,6 +4,7 @@ import { type Focusable, fuzzyFilter, getKeybindings, + getSelectPageIndex, Input, Spacer, TruncatedText, @@ -52,10 +53,13 @@ export function formatAuthSelectorProviderStatus(provider: AuthSelectorProvider) return theme.fg("success", ` ✓ ${source}`); } +const MAX_VISIBLE = 8; + /** * Component that renders an auth provider selector */ export class OAuthSelectorComponent extends Container implements Focusable { + readonly capturesViewportInput: boolean = true; private searchInput: Input; // Focusable implementation - propagate to search input for IME cursor positioning @@ -143,12 +147,11 @@ export class OAuthSelectorComponent extends Container implements Focusable { private updateList(): void { this.listContainer.clear(); - const maxVisible = 8; const startIndex = Math.max( 0, - Math.min(this.selectedIndex - Math.floor(maxVisible / 2), this.filteredProviders.length - maxVisible), + Math.min(this.selectedIndex - Math.floor(MAX_VISIBLE / 2), this.filteredProviders.length - MAX_VISIBLE), ); - const endIndex = Math.min(startIndex + maxVisible, this.filteredProviders.length); + const endIndex = Math.min(startIndex + MAX_VISIBLE, this.filteredProviders.length); for (let i = startIndex; i < endIndex; i++) { const provider = this.filteredProviders[i]; @@ -192,6 +195,7 @@ export class OAuthSelectorComponent extends Container implements Focusable { handleInput(keyData: string): void { const kb = getKeybindings(); + const pageIndex = getSelectPageIndex(kb, keyData, this.selectedIndex, this.filteredProviders.length, MAX_VISIBLE); // Up arrow if (kb.matches(keyData, "tui.select.up")) { if (this.filteredProviders.length === 0) return; @@ -204,6 +208,12 @@ export class OAuthSelectorComponent extends Container implements Focusable { this.selectedIndex = Math.min(this.filteredProviders.length - 1, this.selectedIndex + 1); this.updateList(); } + // Page up/down + else if (pageIndex !== undefined) { + if (this.filteredProviders.length === 0) return; + this.selectedIndex = pageIndex; + this.updateList(); + } // Enter else if (kb.matches(keyData, "tui.select.confirm")) { const selectedProvider = this.filteredProviders[this.selectedIndex]; diff --git a/packages/coding-agent/src/modes/interactive/components/scoped-models-selector.ts b/packages/coding-agent/src/modes/interactive/components/scoped-models-selector.ts index 54a6f97cc..9a63ec294 100644 --- a/packages/coding-agent/src/modes/interactive/components/scoped-models-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/scoped-models-selector.ts @@ -4,6 +4,7 @@ import { type Focusable, fuzzyFilter, getKeybindings, + getSelectPageIndex, Input, Key, matchesKey, @@ -95,6 +96,7 @@ export interface ModelsCallbacks { * Changes are session-only until explicitly persisted with Ctrl+S. */ export class ScopedModelsSelectorComponent extends Container implements Focusable { + readonly capturesViewportInput: boolean = true; private modelsById: Map> = new Map(); private allIds: string[] = []; private enabledIds: EnabledIds = null; @@ -295,6 +297,13 @@ export class ScopedModelsSelectorComponent extends Container implements Focusabl this.updateList(); return; } + const pageIndex = getSelectPageIndex(kb, data, this.selectedIndex, this.filteredItems.length, this.maxVisible); + if (pageIndex !== undefined) { + if (this.filteredItems.length === 0) return; + this.selectedIndex = pageIndex; + this.updateList(); + return; + } // Reorder enabled models const reorderUp = kb.matches(data, "app.models.reorderUp"); diff --git a/packages/coding-agent/src/modes/interactive/components/session-selector.ts b/packages/coding-agent/src/modes/interactive/components/session-selector.ts index d60eaffc8..baa66e991 100644 --- a/packages/coding-agent/src/modes/interactive/components/session-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/session-selector.ts @@ -692,6 +692,10 @@ async function deleteSessionFile( * Component that renders a session selector */ export class SessionSelectorComponent extends Container implements Focusable { + get capturesViewportInput(): boolean { + return this.mode !== "rename"; + } + handleInput(data: string): void { if (this.mode === "rename") { const kb = getKeybindings(); diff --git a/packages/coding-agent/src/modes/interactive/components/settings-submenu.ts b/packages/coding-agent/src/modes/interactive/components/settings-submenu.ts index 81703323a..5da1d6fcc 100644 --- a/packages/coding-agent/src/modes/interactive/components/settings-submenu.ts +++ b/packages/coding-agent/src/modes/interactive/components/settings-submenu.ts @@ -123,6 +123,8 @@ export class SelectSubmenu extends Container { const isNav = kb.matches(data, "tui.select.up") || kb.matches(data, "tui.select.down") || + kb.matches(data, "tui.select.pageUp") || + kb.matches(data, "tui.select.pageDown") || kb.matches(data, "tui.select.confirm") || kb.matches(data, "tui.select.cancel"); if (isNav) { diff --git a/packages/coding-agent/src/modes/interactive/components/tree-selector.ts b/packages/coding-agent/src/modes/interactive/components/tree-selector.ts index 767740c87..73255479b 100644 --- a/packages/coding-agent/src/modes/interactive/components/tree-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/tree-selector.ts @@ -1414,6 +1414,10 @@ export class TreeSelectorComponent extends Container implements Focusable { this.treeContainer.addChild(this.treeList); } + get capturesViewportInput(): boolean { + return this.labelInput === null; + } + handleInput(keyData: string): void { if (this.labelInput) { this.labelInput.handleInput(keyData); diff --git a/packages/coding-agent/src/modes/interactive/components/user-message-selector.ts b/packages/coding-agent/src/modes/interactive/components/user-message-selector.ts index 69fbbc82a..d187ad5b6 100644 --- a/packages/coding-agent/src/modes/interactive/components/user-message-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/user-message-selector.ts @@ -1,4 +1,12 @@ -import { type Component, Container, getKeybindings, Spacer, Text, truncateToWidth } from "@earendil-works/pi-tui"; +import { + type Component, + Container, + getKeybindings, + getSelectPageIndex, + Spacer, + Text, + truncateToWidth, +} from "@earendil-works/pi-tui"; import { theme } from "../theme/theme.ts"; import { DynamicBorder } from "./dynamic-border.ts"; @@ -12,6 +20,7 @@ interface UserMessageItem { * Custom user message list component with selection */ class UserMessageList implements Component { + readonly capturesViewportInput: boolean = true; private messages: UserMessageItem[] = []; private selectedIndex: number = 0; public onSelect?: (entryId: string) => void; @@ -80,6 +89,7 @@ class UserMessageList implements Component { handleInput(keyData: string): void { const kb = getKeybindings(); + const pageIndex = getSelectPageIndex(kb, keyData, this.selectedIndex, this.messages.length, this.maxVisible); // Up arrow - go to previous (older) message, wrap to bottom when at top if (kb.matches(keyData, "tui.select.up")) { this.selectedIndex = this.selectedIndex === 0 ? this.messages.length - 1 : this.selectedIndex - 1; @@ -88,6 +98,11 @@ class UserMessageList implements Component { else if (kb.matches(keyData, "tui.select.down")) { this.selectedIndex = this.selectedIndex === this.messages.length - 1 ? 0 : this.selectedIndex + 1; } + // Page up/down + else if (pageIndex !== undefined) { + if (this.messages.length === 0) return; + this.selectedIndex = pageIndex; + } // Enter - select message and branch else if (kb.matches(keyData, "tui.select.confirm")) { const selected = this.messages[this.selectedIndex]; diff --git a/packages/coding-agent/test/custom-editor-history-keybindings.test.ts b/packages/coding-agent/test/custom-editor-history-keybindings.test.ts deleted file mode 100644 index a4de5200d..000000000 --- a/packages/coding-agent/test/custom-editor-history-keybindings.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { setKeybindings, TuiMainScreen } from "@earendil-works/pi-tui"; -import { afterEach, describe, expect, it } from "vitest"; -import { defaultEditorTheme } from "../../tui/test/test-themes.ts"; -import { VirtualTerminal } from "../../tui/test/virtual-terminal.ts"; -import { KeybindingsManager } from "../src/core/keybindings.ts"; -import { CustomEditor } from "../src/modes/interactive/components/custom-editor.ts"; - -afterEach(() => { - setKeybindings(new KeybindingsManager()); -}); - -describe("CustomEditor prompt history keybindings", () => { - it("gives an explicit history binding precedence over model cycling", () => { - const keybindings = new KeybindingsManager({ - "tui.editor.historyPrevious": "ctrl+p", - "tui.editor.historyNext": "ctrl+n", - }); - setKeybindings(keybindings); - const editor = new CustomEditor(new TuiMainScreen(new VirtualTerminal()), defaultEditorTheme, keybindings); - let modelCycles = 0; - editor.onAction("app.model.cycleForward", () => { - modelCycles++; - }); - editor.addToHistory("previous prompt"); - editor.setText("draft"); - - editor.handleInput("\x10"); // Ctrl+P - expect(editor.getText()).toBe("previous prompt"); - expect(modelCycles).toBe(0); - - editor.handleInput("\x0e"); // Ctrl+N - expect(editor.getText()).toBe("draft"); - }); -}); diff --git a/packages/coding-agent/test/custom-editor-keybindings.test.ts b/packages/coding-agent/test/custom-editor-keybindings.test.ts new file mode 100644 index 000000000..355951a15 --- /dev/null +++ b/packages/coding-agent/test/custom-editor-keybindings.test.ts @@ -0,0 +1,79 @@ +import { setKeybindings, TuiMainScreen } from "@earendil-works/pi-tui"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { defaultEditorTheme } from "../../tui/test/test-themes.ts"; +import { VirtualTerminal } from "../../tui/test/virtual-terminal.ts"; +import { KeybindingsManager } from "../src/core/keybindings.ts"; +import { CustomEditor } from "../src/modes/interactive/components/custom-editor.ts"; + +afterEach(() => { + setKeybindings(new KeybindingsManager()); +}); + +describe("CustomEditor prompt history keybindings", () => { + it("gives an explicit history binding precedence over model cycling", () => { + const keybindings = new KeybindingsManager({ + "tui.editor.historyPrevious": "ctrl+p", + "tui.editor.historyNext": "ctrl+n", + }); + setKeybindings(keybindings); + const editor = new CustomEditor(new TuiMainScreen(new VirtualTerminal()), defaultEditorTheme, keybindings); + let modelCycles = 0; + editor.onAction("app.model.cycleForward", () => { + modelCycles++; + }); + editor.addToHistory("previous prompt"); + editor.setText("draft"); + + editor.handleInput("\x10"); // Ctrl+P + expect(editor.getText()).toBe("previous prompt"); + expect(modelCycles).toBe(0); + + editor.handleInput("\x0e"); // Ctrl+N + expect(editor.getText()).toBe("draft"); + }); +}); + +describe("CustomEditor selection page keybindings", () => { + it("pages an open autocomplete menu before pasting an image with the same key (#7629)", async () => { + // Bind both explicitly: the paste default is Alt+V on Windows and WSL. + const keybindings = new KeybindingsManager({ + "tui.select.pageDown": "ctrl+v", + "app.clipboard.pasteImage": "ctrl+v", + }); + setKeybindings(keybindings); + const editor = new CustomEditor(new TuiMainScreen(new VirtualTerminal()), defaultEditorTheme, keybindings); + let pastes = 0; + editor.onPasteImage = () => { + pastes++; + }; + let submitted = ""; + editor.onSubmit = (text) => { + submitted = text; + }; + editor.setAutocompleteProvider({ + getSuggestions: async () => ({ + items: Array.from({ length: 12 }, (_, index) => ({ + value: `/command-${index}`, + label: `command-${index}`, + })), + prefix: "/", + }), + applyCompletion: (lines, cursorLine, _cursorCol, item) => ({ + lines: lines.map((line, index) => (index === cursorLine ? item.value : line)), + cursorLine, + cursorCol: item.value.length, + }), + }); + + editor.handleInput("/"); + await vi.waitFor(() => expect(editor.isShowingAutocomplete()).toBe(true)); + editor.handleInput("\x16"); // Ctrl+V + editor.handleInput("\r"); + + expect(pastes).toBe(0); + expect(submitted).toBe("/command-5"); + + editor.handleInput("\x16"); // Ctrl+V with autocomplete closed pastes again + expect(pastes).toBe(1); + }); +}); diff --git a/packages/coding-agent/test/mcp-manager-view.test.ts b/packages/coding-agent/test/mcp-manager-view.test.ts index ede09a8c9..ea7619717 100644 --- a/packages/coding-agent/test/mcp-manager-view.test.ts +++ b/packages/coding-agent/test/mcp-manager-view.test.ts @@ -38,3 +38,24 @@ describe("MCP manager status screen", () => { view.handleInput(ESCAPE); }); }); + +describe("MCP manager menu", () => { + beforeAll(() => { + initTheme("dark"); + }); + + // #7629 + test("captures fullscreen viewport keys only while a menu list is shown", async () => { + const view = new McpManagerView(tui, theme, new KeybindingsManager()); + const items = Array.from({ length: 20 }, (_, index) => ({ value: `server-${index}`, label: `server-${index}` })); + const result = view.menu(() => ({ title: "MCP servers", items, confirmLabel: "open", cancelLabel: "close" })); + expect(view.capturesViewportInput).toBe(true); + + view.handleInput("\x1b[6~"); // PageDown + view.handleInput("\r"); + await expect(result).resolves.toBe("server-12"); + + view.status("Sign in to issues", "Connecting…"); + expect(view.capturesViewportInput).toBe(false); + }); +}); diff --git a/packages/coding-agent/test/model-selector.test.ts b/packages/coding-agent/test/model-selector.test.ts index 7ba204ac7..df1a570e6 100644 --- a/packages/coding-agent/test/model-selector.test.ts +++ b/packages/coding-agent/test/model-selector.test.ts @@ -82,6 +82,30 @@ describe("model selector", () => { selector.dispose(); }); + it("moves by one page with remapped selection page keys (#7629)", async () => { + setKeybindings(new KeybindingsManager({ "tui.select.pageUp": "alt+v", "tui.select.pageDown": "ctrl+v" })); + harness = await createHarness({ + models: Array.from({ length: 12 }, (_, index) => ({ id: `model-${String(index).padStart(2, "0")}` })), + }); + const onSelect = vi.fn(); + const selector = new ModelSelectorComponent( + createFakeTui(), + undefined, + harness.session.modelRuntime, + [], + onSelect, + () => {}, + ); + + selector.handleInput("\x16"); // Ctrl+V + selector.handleInput("\x16"); + selector.handleInput("\x1bv"); // Alt+V + selector.handleInput("\r"); + + expect(onSelect).toHaveBeenCalledWith(expect.objectContaining({ id: "model-01" })); + selector.dispose(); + }); + it("uses the configured save binding", async () => { setKeybindings(new KeybindingsManager({ "app.models.save": "ctrl+r" })); harness = await createHarness(); diff --git a/packages/coding-agent/test/settings-selector.test.ts b/packages/coding-agent/test/settings-selector.test.ts index 35b9bfe23..6a39219cb 100644 --- a/packages/coding-agent/test/settings-selector.test.ts +++ b/packages/coding-agent/test/settings-selector.test.ts @@ -6,6 +6,7 @@ import { type SettingsConfig, SettingsSelectorComponent, } from "../src/modes/interactive/components/settings-selector.ts"; +import { SelectSubmenu } from "../src/modes/interactive/components/settings-submenu.ts"; import { initTheme } from "../src/modes/interactive/theme/theme.ts"; import { stripAnsi } from "../src/utils/ansi.ts"; import { createHarness, type Harness } from "./suite/harness.ts"; @@ -165,4 +166,23 @@ describe("SettingsSelectorComponent", () => { expect(output).toContain(" ✓ medium"); expect(output).toContain("→ high"); }); + + it("pages a searchable submenu with selection page keys (#7629)", () => { + const onSelect = vi.fn(); + const submenu = new SelectSubmenu( + "Theme", + "", + Array.from({ length: 20 }, (_, index) => ({ value: `theme-${index}`, label: `theme-${index}` })), + "theme-0", + onSelect, + () => {}, + undefined, + { searchable: true }, + ); + + submenu.handleInput("\x1b[6~"); // PageDown + submenu.handleInput("\r"); + + expect(onSelect).toHaveBeenCalledWith("theme-10"); + }); }); diff --git a/packages/tui/CHANGELOG.md b/packages/tui/CHANGELOG.md index 23cb8dc2f..2f0b69e9a 100644 --- a/packages/tui/CHANGELOG.md +++ b/packages/tui/CHANGELOG.md @@ -5,6 +5,8 @@ ### Added - Added `replaceControlCharacters()` to replace runs of C0 and C1 control characters with a space +- Added `Component.capturesViewportInput`. While a component that sets it has focus, `TuiAltScreen` sends viewport keys such as PageUp/PageDown to it instead of scrolling the transcript, as it already does for focused overlays. `SelectList`, `SettingsList`, and `Editor` with open autocomplete set it ([#7629](https://github.com/earendil-works/pi/issues/7629), based on [#7680](https://github.com/earendil-works/pi/pull/7680) by [@muyiyr](https://github.com/muyiyr)) +- Added `getSelectPageIndex()` to compute the selection after a `tui.select.pageUp` or `tui.select.pageDown` key ([#7629](https://github.com/earendil-works/pi/issues/7629), based on [#7680](https://github.com/earendil-works/pi/pull/7680) by [@muyiyr](https://github.com/muyiyr)) ### Fixed @@ -12,6 +14,7 @@ - Fixed `fuzzyMatch()` ranking a query scattered over an earlier part of the text above a contiguous match later in it. It now also scores each contiguous occurrence of the query and keeps the best alignment ([#11220](https://github.com/earendil-works/pi/issues/11220), [#11296](https://github.com/earendil-works/pi/pull/11296)) - Fixed `ProcessTerminal.setTitle()` writing control characters from the title, which could end the title sequence and send the rest to the terminal as escape sequences - Fixed display math wider than the content width being wrapped row by row, which split fractions and sums apart. It now falls back to the raw LaTeX source ([#11483](https://github.com/earendil-works/pi/issues/11483)) +- Fixed `tui.select.pageUp` and `tui.select.pageDown` being ignored by `SelectList`, `SettingsList`, and editor autocomplete ([#7629](https://github.com/earendil-works/pi/issues/7629), based on [#7680](https://github.com/earendil-works/pi/pull/7680) by [@muyiyr](https://github.com/muyiyr)) ## [1.4.0] - 2026-10-21 diff --git a/packages/tui/README.md b/packages/tui/README.md index 1150ebcf5..6ed8a26b3 100644 --- a/packages/tui/README.md +++ b/packages/tui/README.md @@ -251,6 +251,7 @@ interface Component { render(width: number): string[]; handleInput?(data: string): void; handleMouse?(event: TuiMouseEvent): TuiMouseEventResult | undefined; + capturesViewportInput?: boolean; invalidate(): void; } ``` @@ -260,6 +261,7 @@ interface Component { | `render(width)` | Returns an array of strings, one per line. Each line **must not exceed `width`** or the TUI will error. Use `truncateToWidth()` or manual wrapping to ensure this. | | `handleInput?(data)` | Called when the component has focus and receives keyboard input. The `data` string contains raw terminal input (may include ANSI escape sequences). | | `handleMouse?(event)` | Called by `TuiAltScreen` for normalized pointer input targeted at the component. | +| `capturesViewportInput?` | If true while the component has focus, `TuiAltScreen` passes its viewport scrolling and prompt-jump keys (`tui.altScreen.*` except search) to it instead of scrolling the viewport, as it does for a focused overlay. Set by `SelectList`, `SettingsList`, and `Editor` while autocomplete is open, so PageUp/PageDown move the selection. Only the focused component is checked: a wrapper that forwards input to a list must expose the list's value itself. Use `getSelectPageIndex(keybindings, data, selectedIndex, itemCount, pageSize)` to compute the new selection. | | `invalidate()` | Required. Clear any cached render state so the next `render()` starts from scratch. Components without cached render state can use an empty implementation. | The TUI appends a full SGR reset and OSC 8 reset at the end of each rendered line. Styles do not carry across lines. If you emit multi-line text with styling, reapply styles per line or use `wrapTextWithAnsi()` so styles are preserved for each wrapped line. diff --git a/packages/tui/src/components/editor.ts b/packages/tui/src/components/editor.ts index 5d453e3cd..9fd3677e9 100644 --- a/packages/tui/src/components/editor.ts +++ b/packages/tui/src/components/editor.ts @@ -304,6 +304,11 @@ export class Editor implements Component, Focusable { /** Focusable interface - set by TUI when focus changes */ focused: boolean = false; + /** The open autocomplete menu receives fullscreen viewport keys, such as PageUp/PageDown. */ + get capturesViewportInput(): boolean { + return this.autocompleteState !== null; + } + protected tui: TUI; private theme: EditorTheme; private paddingX: number = 0; @@ -761,7 +766,12 @@ export class Editor implements Component, Focusable { return; } - if (kb.matches(data, "tui.select.up") || kb.matches(data, "tui.select.down")) { + if ( + kb.matches(data, "tui.select.up") || + kb.matches(data, "tui.select.down") || + kb.matches(data, "tui.select.pageUp") || + kb.matches(data, "tui.select.pageDown") + ) { this.autocompleteList.handleInput(data); return; } diff --git a/packages/tui/src/components/select-list.ts b/packages/tui/src/components/select-list.ts index f4f4354dd..c196ab4ab 100644 --- a/packages/tui/src/components/select-list.ts +++ b/packages/tui/src/components/select-list.ts @@ -1,4 +1,4 @@ -import { getKeybindings } from "../keybindings.ts"; +import { getKeybindings, getSelectPageIndex } from "../keybindings.ts"; import type { Component, TuiMouseEvent, TuiMouseEventResult } from "../tui.ts"; import { truncateToWidth, visibleWidth } from "../utils.ts"; @@ -38,6 +38,7 @@ export interface SelectListLayoutOptions { } export class SelectList implements Component { + readonly capturesViewportInput: boolean = true; private items: SelectItem[] = []; private filteredItems: SelectItem[] = []; private selectedIndex: number = 0; @@ -144,6 +145,7 @@ export class SelectList implements Component { handleInput(keyData: string): void { const kb = getKeybindings(); + const pageIndex = getSelectPageIndex(kb, keyData, this.selectedIndex, this.filteredItems.length, this.maxVisible); // Up arrow - wrap to bottom when at top if (kb.matches(keyData, "tui.select.up")) { this.selectedIndex = this.selectedIndex === 0 ? this.filteredItems.length - 1 : this.selectedIndex - 1; @@ -154,6 +156,12 @@ export class SelectList implements Component { this.selectedIndex = this.selectedIndex === this.filteredItems.length - 1 ? 0 : this.selectedIndex + 1; this.notifySelectionChange(); } + // Page up/down - move by one visible page, clamped to the list bounds + else if (pageIndex !== undefined) { + if (this.filteredItems.length === 0 || pageIndex === this.selectedIndex) return; + this.selectedIndex = pageIndex; + this.notifySelectionChange(); + } // Enter else if (kb.matches(keyData, "tui.select.confirm")) { const selectedItem = this.filteredItems[this.selectedIndex]; diff --git a/packages/tui/src/components/settings-list.ts b/packages/tui/src/components/settings-list.ts index b1b8366bf..b5e4d1b8c 100644 --- a/packages/tui/src/components/settings-list.ts +++ b/packages/tui/src/components/settings-list.ts @@ -1,5 +1,5 @@ import { fuzzyFilter } from "../fuzzy.ts"; -import { getKeybindings } from "../keybindings.ts"; +import { getKeybindings, getSelectPageIndex } from "../keybindings.ts"; import type { Component, TuiMouseEvent, TuiMouseEventResult } from "../tui.ts"; import { truncateToWidth, visibleWidth, wrapTextWithAnsi } from "../utils.ts"; import { Input } from "./input.ts"; @@ -36,6 +36,7 @@ export interface SettingsListOptions { } export class SettingsList implements Component { + readonly capturesViewportInput: boolean = true; private items: SettingItem[]; private filteredItems: SettingItem[]; private theme: SettingsListTheme; @@ -230,12 +231,16 @@ export class SettingsList implements Component { // Main list input handling const kb = getKeybindings(); const displayItems = this.getDisplayItems(); + const pageIndex = getSelectPageIndex(kb, data, this.selectedIndex, displayItems.length, this.maxVisible); if (kb.matches(data, "tui.select.up")) { if (displayItems.length === 0) return; this.selectedIndex = this.selectedIndex === 0 ? displayItems.length - 1 : this.selectedIndex - 1; } else if (kb.matches(data, "tui.select.down")) { if (displayItems.length === 0) return; this.selectedIndex = this.selectedIndex === displayItems.length - 1 ? 0 : this.selectedIndex + 1; + } else if (pageIndex !== undefined) { + if (displayItems.length === 0) return; + this.selectedIndex = pageIndex; } else if ( kb.matches(data, "tui.select.confirm") || (data === " " && (!this.searchEnabled || this.searchInput?.getValue().length === 0)) diff --git a/packages/tui/src/index.ts b/packages/tui/src/index.ts index 61e436795..8b4f9ea4a 100644 --- a/packages/tui/src/index.ts +++ b/packages/tui/src/index.ts @@ -83,6 +83,7 @@ export { type FuzzyMatch, fuzzyFilter, fuzzyMatch } from "./fuzzy.ts"; // Keybindings export { getKeybindings, + getSelectPageIndex, type Keybinding, type KeybindingConflict, type KeybindingDefinition, diff --git a/packages/tui/src/keybindings.ts b/packages/tui/src/keybindings.ts index ed6e95171..652399f4b 100644 --- a/packages/tui/src/keybindings.ts +++ b/packages/tui/src/keybindings.ts @@ -318,3 +318,22 @@ export function getKeybindings(): KeybindingsManager { } return globalKeybindings; } + +/** + * Returns the selection index after a `tui.select.pageUp` or `tui.select.pageDown` key, moved by `pageSize` and + * clamped to `0..itemCount - 1`. Returns undefined when `data` is not a selection page key, and 0 for a page key on + * an empty list, so callers must still check that the list has items before selecting. + */ +export function getSelectPageIndex( + keybindings: KeybindingsManager, + data: string, + selectedIndex: number, + itemCount: number, + pageSize: number, +): number | undefined { + let direction: number; + if (keybindings.matches(data, "tui.select.pageUp")) direction = -1; + else if (keybindings.matches(data, "tui.select.pageDown")) direction = 1; + else return undefined; + return Math.max(0, Math.min(itemCount - 1, selectedIndex + direction * pageSize)); +} diff --git a/packages/tui/src/tui-alt-screen.ts b/packages/tui/src/tui-alt-screen.ts index 9596146a6..61d5b4825 100644 --- a/packages/tui/src/tui-alt-screen.ts +++ b/packages/tui/src/tui-alt-screen.ts @@ -678,7 +678,9 @@ export class TuiAltScreen extends TuiBase implements ViewportTUI { this.flashes.flash(message, durationMs); } - private shouldDeferViewportInputToOverlay(): boolean { + /** Focused overlays and components that capture viewport input receive viewport keys instead of the transcript. */ + private shouldDeferViewportInput(): boolean { + if (this.getFocusedComponent()?.capturesViewportInput) return true; return this.isOverlayFocused() && this.activeSearch?.overlay?.isFocused() !== true; } @@ -757,7 +759,7 @@ export class TuiAltScreen extends TuiBase implements ViewportTUI { return { consume: true }; } } - if (this.shouldDeferViewportInputToOverlay()) return undefined; + if (this.shouldDeferViewportInput()) return undefined; if (keybindings.matches(data, "tui.altScreen.pageUp")) { if (!isRelease) { this.scrollBy(-Math.max(1, this.getPrimaryScrollView().viewportHeight - PAGE_SCROLL_OVERLAP)); diff --git a/packages/tui/src/tui.ts b/packages/tui/src/tui.ts index d888940e9..ad11a36c5 100644 --- a/packages/tui/src/tui.ts +++ b/packages/tui/src/tui.ts @@ -134,6 +134,12 @@ export interface Component { */ wantsKeyRelease?: boolean; + /** + * If true while focused, fullscreen viewport keys (`tui.altScreen.*` scrolling and prompt jumps) go to this + * component instead of scrolling the transcript, as they do for a focused overlay. Default is false. + */ + capturesViewportInput?: boolean; + /** * Invalidate any cached rendering state. * Called when theme changes or when component needs to re-render from scratch. diff --git a/packages/tui/test/editor.test.ts b/packages/tui/test/editor.test.ts index 552c654b4..5b2683d20 100644 --- a/packages/tui/test/editor.test.ts +++ b/packages/tui/test/editor.test.ts @@ -2132,6 +2132,36 @@ describe("Editor component", () => { }); describe("Autocomplete", () => { + it("moves the autocomplete selection with selection page keys (#7629)", async () => { + const editor = new Editor(createTestTUI(), defaultEditorTheme); + let submitted = ""; + editor.onSubmit = (text) => { + submitted = text; + }; + editor.setAutocompleteProvider({ + getSuggestions: async () => ({ + items: Array.from({ length: 12 }, (_, index) => ({ + value: `/command-${index}`, + label: `command-${index}`, + })), + prefix: "/", + }), + applyCompletion, + }); + + assert.strictEqual(editor.capturesViewportInput, false); + editor.handleInput("/"); + await flushAutocomplete(); + assert.strictEqual(editor.capturesViewportInput, true); + + editor.handleInput("\x1b[6~"); + editor.handleInput("\x1b[6~"); + editor.handleInput("\x1b[5~"); + editor.handleInput("\r"); + + assert.strictEqual(submitted, "/command-5"); + }); + it("triggers and debounces symbol completion after CJK punctuation", async (t) => { t.mock.timers.enable({ apis: ["setTimeout"] }); for (const before of [ diff --git a/packages/tui/test/select-list.test.ts b/packages/tui/test/select-list.test.ts index b5c24212c..b2ae5e564 100644 --- a/packages/tui/test/select-list.test.ts +++ b/packages/tui/test/select-list.test.ts @@ -113,4 +113,19 @@ describe("SelectList", () => { assert.ok(rendered[0].includes("…")); assert.equal(visibleIndexOf(rendered[0], "first"), visibleIndexOf(rendered[1], "second")); }); + + it("moves by one visible page with selection page keys (#7629)", () => { + const items = Array.from({ length: 12 }, (_, index) => ({ value: `item-${index}`, label: `Item ${index}` })); + const list = new SelectList(items, 5, testTheme); + const selections: string[] = []; + list.onSelectionChange = (item) => selections.push(item.value); + + for (const key of ["\x1b[6~", "\x1b[6~", "\x1b[6~", "\x1b[6~", "\x1b[5~"]) list.handleInput(key); + + assert.deepStrictEqual(selections, ["item-5", "item-10", "item-11", "item-6"]); + + list.setFilter("no match"); + list.handleInput("\x1b[6~"); + assert.deepStrictEqual(selections, ["item-5", "item-10", "item-11", "item-6"]); + }); }); diff --git a/packages/tui/test/settings-list.test.ts b/packages/tui/test/settings-list.test.ts index 07a7ff09f..08f94edc2 100644 --- a/packages/tui/test/settings-list.test.ts +++ b/packages/tui/test/settings-list.test.ts @@ -55,4 +55,26 @@ describe("SettingsList", () => { assert.deepStrictEqual(changes, [{ id: "tui-mode", value: "fullscreen" }]); }); + + it("moves by one visible page with selection page keys (#7629)", () => { + const list = new SettingsList( + Array.from({ length: 12 }, (_, index) => ({ + id: `setting-${index}`, + label: `Setting ${index}`, + currentValue: String(index), + })), + 5, + testTheme, + () => {}, + () => {}, + ); + + list.handleInput("\x1b[6~"); + assert.match(list.render(80).join("\n"), /> Setting 5/); + list.handleInput("\x1b[6~"); + list.handleInput("\x1b[6~"); + assert.match(list.render(80).join("\n"), /> Setting 11/); + list.handleInput("\x1b[5~"); + assert.match(list.render(80).join("\n"), /> Setting 6/); + }); }); diff --git a/packages/tui/test/tui-alt-screen.test.ts b/packages/tui/test/tui-alt-screen.test.ts index 41402d5c7..6ea551419 100644 --- a/packages/tui/test/tui-alt-screen.test.ts +++ b/packages/tui/test/tui-alt-screen.test.ts @@ -6,6 +6,7 @@ import { findAltScreenSearchMatches, } from "../src/alt-screen-search.ts"; import { Box } from "../src/components/box.ts"; +import { Editor } from "../src/components/editor.ts"; import { HStack } from "../src/components/h-stack.ts"; import { Image } from "../src/components/image.ts"; import { MouseRegion } from "../src/components/mouse-region.ts"; @@ -24,6 +25,7 @@ import { import { renderFakeCursor, type TuiMouseEvent } from "../src/tui.ts"; import { TuiAltScreen } from "../src/tui-alt-screen.ts"; import { stripTerminalSequences, visibleWidth } from "../src/utils.ts"; +import { defaultEditorTheme } from "./test-themes.ts"; import { VirtualTerminal } from "./virtual-terminal.ts"; const OSC133_ZONE_START = "\x1b]133;A\x07"; @@ -2147,4 +2149,81 @@ describe("TuiAltScreen", () => { assert.ok(terminal.getViewport().some((line) => line.includes("↑ ↓"))); tui.stop(); }); + + it("defers viewport keys to a focused list instead of scrolling the transcript (#7629)", async () => { + const terminal = new VirtualTerminal(20, 6); + const tui = new TuiAltScreen(terminal); + const list = new SelectList( + Array.from({ length: 12 }, (_, index) => ({ value: `item-${index}`, label: `item-${index}` })), + 3, + { + selectedPrefix: (text) => text, + selectedText: (text) => text, + description: (text) => text, + scrollInfo: (text) => text, + noMatch: (text) => text, + }, + ); + tui.addChild(new Text(Array.from({ length: 12 }, (_, index) => `line ${index + 1}`).join("\n"), 0, 0)); + tui.addChild(list); + tui.setFocus(list); + tui.start(); + await terminal.waitForRender(); + const topBefore = tui.viewportTop; + + terminal.sendInput("\x1b[6~"); // PageDown + terminal.sendInput("\x1b[1;5H"); // Ctrl+Home + await terminal.waitForRender(); + assert.strictEqual(list.getSelectedItem()?.value, "item-3"); + assert.strictEqual(tui.viewportTop, topBefore); + tui.stop(); + }); + + it("gives viewport keys back to the transcript when editor autocomplete closes (#7629)", async () => { + const terminal = new VirtualTerminal(40, 8); + const tui = new TuiAltScreen(terminal); + const editor = new Editor(tui, defaultEditorTheme); + let submitted = ""; + editor.onSubmit = (text) => { + submitted = text; + }; + editor.setAutocompleteProvider({ + getSuggestions: async () => ({ + items: Array.from({ length: 12 }, (_, index) => ({ + value: `/command-${index}`, + label: `command-${index}`, + })), + prefix: "/", + }), + applyCompletion: (lines, cursorLine, _cursorCol, item) => ({ + lines: lines.map((line, index) => (index === cursorLine ? item.value : line)), + cursorLine, + cursorCol: item.value.length, + }), + }); + tui.addChild(new Text(Array.from({ length: 30 }, (_, index) => `line ${index + 1}`).join("\n"), 0, 0)); + tui.addChild(editor); + tui.setFocus(editor); + tui.start(); + await terminal.waitForRender(); + + terminal.sendInput("/"); + await new Promise((resolve) => setImmediate(resolve)); + await terminal.waitForRender(); + assert.strictEqual(editor.isShowingAutocomplete(), true); + const topWithMenu = tui.viewportTop; + terminal.sendInput("\x1b[6~"); // PageDown pages the menu + await terminal.waitForRender(); + assert.strictEqual(tui.viewportTop, topWithMenu); + + terminal.sendInput("\r"); + await terminal.waitForRender(); + assert.strictEqual(submitted, "/command-5"); + assert.strictEqual(editor.isShowingAutocomplete(), false); + const topWithoutMenu = tui.viewportTop; + terminal.sendInput("\x1b[5~"); // PageUp scrolls the transcript again + await terminal.waitForRender(); + assert.ok(tui.viewportTop < topWithoutMenu); + tui.stop(); + }); }); From cc41246b9d20aedeef5ab3e89138151adfde312f Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 18:10:01 +0000 Subject: [PATCH 105/271] docs(chord): design audit of slices 1-6a, decisions 6a and 7a --- packages/chord/docs/next/decisions.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 9151ef419..565b50cf0 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1769,6 +1769,17 @@ validation and copies the v2 kernel added around Delta. the registry has a value (served and composed once; an empty table composes too, at the serve), bounded by `readyTimeout`; without one in time the object fails `transient` (`unavailable/undeclared`) and is retried when the registry gets its first value (R18); once live, a later failure of the registry changes nothing (last good value). +- **Design audit of slices 1-6a (owner, 2026-10-27; `/tmp/design-audit-{opus,sol}.md`):** the built code follows the + design except for five bugs, fixed after 6b's commit with the auditors' reproductions as tests: on a failed + activation references are released after the natives closed (§6, §13); a dependency going live while the dependent + activates retries it (R18, the wake remembered); a call on a `local` object runs with the host's incarnation (§4); a + retirement of an unloaded document checks its stored version and fingerprint (§9, R15); `ref.retry()` holds a refused + hold again with a new id. Two choices: (6, "a") a link drop while a remote dependency is still loading leaves the + dependent `loading` until `readyTimeout` (§6; supersedes slice-3b race 18's transient failure); (7, "a") `host.hold` + and `host.get` check params at once, also before `load`; "fails once `load` resolved" stays only for a local type + not served here (§7). Stale design text updated: `mapShared` per array, job runs keeping a failed object alive only + while an attempt is live or activating, `ctx.state.change(fn, ctx)`, revocation closes child links, routes take a + context, §22's `revoke`/`chain`/`limit`, native watches collapsing only past 100 pending values. - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the From 97553d10f2cb8be11a4d0fda331fe06967139643 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 18:10:32 +0000 Subject: [PATCH 106/271] docs(chord): design text after the audit of slices 1-6a --- packages/chord/docs/next/design.md | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index 4f920dcee..8c6aed5a5 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -201,7 +201,7 @@ parent link closes. `resolve` returns the link before the worker has loaded (the | `load([{ name, facet }], context)` | facets with their plugin names (the plugin's `package.json` name, npm style; duplicates and invalid names are refused); runs every facet's setup, then waits for the start barrier (section 21); a throwing setup rejects and closes the host; runs once. Loaders are adapters; base code passes its core facet the same way | | `connect(link, { principal, trusted?, codec? }, context)` | the link to the parent, at most one open; once it closed (a drop), `connect` takes the next, and every remote reference over the parent is held again on it. `principal` = who the parent is | | `accept(link, { principal, trusted?, codec? }, context)` | a link from a child; holds and calls on it act as `principal` | -| `route(Kind, { authorize, resolve })` | how to reach hosts of a kind (below); called before `load`, once per kind | +| `route(Kind, { authorize, resolve }, context)` | how to reach hosts of a kind (below); called before `load`, once per kind; work arriving on its links runs under `context` | | `get(Type, params, context, { readyTimeout? })` | a reference once its object is live (section 7) | | `hold(Type, params, context, { readyTimeout? })` | a reference at once that follows its object until released (section 7) | | `close(context)` | closes the links (children, routes, then the parent), then ends every object at once and tears them down, the last to go live first (reason `close`); rejects with an `AggregateError` of the native closes that threw | @@ -452,7 +452,7 @@ drops, folds or resets. server.revoke({ grant: "g1" }, ctx); ``` - `host.revoke(r)` ends every object and closes every link whose principal `p` has `r` in `revokedBy(p)` (compared by + `host.revoke(r)` ends every object and closes every child link whose principal `p` has `r` in `revokedBy(p)` (compared by canonical JSON of each value `revokedBy` returns, computed once per principal; never the host's own principal), and propagates `r` to the children as a `revoke` message on its route links, honoured only on a child's parent link, where each applies its own `revokedBy` (the same app code; nothing is replayed to a child connected later). Holders of a revoked object get `gone` with `forbidden/revoked` (owner, @@ -574,7 +574,8 @@ host.serve(Notes, ({ session }, principal, self) => `remote` type lives while referenced (`linger` keeps it briefly after the last reference). No remote object has host lifetime: its identity includes the principal, and long-lived parts belong in a local shared store. Below, "host-lifetime object" means a `local` object with empty params. Running and finishing job runs keep their - object alive as a count, not as references; when the count reaches zero with no holders, the object ends (§15). + object alive as a count, not as references, while an attempt is live or activating ("Slice 6a small items" (12)); when + the count reaches zero with no holders, the object ends (§15). - **Activation** (one fresh attempt each time, bounded by `readyTimeout`, its wait for the previous instance's teardown included): the factory runs; singleton documents take `initial()` and, on the object's first activation, persisted documents load and its tables' definitions are checked and migrated (slice 6a; the deadline paused during @@ -604,7 +605,10 @@ host.serve(Notes, ({ session }, principal, self) => reload or `ref.retry()`; live dependents stay live. The members freeze only after the commits admitted before the seal adopted and published once, and a retry's factory waits for that (2026-10-27, slice 6a build reviews). - **Waiting for dependencies (R18):** an object whose references are not live yet stays `loading` and goes live as - soon as they are (bounded by `readyTimeout`). A dependency that fails (`broken` or `transient`), or does not become + soon as they are (bounded by `readyTimeout`; a link drop while a dependency is still loading is no failure: the + reference re-holds on the next link and the dependent keeps waiting, design audit 6, 2026-10-27). A dependency + going live while the dependent still activates is remembered and retries it at once if that attempt fails. A + dependency that fails (`broken` or `transient`), or does not become live in time, fails the dependent **`transient`** with the dependency's cause inside, never `broken`: the cause is the dependency's. The dependent keeps its references while failed, and Chord installs their state as an automatic retry trigger: the moment the dependency goes live, the dependent is retried at once (the remaining backoff is @@ -713,7 +717,7 @@ location are dropped. of its variant has a value). A broken failure rejects at once; through transient ones it waits up to its `readyTimeout`, then rejects with the last failure's error; a getter that gives up releases its reference. **`host.hold`** returns a reference at once that follows its object until released (loading, live, failed and - retried as its object is, loading again while its link is down) and never rejects; invalid params and a local type + retried as its object is, loading again while its link is down) and never rejects; invalid params (checked at once, also before `load`: design audit 7, 2026-10-27) and a local type not served here throw at once (before `load` resolved, these fail the reference broken once it did); its `variant` is `undefined` until its first binding (slice 3b). There is no `state: false` option (C8) and no `optional` flag. - **Narrowing by variant:** `ref.variant` is the discriminant (C15: no `inVariant`/`expectVariant`); `RefIn` @@ -1149,7 +1153,7 @@ type NativeWatchCall = { kind: "value"; value: T } | { kind: "failed"; error: map; `r` drops everything before it); past 10,000 pending ops only "replace with the current value" is kept (no size, text or time caps; no folding of list appends or trims; ops come only from Chord's tracker and `diffRevisions`, so there are no defences against hand-written ops). `PendingOps` is Chord's shared Delta code. The - same queue holds a link hold's ops while its window is full (section 3). Natives: the newest value. Durable's + same queue holds a link hold's ops while its window is full (section 3). Natives: every value in order, past 100 pending values one call with the newest (W4). Durable's event stream keeps its own queue (events are not state); `stop()` is idempotent, drops pending frames, never aborts or awaits a running call; `stop()` and `closed` return the end, tagged by `kind` (owner, 2026-10-27): `stopped`, `aborted` (its context aborted), `ended`, `listener_error { error }`, `failed { error }` (a final failure; a recoverable one is a call). Durable's synchronous @@ -1210,8 +1214,8 @@ source (W1); a dirty derive nobody holds recomputes on its next read. A throwing comparisons, not content comparisons: ```ts - const strip = mapShared((n: Note) => ({ text: n.text, at: n.at })); - self.derive([notes], ([n]) => ({ list: n.list.map(strip) })); + const strip = mapShared((n: Note) => ({ text: n.text, at: n.at })); // a mapper over the whole array + self.derive([notes], ([n]) => ({ list: strip(n.list) })); // one appended note: one new object, 2,000 identical references, diffRevisions emits one insert ``` @@ -1346,7 +1350,7 @@ host.serve(ChatStore, ({ conversation }, self) => { // local: no princi const shell = self.job(ShellJob, { run: async ({ command }, ctx) => { const out = new OutputBuffer({ maxLines: 2000, retain: "tail" }); - const progress = throttle(100, () => ctx.state.change((d) => { d.output = out.snapshot().text; })); + const progress = throttle(100, () => ctx.state.change((d) => { d.output = out.snapshot().text; }, ctx)); try { const r = await runProcess(command, ctx.abortSignal, (chunk) => { out.push(chunk); progress.schedule(); }); return { exit: r.code, output: out.snapshot().text }; @@ -1385,7 +1389,7 @@ host.serve(ChatStore, ({ conversation }, self) => { // local: no princi the run, both or neither). `run` starts after the commit. - **2. Run:** `run(args, ctx)` may do anything, outside transactions. `ctx` is a `Ctx` (the object's principal for a remote type, the host's for a local one) plus `jobId` and `state`: `ctx.state.value` (synchronous) and - `ctx.state.change(fn)` (a transaction on the owner's line on the run's document, without the run's abort signal). + `ctx.state.change(fn, ctx)` (a transaction on the owner's line on the run's document, without the run's abort signal). State changes are accepted until `run` settles, also after an abort (so output written while stopping still reaches `outcome.state`), and are refused (`released`) afterwards, never recreating a retired document. Each change is a commit and, while someone watches, a message: authors throttle frequent changes themselves (Chord does not coalesce them); awaiting or @@ -1785,7 +1789,8 @@ validation, no clone of snapshots or ops. | Kind | Direction | Fields | | --- | --- | --- | | `hello` | both, first | `protocol`, `incarnation`, `calls` (the call window N) | - | `hold` | to provider | `origin?`, `hold`, `type`, `version`, `fingerprint`, `params` (part), `address?`, `principal?`, `incarnation?` | + | `hold` | to provider | `origin?`, `hold`, `type`, `version`, `fingerprint`, `params` (part), `address?`, `principal?`, `incarnation?`, `chain?` (activation chain, Q5), `limit?` (decoded size limit, 4b) | + | `revoke` | parent to child (route link) | the revocation value (4b) | | `release`, `retry` | to provider | `origin?`, `hold` | | `call` | to provider | `origin?`, `call`, `hold`, `method` (index), `args?` (part; absent: no arguments), `segments?` | | `abort` | to provider | `origin?`, `call` (named `cancel` until 2026-10-27) | From ec2c8ed311eee3d3100fe35eb01fb4d0aaa7f0c2 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 18:51:30 +0000 Subject: [PATCH 107/271] docs(chord): slice 6b reopen and commit numbers accepted --- packages/chord/docs/next/decisions.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 565b50cf0..7e7a96838 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1780,6 +1780,11 @@ validation and copies the v2 kernel added around Delta. not served here (§7). Stale design text updated: `mapShared` per array, job runs keeping a failed object alive only while an attempt is live or activating, `ctx.state.change(fn, ctx)`, revocation closes child links, routes take a context, §22's `revoke`/`chain`/`limit`, native watches collapsing only past 100 pending values. +- **Slice 6b: two numbers accepted (owner, 2026-10-27: "yeah the reopen times are ok"):** reopening a store with + `StoreStorage` costs about 0.15 ms more than `SqliteStorage` once per open (0.28 to 0.43 ms: six table declarations, + the allocator read, compiling the reads); the one-entry commit's mean is raised by one outlier among 20 samples while + its p75 is equal or lower. Neither is a hot path or a steady regression. `StoreStorage.close` waits for its commits + only (its reads go straight to the store, which stays open). - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the From 33385f9572303e79bfb3811cc92cc8dd65e22261 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 18:53:50 +0000 Subject: [PATCH 108/271] docs(chord): attribution of the StoreStorage.close rule --- packages/chord/docs/next/decisions.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 7e7a96838..7c7b30647 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1783,8 +1783,9 @@ validation and copies the v2 kernel added around Delta. - **Slice 6b: two numbers accepted (owner, 2026-10-27: "yeah the reopen times are ok"):** reopening a store with `StoreStorage` costs about 0.15 ms more than `SqliteStorage` once per open (0.28 to 0.43 ms: six table declarations, the allocator read, compiling the reads); the one-entry commit's mean is raised by one outlier among 20 samples while - its p75 is equal or lower. Neither is a hot path or a steady regression. `StoreStorage.close` waits for its commits - only (its reads go straight to the store, which stays open). + its p75 is equal or lower. Neither is a hot path or a steady regression. Accepted by the redesign session in fix + round 1 and shown to the owner (not his quote): `StoreStorage.close` waits for its commits only (its reads go straight + to the store, which stays open). - **The performance and memory bar, and converging reviews (owner, 2026-10-27: "yes"):** supersedes "as good as HEAD in everything" for Chord v2 slices and pi changes. CPU: each hot path (calls, changes, watches, links) within about 5% of HEAD, measured in bursts with alternating processes pinned to one CPU; worse is fixed or goes to the From 4e5cd1a973e83ff454ea6f06854664a5d6612462 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 19:03:49 +0000 Subject: [PATCH 109/271] docs(chord): slice 6b resume watch cost accepted --- packages/chord/docs/next/decisions.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 7c7b30647..c437875fe 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1780,7 +1780,9 @@ validation and copies the v2 kernel added around Delta. not served here (§7). Stale design text updated: `mapShared` per array, job runs keeping a failed object alive only while an attempt is live or activating, `ctx.state.change(fn, ctx)`, revocation closes child links, routes take a context, §22's `revoke`/`chain`/`limit`, native watches collapsing only past 100 pending values. -- **Slice 6b: two numbers accepted (owner, 2026-10-27: "yeah the reopen times are ok"):** reopening a store with +- **Slice 6b: numbers accepted (owner, 2026-10-27: "yeah the reopen times are ok"; "i accept the 12%": resuming a + conversation watch 20 entries back costs about 12% more CPU on `StoreStorage`, once per reconnect, about 0.1 ms per + mount, from the extra promise per read in the fork scan):** reopening a store with `StoreStorage` costs about 0.15 ms more than `SqliteStorage` once per open (0.28 to 0.43 ms: six table declarations, the allocator read, compiling the reads); the one-entry commit's mean is raised by one outlier among 20 samples while its p75 is equal or lower. Neither is a hot path or a steady regression. Accepted by the redesign session in fix From b5fc6dc84f28a103f85db6f201f40c0a32ee76b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Hou=C5=A1ka?= Date: Wed, 28 Oct 2026 20:09:32 +0100 Subject: [PATCH 110/271] fix(coding-agent): strip private-parameter CSI sequences in stripAnsi stripAnsi treated <, = and > after ESC [ as final bytes, so ESC[<35;59;1M (SGR mouse report) left 35;59;1M in tool output. Accept one private prefix byte only directly after a CSI introducer (ESC [ or 0x9B). After a bare ESC these bytes stay final bytes, so ESC= and ESC> (keypad modes) keep the digits of the text that follows. closes #11377 --- packages/coding-agent/CHANGELOG.md | 1 + packages/coding-agent/src/utils/ansi.ts | 7 +++-- packages/coding-agent/test/ansi-utils.test.ts | 27 ++++++++++++++++++- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 1ed5d448d..31571b8e9 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -26,6 +26,7 @@ - Fixed streamed tool calls re-running their call renderer on every argument delta; the interactive mode now renders the latest arguments once per frame, so a large streamed `write` no longer re-highlights its whole content per delta ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed the login dialog opening sign-in URLs that are not web URLs, and printing control characters from sign-in, device-code, and info URLs, which could end a terminal hyperlink and send escape sequences to the terminal - Fixed `tui.select.pageUp` and `tui.select.pageDown` being ignored in slash command and file autocomplete, `/settings`, and the model, scoped models, login, fork, `/mcp`, and llama.cpp selectors. In fullscreen mode, PageUp/PageDown now move an open list or autocomplete menu instead of scrolling the transcript, and a page key bound to Ctrl+V pages autocomplete instead of pasting an image ([#7629](https://github.com/earendil-works/pi/issues/7629), based on [#7680](https://github.com/earendil-works/pi/pull/7680) by [@muyiyr](https://github.com/muyiyr)) +- Fixed tool output and `!` command output showing the parameters of control sequences with a `<`, `=` or `>` prefix as text, for example `35;59;1M` from a mouse report `ESC[<35;59;1M` or `0;276;0c` from a terminal reply `ESC[>0;276;0c`. Text after the keypad-mode sequences `ESC=` and `ESC>` is still kept ([#11377](https://github.com/earendil-works/pi/issues/11377)) ## [1.4.0] - 2026-10-21 diff --git a/packages/coding-agent/src/utils/ansi.ts b/packages/coding-agent/src/utils/ansi.ts index cad5eb65b..6d49fc1bc 100644 --- a/packages/coding-agent/src/utils/ansi.ts +++ b/packages/coding-agent/src/utils/ansi.ts @@ -32,8 +32,11 @@ const ST = "(?:\\u0007|\\u001B\\u005C|\\u009C)"; // OSC sequences: ESC ] ... ST const OSC_START = "\\u001B\\]"; -// CSI and related: ESC/C1, optional intermediates, optional params (supports ; and :), then final byte -const CSI_START = "[\\u001B\\u009B][[\\]()#;?]*(?:\\d{1,4}(?:[;:]\\d{0,4})*)?"; +// CSI and related: ESC/C1, optional intermediates, optional params (supports ; and :), then final byte. +// A private prefix (<, =, >) is accepted only right after a real CSI introducer (ESC [ or 0x9B), so +// ESC[<35;59;1M (SGR mouse report) is stripped whole. After a bare ESC these bytes stay final bytes: +// ESC = and ESC > (keypad modes) must not swallow the digits of the text that follows. +const CSI_START = "(?:(?:\\u001B\\[|\\u009B)[<=>]|[\\u001B\\u009B][[\\]()#;?]*)(?:\\d{1,4}(?:[;:]\\d{0,4})*)?"; const CSI_FINAL = "[\\dA-PR-TZcf-nq-uy=><~]"; // Complete sequences. OSC is non-greedy until the first ST. diff --git a/packages/coding-agent/test/ansi-utils.test.ts b/packages/coding-agent/test/ansi-utils.test.ts index 15bb43140..f108e482c 100644 --- a/packages/coding-agent/test/ansi-utils.test.ts +++ b/packages/coding-agent/test/ansi-utils.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { stripAnsi } from "../src/utils/ansi.ts"; +import { splitIncompleteAnsiSuffix, stripAnsi } from "../src/utils/ansi.ts"; function referenceAnsiRegex(): RegExp { const ST = "(?:\\u0007|\\u001B\\u005C|\\u009C)"; @@ -107,4 +107,29 @@ describe("stripAnsi", () => { const input = "a\x1b[31mred\x1b[0m\x1b]8;;https://example.com\x07link\x1b]8;;\x07z"; expect(stripAnsi(input)).toBe("aredlinkz"); }); + + // https://github.com/earendil-works/pi/issues/11377 + it("strips private-parameter CSI sequences with their parameters", () => { + expect(stripAnsi("\x1b[<35;59;1Mstep 1")).toBe("step 1"); + expect(stripAnsi("a\x1b[<0;10;20mb")).toBe("ab"); + expect(stripAnsi("a\x1b[>0;276;0cb")).toBe("ab"); + expect(stripAnsi("a\x1b[>cb")).toBe("ab"); + expect(stripAnsi("a\x1b[=5hb")).toBe("ab"); + expect(stripAnsi("a\x9b<35;59;1Mb")).toBe("ab"); + }); + + // https://github.com/earendil-works/pi/issues/11377 + it("keeps digits after bare ESC keypad-mode sequences", () => { + expect(stripAnsi("\x1b[?1h\x1b=123 tests passed")).toBe("123 tests passed"); + expect(stripAnsi("\x1b[?1l\x1b>42 files")).toBe("42 files"); + expect(stripAnsi("\x1b<10 items")).toBe("10 items"); + }); +}); + +describe("splitIncompleteAnsiSuffix", () => { + // https://github.com/earendil-works/pi/issues/11377 + it("holds back unfinished private-parameter CSI sequences", () => { + expect(splitIncompleteAnsiSuffix("x\x1b[<35;59")).toEqual({ complete: "x", pending: "\x1b[<35;59" }); + expect(splitIncompleteAnsiSuffix("x\x1b=")).toEqual({ complete: "x\x1b=", pending: "" }); + }); }); From 14b4c56e5796e995706fec5b5124181a0f60bc35 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 19:18:27 +0000 Subject: [PATCH 111/271] feat(chord,durable): Chord v2 slice 6b, Durable and jobs on the store StoreStorage (Durable's storage on Chord v2's store, @earendil-works/pi-durable/storage/store) passes Durable's storage conformance suite and runs Durable's harness tests (npm run test:store). The store gets owner names and claims, one id allocator, Durable's Storage semantics for refused steps, history none/latest/rewindable with reads at a point, archive revisions and applyBatch. self.follow follows Durable's synchronous observers as Chord sources. Persisted jobs: records and state in the owner's store, restarts as Durable's tasks, abort stored, deferred state. Chord's ./next export returns for the adapter surface. --- packages/chord/CHANGELOG.md | 1 + packages/chord/docs/next/benchmarks.md | 93 ++ packages/chord/docs/next/slices.md | 19 +- packages/chord/package.json | 5 + .../chord/src/next/documents/definition.ts | 9 + .../chord/src/next/documents/documents.ts | 18 +- packages/chord/src/next/documents/index.ts | 2 +- packages/chord/src/next/documents/internal.ts | 10 +- .../chord/src/next/documents/keyed-slot.ts | 22 +- .../chord/src/next/documents/persistence.ts | 67 +- packages/chord/src/next/documents/slot.ts | 38 +- .../chord/src/next/documents/transaction.ts | 65 +- packages/chord/src/next/host/attempt.ts | 85 +- packages/chord/src/next/host/follow.ts | 248 ++++ packages/chord/src/next/host/host.ts | 10 +- packages/chord/src/next/host/index.ts | 1 + packages/chord/src/next/host/object.ts | 70 +- packages/chord/src/next/host/routines.ts | 1 + packages/chord/src/next/index.ts | 50 + packages/chord/src/next/jobs/README.md | 14 +- packages/chord/src/next/jobs/definition.ts | 68 +- packages/chord/src/next/jobs/index.ts | 2 +- packages/chord/src/next/jobs/job.ts | 174 ++- packages/chord/src/next/jobs/persisted.ts | 100 ++ packages/chord/src/next/jobs/records.ts | 524 ++++++-- packages/chord/src/next/jobs/run-states.ts | 243 ++++ packages/chord/src/next/jobs/run.ts | 141 ++- packages/chord/src/next/jobs/running.ts | 3 +- packages/chord/src/next/jobs/stored.ts | 106 ++ packages/chord/src/next/remote/hold.ts | 1 + packages/chord/src/next/state/batch.ts | 27 + packages/chord/src/next/state/watchable.ts | 17 + packages/chord/src/next/store/README.md | 12 + packages/chord/src/next/store/adapter.ts | 152 +++ packages/chord/src/next/store/allocator.ts | 33 + packages/chord/src/next/store/applier.ts | 297 ++++- packages/chord/src/next/store/apply-batch.ts | 99 ++ packages/chord/src/next/store/archive.ts | 168 ++- packages/chord/src/next/store/claims.ts | 57 + packages/chord/src/next/store/engine.ts | 462 +++++-- packages/chord/src/next/store/export.ts | 39 +- packages/chord/src/next/store/import.ts | 105 +- packages/chord/src/next/store/outbox.ts | 17 +- packages/chord/src/next/store/refusal.ts | 43 + packages/chord/src/next/store/schema.ts | 66 +- packages/chord/src/next/store/sql.ts | 30 +- packages/chord/src/next/store/step.ts | 45 +- packages/chord/src/next/store/store.ts | 296 ++++- packages/chord/src/next/store/table-writes.ts | 12 +- packages/chord/test/next/bench/jobs.ts | 164 +++ packages/chord/test/next/bench/store.ts | 8 +- .../test/next/documents/persistence.test.ts | 34 +- .../test/next/documents/transaction.test.ts | 30 +- packages/chord/test/next/examples/19-jobs.ts | 2 +- .../test/next/examples/28-persisted-jobs.ts | 125 ++ packages/chord/test/next/host/attempt.test.ts | 4 +- packages/chord/test/next/host/follow.test.ts | 345 ++++++ packages/chord/test/next/host/host.test.ts | 17 +- .../test/next/host/member-source.test.ts | 1 + .../chord/test/next/jobs/definition.test.ts | 31 +- packages/chord/test/next/jobs/job.test.ts | 20 +- .../chord/test/next/jobs/persisted.test.ts | 1066 +++++++++++++++++ packages/chord/test/next/jobs/records.test.ts | 4 +- packages/chord/test/next/jobs/run.test.ts | 18 +- .../chord/test/next/state/watchable.test.ts | 1 + .../chord/test/next/store/allocator.test.ts | 24 + .../chord/test/next/store/applier.test.ts | 20 +- .../chord/test/next/store/apply-batch.test.ts | 290 +++++ .../chord/test/next/store/catalog.test.ts | 12 +- packages/chord/test/next/store/claims.test.ts | 72 ++ packages/chord/test/next/store/engine.test.ts | 181 ++- packages/chord/test/next/store/export.test.ts | 85 +- packages/chord/test/next/store/import.test.ts | 92 +- packages/chord/test/next/store/jsonl.test.ts | 4 +- packages/chord/test/next/store/outbox.test.ts | 9 +- packages/chord/test/next/store/query.test.ts | 28 +- .../chord/test/next/store/refusal.test.ts | 65 + packages/chord/test/next/store/sql.test.ts | 2 +- packages/chord/test/next/store/store.test.ts | 191 ++- .../test/next/store/table-changes.test.ts | 2 +- .../test/next/store/table-handle.test.ts | 8 +- .../test/next/store/table-migration.test.ts | 2 +- .../test/next/store/table-writes.test.ts | 9 +- packages/chord/test/next/support.ts | 64 +- packages/chord/test/sqlite/cloudflare.test.ts | 74 ++ packages/chord/test/sqlite/node.test.ts | 47 +- packages/durable/CHANGELOG.md | 1 + packages/durable/README.md | 2 + packages/durable/package.json | 9 +- packages/durable/src/storage/store/commit.ts | 327 +++++ .../durable/src/storage/store/documents.ts | 143 +++ packages/durable/src/storage/store/index.ts | 1 + packages/durable/src/storage/store/queries.ts | 137 +++ packages/durable/src/storage/store/storage.ts | 557 +++++++++ packages/durable/src/storage/store/tables.ts | 122 ++ .../src/testing/storage-conformance.ts | 24 + .../durable/test/conversation-view-bench.ts | 10 +- .../test/examples/35-durable-on-the-store.ts | 108 ++ .../durable/test/examples/36-durable-views.ts | 91 ++ packages/durable/test/storage-memory.ts | 11 +- packages/durable/test/storage.bench.ts | 15 +- packages/durable/test/store-storage-node.ts | 23 + .../durable/test/store-storage-support.ts | 46 + packages/durable/test/store-storage.test.ts | 592 +++++++++ packages/durable/vitest.store.config.ts | 47 + tsconfig.json | 1 + 106 files changed, 8714 insertions(+), 871 deletions(-) create mode 100644 packages/chord/src/next/host/follow.ts create mode 100644 packages/chord/src/next/index.ts create mode 100644 packages/chord/src/next/jobs/persisted.ts create mode 100644 packages/chord/src/next/jobs/run-states.ts create mode 100644 packages/chord/src/next/jobs/stored.ts create mode 100644 packages/chord/src/next/store/adapter.ts create mode 100644 packages/chord/src/next/store/allocator.ts create mode 100644 packages/chord/src/next/store/apply-batch.ts create mode 100644 packages/chord/src/next/store/claims.ts create mode 100644 packages/chord/src/next/store/refusal.ts create mode 100644 packages/chord/test/next/bench/jobs.ts create mode 100644 packages/chord/test/next/examples/28-persisted-jobs.ts create mode 100644 packages/chord/test/next/host/follow.test.ts create mode 100644 packages/chord/test/next/jobs/persisted.test.ts create mode 100644 packages/chord/test/next/store/allocator.test.ts create mode 100644 packages/chord/test/next/store/apply-batch.test.ts create mode 100644 packages/chord/test/next/store/claims.test.ts create mode 100644 packages/chord/test/next/store/refusal.test.ts create mode 100644 packages/durable/src/storage/store/commit.ts create mode 100644 packages/durable/src/storage/store/documents.ts create mode 100644 packages/durable/src/storage/store/index.ts create mode 100644 packages/durable/src/storage/store/queries.ts create mode 100644 packages/durable/src/storage/store/storage.ts create mode 100644 packages/durable/src/storage/store/tables.ts create mode 100644 packages/durable/test/examples/35-durable-on-the-store.ts create mode 100644 packages/durable/test/examples/36-durable-views.ts create mode 100644 packages/durable/test/store-storage-node.ts create mode 100644 packages/durable/test/store-storage-support.ts create mode 100644 packages/durable/test/store-storage.test.ts create mode 100644 packages/durable/vitest.store.config.ts diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index 5724e1a42..f475c4c8a 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -13,6 +13,7 @@ ### Added +- Chord v2 has a package export again, `@earendil-works/chord/next` (experimental): every layer's API for apps (contracts, data, documents, hosts, jobs, links, state, the store), and, marked internal, the store's surface for adapters (`StoreOwnerTables`, `OwnerTable`, the documents engine's reads and writes, the commit step, `applyBatch`). v2's store gains one id allocator for every id in it, claims per table and per document namespace, Durable's Storage semantics for a refused commit (it rejects that commit only; an unknown outcome stops the store), document histories `none`, `latest` and `rewindable` with reads at a point, archives carrying document revisions, and batch apply; v2 objects gain `self.follow` (another runtime's synchronous observers as sources) and persisted jobs (`persist: { resume }`, a stored `abort(id, ctx)`, deferred state changes). - Chord v2, experimental and not exported yet: the next object runtime in `src/next` (contracts, hosts and objects, documents, derives, links and relays, routines and jobs, the store), with its tests and runnable examples in `test/next` and its design in `docs/next` (`design.md`, the decision log `decisions.md`, the slice plan `slices.md`). Its API changes with every slice until it gets an entry point. - `NativeWatchable`: an in-process value that changes and can be watched (`value`, `watch(context)` returning a `NativeWatchHandle`), the in-process counterpart of `Watchable`. - `WatchHandle` and `WatchEnd`, the one watch API, moved from Durable: a serialized observation with `value`, `start(listener)`, `stop()`, and `closed`; the listener gets a `WatchChange` (`{ value, ops, retired }`) and the context. `WatchEnd` reasons are `stopped`, `cancelled`, `ended`, `listener_error`, and `failed`. diff --git a/packages/chord/docs/next/benchmarks.md b/packages/chord/docs/next/benchmarks.md index 4add7d22b..8e63d8d36 100644 --- a/packages/chord/docs/next/benchmarks.md +++ b/packages/chord/docs/next/benchmarks.md @@ -490,3 +490,96 @@ Observations: - Late migration chunks cost what early ones do (paging forward by the last key keeps each chunk's read bounded); the spec's estimate of about 15 µs per migrated row holds with room. - A bulk write's step is the one place a single commit holds every owner's commits for milliseconds; the guide says so. + +## Slice 6b: Durable on the store, persisted jobs (2026-10-28) + +Node 24.4.1, one CPU (`taskset -c 6`), SQLite files in `tmpdir()`. HEAD is `b2fcfee08`, extracted at a path of the same +length (`/home/badlogic/pi-6h`); runs alternate HEAD and the build. + +**Durable's storage benchmark** (`packages/durable/test/storage.bench.ts`, `TIMING_SCALE`), `StoreStorage` (outbox off) +against `SqliteStorage` in the same process, mean ms per operation over three runs (p75 where the mean is one outlier): + +| Case | SqliteStorage | StoreStorage | | +| --- | --- | --- | --- | +| exact entry lookup | 0.0030 | 0.0030 | 0% | +| entry page scan (100) | 0.0942 | 0.0940 | 0% | +| filtered task scan (50) | 0.0206 (p75 0.0200) | 0.0220 (p75 0.0207) | +7% mean, +3% p75 | +| exact document address among many | 0.0037 | 0.0037 | 0% | +| document replay tail (0) | 0.0089 (p75 0.0084) | 0.0092 (p75 0.0085) | +3% mean, +1% p75 | +| document replay tail (16 / 128 / 1024) | 0.0300 / 0.1726 / 1.224 | 0.0270 / 0.1522 / 1.135 | -10% / -12% / -7% | +| ancient / recent historical read | 0.1681 / 0.1611 | 0.1541 / 0.1513 | -8% / -6% | +| fork-depth history scan (100) | 0.1177 (p75 0.1156) | 0.1208 (p75 0.1159) | +3% mean, 0% p75 | +| fork-depth head lookup | 0.0361 | 0.0332 | -8% | +| commit one entry (20 samples) | 0.0439 (p75 0.0416) | 0.0601 (p75 0.0373) | +37% mean, -10% p75 | +| commit 100 entries | 0.743 | 0.620 | -17% | +| commit mixed entry/task/submission/document | 0.195 | 0.187 | -4% | +| reopen and first exact read | 0.294 | 0.443 | +51% | + +Warm micro-benchmarks behind the same cases (each read 20,000 times; commits 10,000 after 2,000 warm-up): page scan +91.2 vs 90.6 µs, task scan 19.9 vs 20.1 µs, fork-depth scan 114.1 vs 114.9 µs, head lookup 35.3 vs 32.9 µs, document +address 3.57 vs 3.68 µs, replay tail (0) 8.41 vs 8.44 µs; one entry per commit 29.6 vs 27.8 µs mean, 18.4 vs 17.1 µs +median. What it took, beyond the first build's compiled reads, point lookups without a bound `LIMIT`, version columns +selected only past version 1 and the counting read guard: + +- **Reads select only the record.** A scan selected an entry's `commit_seq` too (only `entry(id)` needs it): 7% of a + page scan. +- **No index for head markers on every entry.** `entries.by_head (conversation_id, has_head, id)` cost about 5.5 µs per + commit (a raw insert: 39.7 vs 45.2 µs); `SqliteStorage` has a partial index (`WHERE head IS NOT NULL`), which the + table vocabulary does not have. The head markers are an `entry_heads` table instead, written only with an entry that + has one: a commit without one writes no second index, and a head lookup reads the newest head's id, then the entry. +- **A document record is one JSON text.** `meta` keeps the whole Durable record, its creation seq included; a read + selects `meta` (and `retired_at` only at a point). +- **A store already at the newest schema opens with one read** (schema version, encoding and metadata row), without + the migrations' write transaction: 84 → 71 µs on a fresh connection. + +Commit one entry: the mean is one sample (Maglev compiling the commit path on the first fixtures: 130 and 386 µs at +fixtures 25 and 30 of the first store round of a 60-fixture replay, none in the second round, no GC or deopt traced); +the median and p75 are lower than `SqliteStorage`'s. Reopen: a fresh connection on a copied file, measured in parts +(`openNodeSqliteDatabase` is the same for both): `SqliteStorage.open` 42-60 µs; `openStore` 63-71 µs (the schema read, +the catalog with its JSON parse) plus `StoreStorage.open` 75 µs (declaring six tables: their SQL rendered, claims; the +catalog check of the table preparation; `max(record_ids.id)` for the allocator, as §2.1 asks; compiling the point +reads), first read +3 µs: about 0.1-0.15 ms once per open, cold code, not on any hot path. + +**Statements per commit** (cost guard in `packages/durable/test/store-storage.test.ts`): an entry append and a document +delta take 4 each (`SqliteStorage`: 5 and about 7). + +**Memory** (`packages/durable/test/storage-memory.ts`): heap after seeding 0.16 → 0.30 MiB (1k), 0.17 → 0.32 MiB (10k): +a fixed cost of the store (catalog, compiled statements, claims, the store's code), the same at both scales; heap after +reads 0.13 → 0.16-0.18 MiB. Disk 0.61 → 0.58 MiB (1k), 4.34 → 4.38 MiB (10k). + +**Chord's commit gate** (`test/next/bench/store.ts`), HEAD → build, two alternating runs each, medians: + +| Case | HEAD | build | +| --- | --- | --- | +| (a) one document delta per commit | 13.6-13.7 µs | 13.8-14.0 µs (+2%) | +| (b) one table put per commit | 12.8-13.3 µs | 13.0-13.2 µs | +| (c) whole transactions (no watch / watched) | 17.2-17.3 / 17.1-17.2 µs | 17.4-17.7 / 17.2-17.7 µs (+2%) | +| (d) one in-memory document (median / burst) | 2.1 / 2.9-3.0 µs | 2.1 / 4.9 µs | +| heap after GC | 2940 KiB | 2996 KiB | + +The revisions index is a partial index on bases (`(document_id, seq) WHERE kind = 'base'`): a delta writes no index +entry (the first build's `(document_id, kind, seq)` cost (a) 15%). (d)'s burst is not the in-memory path: run alone +(only section (d), three alternating runs) HEAD gives 4.9-5.1 µs and the build 4.9-5.0 µs per commit, and 40 bursts of +1,000 in a fresh process 2.17 vs 2.18-2.38 µs; in the full gate, HEAD's in-memory path runs warm from the persisted +sections before it, the build's does not share as much of their code (`StoredChange`'s variants). + +**Durable's conversation view benchmark** (`test/conversation-view-bench.ts`, `PI_DURABLE_BENCH_STORAGE`), two runs +each, `SqliteStorage` → `StoreStorage`: append one entry per commit 56.1-57.2 → 56.2-57.0 µs with one watch, 61.0-62.1 +→ 62.7-63.1 µs with four (+2%); bursts of 50 884-888 → 891 µs; `pi.live` only 62.7-63.0 → 65.1-66.6 µs (+4-6%); a +long-lived watch over 20,000 appends 0.83-0.85 → 0.84-0.85 s CPU; heaps 21.5 → 21.7 MiB. + +**Fix round 1** (2026-10-28, after the build reviews): `StoreStorage`'s reads take one promise besides the store's guard +(no `#call` wrapper; `close` waits for its commits, a read changes nothing), their request errors in a tagged result. +Warm micro-benchmark (20,000 runs, two runs each, `SqliteStorage` → `StoreStorage`): filtered task scan 19.8 → 19.8-20.1 +µs, `entry(conversation, id)` over the fork chain 23.1 → 21.9-22.3 µs. `storage.bench.ts` (two runs): task scan mean +0.0207 → 0.0217-0.0218 ms (p75 0.0200-0.0203 → 0.0204-0.0206), fork-depth scan 0.1177-0.1188 → 0.1210-0.1242 (p75 +0.1154-0.1167 → 0.1160-0.1193), the rest as above. +`conversation-view-bench` "resume 20 back", three runs: CPU 4.23-4.47 → 4.50-5.14 ms for a 3 ms scenario (one sample +each; the head lookup and scans under it are at parity or faster in the micro-benchmark). The persisted-job fixes +(one invocation per call of `run`, held state folded in a `PendingOps`): start 55.5 → 55.2 µs, immediate state change +21.3 → 21.2 µs, deferred 3.2 → 3.5 µs (the fold), in memory unchanged. + +**Persisted jobs** (`test/next/bench/jobs.ts`, 7 bursts of 500, medians): a start (resolved after its commit) 8.4 µs in +memory, 55.5 µs persisted (two commits: the record and state created, then the reservation); a state change inside +`run` 2.7 µs in memory, 21.3 µs persisted (one delta commit, as the gate's (c)), 3.2 µs deferred (stored with the +outcome). diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index c2498ecba..87d395336 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -27,7 +27,7 @@ slice spec's section citations are for orientation, not a reading limit (owner, - **L1** (`src/next/contracts`, `src/next/data`): contracts, params and identity, argument checks, principal keys, errors. `index.ts` for apps, `internal.ts` for Chord's own code. -- **Slices 1, 2, 3a, 3b, 4a, 4b, 5, 6a** (committed; 5 `4a5b45c45`, 6a 2026-10-27): `src/next/documents`, `state`, `host`, `jobs`, `link`, +- **Slices 1, 2, 3a, 3b, 4a, 4b, 5, 6a, 6b** (committed; 5 `4a5b45c45`, 6a and 6b 2026-10-27/28): `src/next/documents`, `state`, `host`, `jobs`, `link`, `remote`, `store`; examples 01-25; benchmarks in `benchmarks.md`. - **Chord's shared code** (upstream pi): `src/watch.ts` (`Watch`, `NativeWatch`, `WatchHandle`, `WatchEnd`, `NativeWatchable`), Delta (`track`, `diffRevisions`, `applyImmutable`, encoders, `PendingOps`, `appendText`), @@ -161,19 +161,6 @@ deleted (the slice's spec and code are the record). Moved here from `carry-over. - **`map(bytes())` is refused at declaration** ("unsupported keyword `~refine`"): TypeBox's `Record` copies the bytes schema's check as an enumerable key, which the snapshot walk refuses. An L1 bug found in slice 3a, not fixed yet. -### Slice 6b: Durable and jobs on the store (from slice 5 and L3a-d) - -Specified in `slice-6b.md` (decided 2026-10-27, D1-D9, R1-R4, "owner names and claims"); each entry below is taken there. - -- **Persisted jobs** (from slice 5): `persist: { resume }`, records and state documents in the owner's store (D5), the - restart limit (D6), `abort` stored (D7), deferred state (D8), the seal as a clean stop (R3), the `finish` failure - count persisted, the job fingerprint and the start trace carrier stored. -- **`StoreStorage`** (D1-D4, R1, R2, claims): Durable's `Storage` on the store, passing its conformance suite; one id - allocator (D3); Durable's checks in the step; fork copies resolved before the step (S11); `adopt` in the host's batch. -- **`self.follow`** (D9, R4): a Chord source over Durable's synchronous observers. Reaching Durable from other objects - (the harness in a registry entry) moves to slice 8. -- **The applier for batch apply**: every `StoreWrite` kind, `applyBatch`, one code path with import. - ### Slice 7: reload and rollout - **A native restart drains first** (owner, 2026-10-26): park new calls, await running calls (bounded by @@ -185,6 +172,9 @@ Specified in `slice-6b.md` (decided 2026-10-27, D1-D9, R1-R4, "owner names and c per routine invocation; holds unsubscribed at seal and re-bound by member name; keyed members' versions mapped per key; in-memory `migrate` wired to reload (6a builds the slot version check and the migrate); `rebuild` (key and column-type changes) if a case needs it. +- **From slice 6b:** a persisted run of another job version or fingerprint stays `blocked { version }` with its record + as stored (resolve it on reload); blocked persisted runs are retried on reload; runs of jobs the code no longer + declares (below). - **A run whose job the new code no longer declares** (slice 6a builder, 2026-10-27): 6a keeps it in its phase, waiting, not invoked (unreachable without reload); slice 7 decides `blocked` with a reason (design §15's `blocked { version }` family). @@ -233,6 +223,7 @@ Specified in `slice-6b.md` (decided 2026-10-27, D1-D9, R1-R4, "owner names and c ### Slice 9: telemetry - Spans for routine calls and job runs (from slice 5). +- The use of a persisted run's stored start trace carrier (`trace`, stored since slice 6b). - Round-3 cut to decide: C11 (`ctx.trace`). diff --git a/packages/chord/package.json b/packages/chord/package.json index 57dbb5406..895c16523 100644 --- a/packages/chord/package.json +++ b/packages/chord/package.json @@ -36,6 +36,11 @@ "types": "./dist/telemetry-sqlite/index.d.ts", "import": "./dist/telemetry-sqlite/index.js" }, + "./next": { + "source": "./src/next/index.ts", + "types": "./dist/next/index.d.ts", + "import": "./dist/next/index.js" + }, "./sqlite": { "source": "./src/sqlite/index.ts", "types": "./dist/sqlite/index.d.ts", diff --git a/packages/chord/src/next/documents/definition.ts b/packages/chord/src/next/documents/definition.ts index 77906479b..e9a2eecf1 100644 --- a/packages/chord/src/next/documents/definition.ts +++ b/packages/chord/src/next/documents/definition.ts @@ -12,6 +12,15 @@ import { validatorFor } from "../data/internal.ts"; /** A document root: a JSON object. */ export type JsonObject = { [key: string]: JsonValue }; +/** + * When a commit reaches the store (Durable's `CommitPersistence`): `immediate`, the default, before the commit + * resolves; `deferred` (a persisted run's state change, D8): adopted and published at once, stored with the next + * immediate change, the run's outcome, the object's close, or about `withinMs` (default 1,000 ms) later. + */ +export type CommitPersistence = + | { readonly kind: "immediate" } + | { readonly kind: "deferred"; readonly withinMs?: number }; + /** What a persisted document's `checkpointWhen` learns of its stored revisions. */ export type CheckpointInfo = { /** The deltas stored after its newest base, without the change being decided. */ diff --git a/packages/chord/src/next/documents/documents.ts b/packages/chord/src/next/documents/documents.ts index 6ca0f5e82..cf9a22596 100644 --- a/packages/chord/src/next/documents/documents.ts +++ b/packages/chord/src/next/documents/documents.ts @@ -6,7 +6,7 @@ import type { Context } from "../../context/index.ts"; import { ChordError } from "../data/index.ts"; import type { Batch } from "../state/internal.ts"; import type { ChordStore } from "../store/store.ts"; -import type { JsonObject, StateDefinition } from "./definition.ts"; +import type { CommitPersistence, JsonObject, StateDefinition } from "./definition.ts"; import { SealToken } from "./fence.ts"; import { type HandleOf, type KeyDeclarer, Keyed, Singleton } from "./handles.ts"; import { KeyedSlot } from "./keyed-slot.ts"; @@ -23,7 +23,7 @@ export type DocumentsOwner = { readonly batch: Batch; /** The object's context: shared cold loads run under it (rule 11). */ readonly context: Context; - /** The host's store and the object's owner string in it (`:`, W7), when it has one. */ + /** The host's store and the object's owner name in it (its type, R2), when it has one. */ readonly store: { readonly store: ChordStore; readonly owner: string } | undefined; /** A document the code cannot read fails the live object `broken` (slice 6a §1.4); ignored when it is not live. */ fail(error: ChordError): void; @@ -137,13 +137,19 @@ export class Documents { /** * Runs `fn` as one transaction on the line under `token` and commits what it drafted (§9): a throw, an abort of - * `context`, the token's seal or the object's end discards everything. + * `context`, the token's seal or the object's end discards everything. `persistence` defers storing a run state's + * change (D8). */ - async tx(fn: (tx: Transaction) => R | Promise, token: SealToken, context: Context): Promise { + async tx( + fn: (tx: Transaction) => R | Promise, + token: SealToken, + context: Context, + persistence?: CommitPersistence, + ): Promise { this.batch.assertOutside("tx"); this.assertAdmitted(token); const release = await this.line.acquire(); - const tx = new Transaction(this, token, release, context); + const tx = new Transaction(this, token, release, context, persistence); try { this.assertAdmitted(token); this.#holder = tx; @@ -232,6 +238,6 @@ export class Documents { /** The store side of persisted documents named `name`; the attempt checked the store first (P3). */ #persistence(name: string): Persistence { const { store, owner } = this.#owner.store!; - return new Persistence(store, owner, name); + return new Persistence(store, owner, "chord", name); } } diff --git a/packages/chord/src/next/documents/index.ts b/packages/chord/src/next/documents/index.ts index e7568d8f0..eaae754eb 100644 --- a/packages/chord/src/next/documents/index.ts +++ b/packages/chord/src/next/documents/index.ts @@ -1,5 +1,5 @@ // Documents for apps: definitions, handles and transactions. -export { defineState, type JsonObject, type StateDefinition } from "./definition.ts"; +export { type CommitPersistence, defineState, type JsonObject, type StateDefinition } from "./definition.ts"; export type { HandleOf, KeyedHandle, SingletonHandle } from "./handles.ts"; export type { Tx } from "./transaction.ts"; diff --git a/packages/chord/src/next/documents/internal.ts b/packages/chord/src/next/documents/internal.ts index b2eee6b2b..1bd15a893 100644 --- a/packages/chord/src/next/documents/internal.ts +++ b/packages/chord/src/next/documents/internal.ts @@ -5,5 +5,13 @@ export { Documents, type DocumentsOwner } from "./documents.ts"; export { SealToken } from "./fence.ts"; export { type KeyDeclarer, Keyed, Singleton } from "./handles.ts"; export { type KeyedSlot, KeySource } from "./keyed-slot.ts"; -export type { AnyDefinition, DocumentSlot } from "./slot.ts"; +export { + type DocumentChange, + type Held, + Persistence, + type Stored, + type StoredChange, + storedAfter, +} from "./persistence.ts"; +export { type AnyDefinition, type DocumentSlot, type PersistedSlot, SlotDefinition } from "./slot.ts"; export type { Staged, Transaction } from "./transaction.ts"; diff --git a/packages/chord/src/next/documents/keyed-slot.ts b/packages/chord/src/next/documents/keyed-slot.ts index 6575fd8ec..6faec91f0 100644 --- a/packages/chord/src/next/documents/keyed-slot.ts +++ b/packages/chord/src/next/documents/keyed-slot.ts @@ -8,12 +8,20 @@ import { type Prepared, type Tracker, track } from "../../delta/index.ts"; import type { Watch } from "../../watch.ts"; import { ChordError } from "../data/index.ts"; import { asChordError } from "../data/internal.ts"; +import type { StagedWrite } from "../store/step.ts"; import { compareText, storedText } from "../store/text.ts"; import { AddressSource } from "./address-source.ts"; import type { JsonObject } from "./definition.ts"; import type { Documents } from "./documents.ts"; import type { SealToken } from "./fence.ts"; -import type { Loaded, Persistence, Stored } from "./persistence.ts"; +import { + type DocumentChange, + type Loaded, + type Persistence, + type Stored, + type StoredChange, + storedAfter, +} from "./persistence.ts"; import { Resident } from "./residency.ts"; import { type AnyDefinition, type DocumentSlot, type PersistedSlot, SlotDefinition } from "./slot.ts"; @@ -222,6 +230,10 @@ export class KeyedSlot implements DocumentSlot, PersistedSlot { return this.#residents.get(key)?.stored; } + stage(key: string, change: DocumentChange, was: Stored | undefined, writes: StagedWrite[]): StoredChange { + return this.persistence.stage(key, this.definition.current, was, change, writes); + } + /** * Takes the attempt's declaration: a newer version migrates each document in memory on its next access (persisted * ones store a base with their next change); the pinned keys are watched, so they migrate now, all or none, and @@ -251,7 +263,13 @@ export class KeyedSlot implements DocumentSlot, PersistedSlot { return this.definition.current.initial(context); } - adopt(key: string, tracker: Tracker, prepared: Prepared, stored: Stored | undefined): void { + adopt( + key: string, + tracker: Tracker, + prepared: Prepared, + change: StoredChange | undefined, + ): void { + const stored = change === undefined ? undefined : storedAfter(change); const source = this.#sources.get(key); const before = source?.peek(); tracker.adopt(prepared); diff --git a/packages/chord/src/next/documents/persistence.ts b/packages/chord/src/next/documents/persistence.ts index 7b6082286..3efc45308 100644 --- a/packages/chord/src/next/documents/persistence.ts +++ b/packages/chord/src/next/documents/persistence.ts @@ -3,7 +3,7 @@ // checked `migrate`, and the writes one commit stages for a changed or retired document. import type { Context } from "../../context/index.ts"; -import type { Prepared } from "../../delta/index.ts"; +import type { Op, Prepared } from "../../delta/index.ts"; import { changeDocument, createDocument, @@ -32,25 +32,35 @@ export type Stored = { /** A loaded document: its value at the definition's version, and what is stored of it. */ export type Loaded = { readonly value: JsonObject; readonly stored: Stored }; +/** One transaction's change of one document: its prepared draft (if changed), and whether it retires it. */ +export type DocumentChange = { readonly prepared: Prepared | undefined; readonly retires: boolean }; + +/** A run state's held ops (D8), stored before the next change's own; `base` when a held change asked for a base. */ +export type Held = { readonly ops: readonly Op[]; readonly base: boolean }; + /** * What one commit stores of a document, in the store writes' words: a retirement (nothing stored after it), a - * creation whose id the step assigns (`create`, its position among the commit's creations), or a change of the record - * `stored` was prepared on. + * creation whose id the step assigns (`create`: `created.id` once it applied), a change of the record `stored` was + * prepared on, or nothing yet (`held`: a deferred change of a run's state, D8, kept in memory over `stored`). */ export type StoredChange = | { readonly kind: "retire" } - | { readonly kind: "create"; readonly version: number; readonly index: number } - | { readonly kind: "change"; readonly stored: Stored }; + | { readonly kind: "create"; readonly version: number; readonly created: { id: number } } + | { readonly kind: "change"; readonly stored: Stored } + | { readonly kind: "held"; readonly stored: Stored; readonly withinMs: number }; -/** The documents of one persisted definition name on one object: `(chord, :, name, key)`. */ +/** + * The documents of one persisted name on one object: `(, namespace, name, key)`, the namespace `chord` for + * `self.state`, `chord.jobs` and `chord.job-states` for persisted job runs (slice 6b D5). + */ export class Persistence { readonly store: ChordStore; readonly #address: Omit; - /** `owner` is `:` (W7); `kind` the definition's name. */ - constructor(store: ChordStore, owner: string, kind: string) { + /** `owner` is the object's type (R2); `kind` the definition's or the job's name. */ + constructor(store: ChordStore, owner: string, namespace: string, kind: string) { this.store = store; - this.#address = { namespace: "chord", owner, kind }; + this.#address = { owner, namespace, kind }; } /** @@ -58,7 +68,7 @@ export class Persistence { * as the store's). Counted (`chord.store.loads`). */ async read(key: string, context: Context): Promise { - const address = this.#at(key); + const address = this.at(key); const document = await this.store.read(async (executor) => { const record = await findDocument(executor, address); return record === undefined ? undefined : loadDocument(executor, record); @@ -92,7 +102,7 @@ export class Persistence { /** What is stored of the live document at `key` without loading it (a retirement's find), `undefined` when absent. */ async find(key: string, context: Context): Promise { - const record = await this.store.read((executor) => findDocument(executor, this.#at(key)), context); + const record = await this.store.read((executor) => findDocument(executor, this.at(key)), context); if (record === undefined) return undefined; // The change counter is all a retirement needs; `deltasSinceBase` is not read for one. return { id: record.id, revision: record.revision, version: record.version, deltasSinceBase: 0 }; @@ -106,16 +116,16 @@ export class Persistence { /** * Stages one commit's writes of the document at `key` into `writes` (rule 8: serialized here, before the queue): * its retirement when `retired` is stored, then its creation, or the change of `stored` (a base when created, when - * the stored version is older, or when `checkpointWhen` asks; a delta otherwise). `creations` counts the commit's - * creations so far (the step's ids follow their order). Returns what the commit stores. + * the stored version is older, or when `checkpointWhen` asks; a delta otherwise), `held` ops stored before the + * change's own (D8: a held base makes it a base). Returns what the commit stores. */ stage( key: string, definition: AnyDefinition, stored: Stored | undefined, - change: { readonly prepared: Prepared | undefined; readonly retires: boolean }, + change: DocumentChange, writes: StagedWrite[], - creations: number, + held?: Held, ): StoredChange { const { prepared, retires } = change; if (retires && stored !== undefined) writes.push(retireDocument(stored.id, stored.revision)); @@ -123,34 +133,43 @@ export class Persistence { const { version, fingerprint } = definition; if (retires || stored === undefined) { const base = { version, fingerprint, value: prepared.value }; - writes.push(createDocument(this.#at(key), "latest", base, JSON.stringify(prepared.value))); - return { kind: "create", version, index: creations }; + const created = { id: 0 }; + const write = createDocument(this.at(key), base, JSON.stringify(prepared.value), (id) => { + created.id = id; + }); + writes.push(write); + return { kind: "create", version, created }; } const info = { deltasSinceBase: stored.deltasSinceBase }; - const base = stored.version !== version || definition.checkpointWhen(prepared.value, prepared.ops, info); + const base = + held?.base === true || + stored.version !== version || + definition.checkpointWhen(prepared.value, prepared.ops, info); + const ops = held === undefined ? prepared.ops : [...held.ops, ...prepared.ops]; const content: DocumentContent = base ? { kind: "base", version, fingerprint, value: prepared.value } - : { kind: "delta", version, ops: prepared.ops }; - const text = JSON.stringify(base ? prepared.value : prepared.ops); + : { kind: "delta", version, ops }; + const text = JSON.stringify(base ? prepared.value : ops); writes.push(changeDocument(stored.id, stored.revision, content, text)); const deltasSinceBase = base ? 0 : stored.deltasSinceBase + 1; return { kind: "change", stored: { id: stored.id, revision: stored.revision + 1, version, deltasSinceBase } }; } /** The address of `key`: a singleton's (`""`) has no key. */ - #at(key: string): DocumentAddress { + at(key: string): DocumentAddress { return key === "" ? this.#address : { ...this.#address, key }; } } -/** What is stored after a commit's `change` was adopted with the commit's created ids. */ -export function storedAfter(change: StoredChange, ids: readonly number[]): Stored | undefined { +/** What is stored after a commit's `change` was adopted. */ +export function storedAfter(change: StoredChange): Stored | undefined { switch (change.kind) { case "retire": return undefined; case "create": - return { id: ids[change.index]!, revision: 1, version: change.version, deltasSinceBase: 0 }; + return { id: change.created.id, revision: 1, version: change.version, deltasSinceBase: 0 }; case "change": + case "held": return change.stored; } } diff --git a/packages/chord/src/next/documents/slot.ts b/packages/chord/src/next/documents/slot.ts index 657e8d90d..43d0e01cd 100644 --- a/packages/chord/src/next/documents/slot.ts +++ b/packages/chord/src/next/documents/slot.ts @@ -6,11 +6,12 @@ import type { Context } from "../../context/index.ts"; import { type Prepared, type Tracker, track } from "../../delta/index.ts"; import { ChordError } from "../data/index.ts"; +import type { StagedWrite } from "../store/step.ts"; import { AddressSource } from "./address-source.ts"; -import type { JsonObject, StateDefinition } from "./definition.ts"; +import type { CommitPersistence, JsonObject, StateDefinition } from "./definition.ts"; import type { Documents } from "./documents.ts"; import type { SealToken } from "./fence.ts"; -import type { Persistence, Stored } from "./persistence.ts"; +import { type DocumentChange, type Persistence, type Stored, type StoredChange, storedAfter } from "./persistence.ts"; /** A document definition, whatever its value and keyedness. */ export type AnyDefinition = StateDefinition; @@ -27,8 +28,13 @@ export interface DocumentSlot { initial(context: Context): JsonObject; /** The persisted slot's store side; `undefined` for documents in memory only. */ readonly persisted: PersistedSlot | undefined; - /** Adopts a commit of the document at `key` (inside the batch), with what is stored of it after the commit. */ - adopt(key: string, tracker: Tracker, prepared: Prepared, stored: Stored | undefined): void; + /** Adopts a commit of the document at `key` (inside the batch), with what the commit stored of it. */ + adopt( + key: string, + tracker: Tracker, + prepared: Prepared, + stored: StoredChange | undefined, + ): void; /** Retires the document at `key` (inside the batch). */ retire(key: string): void; } @@ -41,6 +47,17 @@ export interface PersistedSlot { cold(key: string): boolean; /** What is stored of the document at `key` as memory knows it; `undefined` when absent or cold. */ stored(key: string): Stored | undefined; + /** + * Stages one commit's writes of the document at `key` into `writes` (`was`: what is stored of it, a cold one's as + * the transaction found it); `persistence` is the transaction's (a deferred change of a run's state is held, D8). + */ + stage( + key: string, + change: DocumentChange, + was: Stored | undefined, + writes: StagedWrite[], + persistence: CommitPersistence, + ): StoredChange; /** * Loads the document at `key` (its caller holds the line): installed when present and still absent from memory, * and, for a transaction's load (`token`), while the token is not sealed (a cut transaction released the line, so @@ -205,6 +222,10 @@ export class SingletonSlot extends AddressSource implements DocumentSlot, Persis return this.#stored; } + stage(key: string, change: DocumentChange, was: Stored | undefined, writes: StagedWrite[]): StoredChange { + return this.persistence.stage(key, this.definition.current, was, change, writes); + } + current(_key: string): Tracker | undefined { return this.#tracker; } @@ -214,12 +235,17 @@ export class SingletonSlot extends AddressSource implements DocumentSlot, Persis return this.#initial!; } - adopt(_key: string, tracker: Tracker, prepared: Prepared, stored: Stored | undefined): void { + adopt( + _key: string, + tracker: Tracker, + prepared: Prepared, + stored: StoredChange | undefined, + ): void { // Unchecked: a step admitted before the object ended adopts after its seal (slice 6a §1.3). const before = this.#tracker?.value ?? this.#initial!; tracker.adopt(prepared); this.#tracker = tracker; - this.#stored = stored; + this.#stored = stored === undefined ? undefined : storedAfter(stored); this.written(before, prepared); } diff --git a/packages/chord/src/next/documents/transaction.ts b/packages/chord/src/next/documents/transaction.ts index 0dfb6c1bb..506c15468 100644 --- a/packages/chord/src/next/documents/transaction.ts +++ b/packages/chord/src/next/documents/transaction.ts @@ -9,17 +9,18 @@ import type { Job } from "../jobs/internal.ts"; import type { Columns, PutRowOf, RowOf } from "../store/columns.ts"; import type { Condition, SetOf } from "../store/expression.ts"; import type { ColumnRefs, Query } from "../store/query.ts"; +import { refusalOf } from "../store/refusal.ts"; import type { StagedWrite } from "../store/step.ts"; import type { ChordStore } from "../store/store.ts"; import type { KeyOf } from "../store/table.ts"; import { TableChanges } from "../store/table-changes.ts"; import { type Page, type PageOptions, ROWS_PAGE, type Table, TableHandle } from "../store/table-handle.ts"; -import type { JsonObject } from "./definition.ts"; +import type { CommitPersistence, JsonObject } from "./definition.ts"; import type { Documents } from "./documents.ts"; import type { SealToken } from "./fence.ts"; import type { Keyed, KeyedHandle, Singleton, SingletonHandle } from "./handles.ts"; import type { KeyedSlot } from "./keyed-slot.ts"; -import { type Stored, type StoredChange, storedAfter } from "./persistence.ts"; +import type { Stored, StoredChange } from "./persistence.ts"; import type { DocumentSlot, SingletonSlot } from "./slot.ts"; /** What `self.tx` gives its callback. Inside it: only `await tx.*`, no other `self.tx`, no calls to other objects. */ @@ -98,6 +99,8 @@ type Entry = { found: Stored | undefined; }; +const IMMEDIATE: CommitPersistence = { kind: "immediate" }; + /** * One document change a commit adopts: a removal, a draft's prepared change, or both; and for a persisted document * what the commit stores of it. @@ -109,9 +112,6 @@ type Adoption = { readonly stored: StoredChange | undefined; }; -/** The created documents' ids of a commit without store writes: none. */ -const NO_IDS: readonly number[] = []; - /** * One transaction of an object, under the seal token of whoever runs it (an attempt, a job run's invocation, the * object). Commit prepares every draft (Delta records ops and checks only placed values) and adopts the changed @@ -135,15 +135,33 @@ export class Transaction implements Tx { #tables: Map | undefined; /** `tx.*` reads underway: a callback settling with one is refused (rule 3, as Durable's). */ #pending = 0; + /** When its commit reaches the store (a run state's deferred change, D8). */ + readonly #persistence: CommitPersistence; #state: "open" | "committing" | "stepping" | "committed" | "discarded" | "cut" = "open"; /** Rejected by `cut` while the commit waits in the store's queue: its caller is told at once (rule 3). */ #queued: PromiseWithResolvers | undefined; - constructor(documents: Documents, token: SealToken, release: () => void, context: Context) { + constructor( + documents: Documents, + token: SealToken, + release: () => void, + context: Context, + persistence: CommitPersistence = IMMEDIATE, + ) { this.#documents = documents; this.token = token; this.#release = release; this.#context = context; + this.#persistence = persistence; + } + + /** + * The store writes of this transaction, which `write`s join (Chord's own documents: a job's run records): made by the + * first, which sends the commit through `store`'s step. + */ + stageStore(store: ChordStore): StagedWrite[] { + this.#writes ??= { store, staged: [] }; + return this.#writes.staged; } /** The object whose transaction this is. */ @@ -311,7 +329,6 @@ export class Transaction implements Tx { this.#state = "committing"; this.#context.abortSignal?.throwIfAborted(); const adoptions: Adoption[] = []; - let creations = 0; for (const entries of this.#entries.values()) { for (const entry of entries.values()) { let prepared = entry.draft?.change.prepare(); @@ -325,19 +342,16 @@ export class Transaction implements Tx { const persisted = slot.persisted; let stored: StoredChange | undefined; if (persisted !== undefined) { - const { persistence, definition } = persisted; - this.#writes ??= { store: persistence.store, staged: [] }; - const change = { prepared, retires }; + const writes = this.stageStore(persisted.persistence.store); const was = persisted.stored(key) ?? entry.found; - stored = persistence.stage(key, definition.current, was, change, this.#writes.staged, creations); - if (stored.kind === "create") creations++; + stored = persisted.stage(key, { prepared, retires }, was, writes, this.#persistence); } adoptions.push({ entry, prepared, retires, stored }); } } const writes = this.#writes; if (writes === undefined) { - this.#adopt(adoptions, NO_IDS); + this.#adopt(adoptions); this.#state = "committed"; return; } @@ -356,7 +370,7 @@ export class Transaction implements Tx { this.#state = "stepping"; return true; }, - adopt: (_seq: number, ids: readonly number[]) => this.#adopt(adoptions, ids), + adopt: () => this.#adopt(adoptions), }; this.#queued = Promise.withResolvers(); try { @@ -364,7 +378,7 @@ export class Transaction implements Tx { await Promise.race([store.commit(staged, fence, withoutAbortSignal(this.#context)), this.#queued.promise]); } catch (error) { if (this.#state === "cut") throw this.#cutError(); - throw error; + throw this.#refused(error); } finally { this.#queued = undefined; } @@ -372,18 +386,29 @@ export class Transaction implements Tx { } /** - * Adopts the changed documents (persisted ones with what is stored of them, `ids` being the commit's created - * documents' ids) and the staged changes in one batch; nothing when nothing changed. + * The error a commit the store rejected rejects with (D4): the store's own (`released`, `storage_failed`) as it + * is; a refused step classified, an `int` `CHECK` failing only this transaction (`invalid/args`), any other refusal + * failing the live object `broken` too (memory and disk disagree). */ - #adopt(adoptions: readonly Adoption[], ids: readonly number[]): void { + #refused(error: unknown): unknown { + const refusal = refusalOf(error, this.#documents.label); + if (refusal === undefined) return error; + if (refusal.kind === "broken") this.#documents.fail(refusal.error); + return refusal.error; + } + + /** + * Adopts the changed documents (persisted ones with what the commit stored of them) and the staged changes in one + * batch; nothing when nothing changed. + */ + #adopt(adoptions: readonly Adoption[]): void { const staged = this.#staged; if (adoptions.length === 0 && staged === undefined) return; this.#documents.batch.commit(() => { for (const { entry, prepared, retires, stored } of adoptions) { if (retires) entry.slot.retire(entry.key); if (prepared === undefined) continue; - const after = stored === undefined ? undefined : storedAfter(stored, ids); - entry.slot.adopt(entry.key, entry.draft!.tracker, prepared, after); + entry.slot.adopt(entry.key, entry.draft!.tracker, prepared, stored); } for (const each of staged?.values() ?? []) each.adopt(); }, this.#context); diff --git a/packages/chord/src/next/host/attempt.ts b/packages/chord/src/next/host/attempt.ts index aa9071dbd..a424b5ff6 100644 --- a/packages/chord/src/next/host/attempt.ts +++ b/packages/chord/src/next/host/attempt.ts @@ -5,7 +5,7 @@ import { awaitWithContext, type Context, withAbortSignal } from "../../context/index.ts"; import type { JsonValue } from "../../json.ts"; -import type { NativeWatchable, Watchable } from "../../watch.ts"; +import type { NativeWatchable, Watchable, WatchEnd } from "../../watch.ts"; import type { Contract, Implementation, ParamsOf } from "../contracts/index.ts"; import { contractData, type ShapeOf } from "../contracts/internal.ts"; import { ChordError, type FailureCause, type Principal } from "../data/index.ts"; @@ -30,6 +30,7 @@ import { type Table, TableHandle } from "../store/table-handle.ts"; import { prepareTables } from "../store/table-migration.ts"; import { CallContext, type Caller, type Ctx, withLinkedAbortSignal } from "./ctx.ts"; import { Deadline } from "./deadline.ts"; +import { type FollowAttach, FollowSource } from "./follow.ts"; import type { HostError } from "./host.ts"; import { attachDerives, checkImplementation, type KeyedBinding, type Members } from "./members.ts"; import { @@ -77,6 +78,13 @@ export interface Self

extends RoutineDeclaratio ): Native; /** Runs `fn` as one transaction of this object (§9). */ tx(fn: (tx: Tx) => R | Promise, context: Context): Promise; + /** + * A source following another runtime's synchronous observer (Durable's views, slice 6b §3): `attach` runs during + * activation once the natives opened (a rejection or `undefined` fails the activation `transient`), and the observer + * is detached when the attempt ends. Its runtime failing fails this object `transient`; its `ended` is final, the + * source keeping its last value. A state member, a derive source, an `on` source. + */ + follow(attach: FollowAttach): Watchable; /** * This object's job `definition` (§15), once per job name: `run` runs each started run outside transactions, * `finish` records its outcome in one. Starts are taken once the object is live. @@ -143,6 +151,8 @@ export class Attempt implements StoreSelf, KeyDeclarer { #members: Members | undefined; #variant: string | undefined; readonly #derives: Derive[] = []; + /** Its followed sources, attached in declaration order after the natives. */ + readonly #follows: FollowSource[] = []; readonly #routines = new Routines(); readonly #jobs = new Map(); /** The tables it declared, by name. */ @@ -162,6 +172,8 @@ export class Attempt implements StoreSelf, KeyDeclarer { #previous: Attempt | undefined; /** What activation returns once the attempt was abandoned: a timeout, a cycle through its object, or its end. */ #abandoned: Exclude | undefined; + /** A followed source's end while this attempt activated: it fails as it goes live (if its activation returned). */ + #endedEarly: ChordError | undefined; constructor(object: ChordObject, previous: Attempt | undefined, context: Context) { this.#object = object; @@ -173,8 +185,11 @@ export class Attempt implements StoreSelf, KeyDeclarer { state>(definition: D): HandleOf { this.#assertDeclaring("state"); const what = `state "${definition.name}"`; - if (definition.persist) this.#assertPersistable(what); if (this.#declared.has(definition.name)) throw new Error(`${this.#object.label}: ${what} is declared twice`); + if (definition.persist) { + this.#assertPersistable(what); + this.#object.claimDocuments("chord"); + } this.#declared.set(definition.name, definition); return this.#object.documents.declare(definition, this.#token, this); } @@ -189,15 +204,16 @@ export class Attempt implements StoreSelf, KeyDeclarer { table(definition: TableDefinition): Table { this.#assertDeclaring("table"); const object = this.#object; - const { host, served, label } = object; + const { served, label } = object; const store = this.#assertPersistable(`table "${definition.name}"`); if (this.#tables.has(definition.name)) throw new Error(`${label}: table "${definition.name}" is declared twice`); - const sqlName = tableSqlName(host.address?.kind ?? "client", served.contract.name, definition.name); + const sqlName = tableSqlName(served.contract.name, definition.name); + object.claimTable(sqlName); const owner = { label, scope: object.documents, assertLive: () => object.documents.assertLive(), - fail: (error: ChordError) => object.failLive(this, error), + fail: (error: ChordError) => object.failLive(this, { kind: "broken", error }), }; // The handle's rows are the definition's, typed by `C` and `K`. const handle = new TableHandle(definition as unknown as TableDefinition, sqlName, store, owner); @@ -249,6 +265,16 @@ export class Attempt implements StoreSelf, KeyDeclarer { return this.#object.documents.tx(fn, this.#token, context); } + follow(attach: FollowAttach): Watchable { + this.#assertDeclaring("follow"); + const object = this.#object; + const owner = { label: object.label, context: this.#context, ended: (end: WatchEnd) => this.#followEnded(end) }; + // The source holds what the runtime gives it, `T`, as Chord's JSON sources do. + const source = new FollowSource(attach as unknown as FollowAttach, owner, object.host.batch); + this.#follows.push(source); + return source as unknown as Watchable; + } + on(source: Watchable, fn: OnHandler): void; on(source: NativeWatchable, fn: OnNativeHandler): void; on(source: Watchable | NativeWatchable, fn: (...args: never[]) => void | Promise): void { @@ -267,6 +293,11 @@ export class Attempt implements StoreSelf, KeyDeclarer { if (this.#jobs.has(definition.name)) { throw new Error(`${object.label}: job "${definition.name}" is declared twice`); } + if (definition.persist !== undefined) { + this.#assertPersistable(`job "${definition.name}"`); + object.claimDocuments("chord.jobs"); + object.claimDocuments("chord.job-states"); + } const owner = { host: object.host, caller: this.#caller(), @@ -314,9 +345,34 @@ export class Attempt implements StoreSelf, KeyDeclarer { this.#abort.abort(outcome.kind === "ended" ? outcome.error : outcome.cause.error); } + /** + * A followed runtime ended its observer (R4): `failed` fails this attempt `transient`, live or activating; `ended` + * does only while it activates (it would go live over a source that ended). An activation that already returned + * fails as it goes live. + */ + #followEnded(end: WatchEnd): void { + if (end.kind === "ended" && this.#isLive) return; + const { label } = this.#object; + const error = + end.kind === "failed" + ? asChordError(end.error, label) + : new ChordError({ code: "unavailable", message: `${label}: a followed source ended while it activated` }); + if (this.#isLive) { + this.#object.failLive(this, { kind: "transient", error }); + return; + } + this.#endedEarly ??= error; + this.abandon({ kind: "failed", cause: { kind: "transient", error } }); + } + /** The object went live: its jobs take starts and its routines start, in this turn; both stop when it is sealed. */ live(): void { this.#isLive = true; + const early = this.#endedEarly; + if (early !== undefined) { + this.#object.failLive(this, { kind: "transient", error: early }); + return; + } for (const job of this.#jobs.values()) job.live(); const { host, label } = this.#object; this.#routines.start({ @@ -421,6 +477,8 @@ export class Attempt implements StoreSelf, KeyDeclarer { if ((await drained(this.#running.keys(), this.#running.size, host.timeouts.close)) === "timeout") { host.report({ kind: "drain_timeout", object: label }); } + // The close writer: the persisted runs the seal stopped, and the held state changes (slice 6b §4.2). + for (const job of this.#jobs.values()) await job.writeAtClose(this.#context); // Work stuck past the drain keeps running, but no longer counts as a call or holds its `ctx.get` references. for (const [running, cleanup] of this.#running) { this.#running.delete(running); @@ -434,10 +492,14 @@ export class Attempt implements StoreSelf, KeyDeclarer { return failures; } - /** Detaches its derives (once their object no longer needs them): reads throw `released`, their watches end. */ + /** + * Detaches its derives (once their object no longer needs them: reads throw `released`, their watches end) and its + * followed sources (their observers detached, their watches ended). + */ detach(): void { const ended = this.#object.ended(); for (const derive of this.#derives) derive.detach(ended); + for (const follow of this.#follows) follow.detach(); } /** Releases its references, and the previous attempt's if this one never bound its own. */ @@ -458,6 +520,8 @@ export class Attempt implements StoreSelf, KeyDeclarer { const loading = documents.load(this.#opening); if (loading !== undefined) await loading; if (this.#tables.size > 0) await this.#prepareTables(deadline); + // A persisted job's runs, loaded once per object: rebuilt before anything runs. + for (const job of this.#jobs.values()) await job.load(this.#opening); const { host } = this.#object; dependencies = this.#references.map(({ binding, contract, params }) => host.bind(binding, contract, params, this.#opening), @@ -508,6 +572,15 @@ export class Attempt implements StoreSelf, KeyDeclarer { return this.#abandoned ?? { kind: "failed", cause: openFailure(error, this.#object.label) }; } } + for (const follow of this.#follows) { + if (this.#abandoned !== undefined) return this.#abandoned; + try { + await awaitWithContext(follow.attach(this.#opening), this.#opening); + } catch (error) { + const cause: FailureCause = { kind: "transient", error: asChordError(error, this.#object.label) }; + return this.#abandoned ?? { kind: "failed", cause }; + } + } try { const checked = checkImplementation(this.#object.served.contract, implementation); this.#members = checked.members; diff --git a/packages/chord/src/next/host/follow.ts b/packages/chord/src/next/host/follow.ts new file mode 100644 index 000000000..d4131ca3d --- /dev/null +++ b/packages/chord/src/next/host/follow.ts @@ -0,0 +1,248 @@ +// Followed sources (slice 6b D9, R4, §3): `self.follow(attach)`, a Chord source over a synchronous observer of another +// runtime (Durable's `observeConversation`, `observeTaskGraph`, `observeDoc`). Attached during its attempt's +// activation, detached at the attempt's end. Calls inside a running batch (Durable adopting in the store's commit step) +// are marked in that batch, calls outside one are grouped into one batch on the next microtask, so a derive over two +// views of one commit computes once with both changes. + +import type { Context } from "../../context/index.ts"; +import type { Op } from "../../delta/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { Watch, WatchCall, WatchEnd } from "../../watch.ts"; +import { ChordError } from "../data/index.ts"; +import { type Batch, Source, type Subscriber } from "../state/internal.ts"; + +/** The observer a followed runtime is given: Chord's watch shapes (Durable's `Observer`). */ +export type FollowObserver = { + /** One change, synchronously and quickly: Chord only records it. */ + advance(call: WatchCall, context: Context): void; + /** The followed source ended: `failed` (its runtime failed), or `ended` for good. */ + end(end: WatchEnd): void; +}; + +/** What attaching gives: the value every later call starts from, and how to detach (Durable's `Observation`). */ +export type FollowAttachment = { readonly value: T; detach(): void }; + +/** + * What `self.follow` takes: attaches `observer`, resolving with the attachment, `undefined` for an absent address. + * The observer takes calls of any value (it only records them), so `T` is inferred from the attachment alone. + */ +export type FollowAttach = ( + observer: FollowObserver, + context: Context, +) => Promise | undefined>; + +/** Who a followed source belongs to: its attempt (R4) and its context. */ +export type FollowOwner = { + readonly label: string; + readonly context: Context; + /** + * The followed runtime ended its observer: `failed` fails the attempt `transient` (live or activating, its retry + * attaches again); `ended` while the attempt activates fails it `transient` too (it would go live on a source that + * ended); `ended` while live is final for the source only. + */ + ended(end: WatchEnd): void; +}; + +/** + * Where a followed source stands: waiting for its attachment (with the runtime's end when it came first), attached, + * ended by the runtime (its value stays; detached at its attempt's end), or detached by its attempt. + */ +type FollowState = + | { readonly kind: "attaching"; readonly end: WatchEnd | undefined } + | { readonly kind: "attached"; readonly detach: () => void } + | { readonly kind: "ended"; readonly detach: () => void } + | { readonly kind: "detached" }; + +const NO_OPS: readonly Op[] = []; + +const ENDED: WatchEnd = { kind: "ended" }; + +/** A followed source: the runtime's value, its calls applied in order, its frames delivered in the host's batch. */ +export class FollowSource extends Source { + readonly #attach: FollowAttach; + readonly #owner: FollowOwner; + #state: FollowState = { kind: "attaching", end: undefined }; + #value: JsonValue | undefined; + /** Calls not applied yet: before the attachment resolved, and outside a batch until the next microtask. */ + #queued: WatchCall[] = []; + /** Whether a microtask is queued to apply `#queued` in a batch of its own. */ + #scheduled = false; + /** + * The runtime's end, still to reach the watches: `queued` after the queued calls' batch, `frame` after the current + * batch's frame (a retirement and an end in one batch: the retirement first). + */ + #ending: "none" | "queued" | "frame" = "none"; + /** The ops of the current batch's frame (kept only while watched), and whether a retirement came in it. */ + #ops: Op[] = []; + #retired = false; + #listed = false; + + constructor(attach: FollowAttach, owner: FollowOwner, batch: Batch) { + super(batch); + this.#attach = attach; + this.#owner = owner; + } + + get value(): JsonValue { + if (this.#value === undefined) { + throw new ChordError({ + code: "not_ready", + message: `${this.#owner.label}: a followed source read before it attached`, + }); + } + return this.#value; + } + + /** + * Attaches (activation, after the natives opened): the calls that came meanwhile apply on the attachment's value, + * and an end that came meanwhile takes effect then (the value stays). A rejected attachment, or `undefined` (an + * absent address), throws; an attachment arriving after its attempt detached the source is detached at once. + */ + async attach(context: Context): Promise { + const observer: FollowObserver = { + advance: (call) => this.#advance(call), + end: (end) => this.#end(end), + }; + const attached = await this.#attach(observer, context); + if (attached === undefined) { + throw new ChordError({ code: "unavailable", message: `${this.#owner.label}: the followed address is absent` }); + } + const state = this.#state; + const detach = () => attached.detach(); + if (state.kind !== "attaching") { + detach(); + return; + } + this.#value = attached.value; + // Nothing watches it yet (members are checked after this): the queued calls only move the value on. + for (const call of this.#queued) if (call.kind === "value") this.#value = call.value; + this.#queued = []; + this.#state = state.end?.kind === "ended" ? { kind: "ended", detach } : { kind: "attached", detach }; + } + + /** Its attempt ended: the observer is detached (also after the runtime ended it) and its watches end `ended`. */ + detach(): void { + const state = this.#state; + this.#state = { kind: "detached" }; + this.#queued = []; + if (state.kind === "attached" || state.kind === "ended") state.detach(); + this.endWatches(); + } + + /** A watch of an ended source opens at its value and ends `ended` at once. */ + override openWatch(context: Context): Watch { + const watch = super.openWatch(context); + if (this.#over) watch.end(ENDED); + return watch; + } + + /** A subscriber of an ended source (a reference's watch through its member source) ends `ended` at once. */ + override subscribe(subscriber: Subscriber): void { + if (this.#over) subscriber.end(ENDED); + else super.subscribe(subscriber); + } + + deliver(context: Context): void { + const ops = this.#ops; + const retired = this.#retired; + this.#ops = []; + this.#retired = false; + this.#listed = false; + this.publish(this.#value!, ops.length === 0 ? NO_OPS : ops, retired, context); + if (this.#ending === "frame") { + this.#ending = "none"; + this.endWatchesAfterDelivery(); + } + } + + /** Whether nothing reaches its watches any more: ended by the runtime (what it gave delivered) or detached. */ + get #over(): boolean { + const { kind } = this.#state; + return (kind === "ended" && this.#ending === "none") || kind === "detached"; + } + + /** + * A call of the runtime: before the attachment, queued; inside a running batch, applied there (after what was + * queued); outside one, queued for one batch on the next microtask. + */ + #advance(call: WatchCall): void { + const { kind } = this.#state; + if (kind === "ended" || kind === "detached") return; + if (kind === "attaching" || !this.batch.running) { + this.#queued.push(call); + if (kind === "attached") this.#schedule(); + return; + } + this.#applyQueued(); + this.#apply(call); + } + + /** + * The runtime ended its observer (R4); its attempt is told. `ended` is final: what came before reaches the watches + * first (the queued calls in their batch, a call of the running batch in its frame), then they end, the value + * staying. A `failed` attempt detaches the source. + */ + #end(end: WatchEnd): void { + const state = this.#state; + if (state.kind === "ended" || state.kind === "detached") return; + if (state.kind === "attaching") { + if (state.end === undefined) this.#state = { kind: "attaching", end }; + this.#owner.ended(end); + return; + } + if (end.kind === "ended") { + this.#state = { kind: "ended", detach: state.detach }; + if (this.#queued.length > 0 && !this.batch.running) this.#ending = "queued"; + else { + // Inside a running batch the queued calls join it (none is queued outside one: they wait for their batch). + this.#applyQueued(); + this.#endAfterFrame(); + } + } + this.#owner.ended(end); + } + + /** The watches end once this batch's frame reached them, at once when the source is in none. */ + #endAfterFrame(): void { + // No frame to wait for (nothing watches it): the end is over now, and a later watch ends at once. + this.#ending = this.#listed ? "frame" : "none"; + if (!this.#listed) this.endWatchesAfterDelivery(); + } + + #schedule(): void { + if (this.#scheduled) return; + this.#scheduled = true; + this.batch.soon(() => { + this.#scheduled = false; + const { kind } = this.#state; + if (kind === "attached" || kind === "ended") this.#applyQueued(); + if (this.#ending === "queued") this.#endAfterFrame(); + }, this.#owner.context); + } + + #applyQueued(): void { + const queued = this.#queued; + if (queued.length === 0) return; + this.#queued = []; + for (const call of queued) this.#apply(call); + } + + /** Applies one value call inside the running batch: the new value, its ops for the frame, the source marked once. */ + #apply(call: WatchCall): void { + // The runtime's failures are final: it ends its observer instead of sending one. + if (call.kind !== "value") return; + this.#value = call.value; + if (call.retired) this.#retired = true; + if (!this.watched) { + this.changed(); + if (call.retired) this.retiredInBatch(); + return; + } + for (const op of call.ops) this.#ops.push(op); + if (!this.#listed) { + this.#listed = true; + this.changed(); + } + if (call.retired) this.retiredInBatch(); + } +} diff --git a/packages/chord/src/next/host/host.ts b/packages/chord/src/next/host/host.ts index 8312e70bd..ecfad05ed 100644 --- a/packages/chord/src/next/host/host.ts +++ b/packages/chord/src/next/host/host.ts @@ -126,6 +126,14 @@ export type HostError = readonly id: string; readonly error: unknown; } + /** A persisted run was cut off five times in a row by crashes (D6): it ends `failed { restart_limit }`, `finish` runs. */ + | { + readonly kind: "job_interrupted"; + readonly object: string; + readonly job: string; + readonly id: string; + readonly restarts: number; + } /** A blocked run was dropped without `finish`: aborted, or its object ended. */ | { readonly kind: "job_dropped"; readonly object: string; readonly job: string; readonly id: string }; @@ -303,7 +311,7 @@ export class ChordHost

this.#storageFailed(error)); + this.store?.attach({ failed: (error) => this.#storageFailed(error), batch: this.batch }); } async load(facets: readonly { readonly name: string; readonly facet: Facet

}[], context: Context): Promise { diff --git a/packages/chord/src/next/host/index.ts b/packages/chord/src/next/host/index.ts index 6eb1a040c..3d7611c5d 100644 --- a/packages/chord/src/next/host/index.ts +++ b/packages/chord/src/next/host/index.ts @@ -5,6 +5,7 @@ export type { ActivityCounts } from "./activity.ts"; export type { Factory, Self, StoreSelf } from "./attempt.ts"; export type { Ctx } from "./ctx.ts"; export { defineFacet, type Facet, type FacetHost, type ServeOptions } from "./facet.ts"; +export type { FollowAttach, FollowAttachment, FollowObserver } from "./follow.ts"; export { createHost, type GetOptions, diff --git a/packages/chord/src/next/host/object.ts b/packages/chord/src/next/host/object.ts index ebdf37b4a..43e4d80fd 100644 --- a/packages/chord/src/next/host/object.ts +++ b/packages/chord/src/next/host/object.ts @@ -142,6 +142,8 @@ export class ChordObject implements Target { readonly #members = new Map(); /** The keys of its keyed members that references hold, across attempts. */ readonly #keys: KeyedMembers; + /** The one timer storing its jobs' held state changes by their earliest deadline (D8), while one is held. */ + #flush: { readonly deadline: number; readonly timer: ReturnType } | undefined; constructor( host: AnyHost, @@ -167,12 +169,9 @@ export class ChordObject implements Target { label: this.label, batch: host.batch, context, - store: - store === undefined - ? undefined - : { store, owner: `${host.address?.kind ?? "client"}:${served.contract.name}` }, + store: store === undefined ? undefined : { store, owner: served.contract.name }, fail: (error) => { - if (this.#state.kind === "live") this.failLive(this.#state.attempt, error); + if (this.#state.kind === "live") this.failLive(this.#state.attempt, { kind: "broken", error }); }, }); } @@ -327,12 +326,15 @@ export class ChordObject implements Target { jobRecords(definition: JobDefinition): JobRecords { const records = this.#jobs.get(definition.name); if (records === undefined) { + const { store } = this.host; const owner = { label: this.label, host: this.host, context: this.#context, documents: this.documents, + store: store === undefined ? undefined : { store, owner: this.served.contract.name }, countRuns: (delta: number) => this.countRuns(delta), + held: (deadline: number) => this.#heldUntil(deadline), }; const made = new JobRecords(definition, owner); this.#jobs.set(definition.name, made); @@ -347,19 +349,34 @@ export class ChordObject implements Target { } /** - * A runtime failure of the live attempt `attempt` (a `migrate` failing on access, a stored version newer than the - * code): the attempt is sealed and the object `failed` with the broken `cause`; calls reject with it, holders are - * told, the state members keep their values once the admitted commits adopted, the natives close in the background - * (then its references are released), `onError` gets `object_failed` once. Documents and job runs stay. Recovery: - * `ref.retry()` (a new attempt, whose members the member sources follow once it is live). A failure of a sealed - * attempt is ignored. + * Claims table `table` (its SQL name) in the host's store for this object, until its end (slice 6b claims); held by + * another owner: `invalid/args`, which fails the declaring factory. */ - failLive(attempt: Attempt, error: ChordError): void { + claimTable(table: string): void { + this.host.store!.claims.table(table, this); + } + + /** Claims this object's documents in `namespace` of the host's store, as `claimTable`. */ + claimDocuments(namespace: string): void { + this.host.store!.claims.documents(this.served.contract.name, namespace, this); + } + + /** + * A runtime failure of the live attempt `attempt`: `broken` (a `migrate` failing on access, a stored version newer + * than the code, a refused commit) or `transient` (a followed runtime failed, R4): the attempt is sealed and the + * object `failed` with `cause`; calls reject with it, holders are told, the state members keep their values once the + * admitted commits adopted, the natives close in the background (then its references are released), `onError` gets + * `object_failed` once. Documents and job runs stay. Recovery: a broken failure by `ref.retry()`, a transient one + * after its identity's backoff too (a new attempt, whose members the member sources follow once it is live). A + * failure of a sealed attempt is ignored, so several at once fail it once. + */ + failLive(attempt: Attempt, cause: FailureCause): void { const state = this.#state; if (state.kind !== "live" || state.attempt !== attempt) return; - const cause: FailureCause = { kind: "broken", error }; - this.host.backoff.delete(this.identity); - const failure: Failed = { kind: "failed", cause, retryAt: undefined }; + const { error } = cause; + if (cause.kind === "transient") this.host.backoff.failed(this.identity); + else this.host.backoff.delete(this.identity); + const failure: Failed = { kind: "failed", cause, retryAt: this.host.backoff.until(this.identity) }; this.#state = { kind: "failed", failure, holding: undefined }; this.#lastFailure = error; this.#outcome = settlement(); @@ -382,6 +399,26 @@ export class ChordObject implements Target { this.#idle(); } + /** A held state change must be stored by `deadline`: the timer is set for the earliest one. */ + #heldUntil(deadline: number): void { + if (this.#flush !== undefined && this.#flush.deadline <= deadline) return; + if (this.#flush !== undefined) clearTimeout(this.#flush.timer); + // A foreground timer: what is held is stored before the process may exit. + const timer = setTimeout(() => void this.#flushHeld(), Math.max(0, deadline - Date.now())); + this.#flush = { deadline, timer }; + } + + /** The timer fired: every held change past its deadline is stored, and the timer set for the next one. */ + async #flushHeld(): Promise { + this.#flush = undefined; + if (this.#state.kind === "ended") return; + const now = Date.now(); + const jobs = [...this.#jobs.values()]; + await Promise.all(jobs.map((records) => records.flushHeld(now))); + const next = Math.min(...jobs.map((records) => records.nextDeadline())); + if (Number.isFinite(next)) this.#heldUntil(next); + } + /** * Adds `delta` to its running and finishing job runs (and `host.activity.jobs`). A new run keeps a lingering object; * the last one settling on an object without references ends it as its last release would (`#idle`). @@ -580,6 +617,8 @@ export class ChordObject implements Target { const previous = this.host.endingOf(this.identity); this.#state = { kind: "ended", ended: { kind: "ended", error } }; for (const records of this.#jobs.values()) records.end(error); + if (this.#flush !== undefined) clearTimeout(this.#flush.timer); + this.#flush = undefined; this.documents.seal(); this.#stopLingering(); this.host.forget(this); @@ -648,6 +687,7 @@ export class ChordObject implements Target { if (teardown.kind !== "end") teardown.attempt.releaseReferences(); await this.documents.settled(); this.documents.end(); + this.host.store?.claims.release(this); for (const records of this.#jobs.values()) records.drop(); this.#jobs.clear(); return failures; diff --git a/packages/chord/src/next/host/routines.ts b/packages/chord/src/next/host/routines.ts index 824e0ee38..ea924ec7b 100644 --- a/packages/chord/src/next/host/routines.ts +++ b/packages/chord/src/next/host/routines.ts @@ -306,6 +306,7 @@ class OnRoutine { if (call.kind === "value") done(true); }, end: () => done(false), + endAfterDelivery: () => done(false), }; const aborted = (): void => done(false); signal?.addEventListener("abort", aborted, { once: true }); diff --git a/packages/chord/src/next/index.ts b/packages/chord/src/next/index.ts new file mode 100644 index 000000000..8e759f345 --- /dev/null +++ b/packages/chord/src/next/index.ts @@ -0,0 +1,50 @@ +// Chord v2 (`@earendil-works/chord/next`, experimental): every layer's API for apps, and the store's surface for +// adapters (Durable's `StoreStorage`), marked internal. + +export * from "./contracts/index.ts"; +export * from "./data/index.ts"; +export * from "./documents/index.ts"; +export * from "./host/index.ts"; +export * from "./jobs/index.ts"; +export * from "./link/index.ts"; +export * from "./state/index.ts"; +/** @internal The store's surface for adapters (slice 6b §2.1). */ +export { type DecodedRow, OwnerTable, StoreOwnerTables } from "./store/adapter.ts"; +/** @internal The sync layer's batch apply (slice 6b §1.6). */ +export { applyBatch } from "./store/apply-batch.ts"; +/** @internal */ +export type { Claimant } from "./store/claims.ts"; +/** @internal */ +export { + type DocumentAddress, + type DocumentContent, + type DocumentEntry, + type DocumentHistory, + type DocumentPoint, + type DocumentRecord, + type DocumentScope, + findDocument, + findDocumentEntry, + insertDocument, + listDocumentEntries, + listDocuments, + loadContent, + loadDocument, + type OwnedDocumentEntry, + readDocument, + readDocumentEntry, + retireRecord, + type StoredDocument, + writeContent, +} from "./store/engine.ts"; +/** @internal */ +export type { WhereNode } from "./store/expression.ts"; +export * from "./store/index.ts"; +/** @internal */ +export type { EncodedRow } from "./store/rows.ts"; +/** @internal */ +export type { Order } from "./store/sql.ts"; +/** @internal */ +export type { StagedWrite, Step } from "./store/step.ts"; +/** @internal */ +export type { ChordStore } from "./store/store.ts"; diff --git a/packages/chord/src/next/jobs/README.md b/packages/chord/src/next/jobs/README.md index 3fba3ca35..13594c9c4 100644 --- a/packages/chord/src/next/jobs/README.md +++ b/packages/chord/src/next/jobs/README.md @@ -105,8 +105,18 @@ host.serve(ChatStore, (_, self) => { only when it records something (`Transaction.staged` reads what it staged so far). - `JobOwner`: what a job needs of its object: label, host, caller, the object's context (without an abort signal), documents, `admit` (the drain) and `countRuns` (keep-alive and `host.activity.jobs`). -- Inside: `JobRun` (one run: record, phase, abort signal, invocation fence closed as `run` settles, `running` entry, - the `Run` handle over its detachable waiters), the +- Inside: `JobRun` (one run: record, phase, its current `Invocation`, `running` entry, the `Run` handle over its + detachable waiters; an `Invocation` is one call of `run` with its own seal token, abort signal and join, captured by + its ctx and its settling, so an invocation a seal replaced never writes or records for the next one), the run states (a keyed `DocumentHandle` by run id, apart from `self.state`'s, never drafted while absent) and `Running` (`running`'s list, copied per change; while watched it records the ops of each change, a run's state ops re-pathed under its entry, delivered as one frame per batch). + +Persisted jobs (slice 6b): `persist: { resume: "rerun" | "abandon" }` in a `local` type with empty params. Each +unsettled run's record (`stored.ts`) is a document in `chord.jobs`, its state one in `chord.job-states` +(`persisted.ts`, `run-states.ts`); a run is reserved (`running`) before `run`, counted after a crash (five in a row end it +`restart_limit`), written back `stopped` by the close writer after the drain, resumed on the next live attempt (per +`resume`); the records load once per object, again only after a load failed, a record of another version or fingerprint +blocked before its state is decoded; `abort(id, ctx)` is stored and joins the current invocation; +`ctx.state.change(fn, ctx, { persistence: { kind: "deferred" } })` holds a change (its ops folded in a `PendingOps`) until +the next immediate one, the outcome, the close writer or its deadline (one timer per object). diff --git a/packages/chord/src/next/jobs/definition.ts b/packages/chord/src/next/jobs/definition.ts index c0dc86082..852d4e80d 100644 --- a/packages/chord/src/next/jobs/definition.ts +++ b/packages/chord/src/next/jobs/definition.ts @@ -2,7 +2,7 @@ import { checkKeys, entriesOf } from "../contracts/declare.ts"; import type { Static, TSchema } from "../contracts/index.ts"; -import { checkName } from "../contracts/internal.ts"; +import { checkName, hash64 } from "../contracts/internal.ts"; import { type Snapshot, snapshotArguments, snapshotSchema } from "../contracts/snapshot.ts"; import type { ChordError, PluginError } from "../data/index.ts"; import type { JsonObject, StateDefinition } from "../documents/index.ts"; @@ -11,7 +11,7 @@ import type { JsonObject, StateDefinition } from "../documents/index.ts"; export type JobOptions< A extends TSchema, R extends TSchema, - D extends StateDefinition | undefined, + D extends StateDefinition | undefined, > = { /** The job's version; every change of its args, result or state bumps it. */ readonly version: number; @@ -19,23 +19,33 @@ export type JobOptions< readonly args: A; /** What `run` returns; not validated. */ readonly result: R; - /** A singleton document definition: one document per run, `ctx.state` in `run`, `outcome.state` in `finish`. */ + /** + * A singleton document definition: one document per run, `ctx.state` in `run`, `outcome.state` in `finish`; a + * persisted job's is persisted (`persist: true`, its `checkpointWhen`). + */ readonly state?: D; /** At most this many runs running or finishing per object; a start beyond it rejects `unavailable/busy`. */ readonly concurrency?: number; /** Per older version, how its args and result map forwards (used from slice 7 on; checked now). */ readonly accepts?: JobAccepts; - /** How a persisted run resumes after a crash; refused before slice 6. */ - readonly persist?: { readonly resume: "rerun" | "abandon" }; + /** + * Persists the runs in the object's store (slice 6b; only in `local` types with empty params, P3): how a run cut + * off by a crash or a stop resumes, `rerun` (its `run` again: idempotent work) or `abandon` (`finish` gets `failed { + * aborted, reason: "restart" }`). Its result is JSON only (no `bytes()`). + */ + readonly persist?: JobPersistence; }; +/** How a persisted job's runs resume after a crash or a stop. */ +export type JobPersistence = { readonly resume: "rerun" | "abandon" }; + /** Per older version, its mappings forwards: its args to the current args, its result to the current result. */ export type JobAccepts = { readonly [version: number]: { readonly args: (old: never) => Static; readonly result: (old: never) => Static }; }; /** The value of a job's state document: the definition's document, `undefined` without one. */ -type StateOf = D extends StateDefinition ? T : undefined; +type StateOf = D extends StateDefinition ? T : undefined; /** What a run ended with, as `finish` and `outcome` get it: its args and state, and its result or its error. */ export type Outcome = { readonly id: string; readonly args: A; readonly state: S } & ( @@ -45,18 +55,20 @@ export type Outcome = { readonly id: string; readonly args: A; readonly /** * Why a run failed: `run` threw (reported as `job_failed`; anything but a ChordError or PluginError becomes a - * PluginError `internal` with its message), or it was aborted (`abort`: `abort(id)`; `reload` and `restart` come with - * slices 7 and 6). + * PluginError `internal` with its message), it was aborted (`abort`: `abort(id)`; `restart`: cut off by its attempt's + * seal while the object continues, or a persisted run abandoned after a crash; `reload` comes with slice 7), or a + * persisted run was cut off five times in a row (`restart_limit`, D6). */ export type JobError = | { readonly kind: "threw"; readonly error: ChordError | PluginError } - | { readonly kind: "aborted"; readonly reason: "abort" | "reload" | "restart" }; + | { readonly kind: "aborted"; readonly reason: "abort" | "reload" | "restart" } + | { readonly kind: "restart_limit"; readonly restarts: number }; /** Declares a job: owned work a call starts that outlives it (§15). */ export function defineJob< const A extends TSchema, const R extends TSchema, - D extends StateDefinition | undefined = undefined, + D extends StateDefinition | undefined = undefined, >(name: string, options: JobOptions): JobDefinition, Static, StateOf> { // Declaration policy: JavaScript callers reach every check below. checkName(name, "A job name"); @@ -67,12 +79,32 @@ export function defineJob< if (args.bytes) throw new Error(`${name}.args: job arguments are JSON (running carries them); no bytes()`); const result = snapshotSchema(options.result, `${name}.result`); if (state?.keyed) throw new Error(`${name}.state: expected a singleton document definition`); + if (persist !== undefined) checkPersistence(persist, state, result, name); if (concurrency !== undefined && !(Number.isSafeInteger(concurrency) && concurrency > 0)) { throw new Error(`${name}.concurrency: expected a positive integer`); } if (accepts !== undefined) checkJobAccepts(accepts, version, `${name}.accepts`); - if (persist !== undefined) throw new Error(`${name}.persist: persisted jobs are not supported before slice 6`); - return new JobDefinition(name, version, args, result, state, concurrency, accepts ?? {}); + const fingerprint = hash64( + JSON.stringify({ args: args.normal, result: result.normal, state: state?.fingerprint ?? null }), + ); + return new JobDefinition(name, version, args, result, state, concurrency, accepts ?? {}, persist, fingerprint); +} + +/** A persisted job's declaration: a resume rule, a persisted state document (if any), a JSON-only result. */ +function checkPersistence( + persist: unknown, + state: StateDefinition | undefined, + result: Snapshot, + name: string, +): void { + checkKeys(persist, ["resume"], `${name}.persist`); + const { resume } = persist as { readonly resume?: unknown }; + if (resume !== "rerun" && resume !== "abandon") + throw new Error(`${name}.persist.resume: expected "rerun" or "abandon"`); + if (state !== undefined && !state.persist) { + throw new Error(`${name}.state: a persisted job's state is a persisted document (persist: true)`); + } + if (result.bytes) throw new Error(`${name}.result: a persisted job's result is JSON (it is stored); no bytes()`); } /** Checks an accepts table's shape: older versions only, each with an `args` and a `result` function. */ @@ -96,10 +128,14 @@ export class JobDefinition { /** The args schema, Chord's copy; its compiled check is cached by this snapshot. */ readonly args: Snapshot; readonly result: Snapshot; - readonly state: StateDefinition | undefined; + readonly state: StateDefinition | undefined; readonly concurrency: number | undefined; /** The older versions' mappings, checked at declaration; slice 7 maps older runs with them. */ readonly accepts: JobAccepts; + /** How its runs persist; `undefined` for a job in memory. */ + readonly persist: JobPersistence | undefined; + /** `hash64` of its args', result's and state's normal forms: a stored run of another fingerprint is blocked. */ + readonly fingerprint: string; /** Type only: the args, result and state types. */ declare readonly "~types"?: { readonly args: A; readonly result: R; readonly state: S }; @@ -108,9 +144,11 @@ export class JobDefinition { version: number, args: Snapshot, result: Snapshot, - state: StateDefinition | undefined, + state: StateDefinition | undefined, concurrency: number | undefined, accepts: JobAccepts, + persist: JobPersistence | undefined, + fingerprint: string, ) { this.name = name; this.version = version; @@ -119,5 +157,7 @@ export class JobDefinition { this.state = state; this.concurrency = concurrency; this.accepts = accepts; + this.persist = persist; + this.fingerprint = fingerprint; } } diff --git a/packages/chord/src/next/jobs/index.ts b/packages/chord/src/next/jobs/index.ts index 5a8cd4739..b2472d220 100644 --- a/packages/chord/src/next/jobs/index.ts +++ b/packages/chord/src/next/jobs/index.ts @@ -9,5 +9,5 @@ export { type Outcome, } from "./definition.ts"; export type { JobCtx, JobHandle, JobImplementation, RunState, StartOptions } from "./job.ts"; -export type { Run } from "./run.ts"; +export type { BlockedReason, Run } from "./run.ts"; export type { RunningEntry } from "./running.ts"; diff --git a/packages/chord/src/next/jobs/job.ts b/packages/chord/src/next/jobs/job.ts index 12e4b7403..70d6703fd 100644 --- a/packages/chord/src/next/jobs/job.ts +++ b/packages/chord/src/next/jobs/job.ts @@ -7,13 +7,13 @@ import type { Draft } from "../../delta/index.ts"; import type { JsonValue } from "../../json.ts"; import type { Watchable } from "../../watch.ts"; import { ChordError, isChordError, PluginError, type Principal } from "../data/index.ts"; -import type { JsonObject, Tx } from "../documents/index.ts"; +import type { CommitPersistence, JsonObject, Tx } from "../documents/index.ts"; import type { Documents, SealToken, Transaction } from "../documents/internal.ts"; import { type Ctx, sleep } from "../host/index.ts"; import { type AnyHost, CallContext, type Caller, nextDelay } from "../host/internal.ts"; import type { Outcome } from "./definition.ts"; import type { JobRecords, Settled } from "./records.ts"; -import type { AnyOutcome, JobRun, Run } from "./run.ts"; +import type { AnyOutcome, Invocation, JobRun, Run } from "./run.ts"; import type { RunningEntry } from "./running.ts"; /** A job's handle, `self.job(Job, { run, finish? })`; `A`, `R` and `S` are its args, result and state. */ @@ -27,10 +27,12 @@ export interface JobHandle { /** Waits for the outcome of the unsettled run `id`; a settled or unknown id rejects `gone`. */ outcome(id: string, context: Context): Promise>; /** - * Fires a running run's abort signal; drops a blocked run (no `finish`; its waiters reject `aborted`); ignored for - * finishing runs and unknown ids. + * Aborts run `id` (D7, as Durable's `abortTask`): marks it (stored for a persisted job) and fires its `run`'s + * signal, resolving `"marked"` once its `run` settled (a run not invoked yet ends `aborted` without one); a + * finishing run is only marked, a blocked one dropped (no `finish`; its waiters reject `aborted`). `"terminal"` when + * no unsettled run has that id. `context` ends only this wait. */ - abort(id: string): void; + abort(id: string, context: Context): Promise<"marked" | "terminal">; /** The unsettled runs, one entry each in start order, tagged by phase (a JSON watchable, a state member). */ readonly running: Watchable[]>; } @@ -60,9 +62,15 @@ export interface RunState { readonly value: S; /** * One transaction on the object's line drafting only this document, under `context` without its abort signal: - * admitted while `run` runs (also after an abort, with the run's own ctx), refused `released` once it settled. + * admitted while `run` runs (also after an abort, with the run's own ctx), refused `released` once it settled. A + * persisted job's change may be `deferred` (D8): adopted and published at once, stored within `withinMs` (1,000 ms by + * default) or earlier; a crash loses it. In memory the option changes nothing. */ - change(fn: (draft: Draft) => T, context: Context): Promise; + change( + fn: (draft: Draft) => T, + context: Context, + options?: { readonly persistence?: CommitPersistence }, + ): Promise; } /** Where a job's handle runs its work: its attempt, as the job sees it. */ @@ -105,6 +113,8 @@ export class Job implements JobHandle #state: "declared" | "live" | "sealed" = "declared"; /** Runs whose `run` runs on this attempt. */ readonly #invocations = new Set(); + /** The seal's error, once sealed: a reservation committing after it cuts its run off with it. */ + #sealed: ChordError | undefined; /** Runs whose finish runs on this attempt. */ readonly #finishing = new Set(); @@ -128,8 +138,18 @@ export class Job implements JobHandle return this.#records.outcome(id, context); } - abort(id: string): void { - this.#records.abort(id); + abort(id: string, context: Context): Promise<"marked" | "terminal"> { + return this.#records.abort(id, context); + } + + /** A persisted job's runs, loaded once per object at its first activation (before it goes live). */ + load(context: Context): Promise { + return this.#records.load(context); + } + + /** The close writer of this job's runs, after the attempt's drain (slice 6b §4.2). */ + writeAtClose(context: Context): Promise { + return this.#records.writeAtClose(context); } /** The attempt went live: starts are taken, and the records' waiting runs are invoked or finished here. */ @@ -170,6 +190,7 @@ export class Job implements JobHandle seal(error: ChordError): void { if (this.#state === "sealed") return; this.#state = "sealed"; + this.#sealed = error; this.#records.detach(this, this.#finishing); this.#finishing.clear(); for (const run of this.#invocations) this.#records.cut(run, error); @@ -177,30 +198,70 @@ export class Job implements JobHandle } /** - * Runs `run` for a committed run (with its signal aborted already after an early abort). The invocation and the + * Runs `run` for a committed run (with its signal aborted already after an early abort); a persisted run's + * reservation commits first (`running`: a crash from then on counts, D6). The reservation, the invocation and the * run's finish are one piece of work in the drain, admitted before plugin code runs: `run` may end its own object * before its first await. */ invoke(run: JobRun): void { + const invocation = run.begin(); + if (!this.#records.persisted || run.phase === "running") { + this.#start(run, invocation); + return; + } + const { documents, token, context } = this.#owner; + const reserving = documents + .tx((tx) => this.#records.reserve(tx, run), token, context) + .then( + () => { + if (this.#state !== "sealed") { + this.#start(run, invocation); + return; + } + this.#records.cut(run, this.#sealed!); + // No `run` started: nothing else ends it, and an abort joined to it returns. + invocation.ended(); + }, + () => { + invocation.settle(); + invocation.ended(); + // Refused because this attempt was sealed (at admission, or queued past the seal): the run is pending still, + // for the next live attempt (this one is detached already). Any other failure (the store closed or failed, + // the object ended) is not retried here: retrying on this attempt would only fail again. + if (this.#state === "sealed") this.#records.requeue(run); + }, + ); + this.#owner.admit(reserving); + } + + /** Starts `invocation` of `run` on this attempt: its ctx and its settling act for this invocation only. */ + #start(run: JobRun, invocation: Invocation): void { const { context, caller, host } = this.#owner; - const ctx = new JobContext(withAbortSignal(run.abort.signal, context), caller, host, run.id, this.#stateOf(run)); + const signal = invocation.abort.signal; + const ctx = new JobContext( + withAbortSignal(signal, context), + caller, + host, + run.id, + this.#stateOf(run, invocation), + ); const work = Promise.withResolvers(); this.#owner.admit(work.promise, () => ctx.settle()); this.#invocations.add(run); - run.invocation = "running"; - const invocation = (async () => { + const running = (async () => { try { return await this.#implementation.run(run.args, ctx); } finally { // The fence closes as `run` settles: a change made later is refused. - run.invocation = "settled"; + invocation.settle(); } })(); - invocation + running .then( - (result) => this.#settled(run, ctx, { kind: "done", result }), - (error: unknown) => this.#settled(run, ctx, this.#failed(run, error)), + (result) => this.#settled(run, invocation, ctx, { kind: "done", result }), + (error: unknown) => this.#settled(run, invocation, ctx, this.#failed(run, invocation, error)), ) + .finally(() => invocation.ended()) .then(work.resolve); } @@ -214,21 +275,22 @@ export class Job implements JobHandle this.#owner.admit(work); } - /** How a throwing `run` ended: `aborted` after its signal fired, otherwise `threw` (reported). */ - #failed(run: JobRun, error: unknown): Settled { - if (run.abort.signal.aborted) return { kind: "failed", error: { kind: "aborted", reason: "abort" } }; + /** How a throwing `run` ended: `aborted` after its invocation's signal fired, otherwise `threw` (reported). */ + #failed(run: JobRun, invocation: Invocation, error: unknown): Settled { + if (invocation.abort.signal.aborted) return { kind: "failed", error: { kind: "aborted", reason: "abort" } }; this.#records.report({ kind: "job_failed", id: run.id, error }); const thrown = isChordError(error) || error instanceof PluginError ? error : internal(error); return { kind: "failed", error: { kind: "threw", error: thrown } }; } /** - * `run` settled: its `ctx.get` references go; its outcome is recorded unless the attempt's seal cut it off (which - * recorded its own), then the run finishes here while this attempt is live. + * `invocation` of `run` settled: its `ctx.get` references go; its outcome is recorded unless the attempt's seal cut + * it off (which recorded its own, or stopped the run for a later invocation), then the run finishes here while this + * attempt is live. */ - async #settled(run: JobRun, ctx: JobContext, ended: Settled): Promise { + async #settled(run: JobRun, invocation: Invocation, ctx: JobContext, ended: Settled): Promise { ctx.settle(); - if (run.token.sealed) return; + if (invocation.token.sealed || run.invocation !== invocation) return; this.#invocations.delete(run); this.#records.settled(run, ended); } @@ -238,14 +300,14 @@ export class Job implements JobHandle this.#finishing.add(run); let delay: number | undefined; try { - for (let failures = 1; ; failures++) { + for (;;) { try { return await documents.tx( async (tx) => { const staging = this.#records.staging(tx); staging.finishing = run; await this.#implementation.finish?.(run.outcome!, tx); - staging.changes.push({ kind: "delete", run }); + this.#records.change(tx, { kind: "delete", run }); }, token, context, @@ -253,7 +315,12 @@ export class Job implements JobHandle } catch (error) { if (this.#state === "sealed") return; this.#records.report({ kind: "job_finish_failed", id: run.id, error }); - if (failures === FINISH_ATTEMPTS) return await this.#records.record(run, "block"); + // The count is the run's (stored for a persisted job): failures count across a crash. + const finishFailures = run.finishFailures + 1; + if (finishFailures >= FINISH_ATTEMPTS) { + return await this.#records.record(run, { kind: "block", run, reason: "finish_failed" }); + } + await this.#records.record(run, { kind: "update", run, fields: { finishFailures } }); delay = nextDelay(delay); try { await sleep(delay, withAbortSignal(signal, context)); @@ -267,8 +334,8 @@ export class Job implements JobHandle } } - /** `ctx.state` of `run`, when the job has a state: changes under the run's own token while `run` runs. */ - #stateOf(run: JobRun): RunState | undefined { + /** `ctx.state` of `invocation` of `run`, when the job has a state: changes under its token while it runs. */ + #stateOf(run: JobRun, invocation: Invocation): RunState | undefined { const records = this.#records; const states = records.states; if (states === undefined) return undefined; @@ -285,16 +352,59 @@ export class Job implements JobHandle if (value === undefined) throw refused(); return value; }, - change: (fn: (draft: Draft) => T, context: Context): Promise => { - if (run.invocation !== "running") return Promise.reject(refused()); + change: ( + fn: (draft: Draft) => T, + context: Context, + options: { readonly persistence?: CommitPersistence } = {}, + ): Promise => { + if (invocation.state !== "running") return Promise.reject(refused()); + let persistence: CommitPersistence; + try { + persistence = checkedPersistence(options.persistence, records.name); + } catch (error) { + return Promise.reject(error); + } // Never the abort signal: a run records its progress also after an abort. const unaborted = withoutAbortSignal(context); - return documents.tx(async (tx) => fn(await tx.draftSlot(states, run.id)), run.token, unaborted); + // A change resetting the restart count is stored at once: the reset must be durable (D8). + const now = run.restarts === 0 ? persistence : IMMEDIATE; + return documents.tx( + async (tx) => { + const value = fn(await tx.draftSlot(states, run.id)); + records.progressed(tx, run); + return value; + }, + invocation.token, + unaborted, + now, + ); }, }; } } +const IMMEDIATE: CommitPersistence = { kind: "immediate" }; + +/** + * A state change's `persistence` (a trust boundary: plugin code): `immediate` when omitted; a deferred one's `withinMs` + * a number from 0 (`Infinity` waits for the others), `invalid/args` otherwise. + */ +function checkedPersistence(persistence: CommitPersistence | undefined, what: string): CommitPersistence { + if (persistence === undefined || persistence.kind === "immediate") return IMMEDIATE; + const { withinMs } = persistence; + if ( + persistence.kind !== "deferred" || + (withinMs !== undefined && !(typeof withinMs === "number" && withinMs >= 0)) + ) { + throw new ChordError({ + code: "invalid", + reason: "args", + message: `${what}: persistence is immediate or deferred with withinMs >= 0`, + }); + } + return persistence; +} + /** A plugin's throw that is neither a ChordError nor a PluginError, as a run's outcome keeps it. */ function internal(error: unknown): PluginError { return new PluginError("internal", error instanceof Error ? error.message : String(error)); diff --git a/packages/chord/src/next/jobs/persisted.ts b/packages/chord/src/next/jobs/persisted.ts new file mode 100644 index 000000000..9802ad441 --- /dev/null +++ b/packages/chord/src/next/jobs/persisted.ts @@ -0,0 +1,100 @@ +// A persisted job's run records in its object's store (slice 6b §4.1, D5): the record document of each unsettled run +// (namespace `chord.jobs`, kind the job name, key the run id, `history: none`), written whole in the transactions that +// change the run, and read once per object when it first activates; for a run of another job version, also what is +// stored of its state document (`chord.job-states`), never decoded, so its drop retires it whatever the code declares. + +import type { Context } from "../../context/index.ts"; +import type { JsonObject } from "../documents/index.ts"; +import type { Persistence, Stored } from "../documents/internal.ts"; +import { changeDocument, createDocument, retireDocument } from "../store/engine.ts"; +import type { StagedWrite } from "../store/step.ts"; +import { RUN_RECORD, recordOf, type StoredRun } from "./stored.ts"; + +/** A run record as a load found it: its run id, the record and what is stored of it. */ +export type LoadedRecord = { readonly id: string; readonly record: StoredRun; readonly stored: Stored }; + +/** The keys a load reads per page. */ +const KEYS_PAGE = 100; + +/** The record documents of one persisted job's runs, and what is stored of each. */ +export class PersistedRuns { + readonly #records: Persistence; + readonly #states: Persistence; + readonly #stored = new Map(); + /** What is stored of the state documents of runs of another job version, by run id. */ + readonly #blockedStates = new Map(); + + /** `records` and `states` are the job's record and state documents' store sides (`chord.jobs`, `chord.job-states`). */ + constructor(records: Persistence, states: Persistence) { + this.#records = records; + this.#states = states; + } + + /** + * Stages run `id`'s record `record` into `writes` (rule 8: serialized now): its creation, or a new base of what is + * stored; returns the adoption, which takes what the commit stored. + */ + stage(id: string, record: StoredRun, writes: StagedWrite[]): () => void { + const text = JSON.stringify(record); + const stored = this.#stored.get(id); + const { version, fingerprint } = RUN_RECORD; + const value = record as unknown as JsonObject; + if (stored === undefined) { + const created = { id: 0 }; + writes.push( + createDocument(this.#records.at(id), { version, fingerprint, value }, text, (made) => { + created.id = made; + }), + ); + return () => this.#stored.set(id, { id: created.id, revision: 1, version, deltasSinceBase: 0 }); + } + writes.push(changeDocument(stored.id, stored.revision, { kind: "base", version, fingerprint, value }, text)); + return () => this.#stored.set(id, { ...stored, revision: stored.revision + 1 }); + } + + /** + * Stages the retirement of run `id`'s record into `writes` (the run settled or was dropped), and of its state + * document when the run is of another job version; the adoption forgets them. + */ + stageRetire(id: string, writes: StagedWrite[]): () => void { + const stored = this.#stored.get(id); + if (stored !== undefined) writes.push(retireDocument(stored.id, stored.revision)); + const state = this.#blockedStates.get(id); + if (state !== undefined) writes.push(retireDocument(state.id, state.revision)); + return () => { + this.#stored.delete(id); + this.#blockedStates.delete(id); + }; + } + + /** Finds what is stored of run `id`'s state document (a run of another job version), keeping it for the drop. */ + async findBlockedState(id: string, context: Context): Promise { + const stored = await this.#states.find(id, context); + if (stored !== undefined) this.#blockedStates.set(id, stored); + } + + /** Every stored record of the job, in run id order, page by page with plain reads (activation holds the line). */ + async load(context: Context): Promise { + const loaded: LoadedRecord[] = []; + let after: string | undefined; + for (;;) { + const keys = await this.#records.keys(after, KEYS_PAGE, context); + for (const id of keys) { + const document = await this.#records.read(id, context); + if (document === undefined) continue; + const { record: stored, value, deltasSinceBase } = document; + const found = { id: stored.id, revision: stored.revision, version: stored.version, deltasSinceBase }; + this.#stored.set(id, found); + loaded.push({ id, record: recordOf(value as JsonObject), stored: found }); + } + if (keys.length < KEYS_PAGE) return loaded; + after = keys.at(-1); + } + } + + /** The object's documents are dropped: nothing is known of the store any more. */ + drop(): void { + this.#stored.clear(); + this.#blockedStates.clear(); + } +} diff --git a/packages/chord/src/next/jobs/records.ts b/packages/chord/src/next/jobs/records.ts index b1e8e306b..73cc9f91e 100644 --- a/packages/chord/src/next/jobs/records.ts +++ b/packages/chord/src/next/jobs/records.ts @@ -1,20 +1,27 @@ -// A job's records on its object (§15, slice 6a §1.2): its runs by id and by key, the active count, the run state -// documents, `running` and the runs waiting for a live attempt, across the object's attempts. Records and state -// documents change only in transactions on the object's line (staged changes adopted in the commit step). +// A job's records on its object (§15, slice 6a §1.2, slice 6b §4): its runs by id and by key, the active count, the +// run state documents, `running` and the runs waiting for a live attempt, across the object's attempts. Records and +// state documents change only in transactions on the object's line (staged changes adopted in the commit step); a +// persisted job's also in its object's store, in the same commits, loaded once per object, and written back stopped by +// the close writer when a clean stop cuts its runs off. -import type { Context } from "../../context/index.ts"; -import { type Op, type Prepared, type Tracker, track } from "../../delta/index.ts"; +import { type Context, withoutAbortSignal } from "../../context/index.ts"; +import type { Op } from "../../delta/index.ts"; import type { JsonValue } from "../../json.ts"; import { ChordError } from "../data/index.ts"; import { check, validatorFor } from "../data/internal.ts"; -import type { JsonObject, StateDefinition } from "../documents/index.ts"; -import type { DocumentSlot, Documents, Staged, Transaction } from "../documents/internal.ts"; +import type { JsonObject } from "../documents/index.ts"; +import { type Documents, Persistence, type Staged, type Transaction } from "../documents/internal.ts"; import type { AnyHost } from "../host/internal.ts"; import { uuidv7 } from "../host/internal.ts"; +import type { StagedWrite } from "../store/step.ts"; +import type { ChordStore } from "../store/store.ts"; import type { JobDefinition, JobError } from "./definition.ts"; import type { Job, StartOptions } from "./job.ts"; -import { type AnyOutcome, JobRun, type Run } from "./run.ts"; +import { type LoadedRecord, PersistedRuns } from "./persisted.ts"; +import { type AnyOutcome, type BlockedReason, JobRun, type Phase, type Run } from "./run.ts"; +import { RunStates } from "./run-states.ts"; import { Running } from "./running.ts"; +import { outcomeOf, type StoredFields, storedRun } from "./stored.ts"; /** What a job's records need of their object. */ export type RecordsOwner = { @@ -23,8 +30,12 @@ export type RecordsOwner = { /** The object's context, without an abort signal: outcomes are recorded under it. */ readonly context: Context; readonly documents: Documents; + /** The host's store and the object's owner name in it, when the object may persist (P3). */ + readonly store: { readonly store: ChordStore; readonly owner: string } | undefined; /** Counts running and finishing runs: they keep the object alive (`host.activity.jobs`). */ countRuns(delta: number): void; + /** A held state change must be stored by `deadline`: the object's one timer flushes it (D8). */ + held(deadline: number): void; }; /** How a run's invocation ended, before its outcome is recorded. */ @@ -32,21 +43,45 @@ export type Settled = | { readonly kind: "done"; readonly result: unknown } | { readonly kind: "failed"; readonly error: JobError }; +/** What a change sets on a run's record besides its phase transitions below (D6, D7, the finish failure count). */ +type RunFields = { + readonly phase?: Phase; + readonly restarts?: number; + readonly finishFailures?: number; + readonly abortMarked?: true; +}; + +/** A loaded run and the change its activation writes, if any. */ +type Restored = { readonly run: JobRun; readonly change: RecordChange | undefined }; + /** A change of the records, staged in a transaction and adopted in its commit step. */ type RecordChange = - | { readonly kind: "create"; readonly run: JobRun; readonly state: JsonObject | undefined } + | { readonly kind: "create"; readonly run: JobRun } + | { readonly kind: "update"; readonly run: JobRun; readonly fields: RunFields } | { readonly kind: "outcome"; readonly run: JobRun; readonly outcome: AnyOutcome } - | { readonly kind: "delete" | "block" | "drop"; readonly run: JobRun }; + | { readonly kind: "block"; readonly run: JobRun; readonly reason: BlockedReason } + | { readonly kind: "delete"; readonly run: JobRun } + | { readonly kind: "drop"; readonly run: JobRun }; + +/** What a commit step adopts of the records: a change, or a loaded run taken into memory as stored. */ +type Adopted = RecordChange | { readonly kind: "restore"; readonly run: JobRun }; /** One transaction's changes of one job: the runs it started (dedupe, concurrency) and whose finish it is. */ export type Staging = Staged & { readonly started: JobRun[]; - readonly changes: RecordChange[]; + readonly changes: Adopted[]; + /** What the transaction's store writes leave in memory, adopted with the changes. */ + readonly adopts: (() => void)[]; finishing: JobRun | undefined; }; const RESTART: Settled = { kind: "failed", error: { kind: "aborted", reason: "restart" } }; +const ABORTED: Settled = { kind: "failed", error: { kind: "aborted", reason: "abort" } }; + +/** A persisted run cut off this many times in a row ends `restart_limit` (D6, Durable's limit). */ +const RESTART_LIMIT = 5; + /** * The records of one job name on one object. The live attempt's `Job` invokes and finishes its runs; between live * attempts, committed runs not yet invoked and runs whose outcome is recorded wait for the next one. @@ -56,6 +91,8 @@ export class JobRecords { readonly running: Running; readonly #owner: RecordsOwner; readonly #states: RunStates | undefined; + /** The store side of a persisted job's runs; `undefined` in memory. */ + readonly #persisted: PersistedRuns | undefined; readonly #runs = new Map(); readonly #keys = new Map(); /** Running and finishing runs: concurrency and keep-alive count them. */ @@ -64,17 +101,30 @@ export class JobRecords { #job: Job | undefined; /** Runs waiting for a live attempt: to be invoked (`invocation` pending) or finished (phase `finishing`). */ readonly #waiting = new Set(); + /** Runs a clean stop cut off whose record the close writer has not written `stopped` yet. */ + readonly #stopped = new Set(); + /** Whether a persisted job's records were loaded (once per object). */ + #loaded = false; /** The object's end error, once it ended. */ #ended: ChordError | undefined; constructor(definition: JobDefinition, owner: RecordsOwner) { this.definition = definition; this.#owner = owner; - const { state } = definition; - this.#states = - state === undefined + const { state, persist, name } = definition; + const store = persist === undefined ? undefined : owner.store!; + const states = { + documents: owner.documents, + changed: (id: string, ops: readonly Op[]) => this.#stateChanged(id, ops), + held: (deadline: number) => owner.held(deadline), + }; + const persistence = + store === undefined ? undefined : new Persistence(store.store, store.owner, "chord.job-states", name); + this.#states = state === undefined ? undefined : new RunStates(state, states, persistence); + this.#persisted = + store === undefined ? undefined - : new RunStates(state, owner.documents, (id, ops) => this.#stateChanged(id, ops)); + : new PersistedRuns(new Persistence(store.store, store.owner, "chord.jobs", name), persistence!); this.running = new Running(owner.documents.batch, this.name); } @@ -92,15 +142,24 @@ export class JobRecords { return run.handle.outcome(context); } - /** Fires a running run's signal; drops a blocked run; ignored for finishing runs, unknown ids and after the end. */ - abort(id: string): void { + /** + * Aborts run `id` (D7, Durable's `abortTask`): a blocked run is dropped; a finishing one is only marked; any other + * is marked (stored for a persisted job) in a transaction, its signal fired after the commit, outside the batch and + * the line, and the abort joins its invocation. `"marked"`, or `"terminal"` when no unsettled run has that id. + */ + async abort(id: string, context: Context): Promise<"marked" | "terminal"> { const run = this.#runs.get(id); - if (run === undefined || this.#ended !== undefined) return; - if (run.phase === "running") { - run.abort.abort(new ChordError({ code: "aborted", message: `${this.name}: run ${id} aborted` })); - } else if (run.phase === "blocked") { - void this.record(run, "drop"); + if (run === undefined || this.#ended !== undefined) return "terminal"; + if (run.phase === "blocked") { + await this.record(run, { kind: "drop", run }); + return "marked"; } + if (!run.abortMarked) await this.record(run, { kind: "update", run, fields: { abortMarked: true } }); + // A finishing run's outcome stays: it is only marked. + if (run.phase === "finishing") return "marked"; + run.invocation?.abort.abort(new ChordError({ code: "aborted", message: `${this.name}: run ${id} aborted` })); + await run.over(context); + return "marked"; } /** See `Job.stage`: the checks and the staged creation, against the object's runs. */ @@ -120,26 +179,62 @@ export class JobRecords { const message = `${this.name}: at its concurrency limit (${concurrency})`; throw new ChordError({ code: "unavailable", reason: "busy", message }); } - const state = definition.state?.initial(context); + const persisted = this.#persisted !== undefined; + const trace = persisted ? context.telemetry.propagation?.inject(context.telemetry) : undefined; // Checked above: the args are JSON (the definition refuses bytes). - const run = new JobRun(uuidv7(), definition.version, key, args as JsonValue); + const run = new JobRun( + uuidv7(), + definition.version, + key, + args as JsonValue, + persisted ? "pending" : "running", + trace, + ); const staging = this.staging(tx); staging.started.push(run); - staging.changes.push({ kind: "create", run, state }); + this.change(tx, { kind: "create", run }); + const state = definition.state?.initial(context); + if (state !== undefined) staging.adopts.push(this.#states!.stageCreate(run.id, state, () => this.#store(tx))); return run.handle; } + /** + * Stages `change` in `tx`: in the job's staging, adopted in the commit step, and for a persisted job the run's record + * written with it (rule 8: serialized now), and for a settled run its state document retired. + */ + change(tx: Transaction, change: RecordChange): void { + const staging = this.staging(tx); + staging.changes.push(change); + const { run } = change; + const states = this.#states; + const retires = change.kind === "delete" || change.kind === "drop"; + if (retires && states?.current(run.id) !== undefined) tx.retireSlot(states, run.id); + const persisted = this.#persisted; + if (persisted === undefined) return; + const writes = this.#store(tx); + if (retires) { + staging.adopts.push(persisted.stageRetire(run.id, writes)); + return; + } + const fields = recordFields(change); + staging.adopts.push(persisted.stage(run.id, storedRun(run, this.definition.fingerprint, fields), writes)); + } + /** This job's changes in `tx`, made on the first. */ staging(tx: Transaction): Staging { return tx.stage(this, () => { - const changes: RecordChange[] = []; + const changes: Adopted[] = []; + const adopts: (() => void)[] = []; return { started: [], changes, + adopts, finishing: undefined, adopt: () => { // A step admitted before the object ended adopts after it: the records are not changed any more. if (this.#ended !== undefined) return; + // What is stored first: a new run's state document exists when its `running` entry is made. + runAll(adopts); for (const change of changes) this.#adopt(change); }, committed: () => { @@ -158,6 +253,49 @@ export class JobRecords { }); } + /** + * A persisted job's runs, loaded once per object at its first activation (slice 6b §4.2), again only after a load + * failed (design §9): each record read; a record of another version or fingerprint blocked (`version`) as stored, + * its state document left undecoded; the others' state documents read and decoded, a run cut off while `running` + * counted (`restarts + 1`; at the limit it ends `restart_limit`, reported `job_interrupted`), `stopped` and + * `pending` ones made `pending` again. The runs and their activation changes are adopted in one commit, all before + * anything runs; a failure leaves nothing loaded. + */ + async load(context: Context): Promise { + const persisted = this.#persisted; + if (persisted === undefined || this.#loaded) return; + const restored: Restored[] = []; + try { + for (const loaded of await persisted.load(context)) restored.push(await this.#restored(loaded, context)); + if (restored.length > 0) { + const { documents } = this.#owner; + await documents.tx((tx) => this.#restoreAll(tx, restored), documents.own, context); + } + } catch (error) { + persisted.drop(); + for (const { run } of restored) this.#states?.retire(run.id); + throw error; + } + this.#loaded = true; + for (const { run, change } of restored) { + if ( + change?.kind === "outcome" && + change.outcome.kind === "failed" && + change.outcome.error.kind === "restart_limit" + ) { + this.report({ kind: "job_interrupted", id: run.id, restarts: change.outcome.error.restarts }); + } + } + for (const { run } of restored) this.#need(run); + } + + /** Stages the loaded runs in `tx`: each restored, then the activation changes (adopted after every restore). */ + #restoreAll(tx: Transaction, restored: readonly Restored[]): void { + const staging = this.staging(tx); + for (const { run } of restored) staging.changes.push({ kind: "restore", run }); + for (const { change } of restored) if (change !== undefined) this.change(tx, change); + } + /** `job`, of the attempt that went live, invokes and finishes the runs from now on; the waiting ones first. */ attach(job: Job): void { if (this.#ended !== undefined) return; @@ -176,6 +314,24 @@ export class JobRecords { for (const run of finishing) this.#need(run); } + /** + * A run whose reservation its sealed attempt never committed waits for the next live attempt (dispatched now when one + * is attached already: the sealed attempt was detached at its seal). + */ + requeue(run: JobRun): void { + this.#need(run); + } + + /** Records `run` running before its `run` is invoked (a persisted run's reservation: a crash after it counts). */ + reserve(tx: Transaction, run: JobRun): void { + this.change(tx, { kind: "update", run, fields: { phase: "running" } }); + } + + /** A state change of a run with restarts counted resets them in its commit (progress, D6). */ + progressed(tx: Transaction, run: JobRun): void { + if (run.restarts !== 0) this.change(tx, { kind: "update", run, fields: { restarts: 0 } }); + } + /** * A run's invocation settled: its outcome is recorded (phase `finishing`) in a transaction of the object behind the * state changes admitted before, so `outcome.state` has them; then it is finished on the live attempt. @@ -185,30 +341,38 @@ export class JobRecords { } /** - * A running run whose attempt was sealed while the object continues: its signal fires with `error`, its invocation - * is fenced (later state changes refused, the open one discarded, its settling ignored), and it ends `failed { - * aborted, reason: "restart" }`, finished on the next live attempt. + * A running run whose attempt was sealed: its signal fires with `error`, its invocation is fenced (later state + * changes refused, the open one discarded, its settling ignored). In memory it ends `failed { aborted, reason: + * "restart" }`, finished on the next live attempt; persisted it is stopped (R3), written back by the close writer and + * resumed by the next live attempt as after a clean close. */ cut(run: JobRun, error: ChordError): void { - run.invocation = "settled"; - this.#owner.documents.fence(run.token); - run.abort.abort(error); + // Only an invoked run is cut: its current invocation is the one the seal stops. + const invocation = run.invocation!; + invocation.settle(); + this.#owner.documents.fence(invocation.token); + invocation.abort.abort(error); + if (this.#persisted !== undefined) { + run.phase = "stopped"; + // Resumed per the job's `resume` (an `abandon` job's run ends `failed { aborted, restart }`). + run.resuming = true; + this.#stopped.add(run); + if (this.#ended === undefined) this.#need(run); + return; + } if (this.#ended === undefined) void this.#recordOutcome(run, RESTART); } /** - * Records `kind` for `run` in a transaction of the object: `block` a finishing run, `drop` a blocked one (its - * record, key and state document go), if it still is one there. The object's end ends it. + * Records `change` (a run's finish failure, its block, its drop) in a transaction of the object, if `run` is still + * one of its runs there. The object's end ends it. */ - async record(run: JobRun, kind: "block" | "drop"): Promise { + async record(run: JobRun, change: RecordChange): Promise { const { documents, context } = this.#owner; try { await documents.tx( (tx) => { - const current = this.#runs.get(run.id) === run; - if (current && run.phase === (kind === "block" ? "finishing" : "blocked")) { - this.staging(tx).changes.push({ kind, run }); - } + if (this.#runs.get(run.id) === run) this.change(tx, change); }, documents.own, context, @@ -218,19 +382,81 @@ export class JobRecords { } } + /** + * The close writer (slice 6b §4.2, Durable's `writeAtClose`): one commit past the seal writing the runs a clean stop + * cut off `stopped` and every held state change, straight to the store (the object's transactions are refused by + * now). Its failure is the store's; the records stay as they were (a crash counts them). + */ + async writeAtClose(context: Context): Promise { + const persisted = this.#persisted; + if (persisted === undefined) return; + const writes: StagedWrite[] = []; + const adopts: (() => void)[] = []; + for (const run of this.#stopped) { + adopts.push(persisted.stage(run.id, storedRun(run, this.definition.fingerprint), writes)); + } + this.#stopped.clear(); + for (const id of this.#states?.heldIds ?? []) { + const adopt = this.#states!.stageHeld(id, writes); + if (adopt !== undefined) adopts.push(adopt); + } + if (writes.length === 0) return; + const fence = { admit: () => true, adopt: () => runAll(adopts) }; + try { + await this.#owner.store!.store.commit(writes, fence, withoutAbortSignal(context)); + } catch { + // The store failed or refused it (reported as the store's): a crash loses as much. + } + } + + /** Stores every held state change whose deadline passed (`now`), in one transaction of the object (D8). */ + async flushHeld(now: number): Promise { + const states = this.#states; + if (states === undefined || states.heldIds.length === 0) return; + const { documents, context } = this.#owner; + try { + await documents.tx( + (tx) => { + const staging = this.staging(tx); + for (const id of states.heldIds) { + if (states.deadline(id) > now) continue; + const adopt = states.stageHeld(id, this.#store(tx)); + if (adopt !== undefined) staging.adopts.push(adopt); + } + }, + documents.own, + context, + ); + } catch { + // The object is ending: its close writer stores what is held. + } + } + + /** The earliest deadline of a held state change (`Infinity`: none). */ + nextDeadline(): number { + const states = this.#states; + if (states === undefined) return Number.POSITIVE_INFINITY; + return Math.min(Number.POSITIVE_INFINITY, ...states.heldIds.map((id) => states.deadline(id))); + } + /** The value of run `id`'s state document, `undefined` without one or once retired. */ stateOf(id: string): JsonObject | undefined { return this.#states?.value(id); } /** The run state documents, for `ctx.state.change`; `undefined` without a state definition. */ - get states(): DocumentSlot | undefined { + get states(): RunStates | undefined { return this.#states; } + /** Whether the job persists its runs. */ + get persisted(): boolean { + return this.#persisted !== undefined; + } + /** - * The object ends: running runs' signals fire, every waiter rejects with `error`, blocked runs are dropped - * (reported); nothing finishes. The records stay readable until `drop`. + * The object ends: running runs' signals fire, every waiter rejects with `error`; blocked in-memory runs are + * dropped (reported), persisted ones keep their records; nothing finishes. The records stay readable until `drop`. */ end(error: ChordError): void { if (this.#ended !== undefined) return; @@ -238,9 +464,11 @@ export class JobRecords { this.#job = undefined; this.#waiting.clear(); for (const run of this.#runs.values()) { - if (run.phase === "blocked") this.report({ kind: "job_dropped", id: run.id }); - run.abort.abort(error); + if (run.phase === "blocked" && this.#persisted === undefined) this.report({ kind: "job_dropped", id: run.id }); + const { invocation } = run; + invocation?.abort.abort(error); run.reject(error); + if (invocation?.state !== "running") invocation?.ended(); } if (this.#active > 0) this.#owner.countRuns(-this.#active); } @@ -250,26 +478,87 @@ export class JobRecords { this.#runs.clear(); this.#keys.clear(); this.#states?.drop(); + this.#persisted?.drop(); this.running.end(); } - /** Reports a run's failure or drop to the host. */ + /** Reports a run's failure, drop or interruption to the host. */ report( error: | { readonly kind: "job_failed" | "job_finish_failed"; readonly id: string; readonly error: unknown } - | { readonly kind: "job_dropped"; readonly id: string }, + | { readonly kind: "job_dropped"; readonly id: string } + | { readonly kind: "job_interrupted"; readonly id: string; readonly restarts: number }, ): void { this.#owner.host.report({ ...error, object: this.#owner.label, job: this.definition.name }); } - /** Records the outcome of `run`, then has it finished on the live attempt. */ + /** The store writes of `tx` (Chord's own documents of a persisted job). */ + #store(tx: Transaction): StagedWrite[] { + return tx.stageStore(this.#owner.store!.store); + } + + /** + * The run a loaded record stands for and the change its activation writes (D6): a record of another version or + * fingerprint blocked as stored (its state document only found, never decoded: slice 7 maps it); otherwise its state + * document read and decoded (migrated), pinned while it is unsettled, a crash counted, at the limit its outcome, + * marked resuming unless blocked or finishing. + */ + async #restored(loaded: LoadedRecord, context: Context): Promise { + const { id, record } = loaded; + const { definition } = this; + const run = new JobRun(id, record.version, record.key, record.args, record.phase, record.trace); + run.restarts = record.restarts; + run.finishFailures = record.finishFailures; + run.abortMarked = record.abort === true; + const states = this.#states; + if (record.version !== definition.version || record.fingerprint !== definition.fingerprint) { + run.phase = "blocked"; + run.blocked = "version"; + // What is stored of its state (also when this version declares none), so an abort drops it with the record. + await this.#persisted!.findBlockedState(id, context); + return { run, change: undefined }; + } + const document = await states?.persistence.read(id, context); + if (document !== undefined) { + const { value, stored } = states!.persistence.decode(document, states!.definition.current, context); + states!.create(id, value, stored); + } + if (record.outcome !== undefined) run.outcome = outcomeOf(record.outcome, id, record.args, this.stateOf(id)); + run.blocked = record.blocked; + switch (record.phase) { + case "running": { + const restarts = record.restarts + 1; + run.resuming = true; + if (restarts < RESTART_LIMIT) { + return { run, change: { kind: "update", run, fields: { phase: "pending", restarts } } }; + } + run.restarts = restarts; + const outcome = this.#outcome(run, { kind: "failed", error: { kind: "restart_limit", restarts } }); + return { run, change: { kind: "outcome", run, outcome } }; + } + case "stopped": + run.resuming = true; + return { run, change: { kind: "update", run, fields: { phase: "pending" } } }; + case "pending": + run.resuming = true; + return { run, change: undefined }; + default: + return { run, change: undefined }; + } + } + + /** Records the outcome of `run` (with what its state holds, D8), then has it finished on the live attempt. */ async #recordOutcome(run: JobRun, ended: Settled): Promise { const { documents, context } = this.#owner; try { await documents.tx( (tx) => { - const outcome = { id: run.id, args: run.args, state: this.stateOf(run.id), ...ended }; - this.staging(tx).changes.push({ kind: "outcome", run, outcome }); + const outcome = this.#outcome(run, ended); + this.change(tx, { kind: "outcome", run, outcome }); + // What its state holds is stored before the outcome is recorded (D8). + const adopt = + this.#persisted === undefined ? undefined : this.#states?.stageHeld(run.id, this.#store(tx)); + if (adopt !== undefined) this.staging(tx).adopts.push(adopt); }, documents.own, context, @@ -277,10 +566,18 @@ export class JobRecords { } catch { // The object ended: nothing finishes. return; + } finally { + // A run ending without an invocation (marked aborted, abandoned, cut off in memory) has none to wait for. + run.invocation?.ended(); } this.#need(run); } + /** `run`'s outcome for `ended`, with its args and its state as it is now. */ + #outcome(run: JobRun, ended: Settled): AnyOutcome { + return { id: run.id, args: run.args, state: this.stateOf(run.id), ...ended }; + } + /** `run` needs a live attempt (to be invoked or finished): now, or once one attaches. */ #need(run: JobRun): void { if (this.#ended !== undefined || this.#runs.get(run.id) !== run) return; @@ -289,30 +586,53 @@ export class JobRecords { else this.#dispatch(run, job); } - /** Invokes or finishes `run` on `job`, as its record stands. */ + /** + * Invokes or finishes `run` on `job`, as its record stands: a finishing run is finished; a run not invoked yet is + * invoked, unless it is marked aborted (it ends `aborted` without `run`) or resumes abandoned (`failed { aborted, + * reason: "restart" }`). + */ #dispatch(run: JobRun, job: Job): void { if (this.#runs.get(run.id) !== run) return; - if (run.phase === "finishing") job.finish(run); - else if (run.phase === "running" && run.invocation === "pending") job.invoke(run); + if (run.phase === "finishing") { + job.finish(run); + return; + } + if (run.phase === "blocked" || run.invoked) return; + if (run.abortMarked) { + void this.#recordOutcome(run, ABORTED); + return; + } + if (run.resuming && this.definition.persist?.resume === "abandon") { + void this.#recordOutcome(run, RESTART); + return; + } + run.resuming = false; + job.invoke(run); } /** - * Adopts one record change, inside the commit step: the maps, the active count, the state document, `running`. A - * new run is counted for its object here, before the commit's publication (a link closing while it sends may - * release the object's last reference) and before any run of the same commit settles in `#committed`. + * Adopts one record change, inside the commit step: the maps, the active count, `running`. A new run is counted for + * its object here, before the commit's publication (a link closing while it sends may release the object's last + * reference) and before any run of the same commit settles in `#committed`. */ - #adopt(change: RecordChange): void { + #adopt(change: Adopted): void { const { run } = change; switch (change.kind) { case "create": + case "restore": this.#runs.set(run.id, run); if (run.key !== undefined) this.#keys.set(run.key, run); - this.#active++; - this.#owner.countRuns(1); - if (change.state !== undefined) this.#states?.create(run.id, change.state); - run.entry = run.entryFor(change.state); + if (run.phase !== "blocked") { + this.#active++; + this.#owner.countRuns(1); + } + run.entry = run.entryFor(this.stateOf(run.id)); this.running.add(run.entry); return; + case "update": + Object.assign(run, change.fields); + this.#replaceEntry(run); + return; case "outcome": run.outcome = change.outcome; run.phase = "finishing"; @@ -320,6 +640,7 @@ export class JobRecords { return; case "block": run.phase = "blocked"; + run.blocked = change.reason; this.#active--; this.#replaceEntry(run); return; @@ -329,7 +650,6 @@ export class JobRecords { // The finishing run's key may name its successor already, started in the same commit. if (run.key !== undefined && this.#keys.get(run.key) === run) this.#keys.delete(run.key); if (change.kind === "delete") this.#active--; - this.#states?.retire(run.id); this.running.remove(run.entry!); run.entry = undefined; return; @@ -337,13 +657,15 @@ export class JobRecords { } /** The actions after a record change committed. */ - #committed(change: RecordChange): void { + #committed(change: Adopted): void { const { run } = change; switch (change.kind) { case "create": // After the commit, outside the batch, once the transaction settled; on the live attempt, or the next. queueMicrotask(() => this.#need(run)); return; + case "restore": + case "update": case "outcome": return; case "delete": @@ -362,8 +684,10 @@ export class JobRecords { /** A run's state document changed by `ops`, inside the commit step: its entry is replaced. */ #stateChanged(id: string, ops: readonly Op[]): void { - const run = this.#runs.get(id)!; - const before = run.entry!; + const run = this.#runs.get(id); + // Its record may not be adopted yet (a start creates both in one commit; its entry takes the state then). + if (run?.entry === undefined) return; + const before = run.entry; run.entry = run.entryFor(this.stateOf(id)); this.running.stateChanged(before, run.entry, ops); } @@ -384,63 +708,21 @@ export class JobRecords { } } -/** - * A job's state documents, one per run by its id: Chord's own keyed slot, apart from the object's `self.state` slots. - * Created with the run and retired with its record; never drafted while absent, so a retired one is never recreated. - * Each commit tells the job, whose `running` carries the state. - */ -class RunStates implements DocumentSlot { - readonly owner: Documents; - /** In memory only (persisted jobs are slice 6b's). */ - readonly persisted = undefined; - readonly #definition: StateDefinition; - readonly #trackers = new Map>(); - readonly #changed: (id: string, ops: readonly Op[]) => void; +/** Runs every adoption of `adopts`. */ +function runAll(adopts: readonly (() => void)[]): void { + for (const adopt of adopts) adopt(); +} - constructor( - definition: StateDefinition, - owner: Documents, - changed: (id: string, ops: readonly Op[]) => void, - ) { - this.#definition = definition; - this.owner = owner; - this.#changed = changed; - } - - /** The value of run `id`'s document; `undefined` once retired. */ - value(id: string): JsonObject | undefined { - return this.#trackers.get(id)?.value; - } - - current(key: string): Tracker | undefined { - return this.#trackers.get(key); - } - - initial(_context: Context): JsonObject { - throw new ChordError({ - code: "released", - reason: "settled", - message: `${this.#definition.name}: the run's document is retired`, - }); - } - - adopt(key: string, tracker: Tracker, prepared: Prepared): void { - tracker.adopt(prepared); - this.#changed(key, prepared.ops); - } - - /** Retires run `key`'s document (its record's deletion). */ - retire(key: string): void { - this.#trackers.delete(key); - } - - /** Creates run `id`'s document with `value` (its start's commit). */ - create(id: string, value: JsonObject): void { - this.#trackers.set(id, track(value)); - } - - /** Drops every document (the object ended). */ - drop(): void { - this.#trackers.clear(); +/** The record fields a change writes, its phase transitions included. */ +function recordFields(change: Exclude): StoredFields { + switch (change.kind) { + case "create": + return {}; + case "update": + return change.fields; + case "outcome": + return { phase: "finishing", outcome: change.outcome }; + case "block": + return { phase: "blocked", blocked: change.reason }; } } diff --git a/packages/chord/src/next/jobs/run-states.ts b/packages/chord/src/next/jobs/run-states.ts new file mode 100644 index 000000000..f8305c44c --- /dev/null +++ b/packages/chord/src/next/jobs/run-states.ts @@ -0,0 +1,243 @@ +// A job's run state documents (§15, slice 6b §4.1, §4.3): one per run by its id, Chord's own slot apart from the +// object's `self.state` slots. In memory, or persisted in the object's store (namespace `chord.job-states`), always in +// memory while the run is unsettled (`ctx.state.value` reads it synchronously). A persisted run's deferred changes +// (D8) are adopted and published at once and held, stored before the next change's own, at the outcome, at close or +// when their deadline passes. + +import type { Context } from "../../context/index.ts"; +import { type Op, PendingOps, type Prepared, type Tracker, track } from "../../delta/index.ts"; +import { ChordError } from "../data/index.ts"; +import type { CommitPersistence, JsonObject, StateDefinition } from "../documents/index.ts"; +import { + type DocumentChange, + type DocumentSlot, + type Documents, + type Held, + type PersistedSlot, + type Persistence, + SlotDefinition, + type Stored, + type StoredChange, + storedAfter, +} from "../documents/internal.ts"; +import { changeDocument, createDocument } from "../store/engine.ts"; +import type { StagedWrite } from "../store/step.ts"; + +/** + * A held state's ops, folded as they come (`PendingOps`: repeated writes of one leaf keep one op; past its bound only + * "store a base" stays), whether a held change asked for a base, and when it must be stored at the latest. + */ +type HeldState = { readonly pending: PendingOps; base: boolean; deadline: number }; + +/** What the run states tell their job: a document changed (`running` follows), a held change's deadline. */ +export type RunStatesOwner = { + readonly documents: Documents; + /** Run `id`'s document changed by `ops`, inside the commit step. */ + changed(id: string, ops: readonly Op[]): void; + /** A held change must be stored by `deadline` (`Date.now()` time). */ + held(deadline: number): void; +}; + +/** How long a deferred change is held at most by default (Durable's `progress.persistWithinMs`). */ +const DEFAULT_WITHIN_MS = 1_000; + +/** A run's state documents. Created with the run, retired with its record; never drafted while absent. */ +export class RunStates implements DocumentSlot, PersistedSlot { + readonly owner: Documents; + /** The store side when the job persists; `undefined` in memory. */ + readonly persisted: PersistedSlot | undefined; + readonly definition: SlotDefinition; + readonly #persistence: Persistence | undefined; + readonly #states: RunStatesOwner; + readonly #trackers = new Map>(); + readonly #stored = new Map(); + readonly #held = new Map(); + + constructor( + definition: StateDefinition, + states: RunStatesOwner, + persistence: Persistence | undefined, + ) { + this.definition = new SlotDefinition(definition); + this.owner = states.documents; + this.#states = states; + this.#persistence = persistence; + this.persisted = persistence === undefined ? undefined : this; + } + + get persistence(): Persistence { + // Only reached through `persisted`, which a persisted job's states alone have. + return this.#persistence!; + } + + /** The value of run `id`'s document; `undefined` once retired. */ + value(id: string): JsonObject | undefined { + return this.#trackers.get(id)?.value; + } + + current(key: string): Tracker | undefined { + return this.#trackers.get(key); + } + + initial(_context: Context): JsonObject { + throw new ChordError({ + code: "released", + reason: "settled", + message: `${this.definition.current.name}: the run's document is retired`, + }); + } + + /** Pinned in memory while the run is unsettled: never cold. */ + cold(_key: string): boolean { + return false; + } + + stored(key: string): Stored | undefined { + return this.#stored.get(key); + } + + async load(_key: string, _context: Context): Promise {} + + /** + * A change of run `key`'s document: a deferred one of a stored document is held (nothing written); an immediate + * one stores what is held with its own ops, as one delta on the stored revision or a base. + */ + stage( + key: string, + change: DocumentChange, + was: Stored | undefined, + writes: StagedWrite[], + persistence: CommitPersistence, + ): StoredChange { + if (persistence.kind === "deferred" && was !== undefined && !change.retires && change.prepared !== undefined) { + return { kind: "held", stored: was, withinMs: persistence.withinMs ?? DEFAULT_WITHIN_MS }; + } + const held = this.#held.get(key); + return this.persistence.stage(key, this.definition.current, was, change, writes, held && heldOps(held)); + } + + adopt( + key: string, + tracker: Tracker, + prepared: Prepared, + change: StoredChange | undefined, + ): void { + tracker.adopt(prepared); + if (change?.kind === "held") this.#hold(key, prepared, change); + else if (change !== undefined) this.#settle(key, storedAfter(change)); + this.#states.changed(key, prepared.ops); + } + + /** Retires run `key`'s document (its record's deletion): what is held goes with it. */ + retire(key: string): void { + this.#trackers.delete(key); + this.#stored.delete(key); + this.#held.delete(key); + } + + /** Run `id`'s document with `value`, as its start created it or a load found it (`stored`: persisted). */ + create(id: string, value: JsonObject, stored: Stored | undefined): void { + this.#trackers.set(id, track(value)); + if (stored !== undefined) this.#stored.set(id, stored); + } + + /** + * Stages the creation of run `id`'s document with `value` into `writes` (its start); returns the adoption, which + * creates it in memory with what is stored. + */ + stageCreate(id: string, value: JsonObject, writes: () => StagedWrite[]): () => void { + const persistence = this.#persistence; + if (persistence === undefined) return () => this.create(id, value, undefined); + const { version, fingerprint } = this.definition.current; + const created = { id: 0 }; + const base = { version, fingerprint, value }; + writes().push( + createDocument(persistence.at(id), base, JSON.stringify(value), (stored) => { + created.id = stored; + }), + ); + return () => this.create(id, value, { id: created.id, revision: 1, version, deltasSinceBase: 0 }); + } + + /** When run `id`'s held changes must be stored at the latest (`Infinity`: none held, or none with a deadline). */ + deadline(id: string): number { + return this.#held.get(id)?.deadline ?? Number.POSITIVE_INFINITY; + } + + /** The runs whose documents hold changes not stored yet. */ + get heldIds(): readonly string[] { + return [...this.#held.keys()]; + } + + /** + * Stages what run `id`'s document holds into `writes` (a flush: its deadline, the outcome, the close writer), as one + * delta on its stored revision or a base; returns the adoption, `undefined` when nothing is held. + */ + stageHeld(id: string, writes: StagedWrite[]): (() => void) | undefined { + const held = this.#held.get(id); + const stored = this.#stored.get(id); + const tracker = this.#trackers.get(id); + if (held === undefined || stored === undefined || tracker === undefined) return undefined; + const { version, fingerprint } = this.definition.current; + const value = tracker.value; + const { ops, base: asked } = heldOps(held); + const base = asked || stored.version !== version; + const content = base + ? ({ kind: "base", version, fingerprint, value } as const) + : ({ kind: "delta", version, ops } as const); + writes.push(changeDocument(stored.id, stored.revision, content, JSON.stringify(base ? value : ops))); + const deltasSinceBase = base ? 0 : stored.deltasSinceBase + 1; + const after: Stored = { id: stored.id, revision: stored.revision + 1, version, deltasSinceBase }; + return () => { + if (this.#held.get(id) === held) this.#settle(id, after); + }; + } + + /** Drops every document (the object ended): what is held is lost, as a crash loses it. */ + drop(): void { + this.#trackers.clear(); + this.#stored.clear(); + this.#held.clear(); + } + + /** What is stored of run `key`'s document after a commit stored it: nothing is held any more. */ + #settle(key: string, stored: Stored | undefined): void { + if (stored === undefined) this.#stored.delete(key); + else this.#stored.set(key, stored); + this.#held.delete(key); + } + + /** A deferred change adopted: its ops held (a base when `checkpointWhen` asks), its deadline told. */ + #hold(key: string, prepared: Prepared, change: Extract): void { + const held = this.#held.get(key) ?? { + pending: new PendingOps(), + base: false, + deadline: Number.POSITIVE_INFINITY, + }; + const deltasSinceBase = change.stored.deltasSinceBase + (held.pending.size > 0 ? 1 : 0); + if (this.definition.current.checkpointWhen(prepared.value, prepared.ops, { deltasSinceBase })) held.base = true; + // A base stores the whole value: no op needs keeping for it. + if (held.base) held.pending.take(); + else held.pending.push(prepared.ops); + held.deadline = Math.min(held.deadline, Date.now() + change.withinMs); + this.#held.set(key, held); + if (Number.isFinite(held.deadline)) this.#states.held(held.deadline); + } +} + +/** + * What `held` stores, its queue left as it was (a commit may still be refused): its folded ops, or a base once a held + * change asked for one or the queue overflowed. Taking and pushing back one batch keeps it by reference: no copy. + */ +function heldOps(held: HeldState): Held { + if (held.base) return NO_HELD_OPS_BASE; + const taken = held.pending.take(); + if (taken.kind === "replace") { + held.base = true; + return NO_HELD_OPS_BASE; + } + held.pending.push(taken.ops); + return { ops: taken.ops, base: false }; +} + +const NO_HELD_OPS_BASE: Held = { ops: [], base: true }; diff --git a/packages/chord/src/next/jobs/run.ts b/packages/chord/src/next/jobs/run.ts index 688e0efa1..cff213177 100644 --- a/packages/chord/src/next/jobs/run.ts +++ b/packages/chord/src/next/jobs/run.ts @@ -1,5 +1,7 @@ -// One run of a job (§15): its record (id, version, key, args, phase, outcome), its abort signal, the fence of its -// invocation (its seal token), its `running` entry and the `Run` handle its callers wait on. +// One run of a job (§15, slice 6b §4.1): its record (id, version, key, args, phase, restarts, finish failures, abort +// mark, outcome), its current invocation (the fence of its seal token, its abort signal, the end an abort joins: one +// per invocation, as a persisted run is invoked again after a seal, R3), its `running` entry and the `Run` handle its +// callers wait on. import type { Context } from "../../context/index.ts"; import type { JsonValue } from "../../json.ts"; @@ -23,13 +25,15 @@ export interface Run { export type AnyOutcome = Outcome; /** - * Where a run's record stands. `running` from its start's commit (its `run` starts after it), `finishing` once its - * outcome is recorded, `blocked` after five failed finishes. + * Where a run's record stands. A persisted run is `pending` from its start's commit until its reservation commit + * (`running`, then its `run` starts), `stopped` once a clean stop cut it off; an in-memory run is `running` from its + * start's commit. `finishing` once its outcome is recorded, `blocked` after five failed finishes or for another + * version. */ -type Phase = "running" | "finishing" | "blocked"; +export type Phase = "pending" | "running" | "stopped" | "finishing" | "blocked"; -/** Where a run's invocation stands: `ctx.state.change` is admitted only while it runs (the invocation fence). */ -type Invocation = "pending" | "running" | "settled"; +/** Why a run is blocked: its `finish` failed five times in a row, or it was stored by another job version (slice 7). */ +export type BlockedReason = "finish_failed" | "version"; /** How a run settled for its waiters. */ type Answer = @@ -45,13 +49,21 @@ export class JobRun { readonly version: number; readonly key: string | undefined; readonly args: JsonValue; - /** `run`'s abort signal: `abort(id)`, the object's end and `host.close` fire it; an early abort pre-marks it. */ - readonly abort = new AbortController(); readonly handle: Run; - /** The seal token of its invocation: sealed when the invocation is cut off by its attempt's seal (`ctx.state.change`). */ - readonly token = new SealToken(); - phase: Phase = "running"; - invocation: Invocation = "pending"; + /** Its current invocation, from its reservation (or its start in memory) on; a later one replaces it. */ + invocation: Invocation | undefined; + phase: Phase; + /** A persisted run's crashes in a row while running (D6); reset by a state change. */ + restarts = 0; + /** Failed finishes in a row (stored for a persisted run, so they count across a crash). */ + finishFailures = 0; + /** Set by `abort(id)` (stored for a persisted run, D7): a run not invoked yet ends `aborted` without `run`. */ + abortMarked = false; + /** A persisted run cut off by a crash or a clean stop, to be resumed per its job's `resume` (no new start). */ + resuming = false; + blocked: BlockedReason | undefined; + /** The start's trace carrier, when the adapter propagates (stored, slice 9 uses it). */ + readonly trace: { readonly [key: string]: string } | undefined; /** What it ended with, once recorded (finishing and blocked). */ outcome: AnyOutcome | undefined; /** Its entry in `running`, while its record exists; replaced when its record or state document changes. */ @@ -59,29 +71,62 @@ export class JobRun { /** Its waiters: apart from the run, so a handle kept by a caller keeps only how the run settled. */ readonly #waiters = new Waiters(); - constructor(id: string, version: number, key: string | undefined, args: JsonValue) { + constructor( + id: string, + version: number, + key: string | undefined, + args: JsonValue, + phase: Phase, + trace?: { readonly [key: string]: string }, + ) { this.id = id; this.version = version; this.key = key; this.args = args; + this.phase = phase; + this.trace = trace; const waiters = this.#waiters; this.handle = { id, outcome: (context) => waiters.wait(context) }; } - /** Its `running` entry for its current record and `state` (the job's state document, `undefined` without one). */ + /** + * Its `running` entry for its current record and `state` (the job's state document, `undefined` without one): a + * pending or stopped run shows as `running` (it runs, or will run again). + */ entryFor(state: JsonValue | undefined): JsonValue { const { id, version, key, args, phase } = this; - if (phase === "blocked") return { id, version, phase, reason: "finish_failed" }; + if (phase === "blocked") return { id, version, phase, reason: this.blocked! }; return { id, version, ...(key === undefined ? {} : { key }), args, ...(state === undefined ? {} : { state }), - phase, + phase: phase === "finishing" ? "finishing" : "running", }; } + /** A new invocation, the current one from now on (a run marked aborted is never invoked: it ends without one). */ + begin(): Invocation { + const invocation = new Invocation(); + this.invocation = invocation; + return invocation; + } + + /** Whether its current invocation runs (or is reserved): it is not invoked again meanwhile. */ + get invoked(): boolean { + return this.invocation?.state === "running"; + } + + /** + * Resolves once its current invocation ended, at once when none runs (`abort` joins it; one a seal cut off is + * fenced and is not waited for); `context` ends only the wait. + */ + over(context: Context): Promise { + const invocation = this.invocation; + return invocation?.state === "running" ? invocation.over(context) : Promise.resolve(); + } + /** Its waiters get the outcome. */ resolve(outcome: AnyOutcome): void { this.#waiters.settle({ kind: "done", outcome }); @@ -93,6 +138,35 @@ export class JobRun { } } +/** + * One invocation of a run's `run`: its fence (`ctx.state.change` admitted while it runs, under its own seal token, + * which its attempt's seal fences), its abort signal and the waits for its end. Captured by its ctx and its settling, + * so a replaced invocation never acts for its successor. + */ +export class Invocation { + readonly token = new SealToken(); + /** `run`'s abort signal: `abort(id)`, the attempt's seal, the object's end and `host.close` fire it. */ + readonly abort = new AbortController(); + /** `running` from its reservation until `run` settled or a seal cut it off. */ + state: "running" | "settled" = "running"; + readonly #joins = new Joins(); + + /** The fence closes: a change made later is refused. */ + settle(): void { + this.state = "settled"; + } + + /** Nothing of it runs here any more (its `run` settled, or the object ended): every join resolves. */ + ended(): void { + this.#joins.end(); + } + + /** Resolves once it ended; `context` ends only the wait. */ + over(context: Context): Promise { + return this.#joins.wait(context); + } +} + /** A run's waits for its outcome: each removed when its context aborts; late ones answered at once. */ class Waiters { /** How the run settled, once it did. */ @@ -135,3 +209,36 @@ class Waiters { function answer(settled: Answer): Promise { return settled.kind === "done" ? Promise.resolve(settled.outcome) : Promise.reject(settled.error); } + +/** The waits for a run's invocation to be over: each removed when its context aborts, so a cancelled one keeps nothing. */ +class Joins { + #over = false; + readonly #waiting = new Set<() => void>(); + + /** The invocation is over (once): every wait resolves. */ + end(): void { + if (this.#over) return; + this.#over = true; + for (const resolve of this.#waiting) resolve(); + this.#waiting.clear(); + } + + /** A wait: at once when over; otherwise until it is or `context` aborts (rejecting with its reason). */ + wait(context: Context): Promise { + if (this.#over) return Promise.resolve(); + const signal = context.abortSignal; + if (signal?.aborted) return Promise.reject(signal.reason); + const { promise, resolve, reject } = Promise.withResolvers(); + const abort = (): void => { + this.#waiting.delete(done); + reject(signal?.reason); + }; + const done = (): void => { + signal?.removeEventListener("abort", abort); + resolve(); + }; + this.#waiting.add(done); + signal?.addEventListener("abort", abort, { once: true }); + return promise; + } +} diff --git a/packages/chord/src/next/jobs/running.ts b/packages/chord/src/next/jobs/running.ts index 3f24ccd24..098cb2804 100644 --- a/packages/chord/src/next/jobs/running.ts +++ b/packages/chord/src/next/jobs/running.ts @@ -7,6 +7,7 @@ import type { NonEmptyPath, Op, Path } from "../../delta/index.ts"; import type { JsonValue } from "../../json.ts"; import { ChordError } from "../data/index.ts"; import { type Batch, Source } from "../state/internal.ts"; +import type { BlockedReason } from "./run.ts"; /** * One entry of `running`: a running or finishing run with its args (and state, when the job has a state document), or @@ -20,7 +21,7 @@ export type RunningEntry = readonly args: A; readonly phase: "running" | "finishing"; } & ([S] extends [undefined] ? unknown : { readonly state: S })) - | { readonly id: string; readonly version: number; readonly phase: "blocked"; readonly reason: "finish_failed" }; + | { readonly id: string; readonly version: number; readonly phase: "blocked"; readonly reason: BlockedReason }; const EMPTY: readonly JsonValue[] = []; diff --git a/packages/chord/src/next/jobs/stored.ts b/packages/chord/src/next/jobs/stored.ts new file mode 100644 index 000000000..e2f14405f --- /dev/null +++ b/packages/chord/src/next/jobs/stored.ts @@ -0,0 +1,106 @@ +// A persisted run's record (slice 6b §4.1, D5): the document each unsettled run of a persisted job keeps in its +// object's store (namespace `chord.jobs`, kind the job name, key the run id), and the conversion of an outcome to and +// from it. Errors are stored in the wire error format, so a `PluginError`'s code, message and details survive. + +import type { JsonValue } from "../../json.ts"; +import type { JsonObject } from "../documents/index.ts"; +import { errorFromWire, wireError } from "../link/internal.ts"; +import type { WireError } from "../link/message.ts"; +import type { JobError } from "./definition.ts"; +import type { AnyOutcome, BlockedReason, JobRun, Phase } from "./run.ts"; + +/** How a run failed, as stored: a thrown error in the wire format, an abort, the restart limit. */ +export type StoredJobError = + | { readonly kind: "threw"; readonly error: WireError } + | { readonly kind: "aborted"; readonly reason: "abort" | "reload" | "restart" } + | { readonly kind: "restart_limit"; readonly restarts: number }; + +/** A run's outcome as stored: its result, or how it failed. */ +export type StoredOutcome = + | { readonly kind: "done"; readonly result: JsonValue } + | { readonly kind: "failed"; readonly error: StoredJobError }; + +/** The record document of one unsettled persisted run. */ +export type StoredRun = { + readonly version: number; + readonly fingerprint: string; + readonly args: JsonValue; + readonly key?: string; + readonly trace?: { readonly [key: string]: string }; + readonly phase: Phase; + readonly restarts: number; + readonly finishFailures: number; + readonly abort?: true; + readonly blocked?: BlockedReason; + readonly outcome?: StoredOutcome; +}; + +/** + * The record documents' version and fingerprint (Chord's own documents: no definition, no `migrate`; a new record + * shape raises the version). + */ +export const RUN_RECORD = { version: 1, fingerprint: "chord.job-run.1" } as const; + +/** What a change sets on a run's record; the rest is the run's own. */ +export type StoredFields = { + readonly phase?: Phase; + readonly restarts?: number; + readonly finishFailures?: number; + readonly abortMarked?: true; + readonly blocked?: BlockedReason; + readonly outcome?: AnyOutcome; +}; + +/** `run`'s record as stored with `fields` changed, with the job's `fingerprint`. */ +export function storedRun(run: JobRun, fingerprint: string, fields: StoredFields = {}): StoredRun { + const { version, args, key, trace } = run; + const { phase, restarts, finishFailures, abortMarked, blocked, outcome } = { ...pick(run), ...fields }; + return { + version, + fingerprint, + args, + ...(key === undefined ? {} : { key }), + ...(trace === undefined ? {} : { trace }), + phase, + restarts, + finishFailures, + ...(abortMarked ? { abort: true as const } : {}), + ...(blocked === undefined ? {} : { blocked }), + ...(outcome === undefined ? {} : { outcome: storedOutcome(outcome) }), + }; +} + +/** The fields of `run` a record stores that change. */ +function pick(run: JobRun): Required> & StoredFields { + const { phase, restarts, finishFailures, blocked, outcome } = run; + return { phase, restarts, finishFailures, ...(run.abortMarked ? { abortMarked: true } : {}), blocked, outcome }; +} + +/** An outcome as stored: a result as it is (JSON: persisted jobs refuse `bytes()`), an error in the wire format. */ +function storedOutcome(outcome: AnyOutcome): StoredOutcome { + if (outcome.kind === "done") return { kind: "done", result: outcome.result as JsonValue }; + return { kind: "failed", error: storedError(outcome.error) }; +} + +function storedError(error: JobError): StoredJobError { + return error.kind === "threw" ? { kind: "threw", error: wireError(error.error) } : error; +} + +/** The outcome a stored one stands for, rebuilt as the in-memory one (`id`, `args` and `state` from the run). */ +export function outcomeOf( + stored: StoredOutcome, + id: string, + args: JsonValue, + state: JsonValue | undefined, +): AnyOutcome { + if (stored.kind === "done") return { id, args, state, kind: "done", result: stored.result }; + const { error } = stored; + const rebuilt: JobError = error.kind === "threw" ? { kind: "threw", error: errorFromWire(error.error) } : error; + return { id, args, state, kind: "failed", error: rebuilt }; +} + +/** A stored record as Chord wrote it (the record definition's value, a JSON object). */ +export function recordOf(value: JsonObject): StoredRun { + // Chord's own document: `storedRun` wrote it. + return value as unknown as StoredRun; +} diff --git a/packages/chord/src/next/remote/hold.ts b/packages/chord/src/next/remote/hold.ts index a87c4aaa8..232d2122a 100644 --- a/packages/chord/src/next/remote/hold.ts +++ b/packages/chord/src/next/remote/hold.ts @@ -372,6 +372,7 @@ export class Hold implements Holder { }, // The member's source ended with its object, which ends this hold. end: () => {}, + endAfterDelivery: () => {}, }, encoder: encoder(), pending: new PendingOps(), diff --git a/packages/chord/src/next/state/batch.ts b/packages/chord/src/next/state/batch.ts index d58c5c347..66dbd5d5a 100644 --- a/packages/chord/src/next/state/batch.ts +++ b/packages/chord/src/next/state/batch.ts @@ -27,6 +27,8 @@ export class Batch { #sending = false; /** Changes `within` deferred while the flushes sent, for the next commit. */ #later: (() => void)[] = []; + /** Changes `soon` collected for the commit on the next microtask, while one is queued. */ + #soon: (() => void)[] | undefined; constructor(report: (error: BatchError) => void, metrics: TelemetryMetrics | undefined) { this.#report = report; @@ -34,6 +36,11 @@ export class Batch { this.#metrics = metrics; } + /** Whether a commit's adoption, a delivery or a derive compute runs now: a change made now lands in this batch. */ + get running(): boolean { + return this.#running; + } + /** Throws `invalid/reentrant` inside the batch: `operation` may not run during adoptions or derive computes. */ assertOutside(operation: string): void { if (this.#running) throw invalid("reentrant", `${operation} inside a commit or a derive compute`); @@ -101,6 +108,26 @@ export class Batch { else this.commit(adopt, context); } + /** + * Runs `adopt` in one commit on the next microtask, together with every other `soon` until then (changes arriving + * outside any commit, as a followed runtime's calls off its commit step: one batch for all of them). `context` is + * the first one's. + */ + soon(adopt: () => void, context: Context): void { + if (this.#soon !== undefined) { + this.#soon.push(adopt); + return; + } + const soon = [adopt]; + this.#soon = soon; + queueMicrotask(() => { + this.#soon = undefined; + this.commit(() => { + for (const each of soon) each(); + }, context); + }); + } + /** Lists a watched source the current commit changed. */ changed(source: Source): void { this.#changed.push(source); diff --git a/packages/chord/src/next/state/watchable.ts b/packages/chord/src/next/state/watchable.ts index 8bdbac622..a7fd841ae 100644 --- a/packages/chord/src/next/state/watchable.ts +++ b/packages/chord/src/next/state/watchable.ts @@ -20,6 +20,11 @@ export interface Subscriber { advance(call: WatchCall, context: Context): void; /** The source ended for good with `end`; nothing follows, and the subscriber is dropped. */ end(end: WatchEnd): void; + /** + * The source ended for good after its last call (a followed runtime ended): a subscriber delivering later (a watch) + * ends `ended` once it delivered what it was given, others end now. + */ + endAfterDelivery(): void; } /** @@ -147,6 +152,18 @@ export abstract class Source implements Watchable { this.forward({ kind: "failed", error: handedOut(error) }, context); } + /** + * Ends every watch and subscriber for good once what they were given is delivered (a followed runtime ended after a + * last call, Durable's `endAfterDelivery`): `ended`; nothing follows. + */ + protected endWatchesAfterDelivery(): void { + for (const watch of [...this.#watches]) watch.endAfterDelivery(); + if (this.#subscribers.size === 0) return; + const subscribers = [...this.#subscribers]; + this.#subscribers.clear(); + for (const subscriber of subscribers) subscriber.endAfterDelivery(); + } + /** Ends every watch and subscriber for good (its owner ended): `ended`; nothing follows. */ protected endWatches(): void { // Ending detaches, which deletes the current entry (safe while iterating) and runs no listener. diff --git a/packages/chord/src/next/store/README.md b/packages/chord/src/next/store/README.md index 933fb6df6..b1fcb2975 100644 --- a/packages/chord/src/next/store/README.md +++ b/packages/chord/src/next/store/README.md @@ -52,6 +52,18 @@ const copy = await importStore(sqliteBackend(db), decodeJsonl(bytes), ctx); checks every part (a trust boundary) and applies the archive in one backend transaction; JSONL via `encodeJsonl` and `decodeJsonl`. +## For adapters (`../index.ts`, marked internal) + +Durable's `StoreStorage` keeps its records in the store: `StoreOwnerTables` declares and prepares an owner's tables +(claimed, `":

"`), `OwnerTable` encodes rows at staging, writes them in a step (`put`, with a step-stamped +column) and reads them through queries compiled once (`prepare`, `prepareFirst`, `prepareGet`, `reader`); the documents +engine's writes (`insertDocument`, `writeContent`, `retireRecord`) and reads at a point (`findDocument`, `loadContent`, +`listDocumentEntries`, …); `store.commitInBatch` adopts in the attached host's batch; `store.mintId()` and `store.ids` +are the one allocator (D3); `store.claims` holds what can collide; `store.readTransaction` reads several queries at one +state. A step's own error is a refusal (the commit rejects with it, the store goes on); `refusalOf` classifies it for +Chord's owners. `applyBatch(store, commits, ctx)` replays outbox commits (the sync layer's entry point): refused at its +call and at its turn while an owner holds a claim or once the store diverged, and no claim is taken while it applies. + ## For Chord's layers `ChordStore` (`store.ts`: `commit(staged, fence, ctx)`, `read(fn, ctx)` guarded as Durable's `GuardedStorage`, diff --git a/packages/chord/src/next/store/adapter.ts b/packages/chord/src/next/store/adapter.ts new file mode 100644 index 000000000..6e5eec4b6 --- /dev/null +++ b/packages/chord/src/next/store/adapter.ts @@ -0,0 +1,152 @@ +// The store for adapters (slice 6b §2.1, D1, D2): what an owner outside Chord's objects (Durable's `StoreStorage`) uses +// to keep its data in the store as declared tables and engine documents: its tables declared and prepared under its +// owner name (claimed, the path `self.table` takes), rows encoded at staging and written in a step with what only the +// step knows, and its reads. Internal for adapters; apps use `openStore` and objects. + +import type { Context } from "../../context/index.ts"; +import type { SqliteExecutor, SqliteValue } from "../../sqlite/index.ts"; +import { checkName } from "../contracts/internal.ts"; +import type { Claimant } from "./claims.ts"; +import type { WhereNode } from "./expression.ts"; +import type { EncodedRow } from "./rows.ts"; +import { type Order, TableSql, tableSqlName } from "./sql.ts"; +import type { Step } from "./step.ts"; +import type { ChordStore } from "./store.ts"; +import type { TableDefinition } from "./table.ts"; +import { prepareTables } from "./table-migration.ts"; +import type { TableWrite } from "./table-writes.ts"; + +/** A stored row as reads decode it. */ +export type DecodedRow = { readonly [column: string]: unknown }; + +/** + * The tables of one owner name in a store: each declared once (its SQL name `":
"` claimed for + * `claimant`), then prepared together (created at their newest version or migrated, as `self.table`'s are). + */ +export class StoreOwnerTables { + readonly store: ChordStore; + readonly owner: string; + readonly #claimant: Claimant; + readonly #tables: OwnerTable[] = []; + + /** `owner` is a restricted name (lowercase `[a-z0-9@/._~-]`): it becomes the first part of SQL names. */ + constructor(store: ChordStore, owner: string, claimant: Claimant) { + checkName(owner, "An owner name"); + this.store = store; + this.owner = owner; + this.#claimant = claimant; + } + + /** Declares `definition` under the owner, claiming its SQL name (held by another: `invalid/args`). */ + declare(definition: TableDefinition): OwnerTable { + const table = new OwnerTable(definition, tableSqlName(this.owner, definition.name)); + this.store.claims.table(table.sql.table, this.#claimant); + this.#tables.push(table); + return table; + } + + /** Brings every declared table to its definition: created fresh, or migrated eagerly in their commits. */ + prepare(context: Context): Promise { + const tables = this.#tables.map(({ definition, sql }) => ({ definition, table: sql.table })); + const fence = { admit: () => true, adopt: () => {} }; + return prepareTables(tables, this.store, fence, this.owner, context); + } +} + +/** One declared table of an owner: its rows encoded and written in a step, its reads by key and by query. */ +export class OwnerTable { + readonly definition: TableDefinition; + readonly sql: TableSql; + + constructor(definition: TableDefinition, table: string) { + this.definition = definition; + this.sql = new TableSql(table, definition.columns, definition.rows.physical, definition.key); + } + + /** `row` checked and encoded for a `put` (a trust boundary, rule 8: at staging, before the queue). */ + encode(row: unknown): EncodedRow { + return this.definition.rows.put(row, this.definition.version); + } + + /** + * Writes `encoded` in `step` (an upsert) and returns the `put` as recorded; `stamp` names an `int` column that takes + * the step's seq (Durable's `entries.commit_seq`), in the row and in the record. + */ + async put(step: Step, encoded: EncodedRow, stamp?: string): Promise { + const { version } = this.definition; + const { table } = this.sql; + if (stamp === undefined) { + await step.run(this.sql.put, ...encoded.params); + return { kind: "put", table, row: encoded.row, version, columns: encoded.columns }; + } + const params = [...encoded.params]; + params[this.definition.rows.physical.indexOf(stamp)] = step.seq; + await step.run(this.sql.put, ...params); + return { kind: "put", table, row: { ...encoded.row, [stamp]: step.seq }, version, columns: encoded.columns }; + } + + /** The row with the key `params` (in key order), decoded. */ + async get( + executor: Pick, + params: readonly SqliteValue[], + context: Context, + ): Promise { + const stored = await executor.get<{ [column: string]: SqliteValue }>(this.sql.get, ...params); + return stored === undefined ? undefined : this.definition.rows.decode(stored, context); + } + + /** + * A read compiled once (the store's query vocabulary rendered to SQL): `columns` of the rows `where` matches in + * `order`. The values in `where` only shape it: each is a `?` the caller binds in the order the conditions take + * them, then the limit. Adapters keep their fixed reads so: no SQL is rendered per call. + */ + prepare(columns: readonly string[], where: WhereNode | undefined, order: Order): string { + return this.sql.select(where, order, undefined, 0, columns).sql; + } + + /** `prepare` for the first row only: the limit is a literal `1` (a bound one costs SQLite a plan step per call). */ + prepareFirst(columns: readonly string[], where: WhereNode | undefined, order: Order): string { + return this.sql.select(where, order, undefined, 1, columns).sql.replace(/LIMIT \?$/, "LIMIT 1"); + } + + /** + * The version of `json(...)` column `column`: at 1 every stored cell is at 1 (none is older, a newer one fails the + * owner when its table is prepared), so a read may leave the version column out. + */ + jsonVersion(column: string): number { + const found = this.definition.column(column, "jsonVersion"); + if (found.type !== "json") throw new Error(`${this.definition.name}: column "${column}" is no json(...) column`); + return found.json.version; + } + + /** A point lookup compiled once: `columns` of the row whose key the caller binds, in key order. */ + prepareGet(columns: readonly string[]): string { + return this.sql.getColumns(columns); + } + + /** + * The read of `json(...)` column `column`'s stored cells (`text` written at `version`, migrated on access), made + * once: an adapter selecting the cell itself reads it so. + */ + reader(column: string): (text: string, version: number, context: Context) => unknown { + const found = this.definition.column(column, "reader"); + if (found.type !== "json") throw new Error(`${this.definition.name}: column "${column}" is no json(...) column`); + // At version 1 a cell is the JSON it was written as: no version to compare, nothing to migrate. + if (found.json.version === 1) return (text) => JSON.parse(text) as unknown; + const what = `${this.sql.table}.${column}`; + return (text, version, context) => found.json.read(text, version, what, context); + } + + /** Up to `limit` rows matching `where` in `order`, decoded. */ + async select( + executor: Pick, + where: WhereNode | undefined, + order: Order, + limit: number, + context: Context, + ): Promise { + const statement = this.sql.select(where, order, undefined, limit); + const stored = await executor.all<{ [column: string]: SqliteValue }>(statement.sql, ...statement.params); + return stored.map((row) => this.definition.rows.decode(row, context)); + } +} diff --git a/packages/chord/src/next/store/allocator.ts b/packages/chord/src/next/store/allocator.ts new file mode 100644 index 000000000..6ff6d120a --- /dev/null +++ b/packages/chord/src/next/store/allocator.ts @@ -0,0 +1,33 @@ +// The store's id allocator (slice 6b D3, §1.2): one counter for every id in the store (Durable's `mintId`, Chord's +// document ids), synchronous, raised by explicit ids, saturating at `Number.MAX_SAFE_INTEGER`. + +/** The first id the allocator gives: id 1 is Durable's root conversation. */ +export const FIRST_ID = 2; + +/** + * The allocator: `value` is the next id it gives. An id raised to `Number.MAX_SAFE_INTEGER` is accepted, the value + * stays there, and every later `mint` throws: no id past the safe range (it would stop counting exactly and repeat). + */ +export class IdAllocator { + #next: number; + + constructor(next: number) { + this.#next = next; + } + + /** The next id `mint` gives; what each commit persists. */ + get value(): number { + return this.#next; + } + + /** Takes the next id; throws "ID space is exhausted" once saturated (Durable's message). */ + mint(): number { + if (this.#next >= Number.MAX_SAFE_INTEGER) throw new Error("ID space is exhausted"); + return this.#next++; + } + + /** Raises the counter above `id`, an id written explicitly (Durable mints ahead), saturating. */ + raise(id: number): void { + if (id >= this.#next) this.#next = Math.min(id + 1, Number.MAX_SAFE_INTEGER); + } +} diff --git a/packages/chord/src/next/store/applier.ts b/packages/chord/src/next/store/applier.ts index f6e55e214..a1bc6c862 100644 --- a/packages/chord/src/next/store/applier.ts +++ b/packages/chord/src/next/store/applier.ts @@ -1,25 +1,37 @@ -// The applier (slice 6a §13): recorded writes applied as they were recorded, inside a backend transaction, checked -// against the store's own records (a trust boundary: archives, later a replica's batches). One code path for import, -// bootstrap and, from slice 6b, batch apply; 6a applies the kinds an archive holds. +// The applier (slice 6a §13, slice 6b §1.6): recorded writes applied as they were recorded, inside a backend +// transaction, checked against the store's own records (a trust boundary: archives, a replica's batches). One code +// path for import, bootstrap and batch apply, for every write kind; a recorded count the store does not reproduce +// stops applying (`Diverged`, S17). import type { SqliteValue } from "../../sqlite/index.ts"; import { checkName } from "../contracts/internal.ts"; import { invalid } from "../data/internal.ts"; -import type { ArchivePart } from "./archive.ts"; import type { TableRecord } from "./catalog.ts"; import { COLUMN_NAME, type ColumnRecord } from "./columns.ts"; -import { insertDocument } from "./engine.ts"; +import { + type DocumentContent, + type DocumentRecord, + type DocumentWrite, + insertDocument, + insertRevision, + type RevisionContent, + readDocument, + retireRecord, + writeContent, +} from "./engine.ts"; +import type { ExprNode, SetNode, WhereNode } from "./expression.ts"; import { checkScalar, ROW_VERSION, toSqlite, versionColumn } from "./rows.ts"; -import { recordedPut } from "./sql.ts"; +import { recordedPut, TableSql } from "./sql.ts"; import type { Step } from "./step.ts"; import { INDEX_NAME } from "./table.ts"; -import { applyIndex, applyTable, type TableWrite } from "./table-writes.ts"; +import { applyAlter, applyDropIndex, applyIndex, applyTable, type StoredKey, type TableWrite } from "./table-writes.ts"; +import type { StoreWrite } from "./writes.ts"; -/** A recorded write the applier takes: the writes of an archive. */ -export type AppliedWrite = Exclude; +/** A recorded write the store does not reproduce (a count, a revision): applying stops, the store diverged (S17). */ +export class Diverged extends Error {} -/** Applies `write` in `step`; a write the records refuse throws `invalid/args` (`what` names its place). */ -export async function applyWrite(step: Step, write: AppliedWrite, what: string): Promise { +/** Applies `write` in `step`; a write the records refuse throws `invalid/args`, one they do not reproduce `Diverged`. */ +export async function applyWrite(step: Step, write: StoreWrite, what: string): Promise { switch (write.kind) { case "table": checkTable(write, what); @@ -29,29 +41,238 @@ export async function applyWrite(step: Step, write: AppliedWrite, what: string): checkIndex(recordOf(step, write.table, what), write, what); await applyIndex(step, write); return; + case "dropIndex": + if (!Object.hasOwn(recordOf(step, write.table, what).indexes, write.name)) { + throw invalid("args", `${what}: no index ${JSON.stringify(write.name)}`); + } + await applyDropIndex(step, write); + return; + case "alter": + checkAlter(recordOf(step, write.table, what), write, what); + await applyAlter(step, write); + return; case "put": { const record = recordOf(step, write.table, what); const { sql, physical } = recordedPut(write.table, record.columns); await step.run(sql, ...putParams(record, write, physical, what)); return; } - case "document.create": { - let text: string; - try { - text = JSON.stringify(write.base.value); - } catch (error) { - throw invalid("args", `${what}: the base is not JSON (${String(error)})`); - } - await insertDocument(step, write.record, text); - return; - } + case "update": + case "delete": + case "updateWhere": + case "deleteWhere": + return applyCounted(step, write, what); + default: + return applyDocument(step, write, what); } } -/** A recorded table's SQL name (three names joined by `:`), columns, key and json versions, by the name rules. */ +/** Applies a document write: a creation, a change or a retirement on its recorded revision, an archive's revision. */ +async function applyDocument(step: Step, write: DocumentWrite, what: string): Promise { + switch (write.kind) { + case "document.create": + await insertDocument(step, write.record, write.base === undefined ? undefined : json(write.base.value, what)); + return; + case "document.revision": { + const { content } = write; + await insertRevision(step, write.id, write.seq, content.kind, content.version, contentText(content, what)); + return; + } + case "document.change": { + const record = await liveAt(step, write.id, write.revision, what); + const { content } = write; + const text = contentText(content, what); + await writeContent(step, write.id, record.history, write.revision, content, text); + return; + } + case "document.retire": + await retireRecord(step, await liveAt(step, write.id, write.revision, what)); + return; + } +} + +/** A content's stored text: a base's value or a delta's ops, checked as JSON. */ +function contentText(content: DocumentContent | RevisionContent, what: string): string { + return json(content.kind === "base" ? content.value : content.ops, what); +} + +/** The live record `id` at `revision`; another revision, or none, is a divergence. */ +async function liveAt(step: Step, id: number, revision: number, what: string): Promise { + const record = await readDocument(step, id); + if (record === undefined || record.retiredAt !== undefined || record.revision !== revision) { + throw new Diverged(`${what}: document ${id} is not live at revision ${revision}`); + } + return record; +} + +/** `value` serialized; one `JSON.stringify` cannot encode is refused. */ +function json(value: unknown, what: string): string { + let text: string | undefined; + try { + text = JSON.stringify(value); + } catch (error) { + throw invalid("args", `${what}: not JSON (${String(error)})`); + } + if (text === undefined) throw invalid("args", `${what}: not JSON`); + return text; +} + +/** + * Applies a counted table write: its statement rendered from the record's columns and the recorded `key`, `where` and + * `set` (checked: only recorded columns, well-formed nodes), and its count compared with the recorded one. + */ +async function applyCounted( + step: Step, + write: Extract, + what: string, +): Promise { + const record = recordOf(step, write.table, what); + const columns = Object.entries(record.columns).map(([name, column]) => ({ name, type: column.type })); + const sql = new TableSql(write.table, columns, [], record.key); + let statement: { readonly sql: string; readonly params: SqliteValue[] }; + switch (write.kind) { + case "update": + statement = sql.update( + setOf(record, write, what), + keyParams(record, write.key, what), + versionsOf(record, write, what), + ); + break; + case "delete": + statement = { sql: sql.delete, params: keyParams(record, write.key, what) }; + break; + case "updateWhere": + checkWhere(record, write.where, what); + statement = sql.updateWhere(write.where, setOf(record, write, what), versionsOf(record, write, what)); + break; + case "deleteWhere": + checkWhere(record, write.where, what); + statement = sql.deleteWhere(write.where); + break; + } + const count = await step.runCounted(statement.sql, ...statement.params); + if (count !== write.count) throw new Diverged(`${what}: changed ${count} rows, recorded ${write.count}`); +} + +/** The recorded key as parameters in key order: exactly the key columns, each of its type. */ +function keyParams(record: TableRecord, key: StoredKey, what: string): SqliteValue[] { + if (Object.keys(key).length !== record.key.length) throw invalid("args", `${what}: not a key of ${record.table}`); + return record.key.map((name) => { + if (!Object.hasOwn(key, name)) throw invalid("args", `${what}: key column ${JSON.stringify(name)} is missing`); + return toSqlite(checkScalar(scalarType(record.columns[name]!), key[name], `${what}: key`)); + }); +} + +/** A recorded `set` with the migration stamp `$v` when it carries one; key columns never change (rule 13). */ +function setOf( + record: TableRecord, + write: Extract, + what: string, +): SetNode { + const { set } = write; + if (typeof set !== "object" || set === null || Object.keys(set).length === 0) { + throw invalid("args", `${what}: an empty or malformed set`); + } + for (const [column, node] of Object.entries(set)) { + if (!Object.hasOwn(record.columns, column) || record.key.includes(column)) { + throw invalid("args", `${what}: set writes ${JSON.stringify(column)}, no non-key column`); + } + checkExpr(record, node, what); + } + return write.version === undefined ? set : { ...set, [ROW_VERSION]: write.version }; +} + +/** The `json(...)` versions a `set` stamps: the recorded ones, of `json(...)` columns it sets, up to theirs. */ +function versionsOf( + record: TableRecord, + write: Extract, + what: string, +): Map { + const versions = new Map(); + for (const [name, column] of Object.entries(record.columns)) { + if (column.type !== "json" || !Object.hasOwn(write.set, name)) continue; + // A clear stamps `NULL` whatever the version; a value stamps its recorded version. + const version = write.columns[name] ?? column.version!; + if (!Number.isSafeInteger(version) || version < 1 || version > column.version!) { + throw invalid("args", `${what}: no readable version for ${JSON.stringify(name)}`); + } + versions.set(name, version); + } + return versions; +} + +/** A scalar column's type; `json(...)` columns are not scalars. */ +function scalarType(column: ColumnRecord): "str" | "int" | "num" | "bool" { + return column.type === "json" ? "str" : column.type; +} + +const COMPARISONS = ["eq", "ne", "lt", "lte", "gt", "gte"] as const; + +/** A recorded condition: well-formed nodes over recorded scalar columns (and the row's `$v`). */ +function checkWhere(record: TableRecord, node: unknown, what: string): asserts node is WhereNode { + const refuse = (): never => { + throw invalid("args", `${what}: a malformed condition`); + }; + if (typeof node !== "object" || node === null || Object.keys(node).length !== 1) return refuse(); + const [[op, operand]] = Object.entries(node) as [[string, unknown]]; + const column = (name: unknown): void => { + if (name === ROW_VERSION) return; + if (typeof name !== "string" || record.columns[name] === undefined || record.columns[name]!.type === "json") { + refuse(); + } + }; + if (op === "and" || op === "or") { + if (!Array.isArray(operand)) return refuse(); + for (const each of operand) checkWhere(record, each, what); + } else if (op === "not") checkWhere(record, operand, what); + else if (op === "isNull") column(operand); + else if (op === "inArray" || op === "prefix" || (COMPARISONS as readonly string[]).includes(op)) { + if (!Array.isArray(operand) || operand.length !== 2) return refuse(); + column(operand[0]); + const values = op === "inArray" ? operand[1] : [operand[1]]; + if (!Array.isArray(values)) return refuse(); + for (const value of values) if (!isScalar(value) || (op === "prefix" && typeof value !== "string")) refuse(); + } else refuse(); +} + +const OPERATIONS = ["add", "sub", "max", "min", "concat", "coalesce"] as const; + +/** A recorded expression: scalars, `null`, JSON literals, recorded columns and the vocabulary's operations. */ +function checkExpr(record: TableRecord, node: unknown, what: string): asserts node is ExprNode { + if (node === null || isScalar(node)) return; + const refuse = (): never => { + throw invalid("args", `${what}: a malformed expression`); + }; + if (typeof node !== "object" || Object.keys(node).length !== 1) return refuse(); + const [[op, operand]] = Object.entries(node) as [[string, unknown]]; + if (op === "col") { + if (typeof operand !== "string" || !Object.hasOwn(record.columns, operand)) refuse(); + return; + } + if (op === "json") return; + if (!Array.isArray(operand)) return refuse(); + if ((OPERATIONS as readonly string[]).includes(op) && operand.length === 2) { + for (const each of operand) checkExpr(record, each, what); + } else if (op === "replacePrefix" && operand.length === 3) { + checkExpr(record, operand[0], what); + if (typeof operand[1] !== "string" || typeof operand[2] !== "string") refuse(); + } else if (op === "when" && operand.length === 3) { + checkWhere(record, operand[0], what); + checkExpr(record, operand[1], what); + checkExpr(record, operand[2], what); + } else refuse(); +} + +function isScalar(value: unknown): boolean { + return ( + typeof value === "string" || typeof value === "boolean" || (typeof value === "number" && Number.isFinite(value)) + ); +} + +/** A recorded table's SQL name (two names joined by `:`, R2), columns, key and json versions, by the name rules. */ function checkTable(write: Extract, what: string): void { const parts = write.table.split(":"); - if (parts.length !== 3) throw invalid("args", `${what}: table ${JSON.stringify(write.table)}: not kind:type:table`); + if (parts.length !== 2) throw invalid("args", `${what}: table ${JSON.stringify(write.table)}: not owner:table`); try { for (const part of parts) checkName(part, `${what}: table name part`); } catch (error) { @@ -80,6 +301,30 @@ function checkColumn(column: ColumnRecord, what: string): void { checkScalar(column.type, column.default, `${what}: the default`); } +/** A recorded column edit of `record`'s next version or its own (the edits after a data step). */ +function checkAlter(record: TableRecord, write: Extract, what: string): void { + if (write.version !== record.version && write.version !== record.version + 1) { + throw invalid("args", `${what}: an edit of version ${write.version} on version ${record.version}`); + } + const { edit } = write; + const has = (name: string): boolean => Object.hasOwn(record.columns, name); + const fresh = (name: string): boolean => + COLUMN_NAME.test(name) && !Object.keys(record.columns).some((each) => each.toLowerCase() === name.toLowerCase()); + const indexed = (name: string): boolean => + record.key.includes(name) || Object.values(record.indexes).some((columns) => columns.includes(name)); + const valid = + edit.kind === "rename" + ? has(edit.from) && fresh(edit.to) + : edit.kind === "add" + ? fresh(edit.name) && (edit.column.optional === true || edit.column.default !== undefined) + : edit.kind === "drop" + ? has(edit.name) && !indexed(edit.name) + : has(edit.name) && record.columns[edit.name]!.type === "json" && edit.column.type === "json"; + if (!valid) throw invalid("args", `${what}: an ${edit.kind} edit the table cannot take`); + if (edit.kind === "add" || edit.kind === "json") + checkColumn(edit.column, `${what}: column ${JSON.stringify(edit.name)}`); +} + /** `names` are distinct, recorded, non-`opt` `str` or `int` columns. */ function checkOrderColumns( columns: { readonly [name: string]: ColumnRecord }, @@ -152,11 +397,7 @@ function putParams( } const version = versions[name]; if (version === undefined || version > column.version!) throw invalid("args", `${where}: no readable version`); - try { - cells.set(name, JSON.stringify(value)); - } catch (error) { - throw invalid("args", `${where}: not JSON (${String(error)})`); - } + cells.set(name, json(value, where)); cells.set(versionColumn(name), version); } return physical.map((name) => (name === ROW_VERSION ? version : (cells.get(name) ?? null))); diff --git a/packages/chord/src/next/store/apply-batch.ts b/packages/chord/src/next/store/apply-batch.ts new file mode 100644 index 000000000..be4be9ad2 --- /dev/null +++ b/packages/chord/src/next/store/apply-batch.ts @@ -0,0 +1,99 @@ +// Batch apply (slice 6b §1.6, S5, S17): a replica takes commits `from..to` as an outbox gave them, in one backend +// transaction through the store's commit queue, each write applied with its commit's seq by the applier (a trust +// boundary), `lastSeq` and the allocator set from the last commit. Internal: the sync layer's entry point. + +import type { Context } from "../../context/index.ts"; +import type { SqliteExecutor } from "../../sqlite/index.ts"; +import { invalid } from "../data/internal.ts"; +import { IdAllocator } from "./allocator.ts"; +import { applyWrite, Diverged } from "./applier.ts"; +import { checkWrite } from "./archive.ts"; +import { type TableRecord, writeCatalog } from "./catalog.ts"; +import { OUTBOX_FORMAT, type OutboxCommit } from "./outbox.ts"; +import { TABLES } from "./schema.ts"; +import { Step } from "./step.ts"; +import type { ChordStore, Replayed } from "./store.ts"; + +/** + * Applies `commits` (from an outbox, in seq order) to `store`: refused while an owner holds a claim on it, once it + * diverged, or when the first commit does not follow its `lastSeq` or one does not follow the one before; a write the + * store's records refuse rolls the whole batch back (`invalid/args`); a recorded count the store does not reproduce + * rolls it back and marks the store diverged, so every later batch is refused. Commits written to the store's outbox + * too when it has one. + */ +export function applyBatch(store: ChordStore, commits: readonly OutboxCommit[], context: Context): Promise { + let checked: OutboxCommit[]; + try { + checked = commits.map((commit, index) => checkCommit(commit, `batch: commit ${index}`, context)); + } catch (error) { + return Promise.reject(error); + } + return store.replay( + (executor, catalog) => replay(store, checked, executor, catalog), + (error) => error instanceof Diverged, + ); +} + +/** One commit's envelope and writes, checked (a trust boundary). */ +function checkCommit(commit: OutboxCommit, what: string, context: Context): OutboxCommit { + const { seq, nextId, writes } = commit as Partial; + if (!Number.isSafeInteger(seq) || !Number.isSafeInteger(nextId) || !Array.isArray(writes)) { + throw invalid("args", `${what}: not an outbox commit`); + } + return { + seq: seq!, + nextId: nextId!, + writes: writes.map((write, index) => checkWrite(write, `${what}: write ${index}`, context)), + }; +} + +/** The batch inside the store's turn: seqs continuous from `lastSeq`, allocator values never falling. */ +async function replay( + store: ChordStore, + commits: readonly OutboxCommit[], + executor: SqliteExecutor, + catalog: ReadonlyMap, +): Promise { + const tables = new Map(catalog); + const changed = new Map(); + let lastSeq = store.lastSeq; + let nextId = store.nextId; + for (const [index, commit] of commits.entries()) { + const what = `batch: commit ${index} (seq ${commit.seq})`; + if (commit.seq !== lastSeq + 1) throw invalid("args", `${what}: does not follow seq ${lastSeq}`); + if (commit.nextId < nextId) throw invalid("args", `${what}: its next id ${commit.nextId} is below ${nextId}`); + const step = new Step(executor, commit.seq, new IdAllocator(commit.nextId), tables); + for (const [position, write] of commit.writes.entries()) { + const where = `${what}: write ${position} (${write.kind})`; + if (write.kind === "document.create") checkCreated(write.record.id, commit.nextId, where); + await applyWrite(step, write, where); + } + await writeCatalog(step); + for (const [name, record] of step.tables) { + tables.set(name, record); + changed.set(name, record); + } + if (store.recordsOutbox) { + await executor.run( + `INSERT INTO ${TABLES.outbox} (seq, format, next_id, writes) VALUES (?, ?, ?, ?)`, + commit.seq, + OUTBOX_FORMAT, + commit.nextId, + JSON.stringify(commit.writes), + ); + } + lastSeq = commit.seq; + nextId = commit.nextId; + } + await executor.run(`UPDATE ${TABLES.meta} SET last_seq = ?, next_id = ? WHERE singleton = 1`, lastSeq, nextId); + return { lastSeq, nextId, tables: changed }; +} + +/** + * A created document's id is below its commit's next id (the allocator never mints it again), or equal to a saturated + * next id (`Number.MAX_SAFE_INTEGER` may be written, as import allows). + */ +function checkCreated(id: number, nextId: number, what: string): void { + if (id < nextId || (nextId === Number.MAX_SAFE_INTEGER && id === nextId)) return; + throw invalid("args", `${what}: id ${id} is not below the commit's next id ${nextId}`); +} diff --git a/packages/chord/src/next/store/archive.ts b/packages/chord/src/next/store/archive.ts index 963cfae48..1db668048 100644 --- a/packages/chord/src/next/store/archive.ts +++ b/packages/chord/src/next/store/archive.ts @@ -1,6 +1,6 @@ -// Archives (slice 6a §13; decisions "Export archives are operations", "Writes carry their versions; archives are -// lossless"): the parts an export yields and an import takes, their order, and the one compiled schema every -// imported part is checked against first. +// Archives and recorded commits as they cross a trust boundary (slice 6a §13, slice 6b §1.6; decisions "Export archives +// are operations", "Writes carry their versions; archives are lossless"): the parts an export yields and an import +// takes, their order, and the compiled schemas every imported part and every applied write is checked against first. import Type, { type TInteger } from "typebox"; import type { Context } from "../../context/index.ts"; @@ -8,28 +8,29 @@ import { arr, bool, int, lit, num, obj, opt, str, union, unknown } from "../cont import { invalid, validatorFor } from "../data/internal.ts"; import type { DocumentWrite } from "./engine.ts"; import type { TableWrite } from "./table-writes.ts"; +import type { StoreWrite } from "./writes.ts"; /** The archive format `export` writes; `import` takes this one only. */ export const ARCHIVE_FORMAT = 1; -/** The first part: the format, the store's schema version, its `lastSeq` and the next document id (rule 24). */ +/** The first part: the format, the store's schema version, its `lastSeq` and the allocator's next id (rule 24, D3). */ export type ArchiveManifest = { readonly kind: "manifest"; readonly format: number; readonly schema: number; readonly lastSeq: number; - readonly nextDocumentId: number; + readonly nextId: number; }; /** * One part of an archive, in this order: the manifest; per table (by SQL name) its `table` write, its `put`s and its - * `index` writes; the live documents' `document.create`s (by id, the current value as the base); `end` with the - * count of the parts between. + * `index` writes; per document record (by id, live and retired) its `document.create` without a base, then its kept + * revisions in seq order; `end` with the count of the parts between. */ export type ArchivePart = | ArchiveManifest | Extract - | Extract + | Extract | { readonly kind: "end"; readonly writes: number }; /** An integer from `min` up to 2^53 - 1: seqs, ids, versions and counts stay exact (a seq after 2^53 would repeat). */ @@ -47,54 +48,113 @@ const column = obj({ fingerprint: opt(str()), }); +const versions = Type.Record(Type.String(), version); + +const TABLE = obj({ + kind: lit("table"), + table: str(), + version, + state: union([lit("ready"), lit("migrating")]), + columns: Type.Record(Type.String(), column), + key: arr(str()), + fingerprint: str(), +}); + +const PUT = obj({ + kind: lit("put"), + table: str(), + row: Type.Record(Type.String(), unknown()), + version, + columns: versions, +}); + +const INDEX = obj({ kind: lit("index"), table: str(), name: str(), columns: arr(str()) }); + +const RECORD = obj({ + id: safe(1), + owner: str(), + namespace: str(), + kind: str(), + // Absent for a singleton. + key: opt(str()), + history: union([lit("none"), lit("latest"), lit("rewindable")]), + createdAt: safe(1), + retiredAt: opt(safe(1)), + meta: opt(str()), + revision: safe(1), + version, + fingerprint: str(), +}); + +const BASE = obj({ version, fingerprint: str(), value: unknown() }); + +/** A delta's ops; their shapes are checked where they are applied. */ +const ops = arr(unknown()); + +const REVISION = obj({ + kind: lit("document.revision"), + id: safe(1), + seq: safe(1), + content: union([obj({ kind: lit("base"), version, value: unknown() }), obj({ kind: lit("delta"), version, ops })]), +}); + const PART = union([ - obj({ - kind: lit("manifest"), - format: safe(1), - schema: safe(1), - lastSeq: safe(0), - nextDocumentId: safe(1), - }), - obj({ - kind: lit("table"), - table: str(), - version, - state: union([lit("ready"), lit("migrating")]), - columns: Type.Record(Type.String(), column), - key: arr(str()), - fingerprint: str(), - }), - obj({ - kind: lit("put"), - table: str(), - row: Type.Record(Type.String(), unknown()), - version, - columns: Type.Record(Type.String(), version), - }), - obj({ kind: lit("index"), table: str(), name: str(), columns: arr(str()) }), - obj({ - kind: lit("document.create"), - record: obj({ - id: safe(1), - namespace: str(), - owner: str(), - kind: str(), - // Absent for a singleton. - key: opt(str()), - history: union([lit("latest"), lit("rewindable")]), - createdAt: safe(1), - meta: opt(str()), - revision: safe(1), - version, - fingerprint: str(), - }), - base: obj({ version, fingerprint: str(), value: unknown() }), - }), + obj({ kind: lit("manifest"), format: safe(1), schema: safe(1), lastSeq: safe(0), nextId: safe(1) }), + TABLE, + PUT, + INDEX, + obj({ kind: lit("document.create"), record: RECORD }), + REVISION, obj({ kind: lit("end"), writes: safe(0) }), ]); +const count = safe(0); + +const edit = union([ + obj({ kind: lit("rename"), from: str(), to: str() }), + obj({ kind: lit("add"), name: str(), column }), + obj({ kind: lit("drop"), name: str() }), + obj({ kind: lit("json"), name: str(), column }), +]); + +const key = Type.Record(Type.String(), union([str(), num(), bool()])); + +/** One recorded write of a commit (a batch's); `where` and `set` nodes are checked by the applier. */ +const WRITE = union([ + TABLE, + PUT, + INDEX, + obj({ kind: lit("dropIndex"), table: str(), name: str() }), + obj({ kind: lit("alter"), table: str(), version, edit }), + obj({ kind: lit("update"), table: str(), key, set: unknown(), columns: versions, count, version: opt(version) }), + obj({ kind: lit("delete"), table: str(), key, count }), + obj({ + kind: lit("updateWhere"), + table: str(), + where: unknown(), + set: unknown(), + columns: versions, + count, + version: opt(version), + }), + obj({ kind: lit("deleteWhere"), table: str(), where: unknown(), count }), + obj({ kind: lit("document.create"), record: RECORD, base: BASE }), + obj({ + kind: lit("document.change"), + id: safe(1), + revision: safe(1), + content: union([ + obj({ kind: lit("base"), version, fingerprint: str(), value: unknown() }), + obj({ kind: lit("delta"), version, ops }), + ]), + }), + obj({ kind: lit("document.retire"), id: safe(1), revision: safe(1) }), +]); + const PART_SCHEMA = { schema: PART }; +const WRITE_SCHEMA = { schema: WRITE }; + /** Checks one imported part against the archive schema (a trust boundary); `invalid/args` naming its position. */ export function checkPart(part: unknown, position: number, context: Context): ArchivePart { context.telemetry.metrics?.counter("chord.data.validations").add(1); @@ -104,3 +164,11 @@ export function checkPart(part: unknown, position: number, context: Context): Ar // The check above establishes the type. return part as ArchivePart; } + +/** Checks one write of a batch's commit against the write schema (a trust boundary); `what` names its place. */ +export function checkWrite(write: unknown, what: string, context: Context): StoreWrite { + context.telemetry.metrics?.counter("chord.data.validations").add(1); + if (!validatorFor(WRITE_SCHEMA, context).Check(write)) throw invalid("args", `${what}: not a store write`); + // The check above establishes the type, the nodes aside (the applier checks them). + return write as StoreWrite; +} diff --git a/packages/chord/src/next/store/claims.ts b/packages/chord/src/next/store/claims.ts new file mode 100644 index 000000000..2f753f18f --- /dev/null +++ b/packages/chord/src/next/store/claims.ts @@ -0,0 +1,57 @@ +// Claims (slice 6b "owner names and claims", "claims per table and per document namespace"): what can collide in a +// store is claimed once, in memory: a table by its SQL name, documents by `(owner, namespace)`. A claim is held by its +// claimant (an object, a `StoreStorage`) until its end releases every claim it holds. While a batch applies, nothing +// is claimed: batch apply and owners exclude each other both ways. + +import { invalid } from "../data/internal.ts"; + +/** Who holds claims: named in the refusal of a second claim. */ +export type Claimant = { readonly label: string }; + +/** A store's claims, by key; one claimant per key. */ +export class Claims { + readonly #holders = new Map(); + readonly #keys = new Map(); + /** Whether a batch applies now: no claim is taken meanwhile. */ + #locked = false; + + /** Whether any claimant holds a claim (batch apply is refused then: its memory would go stale). */ + get held(): boolean { + return this.#keys.size > 0; + } + + /** Claims table `table` (its SQL name) for `claimant`; held by another: `invalid/args`. */ + table(table: string, claimant: Claimant): void { + this.#claim(`table ${JSON.stringify(table)}`, claimant); + } + + /** Claims the documents of `owner` in `namespace` for `claimant`; held by another: `invalid/args`. */ + documents(owner: string, namespace: string, claimant: Claimant): void { + this.#claim(`documents ${JSON.stringify([owner, namespace])}`, claimant); + } + + /** Refuses every claim (`invalid/args`) until the returned unlock: a batch applies meanwhile. */ + lock(): () => void { + this.#locked = true; + return () => { + this.#locked = false; + }; + } + + /** Releases every claim `claimant` holds. */ + release(claimant: Claimant): void { + for (const key of this.#keys.get(claimant) ?? []) this.#holders.delete(key); + this.#keys.delete(claimant); + } + + #claim(key: string, claimant: Claimant): void { + if (this.#locked) throw invalid("args", `store: ${key} cannot be claimed while a batch applies`); + const holder = this.#holders.get(key); + if (holder === claimant) return; + if (holder !== undefined) throw invalid("args", `store: ${key} is claimed by ${holder.label}`); + this.#holders.set(key, claimant); + const keys = this.#keys.get(claimant); + if (keys === undefined) this.#keys.set(claimant, [key]); + else keys.push(key); + } +} diff --git a/packages/chord/src/next/store/engine.ts b/packages/chord/src/next/store/engine.ts index b5fbc8f80..b30b70b2a 100644 --- a/packages/chord/src/next/store/engine.ts +++ b/packages/chord/src/next/store/engine.ts @@ -1,6 +1,7 @@ -// The documents engine (S13, slice 6a §8): one record per incarnation of an address, revisions made of a base and -// later deltas; its writes inside the commit step, checked by the writes themselves (the partial unique index on the -// live address, the counted conditional `UPDATE` on the record's revision), and its plain reads. +// The documents engine (S13, slice 6a §8, slice 6b §1.5): one record per incarnation of an address, revisions made of +// bases and deltas, a history rule per record (R1); its writes inside the commit step, checked by the writes +// themselves (the partial unique index on the live address, the counted conditional `UPDATE` on the record's +// revision), and its reads, of the current state or at a point. import { apply, type Op } from "../../delta/index.ts"; import type { JsonValue } from "../../json.ts"; @@ -9,23 +10,33 @@ import { TABLES } from "./schema.ts"; import type { StagedWrite, Step } from "./step.ts"; /** - * A document's address: the adapter's namespace, the owner string, the definition's name and, for a keyed document, its - * key; a singleton has none (as Durable's addresses). The column keeps `""` for it, which a keyed document's empty key - * shares (its definition says which it is): a record read back gives no key for either. + * A document's address: the owner name, the adapter's namespace (claimed with the owner: Chord's `chord`, `chord.jobs`, + * `chord.job-states`; Durable's scopes), the kind and, for a keyed document, its key; a singleton has none. The column + * keeps `""` for it, which a keyed document's empty key shares (its definition says which it is). */ export type DocumentAddress = { - readonly namespace: string; readonly owner: string; + readonly namespace: string; readonly kind: string; readonly key?: string; }; -/** A stored record: one incarnation of an address. Chord keeps only the latest revision (`history: "latest"`). */ +/** + * What a record keeps (R1), applied at a new base and at retirement: `none` (Chord) drops older revisions at a new base + * and the row and its revisions at retirement; `latest` drops older revisions at a new base and its revisions at + * retirement, keeping the row; `rewindable` keeps everything. + */ +export type DocumentHistory = "none" | "latest" | "rewindable"; + +/** A stored record: one incarnation of an address, live from `createdAt` until `retiredAt` (half-open). */ export type DocumentRecord = DocumentAddress & { readonly id: number; - readonly history: "latest" | "rewindable"; + readonly history: DocumentHistory; /** The seq that created it. */ readonly createdAt: number; + /** The seq that retired it, once retired. */ + readonly retiredAt?: number; + /** The adapter's details (Durable: its fork policy and history). */ readonly meta?: string; /** Counts its changes: a change names the revision it was prepared on. */ readonly revision: number; @@ -42,130 +53,165 @@ export type DocumentContent = | ({ readonly kind: "base" } & DocumentBase) | { readonly kind: "delta"; readonly version: number; readonly ops: readonly Op[] }; -/** The engine's writes as the outbox and archives record them. */ +/** A stored revision's content: a base's value or a delta's ops, at a version (a revision keeps no fingerprint). */ +export type RevisionContent = + | { readonly kind: "base"; readonly version: number; readonly value: JsonValue } + | { readonly kind: "delta"; readonly version: number; readonly ops: readonly Op[] }; + +/** One stored revision as archives carry it: its document, its seq, its content. */ +export type DocumentRevision = { readonly id: number; readonly seq: number; readonly content: RevisionContent }; + +/** + * The engine's writes as the outbox and archives record them. A creation carries its base in a commit (stored at + * `createdAt`); in an archive it has none and its `document.revision`s follow it. + */ export type DocumentWrite = - | { readonly kind: "document.create"; readonly record: DocumentRecord; readonly base: DocumentBase } + | { readonly kind: "document.create"; readonly record: DocumentRecord; readonly base?: DocumentBase } | { readonly kind: "document.change"; readonly id: number; readonly revision: number; readonly content: DocumentContent; } - | { readonly kind: "document.retire"; readonly id: number; readonly revision: number }; + | { readonly kind: "document.retire"; readonly id: number; readonly revision: number } + | ({ readonly kind: "document.revision" } & DocumentRevision); -/** A loaded document: its record and its value (base plus deltas applied once). */ +/** A loaded document: its record and its value (a base plus deltas applied once) at a point. */ export type StoredDocument = { readonly record: DocumentRecord; readonly value: JsonValue; - /** The deltas stored after its base (what `checkpointWhen` sees). */ + /** The version of the base the value was built from. */ + readonly version: number; + /** The deltas applied after its base (what `checkpointWhen` sees). */ readonly deltasSinceBase: number; }; +/** Where a read looks: the current state, or the state after a seq. */ +export type DocumentPoint = number | "current"; + /** - * Stages the creation of a document at `address` with `base`; its id comes from the step. `text` is the base's value - * serialized before the queue (rule 8). + * Stages the creation of a Chord document at `address` with `base`; its id is taken from the allocator when it + * applies (a saturated allocator refuses the commit) and handed to `created` (its owner adopts it after `COMMIT`). + * `text` is the base's value serialized before the queue (rule 8). */ export function createDocument( address: DocumentAddress, - history: "latest" | "rewindable", base: DocumentBase, text: string, + created?: (id: number) => void, ): StagedWrite { return { order: "create", apply: async (step) => { - // The next id is stored too: it stays in the safe range, or ids would stop counting exactly and repeat. - if (step.nextDocumentId >= Number.MAX_SAFE_INTEGER) { - throw new Error("store: no document id past Number.MAX_SAFE_INTEGER"); - } - const id = step.nextDocumentId++; + const id = step.ids.mint(); step.created.push(id); + created?.(id); const { version, fingerprint } = base; const record: DocumentRecord = { id, ...address, - history, + history: "none", createdAt: step.seq, revision: 1, version, fingerprint, }; await insertDocument(step, record, text); - return { kind: "document.create", record, base }; + return [{ kind: "document.create", record, base }]; }, }; } /** - * Inserts `record` as it is (id, `createdAt`, revision kept: a creation, or an import) with its base, `text` being - * the base's value serialized. + * Inserts `record` as it is (id, seqs and revision kept: a creation, an import, a batch), with its base at `createdAt` + * when `text` (the base's value serialized) is given; a record created retired (an empty lifetime) keeps what its + * history keeps. */ -export async function insertDocument(step: Step, record: DocumentRecord, text: string): Promise { - const { id, namespace, owner, kind, key, history, createdAt, meta, revision, version, fingerprint } = record; +export async function insertDocument(step: Step, record: DocumentRecord, text: string | undefined): Promise { + const { id, namespace, owner, kind, key, history, createdAt, retiredAt, meta, revision, version, fingerprint } = + record; await step.run( `INSERT INTO ${TABLES.documents} - (id, namespace, owner, kind, key, history, created_at, meta, revision, version, fingerprint) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + (id, owner, namespace, kind, key, history, created_at, retired_at, meta, revision, version, fingerprint) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, id, - namespace, owner, + namespace, kind, key ?? "", history, createdAt, + retiredAt ?? null, meta ?? null, revision, version, fingerprint, ); - await insertRevision(step, id, "base", version, text); + if (text !== undefined) await insertRevision(step, id, createdAt, "base", version, text); + if (retiredAt !== undefined) await dropRetired(step, id, history); } /** - * Stages a change of document `id` prepared on `revision`: a delta on the current base's version, or a new base - * (which prunes the older revisions of a `latest` record). `text` is the content serialized before the queue. + * Stages a change of Chord document `id` prepared on `revision`: a delta on the current base's version, or a new base. + * `text` is the content serialized before the queue. */ export function changeDocument(id: number, revision: number, content: DocumentContent, text: string): StagedWrite { const write: DocumentWrite = { kind: "document.change", id, revision, content }; return { order: "write", apply: async (step) => { - // As ids: a revision past the safe range would stop counting exactly. - if (revision >= Number.MAX_SAFE_INTEGER) throw new Error(`document ${id}: no revision past the safe range`); - await insertRevision(step, id, content.kind, content.version, text); - const changed = - content.kind === "delta" - ? await step.runCounted( - `UPDATE ${TABLES.documents} SET revision = revision + 1 - WHERE id = ? AND revision = ? AND version = ? AND retired_at IS NULL`, - id, - revision, - content.version, - ) - : await step.runCounted( - `UPDATE ${TABLES.documents} SET revision = revision + 1, version = ?, fingerprint = ? - WHERE id = ? AND revision = ? AND retired_at IS NULL`, - content.version, - content.fingerprint, - id, - revision, - ); - // Memory and disk disagree: the step fails, which stops the store (Durable's rule). - if (changed !== 1) throw new Error(`document ${id}: not live at revision ${revision}`); - if (content.kind === "base") { - await step.run(`DELETE FROM ${TABLES.revisions} WHERE document_id = ? AND seq < ?`, id, step.seq); - } - return write; + await writeContent(step, id, "none", revision, content, text); + return [write]; }, }; } -/** Stages the retirement of `latest` document `id` at `revision`: its record and revisions go (rule 19). */ +/** + * Writes `content` to document `id` (of `history`) prepared on `revision`, in the step's seq: its revision, the + * counted conditional `UPDATE` (a delta only on the base's version), and at a new base the older revisions a + * `none` or `latest` record drops. A record that is not live at `revision` throws (memory and disk disagree). + */ +export async function writeContent( + step: Step, + id: number, + history: DocumentHistory, + revision: number, + content: DocumentContent, + text: string, +): Promise { + // As ids: a revision past the safe range would stop counting exactly. + if (revision >= Number.MAX_SAFE_INTEGER) throw new Error(`document ${id}: no revision past the safe range`); + await insertRevision(step, id, step.seq, content.kind, content.version, text); + const changed = + content.kind === "delta" + ? await step.runCounted( + `UPDATE ${TABLES.documents} SET revision = revision + 1 + WHERE id = ? AND revision = ? AND version = ? AND retired_at IS NULL`, + id, + revision, + content.version, + ) + : await step.runCounted( + `UPDATE ${TABLES.documents} SET revision = revision + 1, version = ?, fingerprint = ? + WHERE id = ? AND revision = ? AND retired_at IS NULL`, + content.version, + content.fingerprint, + id, + revision, + ); + if (changed !== 1) throw new Error(`document ${id}: not live at revision ${revision}`); + if (content.kind === "base" && history !== "rewindable") { + await step.run(`DELETE FROM ${TABLES.revisions} WHERE document_id = ? AND seq < ?`, id, step.seq); + } +} + +/** Stages the retirement of Chord document `id` (history `none`) at `revision`: its row and its revisions go. */ export function retireDocument(id: number, revision: number): StagedWrite { const write: DocumentWrite = { kind: "document.retire", id, revision }; return { order: "retire", apply: async (step) => { + // A Chord document keeps no history (`none`): its row goes with a conditional, counted DELETE (6a's). const deleted = await step.runCounted( `DELETE FROM ${TABLES.documents} WHERE id = ? AND revision = ? AND retired_at IS NULL`, id, @@ -173,43 +219,141 @@ export function retireDocument(id: number, revision: number): StagedWrite { ); if (deleted !== 1) throw new Error(`document ${id}: not live at revision ${revision}`); await step.run(`DELETE FROM ${TABLES.revisions} WHERE document_id = ?`, id); - return write; + return [write]; }, }; } +/** + * Retires live `record` in the step's seq by its history (R1): a `none` record's row and revisions go, a `latest` one + * keeps its row with `retired_at` and drops its revisions, a `rewindable` one keeps everything. + */ +export async function retireRecord(step: Step, record: DocumentRecord): Promise { + const { id, history } = record; + if (history !== "none") await step.run(`UPDATE ${TABLES.documents} SET retired_at = ? WHERE id = ?`, step.seq, id); + await dropRetired(step, id, history); +} + +/** What a retired record of `history` drops: a `none` row too, every revision unless `rewindable`. */ +async function dropRetired(step: Step, id: number, history: DocumentHistory): Promise { + if (history === "rewindable") return; + if (history === "none") await step.run(`DELETE FROM ${TABLES.documents} WHERE id = ?`, id); + await step.run(`DELETE FROM ${TABLES.revisions} WHERE document_id = ?`, id); +} + +/** Inserts one revision of document `id` at `seq`, `text` its serialized content. */ +export function insertRevision( + step: Step, + id: number, + seq: number, + kind: "base" | "delta", + version: number, + text: string, +): Promise { + return step.run( + `INSERT INTO ${TABLES.revisions} (document_id, seq, kind, version, content) VALUES (?, ?, ?, ?, ?)`, + id, + seq, + kind, + version, + text, + ); +} + type RecordRow = { readonly id: number; - readonly namespace: string; readonly owner: string; + readonly namespace: string; readonly kind: string; readonly key: string; - readonly history: "latest" | "rewindable"; + readonly history: DocumentHistory; readonly created_at: number; + readonly retired_at: number | null; readonly meta: string | null; readonly revision: number; readonly version: number; readonly fingerprint: string; }; -const RECORD_COLUMNS = "id, namespace, owner, kind, key, history, created_at, meta, revision, version, fingerprint"; +const RECORD_COLUMNS = + "id, owner, namespace, kind, key, history, created_at, retired_at, meta, revision, version, fingerprint"; -/** The live record at `address`, if any. */ -export async function findDocument( - executor: SqliteExecutor, - address: DocumentAddress, +const READ = `SELECT ${RECORD_COLUMNS} FROM ${TABLES.documents} WHERE id = ?`; + +/** The record with `id`, live or retired, if there is one. */ +export async function readDocument( + executor: Pick, + id: number, ): Promise { - const row = await executor.get( - `SELECT ${RECORD_COLUMNS} FROM ${TABLES.documents} - WHERE namespace = ? AND owner = ? AND kind = ? AND key = ? AND retired_at IS NULL`, - address.namespace, - address.owner, - address.kind, - address.key ?? "", - ); + const row = await executor.get(READ, id); return row === undefined ? undefined : recordOf(row); } +const FIND_CURRENT = `SELECT ${RECORD_COLUMNS} FROM ${TABLES.documents} + WHERE owner = ? AND namespace = ? AND kind = ? AND key = ? AND retired_at IS NULL`; + +const FIND_AT = `SELECT ${RECORD_COLUMNS} FROM ${TABLES.documents} + WHERE owner = ? AND namespace = ? AND kind = ? AND key = ? AND created_at <= ? AND (retired_at IS NULL OR retired_at > ?) + ORDER BY created_at DESC LIMIT 1`; + +/** The record at `address` live at `at` (the half-open lifetime `createdAt <= at < retiredAt`), if any. */ +export async function findDocument( + executor: Pick, + address: DocumentAddress, + at: DocumentPoint = "current", +): Promise { + const { owner, namespace, kind } = address; + const key = address.key ?? ""; + const row = + at === "current" + ? await executor.get(FIND_CURRENT, owner, namespace, kind, key) + : await executor.get(FIND_AT, owner, namespace, kind, key, at, at); + return row === undefined ? undefined : recordOf(row); +} + +/** + * A record as an adapter that keeps its own fields in `meta` reads it: the meta, and, where a read can find retired + * records (at a point, by id), when it was retired. One or two columns: each column a read returns costs. + */ +export type DocumentEntry = { readonly meta: string | null; readonly retired_at?: number | null }; + +/** A document entry with what tells whose it is and what it keeps, as a read by id needs. */ +export type OwnedDocumentEntry = DocumentEntry & { + readonly owner: string; + readonly namespace: string; + readonly history: DocumentHistory; +}; + +const READ_ENTRY = `SELECT meta, retired_at, owner, namespace, history FROM ${TABLES.documents} WHERE id = ?`; + +/** The entry of the record with `id`, live or retired, with its owner, namespace and history; `undefined` if none. */ +export function readDocumentEntry( + executor: Pick, + id: number, +): Promise { + return executor.get(READ_ENTRY, id); +} + +const FIND_ENTRY_CURRENT = `SELECT meta FROM ${TABLES.documents} + WHERE owner = ? AND namespace = ? AND kind = ? AND key = ? AND retired_at IS NULL`; + +const FIND_ENTRY_AT = `SELECT meta, retired_at FROM ${TABLES.documents} + WHERE owner = ? AND namespace = ? AND kind = ? AND key = ? AND created_at <= ? AND (retired_at IS NULL OR retired_at > ?) + ORDER BY created_at DESC LIMIT 1`; + +/** `findDocument` reading only what an adapter keeping its fields in `meta` needs (fewer columns, no record built). */ +export function findDocumentEntry( + executor: Pick, + address: DocumentAddress, + at: DocumentPoint, +): Promise { + const { owner, namespace, kind } = address; + const key = address.key ?? ""; + return at === "current" + ? executor.get(FIND_ENTRY_CURRENT, owner, namespace, kind, key) + : executor.get(FIND_ENTRY_AT, owner, namespace, kind, key, at, at); +} + type RevisionRow = { readonly seq: number; readonly kind: "base" | "delta"; @@ -217,22 +361,37 @@ type RevisionRow = { readonly content: string; }; -/** Loads the live document `record` found: its latest base and the deltas after it, applied once. */ -export async function loadDocument(executor: SqliteExecutor, record: DocumentRecord): Promise { - const revisions = await executor.all( - `SELECT seq, kind, version, content FROM ${TABLES.revisions} WHERE document_id = ? AND seq >= ( - SELECT max(seq) FROM ${TABLES.revisions} WHERE document_id = ? AND kind = 'base') ORDER BY seq`, - record.id, - record.id, - ); +const LOAD = `SELECT seq, kind, version, content FROM ${TABLES.revisions} WHERE document_id = ? AND seq <= ? AND seq >= ( + SELECT max(seq) FROM ${TABLES.revisions} WHERE document_id = ? AND kind = 'base' AND seq <= ?) ORDER BY seq`; + +/** + * Loads `record` at `at`: its newest base at or below it and the deltas after that base up to it, applied once; the + * version is the base's. A record without such a base (one that keeps no history from there) throws. + */ +export async function loadDocument( + executor: Pick, + record: DocumentRecord, + at: DocumentPoint = "current", +): Promise { + return { record, ...(await loadContent(executor, record.id, at)) }; +} + +/** `loadDocument`'s content of document `id` at `at`, without its record (an adapter reads the fields it keeps). */ +export async function loadContent( + executor: Pick, + id: number, + at: DocumentPoint, +): Promise> { + const upper = at === "current" ? Number.MAX_SAFE_INTEGER : at; + const revisions = await executor.all(LOAD, id, upper, id, upper); const [base, ...deltas] = revisions; - if (base === undefined || base.kind !== "base") throw new Error(`document ${record.id}: no base`); + if (base === undefined || base.kind !== "base") throw new Error(`document ${id}: no base`); let value = JSON.parse(base.content) as JsonValue; for (const delta of deltas) { - if (delta.version !== base.version) throw new Error(`document ${record.id}: a delta crosses a version`); + if (delta.version !== base.version) throw new Error(`document ${id}: a delta crosses a version`); value = apply(value, JSON.parse(delta.content) as Op[]); } - return { record, value, deltasSinceBase: deltas.length }; + return { value, version: base.version, deltasSinceBase: deltas.length }; } /** Up to `limit` keys of live documents of `kind` under `owner` after `after` (from the first), in code point order. */ @@ -245,10 +404,10 @@ export async function listDocumentKeys( // The first page has no lower bound: `""` is a key too. const from = after === undefined ? "" : " AND key > ?"; const rows = await executor.all<{ readonly key: string }>( - `SELECT key FROM ${TABLES.documents} WHERE namespace = ? AND owner = ? AND kind = ?${from} + `SELECT key FROM ${TABLES.documents} WHERE owner = ? AND namespace = ? AND kind = ?${from} AND retired_at IS NULL ORDER BY key LIMIT ?`, - address.namespace, address.owner, + address.namespace, address.kind, ...(after === undefined ? [] : [after]), limit, @@ -256,40 +415,131 @@ export async function listDocumentKeys( return rows.map(({ key }) => key); } -/** Up to `limit` live records with ids above `after`, by id (an export's pages). */ -export async function listDocuments(executor: SqliteExecutor, after: number, limit: number): Promise { +/** What `listDocuments` scans: one owner's namespace, optionally one kind. */ +export type DocumentScope = { readonly owner: string; readonly namespace: string; readonly kind?: string }; + +/** The statements of `listDocuments`, by its shape: with a kind or not, current or at a point. */ +const LIST = (() => { + const from = `SELECT ${RECORD_COLUMNS} FROM ${TABLES.documents} WHERE owner = ? AND namespace = ? AND id > ?`; + const current = " AND retired_at IS NULL ORDER BY id LIMIT ?"; + const at = " AND created_at <= ? AND (retired_at IS NULL OR retired_at > ?) ORDER BY id LIMIT ?"; + return { + current: `${from}${current}`, + at: `${from}${at}`, + kindCurrent: `${from} AND kind = ?${current}`, + kindAt: `${from} AND kind = ?${at}`, + }; +})(); + +/** Up to `limit` records of `scope` live at `at` with ids above `after`, by id. */ +export async function listDocuments( + executor: Pick, + scope: DocumentScope, + at: DocumentPoint, + after: number, + limit: number, +): Promise { + const { owner, namespace, kind } = scope; + let rows: RecordRow[]; + if (kind === undefined) { + rows = + at === "current" + ? await executor.all(LIST.current, owner, namespace, after, limit) + : await executor.all(LIST.at, owner, namespace, after, at, at, limit); + } else { + rows = + at === "current" + ? await executor.all(LIST.kindCurrent, owner, namespace, after, kind, limit) + : await executor.all(LIST.kindAt, owner, namespace, after, kind, at, at, limit); + } + return rows.map(recordOf); +} + +/** The statements of `listDocumentEntries`, by shape, as `LIST`'s. */ +const LIST_ENTRIES = (() => { + const from = (columns: string) => + `SELECT ${columns} FROM ${TABLES.documents} WHERE owner = ? AND namespace = ? AND id > ?`; + const current = " AND retired_at IS NULL ORDER BY id LIMIT ?"; + const at = " AND created_at <= ? AND (retired_at IS NULL OR retired_at > ?) ORDER BY id LIMIT ?"; + return { + current: `${from("meta")}${current}`, + at: `${from("meta, retired_at")}${at}`, + kindCurrent: `${from("meta")} AND kind = ?${current}`, + kindAt: `${from("meta, retired_at")} AND kind = ?${at}`, + }; +})(); + +/** `listDocuments` reading only what an adapter keeping its fields in `meta` needs. */ +export function listDocumentEntries( + executor: Pick, + scope: DocumentScope, + at: DocumentPoint, + after: number, + limit: number, +): Promise { + const { owner, namespace, kind } = scope; + if (kind === undefined) { + return at === "current" + ? executor.all(LIST_ENTRIES.current, owner, namespace, after, limit) + : executor.all(LIST_ENTRIES.at, owner, namespace, after, at, at, limit); + } + return at === "current" + ? executor.all(LIST_ENTRIES.kindCurrent, owner, namespace, after, kind, limit) + : executor.all(LIST_ENTRIES.kindAt, owner, namespace, after, kind, at, at, limit); +} + +/** Up to `limit` stored records, live and retired, with ids above `after`, by id (an export's pages). */ +export async function listRecords(executor: SqliteExecutor, after: number, limit: number): Promise { const rows = await executor.all( - `SELECT ${RECORD_COLUMNS} FROM ${TABLES.documents} WHERE id > ? AND retired_at IS NULL ORDER BY id LIMIT ?`, + `SELECT ${RECORD_COLUMNS} FROM ${TABLES.documents} WHERE id > ? ORDER BY id LIMIT ?`, after, limit, ); return rows.map(recordOf); } +/** Up to `limit` stored revisions of document `id` after seq `after`, in seq order (an export's page). */ +export async function listRevisions( + executor: SqliteExecutor, + id: number, + after: number, + limit: number, +): Promise { + const rows = await executor.all( + `SELECT seq, kind, version, content FROM ${TABLES.revisions} WHERE document_id = ? AND seq > ? ORDER BY seq LIMIT ?`, + id, + after, + limit, + ); + return rows.map(({ seq, kind, version, content }) => { + // Written by `insertRevision`: a base's value or a delta's ops, as JSON. + const parsed = JSON.parse(content) as unknown; + return { + id, + seq, + content: + kind === "base" + ? { kind, version, value: parsed as JsonValue } + : { kind, version, ops: parsed as readonly Op[] }, + }; + }); +} + function recordOf(row: RecordRow): DocumentRecord { - const { id, namespace, owner, kind, key, history, created_at, meta, revision, version, fingerprint } = row; + const { id, namespace, owner, kind, key, history, created_at, retired_at, meta, revision, version, fingerprint } = + row; return { id, - namespace, owner, + namespace, kind, ...(key === "" ? {} : { key }), history, createdAt: created_at, + ...(retired_at === null ? {} : { retiredAt: retired_at }), ...(meta === null ? {} : { meta }), revision, version, fingerprint, }; } - -function insertRevision(step: Step, id: number, kind: "base" | "delta", version: number, text: string): Promise { - return step.run( - `INSERT INTO ${TABLES.revisions} (document_id, seq, kind, version, content) VALUES (?, ?, ?, ?, ?)`, - id, - step.seq, - kind, - version, - text, - ); -} diff --git a/packages/chord/src/next/store/export.ts b/packages/chord/src/next/store/export.ts index cdadc0424..78554d6e2 100644 --- a/packages/chord/src/next/store/export.ts +++ b/packages/chord/src/next/store/export.ts @@ -10,7 +10,7 @@ import type { SqliteValue } from "../../sqlite/index.ts"; import { ChordError } from "../data/index.ts"; import { ARCHIVE_FORMAT, type ArchivePart } from "./archive.ts"; import type { TableRecord } from "./catalog.ts"; -import { listDocuments, loadDocument } from "./engine.ts"; +import { listRecords, listRevisions } from "./engine.ts"; import { ROW_VERSION, type Scalar, versionColumn } from "./rows.ts"; import { STORE_SCHEMA } from "./schema.ts"; import { quote, recordedPut } from "./sql.ts"; @@ -116,8 +116,8 @@ async function* parts( hold(release); try { if (context.abortSignal?.aborted) throw aborted(context); - const { lastSeq, nextDocumentId } = store; - yield { kind: "manifest", format: ARCHIVE_FORMAT, schema: STORE_SCHEMA, lastSeq, nextDocumentId }; + const { lastSeq, nextId } = store; + yield { kind: "manifest", format: ARCHIVE_FORMAT, schema: STORE_SCHEMA, lastSeq, nextId }; let writes = 0; const tables = [...store.tables.values()].sort((a, b) => (a.table < b.table ? -1 : 1)); for (const record of tables) { @@ -133,8 +133,8 @@ async function* parts( writes++; } } - for await (const create of documents(store, context)) { - yield create; + for await (const part of documents(store, context)) { + yield part; writes++; } yield { kind: "end", writes }; @@ -188,20 +188,37 @@ function putOf( return { kind: "put", table: record.table, row, version: stored[ROW_VERSION] as number, columns: versions }; } -/** The live documents by id as their creations: the record and the current value as the base. */ +/** + * Every stored record by id, live and retired, as its creation without a base, then its kept revisions in seq order: + * a `none` or `latest` record's current base and its deltas, a retired `latest` one none, a `rewindable` one all. + */ async function* documents( store: ChordStore, context: Context, -): AsyncGenerator> { +): AsyncGenerator> { let after = 0; for (;;) { - const records = await store.read((executor) => listDocuments(executor, after, EXPORT_PAGE), context); + const records = await store.read((executor) => listRecords(executor, after, EXPORT_PAGE), context); for (const record of records) { - const { value } = await store.read((executor) => loadDocument(executor, record), context); - const { version, fingerprint } = record; - yield { kind: "document.create", record, base: { version, fingerprint, value } }; + yield { kind: "document.create", record }; + yield* revisions(store, record.id, context); } if (records.length < EXPORT_PAGE) return; after = records.at(-1)!.id; } } + +/** Document `id`'s kept revisions in seq order, read page by page (a long history is never one array). */ +async function* revisions( + store: ChordStore, + id: number, + context: Context, +): AsyncGenerator> { + let after = 0; + for (;;) { + const page = await store.read((executor) => listRevisions(executor, id, after, EXPORT_PAGE), context); + for (const revision of page) yield { kind: "document.revision", ...revision }; + if (page.length < EXPORT_PAGE) return; + after = page.at(-1)!.seq; + } +} diff --git a/packages/chord/src/next/store/import.ts b/packages/chord/src/next/store/import.ts index c53ca4e77..46d4d290d 100644 --- a/packages/chord/src/next/store/import.ts +++ b/packages/chord/src/next/store/import.ts @@ -1,17 +1,20 @@ // `importStore(backend, parts, ctx, { outbox? })` (slice 6a §13, rules 12, 24, 25): a store made from an archive, a // trust boundary. Every part is checked against the archive schema, its place in the archive's order and its content -// (names, rows against their table's recorded columns, ids and seqs, one live record per address, versions, -// duplicates, the `end` count, the manifest); everything is applied in one backend transaction with the applier, so a -// refused import leaves the backend as it was. An input that stalls is cancelled by `ctx`. +// (names, rows against their table's recorded columns, ids and seqs, lifetimes at one address never overlapping, +// revisions in seq order from a base, versions, duplicates, the `end` count, the manifest); everything is applied in +// one backend transaction with the applier, so a refused import leaves the backend as it was. An input that stalls is +// cancelled by `ctx`. import { awaitWithContext, type Context } from "../../context/index.ts"; import type { SqliteDatabase, SqliteExecutor } from "../../sqlite/index.ts"; import { ChordError } from "../data/index.ts"; import { invalid } from "../data/internal.ts"; +import { IdAllocator } from "./allocator.ts"; import { applyWrite } from "./applier.ts"; import { ARCHIVE_FORMAT, type ArchiveManifest, type ArchivePart, checkPart } from "./archive.ts"; import type { Backend } from "./backend.ts"; import { readCatalog, writeCatalog } from "./catalog.ts"; +import type { DocumentRecord } from "./engine.ts"; import { applyStoreSchema, type MetaRow, STORE_SCHEMA, TABLES } from "./schema.ts"; import { Step } from "./step.ts"; import { ChordStore, type Store, type StoreOptions } from "./store.ts"; @@ -83,7 +86,10 @@ class ArchiveImport { #table: string | undefined; readonly #tables = new Set(); readonly #ids = new Set(); - readonly #addresses = new Set(); + /** The lifetimes seen at each address, `[createdAt, retiredAt)` (`Infinity` while live). */ + readonly #lifetimes = new Map(); + /** The record whose revisions may follow, with the last revision's seq and its base's version. */ + #document: { readonly record: DocumentRecord; seq: number; base: number | undefined } | undefined; #writes = 0; constructor(executor: SqliteExecutor, context: Context) { @@ -105,6 +111,7 @@ class ArchiveImport { if (part.writes !== this.#writes) { throw invalid("args", `${what}: counts ${part.writes} writes, the archive has ${this.#writes}`); } + this.#endDocument(what); this.#place = "ended"; return; } @@ -113,24 +120,24 @@ class ArchiveImport { await applyWrite(this.#step, part, what); } - /** The stored metadata once the archive ended: its `lastSeq`, next document id and the outbox setting. */ + /** The stored metadata once the archive ended: its `lastSeq`, the allocator's next id and the outbox setting. */ async finish(meta: MetaRow, outbox: boolean): Promise { if (this.#place !== "ended" || this.#step === undefined || this.#manifest === undefined) { throw invalid("args", "import: the archive has no end part"); } - const { lastSeq, nextDocumentId } = this.#manifest; + const { lastSeq, nextId } = this.#manifest; await writeCatalog(this.#step); await this.#executor.run( - `UPDATE ${TABLES.meta} SET last_seq = ?, next_document_id = ?, outbox = ?, outbox_floor = ? WHERE singleton = 1`, + `UPDATE ${TABLES.meta} SET last_seq = ?, next_id = ?, outbox = ?, outbox_floor = ? WHERE singleton = 1`, lastSeq, - nextDocumentId, + nextId, outbox ? 1 : 0, lastSeq, ); return { ...meta, last_seq: lastSeq, - next_document_id: nextDocumentId, + next_id: nextId, outbox: outbox ? 1 : 0, outbox_floor: lastSeq, }; @@ -145,7 +152,7 @@ class ArchiveImport { if (part.schema > STORE_SCHEMA) throw invalid("args", `${what}: schema ${part.schema} is newer than ${STORE_SCHEMA}`); this.#manifest = part; - this.#step = new Step(this.#executor, part.lastSeq, part.nextDocumentId, new Map()); + this.#step = new Step(this.#executor, part.lastSeq, new IdAllocator(part.nextId), new Map()); } /** Where `part` puts the archive, refused when out of order, a duplicate, or outside the manifest's ranges. */ @@ -171,21 +178,27 @@ class ArchiveImport { case "document.create": this.#checkDocument(part, what); return "documents"; + case "document.revision": + this.#checkRevision(part, what); + return "documents"; } } /** - * A record within the manifest's ids and seqs, unique, alone at its address, its base at its version; its address - * and meta well-formed text without NUL (SQLite would cut a NUL off, slice 6a §8). + * A record within the manifest's ids and seqs (an id equal to a saturated `nextId` too), unique, its lifetime + * overlapping no other at its address; its address and meta well-formed text without NUL (SQLite would cut a NUL off, + * slice 6a §8). The record before it must have ended well (`#endDocument`). */ #checkDocument(part: Extract, what: string): void { - const { record, base } = part; - const { lastSeq, nextDocumentId } = this.#manifest!; - if (record.id >= nextDocumentId) throw invalid("args", `${what}: id ${record.id} is not below ${nextDocumentId}`); + this.#endDocument(what); + const { record } = part; + const { lastSeq, nextId } = this.#manifest!; + const saturated = nextId === Number.MAX_SAFE_INTEGER && record.id === nextId; + if (record.id >= nextId && !saturated) throw invalid("args", `${what}: id ${record.id} is not below ${nextId}`); if (this.#ids.has(record.id)) throw invalid("args", `${what}: id ${record.id} occurs twice`); - if (record.createdAt > lastSeq) throw invalid("args", `${what}: created after the manifest's lastSeq`); - if (base.version !== record.version || base.fingerprint !== record.fingerprint) { - throw invalid("args", `${what}: the base is not at the record's version and fingerprint`); + const { createdAt, retiredAt } = record; + if (createdAt > lastSeq || (retiredAt !== undefined && (retiredAt < createdAt || retiredAt > lastSeq))) { + throw invalid("args", `${what}: a lifetime outside the manifest's lastSeq`); } const { namespace, owner, kind, meta } = record; // A singleton's record has no key; its column keeps `""`. @@ -196,10 +209,60 @@ class ArchiveImport { if (meta !== undefined && storedText(meta, what) !== meta) { throw invalid("args", `${what}: the meta is not well-formed text`); } - const address = JSON.stringify([namespace, owner, kind, key]); - if (this.#addresses.has(address)) throw invalid("args", `${what}: a second live record at one address`); + const address = JSON.stringify([owner, namespace, kind, key]); + const lifetime: [number, number] = [createdAt, retiredAt ?? Number.POSITIVE_INFINITY]; + const lifetimes = this.#lifetimes.get(address) ?? []; + // Two live records are two infinite lifetimes, which overlap; an empty one (`createdAt = retiredAt`) overlaps none. + if (lifetime[0] < lifetime[1] && lifetimes.some(([from, to]) => from < lifetime[1] && lifetime[0] < to)) { + throw invalid("args", `${what}: a second record at one address for an overlapping lifetime`); + } + lifetimes.push(lifetime); + this.#lifetimes.set(address, lifetimes); this.#ids.add(record.id); - this.#addresses.add(address); + this.#document = { record, seq: 0, base: undefined }; + } + + /** + * A revision of the record before it, in seq order within its lifetime, from a base, each delta at its base's + * version. + */ + #checkRevision(part: Extract, what: string): void { + const document = this.#document; + if (document === undefined || document.record.id !== part.id) { + throw invalid("args", `${what}: a revision follows its record's create`); + } + const { record } = document; + if (record.retiredAt !== undefined && record.history !== "rewindable") { + throw invalid("args", `${what}: a retired ${record.history} record keeps no revisions`); + } + if ( + part.seq <= document.seq || + part.seq < record.createdAt || + part.seq > (record.retiredAt ?? this.#manifest!.lastSeq) + ) { + throw invalid("args", `${what}: a revision out of seq order or outside its lifetime`); + } + const { content } = part; + if (content.kind === "base") document.base = content.version; + else if (document.base !== content.version) { + throw invalid("args", `${what}: a delta without a base at its version`); + } + document.seq = part.seq; + } + + /** + * The record before ends well: its last base is at the record's version; only a retired `latest` record has none + * (its content was dropped at retirement; a retired `rewindable` one keeps its history). + */ + #endDocument(what: string): void { + const document = this.#document; + this.#document = undefined; + if (document === undefined) return; + const { record, base } = document; + const contentless = record.retiredAt !== undefined && record.history === "latest"; + if (base === undefined ? !contentless : base !== record.version) { + throw invalid("args", `${what}: record ${record.id} has no base at its version`); + } } } diff --git a/packages/chord/src/next/store/outbox.ts b/packages/chord/src/next/store/outbox.ts index 642feba73..c9b1622e3 100644 --- a/packages/chord/src/next/store/outbox.ts +++ b/packages/chord/src/next/store/outbox.ts @@ -11,8 +11,8 @@ import type { StoreWrite } from "./writes.ts"; /** The write vocabulary's version an outbox commit is stored in; a newer one is refused, data intact. */ export const OUTBOX_FORMAT = 1; -/** One commit as the outbox gives it. */ -export type OutboxCommit = { readonly seq: number; readonly writes: readonly StoreWrite[] }; +/** One commit as the outbox gives it: its seq, the allocator's value it persisted (D3) and its writes in applied order. */ +export type OutboxCommit = { readonly seq: number; readonly nextId: number; readonly writes: readonly StoreWrite[] }; const NOTHING = (): void => {}; @@ -73,19 +73,24 @@ export class Outbox { } const rows = await this.#store.read( (executor) => - executor.all<{ readonly seq: number; readonly format: number; readonly writes: string }>( - `SELECT seq, format, writes FROM ${TABLES.outbox} WHERE seq > ? ORDER BY seq LIMIT ?`, + executor.all<{ + readonly seq: number; + readonly format: number; + readonly next_id: number; + readonly writes: string; + }>( + `SELECT seq, format, next_id, writes FROM ${TABLES.outbox} WHERE seq > ? ORDER BY seq LIMIT ?`, afterSeq, limit, ), context, ); - return rows.map(({ seq, format, writes }) => { + return rows.map(({ seq, format, next_id, writes }) => { if (format !== OUTBOX_FORMAT) { const message = `outbox: commit ${seq} is in format ${format}, newer than ${OUTBOX_FORMAT}`; throw new ChordError({ code: "unavailable", message }); } - return { seq, writes: JSON.parse(writes) as StoreWrite[] }; + return { seq, nextId: next_id, writes: JSON.parse(writes) as StoreWrite[] }; }); } diff --git a/packages/chord/src/next/store/refusal.ts b/packages/chord/src/next/store/refusal.ts new file mode 100644 index 000000000..2ccb3ce52 --- /dev/null +++ b/packages/chord/src/next/store/refusal.ts @@ -0,0 +1,43 @@ +// A refused commit as Chord's owners take it (slice 6b D4, §1.4): the store rejects a step it rolled back with the +// step's own error, unchanged (Durable's Storage semantics); Chord classifies it: an `int` column's `CHECK` (a +// plugin's value out of range) fails only its transaction, any other refusal means memory and disk disagree. + +import { ChordError, isChordError } from "../data/index.ts"; + +/** How a Chord commit ended that the store rejected: `check` fails the transaction only, `broken` its object too. */ +export type Refusal = + | { readonly kind: "check"; readonly error: ChordError } + | { readonly kind: "broken"; readonly error: ChordError }; + +/** SQLite's extended result code `SQLITE_CONSTRAINT_CHECK`, which Node's binding reports as `errcode`. */ +const SQLITE_CONSTRAINT_CHECK = 275; + +/** + * The refusal behind `error`, a store commit's rejection; `undefined` when it is the store's own (`released` at + * admission or after close, `unavailable/storage_failed` once stopped), which the owner rethrows as it is. + */ +export function refusalOf(error: unknown, label: string): Refusal | undefined { + if (isChordError(error, "released") || (isChordError(error, "unavailable") && error.reason === "storage_failed")) { + return undefined; + } + const message = error instanceof Error ? error.message : String(error); + if (isCheck(error, message)) { + const refused = `${label}: a value is out of its column's range (${message})`; + return { + kind: "check", + error: new ChordError({ code: "invalid", reason: "args", message: refused }, { cause: error }), + }; + } + const refused = `${label}: the store refused the commit (${message})`; + return { kind: "broken", error: new ChordError({ code: "unavailable", message: refused }, { cause: error }) }; +} + +/** + * Whether a refusal is a `CHECK` failure: SQLite's extended code where the binding has one (Node), the message + * otherwise (the Durable Object facade has no extended codes). + */ +function isCheck(error: unknown, message: string): boolean { + const code = (error as { readonly errcode?: unknown } | undefined)?.errcode; + if (typeof code === "number") return code === SQLITE_CONSTRAINT_CHECK; + return message.includes("CHECK constraint failed"); +} diff --git a/packages/chord/src/next/store/schema.ts b/packages/chord/src/next/store/schema.ts index c528360ce..249d177f7 100644 --- a/packages/chord/src/next/store/schema.ts +++ b/packages/chord/src/next/store/schema.ts @@ -1,6 +1,6 @@ // The store's own SQL schema (slice 6a §2): its metadata, the recorded table definitions, the documents engine's // records and revisions, and the outbox; versioned by `applySqliteMigrations`. Declared tables are named -// `"::
"`, so no name here can meet one of them. +// `":
"`, so no name here can meet one of them. import { applySqliteMigrations, type SqliteDatabase, type SqliteMigration } from "../../sqlite/index.ts"; @@ -14,10 +14,13 @@ export const TABLES = { } as const; /** - * The schema history; never change an entry, append one. `chord_meta` is one row: `last_seq`, the next document id - * (S10), and the outbox setting with its floor (the seq it starts after, advanced by `trim`). A document record keeps - * its current base's version and fingerprint and a `revision` counter for the conditional `UPDATE`; the partial - * unique index refuses two live records at one address. + * The schema history; never change an entry, append one (nothing is deployed yet: slice 6b changed entry 1 in place). + * `chord_meta` is one row: `last_seq`, the allocator's next id (D3; from 2, id 1 is Durable's root conversation), and + * the outbox setting with its floor (the seq it starts after, advanced by `trim`). A document record keeps its + * history rule (R1), its current base's version and fingerprint and a `revision` counter for the conditional + * `UPDATE`; the partial unique index refuses two live records at one address. Entry 2 adds the indexes of reads at a + * point (slice 6b §1.5): an address's incarnations by creation, an owner's namespace by id, a document's bases (a + * partial index: deltas write no entry). */ const MIGRATIONS: readonly SqliteMigration[] = [ { @@ -26,19 +29,19 @@ const MIGRATIONS: readonly SqliteMigration[] = [ `CREATE TABLE ${TABLES.meta} ( singleton INTEGER PRIMARY KEY CHECK (singleton = 1), last_seq INTEGER NOT NULL, - next_document_id INTEGER NOT NULL, + next_id INTEGER NOT NULL, outbox INTEGER NOT NULL CHECK (outbox IN (0, 1)), outbox_floor INTEGER NOT NULL ) STRICT`, - `INSERT INTO ${TABLES.meta} (singleton, last_seq, next_document_id, outbox, outbox_floor) VALUES (1, 0, 1, 0, 0)`, + `INSERT INTO ${TABLES.meta} (singleton, last_seq, next_id, outbox, outbox_floor) VALUES (1, 0, 2, 0, 0)`, `CREATE TABLE ${TABLES.tables} (name TEXT PRIMARY KEY, definition TEXT NOT NULL) STRICT, WITHOUT ROWID`, `CREATE TABLE ${TABLES.documents} ( id INTEGER PRIMARY KEY, - namespace TEXT NOT NULL, owner TEXT NOT NULL, + namespace TEXT NOT NULL, kind TEXT NOT NULL, key TEXT NOT NULL, - history TEXT NOT NULL CHECK (history IN ('latest', 'rewindable')), + history TEXT NOT NULL CHECK (history IN ('none', 'latest', 'rewindable')), created_at INTEGER NOT NULL, retired_at INTEGER, meta TEXT, @@ -46,7 +49,7 @@ const MIGRATIONS: readonly SqliteMigration[] = [ version INTEGER NOT NULL, fingerprint TEXT NOT NULL ) STRICT`, - `CREATE UNIQUE INDEX chord_documents_live ON ${TABLES.documents} (namespace, owner, kind, key) + `CREATE UNIQUE INDEX chord_documents_live ON ${TABLES.documents} (owner, namespace, kind, key) WHERE retired_at IS NULL`, `CREATE TABLE ${TABLES.revisions} ( document_id INTEGER NOT NULL, @@ -56,7 +59,21 @@ const MIGRATIONS: readonly SqliteMigration[] = [ content TEXT NOT NULL, PRIMARY KEY (document_id, seq) ) STRICT, WITHOUT ROWID`, - `CREATE TABLE ${TABLES.outbox} (seq INTEGER PRIMARY KEY, format INTEGER NOT NULL, writes TEXT NOT NULL) STRICT`, + `CREATE TABLE ${TABLES.outbox} ( + seq INTEGER PRIMARY KEY, + format INTEGER NOT NULL, + next_id INTEGER NOT NULL, + writes TEXT NOT NULL + ) STRICT`, + ], + }, + { + version: 2, + statements: [ + `CREATE INDEX chord_documents_address ON ${TABLES.documents} (owner, namespace, kind, key, created_at)`, + `CREATE INDEX chord_documents_scope ON ${TABLES.documents} (owner, namespace, id)`, + // Only bases are looked up by it (a load's newest base at or below a point): deltas write no entry. + `CREATE INDEX chord_revisions_bases ON ${TABLES.revisions} (document_id, seq) WHERE kind = 'base'`, ], }, ]; @@ -67,23 +84,44 @@ export const STORE_SCHEMA = MIGRATIONS.at(-1)!.version; /** The store's metadata row, as `openStore` reads it once. */ export type MetaRow = { readonly last_seq: number; - readonly next_document_id: number; + readonly next_id: number; readonly outbox: number; readonly outbox_floor: number; }; /** * Brings `db` to the store's newest schema (one transaction) and checks that it stores text as UTF-8 (the store - * compares text by its bytes); returns the metadata row. + * compares text by its bytes); returns the metadata row. A store already there is read in one statement, without the + * migrations' write transaction (most opens: the cost of a reopen). */ export async function applyStoreSchema(db: SqliteDatabase): Promise { + const current = await currentMeta(db); + if (current !== undefined) return current; const encoding = await db.get<{ readonly encoding: string }>("PRAGMA encoding"); if (encoding?.encoding !== "UTF-8") throw new Error(`store: the database encoding is ${encoding?.encoding}, not UTF-8`); await applySqliteMigrations(db, MIGRATIONS, { table: "chord_store_schema" }); const meta = await db.get( - `SELECT last_seq, next_document_id, outbox, outbox_floor FROM ${TABLES.meta} WHERE singleton = 1`, + `SELECT last_seq, next_id, outbox, outbox_floor FROM ${TABLES.meta} WHERE singleton = 1`, ); if (meta === undefined) throw new Error("store: the metadata row is missing"); return meta; } + +/** The schema version, the encoding and the metadata row of a store, in one read. */ +const CURRENT_META = `SELECT s.version, e.encoding, m.last_seq, m.next_id, m.outbox, m.outbox_floor + FROM chord_store_schema s, pragma_encoding e, ${TABLES.meta} m WHERE s.singleton = 1 AND m.singleton = 1`; + +/** The metadata row of a store at the newest schema in UTF-8; `undefined` for any other database. */ +async function currentMeta(db: SqliteDatabase): Promise { + let row: (MetaRow & { readonly version: number; readonly encoding: string }) | undefined; + try { + row = await db.get(CURRENT_META); + } catch { + // A database without the store's tables (new, or not a store) or a backend without `pragma_encoding`: the + // migrations' path creates, refuses or reads it. + return undefined; + } + if (row === undefined || row.version !== STORE_SCHEMA || row.encoding !== "UTF-8") return undefined; + return { last_seq: row.last_seq, next_id: row.next_id, outbox: row.outbox, outbox_floor: row.outbox_floor }; +} diff --git a/packages/chord/src/next/store/sql.ts b/packages/chord/src/next/store/sql.ts index 95529eb55..867b15bd5 100644 --- a/packages/chord/src/next/store/sql.ts +++ b/packages/chord/src/next/store/sql.ts @@ -19,13 +19,14 @@ export function quote(name: string): string { return `"${name}"`; } -/** The SQL name of an object's table: `::
`, injective since no part contains `:`. */ -export function tableSqlName(hostKind: string, objectType: string, table: string): string { - return `${hostKind}:${objectType}:${table}`; +/** The SQL name of an owner's table: `:
` (R2), injective since neither part contains `:`. */ +export function tableSqlName(owner: string, table: string): string { + return `${owner}:${table}`; } /** - * `CREATE TABLE` for `columns` (as recorded) and `key`: `STRICT`, `WITHOUT ROWID`, the key as primary key, an `int` + * `CREATE TABLE` for `columns` (as recorded) and `key`: `STRICT`, the key as primary key (`WITHOUT ROWID`, except + * for a single `int` key, which becomes the rowid: the fastest point lookups, and rows of any size), an `int` * column checked within ±(2^53 - 1) (rule 10), a `bool` one within 0 and 1, a `json(...)` one as `TEXT` with its * version column, and the row's `$v`. */ @@ -37,7 +38,8 @@ export function createTableSql( const definitions = Object.entries(columns).flatMap(([name, column]) => columnSql(name, column)); definitions.push(`${quote(ROW_VERSION)} INTEGER NOT NULL`); definitions.push(`PRIMARY KEY (${key.map(quote).join(", ")})`); - return `CREATE TABLE ${quote(table)} (${definitions.join(", ")}) STRICT, WITHOUT ROWID`; + const rowid = key.length === 1 && columns[key[0]!]!.type === "int"; + return `CREATE TABLE ${quote(table)} (${definitions.join(", ")}) STRICT${rowid ? "" : ", WITHOUT ROWID"}`; } const SAFE = Number.MAX_SAFE_INTEGER; @@ -114,7 +116,12 @@ export class TableSql { readonly get: string; readonly delete: string; - constructor(table: string, columns: readonly Column[], physical: readonly string[], key: readonly string[]) { + constructor( + table: string, + columns: readonly Pick[], + physical: readonly string[], + key: readonly string[], + ) { this.table = table; const id = quote(table); this.#select = [ @@ -127,6 +134,11 @@ export class TableSql { this.delete = `DELETE FROM ${id} WHERE ${this.#byKey}`; } + /** `SELECT` of `columns` of one row by key: a point lookup, without the `ORDER BY` and `LIMIT` of a page. */ + getColumns(columns: readonly string[]): string { + return `SELECT ${columns.map(quote).join(", ")} FROM ${quote(this.table)} WHERE ${this.#byKey}`; + } + /** `UPDATE` of one row by key; `versions` gives each `json(...)` column's version to stamp. */ update(set: SetNode, key: readonly SqliteValue[], versions: ReadonlyMap): Statement { const params: SqliteValue[] = []; @@ -151,13 +163,14 @@ export class TableSql { /** * A page: rows matching `where` in `order`, after the order values `after` (row-value comparison; the key in the - * order makes it total), at most `limit` (bound). + * order makes it total), at most `limit` (bound); every column, or only `columns`. */ select( where: WhereNode | undefined, order: Order, after: readonly SqliteValue[] | undefined, limit: number, + columns?: readonly string[], ): Statement { const params: SqliteValue[] = []; const conditions: string[] = []; @@ -171,7 +184,8 @@ export class TableSql { const direction = order.direction === "asc" ? "ASC" : "DESC"; const sorting = order.columns.map((column) => `${quote(column)} ${direction}`).join(", "); params.push(limit); - return { sql: `SELECT ${this.#select} FROM ${quote(this.table)}${filter} ORDER BY ${sorting} LIMIT ?`, params }; + const select = columns === undefined ? this.#select : columns.map(quote).join(", "); + return { sql: `SELECT ${select} FROM ${quote(this.table)}${filter} ORDER BY ${sorting} LIMIT ?`, params }; } /** `count(*)` of the rows matching `where`. */ diff --git a/packages/chord/src/next/store/step.ts b/packages/chord/src/next/store/step.ts index 8a428774a..57be88756 100644 --- a/packages/chord/src/next/store/step.ts +++ b/packages/chord/src/next/store/step.ts @@ -1,30 +1,38 @@ -// The commit step's working state (slice 6a §3) and the shape every write takes into it: a write staged before the -// queue (serialized, rule 8), applied inside the step in rule 9's order. +// The commit step's working state (slice 6a §3, slice 6b §1.3) and the shape every write takes into it: a write staged +// before the queue (serialized, rule 8), applied inside the step, returning the writes it recorded. import type { SqliteExecutor, SqliteValue } from "../../sqlite/index.ts"; +import type { IdAllocator } from "./allocator.ts"; import type { TableRecord } from "./catalog.ts"; import type { StoreWrite } from "./writes.ts"; -/** Where a staged write goes in its commit (rule 9): retirements, creations, then everything else in order. */ +/** + * Where a staged write goes in its commit: retirements, creations, then everything else, each phase in staging order. + * A retirement frees a live address before a creation takes it (the partial unique index); an adapter's composite + * write orders its own parts inside `write`. + */ export type WriteOrder = "retire" | "create" | "write"; -/** A write staged for the commit step: its order and what it does there, returning the write as recorded. */ +/** A write staged for the commit step: its order and what it does there, returning the writes as recorded. */ export type StagedWrite = { readonly order: WriteOrder; - /** Applies the write inside the step's backend transaction; resolves with the write as the outbox records it. */ - apply(step: Step): Promise; + /** + * Applies the write inside the step's backend transaction; resolves with the writes as the outbox records them, in + * the order applied, with what only the step knows resolved (its seq, a record's revision). + */ + apply(step: Step): Promise; }; /** - * One commit step: its backend transaction, its seq, the document ids it assigns (S10, monotonic with seq), the table - * records it changed (the store's catalog takes them after `COMMIT`) and the statements it ran - * (`chord.store.statements`). + * One commit step: its backend transaction, its seq, the store's id allocator (D3: ids are taken from it as the + * writes apply, never copied), the ids of the documents it created, the table records it changed (the store's catalog + * takes them after `COMMIT`) and the statements it ran (`chord.store.statements`). Its reads are the step reader of + * adapters: they see the step's own writes. */ export class Step { readonly executor: SqliteExecutor; readonly seq: number; - /** The next document id; the store adopts it after `COMMIT`. */ - nextDocumentId: number; + readonly ids: IdAllocator; /** The ids of the commit's created documents, in the order of their writes. */ readonly created: number[] = []; /** The table records this step changed, by SQL name. */ @@ -32,15 +40,10 @@ export class Step { readonly #catalog: ReadonlyMap; statements = 0; - constructor( - executor: SqliteExecutor, - seq: number, - nextDocumentId: number, - catalog: ReadonlyMap, - ) { + constructor(executor: SqliteExecutor, seq: number, ids: IdAllocator, catalog: ReadonlyMap) { this.executor = executor; this.seq = seq; - this.nextDocumentId = nextDocumentId; + this.ids = ids; this.#catalog = catalog; } @@ -61,6 +64,12 @@ export class Step { return this.executor.runCounted(sql, ...params); } + /** Runs one query for its first row. */ + get(sql: string, ...params: SqliteValue[]): Promise { + this.statements++; + return this.executor.get(sql, ...params); + } + /** Runs one query. */ all(sql: string, ...params: SqliteValue[]): Promise { this.statements++; diff --git a/packages/chord/src/next/store/store.ts b/packages/chord/src/next/store/store.ts index f24f2d51e..cde65fea2 100644 --- a/packages/chord/src/next/store/store.ts +++ b/packages/chord/src/next/store/store.ts @@ -1,13 +1,19 @@ -// The store (slice 6a §2, §3): opened over a backend, one commit queue that serializes only the commit step (the seal -// fence's admission check, `BEGIN`, the writes in rule 9's order, the outbox commit, `lastSeq`, `COMMIT`, then the -// owner's adoption), reads guarded as Durable's `GuardedStorage` guards them, and the stop after a backend error. +// The store (slice 6a §2, §3, slice 6b §1): opened over a backend, one commit queue that serializes only the commit step +// (the seal fence's admission check, `BEGIN`, the writes in their order, the outbox commit, `lastSeq` and the +// allocator's value, `COMMIT`, then the owner's adoption), the id allocator, the claims of its tables and documents, +// reads guarded as Durable's `GuardedStorage` guards them, Durable's Storage semantics for a refused step (D4), and the +// stop after an unknown outcome. import { awaitWithContext, type Context } from "../../context/index.ts"; import type { SqliteDatabase, SqliteExecutor } from "../../sqlite/index.ts"; import { ChordError } from "../data/index.ts"; +import { invalid } from "../data/internal.ts"; +import type { Batch } from "../state/internal.ts"; +import { IdAllocator } from "./allocator.ts"; import type { ArchivePart } from "./archive.ts"; import type { Backend } from "./backend.ts"; import { readCatalog, type TableRecord, writeCatalog } from "./catalog.ts"; +import { Claims } from "./claims.ts"; import { StoreExport } from "./export.ts"; import { OUTBOX_FORMAT, Outbox } from "./outbox.ts"; import { applyStoreSchema, type MetaRow, TABLES } from "./schema.ts"; @@ -81,6 +87,19 @@ export type CommitFence = { adopt(seq: number, ids: readonly number[]): void; }; +/** What the open host gives the store: its `storage_failed` listener and its batch, in which adapters adopt (§1.3). */ +export type Attachment = { + readonly failed: (error: unknown) => void; + readonly batch: Batch; +}; + +/** Where a replayed batch leaves the store: its last commit's seq and allocator value, the tables it changed. */ +export type Replayed = { + readonly lastSeq: number; + readonly nextId: number; + readonly tables: ReadonlyMap; +}; + /** Where a store stands: open, stopped by a backend error, or closed. */ type StoreState = | { readonly kind: "open" } @@ -91,19 +110,30 @@ const ORDER: readonly WriteOrder[] = ["retire", "create", "write"]; const NOTHING = (): void => {}; +/** The step's own error, until its callback throws one: a distinct value no thrown error can be. */ +const NONE: unique symbol = Symbol("none"); + /** The store behind `Store`, with what Chord's layers use: commits, guarded reads, the host's failure listener. */ export class ChordStore implements Store { readonly #db: SqliteDatabase; #lastSeq: number; - #nextDocumentId: number; + /** The allocator's value as the last commit persisted it (an export's manifest). */ + #committedNextId: number; + readonly #ids: IdAllocator; + /** The claims of its tables and documents (slice 6b). */ + readonly claims = new Claims(); #outbox: Outbox | undefined; #state: StoreState = { kind: "open" }; + /** Set once a batch's recorded count did not reproduce (S17): no batch applies any more. */ + #diverged = false; /** The commit queue's tail: each step starts after the one before it settled. */ #tail: Promise = Promise.resolve(); - /** Reads underway, which `close` waits for. */ - readonly #underway = new Set>(); - /** The open host's listener for `storage_failed`, told once; one host per store (rule 7). */ - #listener: ((error: unknown) => void) | undefined; + /** Reads and maintenance writes underway, which `close` waits for. */ + #underway = 0; + /** Resolves `close`'s wait once no read is underway. */ + #drained: (() => void) | undefined; + /** The open host's listener for `storage_failed` (told once) and batch; one host per store (rule 7). */ + #attachment: Attachment | undefined; #closing: Promise | undefined; /** The recorded tables, by SQL name; changed only after a step's `COMMIT`. */ readonly #catalog: Map; @@ -114,7 +144,8 @@ export class ChordStore implements Store { this.#db = db; this.#catalog = catalog; this.#lastSeq = meta.last_seq; - this.#nextDocumentId = meta.next_document_id; + this.#committedNextId = meta.next_id; + this.#ids = new IdAllocator(meta.next_id); this.#outbox = meta.outbox === 1 ? new Outbox(this, meta.outbox_floor) : undefined; } @@ -126,9 +157,25 @@ export class ChordStore implements Store { return this.#outbox; } - /** The id the next created document gets. */ - get nextDocumentId(): number { - return this.#nextDocumentId; + /** The allocator's value as the last commit persisted it. */ + get nextId(): number { + return this.#committedNextId; + } + + /** The id allocator: `mint` takes the next id, `raise` makes room above an explicit one (D3). */ + get ids(): IdAllocator { + return this.#ids; + } + + /** Takes the next id from the allocator (Durable's `mintId`); refused once the store stopped or closed. */ + mintId(): number { + this.assertUsable(); + return this.#ids.mint(); + } + + /** The open host's batch, in which an adapter adopts its commits; `undefined` without a host. */ + get batch(): Batch | undefined { + return this.#attachment?.batch; } /** Every recorded table, by SQL name, as of the last commit. */ @@ -170,24 +217,26 @@ export class ChordStore implements Store { return this.#catalog.get(name); } - /** Attaches the open host's `storage_failed` listener; a store attached to an open host is refused (rule 7). */ - attach(listener: (error: unknown) => void): void { - if (this.#listener !== undefined) { + /** Attaches the open host; a store attached to an open host is refused (rule 7). */ + attach(attachment: Attachment): void { + if (this.#attachment !== undefined) { throw new ChordError({ code: "invalid", reason: "args", message: "store: already attached to an open host" }); } - this.#listener = listener; + this.#attachment = attachment; } /** The host closed: the store may serve another one. */ detach(): void { - this.#listener = undefined; + this.#attachment = undefined; } /** * Commits `writes` as one seq when the commit queue reaches it: `fence.admit()` at the step's entry, the writes in - * rule 9's order (retirements, creations, the rest), the outbox commit, `lastSeq`, `COMMIT`, then `fence.adopt` - * before the next step starts. Resolves with the seq. A backend error or a throwing `adopt` stops the store - * (`storage_failed`); a refused admission rejects `released`. `context` counts the step (no abort once queued). + * their order (retirements, creations, the rest), the outbox commit, `lastSeq` and the allocator's value, `COMMIT`, + * then `fence.adopt` before the next step starts. Resolves with the seq. A refused admission rejects `released`. A + * step the backend rolled back after the step's own code threw (a refusal, D4) rejects with that error unchanged and + * the store goes on; any other failure (`BEGIN`, `COMMIT`, `ROLLBACK`, a throwing `adopt`) is an unknown outcome + * that stops the store (`storage_failed`). `context` counts the step (no abort once queued). */ commit(writes: readonly StagedWrite[], fence: CommitFence, context: Context): Promise { try { @@ -200,18 +249,48 @@ export class ChordStore implements Store { return step; } + /** + * An adapter's commit (§1.3): `commit` admitting every write, its adoption run in the attached host's batch + * (`batch.within`: it joins a running batch or opens one), plainly without a host. Durable's `adopt` publishes inside + * it, so Chord sources over Durable's views are marked in the step. + */ + commitInBatch(writes: readonly StagedWrite[], adopt: (seq: number) => void, context: Context): Promise { + const fence: CommitFence = { + admit: () => true, + adopt: (seq) => { + const batch = this.#attachment?.batch; + if (batch === undefined) adopt(seq); + else batch.within(() => adopt(seq), context); + }, + }; + return this.commit(writes, fence, context); + } + /** * Runs `read` on the backend under the guard (rule 11, Durable's `GuardedStorage`): refused once the store stopped * or closed; a backend error fails the store unless `context` was aborted (then `aborted`); a read settling after - * the store stopped rejects `unavailable/storage_failed`. + * the store stopped rejects `unavailable/storage_failed`. `read` returns data and never throws for a request: an + * adapter checks a request from what it returned, after the guard (slice 6b §1.4). */ - async read(read: (executor: SqliteExecutor) => Promise, context: Context): Promise { + read(read: (executor: SqliteExecutor) => Promise, context: Context): Promise { + return this.#guarded(() => read(this.#db), context); + } + + /** + * `read` for several queries that must see one state (S4, a document from its record and revisions): one backend + * transaction, guarded as `read`. + */ + readTransaction(read: (executor: SqliteExecutor) => Promise, context: Context): Promise { + return this.#guarded(() => this.#db.transaction(read), context); + } + + async #guarded(run: () => Promise, context: Context): Promise { this.assertUsable(); - const call = (async () => read(this.#db))(); - this.#underway.add(call); + this.#underway++; let result: T; try { - result = await call; + // A read throwing before its first await is the backend's too: inside the `try`. + result = await run(); } catch (error) { this.assertUsable(); const signal = context.abortSignal; @@ -219,27 +298,31 @@ export class ChordStore implements Store { throw new ChordError({ code: "aborted", message: "store: read aborted" }, { cause: signal.reason }); throw this.#fail(error); } finally { - this.#underway.delete(call); + this.#settle(); } this.assertUsable(); return result; } + /** One read or maintenance write settled: `close` goes on once none is underway. */ + #settle(): void { + if (--this.#underway === 0) this.#drained?.(); + } + /** * Runs `write` on the backend outside the commit queue (the outbox's `trim`), in a backend transaction, guarded as * a read without a context: any error stops the store. */ async maintain(write: (executor: SqliteExecutor) => Promise): Promise { this.assertUsable(); - const call = this.#db.transaction(write); - this.#underway.add(call); + this.#underway++; try { - await call; + await this.#db.transaction(write); } catch (error) { this.assertUsable(); throw this.#fail(error); } finally { - this.#underway.delete(call); + this.#settle(); } } @@ -252,7 +335,11 @@ export class ChordStore implements Store { this.#state = { kind: "closed" }; for (const end of [...this.#held]) end(); await this.#tail; - await Promise.allSettled([...this.#underway]); + if (this.#underway > 0) { + await new Promise((resolve) => { + this.#drained = resolve; + }); + } await this.#db.close(); } @@ -273,41 +360,26 @@ export class ChordStore implements Store { const seq = this.#lastSeq + 1; // Past the safe range seqs stop counting exactly (a later one would repeat): the store cannot number further. if (seq > Number.MAX_SAFE_INTEGER) throw this.#fail(new Error("store: no seq past Number.MAX_SAFE_INTEGER")); - const outbox = this.#outbox !== undefined; - let step: Step; + let thrown: unknown = NONE; + let done: { readonly step: Step; readonly nextId: number }; try { - step = await this.#db.transaction(async (executor) => { - const step = new Step(executor, seq, this.#nextDocumentId, this.#catalog); - const recorded: StoreWrite[] | undefined = outbox ? [] : undefined; - for (const order of ORDER) { - for (const write of writes) { - if (write.order !== order) continue; - const done = await write.apply(step); - recorded?.push(done); - } + done = await this.#db.transaction(async (executor) => { + try { + return await this.#write(executor, seq, writes); + } catch (error) { + thrown = error; + throw error; } - await writeCatalog(step); - if (recorded !== undefined) { - await step.run( - `INSERT INTO ${TABLES.outbox} (seq, format, writes) VALUES (?, ?, ?)`, - seq, - OUTBOX_FORMAT, - JSON.stringify(recorded), - ); - } - await step.run( - `UPDATE ${TABLES.meta} SET last_seq = ?, next_document_id = ? WHERE singleton = 1`, - seq, - step.nextDocumentId, - ); - return step; }); } catch (error) { + // The facade rolled back and rejects with the step's own error: a refusal, the store goes on (D4). + if (error === thrown) throw error; throw this.#fail(error); } // `COMMIT` returned: the commit is durable, and memory follows. + const { step, nextId } = done; this.#lastSeq = seq; - this.#nextDocumentId = step.nextDocumentId; + this.#committedNextId = nextId; for (const [name, record] of step.tables) this.#catalog.set(name, record); const metrics = context.telemetry.metrics; metrics?.counter("chord.store.commits").add(1); @@ -320,6 +392,112 @@ export class ChordStore implements Store { return seq; } + /** + * The step's writes inside its backend transaction: each staged write in its phase, the catalog, the outbox commit, + * the metadata. The allocator's value is taken once, after the writes applied (mints meanwhile only raise it). + */ + async #write( + executor: SqliteExecutor, + seq: number, + writes: readonly StagedWrite[], + ): Promise<{ readonly step: Step; readonly nextId: number }> { + const step = new Step(executor, seq, this.#ids, this.#catalog); + const recorded: StoreWrite[] | undefined = this.#outbox !== undefined ? [] : undefined; + for (const order of ORDER) { + for (const write of writes) { + if (write.order !== order) continue; + const done = await write.apply(step); + recorded?.push(...done); + } + } + await writeCatalog(step); + const nextId = this.#ids.value; + if (recorded !== undefined) { + await step.run( + `INSERT INTO ${TABLES.outbox} (seq, format, next_id, writes) VALUES (?, ?, ?, ?)`, + seq, + OUTBOX_FORMAT, + nextId, + JSON.stringify(recorded), + ); + } + await step.run(`UPDATE ${TABLES.meta} SET last_seq = ?, next_id = ? WHERE singleton = 1`, seq, nextId); + return { step, nextId }; + } + + /** + * Batch apply's turn (slice 6b §1.6, S5): refused (`invalid/args`) while an owner holds a claim (its memory would + * go stale) and once the store diverged; otherwise `apply` runs in the commit queue, in one backend transaction, and + * returns the position its last commit leaves (`lastSeq`, the allocator's value) and the tables it changed, which + * memory takes after `COMMIT`. `apply`'s own error rolls back and rejects as a refusal (a `Diverged` one marks the + * store diverged); any other failure stops the store. + */ + replay( + apply: (executor: SqliteExecutor, catalog: ReadonlyMap) => Promise, + isDivergence: (error: unknown) => boolean, + ): Promise { + try { + this.#assertReplayable(); + } catch (error) { + return Promise.reject(error); + } + const turn = this.#tail.then(() => this.#replay(apply, isDivergence)); + this.#tail = turn.then(NOTHING, NOTHING); + return turn; + } + + /** At its call and again at its turn: a claim or a divergence may come between the two. */ + #assertReplayable(): void { + this.assertUsable(); + if (this.#diverged) throw invalid("args", "store: diverged; a batch can no longer apply"); + if (this.claims.held) throw invalid("args", "store: an owner holds claims; a batch would stale its memory"); + } + + async #replay( + apply: (executor: SqliteExecutor, catalog: ReadonlyMap) => Promise, + isDivergence: (error: unknown) => boolean, + ): Promise { + this.#assertReplayable(); + // While it applies, no owner may claim (and read) what it is changing. + const unlock = this.claims.lock(); + try { + await this.#applyReplay(apply, isDivergence); + } finally { + unlock(); + } + } + + async #applyReplay( + apply: (executor: SqliteExecutor, catalog: ReadonlyMap) => Promise, + isDivergence: (error: unknown) => boolean, + ): Promise { + let thrown: unknown = NONE; + let replayed: Replayed; + try { + replayed = await this.#db.transaction(async (executor) => { + try { + return await apply(executor, this.#catalog); + } catch (error) { + thrown = error; + throw error; + } + }); + } catch (error) { + if (error !== thrown) throw this.#fail(error); + if (isDivergence(error)) this.#diverged = true; + throw error; + } + this.#lastSeq = replayed.lastSeq; + this.#committedNextId = replayed.nextId; + this.#ids.raise(replayed.nextId - 1); + for (const [name, record] of replayed.tables) this.#catalog.set(name, record); + } + + /** Whether the store has an outbox, which a replayed batch writes too. */ + get recordsOutbox(): boolean { + return this.#outbox !== undefined; + } + /** Throws `unavailable/storage_failed` once the store stopped, `released` once it closed. */ assertUsable(): void { const state = this.#state; @@ -332,7 +510,7 @@ export class ChordStore implements Store { #fail(error: unknown): ChordError { if (this.#state.kind === "open") { this.#state = { kind: "failed", error }; - this.#listener?.(error); + this.#attachment?.failed(error); } return storageFailed(error); } diff --git a/packages/chord/src/next/store/table-writes.ts b/packages/chord/src/next/store/table-writes.ts index 101c7f8b3..b517b166b 100644 --- a/packages/chord/src/next/store/table-writes.ts +++ b/packages/chord/src/next/store/table-writes.ts @@ -86,12 +86,12 @@ export type TableDefinitionWrite = Extract applyTable(step, write) }; + return { order: "write", apply: async (step) => [await applyTable(step, write)] }; } /** Stages an `index` write. */ export function stageIndex(write: Extract): StagedWrite { - return { order: "write", apply: (step) => applyIndex(step, write) }; + return { order: "write", apply: async (step) => [await applyIndex(step, write)] }; } /** Stages one edit of the migration of `table` to `version` as its `alter`, `index` or `dropIndex` write. */ @@ -101,11 +101,11 @@ export function stageEdit(table: string, version: number, edit: TableEdit): Stag return stageIndex({ kind: "index", table, name: edit.name, columns: edit.columns }); case "dropIndex": { const write = { kind: "dropIndex", table, name: edit.name } as const; - return { order: "write", apply: (step) => applyDropIndex(step, write) }; + return { order: "write", apply: async (step) => [await applyDropIndex(step, write)] }; } default: { const write = { kind: "alter", table, version, edit } as const; - return { order: "write", apply: (step) => applyAlter(step, write) }; + return { order: "write", apply: async (step) => [await applyAlter(step, write)] }; } } } @@ -117,7 +117,7 @@ export function stagePut(sql: string, table: string, row: EncodedRow, version: n order: "write", apply: async (step) => { await step.run(sql, ...row.params); - return write; + return [write]; }, }; } @@ -167,7 +167,7 @@ export function stageDeleteWhere(statement: Statement, table: string, where: Whe function counted(statement: Statement, record: (count: number) => TableWrite): StagedWrite { return { order: "write", - apply: async (step) => record(await step.runCounted(statement.sql, ...statement.params)), + apply: async (step) => [record(await step.runCounted(statement.sql, ...statement.params))], }; } diff --git a/packages/chord/test/next/bench/jobs.ts b/packages/chord/test/next/bench/jobs.ts new file mode 100644 index 000000000..87cce72d0 --- /dev/null +++ b/packages/chord/test/next/bench/jobs.ts @@ -0,0 +1,164 @@ +// Benchmark of persisted jobs (slice 6b §4, §5): a run's start (its record and state created, resolved after the +// commit) and a state change inside `run` (immediate, and deferred), for a persisted job on a SQLite file against the +// same job in memory. Median and p95 per operation over several bursts. A disk commit is not held to the in-memory +// number: the persisted columns show what a run's records cost on the store. +// Run by hand from packages/chord (not in CI): +// node --conditions=source --experimental-strip-types --expose-gc test/next/bench/jobs.ts +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { performance } from "node:perf_hooks"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { defineJob, type JobHandle, type RunState } from "../../../src/next/jobs/index.ts"; +import { openStore, sqliteBackend } from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const OPERATIONS = 500; +const RUNS = 7; + +type Progress = { done: number }; + +const InMemoryProgress = defineState("bench.progress", { + schema: obj({ done: int() }), + version: 1, + initial: () => ({ done: 0 }), +}); + +const PersistedProgress = defineState("bench.progress", { + schema: obj({ done: int() }), + version: 1, + initial: () => ({ done: 0 }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 64, +}); + +const InMemoryImport = defineJob("bench.import", { + version: 1, + args: obj({ path: str() }), + result: obj({ count: int() }), + state: InMemoryProgress, +}); + +const PersistedImport = defineJob("bench.import", { + version: 1, + args: obj({ path: str() }), + result: obj({ count: int() }), + state: PersistedProgress, + persist: { resume: "rerun" }, +}); + +const Worker = local("bench.worker", { version: 1, params: {}, members: {} }); + +/** Where a run's `run` parks: its state, and the gate that ends it. */ +type Parked = { readonly state: RunState; readonly end: () => void }; + +/** A host serving `Worker` with the job (persisted on a fresh SQLite file, or in memory), its runs parked. */ +async function served(persisted: boolean) { + const directory = await mkdtemp(join(tmpdir(), "chord-bench-jobs-")); + const store = persisted + ? await openStore(sqliteBackend(await openNodeSqliteDatabase(join(directory, "store.sqlite"))), context) + : undefined; + const parked: Parked[] = []; + const waiting: ((parked: Parked) => void)[] = []; + let jobs: JobHandle<{ path: string }, { count: number }, Progress> | undefined; + const host = createHost({ store, onError: () => {} }); + const facet = defineFacet((h) => + h.serve(Worker, (_, self) => { + jobs = self.job(persisted ? PersistedImport : InMemoryImport, { + run: (_args, ctx) => + new Promise((resolve) => { + const run = { state: ctx.state, end: () => resolve({ count: ctx.state.value.done }) }; + const next = waiting.shift(); + if (next === undefined) parked.push(run); + else next(run); + }), + }); + return {}; + }), + ); + await host.load([{ name: "@bench/jobs", facet }], context); + const reference = await host.get(Worker, {}, context); + return { + jobs: jobs!, + /** The parked `run` of the next run invoked. */ + next: () => new Promise((resolve) => waiting.push(resolve)), + close: async () => { + reference.release(); + await host.close(context); + await store?.close(context); + await rm(directory, { recursive: true, force: true }); + }, + }; +} + +/** Per-operation times (ms) of `RUNS` bursts, after one warm-up burst; `burst` does `OPERATIONS` timed operations. */ +async function bursts( + persisted: boolean, + burst: ( + worker: Awaited>, + time: (operation: () => Promise) => Promise, + ) => Promise, +) { + const times: number[] = []; + for (let run = -1; run < RUNS; run++) { + const worker = await served(persisted); + await burst(worker, async (operation) => { + const before = performance.now(); + await operation(); + if (run >= 0) times.push(performance.now() - before); + }); + await worker.close(); + } + times.sort((a, b) => a - b); + return { median: times[Math.floor(times.length / 2)]!, p95: times[Math.floor(times.length * 0.95)]! }; +} + +/** Starts: each run's start resolved (record and state committed), its `run` parked, all ended after the burst. */ +function starts(persisted: boolean) { + return bursts(persisted, async ({ jobs, next }, time) => { + const parked: Parked[] = []; + for (let index = 0; index < OPERATIONS; index++) { + const invoked = next(); + await time(() => jobs.start({ path: `p${index}` }, context)); + parked.push(await invoked); + } + for (const run of parked) run.end(); + }); +} + +/** State changes: `OPERATIONS` changes of one run's state, immediate or deferred (stored by the outcome). */ +function changes(persisted: boolean, deferred: boolean) { + const options = deferred ? { persistence: { kind: "deferred", withinMs: Infinity } as const } : undefined; + return bursts(persisted, async ({ jobs, next }, time) => { + const invoked = next(); + await jobs.start({ path: "p" }, context); + const { state, end } = await invoked; + for (let index = 0; index < OPERATIONS; index++) { + await time(() => + state.change( + (draft) => { + draft.done = index + 1; + }, + context, + options, + ), + ); + } + end(); + }); +} + +function format({ median, p95 }: { readonly median: number; readonly p95: number }): string { + return `median ${(median * 1000).toFixed(1)} µs, p95 ${(p95 * 1000).toFixed(1)} µs`; +} + +console.log(`start, ${RUNS} bursts of ${OPERATIONS}:`); +console.log(` in memory: ${format(await starts(false))}`); +console.log(` persisted: ${format(await starts(true))}`); +console.log(`state change, ${RUNS} bursts of ${OPERATIONS}:`); +console.log(` in memory: ${format(await changes(false, false))}`); +console.log(` persisted, immediate: ${format(await changes(true, false))}`); +console.log(` persisted, deferred: ${format(await changes(true, true))}`); diff --git a/packages/chord/test/next/bench/store.ts b/packages/chord/test/next/bench/store.ts index ce2923de9..e4aa4392c 100644 --- a/packages/chord/test/next/bench/store.ts +++ b/packages/chord/test/next/bench/store.ts @@ -78,7 +78,7 @@ function chordDeltas(outbox: boolean) { const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context, { outbox, })) as ChordStore; - const address = { namespace: "chord", owner: "session:bench.store", kind: "note", key: "" }; + const address = { namespace: "chord", owner: "bench.store", kind: "note", key: "" }; const base = { version: 1, fingerprint: "0123456789abcdef", value: VALUE }; let id = 0; const fence = { @@ -87,7 +87,7 @@ function chordDeltas(outbox: boolean) { id = ids[0] ?? id; }, }; - await store.commit([createDocument(address, "latest", base, JSON.stringify(VALUE))], fence, context); + await store.commit([createDocument(address, base, JSON.stringify(VALUE))], fence, context); return { commit: (index: number) => { const ops = [["s", ["n"], index + 1]] as const; @@ -149,7 +149,7 @@ function chordPuts(outbox: boolean) { const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context, { outbox, })) as ChordStore; - const table = tableSqlName("session", "bench.store", Edits.name); + const table = tableSqlName("bench.store", Edits.name); const sql = new TableSql(table, Edits.columns, Edits.rows.physical, Edits.key); const fence = { admit: () => true, adopt: () => {} }; const { version, record: columns, key, fingerprint } = Edits; @@ -329,7 +329,7 @@ console.log(`heap after GC: ${heap() - before} KiB above the start`); async function filledEdits(count: number, body = VALUE.body, definition: typeof Edits = Edits) { const { path, remove } = await file(); const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; - const table = tableSqlName("session", "bench.store", Edits.name); + const table = tableSqlName("bench.store", Edits.name); const fence = { admit: () => true, adopt: () => {} }; await prepareTables([{ definition, table }], store, fence, "bench", context); const handle = new TableHandle(definition, table, store, { diff --git a/packages/chord/test/next/documents/persistence.test.ts b/packages/chord/test/next/documents/persistence.test.ts index 6b97b83a2..13fe83ea8 100644 --- a/packages/chord/test/next/documents/persistence.test.ts +++ b/packages/chord/test/next/documents/persistence.test.ts @@ -27,7 +27,7 @@ let notes: Persistence; beforeEach(async () => { store = await memoryStore({ outbox: true }); - notes = new Persistence(store, "session:app.store", "notes.note"); + notes = new Persistence(store, "app.store", "chord", "notes.note"); }); afterEach(async () => { @@ -47,20 +47,10 @@ async function change( after(draft.state); const prepared = draft.prepare(); const writes: StagedWrite[] = []; - const result = notes.stage(key, definition, stored, { prepared, retires: false }, writes, 0); - let ids: readonly number[] = []; - await store.commit( - writes, - { - admit: () => true, - adopt: (_seq, created) => { - ids = created; - }, - }, - context, - ); + const result = notes.stage(key, definition, stored, { prepared, retires: false }, writes); + await store.commit(writes, { admit: () => true, adopt: () => {} }, context); tracker.adopt(prepared); - return storedAfter(result, ids); + return storedAfter(result); } /** The document at `key` read from the store and decoded by `definition`, `undefined` when absent. */ @@ -259,12 +249,12 @@ describe("staged writes", () => { draft.state.n = n; return draft.prepare(); }; - const first = notes.stage("b", Note, undefined, { prepared: fresh(2), retires: false }, writes, 0); - const again = notes.stage("a", Note, stored, { prepared: fresh(3), retires: true }, writes, 1); - const gone = notes.stage("c", Note, undefined, { prepared: undefined, retires: true }, writes, 2); - expect([first, again, gone]).toEqual([ - { kind: "create", version: 1, index: 0 }, - { kind: "create", version: 1, index: 1 }, + const first = notes.stage("b", Note, undefined, { prepared: fresh(2), retires: false }, writes); + const again = notes.stage("a", Note, stored, { prepared: fresh(3), retires: true }, writes); + const gone = notes.stage("c", Note, undefined, { prepared: undefined, retires: true }, writes); + expect([first, again, gone]).toMatchObject([ + { kind: "create", version: 1 }, + { kind: "create", version: 1 }, { kind: "retire" }, ]); let ids: readonly number[] = []; @@ -279,7 +269,7 @@ describe("staged writes", () => { context, ); expect(ids).toEqual([stored!.id + 1, stored!.id + 2]); - expect(storedAfter(again, ids)).toEqual({ id: stored!.id + 2, revision: 1, version: 1, deltasSinceBase: 0 }); + expect(storedAfter(again)).toEqual({ id: stored!.id + 2, revision: 1, version: 1, deltasSinceBase: 0 }); expect((await outboxKinds()).at(-1)).toEqual(["document.retire", "document.create", "document.create"]); }); @@ -287,6 +277,6 @@ describe("staged writes", () => { // Drafts refuse such values when placed; a prepared change made by hand shows where the text is made. const value = { title: "", n: 1n } as unknown as JsonObject; const prepared = { base: value, value, ops: [], baseRevision: 0, abort: () => {} }; - expect(() => notes.stage("a", Note, undefined, { prepared, retires: false }, [], 0)).toThrow(TypeError); + expect(() => notes.stage("a", Note, undefined, { prepared, retires: false }, [])).toThrow(TypeError); }); }); diff --git a/packages/chord/test/next/documents/transaction.test.ts b/packages/chord/test/next/documents/transaction.test.ts index ff75ea375..e26052957 100644 --- a/packages/chord/test/next/documents/transaction.test.ts +++ b/packages/chord/test/next/documents/transaction.test.ts @@ -435,7 +435,7 @@ describe("tables in a transaction", () => { order: "write", apply: async () => { await gate.promise; - return { kind: "dropIndex", table: "x", name: "y" }; + return [{ kind: "dropIndex", table: "x", name: "y" }]; }, }, ], @@ -533,6 +533,22 @@ describe("persisted documents in a transaction", () => { await store.close(context); }); + it("a keyed retirement takes 6a's statements: the counted DELETE, the revisions, lastSeq (cost guard)", async () => { + const { telemetry, context: counting, counts } = await metered(); + const { self, k, store, host } = await stored(telemetry); + await self.tx(async (tx) => { + (await tx.draft(k, "a")).n = 1; + }, counting); + const before = { ...counts() }; + await self.tx(async (tx) => { + await tx.retire(k, "a"); + }, counting); + // No read of the record first, no second DELETE of its row; plus the outbox commit. + expect(counts()["chord.store.statements"]! - before["chord.store.statements"]!).toBe(4); + await host.close(context); + await store.close(context); + }); + it("assign the created documents' ids by their order in the commit; retire and recreate in one commit", async () => { const { self, k, store, host } = await stored(); await self.tx(async (tx) => { @@ -548,9 +564,9 @@ describe("persisted documents in a transaction", () => { writes.flatMap((write) => (write.kind === "document.create" ? [[write.record.key, write.record.id]] : [])), ); expect(created).toEqual([ - ["b", 1], - ["a", 2], - ["b", 3], + ["b", 2], + ["a", 3], + ["b", 4], ]); expect(commits[1]!.writes.map(({ kind }) => kind)).toEqual(["document.retire", "document.create"]); // What is stored of `b` is the new record's: the next change is a delta on it. @@ -561,7 +577,7 @@ describe("persisted documents in a transaction", () => { }, context, ); - expect((await store.outbox!.read(2, 10, context))[0]!.writes).toMatchObject([{ kind: "document.change", id: 3 }]); + expect((await store.outbox!.read(2, 10, context))[0]!.writes).toMatchObject([{ kind: "document.change", id: 4 }]); await host.close(context); await store.close(context); }); @@ -644,7 +660,7 @@ describe("persisted documents in a transaction", () => { apply: async () => { inStep.resolve(); await gate.promise; - return { kind: "dropIndex", table: "x", name: "y" }; + return [{ kind: "dropIndex", table: "x", name: "y" }]; }, }; return commit([...writes, held], fence, ctx); @@ -700,7 +716,7 @@ describe("persisted documents in a transaction", () => { apply: async () => { inStep.resolve(); await gate.promise; - return { kind: "dropIndex", table: "x", name: "y" }; + return [{ kind: "dropIndex", table: "x", name: "y" }]; }, }; return commit([...writes, held], fence, ctx); diff --git a/packages/chord/test/next/examples/19-jobs.ts b/packages/chord/test/next/examples/19-jobs.ts index f52ee317a..06964dc63 100644 --- a/packages/chord/test/next/examples/19-jobs.ts +++ b/packages/chord/test/next/examples/19-jobs.ts @@ -140,7 +140,7 @@ const facet = defineFacet

((host) => { runShell: async ({ command, lines, requestId }, ctx) => (await shell.start({ command, lines }, ctx, { key: requestId })).id, abortShell: async ({ id }) => { - shell.abort(id); + void shell.abort(id, context); return null; }, }; diff --git a/packages/chord/test/next/examples/28-persisted-jobs.ts b/packages/chord/test/next/examples/28-persisted-jobs.ts new file mode 100644 index 000000000..4750e3ced --- /dev/null +++ b/packages/chord/test/next/examples/28-persisted-jobs.ts @@ -0,0 +1,125 @@ +// Persisted jobs: `persist: { resume }` keeps each unsettled run's record and state document in the object's store. +// A run survives a crash (counted: five in a row end it `restart_limit`), a clean close (written back `stopped`, resumed +// without counting) and its object's retry; `abort` is stored and joins `run`; a state change may be `deferred`. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/28-persisted-jobs.ts +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import { defineJob, type JobHandle } from "../../../src/next/jobs/index.ts"; +import { openStore, type Store, sqliteBackend } from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const context = BACKGROUND_CONTEXT; + +// The run's state: persisted, as a persisted job's must be. +const ImportState = defineState("notes.import.state", { + schema: obj({ done: int() }), + version: 1, + initial: () => ({ done: 0 }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 16, +}); + +const ImportJob = defineJob("notes.import", { + version: 1, + args: obj({ path: str() }), + result: obj({ count: int() }), + state: ImportState, + // After a crash or a stop, `run` runs again: it must be idempotent (only work since the last state change repeats). + persist: { resume: "rerun" }, +}); + +const Notes = local("notes.store", { + version: 1, + params: {}, + members: { start: method(str(), str()), abort: method(str(), str()) }, +}); + +/** How the current process's `run` behaves: hang (to be cut off) or finish. */ +let behaviour: "hang" | "finish" = "hang"; +let jobs: JobHandle<{ path: string }, { count: number }, { done: number }> | undefined; + +const facet = defineFacet((host) => { + host.serve(Notes, (_, self) => { + const imports = self.job(ImportJob, { + run: async ({ path }, ctx) => { + console.log(` run ${path}: from done = ${ctx.state.value.done}`); + // Progress, deferred: published at once, stored within 100 ms (or with the next immediate change). + await ctx.state.change( + (draft) => { + draft.done += 1; + }, + ctx, + { persistence: { kind: "deferred", withinMs: 100 } }, + ); + if (behaviour === "hang") + await new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", resolve)); + return { count: ctx.state.value.done }; + }, + finish: async (outcome) => { + console.log(` finish: ${outcome.kind}`, outcome.kind === "done" ? outcome.result : outcome.error.kind); + }, + }); + jobs = imports; + return { + start: async (path, ctx) => (await imports.start({ path }, ctx)).id, + abort: (id, ctx) => imports.abort(id, ctx), + }; + }); +}); + +/** A process: a store over the file and a host serving `Notes`. */ +async function process(path: string): Promise<{ store: Store; host: Host }> { + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context); + const host = createHost({ + store, + onError: (error) => console.log(" onError:", error.kind), + closeTimeout: 100, + }); + await host.load([{ name: "@example/notes", facet }], context); + return { store, host }; +} + +const settle = () => new Promise((resolve) => setTimeout(resolve, 150)); +const directory = await mkdtemp(join(tmpdir(), "chord-example-28-")); +const path = join(directory, "session.sqlite"); + +// 1. Start a run, then crash: the store closes under the live host. +console.log("1. start, then crash"); +let { store, host } = await process(path); +const id = await (await host.get(Notes, {}, context)).start("inbox", context); +await settle(); +await store.close(context); + +// 2. The next process counts the crash and reruns it (its state as stored: the deferred change was stored in time). +console.log("2. reopen: the run is counted and rerun"); +({ store, host } = await process(path)); +await settle(); + +// 3. A clean close writes the run back `stopped`: the next process resumes it without counting. +console.log("3. clean close, reopen: resumed, uncounted"); +await host.close(context); +await store.close(context); +behaviour = "finish"; +({ store, host } = await process(path)); +await settle(); +console.log(" unsettled runs now:", jobs?.running.value.length); + +// 4. A stored abort: marked, joined. +console.log("4. abort"); +behaviour = "hang"; +const notes = await host.get(Notes, {}, context); +const second = await notes.start("drafts", context); +await settle(); +console.log(" abort:", await notes.abort(second, context)); +await settle(); +console.log(" once settled:", await notes.abort(second, context), "/ the first run:", await notes.abort(id, context)); + +await host.close(context); +await store.close(context); +await rm(directory, { recursive: true, force: true }); diff --git a/packages/chord/test/next/host/attempt.test.ts b/packages/chord/test/next/host/attempt.test.ts index 73983ae0a..867502320 100644 --- a/packages/chord/test/next/host/attempt.test.ts +++ b/packages/chord/test/next/host/attempt.test.ts @@ -604,7 +604,7 @@ describe("tables", () => { }, ); await host.get(Store, {}, context); - expect(store.table("client:app.store:edits")).toMatchObject({ + expect(store.table("app.store:edits")).toMatchObject({ version: 1, state: "ready", indexes: { by_at: ["at"] }, @@ -717,7 +717,7 @@ describe("eager migrations at activation (slice 6a §9)", () => { afterEach(() => { pumping = false; }); - const TABLE = "client:app.store:edits"; + const TABLE = "app.store:edits"; const OPEN = { admit: () => true, adopt: () => {} }; const EditsV1 = defineTable("edits", { version: 1, columns: { id: str(), at: int() }, key: ["id"] }); const Store = local("app.store", { version: 1, params: {}, members: { n: int() } }); diff --git a/packages/chord/test/next/host/follow.test.ts b/packages/chord/test/next/host/follow.test.ts new file mode 100644 index 000000000..de225e098 --- /dev/null +++ b/packages/chord/test/next/host/follow.test.ts @@ -0,0 +1,345 @@ +// Tests of `self.follow` (slice 6b §3, D9, R4): a source over another runtime's synchronous observer, attached during +// activation, its calls marked in the running batch or grouped into one batch outside it, its ends. + +import { describe, expect, expectTypeOf, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import type { Op } from "../../../src/delta/index.ts"; +import { int, local, obj } from "../../../src/next/contracts/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import { FollowSource } from "../../../src/next/host/follow.ts"; +import type { ChordHost } from "../../../src/next/host/host.ts"; +import type { FollowAttach, FollowObserver } from "../../../src/next/host/index.ts"; +import { Batch } from "../../../src/next/state/batch.ts"; +import type { Watchable, WatchEnd } from "../../../src/watch.ts"; +import { changesOf, delivered, expectCollected, instancesOf, loaded, valuesOf } from "../support.ts"; + +const followed = instancesOf(FollowSource, "attach"); + +type Count = { readonly n: number }; + +/** A runtime with synchronous observers, as Durable's views have them. */ +class Runtime { + value: Count = { n: 0 }; + readonly observers = new Set>(); + attachments = 0; + detached = 0; + /** Resolves the next attachment when set (an attachment that takes time). */ + gate: Promise | undefined; + absent = false; + + readonly attach: FollowAttach = async (observer) => { + this.observers.add(observer); + this.attachments++; + await this.gate; + if (this.absent) return undefined; + return { + value: this.value, + detach: () => { + this.observers.delete(observer); + this.detached++; + }, + }; + }; + + /** One change: every observer advanced with its ops. */ + set(n: number, retired = false): void { + const ops: Op[] = [["s", ["n"], n]]; + this.value = { n }; + for (const observer of this.observers) + observer.advance({ kind: "value", value: this.value, ops, retired }, context); + } + + end(end: WatchEnd): void { + for (const observer of [...this.observers]) observer.end(end); + this.observers.clear(); + } +} + +const View = local("app.view", { version: 1, params: {}, members: { a: obj({ n: int() }), sum: int() } }); + +/** Serves `View`, following `a` and `b`, its `sum` a derive over both (its computes counted). */ +async function served(a = new Runtime(), b = new Runtime()) { + let computes = 0; + const { host, errors } = await loaded((h) => + h.serve(View, (_, self) => { + const left = self.follow(a.attach); + const right = self.follow(b.attach); + const sum = self.derive([left, right], ([x, y]) => { + computes++; + return x.n + y.n; + }); + return { a: left, sum }; + }), + ); + const view = await host.get(View, {}, context); + const batch = (host as unknown as ChordHost).batch; + return { host, errors, view, a, b, batch, computes: () => computes }; +} + +describe("calls", () => { + it("inside a running batch: one frame per commit, and a derive over two views computes once with both", async () => { + const { view, a, b, batch, computes } = await served(); + const calls = changesOf(await view.a.watch(context)); + const sums = changesOf(await view.sum.watch(context)); + const before = computes(); + batch.commit(() => { + a.set(1); + a.set(2); + b.set(10); + }, context); + await delivered(); + expect(calls).toEqual([ + { + kind: "value", + value: { n: 2 }, + ops: [ + ["s", ["n"], 1], + ["s", ["n"], 2], + ], + retired: false, + }, + ]); + expect(valuesOf(sums)).toEqual([12]); + expect(computes() - before).toBe(1); + }); + + it("outside any batch: queued and applied in one batch on the next microtask", async () => { + const { view, a, b, computes } = await served(); + const sums = changesOf(await view.sum.watch(context)); + const before = computes(); + a.set(1); + b.set(2); + expect(view.a.value).toEqual({ n: 0 }); + await delivered(); + expect(view.a.value).toEqual({ n: 1 }); + expect(valuesOf(sums)).toEqual([3]); + expect(computes() - before).toBe(1); + }); + + it("before the attachment resolved: applied in order on its value", async () => { + const a = new Runtime(); + const gate = Promise.withResolvers(); + a.gate = gate.promise; + const serving = served(a); + await delivered(); + a.set(5); + a.set(6); + gate.resolve(); + const { view } = await serving; + expect(view.a.value).toEqual({ n: 6 }); + }); +}); + +describe("types", () => { + it("infers the value type from the attachment, without a type argument", async () => { + const runtime = new Runtime(); + // A runtime's own observe, as Durable's `observeConversation` is: typed by its value, not by Chord's observer. + const observe = (observer: FollowObserver) => runtime.attach(observer, context); + await loaded((h) => + h.serve(View, (_, self) => { + const a = self.follow((observer) => observe(observer)); + expectTypeOf(a).toEqualTypeOf>(); + return { a, sum: self.derive([a], ([x]) => x.n) }; + }), + ); + }); +}); + +describe("activation", () => { + it("an absent address fails the activation transient; a failed activation detaches what attached", async () => { + const a = new Runtime(); + const b = new Runtime(); + b.absent = true; + const { host, errors } = await loaded((h) => + h.serve(View, (_, self) => ({ a: self.follow(a.attach), sum: self.derive([self.follow(b.attach)], () => 0) })), + ); + await expect(host.get(View, {}, context, { readyTimeout: 50 })).rejects.toThrow("followed address is absent"); + expect(errors.find((error) => error.kind === "activation_failed")).toMatchObject({ + failure: { kind: "transient" }, + }); + await delivered(); + expect([a.attachments, a.detached]).toEqual([a.attachments, a.attachments]); + await host.close(context); + }); +}); + +describe("ends (R4)", () => { + it("failed: the live object fails transient once, reported, and its retry attaches again", async () => { + const { host, errors, view, a, b } = await served(); + a.end({ kind: "failed", error: new ChordError({ code: "unavailable", message: "session failed" }) }); + b.end({ kind: "failed", error: new ChordError({ code: "unavailable", message: "session failed" }) }); + expect(view.state.value).toMatchObject({ kind: "failed", cause: { kind: "transient" } }); + expect((view.state.value as { retryAt?: number }).retryAt).toBeGreaterThan(0); + expect(errors.filter((error) => error.kind === "object_failed")).toHaveLength(1); + expect(view.a.value).toEqual({ n: 0 }); + await delivered(); + view.retry(); + await delivered(); + await delivered(); + expect(view.state.value).toEqual({ kind: "live" }); + expect([a.attachments, a.detached, b.attachments]).toEqual([2, 1, 2]); + a.set(3); + await delivered(); + expect(view.a.value).toEqual({ n: 3 }); + await host.close(context); + }); + + it("ended: final; a queued retirement reaches the watch first, the value stays, a new watch ends at once", async () => { + const { host, errors, view, a } = await served(); + const watch = await view.a.watch(context); + const calls = changesOf(watch); + a.set(4, true); + a.end({ kind: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(calls).toMatchObject([{ kind: "value", value: { n: 4 }, retired: true }]); + expect(view.a.value).toEqual({ n: 4 }); + const late = await view.a.watch(context); + expect(late.value).toEqual({ n: 4 }); + expect(await late.closed).toEqual({ kind: "ended" }); + expect(errors).toEqual([]); + expect(view.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); +}); + +describe("ends during activation and in one batch (review regressions)", () => { + it("an end before the attachment resolved keeps the attachment's value on the source", async () => { + const source = new FollowSource( + async (observer) => { + observer.end({ kind: "ended" }); + return { value: { n: 7 }, detach() {} }; + }, + { label: "early end", context, ended() {} }, + new Batch(() => {}, undefined), + ); + await source.attach(context); + expect(source.value).toEqual({ n: 7 }); + }); + + it.each([ + ["ended", { kind: "ended" } as const], + ["failed", { kind: "failed", error: new Error("early failure") } as const], + ])("an %s end before the attachment resolved fails the activation transient", async (_, end) => { + const a = new Runtime(); + const gate = Promise.withResolvers(); + a.gate = gate.promise; + // Loading activates the object (local, empty params): its follow attaches meanwhile. + const loading = loaded((h) => + h.serve(View, (_, self) => { + const left = self.follow(a.attach); + return { a: left, sum: self.derive([left], ([x]) => x.n) }; + }), + ); + await delivered(); + a.end(end); + gate.resolve(); + const { host, errors } = await loading; + await delivered(); + expect(errors.find((error) => error.kind === "activation_failed")).toMatchObject({ + failure: { kind: "transient" }, + }); + await host.close(context); + expect(a.detached).toBe(a.attachments); + }); + + it("an end while the object activates (after its sources attached) fails the activation transient", async () => { + const a = new Runtime(); + let ended = false; + const { host, errors } = await loaded((h) => + h.serve(View, (_, self) => { + const left = self.follow(a.attach); + // The activation computes the derives after the follows attached: the runtime ends then. + const sum = self.derive([left], ([x]) => { + if (!ended) { + ended = true; + a.end({ kind: "ended" }); + } + return x.n; + }); + return { a: left, sum }; + }), + ); + await expect(host.get(View, {}, context, { readyTimeout: 50 })).rejects.toThrow(); + expect(ended).toBe(true); + expect(errors.find((error) => error.kind === "activation_failed")).toMatchObject({ + failure: { kind: "transient" }, + }); + await host.close(context); + }); + + it("a retirement and an end inside one batch: the watch gets the retirement, then ends", async () => { + const batch = new Batch(() => {}, undefined); + let observer: FollowObserver | undefined; + const source = new FollowSource( + async (attached) => { + observer = attached; + return { value: { n: 7 }, detach() {} }; + }, + { label: "retirement", context, ended() {} }, + batch, + ); + await source.attach(context); + const watch = await source.watch(context); + const calls = changesOf(watch); + batch.commit(() => { + observer!.advance({ kind: "value", value: { n: 7 }, ops: [], retired: true }, context); + observer!.end({ kind: "ended" }); + }, context); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(calls).toEqual([{ kind: "value", value: { n: 7 }, ops: [], retired: true }]); + }); + + it("a call and an end outside a batch with nothing watching: a later watch opens at the value and ends", async () => { + let observer: FollowObserver | undefined; + const source = new FollowSource( + async (attached) => { + observer = attached; + return { value: { n: 0 }, detach() {} }; + }, + { label: "unwatched", context, ended() {} }, + new Batch(() => {}, undefined), + ); + await source.attach(context); + observer!.advance({ kind: "value", value: { n: 1 }, ops: [["s", ["n"], 1]], retired: true }, context); + observer!.end({ kind: "ended" }); + await delivered(); + const watch = await source.watch(context); + expect(watch.value).toEqual({ n: 1 }); + expect(await watch.closed).toEqual({ kind: "ended" }); + source.detach(); + }); + + it("an end outside a batch after calls of two views: the views of one commit are delivered together", async () => { + const { host, view, a, b, computes } = await served(); + const sums = changesOf(await view.sum.watch(context)); + const before = computes(); + a.set(1, true); + b.set(2); + a.end({ kind: "ended" }); + await delivered(); + expect(valuesOf(sums)).toEqual([3]); + expect(computes() - before).toBe(1); + await host.close(context); + }); +}); + +describe("GC", () => { + it("a source its runtime ended is still detached at the object's end, then collected", async () => { + const a = new Runtime(); + const { host } = await served(a); + a.end({ kind: "ended" }); + await delivered(); + expect(a.detached).toBe(0); + await host.close(context); + expect(a.detached).toBe(1); + await expectCollected(...followed.splice(0)); + }); + + it("an ended object's followed sources are collected, and their observers detached", async () => { + const a = new Runtime(); + const { host } = await served(a); + await host.close(context); + expect(a.observers.size).toBe(0); + await expectCollected(...followed.splice(0)); + }); +}); diff --git a/packages/chord/test/next/host/host.test.ts b/packages/chord/test/next/host/host.test.ts index 96d2482c4..07c4eb751 100644 --- a/packages/chord/test/next/host/host.test.ts +++ b/packages/chord/test/next/host/host.test.ts @@ -21,6 +21,7 @@ import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; import { failures, fakeTimers, + faultyStore, flushed, forEachCodec, linked, @@ -921,20 +922,16 @@ describe("the host's store", () => { }); it("reports storage_failed once and closes the host; the app still closes the store", async () => { - const store = await memoryStore(); + const { store, fail } = await faultyStore(); const errors: HostError[] = []; const host = createHost({ store, onError: (error) => errors.push(error) }); await host.load([], context); - const broken: StagedWrite = { - order: "write", - apply: async (step) => { - await step.run("INSERT INTO no_such_table VALUES (1)"); - return { kind: "document.retire", id: 0, revision: 0 }; - }, - }; + const write: StagedWrite = { order: "write", apply: async () => [] }; const fence = { admit: () => true, adopt: () => {} }; - await expect(store.commit([broken], fence, context)).rejects.toMatchObject({ reason: "storage_failed" }); - await expect(store.commit([broken], fence, context)).rejects.toMatchObject({ reason: "storage_failed" }); + // A failed `COMMIT`: an unknown outcome (D4). + fail("COMMIT"); + await expect(store.commit([write], fence, context)).rejects.toMatchObject({ reason: "storage_failed" }); + await expect(store.commit([write], fence, context)).rejects.toMatchObject({ reason: "storage_failed" }); // The host closed itself; the app's own close returns that closing. expect((host as ChordHost).closing).toBe(true); await host.close(context); diff --git a/packages/chord/test/next/host/member-source.test.ts b/packages/chord/test/next/host/member-source.test.ts index 77939396b..4a804cdb7 100644 --- a/packages/chord/test/next/host/member-source.test.ts +++ b/packages/chord/test/next/host/member-source.test.ts @@ -386,6 +386,7 @@ describe("subscribers, directly", () => { advance: (call) => void events.push(call.kind === "value" ? [call.value, call.ops] : (call.error as Error).message), end: (end) => void events.push(end.kind), + endAfterDelivery: () => void events.push("ended"), }; expect(source.value).toBe(0); source.subscribe(subscriber); diff --git a/packages/chord/test/next/jobs/definition.test.ts b/packages/chord/test/next/jobs/definition.test.ts index 4f16502ac..aea86122d 100644 --- a/packages/chord/test/next/jobs/definition.test.ts +++ b/packages/chord/test/next/jobs/definition.test.ts @@ -96,11 +96,34 @@ describe("defineJob", () => { ); }); - it("refuses persist before slice 6, in its final shape", () => { - expect(() => defineJob("j", { version: 1, args, result: int(), persist: { resume: "rerun" } })).toThrow( - "j.persist: persisted jobs are not supported before slice 6", + it("takes persist with a resume rule, a persisted state and a JSON-only result (slice 6b §4)", () => { + const Persisted = defineState("p.state", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: 0 }), + persist: true, + checkpointWhen: () => true, + }); + const InMemory = defineState("m.state", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + const job = defineJob("j", { version: 1, args, result: int(), state: Persisted, persist: { resume: "rerun" } }); + expect(job.persist).toEqual({ resume: "rerun" }); + expect(() => + defineJob("j", { version: 1, args, result: int(), state: InMemory, persist: { resume: "abandon" } }), + ).toThrow("a persisted job's state is a persisted document"); + expect(() => defineJob("j", { version: 1, args, result: bytes(), persist: { resume: "rerun" } })).toThrow( + "a persisted job's result is JSON", ); + // An in-memory job may return bytes. + expect(() => defineJob("j", { version: 1, args, result: bytes() })).not.toThrow(); // @ts-expect-error: resume is required - expect(() => defineJob("j", { version: 1, args, result: int(), persist: {} })).toThrow(); + expect(() => defineJob("j", { version: 1, args, result: int(), persist: {} })).toThrow( + 'expected "rerun" or "abandon"', + ); + }); + + it("fingerprints args, result and state: a schema change without a version bump changes it", () => { + const one = defineJob("j", { version: 1, args, result: int() }); + expect(defineJob("j", { version: 1, args, result: int() }).fingerprint).toBe(one.fingerprint); + expect(defineJob("j", { version: 1, args, result: str() }).fingerprint).not.toBe(one.fingerprint); }); }); diff --git a/packages/chord/test/next/jobs/job.test.ts b/packages/chord/test/next/jobs/job.test.ts index a868d0f04..4fd9aadcb 100644 --- a/packages/chord/test/next/jobs/job.test.ts +++ b/packages/chord/test/next/jobs/job.test.ts @@ -325,23 +325,27 @@ describe("start", () => { expect(log.value.list).toEqual(["written with the start", "from run a"]); }); - it("starts run with its signal aborted after an abort between the commit and its start", async () => { + it("an abort between the commit and the start is marked, then the signal fires; the abort joins run (D7)", async () => { let signalled: boolean | undefined; const { job } = await declared(Echo, () => ({ run: async ({ text }, ctx) => { + await new Promise((resolve) => setTimeout(resolve, 20)); signalled = ctx.abortSignal?.aborted; return text; }, })); const watch = await job.running.watch(context); + const aborts: Promise[] = []; // The watch's frame of the start's commit arrives before `run` starts. watch.start(async (call) => { - if (call.kind !== "value") return; - for (const entry of call.value) job.abort(entry.id); + if (call.kind !== "value" || aborts.length > 0) return; + for (const entry of call.value) aborts.push(job.abort(entry.id, context)); }); const run = await job.start({ text: "a" }, context); expect(await run.outcome(context)).toMatchObject({ kind: "done", result: "a" }); expect(signalled).toBe(true); + expect(await Promise.all(aborts)).toEqual(["marked"]); + expect(await job.abort(run.id, context)).toBe("terminal"); await watch.stop(); }); @@ -578,7 +582,7 @@ describe("run", () => { })); const run = await job.start({ command: "a" }, context); await flushed(); - job.abort(run.id); + void job.abort(run.id, context); expect(await run.outcome(context)).toEqual({ id: run.id, args: { command: "a" }, @@ -598,7 +602,7 @@ describe("run", () => { })); const run = await job.start({ text: "kept" }, context); await flushed(); - job.abort(run.id); + void job.abort(run.id, context); expect(await run.outcome(context)).toMatchObject({ kind: "done", result: "kept" }); }); @@ -779,7 +783,7 @@ describe("finish", () => { const other = await job.start({ command: "c" }, context, { key: "other" }); await vi.advanceTimersByTimeAsync(0); expect(job.running.value.map((entry) => entry.phase)).toEqual(["blocked", "finishing"]); - job.abort(run.id); + void job.abort(run.id, context); await vi.advanceTimersByTimeAsync(0); expect(await dropped).toMatchObject({ code: "aborted" }); expect(job.running.value.map((entry) => entry.id)).toEqual([other.id]); @@ -819,7 +823,7 @@ describe("finish", () => { })); const run = await job.start({ text: "a" }, context); await flushed(); - job.abort(run.id); + void job.abort(run.id, context); gate.resolve(); expect((await run.outcome(context)).kind).toBe("done"); }); @@ -1299,7 +1303,7 @@ describe("GC", () => { () => new Error("expected a rejection"), (error: Error) => error, ); - job.abort(run.id); + void job.abort(run.id, context); await vi.advanceTimersByTimeAsync(0); expect(await dropped).toMatchObject({ code: "aborted" }); expect(job.running.value).toEqual([]); diff --git a/packages/chord/test/next/jobs/persisted.test.ts b/packages/chord/test/next/jobs/persisted.test.ts new file mode 100644 index 000000000..80751d207 --- /dev/null +++ b/packages/chord/test/next/jobs/persisted.test.ts @@ -0,0 +1,1066 @@ +// Tests of persisted jobs (slice 6b §4, D5-D8, R3): run records and state documents in the object's store, every +// transition of §4.2 across a crash (the store closed under a live host, reopened by a new one), the clean stop and +// the close writer, a seal and its retry, the restart limit and its reset, the finish failure count, a stored abort, +// deferred state. + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { MAX_PENDING_OPS } from "../../../src/delta/pending.ts"; +import { arr, int, local, num, obj, str } from "../../../src/next/contracts/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { RETENTION_MS, Residency } from "../../../src/next/documents/residency.ts"; +import { createHost, defineFacet, type HostError, type StoreSelf } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { + defineJob, + type JobHandle, + type JobImplementation, + type Outcome, + type RunState, +} from "../../../src/next/jobs/index.ts"; +import { JobRun } from "../../../src/next/jobs/run.ts"; +import { sqliteBackend } from "../../../src/next/store/backend.ts"; +import { type ChordStore, openStore } from "../../../src/next/store/store.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { delivered, expectCollected, failLive, instancesOf } from "../support.ts"; + +const attempted = instancesOf(ChordObject, "attempt"); + +const Progress = defineState("import.progress", { + schema: obj({ done: int() }), + version: 1, + initial: () => ({ done: 0 }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 8, +}); + +const Import = defineJob("notes.import", importOptions()); + +/** `Import`'s declaration, which the other versions below change. */ +function importOptions() { + return { + version: 1, + args: obj({ path: str() }), + result: obj({ count: int() }), + state: Progress, + persist: { resume: "rerun" }, + } as const; +} + +/** `Import` at the next version, as a deploy changes it. */ +const ImportV2 = defineJob("notes.import", { ...importOptions(), version: 2 }); + +/** `Import` at its version with another result schema (`num` for `int`): another fingerprint. */ +const ImportReshaped = defineJob("notes.import", { ...importOptions(), result: obj({ count: num() }) }); + +const Abandoned = defineJob("notes.abandoned", { + version: 1, + args: obj({ path: str() }), + result: obj({ count: int() }), + persist: { resume: "abandon" }, +}); + +const Store = local("notes.store", { version: 1, params: {}, members: {} }); + +type ImportArgs = { readonly path: string }; +type ImportResult = { readonly count: number }; +type ImportOutcome = Outcome; + +let directory: string | undefined; + +afterEach(async () => { + if (directory !== undefined) await rm(directory, { recursive: true, force: true }); + directory = undefined; +}); + +/** A store file opened by `open()` again after each crash or close. */ +async function storeFile() { + directory = await mkdtemp(join(tmpdir(), "chord-jobs-")); + const path = join(directory, "store.sqlite"); + return async () => (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; +} + +/** What one host over the store sees of the store object's jobs. */ +type Opened = { + readonly store: ChordStore; + readonly errors: HostError[]; + readonly imports: JobHandle; + readonly abandoned: JobHandle; + readonly finished: ImportOutcome[]; + readonly self: StoreSelf; + /** `ref.retry()` of the store object. */ + retry(): Promise; + close(): Promise; + /** A crash: the store's backend closes under the live host, which is left behind. */ + crash(): Promise; +}; + +/** A host over `store` serving `Store` with both jobs (`Import` or another declaration of it), `run` and `finish` given. */ +async function opened( + store: ChordStore, + run: JobImplementation["run"], + finish?: (outcome: ImportOutcome) => void, + declared: typeof Import = Import, +): Promise { + const errors: HostError[] = []; + const finished: ImportOutcome[] = []; + const host = createHost({ store, onError: (error) => errors.push(error), closeTimeout: 200 }); + let found: Pick | undefined; + await host.load( + [ + { + name: "notes", + facet: defineFacet((h) => + h.serve(Store, (_, self) => { + const imports = self.job(declared, { + run, + finish: async (outcome) => { + finish?.(outcome); + finished.push(outcome); + }, + }); + const abandoned = self.job(Abandoned, { + run: async () => ({ count: 0 }), + finish: async (outcome) => void finished.push(outcome as unknown as ImportOutcome), + }); + found = { imports, abandoned, self }; + return {}; + }), + ), + }, + ], + context, + ); + return { + store, + errors, + finished, + ...found!, + retry: async () => (await host.hold(Store, {}, context)).retry(), + close: async () => { + await host.close(context); + await store.close(context); + }, + crash: () => store.close(context), + }; +} + +/** The stored revisions of the run state documents, in seq order: their kinds and contents. */ +async function stateRevisions(store: ChordStore): Promise<[string, unknown][]> { + const rows = await store.read( + (executor) => + executor.all<{ kind: string; content: string }>( + `SELECT r.kind, r.content FROM chord_revisions r JOIN chord_documents d ON d.id = r.document_id + WHERE d.namespace = 'chord.job-states' ORDER BY r.seq`, + ), + context, + ); + return rows.map(({ kind, content }) => [kind, JSON.parse(content)]); +} + +/** Waits `ms` of real time (a drain's bound passing, a wait that must not settle). */ +function sleepMs(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** The records the store keeps of `namespace`'s documents: their kinds and keys. */ +async function stored(store: ChordStore, namespace: string): Promise { + return store.read( + (executor) => executor.all("SELECT kind, key FROM chord_documents WHERE namespace = ? ORDER BY id", namespace), + context, + ); +} + +/** The current record of run `id`. */ +async function record(store: ChordStore, id: string): Promise | undefined> { + const row = await store.read( + (executor) => + executor.get<{ content: string }>( + `SELECT r.content FROM chord_revisions r JOIN chord_documents d ON d.id = r.document_id + WHERE d.namespace = 'chord.jobs' AND d.key = ? ORDER BY r.seq DESC LIMIT 1`, + id, + ), + context, + ); + return row === undefined ? undefined : (JSON.parse(row.content) as Record); +} + +describe("records", () => { + it("a run's record and state live in the store while it is unsettled; finish deletes the record, retires the state", async () => { + const open = await storeFile(); + const gate = Promise.withResolvers(); + const jobs = await opened(await open(), async (_args, ctx) => { + await ctx.state.change((draft) => { + draft.done = 3; + }, ctx); + await gate.promise; + return { count: 3 }; + }); + const run = await jobs.imports.start({ path: "a" }, context, { key: "k" }); + await delivered(); + expect(await stored(jobs.store, "chord.jobs")).toEqual([{ kind: "notes.import", key: run.id }]); + expect(await stored(jobs.store, "chord.job-states")).toEqual([{ kind: "notes.import", key: run.id }]); + expect(await record(jobs.store, run.id)).toMatchObject({ + phase: "running", + restarts: 0, + key: "k", + args: { path: "a" }, + }); + gate.resolve(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 3 }, state: { done: 3 } }); + expect(await stored(jobs.store, "chord.jobs")).toEqual([]); + expect(await stored(jobs.store, "chord.job-states")).toEqual([]); + await jobs.close(); + }); +}); + +describe("crashes and restarts (D6)", () => { + it("a run cut off while running is counted and rerun; a state change resets the count; five in a row end it", async () => { + const open = await storeFile(); + const hang = (): Promise => new Promise(() => {}); + let jobs = await opened(await open(), hang); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + for (let restart = 1; restart < 5; restart++) { + await jobs.crash(); + jobs = await opened(await open(), hang); + await delivered(); + expect(await record(jobs.store, run.id)).toMatchObject({ phase: "running", restarts: restart }); + } + await jobs.crash(); + jobs = await opened(await open(), hang); + await delivered(); + await delivered(); + expect(jobs.finished).toMatchObject([{ kind: "failed", error: { kind: "restart_limit", restarts: 5 } }]); + expect(jobs.errors).toContainEqual( + expect.objectContaining({ kind: "job_interrupted", job: "notes.import", id: run.id, restarts: 5 }), + ); + await jobs.close(); + }); + + it("a state change resets the restart count, stored at once", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + jobs = await opened(await open(), async (_args, ctx) => { + await ctx.state.change( + (draft) => { + draft.done = 1; + }, + ctx, + { persistence: { kind: "deferred" } }, + ); + return new Promise(() => {}); + }); + await delivered(); + await delivered(); + expect(await record(jobs.store, run.id)).toMatchObject({ phase: "running", restarts: 0 }); + await jobs.crash(); + }); + + it("abandon: a run cut off by a crash finishes failed { aborted, restart } without run", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), async () => ({ count: 0 })); + const gate = Promise.withResolvers(); + // Its run never returns before the crash. + void gate; + const run = await jobs.abandoned.start({ path: "a" }, context); + await jobs.crash(); + jobs = await opened(await open(), async () => ({ count: 0 })); + await delivered(); + await delivered(); + expect(jobs.finished.filter((outcome) => outcome.id === run.id)).toMatchObject([ + { kind: "failed", error: { kind: "aborted", reason: "restart" } }, + ]); + await jobs.close(); + }); +}); + +describe("versions", () => { + it.each([ + ["another version", ImportV2], + ["the same version with another fingerprint", ImportReshaped], + ])("a run stored by %s is blocked (version): kept as stored, never run, listed blocked", async (_, declared) => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + let runs = 0; + jobs = await opened( + await open(), + async () => { + runs++; + return { count: 1 }; + }, + undefined, + declared, + ); + await delivered(); + expect(runs).toBe(0); + expect(jobs.imports.running.value).toEqual([{ id: run.id, version: 1, phase: "blocked", reason: "version" }]); + // Kept as stored: the crash is not counted, slice 7 maps or resolves it. + expect(await record(jobs.store, run.id)).toMatchObject({ version: 1, phase: "running", restarts: 0 }); + await jobs.close(); + }); +}); + +describe("versions with incompatible state (review regression)", () => { + it("a run of another version is blocked before its state is decoded: no migration, no activation failure", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + const changed = defineState("import.progress", { + schema: obj({ done: str() }), + version: 2, + initial: () => ({ done: "new" }), + persist: true, + checkpointWhen: () => false, + migrate: () => { + throw new Error("must not migrate an unsupported run"); + }, + }); + const declared = defineJob("notes.import", { ...importOptions(), version: 2, state: changed }); + // The declaration's state type differs from `Import`'s: this host only lists the blocked run. + jobs = await opened(await open(), async () => ({ count: 0 }), undefined, declared as unknown as typeof Import); + expect(jobs.errors).toEqual([]); + expect(jobs.imports.running.value).toEqual([{ id: run.id, version: 1, phase: "blocked", reason: "version" }]); + // An abort drops it: its record and its undecoded state document are retired. + expect(await jobs.imports.abort(run.id, context)).toBe("marked"); + expect(await stored(jobs.store, "chord.jobs")).toEqual([]); + expect(await stored(jobs.store, "chord.job-states")).toEqual([]); + await jobs.close(); + }); +}); + +describe("versions without a state (review regression)", () => { + it("an abort of a run blocked under a version without a state retires the old state document too", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + const stateless = defineJob("notes.import", { ...importOptions(), version: 2, state: undefined }); + // The declaration's state type differs from `Import`'s: this host only lists and drops the blocked run. + jobs = await opened(await open(), async () => ({ count: 0 }), undefined, stateless as unknown as typeof Import); + expect(jobs.imports.running.value).toEqual([{ id: run.id, version: 1, phase: "blocked", reason: "version" }]); + expect(await jobs.imports.abort(run.id, context)).toBe("marked"); + expect(await stored(jobs.store, "chord.jobs")).toEqual([]); + expect(await stored(jobs.store, "chord.job-states")).toEqual([]); + await jobs.close(); + }); +}); + +describe("a seal during the reservation (review regressions)", () => { + it("a reservation refused at admission after the seal: the next live attempt invokes the run", async () => { + const open = await storeFile(); + let invoked = 0; + const jobs = await opened(await open(), async () => ({ count: ++invoked })); + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + const commit = jobs.store.commit.bind(jobs.store); + let commits = 0; + // The second commit is the reservation: it reaches the store only after the seal. + jobs.store.commit = (writes, fence, ctx) => { + if (++commits !== 2) return commit(writes, fence, ctx); + entered.resolve(); + return gate.promise.then(() => commit(writes, fence, ctx)); + }; + const run = await jobs.imports.start({ path: "p" }, context); + await entered.promise; + failLive(attempted, jobs.self, new ChordError({ code: "unavailable", message: "seal during reservation" })); + await jobs.retry(); + gate.resolve(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 1 } }); + expect(invoked).toBe(1); + await jobs.close(); + }); + + it("a reservation queued behind a held commit queue past the seal: the next live attempt invokes the run", async () => { + const open = await storeFile(); + let invoked = 0; + const jobs = await opened(await open(), async () => ({ count: ++invoked })); + const held = Promise.withResolvers<() => void>(); + const commit = jobs.store.commit.bind(jobs.store); + let first = true; + // The start's commit takes the queue as it adopts (an export would): the reservation waits behind it. + jobs.store.commit = (writes, fence, ctx) => { + if (!first) return commit(writes, fence, ctx); + first = false; + const holding = { + admit: () => fence.admit(), + adopt: (seq: number, ids: readonly number[]) => { + fence.adopt(seq, ids); + void jobs.store.take(() => {}).then(held.resolve); + }, + }; + return commit(writes, holding, ctx); + }; + const run = await jobs.imports.start({ path: "p" }, context); + const release = await held.promise; + await delivered(); + failLive(attempted, jobs.self, new ChordError({ code: "unavailable", message: "seal queued reservation" })); + await jobs.retry(); + release(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 1 } }); + expect(invoked).toBe(1); + await jobs.close(); + }); + + it("an abort joined to a reservation that commits after the seal returns marked", async () => { + const open = await storeFile(); + const jobs = await opened(await open(), async () => ({ count: 1 })); + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + const commit = jobs.store.commit.bind(jobs.store); + let commits = 0; + // The reservation's commit is admitted, then held past the seal (its continuation sees the attempt sealed). + jobs.store.commit = (writes, fence, ctx) => { + const committed = commit(writes, fence, ctx); + if (++commits !== 2) return committed; + entered.resolve(); + return committed.then(async (seq) => { + await gate.promise; + return seq; + }); + }; + const run = await jobs.imports.start({ path: "p" }, context); + await entered.promise; + const aborting = jobs.imports.abort(run.id, context); + await delivered(); + failLive(attempted, jobs.self, new ChordError({ code: "unavailable", message: "seal after reservation" })); + gate.resolve(); + expect(await aborting).toBe("marked"); + await jobs.close(); + }); +}); + +describe("the clean stop and the close writer", () => { + it("host close writes the stopped runs back: the next start resumes them without counting", async () => { + const open = await storeFile(); + let jobs = await opened( + await open(), + (_args, ctx) => + new Promise((_, reject) => ctx.abortSignal?.addEventListener("abort", () => reject(new Error("stopped")))), + ); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.close(); + const reopened = await open(); + expect(await record(reopened, run.id)).toMatchObject({ phase: "stopped", restarts: 0 }); + jobs = await opened(reopened, async () => ({ count: 7 })); + await delivered(); + await delivered(); + expect(jobs.finished).toMatchObject([{ kind: "done", result: { count: 7 } }]); + await jobs.close(); + }); + + it("a seal (a live failure) stops the run; the retry resumes it once its old invocation settled, uncounted (R3)", async () => { + const open = await storeFile(); + let invocations = 0; + const old = Promise.withResolvers(); + const jobs = await opened(await open(), async () => { + invocations++; + if (invocations === 1) { + // Ignores its signal until released. + await old.promise; + return { count: -1 }; + } + return { count: invocations }; + }); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + failLive(attempted, jobs.self, new ChordError({ code: "unavailable", message: "boom" })); + await jobs.retry(); + await delivered(); + // The retry waits for the old invocation (the drain, bounded by closeTimeout): no second run beside it. + expect(invocations).toBe(1); + old.resolve(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 2 } }); + expect(jobs.finished.map(({ kind }) => kind)).toEqual(["done"]); + await jobs.close(); + }); + + it("a close writer that cannot commit leaves the records running: the next start counts a restart", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + await jobs.close(); + jobs = await opened(await open(), () => new Promise(() => {})); + await delivered(); + expect(await record(jobs.store, run.id)).toMatchObject({ phase: "running", restarts: 1 }); + await jobs.crash(); + }); +}); + +describe("invocations after a seal (R3, review regressions)", () => { + /** A store object whose first invocation ignores its signal until `old` resolves, the next ones run `next`. */ + async function sealed(next: (ctx: { readonly abortSignal?: AbortSignal }) => Promise) { + const open = await storeFile(); + const old = Promise.withResolvers(); + const signals: boolean[] = []; + let invocations = 0; + const jobs = await opened(await open(), async (_args, ctx) => { + signals.push(ctx.abortSignal?.aborted ?? false); + if (++invocations > 1) return next(ctx); + await old.promise; + const wrote = await ctx.state + .change((draft) => { + draft.done = 99; + }, ctx) + .then( + () => "accepted", + (error: unknown) => error, + ); + return { count: wrote === "accepted" ? -2 : -1 }; + }); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + failLive(attempted, jobs.self, new ChordError({ code: "unavailable", message: "boom" })); + return { jobs, run, old, signals, invocations: () => invocations }; + } + + it("an invocation replaced after the bounded drain writes nothing and records nothing", async () => { + const current = Promise.withResolvers(); + const { jobs, run, old, invocations } = await sealed(async () => { + await current.promise; + return { count: 2 }; + }); + await jobs.retry(); + // The drain (closeTimeout 200 ms) ends without the old invocation: the retry invokes the run again. + await sleepMs(300); + expect(invocations()).toBe(2); + old.resolve(); + await delivered(); + await delivered(); + expect(jobs.finished).toEqual([]); + expect(jobs.imports.running.value).toMatchObject([{ id: run.id, state: { done: 0 } }]); + current.resolve(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 2 } }); + await jobs.close(); + }); + + it("a resumed invocation gets a fresh abort signal", async () => { + const { jobs, run, old, signals } = await sealed(async () => ({ count: 1 })); + await jobs.retry(); + old.resolve(); + await run.outcome(context); + expect(signals).toEqual([false, false]); + await jobs.close(); + }); + + it("abort joins the resumed invocation, not the first", async () => { + const second = Promise.withResolvers(); + const { jobs, run, old } = await sealed(async () => { + await second.promise; + return { count: 2 }; + }); + old.resolve(); + await jobs.retry(); + await sleepMs(20); + let settled = false; + const aborting = jobs.imports.abort(run.id, context).then((answer) => { + settled = true; + return answer; + }); + await sleepMs(50); + expect(settled).toBe(false); + second.resolve(); + expect(await aborting).toBe("marked"); + await jobs.close(); + }); + + it("resume abandon: a run the seal stopped finishes failed { aborted, restart } without a second run", async () => { + const open = await storeFile(); + const old = Promise.withResolvers(); + let calls = 0; + const declared = defineJob("notes.import", { ...importOptions(), persist: { resume: "abandon" } }); + const jobs = await opened( + await open(), + async () => { + calls++; + if (calls === 1) await old.promise; + return { count: calls }; + }, + undefined, + declared, + ); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + failLive(attempted, jobs.self, new ChordError({ code: "unavailable", message: "boom" })); + await jobs.retry(); + old.resolve(); + expect(await run.outcome(context)).toMatchObject({ + kind: "failed", + error: { kind: "aborted", reason: "restart" }, + }); + expect(calls).toBe(1); + await jobs.close(); + }); +}); + +describe("loading (review regressions)", () => { + it("a refused recovery commit loses nothing: the retry loads the runs again and counts the crash once", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + const store = await open(); + const refused = vi.spyOn(store, "commit").mockRejectedValueOnce(new Error("refused recovery")); + let invocations = 0; + jobs = await opened(store, () => { + invocations++; + return new Promise(() => {}); + }); + expect(jobs.errors).toContainEqual(expect.objectContaining({ kind: "activation_failed" })); + refused.mockRestore(); + await jobs.retry(); + await delivered(); + await delivered(); + expect(invocations).toBe(1); + expect(jobs.imports.running.value).toMatchObject([{ id: run.id }]); + expect(await record(store, run.id)).toMatchObject({ phase: "running", restarts: 1 }); + await jobs.close(); + }); + + it("a state document that fails to load fails the activation; the retry with fixed code loads the run", async () => { + const open = await storeFile(); + const hang = (): Promise => new Promise(() => {}); + const failing = { fail: true }; + const ProgressV2 = defineState("import.progress", { + schema: obj({ done: int() }), + version: 2, + migrate: (old) => { + if (failing.fail) throw new Error("migrate failed"); + return old as { done: number }; + }, + initial: () => ({ done: 0 }), + persist: true, + checkpointWhen: () => false, + }); + let jobs = await opened(await open(), hang); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + jobs = await opened( + await open(), + hang, + undefined, + defineJob("notes.import", { ...importOptions(), state: ProgressV2 }), + ); + expect(jobs.errors.map(({ kind }) => kind)).toContain("activation_failed"); + failing.fail = false; + await jobs.retry(); + await delivered(); + expect(jobs.imports.running.value).toMatchObject([{ id: run.id, state: { done: 0 } }]); + await jobs.close(); + }); +}); + +describe("finish failures", () => { + it("are counted on the record, across a crash", async () => { + const open = await storeFile(); + let jobs = await opened( + await open(), + async () => ({ count: 1 }), + () => { + throw new Error("finish failed"); + }, + ); + const run = await jobs.imports.start({ path: "a" }, context); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(await record(jobs.store, run.id)).toMatchObject({ phase: "finishing", finishFailures: 1 }); + await jobs.crash(); + const failing = () => { + throw new Error("finish failed"); + }; + jobs = await opened(await open(), async () => ({ count: 1 }), failing); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(await record(jobs.store, run.id)).toMatchObject({ phase: "finishing", finishFailures: 2 }); + await jobs.crash(); + jobs = await opened(await open(), async () => ({ count: 1 })); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(jobs.finished).toMatchObject([{ kind: "done", result: { count: 1 } }]); + await jobs.close(); + }); +}); + +describe("abort (D7)", () => { + it("is stored and joined: marked once run settled, terminal for an unknown id", async () => { + const open = await storeFile(); + const jobs = await opened( + await open(), + (_args, ctx) => + new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", () => resolve({ count: -1 }))), + ); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + expect(await jobs.imports.abort(run.id, context)).toBe("marked"); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: -1 } }); + expect(await jobs.imports.abort(run.id, context)).toBe("terminal"); + await jobs.close(); + }); + + it("an abort-marked run not invoked yet finishes aborted without run after a crash", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + const aborting = jobs.imports.abort(run.id, context); + await delivered(); + await jobs.crash(); + void aborting.catch(() => {}); + let ran = false; + jobs = await opened(await open(), async () => { + ran = true; + return { count: 1 }; + }); + await delivered(); + await delivered(); + expect(ran).toBe(false); + expect(jobs.finished).toMatchObject([{ kind: "failed", error: { kind: "aborted", reason: "abort" } }]); + await jobs.close(); + }); +}); + +describe("abort of a finishing run (review regression)", () => { + it("stores the mark and keeps the recorded outcome", async () => { + const open = await storeFile(); + const jobs = await opened( + await open(), + async () => ({ count: 1 }), + () => { + throw new Error("retry finish"); + }, + ); + const run = await jobs.imports.start({ path: "a" }, context); + await sleepMs(50); + expect(await record(jobs.store, run.id)).toMatchObject({ phase: "finishing" }); + expect(await jobs.imports.abort(run.id, context)).toBe("marked"); + expect(await record(jobs.store, run.id)).toMatchObject({ + phase: "finishing", + abort: true, + outcome: { kind: "done", result: { count: 1 } }, + }); + await jobs.crash(); + }); +}); + +describe("abort racing an outcome", () => { + it("an abort that comes as run returns keeps its result: marked or terminal, the outcome done", async () => { + const open = await storeFile(); + const jobs = await opened(await open(), async () => ({ count: 5 })); + const run = await jobs.imports.start({ path: "a" }, context); + const aborted = await jobs.imports.abort(run.id, context); + expect(["marked", "terminal"]).toContain(aborted); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 5 } }); + await jobs.close(); + }); +}); + +describe("pinning", () => { + it("an unsettled run's state never idles: past the residency retention it is in memory, read synchronously", async () => { + const open = await storeFile(); + const touched = vi.spyOn(Residency.prototype, "touch"); + const gate = Promise.withResolvers(); + const values: number[] = []; + const jobs = await opened(await open(), async (_args, ctx) => { + await ctx.state.change((draft) => { + draft.done = 3; + }, ctx); + await gate.promise; + values.push(ctx.state.value.done); + return { count: ctx.state.value.done }; + }); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + try { + vi.advanceTimersByTime(2 * RETENTION_MS); + } finally { + vi.useRealTimers(); + } + await delivered(); + // The run's documents never entered the idle structure the retention evicts from. + expect(touched).not.toHaveBeenCalled(); + expect(jobs.imports.running.value).toMatchObject([{ id: run.id, state: { done: 3 } }]); + gate.resolve(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 3 } }); + expect(values).toEqual([3]); + touched.mockRestore(); + await jobs.close(); + }); +}); + +describe("deferred state (D8)", () => { + it("is adopted at once, held, and stored with the outcome", async () => { + const open = await storeFile(); + const gate = Promise.withResolvers(); + let seen: number | undefined; + const jobs = await opened(await open(), async (_args, ctx) => { + await ctx.state.change( + (draft) => { + draft.done = 1; + }, + ctx, + { + persistence: { kind: "deferred", withinMs: Infinity }, + }, + ); + seen = ctx.state.value.done; + await gate.promise; + return { count: 1 }; + }); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + expect(seen).toBe(1); + const state = await jobs.store.read( + (executor) => + executor.all( + "SELECT r.kind FROM chord_revisions r JOIN chord_documents d ON d.id = r.document_id WHERE d.namespace = 'chord.job-states'", + ), + context, + ); + expect(state).toEqual([{ kind: "base" }]); + gate.resolve(); + expect(await run.outcome(context)).toMatchObject({ state: { done: 1 } }); + await jobs.close(); + }); + + it("merges held ops into the next immediate change: one delta on the stored revision", async () => { + const open = await storeFile(); + const gate = Promise.withResolvers(); + const jobs = await opened(await open(), async (_args, ctx) => { + const deferred = { persistence: { kind: "deferred", withinMs: Infinity } } as const; + await ctx.state.change( + (draft) => { + draft.done = 1; + }, + ctx, + deferred, + ); + await ctx.state.change( + (draft) => { + draft.done = 2; + }, + ctx, + deferred, + ); + await ctx.state.change((draft) => { + draft.done = 3; + }, ctx); + await gate.promise; + return { count: 3 }; + }); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await delivered(); + const revisions = await jobs.store.read( + (executor) => + executor.all<{ kind: string; content: string }>( + "SELECT r.kind, r.content FROM chord_revisions r JOIN chord_documents d ON d.id = r.document_id WHERE d.namespace = 'chord.job-states' ORDER BY r.seq", + ), + context, + ); + expect(revisions.map(({ kind, content }) => [kind, JSON.parse(content)])).toEqual([ + ["base", { done: 0 }], + [ + "delta", + // The held sets of one leaf folded (`PendingOps`), then the change's own. + [ + ["s", ["done"], 2], + ["s", ["done"], 3], + ], + ], + ]); + gate.resolve(); + await run.outcome(context); + await jobs.close(); + }); + + it("folds held writes: 100 sets of one leaf keep one op; past the pending bound the next change stores a base", async () => { + const open = await storeFile(); + const store = await open(); + const changed = Promise.withResolvers(); + const deferred = { persistence: { kind: "deferred", withinMs: Infinity } } as const; + const jobs = await opened(store, async (_args, ctx) => { + for (let n = 1; n <= 100; n++) { + await ctx.state.change( + (draft) => { + draft.done = n; + }, + ctx, + deferred, + ); + } + await ctx.state.change((draft) => { + draft.done = 101; + }, ctx); + changed.resolve(); + return new Promise(() => {}); + }); + await jobs.imports.start({ path: "a" }, context); + await changed.promise; + expect((await stateRevisions(store)).at(-1)).toEqual([ + "delta", + [ + ["s", ["done"], 100], + ["s", ["done"], 101], + ], + ]); + await jobs.close(); + }); + + it("an overflow of held ops stores the next change as a base", async () => { + const open = await storeFile(); + const store = await open(); + const Items = defineState("import.progress", { + schema: obj({ done: int(), items: arr(int()) }), + version: 1, + initial: () => ({ done: 0, items: new Array(MAX_PENDING_OPS + 2).fill(0) }), + persist: true, + checkpointWhen: () => false, + }); + const declared = defineJob("notes.import", { ...importOptions(), state: Items }); + const changed = Promise.withResolvers(); + const deferred = { persistence: { kind: "deferred", withinMs: Infinity } } as const; + // The declaration's state type differs from `Import`'s; this run writes only `Items`' shape. + const jobs = await opened( + store, + async (_args, ctx) => { + const state = ctx.state as unknown as RunState<{ done: number; items: number[] }>; + // One leaf each: nothing folds, and the queue passes its bound. + for (let n = 0; n <= MAX_PENDING_OPS; n++) { + await state.change( + (draft) => { + draft.items[n] = 1; + }, + ctx, + deferred, + ); + } + await state.change((draft) => { + draft.done = 1; + }, ctx); + changed.resolve(); + return new Promise(() => {}); + }, + undefined, + declared as unknown as typeof Import, + ); + await jobs.imports.start({ path: "a" }, context); + await changed.promise; + // A new base prunes the older revisions (`history: none`): the start's base is gone. + expect(await stateRevisions(store)).toEqual([["base", expect.objectContaining({ done: 1 })]]); + await jobs.close(); + }); + + it("is stored by the close writer at a clean stop, and after withinMs by the object's timer", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), async (_args, ctx) => { + await ctx.state.change( + (draft) => { + draft.done = 4; + }, + ctx, + { persistence: { kind: "deferred", withinMs: Infinity } }, + ); + return new Promise(() => {}); + }); + await jobs.imports.start({ path: "a" }, context); + await delivered(); + await delivered(); + await jobs.close(); + jobs = await opened(await open(), async (_args, ctx) => { + expect(ctx.state.value).toEqual({ done: 4 }); + await ctx.state.change( + (draft) => { + draft.done = 5; + }, + ctx, + { persistence: { kind: "deferred", withinMs: 10 } }, + ); + return new Promise(() => {}); + }); + await new Promise((resolve) => setTimeout(resolve, 60)); + await jobs.crash(); + jobs = await opened(await open(), async (_args, ctx) => { + expect(ctx.state.value).toEqual({ done: 5 }); + return { count: 0 }; + }); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(jobs.finished).toMatchObject([{ kind: "done", state: { done: 5 } }]); + await jobs.close(); + }); + + it("refuses a negative withinMs", async () => { + const open = await storeFile(); + let refused: unknown; + const jobs = await opened(await open(), async (_args, ctx) => { + refused = await ctx.state + .change(() => {}, ctx, { persistence: { kind: "deferred", withinMs: -1 } }) + .catch((error: unknown) => error); + return { count: 0 }; + }); + const run = await jobs.imports.start({ path: "a" }, context); + await run.outcome(context); + expect(refused).toMatchObject({ code: "invalid", reason: "args" }); + await jobs.close(); + }); +}); + +describe("GC", () => { + const runs = instancesOf(JobRun, "entryFor"); + + it("settled runs, cancelled abort waits, and an ended object with a held change and its timer are collected", async () => { + const open = await storeFile(); + /** Runs one settled run and leaves one held change behind a closed host: what must be collected. */ + const ended = async (): Promise[]> => { + // Earlier tests leave crashed hosts behind: only this one's objects and runs count. + runs.splice(0); + attempted.splice(0); + const jobs = await opened(await open(), async (_args, ctx) => { + await ctx.state.change( + (draft) => { + draft.done = 1; + }, + ctx, + { persistence: { kind: "deferred", withinMs: 60_000 } }, + ); + return new Promise(() => {}); + }); + const settled = await jobs.abandoned.start({ path: "a" }, context); + await settled.outcome(context); + await jobs.imports.start({ path: "b" }, context); + await delivered(); + await delivered(); + const refs = [...runs.splice(0), ...attempted.splice(0)]; + await jobs.close(); + return refs; + }; + await expectCollected(...(await ended())); + const hanging = await opened(await open(), () => new Promise(() => {})); + const run = await hanging.imports.start({ path: "c" }, context); + await delivered(); + const cancel = withCancel(context); + let waiting: Promise | undefined = hanging.imports.abort(run.id, cancel.context).catch(() => {}); + const waitRef = new WeakRef(waiting); + await delivered(); + cancel.cancel(); + await waiting; + waiting = undefined; + await expectCollected(waitRef); + await hanging.close(); + }); +}); diff --git a/packages/chord/test/next/jobs/records.test.ts b/packages/chord/test/next/jobs/records.test.ts index 736791615..d8d2fca8b 100644 --- a/packages/chord/test/next/jobs/records.test.ts +++ b/packages/chord/test/next/jobs/records.test.ts @@ -114,7 +114,7 @@ describe("runs across attempts", () => { }); const run = await jobs[0]!.start({ text: "a" }, context); await flushed(); - jobs[0]!.abort(run.id); + void jobs[0]!.abort(run.id, context); await flushed(); expect(attempts).toEqual([1]); await restart(); @@ -176,7 +176,7 @@ describe("the object's end during an admitted commit", () => { apply: async () => { inStep.resolve(); await gate.promise; - return { kind: "dropIndex", table: "x", name: "y" }; + return [{ kind: "dropIndex", table: "x", name: "y" }]; }, }; return commit([...writes, held], fence, ctx); diff --git a/packages/chord/test/next/jobs/run.test.ts b/packages/chord/test/next/jobs/run.test.ts index 0899a33f7..1c2bb89a1 100644 --- a/packages/chord/test/next/jobs/run.test.ts +++ b/packages/chord/test/next/jobs/run.test.ts @@ -7,8 +7,8 @@ import { JobRun } from "../../../src/next/jobs/run.ts"; import { rejection } from "../support.ts"; describe("a run", () => { - it("has an entry per phase: args and state while running or finishing, neither when blocked", () => { - const run = new JobRun("r1", 2, "k", { command: "ls" }); + it("has an entry per phase: args and state while running or finishing, neither when blocked, with its reason", () => { + const run = new JobRun("r1", 2, "k", { command: "ls" }, "running"); expect(run.entryFor({ output: "" })).toEqual({ id: "r1", version: 2, @@ -18,23 +18,31 @@ describe("a run", () => { phase: "running", }); run.phase = "finishing"; - expect(new JobRun("r2", 1, undefined, { a: 1 }).entryFor(undefined)).toEqual({ + expect(new JobRun("r2", 1, undefined, { a: 1 }, "running").entryFor(undefined)).toEqual({ id: "r2", version: 1, args: { a: 1 }, phase: "running", }); run.phase = "blocked"; + run.blocked = "finish_failed"; expect(run.entryFor({ output: "x" })).toEqual({ id: "r1", version: 2, phase: "blocked", reason: "finish_failed", }); + run.blocked = "version"; + expect(run.entryFor(undefined)).toMatchObject({ phase: "blocked", reason: "version" }); + // A persisted run waiting for its reservation, or stopped, runs (again): `running`. + run.phase = "pending"; + expect(run.entryFor(undefined)).toMatchObject({ phase: "running" }); + run.phase = "stopped"; + expect(run.entryFor(undefined)).toMatchObject({ phase: "running" }); }); it("gives every waiter the outcome, late ones too; a waiter's context ends only its wait", async () => { - const run = new JobRun("r1", 1, undefined, {}); + const run = new JobRun("r1", 1, undefined, {}, "running"); const { context: waiting, cancel } = withCancel(context); const cancelled = rejection(run.handle.outcome(waiting)); cancel(new Error("left")); @@ -42,7 +50,7 @@ describe("a run", () => { const outcome = { id: "r1", args: {}, state: undefined, kind: "done", result: 1 } as const; run.resolve(outcome); expect(await run.handle.outcome(context)).toBe(outcome); - const rejected = new JobRun("r2", 1, undefined, {}); + const rejected = new JobRun("r2", 1, undefined, {}, "running"); rejected.reject(new ChordError({ code: "aborted", message: "dropped" })); expect(await rejection(rejected.handle.outcome(context))).toMatchObject({ code: "aborted" }); }); diff --git a/packages/chord/test/next/state/watchable.test.ts b/packages/chord/test/next/state/watchable.test.ts index f59f113c1..35184f31f 100644 --- a/packages/chord/test/next/state/watchable.test.ts +++ b/packages/chord/test/next/state/watchable.test.ts @@ -119,6 +119,7 @@ describe("subscribers", () => { const subscriber: Subscriber = { advance: (call) => void (call.kind === "value" && frames.push([call.value, call.ops])), end: () => {}, + endAfterDelivery: () => {}, }; source.subscribe(subscriber); expect(source.watched).toBe(true); diff --git a/packages/chord/test/next/store/allocator.test.ts b/packages/chord/test/next/store/allocator.test.ts new file mode 100644 index 000000000..518787f6d --- /dev/null +++ b/packages/chord/test/next/store/allocator.test.ts @@ -0,0 +1,24 @@ +// Tests of the id allocator (slice 6b D3, §1.2): from 2, raised by explicit ids, saturating at the safe range. + +import { describe, expect, it } from "vitest"; +import { FIRST_ID, IdAllocator } from "../../../src/next/store/allocator.ts"; + +describe("the allocator", () => { + it("mints from its value, and an explicit id raises it above itself, never lowering it", () => { + const ids = new IdAllocator(FIRST_ID); + expect([ids.mint(), ids.mint()]).toEqual([2, 3]); + ids.raise(100); + expect(ids.mint()).toBe(101); + ids.raise(50); + expect(ids.value).toBe(102); + }); + + it("saturates at Number.MAX_SAFE_INTEGER: that id may be written, every later mint is refused", () => { + const ids = new IdAllocator(Number.MAX_SAFE_INTEGER - 1); + expect(ids.mint()).toBe(Number.MAX_SAFE_INTEGER - 1); + expect(() => ids.mint()).toThrow("ID space is exhausted"); + ids.raise(Number.MAX_SAFE_INTEGER); + expect(ids.value).toBe(Number.MAX_SAFE_INTEGER); + expect(() => ids.mint()).toThrow("ID space is exhausted"); + }); +}); diff --git a/packages/chord/test/next/store/applier.test.ts b/packages/chord/test/next/store/applier.test.ts index 00d8e61f5..5cad8bfe8 100644 --- a/packages/chord/test/next/store/applier.test.ts +++ b/packages/chord/test/next/store/applier.test.ts @@ -3,14 +3,16 @@ // as stored, booleans, json(...) cells as JSON text) and documents with their ids kept. import { describe, expect, it } from "vitest"; -import { type AppliedWrite, applyWrite } from "../../../src/next/store/applier.ts"; +import { IdAllocator } from "../../../src/next/store/allocator.ts"; +import { applyWrite } from "../../../src/next/store/applier.ts"; import { Step } from "../../../src/next/store/step.ts"; +import type { StoreWrite } from "../../../src/next/store/writes.ts"; import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; import { rejection } from "../support.ts"; -const TABLE = "client:app.store:edits"; +const TABLE = "app.store:edits"; -const table: AppliedWrite = { +const table: StoreWrite = { kind: "table", table: TABLE, version: 2, @@ -26,11 +28,11 @@ const table: AppliedWrite = { }; /** Applies `writes` in one transaction of a fresh database; resolves with the rows of the table. */ -async function applied(...writes: AppliedWrite[]): Promise { +async function applied(...writes: StoreWrite[]): Promise { const db = await openNodeSqliteDatabase(":memory:"); try { return await db.transaction(async (executor) => { - const step = new Step(executor, 1, 1, new Map()); + const step = new Step(executor, 1, new IdAllocator(2), new Map()); for (const write of writes) await applyWrite(step, write, "test"); return executor.all(`SELECT * FROM "${TABLE}"`); }); @@ -51,7 +53,7 @@ describe("applied writes", () => { expect(rows).toEqual([{ id: "a", done: 1, note: null, meta: '{"x":[1]}', meta$v: 1, $v: 1 }]); }); - it.each<[string, AppliedWrite[], string]>([ + it.each<[string, StoreWrite[], string]>([ [ "a put for a table not recorded", [{ kind: "put", table: TABLE, row: {}, version: 1, columns: {} }], @@ -59,17 +61,17 @@ describe("applied writes", () => { ], [ "a key over an opt column", - [{ ...table, key: ["note"] } as AppliedWrite], + [{ ...table, key: ["note"] } as StoreWrite], '"note" is not a non-opt str or int column', ], [ "a json column without a fingerprint", - [{ ...table, columns: { id: { type: "str" }, meta: { type: "json", version: 1 } } } as AppliedWrite], + [{ ...table, columns: { id: { type: "str" }, meta: { type: "json", version: 1 } } } as StoreWrite], "a json(...) column has a version and a fingerprint", ], [ "a default of another type", - [{ ...table, columns: { id: { type: "str" }, n: { type: "int", default: "x" } } } as AppliedWrite], + [{ ...table, columns: { id: { type: "str" }, n: { type: "int", default: "x" } } } as StoreWrite], "is not a int value", ], [ diff --git a/packages/chord/test/next/store/apply-batch.test.ts b/packages/chord/test/next/store/apply-batch.test.ts new file mode 100644 index 000000000..762b4f4d6 --- /dev/null +++ b/packages/chord/test/next/store/apply-batch.test.ts @@ -0,0 +1,290 @@ +// Tests of batch apply (slice 6b §1.6, S5, S17): a store's outbox replayed into an empty store gives the same archive, +// manifest included; every write kind (an adapter's create-plus-retire and change-plus-retire too); the refusals (claims held, seqs that do not follow, writes the records refuse) +// and the divergence of a recorded count. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, str } from "../../../src/next/contracts/index.ts"; +import { applyBatch } from "../../../src/next/store/apply-batch.ts"; +import { + changeDocument, + createDocument, + type DocumentRecord, + type DocumentWrite, + insertDocument, + readDocument, + retireDocument, + retireRecord, + writeContent, +} from "../../../src/next/store/engine.ts"; +import { add, type ColumnRef, lt, prefix } from "../../../src/next/store/expression.ts"; +import type { OutboxCommit } from "../../../src/next/store/outbox.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import { evolveTable } from "../../../src/next/store/table-evolve.ts"; +import { prepareTables } from "../../../src/next/store/table-migration.ts"; +import { archiveStore, exported, memoryStore, rejection, tableIn } from "../support.ts"; + +const OPEN = { admit: () => true, adopt: () => {} }; + +/** The column references a table's callbacks get (the changes are untyped below the transaction). */ +type Refs = { readonly [column: string]: ColumnRef | undefined }; + +const CountsV1 = defineTable("counts", { + version: 1, + columns: { id: str(), n: int(), tag: str() }, + key: ["id"], + indexes: { by_tag: ["tag"] }, +}); + +const CountsV2 = evolveTable(CountsV1, (t) => t.add("extra", int({ default: 0 })).dropIndex("by_tag")); + +/** A store with an outbox from its first commit holding every write kind: an archive store, then counted writes. */ +async function everyKind(): Promise { + const store = await archiveStore({ outbox: true }); + const counts = await tableIn(store, CountsV1); + const staged: StagedWrite[] = []; + const changes = new TableChanges(counts, staged); + for (let i = 0; i < 10; i++) changes.put({ id: `c${i}`, n: i, tag: i % 2 === 0 ? "even" : "odd" }); + await store.commit(staged, OPEN, context); + const bulk: StagedWrite[] = []; + const more = new TableChanges(counts, bulk); + more.update({ id: "c1" }, (c: Refs) => ({ n: add(c.n!, 10) })); + more.delete({ id: "c2" }); + more.updateWhere((c: Refs) => prefix(c.tag!, "ev"), { tag: "EVEN" }); + more.deleteWhere((c: Refs) => lt(c.n!, 1)); + await store.commit(bulk, OPEN, context); + await prepareTables([{ definition: CountsV2, table: "app.store:counts" }], store, OPEN, "app.store", context); + const created: number[] = []; + const fence = { admit: () => true, adopt: (_: number, ids: readonly number[]) => void created.push(...ids) }; + const address = { owner: "app.store", namespace: "chord", kind: "replayed", key: "x" }; + const base = { version: 1, fingerprint: "f1", value: { n: 0 } }; + await store.commit([createDocument(address, base, '{"n":0}')], fence, context); + const ops = [["s", ["n"], 1]] as const; + await store.commit( + [changeDocument(created[0]!, 1, { kind: "delta", version: 1, ops }, JSON.stringify(ops))], + OPEN, + context, + ); + await store.commit([retireDocument(created[0]!, 2)], OPEN, context); + await durableShaped(store); + return store; +} + +/** + * What only an adapter writes (Durable's commits): a rewindable creation retired in its own commit (an empty + * lifetime), and a change with the retirement of the same record in one commit. + */ +async function durableShaped(store: ChordStore): Promise { + const address = { owner: "app.durable", namespace: "session", kind: "notes" }; + const base = { version: 1, fingerprint: "", value: { n: 0 } }; + /** A record created at `seq`, retired then too when `retired`. */ + const made = (id: number, key: string, seq: number, retired: boolean): DocumentRecord => ({ + id, + ...address, + key, + history: "rewindable", + createdAt: seq, + ...(retired ? { retiredAt: seq } : {}), + meta: "{}", + revision: 1, + version: 1, + fingerprint: "", + }); + const create = (key: string, retired: boolean): StagedWrite => ({ + order: "write", + apply: async (step) => { + const record = made(step.ids.mint(), key, step.seq, retired); + await insertDocument(step, record, '{"n":0}'); + return [{ kind: "document.create", record, base }]; + }, + }); + await store.commit([create("empty", true), create("kept", false)], OPEN, context); + const kept = (await store.read( + (executor) => executor.get<{ id: number }>("SELECT id FROM chord_documents WHERE key = 'kept'"), + context, + ))!.id; + const ops = [["s", ["n"], 1]] as const; + const changeAndRetire: StagedWrite = { + order: "write", + apply: async (step) => { + const record = (await readDocument(step, kept))!; + const content = { kind: "delta", version: 1, ops } as const; + await writeContent(step, kept, "rewindable", 1, content, JSON.stringify(ops)); + await retireRecord(step, record); + const writes: DocumentWrite[] = [ + { kind: "document.change", id: kept, revision: 1, content }, + { kind: "document.retire", id: kept, revision: 2 }, + ]; + return writes; + }, + }; + await store.commit([changeAndRetire], OPEN, context); +} + +/** Every commit of `store`'s outbox. */ +async function commits(store: ChordStore): Promise { + return store.outbox!.read(store.outbox!.floor, 10_000, context); +} + +describe("replay", () => { + it("a store's outbox replayed into an empty store gives the same archive, manifest included", async () => { + const source = await everyKind(); + const all = await commits(source); + const kinds = new Set(all.flatMap(({ writes }) => writes.map(({ kind }) => kind))); + expect([...kinds].sort()).toEqual( + [ + "alter", + "document.change", + "document.create", + "document.retire", + "dropIndex", + "index", + "put", + "table", + "update", + "updateWhere", + "delete", + "deleteWhere", + ].sort(), + ); + const replica = await memoryStore(); + await applyBatch(replica, all.slice(0, 3), context); + await applyBatch(replica, all.slice(3), context); + expect(await exported(replica)).toEqual(await exported(source)); + expect([replica.lastSeq, replica.nextId]).toEqual([source.lastSeq, source.nextId]); + }); + + it("restores the allocator: a replica mints above every id it holds", async () => { + const source = await everyKind(); + const replica = await memoryStore(); + await applyBatch(replica, await commits(source), context); + expect(replica.mintId()).toBe(source.nextId); + }); + + it("writes the applied commits to the replica's own outbox when it has one", async () => { + const source = await everyKind(); + const all = await commits(source); + const replica = await memoryStore({ outbox: true }); + await applyBatch(replica, all, context); + expect(await commits(replica)).toEqual(all); + }); +}); + +describe("refusals", () => { + it("while an owner holds a claim on the store (its memory would go stale)", async () => { + const replica = await memoryStore(); + replica.claims.documents("app.store", "chord", { label: "an object" }); + expect(await rejection(applyBatch(replica, [], context))).toMatchObject({ code: "invalid", reason: "args" }); + }); + + it("a claim taken between the call and its turn in the queue refuses the batch; a claim is refused while it applies", async () => { + const replica = await memoryStore(); + const held = await replica.take(() => {}); + const replaying = applyBatch(replica, [], context); + const owner = { label: "an object" }; + replica.claims.documents("app.store", "chord", owner); + held(); + expect(await rejection(replaying)).toMatchObject({ code: "invalid", reason: "args" }); + expect(replica.lastSeq).toBe(0); + replica.claims.release(owner); + const all = await commits(await everyKind()); + let applied = false; + const applying = applyBatch(replica, all, context).then(() => { + applied = true; + }); + // A claim made and released in one turn holds nothing; once the batch applies (across awaits), one is refused. + let refused: unknown; + while (!applied && refused === undefined) { + await Promise.resolve(); + try { + replica.claims.documents("app.store", "chord", owner); + replica.claims.release(owner); + } catch (error) { + refused = error; + } + } + expect(refused).toMatchObject({ code: "invalid", reason: "args" }); + await applying; + expect(replica.lastSeq).toBe(all.at(-1)!.seq); + replica.claims.documents("app.store", "chord", owner); + }); + + it("a batch queued behind one that diverges is refused at its turn", async () => { + const all = await commits(await everyKind()); + const diverging = structuredClone(all); + for (const commit of diverging) { + for (const write of commit.writes) if (write.kind === "updateWhere") (write as { count: number }).count += 1; + } + const replica = await memoryStore(); + const first = rejection(applyBatch(replica, diverging, context)); + const second = rejection(applyBatch(replica, all, context)); + expect(String(await first)).toContain("recorded"); + expect(await second).toMatchObject({ code: "invalid", reason: "args" }); + expect(replica.lastSeq).toBe(0); + }); + + it("a commit whose next id is not above a document it creates (a saturated id aside)", async () => { + const replica = await memoryStore(); + const record = { + id: 100, + owner: "app.store", + namespace: "chord", + kind: "x", + history: "rewindable" as const, + createdAt: 1, + revision: 1, + version: 1, + fingerprint: "", + }; + const base = { version: 1, fingerprint: "", value: {} }; + const create = { kind: "document.create", record, base } as const; + expect(await rejection(applyBatch(replica, [{ seq: 1, nextId: 2, writes: [create] }], context))).toMatchObject({ + code: "invalid", + reason: "args", + }); + expect(await rejection(applyBatch(replica, [{ seq: 1, nextId: 100, writes: [create] }], context))).toMatchObject({ + reason: "args", + }); + await applyBatch(replica, [{ seq: 1, nextId: 101, writes: [create] }], context); + const saturated = { ...create, record: { ...record, id: Number.MAX_SAFE_INTEGER, key: "max" } }; + await applyBatch(replica, [{ seq: 2, nextId: Number.MAX_SAFE_INTEGER, writes: [saturated] }], context); + expect(replica.lastSeq).toBe(2); + }); + + it("a first commit that does not follow lastSeq, or one that does not follow the one before; nothing applied", async () => { + const all = await commits(await everyKind()); + const replica = await memoryStore(); + expect(await rejection(applyBatch(replica, all.slice(1), context))).toMatchObject({ reason: "args" }); + expect(await rejection(applyBatch(replica, [all[0]!, all[2]!], context))).toMatchObject({ reason: "args" }); + expect(replica.lastSeq).toBe(0); + await applyBatch(replica, all, context); + expect(replica.lastSeq).toBe(all.at(-1)!.seq); + }); + + it("a write the records refuse rolls the batch back, and the store goes on", async () => { + const all = await commits(await everyKind()); + const replica = await memoryStore(); + const broken = structuredClone(all); + (broken[0]!.writes[0] as { table: string }).table = "no:such:table"; + expect(await rejection(applyBatch(replica, broken, context))).toMatchObject({ code: "invalid", reason: "args" }); + const forged = structuredClone(all); + (forged[0]!.writes as unknown[]).push({ kind: "put", table: "x", row: {}, version: 0 }); + expect(await rejection(applyBatch(replica, forged, context))).toMatchObject({ code: "invalid", reason: "args" }); + await applyBatch(replica, all, context); + }); + + it("a recorded count the store does not reproduce stops applying and marks the store diverged (S17)", async () => { + const all = await commits(await everyKind()); + const index = all.findIndex(({ writes }) => writes.some(({ kind }) => kind === "updateWhere")); + const diverging = structuredClone(all); + const write = diverging[index]!.writes.find(({ kind }) => kind === "updateWhere") as { count: number }; + write.count += 1; + const replica = await memoryStore(); + expect(String(await rejection(applyBatch(replica, diverging, context)))).toContain("recorded"); + expect(await rejection(applyBatch(replica, all, context))).toMatchObject({ code: "invalid", reason: "args" }); + expect(replica.lastSeq).toBe(0); + }); +}); diff --git a/packages/chord/test/next/store/catalog.test.ts b/packages/chord/test/next/store/catalog.test.ts index 451c45cb2..11761b30a 100644 --- a/packages/chord/test/next/store/catalog.test.ts +++ b/packages/chord/test/next/store/catalog.test.ts @@ -20,7 +20,7 @@ const Edits = defineTable("edits", { key: ["id"], indexes: { by_at: ["at"] }, }); -const TABLE = "session:app.store:edits"; +const TABLE = "app.store:edits"; describe("the catalog", () => { it("is read once at open: a reopened store knows its tables", async () => { @@ -51,14 +51,12 @@ describe("the catalog", () => { apply: (step) => step .run("INSERT INTO no_such_table VALUES (1)") - .then(() => ({ kind: "dropIndex", table: TABLE, name: "x" })), + .then(() => [{ kind: "dropIndex", table: TABLE, name: "x" }]), }; const table = stageTable({ kind: "table", table: TABLE, version, state: "ready", columns, key, fingerprint }); - await expect( - store.commit([table, failing], { admit: () => true, adopt: () => {} }, context), - ).rejects.toMatchObject({ - reason: "storage_failed", - }); + await expect(store.commit([table, failing], { admit: () => true, adopt: () => {} }, context)).rejects.toThrow( + "no such table", + ); expect(store.table(TABLE)).toBeUndefined(); await store.close(context); }); diff --git a/packages/chord/test/next/store/claims.test.ts b/packages/chord/test/next/store/claims.test.ts new file mode 100644 index 000000000..3974487e3 --- /dev/null +++ b/packages/chord/test/next/store/claims.test.ts @@ -0,0 +1,72 @@ +// Tests of claims (slice 6b "owner names and claims", "claims per table and per document namespace"): a table by its +// SQL name and documents by owner and namespace, once per store; released by their claimant's end; none taken while a +// batch applies. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { Claims } from "../../../src/next/store/claims.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { loaded, memoryStore } from "../support.ts"; + +describe("claims", () => { + it("refuse a second claimant of a table or of an owner's namespace; one claimant may claim again", () => { + const claims = new Claims(); + const a = { label: "A" }; + const b = { label: "B" }; + claims.table("x:edits", a); + claims.table("x:edits", a); + claims.documents("x", "chord", a); + expect(() => claims.table("x:edits", b)).toThrow("claimed by A"); + expect(() => claims.documents("x", "chord", b)).toThrow("claimed by A"); + // Another namespace of the same owner, another table: no collision. + claims.documents("x", "session", b); + claims.table("x:entries", b); + expect(claims.held).toBe(true); + claims.release(a); + claims.table("x:edits", b); + claims.release(b); + expect(claims.held).toBe(false); + }); + + it("are refused while locked (a batch applies), taken again once unlocked", () => { + const claims = new Claims(); + const owner = { label: "A" }; + const unlock = claims.lock(); + expect(() => claims.table("x:edits", owner)).toThrow("while a batch applies"); + expect(() => claims.documents("x", "chord", owner)).toThrow("while a batch applies"); + expect(claims.held).toBe(false); + unlock(); + claims.table("x:edits", owner); + expect(claims.held).toBe(true); + }); + + it("an object claims its tables and documents when it declares them, and releases them at its end", async () => { + const store = await memoryStore(); + const Note = defineState("note", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: 0 }), + persist: true, + checkpointWhen: () => true, + }); + const Edits = defineTable("edits", { version: 1, columns: { id: str() }, key: ["id"] }); + const Notes = local("notes.store", { version: 1, params: {}, members: {} }); + const { host } = await loaded( + (h) => + h.serve(Notes, (_, self) => { + self.state(Note); + self.table(Edits); + return {}; + }), + { store }, + ); + await host.get(Notes, {}, context); + expect(() => store.claims.table("notes.store:edits", { label: "other" })).toThrow("claimed by notes.store"); + expect(() => store.claims.documents("notes.store", "chord", { label: "other" })).toThrow("claimed"); + await host.close(context); + expect(store.claims.held).toBe(false); + store.claims.table("notes.store:edits", { label: "other" }); + }); +}); diff --git a/packages/chord/test/next/store/engine.test.ts b/packages/chord/test/next/store/engine.test.ts index 78a41dab0..00b28eae4 100644 --- a/packages/chord/test/next/store/engine.test.ts +++ b/packages/chord/test/next/store/engine.test.ts @@ -1,4 +1,5 @@ -// The engine (slice 6a §10, §11): persisted documents as bases and deltas, their keys in code point order, retirement. +// The engine (slice 6a §10, §11, slice 6b §1.5): documents as bases and deltas, their keys in code point order, the +// history rules at a new base and at retirement (R1), reads at a point, records inserted as they are. import { describe, expect, it } from "vitest"; import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; @@ -9,10 +10,17 @@ import { changeDocument, createDocument, type DocumentAddress, + type DocumentHistory, + type DocumentRecord, findDocument, + insertDocument, listDocumentKeys, + listDocuments, loadDocument, + readDocument, retireDocument, + retireRecord, + writeContent, } from "../../../src/next/store/engine.ts"; import { importStore } from "../../../src/next/store/import.ts"; import { TABLES } from "../../../src/next/store/schema.ts"; @@ -21,7 +29,7 @@ import { type ChordStore, openStore } from "../../../src/next/store/store.ts"; import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; import { exported, rejection } from "../support.ts"; -const OWNER = { namespace: "chord", owner: "session:app.store", kind: "notes" }; +const OWNER = { owner: "app.store", namespace: "chord", kind: "notes" }; /** A store on `:memory:` (or `on`) and a commit helper resolving with the created ids. */ async function engine(on?: ChordStore) { @@ -47,7 +55,7 @@ async function engine(on?: ChordStore) { fingerprint: `f${version}`.padEnd(16, "0"), value: value as never, }); - const create = (key: string, value: object) => createDocument(at(key), "latest", base(value), JSON.stringify(value)); + const create = (key: string, value: object) => createDocument(at(key), base(value), JSON.stringify(value)); const delta = (id: number, revision: number, ops: Op[]) => changeDocument(id, revision, { kind: "delta", version: 1, ops }, JSON.stringify(ops)); const read = (fn: Parameters[0]) => store.read(fn, context) as Promise; @@ -60,6 +68,16 @@ async function reimported(store: ChordStore, edit: (parts: ArchivePart[]) => unk return (await importStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), parts, context)) as ChordStore; } +/** A staged write applying `apply` in the step (an adapter's composite). */ +function staged(apply: StagedWrite["apply"]): StagedWrite { + return { order: "write", apply }; +} + +/** An adapter's record of `history` at `key`, with an explicit id and the step's seq. */ +function record(id: number, key: string, history: DocumentHistory, createdAt: number): DocumentRecord { + return { id, ...OWNER, key, history, createdAt, revision: 1, version: 1, fingerprint: "" }; +} + const LIMIT = Number.MAX_SAFE_INTEGER; describe("documents", () => { @@ -68,12 +86,12 @@ describe("documents", () => { const [id] = await commit(create("a", { n: 0 })); await commit(delta(id!, 1, [["s", ["n"], 1]])); await commit(delta(id!, 2, [["s", ["n"], 2]])); - const record = (await read>>((e) => findDocument(e, at("a"))))!; - expect(record).toMatchObject({ id, revision: 3, version: 1, history: "latest", createdAt: 1 }); - expect(await read((e) => loadDocument(e, record))).toMatchObject({ value: { n: 2 }, deltasSinceBase: 2 }); + const found = (await read((e) => findDocument(e, at("a"))))!; + expect(found).toMatchObject({ id, revision: 3, version: 1, history: "none", createdAt: 1 }); + expect(await read((e) => loadDocument(e, found))).toMatchObject({ value: { n: 2 }, deltasSinceBase: 2 }); const next = base({ m: 9 }, 2); await commit(changeDocument(id!, 3, { kind: "base", ...next }, JSON.stringify(next.value))); - const rebased = (await read>>((e) => findDocument(e, at("a"))))!; + const rebased = (await read((e) => findDocument(e, at("a"))))!; expect(rebased).toMatchObject({ revision: 4, version: 2, fingerprint: next.fingerprint }); expect(await read((e) => loadDocument(e, rebased))).toMatchObject({ value: { m: 9 }, deltasSinceBase: 0 }); expect(await read((e) => e.all(`SELECT seq FROM ${TABLES.revisions}`))).toEqual([{ seq: 4 }]); @@ -86,45 +104,152 @@ describe("documents", () => { expect(await read((e) => listDocumentKeys(e, OWNER, "b", 1))).toEqual(["\uFF5E"]); }); - it("retire a latest record with its revisions (rule 19); the address takes a new incarnation", async () => { + it("retire a none record with its row and revisions (R1); the address takes a new incarnation", async () => { const { commit, at, create, read } = await engine(); const [id] = await commit(create("a", { n: 0 })); const [next] = await commit(retireDocument(id!, 1), create("a", { n: 1 })); - expect(next).toBe(2); - expect(await read((e) => e.all(`SELECT id FROM ${TABLES.documents}`))).toEqual([{ id: 2 }]); - expect(await read((e) => e.all(`SELECT document_id FROM ${TABLES.revisions}`))).toEqual([{ document_id: 2 }]); - expect(await read((e) => findDocument(e, at("a")))).toMatchObject({ id: 2 }); + expect(next).toBe(3); + expect(await read((e) => e.all(`SELECT id FROM ${TABLES.documents}`))).toEqual([{ id: 3 }]); + expect(await read((e) => e.all(`SELECT document_id FROM ${TABLES.revisions}`))).toEqual([{ document_id: 3 }]); + expect(await read((e) => findDocument(e, at("a")))).toMatchObject({ id: 3 }); }); - it("refuse two live records at one address and a delta on another version: storage_failed", async () => { + it("refuse two live records at one address and a delta on another version: the step is refused", async () => { const first = await engine(); await first.commit(first.create("a", {})); - expect(await rejection(first.commit(first.create("a", {})))).toMatchObject({ reason: "storage_failed" }); + expect(String(await rejection(first.commit(first.create("a", {}))))).toContain("UNIQUE"); const second = await engine(); const [id] = await second.commit(second.create("a", {})); const crossing = changeDocument(id!, 1, { kind: "delta", version: 2, ops: [] }, "[]"); - expect(await rejection(second.commit(crossing))).toMatchObject({ reason: "storage_failed" }); + expect(String(await rejection(second.commit(crossing)))).toContain("not live at revision 1"); + expect(await second.read(async () => 1)).toBe(1); + }); +}); + +describe("history rules (R1) and reads at a point", () => { + /** Creates adapter record `id` at "a" (its base `{ n: 0 }`), then changes and retires it in later commits. */ + async function lifetime(history: DocumentHistory) { + const { store, commit, at, read } = await engine(); + await commit( + staged(async (step) => { + const made = record(10, "a", history, step.seq); + await insertDocument(step, made, '{"n":0}'); + return []; + }), + ); + await commit( + staged(async (step) => { + const content = { kind: "base", version: 1, fingerprint: "", value: { n: 1 } } as const; + await writeContent(step, 10, history, 1, content, '{"n":1}'); + return []; + }), + ); + await commit( + staged(async (step) => { + await writeContent( + step, + 10, + history, + 2, + { kind: "delta", version: 1, ops: [["s", ["n"], 2]] }, + '[["s",["n"],2]]', + ); + return []; + }), + ); + // An adapter's retirement (Durable's): the record read in the step, retired by its history's rule. + await commit( + staged(async (step) => { + await retireRecord(step, (await readDocument(step, 10))!); + return []; + }), + ); + return { store, read, at }; + } + + it("keep a retired latest record's row without revisions, and every revision of a rewindable one", async () => { + const latest = await lifetime("latest"); + expect(await latest.read((e) => e.all(`SELECT id, retired_at FROM ${TABLES.documents}`))).toEqual([ + { id: 10, retired_at: 4 }, + ]); + expect(await latest.read((e) => e.all(`SELECT seq FROM ${TABLES.revisions}`))).toEqual([]); + const rewindable = await lifetime("rewindable"); + expect(await rewindable.read((e) => e.all(`SELECT seq, kind FROM ${TABLES.revisions} ORDER BY seq`))).toEqual([ + { seq: 1, kind: "base" }, + { seq: 2, kind: "base" }, + { seq: 3, kind: "delta" }, + ]); + }); + + it("find a record at a point by its half-open lifetime, and load a rewindable one at any point inside it", async () => { + const { read, at } = await lifetime("rewindable"); + const found = await Promise.all( + [0, 1, 3, 4, "current" as const].map((point) => + read((e) => findDocument(e, at("a"), point)), + ), + ); + expect(found.map((each) => each?.id)).toEqual([undefined, 10, 10, undefined, undefined]); + const record = found[1]!; + expect(record.retiredAt).toBe(4); + for (const [point, n] of [ + [1, 0], + [2, 1], + [3, 2], + ] as const) { + expect(await read((e) => loadDocument(e, record, point))).toMatchObject({ value: { n }, version: 1 }); + } + expect(await read((e) => listDocuments(e, OWNER, 3, 0, 10))).toMatchObject([{ id: 10 }]); + expect(await read((e) => listDocuments(e, OWNER, "current", 0, 10))).toEqual([]); + }); + + it("find a retired latest record at a point, whose content is gone", async () => { + const { read, at } = await lifetime("latest"); + const found = (await read((e) => findDocument(e, at("a"), 2)))!; + expect(found).toMatchObject({ id: 10, retiredAt: 4, history: "latest" }); + // A read returns data (the store stops on a read's throw, slice 6b §1.4): the outcome as a value. + const outcome = read((e) => + loadDocument(e, found, 2).then( + () => "loaded", + (error: unknown) => String(error), + ), + ); + expect(await outcome).toContain("no base"); + }); + + it("insert a create-plus-retire as an empty lifetime that never holds the live address", async () => { + const { commit, read, at, create } = await engine(); + const [live] = await commit(create("a", {})); + await commit( + staged(async (step) => { + await insertDocument(step, { ...record(20, "a", "latest", step.seq), retiredAt: step.seq }, "{}"); + return []; + }), + ); + expect(await read((e) => findDocument(e, at("a")))).toMatchObject({ id: live }); + expect(await read((e) => e.all(`SELECT document_id FROM ${TABLES.revisions} WHERE document_id = 20`))).toEqual( + [], + ); }); }); describe("counters at the safe-integer limit", () => { - it("allocate document ids up to the safe range after an import near it; past it a commit fails storage_failed", async () => { + it("allocate document ids up to the safe range after an import near it; past it the commit is refused", async () => { const { store: source } = await engine(); const near = (next: number) => - reimported(source, ([manifest, ...rest]) => [{ ...manifest, nextDocumentId: next }, ...rest]); + reimported(source, ([manifest, ...rest]) => [{ ...manifest, nextId: next }, ...rest]); const { store, commit, create } = await engine(await near(LIMIT - 1)); expect(await commit(create("a", { n: 0 }))).toEqual([LIMIT - 1]); - expect(store.nextDocumentId).toBe(LIMIT); - expect(await rejection(commit(create("b", { n: 0 })))).toMatchObject({ reason: "storage_failed" }); + expect(store.nextId).toBe(LIMIT); + expect(String(await rejection(commit(create("b", { n: 0 }))))).toContain("ID space is exhausted"); // Two creations in one commit where only one id is left: nothing of it is written. const both = await engine(await near(LIMIT - 1)); - expect(await rejection(both.commit(both.create("a", { n: 0 }), both.create("b", { n: 0 })))).toMatchObject({ - reason: "storage_failed", - }); - expect([both.store.lastSeq, both.store.nextDocumentId]).toEqual([0, LIMIT - 1]); + expect(String(await rejection(both.commit(both.create("a", { n: 0 }), both.create("b", { n: 0 }))))).toContain( + "ID space is exhausted", + ); + expect([both.store.lastSeq, both.store.nextId]).toEqual([0, LIMIT - 1]); }); - it("count revisions up to the safe range after an import near it; a change past it fails storage_failed", async () => { + it("count revisions up to the safe range after an import near it; a change past it is refused", async () => { const { store: source, commit: seed, create } = await engine(); await seed(create("a", { n: 0 })); const store = await reimported(source, (parts) => @@ -133,10 +258,8 @@ describe("counters at the safe-integer limit", () => { ), ); const { commit, delta, read, at } = await engine(store); - const record = (await read>>((e) => findDocument(e, at("a"))))!; - await commit(delta(record.id, LIMIT - 1, [["s", ["n"], 1]])); - expect(await rejection(commit(delta(record.id, LIMIT, [["s", ["n"], 2]])))).toMatchObject({ - reason: "storage_failed", - }); + const found = (await read((e) => findDocument(e, at("a"))))!; + await commit(delta(found.id, LIMIT - 1, [["s", ["n"], 1]])); + expect(String(await rejection(commit(delta(found.id, LIMIT, [["s", ["n"], 2]]))))).toContain("safe range"); }); }); diff --git a/packages/chord/test/next/store/export.test.ts b/packages/chord/test/next/store/export.test.ts index e0e743023..9a8d70076 100644 --- a/packages/chord/test/next/store/export.test.ts +++ b/packages/chord/test/next/store/export.test.ts @@ -7,6 +7,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; import { int, str } from "../../../src/next/contracts/index.ts"; import { ARCHIVE_FORMAT } from "../../../src/next/store/archive.ts"; +import { insertDocument, insertRevision } from "../../../src/next/store/engine.ts"; import { EXPORT_PAGE } from "../../../src/next/store/export.ts"; import { STORE_SCHEMA } from "../../../src/next/store/schema.ts"; import type { StagedWrite } from "../../../src/next/store/step.ts"; @@ -32,7 +33,7 @@ afterEach(async () => { }); describe("parts", () => { - it("come in archive order: manifest, per table its table write, puts and indexes, the live documents, end", async () => { + it("come in archive order: manifest, per table its table write, puts and indexes, each record and its revisions, end", async () => { store = await archiveStore(); const parts = await exported(store); expect(parts[0]).toEqual({ @@ -40,11 +41,11 @@ describe("parts", () => { format: ARCHIVE_FORMAT, schema: STORE_SCHEMA, lastSeq: store.lastSeq, - nextDocumentId: store.nextDocumentId, + nextId: store.nextId, }); const kinds = parts.map((part) => part.kind === "put" || part.kind === "table" || part.kind === "index" - ? `${part.kind} ${part.table.split(":")[2]}` + ? `${part.kind} ${part.table.split(":")[1]}` : part.kind, ); const runs: [string, number][] = []; @@ -60,27 +61,32 @@ describe("parts", () => { ["index edits", 1], ["table notes", 1], ["put notes", 150], - ["document.create", 3], + ["document.create", 1], + ["document.revision", 1], + ["document.create", 1], + ["document.revision", 3], + ["document.create", 1], + ["document.revision", 2], ["end", 1], ]); expect(parts.at(-1)).toEqual({ kind: "end", writes: parts.length - 2 }); }); - it("are lossless: stored values and versions, NULL cells absent, a migrating table's state, deltas folded into the base", async () => { + it("are lossless: stored values and versions, NULL cells absent, a migrating table's state, revisions as stored", async () => { store = await archiveStore(); const parts = await exported(store); const edits = parts.filter((part) => part.kind === "put" && part.table.endsWith(":edits")); expect(edits.slice(0, 2)).toEqual([ { kind: "put", - table: "client:app.store:edits", + table: "app.store:edits", row: { id: "e000", path: "p/0", at: 0, note: "n0", meta: { a: 0 }, done: false }, version: 1, columns: { meta: 1 }, }, { kind: "put", - table: "client:app.store:edits", + table: "app.store:edits", row: { id: "e001", path: "p/1", at: 1, done: false }, version: 1, columns: {}, @@ -92,20 +98,67 @@ describe("parts", () => { }); const notes = parts.filter((part) => part.kind === "put" && part.table.endsWith(":notes")); expect(notes.map((part) => (part.kind === "put" ? part.version : 0)).filter((v) => v === 2)).toHaveLength(100); - const documents = parts.filter((part) => part.kind === "document.create"); - expect( - documents.map((part) => - part.kind === "document.create" ? [part.record.key, part.record.revision, part.base.value] : [], - ), - ).toEqual([ + const documents = parts.flatMap((part): unknown[][] => + part.kind === "document.create" + ? [[part.record.key, part.record.revision, part.record.history]] + : part.kind === "document.revision" + ? [[part.content.kind, part.content.kind === "base" ? part.content.value : part.content.ops]] + : [], + ); + expect(documents).toEqual([ // A singleton's record has no key. - [undefined, 1, { title: "" }], - ["a", 3, { title: "a3" }], - ["b", 2, { title: "b2" }], + [undefined, 1, "none"], + ["base", { title: "" }], + ["a", 3, "none"], + ["base", { title: "a" }], + ["delta", [["s", ["title"], "a2"]]], + ["delta", [["s", ["title"], "a3"]]], + ["b", 2, "none"], + ["base", { title: "b" }], + ["delta", [["s", ["title"], "b2"]]], ]); }); }); +describe("revisions", () => { + it("a history longer than a page is read page by page and exported whole, in seq order", async () => { + store = await memoryStore(); + const count = EXPORT_PAGE * 2 + 1; + const record = { + id: 2, + owner: "app.store", + namespace: "session", + kind: "long", + history: "rewindable" as const, + createdAt: 1, + revision: 1, + version: 1, + fingerprint: "", + }; + const create: StagedWrite = { + order: "write", + apply: async (step) => { + await insertDocument(step, record, "{}"); + return []; + }, + }; + await store.commit([create], OPEN, context); + for (let n = 1; n < count; n++) { + const ops = JSON.stringify([["s", ["n"], n]]); + const delta: StagedWrite = { + order: "write", + apply: async (step) => { + await insertRevision(step, record.id, step.seq, "delta", 1, ops); + return []; + }, + }; + await store.commit([delta], OPEN, context); + } + const revisions = (await exported(store)).flatMap((part) => (part.kind === "document.revision" ? [part] : [])); + expect(revisions.map(({ seq }) => seq)).toEqual(Array.from({ length: count }, (_, index) => index + 1)); + }); +}); + describe("the commit queue", () => { it("is held from the first next() until the export ends: a commit waits", async () => { store = await archiveStore(); diff --git a/packages/chord/test/next/store/import.test.ts b/packages/chord/test/next/store/import.test.ts index d440a901e..02350a696 100644 --- a/packages/chord/test/next/store/import.test.ts +++ b/packages/chord/test/next/store/import.test.ts @@ -50,7 +50,7 @@ describe("round trips", () => { const store = await imported(parts); expect(await exported(store)).toEqual(parts); const manifest = parts[0] as Extract; - expect([store.lastSeq, store.nextDocumentId]).toEqual([manifest.lastSeq, manifest.nextDocumentId]); + expect([store.lastSeq, store.nextId]).toEqual([manifest.lastSeq, manifest.nextId]); }); it("the store keeps its ids, seqs and folded documents; the outbox, when asked, starts after lastSeq", async () => { @@ -59,8 +59,7 @@ describe("round trips", () => { expect(await store.outbox!.read(store.lastSeq, 10, context)).toEqual([]); expect(await rejection(store.outbox!.read(0, 10, context))).toMatchObject({ code: "invalid" }); const record = await store.read( - (executor) => - findDocument(executor, { namespace: "chord", owner: "client:app.store", kind: "memo", key: "a" }), + (executor) => findDocument(executor, { namespace: "chord", owner: "app.store", kind: "memo", key: "a" }), context, ); const original = parts.find((part) => part.kind === "document.create" && part.record.key === "a"); @@ -71,12 +70,12 @@ describe("round trips", () => { it("a migrating table resumes on the imported store; rows written at version 1 read as the table's", async () => { const store = await imported(await archive()); const { EditsV2, NotesV2 } = ArchiveTables; - const notes = "client:app.store:notes"; + const notes = "app.store:notes"; const fence = { admit: () => true, adopt: () => {} }; await expect( prepareTables([{ definition: NotesV2, table: notes }], store, fence, "app", context), ).rejects.toThrow("crash"); - const edits = new TableHandle(EditsV2, "client:app.store:edits", store, tableOwner()); + const edits = new TableHandle(EditsV2, "app.store:edits", store, tableOwner()); expect(await edits.get({ id: "e003" }, context)).toEqual({ id: "e003", path: "p/3", @@ -163,10 +162,20 @@ interface Mutable { } /** Part `index` as a writable object. */ +/** The positions of every part of `kind`. */ +function all(parts: readonly ArchivePart[], kind: ArchivePart["kind"]): number[] { + return parts.flatMap((part, index) => (part.kind === kind ? [index] : [])); +} + function at(parts: unknown[], index: number): Mutable { return parts[index] as Mutable; } +/** The content of the revision part at `index`, writable. */ +function revisionContent(parts: unknown[], index: number): Mutable { + return at(parts, index).content as Mutable; +} + describe("refusals", () => { it.each<[string, (parts: ArchivePart[]) => unknown[], string]>([ [ @@ -258,12 +267,12 @@ describe("refusals", () => { "tables come before the documents", ], [ - "a table name that is not kind:type:table", + "a table name that is not owner:table", (parts) => { at(parts, first(parts, "table")).table = "edits"; return parts; }, - "not kind:type:table", + "not owner:table", ], [ "a column name breaking the rules", @@ -325,8 +334,8 @@ describe("refusals", () => { [ "an id twice", (parts) => { - const document = first(parts, "document.create"); - (at(parts, document + 1).record as Mutable).id = (at(parts, document).record as Mutable).id!; + const [one, two] = all(parts, "document.create"); + (at(parts, two!).record as Mutable).id = (at(parts, one!).record as Mutable).id!; return parts; }, "occurs twice", @@ -337,24 +346,75 @@ describe("refusals", () => { (at(parts, first(parts, "document.create")).record as Mutable).createdAt = 10_000; return parts; }, - "created after the manifest's lastSeq", + "a lifetime outside the manifest's lastSeq", ], [ - "a base at another version", + "a base at another version than its record's", (parts) => { - (at(parts, first(parts, "document.create")).base as Mutable).version = 2; + revisionContent(parts, first(parts, "document.revision")).version = 2; return parts; }, - "the base is not at the record's version", + "has no base at its version", + ], + [ + "a delta without a base at its version", + (parts) => { + const delta = parts.findIndex((part) => part.kind === "document.revision" && part.content.kind === "delta"); + revisionContent(parts, delta).version = 2; + return parts; + }, + "a delta without a base at its version", + ], + [ + "a revision out of seq order", + (parts) => { + const delta = parts.findIndex((part) => part.kind === "document.revision" && part.content.kind === "delta"); + at(parts, delta).seq = 1; + return parts; + }, + "out of seq order", + ], + [ + "a revision of another record", + (parts) => { + at(parts, first(parts, "document.revision")).id = 9_999; + return parts; + }, + "follows its record's create", + ], + [ + "a live record without revisions", + (parts) => parts.filter((_, index) => index !== first(parts, "document.revision")), + "has no base at its version", + ], + [ + "revisions of a retired none record", + (parts) => { + const record = at(parts, first(parts, "document.create")).record as Mutable; + record.retiredAt = record.createdAt!; + return parts; + }, + "keeps no revisions", + ], + [ + "a retired rewindable record without its history", + (parts) => { + const record = at(parts, first(parts, "document.create")).record as Mutable; + record.history = "rewindable"; + record.retiredAt = record.createdAt!; + const id = record.id; + return parts.filter((part) => part.kind !== "document.revision" || part.id !== id); + }, + "has no base at its version", ], [ "two live records at one address", (parts) => { - const document = first(parts, "document.create"); - (at(parts, document + 1).record as Mutable).key = (at(parts, document).record as Mutable).key!; + const [one, two] = all(parts, "document.create"); + (at(parts, two!).record as Mutable).key = (at(parts, one!).record as Mutable).key!; return parts; }, - "a second live record at one address", + "a second record at one address for an overlapping lifetime", ], [ "a record's meta with a NUL", diff --git a/packages/chord/test/next/store/jsonl.test.ts b/packages/chord/test/next/store/jsonl.test.ts index 3d36e2321..9664737d3 100644 --- a/packages/chord/test/next/store/jsonl.test.ts +++ b/packages/chord/test/next/store/jsonl.test.ts @@ -17,8 +17,8 @@ async function collect(values: AsyncIterable): Promise { } const PARTS: ArchivePart[] = [ - { kind: "manifest", format: 1, schema: 1, lastSeq: 3, nextDocumentId: 2 }, - { kind: "put", table: "client:app.store:edits", row: { id: "ü€😀", n: 1 }, version: 1, columns: {} }, + { kind: "manifest", format: 1, schema: 1, lastSeq: 3, nextId: 2 }, + { kind: "put", table: "app.store:edits", row: { id: "ü€😀", n: 1 }, version: 1, columns: {} }, { kind: "end", writes: 1 }, ]; diff --git a/packages/chord/test/next/store/outbox.test.ts b/packages/chord/test/next/store/outbox.test.ts index 2a720c080..4b5196524 100644 --- a/packages/chord/test/next/store/outbox.test.ts +++ b/packages/chord/test/next/store/outbox.test.ts @@ -14,13 +14,13 @@ import { type ChordStore, openStore } from "../../../src/next/store/store.ts"; import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; import { memoryStore, rejection } from "../support.ts"; -const ADDRESS = { namespace: "chord", owner: "session:app.store", kind: "notes", key: "" }; +const ADDRESS = { namespace: "chord", owner: "app.store", kind: "notes", key: "" }; const BASE = { version: 1, fingerprint: "0000000000000000", value: { n: 0 } }; const OPEN = { admit: () => true, adopt: () => {} }; /** Creates the singleton at `ADDRESS` (with `key`) in one commit. */ async function created(store: ChordStore, key = ""): Promise { - await store.commit([createDocument({ ...ADDRESS, key }, "latest", BASE, JSON.stringify(BASE.value))], OPEN, context); + await store.commit([createDocument({ ...ADDRESS, key }, BASE, JSON.stringify(BASE.value))], OPEN, context); } let directory: string | undefined; @@ -46,7 +46,7 @@ describe("the outbox", () => { await store.close(context); store = await open(true); expect(store.outbox?.floor).toBe(1); - const id = 1; + const id = 2; const ops = [["s", ["n"], 2]] as const; await store.commit( [changeDocument(id, 1, { kind: "delta", version: 1, ops }, JSON.stringify(ops))], @@ -58,6 +58,7 @@ describe("the outbox", () => { expect(await store.outbox!.read(1, 10, context)).toEqual([ { seq: 2, + nextId: 3, writes: [{ kind: "document.change", id, revision: 1, content: { kind: "delta", version: 1, ops } }], }, ]); @@ -75,7 +76,7 @@ describe("the outbox", () => { const [commit] = await store.outbox!.read(0, 10, context); expect(commit!.writes[0]).toMatchObject({ kind: "document.create", - record: { id: 1, createdAt: 1, ...ADDRESS }, + record: { id: 2, createdAt: 1, ...ADDRESS }, base: BASE, }); await store.outbox!.trim(1, context); diff --git a/packages/chord/test/next/store/query.test.ts b/packages/chord/test/next/store/query.test.ts index 9dfb93473..2c6875a5f 100644 --- a/packages/chord/test/next/store/query.test.ts +++ b/packages/chord/test/next/store/query.test.ts @@ -51,36 +51,32 @@ describe("cursors", () => { const order = query(T).orderBy((e) => e.at, "desc").order; it("round-trip the order values, also across conditions with the same order", () => { - const cursor = encodeCursor("client:app:edits", order, [5, "a\u{1F600}", 2]); + const cursor = encodeCursor("app:edits", order, [5, "a\u{1F600}", 2]); expect(cursor).toMatch(/^[A-Za-z0-9_-]+$/); - expect(decodeCursor(T, "client:app:edits", order, cursor)).toEqual([5, "a\u{1F600}", 2]); + expect(decodeCursor(T, "app:edits", order, cursor)).toEqual([5, "a\u{1F600}", 2]); const other = query(T) .where((e) => eq(e.id, "x")) .orderBy((e) => e.at, "desc").order; - expect(decodeCursor(T, "client:app:edits", other, cursor)).toEqual([5, "a\u{1F600}", 2]); + expect(decodeCursor(T, "app:edits", other, cursor)).toEqual([5, "a\u{1F600}", 2]); }); it("are a trust boundary: another table, order or direction, garbage and bad values are invalid/args", () => { - const cursor = encodeCursor("client:app:edits", order, [5, "a", 2]); - expect(() => decodeCursor(T, "client:app:edits", query(T).order, cursor)).toThrow(args); - expect(() => decodeCursor(T, "client:app:edits", query(T).orderBy((e) => e.at).order, cursor)).toThrow(args); + const cursor = encodeCursor("app:edits", order, [5, "a", 2]); + expect(() => decodeCursor(T, "app:edits", query(T).order, cursor)).toThrow(args); + expect(() => decodeCursor(T, "app:edits", query(T).orderBy((e) => e.at).order, cursor)).toThrow(args); expect(() => decodeCursor( defineTable("other", { version: 1, columns: { id: str(), entry: int(), at: int() }, key: ["id", "entry"] }), - "client:app:other", + "app:other", order, cursor, ), ).toThrow(args); // The same definition in another object's table. - expect(() => decodeCursor(T, "client:other:edits", order, cursor)).toThrow(args); - expect(() => decodeCursor(T, "client:app:edits", order, "%%%")).toThrow(args); - expect(() => decodeCursor(T, "client:app:edits", order, 42)).toThrow(args); - expect(() => - decodeCursor(T, "client:app:edits", order, encodeCursor("client:app:edits", order, ["x", "a", 2])), - ).toThrow(args); - expect(() => - decodeCursor(T, "client:app:edits", order, encodeCursor("client:app:edits", order, [5, "a"])), - ).toThrow(args); + expect(() => decodeCursor(T, "other:edits", order, cursor)).toThrow(args); + expect(() => decodeCursor(T, "app:edits", order, "%%%")).toThrow(args); + expect(() => decodeCursor(T, "app:edits", order, 42)).toThrow(args); + expect(() => decodeCursor(T, "app:edits", order, encodeCursor("app:edits", order, ["x", "a", 2]))).toThrow(args); + expect(() => decodeCursor(T, "app:edits", order, encodeCursor("app:edits", order, [5, "a"]))).toThrow(args); }); }); diff --git a/packages/chord/test/next/store/refusal.test.ts b/packages/chord/test/next/store/refusal.test.ts new file mode 100644 index 000000000..760273954 --- /dev/null +++ b/packages/chord/test/next/store/refusal.test.ts @@ -0,0 +1,65 @@ +// Tests of refusals as Chord's owners take them (slice 6b D4, §1.4): an `int` CHECK fails only its transaction, any +// other refusal its object; the store's own errors pass through. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import { add } from "../../../src/next/store/expression.ts"; +import { refusalOf } from "../../../src/next/store/refusal.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { loaded, memoryStore, rejection } from "../support.ts"; + +describe("classification", () => { + it("a CHECK failure by SQLite's extended code or by its message; everything else broken; the store's own errors pass", () => { + const node = Object.assign(new Error("CHECK constraint failed: n"), { errcode: 275 }); + const other = Object.assign(new Error("UNIQUE constraint failed"), { errcode: 2067 }); + expect(refusalOf(node, "x")).toMatchObject({ kind: "check", error: { code: "invalid", reason: "args" } }); + expect(refusalOf(new Error("CHECK constraint failed: n"), "x")).toMatchObject({ kind: "check" }); + expect(refusalOf(other, "x")).toMatchObject({ kind: "broken", error: { code: "unavailable" } }); + const stopped = new ChordError({ + code: "unavailable", + reason: "storage_failed", + message: "store: storage failed", + }); + expect(refusalOf(stopped, "x")).toBeUndefined(); + expect(refusalOf(new ChordError({ code: "released", message: "closed" }), "x")).toBeUndefined(); + }); +}); + +describe("Chord's owners", () => { + const Counts = defineTable("counts", { version: 1, columns: { id: str(), n: int() }, key: ["id"] }); + const Counter = local("app.counter", { + version: 1, + params: {}, + members: { bump: method(obj({ by: int() })), reset: method() }, + }); + + it("an int out of range fails only its transaction; the object stays live", async () => { + const store = await memoryStore(); + const { host, errors } = await loaded( + (h) => + h.serve(Counter, (_, self) => { + const counts = self.table(Counts); + return { + bump: async ({ by }, ctx) => { + await self.tx(async (tx) => tx.update(counts, { id: "a" }, (c) => ({ n: add(c.n, by) })), ctx); + return null; + }, + reset: async (ctx) => { + await self.tx(async (tx) => tx.put(counts, { id: "a", n: Number.MAX_SAFE_INTEGER }), ctx); + return null; + }, + }; + }), + { store }, + ); + const counter = await host.get(Counter, {}, context); + await counter.reset(context); + expect(await rejection(counter.bump({ by: 1 }, context))).toMatchObject({ code: "invalid", reason: "args" }); + expect(counter.state.value).toEqual({ kind: "live" }); + expect(errors).toEqual([]); + await counter.bump({ by: -1 }, context); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/store/sql.test.ts b/packages/chord/test/next/store/sql.test.ts index e8adaa0eb..38b0c8623 100644 --- a/packages/chord/test/next/store/sql.test.ts +++ b/packages/chord/test/next/store/sql.test.ts @@ -29,7 +29,7 @@ const T = defineTable("edits", { describe("names", () => { it("renders ::

, quoted", () => { - expect(tableSqlName("session", "@pi/file-history.store", "edits")).toBe("session:@pi/file-history.store:edits"); + expect(tableSqlName("@pi/file-history.store", "edits")).toBe("@pi/file-history.store:edits"); expect(quote("session:a:b")).toBe('"session:a:b"'); }); }); diff --git a/packages/chord/test/next/store/store.test.ts b/packages/chord/test/next/store/store.test.ts index 03f1c0087..e94641317 100644 --- a/packages/chord/test/next/store/store.test.ts +++ b/packages/chord/test/next/store/store.test.ts @@ -1,19 +1,22 @@ -// The store (slice 6a §2, §3): opening, the commit step, storage failures, one host per store (rule 7), close. +// The store (slice 6a §2, §3, slice 6b §1): opening, the commit step, refusals and unknown outcomes (D4), the id +// allocator (D3), one host per store (rule 7), close. import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { Batch } from "../../../src/next/state/internal.ts"; import { sqliteBackend } from "../../../src/next/store/backend.ts"; import { changeDocument, createDocument, retireDocument } from "../../../src/next/store/engine.ts"; import { importStore } from "../../../src/next/store/import.ts"; import type { StagedWrite } from "../../../src/next/store/step.ts"; import { type ChordStore, openStore, type Store } from "../../../src/next/store/store.ts"; import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; -import { exported, metered, rejection } from "../support.ts"; +import { exported, faultyStore, holdNextCommit, metered, rejection } from "../support.ts"; -const ADDRESS = { namespace: "chord", owner: "session:app.store", kind: "notes", key: "" }; +const ADDRESS = { namespace: "chord", owner: "app.store", kind: "notes", key: "" }; const BASE = { version: 1, fingerprint: "0000000000000000", value: { n: 0 } }; /** A store on a fresh `:memory:` database, as Chord's layers see it. */ @@ -32,15 +35,15 @@ function recording(adopted: [number, readonly number[]][] = []) { /** Creates the singleton at `ADDRESS` and resolves with its id. */ async function created(store: ChordStore): Promise { const { adopted, fence } = recording(); - await store.commit([createDocument(ADDRESS, "latest", BASE, JSON.stringify(BASE.value))], fence, context); + await store.commit([createDocument(ADDRESS, BASE, JSON.stringify(BASE.value))], fence, context); return adopted[0]![1][0]!; } -/** A staged write whose step fails in the backend. */ +/** A staged write whose step fails in the backend: its own code throws, a refusal. */ const BROKEN: StagedWrite = { order: "write", apply: (step) => - step.run("INSERT INTO no_such_table VALUES (1)").then(() => ({ kind: "document.retire", id: 0, revision: 0 })), + step.run("INSERT INTO no_such_table VALUES (1)").then(() => [{ kind: "document.retire", id: 0, revision: 0 }]), }; let directory: string | undefined; @@ -56,13 +59,13 @@ describe("opening", () => { const path = join(directory, "store.sqlite"); const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; expect([store.lastSeq, store.outbox]).toEqual([0, undefined]); - expect(await created(store)).toBe(1); + expect(await created(store)).toBe(2); await store.close(context); const reopened = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; expect(reopened.lastSeq).toBe(1); const { adopted, fence } = recording(); - await reopened.commit([createDocument({ ...ADDRESS, key: "b" }, "latest", BASE, "{}")], fence, context); - expect(adopted).toEqual([[2, [2]]]); + await reopened.commit([createDocument({ ...ADDRESS, key: "b" }, BASE, "{}")], fence, context); + expect(adopted).toEqual([[2, [3]]]); await reopened.close(context); }); @@ -84,24 +87,24 @@ describe("the commit step", () => { adopt: (seq: number, ids: readonly number[]) => void events.push(`${name} ${seq} ${ids}`), }); const first = store.commit( - [createDocument(ADDRESS, "latest", BASE, "{}"), retireDocument(id, 1)], + [createDocument(ADDRESS, BASE, "{}"), retireDocument(id, 1)], fence("recreate"), context, ); const second = store.commit( - [changeDocument(2, 1, { kind: "delta", version: 1, ops: [["s", ["n"], 1]] }, '[["s",["n"],1]]')], + [changeDocument(3, 1, { kind: "delta", version: 1, ops: [["s", ["n"], 1]] }, '[["s",["n"],1]]')], fence("change"), context, ); expect(await Promise.all([first, second])).toEqual([2, 3]); - expect(events).toEqual(["recreate 2 2", "change 3 "]); + expect(events).toEqual(["recreate 2 3", "change 3 "]); expect(store.lastSeq).toBe(3); }); it("rejects a commit its fence refuses at the entry released, writing nothing; an empty commit takes no seq", async () => { const store = await memoryStore(); const refused = store.commit( - [createDocument(ADDRESS, "latest", BASE, "{}")], + [createDocument(ADDRESS, BASE, "{}")], { admit: () => false, adopt: () => {} }, context, ); @@ -109,7 +112,7 @@ describe("the commit step", () => { const { adopted, fence } = recording(); expect(await store.commit([], fence, context)).toBe(0); expect(adopted).toEqual([[0, []]]); - expect(await created(store)).toBe(1); + expect(await created(store)).toBe(2); }); it("numbers up to Number.MAX_SAFE_INTEGER after an import near it, then fails storage_failed: no seq repeats", async () => { @@ -120,13 +123,9 @@ describe("the commit step", () => { parts, context, )) as ChordStore; - expect(await created(store)).toBe(1); + expect(await created(store)).toBe(2); expect(store.lastSeq).toBe(Number.MAX_SAFE_INTEGER); - const refused = store.commit( - [createDocument({ ...ADDRESS, key: "b" }, "latest", BASE, "{}")], - recording().fence, - context, - ); + const refused = store.commit([createDocument({ ...ADDRESS, key: "b" }, BASE, "{}")], recording().fence, context); expect(await rejection(refused)).toMatchObject({ code: "unavailable", reason: "storage_failed" }); expect(store.lastSeq).toBe(Number.MAX_SAFE_INTEGER); expect(await rejection(store.read(async () => null, context))).toMatchObject({ reason: "storage_failed" }); @@ -151,26 +150,49 @@ describe("the commit step", () => { }); }); -describe("storage failures", () => { - it("stop the store: the commit rejects storage_failed, later commits and reads too; the host is told once", async () => { +describe("refusals and unknown outcomes (D4)", () => { + it("a step whose own code throws is refused: the commit rejects with that error, nothing is adopted, the store goes on", async () => { const store = await memoryStore(); const told: unknown[] = []; - store.attach((error) => told.push(error)); - expect(await rejection(store.commit([BROKEN], recording().fence, context))).toMatchObject({ - code: "unavailable", - reason: "storage_failed", - }); - expect(await rejection(store.commit([BROKEN], recording().fence, context))).toMatchObject({ - reason: "storage_failed", - }); - expect(await rejection(store.read(async () => 1, context))).toMatchObject({ reason: "storage_failed" }); - expect(told.length).toBe(1); - expect(store.lastSeq).toBe(0); - await store.close(context); - await store.close(context); + store.attach({ failed: (error) => told.push(error), batch: new Batch(() => {}, undefined) }); + const { adopted, fence } = recording(); + const error = await rejection(store.commit([BROKEN], fence, context)); + expect(String(error)).toContain("no such table"); + expect(isChordError(error)).toBe(false); + expect([adopted, told, store.lastSeq]).toEqual([[], [], 0]); + expect(await created(store)).toBe(2); + expect(store.lastSeq).toBe(1); }); - it("include an adopt that throws and a conditional update that finds another revision", async () => { + it.each([ + ["BEGIN IMMEDIATE", undefined], + ["COMMIT", undefined], + ["ROLLBACK", BROKEN], + ] as const)( + "a failed %s is an unknown outcome: the store stops storage_failed, its host told once", + async (at, write) => { + const { store, fail } = await faultyStore(); + const told: unknown[] = []; + store.attach({ failed: (error) => told.push(error), batch: new Batch(() => {}, undefined) }); + fail(at); + const writes = write === undefined ? [createDocument(ADDRESS, BASE, "{}")] : [write]; + expect(await rejection(store.commit(writes, recording().fence, context))).toMatchObject({ + code: "unavailable", + reason: "storage_failed", + }); + expect( + await rejection(store.commit([createDocument(ADDRESS, BASE, "{}")], recording().fence, context)), + ).toMatchObject({ + reason: "storage_failed", + }); + expect(await rejection(store.read(async () => 1, context))).toMatchObject({ reason: "storage_failed" }); + expect(told.length).toBe(1); + await store.close(context); + await store.close(context); + }, + ); + + it("an adopt that throws stops the store, an empty commit's too; a conditional update on another revision is refused", async () => { const store = await memoryStore(); const id = await created(store); const throwing = { @@ -182,40 +204,109 @@ describe("storage failures", () => { expect(await rejection(store.commit([retireDocument(id, 1)], throwing, context))).toMatchObject({ reason: "storage_failed", }); - // An empty commit's adopt too. const empty = await memoryStore(); expect(await rejection(empty.commit([], throwing, context))).toMatchObject({ reason: "storage_failed" }); expect(await rejection(empty.read(async () => 1, context))).toMatchObject({ reason: "storage_failed" }); const other = await memoryStore(); const otherId = await created(other); const stale = changeDocument(otherId, 7, { kind: "delta", version: 1, ops: [] }, "[]"); - expect(await rejection(other.commit([stale], recording().fence, context))).toMatchObject({ - reason: "storage_failed", - }); + expect(String(await rejection(other.commit([stale], recording().fence, context)))).toContain( + "not live at revision 7", + ); expect(other.lastSeq).toBe(1); + expect(await other.read(async () => 1, context)).toBe(1); }); it("spare a read whose context was aborted (aborted), and reject a read that settles after the stop", async () => { - const store = await memoryStore(); + const { store, fail } = await faultyStore(); const cancel = withCancel(context); cancel.cancel(); const failing = store.read(() => Promise.reject(new Error("backend")), cancel.context); expect(await rejection(failing)).toMatchObject({ code: "aborted" }); const gate = Promise.withResolvers(); const late = store.read(() => gate.promise, context); - await rejection(store.commit([BROKEN], recording().fence, context)); + fail("COMMIT"); + await rejection(store.commit([createDocument(ADDRESS, BASE, "{}")], recording().fence, context)); gate.resolve(1); expect(await rejection(late)).toMatchObject({ reason: "storage_failed" }); }); + + it("runs a read transaction's queries on one state, guarded as a read", async () => { + const store = await memoryStore(); + await created(store); + const counts = await store.readTransaction( + async (executor) => [ + await executor.get("SELECT count(*) AS n FROM chord_documents"), + await executor.get("SELECT 1 AS one"), + ], + context, + ); + expect(counts).toEqual([{ n: 1 }, { one: 1 }]); + expect(await rejection(store.readTransaction(() => Promise.reject(new Error("backend")), context))).toMatchObject( + { + reason: "storage_failed", + }, + ); + }); +}); + +describe("the id allocator (D3)", () => { + /** A staged write raising the allocator above `id`, as an adapter's explicit id does. */ + const raising = (id: number): StagedWrite => ({ + order: "write", + apply: async (step) => { + step.ids.raise(id); + return []; + }, + }); + + it("starts at 2, takes a creation's id as it applies, and gives a mint during a running step the next one", async () => { + const store = await memoryStore({ outbox: true }); + const held = holdNextCommit(store); + const commit = store.commit([createDocument(ADDRESS, BASE, "{}")], recording().fence, context); + await held.entered; + expect(store.mintId()).toBe(3); + held.release(); + await commit; + expect(store.mintId()).toBe(4); + const [first] = await store.outbox!.read(0, 10, context); + expect([first!.nextId, (first!.writes[0] as { record: { id: number } }).record.id]).toEqual([4, 2]); + expect(store.nextId).toBe(4); + }); + + it("leaves a refused step's ids unused and keeps the raised value", async () => { + const store = await memoryStore(); + await rejection(store.commit([createDocument(ADDRESS, BASE, "{}"), BROKEN], recording().fence, context)); + expect(await created(store)).toBe(3); + }); + + it("saturates at Number.MAX_SAFE_INTEGER across reopen: mints and Chord creations are refused, the store goes on", async () => { + directory = await mkdtemp(join(tmpdir(), "chord-store-")); + const path = join(directory, "store.sqlite"); + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; + await store.commit([raising(Number.MAX_SAFE_INTEGER)], recording().fence, context); + expect(store.nextId).toBe(Number.MAX_SAFE_INTEGER); + await store.close(context); + const reopened = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; + expect(reopened.nextId).toBe(Number.MAX_SAFE_INTEGER); + expect(() => reopened.mintId()).toThrow("ID space is exhausted"); + const refused = reopened.commit([createDocument(ADDRESS, BASE, "{}")], recording().fence, context); + expect(String(await rejection(refused))).toContain("ID space is exhausted"); + expect(await reopened.read(async () => 1, context)).toBe(1); + await reopened.close(context); + }); }); describe("one host per store (rule 7)", () => { - it("refuses a second listener until the first detaches", async () => { + it("refuses a second attachment until the first detaches", async () => { const store = await memoryStore(); - store.attach(() => {}); - expect(() => store.attach(() => {})).toThrow("already attached"); + const attachment = { failed: () => {}, batch: new Batch(() => {}, undefined) }; + store.attach(attachment); + expect(store.batch).toBe(attachment.batch); + expect(() => store.attach(attachment)).toThrow("already attached"); store.detach(); - store.attach(() => {}); + expect(store.batch).toBeUndefined(); + store.attach(attachment); }); }); @@ -224,7 +315,7 @@ describe("close", () => { const store = await memoryStore(); const gate = Promise.withResolvers(); const entered = Promise.withResolvers(); - const create = createDocument(ADDRESS, "latest", BASE, "{}"); + const create = createDocument(ADDRESS, BASE, "{}"); const held: StagedWrite = { order: "create", apply: async (step) => { @@ -235,11 +326,7 @@ describe("close", () => { }; const running = store.commit([held], recording().fence, context); await entered.promise; - const queued = store.commit( - [createDocument({ ...ADDRESS, key: "q" }, "latest", BASE, "{}")], - recording().fence, - context, - ); + const queued = store.commit([createDocument({ ...ADDRESS, key: "q" }, BASE, "{}")], recording().fence, context); const closed = (store as Store).close(context); gate.resolve(); expect(await running).toBe(1); diff --git a/packages/chord/test/next/store/table-changes.test.ts b/packages/chord/test/next/store/table-changes.test.ts index 2f51e3e37..8f3e75047 100644 --- a/packages/chord/test/next/store/table-changes.test.ts +++ b/packages/chord/test/next/store/table-changes.test.ts @@ -43,7 +43,7 @@ describe("staging", () => { changes.delete({ id: "e9" }); expect(staged).toHaveLength(3); expect( - await store.read((executor) => executor.get(`SELECT count(*) AS n FROM "session:app.store:edits"`), context), + await store.read((executor) => executor.get(`SELECT count(*) AS n FROM "app.store:edits"`), context), ).toEqual({ n: 1 }); }); }); diff --git a/packages/chord/test/next/store/table-handle.test.ts b/packages/chord/test/next/store/table-handle.test.ts index 7a51ef960..0a572d37c 100644 --- a/packages/chord/test/next/store/table-handle.test.ts +++ b/packages/chord/test/next/store/table-handle.test.ts @@ -125,12 +125,12 @@ describe("unreadable cells", () => { it("a json cell of a newer version fails the owner and the read, the data intact", async () => { const owner = tableOwner(); const edits = await filled(1, owner); - await store.read((executor) => executor.run(`UPDATE "session:app.store:edits" SET "meta$v" = 2`), context); + await store.read((executor) => executor.run(`UPDATE "app.store:edits" SET "meta$v" = 2`), context); const error = await rejection(edits.get({ id: "e000" }, context)); expect(error).toMatchObject({ code: "unavailable" }); expect(owner.failures).toHaveLength(1); - expect( - await store.read((executor) => executor.get(`SELECT meta FROM "session:app.store:edits"`), context), - ).toEqual({ meta: '{"a":0}' }); + expect(await store.read((executor) => executor.get(`SELECT meta FROM "app.store:edits"`), context)).toEqual({ + meta: '{"a":0}', + }); }); }); diff --git a/packages/chord/test/next/store/table-migration.test.ts b/packages/chord/test/next/store/table-migration.test.ts index c55ae4b96..d58bef1b7 100644 --- a/packages/chord/test/next/store/table-migration.test.ts +++ b/packages/chord/test/next/store/table-migration.test.ts @@ -17,7 +17,7 @@ import { TableHandle } from "../../../src/next/store/table-handle.ts"; import { MIGRATION_CHUNK, prepareTables } from "../../../src/next/store/table-migration.ts"; import { memoryStore, rejection, tableOwner } from "../support.ts"; -const TABLE = "client:app.store:edits"; +const TABLE = "app.store:edits"; const OPEN = { admit: () => true, adopt: () => {} }; const EditsV1 = defineTable("edits", { diff --git a/packages/chord/test/next/store/table-writes.test.ts b/packages/chord/test/next/store/table-writes.test.ts index d1bd06c0a..7fae21403 100644 --- a/packages/chord/test/next/store/table-writes.test.ts +++ b/packages/chord/test/next/store/table-writes.test.ts @@ -26,7 +26,7 @@ const Edits = defineTable("edits", { key: ["id"], indexes: { by_path: ["path"] }, }); -const TABLE = "session:app.store:edits"; +const TABLE = "app.store:edits"; const FENCE = { admit: () => true, adopt: () => {} }; let store: ChordStore; @@ -71,10 +71,11 @@ describe("schema writes", () => { expect(indexes).toContainEqual({ name: `${TABLE}:by_path` }); }); - it("an index write for a table the store does not have fails the step (storage_failed)", async () => { + it("an index write for a table the store does not have refuses the step (D4: the store goes on)", async () => { await opened(); - const write = stageIndex({ kind: "index", table: "session:app.store:nope", name: "x", columns: ["id"] }); - await expect(store.commit([write], FENCE, context)).rejects.toMatchObject({ reason: "storage_failed" }); + const write = stageIndex({ kind: "index", table: "app.store:nope", name: "x", columns: ["id"] }); + await expect(store.commit([write], FENCE, context)).rejects.toThrow("not in the store"); + expect(await store.read(async () => 1, context)).toBe(1); }); }); diff --git a/packages/chord/test/next/support.ts b/packages/chord/test/next/support.ts index 7ad2b9cd7..428dea36f 100644 --- a/packages/chord/test/next/support.ts +++ b/packages/chord/test/next/support.ts @@ -1,6 +1,7 @@ // Test support: cost counters through the in-memory telemetry adapter, GC checks, and hosts, links, watches and // rejections for the slice tests. +import { DatabaseSync } from "node:sqlite"; import { setFlagsFromString } from "node:v8"; import { runInNewContext } from "node:vm"; import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; @@ -65,7 +66,7 @@ import { evolveTable } from "../../src/next/store/table-evolve.ts"; import { TableHandle, type TableOwner } from "../../src/next/store/table-handle.ts"; import { prepareTables } from "../../src/next/store/table-migration.ts"; import { nodeCompression } from "../../src/next-node/compression.ts"; -import { openNodeSqliteDatabase } from "../../src/sqlite/node.ts"; +import { NodeSqliteDatabase, openNodeSqliteDatabase } from "../../src/sqlite/node.ts"; import type { NativeWatchHandle, WatchCall, WatchHandle } from "../../src/watch.ts"; /** Counts of the `chord.data.*` counters. */ @@ -158,7 +159,7 @@ export function valuesOf(calls: readonly WatchCall[]): (T | { readonly fai */ export function failLive(objects: readonly WeakRef[], self: Self, error: ChordError): void { // The factory's `self` is its attempt. - for (const object of objects) object.deref()?.failLive(self as unknown as Attempt, error); + for (const object of objects) object.deref()?.failLive(self as unknown as Attempt, { kind: "broken", error }); } /** A local type whose objects' attempts a test ends by hand (`restartable`); it ends with its last reference. */ @@ -252,6 +253,37 @@ export async function memoryStore(options?: { readonly outbox?: boolean }): Prom return (await openStore(sqliteBackend(db), BACKGROUND_CONTEXT, options)) as ChordStore; } +/** A statement of the Node facade's transactions a test can make fail. */ +export type TransactionStatement = "BEGIN IMMEDIATE" | "COMMIT" | "ROLLBACK"; + +/** + * A Node SQLite facade on `:memory:` whose next `statement` throws, once per `fail` (the D4 matrix: a failed + * `BEGIN`, `COMMIT` or `ROLLBACK`, as the real facade sees one). + */ +export function faultyDatabase(): { readonly db: NodeSqliteDatabase; fail(statement: TransactionStatement): void } { + const database = new DatabaseSync(":memory:"); + const failing = new Set(); + const exec = database.exec.bind(database); + database.exec = (sql: string): void => { + if (!failing.delete(sql)) { + exec(sql); + return; + } + // A failed `ROLLBACK` here still rolls back, so the database stays usable for the rest of the test. + if (sql === "ROLLBACK") exec(sql); + throw new Error(`injected: ${sql} failed`); + }; + return { db: new NodeSqliteDatabase(database), fail: (statement) => void failing.add(statement) }; +} + +/** A store over `faultyDatabase()`, and its `fail`. */ +export async function faultyStore(options?: { + readonly outbox?: boolean; +}): Promise<{ readonly store: ChordStore; fail(statement: TransactionStatement): void }> { + const { db, fail } = faultyDatabase(); + return { store: (await openStore(sqliteBackend(db), BACKGROUND_CONTEXT, options)) as ChordStore, fail }; +} + /** * Holds `store`'s next commit step inside its backend transaction, after its admission: `entered` resolves there, * `release()` lets it finish; later commits run as usual. @@ -267,7 +299,7 @@ export function holdNextCommit(store: ChordStore): { readonly entered: Promise { entered.resolve(); await gate.promise; - return { kind: "dropIndex", table: "x", name: "y" }; + return [{ kind: "dropIndex", table: "x", name: "y" }]; }, }; return commit([...writes, held], fence, ctx); @@ -280,13 +312,13 @@ export function tableOwner(failures: ChordError[] = []): TableOwner & { readonly return { label: "app.store", scope: {}, assertLive: () => {}, fail: (error) => void failures.push(error), failures }; } -/** Creates `definition` (and its indexes) in `store` as `session:app.store:` and returns a handle to it. */ +/** Creates `definition` (and its indexes) in `store` as `app.store:` and returns a handle to it. */ export async function tableIn( store: ChordStore, definition: TableDefinition, owner = tableOwner(), ): Promise { - const table = `session:app.store:${definition.name}`; + const table = `app.store:${definition.name}`; const fence = { admit: () => true, adopt: () => {} }; await prepareTables([{ definition, table }], store, fence, "app.store", BACKGROUND_CONTEXT); return new TableHandle(definition, table, store, owner); @@ -886,8 +918,8 @@ export const ArchiveTables = (() => { })(); /** - * A store holding a bit of everything an archive carries: `client:app.store:edits` at version 2 with 150 rows (two - * export pages; some written at version 1, `NULL` and `json(...)` cells, an index), `client:app.store:notes` left + * A store holding a bit of everything an archive carries: `app.store:edits` at version 2 with 150 rows (two + * export pages; some written at version 1, `NULL` and `json(...)` cells, an index), `app.store:notes` left * `migrating` at version 2 after one committed chunk, and documents: a singleton, keyed ones with deltas after their * base, and a retired one. */ @@ -895,8 +927,8 @@ export async function archiveStore(options?: { readonly outbox?: boolean }): Pro const { EditsV1, EditsV2, NotesV1, NotesV2 } = ArchiveTables; const store = await memoryStore(options); const fence = { admit: () => true, adopt: () => {} }; - const edits = await tableAt(store, "client:app.store:edits", EditsV1); - const notes = await tableAt(store, "client:app.store:notes", NotesV1); + const edits = await tableAt(store, "app.store:edits", EditsV1); + const notes = await tableAt(store, "app.store:notes", NotesV1); const staged: StagedWrite[] = []; const editChanges = new TableChanges(edits, staged); for (let i = 0; i < 150; i++) { @@ -911,21 +943,15 @@ export async function archiveStore(options?: { readonly outbox?: boolean }): Pro const noteChanges = new TableChanges(notes, staged); for (let i = 0; i < 150; i++) noteChanges.put({ id: i, text: `t${i}` }); await store.commit(staged, fence, BACKGROUND_CONTEXT); + await prepareTables([{ definition: EditsV2, table: "app.store:edits" }], store, fence, "app", BACKGROUND_CONTEXT); await prepareTables( - [{ definition: EditsV2, table: "client:app.store:edits" }], - store, - fence, - "app", - BACKGROUND_CONTEXT, - ); - await prepareTables( - [{ definition: NotesV2, table: "client:app.store:notes" }], + [{ definition: NotesV2, table: "app.store:notes" }], store, fence, "app", BACKGROUND_CONTEXT, ).catch(() => {}); - const address = { namespace: "chord", owner: "client:app.store", kind: "memo" }; + const address = { namespace: "chord", owner: "app.store", kind: "memo" }; const base = (value: JsonValue) => ({ version: 1, fingerprint: "f1", value }); let ids: readonly number[] = []; const created = { @@ -936,7 +962,7 @@ export async function archiveStore(options?: { readonly outbox?: boolean }): Pro }; await store.commit( ["", "a", "b", "gone"].map((key) => - createDocument({ ...address, key }, "latest", base({ title: key }), JSON.stringify({ title: key })), + createDocument({ ...address, key }, base({ title: key }), JSON.stringify({ title: key })), ), created, BACKGROUND_CONTEXT, diff --git a/packages/chord/test/sqlite/cloudflare.test.ts b/packages/chord/test/sqlite/cloudflare.test.ts index 41a270524..1202d8f49 100644 --- a/packages/chord/test/sqlite/cloudflare.test.ts +++ b/packages/chord/test/sqlite/cloudflare.test.ts @@ -1,5 +1,10 @@ import { DatabaseSync } from "node:sqlite"; import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT } from "../../src/context/index.ts"; +import { sqliteBackend } from "../../src/next/store/backend.ts"; +import { refusalOf } from "../../src/next/store/refusal.ts"; +import type { StagedWrite } from "../../src/next/store/step.ts"; +import { type ChordStore, openStore } from "../../src/next/store/store.ts"; import { DurableObjectSqliteDatabase } from "../../src/sqlite/cloudflare.ts"; import { durableObjectStorage, RUN_CHANGES, runChanges } from "./helpers.ts"; @@ -123,3 +128,72 @@ describe("Durable Object SQLite driver", () => { expect(sqlite.prepare("SELECT id FROM items").all()).toEqual([]); }); }); + +describe("the outcome of a transaction (slice 6b D4)", () => { + /** The test storage, its `transaction` failing after the closure resolved (a failed commit) when asked. */ + function failingCommit(): { storage: ReturnType; fail(): void } { + const storage = durableObjectStorage(); + const transaction = storage.transaction.bind(storage); + let failing = false; + return { + storage: { + ...storage, + transaction: async (closure) => { + const result = await transaction(closure); + if (!failing) return result; + failing = false; + throw new Error("commit failed"); + }, + }, + fail: () => { + failing = true; + }, + }; + } + + it("rejects with the callback's own error after a rollback, another one when the commit fails", async () => { + const { storage, fail } = failingCommit(); + const database = new DurableObjectSqliteDatabase(storage); + const own = new Error("refused"); + await expect(database.transaction(async () => Promise.reject(own))).rejects.toBe(own); + fail(); + await expect(database.transaction(async () => 1)).rejects.toThrow("commit failed"); + }); + + it("a store on it tells a CHECK refusal by its message, its error having no extended code", async () => { + const storage = durableObjectStorage(); + const exec = storage.sql.exec.bind(storage.sql); + // The Durable Object binding's errors carry a message only. + const withoutCodes: typeof storage = { + ...storage, + sql: { + exec: (query, ...bindings) => { + try { + return exec(query, ...bindings); + } catch (error) { + throw new Error((error as Error).message); + } + }, + }, + }; + const store = (await openStore( + sqliteBackend(new DurableObjectSqliteDatabase(withoutCodes)), + BACKGROUND_CONTEXT, + )) as ChordStore; + await store.read( + (executor) => executor.exec('CREATE TABLE "checked" (n INTEGER CHECK (n < 10))'), + BACKGROUND_CONTEXT, + ); + const refused: StagedWrite = { + order: "write", + apply: async (step) => { + await step.run('INSERT INTO "checked" VALUES (11)'); + return []; + }, + }; + const fence = { admit: () => true, adopt: () => {} }; + const error = await store.commit([refused], fence, BACKGROUND_CONTEXT).catch((thrown: unknown) => thrown); + expect(refusalOf(error, "x")).toMatchObject({ kind: "check" }); + expect(await store.read(async () => 1, BACKGROUND_CONTEXT)).toBe(1); + }); +}); diff --git a/packages/chord/test/sqlite/node.test.ts b/packages/chord/test/sqlite/node.test.ts index 96ad09a96..8666b3f5d 100644 --- a/packages/chord/test/sqlite/node.test.ts +++ b/packages/chord/test/sqlite/node.test.ts @@ -1,7 +1,7 @@ import { DatabaseSync } from "node:sqlite"; import { describe, expect, it, vi } from "vitest"; import type { SqliteExecutor } from "../../src/sqlite/index.ts"; -import { openNodeSqliteDatabase } from "../../src/sqlite/node.ts"; +import { NodeSqliteDatabase, openNodeSqliteDatabase } from "../../src/sqlite/node.ts"; import { RUN_CHANGES, runChanges } from "./helpers.ts"; describe("Node SQLite driver", () => { @@ -198,3 +198,48 @@ describe("Node SQLite driver", () => { await database.close(); }); }); + +describe("the outcome of a transaction (slice 6b D4)", () => { + /** A facade on `:memory:` whose next `statement` throws (and a `ROLLBACK` still rolls back). */ + function faulty(statement?: "BEGIN IMMEDIATE" | "COMMIT" | "ROLLBACK") { + const database = new DatabaseSync(":memory:"); + const exec = database.exec.bind(database); + let failing = statement; + database.exec = (sql: string): void => { + if (sql !== failing) { + exec(sql); + return; + } + failing = undefined; + if (sql === "ROLLBACK") exec(sql); + throw new Error(`${sql} failed`); + }; + return new NodeSqliteDatabase(database); + } + + it("rejects with the callback's own error after a rollback: the caller knows it never committed", async () => { + const database = faulty(); + const own = new Error("refused"); + await database.exec("CREATE TABLE t (n INTEGER)"); + const outcome = database.transaction(async (handle) => { + await handle.run("INSERT INTO t VALUES (1)"); + throw own; + }); + await expect(outcome).rejects.toBe(own); + expect(await database.all("SELECT n FROM t")).toEqual([]); + }); + + it.each(["BEGIN IMMEDIATE", "COMMIT"] as const)("rejects with another error when %s fails", async (statement) => { + const database = faulty(statement); + const outcome = database.transaction(async () => 1); + await expect(outcome).rejects.toThrow(`${statement} failed`); + }); + + it("rejects with an AggregateError, not the callback's error, when its ROLLBACK fails", async () => { + const database = faulty("ROLLBACK"); + const own = new Error("refused"); + const error = await database.transaction(async () => Promise.reject(own)).catch((thrown: unknown) => thrown); + expect(error).toBeInstanceOf(AggregateError); + expect(error).not.toBe(own); + }); +}); diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 46658afba..56763148d 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -100,6 +100,7 @@ ### Added +- `StoreStorage` (`@earendil-works/pi-durable/storage/store`): Durable's `Storage` on Chord v2's store. `StoreStorage.open(store, owner, context)` keeps a Session's records as declared tables of the store under `owner` and its documents in the store's documents engine, takes its ids from the store's one allocator, and commits each batch as one write of the store's commit step, adopted in the batch of the host the store serves, so a Chord object and a Durable Session commit through one queue and one outbox. It passes the storage conformance suite; `close()` leaves the store open (the app owns it). - Deferred commits: `Session.commit()`, `Conversation.commit()`, `TaskRuntime.commit()`, and a tool's `api.commit()` take `{ persistence: { kind: "deferred", withinMs? } }`. A commit that changes only existing conversation documents with `history: "latest"` and `fork: "initial"` is then adopted in memory and shown to watches at once, and stored with the next commit that reads or writes the document, before the document leaves memory, at close, or within `withinMs` (default `settings.progress.persistWithinMs`, or `createSession(storage, { persistWithinMs })`); any other write makes it immediate. A crash loses what is held. See spec §4.1. - Synchronous observers for adapters, such as a Chord source over a Durable view: `Session.observeDoc(token, ...scope, observer, context)`, `Harness.observeConversation(id, observer, context)`, and `Harness.observeTaskGraph(observer, context)` attach an `Observer` (`advance(change, context)`, `end(failure)`) atomically on the Session line, like a watch, and return an `Observation` (`value`, `detach()`; `observeDoc` gives `undefined` for an absent document). `advance` gets every change unfolded as memory adopts it, for a stored commit inside Storage's `adopt` and for a deferred one before it is stored, so it must only record it: no awaits, no Session calls, no throws. A document observer gets one call per change at its address (two for a commit that retires and recreates it), the task graph also a call outside any commit when a registry change or a failed migration changes a `blocked` reason, and a change can arrive before the attaching promise resolves. The attaching context governs only the attachment; a closed Session or an invalid argument rejects the promise. One that throws is reported as `listener_error`, detached, and ended with the error. `end` comes once, at Session close or failure, or after a task document's retirement with its task. Document, view, and task graph watches are built on them (spec §9.1). - `ConversationTranscript` keeps what a client has of one conversation over any `ConversationView`: `merge(view)` returns the view with `items`, the client's whole transcript (entries, a `gap` where a later window did not continue it, and an `earlier` marker until the start is loaded), instead of the window; `loadEarlier(context)` and `fillGap(gap, context)` page entries through a loader such as `Conversation.entries()`, and `resume` is the window that continues it after a drop: `conversation.watch(context, transcript.resume)`. A merge costs O(items) and returns the same `items` array when it adds nothing; concurrent loads of the same page are shared ([#11327](https://github.com/earendil-works/pi/issues/11327)). diff --git a/packages/durable/README.md b/packages/durable/README.md index bdf59282e..d87c54d34 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -956,6 +956,8 @@ node --conditions=source --experimental-strip-types test/examples/13-chat.ts | [32-tests-after-edits](test/examples/32-tests-after-edits.ts) | An `afterRun` hook that runs the tests after every run that edited files, a fix run when they fail, and input waiting for them | | [33-turn-limit-and-continue](test/examples/33-turn-limit-and-continue.ts) | A per-conversation turn limit, `runFailure`, and continuing without a user message | | [34-submit-in-commit](test/examples/34-submit-in-commit.ts) | Admitting a visitor's steer inside the host's own commit with `submit(tx, draft)` and `whenIdle: "reject"` | +| [35-durable-on-the-store](test/examples/35-durable-on-the-store.ts) | A Harness on `StoreStorage` in a Chord host's store, next to a Chord object's own table: one commit queue, one id allocator, one export | +| [36-durable-views](test/examples/36-durable-views.ts) | A Chord object following a Durable view with `self.follow`: its frames and a member reading the followed value | | [00](test/examples/00-conversation.ts)–[12](test/examples/12-recovery.ts) | The layers underneath: sessions, documents, forks, watches, the Harness, agent configuration, reload, extension state, tasks, recovery | Examples that call OpenAI need `OPENAI_API_KEY`; most use the faux provider otherwise. diff --git a/packages/durable/package.json b/packages/durable/package.json index 106512d21..933fe6db7 100644 --- a/packages/durable/package.json +++ b/packages/durable/package.json @@ -84,6 +84,12 @@ "import": "./dist/storage/jsonl/node.js", "default": "./dist/storage/jsonl/node.js" }, + "./storage/store": { + "source": "./src/storage/store/index.ts", + "types": "./dist/storage/store/index.d.ts", + "import": "./dist/storage/store/index.js", + "default": "./dist/storage/store/index.js" + }, "./storage/sqlite": { "source": "./src/storage/sqlite/index.ts", "types": "./dist/storage/sqlite/index.d.ts", @@ -119,7 +125,8 @@ "scripts": { "clean": "shx rm -rf dist", "build": "tsc -p tsconfig.build.json", - "test": "vitest --run && PI_DURABLE_DOCUMENT_RETENTION_MS=0 vitest --run && npm run test:workers", + "test": "vitest --run && PI_DURABLE_DOCUMENT_RETENTION_MS=0 vitest --run && npm run test:store && npm run test:workers", + "test:store": "vitest --run --config vitest.store.config.ts", "test:workers": "vitest --run --config test-workers/vitest.config.ts", "bench:storage": "vitest bench --config vitest.benchmark.config.ts", "bench:storage:memory": "node --conditions=source --experimental-strip-types test/storage-memory.ts", diff --git a/packages/durable/src/storage/store/commit.ts b/packages/durable/src/storage/store/commit.ts new file mode 100644 index 000000000..cc53f5418 --- /dev/null +++ b/packages/durable/src/storage/store/commit.ts @@ -0,0 +1,327 @@ +// One Durable commit as one composite write of the store's commit step (slice 6b §2.3): staged before the queue (rows +// encoded, contents serialized, explicit ids raising the store's allocator, fork copies resolved from their sources), +// then inside the step what `SqliteStorage` does inside its transaction, with the step's reader: Durable's checks of +// ids and documents (its messages), the table puts, and the documents in two passes, existing ones first (their content, +// then their retirement, which frees the addresses this commit's creations take), then the creations and copies. + +import type { Context, JsonValue } from "@earendil-works/chord"; +import { + type EncodedRow, + type DocumentContent as EngineContent, + type DocumentRecord as EngineRecord, + findDocument, + insertDocument, + type OwnerTable, + readDocument, + retireRecord, + type StagedWrite, + type Step, + type StoreWrite, + writeContent, +} from "@earendil-works/chord/next"; +import type { + DocumentContent, + DocumentCopySource, + DocumentCreate, + DocumentId, + DocumentRecord, + Id, + JsonObject, + StorageWrite, +} from "../../types.ts"; +import { addressOf, durableRecord, engineRecord } from "./documents.ts"; +import { conversationRow, entryRow, submissionRow, taskRow } from "./tables.ts"; + +/** Durable's record types, as `record_ids` names them. */ +export type RecordType = "conversation" | "entry" | "task" | "submission" | "document"; + +/** The storage's tables, as a commit writes them. */ +export type CommitTables = { + readonly recordIds: OwnerTable; + readonly conversations: OwnerTable; + readonly entries: OwnerTable; + readonly entryHeads: OwnerTable; + readonly tasks: OwnerTable; + readonly submissions: OwnerTable; +}; + +/** A table write of the commit: its record's id and type, its table and its row encoded at staging. */ +type RowWrite = { + readonly id: number; + readonly type: Exclude; + readonly table: OwnerTable; + readonly row: EncodedRow; +}; + +/** A base as the commit stores it: its version and value, and the value serialized. */ +type StagedBase = { readonly version: number; readonly value: JsonObject; readonly text: string }; + +/** What one commit does with one document, as `SqliteStorage`'s action map has it. */ +type DocumentAction = { + create?: DocumentCreate; + copy?: DocumentCopySource; + /** A creation's base: the write's own, or a copy's once resolved. */ + base?: StagedBase; + content?: { readonly content: DocumentContent; readonly text: string }; + retire: boolean; +}; + +/** Reads a copy's source at its point: its stored document, `undefined` when it cannot be read there. */ +export type CopySource = ( + source: DocumentCopySource, +) => Promise<{ readonly record: DocumentRecord; readonly version: number; readonly value: JsonObject } | undefined>; + +/** A Durable commit staged for the store's step. */ +export class DurableCommit implements StagedWrite { + readonly order = "write"; + readonly #owner: string; + readonly #tables: CommitTables; + readonly #context: Context; + /** Every id a write claims, in write order (documents created or copied included), for the id checks. */ + readonly #claims: { readonly id: number; readonly type: RecordType }[] = []; + readonly #rows: RowWrite[] = []; + readonly #actions = new Map(); + + /** + * Stages `writes` (rule 8: rows and contents encoded here, before the queue); `raise` makes room above each explicit + * id. A malformed batch (two content commands for one document, a second retirement) throws Durable's message. + */ + constructor( + writes: readonly StorageWrite[], + owner: string, + tables: CommitTables, + raise: (id: number) => void, + context: Context, + ) { + this.#owner = owner; + this.#tables = tables; + this.#context = context; + for (const write of writes) this.#stage(write, raise); + } + + /** + * Resolves the copies before the step (S11): each source read at its point and checked against the copy's record, + * entered as a creation carrying the source's value as its base. + */ + async resolveCopies(read: CopySource): Promise { + for (const [id, action] of this.#actions) { + const source = action.copy; + if (source === undefined) continue; + if (this.#actions.has(source.id)) throw new Error(`Document copy ${id} source is changed in the copy batch`); + try { + const stored = await read(source); + if (stored === undefined) throw new Error(`Fork source document ${source.id} cannot be read`); + const create = action.create!; + if ( + stored.record.scope.kind !== "conversation" || + create.scope.kind !== "conversation" || + stored.record.kind !== create.kind || + stored.record.key !== create.key || + stored.record.history !== create.history || + stored.record.fork !== create.fork + ) { + throw new Error(`Fork source document ${source.id} does not match the copied record`); + } + action.base = { version: stored.version, value: stored.value, text: JSON.stringify(stored.value) }; + } catch (error) { + throw new Error(`Document copy ${id} was rejected`, { cause: error }); + } + } + } + + async apply(step: Step): Promise { + const recorded: StoreWrite[] = []; + const claimed = await this.#checkIds(step); + const existing = await this.#checkDocuments(step); + for (const write of this.#rows) { + if (claimed.get(write.id) === undefined) recorded.push(await this.#claim(step, write.id, write.type)); + claimed.set(write.id, write.type); + const stamp = write.table === this.#tables.entries ? "commit_seq" : undefined; + recorded.push(await write.table.put(step, write.row, stamp)); + } + // Existing documents first: a retirement frees its address before a creation of this commit takes it. + for (const [id, record] of existing) recorded.push(...(await this.#change(step, record, this.#actions.get(id)!))); + for (const [id, action] of this.#actions) { + if (action.create === undefined) continue; + recorded.push(await this.#claim(step, id, "document")); + recorded.push(await this.#create(step, action.create, action.base!, action.retire)); + } + return recorded; + } + + /** Stages one write: a table row encoded, a document command into its action. */ + #stage(write: StorageWrite, raise: (id: number) => void): void { + const tables = this.#tables; + switch (write.type) { + case "conversation": + this.#row(write.value.id, write.type, tables.conversations, conversationRow(write.value), raise); + return; + case "entry": { + const { id, conversationId, head } = write.value; + this.#row(id, write.type, tables.entries, entryRow(write.value), raise); + // Its claim is the entry's: `apply` claims an id once. + if (head !== undefined) { + const row = tables.entryHeads.encode({ id, conversation_id: conversationId }); + this.#rows.push({ id, type: write.type, table: tables.entryHeads, row }); + } + return; + } + case "task": + this.#row(write.value.id, write.type, tables.tasks, taskRow(write.value), raise); + return; + case "submission": + this.#row(write.value.id, write.type, tables.submissions, submissionRow(write.value), raise); + return; + case "document.create": + case "document.copy": { + const { id } = write.record; + const action = this.#action(id); + if (action.create !== undefined || action.content !== undefined || action.copy !== undefined) { + throw new Error(`Document ${id} has more than one content command`); + } + action.create = write.record; + if (write.type === "document.copy") action.copy = write.source; + else action.base = { ...write.content, text: JSON.stringify(write.content.value) }; + this.#claims.push({ id, type: "document" }); + raise(id); + return; + } + case "document.change": { + const action = this.#action(write.id); + // A creation's base is its content command (`SqliteStorage` counts it so too), in either order. + if (action.create !== undefined || action.content !== undefined || action.copy !== undefined) { + throw new Error(`Document ${write.id} has more than one content command`); + } + const { content } = write; + action.content = { content, text: JSON.stringify(content.kind === "base" ? content.value : content.ops) }; + return; + } + case "document.retire": { + const action = this.#action(write.id); + if (action.retire) throw new Error(`Document ${write.id} is retired more than once`); + action.retire = true; + return; + } + } + } + + #row(id: Id, type: RowWrite["type"], table: OwnerTable, row: object, raise: (id: number) => void): void { + this.#claims.push({ id, type }); + this.#rows.push({ id, type, table, row: table.encode(row) }); + raise(id); + } + + #action(id: DocumentId): DocumentAction { + let action = this.#actions.get(id); + if (action === undefined) { + action = { retire: false }; + this.#actions.set(id, action); + } + return action; + } + + /** + * Durable's global id checks (`SqliteStorage.checkGlobalIds`): a conversation, entry or document id claimed by + * anything, or written twice; a task or submission id claimed by another type. Returns the stored types. + */ + async #checkIds(step: Step): Promise> { + const stored = new Map(); + const claimed = new Map(); + for (const { id, type } of this.#claims) { + let existing = stored.get(id); + if (!stored.has(id)) { + const row = await this.#tables.recordIds.get(step, [id], this.#context); + existing = row?.record_type as RecordType | undefined; + stored.set(id, existing); + } + const earlier = claimed.get(id); + if (type === "conversation" || type === "entry" || type === "document") { + if (existing !== undefined) throw new Error(`ID ${id} already belongs to ${existing}`); + if (earlier !== undefined) throw new Error(`ID ${id} is written more than once`); + } else { + if (existing !== undefined && existing !== type) throw new Error(`ID ${id} already belongs to ${existing}`); + if (earlier !== undefined && earlier !== type) throw new Error(`ID ${id} is written as two record types`); + } + claimed.set(id, type); + } + const types = new Map(); + for (const [id, type] of stored) if (type !== undefined) types.set(id, type); + return types; + } + + /** + * Durable's document checks (`SqliteStorage.checkDocumentActions`): a change or retirement of an unknown or retired + * document, a delta on another version, more than one live incarnation per address after the commit. Returns the + * existing records the commit changes or retires, in action order. + */ + async #checkDocuments(step: Step): Promise> { + const existing = new Map(); + for (const [id, action] of this.#actions) { + if (action.create !== undefined) continue; + const stored = await readDocument(step, id); + const record = stored === undefined ? undefined : durableRecord(stored, this.#owner); + if (record === undefined) throw new Error(`Unknown document: ${id}`); + if (record.retiredAt !== undefined) throw new Error(`Document ${id} is retired`); + const content = action.content?.content; + if (content?.kind === "delta" && content.version !== stored!.version) { + throw new Error(`Document ${id} version transition requires a base`); + } + existing.set(id, stored!); + } + /** Per address a live creation takes: its live incarnations after the commit so far. */ + const live = new Map(); + for (const action of this.#actions.values()) { + if (action.create === undefined || action.retire) continue; + const address = addressOf(action.create, this.#owner); + const key = JSON.stringify([address.namespace, address.kind, address.key]); + let count = live.get(key); + if (count === undefined) { + // The incarnation live there now counts unless this commit retires it. + const current = await findDocument(step, address); + count = current === undefined || this.#actions.get(current.id as DocumentId)?.retire === true ? 0 : 1; + } + if (++count > 1) throw new Error("Document address already has a current incarnation"); + live.set(key, count); + } + return existing; + } + + /** Puts `id`'s `record_ids` row: its record type. */ + #claim(step: Step, id: number, type: RecordType): Promise { + const table = this.#tables.recordIds; + return table.put(step, table.encode({ id, record_type: type })); + } + + /** An existing document's content, then its retirement, as their writes. */ + async #change(step: Step, record: EngineRecord, action: DocumentAction): Promise { + const writes: StoreWrite[] = []; + let { revision } = record; + if (action.content !== undefined) { + const { content, text } = action.content; + const stored = engineContent(content); + await writeContent(step, record.id, record.history, revision, stored, text); + writes.push({ kind: "document.change", id: record.id, revision, content: stored }); + revision++; + } + if (action.retire) { + await retireRecord(step, record); + writes.push({ kind: "document.retire", id: record.id, revision }); + } + return writes; + } + + /** A creation or a resolved copy (retired at once for a create-plus-retire, an empty lifetime), as its write. */ + async #create(step: Step, create: DocumentCreate, base: StagedBase, retired: boolean): Promise { + const record = engineRecord(create, this.#owner, base.version, step.seq, retired); + await insertDocument(step, record, base.text); + const value = base.value as JsonValue; + return { kind: "document.create", record, base: { version: base.version, fingerprint: "", value } }; + } +} + +/** A Durable content as the engine stores it: its documents have no schema fingerprint. */ +function engineContent(content: DocumentContent): EngineContent { + return content.kind === "base" + ? { kind: "base", version: content.version, fingerprint: "", value: content.value } + : { kind: "delta", version: content.version, ops: content.ops }; +} diff --git a/packages/durable/src/storage/store/documents.ts b/packages/durable/src/storage/store/documents.ts new file mode 100644 index 000000000..d81f94d2d --- /dev/null +++ b/packages/durable/src/storage/store/documents.ts @@ -0,0 +1,143 @@ +// Durable's documents in the store's documents engine (slice 6b §1.5, R1): a Durable address as an engine address +// under the storage's owner name (its scope as the namespace, its kind and key JSON-encoded as Durable's indexed +// strings), a Durable record as an engine record and back (the Durable record in `meta`), and a document read at a +// point. + +import { + type DocumentEntry, + type DocumentHistory, + type DocumentAddress as EngineAddress, + type DocumentPoint as EnginePoint, + type DocumentRecord as EngineRecord, + loadContent, + readDocumentEntry, +} from "@earendil-works/chord/next"; +import type { SqliteExecutor } from "@earendil-works/chord/sqlite"; +import { seqFromNumber } from "../../ids.ts"; +import type { + DocumentAddress, + DocumentCreate, + DocumentId, + DocumentPoint, + DocumentRecord, + JsonObject, + StoredDocument, +} from "../../types.ts"; +import { encodeIndexed } from "./tables.ts"; + +/** The namespaces a storage claims under its owner: Durable's scopes (Chord's namespaces never contain a `:`). */ +export const SCOPE_NAMESPACES = ["session", "conversation:", "task:"] as const; + +/** The engine namespace of a Durable scope: `session`, `conversation:`, `task:`. */ +export function namespaceOf(scope: DocumentRecord["scope"]): string { + switch (scope.kind) { + case "session": + return "session"; + case "conversation": + return `conversation:${scope.conversationId}`; + case "task": + return `task:${scope.taskId}`; + } +} + +/** The engine address of a Durable address under `owner`. */ +export function addressOf(address: DocumentAddress | DocumentCreate, owner: string): EngineAddress { + const at = { owner, namespace: namespaceOf(address.scope), kind: encodeIndexed(address.kind) }; + return address.key === undefined ? at : { ...at, key: encodeIndexed(address.key) }; +} + +/** What a record of `scope` keeps: session and task documents and latest conversation ones their latest state. */ +export function historyOf(record: DocumentCreate | DocumentRecord): DocumentHistory { + return record.scope.kind === "conversation" && record.history === "rewindable" ? "rewindable" : "latest"; +} + +/** + * The engine record of a Durable creation at `seq` (retired at once for an empty lifetime): the Durable record itself, + * its creation seq included, in `meta`, so a read of a live record parses one JSON text from one column, as + * `SqliteStorage`'s does (a retirement later sets only the engine's `retired_at`). + */ +export function engineRecord( + create: DocumentCreate, + owner: string, + version: number, + seq: number, + retired: boolean, +): EngineRecord { + return { + id: create.id, + ...addressOf(create, owner), + history: historyOf(create), + createdAt: seq, + ...(retired ? { retiredAt: seq } : {}), + meta: JSON.stringify({ ...create, createdAt: seq }), + revision: 1, + version, + fingerprint: "", + }; +} + +/** + * The Durable record of an engine record of this storage's owner and scopes, `undefined` for anyone else's (a Chord + * document, another owner's): its id is unknown to this storage. + */ +export function durableRecord(record: EngineRecord, owner: string): DocumentRecord | undefined { + const { meta } = record; + if (meta === undefined || !ownedHere(record, owner)) return undefined; + return recordOfEntry({ meta, retired_at: record.retiredAt ?? null }); +} + +/** The Durable record of an engine entry read at one of this storage's addresses or scopes (the record in `meta`). */ +export function recordOfEntry(entry: DocumentEntry): DocumentRecord { + // Written by `engineRecord`, parsed fresh: a retirement since is set on it in place. + const record = JSON.parse(entry.meta!) as { -readonly [K in keyof DocumentRecord]: DocumentRecord[K] }; + if (entry.retired_at !== undefined && entry.retired_at !== null) record.retiredAt = seqFromNumber(entry.retired_at); + return record as DocumentRecord; +} + +/** Whether a record is this storage's: `owner`'s, in one of Durable's scopes (not a Chord document of the owner). */ +function ownedHere(record: { readonly owner: string; readonly namespace: string }, owner: string): boolean { + const { namespace } = record; + if (record.owner !== owner) return false; + return namespace === "session" || namespace.startsWith("conversation:") || namespace.startsWith("task:"); +} + +/** Whether a document is alive at `at` (the half-open lifetime). */ +export function aliveAt(record: DocumentRecord, at: DocumentPoint): boolean { + if (at === "current") return record.retiredAt === undefined; + return record.createdAt <= at && (record.retiredAt === undefined || at < record.retiredAt); +} + +/** + * A document read at a point, as a tagged result: request errors are told after the guarded read (slice 6b §1.4), + * never thrown inside it. + */ +export type DocumentRead = + | { readonly kind: "found"; readonly document: StoredDocument | undefined } + | { readonly kind: "current_only"; readonly id: DocumentId }; + +/** + * Document `id` of `owner` at `at`: absent (unknown, someone else's, not alive at `at`), a record whose history does + * not reach back to `at`, or its value from the base at or below `at` and the deltas up to it. + */ +export async function readAt( + executor: Pick, + id: DocumentId, + at: DocumentPoint, + owner: string, +): Promise { + const entry = await readDocumentEntry(executor, id); + if (entry === undefined || entry.meta === null || !ownedHere(entry, owner)) { + return { kind: "found", document: undefined }; + } + const record = recordOfEntry(entry); + if (at !== "current" && entry.history !== "rewindable") return { kind: "current_only", id }; + if (!aliveAt(record, at)) return { kind: "found", document: undefined }; + const point: EnginePoint = at; + const loaded = await loadContent(executor, id, point); + // The engine stores what Durable's commits gave it: JSON objects. + const value = loaded.value as JsonObject; + return { + kind: "found", + document: { record, version: loaded.version, value, deltasSinceBase: loaded.deltasSinceBase }, + }; +} diff --git a/packages/durable/src/storage/store/index.ts b/packages/durable/src/storage/store/index.ts new file mode 100644 index 000000000..0ddf86cdc --- /dev/null +++ b/packages/durable/src/storage/store/index.ts @@ -0,0 +1 @@ +export { StoreStorage } from "./storage.ts"; diff --git a/packages/durable/src/storage/store/queries.ts b/packages/durable/src/storage/store/queries.ts new file mode 100644 index 000000000..8491270b7 --- /dev/null +++ b/packages/durable/src/storage/store/queries.ts @@ -0,0 +1,137 @@ +// StoreStorage's reads as the store's queries (slice 6b D2, §2.4), compiled once from the store's query vocabulary +// into SQL over the declared tables: no SQL is rendered per call, and a read selects only the record cell (by id, an +// entry's commit seq too; each column returned costs) and parses one JSON text per record, as `SqliteStorage`'s do. + +import type { Context } from "@earendil-works/chord"; +import type { Order, OwnerTable, WhereNode } from "@earendil-works/chord/next"; +import type { SqliteExecutor, SqliteValue } from "@earendil-works/chord/sqlite"; +import type { CommitTables } from "./commit.ts"; + +/** + * A stored row as a query selects it: the record cell, its version (selected only once the column has versions past + * 1), and an entry's commit seq. + */ +type RecordRow = { readonly record: string; readonly record$v?: number; readonly commit_seq?: number }; + +/** One condition of a read: a column, how it compares, its value; a call site keeps one order of them. */ +export type Condition = readonly [string, "eq" | "gt" | "lt" | "gte" | "lte", SqliteValue]; + +const ASCENDING: Order = { columns: ["id"], direction: "asc" }; +const DESCENDING: Order = { columns: ["id"], direction: "desc" }; + +/** The columns of a read of ids only. */ +const ID = ["id"]; + +/** A placeholder value: a query's shape binds its values at each call. */ +const ANY = 0; + +/** + * One table's compiled reads: by id (with the `byIdExtra` columns), and by shape (conditions in binding order, an + * order; the record only), each shape compiled at its first use. + */ +export class TableQueries { + readonly #table: OwnerTable; + /** The record column's read of a cell at a version past 1; `undefined` while the column is at version 1. */ + readonly #versioned: ((text: string, version: number, context: Context) => unknown) | undefined; + readonly #byId: string; + readonly #shapes = new Map(); + readonly #columns: readonly string[]; + + constructor(table: OwnerTable, byIdExtra: readonly string[] = []) { + this.#table = table; + const versioned = table.jsonVersion("record") > 1; + this.#versioned = versioned ? table.reader("record") : undefined; + this.#columns = versioned ? ["record", "record$v"] : ["record"]; + this.#byId = table.prepareGet([...this.#columns, ...byIdExtra]); + } + + /** The row with `id`, `undefined` when absent. */ + byId(executor: Pick, id: number): Promise { + return executor.get(this.#byId, id); + } + + /** Up to `limit` rows whose `conditions` hold, by id in `direction`. */ + select( + executor: Pick, + conditions: readonly Condition[], + direction: "ascending" | "descending", + limit: number, + ): Promise { + const sql = this.#shape(conditions, direction, "many"); + return executor.all(sql, ...conditions.map(([, , value]) => value), limit); + } + + /** The first row whose `conditions` hold, by id in `direction`; `undefined` when none does. */ + first( + executor: Pick, + conditions: readonly Condition[], + direction: "ascending" | "descending", + ): Promise { + const sql = this.#shape(conditions, direction, "first"); + return executor.get(sql, ...conditions.map(([, , value]) => value)); + } + + /** A row's record, read as its column's (version checked, migrated on access). */ + record(row: RecordRow, context: Context): T { + const versioned = this.#versioned; + // The cell holds a record of this table the storage wrote; at version 1 it is that JSON (`OwnerTable.reader`). + return (versioned === undefined ? JSON.parse(row.record) : versioned(row.record, row.record$v!, context)) as T; + } + + /** Appends the records of `rows` to `into`, in order, each read as `record` reads it (one check per call). */ + records(rows: readonly RecordRow[], into: T[], context: Context): void { + const versioned = this.#versioned; + if (versioned === undefined) { + for (const row of rows) into.push(JSON.parse(row.record) as T); + return; + } + for (const row of rows) into.push(versioned(row.record, row.record$v!, context) as T); + } + + /** The compiled read of a shape: its conditions' columns and comparisons, the direction, one row or many. */ + #shape(conditions: readonly Condition[], direction: "ascending" | "descending", rows: "first" | "many"): string { + let key = `${rows} ${direction}`; + for (const [column, condition] of conditions) key += ` ${column} ${condition}`; + let sql = this.#shapes.get(key); + if (sql !== undefined) return sql; + // One comparison per condition: the vocabulary's node of that name, its value a placeholder. + const nodes = conditions.map(([column, condition]) => ({ [condition]: [column, ANY] }) as unknown as WhereNode); + const where: WhereNode | undefined = nodes.length === 0 ? undefined : { and: nodes }; + const order = direction === "ascending" ? ASCENDING : DESCENDING; + sql = + rows === "first" + ? this.#table.prepareFirst(this.#columns, where, order) + : this.#table.prepare(this.#columns, where, order); + this.#shapes.set(key, sql); + return sql; + } +} + +/** The compiled reads of the storage's tables. */ +export type StorageQueries = { + readonly conversations: TableQueries; + readonly entries: TableQueries; + /** The id of a conversation's newest entry with a head marker: any, or at or before an id (bound second). */ + readonly latestHead: { readonly any: string; readonly atOrBefore: string }; + readonly tasks: TableQueries; + readonly submissions: TableQueries; +}; + +/** The reads of `tables`. */ +export function storageQueries(tables: CommitTables): StorageQueries { + const heads = tables.entryHeads; + return { + conversations: new TableQueries(tables.conversations), + entries: new TableQueries(tables.entries, ["commit_seq"]), + latestHead: { + any: heads.prepareFirst(ID, { eq: ["conversation_id", ANY] }, DESCENDING), + atOrBefore: heads.prepareFirst( + ID, + { and: [{ eq: ["conversation_id", ANY] }, { lte: ["id", ANY] }] }, + DESCENDING, + ), + }, + tasks: new TableQueries(tables.tasks), + submissions: new TableQueries(tables.submissions), + }; +} diff --git a/packages/durable/src/storage/store/storage.ts b/packages/durable/src/storage/store/storage.ts new file mode 100644 index 000000000..554979b89 --- /dev/null +++ b/packages/durable/src/storage/store/storage.ts @@ -0,0 +1,557 @@ +// Durable's `Storage` on Chord's store (slice 6b D1, D2, §2): opening under an owner name (claims, tables, the +// allocator), commits as one composite write of the store's commit step, and the reads, each one guarded read of the +// store (several queries at one state in a read transaction), its request errors in a tagged result told after it. + +import type { Context, JsonValue } from "@earendil-works/chord"; +import { + type ChordStore, + type Claimant, + findDocumentEntry, + listDocumentEntries, + type Store, + StoreOwnerTables, +} from "@earendil-works/chord/next"; +import type { SqliteExecutor } from "@earendil-works/chord/sqlite"; +import { StorageRequestError } from "../../errors.ts"; +import { idFromNumber, seqFromNumber } from "../../ids.ts"; +import type { + ConversationId, + ConversationQuery, + ConversationRecord, + Cursor, + DocumentAddress, + DocumentCopySource, + DocumentId, + DocumentPoint, + DocumentQuery, + DocumentRecord, + EntryId, + EntryQuery, + EntryRecord, + Id, + Page, + ScanOrder, + Seq, + Storage, + StorageWrite, + StoredDocument, + SubmissionId, + SubmissionQuery, + SubmissionRecord, + TaskId, + TaskQuery, + TaskRecord, +} from "../../types.ts"; +import { nextCursor, type ScanStart, scanStart } from "../scan.ts"; +import { type CommitTables, DurableCommit } from "./commit.ts"; +import { addressOf, type DocumentRead, namespaceOf, readAt, recordOfEntry, SCOPE_NAMESPACES } from "./documents.ts"; +import { type Condition, type StorageQueries, storageQueries, type TableQueries } from "./queries.ts"; +import { Conversations, Entries, EntryHeads, encodeIndexed, RecordIds, Submissions, Tasks } from "./tables.ts"; + +type StoredTask = TaskRecord; +type StoredEntry = { readonly entry: EntryRecord; readonly commitSeq: Seq }; + +/** + * A guarded read's result: its data, or its request error, which is thrown after the guarded read (slice 6b §1.4), + * never inside it. + */ +type Read = { readonly kind: "found"; readonly value: T } | { readonly kind: "refused"; readonly message: string }; + +/** The id condition of a table scan from `start`. */ +function scanFrom(start: ScanStart): Condition { + return start.order === "ascending" + ? ["id", "gt", start.after ?? -1] + : ["id", "lt", start.after ?? Number.MAX_SAFE_INTEGER]; +} + +function page }>( + values: readonly T[], + limit: number, + order: ScanOrder, +): Page { + const items = values.slice(0, limit); + if (values.length <= limit) return { items }; + return { items, next: nextCursor(items.at(-1)!.id, order) }; +} + +function cursorId(cursor: Cursor | undefined): number | undefined { + const after = cursor?.after; + if (after === undefined) return undefined; + if (typeof after !== "number" || !Number.isSafeInteger(after)) + throw new StorageRequestError("Invalid storage cursor"); + return after; +} + +/** + * Durable's `Storage` on Chord's store (slice 6b D1, D2, §2): its records as declared tables under an owner name, its + * documents in the store's documents engine, its ids from the store's one allocator, its commits as one composite + * write of the store's commit step, adopted in the host's batch. Several may share a store under distinct owner names + * (claims); the app owns the store, and `close` leaves it open. + */ +export class StoreStorage implements Storage { + readonly #store: ChordStore; + readonly #owner: string; + readonly #claimant: Claimant; + readonly #tables: CommitTables; + readonly #queries: StorageQueries; + #closed = false; + #closing: Promise | undefined; + /** Calls underway, which `close` waits for before it releases the claims (`#drained` once they reach 0). */ + #underway = 0; + #drained: (() => void) | undefined; + + private constructor(store: ChordStore, owner: string, claimant: Claimant, tables: CommitTables) { + this.#store = store; + this.#owner = owner; + this.#claimant = claimant; + this.#tables = tables; + this.#queries = storageQueries(tables); + } + + /** + * Opens Durable's storage in `store` under `owner` (a lowercase `[a-z0-9@/._~-]` name): claims its tables and its + * document namespaces (a second claim is refused `invalid/args`), creates or migrates its tables, and raises the + * store's allocator above every id it holds (one index read). A failure releases what it claimed. + */ + static async open(store: Store, owner: string, context: Context): Promise { + // Chord's own: `openStore` or `importStore` made it. + const chord = store as ChordStore; + const claimant: Claimant = { label: `StoreStorage(${owner})` }; + try { + const declared = new StoreOwnerTables(chord, owner, claimant); + const tables: CommitTables = { + recordIds: declared.declare(RecordIds), + conversations: declared.declare(Conversations), + entries: declared.declare(Entries), + entryHeads: declared.declare(EntryHeads), + tasks: declared.declare(Tasks), + submissions: declared.declare(Submissions), + }; + for (const namespace of SCOPE_NAMESPACES) chord.claims.documents(owner, namespace, claimant); + await declared.prepare(context); + const highest = await chord.read( + (executor) => + executor.get<{ readonly id: number | null }>( + `SELECT max("id") AS id FROM "${tables.recordIds.sql.table}"`, + ), + context, + ); + if (highest?.id != null) chord.ids.raise(highest.id); + return new StoreStorage(chord, owner, claimant, tables); + } catch (error) { + chord.claims.release(claimant); + throw error; + } + } + + commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, context: Context): Promise { + return this.#call(async () => { + const store = this.#store; + const commit = new DurableCommit(writes, this.#owner, this.#tables, (id) => store.ids.raise(id), context); + await commit.resolveCopies((source) => this.#copySource(source, context)); + // An empty commit stages nothing: it takes no seq, adopted at `lastSeq`. + const staged = writes.length === 0 ? [] : [commit]; + return seqFromNumber(await store.commitInBatch(staged, (seq) => adopt(seqFromNumber(seq)), context)); + }); + } + + mintId>(): Promise { + return this.#call(async () => idFromNumber(this.#store.mintId())); + } + + conversation(id: ConversationId, context: Context): Promise { + return this.#read((executor) => this.#conversation(executor, id, context), context); + } + + scanConversations( + query: ConversationQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#reading(() => { + const start = scanStart(query.order, cursor, "ascending"); + const conditions: Condition[] = [scanFrom(start)]; + if (query.ownerConversationId !== undefined) { + conditions.push(["owner_conversation_id", "eq", query.ownerConversationId]); + } + if (query.ownerTaskId !== undefined) conditions.push(["owner_task_id", "eq", query.ownerTaskId]); + return this.#scan(this.#queries.conversations, conditions, start, limit, context); + }); + } + + entry(id: EntryId, context: Context): Promise; + entry(conversationId: ConversationId, id: EntryId, context: Context): Promise; + entry( + idOrConversationId: EntryId | ConversationId, + idOrContext: EntryId | Context, + maybeContext?: Context, + ): Promise { + if (maybeContext === undefined) { + const context = idOrContext as Context; + return this.#read((executor) => this.#entry(executor, idOrConversationId as EntryId, context), context); + } + const context = maybeContext; + const id = typeof idOrContext === "number" ? idFromNumber(idOrContext) : undefined; + return this.#reading(() => { + if (id === undefined) throw new StorageRequestError("Storage.entry() requires an entry ID"); + const conversationId = idOrConversationId as ConversationId; + return this.#store + .read((executor) => this.#visibleEntry(executor, conversationId, id, context), context) + .then(told); + }); + } + + findLatestHeadMarker( + conversationId: ConversationId, + atOrBeforeEntryId: EntryId | undefined, + context: Context, + ): Promise<(EntryRecord & { readonly head: EntryId }) | undefined> { + return this.#reading(() => + this.#store + .read((executor) => this.#latestHeadMarker(executor, conversationId, atOrBeforeEntryId, context), context) + .then(told), + ); + } + + scanEntries( + query: EntryQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#reading(() => { + const { order, after } = scanStart(query.order, cursor, "descending"); + const read = + order === "ascending" + ? (executor: SqliteExecutor) => this.#entriesAscending(executor, query, limit, after, context) + : (executor: SqliteExecutor) => this.#entriesDescending(executor, query, limit, after, context); + return this.#store.read(read, context).then(told); + }); + } + + task(id: TaskId, context: Context): Promise { + const tasks = this.#queries.tasks; + return this.#read(async (executor) => { + const row = await tasks.byId(executor, id); + return row === undefined ? undefined : tasks.record(row, context); + }, context); + } + + scanTasks( + query: TaskQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#reading(() => { + const start = scanStart(query.order, cursor, "ascending"); + const conditions: Condition[] = [scanFrom(start)]; + if (query.conversationId !== undefined) conditions.push(["conversation_id", "eq", query.conversationId]); + if (query.kind !== undefined) conditions.push(["kind", "eq", encodeIndexed(query.kind)]); + if (query.status !== undefined) conditions.push(["status", "eq", query.status]); + if (query.abortRequested !== undefined) + conditions.push(["abort_requested", "eq", query.abortRequested ? 1 : 0]); + if (query.background !== undefined) conditions.push(["background", "eq", query.background ? 1 : 0]); + return this.#scan(this.#queries.tasks, conditions, start, limit, context); + }); + } + + submission(id: SubmissionId, context: Context): Promise { + const submissions = this.#queries.submissions; + return this.#read(async (executor) => { + const row = await submissions.byId(executor, id); + return row === undefined ? undefined : submissions.record(row, context); + }, context); + } + + scanSubmissions( + query: SubmissionQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#reading(() => { + const start = scanStart(query.order, cursor, "ascending"); + const conditions: Condition[] = [scanFrom(start)]; + if (query.conversationId !== undefined) conditions.push(["conversation_id", "eq", query.conversationId]); + if (query.status !== undefined) conditions.push(["status", "eq", query.status]); + return this.#scan(this.#queries.submissions, conditions, start, limit, context); + }); + } + + submissionByRequest( + conversationId: ConversationId, + requestId: string, + context: Context, + ): Promise { + const submissions = this.#queries.submissions; + const conditions: Condition[] = [ + ["conversation_id", "eq", conversationId], + ["request_id", "eq", encodeIndexed(requestId)], + ]; + return this.#read(async (executor) => { + const row = await submissions.first(executor, conditions, "ascending"); + return row === undefined ? undefined : submissions.record(row, context); + }, context); + } + + findDocument(address: DocumentAddress, at: DocumentPoint, context: Context): Promise { + return this.#read(async (executor) => { + const entry = await findDocumentEntry(executor, addressOf(address, this.#owner), at); + return entry === undefined ? undefined : recordOfEntry(entry); + }, context); + } + + document(id: DocumentId, at: DocumentPoint, context: Context): Promise { + // The record and its revisions must come from one committed state (S4). + return this.#reading(() => + this.#store.readTransaction((executor) => readAt(executor, id, at, this.#owner), context).then(documentOf), + ); + } + + scanDocuments( + query: DocumentQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#reading(() => { + const after = cursorId(cursor) ?? -1; + const scope = { + owner: this.#owner, + namespace: namespaceOf(query.scope), + ...(query.kind === undefined ? {} : { kind: encodeIndexed(query.kind) }), + }; + return this.#store + .read((executor) => listDocumentEntries(executor, scope, query.at, after, limit + 1), context) + .then((found) => page(found.map(recordOfEntry), limit, "ascending")); + }); + } + + /** + * Refuses new calls, waits for its commits underway, then releases its claims; the store stays open (the app's), + * and finishes the reads underway, which change nothing. + */ + close(_context: Context): Promise { + if (this.#closing === undefined) { + this.#closed = true; + this.#closing = (async () => { + if (this.#underway > 0) { + await new Promise((resolve) => { + this.#drained = resolve; + }); + } + this.#store.claims.release(this.#claimant); + })(); + } + return this.#closing; + } + + /** Runs one commit or mint of this storage: refused once it closed, counted until it settles (`close` waits). */ + async #call(run: () => Promise): Promise { + if (this.#closed) throw new Error("StoreStorage is closed"); + this.#underway++; + try { + return await run(); + } finally { + if (--this.#underway === 0) this.#drained?.(); + } + } + + /** + * A call that is one guarded read of the store: refused once closed; not waited for by `close` (a read changes + * nothing, and the store, which stays open, finishes it). + */ + #read(read: (executor: SqliteExecutor) => Promise, context: Context): Promise { + return this.#reading(() => this.#store.read(read, context)); + } + + /** + * A read call: `read` prepares its guarded read (a request error thrown then, before it) and returns it. Refused + * once closed; one promise per call besides the guard's (no async wrapper). + */ + #reading(read: () => Promise): Promise { + try { + if (this.#closed) throw new Error("StoreStorage is closed"); + return read(); + } catch (error) { + return Promise.reject(error); + } + } + + async #conversation( + executor: SqliteExecutor, + id: ConversationId, + context: Context, + ): Promise { + const conversations = this.#queries.conversations; + const row = await conversations.byId(executor, id); + return row === undefined ? undefined : conversations.record(row, context); + } + + #entry(executor: SqliteExecutor, id: EntryId, context: Context): Promise { + const entries = this.#queries.entries; + return entries.byId(executor, id).then((row) => { + if (row === undefined) return undefined; + return { entry: entries.record(row, context), commitSeq: seqFromNumber(row.commit_seq!) }; + }); + } + + /** Entry `id` when visible through `conversationId`'s fork chain (`SqliteStorage.readEntry`). */ + async #visibleEntry( + executor: SqliteExecutor, + conversationId: ConversationId, + id: EntryId, + context: Context, + ): Promise> { + let conversation = await this.#conversation(executor, conversationId, context); + if (conversation === undefined) return unknownConversation(conversationId); + const stored = await this.#entry(executor, id, context); + if (stored === undefined) return NOTHING_FOUND; + let upper = Number.POSITIVE_INFINITY; + while (conversation.id !== stored.entry.conversationId) { + if (conversation.parent === undefined) return NOTHING_FOUND; + upper = Math.min(upper, conversation.parent.at); + conversation = (await this.#conversation(executor, conversation.parent.conversationId, context))!; + } + return stored.entry.id > upper ? NOTHING_FOUND : { kind: "found", value: stored }; + } + + async #latestHeadMarker( + executor: SqliteExecutor, + conversationId: ConversationId, + atOrBeforeEntryId: EntryId | undefined, + context: Context, + ): Promise> { + let conversation = await this.#conversation(executor, conversationId, context); + if (conversation === undefined) return unknownConversation(conversationId); + let upper: number | undefined = atOrBeforeEntryId; + while (true) { + const { latestHead } = this.#queries; + const head = + upper === undefined + ? await executor.get<{ readonly id: number }>(latestHead.any, conversation.id) + : await executor.get<{ readonly id: number }>(latestHead.atOrBefore, conversation.id, upper); + if (head !== undefined) { + // Written with its entry, in the same commit. + const row = (await this.#queries.entries.byId(executor, head.id))!; + const value = this.#queries.entries.record(row, context); + return { kind: "found", value }; + } + if (conversation.parent === undefined) return NOTHING_FOUND; + upper = upper === undefined ? conversation.parent.at : Math.min(upper, conversation.parent.at); + conversation = (await this.#conversation(executor, conversation.parent.conversationId, context))!; + } + } + + /** Newest first: each conversation of the fork chain up to its fork point (`SqliteStorage.readEntries`). */ + async #entriesDescending( + executor: SqliteExecutor, + query: EntryQuery, + limit: number, + after: number | undefined, + context: Context, + ): Promise>> { + let conversation = await this.#conversation(executor, query.conversationId, context); + if (conversation === undefined) return unknownConversation(query.conversationId); + let upper: number | undefined = query.maxEntryId; + if (after !== undefined) upper = Math.min(upper ?? Number.MAX_SAFE_INTEGER, after - 1); + const values: EntryRecord[] = []; + while (true) { + const range = this.#range(conversation.id, query.minEntryId, upper); + const rows = await this.#queries.entries.select(executor, range, "descending", limit + 1 - values.length); + this.#queries.entries.records(rows, values, context); + if (values.length > limit || conversation.parent === undefined) break; + upper = upper === undefined ? conversation.parent.at : Math.min(upper, conversation.parent.at); + if (query.minEntryId !== undefined && upper < query.minEntryId) break; + conversation = (await this.#conversation(executor, conversation.parent.conversationId, context))!; + } + return { kind: "found", value: page(values, limit, "descending") }; + } + + /** Oldest first: the fork chain's segments from the root forward, each up to its fork point. */ + async #entriesAscending( + executor: SqliteExecutor, + query: EntryQuery, + limit: number, + after: number | undefined, + context: Context, + ): Promise>> { + const segments: { readonly conversationId: ConversationId; readonly upper: number | undefined }[] = []; + let conversation = await this.#conversation(executor, query.conversationId, context); + if (conversation === undefined) return unknownConversation(query.conversationId); + let upper: number | undefined = query.maxEntryId; + while (true) { + segments.push({ conversationId: conversation.id, upper }); + if (conversation.parent === undefined) break; + upper = upper === undefined ? conversation.parent.at : Math.min(upper, conversation.parent.at); + if (query.minEntryId !== undefined && upper < query.minEntryId) break; + conversation = (await this.#conversation(executor, conversation.parent.conversationId, context))!; + } + let lower: number | undefined = query.minEntryId; + if (after !== undefined) lower = Math.max(lower ?? after + 1, after + 1); + const values: EntryRecord[] = []; + for (const segment of segments.reverse()) { + const range = this.#range(segment.conversationId, lower, segment.upper); + const rows = await this.#queries.entries.select(executor, range, "ascending", limit + 1 - values.length); + this.#queries.entries.records(rows, values, context); + if (values.length > limit) break; + } + return { kind: "found", value: page(values, limit, "ascending") }; + } + + /** One conversation's entries with ids from `lower` to `upper` (inclusive, each optional). */ + #range(conversationId: ConversationId, lower: number | undefined, upper: number | undefined): Condition[] { + const conditions: Condition[] = [["conversation_id", "eq", conversationId]]; + if (lower !== undefined) conditions.push(["id", "gte", lower]); + if (upper !== undefined) conditions.push(["id", "lte", upper]); + return conditions; + } + + /** A page of a table scan: up to `limit` records of `queries`' table whose `conditions` hold, from `start`. */ + #scan }>( + queries: TableQueries, + conditions: readonly Condition[], + start: ScanStart, + limit: number, + context: Context, + ): Promise> { + return this.#store + .read((executor) => queries.select(executor, conditions, start.order, limit + 1), context) + .then((rows) => { + const values: T[] = []; + queries.records(rows, values, context); + return page(values, limit, start.order); + }); + } + + /** A copy's source read at its point, before the step (S11); a request error is thrown here, outside the read. */ + async #copySource(source: DocumentCopySource, context: Context) { + const read = await this.#store.readTransaction( + (executor) => readAt(executor, source.id, source.at, this.#owner), + context, + ); + return documentOf(read); + } +} + +/** A read that found nothing. */ +const NOTHING_FOUND: Read = { kind: "found", value: undefined }; + +function unknownConversation(id: ConversationId): Read { + return { kind: "refused", message: `Unknown conversation: ${id}` }; +} + +/** A read's data, its request error thrown now that the guarded read returned (slice 6b §1.4). */ +function told(read: Read): T { + if (read.kind === "refused") throw new StorageRequestError(read.message); + return read.value; +} + +/** A document read's result, a record that keeps no history at the point thrown as Durable's request error. */ +function documentOf(read: DocumentRead): StoredDocument | undefined { + if (read.kind === "current_only") + throw new StorageRequestError(`Document ${read.id} does not retain historical content`); + return read.document; +} diff --git a/packages/durable/src/storage/store/tables.ts b/packages/durable/src/storage/store/tables.ts new file mode 100644 index 000000000..49523292d --- /dev/null +++ b/packages/durable/src/storage/store/tables.ts @@ -0,0 +1,122 @@ +// Durable's records as declared tables of the store (slice 6b D2, §2.2): every row carries its record's indexed columns +// and the whole record as one `json(...)` cell. Indexed strings that must stay lossless (`kind`, `request_id`) hold +// Durable's JSON encoding; index columns are never `opt`, so absent values take sentinels no encoded value can be +// (`0` for an absent owner, which is never an id; `""` for an absent request id: an encoded string starts with `"`). + +import type { JsonValue } from "@earendil-works/chord"; +import { defineTable, int, json, str, unknown } from "@earendil-works/chord/next"; +import type { ConversationRecord, EntryRecord, SubmissionRecord, TaskRecord } from "../../types.ts"; + +/** A record as one `json(...)` cell. */ +const record = () => json(unknown(), { version: 1 }); + +/** Which record type owns an id: Durable's one namespace across its five record types. */ +export const RecordIds = defineTable("record_ids", { + version: 1, + columns: { id: int(), record_type: str() }, + key: ["id"], +}); + +export const Conversations = defineTable("conversations", { + version: 1, + columns: { id: int(), owner_conversation_id: int(), owner_task_id: int(), record: record() }, + key: ["id"], + indexes: { by_owner_conversation: ["owner_conversation_id"], by_owner_task: ["owner_task_id"] }, +}); + +export const Entries = defineTable("entries", { + version: 1, + columns: { id: int(), conversation_id: int(), commit_seq: int(), record: record() }, + key: ["id"], + indexes: { by_conversation: ["conversation_id"] }, +}); + +/** + * The entries with a head marker, by conversation: what `SqliteStorage`'s partial index on `head IS NOT NULL` is, + * as rows written only with such an entry, so a commit of an entry without one updates no second index. + */ +export const EntryHeads = defineTable("entry_heads", { + version: 1, + columns: { id: int(), conversation_id: int() }, + key: ["id"], + indexes: { by_conversation: ["conversation_id"] }, +}); + +export const Tasks = defineTable("tasks", { + version: 1, + columns: { + id: int(), + conversation_id: int(), + kind: str(), + status: str(), + abort_requested: int(), + background: int(), + record: record(), + }, + key: ["id"], + indexes: { + by_status: ["status"], + by_conversation: ["conversation_id"], + by_kind: ["kind"], + by_abort_requested: ["abort_requested"], + by_background: ["background"], + }, +}); + +export const Submissions = defineTable("submissions", { + version: 1, + columns: { id: int(), conversation_id: int(), request_id: str(), status: str(), record: record() }, + key: ["id"], + indexes: { + by_request: ["conversation_id", "request_id"], + by_conversation: ["conversation_id"], + by_status: ["status"], + }, +}); + +/** Durable's JSON encoding of an indexed string: lossless whatever the string (lone surrogates, NUL). */ +export const encodeIndexed = (value: string): string => JSON.stringify(value); + +/** The row of a conversation record. */ +export function conversationRow(value: ConversationRecord): object { + return { + id: value.id, + owner_conversation_id: value.owner?.conversationId ?? 0, + owner_task_id: value.owner?.taskId ?? 0, + record: value, + }; +} + +/** The row of an entry record; `commit_seq` is stamped by the step. */ +export function entryRow(value: EntryRecord): object { + return { + id: value.id, + conversation_id: value.conversationId, + commit_seq: 0, + record: value, + }; +} + +/** The row of a task record. */ +export function taskRow(value: TaskRecord): object { + return { + id: value.id, + conversation_id: value.conversationId, + kind: encodeIndexed(value.kind), + status: value.state.status, + abort_requested: value.abortRequested ? 1 : 0, + background: value.background ? 1 : 0, + record: value, + }; +} + +/** The row of a submission record. */ +export function submissionRow(value: SubmissionRecord): object { + return { + id: value.id, + conversation_id: value.conversationId, + request_id: value.requestId === undefined ? "" : encodeIndexed(value.requestId), + status: value.status, + record: value, + }; +} diff --git a/packages/durable/src/testing/storage-conformance.ts b/packages/durable/src/testing/storage-conformance.ts index 2797c0462..91b9cf057 100644 --- a/packages/durable/src/testing/storage-conformance.ts +++ b/packages/durable/src/testing/storage-conformance.ts @@ -1646,6 +1646,30 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, ), + createCase( + options, + "refuses a creation and a change of one document in one commit, in either order", + async (storage) => { + await createRoot(storage); + for (const order of ["create first", "change first"] as const) { + const id = await storage.mintId(); + const create: StorageWrite = { + type: "document.create", + record: { id, kind: "twice", key: order, scope: { kind: "session" } }, + content: { kind: "base", version: 1, value: { x: 1 } }, + }; + const change: StorageWrite = { + type: "document.change", + id, + content: { kind: "delta", version: 1, ops: [["s", ["x"], 2]] }, + }; + const writes = order === "create first" ? [create, change] : [change, create]; + await expect(storage.commit(writes, adopt, context)).rejects.toThrow("more than one content command"); + expect(await storage.document(id, "current", context)).toBeUndefined(); + } + }, + ), + createCase( options, "rolls back record tables and secondary indexes when a document command fails", diff --git a/packages/durable/test/conversation-view-bench.ts b/packages/durable/test/conversation-view-bench.ts index d9096293f..e3b781850 100644 --- a/packages/durable/test/conversation-view-bench.ts +++ b/packages/durable/test/conversation-view-bench.ts @@ -3,7 +3,8 @@ * * node --conditions=source --experimental-strip-types --expose-gc test/conversation-view-bench.ts * - * Scenarios, each in its own child process so heap numbers are its own, on SQLite so history stays out of the heap: + * Scenarios, each in its own child process so heap numbers are its own, on SQLite so history stays out of the heap + * (`SqliteStorage`; with `PI_DURABLE_BENCH_STORAGE=store`, Durable's storage on Chord's store, `StoreStorage`): * - `start`: acquire a watch of a conversation of N entries (the mount is built every time; the watch is stopped). * - `append`: K started watches; commit entries one per commit, or in bursts of 50 per commit, and wait for delivery. * - `resume`: a watch with `from` 20 entries back, as a client that reconnects after a short drop. @@ -22,6 +23,7 @@ import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; import { createModels } from "@earendil-works/pi-ai/models"; import { type Conversation, createRegistry, type EntryId, Harness, LiveDoc, type Storage } from "../src/index.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { openStoreStorage } from "./store-storage-support.ts"; type Scenario = | { kind: "start"; entries: number } @@ -114,7 +116,11 @@ async function run(scenario: Scenario): Promise> { const directory = mkdtempSync(join(tmpdir(), "pi-durable-view-bench-")); let storage: Storage | undefined; try { - storage = await openNodeSqliteStorage(join(directory, "session.sqlite")); + const path = join(directory, "session.sqlite"); + storage = + process.env.PI_DURABLE_BENCH_STORAGE === "store" + ? await openStoreStorage(path) + : await openNodeSqliteStorage(path); const models = createModels(); const harness = await Harness.open(storage, { models, registry: createRegistry() }, context); const root = await harness.root(context); diff --git a/packages/durable/test/examples/35-durable-on-the-store.ts b/packages/durable/test/examples/35-durable-on-the-store.ts new file mode 100644 index 000000000..958a9d1b4 --- /dev/null +++ b/packages/durable/test/examples/35-durable-on-the-store.ts @@ -0,0 +1,108 @@ +// Durable on the store: a session's worker keeps everything in one store. Durable's Session (a Harness) opens +// `StoreStorage` over the host's store under its object's type name, next to a Chord object's own table and documents; +// both commit through the store's one commit queue (one seq each), one id allocator numbers them all, and one export +// carries both. +// Run from packages/durable: +// node --conditions=source --experimental-strip-types test/examples/35-durable-on-the-store.ts +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { + createHost, + defineFacet, + defineTable, + local, + method, + openStore, + sqliteBackend, + str, +} from "@earendil-works/chord/next"; +import { openNodeSqliteDatabase } from "@earendil-works/chord/sqlite/node"; +import { createModels } from "@earendil-works/pi-ai"; +import { createRegistry, Harness, ROOT_CONVERSATION_ID } from "../../src/index.ts"; +import { StoreStorage } from "../../src/storage/store/index.ts"; + +const context = BACKGROUND_CONTEXT; + +const Notes = defineTable("notes", { version: 1, columns: { id: str(), text: str() }, key: ["id"] }); + +// The harness object: local with empty params, so it lives as long as the host and may persist (P3). +const Chat = local("example.chat", { + version: 1, + params: {}, + members: { note: method(str()), say: method(str()) }, +}); + +const facet = defineFacet((host) => { + host.serve(Chat, (_, self) => { + // Its own table, under its type name ... + const notes = self.table(Notes); + // ... and Durable's records under the same owner name: claims per table and per document namespace. + const harness = self.open( + async (open) => { + const opened = await Harness.open( + await StoreStorage.open(store, Chat.name, open), + { models: createModels(), registry: createRegistry() }, + open, + ); + await opened.root(open); + return opened; + }, + (h, _reason, close) => h.close(close), + ); + return { + note: async (text, ctx) => { + await self.tx(async (tx) => tx.put(notes, { id: `n${text.length}`, text }), ctx); + return null; + }, + say: async (text, ctx) => { + const root = await harness.value.conversation(ROOT_CONVERSATION_ID, ctx); + await root!.submit({ type: "write", entry: { kind: "example.said", data: { text } } }, ctx); + return null; + }, + }; + }); +}); + +const directory = await mkdtemp(join(tmpdir(), "chord-example-26-")); +const path = join(directory, "session.sqlite"); +// 1. The worker's start code opens the store and gives it to its host. +const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context); +const host = createHost({ store, onError: (error) => console.log("onError", error.kind) }); +await host.load([{ name: "@example/core", facet }], context); +const chat = await host.get(Chat, {}, context); + +// 2. Chord's commit and Durable's commit, each one seq of the same store. +const before = store.lastSeq; +await chat.note("hello", context); +await chat.say("hello, Durable", context); +console.log("seqs taken:", store.lastSeq - before); + +// 3. One archive holds both: Chord's table, Durable's tables and documents, all under the owner name. +const owners = new Map(); +for await (const part of store.export(context)) { + const owner = + part.kind === "table" || part.kind === "put" + ? part.table.split(":")[0] + : part.kind === "document.create" + ? part.record.owner + : undefined; + if (owner !== undefined) owners.set(`${owner} ${part.kind}`, (owners.get(`${owner} ${part.kind}`) ?? 0) + 1); + if (part.kind === "manifest") console.log("manifest:", { lastSeq: part.lastSeq, nextId: part.nextId }); +} +console.log("parts by owner and kind:", Object.fromEntries(owners)); + +// 4. The app closes the host, then the store; a reopen finds both. +await host.close(context); +await store.close(context); +const reopened = await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context); +const storage = await StoreStorage.open(reopened, Chat.name, context); +const page = await storage.scanEntries({ conversationId: ROOT_CONVERSATION_ID }, 10, undefined, context); +console.log( + "Durable entries after reopen:", + page.items.map((entry) => entry.kind), +); +await storage.close(context); +await reopened.close(context); +await rm(directory, { recursive: true, force: true }); diff --git a/packages/durable/test/examples/36-durable-views.ts b/packages/durable/test/examples/36-durable-views.ts new file mode 100644 index 000000000..2f2ace488 --- /dev/null +++ b/packages/durable/test/examples/36-durable-views.ts @@ -0,0 +1,91 @@ +// Durable's views as Chord sources: `self.follow(attach)` turns Durable's synchronous observers (the conversation view, +// the task graph) into watchables, members and derive sources of the object that opened Durable. With Durable on the +// host's store, Durable adopts its commits inside the store's commit step, in the host's batch: a derive over two views +// of one commit computes once, and a watcher gets one frame per commit. +// Run from packages/durable: +// node --conditions=source --experimental-strip-types test/examples/36-durable-views.ts + +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { + arr, + createHost, + defineFacet, + int, + local, + method, + obj, + openStore, + sqliteBackend, + str, +} from "@earendil-works/chord/next"; +import { openNodeSqliteDatabase } from "@earendil-works/chord/sqlite/node"; +import { createModels } from "@earendil-works/pi-ai"; +import { createRegistry, Harness, ROOT_CONVERSATION_ID } from "../../src/index.ts"; +import { StoreStorage } from "../../src/storage/store/index.ts"; + +const context = BACKGROUND_CONTEXT; + +const Chat = local("example.chat", { + version: 1, + params: {}, + members: { + // What a UI shows: the kinds of the transcript's entries and the number of tasks, from two views. + summary: obj({ kinds: arr(str()), tasks: int() }), + write: method(str()), + }, +}); + +const facet = defineFacet((host) => { + host.serve(Chat, (_, self) => { + const harness = self.open( + async (open) => { + const opened = await Harness.open( + await StoreStorage.open(store, Chat.name, open), + { models: createModels(), registry: createRegistry() }, + open, + ); + await opened.root(open); + return opened; + }, + (h, _reason, close) => h.close(close), + ); + // 1. Follow two of Durable's views; they attach once the natives opened. + const view = self.follow((observer, ctx) => + harness.value.observeConversation(ROOT_CONVERSATION_ID, observer, ctx, { min: 50 }), + ); + const graph = self.follow((observer, ctx) => harness.value.observeTaskGraph(observer, ctx)); + // 2. A derive over both: one compute per Durable commit. + const summary = self.derive([view, graph], ([v, g]) => ({ + kinds: v.entries.map((entry) => entry.kind), + tasks: Object.keys(g.tasks).length, + })); + return { + summary, + write: async (kind, ctx) => { + const root = await harness.value.conversation(ROOT_CONVERSATION_ID, ctx); + await root!.submit({ type: "write", entry: { kind } }, ctx); + return null; + }, + }; + }); +}); + +const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context); +const host = createHost({ store, onError: (error) => console.log("onError", error.kind) }); +await host.load([{ name: "@example/core", facet }], context); +const chat = await host.get(Chat, {}, context); + +// 3. Watch the member: one frame per Durable commit, with its ops. +const watch = await chat.summary.watch(context); +watch.start(async (call) => { + if (call.kind === "value") console.log("frame:", call.value.kinds, JSON.stringify(call.ops)); +}); +await chat.write("example.first", context); +await chat.write("example.second", context); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("member value:", chat.summary.value); + +// 4. Closing the host detaches the observers; the store is the app's. +await watch.stop(); +await host.close(context); +await store.close(context); diff --git a/packages/durable/test/storage-memory.ts b/packages/durable/test/storage-memory.ts index 0e105fdc8..49828a4b0 100644 --- a/packages/durable/test/storage-memory.ts +++ b/packages/durable/test/storage-memory.ts @@ -17,8 +17,9 @@ import { openNodeJsonlStorage } from "../src/storage/jsonl/node.ts"; import { MemoryStorage } from "../src/storage/memory.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import type { Storage } from "../src/types.ts"; +import { openStoreStorage } from "./store-storage-support.ts"; -const STORAGE_BENCHMARK_BACKENDS = ["memory", "sqlite", "jsonl"] as const; +const STORAGE_BENCHMARK_BACKENDS = ["memory", "sqlite", "store", "jsonl"] as const; type StorageBenchmarkBackend = (typeof STORAGE_BENCHMARK_BACKENDS)[number]; type MemorySnapshot = { @@ -125,11 +126,13 @@ async function runWorker(backend: StorageBenchmarkBackend, scale: StorageBenchma storage = new MemoryStorage(); } else { directory = await mkdtemp(join(tmpdir(), `pi-durable-${backend}-memory-`)); - path = join(directory, backend === "sqlite" ? "storage.sqlite" : "storage"); + path = join(directory, backend === "jsonl" ? "storage" : "storage.sqlite"); storage = backend === "sqlite" ? await openNodeSqliteStorage(path) - : await openNodeJsonlStorage(path, BACKGROUND_CONTEXT); + : backend === "store" + ? await openStoreStorage(path) + : await openNodeJsonlStorage(path, BACKGROUND_CONTEXT); } try { @@ -151,7 +154,7 @@ async function runWorker(backend: StorageBenchmarkBackend, scale: StorageBenchma collectGarbage(); const postRead = snapshot(); let disk: DiskFootprint | undefined; - if (path !== undefined && backend === "sqlite") { + if (path !== undefined && backend !== "jsonl") { disk = { ...sqliteMetrics(path), mainBytes: await fileSize(path), diff --git a/packages/durable/test/storage.bench.ts b/packages/durable/test/storage.bench.ts index f24e915eb..68fdb06a3 100644 --- a/packages/durable/test/storage.bench.ts +++ b/packages/durable/test/storage.bench.ts @@ -14,8 +14,9 @@ import { openNodeJsonlStorage } from "../src/storage/jsonl/node.ts"; import { MemoryStorage } from "../src/storage/memory.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import type { EntryId, Storage } from "../src/types.ts"; +import { openStoreStorage } from "./store-storage-support.ts"; -const STORAGE_BENCHMARK_BACKENDS = ["memory", "sqlite", "jsonl"] as const; +const STORAGE_BENCHMARK_BACKENDS = ["memory", "sqlite", "store", "jsonl"] as const; type StorageBenchmarkBackend = (typeof STORAGE_BENCHMARK_BACKENDS)[number]; const READ_OPTIONS = { time: 300, iterations: 10, warmupTime: 75, warmupIterations: 3 } as const; @@ -38,9 +39,10 @@ async function createFixture(backend: StorageBenchmarkBackend): Promise } const directory = await mkdtemp(join(tmpdir(), "pi-durable-benchmark-")); directories.push(directory); - if (backend === "sqlite") { + if (backend === "sqlite" || backend === "store") { const path = join(directory, "storage.sqlite"); - const fixture = { backend, storage: await openNodeSqliteStorage(path) } satisfies Fixture; + const storage = backend === "sqlite" ? await openNodeSqliteStorage(path) : await openStoreStorage(path); + const fixture = { backend, storage } satisfies Fixture; fixtures.push(fixture); return fixture; } @@ -123,11 +125,12 @@ type ReopenFixture = { }; async function openPersistentStorage(backend: PersistentBackend, path: string): Promise { + if (backend === "store") return openStoreStorage(path); return backend === "sqlite" ? openNodeSqliteStorage(path) : openNodeJsonlStorage(path, BACKGROUND_CONTEXT); } async function copyPersistentStorage(backend: PersistentBackend, source: string, destination: string): Promise { - if (backend === "sqlite") await copyFile(source, destination); + if (backend !== "jsonl") await copyFile(source, destination); else await cp(source, destination, { recursive: true }); } @@ -136,7 +139,7 @@ for (const backend of STORAGE_BENCHMARK_BACKENDS) { if (backend === "memory") continue; const seedDirectory = await mkdtemp(join(tmpdir(), `pi-durable-${backend}-reopen-benchmark-`)); directories.push(seedDirectory); - const seedPath = join(seedDirectory, backend === "sqlite" ? "storage.sqlite" : "storage"); + const seedPath = join(seedDirectory, backend === "jsonl" ? "storage" : "storage.sqlite"); const seed = await openPersistentStorage(backend, seedPath); const dataset = await seedStorageBenchmark(seed); await seed.close(BACKGROUND_CONTEXT); @@ -144,7 +147,7 @@ for (const backend of STORAGE_BENCHMARK_BACKENDS) { Array.from({ length: REOPEN_OPTIONS.iterations + REOPEN_OPTIONS.warmupIterations }, async () => { const directory = await mkdtemp(join(tmpdir(), `pi-durable-${backend}-reopen-sample-`)); directories.push(directory); - const path = join(directory, backend === "sqlite" ? "storage.sqlite" : "storage"); + const path = join(directory, backend === "jsonl" ? "storage" : "storage.sqlite"); await copyPersistentStorage(backend, seedPath, path); return path; }), diff --git a/packages/durable/test/store-storage-node.ts b/packages/durable/test/store-storage-node.ts new file mode 100644 index 000000000..3dff775d4 --- /dev/null +++ b/packages/durable/test/store-storage-node.ts @@ -0,0 +1,23 @@ +// What `vitest.store.config.ts` puts in the place of `../src/storage/sqlite/node.ts`: the same exports, with +// `openNodeSqliteStorage(path)` opening Durable's storage on Chord's store over that file (`StoreStorage`), so the +// harness test files that take a SQLite storage run unchanged on the store. + +import type { NodeSqliteDatabaseOptions } from "@earendil-works/chord/sqlite/node"; +import type { SqliteStorageOptions } from "../src/storage/sqlite/index.ts"; +import type { Storage } from "../src/types.ts"; +import { openStoreStorage } from "./store-storage-support.ts"; + +export { + NodeSqliteDatabase, + type NodeSqliteDatabaseOptions, + openNodeSqliteDatabase, +} from "@earendil-works/chord/sqlite/node"; + +/** Node SQLite connection settings plus durable storage options, as the original takes them. */ +export type NodeSqliteStorageOptions = NodeSqliteDatabaseOptions & SqliteStorageOptions; + +/** A `StoreStorage` on a store over the file at `path`; a table prefix becomes nothing (the owner name separates). */ +export function openNodeSqliteStorage(path: string, options: NodeSqliteStorageOptions = {}): Promise { + const { prefix: _prefix, ...database } = options; + return openStoreStorage(path, database); +} diff --git a/packages/durable/test/store-storage-support.ts b/packages/durable/test/store-storage-support.ts new file mode 100644 index 000000000..1143d6eaf --- /dev/null +++ b/packages/durable/test/store-storage-support.ts @@ -0,0 +1,46 @@ +// StoreStorage over a SQLite file for tests: a store opened on the file, Durable's storage in it under a test owner +// name, and a close that closes the store too (the app's part: in tests the storage is the store's only owner). + +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { openStore, sqliteBackend } from "@earendil-works/chord/next"; +import { type NodeSqliteDatabaseOptions, openNodeSqliteDatabase } from "@earendil-works/chord/sqlite/node"; +import { StoreStorage } from "../src/storage/store/index.ts"; +import type { Storage } from "../src/types.ts"; + +/** The owner name the test storages use. */ +export const TEST_OWNER = "durable.test"; + +/** A `StoreStorage` on a store over the SQLite file at `path` (`:memory:` too) whose `close` closes the store too. */ +export async function openStoreStorage(path: string, options: NodeSqliteDatabaseOptions = {}): Promise { + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(path, options)), BACKGROUND_CONTEXT); + let storage: StoreStorage; + try { + storage = await StoreStorage.open(store, TEST_OWNER, BACKGROUND_CONTEXT); + } catch (error) { + await store.close(BACKGROUND_CONTEXT); + throw error; + } + let closing: Promise | undefined; + const close = (context: Parameters[0]): Promise => { + closing ??= storage.close(context).then(() => store.close(context)); + return closing; + }; + // StoreStorage keeps its state in private fields: its methods run on the storage itself, bound once. + const bound = new Map([["close", close]]); + return new Proxy(storage, { + get(target, property) { + let value = bound.get(property); + if (value !== undefined) return value; + value = Reflect.get(target, property, target); + if (typeof value !== "function") return value; + value = value.bind(target); + bound.set(property, value); + return value; + }, + // A test replacing a method (`storage.commit = …`) gets its own on the next read. + set(target, property, value) { + bound.delete(property); + return Reflect.set(target, property, value, target); + }, + }); +} diff --git a/packages/durable/test/store-storage.test.ts b/packages/durable/test/store-storage.test.ts new file mode 100644 index 000000000..94b7dee3a --- /dev/null +++ b/packages/durable/test/store-storage.test.ts @@ -0,0 +1,592 @@ +// StoreStorage (slice 6b §2): Durable's storage conformance suite on SQLite `:memory:` and across reopen, then what is +// its own: claims and close, owners sharing a store, the store's semantics for a refused commit, the allocator. + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import type { Context } from "@earendil-works/chord"; +import { BACKGROUND_CONTEXT, withTelemetry } from "@earendil-works/chord/context"; +import { + applyBatch, + type ChordStore, + createHost, + defineFacet, + defineState, + importStore, + int, + local, + method, + obj, + openStore, + type Store, + sqliteBackend, + str, +} from "@earendil-works/chord/next"; +import { openNodeSqliteDatabase } from "@earendil-works/chord/sqlite/node"; +import { createModels } from "@earendil-works/pi-ai"; +import { registerStorageConformance } from "@earendil-works/pi-durable/testing"; +import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; +import { describe, expect, it } from "vitest"; +import { StorageRequestError } from "../src/errors.ts"; +import { idFromNumber } from "../src/ids.ts"; +import { type CompactionResult, createRegistry, Harness, LiveDoc, type Tx } from "../src/index.ts"; +import { StoreStorage } from "../src/storage/store/index.ts"; +import type { ConversationId, DocumentId, EntryId, Id, Seq, Storage, StorageWrite, TaskId } from "../src/types.ts"; +import { ROOT_CONVERSATION_ID } from "../src/types.ts"; +import { openStoreStorage } from "./store-storage-support.ts"; + +const context = BACKGROUND_CONTEXT; +const adopt = (): void => {}; + +/** A storage that closes and reopens its store file after every commit (Durable's reopen harness). */ +class ReopeningStorage implements Storage { + #current: Storage; + readonly #path: string; + #closed = false; + + constructor(current: Storage, path: string) { + this.#current = current; + this.#path = path; + } + + async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, commitContext: Context): Promise { + if (this.#closed) return this.#current.commit(writes, adopt, commitContext); + try { + return await this.#current.commit(writes, adopt, commitContext); + } finally { + await this.#current.close(context); + this.#current = await openStoreStorage(this.#path); + } + } + + mintId>(): Promise { + return this.#current.mintId(); + } + conversation: Storage["conversation"] = (id, c) => this.#current.conversation(id, c); + scanConversations: Storage["scanConversations"] = (q, l, cursor, c) => + this.#current.scanConversations(q, l, cursor, c); + entry(id: EntryId, c: Context): ReturnType; + entry(conversationId: ConversationId, id: EntryId, c: Context): ReturnType; + entry(first: EntryId | ConversationId, second: EntryId | Context, third?: Context) { + if (third === undefined) return this.#current.entry(idFromNumber(first), second as Context); + return this.#current.entry(idFromNumber(first), second as EntryId, third); + } + findLatestHeadMarker: Storage["findLatestHeadMarker"] = (id, at, c) => this.#current.findLatestHeadMarker(id, at, c); + scanEntries: Storage["scanEntries"] = (q, l, cursor, c) => this.#current.scanEntries(q, l, cursor, c); + task: Storage["task"] = (id, c) => this.#current.task(id, c); + scanTasks: Storage["scanTasks"] = (q, l, cursor, c) => this.#current.scanTasks(q, l, cursor, c); + submission: Storage["submission"] = (id, c) => this.#current.submission(id, c); + scanSubmissions: Storage["scanSubmissions"] = (q, l, cursor, c) => this.#current.scanSubmissions(q, l, cursor, c); + submissionByRequest: Storage["submissionByRequest"] = (id, request, c) => + this.#current.submissionByRequest(id, request, c); + findDocument: Storage["findDocument"] = (address, at, c) => this.#current.findDocument(address, at, c); + document: Storage["document"] = (id, at, c) => this.#current.document(id, at, c); + scanDocuments: Storage["scanDocuments"] = (q, l, cursor, c) => this.#current.scanDocuments(q, l, cursor, c); + + async close(closeContext: Context): Promise { + if (this.#closed) return; + this.#closed = true; + await this.#current.close(closeContext); + } +} + +registerStorageConformance({ describe, expect, it }, "StoreStorage", async (use) => { + const storage = await openStoreStorage(":memory:"); + try { + await use(storage); + } finally { + await storage.close(context); + } +}); + +registerStorageConformance({ describe, expect, it }, "StoreStorage across reopen", async (use) => { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-store-conformance-")); + const path = join(directory, "store.sqlite"); + const storage = new ReopeningStorage(await openStoreStorage(path), path); + try { + await use(storage); + } finally { + await storage.close(context); + await rm(directory, { recursive: true, force: true }); + } +}); + +/** A store on `:memory:`. */ +async function memoryStore() { + return openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context); +} + +describe("claims and close", () => { + it("refuse a second StoreStorage on one owner until the first closed; two owners share a store", async () => { + const store = await memoryStore(); + const first = await StoreStorage.open(store, "harness", context); + await expect(StoreStorage.open(store, "harness", context)).rejects.toMatchObject({ + code: "invalid", + reason: "args", + }); + const other = await StoreStorage.open(store, "other", context); + await first.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + expect(await other.conversation(ROOT_CONVERSATION_ID, context)).toBeUndefined(); + await first.close(context); + await expect(first.conversation(ROOT_CONVERSATION_ID, context)).rejects.toThrow("StoreStorage is closed"); + const reopened = await StoreStorage.open(store, "harness", context); + expect(await reopened.conversation(ROOT_CONVERSATION_ID, context)).toEqual({ id: ROOT_CONVERSATION_ID }); + await store.close(context); + }); + + it("waits at close for its calls underway, then releases its claims", async () => { + const store = await memoryStore(); + const storage = await StoreStorage.open(store, "harness", context); + const commit = storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + const closing = storage.close(context); + await commit; + await closing; + await StoreStorage.open(store, "harness", context); + await store.close(context); + }); +}); + +describe("the store's semantics (D4)", () => { + it("a refused Durable commit rejects with Durable's message and keeps the store usable", async () => { + const store = await memoryStore(); + const storage = await StoreStorage.open(store, "harness", context); + const root = { type: "conversation", value: { id: ROOT_CONVERSATION_ID } } as const; + await storage.commit([root], adopt, context); + await expect(storage.commit([root], adopt, context)).rejects.toThrow("ID 1 already belongs to conversation"); + const entry = { + id: idFromNumber(await storage.mintId()), + conversationId: ROOT_CONVERSATION_ID, + kind: "m", + }; + await storage.commit([{ type: "entry", value: entry }], adopt, context); + expect((await storage.entry(entry.id, context))?.entry).toEqual(entry); + await store.close(context); + }); + + it("raises the allocator above every id it holds when it opens (a store built by import)", async () => { + const store = await memoryStore(); + const storage = await StoreStorage.open(store, "harness", context); + const entry = { id: idFromNumber(500), conversationId: ROOT_CONVERSATION_ID, kind: "m" }; + await storage.commit( + [ + { type: "conversation", value: { id: ROOT_CONVERSATION_ID } }, + { type: "entry", value: entry }, + ], + adopt, + context, + ); + expect(await storage.mintId()).toBe(501); + await store.close(context); + }); +}); + +const MemoDoc = defineState("memo", { + schema: obj({ m: int() }), + version: 1, + initial: () => ({ m: 1 }), + persist: true, + checkpointWhen: () => true, +}); + +const Memo = local("app.memo", { version: 1, params: {}, members: { memo: obj({ m: int() }), set: method() } }); + +/** A store on `:memory:` with an outbox from its first commit, and Durable's storage in it. */ +async function recorded() { + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context, { + outbox: true, + })) as ChordStore; + const storage = await StoreStorage.open(store, "harness", context); + await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + return { store, storage }; +} + +/** Rewindable history, a retired latest document, a create-plus-retire and a change-plus-retire. */ +async function history(storage: Storage): Promise<{ readonly rewindable: DocumentId; readonly seqs: Seq[] }> { + const scope = { kind: "conversation", conversationId: ROOT_CONVERSATION_ID } as const; + const rewindable = await storage.mintId(); + const latest = await storage.mintId(); + const empty = await storage.mintId(); + const seqs: Seq[] = []; + const base = (n: number) => ({ kind: "base", version: 1, value: { n } }) as const; + seqs.push( + await storage.commit( + [ + { + type: "document.create", + record: { id: rewindable, kind: "r", scope, history: "rewindable", fork: "asOf" }, + content: base(0), + }, + { + type: "document.create", + record: { id: latest, kind: "l", scope, history: "latest", fork: "current" }, + content: base(0), + }, + ], + adopt, + context, + ), + ); + seqs.push( + await storage.commit( + [{ type: "document.change", id: rewindable, content: { kind: "delta", version: 1, ops: [["s", ["n"], 1]] } }], + adopt, + context, + ), + ); + seqs.push( + await storage.commit( + [ + { type: "document.change", id: latest, content: { kind: "delta", version: 1, ops: [["s", ["n"], 2]] } }, + { type: "document.retire", id: latest }, + { type: "document.create", record: { id: empty, kind: "e", scope: { kind: "session" } }, content: base(3) }, + { type: "document.retire", id: empty }, + ], + adopt, + context, + ), + ); + return { rewindable, seqs }; +} + +describe("archives and batches of Durable data", () => { + it("export and import keep rewindable history, retired records and Chord's own data: reads at a point agree", async () => { + const { store, storage } = await recorded(); + const { rewindable, seqs } = await history(storage); + // Chord's own data in the same store: an object's persisted document. + const host = createHost({ store, onError: () => {} }); + const facet = defineFacet((h) => + h.serve(Memo, (_, self) => { + const memo = self.state(MemoDoc); + return { + memo, + set: async (ctx) => { + await memo.change((draft) => { + draft.m = 2; + }, ctx); + return null; + }, + }; + }), + ); + await host.load([{ name: "app", facet }], context); + await (await host.get(Memo, {}, context)).set(context); + await host.close(context); + const parts = []; + for await (const part of store.export(context)) parts.push(part); + expect(parts.filter((part) => part.kind === "document.create").map(({ record }) => record.owner)).toContain( + "app.memo", + ); + const copy = await importStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), parts, context); + const imported = await StoreStorage.open(copy, "harness", context); + for (const at of [seqs[0]!, seqs[1]!, "current" as const]) { + expect(await imported.document(rewindable, at, context)).toEqual( + await storage.document(rewindable, at, context), + ); + } + const scope = { kind: "conversation", conversationId: ROOT_CONVERSATION_ID } as const; + const at = seqs[1]!; + expect(await imported.scanDocuments({ scope, at }, 10, undefined, context)).toEqual( + await storage.scanDocuments({ scope, at }, 10, undefined, context), + ); + const again = []; + for await (const part of copy.export(context)) again.push(part); + expect(again).toEqual(parts); + expect(await imported.mintId()).toBe(await storage.mintId()); + await copy.close(context); + await store.close(context); + }); + + it("an outbox of Durable commits replayed into an empty store gives the same archive (create-plus-retire included)", async () => { + const { store, storage } = await recorded(); + await history(storage); + const commits = await (store as ChordStore).outbox!.read(0, 1_000, context); + const replica = (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context)) as ChordStore; + await applyBatch(replica, commits, context); + const exported = async (from: Store) => { + const parts = []; + for await (const part of from.export(context)) parts.push(part); + return parts; + }; + expect(await exported(replica)).toEqual(await exported(store)); + await replica.close(context); + await store.close(context); + }); +}); + +describe("head markers (the entry_heads table)", () => { + it("a head entry is found after reopen, in an imported archive and in a replayed outbox", async () => { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-store-heads-")); + const path = join(directory, "store.sqlite"); + const open = async () => + (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context, { outbox: true })) as ChordStore; + let store = await open(); + let storage = await StoreStorage.open(store, "harness", context); + await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + const entry = async (head?: EntryId): Promise => { + const id = await storage.mintId(); + const value = { id, conversationId: ROOT_CONVERSATION_ID, kind: "k", data: {}, ...(head ? { head } : {}) }; + await storage.commit([{ type: "entry", value }], adopt, context); + return id; + }; + const first = await entry(); + const marker = await entry(first); + await entry(); + /** The newest head marker, and the one at or before the first entry (none). */ + const heads = async (of: Storage) => [ + (await of.findLatestHeadMarker(ROOT_CONVERSATION_ID, undefined, context))?.id, + (await of.findLatestHeadMarker(ROOT_CONVERSATION_ID, first, context))?.id, + ]; + expect(await heads(storage)).toEqual([marker, undefined]); + await storage.close(context); + await store.close(context); + store = await open(); + storage = await StoreStorage.open(store, "harness", context); + expect(await heads(storage)).toEqual([marker, undefined]); + const parts = []; + for await (const part of store.export(context)) parts.push(part); + expect(parts.filter((part) => part.kind === "put" && part.table === "harness:entry_heads")).toEqual([ + expect.objectContaining({ row: { id: marker, conversation_id: ROOT_CONVERSATION_ID } }), + ]); + const copy = await importStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), parts, context); + expect(await heads(await StoreStorage.open(copy, "harness", context))).toEqual([marker, undefined]); + const replica = (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context)) as ChordStore; + await applyBatch(replica, await store.outbox!.read(0, 1_000, context), context); + expect(await heads(await StoreStorage.open(replica, "harness", context))).toEqual([marker, undefined]); + for (const each of [copy, replica, store]) await each.close(context); + await rm(directory, { recursive: true, force: true }); + }); +}); + +describe("reads (slice 6b §1.4, §2.4)", () => { + it("a request error inside a multi-query read rejects that read only; the store goes on", async () => { + const { store, storage } = await recorded(); + await expect( + storage.scanEntries({ conversationId: idFromNumber(999) }, 10, undefined, context), + ).rejects.toBeInstanceOf(StorageRequestError); + await expect(storage.document(idFromNumber(999), "current", context)).resolves.toBeUndefined(); + expect(await storage.conversation(ROOT_CONVERSATION_ID, context)).toEqual({ id: ROOT_CONVERSATION_ID }); + await store.close(context); + }); +}); + +describe("commits (slice 6b §1.3, §2.5)", () => { + it("adopt in the attached host's batch, and count four statements for an entry append and for a delta", async () => { + const telemetry = new InMemoryTelemetryContext(); + const span = await telemetry.startSpan({ name: "test" }, (each) => each); + const metered = withTelemetry(span, context); + const counts = (): number => + telemetry + .getMetrics() + .filter(({ name }) => name === "chord.store.statements") + .reduce((total, { value }) => total + value, 0); + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context)) as ChordStore; + const host = createHost({ store, onError: () => {} }); + const storage = await StoreStorage.open(store, "harness", context); + await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + let running: boolean | undefined; + const entry = { + id: idFromNumber(await storage.mintId()), + conversationId: ROOT_CONVERSATION_ID, + kind: "m", + }; + const before = counts(); + await storage.commit( + [{ type: "entry", value: entry }], + () => { + running = store.batch!.running; + }, + metered, + ); + const appended = counts() - before; + expect(running).toBe(true); + const id = await storage.mintId(); + const record = { id, kind: "s", scope: { kind: "session" } } as const; + await storage.commit( + [{ type: "document.create", record, content: { kind: "base", version: 1, value: { n: 0 } } }], + adopt, + context, + ); + const changed = counts(); + await storage.commit( + [{ type: "document.change", id, content: { kind: "delta", version: 1, ops: [["s", ["n"], 1]] } }], + adopt, + metered, + ); + expect([appended, counts() - changed]).toEqual([4, 4]); + await host.close(context); + await store.close(context); + }); +}); + +describe("Durable's views as Chord sources on the store (slice 6b §3)", () => { + const Chat = local("app.chat", { + version: 1, + params: {}, + members: { entries: int(), note: method(obj({ kind: str() })) }, + }); + + it("a derive over the conversation view and the task graph: one frame per Durable commit, both views in one batch", async () => { + const store = await memoryStore(); + const host = createHost({ store, onError: () => {} }); + let computes = 0; + const facet = defineFacet((h) => + h.serve(Chat, (_, self) => { + const harness = self.open( + async (open) => { + const opened = await Harness.open( + await StoreStorage.open(store, Chat.name, open), + { models: createModels(), registry: createRegistry() }, + open, + ); + await opened.root(open); + return opened; + }, + (h, _reason, close) => h.close(close), + ); + const view = self.follow((observer, ctx) => + harness.value.observeConversation(ROOT_CONVERSATION_ID, observer, ctx), + ); + const graph = self.follow((observer, ctx) => harness.value.observeTaskGraph(observer, ctx)); + const entries = self.derive([view, graph], ([v]) => { + computes++; + return v.entries.length; + }); + return { + entries, + note: async ({ kind }, ctx) => { + const root = await harness.value.conversation(ROOT_CONVERSATION_ID, ctx); + await root!.submit({ type: "write", entry: { kind } }, ctx); + return null; + }, + }; + }), + ); + await host.load([{ name: "app", facet }], context); + const chat = await host.get(Chat, {}, context); + const calls: unknown[] = []; + const watch = await chat.entries.watch(context); + watch.start(async (call) => void calls.push(call.kind === "value" ? call.value : call.kind)); + const before = computes; + await chat.note({ kind: "test.note" }, context); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(chat.entries.value).toBeGreaterThan(0); + expect(calls).toEqual([chat.entries.value]); + expect(computes - before).toBe(1); + await host.close(context); + await store.close(context); + }); + + it("a deferred Durable commit, then an immediate one, reach the followed document in order", async () => { + const store = await memoryStore(); + const host = createHost({ store, onError: () => {} }); + const Live = local("app.live", { + version: 1, + params: {}, + members: { n: int(), bump: method() }, + }); + /** A commit setting the root's `LiveDoc` failure note to `value`. */ + const set = (value: number) => async (tx: Tx) => { + (await tx.doc(LiveDoc, ROOT_CONVERSATION_ID)).compactionFailure = { + taskId: idFromNumber>(1), + trigger: "manual", + error: String(value), + }; + }; + const facet = defineFacet((h) => + h.serve(Live, (_, self) => { + const harness = self.open( + async (open) => { + const opened = await Harness.open( + await StoreStorage.open(store, Live.name, open), + { models: createModels(), registry: createRegistry() }, + open, + ); + await (await opened.root(open)).commit(set(0), open); + return opened; + }, + (h, _reason, close) => h.close(close), + ); + const live = self.follow((observer, ctx) => + harness.value.observeDoc(LiveDoc, ROOT_CONVERSATION_ID, observer, ctx), + ); + return { + n: self.derive([live], ([value]) => Number(value.compactionFailure?.error ?? -1)), + bump: async (ctx) => { + const root = (await harness.value.conversation(ROOT_CONVERSATION_ID, ctx))!; + await root.commit(set(1), ctx, { persistence: { kind: "deferred", withinMs: 10_000 } }); + await root.commit(set(2), ctx); + return null; + }, + }; + }), + ); + await host.load([{ name: "app", facet }], context); + const live = await host.get(Live, {}, context); + const seen: unknown[] = []; + const watch = await live.n.watch(context); + watch.start(async (call) => void seen.push(call.kind === "value" ? call.value : call.kind)); + await live.bump(context); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(live.n.value).toBe(2); + expect(seen.at(-1)).toBe(2); + await host.close(context); + await store.close(context); + }); +}); + +describe("review regressions", () => { + it('a head-marker entry whose kind is "refused" is returned, not taken for a request error', async () => { + const storage = await openStoreStorage(":memory:"); + await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + const first = await storage.mintId(); + const marker = await storage.mintId(); + await storage.commit( + [ + { type: "entry", value: { id: first, conversationId: ROOT_CONVERSATION_ID, kind: "message" } }, + { + type: "entry", + value: { id: marker, conversationId: ROOT_CONVERSATION_ID, kind: "refused", head: first }, + }, + ], + adopt, + context, + ); + expect((await storage.findLatestHeadMarker(ROOT_CONVERSATION_ID, undefined, context))?.id).toBe(marker); + const page = await storage.scanEntries({ conversationId: ROOT_CONVERSATION_ID }, 10, undefined, context); + expect(page.items.map(({ kind }) => kind)).toEqual(["refused", "message"]); + await storage.close(context); + }); + + it("an empty commit takes no seq", async () => { + const { store, storage } = await recorded(); + const before = store.lastSeq; + const adopted: Seq[] = []; + expect(await storage.commit([], (seq) => adopted.push(seq), context)).toBe(before); + expect(store.lastSeq).toBe(before); + expect(adopted).toEqual([before]); + await store.close(context); + }); +}); + +describe("GC", () => { + it("a closed StoreStorage is collected while its store stays open", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const store = await memoryStore(); + const opened = async (): Promise> => { + const storage = await StoreStorage.open(store, "harness", context); + await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + await storage.close(context); + return new WeakRef(storage); + }; + const ref = await opened(); + for (let round = 0; round < 10 && ref.deref() !== undefined; round++) { + await new Promise((resolve) => setImmediate(resolve)); + gc(); + } + expect(ref.deref()).toBeUndefined(); + await store.close(context); + }); +}); diff --git a/packages/durable/vitest.store.config.ts b/packages/durable/vitest.store.config.ts new file mode 100644 index 000000000..b98674172 --- /dev/null +++ b/packages/durable/vitest.store.config.ts @@ -0,0 +1,47 @@ +// The harness test files that take a SQLite storage, run on Durable's storage on Chord's store (`StoreStorage`, slice +// 6b): `../src/storage/sqlite/node.ts`'s opener is replaced by `test/store-storage-node.ts`; the files are unchanged. +// Four tests edit or read `SqliteStorage`'s own tables with raw SQL (`tasks`, `durable_metadata`) and are left out. + +import { fileURLToPath } from "node:url"; +import { defineConfig, mergeConfig } from "vitest/config"; +import base from "./vitest.config.ts"; + +const storeOpener = fileURLToPath(new URL("./test/store-storage-node.ts", import.meta.url)); + +export default mergeConfig( + base, + defineConfig({ + test: { + testNamePattern: + /^(?!.*(resumes a retry backoff after reopen|sleeps again for a retry stored as a running phase|polls a checkpoint stored without a thinking level|refuses to create the root of an empty storage))/, + include: [ + "test/harness-after-run.test.ts", + "test/harness-blocked.test.ts", + "test/harness-compaction.test.ts", + "test/harness-continue.test.ts", + "test/harness-conversations.test.ts", + "test/harness-generation-recovery.test.ts", + "test/harness-inbox.test.ts", + "test/harness-input-entries.test.ts", + "test/harness-lifecycle.test.ts", + "test/harness-nested-tools-restart.test.ts", + "test/harness-nested-tools.test.ts", + "test/harness-ownership.test.ts", + "test/harness-read-only.test.ts", + "test/harness-restart-failure.test.ts", + "test/harness-scheduler-indexes.test.ts", + "test/harness-scheduler-retention.test.ts", + "test/harness-sleep.test.ts", + "test/harness-structured.test.ts", + "test/harness-submissions.test.ts", + "test/harness-task-graph.test.ts", + "test/harness-tasks-recovery.test.ts", + "test/harness-tool-hooks.test.ts", + "test/harness-tools-recovery.test.ts", + "test/harness-turn-limit.test.ts", + "test/harness-view.test.ts", + ], + }, + resolve: { alias: [{ find: /^\.\.\/src\/storage\/sqlite\/node\.ts$/, replacement: storeOpener }] }, + }), +); diff --git a/tsconfig.json b/tsconfig.json index 8a90126b1..eea4c716b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -8,6 +8,7 @@ "@earendil-works/chord": ["./packages/chord/src/index.ts"], "@earendil-works/chord/context": ["./packages/chord/src/context/index.ts"], "@earendil-works/chord/delta": ["./packages/chord/src/delta/index.ts"], + "@earendil-works/chord/next": ["./packages/chord/src/next/index.ts"], "@earendil-works/chord/node": ["./packages/chord/src/node.ts"], "@earendil-works/chord/output": ["./packages/chord/src/output.ts"], "@earendil-works/chord/telemetry-sqlite": ["./packages/chord/src/telemetry-sqlite/index.ts"], From 9017ce0813307c73c80e0f5f1378aebf89f0832e Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 19:27:58 +0000 Subject: [PATCH 112/271] docs(chord): slice 6b built --- packages/chord/docs/next/slice-6b.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/packages/chord/docs/next/slice-6b.md b/packages/chord/docs/next/slice-6b.md index 78f3a71b3..45e75f5f0 100644 --- a/packages/chord/docs/next/slice-6b.md +++ b/packages/chord/docs/next/slice-6b.md @@ -1,12 +1,7 @@ # Slice 6b: Durable and jobs on the store (spec) -Status: **ready to build** (2026-10-27): D1-D9, R1-R4 and the claims decided by the owner (decision log, entries -"Slice 6b …" and "owner names and claims"); two spec reviews (`/tmp/s6b-review{,2}-{astra,fable,opus,sol}.md`) -kept every decision; this version closes their findings. Written by the redesign session after reading, in full, -Durable's `types.ts`, its SQLite storage, schema and migrations, the storage conformance suite and benchmark, -`session/session.ts`, `session/transaction.ts`, `session/forks.ts`, the scheduler's restarts, `writeAtClose` and `abort`, -and Chord v2's store, engine, applier, commit step and jobs. Durable's memory, JSONL and IndexedDB storages stay as they -are (S15). +Status: **built** (2026-10-28, pi `14b4c56e5`: one build, a build review and two fix rounds by the four spec reviewers; +reopen, the one-entry commit's mean and the resume watch's CPU accepted by the owner). Was: ready to build (2026-10-27). The single source for building 6b, together with `design.md` read in full; the decision log `decisions.md` (later entries win), in particular "The store", S1, S2, S4, S10, S11, S13, S15, W1, W12, "Slice 6 first decisions", "Durable's From e613c7487b3f4692f7c22f8d60d5836e2d6dee06 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 19:41:25 +0000 Subject: [PATCH 113/271] =?UTF-8?q?docs(chord):=20design=20=C2=A73=20place?= =?UTF-8?q?ment:=20a=20parent=20link=20that=20is=20down=20waits?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/chord/docs/next/design.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index 8c6aed5a5..03bbadc47 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -132,7 +132,8 @@ export const Catalog = remote("catalog", { placement: () => Server(), … }); | an arriving hold for a served type whose params place it on another host | broken `invalid/placement`; the address the hold names only routes it | | the hold names an address of a kind this host has a route for | over that route's link (the address's params checked against the kind, then `authorize`) | | otherwise (a remote type) | over the parent link; a hold that arrived *on* the parent link is never sent back to it: `unavailable` | - | no parent link | `unavailable` | + | no parent link: the host never had one | `unavailable` | + | no parent link: it is down (a drop) | waits on the parent path, `loading`, until the next link (design audit, 2026-10-27) | A `placement` or `authorize` that throws fails the hold broken with that error; an address whose params fail the kind's check is broken `invalid/placement`. From 6975eb18a62ea7d2082eaa73aa6f8f883414697a Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 19:44:10 +0000 Subject: [PATCH 114/271] =?UTF-8?q?docs(chord):=20design=20=C2=A73=20route?= =?UTF-8?q?=20context,=20=C2=A77=20retry=20of=20a=20refused=20hold?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/chord/docs/next/design.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index 03bbadc47..7429a66e9 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -273,8 +273,7 @@ host.route(Session, { completed its handshake resets the backoff. Without holds for the address nothing is resolved again, so passive viewers recover without a user action while they hold. - Chord never closes a route link itself except at `host.close`; stopping idle workers is the app's (`host.activity`). -- Route links have no app context yet: work arriving on them runs under the background context without telemetry - (slice 4b gives routes a context). +- Work arriving on a route's links runs under the context given to `host.route(Kind, options, context)` (slice 4b). ### Links @@ -749,7 +748,8 @@ location are dropped. | object `failed` | reject at once with the cause | `failed` with cause and `retryAt` | - A failed reference stays alive: while held, Chord retries per the backoff; `ref.retry()` tries at once (a Retry - button; over a link the `retry { hold }` message); success goes `loading`, then `live`. A provider tells a pending + button; over a link the `retry { hold }` message, which also re-runs a refused hold's start: authorize, placement, + binding; design audit 2026-10-27); success goes `loading`, then `live`. A provider tells a pending hold's consumer each failure (`failed`, with the rest of its backoff window) and the next attempt (`loading`). - **Parked calls** (slice 3b): calls made while the reference is loading are queued in order, each waiting at most the reference's `readyTimeout` from the moment it was made (never renewed), then fail `unavailable/park_timeout`; they From 92c1da1f62a5fd2ba18c70f6f8691014793fd5fb Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:44:25 +0100 Subject: [PATCH 115/271] fix(env): fail exec and pty with spawn_error when a helper thread cannot start (#11540) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pi-env daemon started an `exec`'s stdout and stderr readers and its waiter with `thread::spawn`. `thread::spawn` panics when no thread can be created, for example when a cgroup `TasksMax` or `RLIMIT_NPROC` limit is reached. When that happened after the child was already running: - only the request thread unwound, so the exec never got a reply; - its control channel was dropped, so `cancel` and `kill` reached nobody; - the child kept running, even after the daemon exited. `pty` started its reader, input writer and waiter threads the same way. ## Changes - `exec.rs`: `spawn_reader` now uses `thread::Builder::spawn` and returns an `io::Result`. If a reader or the waiter cannot start, `run` sets `stop`, interrupts any readers already started, kills the process group, reaps the child, removes it from `groups`, and returns `spawn_error`. - The waiter thread now gets the `Child` through a channel once it is running. If the waiter cannot start, the child stays with the caller, which can still reap it. Before, the failed thread's closure dropped the child without waiting for it. - `pty.rs`: the same handling for the reader, writer and waiter threads. On failure the terminal is also removed from `terminals`. ## Testing - Manual reproduction with a script that runs the daemon as `prlimit --nproc=N -- pi-env`. It starts 30 `sleep 30` execs 200 ms apart, aborts them all, then closes the connection. N ran from 37 to 70. - Before: for most N, one or two execs stayed pending forever after abort, and a `sleep 30` kept running after the daemon was closed. - After: every exec settles, either `aborted` or `spawn_error`, and no `sleep` is left running. - `cargo build`, `cargo clippy --all-targets`, `cargo fmt --check` and `cargo test` all pass. The `packages/env` vitest suite passes (78 passed), and `npm run check` passes. - There is no automated regression test. A thread-creation failure cannot be triggered reliably in a test: `RLIMIT_NPROC` counts all of the user's tasks and does not apply to root, and the daemon has no fault-injection hook. - The Windows build was not compiled. The `exec.rs` change uses only existing cross-platform calls, and the `pty.rs` change is Unix-only. Fixes #11433 Requested by @petrroll --------- Co-authored-by: triagent Co-authored-by: Petr Houška --- packages/env/CHANGELOG.md | 4 +++ packages/env/daemon/src/exec.rs | 55 +++++++++++++++++++++------------ packages/env/daemon/src/pty.rs | 39 ++++++++++++++++++----- 3 files changed, 72 insertions(+), 26 deletions(-) diff --git a/packages/env/CHANGELOG.md b/packages/env/CHANGELOG.md index 372e1aaea..c484db660 100644 --- a/packages/env/CHANGELOG.md +++ b/packages/env/CHANGELOG.md @@ -6,6 +6,10 @@ - `RemoteWatcher` follows Durable's `FileWatcher`: `close(context)` is `stop(context)`, and its changes are tagged (`{ kind: "paths", paths }`, `{ kind: "overflow" }`, `{ kind: "failed", error }`). The daemon's `watch` protocol is unchanged. Migration: `await watcher.close(ctx)` becomes `await watcher.stop(ctx)`; test `change.kind` instead of `"overflow" in change` or `"error" in change`. +### Fixed + +- `exec` and `pty` fail with `spawn_error` instead of hanging when the daemon cannot start a helper thread (for example at a cgroup task limit); the started command is killed and reaped ([#11433](https://github.com/earendil-works/pi/issues/11433)). + ## [1.4.0] - 2026-10-21 ## [1.3.0] - 2026-10-15 diff --git a/packages/env/daemon/src/exec.rs b/packages/env/daemon/src/exec.rs index 1d8b03845..d92c51279 100644 --- a/packages/env/daemon/src/exec.rs +++ b/packages/env/daemon/src/exec.rs @@ -217,8 +217,8 @@ fn spawn_reader( sender: Sender, stop: Arc, output: Option>, -) -> JoinHandle<()> { - thread::spawn(move || { +) -> std::io::Result> { + thread::Builder::new().spawn(move || { sys::read_pipe(pipe, &stop, |bytes| { let delivered = sender.send(Message::Data(stream, bytes.to_vec())).is_ok(); if let Some(output) = &output { @@ -298,29 +298,46 @@ pub fn run( } else { None }; - let readers = [ - spawn_reader( - 0, - child.stdout.take().unwrap(), - sender.clone(), - stop.clone(), - gate.clone(), - ), - spawn_reader( - 1, - child.stderr.take().unwrap(), - sender.clone(), - stop.clone(), - gate, - ), - ]; + // A thread that cannot start (a task limit) fails the command; the started child is killed and reaped. + let abandon = |mut child: Child, readers: &[JoinHandle<()>], error: std::io::Error| { + stop.store(true, Ordering::SeqCst); + for reader in readers { + sys::interrupt_reader(reader); + } + kill_group(pid); + // On Windows the group kill spawns taskkill, which can fail under the same limit; kill the child directly. + let _ = child.kill(); + let _ = child.wait(); + groups.lock().unwrap().remove(&pid); + Failure::new("spawn_error", error.to_string()) + }; + let mut readers = Vec::with_capacity(2); + let stdout = child.stdout.take().unwrap(); + match spawn_reader(0, stdout, sender.clone(), stop.clone(), gate.clone()) { + Ok(reader) => readers.push(reader), + Err(error) => return Err(abandon(child, &readers, error)), + } + let stderr = child.stderr.take().unwrap(); + match spawn_reader(1, stderr, sender.clone(), stop.clone(), gate) { + Ok(reader) => readers.push(reader), + Err(error) => return Err(abandon(child, &readers, error)), + } + // The waiter receives the child once it runs, so a waiter that cannot start leaves the child here to reap. + let (give, take) = mpsc::channel::(); let waiter = sender.clone(); - thread::spawn(move || { + let started = thread::Builder::new().spawn(move || { + let Ok(mut child) = take.recv() else { + return; + }; let _ = waiter.send(match child.wait() { Ok(status) => Message::Exit(status), Err(error) => Message::WaitFailed(error.to_string()), }); }); + if let Err(error) = started { + return Err(abandon(child, &readers, error)); + } + let _ = give.send(child); drop(sender); let mut decoders = [StreamDecoder::new(), StreamDecoder::new()]; diff --git a/packages/env/daemon/src/pty.rs b/packages/env/daemon/src/pty.rs index 00cc566aa..e8be9e222 100644 --- a/packages/env/daemon/src/pty.rs +++ b/packages/env/daemon/src/pty.rs @@ -182,7 +182,7 @@ pub fn run( format!("Working directory does not exist: {}", request.cwd), )); } - let (master, mut child) = + let (master, child) = unix::spawn(&request).map_err(|error| Failure::new("spawn_error", error.to_string()))?; let master = Arc::new(master); let pid = child.id(); @@ -203,12 +203,21 @@ pub fn run( )); let stop = Arc::new(AtomicBool::new(false)); - let reader = { + // A thread that cannot start (a task limit) fails the terminal; the started program is killed and reaped. + let abandon = |mut child: std::process::Child, error: std::io::Error| { + stop.store(true, Ordering::SeqCst); + kill_group(pid); + let _ = child.wait(); + terminals.lock().unwrap().remove(&id); + groups.lock().unwrap().remove(&pid); + Failure::new("spawn_error", error.to_string()) + }; + let started = { let master = master.clone(); let sender = sender.clone(); let stop = stop.clone(); let output = output.clone(); - thread::spawn(move || { + thread::Builder::new().spawn(move || { unix::read(&master, &stop, |bytes| { let delivered = sender.send(Message::Data(0, bytes.to_vec())).is_ok(); // Like a full pipe: a program that writes faster than the link waits. @@ -218,24 +227,40 @@ pub fn run( let _ = sender.send(Message::Eof(0)); }) }; - { + let reader = match started { + Ok(reader) => reader, + Err(error) => return Err(abandon(child, error)), + }; + let started = { let master = master.clone(); let stop = stop.clone(); - thread::spawn(move || { + thread::Builder::new().spawn(move || { for bytes in inputs { if !unix::write_all(&master, &bytes, &stop) { return; } } - }); + }) + }; + if let Err(error) = started { + return Err(abandon(child, error)); } + // The waiter receives the child once it runs, so a waiter that cannot start leaves the child here to reap. + let (give, take) = mpsc::channel::(); let waiter = sender.clone(); - thread::spawn(move || { + let started = thread::Builder::new().spawn(move || { + let Ok(mut child) = take.recv() else { + return; + }; let _ = waiter.send(match child.wait() { Ok(status) => Message::Exit(status), Err(error) => Message::WaitFailed(error.to_string()), }); }); + if let Err(error) = started { + return Err(abandon(child, error)); + } + let _ = give.send(child); drop(sender); let mut status = None; From 391e83fb17f01087b7037e1f7fe42bc35e62add1 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 19:50:41 +0000 Subject: [PATCH 116/271] fix(chord): Chord v2 follows its design where the audit of slices 1-6a found it did not A failed activation releases its references after its natives closed; a dependency going live while the dependent activates retries it at once; a call on a local object runs with the host's principal and incarnation; retiring an unloaded document checks its stored version and fingerprint, and a cut retirement no longer fails its successor; ref.retry() re-runs a refused hold's start on the same id (the provider keeps refused holds), and ends released/closed on a closed host; a relayed hold waits while the parent link is down; a link drop while a dependency is still loading is no failure. --- .../chord/src/next/documents/persistence.ts | 41 +++++-- .../chord/src/next/documents/transaction.ts | 25 +++- packages/chord/src/next/host/attempt.ts | 7 +- packages/chord/src/next/host/host.ts | 20 +-- packages/chord/src/next/host/object.ts | 22 +++- packages/chord/src/next/host/placement.ts | 6 +- packages/chord/src/next/host/reference.ts | 24 +++- packages/chord/src/next/jobs/persisted.ts | 4 +- packages/chord/src/next/remote/hold.ts | 34 ++++-- packages/chord/src/next/remote/path.ts | 8 ++ .../chord/src/next/remote/remote-object.ts | 42 +++---- .../test/next/documents/keyed-slot.test.ts | 90 ++++++++++++++ .../test/next/documents/persistence.test.ts | 5 +- packages/chord/test/next/host/ctx.test.ts | 28 ++++- packages/chord/test/next/host/host.test.ts | 19 ++- packages/chord/test/next/host/object.test.ts | 59 +++++++++ packages/chord/test/next/host/route.test.ts | 34 ++++++ packages/chord/test/next/remote/relay.test.ts | 36 ++++++ .../test/next/remote/remote-object.test.ts | 114 +++++++++++++----- 19 files changed, 514 insertions(+), 104 deletions(-) diff --git a/packages/chord/src/next/documents/persistence.ts b/packages/chord/src/next/documents/persistence.ts index 3efc45308..46aebb93e 100644 --- a/packages/chord/src/next/documents/persistence.ts +++ b/packages/chord/src/next/documents/persistence.ts @@ -29,6 +29,9 @@ export type Stored = { readonly deltasSinceBase: number; }; +/** A live document found without loading it: what is stored of it and the fingerprint it was stored with. */ +export type FoundDocument = { readonly stored: Stored; readonly fingerprint: string }; + /** A loaded document: its value at the definition's version, and what is stored of it. */ export type Loaded = { readonly value: JsonObject; readonly stored: Stored }; @@ -84,28 +87,27 @@ export class Persistence { */ decode(document: StoredDocument, definition: AnyDefinition, context: Context): Loaded { const { record, deltasSinceBase } = document; - const { name, version, fingerprint } = definition; - if (record.version > version) { - throw new Error(`${name}: stored at version ${record.version}, newer than the code's ${version}`); - } - if (record.version === version && record.fingerprint !== fingerprint) { - throw new Error(`${name}: stored with another schema at version ${version} (a schema changed without a bump)`); - } + checkReadable(record, definition); // The engine stores what commits gave it: JSON objects of the record's version. const stored = document.value as JsonObject; - const value = record.version === version ? stored : definition.migrate(stored, record.version, context); + const value = + record.version === definition.version ? stored : definition.migrate(stored, record.version, context); return { value, stored: { id: record.id, revision: record.revision, version: record.version, deltasSinceBase }, }; } - /** What is stored of the live document at `key` without loading it (a retirement's find), `undefined` when absent. */ - async find(key: string, context: Context): Promise { + /** + * What is stored of the live document at `key` without loading it (a retirement's find) and the schema fingerprint + * it was stored with (`checkReadable` takes both), `undefined` when absent. + */ + async find(key: string, context: Context): Promise { const record = await this.store.read((executor) => findDocument(executor, this.at(key)), context); if (record === undefined) return undefined; // The change counter is all a retirement needs; `deltasSinceBase` is not read for one. - return { id: record.id, revision: record.revision, version: record.version, deltasSinceBase: 0 }; + const stored = { id: record.id, revision: record.revision, version: record.version, deltasSinceBase: 0 }; + return { stored, fingerprint: record.fingerprint }; } /** Up to `limit` keys of live documents after `after`, in code point order. */ @@ -161,6 +163,23 @@ export class Persistence { } } +/** + * Throws when `definition` cannot read a document stored at `stored`'s version and fingerprint: a newer version than + * the code's, or the same version with another fingerprint (D9). A load and a retirement check alike (R15). + */ +export function checkReadable( + stored: { readonly version: number; readonly fingerprint: string }, + definition: AnyDefinition, +): void { + const { name, version, fingerprint } = definition; + if (stored.version > version) { + throw new Error(`${name}: stored at version ${stored.version}, newer than the code's ${version}`); + } + if (stored.version === version && stored.fingerprint !== fingerprint) { + throw new Error(`${name}: stored with another schema at version ${version} (a schema changed without a bump)`); + } +} + /** What is stored after a commit's `change` was adopted. */ export function storedAfter(change: StoredChange): Stored | undefined { switch (change.kind) { diff --git a/packages/chord/src/next/documents/transaction.ts b/packages/chord/src/next/documents/transaction.ts index 506c15468..af47dd2e4 100644 --- a/packages/chord/src/next/documents/transaction.ts +++ b/packages/chord/src/next/documents/transaction.ts @@ -4,6 +4,7 @@ import { type Context, withoutAbortSignal } from "../../context/index.ts"; import { type Change, type Draft, type Prepared, type Tracker, track } from "../../delta/index.ts"; import { ChordError } from "../data/index.ts"; +import { asChordError } from "../data/internal.ts"; import type { JobHandle, Outcome, Run, StartOptions } from "../jobs/index.ts"; import type { Job } from "../jobs/internal.ts"; import type { Columns, PutRowOf, RowOf } from "../store/columns.ts"; @@ -20,8 +21,8 @@ import type { Documents } from "./documents.ts"; import type { SealToken } from "./fence.ts"; import type { Keyed, KeyedHandle, Singleton, SingletonHandle } from "./handles.ts"; import type { KeyedSlot } from "./keyed-slot.ts"; -import type { Stored, StoredChange } from "./persistence.ts"; -import type { DocumentSlot, SingletonSlot } from "./slot.ts"; +import { checkReadable, type FoundDocument, type Stored, type StoredChange } from "./persistence.ts"; +import type { DocumentSlot, PersistedSlot, SingletonSlot } from "./slot.ts"; /** What `self.tx` gives its callback. Inside it: only `await tx.*`, no other `self.tx`, no calls to other objects. */ export interface Tx { @@ -213,9 +214,14 @@ export class Transaction implements Tx { const at = key === "" ? key : slot.key(key); const entry = this.retireSlot(slot, at); const persisted = slot.persisted; - // A cold persisted document is found, not loaded (one read; absent: a no-op, R13). + // A cold persisted document is found, not loaded (one read; absent: a no-op, R13); a version or fingerprint + // the code cannot read fails the owner as a load would (R15): a retirement of it is no exception. if (entry.found === undefined && persisted?.cold(at)) { - entry.found = await this.#track(persisted.persistence.find(at, this.#context)); + const found = await this.#track(persisted.persistence.find(at, this.#context)); + // Cut meanwhile: its attempt was sealed, and failing the owner now would fail its successor. + this.#assertOpen(); + if (found !== undefined) checkFound(found, persisted, slot); + entry.found = found?.stored; } } @@ -533,3 +539,14 @@ export class Transaction implements Tx { return entry.draft.change.state; } } + +/** Throws, failing `slot`'s owner `broken`, when its definition cannot read the cold document `found` (R15). */ +function checkFound(found: FoundDocument, persisted: PersistedSlot, slot: DocumentSlot): void { + try { + checkReadable({ version: found.stored.version, fingerprint: found.fingerprint }, persisted.definition.current); + } catch (error) { + const failure = asChordError(error, slot.owner.label); + slot.owner.fail(failure); + throw failure; + } +} diff --git a/packages/chord/src/next/host/attempt.ts b/packages/chord/src/next/host/attempt.ts index a424b5ff6..b6d04a898 100644 --- a/packages/chord/src/next/host/attempt.ts +++ b/packages/chord/src/next/host/attempt.ts @@ -408,9 +408,10 @@ export class Attempt implements StoreSelf, KeyDeclarer { const checked = checkArgs(served.provider, method, args, served.contract.version, context); const linked = withLinkedAbortSignal(this.#abort.signal, context); const callContext = linked.context; - // A local object's handlers act as the host; a remote one's as their caller. - const principal = served.kind === "remote" ? caller.principal : host.principal; - const ctx = new CallContext(callContext, { principal, incarnation: caller.incarnation }, host); + // A local object's handlers act as the host, its principal and its incarnation (§4); a remote one's as their + // caller. The caller stays the reference's (presence, §19). + const acting = served.kind === "remote" ? caller : { principal: host.principal, incarnation: host.incarnation }; + const ctx = new CallContext(callContext, acting, host); // Admitted to the drain before the handler runs: the handler may end its own object before it returns. const running = Promise.withResolvers(); host.count("calls", 1); diff --git a/packages/chord/src/next/host/host.ts b/packages/chord/src/next/host/host.ts index ecfad05ed..cbafd799b 100644 --- a/packages/chord/src/next/host/host.ts +++ b/packages/chord/src/next/host/host.ts @@ -181,8 +181,9 @@ export interface Host

(Type: C, params: ParamsOf, context: Context, options?: GetOptions): Promise>; /** * A reference at once, following its object until released: loading, live, failed (retried as its object is) and - * loading again while its link is down; it never rejects. Throws at once for invalid params and a local type not - * served here; before `load` resolved these are decided once it did, failing the reference broken. Its `variant` is `undefined` until its first binding. + * loading again while its link is down; it never rejects. Throws at once for invalid params (also before `load`) + * and a local type not served here; before `load` resolved, a type not served here fails the reference broken once + * it did. Its `variant` is `undefined` until its first binding. */ hold(Type: C, params: ParamsOf, context: Context, options?: GetOptions): Held; /** @@ -641,9 +642,9 @@ export class ChordHost

+ this.#state === "closed" + ? binding.end(this.#closedError()) + : this.#bindOrFail(binding, Type, checked, context); + binding.refuse({ kind: "broken", error: asChordError(error, Type.name) }, again); } } diff --git a/packages/chord/src/next/host/object.ts b/packages/chord/src/next/host/object.ts index 43e4d80fd..f422a58fb 100644 --- a/packages/chord/src/next/host/object.ts +++ b/packages/chord/src/next/host/object.ts @@ -125,6 +125,8 @@ export class ChordObject implements Target { readonly #holders = new Set(); /** Objects this object's activation waits for; followed to find cycles (Q5). */ readonly #waitingFor = new Set(); + /** The attempt a dependency went live during (R18): if it fails on a dependency, it is retried at once. */ + #woken: Attempt | undefined; /** The current attempt's outcome: resolves when it goes live, rejects with its error; renewed by every retry. */ #outcome = settlement(); /** The error of the last failed attempt, until one goes live. */ @@ -474,9 +476,17 @@ export class ChordObject implements Target { } } - /** A dependency went live: retried at once (backoff window skipped) when it failed on a dependency (R18). */ + /** + * A dependency went live: retried at once (backoff window skipped) when it failed on a dependency (R18). A wake + * while it activates is remembered: that activation may still fail on the dependency's earlier failure. + */ dependencyLive(): void { - if (this.#state.kind !== "failed" || this.#state.holding === undefined) return; + const state = this.#state; + if (state.kind === "activating") { + this.#woken = state.attempt; + return; + } + if (state.kind !== "failed" || state.holding === undefined) return; this.host.backoff.skip(this.identity); this.attempt(); } @@ -519,6 +529,7 @@ export class ChordObject implements Target { } #wentLive(attempt: Attempt): void { + this.#woken = undefined; this.#state = { kind: "live", attempt }; this.#exposed = attempt; if (this.#members.size > 0 || !this.#keys.empty) @@ -538,6 +549,8 @@ export class ChordObject implements Target { * broken: no automatic retry. */ #failed(attempt: Attempt, cause: FailureCause, holding: Attempt | undefined): void { + const woken = this.#woken === attempt; + this.#woken = undefined; // What its activation loaded for the pinned keys goes; they stay frozen. this.#keys.discard(); if (cause.kind === "transient") this.host.backoff.failed(this.identity); @@ -546,9 +559,10 @@ export class ChordObject implements Target { this.#state = { kind: "failed", failure, holding }; this.#lastFailure = cause.error; attempt.seal(this.ended()); - if (holding === undefined) attempt.releaseReferences(); const closing = attempt.close(END).then((failures) => { attempt.detach(); + // Released once its natives closed (a close may use them, §6, §13); R18's attempt keeps them for the retry. + if (holding === undefined) attempt.releaseReferences(); return failures; }); this.host.ending(this.identity, this.#reported(closing)); @@ -557,6 +571,8 @@ export class ChordObject implements Target { this.#notify(); // Job runs kept it while it activated; failed, they keep it no longer. this.#idle(); + // A dependency went live while this attempt activated: the wake it missed retries it now (R18). + if (holding !== undefined && woken) this.dependencyLive(); } /** The member source of `member`, made bound to `exposed`'s member by its first use. */ diff --git a/packages/chord/src/next/host/placement.ts b/packages/chord/src/next/host/placement.ts index 67f4215ca..e6011168f 100644 --- a/packages/chord/src/next/host/placement.ts +++ b/packages/chord/src/next/host/placement.ts @@ -51,15 +51,15 @@ export class Placement { /** * Where a hold arriving on `link` acting as `principal` goes: here when its type is served here; over the route of - * the address it names; else over the parent link, unless it came on the parent link or there is none - * (`unavailable`). Throws as `reference` does. + * the address it names; else over the parent link (waiting for it while it is down), unless it came on the parent + * link or this host never had one (`unavailable`). Throws as `reference` does. */ hold(request: HoldRequest, link: HostLink, principal: Principal | undefined, context: Context): Placed { if (this.#types.has(request.type)) return HERE; const { address } = request; const routed = this.#pathTo(address, principal, context); if (routed !== undefined) return { kind: "path", path: routed, address }; - if (link.role === "parent" || this.#parent.link === undefined) { + if (link.role === "parent" || !this.#parent.linked) { throw new ChordError({ code: "unavailable", message: `${request.type} is not served here` }); } return { kind: "path", path: this.#parent, address }; diff --git a/packages/chord/src/next/host/reference.ts b/packages/chord/src/next/host/reference.ts index 142dae425..4b9ed1ff8 100644 --- a/packages/chord/src/next/host/reference.ts +++ b/packages/chord/src/next/host/reference.ts @@ -123,7 +123,7 @@ type BindingState = readonly dependent: ChordObject | undefined; readonly target: Target; } - | { readonly kind: "refused"; readonly batch: Batch; readonly failure: Failed } + | { readonly kind: "refused"; readonly batch: Batch; readonly failure: Failed; readonly bind: () => void } | { readonly kind: "released"; readonly state: Ended }; /** What a wait for a state decides at each change. */ @@ -240,11 +240,15 @@ export class Binding implements Holder { this.moved(); } - /** A `host.hold` that cannot be bound once `load` resolved: failed with `cause`, which no retry changes. */ - refuse(cause: FailureCause): void { + /** + * A `host.hold` that cannot be bound (where it goes refused it): failed with `cause` until `ref.retry()` runs `bind` + * again (a route's `authorize` may admit it later). + */ + refuse(cause: FailureCause, bind: () => void): void { const state = this.#state; if (state.kind !== "unbound") return; - this.#state = { kind: "refused", batch: state.batch, failure: { kind: "failed", cause, retryAt: undefined } }; + const failure: Failed = { kind: "failed", cause, retryAt: undefined }; + this.#state = { kind: "refused", batch: state.batch, failure, bind }; this.moved(); } @@ -388,9 +392,17 @@ export class Binding implements Holder { return this.#ended.promise; } - /** `ref.retry()`: the object is asked for an attempt now; it ignores this unless failed. */ + /** `ref.retry()`: the object is asked for an attempt now (ignored unless failed); a refused hold is bound again. */ retry(): void { - if (this.#state.kind === "bound") this.#state.target.retry(); + const state = this.#state; + if (state.kind === "bound") { + state.target.retry(); + return; + } + if (state.kind !== "refused") return; + this.#state = { kind: "unbound", batch: state.batch, dependent: undefined }; + this.moved(); + state.bind(); } /** Releases the reference (idempotent); the object may end when it was its last. */ diff --git a/packages/chord/src/next/jobs/persisted.ts b/packages/chord/src/next/jobs/persisted.ts index 9802ad441..6530ec58d 100644 --- a/packages/chord/src/next/jobs/persisted.ts +++ b/packages/chord/src/next/jobs/persisted.ts @@ -69,8 +69,8 @@ export class PersistedRuns { /** Finds what is stored of run `id`'s state document (a run of another job version), keeping it for the drop. */ async findBlockedState(id: string, context: Context): Promise { - const stored = await this.#states.find(id, context); - if (stored !== undefined) this.#blockedStates.set(id, stored); + const found = await this.#states.find(id, context); + if (found !== undefined) this.#blockedStates.set(id, found.stored); } /** Every stored record of the job, in run id order, page by page with plain reads (activation holds the line). */ diff --git a/packages/chord/src/next/remote/hold.ts b/packages/chord/src/next/remote/hold.ts index 232d2122a..59ee856c9 100644 --- a/packages/chord/src/next/remote/hold.ts +++ b/packages/chord/src/next/remote/hold.ts @@ -72,6 +72,8 @@ type HoldState = /** The members it follows, in the order they were subscribed. */ readonly followed: Followed[]; } + /** Refused where it arrived (placement, `authorize`, a mismatch, invalid params, a cycle): `retry` starts it again. */ + | { readonly kind: "refused"; readonly request: HoldRequest } | { readonly kind: "released" }; /** A bound hold's state. */ @@ -84,8 +86,8 @@ const BINDING: HoldState = { kind: "binding" }; /** * One hold served here: a reference to its object acting as its caller (the hold's principal). While pending, each * failure of its object is sent as `failed` (with the rest of its backoff window) and the next attempt as `loading`; a - * hold that cannot be taken (a refused type, a mismatch, invalid params or placement) is a broken `failed` that ends - * it. Once live it binds when its link's window has a slot, then sends a part per batch that changed its members + * hold that cannot be taken (a refused type, a mismatch, invalid params or placement) is a broken `failed` that holds + * nothing until a `retry` starts it again. Once live it binds when its link's window has a slot, then sends a part per batch that changed its members * (or keeps their ops pending while the window is full), and `gone` when its object ends. */ export class Hold implements Holder { @@ -137,7 +139,7 @@ export class Hold implements Holder { * Admits the hold once `load` resolved: its params within the link's part limit (`too_large/part`), then relayed * (it is replaced by a relayed hold), or served here, its params opened and parsed (a malformed part is a protocol * error), its object taken and followed (`moved`). A hold that cannot go anywhere, a type that does not match, - * invalid params, a wrong placement or a cycle (Q5) is a broken `failed`, which ends it. + * invalid params, a wrong placement or a cycle (Q5) is a broken `failed`; the hold is kept, refused, for a `retry`. */ async start(request: HoldRequest): Promise { const { link } = this.#provider; @@ -169,7 +171,7 @@ export class Hold implements Holder { } object = link.host.serving(request, params, this.caller.principal!, link.context); } catch (error) { - return this.#refuse(asChordError(error, request.type)); + return this.#refuse(request, asChordError(error, request.type)); } // Counted before the object lists it in its presence: a derive over that may close this link synchronously. this.#object = object; @@ -203,9 +205,19 @@ export class Hold implements Holder { } } - /** A `retry` message: the object is asked for an attempt now (it ignores this unless failed). */ + /** + * A `retry` message: the object is asked for an attempt now (it ignores this unless failed); a refused hold is + * started again (a route's `authorize` may admit it now, a type may be served here now). + */ retry(): void { - this.#object?.retry(); + const state = this.#state; + if (state.kind !== "refused") { + this.#object?.retry(); + return; + } + this.#state = PENDING; + this.#provider.send({ kind: "loading", hold: this.id }); + void this.start(state.request); } /** Calls a method of the held object as the hold's caller. */ @@ -413,11 +425,13 @@ export class Hold implements Holder { this.#provider.touched(this); } - /** Refuses the hold: a broken `failed`, final; the provider forgets it. */ - #refuse(error: ChordError): void { - this.#provider.forget(this); + /** Refuses the hold: a broken `failed`; it keeps `request` for a `retry`, holding nothing until then. */ + #refuse(request: HoldRequest, error: ChordError): void { + const { params } = request; + // Its params owned, not a view of the message they came in (which the link's buffers may hold). + const owned = typeof params === "string" ? params : (params as Uint8Array).slice(); + this.#state = { kind: "refused", request: { ...request, params: owned } }; this.#provider.send({ kind: "failed", hold: this.id, cause: { kind: "broken", error: wireError(error) } }); - this.release(); } } diff --git a/packages/chord/src/next/remote/path.ts b/packages/chord/src/next/remote/path.ts index b1050ec7d..77efb0755 100644 --- a/packages/chord/src/next/remote/path.ts +++ b/packages/chord/src/next/remote/path.ts @@ -18,6 +18,8 @@ export class Path { readonly #holders = new Set(); readonly #changed: () => void; #link: HostLink | undefined; + /** Whether it ever had a link: a host never connected to a parent has no parent path to wait on. */ + #linked = false; constructor(changed: () => void) { this.#changed = changed; @@ -28,6 +30,11 @@ export class Path { return this.#link; } + /** Whether it had a link once (it may be down now: its holders wait for the next). */ + get linked(): boolean { + return this.#linked; + } + /** Whether anything is held over this path. */ get held(): boolean { return this.#holders.size > 0; @@ -49,6 +56,7 @@ export class Path { /** `link` is the path's link now: every holder is held on it. */ connected(link: HostLink): void { this.#link = link; + this.#linked = true; for (const holder of this.#holders) holder.attach(link); } diff --git a/packages/chord/src/next/remote/remote-object.ts b/packages/chord/src/next/remote/remote-object.ts index 450c051e1..1765b7d1d 100644 --- a/packages/chord/src/next/remote/remote-object.ts +++ b/packages/chord/src/next/remote/remote-object.ts @@ -153,10 +153,10 @@ export class RemoteObject implements Target { this.#hold = { side: link.consumer, id: link.consumer.attach(this, chain) }; } - /** The link dropped with `error`: loading until a new link's hold is bound (`#startOver`). */ - dropped(error: ChordError): void { + /** The link dropped: loading until a new link's hold is bound (`#startOver`). */ + dropped(_error: ChordError): void { this.#hold = undefined; - this.#startOver(error); + this.#startOver(); } /** @@ -214,16 +214,21 @@ export class RemoteObject implements Target { } /** - * `ref.retry()`: a failed object's provider is asked for an attempt; one that failed for a part past the link's - * limits is placed anew and held with a new id, now or on its path's next link (a placement that throws fails it - * broken again); ignored otherwise and without a link. + * `ref.retry()`: a failed object's provider is asked for an attempt (a hold it refused is started there again). One + * that failed for a part past the link's limits is placed anew and held with a new id, now or on its path's next + * link (a placement that throws fails it broken again). */ retry(): void { if (this.#state.kind !== "failed") return; - if (this.#path !== this.#host.unplaced) { - this.#hold?.side.retry(this.#hold.id); + if (this.#path === this.#host.unplaced) { + this.#place(); return; } + this.#hold?.side.retry(this.#hold.id); + } + + /** Places a reference whose part was past the limits anew, on its path now (a placement that throws fails it). */ + #place(): void { let path: Path; try { path = this.#host.pathOf(this.contract, this.#params, this.caller.principal); @@ -334,9 +339,7 @@ export class RemoteObject implements Target { * (`#startOver`). */ loading(): void { - this.#startOver( - new ChordError({ code: "unavailable", message: `${this.label}: its relay's upstream link dropped` }), - ); + this.#startOver(); } /** @@ -402,18 +405,13 @@ export class RemoteObject implements Target { } /** - * Loading again until the next binding; the members keep their last values. A dependent's activation waiting for - * this hold while it was loading already fails at once with `error` (transient) and is retried when a new hold is - * bound (R18). + * Loading again until the next binding; the members keep their last values. A drop while it is loading already + * changes nothing: a dependent's activation waiting for it goes on waiting for the next link's binding, bounded by + * its `readyTimeout` (§6, R18; a drop is not a dependency failure). */ - #startOver(error: ChordError): void { - if (this.#state.kind === "ended") return; - if (this.#state === LOADING) { - this.#settled.reject(error); - this.#settled = settlement(); - } else { - this.#move(LOADING); - } + #startOver(): void { + const { kind } = this.#state; + if (kind !== "ended" && kind !== "loading") this.#move(LOADING); } /** Moves to `state` and tells the reference; `settled` follows (a new wait each time it is loading again). */ diff --git a/packages/chord/test/next/documents/keyed-slot.test.ts b/packages/chord/test/next/documents/keyed-slot.test.ts index 674ff1e85..fbc3c2a3e 100644 --- a/packages/chord/test/next/documents/keyed-slot.test.ts +++ b/packages/chord/test/next/documents/keyed-slot.test.ts @@ -449,6 +449,96 @@ describe("cold keyed loads on the line", () => { await host.close(context); }); + it.each([ + ["a newer version", 2, { title: str(), n: str() }, "stored at version 2, newer than the code's 1"], + ["another fingerprint at its version", 1, { title: str(), n: str() }, "stored with another schema at version 1"], + ])( + "tx.retire of a cold key stored at %s fails the owner broken as a read would and keeps it (R15, audit)", + async (_, version, schema, message) => { + const store = await freshStore(); + // Written by other code: a newer version, or the same version with another schema. + const Other = defineState("notes.note", { + schema: obj(schema), + version, + initial: () => ({ title: "", n: "" }), + keyed: true, + persist: true, + checkpointWhen: () => false, + }); + const writer = await storeRestartable( + objects, + (_, self) => { + void self + .state(Other) + .change("a", (draft) => void Object.assign(draft, { title: "kept", n: "1" }), context); + }, + { store }, + ); + await flushed(); + await writer.host.close(context); + const { host, notes, errors, reference } = await persisted(store); + expect(await rejection(notes().retire("a", context))).toMatchObject({ + message: expect.stringContaining(message), + }); + expect(errors.map(({ kind }) => kind)).toEqual(["object_failed"]); + expect(reference.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + await host.close(context); + const reader = await storeRestartable(objects, (_, self) => void self.state(Other), { store }); + expect(await store.read((executor) => executor.all("SELECT key FROM chord_documents"), context)).toEqual([ + { key: "a" }, + ]); + await reader.host.close(context); + }, + ); + + it("a retirement cut by its attempt's seal while it finds the document fails no successor (audit)", async () => { + const store = await freshStore(); + const Newer = defineState("notes.note", { + schema: obj({ title: str() }), + version: 2, + initial: () => ({ title: "" }), + keyed: true, + persist: true, + checkpointWhen: () => false, + }); + const writer = await storeRestartable( + objects, + (_, self) => + void self.state(Newer).change( + "a", + (draft) => { + draft.title = "kept"; + }, + context, + ), + { store }, + ); + await flushed(); + await writer.host.close(context); + const { host, notes, errors, reference, fail, retry } = await persisted(store); + // The find's read is held past the seal: the retirement resumes on a cut transaction. + const read = store.read.bind(store); + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + store.read = async (fn, ctx) => { + store.read = read; + const value = await read(fn, ctx); + entered.resolve(); + await gate.promise; + return value; + }; + const retired = rejection(notes().retire("a", context)); + await entered.promise; + fail(); + await retry(); + gate.resolve(); + expect(await retired).toMatchObject({ code: "released" }); + await flushed(); + expect(reference.state.value).toEqual({ kind: "live" }); + expect(errors.filter(({ kind }) => kind === "object_failed")).toHaveLength(1); + await host.close(context); + }); + it("a sealed handle reads cached documents; a cold read is refused released", async () => { const store = await freshStore(); await seed(store, { a: "x", b: "y" }); diff --git a/packages/chord/test/next/documents/persistence.test.ts b/packages/chord/test/next/documents/persistence.test.ts index 13fe83ea8..b4be944bc 100644 --- a/packages/chord/test/next/documents/persistence.test.ts +++ b/packages/chord/test/next/documents/persistence.test.ts @@ -167,7 +167,10 @@ describe("loads", () => { }, undefined, ); - expect(await notes.find("a", context)).toMatchObject({ revision: 1, version: 1 }); + expect(await notes.find("a", context)).toMatchObject({ + stored: { revision: 1, version: 1 }, + fingerprint: expect.any(String), + }); expect(await notes.keys(undefined, 10, context)).toEqual(["a", "b", "\uFFFF", "\u{1F600}"]); expect(await notes.keys("b", 1, context)).toEqual(["\uFFFF"]); }); diff --git a/packages/chord/test/next/host/ctx.test.ts b/packages/chord/test/next/host/ctx.test.ts index 0d4cc480b..732ed2d3a 100644 --- a/packages/chord/test/next/host/ctx.test.ts +++ b/packages/chord/test/next/host/ctx.test.ts @@ -5,7 +5,7 @@ import { withLinkedAbortSignal } from "../../../src/next/host/ctx.ts"; import type { ChordHost } from "../../../src/next/host/host.ts"; import type { Ctx, Host, Reference } from "../../../src/next/host/index.ts"; import { constant } from "../../../src/next/state/index.ts"; -import { flushed, loaded, rawParent, rejection, type TestPrincipal } from "../support.ts"; +import { flushed, incarnationOf, linked, loaded, rawParent, rejection, type TestPrincipal } from "../support.ts"; const Item = local("app.item", { version: 1, params: { id: int() }, members: { id: int() } }); const Desk = local("app.desk", { @@ -140,6 +140,32 @@ describe("ctx", () => { }); }); +describe("local handlers act as the host (§4, audit)", () => { + it("a local object called from a remote handler's ctx.get runs with this host's principal and incarnation", async () => { + const Store = local("app.local-store", { version: 1, params: {}, members: { who: method(undefined, str()) } }); + const Gate = remote("app.gate", { + version: 1, + params: {}, + variants: { v: { methods: { who: method(undefined, str()), through: method(undefined, str()) } } }, + }); + const who = (ctx: Ctx) => `${ctx.principal?.kind ?? "none"} ${ctx.incarnation}`; + const { server, client } = await linked((h) => { + h.serve(Store, () => ({ who: async (ctx) => who(ctx) })); + h.serve(Gate, () => ({ + variant: "v", + members: { + who: async (ctx) => who(ctx), + through: async (ctx) => (await ctx.get(Store, {})).who(ctx), + }, + })); + }); + const gate = await client.get(Gate, {}, context); + // The remote handler acts as its caller; the local object it reaches acts as this host, both halves. + expect(await gate.who(context)).toBe(`owner ${incarnationOf(client)}`); + expect(await gate.through(context)).toBe(`server ${incarnationOf(server)}`); + }); +}); + describe("linked abort signals", () => { it("abort with the caller's signal or the call's, the caller's reason first", () => { for (const first of ["caller", "call"] as const) { diff --git a/packages/chord/test/next/host/host.test.ts b/packages/chord/test/next/host/host.test.ts index 07c4eb751..64a291b43 100644 --- a/packages/chord/test/next/host/host.test.ts +++ b/packages/chord/test/next/host/host.test.ts @@ -694,6 +694,16 @@ describe("acquiring: hold and get", () => { expect(await refused).toEqual(error); }); + it("invalid params before load resolved: hold throws and get rejects invalid/params at once (audit, 7a)", async () => { + const host = createHost(noErrors); + expect(() => host.hold(Page, { id: "x" } as never, context)).toThrow("params"); + expect(await rejection(host.get(Page, { id: "x" } as never, context))).toMatchObject({ + code: "invalid", + reason: "params", + }); + await host.load([], context); + }); + it("a hold released before load resolved makes nothing once it did", async () => { const factory = vi.fn(({ id }: { id: number }) => ({ id: constant(id) })); const host = createHost(noErrors); @@ -805,10 +815,10 @@ forEachCodec(() => { } /** Sends a hold of `PlacedNotes` (or `type`) for `session`, naming `address` when given. */ - function holdOf(peer: RawPeer, session: string, address?: string, type: string = PlacedNotes.name) { + function holdOf(peer: RawPeer, session: string, address?: string, type: string = PlacedNotes.name, hold = 0) { peer.send({ kind: "hold", - hold: 0, + hold, type, version: PlacedNotes.version, fingerprint: PlacedNotes.fingerprint, @@ -836,9 +846,10 @@ forEachCodec(() => { hold: 0, cause: { kind: "broken", error: { kind: "chord", error: misplaced } }, }); - holdOf(child, "S2"); + // A refused hold is kept (a retry starts it again): the next hold takes the next id. + holdOf(child, "S2", undefined, PlacedNotes.name, 1); expect(await child.next("failed")).toMatchObject({ - hold: 0, + hold: 1, cause: { kind: "broken", error: { kind: "chord", error: misplaced } }, }); }); diff --git a/packages/chord/test/next/host/object.test.ts b/packages/chord/test/next/host/object.test.ts index 2df9253e8..d492240f0 100644 --- a/packages/chord/test/next/host/object.test.ts +++ b/packages/chord/test/next/host/object.test.ts @@ -311,6 +311,30 @@ describe("dependency recovery (R18)", () => { ]); }); + it("remembers a dependency going live while the dependent activates: the dependent is retried at once (audit)", async () => { + const viewer = view(Core); + let cores = 0; + const { host } = await loaded((h) => { + // Its first factory throws (broken); the second goes live at once, without natives. + h.serve(Core, () => { + if (++cores === 1) throw new Error("first factory"); + return { n: constant(2) }; + }); + h.serve(View, viewer.factory); + }); + await flushed(); + // The view activates on the core's failure while the core is retried and goes live at once. + const dependency = host.hold(Core, {}, context); + const dependent = host.hold(View, { id: 1 }, context); + dependency.retry(); + await flushed(); + expect(dependency.state.value).toEqual({ kind: "live" }); + expect(dependent.state.value).toEqual({ kind: "live" }); + expect(viewer.times).toEqual([0, 0]); + dependent.release(); + dependency.release(); + }); + it("ignores a trigger while an attempt runs: simultaneous triggers give one attempt", async () => { const gate = Promise.withResolvers(); const svc = flaky((attempt) => attempt === 1, gate.promise); @@ -889,6 +913,41 @@ function nearHold(hold: number): Extract { return { kind: "hold", hold, type, version, fingerprint, params: jsonPart({ session: "S1" }) }; } +describe("a failed activation's cleanup (audit)", () => { + it("closes the natives that opened before it releases its references: a close may use them", async () => { + const Dependent = local("app.dependent", { version: 1, params: { id: int() }, members: { n: int() } }); + const seen: unknown[] = []; + const { host, errors } = await loaded((h) => { + h.serve(Core, () => ({ n: constant(5) })); + h.serve(Dependent, (_, self) => { + const core = self.get(Core, {}); + self.open( + async () => null, + () => { + try { + seen.push(core.n.value); + } catch (error) { + seen.push(String(error)); + } + }, + ); + self.open( + async () => { + throw broken(new Error("open failed")); + }, + () => {}, + ); + return { n: core.n }; + }); + }); + await expect(host.get(Dependent, { id: 1 }, context)).rejects.toThrow("open failed"); + await flushed(); + expect(errors.find(({ kind }) => kind === "activation_failed")).toMatchObject({ failure: { kind: "broken" } }); + expect(seen).toEqual([5]); + await host.close(context); + }); +}); + describe("a live object failing (slice 6a §1.4)", () => { it("fails broken: calls reject with the cause, one object_failed report, natives close, then references go", async () => { const events: string[] = []; diff --git a/packages/chord/test/next/host/route.test.ts b/packages/chord/test/next/host/route.test.ts index dc4485504..eb3d6413f 100644 --- a/packages/chord/test/next/host/route.test.ts +++ b/packages/chord/test/next/host/route.test.ts @@ -56,6 +56,40 @@ forEachCodec(() => { expect((await owner.tab.get(PlacedNotes, { session: "S2" }, context)).variant).toBe("editor"); }); + it("ref.retry() holds a refused hold again: authorize admitting it later, over a link and on this host (audit)", async () => { + let admit = false; + const { tab, server } = await relayed({ authorize: () => admit }); + const overLink = tab.hold(PlacedNotes, { session: "S1" }, context); + const here = server.hold(PlacedNotes, { session: "S2" }, context); + await transported(); + for (const ref of [overLink, here]) { + expect(ref.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "broken", error: { code: "forbidden" } }, + }); + } + admit = true; + overLink.retry(); + here.retry(); + await transported(); + expect(overLink.state.value).toEqual({ kind: "live" }); + expect(here.state.value).toEqual({ kind: "live" }); + }); + + it("ref.retry() of a reference refused on this host after host.close ends it released, nothing resolved (audit)", async () => { + let admit = false; + const { server, resolves } = await relayed({ authorize: () => admit }); + const ref = server.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + expect(ref.state.value).toMatchObject({ kind: "failed", cause: { error: { code: "forbidden" } } }); + await server.close(context); + admit = true; + ref.retry(); + await transported(); + expect(ref.state.value).toMatchObject({ kind: "ended", error: { code: "released", reason: "closed" } }); + expect(resolves).toEqual([]); + }); + it("fails a hold broken with what authorize throws", async () => { const { tab } = await relayed({ authorize: () => { diff --git a/packages/chord/test/next/remote/relay.test.ts b/packages/chord/test/next/remote/relay.test.ts index 78fbddb0c..701b2adaf 100644 --- a/packages/chord/test/next/remote/relay.test.ts +++ b/packages/chord/test/next/remote/relay.test.ts @@ -15,11 +15,13 @@ import { fakeTimers, forEachCodec, instancesOf, + Notes, PlacedNotes, RawPeer, rawWorkers, rejection, relayed, + serveNotes, type TestPrincipal, testCodec, transported, @@ -375,6 +377,40 @@ forEachCodec(() => { }); }); + describe("a parent link that is down (audit)", () => { + it("holds a hold arriving meanwhile on the parent path: it waits loading and binds on the next parent link", async () => { + const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + await server.load( + [{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], + context, + ); + const middle = createHost({ principal: { kind: "server" }, onError: () => {} }); + await middle.load([], context); + /** Connects the middle host to the server over a new link; returns the server's end. */ + const up = (): Link => { + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + middle.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); + return a; + }; + const first = up(); + await transported(); + first.close(); + await transported(); + const tab = createHost({ onError: () => {} }); + await tab.load([], context); + const [a, b] = memoryLinkPair(); + middle.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + tab.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); + const notes = tab.hold(Notes, { session: "S1" }, context); + await transported(); + expect(notes.state.value).toEqual({ kind: "loading" }); + up(); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + }); + }); + describe("a downstream release", () => { it("before the upstream hold was sent drops it; after, releases it upstream; messages crossing it are dropped (race 14)", async () => { let finish: () => void = () => {}; diff --git a/packages/chord/test/next/remote/remote-object.test.ts b/packages/chord/test/next/remote/remote-object.test.ts index e5e3d69b7..03baaa01c 100644 --- a/packages/chord/test/next/remote/remote-object.test.ts +++ b/packages/chord/test/next/remote/remote-object.test.ts @@ -335,7 +335,7 @@ forEachCodec(() => { }); describe("self.get through a relay whose upstream dropped (R18)", () => { - it("a loading notice while loading fails the dependent at once, transient; bound retries it", async () => { + it("a loading notice while loading changes nothing: the dependent goes on waiting and goes live when bound", async () => { const View = local("app.view", { version: 1, params: {}, members: { count: int() } }); const errors: HostError[] = []; const client = createHost({ onError: (error) => errors.push(error) }); @@ -347,19 +347,12 @@ forEachCodec(() => { ); const loading = client.load([{ name: "@test/view", facet }], context); const { hold } = await peer.next("hold"); - // The relay's upstream link dropped before the hold was bound: no wait for the readyTimeout. + // The relay's upstream link dropped before the hold was bound: a drop is not a dependency failure (audit). peer.send({ kind: "loading", hold }); + await transported(); + bindHold(peer, hold, [{ text: "a" }]); await loading; - expect(errors).toMatchObject([ - { - kind: "activation_failed", - object: "app.view", - failure: { kind: "transient", error: { message: expect.stringContaining("app.notes") } }, - }, - ]); - bindHold(peer, hold); - const retried = await bindNext(peer, [{ text: "a" }]); - expect(retried).toBe(1); + expect(errors).toEqual([]); const view = await client.get(View, {}, context); expect(view.count.value).toBe(1); }); @@ -631,6 +624,64 @@ forEachCodec(() => { expect(notes.state.value).toEqual({ kind: "live" }); }); + it("ref.retry() of a broken failure sends retry on the same hold: no release, no new hold (audit)", async () => { + const { client, peer } = await rawProvider(); + const notes = client.hold(Notes, { session: "S1" }, context); + const { hold } = await peer.next("hold"); + const error = { code: "forbidden", message: "not authorized" } as const; + peer.send({ kind: "failed", hold, cause: { kind: "broken", error: { kind: "chord", error } } }); + await transported(); + expect(notes.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + notes.retry(); + expect(await peer.next("retry")).toEqual({ kind: "retry", hold }); + bindHold(peer, hold); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + expect(peer.received.filter(({ kind }) => kind === "hold" || kind === "release")).toEqual([]); + }); + + it("ref.retry() of an object whose factory threw, held twice: failed, loading, live; a call made then succeeds (audit)", async () => { + const Flaky = remote("app.flaky", { + version: 1, + params: {}, + variants: { only: { state: {}, methods: { ping: method() } } }, + }); + let factories = 0; + const { client } = await linked((h) => + h.serve(Flaky, () => { + if (++factories === 1) throw new Error("bad factory"); + return { variant: "only", members: { ping: async () => null } }; + }), + ); + const one = client.hold(Flaky, {}, context); + const two = client.hold(Flaky, {}, context); + await transported(); + const states: string[] = []; + const watch = await one.state.watch(context); + states.push(watch.value.kind); + watch.start(async (call) => { + if (call.kind === "value") states.push(call.value.kind); + }); + one.retry(); + // A call made while it loads again waits for the binding; none sees the old failure. + let called: Promise | undefined; + const unwatch = await one.state.watch(context); + unwatch.start(async (call) => { + if (call.kind === "value" && call.value.kind === "loading" && called === undefined) { + called = one.ping(context).then( + () => "ok", + (error: Error) => `rejected: ${error.message}`, + ); + } + }); + await transported(); + await transported(); + expect(states).toEqual(["failed", "loading", "live"]); + expect(await called).toBe("ok"); + expect(two.state.value).toEqual({ kind: "live" }); + expect(factories).toBe(2); + }); + it("a drop while failed: loading, held again on the next link (race 19)", async () => { const { client, peer } = await rawProvider(); const notes = client.hold(Notes, { session: "S1" }, context); @@ -692,7 +743,7 @@ forEachCodec(() => { }); }); - describe("an activating dependent whose remote dependency's link drops (race 18)", () => { + describe("an activating dependent whose remote dependency's link drops (audit; supersedes race 18)", () => { fakeTimers(); const View = local("app.view", { version: 1, params: {}, members: { count: int() } }); @@ -709,27 +760,36 @@ forEachCodec(() => { expect((await client.get(View, {}, context, { readyTimeout: 1 })).count.value).toBe(1); } - it("fails transiently at once with the drop, keeps the reference, and is retried at once when it is bound", async () => { + it("stays loading through the drop, no failure reported, and goes live when the next link binds it", async () => { const errors: HostError[] = []; const client = createHost({ onError: (error) => errors.push(error) }); const peer = await rawParent(client); const loading = client.load([{ name: "@test/view", facet }], context); await peer.next("hold"); peer.close(); + await transported(); + const view = client.hold(View, {}, context); + expect(view.state.value).toEqual({ kind: "loading" }); + const next = await rawParent(client); + expect(await bindNext(next, [{ text: "a" }])).toBe(0); await loading; - expect(errors).toMatchObject([ - { - kind: "activation_failed", - failure: { - kind: "transient", - error: { - message: "app.view: dependency app.notes(S1) failed", - cause: { message: "the link closed" }, - }, - }, - }, - ]); - await retried(client, await rawParent(client)); + await transported(); + expect(view.state.value).toEqual({ kind: "live" }); + expect(errors).toEqual([]); + }); + + it("dropped and never bound again, fails transient at its readyTimeout (the bound of the wait)", async () => { + const errors: HostError[] = []; + const client = createHost({ readyTimeout: 1_000, onError: (error) => errors.push(error) }); + const peer = await rawParent(client); + const loading = client.load([{ name: "@test/view", facet }], context); + await peer.next("hold"); + peer.close(); + await transported(); + expect(errors).toEqual([]); + await vi.advanceTimersByTimeAsync(1_000); + await loading; + expect(failures(errors)).toEqual(["app.view: transient"]); }); it("without any link fails at its readyTimeout, keeps the reference, and is retried when the first link binds it", async () => { From b6d5c2ecdc9bf33446fbbcef3aca016f6c952753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Hou=C5=A1ka?= Date: Wed, 28 Oct 2026 20:26:25 +0100 Subject: [PATCH 117/271] fix(env): kill the daemon's commands when it is terminated Connection.close() ends stdin and sends SIGTERM at once. The daemon did not handle SIGTERM, so it died before its end-of-input kill_all, and running commands were reparented to init. The daemon now blocks SIGTERM, SIGINT and SIGHUP before starting any thread and waits for them on its own thread, which kills every process group it started and exits. On Windows the job object already kills commands with the daemon. --- packages/env/CHANGELOG.md | 1 + packages/env/daemon/src/main.rs | 11 ++++++++++- packages/env/daemon/src/sys/unix.rs | 23 +++++++++++++++++++++++ packages/env/daemon/src/sys/windows.rs | 3 +++ packages/env/docs/protocol.md | 3 ++- packages/env/test/remote.test.ts | 25 +++++++++++++++++++++++++ 6 files changed, 64 insertions(+), 2 deletions(-) diff --git a/packages/env/CHANGELOG.md b/packages/env/CHANGELOG.md index c484db660..96bd4e91e 100644 --- a/packages/env/CHANGELOG.md +++ b/packages/env/CHANGELOG.md @@ -9,6 +9,7 @@ ### Fixed - `exec` and `pty` fail with `spawn_error` instead of hanging when the daemon cannot start a helper thread (for example at a cgroup task limit); the started command is killed and reaped ([#11433](https://github.com/earendil-works/pi/issues/11433)). +- `Connection.close()` on Unix no longer leaves running commands behind: the daemon used to die from the client's `SIGTERM` before killing its process groups. It now kills them on `SIGTERM`, `SIGINT` or `SIGHUP` before exiting. ## [1.4.0] - 2026-10-21 diff --git a/packages/env/daemon/src/main.rs b/packages/env/daemon/src/main.rs index 4457fef4d..ea720eabb 100644 --- a/packages/env/daemon/src/main.rs +++ b/packages/env/daemon/src/main.rs @@ -470,6 +470,15 @@ fn serve_frames(server: &Arc, input: &mut impl Read) -> io::Result<()> { } fn serve(token: &str) -> io::Result<()> { + let groups: exec::Groups = Arc::new(Mutex::new(Default::default())); + { + // A client that stops the daemon (Node's `child.kill()`), or a hangup, leaves nothing running either. + let groups = groups.clone(); + sys::on_exit_signal(move |signal| { + kill_all(&groups); + std::process::exit(128 + signal); + }); + } { let mut stdout = io::stdout().lock(); writeln!(stdout, "PI-ENV {token}")?; @@ -488,7 +497,7 @@ fn serve(token: &str) -> io::Result<()> { handles: Mutex::new(HashMap::new()), next_handle: AtomicU64::new(1), controls: Mutex::new(HashMap::new()), - groups: Arc::new(Mutex::new(Default::default())), + groups, terminals: Arc::new(Mutex::new(HashMap::new())), }); let last_seen = Arc::new(AtomicU64::new(now_ms())); diff --git a/packages/env/daemon/src/sys/unix.rs b/packages/env/daemon/src/sys/unix.rs index 764cad1d1..4b0637f0e 100644 --- a/packages/env/daemon/src/sys/unix.rs +++ b/packages/env/daemon/src/sys/unix.rs @@ -283,6 +283,29 @@ pub fn configure( pub fn after_spawn(_child: &Child) {} +/// Run `on_signal` with the signal on its own thread when `SIGTERM`, `SIGINT` or `SIGHUP` arrives, instead of dying +/// at once. Call before starting any other thread: threads inherit the blocked signals, so only this one receives +/// them. Commands reset their signal mask before they run. +pub fn on_exit_signal(on_signal: impl FnOnce(i32) + Send + 'static) { + // SAFETY: a zeroed sigset_t initialized by sigemptyset, and plain signal mask calls. + let set = unsafe { + let mut set: libc::sigset_t = std::mem::zeroed(); + libc::sigemptyset(&mut set); + for signal in [libc::SIGTERM, libc::SIGINT, libc::SIGHUP] { + libc::sigaddset(&mut set, signal); + } + libc::pthread_sigmask(libc::SIG_BLOCK, &set, std::ptr::null_mut()); + set + }; + std::thread::spawn(move || { + let mut signal = 0; + // SAFETY: waits for one of the signals blocked above. + if unsafe { libc::sigwait(&set, &mut signal) } == 0 { + on_signal(signal); + } + }); +} + /// Kill the command's process group, or the process if the group is gone. pub fn kill_tree(pid: u32) { let pid = pid as i32; diff --git a/packages/env/daemon/src/sys/windows.rs b/packages/env/daemon/src/sys/windows.rs index f43a4bacf..de39fd4c3 100644 --- a/packages/env/daemon/src/sys/windows.rs +++ b/packages/env/daemon/src/sys/windows.rs @@ -550,6 +550,9 @@ pub fn after_spawn(child: &Child) { } } +/// Nothing to do: the daemon cannot catch `TerminateProcess`, and the job object kills its commands with it. +pub fn on_exit_signal(_on_signal: impl FnOnce(i32) + Send + 'static) {} + /// Node's `killProcessTree` on Windows: `taskkill /F /T`, not waited for. pub fn kill_tree(pid: u32) { let system_root = std::env::var("SystemRoot").unwrap_or_else(|_| "C:\\Windows".into()); diff --git a/packages/env/docs/protocol.md b/packages/env/docs/protocol.md index ae43fb767..f8ca5b4e1 100644 --- a/packages/env/docs/protocol.md +++ b/packages/env/docs/protocol.md @@ -62,7 +62,8 @@ command as a full pipe would. ## Liveness Each side sends `ping` every 5 seconds. The daemon kills every process group it started and exits after 30 seconds -without receiving any bytes, or when stdin ends. A dropped connection therefore stops remote commands. The client +without receiving any bytes, when stdin ends, or on `SIGTERM`, `SIGINT` or `SIGHUP` (on Windows, a job object kills +its commands with it). A dropped connection or a stopped daemon therefore stops remote commands. The client fails requests of a lost connection with `{ code: "unknown", lost: true }` and starts a new daemon on the next request; handles belong to the daemon that opened them, so requests on them fail instead of reaching the new one. diff --git a/packages/env/test/remote.test.ts b/packages/env/test/remote.test.ts index 4b1241148..4f7370679 100644 --- a/packages/env/test/remote.test.ts +++ b/packages/env/test/remote.test.ts @@ -117,6 +117,31 @@ describe("RemoteExecutionEnv", () => { expect(Date.now() - started).toBeLessThan(3000); }); + // The daemon used to die on Connection.close()'s SIGTERM before killing its commands. + it.skipIf(process.platform === "win32")("kills running commands when the connection closes", async () => { + const { env, connection } = environment(); + const pid = Promise.withResolvers(); + const running = env.exec( + ["sh", "-c", "echo $$; exec sleep 30"], + { onOutput: (text: string) => pid.resolve(Number.parseInt(text, 10)) }, + context, + ); + const command = await pid.promise; + connection.close(); + await running; + const alive = () => { + try { + process.kill(command, 0); + return true; + } catch { + return false; + } + }; + const deadline = Date.now() + 5000; + while (alive() && Date.now() < deadline) await new Promise((done) => setTimeout(done, 50)); + expect(alive()).toBe(false); + }); + it("fails requests on handles of a lost connection instead of reaching the new daemon", async () => { const { env, connection } = environment(); getOrThrow(await env.writeFile("a.txt", "AAA", context)); From a87b742ff4ce262c07b7830a25c73c1442dab41b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Hou=C5=A1ka?= Date: Wed, 28 Oct 2026 21:03:07 +0100 Subject: [PATCH 118/271] docs(env): Windows commands' descendants survive a terminated daemon The job object uses silent breakaway, as libuv does, so it only kills each command's own process. --- packages/env/daemon/src/sys/windows.rs | 3 ++- packages/env/docs/protocol.md | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/env/daemon/src/sys/windows.rs b/packages/env/daemon/src/sys/windows.rs index de39fd4c3..401719369 100644 --- a/packages/env/daemon/src/sys/windows.rs +++ b/packages/env/daemon/src/sys/windows.rs @@ -550,7 +550,8 @@ pub fn after_spawn(child: &Child) { } } -/// Nothing to do: the daemon cannot catch `TerminateProcess`, and the job object kills its commands with it. +/// Nothing to do: the daemon cannot catch `TerminateProcess`. When it ends, the job object kills each command's own +/// process; processes those start leave the job (silent breakaway, as in libuv) and keep running. pub fn on_exit_signal(_on_signal: impl FnOnce(i32) + Send + 'static) {} /// Node's `killProcessTree` on Windows: `taskkill /F /T`, not waited for. diff --git a/packages/env/docs/protocol.md b/packages/env/docs/protocol.md index f8ca5b4e1..c5e482856 100644 --- a/packages/env/docs/protocol.md +++ b/packages/env/docs/protocol.md @@ -62,10 +62,11 @@ command as a full pipe would. ## Liveness Each side sends `ping` every 5 seconds. The daemon kills every process group it started and exits after 30 seconds -without receiving any bytes, when stdin ends, or on `SIGTERM`, `SIGINT` or `SIGHUP` (on Windows, a job object kills -its commands with it). A dropped connection or a stopped daemon therefore stops remote commands. The client -fails requests of a lost connection with `{ code: "unknown", lost: true }` and starts a new daemon on the next request; -handles belong to the daemon that opened them, so requests on them fail instead of reaching the new one. +without receiving any bytes, when stdin ends, or on `SIGTERM`, `SIGINT` or `SIGHUP`. A dropped connection or a +stopped daemon therefore stops remote commands. On Windows, a terminated daemon cannot clean up: a job object kills +each command's own process, but not processes that one started. The client fails requests of a lost connection with +`{ code: "unknown", lost: true }` and starts a new daemon on the next request; handles belong to the daemon that opened +them, so requests on them fail instead of reaching the new one. ## Operations From 1ce44bedebc66acde3579425ba16bc318c9b9270 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 21:05:01 +0000 Subject: [PATCH 119/271] docs(chord): slice 8 S2 and S3 wording as refined --- packages/chord/docs/next/decisions.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index c437875fe..f425d5bbf 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1763,12 +1763,14 @@ validation and copies the v2 kernel added around Delta. back: `host.serve(R, { compose, accepts: { [old]: { contribution?: (old) => current, value?: (current) => old } } })`; a reader bundling an older token gets the composed value mapped to its version (computed once per composition and version, only while such a reader exists); without a `value` mapping, or for a reader newer than the provider, its - value is failed `contract_mismatch` (reported). + value is failed `contract_mismatch` (reported); such a reader fails `broken`, not `transient` (nothing heals it before + a reload, which retries broken objects; build review 2026-10-27). - **Slice 8 S3: `self.registry` is a dependency, as `self.get` (owner, 2026-10-27: "isn't self.registry meant to wait like self.get?"; my spec had it as a plain read, a mistake against design §6, §7 and §17):** activation waits until - the registry has a value (served and composed once; an empty table composes too, at the serve), bounded by - `readyTimeout`; without one in time the object fails `transient` (`unavailable/undeclared`) and is retried when the - registry gets its first value (R18); once live, a later failure of the registry changes nothing (last good value). + the registry has a value (served and composed once; an empty table composes too, at the serve); after `load`'s + setups a registry without a value cannot get one before a reload, so a reader then fails at once `transient` + (`unavailable/undeclared`) and is retried when the registry gets its first value (R18), as `self.get` of a type not + served here fails at once (refined 2026-10-27 after the second spec review: no `readyTimeout` wait at start); once live, a later failure of the registry changes nothing (last good value). - **Design audit of slices 1-6a (owner, 2026-10-27; `/tmp/design-audit-{opus,sol}.md`):** the built code follows the design except for five bugs, fixed after 6b's commit with the auditors' reproductions as tests: on a failed activation references are released after the natives closed (§6, §13); a dependency going live while the dependent From 790a98b9ab39653def35100463fc24408a3346dc Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 21:07:05 +0000 Subject: [PATCH 120/271] =?UTF-8?q?docs(chord):=20design=20=C2=A717=20comp?= =?UTF-8?q?ose=20failure=20and=20helper=20names=20as=20built?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/chord/docs/next/design.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index 7429a66e9..ba7b8378b 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -1518,9 +1518,10 @@ const router = self.registry(Http).value; // live (host.registry(Http) outsi `compose(table, { report })` is pure and runs whenever the table changes (after a commit, when a contributor goes live or ends, when a dynamic contribution changes). It always delivers a value with its fallback built in (e.g. the first by plugin name wins) and reports conflicts through `report` (owner notices and telemetry events); readers get - a plain value and never handle conflicts. Only if `compose` throws is the registry in an error state, and readers - get an error when they use it. Work that started earlier keeps the value it read (a pinned Durable phase keeps its - old table). Helpers: `keyed(table, entriesOf)`, `ordered(table, itemsOf, orderOf?)`. + a plain value and never handle conflicts. If `compose` throws, the failure is reported and its + watches get a `failed` call, while `.value` keeps the last good composition (slice 8); only before any good + composition do readers get the error. Work that started earlier keeps the value it read (a pinned Durable phase keeps + its old table). Helpers: `keyedEntries(table, entriesOf, report)`, `orderedEntries(table, itemsOf, orderOf?)`. - **Who contributes (R21):** only facets and host-lifetime objects (`local`, empty params); their factory's `self` has `contribute`, a referenced object's `self` does not (a type error). Reading a registry's composed value stays open to every object. - **A contribution is in the table exactly while its contributor is live:** a facet's while its plugin is loaded; an From edab3efa834709a5ecfa191db751003e21e0a596 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 21:07:42 +0000 Subject: [PATCH 121/271] docs(chord): slice 8: only Chord watchables are followed as contributions --- packages/chord/docs/next/slice-8.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/chord/docs/next/slice-8.md b/packages/chord/docs/next/slice-8.md index 2bd988d7a..64c682534 100644 --- a/packages/chord/docs/next/slice-8.md +++ b/packages/chord/docs/next/slice-8.md @@ -58,8 +58,9 @@ await host.root(async (ctx) => (await ctx.get(Notes, { session })).add(note, ctx when the table is built, an older one without an adapter or a newer one is left out and reported (`HostError` `contribution_refused { registry, plugin, version }`). - **Contributions:** a plain value, or a Chord watchable (a `writable`, a derive, a document member; another - registry's value too, a cycle of registries reading each other refused when the second one serves or contributes), followed as a dependent of the table (its change recomposes); a foreign watchable (a Durable watch) - is refused at `contribute` (declaration policy). The table's entry is `{ plugin, type?, version, value }` (`type` for + registry's value too, a cycle of registries reading each other refused when the second one serves or contributes), followed as a dependent of the table (its change recomposes), recognised by Chord's own mark; anything + else is a plain value, whatever methods it has (a harness, a conversation handle or a runtime is contributed as a plain + value, not followed; build review 2026-10-27, replacing "a foreign watchable is refused"). The table's entry is `{ plugin, type?, version, value }` (`type` for an object contributor; served types carry their plugin's name). - **`compose(table, { report })`:** pure, lazy as a derive: the registry's value is marked when its table changes (a contributor goes live or ends, a watchable contribution changes) and composed when read or, while watched, in the From 6eb99ad65ba38d4e058f69ee573c390c40abe277 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 21:33:59 +0000 Subject: [PATCH 122/271] feat(chord): Chord v2 slice 8, registries and host.root Registries: shared tokens (defineRegistry), one provider per host (host.serve(R, { compose, accepts })), contributions from facets and host-lifetime objects, plain values or followed Chord watchables, a table sorted by plugin, lazy compose in the batch that changed the table, failures left out and reported with the last good value kept; self.registry as a declared dependency, host.registry outside objects, older readers mapped by accepts[old].value; keyedEntries and orderedEntries. Native sources: self.writable, registry values and ref.state. self.retryOn wakes failed objects on registry values. host.root gives code outside objects a call scope sealed at settle, abort and host.close. Durable: registryReader(value, context); a Session close ends synchronous observers failed ("session closed"), watches still end ended. --- packages/chord/CHANGELOG.md | 1 + packages/chord/docs/next/benchmarks.md | 50 ++ packages/chord/docs/next/design.md | 23 +- packages/chord/docs/next/slice-8.md | 3 +- packages/chord/docs/next/slices.md | 24 +- packages/chord/src/next/data/errors.ts | 3 +- packages/chord/src/next/host/README.md | 59 +- packages/chord/src/next/host/attempt.ts | 217 +++++- packages/chord/src/next/host/ctx.ts | 8 +- packages/chord/src/next/host/facet.ts | 89 ++- packages/chord/src/next/host/host.ts | 121 +++- packages/chord/src/next/host/index.ts | 5 +- packages/chord/src/next/host/members.ts | 43 +- packages/chord/src/next/host/object.ts | 70 +- packages/chord/src/next/host/ref-state.ts | 69 +- packages/chord/src/next/host/reference.ts | 21 +- packages/chord/src/next/host/retry-on.ts | 55 ++ packages/chord/src/next/host/root.ts | 76 ++ packages/chord/src/next/host/routines.ts | 13 +- packages/chord/src/next/index.ts | 1 + packages/chord/src/next/registries/README.md | 81 +++ .../chord/src/next/registries/contribution.ts | 158 +++++ packages/chord/src/next/registries/entries.ts | 54 ++ packages/chord/src/next/registries/index.ts | 16 + .../chord/src/next/registries/internal.ts | 5 + packages/chord/src/next/registries/record.ts | 301 ++++++++ .../chord/src/next/registries/registry.ts | 89 +++ packages/chord/src/next/registries/value.ts | 177 +++++ packages/chord/src/next/state/README.md | 31 +- packages/chord/src/next/state/batch.ts | 43 +- packages/chord/src/next/state/derive.ts | 24 +- packages/chord/src/next/state/index.ts | 4 +- packages/chord/src/next/state/internal.ts | 15 +- .../chord/src/next/state/native-source.ts | 291 ++++++++ packages/chord/src/next/state/writable.ts | 78 +++ packages/chord/test/next/bench/calls.ts | 90 +++ packages/chord/test/next/bench/registries.ts | 108 +++ packages/chord/test/next/data/errors.test.ts | 6 +- .../chord/test/next/examples/29-registries.ts | 115 ++++ .../chord/test/next/examples/30-host-root.ts | 85 +++ packages/chord/test/next/host/attempt.test.ts | 33 +- packages/chord/test/next/host/ctx.test.ts | 6 +- packages/chord/test/next/host/facet.test.ts | 28 +- packages/chord/test/next/host/members.test.ts | 4 +- .../chord/test/next/host/retry-on.test.ts | 204 ++++++ packages/chord/test/next/host/root.test.ts | 175 +++++ .../test/next/registries/entries.test.ts | 49 ++ .../chord/test/next/registries/record.test.ts | 649 ++++++++++++++++++ .../test/next/registries/registry.test.ts | 67 ++ .../chord/test/next/registries/value.test.ts | 398 +++++++++++ .../test/next/state/native-source.test.ts | 152 ++++ .../chord/test/next/state/writable.test.ts | 210 ++++++ packages/chord/test/next/support.ts | 18 +- packages/durable/CHANGELOG.md | 4 + packages/durable/docs/spec.md | 8 +- .../durable/src/harness/registry-reader.ts | 34 + packages/durable/src/harness/task-graph.ts | 17 +- packages/durable/src/harness/view.ts | 8 +- packages/durable/src/index.ts | 1 + packages/durable/src/session/session.ts | 28 +- packages/durable/src/types.ts | 7 +- .../test/examples/37-harness-registry.ts | 172 +++++ .../durable/test/harness-observers.test.ts | 5 +- .../test/harness-registry-reader.test.ts | 198 ++++++ .../durable/test/session-observers.test.ts | 30 +- scripts/check-entry-graphs.mjs | 5 +- 66 files changed, 5036 insertions(+), 196 deletions(-) create mode 100644 packages/chord/src/next/host/retry-on.ts create mode 100644 packages/chord/src/next/host/root.ts create mode 100644 packages/chord/src/next/registries/README.md create mode 100644 packages/chord/src/next/registries/contribution.ts create mode 100644 packages/chord/src/next/registries/entries.ts create mode 100644 packages/chord/src/next/registries/index.ts create mode 100644 packages/chord/src/next/registries/internal.ts create mode 100644 packages/chord/src/next/registries/record.ts create mode 100644 packages/chord/src/next/registries/registry.ts create mode 100644 packages/chord/src/next/registries/value.ts create mode 100644 packages/chord/src/next/state/native-source.ts create mode 100644 packages/chord/src/next/state/writable.ts create mode 100644 packages/chord/test/next/bench/calls.ts create mode 100644 packages/chord/test/next/bench/registries.ts create mode 100644 packages/chord/test/next/examples/29-registries.ts create mode 100644 packages/chord/test/next/examples/30-host-root.ts create mode 100644 packages/chord/test/next/host/retry-on.test.ts create mode 100644 packages/chord/test/next/host/root.test.ts create mode 100644 packages/chord/test/next/registries/entries.test.ts create mode 100644 packages/chord/test/next/registries/record.test.ts create mode 100644 packages/chord/test/next/registries/registry.test.ts create mode 100644 packages/chord/test/next/registries/value.test.ts create mode 100644 packages/chord/test/next/state/native-source.test.ts create mode 100644 packages/chord/test/next/state/writable.test.ts create mode 100644 packages/durable/src/harness/registry-reader.ts create mode 100644 packages/durable/test/examples/37-harness-registry.ts create mode 100644 packages/durable/test/harness-registry-reader.test.ts diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index f475c4c8a..d0c9d0eff 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -14,6 +14,7 @@ ### Added - Chord v2 has a package export again, `@earendil-works/chord/next` (experimental): every layer's API for apps (contracts, data, documents, hosts, jobs, links, state, the store), and, marked internal, the store's surface for adapters (`StoreOwnerTables`, `OwnerTable`, the documents engine's reads and writes, the commit step, `applyBatch`). v2's store gains one id allocator for every id in it, claims per table and per document namespace, Durable's Storage semantics for a refused commit (it rejects that commit only; an unknown outcome stops the store), document histories `none`, `latest` and `rewindable` with reads at a point, archives carrying document revisions, and batch apply; v2 objects gain `self.follow` (another runtime's synchronous observers as sources) and persisted jobs (`persist: { resume }`, a stored `abort(id, ctx)`, deferred state changes). +- Chord v2 (`@earendil-works/chord/next`) gains registries, writables, wake signals and root scopes (slice 8): `defineRegistry(name, { version })` tokens shared by plugins, served by the defining plugin's facet with `host.serve(R, { compose, accepts? })`; contributions from facets (`defineFacet((host, self) => self.contribute(R, value))`) and host-lifetime objects (`StoreSelf.contribute`), plain values or Chord watchables, in a table sorted by plugin name and composed lazily (once at the end of `load`'s setups, then when read or followed, in the batch that changed the table); readers through `self.registry(R)` (a declared dependency: an object activates once the registry has a value, and fails `transient` with the new reason `unavailable/undeclared` when nobody serves it) and `host.registry(R, ctx)`, a `RegistryValue` with `observe(observer, ctx)`; older contributions and readers adapted by `accepts`; the helpers `keyedEntries` and `orderedEntries`; the `onError` kinds `registry_conflict`, `registry_failed`, `contribution_failed` and `contribution_refused`. `self.writable(initial)` is an in-process value an attempt sets (identity compared); native sources (writables, registry values, `ref.state`) are derive sources. `self.retryOn(...sources)` retries a failed object when a registry value changes. `host.root(fn, context, { principal? })` runs code outside objects with a handler's `ctx`, sealed when `fn` settles, `context` aborts or the host closes. - Chord v2, experimental and not exported yet: the next object runtime in `src/next` (contracts, hosts and objects, documents, derives, links and relays, routines and jobs, the store), with its tests and runnable examples in `test/next` and its design in `docs/next` (`design.md`, the decision log `decisions.md`, the slice plan `slices.md`). Its API changes with every slice until it gets an entry point. - `NativeWatchable`: an in-process value that changes and can be watched (`value`, `watch(context)` returning a `NativeWatchHandle`), the in-process counterpart of `Watchable`. - `WatchHandle` and `WatchEnd`, the one watch API, moved from Durable: a serialized observation with `value`, `start(listener)`, `stop()`, and `closed`; the listener gets a `WatchChange` (`{ value, ops, retired }`) and the context. `WatchEnd` reasons are `stopped`, `cancelled`, `ended`, `listener_error`, and `failed`. diff --git a/packages/chord/docs/next/benchmarks.md b/packages/chord/docs/next/benchmarks.md index 8e63d8d36..916c70838 100644 --- a/packages/chord/docs/next/benchmarks.md +++ b/packages/chord/docs/next/benchmarks.md @@ -583,3 +583,53 @@ each; the head lookup and scans under it are at parity or faster in the micro-be memory, 55.5 µs persisted (two commits: the record and state created, then the reservation); a state change inside `run` 2.7 µs in memory, 21.3 µs persisted (one delta commit, as the gate's (c)), 3.2 µs deferred (stored with the outcome). + +## Slice 8: registries and `host.root` (2026-10-28) + +Node 24.4.1, one CPU (`taskset -c 6`), runs alternating. HEAD is `391e83fb1`, extracted at a path of the same length +(`/home/badlogic/pi-h8`); the build was measured in the worktree and, where the worktree itself skewed a result (below), +in an extract of the same sources at `/home/badlogic/pi-x8`. + +**Paths under every object** (`test/next/bench/calls.ts`, new: runs on HEAD as it is), median per operation over nine +bursts, heap allocated per operation, three alternating runs each, HEAD → build: + +| Case | HEAD | build | +| --- | --- | --- | +| local method call (its ctx) | 2.80-2.81 µs, 6,894 B | 2.81-2.83 µs, 6,894 B | +| commit through a watched derive (`batch.compute`) | 10.34-10.43 µs, 15,312 B | 10.46-10.51 µs, 15,310 B | +| activation churn: get, live, release (an object with params) | 57.5-59.3 µs, 43,696-43,717 B | 58.1-58.9 µs, 43,782-43,787 B | +| retained per live object (10,000 kept) | 6,586 B | 6,602 B | + +The attempt keeps its registry reads, contributions, wake signals and writables in one record made by the first such +declaration, and iterates nothing when there is none: the first build allocated 1,020 B more per activation (six +arrays per attempt, an array iterator per `for … of` over an empty list, a copy from `splice`) and retained 72 B more +per live object; now +70 B allocated and +16 B retained (one field on the attempt, one on the object). + +**Existing benchmarks**, HEAD → build: the derive diamond (`derives.ts`, three runs) 9.4-9.8 → 9.5-9.9 µs median; its +pooled p95 16.5-17.1 → 18.4-20.8 µs comes from the run in which V8 tiers the commit path up (per run, after it: HEAD +10.7-11.3 µs, build 10.1-11.3 µs; the first measured runs 15.0-25.6 µs on either side, in another run each), the same +with HEAD's `batch.ts` and `derive.ts` in the build. The store gate (`store.ts`, three runs): every median within +0.4 µs (Chord store 13.0-13.9 → 13.0-14.2 µs, Chord host 17.3-18.2 → 17.3-17.8 µs, in-memory document 2.1 → 2.2 µs), +heap 3,498 → 3,144 KiB. Jobs (`jobs.ts`): start 8.4 → 8.5 µs in memory, 55.4 → 56.2 µs persisted; state change 2.7 → +2.7, 21.3 → 21.8, 3.4 → 3.5 µs. Links (`links.ts`, three runs): wire sizes, encodes, decodes and heap per message +unchanged; its round trips are one cold burst each and vary as much between HEAD runs (commit to frame, text codec, +0.055-0.076 ms). A warmed remote call (2,000 calls per burst, nine bursts, a memory link): binary 7.16-7.24 → 7.17-7.29 +µs, text 7.66-7.72 → 7.70-7.83 µs in the extracts; the worktree itself runs the same sources at 7.8-7.9 µs (binary), +an effect of the tree, not the code. + +**Registries and writables** (`test/next/bench/registries.ts`, new), median over nine bursts: `writable.set` delivered +to one watch 0.81-0.91 µs (2.9 KB allocated: the batch and the watch's call); a set of the same value 0.008 µs, 0 B +(one comparison: its batch work is a method of its own, so `set` allocates no closure context); one of 10 contributions +changed while the registry is watched 0.74-0.77 µs, of 100 1.52-1.59 µs (the table built again over every entry, one +compose per set: the counted `chord.registry.composes` is 1.00 per set). + +**Fix round 1** (2026-10-28), HEAD extract → build extract (`/home/badlogic/pi-x8`), CPU 6. `calls.ts`, two runs: local +method call 2.80 → 2.80-2.81 µs, 6,894 B both; commit through a watched derive 10.51-10.65 → 10.72 µs; activation churn +57.9-59.0 → 59.3-60.0 µs, 43,714 → 43,769-43,787 B. The derive diamond (five runs): median 9.4-9.8 → 9.5 µs, pooled p95 +16.4-16.7 → 19.0-19.2 µs. Per run, both have one slow run while V8 tiers the commit path up (HEAD its second, median/p95 +15.5/22.6-22.9 µs; the build its first, 16.4/26.7-27.2 µs) and equal runs after it (HEAD 8.8-9.6/9.6-11.2 µs, build +8.9-9.8/9.7-10.6 µs). `batch.compute` is not the cause: HEAD's `batch.ts` and `derive.ts` in the build give the same +p95, and swapping host files moves it without a pattern (code layout, not work on the path). `registries.ts`: +`writable.set` 0.87 µs, the same value 0.011-0.016 µs and 0 B (the seal check now comes first: a set of an ended +attempt's writable throws whatever its value); one of 10 contributions set 0.71-0.78 µs, of 100 1.43-1.61 µs (one run +in six 3.4 µs on either variant tried: JIT). diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index ba7b8378b..b3ec994d1 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -207,7 +207,8 @@ parent link closes. `resolve` returns the link before the worker has loaded (the | `hold(Type, params, context, { readyTimeout? })` | a reference at once that follows its object until released (section 7) | | `close(context)` | closes the links (children, routes, then the parent), then ends every object at once and tears them down, the last to go live first (reason `close`); rejects with an `AggregateError` of the native closes that threw | | `revoke(r, context)` | revocation (section 4; slice 4b) | -| `root(fn, …)` | a call scope for code outside objects (a Durable hook, a CLI); built in slice 8 (**decision needed** on its signature, section 26) | +| `root(fn, context, { principal? })` | a call scope for code outside objects (a Durable hook, a CLI): `fn` gets a handler's `ctx`, sealed when `fn` settles, `context` aborts or the host closes (slice 8 D3, section 8) | +| `registry(R, context)` | registry `R`'s composed value for code outside objects; waits for `load`, rejects when it has none (slice 8, section 17) | **Options of `createHost`:** `address`, `principal`, `principalKey` (section 4; the same function on every host), `telemetry` (section 29), `onError`, `closeTimeout`, `readyTimeout`, `heartbeat: { ping?, close? }`; later `revokedBy` @@ -216,7 +217,7 @@ and closed by the app with `store.close(ctx)` after `host.close`; the host never background, as a tagged `HostError`: `listener_error` (a watch listener threw), `delivery_failed` (a frame delivery threw), `activation_failed { object, failure }`, `close_failed { object, native, error }`, `close_timeout { object, native }`, `drain_timeout { object }`, `protocol_error { error }`, `resolve_failed { address, error }`; later routine -errors (slice 5), `derive_failed { object, member?, error }` (a derive's compute threw, once per failure episode, slice 6a), `object_failed { object, failure }` (a live object failing, slice 6a, `broken`; `transient` when a `self.follow` source failed, slice 6b R4; `failure` where the value is a `FailureCause`, `error` otherwise, 2026-10-27), `job_interrupted { object, job, id, restarts }` (a persisted run ended at the restart limit, slice 6b D6) and `storage_failed { error }` +errors (slice 5), `derive_failed { object, member?, error }` (a derive's compute threw, once per failure episode, slice 6a), the registries' `registry_conflict { registry, message, plugins }`, `registry_failed { registry, error }`, `contribution_failed { registry, plugin, type?, error }` and `contribution_refused { registry, plugin, type?, version }` (each once per episode, slice 8), `object_failed { object, failure }` (a live object failing, slice 6a, `broken`; `transient` when a `self.follow` source failed, slice 6b R4; `failure` where the value is a `FailureCause`, `error` otherwise, 2026-10-27), `job_interrupted { object, job, id, restarts }` (a persisted run ended at the restart limit, slice 6b D6) and `storage_failed { error }` (slice 6a), which every app must handle: Chord never ends the process, the app exits with its own code for its supervisor or reopens. @@ -546,7 +547,7 @@ host.serve(Notes, ({ session }, principal, self) => link protocol (adding one bumps the protocol version; a receiver rejects unknown reasons; a test pins the list): `invalid` (`args`, `params`, `member`, `placement`, `reentrant`, `read_after_bulk`, `cycle`), `contract_mismatch` (`updating`), `forbidden` (`revoked`), `gone`, `not_ready`, `released` (`ended`, `closed`, `settled`, `version`, `discarded`; none for a holder's own release), `too_large` (`part`), `unavailable` (`locked`, - `park_timeout`, `busy`, `storage_failed`), `aborted`. A failed check names TypeBox's instance path ("unknown field", "missing", else TypeBox's + `park_timeout`, `busy`, `storage_failed`, `undeclared`: a registry no plugin serves here, slice 8), `aborted`. A failed check names TypeBox's instance path ("unknown field", "missing", else TypeBox's message; keys containing `/` or `~` make the path ambiguous, documented). Plugins throw `PluginError(code, message, details?)` for codes of their own; anything else crosses a link as the plugin code `internal` with its message, never a stack or a cause; plugin errors never use Chord's codes. An error rebuilt from the wire is not copied and only its @@ -1505,7 +1506,7 @@ host.serve(Http, { // the defining p accepts: { 1: { contribution: (old: HttpContributionV1) => upgrade(old), value: (router) => routerV1(router) } }, }); self.contribute(Http, { routes: { … }, wraps: [ … ] }); -const router = self.registry(Http).value; // live (host.registry(Http) outside objects) +const router = self.registry(Http); // a declared dependency; read router.value at use (await host.registry(Http, ctx) outside objects) ``` - Each contributor (a facet or an object) contributes **at most one value per registry**; the table is an immutable, @@ -1521,7 +1522,9 @@ const router = self.registry(Http).value; // live (host.registry(Http) outsi a plain value and never handle conflicts. If `compose` throws, the failure is reported and its watches get a `failed` call, while `.value` keeps the last good composition (slice 8); only before any good composition do readers get the error. Work that started earlier keeps the value it read (a pinned Durable phase keeps - its old table). Helpers: `keyedEntries(table, entriesOf, report)`, `orderedEntries(table, itemsOf, orderOf?)`. + its old table). Helpers: `keyedEntries(table, entriesOf, report)` (a map, the first by plugin name wins, a duplicate + reported), `orderedEntries(table, itemsOf, orderOf?)`; both keep each item's plugin (named so beside the contracts' + `keyed`, slice 8 build). - **Who contributes (R21):** only facets and host-lifetime objects (`local`, empty params); their factory's `self` has `contribute`, a referenced object's `self` does not (a type error). Reading a registry's composed value stays open to every object. - **A contribution is in the table exactly while its contributor is live:** a facet's while its plugin is loaded; an @@ -1962,14 +1965,20 @@ loop. Chord has no stream type (it would need its own rules for drops, cancellat loading object (refines R17). A missing tool gives its call an error result the model sees; a missing model makes the generation report "model unavailable" and wait (not fail) until a model contribution appears; a task whose definition is missing stays blocked (`missing_task`) until the plugin is installed or the task is aborted, as Durable does. +- **Durable's views from other objects** (slice 8 §5): the harness object contributes its `Native` to a + registry of p3's; another object declares that registry (`self.registry`), wakes on it (`self.retryOn`) and follows + a view through its value (`self.follow`). A Session close ends Durable's synchronous observers `failed` ("session + closed"; decision "Slice 8 S1"; app watches still end `ended`), so the follower fails `transient` and follows the + next Harness once its contribution arrives. `registryReader`'s listener runs inside Chord's batch: it only wakes + Durable's scheduler (a Chord call there throws `invalid/reentrant`). - `HarnessModels` and `HarnessEnvironments` start with one contribution each, from the models and envs plugins' objects that own the natives; several providers or environment kinds are their `compose`. After a models reload the next request uses the new runtime; a running stream finishes on the old one. - **Chord from Durable code (Q6):** tools, hooks and sections get Durable's `ToolExecutionApi` and a `Context`, never a Chord `ctx`. p3's core opens the harness with a base context holding the host under a typed key (`withContextValue(ChordHostKey, host, …)`); every phase context inherits it. A core helper opens a call scope from - any Durable context: `chord(async (ctx) => …, context)` = `host.root(fn, { signal: context.abortSignal, telemetry: - context.telemetry })` (host principal; attribution as data; references released when `fn` settles). No wrapping of + any Durable context: `chord(async (ctx) => …, context)` = `host.root(fn, context)` (host principal; attribution as + data; references released when `fn` settles; slice 8 D3). No wrapping of tools and no p3 `defineTool`. Clients are never reachable this way (they are not addressable): a tool that needs the user goes through an owner's state, e.g. the Questions service (`apps/p2/docs/questions.md`): the model-facing `ask_user` tool and any other tool call `Questions.ask`/`wait`, tabs render the open questions and `answer` them. diff --git a/packages/chord/docs/next/slice-8.md b/packages/chord/docs/next/slice-8.md index 64c682534..65f1c494f 100644 --- a/packages/chord/docs/next/slice-8.md +++ b/packages/chord/docs/next/slice-8.md @@ -1,6 +1,7 @@ # Slice 8: registries and `host.root` (spec) -Status: **ready to build** (2026-10-27; D1-D3, the registry failures and S1-S3 decided by the owner; two spec reviews, +Status: **built** (2026-10-28; build review and two fix rounds by the same four reviewers, +`/tmp/s8-build-review-*.md`, `/tmp/s8-build-rereview-*.md`). Before: ready to build (2026-10-27; D1-D3, the registry failures and S1-S3 decided by the owner; two spec reviews, `/tmp/s8-review{,2}-{astra,fable,opus,sol}.md`, found no remaining blocker; S3 refined after the second: with no value after `load`, a reader fails at once, as `self.get` of a type not served here) diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index 87d395336..cf346eb9e 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -27,15 +27,16 @@ slice spec's section citations are for orientation, not a reading limit (owner, - **L1** (`src/next/contracts`, `src/next/data`): contracts, params and identity, argument checks, principal keys, errors. `index.ts` for apps, `internal.ts` for Chord's own code. -- **Slices 1, 2, 3a, 3b, 4a, 4b, 5, 6a, 6b** (committed; 5 `4a5b45c45`, 6a and 6b 2026-10-27/28): `src/next/documents`, `state`, `host`, `jobs`, `link`, - `remote`, `store`; examples 01-25; benchmarks in `benchmarks.md`. +- **Slices 1, 2, 3a, 3b, 4a, 4b, 5, 6a, 6b, 8** (committed; 5 `4a5b45c45`, 6a and 6b 2026-10-27/28, 8 2026-10-28): `src/next/documents`, `state`, `host`, `jobs`, `link`, + `remote`, `store`, `registries`; examples 01-30 (Durable's 35-37); benchmarks in `benchmarks.md`. - **Chord's shared code** (upstream pi): `src/watch.ts` (`Watch`, `NativeWatch`, `WatchHandle`, `WatchEnd`, `NativeWatchable`), Delta (`track`, `diffRevisions`, `applyImmutable`, encoders, `PendingOps`, `appendText`), context, JSON, output, SQLite drivers. ## The slices -Order (owner, 2026-10-22: routines and jobs before the store and reload). +Order (owner, 2026-10-22: routines and jobs before the store and reload; 2026-10-27: slice 8 before slice 7, whose +reload has to reload real registries). | # | Slice | In short | | --- | --- | --- | @@ -48,8 +49,8 @@ Order (owner, 2026-10-22: routines and jobs before the store and reload). | 5 | **Routines and jobs** | `self.routines`, `self.job` | | 6a | **The store** | the store (backends, tables, queries, writes, commit queue, outbox, documents engine, migrations, export), persisted documents, `self.table`, keyed watches and retirement | | 6b | **Durable and jobs on the store** | persisted jobs, Durable's `StoreStorage`, the Chord source over Durable's views (`self.follow`) | +| 8 | **Registries and `host.root`** | registries, `self.writable`, `retryOn`, Durable's `registryReader`, `host.root` (`chord(fn, context)` and the harness object are p3's base code) | | 7 | **Reload and rollout** | plugin loading, staging, swaps and restarts, revisions, `accepts` in use | -| 8 | **Registries and `host.root`** | registries, `self.writable`, `retryOn`, a `RegistryReader` for Durable, `host.root` (`chord(fn, context)` and the harness object are p3's base code) | | 9 | **Telemetry** | spans, events, metrics across hosts | The playground (`apps/chord-playground`) moves to the new code once slice 4 exists. Round-3 cuts (C3-C16) and @@ -211,14 +212,13 @@ deleted (the slice's spec and code are the record). Moved here from `carry-over. internal ones are rebuilt; add matching only if a real need appears. (W1.) - **Retrying broken failures:** reload retries a broken object as `ref.retry()` does (`ChordObject.retry`: the backoff skipped, an attempt now; holders are told `loading`). (Slices 2, 3b.) - -### Slice 8: registries and `host.root` - -- **Durable's views from other objects** (from slice 6b): the harness carried in a registry entry, so an object other - than the one that opened Durable can `self.follow` its views. - -- Specified in `slice-8.md` (decided 2026-10-27): `host.root` (D3), `retryOn` (D2), registries recomposing in the - owner's flush (W4); `host.activity` on the facet's host dropped (nothing needs it). +- **Registries on reload** (from slice 8 §6): a reload replaces a plugin's contributions and a provider atomically, in + one batch (staging composes nothing, decision D1 of "Review round 1"); the record keeps the provider, facet + contributions by plugin and object contributions by object apart for it (`RegistryRecord.serve` refuses a second + serve today). Older registry readers across a reload of the provider (their mapped values follow the new + `accepts[old].value`). A `writable` across a pure swap: the new factory makes a new `self.writable(initial)`, so a + contribution goes back to its initial value until the restarted routine sets it (MCP's tools would vanish briefly). + `retryOn` watches across a reload (a reload retries failed objects anyway). (Slice 8 spec.) ### Slice 9: telemetry diff --git a/packages/chord/src/next/data/errors.ts b/packages/chord/src/next/data/errors.ts index 697f3b578..623c05fe2 100644 --- a/packages/chord/src/next/data/errors.ts +++ b/packages/chord/src/next/data/errors.ts @@ -18,7 +18,8 @@ export const CHORD_ERROR_REASONS = { // `discarded`: a transaction cut at the seal. released: ["ended", "closed", "settled", "version", "discarded"], too_large: ["part"], - unavailable: ["locked", "park_timeout", "busy", "storage_failed"], + // `undeclared`: a registry no plugin serves on this host. + unavailable: ["locked", "park_timeout", "busy", "storage_failed", "undeclared"], aborted: [], } as const; diff --git a/packages/chord/src/next/host/README.md b/packages/chord/src/next/host/README.md index 2474ca16a..da13258bc 100644 --- a/packages/chord/src/next/host/README.md +++ b/packages/chord/src/next/host/README.md @@ -4,8 +4,10 @@ Hosts, facets, objects, natives and references: hosts serving `local` and `remot principals and variants, links to children, to a parent and over routes to addressable hosts (with relays between them), the heartbeat, the `ctx` of method handlers, (slice 4b) revocation, cycles across hosts, presence, keep-alive counts, size limits per part and compression at the origin, (slice 5) routines and the objects' jobs -(declared here, built in `jobs/`), and (slice 6a) the host's store: tables, persisted documents and eager table -migrations at activation (built in `store/` and `documents/`), and live objects that fail. +(declared here, built in `jobs/`), (slice 6a) the host's store: tables, persisted documents and eager table +migrations at activation (built in `store/` and `documents/`), and live objects that fail, and (slice 8) registries +(served, contributed and read here, built in `registries/`), writables, wake signals (`retryOn`) and root scopes +(`host.root`). ## For apps (`index.ts`) @@ -189,8 +191,41 @@ server.accept(webSocketLink(ws), { principal, limits: { part: 16 * MiB, decoded: route's `resolve` rejected; the address waits its backoff), `routine_failed` (`object`, `routine`: its position, `error`: a routine call threw, or its watch could not be taken; a source's failure calls never reach a routine), `job_failed` (`object`, `job`, `id`, `error`: a run threw, not after an abort), `job_finish_failed` (the same: a `finish` threw; it is retried) and - `job_dropped` (`object`, `job`, `id`: a blocked run dropped without `finish`, by `abort` or its object's end). A - throwing `onError` is ignored. + `job_dropped` (`object`, `job`, `id`: a blocked run dropped without `finish`, by `abort` or its object's end), and + the registries' (slice 8, each once per episode) `registry_conflict` (`registry`, `message`, `plugins`: a conflict + `compose` resolved), `registry_failed` (`registry`, `error`: `compose` or an older version's `value` adapter threw), + `contribution_failed` (`registry`, `plugin`, `type?`, `error`: a watchable contribution failed or ended, or its + adapter threw) and `contribution_refused` (`registry`, `plugin`, `type?`, `version`). A throwing `onError` is + ignored. +- **Registries** (slice 8, `registries/`): a facet serves a registry it defines with `host.serve(R, { compose, + accepts? })` (one provider per host; a second serve throws at `load`); a facet's `self.contribute(R, value)` (its + setup's second argument, `FacetSelf`) is in the table while the plugin is loaded, a host-lifetime object's (`local`, + empty params: `StoreSelf.contribute`; any other factory throws, R21) from `live` until it fails, ends or restarts, + entering before anyone sees it live and leaving at its seal. One value per registry each; a Chord watchable (a + source, a constant, a document handle, a reference member, a writable, a registry value) is followed through the + batch, anything else is a plain value whatever methods it has (registries carry natives); a registry value closing + a cycle of registries (through the tables and the factory's own earlier contributions) is refused `invalid/cycle` + at `contribute` (an edge lives as long as its contribution is in a table). Every served registry composes once at the end of `load`'s setups, before any object activates. + `self.registry(R)` is a declared dependency: with no value at activation (no provider, a first `compose` that threw, + a version not served here) the attempt fails `dependency_failed` (`transient`, the registry's error as the cause, + the references kept) at once, and the registry's first value retries it (R18); a version the provider does not + serve fails it `broken` (`contract_mismatch`: only a reload helps). `host.registry(R, ctx)` waits for `load` and + rejects without a value. +- **Wake signals** (`self.retryOn(...sources)`, D2): registry values (constants are accepted and never wake; anything + else throws: a reference's member, the attempt's writable, `ref.state`). The object follows them from the factory's + return (before hydration, references and natives, so a failed open still has them) until the next attempt's + declaration replaces them or it ends. A new value retries a failed object (`transient` or `broken`) as `ref.retry()` + does, after the batch; one while it activates is remembered and retries it at once if that attempt fails; one while + live is ignored; failure calls wake nothing; a composition made by the object's own contributions leaving (it + failed live) does not wake it. There is no backoff between wakes: wake an object on sources that change rarely + (models, a harness), not on every event (a writable set per message retries a failing object per message). +- `host.root(fn, ctx, { principal? })` (slice 8 D3): `fn(ctx)` with a handler's `Ctx

`: the host's principal or the + given one, this host's incarnation, `ctx`'s telemetry, a signal linked to `ctx`'s, `ctx.get`. When `fn` settles, + `ctx` aborts or the host closes, the scope is sealed: its references released (their calls reject `released`), a + waiting `ctx.get` gives up, a later one is refused `released/settled`, its signal aborted. An abort rejects + `host.root` at once (an uncooperative `fn` runs on and fails at its next use of Chord); an aborted `ctx` runs no + `fn`; `host.close` rejects it at once `released/closed`, as a method's caller when its object ends; after `close` + it rejects `released/closed`. Not counted in `host.activity`. - `host.load([{ name, facet }], ctx)`: once per host; runs every setup, then activates every type with empty params and resolves when each has had its first outcome, live or failed (the start barrier; later retries never hold it). A setup that throws rejects it with that error and closes the host as `close` does, its links included. Gets wait @@ -281,7 +316,8 @@ server.accept(webSocketLink(ws), { principal, limits: { part: 16 * MiB, decoded: `broken`; a live object whose read fails so fails while live (`object_failed`), keeping its documents for the next attempt. - A factory `(params, self) => implementation` declares with `self.state`, `self.derive`, `self.get`, `self.open`, - `self.on`, `self.onLive` and `self.job` (`jobs/`), and writes with `self.tx`. It runs once per attempt. The implementation has exactly the contract's members: state + `self.on`, `self.onLive`, `self.job` (`jobs/`), `self.registry`, `self.writable` and `self.retryOn` (below), and + writes with `self.tx`. It runs once per attempt. The implementation has exactly the contract's members: state members are JSON watchables (document handles, derives, reference members, constants), methods `(args, ctx) => Promise`, or `(ctx) => Promise` for a method without arguments (the reference omits them too: `notes.clear(ctx)`). A remote factory `(params, principal, self) => ({ variant, members })` gets the principal its @@ -456,8 +492,9 @@ caller), `Placement` (where a reference or an arriving hold goes: here, a route' served here is placed here), `Route` (one per routed kind: per address its `Path`, at most one running `resolve`, the backoff per address; an address is kept while it has holders, a link or a running `resolve`), `ChordObject` (one per identity: its state across attempts, holders, retries, cycle checks with its instance id and -the chain it was created for, lingering, ending and revocation, and the count of its running and finishing job runs, -which keeps it like a reference: `countRuns`), `Activity` (`host.activity`: counters, one +the chain it was created for, lingering, ending and revocation, the count of its running and finishing job runs, +which keeps it like a reference: `countRuns`, its contributions while live and its wake signals), `RetryWakes` (an +object's `retryOn` subscriptions), `RootScopes` (the open `host.root` scopes, sealed once each), `Activity` (`host.activity`: counters, one publication per microtask turn), `References` (one type's presence list), `RevocationKeys` (each principal's keys, in a `WeakMap`), `Attempt` (one activation: it is the @@ -471,9 +508,9 @@ native: opened once, closed once, a late value closed at once; `within` bounds a windows per identity and when each ends, holding no object), `Binding` (a reference's link to its object or remote object: its state, the object that declared it, its per-reference watches, calls, parked calls and the waits for a state; released, it keeps only its name), `ParkedCalls` (a reference's queue of calls made while loading, a timer -each), `ReferenceState` (`ref.state`: its native watches, told of each change), `CallContext` (a handler's -`ctx`), `withLinkedAbortSignal` (a call's or a get's context aborted by the caller and by a long-lived signal, its +each), `ReferenceState` (`ref.state`, a native source: its native watches, told of each change; derives over it +marked), `CallContext` (a handler's or a root scope's `ctx`), `withLinkedAbortSignal` (a call's or a get's context aborted by the caller and by a long-lived signal, its reference's or its attempt's, linked only until it settles: `AbortSignal.any` keeps cleanup entries per call while its -sources live before Node 24.16.0), `checkImplementation` (the variant and member check), `setUp` (runs one facet's setup) and -`resolveWatchable` (the member check, Q14), and `nextDelay` (the 1 s to 60 s doubling every backoff shares). Cycle checks follow the waits of activating objects with a visited set, +sources live before Node 24.16.0), `checkImplementation` (the variant and member check), `setUp` (runs one facet's setup), +`resolveWatchable` (the member check, Q14), `contributionOf` (a contribution as Chord follows it, the cycle check), and `nextDelay` (the 1 s to 60 s doubling every backoff shares). Cycle checks follow the waits of activating objects with a visited set, linear in the waits. diff --git a/packages/chord/src/next/host/attempt.ts b/packages/chord/src/next/host/attempt.ts index b6d04a898..259b6a0cf 100644 --- a/packages/chord/src/next/host/attempt.ts +++ b/packages/chord/src/next/host/attempt.ts @@ -8,7 +8,7 @@ import type { JsonValue } from "../../json.ts"; import type { NativeWatchable, Watchable, WatchEnd } from "../../watch.ts"; import type { Contract, Implementation, ParamsOf } from "../contracts/index.ts"; import { contractData, type ShapeOf } from "../contracts/internal.ts"; -import { ChordError, type FailureCause, type Principal } from "../data/index.ts"; +import { ChordError, type FailureCause, isChordError, type Principal } from "../data/index.ts"; import { asChordError, checkArgs, checkParams, invalid, openFailure } from "../data/internal.ts"; import type { HandleOf, JsonObject, StateDefinition, Tx } from "../documents/index.ts"; import { @@ -21,7 +21,10 @@ import { } from "../documents/internal.ts"; import type { JobDefinition, JobHandle, JobImplementation } from "../jobs/index.ts"; import { type AnyJobImplementation, Job } from "../jobs/internal.ts"; -import { Derive, type DeriveSource } from "../state/internal.ts"; +import type { Contribution, Registry, RegistryValue } from "../registries/index.ts"; +import { type RegistryContribution, type RegistryEdge, RegistryValueSource } from "../registries/internal.ts"; +import type { Writable } from "../state/index.ts"; +import { Derive, type DeriveSource, WritableSource } from "../state/internal.ts"; import type { Columns } from "../store/columns.ts"; import { tableSqlName } from "../store/sql.ts"; import type { ChordStore } from "../store/store.ts"; @@ -32,7 +35,7 @@ import { CallContext, type Caller, type Ctx, withLinkedAbortSignal } from "./ctx import { Deadline } from "./deadline.ts"; import { type FollowAttach, FollowSource } from "./follow.ts"; import type { HostError } from "./host.ts"; -import { attachDerives, checkImplementation, type KeyedBinding, type Members } from "./members.ts"; +import { attachDerives, checkImplementation, contributionOf, type KeyedBinding, type Members } from "./members.ts"; import { type CloseFailed, drained, @@ -44,6 +47,7 @@ import { } from "./native.ts"; import type { ChordObject } from "./object.ts"; import type { Binding, CheckedParams, Reference, Target } from "./reference.ts"; +import { type RetrySource, retrySource } from "./retry-on.ts"; import { type OnHandler, type OnLiveHandler, @@ -53,19 +57,48 @@ import { } from "./routines.ts"; /** - * What a factory declares with. Declarations (`state`, `derive`, `get`, `open`, `on`, `onLive`, `job`) are valid only - * while it runs. `P` is the app's principal type, which the ctx of routine calls and job runs carry. + * What a factory declares with. Declarations (`state`, `derive`, `get`, `registry`, `writable`, `retryOn`, `open`, + * `on`, `onLive`, `job`) are valid only while it runs. `P` is the app's principal type, which the ctx of routine calls + * and job runs carry. */ export interface Self

extends RoutineDeclarations

{ /** This object's documents of `definition`, in memory for its life (across its attempts). */ state>(definition: D): HandleOf; - /** A value computed from `sources` by the pure, synchronous `compute`. */ - derive[], T extends JsonValue>( + /** + * A value computed from `sources` (JSON watchables and native ones: a `writable`, a registry value, `ref.state`) by + * the pure, synchronous `compute`; always JSON. + */ + derive | NativeWatchable)[], T extends JsonValue>( sources: S, - compute: (values: { readonly [I in keyof S]: S[I] extends Watchable ? V : never }) => T, + compute: ( + values: { + readonly [I in keyof S]: S[I] extends Watchable + ? V + : S[I] extends NativeWatchable + ? V + : never; + }, + ) => T, ): Watchable; /** A reference to another object, live once this object is, released when it ends. */ get(Type: C, params: ParamsOf): Reference; + /** + * Registry `R`'s composed value as this plugin's version of `R` sees it: a declared dependency, as `get` is + * (activation waits until it has a value; with none it fails `transient` and is retried when one arrives, R18). + * Read `.value` at use; once live, a later failure of the registry keeps its last good value. + */ + registry(R: Registry): RegistryValue; + /** + * An in-process value this attempt sets (`set`, identity compared): a contribution, a derive source and an `on` + * source, never a state member; its watches end with the attempt, and `set` after its end throws `released`. + */ + writable(initial: T): Writable; + /** + * Wake signals: a new value of any of `sources` (registry values; constants never change) retries this object + * while it is failed, as `ref.retry()` does, and once more at once if its current activation fails. Followed from + * the factory's return until the next attempt's declaration or the object's end; failures wake nothing. + */ + retryOn(...sources: readonly (Watchable | NativeWatchable)[]): void; /** * A native, opened during activation after the references are live and the earlier natives are open, and closed * in reverse when the attempt ends. `open` gets a context aborted when the attempt is abandoned; `close` one aborted @@ -101,6 +134,12 @@ export interface StoreSelf

extends Self

{ state>(definition: D): HandleOf; /** This object's table `definition` in the host's store, once per table name. */ table(definition: TableDefinition): Table; + /** + * Contributes `value` to registry `R` from the moment this object is live until it ends, fails or restarts (R21: a + * host-lifetime object): a plain value, or a Chord watchable of one (followed: its change recomposes). At most one + * per registry. + */ + contribute(R: Registry, value: Contribution): void; } /** @@ -111,6 +150,27 @@ export type Factory = Shape ? (params: ParamsOf, self: keyof ParamsOf extends never ? StoreSelf

: Self

) => Implementation> : (params: ParamsOf, principal: P, self: Self

) => Implementation>; +/** How an activation fails on a declared registry without a value (`#readRegistries`). */ +type UnreadRegistry = Extract; + +/** No declarations of a kind: the shared empty list. */ +const NONE: readonly never[] = []; + +/** An attempt's registry reads, contributions and wake signals, and its writables (slice 8). */ +type RegistryDeclarations = { + /** The registry values it declared (`self.registry`), registered as read during activation (`reads`). */ + readonly registries: RegistryValueSource[]; + /** Releases its registry reads (with its references). */ + readonly reads: (() => void)[]; + /** Its wake signals (`self.retryOn`), handed to its object right after the factory. */ + readonly retrySources: RetrySource[]; + /** Its writables, whose watches end with it. */ + readonly writables: { detach(error: Error): void }[]; + /** Its contributions as declared, resolved during activation into `contributions`. */ + readonly declared: { readonly registry: Registry; readonly value: unknown }[]; + contributions: readonly RegistryContribution[]; +}; + /** How an attempt's activation ended. `dependency_failed` (R18) keeps the attempt's references. */ type Activation = | { readonly kind: "live" } @@ -174,6 +234,8 @@ export class Attempt implements StoreSelf, KeyDeclarer { #abandoned: Exclude | undefined; /** A followed source's end while this attempt activated: it fails as it goes live (if its activation returned). */ #endedEarly: ChordError | undefined; + /** Its registry reads, contributions, wake signals and writables, made by the first: most attempts have none. */ + #registryDeclarations: RegistryDeclarations | undefined; constructor(object: ChordObject, previous: Attempt | undefined, context: Context) { this.#object = object; @@ -221,13 +283,21 @@ export class Attempt implements StoreSelf, KeyDeclarer { return handle as unknown as Table; } - derive[], T extends JsonValue>( + derive | NativeWatchable)[], T extends JsonValue>( sources: S, - compute: (values: { readonly [I in keyof S]: S[I] extends Watchable ? V : never }) => T, + compute: ( + values: { + readonly [I in keyof S]: S[I] extends Watchable + ? V + : S[I] extends NativeWatchable + ? V + : never; + }, + ) => T, ): Watchable { this.#assertDeclaring("derive"); // The values are the sources' values, in order; the result is `T`. - const derive = new Derive(sources, compute as (values: readonly JsonValue[]) => T, this.#object.host.batch); + const derive = new Derive(sources, compute as (values: readonly unknown[]) => T, this.#object.host.batch); this.#derives.push(derive); return derive as unknown as Watchable; } @@ -242,6 +312,50 @@ export class Attempt implements StoreSelf, KeyDeclarer { return binding.reference as Reference; } + registry(R: Registry): RegistryValue { + this.#assertDeclaring("registry"); + const value = this.#object.host.registryRecord(R.name).value(R.version); + this.#declarations().registries.push(value); + // Its values are the composition as `R`'s version has it, `V`. + return value as RegistryValue; + } + + writable(initial: T): Writable { + this.#assertDeclaring("writable"); + const owner = { + context: this.#context, + assertUsable: () => { + if (this.#token.sealed) throw this.#object.ended(); + }, + }; + const writable = new WritableSource(initial, owner, this.#object.host.batch); + this.#declarations().writables.push(writable); + return writable; + } + + retryOn(...sources: readonly (Watchable | NativeWatchable)[]): void { + this.#assertDeclaring("retryOn"); + const { retrySources } = this.#declarations(); + for (const source of sources) retrySources.push(retrySource(source, this.#object.label)); + } + + contribute(R: Registry, value: Contribution): void { + this.#assertDeclaring("contribute"); + const { served, label, host } = this.#object; + // R21: the types allow it only to host-lifetime objects; plugin code is bundled without type checks. + if (served.kind !== "local" || !served.hostLifetime) { + throw new Error(`${label}: self.contribute is valid only in a local type with empty params`); + } + const { declared } = this.#declarations(); + if (declared.some(({ registry }) => registry.name === R.name)) { + throw new Error(`${label}: registry ${R.name} is contributed to twice`); + } + // Refused at `contribute` (declaration policy), also a cycle within this factory's own declarations; the edges + // live with the contributions in the tables, from live on. + if (value instanceof RegistryValueSource) host.registryRecord(R.name).assertNoCycle(value, this.#pendingEdges()); + declared.push({ registry: R as Registry, value }); + } + open( open: (context: Context) => Promise, close: (value: T, reason: NativeCloseReason, context: Context) => void | Promise, @@ -384,6 +498,11 @@ export class Attempt implements StoreSelf, KeyDeclarer { }); } + /** Its contributions, resolved by its activation: in the registries' tables while its object is live. */ + get contributions(): readonly RegistryContribution[] { + return this.#registryDeclarations?.contributions ?? NONE; + } + /** The variant a remote object's factory chose; `undefined` for a local object. */ get variant(): string | undefined { return this.#variant; @@ -494,27 +613,34 @@ export class Attempt implements StoreSelf, KeyDeclarer { } /** - * Detaches its derives (once their object no longer needs them: reads throw `released`, their watches end) and its - * followed sources (their observers detached, their watches ended). + * Detaches its derives (once their object no longer needs them: reads throw `released`, their watches end), its + * followed sources (their observers detached, their watches ended) and its writables (their watches ended). */ detach(): void { const ended = this.#object.ended(); for (const derive of this.#derives) derive.detach(ended); for (const follow of this.#follows) follow.detach(); + const declarations = this.#registryDeclarations; + if (declarations !== undefined) for (const writable of declarations.writables) writable.detach(ended); } - /** Releases its references, and the previous attempt's if this one never bound its own. */ + /** Releases its references and registry reads, and the previous attempt's if this one never bound its own. */ releaseReferences(): void { this.#releasePrevious(); for (const { binding } of this.#references) binding.release(); + const declarations = this.#registryDeclarations; + if (declarations !== undefined) for (const release of declarations.reads.splice(0)) release(); } /** The activation steps; each catches its own failures with their kind (the table in slice 2's spec). */ async #activate(deadline: Deadline): Promise { let implementation: unknown; let dependencies: Target[]; + let unread: UnreadRegistry | undefined; try { implementation = this.#declare(); + // Right after the factory, before anything can fail: a failed attempt keeps its wake signals. + this.#object.retryOn(this.#registryDeclarations?.retrySources ?? NONE); const { documents } = this.#object; // The object holds the pins from now on. documents.hydrate(this.#declared, this.#pins.splice(0), this.#opening); @@ -528,12 +654,15 @@ export class Attempt implements StoreSelf, KeyDeclarer { host.bind(binding, contract, params, this.#opening), ); for (const dependency of dependencies) this.#object.assertNoCycle(dependency, this.#opening); + if (this.#registryDeclarations !== undefined) unread = this.#readRegistries(this.#registryDeclarations); } catch (error) { return this.#broken(error); } finally { // Bound first: a dependency held only by the previous attempt never ends in between. this.#releasePrevious(); } + // No value after `load`'s setups: none comes before a first composition (which retries it, R18) or a reload. + if (unread !== undefined) return this.#abandoned ?? unread; try { await Promise.all(dependencies.map((dependency) => this.#object.waitFor(dependency, this.#opening))); } catch (error) { @@ -586,6 +715,8 @@ export class Attempt implements StoreSelf, KeyDeclarer { const checked = checkImplementation(this.#object.served.contract, implementation); this.#members = checked.members; this.#variant = checked.variant; + if (this.#registryDeclarations !== undefined) + this.#registryDeclarations.contributions = this.#resolveContributions(this.#registryDeclarations); const { host, label } = this.#object; attachDerives(this.#derives, this.#members.state, (member) => (error) => { // During activation a throwing derive fails the attempt, which `activation_failed` reports alone. @@ -607,6 +738,41 @@ export class Attempt implements StoreSelf, KeyDeclarer { return this.#abandoned ?? { kind: "live" }; } + /** + * Registers this attempt's object as a reader of every registry it declared (woken by a first value, R18, until its + * references are released); returns how the activation fails on the first with no value, `undefined` when every + * one has a value. A version the provider does not serve (S2) is final until a reload: broken, the + * `contract_mismatch` itself the cause. Otherwise (no provider, a first `compose` that threw) a value may still come: + * `transient` (R18, the references kept), naming the registry, its error as the cause. + */ + #readRegistries({ registries, reads }: RegistryDeclarations): UnreadRegistry | undefined { + let unread: UnreadRegistry | undefined; + for (const value of registries) { + reads.push(value.read(this.#object)); + const start = value.start(); + if (unread !== undefined || start.kind === "value" || start.last !== undefined) continue; + const { error } = start; + if (isChordError(error, "contract_mismatch")) { + unread = { kind: "failed", cause: { kind: "broken", error } }; + continue; + } + const message = `${this.#object.label}: registry ${value.name} has no value`; + const cause = new ChordError({ code: "unavailable", message }, { cause: error }); + unread = { kind: "dependency_failed", cause: { kind: "transient", error: cause } }; + } + return unread; + } + + /** Its declared contributions, as Chord follows them (its references are live now), by its plugin and type. */ + #resolveContributions({ declared }: RegistryDeclarations): readonly RegistryContribution[] { + const { served, host } = this.#object; + const contributor = { plugin: served.plugin, type: served.contract.name }; + const pending = this.#pendingEdges(); + return declared.map(({ registry, value }) => + contributionOf(host.registryRecord(registry.name), contributor, registry.version, value, pending), + ); + } + /** * Brings the declared tables to their definitions before the references are bound (slice 6a §9): created fresh or * migrated eagerly, with the `readyTimeout` deadline paused meanwhile (resumed with the budget it had left). @@ -686,6 +852,29 @@ export class Attempt implements StoreSelf, KeyDeclarer { this.#previous = undefined; } + /** The registry edges its declared contributions would add (another registry's value contributed). */ + #pendingEdges(): RegistryEdge[] { + const edges: RegistryEdge[] = []; + const { host } = this.#object; + for (const { registry, value } of this.#registryDeclarations?.declared ?? NONE) { + const to = value instanceof RegistryValueSource ? value.record : undefined; + if (to !== undefined) edges.push({ from: host.registryRecord(registry.name), to }); + } + return edges; + } + + #declarations(): RegistryDeclarations { + this.#registryDeclarations ??= { + registries: [], + reads: [], + retrySources: [], + writables: [], + declared: [], + contributions: NONE, + }; + return this.#registryDeclarations; + } + #assertDeclaring(declaration: string): void { if (!this.#declaring) { throw new Error(`${this.#object.label}: self.${declaration} is valid only in the factory`); diff --git a/packages/chord/src/next/host/ctx.ts b/packages/chord/src/next/host/ctx.ts index 4eb44020b..d14d3470c 100644 --- a/packages/chord/src/next/host/ctx.ts +++ b/packages/chord/src/next/host/ctx.ts @@ -23,7 +23,7 @@ export interface Ctx

extends Context { /** Who makes a call: the principal it acts as and its host's incarnation. */ export type Caller = { readonly principal: Principal | undefined; readonly incarnation: string }; -/** The ctx of one handler run: its references are released by `settle` when the handler settled. */ +/** The ctx of one handler run (or root scope): its references are released by `settle` when it settled. */ export class CallContext implements Ctx { readonly principal: Principal | undefined; readonly incarnation: string; @@ -58,9 +58,9 @@ export class CallContext implements Ctx { } async get(Type: C, params: ParamsOf, options: GetOptions = {}): Promise> { - // An ended handler creates nothing: its call's abort signal is checked before anything else. - this.#context.abortSignal?.throwIfAborted(); + // A settled scope refuses `released` whatever aborted it; an aborted call creates nothing. if (this.#references === undefined) throw settled(Type); + this.#context.abortSignal?.throwIfAborted(); // Made for the caller: its principal and its origin's incarnation travel with the reference's holds. const caller = { principal: this.principal, incarnation: this.incarnation }; this.#settling ??= new AbortController(); @@ -96,7 +96,7 @@ function settled(Type: Contract): ChordError { return new ChordError({ code: "released", reason: "settled", - message: `${Type.name}: ctx.get after its handler settled`, + message: `${Type.name}: ctx.get after its call settled`, }); } diff --git a/packages/chord/src/next/host/facet.ts b/packages/chord/src/next/host/facet.ts index 0055285ff..4e9615852 100644 --- a/packages/chord/src/next/host/facet.ts +++ b/packages/chord/src/next/host/facet.ts @@ -1,16 +1,23 @@ -// Facets (§16): a plugin's setup for one host, typed by the app's principal type. A setup only declares: the types it -// serves and the host's presence lists; it has no documents, routines or natives, so a facet cannot fail after `load`. +// Facets (§16): a plugin's setup for one host, typed by the app's principal type. A setup only declares: the types and +// registries it serves, its contributions to registries (slice 8) and the host's presence lists; it has no documents, +// routines or natives, so a facet cannot fail after `load`. import type { Watchable } from "../../watch.ts"; import type { Contract } from "../contracts/index.ts"; import type { Principal } from "../data/index.ts"; +import type { Contribution, Registry, RegistryProvider } from "../registries/index.ts"; import type { Factory } from "./attempt.ts"; import type { Presence } from "./presence.ts"; -/** What a setup serves types with (valid only while the setup runs), and the host's presence lists (always). */ +/** What a setup serves types and registries with (valid only while the setup runs), and the host's presence lists. */ export interface FacetHost

{ /** Serves a type on this host with `factory`; a remote type's factory gets the principal its object is for. */ serve(Type: C, factory: Factory, options?: ServeOptions): void; + /** + * Serves registry `R` on this host (its defining plugin's facet, the one provider, D1): `compose` builds the value + * from the table, `accepts` adapts older versions. A second serve of a registry is refused at `load`. + */ + serve(R: Registry, provider: RegistryProvider): void; /** `host.references(Type)`: the references to objects of `Type` on this host (§19), for factories to derive from. */ references(Type: C): Watchable[]>; } @@ -24,48 +31,92 @@ export type ServeOptions = { readonly linger?: number; }; -/** A type a setup served: its contract, its factory and its options. */ -export type ServedEntry = { - readonly contract: Contract; - readonly factory: ServedFactory; - readonly options: ServeOptions; -}; +/** What a facet's setup declares with besides the host: its contributions to registries (R21). */ +export interface FacetSelf { + /** + * Contributes `value` to registry `R` while this plugin is loaded: a plain value, or a Chord watchable of one + * (followed: its change recomposes). At most one per registry. + */ + contribute(R: Registry, value: Contribution): void; +} + +/** What a setup served, by kind: a type (its contract, factory and options), or a registry (its provider). */ +export type ServedEntry = + | { + readonly kind: "type"; + readonly contract: Contract; + readonly factory: ServedFactory; + readonly options: ServeOptions; + } + | { + readonly kind: "registry"; + readonly registry: Registry; + readonly provider: RegistryProvider; + }; + +/** A contribution a setup declared. */ +export type FacetContribution = { readonly registry: Registry; readonly value: unknown }; /** A factory as a setup served it, of either kind (the host dispatches on its contract's kind). */ type ServedFactory = (params: never, principalOrSelf: never, self: never) => unknown; /** A plugin's facet for hosts of the app's principal type `P` (type only: nothing is checked at runtime). */ export type Facet

= { - readonly setup: (host: FacetHost

) => void; + readonly setup: (host: FacetHost

, self: FacetSelf) => void; /** Type only: the principal type, invariant, so a host takes only facets typed for exactly its `P`. */ readonly "~principal"?: (principal: P) => P; }; /** Defines a facet; base code binds `P` once, next to its principal type. */ -export function defineFacet

(setup: (host: FacetHost

) => void): Facet

{ +export function defineFacet

(setup: (host: FacetHost

, self: FacetSelf) => void): Facet

{ return { setup }; } -/** Runs a facet's setup and returns the types it served, in order; `references` is the host's. */ +/** + * Runs a facet's setup and returns the types and registries it served and its contributions, in order; `references` + * is the host's. + */ export function setUp

( name: string, facet: Facet

, references: (Type: C) => Watchable[]>, -): readonly ServedEntry[] { +): { readonly served: readonly ServedEntry[]; readonly contributions: readonly FacetContribution[] } { const served: ServedEntry[] = []; + const contributions: FacetContribution[] = []; let open = true; + const assertOpen = (what: string): void => { + if (!open) throw new Error(`${name}: ${what} is valid only in the facet's setup`); + }; const host: FacetHost

= { - serve: (Type, factory, options = {}) => { - if (!open) throw new Error(`${name}: host.serve is valid only in the facet's setup`); - // Served types are dispatched by name and kind; the factory matches its own contract. - served.push({ contract: Type, factory: factory as ServedFactory, options }); + serve: (Type: Contract | Registry, implementation: unknown, options: ServeOptions = {}) => { + assertOpen("host.serve"); + // Served types and registries are dispatched by their kind; a factory matches its own contract, a provider + // its own registry. + if (Type.kind === "registry") { + served.push({ + kind: "registry", + registry: Type, + provider: implementation as RegistryProvider, + }); + } else { + served.push({ kind: "type", contract: Type, factory: implementation as ServedFactory, options }); + } }, references, }; + const self: FacetSelf = { + contribute: (registry, value) => { + assertOpen("self.contribute"); + if (contributions.some((each) => each.registry.name === registry.name)) { + throw new Error(`${name}: registry ${registry.name} is contributed to twice`); + } + contributions.push({ registry: registry as Registry, value }); + }, + }; try { - facet.setup(host); + facet.setup(host, self); } finally { open = false; } - return served; + return { served, contributions }; } diff --git a/packages/chord/src/next/host/host.ts b/packages/chord/src/next/host/host.ts index cbafd799b..5b360e4ba 100644 --- a/packages/chord/src/next/host/host.ts +++ b/packages/chord/src/next/host/host.ts @@ -1,10 +1,16 @@ // Hosts (§3): `createHost`, loading facets and the start barrier, links to children, the parent (connected again after // a drop) and routes, the origins it relays holds for, getting and holding references as a principal, the heartbeat -// of every link, revocation (§4), presence (§19) and keep-alive counts, closing; one batch per host, the backoff map -// and the endings new instances wait for. +// of every link, revocation (§4), presence (§19) and keep-alive counts, registries and root scopes (slice 8), closing; +// one batch per host, the backoff map and the endings new instances wait for. import type { TelemetryContext } from "@earendil-works/pi-telemetry"; -import { BACKGROUND_CONTEXT, type Context, withAbortSignal, withoutAbortSignal } from "../../context/index.ts"; +import { + awaitWithContext, + BACKGROUND_CONTEXT, + type Context, + withAbortSignal, + withoutAbortSignal, +} from "../../context/index.ts"; import type { JsonValue } from "../../json.ts"; import type { Watchable } from "../../watch.ts"; import type { Address, Contract, HostKind, ParamsOf } from "../contracts/index.ts"; @@ -21,6 +27,13 @@ import { } from "../data/internal.ts"; import { binaryCodec, type Codec, type Compression, type Link, type PartLimits } from "../link/index.ts"; import { linkLimits } from "../link/internal.ts"; +import type { Registry, RegistryValue } from "../registries/index.ts"; +import { + type RegistryContribution, + type RegistryEdge, + type RegistryError, + RegistryRecord, +} from "../registries/internal.ts"; import { type HoldRequest, HostLink, @@ -34,14 +47,16 @@ import type { ChordStore, Store } from "../store/store.ts"; import { Activity, type ActivityCounts } from "./activity.ts"; import { notLive } from "./attempt.ts"; import { Backoff } from "./backoff.ts"; -import { type Caller, withLinkedAbortSignal } from "./ctx.ts"; -import { type Facet, type ServedEntry, setUp } from "./facet.ts"; +import { type Caller, type Ctx, withLinkedAbortSignal } from "./ctx.ts"; +import { type Facet, type FacetContribution, type ServedEntry, setUp } from "./facet.ts"; +import { contributionOf } from "./members.ts"; import type { CloseFailed } from "./native.ts"; import { ChordObject, type Holder, type LocalContract, type RemoteContract, type Served } from "./object.ts"; import { type Placed, Placement } from "./placement.ts"; import { type Presence, References } from "./presence.ts"; import { Binding, type CheckedParams, type Held, type Reference, type Target } from "./reference.ts"; import { RevocationKeys, type RevokedBy } from "./revocation.ts"; +import { type RootOptions, RootScopes } from "./root.ts"; import { Route, type RouteOptions } from "./route.ts"; import { uuidv7 } from "./uuid.ts"; @@ -93,6 +108,7 @@ export type Position = { readonly kind: "on" | "onLive" | "native"; readonly pos */ export type HostError = | BatchError + | RegistryError | { readonly kind: "activation_failed"; readonly object: string; readonly failure: FailureCause } /** A live object failed at runtime (a `migrate` failing on access, a stored version newer than the code). */ | { readonly kind: "object_failed"; readonly object: string; readonly failure: FailureCause } @@ -163,7 +179,7 @@ export type GetOptions = { * A host whatever its principal type, as Chord's own parts use it: `ChordHost

` is invariant in `P` (`load` takes * facets typed for exactly `P`). */ -export type AnyHost = Omit; +export type AnyHost = Omit; /** A host. `P` is the app's principal type, `R` its revocation type. */ export interface Host

{ @@ -228,8 +244,21 @@ export interface Host

; /** - * Closes its links, then ends every object at once, then drains and closes them one by one, the last to go live - * first (reason `close`). Rejects with an `AggregateError` of the closes that threw. + * Registry `R`'s composed value as `R`'s version sees it, for code outside objects; waits for `load` first. Rejects + * when it has no value then (`unavailable/undeclared` without a provider, a first `compose` that threw, a version + * not served here: `contract_mismatch`); none can come before a reload. + */ + registry(R: Registry, context: Context): Promise>; + /** + * Runs `fn` with a call context for code outside objects (a Durable tool, a CLI): the host's principal (or + * `options.principal`), this host's incarnation, `context`'s telemetry and signal, `ctx.get`. When `fn` settles or + * `context` aborts, the context is sealed: its signal aborted, `ctx.get` refused `released`, every reference it took + * released; an abort rejects at once. `host.close` seals the open ones; after it, `root` rejects `released`. + */ + root(fn: (ctx: Ctx

) => Promise, context: Context, options?: RootOptions

): Promise; + /** + * Closes its links, seals its root scopes, then ends every object at once, then drains and closes them one by one, + * the last to go live first (reason `close`). Rejects with an `AggregateError` of the closes that threw. */ close(context: Context): Promise; } @@ -241,6 +270,9 @@ export function createHost

(options); } +/** No registry edges pending (a facet's contributions). */ +const NO_EDGES: readonly RegistryEdge[] = []; + /** No activation chain (Q5). */ const NO_CHAIN: readonly string[] = []; @@ -291,6 +323,9 @@ export class ChordHost

(); + readonly #roots = new RootScopes(this); /** The one heartbeat timer, running while the host has a link. */ #ticking: ReturnType | undefined; #state: "new" | "loading" | "loaded" | "closed" = "new"; @@ -321,7 +356,13 @@ export class ChordHost

{ + for (const contribution of contributions) contribution.record.add(contribution); + for (const record of this.#registries.values()) record.composeServed(); + }, context); } catch (error) { // The host closes as `close` would: its links too. Nothing was activated, so no close can fail. this.#closing ??= this.#close(); @@ -422,6 +463,35 @@ export class ChordHost

[]>; } + async registry(R: Registry, context: Context): Promise> { + this.batch.assertOutside("registry"); + const loading = this.pendingLoad; + if (loading !== undefined) await awaitWithContext(loading, context); + this.#assertOpen(); + const value = this.registryRecord(R.name).value(R.version); + const start = value.start(); + if (start.kind === "failed" && start.last === undefined) throw start.error; + // Its values are the composition as `R`'s version has it, `V`. + return value as RegistryValue; + } + + async root(fn: (ctx: Ctx

) => Promise, context: Context, options: RootOptions

= {}): Promise { + this.batch.assertOutside("root"); + this.#assertOpen(); + // The scope's ctx acts as `principal`, a `P`. + return this.#roots.run(fn as (ctx: Ctx) => Promise, options.principal ?? this.principal, context); + } + + /** The record of registry `name` on this host, made by its first use. */ + registryRecord(name: string): RegistryRecord { + let record = this.#registries.get(name); + if (record === undefined) { + record = new RegistryRecord(name, this.batch, (error) => this.report(error)); + this.#registries.set(name, record); + } + return record; + } + async close(_context: Context): Promise { this.batch.assertOutside("close"); this.#closing ??= this.#close(); @@ -782,6 +852,8 @@ export class ChordHost

}[]): void { + /** + * Runs every facet's setup, checking the plugin names (npm names, each once), and serves what they declared; returns + * their contributions, which enter the registries' tables once the setups ran. + */ + #setUp(facets: readonly { readonly name: string; readonly facet: Facet

}[]): RegistryContribution[] { const names = new Set(); + const made: RegistryContribution[] = []; for (const { name, facet } of facets) { if (!PLUGIN_NAME.test(name)) throw new Error(`"${name}" is not a valid plugin name`); if (names.has(name)) throw new Error(`plugin "${name}" is loaded twice`); names.add(name); - for (const entry of setUp(name, facet, (Type) => this.references(Type))) this.#serve(entry); + const { served, contributions } = setUp(name, facet, (Type) => this.references(Type)); + for (const entry of served) this.#serve(name, entry); + for (const contribution of contributions) made.push(this.#contribution(name, contribution)); } + return made; } - #serve({ contract, factory, options }: ServedEntry): void { + /** A facet's contribution, checked: in its registry's table while the host lives (its plugin stays loaded). */ + #contribution(plugin: string, { registry, value }: FacetContribution): RegistryContribution { + // A facet contributes no registry's value (it reads none): no edges of its own. + const record = this.registryRecord(registry.name); + return contributionOf(record, { plugin, type: undefined }, registry.version, value, NO_EDGES); + } + + #serve(plugin: string, entry: ServedEntry): void { + if (entry.kind === "registry") { + this.registryRecord(entry.registry.name).serve(entry.registry.version, entry.provider); + return; + } + const { contract, factory, options } = entry; if (this.#types.has(contract.name)) throw new Error(`${contract.name} is served twice`); // A client serves only local types (§3): a remote object is owned by a principal of the host serving it. if (contract.kind === "remote" && this.principal === undefined) { throw new Error(`${contract.name}: a host without a principal serves only local types`); } - const common = { provider: bindProvider(contract, {}), linger: options.linger ?? 0 }; + const common = { plugin, provider: bindProvider(contract, {}), linger: options.linger ?? 0 }; // The contract's kind decides the factory's signature and the contract's shape. const served: Served = contract.kind === "local" diff --git a/packages/chord/src/next/host/index.ts b/packages/chord/src/next/host/index.ts index 3d7611c5d..79b8a2634 100644 --- a/packages/chord/src/next/host/index.ts +++ b/packages/chord/src/next/host/index.ts @@ -1,10 +1,10 @@ // Hosts for apps: creating a host, facets, factories, natives, references and the ctx of method handlers, routines and -// their waiting helpers, revocation, presence and keep-alive counts. +// their waiting helpers, revocation, presence and keep-alive counts, root scopes. export type { ActivityCounts } from "./activity.ts"; export type { Factory, Self, StoreSelf } from "./attempt.ts"; export type { Ctx } from "./ctx.ts"; -export { defineFacet, type Facet, type FacetHost, type ServeOptions } from "./facet.ts"; +export { defineFacet, type Facet, type FacetHost, type FacetSelf, type ServeOptions } from "./facet.ts"; export type { FollowAttach, FollowAttachment, FollowObserver } from "./follow.ts"; export { createHost, @@ -20,6 +20,7 @@ export type { Presence } from "./presence.ts"; export type { RefState } from "./ref-state.ts"; export type { Held, Reference, RefIn } from "./reference.ts"; export type { RevokedBy } from "./revocation.ts"; +export type { RootOptions } from "./root.ts"; export type { RouteOptions } from "./route.ts"; export type { OnHandler, OnLiveHandler, OnNativeHandler, RoutineDeclarations } from "./routines.ts"; export { sleep, untilAborted } from "./sleep.ts"; diff --git a/packages/chord/src/next/host/members.ts b/packages/chord/src/next/host/members.ts index 7eaf1fc73..ed2222cd9 100644 --- a/packages/chord/src/next/host/members.ts +++ b/packages/chord/src/next/host/members.ts @@ -1,5 +1,6 @@ // The member check (Q14): an implementation is one variant with exactly that variant's members, methods being -// functions and state members JSON watchables; and the resolution of every derive's declared sources. +// functions and state members JSON watchables; and the resolution of what Chord follows behind every derive's declared +// sources (JSON and native) and every contribution to a registry (slice 8). // // The rule for watches across attempts (slice 6a §1.2, §1.4): references and other objects' derives follow a local // object's state member through its member source (`member-source.ts`), and a key of a keyed member they hold (`at`, @@ -17,7 +18,15 @@ import type { Context } from "../../context/index.ts"; import type { Contract } from "../contracts/index.ts"; import { currentVariants, LOCAL, takesNoArgs, type VariantData } from "../contracts/internal.ts"; import type { KeyedSlot } from "../documents/internal.ts"; -import type { Derive, DeriveSource } from "../state/internal.ts"; +import { + type ContributionContent, + type Contributor, + RegistryContribution, + type RegistryEdge, + type RegistryRecord, + RegistryValueSource, +} from "../registries/internal.ts"; +import { type Derive, type DeriveInput, type DeriveSource, NativeSource } from "../state/internal.ts"; import { type Binding, resolveKeyed, resolveWatchable } from "./reference.ts"; /** A checked implementation, as Chord calls and follows it. */ @@ -114,11 +123,35 @@ export function attachDerives( derives.forEach((derive, position) => { const member = exposed.get(derive); const name = member ?? `derive #${position + 1}`; - const resolve = (source: unknown, index: number): DeriveSource => { - const resolved = resolveWatchable(source); - if (resolved === undefined) throw new Error(`${name}: source ${index} is not a JSON watchable`); + const resolve = (source: unknown, index: number): DeriveInput => { + const resolved = resolveWatchable(source) ?? (source instanceof NativeSource ? source : undefined); + if (resolved === undefined) throw new Error(`${name}: source ${index} is not a Chord watchable`); return resolved; }; derive.attach(resolve, report(member)); }); } + +/** + * A contribution of `contributor` at `version` to `record`, as Chord follows it: a Chord watchable (a source, a + * constant, a document handle, a reference member, a native source) is followed, anything else is a plain value, + * whatever methods it has (registries carry natives). Refused (`invalid/cycle`) when it is a registry's value that + * closes a cycle of registries, through the tables and the contributor's `pending` edges. + */ +export function contributionOf( + record: RegistryRecord, + contributor: Contributor, + version: number, + value: unknown, + pending: readonly RegistryEdge[], +): RegistryContribution { + if (value instanceof RegistryValueSource) record.assertNoCycle(value, pending); + return new RegistryContribution(record, contributor, version, contentOf(value)); +} + +/** What Chord follows behind a contribution: its JSON or native source when it is a Chord watchable. */ +function contentOf(value: unknown): ContributionContent { + if (value instanceof NativeSource) return { kind: "native", source: value }; + const source = resolveWatchable(value); + return source === undefined ? { kind: "plain", value } : { kind: "json", source }; +} diff --git a/packages/chord/src/next/host/object.ts b/packages/chord/src/next/host/object.ts index f422a58fb..91e0bd2b8 100644 --- a/packages/chord/src/next/host/object.ts +++ b/packages/chord/src/next/host/object.ts @@ -2,7 +2,8 @@ // and job runs, its member sources, slice 6a §1.2), its holders, retries after failures (backoff, dependency recovery // R18, `ref.retry()`), a live object's failure (slice 6a §1.4), the waits and cycle checks of activations (Q5, its // instance id and the activation chain it was created for), lingering after its last reference (§3, D8), the job runs -// that keep it alive (§15), and its ending, revocation included (§4, R11). +// that keep it alive (§15), its contributions to registries while live and its wake signals (`retryOn`, slice 8), and +// its ending, revocation included (§4, R11). import { awaitWithContext, type Context } from "../../context/index.ts"; import type { Contract, ParamsOf } from "../contracts/index.ts"; @@ -23,6 +24,7 @@ import type { KeyedBinding } from "./members.ts"; import type { CloseFailed, NativeCloseReason } from "./native.ts"; import { type Ended, type Failed, LIVE, LOADING, type RefState } from "./ref-state.ts"; import type { Target } from "./reference.ts"; +import { type RetrySource, RetryWakes } from "./retry-on.ts"; import { uuidv7 } from "./uuid.ts"; /** A `local` contract, as a served type has it. */ @@ -48,6 +50,8 @@ export type Served = }); type ServedType = { + /** The plugin whose facet served it: its objects contribute to registries by it. */ + readonly plugin: string; readonly contract: C; readonly provider: Provider; /** How long a live object stays live after its last reference, in milliseconds (`serve`'s `linger`, 0 by default). */ @@ -146,6 +150,8 @@ export class ChordObject implements Target { readonly #keys: KeyedMembers; /** The one timer storing its jobs' held state changes by their earliest deadline (D8), while one is held. */ #flush: { readonly deadline: number; readonly timer: ReturnType } | undefined; + /** Its wake signals (`retryOn`), the last attempt's declaration; made by the first. */ + #wakes: RetryWakes | undefined; constructor( host: AnyHost, @@ -264,6 +270,17 @@ export class ChordObject implements Target { this.attempt(); } + /** + * Follows `sources` as its wake signals instead of the last attempt's (`self.retryOn`, right after a factory + * returned): a new value retries it while failed, as `ref.retry()` does; one while it activates retries it at once + * if that attempt fails; one while live does nothing. + */ + retryOn(sources: readonly RetrySource[]): void { + if (this.#wakes === undefined && sources.length === 0) return; + this.#wakes ??= new RetryWakes(); + this.#wakes.replace(sources, () => this.#wake()); + } + /** Calls a method of the live object for `caller`. */ async call(method: string, args: unknown, context: Context, caller: Caller): Promise { return this.#live().call(method, args, context, caller); @@ -383,6 +400,7 @@ export class ChordObject implements Target { this.#lastFailure = error; this.#outcome = settlement(); this.#outcome.reject(error); + this.#withdraw(attempt); attempt.seal(error); // Members following another object's sources freeze now: no commit of this object changes them. this.#freeze(attempt, "foreign"); @@ -530,10 +548,18 @@ export class ChordObject implements Target { #wentLive(attempt: Attempt): void { this.#woken = undefined; + if (this.#wakes !== undefined) this.#wakes.during = undefined; this.#state = { kind: "live", attempt }; this.#exposed = attempt; if (this.#members.size > 0 || !this.#keys.empty) this.host.batch.within(() => this.#rebind(attempt), this.#context); + const { contributions } = attempt; + // In the registries' tables before anyone sees it live (a getter released now reads them recomposed). + if (contributions.length > 0) { + this.host.batch.within(() => { + for (const contribution of contributions) contribution.record.add(contribution); + }, this.#context); + } this.#lastFailure = undefined; this.host.wentLive(this); this.#outcome.resolve(); @@ -551,6 +577,8 @@ export class ChordObject implements Target { #failed(attempt: Attempt, cause: FailureCause, holding: Attempt | undefined): void { const woken = this.#woken === attempt; this.#woken = undefined; + const wokenBy = this.#wakes?.during === attempt; + if (this.#wakes !== undefined) this.#wakes.during = undefined; // What its activation loaded for the pinned keys goes; they stay frozen. this.#keys.discard(); if (cause.kind === "transient") this.host.backoff.failed(this.identity); @@ -573,6 +601,44 @@ export class ChordObject implements Target { this.#idle(); // A dependency went live while this attempt activated: the wake it missed retries it now (R18). if (holding !== undefined && woken) this.dependencyLive(); + // A wake signal arrived while it activated: it retries it now. + else if (wokenBy) this.retry(); + } + + /** + * A wake signal's new value (inside its batch): retries a failed object (after the batch: a factory may not run in + * one), remembered while it activates, ignored while live and when its own contributions' leaving made it. + */ + #wake(): void { + const state = this.#state; + // Called by its wakes only. + const wakes = this.#wakes!; + if (state.kind === "activating") { + wakes.during = state.attempt; + return; + } + if (state.kind !== "failed" || wakes.leaving) return; + queueMicrotask(() => this.retry()); + } + + /** + * The live attempt `attempt` leaves the registries' tables (it fails or ends), inside a batch; a composition this + * leaving makes wakes no wake signal of this object. + */ + #withdraw(attempt: Attempt): void { + const { contributions } = attempt; + if (contributions.length === 0) return; + const { batch } = this.host; + const wakes = this.#wakes; + batch.within(() => { + if (wakes !== undefined) { + wakes.leaving = true; + batch.flushing(() => { + wakes.leaving = false; + }); + } + for (const contribution of contributions) contribution.record.remove(contribution); + }, this.#context); } /** The member source of `member`, made bound to `exposed`'s member by its first use. */ @@ -637,6 +703,7 @@ export class ChordObject implements Target { this.#flush = undefined; this.documents.seal(); this.#stopLingering(); + this.#wakes?.stop(); this.host.forget(this); this.#outcome.reject(error); // In a batch: the derives over its members are marked, so none keeps a cached value. @@ -656,6 +723,7 @@ export class ChordObject implements Target { const { holding } = state; return holding === undefined ? { kind: "end", previous } : { kind: "release", attempt: holding, previous }; } + if (state.kind === "live") this.#withdraw(state.attempt); state.attempt.seal(error); return { kind: "close", attempt: state.attempt, previous }; } diff --git a/packages/chord/src/next/host/ref-state.ts b/packages/chord/src/next/host/ref-state.ts index 75f9fa42e..9e5720405 100644 --- a/packages/chord/src/next/host/ref-state.ts +++ b/packages/chord/src/next/host/ref-state.ts @@ -1,9 +1,10 @@ // The state of a reference (§7): loading, live, failed or ended; and `ref.state`, the native watchable that follows // it. -import type { Context } from "../../context/index.ts"; -import { NativeWatch, type NativeWatchable } from "../../watch.ts"; +import { BACKGROUND_CONTEXT, type Context } from "../../context/index.ts"; +import { NativeWatch } from "../../watch.ts"; import type { ChordError, FailureCause } from "../data/index.ts"; +import { type Batch, NativeSource } from "../state/internal.ts"; /** * Where a reference stands. `loading`: its object activates or retries, it waits for `load` or a parent link, or its @@ -29,27 +30,21 @@ export const LOADING: RefState = { kind: "loading" }; export const LIVE: RefState = { kind: "live" }; -const IGNORE = (): void => {}; - /** - * `ref.state`: its value is the reference's state, read through `read`; each watch gets every new state, in its own - * context, and ends `ended` when the reference is released (the end is the notification). + * `ref.state`, a native source: its value is the reference's state, read through `read`; each watch gets every new + * state, in its own context (not one call per batch, as a followed source's), and ends `ended` when the reference is + * released (the end is the notification). Derives reading it are marked in a batch of its own when it changes. */ -export class ReferenceState implements NativeWatchable { +export class ReferenceState extends NativeSource { readonly #read: () => RefState; - /** Where a listener's throw goes (the host); dropped on release, so a released reference keeps no host. */ - #report: (error: unknown) => void; - /** Throws `invalid/reentrant` inside the host's batch (a watch taken in a derive compute); dropped on release. */ - #assertOutside: () => void; /** The watches with the contexts they were taken in, which their frames carry. */ readonly #watches = new Map, Context>(); /** The state the watches have: a change is announced only when the state is another object. */ #last: RefState; - constructor(read: () => RefState, report: (error: unknown) => void, assertOutside: () => void) { + constructor(read: () => RefState, batch: Batch | undefined) { + super(batch); this.#read = read; - this.#report = report; - this.#assertOutside = assertOutside; this.#last = read(); } @@ -57,35 +52,57 @@ export class ReferenceState implements NativeWatchable { return this.#read(); } + /** Every state in order: a watch sees each step it went through. */ async watch(context: Context): Promise> { - this.#assertOutside(); const state = this.#read(); - const watch: NativeWatch = new NativeWatch(state, () => this.#watches.delete(watch), this.#report); - // An ended reference has nothing more to announce. - if (state.kind === "ended") { - watch.end({ kind: "ended" }); - return watch; + const batch = this.batch; + // An ended reference has nothing more to announce, and keeps no host to report to. + if (batch === undefined || state.kind === "ended") { + const ended = new NativeWatch(state, IGNORE, IGNORE); + ended.end({ kind: "ended" }); + return ended; } + batch.assertOutside("watch"); + const watch = new NativeWatch(state, () => this.#watches.delete(watch), batch.reportListener); this.#watches.set(watch, context); if (context.abortSignal !== undefined) watch.observeAbort(context.abortSignal); return watch; } - /** The reference's state may have changed: every watch gets it when it is another state. */ - changed(): void { + /** The reference's state may have changed: every watch gets it when it is another state, derives are marked. */ + moved(): void { const state = this.#read(); if (state === this.#last) return; this.#last = state; for (const [watch, context] of this.#watches) watch.advance(state, context); + if (this.dependents.size > 0) this.#mark(); } - /** The reference ended (`ended`, the reference's state): every watch ends `ended`. */ - end(ended: Ended): void { + /** The reference ended (`ended`, the reference's state): every watch ends `ended`, derives are marked a last time. */ + released(ended: Ended): void { // An ended reference keeps nothing of its object: not the last failure's error, not the host. this.#last = ended; - this.#report = IGNORE; - this.#assertOutside = IGNORE; // Ending detaches, which deletes the current entry (safe while iterating) and runs no listener. for (const watch of this.#watches.keys()) watch.end({ kind: "ended" }); + const batch = this.batch; + if (batch === undefined || this.dependents.size === 0) { + this.end(); + return; + } + // Ended where the marks run: a release while a batch's flushes send marks in the next commit, after this. + batch.within(() => { + this.changed(); + this.end(); + }, BACKGROUND_CONTEXT); + } + + /** + * Marks the derives reading it, in the batch running now or one of its own (apart from `moved`: a closure there + * would allocate on every state change). A reference has no context of its own: the background context. + */ + #mark(): void { + this.batch?.within(() => this.changed(), BACKGROUND_CONTEXT); } } + +const IGNORE = (): void => {}; diff --git a/packages/chord/src/next/host/reference.ts b/packages/chord/src/next/host/reference.ts index 4b9ed1ff8..65da7ab33 100644 --- a/packages/chord/src/next/host/reference.ts +++ b/packages/chord/src/next/host/reference.ts @@ -136,8 +136,6 @@ const WAIT: Decision = { kind: "wait" }; const DONE: Decision = { kind: "done" }; -const IGNORE = (): void => {}; - /** A wait for a state: checked at every change, rejected with the reference's end. */ type Waiter = { readonly check: () => void; readonly end: (error: ChordError) => void }; @@ -257,7 +255,7 @@ export class Binding implements Holder { * rejects them with the cause; waits for a state decide. */ moved(): void { - this.#refState?.changed(); + this.#refState?.moved(); const state = this.refState; if (state.kind === "live") { this.#parked?.flush( @@ -430,7 +428,7 @@ export class Binding implements Holder { this.#parked?.reject(error); this.#parked = undefined; for (const watch of this.#watches) watch.end({ kind: "ended" }); - this.#refState?.end(ended); + this.#refState?.released(ended); // Each removes itself; promise reactions run later. for (const waiter of this.#waiters) waiter.end(error); this.#ended?.resolve(); @@ -495,13 +493,14 @@ export class Binding implements Holder { #watchableState(): ReferenceState { if (this.#refState !== undefined) return this.#refState; const state = this.#state; - // A released reference's state never changes: its watches end at once, with no listener to report. Bound, not a - // closure: `read` would share the closure's scope and keep the batch after release. - const [report, assertOutside] = - state.kind === "released" - ? [IGNORE, IGNORE] - : [state.batch.reportListener, state.batch.assertOutside.bind(state.batch, "watch")]; - this.#refState = new ReferenceState(() => this.refState, report, assertOutside); + // A released reference's state never changes: it is made ended, keeping no batch. + if (state.kind === "released") { + const made = new ReferenceState(() => this.refState, undefined); + made.released(state.state); + this.#refState = made; + return made; + } + this.#refState = new ReferenceState(() => this.refState, state.batch); return this.#refState; } diff --git a/packages/chord/src/next/host/retry-on.ts b/packages/chord/src/next/host/retry-on.ts new file mode 100644 index 000000000..84f997a11 --- /dev/null +++ b/packages/chord/src/next/host/retry-on.ts @@ -0,0 +1,55 @@ +// Wake signals (slice 8 §3, D2): `self.retryOn(...sources)`. The object follows each declared source while it lives, +// replaced by each attempt's declaration; a new value of a source wakes it (the object decides what a wake does: a +// retry while failed, remembered while it activates, nothing while live). Failures of a source wake nothing. + +import { RegistryValueSource } from "../registries/internal.ts"; +import { Constant, type NativeObserver } from "../state/internal.ts"; + +/** What `retryOn` takes: sources that live independently of the attempt (a registry value; a constant never wakes). */ +export type RetrySource = RegistryValueSource | Constant; + +/** + * Checks a `retryOn` source (declaration policy: TypeScript cannot tell a reference's member or this attempt's + * `writable` from a registry value or a constant). + */ +export function retrySource(source: unknown, label: string): RetrySource { + if (source instanceof RegistryValueSource || source instanceof Constant) return source; + throw new Error(`${label}: self.retryOn takes registry values and constants (sources that outlive the attempt)`); +} + +/** + * The wake signals of one object: its subscriptions (one set at a time, the last attempt's declaration) and what its + * object decides a wake by. + */ +export class RetryWakes { + /** The attempt a wake arrived during, while it activates: if it fails, it is retried at once. */ + during: object | undefined; + /** + * Set from its object's contributions leaving until that batch delivered: a composition its own leaving made does + * not wake it (an object failing live would retry itself without backoff). + */ + leaving = false; + #subscribed: { readonly source: RegistryValueSource; readonly observer: NativeObserver }[] = []; + + /** Follows `sources` instead of the previous set; `wake` runs inside the batch of each new value (it only records). */ + replace(sources: readonly RetrySource[], wake: () => void): void { + this.stop(); + const observer: NativeObserver = { + advance: (call) => { + if (call.kind === "value") wake(); + }, + end: () => {}, + }; + for (const source of sources) { + if (!(source instanceof RegistryValueSource)) continue; + source.subscribe(observer); + this.#subscribed.push({ source, observer }); + } + } + + /** Stops following (the object ended, or a new declaration replaces the set). */ + stop(): void { + for (const { source, observer } of this.#subscribed) source.unsubscribe(observer); + this.#subscribed = []; + } +} diff --git a/packages/chord/src/next/host/root.ts b/packages/chord/src/next/host/root.ts new file mode 100644 index 000000000..3ebdf3d4f --- /dev/null +++ b/packages/chord/src/next/host/root.ts @@ -0,0 +1,76 @@ +// Root scopes (slice 8 §4, D3, design §8): `host.root(fn, context, options?)`, a call context for code outside objects +// (a Durable tool, a CLI), sealed when `fn` settles, when `context` aborts and when the host closes: its signal aborted, +// its `ctx.get` refused `released`, every reference it took released. + +import { awaitWithContext, type Context } from "../../context/index.ts"; +import { ChordError, type Principal } from "../data/index.ts"; +import { CallContext, type Ctx, withLinkedAbortSignal } from "./ctx.ts"; +import type { AnyHost } from "./host.ts"; + +/** What `host.root` takes besides `fn` and the context. */ +export type RootOptions

= { + /** Whom the scope acts as: the host's principal by default; a trusted same-host caller's choice (base code). */ + readonly principal?: P; +}; + +/** An open root scope: its seal, and the rejection its caller gets when the host closes. */ +type OpenScope = { readonly seal: (reason: unknown) => void; readonly closed: PromiseWithResolvers }; + +/** The open root scopes of a host, each sealed once; `close` seals the rest and rejects their callers. */ +export class RootScopes { + readonly #host: AnyHost; + /** Each open scope: how it is sealed with a reason, and its caller's rejection when the host closes. */ + readonly #open = new Set(); + + constructor(host: AnyHost) { + this.#host = host; + } + + /** + * Runs `fn` with a call context like a handler's (`principal`, this host's incarnation, `context`'s telemetry, a + * signal linked to `context`'s), sealed when `fn` settles, `context` aborts or the host closes. Resolves as `fn` + * does; rejects at once when `context` aborts or the host closes (an uncooperative `fn` runs on and fails at its + * next use of Chord), or when `context` was aborted already (`fn` does not run). + */ + run(fn: (ctx: Ctx) => Promise, principal: Principal | undefined, context: Context): Promise { + context.abortSignal?.throwIfAborted(); + const host = this.#host; + const controller = new AbortController(); + const linked = withLinkedAbortSignal(controller.signal, context); + const scope = linked.context; + const ctx = new CallContext(scope, { principal, incarnation: host.incarnation }, host); + const open: OpenScope = { + seal: (reason) => { + if (!this.#open.delete(open)) return; + scope.abortSignal?.removeEventListener("abort", aborted); + // References first: a `get` still waiting gives up with `released`, not with the abort's reason. + ctx.settle(); + // While still linked, so the scope's own signal (a caller's context has one) aborts too. + controller.abort(reason); + linked.release(); + }, + closed: Promise.withResolvers(), + }; + const aborted = (): void => open.seal(scope.abortSignal?.reason); + this.#open.add(open); + scope.abortSignal?.addEventListener("abort", aborted, { once: true }); + // Plugin code: a synchronous throw becomes the rejection. + const running = (async () => fn(ctx))(); + const settled = (): void => open.seal(SETTLED); + running.then(settled, settled); + // A method's caller gets `released` when its object ends, whatever the handler still does: so does this one. One + // wait on the caller's signal around both, so the close removes its listener too. + return awaitWithContext(Promise.race([running, open.closed.promise]), context); + } + + /** The host closes: every open scope is sealed with `reason`, its caller rejected with it. */ + close(reason: unknown): void { + for (const open of [...this.#open]) { + open.seal(reason); + open.closed.reject(reason); + } + } +} + +/** The abort reason of a scope whose `fn` settled. */ +const SETTLED = new ChordError({ code: "released", reason: "settled", message: "host.root: its fn settled" }); diff --git a/packages/chord/src/next/host/routines.ts b/packages/chord/src/next/host/routines.ts index ea924ec7b..837551b9e 100644 --- a/packages/chord/src/next/host/routines.ts +++ b/packages/chord/src/next/host/routines.ts @@ -14,7 +14,7 @@ import { type WatchEnd, } from "../../watch.ts"; import type { Principal } from "../data/index.ts"; -import { Source } from "../state/internal.ts"; +import { NativeSource, Source } from "../state/internal.ts"; import { nextDelay } from "./backoff.ts"; import { CallContext, type Caller, type Ctx } from "./ctx.ts"; import type { AnyHost, Position } from "./host.ts"; @@ -94,6 +94,9 @@ type Called = { readonly kind: "done" } | { readonly kind: "failed"; readonly er const DONE: Called = { kind: "done" }; +/** What `#failing` says of a source failed now. */ +const FAILING = { kind: "failed" } as const; + /** One run of an `onLive` that lasted this long resets its backoff. */ const LONG_RUN = 60_000; @@ -267,10 +270,14 @@ class OnRoutine { } /** - * Whether the source is a Chord source failed now (`failed`), or one with no value yet (`no_value`, with the source to - * follow); `undefined` otherwise (natives and Durable's views tell their failures through their watches). + * Whether the source is a Chord source failed now (`failed`), or a JSON one with no value yet (`no_value`, with the + * source to follow); `undefined` otherwise (Durable's views tell their failures through their watches). */ #failing(): { readonly kind: "failed" } | { readonly kind: "no_value"; readonly source: Source } | undefined { + const declared = this.#declaration.source; + // A native source opens its watch at its last good value while failed (a registry whose `compose` threw): its + // first call is the failure, not that stale value. + if (declared instanceof NativeSource) return declared.start().kind === "failed" ? FAILING : undefined; let source: unknown; try { source = resolveWatchable(this.#declaration.source); diff --git a/packages/chord/src/next/index.ts b/packages/chord/src/next/index.ts index 8e759f345..1055a2cf6 100644 --- a/packages/chord/src/next/index.ts +++ b/packages/chord/src/next/index.ts @@ -7,6 +7,7 @@ export * from "./documents/index.ts"; export * from "./host/index.ts"; export * from "./jobs/index.ts"; export * from "./link/index.ts"; +export * from "./registries/index.ts"; export * from "./state/index.ts"; /** @internal The store's surface for adapters (slice 6b §2.1). */ export { type DecodedRow, OwnerTable, StoreOwnerTables } from "./store/adapter.ts"; diff --git a/packages/chord/src/next/registries/README.md b/packages/chord/src/next/registries/README.md new file mode 100644 index 000000000..070c98877 --- /dev/null +++ b/packages/chord/src/next/registries/README.md @@ -0,0 +1,81 @@ +# registries + +Registries (design §17, slice 8): an extension point between plugins. The defining plugin shares a token and serves +it with `compose`; facets and host-lifetime objects contribute; readers follow the composed value. + +## For apps (`index.ts`) + +```ts +import { defineRegistry, keyedEntries } from "./index.ts"; + +// Shared by every plugin, as a contract: identity and types only. +export const Http = defineRegistry("chord.http", { version: 2 }); + +defineFacet((host, self) => { + host.serve(Http, { // the defining plugin: the one provider + compose: (table, { report }) => routerOf(keyedEntries(table, (c) => c.routes, report)), + accepts: { 1: { contribution: (old: HttpV1) => upgrade(old), value: (router) => routerV1(router) } }, + }); + self.contribute(Http, { routes: { "GET /notes": exportNotes } }); // while this plugin is loaded +}); +host.serve(Mcp, (_, self) => { // local, empty params: may contribute (R21) + const ext = self.writable(mcpExtension([])); + self.contribute(Extensions, ext); // from live until it fails, ends or restarts + self.retryOn(self.registry(Models)); // a new model wakes it while failed (rare changes) + return {}; +}); +host.serve(Server, (_, self) => { + const router = self.registry(Http); // a dependency; `.value` at use + return { request: async (req, ctx) => router.value.route(req, ctx) }; +}); +``` + +- `defineRegistry(name, { version })`: a token. `name` follows the type names' rule (`[a-z0-9@/._~-]`), + `version` is the version this plugin is built against. `C` is what contributors give, `V` what readers get. +- `host.serve(R, { compose, accepts? })` in the defining plugin's facet (`host/`). `compose(table, { report })` is pure + and synchronous and always returns a value (a fallback built in); `report({ message, plugins })` reports a conflict + it resolved, as `registry_conflict`, once per episode (a conflict reported by the last good composition is not + reported again). It runs at the end of `load`'s setups (every served registry once, an empty table too), then lazily: + a change of the table marks it, and it composes when read or, while someone watches, observes or waits for it, in + the batch that changed the table (after the commit's adoptions); an unchanged table (every entry the same) composes + nothing. A `compose` that throws is the owner's bug: `registry_failed` once per episode, watches get one `failed` + call, `.value` keeps the last good composition (before any, it throws the error); the next good composition is a + value call, also when its value is the same. +- The table (`RegistryTable`): one `RegistryEntry { plugin, type?, version, value }` per live contributor, sorted by + plugin name, then type (a facet's entry first): never load order. A contribution of an older version is adapted with + `accepts[version].contribution` once per contribution value; a newer one, or an older one without an adapter, is left + out and reported `contribution_refused` once. A Chord watchable contribution (a writable, a derive, a document, a + reference member, another registry's value) is followed: its change marks the table; one that fails or ends (a + registry whose `compose` threw too, though its readers keep its last good value), or whose adapter throws, is left + out and reported `contribution_failed` once per episode, and is back when it heals. Anything else is a plain value, + whatever methods it has (`undefined` included). +- Readers: `self.registry(R)` (a declared dependency, `host/`) and `await host.registry(R, ctx)` give a + `RegistryValue`: a native watchable (`.value`, `watch`) with `observe(observer, ctx)`, a synchronous observer of + its compositions (Durable's shape, for `registryReader`). A reader whose token is older than the provider's gets the + composition mapped back with `accepts[version].value`, once per composition, only while read or followed; a mapping + that throws fails that version only (`registry_failed`, its last good mapping kept). A reader without a mapping, or + newer than the provider, gets `contract_mismatch` (its object fails `broken`: only a reload helps). The mapping is + kept only while a reader or follower is there. Without a provider a registry has no value + (`unavailable/undeclared`); its contributions wait in the table. +- `keyedEntries(table, entriesOf, report)`: a map of every entry's keyed items, the first by plugin name keeping a key, + a duplicate reported through `compose`'s `report`; `orderedEntries(table, itemsOf, orderOf?)`: every entry's items in + table order, or by `orderOf`. Both keep each item's `plugin` (and `type`): a contribution is another plugin's code, so + a use site catches its throws and rejections, reports the plugin, and goes on. +- Registries carry natives: an entry lives exactly as long as its contributor. A last good composition kept after a + throwing `compose` may still name a contributor that left: use sites look up at each use and keep nothing. + +## For Chord's layers (`internal.ts`) + +- `RegistryRecord`: one per registry and host (`ChordHost.registryRecord(name)`, made by its first use): the provider + (`serve`), the contributions in table order (`add`, `remove`, inside a batch), the last table and composition + (`start()`: composed now when dirty; a read while composing is a cycle, `invalid/cycle`), the episode reports, the + direct edges between registries whose values are contributions in its table (counted, kept while those + contributions are in it; `assertNoCycle` refuses a cycle at `contribute`, through those edges and the + contributing attempt's own earlier declarations), and its values by reader version + (`value(version)`); `close()` at the host's close. +- `RegistryContribution` (`contribution.ts`): one contributor's contribution, a `Dependent` of its source; + `entry(version, provider)` is its table entry (the same object while its value is), or `undefined` when left out. +- `RegistryValueSource`: a `FollowedSource` per reader version; `read(waiter)` registers an activation's object + (`RegistryWaiter`), woken with `dependencyLive()` (on a microtask) when the value gets its first good value; readers + waiting for one count as followers, so a change composes for them. At the host's close it keeps its last start and + lets the record (and the host) go. diff --git a/packages/chord/src/next/registries/contribution.ts b/packages/chord/src/next/registries/contribution.ts new file mode 100644 index 000000000..7572a40a7 --- /dev/null +++ b/packages/chord/src/next/registries/contribution.ts @@ -0,0 +1,158 @@ +// One contributor's contribution to a registry (slice 8 §1): its table entry, made once per value (an older version's +// adapted once per value), followed when it is a Chord watchable, left out and reported when it fails or is refused. + +import { type Dependent, type DeriveSource, handedOut, type NativeSource, Source } from "../state/internal.ts"; +import type { RegistryRecord } from "./record.ts"; +import type { RegistryEntry, RegistryProvider } from "./registry.ts"; +import { RegistryValueSource } from "./value.ts"; + +/** Who contributes: a facet (by plugin) or a host-lifetime object (by plugin and type). */ +export type Contributor = { readonly plugin: string; readonly type: string | undefined }; + +/** + * What a contribution is: a plain value, a Chord JSON watchable (a source or a constant), or a native source; the + * watchables followed for their value. + */ +export type ContributionContent = + | { readonly kind: "plain"; readonly value: unknown } + | { readonly kind: "json"; readonly source: DeriveSource } + | { readonly kind: "native"; readonly source: NativeSource }; + +/** + * One contributor's contribution to a registry: a plain value, or a Chord source it follows as a dependent (a change + * marks the registry's table). Its table entry is made once per value (an older version's adapted once per value); + * a failure or refusal leaves it out, reported once per episode. + */ +export class RegistryContribution implements Dependent { + /** The registry it contributes to. */ + readonly record: RegistryRecord; + readonly contributor: Contributor; + /** The registry version its contributor was built against. */ + readonly version: number; + readonly #content: ContributionContent; + /** The last adaptation of an older contribution: the value it was made from and its result. */ + #adapted: { readonly from: unknown; readonly value: unknown } | undefined; + #entry: RegistryEntry | undefined; + /** Left out since its last entry: failing (reported once), or refused (reported once). */ + #out: "none" | "failed" | "refused" = "none"; + + constructor(record: RegistryRecord, contributor: Contributor, version: number, content: ContributionContent) { + this.record = record; + this.contributor = contributor; + this.version = version; + this.#content = content; + } + + /** Its source changed (inside a batch): the registry's table is marked. */ + invalidate(): void { + this.record.contributionChanged(); + } + + /** A retirement of its source's document changes nothing beyond the value. */ + sourceRetired(): void {} + + /** Starts following its source, if it has one. */ + follow(): void { + const source = this.#followed(); + source?.dependents.add(this); + } + + /** Stops following its source. */ + unfollow(): void { + this.#followed()?.dependents.delete(this); + } + + /** The registry whose value it is, when it is one (its edge in the cycle check, while it is in the table). */ + get reads(): RegistryRecord | undefined { + const content = this.#content; + return content.kind === "native" && content.source instanceof RegistryValueSource + ? content.source.record + : undefined; + } + + /** Its source when it changes (a constant never does). */ + #followed(): Source | NativeSource | undefined { + const content = this.#content; + if (content.kind === "native") return content.source; + return content.kind === "json" && content.source instanceof Source ? content.source : undefined; + } + + /** + * Its entry in a table of the provider at `version`: the same object while its value is the same; `undefined` when + * left out (a refused version, a failing source, a throwing adapter). + */ + entry(version: number, provider: RegistryProvider): RegistryEntry | undefined { + const adapter = this.version < version ? provider.accepts?.[this.version]?.contribution : undefined; + if (this.version > version || (this.version < version && adapter === undefined)) { + if (this.#out !== "refused") this.#refused(); + return undefined; + } + let value: unknown; + try { + const raw = this.#raw(); + value = adapter === undefined ? raw : this.#adapt(raw, adapter); + } catch (error) { + if (this.#out !== "failed") this.#failed(error); + return undefined; + } + this.#out = "none"; + const entry = this.#entry; + // `undefined` is a value: the entry, not its field, says whether there was one. + if (entry !== undefined && entry.value === value) return entry; + const { plugin, type } = this.contributor; + this.#entry = { plugin, ...(type === undefined ? {} : { type }), version: this.version, value }; + return this.#entry; + } + + /** + * The contributed value now: a JSON source's value throws while it is failed or ended; a native source that failed + * (a registry whose `compose` threw) keeps a last good value for its readers, but is left out here as failed. + */ + #raw(): unknown { + const content = this.#content; + switch (content.kind) { + case "plain": + return content.value; + case "json": + return content.source.value; + case "native": + return content.source.current(); + } + } + + #adapt(raw: unknown, adapter: (old: never) => unknown): unknown { + const adapted = this.#adapted; + if (adapted !== undefined && adapted.from === raw) return adapted.value; + const value = this.record.adapt(() => adapter(raw as never)); + this.#adapted = { from: raw, value }; + return value; + } + + #refused(): void { + this.#out = "refused"; + const { plugin, type } = this.contributor; + const registry = this.record.name; + const where = type === undefined ? {} : { type }; + this.record.reportContribution({ + kind: "contribution_refused", + registry, + plugin, + ...where, + version: this.version, + }); + } + + #failed(error: unknown): void { + this.#out = "failed"; + const { plugin, type } = this.contributor; + const registry = this.record.name; + const where = type === undefined ? {} : { type }; + this.record.reportContribution({ + kind: "contribution_failed", + registry, + plugin, + ...where, + error: handedOut(error), + }); + } +} diff --git a/packages/chord/src/next/registries/entries.ts b/packages/chord/src/next/registries/entries.ts new file mode 100644 index 000000000..069363ec8 --- /dev/null +++ b/packages/chord/src/next/registries/entries.ts @@ -0,0 +1,54 @@ +// Table helpers for `compose` (slice 8 §1, Q8): a map by key (the first by plugin name wins, duplicates reported) and +// an ordered list; both keep each item's plugin, so a use site reports the contributor of an entry that throws. + +import type { ComposeContext, RegistryTable } from "./registry.ts"; + +/** An item taken from a table, with the contributor it came from. */ +export type TableItem = { readonly value: I; readonly plugin: string; readonly type?: string }; + +/** + * A map of every entry's keyed items (`entriesOf`, e.g. its routes), in table order: a key taken already is reported + * through `report` (`compose`'s) naming both plugins, and the first by plugin name keeps it. + */ +export function keyedEntries( + table: RegistryTable, + entriesOf: (value: C) => Readonly> | undefined, + report: ComposeContext["report"], +): ReadonlyMap> { + const map = new Map>(); + for (const entry of table) { + const items = entriesOf(entry.value); + if (items === undefined) continue; + for (const [key, value] of Object.entries(items)) { + const taken = map.get(key); + if (taken !== undefined) { + report({ message: `"${key}" is contributed twice`, plugins: [taken.plugin, entry.plugin] }); + continue; + } + map.set(key, item(value, entry.plugin, entry.type)); + } + } + return map; +} + +/** + * Every entry's items (`itemsOf`) in one list: in table order, or sorted by `orderOf` (a phase's index, for example), + * table order kept among equals. + */ +export function orderedEntries( + table: RegistryTable, + itemsOf: (value: C) => readonly I[] | undefined, + orderOf?: (item: I) => number, +): readonly TableItem[] { + const items: TableItem[] = []; + for (const entry of table) { + for (const value of itemsOf(entry.value) ?? []) items.push(item(value, entry.plugin, entry.type)); + } + // `sort` is stable: equal orders keep table order. + if (orderOf !== undefined) items.sort((a, b) => orderOf(a.value) - orderOf(b.value)); + return items; +} + +function item(value: I, plugin: string, type: string | undefined): TableItem { + return type === undefined ? { value, plugin } : { value, plugin, type }; +} diff --git a/packages/chord/src/next/registries/index.ts b/packages/chord/src/next/registries/index.ts new file mode 100644 index 000000000..8c9d8154c --- /dev/null +++ b/packages/chord/src/next/registries/index.ts @@ -0,0 +1,16 @@ +// Registries for apps: tokens, what a provider serves them with, the table `compose` gets and its helpers, and the +// composed value readers follow. + +export { keyedEntries, orderedEntries, type TableItem } from "./entries.ts"; +export { + type ComposeContext, + type Contribution, + defineRegistry, + type Registry, + type RegistryAccepts, + type RegistryConflict, + type RegistryEntry, + type RegistryProvider, + type RegistryTable, + type RegistryValue, +} from "./registry.ts"; diff --git a/packages/chord/src/next/registries/internal.ts b/packages/chord/src/next/registries/internal.ts new file mode 100644 index 000000000..1a6969089 --- /dev/null +++ b/packages/chord/src/next/registries/internal.ts @@ -0,0 +1,5 @@ +// Registries for Chord's layers: a host's record of each registry, its contributions and its values by version. + +export { type ContributionContent, type Contributor, RegistryContribution } from "./contribution.ts"; +export { type RegistryEdge, type RegistryError, RegistryRecord } from "./record.ts"; +export { RegistryValueSource, type RegistryWaiter } from "./value.ts"; diff --git a/packages/chord/src/next/registries/record.ts b/packages/chord/src/next/registries/record.ts new file mode 100644 index 000000000..f879132f9 --- /dev/null +++ b/packages/chord/src/next/registries/record.ts @@ -0,0 +1,301 @@ +// A registry on a host (slice 8 §1): its provider, its contributions in table order (`contribution.ts`), the table +// and its lazy composition (marked when the table changes, composed when read or followed, in the batch's +// deliveries), the reports of conflicts and failures (once per episode), and the values its readers follow by +// version. + +import { ChordError } from "../data/index.ts"; +import { invalid } from "../data/internal.ts"; +import { type Batch, handedOut, type NativeStart } from "../state/internal.ts"; +import type { RegistryContribution } from "./contribution.ts"; +import type { ComposeContext, RegistryConflict, RegistryEntry, RegistryProvider, RegistryTable } from "./registry.ts"; +import { RegistryValueSource } from "./value.ts"; + +/** What a registry reports to its host's `onError`, by kind; each once per episode. */ +export type RegistryError = + /** `compose` reported a conflict it resolved. */ + | { + readonly kind: "registry_conflict"; + readonly registry: string; + readonly message: string; + readonly plugins: readonly string[]; + } + /** `compose` (or an older version's `value` adapter) threw: the last good composition stays. */ + | { readonly kind: "registry_failed"; readonly registry: string; readonly error: unknown } + /** A watchable contribution failed or ended, or its `contribution` adapter threw: it is left out until it heals. */ + | { + readonly kind: "contribution_failed"; + readonly registry: string; + readonly plugin: string; + readonly type?: string; + readonly error: unknown; + } + /** A contribution of a version the provider does not accept (newer, or older without an adapter) is left out. */ + | { + readonly kind: "contribution_refused"; + readonly registry: string; + readonly plugin: string; + readonly type?: string; + readonly version: number; + }; + +/** One registry's value contributed to another: `from`'s table reads `to`. */ +export type RegistryEdge = { readonly from: RegistryRecord; readonly to: RegistryRecord }; + +/** The provider as served: the defining plugin's version, `compose` and adapters. */ +type Provider = { readonly version: number; readonly provider: RegistryProvider }; + +/** Where a composition stands: never composed, its last good value, and its failure while `compose` throws. */ +type Composition = { + readonly good: { readonly value: unknown } | undefined; + readonly failure: { readonly error: unknown } | undefined; +}; + +const NEVER: Composition = { good: undefined, failure: undefined }; + +/** One registry of a host. */ +export class RegistryRecord { + readonly name: string; + readonly #batch: Batch; + readonly #report: (error: RegistryError) => void; + #provider: Provider | undefined; + /** The contributions of live contributors, sorted by plugin, then type (a facet's first). */ + readonly #contributions: RegistryContribution[] = []; + /** The last table composed, kept so an unchanged table composes nothing. */ + #table: RegistryTable | undefined; + #dirty = true; + #composition: Composition = NEVER; + /** Set while its table is built and composed: a read then is a cycle of registries. */ + #composing = false; + /** The conflicts the last good composition reported, by key: a conflict is reported once per episode. */ + #conflicts = new Set(); + /** + * Registries whose values are contributions in this one's table, each with how many: a contribution that would + * close a cycle of them is refused. An edge lives as long as its contribution is in the table. + */ + readonly #reads = new Map(); + /** The values readers follow, by version; made by the first reader of a version. */ + readonly #values = new Map(); + + constructor(name: string, batch: Batch, report: (error: RegistryError) => void) { + this.name = name; + this.#batch = batch; + this.#report = report; + } + + /** The provider's version; `undefined` while none serves it. */ + get providedVersion(): number | undefined { + return this.#provider?.version; + } + + /** The value readers of `version` follow. */ + value(version: number): RegistryValueSource { + let value = this.#values.get(version); + if (value === undefined) { + value = new RegistryValueSource(this, version, this.#batch); + this.#values.set(version, value); + } + return value; + } + + /** Serves the registry with `provider` at `version` (the defining plugin's facet); a second serve is refused. */ + serve(version: number, provider: RegistryProvider): void { + if (this.#provider !== undefined) throw new Error(`registry ${this.name} is served twice`); + // Declaration policy: plugin code is bundled without type checks. + if (typeof provider?.compose !== "function") throw new Error(`registry ${this.name}: compose is a function`); + this.#provider = { version, provider }; + } + + /** The first composition, at the end of `load`'s setups, inside one batch for every served registry. */ + composeServed(): void { + if (this.#provider === undefined) return; + this.#invalidate(); + this.start(); + } + + /** + * Refuses a contribution that is `source`'s value when it would close a cycle (declaration policy, at + * `contribute`): through the registries in the tables now and the edges `pending` (the contributing attempt's + * earlier declarations, which enter the tables only once it is live). + */ + assertNoCycle(source: RegistryValueSource, pending: readonly RegistryEdge[]): void { + const { record } = source; + if (record === undefined) return; + if (record.#reaches(this, new Set(), pending)) { + throw invalid("cycle", `registries ${this.name} and ${record.name} read each other`); + } + } + + /** Adds a live contributor's contribution, inside a batch: the table changes. */ + add(contribution: RegistryContribution): void { + const at = this.#contributions.findIndex((other) => compareContributors(contribution, other) < 0); + this.#contributions.splice(at < 0 ? this.#contributions.length : at, 0, contribution); + const { reads } = contribution; + if (reads !== undefined) this.#reads.set(reads, (this.#reads.get(reads) ?? 0) + 1); + contribution.follow(); + this.#invalidate(); + } + + /** Removes an ending contributor's contribution, inside a batch: the table changes. */ + remove(contribution: RegistryContribution): void { + const at = this.#contributions.indexOf(contribution); + if (at < 0) return; + this.#contributions.splice(at, 1); + const { reads } = contribution; + if (reads !== undefined) { + const left = this.#reads.get(reads)! - 1; + if (left === 0) this.#reads.delete(reads); + else this.#reads.set(reads, left); + } + contribution.unfollow(); + this.#invalidate(); + } + + /** A followed contribution changed, inside a batch: the table is marked, composed when read or followed. */ + contributionChanged(): void { + this.#invalidate(); + } + + /** + * The current composition, composed now when the table changed: the value, or `compose`'s failure with the last + * good one; `unavailable/undeclared` while no provider serves it. A read while it composes is a cycle. + */ + start(): NativeStart { + if (this.#composing) throw invalid("cycle", `registry ${this.name} is read while it composes`); + if (this.#provider === undefined) { + const error = new ChordError({ + code: "unavailable", + reason: "undeclared", + message: `registry ${this.name}: no plugin serves it here`, + }); + return { kind: "failed", error, last: undefined }; + } + if (this.#dirty) this.#recompose(this.#provider); + const { good, failure } = this.#composition; + if (failure !== undefined) return { kind: "failed", error: failure.error, last: good }; + // A composition without a failure has a value. + return { kind: "value", value: good!.value }; + } + + /** The adapter mapping the composed value back for readers of `version`; `undefined` when there is none. */ + valueAdapter(version: number): ((current: never) => unknown) | undefined { + return this.#provider?.provider.accepts?.[version]?.value; + } + + /** Runs an adapter of the provider as a pure compute, counted. */ + adapt(adapter: () => T): T { + this.#batch.count("chord.registry.adapts"); + return this.#batch.compute(adapter); + } + + /** Reports a contribution left out (`contribution_failed`, `contribution_refused`; the caller keeps the episode). */ + reportContribution(error: RegistryError): void { + this.#report(error); + } + + /** Reports a throw of `compose` or of a value adapter (`registry_failed`; the caller keeps the episode). */ + failed(error: unknown): void { + this.#report({ kind: "registry_failed", registry: this.name, error: handedOut(error) }); + } + + /** + * The host closed: the contributions stop following their sources, the values end (a value kept by a reader still + * reads the last composition). + */ + close(): void { + for (const contribution of this.#contributions) contribution.unfollow(); + this.#contributions.length = 0; + this.#reads.clear(); + this.#table = undefined; + this.#dirty = false; + for (const value of this.#values.values()) value.close(); + this.#values.clear(); + } + + /** + * The table changed: every version's value is marked and listed (each lists itself once per batch); while it is + * not composed since an earlier change, only listed (its dependents are marked already). + */ + #invalidate(): void { + if (this.#dirty) { + for (const value of this.#values.values()) value.relisted(); + return; + } + this.#dirty = true; + for (const value of this.#values.values()) value.recomposed(); + } + + /** + * Builds the table and composes it, unless it is the last one composed, entry for entry (`compose` is pure). A + * throw is a failure episode: reported once, the last good composition kept; conflicts are reported once per + * episode, as their message and plugins. + */ + #recompose({ version, provider }: Provider): void { + this.#dirty = false; + this.#composing = true; + try { + const table = this.#build(version, provider); + if (table === this.#table && this.#composition !== NEVER) return; + this.#table = table; + const conflicts = new Map(); + const context: ComposeContext = { + report: (conflict) => + conflicts.set(`${conflict.message}\u0000${conflict.plugins.join("\u0000")}`, conflict), + }; + let value: unknown; + try { + this.#batch.count("chord.registry.composes"); + value = this.#batch.compute(() => provider.compose(table, context)); + } catch (error) { + if (this.#composition.failure === undefined) this.failed(error); + this.#composition = { good: this.#composition.good, failure: { error } }; + return; + } + this.#composition = { good: { value }, failure: undefined }; + for (const [key, conflict] of conflicts) { + if (this.#conflicts.has(key)) continue; + this.#report({ kind: "registry_conflict", registry: this.name, ...conflict }); + } + this.#conflicts = new Set(conflicts.keys()); + } finally { + this.#composing = false; + } + } + + /** The table of the live contributions now; the last one itself when every entry is the same object. */ + #build(version: number, provider: RegistryProvider): RegistryTable { + const previous = this.#table; + const table: RegistryEntry[] = []; + let same = previous !== undefined; + for (const contribution of this.#contributions) { + const entry = contribution.entry(version, provider); + if (entry === undefined) continue; + if (same && previous![table.length] !== entry) same = false; + table.push(entry); + } + return same && previous!.length === table.length ? previous! : table; + } + + /** Whether `target` is reached from here through the tables' edges and `pending`. */ + #reaches(target: RegistryRecord, visited: Set, pending: readonly RegistryEdge[]): boolean { + if (this === target) return true; + visited.add(this); + for (const next of this.#reads.keys()) { + if (!visited.has(next) && next.#reaches(target, visited, pending)) return true; + } + for (const { from, to } of pending) { + if (from === this && !visited.has(to) && to.#reaches(target, visited, pending)) return true; + } + return false; + } +} + +/** Table order: by plugin name, then type, a facet (no type) first; code point order, never load order. */ +function compareContributors(a: RegistryContribution, b: RegistryContribution): number { + const left = a.contributor; + const right = b.contributor; + if (left.plugin !== right.plugin) return left.plugin < right.plugin ? -1 : 1; + const leftType = left.type ?? ""; + const rightType = right.type ?? ""; + if (leftType === rightType) return 0; + return leftType < rightType ? -1 : 1; +} diff --git a/packages/chord/src/next/registries/registry.ts b/packages/chord/src/next/registries/registry.ts new file mode 100644 index 000000000..6b79109b8 --- /dev/null +++ b/packages/chord/src/next/registries/registry.ts @@ -0,0 +1,89 @@ +// Registry tokens (design §17, slice 8 D1): `defineRegistry`, shared by every plugin as a contract is (identity and +// types only), and what the defining plugin's facet serves it with (`compose`, `accepts`); the table `compose` gets and +// the composed value readers follow. + +import type { Context } from "../../context/index.ts"; +import type { NativeWatchable, Watchable } from "../../watch.ts"; +import { checkName } from "../contracts/names.ts"; +import type { NativeObservation, NativeObserver } from "../state/index.ts"; + +/** + * A registry token: its name (as type names, `[a-z0-9@/._~-]`), the version the bundling plugin was built against, the + * contribution type `C` and the composed value type `V`. Shared: every contributor and reader bundles its own copy. + */ +export type Registry = { + readonly kind: "registry"; + readonly name: string; + readonly version: number; + /** Type only: `C` in, `V` out. */ + readonly "~types"?: (contribution: C) => V; +}; + +/** One contribution in a registry's table: its plugin, its object type for an object contributor, its version. */ +export type RegistryEntry = { + readonly plugin: string; + /** The contributing object's type; absent for a facet. */ + readonly type?: string; + /** The registry version its contributor was built against (an older one adapted to `C`). */ + readonly version: number; + readonly value: C; +}; + +/** A registry's table: one entry per live contributor, sorted by plugin name, then type (a facet's first). */ +export type RegistryTable = readonly RegistryEntry[]; + +/** A conflict `compose` resolved with its fallback: reported as `registry_conflict`, once per conflict episode. */ +export type RegistryConflict = { readonly message: string; readonly plugins: readonly string[] }; + +/** What `compose` gets besides the table. */ +export type ComposeContext = { + /** Reports a conflict `compose` resolved (the first by plugin name wins, for example). */ + report(conflict: RegistryConflict): void; +}; + +/** + * Adapters of older versions, by version, the provider's code: `contribution` maps an older contribution to `C` (once + * per contribution value), `value` maps the composed value back for a reader of that version (once per composition). + * A plugin annotates the old contribution's type (`(old: HttpV1) => upgrade(old)`). + */ +export type RegistryAccepts = { + readonly [version: number]: { + readonly contribution?: (old: never) => C; + readonly value?: (current: V) => unknown; + }; +}; + +/** What the defining plugin's facet serves a registry with (`host.serve(R, provider)`). */ +export type RegistryProvider = { + /** + * Composes the table: pure and synchronous, always a value (a fallback built in), conflicts reported. Runs when the + * table changed and someone reads or follows the value; a throw is the owner's bug (`registry_failed`, the last + * good composition stays). + */ + readonly compose: (table: RegistryTable, context: ComposeContext) => V; + readonly accepts?: RegistryAccepts; +}; + +/** What a contributor contributes: a plain value, or a Chord watchable of one (followed: its change recomposes). */ +export type Contribution = C | Watchable | NativeWatchable; + +/** + * A registry's composed value as a reader of one version sees it (`self.registry`, `host.registry`): a native + * watchable; `.value` is the last good composition (a later throwing `compose` keeps it), a watch gets each new one. + */ +export interface RegistryValue extends NativeWatchable { + /** + * A synchronous observer of its compositions (as Durable's `observeXXX`): `advance` runs inside the batch that + * composed, so it only records; an attached observer counts as watching, so the composition runs for it. + */ + observe(observer: NativeObserver, context: Context): NativeObservation; +} + +/** Declares a registry token: `name` and the `version` this plugin is built against. */ +export function defineRegistry(name: string, options: { readonly version: number }): Registry { + checkName(name, "registry"); + const { version } = options; + // Declaration policy: JavaScript callers reach this check. + if (!Number.isSafeInteger(version) || version < 1) throw new Error(`${name}.version: expected a positive integer`); + return { kind: "registry", name, version }; +} diff --git a/packages/chord/src/next/registries/value.ts b/packages/chord/src/next/registries/value.ts new file mode 100644 index 000000000..7b3b8e1db --- /dev/null +++ b/packages/chord/src/next/registries/value.ts @@ -0,0 +1,177 @@ +// Registry values (slice 8 §1 "Reading", S2, S3): a registry's composed value as the readers of one version see it, a +// native source. The provider's version reads the composition itself; an older one reads it mapped back with +// `accepts[old].value`, once per composition, kept only while someone reads or follows it; a version the provider +// cannot serve fails `contract_mismatch`. Readers declared by attempts are woken (R18) when the value gets its first +// good value. + +import { ChordError } from "../data/index.ts"; +import { type Batch, FollowedSource, type NativeStart } from "../state/internal.ts"; +import type { RegistryRecord } from "./record.ts"; + +/** An object waiting for a registry's value (`self.registry`): retried at once when its first value arrives (R18). */ +export interface RegistryWaiter { + dependencyLive(): void; +} + +/** The last good mapping for an older version, and the composition it was made from. */ +type Mapped = { readonly from: unknown; readonly value: unknown }; + +/** A failed mapping of composition `from`, until another composition maps. */ +type MapFailure = { readonly from: unknown; readonly error: unknown }; + +/** + * Where a value's registry stands for it: on the host, or closed with the host (its last start kept, the record let + * go, so a value an app keeps keeps no closed host). + */ +type Registered = + | { readonly kind: "open"; readonly record: RegistryRecord } + | { readonly kind: "closed"; readonly last: NativeStart }; + +/** One version's view of a registry's composed value. */ +export class RegistryValueSource extends FollowedSource { + readonly name: string; + readonly version: number; + #registered: Registered; + /** Readers declared by attempts, each a token of its own (one object may read twice), until released. */ + readonly #readers = new Set<{ readonly waiter: RegistryWaiter }>(); + /** Whether it had a good value: the first one wakes the readers. */ + #valued = false; + #mapped: Mapped | undefined; + #mapFailure: MapFailure | undefined; + + constructor(record: RegistryRecord, version: number, batch: Batch) { + super(batch); + this.name = record.name; + this.version = version; + this.#registered = { kind: "open", record }; + } + + /** Its registry's record while the host is open. */ + get record(): RegistryRecord | undefined { + const registered = this.#registered; + return registered.kind === "open" ? registered.record : undefined; + } + + /** The composition (or its mapping) at its last good value; throws the failure when there is none. */ + get value(): unknown { + const start = this.start(); + if (start.kind === "value") return start.value; + if (start.last === undefined) throw start.error; + return start.last.value; + } + + /** Its composition (mapped for an older version), composed now if its table changed; a first value wakes readers. */ + override start(): NativeStart { + const registered = this.#registered; + if (registered.kind === "closed") return registered.last; + const start = this.#start(registered.record); + if (!this.#valued && (start.kind === "value" || start.last !== undefined)) { + this.#valued = true; + // Not inside the batch that composed: a retry runs a factory, which may not run in a batch. + for (const { waiter } of this.#readers) queueMicrotask(() => waiter.dependencyLive()); + } + return start; + } + + /** Its composition (or mapping) now; throws while it is failed, though `.value` keeps the last good one. */ + override current(): unknown { + const start = this.start(); + if (start.kind === "failed") throw start.error; + return start.value; + } + + /** Followed while someone watches or observes it, and while readers wait for its first value. */ + override get watched(): boolean { + return super.watched || (this.#readers.size > 0 && !this.#valued); + } + + /** Registers `waiter` (an activation's declared dependency) until the returned release. */ + read(waiter: RegistryWaiter): () => void { + const token = { waiter }; + this.#readers.add(token); + return () => { + this.#readers.delete(token); + if (this.#readers.size === 0 && !super.watched) this.#dropMapping(); + }; + } + + /** The registry's table changed (inside a batch): its dependents are marked, its followers get the next frame. */ + recomposed(): void { + this.changed(); + } + + /** + * The registry's table changed again while not composed since (inside a batch): its dependents are marked already + * (none read it meanwhile, or it would be composed); its followers get the next frame. Marking stops here, so a + * cycle of registries through their values ends. + */ + relisted(): void { + this.list(); + } + + /** The host closed: its watches and observers end, its readers and mapping go; it keeps its last start only. */ + close(): void { + let last: NativeStart; + try { + last = this.start(); + } catch (error) { + last = { kind: "failed", error, last: undefined }; + } + this.#registered = { kind: "closed", last }; + this.#readers.clear(); + this.#dropMapping(); + this.end(); + } + + /** Nobody follows it: without readers, its mapping goes (computed again on the next read). */ + protected override unfollowed(): void { + if (this.#readers.size === 0) this.#dropMapping(); + } + + #dropMapping(): void { + this.#mapped = undefined; + this.#mapFailure = undefined; + } + + #start(record: RegistryRecord): NativeStart { + const provided = record.providedVersion; + if (provided === undefined || provided === this.version) return record.start(); + const map = provided > this.version ? record.valueAdapter(this.version) : undefined; + if (map === undefined) { + const message = `registry ${this.name}: version ${this.version} is not served here (version ${provided} is)`; + return { kind: "failed", error: new ChordError({ code: "contract_mismatch", message }), last: undefined }; + } + return this.#mappedStart(record, record.start(), map); + } + + /** + * The composition `current` mapped back with `map`, once per composition; a throwing mapping fails this version + * only (reported once per episode), keeping its last good mapping; a composition mapped before heals it. + */ + #mappedStart( + record: RegistryRecord, + current: NativeStart, + map: (current: never) => unknown, + ): NativeStart { + if (current.kind === "failed" && current.last === undefined) return current; + const from = current.kind === "value" ? current.value : current.last!.value; + // `undefined` is a composition too: the records, not their fields, say whether there is one. + const mapped = this.#mapped; + const failure = this.#mapFailure; + if (mapped !== undefined && mapped.from === from) { + this.#mapFailure = undefined; + } else if (failure === undefined || failure.from !== from) { + try { + this.#mapped = { from, value: record.adapt(() => map(from as never)) }; + this.#mapFailure = undefined; + } catch (error) { + if (failure === undefined) record.failed(error); + this.#mapFailure = { from, error }; + } + } + const last = this.#mapped === undefined ? undefined : { value: this.#mapped.value }; + if (this.#mapFailure !== undefined) return { kind: "failed", error: this.#mapFailure.error, last }; + if (current.kind === "failed") return { kind: "failed", error: current.error, last }; + return { kind: "value", value: last!.value }; + } +} diff --git a/packages/chord/src/next/state/README.md b/packages/chord/src/next/state/README.md index b386d167b..c10600a71 100644 --- a/packages/chord/src/next/state/README.md +++ b/packages/chord/src/next/state/README.md @@ -1,6 +1,7 @@ # state -Watchables, derives and the batch: how values change, are computed from each other and reach their watches. +Watchables (JSON and native), derives, writables and the batch: how values change, are computed from each other and +reach their watches. ## For apps (`index.ts`) @@ -25,6 +26,17 @@ watch.start(async (call, ctx) => { `[["r", value]]`). Values are JSON. A watch ends only for good (its owner ended, `stop()`, an aborted context, a throwing listener). - `constant(value)`: a watchable that never changes; a state member or a derive source. +- Native sources (`NativeWatchable`, W4): in-process values compared by identity, never ops, never state members (a + factory returning one fails `broken`, Q14): `self.writable(initial)`, a registry's composed value + (`self.registry`, `registries/`), and `ref.state`. They are derive sources (`self.derive([writable, doc], …)` takes + both kinds; the result is JSON) and `on` sources. A watch gets one call per batch that changed the value, each new + value in order; a failure (a registry whose `compose` threw) is one `failed` call per episode, the watch stays open. + `observe(observer, ctx)` attaches a synchronous observer (Durable's shape: `advance(call, ctx)` inside the batch, + record only; `end(end)`), which counts as watching. `ref.state` gives every state to its watches, not one per batch. +- `Writable` (`self.writable(initial)`): `set(value)`; the same value (by identity) notifies nobody and costs one + comparison; inside a batch it is marked there, outside it opens its own; inside a derive compute or a `compose` it + throws `invalid/reentrant`. It belongs to its attempt: `set` after the attempt's end throws `released`, its watches + end `ended`, its value stays readable. - `mapShared(f)`: maps an array, calling `f` once per object element (by identity) and reusing its result while the element stays the same object, so a derive over a list keeps unchanged parts identical and its diff cheap. @@ -55,8 +67,10 @@ Rules: the flushes listed with `flushing(flush)` during it (a link sends one state message per batch, W1); `within(adopt, ctx)` adopts a change inside the commit running now, or as a commit of its own, or, while a commit's flushes send, as the next commit (presence and keep-alive counts: a departure found while sending lands there); - `run(work)` sets the reentrancy flag (nested); `assertOutside(operation)`; `count(counter)` counts derive work on - the host's metrics (`chord.derive.computes`, `chord.derive.diffs`). + `run(work)` sets the reentrancy flag (nested); `compute(work)` also marks a pure compute (derives, `compose`), in + which `assertNotComputing(operation)` refuses changes; `assertOutside(operation)`; `count(counter)` counts derive and + registry work on the host's metrics (`chord.derive.computes`, `chord.derive.diffs`, `chord.registry.composes`, + `chord.registry.adapts`). `changed(source)` takes any `Delivered` (JSON or native source). - `Source`: a changing value on a host (a singleton document, a derive, a replica) with its watches, subscribers and dependents (`Dependent`: derives, and member sources passing changes on to theirs). Subclasses call `changed()` when a commit changes them and implement `deliver(ctx)` (`publish` a value, `failWatches` a recoverable failure, @@ -67,4 +81,13 @@ Rules: - `Derive`: declared with its raw sources; `attach(resolve, report)` resolves them at activation (and drops the raw list; `report` gets a failure of its compute, once per episode), `detach(error)` when its object ends (later reads throw `error` for good), `invalidate()` from its sources (a failed derive too: it recomputes on its next read). -- `Constant`: the class behind `constant()`. `DeriveSource` is what a derive reads: a `Source` or a `Constant`. +- `Constant`: the class behind `constant()`. `DeriveSource` is a JSON watchable as Chord follows it (a state member): + a `Source` or a `Constant`; `DeriveInput` is what a derive reads: those and `NativeSource`s. +- `NativeSource`: the base of native sources, what derives and registry tables read: `dependents`, `start()` + (`NativeStart`: a value, or a failure with its last good value), `changed()` (inside a batch: marks the dependents) + and `end()` (drops them and the batch). `FollowedSource` adds followers through the batch: `openWatch`, `observe` + (its abort listener removed at the end), `subscribe(observer)` (Chord's own followers, also without a value: + `retryOn`), `deliver(ctx)` (the new value when its followers have another, a failure once per episode; nothing is + kept once nobody follows). `WritableSource` is the writable, its owner (`WritableOwner`) the attempt's context and + seal check, let go when the attempt ends; `ref.state` (`host/`) is a plain `NativeSource` giving each state to its + own watches. diff --git a/packages/chord/src/next/state/batch.ts b/packages/chord/src/next/state/batch.ts index 66dbd5d5a..38eeb0901 100644 --- a/packages/chord/src/next/state/batch.ts +++ b/packages/chord/src/next/state/batch.ts @@ -4,13 +4,18 @@ import type { TelemetryMetrics } from "@earendil-works/pi-telemetry"; import type { Context } from "../../context/index.ts"; import { invalid } from "../data/internal.ts"; -import type { Source } from "./watchable.ts"; /** What a batch reports: a watch listener that threw, or a frame delivery that threw (a derive's value is no JSON). */ export type BatchError = | { readonly kind: "listener_error"; readonly error: unknown } | { readonly kind: "delivery_failed"; readonly error: unknown }; +/** What a commit lists to get its frame delivered: a watched source it changed (JSON or native). */ +export interface Delivered { + /** Delivers this batch's frame to its watches and subscribers; called once per batch that listed it. */ + deliver(context: Context): void; +} + /** A host's batch: the reentrancy flag, the watched sources a commit changed, and where background errors go. */ export class Batch { /** Reports a throwing watch listener's error (to the host's `onError`); what watches get as their `report`. */ @@ -19,8 +24,10 @@ export class Batch { readonly #metrics: TelemetryMetrics | undefined; /** Set while adopting, delivering or computing a derive; one flag per host. */ #running = false; + /** Set while a pure compute runs (a derive's, a registry's `compose`): nothing may change a source then. */ + #computing = false; /** The watched sources the current commit changed, each listed once by its `changed`. */ - #changed: Source[] = []; + #changed: Delivered[] = []; /** What runs once the current commit's frames were delivered, each listed once by its `flushing`. */ #flushes: ((context: Context) => void)[] = []; /** Set while the current commit's flushes run (its links send). */ @@ -46,6 +53,28 @@ export class Batch { if (this.#running) throw invalid("reentrant", `${operation} inside a commit or a derive compute`); } + /** + * Throws `invalid/reentrant` inside a pure compute (a derive's, a registry's `compose`): `operation` would change a + * source while one is computed from it. Changes inside an adoption are fine (they land in its batch). + */ + assertNotComputing(operation: string): void { + if (this.#computing) throw invalid("reentrant", `${operation} inside a derive compute or a compose`); + } + + /** Runs the pure compute `work` (`run`, and changes refused: `assertNotComputing`); nests. */ + compute(work: () => T): T { + const running = this.#running; + const computing = this.#computing; + this.#running = true; + this.#computing = true; + try { + return work(); + } finally { + this.#running = running; + this.#computing = computing; + } + } + /** Runs `work` with reentrant calls refused; nests (a derive compute inside a commit). */ run(work: () => T): T { const outer = this.#running; @@ -129,7 +158,7 @@ export class Batch { } /** Lists a watched source the current commit changed. */ - changed(source: Source): void { + changed(source: Delivered): void { this.#changed.push(source); } @@ -138,8 +167,12 @@ export class Batch { this.#flushes.push(flush); } - /** Counts one unit of derive work on the host's metrics (a lazy read has no context to count through). */ - count(counter: "chord.derive.computes" | "chord.derive.diffs"): void { + /** + * Counts one unit of derive or registry work on the host's metrics (a lazy read has no context to count through). + */ + count( + counter: "chord.derive.computes" | "chord.derive.diffs" | "chord.registry.composes" | "chord.registry.adapts", + ): void { this.#metrics?.counter(counter).add(1); } } diff --git a/packages/chord/src/next/state/derive.ts b/packages/chord/src/next/state/derive.ts index 6bb86f617..df364f10d 100644 --- a/packages/chord/src/next/state/derive.ts +++ b/packages/chord/src/next/state/derive.ts @@ -5,11 +5,15 @@ import type { Context } from "../../context/index.ts"; import { diffRevisions, type Op } from "../../delta/index.ts"; import type { JsonValue } from "../../json.ts"; import type { Batch } from "./batch.ts"; +import { NativeSource } from "./native-source.ts"; import { type Constant, type Dependent, handedOut, Source, type WatchStart } from "./watchable.ts"; -/** What a derive reads: a source it depends on, or a constant. */ +/** A JSON watchable as Chord follows it (a state member, a JSON derive source): a source, or a constant. */ export type DeriveSource = Source | Constant; +/** What a derive reads: a JSON source or constant, or a native source (a `writable`, a registry value, `ref.state`). */ +export type DeriveInput = DeriveSource | NativeSource; + /** * Where a derive stands. `dirty`: a source changed since the last compute (or there was none). A failed derive (its * compute threw) throws its error until a source it reads changes or is re-bound, then recomputes (self-healing, design @@ -34,10 +38,10 @@ const NO_VALUE: unique symbol = Symbol("no value"); * it. */ export class Derive extends Source implements Dependent { - readonly #compute: (values: readonly JsonValue[]) => JsonValue; + readonly #compute: (values: readonly unknown[]) => JsonValue; /** The sources as declared, until `attach` resolves them. */ #declared: readonly unknown[] | undefined; - #sources: readonly DeriveSource[] = []; + #sources: readonly DeriveInput[] = []; #state: DeriveState = { kind: "dirty" }; /** The value this derive's watches have, kept from its marking to its frame in the batch. */ #framed: JsonValue | typeof NO_VALUE | undefined; @@ -49,7 +53,7 @@ export class Derive extends Source implements Dependent { /** Its last good value, which a watch opened while it is failed starts at (`NO_VALUE` before the first). */ #good: JsonValue | typeof NO_VALUE = NO_VALUE; - constructor(sources: readonly unknown[], compute: (values: readonly JsonValue[]) => JsonValue, batch: Batch) { + constructor(sources: readonly unknown[], compute: (values: readonly unknown[]) => JsonValue, batch: Batch) { super(batch); this.#declared = sources; this.#compute = compute; @@ -77,16 +81,20 @@ export class Derive extends Source implements Dependent { * `report` gets a throw of its compute, once per failure episode (a value ends it); a failing source's error is its * source's to report. */ - attach(resolve: (source: unknown, index: number) => DeriveSource, report: (error: unknown) => void): void { + attach(resolve: (source: unknown, index: number) => DeriveInput, report: (error: unknown) => void): void { this.#report = report; this.#sources = this.#declared!.map(resolve); this.#declared = undefined; - for (const source of this.#sources) if (source instanceof Source) source.dependents.add(this); + for (const source of this.#sources) { + if (source instanceof Source || source instanceof NativeSource) source.dependents.add(this); + } } /** Its object ended: no source keeps it, its watches end `ended`, reads throw `error` (the object's). */ detach(error: Error): void { - for (const source of this.#sources) if (source instanceof Source) source.dependents.delete(this); + for (const source of this.#sources) { + if (source instanceof Source || source instanceof NativeSource) source.dependents.delete(this); + } this.#sources = []; this.#declared = undefined; this.#state = { kind: "detached", error }; @@ -146,7 +154,7 @@ export class Derive extends Source implements Dependent { } #recompute(): JsonValue { - return this.batch.run(() => { + return this.batch.compute(() => { try { this.batch.count("chord.derive.computes"); // A failing source throws its own error here, so a derive over a failed derive fails with the same error. diff --git a/packages/chord/src/next/state/index.ts b/packages/chord/src/next/state/index.ts index 8fbd9ad1e..b6ffb24db 100644 --- a/packages/chord/src/next/state/index.ts +++ b/packages/chord/src/next/state/index.ts @@ -1,5 +1,7 @@ -// State for apps: watchables, constants, and the derive helper `mapShared`. +// State for apps: watchables, constants, writables, observers of native sources, and the derive helper `mapShared`. export type { Watchable } from "../../watch.ts"; export { mapShared } from "./derive.ts"; +export type { NativeObservation, NativeObserver } from "./native-source.ts"; export { constant } from "./watchable.ts"; +export type { Writable } from "./writable.ts"; diff --git a/packages/chord/src/next/state/internal.ts b/packages/chord/src/next/state/internal.ts index 5ecd3641f..f4c0475cc 100644 --- a/packages/chord/src/next/state/internal.ts +++ b/packages/chord/src/next/state/internal.ts @@ -1,5 +1,14 @@ -// State for Chord's layers: the batch, sources, derives and constants behind the app's watchables. +// State for Chord's layers: the batch, sources (JSON and native), derives, constants and writables behind the +// app's watchables. -export { Batch, type BatchError } from "./batch.ts"; -export { Derive, type DeriveSource } from "./derive.ts"; +export { Batch, type BatchError, type Delivered } from "./batch.ts"; +export { Derive, type DeriveInput, type DeriveSource } from "./derive.ts"; +export { + FollowedSource, + type NativeObservation, + type NativeObserver, + NativeSource, + type NativeStart, +} from "./native-source.ts"; export { Constant, type Dependent, handedOut, Source, type Subscriber, type WatchStart } from "./watchable.ts"; +export { type WritableOwner, WritableSource } from "./writable.ts"; diff --git a/packages/chord/src/next/state/native-source.ts b/packages/chord/src/next/state/native-source.ts new file mode 100644 index 000000000..9d6308801 --- /dev/null +++ b/packages/chord/src/next/state/native-source.ts @@ -0,0 +1,291 @@ +// Native sources (W4, slice 8 §2): in-process values that change, compared by identity (a `writable`, a registry's +// composed value, `ref.state`). Derives and registry tables depend on them and the batch marks them; the followed +// ones (writables, registry values) give their watches and observers one call per batch that changed them. + +import type { Context } from "../../context/index.ts"; +import { + NativeWatch, + type NativeWatchable, + type NativeWatchCall, + type NativeWatchHandle, + type WatchEnd, +} from "../../watch.ts"; +import type { Batch } from "./batch.ts"; +import { type Dependent, handedOut } from "./watchable.ts"; + +/** + * A synchronous observer of a native source (Durable's observer shape): `advance` is called inside the batch that + * changed the source, so it only records; `end` once, when the source ended (`ended`) or the observer's context was + * aborted (`aborted`). + */ +export interface NativeObserver { + /** The source's call of this batch: its new value, or a failure it may recover from. */ + advance(call: NativeWatchCall, context: Context): void; + /** The observer is detached for good with `end`. */ + end(end: WatchEnd): void; +} + +/** What `observe` gives: the value the observer starts at, and how to detach it (Durable's `Observation`). */ +export type NativeObservation = { readonly value: T; detach(): void }; + +/** + * What a new watch of a native source starts at: its value, or while it is failed, the failure and its last good value + * (`last` undefined when it never had one: a watch cannot start then). + */ +export type NativeStart = + | { readonly kind: "value"; readonly value: T } + | { readonly kind: "failed"; readonly error: unknown; readonly last: { readonly value: T } | undefined }; + +/** + * A native source as derives and registry tables read it: its value (`start()` while it may fail), the dependents a + * change marks (`changed()`, inside a batch), and the host's batch until it ended. How its watches get their calls is + * the subclass's (`FollowedSource` for writables and registry values; `ref.state` gives each state to its watches). + */ +export abstract class NativeSource implements NativeWatchable { + /** Derives and registry tables reading this source; each removes itself when it stops. */ + readonly dependents = new Set(); + /** The host's batch; dropped when the source ended, so an ended source keeps no host. */ + #batch: Batch | undefined; + + /** `batch` undefined: made ended (a released reference's state). */ + constructor(batch: Batch | undefined) { + this.#batch = batch; + } + + abstract get value(): T; + + abstract watch(context: Context): Promise>; + + /** What a new watch starts at; a source that can fail and recover tells its failure. */ + start(): NativeStart { + return { kind: "value", value: this.value }; + } + + /** + * Its value now as a contribution reads it: throws its failure while it is failed, where `.value` keeps the last good + * value for readers (a registry whose `compose` threw). + */ + current(): T { + return this.value; + } + + /** The host's batch, `undefined` once the source ended. */ + protected get batch(): Batch | undefined { + return this.#batch; + } + + /** Records a change inside the current batch: every dependent marked. */ + protected changed(): void { + for (const dependent of this.dependents) dependent.invalidate(); + } + + /** Ends the source for good: its dependents dropped, the batch let go. */ + protected end(): void { + this.#batch = undefined; + this.dependents.clear(); + } +} + +/** + * What the watches and observers were last given: a value, a failure (one call per failure episode), or nothing yet + * (a source with no value, followed for its first one). + */ +type Shown = { readonly kind: "value"; readonly value: T } | { readonly kind: "failed" } | { readonly kind: "none" }; + +const ENDED: WatchEnd = { kind: "ended" }; + +const ABORTED: WatchEnd = { kind: "aborted" }; + +const FAILED: Shown = { kind: "failed" }; + +const NONE = { kind: "none" } as const; + +const NOTHING = (): void => {}; + +/** + * A native source its followers get through the batch (a writable, a registry value): a change lists it for one frame + * when someone follows it; the frame is its value then, delivered only when it is another value (identity) than the + * one its followers have, or after a failure, whose call comes once per episode. + */ +export abstract class FollowedSource extends NativeSource { + readonly #watches = new Set>(); + /** Synchronous observers and Chord's own subscribers (`retryOn`), fed inside the batch, each with its cleanup. */ + readonly #observers = new Map, () => void>(); + /** What the followers have; `undefined` while nobody follows (so no old value is kept). */ + #shown: Shown | undefined; + /** Listed for a frame in the current batch. */ + #listed = false; + + /** Whether a watch or an observer follows it (a registry value also while readers wait for its first value). */ + get watched(): boolean { + return this.#watches.size > 0 || this.#observers.size > 0; + } + + async watch(context: Context): Promise> { + return this.openWatch(context); + } + + /** + * `watch` without the Promise. On a failed source it opens at the last good value and its first call is the + * failure; with no value to start at it throws the failure. An ended source gives a watch that has ended. + */ + openWatch(context: Context): NativeWatch { + const batch = this.batch; + if (batch === undefined) { + const ended = new NativeWatch(this.value, NOTHING, NOTHING); + ended.end(ENDED); + return ended; + } + batch.assertOutside("watch"); + const start = this.start(); + if (start.kind === "failed" && start.last === undefined) throw start.error; + const value = start.kind === "value" ? start.value : start.last!.value; + this.#follow(start); + const watch: NativeWatch = new NativeWatch(value, () => this.#unfollow(watch), batch.reportListener); + this.#watches.add(watch); + if (context.abortSignal !== undefined) watch.observeAbort(context.abortSignal); + if (start.kind === "failed") watch.fail(handedOut(start.error), context); + return watch; + } + + /** + * Attaches a synchronous observer at the current value (Durable's `observeXXX`); it counts as following, so a lazy + * value is computed in the batch that changed it. Throws the failure when there is no value to start at; aborting + * `context` detaches it (`aborted`); the source's end detaches it too (`ended`), its abort listener removed. + */ + observe(observer: NativeObserver, context: Context): NativeObservation { + this.batch?.assertOutside("observe"); + const start = this.start(); + if (start.kind === "failed" && start.last === undefined) throw start.error; + const value = start.kind === "value" ? start.value : start.last!.value; + if (this.batch === undefined) { + observer.end(ENDED); + return { value, detach: NOTHING }; + } + const signal = context.abortSignal; + if (signal?.aborted) { + observer.end(ABORTED); + return { value, detach: NOTHING }; + } + const aborted = (): void => { + if (!this.#observers.has(observer)) return; + this.unsubscribe(observer); + observer.end(ABORTED); + }; + const cleanup = signal === undefined ? NOTHING : () => signal.removeEventListener("abort", aborted); + signal?.addEventListener("abort", aborted, { once: true }); + this.#follow(start); + this.#observers.set(observer, cleanup); + return { value, detach: () => this.unsubscribe(observer) }; + } + + /** + * Feeds `observer` every call from now on, inside each batch, also while the source has no value (Chord's own + * followers: `retryOn`). Nothing is delivered for the current value. + */ + subscribe(observer: NativeObserver): void { + if (this.batch === undefined) return; + this.#follow(this.#startOrNone()); + this.#observers.set(observer, NOTHING); + } + + unsubscribe(observer: NativeObserver): void { + const cleanup = this.#observers.get(observer); + if (cleanup === undefined) return; + this.#observers.delete(observer); + cleanup(); + this.#settleFollowers(); + } + + /** Delivers this batch's frame: the new value when the followers have another, or a failure once per episode. */ + deliver(context: Context): void { + this.#listed = false; + const shown = this.#shown; + if (shown === undefined) return; + this.#deliver(shown, context); + // Readers waiting for a first value pulled it through this delivery: nothing is kept for them. + this.#settleFollowers(); + } + + /** + * Records a change inside the current batch: every dependent marked, and the source listed for a frame when someone + * follows it (or `watched` says so). + */ + protected override changed(): void { + super.changed(); + this.list(); + } + + /** + * Lists the source for a frame in the current batch when someone follows it (or `watched` says so), its dependents + * left as they are (marked already). + */ + protected list(): void { + if (this.#listed || !this.watched || this.batch === undefined) return; + this.#listed = true; + // Only readers waiting for a first value follow it without a follower's record: they have none. Not computed + // here: marking runs inside adoptions, the value is pulled in the delivery. + this.#shown ??= NONE; + this.batch.changed(this); + } + + /** Nothing follows it any more (no watch, no observer): a subclass drops what it kept for followers. */ + protected unfollowed(): void {} + + /** Ends the source for good: watches and observers end `ended` (their abort listeners removed). */ + protected override end(): void { + super.end(); + this.#shown = undefined; + // Ending detaches, which deletes the current entry (safe while iterating) and runs no listener. + for (const watch of this.#watches) watch.end(ENDED); + const observers = [...this.#observers]; + this.#observers.clear(); + for (const [observer, cleanup] of observers) { + cleanup(); + observer.end(ENDED); + } + } + + #deliver(shown: Shown, context: Context): void { + const start = this.#startOrNone(); + if (start.kind === "failed") { + // A failure reaches followers that had a value: those without one wait for the first. + if (shown.kind !== "value") return; + this.#shown = FAILED; + const error = handedOut(start.error); + for (const watch of this.#watches) watch.fail(error, context); + for (const observer of [...this.#observers.keys()]) observer.advance({ kind: "failed", error }, context); + return; + } + if (start.kind !== "value" || (shown.kind === "value" && shown.value === start.value)) return; + const { value } = start; + this.#shown = start; + for (const watch of this.#watches) watch.advance(value, context); + // A copy: an observer may detach itself in its call. + for (const observer of [...this.#observers.keys()]) observer.advance({ kind: "value", value }, context); + } + + /** Its start, `none` while it has no value. */ + #startOrNone(): NativeStart | { readonly kind: "none" } { + const start = this.start(); + return start.kind === "failed" && start.last === undefined ? NONE : start; + } + + /** A follower comes: the first one records what followers have (a later one opened at the same current value). */ + #follow(start: NativeStart | { readonly kind: "none" }): void { + if (this.#shown !== undefined) return; + this.#shown = start.kind === "failed" ? FAILED : start; + } + + #unfollow(watch: NativeWatch): void { + this.#watches.delete(watch); + this.#settleFollowers(); + } + + /** No follower left: nothing is shown any more, so no old value is kept. */ + #settleFollowers(): void { + if (this.#watches.size > 0 || this.#observers.size > 0) return; + this.#shown = undefined; + this.unfollowed(); + } +} diff --git a/packages/chord/src/next/state/writable.ts b/packages/chord/src/next/state/writable.ts new file mode 100644 index 000000000..e8760b409 --- /dev/null +++ b/packages/chord/src/next/state/writable.ts @@ -0,0 +1,78 @@ +// Writables (slice 8 §2, design §17): `self.writable(initial)`, an in-process value its attempt sets, identity +// compared; a contribution, a derive source and an `on` source, never a state member. + +import type { Context } from "../../context/index.ts"; +import type { NativeWatchable } from "../../watch.ts"; +import type { Batch } from "./batch.ts"; +import { FollowedSource } from "./native-source.ts"; + +/** An in-process value that changes when its owner sets it; `set` of the same value notifies nobody. */ +export interface Writable extends NativeWatchable { + /** + * Sets the value: a new value (by identity) is a new frame for its watches and marks the derives and registry + * tables reading it, in the batch running now or in one of its own. Throws `released` once its attempt ended, and + * `invalid/reentrant` inside a derive compute or a `compose`. + */ + set(value: T): void; +} + +/** Who owns a writable: the context its own batches run in, and the check that its attempt still runs. */ +export type WritableOwner = { + readonly context: Context; + /** Throws `released` once the attempt that made it was sealed. */ + assertUsable(): void; +}; + +/** Whom a writable belongs to: its running attempt, or nothing once that attempt ended (its error kept). */ +type Ownership = + | { readonly kind: "owned"; readonly owner: WritableOwner } + | { readonly kind: "ended"; readonly error: Error }; + +/** A writable: its value, set by its owner's code, delivered through the host's batch. */ +export class WritableSource extends FollowedSource implements Writable { + #value: T; + #ownership: Ownership; + + constructor(initial: T, owner: WritableOwner, batch: Batch) { + super(batch); + this.#value = initial; + this.#ownership = { kind: "owned", owner }; + } + + get value(): T { + return this.#value; + } + + set(value: T): void { + const ownership = this.#ownership; + if (ownership.kind === "ended") throw ownership.error; + ownership.owner.assertUsable(); + // The cost guard: setting the value it has costs one comparison past the seal check. + if (value === this.#value) return; + const batch = this.batch!; + batch.assertNotComputing("writable.set"); + this.#change(value, batch, ownership.owner.context); + } + + /** + * Adopts `value` in the batch running now, or in one of its own. Apart from `set`: a closure there would make every + * `set` allocate its context, the same value's too. + */ + #change(value: T, batch: Batch, context: Context): void { + batch.within(() => { + this.#value = value; + this.changed(); + }, context); + } + + /** + * Its attempt ended with `error`: its watches end `ended`, `set` throws `error`, its value stays readable; it keeps + * neither its attempt nor the host. + */ + detach(error: Error): void { + // Kept by `set`: its stack is made text now, so it holds none of the frames' objects. + void error.stack; + this.#ownership = { kind: "ended", error }; + this.end(); + } +} diff --git a/packages/chord/test/next/bench/calls.ts b/packages/chord/test/next/bench/calls.ts new file mode 100644 index 000000000..d8e5dc0f7 --- /dev/null +++ b/packages/chord/test/next/bench/calls.ts @@ -0,0 +1,90 @@ +// Benchmark of the paths slice 8 touched under every object (no registries involved): a local method call (its ctx), +// a commit through a watched derive (the batch, a derive's compute) and activation churn (an object with params made, +// live and ended per get: its attempt's declarations). Median µs per operation over bursts, and heap allocated per +// operation (growth plus what the collections during the burst freed). Runs on HEAD as it is (copy it there). +// Run by hand from packages/chord (not in CI), pinned for comparisons: +// taskset -c 6 node --conditions=source --experimental-strip-types --expose-gc test/next/bench/calls.ts +import { performance } from "node:perf_hooks"; +import { GCProfiler } from "node:v8"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method, obj } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; + +const BURSTS = 9; + +const gc = (globalThis as { gc?: () => void }).gc!; + +const Counter = defineState("bench.counter", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); +const Store = local("bench.store", { + version: 1, + params: {}, + members: { doubled: int(), bump: method(), set: method(int()) }, +}); +const Item = local("bench.item", { version: 1, params: { id: int() }, members: { n: int() } }); + +const host = createHost({ onError: (error) => console.log("onError", error.kind) }); +await host.load( + [ + { + name: "@bench/calls", + facet: defineFacet((h) => { + h.serve(Store, (_, self) => { + const counter = self.state(Counter); + return { + doubled: self.derive([counter], ([c]) => c.n * 2), + bump: async () => null, + set: async (n, ctx) => { + await counter.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }); + h.serve(Item, ({ id }, self) => ({ n: self.derive([self.get(Store, {}).doubled], ([d]) => d + id) })); + }), + }, + ], + context, +); + +const store = await host.get(Store, {}, context); +const watch = await store.doubled.watch(context); +watch.start(async () => {}); + +/** Median µs per operation over the bursts (after one warm-up burst), and heap bytes allocated per operation. */ +async function measure(runs: number, operation: (i: number) => Promise): Promise { + const times: number[] = []; + let bytes = 0; + let counted = 0; + for (let burst = -1; burst < BURSTS; burst++) { + gc(); + const before = process.memoryUsage().heapUsed; + const profiler = new GCProfiler(); + profiler.start(); + const started = performance.now(); + for (let i = 0; i < runs; i++) await operation(i); + const elapsed = performance.now() - started; + const after = process.memoryUsage().heapUsed; + let freed = 0; + for (const { beforeGC, afterGC } of profiler.stop().statistics) { + freed += beforeGC.heapStatistics.usedHeapSize - afterGC.heapStatistics.usedHeapSize; + } + if (burst < 0) continue; + times.push((elapsed * 1000) / runs); + bytes += after - before + freed; + counted += runs; + } + times.sort((a, b) => a - b); + return `${times[Math.floor(times.length / 2)]!.toFixed(2)} µs, ${Math.round(bytes / counted)} B allocated`; +} + +console.log("local method call:", await measure(20_000, () => store.bump(context))); +console.log("commit through a watched derive:", await measure(10_000, (i) => store.set(i + 1, context))); +console.log( + "activation churn (get, live, release):", + await measure(2_000, async (i) => (await host.get(Item, { id: i }, context)).release()), +); +await watch.stop(); +await host.close(context); diff --git a/packages/chord/test/next/bench/registries.ts b/packages/chord/test/next/bench/registries.ts new file mode 100644 index 000000000..5eb75504b --- /dev/null +++ b/packages/chord/test/next/bench/registries.ts @@ -0,0 +1,108 @@ +// Benchmark of slice 8's registries and writables: a `writable.set` delivered to one watch, a set of the same value +// (one comparison), and one contribution of N (10, 100) changed while the registry is watched (one table build and one +// compose per set, over N entries). Median µs per operation over bursts, heap allocated per operation, and the +// registry's compose count per set (the cost guard: one). +// Run by hand from packages/chord (not in CI), pinned for comparisons: +// taskset -c 6 node --conditions=source --experimental-strip-types --expose-gc test/next/bench/registries.ts +import { performance } from "node:perf_hooks"; +import { GCProfiler } from "node:v8"; +import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; +import type { Writable } from "../../../src/next/state/index.ts"; + +const BURSTS = 9; + +const gc = (globalThis as { gc?: () => void }).gc!; + +/** Median µs per operation over the bursts (after one warm-up burst), and heap bytes allocated per operation. */ +async function measure(runs: number, operation: (i: number) => void): Promise { + const times: number[] = []; + let bytes = 0; + let counted = 0; + for (let burst = -1; burst < BURSTS; burst++) { + gc(); + const before = process.memoryUsage().heapUsed; + const profiler = new GCProfiler(); + profiler.start(); + const started = performance.now(); + for (let i = 0; i < runs; i++) operation(i); + // The watches' listeners run on microtasks: their work counts too. + await new Promise((resolve) => setImmediate(resolve)); + const elapsed = performance.now() - started; + const after = process.memoryUsage().heapUsed; + let freed = 0; + for (const { beforeGC, afterGC } of profiler.stop().statistics) { + freed += beforeGC.heapStatistics.usedHeapSize - afterGC.heapStatistics.usedHeapSize; + } + if (burst < 0) continue; + times.push((elapsed * 1000) / runs); + bytes += after - before + freed; + counted += runs; + } + times.sort((a, b) => a - b); + return `${times[Math.floor(times.length / 2)]!.toFixed(3)} µs, ${Math.round(bytes / counted)} B allocated`; +} + +const Sum = defineRegistry("bench.sum", { version: 1 }); + +/** A host with `contributors` host-lifetime objects each contributing a writable number to `Sum`, watched. */ +async function registryOf(contributors: number) { + const telemetry = new InMemoryTelemetryContext(); + const writables: Writable[] = []; + const types = Array.from({ length: contributors }, (_, i) => + local(`bench.contributor-${i}`, { version: 1, params: {}, members: { n: int() } }), + ); + const host = createHost({ telemetry, onError: (error) => console.log("onError", error.kind) }); + const facet = defineFacet((h) => { + h.serve(Sum, { compose: (table) => table.reduce((sum, { value }) => sum + value, 0) }); + for (const Type of types) { + h.serve(Type, (_, self) => { + const value = self.writable(0); + writables.push(value); + self.contribute(Sum, value); + return { n: self.derive([value], ([n]) => n) }; + }); + } + }); + await host.load([{ name: "@bench/registries", facet }], context); + const sum = await host.registry(Sum, context); + const watch = await sum.watch(context); + watch.start(async () => {}); + const composes = (): number => + telemetry + .getMetrics() + .filter(({ name }) => name === "chord.registry.composes") + .reduce((n, { value }) => n + value, 0); + return { host, writables, watch, composes }; +} + +const one = await registryOf(1); +const writable = one.writables[0]!; +const watch = await writable.watch(context); +watch.start(async () => {}); +console.log("writable.set, one watch:", await measure(20_000, (i) => writable.set(i + 1))); +// The listeners' backlogs dropped first: what follows measures the set alone. +await watch.stop(); +await one.watch.stop(); +writable.set(-1); +console.log("writable.set of the same value:", await measure(1_000_000, () => writable.set(-1))); +console.log("an empty operation (the loop's own cost):", await measure(1_000_000, () => {})); +await one.host.close(context); + +for (const contributors of [10, 100]) { + const { host, writables, watch, composes } = await registryOf(contributors); + const before = composes(); + const target = writables[Math.floor(contributors / 2)]!; + const sets = 5_000; + const result = await measure(sets, (i) => target.set(i + 1)); + console.log( + `one of ${contributors} contributions set, registry watched:`, + result, + `${((composes() - before) / (sets * (BURSTS + 1))).toFixed(2)} composes per set`, + ); + await watch.stop(); + await host.close(context); +} diff --git a/packages/chord/test/next/data/errors.test.ts b/packages/chord/test/next/data/errors.test.ts index 65dee086a..c5d955a99 100644 --- a/packages/chord/test/next/data/errors.test.ts +++ b/packages/chord/test/next/data/errors.test.ts @@ -25,7 +25,9 @@ describe("ChordError", () => { const error = new ChordError({ code: "too_large", reason: "part", message: "m" }) as ChordError; if (error.code === "invalid") expectTypeOf(error.reason).toEqualTypeOf>(); if (error.code === "unavailable") - expectTypeOf(error.reason).toEqualTypeOf<"locked" | "park_timeout" | "busy" | "storage_failed" | undefined>(); + expectTypeOf(error.reason).toEqualTypeOf< + "locked" | "park_timeout" | "busy" | "storage_failed" | "undeclared" | undefined + >(); if (error.code === "gone") expectTypeOf(error.reason).toEqualTypeOf(); expect(error).toBeInstanceOf(Error); expect(error.name).toBe("ChordError"); @@ -83,7 +85,7 @@ describe("protocol table", () => { not_ready: [], released: ["ended", "closed", "settled", "version", "discarded"], too_large: ["part"], - unavailable: ["locked", "park_timeout", "busy", "storage_failed"], + unavailable: ["locked", "park_timeout", "busy", "storage_failed", "undeclared"], aborted: [], }, }; diff --git a/packages/chord/test/next/examples/29-registries.ts b/packages/chord/test/next/examples/29-registries.ts new file mode 100644 index 000000000..462bd20b0 --- /dev/null +++ b/packages/chord/test/next/examples/29-registries.ts @@ -0,0 +1,115 @@ +// Registries: an extension point between plugins. The defining plugin shares a token (`defineRegistry`: a name, a +// version, the contribution and value types) and serves it with `compose`, a pure function from the table of +// contributions (sorted by plugin name, never load order) to one value, conflicts resolved by a fallback and reported +// (`registry_conflict`). Facets and host-lifetime objects contribute one value each: a plain value, or a watchable +// (`self.writable`) whose changes recompose. Readers follow the composed value (`self.registry`, a dependency like +// `self.get`; `host.registry` outside objects) and guard each contribution where they call it. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/29-registries.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { defineRegistry, keyedEntries, type TableItem } from "../../../src/next/registries/index.ts"; + +const context = BACKGROUND_CONTEXT; + +// 1. The token, shared by every plugin (bundled like a contract): routes in, a router out. +type Handler = (path: string) => string; +type Routes = { readonly routes: Readonly> }; +type Router = ReadonlyMap>; +const Http = defineRegistry("example.http", { version: 1 }); + +/** The HTTP server: it reads the router at each request and guards the contribution it calls. */ +const Server = local("example.server", { + version: 1, + params: {}, + members: { request: method(obj({ path: str() }), str()) }, +}); + +/** A plugin whose routes are known only at runtime: it contributes a writable it sets. */ +const Mounts = local("example.mounts", { version: 1, params: {}, members: { mount: method(obj({ path: str() })) } }); + +// 2. The defining plugin serves the registry: the first route by plugin name wins, a duplicate is reported. +const httpPlugin = defineFacet((host) => { + host.serve(Http, { compose: (table, { report }) => keyedEntries(table, (entry) => entry.routes, report) }); + host.serve(Server, (_, self) => { + const router = self.registry(Http); + return { + request: async ({ path }) => { + const route = router.value.get(path); + if (route === undefined) return "404"; + // A contribution is another plugin's code: catch, report its plugin, go on. + try { + return route.value(path); + } catch (error) { + console.log(`route ${path} of ${route.plugin} threw:`, (error as Error).message); + return "500"; + } + }, + }; + }); +}); + +// 3. Two plugins contribute from their facets: plain values, in the table while the plugin is loaded. +const notesPlugin = defineFacet((_, self) => { + self.contribute(Http, { routes: { "/notes": () => "notes", "/broken": () => JSON.parse("{") } }); +}); +const searchPlugin = defineFacet((_, self) => { + self.contribute(Http, { routes: { "/search": () => "results", "/notes": () => "search's notes" } }); +}); + +// 4. A host-lifetime object contributes a writable: its changes recompose the router. +const mountsPlugin = defineFacet((host) => { + host.serve(Mounts, (_, self) => { + const routes = self.writable({ routes: {} }); + self.contribute(Http, routes); + return { + mount: async ({ path }) => { + routes.set({ routes: { ...routes.value.routes, [path]: () => `mounted ${path}` } }); + return null; + }, + }; + }); +}); + +const host = createHost({ + onError: (error) => { + if (error.kind === "registry_conflict") console.log("conflict:", error.message, error.plugins); + else console.log("onError:", error.kind); + }, +}); + +// 5. A reader before `load` waits: the provider comes with `load`, which composes every registry before any object. +const early = host.registry(Http, context); +await host.load( + [ + { name: "@example/http", facet: httpPlugin }, + { name: "@example/notes", facet: notesPlugin }, + { name: "@example/search", facet: searchPlugin }, + { name: "@example/mounts", facet: mountsPlugin }, + ], + context, +); +const router = await early; +console.log( + "routes:", + [...router.value].map(([path, { plugin }]) => `${path} (${plugin})`), +); + +// 6. Requests: a duplicate went to the first plugin by name; a throwing contribution is guarded at the use site. +const server = await host.get(Server, {}, context); +console.log("/notes ->", await server.request({ path: "/notes" }, context)); +console.log("/broken ->", await server.request({ path: "/broken" }, context)); + +// 7. A watch of the composed value gets each new router; mounting recomposes it. +const watch = await router.watch(context); +watch.start(async (call) => { + if (call.kind === "value") console.log("router now has:", [...call.value.keys()]); +}); +const mounts = await host.get(Mounts, {}, context); +await mounts.mount({ path: "/files" }, context); +await new Promise((resolve) => setTimeout(resolve, 0)); +console.log("/files ->", await server.request({ path: "/files" }, context)); + +await watch.stop(); +await host.close(context); diff --git a/packages/chord/test/next/examples/30-host-root.ts b/packages/chord/test/next/examples/30-host-root.ts new file mode 100644 index 000000000..885ab4414 --- /dev/null +++ b/packages/chord/test/next/examples/30-host-root.ts @@ -0,0 +1,85 @@ +// `host.root(fn, context)`: a call scope for code outside objects (a Durable tool, a CLI command). `fn` gets a ctx like a +// method handler's: the host's principal (or one a trusted caller names), this host's incarnation, the context's +// telemetry and signal, and `ctx.get` for references. When `fn` settles, or the context aborts, or the host closes, the +// scope is sealed: its signal aborts, its references are released, and a late `ctx.get` is refused `released`; so code +// that ignores its signal fails at its next use of Chord. (p3 wraps it as `chord(fn, context)` for Durable's tools.) +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/30-host-root.ts +import { BACKGROUND_CONTEXT, withCancel } from "../../../src/context/index.ts"; +import { arr, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { type Ctx, createHost, defineFacet, sleep } from "../../../src/next/host/index.ts"; + +const context = BACKGROUND_CONTEXT; + +type Principal = { readonly kind: "server" } | { readonly kind: "owner"; readonly name: string }; + +const Notes = local("example.notes", { + version: 1, + params: { session: str() }, + members: { list: arr(str()), add: method(obj({ text: str() })) }, +}); + +const List = defineState("example.list", { + schema: obj({ list: arr(str()) }), + version: 1, + initial: () => ({ list: [] }), +}); + +const facet = defineFacet((host) => { + host.serve(Notes, (_, self) => { + const notes = self.state(List); + return { + list: self.derive([notes], ([n]) => n.list), + add: async ({ text }, ctx) => { + await notes.change((draft) => { + draft.list.push(`${text} (by ${ctx.principal?.kind})`); + }, ctx); + return null; + }, + }; + }); +}); + +const host = createHost({ principal: { kind: "server" }, onError: (error) => console.log(error.kind) }); + +// 1. A scope made before `load` resolved: its `ctx.get` parks until the host is loaded. +const early = host.root(async (ctx) => (await ctx.get(Notes, { session: "s1" })).list.value, context); +await host.load([{ name: "@example/notes", facet }], context); +console.log("before load, read after it:", await early); + +// 2. A tool's call: get, call, return plain data. The reference is released when fn settles. +const addNote = (text: string) => + host.root(async (ctx) => { + const notes = await ctx.get(Notes, { session: "s1" }); + await notes.add({ text }, ctx); + return notes.list.value; + }, context); +console.log("after a tool call:", await addNote("first")); + +// 3. Base code acting for an authenticated user names the principal. +const owner: Principal = { kind: "owner", name: "ada" }; +console.log("as the owner:", await host.root(async (ctx) => ctx.principal, context, { principal: owner })); + +// 4. A kept ctx is sealed once fn settled. +let kept: Ctx | undefined; +await host.root(async (ctx) => { + kept = ctx; +}, context); +console.log("kept ctx aborted:", kept!.abortSignal?.aborted); +await kept!.get(Notes, { session: "s1" }).catch((error: Error) => console.log("kept ctx.get:", error.message)); + +// 5. Cancelling the context rejects at once and seals the scope; an fn that ignores its signal fails at its next +// use of Chord. +const { context: cancellable, cancel } = withCancel(context); +const stubborn = host.root(async (ctx) => { + await new Promise((resolve) => setTimeout(resolve, 20)); + await ctx.get(Notes, { session: "s1" }); +}, cancellable); +cancel(new Error("user pressed stop")); +await stubborn.catch((error: Error) => console.log("cancelled:", error.message)); + +// 6. Closing the host seals the scopes still open and rejects their callers at once: an fn sleeping on its ctx ends. +const sleeping = host.root((ctx) => sleep(60_000, ctx), context).catch((error: Error) => `closed: ${error.message}`); +await host.close(context); +console.log(await sleeping); diff --git a/packages/chord/test/next/host/attempt.test.ts b/packages/chord/test/next/host/attempt.test.ts index 867502320..4066748e7 100644 --- a/packages/chord/test/next/host/attempt.test.ts +++ b/packages/chord/test/next/host/attempt.test.ts @@ -1,5 +1,5 @@ import { createHook } from "node:async_hooks"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, expectTypeOf, it, vi } from "vitest"; import { type Context, BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; import { arr, int, local, method, obj, opt, remote, str } from "../../../src/next/contracts/index.ts"; import { broken, isChordError } from "../../../src/next/data/index.ts"; @@ -12,6 +12,7 @@ import { type Self, type StoreSelf, } from "../../../src/next/host/index.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; import { constant } from "../../../src/next/state/index.ts"; import type { StagedWrite } from "../../../src/next/store/step.ts"; import type { ChordStore } from "../../../src/next/store/store.ts"; @@ -425,6 +426,36 @@ describe("declarations", () => { () => {}, ), ).toThrow("self.open is valid only in the factory"); + const Names = defineRegistry("app.names", { version: 1 }); + expect(() => later!.registry(Names)).toThrow("self.registry is valid only in the factory"); + expect(() => later!.writable(1)).toThrow("self.writable is valid only in the factory"); + expect(() => later!.retryOn()).toThrow("self.retryOn is valid only in the factory"); + }); + + it("contribute only in host-lifetime objects (R21): `Self` has no `contribute`, a declaration error at runtime", async () => { + expectTypeOf().not.toHaveProperty("contribute"); + expectTypeOf().toHaveProperty("contribute"); + const Names = defineRegistry("app.names", { version: 1 }); + const Store = local("app.store", { version: 1, params: {}, members: {} }); + const { host } = await loaded((h) => { + h.serve(Names, { compose: (table) => table.length }); + h.serve(Page, (_, self) => { + (self as unknown as StoreSelf).contribute(Names, "page"); + return { label: constant(""), slow: async () => null }; + }); + h.serve(Store, (_, self) => { + self.contribute(Names, "a"); + self.contribute(Names, "b"); + return {}; + }); + }); + await expect(host.get(Page, { id: 1 }, context)).rejects.toThrow( + "app.page(1): self.contribute is valid only in a local type with empty params", + ); + await expect(host.get(Store, {}, context)).rejects.toThrow( + "app.store: registry app.names is contributed to twice", + ); + await host.close(context); }); it("refuse a document definition declared twice", async () => { diff --git a/packages/chord/test/next/host/ctx.test.ts b/packages/chord/test/next/host/ctx.test.ts index 732ed2d3a..df259c227 100644 --- a/packages/chord/test/next/host/ctx.test.ts +++ b/packages/chord/test/next/host/ctx.test.ts @@ -61,7 +61,7 @@ describe("ctx", () => { expect(await rejection(seen.ctx!.get(Item, { id: 1 }))).toEqual({ code: "released", reason: "settled", - message: "app.item: ctx.get after its handler settled", + message: "app.item: ctx.get after its call settled", }); }); @@ -103,7 +103,7 @@ describe("ctx", () => { expect(await seen.late).toMatchObject({ code: "released", reason: "settled", - message: "app.item: ctx.get after its handler settled", + message: "app.item: ctx.get after its call settled", }); await flushed(); // The late reference's object ended with it. @@ -130,7 +130,7 @@ describe("ctx", () => { expect(await pending).toMatchObject({ code: "released", reason: "settled", - message: "app.far: ctx.get after its handler settled", + message: "app.far: ctx.get after its call settled", }); }); diff --git a/packages/chord/test/next/host/facet.test.ts b/packages/chord/test/next/host/facet.test.ts index 54fe1113d..0dd05e818 100644 --- a/packages/chord/test/next/host/facet.test.ts +++ b/packages/chord/test/next/host/facet.test.ts @@ -1,6 +1,7 @@ import { describe, expect, expectTypeOf, it } from "vitest"; import { int, local } from "../../../src/next/contracts/index.ts"; -import { defineFacet, type Facet, type FacetHost, type Host } from "../../../src/next/host/index.ts"; +import { defineFacet, type Facet, type FacetHost, type FacetSelf, type Host } from "../../../src/next/host/index.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; import { constant } from "../../../src/next/state/index.ts"; import { loaded } from "../support.ts"; @@ -17,13 +18,34 @@ describe("facets", () => { ); }); - it("only declare: a setup takes the host alone, which has no documents, routines or natives", () => { - expectTypeOf["setup"]>>().toEqualTypeOf<[FacetHost<{ kind: "owner" }>]>(); + it("only declare: a setup takes the host and a self that only contributes; no documents, routines or natives", () => { + expectTypeOf["setup"]>>().toEqualTypeOf< + [FacetHost<{ kind: "owner" }>, FacetSelf] + >(); + expectTypeOf().toEqualTypeOf<"contribute">(); expectTypeOf().not.toHaveProperty("state"); expectTypeOf().not.toHaveProperty("on"); expectTypeOf().not.toHaveProperty("onLive"); }); + it("contribute at most one value per registry, only while their setup runs", async () => { + const Names = defineRegistry("app.names", { version: 1 }); + await expect( + loaded((_, self) => { + self.contribute(Names, "a"); + self.contribute(Names, "b"); + }), + ).rejects.toThrow("@test/app: registry app.names is contributed to twice"); + let later: FacetSelf | undefined; + await loaded((host, self) => { + host.serve(Names, { compose: (table) => table.length }); + later = self; + }); + expect(() => later!.contribute(Names, "late")).toThrow( + "@test/app: self.contribute is valid only in the facet's setup", + ); + }); + it("are typed by exactly the app's principal type", () => { type App = { kind: "owner" } | { kind: "guest"; session: string }; const facet = defineFacet(() => {}); diff --git a/packages/chord/test/next/host/members.test.ts b/packages/chord/test/next/host/members.test.ts index 2efc060e0..9c5e40793 100644 --- a/packages/chord/test/next/host/members.test.ts +++ b/packages/chord/test/next/host/members.test.ts @@ -22,10 +22,10 @@ describe("the member check (Q14)", () => { }), ); await expect(host.get(Named, { n: 1 }, context)).rejects.toThrow( - "app.named(1): label: source 1 is not a JSON watchable", + "app.named(1): label: source 1 is not a Chord watchable", ); await expect(host.get(Named, { n: 2 }, context)).rejects.toThrow( - "app.named(2): derive #1: source 0 is not a JSON watchable", + "app.named(2): derive #1: source 0 is not a Chord watchable", ); }); }); diff --git a/packages/chord/test/next/host/retry-on.test.ts b/packages/chord/test/next/host/retry-on.test.ts new file mode 100644 index 000000000..5677b6c28 --- /dev/null +++ b/packages/chord/test/next/host/retry-on.test.ts @@ -0,0 +1,204 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method } from "../../../src/next/contracts/index.ts"; +import { broken, ChordError } from "../../../src/next/data/index.ts"; +import type { FacetHost, Self, StoreSelf } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { expectCollected, failLive, flushed, instancesOf, loaded } from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +/** The models registry: the number of models contributed; it throws when told to. */ +const Models = defineRegistry("app.models", { version: 1 }); + +const Runtime = local("app.runtime", { version: 1, params: {}, members: { set: method(int()) } }); + +/** Serves `Models` (throwing for a negative sum) and `Runtime`, contributing a writable count of models. */ +function serveModels(h: FacetHost): void { + h.serve(Models, { + compose: (table) => { + const sum = table.reduce((n, { value }) => n + value, 0); + if (sum < 0) throw new Error("negative"); + return sum; + }, + }); + h.serve(Runtime, (_, self) => { + const count = self.writable(0); + self.contribute(Models, count); + return { + set: async (n: number) => { + count.set(n); + return null; + }, + }; + }); +} + +const User = local("app.user", { version: 1, params: { id: int() }, members: {} }); + +/** + * Serves `User`, whose open reads the models first and fails broken when there were none (after `wait`, if given); + * `opens` counts its attempts. + */ +function serveUser(h: FacetHost, opens: { n: number }, wait?: () => Promise): void { + h.serve(User, (_, self) => { + const models = self.registry(Models); + self.retryOn(models); + self.open( + async () => { + const seen = models.value; + opens.n++; + await wait?.(); + if (seen === 0) throw broken(new ChordError({ code: "unavailable", message: "no model" })); + }, + () => {}, + ); + return {}; + }); +} + +describe("retryOn", () => { + it("retries a failed object (a broken open) when a source gets a new value: the watches were installed before the open", async () => { + const opens = { n: 0 }; + const { host } = await loaded((h) => { + serveModels(h); + serveUser(h, opens); + }); + const user = host.hold(User, { id: 1 }, context); + await flushed(); + expect(user.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + await (await host.get(Runtime, {}, context)).set(1, context); + await flushed(); + expect(user.state.value).toEqual({ kind: "live" }); + expect(opens.n).toBe(2); + await host.close(context); + }); + + it("wakes on value calls only: a throwing compose wakes nothing, its next good composition does", async () => { + const opens = { n: 0 }; + const { host } = await loaded((h) => { + serveModels(h); + serveUser(h, opens); + }); + const user = host.hold(User, { id: 1 }, context); + await flushed(); + const runtime = await host.get(Runtime, {}, context); + await runtime.set(-1, context); + await flushed(); + expect(opens.n).toBe(1); + await runtime.set(2, context); + await flushed(); + expect(opens.n).toBe(2); + expect(user.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); + + it("remembers a wake while it activates: that attempt failing is retried at once", async () => { + const opens = { n: 0 }; + const gate = Promise.withResolvers(); + const { host } = await loaded((h) => { + serveModels(h); + serveUser(h, opens, async () => { + if (opens.n === 1) await gate.promise; + }); + }); + const runtime = await host.get(Runtime, {}, context); + const user = host.hold(User, { id: 1 }, context); + await flushed(); + // The first attempt read no model and waits; a model arrives meanwhile; then that attempt fails broken. + await runtime.set(1, context); + expect(user.state.value).toEqual({ kind: "loading" }); + gate.resolve(); + await flushed(); + expect(opens.n).toBe(2); + expect(user.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); + + it("ignores wakes while live", async () => { + const opens = { n: 0 }; + const { host } = await loaded((h) => { + serveModels(h); + h.serve(User, (_, self) => { + self.retryOn(self.registry(Models)); + self.open( + async () => void opens.n++, + () => {}, + ); + return {}; + }); + }); + const user = host.hold(User, { id: 1 }, context); + await flushed(); + await (await host.get(Runtime, {}, context)).set(3, context); + await flushed(); + expect(opens.n).toBe(1); + expect(user.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); + + it("refuses sources that end with an attempt: a reference's member, its own writable, `ref.state`", async () => { + const Other = local("app.other", { version: 1, params: {}, members: { n: int() } }); + const Picky = local("app.picky", { version: 1, params: { which: int() }, members: {} }); + const { host } = await loaded((h) => { + h.serve(Other, () => ({ n: constant(1) })); + h.serve(Picky, ({ which }, self) => { + const other = self.get(Other, {}); + const sources = [other.n, self.writable(1), other.state, constant(1)]; + self.retryOn(sources[which]!); + return {}; + }); + }); + for (const which of [0, 1, 2]) { + await expect(host.get(Picky, { which }, context)).rejects.toThrow( + `app.picky(${which}): self.retryOn takes registry values and constants (sources that outlive the attempt)`, + ); + } + (await host.get(Picky, { which: 3 }, context)).release(); + await host.close(context); + }); + + it("are the object's: a failed object that ends releases them, and is collected while the registry lives", async () => { + const opens = { n: 0 }; + const { host } = await loaded((h) => { + serveModels(h); + serveUser(h, opens); + }); + const refs = await (async () => { + const user = host.hold(User, { id: 1 }, context); + await flushed(); + user.release(); + await flushed(); + return objects.filter((ref) => ref.deref()?.label === "app.user(1)"); + })(); + expect(refs.length).toBeGreaterThan(0); + await expectCollected(...refs); + await host.close(context); + }); + + it("does not wake an object by its own contributions leaving (it failed live: no retry without backoff)", async () => { + const Contributor = local("app.contributor", { version: 1, params: {}, members: {} }); + let contributor: StoreSelf | undefined; + const { host } = await loaded((h) => { + serveModels(h); + h.serve(Contributor, (_, self) => { + contributor = self; + self.contribute(Models, 1); + self.retryOn(self.registry(Models)); + return {}; + }); + }); + const models = await host.registry(Models, context); + const watched: number[] = []; + (await models.watch(context)).start(async (call) => void (call.kind === "value" && watched.push(call.value))); + failLive(objects, contributor as Self, new ChordError({ code: "unavailable", message: "boom" })); + await flushed(); + const reference = host.hold(Contributor, {}, context); + await flushed(); + expect(reference.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + expect(watched).toEqual([0]); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/root.test.ts b/packages/chord/test/next/host/root.test.ts new file mode 100644 index 000000000..76c1c2b7e --- /dev/null +++ b/packages/chord/test/next/host/root.test.ts @@ -0,0 +1,175 @@ +import { getEventListeners } from "node:events"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withAbortSignal, withCancel } from "../../../src/context/index.ts"; +import { int, local, method } from "../../../src/next/contracts/index.ts"; +import { type Ctx, createHost, defineFacet, sleep } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { expectCollected, flushed, loaded, rejection, type TestPrincipal } from "../support.ts"; + +const Info = local("app.info", { version: 1, params: { id: int() }, members: { id: int(), wait: method() } }); + +/** Serves `Info`; its `wait` waits until the caller's context aborts. */ +function serveInfo(h: Parameters[0]>[0]): void { + h.serve(Info, ({ id }) => ({ + id: constant(id), + wait: async (ctx) => { + await sleep(60_000, ctx); + return null; + }, + })); +} + +describe("host.root", () => { + it("gives fn a call context: the host's principal (or the given one), this host's incarnation, `ctx.get`", async () => { + const host = createHost({ principal: { kind: "server" }, onError: () => {} }); + await host.load([{ name: "@app/info", facet: defineFacet(serveInfo) }], context); + const seen = await host.root(async (ctx) => { + const info = await ctx.get(Info, { id: 7 }); + return { principal: ctx.principal, incarnation: ctx.incarnation, id: info.id.value }; + }, context); + expect(seen).toMatchObject({ principal: { kind: "server" }, id: 7 }); + const owner = { kind: "owner", access: "local" } as const; + expect(await host.root(async (ctx) => ctx.principal, context, { principal: owner })).toEqual(owner); + await host.close(context); + }); + + it("seals the context when fn settles: its references released, its signal aborted, `ctx.get` refused `released`", async () => { + const { host } = await loaded(serveInfo); + let kept: Ctx | undefined; + const info = await host.root(async (ctx) => { + kept = ctx; + return ctx.get(Info, { id: 1 }); + }, context); + expect(info.state.value).toMatchObject({ kind: "ended", error: { code: "released" } }); + expect(kept!.abortSignal?.aborted).toBe(true); + expect(await rejection(kept!.get(Info, { id: 1 }))).toMatchObject({ code: "released", reason: "settled" }); + // A throwing fn seals too, and rejects with its error. + await expect( + host.root(async () => { + throw new Error("fn failed"); + }, context), + ).rejects.toThrow("fn failed"); + await host.close(context); + }); + + it("an abort rejects at once and seals: a running call through its reference rejects, a later get is `released`", async () => { + const { host } = await loaded(serveInfo); + const { context: cancellable, cancel } = withCancel(context); + let kept: Ctx | undefined; + let call: Promise | undefined; + const running = host.root(async (ctx) => { + kept = ctx; + const info = await ctx.get(Info, { id: 1 }); + call = info.wait(context); + // Uncooperative: it waits on nothing of Chord's. + await new Promise(() => {}); + }, cancellable); + await flushed(); + cancel(new Error("stop")); + expect(await rejection(running)).toEqual(new Error("stop")); + expect(await rejection(call!)).toMatchObject({ code: "released" }); + expect(await rejection(kept!.get(Info, { id: 2 }))).toMatchObject({ code: "released", reason: "settled" }); + await host.close(context); + }); + + it("an aborted context rejects before fn runs; a synchronous throw is a rejection", async () => { + const { host } = await loaded(serveInfo); + const { context: aborted, cancel } = withCancel(context); + cancel(new Error("gone")); + let ran = false; + const fn = async (): Promise => { + ran = true; + }; + await expect(host.root(fn, aborted)).rejects.toThrow("gone"); + expect(ran).toBe(false); + await expect( + host.root( + (() => { + throw new Error("sync"); + }) as never, + context, + ), + ).rejects.toThrow("sync"); + await host.close(context); + }); + + it("a `ctx.get` before `load` resolved parks until it did", async () => { + const host = createHost({ onError: () => {} }); + const result = host.root(async (ctx) => (await ctx.get(Info, { id: 3 })).id.value, context); + await flushed(); + await host.load([{ name: "@app/info", facet: defineFacet(serveInfo) }], context); + expect(await result).toBe(3); + await host.close(context); + }); + + it("`host.close` seals every open scope (an fn sleeping on its ctx ends); after it, `root` rejects `released`", async () => { + const { host } = await loaded(serveInfo); + const sleeping = rejection(host.root((ctx) => sleep(60_000, ctx), context)); + await flushed(); + await host.close(context); + expect(await sleeping).toMatchObject({ code: "released", reason: "closed" }); + expect(await rejection(host.root(async () => 1, context))).toMatchObject({ code: "released", reason: "closed" }); + }); + + it("aborts the scope's own signal when fn settles and at `host.close`, the caller's context having one (Fable, Opus, Sol)", async () => { + const { host } = await loaded(serveInfo); + const settled = withCancel(context); + let kept: Ctx | undefined; + await host.root(async (ctx) => { + kept = ctx; + }, settled.context); + expect(kept!.abortSignal?.aborted).toBe(true); + expect(settled.context.abortSignal?.aborted).toBe(false); + const closing = withCancel(context); + let slept: "running" | "ended" = "running"; + const sleeping = host.root(async (ctx) => { + try { + await sleep(60_000, ctx); + } finally { + slept = "ended"; + } + }, closing.context); + const outcome = rejection(sleeping); + await flushed(); + await host.close(context); + await flushed(); + expect(slept).toBe("ended"); + expect(closing.context.abortSignal?.aborted).toBe(false); + expect(await outcome).toMatchObject({ code: "released", reason: "closed" }); + }); + + it("`host.close` rejects the caller at once, whatever an uncooperative fn still does (Sol)", async () => { + const { host } = await loaded(serveInfo); + const running = host.root(() => new Promise(() => {}), context); + const outcome = running.then( + () => "resolved", + (error: unknown) => error, + ); + await host.close(context); + expect(await outcome).toMatchObject({ code: "released", reason: "closed" }); + }); + + it("leaves no listener on the caller's signal once `host.close` rejected an fn that never settles (Astra)", async () => { + const controller = new AbortController(); + const host = createHost({ onError: () => {} }); + const running = host.root(async () => new Promise(() => {}), withAbortSignal(controller.signal, context)); + const rejected = running.catch((error: unknown) => error); + await host.close(context); + expect(await rejected).toMatchObject({ code: "released", reason: "closed" }); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + }); + + it("keeps nothing once sealed: its scope and references are collected while the caller's signal lives on", async () => { + const { host } = await loaded(serveInfo); + const { context: long } = withCancel(context); + const refs = await (async () => { + const made: WeakRef[] = []; + await host.root(async (ctx) => { + made.push(new WeakRef(ctx), new WeakRef(await ctx.get(Info, { id: 1 }))); + }, long); + return made; + })(); + await expectCollected(...refs); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/registries/entries.test.ts b/packages/chord/test/next/registries/entries.test.ts new file mode 100644 index 000000000..88bd61003 --- /dev/null +++ b/packages/chord/test/next/registries/entries.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { + keyedEntries, + orderedEntries, + type RegistryConflict, + type RegistryTable, +} from "../../../src/next/registries/index.ts"; + +const table: RegistryTable<{ readonly routes?: Readonly>; readonly items?: readonly string[] }> = + [ + { plugin: "@a/one", version: 1, value: { routes: { "/x": "one", "/y": "one" }, items: ["late", "first"] } }, + { plugin: "@b/two", type: "b.store", version: 1, value: { routes: { "/x": "two" }, items: ["second"] } }, + { plugin: "@c/three", version: 1, value: {} }, + ]; + +describe("keyedEntries", () => { + it("maps every entry's keys with their plugin; a duplicate is reported and the first by plugin name keeps it", () => { + const conflicts: RegistryConflict[] = []; + const map = keyedEntries( + table, + (value) => value.routes, + (conflict) => conflicts.push(conflict), + ); + expect([...map]).toEqual([ + ["/x", { value: "one", plugin: "@a/one" }], + ["/y", { value: "one", plugin: "@a/one" }], + ]); + expect(conflicts).toEqual([{ message: '"/x" is contributed twice', plugins: ["@a/one", "@b/two"] }]); + }); +}); + +describe("orderedEntries", () => { + it("lists every entry's items in table order, with their plugin and type", () => { + expect(orderedEntries(table, (value) => value.items)).toEqual([ + { value: "late", plugin: "@a/one" }, + { value: "first", plugin: "@a/one" }, + { value: "second", plugin: "@b/two", type: "b.store" }, + ]); + }); + + it("sorts by `orderOf`, table order kept among equals", () => { + const order = (item: string) => (item === "first" ? 0 : 1); + expect(orderedEntries(table, (value) => value.items, order).map(({ value }) => value)).toEqual([ + "first", + "late", + "second", + ]); + }); +}); diff --git a/packages/chord/test/next/registries/record.test.ts b/packages/chord/test/next/registries/record.test.ts new file mode 100644 index 000000000..c25b27fcf --- /dev/null +++ b/packages/chord/test/next/registries/record.test.ts @@ -0,0 +1,649 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { bool, int, local, method, obj } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import type { HostError } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { + defineRegistry, + keyedEntries, + type Registry, + type RegistryTable, + type TableItem, +} from "../../../src/next/registries/index.ts"; +import type { NativeWatchCall } from "../../../src/watch.ts"; +import { + delivered, + expectCollected, + failures, + flushed, + instancesOf, + loadedPlugins, + metered, + storeRestartable, +} from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +/** A registry whose composition is its table as `plugin:type=value` strings. */ +const Names = defineRegistry("app.names", { version: 1 }); + +const listed = (table: RegistryTable): readonly string[] => + table.map(({ plugin, type, value }) => `${plugin}:${type ?? ""}=${value}`); + +/** A host-lifetime object contributing a writable it sets with `set`. */ +const Setter = local("app.setter", { version: 1, params: {}, members: { set: method(int()) } }); + +/** Serves `Setter` contributing a writable of `R`'s contribution type, `initial` first, `set(n)` makes `of(n)`. */ +function setter(R: Registry, initial: C, of: (n: number) => C) { + return (host: Parameters[0][string]>[0]) => + host.serve(Setter, (_, self) => { + const value = self.writable(initial); + self.contribute(R, value); + return { + set: async (n: number) => { + value.set(of(n)); + return null; + }, + }; + }); +} + +function kinds(errors: readonly HostError[]): string[] { + return errors.map((error) => error.kind); +} + +describe("the table", () => { + it("is sorted by plugin name, then type, a facet first: never load order", async () => { + const { host } = await loadedPlugins({ + "@z/late": (_, self) => self.contribute(Names, "z"), + "@a/early": (host, self) => { + self.contribute(Names, "a"); + setter(Names, "store", String)(host); + }, + "@m/provider": (host) => host.serve(Names, { compose: listed }), + }); + const value = await host.registry(Names, context); + expect(value.value).toEqual(["@a/early:=a", "@a/early:app.setter=store", "@z/late:=z"]); + await host.close(context); + }); + + it("adapts an older contribution once per value; a newer one, or an older one without an adapter, is left out and reported once", async () => { + const V3 = defineRegistry("app.versions", { version: 3 }); + const V2 = defineRegistry<{ old: string }, unknown>("app.versions", { version: 2 }); + const V1 = defineRegistry("app.versions", { version: 1 }); + const V4 = defineRegistry("app.versions", { version: 4 }); + const adapted: string[] = []; + const { host, errors } = await loadedPlugins({ + "@app/provider": (host) => + host.serve(V3, { + compose: listed, + accepts: { + 2: { + contribution: (old: { old: string }) => { + adapted.push(old.old); + return `adapted ${old.old}`; + }, + }, + }, + }), + "@app/one": (_, self) => self.contribute(V1, "one"), + "@app/two": setter(V2, { old: "first" }, (n) => ({ old: `${n}` })), + "@app/three": (_, self) => self.contribute(V3, "three"), + "@app/four": (_, self) => self.contribute(V4, "four"), + }); + const value = await host.registry(V3, context); + expect(value.value).toEqual(["@app/three:=three", "@app/two:app.setter=adapted first"]); + expect(errors).toEqual([ + { kind: "contribution_refused", registry: "app.versions", plugin: "@app/four", version: 4 }, + { kind: "contribution_refused", registry: "app.versions", plugin: "@app/one", version: 1 }, + ]); + const setters = await host.get(Setter, {}, context); + await setters.set(5, context); + expect(value.value).toEqual(["@app/three:=three", "@app/two:app.setter=adapted 5"]); + // Each value adapted once, though the table was built again; no refusal reported twice. + expect(adapted).toEqual(["first", "5"]); + expect(kinds(errors)).toEqual(["contribution_refused", "contribution_refused"]); + await host.close(context); + }); + + it("takes a contribution declared before its provider's setup into the first composition", async () => { + let composed: readonly string[] | undefined; + const { host } = await loadedPlugins({ + "@app/a": (_, self) => self.contribute(Names, "first"), + "@app/provider": (host) => + host.serve(Names, { + compose: (table) => { + composed = listed(table); + return composed; + }, + }), + }); + expect(composed).toEqual(["@app/a:=first"]); + await host.close(context); + }); +}); + +describe("compose", () => { + it("composes every served registry once at the end of load's setups, an empty table too, before any object activates", async () => { + const { telemetry, counts } = await metered(); + const Other = defineRegistry("app.other", { version: 1 }); + const Reader = local("app.reader", { version: 1, params: {}, members: { seen: int() } }); + let seen: readonly string[] | undefined; + const { host } = await loadedPlugins( + { + "@app/provider": (host, self) => { + host.serve(Names, { compose: listed }); + host.serve(Other, { compose: (table) => table.length }); + self.contribute(Names, "a"); + host.serve(Reader, (_, self) => { + const names = self.registry(Names); + self.open( + async () => { + seen = names.value; + }, + () => {}, + ); + return { seen: self.derive([self.registry(Other)], ([n]) => n) }; + }); + }, + }, + { telemetry }, + ); + expect(seen).toEqual(["@app/provider:=a"]); + expect((await host.get(Reader, {}, context)).seen.value).toBe(0); + expect(counts()["chord.registry.composes"]).toBe(2); + await host.close(context); + }); + + it("is lazy: a change marks the table, a read composes it once; no compose for an unchanged table", async () => { + const { telemetry, counts } = await metered(); + const Flags = defineRegistry("app.flags", { version: 1 }); + const Doc = defineState("app.flag", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 1 }) }); + const Flagger = local("app.flagger", { version: 1, params: {}, members: { set: method(int()) } }); + const { host } = await loadedPlugins( + { + "@app/flags": (host) => { + host.serve(Flags, { compose: (table) => table.filter(({ value }) => value).length }); + host.serve(Flagger, (_, self) => { + const doc = self.state(Doc); + self.contribute( + Flags, + self.derive([doc], ([d]) => d.n > 0), + ); + return { + set: async (n: number, ctx) => { + await doc.change((d) => { + d.n = n; + }, ctx); + return null; + }, + }; + }); + }, + }, + { telemetry }, + ); + const flags = await host.registry(Flags, context); + const flagger = await host.get(Flagger, {}, context); + expect(flags.value).toBe(1); + const base = counts()["chord.registry.composes"]!; + expect(flags.value).toBe(1); + expect(counts()["chord.registry.composes"]).toBe(base); + // The derive changes, its value does not: the table stays, nothing composes. + await flagger.set(2, context); + await flagger.set(3, context); + expect(flags.value).toBe(1); + expect(counts()["chord.registry.composes"]).toBe(base); + // Nobody reads it: nothing composes until a read. + await flagger.set(-1, context); + await flagger.set(-2, context); + expect(counts()["chord.registry.composes"]).toBe(base); + expect(flags.value).toBe(0); + expect(counts()["chord.registry.composes"]).toBe(base + 1); + await host.close(context); + }); + + it("composes a watched registry in the batch of the commit that changed a contribution, once per batch", async () => { + const { telemetry, counts } = await metered(); + const Doc = defineState("app.names", { + schema: obj({ a: int(), b: int() }), + version: 1, + initial: () => ({ a: 0, b: 0 }), + }); + const Two = local("app.two", { version: 1, params: {}, members: { both: method(int()), sum: int() } }); + const Sum = defineRegistry<{ readonly a: number; readonly b: number }, number>("app.sum", { version: 1 }); + const { host } = await loadedPlugins( + { + "@app/sum": (host) => { + host.serve(Sum, { compose: (table) => table.reduce((n, { value }) => n + value.a + value.b, 0) }); + host.serve(Two, (_, self) => { + const doc = self.state(Doc); + self.contribute(Sum, doc); + return { + sum: self.derive([self.registry(Sum)], ([n]) => n), + both: async (n: number, ctx) => { + await self.tx(async (tx) => { + const d = await tx.draft(doc); + d.a = n; + d.b = n; + }, ctx); + return null; + }, + }; + }); + }, + }, + { telemetry }, + ); + const two = await host.get(Two, {}, context); + const sums: number[] = []; + const watch = await two.sum.watch(context); + watch.start(async (call) => { + if (call.kind === "value") sums.push(call.value); + }); + const base = counts()["chord.registry.composes"]!; + await two.both(2, context); + await delivered(); + // One commit, both fields: one compose in its batch, one frame of the derive over it. + expect(sums).toEqual([4]); + expect(counts()["chord.registry.composes"]).toBe(base + 1); + await host.close(context); + }); + + it("reports a conflict once per episode", async () => { + const Routes = defineRegistry>, ReadonlyMap>>( + "app.routes", + { version: 1 }, + ); + const { host, errors } = await loadedPlugins({ + "@app/a": (_, self) => self.contribute(Routes, { "/x": "a" }), + "@app/b": setter(Routes, { "/x": "b" } as Readonly>, (n) => ({ + "/x": "b", + [`/${n}`]: "b", + })), + "@app/http": (host) => + host.serve(Routes, { compose: (table, { report }) => keyedEntries(table, (c) => c, report) }), + }); + const routes = await host.registry(Routes, context); + expect(routes.value.get("/x")).toEqual({ value: "a", plugin: "@app/a" }); + const setters = await host.get(Setter, {}, context); + await setters.set(1, context); + void routes.value; + await setters.set(2, context); + void routes.value; + expect(errors).toEqual([ + { + kind: "registry_conflict", + registry: "app.routes", + message: '"/x" is contributed twice', + plugins: ["@app/a", "@app/b"], + }, + ]); + await host.close(context); + }); + + it("a throwing compose: reported once, a failed call, `.value` keeps the last good composition; the next good one heals", async () => { + const Count = defineRegistry("app.count", { version: 1 }); + const { host, errors } = await loadedPlugins({ + "@app/count": (host) => { + host.serve(Count, { + compose: (table) => { + const sum = table.reduce((n, { value }) => n + value, 0); + if (sum < 0) throw new Error(`negative: ${sum}`); + return sum; + }, + }); + setter(Count, 1, (n) => n)(host); + }, + }); + const count = await host.registry(Count, context); + const calls: NativeWatchCall[] = []; + const watch = await count.watch(context); + watch.start(async (call) => void calls.push(call)); + const setters = await host.get(Setter, {}, context); + await setters.set(-1, context); + await setters.set(-2, context); + await delivered(); + expect(count.value).toBe(1); + expect(errors).toEqual([{ kind: "registry_failed", registry: "app.count", error: new Error("negative: -1") }]); + await setters.set(1, context); + await delivered(); + // The heal is a value call, also when its value is the last good one. + expect(calls).toEqual([ + { kind: "failed", error: new Error("negative: -1") }, + { kind: "value", value: 1 }, + ]); + await setters.set(3, context); + await delivered(); + expect(calls.at(-1)).toEqual({ kind: "value", value: 3 }); + await host.close(context); + }); + + it("before any good composition, `.value` and `host.registry` throw the error", async () => { + const Broken = defineRegistry("app.broken", { version: 1 }); + const { host, errors } = await loadedPlugins({ + "@app/broken": (host) => + host.serve(Broken, { + compose: () => { + throw new Error("bug"); + }, + }), + }); + await expect(host.registry(Broken, context)).rejects.toThrow("bug"); + expect(kinds(errors)).toEqual(["registry_failed"]); + await host.close(context); + }); +}); + +describe("contributions", () => { + it("a watchable contribution that fails is left out and reported once, and is back once it heals", async () => { + const Doc = defineState("app.n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 1 }) }); + const Failing = local("app.failing", { version: 1, params: {}, members: { set: method(int()) } }); + const { host, errors } = await loadedPlugins({ + "@app/names": (host, self) => { + host.serve(Names, { compose: listed }); + self.contribute(Names, "facet"); + host.serve(Failing, (_, self) => { + const doc = self.state(Doc); + self.contribute( + Names, + self.derive([doc], ([d]) => { + if (d.n < 0) throw new Error("negative"); + return `n${d.n}`; + }), + ); + return { + set: async (n: number, ctx) => { + await doc.change((d) => { + d.n = n; + }, ctx); + return null; + }, + }; + }); + }, + }); + const names = await host.registry(Names, context); + const failing = await host.get(Failing, {}, context); + expect(names.value).toEqual(["@app/names:=facet", "@app/names:app.failing=n1"]); + await failing.set(-1, context); + expect(names.value).toEqual(["@app/names:=facet"]); + await failing.set(-2, context); + expect(names.value).toEqual(["@app/names:=facet"]); + await failing.set(2, context); + expect(names.value).toEqual(["@app/names:=facet", "@app/names:app.failing=n2"]); + expect(errors.filter(({ kind }) => kind === "contribution_failed")).toEqual([ + { + kind: "contribution_failed", + registry: "app.names", + plugin: "@app/names", + type: "app.failing", + error: new Error("negative"), + }, + ]); + await host.close(context); + }); + + it("a contribution adapter that throws leaves that contribution out (`contribution_failed`)", async () => { + const V2 = defineRegistry("app.adapted", { version: 2 }); + const V1 = defineRegistry("app.adapted", { version: 1 }); + const { host, errors } = await loadedPlugins({ + "@app/provider": (host, self) => { + host.serve(V2, { + compose: listed, + accepts: { + 1: { + contribution: () => { + throw new Error("cannot adapt"); + }, + }, + }, + }); + self.contribute(V2, "current"); + }, + "@app/old": (_, self) => self.contribute(V1, "old"), + }); + expect((await host.registry(V2, context)).value).toEqual(["@app/provider:=current"]); + expect(errors).toEqual([ + { kind: "contribution_failed", registry: "app.adapted", plugin: "@app/old", error: new Error("cannot adapt") }, + ]); + await host.close(context); + }); + + it("refuses a contribution closing a cycle with the registries' tables; one through a derive is cut and reported", async () => { + const A = defineRegistry("app.a", { version: 1 }); + const B = defineRegistry("app.b", { version: 1 }); + const Direct = local("app.direct", { version: 1, params: {}, members: {} }); + const Late = local("app.late", { version: 1, params: {}, members: {} }); + const Derived = local("app.derived", { version: 1, params: {}, members: {} }); + const Back = local("app.back", { version: 1, params: {}, members: {} }); + let lateAttempts = 0; + const { host, errors } = await loadedPlugins({ + "@app/cycles": (host) => { + host.serve(A, { compose: (table) => table.length }); + host.serve(B, { compose: (table) => table.length }); + host.serve(Direct, (_, self) => { + self.contribute(A, self.registry(B)); + return {}; + }); + // Its first attempt is discarded (the factory throws), then, once `Direct` is live, it closes a cycle. + host.serve(Late, (_, self) => { + self.contribute(B, self.registry(A)); + if (++lateAttempts === 1) throw new Error("first attempt"); + return {}; + }); + host.serve(Derived, (_, self) => { + self.contribute( + A, + self.derive([self.registry(B)], ([n]) => n as number), + ); + return {}; + }); + }, + "@app/back": (host) => + host.serve(Back, (_, self) => { + self.contribute( + B, + self.derive([self.registry(A)], ([n]) => n as number), + ); + return {}; + }), + }); + const late = host.hold(Late, {}, context); + late.retry(); + await flushed(); + expect(late.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "broken", error: { code: "invalid", reason: "cycle" } }, + }); + // A reads B's derive, which reads A while A composes: that contribution is cut, A composes without the cycle. + expect((await host.registry(A, context)).value).toBe(2); + expect(errors.find((error) => error.kind === "contribution_failed")).toMatchObject({ + registry: "app.b", + plugin: "@app/back", + error: { code: "invalid", reason: "cycle" }, + }); + late.release(); + await host.close(context); + }); + + it("refuses a cycle declared within one factory: broken, invalid/cycle (Sol)", async () => { + const A = defineRegistry("app.fix-a", { version: 1 }); + const B = defineRegistry("app.fix-b", { version: 1 }); + const Both = local("app.direct-cycle", { version: 1, params: {}, members: {} }); + const { host, errors } = await loadedPlugins({ + "@app/cycle": (h) => { + h.serve(A, { compose: (table) => table.length }); + h.serve(B, { compose: (table) => table.length }); + h.serve(Both, (_, self) => { + self.contribute(A, self.registry(B)); + self.contribute(B, self.registry(A)); + return {}; + }); + }, + }); + expect( + errors.find((error) => error.kind === "activation_failed" && error.object === "app.direct-cycle"), + ).toMatchObject({ failure: { kind: "broken", error: { code: "invalid", reason: "cycle" } } }); + await host.close(context); + }); + + it("leaves no cycle edge from a contribution that never entered a table (Sol)", async () => { + const A = defineRegistry("app.edge-a", { version: 1 }); + const B = defineRegistry("app.edge-b", { version: 1 }); + const Failed = local("app.failed", { version: 1, params: {}, members: {} }); + const Good = local("app.good", { version: 1, params: {}, members: {} }); + const { host, errors } = await loadedPlugins({ + "@app/a": (h) => { + h.serve(A, { compose: (table) => table.length }); + h.serve(B, { compose: (table) => table.length }); + h.serve(Failed, (_, self) => { + self.contribute(A, self.registry(B)); + throw new Error("factory discarded"); + }); + h.serve(Good, (_, self) => { + self.contribute(B, self.registry(A)); + return {}; + }); + }, + }); + expect(failures(errors)).toEqual(["app.failed: broken"]); + await host.close(context); + }); + + it("follows only Chord watchables: anything else is a plain value, whatever methods it has (Sol)", async () => { + const Natives = defineRegistry<{ watch(): string; readonly value: string }, readonly string[]>("app.natives", { + version: 1, + }); + const native = { value: "a handle", watch: () => "a native operation" }; + const { host } = await loadedPlugins({ + "@app/x": (h, self) => { + h.serve(Natives, { compose: (table) => table.map(({ value }) => value.watch()) }); + self.contribute(Natives, native); + }, + }); + expect((await host.registry(Natives, context)).value).toEqual(["a native operation"]); + await host.close(context); + }); + + it("keeps an `undefined` contribution in the table (Sol, Astra)", async () => { + const Nothing = defineRegistry("app.undefined", { version: 1 }); + const { host } = await loadedPlugins({ + "@app/a": (h, self) => { + h.serve(Nothing, { compose: (table) => table.length }); + self.contribute(Nothing, undefined); + }, + }); + expect((await host.registry(Nothing, context)).value).toBe(1); + await host.close(context); + }); + + it("leaves out a registry's value contributed to another while that registry's compose throws (Sol, Astra)", async () => { + const Source = defineRegistry("app.source", { version: 1 }); + const Target = defineRegistry("app.target", { version: 1 }); + const Bridge = local("app.bridge", { version: 1, params: {}, members: {} }); + const { host, errors } = await loadedPlugins({ + "@app/a": (h) => { + h.serve(Source, { + compose: (table) => { + const n = table.reduce((sum, { value }) => sum + value, 0); + if (n < 0) throw new Error("negative"); + return n; + }, + }); + h.serve(Target, { compose: (table) => table.map(({ value }) => value) }); + setter(Source, 1, (n) => n)(h); + h.serve(Bridge, (_, self) => { + self.contribute(Target, self.registry(Source)); + return {}; + }); + }, + }); + const target = await host.registry(Target, context); + expect(target.value).toEqual([1]); + const setters = await host.get(Setter, {}, context); + await setters.set(-1, context); + // The source's readers keep its last good value; as a contribution it is failed: left out, reported. + expect((await host.registry(Source, context)).value).toBe(1); + expect(target.value).toEqual([]); + expect(errors.find((error) => error.kind === "contribution_failed")).toMatchObject({ + registry: "app.target", + plugin: "@app/a", + type: "app.bridge", + error: new Error("negative"), + }); + await setters.set(3, context); + expect(target.value).toEqual([3]); + await host.close(context); + }); + + it("a followed contribution whose runtime ended its observer stays with its last value (R4: the source keeps it)", async () => { + let end: (() => void) | undefined; + const Follower = local("app.follower", { version: 1, params: {}, members: {} }); + const { host, errors } = await loadedPlugins({ + "@app/a": (h) => { + h.serve(Names, { compose: listed }); + h.serve(Follower, (_, self) => { + self.contribute( + Names, + self.follow(async (observer) => { + end = () => observer.end({ kind: "ended" }); + return { value: "followed", detach: () => {} }; + }), + ); + return {}; + }); + }, + }); + const names = await host.registry(Names, context); + expect(names.value).toEqual(["@app/a:app.follower=followed"]); + end!(); + await flushed(); + expect(names.value).toEqual(["@app/a:app.follower=followed"]); + expect(errors.filter(({ kind }) => kind === "contribution_failed")).toEqual([]); + await host.close(context); + }); + + it("an object's contribution is in the table from live until it fails, again once retried, until it ends", async () => { + const { host, fail, retry } = await storeRestartable( + objects, + (attempt, self) => self.contribute(Names, `attempt ${attempt}`), + { serve: (host) => host.serve(Names, { compose: listed }) }, + ); + const names = await host.registry(Names, context); + expect(names.value).toEqual(["@test/app:app.store=attempt 1"]); + fail(); + expect(names.value).toEqual([]); + await retry(); + expect(names.value).toEqual(["@test/app:app.store=attempt 2"]); + await host.close(context); + // Closed: nothing composes any more; a value a reader keeps reads the last composition. + expect(names.value).toEqual(["@test/app:app.store=attempt 2"]); + }); +}); + +describe("the table's memory", () => { + it("keeps nothing of a closed host: records, tables and followed contributions go with it", async () => { + const Doc = defineState("app.kept", { schema: obj({ on: bool() }), version: 1, initial: () => ({ on: true }) }); + const Kept = local("app.kept", { version: 1, params: {}, members: {} }); + const refs = await (async () => { + const { host } = await loadedPlugins({ + "@app/kept": (host) => { + host.serve(Names, { compose: listed }); + host.serve(Kept, (_, self) => { + self.contribute( + Names, + self.derive([self.state(Doc)], ([d]) => `${d.on}`), + ); + return {}; + }); + }, + }); + const names = new WeakRef(await host.registry(Names, context)); + await host.close(context); + return [names, new WeakRef(host)]; + })(); + await expectCollected(...refs); + }); +}); diff --git a/packages/chord/test/next/registries/registry.test.ts b/packages/chord/test/next/registries/registry.test.ts new file mode 100644 index 000000000..61db2cd14 --- /dev/null +++ b/packages/chord/test/next/registries/registry.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, expectTypeOf, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { defineRegistry, type Registry, type RegistryValue } from "../../../src/next/registries/index.ts"; +import { loadedPlugins } from "../support.ts"; + +describe("tokens", () => { + it("carry a name, a version and the types only; names and versions are checked at declaration", () => { + const Http = defineRegistry<{ readonly path: string }, readonly string[]>("chord.http", { version: 2 }); + expect(Http).toEqual({ kind: "registry", name: "chord.http", version: 2 }); + expectTypeOf(Http).toEqualTypeOf>(); + expect(() => defineRegistry("Chord.Http", { version: 1 })).toThrow('registry "Chord.Http": expected lowercase'); + expect(() => defineRegistry("chord.http", { version: 0 })).toThrow( + "chord.http.version: expected a positive integer", + ); + }); +}); + +describe("providers", () => { + const Names = defineRegistry("app.names", { version: 1 }); + + it("one per host: a second serve is refused at load", async () => { + const host = createHost({ onError: () => {} }); + const serve = defineFacet((h) => h.serve(Names, { compose: (table) => table.length })); + await expect( + host.load( + [ + { name: "@app/a", facet: serve }, + { name: "@app/b", facet: serve }, + ], + context, + ), + ).rejects.toThrow("registry app.names is served twice"); + }); + + it("a provider without compose is refused at load (plugin code is not type-checked)", async () => { + await expect(loadedPlugins({ "@app/a": (h) => h.serve(Names, {} as never) })).rejects.toThrow( + "registry app.names: compose is a function", + ); + }); + + it("a registry whose provider comes with `load` heals a `host.registry` made before it", async () => { + const host = createHost({ onError: () => {} }); + const waiting = host.registry(Names, context); + await host.load( + [{ name: "@app/a", facet: defineFacet((h) => h.serve(Names, { compose: (t) => t.length })) }], + context, + ); + const value = await waiting; + expectTypeOf(value).toEqualTypeOf>(); + expect(value.value).toBe(0); + await host.close(context); + }); + + it("are served only while the setup runs", async () => { + let later: Parameters[0]>[0] | undefined; + const { host } = await loadedPlugins({ + "@app/a": (h) => { + later = h; + }, + }); + expect(() => later!.serve(Names, { compose: (t) => t.length })).toThrow( + "@app/a: host.serve is valid only in the facet's setup", + ); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/registries/value.test.ts b/packages/chord/test/next/registries/value.test.ts new file mode 100644 index 000000000..e23722948 --- /dev/null +++ b/packages/chord/test/next/registries/value.test.ts @@ -0,0 +1,398 @@ +import { getEventListeners } from "node:events"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withAbortSignal, withCancel } from "../../../src/context/index.ts"; +import { int, local, method, str } from "../../../src/next/contracts/index.ts"; +import type { defineFacet } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { defineRegistry, type Registry } from "../../../src/next/registries/index.ts"; +import type { NativeWatchCall, WatchEnd } from "../../../src/watch.ts"; +import { + expectCollected, + failures, + flushed, + instancesOf, + loadedPlugins, + metered, + rejection, + settledHeap, +} from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +/** The provider's version 2: contributions are numbers, the value their sum. */ +const Sum = defineRegistry("app.sum", { version: 2 }); + +/** A reader built against version 1: its value was a plain number. */ +const SumV1 = defineRegistry("app.sum", { version: 1 }); + +/** A reader built against a version the provider does not know yet. */ +const SumV3 = defineRegistry("app.sum", { version: 3 }); + +const Setter = local("app.setter", { version: 1, params: {}, members: { set: method(int()) } }); + +/** Serves `Setter`, contributing a writable number to `Sum`. */ +function setter(host: Parameters[0]>[0], initial = 1): void { + host.serve(Setter, (_, self) => { + const value = self.writable(initial); + self.contribute(Sum, value); + return { + set: async (n: number) => { + value.set(n); + return null; + }, + }; + }); +} + +const total = (table: readonly { readonly value: number }[]) => ({ + total: table.reduce((n, { value }) => n + value, 0), +}); + +describe("older readers (S2)", () => { + it("get the composition mapped back with accepts[old].value, once per composition", async () => { + const { telemetry, counts } = await metered(); + const mapped: number[] = []; + const { host } = await loadedPlugins( + { + "@app/sum": (host) => { + host.serve(Sum, { + compose: total, + accepts: { + 1: { + value: (current) => { + mapped.push(current.total); + return current.total; + }, + }, + }, + }); + setter(host); + }, + }, + { telemetry }, + ); + const old = await host.registry(SumV1, context); + const current = await host.registry(Sum, context); + expect(old.value).toBe(1); + expect(old.value).toBe(1); + expect(current.value).toEqual({ total: 1 }); + await (await host.get(Setter, {}, context)).set(4, context); + expect(old.value).toBe(4); + expect(mapped).toEqual([1, 4]); + expect(counts()["chord.registry.adapts"]).toBe(2); + await host.close(context); + }); + + it("without a value mapping, and for a reader newer than the provider, fail `contract_mismatch`: broken, reported", async () => { + const Newer = local("app.newer", { version: 1, params: {}, members: { total: int() } }); + const Older = local("app.older", { version: 1, params: {}, members: { total: int() } }); + const { host, errors } = await loadedPlugins({ + "@app/sum": (host) => { + host.serve(Sum, { compose: total }); + host.serve(Newer, (_, self) => ({ total: self.derive([self.registry(SumV3)], ([v]) => v.total) })); + host.serve(Older, (_, self) => ({ total: self.derive([self.registry(SumV1)], ([v]) => v) })); + }, + }); + expect(await rejection(host.registry(SumV1, context))).toMatchObject({ code: "contract_mismatch" }); + expect(await rejection(host.registry(SumV3, context))).toMatchObject({ code: "contract_mismatch" }); + // Only a reload helps: no backoff, the mismatch itself the cause. + expect(failures(errors)).toEqual(["app.newer: broken", "app.older: broken"]); + expect(errors[0]).toMatchObject({ failure: { error: { code: "contract_mismatch" } } }); + await host.close(context); + }); + + it("maps an `undefined` composition (Sol, Astra)", async () => { + const Nothing = defineRegistry("app.nothing", { version: 2 }); + const NothingV1 = defineRegistry("app.nothing", { version: 1 }); + const { host } = await loadedPlugins({ + "@app/a": (h) => h.serve(Nothing, { compose: () => undefined, accepts: { 1: { value: () => "mapped" } } }), + }); + expect((await host.registry(NothingV1, context)).value).toBe("mapped"); + await host.close(context); + }); + + it("a failed mapping heals when the composition is the one mapped before (Astra)", async () => { + const Count = defineRegistry("app.count", { version: 2 }); + const CountV1 = defineRegistry("app.count", { version: 1 }); + const Counter = local("app.counter", { version: 1, params: {}, members: { set: method(int()) } }); + const { host } = await loadedPlugins({ + "@app/count": (h) => { + h.serve(Count, { + compose: (table) => table.reduce((n, { value }) => n + value, 0), + accepts: { + 1: { + value: (n) => { + if (n < 0) throw new Error("negative"); + return n; + }, + }, + }, + }); + h.serve(Counter, (_, self) => { + const value = self.writable(1); + self.contribute(Count, value); + return { + set: async (n: number) => { + value.set(n); + return null; + }, + }; + }); + }, + }); + const old = await host.registry(CountV1, context); + const calls: NativeWatchCall[] = []; + old.observe({ advance: (call) => calls.push(call), end: () => {} }, context); + const counter = await host.get(Counter, {}, context); + await counter.set(-1, context); + // Back to the composition mapped before (the same number): the mapping heals. + await counter.set(1, context); + expect(calls).toEqual([ + { kind: "failed", error: new Error("negative") }, + { kind: "value", value: 1 }, + ]); + await host.close(context); + }); +}); + +describe("readers (S3)", () => { + const Reader = local("app.reader", { version: 1, params: {}, members: { total: int() } }); + const serveReader = ( + host: Parameters[0]>[0], + R: Registry = Sum, + ) => host.serve(Reader, (_, self) => ({ total: self.derive([self.registry(R)], ([v]) => v.total) })); + + it("activate once the registry has a value; a reader of a registry nobody serves fails at once, `load` waits for nothing", async () => { + const started = Date.now(); + const { host, errors } = await loadedPlugins({ "@app/reader": (host) => serveReader(host) }); + expect(Date.now() - started).toBeLessThan(1_000); + expect(errors).toMatchObject([ + { + kind: "activation_failed", + object: "app.reader", + failure: { + kind: "transient", + error: { + code: "unavailable", + message: "app.reader: registry app.sum has no value", + cause: { code: "unavailable", reason: "undeclared" }, + }, + }, + }, + ]); + expect(await rejection(host.registry(Sum, context))).toMatchObject({ code: "unavailable", reason: "undeclared" }); + await host.close(context); + }); + + it("a reader failed on a first `compose` that threw is retried at once by the first good composition (R18)", async () => { + const { host, errors } = await loadedPlugins({ + "@app/sum": (host) => { + host.serve(Sum, { + compose: (table) => { + const sum = total(table); + if (sum.total === 0) throw new Error("empty"); + return sum; + }, + }); + serveReader(host); + }, + "@app/setter": (host) => setter(host, 0), + }); + expect(failures(errors)).toEqual(["app.reader: transient"]); + const setters = await host.get(Setter, {}, context); + await setters.set(3, context); + // The reader's backoff is a second at least: the composition retried it. + const reader = await host.get(Reader, {}, context, { readyTimeout: 500 }); + expect(reader.total.value).toBe(3); + await host.close(context); + }); + + it("once live, a later throwing `compose` changes nothing for the reader: `.value` keeps the last good one", async () => { + const { host } = await loadedPlugins({ + "@app/sum": (host) => { + host.serve(Sum, { + compose: (table) => { + const sum = total(table); + if (sum.total < 0) throw new Error("negative"); + return sum; + }, + }); + serveReader(host); + setter(host, 2); + }, + }); + const reader = await host.get(Reader, {}, context); + const setters = await host.get(Setter, {}, context); + expect(reader.total.value).toBe(2); + await setters.set(-1, context); + expect(reader.total.value).toBe(2); + expect(reader.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); +}); + +describe("readers' memory", () => { + it("an ended reader is collected while the registry lives: its derive and its read leave the value", async () => { + const Reader = local("app.reader", { version: 1, params: { id: int() }, members: { total: int() } }); + const { host } = await loadedPlugins({ + "@app/sum": (host) => { + host.serve(Sum, { compose: total, accepts: { 1: { value: ({ total }) => total } } }); + host.serve(Reader, (_, self) => ({ + total: self.derive([self.registry(Sum), self.registry(SumV1)], ([v, old]) => v.total + old), + })); + setter(host); + }, + }); + const refs = await (async () => { + const reader = await host.get(Reader, { id: 1 }, context); + expect(reader.total.value).toBe(2); + reader.release(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return objects.filter((ref) => ref.deref()?.label === "app.reader(1)"); + })(); + expect(refs.length).toBeGreaterThan(0); + await (await host.get(Setter, {}, context)).set(5, context); + await expectCollected(...refs); + await host.close(context); + }); +}); + +describe("values' memory", () => { + it("drops an older version's mapping once its last reader ended (Sol)", async () => { + const New = defineRegistry("app.version-gc", { version: 2 }); + const Old = defineRegistry("app.version-gc", { + version: 1, + }); + const Reader = local("app.old-reader", { version: 1, params: { id: int() }, members: { n: int() } }); + let mapped: WeakRef | undefined; + const { host } = await loadedPlugins({ + "@app/a": (h) => { + h.serve(New, { + compose: () => 1, + accepts: { + 1: { + value: (n) => { + const value = { n, payload: new Uint8Array(1_000_000) }; + mapped = new WeakRef(value); + return value; + }, + }, + }, + }); + h.serve(Reader, (_, self) => ({ n: self.derive([self.registry(Old)], ([v]) => v.n) })); + }, + }); + const reader = await host.get(Reader, { id: 1 }, context); + expect(reader.n.value).toBe(1); + reader.release(); + await flushed(); + await expectCollected(mapped!); + await host.close(context); + }); + + it("keeps no old composition once its readers stopped waiting and nobody follows it (Opus)", async () => { + const Big = defineRegistry("app.big", { version: 1 }); + let first = true; + const made: WeakRef[] = []; + const Waiting = local("app.waiting", { version: 1, params: {}, members: {} }); + const Counter = local("app.counter", { version: 1, params: {}, members: { set: method(int()) } }); + const { host } = await loadedPlugins({ + "@app/a": (h) => { + h.serve(Big, { + compose: (table) => { + if (first) throw new Error("first"); + const value = { payload: new Array(1000).fill(table.length) }; + made.push(new WeakRef(value)); + return value; + }, + }); + h.serve(Counter, (_, self) => { + const value = self.writable(0); + self.contribute(Big, value); + return { + set: async (n: number) => { + value.set(n); + return null; + }, + }; + }); + h.serve(Waiting, (_, self) => { + void self.registry(Big); + return {}; + }); + }, + }); + const counter = await host.get(Counter, {}, context); + first = false; + // The waiting reader's composition (pulled in the batch of this set) wakes it. + await counter.set(1, context); + await flushed(); + await counter.set(2, context); + void (await host.registry(Big, context)).value; + await settledHeap(); + // Only the current composition is alive. + expect(made.map((ref) => ref.deref() !== undefined)).toEqual([false, true]); + await host.close(context); + }); + + it("a value an app keeps after `host.close` keeps no closed host, and reads its last composition (Sol, Astra)", async () => { + const Kept = defineRegistry("app.kept", { version: 1 }); + const signal = new AbortController(); + const { weak, value } = await (async () => { + const { host } = await loadedPlugins({ "@app/a": (h) => h.serve(Kept, { compose: () => 7 }) }); + const value = await host.registry(Kept, context); + value.observe({ advance: () => {}, end: () => {} }, withAbortSignal(signal.signal, context)); + expect(getEventListeners(signal.signal, "abort")).toHaveLength(1); + await host.close(context); + // The observation ended with the host: its abort listener went with it. + expect(getEventListeners(signal.signal, "abort")).toHaveLength(0); + return { weak: new WeakRef(host), value }; + })(); + await expectCollected(weak); + expect(value.value).toBe(7); + }); +}); + +describe("observe", () => { + it("calls a synchronous observer inside the batch that composed; it counts as watching", async () => { + const Labels = defineRegistry("app.labels", { version: 1 }); + const Label = local("app.label", { version: 1, params: {}, members: { set: method(str()) } }); + const { host } = await loadedPlugins({ + "@app/labels": (host) => { + host.serve(Labels, { compose: (table) => table.map(({ value }) => value).join(",") }); + host.serve(Label, (_, self) => { + const label = self.writable("a"); + self.contribute(Labels, label); + return { + set: async (text: string) => { + label.set(text); + return null; + }, + }; + }); + }, + }); + const labels = await host.registry(Labels, context); + const calls: (NativeWatchCall | WatchEnd)[] = []; + const { context: cancellable, cancel } = withCancel(context); + const observation = labels.observe( + { advance: (call) => calls.push(call), end: (end) => calls.push(end) }, + cancellable, + ); + expect(observation.value).toBe("a"); + const label = await host.get(Label, {}, context); + await label.set("b", context); + // Already called: synchronously, in the set's batch. + expect(calls).toEqual([{ kind: "value", value: "b" }]); + cancel(); + await label.set("c", context); + expect(calls).toEqual([{ kind: "value", value: "b" }, { kind: "aborted" }]); + const again = labels.observe({ advance: (call) => calls.push(call), end: (end) => calls.push(end) }, context); + again.detach(); + await label.set("d", context); + expect(calls).toHaveLength(2); + labels.observe({ advance: () => {}, end: (end) => calls.push(end) }, context); + await host.close(context); + expect(calls.at(-1)).toEqual({ kind: "ended" }); + }); +}); diff --git a/packages/chord/test/next/state/native-source.test.ts b/packages/chord/test/next/state/native-source.test.ts new file mode 100644 index 000000000..cb31e8770 --- /dev/null +++ b/packages/chord/test/next/state/native-source.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, expectTypeOf, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method, str } from "../../../src/next/contracts/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import type { Self } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import type { NativeWatchCall, Watchable } from "../../../src/watch.ts"; +import { delivered, failLive, flushed, instancesOf, loaded, rejection } from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +const Count = defineRegistry("app.count", { version: 1 }); + +const Setter = local("app.setter", { version: 1, params: {}, members: { set: method(int()) } }); + +/** Serves `Count` (a sum that throws when negative) and `Setter`, which contributes a writable number. */ +function serveCount(h: Parameters[0]>[0]): void { + h.serve(Count, { + compose: (table) => { + const sum = table.reduce((n, { value }) => n + value, 0); + if (sum < 0) throw new Error(`negative: ${sum}`); + return sum; + }, + }); + h.serve(Setter, (_, self) => { + const value = self.writable(1); + self.contribute(Count, value); + return { + set: async (n: number) => { + value.set(n); + return null; + }, + }; + }); +} + +describe("native sources as derive sources", () => { + it("a derive reads JSON and native sources alike, marked when a native one changes", async () => { + const Mixed = local("app.mixed", { version: 1, params: {}, members: { label: str(), name: method(str()) } }); + const { host } = await loaded((h) => { + serveCount(h); + h.serve(Mixed, (_, self) => { + const name = self.writable("a"); + return { + label: self.derive([name, self.registry(Count)], ([n, count]) => `${n}:${count}`), + name: async (text: string) => { + name.set(text); + return null; + }, + }; + }); + }); + const mixed = await host.get(Mixed, {}, context); + await (await host.get(Setter, {}, context)).set(2, context); + expect(mixed.label.value).toBe("a:2"); + await mixed.name("b", context); + expect(mixed.label.value).toBe("b:2"); + await host.close(context); + }); + + it("`ref.state` is a derive source: a state member can show a dependency's reachability", async () => { + const Flaky = local("app.flaky", { version: 1, params: { id: int() }, members: {} }); + const Watcher = local("app.watcher", { version: 1, params: {}, members: { flaky: str() } }); + let flakySelf: Self | undefined; + const { host } = await loaded((h) => { + h.serve(Flaky, (_, self) => { + flakySelf = self; + return {}; + }); + h.serve(Watcher, (_, self) => { + const flaky = self.get(Flaky, { id: 1 }); + return { flaky: self.derive([flaky.state], ([state]) => state.kind) }; + }); + }); + const watcher = await host.get(Watcher, {}, context); + const kinds: string[] = []; + (await watcher.flaky.watch(context)).start( + async (call) => void (call.kind === "value" && kinds.push(call.value)), + ); + expect(watcher.flaky.value).toBe("live"); + failLive(objects, flakySelf!, new ChordError({ code: "unavailable", message: "down" })); + expect(watcher.flaky.value).toBe("failed"); + await delivered(); + expect(kinds).toEqual(["failed"]); + await host.close(context); + }); + + it("types a derive's sources: watchables of either kind, never a native's handle (Fable)", () => { + const declare = (self: Self): void => { + const handle = self.open( + async () => 1, + () => {}, + ); + // @ts-expect-error a native handle has a `value` but is no watchable + self.derive([handle], ([n]) => n); + expectTypeOf(self.derive([self.writable(1), constant("a")], ([n, s]) => s.length + n)).toEqualTypeOf< + Watchable + >(); + }; + expect(declare).toBeTypeOf("function"); + }); + + it("refuses a native source as a state member (Q14)", async () => { + const Bad = local("app.bad", { version: 1, params: {}, members: { n: int() } }); + const { host } = await loaded((h) => h.serve(Bad, (_, self) => ({ n: self.writable(1) as never }))); + expect(await rejection(host.get(Bad, {}, context))).toMatchObject({ + message: "app.bad: n: a state member is a JSON watchable", + }); + await host.close(context); + }); +}); + +describe("watching a native source that fails", () => { + it("a watch taken while it is failed opens at its last good value and gets the failure first", async () => { + const { host } = await loaded(serveCount); + const count = await host.registry(Count, context); + await (await host.get(Setter, {}, context)).set(-1, context); + const calls: NativeWatchCall[] = []; + const watch = await count.watch(context); + expect(watch.value).toBe(1); + watch.start(async (call) => void calls.push(call)); + await delivered(); + expect(calls).toEqual([{ kind: "failed", error: new Error("negative: -1") }]); + await host.close(context); + }); + + it("an `on` routine started while its source is failed waits for its next good value, never calls with the stale one", async () => { + const Follower = local("app.follower", { version: 1, params: { id: int() }, members: {} }); + const seen: number[] = []; + const { host } = await loaded((h) => { + serveCount(h); + h.serve(Follower, (_, self) => { + self.on(self.registry(Count), ({ value }) => void seen.push(value)); + return {}; + }); + }); + const setters = await host.get(Setter, {}, context); + await setters.set(-1, context); + // Read: the failure is composed before the follower starts. + void (await host.registry(Count, context)).value; + const follower = await host.get(Follower, { id: 1 }, context); + await flushed(); + expect(seen).toEqual([]); + await setters.set(4, context); + await delivered(); + expect(seen).toEqual([4]); + follower.release(); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/state/writable.test.ts b/packages/chord/test/next/state/writable.test.ts new file mode 100644 index 000000000..30b27309f --- /dev/null +++ b/packages/chord/test/next/state/writable.test.ts @@ -0,0 +1,210 @@ +import { getEventListeners } from "node:events"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withAbortSignal } from "../../../src/context/index.ts"; +import { int, local, method } from "../../../src/next/contracts/index.ts"; +import type { ChordHost } from "../../../src/next/host/host.ts"; +import type { Self } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; +import type { Writable } from "../../../src/next/state/index.ts"; +import type { WritableSource } from "../../../src/next/state/internal.ts"; +import type { NativeWatchCall, WatchEnd } from "../../../src/watch.ts"; +import { delivered, expectCollected, instancesOf, loaded, metered, rejection, restartable } from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +const Counter = local("app.counter", { version: 1, params: {}, members: { doubled: int(), set: method(int()) } }); + +/** Serves `Counter` with a writable number, its double as a state member; the writable is handed out to the test. */ +async function counter(options: Parameters[1] = {}) { + let writable: Writable | undefined; + let self: Self | undefined; + const { host, errors } = await loaded((h) => { + h.serve(Counter, (_, made) => { + self = made; + const value = made.writable(1); + writable = value; + return { + doubled: made.derive([value], ([n]) => n * 2), + set: async (n: number) => { + value.set(n); + return null; + }, + }; + }); + }, options); + const reference = await host.get(Counter, {}, context); + return { host, errors, reference, writable: writable!, self: self! }; +} + +describe("writable", () => { + it("a new value (by identity) is a new frame; the same value notifies nobody and costs one comparison", async () => { + const { telemetry, counts } = await metered(); + const { host, reference, writable } = await counter({ telemetry }); + const calls: NativeWatchCall[] = []; + (await writable.watch(context)).start(async (call) => void calls.push(call)); + const doubled: number[] = []; + (await reference.doubled.watch(context)).start( + async (call) => void (call.kind === "value" && doubled.push(call.value)), + ); + const computes = counts()["chord.derive.computes"]; + writable.set(1); + await delivered(); + expect(calls).toEqual([]); + expect(counts()["chord.derive.computes"]).toBe(computes); + writable.set(2); + writable.set(3); + await delivered(); + expect(calls).toEqual([ + { kind: "value", value: 2 }, + { kind: "value", value: 3 }, + ]); + // A JSON watch folds what came while it waited: the derive's last value. + expect(doubled).toEqual([6]); + expect(writable.value).toBe(3); + await host.close(context); + }); + + it("inside a batch, sets mark it there: one frame with the last value, the derives over it computed once", async () => { + const { telemetry, counts } = await metered(); + const { host, reference, writable } = await counter({ telemetry }); + const calls: NativeWatchCall[] = []; + (await writable.watch(context)).start(async (call) => void calls.push(call)); + const doubled: number[] = []; + (await reference.doubled.watch(context)).start( + async (call) => void (call.kind === "value" && doubled.push(call.value)), + ); + const computes = counts()["chord.derive.computes"]!; + (host as unknown as ChordHost).batch.commit(() => { + writable.set(5); + writable.set(6); + }, context); + await delivered(); + expect(calls).toEqual([{ kind: "value", value: 6 }]); + expect(doubled).toEqual([12]); + expect(counts()["chord.derive.computes"]).toBe(computes + 1); + await host.close(context); + }); + + it("refuses a set inside a derive compute or a compose (`invalid/reentrant`)", async () => { + const Echo = defineRegistry("app.echo", { version: 1 }); + let inner: Writable | undefined; + const Bad = local("app.bad", { version: 1, params: {}, members: { bad: int() } }); + const Good = local("app.good", { version: 1, params: {}, members: {} }); + const { host, errors } = await loaded((h) => { + h.serve(Echo, { + compose: (table) => { + inner?.set(table.length + 100); + return table.length; + }, + }); + h.serve(Good, (_, self) => { + inner = self.writable(0); + self.contribute(Echo, inner); + return {}; + }); + h.serve(Bad, (_, self) => { + const own = self.writable(0); + return { + bad: self.derive([own], ([n]) => { + own.set(n + 1); + return n; + }), + }; + }); + }); + expect((await host.registry(Echo, context)).value).toBe(0); + expect(errors.find((error) => error.kind === "registry_failed")).toMatchObject({ + error: { code: "invalid", reason: "reentrant" }, + }); + expect(errors.find((error) => error.kind === "activation_failed")).toMatchObject({ + object: "app.bad", + failure: { kind: "broken", error: { code: "invalid", reason: "reentrant" } }, + }); + await host.close(context); + }); + + it("is the attempt's: `set` after its end throws `released`, its watches end `ended`, its value stays readable", async () => { + let writable: Writable | undefined; + const { host, restart } = await restartable(objects, (attempt, self) => { + if (attempt === 1) writable = self.writable(1); + }); + const watch = await writable!.watch(context); + await restart(); + expect(await watch.closed).toEqual({ kind: "ended" } satisfies WatchEnd); + expect(() => writable!.set(2)).toThrow("app.restartable(1): ended"); + expect(writable!.value).toBe(1); + await host.close(context); + }); + + it("is a contribution, a derive source and an `on` source", async () => { + const Values = defineRegistry("app.values", { version: 1 }); + const Source = local("app.source", { version: 1, params: {}, members: { set: method(int()), seen: int() } }); + const seen: number[] = []; + const { host } = await loaded((h) => { + h.serve(Values, { compose: (table) => table.map(({ value }) => value) }); + h.serve(Source, (_, self) => { + const value = self.writable(1); + self.contribute(Values, value); + self.on(value, ({ value: n }) => void seen.push(n)); + return { + seen: self.derive([value], ([n]) => n), + set: async (n: number) => { + value.set(n); + return null; + }, + }; + }); + }); + const source = await host.get(Source, {}, context); + const values = await host.registry(Values, context); + await source.set(2, context); + await delivered(); + expect(values.value).toEqual([2]); + expect(source.seen.value).toBe(2); + expect(seen).toEqual([1, 2]); + await host.close(context); + }); + + it("is collected with its attempt", async () => { + const refs = await (async () => { + const made: WeakRef[] = []; + const { host, restart } = await restartable(objects, (attempt, self) => { + if (attempt === 1) made.push(new WeakRef(self.writable({ big: new Array(1000).fill(0) }))); + }); + await restart(); + await host.close(context); + return made; + })(); + await expectCollected(...refs); + }); + + it("a set of the same value after its attempt ended throws `released` too (Sol, Astra)", async () => { + const { host, writable } = await counter(); + await host.close(context); + expect(() => writable.set(1)).toThrow("app.counter: ended"); + }); + + it("a writable an app keeps after `host.close` keeps no closed host; its observers' abort listeners go (Sol)", async () => { + const signal = new AbortController(); + const { weak, writable } = await (async () => { + const { host, writable } = await counter(); + (writable as WritableSource).observe( + { advance: () => {}, end: () => {} }, + withAbortSignal(signal.signal, context), + ); + expect(getEventListeners(signal.signal, "abort")).toHaveLength(1); + await host.close(context); + expect(getEventListeners(signal.signal, "abort")).toHaveLength(0); + return { weak: new WeakRef(host), writable }; + })(); + await expectCollected(weak); + expect(writable.value).toBe(1); + }); + + it("a late callback's set after the host closed throws `released`", async () => { + const { host, writable } = await counter(); + await host.close(context); + expect(await rejection(Promise.resolve().then(() => writable.set(9)))).toMatchObject({ code: "released" }); + }); +}); diff --git a/packages/chord/test/next/support.ts b/packages/chord/test/next/support.ts index 428dea36f..9796d534f 100644 --- a/packages/chord/test/next/support.ts +++ b/packages/chord/test/next/support.ts @@ -35,6 +35,7 @@ import { createHost, defineFacet, type FacetHost, + type FacetSelf, type Host, type HostError, type RefState, @@ -231,7 +232,7 @@ async function restartableOf void, + setup: (host: FacetHost, self: FacetSelf) => void, options: { readonly telemetry?: InMemoryTelemetryContext; readonly context?: Context; @@ -247,6 +248,21 @@ export async function loaded( return { host, errors }; } +/** + * A host with background errors collected, loaded with one facet per plugin, in the order given (`setups` by plugin + * name). + */ +export async function loadedPlugins( + setups: Readonly void>>, + options: { readonly telemetry?: InMemoryTelemetryContext; readonly readyTimeout?: number } = {}, +): Promise<{ readonly host: Host; readonly errors: HostError[] }> { + const errors: HostError[] = []; + const host = createHost({ ...options, onError: (error) => errors.push(error) }); + const facets = Object.entries(setups).map(([name, setup]) => ({ name, facet: defineFacet(setup) })); + await host.load(facets, BACKGROUND_CONTEXT); + return { host, errors }; +} + /** A store on a fresh SQLite `:memory:` database, as Chord's layers see it. */ export async function memoryStore(options?: { readonly outbox?: boolean }): Promise { const db = await openNodeSqliteDatabase(":memory:"); diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 56763148d..92430f3e4 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -98,6 +98,8 @@ - `RunFailureReason` (`pi.live.runFailure.reason`) gains `nothing_to_answer` and `turn_limit`, also an `unanswered` submission's `reason`: exhaustive switches over it need cases ([#11043](https://github.com/earendil-works/pi/issues/11043)) - `pi.live.run` has a required `turns` field, the run's tool rounds so far; code that writes `run` itself sets it (`turns: 0`). `pi.live` is version 3: its `migrate` adds `turns: 0` to a stored run without it, version 1 or a version 2 an earlier unreleased build wrote, whose run in flight otherwise failed its next tool round with "Value contains a non-finite number" ([#11043](https://github.com/earendil-works/pi/issues/11043)) +- A Session's close ends synchronous observers (`observeDoc()`, `observeConversation()`, `observeTaskGraph()`, `observeCommits()`) `failed` with the error "session closed" instead of `ended`, so an adapter following a Session (Chord's `self.follow`) fails and follows the next one; `ended` stays for an observed owner that ended (a task document's task), and watches still end `ended`. Before: `end: (end) => { if (end.kind === "ended") done(); }`. After: `end: (end) => { if (end.kind === "ended" || end.kind === "failed") done(); }`. + ### Added - `StoreStorage` (`@earendil-works/pi-durable/storage/store`): Durable's `Storage` on Chord v2's store. `StoreStorage.open(store, owner, context)` keeps a Session's records as declared tables of the store under `owner` and its documents in the store's documents engine, takes its ids from the store's one allocator, and commits each batch as one write of the store's commit step, adopted in the batch of the host the store serves, so a Chord object and a Durable Session commit through one queue and one outbox. It passes the storage conformance suite; `close()` leaves the store open (the app owns it). @@ -115,6 +117,8 @@ - `HookApi.entry()` and `entries()`, and `TaskRuntime.entries()`: a hook or task reads the conversation's raw history, which compaction never changes, so a range bounded by entry IDs reads the same before and after a compaction and on a rerun ([#11141](https://github.com/earendil-works/pi/issues/11141)) - Run-end tasks: a `GenerationHooks.afterRun(run, api)` hook returns `{ tasks }`, built with the typed `runEndTask(task, input)`, after a run answered (a final answer or a `terminate` round; not when an `onYield` continuation is taken, not after a failure or an abort). The answer is committed and the run's inputs settle `done` first, so `Submission.wait()` resolves at the answer; the generation then creates the tasks as its children and waits for them, keeping `pi.live.run` (with `run.afterRun`) and the conversation busy, so idle waits include them, Stop aborts them, and a crash resumes them. Input queued meanwhile, steers included, is placed when they are terminal and starts the next run; a task admits a model-visible write or a fix-run follow-up in its own commit with `submit(tx, draft)`. A crash before the tasks exist asks the hooks again; a broken request (an uninstalled or built-in task, a non-JSON input, a throwing `initial()`) is reported and skipped. `run` names the run's entries: `first` (`pi.live.run.first`, carried into `runFailure.first` so a continue covers a failed run's history), `answer`, and `last` ([#11141](https://github.com/earendil-works/pi/issues/11141)) +- `registryReader(value, context)`: a `RegistryReader` over a Chord v2 registry value whose composition is a registry snapshot (`RegistryValue`): `snapshot()` reads the current composition, `subscribe` observes its compositions synchronously inside Chord's batch, so a Harness opened in a Chord object takes its extensions from a Chord registry. + ### Changed - Document, conversation view, and task graph watches use Chord's watch handles (`Watch`). diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index 4f3795e33..385f8e948 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -5431,9 +5431,11 @@ advances without a commit, when a registry change or a failed migration changes a task's `blocked` reason (section 9.4): that call runs on the Session line, outside any `adopt`, with a context of its own. A document observer follows its address as a document watch does, and `observeDoc()` gives `undefined` for an -absent one. `end` comes once: when the Session closes (`ended`) or fails -(`failed` with the error), or, for a task document, right after the retirement change of -the commit that makes its task terminal, inside that commit's `adopt` (`ended`). +absent one. `end` comes once: when the Session closes (`failed` with the error +"session closed": an adapter following the Session, such as Chord's `self.follow`, +fails and follows the next Session; watches end `ended`) or fails (`failed` with +the error), or, for a task document, right after the retirement change of the +commit that makes its task terminal, inside that commit's `adopt` (`ended`). `detach()` is idempotent, and nothing follows it, not even the rest of the commit underway; a detached observer is referenced by nothing of the Session. `context` governs only the attachment: cancelling it while attaching rejects diff --git a/packages/durable/src/harness/registry-reader.ts b/packages/durable/src/harness/registry-reader.ts new file mode 100644 index 000000000..97f8a44bd --- /dev/null +++ b/packages/durable/src/harness/registry-reader.ts @@ -0,0 +1,34 @@ +// A Harness's registry over a Chord registry (Chord v2 slice 8 §5): the composed value of p3's extensions registry, +// read as Durable's `RegistryReader`. + +import type { Context } from "@earendil-works/chord/context"; +import type { RegistryValue } from "@earendil-works/chord/next"; +import type { RegistryReader, RegistrySnapshot, ToolRegistration } from "./types.ts"; + +/** + * A `RegistryReader` over a Chord registry value whose composition is a registry snapshot: `snapshot()` reads the + * current composition (the last good one while `compose` fails), `subscribe` observes its compositions, its listener + * called synchronously inside the Chord batch that composed (so it only wakes the scheduler; a Chord call there + * throws `invalid/reentrant`). `context` is the observation's: aborting it detaches every listener. + */ +export function registryReader( + value: RegistryValue>, + context: Context, +): RegistryReader { + return { + snapshot: () => value.value, + subscribe: (listener) => { + const observation = value.observe( + { + advance: (call) => { + // A failed composition keeps the last snapshot: nothing to reconsider. + if (call.kind === "value") listener(); + }, + end: () => {}, + }, + context, + ); + return () => observation.detach(); + }, + }; +} diff --git a/packages/durable/src/harness/task-graph.ts b/packages/durable/src/harness/task-graph.ts index 2e0b78875..943eba33a 100644 --- a/packages/durable/src/harness/task-graph.ts +++ b/packages/durable/src/harness/task-graph.ts @@ -1,16 +1,10 @@ -import { - type Context, - type JsonValue, - Watch, - type WatchCall, - type WatchEnd, - type WatchHandle, -} from "@earendil-works/chord"; +import { type Context, type JsonValue, Watch, type WatchCall, type WatchHandle } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT, withoutAbortSignal } from "@earendil-works/chord/context"; import { applyImmutable, type Op } from "@earendil-works/chord/delta"; import { + closeEndOf, + closeEnds, contained, - ENDED, guardObserver, type ObservedCommit, observation, @@ -110,11 +104,10 @@ export class TaskGraphView { blocking.subscribeBlocked(() => this.#refreshBlocked()); session.subscribeClose(() => { this.#closed = true; - const failure = this.#session.failure; - const end: WatchEnd = failure === undefined ? ENDED : { kind: "failed", error: failure.error }; + const ends = closeEnds(this.#session.failure); for (const observer of [...(this.#mount?.observers ?? [])]) { contained( - () => observer.end(end), + () => observer.end(closeEndOf(observer, ends)), (error) => session.report(error), ); } diff --git a/packages/durable/src/harness/view.ts b/packages/durable/src/harness/view.ts index d2999c47f..72cd9c890 100644 --- a/packages/durable/src/harness/view.ts +++ b/packages/durable/src/harness/view.ts @@ -10,8 +10,9 @@ import { withoutAbortSignal } from "@earendil-works/chord/context"; import { applyImmutable, type NonEmptyPath, type Op, type Path } from "@earendil-works/chord/delta"; import { idFromNumber } from "../ids.ts"; import { + closeEndOf, + closeEnds, contained, - ENDED, guardObserver, type ObservedCommit, observation, @@ -158,12 +159,11 @@ export class ConversationViews { }); session.subscribeClose(() => { this.#closed = true; - const failure = this.#session.failure; - const end: WatchEnd = failure === undefined ? ENDED : { kind: "failed", error: failure.error }; + const ends = closeEnds(this.#session.failure); for (const mount of this.#mounts.values()) for (const windowed of [...mount.observers]) { contained( - () => windowed.observer.end(end), + () => windowed.observer.end(closeEndOf(windowed.observer, ends)), (error) => this.report(error), ); } diff --git a/packages/durable/src/index.ts b/packages/durable/src/index.ts index 4b4671129..59068ad12 100644 --- a/packages/durable/src/index.ts +++ b/packages/durable/src/index.ts @@ -54,6 +54,7 @@ export { } from "./harness/live.ts"; export { ProviderDoc, type ProviderState } from "./harness/provider.ts"; export { createRegistry } from "./harness/registry.ts"; +export { registryReader } from "./harness/registry-reader.ts"; export type { BlockedReason } from "./harness/scheduler.ts"; export type { TaskGraph, TaskGraphNode, TaskGraphState, TaskGraphWatch } from "./harness/task-graph.ts"; export { DURABLE_TELEMETRY_SCHEMA } from "./harness/telemetry.ts"; diff --git a/packages/durable/src/session/session.ts b/packages/durable/src/session/session.ts index 5f5df8778..a6bb44556 100644 --- a/packages/durable/src/session/session.ts +++ b/packages/durable/src/session/session.ts @@ -612,8 +612,8 @@ export class SessionImpl implements Session { const observers = [...this.#commitListeners]; this.#commitListeners.clear(); // After the last publication: the commits admitted before the seal have settled, held changes are stored. - const end: WatchEnd = - this.#failure === undefined ? ENDED : { kind: "failed", error: this.#failure.error }; + // Commit observers are synchronous observers, all of them. + const end = closeEnds(this.#failure).observer; for (const observer of observers) observer.end(end); this.#internalListeners.clear(); this.#documents.clear(); @@ -1269,7 +1269,7 @@ export class SessionImpl implements Session { }); unsubscribeClose = this.subscribeClose(() => { detach(); - observer.end(this.#failure === undefined ? ENDED : { kind: "failed", error: this.#failure.error }); + observer.end(closeEndOf(observer, closeEnds(this.#failure))); }); return observer; } @@ -1419,6 +1419,28 @@ function toError(error: unknown): Error { /** The end of an observer whose source closed. */ export const ENDED: WatchEnd = { kind: "ended" }; +/** + * How a Session's close ends what follows it (decision "Slice 8 S1"), made once per close: an app's watch `ended`; a + * synchronous observer (an adapter following the Session, which may open it again) `failed` with "session closed", so + * it follows the next Session; `ended` stays for an observed owner that ended (a task document's task). A failed + * Session ends both `failed` with its error. + */ +export function closeEnds(failure: { readonly error: unknown } | undefined): { + readonly watch: WatchEnd; + readonly observer: WatchEnd; +} { + if (failure !== undefined) { + const failed: WatchEnd = { kind: "failed", error: failure.error }; + return { watch: failed, observer: failed }; + } + return { watch: ENDED, observer: { kind: "failed", error: new Error("session closed") } }; +} + +/** The end `ends` give `follower`: a watch's, or an observer's. */ +export function closeEndOf(follower: unknown, ends: ReturnType): WatchEnd { + return follower instanceof Watch ? ends.watch : ends.observer; +} + /** * The `Observation` of a guarded observer attached at `value`. Built outside the attaching closure, so a detached * observation keeps nothing but `value` and the spent guard alive. diff --git a/packages/durable/src/types.ts b/packages/durable/src/types.ts index 4f17a3711..c2aa21688 100644 --- a/packages/durable/src/types.ts +++ b/packages/durable/src/types.ts @@ -1101,8 +1101,9 @@ export type DocumentWatch = WatchHandle>; * may come before the attaching promise resolves; their ops apply from `Observation.value`. `advance` must only record * the call, synchronously and quickly, never await, call the Session, or throw. A throw is reported as * `listener_error`, detaches the observer, and ends it `listener_error` with the error. `end` is called once: `ended` - * when the Session closes or the observed scope's owner ends (a task document's task, after its retirement change, - * inside that commit's `adopt`), `failed` with the error that failed the Session; never after `detach()`. + * when the observed scope's owner ends (a task document's task, after its retirement change, inside that commit's + * `adopt`), `failed` with "session closed" when the Session closes (an adapter follows the next Session) or with the + * error that failed the Session; never after `detach()`. */ export type Observer = { advance(call: WatchCall, context: Context): void; @@ -1111,7 +1112,7 @@ export type Observer = { /** * A synchronous observer of stored commits (`Session.observeCommits()`): `advance` gets each `CommitPublication`, `end` - * the Session's close. + * the Session's close (`failed` with "session closed", or with the error that failed the Session). */ export type CommitObserver = { advance(publication: CommitPublication, context: Context): void; diff --git a/packages/durable/test/examples/37-harness-registry.ts b/packages/durable/test/examples/37-harness-registry.ts new file mode 100644 index 000000000..404a61a0d --- /dev/null +++ b/packages/durable/test/examples/37-harness-registry.ts @@ -0,0 +1,172 @@ +// A Durable harness in Chord objects, p3's pattern (Chord v2 slice 8): the object that opens Durable on the host's store +// takes its extensions from a Chord registry (`registryReader`), so any plugin extends the harness by contributing an +// extension; and it contributes its Harness to a second registry, through which another object follows its views +// (`self.follow`). When that Harness's Session closes, the follower fails and follows the next Harness, woken by its +// contribution (`self.retryOn`). +// Run from packages/durable: +// node --conditions=source --experimental-strip-types test/examples/37-harness-registry.ts + +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { + arr, + createHost, + defineFacet, + defineRegistry, + local, + method, + type Native, + openStore, + type RegistryTable, + sqliteBackend, + str, +} from "@earendil-works/chord/next"; +import { openNodeSqliteDatabase } from "@earendil-works/chord/sqlite/node"; +import { createModels } from "@earendil-works/pi-ai"; +import { + createRegistry, + defineExtension, + defineTask, + type Extension, + Harness, + type RegistrySnapshot, + ROOT_CONVERSATION_ID, + registryReader, +} from "../../src/index.ts"; +import { StoreStorage } from "../../src/storage/store/index.ts"; + +const context = BACKGROUND_CONTEXT; + +// 1. The core's registries: extensions in, Durable's registry snapshot out; and the Harness, for other objects. +const Extensions = defineRegistry("example.extensions", { version: 1 }); +const Harnesses = defineRegistry, Harness | undefined>("example.harnesses", { version: 1 }); + +/** Builds Durable's snapshot without throwing: an extension Durable refuses (a task name taken) is reported, left out. */ +function snapshotOf( + table: RegistryTable, + report: (conflict: { message: string; plugins: string[] }) => void, +) { + const registry = createRegistry(); + for (const { value, plugin } of table) { + try { + registry.install(value); + } catch (error) { + report({ message: (error as Error).message, plugins: [plugin] }); + } + } + return registry.snapshot(); +} + +const Core = local("example.harness", { + version: 1, + params: {}, + members: { greet: method(undefined, str()), close: method() }, +}); +const Transcript = local("example.transcript", { version: 1, params: {}, members: { kinds: arr(str()) } }); + +// 2. The core facet: the harness object opens Durable on the store with its extensions read from the registry. +const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context); +const core = defineFacet((host) => { + host.serve(Extensions, { compose: (table, { report }) => snapshotOf(table, report) }); + host.serve(Harnesses, { compose: (table) => table.at(-1)?.value.value }); + host.serve(Core, (_, self) => { + // A declared dependency: the object activates (and opens Durable) once the registry has a value. + const extensions = self.registry(Extensions); + const harness = self.open( + async (open) => { + const opened = await Harness.open( + await StoreStorage.open(store, Core.name, open), + { models: createModels(), registry: registryReader(extensions, open) }, + open, + ); + await opened.root(open); + return opened; + }, + (h, _reason, close) => h.close(close), + ); + self.contribute(Harnesses, harness); + // Following its own task graph, the object fails with its Session (S1), and its retry opens the next Harness. + self.follow((observer, ctx) => harness.value.observeTaskGraph(observer, ctx)); + return { + greet: async (ctx) => { + // The composed snapshot, read at use: is the task's extension installed? + if (extensions.value.task(Greet.definition.name) === undefined) return "no greeter installed"; + const root = await harness.value.conversation(ROOT_CONVERSATION_ID, ctx); + const id = await root!.commit( + (tx) => tx.createTask(Greet, undefined, { ownership: { kind: "conversation" } }), + ctx, + ); + harness.value.resume(); + return JSON.stringify((await harness.value.waitForTask(id, ctx)).state.outcome); + }, + close: async (ctx) => { + await harness.value.close(ctx); + return null; + }, + }; + }); + // 3. Another object follows the harness's conversation view through the registry. + host.serve(Transcript, (_, self) => { + const harness = self.registry(Harnesses); + self.retryOn(harness); + const view = self.follow(async (observer, ctx) => + harness.value?.observeConversation(ROOT_CONVERSATION_ID, observer, ctx), + ); + return { kinds: self.derive([view], ([v]) => v.entries.map((entry) => entry.kind)) }; + }); +}); + +// 4. Two plugins contribute extensions with the same task name, which Durable refuses: the first by plugin name +// (copycat) is installed, the other reported. +const Greet = defineTask({ + name: "example.greet", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (_task, runtime, taskContext) => { + await runtime.commit(async (tx) => { + await tx.appendEntry(ROOT_CONVERSATION_ID, { kind: "example.greeted", data: {} }); + return { status: "terminal", outcome: { status: "completed", result: "hello" } }; + }, taskContext); + }, + }, + abort: async () => {}, +}); +const greeter = defineFacet((_, self) => + self.contribute(Extensions, defineExtension({ name: "greeter", tasks: [Greet] })), +); +const copycat = defineFacet((_, self) => + self.contribute(Extensions, defineExtension({ name: "copycat", tasks: [Greet] })), +); + +const host = createHost({ + store, + onError: (error) => console.log("onError:", error.kind, error.kind === "registry_conflict" ? error.message : ""), +}); +await host.load( + [ + { name: "@example/core", facet: core }, + { name: "@example/greeter", facet: greeter }, + { name: "@example/copycat", facet: copycat }, + ], + context, +); + +// 5. The task comes from a contributed extension; the transcript object sees its entry. (It failed once at start, before +// the harness object contributed its Harness: `activation_failed`, then the contribution woke it.) +const harness = host.hold(Core, {}, context); +const transcript = host.hold(Transcript, {}, context); +console.log("greet:", await harness.greet(context)); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("transcript:", transcript.kinds.value); + +// 6. The Session closes: both objects fail; a retry of the harness object opens the next Harness (on the same store: +// the transcript is still there), whose contribution wakes the follower. +await harness.close(context).catch((error: Error) => console.log("close:", error.message)); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("transcript after the close:", transcript.state.value.kind); +harness.retry(); +await new Promise((resolve) => setTimeout(resolve, 50)); +console.log("transcript after the retry:", transcript.state.value.kind, transcript.kinds.value); + +await host.close(context); +await store.close(context); diff --git a/packages/durable/test/harness-observers.test.ts b/packages/durable/test/harness-observers.test.ts index fdd6d7787..bbd3db12e 100644 --- a/packages/durable/test/harness-observers.test.ts +++ b/packages/durable/test/harness-observers.test.ts @@ -124,8 +124,11 @@ describe("observeConversation and observeTaskGraph", () => { await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); expect(view.calls).toHaveLength(1); await watch.stop(); + const graphWatch = await harness.watchTaskGraph(context); await harness.close(context); - expect(graph.calls).toEqual([{ end: { kind: "ended" } }]); + // S1: a Session close fails an observer (it may follow the next Session); a watch ends `ended`. + expect(graph.calls).toEqual([{ end: { kind: "failed", error: new Error("session closed") } }]); + expect(await graphWatch.closed).toEqual({ kind: "ended" }); }); it("end with the error that failed the Harness", async () => { diff --git a/packages/durable/test/harness-registry-reader.test.ts b/packages/durable/test/harness-registry-reader.test.ts new file mode 100644 index 000000000..b2a82da35 --- /dev/null +++ b/packages/durable/test/harness-registry-reader.test.ts @@ -0,0 +1,198 @@ +import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; +import { + arr, + createHost, + defineFacet, + defineRegistry, + type FacetHost, + local, + method, + type Native, + type RegistryTable, + str, +} from "@earendil-works/chord/next"; +import { createModels } from "@earendil-works/pi-ai/models"; +import { describe, expect, it } from "vitest"; +import { + createRegistry, + defineExtension, + defineTask, + type Extension, + Harness, + MemoryStorage, + type RegistrySnapshot, + ROOT_CONVERSATION_ID, + registryReader, +} from "../src/index.ts"; + +/** p3's pattern: the extensions registry composes Durable's snapshot, leaving out an extension Durable refuses. */ +const Extensions = defineRegistry("app.harness.extensions", { version: 1 }); + +function compose(table: RegistryTable, report: (conflict: { message: string; plugins: string[] }) => void) { + const registry = createRegistry(); + for (const { value, plugin } of table) { + try { + registry.install(value); + } catch (error) { + report({ message: (error as Error).message, plugins: [plugin] }); + } + } + return registry.snapshot(); +} + +const Done = defineTask({ + name: "app.done", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (_task, runtime, taskContext) => { + await runtime.commit( + () => ({ status: "terminal", outcome: { status: "completed", result: null } }), + taskContext, + ); + }, + }, + abort: async () => {}, +}); + +/** A host-lifetime object contributing an extension it replaces with `install` (one with the task `app.done`). */ +const Plugin = local("app.plugin", { version: 1, params: {}, members: { install: method() } }); + +function serve(host: FacetHost): void { + host.serve(Extensions, { compose: (table, { report }) => compose(table, report) }); + host.serve(Plugin, (_, self) => { + const extension = self.writable(defineExtension({ name: "app.plugin" })); + self.contribute(Extensions, extension); + return { + install: async () => { + extension.set(defineExtension({ name: "app.plugin", tasks: [Done] })); + return null; + }, + }; + }); +} + +describe("registryReader", () => { + it("reads the composed snapshot; a listener is called synchronously, inside the batch that composed", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/core", facet: defineFacet(serve) }], context); + const value = await host.registry(Extensions, context); + const reader = registryReader(value, context); + expect(reader.snapshot().extension("app.plugin")?.tasks).toBeUndefined(); + const calls: string[] = []; + const unsubscribe = reader.subscribe(() => { + // Inside Chord's batch: a Chord call here is refused, so a listener only wakes its scheduler. + try { + value.observe({ advance: () => {}, end: () => {} }, context); + calls.push("outside"); + } catch (error) { + calls.push((error as { reason?: string }).reason ?? "other"); + } + }); + const plugin = await host.get(Plugin, {}, context); + await plugin.install(context); + expect(calls).toEqual(["reentrant"]); + expect(reader.snapshot().task("app.done")).toBe(Done); + unsubscribe(); + await host.close(context); + }); + + it("wakes a Harness's scheduler: a task blocked on its missing definition runs once a contribution brings it", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/core", facet: defineFacet(serve) }], context); + const harness = await Harness.open( + new MemoryStorage(), + { models: createModels(), registry: registryReader(await host.registry(Extensions, context), context) }, + context, + ); + harness.resume(); + const root = await harness.root(context); + const id = await root.commit( + (tx) => tx.createTask(Done, undefined, { ownership: { kind: "conversation" } }), + context, + ); + const done = harness.waitForTask(id, context); + await new Promise((resolve) => setTimeout(resolve, 10)); + await (await host.get(Plugin, {}, context)).install(context); + expect((await done).state.outcome).toEqual({ status: "completed", result: null }); + await harness.close(context); + await host.close(context); + }); +}); + +/** p3's pattern for Durable's views from other objects: the object that opens Durable contributes its Harness. */ +const Harnesses = defineRegistry, Harness | undefined>("app.harnesses", { version: 1 }); + +/** Opens a Harness (a new one per attempt), contributes it, and follows its task graph: its Session closing fails it. */ +const Durable = local("app.durable", { version: 1, params: {}, members: { close: method(), write: method(str()) } }); + +/** Follows the root conversation's entry kinds through the contributed Harness, from another object. */ +const Viewer = local("app.viewer", { version: 1, params: {}, members: { kinds: arr(str()) } }); + +function serveViews(host: FacetHost): void { + host.serve(Harnesses, { compose: (table) => table.at(-1)?.value.value }); + host.serve(Durable, (_, self) => { + const harness = self.open( + async (open) => { + const opened = await Harness.open( + new MemoryStorage(), + { models: createModels(), registry: createRegistry() }, + open, + ); + await opened.root(open); + return opened; + }, + (h, _reason, close) => h.close(close), + ); + self.contribute(Harnesses, harness); + self.follow((observer, ctx) => harness.value.observeTaskGraph(observer, ctx)); + return { + close: async (ctx) => { + await harness.value.close(ctx); + return null; + }, + write: async (kind, ctx) => { + const root = await harness.value.conversation(ROOT_CONVERSATION_ID, ctx); + await root!.submit({ type: "write", entry: { kind } }, ctx); + return null; + }, + }; + }); + host.serve(Viewer, (_, self) => { + const harness = self.registry(Harnesses); + self.retryOn(harness); + const view = self.follow(async (observer, ctx) => + harness.value === undefined + ? undefined + : harness.value.observeConversation(ROOT_CONVERSATION_ID, observer, ctx), + ); + return { kinds: self.derive([view], ([v]) => v.entries.map((entry) => entry.kind)) }; + }); +} + +describe("Durable's views from other objects (slice 8 §5, S1)", () => { + it("a follower of a contributed Harness fails when that Harness's Session closes, and follows the next one", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/core", facet: defineFacet(serveViews) }], context); + const durable = host.hold(Durable, {}, context); + const viewer = host.hold(Viewer, {}, context); + await new Promise((resolve) => setTimeout(resolve, 10)); + await durable.write("app.first", context); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(viewer.kinds.value).toContain("app.first"); + // The Session closes: both followers fail `transient` (S1); a retry opens the next Harness, whose contribution + // wakes the viewer (`retryOn`), which follows it. + // The call is cut off by its object's failure: the Session's close fails the object that opened it. + await expect(durable.close(context)).rejects.toThrow("session closed"); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(viewer.state.value).toMatchObject({ kind: "failed", cause: { kind: "transient" } }); + durable.retry(); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(viewer.state.value).toEqual({ kind: "live" }); + expect(viewer.kinds.value).not.toContain("app.first"); + await durable.write("app.second", context); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(viewer.kinds.value).toContain("app.second"); + await host.close(context); + }); +}); diff --git a/packages/durable/test/session-observers.test.ts b/packages/durable/test/session-observers.test.ts index 0dcbc324a..a22731cef 100644 --- a/packages/durable/test/session-observers.test.ts +++ b/packages/durable/test/session-observers.test.ts @@ -173,13 +173,31 @@ describe("observeDoc", () => { expect(calls).toEqual([]); }); - it("ends once when the Session closes", async () => { + it("ends once when the Session closes, `failed` (S1: an adapter follows the next Session), a watch `ended`", async () => { const { storage, session } = open(); await add(session); const { calls, observer } = recorder(storage); await session.observeDoc(SessionCounter, observer, context); + const watch = (await session.watchDoc(SessionCounter, context))!; await session.close(context); - expect(calls).toEqual([{ end: { kind: "ended" } }]); + expect(calls).toEqual([{ end: { kind: "failed", error: new Error("session closed") } }]); + expect(await watch.closed).toEqual({ kind: "ended" }); + }); + + it("ends each close's observers with an error of that close, not one shared by every Session", async () => { + const errors: unknown[] = []; + for (let i = 0; i < 2; i++) { + const { session } = open(); + await add(session); + await session.observeDoc( + SessionCounter, + { advance: () => {}, end: (end) => errors.push(end.kind === "failed" ? end.error : end) }, + context, + ); + await session.close(context); + } + expect(errors).toEqual([new Error("session closed"), new Error("session closed")]); + expect(errors[0]).not.toBe(errors[1]); }); it("ends with the error that failed the Session", async () => { @@ -368,14 +386,18 @@ describe("observeCommits", () => { return { calls, observer }; } - it("gets every stored commit, then `ended` once the Session closed, after its last publication", async () => { + it("gets every stored commit, then `failed` once the Session closed (S1), after its last publication", async () => { const { session } = open(); const { calls, observer } = commits(); session.observeCommits(observer); await add(session); await add(session); await session.close(context); - expect(calls).toEqual([{ seq: expect.any(Number) }, { seq: expect.any(Number) }, { end: { kind: "ended" } }]); + expect(calls).toEqual([ + { seq: expect.any(Number) }, + { seq: expect.any(Number) }, + { end: { kind: "failed", error: new Error("session closed") } }, + ]); }); it("ends `failed` with the error that failed the Session", async () => { diff --git a/scripts/check-entry-graphs.mjs b/scripts/check-entry-graphs.mjs index b3b033a73..388eb0012 100644 --- a/scripts/check-entry-graphs.mjs +++ b/scripts/check-entry-graphs.mjs @@ -67,8 +67,9 @@ const BUDGETS = { // The Harness's request identity reaches pi-ai's utils/identity.ts and its version constant. Partial commits // parse streamed tool-call arguments with pi-ai's utils/streaming-json.ts (and its json-parse.ts) to append what // argument strings grew by (harness/partial-changes.ts). Chord's watches fold pending ops with Delta's - // pending queue (delta/pending.ts). - maxFiles: 77, + // pending queue (delta/pending.ts). `registryReader` (harness/registry-reader.ts) reads a Chord v2 registry + // value through type-only imports. + maxFiles: 78, forbid: [ "packages/ai/src/index.ts", "packages/ai/src/utils/typebox-helpers.ts", From c466a35957e73876651d014d6c36d5ea47148344 Mon Sep 17 00:00:00 2001 From: Christian Klotz Date: Thu, 29 Oct 2026 00:13:41 +0200 Subject: [PATCH 123/271] fix(coding-agent): cap pi.exec output at 256 MiB (#11546) Closes #11360. ## Summary - Cap combined stdout/stderr with `ExecOptions.maxBuffer`, defaulting to the smaller of 256 MiB and the runtime string limit. Invalid explicit limits reject before spawning. - Terminate oversized commands and return `truncated: true`, `killed: true`, and a nonzero exit code. Close output pipes so writing descendants cannot keep the call pending, and discard incomplete UTF-8 suffixes rather than adding replacement characters. - Fix SIGKILL escalation for children that ignore SIGTERM, and simplify completion to `await` with one cleanup block and result. - Document the API and add regressions for byte limits, shell descendants, UTF-8 chunk boundaries, and 32-bit runtime limits. `ExecResult.truncated` is required. Existing extensions reading execution results continue working; typed mocks or wrappers constructing results must add the field. ## Verification - `npm run build` and the post-rebase `npm run build:offline` - `npm run check` - 39 focused execution and extension tests - `./test.sh` passes after building the env daemon. Earlier runs hit remote-cleanup and reftable-watcher timeouts; both passed in focused reruns and the final full run. --- packages/coding-agent/CHANGELOG.md | 3 + packages/coding-agent/docs/extensions.md | 6 +- packages/coding-agent/src/core/exec.ts | 190 +++++++++++------- .../test/exec-runtime-limit.test.ts | 25 +++ .../test/exec-utf8-chunks.test.ts | 53 +++++ packages/coding-agent/test/exec.test.ts | 108 +++++++++- .../git-merge-and-resolve-extension.test.ts | 4 +- .../input-transform-streaming-example.test.ts | 6 +- 8 files changed, 313 insertions(+), 82 deletions(-) create mode 100644 packages/coding-agent/test/exec-runtime-limit.test.ts create mode 100644 packages/coding-agent/test/exec-utf8-chunks.test.ts diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 31571b8e9..6f09e904e 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -5,6 +5,7 @@ ### Added - Extension packages can now show their changelog after an update. Pi reads `pi.changelogPath` from the package's `package.json`, or `CHANGELOG.md` in the package root, and on the next fresh launch shows entries newer than the last seen version under "Extension Updates". The `collapseChangelog` setting condenses this to one line per package ([#5958](https://github.com/earendil-works/pi/issues/5958), [#8790](https://github.com/earendil-works/pi/pull/8790)) +- `pi.exec()` accepts a `maxBuffer` option that limits stdout and stderr combined, and its result has a `truncated` field that is true when output exceeded the limit ([#11360](https://github.com/earendil-works/pi/issues/11360)) - Device-code login (for example GitHub Copilot) now opens the verification page and copies the code when you press Enter. Providers can turn either action off with `openBrowserOnConfirm: false` or `copyCodeOnConfirm: false` ([#9282](https://github.com/earendil-works/pi/issues/9282), [#9301](https://github.com/earendil-works/pi/pull/9301)) ### Changed @@ -15,6 +16,8 @@ ### Fixed - Fixed a redundant system prompt update (tools, rules) when an extension set the same active tools in a different order, for example a reversed `pi.setActiveTools()` list. Tools that stay active now keep their order and newly activated tools are appended ([#11495](https://github.com/earendil-works/pi/issues/11495)) +- Fixed `pi.exec()` crashing pi with `RangeError: Invalid string length` when a command printed more output than fits in a string. The default output cap is the smaller of 256 MiB and the runtime string limit; past the cap, the command is terminated and the result has `truncated: true` and a nonzero exit code ([#11360](https://github.com/earendil-works/pi/issues/11360)) +- Fixed `pi.exec()` never returning when a command ignored SIGTERM after a timeout or abort. pi now sends SIGKILL 5 seconds later, as intended - Fixed `pi install` and `pi remove` adding ever deeper nested entries to the npm `package-lock.json` when the agent directory is reached through a symlink, such as `~/.pi` linked into a dotfiles repo. pi now passes the resolved real path to npm ([#11358](https://github.com/earendil-works/pi/issues/11358)) - MCP sign-in now rejects authorization endpoints and other OAuth metadata URLs that are not `http:` or `https:`. Before, a malicious MCP server could supply a `file:` or custom protocol URL, which pi passed to the platform URL handler. - Automatic `fd` and `rg` downloads no longer fetch x86-64 binaries on ARM32 and other architectures without a prebuilt release. pi now warns and recommends installing the tool with the system package manager ([#11209](https://github.com/earendil-works/pi/issues/11209)) diff --git a/packages/coding-agent/docs/extensions.md b/packages/coding-agent/docs/extensions.md index 2439d1b45..7666d5645 100644 --- a/packages/coding-agent/docs/extensions.md +++ b/packages/coding-agent/docs/extensions.md @@ -1868,13 +1868,15 @@ pi.on("session_start", () => { ### pi.exec(command, args, options?) -Execute a shell command. +Execute a command without a shell. ```typescript const result = await pi.exec("git", ["status"], { signal, timeout: 5000 }); -// result.stdout, result.stderr, result.code, result.killed +// result.stdout, result.stderr, result.code, result.killed, result.truncated ``` +Options: `signal` and `timeout` terminate the command, `cwd` sets the working directory, and `maxBuffer` limits stdout and stderr combined (default 256 MiB). When output exceeds `maxBuffer`, pi terminates the command and returns the output up to the limit with `truncated: true`, `killed: true`, and a nonzero `code`. `killed` is true whenever pi terminated the command: timeout, abort, or `maxBuffer`. A command that ignores SIGTERM gets SIGKILL after 5 seconds. + ### pi.getActiveTools() / pi.getAllTools() / pi.setActiveTools(names) Manage active tools. This works for both built-in tools and dynamically registered tools. `pi.getActiveTools()` returns the active tool names as `string[]`; `pi.getAllTools()` returns metadata for all configured tools. diff --git a/packages/coding-agent/src/core/exec.ts b/packages/coding-agent/src/core/exec.ts index cf4205f7f..b6e845f45 100644 --- a/packages/coding-agent/src/core/exec.ts +++ b/packages/coding-agent/src/core/exec.ts @@ -2,10 +2,14 @@ * Shared command execution utilities for extensions and custom tools. */ +import { constants as bufferConstants } from "node:buffer"; import { spawn } from "node:child_process"; import { StringDecoder } from "node:string_decoder"; import { waitForChildProcess } from "../utils/child-process.ts"; +const DEFAULT_EXEC_MAX_BUFFER = Math.min(256 * 1024 * 1024, bufferConstants.MAX_STRING_LENGTH); +const FORCE_KILL_DELAY_MS = 5000; + /** * Options for executing shell commands. */ @@ -16,6 +20,11 @@ export interface ExecOptions { timeout?: number; /** Working directory */ cwd?: string; + /** + * Combined stdout/stderr byte limit; exceeding it terminates the command with `truncated: true`. + * Nonnegative integer up to the runtime string limit. Default: 256 MiB, capped by that limit. + */ + maxBuffer?: number; } /** @@ -25,12 +34,15 @@ export interface ExecResult { stdout: string; stderr: string; code: number; + /** The command was terminated by pi: timeout, abort, or output over `maxBuffer`. */ killed: boolean; + /** Output exceeded `maxBuffer`; stdout and stderr are incomplete and `code` is nonzero. */ + truncated: boolean; } /** * Execute a shell command and return stdout/stderr/code. - * Supports timeout and abort signal. + * Supports timeout, abort signal, and an output cap (`maxBuffer`). */ export async function execCommand( command: string, @@ -38,79 +50,109 @@ export async function execCommand( cwd: string, options?: ExecOptions, ): Promise { - return new Promise((resolve) => { - const proc = spawn(command, args, { - cwd, - shell: false, - stdio: ["ignore", "pipe", "pipe"], - }); + const maxBuffer = options?.maxBuffer ?? DEFAULT_EXEC_MAX_BUFFER; + if (!Number.isInteger(maxBuffer) || maxBuffer < 0 || maxBuffer > bufferConstants.MAX_STRING_LENGTH) { + throw new RangeError(`maxBuffer must be an integer between 0 and ${bufferConstants.MAX_STRING_LENGTH} bytes`); + } - let stdout = ""; - let stderr = ""; - // Decode across chunks: a multi-byte UTF-8 character can be split between two chunks, - // and decoding each chunk separately would corrupt it (earendil-works/pi#11312). - const stdoutDecoder = new StringDecoder("utf8"); - const stderrDecoder = new StringDecoder("utf8"); - let killed = false; - let timeoutId: NodeJS.Timeout | undefined; - - const killProcess = () => { - if (!killed) { - killed = true; - proc.kill("SIGTERM"); - // Force kill after 5 seconds if SIGTERM doesn't work - setTimeout(() => { - if (!proc.killed) { - proc.kill("SIGKILL"); - } - }, 5000); - } - }; - - // Handle abort signal - if (options?.signal) { - if (options.signal.aborted) { - killProcess(); - } else { - options.signal.addEventListener("abort", killProcess, { once: true }); - } - } - - // Handle timeout - if (options?.timeout && options.timeout > 0) { - timeoutId = setTimeout(() => { - killProcess(); - }, options.timeout); - } - - proc.stdout?.on("data", (data) => { - stdout += stdoutDecoder.write(data); - }); - - proc.stderr?.on("data", (data) => { - stderr += stderrDecoder.write(data); - }); - - // Wait for process termination without hanging on inherited stdio handles - // held open by detached descendants. - waitForChildProcess(proc) - .then((code) => { - if (timeoutId) clearTimeout(timeoutId); - stdout += stdoutDecoder.end(); - stderr += stderrDecoder.end(); - if (options?.signal) { - options.signal.removeEventListener("abort", killProcess); - } - resolve({ stdout, stderr, code: code ?? 0, killed }); - }) - .catch((error: Error) => { - if (timeoutId) clearTimeout(timeoutId); - stdout += stdoutDecoder.end(); - stderr += stderrDecoder.end(); - if (options?.signal) { - options.signal.removeEventListener("abort", killProcess); - } - resolve({ stdout, stderr: stderr || error.message, code: 1, killed }); - }); + const proc = spawn(command, args, { + cwd, + shell: false, + stdio: ["ignore", "pipe", "pipe"], }); + + let stdout = ""; + let stderr = ""; + // Decode across chunks: a multi-byte UTF-8 character can be split between two chunks, + // and decoding each chunk separately would corrupt it (earendil-works/pi#11312). + const stdoutDecoder = new StringDecoder("utf8"); + const stderrDecoder = new StringDecoder("utf8"); + let outputBytes = 0; + let truncated = false; + let killed = false; + let timeoutId: NodeJS.Timeout | undefined; + let forceKillId: NodeJS.Timeout | undefined; + + const killProcess = () => { + if (killed) return; + killed = true; + proc.kill("SIGTERM"); + forceKillId = setTimeout(() => { + // `proc.killed` only says a signal was sent, so check whether the child actually exited. + if (proc.exitCode === null && proc.signalCode === null) { + proc.kill("SIGKILL"); + } + }, FORCE_KILL_DELAY_MS); + }; + + if (options?.signal) { + if (options.signal.aborted) { + killProcess(); + } else { + options.signal.addEventListener("abort", killProcess, { once: true }); + } + } + + if (options?.timeout && options.timeout > 0) { + timeoutId = setTimeout(killProcess, options.timeout); + } + + // Keep only bytes within the combined cap (#11360). Close both pipes on overflow so writing + // descendants cannot keep waitForChildProcess busy after the child exits. Pausing is not + // enough: Node resumes the child's stdio on exit. + const takeWithinCap = (data: Buffer): Buffer | undefined => { + if (truncated) return undefined; + const remaining = maxBuffer - outputBytes; + if (data.length <= remaining) { + outputBytes += data.length; + return data; + } + truncated = true; + proc.stdout?.destroy(); + proc.stderr?.destroy(); + killProcess(); + return remaining > 0 ? data.subarray(0, remaining) : undefined; + }; + + proc.stdout?.on("data", (data: Buffer) => { + const chunk = takeWithinCap(data); + if (chunk) stdout += stdoutDecoder.write(chunk); + }); + + proc.stderr?.on("data", (data: Buffer) => { + const chunk = takeWithinCap(data); + if (chunk) stderr += stderrDecoder.write(chunk); + }); + + let code: number | null; + try { + // Do not hang on inherited stdio handles held open by detached descendants. + code = await waitForChildProcess(proc); + } catch (error) { + if (!(error instanceof Error)) throw error; + // Child-process errors (e.g. ENOENT) are failed executions, not host failures. + code = 1; + if (!stderr) stderr = error.message; + } finally { + clearTimeout(timeoutId); + clearTimeout(forceKillId); + options?.signal?.removeEventListener("abort", killProcess); + } + + // On truncation, discard any incomplete UTF-8 suffix buffered by either decoder. + // Flushing it would append a replacement character that was not in the accepted output. + if (!truncated) { + stdout += stdoutDecoder.end(); + stderr += stderrDecoder.end(); + } + + const exitCode = code ?? 0; + return { + stdout, + stderr, + // A truncated result must never read as success. + code: truncated && exitCode === 0 ? 1 : exitCode, + killed, + truncated, + }; } diff --git a/packages/coding-agent/test/exec-runtime-limit.test.ts b/packages/coding-agent/test/exec-runtime-limit.test.ts new file mode 100644 index 000000000..879a45911 --- /dev/null +++ b/packages/coding-agent/test/exec-runtime-limit.test.ts @@ -0,0 +1,25 @@ +import * as buffer from "node:buffer"; +import { expect, it, vi } from "vitest"; +import { execCommand } from "../src/core/exec.ts"; + +vi.mock("node:buffer", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + // 32-bit Node's maximum is 16 bytes below 256 MiB. + constants: { ...actual.constants, MAX_STRING_LENGTH: 268_435_440 }, + }; +}); + +// Regression tests for https://github.com/earendil-works/pi/issues/11360 +it("executes with the default cap when the runtime string limit is below 256 MiB", async () => { + const result = await execCommand(process.execPath, ["-e", 'process.stdout.write("ok")'], process.cwd()); + + expect(result).toEqual({ stdout: "ok", stderr: "", code: 0, killed: false, truncated: false }); +}); + +it("still rejects an explicit cap above the runtime string limit", async () => { + await expect( + execCommand(process.execPath, ["-e", ""], process.cwd(), { maxBuffer: buffer.constants.MAX_STRING_LENGTH + 1 }), + ).rejects.toThrow(RangeError); +}); diff --git a/packages/coding-agent/test/exec-utf8-chunks.test.ts b/packages/coding-agent/test/exec-utf8-chunks.test.ts new file mode 100644 index 000000000..5dad0aae0 --- /dev/null +++ b/packages/coding-agent/test/exec-utf8-chunks.test.ts @@ -0,0 +1,53 @@ +import type * as ChildProcessModule from "node:child_process"; +import { ChildProcess, spawn } from "node:child_process"; +import { PassThrough } from "node:stream"; +import { setImmediate } from "node:timers/promises"; +import { expect, it, vi } from "vitest"; +import { execCommand } from "../src/core/exec.ts"; + +vi.mock("node:child_process", async (importOriginal) => ({ + ...(await importOriginal()), + spawn: vi.fn(), +})); + +// #11360: real streams control chunk boundaries without relying on OS pipe timing. +// Only process creation and termination are stubbed; decoding and process waiting are real. +it.each(["stdout", "stderr"] as const)( + "discards pending %s UTF-8 bytes when either stream exceeds the cap", + async (stream) => { + const child = new ChildProcess(); + const stdout = new PassThrough(); + const stderr = new PassThrough(); + child.stdout = stdout; + child.stderr = stderr; + vi.mocked(spawn).mockReturnValueOnce(child); + vi.spyOn(child, "kill").mockImplementation(() => { + queueMicrotask(() => { + child.emit("exit", null, "SIGTERM"); + child.emit("close", null, "SIGTERM"); + }); + return true; + }); + + const resultPromise = execCommand("test-command", [], process.cwd(), { maxBuffer: 3 }); + const output = stream === "stdout" ? stdout : stderr; + const otherOutput = stream === "stdout" ? stderr : stdout; + // The decoder first returns "a" and buffers the first byte of €. + output.write(Buffer.from([0x61, 0xe2])); + await setImmediate(); + // The other decoder also buffers the first byte of € before the shared cap is exceeded. + otherOutput.write(Buffer.from([0xe2])); + await setImmediate(); + output.write(Buffer.from([0x82, 0xac])); + + const result = await resultPromise; + expect(result).toMatchObject({ + stdout: stream === "stdout" ? "a" : "", + stderr: stream === "stderr" ? "a" : "", + truncated: true, + killed: true, + }); + expect(result.code).not.toBe(0); + expect(Buffer.byteLength(result.stdout) + Buffer.byteLength(result.stderr)).toBeLessThanOrEqual(3); + }, +); diff --git a/packages/coding-agent/test/exec.test.ts b/packages/coding-agent/test/exec.test.ts index 96232a8b3..df5a63817 100644 --- a/packages/coding-agent/test/exec.test.ts +++ b/packages/coding-agent/test/exec.test.ts @@ -1,3 +1,4 @@ +import { constants as bufferConstants } from "node:buffer"; import { describe, expect, it } from "vitest"; import { execCommand } from "../src/core/exec.ts"; @@ -9,6 +10,7 @@ describe("execCommand", () => { expect(result).toMatchObject({ code: 1, killed: false, + truncated: false, stdout: "", }); expect(result.stderr).toMatch(/ENOENT/); @@ -17,6 +19,110 @@ describe("execCommand", () => { it("keeps stderr empty when a command exits silently", async () => { const result = await execCommand(process.execPath, ["-e", "process.exit(1)"], process.cwd()); - expect(result).toEqual({ stdout: "", stderr: "", code: 1, killed: false }); + expect(result).toEqual({ stdout: "", stderr: "", code: 1, killed: false, truncated: false }); }); + + // Regression tests for https://github.com/earendil-works/pi/issues/11360 + it.each([ + -1, + 0.5, + Number.NaN, + Number.POSITIVE_INFINITY, + Number.NEGATIVE_INFINITY, + bufferConstants.MAX_STRING_LENGTH + 1, + ])("rejects invalid maxBuffer %s instead of attempting execution", async (maxBuffer) => { + // A missing command normally resolves to a failed result; invalid options must reject instead. + await expect( + execCommand("pi-command-that-does-not-exist-11360", [], process.cwd(), { maxBuffer }), + ).rejects.toThrow(RangeError); + }); + + it.each([0, bufferConstants.MAX_STRING_LENGTH])("accepts maxBuffer boundary %s", async (maxBuffer) => { + const result = await execCommand(process.execPath, ["-e", ""], process.cwd(), { maxBuffer }); + + expect(result).toEqual({ stdout: "", stderr: "", code: 0, killed: false, truncated: false }); + }); + + it("truncates all output when maxBuffer is zero", async () => { + const result = await execCommand(process.execPath, ["-e", 'process.stdout.write("x")'], process.cwd(), { + maxBuffer: 0, + }); + + expect(result).toMatchObject({ stdout: "", stderr: "", truncated: true, killed: true }); + expect(result.code).not.toBe(0); + }); + + it("caps combined output at maxBuffer, terminates the command, and reports truncation", async () => { + const script = + 'process.stderr.write("e".repeat(600)); process.stdout.write("o".repeat(600)); setInterval(() => process.stdout.write("x".repeat(1000)), 10);'; + const result = await execCommand(process.execPath, ["-e", script], process.cwd(), { maxBuffer: 1000 }); + + expect(result).toMatchObject({ truncated: true, killed: true }); + expect(result.code).not.toBe(0); + expect(Buffer.byteLength(result.stdout) + Buffer.byteLength(result.stderr)).toBe(1000); + }); + + // #11360: a byte cap must not turn a partial UTF-8 character into replacement output. + it.each([ + ["stdout", "€", 1, ""], + ["stderr", "€", 1, ""], + ["stdout", "a€🙂", 5, "a€"], + ["stderr", "a€🙂", 5, "a€"], + ] as const)( + "keeps complete UTF-8 characters from %s output %s capped at %i bytes", + async (stream, output, maxBuffer, expected) => { + const script = `process.${stream}.write(${JSON.stringify(output)})`; + const result = await execCommand(process.execPath, ["-e", script], process.cwd(), { maxBuffer }); + + expect(result).toMatchObject({ [stream]: expected, truncated: true, killed: true }); + expect(result.code).not.toBe(0); + expect(Buffer.byteLength(result.stdout) + Buffer.byteLength(result.stderr)).toBeLessThanOrEqual(maxBuffer); + }, + ); + + it("still flushes incomplete UTF-8 at normal EOF when output was not truncated", async () => { + const script = "process.stdout.write(Buffer.from([0xe2])); process.stderr.write(Buffer.from([0xf0]));"; + const result = await execCommand(process.execPath, ["-e", script], process.cwd()); + + expect(result).toEqual({ stdout: "�", stderr: "�", code: 0, killed: false, truncated: false }); + }); + + it("leaves output up to maxBuffer unchanged", async () => { + const result = await execCommand( + process.execPath, + ["-e", 'process.stdout.write("a".repeat(1000))'], + process.cwd(), + { + maxBuffer: 1000, + }, + ); + + expect(result).toEqual({ stdout: "a".repeat(1000), stderr: "", code: 0, killed: false, truncated: false }); + }); + + it.skipIf(process.platform === "win32")( + "returns when a shell's child keeps writing after the shell is terminated", + async () => { + // SIGTERM kills only `sh`; `yes` keeps the output pipes open and writing. + const result = await execCommand("sh", ["-c", "yes; echo done"], process.cwd(), { maxBuffer: 1000 }); + + expect(result).toMatchObject({ killed: true, truncated: true }); + expect(result.stdout).toBe("y\n".repeat(500)); + }, + 10_000, + ); + + it.skipIf(process.platform === "win32")( + "force-kills a command that ignores SIGTERM", + async () => { + // The handler is installed before the first write, so the kill triggered by maxBuffer + // cannot arrive before the child ignores SIGTERM. + const script = 'process.on("SIGTERM", () => {}); process.stdout.write("xx"); setInterval(() => {}, 1000);'; + const result = await execCommand(process.execPath, ["-e", script], process.cwd(), { maxBuffer: 1 }); + + expect(result).toMatchObject({ stdout: "x", killed: true, truncated: true }); + expect(result.code).not.toBe(0); + }, + 15_000, + ); }); diff --git a/packages/coding-agent/test/git-merge-and-resolve-extension.test.ts b/packages/coding-agent/test/git-merge-and-resolve-extension.test.ts index 7c8741c7e..7903a4078 100644 --- a/packages/coding-agent/test/git-merge-and-resolve-extension.test.ts +++ b/packages/coding-agent/test/git-merge-and-resolve-extension.test.ts @@ -7,8 +7,8 @@ import type { ExecResult, ExtensionAPI, ExtensionContext } from "../src/core/ext type AgentEndHandler = (event: { type: "agent_end" }, ctx: ExtensionContext) => Promise; -const ok: ExecResult = { stdout: "", stderr: "", code: 0, killed: false }; -const fail: ExecResult = { stdout: "", stderr: "error", code: 1, killed: false }; +const ok: ExecResult = { stdout: "", stderr: "", code: 0, killed: false, truncated: false }; +const fail: ExecResult = { stdout: "", stderr: "error", code: 1, killed: false, truncated: false }; /** Standard exec results for a clean repo tracking origin/main, not in a merge. */ function withUpstream(results: Map): Map { diff --git a/packages/coding-agent/test/input-transform-streaming-example.test.ts b/packages/coding-agent/test/input-transform-streaming-example.test.ts index faf95b6ba..0eb1ccf60 100644 --- a/packages/coding-agent/test/input-transform-streaming-example.test.ts +++ b/packages/coding-agent/test/input-transform-streaming-example.test.ts @@ -35,9 +35,9 @@ function setup(execResult: ExecResult) { describe("input-transform-streaming example", () => { const diffOutput = " src/index.ts | 5 ++---\n 1 file changed, 2 insertions(+), 3 deletions(-)"; - const gitSuccess: ExecResult = { stdout: diffOutput, stderr: "", code: 0, killed: false }; - const gitEmpty: ExecResult = { stdout: "", stderr: "", code: 0, killed: false }; - const gitFail: ExecResult = { stdout: "", stderr: "not a git repo", code: 128, killed: false }; + const gitSuccess: ExecResult = { stdout: diffOutput, stderr: "", code: 0, killed: false, truncated: false }; + const gitEmpty: ExecResult = { stdout: "", stderr: "", code: 0, killed: false, truncated: false }; + const gitFail: ExecResult = { stdout: "", stderr: "not a git repo", code: 128, killed: false, truncated: false }; it("skips exec during steering", async () => { const { emit, exec } = setup(gitSuccess); From 556860a386a03113eca146abd9cbb08e9391b7fb Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 23:16:28 +0000 Subject: [PATCH 124/271] docs(chord): no client providers for now --- packages/chord/docs/next/decisions.md | 9 +++++++++ packages/chord/docs/next/design.md | 4 +++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index f425d5bbf..59893c07b 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1771,6 +1771,15 @@ validation and copies the v2 kernel added around Delta. setups a registry without a value cannot get one before a reload, so a reader then fails at once `transient` (`unavailable/undeclared`) and is retried when the registry gets its first value (R18), as `self.get` of a type not served here fails at once (refined 2026-10-27 after the second spec review: no `readyTimeout` wait at start); once live, a later failure of the registry changes nothing (last good value). +- **No client providers for now (owner, 2026-10-28: "yeah, no client providers for now"; reviews + `/tmp/barrier-{astra,fable,opus,sol}.md`):** clients stay unaddressable and serve only `local` types (design §3). + Client-side capabilities (a phone's camera, clipboard, a page's WebMCP tools) go through an owner's state, as the + Questions service: a tool asks an owner store (`request`, `wait`), clients watching its open requests carry them out + and answer with a method call (the answer is method arguments, checked against the owner's schema; bytes as `bytes()` + leaves, chunked if large); a client may advertise the tools it can run to such a store, tied to its presence. Why: + a client as a provider would feed untrusted results, state and ops into server and worker objects (consumers never + validate them), needs client addresses, routing down through relays and a principal for clients; the owner pattern + needs nothing new in Chord and survives link drops and several devices by the owner's rule. - **Design audit of slices 1-6a (owner, 2026-10-27; `/tmp/design-audit-{opus,sol}.md`):** the built code follows the design except for five bugs, fixed after 6b's commit with the auditors' reproductions as tests: on a failed activation references are released after the natives closed (§6, §13); a dependency going live while the dependent diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index b3ec994d1..dc38b5642 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -89,7 +89,9 @@ worker worker worker addressable by a typed address, started on demand | worker | yes, many | stamped by the parent | started on demand by the app | Clients serve only `local` types (they are not addressable): a host without a principal refuses to serve a remote type -(`load` throws; a remote object is owned by a principal of the host serving it; owner, 2026-10-24). A client's local +(`load` throws; a remote object is owned by a principal of the host serving it; owner, 2026-10-24). No client providers for now (owner, +2026-10-28): client-side capabilities (a camera, WebMCP tools) go through an owner's requests that clients answer +(section 20, the Questions service). A client's local calls have no principal; what a client calls on the server is stamped with the client's authenticated principal. Clients reach workers only through the server, which relays; workers reach each other through the server too. A detached process (e.g. a terminal host) would be another child kind of the server. Things that are not hosts: a code From e65b3b88c08412bb8ffbd21fb5dc5756064a6e5a Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 29 Oct 2026 00:51:57 +0000 Subject: [PATCH 125/271] docs(chord): dependency model proposal for slice 7 and its explainer (Parked Until Resolved) --- packages/chord/docs/next/dependencies/book.md | 1674 +++++++++++ .../chord/docs/next/dependencies/index.html | 2647 +++++++++++++++++ .../chord/docs/next/dependencies/model.md | 419 +++ 3 files changed, 4740 insertions(+) create mode 100644 packages/chord/docs/next/dependencies/book.md create mode 100644 packages/chord/docs/next/dependencies/index.html create mode 100644 packages/chord/docs/next/dependencies/model.md diff --git a/packages/chord/docs/next/dependencies/book.md b/packages/chord/docs/next/dependencies/book.md new file mode 100644 index 000000000..19b550bdf --- /dev/null +++ b/packages/chord/docs/next/dependencies/book.md @@ -0,0 +1,1674 @@ +--- +title: Parked Until Resolved: one model for dependencies in Chord v2 +cover: Parked / Until Resolved +kicker: A design proposal · Chord v2 · slice 7 · 2026-10-29 +subtitle: What happens when what an object needs is not there yet, is somewhere else, has gone away or does not fit, told through the scenarios that decide it +stats: + - 5 → 1 | answers today to "the provider is not there" + - 0 | deadlines on waiting for a dependency + - 300+ | scenarios played through by four reviewers + - 6 | decisions left for the owner +note: | + A proposal, not a decision. It states the dependency model the redesign session drafted on the night of + 2026-10-28/29 from the owner's direction ("park until resolved"; "no difference between local and remote + resolution: a link going is like a plugin unloading") and five rounds of review by Astra, Fable, Opus and Sol, who + played more than 300 scenarios from p2's plugins and their own against it. The working document is + `/tmp/model/model.md`; the reviews are in `/tmp/dep-graph-*.md`, `/tmp/barrier-*.md` and `/tmp/model/review*-*.md`. + Code references are to pi `main` at `358f7d46a` (`packages/chord/src/next`). Notes marked *review* record a point + the reviewers found or still disagree on; *Owner* marks a decision that is his. +colophon: Parked Until Resolved · first edition · 2026-10-29 · Chord v2 design proposal for slice 7 +--- + +# Before we begin {.preface} + +It is a Tuesday morning and Mario opens a session on his phone. The server starts a worker for it. The worker +connects to the server, loads its plugins and opens the agent's harness. One of the plugins, code mode, needs the +harness before it can contribute its tools. Another, search, needs a feed that lives on the server. A third, sharing, +was removed from the worker's plugin set last night, but the notes view still asks for it. At the same moment the +browser tab holds the session's notes, which live on the worker that is still starting. + +Every one of these is the same question asked in different places: *what I need is not there yet; what now?* Chord +v2 answers it today in five different ways, depending on how the dependency was declared and where its provider +lives. A missing type on the same host breaks the object for good; the same type missing on another host makes it +retry every few seconds forever; a missing registry waits for a value; a reload that removes a type ends every +reference to it. And when two hosts start at once and need each other's objects in a chain that has no cycle at all, +both wait until a 60-second deadline breaks the knot. + +The owner's direction was short: an object should be **parked until its dependencies resolve**, and there should be +no difference between a dependency on this host and one on another: a link going away is like the provider's plugin +being unloaded, and a link coming back is like it being loaded. This book works out what that means, rule by rule, +and plays it through the scenarios that decide whether it holds. + +## Eight things to know + +:::findings +1. **A request outlives its provider.** A reference stands for a request (type, params, principal, version). It ends + only when its holder ends it, never because the provider is missing, went away or does not fit. +2. **One resolver for local and remote.** The address decides where a request goes; the provider found there is + checked; a missing provider here and a missing provider on another host give the same answer. +3. **Two kinds of not-yet.** Something under way (activating, reconnecting, installing, not published yet) shows as + `loading`; nothing under way (not served, no fitting version, a dependency failed) shows as `failed` with the new + cause `waiting`. +4. **Dependents park.** An object whose dependencies are not all live waits with no timer and no backoff, and is + woken by events: a dependency changing state, or a name becoming usable on a host. +5. **`broken` means the object's own fault.** Only its own code and cycles break an object; whatever it waits for is + `waiting`. +6. **Startup needs one short gate, not a barrier.** Requests wait only for the host's facets to be published, which + is synchronous; that ends the startup deadlock. Whether `load` then also waits for first outcomes is the owner's + call. +7. **One gap in cycle detection needs a choice.** Objects on two hosts that start at the same moment and need each + other are caught today only by a deadline. A probe can walk the waiting edges and find the ring without failing + anything (recommended); a narrowed deadline is simpler but polls, misses a ring, or never finds the cycle at all. +8. **Only callers and own work keep clocks.** Getters, parked calls and an object's own steps keep their deadlines; + waiting for a dependency has none. +::: + +## Words used throughout + +A **host** is one participant (the server, a session worker, a browser tab). A **provider** is what serves a type or +a registry on a host: a facet registered it with `host.serve`. A **request** is what a reference stands for: a type +or registry, its params, the principal it acts as and the consumer's contract version. Its **holder** made it and +owns it: an object's attempt (`self.get`, `self.registry`), a call (`ctx.get`), outside code (`host.get`, +`host.hold`) or a peer host (a `hold` arriving over a link). A request is **bound** when it points at an object (here) +or a hold (over a link), and **unbound** with a reason otherwise. An **attempt** is one try at activating an object: +its factory, its requests, its natives. A **dependency** is a request an attempt needs live before it may go live. An +attempt is **parked** while one of its dependencies is not live. A **publication** is the moment a host makes its +facets' declarations visible: the setups of `load`, the commit of a reload, the commit of an install. A **wake** is an +event that makes the resolver look at a request again. + +The running example is a small p3: a server **S** with the session catalog and the search feed; a worker **W** for +session `S1` with the harness, code mode, the notes store and the search indexer; and a browser tab **T** that shows +the notes. The plugins are `core`, `models`, `codemode`, `notes`, `search` and `sharing`. + +# Five Answers to One Question {#today lede="What Chord v2 does today when a dependency is missing, and why the answers disagree."} + +Chord v2 was built slice by slice, and each slice answered "the provider is not there" for the case in front of it. +Slice 1 made `self.get` of a type nobody serves throw, because a typo in a type name should fail loudly. Slice 3b made +remote holds survive link drops, so a missing provider on another host became a refused hold that the consumer retries. +Slice 8 made `self.registry` a dependency that waits for a value. Each answer was reasonable alone. Together they +give five outcomes for one situation. + +| Declared as | The provider is missing | Today | +| --- | --- | --- | +| `self.get(LocalType)` | no facet serves it | `self.get` throws, so the factory throws: the object is `broken` | +| `self.get(RemoteType)` | the worker does not serve it | the hold is refused `broken`; the dependent is `transient` and retried every 1 to 60 seconds, forever | +| `self.registry(R)` | no plugin serves `R` | `transient`, woken at `R`'s first value only | +| `self.registry(R)`, the reader newer than the provider | | `broken` | +| a reload removes the type | | every holder gets `gone`: ended for good | + +The differences are not about the situation. Whether the notes plugin is missing on this host or on the worker, the +notes view cannot work until someone installs it. Yet in one case the view is broken until a reload, in the other it +polls the worker every minute for the rest of the session, and the reload that finally brings the plugin back cannot +revive a reference that got `gone`. + +## The deadlock without a cycle + +The startup barrier has a worse problem. Sol reproduced it against the current code (`/tmp/dep-graph-sol-barrier.log`). + +:::sequence animate | The startup deadlock today. Nothing in the object graph is cyclic; the two barriers block each other. +actor s "server S" "load: waits for A" +actor w "worker W" "load: waits for BStore" +s -> s: setups: serve C; start eager A +s => w: A needs B: hold B +w -> w: setups: serve B; start eager BStore +w => s: BStore needs C: hold C +s -> s: Hold.start waits for S.pendingLoad +note s..w: S's load waits for A, A for B, B for BStore, BStore for C, C's hold for S's load +divider "60 s later" +s -> s: A's activation deadline fires, A fails +s --> w: C binds, BStore and B go live, A is retried +::: + +`C` has no dependencies and is registered the moment S's setups ran. Its hold waits anyway, because incoming holds +wait for `pendingLoad`, and `pendingLoad` is the whole of `load`: the setups and the barrier that waits for every +host-lifetime object, `A` included. Two jobs share one promise. Holds only need the first ("are the types +registered?"); the second ("has every plugin had its first try?") is for the app. Mixing them makes an acyclic chain +across two hosts wait until a timeout fires. + +## Timers standing in for events + +Both problems have the same root. Where Chord does not know when something will change, it polls or waits for a +deadline: the identity backoff retries a dependent whose remote provider refused it; the activation deadline breaks the +startup knot; the same deadline is the only answer to two objects on two hosts that start at the same moment and need +each other, a cycle the activation chains cannot see (design §6, "Known limitation"). + +A timer is the right tool when nothing else can tell Chord that the world changed: a disk that may be back, a worker +process that may start this time, a silent transport. It is the wrong tool when an event exists or can exist. A +missing plugin comes back through a reload, which is an event on the host that serves it. A dependency going live is +an event. A link opening is an event. The model below uses those events and keeps timers only where no event can +exist. + +# Requests That Wait {#requests lede="A reference is a request with a holder; it outlives every provider it meets."} + +The first change is a separation the code makes only halfway today: between what a reference **asks for** and what +currently **answers** it. When the notes view on W does `self.get(NoteStore, {})`, it asks for "the NoteStore of this +host, as the host, at contract version 3". Which object answers is a second question, and the answer can change: no +object yet, an object activating, a live one, a failed one, none again after a reload, a new one after the next. + +## A request and its holder + +A **request** is the asking part: type (or registry), params, principal and the consumer's contract version. Every +request has exactly one **holder**, which made it and is the only one that ends it. + +| Holder | Made by | Ends when | +| --- | --- | --- | +| an attempt | `self.get`, `self.registry`, a keyed pin `ref.notes.at(key)` | the attempt ends (its object ended, or a reload replaced it) | +| a call | `ctx.get` | the call settles | +| outside code | `host.get`, `host.hold`, `host.registry` | `release()`, or the getter gives up | +| a peer host | a `hold` arriving over a link | the peer releases it, or its link or origin closes | +| any of them | | revocation of its principal; its own host closes | + +That list is the whole of how a request ends. **A request never ends because its provider is missing, went away or +does not fit.** Today it does in two places: a `self.get` of a type nobody serves throws before the request exists, +and a reload that removes a type sends `gone`. Both go. + +## The address decides where + +A request goes to one place, and the address decides which. For a `local` type, the place is this +host. For a `remote` type, the consumer computes the address from the params with its own contract's +placement function (design §3): `Notes { session: "S1" }` is placed on `Session { id: "S1" }`. If that is this host's +address, the request goes here; otherwise over the route for that kind, or over the parent link. + +Today the code checks something else first: whether the type is registered here. A type placed on this host but not +registered yet is sent over the parent link, which is wrong twice. Sol's example: the server has address `Server`, +the catalog type is placed on `Server`, and no facet serves it yet. Today's `Placement.reference` sends the server's +own request over a parent link the server never had. And when the catalog plugin is installed later, nothing wakes +that request, because it was never registered as waiting here. + +:::flow wide | Placement by address. The destination follows from the address alone; only then is the provider found there checked. +row "Where" accent +box addr "the request's address" "from the consumer's own placement function" accent +box here "here" "a local type, or this host's address" +box away "a route or the parent" "every other address" blue +row "Then" muted +box check "check the provider here" "served? fitting version? authorize, own placement" +box hold "hold over the path" "the provider host checks it the same way" blue +arrow addr -> here +arrow addr -> away +arrow here -> check +arrow away -> hold +::: + +The rule, from the reviews (Sol B1, Astra F1): placement never looks at what is registered when the type names an +address. A request placed here whose provider is missing is **unbound here**, waiting for the name. A version check +compares the consumer with the provider actually selected, never with a same-name registration on the consumer's own +host (today `#assertRoute` does that before placement, so worker W1 serving `Notes` v1 rejects a `Notes` v2 request +placed on W2, which serves v2). + +One case keeps today's rule: a `remote` type declared **without placement** (allowed by `remote()`, used in tests and +for server types a tab reaches through its parent). It goes here if it is served here, otherwise over the parent. A +request of that kind made before the host has a parent link (a tab loads before it connects) waits here as not served +and is placed again when the host's first parent link connects (Opus, Sol); on a host that never connects, the root, +it stays here. Opus would rather require a placement on every remote type, which is simpler; it removes a supported +declaration, and the owner asked for no feature to go, so the draft keeps the fallback. + +## What the check finds + +Once placed, the provider found there is checked: served, a fitting version (the current one or an accepted older one, +with the fingerprint computed from the provider's own history), `authorize`, and the provider's own placement check. +The request is then **bound** (to an object here, or to a hold over a path), or **unbound** with a reason: + +| The check finds | The request | Shows | +| --- | --- | --- | +| a fitting provider | bound | its object's state | +| this host has not published its facets yet | unbound `unpublished` | `loading` | +| nobody serves the name at its address | unbound `not_served` | `failed { waiting }` | +| no fitting version, a fingerprint that differs at the same version, a placement the provider computes elsewhere | unbound `version` | `failed { waiting }` | +| the registry is served, but its first `compose`, or the reader's version mapping, threw | unbound `compose` | `failed { waiting }` | +| the provider host is installing, and the hold does not fit its current code | unbound `installing`, at the provider | `loading` | +| the path has no link (a drop, a route resolving) | waits on the path | `loading` | +| `authorize` refused, or the params are invalid | refused | `failed { broken }`, as today | + +The reasons that wait for a name (`unpublished`, `not_served`, `version`, `compose`) are kept in an index under that +name, on the host that decided them: the consumer's host for a local request, the provider's host for a hold that +arrived over a link. The provider already keeps such holds today as **refused** (`remote/hold.ts`), and `retry` re-runs +them. `installing` belongs to the provider host's own install; `no_link` lives on the path, where the next link holds +the request again, as today. Every index drops an entry when its request ends, so a released getter, a closed tab or a +revoked guest leaves nothing behind. A refused request (no authority, bad params) is never indexed: a later +publication cannot grant authority. + +## The publication gate + +There is one short gate. Before a host has run its facets' setups, it does not know what it serves, so it cannot tell +"not served" from "not registered yet". A worker connects to its parent before it loads (W6), so holds can arrive in +that window, and p2's web shell holds references before its own `load` resolved. + +So a request made on a host, or arriving on its links, before the host's **first publication** is not placed at all: +it is unbound `unpublished` and shows `loading`. The publication places it. Without this gate the request would be +placed by a half-empty registration table (Fable's trace: W1's own `Notes` sent over the parent, relayed back by the +server, forever), and a tab would see "not served" during every worker start (Opus's trace). + +The gate is not the barrier. It lasts only until the setups ran, and the setups are synchronous code at the start of +`load`: they register types and registries and contribute, and wait for nothing. That is the difference that ends +the startup deadlock (chapter VI). + +# Loading or Waiting {#states lede="What a reference shows while it is not live, and why a missing plugin is an answer rather than a spinner."} + +`ref.state` keeps its four kinds: `loading`, `live`, `failed`, `ended`. What changes is one new kind of failure cause. + +```ts +type RefState = + | { kind: "loading" } + | { kind: "live" } + | { kind: "failed"; cause: FailureCause; retryAt: number | undefined } + | { kind: "ended"; error: ChordError }; + +type FailureCause = + | { kind: "transient"; error: ChordError } // the object's own effect failed (an open, a follow): backoff + | { kind: "broken"; error: ChordError } // the object's own code, a cycle, or authority: retry() or a reload + | { kind: "waiting"; error: ChordError }; // not the object's fault: woken by an event, never by a timer +``` + +## The mapping + +Whether a request that is not live shows `loading` or `failed { waiting }` follows from what it waits for: + +| The request is | `ref.state` | Calls on it | +| --- | --- | --- | +| bound, its object activating, or parked with no failed blocker | `loading` | park, bounded | +| unbound `unpublished` or `installing`, or its path has no link | `loading` | park, bounded | +| unbound `not_served`, `version` or `compose` | `failed { waiting }`, no `retryAt` | reject at once with the cause | +| bound, its object failed | `failed` with the object's own cause and `retryAt`, as today | reject at once | +| bound, its object live | `live` | run | +| ended by its holder, revoked, or its host closed | `ended` | reject | + +The intuition is "is something under way?": an activation, a reconnect, an install, a load. When something is under +way, the reference waits and calls park; when nothing is, the reference says what is missing and calls fail fast. +The reviewers pointed out that "under way" is not something Chord can observe in general (a parent reconnect is the +app's, a `resolve` may hang), so the table is the rule and the intuition only explains it. + +## A link drop and an unload, side by side + +The owner's point was that a link going away is like the provider's plugin unloading. For the request they are the +same: it stays, it keeps its last values readable, it binds again when a provider is reachable, and its holder +notices nothing but the state. They differ in what is under way. + +:::steps | The notes view on T, once through a link drop and once through an unload on W ++++ Live +T's notes view holds `Notes { S1 }` on W through the server. `ref.state` is `live`; the view renders the notes. ++++ The link drops +W's link to the server closes (a crash, a network change). The request on T stays; the path has no link: unbound +`no_link`, `ref.state` `loading`. The last notes stay readable. A call made now parks, bounded by `readyTimeout`. +Calls already sent fail and are never resent: their outcome is unknown. ++++ The link comes back +The route resolves again, the request is held anew on the new link, `bound` arrives with a fresh snapshot. `live`. +Parked calls go out in order. ++++ The plugin is removed on W +A reload on W removes the `notes` plugin. The provider's object ends as today: its natives close, its callers are +rejected at once, its handlers drain. The request on T does not end: W sends `failed { waiting, not_served }` on +the bound hold, which unbinds it. `ref.state` is `failed { waiting }`; the last notes stay readable; new calls fail at once with "notes is +not served on Session S1". ++++ The plugin comes back +A later reload on W serves `Notes` again. The publication re-runs W's refused holds for that name: `loading`, then +`bound`. T's view is live again without having done anything. +::: + +One difference must stay: calls already sent. When a link drops, a sent call's outcome is unknown, so it fails and +is not resent. When a plugin unloads on the same host, its handlers are drained on the old code (design §21). Location +decides nothing else. + +:::review | Review: a third cause kind, or not +Opus and Fable recommend `waiting` as above. Astra would keep two kinds and mark event-waited failures as `transient` +with no `retryAt` (today's wire already carries a `transient` cause without `retryIn`, `remote/remote-object.ts`), so +no kind is added; the cost is that `transient` then means two things, a backoff under way or an event awaited. Sol would show `loading` with a diagnostic; that parks every call on a missing plugin for 60 +seconds and spins forever in the UI. The draft recommends `waiting`. **Owner** decision 3. +::: + +# Parked {#parked lede="An object whose dependencies are not all live waits, with no timer, and goes on the moment they are."} + +The owner's instinct was that an object is parked until its dependencies all cleanly resolve. This chapter makes that +exact: what a dependency is, what the parked object shows, what it keeps, and where parking ends. + +## What counts as a dependency + +A **dependency** is a request an attempt makes in its factory and needs live before the object may go live: + +- `self.get(Type, params)`, local or remote: one rule for both; +- `self.registry(R)`: a registry at the reader's version (chapter X); +- a keyed pin, `ref.notes.at(key)` on another object's keyed member: the key must be loaded. + +Two things are not dependencies. `self.follow(attach)` is the attempt's own step: its `attach` runs Durable's code +and may fail like an `open`, with the object's own `transient` failure and backoff (a follower that wants to recover +the moment a new harness appears declares `self.retryOn` on the registry it reads the harness through). And a +`ctx.get` in a method is not part of the object's activation at all: it is the call's request. + +## The dependency check + +An attempt runs its factory, hydrates its persisted singletons (they are the object's documents, kept across +attempts, as today), makes its requests, and is **parked** while any dependency is not live (and hydrated). What it +shows follows its dependencies: + +| Its dependencies | The parked object shows | +| --- | --- | +| any `loading`, none failed | `loading` | +| any `failed` | `failed { waiting, "waiting for X: " }`, no `retryAt`; the message keeps the nearest blocker and the root cause within the 500-character rule | +| one failed with `invalid/cycle` (a dependent off the ring) | `broken` with the same error: it would wait on a ring that never heals by itself | +| one `ended` (revoked, or its host closed) | `broken` with that error: the request cannot come back; `retry()` or a reload makes new requests | + +When every dependency is live, the attempt passes the **dependency check** and runs its own steps: its natives in +declaration order, its follow attaches, the member checks, its derives. Those are +bounded by its own `readyTimeout` deadline, as today. Then it goes live. + +The dependency check is the boundary between waiting and working (Astra's F3). Before it, the attempt holds no +natives; after it, a dependency that leaves `live` is treated as after live (R18): the attempt goes on, and its calls +through that reference meet the reference's state. So a parked attempt never holds a native, and an attempt that +opened a native is never parked. This is a deliberate rule, and Astra called it what it is: a weakening of "live +means every dependency is live". An object can finish opening after a dependency went away. Without that boundary a contributor unloading while the attempt opens its natives +would leave it half open and parked, or force a seal, a drain and a fresh attempt for a dependency the object may not +even use at once. + +:::sequence animate | Code mode on W parks on the harness and goes on the moment it is live. +actor cm "CodeMode" "host-lifetime, local" +actor h "Harness" "host-lifetime, local" +actor m "HarnessModels" "registry" +cm -> h: self.get(Harness): loading +note cm: parked: shows loading, no timer +h -> m: self.registry(HarnessModels): usable +h -> h: own steps: open Durable +h => cm: Harness live: a state change +cm -> cm: dependency check passes; open the sandbox +cm -> cm: live; contribute tools +::: + +## No clock while parked + +A parked attempt has no dependency deadline and no backoff. Its `readyTimeout` deadline is paused while it is +parked, as eager migrations already pause it (`attempt.ts`, `deadline.pause()`), and resumes with the budget left when +the attempt passes the check. Today a dependency that does not go live within 60 seconds fails its dependent +`transient`, which then retries with backoff from 1 to 60 seconds. Both go: the dependent is woken when the dependency +changes, and the time a dependency takes is bounded where it runs, by that dependency's own steps. + +## What a parked attempt keeps + +It keeps its documents (the object's), its declarations and its requests (R18's holding today), and its seal token. +It keeps no native. It ends with its object: when its last holder releases it, or, for a host-lifetime object, when +its plugin is removed or the host closes. A parked object +with params that nobody holds any more ends and releases its requests, so a whole parked chain goes when its top is +released. + +## `broken` is the object's own fault + +After the model, `broken` means exactly three things: the object's own code failed (a throwing factory, a derive that +throws at activation, a hydration or `migrate` that fails, a document fingerprint mismatch, an `open` that throws +`broken(...)`), the object is on a cycle, or it lost its authority (`forbidden`, a dependency that ended). Everything an object waits for, including a provider that is itself +broken, makes it `waiting`. The difference matters for recovery: a broken object needs `ref.retry()` or a reload of its +own plugin; a waiting one recovers by itself when what it waits for recovers. + +## Retry passes on + +A Retry button on a waiting object should do something. Opus's trace: the tab's `Panel` waits for the server's `Dash`, +which waits for `Catalog`, which is broken by a factory bug that a configuration fix would heal at the next attempt. +The user sees "waiting for Dash: waiting for Catalog: …" and presses Retry. Today `Panel` would re-check, find `Dash` +still failed and do nothing. + +So `ref.retry()` on a `failed { waiting }` object retries its failed blockers, hop by hop, with the existing `retry` +message over links: it retries the blockers that are `broken` or `transient` (the only ones a retry can change), +passes through the `waiting` ones, and leaves the rest alone. Each retry carries an operation id, so a diamond of +dependencies retries a shared leaf once. Retry of a `broken` or `transient` object starts a new attempt, as today. A +`retryOn` value call still retries a failed object as slice 8 decided; while the object is parked `waiting`, it does +nothing: its dependencies decide. + +## A reload of the parked object itself + +The parked attempt's factory never runs again while it is parked, with one exception: a reload of the object's own +plugin replaces the parked attempt with the new factory's attempt, as one handover: the new requests are made and +bound first, then the old ones released (Astra: a dependency held only by this object, unchanged by the reload, must +not end in between; today's `#activate` already binds before `#releasePrevious`). If the new code no longer declares +the missing dependency, the object can go live at once. + +## After live, nothing changes + +R18's second half stays: once an object is live, a dependency that leaves `live` (a link drop, a removal) leaves the +object live. Its calls that use the dependency get the dependency's state: they park while it is `loading` and reject +while it is `failed`. Reactivating every live consumer on every dependency loss would turn one worker disconnect into +closed natives, withdrawn contributions and aborted jobs on the server; the reviewers agreed it should not. + +# What Wakes Whom {#wakes lede="Two kinds of event replace every poll and every dependency deadline."} + +A model without timers lives or dies by its wakes: a wake that is lost leaves an object parked forever. So the wakes +are kept to two kinds of event, each with one owner, and every unbound request and every parked attempt is +subscribed to exactly the events that can change it. + +| Event | Who is told | Exists today? | +| --- | --- | --- | +| **a target changes state**: an object goes live, loading or failed, or gets a new cause; a hold receives `bound`, `loading` or `failed`, also after a re-hold on a new link | the requests bound to it, and the attempts that depend on those requests | partly: `dependencyLive` covers only "goes live" | +| **a name becomes usable on a host**: a publication (`load`'s setups, a reload's commit, an install's commit) serves a type, a registry or a version; a registry composes or maps a value at a version | the unbound requests indexed under that name on that host; a refused hold re-runs its start and sends `loading`, then `bound` or `failed` | partly: refused holds and `retry` exist; the re-run at publication is new; a registry wakes only at its first value | + +## Every change, not only "live" + +Draft 1 woke a parked attempt only when a dependency went live. Astra, Fable and Sol found the same hole: a dependency +that moves from `loading` to `failed`, or from one cause to another, leaves its dependent showing a stale state, and a +dependent that should now show `failed { waiting }` keeps parking calls. So a parked attempt re-evaluates on every +state change of any of its dependencies and updates what it shows. It leaves parking only when all of them are live. +Its factory never re-runs on a wake. + +Three details keep this correct and cheap: + +- **Coalesced, after the batch.** Wakes run after the batch that caused them (a publication, a commit, an inbound + `bound`), once per object, so a commit that changes three dependencies wakes their dependent once, and no factory + runs inside another owner's batch. +- **Remembered.** A change that arrives while the attempt runs its own steps or tears down is remembered and applied + after (today's R18 wake memory, generalized). +- **Indexed by name.** A publication touches only the unbound requests waiting for the names it serves. A reload of + `notes` never looks at requests waiting for `search`. + +## Across a link: no new message + +The wake across hosts needs no new message kind. Today a provider keeps a hold it could not serve as refused, and a +`retry` from the consumer re-runs its start. The model adds one thing: the provider re-runs its refused holds itself, +at its own publication, for the names that publication serves. + +:::sequence animate | The search indexer on W waits for the server's feed, which a reload on S adds. +actor w "worker W" "Indexer, parked" +actor s "server S" "search plugin missing" +w -> s: hold SearchFeed +s --> w: failed { waiting, not_served } +note w: Indexer shows failed { waiting }; no timer, no retries +divider "a reload on S adds the search plugin" +s -> s: publication serves SearchFeed; re-run refused holds for it +s --> w: loading +s --> w: bound (snapshot) +w => w: a target changed state: Indexer re-evaluates, passes the check, goes live +::: + +Two existing messages change: `failed` carries the new cause kind `waiting`, and `loading` and a `waiting` `failed` +carry the provider's attempt id for the cycle probe (chapter VIII). The pinned list of causes and its test change with +them, and `PROTOCOL_VERSION` stays 1 until p3 ships. + +## Registries + +A registry reader is woken at every transition of the registry's value at its version to usable: when the registry +is served, when it composes for the first time, when an older reader's mapping succeeds again after it failed. Today +`RegistryValueSource` wakes only at the very first good value (`#valued`), which a registry whose provider is removed +and added again would never pass a second time. + +# Starting Up {#startup lede="The barrier did two jobs with one promise; separated, the deadlock disappears."} + +`load` does three things today: it runs every facet's setup, it publishes them (types served, registries composed +once, contributions in their tables), and it waits until every host-lifetime object has had its first outcome. The +promise of the whole thing, `pendingLoad`, is also what incoming holds and `host.hold` wait for. That is the startup +deadlock of chapter I. + +## Publication first + +The model splits `load` at the publication: + +1. **Setups.** Every facet's setup runs. Setups only declare: `host.serve`, `self.contribute`, `host.references`. + They are synchronous and wait for nothing. +2. **Publication.** All declarations become visible at once, in one batch: types served, registries served and + composed once, contributions in their tables. Nothing activates inside that batch. +3. **Wakes.** After the batch, every request that waited for the publication (unbound `unpublished`) is placed, and + every unbound request for a name the publication serves is re-evaluated. +4. **First outcomes.** The host-lifetime objects activate. Whether `load` waits for them is the barrier question below. + +Requests wait for step 2 only. Since the setups wait for nothing, a host always reaches step 2, and a request from +another host can never wait for this host's activations. Chapter I's knot comes apart: + +:::sequence animate | The same startup as in chapter I, under the model. No deadline is needed. +actor s "server S" "" +actor w "worker W" "" +s -> s: setups, publication: C served +s => w: A needs B: hold B +w -> w: setups, publication: B served +w => s: BStore needs C: hold C +s --> w: bound C (S has published; C activates and goes live) +w -> w: BStore live, B live +w --> s: bound B +s -> s: A live; S's first pass done +::: + +## The barrier: (a) or (b) + +What is left of the barrier is the app's question "has every plugin had its first try?". R17 needs it: p3's base code +resumes the harness after `load`, so the first turn sees the tools of every plugin that loaded. Two shapes are on the +table. + +**(a) Chord keeps it.** `load` resolves when every host-lifetime object of this `load` has a **first outcome**: it is +live, failed by its own cause, or parked with no blocker that is a local object still without its first outcome. In +code, per object: + +```ts +const firstOutcome = (o: Obj): boolean => + o.live || o.ownFailure || (o.parked && o.blockers.every((b) => !b.local || firstOutcome(b))); +``` + +So `CodeMode → Harness`, both local, waits until the harness had its outcome: code mode is parked on a local object +that is still in its first attempt, so code mode is not done. A wait on another host, or on a dependency that failed, +ends the first pass: the barrier never waits for another host, a retry, a later contribution or a reload. The signal +is separate from the readiness dependents wait for (Sol: resolving that for a parked object would let its dependents +open natives too early), and it is counted once per object, never reopened by retries. An object of the cohort that +ends before its first outcome (its plugin removed, the host closing) settles its entry as ended, and `host.close` +settles a `load` that is still waiting (Astra). + +**(b) The app keeps it.** `load` resolves after the publication. Chord adds `host.startup`, a watchable of the +host-lifetime objects and their first outcomes with causes; base code decides when to resume (Fable). The tracker is +the same; only who awaits it changes. + +Three of the four reviewers recommend (a): the activation graph is Chord's, and every app that wants R17's behaviour +would otherwise build the same cohort tracker. Fable recommends (b): the policy belongs to the app, and p2 already has +three different ones. **Owner** decision 1. + +## What the barrier does not promise + +Neither shape can promise that every tool is there for the first turn, and the model does not pretend it can. Sol +reproduced it against today's code (`/tmp/barrier-sol-first-outcome.log`): object A's `open` fails on its first +attempt (A's first outcome); B goes live and contributes; the registry changes and `retryOn` wakes A for a second, +slow `open`; `load` resolves with only B's contribution in the table. Tools found in `onLive` (MCP) arrive later by +design. What a first turn really needs (a model, an environment, a reconciled working directory) is p3's explicit +admission check, as p2's `whenRegistered` and its harness prerequisites are today. + +:::note | What p2 does today +p2 does not wait for all host objects before the first turn. Its worker (`apps/p2/src/hosts/worker.ts`) loads with +`awaitHostObjects: false`, waits for the harness, resumes Durable, tells its parent `control.ready()` and only then +awaits `host.ready()`. It has to: its subagent tools and the search indexer need server types, and the server admits +the worker's link only after `control.ready()`. A barrier before that point would deadlock, exactly as in chapter I +(Astra and Sol, independently). +::: + +# Plugins Coming and Going {#reload lede="Removals, versions and rollouts across hosts, as events the requests wait for."} + +Slice 7 brings reload and rollout, and with them the cases where a provider disappears on purpose. Today's design +handles them with their own mechanisms: `gone` when a type is removed, a final `contract_mismatch` when a version is +dropped, `updating` with a deadline during a rollout. Under the model all of them are one thing: a request becomes +unbound with a reason, and the event that fixes the reason wakes it. + +## A type removed, and served again + +A reload that removes a type ends its objects as today: natives close with reason `end`, callers of running calls are +rejected at once, handlers drain. What changes is what happens to the requests that pointed at those objects. + +- **Over a link,** the provider sends `failed { waiting, not_served }` on the bound hold. A `failed` on a bound hold + unbinds it (Fable: one message, one rule; the consumer applies it as today's `#startOver` plus `failed`). The + provider detaches the hold's sources and drops its unsent pending ops and any queued `bound`. Two things it must not + do (Sol): free the ack slots of `bound` and `state` messages already sent, which stay counted until their `ack` or + the link's loss, or resend calls, which fail and free their slots once. +- **On the same host,** a `Binding` whose target ended by removal becomes unbound and accepts a second `bind`. Its + member sources re-bind by name (one frame from the last value, as on a restart today), keyed pins re-pin, parked + calls flush or reject, presence lists it again. This is the one genuinely new piece of machinery in the model + (Fable, Sol): today `bind` runs once per binding, and the ended object is forgotten. + +When a later commit serves the name again, the publication wakes every request indexed under it, on every host that +holds one, and they bind as if nothing had happened. + +## A version no longer accepted + +A reload can also keep the type and stop accepting an old version (it drops `accepts[1]`). Draft 1 ended the provider +object then; Sol and Astra caught it: the object, its documents, its jobs and its holders on the current version are +fine. Only the requests on version 1 become unbound `version`. The object's own plugin reload still restarts it if it +has natives (§21), as for any reload. + +## A type that moved + +A plugin can move a type from the worker to the server by changing its placement. A consumer's request carries the +address its own code computed, so the provider appearing on the server cannot wake it: the request is not waiting +there. The old host answers `not_served`; the consumer's own reload, which brings the new placement, makes a new +request. The model adds no redirect: moving state between hosts is a deliberate migration, not something Chord guesses. + +## Rollouts across hosts + +A rollout installs a revision on the server first, then on the workers, then on the tabs (design §21). Between those +installs, hosts run different code. The model needs only one rule for that: **each host knows its own install, and +nothing more.** + +- A provider host that is **installing** keeps holds that do not fit its current code as unbound `installing`: the + consumer sees `loading`, its calls park. When the install ends, by its commit or by its rejection, the host + re-evaluates them against the code it now runs (Astra, Sol: a rejected install must not leave them asleep). +- A provider host that is **queued** but not installing yet answers a mismatch with `failed { waiting, version }` + until its install commits and wakes the request. That is a visible blip of at most one install; Opus judged it + acceptable, and avoiding it would need revision fields in every answer. +- **Installs never wait for activations.** An install that waited for its objects to go live would deadlock a rollout + that runs in the other order: the worker's objects wait for the server's new types, the server's install waits for + the worker. +- A host whose **own** install was rejected fails its own requests at once with the reason (design §21). + +:::steps | The sharing plugin's contract goes from v1 to v2; the server installs first ++++ Before +S and W run revision R. W's `ShareStatus` holds the server's `Sharing` v1; T's tab holds W's `ShareBadge`. All `live`. ++++ S installs R+1 +R+1 makes `Sharing` v2, with no `accepts[1]`. While S stages it, S still runs R: W's v1 hold fits the running code +and stays `live`. A hold that arrived now from a host already on R+1 (a v2 hold) would not fit R: it would wait +`installing` (`loading`, calls park) until the install ends. ++++ S commits R+1 +The install ends. S re-evaluates: v1 is not served. W's hold becomes `failed { waiting, version: asked 1, served 2 }`. +`ShareStatus` stays live (R18 after live); its calls that use sharing reject at once with the cause. ++++ W installs R+1 +W's new code bundles `Sharing` v2. Its own reload's handover makes the new request at v2 and releases the old one. The +new hold binds; `ShareStatus` is fully back. T's `ShareBadge` never noticed anything but one stale value. +::: + +## Old calls + +Calls that were running when the reload came follow §21 unchanged: a pure swap lets them finish on their old handles; +a native restart drains them first; a removal rejects their callers at once and drains their handlers. Chord never +resends a call. + +# Finding Cycles {#cycles lede="Without the deadline, the one gap left in cycle detection needs an answer: a narrowed deadline, or a probe."} + +A provider that never appears keeps its dependents parked, and that is visible: they say what they wait for. A cycle is +different: A waits for B, B for A, every provider exists, and still neither can go live first. Today Chord has two detectors and a deadline. On one host, `self.get` follows the waiting edges of +activating objects. Across hosts, a hold carries the activation chain it was made in, and a provider refuses a hold +whose chain contains the object being activated. The gap is documented (design §6): when A on the server and B on a +worker start at the same moment, the holds carry `[A]` and `[B]`, neither finds the other, and both fail at the 60-second +deadline. The model removes the deadline, so the gap must close. + +## Two answers + +The owner decided this gap once already. On 2026-10-24, after the slice 4b reviews, he made simultaneous starts across +hosts "a documented limitation: they time out transient and retry; no probe message." That decision assumed the +activation deadline. The model removes the deadline for dependency waits, so the decision needs one of two +continuations, and this one is his (**Owner** decision 6): + +- **(A) The 4b decision, narrowed.** One deadline stays: an attempt parked on a dependency held over a link, whose + target attempt is known and still activating or parked, fails `transient` after its `readyTimeout` and retries with + backoff; its next attempt usually starts after the other side, and the activation chain in its hold finds the cycle. + Working it through showed the catch. A target that is *parked* is the common case of a chain across hosts waiting + for a missing plugin: a tab's view on a worker object that waits for an uninstalled type. Those dependents would time + out and poll again, as today, so "local = remote" would stop at the link. Narrowing the deadline to targets that are + `loading` avoids the polling but misses a real ring: A and B on two hosts wait on each other, and B also on a missing + X. B is `failed { waiting }`, so A is too; when X appears, both stay `failed { waiting }` on each other, with no + deadline to break it and no chain to see it. Worse (Fable, round 6): an object's activation chain is fixed + when it is created and its instance id survives retries, so a retried attempt sends the same hold with the same + chain; two simultaneous starts time out and retry in lockstep and are never found. The deadline would poll without + finding what it is for. +- **(B) The probe (recommended, pending review)** described in the rest of this chapter: no deadline at all, one new + message kind, and the most intricate rules in the model. Five review rounds found a counterexample in each of the + first four drafts; the version below answers all of them, with one choice left in rule 5. Slice 7 + would prove it with tests before the old deadline goes. + +The rest of the chapter describes (B): it is the answer that keeps one rule on both sides of a link, and its review +history shows what a complete answer takes. + +## Why a growing chain is not enough + +Draft 1 proposed letting chains grow: an activating object that receives a hold with a chain sends the grown chain on +to its own dependencies. Astra and Sol showed two flaws. Chains are cut at 32 entries today, so a ring of 33 objects +started at once is never found: the chain that reaches each object is always missing that object's own id. And a +chain records ancestry, not the current wait graph: once A stops waiting for B, B's chain still names A, and a later +wait of B on A looks like a cycle that is not there. + +## (B) The probe + +The model uses an edge-chasing probe (the idea of Chandy, Misra and Haas), for the rings that cross a link; on one host the `self.get` +walk stays, synchronous and exact, as today (Opus). +Five rounds of review shaped it; the rules below are the result. + +1. **Attempts, edges, tokens.** Every attempt has its own id, minted per attempt: a retry or a reload is a new + attempt (today the id is per instance and survives retries). A **wait edge** runs from a parked attempt to the + activating or parked attempt it waits for, through a bound request. Unbound waits and registry reads are never + edges: they are not deadlocks, they end when a provider appears. Every edge carries an **edge token**: a number from a counter the + attempt keeps, never reused, replaced by a fresh one on any interruption or change of that edge (Astra, Sol: two + requests to one object are two edges): a new target attempt (also when a local + target retries without a re-bind), any `loading` on a bound or pending hold (a relay re-holding upstream included), + an unbinding, a rebinding, a re-hold after a reconnect. An attempt is parked for one interval only (once past its + dependency check it never parks again), and attempt ids are never reused. A provider tells its consumers which attempt they wait for on every answer that is not `bound`: + `loading { hold, attempt }`, and a `waiting` `failed { hold, cause, attempt }`. +2. **A search per edge change.** Whenever a parked attempt gets a new edge, or a new token on one, it starts a + search: `probe { kind: "search", round, initiator, trail }`, with the next number of a counter the attempt keeps, + its own attempt id, and the trail of `(attempt, edge token)` hops crossed so far, starting with its + own. A ring closes at its last edge, + and that edge's source starts the search that finds it (Sol's simplification after Astra's counterexample: a ring + that forms only after a reconnect, with unchanged attempts, is otherwise missed). Searches are coalesced per batch. + On the same host a probe is a call on the target attempt, forwarded through a queue rather than recursion; over a + link it is a new message, `probe { hold, to, ... }`, consumer to provider, relayed by hold id with the origin. + **Probes are honoured on parent and route links and dropped on child links** (`accept`), whatever `trusted` says + (Opus): a tab learns attempt ids from `loading` and could otherwise break an object with a forged probe, while a + worker whose transport is untrusted must still take part. Clients serve no remote types, so no real cycle runs + through one. +3. **Forwarding.** An attempt that receives a search drops it if `to` is not its current attempt or if it is not + parked. If the initiator is itself, the search came back. If the attempt is already in the trail, it drops it: a ring + that does not contain the initiator is found by its own members' searches (Opus). Otherwise it forwards it along + every wait edge, appending itself and the edge's token to the trail, but only if its round number is higher than the one it stored for + that initiator; then it stores it. A late, older round is dropped even if this attempt never saw it: its initiator + has moved on (Astra, Opus). The seen map holds one number per initiator, up to 256 initiators, and goes with the + attempt. +4. **The check.** The initiator accepts a returned search while it is still parked, whatever round it carries (Astra: + a re-search must not invalidate an older search still under way); the check decides. It verifies its own hop, then + sends a + `probe { kind: "check", round, trail }` along the recorded trail: each hop passes it on only if it is still that + attempt, still parked, still holding the recorded edge token, pointing at the next hop's attempt. The check follows the + trail by position, ignores the seen map and dies on any mismatch. When it returns, the initiator verifies its own hop + once more. If the check comes + back, the initiator fails `broken invalid/cycle`, naming the path from the trail, and today's propagation breaks + the other activating objects on the ring and the dependents off it. +5. **No silent loss.** Astra found the last hole: a search can reach an attempt first by a shortcut and be dropped + there when it arrives again along the ring itself; if the shortcut breaks before the check, the check dies and + nothing is left to find the ring. Round 6 tried `dead` notices sent back to the initiator. Sol and Astra showed + that reliable notices need recovery state per search that the seen map's eviction may not drop, and that a long- + parked attempt would otherwise keep for every initiator that ever passed: a leak, unless every search gets a full + lifecycle with completion and cancellation. So rule 5 is a choice, part of **Owner** decision 6: + - **(B1) Re-search with backoff (recommended).** An attempt parked on a remote attempt starts a fresh search again + after a backoff, 1 second doubling to 60, reset by an edge change, whether it shows `loading` or `failed + { waiting }`; the timer stops when the attempt leaves parking. No `dead` messages and no recovery state; + eviction only costs work; a new search starts within one backoff period, so every stable ring that + crosses a link is eventually found (no wall-clock bound: delivery is only fair). The + timer runs as long as the attempt stays parked on a remote attempt: a view on a worker object that waits for a + missing plugin re-searches every minute until a reload (Fable). The timer sends probes only: it + never fails an object, never re-holds, never polls a provider. Its cost is one probe per parked remote dependent + per period, which dies at the leaves of the downstream wait graph. + - **(B2) Events only.** Every search is owned until it settles: branches report completion or abort along their + trails, a dropped probe answers `dead { initiator, round }` by a reverse route, edges remember the searches they + carry until those settle, and an initiator that ends or moves on cancels its old search. Complete without any + timer, bounded by the searches in flight, and the most intricate protocol in Chord; not yet specified to the + reviewers' satisfaction. + + Sol's correctness reference, enumerating every simple trail without dedupe, never suppresses a ring copy, but its + work is exponential on chains of diamonds. + +:::sequence animate | Two objects on two hosts start at the same moment and need each other. The probe finds it; nothing times out. +actor a "A on server S" "attempt a1" +actor b "B on worker W" "attempt b1" +a -> b: hold B (A activating) +b -> a: hold A (B activating) +b --> a: loading { attempt: b1 } +a --> b: loading { attempt: a1 } +a => b: search { to: b1, round: 1, initiator: a1, trail: [a1·t1] } +b => a: forward: search { to: a1, round: 1, trail: [a1·t1, b1·t1] } +a -> a: search 1 came back: check the trail +a => b: check { round: 1, trail }: b1 still waits on a1 with token t1 +b => a: check passes on +a -> a: check came back: broken invalid/cycle "A → B → A" +::: + +## Why a reported ring existed + +Draft 4 confirmed a cycle with a second, independent round. Sol showed a schedule in which two rounds pass through two +different stale paths, each of them half replaced by reloads, and together report a ring that never existed at any +moment (`/tmp/model/sol-probe4.log`). The check closes that: it walks the very path the search recorded and demands +that every edge still carry the token it had when the search crossed it. A token changes on every interruption and is +never reused, and an attempt is parked for one interval only, so an edge cannot vanish and come back unnoticed (Fable and Opus +checked the argument in round 5). Every edge of the trail therefore existed from the moment the +search crossed it until the moment the check crossed it; the search crossed all of them before the check began, so +when the check began, every edge of the ring existed at once. In Sol's schedule the check dies at `B`, whose edge now +points at another attempt. + +## Why every ring is found + +Under an eventually stable wait graph and fair delivery (endless edge churn can keep invalidating searches), and with +rule 5's (B1) or (B2) for a search suppressed behind a path that broke, a ring of parked attempts is closed by its last +edge. When that edge appears, its source starts a search; every other edge of the +ring exists at that moment, and every attempt on the ring waits, directly or not, on the source, so none of them can +go live before the search and its check come back. Rings of any length are found, through local and remote edges +alike. On a graph without cycles, searches die at the leaves; on a diamond, the seen map drops the second arrival. + +## What it does not promise + +- **A ring broken up just after.** A ring that existed when the check began may be broken up by a reload, a retry, an + unbinding or a revocation a moment later; it is still reported, because it did exist. The objects it breaks recover + as every broken object does, through `ref.retry()` or a reload of their plugin. +- **Linear work under every schedule.** With more than 256 initiators passing one attempt at once, an evicted entry lets + a search be forwarded again, so in an adversarial schedule the work can grow with the number of simple paths (Astra, Sol); never endlessly, since + a search never revisits an attempt already in its trail. + The seen map's memory stays bounded. Each edge change costs one search that crosses each downstream edge once, + and one check per returned search; with (B1), every attempt parked on a remote attempt adds one search per backoff + period, also while an acyclic chain only waits for a missing plugin. +- **Constant message size.** A probe carries its trail, so it grows with the length of the path it crossed; a search + that returns carries a trail as long as its ring. That is the price of the check; chains were cut at 32, trails are + not, because the trail is the proof. + +:::review | Review: the state of (B) +The check's argument (a reported ring existed at the moment the check began) was confirmed by all four reviewers in +rounds 5 and 6, given exact edge identity (the tokens). Completeness under an eventually stable graph holds with +(B1); with (B2) it holds once the search lifecycle is specified, which no draft has done yet. If the owner chooses (B), +slice 7 keeps today's deadline for activations waiting across links until the probe's tests pass: long rings, +diamonds, a ring closed after a reconnect with unchanged attempts, Sol's phantom schedule, Astra's broken shortcut, +Sol's eviction schedule, 300 concurrent initiators, edges added and removed, reloads, relays, forged probes. +::: + +# Callers {#callers lede="Getters, holds and calls get one rule: wait while something is under way, answer when nothing is."} + +Objects park; callers do not. A caller is code that wants an answer: a method that needs a reference now, a tool that +runs inside a turn, a tab that wants to render. The model keeps their deadlines and gives them one rule. + +## `host.hold` + +`host.hold` returns a reference at once and never rejects; only invalid params throw, also before `load`. That was +already the design; the model removes today's inconsistency: a missing local type throws after `load` but fails the +reference before it. Now a missing local type gives `loading` before the publication and `failed { waiting }` after +it, and the reference binds when a provider appears. A UI that wants to wait for something that may never come +holds it and shows its state. + +## `host.get`, `ctx.get`, `host.registry` + +The getters resolve when the reference is live. While it is `loading`, or `failed { transient }` (its object's own +retry is under way), they wait, bounded by their `readyTimeout`. On `failed { waiting }` or `failed { broken }` they +reject at once. On success, rejection or timeout their temporary wait goes; a rejected getter releases its request. +`host.registry` returns the `RegistryValue`, whose watches and observers are owned as they are today; it has no +`release()`. + +Opus's case decided the recommendation. A Durable tool runs `chord((ctx) => ctx.get(SharingView, { session }), ctx)` +while sharing is not installed. If getters waited through `waiting`, the tool, and with it the agent's turn, would +hang for 60 seconds and then fail; a call on the same reference would fail at once. With the rule above, the getter +and the call give the same answer at the same moment. The rule has a cost Opus named: a dependent whose only blocker is +an object recovering by itself (an SSH environment coming back, `transient` with its own backoff) is `waiting`, so a +getter on the dependent rejects at once. Code that wants to ride that out holds the reference instead of getting it. + +:::review | Owner decision 2: getters on `waiting` +Recommended: reject at once (Opus; Fable agreed in round 2). The alternative waits through `waiting` up to +`readyTimeout`, which keeps "local = remote" just as well but makes every getter on a missing plugin cost its full +deadline. +::: + +## Calls + +Calls on a `loading` reference park in order, each bounded by the reference's `readyTimeout` from the moment it was +made (never renewed by a reconnect or a reload). Calls on a `failed` reference reject at once with the cause. A call +already sent when its link drops, or when its object ends, fails and is never resent; its outcome is unknown, and +idempotency is the app's (a `requestId`, as p2's chat does). + +## The rule in one table + +| The reference | `host.hold` | getters | calls | +| --- | --- | --- | --- | +| `loading` | returns it | wait, bounded | park, bounded | +| `failed { transient }` | returns it | wait, bounded | reject | +| `failed { waiting }` | returns it | reject at once | reject | +| `failed { broken }` | returns it | reject at once | reject | +| `live` | returns it | resolve | run | + +# Registries {#registries lede="The same resolver on one host, with versions that can differ inside one revision."} + +A registry is host-local: its contributions are in-process values, often natives. Slice 8 made `self.registry(R)` a +dependency that waits for a value. The model keeps that and folds it into the one resolver. + +## A reader's request + +A reader asks for `R` at the version its own bundle was built against. The request is bound while `R` is served and +has a usable value at that version: composed, and for an older reader mapped back by the provider's +`accepts[old].value`. Otherwise it is unbound: `not_served` (no plugin serves `R` here), `version` (the provider does +not accept the reader's version, or the reader is newer), `compose` (`R` is served, but its first composition, or this +version's mapping, threw), or `loading` before the publication. The reader is woken at every transition of the value +at its version to usable. S2's `broken` for a newer reader and S3's backoff go; so does `#valued`, the latch that woke +readers only at the very first good value. + +A registry waits for a value, never for a particular contribution: an empty table composes. "Wait until model X is +there" is the app's admission check, not a registry rule; Durable's own "model unavailable" wait covers the harness. + +## Versions inside one host + +Draft 1 claimed that a registry's provider and its contributors always run the same version, because a host installs +a revision atomically. Astra corrected it: a revision is a set of plugin bundles, and a reload of the plugin that owns +the registry leaves every other plugin's contributors and readers on the version they were built against. So the +provider's adapters do real work inside one host: `accepts[old].contribution` maps an older contribution, `accepts[old]. +value` maps the composition back for an older reader. A contribution without an adapter is left out and reported; a +reader without one is `waiting` with `version`. + +## The last good value, and natives + +Slice 8 decided that a throwing `compose` keeps the last good composition, so an extension point stays available while +its owner fixes a bug. Astra found the case where that hands out something dead: the harness contributes its native +`H`; a reader is live; the harness ends; the next composition throws for an unrelated reason; `.value` still returns +the composition that contains the closed `H`. + +The model narrows the rule: `.value` keeps the last good composition only while every contribution it was built from +is still in the table. When one of them leaves (its contributor ended or was reloaded), the last good value is dropped +and `.value` throws until `compose` succeeds again. And `compose` represents a missing contributor as a value +(absent), never by throwing. + +:::review | Review: drop the last good value, or fence the reads +Sol would keep the last good value and instead require natives in a registry to be looked up at use through their +owner, without the guarantee. The draft takes Astra's narrower rule, because it keeps "a registry never hands out a +native whose contributor ended" without asking every contributor to wrap its natives. It narrows a decision of slice 8, +so it is the owner's: **Owner** decision 5. +::: + +# Keep-alive, Presence, Revocation {#housekeeping lede="What a waiting request costs, who counts it, and who may end it."} + +A parked object and an unbound request are resources: they hold memory, they may keep a worker process alive, and +they act for a principal. The model says how each of those works while nothing is bound. + +## Keep-alive and presence + +A worker stays alive while `host.activity` shows work: references held over links, calls, jobs, lingering objects. A +request that arrived over a link counts once in `host.activity.references` from its arrival to its end, bound or +unbound. Without that, a session whose tab waits for a plugin that is not installed would see its worker stopped every +idle period (five minutes in p2) and restarted by the next re-hold, forever (Opus). Presence is different: +`host.references` lists only bound references, so "who is looking at these notes" counts viewers, not waiting +requests. + +## Revocation + +Revocation ends the requests of a revoked principal, bound and unbound, at every hop. A guest whose grant is revoked +while their tab waits for a missing plugin does not get a reference later when the plugin comes back: the request is +gone, and nothing revives an ended request (no publication, no Retry, no new grant). A refused request (no authority) +is never indexed in the first place. + +## `gone` + +`gone` remains for one thing: revocation. A provider host that closes closes its links and sends nothing (today's +`#close`); its consumers see a link drop, `loading`, and hold again when the host comes back. A reload that removes a +type sends `failed { waiting }` instead of `gone` (chapter VII). + +## Parked objects and their jobs + +A parked object with params lives while it is referenced, like any object; when its last holder releases it, it ends +and releases its requests, so a parked chain goes from the top down. A host-lifetime object parks until its plugin is +removed or the host closes. Persisted job runs of a parked host-lifetime object are loaded with its documents, before +it parks; `run` and `finish` start only once it is live, and the run count keeps the object as today. + +# The Clock {#clock lede="Which timers stay, and why each of them has no event to replace it."} + +The owner expected the model to remove almost every timeout, "except maybe `readyTimeout`". That is what it does for +dependencies. Every remaining timer bounds something no event can end: a caller's patience, an object's own code, a +silent transport, a process that may start this time. + +| Timeout or backoff | After the model | Why | +| --- | --- | --- | +| `readyTimeout` of an attempt | only its own steps (natives, follow attaches, checks, derives); paused while parked | a slow dependency is bounded where it runs, by its own steps | +| `readyTimeout` of `host.get`, `ctx.get`, `host.registry` | kept | a caller must get an answer | +| `readyTimeout` of a parked call | kept | an action must not run long after it was asked for | +| identity backoff 1 to 60 s | only for the object's own `transient` failures (an `open`, a follow) | nothing tells Chord that a disk or a network came back | +| dependency backoff, polling of refused holds | **removed** | a dependency's recovery is an event | +| dependency deadline for waits across links | answer (A): kept only for an attempt waiting on a remote attempt that stays activating; answer (B): kept until the probe's tests pass, then **removed** | the simultaneous-cycle gap (chapter VIII) | +| probe re-search backoff, only with answer (B1) | new: 1 to 60 s while parked on a remote attempt | finds a ring whose search was suppressed; sends probes only, never fails or re-holds | +| route backoff per address | kept | a failed worker start has no event; it avoids fork storms | +| heartbeat 15 and 30 s | kept | the only detector of a silent transport | +| `closeTimeout` (natives, the drain, the routine handover) | kept | bounds other code's teardown | +| routine and job `finish` backoffs | kept | the object's own code throwing | +| `linger` | kept | a policy for warm objects, not a wait | +| rollout install timeout (designed, not built) | a watchdog on the install only | never a deadline for dependencies | + +So `readyTimeout` stays, but its meaning narrows to two things: how long a caller waits, and how long an object's own +activation steps may take. Waiting for someone else never counts against it. + +# A Morning, Step by Step {#morning lede="The preface's Tuesday morning, played through under the model from the first click to the afternoon's rollout."} + +The rules of the earlier chapters meet in one story. Mario opens session `S1` on his phone; the server S has the catalog +and the search feed; the worker W for `S1` has not started. W's plugin set has `core`, `models`, `codemode`, `notes` +and `search`, but not `sharing`, which was removed last night; the notes view still asks for it. Later in the day a +rollout brings sharing back, with a new contract. + +:::steps | Session S1 on Tuesday ++++ 1. The tab asks for the notes +T calls `host.hold(Notes, { session: "S1" })`. The address is `Session { id: "S1" }`, not T's own, so the request goes +over T's parent, the server. S has a route for `Session`: it resolves the address by starting W. T's reference shows +`loading`; the request waits on S's path until W's link exists. ++++ 2. W connects before it loads +W connects to S first (W6), then starts `load`. The hold for `Notes` arrives before W has published its facets: it is +unbound `unpublished` and waits, still `loading` for T. Nothing is placed by a half-empty table, so the hold is not +sent back to S. ++++ 3. W publishes +W's setups run, synchronously, and are published in one batch: `Notes`, `NoteStore`, `Harness`, `CodeMode`, `Indexer` +are served; `HarnessModels` and `HarnessExtensions` are served and composed once. After that batch, the waiting hold +for `Notes` is placed and binds; the host-lifetime objects start. ++++ 4. Code mode parks on the harness +`CodeMode` declared `self.get(Harness)`. The harness is still opening Durable, so `CodeMode` parks: `loading`, no timer. +The harness reads `HarnessModels`, which has a value (composed at the publication; the model runtime's contribution may arrive +after), finishes its own steps +and goes live. That state change wakes `CodeMode`, which passes its dependency check, opens its sandbox, goes live and +contributes its tools. ++++ 5. The indexer waits for the server +`Indexer` needs the server's `SearchFeed`. The hold goes over W's parent; S published long ago, so it binds at once, +`SearchFeed` is live, and `Indexer` goes live. Had S been starting at the same moment, the hold would have waited only +for S's publication, never for S's activations: the startup deadlock of chapter I cannot form. ++++ 6. The first pass +With barrier (a), W's `load` resolves once `Harness`, `CodeMode` and `Indexer` had their first outcomes. `CodeMode` +waited for the local harness, so its tools are in the table; base code resumes the harness, and the first turn sees +them. With (b), base code would read `host.startup` and decide the same. ++++ 7. The notes view asks for sharing +`Notes`'s factory declared `self.get(ShareStatus)`, a local type of the sharing plugin. Nobody serves it on W: the +request is unbound `not_served`, indexed under the name. `Notes` parks `failed { waiting, "waiting for ShareStatus: not +served on Session S1" }`. T's reference shows the same; a call from T rejects at once with that cause. No backoff, no +polling, no 60-second wait. ++++ 8. The phone changes networks +T's link to S drops. T's reference goes `loading`: its request stays, its last value stays readable, a call made now +parks. A call that was in flight fails and is not resent. T reconnects; the request is held again on the new link; W +answers as before: `failed { waiting }`. ++++ 9. Sharing comes back, with a new contract +In the afternoon a rollout brings the sharing plugin back at revision R+1, where `Sharing` on the server is v2. S +installs first and serves `Sharing` v2. W's install is queued; nothing on W asks for `Sharing` yet, because the sharing +plugin is not installed there, so nothing changes for W. (Had W still run a sharing facet built against v1 without an +adapter on S, its requests would show `failed { waiting, version }` until W installed: the ordinary window of a +server-first rollout.) ++++ 10. W installs R+1 +W's install commits: a publication. It serves `ShareStatus` again: the index wakes `Notes`'s request, which binds. +`ShareStatus` activates, holds the server's `Sharing` at v2 (W's new code bundles v2), goes live, and `Notes` passes its +dependency check, opens what it needs and goes live. T's reference becomes `live` without T having done anything. +::: + +What did not happen is the point. Nothing waited for a deadline; nothing retried on a timer; no reference ended +because a plugin was missing; no host waited for another host's activations; and every waiting object said what it +was waiting for. + +# Scenarios: Starting Up {#sc-startup lede="Twenty startups on p2's real hosts, under the model, with the barrier as (a) and as (b)."} + +These scenarios, from Astra, follow p2's real hosts: the server alone, workers started by a tab or by another worker, +web tabs, the TUI, restarts in the middle. Each row says what the model does, with the barrier in both shapes where +they differ, and what today's code does. The pattern across all of them: publication never waits for activation, so +dependency traffic between hosts always moves; what differs between (a) and (b) is only when base code is told that +the first pass is over, and neither shape promises every tool before the first turn. + +The Rule column cites sections of the working document, reproduced in appendix C. + +| # | Scenario | Under the model | Today | Rule | +| --- | --- | --- | --- | --- | +| 1 | The server starts alone. Catalog, ModelConfig, LoginService and ForwardStore initialize before any session worker exists. | Their local references are `loading` during initialization, then `live` or `failed` with an own cause. Under (a), `load` waits for these first outcomes; under (b), it returns after publication. Public admission still needs explicit security prerequisites; a completed first pass is not proof that every service works. | p2 starts its HTTP listener but gates requests and upgrades behind plugin startup (`apps/p2/src/hosts/server.ts`). Current next waits for eager objects' first live/failed outcomes (`host/host.ts:load`). | §§1–3, 6 | +| 2 | A tab's request reaches the server before its first facet publication. Catalog is declared by a later setup in that same load. | The request stays `loading` behind the publication gate. The complete declaration batch places it; Catalog activation then produces `bound` and the reference becomes `live`. Neither (a) nor (b) delays that hold until unrelated eager objects finish. | Current next's `Hold.start` waits for `pendingLoad`, which includes every eager object's first outcome (`remote/hold.ts`). It waits longer than declaration safety requires. | §§1, 4, 6 | +| 3 | Several server facets set up successfully, but the last setup throws. A tab is already waiting for Catalog. | No partial declaration batch may activate objects. Load fails and the host closes; local held references become `ended`, while the tab sees a link drop and remains `loading`. A replacement server's successful publication can satisfy the tab's retained request. | Current next closes the host on a throwing setup and activates no objects (`host/host.ts:load`). Setup failure rejects load directly. | §§1, 6, 10 | +| 4 | A session worker's Harness opens slowly. Its CodemodeRuntime startup object declares Harness and contributes tools only after its own sandbox opens. | Harness and CodemodeRuntime start `loading`; CodemodeRuntime waits without a dependency timer. Harness becoming `live` wakes CodemodeRuntime, which opens its sandbox and becomes `live`. Under (a), this local chain finishes before load returns and resume; under (b), the application must await the same startup outcomes or risk a first turn without those tools. | p2 resumes after Harness is ready, before `host.ready()` (`apps/p2/src/hosts/worker.ts`); CodemodeRuntime depends on Harness (`plugins/codemode/worker.ts`). Current next's load barrier waits for both first outcomes. | §§3, 6, 11 | +| 5 | A tab opens Chat for a stopped session. The server starts that session worker, whose Search Indexer needs server SearchFeed. | Chat remains `loading` while the route starts the worker. The worker connects and publishes before its application barrier; SearchFeed can bind immediately on the already-published server. Indexer becomes `live` when its prerequisites are ready, and Chat receives `bound` once its own prerequisites are ready. Under (a), a still-remote startup blocker can already count as a parked first outcome; neither option guarantees that every remote-backed feature precedes resume. | p2 connects IPC first but the parent admits it only after `control.ready()`; Harness resumes before the later `host.ready()` (`src/hosts/worker.ts`). Current next gates incoming holds on whole-host load (`remote/hold.ts`). | §§1, 3–4, 6 | +| 6 | A running worker calls ConversationBroker on the server. The broker sends to another session whose worker is stopped. | Only the second worker starts a new publication and first-pass sequence. Its PeerConversation reference is `loading` until that view binds; the first worker and broker remain `live`. Under (a), the new worker's scheduler waits for its first-pass report; under (b), its application must supply that admission rule. This call does not reopen the first worker's startup barrier. | p2 gets PeerConversation inside the broker's call and starts its worker through the session route (`plugins/conversations/server.ts`, `src/runtime/server.ts`). The spawned worker uses p2's Harness-before-host-ready ordering. | §§3, 6, 8 | +| 7 | Many tabs open the same stopped session while other tabs open several different sessions. | Requests to one worker share route resolution and provider identities where their principals match. Each tab's reference starts `loading` and becomes `live` on its own binding. Both barrier options remain per host; a slow worker cannot hold back every other worker's first turn. The application shows per-session startup status, not one global ready flag. | Current next shares resolution per route address (`host/route.ts`). p2's SessionHosts also serializes starts per session (`apps/p2/src/runtime/server.ts`). | §§1, 6 | +| 8 | During server startup, an eager Bootstrap object needs a worker's Agent. That worker's eager FeedOwner needs the server's independent Catalog. | Server and worker publication happen independently of their first-pass reports. Catalog binds while Bootstrap is still `loading`; that wakes FeedOwner, then Agent and Bootstrap can become `live`. Both (a) and (b) avoid a host-ready gate on this dependency traffic. The acyclic object graph needs no timeout to make progress. | Current next can deadlock its whole-host load gates until activation deadlines fail attempts: the Agent hold waits for worker load, while the Catalog hold waits for server load (`host/host.ts`, `remote/hold.ts`). This is an architectural example using p2-style server/worker dependencies, not a claim that p2 has a Bootstrap plugin. | §§1, 4, 6 | +| 9 | The worker has a healthy Harness, but its `p2.subagent-tools` startup object still waits for server Subagents. The first generation is ready to run locally. | `p2.subagent-tools` remains `loading` while its remote reference binds, or `failed` with `waiting` if that provider is unavailable. Under (a), the remote wait can settle its first outcome without its contribution; under (b), publication returns even earlier. Server `bound` later wakes `p2.subagent-tools` and publishes its tools for future phases. If the first turn requires those tools, application admission must wait explicitly. | p2 `p2.subagent-tools` declares both Harness and server Subagents (`plugins/subagents/worker.ts`). Its worker resumes Harness before admitting the upstream-dependent host objects (`src/hosts/worker.ts`). | §§2–4, 6 | +| 10 | In a p3 port, the core serves a Models registry whose empty composition is valid. The optional models contributor is absent when the worker starts. | The registry is usable, so this absence alone does not park Harness; Harness can become `live`. Neither (a) nor (b) waits for an uninstalled contributor. The generation's model selection or application admission must report model unavailable and wait for a contribution. Registry publication later wakes that domain-level wait; it is not a missing-provider activation failure. | p2 instead awaits the `HarnessConfig` default entry inside Harness's Durable opener (`src/runtime/worker.ts:serveHarness`). Without the models contribution, that opener does not finish normally. | §§3, 6, 9 | +| 11 | A session is configured for SSH. Harness's execution opener finds that the environment is invalid or cannot open. | Harness becomes `failed` with its own `transient` cause, or `broken` if the opener marks the error permanent. Dependents still before their readiness check become `failed` with `waiting`. Under (a), this is a first outcome, not a healthy Harness; under (b), load already returned but resume still requires a usable Harness. Effect retry or a configuration-triggered retry can later make it `live`; no local-execution fallback is implied. | p2 opens the session machine during Harness activation and fails rather than silently using the local machine (`src/runtime/worker.ts`). Worker startup failures are surfaced and can require explicit retry (`src/runtime/server.ts`). | §§2–3, 6, 11 | +| 12 | Prompt becomes live before its initial local instruction-file read finishes. A user immediately sends the first message. | Prompt's reference can be `live` while its file-loading work continues. Under (a), its InputGate contribution is present before a normal local first pass completes; the gate still controls when local input may proceed. Under (b), the application must ensure that contribution exists before admitting input. The model does not turn background file loading into a host-wide barrier. | p2 contributes InputGates and awaits the initial local load in its prompt sections; remote instructions deliberately load in the background (`plugins/prompt/worker.ts`). Object readiness is already distinct from instruction readiness. | §§3, 6, 9 | +| 13 | Search starts an archived session worker only to rebuild its historical index. Indexer has a long scan to perform. | Once SearchFeed and Harness are usable, Indexer becomes `live` even while its `scanned` state is false. Both (a) and (b) leave scan completion to the explicit IndexScan operation. The archived-worker application keeps Harness scheduling paused, waits for the scan result and then stops the worker. A startup report does not override that resume policy. | p2 passes `resume:false` for archived index-only starts, exposes `IndexScan.scanned`, and stops the archived worker after rebuilding (`plugins/search/server.ts`, `plugins/search/worker.ts`, `src/hosts/worker.ts`). | §§3, 6 | +| 14 | After a worker starts, a tab opens Workspace for a large directory. Its filesystem watcher is slow to initialize. | The on-demand Workspace reference is `loading` until the backend is usable, then `live`. Its background watch may still be starting. Neither barrier option waits for every future Workspace view or directory watch; a component that needs a complete initial scan must use explicit domain readiness. | p2 deliberately returns the Workspace backend before installing a potentially slow watcher (`plugins/workspace/worker.ts`). This work is not a universal worker-start prerequisite. | §§3, 6 | +| 15 | A worker restores durable work that can resume immediately. No new Chat.send call occurs, so there is no incoming-input gate to intercept it. | Harness opens with scheduling paused and can become `live` before all startup contributors. Under (a), base code resumes after the first-pass report and required application checks; under (b), base code must await equivalent outcomes itself. Input gates on new messages do not guard this replay. Each phase pins the contributions present when that phase begins. | p2 explicitly calls Harness.resume before `host.ready()` (`src/hosts/worker.ts`). Its Durable registry adapter snapshots the extension table per phase/version (`src/runtime/durable.ts`). | §§6, 9 | +| 16 | A previously loaded web tab starts while the server is offline. Its local UI facets can render an offline shell. | Publication of the tab's local facets does not require the server. Remote references stay `loading` on the absent path. Under (a), remote blockers can count as parked first outcomes; under (b), declaration publication returns directly. The shell renders, and later connection plus `bound` messages make the affected views `live`. | p2 starts reconnection before facet installation and renders React without awaiting a whole-host ready promise (`src/hosts/web/main.tsx`). It already supports an offline shell after the page has been loaded. | §§1–2, 6 | +| 17 | A TUI candidate loads before its launcher activates BridgeLink. A startup UI object declares a server reference, whose outgoing frames are buffered. | The reference remains `loading` on a remote dependency. Under (a), that remote wait is a parked first outcome, so candidate preparation can finish; under (b), publication alone finishes. Launcher activation flushes the frames, and `bound` wakes the UI object. Do not replace this dependency declaration with an own native opener that waits indefinitely for launcher activation. | p2's setupTui awaits facet load while BridgeLink buffers frames until `activate` (`src/hosts/tui-worker-runtime.ts`). Adding an eager remote-dependent host object under the old full-ready load rule could stall candidate preparation until a deadline. | §§1, 3, 6, 11 | +| 18 | A tab starts a worker, but the process crashes while Harness is still opening. The server launches a replacement. | The tab's Chat reference remains `loading` across the lost path and replacement startup. The replacement has its own publication and first-pass cohort; successful binding makes Chat `live`. Both (a) and (b) must keep transport/start failure distinct from a healthy startup report. A process manager may surface a startup error and require user retry rather than retrying forever. | p2 records worker startup failures and has an explicit retry path; it does not treat every failed boot as an automatically healing object (`src/runtime/server.ts:SessionHosts`). Current next's route layer independently retries failed resolution (`host/route.ts`). | §§1–2, 6, 11 | +| 19 | The server restarts while a worker is still starting. A detached terminal host already owns running PTYs. | Tabs see their server and worker references become `loading`; new hosts publish and bind them again. The application may stop old workers on parent loss, while the detached PTY owner survives. The old server's unfinished first-pass wait settles on close (its entries end). New startup reports must not wait for the detached PTYs to restart. | p2 workers stop on IPC disconnect, while the terminal driver detaches its socket without killing the detached PTYs (`src/hosts/worker.ts`, `plugins/terminals/detached.ts`). Current next settles ended objects through its existing load settlement path. | §§1, 6, 10 | +| 20 | A startup contributor in a v2 port reads a keyed configuration document from a local store. The store is live, but the first requested key still needs hydration. | The contributor remains `loading` until that pin is ready; store-live alone does not make the dependency usable. Under (a), this local key-loading work remains part of the first pass, then the contributor can become `live` and publish before resume. Under (b), publication returns earlier, so the application must await that readiness if the first turn needs the contribution. | Current next loads dependency pins before opening the consumer's natives and before its settled promise completes (`host/attempt.ts:pinKeys`). Its activation deadline currently also bounds this wait. This is a v2-port example, not an existing p2 keyed API. | §§3, 6, 11 | + +# Scenarios: References, Holds and Calls {#sc-refs lede="Getters, holds, calls, link drops, relays, release and revocation, played through."} + +Fable's scenarios use p2's names: `Catalog` and `Hosts` (the sessions plugin, on the server), `Sessions` (on the +server, called from worker tools), `SessionTransfer` (a worker type the server calls for a fork), `Notes { session }` +(a worker type a tab holds), `NoteStore` (local on the worker), `ForwardStore` (ports, on the worker) and `Forwards` +(radius, on the server), and `Harness` (host-lifetime, on the worker). S is the server, W1 the worker of session S1, T +a web tab. The pattern: a request waits where something is under way and answers where nothing is; nothing a holder +does is needed to recover; and an ended holder leaves nothing behind in any index. + +The Rule column cites sections of the working document, reproduced in appendix C. + +| # | Scenario | Under the model | Today | Rule | +| --- | --- | --- | --- | --- | +| 1 | `self.get(NoteStore)` in `Notes { S1 }`'s factory on W1; `NoteStore` is served and activating. | `Notes` makes its request, it binds to the `NoteStore` object, `Notes` parks `loading`. `NoteStore` goes live: a target state change wakes `Notes`, which opens its natives and goes live. No timer ran. | The same wait, bounded by `readyTimeout`; past it `Notes` fails `transient` and backs off (`attempt.ts` `waitFor`). | §3.1, §3.4, §4 | +| 2 | `self.get(Catalog)` from W1's `ForwardStore`; the parent link is up, S has published, `Catalog` is served there. | The request goes over the parent (placement `Server()`); S binds the hold; `bound` arrives; `ForwardStore` was `loading`, now goes on and live. | The same, after S's whole barrier (`Hold.start` awaits `pendingLoad`). | §1, §2 | +| 3 | The same, but `Catalog`'s plugin is not installed on S. | S indexes the hold `not_served` under "p2.catalog" and answers `failed { waiting, not_served }`; `ForwardStore`'s reference shows `failed { waiting }`, the object parks `failed { waiting, "waiting for p2.catalog: not served" }`. A reload on S serving `Catalog` re-runs the hold: `loading`, `bound`; the reference goes live, the dependent wakes. No backoff, no new hold. | The hold is refused `broken`; `ForwardStore` fails `transient` and re-holds every 1-60 s, refused each time (`remote/hold.ts` `#refuse`, `object.ts` backoff). | §1, §3.1, §4, §5 | +| 4 | `self.get(SessionInfo)` on W1 when the core facet is missing (same host, not served). | The request is unbound `not_served` on W1 itself; the dependent parks `failed { waiting }`; a reload serving the type binds it and wakes the dependent. | `#assertRoute` throws in the factory: the object is `broken` until `retry()` or a reload (`host.ts` `reference`). | §1, §3.1, §4 | +| 5 | A tab's `NotesView` declares `Notes { S1 }`; S routes `Session`, `resolve` forks W1, W1 connects and publishes. | The hold waits at T on its path (`no_link` while S resolves: `loading`), is relayed to W1, waits there `unpublished` until W1's setups, then binds; `bound` comes back through the relay. `NotesView` was `loading` the whole time and goes live. | The same chain of waits, each bounded by `readyTimeout`; a slow worker start past it fails the tab object `transient` and polls. | §1, §6.2 | +| 6 | `host.get(Notes, { session }, ctx)` from a tab's page code while W1 is starting. | The getter waits while the reference is `loading`, up to its `readyTimeout`, then rejects with the cause and releases its request. It resolves as soon as `bound` arrives. | The same. | §8 | +| 7 | `host.get(Catalog, {}, ctx)` on W1 when S does not serve `Catalog`. | The reference becomes `failed { waiting }` at the first answer; the getter rejects at once with "not served on server" (recommended; owner's choice to wait instead). | Rejects at once too (a broken failure), after the refused hold's `failed`. | §8 | +| 8 | `host.hold(Catalog, {}, ctx)` on W1 before W1's `load` ran. | The reference is returned at once, `loading` (`unpublished`); W1's publication places it over the parent; it follows the object for its whole life: `loading`, `live`, `failed { waiting }` if S drops the plugin, `live` again when it comes back. | Returned at once; placed once `load` resolved (`host.ts` `#hold` races `load`); a removal ends it for good with `gone`. | §1, §8, §5 | +| 9 | `host.hold(Missing, {}, ctx)` for a local type nobody serves here. | `loading` before the publication, `failed { waiting, not_served }` after it; a reload serving it binds the same reference. | Throws `unavailable` at once after `load`; before `load`, fails the reference broken once it resolved. | §8 | +| 10 | `await host.registry(HarnessModels, ctx)` by base code when the models plugin is missing. | Rejects at once with `unavailable/undeclared` (`waiting`); code that wants to wait declares the registry in an object or holds. A served registry whose first `compose` threw rejects the same way (`compose`). | Rejects at once (slice 8 build). | §8, §9 | +| 11 | `ctx.get(Sessions, {})` inside a worker tool through `host.root` (`plugins/sessions/worker.ts`) while the parent link is down. | The request waits on the path (`no_link`, `loading`); the getter waits up to `readyTimeout`; the link comes back, the path re-holds, `bound`, the getter resolves. The request ends when the tool call settles. | The same (`ctx.ts` `CallContext.get`, `path.ts` re-hold). | §1, §8 | +| 12 | A call `notes.add(args, ctx)` on a reference whose object is `loading` (W1 restarting). | The call parks in the reference's queue with its own `readyTimeout`; `bound` flushes the queue in order; past the deadline it rejects `unavailable/park_timeout`. | The same (`host/parked.ts`). | §8 | +| 13 | A call on a reference that is `failed { waiting }` (the plugin is not there). | Rejects at once with the cause, no parking: nothing is under way. | Rejects at once (the reference is failed). | §2, §8 | +| 14 | A call already sent to W1 when the IPC link drops. | The call fails `unavailable` and is never resent (its outcome is unknown); the reference goes `loading`; calls made after the drop park until the re-hold's `bound`. | The same (`consumer.ts` link close). | §2, §5 | +| 15 | The parent link drops while `ForwardStore` is parked `loading` on `Catalog` (S restarting). | The request stays on the path (`no_link`), `ForwardStore` stays `loading`; the reconnect holds it again, S answers `loading { attempt }` or `bound`; the dependent wakes on the state change. No deadline on the wait. | The dependent stays `loading` until `readyTimeout`, then `transient` with backoff (the audit's choice 6). | §1, §3.3, §4 | +| 16 | A route link to W1 drops while T holds `Notes { S1 }` through S's relay. | The relay sends `loading` downstream, rejects relayed calls in flight `unavailable`, re-resolves the address with the route backoff, holds again upstream with the kept request; `bound` is forwarded. The tab's dependents stay `loading`. | The same (`relay.ts`); the route backoff stays. | §1, §4, §11 | +| 17 | T's own link drops while its hold for `Notes { S1 }` is still unbound at W1 (`not_served`). | S sends `closed { origin }` upstream; W1 drops the indexed hold with the origin: no leak. T reconnects and holds anew. | The same for refused holds (`provider.ts` origin close). | §1, §10 | +| 18 | A tab closes while `NotesView` is parked on `Notes { S1 }`. | Its last holder releases: a parked object ends at once (it is not live, so no `linger`); its requests end, the relayed hold is released upstream, the index entries go. Nothing keeps the chain. | The same (`object.ts` `#idle`). | §1, §3.5, §10 | +| 19 | `host.revoke({ grant })` while a guest's objects on W1 are parked and its tab's holds wait unbound. | Every request of the principal ends, bound and unbound, at every hop: the parked objects end `forbidden/revoked`, the tab gets `gone` for each hold, the child link closes. Nothing revives them; a new login holds anew. | Bound objects end, holds waiting for `load` are ended (`host.ts` `revokeWaiting`); refused holds are released with the link. | §10 | +| 20 | A worker with nothing live: a tab holds `Notes { S1 }` but the notes plugin is missing on W1. | The unbound hold counts in `host.activity.references` from its arrival, so the supervisor does not stop W1 while the tab waits; it is not in `host.references` (presence lists bound references only). | A refused hold is not counted: the worker stops idle, restarts on the next hold, refuses again. | §10 | +| 21 | S holds `SessionTransfer` on W1 for a fork while W1's `Harness` is parked on the models registry (not served). | The hold binds to `SessionTransfer`, whose attempt parks on `Harness`: `loading`, then `failed { waiting, "waiting for p2.harness: waiting for registry …" }`, with the chain in the message; S's `ctx.get` rejects at once with it. A reload on W1 serving the models plugin wakes the harness, then `SessionTransfer`; a later `ctx.get` binds. | `SessionTransfer` fails `transient` after `readyTimeout`, polls; S's getter rejects at its own deadline. | §3.1, §3.4, §8 | +| 22 | `ref.retry()` from a tab's Retry button on a `failed { waiting }` reference whose blocker on W1 is `broken` (a throwing factory). | The retry passes over the link (`retry { hold }`) to the parked object, which passes it to its `broken` blocker: a new attempt; `waiting` blockers are passed through, `not_served` ones untouched. If the blocker goes live, the chain wakes. | `retry` reaches only the object the hold is on; a refused hold is started again. | §3.7 | +| 23 | Keyed pin: `Notes` pins `store.drafts.at(key)` on `NoteStore`; the key's load fails at the provider. | The pin is a dependency: `Notes` parks `failed { waiting on p2.note-store.drafts[key] }`; the provider's re-bind of that member (a retry) wakes it. | `dependency_failed` transient with backoff (`attempt.ts` `pinKeys`). | §3, §4 | + +# Scenarios: Providers, Versions, Registries {#sc-providers lede="Plugins missing, added, removed and broken; versions that do not fit; registries and the harness."} + +Opus's scenarios cover the provider side: plugins that are missing, arrive by a reload, are removed or break; versions +that fit through `accepts` or do not; registries with missing providers, skewed versions and last good values; +`self.follow` over a harness that restarts; and reloads of an object's own code. The pattern: what used to be `broken` +or polled becomes `waiting`, woken by the one event that can fix it, and an object's own code is still the only thing +that breaks it. + +The Rule column cites sections of the working document, reproduced in appendix C. + +| # | Scenario | Under the model | Today | Rule | +| --- | --- | --- | --- | --- | +| 1 | Worker W1 has no drafts plugin. Chat (chat plugin, on W1) declares `self.get(DraftStore)`. | Chat's request is unbound `not_served` and indexed under `DraftStore`. Chat is `failed { waiting, "waiting for DraftStore: not served" }`, with no timer. A tab's Panel holding Chat shows the same cause, and its calls reject at once. | `#assertRoute` throws in the factory, so Chat is `broken`. Only `ref.retry()` or a reload recovers it (`host.ts`). | §1, §2, §3.1 | +| 2 | A reload adds the drafts plugin to W1 while Chat waits (1). | The reload's commit publishes `DraftStore`. The index wakes Chat's request, which binds; DraftStore activates and goes live. Chat leaves parking, opens its natives and goes live; Panel goes `live`. | Chat stays `broken` unless the reload retries broken objects. | §4, §5 | +| 3 | Server S has no models plugin. On W1, the models plugin's host-lifetime ModelRuntime declares `self.get(ModelConfig)` (placed on S). | S keeps W1's hold as refused, `not_served`. ModelRuntime is `failed { waiting }`. W1's `load` counts it as a first outcome (parked on another host). The harness runs, and its generations report "model unavailable". | S refuses `broken` (`remote/hold.ts #refuse`). ModelRuntime is `transient` and polls: each backoff tick (1-60 s) makes a new hold across the link and reports again, for the host's life. | §1, §3, §6.3 | +| 4 | S gets the models plugin by a reload (3, continued). | S's commit re-runs its refused hold and sends `loading { hold, attempt }`, then `bound`. ModelRuntime wakes, opens its native and contributes to `HarnessModels`. The harness's next generation finds a model. | The next backoff tick re-holds; recovery waits up to 60 s. | §4 | +| 5 | A reload removes the ports plugin from S. Tabs hold `PortsView`; Radius (live) reads the `radius.forwards` registry that Ports contributed to. | PortsView's objects end, and their holds become unbound `not_served`: the provider sends `failed` on the bound hold, and the tabs show `failed { waiting }`. Radius gets a table without Ports, composes the empty value and keeps working. A later re-add re-runs the kept holds, which bind again. | The holds get `gone` and the references end for good (design §21). A re-add cannot revive them. | §5, §9 | +| 6 | A reload removes W1's file-index plugin. The live Files object (files plugin) holds `self.get(FileIndex)` locally. | Files' `Binding` becomes unbound `not_served`. Files stays live (R18). Its member sources keep their last values, and calls through that reference reject with the cause. When file-index is re-added, the binding binds again and the members re-bind by name (one frame). | The reference ends `released`; Files can never use FileIndex again without a restart. | §5 (locally), §3.10 | +| 7 | After a sessions plugin update, S's Catalog factory throws. W1's Chat waits on Catalog; a tab's Panel waits on Chat. The user presses Retry on Panel. | Catalog is `broken`; Chat and Panel are `failed { waiting, "waiting for Catalog: …" }`. Retry passes on hop by hop (T, then S's Catalog, through W1) and retries Catalog once. If the cause is gone (the config is fixed), Catalog goes live and Chat, then Panel, wake. | Chat polls by backoff with new holds. Retry on Panel retries only Panel, which is still blocked. | §3.7 | +| 8 | S's EnvManager (envs plugin) opens an SSH connection that fails because the remote host is offline. W1's environment objects wait on S's env views, which wait on EnvManager. | Only EnvManager backs off (`transient`, its own `retryAt`). Its dependents are `failed { waiting }` with no timers. When SSH comes back, EnvManager goes live and the chain wakes in order. A `host.get` on a dependent rejects at once (it is `waiting`); a hold just shows it. | Every level runs its own 1-60 s backoff and re-holds across the link at every tick. | §3.1, §3.3, §8 | +| 9 | A reload of the terminals plugin restarts S's TerminalService (natives: the terminal host process D). Terminals{session} objects are live and depend on it. | Native restart as in §21: new calls park while it is `loading`; running calls drain; natives close with reason `reload`; the factory runs; natives reopen. Its dependents stay live (R18); their calls park while it is `loading`. | The same; this is design §21, unchanged. | §5 (old calls), §3.10 | +| 10 | Worker W1 crashes while a tab views Chat{S1} through S. | S's relayed hold goes to its path, `no_link`, so the tab's reference is `loading` and its calls park, bounded. The route re-resolves (route backoff); the new W1 publishes, the held request binds, and a fresh `bound` reaches the tab. | The same, except a dependent that is activating fails at its 60 s deadline in between. | §1, §2 | +| 11 | A tab on Chat v1 meets W1 serving Chat v2 with `accepts[1]`. | The check accepts v1; the hold binds through the mappings (slice 7). The tab is `live`. | The same once slice 7 builds `accepts`. | §1, §5 | +| 12 | A tab on Chat v1 meets W1 serving Chat v3 without `accepts[1]`. | The hold is unbound `version`, so the tab shows `failed { waiting }` ("update the page") and its calls reject. The tab's own reload makes a new v3 request and releases the old one. If W1 rolled back to v2, the old hold would bind again in place. | Refused `broken`, or `gone` at the reload that dropped v1: ended for good. | §1, §5 | +| 13 | A tab reloaded to v2 holds Chat while W1 is installing v2. | W1 keeps the hold `installing`: the tab is `loading` and its calls park, bounded. W1's install commit re-evaluates the hold, which binds v2. If W1's install is rejected, the hold is re-evaluated at that end and becomes `failed { waiting, version }`. | Refused `broken` (`contract_mismatch`); design §21's `updating` is not built. | §5 | +| 14 | A plugin changed Chat's contract without bumping the version: a tab on v1 (fingerprint A), W1 on v1 (fingerprint B). | Unbound `version` (a fingerprint that differs at the same version), so `failed { waiting }`. It is not `broken`: the consumer did nothing wrong, and its next reload or a fixed provider heals it. | Refused `broken`, `contract_mismatch`. | §1 | +| 15 | A reload of chat makes Chat v4 accept v3 but drop v2. Tabs hold Chat at v2 and at v3. | Only the v2 holds unbind (`version`). The object, its v3 holds, documents, jobs and natives stay. | Design §21: the version's holds end with `gone`. | §5 | +| 16 | S has no web-shell plugin. Files, artifacts and terminals contribute routes to `Http`; proxy-marker reads `Http`. | The contributions wait in the table. Proxy-marker's reader is unbound `not_served` at once after the publication: `failed { waiting }`. A reload that adds web-shell serves and composes `Http`, the index wakes the reader, and it goes live. | Slice 8 S3: the reader fails `transient` (`unavailable/undeclared`) at once and is woken at the first value only. | §9 | +| 17 | The codemode plugin is reloaded alone with a `HarnessExtensions` token v3; the core harness serves v2. | Codemode's v3 contribution is left out and reported (`contribution_refused`). A reader bundling v3 is unbound `version`: `failed { waiting }`. A core reload to v3 makes it usable, which wakes the reader. | The contribution is refused the same way; the reader is `broken` (S2). | §9 | +| 18 | Core is reloaded alone, serving `HarnessExtensions` v3 with `accepts[2]` (contribution and value). | v2 contributions are adapted once per value; v2 readers get the composition mapped back once per composition. A reader at v1 (no mapping) is `failed { waiting, version }`. | The same, except the v1 reader is `broken` (S2). | §9 | +| 19 | The harness's `HarnessExtensions` compose throws at the first composition (an owner bug). | The registry is served but has no usable value, so the Harness's request is unbound `compose`: `failed { waiting }`. The barrier counts its first outcome. A reload that fixes compose makes the value usable, and the Harness wakes and activates. | S3: the Harness fails `transient` at once, woken at the first good value. | §1, §9 | +| 20 | Codemode contributes its sandbox (a native) to `HarnessExtensions`. Later another contributor's entry makes compose throw, and then codemode restarts. | While compose throws, `.value` keeps the last good composition. When codemode's contribution leaves, that last good value is dropped (it holds the closed sandbox), and `.value` throws until compose succeeds again. No reader is handed a closed native. | Slice 8 keeps the last good value regardless, so a reader could use the closed sandbox. | §9 | +| 21 | Transcript (on W1) follows the harness through `Harnesses.value.observeConversation(...)` and declares `retryOn(Harnesses)`. The harness's Session fails and the harness object restarts. | `self.follow` is Transcript's own step: S1 ends the observer `failed`, so Transcript fails `transient` (its own backoff). When the new harness contributes, `retryOn` wakes it, and it attaches to the new harness and gets one fresh frame. | The same (slice 8 S1 and `retryOn`); unchanged by the model. | §3 (follow), §3.8 | +| 22 | A reload of chat swaps the live Chat in place (no natives). Its new factory adds a `self.get(Sharing)`, and the sharing plugin is absent. | Owner item 4, recommended: Chat parks. Its holders see `failed { waiting, "waiting for Sharing: not served" }` and its calls reject until sharing appears. The alternative keeps the old code until the dependency resolves. | Not built (slice 7); design §21 swaps the object and opens its gate. | §3.9, §13.4 | +| 23 | Chat is parked on DraftStore (1). A chat reload ships code that no longer needs DraftStore. | One handover: the new attempt's requests bind first, then the old ones are released. Chat activates on the new code and goes live; the old `not_served` index entry goes. | Not built; today a broken Chat is retried by the reload (slice 7 plan). | §3.9 | +| 24 | A reload removes the models plugin from W1 while ModelRuntime is parked on S's ModelConfig (3). | ModelRuntime (host-lifetime) ends with its plugin. Its requests end, and W1's refused hold at S is released (a `release`). Nothing stays waiting. | The type's objects end (§21), and its backoff polling stops with it. | §3.5, §1 | + +# Scenarios: Rollouts and Cycles {#sc-rollout lede="Delayed facets across hosts, rollout windows, chains over three hosts, and the probe at work."} + +Sol's scenarios cover the windows in which hosts run different code: a server facet ahead of its worker facet and the +reverse, tabs older or newer than the server, installs that are queued, staged or rejected, a type moved between host +kinds; then chains across three hosts, diamonds and rings, the probe's own corner cases and a forged probe. Row 20 is a +case round 3 found against the previous draft; the rule that fixes it is in chapter VIII. The pattern: a version that +does not fit is a request waiting for a publication, never an error that ends it; an install answers only for its own +host; and a cycle is found by the attempt whose edge closed it. + +The Rule column cites sections of the working document, reproduced in appendix C. + +| # | Scenario | Under the model | Today | Rule | +| --- | --- | --- | --- | --- | +| 1 | The models plugin's server facet upgrades first. Browser and TUI model pickers still use its older Models contract, which the new provider accepts. | Their references remain usable through the old-version adapters. If an owning native restarts, affected references show `loading` until its fresh binding is ready. A caller that later installs the new contract releases its old request and makes a new one. | `host/host.ts:#assertMatch` accepts only exact versions and fingerprints. Accepted-version serving across reload is not built in next; p2 uses its older cluster reload runtime. | §1, §5 | +| 2 | A worker's new search facet makes SearchIndexer depend on SearchFeed v2. The server still serves v1; its search facet installs later. | SearchFeed's request is `failed { waiting }` for the version mismatch. SearchIndexer is also `failed { waiting }`, without a dependency retry timer. The server's later publication rechecks the request; SearchFeed becomes `live` and wakes the indexer. | A remote mismatch is refused as broken; its dependent fails transient and retries with backoff. See `host/attempt.ts` and `remote/hold.ts`; p2's real indexer declares this server dependency in `plugins/search/worker.ts`. | §2–5 | +| 3 | A new web workspace facet arrives before the selected worker's matching Workspace contract. The user opens the workspace panel during this window. | The panel's held Workspace reference becomes `failed { waiting }`; an awaited getter rejects under the recommended getter rule. The old worker object and compatible readers are unaffected. Installing the worker facet wakes the held request and supplies a fresh binding. | Exact-match refusal reaches the client; a dependent object can enter R18 backoff. Next has no install-aware wake protocol yet; p2's workspace UI and worker facets live in `plugins/workspace/`. | §2, §5, §8 | +| 4 | A tab sleeps while the worker upgrades Workspace and accepts its old version. The tab reconnects without updating its code. | The retained request shows `loading` while its path is down. Re-holding asks for the tab's original version, and `bound` supplies the mapped snapshot, returning it to `live`. No old method index is interpreted as a new one. | Path re-holding already exists in `remote/path.ts`, but unequal versions still fail in `host/host.ts`. Reload-time adapters are slice 7 work. | §1, §5 | +| 5 | The server publishes a new Models contract without accepting the old TUI version. The TUI launcher downloads its replacement bundle later. | The old TUI request stays held but shows `failed { waiting }`. The old runtime releases it when the launcher swaps workers. The replacement runtime makes a new-version request and can become `live`; the old request is not silently relabeled. | p2 polls for TUI bundles and swaps sandbox workers in `hosts/tui-client.ts`. Next does not yet implement reload-time version recovery. | §1, §5, §8 | +| 6 | The server and tabs already use revision R5 when a route starts a worker from an older plugin set. The worker connects before loading its current facets. | Business requests stay `loading` until publication or show `failed { waiting }` against published incompatible code. The loader installs the desired code set without awaiting activation. Publication then rechecks waiting requests and the actual objects activate; a route must not wait for the worker's activation barrier before admitting its link. | p2 sends selected worker facets through WorkerInit and uses a separate control-ready sequence. Next has the publication/activation admission deadlock described in `host/host.ts:load` and `remote/hold.ts:start`. | §1, §5–6 | +| 7 | A tab installs new workspace code while the worker's matching install is queued but has not started. | The old worker answers `failed { waiting }`, not `loading` merely because another host knows an upgrade is planned. A held reference survives; a getter rejects. The worker's own install and its completion re-evaluate the request, so it can later become `live`. | Next has no revision/install status handling; it returns an exact-match failure. The model deliberately accepts this rollout-window blip instead of adding revision fields to mismatch messages. | §2, §5, §8 | +| 8 | The worker's search facet stages a new contract, so an incompatible request is held as installing. Staging fails and the old search code remains published. | The request was `loading` during the install. Rejection is an install-end event even without publication, so the request is checked against the old provider and becomes `failed { waiting }` for the remaining mismatch. Its getter rejects rather than waiting forever for a commit that will not occur. | Next has no installer. p2's `hosts/server.ts:performReload` retains published code and bundles when preparation fails, through a different reload protocol. | §4–5, §8 | +| 9 | A worker's own envs facet install is rejected while it needs a capability from the new code set. A provider elsewhere becoming ready cannot repair that rejected local install. | Requests blocked by the worker's own rejected install fail promptly with its reason. Published old code is not replaced by half-installed new code. A later successful local install or explicit retry is needed; waiting for a remote activation is not an install strategy. | Next has no own-install state. Design §21 specifies the prompt own-rejection rule; p2 reports reload refusal while retaining its previous selection. | §5 | +| 10 | The worker ports tool facet is added before the server ports facet. The tool is registered successfully, but the first invocation looks up server Ports with `ctx.get`. | The tool owner remains `live`: this is a call-time lookup, not a declared factory dependency. The missing Ports request is `failed { waiting }`, so the invocation reports unavailable and its call-scoped request ends. After the server facet publishes, a later invocation can acquire a live Ports object. | This is how `plugins/ports/worker.ts` declares the tool: its callback does `ctx.get(Ports)`. Next's remote lookup currently receives a broken refusal; no provider-publication recovery is built. | §1–3, §5, §8 | +| 11 | The server removes Ports first, while old worker tool code and an old browser Ports sheet remain loaded. The worker and browser facets update afterward. | The browser's held request becomes `failed { waiting }` without ending permanently. Ordinary live worker owners stay `live`; their next Ports lookup fails while the provider is absent. Updating those consumers removes obsolete surfaces, or a later provider publication can restore a still-held request. | p2 normally refuses removal when outside-batch consumers still hold a removed type (`hosts/server.ts:refuseHeldRemoval`). Next's target design originally ended removed-type references; the retained-request behavior is new. | §3.10, §5 | +| 12 | Notes(session=7) moves from a server facet to a worker facet. An old tab still computes the server address; a new tab computes worker 7. | The old request stays at the old address and shows `failed { waiting }`; publishing Notes on the worker cannot wake it there. The new tab routes to worker 7 and can become `live`. Updating the old consumer computes a new address and creates a new request; the model adds no redirect. | Placement is already computed by the caller and checked by the provider in `host/placement.ts`. Next has no relocation reload implementation. | §1, §5 | +| 13 | Worker 1 still serves Notes v1 for its own session. Its new code requests Notes v2 placed on worker 2, which already serves v2. | Resolution chooses worker 2 before checking its version. The request can become `live` without waiting for worker 1's unrelated registration to upgrade. Worker 2's `bound` wakes any dependent waiting on that request. | `host/host.ts:#assertRoute` checks a same-name local registration before `Placement.reference`, so worker 1's v1 can wrongly reject this v2 request. | §1 | +| 14 | The models plugin's server facet updates now, but the worker facet updates hours later. Each host has its own native model registry and compatible local contributors. | Each host composes its own table and its local consumers can stay `live`. A worker does not wait for the server's registry revision: native registries are host-local. Only an actual remote object contract used between those facets creates a cross-host version wait. | p2 also has independent server ModelConfig and WorkerModels resources (`plugins/models/server.ts`, `worker.ts`). Next's registry adapters already handle local reader/contributor versions; atomic registry replacement is not built. | §5, §9 | +| 15 | A tab Panel depends on a worker SessionView, which depends on a server Catalog view. Catalog's new facet arrives last. | Catalog's missing request is `failed { waiting }`; SessionView and Panel show waiting with the nearest blocker and root cause. The server publication binds Catalog and makes it `live`. Its state change wakes SessionView; that provider's `bound` then wakes Panel, with no dependent backoff loops. | R18 retains dependency references but fails dependents transient and uses identity backoff. `host/object.ts:dependencyLive` already skips that backoff on a live event; registration-based recovery is missing. | §1–4 | +| 16 | A server Summary waits for views on two workers. Both views reach a shared local CatalogStore on the server, whose own required provider is absent: a diamond, not a cycle. | CatalogStore is `failed { waiting }`, and waiting propagates through the views to Summary. Probes reach the common store, but its unavailable request has no target attempt and no edge back to Summary. It must not report `invalid/cycle`; adding the missing provider later wakes the chain. | Current local waiting-edge traversal and trusted activation chains distinguish cycles from shared dependencies. Dependency failures still use R18's transient/backoff path (`host/object.ts`, `host/attempt.ts`). | §3–4, §7 | +| 17 | Thirty-three object attempts on trusted server/worker links form a ring and start together. Each reference resolves to the next attempt in that ring. | All participants stay pre-activation, showing `loading` or waiting according to their blockers. With answer (A), the attempts waiting across links time out after `readyTimeout`, retry with backoff, and the activation chains of the later attempts find the ring. With answer (B): the attempt whose edge closed the ring last starts a search; it returns, a check along its recorded path returns, and the initiator fails `broken invalid/cycle`; propagation breaks the others and the dependents off the ring. The probe carries its trail of 33 hops; no cap cuts it. | Concurrent cross-host cycles can escape fixed activation chains and rely on readyTimeout/backoff. `host/object.ts:assertNoCycle` and design §6 document that limitation; next has no probe. | §3.1, §7 | +| 18 | A ring is being confirmed when one worker reloads its facet and removes its outgoing dependency, repairing the ring. Some probe returns are already in flight. | With answer (B): New attempts have new ids, and obsolete target ids are discarded. If the check passed before the repair, the ring is reported (it existed when the check began); if the repair came first, the check dies at the replaced attempt and nothing is reported. Unchanged attempts can therefore become `broken invalid/cycle` and need `ref.retry()` or their own plugin reload; provider readiness alone does not retry broken objects. | Next has no probe or reload implementation. Broken cycle failures already require retry, and `host/object.ts:retry` starts another attempt. | §3.6–9, §7 | +| 19 | A malicious tab learns a worker attempt id and sends a forged probe naming that same attempt as its initiator through the server relay. | With answer (B): The server drops the probe because it arrives on a child link, before any forwarding. The worker object and legitimate references keep their states. A child link never carries cycle evidence. | There is no probe message today. `remote/provider.ts` already strips activation chains from untrusted holds rather than trusting a child to declare ancestry. | §7.2 | +| 20 | A→B→C wait while C's return path to A is disconnected. Sixty-five unrelated initiating rounds pass through C before reconnect makes C→A a known edge. | With answer (B): Found in round 3 against draft 3 (C could have evicted every round from the ring). Under the model the re-hold gives C a new edge token, so C starts its own search; it crosses A and B and returns to C, the check along its path confirms, and C fails `broken invalid/cycle`. Foreign rounds and evictions play no part. | Current re-hold carries an activation chain (`remote/remote-object.ts:attach`), which can detect a small newly reachable ring; otherwise the dependency deadline remains its fallback. No probe-round cache exists today. | §7.2–3 | + +# Open Questions {.epilogue lede="What the owner decides, and what slice 7's spec must still pin down."} + +The model replaces five answers to "the provider is not there" with one: the request stays, the dependent parks, an +event wakes it. It ends the startup deadlock with a gate that lasts only for the synchronous setups, finds cycles with a probe +instead of a deadline (or, if the owner prefers, one narrowed deadline), and keeps timers only where no event can exist. Four reviewers played more than 300 +scenarios against five drafts; the catalog in the chapters before this one is a selection. + +## Decisions for the owner + +1. **The barrier** (chapter VI): (a) Chord's `load` waits for the first outcomes of this host's host-lifetime objects, + with the local-blocker rule; or (b) `load` resolves after the publication and the app reads `host.startup`. + Recommended (a); Astra, Opus and Sol for (a), Fable for (b). +2. **Getters on `waiting`** (chapter IX): reject at once (recommended) or wait up to `readyTimeout`. +3. **`waiting` as a cause kind** (chapter III): a new kind (recommended; Opus, Fable), `transient` without `retryAt` + (Astra), or `loading` with a diagnostic (Sol). +4. **A pure swap whose new factory needs a dependency that is not there:** the object parks (its gate closes, its + holders see `loading` or `failed { waiting }`, calls park or reject; recommended), or it stays on the old code until + the dependency resolves (old and new code would then overlap without bound, against §21's "a host is at R or R−1"). +5. **The last good value** (chapter X): kept only while every contribution it was built from is still in the table + (recommended; Astra), or kept as slice 8 decided, with natives in a registry looked up at use through their owner + (Sol). +6. **Cycles across hosts** (chapter VIII): (B) the probe, with (B1) a re-search backoff that sends probes only + (recommended) or (B2) a full search lifecycle with no timer at all; or (A) the slice 4b decision narrowed to one + deadline, which polls chains across hosts again, misses a ring of waiting attempts, and retries in lockstep without + finding the cycle it is for. + +## For slice 7's spec + +- The test list: the startup deadlock resolving without `readyTimeout`; an early hold placed here, not through the + parent; remove-then-serve with a live dependent's calls recovering; a 1,000-hold publication wave under the window; + the probe on long rings, diamonds, edge changes, reloads, relays, reconnects and forged probes; GC of a parked + three-level chain after its top is released; every index entry gone after release, revocation and link loss. +- The local re-binding of a `Binding` (member sources, keyed pins, parked calls, presence) and the unbinding of a live + hold under flow control, both with their race tests. +- The wire: `waiting` in the `failed` cause schema, the attempt id on `loading` and `failed`, the `probe` message, the + pinned list; `PROTOCOL_VERSION` stays 1 until p3 ships. +- The design changes listed in appendix A, folded into `design.md` and the decision log in the same commit as the + owner's decisions. + +# What Changes {.appendix} + +| Where | Today | Under the model | +| --- | --- | --- | +| R18, activation | a dependency that does not go live within `readyTimeout` fails the dependent `transient`; backoff 1 to 60 s | the dependent parks with no timer; `failed { waiting }` when a dependency failed; woken on every state change of a dependency; the deadline paused while parked | +| R18, the boundary | "live" means every dependency was live at the end of activation | the dependency check is the boundary: after it, a dependency leaving `live` is treated as after live | +| `self.get` of a missing `local` type | throws in the factory: `broken` | unbound `not_served`: `failed { waiting }`, woken by the publication that serves it | +| a remote hold the provider cannot serve | refused `broken`; the dependent polls by backoff | refused and kept at the provider, re-run at its publication; `failed { waiting }` | +| design §3, placement | "served here" decides "here" | the address decides; a remote type without placement keeps today's rule | +| the start barrier, `pendingLoad` | holds wait for all of `load` | holds wait only for the publication; the barrier is (a) or (b) | +| S2, a registry reader newer than its provider | `broken` | `failed { waiting, version }` | +| S3, a registry without a value | `transient`, woken at the first value only | `failed { waiting }`, woken at every transition to usable | +| slice 8, the last good value | kept whenever `compose` throws | kept only while every contribution it was built from is still in the table | +| design §6, cycles | a walk on one host, chains across hosts, a documented gap closed by the deadline | the gap of simultaneous starts across hosts: (A) one narrowed deadline, or (B) a probe: a search per edge change, a check along the recorded path, edge tokens, `dead` notices | +| design §7, `host.hold` | throws for a missing local type after `load` | always returns a reference | +| design §7, getters | wait through every failure up to `readyTimeout`, reject on `broken` | wait through `loading` and `transient`; reject at once on `waiting` and `broken` | +| design §21, removal | `gone` to every holder | `failed { waiting, not_served }`; the request binds again when the name is served | +| design §21, a dropped version | `contract_mismatch`, final | only that version's requests unbind; the object stays | +| design §21, rollout | `updating` with revision fields and a deadline | each host knows its own install; `installing` re-evaluated at the install's end; installs never await activations | +| `FailureCause` | `transient`, `broken` | adds `waiting` | +| the wire | | `failed` cause `waiting`; the attempt id on `loading` and `failed`; with (B), one new message, `probe` (`search`, `check`, `dead`) | +| unchanged | | an object's own code reload (§21); R18 after live; `retryOn` outside parking; `self.follow`'s R4; the call window and flow control; revocation's semantics; no client providers | + +# Glossary {.appendix} + +**Attempt:** one try at activating an object: its factory, requests, natives; its own id since the model. **Bound:** a +request pointing at an object here or a hold over a path. **Dependency:** a request an attempt needs live before it +may go live (`self.get`, `self.registry`, a keyed pin). **Dependency check:** the moment all of an attempt's +dependencies are live; after it the attempt runs its own steps. **Edge generation:** a number of a wait edge, incremented by its +source whenever the request binds anew. **First outcome:** an object is live, failed by its own cause, or parked with no local +blocker still in its first attempt. **Holder:** whoever made a request and alone ends it. **Parked:** an attempt +waiting for a dependency, with no timer. **Probe:** the message that walks wait edges to find cycles: a `search`, then a `check` along the recorded path. **Publication:** +a host making its facets' declarations visible: `load`'s setups, a reload's commit, an install's commit. **Refused:** a +request without authority or with invalid params; never indexed. **Request:** what a reference stands for: type or +registry, params, principal, version. **Round:** the number of one search, from a counter per attempt. **Seen set:** the newest round number an +attempt has forwarded, per initiator (up to 256). **Unbound:** a request with no provider at the moment, with a reason (`unpublished`, `not_served`, +`version`, `compose`, `installing`, no link). **Waiting:** the cause of a failure that is not the object's own, woken by +an event. **Wait edge:** a parked attempt waiting for another attempt. **Wake:** an event that makes the resolver look +at a request again. + +# The Working Document {.appendix} + +The proposal as the reviewers checked it, `/tmp/model/model.md`, draft 4. Its section numbers are the ones the +scenario catalogs cite. + +Status: proposal for the owner, not decided. Built in slice 7 with reload and rollout. Sources: the owner's direction +("park until resolved"; "no difference between local and remote resolution: a link going is like a plugin +unloading"), the reviews (`/tmp/dep-mechanism-*.md`, `/tmp/dep-graph-*.md`, `/tmp/barrier-*.md`, +`/tmp/model/review{1,2}-*.md`), the code at pi `358f7d46a`. **Owner** marks an open decision. + +Draft 7 (round 6): (A) also retries in lockstep without finding the cycle (Fable); the probe's recovery after a +suppressed search becomes a choice between (B1) a re-search backoff for probes only (recommended) and (B2) a full +search lifecycle without timers, since `dead` notices alone need recovery state that eviction may not drop (Astra, Sol). +Causes are compared by value on wakes (Opus). +Round 7: a returned search is accepted while its initiator is parked, whatever its round (Astra: re-searches must not +invalidate searches under way); detection is eventual, not within one backoff period; the local walk stays (Opus). + +Draft 6 (round 5): cycles across hosts become an owner choice between (A) the owner's slice 4b decision (simultaneous +starts across hosts time out, no probe message), narrowed to one deadline, and (B) the probe, completed with edge +tokens, no revisits, and `dead` notices back to the initiator (Astra: a check that dies silently can lose the only +search that survived dedupe). (A) turned out to either poll chains across hosts again or miss a ring of `waiting` +attempts, so (B) is recommended, pending its review. + +Draft 5 (round 4, Sol's phantom cycle): the second round is replaced by a check along the recorded trail of +`(attempt, edge generation)` hops; a reported ring existed at one moment; probes grow with their path. + +Draft 4 after round 4 (probe only): round numbers per attempt, forwarded only when newer than the stored one; probes +honoured on parent and route links, dropped on child links; completeness stated under an eventually stable graph; the +repair caveat broadened to any concurrent edge removal; a new edge generation resets confirmation; a bounded display +trail. + +Draft 4 changes (round 3): the probe starts a fresh round on every new or changed wait edge (a ring closes at its last +edge, whose source finds it), keeps one seen entry per initiator, accepts only its current round, and states its +cost honestly; every host drops probes from untrusted links; local forwarding is iterative; no-placement remote +requests are re-placed when a host's first parent link connects; a cohort object that ends before its first outcome +settles the barrier; the last-good refinement is marked for the owner. + +Draft 3 changes (round 2): remote types without placement keep today's fallback (here if served, else the parent); +hydration stays before parking; no copied `retryAt`; a dependency `ended` breaks its dependent; a reload's handover +binds new requests before releasing old ones; retry passes only to `broken` and `transient` blockers, once per +operation; `installing` re-evaluated at every install end (commit or rejection); a registry's last good value dropped +when one of its contributors leaves; the probe with round ids, bounded seen sets, edge-generation triggers, attempt +ids on every non-live answer and relays dropping probes from untrusted links; one cycle mechanism, local and remote. + +## 0. The problem + +The same situation, "what I need is not there", has five outcomes today, depending only on how it is declared: + +| Declared as | Provider missing | Today | +| --- | --- | --- | +| `self.get(LocalType)` | no facet serves it | `self.get` throws, the factory throws: `broken` | +| `self.get(RemoteType)` | the worker does not serve it | the hold is refused `broken`; the dependent `transient`, polled 1-60 s forever | +| `self.registry(R)` | no provider | `transient`, woken at R's first value only | +| `self.registry(R)`, reader newer | | `broken` | +| a reload removes the type | | holders get `gone`: ended for good | + +And the startup barrier deadlocks an acyclic graph across two hosts (Sol, `/tmp/dep-graph-sol-barrier.log`): S's eager +A needs W's B, B needs S's C; W's hold for C waits for S's whole `load`, which waits for A. Only `readyTimeout` ends it. + +## 1. Requests and one resolver + +- **A request** is what a reference stands for: type (or registry), params, principal, the consumer's contract version. + Its holder makes it and owns it: an attempt (`self.get`, `self.registry`, keyed pins), a call (`ctx.get`), outside + code (`host.get`, `host.hold`, `host.registry`), a link's peer (`hold`). **It ends only when its holder ends it** (the + attempt ends, the call settles, `release()`, a getter gives up, the peer's link or origin closes, revocation, the + holding host closes), never because its provider is missing, went away or does not fit. +- **Placement by address** (round 1): a `local` type goes here; a `remote` type with a placement goes to the address + its placement (computed from the consumer's own contract) gives: here if that is this host's address, whether served + here or not, else over the route for that kind or the parent (design §3). A `remote` type **without placement** keeps + today's rule (round 2: Fable, Opus, Astra; no feature removed): here if served here, otherwise over the parent. A + request of that kind made before the host has a parent link waits here `not_served` and is **re-placed when the + host's first parent link connects** (round 3: Opus N5, Sol 3: a tab loads before it connects); on a host that never + connects (the root), it stays here. Opus would rather require a placement on every `remote()`; that removes a + supported declaration, so the draft keeps the fallback. A hold arriving for an address that is not this host's is relayed or + refused (`invalid/placement`) as today; a relay never captures a hold because it serves the same name. +- **Then the selected provider is checked** (served, version or accepted version and fingerprint, `authorize`, the + provider's own placement check). Version checks never look at a same-name registration elsewhere. +- **The publication gate:** a request made on this host, or arriving on its links, before the host's first + publication (§6) is not placed yet; it shows `loading` and is placed by the publication. The gate lasts only for the + synchronous setups, never for activation, so it cannot deadlock. +- **Outcomes of the check:** + + | The check finds | Request | + | --- | --- | + | a fitting provider | bound (to the object here, or to the hold over a path) | + | not published yet | unbound `unpublished`: `loading` | + | nobody serves the name at its address | unbound `not_served`: `failed { waiting }` | + | no fitting version, a fingerprint that differs at the same version, a placement the provider computes elsewhere (code skew) | unbound `version`: `failed { waiting }` | + | the registry is served, but its first `compose` (or the reader's version mapping) threw | unbound `compose`: `failed { waiting }` | + | the provider host is installing a revision and the hold does not fit its current code | unbound `installing` at the provider: `loading` | + | the path has no link (a drop, a route resolving) | on the path: `loading` | + | `authorize` refused, invalid params | `failed { broken, forbidden \| invalid }`, as today, never indexed | + + The unbound reasons that wait for a name (`unpublished`, `not_served`, `version`, `compose`) are indexed under it on + the host that decided them: the consumer's host for a local request; the provider's host for a hold that arrived + over a link (it already keeps such holds as refused: `remote/hold.ts`). `installing` is the provider host's own, + re-evaluated at its install's end; `no_link` lives on the path, where the next link holds it again. Indexes drop an + entry when its request ends; GC-tested. + +## 2. States + +`ref.state` keeps its four kinds; `FailureCause` gains one kind, `waiting`: + +```ts +type RefState = + | { kind: "loading" } + | { kind: "live" } + | { kind: "failed"; cause: FailureCause; retryAt: number | undefined } + | { kind: "ended"; error: ChordError }; + +type FailureCause = + | { kind: "transient"; error: ChordError } // the object's own effect failed (an open, a follow): backoff + | { kind: "broken"; error: ChordError } // the object's own code, a cycle, or authority: retry() or a reload + | { kind: "waiting"; error: ChordError }; // not the object's fault: no provider, no fitting version, a + // dependency failed; retryAt undefined, woken by an event +``` + +| The request is | `ref.state` | Calls | +| --- | --- | --- | +| bound, its object activating or parked with no failed blocker | `loading` | park, bounded | +| unbound `unpublished` or `installing`, or its path has no link | `loading` | park, bounded | +| unbound `not_served`, `version` or `compose` | `failed { waiting }` | reject at once | +| bound, its object failed | `failed` with the object's cause, as today | reject at once | +| bound, its object live | `live` | run | +| ended by its holder, revoked, or its host closed | `ended` | reject | + +**Owner:** `waiting` as a new cause kind (Opus, Fable; recommended: the cause says "not your fault, an event will wake +you" and a UI can say what is missing) or no new kind: `transient` with no `retryAt` (Astra; today's wire already +carries a `transient` cause without `retryIn`, `remote-object.ts`), or `loading` with a diagnostic (Sol; parks every +call on a missing plugin for its full `readyTimeout`). + +## 3. Dependents: parked until their dependencies resolve + +A **dependency** is what an attempt declares and needs before it goes live: `self.get` (local or remote), +`self.registry`, keyed pins (`at(key)` on another object's keyed member). `self.follow` is the attempt's own step (its +attach runs Durable's code), not a dependency; a follower declares `retryOn` on the registry it reads the harness +through, or recovers by its own backoff. + +1. **Order, as today up to the check:** the factory runs; persisted singletons hydrate (the object's documents, kept + across attempts); the requests are made. The attempt is **parked** while any dependency is not `live` (hydrated): + + | Its dependencies | The dependent shows | + | --- | --- | + | any `loading`, none failed | `loading` | + | any `failed` | `failed { waiting, "waiting for X: " }`, `retryAt` undefined; the message keeps the nearest blocker and the root cause within the 500-character rule | + | one failed with `invalid/cycle` (a dependent off the ring) | `broken` with the same error: it would wait on a ring that never heals by itself | + | one `ended` (revoked, or its host closed) | `broken` with that error: the request cannot come back; `retry()` or a reload makes new requests (a guard: a dependency ends only with its holder's principal or host, so the dependent ends too; Fable) | + +2. **The dependency check is the activation boundary:** once every dependency is `live`, the attempt runs its own + steps (natives in order, follow attaches, member checks, derives), bounded by its own `readyTimeout` deadline, and + goes live. A dependency that leaves `live` after the check is treated as after live (R18): the attempt goes on. This + is a deliberate rule (Astra: a weakening of "live means every dependency is live"): an object can finish opening + after a dependency went away, and its calls through that reference meet its state. So a parked attempt holds no + natives, and an attempt that opened natives is never parked. +3. **No timer while parked:** no dependency deadline, no backoff. The attempt's deadline is paused while parked (as + eager migrations pause it) and resumes with the budget left. +4. **Wakes, one rule:** the dependent re-evaluates on every state change of any of its dependencies (live, loading, + failed, a new cause compared by value, ended; Opus: otherwise a ring of waiting attempts passes its causes round and + round), updating what it shows; it leaves parking when all are live. The factory never re-runs + while parked. Wakes are coalesced per object and run after the batch that caused them; a change during the + attempt's own steps or its teardown is remembered. +5. **Kept while parked:** its documents (the object's), its declarations and requests, its seal token; no natives. + Ended with the object (its last holder; a host-lifetime object when its plugin is removed or the host closes). +6. **`broken` only for the object's own code** (a throwing factory, a derive throwing at activation, hydration, + `migrate`, a document fingerprint mismatch, an `open` throwing `broken(...)`), **cycles** (§7) and **authority** + (`forbidden`, an ended dependency). +7. **Retry passes on:** `ref.retry()` on a `failed { waiting }` object retries its `broken` and `transient` blockers, + hop by hop (the existing `retry` message over links), once per retry operation (an id dedupes diamonds); `waiting` + blockers pass it on; nothing else is touched. Retry of a `broken` or `transient` object starts a new attempt, as + today. +8. **`retryOn`** wakes failed objects as decided in slice 8 (a value call retries a `transient` or `broken` object); + while the object is parked `waiting`, a value call does nothing: its dependencies decide. +9. **Own-code reload of a parked object:** its plugin's reload replaces the parked attempt with the new factory's, + as one handover: the new requests are made and bound first, then the old ones released (round 2, Astra N4: an + unchanged dependency held only by this object must not end in between; today's `#activate` binds before + `#releasePrevious`). +10. **After live, R18 unchanged:** a dependency leaving `live` leaves its dependent live. + +## 4. Wakes: two events + +| Event | Re-evaluates | +| --- | --- | +| **a target changes state**: a dependency's object goes live, loading, fails, gets a new cause or ends; a hold's `bound`, `loading`, `failed` arrive, also after a re-hold on a new link | the requests bound to it, and their dependents (§3.4) | +| **a name becomes usable, an install ends, or the first parent link connects, on a host**: a publication (`load`'s setups, a reload's commit, an install's commit) serves a type, a registry, a version; a registry composes or maps a value at a version; an install ends without a commit (rejected) | the unbound requests indexed under those names; the provider's `installing` holds; at the first parent link, the +no-placement remote requests waiting here; a refused hold re-runs its start and sends `loading`, then `bound` or `failed` | + +No new message kind for wakes: `loading`, `bound`, `failed` and `retry` exist. The wire's `failed` carries the new cause +kind `waiting`; `loading` and a `waiting` `failed` carry the provider's attempt id for the probe (§7). + +## 5. Removal, versions, rollout + +- **A reload that removes a type:** its objects end as today (natives close, reason `end`; callers rejected at once; + handlers drain). Their requests are not ended: each becomes unbound `not_served` and binds again if a later commit + serves the name. + - **Locally:** a `Binding` whose target ended by removal becomes unbound and accepts a second `bind`: member sources + re-bind by name (one frame from the last value, as on a restart), keyed pins re-pin, parked calls flush or + reject, presence re-lists. The one new piece of machinery. + - **Over a link:** a `failed` on a bound hold unbinds it (Fable: one message, one rule; the consumer applies it as + `#startOver` plus `failed`); the provider detaches the hold's sources and drops its unsent pending ops and queued + `bound`; the ack slots of `bound`/`state` messages already sent stay counted until their `ack` or the link's loss; + calls in flight on it are rejected and free their slots once. +- **A reload that stops accepting a version** unbinds only the requests on that version (`version`); the object stays + (its own plugin's reload still restarts it if it has natives, §21). +- **A type moved to another host kind:** the request carries the address its own consumer code computed; the old host + answers `not_served`; the consumer's own reload makes a new request. No redirect. +- **A consumer whose code moves to a new version** releases its old request and makes a new one (today). +- **Rollout, without revision fields:** each host knows only its own install. A provider host that installs keeps + holds that do not fit its current code `installing` (`loading`) and re-evaluates them when the install ends, by its + commit or by its rejection. A provider host that is queued but not installing yet answers a mismatch `failed + { waiting, version }` until its install commits and wakes it: a visible blip of at most one install, accepted (Opus). + An install never waits for activations (a reverse-order rollout would deadlock); a host whose own install was + rejected fails its own requests at once with the reason (design §21). The loader keeps its own duties (the desired + code set, digests, serialized installs). +- **Old calls across a reload** follow §21 unchanged: a pure swap lets them finish on old handles; a native restart + drains them first; a removal rejects their callers at once and drains the handlers. Calls are never resent. + +## 6. Startup: publication, then first outcomes + +1. **Publication:** `load` runs every facet's setup, then publishes all at once in one batch (types served, + registries served and composed once, contributions in their tables), then wakes (§4) after that batch. Nothing + activates inside it. +2. **No gate beyond it:** holds and `host.hold` wait only for the publication, never for the barrier. This ends the + startup deadlock: W's hold for C binds as soon as S's setups published; B goes live, then A. +3. **The barrier** (**Owner:** (a) or (b); recommended (a), three reviewers of four): + - **(a):** `load` resolves when every host-lifetime object of this `load` has a first outcome: + `firstOutcome(o) = o.live || o.ownFailure || (o.parked && o.blockers.every((b) => !b.local || firstOutcome(b)))`, + where `local` means executing on this host (an object here, a keyed pin still loading here), not the contract's + kind. So `CodeMode → Harness` waits for the harness's outcome; a wait on another host or on a failed dependency + ends the first pass. One `firstOutcome` signal per object, separate from the readiness dependents wait for (Sol), + counted once, never reopened by retries. An object of the cohort that ends before its first outcome (its plugin + removed, the host closing) settles its entry as ended; `host.close` settles the waiting `load` (Astra R3). + - **What (a) promises:** every host-lifetime object on this host had its first try, and the contributions of those + that went live are in their tables. **Not:** every tool that will arrive (a parked or slow contributor, tools found + in `onLive`, MCP; Sol's probe `/tmp/barrier-sol-first-outcome.log`). What a first turn needs is p3's explicit + admission check. + - **(b):** `load` resolves after the publication; Chord adds `host.startup`, a watchable of the host-lifetime + objects' first outcomes with their causes; the app decides (Fable). The same tracker, awaited by the app. + - p2 today resumes its harness without waiting for all host objects (`apps/p2/src/hosts/worker.ts`), because its + worker objects that need server types are reachable only after `control.ready()`. + +## 7. Cycles + +On one host, cycles are found exactly as today: at `self.get`, following the waiting edges of activating and parked +attempts. Across hosts, a hold carries the activation chain it was made in (today), which finds every cycle whose +objects start one after another. What remains is the documented gap (design §6): objects on two hosts that start at +the same moment and need each other. The owner decided on 2026-10-24 (slice 4b review decisions (2)) that this is a +documented limitation that times out and retries, with no probe message. Without dependency deadlines that timeout no +longer exists, so the gap needs one of two answers (**Owner**): + +- **(A) The 4b decision, narrowed.** One deadline stays: an attempt parked on a dependency held over a link whose target + attempt is known and still activating or parked fails `transient` after its `readyTimeout` and retries with backoff; + its next attempt usually starts after the other side and its chain finds the cycle. It has a cost that decides + against it: the target being *parked* is the common case of a chain across hosts waiting for a missing plugin (a + tab's view on a worker object that waits for an uninstalled type), and those dependents would time out and poll + again, as today, so "local = remote" would not hold across hosts. Narrowing it to targets that are `loading` (not + `failed { waiting }`) avoids that but misses a real ring: A and B on two hosts wait on each other, B also on a + missing X; B is `failed { waiting }`, so A is too; when X appears, both stay `failed { waiting }` on each other, with + no deadline and no chain to find it. And even the timeout does not find what it is meant to (round 6, Fable): an + object's chain is fixed when it is created and its instance id survives retries, so a retried attempt sends the same + hold with the same chain; two simultaneous starts time out and retry in lockstep and are never found. +- **(B) The probe (recommended)**, below: no deadline at all, one new message kind, the most intricate rule set in the + model. Five + review rounds found a counterexample in each of the first four drafts; draft 6's version answers all of them, but it + has not been reviewed again, and slice 7 would have to prove it with tests before the deadline could go. + +### (B) The probe + +Replaces the activation chain in holds. On one host the `self.get` walk stays (it is synchronous and exact: a ring +with no remote edge is found there, where (B1)'s timer does not run; Opus, round 7); the probe serves the rings that +cross a link. It replaces draft 1's bounded chain too +draft 1's bounded chain (a ring of 33 is never found with chains cut at 32; chains keep stale ancestry). Draft 5 replaces the +second round by a check along the recorded path (round 4, Sol: two rounds through different stale paths can report a +ring that never existed, `/tmp/model/sol-probe4.log`). + +1. **Attempts, edges, tokens.** Every attempt has its own id (uuidv7 per attempt: a retry or reload is a new + attempt). A **wait edge** runs from a parked attempt to the parked or activating attempt it waits for, through a + bound request (unbound waits and registry reads are never edges; they are not deadlocks). Each edge carries an **edge + token** (round 5, Astra, Sol): a number from a counter per attempt, never reused, replaced by a fresh one on any + interruption or change of that edge, so two requests to one object are two edges: a new target attempt (also a + local target's retry, without a re-bind), any `loading` on a bound or pending hold (a relay's upstream re-hold + included), an unbinding, a rebinding, a re-hold after a reconnect. An attempt is parked for one interval only (one + that passed its dependency check never parks again) and ids are never reused. A provider tells its consumers the attempt they wait for on + every non-live answer: `loading { hold, attempt }` and a `waiting` `failed { hold, cause, attempt }`. +2. **A search per edge change.** Whenever a parked attempt gets a new edge or a new token on one, it starts a + search: `probe { kind: "search", round, initiator, trail }` along each of its wait edges, `round` the next number of + a counter the attempt keeps, `trail` the list of `(attempt, edge token)` hops it crossed, starting with + the initiator's own. A ring closes at its last + edge, and that edge's source starts the search that finds it. Coalesced per batch. Locally a call on the target + attempt, forwarded through a queue (no recursion); over a link the message `probe { hold, to, ... }`, consumer to + provider, relayed by hold id with the origin. **Probes are honoured on parent and route links and dropped on child + links** (`accept`), whatever `trusted` says. +3. **Forwarding.** An attempt receiving a search drops it if `to` is not its current attempt or it is not parked. If + the initiator is itself, the search returned (4). If the attempt is already in the trail, it drops it (a ring that does not contain the initiator is + found by its own members' searches; Opus). Otherwise it forwards it along every wait edge, appending `(itself, the edge's + token)` to the trail, only if the round number is higher than the one it stored for that + initiator (older or equal rounds are dropped: the initiator superseded them), and stores it. The seen map holds one + number per initiator, up to 256 initiators, and goes with the attempt. +4. **The check.** A returned search is accepted while its initiating attempt is still parked, whatever round it + carries (round 7, Astra: a re-search must not invalidate an older search still under way); the check decides. The + initiator then sends `probe { kind: "check", round, trail }` along the recorded trail: the initiator first verifies its own hop; each hop passes it on only if it is still + that attempt, still parked, still has the recorded edge token, pointing at the next hop's attempt; otherwise the + check dies (also on a `to` mismatch). The check follows the trail by position and bypasses the seen + map. The initiator verifies its own hop again when the check returns. If the check returns, the initiator fails `broken invalid/cycle`, the path named for display from the trail, + and today's propagation breaks the other activating objects on the ring and the dependents off it. 5. **No silent loss.** Dedupe can drop the copy of a search that travelled the ring in favour of a copy whose path + later breaks (Astra, round 5); then the check dies and nothing is left to find the ring. Round 6 tried `dead` + notices back to the initiator; Sol and Astra showed that reliable notices need recovery state per search that + eviction may not drop and that a long-parked attempt would otherwise keep for every initiator that ever passed (a + leak), i.e. a full search lifecycle with cancellation. Two ways (**Owner**, part of decision 6): + - **(B1) Re-search with backoff (recommended).** An attempt parked on a remote attempt starts a fresh search again + after a backoff (1 s doubling to 60 s, reset by an edge change), whether it shows `loading` or `failed { waiting }`; + the timer stops when the attempt leaves parking. No `dead` messages, no recovery state, eviction + only costs work; a new search starts within one backoff period, so every stable ring that crosses a link is + eventually found (no wall-clock bound: delivery is only fair; a ring on one host is the walk's). The timer runs as long as the attempt stays parked on a remote attempt: a view on a worker object that + waits for a missing plugin re-searches every 60 s until a reload (Fable: cheap, but for as long as it waits). The timer sends probes only: it never + fails an object, re-holds or polls a provider. Cost: one probe per parked remote dependent per period, which + crosses the downstream wait graph and dies at its leaves (an unbound wait is not an edge). + - **(B2) Events only.** Each search is owned until it settles: branches report completion or abort back along their + trails, a dropped probe answers `dead { initiator, round }` with a reverse route, an edge records the searches it + carries until they settle, and an initiator that ends or starts a new round cancels its old one along the same + routes. Complete without a timer, bounded by the searches in flight, and the most intricate protocol in Chord; not + yet specified to the reviewers' satisfaction. + Rejected: enumerating every simple trail without dedupe (Sol's correctness reference): no ring copy is ever + suppressed, but work is exponential on chains of diamonds. +- **Why a reported ring existed:** every edge of the trail kept its token from the moment the search crossed it + to the moment the check crossed it; the search crossed all of them before the check started, so at the moment the + check started every edge of the ring existed at once. A token changes on every interruption and is never reused, so an edge cannot + vanish and come back unnoticed. Sol's counterexample fails at the check: `C1` is no longer `B`'s target. +- **Why every ring is found** (under an eventually stable wait graph and fair delivery; endless edge churn can keep + invalidating searches; with rule 5's (B1) or (B2) for a search suppressed behind a path that broke): a ring is closed by its last edge; its source starts a search when the edge appears; every + other edge exists then and every attempt on the ring waits transitively on the source, so none goes live before the + search and the check return. Rings of any length, local and remote mixed. On a DAG searches die at the leaves; on a + diamond the second arrival is dropped. +- **What it does not promise:** (1) a ring that existed when the check started may be broken up by a concurrent reload, + retry, unbinding or revocation just after; it is still reported (it did exist). Its broken objects recover as every + broken object does: `ref.retry()` or a reload of their plugin. (2) Linear work under every schedule: with more than + 256 initiators passing one attempt at once, an evicted entry lets a search be forwarded again, so work can grow with + the number of simple paths (never endlessly: a search never revisits an attempt in its trail). (3) Constant message size: a probe carries its trail, so its size grows with the length of the + path it crossed (a returning search's trail is as long as its ring); no cap cuts it, since the trail is the proof. +- **Cost:** one search per edge change, crossing each downstream edge once, one check per returned search; with (B1), + one more search per backoff period for every attempt parked on a remote attempt, also while an acyclic chain waits + for a missing plugin, plus the trail bytes. The bounded memory is the seen map's; probes in flight are bounded by + the searches started. +- **Transition:** today's dependency deadline for activations waiting across links stays until the probe's tests pass + in slice 7 (long rings, diamonds, a ring closed after a reconnect with unchanged attempt ids, Sol's phantom schedule, + 300 concurrent initiators, edges added and removed, reloads, relays, forged probes); then it goes in the same slice. + +## 8. Getters, holds, calls: one rule + +- **`host.hold`** returns a reference at once; only invalid params throw (also before `load`). A missing local type + gives `failed { waiting }` after the publication, `loading` before it. +- **`host.get`, `ctx.get` and `host.registry`:** resolve when live; wait while `loading`, or `failed { transient }` + (its own retry is under way), up to their `readyTimeout`; reject at once on `failed { waiting }` or `{ broken }`. On + success, rejection or timeout their temporary wait goes; `host.registry` returns the `RegistryValue` with its + ordinary watch ownership (no `release()`). **Owner:** recommended (nobody waits 60 s for a plugin that is not + there; code that wants to wait holds instead), or wait through `waiting` too. +- **Calls** on a `loading` reference park, each bounded by the reference's `readyTimeout` from when it was made; on a + `failed` one they reject at once with the cause; sent calls are never resent. + +## 9. Registries + +- The same resolver on one host: a reader's request for R at its version is bound while R is served and has a usable + value at that version (composed; mapped by `accepts[old].value` for an older reader), otherwise unbound + `not_served`, `version` or `compose`, or `loading` before the publication. Woken at every transition to usable + (`#valued`'s first-value latch goes). S2's `broken` and S3's backoff go. +- **Version skew within one host:** a reload of the registry's plugin alone leaves other plugins' contributors and + readers on their bundled versions; `accepts[old].contribution` and `.value` keep them working, unsupported + contributions are left out and reported, unsupported readers are `waiting` (`version`). +- A registry waits for a value, never for a particular contribution (an empty table composes). After live, a later + `compose` failure changes nothing for the reader: `.value` keeps the last good composition, **but only while every + contribution it was built from is still in the table** (round 2, Astra N2): when one of them leaves (its + contributor ended or was reloaded), the last good value is dropped and `.value` throws until `compose` succeeds, so + it never hands out a native whose contributor ended. `compose` represents a missing contributor as a value. +- Registries stay host-local (contributions are in-process values). + +## 10. Keep-alive, presence, revocation, jobs + +- A request over a link counts once in `host.activity.references` from its arrival to its end, bound or unbound; + presence (`host.references`) lists only bound references. +- Revocation ends the requests of a revoked principal, bound and unbound, at every hop; nothing revives them. +- `gone` stays only for revocation (a provider host closing closes its links and sends nothing; its consumers see a + drop, `loading`, and hold again on the next link). +- Persisted job runs of a parked host-lifetime object are loaded with its documents; `run` and `finish` start only + once it is live; the run count keeps the object as today. + +## 11. Timeouts after the model + +| Timeout or backoff | After | Why | +| --- | --- | --- | +| `readyTimeout` of an attempt | only its own steps; paused while parked | a slow dependency is bounded where it runs | +| `readyTimeout` of `host.get`, `ctx.get`, `host.registry` | kept | a caller gets an answer | +| `readyTimeout` of a parked call | kept | an action must not run long after it was asked for | +| identity backoff 1-60 s | only for the object's own `transient` failures | nothing else says the disk or network is back | +| dependency backoff, polling of refused holds | removed | recovery is an event | +| dependency deadline across links | (A): only for a parked attempt whose remote target attempt stays activating or parked; (B): until the probe's tests pass, then removed | the simultaneous-cycle gap | +| probe re-search backoff (B1) | new, only with (B1): 1 s doubling to 60 s while parked on a remote attempt | finds a ring whose search was suppressed; sends probes only, never fails or re-holds | +| route backoff, heartbeat, `closeTimeout`, routine and `finish` backoffs, `linger` | kept | not dependency waits | +| rollout install timeout (not built) | a watchdog on the install only | never a deadline for dependencies | + +## 12. What changes in the design + +R18 (dependency failures `waiting`, no backoff, the deadline paused while parked, the dependency check as boundary, +wakes on every change, an ended dependency breaks), S2 and S3 (`waiting`, woken at every usable value), slice 8's +last-good rule (dropped when a contribution leaves), §3's placement table (by address; the publication gate), §6 (the +simultaneous-cycle gap: (A) one narrowed deadline or (B) the probe), §7 (`host.hold`, getters, calls), §21 (no `gone` on +removal; a dropped version unbinds only its requests; installs never await activations; rollout without revision +fields; the reload handover), the start barrier (R17, W6), `FailureCause` (+ `waiting`), `loading`/`failed` with the +attempt id, one message kind (`probe`, tagged `search` or `check`). Unchanged: what an object's own code reload does (§21), R18 after live, +`retryOn` outside parking, `self.follow`'s R4, the call window and flow control, revocation, no client providers. + +## 13. Open for the owner + +1. The barrier: (a) or (b) (§6.3). +2. Getters on `waiting`: reject at once (recommended) or wait up to `readyTimeout` (§8). +3. `waiting` as a new cause kind (recommended) or not (§2). +4. A pure swap whose new factory declares a dependency that is not there: the object parks (its gate closes, holders + see `loading` or `failed { waiting }`, calls park or reject; recommended) or stays on the old code until it + resolves (old and new code would overlap without bound, against §21's "a host is at R or R−1"). +5. **The last good value** (§9): kept only while every contribution it was built from is still in the table (draft; + Astra), or kept as slice 8 decided, with natives in a registry looked up at use through their owner (Sol). +6. **Cycles across hosts** (§7): (B) the probe, with (B1) a re-search backoff for probes only (recommended) or (B2) a + full search lifecycle without any timer; or (A) the slice 4b decision narrowed to one deadline, which polls chains + across hosts again, misses a ring of `waiting` attempts, and retries in lockstep without finding the cycle it is for. + diff --git a/packages/chord/docs/next/dependencies/index.html b/packages/chord/docs/next/dependencies/index.html new file mode 100644 index 000000000..3bc8daa6f --- /dev/null +++ b/packages/chord/docs/next/dependencies/index.html @@ -0,0 +1,2647 @@ + + +Parked Until Resolved: one model for dependencies in Chord v2 + + + + +
+
+
A design proposal · Chord v2 · slice 7 · 2026-10-29
+

Parked
Until Resolved

+

What happens when what an object needs is not there yet, is somewhere else, has gone away or does not fit, told through the scenarios that decide it

+
5 → 1answers today to "the provider is not there"
0deadlines on waiting for a dependency
300+scenarios played through by four reviewers
6decisions left for the owner
+

A proposal, not a decision. It states the dependency model the redesign session drafted on the night of +2026-10-28/29 from the owner's direction ("park until resolved"; "no difference between local and remote +resolution: a link going is like a plugin unloading") and five rounds of review by Astra, Fable, Opus and Sol, who +played more than 300 scenarios from p2's plugins and their own against it. The working document is +/tmp/model/model.md; the reviews are in /tmp/dep-graph-*.md, /tmp/barrier-*.md and /tmp/model/review*-*.md. +Code references are to pi main at 358f7d46a (packages/chord/src/next). Notes marked review record a point +the reviewers found or still disagree on; Owner marks a decision that is his.

+
+
+ + +
+

Before we begin

+

It is a Tuesday morning and Mario opens a session on his phone. The server starts a worker for it. The worker +connects to the server, loads its plugins and opens the agent's harness. One of the plugins, code mode, needs the +harness before it can contribute its tools. Another, search, needs a feed that lives on the server. A third, sharing, +was removed from the worker's plugin set last night, but the notes view still asks for it. At the same moment the +browser tab holds the session's notes, which live on the worker that is still starting.

+

Every one of these is the same question asked in different places: what I need is not there yet; what now? Chord +v2 answers it today in five different ways, depending on how the dependency was declared and where its provider +lives. A missing type on the same host breaks the object for good; the same type missing on another host makes it +retry every few seconds forever; a missing registry waits for a value; a reload that removes a type ends every +reference to it. And when two hosts start at once and need each other's objects in a chain that has no cycle at all, +both wait until a 60-second deadline breaks the knot.

+

The owner's direction was short: an object should be parked until its dependencies resolve, and there should be +no difference between a dependency on this host and one on another: a link going away is like the provider's plugin +being unloaded, and a link coming back is like it being loaded. This book works out what that means, rule by rule, +and plays it through the scenarios that decide whether it holds.

+

Eight things to know

+
    +
  1. A request outlives its provider. A reference stands for a request (type, params, principal, version). It ends +only when its holder ends it, never because the provider is missing, went away or does not fit.
  2. +
  3. One resolver for local and remote. The address decides where a request goes; the provider found there is +checked; a missing provider here and a missing provider on another host give the same answer.
  4. +
  5. Two kinds of not-yet. Something under way (activating, reconnecting, installing, not published yet) shows as +loading; nothing under way (not served, no fitting version, a dependency failed) shows as failed with the new +cause waiting.
  6. +
  7. Dependents park. An object whose dependencies are not all live waits with no timer and no backoff, and is +woken by events: a dependency changing state, or a name becoming usable on a host.
  8. +
  9. broken means the object's own fault. Only its own code and cycles break an object; whatever it waits for is +waiting.
  10. +
  11. Startup needs one short gate, not a barrier. Requests wait only for the host's facets to be published, which +is synchronous; that ends the startup deadlock. Whether load then also waits for first outcomes is the owner's +call.
  12. +
  13. One gap in cycle detection needs a choice. Objects on two hosts that start at the same moment and need each +other are caught today only by a deadline. A probe can walk the waiting edges and find the ring without failing +anything (recommended); a narrowed deadline is simpler but polls, misses a ring, or never finds the cycle at all.
  14. +
  15. Only callers and own work keep clocks. Getters, parked calls and an object's own steps keep their deadlines; +waiting for a dependency has none.
  16. +
+

Words used throughout

+

A host is one participant (the server, a session worker, a browser tab). A provider is what serves a type or +a registry on a host: a facet registered it with host.serve. A request is what a reference stands for: a type +or registry, its params, the principal it acts as and the consumer's contract version. Its holder made it and +owns it: an object's attempt (self.get, self.registry), a call (ctx.get), outside code (host.get, +host.hold) or a peer host (a hold arriving over a link). A request is bound when it points at an object (here) +or a hold (over a link), and unbound with a reason otherwise. An attempt is one try at activating an object: +its factory, its requests, its natives. A dependency is a request an attempt needs live before it may go live. An +attempt is parked while one of its dependencies is not live. A publication is the moment a host makes its +facets' declarations visible: the setups of load, the commit of a reload, the commit of an install. A wake is an +event that makes the resolver look at a request again.

+

The running example is a small p3: a server S with the session catalog and the search feed; a worker W for +session S1 with the harness, code mode, the notes store and the search indexer; and a browser tab T that shows +the notes. The plugins are core, models, codemode, notes, search and sharing.

+
+ +
I

Five Answers to One Question

What Chord v2 does today when a dependency is missing, and why the answers disagree.

+

Chord v2 was built slice by slice, and each slice answered "the provider is not there" for the case in front of it. +Slice 1 made self.get of a type nobody serves throw, because a typo in a type name should fail loudly. Slice 3b made +remote holds survive link drops, so a missing provider on another host became a refused hold that the consumer retries. +Slice 8 made self.registry a dependency that waits for a value. Each answer was reasonable alone. Together they +give five outcomes for one situation.

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Declared asThe provider is missingToday
self.get(LocalType)no facet serves itself.get throws, so the factory throws: the object is broken
self.get(RemoteType)the worker does not serve itthe hold is refused broken; the dependent is transient and retried every 1 to 60 seconds, forever
self.registry(R)no plugin serves Rtransient, woken at R's first value only
self.registry(R), the reader newer than the providerbroken
a reload removes the typeevery holder gets gone: ended for good
+

The differences are not about the situation. Whether the notes plugin is missing on this host or on the worker, the +notes view cannot work until someone installs it. Yet in one case the view is broken until a reload, in the other it +polls the worker every minute for the rest of the session, and the reload that finally brings the plugin back cannot +revive a reference that got gone.

+

The deadlock without a cycle

+

The startup barrier has a worse problem. Sol reproduced it against the current code (/tmp/dep-graph-sol-barrier.log).

+
server Sload: waits for Aworker Wload: waits for BStoresetups: serve C; start eager AA needs B: hold Bsetups: serve B; start eager BStoreBStore needs C: hold CHold.start waits for S.pendingLoadS's load waits for A, A for B, B for BStore, BStore for C, C's hold for S's load60 S LATERA's activation deadline fires, A failsC binds, BStore and B go live, A is retried
Figure 1. The startup deadlock today. Nothing in the object graph is cyclic; the two barriers block each other.
+

C has no dependencies and is registered the moment S's setups ran. Its hold waits anyway, because incoming holds +wait for pendingLoad, and pendingLoad is the whole of load: the setups and the barrier that waits for every +host-lifetime object, A included. Two jobs share one promise. Holds only need the first ("are the types +registered?"); the second ("has every plugin had its first try?") is for the app. Mixing them makes an acyclic chain +across two hosts wait until a timeout fires.

+

Timers standing in for events

+

Both problems have the same root. Where Chord does not know when something will change, it polls or waits for a +deadline: the identity backoff retries a dependent whose remote provider refused it; the activation deadline breaks the +startup knot; the same deadline is the only answer to two objects on two hosts that start at the same moment and need +each other, a cycle the activation chains cannot see (design §6, "Known limitation").

+

A timer is the right tool when nothing else can tell Chord that the world changed: a disk that may be back, a worker +process that may start this time, a silent transport. It is the wrong tool when an event exists or can exist. A +missing plugin comes back through a reload, which is an event on the host that serves it. A dependency going live is +an event. A link opening is an event. The model below uses those events and keeps timers only where no event can +exist.

+ + +
II

Requests That Wait

A reference is a request with a holder; it outlives every provider it meets.

+

The first change is a separation the code makes only halfway today: between what a reference asks for and what +currently answers it. When the notes view on W does self.get(NoteStore, {}), it asks for "the NoteStore of this +host, as the host, at contract version 3". Which object answers is a second question, and the answer can change: no +object yet, an object activating, a live one, a failed one, none again after a reload, a new one after the next.

+

A request and its holder

+

A request is the asking part: type (or registry), params, principal and the consumer's contract version. Every +request has exactly one holder, which made it and is the only one that ends it.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
HolderMade byEnds when
an attemptself.get, self.registry, a keyed pin ref.notes.at(key)the attempt ends (its object ended, or a reload replaced it)
a callctx.getthe call settles
outside codehost.get, host.hold, host.registryrelease(), or the getter gives up
a peer hosta hold arriving over a linkthe peer releases it, or its link or origin closes
any of themrevocation of its principal; its own host closes
+

That list is the whole of how a request ends. A request never ends because its provider is missing, went away or +does not fit. Today it does in two places: a self.get of a type nobody serves throws before the request exists, +and a reload that removes a type sends gone. Both go.

+

The address decides where

+

A request goes to one place, and the address decides which. For a local type, the place is this +host. For a remote type, the consumer computes the address from the params with its own contract's +placement function (design §3): Notes { session: "S1" } is placed on Session { id: "S1" }. If that is this host's +address, the request goes here; otherwise over the route for that kind, or over the parent link.

+

Today the code checks something else first: whether the type is registered here. A type placed on this host but not +registered yet is sent over the parent link, which is wrong twice. Sol's example: the server has address Server, +the catalog type is placed on Server, and no facet serves it yet. Today's Placement.reference sends the server's +own request over a parent link the server never had. And when the catalog plugin is installed later, nothing wakes +that request, because it was never registered as waiting here.

+
WHERETHENthe request's addressfrom the consumer's own placementfunctionherea local type, or this host's addressa route or the parentevery other addresscheck the provider hereserved? fitting version? authorize,own placementhold over the paththe provider host checks it the sameway
Figure 2. Placement by address. The destination follows from the address alone; only then is the provider found there checked.
+

The rule, from the reviews (Sol B1, Astra F1): placement never looks at what is registered when the type names an +address. A request placed here whose provider is missing is unbound here, waiting for the name. A version check +compares the consumer with the provider actually selected, never with a same-name registration on the consumer's own +host (today #assertRoute does that before placement, so worker W1 serving Notes v1 rejects a Notes v2 request +placed on W2, which serves v2).

+

One case keeps today's rule: a remote type declared without placement (allowed by remote(), used in tests and +for server types a tab reaches through its parent). It goes here if it is served here, otherwise over the parent. A +request of that kind made before the host has a parent link (a tab loads before it connects) waits here as not served +and is placed again when the host's first parent link connects (Opus, Sol); on a host that never connects, the root, +it stays here. Opus would rather require a placement on every remote type, which is simpler; it removes a supported +declaration, and the owner asked for no feature to go, so the draft keeps the fallback.

+

What the check finds

+

Once placed, the provider found there is checked: served, a fitting version (the current one or an accepted older one, +with the fingerprint computed from the provider's own history), authorize, and the provider's own placement check. +The request is then bound (to an object here, or to a hold over a path), or unbound with a reason:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
The check findsThe requestShows
a fitting providerboundits object's state
this host has not published its facets yetunbound unpublishedloading
nobody serves the name at its addressunbound not_servedfailed { waiting }
no fitting version, a fingerprint that differs at the same version, a placement the provider computes elsewhereunbound versionfailed { waiting }
the registry is served, but its first compose, or the reader's version mapping, threwunbound composefailed { waiting }
the provider host is installing, and the hold does not fit its current codeunbound installing, at the providerloading
the path has no link (a drop, a route resolving)waits on the pathloading
authorize refused, or the params are invalidrefusedfailed { broken }, as today
+

The reasons that wait for a name (unpublished, not_served, version, compose) are kept in an index under that +name, on the host that decided them: the consumer's host for a local request, the provider's host for a hold that +arrived over a link. The provider already keeps such holds today as refused (remote/hold.ts), and retry re-runs +them. installing belongs to the provider host's own install; no_link lives on the path, where the next link holds +the request again, as today. Every index drops an entry when its request ends, so a released getter, a closed tab or a +revoked guest leaves nothing behind. A refused request (no authority, bad params) is never indexed: a later +publication cannot grant authority.

+

The publication gate

+

There is one short gate. Before a host has run its facets' setups, it does not know what it serves, so it cannot tell +"not served" from "not registered yet". A worker connects to its parent before it loads (W6), so holds can arrive in +that window, and p2's web shell holds references before its own load resolved.

+

So a request made on a host, or arriving on its links, before the host's first publication is not placed at all: +it is unbound unpublished and shows loading. The publication places it. Without this gate the request would be +placed by a half-empty registration table (Fable's trace: W1's own Notes sent over the parent, relayed back by the +server, forever), and a tab would see "not served" during every worker start (Opus's trace).

+

The gate is not the barrier. It lasts only until the setups ran, and the setups are synchronous code at the start of +load: they register types and registries and contribute, and wait for nothing. That is the difference that ends +the startup deadlock (chapter VI).

+
+ +
III

Loading or Waiting

What a reference shows while it is not live, and why a missing plugin is an answer rather than a spinner.

+

ref.state keeps its four kinds: loading, live, failed, ended. What changes is one new kind of failure cause.

+
type RefState =
+  | { kind: "loading" }
+  | { kind: "live" }
+  | { kind: "failed"; cause: FailureCause; retryAt: number | undefined }
+  | { kind: "ended"; error: ChordError };
+
+type FailureCause =
+  | { kind: "transient"; error: ChordError }   // the object's own effect failed (an open, a follow): backoff
+  | { kind: "broken"; error: ChordError }      // the object's own code, a cycle, or authority: retry() or a reload
+  | { kind: "waiting"; error: ChordError };    // not the object's fault: woken by an event, never by a timer
+

The mapping

+

Whether a request that is not live shows loading or failed { waiting } follows from what it waits for:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
The request isref.stateCalls on it
bound, its object activating, or parked with no failed blockerloadingpark, bounded
unbound unpublished or installing, or its path has no linkloadingpark, bounded
unbound not_served, version or composefailed { waiting }, no retryAtreject at once with the cause
bound, its object failedfailed with the object's own cause and retryAt, as todayreject at once
bound, its object liveliverun
ended by its holder, revoked, or its host closedendedreject
+

The intuition is "is something under way?": an activation, a reconnect, an install, a load. When something is under +way, the reference waits and calls park; when nothing is, the reference says what is missing and calls fail fast. +The reviewers pointed out that "under way" is not something Chord can observe in general (a parent reconnect is the +app's, a resolve may hang), so the table is the rule and the intuition only explains it.

+ +

The owner's point was that a link going away is like the provider's plugin unloading. For the request they are the +same: it stays, it keeps its last values readable, it binds again when a provider is reachable, and its holder +notices nothing but the state. They differ in what is under way.

+
The notes view on T, once through a link drop and once through an unload on W

Live

T's notes view holds Notes { S1 } on W through the server. ref.state is live; the view renders the notes.

+

The link drops

W's link to the server closes (a crash, a network change). The request on T stays; the path has no link: unbound +no_link, ref.state loading. The last notes stay readable. A call made now parks, bounded by readyTimeout. +Calls already sent fail and are never resent: their outcome is unknown.

+

The link comes back

The route resolves again, the request is held anew on the new link, bound arrives with a fresh snapshot. live. +Parked calls go out in order.

+

The plugin is removed on W

A reload on W removes the notes plugin. The provider's object ends as today: its natives close, its callers are +rejected at once, its handlers drain. The request on T does not end: W sends failed { waiting, not_served } on +the bound hold, which unbinds it. ref.state is failed { waiting }; the last notes stay readable; new calls fail at once with "notes is +not served on Session S1".

+

The plugin comes back

A later reload on W serves Notes again. The publication re-runs W's refused holds for that name: loading, then +bound. T's view is live again without having done anything.

+
+

One difference must stay: calls already sent. When a link drops, a sent call's outcome is unknown, so it fails and +is not resent. When a plugin unloads on the same host, its handlers are drained on the old code (design §21). Location +decides nothing else.

+
Review: a third cause kind, or not

Opus and Fable recommend waiting as above. Astra would keep two kinds and mark event-waited failures as transient +with no retryAt (today's wire already carries a transient cause without retryIn, remote/remote-object.ts), so +no kind is added; the cost is that transient then means two things, a backoff under way or an event awaited. Sol would show loading with a diagnostic; that parks every call on a missing plugin for 60 +seconds and spins forever in the UI. The draft recommends waiting. Owner decision 3.

+
+
+ +
IV

Parked

An object whose dependencies are not all live waits, with no timer, and goes on the moment they are.

+

The owner's instinct was that an object is parked until its dependencies all cleanly resolve. This chapter makes that +exact: what a dependency is, what the parked object shows, what it keeps, and where parking ends.

+

What counts as a dependency

+

A dependency is a request an attempt makes in its factory and needs live before the object may go live:

+
    +
  • self.get(Type, params), local or remote: one rule for both;
  • +
  • self.registry(R): a registry at the reader's version (chapter X);
  • +
  • a keyed pin, ref.notes.at(key) on another object's keyed member: the key must be loaded.
  • +
+

Two things are not dependencies. self.follow(attach) is the attempt's own step: its attach runs Durable's code +and may fail like an open, with the object's own transient failure and backoff (a follower that wants to recover +the moment a new harness appears declares self.retryOn on the registry it reads the harness through). And a +ctx.get in a method is not part of the object's activation at all: it is the call's request.

+

The dependency check

+

An attempt runs its factory, hydrates its persisted singletons (they are the object's documents, kept across +attempts, as today), makes its requests, and is parked while any dependency is not live (and hydrated). What it +shows follows its dependencies:

+ + + + + + + + + + + + + + + + + + + + + + + +
Its dependenciesThe parked object shows
any loading, none failedloading
any failedfailed { waiting, "waiting for X: <X's cause>" }, no retryAt; the message keeps the nearest blocker and the root cause within the 500-character rule
one failed with invalid/cycle (a dependent off the ring)broken with the same error: it would wait on a ring that never heals by itself
one ended (revoked, or its host closed)broken with that error: the request cannot come back; retry() or a reload makes new requests
+

When every dependency is live, the attempt passes the dependency check and runs its own steps: its natives in +declaration order, its follow attaches, the member checks, its derives. Those are +bounded by its own readyTimeout deadline, as today. Then it goes live.

+

The dependency check is the boundary between waiting and working (Astra's F3). Before it, the attempt holds no +natives; after it, a dependency that leaves live is treated as after live (R18): the attempt goes on, and its calls +through that reference meet the reference's state. So a parked attempt never holds a native, and an attempt that +opened a native is never parked. This is a deliberate rule, and Astra called it what it is: a weakening of "live +means every dependency is live". An object can finish opening after a dependency went away. Without that boundary a contributor unloading while the attempt opens its natives +would leave it half open and parked, or force a seal, a drain and a fresh attempt for a dependency the object may not +even use at once.

+
CodeModehost-lifetime, localHarnesshost-lifetime, localHarnessModelsregistryself.get(Harness): loadingparked: shows loading, no timerself.registry(HarnessModels): usableown steps: open DurableHarness live: a state changedependency check passes; open the sandboxlive; contribute tools
Figure 3. Code mode on W parks on the harness and goes on the moment it is live.
+

No clock while parked

+

A parked attempt has no dependency deadline and no backoff. Its readyTimeout deadline is paused while it is +parked, as eager migrations already pause it (attempt.ts, deadline.pause()), and resumes with the budget left when +the attempt passes the check. Today a dependency that does not go live within 60 seconds fails its dependent +transient, which then retries with backoff from 1 to 60 seconds. Both go: the dependent is woken when the dependency +changes, and the time a dependency takes is bounded where it runs, by that dependency's own steps.

+

What a parked attempt keeps

+

It keeps its documents (the object's), its declarations and its requests (R18's holding today), and its seal token. +It keeps no native. It ends with its object: when its last holder releases it, or, for a host-lifetime object, when +its plugin is removed or the host closes. A parked object +with params that nobody holds any more ends and releases its requests, so a whole parked chain goes when its top is +released.

+

broken is the object's own fault

+

After the model, broken means exactly three things: the object's own code failed (a throwing factory, a derive that +throws at activation, a hydration or migrate that fails, a document fingerprint mismatch, an open that throws +broken(...)), the object is on a cycle, or it lost its authority (forbidden, a dependency that ended). Everything an object waits for, including a provider that is itself +broken, makes it waiting. The difference matters for recovery: a broken object needs ref.retry() or a reload of its +own plugin; a waiting one recovers by itself when what it waits for recovers.

+

Retry passes on

+

A Retry button on a waiting object should do something. Opus's trace: the tab's Panel waits for the server's Dash, +which waits for Catalog, which is broken by a factory bug that a configuration fix would heal at the next attempt. +The user sees "waiting for Dash: waiting for Catalog: …" and presses Retry. Today Panel would re-check, find Dash +still failed and do nothing.

+

So ref.retry() on a failed { waiting } object retries its failed blockers, hop by hop, with the existing retry +message over links: it retries the blockers that are broken or transient (the only ones a retry can change), +passes through the waiting ones, and leaves the rest alone. Each retry carries an operation id, so a diamond of +dependencies retries a shared leaf once. Retry of a broken or transient object starts a new attempt, as today. A +retryOn value call still retries a failed object as slice 8 decided; while the object is parked waiting, it does +nothing: its dependencies decide.

+

A reload of the parked object itself

+

The parked attempt's factory never runs again while it is parked, with one exception: a reload of the object's own +plugin replaces the parked attempt with the new factory's attempt, as one handover: the new requests are made and +bound first, then the old ones released (Astra: a dependency held only by this object, unchanged by the reload, must +not end in between; today's #activate already binds before #releasePrevious). If the new code no longer declares +the missing dependency, the object can go live at once.

+

After live, nothing changes

+

R18's second half stays: once an object is live, a dependency that leaves live (a link drop, a removal) leaves the +object live. Its calls that use the dependency get the dependency's state: they park while it is loading and reject +while it is failed. Reactivating every live consumer on every dependency loss would turn one worker disconnect into +closed natives, withdrawn contributions and aborted jobs on the server; the reviewers agreed it should not.

+
+ +
V

What Wakes Whom

Two kinds of event replace every poll and every dependency deadline.

+

A model without timers lives or dies by its wakes: a wake that is lost leaves an object parked forever. So the wakes +are kept to two kinds of event, each with one owner, and every unbound request and every parked attempt is +subscribed to exactly the events that can change it.

+ + + + + + + + + + + + + + + + + + +
EventWho is toldExists today?
a target changes state: an object goes live, loading or failed, or gets a new cause; a hold receives bound, loading or failed, also after a re-hold on a new linkthe requests bound to it, and the attempts that depend on those requestspartly: dependencyLive covers only "goes live"
a name becomes usable on a host: a publication (load's setups, a reload's commit, an install's commit) serves a type, a registry or a version; a registry composes or maps a value at a versionthe unbound requests indexed under that name on that host; a refused hold re-runs its start and sends loading, then bound or failedpartly: refused holds and retry exist; the re-run at publication is new; a registry wakes only at its first value
+

Every change, not only "live"

+

Draft 1 woke a parked attempt only when a dependency went live. Astra, Fable and Sol found the same hole: a dependency +that moves from loading to failed, or from one cause to another, leaves its dependent showing a stale state, and a +dependent that should now show failed { waiting } keeps parking calls. So a parked attempt re-evaluates on every +state change of any of its dependencies and updates what it shows. It leaves parking only when all of them are live. +Its factory never re-runs on a wake.

+

Three details keep this correct and cheap:

+
    +
  • Coalesced, after the batch. Wakes run after the batch that caused them (a publication, a commit, an inbound +bound), once per object, so a commit that changes three dependencies wakes their dependent once, and no factory +runs inside another owner's batch.
  • +
  • Remembered. A change that arrives while the attempt runs its own steps or tears down is remembered and applied +after (today's R18 wake memory, generalized).
  • +
  • Indexed by name. A publication touches only the unbound requests waiting for the names it serves. A reload of +notes never looks at requests waiting for search.
  • +
+ +

The wake across hosts needs no new message kind. Today a provider keeps a hold it could not serve as refused, and a +retry from the consumer re-runs its start. The model adds one thing: the provider re-runs its refused holds itself, +at its own publication, for the names that publication serves.

+
worker WIndexer, parkedserver Ssearch plugin missinghold SearchFeedfailed { waiting, not_served }Indexer shows failed { waiting }; no timer, no retriesA RELOAD ON S ADDS THE SEARCH PLUGINpublication serves SearchFeed; re-run refused holds for itloadingbound (snapshot)a target changed state: Indexer re-evaluates, passes the check, goes live
Figure 4. The search indexer on W waits for the server's feed, which a reload on S adds.
+

Two existing messages change: failed carries the new cause kind waiting, and loading and a waiting failed +carry the provider's attempt id for the cycle probe (chapter VIII). The pinned list of causes and its test change with +them, and PROTOCOL_VERSION stays 1 until p3 ships.

+

Registries

+

A registry reader is woken at every transition of the registry's value at its version to usable: when the registry +is served, when it composes for the first time, when an older reader's mapping succeeds again after it failed. Today +RegistryValueSource wakes only at the very first good value (#valued), which a registry whose provider is removed +and added again would never pass a second time.

+
+ +
VI

Starting Up

The barrier did two jobs with one promise; separated, the deadlock disappears.

+

load does three things today: it runs every facet's setup, it publishes them (types served, registries composed +once, contributions in their tables), and it waits until every host-lifetime object has had its first outcome. The +promise of the whole thing, pendingLoad, is also what incoming holds and host.hold wait for. That is the startup +deadlock of chapter I.

+

Publication first

+

The model splits load at the publication:

+
    +
  1. Setups. Every facet's setup runs. Setups only declare: host.serve, self.contribute, host.references. +They are synchronous and wait for nothing.
  2. +
  3. Publication. All declarations become visible at once, in one batch: types served, registries served and +composed once, contributions in their tables. Nothing activates inside that batch.
  4. +
  5. Wakes. After the batch, every request that waited for the publication (unbound unpublished) is placed, and +every unbound request for a name the publication serves is re-evaluated.
  6. +
  7. First outcomes. The host-lifetime objects activate. Whether load waits for them is the barrier question below.
  8. +
+

Requests wait for step 2 only. Since the setups wait for nothing, a host always reaches step 2, and a request from +another host can never wait for this host's activations. Chapter I's knot comes apart:

+
server Sworker Wsetups, publication: C servedA needs B: hold Bsetups, publication: B servedBStore needs C: hold Cbound C (S has published; C activates and goes live)BStore live, B livebound BA live; S's first pass done
Figure 5. The same startup as in chapter I, under the model. No deadline is needed.
+

The barrier: (a) or (b)

+

What is left of the barrier is the app's question "has every plugin had its first try?". R17 needs it: p3's base code +resumes the harness after load, so the first turn sees the tools of every plugin that loaded. Two shapes are on the +table.

+

(a) Chord keeps it. load resolves when every host-lifetime object of this load has a first outcome: it is +live, failed by its own cause, or parked with no blocker that is a local object still without its first outcome. In +code, per object:

+
const firstOutcome = (o: Obj): boolean =>
+  o.live || o.ownFailure || (o.parked && o.blockers.every((b) => !b.local || firstOutcome(b)));
+

So CodeMode → Harness, both local, waits until the harness had its outcome: code mode is parked on a local object +that is still in its first attempt, so code mode is not done. A wait on another host, or on a dependency that failed, +ends the first pass: the barrier never waits for another host, a retry, a later contribution or a reload. The signal +is separate from the readiness dependents wait for (Sol: resolving that for a parked object would let its dependents +open natives too early), and it is counted once per object, never reopened by retries. An object of the cohort that +ends before its first outcome (its plugin removed, the host closing) settles its entry as ended, and host.close +settles a load that is still waiting (Astra).

+

(b) The app keeps it. load resolves after the publication. Chord adds host.startup, a watchable of the +host-lifetime objects and their first outcomes with causes; base code decides when to resume (Fable). The tracker is +the same; only who awaits it changes.

+

Three of the four reviewers recommend (a): the activation graph is Chord's, and every app that wants R17's behaviour +would otherwise build the same cohort tracker. Fable recommends (b): the policy belongs to the app, and p2 already has +three different ones. Owner decision 1.

+

What the barrier does not promise

+

Neither shape can promise that every tool is there for the first turn, and the model does not pretend it can. Sol +reproduced it against today's code (/tmp/barrier-sol-first-outcome.log): object A's open fails on its first +attempt (A's first outcome); B goes live and contributes; the registry changes and retryOn wakes A for a second, +slow open; load resolves with only B's contribution in the table. Tools found in onLive (MCP) arrive later by +design. What a first turn really needs (a model, an environment, a reconciled working directory) is p3's explicit +admission check, as p2's whenRegistered and its harness prerequisites are today.

+
What p2 does today

p2 does not wait for all host objects before the first turn. Its worker (apps/p2/src/hosts/worker.ts) loads with +awaitHostObjects: false, waits for the harness, resumes Durable, tells its parent control.ready() and only then +awaits host.ready(). It has to: its subagent tools and the search indexer need server types, and the server admits +the worker's link only after control.ready(). A barrier before that point would deadlock, exactly as in chapter I +(Astra and Sol, independently).

+
+
+ +
VII

Plugins Coming and Going

Removals, versions and rollouts across hosts, as events the requests wait for.

+

Slice 7 brings reload and rollout, and with them the cases where a provider disappears on purpose. Today's design +handles them with their own mechanisms: gone when a type is removed, a final contract_mismatch when a version is +dropped, updating with a deadline during a rollout. Under the model all of them are one thing: a request becomes +unbound with a reason, and the event that fixes the reason wakes it.

+

A type removed, and served again

+

A reload that removes a type ends its objects as today: natives close with reason end, callers of running calls are +rejected at once, handlers drain. What changes is what happens to the requests that pointed at those objects.

+
    +
  • Over a link, the provider sends failed { waiting, not_served } on the bound hold. A failed on a bound hold +unbinds it (Fable: one message, one rule; the consumer applies it as today's #startOver plus failed). The +provider detaches the hold's sources and drops its unsent pending ops and any queued bound. Two things it must not +do (Sol): free the ack slots of bound and state messages already sent, which stay counted until their ack or +the link's loss, or resend calls, which fail and free their slots once.
  • +
  • On the same host, a Binding whose target ended by removal becomes unbound and accepts a second bind. Its +member sources re-bind by name (one frame from the last value, as on a restart today), keyed pins re-pin, parked +calls flush or reject, presence lists it again. This is the one genuinely new piece of machinery in the model +(Fable, Sol): today bind runs once per binding, and the ended object is forgotten.
  • +
+

When a later commit serves the name again, the publication wakes every request indexed under it, on every host that +holds one, and they bind as if nothing had happened.

+

A version no longer accepted

+

A reload can also keep the type and stop accepting an old version (it drops accepts[1]). Draft 1 ended the provider +object then; Sol and Astra caught it: the object, its documents, its jobs and its holders on the current version are +fine. Only the requests on version 1 become unbound version. The object's own plugin reload still restarts it if it +has natives (§21), as for any reload.

+

A type that moved

+

A plugin can move a type from the worker to the server by changing its placement. A consumer's request carries the +address its own code computed, so the provider appearing on the server cannot wake it: the request is not waiting +there. The old host answers not_served; the consumer's own reload, which brings the new placement, makes a new +request. The model adds no redirect: moving state between hosts is a deliberate migration, not something Chord guesses.

+

Rollouts across hosts

+

A rollout installs a revision on the server first, then on the workers, then on the tabs (design §21). Between those +installs, hosts run different code. The model needs only one rule for that: each host knows its own install, and +nothing more.

+
    +
  • A provider host that is installing keeps holds that do not fit its current code as unbound installing: the +consumer sees loading, its calls park. When the install ends, by its commit or by its rejection, the host +re-evaluates them against the code it now runs (Astra, Sol: a rejected install must not leave them asleep).
  • +
  • A provider host that is queued but not installing yet answers a mismatch with failed { waiting, version } +until its install commits and wakes the request. That is a visible blip of at most one install; Opus judged it +acceptable, and avoiding it would need revision fields in every answer.
  • +
  • Installs never wait for activations. An install that waited for its objects to go live would deadlock a rollout +that runs in the other order: the worker's objects wait for the server's new types, the server's install waits for +the worker.
  • +
  • A host whose own install was rejected fails its own requests at once with the reason (design §21).
  • +
+
The sharing plugin's contract goes from v1 to v2; the server installs first

Before

S and W run revision R. W's ShareStatus holds the server's Sharing v1; T's tab holds W's ShareBadge. All live.

+

S installs R+1

R+1 makes Sharing v2, with no accepts[1]. While S stages it, S still runs R: W's v1 hold fits the running code +and stays live. A hold that arrived now from a host already on R+1 (a v2 hold) would not fit R: it would wait +installing (loading, calls park) until the install ends.

+

S commits R+1

The install ends. S re-evaluates: v1 is not served. W's hold becomes failed { waiting, version: asked 1, served 2 }. +ShareStatus stays live (R18 after live); its calls that use sharing reject at once with the cause.

+

W installs R+1

W's new code bundles Sharing v2. Its own reload's handover makes the new request at v2 and releases the old one. The +new hold binds; ShareStatus is fully back. T's ShareBadge never noticed anything but one stale value.

+
+

Old calls

+

Calls that were running when the reload came follow §21 unchanged: a pure swap lets them finish on their old handles; +a native restart drains them first; a removal rejects their callers at once and drains their handlers. Chord never +resends a call.

+
+ +
VIII

Finding Cycles

Without the deadline, the one gap left in cycle detection needs an answer: a narrowed deadline, or a probe.

+

A provider that never appears keeps its dependents parked, and that is visible: they say what they wait for. A cycle is +different: A waits for B, B for A, every provider exists, and still neither can go live first. Today Chord has two detectors and a deadline. On one host, self.get follows the waiting edges of +activating objects. Across hosts, a hold carries the activation chain it was made in, and a provider refuses a hold +whose chain contains the object being activated. The gap is documented (design §6): when A on the server and B on a +worker start at the same moment, the holds carry [A] and [B], neither finds the other, and both fail at the 60-second +deadline. The model removes the deadline, so the gap must close.

+

Two answers

+

The owner decided this gap once already. On 2026-10-24, after the slice 4b reviews, he made simultaneous starts across +hosts "a documented limitation: they time out transient and retry; no probe message." That decision assumed the +activation deadline. The model removes the deadline for dependency waits, so the decision needs one of two +continuations, and this one is his (Owner decision 6):

+
    +
  • (A) The 4b decision, narrowed. One deadline stays: an attempt parked on a dependency held over a link, whose +target attempt is known and still activating or parked, fails transient after its readyTimeout and retries with +backoff; its next attempt usually starts after the other side, and the activation chain in its hold finds the cycle. +Working it through showed the catch. A target that is parked is the common case of a chain across hosts waiting +for a missing plugin: a tab's view on a worker object that waits for an uninstalled type. Those dependents would time +out and poll again, as today, so "local = remote" would stop at the link. Narrowing the deadline to targets that are +loading avoids the polling but misses a real ring: A and B on two hosts wait on each other, and B also on a missing +X. B is failed { waiting }, so A is too; when X appears, both stay failed { waiting } on each other, with no +deadline to break it and no chain to see it. Worse (Fable, round 6): an object's activation chain is fixed +when it is created and its instance id survives retries, so a retried attempt sends the same hold with the same +chain; two simultaneous starts time out and retry in lockstep and are never found. The deadline would poll without +finding what it is for.
  • +
  • (B) The probe (recommended, pending review) described in the rest of this chapter: no deadline at all, one new +message kind, and the most intricate rules in the model. Five review rounds found a counterexample in each of the +first four drafts; the version below answers all of them, with one choice left in rule 5. Slice 7 +would prove it with tests before the old deadline goes.
  • +
+

The rest of the chapter describes (B): it is the answer that keeps one rule on both sides of a link, and its review +history shows what a complete answer takes.

+

Why a growing chain is not enough

+

Draft 1 proposed letting chains grow: an activating object that receives a hold with a chain sends the grown chain on +to its own dependencies. Astra and Sol showed two flaws. Chains are cut at 32 entries today, so a ring of 33 objects +started at once is never found: the chain that reaches each object is always missing that object's own id. And a +chain records ancestry, not the current wait graph: once A stops waiting for B, B's chain still names A, and a later +wait of B on A looks like a cycle that is not there.

+

(B) The probe

+

The model uses an edge-chasing probe (the idea of Chandy, Misra and Haas), for the rings that cross a link; on one host the self.get +walk stays, synchronous and exact, as today (Opus). +Five rounds of review shaped it; the rules below are the result.

+
    +
  1. Attempts, edges, tokens. Every attempt has its own id, minted per attempt: a retry or a reload is a new +attempt (today the id is per instance and survives retries). A wait edge runs from a parked attempt to the +activating or parked attempt it waits for, through a bound request. Unbound waits and registry reads are never +edges: they are not deadlocks, they end when a provider appears. Every edge carries an edge token: a number from a counter the +attempt keeps, never reused, replaced by a fresh one on any interruption or change of that edge (Astra, Sol: two +requests to one object are two edges): a new target attempt (also when a local +target retries without a re-bind), any loading on a bound or pending hold (a relay re-holding upstream included), +an unbinding, a rebinding, a re-hold after a reconnect. An attempt is parked for one interval only (once past its +dependency check it never parks again), and attempt ids are never reused. A provider tells its consumers which attempt they wait for on every answer that is not bound: +loading { hold, attempt }, and a waiting failed { hold, cause, attempt }.

    +
  2. +
  3. A search per edge change. Whenever a parked attempt gets a new edge, or a new token on one, it starts a +search: probe { kind: "search", round, initiator, trail }, with the next number of a counter the attempt keeps, +its own attempt id, and the trail of (attempt, edge token) hops crossed so far, starting with its +own. A ring closes at its last edge, +and that edge's source starts the search that finds it (Sol's simplification after Astra's counterexample: a ring +that forms only after a reconnect, with unchanged attempts, is otherwise missed). Searches are coalesced per batch. +On the same host a probe is a call on the target attempt, forwarded through a queue rather than recursion; over a +link it is a new message, probe { hold, to, ... }, consumer to provider, relayed by hold id with the origin. +Probes are honoured on parent and route links and dropped on child links (accept), whatever trusted says +(Opus): a tab learns attempt ids from loading and could otherwise break an object with a forged probe, while a +worker whose transport is untrusted must still take part. Clients serve no remote types, so no real cycle runs +through one.

    +
  4. +
  5. Forwarding. An attempt that receives a search drops it if to is not its current attempt or if it is not +parked. If the initiator is itself, the search came back. If the attempt is already in the trail, it drops it: a ring +that does not contain the initiator is found by its own members' searches (Opus). Otherwise it forwards it along +every wait edge, appending itself and the edge's token to the trail, but only if its round number is higher than the one it stored for +that initiator; then it stores it. A late, older round is dropped even if this attempt never saw it: its initiator +has moved on (Astra, Opus). The seen map holds one number per initiator, up to 256 initiators, and goes with the +attempt.

    +
  6. +
  7. The check. The initiator accepts a returned search while it is still parked, whatever round it carries (Astra: +a re-search must not invalidate an older search still under way); the check decides. It verifies its own hop, then +sends a +probe { kind: "check", round, trail } along the recorded trail: each hop passes it on only if it is still that +attempt, still parked, still holding the recorded edge token, pointing at the next hop's attempt. The check follows the +trail by position, ignores the seen map and dies on any mismatch. When it returns, the initiator verifies its own hop +once more. If the check comes +back, the initiator fails broken invalid/cycle, naming the path from the trail, and today's propagation breaks +the other activating objects on the ring and the dependents off it.

    +
  8. +
  9. No silent loss. Astra found the last hole: a search can reach an attempt first by a shortcut and be dropped +there when it arrives again along the ring itself; if the shortcut breaks before the check, the check dies and +nothing is left to find the ring. Round 6 tried dead notices sent back to the initiator. Sol and Astra showed +that reliable notices need recovery state per search that the seen map's eviction may not drop, and that a long- +parked attempt would otherwise keep for every initiator that ever passed: a leak, unless every search gets a full +lifecycle with completion and cancellation. So rule 5 is a choice, part of Owner decision 6:

    +
      +
    • (B1) Re-search with backoff (recommended). An attempt parked on a remote attempt starts a fresh search again +after a backoff, 1 second doubling to 60, reset by an edge change, whether it shows loading or failed { waiting }; the timer stops when the attempt leaves parking. No dead messages and no recovery state; +eviction only costs work; a new search starts within one backoff period, so every stable ring that +crosses a link is eventually found (no wall-clock bound: delivery is only fair). The +timer runs as long as the attempt stays parked on a remote attempt: a view on a worker object that waits for a +missing plugin re-searches every minute until a reload (Fable). The timer sends probes only: it +never fails an object, never re-holds, never polls a provider. Its cost is one probe per parked remote dependent +per period, which dies at the leaves of the downstream wait graph.
    • +
    • (B2) Events only. Every search is owned until it settles: branches report completion or abort along their +trails, a dropped probe answers dead { initiator, round } by a reverse route, edges remember the searches they +carry until those settle, and an initiator that ends or moves on cancels its old search. Complete without any +timer, bounded by the searches in flight, and the most intricate protocol in Chord; not yet specified to the +reviewers' satisfaction.
    • +
    +

    Sol's correctness reference, enumerating every simple trail without dedupe, never suppresses a ring copy, but its +work is exponential on chains of diamonds.

    +
  10. +
+
A on server Sattempt a1B on worker Wattempt b1hold B (A activating)hold A (B activating)loading { attempt: b1 }loading { attempt: a1 }search { to: b1, round: 1, initiator: a1, trail: [a1·t1] }forward: search { to: a1, round: 1, trail: [a1·t1, b1·t1] }search 1 came back: check the trailcheck { round: 1, trail }: b1 still waits on a1 with token t1check passes oncheck came back: broken invalid/cycle "A → B → A"
Figure 6. Two objects on two hosts start at the same moment and need each other. The probe finds it; nothing times out.
+

Why a reported ring existed

+

Draft 4 confirmed a cycle with a second, independent round. Sol showed a schedule in which two rounds pass through two +different stale paths, each of them half replaced by reloads, and together report a ring that never existed at any +moment (/tmp/model/sol-probe4.log). The check closes that: it walks the very path the search recorded and demands +that every edge still carry the token it had when the search crossed it. A token changes on every interruption and is +never reused, and an attempt is parked for one interval only, so an edge cannot vanish and come back unnoticed (Fable and Opus +checked the argument in round 5). Every edge of the trail therefore existed from the moment the +search crossed it until the moment the check crossed it; the search crossed all of them before the check began, so +when the check began, every edge of the ring existed at once. In Sol's schedule the check dies at B, whose edge now +points at another attempt.

+

Why every ring is found

+

Under an eventually stable wait graph and fair delivery (endless edge churn can keep invalidating searches), and with +rule 5's (B1) or (B2) for a search suppressed behind a path that broke, a ring of parked attempts is closed by its last +edge. When that edge appears, its source starts a search; every other edge of the +ring exists at that moment, and every attempt on the ring waits, directly or not, on the source, so none of them can +go live before the search and its check come back. Rings of any length are found, through local and remote edges +alike. On a graph without cycles, searches die at the leaves; on a diamond, the seen map drops the second arrival.

+

What it does not promise

+
    +
  • A ring broken up just after. A ring that existed when the check began may be broken up by a reload, a retry, an +unbinding or a revocation a moment later; it is still reported, because it did exist. The objects it breaks recover +as every broken object does, through ref.retry() or a reload of their plugin.
  • +
  • Linear work under every schedule. With more than 256 initiators passing one attempt at once, an evicted entry lets +a search be forwarded again, so in an adversarial schedule the work can grow with the number of simple paths (Astra, Sol); never endlessly, since +a search never revisits an attempt already in its trail. +The seen map's memory stays bounded. Each edge change costs one search that crosses each downstream edge once, +and one check per returned search; with (B1), every attempt parked on a remote attempt adds one search per backoff +period, also while an acyclic chain only waits for a missing plugin.
  • +
  • Constant message size. A probe carries its trail, so it grows with the length of the path it crossed; a search +that returns carries a trail as long as its ring. That is the price of the check; chains were cut at 32, trails are +not, because the trail is the proof.
  • +
+
Review: the state of (B)

The check's argument (a reported ring existed at the moment the check began) was confirmed by all four reviewers in +rounds 5 and 6, given exact edge identity (the tokens). Completeness under an eventually stable graph holds with +(B1); with (B2) it holds once the search lifecycle is specified, which no draft has done yet. If the owner chooses (B), +slice 7 keeps today's deadline for activations waiting across links until the probe's tests pass: long rings, +diamonds, a ring closed after a reconnect with unchanged attempts, Sol's phantom schedule, Astra's broken shortcut, +Sol's eviction schedule, 300 concurrent initiators, edges added and removed, reloads, relays, forged probes.

+
+
+ +
IX

Callers

Getters, holds and calls get one rule: wait while something is under way, answer when nothing is.

+

Objects park; callers do not. A caller is code that wants an answer: a method that needs a reference now, a tool that +runs inside a turn, a tab that wants to render. The model keeps their deadlines and gives them one rule.

+

host.hold

+

host.hold returns a reference at once and never rejects; only invalid params throw, also before load. That was +already the design; the model removes today's inconsistency: a missing local type throws after load but fails the +reference before it. Now a missing local type gives loading before the publication and failed { waiting } after +it, and the reference binds when a provider appears. A UI that wants to wait for something that may never come +holds it and shows its state.

+

host.get, ctx.get, host.registry

+

The getters resolve when the reference is live. While it is loading, or failed { transient } (its object's own +retry is under way), they wait, bounded by their readyTimeout. On failed { waiting } or failed { broken } they +reject at once. On success, rejection or timeout their temporary wait goes; a rejected getter releases its request. +host.registry returns the RegistryValue, whose watches and observers are owned as they are today; it has no +release().

+

Opus's case decided the recommendation. A Durable tool runs chord((ctx) => ctx.get(SharingView, { session }), ctx) +while sharing is not installed. If getters waited through waiting, the tool, and with it the agent's turn, would +hang for 60 seconds and then fail; a call on the same reference would fail at once. With the rule above, the getter +and the call give the same answer at the same moment. The rule has a cost Opus named: a dependent whose only blocker is +an object recovering by itself (an SSH environment coming back, transient with its own backoff) is waiting, so a +getter on the dependent rejects at once. Code that wants to ride that out holds the reference instead of getting it.

+
Owner decision 2: getters on waiting

Recommended: reject at once (Opus; Fable agreed in round 2). The alternative waits through waiting up to +readyTimeout, which keeps "local = remote" just as well but makes every getter on a missing plugin cost its full +deadline.

+
+

Calls

+

Calls on a loading reference park in order, each bounded by the reference's readyTimeout from the moment it was +made (never renewed by a reconnect or a reload). Calls on a failed reference reject at once with the cause. A call +already sent when its link drops, or when its object ends, fails and is never resent; its outcome is unknown, and +idempotency is the app's (a requestId, as p2's chat does).

+

The rule in one table

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
The referencehost.holdgetterscalls
loadingreturns itwait, boundedpark, bounded
failed { transient }returns itwait, boundedreject
failed { waiting }returns itreject at oncereject
failed { broken }returns itreject at oncereject
livereturns itresolverun
+
+ +
X

Registries

The same resolver on one host, with versions that can differ inside one revision.

+

A registry is host-local: its contributions are in-process values, often natives. Slice 8 made self.registry(R) a +dependency that waits for a value. The model keeps that and folds it into the one resolver.

+

A reader's request

+

A reader asks for R at the version its own bundle was built against. The request is bound while R is served and +has a usable value at that version: composed, and for an older reader mapped back by the provider's +accepts[old].value. Otherwise it is unbound: not_served (no plugin serves R here), version (the provider does +not accept the reader's version, or the reader is newer), compose (R is served, but its first composition, or this +version's mapping, threw), or loading before the publication. The reader is woken at every transition of the value +at its version to usable. S2's broken for a newer reader and S3's backoff go; so does #valued, the latch that woke +readers only at the very first good value.

+

A registry waits for a value, never for a particular contribution: an empty table composes. "Wait until model X is +there" is the app's admission check, not a registry rule; Durable's own "model unavailable" wait covers the harness.

+

Versions inside one host

+

Draft 1 claimed that a registry's provider and its contributors always run the same version, because a host installs +a revision atomically. Astra corrected it: a revision is a set of plugin bundles, and a reload of the plugin that owns +the registry leaves every other plugin's contributors and readers on the version they were built against. So the +provider's adapters do real work inside one host: accepts[old].contribution maps an older contribution, accepts[old]. value maps the composition back for an older reader. A contribution without an adapter is left out and reported; a +reader without one is waiting with version.

+

The last good value, and natives

+

Slice 8 decided that a throwing compose keeps the last good composition, so an extension point stays available while +its owner fixes a bug. Astra found the case where that hands out something dead: the harness contributes its native +H; a reader is live; the harness ends; the next composition throws for an unrelated reason; .value still returns +the composition that contains the closed H.

+

The model narrows the rule: .value keeps the last good composition only while every contribution it was built from +is still in the table. When one of them leaves (its contributor ended or was reloaded), the last good value is dropped +and .value throws until compose succeeds again. And compose represents a missing contributor as a value +(absent), never by throwing.

+
Review: drop the last good value, or fence the reads

Sol would keep the last good value and instead require natives in a registry to be looked up at use through their +owner, without the guarantee. The draft takes Astra's narrower rule, because it keeps "a registry never hands out a +native whose contributor ended" without asking every contributor to wrap its natives. It narrows a decision of slice 8, +so it is the owner's: Owner decision 5.

+
+
+ +
XI

Keep-alive, Presence, Revocation

What a waiting request costs, who counts it, and who may end it.

+

A parked object and an unbound request are resources: they hold memory, they may keep a worker process alive, and +they act for a principal. The model says how each of those works while nothing is bound.

+

Keep-alive and presence

+

A worker stays alive while host.activity shows work: references held over links, calls, jobs, lingering objects. A +request that arrived over a link counts once in host.activity.references from its arrival to its end, bound or +unbound. Without that, a session whose tab waits for a plugin that is not installed would see its worker stopped every +idle period (five minutes in p2) and restarted by the next re-hold, forever (Opus). Presence is different: +host.references lists only bound references, so "who is looking at these notes" counts viewers, not waiting +requests.

+

Revocation

+

Revocation ends the requests of a revoked principal, bound and unbound, at every hop. A guest whose grant is revoked +while their tab waits for a missing plugin does not get a reference later when the plugin comes back: the request is +gone, and nothing revives an ended request (no publication, no Retry, no new grant). A refused request (no authority) +is never indexed in the first place.

+

gone

+

gone remains for one thing: revocation. A provider host that closes closes its links and sends nothing (today's +#close); its consumers see a link drop, loading, and hold again when the host comes back. A reload that removes a +type sends failed { waiting } instead of gone (chapter VII).

+

Parked objects and their jobs

+

A parked object with params lives while it is referenced, like any object; when its last holder releases it, it ends +and releases its requests, so a parked chain goes from the top down. A host-lifetime object parks until its plugin is +removed or the host closes. Persisted job runs of a parked host-lifetime object are loaded with its documents, before +it parks; run and finish start only once it is live, and the run count keeps the object as today.

+
+ +
XII

The Clock

Which timers stay, and why each of them has no event to replace it.

+

The owner expected the model to remove almost every timeout, "except maybe readyTimeout". That is what it does for +dependencies. Every remaining timer bounds something no event can end: a caller's patience, an object's own code, a +silent transport, a process that may start this time.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Timeout or backoffAfter the modelWhy
readyTimeout of an attemptonly its own steps (natives, follow attaches, checks, derives); paused while parkeda slow dependency is bounded where it runs, by its own steps
readyTimeout of host.get, ctx.get, host.registrykepta caller must get an answer
readyTimeout of a parked callkeptan action must not run long after it was asked for
identity backoff 1 to 60 sonly for the object's own transient failures (an open, a follow)nothing tells Chord that a disk or a network came back
dependency backoff, polling of refused holdsremoveda dependency's recovery is an event
dependency deadline for waits across linksanswer (A): kept only for an attempt waiting on a remote attempt that stays activating; answer (B): kept until the probe's tests pass, then removedthe simultaneous-cycle gap (chapter VIII)
probe re-search backoff, only with answer (B1)new: 1 to 60 s while parked on a remote attemptfinds a ring whose search was suppressed; sends probes only, never fails or re-holds
route backoff per addresskepta failed worker start has no event; it avoids fork storms
heartbeat 15 and 30 skeptthe only detector of a silent transport
closeTimeout (natives, the drain, the routine handover)keptbounds other code's teardown
routine and job finish backoffskeptthe object's own code throwing
lingerkepta policy for warm objects, not a wait
rollout install timeout (designed, not built)a watchdog on the install onlynever a deadline for dependencies
+

So readyTimeout stays, but its meaning narrows to two things: how long a caller waits, and how long an object's own +activation steps may take. Waiting for someone else never counts against it.

+
+ +
XIII

A Morning, Step by Step

The preface's Tuesday morning, played through under the model from the first click to the afternoon's rollout.

+

The rules of the earlier chapters meet in one story. Mario opens session S1 on his phone; the server S has the catalog +and the search feed; the worker W for S1 has not started. W's plugin set has core, models, codemode, notes +and search, but not sharing, which was removed last night; the notes view still asks for it. Later in the day a +rollout brings sharing back, with a new contract.

+
Session S1 on Tuesday

1. The tab asks for the notes

T calls host.hold(Notes, { session: "S1" }). The address is Session { id: "S1" }, not T's own, so the request goes +over T's parent, the server. S has a route for Session: it resolves the address by starting W. T's reference shows +loading; the request waits on S's path until W's link exists.

+

2. W connects before it loads

W connects to S first (W6), then starts load. The hold for Notes arrives before W has published its facets: it is +unbound unpublished and waits, still loading for T. Nothing is placed by a half-empty table, so the hold is not +sent back to S.

+

3. W publishes

W's setups run, synchronously, and are published in one batch: Notes, NoteStore, Harness, CodeMode, Indexer +are served; HarnessModels and HarnessExtensions are served and composed once. After that batch, the waiting hold +for Notes is placed and binds; the host-lifetime objects start.

+

4. Code mode parks on the harness

CodeMode declared self.get(Harness). The harness is still opening Durable, so CodeMode parks: loading, no timer. +The harness reads HarnessModels, which has a value (composed at the publication; the model runtime's contribution may arrive +after), finishes its own steps +and goes live. That state change wakes CodeMode, which passes its dependency check, opens its sandbox, goes live and +contributes its tools.

+

5. The indexer waits for the server

Indexer needs the server's SearchFeed. The hold goes over W's parent; S published long ago, so it binds at once, +SearchFeed is live, and Indexer goes live. Had S been starting at the same moment, the hold would have waited only +for S's publication, never for S's activations: the startup deadlock of chapter I cannot form.

+

6. The first pass

With barrier (a), W's load resolves once Harness, CodeMode and Indexer had their first outcomes. CodeMode +waited for the local harness, so its tools are in the table; base code resumes the harness, and the first turn sees +them. With (b), base code would read host.startup and decide the same.

+

7. The notes view asks for sharing

Notes's factory declared self.get(ShareStatus), a local type of the sharing plugin. Nobody serves it on W: the +request is unbound not_served, indexed under the name. Notes parks failed { waiting, "waiting for ShareStatus: not served on Session S1" }. T's reference shows the same; a call from T rejects at once with that cause. No backoff, no +polling, no 60-second wait.

+

8. The phone changes networks

T's link to S drops. T's reference goes loading: its request stays, its last value stays readable, a call made now +parks. A call that was in flight fails and is not resent. T reconnects; the request is held again on the new link; W +answers as before: failed { waiting }.

+

9. Sharing comes back, with a new contract

In the afternoon a rollout brings the sharing plugin back at revision R+1, where Sharing on the server is v2. S +installs first and serves Sharing v2. W's install is queued; nothing on W asks for Sharing yet, because the sharing +plugin is not installed there, so nothing changes for W. (Had W still run a sharing facet built against v1 without an +adapter on S, its requests would show failed { waiting, version } until W installed: the ordinary window of a +server-first rollout.)

+

10. W installs R+1

W's install commits: a publication. It serves ShareStatus again: the index wakes Notes's request, which binds. +ShareStatus activates, holds the server's Sharing at v2 (W's new code bundles v2), goes live, and Notes passes its +dependency check, opens what it needs and goes live. T's reference becomes live without T having done anything.

+
+

What did not happen is the point. Nothing waited for a deadline; nothing retried on a timer; no reference ended +because a plugin was missing; no host waited for another host's activations; and every waiting object said what it +was waiting for.

+
+ +
XIV

Scenarios: Starting Up

Twenty startups on p2's real hosts, under the model, with the barrier as (a) and as (b).

+

These scenarios, from Astra, follow p2's real hosts: the server alone, workers started by a tab or by another worker, +web tabs, the TUI, restarts in the middle. Each row says what the model does, with the barrier in both shapes where +they differ, and what today's code does. The pattern across all of them: publication never waits for activation, so +dependency traffic between hosts always moves; what differs between (a) and (b) is only when base code is told that +the first pass is over, and neither shape promises every tool before the first turn.

+

The Rule column cites sections of the working document, reproduced in appendix C.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#ScenarioUnder the modelTodayRule
1The server starts alone. Catalog, ModelConfig, LoginService and ForwardStore initialize before any session worker exists.Their local references are loading during initialization, then live or failed with an own cause. Under (a), load waits for these first outcomes; under (b), it returns after publication. Public admission still needs explicit security prerequisites; a completed first pass is not proof that every service works.p2 starts its HTTP listener but gates requests and upgrades behind plugin startup (apps/p2/src/hosts/server.ts). Current next waits for eager objects' first live/failed outcomes (host/host.ts:load).§§1–3, 6
2A tab's request reaches the server before its first facet publication. Catalog is declared by a later setup in that same load.The request stays loading behind the publication gate. The complete declaration batch places it; Catalog activation then produces bound and the reference becomes live. Neither (a) nor (b) delays that hold until unrelated eager objects finish.Current next's Hold.start waits for pendingLoad, which includes every eager object's first outcome (remote/hold.ts). It waits longer than declaration safety requires.§§1, 4, 6
3Several server facets set up successfully, but the last setup throws. A tab is already waiting for Catalog.No partial declaration batch may activate objects. Load fails and the host closes; local held references become ended, while the tab sees a link drop and remains loading. A replacement server's successful publication can satisfy the tab's retained request.Current next closes the host on a throwing setup and activates no objects (host/host.ts:load). Setup failure rejects load directly.§§1, 6, 10
4A session worker's Harness opens slowly. Its CodemodeRuntime startup object declares Harness and contributes tools only after its own sandbox opens.Harness and CodemodeRuntime start loading; CodemodeRuntime waits without a dependency timer. Harness becoming live wakes CodemodeRuntime, which opens its sandbox and becomes live. Under (a), this local chain finishes before load returns and resume; under (b), the application must await the same startup outcomes or risk a first turn without those tools.p2 resumes after Harness is ready, before host.ready() (apps/p2/src/hosts/worker.ts); CodemodeRuntime depends on Harness (plugins/codemode/worker.ts). Current next's load barrier waits for both first outcomes.§§3, 6, 11
5A tab opens Chat for a stopped session. The server starts that session worker, whose Search Indexer needs server SearchFeed.Chat remains loading while the route starts the worker. The worker connects and publishes before its application barrier; SearchFeed can bind immediately on the already-published server. Indexer becomes live when its prerequisites are ready, and Chat receives bound once its own prerequisites are ready. Under (a), a still-remote startup blocker can already count as a parked first outcome; neither option guarantees that every remote-backed feature precedes resume.p2 connects IPC first but the parent admits it only after control.ready(); Harness resumes before the later host.ready() (src/hosts/worker.ts). Current next gates incoming holds on whole-host load (remote/hold.ts).§§1, 3–4, 6
6A running worker calls ConversationBroker on the server. The broker sends to another session whose worker is stopped.Only the second worker starts a new publication and first-pass sequence. Its PeerConversation reference is loading until that view binds; the first worker and broker remain live. Under (a), the new worker's scheduler waits for its first-pass report; under (b), its application must supply that admission rule. This call does not reopen the first worker's startup barrier.p2 gets PeerConversation inside the broker's call and starts its worker through the session route (plugins/conversations/server.ts, src/runtime/server.ts). The spawned worker uses p2's Harness-before-host-ready ordering.§§3, 6, 8
7Many tabs open the same stopped session while other tabs open several different sessions.Requests to one worker share route resolution and provider identities where their principals match. Each tab's reference starts loading and becomes live on its own binding. Both barrier options remain per host; a slow worker cannot hold back every other worker's first turn. The application shows per-session startup status, not one global ready flag.Current next shares resolution per route address (host/route.ts). p2's SessionHosts also serializes starts per session (apps/p2/src/runtime/server.ts).§§1, 6
8During server startup, an eager Bootstrap object needs a worker's Agent. That worker's eager FeedOwner needs the server's independent Catalog.Server and worker publication happen independently of their first-pass reports. Catalog binds while Bootstrap is still loading; that wakes FeedOwner, then Agent and Bootstrap can become live. Both (a) and (b) avoid a host-ready gate on this dependency traffic. The acyclic object graph needs no timeout to make progress.Current next can deadlock its whole-host load gates until activation deadlines fail attempts: the Agent hold waits for worker load, while the Catalog hold waits for server load (host/host.ts, remote/hold.ts). This is an architectural example using p2-style server/worker dependencies, not a claim that p2 has a Bootstrap plugin.§§1, 4, 6
9The worker has a healthy Harness, but its p2.subagent-tools startup object still waits for server Subagents. The first generation is ready to run locally.p2.subagent-tools remains loading while its remote reference binds, or failed with waiting if that provider is unavailable. Under (a), the remote wait can settle its first outcome without its contribution; under (b), publication returns even earlier. Server bound later wakes p2.subagent-tools and publishes its tools for future phases. If the first turn requires those tools, application admission must wait explicitly.p2 p2.subagent-tools declares both Harness and server Subagents (plugins/subagents/worker.ts). Its worker resumes Harness before admitting the upstream-dependent host objects (src/hosts/worker.ts).§§2–4, 6
10In a p3 port, the core serves a Models registry whose empty composition is valid. The optional models contributor is absent when the worker starts.The registry is usable, so this absence alone does not park Harness; Harness can become live. Neither (a) nor (b) waits for an uninstalled contributor. The generation's model selection or application admission must report model unavailable and wait for a contribution. Registry publication later wakes that domain-level wait; it is not a missing-provider activation failure.p2 instead awaits the HarnessConfig default entry inside Harness's Durable opener (src/runtime/worker.ts:serveHarness). Without the models contribution, that opener does not finish normally.§§3, 6, 9
11A session is configured for SSH. Harness's execution opener finds that the environment is invalid or cannot open.Harness becomes failed with its own transient cause, or broken if the opener marks the error permanent. Dependents still before their readiness check become failed with waiting. Under (a), this is a first outcome, not a healthy Harness; under (b), load already returned but resume still requires a usable Harness. Effect retry or a configuration-triggered retry can later make it live; no local-execution fallback is implied.p2 opens the session machine during Harness activation and fails rather than silently using the local machine (src/runtime/worker.ts). Worker startup failures are surfaced and can require explicit retry (src/runtime/server.ts).§§2–3, 6, 11
12Prompt becomes live before its initial local instruction-file read finishes. A user immediately sends the first message.Prompt's reference can be live while its file-loading work continues. Under (a), its InputGate contribution is present before a normal local first pass completes; the gate still controls when local input may proceed. Under (b), the application must ensure that contribution exists before admitting input. The model does not turn background file loading into a host-wide barrier.p2 contributes InputGates and awaits the initial local load in its prompt sections; remote instructions deliberately load in the background (plugins/prompt/worker.ts). Object readiness is already distinct from instruction readiness.§§3, 6, 9
13Search starts an archived session worker only to rebuild its historical index. Indexer has a long scan to perform.Once SearchFeed and Harness are usable, Indexer becomes live even while its scanned state is false. Both (a) and (b) leave scan completion to the explicit IndexScan operation. The archived-worker application keeps Harness scheduling paused, waits for the scan result and then stops the worker. A startup report does not override that resume policy.p2 passes resume:false for archived index-only starts, exposes IndexScan.scanned, and stops the archived worker after rebuilding (plugins/search/server.ts, plugins/search/worker.ts, src/hosts/worker.ts).§§3, 6
14After a worker starts, a tab opens Workspace for a large directory. Its filesystem watcher is slow to initialize.The on-demand Workspace reference is loading until the backend is usable, then live. Its background watch may still be starting. Neither barrier option waits for every future Workspace view or directory watch; a component that needs a complete initial scan must use explicit domain readiness.p2 deliberately returns the Workspace backend before installing a potentially slow watcher (plugins/workspace/worker.ts). This work is not a universal worker-start prerequisite.§§3, 6
15A worker restores durable work that can resume immediately. No new Chat.send call occurs, so there is no incoming-input gate to intercept it.Harness opens with scheduling paused and can become live before all startup contributors. Under (a), base code resumes after the first-pass report and required application checks; under (b), base code must await equivalent outcomes itself. Input gates on new messages do not guard this replay. Each phase pins the contributions present when that phase begins.p2 explicitly calls Harness.resume before host.ready() (src/hosts/worker.ts). Its Durable registry adapter snapshots the extension table per phase/version (src/runtime/durable.ts).§§6, 9
16A previously loaded web tab starts while the server is offline. Its local UI facets can render an offline shell.Publication of the tab's local facets does not require the server. Remote references stay loading on the absent path. Under (a), remote blockers can count as parked first outcomes; under (b), declaration publication returns directly. The shell renders, and later connection plus bound messages make the affected views live.p2 starts reconnection before facet installation and renders React without awaiting a whole-host ready promise (src/hosts/web/main.tsx). It already supports an offline shell after the page has been loaded.§§1–2, 6
17A TUI candidate loads before its launcher activates BridgeLink. A startup UI object declares a server reference, whose outgoing frames are buffered.The reference remains loading on a remote dependency. Under (a), that remote wait is a parked first outcome, so candidate preparation can finish; under (b), publication alone finishes. Launcher activation flushes the frames, and bound wakes the UI object. Do not replace this dependency declaration with an own native opener that waits indefinitely for launcher activation.p2's setupTui awaits facet load while BridgeLink buffers frames until activate (src/hosts/tui-worker-runtime.ts). Adding an eager remote-dependent host object under the old full-ready load rule could stall candidate preparation until a deadline.§§1, 3, 6, 11
18A tab starts a worker, but the process crashes while Harness is still opening. The server launches a replacement.The tab's Chat reference remains loading across the lost path and replacement startup. The replacement has its own publication and first-pass cohort; successful binding makes Chat live. Both (a) and (b) must keep transport/start failure distinct from a healthy startup report. A process manager may surface a startup error and require user retry rather than retrying forever.p2 records worker startup failures and has an explicit retry path; it does not treat every failed boot as an automatically healing object (src/runtime/server.ts:SessionHosts). Current next's route layer independently retries failed resolution (host/route.ts).§§1–2, 6, 11
19The server restarts while a worker is still starting. A detached terminal host already owns running PTYs.Tabs see their server and worker references become loading; new hosts publish and bind them again. The application may stop old workers on parent loss, while the detached PTY owner survives. The old server's unfinished first-pass wait settles on close (its entries end). New startup reports must not wait for the detached PTYs to restart.p2 workers stop on IPC disconnect, while the terminal driver detaches its socket without killing the detached PTYs (src/hosts/worker.ts, plugins/terminals/detached.ts). Current next settles ended objects through its existing load settlement path.§§1, 6, 10
20A startup contributor in a v2 port reads a keyed configuration document from a local store. The store is live, but the first requested key still needs hydration.The contributor remains loading until that pin is ready; store-live alone does not make the dependency usable. Under (a), this local key-loading work remains part of the first pass, then the contributor can become live and publish before resume. Under (b), publication returns earlier, so the application must await that readiness if the first turn needs the contribution.Current next loads dependency pins before opening the consumer's natives and before its settled promise completes (host/attempt.ts:pinKeys). Its activation deadline currently also bounds this wait. This is a v2-port example, not an existing p2 keyed API.§§3, 6, 11
+
+ +
XV

Scenarios: References, Holds and Calls

Getters, holds, calls, link drops, relays, release and revocation, played through.

+

Fable's scenarios use p2's names: Catalog and Hosts (the sessions plugin, on the server), Sessions (on the +server, called from worker tools), SessionTransfer (a worker type the server calls for a fork), Notes { session } +(a worker type a tab holds), NoteStore (local on the worker), ForwardStore (ports, on the worker) and Forwards +(radius, on the server), and Harness (host-lifetime, on the worker). S is the server, W1 the worker of session S1, T +a web tab. The pattern: a request waits where something is under way and answers where nothing is; nothing a holder +does is needed to recover; and an ended holder leaves nothing behind in any index.

+

The Rule column cites sections of the working document, reproduced in appendix C.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#ScenarioUnder the modelTodayRule
1self.get(NoteStore) in Notes { S1 }'s factory on W1; NoteStore is served and activating.Notes makes its request, it binds to the NoteStore object, Notes parks loading. NoteStore goes live: a target state change wakes Notes, which opens its natives and goes live. No timer ran.The same wait, bounded by readyTimeout; past it Notes fails transient and backs off (attempt.ts waitFor).§3.1, §3.4, §4
2self.get(Catalog) from W1's ForwardStore; the parent link is up, S has published, Catalog is served there.The request goes over the parent (placement Server()); S binds the hold; bound arrives; ForwardStore was loading, now goes on and live.The same, after S's whole barrier (Hold.start awaits pendingLoad).§1, §2
3The same, but Catalog's plugin is not installed on S.S indexes the hold not_served under "p2.catalog" and answers failed { waiting, not_served }; ForwardStore's reference shows failed { waiting }, the object parks failed { waiting, "waiting for p2.catalog: not served" }. A reload on S serving Catalog re-runs the hold: loading, bound; the reference goes live, the dependent wakes. No backoff, no new hold.The hold is refused broken; ForwardStore fails transient and re-holds every 1-60 s, refused each time (remote/hold.ts #refuse, object.ts backoff).§1, §3.1, §4, §5
4self.get(SessionInfo) on W1 when the core facet is missing (same host, not served).The request is unbound not_served on W1 itself; the dependent parks failed { waiting }; a reload serving the type binds it and wakes the dependent.#assertRoute throws in the factory: the object is broken until retry() or a reload (host.ts reference).§1, §3.1, §4
5A tab's NotesView declares Notes { S1 }; S routes Session, resolve forks W1, W1 connects and publishes.The hold waits at T on its path (no_link while S resolves: loading), is relayed to W1, waits there unpublished until W1's setups, then binds; bound comes back through the relay. NotesView was loading the whole time and goes live.The same chain of waits, each bounded by readyTimeout; a slow worker start past it fails the tab object transient and polls.§1, §6.2
6host.get(Notes, { session }, ctx) from a tab's page code while W1 is starting.The getter waits while the reference is loading, up to its readyTimeout, then rejects with the cause and releases its request. It resolves as soon as bound arrives.The same.§8
7host.get(Catalog, {}, ctx) on W1 when S does not serve Catalog.The reference becomes failed { waiting } at the first answer; the getter rejects at once with "not served on server" (recommended; owner's choice to wait instead).Rejects at once too (a broken failure), after the refused hold's failed.§8
8host.hold(Catalog, {}, ctx) on W1 before W1's load ran.The reference is returned at once, loading (unpublished); W1's publication places it over the parent; it follows the object for its whole life: loading, live, failed { waiting } if S drops the plugin, live again when it comes back.Returned at once; placed once load resolved (host.ts #hold races load); a removal ends it for good with gone.§1, §8, §5
9host.hold(Missing, {}, ctx) for a local type nobody serves here.loading before the publication, failed { waiting, not_served } after it; a reload serving it binds the same reference.Throws unavailable at once after load; before load, fails the reference broken once it resolved.§8
10await host.registry(HarnessModels, ctx) by base code when the models plugin is missing.Rejects at once with unavailable/undeclared (waiting); code that wants to wait declares the registry in an object or holds. A served registry whose first compose threw rejects the same way (compose).Rejects at once (slice 8 build).§8, §9
11ctx.get(Sessions, {}) inside a worker tool through host.root (plugins/sessions/worker.ts) while the parent link is down.The request waits on the path (no_link, loading); the getter waits up to readyTimeout; the link comes back, the path re-holds, bound, the getter resolves. The request ends when the tool call settles.The same (ctx.ts CallContext.get, path.ts re-hold).§1, §8
12A call notes.add(args, ctx) on a reference whose object is loading (W1 restarting).The call parks in the reference's queue with its own readyTimeout; bound flushes the queue in order; past the deadline it rejects unavailable/park_timeout.The same (host/parked.ts).§8
13A call on a reference that is failed { waiting } (the plugin is not there).Rejects at once with the cause, no parking: nothing is under way.Rejects at once (the reference is failed).§2, §8
14A call already sent to W1 when the IPC link drops.The call fails unavailable and is never resent (its outcome is unknown); the reference goes loading; calls made after the drop park until the re-hold's bound.The same (consumer.ts link close).§2, §5
15The parent link drops while ForwardStore is parked loading on Catalog (S restarting).The request stays on the path (no_link), ForwardStore stays loading; the reconnect holds it again, S answers loading { attempt } or bound; the dependent wakes on the state change. No deadline on the wait.The dependent stays loading until readyTimeout, then transient with backoff (the audit's choice 6).§1, §3.3, §4
16A route link to W1 drops while T holds Notes { S1 } through S's relay.The relay sends loading downstream, rejects relayed calls in flight unavailable, re-resolves the address with the route backoff, holds again upstream with the kept request; bound is forwarded. The tab's dependents stay loading.The same (relay.ts); the route backoff stays.§1, §4, §11
17T's own link drops while its hold for Notes { S1 } is still unbound at W1 (not_served).S sends closed { origin } upstream; W1 drops the indexed hold with the origin: no leak. T reconnects and holds anew.The same for refused holds (provider.ts origin close).§1, §10
18A tab closes while NotesView is parked on Notes { S1 }.Its last holder releases: a parked object ends at once (it is not live, so no linger); its requests end, the relayed hold is released upstream, the index entries go. Nothing keeps the chain.The same (object.ts #idle).§1, §3.5, §10
19host.revoke({ grant }) while a guest's objects on W1 are parked and its tab's holds wait unbound.Every request of the principal ends, bound and unbound, at every hop: the parked objects end forbidden/revoked, the tab gets gone for each hold, the child link closes. Nothing revives them; a new login holds anew.Bound objects end, holds waiting for load are ended (host.ts revokeWaiting); refused holds are released with the link.§10
20A worker with nothing live: a tab holds Notes { S1 } but the notes plugin is missing on W1.The unbound hold counts in host.activity.references from its arrival, so the supervisor does not stop W1 while the tab waits; it is not in host.references (presence lists bound references only).A refused hold is not counted: the worker stops idle, restarts on the next hold, refuses again.§10
21S holds SessionTransfer on W1 for a fork while W1's Harness is parked on the models registry (not served).The hold binds to SessionTransfer, whose attempt parks on Harness: loading, then failed { waiting, "waiting for p2.harness: waiting for registry …" }, with the chain in the message; S's ctx.get rejects at once with it. A reload on W1 serving the models plugin wakes the harness, then SessionTransfer; a later ctx.get binds.SessionTransfer fails transient after readyTimeout, polls; S's getter rejects at its own deadline.§3.1, §3.4, §8
22ref.retry() from a tab's Retry button on a failed { waiting } reference whose blocker on W1 is broken (a throwing factory).The retry passes over the link (retry { hold }) to the parked object, which passes it to its broken blocker: a new attempt; waiting blockers are passed through, not_served ones untouched. If the blocker goes live, the chain wakes.retry reaches only the object the hold is on; a refused hold is started again.§3.7
23Keyed pin: Notes pins store.drafts.at(key) on NoteStore; the key's load fails at the provider.The pin is a dependency: Notes parks failed { waiting on p2.note-store.drafts[key] }; the provider's re-bind of that member (a retry) wakes it.dependency_failed transient with backoff (attempt.ts pinKeys).§3, §4
+
+ +
XVI

Scenarios: Providers, Versions, Registries

Plugins missing, added, removed and broken; versions that do not fit; registries and the harness.

+

Opus's scenarios cover the provider side: plugins that are missing, arrive by a reload, are removed or break; versions +that fit through accepts or do not; registries with missing providers, skewed versions and last good values; +self.follow over a harness that restarts; and reloads of an object's own code. The pattern: what used to be broken +or polled becomes waiting, woken by the one event that can fix it, and an object's own code is still the only thing +that breaks it.

+

The Rule column cites sections of the working document, reproduced in appendix C.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#ScenarioUnder the modelTodayRule
1Worker W1 has no drafts plugin. Chat (chat plugin, on W1) declares self.get(DraftStore).Chat's request is unbound not_served and indexed under DraftStore. Chat is failed { waiting, "waiting for DraftStore: not served" }, with no timer. A tab's Panel holding Chat shows the same cause, and its calls reject at once.#assertRoute throws in the factory, so Chat is broken. Only ref.retry() or a reload recovers it (host.ts).§1, §2, §3.1
2A reload adds the drafts plugin to W1 while Chat waits (1).The reload's commit publishes DraftStore. The index wakes Chat's request, which binds; DraftStore activates and goes live. Chat leaves parking, opens its natives and goes live; Panel goes live.Chat stays broken unless the reload retries broken objects.§4, §5
3Server S has no models plugin. On W1, the models plugin's host-lifetime ModelRuntime declares self.get(ModelConfig) (placed on S).S keeps W1's hold as refused, not_served. ModelRuntime is failed { waiting }. W1's load counts it as a first outcome (parked on another host). The harness runs, and its generations report "model unavailable".S refuses broken (remote/hold.ts #refuse). ModelRuntime is transient and polls: each backoff tick (1-60 s) makes a new hold across the link and reports again, for the host's life.§1, §3, §6.3
4S gets the models plugin by a reload (3, continued).S's commit re-runs its refused hold and sends loading { hold, attempt }, then bound. ModelRuntime wakes, opens its native and contributes to HarnessModels. The harness's next generation finds a model.The next backoff tick re-holds; recovery waits up to 60 s.§4
5A reload removes the ports plugin from S. Tabs hold PortsView; Radius (live) reads the radius.forwards registry that Ports contributed to.PortsView's objects end, and their holds become unbound not_served: the provider sends failed on the bound hold, and the tabs show failed { waiting }. Radius gets a table without Ports, composes the empty value and keeps working. A later re-add re-runs the kept holds, which bind again.The holds get gone and the references end for good (design §21). A re-add cannot revive them.§5, §9
6A reload removes W1's file-index plugin. The live Files object (files plugin) holds self.get(FileIndex) locally.Files' Binding becomes unbound not_served. Files stays live (R18). Its member sources keep their last values, and calls through that reference reject with the cause. When file-index is re-added, the binding binds again and the members re-bind by name (one frame).The reference ends released; Files can never use FileIndex again without a restart.§5 (locally), §3.10
7After a sessions plugin update, S's Catalog factory throws. W1's Chat waits on Catalog; a tab's Panel waits on Chat. The user presses Retry on Panel.Catalog is broken; Chat and Panel are failed { waiting, "waiting for Catalog: …" }. Retry passes on hop by hop (T, then S's Catalog, through W1) and retries Catalog once. If the cause is gone (the config is fixed), Catalog goes live and Chat, then Panel, wake.Chat polls by backoff with new holds. Retry on Panel retries only Panel, which is still blocked.§3.7
8S's EnvManager (envs plugin) opens an SSH connection that fails because the remote host is offline. W1's environment objects wait on S's env views, which wait on EnvManager.Only EnvManager backs off (transient, its own retryAt). Its dependents are failed { waiting } with no timers. When SSH comes back, EnvManager goes live and the chain wakes in order. A host.get on a dependent rejects at once (it is waiting); a hold just shows it.Every level runs its own 1-60 s backoff and re-holds across the link at every tick.§3.1, §3.3, §8
9A reload of the terminals plugin restarts S's TerminalService (natives: the terminal host process D). Terminals{session} objects are live and depend on it.Native restart as in §21: new calls park while it is loading; running calls drain; natives close with reason reload; the factory runs; natives reopen. Its dependents stay live (R18); their calls park while it is loading.The same; this is design §21, unchanged.§5 (old calls), §3.10
10Worker W1 crashes while a tab views Chat{S1} through S.S's relayed hold goes to its path, no_link, so the tab's reference is loading and its calls park, bounded. The route re-resolves (route backoff); the new W1 publishes, the held request binds, and a fresh bound reaches the tab.The same, except a dependent that is activating fails at its 60 s deadline in between.§1, §2
11A tab on Chat v1 meets W1 serving Chat v2 with accepts[1].The check accepts v1; the hold binds through the mappings (slice 7). The tab is live.The same once slice 7 builds accepts.§1, §5
12A tab on Chat v1 meets W1 serving Chat v3 without accepts[1].The hold is unbound version, so the tab shows failed { waiting } ("update the page") and its calls reject. The tab's own reload makes a new v3 request and releases the old one. If W1 rolled back to v2, the old hold would bind again in place.Refused broken, or gone at the reload that dropped v1: ended for good.§1, §5
13A tab reloaded to v2 holds Chat while W1 is installing v2.W1 keeps the hold installing: the tab is loading and its calls park, bounded. W1's install commit re-evaluates the hold, which binds v2. If W1's install is rejected, the hold is re-evaluated at that end and becomes failed { waiting, version }.Refused broken (contract_mismatch); design §21's updating is not built.§5
14A plugin changed Chat's contract without bumping the version: a tab on v1 (fingerprint A), W1 on v1 (fingerprint B).Unbound version (a fingerprint that differs at the same version), so failed { waiting }. It is not broken: the consumer did nothing wrong, and its next reload or a fixed provider heals it.Refused broken, contract_mismatch.§1
15A reload of chat makes Chat v4 accept v3 but drop v2. Tabs hold Chat at v2 and at v3.Only the v2 holds unbind (version). The object, its v3 holds, documents, jobs and natives stay.Design §21: the version's holds end with gone.§5
16S has no web-shell plugin. Files, artifacts and terminals contribute routes to Http; proxy-marker reads Http.The contributions wait in the table. Proxy-marker's reader is unbound not_served at once after the publication: failed { waiting }. A reload that adds web-shell serves and composes Http, the index wakes the reader, and it goes live.Slice 8 S3: the reader fails transient (unavailable/undeclared) at once and is woken at the first value only.§9
17The codemode plugin is reloaded alone with a HarnessExtensions token v3; the core harness serves v2.Codemode's v3 contribution is left out and reported (contribution_refused). A reader bundling v3 is unbound version: failed { waiting }. A core reload to v3 makes it usable, which wakes the reader.The contribution is refused the same way; the reader is broken (S2).§9
18Core is reloaded alone, serving HarnessExtensions v3 with accepts[2] (contribution and value).v2 contributions are adapted once per value; v2 readers get the composition mapped back once per composition. A reader at v1 (no mapping) is failed { waiting, version }.The same, except the v1 reader is broken (S2).§9
19The harness's HarnessExtensions compose throws at the first composition (an owner bug).The registry is served but has no usable value, so the Harness's request is unbound compose: failed { waiting }. The barrier counts its first outcome. A reload that fixes compose makes the value usable, and the Harness wakes and activates.S3: the Harness fails transient at once, woken at the first good value.§1, §9
20Codemode contributes its sandbox (a native) to HarnessExtensions. Later another contributor's entry makes compose throw, and then codemode restarts.While compose throws, .value keeps the last good composition. When codemode's contribution leaves, that last good value is dropped (it holds the closed sandbox), and .value throws until compose succeeds again. No reader is handed a closed native.Slice 8 keeps the last good value regardless, so a reader could use the closed sandbox.§9
21Transcript (on W1) follows the harness through Harnesses.value.observeConversation(...) and declares retryOn(Harnesses). The harness's Session fails and the harness object restarts.self.follow is Transcript's own step: S1 ends the observer failed, so Transcript fails transient (its own backoff). When the new harness contributes, retryOn wakes it, and it attaches to the new harness and gets one fresh frame.The same (slice 8 S1 and retryOn); unchanged by the model.§3 (follow), §3.8
22A reload of chat swaps the live Chat in place (no natives). Its new factory adds a self.get(Sharing), and the sharing plugin is absent.Owner item 4, recommended: Chat parks. Its holders see failed { waiting, "waiting for Sharing: not served" } and its calls reject until sharing appears. The alternative keeps the old code until the dependency resolves.Not built (slice 7); design §21 swaps the object and opens its gate.§3.9, §13.4
23Chat is parked on DraftStore (1). A chat reload ships code that no longer needs DraftStore.One handover: the new attempt's requests bind first, then the old ones are released. Chat activates on the new code and goes live; the old not_served index entry goes.Not built; today a broken Chat is retried by the reload (slice 7 plan).§3.9
24A reload removes the models plugin from W1 while ModelRuntime is parked on S's ModelConfig (3).ModelRuntime (host-lifetime) ends with its plugin. Its requests end, and W1's refused hold at S is released (a release). Nothing stays waiting.The type's objects end (§21), and its backoff polling stops with it.§3.5, §1
+
+ +
XVII

Scenarios: Rollouts and Cycles

Delayed facets across hosts, rollout windows, chains over three hosts, and the probe at work.

+

Sol's scenarios cover the windows in which hosts run different code: a server facet ahead of its worker facet and the +reverse, tabs older or newer than the server, installs that are queued, staged or rejected, a type moved between host +kinds; then chains across three hosts, diamonds and rings, the probe's own corner cases and a forged probe. Row 20 is a +case round 3 found against the previous draft; the rule that fixes it is in chapter VIII. The pattern: a version that +does not fit is a request waiting for a publication, never an error that ends it; an install answers only for its own +host; and a cycle is found by the attempt whose edge closed it.

+

The Rule column cites sections of the working document, reproduced in appendix C.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#ScenarioUnder the modelTodayRule
1The models plugin's server facet upgrades first. Browser and TUI model pickers still use its older Models contract, which the new provider accepts.Their references remain usable through the old-version adapters. If an owning native restarts, affected references show loading until its fresh binding is ready. A caller that later installs the new contract releases its old request and makes a new one.host/host.ts:#assertMatch accepts only exact versions and fingerprints. Accepted-version serving across reload is not built in next; p2 uses its older cluster reload runtime.§1, §5
2A worker's new search facet makes SearchIndexer depend on SearchFeed v2. The server still serves v1; its search facet installs later.SearchFeed's request is failed { waiting } for the version mismatch. SearchIndexer is also failed { waiting }, without a dependency retry timer. The server's later publication rechecks the request; SearchFeed becomes live and wakes the indexer.A remote mismatch is refused as broken; its dependent fails transient and retries with backoff. See host/attempt.ts and remote/hold.ts; p2's real indexer declares this server dependency in plugins/search/worker.ts.§2–5
3A new web workspace facet arrives before the selected worker's matching Workspace contract. The user opens the workspace panel during this window.The panel's held Workspace reference becomes failed { waiting }; an awaited getter rejects under the recommended getter rule. The old worker object and compatible readers are unaffected. Installing the worker facet wakes the held request and supplies a fresh binding.Exact-match refusal reaches the client; a dependent object can enter R18 backoff. Next has no install-aware wake protocol yet; p2's workspace UI and worker facets live in plugins/workspace/.§2, §5, §8
4A tab sleeps while the worker upgrades Workspace and accepts its old version. The tab reconnects without updating its code.The retained request shows loading while its path is down. Re-holding asks for the tab's original version, and bound supplies the mapped snapshot, returning it to live. No old method index is interpreted as a new one.Path re-holding already exists in remote/path.ts, but unequal versions still fail in host/host.ts. Reload-time adapters are slice 7 work.§1, §5
5The server publishes a new Models contract without accepting the old TUI version. The TUI launcher downloads its replacement bundle later.The old TUI request stays held but shows failed { waiting }. The old runtime releases it when the launcher swaps workers. The replacement runtime makes a new-version request and can become live; the old request is not silently relabeled.p2 polls for TUI bundles and swaps sandbox workers in hosts/tui-client.ts. Next does not yet implement reload-time version recovery.§1, §5, §8
6The server and tabs already use revision R5 when a route starts a worker from an older plugin set. The worker connects before loading its current facets.Business requests stay loading until publication or show failed { waiting } against published incompatible code. The loader installs the desired code set without awaiting activation. Publication then rechecks waiting requests and the actual objects activate; a route must not wait for the worker's activation barrier before admitting its link.p2 sends selected worker facets through WorkerInit and uses a separate control-ready sequence. Next has the publication/activation admission deadlock described in host/host.ts:load and remote/hold.ts:start.§1, §5–6
7A tab installs new workspace code while the worker's matching install is queued but has not started.The old worker answers failed { waiting }, not loading merely because another host knows an upgrade is planned. A held reference survives; a getter rejects. The worker's own install and its completion re-evaluate the request, so it can later become live.Next has no revision/install status handling; it returns an exact-match failure. The model deliberately accepts this rollout-window blip instead of adding revision fields to mismatch messages.§2, §5, §8
8The worker's search facet stages a new contract, so an incompatible request is held as installing. Staging fails and the old search code remains published.The request was loading during the install. Rejection is an install-end event even without publication, so the request is checked against the old provider and becomes failed { waiting } for the remaining mismatch. Its getter rejects rather than waiting forever for a commit that will not occur.Next has no installer. p2's hosts/server.ts:performReload retains published code and bundles when preparation fails, through a different reload protocol.§4–5, §8
9A worker's own envs facet install is rejected while it needs a capability from the new code set. A provider elsewhere becoming ready cannot repair that rejected local install.Requests blocked by the worker's own rejected install fail promptly with its reason. Published old code is not replaced by half-installed new code. A later successful local install or explicit retry is needed; waiting for a remote activation is not an install strategy.Next has no own-install state. Design §21 specifies the prompt own-rejection rule; p2 reports reload refusal while retaining its previous selection.§5
10The worker ports tool facet is added before the server ports facet. The tool is registered successfully, but the first invocation looks up server Ports with ctx.get.The tool owner remains live: this is a call-time lookup, not a declared factory dependency. The missing Ports request is failed { waiting }, so the invocation reports unavailable and its call-scoped request ends. After the server facet publishes, a later invocation can acquire a live Ports object.This is how plugins/ports/worker.ts declares the tool: its callback does ctx.get(Ports). Next's remote lookup currently receives a broken refusal; no provider-publication recovery is built.§1–3, §5, §8
11The server removes Ports first, while old worker tool code and an old browser Ports sheet remain loaded. The worker and browser facets update afterward.The browser's held request becomes failed { waiting } without ending permanently. Ordinary live worker owners stay live; their next Ports lookup fails while the provider is absent. Updating those consumers removes obsolete surfaces, or a later provider publication can restore a still-held request.p2 normally refuses removal when outside-batch consumers still hold a removed type (hosts/server.ts:refuseHeldRemoval). Next's target design originally ended removed-type references; the retained-request behavior is new.§3.10, §5
12Notes(session=7) moves from a server facet to a worker facet. An old tab still computes the server address; a new tab computes worker 7.The old request stays at the old address and shows failed { waiting }; publishing Notes on the worker cannot wake it there. The new tab routes to worker 7 and can become live. Updating the old consumer computes a new address and creates a new request; the model adds no redirect.Placement is already computed by the caller and checked by the provider in host/placement.ts. Next has no relocation reload implementation.§1, §5
13Worker 1 still serves Notes v1 for its own session. Its new code requests Notes v2 placed on worker 2, which already serves v2.Resolution chooses worker 2 before checking its version. The request can become live without waiting for worker 1's unrelated registration to upgrade. Worker 2's bound wakes any dependent waiting on that request.host/host.ts:#assertRoute checks a same-name local registration before Placement.reference, so worker 1's v1 can wrongly reject this v2 request.§1
14The models plugin's server facet updates now, but the worker facet updates hours later. Each host has its own native model registry and compatible local contributors.Each host composes its own table and its local consumers can stay live. A worker does not wait for the server's registry revision: native registries are host-local. Only an actual remote object contract used between those facets creates a cross-host version wait.p2 also has independent server ModelConfig and WorkerModels resources (plugins/models/server.ts, worker.ts). Next's registry adapters already handle local reader/contributor versions; atomic registry replacement is not built.§5, §9
15A tab Panel depends on a worker SessionView, which depends on a server Catalog view. Catalog's new facet arrives last.Catalog's missing request is failed { waiting }; SessionView and Panel show waiting with the nearest blocker and root cause. The server publication binds Catalog and makes it live. Its state change wakes SessionView; that provider's bound then wakes Panel, with no dependent backoff loops.R18 retains dependency references but fails dependents transient and uses identity backoff. host/object.ts:dependencyLive already skips that backoff on a live event; registration-based recovery is missing.§1–4
16A server Summary waits for views on two workers. Both views reach a shared local CatalogStore on the server, whose own required provider is absent: a diamond, not a cycle.CatalogStore is failed { waiting }, and waiting propagates through the views to Summary. Probes reach the common store, but its unavailable request has no target attempt and no edge back to Summary. It must not report invalid/cycle; adding the missing provider later wakes the chain.Current local waiting-edge traversal and trusted activation chains distinguish cycles from shared dependencies. Dependency failures still use R18's transient/backoff path (host/object.ts, host/attempt.ts).§3–4, §7
17Thirty-three object attempts on trusted server/worker links form a ring and start together. Each reference resolves to the next attempt in that ring.All participants stay pre-activation, showing loading or waiting according to their blockers. With answer (A), the attempts waiting across links time out after readyTimeout, retry with backoff, and the activation chains of the later attempts find the ring. With answer (B): the attempt whose edge closed the ring last starts a search; it returns, a check along its recorded path returns, and the initiator fails broken invalid/cycle; propagation breaks the others and the dependents off the ring. The probe carries its trail of 33 hops; no cap cuts it.Concurrent cross-host cycles can escape fixed activation chains and rely on readyTimeout/backoff. host/object.ts:assertNoCycle and design §6 document that limitation; next has no probe.§3.1, §7
18A ring is being confirmed when one worker reloads its facet and removes its outgoing dependency, repairing the ring. Some probe returns are already in flight.With answer (B): New attempts have new ids, and obsolete target ids are discarded. If the check passed before the repair, the ring is reported (it existed when the check began); if the repair came first, the check dies at the replaced attempt and nothing is reported. Unchanged attempts can therefore become broken invalid/cycle and need ref.retry() or their own plugin reload; provider readiness alone does not retry broken objects.Next has no probe or reload implementation. Broken cycle failures already require retry, and host/object.ts:retry starts another attempt.§3.6–9, §7
19A malicious tab learns a worker attempt id and sends a forged probe naming that same attempt as its initiator through the server relay.With answer (B): The server drops the probe because it arrives on a child link, before any forwarding. The worker object and legitimate references keep their states. A child link never carries cycle evidence.There is no probe message today. remote/provider.ts already strips activation chains from untrusted holds rather than trusting a child to declare ancestry.§7.2
20A→B→C wait while C's return path to A is disconnected. Sixty-five unrelated initiating rounds pass through C before reconnect makes C→A a known edge.With answer (B): Found in round 3 against draft 3 (C could have evicted every round from the ring). Under the model the re-hold gives C a new edge token, so C starts its own search; it crosses A and B and returns to C, the check along its path confirms, and C fails broken invalid/cycle. Foreign rounds and evictions play no part.Current re-hold carries an activation chain (remote/remote-object.ts:attach), which can detect a small newly reachable ring; otherwise the dependency deadline remains its fallback. No probe-round cache exists today.§7.2–3
+
+ +
∴

Open Questions

What the owner decides, and what slice 7's spec must still pin down.

+

The model replaces five answers to "the provider is not there" with one: the request stays, the dependent parks, an +event wakes it. It ends the startup deadlock with a gate that lasts only for the synchronous setups, finds cycles with a probe +instead of a deadline (or, if the owner prefers, one narrowed deadline), and keeps timers only where no event can exist. Four reviewers played more than 300 +scenarios against five drafts; the catalog in the chapters before this one is a selection.

+

Decisions for the owner

+
    +
  1. The barrier (chapter VI): (a) Chord's load waits for the first outcomes of this host's host-lifetime objects, +with the local-blocker rule; or (b) load resolves after the publication and the app reads host.startup. +Recommended (a); Astra, Opus and Sol for (a), Fable for (b).
  2. +
  3. Getters on waiting (chapter IX): reject at once (recommended) or wait up to readyTimeout.
  4. +
  5. waiting as a cause kind (chapter III): a new kind (recommended; Opus, Fable), transient without retryAt +(Astra), or loading with a diagnostic (Sol).
  6. +
  7. A pure swap whose new factory needs a dependency that is not there: the object parks (its gate closes, its +holders see loading or failed { waiting }, calls park or reject; recommended), or it stays on the old code until +the dependency resolves (old and new code would then overlap without bound, against §21's "a host is at R or R−1").
  8. +
  9. The last good value (chapter X): kept only while every contribution it was built from is still in the table +(recommended; Astra), or kept as slice 8 decided, with natives in a registry looked up at use through their owner +(Sol).
  10. +
  11. Cycles across hosts (chapter VIII): (B) the probe, with (B1) a re-search backoff that sends probes only +(recommended) or (B2) a full search lifecycle with no timer at all; or (A) the slice 4b decision narrowed to one +deadline, which polls chains across hosts again, misses a ring of waiting attempts, and retries in lockstep without +finding the cycle it is for.
  12. +
+

For slice 7's spec

+
    +
  • The test list: the startup deadlock resolving without readyTimeout; an early hold placed here, not through the +parent; remove-then-serve with a live dependent's calls recovering; a 1,000-hold publication wave under the window; +the probe on long rings, diamonds, edge changes, reloads, relays, reconnects and forged probes; GC of a parked +three-level chain after its top is released; every index entry gone after release, revocation and link loss.
  • +
  • The local re-binding of a Binding (member sources, keyed pins, parked calls, presence) and the unbinding of a live +hold under flow control, both with their race tests.
  • +
  • The wire: waiting in the failed cause schema, the attempt id on loading and failed, the probe message, the +pinned list; PROTOCOL_VERSION stays 1 until p3 ships.
  • +
  • The design changes listed in appendix A, folded into design.md and the decision log in the same commit as the +owner's decisions.
  • +
+
+ +
A

What Changes

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
WhereTodayUnder the model
R18, activationa dependency that does not go live within readyTimeout fails the dependent transient; backoff 1 to 60 sthe dependent parks with no timer; failed { waiting } when a dependency failed; woken on every state change of a dependency; the deadline paused while parked
R18, the boundary"live" means every dependency was live at the end of activationthe dependency check is the boundary: after it, a dependency leaving live is treated as after live
self.get of a missing local typethrows in the factory: brokenunbound not_served: failed { waiting }, woken by the publication that serves it
a remote hold the provider cannot serverefused broken; the dependent polls by backoffrefused and kept at the provider, re-run at its publication; failed { waiting }
design §3, placement"served here" decides "here"the address decides; a remote type without placement keeps today's rule
the start barrier, pendingLoadholds wait for all of loadholds wait only for the publication; the barrier is (a) or (b)
S2, a registry reader newer than its providerbrokenfailed { waiting, version }
S3, a registry without a valuetransient, woken at the first value onlyfailed { waiting }, woken at every transition to usable
slice 8, the last good valuekept whenever compose throwskept only while every contribution it was built from is still in the table
design §6, cyclesa walk on one host, chains across hosts, a documented gap closed by the deadlinethe gap of simultaneous starts across hosts: (A) one narrowed deadline, or (B) a probe: a search per edge change, a check along the recorded path, edge tokens, dead notices
design §7, host.holdthrows for a missing local type after loadalways returns a reference
design §7, getterswait through every failure up to readyTimeout, reject on brokenwait through loading and transient; reject at once on waiting and broken
design §21, removalgone to every holderfailed { waiting, not_served }; the request binds again when the name is served
design §21, a dropped versioncontract_mismatch, finalonly that version's requests unbind; the object stays
design §21, rolloutupdating with revision fields and a deadlineeach host knows its own install; installing re-evaluated at the install's end; installs never await activations
FailureCausetransient, brokenadds waiting
the wirefailed cause waiting; the attempt id on loading and failed; with (B), one new message, probe (search, check, dead)
unchangedan object's own code reload (§21); R18 after live; retryOn outside parking; self.follow's R4; the call window and flow control; revocation's semantics; no client providers
+
+ +
B

Glossary

+

Attempt: one try at activating an object: its factory, requests, natives; its own id since the model. Bound: a +request pointing at an object here or a hold over a path. Dependency: a request an attempt needs live before it +may go live (self.get, self.registry, a keyed pin). Dependency check: the moment all of an attempt's +dependencies are live; after it the attempt runs its own steps. Edge generation: a number of a wait edge, incremented by its +source whenever the request binds anew. First outcome: an object is live, failed by its own cause, or parked with no local +blocker still in its first attempt. Holder: whoever made a request and alone ends it. Parked: an attempt +waiting for a dependency, with no timer. Probe: the message that walks wait edges to find cycles: a search, then a check along the recorded path. Publication: +a host making its facets' declarations visible: load's setups, a reload's commit, an install's commit. Refused: a +request without authority or with invalid params; never indexed. Request: what a reference stands for: type or +registry, params, principal, version. Round: the number of one search, from a counter per attempt. Seen set: the newest round number an +attempt has forwarded, per initiator (up to 256). Unbound: a request with no provider at the moment, with a reason (unpublished, not_served, +version, compose, installing, no link). Waiting: the cause of a failure that is not the object's own, woken by +an event. Wait edge: a parked attempt waiting for another attempt. Wake: an event that makes the resolver look +at a request again.

+
+ +
C

The Working Document

+

The proposal as the reviewers checked it, /tmp/model/model.md, draft 4. Its section numbers are the ones the +scenario catalogs cite.

+

Status: proposal for the owner, not decided. Built in slice 7 with reload and rollout. Sources: the owner's direction +("park until resolved"; "no difference between local and remote resolution: a link going is like a plugin +unloading"), the reviews (/tmp/dep-mechanism-*.md, /tmp/dep-graph-*.md, /tmp/barrier-*.md, +/tmp/model/review{1,2}-*.md), the code at pi 358f7d46a. Owner marks an open decision.

+

Draft 7 (round 6): (A) also retries in lockstep without finding the cycle (Fable); the probe's recovery after a +suppressed search becomes a choice between (B1) a re-search backoff for probes only (recommended) and (B2) a full +search lifecycle without timers, since dead notices alone need recovery state that eviction may not drop (Astra, Sol). +Causes are compared by value on wakes (Opus). +Round 7: a returned search is accepted while its initiator is parked, whatever its round (Astra: re-searches must not +invalidate searches under way); detection is eventual, not within one backoff period; the local walk stays (Opus).

+

Draft 6 (round 5): cycles across hosts become an owner choice between (A) the owner's slice 4b decision (simultaneous +starts across hosts time out, no probe message), narrowed to one deadline, and (B) the probe, completed with edge +tokens, no revisits, and dead notices back to the initiator (Astra: a check that dies silently can lose the only +search that survived dedupe). (A) turned out to either poll chains across hosts again or miss a ring of waiting +attempts, so (B) is recommended, pending its review.

+

Draft 5 (round 4, Sol's phantom cycle): the second round is replaced by a check along the recorded trail of +(attempt, edge generation) hops; a reported ring existed at one moment; probes grow with their path.

+

Draft 4 after round 4 (probe only): round numbers per attempt, forwarded only when newer than the stored one; probes +honoured on parent and route links, dropped on child links; completeness stated under an eventually stable graph; the +repair caveat broadened to any concurrent edge removal; a new edge generation resets confirmation; a bounded display +trail.

+

Draft 4 changes (round 3): the probe starts a fresh round on every new or changed wait edge (a ring closes at its last +edge, whose source finds it), keeps one seen entry per initiator, accepts only its current round, and states its +cost honestly; every host drops probes from untrusted links; local forwarding is iterative; no-placement remote +requests are re-placed when a host's first parent link connects; a cohort object that ends before its first outcome +settles the barrier; the last-good refinement is marked for the owner.

+

Draft 3 changes (round 2): remote types without placement keep today's fallback (here if served, else the parent); +hydration stays before parking; no copied retryAt; a dependency ended breaks its dependent; a reload's handover +binds new requests before releasing old ones; retry passes only to broken and transient blockers, once per +operation; installing re-evaluated at every install end (commit or rejection); a registry's last good value dropped +when one of its contributors leaves; the probe with round ids, bounded seen sets, edge-generation triggers, attempt +ids on every non-live answer and relays dropping probes from untrusted links; one cycle mechanism, local and remote.

+

0. The problem

+

The same situation, "what I need is not there", has five outcomes today, depending only on how it is declared:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Declared asProvider missingToday
self.get(LocalType)no facet serves itself.get throws, the factory throws: broken
self.get(RemoteType)the worker does not serve itthe hold is refused broken; the dependent transient, polled 1-60 s forever
self.registry(R)no providertransient, woken at R's first value only
self.registry(R), reader newerbroken
a reload removes the typeholders get gone: ended for good
+

And the startup barrier deadlocks an acyclic graph across two hosts (Sol, /tmp/dep-graph-sol-barrier.log): S's eager +A needs W's B, B needs S's C; W's hold for C waits for S's whole load, which waits for A. Only readyTimeout ends it.

+

1. Requests and one resolver

+
    +
  • A request is what a reference stands for: type (or registry), params, principal, the consumer's contract version. +Its holder makes it and owns it: an attempt (self.get, self.registry, keyed pins), a call (ctx.get), outside +code (host.get, host.hold, host.registry), a link's peer (hold). It ends only when its holder ends it (the +attempt ends, the call settles, release(), a getter gives up, the peer's link or origin closes, revocation, the +holding host closes), never because its provider is missing, went away or does not fit.

    +
  • +
  • Placement by address (round 1): a local type goes here; a remote type with a placement goes to the address +its placement (computed from the consumer's own contract) gives: here if that is this host's address, whether served +here or not, else over the route for that kind or the parent (design §3). A remote type without placement keeps +today's rule (round 2: Fable, Opus, Astra; no feature removed): here if served here, otherwise over the parent. A +request of that kind made before the host has a parent link waits here not_served and is re-placed when the +host's first parent link connects (round 3: Opus N5, Sol 3: a tab loads before it connects); on a host that never +connects (the root), it stays here. Opus would rather require a placement on every remote(); that removes a +supported declaration, so the draft keeps the fallback. A hold arriving for an address that is not this host's is relayed or +refused (invalid/placement) as today; a relay never captures a hold because it serves the same name.

    +
  • +
  • Then the selected provider is checked (served, version or accepted version and fingerprint, authorize, the +provider's own placement check). Version checks never look at a same-name registration elsewhere.

    +
  • +
  • The publication gate: a request made on this host, or arriving on its links, before the host's first +publication (§6) is not placed yet; it shows loading and is placed by the publication. The gate lasts only for the +synchronous setups, never for activation, so it cannot deadlock.

    +
  • +
  • Outcomes of the check:

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    The check findsRequest
    a fitting providerbound (to the object here, or to the hold over a path)
    not published yetunbound unpublished: loading
    nobody serves the name at its addressunbound not_served: failed { waiting }
    no fitting version, a fingerprint that differs at the same version, a placement the provider computes elsewhere (code skew)unbound version: failed { waiting }
    the registry is served, but its first compose (or the reader's version mapping) threwunbound compose: failed { waiting }
    the provider host is installing a revision and the hold does not fit its current codeunbound installing at the provider: loading
    the path has no link (a drop, a route resolving)on the path: loading
    authorize refused, invalid paramsfailed { broken, forbidden | invalid }, as today, never indexed
    +

    The unbound reasons that wait for a name (unpublished, not_served, version, compose) are indexed under it on +the host that decided them: the consumer's host for a local request; the provider's host for a hold that arrived +over a link (it already keeps such holds as refused: remote/hold.ts). installing is the provider host's own, +re-evaluated at its install's end; no_link lives on the path, where the next link holds it again. Indexes drop an +entry when its request ends; GC-tested.

    +
  • +
+

2. States

+

ref.state keeps its four kinds; FailureCause gains one kind, waiting:

+
type RefState =
+  | { kind: "loading" }
+  | { kind: "live" }
+  | { kind: "failed"; cause: FailureCause; retryAt: number | undefined }
+  | { kind: "ended"; error: ChordError };
+
+type FailureCause =
+  | { kind: "transient"; error: ChordError }   // the object's own effect failed (an open, a follow): backoff
+  | { kind: "broken"; error: ChordError }      // the object's own code, a cycle, or authority: retry() or a reload
+  | { kind: "waiting"; error: ChordError };    // not the object's fault: no provider, no fitting version, a
+                                               //   dependency failed; retryAt undefined, woken by an event
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
The request isref.stateCalls
bound, its object activating or parked with no failed blockerloadingpark, bounded
unbound unpublished or installing, or its path has no linkloadingpark, bounded
unbound not_served, version or composefailed { waiting }reject at once
bound, its object failedfailed with the object's cause, as todayreject at once
bound, its object liveliverun
ended by its holder, revoked, or its host closedendedreject
+

Owner: waiting as a new cause kind (Opus, Fable; recommended: the cause says "not your fault, an event will wake +you" and a UI can say what is missing) or no new kind: transient with no retryAt (Astra; today's wire already +carries a transient cause without retryIn, remote-object.ts), or loading with a diagnostic (Sol; parks every +call on a missing plugin for its full readyTimeout).

+

3. Dependents: parked until their dependencies resolve

+

A dependency is what an attempt declares and needs before it goes live: self.get (local or remote), +self.registry, keyed pins (at(key) on another object's keyed member). self.follow is the attempt's own step (its +attach runs Durable's code), not a dependency; a follower declares retryOn on the registry it reads the harness +through, or recovers by its own backoff.

+
    +
  1. Order, as today up to the check: the factory runs; persisted singletons hydrate (the object's documents, kept +across attempts); the requests are made. The attempt is parked while any dependency is not live (hydrated):

    + + + + + + + + + + + + + + + + + + + + + + + +
    Its dependenciesThe dependent shows
    any loading, none failedloading
    any failedfailed { waiting, "waiting for X: <X's cause>" }, retryAt undefined; the message keeps the nearest blocker and the root cause within the 500-character rule
    one failed with invalid/cycle (a dependent off the ring)broken with the same error: it would wait on a ring that never heals by itself
    one ended (revoked, or its host closed)broken with that error: the request cannot come back; retry() or a reload makes new requests (a guard: a dependency ends only with its holder's principal or host, so the dependent ends too; Fable)
    +
  2. +
  3. The dependency check is the activation boundary: once every dependency is live, the attempt runs its own +steps (natives in order, follow attaches, member checks, derives), bounded by its own readyTimeout deadline, and +goes live. A dependency that leaves live after the check is treated as after live (R18): the attempt goes on. This +is a deliberate rule (Astra: a weakening of "live means every dependency is live"): an object can finish opening +after a dependency went away, and its calls through that reference meet its state. So a parked attempt holds no +natives, and an attempt that opened natives is never parked.

    +
  4. +
  5. No timer while parked: no dependency deadline, no backoff. The attempt's deadline is paused while parked (as +eager migrations pause it) and resumes with the budget left.

    +
  6. +
  7. Wakes, one rule: the dependent re-evaluates on every state change of any of its dependencies (live, loading, +failed, a new cause compared by value, ended; Opus: otherwise a ring of waiting attempts passes its causes round and +round), updating what it shows; it leaves parking when all are live. The factory never re-runs +while parked. Wakes are coalesced per object and run after the batch that caused them; a change during the +attempt's own steps or its teardown is remembered.

    +
  8. +
  9. Kept while parked: its documents (the object's), its declarations and requests, its seal token; no natives. +Ended with the object (its last holder; a host-lifetime object when its plugin is removed or the host closes).

    +
  10. +
  11. broken only for the object's own code (a throwing factory, a derive throwing at activation, hydration, +migrate, a document fingerprint mismatch, an open throwing broken(...)), cycles (§7) and authority +(forbidden, an ended dependency).

    +
  12. +
  13. Retry passes on: ref.retry() on a failed { waiting } object retries its broken and transient blockers, +hop by hop (the existing retry message over links), once per retry operation (an id dedupes diamonds); waiting +blockers pass it on; nothing else is touched. Retry of a broken or transient object starts a new attempt, as +today.

    +
  14. +
  15. retryOn wakes failed objects as decided in slice 8 (a value call retries a transient or broken object); +while the object is parked waiting, a value call does nothing: its dependencies decide.

    +
  16. +
  17. Own-code reload of a parked object: its plugin's reload replaces the parked attempt with the new factory's, +as one handover: the new requests are made and bound first, then the old ones released (round 2, Astra N4: an +unchanged dependency held only by this object must not end in between; today's #activate binds before +#releasePrevious).

    +
  18. +
  19. After live, R18 unchanged: a dependency leaving live leaves its dependent live.

    +
  20. +
+

4. Wakes: two events

+ + + + + + + + + + + + + + + + + + + +
EventRe-evaluates
a target changes state: a dependency's object goes live, loading, fails, gets a new cause or ends; a hold's bound, loading, failed arrive, also after a re-hold on a new linkthe requests bound to it, and their dependents (§3.4)
a name becomes usable, an install ends, or the first parent link connects, on a host: a publication (load's setups, a reload's commit, an install's commit) serves a type, a registry, a version; a registry composes or maps a value at a version; an install ends without a commit (rejected)the unbound requests indexed under those names; the provider's installing holds; at the first parent link, the
no-placement remote requests waiting here; a refused hold re-runs its start and sends loading, then bound or failed
+

No new message kind for wakes: loading, bound, failed and retry exist. The wire's failed carries the new cause +kind waiting; loading and a waiting failed carry the provider's attempt id for the probe (§7).

+

5. Removal, versions, rollout

+
    +
  • A reload that removes a type: its objects end as today (natives close, reason end; callers rejected at once; +handlers drain). Their requests are not ended: each becomes unbound not_served and binds again if a later commit +serves the name.
      +
    • Locally: a Binding whose target ended by removal becomes unbound and accepts a second bind: member sources +re-bind by name (one frame from the last value, as on a restart), keyed pins re-pin, parked calls flush or +reject, presence re-lists. The one new piece of machinery.
    • +
    • Over a link: a failed on a bound hold unbinds it (Fable: one message, one rule; the consumer applies it as +#startOver plus failed); the provider detaches the hold's sources and drops its unsent pending ops and queued +bound; the ack slots of bound/state messages already sent stay counted until their ack or the link's loss; +calls in flight on it are rejected and free their slots once.
    • +
    +
  • +
  • A reload that stops accepting a version unbinds only the requests on that version (version); the object stays +(its own plugin's reload still restarts it if it has natives, §21).
  • +
  • A type moved to another host kind: the request carries the address its own consumer code computed; the old host +answers not_served; the consumer's own reload makes a new request. No redirect.
  • +
  • A consumer whose code moves to a new version releases its old request and makes a new one (today).
  • +
  • Rollout, without revision fields: each host knows only its own install. A provider host that installs keeps +holds that do not fit its current code installing (loading) and re-evaluates them when the install ends, by its +commit or by its rejection. A provider host that is queued but not installing yet answers a mismatch failed { waiting, version } until its install commits and wakes it: a visible blip of at most one install, accepted (Opus). +An install never waits for activations (a reverse-order rollout would deadlock); a host whose own install was +rejected fails its own requests at once with the reason (design §21). The loader keeps its own duties (the desired +code set, digests, serialized installs).
  • +
  • Old calls across a reload follow §21 unchanged: a pure swap lets them finish on old handles; a native restart +drains them first; a removal rejects their callers at once and drains the handlers. Calls are never resent.
  • +
+

6. Startup: publication, then first outcomes

+
    +
  1. Publication: load runs every facet's setup, then publishes all at once in one batch (types served, +registries served and composed once, contributions in their tables), then wakes (§4) after that batch. Nothing +activates inside it.
  2. +
  3. No gate beyond it: holds and host.hold wait only for the publication, never for the barrier. This ends the +startup deadlock: W's hold for C binds as soon as S's setups published; B goes live, then A.
  4. +
  5. The barrier (Owner: (a) or (b); recommended (a), three reviewers of four):
      +
    • (a): load resolves when every host-lifetime object of this load has a first outcome: +firstOutcome(o) = o.live || o.ownFailure || (o.parked && o.blockers.every((b) => !b.local || firstOutcome(b))), +where local means executing on this host (an object here, a keyed pin still loading here), not the contract's +kind. So CodeMode → Harness waits for the harness's outcome; a wait on another host or on a failed dependency +ends the first pass. One firstOutcome signal per object, separate from the readiness dependents wait for (Sol), +counted once, never reopened by retries. An object of the cohort that ends before its first outcome (its plugin +removed, the host closing) settles its entry as ended; host.close settles the waiting load (Astra R3).
    • +
    • What (a) promises: every host-lifetime object on this host had its first try, and the contributions of those +that went live are in their tables. Not: every tool that will arrive (a parked or slow contributor, tools found +in onLive, MCP; Sol's probe /tmp/barrier-sol-first-outcome.log). What a first turn needs is p3's explicit +admission check.
    • +
    • (b): load resolves after the publication; Chord adds host.startup, a watchable of the host-lifetime +objects' first outcomes with their causes; the app decides (Fable). The same tracker, awaited by the app.
    • +
    • p2 today resumes its harness without waiting for all host objects (apps/p2/src/hosts/worker.ts), because its +worker objects that need server types are reachable only after control.ready().
    • +
    +
  6. +
+

7. Cycles

+

On one host, cycles are found exactly as today: at self.get, following the waiting edges of activating and parked +attempts. Across hosts, a hold carries the activation chain it was made in (today), which finds every cycle whose +objects start one after another. What remains is the documented gap (design §6): objects on two hosts that start at +the same moment and need each other. The owner decided on 2026-10-24 (slice 4b review decisions (2)) that this is a +documented limitation that times out and retries, with no probe message. Without dependency deadlines that timeout no +longer exists, so the gap needs one of two answers (Owner):

+
    +
  • (A) The 4b decision, narrowed. One deadline stays: an attempt parked on a dependency held over a link whose target +attempt is known and still activating or parked fails transient after its readyTimeout and retries with backoff; +its next attempt usually starts after the other side and its chain finds the cycle. It has a cost that decides +against it: the target being parked is the common case of a chain across hosts waiting for a missing plugin (a +tab's view on a worker object that waits for an uninstalled type), and those dependents would time out and poll +again, as today, so "local = remote" would not hold across hosts. Narrowing it to targets that are loading (not +failed { waiting }) avoids that but misses a real ring: A and B on two hosts wait on each other, B also on a +missing X; B is failed { waiting }, so A is too; when X appears, both stay failed { waiting } on each other, with +no deadline and no chain to find it. And even the timeout does not find what it is meant to (round 6, Fable): an +object's chain is fixed when it is created and its instance id survives retries, so a retried attempt sends the same +hold with the same chain; two simultaneous starts time out and retry in lockstep and are never found.
  • +
  • (B) The probe (recommended), below: no deadline at all, one new message kind, the most intricate rule set in the +model. Five +review rounds found a counterexample in each of the first four drafts; draft 6's version answers all of them, but it +has not been reviewed again, and slice 7 would have to prove it with tests before the deadline could go.
  • +
+

(B) The probe

+

Replaces the activation chain in holds. On one host the self.get walk stays (it is synchronous and exact: a ring +with no remote edge is found there, where (B1)'s timer does not run; Opus, round 7); the probe serves the rings that +cross a link. It replaces draft 1's bounded chain too +draft 1's bounded chain (a ring of 33 is never found with chains cut at 32; chains keep stale ancestry). Draft 5 replaces the +second round by a check along the recorded path (round 4, Sol: two rounds through different stale paths can report a +ring that never existed, /tmp/model/sol-probe4.log).

+
    +
  1. Attempts, edges, tokens. Every attempt has its own id (uuidv7 per attempt: a retry or reload is a new +attempt). A wait edge runs from a parked attempt to the parked or activating attempt it waits for, through a +bound request (unbound waits and registry reads are never edges; they are not deadlocks). Each edge carries an edge +token (round 5, Astra, Sol): a number from a counter per attempt, never reused, replaced by a fresh one on any +interruption or change of that edge, so two requests to one object are two edges: a new target attempt (also a +local target's retry, without a re-bind), any loading on a bound or pending hold (a relay's upstream re-hold +included), an unbinding, a rebinding, a re-hold after a reconnect. An attempt is parked for one interval only (one +that passed its dependency check never parks again) and ids are never reused. A provider tells its consumers the attempt they wait for on +every non-live answer: loading { hold, attempt } and a waiting failed { hold, cause, attempt }.
  2. +
  3. A search per edge change. Whenever a parked attempt gets a new edge or a new token on one, it starts a +search: probe { kind: "search", round, initiator, trail } along each of its wait edges, round the next number of +a counter the attempt keeps, trail the list of (attempt, edge token) hops it crossed, starting with +the initiator's own. A ring closes at its last +edge, and that edge's source starts the search that finds it. Coalesced per batch. Locally a call on the target +attempt, forwarded through a queue (no recursion); over a link the message probe { hold, to, ... }, consumer to +provider, relayed by hold id with the origin. Probes are honoured on parent and route links and dropped on child +links (accept), whatever trusted says.
  4. +
  5. Forwarding. An attempt receiving a search drops it if to is not its current attempt or it is not parked. If +the initiator is itself, the search returned (4). If the attempt is already in the trail, it drops it (a ring that does not contain the initiator is +found by its own members' searches; Opus). Otherwise it forwards it along every wait edge, appending (itself, the edge's token) to the trail, only if the round number is higher than the one it stored for that +initiator (older or equal rounds are dropped: the initiator superseded them), and stores it. The seen map holds one +number per initiator, up to 256 initiators, and goes with the attempt.
  6. +
  7. The check. A returned search is accepted while its initiating attempt is still parked, whatever round it +carries (round 7, Astra: a re-search must not invalidate an older search still under way); the check decides. The +initiator then sends probe { kind: "check", round, trail } along the recorded trail: the initiator first verifies its own hop; each hop passes it on only if it is still +that attempt, still parked, still has the recorded edge token, pointing at the next hop's attempt; otherwise the +check dies (also on a to mismatch). The check follows the trail by position and bypasses the seen +map. The initiator verifies its own hop again when the check returns. If the check returns, the initiator fails broken invalid/cycle, the path named for display from the trail, +and today's propagation breaks the other activating objects on the ring and the dependents off it. 5. No silent loss. Dedupe can drop the copy of a search that travelled the ring in favour of a copy whose path +later breaks (Astra, round 5); then the check dies and nothing is left to find the ring. Round 6 tried dead +notices back to the initiator; Sol and Astra showed that reliable notices need recovery state per search that +eviction may not drop and that a long-parked attempt would otherwise keep for every initiator that ever passed (a +leak), i.e. a full search lifecycle with cancellation. Two ways (Owner, part of decision 6):
      +
    • (B1) Re-search with backoff (recommended). An attempt parked on a remote attempt starts a fresh search again +after a backoff (1 s doubling to 60 s, reset by an edge change), whether it shows loading or failed { waiting }; +the timer stops when the attempt leaves parking. No dead messages, no recovery state, eviction +only costs work; a new search starts within one backoff period, so every stable ring that crosses a link is +eventually found (no wall-clock bound: delivery is only fair; a ring on one host is the walk's). The timer runs as long as the attempt stays parked on a remote attempt: a view on a worker object that +waits for a missing plugin re-searches every 60 s until a reload (Fable: cheap, but for as long as it waits). The timer sends probes only: it never +fails an object, re-holds or polls a provider. Cost: one probe per parked remote dependent per period, which +crosses the downstream wait graph and dies at its leaves (an unbound wait is not an edge).
    • +
    • (B2) Events only. Each search is owned until it settles: branches report completion or abort back along their +trails, a dropped probe answers dead { initiator, round } with a reverse route, an edge records the searches it +carries until they settle, and an initiator that ends or starts a new round cancels its old one along the same +routes. Complete without a timer, bounded by the searches in flight, and the most intricate protocol in Chord; not +yet specified to the reviewers' satisfaction. +Rejected: enumerating every simple trail without dedupe (Sol's correctness reference): no ring copy is ever +suppressed, but work is exponential on chains of diamonds.
    • +
    +
  8. +
+
    +
  • Why a reported ring existed: every edge of the trail kept its token from the moment the search crossed it +to the moment the check crossed it; the search crossed all of them before the check started, so at the moment the +check started every edge of the ring existed at once. A token changes on every interruption and is never reused, so an edge cannot +vanish and come back unnoticed. Sol's counterexample fails at the check: C1 is no longer B's target.
  • +
  • Why every ring is found (under an eventually stable wait graph and fair delivery; endless edge churn can keep +invalidating searches; with rule 5's (B1) or (B2) for a search suppressed behind a path that broke): a ring is closed by its last edge; its source starts a search when the edge appears; every +other edge exists then and every attempt on the ring waits transitively on the source, so none goes live before the +search and the check return. Rings of any length, local and remote mixed. On a DAG searches die at the leaves; on a +diamond the second arrival is dropped.
  • +
  • What it does not promise: (1) a ring that existed when the check started may be broken up by a concurrent reload, +retry, unbinding or revocation just after; it is still reported (it did exist). Its broken objects recover as every +broken object does: ref.retry() or a reload of their plugin. (2) Linear work under every schedule: with more than +256 initiators passing one attempt at once, an evicted entry lets a search be forwarded again, so work can grow with +the number of simple paths (never endlessly: a search never revisits an attempt in its trail). (3) Constant message size: a probe carries its trail, so its size grows with the length of the +path it crossed (a returning search's trail is as long as its ring); no cap cuts it, since the trail is the proof.
  • +
  • Cost: one search per edge change, crossing each downstream edge once, one check per returned search; with (B1), +one more search per backoff period for every attempt parked on a remote attempt, also while an acyclic chain waits +for a missing plugin, plus the trail bytes. The bounded memory is the seen map's; probes in flight are bounded by +the searches started.
  • +
  • Transition: today's dependency deadline for activations waiting across links stays until the probe's tests pass +in slice 7 (long rings, diamonds, a ring closed after a reconnect with unchanged attempt ids, Sol's phantom schedule, +300 concurrent initiators, edges added and removed, reloads, relays, forged probes); then it goes in the same slice.
  • +
+

8. Getters, holds, calls: one rule

+
    +
  • host.hold returns a reference at once; only invalid params throw (also before load). A missing local type +gives failed { waiting } after the publication, loading before it.
  • +
  • host.get, ctx.get and host.registry: resolve when live; wait while loading, or failed { transient } +(its own retry is under way), up to their readyTimeout; reject at once on failed { waiting } or { broken }. On +success, rejection or timeout their temporary wait goes; host.registry returns the RegistryValue with its +ordinary watch ownership (no release()). Owner: recommended (nobody waits 60 s for a plugin that is not +there; code that wants to wait holds instead), or wait through waiting too.
  • +
  • Calls on a loading reference park, each bounded by the reference's readyTimeout from when it was made; on a +failed one they reject at once with the cause; sent calls are never resent.
  • +
+

9. Registries

+
    +
  • The same resolver on one host: a reader's request for R at its version is bound while R is served and has a usable +value at that version (composed; mapped by accepts[old].value for an older reader), otherwise unbound +not_served, version or compose, or loading before the publication. Woken at every transition to usable +(#valued's first-value latch goes). S2's broken and S3's backoff go.
  • +
  • Version skew within one host: a reload of the registry's plugin alone leaves other plugins' contributors and +readers on their bundled versions; accepts[old].contribution and .value keep them working, unsupported +contributions are left out and reported, unsupported readers are waiting (version).
  • +
  • A registry waits for a value, never for a particular contribution (an empty table composes). After live, a later +compose failure changes nothing for the reader: .value keeps the last good composition, but only while every +contribution it was built from is still in the table (round 2, Astra N2): when one of them leaves (its +contributor ended or was reloaded), the last good value is dropped and .value throws until compose succeeds, so +it never hands out a native whose contributor ended. compose represents a missing contributor as a value.
  • +
  • Registries stay host-local (contributions are in-process values).
  • +
+

10. Keep-alive, presence, revocation, jobs

+
    +
  • A request over a link counts once in host.activity.references from its arrival to its end, bound or unbound; +presence (host.references) lists only bound references.
  • +
  • Revocation ends the requests of a revoked principal, bound and unbound, at every hop; nothing revives them.
  • +
  • gone stays only for revocation (a provider host closing closes its links and sends nothing; its consumers see a +drop, loading, and hold again on the next link).
  • +
  • Persisted job runs of a parked host-lifetime object are loaded with its documents; run and finish start only +once it is live; the run count keeps the object as today.
  • +
+

11. Timeouts after the model

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Timeout or backoffAfterWhy
readyTimeout of an attemptonly its own steps; paused while parkeda slow dependency is bounded where it runs
readyTimeout of host.get, ctx.get, host.registrykepta caller gets an answer
readyTimeout of a parked callkeptan action must not run long after it was asked for
identity backoff 1-60 sonly for the object's own transient failuresnothing else says the disk or network is back
dependency backoff, polling of refused holdsremovedrecovery is an event
dependency deadline across links(A): only for a parked attempt whose remote target attempt stays activating or parked; (B): until the probe's tests pass, then removedthe simultaneous-cycle gap
probe re-search backoff (B1)new, only with (B1): 1 s doubling to 60 s while parked on a remote attemptfinds a ring whose search was suppressed; sends probes only, never fails or re-holds
route backoff, heartbeat, closeTimeout, routine and finish backoffs, lingerkeptnot dependency waits
rollout install timeout (not built)a watchdog on the install onlynever a deadline for dependencies
+

12. What changes in the design

+

R18 (dependency failures waiting, no backoff, the deadline paused while parked, the dependency check as boundary, +wakes on every change, an ended dependency breaks), S2 and S3 (waiting, woken at every usable value), slice 8's +last-good rule (dropped when a contribution leaves), §3's placement table (by address; the publication gate), §6 (the +simultaneous-cycle gap: (A) one narrowed deadline or (B) the probe), §7 (host.hold, getters, calls), §21 (no gone on +removal; a dropped version unbinds only its requests; installs never await activations; rollout without revision +fields; the reload handover), the start barrier (R17, W6), FailureCause (+ waiting), loading/failed with the +attempt id, one message kind (probe, tagged search or check). Unchanged: what an object's own code reload does (§21), R18 after live, +retryOn outside parking, self.follow's R4, the call window and flow control, revocation, no client providers.

+

13. Open for the owner

+
    +
  1. The barrier: (a) or (b) (§6.3).
  2. +
  3. Getters on waiting: reject at once (recommended) or wait up to readyTimeout (§8).
  4. +
  5. waiting as a new cause kind (recommended) or not (§2).
  6. +
  7. A pure swap whose new factory declares a dependency that is not there: the object parks (its gate closes, holders +see loading or failed { waiting }, calls park or reject; recommended) or stays on the old code until it +resolves (old and new code would overlap without bound, against §21's "a host is at R or R−1").
  8. +
  9. The last good value (§9): kept only while every contribution it was built from is still in the table (draft; +Astra), or kept as slice 8 decided, with natives in a registry looked up at use through their owner (Sol).
  10. +
  11. Cycles across hosts (§7): (B) the probe, with (B1) a re-search backoff for probes only (recommended) or (B2) a +full search lifecycle without any timer; or (A) the slice 4b decision narrowed to one deadline, which polls chains +across hosts again, misses a ring of waiting attempts, and retries in lockstep without finding the cycle it is for.
  12. +
+
+ +

Parked Until Resolved · first edition · 2026-10-29 · Chord v2 design proposal for slice 7

+ + + diff --git a/packages/chord/docs/next/dependencies/model.md b/packages/chord/docs/next/dependencies/model.md new file mode 100644 index 000000000..f29f430d0 --- /dev/null +++ b/packages/chord/docs/next/dependencies/model.md @@ -0,0 +1,419 @@ +# Dependency resolution in Chord v2: one model (draft 7, redesign session, 2026-10-29) + +Status: proposal for the owner, not decided. Built in slice 7 with reload and rollout. Sources: the owner's direction +("park until resolved"; "no difference between local and remote resolution: a link going is like a plugin +unloading"), the reviews (`/tmp/dep-mechanism-*.md`, `/tmp/dep-graph-*.md`, `/tmp/barrier-*.md`, +`/tmp/model/review{1,2}-*.md`), the code at pi `358f7d46a`. **Owner** marks an open decision. + +Draft 7 (round 6): (A) also retries in lockstep without finding the cycle (Fable); the probe's recovery after a +suppressed search becomes a choice between (B1) a re-search backoff for probes only (recommended) and (B2) a full +search lifecycle without timers, since `dead` notices alone need recovery state that eviction may not drop (Astra, Sol). +Causes are compared by value on wakes (Opus). +Round 7: a returned search is accepted while its initiator is parked, whatever its round (Astra: re-searches must not +invalidate searches under way); detection is eventual, not within one backoff period; the local walk stays (Opus). + +Draft 6 (round 5): cycles across hosts become an owner choice between (A) the owner's slice 4b decision (simultaneous +starts across hosts time out, no probe message), narrowed to one deadline, and (B) the probe, completed with edge +tokens, no revisits, and `dead` notices back to the initiator (Astra: a check that dies silently can lose the only +search that survived dedupe). (A) turned out to either poll chains across hosts again or miss a ring of `waiting` +attempts, so (B) is recommended, pending its review. + +Draft 5 (round 4, Sol's phantom cycle): the second round is replaced by a check along the recorded trail of +`(attempt, edge generation)` hops; a reported ring existed at one moment; probes grow with their path. + +Draft 4 after round 4 (probe only): round numbers per attempt, forwarded only when newer than the stored one; probes +honoured on parent and route links, dropped on child links; completeness stated under an eventually stable graph; the +repair caveat broadened to any concurrent edge removal; a new edge generation resets confirmation; a bounded display +trail. + +Draft 4 changes (round 3): the probe starts a fresh round on every new or changed wait edge (a ring closes at its last +edge, whose source finds it), keeps one seen entry per initiator, accepts only its current round, and states its +cost honestly; every host drops probes from untrusted links; local forwarding is iterative; no-placement remote +requests are re-placed when a host's first parent link connects; a cohort object that ends before its first outcome +settles the barrier; the last-good refinement is marked for the owner. + +Draft 3 changes (round 2): remote types without placement keep today's fallback (here if served, else the parent); +hydration stays before parking; no copied `retryAt`; a dependency `ended` breaks its dependent; a reload's handover +binds new requests before releasing old ones; retry passes only to `broken` and `transient` blockers, once per +operation; `installing` re-evaluated at every install end (commit or rejection); a registry's last good value dropped +when one of its contributors leaves; the probe with round ids, bounded seen sets, edge-generation triggers, attempt +ids on every non-live answer and relays dropping probes from untrusted links; one cycle mechanism, local and remote. + +## 0. The problem + +The same situation, "what I need is not there", has five outcomes today, depending only on how it is declared: + +| Declared as | Provider missing | Today | +| --- | --- | --- | +| `self.get(LocalType)` | no facet serves it | `self.get` throws, the factory throws: `broken` | +| `self.get(RemoteType)` | the worker does not serve it | the hold is refused `broken`; the dependent `transient`, polled 1-60 s forever | +| `self.registry(R)` | no provider | `transient`, woken at R's first value only | +| `self.registry(R)`, reader newer | | `broken` | +| a reload removes the type | | holders get `gone`: ended for good | + +And the startup barrier deadlocks an acyclic graph across two hosts (Sol, `/tmp/dep-graph-sol-barrier.log`): S's eager +A needs W's B, B needs S's C; W's hold for C waits for S's whole `load`, which waits for A. Only `readyTimeout` ends it. + +## 1. Requests and one resolver + +- **A request** is what a reference stands for: type (or registry), params, principal, the consumer's contract version. + Its holder makes it and owns it: an attempt (`self.get`, `self.registry`, keyed pins), a call (`ctx.get`), outside + code (`host.get`, `host.hold`, `host.registry`), a link's peer (`hold`). **It ends only when its holder ends it** (the + attempt ends, the call settles, `release()`, a getter gives up, the peer's link or origin closes, revocation, the + holding host closes), never because its provider is missing, went away or does not fit. +- **Placement by address** (round 1): a `local` type goes here; a `remote` type with a placement goes to the address + its placement (computed from the consumer's own contract) gives: here if that is this host's address, whether served + here or not, else over the route for that kind or the parent (design §3). A `remote` type **without placement** keeps + today's rule (round 2: Fable, Opus, Astra; no feature removed): here if served here, otherwise over the parent. A + request of that kind made before the host has a parent link waits here `not_served` and is **re-placed when the + host's first parent link connects** (round 3: Opus N5, Sol 3: a tab loads before it connects); on a host that never + connects (the root), it stays here. Opus would rather require a placement on every `remote()`; that removes a + supported declaration, so the draft keeps the fallback. A hold arriving for an address that is not this host's is relayed or + refused (`invalid/placement`) as today; a relay never captures a hold because it serves the same name. +- **Then the selected provider is checked** (served, version or accepted version and fingerprint, `authorize`, the + provider's own placement check). Version checks never look at a same-name registration elsewhere. +- **The publication gate:** a request made on this host, or arriving on its links, before the host's first + publication (§6) is not placed yet; it shows `loading` and is placed by the publication. The gate lasts only for the + synchronous setups, never for activation, so it cannot deadlock. +- **Outcomes of the check:** + + | The check finds | Request | + | --- | --- | + | a fitting provider | bound (to the object here, or to the hold over a path) | + | not published yet | unbound `unpublished`: `loading` | + | nobody serves the name at its address | unbound `not_served`: `failed { waiting }` | + | no fitting version, a fingerprint that differs at the same version, a placement the provider computes elsewhere (code skew) | unbound `version`: `failed { waiting }` | + | the registry is served, but its first `compose` (or the reader's version mapping) threw | unbound `compose`: `failed { waiting }` | + | the provider host is installing a revision and the hold does not fit its current code | unbound `installing` at the provider: `loading` | + | the path has no link (a drop, a route resolving) | on the path: `loading` | + | `authorize` refused, invalid params | `failed { broken, forbidden \| invalid }`, as today, never indexed | + + The unbound reasons that wait for a name (`unpublished`, `not_served`, `version`, `compose`) are indexed under it on + the host that decided them: the consumer's host for a local request; the provider's host for a hold that arrived + over a link (it already keeps such holds as refused: `remote/hold.ts`). `installing` is the provider host's own, + re-evaluated at its install's end; `no_link` lives on the path, where the next link holds it again. Indexes drop an + entry when its request ends; GC-tested. + +## 2. States + +`ref.state` keeps its four kinds; `FailureCause` gains one kind, `waiting`: + +```ts +type RefState = + | { kind: "loading" } + | { kind: "live" } + | { kind: "failed"; cause: FailureCause; retryAt: number | undefined } + | { kind: "ended"; error: ChordError }; + +type FailureCause = + | { kind: "transient"; error: ChordError } // the object's own effect failed (an open, a follow): backoff + | { kind: "broken"; error: ChordError } // the object's own code, a cycle, or authority: retry() or a reload + | { kind: "waiting"; error: ChordError }; // not the object's fault: no provider, no fitting version, a + // dependency failed; retryAt undefined, woken by an event +``` + +| The request is | `ref.state` | Calls | +| --- | --- | --- | +| bound, its object activating or parked with no failed blocker | `loading` | park, bounded | +| unbound `unpublished` or `installing`, or its path has no link | `loading` | park, bounded | +| unbound `not_served`, `version` or `compose` | `failed { waiting }` | reject at once | +| bound, its object failed | `failed` with the object's cause, as today | reject at once | +| bound, its object live | `live` | run | +| ended by its holder, revoked, or its host closed | `ended` | reject | + +**Owner:** `waiting` as a new cause kind (Opus, Fable; recommended: the cause says "not your fault, an event will wake +you" and a UI can say what is missing) or no new kind: `transient` with no `retryAt` (Astra; today's wire already +carries a `transient` cause without `retryIn`, `remote-object.ts`), or `loading` with a diagnostic (Sol; parks every +call on a missing plugin for its full `readyTimeout`). + +## 3. Dependents: parked until their dependencies resolve + +A **dependency** is what an attempt declares and needs before it goes live: `self.get` (local or remote), +`self.registry`, keyed pins (`at(key)` on another object's keyed member). `self.follow` is the attempt's own step (its +attach runs Durable's code), not a dependency; a follower declares `retryOn` on the registry it reads the harness +through, or recovers by its own backoff. + +1. **Order, as today up to the check:** the factory runs; persisted singletons hydrate (the object's documents, kept + across attempts); the requests are made. The attempt is **parked** while any dependency is not `live` (hydrated): + + | Its dependencies | The dependent shows | + | --- | --- | + | any `loading`, none failed | `loading` | + | any `failed` | `failed { waiting, "waiting for X: " }`, `retryAt` undefined; the message keeps the nearest blocker and the root cause within the 500-character rule | + | one failed with `invalid/cycle` (a dependent off the ring) | `broken` with the same error: it would wait on a ring that never heals by itself | + | one `ended` (revoked, or its host closed) | `broken` with that error: the request cannot come back; `retry()` or a reload makes new requests (a guard: a dependency ends only with its holder's principal or host, so the dependent ends too; Fable) | + +2. **The dependency check is the activation boundary:** once every dependency is `live`, the attempt runs its own + steps (natives in order, follow attaches, member checks, derives), bounded by its own `readyTimeout` deadline, and + goes live. A dependency that leaves `live` after the check is treated as after live (R18): the attempt goes on. This + is a deliberate rule (Astra: a weakening of "live means every dependency is live"): an object can finish opening + after a dependency went away, and its calls through that reference meet its state. So a parked attempt holds no + natives, and an attempt that opened natives is never parked. +3. **No timer while parked:** no dependency deadline, no backoff. The attempt's deadline is paused while parked (as + eager migrations pause it) and resumes with the budget left. +4. **Wakes, one rule:** the dependent re-evaluates on every state change of any of its dependencies (live, loading, + failed, a new cause compared by value, ended; Opus: otherwise a ring of waiting attempts passes its causes round and + round), updating what it shows; it leaves parking when all are live. The factory never re-runs + while parked. Wakes are coalesced per object and run after the batch that caused them; a change during the + attempt's own steps or its teardown is remembered. +5. **Kept while parked:** its documents (the object's), its declarations and requests, its seal token; no natives. + Ended with the object (its last holder; a host-lifetime object when its plugin is removed or the host closes). +6. **`broken` only for the object's own code** (a throwing factory, a derive throwing at activation, hydration, + `migrate`, a document fingerprint mismatch, an `open` throwing `broken(...)`), **cycles** (§7) and **authority** + (`forbidden`, an ended dependency). +7. **Retry passes on:** `ref.retry()` on a `failed { waiting }` object retries its `broken` and `transient` blockers, + hop by hop (the existing `retry` message over links), once per retry operation (an id dedupes diamonds); `waiting` + blockers pass it on; nothing else is touched. Retry of a `broken` or `transient` object starts a new attempt, as + today. +8. **`retryOn`** wakes failed objects as decided in slice 8 (a value call retries a `transient` or `broken` object); + while the object is parked `waiting`, a value call does nothing: its dependencies decide. +9. **Own-code reload of a parked object:** its plugin's reload replaces the parked attempt with the new factory's, + as one handover: the new requests are made and bound first, then the old ones released (round 2, Astra N4: an + unchanged dependency held only by this object must not end in between; today's `#activate` binds before + `#releasePrevious`). +10. **After live, R18 unchanged:** a dependency leaving `live` leaves its dependent live. + +## 4. Wakes: two events + +| Event | Re-evaluates | +| --- | --- | +| **a target changes state**: a dependency's object goes live, loading, fails, gets a new cause or ends; a hold's `bound`, `loading`, `failed` arrive, also after a re-hold on a new link | the requests bound to it, and their dependents (§3.4) | +| **a name becomes usable, an install ends, or the first parent link connects, on a host**: a publication (`load`'s setups, a reload's commit, an install's commit) serves a type, a registry, a version; a registry composes or maps a value at a version; an install ends without a commit (rejected) | the unbound requests indexed under those names; the provider's `installing` holds; at the first parent link, the +no-placement remote requests waiting here; a refused hold re-runs its start and sends `loading`, then `bound` or `failed` | + +No new message kind for wakes: `loading`, `bound`, `failed` and `retry` exist. The wire's `failed` carries the new cause +kind `waiting`; `loading` and a `waiting` `failed` carry the provider's attempt id for the probe (§7). + +## 5. Removal, versions, rollout + +- **A reload that removes a type:** its objects end as today (natives close, reason `end`; callers rejected at once; + handlers drain). Their requests are not ended: each becomes unbound `not_served` and binds again if a later commit + serves the name. + - **Locally:** a `Binding` whose target ended by removal becomes unbound and accepts a second `bind`: member sources + re-bind by name (one frame from the last value, as on a restart), keyed pins re-pin, parked calls flush or + reject, presence re-lists. The one new piece of machinery. + - **Over a link:** a `failed` on a bound hold unbinds it (Fable: one message, one rule; the consumer applies it as + `#startOver` plus `failed`); the provider detaches the hold's sources and drops its unsent pending ops and queued + `bound`; the ack slots of `bound`/`state` messages already sent stay counted until their `ack` or the link's loss; + calls in flight on it are rejected and free their slots once. +- **A reload that stops accepting a version** unbinds only the requests on that version (`version`); the object stays + (its own plugin's reload still restarts it if it has natives, §21). +- **A type moved to another host kind:** the request carries the address its own consumer code computed; the old host + answers `not_served`; the consumer's own reload makes a new request. No redirect. +- **A consumer whose code moves to a new version** releases its old request and makes a new one (today). +- **Rollout, without revision fields:** each host knows only its own install. A provider host that installs keeps + holds that do not fit its current code `installing` (`loading`) and re-evaluates them when the install ends, by its + commit or by its rejection. A provider host that is queued but not installing yet answers a mismatch `failed + { waiting, version }` until its install commits and wakes it: a visible blip of at most one install, accepted (Opus). + An install never waits for activations (a reverse-order rollout would deadlock); a host whose own install was + rejected fails its own requests at once with the reason (design §21). The loader keeps its own duties (the desired + code set, digests, serialized installs). +- **Old calls across a reload** follow §21 unchanged: a pure swap lets them finish on old handles; a native restart + drains them first; a removal rejects their callers at once and drains the handlers. Calls are never resent. + +## 6. Startup: publication, then first outcomes + +1. **Publication:** `load` runs every facet's setup, then publishes all at once in one batch (types served, + registries served and composed once, contributions in their tables), then wakes (§4) after that batch. Nothing + activates inside it. +2. **No gate beyond it:** holds and `host.hold` wait only for the publication, never for the barrier. This ends the + startup deadlock: W's hold for C binds as soon as S's setups published; B goes live, then A. +3. **The barrier** (**Owner:** (a) or (b); recommended (a), three reviewers of four): + - **(a):** `load` resolves when every host-lifetime object of this `load` has a first outcome: + `firstOutcome(o) = o.live || o.ownFailure || (o.parked && o.blockers.every((b) => !b.local || firstOutcome(b)))`, + where `local` means executing on this host (an object here, a keyed pin still loading here), not the contract's + kind. So `CodeMode → Harness` waits for the harness's outcome; a wait on another host or on a failed dependency + ends the first pass. One `firstOutcome` signal per object, separate from the readiness dependents wait for (Sol), + counted once, never reopened by retries. An object of the cohort that ends before its first outcome (its plugin + removed, the host closing) settles its entry as ended; `host.close` settles the waiting `load` (Astra R3). + - **What (a) promises:** every host-lifetime object on this host had its first try, and the contributions of those + that went live are in their tables. **Not:** every tool that will arrive (a parked or slow contributor, tools found + in `onLive`, MCP; Sol's probe `/tmp/barrier-sol-first-outcome.log`). What a first turn needs is p3's explicit + admission check. + - **(b):** `load` resolves after the publication; Chord adds `host.startup`, a watchable of the host-lifetime + objects' first outcomes with their causes; the app decides (Fable). The same tracker, awaited by the app. + - p2 today resumes its harness without waiting for all host objects (`apps/p2/src/hosts/worker.ts`), because its + worker objects that need server types are reachable only after `control.ready()`. + +## 7. Cycles + +On one host, cycles are found exactly as today: at `self.get`, following the waiting edges of activating and parked +attempts. Across hosts, a hold carries the activation chain it was made in (today), which finds every cycle whose +objects start one after another. What remains is the documented gap (design §6): objects on two hosts that start at +the same moment and need each other. The owner decided on 2026-10-24 (slice 4b review decisions (2)) that this is a +documented limitation that times out and retries, with no probe message. Without dependency deadlines that timeout no +longer exists, so the gap needs one of two answers (**Owner**): + +- **(A) The 4b decision, narrowed.** One deadline stays: an attempt parked on a dependency held over a link whose target + attempt is known and still activating or parked fails `transient` after its `readyTimeout` and retries with backoff; + its next attempt usually starts after the other side and its chain finds the cycle. It has a cost that decides + against it: the target being *parked* is the common case of a chain across hosts waiting for a missing plugin (a + tab's view on a worker object that waits for an uninstalled type), and those dependents would time out and poll + again, as today, so "local = remote" would not hold across hosts. Narrowing it to targets that are `loading` (not + `failed { waiting }`) avoids that but misses a real ring: A and B on two hosts wait on each other, B also on a + missing X; B is `failed { waiting }`, so A is too; when X appears, both stay `failed { waiting }` on each other, with + no deadline and no chain to find it. And even the timeout does not find what it is meant to (round 6, Fable): an + object's chain is fixed when it is created and its instance id survives retries, so a retried attempt sends the same + hold with the same chain; two simultaneous starts time out and retry in lockstep and are never found. +- **(B) The probe (recommended)**, below: no deadline at all, one new message kind, the most intricate rule set in the + model. Five + review rounds found a counterexample in each of the first four drafts; draft 6's version answers all of them, but it + has not been reviewed again, and slice 7 would have to prove it with tests before the deadline could go. + +### (B) The probe + +Replaces the activation chain in holds. On one host the `self.get` walk stays (it is synchronous and exact: a ring +with no remote edge is found there, where (B1)'s timer does not run; Opus, round 7); the probe serves the rings that +cross a link. It replaces draft 1's bounded chain too +draft 1's bounded chain (a ring of 33 is never found with chains cut at 32; chains keep stale ancestry). Draft 5 replaces the +second round by a check along the recorded path (round 4, Sol: two rounds through different stale paths can report a +ring that never existed, `/tmp/model/sol-probe4.log`). + +1. **Attempts, edges, tokens.** Every attempt has its own id (uuidv7 per attempt: a retry or reload is a new + attempt). A **wait edge** runs from a parked attempt to the parked or activating attempt it waits for, through a + bound request (unbound waits and registry reads are never edges; they are not deadlocks). Each edge carries an **edge + token** (round 5, Astra, Sol): a number from a counter per attempt, never reused, replaced by a fresh one on any + interruption or change of that edge, so two requests to one object are two edges: a new target attempt (also a + local target's retry, without a re-bind), any `loading` on a bound or pending hold (a relay's upstream re-hold + included), an unbinding, a rebinding, a re-hold after a reconnect. An attempt is parked for one interval only (one + that passed its dependency check never parks again) and ids are never reused. A provider tells its consumers the attempt they wait for on + every non-live answer: `loading { hold, attempt }` and a `waiting` `failed { hold, cause, attempt }`. +2. **A search per edge change.** Whenever a parked attempt gets a new edge or a new token on one, it starts a + search: `probe { kind: "search", round, initiator, trail }` along each of its wait edges, `round` the next number of + a counter the attempt keeps, `trail` the list of `(attempt, edge token)` hops it crossed, starting with + the initiator's own. A ring closes at its last + edge, and that edge's source starts the search that finds it. Coalesced per batch. Locally a call on the target + attempt, forwarded through a queue (no recursion); over a link the message `probe { hold, to, ... }`, consumer to + provider, relayed by hold id with the origin. **Probes are honoured on parent and route links and dropped on child + links** (`accept`), whatever `trusted` says. +3. **Forwarding.** An attempt receiving a search drops it if `to` is not its current attempt or it is not parked. If + the initiator is itself, the search returned (4). If the attempt is already in the trail, it drops it (a ring that does not contain the initiator is + found by its own members' searches; Opus). Otherwise it forwards it along every wait edge, appending `(itself, the edge's + token)` to the trail, only if the round number is higher than the one it stored for that + initiator (older or equal rounds are dropped: the initiator superseded them), and stores it. The seen map holds one + number per initiator, up to 256 initiators, and goes with the attempt. +4. **The check.** A returned search is accepted while its initiating attempt is still parked, whatever round it + carries (round 7, Astra: a re-search must not invalidate an older search still under way); the check decides. The + initiator then sends `probe { kind: "check", round, trail }` along the recorded trail: the initiator first verifies its own hop; each hop passes it on only if it is still + that attempt, still parked, still has the recorded edge token, pointing at the next hop's attempt; otherwise the + check dies (also on a `to` mismatch). The check follows the trail by position and bypasses the seen + map. The initiator verifies its own hop again when the check returns. If the check returns, the initiator fails `broken invalid/cycle`, the path named for display from the trail, + and today's propagation breaks the other activating objects on the ring and the dependents off it. 5. **No silent loss.** Dedupe can drop the copy of a search that travelled the ring in favour of a copy whose path + later breaks (Astra, round 5); then the check dies and nothing is left to find the ring. Round 6 tried `dead` + notices back to the initiator; Sol and Astra showed that reliable notices need recovery state per search that + eviction may not drop and that a long-parked attempt would otherwise keep for every initiator that ever passed (a + leak), i.e. a full search lifecycle with cancellation. Two ways (**Owner**, part of decision 6): + - **(B1) Re-search with backoff (recommended).** An attempt parked on a remote attempt starts a fresh search again + after a backoff (1 s doubling to 60 s, reset by an edge change), whether it shows `loading` or `failed { waiting }`; + the timer stops when the attempt leaves parking. No `dead` messages, no recovery state, eviction + only costs work; a new search starts within one backoff period, so every stable ring that crosses a link is + eventually found (no wall-clock bound: delivery is only fair; a ring on one host is the walk's). The timer runs as long as the attempt stays parked on a remote attempt: a view on a worker object that + waits for a missing plugin re-searches every 60 s until a reload (Fable: cheap, but for as long as it waits). The timer sends probes only: it never + fails an object, re-holds or polls a provider. Cost: one probe per parked remote dependent per period, which + crosses the downstream wait graph and dies at its leaves (an unbound wait is not an edge). + - **(B2) Events only.** Each search is owned until it settles: branches report completion or abort back along their + trails, a dropped probe answers `dead { initiator, round }` with a reverse route, an edge records the searches it + carries until they settle, and an initiator that ends or starts a new round cancels its old one along the same + routes. Complete without a timer, bounded by the searches in flight, and the most intricate protocol in Chord; not + yet specified to the reviewers' satisfaction. + Rejected: enumerating every simple trail without dedupe (Sol's correctness reference): no ring copy is ever + suppressed, but work is exponential on chains of diamonds. +- **Why a reported ring existed:** every edge of the trail kept its token from the moment the search crossed it + to the moment the check crossed it; the search crossed all of them before the check started, so at the moment the + check started every edge of the ring existed at once. A token changes on every interruption and is never reused, so an edge cannot + vanish and come back unnoticed. Sol's counterexample fails at the check: `C1` is no longer `B`'s target. +- **Why every ring is found** (under an eventually stable wait graph and fair delivery; endless edge churn can keep + invalidating searches; with rule 5's (B1) or (B2) for a search suppressed behind a path that broke): a ring is closed by its last edge; its source starts a search when the edge appears; every + other edge exists then and every attempt on the ring waits transitively on the source, so none goes live before the + search and the check return. Rings of any length, local and remote mixed. On a DAG searches die at the leaves; on a + diamond the second arrival is dropped. +- **What it does not promise:** (1) a ring that existed when the check started may be broken up by a concurrent reload, + retry, unbinding or revocation just after; it is still reported (it did exist). Its broken objects recover as every + broken object does: `ref.retry()` or a reload of their plugin. (2) Linear work under every schedule: with more than + 256 initiators passing one attempt at once, an evicted entry lets a search be forwarded again, so work can grow with + the number of simple paths (never endlessly: a search never revisits an attempt in its trail). (3) Constant message size: a probe carries its trail, so its size grows with the length of the + path it crossed (a returning search's trail is as long as its ring); no cap cuts it, since the trail is the proof. +- **Cost:** one search per edge change, crossing each downstream edge once, one check per returned search; with (B1), + one more search per backoff period for every attempt parked on a remote attempt, also while an acyclic chain waits + for a missing plugin, plus the trail bytes. The bounded memory is the seen map's; probes in flight are bounded by + the searches started. +- **Transition:** today's dependency deadline for activations waiting across links stays until the probe's tests pass + in slice 7 (long rings, diamonds, a ring closed after a reconnect with unchanged attempt ids, Sol's phantom schedule, + 300 concurrent initiators, edges added and removed, reloads, relays, forged probes); then it goes in the same slice. + +## 8. Getters, holds, calls: one rule + +- **`host.hold`** returns a reference at once; only invalid params throw (also before `load`). A missing local type + gives `failed { waiting }` after the publication, `loading` before it. +- **`host.get`, `ctx.get` and `host.registry`:** resolve when live; wait while `loading`, or `failed { transient }` + (its own retry is under way), up to their `readyTimeout`; reject at once on `failed { waiting }` or `{ broken }`. On + success, rejection or timeout their temporary wait goes; `host.registry` returns the `RegistryValue` with its + ordinary watch ownership (no `release()`). **Owner:** recommended (nobody waits 60 s for a plugin that is not + there; code that wants to wait holds instead), or wait through `waiting` too. +- **Calls** on a `loading` reference park, each bounded by the reference's `readyTimeout` from when it was made; on a + `failed` one they reject at once with the cause; sent calls are never resent. + +## 9. Registries + +- The same resolver on one host: a reader's request for R at its version is bound while R is served and has a usable + value at that version (composed; mapped by `accepts[old].value` for an older reader), otherwise unbound + `not_served`, `version` or `compose`, or `loading` before the publication. Woken at every transition to usable + (`#valued`'s first-value latch goes). S2's `broken` and S3's backoff go. +- **Version skew within one host:** a reload of the registry's plugin alone leaves other plugins' contributors and + readers on their bundled versions; `accepts[old].contribution` and `.value` keep them working, unsupported + contributions are left out and reported, unsupported readers are `waiting` (`version`). +- A registry waits for a value, never for a particular contribution (an empty table composes). After live, a later + `compose` failure changes nothing for the reader: `.value` keeps the last good composition, **but only while every + contribution it was built from is still in the table** (round 2, Astra N2): when one of them leaves (its + contributor ended or was reloaded), the last good value is dropped and `.value` throws until `compose` succeeds, so + it never hands out a native whose contributor ended. `compose` represents a missing contributor as a value. +- Registries stay host-local (contributions are in-process values). + +## 10. Keep-alive, presence, revocation, jobs + +- A request over a link counts once in `host.activity.references` from its arrival to its end, bound or unbound; + presence (`host.references`) lists only bound references. +- Revocation ends the requests of a revoked principal, bound and unbound, at every hop; nothing revives them. +- `gone` stays only for revocation (a provider host closing closes its links and sends nothing; its consumers see a + drop, `loading`, and hold again on the next link). +- Persisted job runs of a parked host-lifetime object are loaded with its documents; `run` and `finish` start only + once it is live; the run count keeps the object as today. + +## 11. Timeouts after the model + +| Timeout or backoff | After | Why | +| --- | --- | --- | +| `readyTimeout` of an attempt | only its own steps; paused while parked | a slow dependency is bounded where it runs | +| `readyTimeout` of `host.get`, `ctx.get`, `host.registry` | kept | a caller gets an answer | +| `readyTimeout` of a parked call | kept | an action must not run long after it was asked for | +| identity backoff 1-60 s | only for the object's own `transient` failures | nothing else says the disk or network is back | +| dependency backoff, polling of refused holds | removed | recovery is an event | +| dependency deadline across links | (A): only for a parked attempt whose remote target attempt stays activating or parked; (B): until the probe's tests pass, then removed | the simultaneous-cycle gap | +| probe re-search backoff (B1) | new, only with (B1): 1 s doubling to 60 s while parked on a remote attempt | finds a ring whose search was suppressed; sends probes only, never fails or re-holds | +| route backoff, heartbeat, `closeTimeout`, routine and `finish` backoffs, `linger` | kept | not dependency waits | +| rollout install timeout (not built) | a watchdog on the install only | never a deadline for dependencies | + +## 12. What changes in the design + +R18 (dependency failures `waiting`, no backoff, the deadline paused while parked, the dependency check as boundary, +wakes on every change, an ended dependency breaks), S2 and S3 (`waiting`, woken at every usable value), slice 8's +last-good rule (dropped when a contribution leaves), §3's placement table (by address; the publication gate), §6 (the +simultaneous-cycle gap: (A) one narrowed deadline or (B) the probe), §7 (`host.hold`, getters, calls), §21 (no `gone` on +removal; a dropped version unbinds only its requests; installs never await activations; rollout without revision +fields; the reload handover), the start barrier (R17, W6), `FailureCause` (+ `waiting`), `loading`/`failed` with the +attempt id, one message kind (`probe`, tagged `search` or `check`). Unchanged: what an object's own code reload does (§21), R18 after live, +`retryOn` outside parking, `self.follow`'s R4, the call window and flow control, revocation, no client providers. + +## 13. Open for the owner + +1. The barrier: (a) or (b) (§6.3). +2. Getters on `waiting`: reject at once (recommended) or wait up to `readyTimeout` (§8). +3. `waiting` as a new cause kind (recommended) or not (§2). +4. A pure swap whose new factory declares a dependency that is not there: the object parks (its gate closes, holders + see `loading` or `failed { waiting }`, calls park or reject; recommended) or stays on the old code until it + resolves (old and new code would overlap without bound, against §21's "a host is at R or R−1"). +5. **The last good value** (§9): kept only while every contribution it was built from is still in the table (draft; + Astra), or kept as slice 8 decided, with natives in a registry looked up at use through their owner (Sol). +6. **Cycles across hosts** (§7): (B) the probe, with (B1) a re-search backoff for probes only (recommended) or (B2) a + full search lifecycle without any timer; or (A) the slice 4b decision narrowed to one deadline, which polls chains + across hosts again, misses a ring of `waiting` attempts, and retries in lockstep without finding the cycle it is for. From 67c63c0f7ff41a55176683b0f55bbd8fb0f4935e Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 29 Oct 2026 06:49:17 +0000 Subject: [PATCH 126/271] docs(chord): dependency model: the dependency check applies R18 after live, not a rule of its own --- packages/chord/docs/next/dependencies/book.md | 11 ++++++----- packages/chord/docs/next/dependencies/index.html | 11 ++++++----- packages/chord/docs/next/dependencies/model.md | 6 +++--- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/packages/chord/docs/next/dependencies/book.md b/packages/chord/docs/next/dependencies/book.md index 19b550bdf..4d9a45678 100644 --- a/packages/chord/docs/next/dependencies/book.md +++ b/packages/chord/docs/next/dependencies/book.md @@ -352,8 +352,9 @@ bounded by its own `readyTimeout` deadline, as today. Then it goes live. The dependency check is the boundary between waiting and working (Astra's F3). Before it, the attempt holds no natives; after it, a dependency that leaves `live` is treated as after live (R18): the attempt goes on, and its calls through that reference meet the reference's state. So a parked attempt never holds a native, and an attempt that -opened a native is never parked. This is a deliberate rule, and Astra called it what it is: a weakening of "live -means every dependency is live". An object can finish opening after a dependency went away. Without that boundary a contributor unloading while the attempt opens its natives +opened a native is never parked. It is not a rule of its own: it is R18's rule for live objects (a dependency that +goes away leaves the object live; same-host calls in flight finish, remote ones fail, new calls meet the reference's +state), applied from the dependency check on. Without that boundary a contributor unloading while the attempt opens its natives would leave it half open and parked, or force a seal, a drain and a fresh attempt for a dependency the object may not even use at once. @@ -1399,9 +1400,9 @@ through, or recovers by its own backoff. 2. **The dependency check is the activation boundary:** once every dependency is `live`, the attempt runs its own steps (natives in order, follow attaches, member checks, derives), bounded by its own `readyTimeout` deadline, and - goes live. A dependency that leaves `live` after the check is treated as after live (R18): the attempt goes on. This - is a deliberate rule (Astra: a weakening of "live means every dependency is live"): an object can finish opening - after a dependency went away, and its calls through that reference meet its state. So a parked attempt holds no + goes live. A dependency that leaves `live` after the check is treated as after live (R18): the attempt goes on. This is + R18's rule for live objects applied from the check on, not a rule of its own (owner, 2026-10-29: a dependency can + go away at any point of an object's life; the object stays live, its calls through that reference meet its state). So a parked attempt holds no natives, and an attempt that opened natives is never parked. 3. **No timer while parked:** no dependency deadline, no backoff. The attempt's deadline is paused while parked (as eager migrations pause it) and resumes with the budget left. diff --git a/packages/chord/docs/next/dependencies/index.html b/packages/chord/docs/next/dependencies/index.html index 3bc8daa6f..3cbb8f533 100644 --- a/packages/chord/docs/next/dependencies/index.html +++ b/packages/chord/docs/next/dependencies/index.html @@ -538,8 +538,9 @@ bounded by its own readyTimeout deadline, as today. Then it goes li

The dependency check is the boundary between waiting and working (Astra's F3). Before it, the attempt holds no natives; after it, a dependency that leaves live is treated as after live (R18): the attempt goes on, and its calls through that reference meet the reference's state. So a parked attempt never holds a native, and an attempt that -opened a native is never parked. This is a deliberate rule, and Astra called it what it is: a weakening of "live -means every dependency is live". An object can finish opening after a dependency went away. Without that boundary a contributor unloading while the attempt opens its natives +opened a native is never parked. It is not a rule of its own: it is R18's rule for live objects (a dependency that +goes away leaves the object live; same-host calls in flight finish, remote ones fail, new calls meet the reference's +state), applied from the dependency check on. Without that boundary a contributor unloading while the attempt opens its natives would leave it half open and parked, or force a seal, a drain and a fresh attempt for a dependency the object may not even use at once.

CodeModehost-lifetime, localHarnesshost-lifetime, localHarnessModelsregistryself.get(Harness): loadingparked: shows loading, no timerself.registry(HarnessModels): usableown steps: open DurableHarness live: a state changedependency check passes; open the sandboxlive; contribute tools
Figure 3. Code mode on W parks on the harness and goes on the moment it is live.
@@ -2256,9 +2257,9 @@ across attempts); the requests are made. The attempt is parked
  • The dependency check is the activation boundary: once every dependency is live, the attempt runs its own steps (natives in order, follow attaches, member checks, derives), bounded by its own readyTimeout deadline, and -goes live. A dependency that leaves live after the check is treated as after live (R18): the attempt goes on. This -is a deliberate rule (Astra: a weakening of "live means every dependency is live"): an object can finish opening -after a dependency went away, and its calls through that reference meet its state. So a parked attempt holds no +goes live. A dependency that leaves live after the check is treated as after live (R18): the attempt goes on. This is +R18's rule for live objects applied from the check on, not a rule of its own (owner, 2026-10-29: a dependency can +go away at any point of an object's life; the object stays live, its calls through that reference meet its state). So a parked attempt holds no natives, and an attempt that opened natives is never parked.

  • No timer while parked: no dependency deadline, no backoff. The attempt's deadline is paused while parked (as diff --git a/packages/chord/docs/next/dependencies/model.md b/packages/chord/docs/next/dependencies/model.md index f29f430d0..172f2a688 100644 --- a/packages/chord/docs/next/dependencies/model.md +++ b/packages/chord/docs/next/dependencies/model.md @@ -145,9 +145,9 @@ through, or recovers by its own backoff. 2. **The dependency check is the activation boundary:** once every dependency is `live`, the attempt runs its own steps (natives in order, follow attaches, member checks, derives), bounded by its own `readyTimeout` deadline, and - goes live. A dependency that leaves `live` after the check is treated as after live (R18): the attempt goes on. This - is a deliberate rule (Astra: a weakening of "live means every dependency is live"): an object can finish opening - after a dependency went away, and its calls through that reference meet its state. So a parked attempt holds no + goes live. A dependency that leaves `live` after the check is treated as after live (R18): the attempt goes on. This is + R18's rule for live objects applied from the check on, not a rule of its own (owner, 2026-10-29: a dependency can + go away at any point of an object's life; the object stays live, its calls through that reference meet its state). So a parked attempt holds no natives, and an attempt that opened natives is never parked. 3. **No timer while parked:** no dependency deadline, no backoff. The attempt's deadline is paused while parked (as eager migrations pause it) and resumes with the budget left. From 1b760044ec6686f669f09500dfeaf8f04d3af49d Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 29 Oct 2026 07:06:37 +0000 Subject: [PATCH 127/271] docs(chord): note: every object may contribute (for later) --- packages/chord/docs/next/decisions.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 59893c07b..17b8e8aa4 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2337,6 +2337,10 @@ Everything else from earlier lists is decided (see the entries above and `packag - **Future goal: deploy Chord apps to Cloudflare Durable Objects and similar platforms.** Hot reload there is not needed; composition through plugins is (plugin set fixed at deploy). Research in `research/chord-on-durable-objects.md`. - Deferred until a concrete case: jobs as objects and where a flow lives (when login is ported); project plugins. +- **Every object may contribute** (owner, 2026-10-29: "note it down as something to consider for the future; can be + changed later without a need for migrations"): lift R21 so objects with params and `remote` objects contribute too; + the entry lives while the object is live, its identity is the object's (type, params, principal key); `contribute` + on every `self`. Not decided; R21 stands. - p3 domain points (decided per domain when p3 is designed): `SessionFiles {session}` (proposed), session delete, connection pools, config files with a single writer, file formats as versioned interfaces, the terminal host process as a revision artifact. From 13dee755297b9b2a3d1366b1296984fb45978636 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 29 Oct 2026 07:17:22 +0000 Subject: [PATCH 128/271] docs(chord): every remote type declares its placement --- packages/chord/docs/next/decisions.md | 6 ++++++ packages/chord/docs/next/design.md | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 17b8e8aa4..d4ba29fe0 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1771,6 +1771,12 @@ validation and copies the v2 kernel added around Delta. setups a registry without a value cannot get one before a reload, so a reader then fails at once `transient` (`unavailable/undeclared`) and is retried when the registry gets its first value (R18), as `self.get` of a type not served here fails at once (refined 2026-10-27 after the second spec review: no `readyTimeout` wait at start); once live, a later failure of the registry changes nothing (last good value). +- **Every remote type declares its placement (owner, 2026-10-29: "yes, let's remove it"):** `placement` is required in + `remote()` (a type error without it, and a declaration error at runtime); a remote type without placement, a leftover + of slice 3a (remote objects over one link, before host kinds in 4a), is gone, and with it the fallback "here if served + here, otherwise the parent" and its re-placement at the first parent link. Every remote request is placed by its + address alone (dependency model, `packages/chord/docs/next/dependencies/`). Tests and examples get `placement: () => + Server()` or their own host kind. - **No client providers for now (owner, 2026-10-28: "yeah, no client providers for now"; reviews `/tmp/barrier-{astra,fable,opus,sol}.md`):** clients stay unaddressable and serve only `local` types (design §3). Client-side capabilities (a phone's camera, clipboard, a page's WebMCP tools) go through an owner's state, as the diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index dc38b5642..2772ee9e8 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -117,8 +117,8 @@ export const Catalog = remote("catalog", { placement: () => Server(), … }); - `hostKind(name, params?)` declares a kind of addressable host; its params follow the params policy of contracts (section 5). `Session({ id: "S1" })` is an **address**: `{ kind, params, key }`, the key built like an identity key (`session|S1`); two addresses are equal when their keys are. No host-name strings appear in app code. -- **`placement`** (owner, 2026-10-21) is a typed function of the params returning an address; `local` contracts have - none. It is not part of the contract fingerprint; a changed rule surfaces as a refusal (below). +- **`placement`** (owner, 2026-10-21) is a typed function of the params returning an address, required on every + `remote` contract (owner, 2026-10-29); `local` contracts have none. It is not part of the contract fingerprint; a changed rule surfaces as a refusal (below). - **`params`** (owner, 2026-10-21; was `input`) select the instance among objects of its type and are part of its identity (section 5). - **The caller computes the address** and sends it in the `hold`; a host that relays routes by that address alone and @@ -129,7 +129,7 @@ export const Catalog = remote("catalog", { placement: () => Server(), … }); | Case | Goes | | --- | --- | - | the type is served here and its placement (computed here from the params) is this host's address, or the host has no address, or the type no placement | here | + | the type is served here and its placement (computed here from the params) is this host's address, or the host has no address | here | | a reference made here (`host.get`/`hold`, `self.get`, `ctx.get`) to a served type placed on another host | on, as if not served here (next rows): worker S1 reaches `Notes { session: "S2" }` through the parent | | an arriving hold for a served type whose params place it on another host | broken `invalid/placement`; the address the hold names only routes it | | the hold names an address of a kind this host has a route for | over that route's link (the address's params checked against the kind, then `authorize`) | From 0ddc953102a06424e9d731d60ace171dfcd19f06 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 29 Oct 2026 07:17:49 +0000 Subject: [PATCH 129/271] docs(chord): dependency model: placement required on remote types --- packages/chord/docs/next/dependencies/book.md | 24 ++++++-------- .../chord/docs/next/dependencies/index.html | 31 +++++++------------ .../chord/docs/next/dependencies/model.md | 12 +++---- 3 files changed, 24 insertions(+), 43 deletions(-) diff --git a/packages/chord/docs/next/dependencies/book.md b/packages/chord/docs/next/dependencies/book.md index 4d9a45678..15665a434 100644 --- a/packages/chord/docs/next/dependencies/book.md +++ b/packages/chord/docs/next/dependencies/book.md @@ -195,12 +195,10 @@ compares the consumer with the provider actually selected, never with a same-nam host (today `#assertRoute` does that before placement, so worker W1 serving `Notes` v1 rejects a `Notes` v2 request placed on W2, which serves v2). -One case keeps today's rule: a `remote` type declared **without placement** (allowed by `remote()`, used in tests and -for server types a tab reaches through its parent). It goes here if it is served here, otherwise over the parent. A -request of that kind made before the host has a parent link (a tab loads before it connects) waits here as not served -and is placed again when the host's first parent link connects (Opus, Sol); on a host that never connects, the root, -it stays here. Opus would rather require a placement on every remote type, which is simpler; it removes a supported -declaration, and the owner asked for no feature to go, so the draft keeps the fallback. +Every `remote` type declares its placement (owner, 2026-10-29). Until then `remote()` allowed leaving it out, a +leftover of slice 3a, which built remote objects over one link before host kinds existed; such a type went "here if +served, otherwise the parent", which needed its own fallback and a re-placement when a tab that loads before it +connects gets its first parent link (Opus, Sol). Opus and Fable proposed making it required; the owner agreed. ## What the check finds @@ -1218,7 +1216,7 @@ scenarios against five drafts; the catalog in the chapters before this one is a | R18, the boundary | "live" means every dependency was live at the end of activation | the dependency check is the boundary: after it, a dependency leaving `live` is treated as after live | | `self.get` of a missing `local` type | throws in the factory: `broken` | unbound `not_served`: `failed { waiting }`, woken by the publication that serves it | | a remote hold the provider cannot serve | refused `broken`; the dependent polls by backoff | refused and kept at the provider, re-run at its publication; `failed { waiting }` | -| design §3, placement | "served here" decides "here" | the address decides; a remote type without placement keeps today's rule | +| design §3, placement | "served here" decides "here"; `placement` optional on `remote()` | the address decides; every remote type declares its placement (owner, 2026-10-29) | | the start barrier, `pendingLoad` | holds wait for all of `load` | holds wait only for the publication; the barrier is (a) or (b) | | S2, a registry reader newer than its provider | `broken` | `failed { waiting, version }` | | S3, a registry without a value | `transient`, woken at the first value only | `failed { waiting }`, woken at every transition to usable | @@ -1318,12 +1316,9 @@ A needs W's B, B needs S's C; W's hold for C waits for S's whole `load`, which w holding host closes), never because its provider is missing, went away or does not fit. - **Placement by address** (round 1): a `local` type goes here; a `remote` type with a placement goes to the address its placement (computed from the consumer's own contract) gives: here if that is this host's address, whether served - here or not, else over the route for that kind or the parent (design §3). A `remote` type **without placement** keeps - today's rule (round 2: Fable, Opus, Astra; no feature removed): here if served here, otherwise over the parent. A - request of that kind made before the host has a parent link waits here `not_served` and is **re-placed when the - host's first parent link connects** (round 3: Opus N5, Sol 3: a tab loads before it connects); on a host that never - connects (the root), it stays here. Opus would rather require a placement on every `remote()`; that removes a - supported declaration, so the draft keeps the fallback. A hold arriving for an address that is not this host's is relayed or + here or not, else over the route for that kind or the parent (design §3). Every `remote` type declares + its placement (owner, 2026-10-29: the optional placement of slice 3a is removed, and with it the fallback "here if + served, otherwise the parent" and its re-placement at the first parent link). A hold arriving for an address that is not this host's is relayed or refused (`invalid/placement`) as today; a relay never captures a hold because it serves the same name. - **Then the selected provider is checked** (served, version or accepted version and fingerprint, `authorize`, the provider's own placement check). Version checks never look at a same-name registration elsewhere. @@ -1433,8 +1428,7 @@ through, or recovers by its own backoff. | Event | Re-evaluates | | --- | --- | | **a target changes state**: a dependency's object goes live, loading, fails, gets a new cause or ends; a hold's `bound`, `loading`, `failed` arrive, also after a re-hold on a new link | the requests bound to it, and their dependents (§3.4) | -| **a name becomes usable, an install ends, or the first parent link connects, on a host**: a publication (`load`'s setups, a reload's commit, an install's commit) serves a type, a registry, a version; a registry composes or maps a value at a version; an install ends without a commit (rejected) | the unbound requests indexed under those names; the provider's `installing` holds; at the first parent link, the -no-placement remote requests waiting here; a refused hold re-runs its start and sends `loading`, then `bound` or `failed` | +| **a name becomes usable, or an install ends, on a host**: a publication (`load`'s setups, a reload's commit, an install's commit) serves a type, a registry, a version; a registry composes or maps a value at a version; an install ends without a commit (rejected) | the unbound requests indexed under those names; the provider's `installing` holds; a refused hold re-runs its start and sends `loading`, then `bound` or `failed` | No new message kind for wakes: `loading`, `bound`, `failed` and `retry` exist. The wire's `failed` carries the new cause kind `waiting`; `loading` and a `waiting` `failed` carry the provider's attempt id for the probe (§7). diff --git a/packages/chord/docs/next/dependencies/index.html b/packages/chord/docs/next/dependencies/index.html index 3cbb8f533..656a4f6fa 100644 --- a/packages/chord/docs/next/dependencies/index.html +++ b/packages/chord/docs/next/dependencies/index.html @@ -327,12 +327,10 @@ address. A request placed here whose provider is missing is unbound here compares the consumer with the provider actually selected, never with a same-name registration on the consumer's own host (today #assertRoute does that before placement, so worker W1 serving Notes v1 rejects a Notes v2 request placed on W2, which serves v2).

    -

    One case keeps today's rule: a remote type declared without placement (allowed by remote(), used in tests and -for server types a tab reaches through its parent). It goes here if it is served here, otherwise over the parent. A -request of that kind made before the host has a parent link (a tab loads before it connects) waits here as not served -and is placed again when the host's first parent link connects (Opus, Sol); on a host that never connects, the root, -it stays here. Opus would rather require a placement on every remote type, which is simpler; it removes a supported -declaration, and the owner asked for no feature to go, so the draft keeps the fallback.

    +

    Every remote type declares its placement (owner, 2026-10-29). Until then remote() allowed leaving it out, a +leftover of slice 3a, which built remote objects over one link before host kinds existed; such a type went "here if +served, otherwise the parent", which needed its own fallback and a re-placement when a tab that loads before it +connects gets its first parent link (Opus, Sol). Opus and Fable proposed making it required; the owner agreed.

    What the check finds

    Once placed, the provider found there is checked: served, a fitting version (the current one or an accepted older one, with the fingerprint computed from the provider's own history), authorize, and the provider's own placement check. @@ -1932,8 +1930,8 @@ owner's decisions.

  • design §3, placement -"served here" decides "here" -the address decides; a remote type without placement keeps today's rule +"served here" decides "here"; placement optional on remote() +the address decides; every remote type declares its placement (owner, 2026-10-29) the start barrier, pendingLoad @@ -2103,12 +2101,9 @@ holding host closes), never because its provider is missing, went away or does n
  • Placement by address (round 1): a local type goes here; a remote type with a placement goes to the address its placement (computed from the consumer's own contract) gives: here if that is this host's address, whether served -here or not, else over the route for that kind or the parent (design §3). A remote type without placement keeps -today's rule (round 2: Fable, Opus, Astra; no feature removed): here if served here, otherwise over the parent. A -request of that kind made before the host has a parent link waits here not_served and is re-placed when the -host's first parent link connects (round 3: Opus N5, Sol 3: a tab loads before it connects); on a host that never -connects (the root), it stays here. Opus would rather require a placement on every remote(); that removes a -supported declaration, so the draft keeps the fallback. A hold arriving for an address that is not this host's is relayed or +here or not, else over the route for that kind or the parent (design §3). Every remote type declares +its placement (owner, 2026-10-29: the optional placement of slice 3a is removed, and with it the fallback "here if +served, otherwise the parent" and its re-placement at the first parent link). A hold arriving for an address that is not this host's is relayed or refused (invalid/placement) as today; a relay never captures a hold because it serves the same name.

  • Then the selected provider is checked (served, version or accepted version and fingerprint, authorize, the @@ -2307,12 +2302,8 @@ unchanged dependency held only by this object must not end in between; today' the requests bound to it, and their dependents (§3.4) -a name becomes usable, an install ends, or the first parent link connects, on a host: a publication (load's setups, a reload's commit, an install's commit) serves a type, a registry, a version; a registry composes or maps a value at a version; an install ends without a commit (rejected) -the unbound requests indexed under those names; the provider's installing holds; at the first parent link, the - - -no-placement remote requests waiting here; a refused hold re-runs its start and sends loading, then bound or failed - +a name becomes usable, or an install ends, on a host: a publication (load's setups, a reload's commit, an install's commit) serves a type, a registry, a version; a registry composes or maps a value at a version; an install ends without a commit (rejected) +the unbound requests indexed under those names; the provider's installing holds; a refused hold re-runs its start and sends loading, then bound or failed

    No new message kind for wakes: loading, bound, failed and retry exist. The wire's failed carries the new cause diff --git a/packages/chord/docs/next/dependencies/model.md b/packages/chord/docs/next/dependencies/model.md index 172f2a688..2b838eb79 100644 --- a/packages/chord/docs/next/dependencies/model.md +++ b/packages/chord/docs/next/dependencies/model.md @@ -63,12 +63,9 @@ A needs W's B, B needs S's C; W's hold for C waits for S's whole `load`, which w holding host closes), never because its provider is missing, went away or does not fit. - **Placement by address** (round 1): a `local` type goes here; a `remote` type with a placement goes to the address its placement (computed from the consumer's own contract) gives: here if that is this host's address, whether served - here or not, else over the route for that kind or the parent (design §3). A `remote` type **without placement** keeps - today's rule (round 2: Fable, Opus, Astra; no feature removed): here if served here, otherwise over the parent. A - request of that kind made before the host has a parent link waits here `not_served` and is **re-placed when the - host's first parent link connects** (round 3: Opus N5, Sol 3: a tab loads before it connects); on a host that never - connects (the root), it stays here. Opus would rather require a placement on every `remote()`; that removes a - supported declaration, so the draft keeps the fallback. A hold arriving for an address that is not this host's is relayed or + here or not, else over the route for that kind or the parent (design §3). Every `remote` type declares + its placement (owner, 2026-10-29: the optional placement of slice 3a is removed, and with it the fallback "here if + served, otherwise the parent" and its re-placement at the first parent link). A hold arriving for an address that is not this host's is relayed or refused (`invalid/placement`) as today; a relay never captures a hold because it serves the same name. - **Then the selected provider is checked** (served, version or accepted version and fingerprint, `authorize`, the provider's own placement check). Version checks never look at a same-name registration elsewhere. @@ -178,8 +175,7 @@ through, or recovers by its own backoff. | Event | Re-evaluates | | --- | --- | | **a target changes state**: a dependency's object goes live, loading, fails, gets a new cause or ends; a hold's `bound`, `loading`, `failed` arrive, also after a re-hold on a new link | the requests bound to it, and their dependents (§3.4) | -| **a name becomes usable, an install ends, or the first parent link connects, on a host**: a publication (`load`'s setups, a reload's commit, an install's commit) serves a type, a registry, a version; a registry composes or maps a value at a version; an install ends without a commit (rejected) | the unbound requests indexed under those names; the provider's `installing` holds; at the first parent link, the -no-placement remote requests waiting here; a refused hold re-runs its start and sends `loading`, then `bound` or `failed` | +| **a name becomes usable, or an install ends, on a host**: a publication (`load`'s setups, a reload's commit, an install's commit) serves a type, a registry, a version; a registry composes or maps a value at a version; an install ends without a commit (rejected) | the unbound requests indexed under those names; the provider's `installing` holds; a refused hold re-runs its start and sends `loading`, then `bound` or `failed` | No new message kind for wakes: `loading`, `bound`, `failed` and `retry` exist. The wire's `failed` carries the new cause kind `waiting`; `loading` and a `waiting` `failed` carry the provider's attempt id for the probe (§7). From 599127ea3629d3704b389db1b93e57f4eb2ee45c Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 29 Oct 2026 07:24:20 +0000 Subject: [PATCH 130/271] docs(chord): decision: hostAddress instead of placement --- packages/chord/docs/next/decisions.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index d4ba29fe0..b7b7a89ca 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1771,6 +1771,12 @@ validation and copies the v2 kernel added around Delta. setups a registry without a value cannot get one before a reload, so a reader then fails at once `transient` (`unavailable/undeclared`) and is retried when the registry gets its first value (R18), as `self.get` of a type not served here fails at once (refined 2026-10-27 after the second spec review: no `readyTimeout` wait at start); once live, a later failure of the registry changes nothing (last good value). +- **`hostAddress` instead of `placement` (owner, 2026-10-29: "yes, rename"):** the field of a `remote` contract that + computes where its object lives is `hostAddress: (params) => Address` (it returns the address of the host that serves + the object, the vocabulary of `createHost({ address })`, routes and holds); supersedes the name of "`placement`, not + `host`" (2026-10-21). The error reason `invalid/placement` is `invalid/host_address`; the internal module that decides + where a request goes (`host/placement.ts`, `Placement`) is `host/destination.ts`, `Destination`. Earlier entries' and + slice specs' "placement" reads "host address"; they keep their wording as history. - **Every remote type declares its placement (owner, 2026-10-29: "yes, let's remove it"):** `placement` is required in `remote()` (a type error without it, and a declaration error at runtime); a remote type without placement, a leftover of slice 3a (remote objects over one link, before host kinds in 4a), is gone, and with it the fallback "here if served From 943f30c90953b2e0d5eed4407db22c06ad5224d9 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 29 Oct 2026 07:25:19 +0000 Subject: [PATCH 131/271] docs(chord): design: hostAddress instead of placement --- packages/chord/docs/next/design.md | 35 +++++++++++++++--------------- packages/chord/docs/next/slices.md | 2 +- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index 2772ee9e8..754c96cc2 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -52,7 +52,7 @@ versions and adapters · 28 building: slices · 29 telemetry · 30 cost model · | link | a connection between two hosts | | host kind / address | typed description of addressable hosts / one host of a kind (`Session({ id })`) | | facet | a plugin's declarations for one host kind: the types it serves, its contributions; no state or behaviour of its own | -| object type | a contract: name, params (which instance), host placement, members, version | +| object type | a contract: name, params (which instance), host address, members, version | | object | one instance per (type, params, principal key); `local`: per (type, params) | | principal | who calls; stamped by links, typed by the app | | reference | what `get` returns; keeps its object alive; live and hydrated when awaited | @@ -99,7 +99,7 @@ mode sandbox (a worker thread inside a worker), remote environment daemons (thei the TUI sandbox behind its launcher (the launcher only passes frames). A Durable Object deployment is a mesh routed by the edge (section 24 and the research note). -### Host kinds, addresses and placement +### Host kinds and addresses ```ts export const Server = hostKind("server"); @@ -108,37 +108,38 @@ export const Session = hostKind("session", { id: str() }); // params like a export const Notes = remote("notes", { version: 1, params: { session: str() }, // flat record of required scalars - placement: ({ session }) => Session({ id: session }), // a typed function of the params + hostAddress: ({ session }) => Session({ id: session }), // a typed function of the params variants: { … }, }); -export const Catalog = remote("catalog", { placement: () => Server(), … }); +export const Catalog = remote("catalog", { hostAddress: () => Server(), … }); ``` - `hostKind(name, params?)` declares a kind of addressable host; its params follow the params policy of contracts (section 5). `Session({ id: "S1" })` is an **address**: `{ kind, params, key }`, the key built like an identity key (`session|S1`); two addresses are equal when their keys are. No host-name strings appear in app code. -- **`placement`** (owner, 2026-10-21) is a typed function of the params returning an address, required on every - `remote` contract (owner, 2026-10-29); `local` contracts have none. It is not part of the contract fingerprint; a changed rule surfaces as a refusal (below). +- **`hostAddress`** (owner, 2026-10-21 as `placement`, renamed 2026-10-29) is a typed function of the params returning + the address of the host that serves the object, required on every `remote` contract (owner, 2026-10-29); `local` + contracts have none. It is not part of the contract fingerprint; a changed rule surfaces as a refusal (below). - **`params`** (owner, 2026-10-21; was `input`) select the instance among objects of its type and are part of its identity (section 5). - **The caller computes the address** and sends it in the `hold`; a host that relays routes by that address alone and never needs the contract or parses params. **The provider computes it again** from the params and refuses an object - that is not its own: broken `invalid/placement` (misrouted holds are never forwarded). During a rollout the refusal + that is not its own: broken `invalid/host_address` (misrouted holds are never forwarded). During a rollout the refusal is handled like a mismatch (provider on another revision: park; same revision: fail; slice 7). - **Where a reference made on a host, or a hold arriving on one of its links, goes** (first match; slice 4a): | Case | Goes | | --- | --- | - | the type is served here and its placement (computed here from the params) is this host's address, or the host has no address | here | + | the type is served here and its host address (computed here from the params) is this host's address, or the host has no address | here | | a reference made here (`host.get`/`hold`, `self.get`, `ctx.get`) to a served type placed on another host | on, as if not served here (next rows): worker S1 reaches `Notes { session: "S2" }` through the parent | - | an arriving hold for a served type whose params place it on another host | broken `invalid/placement`; the address the hold names only routes it | + | an arriving hold for a served type whose params place it on another host | broken `invalid/host_address`; the address the hold names only routes it | | the hold names an address of a kind this host has a route for | over that route's link (the address's params checked against the kind, then `authorize`) | | otherwise (a remote type) | over the parent link; a hold that arrived *on* the parent link is never sent back to it: `unavailable` | | no parent link: the host never had one | `unavailable` | | no parent link: it is down (a drop) | waits on the parent path, `loading`, until the next link (design audit, 2026-10-27) | - A `placement` or `authorize` that throws fails the hold broken with that error; an address whose params fail the - kind's check is broken `invalid/placement`. + A `hostAddress` or `authorize` that throws fails the hold broken with that error; an address whose params fail the + kind's check is broken `invalid/host_address`. ### The low-level host API @@ -343,7 +344,7 @@ host.route(Session, { ### Relays (slice 4a; decision "Relay flow control per origin", owner, 2026-10-24) -A host that receives a hold it cannot serve (the placement table above: a route or the parent) **relays** it: it holds +A host that receives a hold it cannot serve (the table above: a route or the parent) **relays** it: it holds it again on that link (upstream) with the arriving hold's principal and incarnation, and forwards between the two without parsing payloads. The provider keeps the window, the pending queues and the calls per origin; the relay never drops, folds or resets. @@ -474,7 +475,7 @@ drops, folds or resets. ## 5. Object types ```ts -export const Notes = remote("notes", { version, placement, params, variants }); +export const Notes = remote("notes", { version, hostAddress, params, variants }); export const NoteStore = local("notes.store", { version, params, members }); ``` @@ -520,7 +521,7 @@ host.serve(Notes, ({ session }, principal, self) => fingerprint, never a description. The provider binds when both equal its current ones, or (slice 7) when the version is one it accepts and the fingerprint equals the one computed from its own history (section 27). Every contract change, additive ones included, bumps `version`. Anything else is `contract_mismatch` (or `updating` during - a rollout, section 21). The fingerprint covers params, variants and members, not placement. A consumer whose contract + a rollout, section 21). The fingerprint covers params, variants and members, not the host address. A consumer whose contract does not fit gets a mismatch on its own binding; the provider object and its other bindings are not affected. - **Variants:** a member name has the same kind (state or method) in every variant that has it, and a method the same signature (arguments and result) in every variant; state schemas may differ per variant (redaction). Enforced at @@ -547,7 +548,7 @@ host.serve(Notes, ({ session }, principal, self) => binding is held only by the served type's registration and by calls in flight (GC-tested; no global strong cache). - **Errors** (L1, slice 3a): `ChordError` with a code and, for some codes, a reason, a closed union that is part of the link protocol (adding one bumps the protocol version; a receiver rejects unknown reasons; a test pins the list): - `invalid` (`args`, `params`, `member`, `placement`, `reentrant`, `read_after_bulk`, `cycle`), `contract_mismatch` + `invalid` (`args`, `params`, `member`, `host_address`, `reentrant`, `read_after_bulk`, `cycle`), `contract_mismatch` (`updating`), `forbidden` (`revoked`), `gone`, `not_ready`, `released` (`ended`, `closed`, `settled`, `version`, `discarded`; none for a holder's own release), `too_large` (`part`), `unavailable` (`locked`, `park_timeout`, `busy`, `storage_failed`, `undeclared`: a registry no plugin serves here, slice 8), `aborted`. A failed check names TypeBox's instance path ("unknown field", "missing", else TypeBox's message; keys containing `/` or `~` make the path ambiguous, documented). Plugins throw `PluginError(code, message, @@ -751,7 +752,7 @@ location are dropped. | object `failed` | reject at once with the cause | `failed` with cause and `retryAt` | - A failed reference stays alive: while held, Chord retries per the backoff; `ref.retry()` tries at once (a Retry - button; over a link the `retry { hold }` message, which also re-runs a refused hold's start: authorize, placement, + button; over a link the `retry { hold }` message, which also re-runs a refused hold's start: authorize, the host address, binding; design audit 2026-10-27); success goes `loading`, then `live`. A provider tells a pending hold's consumer each failure (`failed`, with the rest of its backoff window) and the next attempt (`loading`). - **Parked calls** (slice 3b): calls made while the reference is loading are queued in order, each waiting at most the @@ -2229,7 +2230,7 @@ registry keys with a `conflict` flag; (`native/` directories, `defineNative`, lints); weakening type checking; "magic" input-field matching for delete; a factory trial; a migrate dry run; principal field patterns and re-stamping for revocation; `storeKey`; a consumer contract check during staging; reload votes; per-plugin rollback within a host; `defaultProjectTrust: "always"`; loader variants A and B; a topology layer above the host API; a stream type in -Chord; callbacks as method arguments; `opaque` members; numeric middleware priorities; placement as data naming an +Chord; callbacks as method arguments; `opaque` members; numeric middleware priorities; host addresses as data naming an input field; host-name strings in app code; host values (start data and platform bindings are core objects); consumer-side linger; registry composition as a staging check; a JSONL document storage; `tx.get` for documents; validating state and documents on commits or reads (migration results are validated once, R8); copying values on same-host calls; op-rewriting projections and a view algebra; keeping natives across reloads by name; Chord ending the process on a storage failure; an `optional` diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index cf346eb9e..2ddb88f60 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -44,7 +44,7 @@ reload has to reload real registries). | 2 | **Failures and natives** | `self.open`, activation failures and recovery, backoff, the drain, timeouts | | 3a | **Remote objects over one link** | remote contracts, principals, variants, a memory link, the message format, holds, calls, one message per commit (no relay, no drops) | | 3b | **Surviving drops** | re-holds, parking, `ref.state`, `host.hold`, `ref.retry()`, `linger` | -| 4a | **Links and relays** | host kinds, placement, routes, two-sided links, relays and principals, IPC and WebSocket, flow control, heartbeat | +| 4a | **Links and relays** | host kinds, host addresses (`placement` then), routes, two-sided links, relays and principals, IPC and WebSocket, flow control, heartbeat | | 4b | **Authority and housekeeping** | revocation, cycles across hosts, `host.references`, `host.activity`, size limits, compression | | 5 | **Routines and jobs** | `self.routines`, `self.job` | | 6a | **The store** | the store (backends, tables, queries, writes, commit queue, outbox, documents engine, migrations, export), persisted documents, `self.table`, keyed watches and retirement | From d0687bce8582a1cc7077585b71e12b24913d9aa9 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 29 Oct 2026 07:29:25 +0000 Subject: [PATCH 132/271] docs(chord): only hosts with an address serve remote types; holds always carry their address --- packages/chord/docs/next/decisions.md | 4 ++++ packages/chord/docs/next/design.md | 9 +++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index b7b7a89ca..e87d429d7 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1771,6 +1771,10 @@ validation and copies the v2 kernel added around Delta. setups a registry without a value cannot get one before a reload, so a reader then fails at once `transient` (`unavailable/undeclared`) and is retried when the registry gets its first value (R18), as `self.get` of a type not served here fails at once (refined 2026-10-27 after the second spec review: no `readyTimeout` wait at start); once live, a later failure of the registry changes nothing (last good value). +- **Only hosts with an address serve remote types (owner, 2026-10-29: "b"):** a host serving a `remote` type needs an + address and a principal; `load` refuses a facet that serves one on a host without an address (until now a host with + a principal but no address served remote types and placed every one of them "here"). So "here" is decided by the + address alone, as the host address of the type. Holds carry their `address` always (required on the wire). - **`hostAddress` instead of `placement` (owner, 2026-10-29: "yes, rename"):** the field of a `remote` contract that computes where its object lives is `hostAddress: (params) => Address` (it returns the address of the host that serves the object, the vocabulary of `createHost({ address })`, routes and holds); supersedes the name of "`placement`, not diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index 754c96cc2..66a76765f 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -88,8 +88,9 @@ worker worker worker addressable by a typed address, started on demand | server | yes, one | its own declaration | runs | | worker | yes, many | stamped by the parent | started on demand by the app | -Clients serve only `local` types (they are not addressable): a host without a principal refuses to serve a remote type -(`load` throws; a remote object is owned by a principal of the host serving it; owner, 2026-10-24). No client providers for now (owner, +Clients serve only `local` types (they are not addressable): a host without a principal or without an address refuses +to serve a remote type (`load` throws; a remote object is owned by a principal of the host serving it, owner, +2026-10-24; and placed by its host address, owner, 2026-10-29). No client providers for now (owner, 2026-10-28): client-side capabilities (a camera, WebMCP tools) go through an owner's requests that clients answer (section 20, the Questions service). A client's local calls have no principal; what a client calls on the server is stamped with the client's authenticated principal. @@ -130,7 +131,7 @@ export const Catalog = remote("catalog", { hostAddress: () => Server(), … }); | Case | Goes | | --- | --- | - | the type is served here and its host address (computed here from the params) is this host's address, or the host has no address | here | + | the type is served here and its host address (computed here from the params) is this host's address | here | | a reference made here (`host.get`/`hold`, `self.get`, `ctx.get`) to a served type placed on another host | on, as if not served here (next rows): worker S1 reaches `Notes { session: "S2" }` through the parent | | an arriving hold for a served type whose params place it on another host | broken `invalid/host_address`; the address the hold names only routes it | | the hold names an address of a kind this host has a route for | over that route's link (the address's params checked against the kind, then `authorize`) | @@ -1797,7 +1798,7 @@ validation, no clone of snapshots or ops. | Kind | Direction | Fields | | --- | --- | --- | | `hello` | both, first | `protocol`, `incarnation`, `calls` (the call window N) | - | `hold` | to provider | `origin?`, `hold`, `type`, `version`, `fingerprint`, `params` (part), `address?`, `principal?`, `incarnation?`, `chain?` (activation chain, Q5), `limit?` (decoded size limit, 4b) | + | `hold` | to provider | `origin?`, `hold`, `type`, `version`, `fingerprint`, `params` (part), `address`, `principal?`, `incarnation?`, `chain?` (activation chain, Q5), `limit?` (decoded size limit, 4b) | | `revoke` | parent to child (route link) | the revocation value (4b) | | `release`, `retry` | to provider | `origin?`, `hold` | | `call` | to provider | `origin?`, `call`, `hold`, `method` (index), `args?` (part; absent: no arguments), `segments?` | From 5ec24188207cfd7c2d19251db1803383caa1e160 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 29 Oct 2026 07:37:37 +0000 Subject: [PATCH 133/271] feat(chord): hostAddress instead of placement, required on remote types; only hosts with an address serve remote types The contract field placement is hostAddress, required on every remote() (type and declaration error); the fallback for remote types without one is gone. invalid/placement is invalid/host_address. host/placement.ts (Placement) is host/destination.ts (Destination); here is decided by address alone, and a host without an address refuses to serve a remote type. A hold always carries its address on the wire (the binary codec's ADDRESS flag is gone, the text codec requires it). Tests and examples declare their host addresses. --- packages/chord/src/next/contracts/README.md | 10 +- packages/chord/src/next/contracts/contract.ts | 8 +- packages/chord/src/next/contracts/declare.ts | 20 ++-- .../chord/src/next/contracts/host-kind.ts | 2 +- packages/chord/src/next/data/README.md | 4 +- packages/chord/src/next/data/errors.ts | 2 +- packages/chord/src/next/data/params.ts | 6 +- packages/chord/src/next/data/validator.ts | 4 +- packages/chord/src/next/host/README.md | 23 ++-- .../host/{placement.ts => destination.ts} | 63 +++++----- packages/chord/src/next/host/host.ts | 42 ++++--- packages/chord/src/next/host/route.ts | 2 +- packages/chord/src/next/link/README.md | 6 +- packages/chord/src/next/link/binary.ts | 25 ++-- packages/chord/src/next/link/message.ts | 4 +- packages/chord/src/next/link/text.ts | 24 ++-- packages/chord/src/next/remote/README.md | 4 +- packages/chord/src/next/remote/hold.ts | 9 +- .../chord/src/next/remote/remote-object.ts | 8 +- packages/chord/test/next/bench/compression.ts | 12 +- packages/chord/test/next/bench/links.ts | 7 +- packages/chord/test/next/bench/relay.ts | 2 +- .../chord/test/next/contracts/accepts.test.ts | 4 + .../test/next/contracts/contract.test.ts | 3 +- .../chord/test/next/contracts/declare.test.ts | 109 ++++++++++++++---- .../chord/test/next/contracts/evolve.test.ts | 11 +- .../test/next/contracts/fingerprint.test.ts | 22 +++- .../test/next/contracts/host-kind.test.ts | 28 +++-- .../chord/test/next/contracts/params.test.ts | 29 ++++- packages/chord/test/next/data/args.test.ts | 3 +- packages/chord/test/next/data/errors.test.ts | 2 +- packages/chord/test/next/data/failure.test.ts | 2 +- packages/chord/test/next/data/params.test.ts | 3 +- .../chord/test/next/examples/08-remote.ts | 14 ++- packages/chord/test/next/examples/09-calls.ts | 12 +- packages/chord/test/next/examples/10-drops.ts | 12 +- .../chord/test/next/examples/12-routes.ts | 7 +- packages/chord/test/next/examples/13-relay.ts | 2 +- .../test/next/examples/14-flow-control.ts | 2 +- .../chord/test/next/examples/15-revocation.ts | 2 +- .../next/examples/16-presence-and-activity.ts | 8 +- .../examples/17-limits-and-compression.ts | 14 ++- packages/chord/test/next/examples/19-jobs.ts | 12 +- .../chord/test/next/host/activity.test.ts | 9 +- packages/chord/test/next/host/attempt.test.ts | 19 ++- packages/chord/test/next/host/ctx.test.ts | 19 ++- packages/chord/test/next/host/host.test.ts | 63 +++++++--- .../test/next/host/member-source.test.ts | 11 +- packages/chord/test/next/host/object.test.ts | 26 ++++- .../chord/test/next/host/presence.test.ts | 54 +++++++-- .../chord/test/next/host/reference.test.ts | 6 +- .../chord/test/next/host/revocation.test.ts | 19 ++- packages/chord/test/next/host/route.test.ts | 16 ++- .../chord/test/next/host/routines.test.ts | 13 ++- packages/chord/test/next/jobs/job.test.ts | 13 ++- packages/chord/test/next/link/channel.test.ts | 10 +- packages/chord/test/next/link/message.test.ts | 72 ++++++++++-- .../chord/test/next/link/websocket.test.ts | 4 +- .../chord/test/next/remote/consumer.test.ts | 4 + packages/chord/test/next/remote/hold.test.ts | 37 ++++-- .../chord/test/next/remote/host-link.test.ts | 8 +- .../chord/test/next/remote/provider.test.ts | 63 ++++++++-- packages/chord/test/next/remote/relay.test.ts | 7 +- .../test/next/remote/remote-object.test.ts | 26 ++++- packages/chord/test/next/support.ts | 20 +++- 65 files changed, 801 insertions(+), 306 deletions(-) rename packages/chord/src/next/host/{placement.ts => destination.ts} (56%) diff --git a/packages/chord/src/next/contracts/README.md b/packages/chord/src/next/contracts/README.md index a6e2a06ba..bd79e9bc0 100644 --- a/packages/chord/src/next/contracts/README.md +++ b/packages/chord/src/next/contracts/README.md @@ -12,7 +12,7 @@ const Note = obj({ text: str() }); export const NotesV1 = remote("notes", { version: 1, params: { session: str() }, - placement: ({ session }) => Session({ id: session }), // where an object lives; not in the fingerprint + hostAddress: ({ session }) => Session({ id: session }), // where an object lives; not in the fingerprint variants: { editor: { state: { notes: arr(Note) }, methods: { add: method(Note, str()), clear: method() } }, reader: { state: { notes: arr(Note) } }, @@ -22,17 +22,17 @@ export const Notes = evolve(NotesV1, (e) => e.args("add", obj({ text: str(), tag type NotesProvider = Implementation; // { variant: "editor"; members: { notes, add(args, context), clear(context) } } | { variant: "reader"; ... } ``` -- `remote(name, { version, params, placement?, variants })`: per-principal variants, each with `state` (schemas) and `methods` +- `remote(name, { version, params, hostAddress, variants })`: per-principal variants, each with `state` (schemas) and `methods` (`method(args?, result?)`; an omitted side is `null`, and a method whose arguments are `null` omits them: `clear(context)`). `local(name, { version, params, members })`: one flat map, where a schema is state and `method(...)` a method. -- `placement(params) => Address` (remote only, optional): the address of the host that serves the object for these +- `hostAddress(params) => Address` (remote only, required; a declaration error without it): the address of the host that serves the object for these params (`host/`'s routing table). The caller computes it and sends it with its hold, so a relay routes by the address alone. It is not part of the normal form or the fingerprint and `evolve` carries it. - `hostKind(name, params?)`: a kind of addressable host (`Server = hostKind("server")`, `Session = hostKind("session", { id: str() })`); `params` follow the params policy below. Calling it makes an `Address { kind, params, key }`, its key built like an identity key (`session|S1`); two addresses are equal when their keys are. The params are checked - where the address is routed or served (`invalid/placement`), not when it is made. No host-name strings in app code. + where the address is routed or served (`invalid/host_address`), not when it is made. No host-name strings in app code. - Schema helpers: `str`, `num`, `int`, `bool`, `nul`, `lit`, `oneOf`, `union`, `obj`, `arr`, `map`, `opt`, `bytes`, `unknown`, and `Static`. They return ordinary TypeBox schemas. - `evolve(previous, (e) => e...)`: the next version from structured edits (`state`, `addState`, `removeState`, `args`, @@ -65,7 +65,7 @@ Rules: - `checkAccepts(contract, accepts)`: validates an accepts table without calling a mapping; returns the accepted versions (version and fingerprint). -- `contractData(contract)`: the Chord-owned data behind a handle (its `placement`, its versions with their +- `contractData(contract)`: the Chord-owned data behind a handle (its `hostAddress`, its versions with their `VariantData`, `MethodData` and schema snapshots, its `ParamsData`); a local contract's members are the variant `LOCAL`. Type-level helpers: `ShapeOf`, `CurrentVariants`, `ContractShape`, `Params` and `Method`. - `currentVariants(contract)`: the current version's variants (a local contract has the one variant `LOCAL`); diff --git a/packages/chord/src/next/contracts/contract.ts b/packages/chord/src/next/contracts/contract.ts index cf113e96f..24a4fb47c 100644 --- a/packages/chord/src/next/contracts/contract.ts +++ b/packages/chord/src/next/contracts/contract.ts @@ -97,8 +97,8 @@ export type ContractData = { readonly kind: Kind; readonly name: string; readonly params: ParamsData; - /** A remote contract's placement: the address of the host serving an object for its params. */ - readonly placement: Placement | undefined; + /** A remote contract's host address: the address of the host serving an object for its params; none for local. */ + readonly hostAddress: HostAddress | undefined; /** Every version of the history, oldest first; the last is current. */ readonly versions: readonly VersionData[]; /** The kind of every member name ever declared in the history. */ @@ -107,8 +107,8 @@ export type ContractData = { export type MemberKind = "state" | "method" | "keyed"; -/** A placement as Chord calls it: with checked params. Plugin code: what it returns is checked where it is routed. */ -export type Placement = (params: ParamsOf) => Address; +/** A `hostAddress` as Chord calls it: with checked params. Plugin code: what it returns is checked where routed. */ +export type HostAddress = (params: ParamsOf) => Address; export type VersionData = { readonly version: number; diff --git a/packages/chord/src/next/contracts/declare.ts b/packages/chord/src/next/contracts/declare.ts index 2e3856c28..8bd3f5533 100644 --- a/packages/chord/src/next/contracts/declare.ts +++ b/packages/chord/src/next/contracts/declare.ts @@ -4,11 +4,11 @@ import { type Contract, createContract, + type HostAddress, type Kind, LOCAL, type MemberKind, type MethodData, - type Placement, type VariantData, } from "./contract.ts"; import { materialize } from "./fingerprint.ts"; @@ -79,7 +79,7 @@ export function remote< readonly version: V; readonly params: P & NoInfer>; /** The address of the host serving an object for these params; never part of the fingerprint. */ - readonly placement?: (params: { [F in keyof P]: Static }) => Address; + readonly hostAddress: (params: { [F in keyof P]: Static }) => Address; readonly variants: D & NoInfer>; }, ): Contract<{ @@ -90,16 +90,14 @@ export function remote< }> { // Declaration policy: JavaScript callers reach every check below. checkName(name, "A contract name"); - checkKeys(declaration, ["version", "params", "placement", "variants"], name); + checkKeys(declaration, ["version", "params", "hostAddress", "variants"], name); const variants = entriesOf(declaration.variants, `${name}.variants`); if (variants.length === 0) throw new Error(`${name}.variants: at least one variant is required`); - const { placement } = declaration; - if (placement !== undefined && typeof placement !== "function") { - throw new Error(`${name}.placement: expected a function of the params`); - } + const { hostAddress } = declaration; + if (typeof hostAddress !== "function") throw new Error(`${name}.hostAddress: expected a function of the params`); const read = variants.map(([variant, members]) => [variant, readVariant(members, `${name}.${variant}`)] as const); - // The placement gets the checked params of this contract. - return declare("remote", name, declaration, new Map(read), placement as Placement | undefined); + // The host address gets the checked params of this contract. + return declare("remote", name, declaration, new Map(read), hostAddress as HostAddress); } /** @@ -165,7 +163,7 @@ function declare( name: string, declaration: { readonly version: number; readonly params: unknown }, variants: ReadonlyMap, - placement?: Placement, + hostAddress?: HostAddress, ): C { const { version } = declaration; if (!Number.isSafeInteger(version) || version < 1) throw new Error(`${name}.version: expected a positive integer`); @@ -173,7 +171,7 @@ function declare( checkMembers(variants, kinds, (variant, member, memberKind) => memberPath(name, kind, variant, member, memberKind)); const params = readParams(entriesOf(declaration.params, `${name}.params`), `${name}.params`); const versions = [materialize(kind, params, version, variants, [])]; - return createContract({ kind, name, params, placement, versions, kinds }); + return createContract({ kind, name, params, hostAddress, versions, kinds }); } /** diff --git a/packages/chord/src/next/contracts/host-kind.ts b/packages/chord/src/next/contracts/host-kind.ts index 3f0e61055..b71152d55 100644 --- a/packages/chord/src/next/contracts/host-kind.ts +++ b/packages/chord/src/next/contracts/host-kind.ts @@ -1,5 +1,5 @@ // Host kinds and addresses (§3): a kind of addressable host with params like a contract's, and the address of one -// host of a kind, which a remote contract's placement computes from its params. +// host of a kind, which a remote contract's `hostAddress` computes from its params. import { checkName } from "./names.ts"; import { joinKey, type Params, type ParamsData, type RequiredFields, readParams } from "./params.ts"; diff --git a/packages/chord/src/next/data/README.md b/packages/chord/src/next/data/README.md index 1b8817496..c3caba867 100644 --- a/packages/chord/src/next/data/README.md +++ b/packages/chord/src/next/data/README.md @@ -33,7 +33,7 @@ try { - `checkParams(contract, raw, context)`: one compiled closed-object check of a get's params and their key; `identityKey(contract, key, principalKey)` (no principal key for `local`); `checkAddress(kind, params, raw, - context)`: the same check of an address's params against its host kind's (`invalid/placement`), and its key. + context)`: the same check of an address's params against its host kind's (`invalid/host_address`), and its key. - `bindProvider(contract, accepts)` once per registration, then `checkArgs(provider, method, raw, boundVersion, context)` per call. It returns `{ kind: "method", args, result? }` (current-shape arguments, and the result mapping back to the caller's version) or `{ kind: "call", args, call }` (old arguments for the provider's replacement). @@ -51,7 +51,7 @@ Rules: them. Results and state are never checked. Nothing in L1 is frozen. - A method without arguments (argument schema `null`) omits the argument in references and implementations; `null` is its internal and wire form, which `checkArgs` checks. Unknown fields fail. -- A failed check is `invalid/args`, `invalid/params` or `invalid/placement` with `details.path`, a JSON Pointer from TypeBox's first error. +- A failed check is `invalid/args`, `invalid/params` or `invalid/host_address` with `details.path`, a JSON Pointer from TypeBox's first error. TypeBox does not escape `~` and `/`, so keys containing them make the path ambiguous. - `checkArgs` refuses an unknown method with `invalid/member` and an unaccepted version or a refused method with `contract_mismatch`. It never invokes a `call` replacement. diff --git a/packages/chord/src/next/data/errors.ts b/packages/chord/src/next/data/errors.ts index 623c05fe2..acb8a7d4e 100644 --- a/packages/chord/src/next/data/errors.ts +++ b/packages/chord/src/next/data/errors.ts @@ -8,7 +8,7 @@ export const PROTOCOL_VERSION = 1; /** Every code with the reasons it may carry. `invalid` requires a reason; every other code may omit it. */ export const CHORD_ERROR_REASONS = { - invalid: ["args", "params", "member", "placement", "reentrant", "read_after_bulk", "cycle"], + invalid: ["args", "params", "member", "host_address", "reentrant", "read_after_bulk", "cycle"], contract_mismatch: ["updating"], forbidden: ["revoked"], gone: [], diff --git a/packages/chord/src/next/data/params.ts b/packages/chord/src/next/data/params.ts index 17ce090e1..be971496f 100644 --- a/packages/chord/src/next/data/params.ts +++ b/packages/chord/src/next/data/params.ts @@ -32,11 +32,11 @@ export function identityKey( } /** - * An address from a kind's name and params as a hold names it or a placement returned it: the params checked against - * the kind's with one compiled closed-object check (`invalid/placement` on failure), the key computed. + * An address from a kind's name and params as a hold names it or a `hostAddress` returned it: the params checked + * against the kind's with one compiled closed-object check (`invalid/host_address` on failure), the key computed. */ export function checkAddress(kind: string, params: ParamsData, raw: unknown, context: Context): Address { - check(validatorFor(params, context), raw, "placement", context); + check(validatorFor(params, context), raw, "host_address", context); // The check above establishes a flat record of the kind's scalar fields. const values = raw as { readonly [field: string]: string | number | boolean }; return { kind, params: values, key: addressKey(kind, params, values) }; diff --git a/packages/chord/src/next/data/validator.ts b/packages/chord/src/next/data/validator.ts index 6c3369a6b..4a16599de 100644 --- a/packages/chord/src/next/data/validator.ts +++ b/packages/chord/src/next/data/validator.ts @@ -34,7 +34,7 @@ const validators = new WeakMap(); export function check( validator: Validator, value: unknown, - reason: "args" | "params" | "placement", + reason: "args" | "params" | "host_address", context: Context, ): void { context.telemetry.metrics?.counter("chord.data.validations").add(1); @@ -48,7 +48,7 @@ export function check( function failedCheck( schema: TSchema, value: unknown, - reason: "args" | "params" | "placement", + reason: "args" | "params" | "host_address", context: Context, ): ChordError { context.telemetry.metrics?.counter("chord.data.walks").add(1); diff --git a/packages/chord/src/next/host/README.md b/packages/chord/src/next/host/README.md index da13258bc..b5c9175fa 100644 --- a/packages/chord/src/next/host/README.md +++ b/packages/chord/src/next/host/README.md @@ -79,7 +79,7 @@ Addressable hosts and routes (slice 4a): a server routes holds of worker types t export const Server = hostKind("server"); export const Session = hostKind("session", { id: str() }); export const Notes = remote("notes", { version: 1, params: { session: str() }, - placement: ({ session }) => Session({ id: session }), variants: { … } }); + hostAddress: ({ session }) => Session({ id: session }), variants: { … } }); const server = createHost

    ({ address: Server(), principal: { kind: "server" }, onError }); server.route(Session, { // before load, one route per kind @@ -125,7 +125,8 @@ server.accept(webSocketLink(ws), { principal, limits: { part: 16 * MiB, decoded: - `createHost({ address?, principal?, principalKey?, telemetry?, onError, closeTimeout?, readyTimeout?, heartbeat?, revokedBy?, compression? })`; `R` is the app's revocation type (JSON). `address` (`Session({ id: "S1" })`, see `contracts/`'s `hostKind`) makes the host addressable: it serves only objects placed there; without one it is a client. A host with a `principal` acts as it for its own requests; a host - without one (a client) has none and serves only local types (`serve` of a remote type throws at `load`). + without one (a client) has none. A host serves remote types only with both a principal and an address (`serve` of a + remote type throws at `load` otherwise). `principalKey` maps a principal to its part of an identity (L1's canonical JSON by default; the same function on every host). `closeTimeout` (30 s) bounds each native's close and the drain of an ending object's calls; `readyTimeout` (60 s) bounds every activation @@ -166,7 +167,7 @@ server.accept(webSocketLink(ws), { principal, limits: { part: 16 * MiB, decoded: (values, ops, results, arguments; never params, never arguments sent up its parent link) and inflates deflated parts it owns in place. Child links refuse compressed parts (a protocol error: clients never send them, and a small part could inflate to 64 MiB). Without one (a browser) it sends uncompressed and inflates asynchronously - (`DecompressionStream`), only parts it owns once their id and placement are known and their wire size passed, + (`DecompressionStream`), only parts it owns once their id and destination are known and their wire size passed, within the smaller of its link's decoded limit and a hold's stamped one, holding later messages of the link in order; acks go out after a message was applied, so a slow inflate is backpressure like a slow consumer. - **Cycles across hosts (Q5):** every object instance gets an instance id (uuidv7) at its first attempt; a new object @@ -235,17 +236,17 @@ server.accept(webSocketLink(ws), { principal, limits: { part: 16 * MiB, decoded: the object activates or retries, while there is no parent link or it is down; `failed` while the object is failed and no attempt runs; `live`. Throws at once for invalid params (`invalid/params`) and a local type not served here (`unavailable`); before `load` resolved - these are decided once it did, and the reference fails `broken` with that error. A placement or `authorize` that - throws or refuses, or a placement whose params fail its kind's check (`invalid/placement`), fails the reference + these are decided once it did, and the reference fails `broken` with that error. A `hostAddress` or `authorize` that + throws or refuses, or a host address whose params fail its kind's check (`invalid/host_address`), fails the reference `broken`. A local type never crosses a link. Where a reference made here (`host.get`/`hold`, `self.get`, `ctx.get`) - or a hold arriving on a link goes, first match (`Placement`; the address is the contract's `placement` of the params, + or a hold arriving on a link goes, first match (`Destination`; the address is the contract's `hostAddress` of the params, computed by the caller; a hold carries it to be routed, and a host serving the type computes it again): | Case | Goes | | --- | --- | - | the type is served here and its address is this host's, or the host or the type has none | here | + | the type is served here and its address is this host's | here | | a reference made here to a served type whose address is another host's | on, as if not served here (rows below): worker S1 reaches `Notes { session: "S2" }` through the parent | - | an arriving hold for a served type whose params place it on another host | broken `invalid/placement` (misrouted; never forwarded) | + | an arriving hold for a served type whose params place it on another host | broken `invalid/host_address` (misrouted; never forwarded) | | the address is of a kind this host has a route for | over that route's link (the params checked against the kind, `authorize` first; `forbidden` when it refuses) | | otherwise (a remote type) | over the parent link, open yet or not; a hold that arrived *on* the parent link is never sent back: `unavailable` | | an arriving hold and no parent link | `unavailable` | @@ -458,7 +459,7 @@ cycles, throwing factories), `06-natives.ts` (opening in order, `not_ready`, clo `08-remote.ts` (two hosts, variants per principal over a shared store, replicas), `09-calls.ts` (calls over a link, bytes, errors, `ctx.principal`, cancellation, release), `10-drops.ts` (a hold before any link, `ref.state`, parked calls, a drop and the re-hold), `11-linger-and-retry.ts` (`linger`, failed references, `ref.retry()`), -`12-routes.ts` (addresses, placement, a route to an in-process worker, a guest's `ctx.get` through it), +`12-routes.ts` (addresses, host addresses, a route to an in-process worker, a guest's `ctx.get` through it), `13-relay.ts` (tab, server and a forked worker over WebSocket and IPC, the worker's start data through its core facet, §18), `14-flow-control.ts` (a slow tab: folding at the provider, and at the worker per origin through a relay while a fast tab keeps receiving), `15-revocation.ts` (a grant revoked: its tabs' links closed, its objects on @@ -488,8 +489,8 @@ delta)` (the keep-alive counts), `cycle(chain, id, closing)` (the `invalid/cycle Inside the directory: `ChordHost` (types, objects by identity, facets, the batch, the backoff map, the endings new instances wait for, its links and the heartbeat timer, its routes and the parent path; `reference` checks a reference's route, `bind` binds it to an object here or a remote object over a path, `acquire` gets one for a -caller), `Placement` (where a reference or an arriving hold goes: here, a route's path or the parent's; whether a hold -served here is placed here), `Route` (one per routed kind: per address its `Path`, at most one running +caller), `Destination` (where a reference or an arriving hold goes: here, a route's path or the parent's; whether a +hold served here is placed here), `Route` (one per routed kind: per address its `Path`, at most one running `resolve`, the backoff per address; an address is kept while it has holders, a link or a running `resolve`), `ChordObject` (one per identity: its state across attempts, holders, retries, cycle checks with its instance id and the chain it was created for, lingering, ending and revocation, the count of its running and finishing job runs, diff --git a/packages/chord/src/next/host/placement.ts b/packages/chord/src/next/host/destination.ts similarity index 56% rename from packages/chord/src/next/host/placement.ts rename to packages/chord/src/next/host/destination.ts index e6011168f..d92fbb0f1 100644 --- a/packages/chord/src/next/host/placement.ts +++ b/packages/chord/src/next/host/destination.ts @@ -1,6 +1,6 @@ -// Placement (§3, section 1's table): where a reference made on a host, or a hold arriving on one of its links, goes: -// here, over the route of its placement's address, or over the parent link; and the check that an object served here -// is placed here. +// Destination (§3, section 1's table): where a reference made on a host, or a hold arriving on one of its links, goes: +// here, over the route of its host address, or over the parent link; and the check that an object served here lives +// here. import type { Context } from "../../context/index.ts"; import type { Address, Contract, ParamsOf } from "../contracts/index.ts"; @@ -12,13 +12,13 @@ import type { HoldRequest, HostLink, Path } from "../remote/internal.ts"; import type { Served } from "./object.ts"; import type { Route } from "./route.ts"; -/** Where something goes: here, or over a path (with the address it was placed at, if its type has placement). */ +/** Where something is placed: here, or over a path (with the host address it was computed or named with). */ export type Placed = | { readonly kind: "here" } - | { readonly kind: "path"; readonly path: Path; readonly address: WireAddress | undefined }; + | { readonly kind: "path"; readonly path: Path; readonly address: WireAddress }; -/** A host's placement: its address, its served types, its routes and its parent path, as the host keeps them. */ -export class Placement { +/** A host's destinations: its address, its served types, its routes and its parent path, as the host keeps them. */ +export class Destination { readonly #address: Address | undefined; readonly #types: ReadonlyMap; readonly #routes: ReadonlyMap; @@ -37,15 +37,16 @@ export class Placement { } /** - * Where a reference to `Type` with `params` made as `principal` goes: here when its type is served here and placed - * here (or this host has no address, or the type no placement); otherwise over the route of its placement's kind, - * or over the parent link (placement decides where an object lives: a worker reaches another worker's object of a - * type it serves too). Throws `invalid/placement` for a placement whose params do not fit its kind, `forbidden` when - * the route refuses, and what a placement or `authorize` throws. + * Where a reference to `Type` with `params` made as `principal` goes: here for a local type (served here: the host + * refused it otherwise) or a remote type served here whose host address is this host's; otherwise over the route of + * its host address's kind, or over the parent link (the host address decides where an object lives: a worker + * reaches another worker's object of a type it serves too). Throws `invalid/host_address` for a host address whose + * params do not fit its kind, `forbidden` when the route refuses, and what a `hostAddress` or `authorize` throws. */ reference(Type: Contract, params: ParamsOf, principal: Principal | undefined, context: Context): Placed { - const address = placementOf(Type, params); - if (this.#types.has(Type.name) && this.#placedHere(address)) return HERE; + if (Type.kind === "local") return HERE; + const address = hostAddressOf(Type, params); + if (this.#types.has(Type.name) && this.#isHere(address)) return HERE; return { kind: "path", path: this.#pathTo(address, principal, context) ?? this.#parent, address }; } @@ -66,40 +67,40 @@ export class Placement { } /** - * Throws `invalid/placement` for a hold of `Type` with `params` (checked) served here but placed on another host - * (misrouted; never forwarded). The placement is computed here from the params: the address a hold names only - * routes it. + * Throws `invalid/host_address` for a hold of `Type` with `params` (checked) served here but whose host address is + * another host (misrouted; never forwarded). The host address is computed here from the params: the address a hold + * names only routes it. */ assertHere(Type: Contract, params: ParamsOf): void { - const address = placementOf(Type, params); - if (this.#placedHere(address)) return; - throw invalid("placement", `${Type.name}: placed on ${address!.kind}, this host is ${this.#address!.key}`); + const address = hostAddressOf(Type, params); + if (this.#isHere(address)) return; + // A host serving a remote type has an address (`serve` refuses one without). + throw invalid("host_address", `${Type.name}: placed on ${address.kind}, this host is ${this.#address!.key}`); } /** * The path to `address` when this host routes its kind (its params checked, `authorize` run for `principal`); - * `undefined` without an address or a route for its kind. + * `undefined` without a route for its kind. */ - #pathTo(address: WireAddress | undefined, principal: Principal | undefined, context: Context): Path | undefined { - if (address === undefined) return undefined; + #pathTo(address: WireAddress, principal: Principal | undefined, context: Context): Path | undefined { return this.#routes.get(address.kind)?.path(address.params, principal, context); } - /** Whether an object placed at `address` lives here: this host has no address, the type no placement, or equal. */ - #placedHere(address: WireAddress | undefined): boolean { + /** Whether an object whose host address is `address` lives here: this host has that address. */ + #isHere(address: WireAddress): boolean { const own = this.#address; - return own === undefined || address === undefined || sameAddress(own, address); + return own !== undefined && sameAddress(own, address); } } const HERE: Placed = { kind: "here" }; -/** The address `Type`'s placement computes for `params`; `undefined` for a type without placement. */ -function placementOf(Type: Contract, params: ParamsOf): WireAddress | undefined { - const placement = contractData(Type).placement; - if (placement === undefined) return undefined; +/** The address a remote `Type`'s `hostAddress` computes for `params`. */ +function hostAddressOf(Type: Contract, params: ParamsOf): WireAddress { + // Every remote contract has one (declaration policy, `remote`). + const hostAddress = contractData(Type).hostAddress!; // Plugin code: what it returns is checked where it is routed or served. - const { kind, params: values } = placement(params) as { readonly kind: string; readonly params: unknown }; + const { kind, params: values } = hostAddress(params) as { readonly kind: string; readonly params: unknown }; return { kind, params: values }; } diff --git a/packages/chord/src/next/host/host.ts b/packages/chord/src/next/host/host.ts index 5b360e4ba..5612e562a 100644 --- a/packages/chord/src/next/host/host.ts +++ b/packages/chord/src/next/host/host.ts @@ -48,11 +48,11 @@ import { Activity, type ActivityCounts } from "./activity.ts"; import { notLive } from "./attempt.ts"; import { Backoff } from "./backoff.ts"; import { type Caller, type Ctx, withLinkedAbortSignal } from "./ctx.ts"; +import { Destination, type Placed } from "./destination.ts"; import { type Facet, type FacetContribution, type ServedEntry, setUp } from "./facet.ts"; import { contributionOf } from "./members.ts"; import type { CloseFailed } from "./native.ts"; import { ChordObject, type Holder, type LocalContract, type RemoteContract, type Served } from "./object.ts"; -import { type Placed, Placement } from "./placement.ts"; import { type Presence, References } from "./presence.ts"; import { Binding, type CheckedParams, type Held, type Reference, type Target } from "./reference.ts"; import { RevocationKeys, type RevokedBy } from "./revocation.ts"; @@ -191,8 +191,8 @@ export interface Host

    }[], context: Context): Promise; /** * A reference once its object is live; waits for `load` first. It acts as the host's principal and goes where its - * type's placement puts it: here, over a route, or over the parent link (`Placement`). A broken failure rejects at - * once; through transient ones it waits until its deadline, then rejects with the last failure's error. + * type's host address puts it: here, over a route, or over the parent link (`Destination`). A broken failure + * rejects at once; through transient ones it waits until its deadline, then rejects with the last failure's error. */ get(Type: C, params: ParamsOf, context: Context, options?: GetOptions): Promise>; /** @@ -316,7 +316,7 @@ export class ChordHost

    {}); readonly #routes = new Map(); - readonly #placement: Placement; + readonly #destination: Destination; /** The downstream origins this host relays holds for, by their number on upstream links (never reused). */ readonly relays = new Map(); #nextOrigin = 1; @@ -344,7 +344,7 @@ export class ChordHost

    | undefined); this.timeouts = { close: options.closeTimeout ?? 30_000, ready: options.readyTimeout ?? 60_000 }; this.#heartbeat = { ping: options.heartbeat?.ping ?? 15_000, close: options.heartbeat?.close ?? 30_000 }; - this.#placement = new Placement(this.address, this.#types, this.#routes, this.#parent); + this.#destination = new Destination(this.address, this.#types, this.#routes, this.#parent); // Chord's own: `openStore` made it. this.store = options.store as ChordStore | undefined; this.store?.attach({ failed: (error) => this.#storageFailed(error), batch: this.batch }); @@ -576,13 +576,13 @@ export class ChordHost

    , principal: Principal | undefined): Path { // A retry has no context of its own: its admission checks count on none. - const placed = this.#placement.reference(Type, params, principal, BACKGROUND_CONTEXT); + const placed = this.#destination.reference(Type, params, principal, BACKGROUND_CONTEXT); if (placed.kind === "path") return placed.path; throw new ChordError({ code: "unavailable", message: `${Type.name} is served here now` }); } @@ -713,8 +713,8 @@ export class ChordHost

    (ADDRESS | PRINCIPAL | INCARNATION | CHAIN | LIMIT)) throw new Error("binary: unknown hold flags"); - const address = flags & ADDRESS ? { address: { kind: reader.string(), params: reader.json() } } : {}; + if (flags > (PRINCIPAL | INCARNATION | CHAIN | LIMIT)) throw new Error("binary: unknown hold flags"); const principal = flags & PRINCIPAL ? { principal: reader.json() as { kind: string } } : {}; const incarnation = flags & INCARNATION ? { incarnation: uuid(reader.raw(16)) } : {}; const chain = flags & CHAIN ? { chain: reader.list(() => uuid(reader.raw(16))) } : {}; @@ -221,7 +216,7 @@ function decodeFields(kind: (typeof KINDS)[number], reader: Reader): Message { version, fingerprint, params, - ...address, + address, ...principal, ...incarnation, ...chain, diff --git a/packages/chord/src/next/link/message.ts b/packages/chord/src/next/link/message.ts index a8c022571..84bab8629 100644 --- a/packages/chord/src/next/link/message.ts +++ b/packages/chord/src/next/link/message.ts @@ -62,8 +62,8 @@ const MESSAGE = union([ version: int({ min: 1 }), fingerprint: str({ pattern: "^[0-9a-f]{16}$" }), params: part, - // The address the caller computed (absent for a type without placement). - address: opt(address), + // The address the caller computed with the type's `hostAddress`: it routes the hold. + address, // Whom the hold acts for and the origin's incarnation; read only on a parent link. principal: opt(principal), incarnation: opt(incarnation), diff --git a/packages/chord/src/next/link/text.ts b/packages/chord/src/next/link/text.ts index c9b7b8e0d..e20c741bb 100644 --- a/packages/chord/src/next/link/text.ts +++ b/packages/chord/src/next/link/text.ts @@ -75,16 +75,10 @@ function ownTuple(message: Message): string { const { address, principal, incarnation, chain, limit } = message; const head = json(["H", message.hold, message.type, message.version, message.fingerprint]).slice(0, -1); // Optional trailing fields: an absent one in the middle is null, absent ones at the end are left out. - const optional = [ - address === undefined ? null : [address.kind, address.params], - principal ?? null, - incarnation ?? null, - chain ?? null, - limit ?? null, - ]; + const optional = [principal ?? null, incarnation ?? null, chain ?? null, limit ?? null]; while (optional.length > 0 && optional.at(-1) === null) optional.pop(); const rest = optional.map((field) => `,${json(field)}`).join(""); - return `${head},${partText(message.params)}${rest}]`; + return `${head},${partText(message.params)},${json([address.kind, address.params])}${rest}]`; } case "release": return json(["R", message.hold]); @@ -188,14 +182,10 @@ function fromFields(fields: unknown[], rest: Uint8Array): Message { arity(3); return { kind: "hello", protocol: field(0), incarnation: field(1), calls: field(2) }; case "H": { - arity(5, 6, 7, 8, 9, 10); - const [address, principal, incarnation, chain, limit] = [5, 6, 7, 8, 9].map((index) => f[index] ?? null); - let wireAddress = {}; - if (address !== null) { - const pair = list(address); - if (pair.length !== 2) throw new Error("text: an address is a pair"); - wireAddress = { address: { kind: pair[0], params: pair[1] } }; - } + arity(6, 7, 8, 9, 10); + const address = list(f[5]); + if (address.length !== 2) throw new Error("text: an address is a pair"); + const [principal, incarnation, chain, limit] = [6, 7, 8, 9].map((index) => f[index] ?? null); return { kind: "hold", hold: field(0), @@ -203,7 +193,7 @@ function fromFields(fields: unknown[], rest: Uint8Array): Message { version: field(2), fingerprint: field(3), params: textPart(f[4]), - ...wireAddress, + address: { kind: address[0] as string, params: address[1] }, ...(principal === null ? {} : { principal: principal as { kind: string } }), ...(incarnation === null ? {} : { incarnation: incarnation as string }), ...(chain === null ? {} : { chain: chain as string[] }), diff --git a/packages/chord/src/next/remote/README.md b/packages/chord/src/next/remote/README.md index cd27471a5..16522f71b 100644 --- a/packages/chord/src/next/remote/README.md +++ b/packages/chord/src/next/remote/README.md @@ -83,7 +83,7 @@ within each link's limits, compression at the origin, activation chains and revo `state` with A's and B's entries: the derive sees both at once, never A new with B old. - `Hold` (one per `hold` served here, a `Holder` of its object): takes its object once `load` resolved; the hold's `params` part is parsed then (malformed: a protocol error). A refused type, a mismatch, invalid params, a wrong - placement or a snapshot that throws is a broken `failed` that ends it (the provider forgets the id). While pending + host address or a snapshot that throws is a broken `failed` that ends it (the provider forgets the id). While pending it follows its object's `state`: each failure, transient or broken, is sent as `failed` with `retryIn`, the rest of the object's backoff window (absent when broken), the current one at once when the object is failed already; the next attempt is announced with `loading`; the hold stays, so a `retry` (from any holder) renews even a broken object @@ -146,7 +146,7 @@ within each link's limits, compression at the origin, activation chains and revo until `retry`; the message's other entries apply. Its call arguments go up a parent link uncompressed (the parent may have accepted it as a child). It keeps its binding's caller: two references with different principals are two remote objects. Its holds carry the params as a - part and the address its placement computed. A `bound` (a first binding, a re-hold; also on a live hold) + part and the address its `hostAddress` computed. A `bound` (a first binding, a re-hold; also on a live hold) replaces the replicas in the message's batch: members it had get the snapshot as one frame `[["r", value]]` and a fresh decoder, members outside a new variant vanish (reads throw `invalid/member`, watches get one `failed` call and stay open, derives over them fail; a later binding with the member brings it back as `[["r", value]]`, as on the diff --git a/packages/chord/src/next/remote/hold.ts b/packages/chord/src/next/remote/hold.ts index 59ee856c9..b018c201b 100644 --- a/packages/chord/src/next/remote/hold.ts +++ b/packages/chord/src/next/remote/hold.ts @@ -72,7 +72,7 @@ type HoldState = /** The members it follows, in the order they were subscribed. */ readonly followed: Followed[]; } - /** Refused where it arrived (placement, `authorize`, a mismatch, invalid params, a cycle): `retry` starts it again. */ + /** Refused where it arrived (host address, `authorize`, a mismatch, invalid params, a cycle): `retry` restarts it. */ | { readonly kind: "refused"; readonly request: HoldRequest } | { readonly kind: "released" }; @@ -86,8 +86,8 @@ const BINDING: HoldState = { kind: "binding" }; /** * One hold served here: a reference to its object acting as its caller (the hold's principal). While pending, each * failure of its object is sent as `failed` (with the rest of its backoff window) and the next attempt as `loading`; a - * hold that cannot be taken (a refused type, a mismatch, invalid params or placement) is a broken `failed` that holds - * nothing until a `retry` starts it again. Once live it binds when its link's window has a slot, then sends a part per batch that changed its members + * hold that cannot be taken (a refused type, a mismatch, invalid params or a wrong host address) is a broken `failed` + * that holds nothing until a `retry` starts it again. Once live it binds when its link's window has a slot, then sends a part per batch that changed its members * (or keeps their ops pending while the window is full), and `gone` when its object ends. */ export class Hold implements Holder { @@ -139,7 +139,8 @@ export class Hold implements Holder { * Admits the hold once `load` resolved: its params within the link's part limit (`too_large/part`), then relayed * (it is replaced by a relayed hold), or served here, its params opened and parsed (a malformed part is a protocol * error), its object taken and followed (`moved`). A hold that cannot go anywhere, a type that does not match, - * invalid params, a wrong placement or a cycle (Q5) is a broken `failed`; the hold is kept, refused, for a `retry`. + * invalid params, a wrong host address or a cycle (Q5) is a broken `failed`; the hold is kept, refused, for a + * `retry`. */ async start(request: HoldRequest): Promise { const { link } = this.#provider; diff --git a/packages/chord/src/next/remote/remote-object.ts b/packages/chord/src/next/remote/remote-object.ts index 1765b7d1d..05ea9cea9 100644 --- a/packages/chord/src/next/remote/remote-object.ts +++ b/packages/chord/src/next/remote/remote-object.ts @@ -86,7 +86,7 @@ export class RemoteObject implements Target { host: AnyHost, contract: Contract, { params, key }: CheckedParams, - address: WireAddress | undefined, + address: WireAddress, caller: Caller, path: Path, dependent: ChordObject | undefined, @@ -100,7 +100,7 @@ export class RemoteObject implements Target { version, fingerprint, params: jsonPart(params), - ...(address === undefined ? {} : { address }), + address, }; this.caller = caller; this.#params = params; @@ -216,7 +216,7 @@ export class RemoteObject implements Target { /** * `ref.retry()`: a failed object's provider is asked for an attempt (a hold it refused is started there again). One * that failed for a part past the link's limits is placed anew and held with a new id, now or on its path's next - * link (a placement that throws fails it broken again). + * link (a `hostAddress` that throws fails it broken again). */ retry(): void { if (this.#state.kind !== "failed") return; @@ -227,7 +227,7 @@ export class RemoteObject implements Target { this.#hold?.side.retry(this.#hold.id); } - /** Places a reference whose part was past the limits anew, on its path now (a placement that throws fails it). */ + /** Places a reference whose part was past the limits anew, on its path now (a throwing `hostAddress` fails it). */ #place(): void { let path: Path; try { diff --git a/packages/chord/test/next/bench/compression.ts b/packages/chord/test/next/bench/compression.ts index 7b74ad8f8..15e7024ce 100644 --- a/packages/chord/test/next/bench/compression.ts +++ b/packages/chord/test/next/bench/compression.ts @@ -4,13 +4,15 @@ // Run by hand from packages/chord (not in CI): // node --conditions=source --experimental-strip-types --expose-gc test/next/bench/compression.ts import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; -import { arr, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { arr, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; import { defineState } from "../../../src/next/documents/index.ts"; import { createHost, defineFacet } from "../../../src/next/host/index.ts"; import { type Compression, type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; import { DeflatedPart, inflateAsync } from "../../../src/next/link/internal.ts"; import { nodeCompression } from "../../../src/next-node/compression.ts"; +const Server = hostKind("server"); + const context = BACKGROUND_CONTEXT; type P = { kind: "owner" } | { kind: "server" }; @@ -23,6 +25,7 @@ const Store = local("bench.store", { version: 1, params: {}, members: { doc: obj const View = remote("bench.view", { version: 1, params: {}, + hostAddress: () => Server(), variants: { owner: { state: { doc: obj({ entries: arr(Entry) }) }, methods: { export: method(undefined, str()) } } }, }); @@ -115,7 +118,12 @@ for (const [name, json] of [ /** A server compressing with `server` and a client with `client` (none: it inflates asynchronously). */ async function linked(server: Compression | undefined, client: Compression | undefined) { - const serverHost = createHost

    ({ principal: { kind: "server" }, compression: server, onError: console.error }); + const serverHost = createHost

    ({ + address: Server(), + principal: { kind: "server" }, + compression: server, + onError: console.error, + }); await serverHost.load([{ name: "@bench/app", facet }], context); const clientHost = createHost

    ({ compression: client, onError: console.error }); await clientHost.load([], context); diff --git a/packages/chord/test/next/bench/links.ts b/packages/chord/test/next/bench/links.ts index 3bbbf6d31..39e7e7c55 100644 --- a/packages/chord/test/next/bench/links.ts +++ b/packages/chord/test/next/bench/links.ts @@ -8,13 +8,15 @@ import { GCProfiler } from "node:v8"; import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; import { BACKGROUND_CONTEXT, type Context, withTelemetry } from "../../../src/context/index.ts"; -import { arr, bytes, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { arr, bytes, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; import { defineState } from "../../../src/next/documents/index.ts"; import { createHost, defineFacet, type Host, type RefState } from "../../../src/next/host/index.ts"; import { type Codec, type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; import { binaryCodec, type Message, textCodec } from "../../../src/next/link/internal.ts"; import type { NativeWatchable } from "../../../src/watch.ts"; +const Server = hostKind("server"); + type P = { kind: "owner" } | { kind: "server" }; const Entry = obj({ id: int(), text: str(), author: str(), at: int() }); @@ -37,6 +39,7 @@ const Store = local("bench.store", { const View = remote("bench.view", { version: 1, params: { name: str() }, + hostAddress: () => Server(), variants: { owner: { state: { doc: Shape } } }, }); @@ -44,6 +47,7 @@ const View = remote("bench.view", { const Files = remote("bench.files", { version: 1, params: {}, + hostAddress: () => Server(), variants: { owner: { methods: { ping: method(obj({ n: int() }), int()), put: method(obj({ data: bytes() }), int()) } }, }, @@ -138,6 +142,7 @@ const document = (): Entry[] => Array.from({ length: 500 }, (_, id) => entry(id, async function setup(name: string, entries: Entry[], codec: Codec, metered = true) { const context = metered ? counting : BACKGROUND_CONTEXT; const server = createHost

    ({ + address: Server(), principal: { kind: "server" }, telemetry: metered ? telemetry : undefined, onError: console.error, diff --git a/packages/chord/test/next/bench/relay.ts b/packages/chord/test/next/bench/relay.ts index 4f2c382b0..c699e9282 100644 --- a/packages/chord/test/next/bench/relay.ts +++ b/packages/chord/test/next/bench/relay.ts @@ -39,7 +39,7 @@ const Store = local("bench.store", { const View = remote("bench.view", { version: 1, params: { session: str() }, - placement: ({ session }) => Session({ id: session }), + hostAddress: ({ session }) => Session({ id: session }), variants: { owner: { state: { doc: Shape }, methods: { stream: method(str()), ping: method(int(), int()) } } }, }); diff --git a/packages/chord/test/next/contracts/accepts.test.ts b/packages/chord/test/next/contracts/accepts.test.ts index 83cbfcca9..7091d9d7e 100644 --- a/packages/chord/test/next/contracts/accepts.test.ts +++ b/packages/chord/test/next/contracts/accepts.test.ts @@ -3,6 +3,7 @@ import { type AcceptsTable, arr, evolve, + hostKind, int, local, method, @@ -13,12 +14,15 @@ import { } from "../../../src/next/contracts/index.ts"; import { checkAccepts } from "../../../src/next/contracts/internal.ts"; +const Server = hostKind("server"); + const Note = obj({ text: str() }); const NoteV2 = obj({ text: str(), tag: str() }); const NotesV1 = remote("notes", { version: 1, params: { session: str() }, + hostAddress: () => Server(), variants: { editor: { state: { notes: arr(Note), count: int() }, methods: { add: method(Note, str()), clear: method() } }, reader: { state: { notes: arr(Note) }, methods: { add: method(Note, str()) } }, diff --git a/packages/chord/test/next/contracts/contract.test.ts b/packages/chord/test/next/contracts/contract.test.ts index c9a6f12ab..e7c409022 100644 --- a/packages/chord/test/next/contracts/contract.test.ts +++ b/packages/chord/test/next/contracts/contract.test.ts @@ -19,11 +19,12 @@ import { import { checkAccepts, currentVariants, LOCAL, takesNoArgs } from "../../../src/next/contracts/internal.ts"; import { bindProvider, checkParams } from "../../../src/next/data/internal.ts"; import type { Watchable } from "../../../src/next/state/index.ts"; -import { expectCollected } from "../support.ts"; +import { expectCollected, Server } from "../support.ts"; const Notes = remote("notes", { version: 1, params: { session: str(), page: int() }, + hostAddress: () => Server(), variants: { editor: { state: { notes: arr(str()) }, methods: { add: method(obj({ text: str() }), str()), clear: method() } }, reader: { state: { notes: arr(str()) } }, diff --git a/packages/chord/test/next/contracts/declare.test.ts b/packages/chord/test/next/contracts/declare.test.ts index 016fb2f90..25e4e0811 100644 --- a/packages/chord/test/next/contracts/declare.test.ts +++ b/packages/chord/test/next/contracts/declare.test.ts @@ -1,10 +1,25 @@ import Type from "typebox"; import { describe, expect, it } from "vitest"; -import { arr, evolve, int, keyed, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { + arr, + evolve, + hostKind, + int, + keyed, + local, + method, + obj, + remote, + str, +} from "../../../src/next/contracts/index.ts"; import { currentVariants, LOCAL } from "../../../src/next/contracts/internal.ts"; const Note = obj({ text: str() }); +const Server = hostKind("server"); + +const hostAddress = () => Server(); + /** Declares through an untyped door, as a JavaScript caller would. */ const untyped = { remote: remote as (name: unknown, declaration: unknown) => unknown, @@ -16,6 +31,7 @@ describe("declaration checks", () => { const Notes = remote("notes", { version: 1, params: { session: str() }, + hostAddress, variants: { editor: { state: { notes: arr(Note) }, methods: { add: method(Note, str()), clear: method() } }, reader: { state: { notes: arr(Note) } }, @@ -31,77 +47,109 @@ describe("declaration checks", () => { expect(Info).toMatchObject({ kind: "local", name: "app.session-info", version: 3 }); }); + // Each case declares a valid `hostAddress` unless it checks that one, so the check it names is the one reached. it.each([ - ["empty name", "", { version: 1, params: {}, variants: { a: {} } }, 'A contract name "": expected lowercase'], + [ + "empty name", + "", + { version: 1, params: {}, hostAddress, variants: { a: {} } }, + 'A contract name "": expected lowercase', + ], [ "name not a string", 1, - { version: 1, params: {}, variants: { a: {} } }, + { version: 1, params: {}, hostAddress, variants: { a: {} } }, "A contract name 1: expected lowercase", ], - ["an uppercase name", "Notes", { version: 1, params: {}, variants: { a: {} } }, "expected lowercase"], - ["a name with a colon", "a:b", { version: 1, params: {}, variants: { a: {} } }, "expected lowercase"], - ["version zero", "n", { version: 0, params: {}, variants: { a: {} } }, "n.version: expected a positive integer"], + [ + "an uppercase name", + "Notes", + { version: 1, params: {}, hostAddress, variants: { a: {} } }, + "expected lowercase", + ], + [ + "a name with a colon", + "a:b", + { version: 1, params: {}, hostAddress, variants: { a: {} } }, + "expected lowercase", + ], + [ + "version zero", + "n", + { version: 0, params: {}, hostAddress, variants: { a: {} } }, + "n.version: expected a positive integer", + ], [ "fractional version", "n", - { version: 1.5, params: {}, variants: { a: {} } }, + { version: 1.5, params: {}, hostAddress, variants: { a: {} } }, "n.version: expected a positive integer", ], [ "unsafe version", "n", - { version: 2 ** 53, params: {}, variants: { a: {} } }, + { version: 2 ** 53, params: {}, hostAddress, variants: { a: {} } }, "n.version: expected a positive integer", ], [ "unknown declaration key", "n", - { version: 1, params: {}, variants: { a: {} }, place: "x" }, + { version: 1, params: {}, hostAddress, variants: { a: {} }, place: "x" }, 'n: unknown key "place"', ], [ - "a placement that is no function", + "a host address that is no function", "n", - { version: 1, params: {}, variants: { a: {} }, placement: "x" }, - "n.placement: expected a function of the params", + { version: 1, params: {}, variants: { a: {} }, hostAddress: "x" }, + "n.hostAddress: expected a function of the params", ], - ["no variants", "n", { version: 1, params: {}, variants: {} }, "n.variants: at least one variant is required"], - ["missing params", "n", { version: 1, variants: { a: {} } }, "n.params: expected an object"], + [ + "no host address", + "n", + { version: 1, params: {}, variants: { a: {} } }, + "n.hostAddress: expected a function of the params", + ], + [ + "no variants", + "n", + { version: 1, params: {}, hostAddress, variants: {} }, + "n.variants: at least one variant is required", + ], + ["missing params", "n", { version: 1, hostAddress, variants: { a: {} } }, "n.params: expected an object"], [ "unknown variant key", "n", - { version: 1, params: {}, variants: { a: { members: {} } } }, + { version: 1, params: {}, hostAddress, variants: { a: { members: {} } } }, 'n.a: unknown key "members"', ], [ "a schema under methods", "n", - { version: 1, params: {}, variants: { a: { methods: { m: str() } } } }, + { version: 1, params: {}, hostAddress, variants: { a: { methods: { m: str() } } } }, "n.a.methods.m: expected method(...)", ], [ "a method under state", "n", - { version: 1, params: {}, variants: { a: { state: { s: method() } } } }, + { version: 1, params: {}, hostAddress, variants: { a: { state: { s: method() } } } }, "n.a.state.s: a method belongs in methods", ], [ "bad state schema", "n", - { version: 1, params: {}, variants: { a: { state: { s: { not: {} } } } } }, + { version: 1, params: {}, hostAddress, variants: { a: { state: { s: { not: {} } } } } }, 'n.a.state.s: unsupported keyword "not"', ], [ "open arguments", "n", - { version: 1, params: {}, variants: { a: { methods: { m: method(Type.Object({})) } } } }, + { version: 1, params: {}, hostAddress, variants: { a: { methods: { m: method(Type.Object({})) } } } }, "n.a.methods.m.args: argument objects must be closed", ], [ "bad result", "n", - { version: 1, params: {}, variants: { a: { methods: { m: method(undefined, { not: {} }) } } } }, + { version: 1, params: {}, hostAddress, variants: { a: { methods: { m: method(undefined, { not: {} }) } } } }, 'n.a.methods.m.result: unsupported keyword "not"', ], ])("remote refuses %s", (_case, name, declaration, message) => { @@ -110,7 +158,7 @@ describe("declaration checks", () => { it.each([ ["unknown key", { version: 1, params: {}, members: {}, variants: {} }, 'i: unknown key "variants"'], - ["a placement", { version: 1, params: {}, members: {}, placement: () => null }, 'i: unknown key "placement"'], + ["a host address", { version: 1, params: {}, members: {}, hostAddress }, 'i: unknown key "hostAddress"'], [ "bad member schema", { version: 1, params: {}, members: { s: { not: {} } } }, @@ -125,6 +173,13 @@ describe("declaration checks", () => { expect(() => untyped.local("i", declaration)).toThrow(message); }); + it("refuse a remote declaration without a host address at the type level", () => { + expect(() => + // @ts-expect-error: every remote type declares its host address + remote("n", { version: 1, params: {}, variants: { a: {} } }), + ).toThrow("n.hostAddress: expected a function of the params"); + }); + it("does not check state for closed objects", () => { expect(() => local("i", { version: 1, params: {}, members: { s: Type.Object({ a: str() }) } })).not.toThrow(); }); @@ -136,6 +191,7 @@ describe("D11: members across variants", () => { remote("n", { version: 1, params: {}, + hostAddress, // @ts-expect-error: `x` is a method in variant b variants: { a: { state: { x: str() } }, b: { methods: { x: method() } } }, }), @@ -147,6 +203,7 @@ describe("D11: members across variants", () => { remote("n", { version: 1, params: {}, + hostAddress, // @ts-expect-error: `x` is both state and a method variants: { a: { state: { x: str() }, methods: { x: method() } } }, }), @@ -158,6 +215,7 @@ describe("D11: members across variants", () => { remote("n", { version: 1, params: {}, + hostAddress, // @ts-expect-error: `m` takes a string in a and an integer in b variants: { a: { methods: { m: method(str()) } }, b: { methods: { m: method(int()) } } }, }), @@ -166,6 +224,7 @@ describe("D11: members across variants", () => { remote("n", { version: 1, params: {}, + hostAddress, variants: { a: { methods: { m: method(str(), str()) } }, b: { methods: { m: method(str({ max: 3 }), str()) } }, @@ -179,6 +238,7 @@ describe("D11: members across variants", () => { remote("n", { version: 1, params: {}, + hostAddress, variants: { editor: { state: { notes: arr(obj({ text: str(), author: str() })) }, methods: { m: method(str()) } }, reader: { state: { notes: arr(obj({ text: str() })) }, methods: { m: method(str()) } }, @@ -190,9 +250,9 @@ describe("D11: members across variants", () => { describe("reserved member names", () => { it.each(["then", "release", "variant", "state", "retry"])("refuses %s in remote, local and evolve", (name) => { - expect(() => remote("n", { version: 1, params: {}, variants: { v: { state: { [name]: str() } } } })).toThrow( - `n.v.state.${name}: "${name}" is reserved: a reference has it`, - ); + expect(() => + remote("n", { version: 1, params: {}, hostAddress, variants: { v: { state: { [name]: str() } } } }), + ).toThrow(`n.v.state.${name}: "${name}" is reserved: a reference has it`); expect(() => local("n", { version: 1, params: {}, members: { [name]: method() } })).toThrow( `n.members.${name}: "${name}" is reserved: a reference has it`, ); @@ -220,6 +280,7 @@ describe("keyed members (slice 6a §12)", () => { remote("app.notes", { version: 1, params: {}, + hostAddress, // @ts-expect-error keyed members are only in local contracts variants: { editor: { state: { notes: keyed(Note) } } }, }), diff --git a/packages/chord/test/next/contracts/evolve.test.ts b/packages/chord/test/next/contracts/evolve.test.ts index abc2263e3..85c1d20eb 100644 --- a/packages/chord/test/next/contracts/evolve.test.ts +++ b/packages/chord/test/next/contracts/evolve.test.ts @@ -7,6 +7,7 @@ import { arr, bool, evolve, + hostKind, type Implementation, int, local, @@ -18,11 +19,14 @@ import { str, } from "../../../src/next/contracts/index.ts"; +const Server = hostKind("server"); + const Note = obj({ text: str() }); const NotesV1 = remote("notes", { version: 1, params: { session: str() }, + hostAddress: () => Server(), variants: { editor: { state: { notes: arr(Note), count: int() }, methods: { add: method(Note, str()), clear: method() } }, reader: { state: { notes: arr(Note) }, methods: { add: method(Note, str()) } }, @@ -164,7 +168,12 @@ describe("kinds and fixed parts", () => { }); it("a new base is a full declaration with an explicit version and no older versions", () => { - const base = remote("notes", { version: 5, params: { session: str(), tenant: str() }, variants: { editor: {} } }); + const base = remote("notes", { + version: 5, + params: { session: str(), tenant: str() }, + hostAddress: () => Server(), + variants: { editor: {} }, + }); expect(contractData(base).versions.map(({ version }) => version)).toEqual([5]); expect(evolve(base, (e) => e).version).toBe(6); }); diff --git a/packages/chord/test/next/contracts/fingerprint.test.ts b/packages/chord/test/next/contracts/fingerprint.test.ts index d28e360fb..f159b7856 100644 --- a/packages/chord/test/next/contracts/fingerprint.test.ts +++ b/packages/chord/test/next/contracts/fingerprint.test.ts @@ -1,12 +1,15 @@ import Type, { type TSchema } from "typebox"; import { describe, expect, it } from "vitest"; -import { evolve, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { evolve, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; import { NORMAL_FORM_VERSION } from "../../../src/next/contracts/internal.ts"; import type { Params } from "../../../src/next/contracts/params.ts"; +const Server = hostKind("server"); + /** Fingerprint of a one-variant remote contract with state `s` and params `params`. */ function fingerprintOf(state: TSchema, params: Params = {}): string { - return remote("n", { version: 1, params, variants: { a: { state: { s: state } } } }).fingerprint; + return remote("n", { version: 1, params, hostAddress: () => Server(), variants: { a: { state: { s: state } } } }) + .fingerprint; } describe("fingerprints", () => { @@ -15,6 +18,7 @@ describe("fingerprints", () => { const Notes = remote("notes", { version: 1, params: { session: str() }, + hostAddress: () => Server(), variants: { editor: { state: { notes: obj({ text: str() }) }, methods: { add: method(obj({ text: str() }), str()) } }, reader: { state: { notes: obj({ text: str() }) } }, @@ -58,7 +62,8 @@ describe("fingerprints", () => { }); it("give the same v1 from a separate declaration, and evolve keeps v1's", () => { - const declare = () => remote("n", { version: 1, params: {}, variants: { a: { state: { s: str() } } } }); + const declare = () => + remote("n", { version: 1, params: {}, hostAddress: () => Server(), variants: { a: { state: { s: str() } } } }); const v1 = declare(); const v2 = evolve(v1, (e) => e.state("a", "s", int())); expect(declare().fingerprint).toBe(v1.fingerprint); @@ -67,14 +72,19 @@ describe("fingerprints", () => { }); it("differ between local and remote", () => { - const asRemote = remote("n", { version: 1, params: {}, variants: { members: { state: { s: str() } } } }); + const asRemote = remote("n", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { members: { state: { s: str() } } }, + }); const asLocal = local("n", { version: 1, params: {}, members: { s: str() } }); expect(asLocal.fingerprint).not.toBe(asRemote.fingerprint); }); it("do not depend on the name or version", () => { - const a = remote("a", { version: 1, params: {}, variants: { v: {} } }); - const b = remote("b", { version: 7, params: {}, variants: { v: {} } }); + const a = remote("a", { version: 1, params: {}, hostAddress: () => Server(), variants: { v: {} } }); + const b = remote("b", { version: 7, params: {}, hostAddress: () => Server(), variants: { v: {} } }); expect(a.fingerprint).toBe(b.fingerprint); }); }); diff --git a/packages/chord/test/next/contracts/host-kind.test.ts b/packages/chord/test/next/contracts/host-kind.test.ts index 3414ffd97..93026d54d 100644 --- a/packages/chord/test/next/contracts/host-kind.test.ts +++ b/packages/chord/test/next/contracts/host-kind.test.ts @@ -30,7 +30,7 @@ describe("host kinds", () => { expect(hostKind("@pi/session.v2_~-").kind).toBe("@pi/session.v2_~-"); }); - it("check an address's params where it is routed, with the kind's compiled check: invalid/placement", () => { + it("check an address's params where it is routed, with the kind's compiled check: invalid/host_address", () => { const params = kindParams(Session); expect(checkAddress("session", params, { id: "S1" }, context).key).toBe("session|S1"); for (const bad of [{ id: 1 }, {}, { id: "S1", extra: 1 }, null]) { @@ -40,7 +40,7 @@ describe("host kinds", () => { } catch (error) { thrown = error; } - expect(isChordError(thrown, "invalid") && thrown.reason).toBe("placement"); + expect(isChordError(thrown, "invalid") && thrown.reason).toBe("host_address"); } }); @@ -54,18 +54,22 @@ describe("host kinds", () => { }); }); -describe("placement", () => { +describe("hostAddress", () => { const variants = { v: { state: { n: int() } } }; it("is not part of the fingerprint and is carried by evolve", () => { - const placement = ({ session }: { session: string }) => Session({ id: session }); - const placed = remote("app.placed", { version: 1, params: { session: str() }, placement, variants }); - const unplaced = remote("app.placed", { version: 1, params: { session: str() }, variants }); - expect(placed.fingerprint).toBe(unplaced.fingerprint); - expect(contractData(placed).placement).toBe(placement); - expect(contractData(unplaced).placement).toBeUndefined(); - const next = evolve(placed, (v) => v.addState("v", "m", int())); - expect(contractData(next).placement).toBe(placement); + const hostAddress = ({ session }: { session: string }) => Session({ id: session }); + const onSession = remote("app.placed", { version: 1, params: { session: str() }, hostAddress, variants }); + const onServer = remote("app.placed", { + version: 1, + params: { session: str() }, + hostAddress: () => Server(), + variants, + }); + expect(onSession.fingerprint).toBe(onServer.fingerprint); + expect(contractData(onSession).hostAddress).toBe(hostAddress); + const next = evolve(onSession, (v) => v.addState("v", "m", int())); + expect(contractData(next).hostAddress).toBe(hostAddress); }); it("is typed by the contract's params", () => { @@ -73,7 +77,7 @@ describe("placement", () => { version: 1, params: { session: str() }, // @ts-expect-error: the params have no `id` - placement: ({ id }) => Session({ id }), + hostAddress: ({ id }) => Session({ id }), variants, }); }); diff --git a/packages/chord/test/next/contracts/params.test.ts b/packages/chord/test/next/contracts/params.test.ts index 159ff72ee..96187b2b0 100644 --- a/packages/chord/test/next/contracts/params.test.ts +++ b/packages/chord/test/next/contracts/params.test.ts @@ -1,6 +1,20 @@ import Type from "typebox"; import { describe, expect, it } from "vitest"; -import { arr, bool, bytes, int, num, obj, oneOf, opt, remote, str } from "../../../src/next/contracts/index.ts"; +import { + arr, + bool, + bytes, + hostKind, + int, + num, + obj, + oneOf, + opt, + remote, + str, +} from "../../../src/next/contracts/index.ts"; + +const Server = hostKind("server"); const variants = { a: { state: { s: str() } } }; @@ -15,16 +29,17 @@ describe("params policy", () => { c: bool(), d: oneOf("chat", "agent"), }, + hostAddress: () => Server(), variants, }), ).not.toThrow(); }); it("refuses optional fields, in the types and at declaration", () => { - // @ts-expect-error: params fields are required - expect(() => remote("n", { version: 1, params: { a: opt(str()) }, variants })).toThrow( - "n.params.a: a param is a required str(), int(), bool() or oneOf(...) of strings", - ); + expect(() => + // @ts-expect-error: params fields are required + remote("n", { version: 1, params: { a: opt(str()) }, hostAddress: () => Server(), variants }), + ).toThrow("n.params.a: a param is a required str(), int(), bool() or oneOf(...) of strings"); }); it.each([ @@ -36,6 +51,8 @@ describe("params policy", () => { ["defaults", Type.String({ default: "x" })], ])("refuses %s, in the types and at declaration", (_case, schema) => { // @ts-expect-error: not a params field schema - expect(() => remote("n", { version: 1, params: { p: schema }, variants })).toThrow("n.params.p: a param is"); + expect(() => remote("n", { version: 1, params: { p: schema }, hostAddress: () => Server(), variants })).toThrow( + "n.params.p: a param is", + ); }); }); diff --git a/packages/chord/test/next/data/args.test.ts b/packages/chord/test/next/data/args.test.ts index 90dc8cae6..896861fc0 100644 --- a/packages/chord/test/next/data/args.test.ts +++ b/packages/chord/test/next/data/args.test.ts @@ -15,7 +15,7 @@ import { } from "../../../src/next/contracts/index.ts"; import { isChordError } from "../../../src/next/data/index.ts"; import { bindProvider, type CheckedArgs, checkArgs, type Value } from "../../../src/next/data/internal.ts"; -import { counted, expectCollected } from "../support.ts"; +import { counted, expectCollected, Server } from "../support.ts"; const Note = obj({ text: str() }); const NoteV2 = obj({ text: str(), tag: str() }); @@ -23,6 +23,7 @@ const NoteV2 = obj({ text: str(), tag: str() }); const NotesV1 = remote("notes", { version: 1, params: { session: str() }, + hostAddress: () => Server(), variants: { editor: { state: { notes: arr(Note) }, diff --git a/packages/chord/test/next/data/errors.test.ts b/packages/chord/test/next/data/errors.test.ts index c5d955a99..bebc4e8e4 100644 --- a/packages/chord/test/next/data/errors.test.ts +++ b/packages/chord/test/next/data/errors.test.ts @@ -78,7 +78,7 @@ describe("protocol table", () => { const pinned = { version: 1, reasons: { - invalid: ["args", "params", "member", "placement", "reentrant", "read_after_bulk", "cycle"], + invalid: ["args", "params", "member", "host_address", "reentrant", "read_after_bulk", "cycle"], contract_mismatch: ["updating"], forbidden: ["revoked"], gone: [], diff --git a/packages/chord/test/next/data/failure.test.ts b/packages/chord/test/next/data/failure.test.ts index a5ddbacf2..b833f4d8a 100644 --- a/packages/chord/test/next/data/failure.test.ts +++ b/packages/chord/test/next/data/failure.test.ts @@ -20,7 +20,7 @@ describe("failure causes", () => { }); it("the kind comes from the stage, never from the error code", () => { - const chord = invalid("placement", "wrong host"); + const chord = invalid("host_address", "wrong host"); expect(openFailure(chord, "open")).toEqual({ kind: "transient", error: chord }); expect(openFailure(broken(chord), "open")).toEqual({ kind: "broken", error: chord }); }); diff --git a/packages/chord/test/next/data/params.test.ts b/packages/chord/test/next/data/params.test.ts index 55b9a8ebc..5848b0378 100644 --- a/packages/chord/test/next/data/params.test.ts +++ b/packages/chord/test/next/data/params.test.ts @@ -5,11 +5,12 @@ import { bool, int, local, oneOf, remote, str } from "../../../src/next/contract import { isChordError } from "../../../src/next/data/index.ts"; import { checkParams, identityKey } from "../../../src/next/data/internal.ts"; import { validatorFor } from "../../../src/next/data/validator.ts"; -import { counted, expectCollected } from "../support.ts"; +import { counted, expectCollected, Server } from "../support.ts"; const Session = remote("session", { version: 1, params: { id: str({ min: 1 }), mode: oneOf("chat", "agent"), page: int({ min: 0 }), open: bool() }, + hostAddress: () => Server(), variants: { owner: {} }, }); diff --git a/packages/chord/test/next/examples/08-remote.ts b/packages/chord/test/next/examples/08-remote.ts index c88d81e15..fb538aecb 100644 --- a/packages/chord/test/next/examples/08-remote.ts +++ b/packages/chord/test/next/examples/08-remote.ts @@ -3,7 +3,7 @@ // Run from packages/chord: // node --conditions=source --experimental-strip-types test/next/examples/08-remote.ts import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; -import { arr, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { arr, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; import { defineState } from "../../../src/next/documents/index.ts"; import { createHost, defineFacet } from "../../../src/next/host/index.ts"; import { memoryLinkPair } from "../../../src/next/link/index.ts"; @@ -17,10 +17,14 @@ type P = { kind: "owner" } | { kind: "guest"; session: string } | { kind: "serve const Note = obj({ text: str(), author: str() }); const PublicNote = obj({ text: str() }); +// The server's host kind: every remote contract names the address of the host serving its objects. +const Server = hostKind("server"); + // A remote contract: one object per (type, params, principal key), in the variant its factory chooses. const Notes = remote("example.notes", { version: 1, params: { session: str() }, + hostAddress: () => Server(), variants: { editor: { state: { notes: arr(Note), count: int() }, methods: { add: method(obj({ text: str() })) } }, reader: { state: { notes: arr(PublicNote), count: int() } }, @@ -68,8 +72,12 @@ const facet = defineFacet

    ((host) => { }); }); -// 1. The server: it acts as `server` for its own requests. -const server = createHost

    ({ principal: { kind: "server" }, onError: (error) => console.error("server:", error) }); +// 1. The server, at its address: it acts as `server` for its own requests. +const server = createHost

    ({ + address: Server(), + principal: { kind: "server" }, + onError: (error) => console.error("server:", error), +}); await server.load([{ name: "@example/notes", facet }], context); // 2. A client (a tab): no principal of its own. The entry point authenticated it as a guest of S1 and stamps that on diff --git a/packages/chord/test/next/examples/09-calls.ts b/packages/chord/test/next/examples/09-calls.ts index 01dfbbe14..5c099fd5d 100644 --- a/packages/chord/test/next/examples/09-calls.ts +++ b/packages/chord/test/next/examples/09-calls.ts @@ -3,7 +3,7 @@ // Run from packages/chord: // node --conditions=source --experimental-strip-types test/next/examples/09-calls.ts import { BACKGROUND_CONTEXT, withCancel } from "../../../src/context/index.ts"; -import { bytes, int, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { bytes, hostKind, int, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; import { isChordError, PluginError } from "../../../src/next/data/index.ts"; import { createHost, defineFacet } from "../../../src/next/host/index.ts"; import { memoryLinkPair } from "../../../src/next/link/index.ts"; @@ -12,9 +12,13 @@ const context = BACKGROUND_CONTEXT; type P = { kind: "owner" } | { kind: "server" }; +// The server's host kind: a remote contract's `hostAddress` names the host serving its objects. +const Server = hostKind("server"); + const Files = remote("example.files", { version: 1, params: {}, + hostAddress: () => Server(), variants: { owner: { methods: { @@ -58,7 +62,11 @@ const facet = defineFacet

    ((host) => { })); }); -const server = createHost

    ({ principal: { kind: "server" }, onError: (error) => console.error("server:", error) }); +const server = createHost

    ({ + address: Server(), + principal: { kind: "server" }, + onError: (error) => console.error("server:", error), +}); await server.load([{ name: "@example/files", facet }], context); const [up, down] = memoryLinkPair(); server.accept(up, { principal: { kind: "owner" } }, context); diff --git a/packages/chord/test/next/examples/10-drops.ts b/packages/chord/test/next/examples/10-drops.ts index 701961c45..f0ca424f7 100644 --- a/packages/chord/test/next/examples/10-drops.ts +++ b/packages/chord/test/next/examples/10-drops.ts @@ -3,7 +3,7 @@ // Run from packages/chord: // node --conditions=source --experimental-strip-types test/next/examples/10-drops.ts import { BACKGROUND_CONTEXT, type Context } from "../../../src/context/index.ts"; -import { arr, int, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { arr, hostKind, int, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; import { defineState } from "../../../src/next/documents/index.ts"; import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; import { type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; @@ -12,9 +12,13 @@ const context = BACKGROUND_CONTEXT; type P = { kind: "owner" } | { kind: "server" }; +// The server's host kind: a remote contract's `hostAddress` names the host serving its objects. +const Server = hostKind("server"); + const Notes = remote("example.notes", { version: 1, params: { session: str() }, + hostAddress: () => Server(), variants: { editor: { state: { notes: arr(str()) }, @@ -47,7 +51,11 @@ const facet = defineFacet

    ((host) => { ); }); -const server = createHost

    ({ principal: { kind: "server" }, onError: (error) => console.error("server:", error) }); +const server = createHost

    ({ + address: Server(), + principal: { kind: "server" }, + onError: (error) => console.error("server:", error), +}); await server.load([{ name: "@example/notes", facet }], context); const client = createHost

    ({ onError: (error) => console.error("client:", error) }); await client.load([], context); diff --git a/packages/chord/test/next/examples/12-routes.ts b/packages/chord/test/next/examples/12-routes.ts index 7074bd979..7f6516887 100644 --- a/packages/chord/test/next/examples/12-routes.ts +++ b/packages/chord/test/next/examples/12-routes.ts @@ -1,4 +1,4 @@ -// Routes: host kinds and addresses, a contract's placement, a server routing a session's objects to an in-process +// Routes: host kinds and addresses, a contract's host address, a server routing a session's objects to an in-process // worker (one `resolve` per address, `authorize` per principal), a guest's `ctx.get` on the server reaching the worker // as the guest, and a worker that drops and is resolved again. // Run from packages/chord: @@ -26,7 +26,7 @@ const Session = hostKind("session", { id: str() }); const Notes = remote("example.notes", { version: 1, params: { session: str() }, - placement: ({ session }) => Session({ id: session }), + hostAddress: ({ session }) => Session({ id: session }), variants: { editor: { state: { notes: arr(str()) }, @@ -36,10 +36,11 @@ const Notes = remote("example.notes", { }, }); -// The lobby lives on the server (no placement): its handler gets notes for whoever calls it. +// The lobby lives on the server: its handler gets notes for whoever calls it. const Lobby = remote("example.lobby", { version: 1, params: {}, + hostAddress: () => Server(), variants: { visitor: { methods: { peek: method(obj({ session: str() }), str()) } } }, }); diff --git a/packages/chord/test/next/examples/13-relay.ts b/packages/chord/test/next/examples/13-relay.ts index 6203d3f52..f132e76f8 100644 --- a/packages/chord/test/next/examples/13-relay.ts +++ b/packages/chord/test/next/examples/13-relay.ts @@ -30,7 +30,7 @@ const SessionInfo = local("example.session-info", { version: 1, params: {}, memb const Notes = remote("example.notes", { version: 1, params: { session: str() }, - placement: ({ session }) => Session({ id: session }), + hostAddress: ({ session }) => Session({ id: session }), variants: { owner: { state: { info: Init, notes: arr(str()) }, methods: { add: method(obj({ text: str() }), int()) } }, }, diff --git a/packages/chord/test/next/examples/14-flow-control.ts b/packages/chord/test/next/examples/14-flow-control.ts index 4eba6e1ee..184133150 100644 --- a/packages/chord/test/next/examples/14-flow-control.ts +++ b/packages/chord/test/next/examples/14-flow-control.ts @@ -28,7 +28,7 @@ const Reply = local("example.reply", { const View = remote("example.view", { version: 1, params: { session: str() }, - placement: ({ session }) => Session({ id: session }), + hostAddress: ({ session }) => Session({ id: session }), variants: { owner: { state: { reply: obj({ text: str() }) } } }, }); diff --git a/packages/chord/test/next/examples/15-revocation.ts b/packages/chord/test/next/examples/15-revocation.ts index 1a004ffa6..9ee2a3648 100644 --- a/packages/chord/test/next/examples/15-revocation.ts +++ b/packages/chord/test/next/examples/15-revocation.ts @@ -33,7 +33,7 @@ const Session = hostKind("session", { id: str() }); const Notes = remote("example.notes", { version: 1, params: { session: str() }, - placement: ({ session }) => Session({ id: session }), + hostAddress: ({ session }) => Session({ id: session }), variants: { all: { state: { notes: arr(str()) }, methods: { add: method(obj({ text: str() })), follow: method() } }, }, diff --git a/packages/chord/test/next/examples/16-presence-and-activity.ts b/packages/chord/test/next/examples/16-presence-and-activity.ts index 9091557e0..ba69803cf 100644 --- a/packages/chord/test/next/examples/16-presence-and-activity.ts +++ b/packages/chord/test/next/examples/16-presence-and-activity.ts @@ -6,7 +6,7 @@ // Run from packages/chord: // node --conditions=source --experimental-strip-types test/next/examples/16-presence-and-activity.ts import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; -import { int, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { hostKind, int, method, remote, str } from "../../../src/next/contracts/index.ts"; import { createHost, defineFacet } from "../../../src/next/host/index.ts"; import { memoryLinkPair } from "../../../src/next/link/index.ts"; @@ -14,15 +14,19 @@ const context = BACKGROUND_CONTEXT; type P = { kind: "user"; name: string } | { kind: "server" }; +// The server's host kind: a remote contract's `hostAddress` names the host serving its objects. +const Server = hostKind("server"); + /** A shared document view: who else is looking (`viewers`), and a slow method. */ const Doc = remote("example.doc", { version: 1, params: { id: str() }, + hostAddress: () => Server(), variants: { v: { state: { viewers: int() }, methods: { think: method(undefined, int()) } } }, }); // 1. The server serves `Doc`; its factory derives the viewer count from `host.references(Doc)` (on the facet's host). -const server = createHost

    ({ principal: { kind: "server" }, onError: console.error }); +const server = createHost

    ({ address: Server(), principal: { kind: "server" }, onError: console.error }); const facet = defineFacet

    ((host) => { host.serve( Doc, diff --git a/packages/chord/test/next/examples/17-limits-and-compression.ts b/packages/chord/test/next/examples/17-limits-and-compression.ts index 970dce626..da3dbe0a9 100644 --- a/packages/chord/test/next/examples/17-limits-and-compression.ts +++ b/packages/chord/test/next/examples/17-limits-and-compression.ts @@ -9,7 +9,7 @@ // Run from packages/chord: // node --conditions=source --experimental-strip-types test/next/examples/17-limits-and-compression.ts import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; -import { arr, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { arr, hostKind, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; import { isChordError } from "../../../src/next/data/index.ts"; import { defineState } from "../../../src/next/documents/index.ts"; import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; @@ -29,10 +29,14 @@ const Store = local("example.store", { members: { entries: arr(Entry), append: method(Entry) }, }); +// The server's host kind: a remote contract's `hostAddress` names the host serving its objects. +const Server = hostKind("server"); + /** What a tab holds: the conversation's entries, and appending. */ const Conversation = remote("example.conversation", { version: 1, params: { id: str() }, + hostAddress: () => Server(), variants: { v: { state: { entries: arr(Entry) }, methods: { append: method(Entry) } } }, }); @@ -78,7 +82,12 @@ function counted(link: Link, bytes: { received: number }): Link { * `limits`; the tab's own link to the server has `tabLimits`. */ async function setUp(compression: Compression | undefined, limits?: PartLimits, tabLimits?: PartLimits) { - const server = createHost

    ({ principal: { kind: "server" }, compression, onError: console.error }); + const server = createHost

    ({ + address: Server(), + principal: { kind: "server" }, + compression, + onError: console.error, + }); await server.load([{ name: "@example/conversation", facet }], context); const tab = createHost

    ({ onError: console.error }); await tab.load([], context); @@ -168,6 +177,7 @@ for (const compression of [undefined, nodeCompression]) { // child, would refuse a compressed part. Its 100 KB argument arrives as JSON, within the server's default limits. { const server = createHost

    ({ + address: Server(), principal: { kind: "server" }, compression: nodeCompression, onError: console.error, diff --git a/packages/chord/test/next/examples/19-jobs.ts b/packages/chord/test/next/examples/19-jobs.ts index 06964dc63..94c9646ca 100644 --- a/packages/chord/test/next/examples/19-jobs.ts +++ b/packages/chord/test/next/examples/19-jobs.ts @@ -6,7 +6,7 @@ // Run from packages/chord: // node --conditions=source --experimental-strip-types test/next/examples/19-jobs.ts import { BACKGROUND_CONTEXT, type Context } from "../../../src/context/index.ts"; -import { arr, int, local, method, obj, remote, type Static, str } from "../../../src/next/contracts/index.ts"; +import { arr, hostKind, int, local, method, obj, remote, type Static, str } from "../../../src/next/contracts/index.ts"; import { defineState } from "../../../src/next/documents/index.ts"; import { createHost, defineFacet, sleep } from "../../../src/next/host/index.ts"; import { defineJob } from "../../../src/next/jobs/index.ts"; @@ -56,10 +56,14 @@ const ChatStore = local("example.chat-store", { }, }); +// The server's host kind: a remote contract's `hostAddress` names the host serving its objects. +const Server = hostKind("server"); + /** Per-principal views: the owner runs shells and sees their output, a guest sees which shells run. */ const Chat = remote("example.chat", { version: 1, params: {}, + hostAddress: () => Server(), variants: { owner: { state: { entries: arr(str()), shells: arr(Shell) }, @@ -170,7 +174,11 @@ const facet = defineFacet

    ((host) => { }); // 2. A server and two tabs, the owner's and a guest's. -const server = createHost

    ({ principal: { kind: "server" }, onError: (error) => console.error(error) }); +const server = createHost

    ({ + address: Server(), + principal: { kind: "server" }, + onError: (error) => console.error(error), +}); await server.load([{ name: "@example/chat", facet }], context); async function tab(principal: P) { const host = createHost

    ({ onError: (error) => console.error(error) }); diff --git a/packages/chord/test/next/host/activity.test.ts b/packages/chord/test/next/host/activity.test.ts index 405089d6d..cdfa929b7 100644 --- a/packages/chord/test/next/host/activity.test.ts +++ b/packages/chord/test/next/host/activity.test.ts @@ -6,9 +6,14 @@ import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; import { bool, int, local, method, remote } from "../../../src/next/contracts/index.ts"; import { createHost, defineFacet } from "../../../src/next/host/index.ts"; import { constant } from "../../../src/next/state/index.ts"; -import { fakeTimers, flushed, framesOf, linked, loaded, Notes, serveNotes, transported } from "../support.ts"; +import { fakeTimers, flushed, framesOf, linked, loaded, Notes, Server, serveNotes, transported } from "../support.ts"; -const Kept = remote("app.kept", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); +const Kept = remote("app.kept", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, +}); describe("host.activity", () => { it("counts holds arriving over links and served here, not this host's own references", async () => { diff --git a/packages/chord/test/next/host/attempt.test.ts b/packages/chord/test/next/host/attempt.test.ts index 4066748e7..d1227bc4a 100644 --- a/packages/chord/test/next/host/attempt.test.ts +++ b/packages/chord/test/next/host/attempt.test.ts @@ -21,7 +21,17 @@ import { TableChanges } from "../../../src/next/store/table-changes.ts"; import { evolveTable } from "../../../src/next/store/table-evolve.ts"; import { TableHandle } from "../../../src/next/store/table-handle.ts"; import { prepareTables } from "../../../src/next/store/table-migration.ts"; -import { failures, fakeTimers, flushed, loaded, memoryStore, metered, rejection, tableOwner } from "../support.ts"; +import { + failures, + fakeTimers, + flushed, + loaded, + memoryStore, + metered, + rejection, + Server, + tableOwner, +} from "../support.ts"; const List = defineState("list", { schema: obj({ items: arr(str()) }), version: 1, initial: () => ({ items: [] }) }); const Page = local("app.page", { version: 1, params: { id: int() }, members: { label: str(), slow: method() } }); @@ -565,6 +575,7 @@ describe("remote variants", () => { const Two = remote("app.two", { version: 1, params: {}, + hostAddress: () => Server(), variants: { full: { state: { n: int() }, methods: { bump: method() } }, view: { state: { n: int() } } }, }); const owner = { kind: "owner" } as const; @@ -572,7 +583,11 @@ describe("remote variants", () => { /** A host acting as `owner` whose factory for `Two` returns `implementation`. */ async function two(implementation: unknown) { const errors: HostError[] = []; - const host = createHost({ principal: owner, onError: (error) => errors.push(error) }); + const host = createHost({ + address: Server(), + principal: owner, + onError: (error) => errors.push(error), + }); const facet = defineFacet((h) => h.serve(Two, () => implementation as never)); await host.load([{ name: "@test/app", facet }], context); return { host, errors }; diff --git a/packages/chord/test/next/host/ctx.test.ts b/packages/chord/test/next/host/ctx.test.ts index df259c227..bf4744b76 100644 --- a/packages/chord/test/next/host/ctx.test.ts +++ b/packages/chord/test/next/host/ctx.test.ts @@ -5,7 +5,16 @@ import { withLinkedAbortSignal } from "../../../src/next/host/ctx.ts"; import type { ChordHost } from "../../../src/next/host/host.ts"; import type { Ctx, Host, Reference } from "../../../src/next/host/index.ts"; import { constant } from "../../../src/next/state/index.ts"; -import { flushed, incarnationOf, linked, loaded, rawParent, rejection, type TestPrincipal } from "../support.ts"; +import { + flushed, + incarnationOf, + linked, + loaded, + rawParent, + rejection, + Server, + type TestPrincipal, +} from "../support.ts"; const Item = local("app.item", { version: 1, params: { id: int() }, members: { id: int() } }); const Desk = local("app.desk", { @@ -111,7 +120,12 @@ describe("ctx", () => { }); it("a ctx.get still waiting when its handler settles gives up: its reference is released at once", async () => { - const Far = remote("app.far", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const Far = remote("app.far", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); const Starter = local("app.starter", { version: 1, params: {}, members: { start: method() } }); let pending: Promise | undefined; const { host } = await loaded((h) => @@ -146,6 +160,7 @@ describe("local handlers act as the host (§4, audit)", () => { const Gate = remote("app.gate", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { methods: { who: method(undefined, str()), through: method(undefined, str()) } } }, }); const who = (ctx: Ctx) => `${ctx.principal?.kind ?? "none"} ${ctx.incarnation}`; diff --git a/packages/chord/test/next/host/host.test.ts b/packages/chord/test/next/host/host.test.ts index 64a291b43..0074d240e 100644 --- a/packages/chord/test/next/host/host.test.ts +++ b/packages/chord/test/next/host/host.test.ts @@ -31,6 +31,8 @@ import { partValue, RawPeer, rejection, + SERVER_ADDRESS, + Server, Session, serveNotes, type TestPrincipal, @@ -482,13 +484,24 @@ describe("getters", () => { forEachCodec(() => { describe("principals and where a reference goes", () => { - const Who = remote("app.who", { version: 1, params: { id: int() }, variants: { v: { state: { who: str() } } } }); - const Outer = remote("app.outer", { version: 1, params: {}, variants: { v: { state: { who: str() } } } }); + const Who = remote("app.who", { + version: 1, + params: { id: int() }, + hostAddress: () => Server(), + variants: { v: { state: { who: str() } } }, + }); + const Outer = remote("app.outer", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { who: str() } } }, + }); const Inner = local("app.inner", { version: 1, params: { id: int() }, members: { who: str() } }); const Desk = local("app.desk", { version: 1, params: {}, members: { who: method(undefined, str()) } }); const Gate = remote("app.gate", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { methods: { who: method(undefined, str()) } } }, }); type P = TestPrincipal; @@ -518,6 +531,12 @@ forEachCodec(() => { await expect(loading).rejects.toThrow("app.who: a host without a principal serves only local types"); }); + it("a host without an address serves no remote type, whatever its principal", async () => { + const host = createHost

    ({ principal: { kind: "server" }, onError: () => {} }); + const loading = host.load([{ name: "@test/app", facet: defineFacet

    ((h) => serveAll(h, [])) }], context); + await expect(loading).rejects.toThrow("app.who: a host without an address serves only local types"); + }); + it("ctx.get acts as ctx.principal: a guest's call gets the guest's view; a local handler's acts as the host", async () => { const { server, client } = await linked((h) => serveAll(h, []), { child: { kind: "guest", session: "S1" } }); expect(await (await client.get(Gate, {}, context)).who(context)).toBe("guest"); @@ -533,6 +552,7 @@ forEachCodec(() => { it("identity includes the principal key: equal keys share one object", async () => { const factories: string[] = []; const host = createHost

    ({ + address: Server(), principal: { kind: "guest", session: "S1" }, principalKey: (principal) => principal.kind, onError: () => {}, @@ -549,6 +569,7 @@ forEachCodec(() => { version: Who.version, fingerprint: Who.fingerprint, params: jsonPart({ id: 1 }), + address: SERVER_ADDRESS, }); await peer.next("bound"); expect(factories).toEqual(["1 guest"]); @@ -562,6 +583,7 @@ forEachCodec(() => { version: Who.version, fingerprint: Who.fingerprint, params: jsonPart({ id: 1 }), + address: SERVER_ADDRESS, }); await owner.next("bound"); expect(factories).toEqual(["1 guest", "1 owner"]); @@ -569,7 +591,12 @@ forEachCodec(() => { it("remote objects with empty params are not activated by load; they live while referenced", async () => { const factories: string[] = []; - const Empty = remote("app.empty", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const Empty = remote("app.empty", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); const { server } = await linked((h) => h.serve(Empty, (_, principal) => { factories.push(principal.kind); @@ -661,7 +688,7 @@ describe("acquiring: hold and get", () => { const offline = { code: "unavailable", message: "app.svc(1): offline" }; it("invalid params and a local type not served here: at once", async () => { - const host = createHost({ ...noErrors, principal: { kind: "server" } }); + const host = createHost({ address: Server(), ...noErrors, principal: { kind: "server" } }); await host.load([{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context); const params = { code: "invalid", reason: "params" }; expect(() => host.hold(Notes, { session: 1 } as never, context)).toThrow("params"); @@ -814,8 +841,8 @@ forEachCodec(() => { return { host, child: new RawPeer(b), parent, errors }; } - /** Sends a hold of `PlacedNotes` (or `type`) for `session`, naming `address` when given. */ - function holdOf(peer: RawPeer, session: string, address?: string, type: string = PlacedNotes.name, hold = 0) { + /** Sends a hold of `PlacedNotes` (or `type`) for `session`, naming the session `address`. */ + function holdOf(peer: RawPeer, session: string, address: string, type: string = PlacedNotes.name, hold = 0) { peer.send({ kind: "hold", hold, @@ -823,13 +850,13 @@ forEachCodec(() => { version: PlacedNotes.version, fingerprint: PlacedNotes.fingerprint, params: jsonPart({ session }), - ...(address === undefined ? {} : { address: { kind: "session", params: { id: address } } }), + address: { kind: "session", params: { id: address } }, }); } const misplaced = { code: "invalid", - reason: "placement", + reason: "host_address", message: "app.placed-notes: placed on session, this host is session|S1", }; @@ -839,7 +866,7 @@ forEachCodec(() => { expect((await child.next("bound")).holds[0]).toMatchObject({ hold: 0 }); }); - it("refuses a hold served here whose params place it elsewhere, whatever address it names: invalid/placement (race 5)", async () => { + it("refuses a hold served here whose params place it elsewhere, whatever address it names: invalid/host_address (race 5)", async () => { const { child } = await worker(); holdOf(child, "S2", "S1"); expect(await child.next("failed")).toMatchObject({ @@ -847,7 +874,7 @@ forEachCodec(() => { cause: { kind: "broken", error: { kind: "chord", error: misplaced } }, }); // A refused hold is kept (a retry starts it again): the next hold takes the next id. - holdOf(child, "S2", undefined, PlacedNotes.name, 1); + holdOf(child, "S2", "S2", PlacedNotes.name, 1); expect(await child.next("failed")).toMatchObject({ hold: 1, cause: { kind: "broken", error: { kind: "chord", error: misplaced } }, @@ -860,7 +887,7 @@ forEachCodec(() => { expect((await child.next("bound")).holds[0]).toMatchObject({ hold: 0 }); }); - it("sends a reference made here whose placement is another host over the parent link, though it serves the type", async () => { + it("sends a reference made here whose host address is another host over the parent link, though it serves the type", async () => { const { host, parent } = await worker({ parent: true }); const elsewhere = host.hold(PlacedNotes, { session: "S2" }, context); const hold = await parent!.next("hold"); @@ -871,7 +898,7 @@ forEachCodec(() => { it("sends a hold it cannot serve or route over the parent link, unparsed", async () => { const { child, parent } = await worker({ parent: true }); - holdOf(child, "S9", undefined, "app.elsewhere"); + holdOf(child, "S9", "S9", "app.elsewhere"); const relayed = await parent!.next("hold"); expect(relayed).toMatchObject({ type: "app.elsewhere" }); expect(partValue(relayed.params)).toEqual({ session: "S9" }); @@ -879,7 +906,7 @@ forEachCodec(() => { it("never sends a hold from the parent back to it: unavailable (race 8)", async () => { const { parent } = await worker({ parent: true }); - holdOf(parent!, "S9", undefined, "app.elsewhere"); + holdOf(parent!, "S9", "S9", "app.elsewhere"); expect(await parent!.next("failed")).toMatchObject({ hold: 0, cause: { @@ -892,25 +919,25 @@ forEachCodec(() => { it("refuses a hold it cannot serve or route without a parent link: unavailable", async () => { const { child } = await worker(); - holdOf(child, "S9", undefined, "app.elsewhere"); + holdOf(child, "S9", "S9", "app.elsewhere"); expect(await child.next("failed")).toMatchObject({ cause: { kind: "broken", error: { kind: "chord", error: { code: "unavailable" } } }, }); }); - it("fails a reference broken with what its placement throws", async () => { + it("fails a reference broken with what its hostAddress throws", async () => { const Throwing = remote("app.throwing", { version: 1, params: {}, - placement: () => { - throw new Error("placement threw"); + hostAddress: () => { + throw new Error("hostAddress threw"); }, variants: { v: { state: { n: int() } } }, }); const client = createHost(noErrors); await client.load([], context); expect(await rejection(client.get(Throwing, {}, context))).toMatchObject({ - message: "app.throwing: placement threw", + message: "app.throwing: hostAddress threw", }); expect(client.hold(Throwing, {}, context).state.value).toMatchObject({ kind: "failed" }); }); diff --git a/packages/chord/test/next/host/member-source.test.ts b/packages/chord/test/next/host/member-source.test.ts index 4a804cdb7..628b84e2c 100644 --- a/packages/chord/test/next/host/member-source.test.ts +++ b/packages/chord/test/next/host/member-source.test.ts @@ -22,6 +22,7 @@ import { loaded, memoryStore, metered, + Server, transported, valuesOf, } from "../support.ts"; @@ -324,7 +325,12 @@ describe("re-binding when a new attempt goes live", () => { describe("subscribers", () => { it("a hold over a link on a remote object re-exposing a local member gets frames again after the local retry", async () => { const Count = local("app.count", { version: 1, params: {}, members: { n: obj({ n: int() }), add: method() } }); - const View = remote("app.view", { version: 1, params: {}, variants: { v: { state: { n: obj({ n: int() }) } } } }); + const View = remote("app.view", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: obj({ n: int() }) } } }, + }); const Counter = defineState("counter", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); const selves: Self[] = []; const { server, client } = await linked((h) => { @@ -406,10 +412,11 @@ describe("a member the new attempt's variant lacks", () => { const Two = remote("app.two", { version: 1, params: {}, + hostAddress: () => Server(), variants: { full: { state: { n: int() } }, view: { state: { m: int() } } }, }); const owner = { kind: "owner" } as const; - const host = createHost({ principal: owner, onError: () => {} }); + const host = createHost({ address: Server(), principal: owner, onError: () => {} }); const selves: Self[] = []; const facet = defineFacet((h) => h.serve(Two, (_, _principal, self) => { diff --git a/packages/chord/test/next/host/object.test.ts b/packages/chord/test/next/host/object.test.ts index d492240f0..ab1675fb2 100644 --- a/packages/chord/test/next/host/object.test.ts +++ b/packages/chord/test/next/host/object.test.ts @@ -26,6 +26,8 @@ import { rejection, relayed, restartable, + SERVER_ADDRESS, + Server, Session, StoreRestartable, storeRestartable, @@ -705,14 +707,19 @@ describe("linger", () => { /** Every object attempted in this file, weakly: the objects of the cycles among them. */ const attempted = instancesOf(ChordObject, "attempt"); -/** On the server (no placement): its activation holds `Far` on the session's worker. */ -const Near = remote("app.near", { version: 1, params: { session: str() }, variants: { v: { state: { n: int() } } } }); +/** On the server: its activation holds `Far` on the session's worker. */ +const Near = remote("app.near", { + version: 1, + params: { session: str() }, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, +}); /** On the session's worker: its activation holds `Near` back on the server, or, with `local`, `Middle`. */ const Far = remote("app.far", { version: 1, params: { session: str() }, - placement: ({ session }) => Session({ id: session }), + hostAddress: ({ session }) => Session({ id: session }), variants: { v: { state: { n: int() } } }, }); @@ -720,6 +727,7 @@ const Far = remote("app.far", { const Middle = remote("app.middle", { version: 1, params: { session: str() }, + hostAddress: () => Server(), variants: { v: { state: { n: int() } } }, }); @@ -730,7 +738,7 @@ const Hub = local("app.hub", { version: 1, params: { session: str() }, members: const Loop = remote("app.loop", { version: 1, params: { here: str(), there: str() }, - placement: ({ here }) => Session({ id: here }), + hostAddress: ({ here }) => Session({ id: here }), variants: { v: { state: { n: int() } } }, }); @@ -910,7 +918,15 @@ function farHold(hold: number): Extract { /** A hold of `Near` for S1, as a worker sends it to the server. */ function nearHold(hold: number): Extract { const { name: type, version, fingerprint } = Near; - return { kind: "hold", hold, type, version, fingerprint, params: jsonPart({ session: "S1" }) }; + return { + kind: "hold", + hold, + type, + version, + fingerprint, + params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, + }; } describe("a failed activation's cleanup (audit)", () => { diff --git a/packages/chord/test/next/host/presence.test.ts b/packages/chord/test/next/host/presence.test.ts index 6260f9360..a43b523bf 100644 --- a/packages/chord/test/next/host/presence.test.ts +++ b/packages/chord/test/next/host/presence.test.ts @@ -22,6 +22,8 @@ import { PlacedNotes, RawPeer, relayed, + SERVER_ADDRESS, + Server, serveNotes, type TestPrincipal, testCodec, @@ -84,9 +86,10 @@ describe("host.references", () => { const Shown = remote("app.shown", { version: 1, params: { session: str() }, + hostAddress: () => Server(), variants: { v: { state: { here: int() } } }, }); - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); await server.load( [ { @@ -130,9 +133,10 @@ describe("host.references", () => { const Viewed = remote("app.viewed", { version: 1, params: { session: str() }, + hostAddress: () => Server(), variants: { v: { state: { count: int(), viewers: int() } } }, }); - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); const facet = defineFacet((host) => { serveNotes(host); host.serve(Viewed, ({ session }, _principal, self) => { @@ -151,7 +155,15 @@ describe("host.references", () => { server.accept(end, { principal: { kind: "owner" }, codec: testCodec() }, context); const peer = new RawPeer(b); const { name: type, version, fingerprint } = Viewed; - peer.send({ kind: "hold", hold: 0, type, version, fingerprint, params: jsonPart({ session: "S1" }) }); + peer.send({ + kind: "hold", + hold: 0, + type, + version, + fingerprint, + params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, + }); return peer; }; const drop = { armed: false }; @@ -176,9 +188,10 @@ describe("host.references", () => { const Viewed = remote("app.viewed", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { viewers: int() } } }, }); - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); const facet = defineFacet((host) => { host.serve( Viewed, @@ -195,13 +208,13 @@ describe("host.references", () => { server.accept(closingOnSend(a, drop), { principal: { kind: "owner" }, codec: testCodec() }, context); const peer = new RawPeer(b); const { name: type, version, fingerprint } = Viewed; - peer.send({ kind: "hold", hold: 0, type, version, fingerprint, params: "{}" }); + peer.send({ kind: "hold", hold: 0, type, version, fingerprint, params: "{}", address: SERVER_ADDRESS }); await peer.next("bound"); peer.send({ kind: "ack" }); await transported(); // The second hold's presence entry changes the first's state: sending it closes the link. drop.armed = true; - peer.send({ kind: "hold", hold: 1, type, version, fingerprint, params: "{}" }); + peer.send({ kind: "hold", hold: 1, type, version, fingerprint, params: "{}", address: SERVER_ADDRESS }); await transported(); expect(peer.closed).toBe(true); expect(server.references(Viewed).value).toEqual([]); @@ -210,8 +223,13 @@ describe("host.references", () => { }); it("completes a release's bookkeeping before its departure publishes: a link closing then leaves no lingering", async () => { - const Viewed = remote("app.viewed", { version: 1, params: {}, variants: { v: { state: { viewers: int() } } } }); - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const Viewed = remote("app.viewed", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { viewers: int() } } }, + }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); const facet = defineFacet((host) => { host.serve( Viewed, @@ -229,7 +247,7 @@ describe("host.references", () => { const peer = new RawPeer(b); const { name: type, version, fingerprint } = Viewed; for (const hold of [0, 1]) { - peer.send({ kind: "hold", hold, type, version, fingerprint, params: "{}" }); + peer.send({ kind: "hold", hold, type, version, fingerprint, params: "{}", address: SERVER_ADDRESS }); await peer.next("bound"); peer.send({ kind: "ack" }); } @@ -248,6 +266,7 @@ describe("host.references", () => { it("removes a bulk departure with one list and one splice op per entry, in one frame (a revoke of many holds)", async () => { const server = createHost({ + address: Server(), principal: { kind: "server" }, revokedBy: (principal) => (principal.kind === "guest" ? [{ session: principal.session }] : []), onError: () => {}, @@ -258,7 +277,15 @@ describe("host.references", () => { const peer = new RawPeer(b); const { name: type, version, fingerprint } = Notes; for (let hold = 0; hold < 4; hold++) { - peer.send({ kind: "hold", hold, type, version, fingerprint, params: jsonPart({ session: "S1" }) }); + peer.send({ + kind: "hold", + hold, + type, + version, + fingerprint, + params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, + }); } await transported(); const references = server.references(Notes); @@ -285,7 +312,12 @@ describe("host.references", () => { }); it("has no entries for lingering objects", async () => { - const Kept = remote("app.kept", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const Kept = remote("app.kept", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); const { server, client } = await linked((h) => h.serve(Kept, () => ({ variant: "v", members: { n: constant(1) } }), { linger: 60_000 }), ); diff --git a/packages/chord/test/next/host/reference.test.ts b/packages/chord/test/next/host/reference.test.ts index adeeb7288..2f2375b1c 100644 --- a/packages/chord/test/next/host/reference.test.ts +++ b/packages/chord/test/next/host/reference.test.ts @@ -29,6 +29,7 @@ import { memoryStore, Notes, rejection, + Server, settledHeap, } from "../support.ts"; @@ -338,12 +339,13 @@ describe("remote references", () => { const Two = remote("app.two", { version: 1, params: {}, + hostAddress: () => Server(), variants: { full: { state: { n: int() }, methods: { bump: method() } }, view: { state: { m: int() } } }, }); const Holder = local("app.holder", { version: 1, params: {}, members: { n: int() } }); const seen: unknown[] = []; const owner = { kind: "owner" } as const; - const host = createHost({ principal: owner, onError: () => {} }); + const host = createHost({ address: Server(), principal: owner, onError: () => {} }); const facet = defineFacet((h) => { h.serve(Two, () => ({ variant: "view", members: { m: constant(1) } })); h.serve(Holder, (_, self) => { @@ -369,6 +371,7 @@ describe("remote reference types", () => { const Two = remote("app.two", { version: 1, params: {}, + hostAddress: () => Server(), variants: { full: { state: { n: int() }, methods: { bump: method() } }, view: { state: { m: int() } } }, }); type Ref = Reference; @@ -384,6 +387,7 @@ describe("remote reference types", () => { const Two = remote("app.two", { version: 1, params: {}, + hostAddress: () => Server(), variants: { full: { state: { n: int() }, methods: { bump: method() } }, view: { state: { m: int() } } }, }); expectTypeOf["variant"]>().toEqualTypeOf<"full" | "view" | undefined>(); diff --git a/packages/chord/test/next/host/revocation.test.ts b/packages/chord/test/next/host/revocation.test.ts index a80fba200..ed5bab268 100644 --- a/packages/chord/test/next/host/revocation.test.ts +++ b/packages/chord/test/next/host/revocation.test.ts @@ -19,6 +19,8 @@ import { rawParent, rawWorkers, relayed, + SERVER_ADDRESS, + Server, Session, serveNotes, type TestPrincipal, @@ -50,9 +52,10 @@ const Barrier = local("app.barrier", { version: 1, params: {}, members: { n: int /** A guest's variant: its `wait` method's index. */ const { wait } = wireNames(Notes, "reader").methods as { wait: number }; -/** A worker of session S1 serving `Notes` (as `serveNotes`), its parent a raw peer whose holds name their principal. */ +/** A worker of session S1 serving `Notes` (as `serveNotes`, at the address `Notes` names), its parent a raw peer whose holds name their principal. */ async function worker(setup: (host: FacetHost) => void = (h) => void serveNotes(h)) { const host = createHost({ + address: Server(), principal: { kind: "worker", session: "S1" }, revokedBy, onError: () => {}, @@ -71,6 +74,7 @@ function holdAs(peer: RawPeer, hold: number, principal: TestPrincipal, Type: typ version: Type.version, fingerprint: Type.fingerprint, params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, principal, }); } @@ -79,6 +83,7 @@ describe("revocation keys", () => { it("are computed once per principal object, compared as canonical JSON", async () => { let calls = 0; const host = createHost({ + address: Server(), principal: { kind: "server" }, revokedBy: () => { calls++; @@ -103,6 +108,7 @@ describe("revocation keys", () => { it("keep no principal alive (a WeakMap keyed by the principal the host cached them for)", async () => { const cached: WeakRef[] = []; const host = createHost({ + address: Server(), principal: { kind: "worker", session: "S1" }, revokedBy: (principal) => { // The principal decoded on this host, the cache's key: not the sender's object. @@ -172,7 +178,12 @@ forEachCodec(() => { }); it("abandons an activation in progress (race 1): its open's context is aborted, its holders get gone", async () => { - const Slow = remote("app.slow", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const Slow = remote("app.slow", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); const aborted: unknown[] = []; const { host, parent } = await worker((h) => h.serve(Slow, (_, _principal, self) => { @@ -196,6 +207,7 @@ forEachCodec(() => { version: Slow.version, fingerprint: Slow.fingerprint, params: "{}", + address: SERVER_ADDRESS, principal: S1, }); await transported(); @@ -217,6 +229,7 @@ forEachCodec(() => { // A worker connects before it loads (W6); a load can wait for an activation (the start barrier). const opened = Promise.withResolvers(); const host = createHost({ + address: Server(), principal: { kind: "worker", session: "S1" }, revokedBy, onError: () => {}, @@ -313,6 +326,7 @@ forEachCodec(() => { describe("links and revoke messages", () => { it("closes the child links whose principal it revokes, never the parent link", async () => { const host = createHost({ + address: Server(), principal: { kind: "server" }, revokedBy, onError: () => {}, @@ -336,6 +350,7 @@ forEachCodec(() => { it("ignores a revoke from a child link (race 6)", async () => { const host = createHost({ + address: Server(), principal: { kind: "server" }, revokedBy, onError: () => {}, diff --git a/packages/chord/test/next/host/route.test.ts b/packages/chord/test/next/host/route.test.ts index eb3d6413f..756ae9429 100644 --- a/packages/chord/test/next/host/route.test.ts +++ b/packages/chord/test/next/host/route.test.ts @@ -12,6 +12,7 @@ import { PlacedNotes, rejection, relayed, + Server, Session, serveNotes, type TestPrincipal, @@ -101,18 +102,18 @@ forEachCodec(() => { }); }); - it("fails a hold broken invalid/placement for an address whose params do not fit the kind", async () => { + it("fails a hold broken invalid/host_address for an address whose params do not fit the kind", async () => { const Misplaced = remote("app.misplaced", { version: 1, params: { n: int() }, - // An address of the right kind with params of the wrong type, as a buggy placement could compute. - placement: ({ n }) => ({ kind: "session", params: { id: n }, key: "" }) as never, + // An address of the right kind with params of the wrong type, as a buggy `hostAddress` could compute. + hostAddress: ({ n }) => ({ kind: "session", params: { id: n }, key: "" }) as never, variants: { v: { state: { n: int() } } }, }); const { tab, resolves } = await relayed(); expect(await rejection(tab.get(Misplaced, { n: 1 }, context))).toMatchObject({ code: "invalid", - reason: "placement", + reason: "host_address", }); expect(resolves).toEqual([]); }); @@ -322,11 +323,16 @@ forEachCodec(() => { const Who = remote("app.who", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { methods: { app: method(undefined, str()) } } }, }); const resolved: (string | undefined)[] = []; let worker: Host | undefined; - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + onError: () => {}, + }); server.route( Session, { diff --git a/packages/chord/test/next/host/routines.test.ts b/packages/chord/test/next/host/routines.test.ts index e3bb69a6e..17dfe5eb1 100644 --- a/packages/chord/test/next/host/routines.test.ts +++ b/packages/chord/test/next/host/routines.test.ts @@ -36,6 +36,7 @@ import { loaded, Notes, rawParent, + Server, serveNotes, type TestPrincipal, transported, @@ -789,9 +790,10 @@ describe("on", () => { const Halves = remote("app.halves", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { half: int() }, methods: { set: method(int()) } } }, }); - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); const serve = defineFacet((h) => h.serve(Halves, (_, _principal, self) => { const doc = self.state(Doc); @@ -844,7 +846,7 @@ describe("on", () => { }); it("follows ref.state: a reference over a link going loading and live again", async () => { - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); await server.load([{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context); const Viewer = local("app.viewer", { version: 1, params: { id: int() }, members: {} }); const states: string[] = []; @@ -1069,7 +1071,12 @@ describe("routines and the end", () => { }); it("act as a remote object's principal", async () => { - const Watcher = remote("app.watcher", { version: 1, params: {}, variants: { v: { state: {} } } }); + const Watcher = remote("app.watcher", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: {} } }, + }); const seen: unknown[] = []; const { client } = await linked( (h) => diff --git a/packages/chord/test/next/jobs/job.test.ts b/packages/chord/test/next/jobs/job.test.ts index 4fd9aadcb..d82d82b3f 100644 --- a/packages/chord/test/next/jobs/job.test.ts +++ b/packages/chord/test/next/jobs/job.test.ts @@ -34,6 +34,7 @@ import { loaded, metered, rejection, + Server, type TestPrincipal, transported, } from "../support.ts"; @@ -1057,11 +1058,12 @@ describe("keep-alive", () => { const View = remote("app.view", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { running: arr(unknown()) } } }, }); let job: JobHandle<{ text: string }, string, undefined> | undefined; let ran = false; - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); const facet = defineFacet((h) => h.serve(View, (_, _principal, self) => { job = self.job(Echo, { @@ -1157,13 +1159,19 @@ describe("keep-alive", () => { describe("object end", () => { type Revocation = { session: string }; - const Jobs = remote("app.jobs", { version: 1, params: {}, variants: { v: { state: {} } } }); + const Jobs = remote("app.jobs", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: {} } }, + }); it("revocation (R11): runs' signals fire with the end's error, the drain waits for them, no finish", async () => { let job: JobHandle<{ text: string }, string, undefined> | undefined; const finished: string[] = []; let reason: unknown; const server = createHost({ + address: Server(), principal: { kind: "server" }, revokedBy: (p) => (p.kind === "guest" ? [{ session: p.session }] : []), onError: () => {}, @@ -1209,6 +1217,7 @@ describe("object end", () => { const gate = Promise.withResolvers(); const events: string[] = []; const server = createHost({ + address: Server(), principal: { kind: "server" }, revokedBy: (p) => (p.kind === "guest" ? [{ session: p.session }] : []), onError: () => {}, diff --git a/packages/chord/test/next/link/channel.test.ts b/packages/chord/test/next/link/channel.test.ts index f5abbe8eb..4179830af 100644 --- a/packages/chord/test/next/link/channel.test.ts +++ b/packages/chord/test/next/link/channel.test.ts @@ -20,11 +20,18 @@ import { RawPeer, rawLinked, rawProvider, + SERVER_ADDRESS, + Server, testCodec, transported, } from "../support.ts"; -const Thing = remote("app.thing", { version: 1, params: {}, variants: { only: { state: { n: int() } } } }); +const Thing = remote("app.thing", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { only: { state: { n: int() } } }, +}); const hold = { kind: "hold", @@ -33,6 +40,7 @@ const hold = { version: 1, fingerprint: "0000000000000000", params: "{}", + address: SERVER_ADDRESS, } as const; const problems = (errors: readonly HostError[]) => diff --git a/packages/chord/test/next/link/message.test.ts b/packages/chord/test/next/link/message.test.ts index 6adf76b4d..968612a2a 100644 --- a/packages/chord/test/next/link/message.test.ts +++ b/packages/chord/test/next/link/message.test.ts @@ -21,12 +21,23 @@ function partText(part: Part): Part { } const incarnation = "0190f5a2-7c3e-7d4a-9b1c-2d3e4f5a6b7c"; + +/** The address a hold names: `Server()` on the wire. */ +const server = { kind: "server", params: {} }; const error = { kind: "chord", error: { code: "released", message: "gone" } } as const; /** One message of every kind, with the optional fields both present and absent; parts are JSON text. */ const EVERY_KIND: Message[] = [ { kind: "hello", protocol: 1, incarnation, calls: 256 }, - { kind: "hold", hold: 300, type: "app.notes", version: 2, fingerprint: "0123456789abcdef", params: '{"id":1}' }, + { + kind: "hold", + hold: 300, + type: "app.notes", + version: 2, + fingerprint: "0123456789abcdef", + params: '{"id":1}', + address: server, + }, { kind: "hold", hold: 1, @@ -45,6 +56,7 @@ const EVERY_KIND: Message[] = [ version: 1, fingerprint: "0123456789abcdef", params: "{}", + address: server, incarnation, }, { kind: "release", hold: 0 }, @@ -106,7 +118,16 @@ const EVERY_KIND: Message[] = [ { kind: "ack" }, { kind: "ack", origin: 1 }, { kind: "closed", origin: 300 }, - { kind: "hold", origin: 2, hold: 0, type: "t", version: 1, fingerprint: "0123456789abcdef", params: "{}" }, + { + kind: "hold", + origin: 2, + hold: 0, + type: "t", + version: 1, + fingerprint: "0123456789abcdef", + params: "{}", + address: server, + }, { kind: "release", origin: 2, hold: 3 }, { kind: "call", origin: 2, call: 1, hold: 3, method: 0, args: "[1]" }, { kind: "abort", origin: 2, call: 1 }, @@ -130,10 +151,20 @@ const EVERY_KIND: Message[] = [ version: 1, fingerprint: "0123456789abcdef", params: "{}", + address: server, chain: [incarnation, incarnation], limit: 65_536, }, - { kind: "hold", hold: 4, type: "t", version: 1, fingerprint: "0123456789abcdef", params: "{}", limit: 0 }, + { + kind: "hold", + hold: 4, + type: "t", + version: 1, + fingerprint: "0123456789abcdef", + params: "{}", + address: server, + limit: 0, + }, { kind: "revoke", revocation: { grant: "g1" } }, { kind: "ping" }, { kind: "pong" }, @@ -181,7 +212,15 @@ forEachCodec(() => { const codec = testCodec(); const deflated = () => new DeflatedPart(new Uint8Array([0, 1, 2, 255])); const messages: Message[] = [ - { kind: "hold", hold: 0, type: "t", version: 1, fingerprint: "0123456789abcdef", params: deflated() }, + { + kind: "hold", + hold: 0, + type: "t", + version: 1, + fingerprint: "0123456789abcdef", + params: deflated(), + address: server, + }, { kind: "call", call: 0, hold: 0, method: 0, args: deflated() }, { kind: "result", call: 0, value: deflated(), segments: [new Uint8Array([7])] }, { @@ -253,6 +292,7 @@ forEachCodec(() => { version: 1, fingerprint: "0123456789abcdef", params: jsonPart({ "\uFEFFkey": "\uFEFFvalue" }), + address: server, }; const decoded = codec.decode(codec.encode(message)) as Extract; expect(plain(decoded)).toEqual(message); @@ -283,9 +323,10 @@ describe("the binary codec", () => { version: 1, fingerprint: "0123456789abcdef", params: "{}", + address: server, }); - // kind, id, type, version, fingerprint, params, the optional fields' flags - expect(hold).toHaveLength(1 + 1 + 2 + 1 + 8 + 3 + 1); + // kind, id, type, version, fingerprint, params, the address's kind and params, the optional fields' flags + expect(hold).toHaveLength(1 + 1 + 2 + 1 + 8 + 3 + 7 + 3 + 1); expect([...binaryCodec.encode({ kind: "ack" })]).toEqual([13]); // An origin: the kind byte's high bit, then the origin as a varint. expect([...binaryCodec.encode({ kind: "ack", origin: 300 })]).toEqual([13 | 0x80, 0xac, 0x02]); @@ -345,6 +386,7 @@ describe("the binary codec", () => { version: 1, fingerprint: "0123456789abcdef", params: "{}", + address: server, }); const forged = hold.slice(); forged[forged.length - 1] = 32; @@ -446,10 +488,12 @@ describe("the text codec", () => { version: 1, fingerprint: "0123456789abcdef", params: "{}", + address: server, } as const; - expect(text(hold)).toBe('["H",0,"t",1,"0123456789abcdef",[0,{}]]'); - expect(text({ ...hold, incarnation })).toBe(`["H",0,"t",1,"0123456789abcdef",[0,{}],null,null,"${incarnation}"]`); - expect(text({ ...hold, limit: 5 })).toBe('["H",0,"t",1,"0123456789abcdef",[0,{}],null,null,null,null,5]'); + const head = '["H",0,"t",1,"0123456789abcdef",[0,{}],["server",{}]'; + expect(text(hold)).toBe(`${head}]`); + expect(text({ ...hold, incarnation })).toBe(`${head},null,"${incarnation}"]`); + expect(text({ ...hold, limit: 5 })).toBe(`${head},null,null,null,5]`); }); it("puts byte segments after the tuple and a newline, their lengths the tuple's last field", () => { @@ -474,6 +518,15 @@ describe("the text codec", () => { expect(() => textCodec.decode(new TextEncoder().encode('["@",1,["@",2,["a"]]]'))).toThrow("wraps one message"); expect(() => textCodec.decode(new TextEncoder().encode('["@",1]'))).toThrow("wraps one message"); }); + + it("refuses a hold without its address: every hold names one", () => { + const decode = (tuple: string) => textCodec.decode(new TextEncoder().encode(tuple)); + expect(() => decode('["H",0,"t",1,"0123456789abcdef",[0,{}]]')).toThrow('a "H" message with 5 fields'); + expect(() => decode('["H",0,"t",1,"0123456789abcdef",[0,{}],null]')).toThrow("expected a list"); + expect(() => decode('["H",0,"t",1,"0123456789abcdef",[0,{}],["server"]]')).toThrow("an address is a pair"); + const hold = { kind: "hold", hold: 0, type: "t", version: 1, fingerprint: "0123456789abcdef", params: "{}" }; + expect(() => checkMessage(hold, context)).toThrow("message schema"); + }); }); describe("describeMessage", () => { @@ -534,6 +587,7 @@ describe("the binary codec's short strings", () => { version: 1, fingerprint: "0123456789abcdef", params: jsonPart(type), + address: server, }; expect(plain(binaryCodec.decode(binaryCodec.encode(message)))).toEqual(message); }); diff --git a/packages/chord/test/next/link/websocket.test.ts b/packages/chord/test/next/link/websocket.test.ts index 821c1468e..c67212b48 100644 --- a/packages/chord/test/next/link/websocket.test.ts +++ b/packages/chord/test/next/link/websocket.test.ts @@ -4,7 +4,7 @@ import { WebSocket, WebSocketServer } from "ws"; import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; import { createHost, defineFacet } from "../../../src/next/host/index.ts"; import { type WebSocketLike, webSocketLink } from "../../../src/next/link/index.ts"; -import { Notes, serveNotes, type TestPrincipal, transported } from "../support.ts"; +import { Notes, Server, serveNotes, type TestPrincipal, transported } from "../support.ts"; /** A socket playing a WebSocket by hand. */ class FakeSocket implements WebSocketLike { @@ -93,7 +93,7 @@ describe("WebSocket links over a fake socket", () => { /** A `ws` server on a free loopback port whose connections a server host accepts, and a URL to it. */ async function loopback(options: { readonly maxPayload?: number } = {}) { - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); await server.load([{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context); const wss = new WebSocketServer({ port: 0, host: "127.0.0.1", maxPayload: options.maxPayload }); await new Promise((resolve) => wss.once("listening", resolve)); diff --git a/packages/chord/test/next/remote/consumer.test.ts b/packages/chord/test/next/remote/consumer.test.ts index 32e4fb839..5f7005a68 100644 --- a/packages/chord/test/next/remote/consumer.test.ts +++ b/packages/chord/test/next/remote/consumer.test.ts @@ -23,6 +23,8 @@ import { rawParent, rawProvider, rejection, + SERVER_ADDRESS, + Server, serveNotes, testCodec, transported, @@ -287,6 +289,7 @@ forEachCodec(() => { version: Notes.version, fingerprint: Notes.fingerprint, params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, }); const [bound] = (await peer.next("bound")).holds; expect(bound!.variant).toBe(EDITOR.variant); @@ -415,6 +418,7 @@ forEachCodec(() => { const Echo = remote("app.echo", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { methods: { echo: method(map(int()), map(int())) } } }, }); const { client } = await linked((h) => diff --git a/packages/chord/test/next/remote/hold.test.ts b/packages/chord/test/next/remote/hold.test.ts index ab1c07b8a..03d27f74c 100644 --- a/packages/chord/test/next/remote/hold.test.ts +++ b/packages/chord/test/next/remote/hold.test.ts @@ -20,6 +20,8 @@ import { linked, RawPeer, rawLinked, + SERVER_ADDRESS, + Server, type TestPrincipal, testCodec, transported, @@ -50,6 +52,7 @@ const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () const Halves = remote("app.halves", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { n: obj({ n: int() }), half: int() }, methods: { set: method(int()), wait: method() } } }, }); @@ -91,18 +94,24 @@ async function halves() { function hold(peer: RawPeer, id: number, Type: { name: string; version: number; fingerprint: string }) { const { name: type, version, fingerprint } = Type; - peer.send({ kind: "hold", hold: id, type, version, fingerprint, params: "{}" }); + peer.send({ kind: "hold", hold: id, type, version, fingerprint, params: "{}", address: SERVER_ADDRESS }); } /** The provider object behind a hold, reached through the host's own lookup (to end it by hand). */ function providerObject(server: unknown) { const host = server as ChordHost; const { name: type, version, fingerprint } = Halves; - return host.serving({ type, version, fingerprint, params: "{}" }, {}, { kind: "owner" }, context); + const request = { type, version, fingerprint, params: "{}", address: SERVER_ADDRESS }; + return host.serving(request, {}, { kind: "owner" }, context); } /** A remote type whose activation never completes: its open waits for its abort. */ -const Pending = remote("app.pending", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); +const Pending = remote("app.pending", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, +}); function servePending(h: Parameters[0]>[0]) { h.serve(Pending, (_, _principal, self) => { @@ -221,7 +230,12 @@ forEachCodec(() => { fakeTimers(); it("a new hold of an object failed transiently is sent the failure at once", async () => { - const Flaky = remote("app.flaky", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const Flaky = remote("app.flaky", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); const { peer } = await rawLinked((h) => h.serve(Flaky, (_, _principal, self) => { self.open( @@ -309,6 +323,7 @@ forEachCodec(() => { const Slow = local("app.slow", { version: 1, params: {}, members: { n: int() } }); const { server, refs } = await (async () => { const server = createHost({ + address: Server(), principal: { kind: "server" }, readyTimeout: 2_000, onError: () => {}, @@ -343,7 +358,12 @@ forEachCodec(() => { }); /** A remote type whose open throws `offline` (broken when `failing.broken`) while `failing.left` is above 0. */ -const Flaky = remote("app.flaky", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); +const Flaky = remote("app.flaky", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, +}); function serveFlaky(h: Parameters[0]>[0], failing: { left: number; broken: boolean }) { h.serve(Flaky, (_, _principal, self) => { @@ -441,11 +461,13 @@ forEachCodec(() => { const Tally = remote("app.tally", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { n: obj({ n: int() }) }, methods: { bump: method() } } }, }); const Fresh = remote("app.fresh", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { n: obj({ n: int() }) }, methods: { bump: method() } } }, }); @@ -521,6 +543,7 @@ forEachCodec(() => { const Retiring = remote("app.retiring", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { doc: obj({ n: int() }), big: bool(), t: int() }, @@ -536,7 +559,7 @@ const retiredAll = [RETIRING.state.doc!, RETIRING.state.big!, RETIRING.state.t!] /** A server serving `Retiring` to a raw peer that acks only when told; `call` calls a method of hold 0. */ async function retiring() { - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); await server.load( [ { @@ -583,7 +606,7 @@ async function retiring() { expect((await peer.next("result")).call).toBe(id); }; const { name: type, version, fingerprint } = Retiring; - peer.send({ kind: "hold", hold: 0, type, version, fingerprint, params: "{}" }); + peer.send({ kind: "hold", hold: 0, type, version, fingerprint, params: "{}", address: SERVER_ADDRESS }); await peer.next("bound"); peer.ack(); return { peer, call }; diff --git a/packages/chord/test/next/remote/host-link.test.ts b/packages/chord/test/next/remote/host-link.test.ts index 7eb0a8551..c5205ea12 100644 --- a/packages/chord/test/next/remote/host-link.test.ts +++ b/packages/chord/test/next/remote/host-link.test.ts @@ -22,6 +22,7 @@ import { rawWorkers, rejection, relayed, + Server, Session, serveNotes, type TestPrincipal, @@ -33,6 +34,7 @@ import { const ServerWho = remote("app.server-who", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { who: str() }, methods: { echo: method(str(), str()) } } }, }); @@ -40,7 +42,7 @@ const ServerWho = remote("app.server-who", { const Asker = remote("app.asker", { version: 1, params: { session: str() }, - placement: ({ session }) => Session({ id: session }), + hostAddress: ({ session }) => Session({ id: session }), variants: { v: { state: { who: str() }, methods: { ask: method(str(), str()) } } }, }); @@ -48,7 +50,7 @@ const Asker = remote("app.asker", { const Neighbour = remote("app.neighbour", { version: 1, params: { session: str(), other: str() }, - placement: ({ session }) => Session({ id: session }), + hostAddress: ({ session }) => Session({ id: session }), variants: { v: { state: { count: int() } } }, }); @@ -141,6 +143,7 @@ forEachCodec(() => { const Gateway = remote("app.gateway", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { n: int() }, methods: { open: method(str(), int()) } } }, }); // With params: not activated by `load`, which would wait for the raw worker. @@ -342,6 +345,7 @@ async function compressing(options: { }) { const errors: HostError[] = []; const server = createHost({ + address: Server(), principal: { kind: "server" }, compression: options.server, onError: (error) => errors.push(error), diff --git a/packages/chord/test/next/remote/provider.test.ts b/packages/chord/test/next/remote/provider.test.ts index b1be51b9e..a93e54c62 100644 --- a/packages/chord/test/next/remote/provider.test.ts +++ b/packages/chord/test/next/remote/provider.test.ts @@ -21,6 +21,8 @@ import { RAW_PEER, RawPeer, rawLinked, + SERVER_ADDRESS, + Server, serveNotes, type TestPrincipal, testCodec, @@ -39,7 +41,16 @@ function hold( ) { const { name: type, version, fingerprint } = { ...Type, ...override }; const from = origin === undefined ? {} : { origin }; - peer.send({ kind: "hold", ...from, hold: id, type, version, fingerprint, params: jsonPart(params) }); + peer.send({ + kind: "hold", + ...from, + hold: id, + type, + version, + fingerprint, + params: jsonPart(params), + address: SERVER_ADDRESS, + }); } /** Sends a call of the method at index `method` of the bound variant, with `args` (none when left out). */ @@ -50,7 +61,12 @@ function call(peer: RawPeer, id: number, holdId: number, method: number, args?: } /** A remote type kept per id. */ -const Kept = remote("app.kept", { version: 1, params: { id: int() }, variants: { v: { state: { n: int() } } } }); +const Kept = remote("app.kept", { + version: 1, + params: { id: int() }, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, +}); /** The wire's indexes of `Notes`'s reader variant. */ const READER = wireNames(Notes, "reader"); @@ -120,7 +136,11 @@ forEachCodec(() => { }); it("arriving before load resolved wait for it", async () => { - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + onError: () => {}, + }); const [a, b] = memoryLinkPair(); server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); const peer = new RawPeer(b); @@ -177,7 +197,12 @@ forEachCodec(() => { }); it("before bound are refused not_ready (race 3)", async () => { - const Slow = remote("app.slow", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const Slow = remote("app.slow", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); const { peer } = await rawLinked((h) => h.serve(Slow, (_, _principal, self) => { self.open( @@ -333,7 +358,11 @@ forEachCodec(() => { it("a hold released while it waits for load takes nothing once load resolves", async () => { const events: string[] = []; - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + onError: () => {}, + }); const [a, b] = memoryLinkPair(); server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); const peer = new RawPeer(b); @@ -354,6 +383,7 @@ forEachCodec(() => { const Slow = remote("app.slow", { version: 1, params: { id: int() }, + hostAddress: () => Server(), variants: { v: { state: { n: int() } } }, }); const aborted: string[] = []; @@ -381,6 +411,7 @@ forEachCodec(() => { const Closing = remote("app.closing", { version: 1, params: { id: int() }, + hostAddress: () => Server(), variants: { v: { state: { n: int() } } }, }); const events: string[] = []; @@ -429,6 +460,7 @@ forEachCodec(() => { const Counter = remote("app.counter", { version: 1, params: { id: int() }, + hostAddress: () => Server(), variants: { v: { state: { n: int(), seen: int() }, methods: { bump: method() } } }, }); const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); @@ -488,6 +520,7 @@ forEachCodec(() => { const Grid = remote("app.grid", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { grid: obj({ cells: map(int()) }), one: int() }, @@ -520,7 +553,7 @@ function serveGrid(h: FacetHost): void { /** A server serving `Notes` and `Grid` to a raw peer that acks only when told, the link's context counting. */ async function slowConsumer() { const counting = await metered(); - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); await server.load( [ { @@ -729,7 +762,12 @@ forEachCodec(() => { }); it("carry constants and failed members, and an empty variant's binding, in a first bound (race 24)", async () => { - const Empty = remote("app.empty", { version: 1, params: {}, variants: { v: {} } }); + const Empty = remote("app.empty", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: {} }, + }); const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); const { peer } = await rawLinked((h) => { h.serve(Empty, () => ({ variant: "v", members: {} })); @@ -836,6 +874,7 @@ forEachCodec(() => { const Halves = remote("app.halves", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { one: int(), half: int() }, methods: { set: method(int()) } } }, }); @@ -968,7 +1007,15 @@ forEachCodec(() => { /** A hold of `Notes` for S1 as a message. */ function notesHold(id: number): Extract { const { name: type, version, fingerprint } = Notes; - return { kind: "hold", hold: id, type, version, fingerprint, params: jsonPart({ session: "S1" }) }; + return { + kind: "hold", + hold: id, + type, + version, + fingerprint, + params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, + }; } const UTF8 = new TextEncoder(); diff --git a/packages/chord/test/next/remote/relay.test.ts b/packages/chord/test/next/remote/relay.test.ts index 701b2adaf..73d5c3b62 100644 --- a/packages/chord/test/next/remote/relay.test.ts +++ b/packages/chord/test/next/remote/relay.test.ts @@ -21,6 +21,7 @@ import { rawWorkers, rejection, relayed, + Server, serveNotes, type TestPrincipal, testCodec, @@ -379,7 +380,11 @@ forEachCodec(() => { describe("a parent link that is down (audit)", () => { it("holds a hold arriving meanwhile on the parent path: it waits loading and binds on the next parent link", async () => { - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + onError: () => {}, + }); await server.load( [{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context, diff --git a/packages/chord/test/next/remote/remote-object.test.ts b/packages/chord/test/next/remote/remote-object.test.ts index 03baaa01c..eeb3fc7a9 100644 --- a/packages/chord/test/next/remote/remote-object.test.ts +++ b/packages/chord/test/next/remote/remote-object.test.ts @@ -35,6 +35,7 @@ import { rawParent, rawProvider, rejection, + Server, serveNotes, type TestPrincipal, testCodec, @@ -48,7 +49,12 @@ const Counter = defineState("counter", { schema: obj({ n: int() }), version: 1, forEachCodec(() => { describe("a hold over the parent link", () => { it("reject a broken failure at once", async () => { - const Broken = remote("app.broken", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const Broken = remote("app.broken", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); const { client } = await linked((h) => h.serve(Broken, () => { throw new Error("factory threw"); @@ -197,6 +203,7 @@ forEachCodec(() => { const Flaky = remote("app.flaky", { version: 1, params: { id: int() }, + hostAddress: () => Server(), variants: { v: { state: { n: int() } } }, }); const opens: string[] = []; @@ -250,6 +257,7 @@ forEachCodec(() => { const Halves = remote("app.halves", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { half: int() }, methods: { set: method(int()) } } }, }); const { server, client } = await linked((h) => @@ -363,7 +371,12 @@ forEachCodec(() => { fakeTimers(); it("fails at once with the real cause when its load holds it (no wait for a timeout)", async () => { - const Flaky = remote("app.flaky", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const Flaky = remote("app.flaky", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); const View = local("app.view", { version: 1, params: {}, members: { n: int() } }); const { server, client: first } = await linked((h) => h.serve(Flaky, (_, _principal, self) => { @@ -406,6 +419,7 @@ forEachCodec(() => { const Shifting = remote("app.shifting", { version: 1, params: {}, + hostAddress: () => Server(), variants: { a: { state: { x: int(), y: int() }, methods: { m: method() } }, b: { state: { y: int(), z: int() } }, @@ -592,10 +606,15 @@ forEachCodec(() => { const NotesV2 = remote("app.notes", { version: 2, params: { session: str() }, + hostAddress: () => Server(), variants: { editor: { state: { count: int() } } }, }); const { server, client, links } = await linked((h) => void serveNotes(h)); - const upgraded = createHost({ principal: { kind: "server" }, onError: () => {} }); + const upgraded = createHost({ + address: Server(), + principal: { kind: "server" }, + onError: () => {}, + }); const facet = defineFacet((h) => h.serve(NotesV2, () => ({ variant: "editor", members: { count: constant(0) } })), ); @@ -644,6 +663,7 @@ forEachCodec(() => { const Flaky = remote("app.flaky", { version: 1, params: {}, + hostAddress: () => Server(), variants: { only: { state: {}, methods: { ping: method() } } }, }); let factories = 0; diff --git a/packages/chord/test/next/support.ts b/packages/chord/test/next/support.ts index 9796d534f..06fb043ac 100644 --- a/packages/chord/test/next/support.ts +++ b/packages/chord/test/next/support.ts @@ -401,6 +401,7 @@ export async function linked( const errors = { server: [] as HostError[], client: [] as HostError[] }; const { telemetry, readyTimeout } = options; const server = createHost({ + address: Server(), principal: { kind: "server" }, telemetry, readyTimeout, @@ -547,6 +548,7 @@ export async function rawLinked( ): Promise<{ readonly server: Host; readonly peer: RawPeer; readonly errors: HostError[] }> { const errors: HostError[] = []; const server = createHost({ + address: Server(), principal: { kind: "server" }, readyTimeout: options.readyTimeout, compression: options.compression, @@ -580,8 +582,19 @@ const NOTES_VARIANTS = { reader: { state: { notes: arr(obj({ text: str() })), count: int() }, methods: { wait: method() } }, }; +/** The host kind of the test servers, where every test type but `PlacedNotes` lives. */ +export const Server = hostKind("server"); + +/** `Server()` as a hold names it on the wire: the address of every test type served on the server. */ +export const SERVER_ADDRESS = { kind: "server", params: {} }; + /** The remote type of the link tests: notes per session, an editor variant for owners, a reader one for guests. */ -export const Notes = remote("app.notes", { version: 1, params: { session: str() }, variants: NOTES_VARIANTS }); +export const Notes = remote("app.notes", { + version: 1, + params: { session: str() }, + hostAddress: () => Server(), + variants: NOTES_VARIANTS, +}); /** The kind of the relay tests' workers: one per session. */ export const Session = hostKind("session", { id: str() }); @@ -590,7 +603,7 @@ export const Session = hostKind("session", { id: str() }); export const PlacedNotes = remote("app.placed-notes", { version: 1, params: { session: str() }, - placement: ({ session }) => Session({ id: session }), + hostAddress: ({ session }) => Session({ id: session }), variants: NOTES_VARIANTS, }); @@ -630,7 +643,7 @@ export function serveNotes( }, }; }); - // The two types differ in their name and placement only. + // The two types differ in their name and host address only. host.serve(Type as typeof Notes, ({ session }, principal, self) => { probe.factories.push(`${session} ${principal.kind}`); const store = self.get(NoteStore, { session }); @@ -807,6 +820,7 @@ export async function relayed( const resolves: string[] = []; const { revokedBy, compression = {} } = options; const server = createHost({ + address: Server(), principal: { kind: "server" }, revokedBy: options.serverRevokedBy ?? revokedBy, compression: compression.server, From 373fbcc706686225c5c4f25cbecf95bae3d7811e Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 29 Oct 2026 07:37:56 +0000 Subject: [PATCH 134/271] docs(chord): dependency model: hostAddress --- packages/chord/docs/next/dependencies/book.md | 41 ++++++++++--------- .../chord/docs/next/dependencies/index.html | 39 +++++++++--------- .../chord/docs/next/dependencies/model.md | 14 +++---- 3 files changed, 48 insertions(+), 46 deletions(-) diff --git a/packages/chord/docs/next/dependencies/book.md b/packages/chord/docs/next/dependencies/book.md index 15665a434..1bafbc4a2 100644 --- a/packages/chord/docs/next/dependencies/book.md +++ b/packages/chord/docs/next/dependencies/book.md @@ -166,22 +166,22 @@ and a reload that removes a type sends `gone`. Both go. A request goes to one place, and the address decides which. For a `local` type, the place is this host. For a `remote` type, the consumer computes the address from the params with its own contract's -placement function (design §3): `Notes { session: "S1" }` is placed on `Session { id: "S1" }`. If that is this host's +`hostAddress` function (design §3; `placement` until 2026-10-29): `Notes { session: "S1" }` is placed on `Session { id: "S1" }`. If that is this host's address, the request goes here; otherwise over the route for that kind, or over the parent link. Today the code checks something else first: whether the type is registered here. A type placed on this host but not registered yet is sent over the parent link, which is wrong twice. Sol's example: the server has address `Server`, -the catalog type is placed on `Server`, and no facet serves it yet. Today's `Placement.reference` sends the server's +the catalog type is placed on `Server`, and no facet serves it yet. Today's `Placement.reference` (`host/placement.ts`) sends the server's own request over a parent link the server never had. And when the catalog plugin is installed later, nothing wakes that request, because it was never registered as waiting here. -:::flow wide | Placement by address. The destination follows from the address alone; only then is the provider found there checked. +:::flow wide | The destination by address. The destination follows from the address alone; only then is the provider found there checked. row "Where" accent -box addr "the request's address" "from the consumer's own placement function" accent +box addr "the request's address" "from the consumer's own `hostAddress`" accent box here "here" "a local type, or this host's address" box away "a route or the parent" "every other address" blue row "Then" muted -box check "check the provider here" "served? fitting version? authorize, own placement" +box check "check the provider here" "served? fitting version? authorize, own host address" box hold "hold over the path" "the provider host checks it the same way" blue arrow addr -> here arrow addr -> away @@ -189,13 +189,14 @@ arrow here -> check arrow away -> hold ::: -The rule, from the reviews (Sol B1, Astra F1): placement never looks at what is registered when the type names an +The rule, from the reviews (Sol B1, Astra F1): the destination never depends on what is registered when the type names an address. A request placed here whose provider is missing is **unbound here**, waiting for the name. A version check compares the consumer with the provider actually selected, never with a same-name registration on the consumer's own -host (today `#assertRoute` does that before placement, so worker W1 serving `Notes` v1 rejects a `Notes` v2 request +host (today `#assertRoute` does that before choosing the destination, so worker W1 serving `Notes` v1 rejects a `Notes` v2 request placed on W2, which serves v2). -Every `remote` type declares its placement (owner, 2026-10-29). Until then `remote()` allowed leaving it out, a +Every `remote` type declares its `hostAddress` (owner, 2026-10-29, who also renamed `placement` to it). Until then +`remote()` allowed leaving it out, a leftover of slice 3a, which built remote objects over one link before host kinds existed; such a type went "here if served, otherwise the parent", which needed its own fallback and a re-placement when a tab that loads before it connects gets its first parent link (Opus, Sol). Opus and Fable proposed making it required; the owner agreed. @@ -203,7 +204,7 @@ connects gets its first parent link (Opus, Sol). Opus and Fable proposed making ## What the check finds Once placed, the provider found there is checked: served, a fitting version (the current one or an accepted older one, -with the fingerprint computed from the provider's own history), `authorize`, and the provider's own placement check. +with the fingerprint computed from the provider's own history), `authorize`, and the provider's own host-address check. The request is then **bound** (to an object here, or to a hold over a path), or **unbound** with a reason: | The check finds | The request | Shows | @@ -211,7 +212,7 @@ The request is then **bound** (to an object here, or to a hold over a path), or | a fitting provider | bound | its object's state | | this host has not published its facets yet | unbound `unpublished` | `loading` | | nobody serves the name at its address | unbound `not_served` | `failed { waiting }` | -| no fitting version, a fingerprint that differs at the same version, a placement the provider computes elsewhere | unbound `version` | `failed { waiting }` | +| no fitting version, a fingerprint that differs at the same version, a host address the provider computes elsewhere | unbound `version` | `failed { waiting }` | | the registry is served, but its first `compose`, or the reader's version mapping, threw | unbound `compose` | `failed { waiting }` | | the provider host is installing, and the hold does not fit its current code | unbound `installing`, at the provider | `loading` | | the path has no link (a drop, a route resolving) | waits on the path | `loading` | @@ -598,9 +599,9 @@ has natives (§21), as for any reload. ## A type that moved -A plugin can move a type from the worker to the server by changing its placement. A consumer's request carries the +A plugin can move a type from the worker to the server by changing its `hostAddress`. A consumer's request carries the address its own code computed, so the provider appearing on the server cannot wake it: the request is not waiting -there. The old host answers `not_served`; the consumer's own reload, which brings the new placement, makes a new +there. The old host answers `not_served`; the consumer's own reload, which brings the new `hostAddress`, makes a new request. The model adds no redirect: moving state between hosts is a deliberate migration, not something Chord guesses. ## Rollouts across hosts @@ -1216,7 +1217,7 @@ scenarios against five drafts; the catalog in the chapters before this one is a | R18, the boundary | "live" means every dependency was live at the end of activation | the dependency check is the boundary: after it, a dependency leaving `live` is treated as after live | | `self.get` of a missing `local` type | throws in the factory: `broken` | unbound `not_served`: `failed { waiting }`, woken by the publication that serves it | | a remote hold the provider cannot serve | refused `broken`; the dependent polls by backoff | refused and kept at the provider, re-run at its publication; `failed { waiting }` | -| design §3, placement | "served here" decides "here"; `placement` optional on `remote()` | the address decides; every remote type declares its placement (owner, 2026-10-29) | +| design §3, the destination | "served here" decides "here"; `placement` optional on `remote()` | the address decides; every remote type declares its `hostAddress`, renamed from `placement` (owner, 2026-10-29) | | the start barrier, `pendingLoad` | holds wait for all of `load` | holds wait only for the publication; the barrier is (a) or (b) | | S2, a registry reader newer than its provider | `broken` | `failed { waiting, version }` | | S3, a registry without a value | `transient`, woken at the first value only | `failed { waiting }`, woken at every transition to usable | @@ -1314,14 +1315,14 @@ A needs W's B, B needs S's C; W's hold for C waits for S's whole `load`, which w code (`host.get`, `host.hold`, `host.registry`), a link's peer (`hold`). **It ends only when its holder ends it** (the attempt ends, the call settles, `release()`, a getter gives up, the peer's link or origin closes, revocation, the holding host closes), never because its provider is missing, went away or does not fit. -- **Placement by address** (round 1): a `local` type goes here; a `remote` type with a placement goes to the address - its placement (computed from the consumer's own contract) gives: here if that is this host's address, whether served +- **The destination by address** (round 1): a `local` type goes here; a `remote` type goes to the address its + `hostAddress` (computed from the consumer's own contract) gives: here if that is this host's address, whether served here or not, else over the route for that kind or the parent (design §3). Every `remote` type declares - its placement (owner, 2026-10-29: the optional placement of slice 3a is removed, and with it the fallback "here if + its `hostAddress` (owner, 2026-10-29: the optional `placement` of slice 3a is removed and the field renamed, and with it the fallback "here if served, otherwise the parent" and its re-placement at the first parent link). A hold arriving for an address that is not this host's is relayed or - refused (`invalid/placement`) as today; a relay never captures a hold because it serves the same name. + refused (`invalid/host_address`) as today; a relay never captures a hold because it serves the same name. - **Then the selected provider is checked** (served, version or accepted version and fingerprint, `authorize`, the - provider's own placement check). Version checks never look at a same-name registration elsewhere. + provider's own host-address check). Version checks never look at a same-name registration elsewhere. - **The publication gate:** a request made on this host, or arriving on its links, before the host's first publication (§6) is not placed yet; it shows `loading` and is placed by the publication. The gate lasts only for the synchronous setups, never for activation, so it cannot deadlock. @@ -1332,7 +1333,7 @@ A needs W's B, B needs S's C; W's hold for C waits for S's whole `load`, which w | a fitting provider | bound (to the object here, or to the hold over a path) | | not published yet | unbound `unpublished`: `loading` | | nobody serves the name at its address | unbound `not_served`: `failed { waiting }` | - | no fitting version, a fingerprint that differs at the same version, a placement the provider computes elsewhere (code skew) | unbound `version`: `failed { waiting }` | + | no fitting version, a fingerprint that differs at the same version, a host address the provider computes elsewhere (code skew) | unbound `version`: `failed { waiting }` | | the registry is served, but its first `compose` (or the reader's version mapping) threw | unbound `compose`: `failed { waiting }` | | the provider host is installing a revision and the hold does not fit its current code | unbound `installing` at the provider: `loading` | | the path has no link (a drop, a route resolving) | on the path: `loading` | @@ -1646,7 +1647,7 @@ ring that never existed, `/tmp/model/sol-probe4.log`). R18 (dependency failures `waiting`, no backoff, the deadline paused while parked, the dependency check as boundary, wakes on every change, an ended dependency breaks), S2 and S3 (`waiting`, woken at every usable value), slice 8's -last-good rule (dropped when a contribution leaves), §3's placement table (by address; the publication gate), §6 (the +last-good rule (dropped when a contribution leaves), §3's destination table (by address; the publication gate), §6 (the simultaneous-cycle gap: (A) one narrowed deadline or (B) the probe), §7 (`host.hold`, getters, calls), §21 (no `gone` on removal; a dropped version unbinds only its requests; installs never await activations; rollout without revision fields; the reload handover), the start barrier (R17, W6), `FailureCause` (+ `waiting`), `loading`/`failed` with the diff --git a/packages/chord/docs/next/dependencies/index.html b/packages/chord/docs/next/dependencies/index.html index 656a4f6fa..81c7ae1fe 100644 --- a/packages/chord/docs/next/dependencies/index.html +++ b/packages/chord/docs/next/dependencies/index.html @@ -314,26 +314,27 @@ and a reload that removes a type sends gone. Both go.

    The address decides where

    A request goes to one place, and the address decides which. For a local type, the place is this host. For a remote type, the consumer computes the address from the params with its own contract's -placement function (design §3): Notes { session: "S1" } is placed on Session { id: "S1" }. If that is this host's +hostAddress function (design §3; placement until 2026-10-29): Notes { session: "S1" } is placed on Session { id: "S1" }. If that is this host's address, the request goes here; otherwise over the route for that kind, or over the parent link.

    Today the code checks something else first: whether the type is registered here. A type placed on this host but not registered yet is sent over the parent link, which is wrong twice. Sol's example: the server has address Server, -the catalog type is placed on Server, and no facet serves it yet. Today's Placement.reference sends the server's +the catalog type is placed on Server, and no facet serves it yet. Today's Placement.reference (host/placement.ts) sends the server's own request over a parent link the server never had. And when the catalog plugin is installed later, nothing wakes that request, because it was never registered as waiting here.

    -
    WHERETHENthe request's addressfrom the consumer's own placementfunctionherea local type, or this host's addressa route or the parentevery other addresscheck the provider hereserved? fitting version? authorize,own placementhold over the paththe provider host checks it the sameway
    Figure 2. Placement by address. The destination follows from the address alone; only then is the provider found there checked.
    -

    The rule, from the reviews (Sol B1, Astra F1): placement never looks at what is registered when the type names an +

    WHERETHENthe request's addressfrom the consumer's own `hostAddress`herea local type, or this host's addressa route or the parentevery other addresscheck the provider hereserved? fitting version? authorize,own host addresshold over the paththe provider host checks it the sameway
    Figure 2. The destination by address. The destination follows from the address alone; only then is the provider found there checked.
    +

    The rule, from the reviews (Sol B1, Astra F1): the destination never depends on what is registered when the type names an address. A request placed here whose provider is missing is unbound here, waiting for the name. A version check compares the consumer with the provider actually selected, never with a same-name registration on the consumer's own -host (today #assertRoute does that before placement, so worker W1 serving Notes v1 rejects a Notes v2 request +host (today #assertRoute does that before choosing the destination, so worker W1 serving Notes v1 rejects a Notes v2 request placed on W2, which serves v2).

    -

    Every remote type declares its placement (owner, 2026-10-29). Until then remote() allowed leaving it out, a +

    Every remote type declares its hostAddress (owner, 2026-10-29, who also renamed placement to it). Until then +remote() allowed leaving it out, a leftover of slice 3a, which built remote objects over one link before host kinds existed; such a type went "here if served, otherwise the parent", which needed its own fallback and a re-placement when a tab that loads before it connects gets its first parent link (Opus, Sol). Opus and Fable proposed making it required; the owner agreed.

    What the check finds

    Once placed, the provider found there is checked: served, a fitting version (the current one or an accepted older one, -with the fingerprint computed from the provider's own history), authorize, and the provider's own placement check. +with the fingerprint computed from the provider's own history), authorize, and the provider's own host-address check. The request is then bound (to an object here, or to a hold over a path), or unbound with a reason:

    @@ -359,7 +360,7 @@ The request is then bound (to an object here, or to a hold over - + @@ -721,9 +722,9 @@ object then; Sol and Astra caught it: the object, its documents, its jobs and it fine. Only the requests on version 1 become unbound version. The object's own plugin reload still restarts it if it has natives (§21), as for any reload.

    A type that moved

    -

    A plugin can move a type from the worker to the server by changing its placement. A consumer's request carries the +

    A plugin can move a type from the worker to the server by changing its hostAddress. A consumer's request carries the address its own code computed, so the provider appearing on the server cannot wake it: the request is not waiting -there. The old host answers not_served; the consumer's own reload, which brings the new placement, makes a new +there. The old host answers not_served; the consumer's own reload, which brings the new hostAddress, makes a new request. The model adds no redirect: moving state between hosts is a deliberate migration, not something Chord guesses.

    Rollouts across hosts

    A rollout installs a revision on the server first, then on the workers, then on the tabs (design §21). Between those @@ -1929,9 +1930,9 @@ owner's decisions.

    - + - + @@ -2099,15 +2100,15 @@ code (host.get, host.hold, host.registry) attempt ends, the call settles, release(), a getter gives up, the peer's link or origin closes, revocation, the holding host closes), never because its provider is missing, went away or does not fit.

    -
  • Placement by address (round 1): a local type goes here; a remote type with a placement goes to the address -its placement (computed from the consumer's own contract) gives: here if that is this host's address, whether served +

  • The destination by address (round 1): a local type goes here; a remote type goes to the address its +hostAddress (computed from the consumer's own contract) gives: here if that is this host's address, whether served here or not, else over the route for that kind or the parent (design §3). Every remote type declares -its placement (owner, 2026-10-29: the optional placement of slice 3a is removed, and with it the fallback "here if +its hostAddress (owner, 2026-10-29: the optional placement of slice 3a is removed and the field renamed, and with it the fallback "here if served, otherwise the parent" and its re-placement at the first parent link). A hold arriving for an address that is not this host's is relayed or -refused (invalid/placement) as today; a relay never captures a hold because it serves the same name.

    +refused (invalid/host_address) as today; a relay never captures a hold because it serves the same name.

  • Then the selected provider is checked (served, version or accepted version and fingerprint, authorize, the -provider's own placement check). Version checks never look at a same-name registration elsewhere.

    +provider's own host-address check). Version checks never look at a same-name registration elsewhere.

  • The publication gate: a request made on this host, or arriving on its links, before the host's first publication (§6) is not placed yet; it shows loading and is placed by the publication. The gate lasts only for the @@ -2134,7 +2135,7 @@ synchronous setups, never for activation, so it cannot deadlock.

  • - + @@ -2565,7 +2566,7 @@ once it is live; the run count keeps the object as today.

    12. What changes in the design

    R18 (dependency failures waiting, no backoff, the deadline paused while parked, the dependency check as boundary, wakes on every change, an ended dependency breaks), S2 and S3 (waiting, woken at every usable value), slice 8's -last-good rule (dropped when a contribution leaves), §3's placement table (by address; the publication gate), §6 (the +last-good rule (dropped when a contribution leaves), §3's destination table (by address; the publication gate), §6 (the simultaneous-cycle gap: (A) one narrowed deadline or (B) the probe), §7 (host.hold, getters, calls), §21 (no gone on removal; a dropped version unbinds only its requests; installs never await activations; rollout without revision fields; the reload handover), the start barrier (R17, W6), FailureCause (+ waiting), loading/failed with the diff --git a/packages/chord/docs/next/dependencies/model.md b/packages/chord/docs/next/dependencies/model.md index 2b838eb79..bc3d606c8 100644 --- a/packages/chord/docs/next/dependencies/model.md +++ b/packages/chord/docs/next/dependencies/model.md @@ -61,14 +61,14 @@ A needs W's B, B needs S's C; W's hold for C waits for S's whole `load`, which w code (`host.get`, `host.hold`, `host.registry`), a link's peer (`hold`). **It ends only when its holder ends it** (the attempt ends, the call settles, `release()`, a getter gives up, the peer's link or origin closes, revocation, the holding host closes), never because its provider is missing, went away or does not fit. -- **Placement by address** (round 1): a `local` type goes here; a `remote` type with a placement goes to the address - its placement (computed from the consumer's own contract) gives: here if that is this host's address, whether served +- **The destination by address** (round 1): a `local` type goes here; a `remote` type goes to the address its + `hostAddress` (computed from the consumer's own contract) gives: here if that is this host's address, whether served here or not, else over the route for that kind or the parent (design §3). Every `remote` type declares - its placement (owner, 2026-10-29: the optional placement of slice 3a is removed, and with it the fallback "here if + its `hostAddress` (owner, 2026-10-29: the optional `placement` of slice 3a is removed and the field renamed, and with it the fallback "here if served, otherwise the parent" and its re-placement at the first parent link). A hold arriving for an address that is not this host's is relayed or - refused (`invalid/placement`) as today; a relay never captures a hold because it serves the same name. + refused (`invalid/host_address`) as today; a relay never captures a hold because it serves the same name. - **Then the selected provider is checked** (served, version or accepted version and fingerprint, `authorize`, the - provider's own placement check). Version checks never look at a same-name registration elsewhere. + provider's own host-address check). Version checks never look at a same-name registration elsewhere. - **The publication gate:** a request made on this host, or arriving on its links, before the host's first publication (§6) is not placed yet; it shows `loading` and is placed by the publication. The gate lasts only for the synchronous setups, never for activation, so it cannot deadlock. @@ -79,7 +79,7 @@ A needs W's B, B needs S's C; W's hold for C waits for S's whole `load`, which w | a fitting provider | bound (to the object here, or to the hold over a path) | | not published yet | unbound `unpublished`: `loading` | | nobody serves the name at its address | unbound `not_served`: `failed { waiting }` | - | no fitting version, a fingerprint that differs at the same version, a placement the provider computes elsewhere (code skew) | unbound `version`: `failed { waiting }` | + | no fitting version, a fingerprint that differs at the same version, a host address the provider computes elsewhere (code skew) | unbound `version`: `failed { waiting }` | | the registry is served, but its first `compose` (or the reader's version mapping) threw | unbound `compose`: `failed { waiting }` | | the provider host is installing a revision and the hold does not fit its current code | unbound `installing` at the provider: `loading` | | the path has no link (a drop, a route resolving) | on the path: `loading` | @@ -393,7 +393,7 @@ ring that never existed, `/tmp/model/sol-probe4.log`). R18 (dependency failures `waiting`, no backoff, the deadline paused while parked, the dependency check as boundary, wakes on every change, an ended dependency breaks), S2 and S3 (`waiting`, woken at every usable value), slice 8's -last-good rule (dropped when a contribution leaves), §3's placement table (by address; the publication gate), §6 (the +last-good rule (dropped when a contribution leaves), §3's destination table (by address; the publication gate), §6 (the simultaneous-cycle gap: (A) one narrowed deadline or (B) the probe), §7 (`host.hold`, getters, calls), §21 (no `gone` on removal; a dropped version unbinds only its requests; installs never await activations; rollout without revision fields; the reload handover), the start barrier (R17, W6), `FailureCause` (+ `waiting`), `loading`/`failed` with the From aee933269ba79b84e7b7454097c1d4c426021d82 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 29 Oct 2026 07:44:21 +0000 Subject: [PATCH 135/271] docs(chord): dependency model: hostAddress wording --- packages/chord/docs/next/dependencies/book.md | 13 +++++++------ packages/chord/docs/next/dependencies/index.html | 13 +++++++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/packages/chord/docs/next/dependencies/book.md b/packages/chord/docs/next/dependencies/book.md index 1bafbc4a2..2269bea41 100644 --- a/packages/chord/docs/next/dependencies/book.md +++ b/packages/chord/docs/next/dependencies/book.md @@ -169,11 +169,12 @@ host. For a `remote` type, the consumer computes the address from the params wit `hostAddress` function (design §3; `placement` until 2026-10-29): `Notes { session: "S1" }` is placed on `Session { id: "S1" }`. If that is this host's address, the request goes here; otherwise over the route for that kind, or over the parent link. -Today the code checks something else first: whether the type is registered here. A type placed on this host but not -registered yet is sent over the parent link, which is wrong twice. Sol's example: the server has address `Server`, -the catalog type is placed on `Server`, and no facet serves it yet. Today's `Placement.reference` (`host/placement.ts`) sends the server's -own request over a parent link the server never had. And when the catalog plugin is installed later, nothing wakes -that request, because it was never registered as waiting here. +Until 2026-10-29 the code checked something else first: whether the type is registered here. A type placed on this host but not +registered yet was sent over the parent link, which is wrong twice. Sol's example: the server has address `Server`, +the catalog type is placed on `Server`, and no facet serves it yet. That code, `Placement.reference` (`host/placement.ts`, now `host/destination.ts`), sent the server's +own request over a parent link the server never had, and when the catalog plugin was installed later, nothing woke +that request, because it was never registered as waiting here. The rename removed the first half; slice 7 builds the +wait. :::flow wide | The destination by address. The destination follows from the address alone; only then is the provider found there checked. row "Where" accent @@ -1076,7 +1077,7 @@ The Rule column cites sections of the working document, reproduced in appendix C | # | Scenario | Under the model | Today | Rule | | --- | --- | --- | --- | --- | | 1 | `self.get(NoteStore)` in `Notes { S1 }`'s factory on W1; `NoteStore` is served and activating. | `Notes` makes its request, it binds to the `NoteStore` object, `Notes` parks `loading`. `NoteStore` goes live: a target state change wakes `Notes`, which opens its natives and goes live. No timer ran. | The same wait, bounded by `readyTimeout`; past it `Notes` fails `transient` and backs off (`attempt.ts` `waitFor`). | §3.1, §3.4, §4 | -| 2 | `self.get(Catalog)` from W1's `ForwardStore`; the parent link is up, S has published, `Catalog` is served there. | The request goes over the parent (placement `Server()`); S binds the hold; `bound` arrives; `ForwardStore` was `loading`, now goes on and live. | The same, after S's whole barrier (`Hold.start` awaits `pendingLoad`). | §1, §2 | +| 2 | `self.get(Catalog)` from W1's `ForwardStore`; the parent link is up, S has published, `Catalog` is served there. | The request goes over the parent (`hostAddress: () => Server()`); S binds the hold; `bound` arrives; `ForwardStore` was `loading`, now goes on and live. | The same, after S's whole barrier (`Hold.start` awaits `pendingLoad`). | §1, §2 | | 3 | The same, but `Catalog`'s plugin is not installed on S. | S indexes the hold `not_served` under "p2.catalog" and answers `failed { waiting, not_served }`; `ForwardStore`'s reference shows `failed { waiting }`, the object parks `failed { waiting, "waiting for p2.catalog: not served" }`. A reload on S serving `Catalog` re-runs the hold: `loading`, `bound`; the reference goes live, the dependent wakes. No backoff, no new hold. | The hold is refused `broken`; `ForwardStore` fails `transient` and re-holds every 1-60 s, refused each time (`remote/hold.ts` `#refuse`, `object.ts` backoff). | §1, §3.1, §4, §5 | | 4 | `self.get(SessionInfo)` on W1 when the core facet is missing (same host, not served). | The request is unbound `not_served` on W1 itself; the dependent parks `failed { waiting }`; a reload serving the type binds it and wakes the dependent. | `#assertRoute` throws in the factory: the object is `broken` until `retry()` or a reload (`host.ts` `reference`). | §1, §3.1, §4 | | 5 | A tab's `NotesView` declares `Notes { S1 }`; S routes `Session`, `resolve` forks W1, W1 connects and publishes. | The hold waits at T on its path (`no_link` while S resolves: `loading`), is relayed to W1, waits there `unpublished` until W1's setups, then binds; `bound` comes back through the relay. `NotesView` was `loading` the whole time and goes live. | The same chain of waits, each bounded by `readyTimeout`; a slow worker start past it fails the tab object `transient` and polls. | §1, §6.2 | diff --git a/packages/chord/docs/next/dependencies/index.html b/packages/chord/docs/next/dependencies/index.html index 81c7ae1fe..cefc248a9 100644 --- a/packages/chord/docs/next/dependencies/index.html +++ b/packages/chord/docs/next/dependencies/index.html @@ -316,11 +316,12 @@ and a reload that removes a type sends gone. Both go.

    host. For a remote type, the consumer computes the address from the params with its own contract's hostAddress function (design §3; placement until 2026-10-29): Notes { session: "S1" } is placed on Session { id: "S1" }. If that is this host's address, the request goes here; otherwise over the route for that kind, or over the parent link.

    -

    Today the code checks something else first: whether the type is registered here. A type placed on this host but not -registered yet is sent over the parent link, which is wrong twice. Sol's example: the server has address Server, -the catalog type is placed on Server, and no facet serves it yet. Today's Placement.reference (host/placement.ts) sends the server's -own request over a parent link the server never had. And when the catalog plugin is installed later, nothing wakes -that request, because it was never registered as waiting here.

    +

    Until 2026-10-29 the code checked something else first: whether the type is registered here. A type placed on this host but not +registered yet was sent over the parent link, which is wrong twice. Sol's example: the server has address Server, +the catalog type is placed on Server, and no facet serves it yet. That code, Placement.reference (host/placement.ts, now host/destination.ts), sent the server's +own request over a parent link the server never had, and when the catalog plugin was installed later, nothing woke +that request, because it was never registered as waiting here. The rename removed the first half; slice 7 builds the +wait.

    WHERETHENthe request's addressfrom the consumer's own `hostAddress`herea local type, or this host's addressa route or the parentevery other addresscheck the provider hereserved? fitting version? authorize,own host addresshold over the paththe provider host checks it the sameway
    Figure 2. The destination by address. The destination follows from the address alone; only then is the provider found there checked.

    The rule, from the reviews (Sol B1, Astra F1): the destination never depends on what is registered when the type names an address. A request placed here whose provider is missing is unbound here, waiting for the name. A version check @@ -1358,7 +1359,7 @@ does is needed to recover; and an ended holder leaves nothing behind in any inde

    - + From a54f250fb40ba9664c62fb013b4aeede70fbe4df Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 29 Oct 2026 08:00:54 +0000 Subject: [PATCH 136/271] docs(chord): decision: getters wait through waiting until their readyTimeout --- packages/chord/docs/next/decisions.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index e87d429d7..065822097 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1771,6 +1771,14 @@ validation and copies the v2 kernel added around Delta. setups a registry without a value cannot get one before a reload, so a reader then fails at once `transient` (`unavailable/undeclared`) and is retried when the registry gets its first value (R18), as `self.get` of a type not served here fails at once (refined 2026-10-27 after the second spec review: no `readyTimeout` wait at start); once live, a later failure of the registry changes nothing (last good value). +- **Getters wait through `waiting` (owner, 2026-10-29: "(b) with a default seems fine; UI stays responsive anyways + because it usually holds instead of get"; dependency model, decision 2):** `host.get`, `ctx.get` and `host.registry` + wait while the reference is `loading`, `failed { transient }` or `failed { waiting }`, bounded by their + `readyTimeout` (default 60 s; callers choose their own), and reject at the deadline with the last cause; only + `failed { broken }` rejects at once. Calls stay as they are: on a `loading` reference they park, bounded; on a + `failed` one (any cause) they reject at once. A UI holds (`host.hold`) and shows `ref.state`; code that needs the + object now gets it. Reviewers: Fable (b, calls included), Opus (a, plus a `whenLive` helper), Sol leaning (c, callers + park until their own context aborts), Astra pending. - **Only hosts with an address serve remote types (owner, 2026-10-29: "b"):** a host serving a `remote` type needs an address and a principal; `load` refuses a facet that serves one on a host without an address (until now a host with a principal but no address served remote types and placed every one of them "here"). So "here" is decided by the From 249f24bfc5008244493272b3736b3591aaf4956a Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 29 Oct 2026 08:03:10 +0000 Subject: [PATCH 137/271] docs(chord): getters decision: reviewer votes --- packages/chord/docs/next/decisions.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 065822097..b05ae7047 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1777,8 +1777,10 @@ validation and copies the v2 kernel added around Delta. `readyTimeout` (default 60 s; callers choose their own), and reject at the deadline with the last cause; only `failed { broken }` rejects at once. Calls stay as they are: on a `loading` reference they park, bounded; on a `failed` one (any cause) they reject at once. A UI holds (`host.hold`) and shows `ref.state`; code that needs the - object now gets it. Reviewers: Fable (b, calls included), Opus (a, plus a `whenLive` helper), Sol leaning (c, callers - park until their own context aborts), Astra pending. + object now gets it. Reviewers (`/tmp/model/stw-*.md`): Fable (b, calls included), Opus (a, plus a `whenLive` + helper), Sol and Astra (c: callers park until their own context aborts, with caller deadlines and caps on pending + calls; (b)'s default deadline is that cap). Sol's p2 note: cleanup in a `finally` after an abort needs its own + bounded context, not the aborted one. - **Only hosts with an address serve remote types (owner, 2026-10-29: "b"):** a host serving a `remote` type needs an address and a principal; `load` refuses a facet that serves one on a host without an address (until now a host with a principal but no address served remote types and placed every one of them "here"). So "here" is decided by the From 6fed3cfb99fbe22540030a4d0f099a873ee8e87a Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 09:24:53 +0100 Subject: [PATCH 138/271] fix(coding-agent): let the SDK stream resolve auth for summaries (#11556) In sessions created by `createAgentSession`, compaction, branch summaries and bug-report summaries resolved the stored OAuth credential in `AgentSession._getSummarizationRequestAuth`, then passed the access token on as `options.apiKey`. The SDK's stream function calls `modelRuntime.streamSimple`, which resolves auth again. Because it was given an explicit `apiKey`, `resolveProviderAuth` took the API-key override branch instead of using stored OAuth. Built-in providers accept the token either way. A provider whose API-key path differs from its OAuth path, such as one that refuses explicit keys, failed every summary while normal prompts worked. `createAgentSession` now marks its stream function with `markRuntimeStreamFunction`. This records it in a module-level `WeakMap` keyed by the function, mapped to its `ModelRuntime`. `_getSummarizationRequestAuth` skips pre-resolution when the agent's stream function is marked for the session's own runtime, so that runtime resolves auth once, as it does for normal prompts. The check runs after virtual-model routing. Bare `streamSimple` and custom `streamFn`s still receive resolved credentials, so the fixes for #6324 and #7579 are unchanged. If a caller replaces `agent.streamFunction`, the old behavior applies. **Testing** - Added `test/suite/regressions/11520-sdk-summary-oauth-auth.test.ts`. It builds an SDK session on a provider that has stored OAuth and also an API-key method, runs a branch summary, and asserts two things: no `getAuth` call carries an `apiKey` override, and the provider receives the OAuth access token. It fails without the change and passes with it. - `npm run check` passes and `./test.sh` passes. I also ran the related tests: 6324 branch-summary auth, compaction model overrides, auto-compaction cancellation, SDK stream options, virtual models, and the compaction and branch-summary tests. - Manual reproduction: an SDK script with an `openai` OAuth credential that logs `ModelRuntime.getAuth` during `navigateTree(..., { summarize: true })`. Before the change there were two calls, the second with `apiKey = `. After the change there is one call, with no override. Fixes #11520 Requested by @badlogic Co-authored-by: triagent --- packages/coding-agent/CHANGELOG.md | 1 + .../coding-agent/src/core/agent-session.ts | 16 +++ packages/coding-agent/src/core/sdk.ts | 4 +- .../11520-sdk-summary-oauth-auth.test.ts | 122 ++++++++++++++++++ 4 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 packages/coding-agent/test/suite/regressions/11520-sdk-summary-oauth-auth.test.ts diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 6f09e904e..7133939a1 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -15,6 +15,7 @@ ### Fixed +- Fixed compaction, branch summaries, and bug-report summaries in SDK sessions passing the stored OAuth token back to the model runtime as an explicit API key. Provider auth that treats explicit keys differently from stored OAuth now sees the OAuth login, as normal prompts do ([#11520](https://github.com/earendil-works/pi/issues/11520)) - Fixed a redundant system prompt update (tools, rules) when an extension set the same active tools in a different order, for example a reversed `pi.setActiveTools()` list. Tools that stay active now keep their order and newly activated tools are appended ([#11495](https://github.com/earendil-works/pi/issues/11495)) - Fixed `pi.exec()` crashing pi with `RangeError: Invalid string length` when a command printed more output than fits in a string. The default output cap is the smaller of 256 MiB and the runtime string limit; past the cap, the command is terminated and the result has `truncated: true` and a nonzero exit code ([#11360](https://github.com/earendil-works/pi/issues/11360)) - Fixed `pi.exec()` never returning when a command ignored SIGTERM after a timeout or abort. pi now sends SIGKILL 5 seconds later, as intended diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index c1529f019..dbe36c365 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -29,6 +29,7 @@ import { type BeforeToolCallResult, type PrepareNextTurnContext, runToolCall, + type StreamFn, type ThinkingLevel, } from "@earendil-works/pi-agent-core"; import { contentText, getCurrentSystemMessage, getCurrentTools, retryDelayMs } from "@earendil-works/pi-ai"; @@ -158,6 +159,18 @@ import { type VirtualModelStateData, } from "./virtual-models.ts"; +/** + * Stream functions that resolve auth through a given ModelRuntime themselves, such as the one + * createAgentSession installs. Summaries sent through them must not pass pre-resolved credentials: + * the runtime would take them as an explicit API key instead of the stored OAuth login (#11520). + */ +const runtimeStreamFunctions = new WeakMap(); + +/** Mark `streamFn` as resolving auth through `modelRuntime` on its own. */ +export function markRuntimeStreamFunction(streamFn: StreamFn, modelRuntime: ModelRuntime): void { + runtimeStreamFunctions.set(streamFn, modelRuntime); +} + // ============================================================================ // Skill Block Parsing // ============================================================================ @@ -590,6 +603,9 @@ export class AgentSession { if (this.agent.streamFunction === streamSimple) { return { ...(await this._getRequiredRequestAuth(model, signal)), thinkingLevel }; } + if (runtimeStreamFunctions.get(this.agent.streamFunction) === this._modelRuntime) { + return { model, thinkingLevel }; + } try { const result = await this._modelRuntime.getAuth(model, { signal }); diff --git a/packages/coding-agent/src/core/sdk.ts b/packages/coding-agent/src/core/sdk.ts index 0bf874fff..dbab1d1d9 100644 --- a/packages/coding-agent/src/core/sdk.ts +++ b/packages/coding-agent/src/core/sdk.ts @@ -4,7 +4,7 @@ import type { ModelsSimpleStreamOptions } from "@earendil-works/pi-ai"; import { clampThinkingLevel, type Message, type Model, streamSimple } from "@earendil-works/pi-ai/compat"; import { getAgentDir } from "../config.ts"; import { resolvePath } from "../utils/paths.ts"; -import { AgentSession } from "./agent-session.ts"; +import { AgentSession, markRuntimeStreamFunction } from "./agent-session.ts"; import { formatNoModelsAvailableMessage } from "./auth-guidance.ts"; import { CacheWarmer } from "./cache-warmer.ts"; import { getClientIdentity } from "./client-identity.ts"; @@ -436,6 +436,8 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} thinkingBudgets: settingsManager.getThinkingBudgets(), maxRetryDelayMs: settingsManager.getProviderRetrySettings().maxRetryDelayMs, }); + // This stream resolves auth through modelRuntime, so summaries must not pre-resolve it (#11520). + markRuntimeStreamFunction(agent.streamFunction, modelRuntime); // Restore missing settings metadata for older sessions. if (hasExistingSession) { diff --git a/packages/coding-agent/test/suite/regressions/11520-sdk-summary-oauth-auth.test.ts b/packages/coding-agent/test/suite/regressions/11520-sdk-summary-oauth-auth.test.ts new file mode 100644 index 000000000..02639a701 --- /dev/null +++ b/packages/coding-agent/test/suite/regressions/11520-sdk-summary-oauth-auth.test.ts @@ -0,0 +1,122 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + type Api, + createAssistantMessageEventStream, + type Model, + type SimpleStreamOptions, +} from "@earendil-works/pi-ai"; +import { afterEach, describe, expect, it } from "vitest"; +import { AuthStorage } from "../../../src/core/auth-storage.ts"; +import { ModelRuntime } from "../../../src/core/model-runtime.ts"; +import { createAgentSession } from "../../../src/core/sdk.ts"; +import { SessionManager } from "../../../src/core/session-manager.ts"; +import { SettingsManager } from "../../../src/core/settings-manager.ts"; +import { assistantMsg, createTestResourceLoader, userMsg } from "../../utilities.ts"; + +// Issue #11520: summaries in SDK sessions passed the resolved OAuth token back to the +// ModelRuntime as an explicit API key, so provider auth took the API-key override branch. +describe("issue #11520 SDK summary OAuth auth", () => { + const tempDirs: string[] = []; + + afterEach(() => { + while (tempDirs.length > 0) rmSync(tempDirs.pop()!, { recursive: true, force: true }); + }); + + it("lets the runtime resolve stored OAuth for branch summaries", async () => { + const cwd = mkdtempSync(join(tmpdir(), "pi-11520-")); + tempDirs.push(cwd); + const credentials = AuthStorage.inMemory({ + "oauth-only-provider": { + type: "oauth", + access: "oauth-access", + refresh: "refresh", + expires: Date.now() + 3_600_000, + }, + }); + const modelRuntime = await ModelRuntime.create({ credentials, modelsPath: null, allowModelNetwork: false }); + const model: Model = { + id: "oauth-model", + name: "OAuth Model", + api: "openai-completions", + provider: "oauth-only-provider", + baseUrl: "https://example.test/v1", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 4096, + }; + const requests: SimpleStreamOptions[] = []; + modelRuntime.registerProvider(model.provider, { + name: "OAuth Only", + baseUrl: model.baseUrl, + api: model.api, + // With an API key method as well, an explicit key would bypass the stored OAuth login. + apiKey: "UNUSED_ENV_KEY", + oauth: { + name: "Subscription", + login: async () => ({ access: "oauth-access", refresh: "refresh", expires: Date.now() + 60_000 }), + refreshToken: async (credential) => credential, + getApiKey: (credential) => credential.access, + }, + models: [model], + streamSimple: (requestModel, _context, options) => { + requests.push(options ?? {}); + const stream = createAssistantMessageEventStream(); + stream.end({ + role: "assistant", + content: [{ type: "text", text: "branch summary text" }], + api: requestModel.api, + provider: requestModel.provider, + model: requestModel.id, + usage: { + input: 1, + output: 1, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 2, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: Date.now(), + }); + return stream; + }, + }); + const authOverrides: Array = []; + const getAuth = modelRuntime.getAuth.bind(modelRuntime); + modelRuntime.getAuth = (async (...args: Parameters) => { + authOverrides.push(args[1]?.apiKey); + return getAuth(...args); + }) as ModelRuntime["getAuth"]; + + const sessionManager = SessionManager.inMemory(cwd); + const { session } = await createAgentSession({ + cwd, + agentDir: cwd, + model, + modelRuntime, + settingsManager: SettingsManager.inMemory(), + sessionManager, + resourceLoader: createTestResourceLoader(), + }); + try { + const targetId = sessionManager.appendMessage(userMsg("first branch")); + sessionManager.appendMessage(assistantMsg("first reply")); + sessionManager.appendMessage(userMsg("abandoned branch work")); + sessionManager.appendMessage(assistantMsg("abandoned reply")); + + const result = await session.navigateTree(targetId, { summarize: true }); + + expect(result.summaryEntry?.summary).toContain("branch summary text"); + expect(requests).toHaveLength(1); + expect(requests[0].apiKey).toBe("oauth-access"); + expect(authOverrides.length).toBeGreaterThan(0); + expect(authOverrides.every((apiKey) => apiKey === undefined)).toBe(true); + } finally { + session.dispose(); + } + }); +}); From a5609aae81ea44439f5cb695d623c914c2f71bd8 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 29 Oct 2026 08:31:15 +0000 Subject: [PATCH 139/271] docs(chord): decision: no cycle probe; parked_long diagnostic and authoring rules --- packages/chord/docs/next/decisions.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index b05ae7047..85c34607d 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1771,6 +1771,21 @@ validation and copies the v2 kernel added around Delta. setups a registry without a value cannot get one before a reload, so a reader then fails at once `transient` (`unavailable/undeclared`) and is retried when the registry gets its first value (R18), as `self.get` of a type not served here fails at once (refined 2026-10-27 after the second spec review: no `readyTimeout` wait at start); once live, a later failure of the registry changes nothing (last good value). +- **Cycles across hosts: no probe, a diagnostic (owner, 2026-10-29: "yes"; dependency model, decision 6):** cycles stay + found as today: on one host by the walk over waiting edges at `self.get` (exact), across hosts by the activation chain + in holds (objects that start one after another). The remaining gap, a ring across hosts whose objects start at the + same moment, is not detected: under the dependency model, which has no dependency deadline, such a ring stays parked. + It is made visible: an attempt parked longer than its `readyTimeout` on a remote attempt that is itself only + `loading` is reported once per parking episode as `HostError` `parked_long { object, waitingFor, since }`; nothing + fails or retries. A cycle is a plugin bug (owner: "if no plugin bug, we don't need it"); the guide's rules for plugin + authors and agents: (1) a factory declares only what the object needs, never a type that needs it back; (2) + back-references and occasional lookups go into methods and routines (`ctx.get`); (3) data flows up through state and + registries, not mutual `self.get`; (4) host-lifetime objects on the server never declare worker types in their + factory. Rejected: the probe (edge-chasing searches with a check along the recorded trail, `/tmp/model/review*`, + seven review rounds; complete only with a re-search timer or a full search lifecycle, the most intricate code in + Chord for a plugin bug) and the slice 4b timeout narrowed (it polls chains across hosts again, misses a ring of + waiting attempts, and retries in lockstep without finding the ring). Supersedes slice 4b review decision (2)'s + timeout once the dependency model is built (slice 7); folded into design §6 with it. - **Getters wait through `waiting` (owner, 2026-10-29: "(b) with a default seems fine; UI stays responsive anyways because it usually holds instead of get"; dependency model, decision 2):** `host.get`, `ctx.get` and `host.registry` wait while the reference is `loading`, `failed { transient }` or `failed { waiting }`, bounded by their From 39b16b2281d6f76d33b4d96abf100ca452e89480 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 29 Oct 2026 08:31:59 +0000 Subject: [PATCH 140/271] docs(chord): the dependency model is decided --- packages/chord/docs/next/decisions.md | 10 ++ .../chord/docs/next/dependencies/model.md | 161 ++++-------------- 2 files changed, 45 insertions(+), 126 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 85c34607d..73b18c6a1 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1771,6 +1771,16 @@ validation and copies the v2 kernel added around Delta. setups a registry without a value cannot get one before a reload, so a reader then fails at once `transient` (`unavailable/undeclared`) and is retried when the registry gets its first value (R18), as `self.get` of a type not served here fails at once (refined 2026-10-27 after the second spec review: no `readyTimeout` wait at start); once live, a later failure of the registry changes nothing (last good value). +- **The dependency model is decided (owner, 2026-10-29: "just build this"):** `packages/chord/docs/next/dependencies/model.md` + (explainer `book.md`, `index.html`) as decided, with its remaining four points taken as recommended: (1) the barrier + is (a), `load` resolves at the first outcomes of this host's host-lifetime objects (local blockers still in their + first attempt are waited for; waits on other hosts or failed dependencies end the first pass); (3) `waiting` is a new + `FailureCause` kind; (4) a pure swap whose new factory needs a missing dependency parks the object, no overlap of old + and new code; (5) a registry's last good value is kept only while every contribution it was built from is still in + the table. Supersedes R18's dependency deadline and backoff, S2's `broken` and S3's backoff, §7's throw for a missing + local type and getters' rejection, §21's `gone` on removal and its rollout `updating` deadline, the start barrier's + admission gate (holds wait only for the publication). Built as slice 7a (`slice-7a.md`); design.md is folded with + the build. - **Cycles across hosts: no probe, a diagnostic (owner, 2026-10-29: "yes"; dependency model, decision 6):** cycles stay found as today: on one host by the walk over waiting edges at `self.get` (exact), across hosts by the activation chain in holds (objects that start one after another). The remaining gap, a ring across hosts whose objects start at the diff --git a/packages/chord/docs/next/dependencies/model.md b/packages/chord/docs/next/dependencies/model.md index bc3d606c8..5843e7bab 100644 --- a/packages/chord/docs/next/dependencies/model.md +++ b/packages/chord/docs/next/dependencies/model.md @@ -1,4 +1,4 @@ -# Dependency resolution in Chord v2: one model (draft 7, redesign session, 2026-10-29) +# Dependency resolution in Chord v2: one model (decided, 2026-10-29) Status: proposal for the owner, not decided. Built in slice 7 with reload and rollout. Sources: the owner's direction ("park until resolved"; "no difference between local and remote resolution: a link going is like a plugin @@ -178,7 +178,7 @@ through, or recovers by its own backoff. | **a name becomes usable, or an install ends, on a host**: a publication (`load`'s setups, a reload's commit, an install's commit) serves a type, a registry, a version; a registry composes or maps a value at a version; an install ends without a commit (rejected) | the unbound requests indexed under those names; the provider's `installing` holds; a refused hold re-runs its start and sends `loading`, then `bound` or `failed` | No new message kind for wakes: `loading`, `bound`, `failed` and `retry` exist. The wire's `failed` carries the new cause -kind `waiting`; `loading` and a `waiting` `failed` carry the provider's attempt id for the probe (§7). +kind `waiting`. ## 5. Removal, versions, rollout @@ -231,121 +231,33 @@ kind `waiting`; `loading` and a `waiting` `failed` carry the provider's attempt - p2 today resumes its harness without waiting for all host objects (`apps/p2/src/hosts/worker.ts`), because its worker objects that need server types are reachable only after `control.ready()`. -## 7. Cycles +## 7. Cycles (decided 2026-10-29: no probe) -On one host, cycles are found exactly as today: at `self.get`, following the waiting edges of activating and parked -attempts. Across hosts, a hold carries the activation chain it was made in (today), which finds every cycle whose -objects start one after another. What remains is the documented gap (design §6): objects on two hosts that start at -the same moment and need each other. The owner decided on 2026-10-24 (slice 4b review decisions (2)) that this is a -documented limitation that times out and retries, with no probe message. Without dependency deadlines that timeout no -longer exists, so the gap needs one of two answers (**Owner**): +On one host, cycles are found as today: at `self.get`, following the waiting edges of activating and parked attempts +(exact). Across hosts, a hold carries the activation chain it was made in, which finds every ring whose objects start +one after another. The remaining gap, a ring across hosts whose objects start at the same moment, is not detected: +without a dependency deadline it stays parked. It is made visible: an attempt parked longer than its `readyTimeout` on +a remote attempt that is itself only `loading` is reported once per parking episode as `HostError` `parked_long +{ object, waitingFor, since }`; nothing fails or retries. A cycle is a plugin bug; the guide gives plugin authors and +agents four rules: (1) a factory declares only what the object needs, never a type that needs it back; (2) +back-references and occasional lookups go into methods and routines (`ctx.get`); (3) data flows up through state and +registries, not mutual `self.get`; (4) host-lifetime objects on the server never declare worker types in their factory. -- **(A) The 4b decision, narrowed.** One deadline stays: an attempt parked on a dependency held over a link whose target - attempt is known and still activating or parked fails `transient` after its `readyTimeout` and retries with backoff; - its next attempt usually starts after the other side and its chain finds the cycle. It has a cost that decides - against it: the target being *parked* is the common case of a chain across hosts waiting for a missing plugin (a - tab's view on a worker object that waits for an uninstalled type), and those dependents would time out and poll - again, as today, so "local = remote" would not hold across hosts. Narrowing it to targets that are `loading` (not - `failed { waiting }`) avoids that but misses a real ring: A and B on two hosts wait on each other, B also on a - missing X; B is `failed { waiting }`, so A is too; when X appears, both stay `failed { waiting }` on each other, with - no deadline and no chain to find it. And even the timeout does not find what it is meant to (round 6, Fable): an - object's chain is fixed when it is created and its instance id survives retries, so a retried attempt sends the same - hold with the same chain; two simultaneous starts time out and retry in lockstep and are never found. -- **(B) The probe (recommended)**, below: no deadline at all, one new message kind, the most intricate rule set in the - model. Five - review rounds found a counterexample in each of the first four drafts; draft 6's version answers all of them, but it - has not been reviewed again, and slice 7 would have to prove it with tests before the deadline could go. - -### (B) The probe - -Replaces the activation chain in holds. On one host the `self.get` walk stays (it is synchronous and exact: a ring -with no remote edge is found there, where (B1)'s timer does not run; Opus, round 7); the probe serves the rings that -cross a link. It replaces draft 1's bounded chain too -draft 1's bounded chain (a ring of 33 is never found with chains cut at 32; chains keep stale ancestry). Draft 5 replaces the -second round by a check along the recorded path (round 4, Sol: two rounds through different stale paths can report a -ring that never existed, `/tmp/model/sol-probe4.log`). - -1. **Attempts, edges, tokens.** Every attempt has its own id (uuidv7 per attempt: a retry or reload is a new - attempt). A **wait edge** runs from a parked attempt to the parked or activating attempt it waits for, through a - bound request (unbound waits and registry reads are never edges; they are not deadlocks). Each edge carries an **edge - token** (round 5, Astra, Sol): a number from a counter per attempt, never reused, replaced by a fresh one on any - interruption or change of that edge, so two requests to one object are two edges: a new target attempt (also a - local target's retry, without a re-bind), any `loading` on a bound or pending hold (a relay's upstream re-hold - included), an unbinding, a rebinding, a re-hold after a reconnect. An attempt is parked for one interval only (one - that passed its dependency check never parks again) and ids are never reused. A provider tells its consumers the attempt they wait for on - every non-live answer: `loading { hold, attempt }` and a `waiting` `failed { hold, cause, attempt }`. -2. **A search per edge change.** Whenever a parked attempt gets a new edge or a new token on one, it starts a - search: `probe { kind: "search", round, initiator, trail }` along each of its wait edges, `round` the next number of - a counter the attempt keeps, `trail` the list of `(attempt, edge token)` hops it crossed, starting with - the initiator's own. A ring closes at its last - edge, and that edge's source starts the search that finds it. Coalesced per batch. Locally a call on the target - attempt, forwarded through a queue (no recursion); over a link the message `probe { hold, to, ... }`, consumer to - provider, relayed by hold id with the origin. **Probes are honoured on parent and route links and dropped on child - links** (`accept`), whatever `trusted` says. -3. **Forwarding.** An attempt receiving a search drops it if `to` is not its current attempt or it is not parked. If - the initiator is itself, the search returned (4). If the attempt is already in the trail, it drops it (a ring that does not contain the initiator is - found by its own members' searches; Opus). Otherwise it forwards it along every wait edge, appending `(itself, the edge's - token)` to the trail, only if the round number is higher than the one it stored for that - initiator (older or equal rounds are dropped: the initiator superseded them), and stores it. The seen map holds one - number per initiator, up to 256 initiators, and goes with the attempt. -4. **The check.** A returned search is accepted while its initiating attempt is still parked, whatever round it - carries (round 7, Astra: a re-search must not invalidate an older search still under way); the check decides. The - initiator then sends `probe { kind: "check", round, trail }` along the recorded trail: the initiator first verifies its own hop; each hop passes it on only if it is still - that attempt, still parked, still has the recorded edge token, pointing at the next hop's attempt; otherwise the - check dies (also on a `to` mismatch). The check follows the trail by position and bypasses the seen - map. The initiator verifies its own hop again when the check returns. If the check returns, the initiator fails `broken invalid/cycle`, the path named for display from the trail, - and today's propagation breaks the other activating objects on the ring and the dependents off it. 5. **No silent loss.** Dedupe can drop the copy of a search that travelled the ring in favour of a copy whose path - later breaks (Astra, round 5); then the check dies and nothing is left to find the ring. Round 6 tried `dead` - notices back to the initiator; Sol and Astra showed that reliable notices need recovery state per search that - eviction may not drop and that a long-parked attempt would otherwise keep for every initiator that ever passed (a - leak), i.e. a full search lifecycle with cancellation. Two ways (**Owner**, part of decision 6): - - **(B1) Re-search with backoff (recommended).** An attempt parked on a remote attempt starts a fresh search again - after a backoff (1 s doubling to 60 s, reset by an edge change), whether it shows `loading` or `failed { waiting }`; - the timer stops when the attempt leaves parking. No `dead` messages, no recovery state, eviction - only costs work; a new search starts within one backoff period, so every stable ring that crosses a link is - eventually found (no wall-clock bound: delivery is only fair; a ring on one host is the walk's). The timer runs as long as the attempt stays parked on a remote attempt: a view on a worker object that - waits for a missing plugin re-searches every 60 s until a reload (Fable: cheap, but for as long as it waits). The timer sends probes only: it never - fails an object, re-holds or polls a provider. Cost: one probe per parked remote dependent per period, which - crosses the downstream wait graph and dies at its leaves (an unbound wait is not an edge). - - **(B2) Events only.** Each search is owned until it settles: branches report completion or abort back along their - trails, a dropped probe answers `dead { initiator, round }` with a reverse route, an edge records the searches it - carries until they settle, and an initiator that ends or starts a new round cancels its old one along the same - routes. Complete without a timer, bounded by the searches in flight, and the most intricate protocol in Chord; not - yet specified to the reviewers' satisfaction. - Rejected: enumerating every simple trail without dedupe (Sol's correctness reference): no ring copy is ever - suppressed, but work is exponential on chains of diamonds. -- **Why a reported ring existed:** every edge of the trail kept its token from the moment the search crossed it - to the moment the check crossed it; the search crossed all of them before the check started, so at the moment the - check started every edge of the ring existed at once. A token changes on every interruption and is never reused, so an edge cannot - vanish and come back unnoticed. Sol's counterexample fails at the check: `C1` is no longer `B`'s target. -- **Why every ring is found** (under an eventually stable wait graph and fair delivery; endless edge churn can keep - invalidating searches; with rule 5's (B1) or (B2) for a search suppressed behind a path that broke): a ring is closed by its last edge; its source starts a search when the edge appears; every - other edge exists then and every attempt on the ring waits transitively on the source, so none goes live before the - search and the check return. Rings of any length, local and remote mixed. On a DAG searches die at the leaves; on a - diamond the second arrival is dropped. -- **What it does not promise:** (1) a ring that existed when the check started may be broken up by a concurrent reload, - retry, unbinding or revocation just after; it is still reported (it did exist). Its broken objects recover as every - broken object does: `ref.retry()` or a reload of their plugin. (2) Linear work under every schedule: with more than - 256 initiators passing one attempt at once, an evicted entry lets a search be forwarded again, so work can grow with - the number of simple paths (never endlessly: a search never revisits an attempt in its trail). (3) Constant message size: a probe carries its trail, so its size grows with the length of the - path it crossed (a returning search's trail is as long as its ring); no cap cuts it, since the trail is the proof. -- **Cost:** one search per edge change, crossing each downstream edge once, one check per returned search; with (B1), - one more search per backoff period for every attempt parked on a remote attempt, also while an acyclic chain waits - for a missing plugin, plus the trail bytes. The bounded memory is the seen map's; probes in flight are bounded by - the searches started. -- **Transition:** today's dependency deadline for activations waiting across links stays until the probe's tests pass - in slice 7 (long rings, diamonds, a ring closed after a reconnect with unchanged attempt ids, Sol's phantom schedule, - 300 concurrent initiators, edges added and removed, reloads, relays, forged probes); then it goes in the same slice. +Rejected (owner, 2026-10-29): the probe (edge-chasing searches with a check along the recorded trail; seven review +rounds, `/tmp/model/review*-*.md`; complete only with a re-search timer or a full search lifecycle, the most intricate +code in Chord, for a plugin bug) and the slice 4b timeout narrowed (it polls chains across hosts again, misses a ring of +`waiting` attempts, and retries in lockstep without finding the ring). ## 8. Getters, holds, calls: one rule - **`host.hold`** returns a reference at once; only invalid params throw (also before `load`). A missing local type gives `failed { waiting }` after the publication, `loading` before it. -- **`host.get`, `ctx.get` and `host.registry`:** resolve when live; wait while `loading`, or `failed { transient }` - (its own retry is under way), up to their `readyTimeout`; reject at once on `failed { waiting }` or `{ broken }`. On +- **`host.get`, `ctx.get` and `host.registry`** (owner, 2026-10-29, decision 2): resolve when live; wait while `loading`, + `failed { transient }` or `failed { waiting }`, up to their `readyTimeout` (default 60 s, the caller's choice), then + reject with the last cause; reject at once only on `failed { broken }`. On success, rejection or timeout their temporary wait goes; `host.registry` returns the `RegistryValue` with its - ordinary watch ownership (no `release()`). **Owner:** recommended (nobody waits 60 s for a plugin that is not - there; code that wants to wait holds instead), or wait through `waiting` too. + ordinary watch ownership (no `release()`). A UI holds and shows `ref.state`; code that needs the object now gets it, with the + patience it passes. - **Calls** on a `loading` reference park, each bounded by the reference's `readyTimeout` from when it was made; on a `failed` one they reject at once with the cause; sent calls are never resent. @@ -384,8 +296,8 @@ ring that never existed, `/tmp/model/sol-probe4.log`). | `readyTimeout` of a parked call | kept | an action must not run long after it was asked for | | identity backoff 1-60 s | only for the object's own `transient` failures | nothing else says the disk or network is back | | dependency backoff, polling of refused holds | removed | recovery is an event | -| dependency deadline across links | (A): only for a parked attempt whose remote target attempt stays activating or parked; (B): until the probe's tests pass, then removed | the simultaneous-cycle gap | -| probe re-search backoff (B1) | new, only with (B1): 1 s doubling to 60 s while parked on a remote attempt | finds a ring whose search was suppressed; sends probes only, never fails or re-holds | +| dependency deadline across links | removed | the simultaneous-cycle gap is a plugin bug, reported by `parked_long` | +| `parked_long` report | new: once per parking episode after `readyTimeout` on a remote attempt that is only `loading` | a diagnostic; nothing fails or retries | | route backoff, heartbeat, `closeTimeout`, routine and `finish` backoffs, `linger` | kept | not dependency waits | | rollout install timeout (not built) | a watchdog on the install only | never a deadline for dependencies | @@ -394,22 +306,19 @@ ring that never existed, `/tmp/model/sol-probe4.log`). R18 (dependency failures `waiting`, no backoff, the deadline paused while parked, the dependency check as boundary, wakes on every change, an ended dependency breaks), S2 and S3 (`waiting`, woken at every usable value), slice 8's last-good rule (dropped when a contribution leaves), §3's destination table (by address; the publication gate), §6 (the -simultaneous-cycle gap: (A) one narrowed deadline or (B) the probe), §7 (`host.hold`, getters, calls), §21 (no `gone` on +simultaneous-cycle gap: no probe, `parked_long`), §7 (`host.hold`, getters, calls), §21 (no `gone` on removal; a dropped version unbinds only its requests; installs never await activations; rollout without revision -fields; the reload handover), the start barrier (R17, W6), `FailureCause` (+ `waiting`), `loading`/`failed` with the -attempt id, one message kind (`probe`, tagged `search` or `check`). Unchanged: what an object's own code reload does (§21), R18 after live, +fields; the reload handover), the start barrier (R17, W6), `FailureCause` (+ `waiting`), `parked_long`. Unchanged: what an object's own code reload does (§21), R18 after live, `retryOn` outside parking, `self.follow`'s R4, the call window and flow control, revocation, no client providers. -## 13. Open for the owner +## 13. Decisions (owner, 2026-10-29) -1. The barrier: (a) or (b) (§6.3). -2. Getters on `waiting`: reject at once (recommended) or wait up to `readyTimeout` (§8). -3. `waiting` as a new cause kind (recommended) or not (§2). -4. A pure swap whose new factory declares a dependency that is not there: the object parks (its gate closes, holders - see `loading` or `failed { waiting }`, calls park or reject; recommended) or stays on the old code until it - resolves (old and new code would overlap without bound, against §21's "a host is at R or R−1"). -5. **The last good value** (§9): kept only while every contribution it was built from is still in the table (draft; - Astra), or kept as slice 8 decided, with natives in a registry looked up at use through their owner (Sol). -6. **Cycles across hosts** (§7): (B) the probe, with (B1) a re-search backoff for probes only (recommended) or (B2) a - full search lifecycle without any timer; or (A) the slice 4b decision narrowed to one deadline, which polls chains - across hosts again, misses a ring of `waiting` attempts, and retries in lockstep without finding the cycle it is for. +1. The barrier: (a), `load` resolves at the first outcomes of this host's host-lifetime objects (§6.3). +2. Getters wait through `waiting` until their `readyTimeout`; calls on `failed` still reject (§8). +3. `waiting` is a new `FailureCause` kind (§2). +4. A pure swap whose new factory declares a dependency that is not there parks the object (its gate closes; holders + see `loading` or `failed { waiting }`; calls park or reject); no overlap of old and new code (§21). +5. The last good registry value is kept only while every contribution it was built from is still in the table (§9). +6. Cycles across hosts: no probe; `parked_long` and four authoring rules (§7). + +1, 3, 4 and 5 are the recommendations, taken when the owner said "just build this" (2, 6 decided in discussion). From babe30a7e6c185346c85859da30b58183e5964f0 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 29 Oct 2026 08:32:56 +0000 Subject: [PATCH 141/271] docs(chord): slice 7a spec: the dependency model; slice 7 split into 7a and 7b --- packages/chord/docs/next/slice-7a.md | 130 +++++++++++++++++++++++++++ packages/chord/docs/next/slices.md | 5 +- 2 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 packages/chord/docs/next/slice-7a.md diff --git a/packages/chord/docs/next/slice-7a.md b/packages/chord/docs/next/slice-7a.md new file mode 100644 index 000000000..a5612ce33 --- /dev/null +++ b/packages/chord/docs/next/slice-7a.md @@ -0,0 +1,130 @@ +# Slice 7a: the dependency model (spec) + +Status: **ready to build** (owner, 2026-10-29: the model decided, "just build this"). Slice 7 is split: 7a builds the +dependency model on what exists (one `load` per host, links, routes, relays); 7b builds reload and rollout on top of it +(removal, dropped versions, installs, own-code reloads, loader D). + +Read in full before writing code: `dependencies/model.md` (the model as decided; its section numbers are cited below as +M§), `design.md`, `decisions.md` (the entries of 2026-10-28/29: "No client providers", "Every remote type declares its +placement", "`hostAddress` instead of `placement`", "Only hosts with an address serve remote types", "Getters wait +through `waiting`", "Cycles across hosts: no probe, a diagnostic", "The dependency model is decided"), `slices.md` +(slice 7's carry-over: 7a takes none of it unless named here; 7b takes the rest), the explainer +`dependencies/book.md` for the scenarios (its catalogs are test material). + +## 1. Requests and the unbound index (M§1) + +- Every reference is a request with one holder (attempt, call, outside code, a link's peer). It ends only when its + holder ends it; never because its provider is missing or does not fit. +- The destination by address is built (`host/destination.ts`, slice 7a's predecessor commit `5ec241882`). Add: version + checks only against the selected provider (no `#assertRoute` comparison with a same-name registration on the + consumer's host); a local type not served here is no longer a throw at `self.get`/`host.hold`/getters but an unbound + request. +- **The publication gate:** a request made on this host, or a hold arriving on its links, before the host's first + publication is not placed: `loading`, placed by the publication. `pendingLoad` no longer gates `Hold.start` or + `#hold` beyond the publication (it ends the startup deadlock: Sol's probe `/tmp/dep-graph-sol-barrier.log` becomes a + regression test that passes with fake timers never advanced). +- **Unbound reasons** (M§1's table): `unpublished` (loading), `not_served` and `version` (`failed { waiting }`), + `compose` for registries (`failed { waiting }`), `no_link` on a path (loading); `authorize` refused or invalid params + stay `broken`, never indexed. Indexes per host by name (local requests; refused holds that arrived over links, kept as + today's refused state in `remote/hold.ts`); entries removed when the request ends (release, call settled, getter gave + up, `closed { origin }`, link close, revocation). GC tests. + +## 2. States (M§2) + +- `FailureCause` gains `{ kind: "waiting"; error }` (no `retryAt`). The wire's `failed` cause schema carries it; the + pinned list and its test change; `PROTOCOL_VERSION` stays 1. +- The mapping of M§2's table: `loading` for activating, unpublished, no link (and `installing`, 7b); `failed + { waiting }` for not served, version, compose, and a dependent whose dependency failed. +- A remote refusal for not served / version is `failed { waiting }` instead of `broken`; the provider keeps the refused + hold (it does today) and the consumer never retries it by itself (no backoff). + +## 3. Parked attempts (M§3) + +- Order: factory, persisted singleton hydration (as today), requests made; the attempt is **parked** while any + dependency (`self.get` local or remote, `self.registry`, keyed pins) is not live and hydrated. `self.follow` is not a + dependency (its attach is an own step). +- What it shows: `loading` while blockers are only `loading`; `failed { waiting, "waiting for X: " }` (nearest + blocker plus root cause, 500-character rule) when one failed; `broken` when one failed with `invalid/cycle` (off-ring + dependents) or ended. +- **No timer while parked:** no dependency deadline, no identity backoff for dependency failures; the attempt's + `readyTimeout` deadline is paused while parked (as eager migrations pause it) and resumes with the budget left. The + identity backoff stays for the object's own `transient` failures only. +- **The dependency check is the boundary:** once all are live, the own steps run (natives, follow attaches, member + checks, derives) under the deadline; a dependency leaving `live` afterwards is treated as after live (R18): the + attempt goes on. +- **Wakes on every state change** of a dependency (live, loading, failed, a new cause compared by value, ended): + re-evaluate what the object shows, leave parking when all are live; never re-run the factory while parked; coalesce + per object after the batch that caused them; remember changes during own steps or teardown. +- **Retry passes on:** `ref.retry()` on a `failed { waiting }` object retries its `broken` and `transient` blockers hop + by hop (the existing `retry` message), once per operation (an id dedupes diamonds); `waiting` blockers pass it on. + A `retryOn` value call while parked `waiting` does nothing. + +## 4. Wakes (M§4) + +Two events: a target changes state (the requests bound to it and their dependents); a name becomes usable on a host +(in 7a: `load`'s publication serving types and registries, a registry's value turning usable at a version): the +unbound requests indexed under it, and refused holds, which re-run their start and send `loading`, then `bound` or +`failed`. No new message kind. + +## 5. Startup (M§6) + +- `load` runs the setups, publishes them in one batch (types, registries served and composed once, contributions), + wakes after that batch, then activates host-lifetime objects. +- **The barrier, decision (a):** `load` resolves when every host-lifetime object of the load has a first outcome: + live, failed by its own cause, or parked with no blocker that is a local object still without its first outcome + (`local` = executing on this host, keyed pins still loading here included). One `firstOutcome` signal per object, + separate from the readiness dependents wait for; counted once; an object of the cohort that ends before its first + outcome settles as ended; `host.close` settles a waiting `load`. + +## 6. Getters, holds, calls (M§8, decision "Getters wait through `waiting`") + +- `host.hold` returns a reference at once; only invalid params throw (also before `load`); a missing local type gives + `loading` before the publication, `failed { waiting }` after it, before and after `load` alike. +- `host.get`, `ctx.get`, `host.registry`: resolve when live; wait through `loading`, `failed { transient }` and + `failed { waiting }` until their `readyTimeout`, then reject with the last cause and release; reject at once only on + `failed { broken }`. +- Calls: park on `loading` (bounded, as today); reject at once on `failed`; never resent. + +## 7. Registries (M§9) + +A reader's request: bound while R is served and has a usable value at its version; otherwise unbound `not_served`, +`version` (S2's newer or unmapped reader: `waiting`, not `broken`) or `compose` (first compose or the version mapping +threw), `loading` before the publication. Woken at every transition to usable (`#valued`'s latch goes). The last good +value is kept only while every contribution it was built from is still in the table (decision 5): when one leaves, +the last good value is dropped and `.value` throws until `compose` succeeds. + +## 8. Keep-alive, revocation, cycles (M§7, M§10) + +- A request over a link counts once in `host.activity.references` from arrival to end, bound or unbound; + `host.references` lists bound references only. +- Revocation ends the requests of a revoked principal, bound and unbound, at every hop. +- `gone` only for revocation (a closing provider host sends nothing, as today). +- **Cycles:** the walk at `self.get` on one host, through parked attempts too; the activation chains across hosts as + today. Remove the dependency deadline (no fallback). Add `HostError` `parked_long { object, waitingFor, since }`: + once per parking episode, when an attempt has been parked longer than its `readyTimeout` on a remote attempt that is + itself only `loading`. The guide (host README) states the four authoring rules of M§7. + +## 9. Not in 7a (7b) + +Removal and re-serving of types (local re-binding of a `Binding`, a `failed` on a bound hold unbinding it, window +slots), a dropped accepted version, `installing` and install ends, own-code reloads and the parked-attempt handover, +pure swaps parking (decision 4), rollout, loader D. 7a's code must leave those paths possible, not build them. + +## 10. Tests and cost + +- One `describe` per mechanism; the four catalogs in `dependencies/book.md` (chapters XIV-XVII) are the scenario list: + every row that 7a's scope reaches becomes a test (memory links; relays; fake timers where a test proves no timer + runs). Must include: the startup deadlock without `readyTimeout`; a hold before the publication placed here, not via + the parent; a missing local type on `self.get`, `host.hold`, `host.get` (waits, then rejects with the cause); a + remote refusal kept and re-run when the provider's late `load` publishes; a parked three-level chain across hosts + woken in order; wakes on every state change with causes updated; retry passing on through a diamond once; the + barrier's first-outcome rule (CodeMode → Harness waits; a remote blocker ends the pass); a cohort object ending + before its outcome; registries' waiting reader, wake after a provider arrives, last good dropped when a contribution + leaves; keep-alive counting unbound holds; revocation of unbound holds; `parked_long` reported once; GC tests for the + indexes and a parked chain released from the top. +- Benchmarks (the bar of "The performance and memory bar"): calls, activation churn, a watched derive, links against + HEAD, pinned, alternating, a HEAD copy at a path of the same length; results in `benchmarks.md`. +- Examples: `31-parked-dependencies.ts` (a missing provider arrives; causes shown; no timers), `32-startup.ts` (two + hosts that would have deadlocked). +- Docs: fold the model into `design.md` (§3, §6, §7, §17, §21's 7a parts) in the same change as the build; mark + `dependencies/model.md` as built for 7a. diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index 2ddb88f60..97883b776 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -50,7 +50,8 @@ reload has to reload real registries). | 6a | **The store** | the store (backends, tables, queries, writes, commit queue, outbox, documents engine, migrations, export), persisted documents, `self.table`, keyed watches and retirement | | 6b | **Durable and jobs on the store** | persisted jobs, Durable's `StoreStorage`, the Chord source over Durable's views (`self.follow`) | | 8 | **Registries and `host.root`** | registries, `self.writable`, `retryOn`, Durable's `registryReader`, `host.root` (`chord(fn, context)` and the harness object are p3's base code) | -| 7 | **Reload and rollout** | plugin loading, staging, swaps and restarts, revisions, `accepts` in use | +| 7a | **The dependency model** | requests that outlive providers, parked attempts without timers, wakes, the publication gate, getters, registries as dependencies, `parked_long` (`slice-7a.md`, `dependencies/model.md`) | +| 7b | **Reload and rollout** | plugin loading (loader D), staging, swaps and restarts, removal and re-serving, revisions, `accepts` in use | | 9 | **Telemetry** | spans, events, metrics across hosts | The playground (`apps/chord-playground`) moves to the new code once slice 4 exists. Round-3 cuts (C3-C16) and @@ -162,7 +163,7 @@ deleted (the slice's spec and code are the record). Moved here from `carry-over. - **`map(bytes())` is refused at declaration** ("unsupported keyword `~refine`"): TypeBox's `Record` copies the bytes schema's check as an enumerable key, which the snapshot walk refuses. An L1 bug found in slice 3a, not fixed yet. -### Slice 7: reload and rollout +### Slice 7b: reload and rollout (7a: the dependency model, `slice-7a.md`) - **A native restart drains first** (owner, 2026-10-26): park new calls, await running calls (bounded by `closeTimeout`), then close the natives and restart; a call outliving the drain is cut off by the seal fence. Running From 339847eb14995b738fced67f0b80649d2ad0f52a Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 29 Oct 2026 11:40:57 +0000 Subject: [PATCH 142/271] feat(chord): Chord v2 slice 7a, the dependency model A reference is a request that outlives its provider: a missing provider or a version that does not fit leaves it unbound, waiting for the name (failed { waiting }, a new FailureCause kind), local and remote alike. An attempt parks until its dependencies are live, with no timer and no backoff, woken on every state change of a dependency; its readyTimeout runs only for its own steps. Requests wait only for the host's publication (the startup deadlock is gone); load resolves at the first outcomes of its host-lifetime objects. Getters wait through waiting until their readyTimeout. Registry readers are waiting dependencies, woken at every usable value; a last good value is dropped when a contribution it was built from leaves. Unbound holds count for keep-alive and are revoked; retry passes on to failed blockers once per operation (operation id and visited list on the wire); the dependency deadline is gone and parked_long reports a long wait on a remote attempt. --- packages/chord/docs/next/benchmarks.md | 40 ++ .../chord/docs/next/dependencies/model.md | 10 +- packages/chord/docs/next/design.md | 258 +++++++---- packages/chord/docs/next/slices.md | 10 + packages/chord/src/next/data/failure.ts | 70 ++- packages/chord/src/next/data/internal.ts | 2 +- packages/chord/src/next/host/README.md | 197 ++++++--- packages/chord/src/next/host/attempt.ts | 285 ++++++++---- packages/chord/src/next/host/backoff.ts | 2 +- packages/chord/src/next/host/dependencies.ts | 142 ++++++ packages/chord/src/next/host/destination.ts | 53 +-- packages/chord/src/next/host/host.ts | 351 +++++++++------ packages/chord/src/next/host/internal.ts | 7 +- packages/chord/src/next/host/object.ts | 274 +++++++----- packages/chord/src/next/host/ref-state.ts | 12 +- packages/chord/src/next/host/reference.ts | 159 +++++-- packages/chord/src/next/host/route.ts | 5 + packages/chord/src/next/host/unbound.ts | 62 +++ packages/chord/src/next/link/binary.ts | 35 +- packages/chord/src/next/link/message.ts | 22 +- packages/chord/src/next/link/text.ts | 22 +- packages/chord/src/next/registries/README.md | 32 +- packages/chord/src/next/registries/record.ts | 43 +- packages/chord/src/next/registries/value.ts | 105 ++++- packages/chord/src/next/remote/README.md | 36 +- packages/chord/src/next/remote/consumer.ts | 8 +- packages/chord/src/next/remote/hold.ts | 181 +++++--- packages/chord/src/next/remote/host-link.ts | 6 +- packages/chord/src/next/remote/provider.ts | 18 +- packages/chord/src/next/remote/relay.ts | 6 +- .../chord/src/next/remote/remote-object.ts | 60 ++- packages/chord/test/next/data/failure.test.ts | 66 ++- .../chord/test/next/examples/05-errors.ts | 5 +- .../chord/test/next/examples/07-failures.ts | 3 +- .../next/examples/31-parked-dependencies.ts | 148 +++++++ .../chord/test/next/examples/32-startup.ts | 104 +++++ .../chord/test/next/host/activity.test.ts | 13 + packages/chord/test/next/host/attempt.test.ts | 97 +++- .../chord/test/next/host/dependencies.test.ts | 294 +++++++++++++ packages/chord/test/next/host/host.test.ts | 413 +++++++++++++++++- .../test/next/host/keyed-members.test.ts | 68 ++- packages/chord/test/next/host/object.test.ts | 180 ++++---- .../chord/test/next/host/reference.test.ts | 4 +- .../chord/test/next/host/retry-on.test.ts | 21 + .../chord/test/next/host/revocation.test.ts | 140 +++++- .../chord/test/next/host/routines.test.ts | 3 +- packages/chord/test/next/host/unbound.test.ts | 171 ++++++++ packages/chord/test/next/link/message.test.ts | 41 +- .../chord/test/next/registries/record.test.ts | 29 +- .../chord/test/next/registries/value.test.ts | 173 +++++++- packages/chord/test/next/remote/hold.test.ts | 171 +++++++- .../chord/test/next/remote/provider.test.ts | 16 +- .../test/next/remote/remote-object.test.ts | 94 ++-- .../test/next/store/table-migration.test.ts | 27 +- 54 files changed, 3832 insertions(+), 962 deletions(-) create mode 100644 packages/chord/src/next/host/dependencies.ts create mode 100644 packages/chord/src/next/host/unbound.ts create mode 100644 packages/chord/test/next/examples/31-parked-dependencies.ts create mode 100644 packages/chord/test/next/examples/32-startup.ts create mode 100644 packages/chord/test/next/host/dependencies.test.ts create mode 100644 packages/chord/test/next/host/unbound.test.ts diff --git a/packages/chord/docs/next/benchmarks.md b/packages/chord/docs/next/benchmarks.md index 916c70838..c72316e7e 100644 --- a/packages/chord/docs/next/benchmarks.md +++ b/packages/chord/docs/next/benchmarks.md @@ -633,3 +633,43 @@ p95, and swapping host files moves it without a pattern (code layout, not work o `writable.set` 0.87 µs, the same value 0.011-0.016 µs and 0 B (the seal check now comes first: a set of an ended attempt's writable throws whatever its value); one of 10 contributions set 0.71-0.78 µs, of 100 1.43-1.61 µs (one run in six 3.4 µs on either variant tried: JIT). + +## Slice 7a: the dependency model (2026-10-29) + +Node 24.4.1, one CPU (`taskset -c 6`), runs alternating. HEAD is `babe30a7e`, extracted at a path of the same length +(`/home/badlogic/pi-h7`); the build in the worktree (`/home/badlogic/pi-7a`). + +**Paths under every object** (`test/next/bench/calls.ts`), median per operation over nine bursts, heap allocated per +operation, three alternating runs each, HEAD → build: + +| Case | HEAD | build | +| --- | --- | --- | +| local method call (its ctx) | 2.81-2.82 µs, 6,894 B | 2.80-2.82 µs, 6,894 B | +| commit through a watched derive | 10.43-10.70 µs, 15,310 B | 10.38-10.53 µs, 15,310 B | +| activation churn: get, live, release (an object with params and one dependency) | 58.5-59.9 µs, 43,716-43,719 B | 53.7-55.4 µs, 41,540-41,559 B | +| retained per live object with one dependency (10,000 kept, an ad-hoc probe) | 7,398-7,406 B | 6,931-6,936 B | + +An activation whose dependencies are live at the check does not park: one check of its bindings, then its own steps a +microtask later (as HEAD's wait did). The first build parked every attempt with a dependency (a waiter list, a +`parked_long` record, a microtask per wake even outside parking) and cost 62.0-63.6 µs and 580 B more per activation; +now a wake outside parking is a flag, the first-outcome promise is made only for whoever waits (the start barrier), +and an object keeps no `Set` of waits and no settlement promise per attempt (R18's), hence the lower numbers. + +**Links** (`links.ts`, three runs each): wire sizes, encodes, decodes, heap per message and retained heap per hold +(176.0 → 175.4-176.0 KiB) unchanged. Hold to live over two hops (mean of 100), binary: 1,000-item list 0.781-0.817 → +0.807-0.811 ms, 500 KB document 1.250-1.278 → 1.289-1.310 ms (+1-3%: the hold counted at its arrival, the index +check of the publication); text 1.545-1.580 → 1.521-1.573 and 3.299-3.379 → 3.317-3.393 ms. Commits to frames, +streamed appends, call round trips and reconnects within the runs' spread. + +**Existing benchmarks**, HEAD → build: the derive diamond (`derives.ts`, three runs) 9.4-10.1 → 9.5-9.6 µs median, +p95 19.3-19.9 → 19.3-19.4 µs. Registries (`registries.ts`, three runs): `writable.set` 0.82-0.97 → 0.77-0.87 µs; one of +10 contributions set 0.72-2.71 → 0.70-1.06 µs; of 100 1.50-3.46 → 1.44-3.40 µs (one slow run per side: JIT); +allocation within 20 B. A registry's last good composition keeps the list of contributions it was built from: the +list is replaced (not spliced) when a contributor comes or goes, so a composition costs no copy. + +**Fix round 1** (2026-10-29: revocation over paths, retry operations on the wire, the provider's own host-address +check for same-host remote requests, mapped values' provenance), `calls.ts`, three alternating runs, HEAD → build: +local call 2.81 → 2.79-2.82 µs (6,894 B both), watched derive commit 10.45-10.70 → 10.35-10.47 µs, activation churn +58.6-59.8 → 53.9-54.2 µs (43,716-43,720 → 41,559 B); retained per live object with one dependency 7,399-7,405 → +6,930-6,938 B. The address check runs only for a remote type whose consumer and provider `hostAddress` differ; a +retry's operation id is made per `ref.retry()`, and an object keeps its handled ids only once it was retried. diff --git a/packages/chord/docs/next/dependencies/model.md b/packages/chord/docs/next/dependencies/model.md index 5843e7bab..367c0612c 100644 --- a/packages/chord/docs/next/dependencies/model.md +++ b/packages/chord/docs/next/dependencies/model.md @@ -1,6 +1,9 @@ # Dependency resolution in Chord v2: one model (decided, 2026-10-29) -Status: proposal for the owner, not decided. Built in slice 7 with reload and rollout. Sources: the owner's direction +Status: decided (owner, 2026-10-29); **built for slice 7a** (§1-4 and §6-11 without what installs and reloads add: +§1's `installing`, §4's install-end and reload wakes, §3.9's reload handover and §5's removal, versions and rollout +are slice 7b's; folded into `design.md` §3, §5, §6, §7, §17, §21's start barrier, §22; `slice-7a.md`). The draft +history below is kept as it was. Sources: the owner's direction ("park until resolved"; "no difference between local and remote resolution: a link going is like a plugin unloading"), the reviews (`/tmp/dep-mechanism-*.md`, `/tmp/dep-graph-*.md`, `/tmp/barrier-*.md`, `/tmp/model/review{1,2}-*.md`), the code at pi `358f7d46a`. **Owner** marks an open decision. @@ -138,6 +141,7 @@ through, or recovers by its own backoff. | any `loading`, none failed | `loading` | | any `failed` | `failed { waiting, "waiting for X: " }`, `retryAt` undefined; the message keeps the nearest blocker and the root cause within the 500-character rule | | one failed with `invalid/cycle` (a dependent off the ring) | `broken` with the same error: it would wait on a ring that never heals by itself | + | one failed with `forbidden` (an `authorize` refused it: authority, §3.6) | `broken` with the same error: only `retry()` or a reload helps (slice 7a fix round 1) | | one `ended` (revoked, or its host closed) | `broken` with that error: the request cannot come back; `retry()` or a reload makes new requests (a guard: a dependency ends only with its holder's principal or host, so the dependent ends too; Fable) | 2. **The dependency check is the activation boundary:** once every dependency is `live`, the attempt runs its own @@ -152,7 +156,9 @@ through, or recovers by its own backoff. failed, a new cause compared by value, ended; Opus: otherwise a ring of waiting attempts passes its causes round and round), updating what it shows; it leaves parking when all are live. The factory never re-runs while parked. Wakes are coalesced per object and run after the batch that caused them; a change during the - attempt's own steps or its teardown is remembered. + attempt's own steps or its teardown is remembered. (As built in 7a, with the parent's note 5: a change while the + parked attempt checks or loads its pinned keys is remembered and checked again; after the dependency check, during + the own steps, a change is ignored, §3.2's R18 rule.) 5. **Kept while parked:** its documents (the object's), its declarations and requests, its seal token; no natives. Ended with the object (its last holder; a host-lifetime object when its plugin is removed or the host closes). 6. **`broken` only for the object's own code** (a throwing factory, a derive throwing at activation, hydration, diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index 66a76765f..3f07b01cc 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -124,23 +124,50 @@ export const Catalog = remote("catalog", { hostAddress: () => Server(), … }); - **`params`** (owner, 2026-10-21; was `input`) select the instance among objects of its type and are part of its identity (section 5). - **The caller computes the address** and sends it in the `hold`; a host that relays routes by that address alone and - never needs the contract or parses params. **The provider computes it again** from the params and refuses an object - that is not its own: broken `invalid/host_address` (misrouted holds are never forwarded). During a rollout the refusal - is handled like a mismatch (provider on another revision: park; same revision: fail; slice 7). -- **Where a reference made on a host, or a hold arriving on one of its links, goes** (first match; slice 4a): + never needs the contract or parses params. **The provider computes it again** from the params: a hold naming this + host whose params the provider's code places elsewhere (code skew) waits unbound `version` (`failed { waiting }`, + `invalid/host_address`, dependency model §1; slice 7a); a hold naming another host's address is relayed, never served + here (misrouted holds are never captured by a relay that serves the same name). +- **Every reference is a request** (dependency model, slice 7a): it has one holder (an attempt, a call, outside code, + a link's peer) and ends only when its holder ends it (release, a call settling, a getter giving up, the peer's link + or origin closing, revocation, the host closing), never because its provider is missing, went away or does not fit. +- **Where a reference made on a host, or a hold arriving on one of its links, goes**, by address alone (slice 4a; + slice 7a: no same-name check on the consumer's host): | Case | Goes | | --- | --- | - | the type is served here and its host address (computed here from the params) is this host's address | here | - | a reference made here (`host.get`/`hold`, `self.get`, `ctx.get`) to a served type placed on another host | on, as if not served here (next rows): worker S1 reaches `Notes { session: "S2" }` through the parent | - | an arriving hold for a served type whose params place it on another host | broken `invalid/host_address`; the address the hold names only routes it | - | the hold names an address of a kind this host has a route for | over that route's link (the address's params checked against the kind, then `authorize`) | + | a `local` type | here | + | a `remote` type whose host address (computed here from the params) is this host's address, served here or not | here | + | an arriving hold naming this host's address | here | + | the address (computed, or named by the hold) is of a kind this host has a route for | over that route's link (the address's params checked against the kind, then `authorize`) | | otherwise (a remote type) | over the parent link; a hold that arrived *on* the parent link is never sent back to it: `unavailable` | | no parent link: the host never had one | `unavailable` | | no parent link: it is down (a drop) | waits on the parent path, `loading`, until the next link (design audit, 2026-10-27) | - A `hostAddress` or `authorize` that throws fails the hold broken with that error; an address whose params fail the - kind's check is broken `invalid/host_address`. + A `hostAddress` or `authorize` that throws fails the request broken with that error; an address whose params fail + the kind's check is broken `invalid/host_address`. +- **The publication gate** (slice 7a): a request made on this host, or a hold arriving on its links, before the + host's first publication (`load`'s batch of declarations) is not placed yet: it is `loading`, and the publication + places it. The gate lasts only for the synchronous setups, never for activation, so it cannot deadlock. +- **Then the selected provider is checked** (here: served, the version and fingerprint, the params, the provider's + own host-address check; over a path: at the provider host). Version checks never look at a same-name registration + elsewhere. What the check finds: + + | The check finds | The request | + | --- | --- | + | a fitting provider | bound (to the object here, or to the hold over a path) | + | not published yet | unbound `unpublished`: `loading` | + | nobody serves the name at its address (or serves it as the other kind) | unbound `not_served`: `failed { waiting }` (`unavailable` "X is not served here") | + | no fitting version, a fingerprint that differs at the same version, a host address the provider computes elsewhere | unbound `version`: `failed { waiting }` (`contract_mismatch`, `invalid/host_address`) | + | a registry served, but its first `compose` (or the reader's version mapping) threw | unbound `compose`: `failed { waiting }` (§17) | + | the path has no link (a drop, a route resolving) | on the path: `loading` | + | `authorize` refused, invalid params, a cycle | `failed { broken }`, never indexed | + + The unbound reasons that wait for a name are indexed under it on the host that decided them: the consumer's host + for a request made there, the provider's host for a hold that arrived over a link (kept refused there; a `retry` + re-runs its start). A publication serving a name (in 7a: `load`'s) wakes its requests, which run their check again + (a refused hold sends `loading`, then `bound` or `failed`); no new message kind. Index entries go when their request + ends (GC-tested). The consumer never re-holds a request by itself: no backoff, no polling. ### The low-level host API @@ -203,16 +230,16 @@ parent link closes. `resolve` returns the link before the worker has loaded (the | Call | Meaning | | --- | --- | | `createHost

    ({ address?, principal?, ...options })` | with an address: addressable, serves only objects placed there, acts as `principal` on its own host; without: a client. `P` is the app's principal type | -| `load([{ name, facet }], context)` | facets with their plugin names (the plugin's `package.json` name, npm style; duplicates and invalid names are refused); runs every facet's setup, then waits for the start barrier (section 21); a throwing setup rejects and closes the host; runs once. Loaders are adapters; base code passes its core facet the same way | +| `load([{ name, facet }], context)` | facets with their plugin names (the plugin's `package.json` name, npm style; duplicates and invalid names are refused); runs every facet's setup, publishes all of them in one batch (types and registries served, registries composed once, contributions; requests waiting for the publication are placed after it), then waits for the start barrier (section 21: the first outcomes of its host-lifetime objects, slice 7a); a throwing setup rejects and closes the host; runs once. Loaders are adapters; base code passes its core facet the same way | | `connect(link, { principal, trusted?, codec? }, context)` | the link to the parent, at most one open; once it closed (a drop), `connect` takes the next, and every remote reference over the parent is held again on it. `principal` = who the parent is | | `accept(link, { principal, trusted?, codec? }, context)` | a link from a child; holds and calls on it act as `principal` | | `route(Kind, { authorize, resolve }, context)` | how to reach hosts of a kind (below); called before `load`, once per kind; work arriving on its links runs under `context` | -| `get(Type, params, context, { readyTimeout? })` | a reference once its object is live (section 7) | +| `get(Type, params, context, { readyTimeout? })` | a reference once its object is live, waiting through `loading`, `failed { transient }` and `failed { waiting }` up to its `readyTimeout` (section 7) | | `hold(Type, params, context, { readyTimeout? })` | a reference at once that follows its object until released (section 7) | | `close(context)` | closes the links (children, routes, then the parent), then ends every object at once and tears them down, the last to go live first (reason `close`); rejects with an `AggregateError` of the native closes that threw | | `revoke(r, context)` | revocation (section 4; slice 4b) | | `root(fn, context, { principal? })` | a call scope for code outside objects (a Durable hook, a CLI): `fn` gets a handler's `ctx`, sealed when `fn` settles, `context` aborts or the host closes (slice 8 D3, section 8) | -| `registry(R, context)` | registry `R`'s composed value for code outside objects; waits for `load`, rejects when it has none (slice 8, section 17) | +| `registry(R, context, { readyTimeout? })` | registry `R`'s composed value for code outside objects once it is usable; waits for the publication, then while it has none up to its `readyTimeout`, then rejects with the last error (slice 8, 7a, section 17) | **Options of `createHost`:** `address`, `principal`, `principalKey` (section 4; the same function on every host), `telemetry` (section 29), `onError`, `closeTimeout`, `readyTimeout`, `heartbeat: { ping?, close? }`; later `revokedBy` @@ -221,7 +248,7 @@ and closed by the app with `store.close(ctx)` after `host.close`; the host never background, as a tagged `HostError`: `listener_error` (a watch listener threw), `delivery_failed` (a frame delivery threw), `activation_failed { object, failure }`, `close_failed { object, native, error }`, `close_timeout { object, native }`, `drain_timeout { object }`, `protocol_error { error }`, `resolve_failed { address, error }`; later routine -errors (slice 5), `derive_failed { object, member?, error }` (a derive's compute threw, once per failure episode, slice 6a), the registries' `registry_conflict { registry, message, plugins }`, `registry_failed { registry, error }`, `contribution_failed { registry, plugin, type?, error }` and `contribution_refused { registry, plugin, type?, version }` (each once per episode, slice 8), `object_failed { object, failure }` (a live object failing, slice 6a, `broken`; `transient` when a `self.follow` source failed, slice 6b R4; `failure` where the value is a `FailureCause`, `error` otherwise, 2026-10-27), `job_interrupted { object, job, id, restarts }` (a persisted run ended at the restart limit, slice 6b D6) and `storage_failed { error }` +errors (slice 5), `derive_failed { object, member?, error }` (a derive's compute threw, once per failure episode, slice 6a), the registries' `registry_conflict { registry, message, plugins }`, `registry_failed { registry, error }`, `contribution_failed { registry, plugin, type?, error }` and `contribution_refused { registry, plugin, type?, version }` (each once per episode, slice 8), `object_failed { object, failure }` (a live object failing, slice 6a, `broken`; `transient` when a `self.follow` source failed, slice 6b R4; `failure` where the value is a `FailureCause`, `error` otherwise, 2026-10-27), `job_interrupted { object, job, id, restarts }` (a persisted run ended at the restart limit, slice 6b D6), `parked_long { object, waitingFor, since }` (an attempt parked longer than its `readyTimeout` on a remote attempt that is only `loading`, once per parking episode: a diagnostic of a ring across hosts started at the same moment, slice 7a, §6) and `storage_failed { error }` (slice 6a), which every app must handle: Chord never ends the process, the app exits with its own code for its supervisor or reopens. @@ -230,7 +257,7 @@ the app exits with its own code for its supervisor or reopens. | Timeout | Bounds | Host default | Override | | --- | --- | --- | --- | | `closeTimeout` | a native's `close` (then reported as a possible leak; the rest continues); the drain of an ending object's running handlers (callers are rejected at once) | 30 s | per native: `self.open(open, close, { closeTimeout })` | -| `readyTimeout` | an activation attempt (its wait for the previous instance's teardown included); a getter's whole wait; how long a call made while its reference is loading waits (the park deadline) | 60 s | per `get`/`hold`: `{ readyTimeout }` | +| `readyTimeout` | an activation attempt's own steps (its wait for the previous instance's teardown included; paused while the attempt is parked on its dependencies, slice 7a); a getter's whole wait; how long a call made while its reference is loading waits (the park deadline) | 60 s | per `get`/`hold`/`registry`: `{ readyTimeout }` | The park deadline's per-type override is not built (only if a real case needs it). Timeouts of later slices (routine handover 10 s, job `finish` retry backoff 1 s to 60 s, rollout install 60 s, provider-side `linger` per type) @@ -251,7 +278,9 @@ bounded by the app. - **Worker keep-alive** works as in p2: the worker counts, the app's supervisor decides. **`host.activity`** (slice 4b; owner, 2026-10-24) is a JSON `Watchable` of `{ references, calls, jobs, lingering }`: holds arriving over links, method calls running on this host, running jobs, provider-side lingering objects. Counters are kept where the work - happens; a change marks it dirty and it is published at most once per microtask turn with changes. Core code + happens; a change marks it dirty and it is published at most once per microtask turn with changes. A hold placed + here counts from its arrival to its end, bound or not (one waiting for a missing provider keeps the worker alive; + slice 7a); a hold this host relays counts at its destination; `host.references` lists bound references only. Core code reports it plus Durable tasks (the core `Harness`); the server's supervisor stops the worker when all are zero for the idle time (p2: 5 minutes, checked every second). Chord provides the counts, no idle logic. @@ -521,9 +550,10 @@ host.serve(Notes, ({ session }, principal, self) => - **Compatibility (strict, like documents; R6):** a hold carries only the consumer's contract `version` and fingerprint, never a description. The provider binds when both equal its current ones, or (slice 7) when the version is one it accepts and the fingerprint equals the one computed from its own history (section 27). Every - contract change, additive ones included, bumps `version`. Anything else is `contract_mismatch` (or `updating` during - a rollout, section 21). The fingerprint covers params, variants and members, not the host address. A consumer whose contract - does not fit gets a mismatch on its own binding; the provider object and its other bindings are not affected. + contract change, additive ones included, bumps `version`. Anything else leaves the request unbound `version`: + `failed { waiting }` with `contract_mismatch` (slice 7a: not the consumer's fault; a publication that fits wakes it, + §3). The fingerprint covers params, variants and members, not the host address. A consumer whose contract does not + fit waits on its own request; the provider object and its other bindings are not affected. - **Variants:** a member name has the same kind (state or method) in every variant that has it, and a method the same signature (arguments and result) in every variant; state schemas may differ per variant (redaction). Enforced at the type level in `remote(...)` and at declaration at runtime. A remote factory returns one variant, `{ variant, @@ -567,7 +597,7 @@ host.serve(Notes, ({ session }, principal, self) => ### Lifecycle ``` -(get) → activating → live → ending → ended +(get) → activating (parked on its dependencies → own steps) → live → ending → ended └→ failed (backoff) → activating … live → failed (a runtime failure of the owner, slice 6a: broken, by reload or retry only) ``` @@ -584,11 +614,11 @@ host.serve(Notes, ({ session }, principal, self) => - **Activation** (one fresh attempt each time, bounded by `readyTimeout`, its wait for the previous instance's teardown included): the factory runs; singleton documents take `initial()` and, on the object's first activation, persisted documents load and its tables' definitions are checked and migrated (slice 6a; the deadline paused during - eager migration work); references are bound (before the - previous attempt's are released, so a dependency held only by it never ends in between) and waited for until live - and hydrated; natives open one after another; the returned members are checked (section 5); derives compute; - routines start (slice 5). The object goes live when all of that succeeded. Calls wait behind its **gate** until - then. + eager migration work); its requests are made (`self.get`, `self.registry`, keyed pins), and the attempt is + **parked** until every one is live and hydrated (below; the deadline paused); then its own steps run under the + deadline: natives open one after another, followed sources attach, the returned members are checked (section 5), + derives compute; routines start (slice 5). The object goes live when all of that succeeded. Calls wait behind its + **gate** until then. - **The seal fence** (owner, 2026-10-26): every attempt has a seal token (a job run invocation its own, for its state changes and outcome). After the seal, new work through the attempt's handles is refused at use (`self.tx`, `ctx.get`, a job or routine start); commits are checked when they enter the store's commit step (sealed: rejected `released`, @@ -609,18 +639,36 @@ host.serve(Notes, ({ session }, principal, self) => with the cause, state members keep their last values, natives close, documents stay on the object; recovery by reload or `ref.retry()`; live dependents stay live. The members freeze only after the commits admitted before the seal adopted and published once, and a retry's factory waits for that (2026-10-27, slice 6a build reviews). -- **Waiting for dependencies (R18):** an object whose references are not live yet stays `loading` and goes live as - soon as they are (bounded by `readyTimeout`; a link drop while a dependency is still loading is no failure: the - reference re-holds on the next link and the dependent keeps waiting, design audit 6, 2026-10-27). A dependency - going live while the dependent still activates is remembered and retries it at once if that attempt fails. A - dependency that fails (`broken` or `transient`), or does not become - live in time, fails the dependent **`transient`** with the dependency's cause inside, never `broken`: the cause is - the dependency's. The dependent keeps its references while failed, and Chord installs their state as an automatic - retry trigger: the moment the dependency goes live, the dependent is retried at once (the remaining backoff is - cancelled; the backoff stays as a safety net). Factory authors write `self.get` and know nothing of this. When a - dependent's last reference goes, it ends and releases its dependencies. After `live`, a dependency leaving `live` - does not change the dependent: it stays live, and its calls that use the dependency get its error (or park while it - is loading). Only R18 failures keep references; any other failure releases them. +- **Parked on its dependencies** (dependency model §3, slice 7a; supersedes R18's dependency deadline and backoff): a + dependency is what an attempt declares and needs before it goes live: `self.get` (local or remote), `self.registry`, + keyed pins (`at(key)` on another object's keyed member); `self.follow` is the attempt's own step, not a dependency. + While any dependency is not live (and hydrated), the attempt is parked and its object shows: + + | Its dependencies | The object shows | + | --- | --- | + | any `loading`, none failed | `loading` | + | any `failed` | `failed { waiting }`, `retryAt` undefined, error `unavailable` "waiting for X: " (X the first failed one in declaration order, the root cause found through its message's "waiting for …: "; always these two, so a ring of waiting attempts settles after one round; within 500 characters, X's label clipped first, fix round 1) | + | one failed with `invalid/cycle` (a dependent off the ring) or `forbidden` (an `authorize` that refused it: authority, M§3.6) | `broken` with that error: nothing heals it but `ref.retry()` or a reload | + | one `ended` (revoked, its host closed) | `broken` with that error: the request cannot come back | + + **No timer runs while parked:** no dependency deadline, no backoff; the attempt's `readyTimeout` deadline is paused + (as eager migrations pause it) and resumes with the budget left; the identity backoff stays for the object's own + `transient` failures only. **Wakes:** the parked attempt checks again at every change of a dependency (live, + loading, failed, a new cause compared by value, ended, and its first outcome, below), coalesced per object after the + batch that caused them (a change while it checks or loads pinned keys is remembered); it never re-runs its factory. + A parked attempt keeps its documents, declarations, requests and seal token, and holds no natives. **The dependency + check is the boundary:** once all are live (and the pinned keys loaded), the own steps run; a dependency that leaves + `live` afterwards changes nothing (R18 after live: the object goes live; its calls through that reference meet its + state). **Retry passes on:** `ref.retry()` (or a `retry` message) on an object parked `failed { waiting }` passes on + to its failed dependencies, once per retry operation: the operation (a uuidv7) travels in the `retry` message + (`operation`) with the instance ids of the objects it passed through (`through`, added hop by hop, at most 32), and + an object already in `through` drops it, so a ring across hosts stops after one lap however many retries run at + once; an object also remembers the last few ids it handled, so a diamond retries it once. A `broken` or + `transient` one starts an attempt, a parked one passes it on, a request waiting unbound in an index is left as it is + (an event wakes it), a refused hold over a link re-runs its start. A `retryOn` wake while parked does nothing. When + a parked object's last reference goes, it ends at once (it is not live: no `linger`) and releases its requests. + `broken` is only for an object's own code, cycles and authority; no failure keeps references (the R18 retry trigger + is gone: parking replaces it). - **Dependency cycles are rejected (Q5):** every object instance has a uuidv7 instance id, minted at its first attempt; an object that restarts gets a new one, so an old chain never matches (slice 4b, the first user; D4, owner, 2026-10-24). On one host, at @@ -629,23 +677,32 @@ host.serve(Notes, ({ session }, principal, self) => (bounded; ignored from untrusted links): the provider rejects the hold when the instance being activated is in the chain (slice 4b; on one host slice 1; at most 32 entries; locally created objects inherit the activating chain). A cycle fails the activation `broken invalid/cycle` with the cycle as cause ("A → B → A"), and every other activating - object on it the same way; no references are kept. A dependency failing with `invalid/cycle` fails its dependent - **broken** with the same error, an exception to R18 (owner, 2026-10-24: otherwise every backoff retry hits the same - cycle); dependents that are not on the cycle break too, and a reload or `ref.retry()` recovers them. Checked again - after a reload. Cost: O(chain) per declared reference, only during activation. - **Known limitation** (owner, 2026-10-24): a chain is fixed when its object is created, so a cycle entering an object - that is already activating is not detected. Example: A on the server and B on a worker start at the same moment, A - declares B and B declares A; the holds carry `[A]` and `[B]`, neither finds the other, both fail `transient` at - `readyTimeout` and retry with backoff (later attempts usually come in order and detect it). A probe along waiting - holds was not built. + object on it the same way; no references are kept. A dependency failing with `invalid/cycle` makes its dependent + **broken** with the same error (owner, 2026-10-24: it would wait on a ring that never heals); dependents that are not + on the cycle break too, and a reload or `ref.retry()` recovers them. The walk on one host follows the waiting edges + of activating and parked attempts (exact). Checked again after a reload. Cost: O(chain) per declared reference, + only during activation. + **The gap: rings across hosts started at the same moment** (owner, 2026-10-29: no probe): a chain is fixed when its + object is created, so a cycle entering an object already activating is not detected. Example: A on the server and B + on a worker start at the same moment, A declares B and B declares A; the holds carry `[A]` and `[B]`, neither finds + the other, and without a dependency deadline both stay parked. It is made visible: an attempt parked longer than its + `readyTimeout` on a remote attempt that is itself only `loading` (its hold on a link; the clock and `since` start + when the hold gets on a link and stop while the path has no link) + is reported once per parking episode as + `parked_long { object, waitingFor, since }`; nothing fails or retries. A cycle is a plugin bug; **the guide's four + rules** for plugin authors and agents (host README): (1) a factory declares only what the object needs, never a type + that needs it back; (2) back-references and occasional lookups go into methods and routines (`ctx.get`); (3) data + flows up through state and registries, not mutual `self.get`; (4) host-lifetime objects on the server never declare + worker types in their factory. - **Failed:** a broken part fails the whole object: a throwing factory, document hydration or `migrate`, a failed - open, a derive that throws on its first computation during activation, or a dependency that did not become live. - After the object is live, a throwing derive fails only its own member (`broken`, W1). The cause is tagged and decided by Chord from where it - failed: + open, a derive that throws on its first computation during activation. After the object is live, a throwing derive + fails only its own member (`broken`, W1). The cause is tagged and decided by Chord from where it failed: ```ts - cause: { kind: "transient"; error: ChordError } // an open threw, a reference was unreachable: automatic backoff - | { kind: "broken"; error: ChordError } // factory, derive, hydration or migrate threw: no automatic retries + cause: { kind: "transient"; error: ChordError } // the object's own effect failed (an open, a follow): backoff + | { kind: "broken"; error: ChordError } // its own code (factory, derive, hydration, migrate), a cycle, authority + | { kind: "waiting"; error: ChordError } // not its fault (slice 7a): no provider, no fitting version, a + // dependency failed; retryAt undefined, woken by an event ``` Only an `open` may choose `broken` itself (`throw broken(error)`, e.g. invalid configuration). **Transient** causes @@ -718,13 +775,17 @@ location are dropped. | a call | `await ctx.get(Type, params, { readyTimeout? })` | when the handler settles | | outside | `await host.get(Type, params, context, { readyTimeout? })` / `host.hold(Type, params, context, { readyTimeout? })` | `release()` | -- **`host.get`** (and `ctx.get`) resolves with a reference once its object is live and **hydrated** (every state member - of its variant has a value). A broken failure rejects at once; through transient ones it waits up to its - `readyTimeout`, then rejects with the last failure's error; a getter that gives up releases its reference. - **`host.hold`** returns a reference at once that follows its object until released (loading, live, failed and - retried as its object is, loading again while its link is down) and never rejects; invalid params (checked at once, also before `load`: design audit 7, 2026-10-27) and a local type - not served here throw at once (before `load` resolved, these fail the reference broken once it did); its `variant` is - `undefined` until its first binding (slice 3b). There is no `state: false` option (C8) and no `optional` flag. +- **`host.get`** (and `ctx.get`, `host.registry`) resolves with a reference once its object is live and **hydrated** + (every state member of its variant has a value). It waits while the reference is `loading`, `failed { transient }` + or `failed { waiting }`, up to its `readyTimeout` (default 60 s, the caller's choice), then rejects with the last + cause; only `failed { broken }` rejects at once (owner, 2026-10-29, decision "Getters wait through `waiting`"). A + getter that gives up releases its reference. **`host.hold`** returns a reference at once that follows its object + until released (loading, live, failed and retried as its object is, loading again while its link is down) and never + rejects; only invalid params throw (checked at once, also before `load`: design audit 7, 2026-10-27). A type not + served where it goes is `loading` before the host's publication and `failed { waiting }` after it, before and after + `load` alike (§3; slice 7a). Its `variant` is `undefined` until its first binding (slice 3b). A UI holds and shows + `ref.state`; code that needs the object now gets it, with the patience it passes. There is no `state: false` option + (C8) and no `optional` flag. - **Narrowing by variant:** `ref.variant` is the discriminant (C15: no `inVariant`/`expectVariant`); `RefIn` narrows to some variants (strict unions, never weakened). - **Members across attempts** (2026-10-27, slice 6a build reviews): on the same host a reference's state member @@ -741,7 +802,8 @@ location are dropped. ```ts | { kind: "loading" } | { kind: "live" } - | { kind: "failed"; cause: FailureCause; retryAt: number | undefined } // retryAt undefined: broken, no automatic retry + | { kind: "failed"; cause: FailureCause; retryAt: number | undefined } // retryAt: a transient cause's backoff; + // undefined for broken and waiting | { kind: "ended"; error: ChordError } // released by its holder, or its object ended ``` @@ -749,13 +811,16 @@ location are dropped. | --- | --- | --- | | link drop | park until reconnect | readable (`loading`) | | object loading or restarting | park until live | readable (`loading`) | -| rollout: provider ahead or installing | park until installed, with a deadline (slice 7) | readable | +| rollout: the provider host installing a revision the hold does not fit (slice 7b) | park until installed, bounded | readable (`loading`) | +| unbound (not served, another version) or parked on a failed dependency | reject at once with the cause | `failed { waiting }` (slice 7a) | | object `failed` | reject at once with the cause | `failed` with cause and `retryAt` | -- A failed reference stays alive: while held, Chord retries per the backoff; `ref.retry()` tries at once (a Retry - button; over a link the `retry { hold }` message, which also re-runs a refused hold's start: authorize, the host address, - binding; design audit 2026-10-27); success goes `loading`, then `live`. A provider tells a pending - hold's consumer each failure (`failed`, with the rest of its backoff window) and the next attempt (`loading`). +- A failed reference stays alive: while held, Chord retries a `transient` failure per the backoff and wakes a + `waiting` one at the event it waits for; `ref.retry()` tries at once (a Retry button; over a link the `retry + { hold }` message, which also re-runs a refused hold's start: the host address, binding; design audit 2026-10-27; + on a parked object it passes on to the failed dependencies, §6); success goes `loading`, then `live`. A provider + tells a pending hold's consumer each failure (`failed`, with the rest of its backoff window for a transient one), + each new cause of an object parked on a failed dependency, and the next attempt (`loading`). - **Parked calls** (slice 3b): calls made while the reference is loading are queued in order, each waiting at most the reference's `readyTimeout` from the moment it was made (never renewed), then fail `unavailable/park_timeout`; they are sent in order when it goes live (a new call never overtakes them) and rejected with the cause when it fails. @@ -1490,7 +1555,7 @@ export default defineFacet((host, self) => { // `self` (contributions) a - A plugin's per-host setup, **declarations only** (owner, 2026-10-26): `host.serve`, `self.contribute` (slice 8), `host.references`; no natives, no documents (C16 accepted), no routines, no jobs. Everything a plugin does at runtime is done through objects: shared or persisted state and standing behaviour live in a `local` store object with empty - params (host lifetime, its own failure domain, R18 recovers its dependents), as p2 does (its 63 facets take only + params (host lifetime, its own failure domain; its dependents park while it is failed and go on when it recovers), as p2 does (its 63 facets take only `host`). A facet cannot fail after `load`. - `host.serve(Type, factory, { linger? })`: `linger` (milliseconds, 0 by default; slice 3b) keeps a live object with params alive that long after its last reference, so a new reference within that time reuses it with its documents @@ -1524,8 +1589,11 @@ const router = self.registry(Http); // a declared dependency; read rou live or ends, when a dynamic contribution changes). It always delivers a value with its fallback built in (e.g. the first by plugin name wins) and reports conflicts through `report` (owner notices and telemetry events); readers get a plain value and never handle conflicts. If `compose` throws, the failure is reported and its - watches get a `failed` call, while `.value` keeps the last good composition (slice 8); only before any good - composition do readers get the error. Work that started earlier keeps the value it read (a pinned Durable phase keeps + watches get a `failed` call, while `.value` keeps the last good composition (slice 8), **but only while every + contribution it was built from is still in the table** (dependency model decision 5, slice 7a): when one leaves (its + contributor ended, failed or restarted), the last good value is dropped and `.value` throws until `compose` + succeeds, so it never hands out a native whose contributor ended; only before any good composition, or after such a + drop, do readers get the error. Work that started earlier keeps the value it read (a pinned Durable phase keeps its old table). Helpers: `keyedEntries(table, entriesOf, report)` (a map, the first by plugin name wins, a duplicate reported), `orderedEntries(table, itemsOf, orderOf?)`; both keep each item's plugin (named so beside the contracts' `keyed`, slice 8 build). @@ -1538,7 +1606,8 @@ const router = self.registry(Http); // a declared dependency; read rou (`compose`, version, `accepts`); other plugins' tokens only identify it by name and types (contribute and read, never bring their own `compose`). A reload of the defining plugin replaces the declaration atomically and the table recomposes. Without a declaration (the defining plugin is not loaded) the registry has no composed value: readers - get an error like a failed reference; contributions wait in the table. + wait `failed { waiting }` (`unavailable/undeclared`), like a request for a type nobody serves; contributions wait in + the table. - **Versions (Q8, the owner-adapter pattern):** a contribution carries the registry version its plugin was built against; older ones are adapted with the defining facet's pure `accepts` when the table is built. A contribution without a matching version or adapter (older without adapter, or newer) is left out of the table and reported (an @@ -1548,12 +1617,16 @@ const router = self.registry(Http); // a declared dependency; read rou use site catches synchronous throws and rejections when it calls an entry, reports naming the contributor (the composed table keeps each entry's plugin name), and continues with the other entries. Durable already does this for extensions (`extension_error`; a throwing tool becomes an error result); `compose` reports through `report`. -- **Slice 8's rules** (decisions "Slice 8 D1" to "Slice 8 S3", `slice-8.md`): the token is shared, the provider serves it - (`host.serve(R, { compose, accepts })`); `self.registry(R)` in a factory is a declared dependency (activation waits - for a value; with none after `load`, the reader fails `transient` and is retried when one arrives); contributions - never fail a registry (a failed one is left out and reported); a throwing `compose` keeps the last good composition; - older readers get the value mapped back (`accepts[old].value`); `retryOn` retries on value calls only; `host.root(fn, - context, { principal? })` seals its context at settle, abort and `host.close`. +- **Slice 8's rules** (decisions "Slice 8 D1" to "Slice 8 S3", `slice-8.md`; S2's `broken` and S3's backoff superseded + by slice 7a): the token is shared, the provider serves it (`host.serve(R, { compose, accepts })`); `self.registry(R)` + in a factory is a declared dependency: the attempt parks until the registry has a usable value at its version + (composed; mapped by `accepts[old].value` for an older reader), otherwise it shows `failed { waiting }` naming the + registry (`not_served`: `unavailable/undeclared`; `version`: a reader newer than the provider or without a mapping, + `contract_mismatch`; `compose`: a first `compose` or the mapping threw; `loading` before the publication), woken at + every change of whether the value is usable; `host.registry` waits the same way up to its `readyTimeout`; + contributions never fail a registry (a failed one is left out and reported); a throwing `compose` keeps the last + good composition (above); `retryOn` retries on value calls only; `host.root(fn, context, { principal? })` seals its + context at settle, abort and `host.close`. - **Registries may carry natives.** An entry lives exactly as long as its contributor; consumers look up at each use or follow the composed value and keep nothing from an entry beyond a use. - **Contributions that change (owner, 2026-10-19; W4):** registries are host-local, so contributions are usually @@ -1716,7 +1789,9 @@ agent asks everyone present. Its store must live in an object (it predates "nati keeps the hold: a pure object re-subscribes it through the version's mappings and sends `[["r", value]]` for each member whose value changed; a restarted object sends a fresh `bound` for the same hold (a `bound` on a bound hold replaces its replicas). A served hold whose object leaves `live` while its `bound` waits for a slot goes back to - pending (slice 7 builds it: nothing in 4a leaves `live`). A reload that drops the version ends its holds with `gone` (`contract_mismatch`) before + pending (slice 7 builds it: nothing in 4a leaves `live`). *(Pending slice 7b's reconciliation with the dependency + model M§5, which unbinds such holds `version` instead of ending them; until then this item and the rollout's + `updating` deadline below are not current rules.)* A reload that drops the version ends its holds with `gone` (`contract_mismatch`) before the new code serves anything. A consumer whose own code moves to a new version re-holds (a new hold id) and releases the old. - **Old-version calls:** `checkArgs`'s `{ kind: "call" }` result runs the provider's replacement; `{ kind: "method"; @@ -1729,14 +1804,22 @@ agent asks everyone present. Its store must live in an object (it predates "nati - **Consumers elsewhere** see a restarting object like any object that briefly goes away. - No factory trial, no migrate dry run, no consumer contract check during staging: incompatible consumers fail at bind with `contract_mismatch` and a notice naming plugin and type. -- **Start barrier** (host start and first load): load facets (the core facet with its start data among them), wait - until every facet and host-lifetime object is live or failed (an object that must - contribute at start, like the model runtime, is a `local` object with empty params). Then the registries hold the contributions of every - plugin that loaded, and dependent work - resumes: base code calls `harness.resume()` after `load` (the harness is an ordinary host-lifetime object that - opened Durable paused during the barrier; owner, 2026-10-21, refines R17). Nothing - waits for specific contributions: what is missing shows where it is used (section 24). Failed plugins are reported - with their error. Plugin reloads need no barrier. +- **Start barrier** (host start and first load; dependency model §6, decision (a), slice 7a): `load` runs the facets' + setups (the core facet with its start data among them) and **publishes** them in one batch (types served, + registries served and composed once, contributions in their tables); requests that waited for the publication are + placed after that batch (holds and `host.hold` wait for the publication only, never for the barrier: two hosts + whose objects need each other's types start without a deadlock). Then it activates the host-lifetime objects and + resolves when each has its **first outcome**: live, failed by its own cause, or parked with no blocker that is a + local object still without its first outcome (`local`: executing on this host, a keyed pin still loading here + included; a wait on another host or on a failed dependency ends the first pass), or ended (`host.close` settles a + waiting `load`). One first-outcome signal per object, separate from the readiness dependents wait for, counted once, + never reopened by retries. So `CodeMode → Harness` waits for the harness's outcome. **What it promises:** every + host-lifetime object here had its first try, and the contributions of those that went live are in their tables; + **not** every tool that will arrive (a parked or slow contributor, tools found in `onLive`, MCP): what a first turn + needs is p3's explicit admission check. Base code calls `harness.resume()` after `load` (the harness is an ordinary + host-lifetime object that opened Durable paused during the barrier; owner, 2026-10-21, refines R17). Nothing waits + for specific contributions: what is missing shows where it is used (section 24). Failed plugins are reported with + their error. Plugin reloads need no barrier. ### Rollout across hosts @@ -1800,14 +1883,15 @@ validation, no clone of snapshots or ops. | `hello` | both, first | `protocol`, `incarnation`, `calls` (the call window N) | | `hold` | to provider | `origin?`, `hold`, `type`, `version`, `fingerprint`, `params` (part), `address`, `principal?`, `incarnation?`, `chain?` (activation chain, Q5), `limit?` (decoded size limit, 4b) | | `revoke` | parent to child (route link) | the revocation value (4b) | - | `release`, `retry` | to provider | `origin?`, `hold` | + | `release` | to provider | `origin?`, `hold` | + | `retry` | to provider | `origin?`, `hold`, `operation?` (the retry it belongs to, a uuidv7), `through?` (with `operation`: the instance ids of the objects it passed through, at most 32; an object in it drops the retry, so a ring stops after one lap); relays forward both (slice 7a fix rounds 1, 2) | | `call` | to provider | `origin?`, `call`, `hold`, `method` (index), `args?` (part; absent: no arguments), `segments?` | | `abort` | to provider | `origin?`, `call` (named `cancel` until 2026-10-27) | | `ack` | to provider | `origin?`; one per applied `bound`/`state` message | | `closed` | relay to provider | `origin`: the downstream origin ended | | `bound` | to consumer | `origin?`, `holds: [{ hold, variant, members: [{ member, value }], failed }]`; one batch | | `state` | to consumer | `origin?`, `holds: [{ hold, members: [{ member, ops }], failed, retired }]`; one batch (`retired`: members retired since the last message, slice 6a) | - | `failed` | to consumer | `origin?`, `hold`, `cause { kind, error }`, `retryIn?` (absent: broken) | + | `failed` | to consumer | `origin?`, `hold`, `cause { kind, error }` (`kind`: `transient`, `broken` or `waiting`, slice 7a), `retryIn?` (a transient cause's backoff; absent: broken, waiting); each new cause of a parked object is a message of its own | | `loading` | to consumer | `origin?`, `hold`: a new attempt after a failure, or a relay's upstream dropped | | `result` | to consumer | `origin?`, `call`, `value?` (part; absent: `null`), `segments?` | | `error` | to consumer | `origin?`, `call`, `error` (a ChordError's JSON or a plugin's code, message, details) | @@ -2132,8 +2216,8 @@ function doing counted work counts it through its context; call sites never coun site labels. **Testing the core:** every slice spec contains an explicit test list, one group per mechanism with its edge cases, -tested individually and exhaustively: activation and dependency recovery (R18: broken dependency, trigger, backoff as -safety net, a dependency failing while live, cycles), revocation over running jobs (R11), compatibility (R6), params +tested individually and exhaustively: activation and parked dependencies (the dependency model: unbound requests, waits without timers, wakes on every +change, retry passing on, a dependency failing while live, cycles, the start barrier's first outcomes), revocation over running jobs (R11), compatibility (R6), params and arguments for every caller (R7), migration results and old calls on migrated documents (R8, R15), flow control (the window per origin, folding, relays forwarding and returning acks), the incarnation fence (R20), the harness start after the barrier (R17), native restart patterns (R16), and W1's publication list (a throwing derive, a chain A→B→C diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index 97883b776..93c5cfc86 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -76,6 +76,7 @@ pattern, no code. Decided cuts are noted where they apply. | 3 | park deadline (= `readyTimeout`, C13) | 3b | | 3 | `host.activity` (keep-alive counts) | 4b | | 3 | `host.root(fn, …)` for code outside objects | 8 | +| 3 | requests that outlive providers: the destination by address, the publication gate, the unbound index, the check against the selected provider only; the start barrier's first outcomes (decision (a)) | 7a | | 4 | principal type `P`, links stamp principals, principal keys in identity | 3a | | 4 | revocation (`revokedBy`, `host.revoke`, propagation to children) | 4b | | 5 | `local` types: params, members, checked arguments (Q15), results unchecked | 1 | @@ -85,6 +86,7 @@ pattern, no code. Decided cuts are noted where they apply. | 6 | lifecycle, one object per identity, lifetime from kind and params (Q-A), factory, activation, ending | 1 | | 6 | failure causes, backoff per identity, dependency recovery (R18) | 2 | | 6 | `retryOn` | 8 (first independently living source: registries) | +| 6 | parked attempts (no dependency deadline or backoff, wakes on every change, retry passing on), `FailureCause` `waiting`, `parked_long` and the authoring rules against cycles | 7a | | 6 | cycle rejection on one host (Q5) | 1 | | 6 | cycle rejection across hosts (activation chain in hold headers) | 4b | | 6 | shared store with per-principal views; worker-wide native owners; natives following a relocation | doc (1, 3, 2) | @@ -92,6 +94,7 @@ pattern, no code. Decided cuts are noted where they apply. | 7 | per-reference watches (released with their reference) | 1 | | 7 | `ctx.get` (call-scoped references) | 3a | | 7 | `ref.state` (tagged), `ref.retry()`, `host.hold`, calls parking, failed references | 3b | +| 7 | getters wait through `waiting`; `host.hold` throws only for invalid params | 7a | | 7 | `ref.variant` as discriminant, `RefIn` (C15: no `inVariant`/`expectVariant`) | 3a | | 7 | `state: false` (C8: cut) | - | | 7 | optional dependencies as registries | doc (8) | @@ -119,6 +122,7 @@ pattern, no code. Decided cuts are noted where they apply. | 15 | `defineJob`, `self.job`, intent/run/finish, state documents, dedupe (Q9), `busy`, `running`, blocked `finish` (C3 decided); persistence, resume, restart limit; `accepts` in use | 5 (persistence: 6b; `accepts` and reload: 7) | | 16 | `defineFacet

    `, facet as declarations only (documents and routines removed in 6a) | 1, 6a | | 17 | registries: `defineRegistry`, contributions, `compose`, versions (Q8), use-site guards | 8 | +| 17 | registry readers as parked dependencies (`waiting`), `host.registry` waiting, the last good value dropped when a contribution leaves | 7a | | 18 | `constant(value)`, start data as a core object (`SessionInfo`) | 1 (core facet pattern: 4a) | | 18 | platform bindings as core objects | doc (8) | | 19 | `host.references` (presence) | 4b | @@ -185,6 +189,12 @@ deleted (the slice's spec and code are the record). Moved here from `carry-over. (`local`, empty params) stays broken until the host restarts; slice 7 maps the records to the new version (or blocks the old runs) on reload. +- **Publications wake per name** (from slice 7a): `UnboundIndex.wakeAll()` wakes every indexed request, right for the + one publication 7a has; a reload's or an install's publication wakes only the requests of the names it serves + (`wake(names)`), or every refused hold on the host would re-run its start. +- **The reload handover of a parked attempt** (from slice 7a): the new attempt's requests bind before the old ones are + released (M§3.9); the mechanism (an attempt releasing its predecessor's references after binding its own) was removed + in 7a with R18's kept references, its only feeder. - **Routines on reload** (from slice 5): the handover per object, bounded by `closeTimeout`; calls do not wait for it. - **Jobs on reload** (from slice 5): `accepts` in use (args and result mapped forwards once per run; `blocked { version }` without an adapter or for a newer version); when an older run's state document may migrate while its diff --git a/packages/chord/src/next/data/failure.ts b/packages/chord/src/next/data/failure.ts index a76190118..2f01fe258 100644 --- a/packages/chord/src/next/data/failure.ts +++ b/packages/chord/src/next/data/failure.ts @@ -1,11 +1,14 @@ -// Failure causes of an open: transient (retry later) or broken (bad configuration). The failing stage decides. +// Failure causes: transient (the object's own effect failed: retry later), broken (its own code, a cycle or authority: +// retry by hand) or waiting (not its fault: no provider, no fitting version, a dependency that failed; woken by an +// event). The failing stage decides, never the error code. -import { asChordError, type ChordError } from "./errors.ts"; +import { asChordError, ChordError } from "./errors.ts"; -/** Why an open failed; the kind comes from the failing stage, never from the error code. */ +/** Why an object, or a request, is failed; the kind comes from the failing stage, never from the error code. */ export type FailureCause = | { readonly kind: "transient"; readonly error: ChordError } - | { readonly kind: "broken"; readonly error: ChordError }; + | { readonly kind: "broken"; readonly error: ChordError } + | { readonly kind: "waiting"; readonly error: ChordError }; /** Marks an open's failure as broken (bad configuration): `throw broken(error)`. */ export function broken(cause: unknown): Error { @@ -18,6 +21,65 @@ export function openFailure(thrown: unknown, label: string): FailureCause { return { kind: "transient", error: asChordError(thrown, label) }; } +/** The longest message a `waiting` cause of a dependent carries (telemetry's rule for error messages). */ +const MESSAGE_LIMIT = 500; + +/** How a dependent's message names its blocker: the nearest one first. */ +const PREFIX = "waiting for "; + +/** The separator after a blocker's label. */ +const SEPARATOR = ": "; + +/** The marker of a clipped label or root. */ +const CLIPPED = "…"; + +/** The most of the 500 characters the root cause takes when both are long: the nearest blocker keeps the rest. */ +const ROOT_LIMIT = 320; + +/** + * The `waiting` cause of a dependent parked on `blocker`, which failed with `error`: `unavailable`, its message always + * "waiting for : " (the root found through any "waiting for …: " of `error`'s message), so a ring + * of waiting attempts passing causes round reaches the same message after one round; within 500 characters, the + * label clipped first, the root only past its share. `error` is its cause. + */ +export function waitingFor(blocker: string, error: ChordError): FailureCause { + let root = rootMessage(error.message); + let label = blocker; + const room = MESSAGE_LIMIT - PREFIX.length - SEPARATOR.length; + if (label.length + root.length > room) { + const rootRoom = Math.max(ROOT_LIMIT, room - label.length); + if (root.length > rootRoom) root = clipped(root, rootRoom); + if (label.length + root.length > room) label = clipped(label, room - root.length); + } + const message = `${PREFIX}${label}${SEPARATOR}${root}`; + return { kind: "waiting", error: new ChordError({ code: "unavailable", message }, { cause: error }) }; +} + +/** `text` within `length` characters, clipped with "…" when longer. */ +function clipped(text: string, length: number): string { + return text.length <= length ? text : `${text.slice(0, length - CLIPPED.length)}${CLIPPED}`; +} + +/** The root cause in a "waiting for X: root" message: past every leading "waiting for …: ". */ +function rootMessage(message: string): string { + let rest = message; + while (rest.startsWith(PREFIX)) { + const end = rest.indexOf(SEPARATOR, PREFIX.length); + if (end < 0) break; + rest = rest.slice(end + SEPARATOR.length); + } + return rest; +} + +/** Whether two failure causes say the same (a wake compares causes by value): kind, code, reason and message. */ +export function sameCause(a: FailureCause, b: FailureCause): boolean { + const { error: left } = a; + const { error: right } = b; + return ( + a.kind === b.kind && left.code === right.code && left.reason === right.reason && left.message === right.message + ); +} + class BrokenOpen extends Error { constructor(cause: unknown) { super(cause instanceof Error ? cause.message : String(cause), { cause }); diff --git a/packages/chord/src/next/data/internal.ts b/packages/chord/src/next/data/internal.ts index d59881ebe..95d03a2cc 100644 --- a/packages/chord/src/next/data/internal.ts +++ b/packages/chord/src/next/data/internal.ts @@ -8,7 +8,7 @@ export { invalid, PROTOCOL_VERSION, } from "./errors.ts"; -export { openFailure } from "./failure.ts"; +export { openFailure, sameCause, waitingFor } from "./failure.ts"; export { checkAddress, checkParams, identityKey } from "./params.ts"; export { canonicalJson, principalKey } from "./principal.ts"; export { check, validatorFor } from "./validator.ts"; diff --git a/packages/chord/src/next/host/README.md b/packages/chord/src/next/host/README.md index b5c9175fa..0bc278cbf 100644 --- a/packages/chord/src/next/host/README.md +++ b/packages/chord/src/next/host/README.md @@ -62,12 +62,13 @@ const host = createHost({ closeTimeout: 30_000, readyTimeout: 60_000, }); -await host.load([{ name: "@app/notes", facet }], ctx); // every host-lifetime object live or failed (reported) -const notes = await host.get(Notes, {}, ctx); // live and hydrated; waits through transient retries +await host.load([{ name: "@app/notes", facet }], ctx); // published, then every host-lifetime object's first outcome +const notes = await host.get(Notes, {}, ctx); // live and hydrated; waits through loading, transient, waiting await notes.add({ text: "milk" }, ctx); notes.release(); const draft = host.hold(Draft, { conversation: "c1" }, ctx); // a reference at once: loading, then live (await draft.state.watch(ctx)).start(async (call) => call.kind === "value" && render(call.value)); // loading, live, failed, ended +// failed { waiting }: "waiting for X: …", not this object's fault; it goes on when X is live if (draft.state.value.kind === "failed") draft.retry(); // a Retry button host.accept(link, { principal: { kind: "guest", session: "S1" } }, ctx); // a child's link, authenticated by its entry point await host.close(ctx); // closes links, then natives with reason "close" @@ -130,7 +131,8 @@ server.accept(webSocketLink(ws), { principal, limits: { part: 16 * MiB, decoded: `principalKey` maps a principal to its part of an identity (L1's canonical JSON by default; the same function on every host). `closeTimeout` (30 s) bounds each native's close and the drain of an ending object's calls; `readyTimeout` (60 s) bounds every activation - attempt (its wait for the previous instance's teardown included) and is the default deadline of a `get`. `heartbeat: { ping?, close? }`: a link silent for `ping` ms (15 s) + attempt (its wait for the previous instance's teardown included; paused while it is parked on its dependencies) and + is the default deadline of a `get` and of `host.registry`. `heartbeat: { ping?, close? }`: a link silent for `ping` ms (15 s) sends a `ping`, one silent for `close` ms (30 s), its handshake included, is closed (its references go `loading`); one interval timer per host (`min(ping, close) / 10`), running only while it has links. A handler that blocks the event loop longer than `close` gets its links closed. @@ -139,8 +141,11 @@ server.accept(webSocketLink(ws), { principal, limits: { part: 16 * MiB, decoded: principal has `r` among its keys ends at once, whatever references remain (R11): each holder gets `forbidden` with reason `revoked` (a UI can say "access revoked"; a hold over a link sends `gone` with it), running calls are aborted with it (callers rejected), an activation in progress is abandoned; teardown follows as for any end (drain, closes - with reason `end`). Holds of the principal that arrived before the revoke but still wait (for `load`, or relayed for - their route's link) end too (`gone`). + with reason `end`). Requests of the principal that hold nothing end too, bound to nothing yet or never: holds that + arrived before the publication, holds refused here (`failed { waiting }`), relayed holds waiting for their route's + link (each gets `gone`), references made here still unbound, and this host's own remote references of the + principal over its paths, a route still resolving or a link down included (`ended`, released at the provider if + held there): no later link holds them again. Then every route link is sent `revoke { revocation }`, which the child applies with its own `revokedBy` (and passes down its own route links), and every child link whose principal it revokes closes (a relay sends `closed { origin }` upstream for it, after the `revoke`). Never this host's own principal, never the parent link; a @@ -157,8 +162,9 @@ server.accept(webSocketLink(ws), { principal, limits: { part: 16 * MiB, decoded: splice op per removal, one for the additions: a revoke of many holds is one list and one frame), otherwise it is built on its next read. A departure discovered while a commit's links send (a transport failing on send) lands in the next batch, so one commit stays one message per link and origin. -- `host.activity`: a JSON watchable `{ references, calls, jobs, lingering }`: holds arriving over links and served - here, method calls running here (local and remote), job runs running or finishing here, objects lingering after their +- `host.activity`: a JSON watchable `{ references, calls, jobs, lingering }`: holds arriving over links and placed + here, from their arrival to their end, bound or not (a hold waiting for a missing provider keeps a worker alive; + `host.references` lists bound references only; a relayed hold counts at its destination), method calls running here (local and remote), job runs running or finishing here, objects lingering after their last reference. Counted where the work happens; the changes of one microtask turn are one publication (`[["r", counts]]`), a slow watcher gets the newest counts; nothing is scheduled while nobody watches or derives from it. Chord counts; the app's supervisor decides. @@ -176,10 +182,12 @@ server.accept(webSocketLink(ws), { principal, limits: { part: 16 * MiB, decoded: here, plus its own id; the last 32); honoured from trusted links only, relays forward it as it is. A provider refuses a hold whose chain names the instance it would bind while that activates: broken `invalid/cycle` naming the cycle by label where known ("app.a(1) → … → app.a(1)"); a local `self.get` of an activation in the chain is refused the same - way. A dependency failing with `invalid/cycle` fails its dependent broken with that error (owner, an exception to - R18's transient dependency failures: a retry would meet the same cycle). Known limitation: a chain is fixed when its - object is created, so a cycle entering an object that is already activating (two activations started at the same - moment) is not detected: both fail `transient` at `readyTimeout` and retry with backoff. + way. A dependency failing with `invalid/cycle` makes its dependent broken with that error (it would wait on a ring + that never heals). The gap (owner, 2026-10-29: no probe): a chain is fixed when its object is created, so a ring + across hosts whose objects start at the same moment is not detected, and without a dependency deadline it stays + parked; an attempt parked longer than its `readyTimeout` on a remote attempt that is only `loading` (its hold on a + link: the clock and `since` run only while it is, so a late link is not reported at once) is reported once per + parking episode (`parked_long`), nothing fails. A cycle is a plugin bug: see the authoring rules below. - `onError(error: HostError)` gets everything that fails in the background, by `kind` (`failure` where the value is a `FailureCause`, `error` otherwise; a position is `{ kind: "on" | "onLive" | "native", position }`, from 1): `activation_failed` (`object`, `failure`: every failed attempt), `object_failed` (`object`, `failure`: a live object @@ -192,7 +200,9 @@ server.accept(webSocketLink(ws), { principal, limits: { part: 16 * MiB, decoded: route's `resolve` rejected; the address waits its backoff), `routine_failed` (`object`, `routine`: its position, `error`: a routine call threw, or its watch could not be taken; a source's failure calls never reach a routine), `job_failed` (`object`, `job`, `id`, `error`: a run threw, not after an abort), `job_finish_failed` (the same: a `finish` threw; it is retried) and - `job_dropped` (`object`, `job`, `id`: a blocked run dropped without `finish`, by `abort` or its object's end), and + `job_dropped` (`object`, `job`, `id`: a blocked run dropped without `finish`, by `abort` or its object's end), + `parked_long` (`object`, `waitingFor`: the remote blocker's label, `since`: when it parked; once per parking episode, + slice 7a), and the registries' (slice 8, each once per episode) `registry_conflict` (`registry`, `message`, `plugins`: a conflict `compose` resolved), `registry_failed` (`registry`, `error`: `compose` or an older version's `value` adapter threw), `contribution_failed` (`registry`, `plugin`, `type?`, `error`: a watchable contribution failed or ended, or its @@ -207,11 +217,13 @@ server.accept(webSocketLink(ws), { principal, limits: { part: 16 * MiB, decoded: batch, anything else is a plain value whatever methods it has (registries carry natives); a registry value closing a cycle of registries (through the tables and the factory's own earlier contributions) is refused `invalid/cycle` at `contribute` (an edge lives as long as its contribution is in a table). Every served registry composes once at the end of `load`'s setups, before any object activates. - `self.registry(R)` is a declared dependency: with no value at activation (no provider, a first `compose` that threw, - a version not served here) the attempt fails `dependency_failed` (`transient`, the registry's error as the cause, - the references kept) at once, and the registry's first value retries it (R18); a version the provider does not - serve fails it `broken` (`contract_mismatch`: only a reload helps). `host.registry(R, ctx)` waits for `load` and - rejects without a value. + `self.registry(R)` is a declared dependency: without a usable value at its version (no provider: + `unavailable/undeclared`; a first `compose` or the version mapping that threw; a version the provider does not serve: + `contract_mismatch`) the attempt parks `failed { waiting }` naming the registry, and every change of whether the value + is usable wakes it. `host.registry(R, ctx, { readyTimeout? })` waits for the publication, then the same way up to its + deadline, and rejects with the last error. The last good composition is kept only while every contribution it was + built from is still in the table (a contributor that fails or ends drops it: `.value` throws until `compose` + succeeds). - **Wake signals** (`self.retryOn(...sources)`, D2): registry values (constants are accepted and never wake; anything else throws: a reference's member, the attempt's writable, `ref.state`). The object follows them from the factory's return (before hydration, references and natives, so a failed open still has them) until the next attempt's @@ -227,37 +239,47 @@ server.accept(webSocketLink(ws), { principal, limits: { part: 16 * MiB, decoded: `host.root` at once (an uncooperative `fn` runs on and fails at its next use of Chord); an aborted `ctx` runs no `fn`; `host.close` rejects it at once `released/closed`, as a method's caller when its object ends; after `close` it rejects `released/closed`. Not counted in `host.activity`. -- `host.load([{ name, facet }], ctx)`: once per host; runs every setup, then activates every type with empty params - and resolves when each has had its first outcome, live or failed (the start barrier; later retries never hold it). - A setup that throws rejects it with that error and closes the host as `close` does, its links included. Gets wait - for it. +- `host.load([{ name, facet }], ctx)`: once per host; runs every setup and publishes them in one batch (types and + registries served, registries composed once, contributions); requests made here and holds arriving before that + waited in the unbound index (`loading`) and are placed after the batch (they never wait for the barrier: two hosts + whose objects need each other's types start without a deadlock). Then it activates every type with empty params and + resolves when each has its first outcome (decision (a)): live, failed by its own cause, parked with no blocker that + is an object here still without its first outcome (a keyed pin still loading counts as here; a wait on another host + or on a failed dependency ends the first pass), or ended (`close` settles a waiting `load`); later retries never + hold it. A setup that throws rejects it with that error and closes the host as `close` does, its links included. - `host.hold(Type, params, ctx, { readyTimeout? })`: a reference at once (`Held`: its `variant` is `undefined` until - its first binding), which follows its object until released and never rejects: `loading` while `load` runs, while - the object activates or retries, while there is no parent link or it is down; `failed` while the object is failed - and no attempt runs; `live`. Throws at once for invalid params (`invalid/params`) and a local type not served here - (`unavailable`); before `load` resolved - these are decided once it did, and the reference fails `broken` with that error. A `hostAddress` or `authorize` that - throws or refuses, or a host address whose params fail its kind's check (`invalid/host_address`), fails the reference - `broken`. A local type never crosses a link. Where a reference made here (`host.get`/`hold`, `self.get`, `ctx.get`) - or a hold arriving on a link goes, first match (`Destination`; the address is the contract's `hostAddress` of the params, - computed by the caller; a hold carries it to be routed, and a host serving the type computes it again): + its first binding), which follows its object until released and never rejects: `loading` before the publication, + while the object activates, retries or is parked on dependencies under way, while there is no parent link or it is + down; `failed` while the object is failed and no attempt runs, or while nothing fits it (`waiting`: not served where + it goes, another version; bound when a publication serves it) or its object is parked on a failed dependency; + `live`. Throws at once only for invalid params (`invalid/params`, also before `load`). A `hostAddress` or + `authorize` that throws or refuses, or a host address whose params fail its kind's check (`invalid/host_address`), + fails the reference `broken`. A local type never crosses a link. Where a reference made here (`host.get`/`hold`, + `self.get`, `ctx.get`) or a hold arriving on a link goes, by address alone (`Destination`; the address is the + contract's `hostAddress` of the params, computed by the caller; a hold carries it to be routed, and a host serving + the type computes it again): | Case | Goes | | --- | --- | - | the type is served here and its address is this host's | here | - | a reference made here to a served type whose address is another host's | on, as if not served here (rows below): worker S1 reaches `Notes { session: "S2" }` through the parent | - | an arriving hold for a served type whose params place it on another host | broken `invalid/host_address` (misrouted; never forwarded) | + | a local type; a remote type whose address is this host's, served here or not; a hold naming this host's address | here, checked against what is served here (below) | | the address is of a kind this host has a route for | over that route's link (the params checked against the kind, `authorize` first; `forbidden` when it refuses) | | otherwise (a remote type) | over the parent link, open yet or not; a hold that arrived *on* the parent link is never sent back: `unavailable` | | an arriving hold and no parent link | `unavailable` | - A hold that arrives and goes on is relayed (`remote/`): this host holds it upstream as the arriving hold's caller - and forwards between the two without parsing params, arguments, results or state. + Placed here, a request is checked against the type served here only (never a same-name registration elsewhere): + not served (or served as the other kind) or not at the caller's version and fingerprint, or params the provider's + own `hostAddress` places on another host (code skew; checked for an arriving hold and for a remote request made + here alike), it waits unbound `failed { waiting }` (`unavailable`, `contract_mismatch`, `invalid/host_address`), + indexed under its name on this host (`unbound.ts`) until a publication wakes it (a refused hold also by a `retry`, + which re-runs its start: `loading`, then `bound` or `failed`); invalid params, a cycle and a provider's + `hostAddress` that throws (its own code) are `broken`, never indexed. A hold that arrives and goes on is relayed (`remote/`): this host + holds it upstream as the arriving hold's caller and forwards between the two without parsing params, arguments, + results or state. - `host.get(Type, params, ctx, { readyTimeout? })`: a hold, awaited until live. Each getter has one deadline for its - whole wait, `load` and reconnects included. A broken failure rejects it at once with its error; through transient - failures and drops it waits, and at its deadline rejects with the last failure's error (`unavailable` "did not - become live within …" when none failed yet). A getter whose deadline passes or whose context is cancelled releases - only its own reference. + whole wait, the publication and reconnects included. A broken failure rejects it at once with its error; through + `loading`, `failed { transient }`, `failed { waiting }` and drops it waits, and at its deadline rejects with the last + failure's error (`unavailable` "did not become live within …" when none failed yet). A getter whose deadline passes + or whose context is cancelled releases only its own reference. - `host.serve(Type, factory, { linger? })` in a facet's setup: a live object with params whose last reference is released stays live for `linger` ms (0 by default), with its documents and dependencies; a reference within that time reuses it, otherwise it ends. An object waiting, activating or failed when its last reference goes ends at once; @@ -298,7 +320,9 @@ server.accept(webSocketLink(ws), { principal, limits: { part: 16 * MiB, decoded: at `host.close`. When this host holds over a route link it sends whom each hold acts for (the table in `remote/`): a guest's handler that `ctx.get`s a worker type reaches the worker as the guest. - `host.close(ctx)`: aborts every route's `resolve`, closes its links first, children, routes, then the parent (no - `gone` goes out), ends every remote reference (`released/closed`, parked calls rejected with it), then ends every + `gone` goes out), ends every remote reference (`released/closed`, parked calls rejected with it), every request + still unbound (waiting for the publication or a provider) and every waiting `host.registry` (`released/closed`, at + once, not after the slow closes below), then ends every object at once (calls fail `released/ended`, `self.tx` throws; documents stay readable until it is done), aborts pending opens, waits for endings already running, then drains and closes object by object, the last to go live first (a dependent before its dependencies), with reason `close`. Every call returns the same closing. It rejects with an `AggregateError` of the closes that threw. It does not wait @@ -366,7 +390,8 @@ server.accept(webSocketLink(ws), { principal, limits: { part: 16 * MiB, decoded: loading or failed; at runtime it carries every member of every variant, and one outside the bound variant fails `invalid/member`. - `ref.state` (a `NativeWatchable`, made on first access): `{ kind: "loading" } | { kind: "live" } | { kind: - "failed"; cause; retryAt }` (`retryAt`: when the object's backoff window ends, `undefined` for a broken failure) `| + "failed"; cause; retryAt }` (`retryAt`: when the object's backoff window ends, `undefined` for a `broken` or + `waiting` cause) `| { kind: "ended"; error }` (for good: `released` without a reason when its holder released it, `released/ended` when its object ended, `released/closed` at host close, `forbidden/revoked`). A watch gets the current state, then every change; it ends `ended` when the reference ends. What a @@ -382,16 +407,18 @@ server.accept(webSocketLink(ws), { principal, limits: { part: 16 * MiB, decoded: Calls made after the reference went live never overtake queued ones (a call a handler makes while they are sent queues behind them). A parked call is checked against its deadline when it is about to be sent, and against the variant it is sent on; it also leaves the queue when its caller cancels (rejected with the reason), the reference - fails (the cause's error) or is released (`released`). A `host.hold` released before `load` resolved makes nothing. + fails (the cause's error) or is released (`released`). A `host.hold` released before the publication makes nothing. - A local reference follows its object's state members through member sources (`member-source.ts`, one per object and member): when a new attempt goes live every member re-binds by name to it (watches get one frame from the last value, other objects' derives over the member recompute, a failed one included), as a remote reference's re-hold does. The keys a reference holds on a keyed member (`at(key)`, `watch(key)`) follow the object the same way (`keyed-members.ts`, one source per member and key; the new attempt's member loads them before it goes live). A key - whose first load is still in flight when its object fails or ends is given up: the activation waiting on it fails - `transient` (R18) and retries on the next attempt; the late pin is released once. + whose first load is still in flight when its object fails or ends is given up: the parked attempt waiting on it shows + `failed { waiting }` and loads it again at the next change of that reference (its object's new attempt going live) + or a `ref.retry()`; the late pin is released once. - `ref.retry()`: a failed object (broken too) gets an attempt now, its backoff window skipped; every reference to it - goes `loading`. Ignored while an attempt runs, once live and once released; over a link it is a `retry` message. + goes `loading`. One parked `failed { waiting }` passes it on (above). Ignored while an attempt runs otherwise, once + live and once released; over a link it is a `retry` message carrying its operation. Rules: @@ -402,23 +429,48 @@ Rules: `self.get` in a remote object's factory as that object's principal, in a local object's factory as the host's. Over a route link they travel with the hold; over the parent link the parent acts as this host (upward is a boundary: a worker that acts for a user towards the server passes the user as data and the server object decides). -- **Activation**, one attempt: the factory; singleton documents hydrated; references bound and live; natives opened one - after another in declaration order (an opener may read documents, references and earlier natives, not derives); - members checked; derives computed; live. An attempt not live within `readyTimeout` is abandoned: its openers' - contexts are aborted, what it opened is closed (`end`), an open resolving later is closed at once. +- **Activation**, one attempt: the factory; singleton documents hydrated; its requests made (references bound, + registries read); parked until every dependency is live (below); then its own steps: natives opened one after + another in declaration order (an opener may read documents, references and earlier natives, not derives), followed + sources attached, members checked, derives computed; live. An attempt whose own steps do not finish within + `readyTimeout` (the time parked not counted) is abandoned: its openers' contexts are aborted, what it opened is + closed (`end`), an open resolving later is closed at once. +- **Parked attempts** (dependency model §3, slice 7a; `dependencies.ts`): a dependency is a `self.get` reference + (local or remote), a `self.registry` value or a keyed pin (`ref.member.at(key)`, loaded once the reference is live); + `self.follow` is an own step. While one is not live the attempt is parked and its object shows `loading` (all under + way), `failed { waiting }` with `unavailable` "waiting for X: " (X the first failed one, the root cause + found through X's message; always these two, so a ring of waiting attempts settles after one round; within 500 + characters, X's label clipped first; X's error as `cause`), or fails `broken` with the error of a dependency that + failed for a cycle, for authority (`forbidden`: an `authorize` refused it) or ended. A getter's deadline rejects + with the last cause it showed, also after the dependency fell back to `loading`. No timer runs (no dependency deadline, no backoff; the `readyTimeout` + deadline paused); the factory never runs again; every change of a dependency (its state, a new cause by value, its + first outcome) makes it check again once after the batch that made it; once all are live it goes on to its own + steps, and a dependency leaving `live` after that changes nothing. `ref.retry()` on a parked `failed { waiting }` + object passes on to its failed dependencies once per retry operation (a uuidv7 that travels in the `retry` + message with the instance ids of the objects it passed through: one already in that list drops it, so a ring across + hosts stops after one lap whatever the number of concurrent retries, and the last few ids an object handled + shortcut a diamond: a `broken` or + `transient` one starts an attempt, a parked one passes it on, a request waiting in an index is left alone, a + refused hold over a link re-runs its start); a `retryOn` wake does nothing while parked. A parked object whose last reference goes ends at once. +- **Authoring rules against cycles** (dependency model §7; a cycle is a plugin bug, and a ring across hosts started at + the same moment is only reported, `parked_long`): (1) a factory declares only what the object needs, never a type + that needs it back; (2) back-references and occasional lookups go into methods and routines (`ctx.get`); (3) data + flows up through state and registries, not mutual `self.get`; (4) host-lifetime objects on the server never declare + worker types in their factory. - **Failures** become a `FailureCause` with a ChordError; the kind comes from where it failed: the factory, a singleton's `initial()`, the member check (a native watchable as state included), a first derive compute or a - dependency cycle are `broken`; a throwing open is `transient` unless it threw `broken(error)`; the `readyTimeout` is - `transient`; a dependency that failed (R18) is `transient` with `unavailable` "A: dependency B failed" and B's error - as `cause`. A keyed document's `initial()` runs in a later transaction and fails only that transaction. A cycle - (`a(1) → b(1) → a(1)`) fails every object on it. + dependency cycle are `broken`; a throwing open or a failed `self.follow` attach is `transient` unless it threw + `broken(error)`; the `readyTimeout` is `transient`; a dependency that failed is no failure of the dependent (it + parks `failed { waiting }`, above), one that failed for a cycle, for authority (`forbidden`) or ended makes it + `broken`. A keyed document's + `initial()` runs in a later transaction and fails only that transaction. A cycle (`a(1) → b(1) → a(1)`) fails every + object on it. - **Retries:** one attempt per object at a time. A transient failure is retried after a backoff per identity: 1 s, doubling to 60 s, reset by a success; a new object for an identity inside its window waits for the window's end. - A broken failure is retried only by `ref.retry()` (and reload, slice 7, `retryOn`, slice 8); a new object after its - end tries at once. Only a dependency failure, or a `readyTimeout` while waiting for dependencies, keeps the object's - references; when that dependency goes live (a remote one: bound again after a drop) the object is retried at once. A dependency failure is transient whatever the dependency's kind: a dependent of a broken dependency is - retried with its backoff for as long as it is held. A retry binds its references before the previous attempt's are - released, so a dependency held only by the retrying object never ends in between. + A broken failure is retried only by `ref.retry()` (and reload, slice 7b, `retryOn`, slice 8); a new object after its + end tries at once. A failed attempt releases its references once its natives closed; the next attempt makes new + ones. A dependent of a failed dependency is not retried at all: it is parked and goes on when the dependency is + live (no backoff of its own). - **Ending:** transactions refused and the callers of running calls rejected `released` at once, their handlers' contexts aborted, routines' and job runs' too (their waiters rejected); then the drain (the handlers', routine calls' and runs' own promises, a run's `finish` included, up to `closeTimeout`), the natives closed in reverse @@ -428,8 +480,8 @@ Rules: - Calls check the method against the bound variant first (`invalid/member`), then the arguments against the provider's current version (`invalid/member`, `invalid/args`); on one host arguments and results pass by reference. - **Release** rejects the calls made through the reference with `released` and aborts their handlers' contexts, on one - host as over a link. A get needs the provider's version and fingerprint (`contract_mismatch`); an unserved - type is `unavailable`, a closed host `released`. + host as over a link. A get needs the provider's version and fingerprint (`contract_mismatch`, `waiting`); an + unserved type is `unavailable` (`waiting`); both wait up to the getter's deadline; a closed host is `released`. - Watches taken through a reference end when it is released, even while others keep the object alive. - A failure's error keeps its stack frames' receivers alive until its `stack` is read (V8), its failed attempt among them (natives closed, documents dropped). An object keeps its last failure's error until it goes live or ends; an @@ -470,30 +522,35 @@ compression bomb), `18-routines.ts` (a reconciling `on` with a retry, a periodic job), and (slice 6a) `20-tables.ts` (tables, writes, pages, counts), `21-persisted-documents.ts` (documents across two hosts on one store file), `22-migrations.ts` (`evolveTable` and an eager migration at activation), `23-keyed-watches.ts` (`watch(key)`, `at(key)`, the `retired` flag), `24-keyed-members.ts` (a keyed member pinned per -key by its consumers) and `25-export-import.ts` (an archive in JSONL and an import). +key by its consumers) and `25-export-import.ts` (an archive in JSONL and an import), and (slice 7a) +`31-parked-dependencies.ts` (a missing provider arrives: causes shown, no timers) and `32-startup.ts` (two hosts that +would have deadlocked at startup). ## For Chord's layers (`internal.ts`) For `remote/`: `ChordHost` (`place` decides where an arriving hold goes, here or relayed over a path; `serving` finds the object a hold placed here binds to; `relays` and `relayOrigin`, the origins this host relays holds for by -their number; `pendingLoad`, `closing`, `linkOpened`, `unlink`), `AnyHost` (a host whatever its principal type: `ChordHost

    ` is invariant in `P`), +their number; `published`, `index`/`unindex` (the unbound index), `closing`, `linkOpened`, `unlink`), `AnyHost` (a host whatever its principal type: `ChordHost

    ` is invariant in `P`), `Target` (what a binding is bound to, implemented by `ChordObject` and by `remote/`'s `RemoteObject`), `ChordObject` (`state`, `retry`) and `Holder` (what holds an object: a binding, or a hold over a link: `moved()` when the object's `state` changed, which a new holder also reads itself; `end(error)`), `Binding` (`moved`, `end`), -`CheckedParams`, `Caller` (principal and incarnation of a reference's maker or a hold's origin), `RefState` with `LOADING`, `LIVE`, `RELEASED`, and -`settlement`. Also on `ChordHost` for `remote/`: `revoke` (a parent's `revoke` message), `compression`, `count(kind, +`CheckedParams`, `Caller` (principal and incarnation of a reference's maker or a hold's origin), `RefState` with `LOADING`, `LIVE`, +`RetryOperation` (one `ref.retry()` passing on), `Serving` (what a hold placed here binds to, or why it waits) and +`UnboundRequest` (what the unbound index holds). Also on `ChordHost` for `remote/`: `revoke` (a parent's `revoke` message), `compression`, `count(kind, delta)` (the keep-alive counts), `cycle(chain, id, closing)` (the `invalid/cycle` error naming a chain by labels), `unplaced` (the path of remote references failed for a part past the limits) and `pathOf(Type, params, principal)` (placing one anew on `retry`). Inside the directory: `ChordHost` (types, objects by identity, facets, the batch, the backoff map, the endings new -instances wait for, its links and the heartbeat timer, its routes and the parent path; `reference` checks a -reference's route, `bind` binds it to an object here or a remote object over a path, `acquire` gets one for a -caller), `Destination` (where a reference or an arriving hold goes: here, a route's path or the parent's; whether a -hold served here is placed here), `Route` (one per routed kind: per address its `Path`, at most one running +instances wait for, its links and the heartbeat timer, its routes and the parent path, the publication and the +unbound index; `reference` makes a request's binding, `bind` binds it to an object here or a remote object over a +path or leaves it waiting, `acquire` gets one for a caller), `UnboundIndex` (requests that cannot bind yet, by name), +`checkDependencies` and `ParkedLong` (what a parked attempt's dependencies say; the `parked_long` report), +`Destination` (where a reference or an arriving hold goes, by address: here, a route's path or the parent's; whether +a hold served here is placed here), `Route` (one per routed kind: per address its `Path`, at most one running `resolve`, the backoff per address; an address is kept while it has holders, a link or a running `resolve`), -`ChordObject` (one per identity: its state across attempts, holders, retries, cycle checks with its instance id and -the chain it was created for, lingering, ending and revocation, the count of its running and finishing job runs, +`ChordObject` (one per identity: its state across attempts (what a parked attempt shows), its first outcome, holders, +retries passed on while parked, cycle checks with its instance id and the chain it was created for, lingering, ending and revocation, the count of its running and finishing job runs, which keeps it like a reference: `countRuns`, its contributions while live and its wake signals), `RetryWakes` (an object's `retryOn` subscriptions), `RootScopes` (the open `host.root` scopes, sealed once each), `Activity` (`host.activity`: counters, one publication per microtask turn), `References` (one type's presence list), `RevocationKeys` (each principal's keys, diff --git a/packages/chord/src/next/host/attempt.ts b/packages/chord/src/next/host/attempt.ts index 259b6a0cf..1fd546056 100644 --- a/packages/chord/src/next/host/attempt.ts +++ b/packages/chord/src/next/host/attempt.ts @@ -8,7 +8,7 @@ import type { JsonValue } from "../../json.ts"; import type { NativeWatchable, Watchable, WatchEnd } from "../../watch.ts"; import type { Contract, Implementation, ParamsOf } from "../contracts/index.ts"; import { contractData, type ShapeOf } from "../contracts/internal.ts"; -import { ChordError, type FailureCause, isChordError, type Principal } from "../data/index.ts"; +import { ChordError, type FailureCause, type Principal } from "../data/index.ts"; import { asChordError, checkArgs, checkParams, invalid, openFailure } from "../data/internal.ts"; import type { HandleOf, JsonObject, StateDefinition, Tx } from "../documents/index.ts"; import { @@ -22,7 +22,12 @@ import { import type { JobDefinition, JobHandle, JobImplementation } from "../jobs/index.ts"; import { type AnyJobImplementation, Job } from "../jobs/internal.ts"; import type { Contribution, Registry, RegistryValue } from "../registries/index.ts"; -import { type RegistryContribution, type RegistryEdge, RegistryValueSource } from "../registries/internal.ts"; +import { + type RegistryContribution, + type RegistryEdge, + RegistryValueSource, + type RegistryWaiter, +} from "../registries/internal.ts"; import type { Writable } from "../state/index.ts"; import { Derive, type DeriveSource, WritableSource } from "../state/internal.ts"; import type { Columns } from "../store/columns.ts"; @@ -33,6 +38,7 @@ import { type Table, TableHandle } from "../store/table-handle.ts"; import { prepareTables } from "../store/table-migration.ts"; import { CallContext, type Caller, type Ctx, withLinkedAbortSignal } from "./ctx.ts"; import { Deadline } from "./deadline.ts"; +import { checkDependencies, type Dependency, ParkedLong, waitingOn } from "./dependencies.ts"; import { type FollowAttach, FollowSource } from "./follow.ts"; import type { HostError } from "./host.ts"; import { attachDerives, checkImplementation, contributionOf, type KeyedBinding, type Members } from "./members.ts"; @@ -46,7 +52,8 @@ import { type NativeSlot, } from "./native.ts"; import type { ChordObject } from "./object.ts"; -import type { Binding, CheckedParams, Reference, Target } from "./reference.ts"; +import type { Failed } from "./ref-state.ts"; +import type { Binding, CheckedParams, Reference, RetryOperation } from "./reference.ts"; import { type RetrySource, retrySource } from "./retry-on.ts"; import { type OnHandler, @@ -84,8 +91,9 @@ export interface Self

    extends RoutineDeclaratio get(Type: C, params: ParamsOf): Reference; /** * Registry `R`'s composed value as this plugin's version of `R` sees it: a declared dependency, as `get` is - * (activation waits until it has a value; with none it fails `transient` and is retried when one arrives, R18). - * Read `.value` at use; once live, a later failure of the registry keeps its last good value. + * (the attempt parks until it has a usable value: `failed { waiting }` while it has none, woken when it has). Read + * `.value` at use; once live, a later failure of the registry keeps its last good value while every contribution it + * was built from is still in the table. */ registry(R: Registry): RegistryValue; /** @@ -150,17 +158,14 @@ export type Factory = Shape ? (params: ParamsOf, self: keyof ParamsOf extends never ? StoreSelf

    : Self

    ) => Implementation> : (params: ParamsOf, principal: P, self: Self

    ) => Implementation>; -/** How an activation fails on a declared registry without a value (`#readRegistries`). */ -type UnreadRegistry = Extract; - /** No declarations of a kind: the shared empty list. */ const NONE: readonly never[] = []; /** An attempt's registry reads, contributions and wake signals, and its writables (slice 8). */ type RegistryDeclarations = { - /** The registry values it declared (`self.registry`), registered as read during activation (`reads`). */ + /** The registry values it declared (`self.registry`): dependencies, as its references are. */ readonly registries: RegistryValueSource[]; - /** Releases its registry reads (with its references). */ + /** Releases its registry reads (with its references): a read keeps an older version's mapping. */ readonly reads: (() => void)[]; /** Its wake signals (`self.retryOn`), handed to its object right after the factory. */ readonly retrySources: RetrySource[]; @@ -171,26 +176,29 @@ type RegistryDeclarations = { contributions: readonly RegistryContribution[]; }; -/** How an attempt's activation ended. `dependency_failed` (R18) keeps the attempt's references. */ +/** How an attempt's activation ended. */ type Activation = | { readonly kind: "live" } | { readonly kind: "failed"; readonly cause: FailureCause } - | { readonly kind: "dependency_failed"; readonly cause: FailureCause } | { readonly kind: "ended"; readonly error: ChordError }; +/** How an activation that did not go live ended. */ +type NotLive = Exclude; + /** The error of an object, or a getter, that waited `ms` without the object going live. */ export function notLive(label: string, ms: number): ChordError { return new ChordError({ code: "unavailable", message: `${label}: did not become live within ${ms} ms` }); } /** - * One attempt of an object. Activation: factory, documents hydrated (the object's slots), references bound (before the - * previous attempt's are released) and live, natives opened one after another, members checked, derives computed; - * bounded by the host's `readyTimeout`. Ending: sealed at once (calls aborted, its transactions refused and the one - * holding the line discarded, its job invocations cut off), then the drain, the closes in reverse, derives detached; - * its object releases its references. Documents and job runs stay on the object. + * One attempt of an object. Activation: factory, documents hydrated (the object's slots), references bound, then parked + * until every dependency is live (no timer: the deadline paused), then its own steps: natives opened one after + * another, followed sources attached, members checked, derives computed; bounded by the host's `readyTimeout`. + * Ending: sealed at once (calls aborted, its transactions refused and the one holding the line discarded, its job + * invocations cut off), then the drain, the closes in reverse, derives detached; its object releases its references. + * Documents and job runs stay on the object. */ -export class Attempt implements StoreSelf, KeyDeclarer { +export class Attempt implements StoreSelf, KeyDeclarer, RegistryWaiter { readonly #object: ChordObject; /** The object's context: the natives' closes run under it. */ readonly #context: Context; @@ -228,18 +236,21 @@ export class Attempt implements StoreSelf, KeyDeclarer { * the call path costs no more than before the cleanups. */ readonly #running = new Map, (() => void) | undefined>(); - /** The previous attempt while it still holds its references (R18): released once this one has bound its own. */ - #previous: Attempt | undefined; /** What activation returns once the attempt was abandoned: a timeout, a cycle through its object, or its end. */ - #abandoned: Exclude | undefined; + #abandoned: NotLive | undefined; + /** Its dependencies while it is parked on them (the dependency check); `undefined` otherwise. */ + #parked: readonly Dependency[] | undefined; + /** A dependency changed since the last check: the parked attempt checks again (one check per batch of changes). */ + #moved = false; + /** What a parked attempt waiting for a change waits on; resolved after the change's batch. */ + #changed: PromiseWithResolvers | undefined; /** A followed source's end while this attempt activated: it fails as it goes live (if its activation returned). */ #endedEarly: ChordError | undefined; /** Its registry reads, contributions, wake signals and writables, made by the first: most attempts have none. */ #registryDeclarations: RegistryDeclarations | undefined; - constructor(object: ChordObject, previous: Attempt | undefined, context: Context) { + constructor(object: ChordObject, context: Context) { this.#object = object; - this.#previous = previous; this.#context = context; this.#opening = withAbortSignal(this.#abort.signal, context); } @@ -453,7 +464,7 @@ export class Attempt implements StoreSelf, KeyDeclarer { } /** Abandons the activation with `outcome` (once): its waits stop and the openers' contexts are aborted. */ - abandon(outcome: Exclude): void { + abandon(outcome: NotLive): void { if (this.#abandoned !== undefined) return; this.#abandoned = outcome; this.#abort.abort(outcome.kind === "ended" ? outcome.error : outcome.cause.error); @@ -498,6 +509,11 @@ export class Attempt implements StoreSelf, KeyDeclarer { }); } + /** Whether it is parked on its dependencies (the dependency check runs). */ + get isParked(): boolean { + return this.#parked !== undefined; + } + /** Its contributions, resolved by its activation: in the registries' tables while its object is live. */ get contributions(): readonly RegistryContribution[] { return this.#registryDeclarations?.contributions ?? NONE; @@ -624,9 +640,8 @@ export class Attempt implements StoreSelf, KeyDeclarer { if (declarations !== undefined) for (const writable of declarations.writables) writable.detach(ended); } - /** Releases its references and registry reads, and the previous attempt's if this one never bound its own. */ + /** Releases its references and registry reads. */ releaseReferences(): void { - this.#releasePrevious(); for (const { binding } of this.#references) binding.release(); const declarations = this.#registryDeclarations; if (declarations !== undefined) for (const release of declarations.reads.splice(0)) release(); @@ -635,8 +650,7 @@ export class Attempt implements StoreSelf, KeyDeclarer { /** The activation steps; each catches its own failures with their kind (the table in slice 2's spec). */ async #activate(deadline: Deadline): Promise { let implementation: unknown; - let dependencies: Target[]; - let unread: UnreadRegistry | undefined; + const dependencies: Dependency[] = []; try { implementation = this.#declare(); // Right after the factory, before anything can fail: a failed attempt keeps its wake signals. @@ -650,49 +664,158 @@ export class Attempt implements StoreSelf, KeyDeclarer { // A persisted job's runs, loaded once per object: rebuilt before anything runs. for (const job of this.#jobs.values()) await job.load(this.#opening); const { host } = this.#object; - dependencies = this.#references.map(({ binding, contract, params }) => - host.bind(binding, contract, params, this.#opening), - ); - for (const dependency of dependencies) this.#object.assertNoCycle(dependency, this.#opening); - if (this.#registryDeclarations !== undefined) unread = this.#readRegistries(this.#registryDeclarations); + for (const { binding, contract, params } of this.#references) { + host.bind(binding, contract, params, this.#opening); + dependencies.push({ kind: "reference", binding }); + } + // Once all are bound: a dependency made here may have bound back to this object meanwhile. + for (const dependency of dependencies) { + const target = dependency.kind === "reference" ? dependency.binding.target : undefined; + if (target !== undefined) this.#object.assertNoCycle(target, this.#opening); + } + const declarations = this.#registryDeclarations; + for (const value of declarations?.registries ?? NONE) { + // Read from here until its references are released: an older version's mapping is kept meanwhile. + declarations?.reads.push(value.read()); + dependencies.push({ kind: "registry", value }); + } } catch (error) { return this.#broken(error); + } + // Parked only when something is not live yet (most activations find their dependencies live). + if ((dependencies.length > 0 && checkDependencies(dependencies).kind !== "live") || this.#pinsKeys()) { + const parked = await this.#park(dependencies, deadline); + if (parked !== undefined) return parked; + } else { + // Its own steps start in a later turn: an object ended in the turn it was made opens nothing. + await undefined; + } + return this.#ownSteps(implementation); + } + + /** + * Parks while a dependency is not live (dependency model §3), with no timer: the deadline paused, `parked_long`'s + * report the only clock. Checks at every change of a dependency (after its batch) and shows what it found on its + * object; once all are live, the keys the factory pinned on them are loaded (a failed load parks it `waiting` + * until the next change or a retry). Resolves `undefined` once all are live and pinned, else with how it ends: + * abandoned, or broken (a cycle, an ended dependency). + */ + async #park(dependencies: readonly Dependency[], deadline: Deadline): Promise { + const object = this.#object; + deadline.pause(); + this.#parked = dependencies; + object.waitsFor(dependencies); + // Waiters on a registry are woken when its value turns usable or its failure changes. + const waits: (() => void)[] = []; + for (const each of dependencies) if (each.kind === "registry") waits.push(each.value.wait(this)); + const long = new ParkedLong(object); + try { + for (;;) { + this.#moved = false; + const check = checkDependencies(dependencies); + if (check.kind === "broken") + return this.#abandoned ?? { kind: "failed", cause: { kind: "broken", error: check.error } }; + if (check.kind === "live") { + const pinned = await this.#pinKeys(); + if (this.#abandoned !== undefined) return this.#abandoned; + // Changed while the keys loaded: checked again. + if (pinned === undefined && !this.#moved) { + // Its own steps follow: `loading` until its outcome (a failure shows as the failure, not this). + object.unparked(this); + return undefined; + } + if (pinned !== undefined) object.parked(this, pinned, true); + long.stop(); + } else { + object.parked(this, check.shown, check.firstPass); + long.waitingOn(check.remoteLoading); + } + if (!(await this.#nextChange())) return this.#abandoned!; + } } finally { - // Bound first: a dependency held only by the previous attempt never ends in between. - this.#releasePrevious(); + long.stop(); + for (const release of waits) release(); + this.#parked = undefined; + object.waitsFor(NONE); + deadline.resume(); } - // No value after `load`'s setups: none comes before a first composition (which retries it, R18) or a reload. - if (unread !== undefined) return this.#abandoned ?? unread; + } + + /** + * Waits for the next change of a dependency (at once when one came during the check); `false` when the attempt was + * abandoned meanwhile (its end, the only thing that stops the wait). + */ + async #nextChange(): Promise { + if (this.#moved) return true; + this.#changed = Promise.withResolvers(); try { - await Promise.all(dependencies.map((dependency) => this.#object.waitFor(dependency, this.#opening))); - } catch (error) { - const abandoned = this.#abandoned; - // A timeout while waiting keeps the references (R18): a dependency that goes live later retries this object - // at once (a remote one whose link dropped and is held again). - if (abandoned?.kind === "failed" && abandoned.cause.kind === "transient") { - return { kind: "dependency_failed", cause: abandoned.cause }; - } - if (abandoned !== undefined) return abandoned; - // `waitFor` rejects with a ChordError naming the dependency, its error as the cause; or with a cycle's - // `invalid/cycle`, which fails this object broken too (proposal, slice 4b: an exception to R18, since every - // retry would meet the same cycle), keeping no references. - const failed = error as ChordError; - if (failed.code === "invalid" && failed.reason === "cycle") { - return { kind: "failed", cause: { kind: "broken", error: failed } }; - } - return { kind: "dependency_failed", cause: { kind: "transient", error: failed } }; + await awaitWithContext(this.#changed.promise, this.#opening); + return true; + } catch { + return false; + } finally { + this.#changed = undefined; } - try { - // The keys the factory pinned on its dependencies' keyed members, loaded before this object goes live. - for (const { binding } of this.#references) { + } + + /** + * Loads the keys the factory pinned on its dependencies' keyed members (`at(key)`); `undefined` once all are, else + * what the parked attempt shows for the first that failed to load (`waiting` on its reference). + */ + async #pinKeys(): Promise { + for (const { binding } of this.#references) { + try { const pinning = binding.pinKeys(this.#opening); if (pinning !== undefined) await pinning; + } catch (error) { + return waitingOn(binding.label, asChordError(error, binding.label)); } - } catch (error) { - // A pin load that failed its provider is a dependency failure (rule 6, R18). - const cause: FailureCause = { kind: "transient", error: asChordError(error, this.#object.label) }; - return this.#abandoned ?? { kind: "dependency_failed", cause }; } + return undefined; + } + + /** + * A dependency of this parked attempt changed (its state, its cause, its first outcome): it checks again after the + * batch that changed it, once however many changed. A change while it checks or loads pinned keys is remembered + * (`#moved`); one outside parking costs a flag. + */ + dependencyMoved(): void { + if (this.#moved) return; + this.#moved = true; + if (this.#changed !== undefined) queueMicrotask(() => this.#changed?.resolve()); + } + + /** A registry it is parked on closed with its host: the parked attempt ends with `error`. */ + closed(error: ChordError): void { + this.abandon({ kind: "ended", error }); + } + + /** Whether its factory pinned keys on its dependencies' keyed members (`at(key)`), which load while it is parked. */ + #pinsKeys(): boolean { + for (const { binding } of this.#references) if (binding.pinsKeys) return true; + return false; + } + + /** + * `ref.retry()` on its parked object (`failed { waiting }`): passed on to every failed dependency (a broken or + * transient one retries, a parked one passes it on, a request waiting in an index is left alone), as `operation`; + * then it checks again (a key that failed to load is loaded again). + */ + retryDependencies(operation: RetryOperation): void { + for (const dependency of this.#parked ?? NONE) { + if (dependency.kind === "reference" && dependency.binding.refState.kind === "failed") { + dependency.binding.retry(operation); + } + } + this.dependencyMoved(); + } + + /** + * The steps after the dependency check (its own: natives, followed sources, the member check, derives, the keys + * references pin on its keyed members), under the deadline. A dependency that leaves `live` meanwhile changes + * nothing (R18 after live). + */ + async #ownSteps(implementation: unknown): Promise { for (const native of this.#natives) { // An attempt abandoned before this open (ended while it waited) closed its natives already: none opens late. if (this.#abandoned !== undefined) return this.#abandoned; @@ -711,14 +834,16 @@ export class Attempt implements StoreSelf, KeyDeclarer { return this.#abandoned ?? { kind: "failed", cause }; } } + let members: Members; try { const checked = checkImplementation(this.#object.served.contract, implementation); - this.#members = checked.members; + members = checked.members; + this.#members = members; this.#variant = checked.variant; if (this.#registryDeclarations !== undefined) this.#registryDeclarations.contributions = this.#resolveContributions(this.#registryDeclarations); const { host, label } = this.#object; - attachDerives(this.#derives, this.#members.state, (member) => (error) => { + attachDerives(this.#derives, members.state, (member) => (error) => { // During activation a throwing derive fails the attempt, which `activation_failed` reports alone. if (!this.#isLive) return; host.report({ kind: "derive_failed", object: label, ...(member === undefined ? {} : { member }), error }); @@ -729,40 +854,15 @@ export class Attempt implements StoreSelf, KeyDeclarer { } try { // The keys references pin on the object's keyed members, loaded on this attempt's before it goes live. - const preparing = this.#object.prepareKeys(this.#members.keyed, this.#opening); + const preparing = this.#object.prepareKeys(members.keyed, this.#opening); if (preparing !== undefined) await preparing; } catch (error) { const cause: FailureCause = { kind: "transient", error: asChordError(error, this.#object.label) }; - return this.#abandoned ?? { kind: "dependency_failed", cause }; + return this.#abandoned ?? { kind: "failed", cause }; } return this.#abandoned ?? { kind: "live" }; } - /** - * Registers this attempt's object as a reader of every registry it declared (woken by a first value, R18, until its - * references are released); returns how the activation fails on the first with no value, `undefined` when every - * one has a value. A version the provider does not serve (S2) is final until a reload: broken, the - * `contract_mismatch` itself the cause. Otherwise (no provider, a first `compose` that threw) a value may still come: - * `transient` (R18, the references kept), naming the registry, its error as the cause. - */ - #readRegistries({ registries, reads }: RegistryDeclarations): UnreadRegistry | undefined { - let unread: UnreadRegistry | undefined; - for (const value of registries) { - reads.push(value.read(this.#object)); - const start = value.start(); - if (unread !== undefined || start.kind === "value" || start.last !== undefined) continue; - const { error } = start; - if (isChordError(error, "contract_mismatch")) { - unread = { kind: "failed", cause: { kind: "broken", error } }; - continue; - } - const message = `${this.#object.label}: registry ${value.name} has no value`; - const cause = new ChordError({ code: "unavailable", message }, { cause: error }); - unread = { kind: "dependency_failed", cause: { kind: "transient", error: cause } }; - } - return unread; - } - /** Its declared contributions, as Chord follows them (its references are live now), by its plugin and type. */ #resolveContributions({ declared }: RegistryDeclarations): readonly RegistryContribution[] { const { served, host } = this.#object; @@ -847,11 +947,6 @@ export class Attempt implements StoreSelf, KeyDeclarer { return this.#abandoned ?? { kind: "failed", cause }; } - #releasePrevious(): void { - this.#previous?.releaseReferences(); - this.#previous = undefined; - } - /** The registry edges its declared contributions would add (another registry's value contributed). */ #pendingEdges(): RegistryEdge[] { const edges: RegistryEdge[] = []; diff --git a/packages/chord/src/next/host/backoff.ts b/packages/chord/src/next/host/backoff.ts index bae0a91df..383b04b42 100644 --- a/packages/chord/src/next/host/backoff.ts +++ b/packages/chord/src/next/host/backoff.ts @@ -58,7 +58,7 @@ export class Backoff { return entry?.timer === undefined ? undefined : entry.until; } - /** Ends the window at once (its object is retried now, R18); the delay stays for the next failure. */ + /** Ends the window at once (its object is retried now: `ref.retry()`, a wake); the delay stays for the next failure. */ skip(identity: string): void { const entry = this.#entries.get(identity); clearTimeout(entry?.timer); diff --git a/packages/chord/src/next/host/dependencies.ts b/packages/chord/src/next/host/dependencies.ts new file mode 100644 index 000000000..73381e42e --- /dev/null +++ b/packages/chord/src/next/host/dependencies.ts @@ -0,0 +1,142 @@ +// Parked attempts (dependency model §3, §7): the dependency check, what an attempt's declared dependencies (`self.get` +// references and `self.registry` values) say at one moment: all live, or what the parked attempt shows while one is not +// (`loading`, `failed { waiting }` naming the nearest blocker and the root cause, or `broken` for a cycle, authority or +// an ended dependency), whether this host's first pass can count it (no blocker here still without its first outcome, decision +// (a)), and the remote blocker that is only loading; and the `parked_long` report a long wait on that blocker makes. + +import { type ChordError, isChordError } from "../data/index.ts"; +import { asChordError, waitingFor } from "../data/internal.ts"; +import type { RegistryValueSource } from "../registries/internal.ts"; +import type { ChordObject } from "./object.ts"; +import { type Failed, LOADING, type RefState } from "./ref-state.ts"; +import type { Binding } from "./reference.ts"; + +/** One declared dependency: a reference (`self.get`) or a registry's value (`self.registry`). */ +export type Dependency = + | { readonly kind: "reference"; readonly binding: Binding } + | { readonly kind: "registry"; readonly value: RegistryValueSource }; + +/** + * What the dependencies say. `live`: every one is live (hydrated: a reference's members have values). `parked`: what + * the attempt shows (`loading`, or `failed { waiting }` naming its first failed blocker), whether its first pass is + * over (`firstPass`: no blocker, loading or failed, is an object here still without its first outcome), and a remote + * blocker whose hold is on a link and only loading (its label), for `parked_long`. `broken`: one failed for a cycle + * (`invalid/cycle`), for authority (`forbidden`) or ended: it never comes back by itself. + */ +export type Check = + | { readonly kind: "live" } + | { + readonly kind: "parked"; + readonly shown: RefState; + readonly firstPass: boolean; + readonly remoteLoading: string | undefined; + } + | { readonly kind: "broken"; readonly error: ChordError }; + +const LIVE: Check = { kind: "live" }; + +/** Checks `dependencies` in declaration order (the first failed one is the one the attempt names). */ +export function checkDependencies(dependencies: readonly Dependency[]): Check { + let failed: Failed | undefined; + let firstPass = true; + let remoteLoading: string | undefined; + let loading = false; + for (const dependency of dependencies) { + if (dependency.kind === "registry") { + const { value } = dependency; + const start = value.start(); + if (start.kind === "value" || start.last !== undefined) continue; + failed ??= waitingOn(`registry ${value.name}`, asChordError(start.error, `registry ${value.name}`)); + continue; + } + const { binding } = dependency; + const state = binding.refState; + switch (state.kind) { + case "live": + continue; + case "ended": + return { kind: "broken", error: state.error }; + case "failed": { + const { error } = state.cause; + if (healsNever(error)) return { kind: "broken", error }; + failed ??= waitingOn(binding.label, error); + // Parked `waiting` on something here still in its first attempt: no first outcome yet (fix round 1). + if (binding.target?.blocksFirstPass === true) firstPass = false; + continue; + } + case "loading": { + loading = true; + const target = binding.target; + if (target?.blocksFirstPass === true) firstPass = false; + if (target?.heldOnLink === true) remoteLoading ??= target.label; + } + } + } + if (failed === undefined && !loading) return LIVE; + return { kind: "parked", shown: failed ?? LOADING, firstPass, remoteLoading }; +} + +/** + * Whether a dependency failed with `error` never heals by itself, so its dependent breaks too: a cycle + * (`invalid/cycle`: it would wait on a ring) or authority (`forbidden`: an `authorize` that refused, M§3.6; only + * `retry()` or a reload helps). + */ +function healsNever(error: ChordError): boolean { + return (isChordError(error, "invalid") && error.reason === "cycle") || isChordError(error, "forbidden"); +} + +/** A parked attempt's state for a blocker `label` that failed with `error`. */ +export function waitingOn(label: string, error: ChordError): Failed { + return { kind: "failed", cause: waitingFor(label, error), retryAt: undefined }; +} + +/** + * The `parked_long` report of one parking episode (dependency model §7): once, when the attempt has waited + * `readyTimeout` on a remote blocker that is only loading with its hold on a link (a ring across hosts started at the + * same moment stays parked so; nothing fails or retries). The clock, and the report's `since`, start when such a + * blocker is there and stop when none is (a link that drops, a path still without one): only the time a remote + * attempt answered for the wait counts. Its one timer runs only meanwhile. + */ +export class ParkedLong { + readonly #object: ChordObject; + /** When the attempt began to wait on a remote blocker that is only loading on a link, while it does. */ + #since: number | undefined; + #timer: ReturnType | undefined; + /** The remote blocker it would name now. */ + #blocker: string | undefined; + #reported = false; + + constructor(object: ChordObject) { + this.#object = object; + } + + /** What the check found: `blocker`, a remote attempt that is only loading on a link, or none. */ + waitingOn(blocker: string | undefined): void { + this.#blocker = blocker; + if (blocker === undefined) { + this.stop(); + return; + } + if (this.#reported || this.#timer !== undefined) return; + this.#since ??= Date.now(); + const due = this.#since + this.#object.host.timeouts.ready - Date.now(); + this.#timer = setTimeout(() => this.#report(), Math.max(0, due)); + } + + /** The episode ended, or no remote blocker is loading on a link now: the clock and its timer stop. */ + stop(): void { + clearTimeout(this.#timer); + this.#timer = undefined; + this.#since = undefined; + } + + #report(): void { + this.#timer = undefined; + const waitingFor = this.#blocker; + const since = this.#since; + if (this.#reported || waitingFor === undefined || since === undefined) return; + this.#reported = true; + const object = this.#object; + object.host.report({ kind: "parked_long", object: object.label, waitingFor, since }); + } +} diff --git a/packages/chord/src/next/host/destination.ts b/packages/chord/src/next/host/destination.ts index d92fbb0f1..8a372b257 100644 --- a/packages/chord/src/next/host/destination.ts +++ b/packages/chord/src/next/host/destination.ts @@ -1,6 +1,6 @@ -// Destination (§3, section 1's table): where a reference made on a host, or a hold arriving on one of its links, goes: -// here, over the route of its host address, or over the parent link; and the check that an object served here lives -// here. +// Destination (§3, section 1's table; dependency model §1): where a reference made on a host, or a hold arriving on one +// of its links, goes, by address alone: here when the address is this host's (served here or not), else over the +// route of its host address's kind, or over the parent link; and the check that an object served here lives here. import type { Context } from "../../context/index.ts"; import type { Address, Contract, ParamsOf } from "../contracts/index.ts"; @@ -9,7 +9,6 @@ import { ChordError, type Principal } from "../data/index.ts"; import { invalid } from "../data/internal.ts"; import type { WireAddress } from "../link/internal.ts"; import type { HoldRequest, HostLink, Path } from "../remote/internal.ts"; -import type { Served } from "./object.ts"; import type { Route } from "./route.ts"; /** Where something is placed: here, or over a path (with the host address it was computed or named with). */ @@ -17,47 +16,42 @@ export type Placed = | { readonly kind: "here" } | { readonly kind: "path"; readonly path: Path; readonly address: WireAddress }; -/** A host's destinations: its address, its served types, its routes and its parent path, as the host keeps them. */ +/** A host's destinations: its address, its routes and its parent path, as the host keeps them. */ export class Destination { readonly #address: Address | undefined; - readonly #types: ReadonlyMap; readonly #routes: ReadonlyMap; readonly #parent: Path; - constructor( - address: Address | undefined, - types: ReadonlyMap, - routes: ReadonlyMap, - parent: Path, - ) { + constructor(address: Address | undefined, routes: ReadonlyMap, parent: Path) { this.#address = address; - this.#types = types; this.#routes = routes; this.#parent = parent; } /** - * Where a reference to `Type` with `params` made as `principal` goes: here for a local type (served here: the host - * refused it otherwise) or a remote type served here whose host address is this host's; otherwise over the route of - * its host address's kind, or over the parent link (the host address decides where an object lives: a worker - * reaches another worker's object of a type it serves too). Throws `invalid/host_address` for a host address whose - * params do not fit its kind, `forbidden` when the route refuses, and what a `hostAddress` or `authorize` throws. + * Where a reference to `Type` with `params` made as `principal` goes: here for a local type, and for a remote type + * whose host address is this host's, served here or not (the check against what is served is the host's); otherwise + * over the route of its host address's kind, or over the parent link (the host address decides where an object + * lives: a worker reaches another worker's object of a type it serves too). Throws `invalid/host_address` for a host + * address whose params do not fit its kind, `forbidden` when the route refuses, and what a `hostAddress` or + * `authorize` throws. */ reference(Type: Contract, params: ParamsOf, principal: Principal | undefined, context: Context): Placed { if (Type.kind === "local") return HERE; const address = hostAddressOf(Type, params); - if (this.#types.has(Type.name) && this.#isHere(address)) return HERE; + if (this.#isHere(address)) return HERE; return { kind: "path", path: this.#pathTo(address, principal, context) ?? this.#parent, address }; } /** - * Where a hold arriving on `link` acting as `principal` goes: here when its type is served here; over the route of - * the address it names; else over the parent link (waiting for it while it is down), unless it came on the parent - * link or this host never had one (`unavailable`). Throws as `reference` does. + * Where a hold arriving on `link` acting as `principal` goes, by the address it names: here when it is this host's + * (a relay never captures a hold for serving the same name); over the route of its kind; else over the parent link + * (waiting for it while it is down), unless it came on the parent link or this host never had one (`unavailable`). + * Throws as `reference` does. */ hold(request: HoldRequest, link: HostLink, principal: Principal | undefined, context: Context): Placed { - if (this.#types.has(request.type)) return HERE; const { address } = request; + if (this.#isHere(address)) return HERE; const routed = this.#pathTo(address, principal, context); if (routed !== undefined) return { kind: "path", path: routed, address }; if (link.role === "parent" || !this.#parent.linked) { @@ -67,15 +61,16 @@ export class Destination { } /** - * Throws `invalid/host_address` for a hold of `Type` with `params` (checked) served here but whose host address is - * another host (misrouted; never forwarded). The host address is computed here from the params: the address a hold - * names only routes it. + * The provider's own check of a request placed here for the remote `Type` it serves, with `params` (checked): + * `invalid/host_address` when its `hostAddress` puts the object on another host (code skew: the consumer computed + * this host), `undefined` when it lives here. A `hostAddress` that throws throws (the provider's own code). The + * address a request names only routes it. */ - assertHere(Type: Contract, params: ParamsOf): void { + misplaced(Type: Contract, params: ParamsOf): ChordError | undefined { const address = hostAddressOf(Type, params); - if (this.#isHere(address)) return; + if (this.#isHere(address)) return undefined; // A host serving a remote type has an address (`serve` refuses one without). - throw invalid("host_address", `${Type.name}: placed on ${address.kind}, this host is ${this.#address!.key}`); + return invalid("host_address", `${Type.name}: placed on ${address.kind}, this host is ${this.#address!.key}`); } /** diff --git a/packages/chord/src/next/host/host.ts b/packages/chord/src/next/host/host.ts index 5612e562a..d7d6ef47c 100644 --- a/packages/chord/src/next/host/host.ts +++ b/packages/chord/src/next/host/host.ts @@ -28,6 +28,7 @@ import { import { binaryCodec, type Codec, type Compression, type Link, type PartLimits } from "../link/index.ts"; import { linkLimits } from "../link/internal.ts"; import type { Registry, RegistryValue } from "../registries/index.ts"; +import type { RegistryValueSource } from "../registries/internal.ts"; import { type RegistryContribution, type RegistryEdge, @@ -41,6 +42,7 @@ import { Path, type ProviderSide, RemoteObject, + type Upstream, } from "../remote/internal.ts"; import { Batch, type BatchError } from "../state/internal.ts"; import type { ChordStore, Store } from "../store/store.ts"; @@ -58,6 +60,7 @@ import { Binding, type CheckedParams, type Held, type Reference, type Target } f import { RevocationKeys, type RevokedBy } from "./revocation.ts"; import { type RootOptions, RootScopes } from "./root.ts"; import { Route, type RouteOptions } from "./route.ts"; +import { UnboundIndex, type UnboundRequest } from "./unbound.ts"; import { uuidv7 } from "./uuid.ts"; /** What `createHost` takes; `R` is the app's revocation type (JSON). */ @@ -151,7 +154,12 @@ export type HostError = readonly restarts: number; } /** A blocked run was dropped without `finish`: aborted, or its object ended. */ - | { readonly kind: "job_dropped"; readonly object: string; readonly job: string; readonly id: string }; + | { readonly kind: "job_dropped"; readonly object: string; readonly job: string; readonly id: string } + /** + * An attempt parked `readyTimeout` (since `since`, `Date.now()` time) while `waitingFor`, a remote attempt, is only + * loading: once per parking episode, a diagnostic of a ring across hosts started at the same moment (nothing fails). + */ + | { readonly kind: "parked_long"; readonly object: string; readonly waitingFor: string; readonly since: number }; /** * What `accept` and `connect` take: the principal of the host at the other end, as this host stamps it; `trusted` skips @@ -184,22 +192,26 @@ export type AnyHost = Omit { /** - * Runs every facet's setup, then activates every served type with empty params and resolves when each has had its - * first outcome, live or failed (the start barrier; failures go to `onError`). A setup that throws rejects it and - * closes the host. Runs once. + * Runs every facet's setup and publishes what they declared in one batch (the types and registries served, the + * registries composed once, the contributions); requests that waited for the publication are placed after that + * batch. Then activates every served type with empty params and resolves when each has had its first outcome (the + * start barrier, decision (a)): live, failed by its own cause, parked with no blocker here still without its first + * outcome, or ended (failures go to `onError`). A setup that throws rejects it and closes the host. Runs once. */ load(facets: readonly { readonly name: string; readonly facet: Facet

    }[], context: Context): Promise; /** - * A reference once its object is live; waits for `load` first. It acts as the host's principal and goes where its - * type's host address puts it: here, over a route, or over the parent link (`Destination`). A broken failure - * rejects at once; through transient ones it waits until its deadline, then rejects with the last failure's error. + * A reference once its object is live (and hydrated). It acts as the host's principal and goes where its type's host + * address puts it: here, over a route, or over the parent link (`Destination`). It waits while the reference is + * `loading` (before the host's publication too), `failed { transient }` or `failed { waiting }` until its + * `readyTimeout`, then rejects with the last cause and releases the reference; a broken failure rejects at once. */ get(Type: C, params: ParamsOf, context: Context, options?: GetOptions): Promise>; /** * A reference at once, following its object until released: loading, live, failed (retried as its object is) and - * loading again while its link is down; it never rejects. Throws at once for invalid params (also before `load`) - * and a local type not served here; before `load` resolved, a type not served here fails the reference broken once - * it did. Its `variant` is `undefined` until its first binding. + * loading again while its link is down; it never rejects. Throws at once only for invalid params (also before + * `load`). A type not served where it goes is `loading` before the host's publication and `failed { waiting }` after + * it (bound when a publication serves it); a `hostAddress` or route that refuses fails it broken. Its `variant` is + * `undefined` until its first binding. */ hold(Type: C, params: ParamsOf, context: Context, options?: GetOptions): Held; /** @@ -244,11 +256,12 @@ export interface Host

    ; /** - * Registry `R`'s composed value as `R`'s version sees it, for code outside objects; waits for `load` first. Rejects - * when it has no value then (`unavailable/undeclared` without a provider, a first `compose` that threw, a version - * not served here: `contract_mismatch`); none can come before a reload. + * Registry `R`'s composed value as `R`'s version sees it, for code outside objects, once it is usable: waits for the + * host's publication, then while it has no value (`unavailable/undeclared` without a provider, a first `compose` + * that threw, a version not served here: `contract_mismatch`) until its `readyTimeout`, then rejects with the last + * error. */ - registry(R: Registry, context: Context): Promise>; + registry(R: Registry, context: Context, options?: GetOptions): Promise>; /** * Runs `fn` with a call context for code outside objects (a Durable tool, a CLI): the host's principal (or * `options.principal`), this host's incarnation, `context`'s telemetry and signal, `ctx.get`. When `fn` settles or @@ -263,6 +276,14 @@ export interface Host

    ; } +/** + * What a hold placed here binds to (`serving`): its object, or why it waits unbound (`failed { waiting }`: not served, + * another version, or placed elsewhere by the provider's code). + */ +export type Serving = + | { readonly kind: "object"; readonly object: ChordObject } + | { readonly kind: "waiting"; readonly cause: FailureCause }; + /** Creates a host. */ export function createHost

    ( options: HostOptions, @@ -305,7 +326,11 @@ export class ChordHost

    (); /** Teardowns still running, by identity: the next instance of an identity waits for its previous ones. */ readonly #endings = new Map>(); - readonly #loaded = Promise.withResolvers(); + /** The requests that cannot bind yet, by name: before the publication all of them, then those left unbound. */ + readonly #unbound = new UnboundIndex(); + /** Resolved at the host's first publication (`load`'s batch), or its close. */ + readonly #publication = Promise.withResolvers(); + #published = false; /** Every open link: children, routes and the parent. */ readonly #links = new Set(); /** Where holds go that are not served here and not routed: the parent link, across reconnects. */ @@ -320,7 +345,7 @@ export class ChordHost

    (); #nextOrigin = 1; - /** Aborted when the host closes: the context of every route's `resolve`. */ + /** Aborted when the host closes: the context of every route's `resolve`, and the end of waiting `host.registry`s. */ readonly #routing = new AbortController(); readonly #heartbeat: { readonly ping: number; readonly close: number }; /** Its registries by name, each made by its first use (a serve, a contribution or a reader). */ @@ -344,7 +369,7 @@ export class ChordHost

    | undefined); this.timeouts = { close: options.closeTimeout ?? 30_000, ready: options.readyTimeout ?? 60_000 }; this.#heartbeat = { ping: options.heartbeat?.ping ?? 15_000, close: options.heartbeat?.close ?? 30_000 }; - this.#destination = new Destination(this.address, this.#types, this.#routes, this.#parent); + this.#destination = new Destination(this.address, this.#routes, this.#parent); // Chord's own: `openStore` made it. this.store = options.store as ChordStore | undefined; this.store?.attach({ failed: (error) => this.#storageFailed(error), batch: this.batch }); @@ -357,8 +382,8 @@ export class ChordHost

    { for (const contribution of contributions) contribution.record.add(contribution); for (const record of this.#registries.values()) record.composeServed(); @@ -369,12 +394,15 @@ export class ChordHost

    {}); throw error; } + // After the batch: what waited for the publication is placed (the gate lasted only for the setups). + this.#published = true; + this.#publication.resolve(); + this.#unbound.wakeAll(); const objects = [...this.#types.values()] .filter((served) => served.kind === "local" && served.hostLifetime) .map((served) => this.#object(served, EMPTY_PARAMS, undefined, context)); - await Promise.allSettled(objects.map(({ settled }) => settled)); + await Promise.all(objects.map(({ firstOutcome }) => firstOutcome)); if (this.#state === "loading") this.#state = "loaded"; - this.#loaded.resolve(); } async get( @@ -435,9 +463,18 @@ export class ChordHost

    []>; } - async registry(R: Registry, context: Context): Promise> { + async registry(R: Registry, context: Context, options: GetOptions = {}): Promise> { this.batch.assertOutside("registry"); - const loading = this.pendingLoad; - if (loading !== undefined) await awaitWithContext(loading, context); this.#assertOpen(); - const value = this.registryRecord(R.name).value(R.version); - const start = value.start(); - if (start.kind === "failed" && start.last === undefined) throw start.error; - // Its values are the composition as `R`'s version has it, `V`. - return value as RegistryValue; + const timeout = options.readyTimeout ?? this.timeouts.ready; + const expired = new AbortController(); + const waiting = withLinkedAbortSignal(expired.signal, context); + let last: unknown; + const deadline = setTimeout(() => expired.abort(last ?? notLive(`registry ${R.name}`, timeout)), timeout); + // The host's close ends the wait at once, `released/closed` (its timer with it). + const closing = (): void => expired.abort(this.#closedError()); + this.#routing.signal.addEventListener("abort", closing, { once: true }); + try { + await awaitWithContext(this.#publication.promise, waiting.context); + this.#assertOpen(); + const value = this.registryRecord(R.name).value(R.version); + // Its values are the composition as `R`'s version has it, `V`. + const usable = (): boolean => { + const start = value.start(); + if (start.kind === "value" || start.last !== undefined) return true; + last = start.error; + return false; + }; + if (!usable()) await untilUsable(value, usable, waiting.context); + return value as RegistryValue; + } finally { + clearTimeout(deadline); + this.#routing.signal.removeEventListener("abort", closing); + waiting.release(); + } } async root(fn: (ctx: Ctx

    ) => Promise, context: Context, options: RootOptions

    = {}): Promise { @@ -499,11 +555,21 @@ export class ChordHost

    | undefined { - return this.#state === "new" || this.#state === "loading" ? this.#loaded.promise : undefined; + get published(): boolean { + return this.#published; + } + + /** `request` cannot bind yet: indexed under `name` (the type or registry it waits for) until woken or ended. */ + index(name: string, request: UnboundRequest): void { + this.#unbound.add(name, request); + } + + /** `request` left the unbound index's wait for `name` (it ended, or a retry checks it again). */ + unindex(name: string, request: UnboundRequest): void { + this.#unbound.delete(name, request); } /** Whether the host closes: its objects end through `close`, not through its links. */ @@ -513,8 +579,9 @@ export class ChordHost

    this.#place(binding, Type, params, context)); + return undefined; + } + // Checked above: served here at the caller's version. + const object = this.#object(served!, params, principal, context, NO_CHAIN, binding.dependent); + binding.bind(object); + return object; } - /** Where a hold arriving on `link` goes (`Destination.hold`); `released` once the host closed. */ + /** Where a hold arriving on `link` goes (`Destination.hold`: by the address it names); `released` once the host closed. */ place(request: HoldRequest, link: HostLink, caller: Caller, context: Context): Placed { this.#assertOpen(); return this.#destination.hold(request, link, caller.principal, context); } /** - * The object a hold placed here binds to, acting as `principal`: a remote type served here (otherwise - * `unavailable`) with the hold's version and fingerprint (`contract_mismatch`), valid params, and placed here, its - * host address computed from the params (`invalid/host_address`); not an activation the hold's chain waits for - * (`invalid/cycle`, Q5). A new object keeps the hold's chain. + * What a hold placed here binds to, acting as `principal`: the object, or why it waits unbound (`failed + * { waiting }`): a remote type not served here (`not_served`), not at the hold's version and fingerprint, or whose + * params put it on another host (code skew: `version`). Throws for a broken refusal: invalid params, an activation + * the hold's chain waits for (`invalid/cycle`, Q5). A new object keeps the hold's chain. */ - serving(request: HoldRequest, params: unknown, principal: Principal, context: Context): ChordObject { + serving(request: HoldRequest, params: unknown, principal: Principal, context: Context): Serving { const { type, version, fingerprint, chain = NO_CHAIN } = request; const served = this.#types.get(type); - if (served?.kind !== "remote") { - throw new ChordError({ code: "unavailable", message: `${type} is not served here` }); - } - this.#assertMatch(served, version, fingerprint); - const checked = checkParams(served.contract, params, context); - this.#destination.assertHere(served.contract, checked.params); - const object = this.#object(served, checked, principal, context, chain); + const unbound = unboundCause(served, "remote", type, version, fingerprint); + if (unbound !== undefined) return { kind: "waiting", cause: unbound }; + // Checked above: a remote type served here. + const remote = served as Extract; + const checked = checkParams(remote.contract, params, context); + const misplaced = this.#destination.misplaced(remote.contract, checked.params); + if (misplaced !== undefined) return { kind: "waiting", cause: { kind: "waiting", error: misplaced } }; + const object = this.#object(remote, checked, principal, context, chain); const activation = object.activationId; if (activation !== undefined && chain.includes(activation)) throw this.cycle(chain, activation, [object.label]); - return object; + return { kind: "object", object }; } /** * The path a remote reference of `Type` with `params` acting as `principal` goes over now: `ref.retry()` places a * reference failed for a part past the limits anew (a route may have forgotten its old path). Throws as - * `Destination.reference` does, and `unavailable` for a type served here now. + * `Destination.reference` does, and `unavailable` for a type placed here now. */ pathOf(Type: Contract, params: ParamsOf, principal: Principal | undefined): Path { // A retry has no context of its own: its admission checks count on none. const placed = this.#destination.reference(Type, params, principal, BACKGROUND_CONTEXT); if (placed.kind === "path") return placed.path; - throw new ChordError({ code: "unavailable", message: `${Type.name} is served here now` }); + throw new ChordError({ code: "unavailable", message: `${Type.name} is placed here now` }); } /** @@ -713,47 +788,55 @@ export class ChordHost

    { - // Released while `load` ran: nothing is made for it. - if (binding.refState.kind === "ended") return; - if (this.#state === "closed") return binding.end(this.#closedError()); - this.#bindOrFail(binding, Type, checked, context); - }); + const place = (): void => this.#place(binding, Type, checked, context); + if (this.#published) place(); + else this.#wait(binding, Type.name, undefined, place); return binding; } /** - * Binds a reference (`bind`); where it cannot go (`#assertRoute`, a `hostAddress` or `authorize` that throws or - * refuses) fails it broken, until `ref.retry()` binds it again. + * Places a reference (`bind`); where it cannot go (a `hostAddress` or `authorize` that throws or refuses) fails it + * broken, until `ref.retry()` places it again. One woken after the host closed ends as the close ended the others. */ - #bindOrFail(binding: Binding, Type: Contract, checked: CheckedParams, context: Context): void { + #place(binding: Binding, Type: Contract, checked: CheckedParams, context: Context): void { + if (this.#state === "closed") { + binding.end(this.#closedError()); + return; + } try { - this.#assertRoute(Type); this.bind(binding, Type, checked, context); } catch (error) { - // A retry after the host closed ends the reference as the close did the others. - const again = () => - this.#state === "closed" - ? binding.end(this.#closedError()) - : this.#bindOrFail(binding, Type, checked, context); + const again = (): void => this.#place(binding, Type, checked, context); binding.refuse({ kind: "broken", error: asChordError(error, Type.name) }, again); } } + /** + * The provider's own host-address check of a remote `Type` placed here by the consumer's contract, against the one + * served (`invalid/host_address`, `waiting`: code skew); none when both have the same `hostAddress` (the consumer + * computed it with the provider's function) or the type is local. A provider's `hostAddress` that throws throws. + */ + #misplaced(Type: Contract, served: Served, params: ParamsOf): FailureCause | undefined { + if (served.kind !== "remote" || contractData(Type).hostAddress === contractData(served.contract).hostAddress) { + return undefined; + } + const error = this.#destination.misplaced(served.contract, params); + return error === undefined ? undefined : { kind: "waiting", error }; + } + + /** Leaves `binding` unbound in the index under `name`: `cause` (`waiting`), or none before the publication. */ + #wait(binding: Binding, name: string, cause: FailureCause | undefined, place: () => void): void { + binding.wait(cause, place, () => this.#unbound.delete(name, binding)); + this.#unbound.add(name, binding); + } + /** A new reference's binding made by `caller`; its parked calls wait its `readyTimeout`. */ #binding(Type: Contract, dependent: ChordObject | undefined, caller: Caller, options: GetOptions): Binding { return new Binding(Type, this.batch, dependent, caller, options.readyTimeout ?? this.timeouts.ready); @@ -791,22 +874,6 @@ export class ChordHost

    route.close()), - ]; - for (const holder of holders) if (holder.kind === "remote") holder.lost(closed); + for (const route of this.#routes.values()) route.close(); + for (const holder of this.#pathHolders()) if (holder.kind === "remote") holder.lost(closed); + // At the fence too: requests still waiting for a publication or a provider end as the others did, and gets + // waiting for a publication that will not come fail now, not after the slow closes below (fix round 1). + this.#unbound.close(closed); + this.#publication.resolve(); this.backoff.clear(); // Every object is sealed before any teardown releases a reference, so none ends in the background. const teardowns = [...this.#objects.values()].reverse().map((object) => object.stop()); @@ -870,8 +937,6 @@ export class ChordHost

    0) { const message = `host.close: ${failures.length} native close(s) threw`; throw new AggregateError( @@ -881,6 +946,12 @@ export class ChordHost

    route.holders()); + return [...this.#parent.holders(), ...this.unplaced.holders(), ...routed]; + } + /** The store stopped after a backend error (it tells the host once): reported, and the host closes. */ #storageFailed(error: unknown): void { this.report({ kind: "storage_failed", error }); @@ -956,13 +1027,6 @@ export class ChordHost

    boolean, context: Context): Promise { + let changed = Promise.withResolvers(); + const release = value.wait({ + dependencyMoved: () => queueMicrotask(() => changed.resolve()), + // The value closed with its host: the wait ends with the close's error. + closed: (error) => changed.reject(error), + }); + try { + while (!usable()) { + await awaitWithContext(changed.promise, context); + changed = Promise.withResolvers(); + } + } finally { + release(); + } +} diff --git a/packages/chord/src/next/host/internal.ts b/packages/chord/src/next/host/internal.ts index a976a73a5..18c907631 100644 --- a/packages/chord/src/next/host/internal.ts +++ b/packages/chord/src/next/host/internal.ts @@ -3,9 +3,10 @@ export { nextDelay } from "./backoff.ts"; export { CallContext, type Caller } from "./ctx.ts"; -export type { AnyHost, ChordHost } from "./host.ts"; +export type { AnyHost, ChordHost, Serving } from "./host.ts"; export type { PinnedKey } from "./keyed-members.ts"; -export { type ChordObject, type Holder, settlement } from "./object.ts"; +export type { ChordObject, Holder } from "./object.ts"; export { LIVE, LOADING, type RefState } from "./ref-state.ts"; -export type { Binding, CheckedParams, Target } from "./reference.ts"; +export { type Binding, type CheckedParams, type RetryOperation, retryOperation, type Target } from "./reference.ts"; +export type { UnboundRequest } from "./unbound.ts"; export { uuidv7 } from "./uuid.ts"; diff --git a/packages/chord/src/next/host/object.ts b/packages/chord/src/next/host/object.ts index 91e0bd2b8..54c2400a9 100644 --- a/packages/chord/src/next/host/object.ts +++ b/packages/chord/src/next/host/object.ts @@ -1,15 +1,16 @@ // Objects (§6): one per identity, from its first attempt to its end. Its state across attempts (its documents, line -// and job runs, its member sources, slice 6a §1.2), its holders, retries after failures (backoff, dependency recovery -// R18, `ref.retry()`), a live object's failure (slice 6a §1.4), the waits and cycle checks of activations (Q5, its -// instance id and the activation chain it was created for), lingering after its last reference (§3, D8), the job runs -// that keep it alive (§15), its contributions to registries while live and its wake signals (`retryOn`, slice 8), and -// its ending, revocation included (§4, R11). +// and job runs, its member sources, slice 6a §1.2), its holders, retries after failures (backoff, `ref.retry()`, passed +// on while parked), what it shows while its attempt is parked on dependencies (dependency model §3), its first outcome +// (the start barrier, decision (a)), a live object's failure (slice 6a §1.4), the waits and cycle checks of activations +// (Q5, its instance id and the activation chain it was created for), lingering after its last reference (§3, D8), the +// job runs that keep it alive (§15), its contributions to registries while live and its wake signals (`retryOn`, slice +// 8), and its ending, revocation included (§4, R11). -import { awaitWithContext, type Context } from "../../context/index.ts"; +import type { Context } from "../../context/index.ts"; import type { Contract, ParamsOf } from "../contracts/index.ts"; import type { ContractShape } from "../contracts/internal.ts"; -import { ChordError, type FailureCause, isChordError, type Principal } from "../data/index.ts"; -import { invalid, type Provider } from "../data/internal.ts"; +import { ChordError, type FailureCause, type Principal } from "../data/index.ts"; +import { invalid, type Provider, sameCause } from "../data/internal.ts"; import { Documents, type KeyedSlot } from "../documents/internal.ts"; import type { JobDefinition } from "../jobs/index.ts"; import { JobRecords } from "../jobs/internal.ts"; @@ -17,13 +18,14 @@ import { CHAIN } from "../link/internal.ts"; import type { DeriveSource } from "../state/internal.ts"; import { Attempt, type Self } from "./attempt.ts"; import type { Caller } from "./ctx.ts"; +import type { Dependency } from "./dependencies.ts"; import type { AnyHost } from "./host.ts"; import { KeyedMembers, type PinnedKey } from "./keyed-members.ts"; import { MemberSource } from "./member-source.ts"; import type { KeyedBinding } from "./members.ts"; import type { CloseFailed, NativeCloseReason } from "./native.ts"; import { type Ended, type Failed, LIVE, LOADING, type RefState } from "./ref-state.ts"; -import type { Target } from "./reference.ts"; +import { type RetryOperation, retryOperation, type Target } from "./reference.ts"; import { type RetrySource, RetryWakes } from "./retry-on.ts"; import { uuidv7 } from "./uuid.ts"; @@ -60,7 +62,7 @@ type ServedType = { /** What holds an object: a reference's binding, or a hold arriving over a link. */ export interface Holder { - /** The object whose attempt declared the reference; retried when this object goes live (R18). */ + /** The object whose attempt declared the reference: it checks its dependencies again when this object changes. */ readonly dependent: ChordObject | undefined; /** Whom the reference acts for, with its origin's incarnation (`host.references`). */ readonly caller: Caller; @@ -71,34 +73,40 @@ export interface Holder { } /** - * Where an object stands. `waiting`: for its identity's backoff window before its first attempt. `failed` has the - * state its references show, and keeps the failed attempt in `holding` after a dependency failure (R18): its - * references keep the dependencies. + * Where an object stands. `waiting`: for its identity's backoff window before its first attempt. `activating` has what + * its references show: `loading`, or `failed { waiting }` while its attempt is parked on a failed dependency. `failed` + * has the state its references show. */ type ObjectState = | { readonly kind: "waiting" } - | { readonly kind: "activating"; readonly attempt: Attempt } + | { readonly kind: "activating"; readonly attempt: Attempt; readonly shown: RefState } | { readonly kind: "live"; readonly attempt: Attempt } - | { readonly kind: "failed"; readonly failure: Failed; readonly holding: Attempt | undefined } + | { readonly kind: "failed"; readonly failure: Failed } | { readonly kind: "ended"; readonly ended: Ended }; /** - * What an ended object still has to do once its previous teardowns finished (`previous`): drain and close its attempt, - * or release a failed attempt's references; then drop its documents and job runs (`end` alone). + * What an ended object still has to do once its previous teardowns finished (`previous`): drain and close its attempt; + * then drop its documents and job runs (`end` alone). */ type Teardown = | { readonly kind: "none" } | { readonly kind: "end"; readonly previous: Promise | undefined } - | { readonly kind: "close" | "release"; readonly attempt: Attempt; readonly previous: Promise | undefined }; + | { readonly kind: "close"; readonly attempt: Attempt; readonly previous: Promise | undefined }; const END: NativeCloseReason = { kind: "end" }; const CLOSE: NativeCloseReason = { kind: "close" }; +/** How many retry ids an object remembers: a diamond's shortcut only (a ring is stopped by a retry's `through`). */ +const RETRIES_KEPT = 8; + +/** No dependencies waited for: the shared empty list. */ +const NO_DEPENDENCIES: readonly Dependency[] = []; + /** - * One object. Each attempt is a fresh `Attempt`; the documents, the line and the job runs, the holders (and, after a - * dependency failure, the failed attempt's references) outlive it. A transient failure is retried after its backoff window, or at once when a - * dependency it failed on goes live or a reference asks (`retry`); a broken one stays failed until a reference asks. + * One object. Each attempt is a fresh `Attempt`; the documents, the line, the job runs and the holders outlive it. An + * attempt parked on its dependencies waits for them without a timer; a transient failure (its own) is retried after its + * backoff window, or at once when a reference asks (`retry`); a broken one stays failed until a reference asks. * The object ends with its last reference, a live one after its type's `linger` (never, for a host-lifetime object), * or when the host closes. */ @@ -127,13 +135,20 @@ export class ChordObject implements Target { #state: ObjectState = { kind: "waiting" }; /** References and holds of this object. */ readonly #holders = new Set(); - /** Objects this object's activation waits for; followed to find cycles (Q5). */ - readonly #waitingFor = new Set(); - /** The attempt a dependency went live during (R18): if it fails on a dependency, it is retried at once. */ - #woken: Attempt | undefined; - /** The current attempt's outcome: resolves when it goes live, rejects with its error; renewed by every retry. */ - #outcome = settlement(); - /** The error of the last failed attempt, until one goes live. */ + /** The dependencies its parked attempt waits for; their objects are followed to find cycles (Q5). */ + #waitingFor: readonly Dependency[] = NO_DEPENDENCIES; + /** + * Its first outcome (decision (a)): live, failed, parked with no blocker here still without one, or ended; once. + * The promise is made by the first who waits for it (the start barrier, for host-lifetime objects). + */ + #firstOutcome: PromiseWithResolvers | undefined; + #hasFirstOutcome = false; + /** + * The ids of the last retry operations it handled (at most `RETRIES_KEPT`, the newest last; made by the first): a + * retry reaching it twice through a diamond is handled once. + */ + #retried: string[] | undefined; + /** The error of the last failed attempt, or the last cause a parked attempt showed, until one goes live. */ #lastFailure: ChordError | undefined; /** Ends the object when its type's `linger` passed after its last reference; cleared by a new one. */ #lingering: ReturnType | undefined; @@ -189,7 +204,8 @@ export class ChordObject implements Target { const state = this.#state; if (state.kind === "failed") return state.failure; if (state.kind === "live") return LIVE; - return state.kind === "ended" ? state.ended : LOADING; + if (state.kind === "ended") return state.ended; + return state.kind === "activating" ? state.shown : LOADING; } /** Whether its state members have values: once live (a local object stays live until it ends). */ @@ -202,12 +218,27 @@ export class ChordObject implements Target { return this.#exposed !== undefined && this.#state.kind !== "ended"; } - /** The current attempt's outcome (the start barrier waits for the first). */ - get settled(): Promise { - return this.#outcome.promise; + /** Resolves at its first outcome (the start barrier waits for it); never rejects. */ + get firstOutcome(): Promise { + if (this.#hasFirstOutcome) return Promise.resolve(); + this.#firstOutcome ??= Promise.withResolvers(); + return this.#firstOutcome.promise; } - /** The error of the last failed attempt while none went live since. */ + /** Whether it holds up the start barrier: it has no first outcome yet. */ + get blocksFirstPass(): boolean { + return !this.#hasFirstOutcome; + } + + /** An object here: it is not held on a link. */ + get heldOnLink(): false { + return false; + } + + /** + * The error of its last failure while none went live since: a failed attempt's, or the cause its parked attempt + * showed last (kept when the dependency falls back to `loading`, so a getter's deadline names it). + */ get lastFailure(): ChordError | undefined { return this.#lastFailure; } @@ -251,21 +282,31 @@ export class ChordObject implements Target { const state = this.#state; if (state.kind !== "waiting" && state.kind !== "failed") return; this.#instance ??= uuidv7(); - const attempt = new Attempt(this, state.kind === "failed" ? state.holding : undefined, this.#context); - this.#state = { kind: "activating", attempt }; - if (state.kind === "failed") { - this.#outcome = settlement(); - this.#notify(); - } + const attempt = new Attempt(this, this.#context); + this.#state = { kind: "activating", attempt, shown: LOADING }; + if (state.kind === "failed") this.#notify(); void this.#run(attempt); } /** - * `ref.retry()` and a `retry` message: an attempt now when failed (a broken one too) or waiting, its backoff window - * skipped; ignored while an attempt runs and once live. + * `ref.retry()` and a `retry` message, as `operation` (handled once): an attempt now when failed (a broken one too) + * or waiting, its backoff window skipped; passed on to the failed dependencies while parked `failed { waiting }`, + * with this object added to the objects it passed through (at most `CHAIN`: a retry stops past 32 hops); ignored + * otherwise (an attempt runs, or it is live), and when it passed through this object already (a ring) or was handled + * here already (a diamond). */ - retry(): void { - if (this.#state.kind !== "waiting" && this.#state.kind !== "failed") return; + retry(operation: RetryOperation = retryOperation()): void { + const instance = this.#instance; + if (instance !== undefined && operation.through.includes(instance)) return; + if (this.#handled(operation.id)) return; + const state = this.#state; + if (state.kind === "activating") { + const { through } = operation; + if (state.shown.kind !== "failed" || instance === undefined || through.length >= CHAIN) return; + state.attempt.retryDependencies({ id: operation.id, through: [...through, instance] }); + return; + } + if (state.kind !== "waiting" && state.kind !== "failed") return; this.host.backoff.skip(this.identity); this.attempt(); } @@ -396,10 +437,8 @@ export class ChordObject implements Target { if (cause.kind === "transient") this.host.backoff.failed(this.identity); else this.host.backoff.delete(this.identity); const failure: Failed = { kind: "failed", cause, retryAt: this.host.backoff.until(this.identity) }; - this.#state = { kind: "failed", failure, holding: undefined }; + this.#state = { kind: "failed", failure }; this.#lastFailure = error; - this.#outcome = settlement(); - this.#outcome.reject(error); this.#withdraw(attempt); attempt.seal(error); // Members following another object's sources freeze now: no commit of this object changes them. @@ -476,37 +515,39 @@ export class ChordObject implements Target { throw error; } - /** - * Waits for `dependency`'s current attempt as part of this object's activation. Rejects at once when it is failed, - * and when its attempt fails, with `unavailable` naming it (R18), its error as the cause; a dependency that failed - * for a cycle rejects with that `invalid/cycle` itself. - */ - async waitFor(dependency: Target, context: Context): Promise { - this.#waitingFor.add(dependency); - try { - await awaitWithContext(dependency.settled, context); - } catch (error) { - if (isChordError(error, "invalid") && error.reason === "cycle") throw error; - const message = `${this.label}: dependency ${dependency.label} failed`; - throw new ChordError({ code: "unavailable", message }, { cause: error }); - } finally { - this.#waitingFor.delete(dependency); - } + /** Its parked attempt waits for `dependencies` (none once it leaves parking): the waits a cycle check follows. */ + waitsFor(dependencies: readonly Dependency[]): void { + this.#waitingFor = dependencies; } /** - * A dependency went live: retried at once (backoff window skipped) when it failed on a dependency (R18). A wake - * while it activates is remembered: that activation may still fail on the dependency's earlier failure. + * A dependency of its current attempt changed (its state, or its first outcome): a parked attempt checks again; an + * attempt past its dependency check ignores it (R18 after live), and so does any other state. */ - dependencyLive(): void { + dependencyMoved(): void { const state = this.#state; - if (state.kind === "activating") { - this.#woken = state.attempt; - return; + if (state.kind === "activating") state.attempt.dependencyMoved(); + } + + /** + * Its attempt `attempt` is parked showing `shown` (`loading` or `failed { waiting }`): holders are told when that + * changed (a cause compared by value). `firstPass`: no blocker here is still without its first outcome, so this + * object has its first outcome now. + */ + parked(attempt: Attempt, shown: RefState, firstPass: boolean): void { + const state = this.#state; + if (state.kind !== "activating" || state.attempt !== attempt) return; + if (!sameState(state.shown, shown)) { + this.#state = { kind: "activating", attempt, shown }; + if (shown.kind === "failed") this.#lastFailure = shown.cause.error; + this.#notify(); } - if (state.kind !== "failed" || state.holding === undefined) return; - this.host.backoff.skip(this.identity); - this.attempt(); + if (firstPass) this.#settleFirstOutcome(); + } + + /** Its attempt `attempt` left parking for its own steps: it shows `loading` until its outcome. */ + unparked(attempt: Attempt): void { + this.parked(attempt, LOADING, false); } /** @@ -540,14 +581,11 @@ export class ChordObject implements Target { case "live": return this.#wentLive(attempt); case "failed": - return this.#failed(attempt, outcome.cause, undefined); - case "dependency_failed": - return this.#failed(attempt, outcome.cause, attempt); + return this.#failed(attempt, outcome.cause); } } #wentLive(attempt: Attempt): void { - this.#woken = undefined; if (this.#wakes !== undefined) this.#wakes.during = undefined; this.#state = { kind: "live", attempt }; this.#exposed = attempt; @@ -562,21 +600,17 @@ export class ChordObject implements Target { } this.#lastFailure = undefined; this.host.wentLive(this); - this.#outcome.resolve(); this.#notify(); - // R18: objects that failed because this one had failed are retried now. - for (const holder of [...this.#holders]) holder.dependent?.dependencyLive(); + this.#settleFirstOutcome(); attempt.live(); } /** - * Records a failed attempt: reported, torn down in the background (reason `end`), its references released unless it - * is `holding` them (R18), its holders told. Transient: a backoff window, whose end is the failure's `retryAt`; - * broken: no automatic retry. + * Records a failed attempt: reported, torn down in the background (reason `end`), its references released once its + * natives closed, its holders told. Transient: a backoff window, whose end is the failure's `retryAt`; broken: no + * automatic retry. */ - #failed(attempt: Attempt, cause: FailureCause, holding: Attempt | undefined): void { - const woken = this.#woken === attempt; - this.#woken = undefined; + #failed(attempt: Attempt, cause: FailureCause): void { const wokenBy = this.#wakes?.during === attempt; if (this.#wakes !== undefined) this.#wakes.during = undefined; // What its activation loaded for the pinned keys goes; they stay frozen. @@ -584,25 +618,23 @@ export class ChordObject implements Target { if (cause.kind === "transient") this.host.backoff.failed(this.identity); else this.host.backoff.delete(this.identity); const failure: Failed = { kind: "failed", cause, retryAt: this.host.backoff.until(this.identity) }; - this.#state = { kind: "failed", failure, holding }; + this.#state = { kind: "failed", failure }; this.#lastFailure = cause.error; attempt.seal(this.ended()); const closing = attempt.close(END).then((failures) => { attempt.detach(); - // Released once its natives closed (a close may use them, §6, §13); R18's attempt keeps them for the retry. - if (holding === undefined) attempt.releaseReferences(); + // Released once its natives closed (a close may use them, §6, §13). + attempt.releaseReferences(); return failures; }); this.host.ending(this.identity, this.#reported(closing)); this.host.report({ kind: "activation_failed", object: this.label, failure: cause }); - this.#outcome.reject(cause.error); this.#notify(); + this.#settleFirstOutcome(); // Job runs kept it while it activated; failed, they keep it no longer. this.#idle(); - // A dependency went live while this attempt activated: the wake it missed retries it now (R18). - if (holding !== undefined && woken) this.dependencyLive(); // A wake signal arrived while it activated: it retries it now. - else if (wokenBy) this.retry(); + if (wokenBy) this.retry(); } /** @@ -614,7 +646,8 @@ export class ChordObject implements Target { // Called by its wakes only. const wakes = this.#wakes!; if (state.kind === "activating") { - wakes.during = state.attempt; + // While parked its dependencies decide (dependency model §3.8). + if (!state.attempt.isParked) wakes.during = state.attempt; return; } if (state.kind !== "failed" || wakes.leaving) return; @@ -661,7 +694,7 @@ export class ChordObject implements Target { for (const [member, source] of attempt.stateMembers) { if (which === "all" || !attempt.owns(source)) this.#memberSource(member, attempt).freeze(); } - // A pin still loading is given up with the failure: a consumer activating on it fails transient (R18). + // A pin still loading is given up with the failure: a consumer parked on it shows `failed { waiting }`. this.#keys.freeze(which, this.#lastFailure ?? this.ended()); } @@ -705,7 +738,8 @@ export class ChordObject implements Target { this.#stopLingering(); this.#wakes?.stop(); this.host.forget(this); - this.#outcome.reject(error); + // An object of the start barrier's cohort that ends before its first outcome settles it as ended. + this.#settleFirstOutcome(); // In a batch: the derives over its members are marked, so none keeps a cached value. this.host.batch.within(() => { for (const source of this.#members.values()) source.objectEnded(error); @@ -717,12 +751,8 @@ export class ChordObject implements Target { this.#holders.clear(); this.host.unreferenced(this, holders, this.#context); for (const holder of holders) holder.end(error); - if (state.kind === "waiting") return { kind: "end", previous }; - if (state.kind === "failed") { - // The failed attempt is closed (or closing) since it failed; only the references it holds (R18) are left. - const { holding } = state; - return holding === undefined ? { kind: "end", previous } : { kind: "release", attempt: holding, previous }; - } + // A failed attempt is closed (or closing) since it failed. + if (state.kind === "waiting" || state.kind === "failed") return { kind: "end", previous }; if (state.kind === "live") this.#withdraw(state.attempt); state.attempt.seal(error); return { kind: "close", attempt: state.attempt, previous }; @@ -767,8 +797,10 @@ export class ChordObject implements Target { if (teardown.kind === "none") return []; await teardown.previous; const failures = teardown.kind === "close" ? await teardown.attempt.close(reason) : []; - if (teardown.kind === "close") teardown.attempt.detach(); - if (teardown.kind !== "end") teardown.attempt.releaseReferences(); + if (teardown.kind === "close") { + teardown.attempt.detach(); + teardown.attempt.releaseReferences(); + } await this.documents.settled(); this.documents.end(); this.host.store?.claims.release(this); @@ -794,12 +826,35 @@ export class ChordObject implements Target { throw new ChordError({ code: "not_ready", message: `${this.label}: not live yet` }); } - /** The chain of waits from this activation to `target` (see `Target`). */ + /** + * Whether the retry operation `id` reached it before (among the last few: the diamond's shortcut; a ring is stopped + * by `through`); records it otherwise. + */ + #handled(id: string): boolean { + this.#retried ??= []; + const retried = this.#retried; + if (retried.includes(id)) return true; + if (retried.length === RETRIES_KEPT) retried.shift(); + retried.push(id); + return false; + } + + /** The settlement of its first outcome (once): dependents parked on it check their first pass again. */ + #settleFirstOutcome(): void { + if (this.#hasFirstOutcome) return; + this.#hasFirstOutcome = true; + this.#firstOutcome?.resolve(); + this.#firstOutcome = undefined; + for (const holder of this.#holders) holder.dependent?.dependencyMoved(); + } + + /** The chain of waits from this activation to `target` (see `Target`), through its parked attempt's dependencies. */ pathTo(target: ChordObject, visited: Set): ChordObject[] | undefined { if (this === target) return [this]; visited.add(this); - for (const next of this.#waitingFor) { - if (visited.has(next)) continue; + for (const dependency of this.#waitingFor) { + const next = dependency.kind === "reference" ? dependency.binding.target : undefined; + if (next === undefined || visited.has(next)) continue; const rest = next.pathTo(target, visited); if (rest !== undefined) return [this, ...rest]; } @@ -807,9 +862,8 @@ export class ChordObject implements Target { } } -/** A settlement nobody may wait for: its rejection is not unhandled. */ -export function settlement(): PromiseWithResolvers { - const settlement = Promise.withResolvers(); - settlement.promise.catch(() => {}); - return settlement; +/** Whether two states of a reference say the same: the same object, or failures with the same cause (by value). */ +function sameState(a: RefState, b: RefState): boolean { + if (a === b) return true; + return a.kind === "failed" && b.kind === "failed" && a.retryAt === b.retryAt && sameCause(a.cause, b.cause); } diff --git a/packages/chord/src/next/host/ref-state.ts b/packages/chord/src/next/host/ref-state.ts index 9e5720405..1c9234238 100644 --- a/packages/chord/src/next/host/ref-state.ts +++ b/packages/chord/src/next/host/ref-state.ts @@ -7,11 +7,13 @@ import type { ChordError, FailureCause } from "../data/index.ts"; import { type Batch, NativeSource } from "../state/internal.ts"; /** - * Where a reference stands. `loading`: its object activates or retries, it waits for `load` or a parent link, or its - * link is down (the reference's coarser view of its object's waiting and activating); `failed`: its object failed and - * no attempt runs (`retryAt` undefined: broken, no automatic retry); `live`: bound; `ended`: for good, with the error - * that says why (`released` without a reason: released by its holder; `released/ended`: its object ended; - * `released/closed`: its host closed; `forbidden/revoked`). + * Where a reference stands. `loading`: its object activates, retries or is parked on dependencies still under way, it + * waits for its host's publication or a parent link, or its link is down (the reference's coarser view of its object's + * waiting and activating); `failed`: its object failed and no attempt runs (`transient`: `retryAt` is when its backoff + * retries it; `broken`: no automatic retry), or nothing fits it or its object is parked on a failed dependency + * (`waiting`: an event wakes it, `retryAt` undefined); `live`: bound; `ended`: for good, with the error that says why + * (`released` without a reason: released by its holder; `released/ended`: its object ended; `released/closed`: its + * host closed; `forbidden/revoked`). */ export type RefState = | { readonly kind: "loading" } diff --git a/packages/chord/src/next/host/reference.ts b/packages/chord/src/next/host/reference.ts index 65da7ab33..07e3b2665 100644 --- a/packages/chord/src/next/host/reference.ts +++ b/packages/chord/src/next/host/reference.ts @@ -14,7 +14,7 @@ import { takesNoArgs, type VariantImplementation, } from "../contracts/internal.ts"; -import { ChordError, type FailureCause } from "../data/index.ts"; +import { ChordError, type FailureCause, type Principal } from "../data/index.ts"; import { Keyed, type KeyedSlot, type SealToken, Singleton } from "../documents/internal.ts"; import { type Batch, Constant, type DeriveSource, Source } from "../state/internal.ts"; import { type Caller, withLinkedAbortSignal } from "./ctx.ts"; @@ -23,6 +23,8 @@ import type { KeyedBinding } from "./members.ts"; import type { ChordObject, Holder } from "./object.ts"; import { ParkedCalls } from "./parked.ts"; import { type Ended, type Failed, LOADING, ReferenceState, type RefState } from "./ref-state.ts"; +import type { UnboundRequest } from "./unbound.ts"; +import { uuidv7 } from "./uuid.ts"; /** * A reference: a local contract's members, or for a remote contract the members of the variant its object chose, @@ -50,7 +52,10 @@ type InVariants = { type Lifecycle = { /** The reference's state: loading, live, failed or released; watches get every change. */ readonly state: NativeWatchable; - /** Asks a failed object for an attempt now (the backoff skipped, a broken one renewed); otherwise nothing. */ + /** + * Asks a failed object for an attempt now (the backoff skipped, a broken one renewed); an object parked on failed + * dependencies passes it on to them, once; otherwise nothing. + */ retry(): void; release(): void; }; @@ -68,12 +73,20 @@ export interface Target { readonly hydrated: boolean; /** The variant bound last; `undefined` for a local object and before the first binding. */ readonly variant: string | undefined; - /** The error of its last failed attempt while none went live since. */ + /** The error of its last failure (an attempt's, or the dependencies it is parked on) while none went live since. */ readonly lastFailure: ChordError | undefined; /** Its instance id while it activates on this host (Q5); `undefined` otherwise and over a link. */ readonly activationId: string | undefined; - /** The current attempt's outcome, which a dependent's activation waits for (R18). */ - readonly settled: Promise; + /** + * Whether it is held on a link now: a remote attempt answers for it there (`parked_long` counts it while it is + * loading); not while its path has no link, and never for an object here. + */ + readonly heldOnLink: boolean; + /** + * Whether it holds up this host's start barrier: an object here without its first outcome (decision (a)); a remote + * object never does. + */ + readonly blocksFirstPass: boolean; /** Counts `binding` as a reference; the binding reads `state` itself. */ hold(binding: Binding): void; /** `binding` was released. */ @@ -97,8 +110,11 @@ export interface Target { * attempts once loaded (`keyed-members.ts`); only a live local object takes it. */ pinKey(member: string, key: string, context: Context): Promise; - /** `ref.retry()`: an attempt now if failed; ignored otherwise. */ - retry(): void; + /** + * `ref.retry()` (`operation` once per retry, deduping diamonds): an attempt now if failed; an object parked on failed + * dependencies passes it on; ignored otherwise. + */ + retry(operation: RetryOperation): void; /** * The chain of activation waits from this object to `target`, both included (Q5); `undefined` when there is none. * `visited` keeps a search over shared dependencies linear in the waits. @@ -106,14 +122,29 @@ export interface Target { pathTo(target: ChordObject, visited: Set): ChordObject[] | undefined; } +/** + * One `ref.retry()` passing on through parked objects, across links too (the `retry` message's `operation` and + * `through`): its id (a uuidv7) and the instance ids of the objects it passed through, added hop by hop (at most + * `CHAIN`). An object in `through` drops it, so a ring of hosts stops after one lap however many retries run at once; + * an object remembers the last few ids it handled too, so a diamond retries it once. + */ +export type RetryOperation = { readonly id: string; readonly through: string[] }; + +/** A new retry operation, through nothing yet (`ref.retry()`, a `retry` message that names none). */ +export function retryOperation(): RetryOperation { + return { id: uuidv7(), through: [] }; +} + /** A reference's params as L1 checked them, with their identity key. */ export type CheckedParams = { readonly params: ParamsOf; readonly key: string }; /** - * Where a binding stands. `unbound`: waiting for `load` (a `host.hold`) or for its object's activation to bind it (a - * `self.get`); `refused`: a `host.hold` whose type turned out not to be gettable here once `load` resolved; a released - * binding keeps nothing but its name and its end (`ref.state`'s `ended`). `dependent` is the object whose attempt declared the reference with `self.get`, - * `undefined` for `host.get`, `host.hold` and `ctx.get`. + * Where a binding stands. `unbound`: waiting for its object's activation to bind it (a `self.get`); `waiting`: in the + * host's unbound index, `place` checking it again when woken: before the publication (no `failure`: `loading`), or after + * a check that found no fitting provider (`failed { waiting }`, `not_served` or `version`); `refused`: a broken refusal + * (a `hostAddress` or `authorize` that threw or refused) until `ref.retry()` places it again; a released binding keeps + * nothing but its name and its end (`ref.state`'s `ended`). `dependent` is the object whose attempt declared the + * reference with `self.get`, `undefined` for `host.get`, `host.hold` and `ctx.get`. */ type BindingState = | { readonly kind: "unbound"; readonly batch: Batch; readonly dependent: ChordObject | undefined } @@ -123,7 +154,22 @@ type BindingState = readonly dependent: ChordObject | undefined; readonly target: Target; } - | { readonly kind: "refused"; readonly batch: Batch; readonly failure: Failed; readonly bind: () => void } + | { + readonly kind: "waiting"; + readonly batch: Batch; + readonly dependent: ChordObject | undefined; + readonly failure: Failed | undefined; + readonly place: () => void; + /** Takes it out of the index (it ended). */ + readonly leave: () => void; + } + | { + readonly kind: "refused"; + readonly batch: Batch; + readonly dependent: ChordObject | undefined; + readonly failure: Failed; + readonly bind: () => void; + } | { readonly kind: "released"; readonly state: Ended }; /** What a wait for a state decides at each change. */ @@ -144,7 +190,7 @@ type Waiter = { readonly check: () => void; readonly end: (error: ChordError) => * change of its state. Calls run while live, park while loading (sent in order once live) and reject while failed. * Released, or ended with its object, it ends the watches taken through it and rejects the calls made through it. */ -export class Binding implements Holder { +export class Binding implements Holder, UnboundRequest { readonly reference: Reference; /** Who calls through this reference: the principal it was got as, and this host's incarnation. */ readonly caller: Caller; @@ -165,8 +211,6 @@ export class Binding implements Holder { readonly #waiters = new Set(); /** `ref.state`, made on first access. */ #refState: ReferenceState | undefined; - /** Resolved when the reference ends; made by the first `ended()`. */ - #ended: PromiseWithResolvers | undefined; /** The keys its object's factory pinned with `ref.member.at(key)`: pinned once bound, released with it. */ readonly #keys: KeyedMemberAt[] = []; @@ -206,26 +250,38 @@ export class Binding implements Holder { /** The object whose attempt declared this reference, until it is released. */ get dependent(): ChordObject | undefined { const state = this.#state; - return state.kind === "unbound" || state.kind === "bound" ? state.dependent : undefined; + return state.kind === "released" ? undefined : state.dependent; + } + + /** Whom it acts for (the unbound index's revocation). */ + get principal(): Principal | undefined { + return this.caller.principal; } /** The reference's state. */ get refState(): RefState { const state = this.#state; if (state.kind === "bound") return state.target.state; + if (state.kind === "waiting") return state.failure ?? LOADING; if (state.kind === "refused") return state.failure; return state.kind === "unbound" ? LOADING : state.state; } + /** The object it is bound to; `undefined` while it is not bound. */ + get target(): Target | undefined { + return this.#state.kind === "bound" ? this.#state.target : undefined; + } + /** The object in errors: its label once bound, else the contract's name. */ get label(): string { return this.#state.kind === "bound" ? this.#state.target.label : this.#name; } - /** The error of the object's last failed attempt while none went live since. */ + /** The error of the object's last failure while none went live since, or of the check that left it unbound. */ get lastFailure(): ChordError | undefined { const state = this.#state; if (state.kind === "refused") return state.failure.cause.error; + if (state.kind === "waiting") return state.failure?.cause.error; return state.kind === "bound" ? state.target.lastFailure : undefined; } @@ -233,26 +289,48 @@ export class Binding implements Holder { bind(target: Target): void { const state = this.#state; if (state.kind !== "unbound") return; - this.#state = { ...state, kind: "bound", target }; + this.#state = { kind: "bound", batch: state.batch, dependent: state.dependent, target }; target.hold(this); this.moved(); } /** - * A `host.hold` that cannot be bound (where it goes refused it): failed with `cause` until `ref.retry()` runs `bind` - * again (a route's `authorize` may admit it later). + * Leaves the reference unbound in the host's unbound index (`leave` takes it out): `loading` before the publication + * (no `cause`), else `failed` with `cause` (`waiting`); `place` checks it again when the index wakes it. + */ + wait(cause: FailureCause | undefined, place: () => void, leave: () => void): void { + const state = this.#state; + if (state.kind !== "unbound") return; + const failure: Failed | undefined = + cause === undefined ? undefined : { kind: "failed", cause, retryAt: undefined }; + this.#state = { kind: "waiting", batch: state.batch, dependent: state.dependent, failure, place, leave }; + // Unchanged while loading: nothing to tell. + if (failure !== undefined) this.moved(); + } + + /** Woken by the unbound index (out of it already): checked again where it goes. */ + wake(): void { + const state = this.#state; + if (state.kind !== "waiting") return; + this.#state = { kind: "unbound", batch: state.batch, dependent: state.dependent }; + state.place(); + } + + /** + * A reference that cannot be bound (where it goes refused it): failed broken with `cause` until `ref.retry()` runs + * `bind` again (a route's `authorize` may admit it later). */ refuse(cause: FailureCause, bind: () => void): void { const state = this.#state; if (state.kind !== "unbound") return; const failure: Failed = { kind: "failed", cause, retryAt: undefined }; - this.#state = { kind: "refused", batch: state.batch, failure, bind }; + this.#state = { kind: "refused", batch: state.batch, dependent: state.dependent, failure, bind }; this.moved(); } /** * The state may have changed: `ref.state`'s watches are told; live sends the parked calls in order, failed - * rejects them with the cause; waits for a state decide. + * rejects them with the cause; waits for a state decide; the object whose attempt declared it re-evaluates. */ moved(): void { this.#refState?.moved(); @@ -266,6 +344,7 @@ export class Binding implements Holder { this.#parked?.reject(state.cause.error); } for (const waiter of [...this.#waiters]) waiter.check(); + this.dependent?.dependencyMoved(); } /** Resolves when the reference is live; rejects with a broken failure's error, or its end. Getters wait for it. */ @@ -313,6 +392,11 @@ export class Binding implements Holder { } case "unbound": throw new ChordError({ code: "not_ready", message: `${this.#name}: not live yet` }); + case "waiting": + if (state.failure === undefined) { + throw new ChordError({ code: "not_ready", message: `${this.#name}: not live yet` }); + } + throw state.failure.cause.error; case "refused": throw state.failure.cause.error; case "released": @@ -369,6 +453,11 @@ export class Binding implements Holder { return at; } + /** Whether its object's factory pinned keys on it (`at(key)`). */ + get pinsKeys(): boolean { + return this.#keys.length > 0; + } + /** * Pins the keys of `at(key)` on the live object's keyed members and loads those not in memory, before the * dependent goes live (slice 6a §12); `undefined` when there are none. A failed load rejects. @@ -381,24 +470,18 @@ export class Binding implements Holder { } /** - * Resolves when the reference ends (released, or its object ended); never rejects. A wait that must not keep the - * reference races it (a `host.hold` waiting for `load`). + * `ref.retry()`, as `operation` (one per `ref.retry()`, passed on through parked objects): the object is asked for + * an attempt now (ignored unless failed; a parked one passes it on); a refused reference is placed again; one + * waiting in the index is left as it is (an event wakes it). */ - ended(): Promise { - if (this.#state.kind === "released") return Promise.resolve(); - this.#ended ??= Promise.withResolvers(); - return this.#ended.promise; - } - - /** `ref.retry()`: the object is asked for an attempt now (ignored unless failed); a refused hold is bound again. */ - retry(): void { + retry(operation: RetryOperation = retryOperation()): void { const state = this.#state; if (state.kind === "bound") { - state.target.retry(); + state.target.retry(operation); return; } if (state.kind !== "refused") return; - this.#state = { kind: "unbound", batch: state.batch, dependent: undefined }; + this.#state = { kind: "unbound", batch: state.batch, dependent: state.dependent }; this.moved(); state.bind(); } @@ -415,13 +498,15 @@ export class Binding implements Holder { /** * Ends the reference without releasing it from its object (it ended): watches end, calls (running and parked) - * reject with `error`, `ref.state` is `ended` with it. + * reject with `error`, `ref.state` is `ended` with it; the object whose attempt declared it checks again. */ end(error: ChordError): void { // Kept by `ref.state`: its stack is made text now, so the error holds none of the frames' objects. void error.stack; const ended: Ended = { kind: "ended", error }; + const previous = this.#state; this.#state = { kind: "released", state: ended }; + if (previous.kind === "waiting") previous.leave(); for (const at of this.#keys) at.release(); this.#calls?.abort(error); this.#calls = undefined; @@ -431,8 +516,8 @@ export class Binding implements Holder { this.#refState?.released(ended); // Each removes itself; promise reactions run later. for (const waiter of this.#waiters) waiter.end(error); - this.#ended?.resolve(); - this.#ended = undefined; + // An ended dependency breaks a parked dependent (it cannot come back). + if (previous.kind !== "released") previous.dependent?.dependencyMoved(); } #call(method: string, args: unknown, context: Context): Promise { diff --git a/packages/chord/src/next/host/route.ts b/packages/chord/src/next/host/route.ts index 7d4d53c07..78a26820c 100644 --- a/packages/chord/src/next/host/route.ts +++ b/packages/chord/src/next/host/route.ts @@ -98,6 +98,11 @@ export class Route { /** The host closes: no address resolves again. Returns every holder of every path, for the host to end. */ close(): Upstream[] { this.#backoff.clear(); + return this.holders(); + } + + /** Every holder of every path of this route, linked or not (revocation ends a revoked principal's). */ + holders(): Upstream[] { return [...this.#addresses.values()].flatMap(({ path }) => path.holders()); } diff --git a/packages/chord/src/next/host/unbound.ts b/packages/chord/src/next/host/unbound.ts new file mode 100644 index 000000000..d9be173f4 --- /dev/null +++ b/packages/chord/src/next/host/unbound.ts @@ -0,0 +1,62 @@ +// The unbound index (dependency model §1, §4): the requests a host decided cannot bind yet, by the name they wait for. +// Before the host's first publication every request made here, and every hold arriving on its links, waits here +// (`unpublished`); after it, those its check left unbound (`not_served`, `version`): a reference made here, a hold that +// arrived over a link and was refused for a name. A publication wakes the requests of the names it serves; a request +// leaves when it ends (released, settled, given up, its link or origin closed, revoked) or binds. + +import type { ChordError, Principal } from "../data/index.ts"; + +/** A request in the index: a reference made here or a hold that arrived over a link. */ +export interface UnboundRequest { + /** Whom it acts for: revocation ends the requests of a revoked principal. */ + readonly principal: Principal | undefined; + /** The name it waits for became usable (or the host published): it runs its check again, leaving the index first. */ + wake(): void; + /** It is revoked, or its host closes: it ends with `error` (a hold sends `gone`), leaving the index. */ + end(error: ChordError): void; +} + +/** A host's unbound requests by name. */ +export class UnboundIndex { + readonly #byName = new Map>(); + + /** Indexes `request` under `name` (once; again after a wake). */ + add(name: string, request: UnboundRequest): void { + let requests = this.#byName.get(name); + if (requests === undefined) { + requests = new Set(); + this.#byName.set(name, requests); + } + requests.add(request); + } + + /** `request` ended or binds: it leaves the index. */ + delete(name: string, request: UnboundRequest): void { + const requests = this.#byName.get(name); + if (requests === undefined || !requests.delete(request)) return; + if (requests.size === 0) this.#byName.delete(name); + } + + /** The host's publication: every request indexed (by any name) runs its check again, out of the index first. */ + wakeAll(): void { + const all = [...this.#byName.values()].flatMap((requests) => [...requests]); + this.#byName.clear(); + for (const request of all) request.wake(); + } + + /** Every indexed request acting for a principal `revoked` names ends with `error`. */ + revoke(revoked: (principal: Principal) => boolean, error: ChordError): void { + for (const requests of [...this.#byName.values()]) { + for (const request of [...requests]) { + if (request.principal !== undefined && revoked(request.principal)) request.end(error); + } + } + } + + /** The host closed: every indexed request ends with `error`. */ + close(error: ChordError): void { + const all = [...this.#byName.values()].flatMap((requests) => [...requests]); + this.#byName.clear(); + for (const request of all) request.end(error); + } +} diff --git a/packages/chord/src/next/link/binary.ts b/packages/chord/src/next/link/binary.ts index df9fbc80e..65be9335c 100644 --- a/packages/chord/src/next/link/binary.ts +++ b/packages/chord/src/next/link/binary.ts @@ -6,6 +6,7 @@ import { type BoundEntry, + CAUSE_KINDS, type Codec, type FailedMember, type Message, @@ -49,6 +50,11 @@ const PRESENT = 1; const SEGMENTS = 2; +/** A retry's optional fields, one bit each: its operation (16 raw bytes), then the ids it passed through (a list). */ +const OPERATION = 1; + +const THROUGH = 2; + /** A hold's optional fields, one bit each. */ const PRINCIPAL = 1; @@ -122,10 +128,20 @@ function encodeFields(message: Message, writer: Writer): void { return; } case "release": - case "retry": case "loading": writer.varint(message.hold); return; + case "retry": { + const { operation, through } = message; + writer.varint(message.hold); + writer.byte((operation === undefined ? 0 : OPERATION) | (through === undefined ? 0 : THROUGH)); + if (operation !== undefined) writer.raw(hex(operation.replaceAll("-", ""), 16)); + if (through !== undefined) { + writer.varint(through.length); + for (const instance of through) writer.raw(hex(instance.replaceAll("-", ""), 16)); + } + return; + } case "call": writer.varint(message.call); writer.varint(message.hold); @@ -162,7 +178,7 @@ function encodeFields(message: Message, writer: Writer): void { return; case "failed": writer.varint(message.hold); - writer.byte(message.cause.kind === "transient" ? 0 : 1); + writer.byte(CAUSE_KINDS.indexOf(message.cause.kind)); writer.json(message.cause.error); // 0 when absent, else one more than the delay. writer.varint(message.retryIn === undefined ? 0 : message.retryIn + 1); @@ -224,9 +240,16 @@ function decodeFields(kind: (typeof KINDS)[number], reader: Reader): Message { }; } case "release": - case "retry": case "loading": return { kind, hold: reader.varint() }; + case "retry": { + const hold = reader.varint(); + const flags = reader.byte(); + if (flags > (OPERATION | THROUGH)) throw new Error("binary: an unknown retry flag"); + const operation = flags & OPERATION ? { operation: uuid(reader.raw(16)) } : {}; + const through = flags & THROUGH ? { through: reader.list(() => uuid(reader.raw(16))) } : {}; + return { kind, hold, ...operation, ...through }; + } case "call": { const [call, hold, method] = [reader.varint(), reader.varint(), reader.varint()]; return { kind, call, hold, method, ...reader.payload("args") }; @@ -255,15 +278,15 @@ function decodeFields(kind: (typeof KINDS)[number], reader: Reader): Message { } case "failed": { const hold = reader.varint(); - const cause = reader.byte(); - if (cause > 1) throw new Error("binary: an unknown failure kind"); + const cause = CAUSE_KINDS[reader.byte()]; + if (cause === undefined) throw new Error("binary: an unknown failure kind"); // The schema check after decoding establishes the wire error. const error = reader.json() as Extract["error"]; const retryIn = reader.varint(); return { kind, hold, - cause: { kind: cause === 0 ? "transient" : "broken", error }, + cause: { kind: cause, error }, ...(retryIn === 0 ? {} : { retryIn: retryIn - 1 }), }; } diff --git a/packages/chord/src/next/link/message.ts b/packages/chord/src/next/link/message.ts index 84bab8629..49302273f 100644 --- a/packages/chord/src/next/link/message.ts +++ b/packages/chord/src/next/link/message.ts @@ -3,7 +3,7 @@ import Type from "typebox"; import type { Context } from "../../context/index.ts"; -import { arr, int, lit, obj, opt, type Static, str, union, unknown } from "../contracts/index.ts"; +import { arr, int, lit, obj, oneOf, opt, type Static, str, union, unknown } from "../contracts/index.ts"; import { validatorFor } from "../data/internal.ts"; import type { Part } from "./part.ts"; @@ -50,6 +50,9 @@ const incarnation = str({ pattern: "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f /** The longest activation chain a hold carries (Q5): a longer one keeps its last entries. */ export const CHAIN = 32; +/** The kinds of a `failed` message's cause, in their codes' order (the binary codec's byte, the text codec's letter). */ +export const CAUSE_KINDS: ["transient", "broken", "waiting"] = ["transient", "broken", "waiting"]; + /** The protocol's messages, one tagged union. Parts are opaque here (the owners parse them). */ const MESSAGE = union([ // `calls`: how many concurrent calls this side accepts (the call window). @@ -75,8 +78,16 @@ const MESSAGE = union([ obj({ kind: lit("release"), origin: opt(origin), hold: id }), obj({ kind: lit("call"), origin: opt(origin), call: id, hold: id, method: index, args: opt(part), segments }), obj({ kind: lit("abort"), origin: opt(origin), call: id }), - // Asks the provider for an attempt of a failed hold's object now (`ref.retry()`). - obj({ kind: lit("retry"), origin: opt(origin), hold: id }), + // Asks the provider for an attempt of a failed hold's object now (`ref.retry()`); `operation` names the one retry it + // belongs to (a uuidv7) and `through` the instance ids of the objects it passed through (with `operation` only), so + // an object it reaches twice (a diamond, a ring) handles it once. Relays forward both. + obj({ + kind: lit("retry"), + origin: opt(origin), + hold: id, + operation: opt(incarnation), + through: opt(Type.Array(incarnation, { maxItems: CHAIN })), + }), // Binds several holds at once (first bindings, re-holds), applied in one batch. obj({ kind: lit("bound"), @@ -103,8 +114,9 @@ const MESSAGE = union([ origin: opt(origin), hold: id, // A failure cause's error in the wire error format (a ChordError, checked by L1's envelope check, or a plugin's). - cause: obj({ kind: union([lit("transient"), lit("broken")]), error: wireError }), - // The rest of the object's backoff window in milliseconds: when it retries by itself (absent: broken). + // `waiting`: not the object's fault (no provider, no fitting version, a dependency failed); an event wakes it. + cause: obj({ kind: oneOf(...CAUSE_KINDS), error: wireError }), + // The rest of the object's backoff window in milliseconds: when it retries by itself (absent: broken, waiting). retryIn: opt(int({ min: 0 })), }), // The failed object of a hold started a new attempt. diff --git a/packages/chord/src/next/link/text.ts b/packages/chord/src/next/link/text.ts index e20c741bb..493657b17 100644 --- a/packages/chord/src/next/link/text.ts +++ b/packages/chord/src/next/link/text.ts @@ -8,6 +8,7 @@ import { type BoundEntry, + CAUSE_KINDS, type Codec, type FailedMember, type Message, @@ -91,7 +92,7 @@ function ownTuple(message: Message): string { case "state": return `["s",[${message.holds.map(stateEntry).join(",")}]]`; case "failed": { - const kind = message.cause.kind === "transient" ? "t" : "b"; + const kind = message.cause.kind[0]; const retryIn = message.retryIn === undefined ? [] : [message.retryIn]; return json(["f", message.hold, kind, message.cause.error, ...retryIn]); } @@ -104,7 +105,10 @@ function ownTuple(message: Message): string { case "revoke": return json(["V", message.revocation]); case "retry": - return json(["T", message.hold]); + // `through` travels only with an `operation` (Chord sends both). + if (message.operation === undefined) return json(["T", message.hold]); + if (message.through === undefined) return json(["T", message.hold, message.operation]); + return json(["T", message.hold, message.operation, message.through]); case "loading": return json(["l", message.hold]); case "ack": @@ -257,10 +261,11 @@ function fromFields(fields: unknown[], rest: Uint8Array): Message { }; case "f": { arity(3, 4); - if (f[1] !== "t" && f[1] !== "b") throw new Error("text: an unknown failure kind"); + const causeKind = CAUSE_KINDS.find((each) => each[0] === f[1]); + if (causeKind === undefined) throw new Error("text: an unknown failure kind"); // The schema check after decoding establishes the wire error. const error = f[2] as Extract["error"]; - const cause = { kind: f[1] === "t" ? "transient" : "broken", error } as const; + const cause = { kind: causeKind, error }; return { kind: "failed", hold: field(0), cause, ...(f.length === 4 ? { retryIn: field(3) } : {}) }; } case "r": @@ -276,8 +281,13 @@ function fromFields(fields: unknown[], rest: Uint8Array): Message { arity(1); return { kind: "revoke", revocation: f[0] }; case "T": - arity(1); - return { kind: "retry", hold: field(0) }; + arity(1, 2, 3); + return { + kind: "retry", + hold: field(0), + ...(f.length >= 2 ? { operation: field(1) } : {}), + ...(f.length === 3 ? { through: field(2) } : {}), + }; case "l": arity(1); return { kind: "loading", hold: field(0) }; diff --git a/packages/chord/src/next/registries/README.md b/packages/chord/src/next/registries/README.md index 070c98877..535973db0 100644 --- a/packages/chord/src/next/registries/README.md +++ b/packages/chord/src/next/registries/README.md @@ -39,8 +39,10 @@ host.serve(Server, (_, self) => { a change of the table marks it, and it composes when read or, while someone watches, observes or waits for it, in the batch that changed the table (after the commit's adoptions); an unchanged table (every entry the same) composes nothing. A `compose` that throws is the owner's bug: `registry_failed` once per episode, watches get one `failed` - call, `.value` keeps the last good composition (before any, it throws the error); the next good composition is a - value call, also when its value is the same. + call, `.value` keeps the last good composition while every contribution it was built from is still in the table + (before any, or once one of them left, it throws the error: a contributor that failed or ended may have closed the + native its entry held; decision 5, slice 7a); the next good composition is a value call, also when its value is the + same. - The table (`RegistryTable`): one `RegistryEntry { plugin, type?, version, value }` per live contributor, sorted by plugin name, then type (a facet's entry first): never load order. A contribution of an older version is adapted with `accepts[version].contribution` once per contribution value; a newer one, or an older one without an adapter, is left @@ -53,16 +55,18 @@ host.serve(Server, (_, self) => { `RegistryValue`: a native watchable (`.value`, `watch`) with `observe(observer, ctx)`, a synchronous observer of its compositions (Durable's shape, for `registryReader`). A reader whose token is older than the provider's gets the composition mapped back with `accepts[version].value`, once per composition, only while read or followed; a mapping - that throws fails that version only (`registry_failed`, its last good mapping kept). A reader without a mapping, or - newer than the provider, gets `contract_mismatch` (its object fails `broken`: only a reload helps). The mapping is - kept only while a reader or follower is there. Without a provider a registry has no value - (`unavailable/undeclared`); its contributions wait in the table. + that throws fails that version only (`registry_failed`, its last good mapping kept while every contribution the + composition it was mapped from was built from is still in the table: decision 5 holds for every version). A reader without a mapping, or + newer than the provider, gets `contract_mismatch`. The mapping is kept only while a reader or follower is there. + Without a provider a registry has no value (`unavailable/undeclared`); its contributions wait in the table. A + reader's attempt without a usable value parks `failed { waiting }` (`host/`), woken when the value turns usable; a + `host.registry` waits the same way up to its `readyTimeout`. - `keyedEntries(table, entriesOf, report)`: a map of every entry's keyed items, the first by plugin name keeping a key, a duplicate reported through `compose`'s `report`; `orderedEntries(table, itemsOf, orderOf?)`: every entry's items in table order, or by `orderOf`. Both keep each item's `plugin` (and `type`): a contribution is another plugin's code, so a use site catches its throws and rejections, reports the plugin, and goes on. -- Registries carry natives: an entry lives exactly as long as its contributor. A last good composition kept after a - throwing `compose` may still name a contributor that left: use sites look up at each use and keep nothing. +- Registries carry natives: an entry lives exactly as long as its contributor, and a last good composition no longer + than the contributions it was built from; use sites look up at each use and keep nothing. ## For Chord's layers (`internal.ts`) @@ -75,7 +79,11 @@ host.serve(Server, (_, self) => { (`value(version)`); `close()` at the host's close. - `RegistryContribution` (`contribution.ts`): one contributor's contribution, a `Dependent` of its source; `entry(version, provider)` is its table entry (the same object while its value is), or `undefined` when left out. -- `RegistryValueSource`: a `FollowedSource` per reader version; `read(waiter)` registers an activation's object - (`RegistryWaiter`), woken with `dependencyLive()` (on a microtask) when the value gets its first good value; readers - waiting for one count as followers, so a change composes for them. At the host's close it keeps its last start and - lets the record (and the host) go. +- `RegistryValueSource`: a `FollowedSource` per reader version; `read()` registers a reader (an attempt that declared + it, until its references are released: an older version's mapping is kept meanwhile); `wait(waiter)` registers a + `RegistryWaiter` (a parked attempt, a waiting `host.registry`), told with `dependencyMoved()` (inside the batch: it + only records) at every change of whether the value is usable or of its failure while it is not (errors compared by + value: a version mismatch is made per read), and with `closed(error)` when the host closes (`released/closed`: the + wait ends); waiters count as followers, so a change composes for them. At the host's close it keeps its last start + and lets the record (and the host) go. `RegistryRecord.composedFrom` and `inTable` give a mapping the provenance + decision 5 checks. diff --git a/packages/chord/src/next/registries/record.ts b/packages/chord/src/next/registries/record.ts index f879132f9..110ad7e51 100644 --- a/packages/chord/src/next/registries/record.ts +++ b/packages/chord/src/next/registries/record.ts @@ -44,9 +44,12 @@ export type RegistryEdge = { readonly from: RegistryRecord; readonly to: Registr /** The provider as served: the defining plugin's version, `compose` and adapters. */ type Provider = { readonly version: number; readonly provider: RegistryProvider }; -/** Where a composition stands: never composed, its last good value, and its failure while `compose` throws. */ +/** + * Where a composition stands: never composed, its last good value (with the contributions it was built from), and its + * failure while `compose` throws. + */ type Composition = { - readonly good: { readonly value: unknown } | undefined; + readonly good: { readonly value: unknown; readonly from: readonly RegistryContribution[] } | undefined; readonly failure: { readonly error: unknown } | undefined; }; @@ -59,7 +62,7 @@ export class RegistryRecord { readonly #report: (error: RegistryError) => void; #provider: Provider | undefined; /** The contributions of live contributors, sorted by plugin, then type (a facet's first). */ - readonly #contributions: RegistryContribution[] = []; + #contributions: readonly RegistryContribution[] = []; /** The last table composed, kept so an unchanged table composes nothing. */ #table: RegistryTable | undefined; #dirty = true; @@ -128,7 +131,8 @@ export class RegistryRecord { /** Adds a live contributor's contribution, inside a batch: the table changes. */ add(contribution: RegistryContribution): void { const at = this.#contributions.findIndex((other) => compareContributors(contribution, other) < 0); - this.#contributions.splice(at < 0 ? this.#contributions.length : at, 0, contribution); + // A new list per change (contributors come and go rarely): a composition keeps the one it was built from. + this.#contributions = this.#contributions.toSpliced(at < 0 ? this.#contributions.length : at, 0, contribution); const { reads } = contribution; if (reads !== undefined) this.#reads.set(reads, (this.#reads.get(reads) ?? 0) + 1); contribution.follow(); @@ -139,7 +143,7 @@ export class RegistryRecord { remove(contribution: RegistryContribution): void { const at = this.#contributions.indexOf(contribution); if (at < 0) return; - this.#contributions.splice(at, 1); + this.#contributions = this.#contributions.toSpliced(at, 1); const { reads } = contribution; if (reads !== undefined) { const left = this.#reads.get(reads)! - 1; @@ -176,6 +180,20 @@ export class RegistryRecord { return { kind: "value", value: good!.value }; } + /** The contributions the current good composition was built from; `undefined` while there is none. */ + get composedFrom(): readonly RegistryContribution[] | undefined { + return this.#composition.good?.from; + } + + /** + * Whether every contribution of `from` (what a composition, or a mapping of it, was built from) is still in the + * table (decision 5: one that left may have closed the native its entry held). + */ + inTable(from: readonly RegistryContribution[]): boolean { + const current = this.#contributions; + return from === current || from.every((each) => current.includes(each)); + } + /** The adapter mapping the composed value back for readers of `version`; `undefined` when there is none. */ valueAdapter(version: number): ((current: never) => unknown) | undefined { return this.#provider?.provider.accepts?.[version]?.value; @@ -203,7 +221,7 @@ export class RegistryRecord { */ close(): void { for (const contribution of this.#contributions) contribution.unfollow(); - this.#contributions.length = 0; + this.#contributions = []; this.#reads.clear(); this.#table = undefined; this.#dirty = false; @@ -226,8 +244,9 @@ export class RegistryRecord { /** * Builds the table and composes it, unless it is the last one composed, entry for entry (`compose` is pure). A - * throw is a failure episode: reported once, the last good composition kept; conflicts are reported once per - * episode, as their message and plugins. + * throw is a failure episode: reported once, the last good composition kept while every contribution it was built + * from is still in the table (decision 5: one whose contributor ended may hold a closed native); conflicts are + * reported once per episode, as their message and plugins. */ #recompose({ version, provider }: Provider): void { this.#dirty = false; @@ -247,10 +266,14 @@ export class RegistryRecord { value = this.#batch.compute(() => provider.compose(table, context)); } catch (error) { if (this.#composition.failure === undefined) this.failed(error); - this.#composition = { good: this.#composition.good, failure: { error } }; + const { good } = this.#composition; + this.#composition = { + good: good !== undefined && this.inTable(good.from) ? good : undefined, + failure: { error }, + }; return; } - this.#composition = { good: { value }, failure: undefined }; + this.#composition = { good: { value, from: this.#contributions }, failure: undefined }; for (const [key, conflict] of conflicts) { if (this.#conflicts.has(key)) continue; this.#report({ kind: "registry_conflict", registry: this.name, ...conflict }); diff --git a/packages/chord/src/next/registries/value.ts b/packages/chord/src/next/registries/value.ts index 7b3b8e1db..f08b25987 100644 --- a/packages/chord/src/next/registries/value.ts +++ b/packages/chord/src/next/registries/value.ts @@ -1,20 +1,40 @@ // Registry values (slice 8 §1 "Reading", S2, S3): a registry's composed value as the readers of one version see it, a // native source. The provider's version reads the composition itself; an older one reads it mapped back with // `accepts[old].value`, once per composition, kept only while someone reads or follows it; a version the provider -// cannot serve fails `contract_mismatch`. Readers declared by attempts are woken (R18) when the value gets its first -// good value. +// cannot serve fails `contract_mismatch`. Its readers (attempts that declared it) keep the mapping; its waiters +// (attempts parked on it, a waiting `host.registry`) are woken at every change of whether it is usable, and of its +// failure while it is not (dependency model §9). -import { ChordError } from "../data/index.ts"; +import { ChordError, isChordError } from "../data/index.ts"; import { type Batch, FollowedSource, type NativeStart } from "../state/internal.ts"; +import type { RegistryContribution } from "./contribution.ts"; import type { RegistryRecord } from "./record.ts"; -/** An object waiting for a registry's value (`self.registry`): retried at once when its first value arrives (R18). */ +/** Who waits for a registry's value (a parked attempt, a `host.registry`): told when it may have changed. */ export interface RegistryWaiter { - dependencyLive(): void; + /** The value may have changed (inside its batch: it only records). */ + dependencyMoved(): void; + /** The value closed with its host: the wait ends with `error` (`released/closed`). */ + closed(error: ChordError): void; } -/** The last good mapping for an older version, and the composition it was made from. */ -type Mapped = { readonly from: unknown; readonly value: unknown }; +const USABLE = { kind: "usable" } as const; + +/** Whether two failures say the same: the same error, or ChordErrors alike (a version mismatch is made per read). */ +function sameError(a: unknown, b: unknown): boolean { + if (a === b) return true; + return isChordError(a) && isChordError(b) && a.code === b.code && a.reason === b.reason && a.message === b.message; +} + +/** + * The last good mapping for an older version, the composition it was made from, and the contributions that composition + * was built from (kept only while all are in the table, decision 5). + */ +type Mapped = { + readonly from: unknown; + readonly value: unknown; + readonly contributions: readonly RegistryContribution[] | undefined; +}; /** A failed mapping of composition `from`, until another composition maps. */ type MapFailure = { readonly from: unknown; readonly error: unknown }; @@ -32,10 +52,15 @@ export class RegistryValueSource extends FollowedSource { readonly name: string; readonly version: number; #registered: Registered; - /** Readers declared by attempts, each a token of its own (one object may read twice), until released. */ - readonly #readers = new Set<{ readonly waiter: RegistryWaiter }>(); - /** Whether it had a good value: the first one wakes the readers. */ - #valued = false; + /** Its readers, each a token of its own (one object may read twice), until released. */ + readonly #readers = new Set(); + /** Its waiters, each a token of its own, until released. */ + readonly #waiters = new Set<{ readonly waiter: RegistryWaiter }>(); + /** + * What its waiters were last told: usable (a value, or a failure with a last good one), or the error it has none + * for; a change wakes them. + */ + #told: { readonly kind: "usable" } | { readonly kind: "failed"; readonly error: unknown } | undefined; #mapped: Mapped | undefined; #mapFailure: MapFailure | undefined; @@ -60,16 +85,15 @@ export class RegistryValueSource extends FollowedSource { return start.last.value; } - /** Its composition (mapped for an older version), composed now if its table changed; a first value wakes readers. */ + /** + * Its composition (mapped for an older version), composed now if its table changed; a change of whether it is + * usable, or of its failure while it is not, wakes its waiters. + */ override start(): NativeStart { const registered = this.#registered; if (registered.kind === "closed") return registered.last; const start = this.#start(registered.record); - if (!this.#valued && (start.kind === "value" || start.last !== undefined)) { - this.#valued = true; - // Not inside the batch that composed: a retry runs a factory, which may not run in a batch. - for (const { waiter } of this.#readers) queueMicrotask(() => waiter.dependencyLive()); - } + if (this.#waiters.size > 0) this.#tell(start); return start; } @@ -80,14 +104,14 @@ export class RegistryValueSource extends FollowedSource { return start.value; } - /** Followed while someone watches or observes it, and while readers wait for its first value. */ + /** Followed while someone watches or observes it, and while waiters wait on it. */ override get watched(): boolean { - return super.watched || (this.#readers.size > 0 && !this.#valued); + return super.watched || this.#waiters.size > 0; } - /** Registers `waiter` (an activation's declared dependency) until the returned release. */ - read(waiter: RegistryWaiter): () => void { - const token = { waiter }; + /** Registers a reader (an attempt that declared it) until the returned release: its mapping is kept meanwhile. */ + read(): () => void { + const token = {}; this.#readers.add(token); return () => { this.#readers.delete(token); @@ -95,6 +119,20 @@ export class RegistryValueSource extends FollowedSource { }; } + /** + * Registers `waiter` (a parked attempt's object, a waiting `host.registry`) until the returned release: it is told + * of every change of whether this value is usable (inside the batch that made it: it only records). + */ + wait(waiter: RegistryWaiter): () => void { + const token = { waiter }; + if (this.#waiters.size === 0) this.#told = undefined; + this.#waiters.add(token); + return () => { + this.#waiters.delete(token); + if (this.#waiters.size === 0) this.#told = undefined; + }; + } + /** The registry's table changed (inside a batch): its dependents are marked, its followers get the next frame. */ recomposed(): void { this.changed(); @@ -119,6 +157,11 @@ export class RegistryValueSource extends FollowedSource { } this.#registered = { kind: "closed", last }; this.#readers.clear(); + const waiters = [...this.#waiters]; + this.#waiters.clear(); + const error = new ChordError({ code: "released", reason: "closed", message: `registry ${this.name}: closed` }); + for (const { waiter } of waiters) waiter.closed(error); + this.#told = undefined; this.#dropMapping(); this.end(); } @@ -128,6 +171,17 @@ export class RegistryValueSource extends FollowedSource { if (this.#readers.size === 0) this.#dropMapping(); } + /** Wakes the waiters when `start` says something else than they were told (by usability, then by the error). */ + #tell(start: NativeStart): void { + const told = this.#told; + const usable = start.kind === "value" || start.last !== undefined; + if (usable ? told?.kind === "usable" : told?.kind === "failed" && sameError(told.error, start.error)) return; + this.#told = usable ? USABLE : { kind: "failed", error: (start as { readonly error: unknown }).error }; + // The first start of a wait wakes nobody: the waiter reads it itself. + if (told === undefined) return; + for (const { waiter } of this.#waiters) waiter.dependencyMoved(); + } + #dropMapping(): void { this.#mapped = undefined; this.#mapFailure = undefined; @@ -156,13 +210,18 @@ export class RegistryValueSource extends FollowedSource { if (current.kind === "failed" && current.last === undefined) return current; const from = current.kind === "value" ? current.value : current.last!.value; // `undefined` is a composition too: the records, not their fields, say whether there is one. + // A mapping built from a contribution that left is no last good value any more (decision 5). + if (this.#mapped?.contributions !== undefined && !record.inTable(this.#mapped.contributions)) { + this.#mapped = undefined; + } const mapped = this.#mapped; const failure = this.#mapFailure; if (mapped !== undefined && mapped.from === from) { this.#mapFailure = undefined; } else if (failure === undefined || failure.from !== from) { try { - this.#mapped = { from, value: record.adapt(() => map(from as never)) }; + const value = record.adapt(() => map(from as never)); + this.#mapped = { from, value, contributions: record.composedFrom }; this.#mapFailure = undefined; } catch (error) { if (failure === undefined) record.failed(error); diff --git a/packages/chord/src/next/remote/README.md b/packages/chord/src/next/remote/README.md index 16522f71b..19265eb12 100644 --- a/packages/chord/src/next/remote/README.md +++ b/packages/chord/src/next/remote/README.md @@ -55,8 +55,8 @@ within each link's limits, compression at the origin, activation chains and revo one, which a relayed hold carries upstream as `limit`. A call on a relayed hold whose arguments are past the link's wire limit, or on a served hold whose arguments are past its limits, is answered `too_large/part` and never admitted. Compressed params (on any link) and compressed arguments from a child are protocol errors: params are never - compressed, and clients never send compressed parts. Holds of a principal being revoked that still wait for `load` - end with `gone` (`revokeWaiting`). + compressed, and clients never send compressed parts. Holds of a principal being revoked that hold nothing (before the + publication, refused, relayed and waiting for their link) end with `gone` (`revokeUnbound`). - **The window** (K = 2, `ProviderSide`): at most two of this host's own `bound` and `state` messages unacknowledged per origin; every other message is control and never waits. The consumer sends `ack` with the origin after applying each. The side keeps the sender of every message in flight in order (`local`, or the upstream link a relayed one came @@ -81,13 +81,18 @@ within each link's limits, compression at the origin, activation chains and revo Trace (tab D through relay R, worker U, a derive over A and B on the tab): U's window for D's origin is full; a commit touching A and B joins U's pending queues. D acks one: R returns the `ack` to U, whose freed slot carries one `state` with A's and B's entries: the derive sees both at once, never A new with B old. -- `Hold` (one per `hold` served here, a `Holder` of its object): takes its object once `load` resolved; the hold's - `params` part is parsed then (malformed: a protocol error). A refused type, a mismatch, invalid params, a wrong - host address or a snapshot that throws is a broken `failed` that ends it (the provider forgets the id). While pending - it follows its object's `state`: each failure, transient or broken, is sent as `failed` with `retryIn`, the rest of - the object's backoff window (absent when broken), the current one at once when the object is failed already; the - next attempt is announced with `loading`; the hold stays, so a `retry` (from any holder) renews even a broken object - for every holder. Live, it binds through the window (`boundEntry`: the variant and each member's value as a part, +- `Hold` (one per `hold` served here, a `Holder` of its object, an `UnboundRequest` of the host's index): before the + host's publication it waits in the unbound index (`unplaced`, nothing sent: the consumer is `loading`) and the + publication starts it; then it takes its object in the turn it arrived; the hold's `params` part is parsed then + (malformed: a protocol error). A type not served here (or as a local type), a mismatch, or params the provider's + code places on another host leave it refused `failed { waiting }`, indexed under its type until a publication re-runs + its start (`loading`, then `bound` or `failed`); invalid params or a cycle leave it refused `failed { broken }`; both + are kept for a `retry`, which re-runs the start. While pending it follows its object's `state`: each failure is sent + as `failed` (with `retryIn`, the rest of the object's backoff window, for a transient one; absent for broken and + waiting), each new cause of an object parked on a failed dependency too, the current one at once when the object is + failed already; the next attempt (or the parked object's next `loading`) is announced with `loading`; the hold stays, + so a `retry` (from any holder) renews even a broken object for every holder (a parked object passes it on, one retry + operation of this host). Live, it binds through the window (`boundEntry`: the variant and each member's value as a part, names as indexes into the bound version). Its members' frames arrive as `Subscriber` calls inside the batch; at its end the hold's entry lists the members that changed with their ops, each encoded with that member's own Delta encoder (`entry`), or queues them (`queue`, later `pendingEntry`). Each entry counts one `chord.link.encodes`. A @@ -97,9 +102,10 @@ within each link's limits, compression at the origin, activation chains and revo dropping the member's pending ops; a value after it replaces it), and a retirement of a failed member waits for its next value (never listed beside its failure). Its object's end sends `gone` with its error (`forbidden/revoked` for a revocation; its running calls aborted with it). A - released pending hold stops its wait for `load` at once. Params past the link's wire limit, or its decoded one, fail - the hold broken `too_large/part`; a cycle its chain closes, broken `invalid/cycle` (`host.serving`). It counts as one - of the host's `references` (`host.activity`) while it keeps its object. + released hold before the publication leaves the index at once. Params past the link's wire limit, or its decoded + one, fail the hold broken `too_large/part`; a cycle its chain closes, broken `invalid/cycle` (`host.serving`). It + counts as one of the host's `references` (`host.activity`) from its arrival to its end, bound or not (relayed, it + counts at its destination instead). - `RelayedHold` (a hold this host neither serves nor parses, `host.place` decided it): id `h` of its downstream origin, held with the same id on its path's current link under that origin's number, as the arriving hold's caller, from its kept `hold` message (params copied once out of the received message, so no view keeps the buffer). `call`, @@ -134,9 +140,9 @@ within each link's limits, compression at the origin, activation chains and revo tells its owner (a route) when it needs a link or lost its last holder. - `RemoteObject` (one per remote reference, owned by its host across the links of its path): its `state` (`loading` until a link's hold is bound and again on every drop or `loading`, `failed` from `failed` with `retryAt = now + - retryIn`, `live` on `bound`, `released`), `settled` (pending while loading, resolved while live, rejected while - failed and by a drop or a relay's `loading` while loading: a dependent's activation waits for it and fails at once - then, R18, and `bound` retries the dependent), `lastFailure`, `variant` (set in `bound`'s batch, before derives recompute) and + retryIn` for a transient cause, `retryAt` undefined for a broken or waiting one, `live` on `bound`, `released`; + every change tells its binding, whose parked dependent checks again: no new hold, no backoff), `remote` and + `blocksFirstPass` (never: a wait on another host ends this host's first pass), `lastFailure`, `variant` (set in `bound`'s batch, before derives recompute) and `member` (the last binding's, kept while loading or failed; outside the variant `invalid/member`), `call` (only while live; outside the variant `invalid/member`), `retry` (sends `retry` while failed on an open link; after a part past the limits, placed anew with `host.pathOf` and held with a new id, now or on that path's next link). Its holds made diff --git a/packages/chord/src/next/remote/consumer.ts b/packages/chord/src/next/remote/consumer.ts index 8d654e291..e4b8516e6 100644 --- a/packages/chord/src/next/remote/consumer.ts +++ b/packages/chord/src/next/remote/consumer.ts @@ -8,6 +8,7 @@ import type { Context } from "../../context/index.ts"; import { type MethodData, takesNoArgs } from "../contracts/internal.ts"; import { ChordError, isChordError } from "../data/index.ts"; import { asChordError } from "../data/internal.ts"; +import type { RetryOperation } from "../host/internal.ts"; import { type BoundEntry, type ConsumerMessage, @@ -125,9 +126,10 @@ export class ConsumerSide { this.forget(hold, error); } - /** `ref.retry()` on a failed hold: the provider is asked for an attempt now. */ - retry(hold: number): void { - this.#link.send({ kind: "retry", hold }); + /** `ref.retry()` on a failed hold, as retry `operation`: the provider is asked for an attempt now. */ + retry(hold: number, { id, through }: RetryOperation): void { + const passed = through.length === 0 ? {} : { through }; + this.#link.send({ kind: "retry", hold, operation: id, ...passed }); } /** Drops hold `hold` from the tables; its calls, in flight and waiting, fail with `error` and free their slots. */ diff --git a/packages/chord/src/next/remote/hold.ts b/packages/chord/src/next/remote/hold.ts index b018c201b..21e5230c6 100644 --- a/packages/chord/src/next/remote/hold.ts +++ b/packages/chord/src/next/remote/hold.ts @@ -1,7 +1,8 @@ -// A hold arriving on a link and served here (§7, §22, W1): where it goes once `load` resolved, the object it keeps, -// the notices of its object's failures and new attempts, the snapshot and subscription taken in one synchronous step -// when a slot of the window is free, and the ops of its members per batch (sent at once, or pending while the window -// is full), each member with its own Delta encoder. +// A hold arriving on a link and served here (§7, §22, W1): where it goes once the host published (waiting in the +// unbound index until then), why it is unbound (refused: kept in the index while it waits for a name, dependency +// model §1), the object it keeps, the notices of its object's failures and new attempts, the snapshot and subscription +// taken in one synchronous step when a slot of the window is free, and the ops of its members per batch (sent at +// once, or pending while the window is full), each member with its own Delta encoder. import type { Context } from "../../context/index.ts"; import { type Encoder, encoder, type Op, PendingOps } from "../../delta/index.ts"; @@ -13,9 +14,16 @@ import { variantNames, variantOrder, } from "../contracts/internal.ts"; -import { type ChordError, isChordError } from "../data/index.ts"; +import { type ChordError, type FailureCause, isChordError, type Principal } from "../data/index.ts"; import { asChordError } from "../data/internal.ts"; -import type { Caller, ChordObject, Holder } from "../host/internal.ts"; +import { + type Caller, + type ChordObject, + type Holder, + retryOperation, + type Serving, + type UnboundRequest, +} from "../host/internal.ts"; import { type BoundEntry, type FailedMember, @@ -56,12 +64,13 @@ type Followed = { }; /** - * Where a hold stands: `pending` until its object is live (waiting for `load`, the activation and its retries; - * `toldFailed` once the consumer was told of a failure, so the next attempt is announced with `loading`); `binding` - * while its object is live and its `bound` waits for a slot; `bound` with the object's variant and the members it - * follows. + * Where a hold stands: `unplaced` in the unbound index until the host's publication places it; `pending` until its + * object is live (the activation, parking and retries; `toldFailed` once the consumer was told of a failure, so the + * next attempt is announced with `loading`); `binding` while its object is live and its `bound` waits for a slot; + * `bound` with the object's variant and the members it follows. */ type HoldState = + | { readonly kind: "unplaced"; readonly request: HoldRequest } | { readonly kind: "pending"; readonly toldFailed: boolean } | { readonly kind: "binding" } | { @@ -72,8 +81,12 @@ type HoldState = /** The members it follows, in the order they were subscribed. */ readonly followed: Followed[]; } - /** Refused where it arrived (host address, `authorize`, a mismatch, invalid params, a cycle): `retry` restarts it. */ - | { readonly kind: "refused"; readonly request: HoldRequest } + /** + * Refused where it arrived, with `cause`: `waiting` (not served here, another version, placed elsewhere by the + * provider's code) in the unbound index until a publication wakes it; `broken` (invalid params, a cycle, a route that + * refused, a part past the limits) until a `retry` restarts it. + */ + | { readonly kind: "refused"; readonly request: HoldRequest; readonly cause: FailureCause } | { readonly kind: "released" }; /** A bound hold's state. */ @@ -86,11 +99,13 @@ const BINDING: HoldState = { kind: "binding" }; /** * One hold served here: a reference to its object acting as its caller (the hold's principal). While pending, each * failure of its object is sent as `failed` (with the rest of its backoff window) and the next attempt as `loading`; a - * hold that cannot be taken (a refused type, a mismatch, invalid params or a wrong host address) is a broken `failed` - * that holds nothing until a `retry` starts it again. Once live it binds when its link's window has a slot, then sends a part per batch that changed its members - * (or keeps their ops pending while the window is full), and `gone` when its object ends. + * hold that cannot be taken holds nothing: `failed { waiting }` (not served, another version) until a publication + * re-runs its start, `failed { broken }` (invalid params, a cycle) until a `retry` does. Once live it binds when its + * link's window has a slot, then sends a part per batch that changed its members (or keeps their ops pending while the + * window is full), and `gone` when its object ends. It counts as one of the host's references from its arrival to its + * end, bound or not. */ -export class Hold implements Holder { +export class Hold implements Holder, UnboundRequest { readonly kind = "served"; readonly id: number; /** A hold is declared by no object on this host. */ @@ -103,11 +118,6 @@ export class Hold implements Holder { #state: HoldState = PENDING; /** The held object, from its binding until the hold is released. */ #object: ChordObject | undefined; - /** - * Resolved on release: a pending hold stops waiting for `load`. A plain resolution, not an abort: an aborted wait - * rejects with an error whose stack keeps this hold, and the waited promise keeps that rejection. - */ - readonly #released = Promise.withResolvers(); /** The members the current batch changed, with their ops, in order. */ #changes: (readonly [Followed, readonly Op[]])[] = []; /** The members that failed since the last message that carried this hold. */ @@ -115,14 +125,15 @@ export class Hold implements Holder { /** The members whose document was retired since the last message that carried this hold (ORed while it waits). */ #retired: readonly number[] = NO_RETIRED; #touched = false; - /** Whether the hold counts as one of the host's references (`host.activity`): while it keeps its object. */ - #counted = false; + /** Whether the hold counts as one of the host's references (`host.activity`): from its arrival until it ends. */ + #counted = true; constructor(id: number, provider: ProviderSide, caller: Caller, limits: Limits) { this.id = id; this.#provider = provider; this.caller = caller; this.limits = limits; + provider.link.host.count("references", 1); } /** Whether the hold is live: calls on it reach its object. */ @@ -130,33 +141,37 @@ export class Hold implements Holder { return this.#state.kind === "bound"; } - /** Whether the hold waits for `load`: admitted, nothing taken yet (revocation ends it, §4). */ - get waiting(): boolean { - return this.#state.kind === "pending" && this.#object === undefined; + /** Whom it acts for (the unbound index's revocation). */ + get principal(): Principal | undefined { + return this.caller.principal; + } + + /** Whether the hold holds no object: before the publication, or refused (revocation ends it, §4). */ + get unbound(): boolean { + return this.#object === undefined && this.#state.kind !== "released"; } /** - * Admits the hold once `load` resolved: its params within the link's part limit (`too_large/part`), then relayed - * (it is replaced by a relayed hold), or served here, its params opened and parsed (a malformed part is a protocol - * error), its object taken and followed (`moved`). A hold that cannot go anywhere, a type that does not match, - * invalid params, a wrong host address or a cycle (Q5) is a broken `failed`; the hold is kept, refused, for a - * `retry`. + * Admits the hold: before the host's publication it waits in the unbound index, placed by the publication. + * Then its params within the link's part limit (`too_large/part`), then relayed (it is replaced by a relayed hold), + * or served here, its params opened and parsed (a malformed part is a protocol error), its object taken and followed + * (`moved`) in the turn it arrived. A hold that cannot be served here waits refused (`#refuse`). */ - async start(request: HoldRequest): Promise { + start(request: HoldRequest): void { const { link } = this.#provider; - // Waits only for a `load` still running (it never rejects): once loaded, the reference is taken in the turn the - // hold arrived. - const pendingLoad = link.host.pendingLoad; - if (pendingLoad !== undefined) await Promise.race([pendingLoad, this.#released.promise]); - // Released meanwhile (the wait may have settled first): nothing is taken. - if (this.#state.kind !== "pending") return; - let object: ChordObject; + if (!link.host.published) { + this.#state = { kind: "unplaced", request: owned(request) }; + link.host.index(request.type, this); + return; + } + let served: Serving; try { const oversize = oversized(request.params, link.limits.part); if (oversize !== undefined) throw oversize; const placed = link.host.place(request, link, this.caller, link.context); if (placed.kind === "path") { - this.#state = { kind: "released" }; + // Counted where it is served: a relayed hold is not this host's reference. + this.drop(); new RelayedHold(this.id, this.#provider, request, this.caller, placed.path).start(); return; } @@ -170,18 +185,35 @@ export class Hold implements Holder { link.close(); return; } - object = link.host.serving(request, params, this.caller.principal!, link.context); + served = link.host.serving(request, params, this.caller.principal!, link.context); } catch (error) { - return this.#refuse(request, asChordError(error, request.type)); + this.#refuse(request, { kind: "broken", error: asChordError(error, request.type) }); + return; } - // Counted before the object lists it in its presence: a derive over that may close this link synchronously. + if (served.kind === "waiting") { + this.#refuse(request, served.cause); + return; + } + const { object } = served; this.#object = object; - this.#counted = true; - link.host.count("references", 1); object.hold(this); this.moved(); } + /** + * Woken by the unbound index (out of it already): before the publication, placed now; refused `waiting`, its start + * runs again, announced with `loading` (then `bound` or `failed`). + */ + wake(): void { + const state = this.#state; + if (state.kind === "unplaced") { + this.#state = PENDING; + this.start(state.request); + } else if (state.kind === "refused") { + this.#restart(state.request); + } + } + /** * The object's state changed, or the hold took it: a pending hold binds once it is live (when a slot is free), * sends each failure, and announces the attempt after a failure with `loading`. A live object stays live until it @@ -195,6 +227,7 @@ export class Hold implements Holder { this.#state = BINDING; this.#provider.bind(this, this.#provider.link.context); } else if (state.kind === "failed") { + // Each failure and each new cause (a parked object's blocker changed) is a message of its own. const { cause, retryAt } = state; const retryIn = retryAt === undefined ? {} : { retryIn: Math.max(0, retryAt - Date.now()) }; const failed = { kind: cause.kind, error: wireError(cause.error) }; @@ -207,18 +240,26 @@ export class Hold implements Holder { } /** - * A `retry` message: the object is asked for an attempt now (it ignores this unless failed); a refused hold is - * started again (a route's `authorize` may admit it now, a type may be served here now). + * A `retry` message, of its retry operation (a new one when it names none) through the objects it names: the object is + * asked for an attempt now (it ignores this unless failed, or when the operation reached it before; parked, it + * passes it on); a refused hold is + * started again (a route's `authorize` may admit it now). */ - retry(): void { + retry({ operation, through = [] }: Extract): void { const state = this.#state; if (state.kind !== "refused") { - this.#object?.retry(); + this.#object?.retry(operation === undefined ? retryOperation() : { id: operation, through }); return; } + if (state.cause.kind === "waiting") this.#provider.link.host.unindex(state.request.type, this); + this.#restart(state.request); + } + + /** Starts a refused hold again, announced with `loading`. */ + #restart(request: HoldRequest): void { this.#state = PENDING; this.#provider.send({ kind: "loading", hold: this.id }); - void this.start(state.request); + this.start(request); } /** Calls a method of the held object as the hold's caller. */ @@ -324,18 +365,21 @@ export class Hold implements Holder { /** Ends the hold without releasing its object (the host closes and ends its objects itself). */ drop(): void { - if (this.#state.kind === "bound") { - for (const { source, subscriber } of this.#state.followed) source.unsubscribe(subscriber); + const state = this.#state; + const { host } = this.#provider.link; + if (state.kind === "bound") { + for (const { source, subscriber } of state.followed) source.unsubscribe(subscriber); + } else if (state.kind === "unplaced" || (state.kind === "refused" && state.cause.kind === "waiting")) { + host.unindex(state.request.type, this); } this.#state = { kind: "released" }; - this.#released.resolve(); - if (this.#counted) this.#provider.link.host.count("references", -1); + if (this.#counted) host.count("references", -1); this.#counted = false; } /** - * The object ended while held (revocation; a reload, slice 7), or the hold was revoked while it waited for `load`: - * the hold sends `gone` with `error`. + * The object ended while held (revocation; a reload, slice 7b), or the hold was revoked (or its host closed) while it + * held no object: the hold sends `gone` with `error`. */ end(error: ChordError): void { if (this.#state.kind === "released") return; @@ -426,16 +470,27 @@ export class Hold implements Holder { this.#provider.touched(this); } - /** Refuses the hold: a broken `failed`; it keeps `request` for a `retry`, holding nothing until then. */ - #refuse(request: HoldRequest, error: ChordError): void { - const { params } = request; - // Its params owned, not a view of the message they came in (which the link's buffers may hold). - const owned = typeof params === "string" ? params : (params as Uint8Array).slice(); - this.#state = { kind: "refused", request: { ...request, params: owned } }; - this.#provider.send({ kind: "failed", hold: this.id, cause: { kind: "broken", error: wireError(error) } }); + /** + * Refuses the hold with `cause`, holding nothing: it keeps `request`, indexed under its type while `waiting` (a + * publication starts it again), for a `retry` otherwise. + */ + #refuse(request: HoldRequest, cause: FailureCause): void { + this.#state = { kind: "refused", request: owned(request), cause }; + if (cause.kind === "waiting") this.#provider.link.host.index(request.type, this); + this.#provider.send({ + kind: "failed", + hold: this.id, + cause: { kind: cause.kind, error: wireError(cause.error) }, + }); } } +/** `request` with its params owned, not a view of the message they came in (which the link's buffers may hold). */ +function owned(request: HoldRequest): HoldRequest { + const { params } = request; + return typeof params === "string" ? request : { ...request, params: (params as Uint8Array).slice() }; +} + /** A `hold` message's fields naming whom it acts for. */ type ActingFor = Pick; diff --git a/packages/chord/src/next/remote/host-link.ts b/packages/chord/src/next/remote/host-link.ts index f0da9e79f..0afe07e20 100644 --- a/packages/chord/src/next/remote/host-link.ts +++ b/packages/chord/src/next/remote/host-link.ts @@ -193,9 +193,9 @@ export class HostLink implements ChannelHandler { } } - /** Ends the holds on this link still waiting for `load` that act for a principal `revoked` names (§4). */ - revokeWaiting(revoked: (principal: Principal) => boolean, error: ChordError): void { - for (const provider of this.#providers.values()) provider.revokeWaiting(revoked, error); + /** Ends the holds on this link that hold nothing and act for a principal `revoked` names (§4). */ + revokeUnbound(revoked: (principal: Principal) => boolean, error: ChordError): void { + for (const provider of this.#providers.values()) provider.revokeUnbound(revoked, error); } closed(): void { diff --git a/packages/chord/src/next/remote/provider.ts b/packages/chord/src/next/remote/provider.ts index 4325f7c12..ca49e18e8 100644 --- a/packages/chord/src/next/remote/provider.ts +++ b/packages/chord/src/next/remote/provider.ts @@ -90,7 +90,7 @@ export class ProviderSide { this.#abort(message.call); break; case "retry": - this.#entries.get(message.hold)?.retry(); + this.#entries.get(message.hold)?.retry(message); break; case "ack": this.#ack(); @@ -128,15 +128,15 @@ export class ProviderSide { } /** - * Ends with `error` every hold of this origin that acts for a principal `revoked` names and still waits: for `load` - * (served here), or for its route's link (relayed). + * Ends with `error` every hold of this origin that acts for a principal `revoked` names and holds nothing: served + * here but unbound (before the publication, refused), or relayed and waiting for its route's link. */ - revokeWaiting(revoked: (principal: Principal) => boolean, error: ChordError): void { + revokeUnbound(revoked: (principal: Principal) => boolean, error: ChordError): void { for (const hold of [...this.#entries.values()]) { // A hold arriving on a link acts for a principal: the link's, or one its parent named. - if (!hold.waiting || !revoked(hold.caller.principal!)) continue; - if (hold.kind === "served") hold.end(error); - else hold.revoked(error); + if (!revoked(hold.caller.principal!)) continue; + if (hold.kind === "served" && hold.unbound) hold.end(error); + else if (hold.kind === "relayed" && hold.waiting) hold.revoked(error); } } @@ -283,7 +283,7 @@ export class ProviderSide { /** * A hold message: a protocol error for a live id or compressed params (params are never compressed), otherwise - * admitted once `load` resolved (`Hold.start`). Its activation chain is kept only from a trusted link (Q5); its + * admitted (`Hold.start`, placed once the host published). Its activation chain is kept only from a trusted link (Q5); its * decoded limit is the link's, or the smaller one a parent stamped (Q2), and travels with it when it is relayed. */ #hold(message: Extract): void { @@ -310,7 +310,7 @@ export class ProviderSide { const limits = limit === link.limits.decoded ? link.limits : { part: link.limits.part, decoded: limit }; const hold = new Hold(id, this, this.#caller(message), limits); this.#entries.set(id, hold); - void hold.start(request); + hold.start(request); } /** diff --git a/packages/chord/src/next/remote/relay.ts b/packages/chord/src/next/remote/relay.ts index b8857992c..982207a42 100644 --- a/packages/chord/src/next/remote/relay.ts +++ b/packages/chord/src/next/remote/relay.ts @@ -109,10 +109,10 @@ export class RelayedHold { upstream?.link.send({ kind: "abort", origin: upstream.origin, call: id }); } - /** A downstream `retry`, forwarded. */ - retry(): void { + /** A downstream `retry`, forwarded with its operation and the objects it passed through. */ + retry(message: Extract): void { const upstream = this.#upstream; - upstream?.link.send({ kind: "retry", origin: upstream.origin, hold: this.id }); + upstream?.link.send({ ...message, origin: upstream.origin }); } /** Released downstream: released upstream (its calls with it) and no later link holds it. */ diff --git a/packages/chord/src/next/remote/remote-object.ts b/packages/chord/src/next/remote/remote-object.ts index 05ea9cea9..855f04867 100644 --- a/packages/chord/src/next/remote/remote-object.ts +++ b/packages/chord/src/next/remote/remote-object.ts @@ -24,7 +24,7 @@ import { LOADING, type PinnedKey, type RefState, - settlement, + type RetryOperation, type Target, } from "../host/internal.ts"; import { @@ -78,8 +78,6 @@ export class RemoteObject implements Target { /** The hold on the path's link while one is open: that link's consumer side and this hold's id on it. */ #hold: { readonly side: ConsumerSide; readonly id: number } | undefined; #binding: Binding | undefined; - /** Pending while loading, resolved while live, rejected while failed: a dependent's activation waits for it (R18). */ - #settled = settlement(); #lastFailure: ChordError | undefined; constructor( @@ -112,9 +110,14 @@ export class RemoteObject implements Target { return this.#state; } - /** Pending while loading; resolves when bound, rejects with a failure or the reference's end. */ - get settled(): Promise { - return this.#settled.promise; + /** Whether its hold is on a link now (a remote attempt answers for it), not while its path has no link. */ + get heldOnLink(): boolean { + return this.#hold !== undefined; + } + + /** An object over a link is no blocker of this host's first pass. */ + get blocksFirstPass(): false { + return false; } /** The error of the last `failed`, until `bound`. */ @@ -151,12 +154,15 @@ export class RemoteObject implements Target { attach(link: HostLink): void { const chain = this.#dependent?.activationChain(); this.#hold = { side: link.consumer, id: link.consumer.attach(this, chain) }; + // Held on a link now: a parked dependent checks again (`parked_long` counts a hold on a link only). + this.#dependent?.dependencyMoved(); } /** The link dropped: loading until a new link's hold is bound (`#startOver`). */ dropped(_error: ChordError): void { this.#hold = undefined; this.#startOver(); + this.#dependent?.dependencyMoved(); } /** @@ -214,17 +220,18 @@ export class RemoteObject implements Target { } /** - * `ref.retry()`: a failed object's provider is asked for an attempt (a hold it refused is started there again). One - * that failed for a part past the link's limits is placed anew and held with a new id, now or on its path's next - * link (a `hostAddress` that throws fails it broken again). + * `ref.retry()`: a failed object's provider is asked for an attempt (a hold it refused is started there again; a + * parked object there passes it on as an operation of its own host). One that failed for a part past the link's + * limits is placed anew and held with a new id, now or on its path's next link (a `hostAddress` that throws fails it + * broken again). */ - retry(): void { + retry(operation: RetryOperation): void { if (this.#state.kind !== "failed") return; if (this.#path === this.#host.unplaced) { this.#place(); return; } - this.#hold?.side.retry(this.#hold.id); + this.#hold?.side.retry(this.#hold.id, operation); } /** Places a reference whose part was past the limits anew, on its path now (a throwing `hostAddress` fails it). */ @@ -313,17 +320,16 @@ export class RemoteObject implements Target { }; } - /** After a `bound` message's batch: the reference goes live (its parked calls are sent) and R18 retries follow. */ + /** After a `bound` message's batch: the reference goes live (its parked calls are sent, a parked dependent goes on). */ wentLive(): void { this.#lastFailure = undefined; this.#move(LIVE); - // R18: a dependent that failed while waiting for this object is retried now. - this.#binding?.dependent?.dependencyLive(); } /** - * `failed`: the provider's object failed and no attempt runs; `retryIn` is the rest of its backoff window. Ignored - * once live (only a pending hold is told). + * `failed`: the provider's object failed and no attempt runs (`retryIn`: the rest of its backoff window), or the hold + * is unbound there or its object parked on a failed dependency (`waiting`), each new cause a new message. Ignored once + * live (only a pending hold is told). */ failed(message: Extract): void { if (this.#state.kind === "live") return; @@ -383,6 +389,17 @@ export class RemoteObject implements Target { }; } + /** + * Its principal was revoked on this host (`host.revoke`): released at its provider if it is held there, then ended + * with `error` as `lost` does; no later link holds it. + */ + revoke(error: ChordError): void { + const hold = this.#hold; + this.#hold = undefined; + hold?.side.release(hold.id, error); + this.lost(error); + } + /** `gone`, or the host closes: the reference ends, its calls reject with `error`. */ lost(error: ChordError): void { const binding = this.#binding; @@ -399,28 +416,23 @@ export class RemoteObject implements Target { this.#dependent = undefined; this.#hold = undefined; this.#bound = undefined; - this.#settled.reject(error); this.#path.detach(this); for (const replica of bound?.replicas.values() ?? []) replica.end(error); } /** * Loading again until the next binding; the members keep their last values. A drop while it is loading already - * changes nothing: a dependent's activation waiting for it goes on waiting for the next link's binding, bounded by - * its `readyTimeout` (§6, R18; a drop is not a dependency failure). + * changes nothing: a dependent parked on it goes on waiting for the next link's binding (a drop is not a dependency + * failure). */ #startOver(): void { const { kind } = this.#state; if (kind !== "ended" && kind !== "loading") this.#move(LOADING); } - /** Moves to `state` and tells the reference; `settled` follows (a new wait each time it is loading again). */ + /** Moves to `state` and tells the reference. */ #move(state: RefState): void { - const previous = this.#state; this.#state = state; - if (state.kind === "loading" && previous.kind !== "loading") this.#settled = settlement(); - if (state.kind === "live") this.#settled.resolve(); - if (state.kind === "failed") this.#settled.reject(state.cause.error); this.#binding?.moved(); } } diff --git a/packages/chord/test/next/data/failure.test.ts b/packages/chord/test/next/data/failure.test.ts index b833f4d8a..c7e88861e 100644 --- a/packages/chord/test/next/data/failure.test.ts +++ b/packages/chord/test/next/data/failure.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { broken } from "../../../src/next/data/index.ts"; -import { invalid, openFailure } from "../../../src/next/data/internal.ts"; +import { broken, ChordError } from "../../../src/next/data/index.ts"; +import { invalid, openFailure, sameCause, waitingFor } from "../../../src/next/data/internal.ts"; describe("failure causes", () => { it("a value thrown by an open is transient unless marked broken", () => { @@ -25,3 +25,65 @@ describe("failure causes", () => { expect(openFailure(broken(chord), "open")).toEqual({ kind: "broken", error: chord }); }); }); + +describe("a dependent's waiting cause (dependency model §3)", () => { + const root = new ChordError({ code: "unavailable", message: "app.catalog is not served here" }); + + it("names the nearest blocker and the root cause, its blocker's error as the cause", () => { + const cause = waitingFor("app.chat(S1)", root); + expect(cause.kind).toBe("waiting"); + expect(cause.error.toJSON()).toEqual({ + code: "unavailable", + message: "waiting for app.chat(S1): app.catalog is not served here", + }); + expect(cause.error.cause).toBe(root); + // A chain names its nearest blocker and its root only (fix round 1, Opus F1). + const chain = waitingFor("app.panel", cause.error); + expect(chain.error.message).toBe("waiting for app.panel: app.catalog is not served here"); + }); + + it("reaches a fixed point when a ring of waiting attempts passes its causes round (Opus F1)", () => { + let b = waitingFor("X", new ChordError({ code: "unavailable", message: "X is not served here" })).error; + let rounds = 0; + for (; rounds < 10; rounds++) { + const next = waitingFor("A", waitingFor("B", b).error).error; + if (next.message === b.message) break; + b = next; + } + expect(rounds).toBeLessThanOrEqual(1); + expect(b.message).toBe("waiting for A: X is not served here"); + }); + + it("keeps the root cause within 500 characters: a long label is clipped first (Sol 8)", () => { + const original = new ChordError({ code: "unavailable", message: "ORIGINAL: missing provider" }); + const message = waitingFor(`sol.item(${"x".repeat(600)})`, original).error.message; + expect(message.length).toBeLessThanOrEqual(500); + expect(message).toContain(original.message); + expect(message.startsWith("waiting for sol.item(xxx")).toBe(true); + }); + + it("keeps a long chain's root cause, never repeated ellipses (Sol 8)", () => { + const original = new ChordError({ code: "unavailable", message: "ORIGINAL: missing provider" }); + let error = original; + for (let i = 0; i < 2000; i++) error = waitingFor(`level-${i}`, error).error; + expect(error.message).toBe("waiting for level-1999: ORIGINAL: missing provider"); + }); + + it("clips a long root past its share, and both when both are long", () => { + const long = new ChordError({ code: "unavailable", message: "y".repeat(600) }); + const short = waitingFor("app.a", long).error.message; + expect(short).toHaveLength(500); + expect(short.startsWith("waiting for app.a: yyy")).toBe(true); + const both = waitingFor("z".repeat(600), long).error.message; + expect(both).toHaveLength(500); + expect(both.endsWith("y…")).toBe(true); + }); + + it("is compared by value: kind, code, reason and message", () => { + expect(sameCause(waitingFor("a", root), waitingFor("a", root))).toBe(true); + expect(sameCause(waitingFor("a", root), waitingFor("b", root))).toBe(false); + expect(sameCause({ kind: "broken", error: root }, { kind: "transient", error: root })).toBe(false); + const other = new ChordError({ code: "unavailable", reason: "undeclared", message: root.message }); + expect(sameCause({ kind: "waiting", error: root }, { kind: "waiting", error: other })).toBe(false); + }); +}); diff --git a/packages/chord/test/next/examples/05-errors.ts b/packages/chord/test/next/examples/05-errors.ts index 7ea09797d..fabdaf3b9 100644 --- a/packages/chord/test/next/examples/05-errors.ts +++ b/packages/chord/test/next/examples/05-errors.ts @@ -78,9 +78,10 @@ await attempt("bad params", () => host.get(Echo, { id: "one" }, context)); // An implementation with a member its contract does not declare fails activation. await attempt("unknown member", () => host.get(Sloppy, { id: 1 }, context)); -// A get needs the provider's version and fingerprint; another declaration of the type does not fit. +// A get needs the provider's version and fingerprint; another declaration of the type does not fit. That is not the +// caller's fault (a reload may bring the version): the get waits up to its readyTimeout, then rejects with the cause. const EchoV2 = local("example.echo", { version: 2, params: { id: int() }, members: { id: int() } }); -await attempt("contract mismatch", () => host.get(EchoV2, { id: 1 }, context)); +await attempt("contract mismatch", () => host.get(EchoV2, { id: 1 }, context, { readyTimeout: 100 })); // A factory that throws fails broken: the get rejects at once with its error, and no object is left behind. await attempt("throwing factory", () => host.get(Broken, { id: 1 }, context)); diff --git a/packages/chord/test/next/examples/07-failures.ts b/packages/chord/test/next/examples/07-failures.ts index 2d8bb9697..31dc81a42 100644 --- a/packages/chord/test/next/examples/07-failures.ts +++ b/packages/chord/test/next/examples/07-failures.ts @@ -36,7 +36,8 @@ const facet = defineFacet((host) => { return { status: constant("connected") }; }); host.serve(Inbox, ({ user }, self) => { - // A dependency that fails fails this object transient too; it is retried the moment the dependency is live. + // A dependency that fails parks this object (`failed { waiting }`, no timer of its own, the factory not run + // again); it goes on the moment the dependency is live (31-parked-dependencies.ts). const remote = self.get(Remote, {}); console.log(`${at()}: inbox ${user}, attempt`); return { status: self.derive([remote.status], ([status]) => `${user}: ${status}`) }; diff --git a/packages/chord/test/next/examples/31-parked-dependencies.ts b/packages/chord/test/next/examples/31-parked-dependencies.ts new file mode 100644 index 000000000..938bd0b63 --- /dev/null +++ b/packages/chord/test/next/examples/31-parked-dependencies.ts @@ -0,0 +1,148 @@ +// Parked dependencies (dependency model, slice 7a): a reference is a request that outlives every provider it meets. An +// object whose dependency is not there yet is parked, not failed: it shows `failed { waiting }` naming what it waits +// for (the nearest blocker and the root cause), no timer runs for it, its factory is not run again, and it goes on the +// moment its dependency is live. Here a tab's panel needs `Notes` from a server that does not serve it yet; the server +// restarts with the notes plugin, the tab reconnects, and the panel goes live without anyone asking. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/31-parked-dependencies.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { arr, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type RefState } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import type { NativeWatchable } from "../../../src/watch.ts"; + +const context = BACKGROUND_CONTEXT; +const started = Date.now(); +const at = (): string => `${((Date.now() - started) / 1000).toFixed(1)} s`; + +type Principal = { readonly kind: "server" } | { readonly kind: "owner" }; + +const Server = hostKind("server"); + +/** The server's notes (the notes plugin), placed on the server. */ +const Notes = remote("example.notes", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { editor: { state: { list: arr(str()) }, methods: { add: method(obj({ text: str() })) } } }, +}); + +/** A panel in the tab: a local object over the server's notes. */ +const Panel = local("example.panel", { version: 1, params: { id: int() }, members: { count: int() } }); + +/** A local object the panel's sibling needs, which no plugin of the tab serves. */ +const Theme = local("example.theme", { version: 1, params: {}, members: { name: str() } }); +const Header = local("example.header", { version: 1, params: { id: int() }, members: { title: str() } }); + +const List = defineState("example.list", { + schema: obj({ list: arr(str()) }), + version: 1, + initial: () => ({ list: [] }), +}); + +/** The notes plugin's server facet. */ +const notesPlugin = defineFacet((host) => { + host.serve(Notes, (_params, _principal, self) => { + const list = self.state(List); + return { + variant: "editor", + members: { + list: self.derive([list], ([l]) => l.list), + add: async ({ text }, ctx) => { + await list.change((draft) => void draft.list.push(text), ctx); + return null; + }, + }, + }; + }); +}); + +/** A server with the given plugins, its link to the tab accepted. */ +async function startServer(plugins: { readonly name: string; readonly facet: typeof notesPlugin }[]) { + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + await server.load(plugins, context); + const [up, down] = memoryLinkPair(); + server.accept(up, { principal: { kind: "owner" } }, context); + return { server, tabLink: down, close: () => up.close() }; +} + +// The tab: its panel declares the notes (a dependency); its header a theme nobody serves. +let panels = 0; +const tab = createHost({ onError: (error) => console.log(`${at()}: reported ${error.kind}`) }); +await tab.load( + [ + { + name: "@example/panel", + facet: defineFacet((host) => { + host.serve(Panel, (_, self) => { + panels++; + const notes = self.get(Notes, {}); + return { count: self.derive([notes.list], ([list]) => list.length) }; + }); + host.serve(Header, (_, self) => ({ title: self.get(Theme, {}).name })); + }), + }, + ], + context, +); + +/** Prints every state of `state` as it changes. */ +async function follow(label: string, state: NativeWatchable): Promise { + const watch = await state.watch(context); + watch.start(async (call) => { + if (call.kind === "value") console.log(`${at()}: ${label} ${describe(call.value)}`); + }); +} + +function describe(state: RefState): string { + if (state.kind !== "failed") return state.kind; + return `failed { ${state.cause.kind} }: ${state.cause.error.message}`; +} + +// 1. The first server has no notes plugin. Its answer is `failed { waiting }`: not the panel's fault. +const first = await startServer([]); +tab.connect(first.tabLink, { principal: { kind: "server" } }, context); +const panel = tab.hold(Panel, { id: 1 }, context); +await follow("panel:", panel.state); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log(`${at()}: panel now ${describe(panel.state.value)}`); + +// 2. A local type nobody serves here is the same answer: the request waits unbound, indexed under the type's name. +const header = tab.hold(Header, { id: 1 }, context); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log(`${at()}: header ${describe(header.state.value)}`); + +// 3. A getter waits through `waiting` up to its readyTimeout, then rejects with the cause (a hold just shows it). +try { + await tab.get(Panel, { id: 1 }, context, { readyTimeout: 200 }); +} catch (error) { + if (isChordError(error)) console.log(`${at()}: get gave up after 200 ms: ${error.message}`); +} + +// 4. Time passes: no timer runs for the parked panel, its factory is not run again. +await new Promise((resolve) => setTimeout(resolve, 1500)); +console.log(`${at()}: still ${panel.state.value.kind}, factory runs: ${panels}`); + +// 5. The server restarts with the notes plugin (the link drop shows as loading); the tab reconnects: its request is +// held again and binds, and the parked panel goes on. +first.close(); +await new Promise((resolve) => setTimeout(resolve, 10)); +const second = await startServer([{ name: "@example/notes", facet: notesPlugin }]); +tab.connect(second.tabLink, { principal: { kind: "server" } }, context); +const live = await tab.get(Panel, { id: 1 }, context); +console.log(`${at()}: panel live, count ${live.count.value}, factory runs: ${panels}`); +// The panel's notes are the owner's object on the server (a remote object per principal): the tab adds through its own. +const notes = await tab.get(Notes, {}, context); +await notes.add({ text: "first" }, context); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log(`${at()}: panel count ${live.count.value}`); + +header.release(); +panel.release(); +live.release(); +notes.release(); +await tab.close(context); +await second.server.close(context); +await first.server.close(context); diff --git a/packages/chord/test/next/examples/32-startup.ts b/packages/chord/test/next/examples/32-startup.ts new file mode 100644 index 000000000..2c2169e10 --- /dev/null +++ b/packages/chord/test/next/examples/32-startup.ts @@ -0,0 +1,104 @@ +// Startup (dependency model §6, slice 7a): `load` publishes every facet's declarations in one batch first, then starts +// the host-lifetime objects; requests made here and holds arriving on links wait only for that publication, never for +// the start barrier. Two hosts whose objects need each other's types (an acyclic graph) used to deadlock: the server's +// eager `Bootstrap` needs the worker's `Agent`, the `Agent` needs the server's `Catalog`, and the worker's hold for +// `Catalog` waited for the server's whole `load`, which waited for `Bootstrap`. Now it starts at once, no timeout. +// `load` resolves when each host-lifetime object had its first outcome: live, failed by its own cause, or parked with +// no blocker here still without one (a wait on another host ends the first pass). +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/32-startup.ts +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { hostKind, int, local, remote, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; + +const context = BACKGROUND_CONTEXT; +const started = Date.now(); +const at = (): string => `${Date.now() - started} ms`; + +type Principal = { readonly kind: "server" } | { readonly kind: "worker"; readonly session: string }; + +const Server = hostKind("server"); +const Session = hostKind("session", { id: str() }); + +/** On the server, a host-lifetime object (local, empty params): it starts at `load`. */ +const Bootstrap = local("example.bootstrap", { version: 1, params: {}, members: { agents: int() } }); + +/** On the session's worker. */ +const Agent = remote("example.agent", { + version: 1, + params: { session: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: { v: { state: { models: int() } } }, +}); + +/** On the server. */ +const Catalog = remote("example.catalog", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { models: int() } } }, +}); + +/** The worker of a session, started by the server's route; it connects first, then loads (W6). */ +function startWorker(id: string) { + const [up, down] = memoryLinkPair(); + const worker = createHost({ + address: Session({ id }), + principal: { kind: "worker", session: id }, + onError: (error) => console.log(`${at()}: worker reported ${error.kind}`), + }); + worker.connect(down, { principal: { kind: "server" }, trusted: true }, context); + const agents = defineFacet((host) => + host.serve(Agent, (_params, _principal, self) => { + console.log(`${at()}: worker: Agent declares the server's Catalog`); + return { variant: "v", members: { models: self.get(Catalog, {}).models } }; + }), + ); + void worker + .load([{ name: "@example/agents", facet: agents }], context) + .then(() => console.log(`${at()}: worker loaded`)); + return { worker, link: up }; +} + +const workers: ReturnType[] = []; +const server = createHost({ + address: Server(), + principal: { kind: "server" }, + // No readyTimeout is needed to break anything: a long one shows nothing waits for it. + readyTimeout: 600_000, + onError: (error) => console.log(`${at()}: server reported ${error.kind}`), +}); +server.route( + Session, + { + authorize: () => true, + resolve: async (address) => { + const started = startWorker(address.params.id); + workers.push(started); + return { link: started.link, principal: { kind: "worker", session: address.params.id }, trusted: true }; + }, + }, + context, +); +const core = defineFacet((host) => { + host.serve(Catalog, () => ({ variant: "v", members: { models: constant(3) } })); + host.serve(Bootstrap, (_, self) => { + console.log(`${at()}: server: Bootstrap declares the worker's Agent`); + return { agents: self.get(Agent, { session: "S1" }).models }; + }); +}); + +// 1. The server's load: publication (Catalog served), then Bootstrap starts and parks on the worker's Agent. A wait on +// another host is a first outcome, so `load` resolves without waiting for the worker. +await server.load([{ name: "@example/core", facet: core }], context); +console.log(`${at()}: server loaded; Bootstrap is ${server.hold(Bootstrap, {}, context).state.value.kind}`); + +// 2. The worker's hold for Catalog bound on the published server; Agent went live, then Bootstrap. +const bootstrap = await server.get(Bootstrap, {}, context); +console.log(`${at()}: Bootstrap live, it sees ${bootstrap.agents.value} models through the worker`); + +bootstrap.release(); +await server.close(context); +for (const { worker } of workers) await worker.close(context); diff --git a/packages/chord/test/next/host/activity.test.ts b/packages/chord/test/next/host/activity.test.ts index cdfa929b7..63ac47af6 100644 --- a/packages/chord/test/next/host/activity.test.ts +++ b/packages/chord/test/next/host/activity.test.ts @@ -27,6 +27,19 @@ describe("host.activity", () => { expect(server.activity.value.references).toBe(0); }); + it("counts a hold placed here from its arrival, bound or not: a missing provider keeps the worker alive (M§10)", async () => { + const { server, client } = await linked(() => {}); + const missing = client.hold(Notes, { session: "S1" }, context); + await transported(); + expect(missing.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + expect(server.activity.value.references).toBe(1); + // Presence lists bound references only. + expect(server.references(Notes).value).toEqual([]); + missing.release(); + await transported(); + expect(server.activity.value.references).toBe(0); + }); + it("counts method calls running on this host, local and remote, until they settle", async () => { const { server, client } = await linked((h) => void serveNotes(h)); const remoteRef = await client.get(Notes, { session: "S1" }, context); diff --git a/packages/chord/test/next/host/attempt.test.ts b/packages/chord/test/next/host/attempt.test.ts index d1227bc4a..088ee54ec 100644 --- a/packages/chord/test/next/host/attempt.test.ts +++ b/packages/chord/test/next/host/attempt.test.ts @@ -2,7 +2,7 @@ import { createHook } from "node:async_hooks"; import { afterEach, beforeEach, describe, expect, expectTypeOf, it, vi } from "vitest"; import { type Context, BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; import { arr, int, local, method, obj, opt, remote, str } from "../../../src/next/contracts/index.ts"; -import { broken, isChordError } from "../../../src/next/data/index.ts"; +import { broken, ChordError, isChordError } from "../../../src/next/data/index.ts"; import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; import { createHost, @@ -12,6 +12,7 @@ import { type Self, type StoreSelf, } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; import { defineRegistry } from "../../../src/next/registries/index.ts"; import { constant } from "../../../src/next/state/index.ts"; import type { StagedWrite } from "../../../src/next/store/step.ts"; @@ -22,9 +23,11 @@ import { evolveTable } from "../../../src/next/store/table-evolve.ts"; import { TableHandle } from "../../../src/next/store/table-handle.ts"; import { prepareTables } from "../../../src/next/store/table-migration.ts"; import { + failLive, failures, fakeTimers, flushed, + instancesOf, loaded, memoryStore, metered, @@ -33,6 +36,8 @@ import { tableOwner, } from "../support.ts"; +const objects = instancesOf(ChordObject, "attempt"); + const List = defineState("list", { schema: obj({ items: arr(str()) }), version: 1, initial: () => ({ items: [] }) }); const Page = local("app.page", { version: 1, params: { id: int() }, members: { label: str(), slow: method() } }); @@ -892,3 +897,93 @@ describe("eager migrations at activation (slice 6a §9)", () => { await store.close(context); }); }); + +describe("parked attempts (dependency model §3)", () => { + fakeTimers(); + const View = local("app.view", { version: 1, params: {}, members: { n: int() } }); + const Missing = local("app.missing", { version: 1, params: {}, members: { n: int() } }); + /** A view with params: no host lifetime, so `load` does not wait for it. */ + const Viewer = local("app.viewer", { version: 1, params: { id: int() }, members: { n: int() } }); + + it("run no timer while parked: no dependency deadline, no backoff, the factory run once", async () => { + const factories = vi.fn(); + const { host, errors } = await loaded((h) => + h.serve(View, (_, self) => { + factories(); + return { n: self.get(Missing, {}).n }; + }), + ); + const view = host.hold(View, {}, context); + await flushed(); + await vi.advanceTimersByTimeAsync(10 * 60_000); + expect(view.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.missing: app.missing is not served here" } }, + retryAt: undefined, + }); + expect(vi.getTimerCount()).toBe(0); + expect([factories.mock.calls.length, errors]).toEqual([1, []]); + view.release(); + }); + + it("pause the readyTimeout deadline while parked: the own steps get the budget left", async () => { + const Slow = local("app.slow", { version: 1, params: { id: int() }, members: { n: int() } }); + const { host, errors } = await loaded( + (h) => { + h.serve(Slow, (_, self) => { + self.open( + () => new Promise((resolve) => setTimeout(resolve, 5_000)), + () => {}, + ); + return { n: constant(1) }; + }); + h.serve(Viewer, (_, self) => { + const slow = self.get(Slow, { id: 1 }); + self.open( + () => new Promise(() => {}), + () => {}, + ); + return { n: slow.n }; + }); + }, + { readyTimeout: 10_000 }, + ); + const view = host.hold(Viewer, { id: 1 }, context); + // Parked 5 s on its dependency; then its own open hangs: it fails 10 s after the boundary, not 10 s after start. + await vi.advanceTimersByTimeAsync(5_000 + 9_999); + expect(failures(errors)).toEqual([]); + await vi.advanceTimersByTimeAsync(1); + expect(failures(errors)).toEqual(["app.viewer(1): transient"]); + view.release(); + }); + + it("go on past the dependency check: a dependency leaving live during the own steps changes nothing (R18)", async () => { + const Dependency = local("app.dependency", { version: 1, params: {}, members: { n: int() } }); + const gate = Promise.withResolvers(); + let dependencySelf: Self | undefined; + const { host, errors } = await loaded((h) => { + h.serve(Dependency, (_, self) => { + dependencySelf = self; + return { n: constant(1) }; + }); + h.serve(Viewer, (_, self) => { + const dependency = self.get(Dependency, {}); + self.open( + () => gate.promise, + () => {}, + ); + return { n: dependency.n }; + }); + }); + const view = host.hold(Viewer, { id: 1 }, context); + await flushed(); + // The view opens its own native; its dependency fails live meanwhile. + failLive(objects, dependencySelf!, new ChordError({ code: "unavailable", message: "offline" })); + await flushed(); + gate.resolve(); + await flushed(); + expect(view.state.value).toEqual({ kind: "live" }); + expect(failures(errors)).toEqual([]); + view.release(); + }); +}); diff --git a/packages/chord/test/next/host/dependencies.test.ts b/packages/chord/test/next/host/dependencies.test.ts new file mode 100644 index 000000000..ca413718e --- /dev/null +++ b/packages/chord/test/next/host/dependencies.test.ts @@ -0,0 +1,294 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, remote } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet, type HostError } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { + bindHold, + failures, + fakeTimers, + flushed, + loaded, + Notes, + rawParent, + rejection, + relayed, + Server, + Session, + statesOf, + type TestPrincipal, + transported, +} from "../support.ts"; + +const View = local("app.view", { version: 1, params: { id: int() }, members: { n: int() } }); + +describe("the dependency check", () => { + it("breaks a dependent off a cycle with the cycle's error: it would wait on a ring that never heals", async () => { + const Ping = local("app.ping", { version: 1, params: {}, members: { n: int() } }); + const Pong = local("app.pong", { version: 1, params: {}, members: { n: int() } }); + const { host, errors } = await loaded((h) => { + h.serve(Ping, (_, self) => ({ n: self.get(Pong, {}).n })); + h.serve(Pong, (_, self) => ({ n: self.get(Ping, {}).n })); + h.serve(View, (_, self) => ({ n: self.get(Ping, {}).n })); + }); + const view = host.hold(View, { id: 1 }, context); + await flushed(); + expect(view.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "broken", error: { code: "invalid", reason: "cycle" } }, + }); + expect(failures(errors)).toContain("app.view(1): broken"); + view.release(); + }); + + it("breaks a dependent whose dependency ended (gone): the request cannot come back", async () => { + const errors: HostError[] = []; + const client = createHost({ onError: (error) => errors.push(error) }); + const peer = await rawParent(client); + await client.load( + [ + { + name: "@test/view", + facet: defineFacet((h) => + h.serve(View, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ), + }, + ], + context, + ); + const view = client.hold(View, { id: 1 }, context); + const { hold } = await peer.next("hold"); + const revoked = { code: "forbidden", reason: "revoked", message: "revoked" } as const; + peer.send({ kind: "gone", hold, error: { kind: "chord", error: revoked } }); + await transported(); + expect(view.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken", error: revoked } }); + expect(failures(errors)).toEqual(["app.view(1): broken"]); + view.release(); + }); + + it("goes live once every dependency is: one parked on two, the second binding last", async () => { + const client = createHost({ onError: () => {} }); + const peer = await rawParent(client); + const facet = defineFacet((h) => + h.serve(View, (_, self) => { + const a = self.get(Notes, { session: "S1" }).count; + const b = self.get(Notes, { session: "S2" }).count; + return { n: self.derive([a, b], ([x, y]) => x + y) }; + }), + ); + await client.load([{ name: "@test/view", facet }], context); + const view = client.hold(View, { id: 1 }, context); + const first = await peer.next("hold"); + const second = await peer.next("hold"); + bindHold(peer, first.hold, [{ text: "a" }]); + await transported(); + expect(view.state.value).toEqual({ kind: "loading" }); + bindHold(peer, second.hold, [{ text: "b" }, { text: "c" }]); + await transported(); + expect([view.state.value, view.n.value]).toEqual([{ kind: "live" }, 3]); + view.release(); + }); + + it("breaks a dependent whose dependency a relay's authorize refused: forbidden heals only by retry() (fix round 1, Fable)", async () => { + const errors: HostError[] = []; + const client = createHost({ onError: (error) => errors.push(error) }); + const peer = await rawParent(client); + const facet = defineFacet((h) => + h.serve(View, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ); + await client.load([{ name: "@test/view", facet }], context); + const view = client.hold(View, { id: 1 }, context); + const { hold } = await peer.next("hold"); + const forbidden = { code: "forbidden", message: "session|S1: not authorized" } as const; + peer.send({ kind: "failed", hold, cause: { kind: "broken", error: { kind: "chord", error: forbidden } } }); + await transported(); + expect(view.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken", error: forbidden } }); + expect(failures(errors)).toEqual(["app.view(1): broken"]); + view.release(); + }); + + it("settles a waiting ring across hosts after one round: the root cause kept (fix round 1, Opus F1)", async () => { + const A = remote("app.a", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); + const B = remote("app.b", { + version: 1, + params: {}, + hostAddress: () => Session({ id: "S1" }), + variants: { v: { state: { n: int() } } }, + }); + const X = local("app.x", { version: 1, params: {}, members: { n: int() } }); + const Kick = local("app.kick", { version: 1, params: {}, members: { n: int() } }); + const Front = local("app.front", { version: 1, params: {}, members: { n: int() } }); + const Back = local("app.back", { version: 1, params: {}, members: { n: int() } }); + const { workers } = await relayed({ + trusted: true, + server: (h) => { + // The server's host object starts the worker; its A and the worker's B then start in the same turns, so + // the activation chains miss the ring (M§7's gap). + h.serve(Kick, (_p, self) => ({ n: self.get(B, {}).n })); + h.serve(A, (_p, _principal, self) => ({ variant: "v", members: { n: self.get(B, {}).n } })); + }, + worker: (h) => { + h.serve(Front, (_p, self) => ({ n: self.get(A, {}).n })); + h.serve(Back, (_p, self) => ({ n: self.get(B, {}).n })); + h.serve(B, (_p, _principal, self) => { + const a = self.get(A, {}); + self.get(X, {}); + return { variant: "v", members: { n: a.n } }; + }); + }, + }); + await transported(); + const worker = workers.get("session|S1")!; + const b = worker.hold(B, {}, context); + for (let round = 0; round < 50; round++) await transported(); + const states = statesOf(await b.state.watch(context)); + for (let round = 0; round < 50; round++) await transported(); + expect(states).toEqual([]); + expect(b.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: expect.stringMatching(/: app\.x is not served here$/) } }, + }); + b.release(); + }); +}); + +describe("parked_long (dependency model §7)", () => { + fakeTimers(); + + it("starts its clock when the hold gets on a link: a late link is not reported at once (fix round 2, Opus F2)", async () => { + const errors: HostError[] = []; + const client = createHost({ readyTimeout: 1_000, onError: (error) => errors.push(error) }); + const facet = defineFacet((h) => + h.serve(View, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ); + await client.load([{ name: "@test/view", facet }], context); + const view = client.hold(View, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(5_000); + const peer = await rawParent(client); + await peer.next("hold"); + const linked = Date.now(); + await vi.advanceTimersByTimeAsync(10); + expect(errors).toEqual([]); + await vi.advanceTimersByTimeAsync(990); + expect(errors).toEqual([ + { kind: "parked_long", object: "app.view(1)", waitingFor: "app.notes(S1)", since: linked }, + ]); + view.release(); + }); + + it("is not reported while the path has no link: no remote attempt answers for it (fix round 1, Opus F2)", async () => { + const errors: HostError[] = []; + const client = createHost({ readyTimeout: 1_000, onError: (error) => errors.push(error) }); + const facet = defineFacet((h) => + h.serve(View, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ); + await client.load([{ name: "@test/view", facet }], context); + const view = client.hold(View, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(5_000); + expect(errors).toEqual([]); + expect(view.state.value).toEqual({ kind: "loading" }); + // Once a link holds it and the remote attempt is only loading, it is. + const peer = await rawParent(client); + await peer.next("hold"); + await vi.advanceTimersByTimeAsync(5_000); + expect(errors.map(({ kind }) => kind)).toEqual(["parked_long"]); + view.release(); + }); + + it("is reported once per parking episode, after readyTimeout on a remote attempt only loading; nothing fails", async () => { + const errors: HostError[] = []; + const client = createHost({ readyTimeout: 1_000, onError: (error) => errors.push(error) }); + const peer = await rawParent(client); + const facet = defineFacet((h) => + h.serve(View, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ); + await client.load([{ name: "@test/view", facet }], context); + const view = client.hold(View, { id: 1 }, context); + await peer.next("hold"); + await vi.advanceTimersByTimeAsync(999); + expect(errors).toEqual([]); + await vi.advanceTimersByTimeAsync(1); + expect(errors).toEqual([{ kind: "parked_long", object: "app.view(1)", waitingFor: "app.notes(S1)", since: 0 }]); + await vi.advanceTimersByTimeAsync(60_000); + expect(errors).toHaveLength(1); + expect(view.state.value).toEqual({ kind: "loading" }); + view.release(); + }); + + it("is not reported for a remote blocker that failed: the parked attempt shows its cause instead", async () => { + const errors: HostError[] = []; + const client = createHost({ readyTimeout: 1_000, onError: (error) => errors.push(error) }); + const peer = await rawParent(client); + const facet = defineFacet((h) => + h.serve(View, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ); + await client.load([{ name: "@test/view", facet }], context); + const view = client.hold(View, { id: 1 }, context); + const { hold } = await peer.next("hold"); + const error = { kind: "chord", error: { code: "unavailable", message: "app.notes is not served here" } } as const; + peer.send({ kind: "failed", hold, cause: { kind: "waiting", error } }); + await vi.advanceTimersByTimeAsync(60_000); + expect(errors).toEqual([]); + expect(view.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + expect(vi.getTimerCount()).toBe(0); + view.release(); + }); + + it("is not reported for a local blocker: its own deadline bounds it", async () => { + const Slow = local("app.slow", { version: 1, params: { id: int() }, members: { n: int() } }); + const { host, errors } = await loaded( + (h) => { + h.serve(Slow, (_, self) => { + self.open( + () => new Promise(() => {}), + () => {}, + ); + return { n: constant(1) }; + }); + h.serve(View, (_, self) => ({ n: self.get(Slow, { id: 1 }).n })); + }, + { readyTimeout: 1_000 }, + ); + const view = host.hold(View, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(5_000); + expect(errors.map(({ kind }) => kind)).not.toContain("parked_long"); + view.release(); + }); +}); + +describe("a getter of a parked object (fix round 1, Astra 6)", () => { + it("rejects with the last parked cause after its dependency fell back to loading, the newest of several", async () => { + const host = createHost({ onError: () => {} }); + const peer = await rawParent(host); + await host.load( + [ + { + name: "@test/view", + facet: defineFacet((h) => + h.serve(View, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ), + }, + ], + context, + ); + const pending = rejection(host.get(View, { id: 1 }, context, { readyTimeout: 80 })); + const { hold } = await peer.next("hold"); + for (const message of ["first absence", "specific provider absence"]) { + const error = { kind: "chord", error: { code: "unavailable", message } } as const; + peer.send({ kind: "failed", hold, cause: { kind: "waiting", error } }); + await transported(); + } + peer.send({ kind: "loading", hold }); + await transported(); + expect(await pending).toMatchObject({ + code: "unavailable", + message: "waiting for app.notes(S1): specific provider absence", + }); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/host.test.ts b/packages/chord/test/next/host/host.test.ts index 0074d240e..8c9e0528f 100644 --- a/packages/chord/test/next/host/host.test.ts +++ b/packages/chord/test/next/host/host.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; -import { int, local, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { hostKind, int, local, method, remote, str } from "../../../src/next/contracts/index.ts"; import { broken } from "../../../src/next/data/index.ts"; import { type Ctx, @@ -13,6 +13,7 @@ import { import type { ChordHost } from "../../../src/next/host/internal.ts"; import { memoryLinkPair } from "../../../src/next/link/index.ts"; import { jsonPart } from "../../../src/next/link/internal.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; import { constant } from "../../../src/next/state/index.ts"; import { sqliteBackend } from "../../../src/next/store/backend.ts"; import type { StagedWrite } from "../../../src/next/store/step.ts"; @@ -31,6 +32,7 @@ import { partValue, RawPeer, rejection, + relayed, SERVER_ADDRESS, Server, Session, @@ -153,19 +155,296 @@ describe("the start barrier", () => { await vi.advanceTimersByTimeAsync(1); expect(done).toBe(true); }); + + it("waits for a local blocker's first outcome: a host object parked on another one here (CodeMode → Harness)", async () => { + const Harness = local("app.harness", { version: 1, params: {}, members: { n: int() } }); + const CodeMode = local("app.codemode", { version: 1, params: {}, members: { n: int() } }); + const opened = Promise.withResolvers(); + const host = createHost(noErrors); + const facet = defineFacet((h) => { + h.serve(Harness, (_, self) => { + self.open( + () => opened.promise, + () => {}, + ); + return { n: constant(1) }; + }); + h.serve(CodeMode, (_, self) => ({ n: self.get(Harness, {}).n })); + }); + let done = false; + const loading = host.load([{ name: "@app/core", facet }], context).then(() => { + done = true; + }); + await flushed(); + expect(done).toBe(false); + opened.resolve(); + await loading; + expect(host.hold(CodeMode, {}, context).state.value).toEqual({ kind: "live" }); + }); + + it("ends its pass at a local blocker's own failure: the dependent parked failed { waiting } has its first outcome", async () => { + const Harness = local("app.harness", { version: 1, params: {}, members: { n: int() } }); + const CodeMode = local("app.codemode", { version: 1, params: {}, members: { n: int() } }); + const { host, errors } = await loaded((h) => { + h.serve(Harness, (_, self) => { + self.open( + async () => { + throw new Error("offline"); + }, + () => {}, + ); + return { n: constant(1) }; + }); + h.serve(CodeMode, (_, self) => ({ n: self.get(Harness, {}).n })); + }); + expect(failures(errors)).toEqual(["app.harness: transient"]); + expect(host.hold(CodeMode, {}, context).state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.harness: app.harness: offline" } }, + }); + }); + + it("ends its pass at a remote blocker: a wait on another host is a first outcome", async () => { + const CodeMode = local("app.codemode", { version: 1, params: {}, members: { n: int() } }); + const host = createHost(noErrors); + const [a, b] = memoryLinkPair(); + host.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + const parent = new RawPeer(b); + const facet = defineFacet((h) => + h.serve(CodeMode, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ); + await host.load([{ name: "@app/codemode", facet }], context); + expect(await parent.next("hold")).toMatchObject({ type: Notes.name }); + expect(host.hold(CodeMode, {}, context).state.value).toEqual({ kind: "loading" }); + }); + + it("is settled by host.close: an object of its cohort that ends before its first outcome settles it as ended", async () => { + const host = createHost(noErrors); + const facet = defineFacet((h) => + h.serve(Info, (_, self) => { + self.open( + () => new Promise(() => {}), + () => {}, + ); + return { title: constant("t") }; + }), + ); + let done = false; + const loading = host.load([{ name: "a", facet }], context).then(() => { + done = true; + }); + await flushed(); + expect(done).toBe(false); + await host.close(context); + await loading; + expect(done).toBe(true); + }); + + it("waits for a local blocker parked failed { waiting } on one still in its first attempt (fix round 1, Sol 4)", async () => { + const Core = local("app.core", { version: 1, params: {}, members: { n: int() } }); + const Middle = local("app.middle", { version: 1, params: { id: int() }, members: { n: int() } }); + const Slow = local("app.slow", { version: 1, params: { id: int() }, members: { n: int() } }); + const Missing = local("app.missing", { version: 1, params: {}, members: { n: int() } }); + const gate = Promise.withResolvers(); + const host = createHost(noErrors); + const facet = defineFacet((h) => { + h.serve(Slow, (_, self) => { + self.open( + () => gate.promise, + () => {}, + ); + return { n: constant(1) }; + }); + h.serve(Middle, ({ id }, self) => { + const slow = self.get(Slow, { id }); + self.get(Missing, {}); + return { n: slow.n }; + }); + h.serve(Core, (_, self) => ({ n: self.get(Middle, { id: 1 }).n })); + }); + let done = false; + const loading = host.load([{ name: "@app/core", facet }], context).then(() => { + done = true; + }); + await flushed(); + expect(host.hold(Middle, { id: 1 }, context).state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting" }, + }); + expect(done).toBe(false); + gate.resolve(); + await loading; + await host.close(context); + }); + + it("waits for it whichever it declares first (fix round 1, Astra 3)", async () => { + const A = local("app.a", { version: 1, params: {}, members: {} }); + const B = local("app.b", { version: 1, params: { id: int() }, members: {} }); + const Slow = local("app.slow", { version: 1, params: { id: int() }, members: {} }); + const Missing = local("app.missing", { version: 1, params: {}, members: {} }); + const gate = Promise.withResolvers(); + const host = createHost(noErrors); + const facet = defineFacet((h) => { + h.serve(A, (_, self) => { + self.get(B, { id: 1 }); + return {}; + }); + h.serve(B, (_, self) => { + self.get(Missing, {}); + self.get(Slow, { id: 1 }); + return {}; + }); + h.serve(Slow, (_, self) => { + self.open( + () => gate.promise, + () => {}, + ); + return {}; + }); + }); + let done = false; + const loading = host.load([{ name: "@app/a", facet }], context).then(() => { + done = true; + }); + await flushed(); + const premature = done; + gate.resolve(); + await loading; + await host.close(context); + expect(premature).toBe(false); + }); +}); + +describe("startup across hosts (dependency model §6)", () => { + // Timers never advance: nothing here may wait for a deadline. + fakeTimers(); + + /** On the server, waiting for a worker's `Agent`. */ + const Bootstrap = local("app.bootstrap", { version: 1, params: {}, members: { n: int() } }); + /** On worker S1, waiting for the server's `Catalog`. */ + const Agent = remote("app.agent", { + version: 1, + params: { session: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: { v: { state: { n: int() } } }, + }); + const Catalog = remote("app.catalog", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); + + it("starts two hosts that would have deadlocked (Sol's probe): an acyclic graph needs no timeout", async () => { + const { server, errors } = await relayed({ + server: (h) => { + h.serve(Bootstrap, (_, self) => ({ n: self.get(Agent, { session: "S1" }).n })); + h.serve(Catalog, () => ({ variant: "v", members: { n: constant(7) } })); + }, + worker: (h) => { + // The worker's own host object waits for the server's catalog during the worker's load. + h.serve(Agent, (_params, _principal, self) => ({ variant: "v", members: { n: self.get(Catalog, {}).n } })); + }, + }); + await transported(); + await transported(); + const bootstrap = server.hold(Bootstrap, {}, context); + expect(bootstrap.state.value).toEqual({ kind: "live" }); + expect(bootstrap.n.value).toBe(7); + expect([errors.server, errors.worker]).toEqual([[], []]); + }); + + it("places a hold arriving before the publication here, not over the parent", async () => { + const worker = createHost({ + address: Session({ id: "S1" }), + principal: { kind: "worker", session: "S1" }, + ...noErrors, + }); + const [a, b] = memoryLinkPair(); + worker.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + const parent = new RawPeer(b); + const [c, d] = memoryLinkPair(); + worker.accept(c, { principal: { kind: "owner" }, codec: testCodec() }, context); + const child = new RawPeer(d); + child.send({ + kind: "hold", + hold: 0, + type: PlacedNotes.name, + version: PlacedNotes.version, + fingerprint: PlacedNotes.fingerprint, + params: jsonPart({ session: "S1" }), + address: { kind: "session", params: { id: "S1" } }, + }); + const mine = worker.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + expect(mine.state.value).toEqual({ kind: "loading" }); + await worker.load( + [{ name: "@test/w", facet: defineFacet((h) => serveNotes(h, PlacedNotes)) }], + context, + ); + expect((await child.next("bound")).holds[0]).toMatchObject({ hold: 0 }); + await transported(); + expect(mine.state.value).toEqual({ kind: "live" }); + expect(parent.kinds()).toEqual(["hello"]); + }); + + it("wakes a parked chain over three hosts in order, its causes passed on (tab → worker → server)", async () => { + const Chat = remote("app.chat", { + version: 1, + params: { session: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: { v: { state: { n: int() } } }, + }); + const order: string[] = []; + let catalogs = 0; + const { tab } = await relayed({ + server: (h) => + h.serve(Catalog, (_params, _principal, self) => { + const attempt = ++catalogs; + self.open( + async () => { + if (attempt === 1) throw new Error("offline"); + }, + () => {}, + ); + self.onLive(async () => void order.push("catalog")); + return { variant: "v", members: { n: constant(attempt) } }; + }), + worker: (h) => + h.serve(Chat, (_params, _principal, self) => { + const catalog = self.get(Catalog, {}); + self.onLive(async () => void order.push("chat")); + return { variant: "v", members: { n: catalog.n } }; + }), + }); + const view = tab.hold(Chat, { session: "S1" }, context); + for (let round = 0; round < 5; round++) await transported(); + expect(view.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.catalog: app.catalog: offline" } }, + retryAt: undefined, + }); + // The catalog's own backoff (1 s) is the only timer: it retries, and the chain goes on in order. + await vi.advanceTimersByTimeAsync(1_000); + for (let round = 0; round < 5; round++) await transported(); + expect(view.state.value).toEqual({ kind: "live" }); + expect(order).toEqual(["catalog", "chat"]); + expect(view.n.value).toBe(2); + }); }); describe("get", () => { - it("refuses an unserved type and a contract mismatch", async () => { + it("waits for an unserved type and a contract mismatch until its readyTimeout, then rejects with the cause", async () => { const { host } = await loaded((h) => h.serve(Info, () => ({ title: constant("t") }))); - expect(await rejection(host.get(Page, { id: 1 }, context))).toEqual({ + const quick = { readyTimeout: 5 }; + expect(await rejection(host.get(Page, { id: 1 }, context, quick))).toEqual({ code: "unavailable", message: "app.page is not served here", }); const InfoV2 = local("app.info", { version: 2, params: {}, members: { title: str() } }); const Changed = local("app.info", { version: 1, params: {}, members: { title: int() } }); - expect(await rejection(host.get(InfoV2, {}, context))).toMatchObject({ code: "contract_mismatch" }); - expect(await rejection(host.get(Changed, {}, context))).toMatchObject({ code: "contract_mismatch" }); + expect(await rejection(host.get(InfoV2, {}, context, quick))).toMatchObject({ code: "contract_mismatch" }); + expect(await rejection(host.get(Changed, {}, context, quick))).toMatchObject({ code: "contract_mismatch" }); const Same = local("app.info", { version: 1, params: {}, members: { title: str() } }); expect((await host.get(Same, {}, context)).title.value).toBe("t"); }); @@ -612,7 +891,11 @@ forEachCodec(() => { it("a local type never crosses a link; a remote type without a parent link waits for one", async () => { const { client } = await linked(() => {}); - expect(await rejection(client.get(Inner, { id: 1 }, context))).toEqual({ + expect(client.hold(Inner, { id: 1 }, context).state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting" }, + }); + expect(await rejection(client.get(Inner, { id: 1 }, context, { readyTimeout: 5 }))).toEqual({ code: "unavailable", message: "app.inner is not served here", }); @@ -645,6 +928,81 @@ forEachCodec(() => { }); }); +describe("host.registry and the host's close (fix round 1)", () => { + const Absent = defineRegistry("app.absent", { version: 1 }); + + it("ends a waiting host.registry at the close with released/closed, its deadline timer with it (Sol 5, Astra 5)", async () => { + const host = createHost(noErrors); + await host.load([], context); + let outcome: unknown; + void rejection(host.registry(Absent, context, { readyTimeout: 60_000 })).then((error) => { + outcome = error; + }); + await flushed(); + await host.close(context); + await flushed(); + expect(outcome).toMatchObject({ code: "released", reason: "closed" }); + }); + + it("ends one still waiting for the publication at the close", async () => { + const host = createHost(noErrors); + const waiting = rejection(host.registry(Absent, context, { readyTimeout: 60_000 })); + await host.close(context); + expect(await waiting).toMatchObject({ code: "released", reason: "closed" }); + }); +}); + +describe("the selected provider's own checks for a request made here (fix round 1)", () => { + const Here = hostKind("app-here"); + const Elsewhere = hostKind("app-elsewhere"); + const shape = { version: 1, params: {}, variants: { view: { state: { n: int() } } } } as const; + + it("waits a same-host remote request the provider's code places elsewhere: invalid/host_address (Astra 4)", async () => { + const Consumer = remote("app.placed", { ...shape, hostAddress: () => Here() }); + const Provider = remote("app.placed", { ...shape, hostAddress: () => Elsewhere() }); + const host = createHost({ address: Here(), principal: { kind: "owner" }, onError: () => {} }); + const facet = defineFacet((h) => h.serve(Provider, () => ({ variant: "view", members: { n: constant(1) } }))); + await host.load([{ name: "@app/provider", facet }], context); + const ref = host.hold(Consumer, {}, context); + await flushed(); + expect(ref.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "invalid", reason: "host_address" } }, + }); + ref.release(); + await host.close(context); + }); + + it("fails broken when the provider's hostAddress throws: its own code (Sol 7)", async () => { + const Consumer = remote("app.remote", { ...shape, hostAddress: () => Server() }); + const Provider = remote("app.remote", { + ...shape, + hostAddress: () => { + throw new Error("provider address bug"); + }, + }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + const client = createHost({ onError: () => {} }); + const facet = defineFacet((h) => h.serve(Provider, () => ({ variant: "view", members: { n: constant(1) } }))); + await server.load([{ name: "@app/provider", facet }], context); + await client.load([], context); + const [down, up] = memoryLinkPair(); + server.accept(down, { principal: { kind: "server" } }, context); + client.connect(up, { principal: { kind: "server" } }, context); + const held = client.hold(Consumer, {}, context); + await transported(); + expect(held.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "broken", error: { message: "app.remote: provider address bug" } }, + }); + // The same throw for a request made on the provider's host fails it broken too. + const here = server.hold(Consumer, {}, context); + expect(here.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + await client.close(context); + await server.close(context); + }); +}); + describe("a load whose setup throws", () => { it("closes the host as close does, links included, and rejects with the setup's error", async () => { const host = createHost({ principal: { kind: "server" }, onError: () => {} }); @@ -687,23 +1045,29 @@ describe("acquiring: hold and get", () => { const offline = { code: "unavailable", message: "app.svc(1): offline" }; - it("invalid params and a local type not served here: at once", async () => { + it("invalid params at once; a local type not served here: hold waits failed, get rejects at its readyTimeout", async () => { const host = createHost({ address: Server(), ...noErrors, principal: { kind: "server" } }); await host.load([{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context); const params = { code: "invalid", reason: "params" }; expect(() => host.hold(Notes, { session: 1 } as never, context)).toThrow("params"); expect(await rejection(host.get(Notes, { session: 1 } as never, context))).toMatchObject(params); const unserved = { code: "unavailable", message: "app.info is not served here" }; - expect(() => host.hold(Info, {}, context)).toThrow(unserved.message); - expect(await rejection(host.get(Info, {}, context))).toEqual(unserved); + expect(host.hold(Info, {}, context).state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: unserved }, + retryAt: undefined, + }); + const got = rejection(host.get(Info, {}, context, { readyTimeout: 5 })); + await vi.advanceTimersByTimeAsync(5); + expect(await got).toEqual(unserved); }); - it("before load resolved: hold is loading, get waits; once it resolved both bind, or an unserved type fails", async () => { + it("before load resolved: hold is loading, get waits; once it resolved both bind, or an unserved type waits", async () => { const host = createHost(noErrors); const held = host.hold(Info, {}, context); const unserved = host.hold(Page, { id: 1 }, context); const got = host.get(Info, {}, context); - const refused = rejection(host.get(Page, { id: 1 }, context)); + const refused = rejection(host.get(Page, { id: 1 }, context, { readyTimeout: 50 })); expect([held.state.value, unserved.state.value]).toEqual([{ kind: "loading" }, { kind: "loading" }]); await host.load( [{ name: "@test/app", facet: defineFacet((h) => h.serve(Info, () => ({ title: constant("t") }))) }], @@ -715,9 +1079,10 @@ describe("acquiring: hold and get", () => { const error = { code: "unavailable", message: "app.page is not served here" }; expect(unserved.state.value).toMatchObject({ kind: "failed", - cause: { kind: "broken", error }, + cause: { kind: "waiting", error }, retryAt: undefined, }); + await vi.advanceTimersByTimeAsync(50); expect(await refused).toEqual(error); }); @@ -866,25 +1231,25 @@ forEachCodec(() => { expect((await child.next("bound")).holds[0]).toMatchObject({ hold: 0 }); }); - it("refuses a hold served here whose params place it elsewhere, whatever address it names: invalid/host_address (race 5)", async () => { + it("keeps a hold naming this host whose params the code here places elsewhere waiting: invalid/host_address (code skew, M§1)", async () => { const { child } = await worker(); holdOf(child, "S2", "S1"); expect(await child.next("failed")).toMatchObject({ hold: 0, - cause: { kind: "broken", error: { kind: "chord", error: misplaced } }, - }); - // A refused hold is kept (a retry starts it again): the next hold takes the next id. - holdOf(child, "S2", "S2", PlacedNotes.name, 1); - expect(await child.next("failed")).toMatchObject({ - hold: 1, - cause: { kind: "broken", error: { kind: "chord", error: misplaced } }, + cause: { kind: "waiting", error: { kind: "chord", error: misplaced } }, }); + // A refused hold is kept (a publication or a retry starts it again): the next hold takes the next id. + holdOf(child, "S2", "S1", PlacedNotes.name, 1); + expect(await child.next("failed")).toMatchObject({ hold: 1, cause: { kind: "waiting" } }); }); - it("places a hold served here by its params, not by the address it names (that only routes it)", async () => { - const { child } = await worker(); - holdOf(child, "S1", "S2"); - expect((await child.next("bound")).holds[0]).toMatchObject({ hold: 0 }); + it("places a hold by the address it names, never by the name it serves (a relay never captures it)", async () => { + const { child, parent } = await worker({ parent: true }); + holdOf(child, "S2", "S2"); + const relayed = await parent!.next("hold"); + expect(relayed).toMatchObject({ type: PlacedNotes.name, address: { kind: "session", params: { id: "S2" } } }); + holdOf(child, "S1", "S2", PlacedNotes.name, 1); + expect(await parent!.next("hold")).toMatchObject({ hold: 1 }); }); it("sends a reference made here whose host address is another host over the parent link, though it serves the type", async () => { diff --git a/packages/chord/test/next/host/keyed-members.test.ts b/packages/chord/test/next/host/keyed-members.test.ts index d2570dd56..1ae93211c 100644 --- a/packages/chord/test/next/host/keyed-members.test.ts +++ b/packages/chord/test/next/host/keyed-members.test.ts @@ -1,18 +1,19 @@ // Tests of keyed members across attempts (slice 6a §1.2, §1.4, §12): the keys references hold on an object's keyed // members, frozen while it is failed, re-bound by name (loaded first) when its next attempt goes live, released once. -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { BACKGROUND_CONTEXT as context, withAbortSignal } from "../../../src/context/index.ts"; import { int, keyed, local, obj, str } from "../../../src/next/contracts/index.ts"; import { ChordError } from "../../../src/next/data/index.ts"; import { defineState, type KeyedHandle } from "../../../src/next/documents/index.ts"; -import type { Self } from "../../../src/next/host/index.ts"; +import { createHost, defineFacet, type Self } from "../../../src/next/host/index.ts"; import { ChordObject } from "../../../src/next/host/object.ts"; import { changesOf, delivered, expectCollected, failLive, + flushed, instancesOf, loaded, memoryStore, @@ -291,7 +292,8 @@ describe("a key whose first pin still loads", () => { await delivered(); expect([viewer.value.value, viewer.double.value]).toEqual([{ n: 2 }, 4]); expect([pinsOf(providers.a!), pinsOf(providers.b!)]).toEqual([undefined, 1]); - expect(errors.filter((error) => error.kind === "activation_failed").length).toBe(1); + // The pin's failed load parked the viewer (a dependency failure is not its own): nothing failed it. + expect(errors.filter((error) => error.kind === "activation_failed").length).toBe(0); const values = changesOf(await viewer.value.watch(context)); await providers.a!.change( "k", @@ -312,10 +314,11 @@ describe("a key whose first pin still loads", () => { await close(); }); - it("the object fails and stays failed: the consumer's activation fails transient, never live on it", async () => { - const { host, errors, exposer, providers, hold, gate, entered, fail, close } = await cold(() => "a"); + it("the object fails and stays failed: the consumer parks failed { waiting } on it, never live on it", async () => { + const { host, exposer, providers, hold, gate, entered, fail, close } = await cold(() => "a"); hold(); const abort = new AbortController(); + const viewer = host.hold(Viewer, { id: 1 }, context); const loading = host.get(Viewer, { id: 1 }, withAbortSignal(abort.signal, context)); await entered.promise; fail(); @@ -323,8 +326,8 @@ describe("a key whose first pin still loads", () => { await delivered(); await new Promise((resolve) => setTimeout(resolve, 10)); expect(exposer.state.value.kind).toBe("failed"); - const failure = errors.find((error) => error.kind === "activation_failed"); - expect(failure).toMatchObject({ failure: { kind: "transient" } }); + expect(viewer.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + viewer.release(); abort.abort(); await expect(loading).rejects.toThrow(); expect(pinsOf(providers.a!)).toBeUndefined(); @@ -450,3 +453,54 @@ describe("chains and releases", () => { await expectCollected(...refs); }); }); + +describe("a key whose first pin still loads at startup (scenario 20, Astra)", () => { + it("keeps the first pass pending during an otherwise-ready dependency's delayed keyed pin", async () => { + const Data = defineState("app.pin-data", { + schema: Schema, + version: 1, + keyed: true, + initial: () => ({ n: 4 }), + }); + const Store = local("app.pin-store", { version: 1, params: {}, members: { notes: keyed(Schema) } }); + const Reader = local("app.pin-reader", { version: 1, params: {}, members: { value: Schema } }); + const gate = Promise.withResolvers(); + const original = ChordObject.prototype.pinKey; + const spy = vi.spyOn(ChordObject.prototype, "pinKey").mockImplementation(async function ( + this: ChordObject, + member, + key, + ctx, + ) { + await gate.promise; + return original.call(this, member, key, ctx); + }); + const host = createHost({ onError: () => {} }); + let done = false; + const loading = host + .load( + [ + { + name: "@test/pin", + facet: defineFacet((h) => { + h.serve(Store, (_, self) => ({ notes: self.state(Data) })); + h.serve(Reader, (_, self) => ({ value: self.get(Store, {}).notes.at("k") })); + }), + }, + ], + context, + ) + .then(() => { + done = true; + }); + await flushed(); + const premature = done; + gate.resolve(); + await loading; + spy.mockRestore(); + const ref = await host.get(Reader, {}, context); + expect(ref.value.value).toEqual({ n: 4 }); + await host.close(context); + expect(premature).toBe(false); + }); +}); diff --git a/packages/chord/test/next/host/object.test.ts b/packages/chord/test/next/host/object.test.ts index ab1675fb2..a6959394a 100644 --- a/packages/chord/test/next/host/object.test.ts +++ b/packages/chord/test/next/host/object.test.ts @@ -30,6 +30,7 @@ import { Server, Session, StoreRestartable, + statesOf, storeRestartable, testCodec, transported, @@ -240,7 +241,7 @@ describe("retries", () => { }); }); -describe("dependency recovery (R18)", () => { +describe("parked on dependencies (dependency model §3)", () => { fakeTimers(); const View = local("app.view", { version: 1, params: { id: int() }, members: { n: int() } }); @@ -254,7 +255,7 @@ describe("dependency recovery (R18)", () => { return { times, factory }; } - it("fails a dependent transient with the dependency's cause and retries it at once when it goes live", async () => { + it("parks a dependent of a failed dependency failed { waiting } naming it, with no timer; it goes on when the dependency is live", async () => { const core = flaky((attempt) => attempt === 1); const viewer = view(Core); const { host, errors } = await loaded((h) => { @@ -262,26 +263,28 @@ describe("dependency recovery (R18)", () => { h.serve(View, viewer.factory); }); await vi.advanceTimersByTimeAsync(500); + const held = host.hold(View, { id: 1 }, context); const got = host.get(View, { id: 1 }, context); await flushed(); - const failed = errors.find((error) => error.kind === "activation_failed" && error.object === "app.view(1)"); - expect(failed).toMatchObject({ - failure: { - kind: "transient", - error: { code: "unavailable", message: "app.view(1): dependency app.core failed" }, + expect(held.state.value).toEqual({ + kind: "failed", + cause: { + kind: "waiting", + error: expect.objectContaining({ message: "waiting for app.core: app.core: offline" }), }, + retryAt: undefined, }); - expect(failed?.kind === "activation_failed" && failed.failure.error.cause).toMatchObject({ - message: "app.core: offline", - }); - // The core's retry at 1 s succeeds; the view is retried then, not at the end of its own window (1.5 s). + // A parked attempt is no failure of its own: nothing is reported for the view. + expect(failures(errors)).toEqual(["app.core: transient"]); + // The core's own backoff retries it at 1 s; the view goes on then, its factory not run again. await vi.advanceTimersByTimeAsync(500); expect((await got).n.value).toBe(2); - await vi.advanceTimersByTimeAsync(5000); - expect(viewer.times).toEqual([500, 1000]); + await vi.advanceTimersByTimeAsync(120_000); + expect(viewer.times).toEqual([500]); + held.release(); }); - it("keeps the dependency while failed; releasing the dependent releases it", async () => { + it("keeps its references while parked; a getter rejects with the waiting cause at its deadline; releasing ends the wait", async () => { const svc = flaky(() => true); const viewer = view(Svc); const { host } = await loaded((h) => { @@ -290,81 +293,77 @@ describe("dependency recovery (R18)", () => { }); const got = rejection(host.get(View, { id: 1 }, context, { readyTimeout: 1500 })); await vi.advanceTimersByTimeAsync(1500); - expect(await got).toEqual({ code: "unavailable", message: "app.view(1): dependency app.svc(1) failed" }); + expect(await got).toEqual({ + code: "unavailable", + message: "waiting for app.svc(1): app.svc(1): offline", + }); await vi.advanceTimersByTimeAsync(10_000); - // Held by the failed view, the dependency was retried at 1 s; released with it at 1.5 s, it is not any more. + // Held by the parked view, the dependency was retried at 1 s; released with it at 1.5 s, it is not any more. expect(svc.times).toEqual([0, 1000]); + expect(viewer.times).toEqual([0]); }); - it("binds a retry's references before releasing the previous attempt's", async () => { - const svc = flaky((attempt) => attempt === 1); - const viewer = view(Svc); + it("shows every change of a dependency: loading, failed, a new cause, loading again, live", async () => { + let cores = 0; + const viewer = view(Core); const { host } = await loaded((h) => { - h.serve(Svc, svc.factory); + h.serve(Core, (_, self) => { + const attempt = ++cores; + self.open( + async () => { + if (attempt < 3) throw new Error(`offline ${attempt}`); + }, + () => {}, + ); + return { n: constant(attempt) }; + }); h.serve(View, viewer.factory); }); - const got = host.get(View, { id: 1 }, context); - await vi.advanceTimersByTimeAsync(1000); - // Held only by the view: released first, the live dependency would have ended and started anew. - expect((await got).n.value).toBe(2); - expect([svc.times, viewer.times]).toEqual([ - [0, 1000], - [0, 1000], - ]); + const held = host.hold(View, { id: 1 }, context); + await flushed(); + expect(held.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.core: app.core: offline 1" } }, + }); + const states = statesOf(await held.state.watch(context)); + await vi.advanceTimersByTimeAsync(10_000); + // The core's retries at 1 s and 3 s: each attempt shows as loading, each failure as a new cause. + expect(states).toEqual(["loading", "failed: waiting for app.core: app.core: offline 2", "loading", "live"]); + expect(viewer.times).toEqual([0]); + held.release(); }); - it("remembers a dependency going live while the dependent activates: the dependent is retried at once (audit)", async () => { - const viewer = view(Core); + it("passes a retry on to its broken dependency, once through a diamond", async () => { + const Top = local("app.top", { version: 1, params: {}, members: { n: int() } }); + const Left = local("app.left", { version: 1, params: { id: int() }, members: { n: int() } }); let cores = 0; const { host } = await loaded((h) => { - // Its first factory throws (broken); the second goes live at once, without natives. h.serve(Core, () => { - if (++cores === 1) throw new Error("first factory"); - return { n: constant(2) }; + if (++cores === 1) throw new Error("bug"); + return { n: constant(cores) }; }); - h.serve(View, viewer.factory); - }); - await flushed(); - // The view activates on the core's failure while the core is retried and goes live at once. - const dependency = host.hold(Core, {}, context); - const dependent = host.hold(View, { id: 1 }, context); - dependency.retry(); - await flushed(); - expect(dependency.state.value).toEqual({ kind: "live" }); - expect(dependent.state.value).toEqual({ kind: "live" }); - expect(viewer.times).toEqual([0, 0]); - dependent.release(); - dependency.release(); - }); - - it("ignores a trigger while an attempt runs: simultaneous triggers give one attempt", async () => { - const gate = Promise.withResolvers(); - const svc = flaky((attempt) => attempt === 1, gate.promise); - const viewer = view(Svc); - const { host } = await loaded((h) => { - h.serve(Svc, svc.factory); - h.serve(View, viewer.factory); - }); - const got = host.get(View, { id: 1 }, context); - // At 1 s both windows end: the dependency's retry waits at its gate, the view's waits for it. - await vi.advanceTimersByTimeAsync(1000); - gate.resolve(); - expect((await got).n.value).toBe(2); - expect(viewer.times).toEqual([0, 1000]); - }); - - it("retries a dependent of a broken dependency with its backoff, for as long as it is held", async () => { - const viewer = view(Core); - const { host } = await loaded((h) => { - h.serve(Core, () => { - throw new Error("bug"); + h.serve(Left, (_, self) => ({ n: self.get(Core, {}).n })); + h.serve(Top, (_, self) => { + const a = self.get(Left, { id: 1 }).n; + const b = self.get(Left, { id: 2 }).n; + return { n: self.derive([a, b], ([x, y]) => x + y) }; }); - h.serve(View, viewer.factory); }); - const got = rejection(host.get(View, { id: 1 }, context, { readyTimeout: 16_000 })); - await vi.advanceTimersByTimeAsync(16_000); - expect(await got).toEqual({ code: "unavailable", message: "app.view(1): dependency app.core failed" }); - expect(viewer.times).toEqual([0, 1000, 3000, 7000, 15000]); + const top = host.hold(Top, {}, context); + await flushed(); + expect(top.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.left(1): app.core: bug" } }, + }); + await vi.advanceTimersByTimeAsync(60_000); + // A broken dependency is not retried by any timer. + expect(cores).toBe(1); + top.retry(); + await flushed(); + expect(cores).toBe(2); + expect(top.state.value).toEqual({ kind: "live" }); + expect(top.n.value).toBe(4); + top.release(); }); it("fails every object of a cycle broken; none is retried", async () => { @@ -517,6 +516,39 @@ describe("GC of failed objects", () => { }); }); +describe("GC of a parked chain", () => { + it("collects a chain parked on a type not served, released from its top: objects, attempts and index entries", async () => { + const Top = local("app.top", { version: 1, params: { id: int() }, members: { n: int() } }); + const Mid = local("app.mid", { version: 1, params: { id: int() }, members: { n: int() } }); + const Missing = local("app.missing", { version: 1, params: {}, members: { n: int() } }); + const refs: WeakRef[] = []; + const { host } = await loaded((h) => { + h.serve(Mid, (_, self) => { + refs.push(new WeakRef(self)); + return { n: self.get(Missing, {}).n }; + }); + h.serve(Top, ({ id }, self) => { + refs.push(new WeakRef(self)); + return { n: self.get(Mid, { id }).n }; + }); + }); + const top = host.hold(Top, { id: 1 }, context); + await flushed(); + expect(top.state.value).toMatchObject({ + kind: "failed", + cause: { + kind: "waiting", + error: { message: "waiting for app.mid(1): app.missing is not served here" }, + }, + }); + top.release(); + await flushed(); + expect(refs).toHaveLength(2); + await expectCollected(...refs); + expect(host).toBeDefined(); + }); +}); + describe("ref.retry()", () => { fakeTimers(); diff --git a/packages/chord/test/next/host/reference.test.ts b/packages/chord/test/next/host/reference.test.ts index 2f2375b1c..04abbfe61 100644 --- a/packages/chord/test/next/host/reference.test.ts +++ b/packages/chord/test/next/host/reference.test.ts @@ -613,7 +613,7 @@ describe("keyed state members (slice 6a §12)", () => { await store.close(context); }); - it("a pin load that fails its provider fails the consumer as a dependency failure (transient)", async () => { + it("a pin load that fails its provider parks the consumer failed { waiting } on it (a dependency failure)", async () => { const store = await seeded(); const { host, errors } = await persistedNotes(store, persistedMemo(2)); const provider = await host.get(NoteStore, {}, context); @@ -621,7 +621,7 @@ describe("keyed state members (slice 6a §12)", () => { await vi.waitFor(() => expect(view.state.value.kind).toBe("failed")); expect(view.state.value).toMatchObject({ kind: "failed", - cause: { kind: "transient", error: { message: expect.stringContaining("cannot migrate") } }, + cause: { kind: "waiting", error: { message: expect.stringContaining("cannot migrate") } }, }); expect(provider.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); expect(errors.map(({ kind }) => kind)).toContain("object_failed"); diff --git a/packages/chord/test/next/host/retry-on.test.ts b/packages/chord/test/next/host/retry-on.test.ts index 5677b6c28..31710c226 100644 --- a/packages/chord/test/next/host/retry-on.test.ts +++ b/packages/chord/test/next/host/retry-on.test.ts @@ -76,6 +76,27 @@ describe("retryOn", () => { await host.close(context); }); + it("does nothing while the object is parked failed { waiting }: its dependencies decide (dependency model §3.8)", async () => { + const Missing = local("app.missing", { version: 1, params: {}, members: {} }); + let factories = 0; + const { host } = await loaded((h) => { + serveModels(h); + h.serve(User, (_, self) => { + factories++; + self.retryOn(self.registry(Models)); + self.get(Missing, {}); + return {}; + }); + }); + const user = host.hold(User, { id: 1 }, context); + await flushed(); + expect(user.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await (await host.get(Runtime, {}, context)).set(1, context); + await flushed(); + expect([user.state.value.kind, factories]).toEqual(["failed", 1]); + await host.close(context); + }); + it("wakes on value calls only: a throwing compose wakes nothing, its next good composition does", async () => { const opens = { n: 0 }; const { host } = await loaded((h) => { diff --git a/packages/chord/test/next/host/revocation.test.ts b/packages/chord/test/next/host/revocation.test.ts index ed5bab268..22bc7a9de 100644 --- a/packages/chord/test/next/host/revocation.test.ts +++ b/packages/chord/test/next/host/revocation.test.ts @@ -1,12 +1,12 @@ // Tests of revocation (§4, R10, R11): revocation keys, \`host.revoke\` on one host (objects, running calls, activations, -// holds still waiting for \`load\`), child links and \`revoke\` messages, GC, and revocation across a relay. +// holds bound or not), child links and \`revoke\` messages, GC, and revocation across a relay. import { describe, expect, it } from "vitest"; import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; -import { int, local, remote } from "../../../src/next/contracts/index.ts"; +import { hostKind, int, local, remote } from "../../../src/next/contracts/index.ts"; import { createHost, defineFacet, type FacetHost, type Host } from "../../../src/next/host/index.ts"; import { ChordObject } from "../../../src/next/host/object.ts"; -import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; import { jsonPart } from "../../../src/next/link/internal.ts"; import { constant } from "../../../src/next/state/index.ts"; import { @@ -18,6 +18,7 @@ import { RawPeer, rawParent, rawWorkers, + rejection, relayed, SERVER_ADDRESS, Server, @@ -312,6 +313,22 @@ forEachCodec(() => { await host.close(context); }); + it("ends an unbound hold of the principal (refused, waiting for a provider): gone, nothing left of it", async () => { + const { host, parent } = await worker(); + const Absent = { ...Notes, name: "app.absent" } as unknown as typeof Notes; + holdAs(parent, 0, S1, Absent); + expect(await parent.next("failed")).toMatchObject({ hold: 0, cause: { kind: "waiting" } }); + expect(host.activity.value.references).toBe(1); + parent.send({ kind: "revoke", revocation: { session: "S1" } }); + expect(await parent.next("gone")).toMatchObject({ + hold: 0, + error: { kind: "chord", error: { code: "forbidden", reason: "revoked" } }, + }); + await transported(); + expect(host.activity.value.references).toBe(0); + await host.close(context); + }); + it("binds a hold made for a revoked principal, then the parent's revoke after it ends it (race 3, R10's window)", async () => { const { host, parent } = await worker(); holdAs(parent, 0, S1); @@ -443,3 +460,120 @@ describe("revocation across a relay", () => { expect(again.state.value.kind).toBe("live"); }); }); + +describe("revocation of this host's own requests over its paths (fix round 1)", () => { + type Guest = { kind: "owner" } | { kind: "guest"; grant: string }; + const There = hostKind("app-there"); + const Remote = remote("app.remote", { + version: 1, + params: {}, + hostAddress: () => There(), + variants: { view: { state: {}, methods: {} } }, + }); + const guestRevokedBy = (p: Guest) => (p.kind === "guest" ? [p.grant] : []); + + it("ends a guest's ctx.get waiting for its route before any link exists, at once (Astra 1, Sol 1)", async () => { + const host = createHost({ + principal: { kind: "owner" }, + revokedBy: guestRevokedBy, + onError: () => {}, + }); + host.route(There, { authorize: () => true, resolve: () => new Promise(() => {}) }, context); + await host.load([], context); + let outcome: unknown; + void rejection( + host.root((ctx) => ctx.get(Remote, {}, { readyTimeout: 60_000 }), context, { + principal: { kind: "guest", grant: "g" }, + }), + ).then((error) => { + outcome = error; + }); + await transported(); + host.revoke("g", context); + await transported(); + expect(outcome).toMatchObject({ code: "forbidden", reason: "revoked" }); + await host.close(context); + }); + + it("never lets the revoked request bind on the route's later link (Astra 1)", async () => { + const host = createHost({ + principal: { kind: "owner" }, + revokedBy: guestRevokedBy, + onError: () => {}, + }); + const gate = Promise.withResolvers<{ link: Link; principal: Guest; trusted: boolean }>(); + host.route(There, { authorize: () => true, resolve: () => gate.promise }, context); + await host.load([], context); + const pending = rejection( + host.root( + async (ctx) => { + await ctx.get(Remote, {}, { readyTimeout: 500 }); + throw new Error("revoked request became live"); + }, + context, + { principal: { kind: "guest", grant: "g" } }, + ), + ); + await transported(); + host.revoke("g", context); + const worker = createHost({ + address: There(), + principal: { kind: "owner" }, + revokedBy: guestRevokedBy, + onError: () => {}, + }); + const [up, down] = memoryLinkPair(); + worker.connect(up, { principal: { kind: "owner" }, trusted: true }, context); + const facet = defineFacet((h) => h.serve(Remote, () => ({ variant: "view", members: {} }))); + await worker.load([{ name: "@test/late", facet }], context); + gate.resolve({ link: down, principal: { kind: "owner" }, trusted: true }); + expect(await pending).toMatchObject({ code: "forbidden", reason: "revoked" }); + await transported(); + expect(worker.activity.value.references).toBe(0); + await host.close(context); + await worker.close(context); + }); + + it("ends a guest's reference over a route link that dropped, and nothing holds it on the next one", async () => { + const routed: RawPeer[] = []; + const host = createHost({ + principal: { kind: "owner" }, + revokedBy: guestRevokedBy, + onError: () => {}, + }); + host.route( + There, + { + authorize: () => true, + resolve: async () => { + const [a, b] = memoryLinkPair(); + routed.push(new RawPeer(b)); + return { link: a, principal: { kind: "owner" }, codec: testCodec() }; + }, + }, + context, + ); + await host.load([], context); + const states: string[] = []; + void host.root( + async (ctx) => { + const held = await ctx.get(Remote, {}, { readyTimeout: 60_000 }).catch((error: Error) => error); + states.push(held instanceof Error ? held.message : "live"); + }, + context, + { principal: { kind: "guest", grant: "g" } }, + ); + await transported(); + const first = routed[0]!; + await first.next("hold"); + first.close(); + await transported(); + host.revoke("g", context); + await transported(); + expect(states).toEqual(["app.remote: revoked"]); + expect(routed.flatMap((peer, i) => (i === 0 ? [] : peer.received.filter(({ kind }) => kind === "hold")))).toEqual( + [], + ); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/routines.test.ts b/packages/chord/test/next/host/routines.test.ts index 17dfe5eb1..7dfa40565 100644 --- a/packages/chord/test/next/host/routines.test.ts +++ b/packages/chord/test/next/host/routines.test.ts @@ -727,8 +727,9 @@ describe("on", () => { }, ctx); }); }); + // 0 → 1 → 2 → 3, then the write of 3 changes nothing and nothing calls again (waited for, not timed: Opus F3). + await vi.waitFor(() => expect(calls).toBe(4), { timeout: 2_000 }); await new Promise((resolve) => setTimeout(resolve, 20)); - // 0 → 1 → 2 → 3, then the write of 3 changes nothing and nothing calls again. expect(calls).toBe(4); }); diff --git a/packages/chord/test/next/host/unbound.test.ts b/packages/chord/test/next/host/unbound.test.ts new file mode 100644 index 000000000..8da3929e4 --- /dev/null +++ b/packages/chord/test/next/host/unbound.test.ts @@ -0,0 +1,171 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, str } from "../../../src/next/contracts/index.ts"; +import { ChordError, type Principal } from "../../../src/next/data/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { Binding } from "../../../src/next/host/reference.ts"; +import { UnboundIndex, type UnboundRequest } from "../../../src/next/host/unbound.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { jsonPart } from "../../../src/next/link/internal.ts"; +import { Hold } from "../../../src/next/remote/hold.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { + delivered, + expectCollected, + instancesOf, + RawPeer, + rejection, + SERVER_ADDRESS, + Server, + type TestPrincipal, + testCodec, + transported, +} from "../support.ts"; + +const bindings = instancesOf(Binding, "release"); +const holds = instancesOf(Hold, "release"); + +const Info = local("app.info", { version: 1, params: {}, members: { title: str() } }); +const Missing = local("app.missing", { version: 1, params: { id: int() }, members: { n: int() } }); + +/** A request recording its wakes and ends. */ +function request(principal?: Principal): UnboundRequest & { readonly log: string[] } { + const log: string[] = []; + return { principal, log, wake: () => log.push("wake"), end: (error) => log.push(`end ${error.message}`) }; +} + +describe("the index", () => { + it("wakes every request at the publication, out of the index first; a deleted one is not woken", () => { + const index = new UnboundIndex(); + const [a, b, c] = [request(), request(), request()]; + index.add("app.a", a); + index.add("app.a", b); + index.add("app.b", c); + index.delete("app.a", b); + index.wakeAll(); + expect([a.log, b.log, c.log]).toEqual([["wake"], [], ["wake"]]); + // Out of it: a second publication wakes nobody. + index.wakeAll(); + expect(a.log).toEqual(["wake"]); + }); + + it("ends the requests of a revoked principal only, and every request at the host's close", () => { + const index = new UnboundIndex(); + const guest = request({ kind: "guest" }); + const owner = request({ kind: "owner" }); + const own = request(); + for (const each of [guest, owner, own]) index.add("app.a", each); + const revoked = new ChordError({ code: "forbidden", reason: "revoked", message: "revoked" }); + index.revoke((principal) => principal.kind === "guest", revoked); + expect([guest.log, owner.log, own.log]).toEqual([["end revoked"], [], []]); + index.close(new ChordError({ code: "released", reason: "closed", message: "closed" })); + expect([owner.log, own.log]).toEqual([["end closed"], ["end closed"]]); + }); +}); + +describe("references waiting in the index", () => { + it("before the publication: loading, placed by it; a type not served then waits failed { waiting }", async () => { + const host = createHost({ onError: () => {} }); + const info = host.hold(Info, {}, context); + const missing = host.hold(Missing, { id: 1 }, context); + expect([info.state.value, missing.state.value]).toEqual([{ kind: "loading" }, { kind: "loading" }]); + await host.load( + [{ name: "@test/a", facet: defineFacet((h) => h.serve(Info, () => ({ title: constant("t") }))) }], + context, + ); + await delivered(); + expect(info.state.value).toEqual({ kind: "live" }); + expect(missing.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "unavailable", message: "app.missing is not served here" } }, + retryAt: undefined, + }); + // A retry leaves it waiting: an event wakes it, nothing else. + missing.retry(); + expect(missing.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await host.close(context); + expect(missing.state.value).toMatchObject({ kind: "ended", error: { reason: "closed" } }); + }); + + it("leave nothing behind once released, before and after the publication (GC)", async () => { + const host = createHost({ onError: () => {} }); + host.hold(Missing, { id: 1 }, context).release(); + await host.load([], context); + host.hold(Missing, { id: 2 }, context).release(); + const got = host.get(Missing, { id: 3 }, context, { readyTimeout: 1 }).catch(() => {}); + await got; + await expectCollected(...bindings); + expect(host).toBeDefined(); + }); + it("end at the start of the host's close, not after its slow native closes (fix round 1, Sol 6)", async () => { + const Core = local("app.core", { version: 1, params: {}, members: { n: int() } }); + const closeGate = Promise.withResolvers(); + const host = createHost({ onError: () => {} }); + const facet = defineFacet((h) => + h.serve(Core, (_, self) => { + self.open( + async () => 1, + () => closeGate.promise, + ); + return { n: constant(1) }; + }), + ); + await host.load([{ name: "@test/a", facet }], context); + let outcome: unknown; + void rejection(host.get(Missing, { id: 1 }, context)).then((error) => { + outcome = error; + }); + const held = host.hold(Missing, { id: 2 }, context); + const closing = host.close(context); + await delivered(); + expect(outcome).toMatchObject({ code: "released", reason: "closed" }); + expect(held.state.value).toMatchObject({ kind: "ended", error: { reason: "closed" } }); + closeGate.resolve(); + await closing; + }); +}); + +describe("holds waiting in the index", () => { + /** A server not loaded yet, a raw consumer peer on a child link, and a hold of `app.missing` it sent. */ + async function early() { + const host = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + const [a, b] = memoryLinkPair(); + host.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + peer.send({ + kind: "hold", + hold: 0, + type: "app.missing", + version: 1, + fingerprint: "0123456789abcdef", + params: jsonPart({}), + address: SERVER_ADDRESS, + }); + await transported(); + return { host, peer, link: a }; + } + + it("wait for the publication without a word, then are refused failed { waiting } and kept", async () => { + const { host, peer } = await early(); + expect(peer.kinds()).toEqual(["hello"]); + await host.load([], context); + expect(await peer.next("failed")).toMatchObject({ hold: 0, cause: { kind: "waiting" } }); + expect(host.activity.value.references).toBe(1); + await host.close(context); + }); + + it("leave nothing behind once their link closed, before and after the publication (GC)", async () => { + const before = await early(); + before.link.close(); + await transported(); + await before.host.load([], context); + const after = await early(); + await after.host.load([], context); + await after.peer.next("failed"); + after.link.close(); + await transported(); + expect([before.host.activity.value.references, after.host.activity.value.references]).toEqual([0, 0]); + await expectCollected(...holds); + expect([before.host, after.host]).toBeDefined(); + }); +}); diff --git a/packages/chord/test/next/link/message.test.ts b/packages/chord/test/next/link/message.test.ts index 968612a2a..ceeb6c599 100644 --- a/packages/chord/test/next/link/message.test.ts +++ b/packages/chord/test/next/link/message.test.ts @@ -100,6 +100,12 @@ const EVERY_KIND: Message[] = [ cause: { kind: "transient", error: { kind: "chord", error: { code: "unavailable", message: "m" } } }, retryIn: 0, }, + // Not the object's fault (dependency model §2): no backoff window, an event wakes it. + { + kind: "failed", + hold: 0, + cause: { kind: "waiting", error: { kind: "chord", error: { code: "contract_mismatch", message: "m" } } }, + }, // The wire error format in `failed` and `gone`: a plugin's error travels too. { kind: "failed", hold: 0, cause: { kind: "broken", error: { kind: "plugin", code: "config", message: "m" } } }, { kind: "gone", hold: 1, error: { kind: "plugin", code: "config", message: "m", details: { at: 1 } } }, @@ -110,6 +116,10 @@ const EVERY_KIND: Message[] = [ retryIn: 60_000, }, { kind: "retry", hold: 300 }, + // The retry operation it belongs to (fix round 1): an object reached twice handles it once. + { kind: "retry", hold: 300, operation: incarnation }, + // And the instance ids of the objects it passed through (fix round 2): a ring stops after one lap. + { kind: "retry", hold: 300, operation: incarnation, through: [incarnation, incarnation] }, { kind: "loading", hold: 300 }, { kind: "result", call: 3, value: '{"n":1}' }, { kind: "result", call: 3 }, @@ -333,6 +343,21 @@ describe("the binary codec", () => { expect([...binaryCodec.encode({ kind: "closed", origin: 1 })]).toEqual([14 | 0x80, 1]); }); + it("writes a failure's cause as one byte, 0 transient, 1 broken, 2 waiting (dependency model §2); refuses another", () => { + const error = { kind: "chord", error: { code: "unavailable", message: "m" } } as const; + const failed = (kind: "transient" | "broken" | "waiting"): Message => ({ + kind: "failed", + hold: 0, + cause: { kind, error }, + }); + expect( + ["transient", "broken", "waiting"].map((kind) => binaryCodec.encode(failed(kind as "waiting"))[2]), + ).toEqual([0, 1, 2]); + const unknown = binaryCodec.encode(failed("waiting")); + unknown[2] = 3; + expect(() => binaryCodec.decode(unknown)).toThrow("an unknown failure kind"); + }); + it("costs a state entry without retirements no byte: the failure count is doubled, plus one when they follow", () => { const state = (retired: number[]): Message => ({ kind: "state", @@ -413,8 +438,16 @@ describe("the binary codec", () => { expect(() => binaryCodec.decode(new Uint8Array([1, 0, 1, 0xff]))).toThrow(); }); - it("writes retry and loading as their kind and hold, and failed's retryIn as one more than it (0: absent)", () => { - expect([...binaryCodec.encode({ kind: "retry", hold: 300 })]).toEqual([11, 0xac, 0x02]); + it("writes retry as its kind, hold, flags, operation (16 raw bytes) and objects passed through, loading as its kind and hold, and failed's retryIn as one more than it (0: absent)", () => { + expect([...binaryCodec.encode({ kind: "retry", hold: 300 })]).toEqual([11, 0xac, 0x02, 0]); + expect(binaryCodec.encode({ kind: "retry", hold: 300, operation: incarnation })).toHaveLength(4 + 16); + const through = [incarnation, incarnation]; + expect(binaryCodec.encode({ kind: "retry", hold: 300, operation: incarnation, through })).toHaveLength( + 4 + 16 + 1 + 32, + ); + const flagged = binaryCodec.encode({ kind: "retry", hold: 300 }); + flagged[3] = 4; + expect(() => binaryCodec.decode(flagged)).toThrow("an unknown retry flag"); expect([...binaryCodec.encode({ kind: "loading", hold: 1 })]).toEqual([12, 1]); const cause = { kind: "broken", error: { kind: "chord", error: { code: "released", message: "" } } } as const; const absent = binaryCodec.encode({ kind: "failed", hold: 0, cause }); @@ -446,6 +479,10 @@ describe("the text codec", () => { expect(text({ kind: "failed", hold: 2, cause, retryIn: 750 })).toBe( '["f",2,"t",{"kind":"chord","error":{"code":"unavailable","message":"m"}},750]', ); + expect(text({ kind: "failed", hold: 2, cause: { ...cause, kind: "waiting" } })).toBe( + '["f",2,"w",{"kind":"chord","error":{"code":"unavailable","message":"m"}}]', + ); + expect(() => textCodec.decode(new TextEncoder().encode('["f",2,"x",{}]'))).toThrow("an unknown failure kind"); expect(text({ kind: "ack" })).toBe('["a"]'); // An origin wraps the message's own tuple. expect(text({ kind: "release", origin: 4, hold: 3 })).toBe('["@",4,["R",3]]'); diff --git a/packages/chord/test/next/registries/record.test.ts b/packages/chord/test/next/registries/record.test.ts index c25b27fcf..5066df8ea 100644 --- a/packages/chord/test/next/registries/record.test.ts +++ b/packages/chord/test/next/registries/record.test.ts @@ -320,7 +320,7 @@ describe("compose", () => { await host.close(context); }); - it("before any good composition, `.value` and `host.registry` throw the error", async () => { + it("before any good composition, `.value` throws the error and `host.registry` rejects with it at its readyTimeout", async () => { const Broken = defineRegistry("app.broken", { version: 1 }); const { host, errors } = await loadedPlugins({ "@app/broken": (host) => @@ -330,7 +330,7 @@ describe("compose", () => { }, }), }); - await expect(host.registry(Broken, context)).rejects.toThrow("bug"); + await expect(host.registry(Broken, context, { readyTimeout: 5 })).rejects.toThrow("bug"); expect(kinds(errors)).toEqual(["registry_failed"]); await host.close(context); }); @@ -623,6 +623,31 @@ describe("contributions", () => { }); }); +describe("the last good composition (decision 5)", () => { + it("is dropped when a contribution it was built from leaves: `.value` throws until `compose` succeeds", async () => { + const Count = defineRegistry("app.count", { version: 1 }); + const compose = (table: RegistryTable): number => { + if (table.length === 0) throw new Error("no contributor"); + return table[0]!.value; + }; + const { host, fail, retry } = await storeRestartable( + objects, + (attempt, self) => self.contribute(Count, attempt), + { + serve: (host) => host.serve(Count, { compose }), + }, + ); + const count = await host.registry(Count, context); + expect(count.value).toBe(1); + // Its contributor failed: the last good value was built from its contribution (a native may have closed with it). + fail(); + expect(() => count.value).toThrow("no contributor"); + await retry(); + expect(count.value).toBe(2); + await host.close(context); + }); +}); + describe("the table's memory", () => { it("keeps nothing of a closed host: records, tables and followed contributions go with it", async () => { const Doc = defineState("app.kept", { schema: obj({ on: bool() }), version: 1, initial: () => ({ on: true }) }); diff --git a/packages/chord/test/next/registries/value.test.ts b/packages/chord/test/next/registries/value.test.ts index e23722948..b2e6c6579 100644 --- a/packages/chord/test/next/registries/value.test.ts +++ b/packages/chord/test/next/registries/value.test.ts @@ -2,7 +2,8 @@ import { getEventListeners } from "node:events"; import { describe, expect, it } from "vitest"; import { BACKGROUND_CONTEXT as context, withAbortSignal, withCancel } from "../../../src/context/index.ts"; import { int, local, method, str } from "../../../src/next/contracts/index.ts"; -import type { defineFacet } from "../../../src/next/host/index.ts"; +import type { FollowObserver } from "../../../src/next/host/follow.ts"; +import { createHost, defineFacet, type Native } from "../../../src/next/host/index.ts"; import { ChordObject } from "../../../src/next/host/object.ts"; import { defineRegistry, type Registry } from "../../../src/next/registries/index.ts"; import type { NativeWatchCall, WatchEnd } from "../../../src/watch.ts"; @@ -83,7 +84,7 @@ describe("older readers (S2)", () => { await host.close(context); }); - it("without a value mapping, and for a reader newer than the provider, fail `contract_mismatch`: broken, reported", async () => { + it("without a value mapping, and for a reader newer than the provider, park readers failed { waiting } (contract_mismatch)", async () => { const Newer = local("app.newer", { version: 1, params: {}, members: { total: int() } }); const Older = local("app.older", { version: 1, params: {}, members: { total: int() } }); const { host, errors } = await loadedPlugins({ @@ -93,11 +94,103 @@ describe("older readers (S2)", () => { host.serve(Older, (_, self) => ({ total: self.derive([self.registry(SumV1)], ([v]) => v) })); }, }); - expect(await rejection(host.registry(SumV1, context))).toMatchObject({ code: "contract_mismatch" }); - expect(await rejection(host.registry(SumV3, context))).toMatchObject({ code: "contract_mismatch" }); - // Only a reload helps: no backoff, the mismatch itself the cause. - expect(failures(errors)).toEqual(["app.newer: broken", "app.older: broken"]); - expect(errors[0]).toMatchObject({ failure: { error: { code: "contract_mismatch" } } }); + const quick = { readyTimeout: 5 }; + expect(await rejection(host.registry(SumV1, context, quick))).toMatchObject({ code: "contract_mismatch" }); + expect(await rejection(host.registry(SumV3, context, quick))).toMatchObject({ code: "contract_mismatch" }); + // Not their fault: waiting (a reload serving the version wakes them), no backoff, nothing failed. + expect(failures(errors)).toEqual([]); + expect(host.hold(Newer, {}, context).state.value).toMatchObject({ + kind: "failed", + cause: { + kind: "waiting", + error: { + message: "waiting for registry app.sum: registry app.sum: version 3 is not served here (version 2 is)", + }, + }, + retryAt: undefined, + }); + await host.close(context); + }); + + it("drops an older reader's last good mapping after a contributor it was built from failed live (fix round 1, Astra 2)", async () => { + const R = defineRegistry("app.mapped", { version: 2 }); + const Old = defineRegistry("app.mapped", { version: 1 }); + const Contributor = local("app.contributor", { version: 1, params: {}, members: { n: int() } }); + let observer: FollowObserver | undefined; + const { host } = await loadedPlugins({ + "@app/mapping": (h) => { + h.serve(R, { + compose: (table) => table.reduce((n, e) => n + e.value, 0), + accepts: { + 1: { + value: (n) => { + if (n === 0) throw new Error("old version cannot map the empty table"); + return n; + }, + }, + }, + }); + h.serve(Contributor, (_, self) => { + self.contribute(R, 42); + return { + n: self.follow(async (followed) => { + observer = followed; + return { value: 42, detach() {} }; + }), + }; + }); + }, + }); + const old = await host.registry(Old, context); + expect(old.value).toBe(42); + observer!.end({ kind: "failed", error: new Error("runtime ended") }); + await flushed(); + expect(() => old.value).toThrow("old version cannot map the empty table"); + await host.close(context); + }); + + it("never hands out a mapped native whose contributor failed and closed it (fix round 1, Sol 2)", async () => { + type Resource = { closed: boolean }; + const Current = defineRegistry, readonly Native[]>("app.resources", { version: 2 }); + const Old = defineRegistry, Native>("app.resources", { version: 1 }); + const Owner = local("app.owner", { version: 1, params: {}, members: {} }); + const resource = { closed: false }; + let failRuntime: () => void = () => {}; + const { host } = await loadedPlugins({ + "@app/resources": (h) => { + h.serve(Current, { + compose: (table) => table.map(({ value }) => value), + accepts: { + 1: { + value: (current) => { + if (current.length === 0) throw new Error("no current resource"); + return current[0]!; + }, + }, + }, + }); + h.serve(Owner, (_, self) => { + const native = self.open( + async () => resource, + () => { + resource.closed = true; + }, + ); + self.contribute(Current, native); + self.follow(async (observer) => { + failRuntime = () => observer.end({ kind: "failed", error: new Error("runtime closed") }); + return { value: 1, detach: () => {} }; + }); + return {}; + }); + }, + }); + const value = await host.registry(Old, context); + expect(value.value.value).toBe(resource); + failRuntime(); + await flushed(); + expect(resource.closed).toBe(true); + expect(() => value.value).toThrow("no current resource"); await host.close(context); }); @@ -162,29 +255,30 @@ describe("readers (S3)", () => { R: Registry = Sum, ) => host.serve(Reader, (_, self) => ({ total: self.derive([self.registry(R)], ([v]) => v.total) })); - it("activate once the registry has a value; a reader of a registry nobody serves fails at once, `load` waits for nothing", async () => { + it("activate once the registry has a value; a reader of a registry nobody serves parks failed { waiting }, `load` waits for nothing", async () => { const started = Date.now(); const { host, errors } = await loadedPlugins({ "@app/reader": (host) => serveReader(host) }); expect(Date.now() - started).toBeLessThan(1_000); - expect(errors).toMatchObject([ - { - kind: "activation_failed", - object: "app.reader", - failure: { - kind: "transient", - error: { - code: "unavailable", - message: "app.reader: registry app.sum has no value", - cause: { code: "unavailable", reason: "undeclared" }, - }, + expect(errors).toEqual([]); + expect(host.hold(Reader, {}, context).state.value).toMatchObject({ + kind: "failed", + cause: { + kind: "waiting", + error: { + code: "unavailable", + message: "waiting for registry app.sum: registry app.sum: no plugin serves it here", + cause: { code: "unavailable", reason: "undeclared" }, }, }, - ]); - expect(await rejection(host.registry(Sum, context))).toMatchObject({ code: "unavailable", reason: "undeclared" }); + }); + expect(await rejection(host.registry(Sum, context, { readyTimeout: 5 }))).toMatchObject({ + code: "unavailable", + reason: "undeclared", + }); await host.close(context); }); - it("a reader failed on a first `compose` that threw is retried at once by the first good composition (R18)", async () => { + it("a reader parked on a first `compose` that threw goes on at the first good composition", async () => { const { host, errors } = await loadedPlugins({ "@app/sum": (host) => { host.serve(Sum, { @@ -198,15 +292,46 @@ describe("readers (S3)", () => { }, "@app/setter": (host) => setter(host, 0), }); - expect(failures(errors)).toEqual(["app.reader: transient"]); + expect(failures(errors)).toEqual([]); + const held = host.hold(Reader, {}, context); + expect(held.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); const setters = await host.get(Setter, {}, context); await setters.set(3, context); - // The reader's backoff is a second at least: the composition retried it. + // Woken by the composition: no timer of its own. const reader = await host.get(Reader, {}, context, { readyTimeout: 500 }); + held.release(); expect(reader.total.value).toBe(3); await host.close(context); }); + it("host.registry before the publication waits for it, then resolves with the provider it served", async () => { + const host = createHost({ onError: () => {} }); + const waiting = host.registry(Sum, context); + await host.load([{ name: "@app/sum", facet: defineFacet((h) => h.serve(Sum, { compose: total })) }], context); + expect((await waiting).value).toEqual({ total: 0 }); + await host.close(context); + }); + + it("host.registry waits through no value until a first good composition, woken by it, its wait then gone", async () => { + const { host } = await loadedPlugins({ + "@app/sum": (host) => { + host.serve(Sum, { + compose: (table) => { + const sum = total(table); + if (sum.total === 0) throw new Error("empty"); + return sum; + }, + }); + setter(host, 0); + }, + }); + const waiting = host.registry(Sum, context, { readyTimeout: 500 }); + await flushed(); + await (await host.get(Setter, {}, context)).set(5, context); + expect((await waiting).value).toEqual({ total: 5 }); + await host.close(context); + }); + it("once live, a later throwing `compose` changes nothing for the reader: `.value` keeps the last good one", async () => { const { host } = await loadedPlugins({ "@app/sum": (host) => { diff --git a/packages/chord/test/next/remote/hold.test.ts b/packages/chord/test/next/remote/hold.test.ts index 03d27f74c..ffbe77fcc 100644 --- a/packages/chord/test/next/remote/hold.test.ts +++ b/packages/chord/test/next/remote/hold.test.ts @@ -2,13 +2,13 @@ import { describe, expect, it, vi } from "vitest"; import { type Context, BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; import type * as Delta from "../../../src/delta/index.ts"; import { decoder } from "../../../src/delta/index.ts"; -import { bool, int, local, method, obj, remote } from "../../../src/next/contracts/index.ts"; +import { bool, hostKind, int, local, method, obj, remote } from "../../../src/next/contracts/index.ts"; import { broken } from "../../../src/next/data/index.ts"; import { defineState } from "../../../src/next/documents/index.ts"; import { createHost, defineFacet, type Host, type Self } from "../../../src/next/host/index.ts"; import type { ChordHost } from "../../../src/next/host/internal.ts"; import { type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; -import { jsonPart } from "../../../src/next/link/internal.ts"; +import { binaryCodec, jsonPart } from "../../../src/next/link/internal.ts"; import { Hold } from "../../../src/next/remote/hold.ts"; import { constant } from "../../../src/next/state/index.ts"; import { @@ -102,7 +102,9 @@ function providerObject(server: unknown) { const host = server as ChordHost; const { name: type, version, fingerprint } = Halves; const request = { type, version, fingerprint, params: "{}", address: SERVER_ADDRESS }; - return host.serving(request, {}, { kind: "owner" }, context); + const served = host.serving(request, {}, { kind: "owner" }, context); + if (served.kind !== "object") throw new Error("Halves is served here"); + return served.object; } /** A remote type whose activation never completes: its open waits for its abort. */ @@ -457,6 +459,65 @@ forEachCodec(() => { }); }); + describe("holds that bind to nothing yet (dependency model §1, §3)", () => { + const Absent = { name: "app.absent", version: 1, fingerprint: "0123456789abcdef" }; + + it("a refused hold waits failed { waiting }, counted; a retry re-runs its start (loading, failed); its link's close drops it", async () => { + const { server, peer } = await rawLinked(() => {}); + hold(peer, 0, Absent); + const failed = await peer.next("failed"); + expect(failed).toMatchObject({ hold: 0, cause: { kind: "waiting" } }); + expect(failed.retryIn).toBeUndefined(); + expect(server.activity.value.references).toBe(1); + expect(server.references(Halves).value).toEqual([]); + peer.send({ kind: "retry", hold: 0 }); + expect(await peer.next("loading")).toEqual({ kind: "loading", hold: 0 }); + expect(await peer.next("failed")).toMatchObject({ hold: 0, cause: { kind: "waiting" } }); + peer.close(); + await transported(); + expect(server.activity.value.references).toBe(0); + }); + + it("a hold on an object parked on a failed dependency is told each new cause, then loading, then bound", async () => { + const Core = local("app.core", { version: 1, params: {}, members: { n: int() } }); + let cores = 0; + const { peer } = await rawLinked((h) => { + h.serve(Core, () => { + if (++cores < 3) throw new Error(`bug ${cores}`); + return { n: constant(cores) }; + }); + h.serve(Halves, (_, _principal, self) => { + const core = self.get(Core, {}); + const n = self.state(N); + return { + variant: "v", + members: { + n, + half: self.derive([core.n], ([c]) => c), + set: async () => null, + wait: async () => null, + }, + }; + }); + }); + hold(peer, 0, Halves); + const waiting = (message: string) => ({ + kind: "failed", + hold: 0, + cause: { kind: "waiting", error: { kind: "chord", error: { code: "unavailable", message } } }, + }); + // The core failed at load (its first outcome): the parked object names it at once. + expect(await peer.next("failed")).toEqual(waiting("waiting for app.core: app.core: bug 1")); + // A retry passes on through the parked object to its broken dependency (one host's retry operation). + // (Its new attempt failed within the batch of changes the parked object checks once: no loading between.) + peer.send({ kind: "retry", hold: 0 }); + expect(await peer.next("failed")).toEqual(waiting("waiting for app.core: app.core: bug 2")); + peer.send({ kind: "retry", hold: 0 }); + await peer.next("loading"); + expect((await peer.next("bound")).holds[0]).toMatchObject({ hold: 0 }); + }); + }); + describe("a reconnecting consumer", () => { const Tally = remote("app.tally", { version: 1, @@ -715,3 +776,107 @@ forEachCodec(() => { }); }); }); + +/** Lets every pending reaction run (a memory link delivers by microtask). */ +async function flush(): Promise { + for (let i = 0; i < 100; i++) await Promise.resolve(); +} + +/** A local type nobody serves. */ +const Missing = local("app.missing", { version: 1, params: {}, members: { n: int() } }); + +describe("retry operations across links (fix rounds 1 and 2, Sol 3)", () => { + it.each([1, 9, 50])( + "quiesce after %i concurrent Retries round a waiting ring across hosts: each stops after one lap", + async (operations) => { + type P = { kind: "server" } | { kind: "worker" }; + const Server = hostKind("app.retry-server"); + const Worker = hostKind("app.retry-worker"); + const LocalA = local("app.a", { version: 1, params: {}, members: { n: int() } }); + const LocalB = local("app.b", { version: 1, params: {}, members: { n: int() } }); + const RemoteA = remote("app.remote-a", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { reader: { state: { n: int() } } }, + }); + const RemoteB = remote("app.remote-b", { + version: 1, + params: {}, + hostAddress: () => Worker(), + variants: { reader: { state: { n: int() } } }, + }); + let retries = 0; + const count = (link: Link): Link => ({ + send: (bytes) => { + if (binaryCodec.decode(bytes).kind === "retry") retries++; + link.send(bytes); + }, + onMessage: (listener) => link.onMessage(listener), + onClose: (listener) => link.onClose(listener), + close: () => link.close(), + }); + const [down, up] = memoryLinkPair(); + const server = createHost

    ({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + const worker = createHost

    ({ address: Worker(), principal: { kind: "worker" }, onError: () => {} }); + server.route( + Worker, + { + authorize: () => true, + resolve: async () => ({ link: count(down), principal: { kind: "worker" }, trusted: true }), + }, + context, + ); + worker.connect(count(up), { principal: { kind: "server" }, trusted: true }, context); + try { + const left = server.load( + [ + { + name: "@test/a", + facet: defineFacet

    ((h) => { + h.serve(LocalA, (_, self) => ({ n: self.get(RemoteB, {}).n })); + h.serve(RemoteA, (_, _principal, self) => ({ + variant: "reader", + members: { n: self.get(LocalA, {}).n }, + })); + }), + }, + ], + context, + ); + const right = worker.load( + [ + { + name: "@test/b", + facet: defineFacet

    ((h) => { + h.serve(LocalB, (_, self) => { + self.get(Missing, {}); + return { n: self.get(RemoteA, {}).n }; + }); + h.serve(RemoteB, (_, _principal, self) => ({ + variant: "reader", + members: { n: self.get(LocalB, {}).n }, + })); + }), + }, + ], + context, + ); + await Promise.all([left, right]); + await flush(); + const a = server.hold(LocalA, {}, context); + expect(a.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + // More at once than an object remembers ids: `through` stops each at the object it passed through already. + for (let i = 0; i < operations; i++) a.retry(); + await flush(); + // Each went round once: server → worker → server, then stopped at the object it started from. + expect(retries).toBe(2 * operations); + await flush(); + expect(retries).toBe(2 * operations); + } finally { + await server.close(context); + await worker.close(context); + } + }, + ); +}); diff --git a/packages/chord/test/next/remote/provider.test.ts b/packages/chord/test/next/remote/provider.test.ts index a93e54c62..dc9f30aed 100644 --- a/packages/chord/test/next/remote/provider.test.ts +++ b/packages/chord/test/next/remote/provider.test.ts @@ -117,14 +117,14 @@ forEachCodec(() => { { fingerprint: "0000000000000000" }, { code: "contract_mismatch", message: "app.notes v1 does not match the served v1" }, ], - ])("refuse %s with a broken failed", async (_, override, error) => { + ])("keep %s refused, failed { waiting }", async (_, override, error) => { const { peer } = await notesServer(); const { type, ...rest } = override as { type?: string }; hold(peer, 0, { ...Notes, name: type ?? Notes.name } as Contract, { session: "S1" }, rest); - expect(await peer.next("failed")).toMatchObject({ - hold: 0, - cause: { kind: "broken", error: { kind: "chord", error } }, - }); + const failed = await peer.next("failed"); + expect(failed).toMatchObject({ hold: 0, cause: { kind: "waiting", error: { kind: "chord", error } } }); + // No backoff window: an event (a publication) wakes it. + expect(failed.retryIn).toBeUndefined(); }); it("refuse invalid params with a broken failed", async () => { @@ -135,7 +135,7 @@ forEachCodec(() => { }); }); - it("arriving before load resolved wait for it", async () => { + it("arriving before the publication wait for it, and bind before the start barrier is over", async () => { const server = createHost({ address: Server(), principal: { kind: "server" }, @@ -505,12 +505,12 @@ forEachCodec(() => { }); describe("local types over a link", () => { - it("never cross one", async () => { + it("never cross one: a hold naming one is not served (failed { waiting })", async () => { const Store = local("app.store", { version: 1, params: {}, members: { n: int() } }); const { peer } = await rawLinked((h) => h.serve(Store, () => ({ n: constant(1) }))); hold(peer, 0, Store, {}); expect(await peer.next("failed")).toMatchObject({ - cause: { kind: "broken", error: { kind: "chord", error: { code: "unavailable" } } }, + cause: { kind: "waiting", error: { kind: "chord", error: { code: "unavailable" } } }, }); }); }); diff --git a/packages/chord/test/next/remote/remote-object.test.ts b/packages/chord/test/next/remote/remote-object.test.ts index eeb3fc7a9..c9a281f44 100644 --- a/packages/chord/test/next/remote/remote-object.test.ts +++ b/packages/chord/test/next/remote/remote-object.test.ts @@ -302,16 +302,20 @@ forEachCodec(() => { }); }); - describe("self.get of a type over the link (R18)", () => { - it("a failed hold fails the dependent transiently; bound retries it at once with a new hold", async () => { + describe("self.get of a type over the link (dependency model §3)", () => { + it("a failed hold parks the dependent failed { waiting }: no new attempt or hold; bound wakes it", async () => { const View = local("app.view", { version: 1, params: {}, members: { count: int() } }); const errors: HostError[] = []; const client = createHost({ onError: (error) => errors.push(error) }); const [a, b] = memoryLinkPair(); client.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); const peer = new RawPeer(b); + let factories = 0; const facet = defineFacet((h) => - h.serve(View, (_, self) => ({ count: self.get(Notes, { session: "S1" }).count })), + h.serve(View, (_, self) => { + factories++; + return { count: self.get(Notes, { session: "S1" }).count }; + }), ); const loading = client.load([{ name: "@test/view", facet }], context); const { hold } = await peer.next("hold"); @@ -320,25 +324,20 @@ forEachCodec(() => { hold, cause: { kind: "transient", error: { kind: "chord", error: { code: "unavailable", message: "warming" } } }, }); + // Parked on a remote blocker: its first outcome, so the start barrier is over. await loading; - expect(errors).toMatchObject([ - { - kind: "activation_failed", - object: "app.view", - failure: { kind: "transient", error: { code: "unavailable" } }, - }, - ]); - const members = [ - { member: EDITOR.state.notes!, value: "[]" }, - { member: EDITOR.state.count!, value: "0" }, - ]; - peer.send({ kind: "bound", holds: [{ hold, variant: EDITOR.variant, members, failed: [] }] }); - // The retry binds its own hold before the failed attempt's is released. - const retried = await bindNext(peer, [{ text: "a" }]); - expect(retried).toBe(1); - expect(await peer.next("release")).toMatchObject({ hold }); - const view = await client.get(View, {}, context); + await transported(); + expect(errors).toEqual([]); + const view = client.hold(View, {}, context); + expect(view.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.notes(S1): warming" } }, + }); + bindHold(peer, hold, [{ text: "a" }]); + await transported(); + expect(view.state.value).toEqual({ kind: "live" }); expect(view.count.value).toBe(1); + expect([factories, peer.kinds().filter((kind) => kind === "hold" || kind === "release")]).toEqual([1, []]); }); }); @@ -370,7 +369,7 @@ forEachCodec(() => { // Timers stand still: neither the backoff window nor a readyTimeout ends while the test runs. fakeTimers(); - it("fails at once with the real cause when its load holds it (no wait for a timeout)", async () => { + it("parks at once on the real cause when its load holds it (no wait for a timeout)", async () => { const Flaky = remote("app.flaky", { version: 1, params: {}, @@ -399,16 +398,12 @@ forEachCodec(() => { second.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); const facet = defineFacet((h) => h.serve(View, (_, self) => ({ n: self.get(Flaky, {}).n }))); await second.load([{ name: "@test/view", facet }], context); - expect(errors).toMatchObject([ - { - kind: "activation_failed", - object: "app.view", - failure: { - kind: "transient", - error: { message: "app.view: dependency app.flaky failed", cause: { message: "app.flaky: offline" } }, - }, - }, - ]); + await transported(); + expect(errors).toEqual([]); + expect(second.hold(View, {}, context).state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.flaky: app.flaky: offline" } }, + }); await first.close(context); await failing; }); @@ -633,7 +628,7 @@ forEachCodec(() => { const toUpgraded = await relink(upgraded, links[0]); expect(notes.state.value).toMatchObject({ kind: "failed", - cause: { kind: "broken", error: { code: "contract_mismatch" } }, + cause: { kind: "waiting", error: { code: "contract_mismatch" } }, retryAt: undefined, }); notes.retry(); @@ -652,7 +647,7 @@ forEachCodec(() => { await transported(); expect(notes.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); notes.retry(); - expect(await peer.next("retry")).toEqual({ kind: "retry", hold }); + expect(await peer.next("retry")).toEqual({ kind: "retry", hold, operation: expect.any(String) }); bindHold(peer, hold); await transported(); expect(notes.state.value).toEqual({ kind: "live" }); @@ -771,15 +766,6 @@ forEachCodec(() => { h.serve(View, (_, self) => ({ count: self.get(Notes, { session: "S1" }).count })), ); - /** Binds the kept hold (0) on `next`; the retry's own hold (1) is bound and the kept one released (R18). */ - async function retried(client: Host, next: RawPeer): Promise { - expect(await bindNext(next, [{ text: "a" }])).toBe(0); - // Retried at once, long before its backoff window (1 s) ends: its new attempt holds anew. - expect(await bindNext(next, [{ text: "a" }])).toBe(1); - expect(await next.next("release")).toEqual({ kind: "release", hold: 0 }); - expect((await client.get(View, {}, context, { readyTimeout: 1 })).count.value).toBe(1); - } - it("stays loading through the drop, no failure reported, and goes live when the next link binds it", async () => { const errors: HostError[] = []; const client = createHost({ onError: (error) => errors.push(error) }); @@ -798,28 +784,32 @@ forEachCodec(() => { expect(errors).toEqual([]); }); - it("dropped and never bound again, fails transient at its readyTimeout (the bound of the wait)", async () => { + it("dropped and never bound again, stays parked loading: nothing fails, no parked_long without a link", async () => { const errors: HostError[] = []; const client = createHost({ readyTimeout: 1_000, onError: (error) => errors.push(error) }); const peer = await rawParent(client); const loading = client.load([{ name: "@test/view", facet }], context); await peer.next("hold"); + // A remote blocker ends the start barrier's first pass. + await loading; peer.close(); await transported(); + await vi.advanceTimersByTimeAsync(10_000); + // No remote attempt answers for it while its path has no link (fix round 1, Opus F2). expect(errors).toEqual([]); - await vi.advanceTimersByTimeAsync(1_000); - await loading; - expect(failures(errors)).toEqual(["app.view: transient"]); + expect(client.hold(View, {}, context).state.value).toEqual({ kind: "loading" }); }); - it("without any link fails at its readyTimeout, keeps the reference, and is retried when the first link binds it", async () => { + it("without any link stays loading past its readyTimeout and goes live when the first link binds its one hold", async () => { const errors: HostError[] = []; const client = createHost({ readyTimeout: 1_000, onError: (error) => errors.push(error) }); - const loading = client.load([{ name: "@test/view", facet }], context); - await vi.advanceTimersByTimeAsync(1_000); - await loading; - expect(failures(errors)).toEqual(["app.view: transient"]); - await retried(client, await rawParent(client)); + await client.load([{ name: "@test/view", facet }], context); + await vi.advanceTimersByTimeAsync(500); + expect(failures(errors)).toEqual([]); + const next = await rawParent(client); + expect(await bindNext(next, [{ text: "a" }])).toBe(0); + expect((await client.get(View, {}, context, { readyTimeout: 1 })).count.value).toBe(1); + expect(next.kinds().filter((kind) => kind === "hold" || kind === "release")).toEqual([]); }); }); diff --git a/packages/chord/test/next/store/table-migration.test.ts b/packages/chord/test/next/store/table-migration.test.ts index d58bef1b7..e5938015b 100644 --- a/packages/chord/test/next/store/table-migration.test.ts +++ b/packages/chord/test/next/store/table-migration.test.ts @@ -218,18 +218,33 @@ describe("eager migrations", () => { }); }); +/** + * Records the `setTimeout` calls the migration's own code makes (its stack passes through `table-migration.ts`); timers + * of the test runner or other work under load pass through unrecorded (fix round 2: the test watched every timer). + */ +function migrationTimers(): { readonly calls: string[]; restore(): void } { + const calls: string[] = []; + const original = globalThis.setTimeout; + const spy = vi.spyOn(globalThis, "setTimeout").mockImplementation(((...args: Parameters) => { + const stack = new Error().stack ?? ""; + if (stack.includes("src/next/store/table-migration.ts")) calls.push(stack); + return original(...args); + }) as typeof setTimeout); + return { calls, restore: () => spy.mockRestore() }; +} + describe("the chunk yield", () => { it("yields between chunks without Node's setImmediate (Workers, browsers), through a message, not a timer", async () => { await filled(3 * MIGRATION_CHUNK); const saved = globalThis.setImmediate; // Runtimes without it: Cloudflare Workers, browsers. Reflect.deleteProperty(globalThis, "setImmediate"); - const timers = vi.spyOn(globalThis, "setTimeout"); + const timers = migrationTimers(); try { await prepared(EditsV2); - expect(timers).not.toHaveBeenCalled(); + expect(timers.calls).toEqual([]); } finally { - timers.mockRestore(); + timers.restore(); globalThis.setImmediate = saved; } expect(store.table(TABLE)).toMatchObject({ version: 2, state: "ready" }); @@ -242,12 +257,12 @@ describe("the chunk yield", () => { setTimeout(() => { fired = true; }, 0); - const timers = vi.spyOn(globalThis, "setTimeout"); + const timers = migrationTimers(); try { await prepared(EditsV2); - expect(timers).not.toHaveBeenCalled(); + expect(timers.calls).toEqual([]); } finally { - timers.mockRestore(); + timers.restore(); } expect(fired).toBe(true); expect(store.table(TABLE)).toMatchObject({ version: 2, state: "ready" }); From 972d7017af55ce15eb05059c56a647c9530c55f5 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 29 Oct 2026 11:54:38 +0000 Subject: [PATCH 143/271] docs(chord): slice 7b brief: reload, decided and open points --- packages/chord/docs/next/slice-7b/book.md | 304 +++++++++++ packages/chord/docs/next/slice-7b/index.html | 500 +++++++++++++++++++ 2 files changed, 804 insertions(+) create mode 100644 packages/chord/docs/next/slice-7b/book.md create mode 100644 packages/chord/docs/next/slice-7b/index.html diff --git a/packages/chord/docs/next/slice-7b/book.md b/packages/chord/docs/next/slice-7b/book.md new file mode 100644 index 000000000..bda87641d --- /dev/null +++ b/packages/chord/docs/next/slice-7b/book.md @@ -0,0 +1,304 @@ +--- +title: Slice 7b: reloading plugins in a running Chord app +cover: Slice 7b / Reload +kicker: A design brief · Chord v2 · slice 7b · 2026-10-29 +subtitle: What happens to objects, references, jobs and registries when a plugin's code changes on one host or across many, what is decided and what the owner still decides +stats: + - 4 | steps of a reload on one host + - 3 | fates of a live object + - 26 | carry-over items from slices 3a to 7a + - 5 | decisions open +note: | + A brief before the slice 7b spec, written by the redesign session from `design.md` §21, the dependency model + (`dependencies/model.md`, decided and built for 7a in pi `339847eb1`), the carry-over in `slices.md` and the code at + `339847eb1`. Marks: **Decided** points cite the decision; **Open** points are the owner's, with a recommendation. +colophon: Slice 7b brief · 2026-10-29 · Chord v2 +--- + +# Before we begin {.preface} + +Mario edits the notes plugin while a session is open: a new method, a renamed field in a document, a second state +member. He presses Reload. The server, the session's worker and two browser tabs run the old notes code; within a +second they should all run the new one, with the notes, the open views and the running export job intact, and without +restarting a single process. + +Chord v2 can do everything around that today: objects, references across hosts, the store, jobs, registries, and since +7a a dependency model in which a missing provider is a wait, not an error. What it cannot do yet is change code. `load` +runs once per host (`host.ts`: "load runs once per host"). Slice 7b adds the reload, on one host and across hosts. + +## Six things to know + +:::findings +1. **A reload is a transaction per host.** Load the bundles, stage the new declarations, commit atomically, then let + each object follow; any code error before the commit leaves the host entirely on its old code. +2. **A live object has three fates.** Pure objects swap in place; objects with natives drain and restart; objects of a + removed type end, and their holders wait. +3. **Old calls finish on old code.** A pure swap keeps the running calls' old handles valid until they settle; only new + calls see the new code. +4. **Versions are the provider's business.** A provider accepts older consumers through `accepts`; a consumer that does + not fit waits (7a), it is not ended. +5. **7a already decided most of the hard part.** Removal, dropped versions and installs are waits woken by a + publication; a reload's commit is such a publication. +6. **Five points are open:** the reload API, the rollout across hosts, loader D, a `writable` across a swap, and the + install watchdog. +::: + +## Words used throughout + +A **reload** changes the code of one or more plugins on one host. A **facet** is a plugin's code for one host kind; a +**bundle** is the built JavaScript of a facet. **Staging** runs the new facets and factories against a recording +`self` without activating anything. The **commit** switches the host to the new code at once and is a **publication** +in 7a's sense. A **pure** object has no natives (`self.open`), in its old or its new code. A **rollout** is a reload +of the same code set on every host, server first. + +# One Host Reloads {#host lede="Four steps, all or nothing up to the commit."} + +Design §21 fixes the shape. **Decided.** + +```text +1. load the new bundles; an import or syntax error rejects +2. staging facet setups and every live object's new factory run with a recording `self` + (declarations only, nothing activated); any throw rejects +3. commit the host switches to the new code atomically: a publication (7a) +4. after each object follows its fate (chapter II) +``` + +Rollback is all or nothing per host, and only for code errors: an import error, a throwing setup or factory. A failed +`open`, a registry conflict or a failing `migrate` after the commit are runtime states of single objects, never a +reason to roll back. A host is at revision R or R−1, never in between. + +Staging runs factories against a recording `self` because a factory only declares: it returns members and calls +`self.state`, `self.get`, `self.open` and so on, and the recording tells Chord what changed (a new dependency, a native +added or removed, a member gone) before anything happens. + +:::sequence animate | A reload of the notes plugin on the worker. Nothing is visible to callers until the commit. +actor app "base code" "reload control" +actor host "worker W" "" +actor objs "notes objects" "live" +app -> host: reload([notes v2], ctx) +host -> host: load bundles +host -> host: staging: setups + new factories (recording self) +host => host: commit: types, registries, contributions swapped; publication +host => objs: each follows its fate +objs --> app: resolved: { reloaded, restarted, ended, failed } +::: + +:::review | Open 1: the API +Today a host has `load(facets, ctx)`, once. Proposal: `host.reload(facets, ctx)` with the plugins' new facets (by +plugin name: added, changed; a plugin missing from the list is removed), resolving with a report of what each object +did, rejecting with the code error and nothing changed. `load` stays the first call. Alternative: one +`host.load` that may be called again. Recommended: the separate `reload`, because its result differs (a report, not a +barrier) and its rollback rule does not apply to the first load (a throwing first setup closes the host). +::: + +# Three Fates {#fates lede="What happens to each live object at the commit."} + +The commit compares each live object's old and new declarations. **Decided** (design §21, owner 2026-10-26, the +dependency model). + +| The object | Fate | Calls | Documents, jobs | +| --- | --- | --- | --- | +| pure, old and new (no `self.open`) | **swapped in place**: new methods, derives and routines; references kept, new ones made | running calls finish on old handles; new calls see new code | stay | +| has natives, old or new | **restarted**: new calls park, running calls drain (`closeTimeout`), natives close (`reload`), the new factory runs, natives reopen | park during the restart | stay; running job runs end `aborted { reload }`, `finish` runs on new code | +| its type is gone | **ended**: natives close (`end`), callers rejected, handlers drain | rejected | in-memory gone; persisted stay until retired | +| a new dependency is missing (any of the above) | **parks** (decision 4): gate closed, `failed { waiting }` | reject | stay | + +```ts +// notes v1 // notes v2: pure, swapped in place +host.serve(NoteStore, (_, self) => { host.serve(NoteStore, (_, self) => { + const notes = self.state(Notes); const notes = self.state(Notes); // same document, v2 migrates it + return { const count = self.derive([notes], ([n]) => n.list.length); + notes, return { + add: (a, ctx) => self.tx(…, ctx), notes, count, // a new exposed member + }; add: (a, ctx) => self.tx(…, ctx), +}); rename: (a, ctx) => self.tx(…, ctx), // a new method + }; + }); +``` + +Rules that come with the fates, all **decided**: + +- **R15, old code and migrated documents:** an old call that touches a document whose version rose throws once; the + document stays intact; the call is not resent. +- **Derives** have no names: exposed derives are matched by their member name, internal ones are rebuilt. +- **Routines** hand over per object: the old ones are aborted, the new ones start once the old calls finished, bounded + by `closeTimeout`; calls do not wait for it. +- **Instance ids:** a restarted object gets a new instance id (an old activation chain must not match it); a swapped one + keeps it. +- **Broken objects** of a reloaded plugin are retried, as `ref.retry()` does. +- **A parked object** of a reloaded plugin is replaced by the new factory's attempt in one handover: the new requests + bind before the old ones are released (the mechanism 7a removed comes back here). +- **In-memory documents** migrate on reload through their definition's `migrate` (6a built the version check). + +:::review | Open 2: a `writable` across a pure swap +A pure swap runs the new factory, so `self.writable(initial)` makes a new writable at its initial value. An object that +contributes MCP's tool list through a writable then contributes an empty list until its restarted routine sets it again: +the tools vanish for a moment. Options: (a) accept it (the routine refills it within one call); (b) keep a writable's +value across a swap when the new factory declares a writable at the same position (as members are matched by name). +Recommended: (a); writables have no names, and position matching would be the first positional rule in Chord. +::: + +# Consumers Across a Reload {#consumers lede="Holds, versions and old calls; 7a decided that nothing is ended for a code change."} + +A reference on another host points at an object whose code may change under it. **Decided** (design §21, §22 and the +dependency model): + +| The provider's reload | The consumer's hold | It sees | +| --- | --- | --- | +| still accepts the hold's version, object swapped | kept; re-subscribed through the version's mappings | one `[["r", value]]` per changed member | +| still accepts it, object restarted | kept | `loading`, then a fresh `bound` | +| stops accepting the version | unbound `version` | `failed { waiting }`; it binds again if a later reload accepts it | +| removes the type | unbound `not_served` | `failed { waiting }`; it binds again when the type comes back | + +A provider accepts older consumers through `accepts`, written once with the provider, never in the shared contract +(Q10): + +```ts +export const NotesV2 = evolve(NotesV1, (e) => e.state("editor", "notes", arr(NoteV2)).args("add", NoteV2)); + +host.serve(NotesV2, factory, { + accepts: { + 1: { + state: { editor: { notes: (current) => current.map(toV1) } }, // what a v1 tab sees, as a diff per commit + methods: { add: { args: (old) => toV2(old) } }, // a v1 tab's call, mapped forward + }, + }, +}); +``` + +- **Old-version calls:** `checkArgs` returns `{ kind: "method", args, result? }` or `{ kind: "call", args, call }`; Chord + runs the handler or the provider's replacement and maps the result back (L1 built the check; 7b invokes it). +- **Adapted bindings get diffs:** Chord keeps the last adapted value per binding and sends `diffRevisions`, so an old + tab never receives a whole state per commit. +- **On the wire** (7b's half of 7a's model): a `failed` on a bound hold unbinds it; the provider drops its unsent + pending work and keeps the ack slots of messages already sent; a consumer whose own code moves to a new version + releases its old hold and makes a new one. +- **A consumer reloaded itself** gets new requests from its new factory; the handover binds them before it releases the + old. + +Today `host.serve` binds every provider with an empty accepts table (`host.ts`: `bindProvider(contract, {})`); 7b passes +the provider's table. Small and **decided** in shape (Q10, `bindProvider`); the option's place is `host.serve(Type, +factory, { accepts, linger })`. + +# Jobs Across a Reload {#jobs lede="Runs belong to the object; their versions are mapped forward."} + +**Decided** (design §15, owner 2026-10-24 and 2026-10-26): + +```ts +const ExportJob = defineJob("notes.export", { + version: 2, + args: obj({ format: oneOf("md", "html") }), + result: obj({ path: str() }), + accepts: { 1: { args: (old) => ({ format: "md" }), result: (old) => ({ path: old.file }) } }, +}); +``` + +- A run started on an older version finishes on its old code across a pure swap; `finish` and `running` see its args + and result mapped forwards, once per run. +- Without an adapter, or for a run of a newer version (a rollback), the run is `blocked { version }` until a reload + brings matching code; blocked runs are retried on every reload. Nothing is aborted or discarded for a version change. +- A native restart aborts running runs (`aborted { reload }`) and `finish` runs on the new code. +- A job the new code no longer declares: its runs are `blocked { version }` (the same family), kept until a reload + declares it again or they are aborted. +- Persisted runs of another version or fingerprint stay blocked with their record as stored, resolved by the reload + that brings their version. + +# Registries Across a Reload {#registries lede="A provider and its contributions are swapped in one batch."} + +**Decided** (slice 8 §6, design §17): the commit replaces a plugin's contributions and a registry's provider in one +batch; staging composes nothing (D1). The record keeps the provider, facet contributions by plugin and object +contributions by object apart, so a reload of one plugin replaces exactly its parts (today `RegistryRecord.serve` +refuses a second serve). Older readers across a reload of the provider follow its new `accepts[old].value`. A reader +waits (7a) when the new provider no longer serves its version. `retryOn` watches are re-declared by the new factory. + +# Across Hosts {#rollout lede="The same reload on every host, in an order that cannot deadlock."} + +**Decided** (design §21 and the dependency model): a revision is the complete code set; the server installs first, +then workers, then clients; installs are serialized per host; a host that starts or reconnects installs the current +revision first; a revision carries a code-set digest so an unchanged server restart reinstalls nothing; an install +never waits for activations (a reverse-order rollout would deadlock); each host knows only its own install: while it +installs, holds that do not fit its running code wait `installing`, re-evaluated when the install ends; a host whose +own install was rejected fails its own requests with the reason. + +:::steps | The notes contract goes from v1 to v2; the server installs first ++++ Before +S, W and T run revision R. T's notes view holds W's `Notes` v1. All live. ++++ S installs R+1 +S has no notes objects of its own; its install commits. T, still on R, holds nothing on S that changed. ++++ W installs R+1 with `accepts[1]` +W's `Notes` v2 swaps in place and still accepts v1: T's hold is re-subscribed through the mapping and gets one diff. ++++ W installs R+1 without `accepts[1]` +T's hold becomes unbound `version`: `failed { waiting }`, last notes readable, calls reject with "update the page". ++++ T reloads its page +T's new code holds v2 with a new hold id; it binds; the old hold is released. +::: + +:::review | Open 3: who runs the rollout +Chord's part is per host: `host.reload`, the `installing` state, and what a hold sees. The rest is distribution: +building bundles, sending them to workers, telling tabs to reload, ordering, the digest. Options: (a) Chord ships a +rollout protocol over its links (an `install` message from the server down, `installed`/`rejected` up, the digest in +`hello`); (b) distribution is the app's (p3's base code calls `host.reload` on each host and orders them), as p2 does +today (`apps/p2/src/hosts/server.ts` `performReload`: prepare every host, commit, publish bundles). Recommended: (b) for +7b, with Chord's per-host guarantees; a protocol only if a second app needs the same code. +::: + +:::review | Open 4: the install watchdog (design §26 #24) +An install that hangs (a bundle that never loads, a staging factory that awaits forever, which it may not) leaves its +holds `installing` forever. Options: (a) a watchdog: `host.reload` rejects after `installTimeout` (default 60 s) and the +host stays on its old code; (b) no Chord timer: the app's context bounds the reload. Recommended: (b): `reload` takes a +context like everything else, and the app passes its deadline; a host option would be the only timer left in reload. +::: + +# Loader D {#loader lede="How a plugin's new code gets into a running process."} + +p2 designed it (`apps/p2/docs/plugin-loading.md` §9, §14); Chord has not built it. **Decided in p2's design:** + +- **One base bundle per surface** holds every host package once: Chord, Durable, pi-ai, pi-tui, TypeBox, React and + `react-dom`, plus the app's runtime. A change to it restarts the hosts. +- **One script per facet**, an esbuild bundle wrapped as a function of its host packages, which are external and + passed in, so there is one Chord and one React per process: + + ```js + // notes/worker.facet.js, built + (require, chord, durable, react) => { + const { defineFacet, local, str } = chord; // the host's instance, not a copy + /* the plugin's own files, contracts included (bundled: their versions are the plugin's) */ + return defineFacet((host, self) => { … }); + } + ``` + +- **Evaluation:** `vm.Script.runInThisContext` on Node (server, workers, TUI worker), a classic ` + From adfa8f3827e92947420d7c85458ff80566759d5e Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 29 Oct 2026 12:15:57 +0000 Subject: [PATCH 144/271] docs(chord): slice 7b brief: plugin layout, building and loading, assets, HTTP, the app kit, measurements --- packages/chord/docs/next/slice-7b/book.md | 379 ++++++++++++++-- packages/chord/docs/next/slice-7b/index.html | 443 ++++++++++++++++--- 2 files changed, 728 insertions(+), 94 deletions(-) diff --git a/packages/chord/docs/next/slice-7b/book.md b/packages/chord/docs/next/slice-7b/book.md index bda87641d..8e2566b9c 100644 --- a/packages/chord/docs/next/slice-7b/book.md +++ b/packages/chord/docs/next/slice-7b/book.md @@ -2,12 +2,13 @@ title: Slice 7b: reloading plugins in a running Chord app cover: Slice 7b / Reload kicker: A design brief · Chord v2 · slice 7b · 2026-10-29 -subtitle: What happens to objects, references, jobs and registries when a plugin's code changes on one host or across many, what is decided and what the owner still decides +subtitle: What a plugin is, how it is built and loaded, how a mini p2 is assembled from Chord, and what happens when a plugin's code changes on one host or across many stats: - 4 | steps of a reload on one host - 3 | fates of a live object - 26 | carry-over items from slices 3a to 7a - - 5 | decisions open + - 11 ms | to rebuild one facet after an edit + - 6 | decisions open note: | A brief before the slice 7b spec, written by the redesign session from `design.md` §21, the dependency model (`dependencies/model.md`, decided and built for 7a in pi `339847eb1`), the carry-over in `slices.md` and the code at @@ -26,7 +27,7 @@ Chord v2 can do everything around that today: objects, references across hosts, 7a a dependency model in which a missing provider is a wait, not an error. What it cannot do yet is change code. `load` runs once per host (`host.ts`: "load runs once per host"). Slice 7b adds the reload, on one host and across hosts. -## Six things to know +## Eight things to know :::findings 1. **A reload is a transaction per host.** Load the bundles, stage the new declarations, commit atomically, then let @@ -39,8 +40,12 @@ runs once per host (`host.ts`: "load runs once per host"). Slice 7b adds the rel not fit waits (7a), it is not ended. 5. **7a already decided most of the hard part.** Removal, dropped versions and installs are waits woken by a publication; a reload's commit is such a publication. -6. **Five points are open:** the reload API, the rollout across hosts, loader D, a `writable` across a swap, and the - install watchdog. +6. **A plugin is a directory with a `package.json` and one file per surface.** Each facet is bundled alone; host + packages are passed in, never copied; a whole plugin set builds in a few hundred milliseconds. +7. **Chord should ship the app kit** (proposed): loader, HTTP, workers, web, TUI and dev reload as optional + components, so a mini p2 is a page of code. +8. **Six points are open:** the reload API, the rollout across hosts, a `writable` across a swap, the install + watchdog, where the loader lives, and the app kit. ::: ## Words used throughout @@ -51,6 +56,332 @@ A **reload** changes the code of one or more plugins on one host. A **facet** is in 7a's sense. A **pure** object has no natives (`self.open`), in its old or its new code. A **rollout** is a reload of the same code set on every host, server first. +# A Plugin, File by File {#plugin lede="What a plugin looks like on disk, and what each file is for."} + +A plugin is a directory. Its name is its `package.json` name; its code is one file per host kind it runs on, a +**facet**; everything else is ordinary TypeScript the facets import. This is p2's layout today (`apps/p2/plugins/*`), +plus the `package.json` that loader D needs (p2's `plugin-loading.md` §8; p2's built-in plugins have none yet and are +named by their directory). + +```text +plugins/notes/ + package.json name = the plugin's name; peerDependencies = host packages; dependencies = bundled npm deps + contract.ts the plugin's object types and registries (shared by its facets and by other plugins) + server.ts facet for the server (default export: defineFacet(...)) + worker.ts facet for session workers + web.tsx facet for browser tabs + tui.ts facet for the TUI + store.ts plain modules the facets import + NoteList.tsx + notes.css + assets/ + template.md files the plugin ships (read at runtime, or offered for download) + icon.svg + skills/notes/SKILL.md optional resources (p2 §4.4) +``` + +```json +{ + "name": "@pi/notes", + "version": "1.4.0", + "type": "module", + "peerDependencies": { + "@earendil-works/chord": "^2", + "@earendil-works/pi-durable": "^1", + "react": "^19" + }, + "dependencies": { + "marked": "16.4.1" + } +} +``` + +- **`name`** identifies the plugin everywhere: its facets, its registry contributions, diagnostics, reloads. +- **`peerDependencies`** are **host packages**: the running app supplies them, one instance per process (one Chord, + one React; `instanceof ChordError` and React hooks only work that way). The bundler never copies them. +- **`dependencies`** are the plugin's own npm packages (`marked`): installed into the plugin's `node_modules` with + `--ignore-scripts` and bundled into every facet that imports them. +- **No `tsconfig.json` is needed.** The bundler strips types and does not type-check; JSX is compiled with the + automatic React runtime, a fixed setting of the loader. A plugin may keep a `tsconfig.json` for its editor and for + `tsc --noEmit` (the SDK ships one to extend); the loader ignores it. + +A facet only declares (chapter III of the model): it serves object types, serves or contributes to registries, and +returns nothing. + +```ts +// plugins/notes/contract.ts +import { defineRegistry, local, remote, str, obj, arr, method } from "@earendil-works/chord/next"; +import { Session } from "@app/core"; // the app's host kinds, a host package +export const NoteStore = local("notes.store", { version: 1, params: {}, members: { /* … */ } }); +export const Notes = remote("notes", { + version: 2, + params: { session: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: { editor: { state: { list: arr(obj({ id: str(), text: str() })) }, methods: { add: method(obj({ text: str() })) } } }, +}); + +// plugins/notes/worker.ts +import { defineFacet } from "@app/core"; // bound to the app's principal type +import { NoteStore, Notes } from "./contract.ts"; +import { serveStore } from "./store.ts"; +export default defineFacet((host) => { + host.serve(NoteStore, serveStore); + host.serve(Notes, ({ session }, principal, self) => { /* … */ }); +}); + +// plugins/notes/web.tsx +import { defineFacet } from "@app/core"; +import { NoteList } from "./NoteList.tsx"; // a React component, bundled +import "./notes.css"; // bundled into the facet's stylesheet +export default defineFacet((host, self) => { + self.contribute(Panels, { notes: NoteList }); // the app's UI registry +}); +``` + +# Building and Loading {#loading lede="How the bundler knows what to pull in, and how the code gets into a running process."} + +Each facet is built on its own, as one script. The bundler starts at the facet file and follows its imports: + +| An import of | Becomes | +| --- | --- | +| the plugin's own files (`./store.ts`, `./NoteList.tsx`) | inlined into the facet's script | +| another plugin's contract (`../sessions/contract.ts`, or `@pi/sessions/contract` for an installed package) | inlined too: a copy, so a plugin carries the contract version it was built against (`accepts` needs that); only contracts and side-effect-free modules may be imported across plugins | +| an npm package in `dependencies` (`marked`) | inlined, resolved from the plugin's own `node_modules` | +| a host package (`@earendil-works/chord/next`, `react`, `@app/core`, …) | **external**: a parameter of the script, the host's instance at load time | +| a Node builtin (`node:fs`) | external, through the `require` the loader passes (server and worker facets only; a web facet that imports one fails to build) | +| CSS (`import "./notes.css"`) | collected into one stylesheet per web facet | +| an asset by URL (`import iconUrl from "./assets/icon.svg?url"`) | a content-hashed file served by the app; the import is its URL | + +The built script is a function of its host packages. That is the whole trick of loader D: there is no module map, no +second copy of Chord, and an old version is garbage-collected once nothing holds its closures. + +```js +// what the bundler emits for plugins/notes/worker.ts (simplified) +(function (require, __files) { + const { defineFacet } = require("@app/core"); // the host's instance + const { local, remote, str } = require("@earendil-works/chord/next"); + const { marked } = /* marked's code, inlined */; + /* contract.ts, store.ts, inlined */ + return defineFacet((host) => { /* … */ }); +}) +``` + +```ts +// the loader (proposed @earendil-works/chord/next/loader), Node side +import { buildFacet, evaluateFacet } from "@earendil-works/chord/next/loader"; + +const HOST_PACKAGES = { // the app's base bundle exports these + "@earendil-works/chord/next": chordNext, "@earendil-works/pi-durable": durable, + "@app/core": core, react, "react/jsx-runtime": jsxRuntime, +}; +const script = await buildFacet({ plugin: "/data/plugins/notes", facet: "worker.ts", hostPackages: Object.keys(HOST_PACKAGES), platform: "node" }); +const facet = evaluateFacet(script, { hostPackages: HOST_PACKAGES, require: createRequire(import.meta.url), dir: "/data/plugins/notes" }); +await host.reload([{ name: "@pi/notes", facet }], ctx); // or host.load the first time +``` + +- `buildFacet` is esbuild with: the facet as entry, the host packages and `node:*` external, `format: "cjs"` wrapped + as the function above, minified, an `onResolve` that refuses anything outside the plugin, its `node_modules`, a + contract of another plugin or a host package. It returns the script, its content hash and esbuild's metafile (the + dev loop knows which files a facet used). +- `evaluateFacet` runs the script with `vm.Script` and a stable filename per (facet, hash) on Node, or as a classic + ` - diff --git a/packages/chord/docs/next/explainer/src/00-front.md b/packages/chord/docs/next/explainer/src/00-front.md deleted file mode 100644 index 36d62e8d0..000000000 --- a/packages/chord/docs/next/explainer/src/00-front.md +++ /dev/null @@ -1,111 +0,0 @@ ---- -title: Chord v2: a guide to building apps that reload -cover: Chord v2 / A Guide -kicker: A guide · Chord v2 target design · 2026-10-20 -subtitle: How to build an app like p2 from nothing on Chord v2 and pi-durable, one object at a time, so that new plugin code takes over while the agent keeps running -stats: - - 11 | layers in the rebuild, L1 to L11 - - ~1,300 | lines built so far (L1 contracts) - - 6–7k | target size of Chord v2, without tests - - 0.004 ms | Delta's apply per small change -note: | - This guide teaches the **designed** API of Chord v2 (`packages/chord/src/next`, export `@earendil-works/chord/next`) - from `packages/chord/docs/next/design.md` and the newer decision log `packages/chord/docs/next/decisions.md`, which wins - where the two differ. Most of it is **not built yet**: a first kernel of about 10k lines is frozen as a reference and - Chord v2 is being rebuilt layer by layer; only the L1 contract layer exists (`src/next/contracts/`, commit - `7d2bb8ea8`). Code in this guide is therefore the target, not something you can run today. Every chapter says what is - built. API names are working names. Notes marked *review* are points that are undecided, or where the sources - disagree; they name the files. The reader is assumed to know pi-durable; Chord is explained from the start. -colophon: Chord v2 · a guide · first edition · 2026-10-20 · based on design.md (target design v2), redesign-decisions.md (review rounds 2 and 3), chord-next-walkthrough.md, layer-1.md, contracts/README.md, the p3 functionality map, questions.md and the pi-durable README ---- - -# Before we begin {.preface} - -It is late in the evening. Mario is on his phone, connected to his laptop at home through the Radius relay. In session -S1 the agent is twenty minutes into a refactoring; right now it is running a test suite through `bash`. A colleague, -Ann, follows the session read-only from her own browser. Mario notices that the notes panel sorts notes the wrong way -round. He fixes one line in the notes plugin on the laptop and presses **Reload**. - -What should happen is easy to say. The notes panel on both screens shows the new order. The test run keeps running. The -agent's turn continues as if nothing happened. Ann's view does not flicker into an error. Nothing that was written is -lost, and no process restarts. - -The obvious ways to get there fail. **Restarting the processes** loses the turn in flight, the open terminal and every -connection; pi-durable would resume the turn, but the test run and the connections are gone. **Hot-swapping modules** -in place, as front-end tooling does, leaves the old code alive in closures, timers, event listeners and sockets that -nobody knows about: the old sort function keeps running inside a callback someone stored, a file watcher opened by the -old code is never closed, and two versions of the plugin now write the same data without any order. - -Chord v2 is the runtime that makes the first outcome normal. It does so by being strict about one thing: **every piece -of state, every open OS handle and every piece of running work has exactly one owner, an object, and objects only -declare what they need.** Chord itself activates, connects, reloads and ends them. Because Chord knows every timer, -watcher, transaction and reference an object has, it can swap the object's code while its data stays, restart only -what touches the operating system, and route calls between browser tabs, the server and per-session worker processes -without the plugin author writing any of that. - -This guide teaches Chord v2 by building an app called **Studio**: a small version of p2 (Pi Mobile), with a server, -one worker process per session, browser tabs as clients, and an agent from pi-durable on each worker. The first -chapters grow one plugin, `@studio/notes`, until it uses every concept; the last three chapters assemble the rest of a -p2-like app from the same parts. - -:::review | Review: what exists today -Nothing in this guide runs yet except the contract layer. `packages/chord/src/next/contracts/` (L1 part 1: `remote`, -`local`, `method`, `evolve`, `checkAccepts`, the schema helpers) is built and reviewed. The rest of L1 (inputs, -argument checks, principals, errors) is a draft spec (`packages/chord/docs/next/layer-1.md`). The frozen kernel in -`src/next/*.ts` implements an older version of the design (host values, a development freeze, JSONL storage, -descriptions on the wire); do not read it as the target. Build on the design, not on the frozen code. -::: - -## Ten things to know - -:::findings -1. **An app is a tree of hosts.** One server, any number of workers it starts on demand, any number of clients (tabs, - TUIs) that connect to it. Clients reach workers only through the server, which relays. -2. **Objects are the unit of everything.** An object type is a contract (name, input, placement, members, version); an - object is one instance per input and caller. The first `get` creates it, later gets share it, the last release ends - it. -3. **Factories declare, Chord activates.** A factory is synchronous: it declares documents, references, derived - values, OS handles, routines and jobs through `self`, and returns the members. Chord opens, binds and starts them, - and undoes them in reverse. -4. **Links stamp who is calling.** A caller's data never says who it is; the link it arrived on does. Each caller gets - its own object and the variant of members it may use. -5. **Shared data lives once, behind views.** A `local` shared store holds the data; per-caller `remote` views expose - what each caller may see. Only stable identities persist documents. -6. **State changes only in transactions.** Documents are JSON values written in `self.tx`; commits are atomic for - every viewer and travel as small operations, never as whole values. -7. **Failures look like disconnects.** A reference survives link drops, restarts and failures: calls park or fail, - values stay readable, and Chord retries by itself. There are no request ids. -8. **Everything long-lived has a declared form.** Standing behaviour is a routine, work that outlives a call is a job, - input from a person is a flow, behaviour from other plugins comes through a registry. Nothing else may hold a timer - or a callback. -9. **Reload swaps pure objects in place and restarts native ones.** Objects without OS handles get the new code with - their data and running work intact; objects with OS handles close and reopen them. A broken change leaves the whole - host on its old code. -10. **pi-durable runs inside a core object.** The agent harness is a host-lifetime object on each worker; models, - environments and tools reach it through registries, and tools reach Chord through `chord(fn, context)`. -::: - -## Words used throughout - -A **host** is one participant of the app: a process, a browser tab or a Durable Object, identified by a logical -identity and an **incarnation** that changes on every start. A **link** connects two hosts. The **server** is the one -addressable root host; a **worker** is a host the server starts for one session; a **client** is a tab or TUI, not -addressable. A **plugin** is a directory with a `package.json`; its code for one kind of host is a **facet**. **Base -code** is the app's own code (Chord calls its facet the core facet); it restarts the hosts when it changes, while -plugins reload. An **object type** (or **contract**) describes objects; an **object** is one instance; its **input** is -its key among objects of that type, and its **placement** computes from the input on which host it lives. A **factory** builds an -object; **`self`** is what a factory declares through. A **principal** is who calls, stamped by the link; a -**variant** is the set of members a principal gets. A **reference** is what `get` returns. A **readable** is a value -plus its ordered changes. A **document** is a named JSON value an object owns, written in a **transaction**. A -**derive** is a value computed from readables. A **native** is an OS-level value (a socket, a watcher, a process) -opened with `self.open`; an object with natives is a **native object**, one without is a **pure object**. A -**routine** is standing behaviour, a **job** is owned work that outlives its call, a **flow** is input into running -work. A **registry** is an owner's extension point, filled with **contributions**. **`ctx`** is what every call -receives: cancellation, caller, call-scoped references. A **reload** swaps a plugin's code inside running hosts; a -**restart** ends a process; a **revision** names the complete code set the hosts run. - -:::note | Reading the code in this guide -Imports are shown once per file and then omitted. `obj`, `str`, `arr` and friends are Chord's schema helpers (they -return ordinary TypeBox schemas). Helper functions that belong to the app (`authenticate`, `watchDir`, `uuidv7`) are -named and not shown. Where the design gives no name, the guide says it uses a working name. -::: diff --git a/packages/chord/docs/next/explainer/src/01-shape.md b/packages/chord/docs/next/explainer/src/01-shape.md deleted file mode 100644 index db18b5cfe..000000000 --- a/packages/chord/docs/next/explainer/src/01-shape.md +++ /dev/null @@ -1,246 +0,0 @@ - -# The Shape of an App {#shape lede="Which processes a Chord app has, how they find each other, and the start code that connects them."} - -Before any plugin exists, Studio needs processes and connections between them. In Chord each participant is a host, -and the app's own start code creates every host, connects them and decides how workers start. Chord deliberately does -none of that itself: a laptop app forks child processes, a test runs everything in one process, a Cloudflare -deployment routes through the edge. This chapter writes Studio's start code: `server.ts`, `worker.ts`, the browser -entry, and a test setup. - -## Three roles - -A Node app on Chord has one shape: - -:::flow wide | The three roles. Clients connect to the one server; the server starts workers and relays between them. -row "Clients: not addressable" muted -box tab1 "browser tab" "owner, local" -box tab2 "browser tab" "guest Ann" -box tui "TUI" "owner" -row "Server: exactly one" accent -gap -box server "server" "`Server()`" accent -gap -row "Workers: one per session, on demand" blue -box w1 "worker" "`Session({ id: \"S1\" })`" blue -box w2 "worker" "`Session({ id: \"S2\" })`" blue -box w3 "worker" "`Session({ id: \"S3\" })`" dashed -arrow tab1 -> server "WebSocket" -arrow tab2 -> server "WebSocket" -arrow tui -> server "WebSocket" -arrow server -> w1 "IPC" -arrow server -> w2 "IPC" -note "A worker for S3 does not exist until something asks for an object that lives there." -::: - -| Role | Addressable | Who it is comes from | Lifecycle | -| --- | --- | --- | --- | -| client | no | the authentication of its link | connects by itself | -| server | yes, exactly one | its own declaration | runs | -| worker | yes, many, by a typed address | stamped by the server, its parent | started on demand by the app | - -The **server** holds what all sessions share (the session list, access, settings) and starts workers. A **worker** -runs one session: its agent, its files, its terminal connections. **Clients** are browser tabs and TUIs. Nobody can -send a request *to* a client, so a client can only serve objects that nobody outside it needs; chapter IV names them -`local` types. - -Things that look like hosts but are not: a code mode sandbox (a worker thread inside a worker, owned by an object), a -remote environment's daemon (it speaks its own protocol, owned by an object), the TUI's sandbox behind its launcher. -A detached process such as a terminal host could become another kind of child of the server. - -## Host kinds and addresses - -Every addressable host has a typed address. The app declares its **host kinds** once, in base code: - -```ts -// studio/src/hosts.ts -import { hostKind, obj, str } from "@earendil-works/chord/next"; - -export const Server = hostKind("server"); -export const Session = hostKind("session", obj({ id: str() })); - -Server(); // the address of the server -Session({ id: "S1" }); // the address of the worker of session S1 -``` - -There are no host-name strings anywhere in app code. Chord serializes addresses internally; your code only calls the -kinds. An object type names its host with a placement function from its input to an address (chapter II), so the -type checker knows that `Session({ id: session })` needs a `session` field. - -## Links - -A **link** is an ordered, two-way channel of messages between two hosts; it only delivers. Chord ships three: a memory -pair for tests, IPC for a Node child process forked with `serialization: "advanced"`, and WebSocket. All three carry -the same binary message format and the same flow control (chapter XVII). Recovery policy belongs to the link adapter: -IPC needs no heartbeat, a WebSocket through a relay needs one, because TCP does not notice that a laptop went to sleep. - -Each link has a **direction**. A host has at most one link to its parent (`connect`) and any number of links from -below (`accept` for clients, `route` for workers). Requests arriving on a link act as the principal the link was given; -chapter III explains why that matters. - -## The server - -```ts -// studio/src/server.ts -import { createHost, ipcLink, webSocketLink } from "@earendil-works/chord/next"; -import { fork } from "node:child_process"; -import { WebSocketServer } from "ws"; -import { Server, Session } from "./hosts.ts"; -import { coreServerFacet } from "./core/server.ts"; -import type { P } from "./principal.ts"; - -const server = createHost

    ({ - address: Server(), - principal: { kind: "server" }, // who this host's own code acts as - storage: await openServerStorage(dataDir), // persisted documents (chapter V); memory if omitted - onError: (e) => { if (e.kind === "storage_failed") process.exit(75); else log(e); }, -}); - -await server.load([ - { name: "@studio/core", facet: coreServerFacet(config) }, // base code, under a fixed name - ...(await loadPlugins("server")), // [{ name, facet }] from the plugin loader -]); - -const wss = new WebSocketServer({ server: http, path: "/chord", maxPayload: 64 * 1024 * 1024 }); -wss.on("connection", (ws, req) => { - const principal = authenticate(req); // app code: cookie, loopback, device token - if (!principal) return ws.close(4401); - server.accept(webSocketLink(ws), { principal }); -}); - -server.route(Session, { - authorize: (p, { id }) => - p.kind === "owner" || p.kind === "server" || (p.kind === "guest" && p.session === id), - resolve: async ({ id }, { signal }) => { - const principal: P = { kind: "worker", session: id }; - const child = fork("worker.ts", [], { serialization: "advanced", signal }); - child.send({ id, principal, init: await initFor(id, signal) }); // the app's own start message - return { link: ipcLink(child), principal }; // Chord stamps `principal` on it - }, -}); -``` - -Each call has one job: - -| Call | Meaning | -| --- | --- | -| `createHost

    ({ address?, principal?, ...options })` | with an address: an addressable host whose own code acts as `principal`; without: a client. `P` is the app's principal type | -| `load([{ name, facet }])` | the facets this host runs, each with its plugin's `package.json` name; duplicates and invalid names are refused. Base code passes its core facet the same way | -| `accept(link, { principal })` | a link from a client; every request on it acts as `principal` | -| `route(Kind, { authorize, resolve })` | how to reach hosts of a kind. `authorize(principal, address)` runs before anything starts; `resolve(address, { signal })` returns `{ link, principal }` | -| `connect(link, { principal })` | the link to the parent, at most one, replaced after a reconnect; `principal` says who the parent is | -| `root(fn, { principal?, signal? })` | a call scope for code outside objects (a CLI command): gives `fn` a `ctx` (chapter VI) | - -Chord has no `listen`. Entry points differ too much: a loopback WebSocket server, a tunnel, the Radius relay -delivering channels, a Durable Object edge, a memory link in a test. Each entry point authenticates and calls -`accept`. Radius, for example, is one more line: - -```ts -radius.onChannel((ws) => - server.accept(webSocketLink(ws), { principal: { kind: "owner", access: "relay" } })); -``` - -## The worker - -```ts -// studio/src/worker.ts -const { id, principal, init } = await startMessage(process); // app code: the first IPC message -const worker = createHost

    ({ - address: Session({ id }), - principal, // { kind: "worker", session: id } - storage: await openSessionStorage(init.dataDir, id), // one storage per session -}); -await worker.load([ - { name: "@studio/core", facet: coreWorkerFacet(init) }, // serves the start data (chapter IX) - ...(await loadPlugins("worker")), -]); -worker.connect(ipcLink(process), { principal: { kind: "server" } }); -``` - -## A browser tab - -```ts -// studio/web/main.tsx -const client = createHost

    (); // no address: a client -await client.load(await loadPlugins("web")); -client.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }); -const notes = await client.get(Notes, { session: "S1" }); // tab → server → worker S1 -``` - -## The first `get`, step by step - -The tab asks for the notes of S1. No worker for S1 runs yet. - -:::sequence animate | The first get for an object on a worker that is not running. The server authorizes, the app starts the worker, Chord stamps the link. -actor tab "browser tab" "owner" -actor server "server" accent -actor app "route(Session)" "app code" muted -actor worker "worker S1" blue -tab -> tab: placement: Session({ id: "S1" }) -tab -> server: hold Notes { session: "S1" } -server -> server: not mine: route(Session) -server -> app: authorize(owner, { id: "S1" }) -app --> server: true -server => app: resolve({ id: "S1" }) -app -> worker: fork + start message -worker -> worker: createHost, load, connect -app --> server: { link, principal: worker S1 } -server => worker: hold (stamped: owner) -worker -> worker: placement check, run factory -worker --> server: bound: snapshot -server --> tab: bound: snapshot -::: - -`resolve` runs once per address while its link lives. When the link drops (the worker crashed or the supervisor -stopped it) and references to that worker still exist, Chord calls `resolve` again by itself, with a per-address -backoff, so a tab that only watches recovers without a click. A worker start that already produced a link finishes -even if the request that triggered it went away. - -**Incarnations** make restarts visible. Chord mints a fresh id (a uuidv7) on every host start and sends it in the link -handshake; the other side sees a restart as a new incarnation of the same identity, and references to it re-hold -(chapter VI). - -## Tests: everything in one process - -The same code runs in a test with memory links. `resolve` builds the worker host in memory and returns one end of the -pair: - -```ts -server.route(Session, { - authorize: () => true, - resolve: async ({ id }) => { - const [up, down] = memoryLinkPair(); - const principal: P = { kind: "worker", session: id }; - const worker = createHost

    ({ address: Session({ id }), principal }); - await worker.load([{ name: "@studio/core", facet: coreWorkerFacet({ id, dataDir }) }, notesWorker]); - worker.connect(up, { principal: { kind: "server" } }); - return { link: down, principal }; - }, -}); -``` - -Memory links deliver the same encoded messages asynchronously, so a test runs the production protocol path. - -## What stays app code - -Starting processes, the start message, authentication, supervising and stopping idle workers are app code. Chord -provides one readable for the last one: `host.activity`, with `{ references, calls, jobs, lingering }` (viewers, calls -in flight, unsettled jobs, objects kept warm after their last reference). Studio's core reports it to the server -together with the agent's running tasks, and the server's supervisor stops a worker when everything has been zero for -five minutes, as p2 does. Chord counts; it has no idle policy. - -## Host options and timeouts - -`createHost` takes the options introduced later in this guide: `storage` (V), `principalKey` and `revokedBy` (III, -VII), `telemetry` (XVII) and `onError`. `onError` receives background errors and `{ kind: "storage_failed" }`; every -app must handle the latter, because Chord never ends the process. Studio exits with its own code so its supervisor -restarts it. - -Timeouts are host defaults with per-thing overrides; the full table is in the appendix. The two you meet first: -`readyTimeout` (60 s: how long an activation and an awaited `get` wait) and the park deadline (60 s: how long a call -waits through a link drop or a restart before failing `unavailable`). - -:::note | Built so far -Nothing in this chapter is built in Chord v2 yet: hosts, links and routes are layers L6 and L7. The frozen kernel has -working versions with the same shape (`createHost`, `connect`, `accept`, `route`, `load`), which the rebuild -simplifies. -::: diff --git a/packages/chord/docs/next/explainer/src/02-first.md b/packages/chord/docs/next/explainer/src/02-first.md deleted file mode 100644 index 5c01e5213..000000000 --- a/packages/chord/docs/next/explainer/src/02-first.md +++ /dev/null @@ -1,187 +0,0 @@ - -# A First Object {#first-object lede="A plugin, a contract, a factory, and a browser tab that calls it on another process."} - -Studio's first feature is session notes: a list of short notes per session, shown in a panel, that anyone allowed can -add to. Several places need "the notes of S1": the owner's tab, a guest's tab, later a download route and an agent -tool. If each built its own copy, the copies would drift apart and nobody would know when to clean them up. In Chord -there is **one object** for "the notes of S1", everyone asks for it by its type and input, and Chord routes them all to it. - -## The plugin's files - -A plugin is a directory with a `package.json`. Its `name` (npm style, globally unique and stable) is the plugin's -identity: Chord finds its stored documents, reports its errors and orders its contributions by it, never by a -directory name or load order. - -| File | Runs on | Holds | -| --- | --- | --- | -| `package.json` | read by the loader | `"name": "@studio/notes"` | -| `contract.ts` | every host that imports it | object types: names, shapes, method signatures; no running code | -| `worker.ts` | every session worker | the worker facet | -| `server.ts` | the server | the server facet (later: the download route) | -| `web.tsx` | browser tabs | the web facet: the panel | - -## The contract - -An object type is a contract: a name, an **input** (the object's key among objects of this type), a **placement** -(on which host it lives, computed from the input), the **members** (state you can read and methods you can call) and a **version**. - -```ts -// notes/contract.ts -import { arr, int, method, obj, remote, str, type Static } from "@earendil-works/chord/next"; -import { Session } from "@studio/core/hosts"; - -export const Note = obj({ id: str(), text: str({ max: 2000 }), by: str(), at: int() }); -export type Note = Static; - -export const Notes = remote("notes", { - version: 1, - input: obj({ session: str() }), // the key: { session: "S1" } - placement: ({ session }) => Session({ id: session }), // input → the host's address - variants: { - editor: { - state: { notes: obj({ items: arr(Note) }) }, - methods: { - add: method(obj({ text: str({ min: 1, max: 2000 }) }), str()), // returns the new id - remove: method(obj({ id: str() })), // no result - }, - }, - }, -}); -``` - -`remote` means objects of this type can be reached from any host, across links (chapter IV introduces `local`). The -schema helpers (`obj`, `str`, `int`, `arr`, `map`, `opt`, `lit`, `oneOf`, `union`, `bool`, `num`, `nul`, `bytes`, -`unknown`) return ordinary TypeBox schemas; short options such as `{ min, max }` bound lengths or values. Schemas must -describe data: no callbacks, no class instances, no custom codecs. - -`method()` has four forms: `method()` (no arguments, no result), `method(args)`, `method(args, result)` and -`method(undefined, result)`. - -A contract contains no code that runs, so every host can import it. When you declare it, Chord materializes its -description and computes a **fingerprint**, a hash of its input, variants and members. Two hosts compare contracts by -version and fingerprint (chapter XVI). Placement is not part of the fingerprint. - -## Placement - -The input says *which* object; `placement` says *where* it lives. The caller runs the placement function on the input and sends the resulting address with its request; the server -routes by it. The provider runs the function too and refuses an object that is not its own with `invalid/placement`. -So a worker for S2 never builds "Notes for S1", even if a caller has an outdated rule. Because placement is a typed -function, a `Session` placement on a type whose input has no `session` field does not compile. - -Inputs are **normalized** before anything else: defaults are applied, unknown fields removed, and a canonical key -computed. `{ session: "S1" }` and `{ session: "S1", extra: 1 }` are the same object. - -## The facet and the factory - -A **facet** is a plugin's setup for one kind of host. It runs once when the plugin loads on that host and tells the -host which object types it serves: - -```ts -// notes/worker.ts -import { defineState } from "@earendil-works/chord/next"; -import { defineFacet } from "@studio/core"; // Studio's defineFacet: principals typed as Studio's P (chapter III) -import { Note, Notes } from "./contract.ts"; - -const NoteList = defineState("notes.list", { // a document (chapter V); in memory for now - schema: obj({ items: arr(Note) }), - version: 1, - initial: () => ({ items: [] }), - checkpointWhen: () => true, -}); - -export default defineFacet((host, self) => { - host.serve(Notes, ({ session }, principal, self) => { - const list = self.state(NoteList); // THE note list of this object - return { - variant: "editor", - notes: list, // a document handle is a readable: expose it as state - add: ({ text }, ctx) => - self.tx(async (tx) => { - const id = uuidv7(); - (await tx.draft(list)).items.push({ id, text, by: "someone", at: Date.now() }); - return id; - }, ctx), - remove: ({ id }, ctx) => - list.change((d) => { d.items = d.items.filter((n) => n.id !== id); }, ctx), - }; - }); -}); -``` - -Read `host.serve(Notes, factory)` as "this host builds `Notes` objects like this". The factory receives the -normalized input, the caller's principal (chapter III) and `self`, through which it declares what the object needs. -It returns the members of one variant, tagged with `variant`. - -**Factories declare; Chord activates.** The factory is synchronous and has no side effects. It does not await, open -files or start timers. `self.state(NoteList)` does not load anything: it declares that the object has this document -and returns a **handle** that stays valid for the object's whole life. Chord then performs the activation: it loads the -document, binds references, opens OS handles, starts routines, and only then lets calls in. The rule exists for -reload: a reload runs the factory again, and anything the factory did itself would happen twice. - -## Calling it - -```tsx -// notes/web.tsx -export function NotesPanel({ session }: { session: string }) { - const notes = useRef(Notes, { session }); // app-side hook over host.get / release - const items = useReadable(notes?.notes)?.items ?? []; - return ( -

    - {items.map((n) =>

    {n.text}

    )} - notes?.add({ text })} /> -
    - ); -} -``` - -Underneath the hooks is the reference API: - -```ts -const notes = await client.get(Notes, { session: "S1" }); // live and hydrated once awaited -notes.notes.value.items; // the current list -notes.notes.subscribe((v) => render(v.items)); // every committed change -const id = await notes.add({ text: "check the flaky test" }); -notes.release(); // the object may end now -``` - -An awaited reference is **live and hydrated**: every state member has a value. State members are **readables**: -`.value` is the current value, `subscribe` is called after every commit. The first `get` for `{ session: "S1" }` runs -the factory; every later `get` for the same input and caller shares that object. The object ends when the last -reference is released (a type may keep it a little longer with `linger`). - -:::sequence animate | A call to add. Arguments are checked once at the provider; the commit reaches the viewer as one small change. -actor tab "browser tab" -actor server "server" muted -actor worker "worker S1" blue -actor obj "Notes { S1 }" accent -tab -> server: call add { text } -server -> worker: relay (header only) -worker -> obj: strip + validate args -obj => obj: self.tx: push note, commit -obj --> worker: result: id -worker --> server: state: ops of the commit -worker --> server: result -server --> tab: state, then result -::: - -## Two rules for members - -**Methods return data, never objects.** A method that wants to point you to another object returns its address -(`{ session: "S2" }`), and you `get` it yourself. That keeps every reference owned by someone who releases it. - -**No callbacks as arguments.** A callback carries code, and everything it closes over, from one object into another. -Suppose an inbox object passed `onFile` to another object, which stored it. After a reload of the inbox's plugin, the -other object still calls the *old* `onFile`; after the inbox ends, it calls into an object whose file watcher is -already closed. Chord forbids this by construction: members are data (JSON plus bytes). Behaviour from elsewhere goes -through registries (chapter XIII), multi-step work through jobs (XI), input from a person through flows (XII). - -## What is wrong with this version - -This first version works for one person. It has two problems, which the next two chapters fix. Everyone gets the -`editor` variant, including a read-only guest. And, less obvious, the owner and a guest do not see the same notes: as -chapter III shows, a remote object exists per caller, so each caller's `Notes` object has its own `NoteList`. - -:::note | Built so far -`remote`, `method`, the schema helpers, `Static` and fingerprints are built (`src/next/contracts/`). Serving, `get`, -references and documents are layers L3 to L5. -::: diff --git a/packages/chord/docs/next/explainer/src/03-principals.md b/packages/chord/docs/next/explainer/src/03-principals.md deleted file mode 100644 index 52a4c1064..000000000 --- a/packages/chord/docs/next/explainer/src/03-principals.md +++ /dev/null @@ -1,190 +0,0 @@ - -# Who Is Calling {#principals lede="Principals: how Chord knows who calls, why a caller cannot lie about it, and how the answer picks the object and its members."} - -The owner may add and remove notes. Ann, a read-only guest, may only read them and should not even see who wrote -what. The object must therefore know who is calling, and the caller must not be able to claim to be someone else. In -Chord the answer is a **principal**, a small JSON value the app defines and the links stamp. - -## The app's principal type - -Chord defines no principal kinds; the app does. Every variant has a `kind`. Studio uses p2's set: - -```ts -// studio/src/principal.ts -export type P = - | { kind: "owner"; access: "local" | "relay" | "device"; device?: string } - | { kind: "guest"; session: string; mode: "read" | "write"; grant: string; revision: number } - | { kind: "server" } - | { kind: "worker"; session: string }; -``` - -| `kind` | Who | -| --- | --- | -| `owner` | the person running the app, on loopback, through the relay, or on a paired device | -| `guest` | someone invited into one session, with a mode and the grant (and its revision) that let them in | -| `server` | the server's own code calling a worker | -| `worker` | a session worker calling the server | - -## Links stamp principals - -A principal never comes from a request's data. The server authenticates a client once, when its WebSocket connects, -and stamps the result on the whole link (`server.accept(link, { principal })`). The server tells Chord who each worker -is when it resolves it (`return { link, principal }`). When the server relays a client's request to a worker, Chord -writes the client's principal into the forwarded request; a principal field coming from a client is always -overwritten, and only a trusted parent link may carry one. - -Where principals come from is app code. Studio's sharing feature creates a share link per session and mode; a guest -who opens it gets a guest record (grant `g1`) and a cookie; `authenticate(req)` turns the cookie into -`{ kind: "guest", session: "S1", mode: "read", grant: "g1", revision: 1 }` on every new connection. Chord never sees -cookies or grants, only the finished principal. - -## The caller is part of the identity - -A `remote` object exists once per **(type, input, principal key)**. The **principal key** is a string computed from -the principal; by default its canonical JSON (sorted keys). The app may pass its own `principalKey` to `createHost`, -and must pass the same function on every host, or identities split silently. - -```ts -const principalKey = (p: P) => - p.kind === "owner" ? `owner:${p.access}:${p.device ?? ""}` - : p.kind === "guest" ? `guest:${p.grant}:r${p.revision}` - : p.kind === "worker" ? `worker:${p.session}` - : p.kind; -``` - -The design wants the key to include how the client is connected and the grant's revision, so that two access paths -never share objects and a changed grant never reuses the old grant's object: - -| Caller | Principal key | `Notes { S1 }` object | -| --- | --- | --- | -| owner, tab A, local | `owner:local:` | 1 | -| owner, tab B, local | `owner:local:` | 1 (shared with tab A) | -| owner through the relay | `owner:relay:` | 2 | -| guest g1, revision 1 | `guest:g1:r1` | 3 | -| guest g1 after a change to write mode, revision 2 | `guest:g1:r2` | 4 (new) | - -## Variants - -The contract lists named sets of members; the factory receives the principal and returns the variant that principal -may use: - -```ts -// notes/contract.ts (version 1, final form for now) -variants: { - editor: { - state: { notes: obj({ items: arr(Note) }) }, - methods: { add: method(obj({ text: str({ min: 1, max: 2000 }) }), str()), remove: method(obj({ id: str() })) }, - }, - reader: { - state: { notes: obj({ items: arr(PublicNote) }) }, // no `by`: authors are hidden - }, -}, -``` - -```ts -// notes/worker.ts -host.serve(Notes, ({ session }, principal, self) => { - if (canWrite(principal, session)) return { variant: "editor", notes: …, add: …, remove: … }; - return { variant: "reader", notes: … }; -}); -``` - -A reader has no `add` at all. Chord refuses a call to a member that is not in the caller's variant, so `add` needs no -check of its own. Three rules keep variants compatible: a member name has the same kind (state or method) in every -variant that has it; a method has the same arguments and result in every variant; only state schemas may differ per -variant (the reader's notes have no authors). `remote(...)` enforces them at the type level, and at runtime for -contracts that arrive as data. - -## How a facet knows `P` - -`P` is Studio's type, not Chord's, and every host is created with it (`createHost

    `). A plugin's facets are written -and compiled separately, then handed to `host.load` as values, so the type checker cannot infer `P` from the host into -the factory's `principal`. Studio's base code therefore exports its own `defineFacet`, already bound to `P`, and every -facet of every plugin imports that one instead of Chord's: - -```ts -// studio/src/facet.ts (base code, exported as @studio/core) -import { bindFacet } from "@earendil-works/chord/next"; -import type { P } from "./principal.ts"; - -export const defineFacet = bindFacet

    (); // a facet's principals are Studio's P -export type { P, Revocation } from "./principal.ts"; -``` - -```ts -// notes/worker.ts (server.ts, web.tsx and tui.ts import the same defineFacet) -import { defineFacet } from "@studio/core"; - -export default defineFacet((host, self) => { - host.serve(Notes, ({ session }, principal, self) => { // principal: P - if (principal.kind === "guest" && principal.mode === "read") return { variant: "reader", notes: … }; - … - }); -}); -``` - -The import is cheap: `@studio/core` is a host package plugins already import at runtime (the contract imports `Session` -from it), `P` is a type-only import, and `bindFacet` is a one-line function. The host checks the pairing: `load` -accepts only facets typed for its own `P`, so a facet compiled against another app's principal type, or one built with -Chord's untyped `defineFacet`, does not compile into Studio's start code. Client facets (`web.tsx`, `tui.ts`) use the -same `defineFacet`; they rarely see a principal, because clients serve only `local` types. - -None of this is checked at runtime, and none of it needs to be: the principal a factory receives is the one the link -stamped. - -:::review | Review: binding facets to the app's principal type (open) -The design shows `defineFacet((host, self) => …)` without saying how `P` reaches the factory. The frozen kernel takes -it as an explicit type parameter, `defineFacet

    (…)`, defaulting to `{ kind: string }` -(`packages/chord/src/next/facet.ts`), and the playground writes `defineFacet(…)`. The pre-bound -`defineFacet` above (`bindFacet` is a working name) is the owner's preferred direction, recorded as an open point in -`packages/chord/docs/next/layer-1.md` §4. -::: - -## Narrowing on the caller's side - -Owner and guest run the same `web.tsx`, and which variant they get is known only at runtime. The types force every -caller to narrow before using variant members: - -```ts -const notes = await client.get(Notes, { session: "S1" }); -notes.add({ text: "hi" }); // type error: `add` exists only in "editor" -if (inVariant(notes, "editor")) await notes.add({ text: "hi" }); -expectVariant(notes, "editor"); // or assert (throws) -function Editor({ notes }: { notes: RefIn }) { … } -``` - -The protection is not the type checker; it is Chord refusing the call at the provider. A client that ignores the -types gets the same error. - -## Admission before a worker starts - -Principals also decide whether a caller may reach a host at all. `route(Session, { authorize })` runs before Chord -resolves an address, so a guest of S1 asking for anything in S2 is refused at the server, and no worker starts for -them. Finer authority lives in the per-principal view's methods; there is no asynchronous "may this caller activate -this object" guard. An object whose address carries authority (say, a login flow by id) may exist; what a caller can -see and do is decided by the variant its factory picks. - -## Inside methods - -`ctx.principal` is the caller of the current call. In Studio's notes the editor's `add` uses it to record the -author: - -```ts -add: ({ text }, ctx) => store.add({ text, by: authorOf(ctx.principal) }, ctx), -``` - -Where principals act, in summary: - -| Purpose | Where | -| --- | --- | -| identity: which object a caller gets | principal key | -| variant: which members | the factory | -| admission: whether a host is reachable | `route.authorize` | -| attribution and fine checks | `ctx.principal` in methods | -| revocation | `revokedBy` (chapter VII) | - -## The bug from chapter II - -Now the bug is visible. The owner's `Notes { S1 }` and Ann's `Notes { S1 }` are two objects (rows 1 and 3 above), and -each declared its own `NoteList`. The owner's notes are not in Ann's list. Per-caller objects must not hold the data; -they must be views over something that exists once. That is the next chapter. diff --git a/packages/chord/docs/next/explainer/src/04-store.md b/packages/chord/docs/next/explainer/src/04-store.md deleted file mode 100644 index 44d3739ff..000000000 --- a/packages/chord/docs/next/explainer/src/04-store.md +++ /dev/null @@ -1,132 +0,0 @@ - -# One Store, Many Views {#shared-store lede="Local types, and the pattern every Chord plugin uses: shared data once in a store, per-caller views in front of it."} - -Every caller has its own `Notes` object, which is what makes per-caller variants possible. But data must exist once. -The fix is the most common pattern in Chord: a **shared store**, a `local` object that holds the data and does the -heavy work once, behind thin per-principal **views**, the `remote` objects callers get. - -## `local` types - -```ts -// notes/contract.ts -export const NoteStore = local("notes.store", { - version: 1, - input: obj({ session: str() }), - members: { - notes: obj({ items: arr(Note) }), // state: a schema - public: obj({ items: arr(PublicNote) }), - add: method(obj({ text: str(), by: str() }), str()), // methods: method(...) - remove: method(obj({ id: str() })), - }, -}); -``` - -A `local` contract has flat `members` and no variants. The differences: - -| | `remote` | `local` | -| --- | --- | --- | -| reachable from | any host, across links | its own host only | -| identity | (type, input, principal key) | (type, input) | -| principals, variants | yes | no | -| factory | `(input, principal, self)` | `(input, self)` | -| arguments | stripped to the provider's schema and validated, for every caller | see the review note below | -| values on the same host | passed by reference, never copied | passed by reference, never copied | -| `ctx.principal` | the caller | none | -| may persist documents | no (chapter V) | yes | - -A local object has no principal: it is trusted code on its own host, acting for the host. So attribution ("who wrote -this note") is passed as data (`by`), by the view that knows who called. Clients serve only local types, because -nobody can address a client. - -## The pattern - -```ts -// notes/worker.ts -export default defineFacet((host, self) => { - host.serve(NoteStore, ({ session }, self) => { - const list = self.state(NoteList); - const pub = self.derive("public", [list], ([l]) => ({ items: l.items.map(stripAuthor) })); - return { - notes: list, - public: pub, // computed once, for every reader - add: ({ text, by }, ctx) => - self.tx(async (tx) => { - const id = uuidv7(); - (await tx.draft(list)).items.push({ id, text, by, at: Date.now() }); - return id; - }, ctx), - remove: ({ id }, ctx) => list.change((d) => { d.items = d.items.filter((n) => n.id !== id); }, ctx), - }; - }); - - host.serve(Notes, ({ session }, principal, self) => { - const store = self.get(NoteStore, { session }); // a reference, declared: held for the view's life - if (canWrite(principal, session)) return { - variant: "editor", - notes: store.notes, // the store's readable, exposed as-is - add: ({ text }, ctx) => store.add({ text, by: authorOf(principal) }, ctx), - remove: (args, ctx) => store.remove(args, ctx), - }; - return { variant: "reader", notes: store.public }; - }); -}); -``` - -:::flow wide | One store, three views. The redaction runs once in the store; both readers read the same value. -row "Per-principal views (remote)" blue -box ve "Notes { S1 }" "owner · editor" blue -box vr1 "Notes { S1 }" "guest Ann · reader" blue -box vr2 "Notes { S1 }" "guest Bob · reader" blue -row "Shared store (local)" accent -gap -box store "NoteStore { S1 }" "notes, public (a derive)" accent -gap -arrow ve -> store "self.get" -arrow vr1 -> store "self.get" -arrow vr2 -> store "self.get" -note "Authority is the view's: the reader variant has no add. The store does not check; it is trusted and acts for the host." -::: - -`self.get(NoteStore, { session })` in a factory declares a **reference** that the view holds for its whole life; Chord -binds it during activation and waits until the store is live (chapter VI). The view exposes the store's readables -directly. Owner, Ann and Bob now give three `Notes` objects, one `NoteStore`, one note list and one redacted list. - -Any heavy or shared part belongs in a store like this: a pi-durable watch, a file watcher, a sandbox, a search index. -Base code uses the same pattern with the same primitives; nothing about it is special to core. - -## Derived values, briefly - -`self.derive(name, sources, fn)` computes a value from readables and recomputes it when a source changes; viewers get -only the difference. The reader's list above is one. Chapter VIII covers derives in depth, including the one footgun -that matters: `stripAuthor` must return the *same* object for an unchanged note, or every commit costs the size of the -whole list. The tool for that is `mapShared`. - -## Arguments are checked at the provider - -Trusted code is not type-checked code: plugins are bundled with esbuild without type checks, and agents write -JavaScript plugins. So the provider checks arguments itself (decision R7): every method's arguments are stripped to -the **provider's own** schema (unknown fields removed) and validated with it, whoever the caller is, and never with a -schema a peer sent. Schemas are compiled lazily on first use and cached per schema object. Bytes are checked as leaves -(`instanceof Uint8Array` plus an optional length bound), never walked. Stripping copies only when a field was removed; -the caller's value is never mutated (proposal Q12, in the L1 draft). Results and state are not validated: they come -from the provider. - -On the same host nothing is copied: arguments and results are passed by reference. Values are **immutable by -convention** (Delta's rule, as in Durable): nobody mutates a value it received, and Chord neither copies nor freezes -values to enforce it. There is no development mode that would freeze them; one behaviour everywhere (R9). - -:::review | Review: are local arguments validated? (Q15, open) -Design §5's table says `local` arguments are "neither validated nor copied"; §3 and R7 say arguments are validated -"for every caller". Review round 3 raised this as Q15 (apply R7 to `local` too, or keep `local` as the one exception), -and `layer-1.md` lists it as open. Until it is decided, write local methods so that they do not depend on either -answer: trust the shape, but never trust content (lengths, ids) that came from a remote caller without the view -checking it. `chord-next-walkthrough.md` concept 6 still describes the older rule ("validated only when the caller is -not a host principal"), which R7 replaced. -::: - -## Choosing between the two - -Use a `remote` type for what callers address: views, things with per-caller authority. Use a `local` type for what -exists once per input on one host: stores, caches, owners of OS handles, and the core objects a host provides -(chapter IX). If a type is reached across a link, it is remote; if a type persists documents, it is local (or a -host-lifetime object, chapter V). diff --git a/packages/chord/docs/next/explainer/src/05-documents.md b/packages/chord/docs/next/explainer/src/05-documents.md deleted file mode 100644 index 2a25a1caa..000000000 --- a/packages/chord/docs/next/explainer/src/05-documents.md +++ /dev/null @@ -1,258 +0,0 @@ - -# Writing It Down {#documents lede="Documents and transactions: how an object stores data, changes it atomically, survives restarts and migrates it."} - -Notes must be stored, changed safely while others read them, sent to viewers as small changes, and survive a worker -restart. An object's data lives in **documents**: named JSON values the object owns, changed only in -**transactions**. The design is a port of pi-durable's store, so most rules will look familiar; this chapter states -them for Chord. - -## Defining a document - -```ts -// notes/worker.ts -const NoteList = defineState("notes.list", { - schema: obj({ items: arr(Note) }), - version: 1, - initial: () => ({ items: [] }), - persist: true, // stored; default: memory only - checkpointWhen: (_value, _ops, info) => info.deltasSinceBase > 64, -}); - -const Draft = defineState("notes.draft", { // one unfinished note per author - schema: obj({ text: str() }), - version: 1, - initial: () => ({ text: "" }), - keyed: true, // many per object, by key - persist: true, - checkpointWhen: () => true, // small: a full base per write -}); -``` - -`checkpointWhen` is required on every definition, with no default (R14, as in Durable since pi `5c1736f82`): it says -when a commit stores a complete base instead of a delta, which bounds how much is replayed on load. Small documents -checkpoint on every write; growing ones after a number of deltas. - -A document's root is always a JSON object (lists live in a field). Documents hold JSON only; bytes are a value and wire -type, not a document type. - -## Singleton and keyed - -```ts -host.serve(NoteStore, ({ session }, self) => { - const list = self.state(NoteList); // THE note list of this object - const drafts = self.state(Draft); // all drafts of this object - … -}); -``` - -| | singleton (no `keyed`) | keyed | -| --- | --- | --- | -| how many | one per object | many per object, by a short key | -| loaded | at activation, kept in memory | on demand | -| read | `list.value` (synchronous) | `await drafts.get(key)`: the value or `undefined` | -| list | (it is one value) | `for await (const key of drafts.keys())` pages keys | -| expose as state | yes: the handle is a readable | no: expose a method or a singleton index | - -There is no live key list. A collection shown live is a singleton (an object holding a list or a map); a large -collection is a method returning pages. - -## Transactions - -All writes go through `self.tx(fn, ctx)`. Inside, `await tx.draft(handle, key?, { tags? })` returns a draft of the -current document, or a draft over `initial()` if it is absent. Drafts record operations (assignments, `push`, -`Object.assign`); asking for the same document twice in one transaction returns the same draft. `tx.retire(handle, -key?)` removes a document. There is no `tx.get`: reading is drafting, and a draft is an overlay over the current -value, so nothing is copied. A draft you do not change commits nothing, so reading never stores an absent document. - -```ts -saveDraft: ({ author, text }, ctx) => - self.tx(async (tx) => { (await tx.draft(drafts, author)).text = text; }, ctx), -``` - -Shorthands for one document: `handle.change(fn, ctx)` (keyed: `handle.change(key, fn, ctx)`) (loads the document, then runs the synchronous `fn` on -its draft) and `handle.retire(ctx)` (keyed: `handle.retire(key, ctx)`). - -### Four patterns - -Two decisions (R12 and R13, both as in Durable) shape how transactions are written. **Drafts are revoked when the -transaction settles**, so a transaction returns at most plain values, never something taken from a draft. And -**there is no existence check inside a transaction.** - -```ts -// 1. Return data: write in the transaction, read after the commit. -await self.tx(async (tx) => { (await tx.draft(list)).items.push(note); }, ctx); -return list.value.items.length; // the committed revision, or a newer one - -// 2. Remove if present: retire of an absent document is a no-op. -await drafts.retire(author, ctx); - -// 3. Create with tags if missing: the draft creates it, only if it changes. -await self.tx(async (tx) => { - (await tx.draft(drafts, author, { tags: [SessionTag(session)] })).text = text; -}, ctx); - -// 4. Branch on existence: read first, outside the transaction. -if ((await drafts.get(author)) === undefined) { … } -``` - -### Durable's rule - -Inside the callback, only `await tx.*`. No other objects, no network, no models, no processes, no second `self.tx`, no -reads past `tx`. Breaking it is not an error but a hang: the object's line is held while the callback waits. Fetch -first; decide and write inside. - -```ts -// Wrong: a fetch inside the transaction holds the object's line. -await self.tx(async (tx) => { (await tx.draft(list)).items.push({ …, title: await fetchTitle(url) }); }, ctx); -// Right: fetch, then write. -const title = await fetchTitle(url, ctx.signal); -await self.tx(async (tx) => { (await tx.draft(list)).items.push({ …, title }); }, ctx); -``` - -## What a commit does - -:::flow wide animate | The commit sequence, Durable's. Nothing is copied, frozen or revalidated; viewers get one batch per commit. -row "Inside self.tx" accent -box line "wait for the line" "per object" -box run "run the callback" "drafts record ops" accent -box prep "prepare" "Delta checks placed values" -box write "storage batch" "deltas or a base" accent -row "After the write" blue -box adopt "adopt" "pointer swap" -box local "notify locally" "once, with the ops" blue -box wire "send" "one batch per link" -box revoke "revoke drafts" "touching one throws" -arrow line -> run -arrow run -> prep -arrow prep -> write -arrow write -> adopt "stored" -arrow adopt -> local -arrow local -> wire -arrow wire -> revoke -note "An error or an abort before the storage write discards everything. Storage commits are serialized per host." -::: - -Lines are per object, never across objects; reads outside a transaction do not take the line. Subscribers on the same -host get the new value and the commit's operations by reference. Remote viewers get the commit's operations, never the -document again; chapter XVII shows how one commit touching several holds still arrives as one message. - -## Tags and deleting by tag - -Tags are typed tokens set when a document is created: - -```ts -export const SessionTag = defineTag("studio.session"); // base code, shared by every plugin -await self.tx(async (tx) => { (await tx.draft(drafts, author, { tags: [SessionTag(session)] })).text = text; }, ctx); -// when the session is deleted -await store.retireByTag(SessionTag, "S1"); // every document tagged S1 -``` - -The design names the operation `store.retireByTag(tag, value)` without saying who holds the store; session delete is -one of the app decisions still open (#18, chapter XIX). - -## Who may persist - -`persist: true` is allowed only in `local` objects and `lifetime: "host"` objects: identities without a principal. A -`remote` object's identity contains the principal key, which changes with the access path and the grant revision. Its -stored documents would scatter across access paths and be orphaned after a reconnect. The rule is enforced at the type -level (`self.state` in a remote factory accepts only non-persisted definitions) and at declaration ("persist needs a -stable identity: put the documents into a local shared store"). This is the second reason for the shared store. - -What survives a host teardown (a crash, a restart) is exactly what is persisted. In-memory documents live as long as -their object: they survive reloads and native restarts of a live object, and go when it ends. Everything else -(natives, derives, routines, references) is rebuilt on activation. - -## Facet documents - -A facet has its own identity (its plugin's name) and may hold documents too, for plugin-wide data such as -preferences. Objects the facet serves may read and write their facet's documents, and one transaction may span the -object's own documents and its facet's (lock order: own line, then the facet's). The facet never touches a served -object's documents. - -## Ownership across plugins - -No plugin reads or writes another plugin's documents, Chord or Durable. Data crosses plugins only through state and -methods. Data that core enforces lives in a format core owns and is handed to core through its API when the data is -created (as with Durable's `createConversation({ init })`), never written into core's documents. - -## Versions and migration - -A change of shape or meaning raises `version` and adds a pure `migrate(old, from)`: - -```ts -const NoteList = defineState("notes.list", { - schema: obj({ items: arr(NoteV2) }), // NoteV2 adds `pinned` - version: 2, - migrate: (old, from) => ({ items: old.items.map((n) => ({ ...n, pinned: false })) }), - initial: () => ({ items: [] }), - persist: true, - checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64, -}); -``` - -Documents migrate on access by the new definition: singletons during activation (hydration counts as access), keyed -ones on their next access. The migrated base is persisted with the next commit; right after a bump, a read-only -transaction writes it. - -Chord does not walk values against schemas. Instead, each definition computes a **schema fingerprint** once -(descriptions and titles ignored), and every stored record keeps the fingerprint it was written with. On access Chord -compares them: the same version with a different fingerprint fails the object `broken` ("schema changed without a -version bump"). So every structural change needs a bump, an additive one with a trivial `migrate`. The one exception -to "no value checks" (R8): the result of `migrate` and of `initial()` is validated against the new schema once, when -produced. A failed check, a throwing `migrate`, or old code meeting a newer document refuses the access, keeps the -stored data and fails the owning object. A changed meaning without a schema change cannot be detected; that is on -you. - -## Storage - -Behind the store is a `DocumentStorage`, passed as `createHost({ storage })`; memory by default. The server has one -shared storage, each worker one for its session. - -```ts -interface DocumentStorage { - commit(writes: readonly DocumentWrite[]): Promise; // create/change/retire, atomic - find(address: DocumentAddress): Promise; - load(id: string): Promise; // version, value, deltasSinceBase - scan(query: { owner: string; definition: string }, limit: number, cursor?: string): - Promise<{ keys: string[]; next?: string }>; - byTag(tag: string, value: string, limit: number, cursor?: string): Promise<{ ids: string[]; next?: string }>; - mintId(): Promise; -} -``` - -Chord ships a memory storage and a SQLite storage (over `@earendil-works/chord/sqlite`, the driver layer that moved -from Durable; Node and Durable Object drivers). Storages check only metadata (incarnation, version, sequence, address -occupancy, fingerprint) and write the supplied delta or base; they never rebuild or validate content on a commit. -Chord documents never commit together with Durable transcripts: Chord has its own storage, not an adapter over -Durable's. Your own storage can prove itself with `registerDocumentStorageConformance(runner, name, withStorage)` from -`@earendil-works/chord/testing`. - -Keyed persisted documents that nobody reads, writes or watches for 10 minutes leave memory and reload on the next -access. A storage error ends the store and closes the host; `onError` receives `{ kind: "storage_failed" }` and the -app decides (chapter I). - -## Kinds of state - -| Kind | For | API | -| --- | --- | --- | -| document | data the object owns | `self.state`, `self.tx` | -| derive | computed values | `self.derive` (chapter VIII) | -| source readable | external data: Durable views, files, logs | value plus ordered changes; feeds derives | -| paged method | large or queryable collections | a method returning pages (chat history) | -| plugin database | full-text search, large blobs | a native opened by its owner (chapter IX) | - -App state lives in Chord documents. Only transcript-bound data, which must fork and rewind with a conversation, lives -in Durable. - -:::review | Review: does an absent singleton read `undefined` or `initial()`? (Q11, open) -Design §9 says "whether a document exists is read outside the transaction (`handle.value` / `handle.get` give -`undefined` for an absent one)" and also "a singleton that does not exist yet reads `initial()` through its handle; -retiring resets it to `initial()`". Review round 3 proposes: singletons always read `initial()`, only absent keyed -documents read `undefined`, and singleton existence needs no API. This guide follows the proposal (`list.value` is -never `undefined`); it is not decided. -::: - -:::note | Built so far -Documents, transactions and storages are layer L3, a port of Durable's session store; not started. The frozen kernel -has a working version with a JSONL storage, which the design rejected. -::: diff --git a/packages/chord/docs/next/explainer/src/06-references.md b/packages/chord/docs/next/explainer/src/06-references.md deleted file mode 100644 index b852916b5..000000000 --- a/packages/chord/docs/next/explainer/src/06-references.md +++ /dev/null @@ -1,185 +0,0 @@ - -# Holding On {#references lede="References, the object lifecycle and ctx: what a caller sees while objects start, fail, restart and move, and why nobody writes retry code."} - -A tab shows the notes of S1 for an hour. During that hour the laptop sleeps, the worker is stopped for idleness and -started again, the notes plugin reloads twice and the store's storage hiccups once. The panel should show the notes the -whole time, say "reconnecting" when it cannot reach them, and recover by itself. In Chord this is what a -**reference** does; plugin code contains no retry loops and no request ids. - -## Three ways to hold - -| Where | API | Released | -| --- | --- | --- | -| a factory | `self.get(Type, input)` | when the object ends | -| a call | `await ctx.get(Type, input, { state? })` | when the call ends | -| outside objects | `await host.get(Type, input, { signal, state?, parkTimeout? })` | `ref.release()` | - -Use `self.get` for what an object needs its whole life (the view's store), `ctx.get` for what one call needs, and -`host.get` only in UI code, tests and entry points. `{ state: false }` binds for methods only, without receiving state. -A reference keeps its object alive while held. - -## The object lifecycle - -```text -(get) → activating → live → ending → ended - └→ failed (backoff) → activating … -``` - -**Activating:** Chord binds the declared references and waits until they are live and hydrated, loads singleton -documents, opens natives (one after another, in declaration order), computes derives and starts routines. The object -goes live when all of that succeeded. Calls wait behind its **gate** until then. - -**Ending:** routines stop, the object's calls and jobs are aborted as their rules say, natives close in reverse order, -references are released. An object with `lifetime: "referenced"` (the default) ends when its last reference goes, -after the type's `linger` if it has one. A `lifetime: "host"` object (only for types without a required input) lives -as long as the host. A running, finishing or blocked job counts as a reference (chapter XI). - -## `ref.state` - -Every reference reports what is going on as a tagged value; listeners get each new one: - -```ts -type RefState = - | { kind: "loading" } // first hold, reconnecting, restarting - | { kind: "live" } - | { kind: "failed"; cause: FailureCause; retryAt: number | undefined } // undefined: no automatic retry - | { kind: "released" }; -``` - -| Situation | Calls | Values | -| --- | --- | --- | -| link drop | park until reconnected | readable, `loading` | -| object loading or restarting | park until live | readable, `loading` | -| rollout: provider ahead or installing | park until installed | readable | -| object `failed` | reject at once with the cause | `failed`, with cause and `retryAt` | - -Values stay readable through all of this: the panel keeps showing the last notes. Parked calls share one deadline (60 -s by default, `parkTimeout` per `get`), then fail `unavailable`. Otherwise a call ends only through its caller's -signal; there is no per-call deadline. A call that was running on an instance that ends fails with an error and is -not resent: Chord cannot know whether it took effect. - -**Errors look like disconnects.** A consumer handles a failed call exactly as a dropped link: show the error, let the -user retry. Work that must not run twice carries its own idempotency key as data (Durable's `requestId`, a job's -`key`). - -:::steps | The worker restarts while a tab watches the notes -+++ Watching -The tab holds `Notes { S1 }`. `ref.state` is `live`; `notes.notes.value` has 40 notes. -+++ The worker goes away -The supervisor stops worker S1 (or it crashes). The server's IPC link to it drops. The server tells the tab that its -hold is gone; the tab's reference goes `loading`. The 40 notes stay readable. -+++ A click meanwhile -Ann's colleague clicks **Add**. The call **parks** on the reference; nothing is sent yet. The deadline is 60 s. -+++ Chord resolves again -Holds on S1 still exist, so Chord calls `route(Session).resolve({ id: "S1" })` again, with the per-address backoff. -The app forks a new worker: a new **incarnation** of `Session({ id: "S1" })`. -+++ Re-hold -The server re-holds every hold on S1 over the new link. The new worker activates `NoteStore { S1 }` (its notes load -from storage) and the views; it sends fresh snapshots. The tab's reference goes `live`. -+++ The parked call runs -The parked `add` is sent now and succeeds. The panel shows 41 notes. Nobody wrote recovery code. -::: - -## Failure and retry - -Any broken part fails the **whole** object: a half-working object could break rules that rely on its documents -changing together. Chord tags the cause by where it failed: - -```ts -type FailureCause = - | { kind: "transient"; error: ChordError } // an open threw, a dependency was unreachable: automatic backoff - | { kind: "broken"; error: ChordError }; // factory, derive, hydration or migrate threw: retried only by a trigger -``` - -**Transient** causes get a backoff per address: 1 s, doubling to 60 s; concurrent gets share one attempt; success -resets it. **Broken** causes would fail the same way again (a code bug, invalid data), so they are retried only by a -**trigger**: a plugin reload, `ref.retry()` (a Retry button), or a readable the factory declared with -`self.retryOn(...readables)`. Only an opener may declare its own failure broken, with `throw broken(error)`, for -example for invalid configuration. A failed reference stays held; on success it goes `loading`, then `live`. - -```ts -host.serve(EnvConnection, ({ env }, self) => { - const config = self.get(EnvConfig, {}); // a facet or registry value, outlives the attempt - self.retryOn(config.envs); // a config change retries a broken open - const ssh = self.open(async ({ signal }) => { - const c = config.envs.value[env]; - if (!c) throw broken(new Error(`unknown environment ${env}`)); - return connect(c, signal); - }, (s) => s.close()); - … -}); -``` - -`retryOn` sources must live independently of the attempt (facet documents, registry values); a member of one of the -object's own references is a type error, since it goes away with the attempt. - -## Waiting for dependencies - -An object whose references are not live yet stays `loading` and goes live the moment they are, bounded by -`readyTimeout` (60 s). The decision for the failure case (R18) is what makes recovery automatic: - -:::sequence animate | Dependency recovery. The view fails transient with the store's cause, keeps its reference, and is retried the moment the store goes live. -actor tab "tab" -actor view "Notes { S1 }" blue -actor store "NoteStore { S1 }" accent -tab -> view: hold -view -> store: self.get (activation) -store -> store: open fails: disk not mounted -store --> view: failed (transient) -view --> tab: failed: transient, cause inside -note view..store: the view keeps its reference; its state is the retry trigger -store -> store: backoff retry: open succeeds -store --> view: live -view -> view: retried at once (backoff cancelled) -view --> tab: live -::: - -A dependency that fails, or does not become live in time, fails the dependent **transient** with the dependency's -cause inside, never broken: the cause is the dependency's. The dependent keeps its references while failed, and Chord -uses their state as a retry trigger. Factory authors write `self.get` and know none of this. After an object is live, -a dependency leaving `live` does not change it: its calls that use the dependency get the dependency's error, or park -while it loads. - -**Cycles are rejected** (Q5). If A's activation gets B and B's gets A, neither would ever lose its last reference. -Every object instance has a uuidv7 instance id; on one host Chord follows the waiting edges of activating objects, and -across hosts a hold made during an activation carries the chain of instance ids. A cycle fails the activation -`broken`, naming it ("A → B → A"), and keeps no references. - -## No optional references - -`get` has no `optional` flag. When a dependency's plugin may be missing, turn the dependency around: the dependent -owns a registry and the other plugin contributes to it. Radius uses forwards from the ports plugin, but ports is -optional, so radius owns `radius.forwards` and ports contributes its forwards; without ports the table is empty and -radius works (chapter XIII). - -## `ctx` - -Every call gets a `ctx` as its last argument: - -| Field | Meaning | -| --- | --- | -| `signal` | fires when the caller cancelled or the object ended; pass it to everything you wait on | -| `principal` | the caller (remote types only) | -| `incarnation` | the calling host's incarnation, for calls that arrived over a link (chapter X) | -| `get` | references released when the call ends | -| `telemetry`, `trace(name, attributes, fn)` | the current span; a child span with a child `ctx` (chapter XVII) | -| `jobId`, `progress` | in a job's `run` (chapter XI) | - -```ts -exportText: async ({}, ctx) => { - const catalog = await ctx.get(Catalog, {}); // released when the call ends - const title = catalog.sessions.value[session]?.title ?? session; - const res = await fetch(renderUrl, { method: "POST", body: render(title, list.value), signal: ctx.signal }); - return await res.text(); -}, -``` - -There are no code versions and no request ids in `ctx`. Durable's `Context` is the same module (`chord/context`): -`ctx` extends it, so passing `ctx` into pi-durable needs no conversion. Code outside objects gets a `ctx` from -`host.root(fn, { principal?, signal? })`; inside Durable tools it uses `chord(fn, context)` (chapter XIV). - -:::note | Built so far -References, `ctx` and the lifecycle are layers L4 and L5. `ChordError` and `FailureCause` are specified in the L1 -draft (`layer-1.md`), with codes `invalid`, `contract_mismatch`, `forbidden`, `gone`, `not_ready`, `released`, -`too_large`, `unavailable` and `aborted`, and a finer `reason` such as `invalid/args`. -::: diff --git a/packages/chord/docs/next/explainer/src/07-revocation.md b/packages/chord/docs/next/explainer/src/07-revocation.md deleted file mode 100644 index 1cacf921b..000000000 --- a/packages/chord/docs/next/explainer/src/07-revocation.md +++ /dev/null @@ -1,113 +0,0 @@ - -# Taking Access Away {#revocation lede="Revocation: how removing a guest ends their objects on every host at once, and why reconnecting does not bring access back."} - -The owner removes Ann from S1. Her tab still has a WebSocket open and a `Notes { S1 }` view on worker S1. Both must end -now, on the server and on the worker, and her tab's automatic reconnect must not restore anything. Chord provides the -mechanism; the app decides who is blocked and remembers it. - -## Labels, not predicates - -The app declares, once, which **revocation labels** each principal carries. Revoking means revoking one label; -everything carrying it is hit. - -```ts -// studio/src/principal.ts -export type Revocation = { grant: string } | { device: string } | { session: string }; - -export const revokedBy = (p: P): Revocation[] => - p.kind === "guest" ? [{ grant: p.grant }, { session: p.session }] - : p.kind === "owner" && p.access === "device" && p.device ? [{ device: p.device }] - : []; - -const server = createHost({ address: Server(), principal: { kind: "server" }, revokedBy }); -// every worker is created with the same revokedBy -``` - -| Principal | Labels | -| --- | --- | -| guest Ann (grant g1) in S1 | `{ grant: "g1" }`, `{ session: "S1" }` | -| guest Bob (grant g2) in S1 | `{ grant: "g2" }`, `{ session: "S1" }` | -| owner on the paired phone | `{ device: "phone" }` | -| owner, local | none | - -| Call | Hits | -| --- | --- | -| `server.revoke({ grant: "g1" })`: remove Ann | Ann | -| `server.revoke({ session: "S1" })`: archive S1 | Ann and Bob | -| `server.revoke({ device: "phone" })`: sign out the phone | the phone | - -A label is plain data because revocation is an event that must travel from the server to the workers; a predicate -function could not be sent. - -## What `revoke` does - -`host.revoke(r)` ends every object and closes every link whose principal `p` has `r` in `revokedBy(p)` (compared by -canonical JSON; never the host's own principal), and passes `r` to its children, which apply their own `revokedBy`. - -:::sequence animate | Removing guest Ann. The label travels down; her objects end on every host; her reconnect is refused by the app's own record. -actor owner "owner tab" -actor server "server" accent -actor worker "worker S1" blue -actor ann "Ann's tab" "guest g1" muted -owner -> server: Sharing.remove({ grant: "g1" }) -server -> server: app: mark g1 blocked (document) -server => server: revoke({ grant: "g1" }) -server -> ann: close link -server => worker: revoke { grant: "g1" } -worker -> worker: end Notes { S1 } as g1 -ann -> server: reconnect -server -> server: authenticate: g1 blocked -server --> ann: refused (4401) -::: - -On worker S1, for `revoke({ grant: "g1" })`: - -| Object | Principal | Hit | -| --- | --- | --- | -| `Notes { S1 }` | owner, local | no | -| `Notes { S1 }` | guest g1 | **yes**: ends | -| `Notes { S1 }` | guest g2 | no | -| `NoteStore { S1 }` | none (local) | no | - -Ending Ann's view releases its reference to the store, which keeps serving the others. - -## Chord persists nothing here - -`revoke` cuts live access only. Lasting denial lives in the app's data (the sharing plugin's guest records), which -`authenticate` checks on every new connection. After a server restart nothing live exists to revoke; every client -authenticates again and blocked grants are refused there. - -**No re-stamping.** A changed grant (Ann moves from read to write) is a revocation of the old grant. Her link closes, -her tab reconnects and authenticates with the new grant revision, which gives a new principal key, and her references -re-hold by themselves, now as an editor. Reconnecting never restores revoked authority. - -**Revocation beats jobs** (R11). Normally a running job keeps its object alive after the last reference is released -(a closed tab does not abort an import). Revoking a principal ends its objects at once regardless; their jobs get the -abort signal and the normal close timeout. - -## Work in shared stores - -Revocation ends objects whose principal carries the label. A shared store has no principal, so work it runs *on -behalf of* someone (a question waiting for Ann's answer, a shell command a guest started) is not ended by `revoke`. -Such a store keeps the grant as data with the work and reconciles from the grant owner's live state with a routine -(chapter X): - -```ts -// in a shared store that runs work for guests -const sharing = self.get(Grants, {}); // the sharing plugin's live grants -self.routines.on(sharing.live, async (live, ctx) => { - for (const run of shell.running.value) - if (run.args.grant && !live[run.args.grant]) shell.abort(run.id); // idempotent -}); -``` - -There is no revocation hook: a hook can be missed (a crash between revoke and hook), a routine over current state -cannot. The window between the revocation and the routine's run is accepted (R10); a step started in it still runs. - -## Who does what - -| Chord | The app | -| --- | --- | -| stamps principals on links | what a grant is; share links, cookies, `authenticate` | -| keeps one object per principal key | defines `Revocation` and `revokedBy` | -| on `revoke(r)`: ends matching objects and links, passes `r` down | decides when to revoke; stores who is blocked | diff --git a/packages/chord/docs/next/explainer/src/08-derives.md b/packages/chord/docs/next/explainer/src/08-derives.md deleted file mode 100644 index 720542ef5..000000000 --- a/packages/chord/docs/next/explainer/src/08-derives.md +++ /dev/null @@ -1,114 +0,0 @@ - -# Computed Values {#derives lede="Derives, the one footgun that makes them expensive, mapShared, and presence: who is looking."} - -The reader's notes without authors, a note count, the number of people viewing: values computed from other values. -Storing copies would go stale; recomputing them on every read would be wasteful and could not be watched. A -**derive** is computed once per change and sent to viewers as a difference. - -## `self.derive` - -```ts -const count = self.derive("count", [list], ([l]) => l.items.length); -const summary = self.derive("summary", [list, store.pinned], ([l, pinned], previous) => ({ - total: l.items.length, - pinned: pinned.length, -})); -``` - -`self.derive(name, sources, (values, previous) => value)`. Sources can be documents, readables, other derives and -state members of references. The value is recomputed when a source changes and diffed into operations for the wire. -Derived values are adopted before ordinary observers run and share their source commit's wire batch, so a viewer -never sees a note list and a count from different commits. - -`previous` is the derive's last value, kept across factory re-runs (reloads) by name. It is a cache, empty after a -host teardown, so a derive must be correct without it. A derive that throws fails its whole owner `broken`. - -## How the difference is found - -Chord compares the new value with the previous one using Delta's `diffRevisions`. Parts that are the **same objects** -as last time cost one `===` each. Parts that are new objects must be compared by content. - -**The footgun.** A derive that builds new objects for unchanged data makes every commit cost the size of the whole -value, per view. The reader list from chapter IV, written naively: - -```ts -self.derive("public", [list], ([l]) => ({ items: l.items.map((n) => ({ id: n.id, text: n.text, at: n.at })) })); -// one note added to 2,000: 2,001 new objects, every one compared by content -``` - -**`mapShared(f)`** keeps one result per source element: `f` runs only for new or changed elements, unchanged ones -return their previous result object. - -```ts -import { mapShared } from "@earendil-works/chord/delta"; - -const stripAuthor = mapShared((n: Note): PublicNote => ({ id: n.id, text: n.text, at: n.at })); -self.derive("public", [list], ([l]) => ({ items: l.items.map(stripAuthor) })); -// one note added: one new object, 2,000 identical references; the diff emits one insert -``` - -:::review | Review: mapShared is planned, not available -Design §12 introduces `mapShared` as coming "from `@earendil-works/chord/delta`", but nothing in `packages/chord/src` -defines it yet, and Delta does not yet treat bytes as atomic leaves in `diffRevisions`. Design §28 lists both as the -step "Delta in pi", before the layered rebuild. Until then, keep a `Map` from source element to result yourself, which -is what `mapShared` will do. -::: - -## Cookbook - -| Case | How | -| --- | --- | -| select a field, look up by key, a small aggregate | compute it; the value is small | -| filter or map a list | `mapShared` over the elements | -| redaction | preferably separate documents per visibility; otherwise `mapShared`, computed once in the shared store, never per view | -| join two sources | `mapShared` plus the looked-up source's revision | -| a window over a large list | take a section of the source elements without rebuilding them; cache per-element measurements by identity; keep the window's start stable so changes arrive as appends | -| a large view combined from several sources | re-path the sources' operations, as Durable's `ConversationView` does; application code never interprets ops | - -**Container width** (R24): structural sharing is O(change) along a path, not across a container's width. Changing one -item of a flat array copies the array's slots, and `mapShared` visits every element once (one identity comparison -each). Large collections are keyed documents or bounded windows. A benchmark of long arrays is to set the documented -limit. - -## Readables that are not documents - -Three small readables round out the toolkit: - -- `constant(value)`: never changes. Start data uses it (chapter IX). -- `self.writable(initial)`: a host-local readable of any in-process value (functions included), compared by identity, - with `set`. It never goes on the wire; it exists for registry contributions that change (chapter XIII). -- Source readables from elsewhere: a Durable view or document watch is already a Chord readable, because Durable and - Chord v2 share the readable type (`chord/src/state.ts`), so it can feed a derive or be exposed as state directly. - -:::review | Review: exposing a writable (Q14, open) -A `writable` may hold functions; nothing yet stops a factory from returning one as a state member, which would reach -the wire encoder. Review round 3 proposes that only members with a JSON schema in the contract are exposable, checked -when the object is served. Undecided. -::: - -## Presence: who is looking - -`host.references(Type)` is a readable of who holds references to objects of a type on this host: input, principal and -client. Client holds carry an opaque origin id, so two tabs are two clients. - -```ts -host.serve(NoteStore, ({ session }, self) => { - const viewers = self.derive("viewers", [host.references(Notes)], ([refs]) => - refs.filter((r) => r.input.session === session).length); - return { …, viewers }; -}); -``` - -A link drop is a departure: accurate and simple. Smoothing (a phone switching networks makes the count flicker) is the -UI's job, for example a debounced count. Counts per person are derived by the app from principals and origins. There -is no consumer-side linger: a closed tab closes its link and keeps nothing. The provider's `linger` keeps a -just-closed object warm, and that also keeps the worker alive through `host.activity`. - -**"Leaving" is never derived from an unmount.** A UI reload unmounts and remounts components; a component unmounted by -a reload has not left. Leaving is explicit navigation, or presence (a dropped reference). This replaces p2's presence, -viewer and login-abandon leases with one mechanism. - -:::note | Built so far -Readables and derives are layer L2 (a readable is today's `replicatedState`, derives reuse today's derived cells over -it). Not started in the rebuild. -::: diff --git a/packages/chord/docs/next/explainer/src/09-natives.md b/packages/chord/docs/next/explainer/src/09-natives.md deleted file mode 100644 index ed7dacbbf..000000000 --- a/packages/chord/docs/next/explainer/src/09-natives.md +++ /dev/null @@ -1,160 +0,0 @@ - -# Things the OS Gives You {#natives lede="Start data, host-lifetime objects and natives: how a plugin opens a file watcher or a socket so that it is closed exactly once, also across reloads."} - -Studio's notes get an inbox: a text file dropped into the session's `inbox/` directory becomes a note. That needs a -file watcher, a **native**: an OS-level value (a file handle, a socket, a process, a database connection) that cannot -be JSON, cannot be copied, must be opened, and must be closed exactly once. It also needs the session's data -directory, which the plugin does not know. This chapter covers both. - -## Start data is an object - -A worker knows things before any plugin loads: its session id, the data directory, init data from the server's start -message. There is no special mechanism for them. Start data is an ordinary `local` object with `lifetime: "host"` and -constant state, served by base code's core facet: - -```ts -// studio/src/core/contract.ts -export const Init = obj({ id: str(), dataDir: str(), cwd: str(), env: opt(str()) }); -export const SessionInfo = local("studio.session-info", { - version: 1, - input: obj({}), - members: { init: Init }, -}); - -// studio/src/core/worker.ts -export const coreWorkerFacet = (init: Static) => defineFacet((host) => { - host.serve(SessionInfo, () => ({ init: constant(init) }), { lifetime: "host" }); - // … the Harness and other core objects (chapter XIV) -}); -``` - -Any plugin on the worker declares a reference and reads it **after activation**: in an opener, a derive, a routine or -a method, never synchronously in the factory. - -```ts -const info = self.get(SessionInfo, {}); -// later: info.init.value.id, info.init.value.dataDir -``` - -Paths are derived by the plugin from generic start data (`join(dataDir, id, "notes-inbox")`), never provided per -plugin. Platform bindings work the same way: on a Durable Object, a core object's `self.open` returns the binding -(`state`, `env`) and its close does nothing. - -## `self.open` - -```ts -const watcher = self.open( - async ({ signal }) => watchDir(path, { signal }), // open: during activation - (w, reason) => w.close(), // close: when the object ends or restarts - { closeTimeout: 10_000 }, // optional; host default 30 s -); -watcher.value; // the watcher; reading it before activation throws not_ready -``` - -`self.open` returns a handle with `.value` (type `Native`), not a proxy. The rules: - -- **Natives live only in objects.** Not at module level, not in the facet. A module-level watcher would survive a - reload with nobody to close it. -- **Order:** natives open one after another in declaration order (a later opener may read an earlier handle's - `.value`) and close in reverse. The object goes live after all opens succeeded. -- **The value never changes during an attempt.** No replacement, no versions, no stale-handle logic. To change a live - native, the object restarts. -- **A failed open** makes the object `failed`: transient with backoff by default, broken if the opener says so - (`throw broken(error)`). A partly failed activation closes the natives already open, with reason `end`; the next - attempt opens all of them anew. -- **`close` gets a reason:** `{ kind: "reload" }` (the plugin reloads and this native object restarts), `{ kind: - "stop" }` (the host shuts down) or `{ kind: "end" }` (the last reference ended, or the type was removed). Each - `close` is bounded by `closeTimeout`, then reported as a possible leak while the rest continues. A throwing `close` - does not stop the others; `host.close()` collects the errors, background ends report to `onError`. -- **Behaviour around a native goes into routines** (chapter X). Listeners on natives outside routines are forbidden. - -An object with at least one `self.open` is a **native object**; one without is a **pure object**. The difference -decides what a reload does (chapter XV): pure objects are swapped in place, native objects restart. - -## The inbox - -The inbox must run as long as the worker, whether anyone looks at the notes or not. So it is a `lifetime: "host"` -object. The facet cannot `self.get` a type it serves, and the inbox has no meaningful input, so it is an object -without input that reads its session from the start data: - -```ts -// notes/contract.ts -export const NoteInbox = local("notes.inbox", { version: 1, input: obj({}), members: {} }); - -// notes/worker.ts -host.serve(NoteInbox, (_input, self) => { - const info = self.get(SessionInfo, {}); - const watcher = self.open(async ({ signal }) => { - const { id, dataDir } = info.init.value; // read in the opener: after activation - return watchDir(join(dataDir, id, "notes-inbox"), { signal }); - }, (w) => w.close()); - - self.routines.loop("import", async (ctx) => { // chapter X - const store = await ctx.get(NoteStore, { session: info.init.value.id }); - for await (const file of watcher.value.files(ctx.signal)) { - await store.add({ text: await readFile(file, "utf8"), by: "inbox" }, ctx); - await rm(file); - } - }); - return {}; -}, { lifetime: "host" }); -``` - -Note `ctx.get(NoteStore, …)` in the routine rather than `self.get` in the factory. The factory runs before the -session id is known (start data is read after activation), so a host-lifetime object without input gets -session-scoped objects with `ctx.get` in a routine or method (R22). A `get` whose input is computed after activation is -to be added only if a real case needs it. - -## Natives with work in flight - -A restart is sequential (R16): close the old native, then open the new one. When the old one still has work running, -use one of two patterns: - -- **Drain in the background.** `close` detaches the old resource and returns at once; it disposes of it when its work - finishes, or after `closeTimeout` at most. Chord opens the new one immediately. Code mode's sandbox does this: a - script running during a reload finishes on the old sandbox (for up to 30 s, Q7; a longer one is aborted and its turn - fails), new scripts start on the new one. -- **Swap through a registry.** The resource is a registry contribution (chapter XIII). Readers always look up the - current one at use; a reader that must wait for one uses a cancellable "wait until present" with the park deadline. - -Chord aborts only the object's own calls and jobs on close. Other work that uses the native (Durable tool calls on a -sandbox) is the owner's business: `close` may drain instead of tearing down. What an old native does after its owner -restarts is also the owner's `close`: a model runtime keeps working until collected, a closed SSH connection fails its -in-flight `exec`. - -## Natives that follow a relocation - -A session can move to another directory. The inbox watcher must follow. Restarting the object on every move would be -heavy-handed; instead the native is a stable manager opened once, and a routine on the session location switches its -internal resource and bumps a generation, so late events from the old location are dropped: - -```ts -const watcher = self.open(async ({ signal }) => new SwitchableWatcher({ signal }), (w) => w.close()); -self.routines.on(location.cwd, (cwd) => watcher.value.switchTo(join(cwd, ".notes-inbox"))); -``` - -## Call-scoped natives - -A native that lives exactly as long as one call (a temporary SSH session for a directory listing, a file handle for a -download) is opened and closed inside the method with `try/finally`. There is no `ctx.open`: - -```ts -list: async ({ path }, ctx) => { - const session = await ssh.open(config, ctx.signal); - try { return await session.readDir(path, ctx.signal); } - finally { session.close(); } -}, -``` - -## Core vs. plugin - -`SessionInfo` comes from base code, `NoteInbox` from a plugin; both use the same primitives. Where code lives decides -which it is. **Base code** (the app's `src/`, imported statically by the start code under a fixed name such as -`@studio/core`) is always there, not switchable, and a change restarts the hosts. **A plugin** (its own directory, -found by listing, named by its `package.json`) is switchable and reloads. Core is what the app always needs and -everything builds on: the harness, the session list, access, the session location. Optional features are plugins; -code mode is a plugin. - -:::note | Built so far -Natives and host-lifetime objects are layer L4. `constant` is decided (R23), not built. -::: diff --git a/packages/chord/docs/next/explainer/src/10-routines.md b/packages/chord/docs/next/explainer/src/10-routines.md deleted file mode 100644 index 4191f21d0..000000000 --- a/packages/chord/docs/next/explainer/src/10-routines.md +++ /dev/null @@ -1,145 +0,0 @@ - -# Standing Behaviour {#routines lede="Routines, reconciling instead of notifying, and how two owners keep a rule between them without a shared transaction."} - -Notes older than 30 days should go every hour. Inbox files should be imported as they arrive. Notes of an archived -session should become read-only. Each of these is behaviour that stands, not a call someone makes. Written as bare -`setInterval`s and event listeners, it would keep running old code after a reload and touch state from nowhere. Chord -has one declared form for it: the **routine**. - -## Four kinds - -```ts -self.routines.every("cleanup", HOUR, (ctx) => dropOlderThan(30 * DAY, ctx)); // due time survives reloads -self.routines.on(settings, async (value, ctx) => applyLimit(value, ctx)); // a readable or event source -self.routines.loop("import", async (ctx) => { // runs until the object ends - while (!ctx.signal.aborted) await importNext(ctx); -}); -self.routines.custom("ticker", ({ signal, run }) => { // anything else - const timer = setInterval(() => run(tick), 1000); - return () => clearInterval(timer); // cleanup undoes the setup -}); -``` - -Routines start when the object goes live, stop when it ends, and restart with the new code on every reload of their -plugin. Each run gets its own `ctx`; errors are caught and reported, never uncaught. Routines are named (`on` -optionally by its source) so Chord recognizes them across reloads; that is how `every` keeps its due time. - -## Serialized per routine - -The new instance of a routine starts only after the old handler in flight has settled; only that routine waits, for at -most the routine handover timeout (10 s), after which it starts anyway and the stuck handler is reported. - -The reason, as a trace: a supervisor routine reconciles pending worker restarts. The old handler is midway through -`restart(W1)` when the plugin reloads. Without serialization the new routine starts at once, sees W1 still pending, and -restarts it a second time. - -## Reconcile, don't track - -A routine **reconciles from the current value**: it makes the world match what it sees now. It writes no "in -progress" flags, and it starts a job for long work (chapter XI). A handler that only cares about the latest value -compares with the current value itself; there is no `overlap` option. Subscribing to the object's own documents is -gapless: every commit is seen. - -No bare timers or callbacks that touch state outside routines, jobs or calls. That is what lets Chord stop all of an -object's behaviour when it ends or reloads. - -## Rules between two owners - -Every thing has exactly one owner, the smallest object it belongs to. Some rules span two: "notes of an archived -session are read-only" involves the catalog (which knows archiving) and `NoteStore` (which holds notes). Two tempting -designs are wrong: - -- **A cross-owner transaction** (archive and lock the notes atomically) would make the catalog know every plugin and - couple their storage. -- **A notification** (`catalog` calls `notes.archive(session)`) can be lost: the worker is not running, the link drops, - the process crashes between the archive and the call. - -The design's rule: **the owner of the dependent data enforces the rule** by reconciling from the other owner's state -with an idempotent routine. - -```ts -// NoteStore: an archived session's notes are read-only -const flags = self.state(NoteFlags); // { archived: boolean }, NoteStore's own document -const catalog = self.get(Catalog, {}); // the server's session list: another owner -self.routines.on(catalog.sessions, async (sessions, ctx) => { - const archived = sessions[session]?.archived ?? false; - if (flags.value.archived !== archived) await flags.change((d) => { d.archived = archived; }, ctx); -}); -// add and remove refuse while flags.value.archived -``` - -After a crash the routine runs again with the current value. The catalog does not know the notes plugin exists. The -same pattern closes a forward of an archived session in the ports plugin, aborts a job whose grant was revoked -(chapter VII), and keeps any invariant between plugins. - -## Reading without starting a worker - -The session list in the sidebar shows each session's note count. Getting `NoteStore { S1 }` for every session would -start every worker. The answer is plain publishing between two objects of one plugin, no Chord mechanism: the worker -side reports a summary to a server-side object by a method call from a routine, and clients read the server object. - -```ts -// notes/contract.ts -export const NoteCounts = remote("notes.counts", { - version: 1, input: obj({}), placement: () => Server(), - variants: { - owner: { state: { counts: map(int()) } }, - worker: { methods: { report: method(obj({ session: str(), count: int() })) } }, - }, -}); - -// notes/server.ts -const Counts = defineState("notes.counts", { - schema: obj({ bySession: map(obj({ count: int(), incarnation: str() })) }), - version: 1, initial: () => ({ bySession: {} }), persist: true, // survives a server restart - checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64, -}); - -host.serve(NoteCountStore, (_input, self) => { // local, lifetime "host" - const counts = self.state(Counts); - return { - counts: self.derive("counts", [counts], ([c]) => mapValues(c.bySession, (e) => e.count)), - report: ({ session, count, incarnation }, ctx) => - counts.change((d) => { - const last = d.bySession[session]; - if (last && last.incarnation > incarnation) return; // uuidv7: an older worker, ignore - d.bySession[session] = { count, incarnation }; - }, ctx), - }; -}, { lifetime: "host" }); - -host.serve(NoteCounts, (_input, principal, self) => { - const store = self.get(NoteCountStore, {}); - if (principal.kind === "worker") return { - variant: "worker", - report: ({ session, count }, ctx) => { - if (session !== principal.session) throw forbidden(); // a worker reports only itself - return store.report({ session, count, incarnation: ctx.incarnation }, ctx); - }, - }; - return { variant: "owner", counts: store.counts }; -}); - -// notes/worker.ts, inside NoteStore -const counts = self.get(NoteCounts, {}); // a reference to the server -self.routines.on(count, (n, ctx) => inVariant(counts, "worker") && counts.report({ session, count: n }, ctx)); -``` - -The server object keeps the latest value per session (in a document if it must survive a server restart) and -**ignores reports from an older worker incarnation** (incarnations are uuidv7, so they order by start time). A worker that was replaced may still deliver a late report; the -fence is the caller's incarnation, which Chord stamps on every call from the link handshake (R20): `ctx.incarnation`, -next to `ctx.principal`, not part of the principal or its key, never sent as data, so it cannot be forged. Session -status, subagent lists, unread marks and artifact lists in p2 all follow this pattern. The store is local and has no -caller incarnation of its own, so the view passes it on as data. - -:::review | Review: `ctx.incarnation` across a relay (Q13, open) -A worker's call to the server arrives directly, so `ctx.incarnation` is the worker's. But a call relayed through the -server (a tab's call to a worker) carries the server's incarnation on the worker's side, because the worker's -handshake is with the server. Review round 3 proposes that relays stamp the origin host's incarnation, as they stamp -the principal. Undecided; the pattern above only needs the direct case. -::: - -:::note | Built so far -Routines are layer L8. The design treats `every` as one of four forms over one runner; review round 3 proposed cutting -to `on` and `loop` only (C4), not decided. -::: diff --git a/packages/chord/docs/next/explainer/src/11-jobs.md b/packages/chord/docs/next/explainer/src/11-jobs.md deleted file mode 100644 index e506c05df..000000000 --- a/packages/chord/docs/next/explainer/src/11-jobs.md +++ /dev/null @@ -1,171 +0,0 @@ - -# Work That Outlives the Call {#jobs lede="Jobs: owned work started by a method that keeps running after its caller leaves, with cancel, dedupe and a recorded outcome."} - -Importing a 50 MB text file into notes takes a minute. The tab that started it may close; the import must go on, be -cancellable, not run twice when the tab retries after a link blip, and its result must be recorded. A method cannot do -this (it dies with its caller), a routine is standing behaviour rather than a request, and a Durable task is for work -that must resume after a crash. Chord's form for it is the **job**, built in pi-durable's sandwich: **intent** (a -transaction), **effect** (anything), **outcome** (a transaction in the current code). - -## Defining and running a job - -```ts -// notes/worker.ts -const ImportJob = defineJob("notes.import", { - version: 1, - args: obj({ path: str() }), - progress: obj({ lines: int() }), - result: obj({ added: int() }), - concurrency: 1, // one import at a time per NoteStore -}); - -const ImportResults = defineState("notes.import-result", { - schema: obj({ outcome: union([obj({ added: int() }), obj({ error: str() })]) }), - version: 1, initial: () => ({ outcome: { added: 0 } }), keyed: true, - checkpointWhen: () => true, -}); - -host.serve(NoteStore, ({ session }, self) => { - const list = self.state(NoteList); - const results = self.state(ImportResults); - const imports = self.job(ImportJob, { - run: async ({ path }, ctx) => { // 2. effect - let lines = 0; - for await (const line of readLines(path, ctx.signal)) { - await addNote({ id: `${ctx.jobId}:${lines}`, text: line, by: "import" }, ctx); // keyed by job id - ctx.progress({ lines: ++lines }); - } - return { added: lines }; - }, - finish: async (outcome, tx) => { // 3. outcome, current code - (await tx.draft(results, outcome.id)).outcome = - outcome.kind === "done" ? outcome.result : { error: outcome.error }; - }, - }); - return { - …, - imports: imports.running, // live list of runs - startImport: ({ path, requestId }, ctx) => - imports.start({ path }, ctx, { key: requestId }), // 1. intent (refined below) - abortImport: ({ id }) => imports.abort(id), - }; -}); -``` - -`defineJob(name, { version, args, progress, result, concurrency?, persist?, accepts? })`; `progress` and the outcome -are typed by the schemas. - -:::sequence animate | The job sandwich. The intent commits before anything runs; the effect runs outside any transaction; the outcome commits in the current code. -actor tab "tab" -actor store "NoteStore { S1 }" accent -actor run "run" "effect" blue -tab -> store: startImport({ path, requestId }) -store => store: 1. intent: busy check, dedupe by key, commit -store --> tab: { id } -store -> run: start after the commit -run -> run: read lines, add notes (keyed by jobId) -run --> store: progress { lines } (throttled) -note tab..store: the tab may close here: the job counts as a reference -run --> store: settles: { kind: "done", result } -store => store: 3. record outcome, finish(outcome, tx), ack -::: - -## The three steps - -**1. Intent.** `job.start(args, ctx, { key? })` is shorthand for `tx.start(job, args, { key? })` inside a -transaction, which may contain other writes. The busy check (`concurrency`) and deduplication (the same key returns the -existing run) are atomic and in the same commit; `run` starts only after the commit. Run ids are uuidv7. - -**2. Effect.** `run(args, ctx)` may do anything: processes, network, other objects. It writes no document itself; it -returns the outcome. External writes carry a dedupe key derived from `ctx.jobId` (above: note ids), so an interrupted -attempt cannot apply them twice. On a reload `run` keeps its old code. - -**3. Outcome.** When `run` settles, Chord records the outcome in its own record, then calls the **current** code's -`finish(outcome, tx)` and acknowledges the run in the same commit. The outcome is tagged: - -```ts -type Outcome = { id: string } & ({ kind: "done"; result: R } | { kind: "failed"; error: "threw" | "aborted" | "lost" }); -``` - -A throwing `finish` changes nothing and is retried with a backoff (1 s to 60 s), and after the next reload. `finish` -is optional and follows Durable's transaction rule. - -## Dedupe ends with the run - -Deduplication covers **unsettled runs only** (Q9). Once `finish` has committed, Chord drops the run's record and key -and releases its waiters' references; it keeps nothing beyond a run's lifetime. Dedupe after completion is the -owner's. Studio records which run a request started, in the same commit as the intent, and `finish` records the -result by run id (`ImportResults` above): - -```ts -const Requests = defineState("notes.import-request", { - schema: obj({ run: str() }), version: 1, initial: () => ({ run: "" }), keyed: true, persist: true, - checkpointWhen: () => true, -}); - -startImport: async ({ path, requestId }, ctx) => { - const prior = await requests.get(requestId); // read outside the transaction (R13) - if (prior) return { id: prior.run }; // started before, maybe finished long ago - return self.tx(async (tx) => { - const run = await tx.start(imports, { path }, { key: requestId }); // dedupes a concurrent retry - (await tx.draft(requests, requestId, { tags: [SessionTag(session)] })).run = run.id; - return { id: run.id }; - }, ctx); -}, -``` - -Two concurrent retries both miss `prior`, but the key makes `tx.start` return the same unsettled run. The client -watches `imports` for progress and reads the result by run id once it is gone from `running`. - -Why Chord does not keep completed runs itself: either 100,000 keyed imports would stay in memory forever, or a -retention limit would let a retried key rerun its effect. The owner knows which results matter. - -## Watching, waiting, aborting - -- **`running`** is a readable of `{ id, version, args, progress, phase }`. `phase` is `running`, `finishing` (outcome - recorded, `finish` not yet committed) or `blocked` (with the reason). `concurrency` counts only active runs. The - panel shows a progress bar from it. -- **Progress** is in memory, throttled and sent as deltas. For output, use `OutputBuffer` and `Progress` from - `@earendil-works/chord/output`: bounded windows, adaptive throttling, no write amplification. -- **Waiting:** `start` returns a run with `outcome(ctx)`. The caller's abort ends only its wait, never the run. -- **Abort is a request:** `job.abort(id)` fires the run's `ctx.signal`; the run decides what it returns (a result - that already arrived stays a success). A run that ignores the signal keeps its slot until it ends. `finish` runs in - Chord's own context. - -## Lifetime, persistence, revocation - -**Jobs keep their owner alive.** A running, finishing or blocked job counts as a reference to its object, which ends -only once all its jobs are settled. Starting an import and closing the tab does not abort it. Unsettled runs also -count as activity for the worker's keep-alive. - -**Revocation takes precedence** (R11). Revoking a principal ends its objects at once, whatever jobs remain; the jobs -get the abort signal and the normal close timeout. A job runs as its object's principal. - -**Persistence:** jobs are in memory by default and gone with the process. `persist: true` (only for stable -identities) keeps the record: a run live at a crash ends `lost` at the next activation and `finish` records it. -Nothing is rerun automatically. - -**Versions:** a run started under an older job version keeps its version; `finish` and `running` see it through pure -`accepts` adapters, and stored outcomes are never rewritten. A version bump without an adapter does not refuse the -reload: the old outcome is `blocked`, kept and reported until a later reload brings the adapter. - -**Native objects:** a job in a native object ends `aborted` when the object restarts on a reload (chapter XV). In a -pure object it keeps running on its old `run` code. - -## What is not a job - -| Work | Use | -| --- | --- | -| request and response that may die with its caller | a method | -| input into running work (a pasted code, an approval) | a flow (chapter XII) | -| work that must resume after a crash | a Durable task | -| standing behaviour | a routine | - -:::review | Review: jobs are the largest open simplification (#9, C3) -Decision #9 (deferred until login is ported): a login or a long relocation is a job whose state consumers want to -address and watch (`LoginFlow { id }`). Options: (a) jobs stay records inside their owner, watched through `running`; -(b) a job is an object of its own, addressed by its id, living until done. Separately, review round 3 proposed cut C3: -shrink jobs to a runner over documents (no job versions, `accepts`, `blocked`, `persist`) or move them out of the -kernel. Not decided. `chord-next-walkthrough.md` concept 17 still shows the older outcome shape `{ ok: true, result }`; -the design tags outcomes by `kind`. -::: diff --git a/packages/chord/docs/next/explainer/src/12-flows.md b/packages/chord/docs/next/explainer/src/12-flows.md deleted file mode 100644 index cf6a986c1..000000000 --- a/packages/chord/docs/next/explainer/src/12-flows.md +++ /dev/null @@ -1,141 +0,0 @@ - -# Asking a Person {#flows lede="Flows and the Questions service: how running work waits for an answer from someone who may be on any tab, may reconnect, and may answer twice."} - -A provider login asks for a pasted code, then an account choice. A dangerous tool call waits for the owner's -approval. The agent asks everyone present a question. In each case running work waits for a person, who may be on any -tab, may lose the connection halfway, may answer late or twice. Two patterns cover this: a **flow** kept as its -owner's state, and the **Questions** service, a shared store for questions. - -## A flow is state, not a call - -The tempting design is a long-held server call, `step()`, that blocks until the next step. A link drop fails it, and -the consumer cannot tell whether it missed a step. Chord's answer is that the owner keeps the flow as **state**, which -re-holds by itself after any drop: - -```ts -// models/contract.ts (sketch from the design) -start: method(obj({ provider: str() }), obj({ id: str() })) // uuidv7 -answer: method(obj({ id: str(), step: int(), value: str() }), Answered) // ok | "step no longer active" -cancel: method(obj({ id: str() })) - -// LoginFlow { id }: state -type FlowState = - | { kind: "running"; step?: { id: number; prompt: LoginPrompt }; notices: Notice[] } - | { kind: "done" | "failed" | "cancelled"; notices: Notice[] }; -``` - -The UI watches the state and renders the current step. A late or second answer to a step that is gone is refused. A -promise-based callback inside the work (pi-ai's `prompt` during a login) stays **inside** the owner: its job keeps the -resolver, and nothing crosses an object boundary. - -Imperative consumers (tests, scripts, a CLI, an agent tool) use a generic client helper over the same state, not a -second protocol: - -```ts -async function* steps(flow: LoginFlowRef, ctx: Ctx) { - let last = 0; - for (;;) { - const s = await until(flow.state, (v) => v.kind !== "running" || (v.step !== undefined && v.step.id > last), ctx); - if (s.kind !== "running") return s.kind; - last = s.step!.id; - yield s.step!; - } -} - -for await (const step of steps(await ctx.get(LoginFlow, { id }), ctx)) - await login.answer({ id, step: step.id, value: answerFor(step) }, ctx); -``` - -`last` keeps steps from being lost or repeated across re-holds. - -:::review | Review: where a flow lives (#11, deferred) -Decision #11, deferred until login is ported: (a) a keyed document of its owner, or (b) an object `LoginFlow { id }` -living until done; tied to #9 (jobs as objects). The `LoginFlow { id }` address above assumes (b). Also undecided is -how "leaving" ends a login (p2 cancels when the starting owner leaves everywhere); with presence it is a dropped -reference, never an unmount (chapter VIII). -::: - -## The Questions service - -The Questions service (`apps/p2/docs/questions.md`) is a p2 design, not a Chord feature, built from Chord's -primitives: a shared store with per-principal views. The asker gets `ask`; viewers get the open questions and -`answer`. - -```ts -export type Question = - | { kind: "text"; message: string; secret?: boolean; help?: string } - | { kind: "select"; message: string; options: readonly { id: string; label: string }[] } - | { kind: "confirm"; message: string; allow: string; deny: string }; - -type Policy = { mode: "first" } | { mode: "quorum"; count: number } | { mode: "all"; join?: "live" | "snapshot" }; - -ask(question, { - audience, // a readable or a list of principals - policy, - key?, // stable per asking flow: a re-ask reattaches to the pending question - deadline?, // then resolve with what is there; `missing` names who did not answer -}, ctx): Promise; -``` - -Its rules: - -- **The store is the truth.** Pending questions and their resolvers live in the store; `open` is a readable over it, - filtered per viewer. Nothing is a document, so a reload or migration cannot leave a ghost question. -- **One settlement point.** Register the waiter and the abort listener before publishing the question; an answer, an - abort or the deadline atomically takes the entry out before resolving. A late or second answer gets "no longer - active". Answers are validated against the stored question (a select id must be one of the options). Never await an - answer inside a transaction. -- **Policies over a live audience.** `first` resolves with the first answer; `quorum(n)` after n answers; `all` when - every member of the audience has answered. With `join: "live"` (the default), someone who joins becomes required and - someone who leaves drops out. -- **Authorization.** The asker is stamped by the host (a worker asks only in its own session's scope; secret questions - only from server askers). Each question stores who may answer it; a guest's send access does not imply approval - rights. Secret answers never appear in `open`, documents or logs. -- **Link drops and crashes.** A rejected `ask` is not a denial. Askers retry with the same `key` until their own signal - aborts; a pending `ask` counts as an active call for keep-alive. -- **Where it lives.** On the host where the asking work runs: session questions on the session's worker, login - questions on the server. UIs reach both through the usual relay. - -The store in Chord v2 terms: - -```ts -host.serve(QuestionStore, ({ scope }, self) => { - const pending = self.open(async () => new PendingQuestions(), (p) => p.rejectAll("store closed")); - const open = self.derive("open", [pending.value.readable], ([all]) => all); // data only, no resolvers - return { open, ask: (args, ctx) => pending.value.ask(args, ctx), answer: (args, ctx) => pending.value.answer(args, ctx) }; -}); - -host.serve(Questions, ({ scope }, principal, self) => { - const store = self.get(QuestionStore, { scope }); - if (isHost(principal)) return { variant: "asker", ask: (args, ctx) => store.ask({ ...args, asker: principal }, ctx) }; - if (!mayView(principal, scope)) throw forbidden(); - return { - variant: "viewer", - open: self.derive("mine", [store.open], ([all]) => openFor(all, principal)), - answer: ({ id, value }, ctx) => store.answer({ id, value, by: principal }, ctx), - }; -}); -``` - -The pending map with its resolvers is in-process state with promises, so it is a native of the store (opened and -closed with it). Its `close` rejects every waiter; askers retry with their key against the new store after a reload. - -:::review | Review: questions.md predates the v2 rules -`apps/p2/docs/questions.md` writes the store as `self.resource(...)` with a `replace` hand-over, which the design -rejected ("resources replaced in place"), and opens Chord call scopes in Durable hooks with `host.root`, which Q6 -replaced with `chord(fn, context)` (chapter XIV). Design §20 says the Questions store "must live in an object (it -predates natives only in objects)". The sketch above is this guide's translation, with the derive per viewer; filtering -per viewer in the store, once per variant, may be cheaper for large audiences. The `ask` signature passes typed -objects as arguments, which is fine (data), but the `audience` readable form cannot cross a link: across links, -audience must be data (principals) or computed by the store. -::: - -## Choosing - -| Case | Use | -| --- | --- | -| a multi-step login in one owner | a flow: the owner's job keeps the resolver, state shows the steps | -| a tool needs the owner's approval | Questions, `first`, audience the owner | -| the agent asks everyone present | Questions, `all`, audience from presence | -| deploy needs 2 of 3 approvals | Questions, `quorum(2)` | -| a notice (auth URL, device code, progress) | not a question: the asking work's own progress state | diff --git a/packages/chord/docs/next/explainer/src/13-registries.md b/packages/chord/docs/next/explainer/src/13-registries.md deleted file mode 100644 index 22efc7019..000000000 --- a/packages/chord/docs/next/explainer/src/13-registries.md +++ /dev/null @@ -1,185 +0,0 @@ - -# Extension Points {#registries lede="The facet, registries and the HTTP component: how plugins add behaviour to each other without callbacks, and how the owner stays in charge."} - -The notes plugin wants a download route, `GET /s/:session/notes.txt`. A logs plugin wants to time every request. -Several plugins add model providers, tools for the agent, commands for the TUI. Someone must combine these -contributions, decide conflicts, and replace one plugin's part as a whole when it reloads. Between plugins exactly two -things pass: object types and **registries**. - -## The facet, completed - -The facet has appeared since chapter II as the place that serves types. It is also itself a **pure object** with the -host's lifetime: - -```ts -// notes/server.ts -export default defineFacet((host, self) => { - const prefs = self.state(NotesPrefs); // plugin-wide documents - self.contribute(Http, { routes: { "GET /s/:session/notes.txt": exportNotes } }); - host.serve(NoteCountStore, …, { lifetime: "host" }); - host.serve(NoteCounts, …); -}); -``` - -- It may hold documents, references, routines and contributions; **no natives** (those need an object with its own - lifetime). -- Its identity is the plugin's name, so its persisted documents survive restarts and reloads. -- Served objects go live after the facet and may read and write its documents (chapter V). -- `host.serve` and `self.contribute` are valid only while the setup runs. The facet never `self.get`s a type it - serves. - -## Registries - -A registry is an extension point owned by someone (a Chord component, the app, a plugin). Each contributor -contributes **at most one value** per registry; the table is an immutable, ordered list of `(contributor, value)`; -the registry's `compose` turns it into the value consumers use. - -```ts -// owned by the HTTP component -export const Http = defineRegistry("chord.http", { - version: 2, - accepts: { 1: (old: HttpContributionV1): HttpContribution => upgrade(old) }, - compose: (table, { report }) => buildRouter(table, report), // pure; always delivers a value -}); - -self.contribute(Http, { routes: { … }, wraps: [ … ] }); // a contributor -const router = host.registry(Http).value; // a consumer: live -``` - -**`compose` runs at runtime, whenever the table changes:** after a reload's commit, when a contributor goes live or -ends, when a contribution changes. It is not a staging check: contributions can be dynamic (known only once a native -is open), so no check before a reload could be complete. `compose` always delivers a value with its fallback built in -(e.g. the first by plugin name wins) and reports conflicts through `report`, which become owner notices and telemetry -events. Readers get a plain value and never handle conflicts. Only if `compose` throws is the registry in an error -state, and readers get an error when they use it. Work that started earlier keeps the value it read. - -**Rules:** - -- **Who contributes** (R21): only facets and `lifetime: "host"` objects; only their `self` has `contribute` (a type - error elsewhere). Every object may read registries. -- **A contribution is in the table exactly while its contributor is live:** a facet's while its plugin is loaded; an - object's from `live` until it ends, fails or restarts. -- **Order** is sorted by plugin name: deterministic on every host and across reloads, and deliberately meaningless. - Meaningful order comes from the owner's composition (HTTP phases, below). Helpers: `keyed(table, entriesOf)`, - `ordered(table, itemsOf, orderOf?)`. -- **One authoritative declaration** (Q8): only the defining plugin's facet declares the registry to the host - (`compose`, version, `accepts`). Other plugins' tokens only identify it by name and types. A reload of the defining - plugin replaces the declaration atomically and the table recomposes. If the defining plugin is not loaded, the - registry has no value (readers get an error like a failed reference) and contributions wait in the table. -- **Versions** (Q8): a contribution carries the registry version its plugin was built against; older ones are adapted - by the defining facet's pure `accepts`. One without a matching version or adapter is left out and reported naming - plugin, registry and versions; its plugin keeps working. Contributions are in-process values (often functions) and - are not schema-checked. -- **Use sites guard contributions.** A contribution is untrusted code (third-party plugins are a goal). Every use site - catches throws and rejections when calling an entry, reports naming the contributor, and continues with the others. -- **The owner decides** conflicts, order and selection. Registries add behaviour; they never replace an owner's checks. - -## Natives through registries - -Registries are the only way a native reaches another plugin. An entry lives exactly as long as its contributor; -consumers look up at each use (or follow the composed value) and keep nothing from an entry beyond a use. The models -plugin's host-lifetime object contributes its runtime; the harness looks it up at every request. After a models reload -the next request gets the new runtime, while a running stream finishes on the old one. - -A handler that stores `const r = entry()` in a module variable and uses it for days has a bug: after the contributor -restarts it holds a closed native. - -## Contributions that change - -Some contributions are known only at runtime: MCP tools appear when a server connects. A contribution is either fixed -for its contributor's life (dynamic parts behind handles read at use) or a **readable**, which Chord subscribes to; -on every change Chord replaces that entry and runs `compose` again. For in-process values, `self.writable(initial)` is -a host-local readable of any value (no JSON, no wire, compared by identity) with `set`: - -```ts -const ext = self.writable(mcpExtension([])); // a Durable Extension, no tools yet -self.contribute(HarnessExtensions, ext); -self.routines.on(mcp.tools, (tools) => ext.set(mcpExtension(tools))); -``` - -There is no remove call. An entry leaves when its contributor ends, or after a reload whose new code no longer -contributes. A live contributor with nothing to offer sets its readable to the registry's empty value (an extension -without tools, empty routes); the table keeps exactly one entry per live contributor. - -## Optional dependencies - -Registries also replace optional references (chapter VI). Radius forwards connections to ports the ports plugin -manages, but ports is optional. So radius owns `radius.forwards` and ports contributes its forwards; without ports the -table is empty and radius works. - -## The HTTP component - -HTTP is an optional Chord component (`@earendil-works/chord/next/http`) built on a registry. A route gets a plain -request summary (method, path params, query, principal, provenance such as local or relayed, selected headers, parsed -and capped JSON bodies, uploads in chunks) and a `ctx` whose `principal` is the request's, so `ctx.get(Notes, …)` gives -exactly the view that person may see. The route returns a description; the component does the HTTP work once (ranges -with 206 and 416, HEAD, download headers with a safe filename, `nosniff`, CSP and its nonce, caching, streaming, abort -on disconnect): - -```ts -type HttpResponse = - | { kind: "json"; value: JsonValue } | { kind: "page"; title: string; body: string } - | { kind: "bytes"; name: string; mime: string; size: number; read(offset: number, length: number): Promise } - | { kind: "redirect"; to: string } | { kind: "error"; status: number }; - -// notes/server.ts -self.contribute(Http, { - routes: { - "GET /s/:session/notes.txt": async (req, ctx) => { - const notes = await ctx.get(Notes, { session: req.params.session }); // the requester's view - const text = new TextEncoder().encode(render(notes.notes.value)); - return { kind: "bytes", name: "notes.txt", mime: "text/plain", size: text.byteLength, - read: async (offset, length) => text.subarray(offset, offset + length) }; - }, - }, -}); -``` - -Routes compose with `keyed`: two contributors with one pattern is a reported conflict, and the first by plugin name -wins. **Wraps** are middleware in phases the owner defines (`observe`, `headers`, `transform`, outermost first), -within a phase in contributor order. No numeric priorities: plugins have no single registration order, so Fastify's -lifecycle phases and encapsulation are the model, not Express. - -```ts -// logs/server.ts -self.contribute(Http, { - wraps: [{ - name: "access-log", phase: "observe", scope: "all", // default scope "own": only own routes - wrap: async (req, next, ctx) => { - const t = Date.now(); - const res = await next(req); - log(req, res, Date.now() - t); - return res; - }, - }], -}); -``` - -`route` narrows a wrap to one route; a `route` that does not exist is a reported conflict. Wraps may add a security -boundary on top of an owner's checks, never instead of them. A request uses the router as it was when it started. -Request resolvers (who sent a request: cookie, loopback, relay) are hooks over the summary. - -## Channels and big bytes - -**Channels** are WebSocket byte channels (`send`, `onData`, `close`) for terminals, separate from object links and -with their own acknowledged windows, so a large state message never holds up keystrokes. - -**Big transfers are chunked by the owner,** not by Chord: bytes are a schema type (`bytes()`), carried natively -(never base64), and a 50 MB upload is a protocol of plain calls: - -```ts -beginUpload({ name, size }, ctx) → { upload } -append({ upload, offset, bytes }, ctx) // about 1 MiB per chunk; the awaited call is the backpressure -finishUpload({ upload }, ctx) → { id } -read({ id, offset, length }, ctx) → bytes -``` - -After a link drop the `append` in flight fails like any call; the writer asks the owner for the stored offset and -resumes. Unfinished uploads expire through the owner's routine. Chord has no stream type: it would need its own rules -for drops, cancellation, relays and restarts, which this offset protocol already survives. - -:::note | Built so far -Registries are layer L9. The HTTP component, channels and the chunked-transfer helper are deferred until after the -kernel and a first real proof (the harness, one tool with an approval pause, a code mode reload); until then the -playground serves its pages with plain app code. Their design stays the target. -::: diff --git a/packages/chord/docs/next/explainer/src/14-agent.md b/packages/chord/docs/next/explainer/src/14-agent.md deleted file mode 100644 index 40b69b555..000000000 --- a/packages/chord/docs/next/explainer/src/14-agent.md +++ /dev/null @@ -1,192 +0,0 @@ - -# The Agent {#agent lede="pi-durable inside Chord: the harness as a core object, models, environments and tools through registries, and how a tool reaches Chord objects and people."} - -Each Studio worker runs a pi-durable harness for its session. The harness must run for the worker's whole life, -whether anyone watches or not. Its models come from the models plugin, its execution environments from the envs -plugin, its tools from every plugin, and all of those reload independently. This chapter assumes you know pi-durable -and shows only how it plugs into Chord. - -## Shared foundations - -Durable and Chord v2 share modules: Durable imports Chord's `context`, `delta`, `json`, `output` and `sqlite`, and v2 -imports the same, so `Context`, `Draft` and `JsonValue` are one type each. A v2 readable is today's -`replicatedState` (`chord/src/state.ts`), so Durable's `viewState()` and document watches can be exposed as Chord state -members without conversion. Only plain data crosses between Durable and Chord objects. - -## The harness is a core object - -```ts -// studio/src/core/harness.ts (base code) -import { Harness as DurableHarness } from "@earendil-works/pi-durable"; - -export const ChordHostKey = createContextKey>("studio.chord-host"); -export const HarnessModels = defineRegistry("studio.harness-models", { version: 1, compose: firstModels }); -export const HarnessEnvironments = defineRegistry("studio.harness-envs", { version: 1, compose: firstEnv }); -export const HarnessExtensions = defineRegistry("studio.harness-extensions", { - version: 1, - compose: (table, { report }) => extensionSnapshot(table, report), // one Durable Extension per plugin -}); -const HarnessRuntime = defineRegistry<() => DurableHarness, () => DurableHarness>("studio.harness-runtime", { - version: 1, compose: onlyEntry, // core-private: one contributor -}); - -host.serve(Harness, (_input, self) => { - const info = self.get(SessionInfo, {}); - const harness = self.open(async ({ signal }) => { - const { id, dataDir } = info.init.value; - const base = withContextValue(ChordHostKey, host, withAbortSignal(signal, BACKGROUND_CONTEXT)); - return DurableHarness.open(await openNodeSqliteStorage(join(dataDir, id, "durable.sqlite")), { - models: () => host.registry(HarnessModels).value, // read at each use - env: (target) => host.registry(HarnessEnvironments).value(target), - settings: () => currentSettings(), - registry: durableRegistry(host.registry(HarnessExtensions)), // a RegistryReader over the composed value - }, base); - }, (h) => h.close(BACKGROUND_CONTEXT)); - self.contribute(HarnessRuntime, () => harness.value); // for core's conversation objects - return { … }; -}, { lifetime: "host" }); -``` - -- **Lifetime `"host"`:** it runs for the worker's whole life, also while nobody holds it, because Durable tasks - continue. -- **Base code:** it lives in the core facet, so a plugin reload never restarts it; only a base-code change (a host - restart) does. -- **Registries, looked up at each use:** Durable's `models` and `settings` accept functions (pi `7ec1f648e`), so each - generation gets the current value. After a models reload the next request uses the new runtime; a running stream - finishes on the old one. -- **Durable's storage is its own.** Chord documents never commit together with Durable transcripts. - -The design writes the call as `Durable.open(storage, { … })`; pi-durable's actual entry point is -`Harness.open(storage, options, context)`, used here as `DurableHarness` to keep it apart from the core object -`Harness`. - -## Start order - -At host start Chord loads every facet and waits until every facet and every `lifetime: "host"` object **other than the -harness** is live or failed: the **start barrier**. Then the registries hold the contributions of every plugin that -loaded, and the harness opens and resumes scheduling (R17). An object that must contribute at start, like the model -runtime, is therefore `lifetime: "host"`. Nothing waits for specific contributions; what is missing shows where it is -used: - -| Missing | What happens | -| --- | --- | -| a tool | its call gets an error result the model sees | -| a model | the generation reports "model unavailable" and waits, not fails, until a model appears | -| a task definition | the task stays blocked (`missing_task`) until the plugin is installed or the task is aborted | -| a plugin that failed | reported with its error; a facet in backoff contributes later and the registries recompose | - -The same holds when a session's database is copied to an instance with other plugins: missing tools, sections and -hooks are skipped, documents of missing plugins stay untouched, stored selections are never rewritten. The UI must -show blocked tasks with their reason and offer to abort them; a turn waiting on one hangs until then. - -:::review | Review: the harness exception to the barrier (C17, R17) -R17 makes the harness start after the barrier, a special case in the barrier's rule. Review round 3 proposed cut C17: -no harness-specific exception; the harness is an ordinary object the app's start code holds after `load` resolves. -Not decided. `chord-next-walkthrough.md` concept 28 still says the harness's activation *is* the start barrier, which -R17 replaced. -::: - -## Tools that call Chord - -Durable tools, hooks and sections receive Durable's `ToolExecutionApi` (or `HookApi`) and a `Context`, never a Chord -`ctx` (Q6). Core put the host into the harness's base context under `ChordHostKey`, and every phase context inherits -it. A core helper opens a Chord call scope from any Durable context: - -```ts -// studio/src/core/chord.ts -export function chord(context: Context, fn: (ctx: Ctx) => Promise): Promise { - const host = context.value(ChordHostKey); - if (!host) throw new Error("not inside the Studio harness"); - return host.root(fn, { signal: context.abortSignal, telemetry: context.telemetry }); -} -``` - -The scope runs as the host principal; attribution is data; references taken with `ctx.get` are released when `fn` -settles. There is no wrapping of tools and no app-specific `defineTool`. The notes plugin gives the agent a tool: - -```ts -// notes/worker.ts -const addNote = defineTool({ - name: "notes_add", // tools are named _ - description: "Add a note to this session's notes panel", - parameters: Type.Object({ text: Type.String({ maxLength: 2000 }) }), - execute: (args, api, context) => chord(async (ctx) => { - const info = await ctx.get(SessionInfo, {}); - const store = await ctx.get(NoteStore, { session: info.init.value.id }); - const id = await store.add({ text: args.text, by: `agent:${api.conversationId}` }, ctx); - return { output: [{ type: "text", text: `added note ${id}` }] }; - }, context), -}); - -export default defineFacet((host, self) => { - self.contribute(HarnessExtensions, defineExtension({ name: "@studio/notes", tools: [addNote] })); - … -}); -``` - -A reload of the notes plugin replaces its extension in the table; Durable pins a registry snapshot per task phase, so -a tool call already running finishes on the old code and the next phase uses the new one. - -## Tools that need a person - -Clients are not addressable, so a tool cannot call the user's tab. A tool that needs the user goes through an owner's -state: the Questions service (chapter XII). The model-facing `ask_user` tool and an approval hook both call -`Questions.ask`; tabs render the open questions and answer them. - -```ts -hook(ToolTask, { - beforeTool: async (call, api, context) => { - if (!needsApproval(call)) return undefined; - const decided = await api.memo("approval", context); // a rerun does not ask again - if (decided !== undefined) return decided ? undefined : { block: "Denied" }; - const allowed = await chord(async (ctx) => { - const q = await ctx.get(Questions, { scope: { session } }); - for (;;) { - try { - const r = await q.ask( - { kind: "confirm", message: `Run ${call.name}?`, allow: "Allow", deny: "Deny" }, - { audience: approvers, policy: { mode: "first" }, key: api.taskId }, - ctx, - ); - return r.answer; - } catch (error) { - if (ctx.signal.aborted) throw error; // a link blip is not a denial: retry - await backoff(ctx); - } - } - }, context); - await api.memo("approval", allowed, context); - return allowed ? undefined : { block: "Denied by the user" }; - }, -}); -``` - -Durable runs `beforeTool` in the tool task's call phase; after the execute checkpoint a crash retries execution -without hooks, so the stored decision covers that retry. The approval must cover the final arguments: put the -approval hook after hooks that rewrite arguments. - -## Conversations for the UI - -Chat panels need a conversation's live view, submit, steer, stop. The design leaves p3's conversation API to the p3 -design; one construction that follows the rules: a core `ConversationStore { conversation }` (local) opens the Durable -watch as its own native, getting the Durable harness from the core-private `HarnessRuntime` registry contribution (a -native may reach another object only through a registry entry), and exposes a bounded window as a derive. Per-principal -`Conversation { session, conversation }` views in front of it check who may send (chapter XIX). - -## Code mode - -Code mode is a plugin. It owns its `pi-codemode` sandbox as a native; its tool entry looks up the **current** sandbox -when `execute()` starts, so a call pinned by a phase before a reload (waiting in an approval hook, say) starts on the -new sandbox. On `reload` and `stop` the old sandbox is sealed and drained in `close`: running scripts finish or hit -their deadline, for at most 30 s (Q7). After a worker stop Durable replays as it does today. - -## What lives where - -| Data | Where | -| --- | --- | -| transcript-bound: entries, documents that fork and rewind with a conversation | Durable | -| app state: session list, notes, drafts, grants, settings | Chord documents | -| data core enforces on conversations (read-only one-shots, cwd followers) | a core per-conversation policy document, written when the conversation is created (`createConversation({ init })`) | - -**Telemetry:** pass the host's adapter as `HarnessOptions.telemetry`; Durable stores trace carriers on submissions and -tasks (chapter XVII). diff --git a/packages/chord/docs/next/explainer/src/15-reload.md b/packages/chord/docs/next/explainer/src/15-reload.md deleted file mode 100644 index 09ab349de..000000000 --- a/packages/chord/docs/next/explainer/src/15-reload.md +++ /dev/null @@ -1,131 +0,0 @@ - -# Reload and Rollout {#reload lede="What happens when you press Reload: per host, per object, and across the server, the workers and every tab."} - -Back to the preface. Mario changed the notes plugin's sort order and pressed Reload. Everything the earlier chapters -asked of plugin authors (synchronous factories, natives only in objects, routines instead of timers, data in -documents) exists so that this moment is safe. This chapter shows what Chord does with it. - -## On one host - -```text -1. load new bundles; an import or syntax error rejects -2. staging facet setups and every live object's new factory run with a recording self - (declarations only, nothing activated); any throw rejects -3. commit the host switches to the new code atomically -4. after per object: pure objects swap, native objects restart -``` - -**Rollback is all-or-nothing per host.** Any load or staging error leaves the whole host on its previous code, so a -host is always completely at one revision and its status is one value. Runtime states (a registry conflict, a failed -open, a migration) are not reasons to roll back; after the commit, errors stay with one object. There is no factory -trial, no migrate dry run, and no consumer contract check in staging: an incompatible consumer fails at bind with -`contract_mismatch` and a notice naming plugin and type. - -:::steps | Reloading @studio/notes on worker S1, object by object -+++ Load and stage -The new bundles load. The facet setup and the factories of `Notes` (three views), `NoteStore { S1 }` and `NoteInbox` -run with a recording `self`: they declare, nothing activates. Nothing throws, so the host commits to the new code. -+++ NoteStore: pure, swapped in place -`NoteStore` has no `self.open`. Its new methods and derives take over; `NoteList`, the drafts, its references and the -running import job stay. Routines restart, serialized: the new `cleanup` waits for the old handler in flight. The gate -opens. -+++ Running calls finish on old code -An `add` that was running when the swap happened finishes on the old code with its old handles; its transaction still -commits. Only new calls get the new handles. Exception (R15): if the reload raised a document's version, old code that -touches the migrated document throws, the document stays intact, and that call fails once. -+++ The import job keeps its old run -The job's `run` keeps the code it started with. When it settles, the **new** `finish` records the outcome. -+++ The views: pure, swapped -The three `Notes` views swap in place. Owner, Ann and Bob keep their references; the reader derive is recomputed with -the new sort and the difference is sent. Nobody sees a reconnect. -+++ NoteInbox: native, restarts -`NoteInbox` has a watcher. Its running work ends `aborted`, the watcher closes with reason `reload`, the new factory -runs, the watcher reopens. Its documents would stay. Whether a file dropped during the restart is seen is the -watcher's business: Studio's `watchDir` scans the directory once when it opens. -::: - -**Why natives restart instead of being handed over:** a native has no line, no transactions and no schema. If old and -new code shared it, they would interleave on it unchecked. In practice restarts are cheap: the Durable harness is base -code, terminal PTYs live in a separate terminal host process, and model runtimes, sandboxes and a search index rebuild -in milliseconds. Only reloading the plugin that owns SSH connections (running remote commands die) or the relay socket -(relayed connections drop) hurts. - -**Removing a type:** its objects end (routines stop, natives close, references end with an error); in-memory documents -go with them; persisted ones stay until retired. **Consumers elsewhere** see a restarting object like any object that -briefly goes away: `loading`, parked calls, then `live`. - -**UI facets reload too.** A reload unmounts and remounts components. That is why "leaving" (abort a login, close an -overlay) is never derived from an unmount (chapter VIII). - -## Across hosts - -A **revision** is (server incarnation, sequence) and names the complete desired code set: every facet with its -digest, not a batch of changes. Every install reconciles everything missing, and a host reports a revision only when it -runs that complete set. A revision also carries the code-set digest (a hash over the facet bundles' content hashes), -so a server restart with unchanged code triggers no reinstall. - -The order is **server, workers, clients**. Each host reports `installed`, `installed_failed` (with the objects that -failed) or `rejected` (with the reason; it stays on its previous code). A server that rejects stops the rollout. A -host that starts or reconnects installs the current revision before anything is routed to it. Installs are serialized -per host and bounded (60 s). - -Calls keep flowing during a rollout. When a call reaches a provider on another revision, the mismatch response -carries the provider's revision and status: - -| Provider | The call | -| --- | --- | -| ahead, or behind and installing | `updating`: values readable, calls park (with the park deadline) | -| rejected | fails at once; retried on the next revision | -| same revision | `contract_mismatch`: a real mismatch, final | - -A host checks its own install status first: if its own install was rejected, a call fails at once with the reason -instead of waiting on a provider. Calls parked on a binding that is released end with an error and are not resent. -There are no cross-host transactions and no votes. - -:::sequence animate | A tab calls into a worker that is still installing. The call parks and runs once the worker reports the revision. -actor tab "tab" "revision 8" -actor server "server" "revision 8" accent -actor worker "worker S1" "installing 8" blue -server => worker: install revision 8 -tab -> server: call Notes.add (new contract) -server -> worker: relay -worker --> server: updating (rev 7, installing) -server --> tab: updating -tab -> tab: park (deadline 60 s) -worker --> server: installed 8 -server --> tab: provider at 8 -tab -> server: call Notes.add (resent from park) -server -> worker: relay -worker --> tab: result -::: - -## Start barrier - -At host start and first load there is one more rule (chapter XIV): load every facet (the core facet with its start -data among them), wait until every facet and `lifetime: "host"` object other than the harness is live or failed, then -resume dependent work. Plugin reloads need no barrier. - -## Loading and the dev loop - -- **Roots:** plugins come from built-in, global (`/plugins`), package (npm or git, installed with - `--ignore-scripts`) and project roots, found by directory listing; no source parsing. The server selects which load. - Dependencies between plugins resolve by contract name at runtime. Names are prefixed: objects `.`, - tools `_`. -- **Loader D:** one base bundle per surface (host packages and the app runtime) plus one script per facet, with host - packages passed in (no module map). Old generations are reclaimed by the garbage collector; a stable filename per - (facet, digest), and V8's compilation cache off on hosts that reload in place. The loader is an adapter: static - imports (tests, Durable Objects) work the same, with the app passing the plugin name to `load`. -- **Dev loop:** a source watcher marks plugins dirty and never applies anything. A reload control (the playground's web - shell, a CLI, an agent tool such as p2's `reload_plugins`) reloads the changed plugins, or restarts all hosts for - base code. Reloads are always explicit. - -:::review | Review: project plugins (#14, deferred) -Per-directory plugins with trust prompts and snapshots (`apps/p2/docs/plugin-loading.md`, still the reference for -roots, trust and scopes) need their own pass over this design. Until then, treat the project root as designed but not -settled. -::: - -:::note | Built so far -Reload and rollout are layer L10, the last before telemetry. The frozen kernel has a reload path with host values and -a development freeze, both removed from the design. -::: diff --git a/packages/chord/docs/next/explainer/src/16-contracts.md b/packages/chord/docs/next/explainer/src/16-contracts.md deleted file mode 100644 index 51e0126cb..000000000 --- a/packages/chord/docs/next/explainer/src/16-contracts.md +++ /dev/null @@ -1,310 +0,0 @@ - -# Changing Contracts {#contracts lede="How a contract gets a new version while old tabs, old workers and old plugins are still talking to it."} - -Chapter XV ended with a tab that calls into a worker on another revision. That case is short-lived: the rollout -finishes and both sides run the same code. Other cases last longer. A phone kept a tab open overnight with yesterday's -web bundle. A plugin from another author was built against `Notes` version 1 and is not updated yet. In both, a -consumer holds a contract that the provider no longer has. This chapter is about what the provider does then. It is -also the one part of Chord v2 that is built: everything in the sections on `evolve`, `checkAccepts` and fingerprints -runs today in `packages/chord/src/next/contracts/`. - -## What has a version - -Everything that is stored or crosses a boundary has a version. Each surface handles an old version its own way: - -| Surface | Version | When an old one shows up | Chapter | -| --- | --- | --- | --- | -| object type (contract) | `version` plus fingerprint | `evolve` and `checkAccepts` on the provider; otherwise `contract_mismatch` | this one | -| document | `version` plus schema fingerprint | pure `migrate` on access; a schema change without a bump fails the object `broken` | V | -| job | `version` | `accepts` on outcomes and `running`; `blocked` without an adapter | XI | -| registry | `version` | `accepts` adapts contributions; one without a match is left out and reported | XIII | -| native | none | the object restarts on reload | IX | -| Durable tasks and documents | Durable's `version` | Durable's `migrate` | XIV | -| link protocol | protocol version | refused; hosts update together, a web page reloads | XVII | -| UI components, Chord and app APIs | none yet | exposed as they are; an SDK will version them later | | - -The rule behind every row is the same as for documents: **every change bumps the version**, additive ones included. -Chord never guesses compatibility from the shape of two schemas. - -## A history, not a file of versions - -A contract history starts with a **base**: a full `remote()` or `local()` declaration with an explicit `version`. The -`Notes` contract from chapters II and III is a base at version 1. Studio now wants pinned notes: notes get a `pinned` -flag and editors get a `pin` method. Instead of editing the declaration, the plugin continues the history: - -```ts -// notes/contract.ts -import { bool, evolve, nul } from "@earendil-works/chord/next"; - -export const NoteV2 = obj({ id: str(), text: str({ max: 2000 }), by: str(), at: int(), pinned: bool() }); -export const PublicNoteV2 = obj({ id: str(), text: str(), at: int(), pinned: bool() }); - -export const NotesV2 = evolve(Notes, (e) => - e.state("editor", "notes", obj({ items: arr(NoteV2) })) - .state("reader", "notes", obj({ items: arr(PublicNoteV2) })) - .addMethod("editor", "pin", obj({ id: str(), pinned: bool() }), nul()), -); -``` - -`evolve(prev, (e) => …)` appends one version to `prev`'s history: `NotesV2.version` is 2, and `NotesV2` still knows -version 1 exactly as it was declared. Every version is **materialized and fingerprinted at declaration**. That is what -makes old code safe: an old plugin bundle that contains its own copy of `Notes` version 1 computes the same -fingerprint as the history inside `NotesV2`. There is no build step, no generated file and no fingerprint copied by -hand. - -The edits: - -| Edit | Changes | Kind | -| --- | --- | --- | -| `e.state(variant, member, schema)` | one state member of one variant | remote | -| `e.addState(variant, member, schema)` / `e.removeState(variant, member)` | adds or removes a state member | remote | -| `e.state(member, schema)`, `e.addState(member, schema)`, `e.removeState(member)` | the same, flat | local | -| `e.args(name, schema)` / `e.result(name, schema)` | one side of a method | both | -| `e.method(name, args, result)` | both sides of a method | both | -| `e.addMethod(variant, name, args, result)` | adds a method (or an existing one to another variant, same signature) | remote | -| `e.addMethod(name, args, result)` | adds a method | local | -| `e.removeMethod(name)` | removes a method from every variant | both | - -Edits chain. The `method(...)` wrapper appears only in base declarations; edits take the schemas directly, so a -method without a result is added with `nul()`. The rules of chapter III still hold across versions: a method has one -signature in every variant, and a member's kind never changes, not even after it was removed and added again. - -**What `evolve` cannot do:** change the input or the set of variants. Both change what an object *is* (its identity, -who gets which members), and no adapter can turn one into the other. That needs a new history base, below. - -## The provider says what it accepts - -`NotesV2` says what changed. It does not say how version 1 consumers are served; the **provider** decides that, with -mappings that live in its own code, never in the contract. The worker facet declares them with `checkAccepts`: - -```ts -// notes/worker.ts -import { checkAccepts } from "@earendil-works/chord/next"; - -const toV1 = ({ pinned, ...note }: Static) => note; -const toPublicV1 = ({ pinned, ...note }: Static) => note; - -const notesAccepts = checkAccepts(NotesV2, { - 1: { - state: { - editor: { notes: (current) => ({ items: current.items.map(toV1) }) }, - reader: { notes: (current) => ({ items: current.items.map(toPublicV1) }) }, - }, - methods: {}, // add and remove did not change; pin is new - }, -}); -// notesAccepts.versions: [{ version: 1, fingerprint: Notes.fingerprint }] - -host.serve(NotesV2, notesFactory, { accepts: notesAccepts }); // working name for the objects-layer option -``` - -`checkAccepts` is **pure**: it checks the mappings without calling them and returns only the version metadata of the -old versions it lists. A provider that lists nothing accepts only its current version. The objects layer later -compares a consumer's version and fingerprint against this list. - -The mappings point in the direction the data flows: - -| Member in the old version | Mapping | Direction | -| --- | --- | --- | -| state, changed | `(currentMember) => oldMember` | provider's current value to what the old consumer expects | -| state, removed | `(currentVariantState) => oldMember`, or `"refuse"` | receives the whole current state of that variant | -| method arguments, changed | `args: (old) => current` | old caller's arguments to current ones | -| method result, changed | `result: (current) => old` | current result back to the old shape | -| method, changed or removed | `{ call: async (oldArgs, ctx) => oldResult }`, or `"refuse"` | replaces the method for old callers | -| member added since | nothing | an old consumer never asks for it | - -A changed state member cannot be refused: the old consumer still has that member and must get a value. A removed one -can be computed from what the variant has now, or refused. `e.method(name, args, result)` replaces both sides and -therefore needs both mappings, even if one side stayed the same (then an identity mapping). - -**Required, at type level and again at runtime.** For every member that exists in the old version and changed or was -removed since, the mapping is required by the type of `checkAccepts`. Because plugins are bundled without type -checks, the same check runs at declaration: a missing mapping, a key for a member that needs none, an unknown version -or a wrong shape throws, naming the member's path (for example `accepts[1].state.reader.notes`). An absent `state` or -`methods` section counts as empty and may be left out only when nothing there needs a mapping. - -**Conservative with no-op edits.** An edit that sets a schema to what it already was, or changes a member and then -changes it back, still requires an identity mapping. TypeScript erases constraints such as `{ max: 2000 }` from the -static type, so two schemas that look the same to the compiler may differ; Chord does not try to tell. - -### Old versions map straight to the current one - -Studio later replaces `remove` with `archive`, which keeps the note in a trash list: - -```ts -export const NotesV3 = evolve(NotesV2, (e) => - e.removeMethod("remove").addMethod("editor", "archive", obj({ id: str() }), nul()), -); - -const notesAccepts = checkAccepts(NotesV3, { - 1: { - state: { editor: { notes: (c) => ({ items: c.items.map(toV1) }) }, reader: { notes: (c) => ({ items: c.items.map(toPublicV1) }) } }, - methods: { remove: { call: async ({ id }, ctx) => { await archiveNote(id, ctx); return null; } } }, - }, - 2: { methods: { remove: { call: async ({ id }, ctx) => { await archiveNote(id, ctx); return null; } } } }, -}); -``` - -Each entry maps **one old version directly to the current one**, not through the versions in between. Version 1 -needs the state mappings (its notes have no `pinned`) and the `remove` replacement; version 2 needs only the -`remove` replacement, and leaves `state` out because its state is unchanged. A member that an old version never had -needs nothing for that version, even if it was added, changed and removed later. Dropping support for version 1 is -deleting its entry: version 1 consumers then get `contract_mismatch`. - -## Binding an old consumer - -A hold carries only the consumer's contract **version and fingerprint**, never a description of the contract (R6). -The provider decides with its own schemas: - -:::flow wide animate | What a provider does with a hold. The decision uses only the provider's own history; a peer's schema is never an input. -row "The hold arrives" -box hold "hold" "version 1, fingerprint F1" blue -box cur "equal to current?" "version and fingerprint" -box acc "accepted old version?" "F1 from the provider's own history" -row "Outcome" accent -box live "bind" "current members" -box adapted "bind adapted" "through the mappings" accent -box mismatch "`contract_mismatch`" "or updating, during a rollout" muted -arrow hold -> cur -arrow cur -> live "yes" -arrow cur -> acc "no" -arrow acc -> adapted "yes" -arrow acc -> mismatch "no" -note "A mismatch fails only that consumer's binding. The provider object and its other bindings are not affected." -::: - -Once a version 1 tab is bound, it is served as if version 1 were current: - -:::steps | A version 1 tab on a version 3 worker -+++ Bind -The tab holds `Notes { S1 }` with version 1 and fingerprint F1. The worker's history computes F1 for version 1, and -`notesAccepts` lists version 1, so the hold binds adapted. The tab gets the `editor` variant as version 1 defined it: -`notes`, `add`, `remove`. It never sees `pin` or `archive`. -+++ State through the mapping -Chord serves the adapted state like a derive of the provider's state: it applies the mapping to the current value -and keeps the result per binding. The snapshot the tab receives has no `pinned` fields. -+++ A commit -The owner pins a note from a current tab. The worker sends the current tab the draft's ops. For the version 1 tab, -Chord applies the mapping to the new value and sends the difference to its last adapted value (`diffRevisions`). The -mapping produced nothing new for a pin, so the version 1 tab receives no change at all. A large list is never resent -whole on every commit. -+++ A call -The tab calls `remove({ id })`. Its arguments are checked against the provider's own version 1 schema, then the -`call` replacement runs `archiveNote` on version 3 code. Its `null` result is what version 1 expects. Had the method -only changed, `args` would have mapped the arguments forward and `result` the result back. -+++ The worker moves on -A later revision drops the version 1 entry. When the tab re-holds (or the worker restarts and the tab binds again), -the hold gets `contract_mismatch`, and Studio's panel shows "reload to update" instead of failing silently. -::: - -## A new history base - -Some changes cannot be adapted. Studio decides that notes belong to a project, not a session: the input changes from -`{ session }` to `{ project }`. That is a new base, written in full, with an explicit version that continues the -published numbering: - -```ts -export const NotesV4 = remote("notes", { - version: 4, // explicit: the published history used 1 to 3 - input: obj({ project: str() }), - placement: () => Server, - variants: { - editor: { state: { notes: obj({ items: arr(NoteV2) }) }, methods: { add: method(…), pin: method(…), archive: method(…) } }, - reader: { state: { notes: obj({ items: arr(PublicNoteV2) }) } }, - }, -}); -// checkAccepts(NotesV4).versions is empty -``` - -A new base has no history before its version and **cannot accept older versions**, not even for an additive change. -Every consumer built against versions 1 to 3 gets `contract_mismatch` and must update. Later `evolve` calls continue -from version 4. The same applies to a change of the variant set. Do not spread an older declaration into a new one or -copy its fingerprint; the contract objects are branded and `evolve` and `checkAccepts` reject copies. - -## Local contracts - -`local` contracts (chapter IV) evolve the same way, with a flat members map and no variants: - -```ts -export const SessionInfoV2 = evolve(SessionInfo, (e) => e.state("init", InitV2)); -checkAccepts(SessionInfoV2, { 1: { state: { init: (current) => toInitV1(current) }, methods: {} } }); -``` - -Same-host consumers of a `local` type are usually reloaded together with their provider, but not always: another -plugin on the same host may be built against an older version. The rules are identical; only the variant argument is -missing. - -## Implementing a contract - -`Implementation` is the type of everything a provider must supply for the current version: one state -getter per member and variant, and one asynchronous method per method, each receiving `(args, ctx)`: - -```ts -const impl: Implementation = { - state: { editor: { notes: () => list.value }, reader: { notes: () => publicList.value } }, - methods: { add: async ({ text }, ctx) => addNote(text, ctx), remove: async ({ id }) => removeNote(id), pin: async (a) => pin(a) }, -}; -``` - -Keep declaration types **inferred** (no annotation on `NotesV2`): the inferred type carries the complete history, and -`checkAccepts` needs it to know what changed. - -:::review | Review: L1's implementation shape and the factory's return value -`Implementation` in `src/next/contracts/types.ts` is `{ state: { variant: { member: getter } }, methods }`. The -design's factories (`design.md` §6, used throughout this guide) return `{ variant, member: value, method: fn }`. -The objects layer (L4) has to connect the two, and also define how `serve` receives the checked accepts (this guide -uses the working name `{ accepts }` as a third argument) and what `"refuse"` does at runtime. The natural reading, -used in the stepper above, is that a refused method call and a hold that needs a refused state member end with -`contract_mismatch`; no source says so. -::: - -## The fingerprint - -The fingerprint is a 64-bit hash (`hash64`, portable across runtimes) of a **normal form** of the contract's -schemas. The normal form decides what counts as a change: - -| Ignored | Significant | -| --- | --- | -| order of object keys, `required` and `enum` (sorted) | order of union and intersection branches (`anyOf`, `oneOf`, `allOf`) and of tuples | -| annotations on schema nodes: `title`, `description`, `examples`, `$comment`, `deprecated` | defaults, formats, patterns, bounds, `$ref` names | -| placement | input, variants, member kinds and every member schema | - -Union order is significant because it is meaning: TypeBox picks defaults and decodes by the first matching branch. -Annotations are dropped only on schema nodes; an object property that happens to be named `title` is data and counts. - -Only data schemas are allowed. One walker copies, validates and normalizes each schema against a **whitelist of -TypeBox keywords**: primitives, arrays and tuples, objects and records, unions and intersections, `$defs` and `$ref` -(use `$ref` for recursive types), constraints and annotations. Conditional and dependent schemas, `not`, -`definitions`, `contentSchema`, `unevaluated*`, `contains`, `propertyNames`, unknown keywords, executable or custom -TypeBox features, arbitrary `Unsafe` and refinements, accessors, symbols and cyclic objects are rejected at -declaration. Bytes are the one special leaf: `bytes({ min, max })` checks a `Uint8Array` and has the data marker -`{ type: "bytes", minLength?, maxLength? }`, so it hashes like any schema; TypeBox's checking still works on it. - -Schemas are deep-copied into a private, frozen history; your own schema objects are copied, not frozen. -`NORMAL_FORM_VERSION` is part of every digest, so a change of the normal form is a link protocol change. - -**Not a security boundary.** A fingerprint says "we were compiled against the same contract"; it proves nothing. A -hostile client can send any fingerprint it likes. Providers therefore always validate arguments with their own -schemas (chapter IV), whatever the fingerprint says. - -## Rules of thumb - -- Treat a published contract like a published API: every change is `evolve`, every `evolve` needs a decision about - old consumers. -- Keep mappings small and pure. A state mapping runs for every commit and every adapted binding; make it a map over - the value, not a query. -- Accept the versions your users still run (one or two back during a normal rollout), and delete entries when the - metrics show no old bindings. -- Change the input or the variants only with a new base, and only when updating every consumer is acceptable. -- Contract, document and job versions are separate and each is decided on its own. Pinned notes needed `NotesV2` - for the wire and `NoteList` version 2 with a `migrate` that adds `pinned: false` (chapter V), but no new - `ImportJob` version, because its outcome did not change. - -:::note | Built so far -This chapter's contract layer is built: `packages/chord/src/next/contracts/` (commit `7d2bb8ea8`, about 1,300 lines -with `remote`, `local`, `method`, `evolve`, `checkAccepts`, `Implementation`, the schema helpers, `fingerprintOf`, -`schemaNormalForm` and `NORMAL_FORM_VERSION`), with nine test files covering evolution, fingerprints, the hash, -local contracts, long chains, schemas, types and review findings. It stands alone: it hosts nothing, calls no -mapping and does not import the frozen runtime. Binding by version and fingerprint, adapted state and calls belong to -the objects and protocol layers, not built yet. -::: diff --git a/packages/chord/docs/next/explainer/src/17-wire.md b/packages/chord/docs/next/explainer/src/17-wire.md deleted file mode 100644 index 67bd21118..000000000 --- a/packages/chord/docs/next/explainer/src/17-wire.md +++ /dev/null @@ -1,251 +0,0 @@ - -# On the Wire {#wire lede="What travels over a link, what a relay does with it, how big things are kept in check, what each hop costs, and how to see it in traces."} - -Most plugin code never thinks about the wire: it declares state, commits transactions and calls methods, and Chord -moves the bytes. You need this chapter when something is slow, when a value is large, when a link misbehaves, or when -you want to read a trace. It also explains why the earlier chapters insisted on small operations, shared structure and -no callbacks: each of those rules removes a cost on some hop. - -## What a state member sends - -Every exposed document, derive or readable is a **state member**. Over a hold it produces one of these: - -| Update | When | -| --- | --- | -| `snap` | on hold and on re-hold: the whole value, once | -| `ops` | per commit: the draft's recorded ops, no diff | -| `ops` from a diff | per change of a derive (and of an adapted binding, chapter XVI) | -| throttled `ops` | job progress | -| `reset` | a migration commit replaced the value (adapted for older consumers) | -| `error` | the member failed | - -**Commits are atomic for consumers.** On the same host, subscribers are notified once, after every document of the -commit is adopted. Remotely, one `state` message per hold carries a batch of member updates that is applied before -listeners run, and a commit that touches several holds on one link travels as **one** `state` message on that link. -A consumer never sees half a commit. - -## Messages - -A message is a **header plus opaque parts** (R1). There is one part per hold the message concerns; the header says -which hold each part belongs to and how large it is. - -| Direction | Kind | Parts | -| --- | --- | --- | -| consumer to provider | `hold`, `release`, `call`, `cancel` | `call` has one (the arguments) | -| provider to consumer | `bound` | one: the snapshot | -| | `state` | one per hold the commit touched on this link | -| | `result` | one | -| | `gone` | none | - -There are no request ids in application code. Hold ids and call ids exist only inside the protocol, per link. - -**One binary format on every link (Q4):** `u32 header length | header JSON | part bytes`, as one `Uint8Array`. A -WebSocket sends it as a binary frame, IPC passes it with `serialization: "advanced"`, a memory link delivers it -asynchronously. Encoding, framing, header checks, id resolution, flow control and size limits are one shared protocol -layer; link adapters only move bytes. Tests on memory links therefore run the production path. - -A part is encoded **once, at its origin**, with that hold's own Delta encoder (paths are interned per hold, end to -end between provider and consumer). A JSON part above a threshold (about 16 KiB) is compressed once at its origin, -whatever the link; byte segments stay raw; relays forward compressed parts as they are; the header names the codec. - -**The header is a TypeBox schema (R4)**, a union tagged by `kind`; the protocol types derive from it. Headers from -untrusted links (client WebSockets) are checked with the schema compiled once; trusted links (IPC between server and -workers) are not checked. **Unknown ids are ignored on both sides (Q1).** They are normal races: a `result` crossing a -`cancel`, a `call` crossing a `gone`, a state part crossing a `release`. The side that ended a hold or call settles -its own pending work: a consumer that receives `gone(h)` rejects its pending calls on `h`, and the order per -direction guarantees nothing else for `h` follows. Only a header that fails the schema is a protocol error, and it -closes that link. - -The link protocol has a version. Hosts with different protocol versions refuse each other; a web page reloads to get -the server's protocol. - -## Through a relay - -A tab never talks to a worker directly (chapter I). The server **relays**, and a relay reads and rewrites only -headers: it maps hold and call ids between its two links, stamps the principal on `hold` and `call` (a client's -principal field is always overwritten; only a trusted parent link may carry one), and groups parts per destination -link, so each link gets only its own holds' parts. It never parses a payload. - -:::sequence animate | The owner adds a note while Ann watches. The commit leaves the worker as one `state` message with two parts; the server splits it by destination without decoding either part. -actor owner "owner's tab" "hold h1" blue -actor ann "Ann's tab" "hold h2" muted -actor server "server" "relay" accent -actor worker "worker S1" "NoteStore, Notes" -owner -> server: call c7 Notes.add (1 part) -server -> server: map ids, stamp owner -server => worker: call c3 (part forwarded as is) -worker -> worker: method runs, self.tx commits -worker => server: state: parts for h1', h2' -server -> server: group parts per link -server --> owner: state (h1 part) -server --> ann: state (h2 part) -worker --> server: result c3 -server --> owner: result c7 -::: - -The `state` message reaches the owner's tab before the `result`, so when `await notes.add(…)` resolves, the new note -is already in `notes.value`. Ann's view is a different variant (reader) and gets its own part from its own derive; the -relay neither knows nor cares. - -## Flow control - -Each link has **one byte window**, counted from each message's `byteLength` (parts are already encoded bytes; nothing -is measured or stringified to count it). The window covers every message, first snapshots included. - -- **Snapshots are always admitted (R19)**, even when larger than the window; they use it up. Whether a hold - **overflows** is decided on the changes queued after the snapshot. -- **Under backpressure** a commit's ops are collected per hold and encoded into one part when credits return. The - consumer still gets the commit atomically. -- **After an overflow** the hold is dropped and established again once the previous snapshot's debt has drained - (credits back at the full window, Q3), and not more often than a minimum interval. So there is at most one snapshot - in flight per hold: no re-hold storm, and no stall for a snapshot larger than the window. -- **Calls** are throttled by call credits the provider grants (a generous window). Beyond them calls wait locally. - Nothing is refused for being "too many", and holds are not limited by count; authorization and input validation - protect them. -- **Byte-heavy traffic** such as terminals uses separate channels (chapter XIII), so a large state message never - holds up keystrokes. - -**Recovery:** after a link drops, the consumer re-holds every hold over the new link and receives snapshots; calls in -flight fail as on any disconnect, calls made meanwhile park. A relay whose window overflows releases the hold and -re-holds; the provider sends a fresh snapshot. After a routed link drops, Chord re-resolves the address (with the -per-address backoff) while holds on it exist, so a passive viewer recovers without anyone pressing anything. - -## Size limits - -Limits apply **per part** (R2, Q2), both to the compressed and the decoded size, and are checked from the sizes in the -header before anything is decoded; zlib's `maxOutputLength` enforces the decoded limit in the one decode function. - -| Situation | What fails | -| --- | --- | -| a part over the limit | only its own hold or call, with `too_large`; the other parts of the message apply | -| a snapshot over the limit | its hold ends `failed`, with no automatic re-hold | -| a WebSocket frame over `maxPayload` (e.g. 64 MiB) | the connection closes (code 1009); this is the transport's hard cap | - -Defaults for client WebSockets are 16 MiB compressed and 64 MiB decoded per part, configurable per link kind. The -decoded limit travels with relayed work: the server stamps the admitting link's limit on the header at admission -(never taken from the client), and the host that decodes enforces it. IPC therefore needs no limit of its own. - -Large data is never a large state member or a large argument. Files go through chunked transfers or the HTTP -component (chapter XIII); long lists are keyed documents or bounded windows (chapter VIII). - -## What each hop costs - -Chord v2's goal is as little compute and memory as possible while staying correct: **O(size of the change)** per -commit and per viewer, independent of the size of the value. A 500 KB value is touched as a whole only when a new -viewer takes its first snapshot. Delta, the value library under documents, was built for this, and its own primitives -are almost free: `diffRevisions` on shared subtrees takes 0.018 ms, `applyImmutable` of a small change 0.004 ms. - -| Hop | Work | -| --- | --- | -| commit | Delta's `prepare` (copies and checks only the values placed in this commit), adopt by pointer swap; storage writes the delta | -| reads in a transaction | the draft overlay, no copy; an unchanged draft writes nothing | -| subscribers on the same host | the same value and ops, by reference | -| derives | compute plus `diffRevisions`; unchanged parts stay the same objects (`mapShared`, chapter VIII) | -| send | one encoding per hold part at its origin, size taken from the encoded bytes | -| relay | header only: ids mapped, principal stamped, parts grouped; parts never parsed | -| receive | compiled header schema on untrusted links; arguments stripped and validated once at the provider (compiled schema); inputs normalized | -| apply | `applyImmutable`, structure shared | -| new hold | the value encoded once (unavoidable) | -| memory | one authoritative value per document per host; replicas and derived values share structure; one cached value per adapted binding | - -These are targets. The frozen kernel measured far worse, which is why the design is so strict here: - -:::stats -- 35 ms | one small change to a 500 KB value in the frozen kernel, no viewer -- +13 ms | per viewer on top (10 viewers: 130 ms) -- 17 / 33 / 101 ms | a tiny SQLite commit after 0 / 10 / 50 earlier deltas -::: - -Nearly all of that came from layers around Delta: whole-value validation on every access, whole-value freezing, -JSON-ness walks on every received frame, double stringification to measure a frame, snapshots cloned per viewer, and -a SQLite storage that replayed history, and schemas rebuilt for every check (a compiled TypeBox check takes -0.012 ms, an interpreted one 5.6 ms). The redesign removes each of them (chapters V, VIII and this one); the -before/after per hop is in `apps/p2/docs/p3-reviews/v2-cost-model.md`. - -:::review | Review: v2-cost-model.md is partly outdated -The measurements and the per-hop "after" targets in `apps/p2/docs/p3-reviews/v2-cost-model.md` are current. Its -closing section "Decisions this needs" is not: items 6 and 7 (validation by the caller's trust; contract descriptions -once per link and fingerprint) were replaced by R7 (validate for every caller) and R6 (no descriptions on the wire), -and §10 still describes the description fetch. -::: - -### Cost guards - -Chord reports **full-value walks, deep copies, encodes, schema validations and materializations** as ordinary -telemetry metrics, always on; with the NOOP adapter they cost an empty call where such work happens. Tests use the -in-memory adapter and assert them per hop. The reference case: one appended note to a 500 KB document with 10 viewers -does no walks, no copies, one encode per hold part at its origin and none at relays, and no validations. A -`vitest bench` suite measures the hops. - -### Ways to make it slow anyway - -| Mistake | Why it costs | Instead | -| --- | --- | --- | -| a derive that builds new objects for unchanged data | every commit diffs and sends O(value) | keep unchanged items identical: `mapShared`, `previous` (chapter VIII) | -| one long array that changes often (R24) | sharing is per path; each change copies one slot per element and visits each | keyed documents or a bounded window | -| bytes as base64 strings | inflated, parsed, compressed as text | `bytes()` leaves; chunked transfers for files | -| awaiting I/O inside `self.tx` | the object's line is blocked for every caller | do the I/O first, then a short transaction | -| storing registry entries or references in documents | stale after a reload, not data | look up the current value at use | -| opening natives at module level or in a facet | no owner: never closed, never restarted | `self.open` in an object (chapter IX) | -| callbacks as arguments | cannot cross a link, keep old code alive | registries, jobs, flows | -| one big object that everybody views | one large snapshot per viewer and per reconnect | split by viewer need; views of the same variant share one value | - -## Telemetry - -Chord reports traces, events and metrics through `@earendil-works/pi-telemetry`. Each host gets **one adapter**, -`createHost({ telemetry })`, NOOP by default; the adapter decides sampling, buffering, export and filtering. Only hosts -hold the adapter, to start root spans. Code sees spans: - -- `ctx.telemetry` is the current span (`Ctx` extends `Context`, so it is the same field Durable and pi-ai read). Every - span carries its adapter's metrics and propagation, so code with a `ctx` can record a metric or inject a carrier. -- `ctx.trace(name, attributes, fn)` runs `fn` in a child span with a child `ctx`. -- App values such as the principal stay out of Durable's context. - -**Across links**, `call` and `hold` carry the caller's trace carrier as a bounded header field (R5: limited key count -and string lengths); the provider runs the method under it, and each relay adds a `chord.relay` span. The call from the -sequence above, as a trace: - -```text -chord.call Notes.add [owner's tab] -└─ chord.relay [server] - └─ chord.method Notes.add [session:S1] principal=owner - · event chord.commit docs=2 ops=5 -``` - -Adapters trust typed input: no validation of malformed values and no try/catch around recording. Only carriers from -another process are checked (keys present, strings), and a bad one falls back to a new root. - -**Shared work** (one activation serving several `get`s, a worker start) is its own root span, linked to the trigger -that started it; each waiting span links to it. Routine runs, job runs, eviction, backoff retries and background -closes are root spans too, linked to their trigger where there is one. - -| Tier | What | -| --- | --- | -| spans | calls, methods, relays, holds, activations, opens, closes, loads, revocations, resolves, routine runs, job runs | -| events | commits inside methods, retries, parking | -| metrics | state messages, WebSocket bytes, commit durations, in-flight calls, active holds, open links, park durations, backpressure waits, the cost guards | - -Ids are attributes of spans, never of metrics. **Nothing is measured or walked for telemetry**; only sizes the code -already has are reported. Errors carry a fixed category and a message cut to 500 characters; cancelling is not an -error. Chord ships a typed telemetry schema, as pi-ai (`AI_TELEMETRY_SCHEMA`) and Durable -(`DURABLE_TELEMETRY_SCHEMA`) do. - -**Storing and reading:** `@earendil-works/chord/telemetry-sqlite` over `chord/sqlite` (Node and Cloudflare) stores one -row per settled span (events and links inline) and metrics aggregated per series per 10 s, flushes in batches every -second, keeps 7 days, and drops (and counts) beyond a bounded queue. The read interface `TelemetryStore` is shared: -`findTraces` (by name, attributes, errors, time window, with limit and cursor) returns complete traces, `getTrace` -one, `findMetrics` metrics. One renderer, `pi-telemetry/render`, prints traces as above; it prints links and never -follows them. The playground's trace viewer reads the same store. - -**Durable** already emits submit and phase spans, stores carriers on submissions and tasks (`TaskOptions.trace`), -and pi-ai emits `pi.ai.request` and `pi.ai.poll`. With the harness given the host's adapter (chapter XIV), an agent -tool that calls `Notes.add` through `chord(fn, context)` shows up as one tree from the model request down to the -commit. - -:::note | Built so far -Nothing of the protocol is built in v2: messages, relays, flow control and limits are layers L6 and L7, telemetry in -Chord is L11. Built in pi: `pi-telemetry` (propagation, links, metrics, the store interface, the renderer), -`chord/context` with `Context.telemetry`, pi-ai and Durable telemetry, and `chord/telemetry-sqlite`. Delta is built; -`mapShared` and bytes as atomic leaves in `diffRevisions` are still to do. The frozen kernel sends descriptions on the -wire and closes a link on an unknown id; both are replaced. -::: diff --git a/packages/chord/docs/next/explainer/src/18-p2-core.md b/packages/chord/docs/next/explainer/src/18-p2-core.md deleted file mode 100644 index 51925055d..000000000 --- a/packages/chord/docs/next/explainer/src/18-p2-core.md +++ /dev/null @@ -1,324 +0,0 @@ - -# Building p2: the Core {#p2-core lede="From a list of what the app must do to a set of owners: the base code every p3 plugin builds on."} - -The notes plugin used every Chord concept. A real app is the same concepts many times over. This chapter and the -next two assemble **p3**, the Chord v2 version of p2, from the parts the guide introduced. p3 is built next to p2 in -`apps/p3`, fully isolated with its own data directory, and **without migrating p2's data** (no import of -`catalog.sqlite`, `shares.sqlite` or the subagents table). Its plugins are derived from what the app must do, not -ported one to one from p2's plugin layout. - -:::review | Review: p3 is not designed yet -The design fixes the rules (`design.md` §25) and lists the p3 decisions still open (#17, #18, #20 to #23); the -per-domain design of p3 has not started. What these three chapters show is one construction that follows the rules, -derived from the functionality map in `apps/p2/docs/p3-functions/`. Object names are working names. Where a domain -has an open decision, the text says which. -::: - -## Start from functions, not from plugins - -The functionality map in `apps/p2/docs/p3-functions/` lists what p2 does, in plain words, with four facts per -function: the data and its size, how durable it is, who may see or do it, and where it must be available. It groups -the functions into 22 domains: - -| # | Domain | # | Domain | -| --- | --- | --- | --- | -| 1 | sessions | 12 | access | -| 2 | session workers | 13 | models, credentials, settings | -| 3 | conversations and agent | 14 | prompt resources | -| 4 | shell commands | 15 | status and presence | -| 5 | chat mode | 16 | search | -| 6 | agents (subagents, one-shots, peer messages) | 17 | remote access and forwards | -| 7 | code mode | 18 | logs and notices | -| 8 | session files | 19 | dev and reload | -| 9 | environments and execution | 20 | web shell | -| 10 | workspace | 21 | TUI shell | -| 11 | terminals | 22 | transport | - -The four facts answer the four questions every Chord object needs, in order: - -:::flow wide animate | From a function in the map to code. Each fact in the map answers one question Chord needs answered. -row "The map says" -box data "data" "what, how big" -box dur "durability" "ephemeral · until restart · persisted · transcript-bound" -box who "who" "owner · guest read · guest write · worker" -box where "where" "server · worker · web · TUI · environment" -row "Chord needs" accent -box owner "the owner" "smallest object it belongs to" accent -box store "where it is kept" "Durable, persisted or in-memory document, tab" -box variant "variants" "and revocation" blue -box host "host and code" "placement; core or plugin" -arrow data -> owner -arrow dur -> store -arrow who -> variant -arrow where -> host -note "Durability decides storage: transcript-bound data is Durable's; persisted app data is a persisted document with a stable identity; until-restart data is an in-memory document or a native; tab-scoped UI state stays in the client." -::: - -**One owner per thing, the smallest object it belongs to.** A guest's grant belongs to the access owner, not to the -session; a draft belongs to one (conversation, author) pair, not to the chat plugin as a whole. Ownership nests, and -rules set at creation are written in the creating commit (Durable's `createConversation({ init })`). - -**Core or plugin.** Core owners are what the app always needs and everything builds on; they are always present, so -other code may depend on them directly. Plugin owners may be missing; anything that depends on an optional feature -does so through a registry it owns itself (chapter XIII). The risk to avoid is a **core god object**: each core owner -owns only what belongs to it. - -## The ownership map - -The result of that procedure for p3, as a starting point: - -| Domain | Owner object(s) | Host | Code | -| --- | --- | --- | --- | -| access | `Access` (grants, devices, share links), `authenticate` | server | core | -| sessions | `Catalog` (session records), lifecycle jobs | server | core | -| session workers | the supervisor (`route(Session)` + idle stop) | server | core (app code) | -| session location | `SessionLocation { session }` | worker | core | -| conversations and agent | `Harness`, `ConversationStore`, `Conversation` views | worker | core | -| status and presence | `SessionStatus` (server copy), reporters on workers | both | core | -| shell commands | `ShellJob` in the chat plugin | worker | plugin | -| chat mode | chat plugin | worker | plugin | -| agents | subagents plugin (Durable child conversations) | worker | plugin | -| code mode | code mode plugin (sandbox native) | worker | plugin | -| session files | `SessionFiles { session }` (proposed, #17) | server | core or plugin, open | -| environments | envs plugin (config, SSH pools #20) | server and worker | plugin | -| workspace | workspace plugin | worker | plugin | -| terminals | terminal host process (#23), terminals plugin | separate process | plugin | -| models, credentials | models plugin, login job + flow | server | plugin | -| settings | `Settings` (single writer, #21) | server | core | -| prompt resources | prompt plugin (watchers in the session's environment) | worker | plugin | -| search | search plugin (index on the server, feed on workers) | both | plugin | -| remote access, forwards | radius plugin, ports plugin | server | plugin | -| logs and notices | `Notices` (one channel), log capture | server | core | -| dev and reload | the reload control, `reload_plugins` tool | server | plugin | -| web and TUI shells | UI facets over the objects above | clients | shells | - -The rest of this chapter builds the core rows. Chapters XIX and XX build the plugins. - -## The base code - -```text -apps/p3/ - src/ - hosts.ts Server, Session host kinds - principal.ts P, Revocation, principalKey, revokedBy, can() - server/main.ts createHost, load, route(Session), WebSocket and Radius entry points, supervisor - worker/main.ts createHost with the start data, connect to the parent - core/ the core facets: Access, Catalog, SessionStatus, Settings, Notices (server); - SessionInfo, SessionLocation, Harness, ConversationStore (worker) - plugins/ chat, subagents, codemode, envs, workspace, terminals, models, login, - prompt, search, ports, radius, files, dev -``` - -`hosts.ts`, `principal.ts`, the `SessionInfo` start data and the `Harness` object are the ones the guide already -built for Studio (chapters I, III, IX and XIV). Base code restarts the hosts when it changes; everything in -`plugins/` reloads. - -## Who is calling: authentication - -Chord stamps principals; the app produces them. p3's server has four ways in, each turned into a principal by -`authenticate(req)` when a WebSocket connects (chapter III): - -| Way in | Principal | -| --- | --- | -| loopback without a cookie (TUI, CLI, first page load) | `{ kind: "owner", access: "local" }` | -| loopback owner browser cookie (a per-port secret file) | `{ kind: "owner", access: "local" }` | -| a device cookie (HMAC-signed, per port, 30 days), obtained with a one-time sign-in token | `{ kind: "owner", access: "device", device }` | -| a guest cookie from a share link | `{ kind: "guest", session, mode, grant, revision }` | -| the Radius relay channel | `{ kind: "owner", access: "relay" }` | - -The checks that are not Chord's stay in this function: origin checks and Fetch Metadata (a browser request from -another site gets no principal, except a top-level navigation), proxy trust, the sign-in token rules (10 minutes, -single use, at most 32 outstanding, memory only), cookie expiry and revocation lists. A failed check closes the socket -before Chord sees it. - -**One permission matrix.** Every function in the map names who may use it, and p2 checks those rules in several -places. p3 writes them once, in `principal.ts`, and every factory and `authorize` uses it: - -```ts -// src/principal.ts -type Action = "read" | "send" | "upload" | "stop" | "workspace" | "terminal" | "manage"; - -export function can(p: P, action: Action, session?: string): boolean { - if (p.kind === "owner" || p.kind === "server") return true; - if (p.kind === "worker") return action !== "manage" && p.session === session; - if (p.kind !== "guest" || p.session !== session) return false; - if (action === "terminal" || action === "manage") return false; // terminals: owner only - if (p.mode === "read") return action === "read"; // no workspace for guest read - return true; // guest write: send, upload, stop, workspace -} -``` - -Factories pick variants from it (`can(p, "send", session) ? "writer" : "reader"`), so the matrix is visible in one -file and the members a caller lacks are simply not there. - -## Access: grants, devices, share links - -`Access` is the core owner of everything that lets someone in: share links per session and mode, guest records, the -device list and the signing key. It lives on the server as the chapter IV pattern: a `local` store that persists the -documents (only stable identities may), and a `remote` view for the owner's settings screen: - -```ts -// src/core/access.ts (server, in the core server facet) -const Links = defineState("access.links", { schema: map(ShareLink), version: 1, persist: true, - initial: () => ({}), checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64 }); -const Guests = defineState("access.guests", { … }); // same shape: map(Guest), persisted -const Devices = defineState("access.devices", { … }); // map(Device), persisted - -host.serve(AccessStore, (_input, self) => { // local: may persist, acts for the host - const links = self.state(Links), guests = self.state(Guests), devices = self.state(Devices); - return { - links, guests, devices, - removeGuest: async ({ grant }, ctx) => { - await self.tx(async (tx) => { (await tx.draft(guests))[grant].blocked = true; }, ctx); - host.revoke({ grant }); // live access ends now - }, - resetLink: async ({ session, mode }, ctx) => { - const blocked = await self.tx((tx) => rotateLink(tx, links, guests, session, mode), ctx); - for (const grant of blocked) host.revoke({ grant }); - }, - signOut: async ({ device }, ctx) => { - await self.tx(async (tx) => { (await tx.draft(devices))[device].revoked = true; }, ctx); - host.revoke({ device }); - }, - … // shareLink, join, signOutAll - }; -}); - -host.serve(Access, (_input, principal, self) => { // remote view: owner only - if (!can(principal, "manage")) throw forbidden(); - const store = self.get(AccessStore, {}); - return { - variant: "owner", - guests: self.derive("guests", [store.guests], ([g]) => publicGuests(g)), // names, modes; no tokens - removeGuest: (args, ctx) => store.removeGuest(args, ctx), - resetLink: (args, ctx) => store.resetLink(args, ctx), - signOut: (args, ctx) => store.signOut(args, ctx), - }; -}); -``` - -`authenticate` is base code on the same host and checks `AccessStore`'s current documents on every new connection -(it holds a reference to the store, taken with `server.root` at start). Live access and lasting denial stay separate, as chapter VII explained: - -| Event | Lasting effect (data) | Live effect | -| --- | --- | --- | -| remove a guest | guest record blocked | `revoke({ grant })` | -| reset a share link | new token; its guests' grants blocked | `revoke({ grant })` for each | -| archive a session | grants of that session void while archived | `revoke({ session })` | -| restore a session | grants valid again; revision bumped | none: guests reconnect | -| sign out a device | device revoked until its expiry | `revoke({ device })` | -| sign out all devices | signing key rotated, tokens cleared | `revoke({ device })` for each | - -A guest who rejoins with the same browser keeps its identity; a removed guest cannot mint a new one, because the -block is in `Guests` and `authenticate` refuses it. Names are unverified, cleaned, at most 40 characters, unique per -session ("Ann (2)"). - -## The catalog - -`Catalog` owns the session records: id, title (and whether it was edited), location, model default, pinned, -archived, timestamps. It lives on the server, persisted, and it **never starts a worker**: listing sessions, showing -their status or their unread marks reads server-side copies only. - -```ts -export const Catalog = remote("p3.catalog", { - version: 1, - input: obj({}), - placement: () => Server(), - variants: { - owner: { - state: { sessions: obj({ items: arr(SessionRow) }) }, - methods: { - create: method(CreateArgs, str()), rename: method(RenameArgs), pin: method(PinArgs), - archive: method(obj({ id: str() })), restore: method(obj({ id: str() })), - }, - }, - guest: { state: { sessions: obj({ items: arr(SessionRow) }) } }, // only the guest's own session, if not archived - }, -}); -``` - -Behind it sits a persisted `local` `CatalogStore`; the guest variant is a derive over it, as in chapter IV. The lifecycle operations (archive, clone, fork, -relocate, delete) are chapter XIX; here only the rule that keeps the catalog small: **the catalog does not know which -plugins exist.** Archiving a session writes `archived: true` and revokes the session; every plugin that keeps -per-session things reconciles from `catalog.sessions` with a routine (chapter X), as the notes plugin did. The ports -plugin closes forwards of archived sessions that way, also after a crash. - -## On the worker: start data, location, harness - -Three core objects exist on every worker from its start: - -- **`SessionInfo`** (chapter IX): the start message as `constant(init)`, with the session id, data directory and the - location the server chose at start. -- **`SessionLocation { session }`** (working name), lifetime host: the current cwd and environment, with a location - revision. Relocation changes it while the worker runs (chapter XIX); natives that depend on the location follow it - with a routine and a generation (chapter IX), and conversations that follow the session's cwd are listed in the core - per-conversation policy document. -- **`Harness`** (chapter XIV): the Durable harness for the session's store, one SQLite database per session, opened - under an exclusive lock keyed by the real data directory and session id. It refuses to start when the saved root - model is not configured, and resumes interrupted Durable work on start. - -`ConversationStore` and the `Conversation` views in front of it are chapter XIX. - -## Workers: start, keep alive, stop - -Starting, supervising and stopping workers is app code (chapter I); Chord only routes and counts. p3's supervisor is -`server.route(Session, { authorize, resolve })` plus a small server object that keeps per-session worker state: - -:::steps | A worker's life, as the supervisor sees it -+++ Authorize, then start -A tab gets `Notes { S1 }`. `authorize` checks `can(p, "read", "S1")` and that the session exists and is not archived -(an archived session refuses, except an index rebuild that starts the worker without resuming work). `resolve` forks -`worker/main.ts` with the start data from the catalog, bounded by a 60 s readiness timeout. -+++ A start failure sticks -If the worker fails to start (an unconfigured environment, a missing model), the error is kept per session and -every `get` fails at once with it, until the owner presses Retry. That clears the error and the related notices. -Guests see that the session is unavailable, not the error text. -+++ Counting -The core facet on the worker reports `host.activity` (`{ references, calls, jobs, lingering }`) and the harness's -running Durable task count to the supervisor object, from a routine whenever either changes. The call carries -`ctx.incarnation`, so a report from an older worker incarnation is ignored (chapter X). -+++ Idle stop -When everything has been zero for five minutes (checked every second, as in p2), the supervisor stops the worker: -the worker closes its host (routines stop, natives close, storage flushes) and exits. Durable work stays persisted -and resumes on the next start. Detached work that must keep the worker alive (a shell command whose caller left) is -a job, so it counts. -+++ Crash -If the worker exits on its own, the link drops. The supervisor records the exit code and a redacted stderr tail -(at most 4,000 characters) for that incarnation and posts one notice. References that still exist make Chord call -`resolve` again with backoff; a passive viewer recovers without a click. -+++ Shutdown -On server shutdown the supervisor stops every worker, with a forced exit after three seconds. Durable work remains -persisted. -::: - -## Status without a worker - -The session list shows per session whether it is running, busy (which conversations), its task count, usage and cost, -and an error. None of that may start a worker. The pattern is chapter X's pub/sub: a reporter on each worker (a -routine over the harness's state) calls `SessionStatus.report(summary)` on the server; `SessionStatus` keeps the latest -value per session and ignores calls whose `ctx.incarnation` is older than the newest it has seen; clients read -`SessionStatus`, which lives on the server. Children lists, unread marks, search index rows and artifact lists use the -same pattern (chapter XIX). - -## Settings and notices - -**Settings** (default model and thinking, run settings, compaction, queue modes, code mode switch, owner name) have -**one writer**: a core `Settings` object on the server owns the file, validates writes and serializes them; every -other process follows its state through a reference instead of watching and rewriting the file itself. p2 watches -settings, models and auth in every process and writes them from two, which is decision #21. - -**Notices** are one channel: one core `Notices` object on the server, written by everything that fails or finishes -(worker crashes, reload results, model errors, login outcomes, grants), keeping one latest notice per source, at most -50. The owner's shell reads one list; dismissing is a method. p2 has five notice channels for the same thing. - -:::review | Review: settings and file formats (#21, #22) -Single writer for settings, models and auth (#21) and versioning on-disk formats such as `envs.json` like documents -(#22) are open p3 decisions in `design.md` §26. The construction above assumes "one owner on the server, others -follow its state" for #21. -::: - -:::note | Built so far -Nothing of p3 exists. The functionality map is complete (`apps/p2/docs/p3-functions/`, five area files plus the -condensed `README.md`), and the design's app rules (`design.md` §25) are decided. p3 starts after the layered rebuild -and the playground's first real proof. -::: diff --git a/packages/chord/docs/next/explainer/src/19-p2-sessions.md b/packages/chord/docs/next/explainer/src/19-p2-sessions.md deleted file mode 100644 index 9d40c2287..000000000 --- a/packages/chord/docs/next/explainer/src/19-p2-sessions.md +++ /dev/null @@ -1,276 +0,0 @@ - -# Sessions and Conversations {#p2-sessions lede="The session lifecycle, status without a worker, chat, shell commands, subagents and search, each mapped to an owner and the Chord parts it uses."} - -This chapter builds the domains a person touches most: the session list and its operations, the conversation view -and composer, shell commands, chat mode, subagents, messages between sessions, search and unread marks. Each section -names the owner, the host, and the Chord parts it uses; code appears only where it shows something new. - -## The session lifecycle - -The catalog (chapter XVIII) owns the session records. Its operations, by the Chord part each needs: - -| Operation | Owner and form | Notes | -| --- | --- | --- | -| create | `Catalog.create`, a method | absolute directory, `~` expanded, must be a directory; on an environment, resolved there through the envs plugin; refuses an unknown model; model and thinking default to the last used | -| rename, pin | methods on `Catalog` | title 1 to 120 characters, whitespace collapsed; an explicit title is never replaced automatically | -| automatic title | a routine on the search feed | the first user message, line breaks to spaces, 100 characters | -| agent rename | an agent tool on the worker calling `Catalog` | root conversation only | -| archive, restore | methods; revocation; reconciling routines | below | -| clone, fork, relocate | jobs on the catalog store | outlive the call, cancellable, one outcome | -| delete | a job; decision #18 | below | - -### Archive and restore - -Archiving must stop the worker whatever it is doing, end guest access, and close the session's port forwards, while -keeping its storage. The catalog does none of the plugin-specific parts: - -:::steps | Archiving S1 -+++ One write -`Catalog.archive({ id: "S1" })` sets `archived: true` in the session record, in one transaction. -+++ Revoke the session -The catalog calls `host.revoke({ session: "S1" })`. Every guest principal of S1 has `{ session: "S1" }` in its -`revokedBy`, so their objects end and their links close on the server and on the worker (chapter VII). New -connections are refused by `authenticate`, because grants of an archived session are void. -+++ Stop the worker -The supervisor reconciles from `catalog.sessions` with a routine: an archived session's worker is stopped, and -`authorize` refuses to start it again. -+++ Plugins reconcile -The ports plugin's routine on `catalog.sessions` closes forwards of archived sessions. The search plugin marks the -session's rows archived. If the server crashed between the write and these steps, the routines run again with the -current value at the next start: nothing is a notification that can be lost. -+++ Restore -`Catalog.restore` clears the flag. Grants are valid again, so guests reconnect and authenticate as before; forwards -stay closed. -::: - -### Clone, fork, relocate - -Each is work that outlives its call, can be cancelled and has one outcome, so each is a **job** (chapter XI) on the -catalog's local store, keyed by the owner's request id. They differ in what they pause: - -- **Clone** copies the session's store, uploads, downloads and artifacts under a new id, with the source worker - paused (refused while work runs) and title "(copy)"; directories `0700`, files `0600`, no links, no overwrite; a - partial copy is removed on failure. -- **Fork** before a user message copies the active context as of the entry before it into a new session (title - "(fork)"), with the message waiting in the new session's editor. It needs two workers: the source reads history, - the new one is seeded empty. A failure archives the copy. -- **Relocate** (cwd, environment or both): refused while the session is busy (naming the running subagents) or when - a subagent has its own cwd and the move crosses machines. The job fences the worker (new admissions wait), updates - the record, and moves the running worker in place by updating `SessionLocation`: natives that follow the location - switch with a generation (chapter IX), and conversations that follow the session's cwd are rewritten in one Durable - commit before work resumes. After 60 s it restarts the worker instead; on failure it releases the fence. - -:::review | Review: jobs as objects (#9) -The design defers whether a long relocation or a login is a record inside its owner (watched through `running`) or an -object of its own (`Relocation { id }`). The construction above uses records. Decide when login is ported -(`design.md` §15, §26). -::: - -### Delete - -p2 has no session delete. p3 needs one that removes the session everywhere: every document tagged with the session -(`store.retireByTag(SessionTag, id)`, chapter V), the worker's databases (the Durable store), the session's files, the -search rows, read marks, children copies, forwards, guest records, and finally the catalog entry. The design commits to -this outcome (`design.md` §25); how plugins take part is open. - -:::review | Review: session delete (#18) -Options in `design.md` §26: per-session objects plus `SessionTag` plus one session directory (delete is retire by tag -and remove a directory, plugins need do nothing), or owner hooks contributed through a catalog registry. Also open: -whether the search index is per session (deleted with it) or global with a hook. Who holds the store for -`retireByTag` is not specified either. -::: - -## Status, children, unread: reports to the server - -The session list must show status, subagents and unread marks for sessions whose workers are not running, and must -not start them. All three follow chapter X's pub/sub pattern: a routine on the worker calls a server object with the -current summary, the server object keeps the latest value per session (persisted where it must survive a restart), -ignores calls from an older `ctx.incarnation`, and clients read the server object. - -| Summary | Source on the worker | Kept on the server | -| --- | --- | --- | -| status: running, error, task count, busy conversations, usage and cost | the harness's task graph and usage | `SessionStatus`, in memory: a stopped worker is "not running" | -| children: name, model, thinking, tools, cwd, last entry, archived | the subagents plugin's child list | `Children`, persisted | -| latest root answer | the search feed | `Unread`: compared with the owner's read mark, persisted, shared by all owner devices | - -Guests see their own session's status without the error text. The owner's read mark is written when the root -conversation is visible, and a new session starts as read. - -## Conversations - -The harness (chapter XIV) owns the Durable store. The UI needs, per conversation: a live window of entries, older -pages, usage numbers, the working indicator, the inbox (queued inputs), and the actions. Two core objects provide them: - -- **`ConversationStore { conversation }`** (local, on the worker): opens the Durable watch as its native, reaching - the harness through the core-private `HarnessRuntime` registry entry, and exposes a **window** as a derive: the - newest entries (at most 100 or 512 KiB, at least 20) plus the conversation's documents. The window's start stays - put while it is within 1.5 times the limits, so new entries arrive as small appends instead of a shifted list - (chapter VIII's window pattern). -- **`Conversation { session, conversation }`** (remote, placement `Session`): the per-principal view. Variants by - `can()`: `owner` (everything, including model, thinking and handoff), `writer` (guest write: send, steer, stop, - compact, shell), `reader` (the window only). - -```ts -host.serve(Conversation, ({ session, conversation }, principal, self) => { - const store = self.get(ConversationStore, { conversation }); - const policy = self.get(ConversationPolicy, { conversation }); // core: readOnly, followsSessionCwd - const author = authorOf(principal); // structured: kind, id, display name - if (!can(principal, "send", session)) return { variant: "reader", view: store.window, page: store.page }; - return { - variant: can(principal, "manage") ? "owner" : "writer", - view: store.window, - page: store.page, // older history: returns data - send: ({ text, attachments, mode, requestId }, ctx) => { - if (policy.value.readOnly) throw forbidden("read-only conversation"); - return store.submit({ type: "input", content: text, attachments, author, whenBusy: mode, requestId }, ctx); - }, - stop: (_args, ctx) => store.stop(author, ctx), // allowed even when read-only - … - }; -}); -``` - -What each function becomes: - -| Function | How | -| --- | --- | -| send, retried send | Durable `submit` with the author's `requestId`: a retry with the same id is admitted once (Durable dedupes, chapter XIV) | -| steer or queue while busy | `whenBusy: "steer"` or a follow-up; the inbox is `docs["pi.inbox"]` in the window | -| attribution | the view derives the author from the principal; the entry stores it structured, the model sees a speaker block; nothing parses text | -| wait for prompt resources | the first input of a fresh worker waits until the prompt plugin's resources are loaded (a registry readable the store follows) | -| stop, dequeue | withdraw queued inputs, abort the run; the caller's texts return to its editor, other authors' to their **restored drafts** | -| restored drafts | a keyed document per (conversation, author), tagged `SessionTag`, persisted in the store; claimed once, first tab wins | -| read-only conversations | the core per-conversation policy document, written at `createConversation({ init })` by whoever creates the conversation; send, dequeue, configure, compact and shell refuse; stop is allowed | -| history pages | a method returning a page (100 entries, 1 MB, 2,000 scanned); compaction and reset markers always included | -| model and thinking | owner variant only; validated against the models plugin's catalog; the root conversation's model is also reported as the session default | -| compaction, handoff | Durable operations through the store | - -**Drafts in the composer** are UI state, not Chord state: the web shell keeps one draft in `sessionStorage`, the TUI -keeps one per conversation across UI replacements, and pending sends keep their request id until acknowledged -("Sending when connected"). Only restored drafts, which the worker produces, are documents. - -## Shell commands - -`!cmd` runs a command in the conversation's cwd on its environment and records the result as an entry; `!!cmd` does -the same but keeps the output out of the model's context. One command at a time per conversation; the live tail is -at most 50 KB or 2,000 lines, sampled every 200 ms; the full output spills to a file; the result entry is placed at the -run's next boundary while the agent is busy and never starts a run. This is the design's job example almost as is: - -```ts -const ShellJob = defineJob("chat.shell", { - version: 1, - args: obj({ command: str({ max: 100_000 }), include: bool(), author: Author }), - progress: obj({ output: str() }), - result: ShellResult, - concurrency: 1, // one per conversation -}); - -const results = self.state(ShellResults); // keyed by run id: dedupe after completion -const shell = self.job(ShellJob, { - run: async ({ command, include, author }, ctx) => { - const out = new OutputBuffer({ maxBytes: 50_000, maxLines: 2000, retain: "tail" }); - const env = await environmentFor(ctx); // the conversation's environment, chapter XX - const r = await env.exec(command, { cwd: env.cwd, signal: ctx.signal, spill: spillPath(ctx.jobId) }, - (chunk) => { out.push(chunk); ctx.progress({ output: out.snapshot().text }); }); - const data = { command, output: out.snapshot().text, exit: r.code, cancelled: ctx.signal.aborted }; - await conversation.submit( - { type: "write", entry: shellEntry(data, author, include), requestId: `shell:${ctx.jobId}` }, ctx); - return data; - }, - finish: async (done, tx) => { - (await tx.draft(results, done.id)).outcome = done.kind === "done" ? done.result : { error: done.error }; - }, -}); -``` - -The request id `shell:${ctx.jobId}` makes the entry idempotent: an attempt interrupted after the write cannot write -it twice. Cancel is `shell.abort(id)`, allowed for the command's author and the owner. A retried `runShell` with the -same request id returns the running job (dedupe of unsettled runs), or the stored result (the owner's own dedupe after -completion, chapter XI). p2's "cancel when nobody watched for 30 s" is dropped: a job keeps its owner alive and does -not depend on viewers. - -## Chat mode - -Chat mode is a per-conversation switch that everyone in the session sees and only the owner changes; forks and new -subagents start with it off. Because it must fork and rewind with the conversation, it is **transcript-bound**: a -Durable conversation document, not a Chord document. Messages without `@pi` become display-only Durable `write` -entries (author, text, attachments, reply quote); a message with `@pi` becomes an input with a digest of the chat -since the last mention (at most 100 messages, 20 files, 4 images), recording which entries it passed on. The reply -quote is derived on the worker from the entry id the client names: the client never supplies author or quote text. - -## Subagents and one-shot agents - -Subagents are Durable child conversations owned by tasks, as in Durable's own background-subagent example: one -`subagent` tool spawns, steers, waits for, stops and lists them; each child is owned by a background anchor task, and -each message is delivered by a reporter task that posts the answer back to the parent once, with request ids so a -restart sends nothing twice. Chord adds only three things: - -- the **child list** for the sidebar is reported to the server (`Children`, above), so it shows without a worker; -- a child that follows the session's cwd, or a read-only one-shot agent created by a code-mode script, is marked in - the core **per-conversation policy document** when it is created, so chat and relocation enforce it without knowing - the subagents plugin (`design.md` §25 porting guideline); -- archive and restore of a child from the UI are methods on the subagents plugin's view, refused for one-shots. - -Subagents cannot spawn subagents; thinking levels are clamped to what the model supports, in one helper shared with -the session's model choice. - -## Messages between sessions - -The root agent can list other sessions and send one an attributed message. The tool runs on the sender's worker and -calls a server object through `chord(fn, context)` (chapter XIV): - -```ts -const peerSend = defineTool({ - name: "session_send", - execute: (args, api, context) => chord(async (ctx) => { - const peers = await ctx.get(Peers, {}); // server, worker variant - await peers.send({ to: args.session, text: args.text, mode: args.mode, requestId: `peer:${api.taskId}` }, ctx); - return { output: [{ type: "text", text: "sent" }] }; - }, context), -}); -``` - -`Peers` checks that the sender is a root conversation of a non-archived session, refuses sending to itself and -archived recipients, then gets the recipient's `Conversation` (which starts its worker) and submits with the -sender as a structured author and the same request id (the tool task's id, stable across restarts), so a retried -tool call delivers once. The sender's -incarnation is `ctx.incarnation`; a call from a replaced worker is refused. Only owner-initiated turns may use the -tool; guests never. - -## Search - -The search plugin indexes user and assistant messages of root and subagent conversations (no shell output, no -Durable internal inputs) for title, cwd and message search across sessions, active or archived. - -:::flow wide animate | The search feed. The worker pushes pages from a cursor; the server owns the index and never starts a worker to answer a query. -row "Worker S1" blue -box commit "Durable commit" "in any conversation" -box wake "wake" "a routine on the harness's commit signal" -box page "read from cursor" "200 entries per page" -row "Server" accent -box index "SearchIndex" "native: SQLite FTS" accent -box cursor "cursor per conversation" "moves after the whole range" -box query "query" "terms, scope, at most 41 hits" -arrow commit -> wake -arrow wake -> page -arrow page -> index "report(page)" -arrow index -> cursor -note "Reports carry ctx.incarnation. Indexing is idempotent: a page sent twice changes nothing." -::: - -The index is a native owned by a host-lifetime object on the server; cursors live with the index, so after a crash -the worker asks for the cursor and continues. Rebuilding one session's index drops its rows and rescans, starting -an archived session's worker without resuming its work. Queries are methods returning data: at most 41 hits, -snippets of 300 characters, queries up to 500 characters. The same feed sets automatic titles, moves "updated at" -forward and supplies the latest root answer for unread marks. - -## Presence - -Viewers per session are `host.references(Conversation)` on the worker, counted per client (chapter VIII), and -reported with the status. Connected guests (name and mode) are the references whose principal is a guest. Presence -is not keep-alive: the supervisor uses `host.activity`, which also counts calls and jobs. - -:::note | Built so far -None of this exists in p3. The parts it relies on are Durable (built: submit with request ids, steer and queue, -child conversations, background tasks) and the Chord layers L3 to L9 (not built). -::: diff --git a/packages/chord/docs/next/explainer/src/20-p2-machines.md b/packages/chord/docs/next/explainer/src/20-p2-machines.md deleted file mode 100644 index 51b9e7546..000000000 --- a/packages/chord/docs/next/explainer/src/20-p2-machines.md +++ /dev/null @@ -1,245 +0,0 @@ - -# Machines, Files, Models {#p2-machines lede="Environments, workspace, terminals, session files, ports and Radius, models and logins, prompt resources, logs, reload and the shells: the rest of p3."} - -The last domains are the ones that touch the outside world: other machines over SSH, the file system and git, -terminals, the relay, model providers and their credentials. Most of them are native objects, and most of them are -places where p2 has the same thing several times (four SSH connections to one environment, five notice channels, -three copies of the environment config parser). The rule that fixes that is the one from chapter XVIII: one owner per -thing. - -## Environments - -The **envs plugin** owns the environment configuration: name, host, user, port, identity file, shell and its -variables, plus a managed ed25519 key per environment and its own `known_hosts` (kept apart from the user's, aliased -per environment name). - -| Function | Owner and form | -| --- | --- | -| list, add, edit, delete (refused while a session, archived too, uses it) | `Envs` on the server: a persisted document behind an owner-only view | -| validation (no leading `-`, no whitespace or control characters in host and user) | the view's methods, before the write | -| a hand-edited `envs.json` is picked up; an invalid file keeps the last valid set and shows an error | a native file watcher (poll fallback) in the envs store; one writer (#21) | -| environment revision: a hash of the config, part of its identity | a derive | -| connection test (connect, deploy the daemon, platform, latency, warnings) | a job; the result per revision in a document | -| unknown or changed host key: fingerprints, trust after confirmation, forget | a flow (chapter XII) on the server | -| browse directories (`~`, relative paths, at most 10k scanned, 1,000 shown) | a method with a **call-scoped** SSH session (chapter IX) or the pooled connection | -| a changed environment restarts its sessions' workers once idle | a routine on the envs store and the status reports | - -**Execution in the agent.** The worker's harness asks `HarnessEnvironments` for an environment per target (chapter -XIV). The envs plugin contributes it from its worker facet: local execution at the session's cwd, or a remote one over -SSH through the `pi-env` daemon, deployed per platform and connecting at the first operation. A remote session never -falls back to local tools. The environment follows `SessionLocation`: after a move, the old connection closes once -the move has committed and watchers reopen at the new location. - -```ts -// envs/worker.ts -export default defineFacet((host, self) => { - const location = self.get(SessionLocation, { session }); - const pool = self.get(EnvPool, {}); // one owner of SSH connections (#20) - self.contribute(HarnessEnvironments, (target) => - target.env === "local" ? localEnv(target.cwd) : pool.environment(target.env, target.cwd)); -}); -``` - -:::review | Review: connection pools (#20) -SSH connections to one environment exist four times in p2 (worker environment, browse connection, test connection, -terminal host). `design.md` §26 asks for one owner with a pool or deliberate per-use connections. `EnvPool` above is -the first option: a native object that closes idle connections after 10 minutes and invalidates them on a config or -key change. Reloading the plugin that owns the pool kills running remote commands (chapter XV), so it should change -rarely. -::: - -## Workspace and file index - -The **workspace plugin** serves the viewed conversation's working tree (a subagent's own cwd included) on the worker: -list a folder (at most 2,000 entries, `.git` hidden), read a file (1 MiB, binary detected), git status, diff, log and -single commits, literal text search, and all paths for name filtering. Every query is a **method returning data**, not -state: results are bounded and requested by the UI. Only change notifications are state: a counter in a document, -bumped by a native watcher of the tree, `HEAD`, the index and config (debounced 100 ms); viewers reload what they show -when it changes. - -Two rules live in one place each instead of three: - -- **Confinement:** path rules, a canonical root, no `.git`, the app's private data directory denied (also excluded - from search). One helper used by listing, reading, uploads and the file index. -- **Git hardening:** no optional locks, no fsmonitor, filters disabled, no prompts or pager, no system config, 10 s - timeout, output caps. - -Downloads (25 MiB, byte ranges for media) go through the HTTP component (chapter XIII) with a route on the server that -reads chunks from the worker; uploads into the project use the chunked protocol of chapter XIII (1 MiB chunks, temp -file then rename, never overwrite, writes serialized). Guest read gets no workspace: the view has no variant for it. - -The **file index** for `@` completion belongs to the same owner, so completion consumes the index instead of scanning -on its own: built from the environment's listing (else git or a walk, at most 50k paths), rebuilt at most once per -second while files change, discarded after a move. Review comments drafted on files and diffs are browser state until -sent; sending them is an ordinary `Conversation.send`. - -## Terminals - -Terminals must survive server and worker restarts, so their PTYs live in a **detached terminal host process**, one per -data directory, with its own process lock; it exits after 10 minutes with no terminals and no clients. The terminals -plugin on the server is the owner the UI talks to: list (title, cwd, environment, size, exit status), open at the -session's cwd and environment (refused for archived sessions), attach, close (asking first if a foreground job runs). -Terminals are **owner only**. - -Bytes do not go over object links. Attaching opens a **channel** (chapter XIII) with a single-use token (30 s): JSON -control on the object, bytes on the channel, with its own acknowledged window (256 KiB, acknowledgements per 64 KiB; a -viewer that falls behind gets a fresh screen snapshot; an 8 MiB backlog disconnects). Attaching gets the current -screen from a headless emulator with 10k lines of scrollback. - -:::review | Review: the terminal host process (#23) -The terminal host runs its own code. Is it part of the revision, reloaded or restarted with it (and running terminals -end), or deliberately outside it with a "host runs older code" notice as in p2? Open in `design.md` §26. -::: - -## Session files - -Attachments (25 MiB, at most 20 per message), sent-file snapshots made by the agent's `send_file` tool, versioned -artifacts (HTML, SVG, Markdown, text; 2 MiB; each update a new immutable version) and shell spill files are all -per-session bytes with caps, immutable ids and range reads; p2 has two upload routes, two caps and two directory -layouts for them. The proposal is one owner: - -```ts -export const SessionFiles = remote("p3.session-files", { - version: 1, - input: obj({ session: str() }), - placement: () => Server(), // readable without a worker - variants: { writer: { … }, reader: { … }, worker: { … } }, -}); -``` - -`SessionFiles { session }` on the server holds the index (per kind, readable without a worker), the chunked upload -and read protocol, the caps per kind, copying on clone and fork, and deletion with the session. Attachments are copied -into a remote environment before an input is admitted, with no fallback on failure. Artifact previews run in a -sandboxed frame with its own CSP and no app identity; ordinary file previews allow only safe types. - -:::review | Review: session files (#17) -Proposed in `design.md` §26, not decided. Open sub-points: one preview limit (p2 has 256 KiB for sent files and -512 KiB for artifacts), what a fork copies of the artifact list, and how attachments are copied into remote -environments. Related: #22, whether on-disk layouts such as downloads and uploads get versions like documents. -::: - -## Ports and Radius - -The **radius plugin** keeps the outbound relay connection, a native in a host-lifetime object on the server: sign-in -by device code from the CLI (credential in a private file), token refresh, a heartbeat in the link adapter (20 s ping, -60 s timeout; TCP does not notice a sleeping laptop), backoff from 1 to 30 s, a disable switch. The relay delivers two -kinds of channels: - -```ts -// radius/server.ts (the entry point: one line, as in chapter I) -radius.onChannel((ws) => host.accept(webSocketLink(ws), { principal: { kind: "owner", access: "relay" } })); -radius.onForward((channel) => proxyToForward(channel, host.registry(RadiusForwards).value)); -``` - -Object links become owner principals with `access: "relay"`: owner, but treated as remote (no localhost-only -powers). p2 had to fake a loopback request with a secret header for this; with `accept` that is gone. - -Forward channels go to `localhost:` (Host, Origin and Location rewritten; 502 when nothing listens; anonymous -visitors only to public forwards). Which forwards exist is not radius's business: it owns a registry, -`radius.forwards`, and the **ports plugin** contributes its forwards (at most 50, random ids, a public toggle that -needs confirmation, an agent tool `ports` for top-level owner-initiated conversations). Without the ports plugin the -table is empty and radius still works: an optional dependency is a registry owned by the dependent (chapter XIII). The -ports plugin closes forwards of archived sessions with a routine on `catalog.sessions` (chapter XIX). - -:::review | Review: may a plugin call `accept`? -`design.md` §3 shows the Radius line in the server's start code and calls entry points "app or adapter code". If -radius is a plugin, its facet needs the host's `accept`, which no other part of this guide gives a plugin. The -alternative is that the start code keeps the entry point and gets channels from the radius object. Not specified. -::: - -## Models and credentials - -The **models plugin** owns the model runtime (pi-ai's models with custom providers and the model cache) in a -host-lifetime object, and the credentials. It contributes the runtime to `HarnessModels` (chapter XIV), so after a -models reload the next request uses the new runtime while a running stream finishes on the old one. **Credentials -never go to clients**: the views expose providers, login methods and which type is signed in, never tokens. Every -secret is registered for redaction in one place, the log capture. - -Views: a model catalog for pickers (names, ids, context windows, thinking levels; at most 300 matches in the web), -logout (refused during a login), refresh a provider's model list. The default model and the cycling scope are -settings (chapter XVIII). - -### Login - -A login is the guide's flow example (chapter XII): a **job** on the server that runs pi-ai's login, and a flow that -shows its steps to the starting owner. - -```ts -const LoginJob = defineJob("models.login", { - version: 1, args: obj({ provider: str(), type: AuthType }), progress: LoginState, result: obj({ ok: bool() }), - concurrency: 1, // one login at a time on the server -}); - -const login = self.job(LoginJob, { - run: async ({ provider, type }, ctx) => { - await runtime.value.login(provider, type, { - signal: ctx.signal, // 10 minute deadline, cancel, leaving - prompt: (prompt) => steps.ask(prompt, ctx), // text, secret, select, manual_code - notify: (event) => steps.notice(event, ctx), // info, auth_url, device_code, progress - }); - return { ok: true }; - }, - finish: async (done, tx) => { /* record the outcome; refresh the model list */ }, -}); -``` - -`prompt` is a promise-based callback inside the work; the resolver stays in the job (`steps.ask` records the step in -the flow state and waits for `answer`), and nothing crosses an object boundary. On a relayed connection the device -code method is selected automatically. The flow is visible only to the owner principal that started it; others are -refused. Leaving ends it: when the starting owner's references are all gone (presence, not an unmount), a routine -aborts the job. Codes, URLs and typed secrets are redacted from logs. - -## Prompt resources - -`AGENTS.md`, skills and templates are read in the session's environment, watched (poll fallback, last valid state), -and expanded on the worker before sending. The **prompt plugin** owns them per session on the worker, contributes the -system prompt section and skills to `HarnessExtensions` as readable contributions (a change recomposes), and exposes a -readiness readable: the first input of a fresh worker waits until it is ready (chapter XIX). After a move the watchers -reopen at the new location. - -## Code mode - -Code mode is a plugin (chapter XIV): its `pi-codemode` sandbox is a native; scripts call tools as nested Durable calls -and replay after a restart; the tool entry looks up the current sandbox when it starts. A reload seals and drains the -old sandbox for at most 30 s. The on/off switch is a setting. - -## Logs and notices - -**Log capture** tees stdout and stderr of the server, workers and detached hosts without changing their behaviour, -redacts known secrets and credential patterns, and keeps bounded rings (at most 128 sources, 1,000 lines or 256 KiB -each). Detached hosts publish redacted snapshots to a private file. Logs are read without waking a stopped worker. -**Notices** are the one channel from chapter XVIII; dismissal stores only notice ids. - -## Dev and reload - -The dev plugin is chapter XV's reload control for p3: a source watcher marks plugins dirty and never applies anything; -the `reload_plugins` agent tool and a control in the owner's shell reload the changed plugins, or restart all hosts for -base code (through the supervisor, reopening the sessions that were running). UI bundles are published only after a -successful backend change; a web page detects a new manifest and reloads with its drafts and navigation state saved; -the TUI swaps its sandboxed UI code with a snapshot of its state. A local Unix socket (`0700`) accepts development -commands without a network admin port. - -## The shells - -The web and TUI shells are UI facets over the objects above. Two rules matter for Chord: - -- **UI state survives navigation, reload and reconnect on purpose.** Composer drafts, pending sends with their request - ids, reading positions and review drafts are kept in tab storage or the TUI's replacement snapshot. A reconnect - re-holds and gets snapshots; nothing is lost that was not in flight. -- **Leaving is never derived from an unmount.** A plugin reload remounts components (chapter XV). Cancelling a login, - closing an overlay or giving up a lease is an explicit action or a dropped reference. - -The TUI launcher keeps downloaded UI code in a permission-dropped worker with no access to files, network or -credentials; local side effects (copy, upload, download, open a URL, an external editor) need a real user gesture or a -confirmation. That is app code around Chord, not part of it. - -## Transport - -The last domain, transport, is chapter XVII: one message format, per-part limits (16 MiB compressed and 64 MiB decoded -on client WebSockets, instead of p2's 1 MiB input cap and 32 pending calls), one byte window per link, re-holds with -snapshots after a reconnect, and a protocol version that makes an outdated web page reload. - -:::note | Built so far -None of p3 exists. pi-ai's login API, pi-codemode and the `pi-env` daemon exist and are used by p2 today; the HTTP -component and channels are deferred until after the kernel's first real proof (chapter XIII). -::: diff --git a/packages/chord/docs/next/explainer/src/20a-compared.md b/packages/chord/docs/next/explainer/src/20a-compared.md deleted file mode 100644 index aa4fcd2c2..000000000 --- a/packages/chord/docs/next/explainer/src/20a-compared.md +++ /dev/null @@ -1,146 +0,0 @@ - -# Other Ways to Do It {#compared lede="Cordis and DeepSeek Harness attack the same problem, plugins that come and go while an agent runs, inside one process. What that buys, and what it costs."} - -Chord v2 is not the only answer to "replace plugin code while the app keeps running". The closest relative is -**Cordis**, a TypeScript plugin framework by Shigma that has been the plugin kernel of the Koishi chatbot framework -since 2019, and its most visible user, **DeepSeek Harness** (`dsh`), DeepSeek's open-source agent harness, whose -motto is "everything is a plugin". This chapter compares them with Chord v2 on the questions the guide cared about: -who owns side effects, how plugins depend on each other, what a reload keeps, what a crash keeps, and how far the model -reaches beyond one process. - -:::note | Sources and status -DeepSeek Harness at commit `687f282` (release 0.2.1-rc.2), which vendors Cordis as `@deepseek-ai/cordis` 4.0.5 (Cordis -RC7 with local changes, `vendor/README.md`): its Cordis tutorial (`docs/cordis-tutorial/`), `docs/architecture.md`, -`docs/subsystems/persistence.md`, `docs/subsystems/core.md` and `packages/boot/hmr/README.md`; the Cordis paper "A -Programming Paradigm for Spatiotemporal Composability" (arXiv 2608.25512). Cordis and dsh are built and in use; Chord v2 -is the design this guide describes. The comparison is between a working system and a design. -::: - -## Cordis in one example - -A Cordis plugin is a function (or an object with `apply`, or a `Service` subclass) that receives a **context**. It may -declare the services it needs with `inject`, and everything it does to the world goes through the context: - -```ts -export const name = "greeter"; -export const inject = ["tools"]; // activate only while a `tools` service exists - -export function apply(ctx: Context) { - ctx.tools.register(defineTool({ name: "greet", /* … */ })); // a registration is an effect: undone on unload - ctx.on("session/start", (s) => log(s.id)); // an effect-bound listener - ctx.effect(() => { // any other side effect, with its disposer - const timer = setInterval(work, 200); - return () => clearInterval(timer); - }); -} -``` - -Each plugin instance is a **fiber** with states `PENDING → LOADING → ACTIVE → UNLOADING → DISPOSED` (or `FAILED`). -Every effect, registration, listener, child plugin and provided service is recorded on the fiber, and unloading runs -the disposers. The paper calls this **temporal composability**: a component can be removed at any time and reverses its -own effects. Its counterpart, **spatial composability**, is the dependency side: `inject` is live, so a plugin starts -when its services appear, and unloads (and later reloads) when a service it depends on goes away or is replaced. The -paper's claim is that independently written components can therefore be combined and replaced without disturbing each -other. The next sections check how far that holds. - -**Services** are named capabilities on the context (`ctx.provide("x", value)` or `class X extends Service`). Names are -runtime strings in one flat namespace per application; TypeScript declaration merging makes `ctx.x` typecheck, but -nothing checks the service's shape at runtime, and optional dependencies are probed with `ctx.get("x")`. Scopes come -from the context tree: `ctx.extend()` derives a child context, `ctx.isolate(name)` gives one subtree its own binding of -a service, so two parts of an app can use two different `shell` providers. - -**Hot reload** follows from the two halves: the reload plugin unloads the old fiber (unwinding all its effects) and -loads the new module with the same config. Nothing held in the old instance's closures or fields survives; a plugin -that needs state across reloads must keep it in a service or on disk. Config edits are diffed by entry id and -mount, unmount or reconfigure only what changed; config fields declared volatile update in place without a remount. - -## DeepSeek Harness on Cordis - -dsh applies "everything is a plugin" to an agent: the model adapters (`ctx.llm`), the tool registry (`ctx.tools`), the -session log (`ctx.sessions`), the agents (`ctx.agents`) and the agent loop itself (`ctx.agentLoop`) are plugins and -services. A running dsh is a plugin tree composed at boot from a **profile** (`web`, `headless`, `sdk`, `acp`) that -stacks **bundles** of Cordis config rows, with user patches on top. There is no privileged core: replacing the agent -loop is a config patch. - -| Concern | How dsh does it | -| --- | --- | -| sessions | a durable, append-only event log per session (JSONL backend, single writer, `flush` as the durability barrier); model history is derived from the events | -| crash mid-turn | the log keeps everything appended; on resume the agent loop closes the open turn by appending missing tool errors and a synthetic `turn/end { reason: "interrupted" }` | -| tools | `ctx.tools.register(defineTool(…))`, an effect of the registering plugin; calls go through pre/execute/post policy waterfalls | -| approval | closed by default; `allowed-once` is the only grant; plugin installation needs full access or per-call approval | -| reload | a source watcher evicts changed modules; Loader entries running affected plugins stop and start again with the new code; a failed replacement is an activation failure, its previous code no longer runs | -| processes | one plugin tree in the Host process; the browser client and Electron's renderer talk to it over RPC streams | -| distribution | a Plugin Manager installs and enables bundles per profile; plugin markets are optional indexes | -| subagents | local subagents with their own durable sessions, and external ones | - -## What the guarantees are in practice - -Reading the Cordis source vendored in dsh and dsh's own documentation, the guarantees are narrower than the paper's -framing. An earlier review for p2 came to the same result (`apps/p2/docs/p3-reviews/dsh-compare.md`). - -- **Reversal is cooperative and unordered.** `_unload` runs all of a fiber's top-level disposers concurrently - (`Promise.all` in `vendor/cordis/src/fiber.ts`); order holds only inside one effect, and the primer tells authors to - "keep the related work in one effect" when teardown order matters. A disposer that throws is logged and skipped. A - disposer that never settles stalls the teardown; dsh's jobs service works around this by force-failing a job whose - cancel throws and warning that "the work may be orphaned" (`packages/jobs/jobs-local/README.md`). Reversal is as - good as every disposer, and nothing checks that a plugin registered all its effects. -- **Behaviour depends on registration order, and a reload changes it.** `emit`, `waterfall`, `serial` and `bail` - listeners run in registration order (`docs/cordis-primer.md`). Registering appends the listener (`push` in - `vendor/cordis/src/events.ts`, unless `prepend`), so when a plugin reloads, its listeners move to the end of the - list. A tool-policy listener that ran first runs last after its own plugin reloads; a waterfall that short-circuits - now sees a different decision first. -- **Dependencies are strings, and every provider reload restarts its dependents.** A fiber's activation key is the - concatenation of its injected providers' fiber ids (`_refresh` in `fiber.ts`), so replacing a provider unloads and - reloads every plugin that injects it, with their in-memory state. Service APIs are "pre-stable; update every - consumer" (`AGENTS.md`): there is no versioned contract between a provider and its consumers. -- **Reloading a core service ends live work.** Unloading the agent provider "stops and drains every live handle it - made" (`packages/core/agent/README.md`), so running agents stop; reloading the session persistence backend under - live sessions invalidates their write handles, and "writes fail loudly until the sessions restart" - (`.agents/notes/implemented/architecture/2026-08-27-handle-based-session-persistence.md`). -- **Registrations leave a gap.** A reloading plugin's tools and other registry entries are removed with its old fiber - and added again by the new one; anything that looks them up in between does not find them. - -None of this is a bug in the narrow sense: each point is documented, and dsh is built around it. But "components -interleave without disturbing each other" holds only for plugins that own nothing other plugins depend on, register -every effect, order nothing by registration, and hold no state that must outlive a reload. - -## Side by side - -| | Cordis and dsh | Chord v2 | -| --- | --- | --- | -| unit | a plugin instance (fiber) in one context tree | an object per (type, input, principal), placed on a host (chapters II, III) | -| side effects | recorded on the fiber with `ctx.effect` and friends; on unload the disposers run concurrently, each as good as its author made it | declared through `self` (documents, natives, routines, jobs, contributions); Chord activates and undoes them (chapters V, IX to XI) | -| dependencies | service names as strings in one flat namespace, via `inject`; a provider reload restarts every dependent | typed contracts via `get(Type, input)`; a reference parks and recovers through restarts (chapter VI); optional dependencies are registries (chapter XIII) | -| ordering | listeners run in registration order, which changes when a plugin reloads | registries compose with a declared `compose` function; nothing depends on load order (chapter XIII) | -| where code runs | one process; remote UIs over RPC to the Host | a tree of hosts (server, workers, tabs); objects reachable across links, relayed by the server (chapters I, XVII) | -| who is calling | not a framework concept | principals stamped by links, variants per caller, revocation (chapters III, VII) | -| state | the plugin's own business; dsh's session log is a service | documents written in transactions, persisted, versioned and migrated (chapter V) | -| reload | unload the old instance, load the new one: in-memory state and running work of that plugin end | pure objects swap their code in place with their documents and running work kept; only objects with natives restart (chapter XV) | -| a running agent turn during reload | entries of changed plugins stop; unloading the agent provider stops its live agents | the harness is base code; Durable pins tools per task phase, so a running call finishes on old code and the next phase uses the new (chapter XIV) | -| crash recovery of a turn | the turn is closed as `interrupted` | pi-durable continues the turn's tasks from their checkpoints; replay-safe tools rerun (chapter XIV) | -| contracts and versions | TypeScript types only, pre-stable APIs, every consumer updated together; config validated with a schema | contract versions with fingerprints and provider-side mappings for old consumers (chapter XVI) | -| status | in production in Koishi since 2019; dsh released | a design with the contract layer built | - -## What the comparison says - -**The core idea is the same.** "Factories declare, Chord activates" (chapter II) and Cordis' reversible effects are -two forms of one rule: a plugin may only change the world through its runtime, so the runtime knows what to undo. -Both reject the module-level timer and the stored callback for the same reason. - -**The difference is what survives a reload.** Cordis reloads at plugin granularity: the old instance unwinds and the -new one starts from nothing, together with every plugin that depends on it. That is simple, but a plugin's in-memory -state and its dependents' state and work in flight end with it, and correctness rests on every disposer and on -registration order. Chord v2 keeps state out of closures on purpose (documents, references, jobs as -records) so that it can swap the code of a pure object while its data and running work stay, and restarts only what -holds OS handles. That is more machinery: transactions, a store, versions on every document and contract. - -**The other difference is reach.** Cordis composes plugins inside one process, and dsh adds RPC to its own clients. -Chord v2 makes the process boundary part of the model: objects live on the host their placement names, callers are -principals, and a reload rolls out across the server, the workers and every tab. An app like p2, with a process per -session and guests in other browsers, needs that; a single-process agent does not. - -:::review | Review: what a reload does to a running tool call in dsh -The sources say that Loader entries running affected plugins stop and that unloading the agent provider stops its live -agents (`packages/boot/hmr/README.md`, `docs/subsystems/core.md`). They do not say what happens to a tool call in -flight when only the tool's plugin reloads; the comparison above does not claim either way. -::: diff --git a/packages/chord/docs/next/explainer/src/21-epilogue.md b/packages/chord/docs/next/explainer/src/21-epilogue.md deleted file mode 100644 index 987fb3c44..000000000 --- a/packages/chord/docs/next/explainer/src/21-epilogue.md +++ /dev/null @@ -1,104 +0,0 @@ - -# What's Next {.epilogue lede="Where Chord v2 stands, what is still undecided, and where the written sources disagree with each other."} - -The guide built Studio from one object to a p2-sized app. Every piece used the same few rules: one owner per thing, -factories that only declare, state that changes only in transactions, long-lived behaviour in declared forms, and -mappings and migrations owned by the provider. Those rules are what let Mario press Reload on his phone in the -preface while the agent keeps running. This last chapter says how much of that exists and what is still open. - -## Status - -:::stats -- ~10k | lines in the frozen first kernel (steps 1 to 6, 443 tests), kept as a reference only -- 1 of 11 | layers of the rebuild started: L1 contracts, built and reviewed -- 6–7k | target size of Chord v2 without tests -::: - -- **The design is decided** except the points marked below. `packages/chord/docs/next/design.md` is the target; - `packages/chord/docs/next/decisions.md` is the log and wins where they differ. -- **The rebuild** goes layer by layer, each with a one-page spec the owner approves, a minimal build with counting and - garbage-collection tests, the cost-guard metrics, and a cleanup loop: L1 data and contracts; L2 readable and - derive; L3 documents and store; L4 objects; L5 references and `ctx`; L6 protocol and links; L7 relay, routes, - revocation; L8 routines and jobs; L9 registries; L10 reload and rollout; L11 telemetry in Chord. -- **Built:** the contract layer (`src/next/contracts/`, commit `7d2bb8ea8`, chapter XVI). In pi: Delta, - `pi-telemetry`, `chord/context`, `chord/telemetry-sqlite`, pi-ai and Durable telemetry, Durable's required - `checkpointWhen`. -- **Next, in order** (`apps/p2/docs/p3-handoff.md`): decide Q12 and Q15, which the L1 spec needs; write the spec for - the rest of L1 (values, inputs and identity, argument checks, principals, errors; draft in - `packages/chord/docs/next/layer-1.md`); then Q11, Q13, Q14 and the cuts C3 to C17 before the layers they affect. -- **In pi:** `mapShared` and bytes as atomic leaves in Delta's `diffRevisions`. -- **Then** the playground web shell (React, esbuild) with the reload control and a preview at `p3.mariozechner.at`, - and the first real proof: the harness, one tool with an approval pause, a code mode reload. Then the HTTP and - channel components, then p3, domain by domain. - -## Deferred decisions - -Kept with stable numbers in `design.md` §26, each waiting for a concrete case: - -| # | Topic | When | Chapter | -| --- | --- | --- | --- | -| 9 | jobs as objects: a record inside the owner, or an object addressed by its id | when login is ported | XI, XIX | -| 11 | where a flow lives: a keyed document of its owner, or an object `LoginFlow { id }` | when login is ported | XII | -| 14 | project plugins: per-directory plugins with trust prompts and snapshots | own pass over `plugin-loading.md` | XV | -| 16 | a Durable Object profile | when a DO target becomes concrete | | -| 17 | session files: one `SessionFiles { session }` owner on the server (proposed) | p3 design | XX | -| 18 | session delete: retire by tag plus a session directory, or owner hooks | p3 design | XIX | -| 20 | connection pools: one owner of SSH connections per environment, or per-use connections | p3 design | XX | -| 21 | config files with a single writer: one owner on the server, others follow | p3 design | XVIII | -| 22 | file formats (`envs.json`, downloads, uploads) as versioned interfaces | p3 design | XX | -| 23 | the terminal host process: inside the revision or deliberately outside | p3 design | XX | - -## Open from review round 3 - -Decisions (`apps/p2/docs/p3-reviews/v2-review3-summary.md`; Q1 to Q10 are decided): - -| # | Problem | Proposal | -| --- | --- | --- | -| Q11 | a singleton that does not exist yet: `undefined` or `initial()`? | singletons read `initial()`; `undefined` only for absent keyed documents | -| Q12 | stripping arguments (R7) against "nothing is copied" | copy on change: a new object only when the argument had extra fields; never mutate the caller's value (proposed in the L1 draft) | -| Q13 | `ctx.incarnation` on relayed calls | relays stamp the origin host's incarnation, as they stamp the principal | -| Q14 | a writable holding functions exposed as a state member | only members with a JSON schema in the contract are exposable, checked at serve | -| Q15 | arguments of `local` methods: validated or not | apply R7 to `local` too, or keep `local` as the one exception | - -Cuts the owner may choose (each removes a primitive this guide teaches): - -| # | Cut | What it costs | -| --- | --- | --- | -| C3 | shrink jobs to a runner over documents (no versions, `accepts`, `blocked`, `persist`), or move jobs out of the kernel | job versioning, or `self.job` for core-only users | -| C4 | routines: `on` and `loop` only | `every`'s due time across reloads | -| C5 | merge `lifetime: "host"` objects into the facet | a plugin's always-on natives share one failure domain | -| C6 | remove `broken()` for opens and `retryOn` | nothing; reload and Retry still work | -| C7 | remove provider-side `linger` | ten lines of app code | -| C8 | remove `state: false` (method-only references) | a method-only consumer pays one snapshot | -| C9 | remove `previous` from derives | derives that used it move to `mapShared` | -| C10 | one `cell(initial)` for `constant` and `writable` | nothing | -| C11 | remove `ctx.trace` (use `traceIn(ctx, …)`) | nothing | -| C12 | remove call credits (the byte window covers calls) | nothing | -| C13 | two timeouts (`closeTimeout`, `readyTimeout`) instead of eight | per-thing overrides | -| C14 | remove the `handle.change` and `handle.retire` shorthands | nothing | -| C15 | remove `inVariant` and `expectVariant` (keep `RefIn`) | nothing | -| C16 | remove facet-owned mutable documents and their transaction exception | atomic writes across an object and its facet move to one owner | -| C17 | no harness-specific start-barrier exception | factories before the barrier cannot depend on the harness | - -## Where the sources disagree - -This guide follows the decision log where it is newer than the design, and the design where it is newer than -everything else. The places that need a fix in the sources: - -:::review | Review: drift in the written sources -- `design.md` §26 lists only the deferred decisions; it omits the open review-3 items Q11 to Q15 and the cuts C3 to - C17. -- `design.md` §3 has a garbled sentence, "Values are method arguments are stripped". -- `apps/p2/docs/p3-handoff.md` §3 is stale: it says all R items are decided and, two lines later, that R2 to R5 and - R7 to R26 are open, and "resume R6 next". -- `kernel.md` and `step-*.md` describe the frozen code: host values, a development freeze, consumer-side linger, - JSONL storage, descriptions on the wire, and a link closed on an unknown id. None of these is the target. -- `chord-next-walkthrough.md` lags the log: validation by trust (now R7, every caller), dev mode and freeze (removed - by R9), descriptions once per link (gone with R6), an IPC frame-count window (now one byte window, Q4), job outcome - `{ ok }` (now `{ kind }`), harness activation as the start barrier (R17), registries "newer refused at load" (Q8), - and hooks using `host.root` (now `chord(fn, context)`, Q6). -- `apps/p2/docs/questions.md` uses `self.resource` and `replace` (both rejected) and `host.root` in hooks. -- `apps/p2/docs/p3-reviews/v2-cost-model.md`: the measurements are current, its "Decisions this needs" section is - not (chapter XVII). -- L1's `Implementation` shape differs from the factory return value used in the design (chapter XVI). -::: diff --git a/packages/chord/docs/next/explainer/src/22-appendix.md b/packages/chord/docs/next/explainer/src/22-appendix.md deleted file mode 100644 index 42eb965f4..000000000 --- a/packages/chord/docs/next/explainer/src/22-appendix.md +++ /dev/null @@ -1,190 +0,0 @@ - -# API Reference {.appendix #api lede="Every name the guide uses, grouped by where you meet it. Working names; only the contract layer is built."} - -## Hosts - -| Call | Meaning | Chapter | -| --- | --- | --- | -| `createHost({ address?, principal?, storage?, principalKey?, revokedBy?, telemetry?, onError?, timeouts })` | a host; with an address it is addressable and its own code acts as `principal`; without, a client | I | -| `host.load([{ name, facet }])` | load facets by plugin name; base code passes its core facet the same way | I | -| `host.connect(link, { principal })` | the link to the parent (at most one) | I | -| `host.accept(link, { principal })` | a link from a client; every request on it acts as `principal` | I | -| `host.route(Kind, { authorize, resolve })` | how to reach hosts of a kind; `resolve` returns `{ link, principal }` | I | -| `host.revoke(r)` | end matching objects and links here and below | VII | -| `host.root(fn, { principal?, signal?, telemetry? })` | a call scope with a `ctx` for code outside objects | VI | -| `host.get(Type, input, { signal, state?, parkTimeout? })`, `host.hold(…)` | a reference from outside objects; `release()` ends it | VI | -| `host.serve(Type, factory, { lifetime?, linger? })` | serve an object type (in a facet) | II | -| `host.registry(Registry)` | the composed value of a registry, a readable | XIII | -| `host.references(Type)` | who holds references to objects of a type on this host (presence) | VIII | -| `host.activity` | `{ references, calls, jobs, lingering }`, for keep-alive | I | -| `host.close()` | end the host: objects, natives, links | I | - -## Inside a factory: `self` - -| Member | Meaning | Chapter | -| --- | --- | --- | -| `self.state(Definition)` | a document handle | V | -| `self.tx(async (tx) => …, ctx)` | a transaction on the object's line | V | -| `self.derive(name, [sources], fn)` | a derived readable | VIII | -| `self.get(Type, input)` | a reference held for the object's life | VI | -| `self.open(open, close, { closeTimeout? })` | a native; `.value` once open | IX | -| `self.routines.on(readable, fn)`, `.every(interval, fn)`, `.loop(fn)`, `.custom(…)` | standing behaviour | X | -| `self.job(Job, { run, finish? })` | owned work that outlives its call | XI | -| `self.contribute(Registry, value)` | a contribution (facets and `lifetime: "host"` objects only) | XIII | -| `self.writable(initial)` | a host-local readable of any in-process value, with `set` | XIII | -| `self.retryOn(readable)` | a trigger that retries a broken activation | VI | - -**Document handles:** `value`, `get(key)` (keyed), `keys`, `change(fn, ctx)` / `change(key, fn, ctx)` and `retire(ctx)` / `retire(key, ctx)` (shorthands for -one-document transactions). **Transactions:** `tx.draft(handle, key?, { tags? })`, `tx.retire(handle, key?)`, -`tx.start(job, args, { key? })`. **Jobs:** `job.start(args, ctx, { key? })`, `job.abort(id)`, `job.running`, -`run.outcome(ctx)`. - -## References and `ctx` - -| Member | Meaning | -| --- | --- | -| `ref.` | a state member (readable: `.value`, subscribe) or a method | -| `ref.state`, `ref.onState(fn)` | `loading`, `live`, `failed` (cause, `retryAt`), `released` | -| `ref.retry()` | retry a failed reference now | -| `ref.release()` | end a reference taken outside objects | -| `ref.members` | the variant's members | -| `ctx.signal`, `ctx.principal`, `ctx.incarnation`, `ctx.get` | cancellation, caller, calling host's incarnation, call-scoped references | -| `ctx.telemetry`, `ctx.trace(name, attributes, fn)` | the current span; a child span with a child `ctx` | -| `ctx.jobId`, `ctx.progress(value)` | in a job's `run` | - -## Definitions - -| Name | What it declares | Chapter | -| --- | --- | --- | -| `hostKind(name, input?)` | a kind of host and its address | I | -| `remote(name, { version, input, placement, variants })` | an object type reachable across links | II | -| `local(name, { version, input, members })` | an object type on its own host | IV | -| `method(args?, result?)` | a method in a base declaration | II | -| `evolve(prev, (e) => …)`, `checkAccepts(contract, accepts)` | the next contract version; the provider's mappings (**built**) | XVI | -| `Implementation` | the type of a provider for the current version (**built**) | XVI | -| `str num int bool nul lit oneOf union obj arr map opt bytes unknown`, `Static` | schema helpers (**built**) | II | -| `defineState(name, { schema, version, initial, checkpointWhen, persist?, keyed?, ordered?, migrate? })` | a document | V | -| `defineTag(name)` | a tag; `store.retireByTag(tag, value)` | V | -| `defineJob(name, { version, args, progress, result, concurrency?, persist?, accepts? })` | a job type | XI | -| `defineRegistry(name, { version, compose, accepts? })` | an extension point | XIII | -| `bindFacet

    ()` | base code's `defineFacet`, with factories' principals typed as the app's `P` (working name) | III | -| `defineFacet((host, self) => …)` | a plugin's setup for one kind of host; imported from base code (`@studio/core`) | II, III | -| `constant(value)` | a readable that never changes (start data) | IX | -| `broken(error)` | a failure that is not retried by the backoff | VI | -| `inVariant(ref, …)`, `expectVariant(ref, …)`, `RefIn<…>` | narrowing a reference by variant | III | -| `mapShared(fn)` | an array map that keeps unchanged items identical (planned in Delta) | VIII | -| `OutputBuffer`, `Progress` | bounded output and throttled progress (`@earendil-works/chord/output`) | XI | -| `memoryLinkPair()`, `ipcLink(child)`, `webSocketLink(ws)` | link adapters | I | -| `DocumentStorage`, `registerDocumentStorageConformance` | the storage port and its shared test suite | V | - -# Timeouts and Defaults {.appendix #defaults} - -| Timeout or default | Bounds | Default | Override | -| --- | --- | --- | --- | -| `closeTimeout` | a native's `close`, then reported as a possible leak | 30 s | per native | -| call drain | an ending object's own call handlers | = `closeTimeout` | per type | -| `readyTimeout` | activation; an awaited `get` | 60 s | per `get` | -| park deadline | parked calls (link drop, loading, rollout) | 60 s | per `get` (`parkTimeout`); per type | -| routine handover | a new routine waiting for the old handler on reload | 10 s | per routine | -| `finish` retry | backoff of a throwing job `finish` | 1 s to 60 s | per job | -| rollout install | one host installing a revision | 60 s | none | -| linger | an object kept alive after its last reference | per type | per type | -| backoff | retries of transient failures, per address | 1 s doubling to 60 s | none | -| eviction | keyed persisted documents idle in memory | 10 minutes | | -| client WebSocket limits | per part | 16 MiB compressed, 64 MiB decoded | per link kind | -| compression | JSON parts compressed at their origin | above about 16 KiB | | -| worker keep-alive (p2, app code) | idle time before the supervisor stops a worker | 5 minutes | app | - -# Versions {.appendix #versions} - -| Surface | Version | On change | Chapter | -| --- | --- | --- | --- | -| object type | `version` plus fingerprint | `evolve` and `checkAccepts`; otherwise `contract_mismatch` | XVI | -| document | `version` plus schema fingerprint | pure `migrate` on access; no bump fails `broken` | V | -| job | `version` | `accepts` on outcomes and `running`; `blocked` without | XI | -| registry | `version` | `accepts` on contributions; unmatched ones left out and reported | XIII | -| native | none | restarts on reload | IX | -| Durable tasks and documents | Durable's `version` | Durable's `migrate` | XIV | -| link protocol | protocol version | refused; hosts update together | XVII | -| UI components, Chord and app APIs | none yet | exposed as they are | | - -# Errors and States {.appendix #errors} - -`ChordError` has a stable `code` and a finer `reason`; on the wire it is `{ code, reason?, message, details? }` -(specified in `layer-1.md`, not built). - -| Code | Meaning | -| --- | --- | -| `invalid` | input, arguments or placement rejected (`invalid/input`, `invalid/args`, `invalid/placement`) | -| `contract_mismatch` | the consumer's version and fingerprint are neither current nor accepted | -| `forbidden` | the caller may not do this | -| `gone` | the object or hold ended | -| `not_ready` | not activated in time | -| `released` | the reference was released | -| `too_large` | a part exceeded its size limit | -| `unavailable` | could not be reached before the deadline (`unavailable/locked` and others) | -| `aborted` | cancelled by a signal | - -| Type | Values | -| --- | --- | -| `FailureCause` | `{ kind: "transient"; error }` (retried with backoff) or `{ kind: "broken"; error }` (retried only by a trigger) | -| reference state | `loading`, `live`, `failed` (cause, `retryAt`), `released` | -| job outcome | `{ kind: "done"; result }` or `{ kind: "failed"; error: "threw" \| "aborted" \| "lost" }` | -| job phase | `running`, `finishing`, `blocked` | -| rollout status of a provider | `updating` (calls park), rejected (fail at once), same revision (`contract_mismatch`) | -| host install report | `installed`, `installed_failed`, `rejected` | - -# Rejected Designs {.appendix #rejected} - -From `design.md` §31, grouped. Each was considered and is not coming back without a new reason. - -| Area | Rejected | -| --- | --- | -| reload | drain-based handover; object restart for state changes; a factory trial; a migrate dry run; a consumer contract check during staging; reload votes; per-plugin rollback within a host; keeping natives across reloads by name; `restart_required`; host values and a development freeze | -| state | version fences without transactions; volatile documents; per-group lines; optimistic transactions; `tx.get`; a JSONL document storage; validating state and documents on commits or reads; copying values on same-host calls; op-rewriting projections and a view algebra; Durable's Session as the object store | -| objects and references | resources replaced in place (`replace`, `deps`, stale handles); revoking proxies on `.value`; consumer-side linger; detached references counting toward presence; an `optional` flag on references; `opaque` members; callbacks as method arguments; generic call or request ids; a stream type in Chord | -| registries | registries created at runtime; entries resolved through mutable indirection; generic registry keys with a `conflict` flag; numeric middleware priorities; registry composition as a staging check | -| principals and placement | principal field patterns and re-stamping for revocation; `storeKey`; placement as data naming an input field; host-name strings in app code; "magic" input-field matching for delete | -| wire | an outbox for transient delivery errors; cross-relay reset coordination; count limits that refuse calls or holds; contracts taken from the running app | -| process and tooling | a topology layer above the host API; loader variants A and B; Chord ending the process on a storage failure; `defaultProjectTrust: "always"`; ceremony (`native/` directories, `defineNative`, lints); weakening type checking | - -# Glossary {.appendix #glossary} - -**Adapted binding:** a consumer bound at an older accepted contract version, served through the provider's mappings. -**Base code:** the app's own code, imported statically; a change restarts the hosts. **Client:** a tab or TUI host, -not addressable. **Contract:** an object type's declaration (name, input, placement, members, version). -**Contribution:** a value a plugin adds to a registry. **`ctx`:** what every call receives. **Derive:** a value -computed from readables. **Document:** a named JSON value an object owns. **Facet:** a plugin's code for one kind of -host. **Factory:** the synchronous function that declares an object. **Fingerprint:** a hash of a contract's or -schema's normal form. **Flow:** input into running work, kept as state by its owner. **Host:** one participant of the -app, with an identity and an incarnation. **Incarnation:** the id a host gets on every start. **Input:** an object's -key among objects of its type; with the type and the caller's principal key it identifies the object. **Job:** owned work that outlives its call. **Link:** an ordered connection between -two hosts. **Local type:** an object type reachable only on its own host, without principals. **Native:** an OS-level -value opened with `self.open`. **Park:** a call waiting for a reference to become live again. **Placement:** the -function from input to host address. **Plugin:** a directory with a `package.json`, reloadable. **Principal:** who -calls, stamped by the link. **Principal key:** the string that identifies a principal in object identities. **Pure -object:** an object without natives; swapped in place on reload. **Readable:** a value plus its ordered changes. -**Reference:** what `get` returns. **Registry:** an owner's extension point. **Relay:** a host forwarding messages -between two links, rewriting only headers. **Reload:** swapping a plugin's code in running hosts. **Restart:** ending a -process. **Revision:** the complete code set the hosts should run. **Revocation:** ending live access for principals -matched by `revokedBy`. **Routine:** standing behaviour of an object. **Server:** the addressable root host. **Shared -store:** a local object that holds data once behind per-principal views. **Tag:** a typed label on documents, used to -retire them together. **Transaction:** an atomic set of document writes on the object's line. **Variant:** the set of -members a principal gets. **View:** a per-principal remote object in front of a shared store. **Worker:** a host the -server starts for one session. - -# Sources {.appendix #sources} - -| Source | Used for | Weight | -| --- | --- | --- | -| `packages/chord/docs/next/decisions.md` | every decision, review rounds 2 and 3 | wins where newer | -| `packages/chord/docs/next/design.md` | the target design, all chapters | authoritative otherwise | -| `packages/chord/src/next/contracts/` and its `README.md` | chapter XVI | built code | -| `packages/chord/docs/next/layer-1.md` | errors, inputs, argument checks | draft spec | -| `apps/p2/docs/p3-reviews/v2-review3-summary.md` | open Q11 to Q15, cuts C3 to C17 | current | -| `apps/p2/docs/p3-reviews/v2-cost-model.md` | measurements, per-hop costs | current except its last section | -| `apps/p2/docs/p3-functions/` | chapters XVIII to XX | current | -| `apps/p2/docs/p3-handoff.md` | status and order of work | §3 partly stale | -| `packages/durable/README.md` | the harness, submit, tasks, subagents | current | -| `chord-next-walkthrough.md`, `kernel.md`, `step-*.md`, `apps/p2/docs/questions.md` | background only | lag the decisions (see What's Next) | -| DeepSeek Harness at `687f282` (its Cordis tutorial, architecture, persistence, core and HMR docs), the Cordis paper (arXiv 2608.25512) | "Other Ways to Do It" | external, built systems | diff --git a/packages/chord/docs/next/explainer/walkthrough.md b/packages/chord/docs/next/explainer/walkthrough.md deleted file mode 100644 index e4bfa5a9e..000000000 --- a/packages/chord/docs/next/explainer/walkthrough.md +++ /dev/null @@ -1,284 +0,0 @@ ---- -title: Chord v2 -kind: walkthrough -kicker: Walkthrough · Chord v2 -subtitle: Build an app like p2 on Chord v2 and pi-durable, one concept at a time, with a tutor that draws the figures and asks the questions -model: anthropic/claude-sonnet-5-5 -sources: - - packages/chord/docs/next/explainer/book.md - - packages/chord/docs/next/design.md - - packages/chord/docs/next/decisions.md - - packages/chord/docs/next/layer-1.md - - packages/chord/src/next/contracts/README.md - - packages/chord/src/next/contracts/*.ts - - packages/durable/README.md -truth: treat Chord v2 as fully built as designed; ignore what the sources say about build status. redesign-decisions.md wins over design.md; design.md wins over the guide (book.md) ---- - -:::tutor -The learner knows pi-durable (harness, conversations, tasks, tools) and nothing about Chord. - -Teach Chord v2 as a finished, working runtime: never say a part is "not built", "planned" or "the target design", even -though the sources say so in places. Open decisions may be mentioned as "still being decided" when a concept depends on -one. - -The running example is Studio, introduced on the page before the first concept (see the introduction). Use it in every -concept and grow it step by step, as the guide book.md does chapter by chapter: read the matching guide chapter before -each concept, and check details in design.md and the decision log. Name the guide chapter in "Read more" lines. - -Keep code short and complete enough to read on its own. Prefer one figure per concept: a sequence for message orders, -a flow for structure, a stepper for lifecycles. -::: - -:::intro -#### What this is about - -Picture **p2** (Pi Mobile): a web app for working with pi agents from any device. It runs on your laptop as one -**server** process. Every chat session gets its own **worker** process, and in each worker a pi-durable agent works -through its turns. You use it from **browser tabs** on your desktop or phone, and you can invite a colleague to watch. -Most features (a notes panel, file browsing, terminals, provider logins) are **plugins**. - -Now you change one line in a plugin and press Reload, while an agent is twenty minutes into a task. The panel should -show the new code on every screen. The agent's turn, the running tool, the open connections and every bit of saved -data should simply continue. No restart, no lost work. - -**Chord v2** is the runtime that makes this normal. An app on Chord is a set of **objects** spread over the server, the -workers and the tabs. Each object owns its data and its OS handles, declares what it needs, and Chord creates, -connects, reloads and ends it. Because Chord knows every piece of state an object has, it can swap the object's code -while its data stays. - -#### The example: Studio - -This walkthrough builds **Studio**, a small p2, on Chord v2 and pi-durable. It has the same three kinds of processes: - -:::flow wide | Studio: one server, a worker per session with its agent, browser tabs as clients. The notes plugin is the first feature we build. -row "Clients" muted -box owner "owner's tab" "phone or desktop" -box guest "guest's tab" "read-only" -row "Server" accent -box server "server" "session list, access" accent -row "Workers" blue -box worker "worker S1" "agent + notes" blue -box worker2 "worker S2" "agent + notes" blue -arrow owner -> server "WebSocket" -arrow guest -> server "WebSocket" -arrow server -> worker "IPC" -arrow server -> worker2 "IPC" -::: - -The first feature is a **notes plugin**: a panel where the owner adds notes to a session, a guest reads them, and the -agent can add notes through a tool. Each concept adds one piece to Studio, until it has everything a p2 needs: who may -do what, saved data, live updates, background jobs, logins, extension points, the agent, and reload. - -#### How it progresses - -1. **The pieces:** processes, objects, callers, shared data, saved data. -2. **Living objects:** references that survive restarts, revocation, computed values, OS handles, standing behaviour, - background jobs, input from people. -3. **The app:** extension points between plugins, and the pi-durable agent inside Chord. -4. **Change:** reloading code while everything runs, evolving contracts, and what goes over the wire. -::: - -# What we're building {#goal part="The pieces"} -Guide preface ("Before we begin", "Ten things to know"). Must cover, briefly and concretely: the reload scenario (phone, -agent mid-turn, guest watching, one-line fix, Reload) and why restarting processes or hot-swapping modules fails; the -one rule behind Chord (every piece of state, OS handle and running work has one owner, an object, and objects only -declare what they need); Studio's pieces from the introduction. Do not go into APIs yet. - -:::check -1. Why is restarting the worker processes not a good way to load new plugin code? -+++ answers -1. It loses the agent's turn in flight, open terminals and connections (pi-durable would resume the turn, but the running tool and the connections are gone). -::: - -# The shape of an app {#shape part="The pieces"} -Guide chapter I. Must cover: hosts (server, workers, clients), links, host kinds and placement, createHost/load/connect/accept/route, -clients reach workers only through the server, incarnations, keep-alive is app code (host.activity). - -:::check -1. A tab wants an object that lives on worker S1, which is not running. What happens, in order? -2. Why does Chord have no listen() call? -+++ answers -1. The tab's call goes to the server; the address is not the server's, so it uses route(Session): authorize, then resolve starts the worker and returns a link plus principal; the server relays the hold, the worker builds the object and answers with a snapshot. -2. Entry points differ too much (loopback WebSocket, Radius channels, a Durable Object edge, memory links); each authenticates and calls accept. -::: - -# A first object {#first-object} -Guide chapter II. Must cover: contract (name, input, placement, variants, version): the input is the object's key -among objects of its type, `placement` computes the host from it; schema helpers, facet and factory (a facet imports -Studio's own defineFacet from @studio/core), serve and get, methods return data, placement checked on both sides. - -:::check -1. Two tabs of the owner ask for Notes of S1. How often does the factory run? -2. A caller with an outdated placement rule asks worker S2 for Notes of S1. What happens? -+++ answers -1. Once per principal key; the owner's two tabs on the same access path share the object, so once. -2. The provider runs the placement function itself and refuses with invalid/placement. -::: - -# Who is calling {#principals} -Guide chapter III. Must cover: the app's principal type, links stamp principals, principal key in the object identity, -variants per principal, narrowing with inVariant/RefIn, and how a facet knows P: base code exports defineFacet bound to -P (bindFacet

    ()), every facet imports it, and host.load accepts only facets typed for its P. - -:::check -1. Can a client send a principal field to claim it is the owner? -2. Why do guests not get an add method that throws "forbidden"? -3. The notes plugin's worker facet narrows principal.kind === "guest". Where does the type checker get Studio's principal union from? -+++ answers -1. No: a client's principal field is always overwritten by the link's stamp; only a trusted parent link may carry one. -2. Variants: the reader variant simply has no add member, and Chord refuses calls to members outside the caller's variant. -3. From Studio's base code: @studio/core exports defineFacet = bindFacet

    (), the facet imports that one, and the worker's host.load accepts only facets typed for the same P. -::: - -# One store, many views {#shared-store} -Guide chapter IV. Must cover: local types, the shared store pattern, attribution as data, argument validation for every caller (R7, Q15 open). - -:::check -1. Where does "who wrote this note" come from if the store is local and has no principal? -+++ answers -1. The remote view knows the caller and passes the author into the store's method as data. -::: - -# Writing it down {#documents} -Guide chapter V. Must cover: defineState, self.tx, drafts and ops, persistence only for stable identities, versions with migrate, tags, storage. - -:::check -1. Why may a remote per-principal object not persist documents? -2. A schema changes but the version is not bumped. What happens? -+++ answers -1. Its identity includes the principal key, which changes (devices, grant revisions); persisted data needs a stable identity, so it lives in a local store or a host-lifetime object. -2. The stored fingerprint differs from the definition's at access, and the object fails broken ("schema changed without a version bump"). -::: - -# Holding on {#references part="Living objects"} -Guide chapter VI. Must cover: reference lifecycle, ref.state, parking, transient vs broken failures, backoff, ctx. - -:::check -1. The link to a worker drops while a tab calls a method. What does the call do? -+++ answers -1. A call already in flight fails like on a disconnect; calls made while the link is down park until the reference is live again (up to the park deadline, 60 s), then fail unavailable. -::: - -# Taking access away {#revocation} -Guide chapter VII. Must cover: revokedBy, host.revoke, propagation to children, no re-stamping, lasting denial in app data. - -:::check -1. After revoke({ grant: "g1" }), Ann reloads her tab. Why does she not get access back? -+++ answers -1. revoke cuts only live access; the app's guest record is blocked and authenticate refuses it on the new connection. -::: - -# Computed values {#derives} -Guide chapter VIII. Must cover: derives, the footgun (new objects for unchanged data), mapShared, container width, presence with host.references. - -:::check -1. Why is items.map(stripAuthor) in a derive expensive on every commit? -+++ answers -1. It builds new objects for every unchanged note, so the diff walks and sends O(value) instead of O(change); mapShared keeps unchanged items identical. -::: - -# Things the OS gives you {#natives} -Guide chapter IX. Must cover: natives with self.open, pure vs native objects, start data with constant, lifetime host, call-scoped natives. - -:::check -1. What happens to a native object on reload, and why? -+++ answers -1. It restarts: its natives close and reopen with the new code, because old and new code sharing an OS handle would interleave unchecked. -::: - -# Standing behaviour {#routines} -Guide chapter X. Must cover: routines (on, every, loop), serialized handlers, reconciling instead of notifying, pub/sub reports with ctx.incarnation. - -:::check -1. Why does archiving a session not call notes.archive(session) directly? -+++ answers -1. A notification can be lost (worker not running, link drops, crash); a routine on the catalog's sessions reconciles from the current value, also after a crash. -::: - -# Work that outlives the call {#jobs} -Guide chapter XI. Must cover: defineJob, the intent / effect / outcome sandwich, dedupe of unsettled runs (Q9), finish in current code, abort as a request. - -:::check -1. A shell job's run crashes after writing the result entry. How is a duplicate entry avoided? -+++ answers -1. The external write carries a key derived from ctx.jobId (requestId shell:jobId), so Durable dedupes it. -::: - -# Asking a person {#flows} -Guide chapter XII. Must cover: a flow is state, not a call; steps helper; the Questions service; decisions #9 and #11 open. - -:::check -1. Why not a long-held step() call that returns the next login step? -+++ answers -1. A link drop fails it and the consumer cannot tell if it missed a step; state re-holds by itself and shows the current step. -::: - -# Extension points {#registries part="The app"} -Guide chapter XIII. Must cover: registries, compose, who may contribute (R21), contributions that change, optional dependencies, HTTP component, channels, chunked transfers. - -:::check -1. Radius needs forwards from the ports plugin, which may be missing. How is that expressed? -+++ answers -1. Radius owns a registry (radius.forwards) and ports contributes to it; without ports the table is empty. -::: - -# The agent {#agent} -Guide chapter XIV. Must cover: the Harness as a core host-lifetime object, HarnessModels / HarnessEnvironments / HarnessExtensions registries, chord(fn, context) for tools, approval via Questions. - -:::check -1. How does a Durable tool get a Chord ctx? -+++ answers -1. Through chord(fn, context): the host is in Durable's context under a context key, and host.root gives fn a ctx. -::: - -# Reload and rollout {#reload part="Change"} -Guide chapter XV. Must cover: load, staging, commit, per-object swap or restart, rollback all-or-nothing per host, revisions, rollout order, updating vs contract_mismatch. - -:::check -1. A staging error happens in one of three plugins being reloaded on a host. What runs afterwards? -+++ answers -1. The whole host stays on its previous code; rollback is all-or-nothing per host. -::: - -# Changing contracts {#contracts} -Guide chapter XVI; the code is in src/next/contracts. Must cover: evolve, checkAccepts and mapping directions, binding by version and fingerprint, new history bases, the fingerprint normal form. - -:::check -1. Which way does a state mapping go, and which way an args mapping? -2. Why can an input change not be adapted? -+++ answers -1. State: current to old (the old consumer reads). Args: old to current (the old caller sends). -2. The input is the object's address; changing it changes what the object is, so it needs a new history base and old consumers get contract_mismatch. -::: - -# On the wire {#wire} -Guide chapter XVII. Must cover: messages and parts, relays rewrite only headers, one state message per commit per link, flow control (R19, Q3), size limits per part, the cost model, telemetry. - -:::check -1. What does the server do with a state part it relays from a worker to a tab? -+++ answers -1. It reads and rewrites only the header (maps ids, groups parts per destination link) and forwards the part without decoding it. -::: - -# Other ways to do it {#compared part="In context"} -Guide chapter "Other Ways to Do It" (after chapter XX). Must cover: Cordis (Koishi's plugin framework since 2019) and -DeepSeek Harness (dsh, everything is a plugin, built on Cordis): reversible effects on a plugin fiber, inject-based -services, reload by unload-and-load; dsh's durable session log and its "interrupted" crash repair. Then what the -guarantees are in practice (the chapter's section of that name): disposers run concurrently and are only as good as -their authors, listener order follows registration order and changes on reload, services are untyped strings and a -provider reload restarts all dependents, and reloading a core service stops live agents. Then the two differences that -matter: what survives a reload (Cordis: nothing of the plugin's or its dependents' in-memory state; Chord: a pure -object's data and running work) and reach (one process vs. a tree of hosts with principals). Say what Chord costs: -more concepts and machinery. A flow or table comparing a reload in both helps. - -:::check -1. A Cordis plugin keeps a list of notes in a field of its instance. What happens to the list when the plugin hot-reloads, and what would the same plugin do on Chord v2? -2. Two Cordis plugins register waterfall listeners on the same tool-policy event; the first one short-circuits. What changes after the first plugin is hot-reloaded? -3. Why does an app like p2 need Chord's multi-host model, when dsh runs its plugin tree in one Host process? -+++ answers -1. The old fiber unloads and the new one starts empty, so the list is gone unless it was kept in a service or on disk. On Chord the notes are a document of a pure object; a reload swaps the object's code and keeps the document. -2. Its listener is registered again at the end of the list, so the other plugin's listener now runs first and the decision can change, unless the listener was registered with prepend. -3. p2 has a worker process per session and clients (including guests) in other browsers; objects must live on the right host, callers must be stamped principals, and reload has to roll out across server, workers and tabs. A single-process tree does not cover that. -::: - From 9b7a44c48d736125e63c4ba3f49a5a0377431111 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 08:22:49 +0000 Subject: [PATCH 195/271] docs(chord): slice 11 draft, P1-P2, one definition per facet kind --- packages/chord/docs/next/decisions.md | 16 +++++ packages/chord/docs/next/design.md | 2 +- packages/chord/docs/next/slice-10.md | 2 +- packages/chord/docs/next/slice-11.md | 93 +++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 packages/chord/docs/next/slice-11.md diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index e28177bbb..d1f4cfd4b 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1988,6 +1988,22 @@ validation and copies the v2 kernel added around Delta. entry lives as long as its contributor and consumers keep nothing beyond a use) A plugin socket is closed (1012, "service restart") when the contribution that handed it out leaves the table (a reload changing or removing it); a `/chord` link or socket whose `Host` is no longer allowed closes (1008) when the allowed hosts change. +- **Slice 11's points (owner, 2026-10-30; `slice-11.md`):** the surfaces are generic plumbing; the app keeps + everything visible (owner: "this one needs to be generic, so apps can do whatever they want, css, tui components, + themes"). (P1, "a") `chord/next/web` ships serving the surface (the base bundle, the plugins' web scripts, + stylesheets and assets, the page through the app's template, all as `Http` contributions), the browser's client host + with its connection loop, and loading and applying the plugins' web facets; not HTML, CSS, components, themes, + navigation, slots or layout. + (P2, "script tag") A plugin's web script is served as a file that registers itself + (`chordFacet("", "", function (require, module, exports) { … })`) and is loaded with ` + From e22468919b33eee0eb29c750204d8de32da2bd5d Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 13:47:28 +0000 Subject: [PATCH 199/271] docs(chord): HTTP server as a native object; the opinionated app kit as the next design pass --- packages/chord/docs/next/decisions.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index dd0f2421b..4c35c4f33 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2051,6 +2051,18 @@ validation and copies the v2 kernel added around Delta. undefined }]`; admission calls them for a request that would be refused for forwarded headers on loopback (p2's Radius relay); a verified one gets `provenance: { kind: "relayed", relay, origin }` and its framer allowed for it, an unverified one stays refused; `authenticate` decides its principal. +- **The HTTP server is a native object (owner, 2026-10-31: "still unconvinced the server can't be a native object; it's a + singleton, unlikely to change"; supersedes D3 and the `httpServer` function of D1):** `chord.http`'s facet serves the + registry `Http` and a host-lifetime object whose native is the listener; its configuration (`HttpConfig`: port, + links path) and `Authenticators` are registries the app (or the app kit) contributes to. The facet is passed as the + same object at every reload, so the listener never closes before the process ends. The port opens right after + `load`'s publication (a few hundred milliseconds later than listening before `load`; a reconnecting tab retries). +- **The app kit is opinionated (owner, 2026-10-31: "the app kit should be super duper opinionated and make for small + app code"; "i suppose"):** after slice 10's fix round and before slice 11, a design pass on `chord/next/app`: one + entry (`startApp`, `startWorker`), conventions for layout, hosts, principals, authentication, storage, plugins, web + and TUI, decided point by point; the slice 10 components are its inside. Inputs from the owner: user management is + part of it ("user management will also need to be a thing we support"), and app shapes without workers (server and + clients only: "a chat app doesn't need workers"). - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From 081f28bad7c40d1ca42370529a77a404dca0bdd7 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 13:50:20 +0000 Subject: [PATCH 200/271] docs(chord): tutorial on hosts and plugins --- .../docs/next/guide-plugins-hosts/book.md | 319 ++++++++++++ .../docs/next/guide-plugins-hosts/index.html | 463 ++++++++++++++++++ 2 files changed, 782 insertions(+) create mode 100644 packages/chord/docs/next/guide-plugins-hosts/book.md create mode 100644 packages/chord/docs/next/guide-plugins-hosts/index.html diff --git a/packages/chord/docs/next/guide-plugins-hosts/book.md b/packages/chord/docs/next/guide-plugins-hosts/book.md new file mode 100644 index 000000000..2ffe01d53 --- /dev/null +++ b/packages/chord/docs/next/guide-plugins-hosts/book.md @@ -0,0 +1,319 @@ +--- +title: Hosts and Plugins: a tutorial for Chord v2 +cover: Hosts / and Plugins +kicker: A guide · Chord v2 · 2026-10-31 +subtitle: Build a small notes app step by step, from one host to plugins on a server and on workers, edited while it runs +stats: + - 3 | kinds of host in an app: server, worker, client + - 1 | file per host kind in a plugin + - 0 | restarts to apply a plugin edit + - 1 | rule: plugins talk only through object types and registries +note: | + A tutorial for people building apps on Chord v2, written by the redesign session from `design.md`, `slice-7c.md`, + `slice-10.md`, the decision log and the code in `packages/chord/src/next` and `src/next-node`. API names are the + current ones; where slice 10's fix round or the app kit (`chord/next/app`, being designed) will change what app code + writes, a note says so. +colophon: Hosts and Plugins · first edition · 2026-10-31 · Chord v2 slices 1-10 +--- + +# Before we begin {.preface} + +You want to build a notes app. You type a note in a browser tab, and every other tab sees it at once. Each project +gets its own notes in its own process, so a project that misbehaves cannot take the others down. And you, or an agent +you work with, add features while the app runs: a search, an export button, a word count, without restarting +anything. + +Chord gives you three things for that. **Hosts**: the processes of your app (the server, a worker per project, the +browser tabs), connected to each other. **Objects**: the state and methods your app is made of, living on one host and +used from any other. **Plugins**: directories of code that declare objects for each kind of host, loaded and +reloaded while the app runs. + +This book builds the app in that order. Each chapter adds one concept and shows complete code. + +## Seven things to know + +:::findings +1. **A host is one participant.** The server, each worker and each browser tab is a host; hosts connect over links. +2. **An object has a type, and a type is a contract.** The contract names the object's state and methods; any host can + ask for an object by type and params, and Chord finds where it lives. +3. **A facet declares, an object acts.** A facet is a plugin's code for one host kind: it says which object types this + host serves. The objects hold the state and do the work. +4. **A plugin is a directory.** A `package.json` with the plugin's name, and one file per host kind: `server.ts`, + `worker.ts`, `web.tsx`. +5. **The server builds, every host evaluates.** The plugins component builds each facet into a small script; the + server, workers and tabs load the scripts for their kind. +6. **Plugins meet through types and registries.** One plugin uses another's objects through its contract, or + contributes to a registry another defines. Nothing else crosses between plugins. +7. **An edit is a rollout.** Change a file, build, roll out: the server, then every worker, swap the plugin's objects in + place; a broken edit keeps the old code running and reports why. +::: + +## Words used throughout + +A **host** is one participant of the app, made with `createHost`. A **host kind** (`Server`, `Session`) describes +addressable hosts; an **address** (`Session({ id: "p1" })`) names one. An **object type** is a contract: `local` (used +on its own host only) or `remote` (used from any host). An **object** is one instance of a type, per params. A +**facet** is a plugin's declarations for one host kind. A **plugin** is a directory with a `package.json`. A +**reference** is what you get when you ask for an object (`host.get`, `self.get`, `ctx.get`). A **registry** is an +extension point: one plugin defines it, others contribute. A **rollout** applies a build to the hosts in order. + +# One Host {#host lede="The smallest app: one process, one object type, one call."} + +Start without plugins. One host, one object that keeps notes: + +```ts +import { createHost, defineFacet, defineState, local, method, obj, arr, str } from "@earendil-works/chord/next"; + +// the contract: what a Notes object looks like from outside +const Notes = local("notes.store", { + version: 1, + params: {}, // one object per host: no params + members: { + list: obj({ items: arr(str()) }), // state anyone can read and watch + add: method(obj({ text: str() })), // a method: arguments checked by Chord + }, +}); + +// the document the object keeps +const NotesDoc = defineState("notes", { schema: obj({ items: arr(str()) }), version: 1, initial: () => ({ items: [] }) }); + +// the facet: which types this host serves, and how to make each object +const facet = defineFacet((host) => { + host.serve(Notes, (_params, self) => { + const doc = self.state(NotesDoc); + return { + list: doc, + add: async ({ text }, ctx) => self.tx(async (tx) => { (await tx.draft(doc)).items.push(text); }, ctx), + }; + }); +}); + +const server = createHost({ onError: console.error }); +await server.load([{ name: "@app/notes", facet }], ctx); +const notes = await server.get(Notes, {}, ctx); // a reference: live, with its state +await notes.add({ text: "buy milk" }, ctx); +console.log(notes.list.value); // { items: ["buy milk"] } +``` + +Four things happened. The **contract** `Notes` says what the object offers. The **facet** says this host serves +`Notes`, with a **factory** that declares the object's parts: a document, and a method writing it in a transaction. +`load` publishes the facet. `get` returns a **reference** once the object is live; its members are the contract's. + +The factory only declares: it is synchronous and opens nothing. Chord activates the object (loads its documents, opens +what it asked to open) and then serves calls. That is why the same factory can run again when the code changes. + +# Several Hosts {#hosts lede="A server, a worker per project, browser tabs: how they find each other's objects."} + +Now give each project its own process. Declare the kinds of addressable host, and make notes a `remote` type that +lives on its project's worker: + +```ts +export const Server = hostKind("server"); +export const Project = hostKind("project", { id: str() }); + +export const Notes = remote("notes", { + version: 1, + params: { project: str() }, + hostAddress: ({ project }) => Project({ id: project }), // where an object of these params lives + variants: { editor: { state: { list: NotesSchema }, methods: { add: method(AddArgs) } } }, +}); +``` + +`hostAddress` is the whole routing story. A tab asks for `Notes { project: "p1" }`. Its host computes the address, +`Project(p1)`, and sends the request to its parent, the server. The server has a **route** for `Project`: it starts (or +finds) the worker for `p1` and passes the request on. The worker serves `Notes`. The tab gets a reference whose state +follows the worker's document, one frame per change. + +:::flow wide animate | A tab holds a project's notes: the server routes by address to the project's worker and relays, without reading the data. +row "Clients" blue +box tab1 "tab A" "holds Notes { project: p1 }" +box tab2 "tab B" "holds Notes { project: p1 }" +row "Server" accent +box server "server" "route Project → worker" +row "Workers" muted +box w1 "worker p1" "serves Notes { p1 }" +box w2 "worker p2" "serves Notes { p2 }" +arrow tab1 -> server "hold" +arrow tab2 -> server "hold" +arrow server -> w1 "relay" +::: + +Each host is created the same way, with what it is: + +```ts +// server.ts +const server = createHost

    ({ address: Server(), principal: { kind: "server" }, onError }); + +// worker.ts (started by the server for one project) +const worker = createHost

    ({ address: Project({ id }), principal: { kind: "worker", project: id }, onError }); +worker.connect(parentLink, { principal: { kind: "server" }, trusted: true }, ctx); + +// a browser tab: no address, it is a client +const tab = createHost

    ({ onError }); +tab.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }, ctx); +``` + +`P` is your app's type of **principal**, who is calling: an owner, a guest, the server, a worker. Links stamp it; +a `remote` object is made per principal, so a guest can get a read-only variant of the same notes. + +:::note | Coming with the app kit +Starting workers, routing a host kind to them, the WebSocket entry and authentication are slice 10's components today +(`createWorkers`, `chord.http`), wired by the app. The opinionated app kit (`chord/next/app`, being designed) will do +this wiring for you: `startApp(...)` and `startWorker()`. +::: + +# Plugins {#plugins lede="Move the code into a directory that can be added, edited and removed while the app runs."} + +So far the facets were part of the app's own code. A **plugin** is the same code in its own directory: + +```text +plugins/notes/ + package.json { "name": "@app/notes" } + contract.ts the Notes contract (other plugins import it) + worker.ts the facet for workers: serves Notes + server.ts the facet for the server: an export route + web.tsx the facet for browser tabs: a notes panel +``` + +The plugin's name is its `package.json` name. Each file named after a **host kind** is that kind's facet, its +default export a `defineFacet(...)`. A plugin has only the files it needs: a plugin that only adds a route has a +`server.ts` and nothing else. + +```ts +// plugins/notes/worker.ts +import { defineFacet } from "@app/core"; +import { Notes } from "./contract.ts"; + +export default defineFacet((host) => { + host.serve(Notes, ({ project }, principal, self) => { + const doc = self.state(NotesDoc); + return { variant: "editor", members: { list: doc, add: async ({ text }, ctx) => /* … */ } }; + }); +}); +``` + +`@app/core` is your app's **base code**, imported by every plugin as a host package (the plugin's build leaves it +out; every host passes its own copy in). It holds what plugins share and never reloads: host kinds, the principal +type, a `defineFacet` bound to it. + +# Loading Plugins {#loading lede="Discover, build, evaluate, load: what happens between a directory and a running object."} + +The **plugins component** does the work. You tell it where plugins are and what each host kind needs: + +```ts +const plugins = createPlugins({ + roots: ["./plugins"], + kinds: { + server: { platform: "node", hostPackages: ["@earendil-works/chord/next", "@app/core"] }, + worker: { platform: "node", hostPackages: ["@earendil-works/chord/next", "@app/core"] }, + web: { platform: "browser", hostPackages: ["@earendil-works/chord/next", "@app/core", "react", "react/jsx-runtime"] }, + }, + onError, +}, ctx); + +const report = await plugins.load(server, { + kind: "server", + core: [{ name: "@app/core", facet: coreServerFacet }], + modules: { "@earendil-works/chord/next": chord, "@app/core": core }, +}, ctx); +console.log(report.failed); // plugins left out, with why: [] when all loaded +``` + +:::steps | What `plugins.load` does ++++ Discover +It lists `./plugins` one level deep and reads each `package.json`. A directory without one, or with two `server` +files, is reported, not loaded. ++++ Build +For each plugin with a `server` facet, esbuild builds one script from `server.ts` and everything it imports: the +plugin's own files, another plugin's contract files, its npm dependencies. Host packages stay out. A syntax error is a +build failure, never a crash. ++++ Evaluate +The script runs and returns the facet. Its `import "@app/core"` gets the server's own `@app/core`, so a `ChordError` +from a plugin is the same class as the server's. ++++ Load +`host.load` runs every facet's setup and publishes all of them at once. A plugin whose build, evaluation or setup +fails is left out and listed in `report.failed`; the others load. Its objects' users wait, and the next edit loads it. +::: + +A worker never builds: the server builds the worker facets too, and each worker evaluates the scripts it is sent. + +# Plugins Using Plugins {#together lede="Two ways one plugin uses another, and the one it may not."} + +A search plugin wants to search the notes. It **imports the contract** and asks for the object: + +```ts +// plugins/search/worker.ts +import { Notes } from "../notes/contract.ts"; // copied into search's script: contracts only + +host.serve(Search, ({ project }, principal, self) => { + const notes = self.get(Notes, { project }); // a dependency: Search goes live once Notes is + const index = self.derive([notes.list], ([list]) => buildIndex(list.items)); + return { variant: "main", members: { find: async ({ q }) => lookup(index.value, q) } }; +}); +``` + +`self.get` in a factory declares a dependency: Chord makes `Search` live only once `Notes` is, and if the notes +plugin is missing, `Search` waits with a clear reason ("waiting for notes: not served here") instead of failing. + +The other way is a **registry**: an extension point one plugin defines and others contribute to. The app's core +defines `Panels`, and every plugin with a UI contributes one: + +```tsx +// plugins/notes/web.tsx +export default defineFacet((host, self) => self.contribute(Panels, { title: "Notes", Component: NotesPanel })); +``` + +What a plugin may not do is reach into another's data. Search never reads notes' documents; it uses the `Notes` +object's state. That rule is what lets the notes plugin change its storage without breaking search. + +# Changing a Plugin {#reload lede="Edit, build, roll out: how new code takes over while every tab stays connected."} + +You change the notes plugin: notes now have a timestamp. The watcher sees the saved file and marks the plugin dirty. +You (or an agent) build and roll out: + +```ts +const built = await plugins.build(plugins.state.value.dirty, ctx); +const result = await plugins.rollout([plugins.step("server", server), workers.step("workers")], built, ctx); +``` + +The rollout goes host by host, server first, then every running worker. On each host Chord compares the new plugin +set with the loaded one, runs the new factories against a recording `self` to see what each object would declare, and +then gives every object of the changed plugin its fate: + +| The object | What happens | +| --- | --- | +| holds no natives (open files, sockets), the new code neither | **swapped** in place: new methods, same documents; running calls finish on the old code | +| holds natives | **restarted**: calls wait, running ones finish, natives close and reopen with the new code | +| its type was removed | **ended**; its users wait until something serves it again | + +The tab's reference follows the new object and gets one frame with the difference. Nothing reconnects. If the edit +is broken (a syntax error, a factory that throws), that plugin keeps its old code on that host and the report says why; +every other plugin of the rollout applies. + +:::note | Versions +A change to a contract (a new field in `Notes`'s state) gets a new `version`. A tab built with the old contract keeps +working if the provider maps the old version (`accepts`); otherwise it waits with "version mismatch" until it reloads. +::: + +# Base Code and Plugins {.epilogue lede="What reloads, what restarts, and where the line is."} + +Everything in `plugins/` reloads: edit, build, roll out. Everything in your base code (`src/`, `@app/core`: host +kinds, principals, the core facet, the app's startup) changes only with a restart, and the watcher tells you so. +Keep base code small: host kinds, principals, and what every plugin needs. Put features in plugins. + +The app kit being designed next (`chord/next/app`) will make the base code nearly empty: `startApp` and `startWorker` +with Chord's conventions for hosts, users, authentication and storage, so an app is mostly plugins. + +# Reference {.appendix} + +| Call | What it does | +| --- | --- | +| `createHost

    ({ address?, principal?, store?, onError })` | a host: addressable with an address, else a client | +| `host.load([{ name, facet }], ctx)` / `host.reload(...)` | publish facets / apply a new plugin set | +| `host.get(Type, params, ctx)` / `host.hold(...)` | a live reference / a reference at once that follows its object | +| `local(name, { version, params, members })` / `remote(name, { version, params, hostAddress, variants })` | contracts | +| `defineFacet((host, self) => …)`, `host.serve(Type, factory)`, `self.contribute(R, value)` | a facet | +| factory `self`: `state`, `derive`, `get`, `registry`, `open`, `on`, `onLive`, `job`, `tx` | an object's parts | +| `createPlugins({ roots, kinds, onError }, ctx)` | the plugins component | +| `plugins.load(host, { kind, core, modules }, ctx)`, `build(names)`, `rollout(steps, built)` | load, build, roll out | +| `hostKind(name, params?)`, `Kind(params)` | host kinds and addresses | diff --git a/packages/chord/docs/next/guide-plugins-hosts/index.html b/packages/chord/docs/next/guide-plugins-hosts/index.html new file mode 100644 index 000000000..b026d7e8e --- /dev/null +++ b/packages/chord/docs/next/guide-plugins-hosts/index.html @@ -0,0 +1,463 @@ + + +Hosts and Plugins: a tutorial for Chord v2 + + + + +

    +
    +
    A guide · Chord v2 · 2026-10-31
    +

    Hosts
    and Plugins

    +

    Build a small notes app step by step, from one host to plugins on a server and on workers, edited while it runs

    +
    3kinds of host in an app: server, worker, client
    1file per host kind in a plugin
    0restarts to apply a plugin edit
    1rule: plugins talk only through object types and registries
    +

    A tutorial for people building apps on Chord v2, written by the redesign session from design.md, slice-7c.md, +slice-10.md, the decision log and the code in packages/chord/src/next and src/next-node. API names are the +current ones; where slice 10's fix round or the app kit (chord/next/app, being designed) will change what app code +writes, a note says so.

    +
    +
    + + +
    +

    Before we begin

    +

    You want to build a notes app. You type a note in a browser tab, and every other tab sees it at once. Each project +gets its own notes in its own process, so a project that misbehaves cannot take the others down. And you, or an agent +you work with, add features while the app runs: a search, an export button, a word count, without restarting +anything.

    +

    Chord gives you three things for that. Hosts: the processes of your app (the server, a worker per project, the +browser tabs), connected to each other. Objects: the state and methods your app is made of, living on one host and +used from any other. Plugins: directories of code that declare objects for each kind of host, loaded and +reloaded while the app runs.

    +

    This book builds the app in that order. Each chapter adds one concept and shows complete code.

    +

    Seven things to know

    +
      +
    1. A host is one participant. The server, each worker and each browser tab is a host; hosts connect over links.
    2. +
    3. An object has a type, and a type is a contract. The contract names the object's state and methods; any host can +ask for an object by type and params, and Chord finds where it lives.
    4. +
    5. A facet declares, an object acts. A facet is a plugin's code for one host kind: it says which object types this +host serves. The objects hold the state and do the work.
    6. +
    7. A plugin is a directory. A package.json with the plugin's name, and one file per host kind: server.ts, +worker.ts, web.tsx.
    8. +
    9. The server builds, every host evaluates. The plugins component builds each facet into a small script; the +server, workers and tabs load the scripts for their kind.
    10. +
    11. Plugins meet through types and registries. One plugin uses another's objects through its contract, or +contributes to a registry another defines. Nothing else crosses between plugins.
    12. +
    13. An edit is a rollout. Change a file, build, roll out: the server, then every worker, swap the plugin's objects in +place; a broken edit keeps the old code running and reports why.
    14. +
    +

    Words used throughout

    +

    A host is one participant of the app, made with createHost. A host kind (Server, Session) describes +addressable hosts; an address (Session({ id: "p1" })) names one. An object type is a contract: local (used +on its own host only) or remote (used from any host). An object is one instance of a type, per params. A +facet is a plugin's declarations for one host kind. A plugin is a directory with a package.json. A +reference is what you get when you ask for an object (host.get, self.get, ctx.get). A registry is an +extension point: one plugin defines it, others contribute. A rollout applies a build to the hosts in order.

    +
    + +
    I

    One Host

    The smallest app: one process, one object type, one call.

    +

    Start without plugins. One host, one object that keeps notes:

    +
    import { createHost, defineFacet, defineState, local, method, obj, arr, str } from "@earendil-works/chord/next";
    +
    +// the contract: what a Notes object looks like from outside
    +const Notes = local("notes.store", {
    +	version: 1,
    +	params: {},                                        // one object per host: no params
    +	members: {
    +		list: obj({ items: arr(str()) }),             // state anyone can read and watch
    +		add: method(obj({ text: str() })),            // a method: arguments checked by Chord
    +	},
    +});
    +
    +// the document the object keeps
    +const NotesDoc = defineState("notes", { schema: obj({ items: arr(str()) }), version: 1, initial: () => ({ items: [] }) });
    +
    +// the facet: which types this host serves, and how to make each object
    +const facet = defineFacet((host) => {
    +	host.serve(Notes, (_params, self) => {
    +		const doc = self.state(NotesDoc);
    +		return {
    +			list: doc,
    +			add: async ({ text }, ctx) => self.tx(async (tx) => { (await tx.draft(doc)).items.push(text); }, ctx),
    +		};
    +	});
    +});
    +
    +const server = createHost({ onError: console.error });
    +await server.load([{ name: "@app/notes", facet }], ctx);
    +const notes = await server.get(Notes, {}, ctx);        // a reference: live, with its state
    +await notes.add({ text: "buy milk" }, ctx);
    +console.log(notes.list.value);                          // { items: ["buy milk"] }
    +

    Four things happened. The contract Notes says what the object offers. The facet says this host serves +Notes, with a factory that declares the object's parts: a document, and a method writing it in a transaction. +load publishes the facet. get returns a reference once the object is live; its members are the contract's.

    +

    The factory only declares: it is synchronous and opens nothing. Chord activates the object (loads its documents, opens +what it asked to open) and then serves calls. That is why the same factory can run again when the code changes.

    +
    + +
    II

    Several Hosts

    A server, a worker per project, browser tabs: how they find each other's objects.

    +

    Now give each project its own process. Declare the kinds of addressable host, and make notes a remote type that +lives on its project's worker:

    +
    export const Server = hostKind("server");
    +export const Project = hostKind("project", { id: str() });
    +
    +export const Notes = remote("notes", {
    +	version: 1,
    +	params: { project: str() },
    +	hostAddress: ({ project }) => Project({ id: project }),   // where an object of these params lives
    +	variants: { editor: { state: { list: NotesSchema }, methods: { add: method(AddArgs) } } },
    +});
    +

    hostAddress is the whole routing story. A tab asks for Notes { project: "p1" }. Its host computes the address, +Project(p1), and sends the request to its parent, the server. The server has a route for Project: it starts (or +finds) the worker for p1 and passes the request on. The worker serves Notes. The tab gets a reference whose state +follows the worker's document, one frame per change.

    +
    CLIENTSSERVERWORKERStab Aholds Notes { project: p1 }tab Bholds Notes { project: p1 }serverroute Project → workerworker p1serves Notes { p1 }worker p2serves Notes { p2 }holdholdrelay
    Figure 1. A tab holds a project's notes: the server routes by address to the project's worker and relays, without reading the data.
    +

    Each host is created the same way, with what it is:

    +
    // server.ts
    +const server = createHost<P>({ address: Server(), principal: { kind: "server" }, onError });
    +
    +// worker.ts (started by the server for one project)
    +const worker = createHost<P>({ address: Project({ id }), principal: { kind: "worker", project: id }, onError });
    +worker.connect(parentLink, { principal: { kind: "server" }, trusted: true }, ctx);
    +
    +// a browser tab: no address, it is a client
    +const tab = createHost<P>({ onError });
    +tab.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }, ctx);
    +

    P is your app's type of principal, who is calling: an owner, a guest, the server, a worker. Links stamp it; +a remote object is made per principal, so a guest can get a read-only variant of the same notes.

    +
    Coming with the app kit

    Starting workers, routing a host kind to them, the WebSocket entry and authentication are slice 10's components today +(createWorkers, chord.http), wired by the app. The opinionated app kit (chord/next/app, being designed) will do +this wiring for you: startApp(...) and startWorker().

    +
    +
    + +
    III

    Plugins

    Move the code into a directory that can be added, edited and removed while the app runs.

    +

    So far the facets were part of the app's own code. A plugin is the same code in its own directory:

    +
    plugins/notes/
    +  package.json        { "name": "@app/notes" }
    +  contract.ts         the Notes contract (other plugins import it)
    +  worker.ts           the facet for workers: serves Notes
    +  server.ts           the facet for the server: an export route
    +  web.tsx             the facet for browser tabs: a notes panel
    +

    The plugin's name is its package.json name. Each file named after a host kind is that kind's facet, its +default export a defineFacet(...). A plugin has only the files it needs: a plugin that only adds a route has a +server.ts and nothing else.

    +
    // plugins/notes/worker.ts
    +import { defineFacet } from "@app/core";
    +import { Notes } from "./contract.ts";
    +
    +export default defineFacet((host) => {
    +	host.serve(Notes, ({ project }, principal, self) => {
    +		const doc = self.state(NotesDoc);
    +		return { variant: "editor", members: { list: doc, add: async ({ text }, ctx) => /* … */ } };
    +	});
    +});
    +

    @app/core is your app's base code, imported by every plugin as a host package (the plugin's build leaves it +out; every host passes its own copy in). It holds what plugins share and never reloads: host kinds, the principal +type, a defineFacet bound to it.

    +
    + +
    IV

    Loading Plugins

    Discover, build, evaluate, load: what happens between a directory and a running object.

    +

    The plugins component does the work. You tell it where plugins are and what each host kind needs:

    +
    const plugins = createPlugins({
    +	roots: ["./plugins"],
    +	kinds: {
    +		server: { platform: "node", hostPackages: ["@earendil-works/chord/next", "@app/core"] },
    +		worker: { platform: "node", hostPackages: ["@earendil-works/chord/next", "@app/core"] },
    +		web: { platform: "browser", hostPackages: ["@earendil-works/chord/next", "@app/core", "react", "react/jsx-runtime"] },
    +	},
    +	onError,
    +}, ctx);
    +
    +const report = await plugins.load(server, {
    +	kind: "server",
    +	core: [{ name: "@app/core", facet: coreServerFacet }],
    +	modules: { "@earendil-works/chord/next": chord, "@app/core": core },
    +}, ctx);
    +console.log(report.failed);   // plugins left out, with why: [] when all loaded
    +
    What plugins.load does

    Discover

    It lists ./plugins one level deep and reads each package.json. A directory without one, or with two server +files, is reported, not loaded.

    +

    Build

    For each plugin with a server facet, esbuild builds one script from server.ts and everything it imports: the +plugin's own files, another plugin's contract files, its npm dependencies. Host packages stay out. A syntax error is a +build failure, never a crash.

    +

    Evaluate

    The script runs and returns the facet. Its import "@app/core" gets the server's own @app/core, so a ChordError +from a plugin is the same class as the server's.

    +

    Load

    host.load runs every facet's setup and publishes all of them at once. A plugin whose build, evaluation or setup +fails is left out and listed in report.failed; the others load. Its objects' users wait, and the next edit loads it.

    +
    +

    A worker never builds: the server builds the worker facets too, and each worker evaluates the scripts it is sent.

    +
    + +
    V

    Plugins Using Plugins

    Two ways one plugin uses another, and the one it may not.

    +

    A search plugin wants to search the notes. It imports the contract and asks for the object:

    +
    // plugins/search/worker.ts
    +import { Notes } from "../notes/contract.ts";   // copied into search's script: contracts only
    +
    +host.serve(Search, ({ project }, principal, self) => {
    +	const notes = self.get(Notes, { project });   // a dependency: Search goes live once Notes is
    +	const index = self.derive([notes.list], ([list]) => buildIndex(list.items));
    +	return { variant: "main", members: { find: async ({ q }) => lookup(index.value, q) } };
    +});
    +

    self.get in a factory declares a dependency: Chord makes Search live only once Notes is, and if the notes +plugin is missing, Search waits with a clear reason ("waiting for notes: not served here") instead of failing.

    +

    The other way is a registry: an extension point one plugin defines and others contribute to. The app's core +defines Panels, and every plugin with a UI contributes one:

    +
    // plugins/notes/web.tsx
    +export default defineFacet((host, self) => self.contribute(Panels, { title: "Notes", Component: NotesPanel }));
    +

    What a plugin may not do is reach into another's data. Search never reads notes' documents; it uses the Notes +object's state. That rule is what lets the notes plugin change its storage without breaking search.

    +
    + +
    VI

    Changing a Plugin

    Edit, build, roll out: how new code takes over while every tab stays connected.

    +

    You change the notes plugin: notes now have a timestamp. The watcher sees the saved file and marks the plugin dirty. +You (or an agent) build and roll out:

    +
    const built = await plugins.build(plugins.state.value.dirty, ctx);
    +const result = await plugins.rollout([plugins.step("server", server), workers.step("workers")], built, ctx);
    +

    The rollout goes host by host, server first, then every running worker. On each host Chord compares the new plugin +set with the loaded one, runs the new factories against a recording self to see what each object would declare, and +then gives every object of the changed plugin its fate:

    +

    failed { waiting }
    no fitting version, a fingerprint that differs at the same version, a placement the provider computes elsewhereno fitting version, a fingerprint that differs at the same version, a host address the provider computes elsewhere unbound version failed { waiting }
    refused and kept at the provider, re-run at its publication; failed { waiting }
    design §3, placementdesign §3, the destination "served here" decides "here"; placement optional on remote()the address decides; every remote type declares its placement (owner, 2026-10-29)the address decides; every remote type declares its hostAddress, renamed from placement (owner, 2026-10-29)
    the start barrier, pendingLoad unbound not_served: failed { waiting }
    no fitting version, a fingerprint that differs at the same version, a placement the provider computes elsewhere (code skew)no fitting version, a fingerprint that differs at the same version, a host address the provider computes elsewhere (code skew) unbound version: failed { waiting }
    2 self.get(Catalog) from W1's ForwardStore; the parent link is up, S has published, Catalog is served there.The request goes over the parent (placement Server()); S binds the hold; bound arrives; ForwardStore was loading, now goes on and live.The request goes over the parent (hostAddress: () => Server()); S binds the hold; bound arrives; ForwardStore was loading, now goes on and live. The same, after S's whole barrier (Hold.start awaits pendingLoad). §1, §2
    + + + + + + + + + + + + + + + + + + +
    The objectWhat happens
    holds no natives (open files, sockets), the new code neitherswapped in place: new methods, same documents; running calls finish on the old code
    holds nativesrestarted: calls wait, running ones finish, natives close and reopen with the new code
    its type was removedended; its users wait until something serves it again
    +

    The tab's reference follows the new object and gets one frame with the difference. Nothing reconnects. If the edit +is broken (a syntax error, a factory that throws), that plugin keeps its old code on that host and the report says why; +every other plugin of the rollout applies.

    +
    Versions

    A change to a contract (a new field in Notes's state) gets a new version. A tab built with the old contract keeps +working if the provider maps the old version (accepts); otherwise it waits with "version mismatch" until it reloads.

    +
    + + +
    ∴

    Base Code and Plugins

    What reloads, what restarts, and where the line is.

    +

    Everything in plugins/ reloads: edit, build, roll out. Everything in your base code (src/, @app/core: host +kinds, principals, the core facet, the app's startup) changes only with a restart, and the watcher tells you so. +Keep base code small: host kinds, principals, and what every plugin needs. Put features in plugins.

    +

    The app kit being designed next (chord/next/app) will make the base code nearly empty: startApp and startWorker +with Chord's conventions for hosts, users, authentication and storage, so an app is mostly plugins.

    +
    + +
    A

    Reference

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    CallWhat it does
    createHost<P>({ address?, principal?, store?, onError })a host: addressable with an address, else a client
    host.load([{ name, facet }], ctx) / host.reload(...)publish facets / apply a new plugin set
    host.get(Type, params, ctx) / host.hold(...)a live reference / a reference at once that follows its object
    local(name, { version, params, members }) / remote(name, { version, params, hostAddress, variants })contracts
    defineFacet((host, self) => …), host.serve(Type, factory), self.contribute(R, value)a facet
    factory self: state, derive, get, registry, open, on, onLive, job, txan object's parts
    createPlugins({ roots, kinds, onError }, ctx)the plugins component
    plugins.load(host, { kind, core, modules }, ctx), build(names), rollout(steps, built)load, build, roll out
    hostKind(name, params?), Kind(params)host kinds and addresses
    +
    + +

    Hosts and Plugins · first edition · 2026-10-31 · Chord v2 slices 1-10

    + + + From a34444d6bd768ecee494ad17261daaa75a6ed808 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 13:56:14 +0000 Subject: [PATCH 201/271] docs(chord): tutorial: overview figure, client hosts --- .../docs/next/guide-plugins-hosts/book.md | 65 ++++++++++++++++++- .../docs/next/guide-plugins-hosts/index.html | 56 +++++++++++++--- 2 files changed, 110 insertions(+), 11 deletions(-) diff --git a/packages/chord/docs/next/guide-plugins-hosts/book.md b/packages/chord/docs/next/guide-plugins-hosts/book.md index 2ffe01d53..83fe5051a 100644 --- a/packages/chord/docs/next/guide-plugins-hosts/book.md +++ b/packages/chord/docs/next/guide-plugins-hosts/book.md @@ -33,7 +33,8 @@ This book builds the app in that order. Each chapter adds one concept and shows ## Seven things to know :::findings -1. **A host is one participant.** The server, each worker and each browser tab is a host; hosts connect over links. +1. **A host is one participant.** The server, each worker, each browser tab and each terminal UI is a host; hosts + connect over links. Tabs and terminal UIs are clients: no address, they connect to the server and ask. 2. **An object has a type, and a type is a contract.** The contract names the object's state and methods; any host can ask for an object by type and params, and Chord finds where it lives. 3. **A facet declares, an object acts.** A facet is a plugin's code for one host kind: it says which object types this @@ -48,6 +49,25 @@ This book builds the app in that order. Each chapter adds one concept and shows place; a broken edit keeps the old code running and reports why. ::: +:::flow wide animate | How it all fits: plugins on disk are built once on the server; each host evaluates the facets of its kind; tabs and the TUI are clients that reach workers' objects through the server. +row "On disk" muted +box disk "plugins/notes/" "`server.ts`, `worker.ts`, `web.tsx`, `tui.ts`, `contract.ts`" muted +box core "src/ (base code)" "`@app/core`: host kinds, principals" muted +row "Server process" accent +box comp "plugins component" "discover, build every facet, watch, roll out" accent +box server "server host" "server facets; routes Project → workers" +row "Clients and workers" +box tab "client: tab or TUI" "web or tui facets, local UI objects; holds `Notes { p1 }`" blue +box w1 "worker p1" "worker facets; serves `Notes { p1 }`" +arrow disk -> comp "read, build" +arrow core -> comp "host packages" +arrow comp -> server "server scripts" +arrow comp -> tab "web or tui scripts" +arrow tab -> server "hold Notes { p1 }" +arrow server -> w1 "start (worker scripts), relay" +note "Each host loads only its own kind's facets. Plugins meet only through object types and registries." +::: + ## Words used throughout A **host** is one participant of the app, made with `createHost`. A **host kind** (`Server`, `Session`) describes @@ -162,6 +182,49 @@ Starting workers, routing a host kind to them, the WebSocket entry and authentic this wiring for you: `startApp(...)` and `startWorker()`. ::: +# Client Hosts {#clients lede="Browser tabs and terminal UIs are hosts too: what they can do, and what they cannot."} + +A browser tab and a terminal UI are hosts like the server, with one difference: they have **no address**. Nobody can +send a request *to* a tab; a tab only connects to the server and asks for things. That is a deliberate rule: a tab +runs on a device the server does not control, so nothing on the server ever waits for, or trusts, a tab's state. + +```ts +// a browser tab (slice 11's page host will do this wiring for you) +const tab = createHost

    ({ onError }); // no address, no principal: a client +tab.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }, ctx); +await tab.load([{ name: "@app/web", facet: webCore }, ...webFacets], ctx); + +const notes = tab.hold(Notes, { project: "p1" }, ctx); // a reference at once, live soon +(await notes.list.watch(ctx)).start(async (call) => { + if (call.kind === "value") render(call.value.items); // every change, as it commits on the worker +}); +await notes.add({ text: "from the tab" }, ctx); // a call: tab → server → worker +``` + +**Who the tab is** is not the tab's to say. The server authenticates the WebSocket when it connects (a cookie, the +owner on loopback, a guest's share link) and stamps that principal on everything the tab asks for. The worker's +`Notes` factory receives it and may give a guest a read-only variant. A tab cannot claim to be the owner. + +**What a tab serves** is only `local` types: state of this tab's UI, never shared. A draft that survives switching +panels, a selection, a tab-local cache: + +```ts +// plugins/notes/web.tsx: the web facet serves a local type, and contributes a panel +export default defineFacet((host, self) => { + host.serve(NoteDraft, (_params, self) => ({ text: self.state(DraftDoc) })); // lives in this tab only + self.contribute(Panels, { title: "Notes", Component: NotesPanel }); +}); +``` + +**When the connection drops** (a phone in a tunnel, a server restart), the tab keeps its references. Their values +stay readable, their `state` says `loading`, calls made meanwhile wait; on reconnect every reference is held again and +gets the current value in one frame. The UI shows a banner from `ref.state`, nothing more. + +**A terminal UI** is the same in Node: a client host connected to the server over a WebSocket, loading the plugins' +`tui` facets, rendering with whatever terminal library the app chose. **What a client never does**: serve a `remote` +type, be routed to, or keep data others rely on. Data that must outlive a tab belongs in an object on the server or a +worker. + # Plugins {#plugins lede="Move the code into a directory that can be added, edited and removed while the app runs."} So far the facets were part of the app's own code. A **plugin** is the same code in its own directory: diff --git a/packages/chord/docs/next/guide-plugins-hosts/index.html b/packages/chord/docs/next/guide-plugins-hosts/index.html index b026d7e8e..ec7ab3f9c 100644 --- a/packages/chord/docs/next/guide-plugins-hosts/index.html +++ b/packages/chord/docs/next/guide-plugins-hosts/index.html @@ -123,10 +123,11 @@ writes, a note says so.

  • Before we begin
  • I. One Host
  • II. Several Hosts
  • -
  • III. Plugins
  • -
  • IV. Loading Plugins
  • -
  • V. Plugins Using Plugins
  • -
  • VI. Changing a Plugin
  • +
  • III. Client Hosts
  • +
  • IV. Plugins
  • +
  • V. Loading Plugins
  • +
  • VI. Plugins Using Plugins
  • +
  • VII. Changing a Plugin
  • Base Code and Plugins
  • Appendix A: Reference
  • @@ -144,7 +145,8 @@ reloaded while the app runs.

    This book builds the app in that order. Each chapter adds one concept and shows complete code.

    Seven things to know

      -
    1. A host is one participant. The server, each worker and each browser tab is a host; hosts connect over links.
    2. +
    3. A host is one participant. The server, each worker, each browser tab and each terminal UI is a host; hosts +connect over links. Tabs and terminal UIs are clients: no address, they connect to the server and ask.
    4. An object has a type, and a type is a contract. The contract names the object's state and methods; any host can ask for an object by type and params, and Chord finds where it lives.
    5. A facet declares, an object acts. A facet is a plugin's code for one host kind: it says which object types this @@ -158,6 +160,7 @@ contributes to a registry another defines. Nothing else crosses between plugins.
    6. An edit is a rollout. Change a file, build, roll out: the server, then every worker, swap the plugin's objects in place; a broken edit keeps the old code running and reports why.
    +
    ON DISKSERVER PROCESSCLIENTS AND WORKERSplugins/notes/server.ts`, `worker.ts`,`web.tsx`, `tui.ts`, `contract.tssrc/ (base code)`@app/core`: host kinds, principalsplugins componentdiscover, build every facet, watch,roll outserver hostserver facets; routes Project →workersclient: tab or TUIweb or tui facets, local UI objects;holds `Notes { p1 }`worker p1worker facets; serves `Notes { p1 }`read, buildhost packagesserver scriptsweb or tui scriptshold Notes { p1 }start (worker scripts), relayEach host loads only its own kind's facets. Plugins meet only through object types and registries.
    Figure 1. How it all fits: plugins on disk are built once on the server; each host evaluates the facets of its kind; tabs and the TUI are clients that reach workers' objects through the server.

    Words used throughout

    A host is one participant of the app, made with createHost. A host kind (Server, Session) describes addressable hosts; an address (Session({ id: "p1" })) names one. An object type is a contract: local (used @@ -223,7 +226,7 @@ lives on its project's worker:

    Project(p1), and sends the request to its parent, the server. The server has a route for Project: it starts (or finds) the worker for p1 and passes the request on. The worker serves Notes. The tab gets a reference whose state follows the worker's document, one frame per change.

    -
    CLIENTSSERVERWORKERStab Aholds Notes { project: p1 }tab Bholds Notes { project: p1 }serverroute Project → workerworker p1serves Notes { p1 }worker p2serves Notes { p2 }holdholdrelay
    Figure 1. A tab holds a project's notes: the server routes by address to the project's worker and relays, without reading the data.
    +
    CLIENTSSERVERWORKERStab Aholds Notes { project: p1 }tab Bholds Notes { project: p1 }serverroute Project → workerworker p1serves Notes { p1 }worker p2serves Notes { p2 }holdholdrelay
    Figure 2. A tab holds a project's notes: the server routes by address to the project's worker and relays, without reading the data.

    Each host is created the same way, with what it is:

    // server.ts
     const server = createHost<P>({ address: Server(), principal: { kind: "server" }, onError });
    @@ -243,7 +246,40 @@ this wiring for you: startApp(...) and startWorker().<
     
     
     
    -
    III

    Plugins

    Move the code into a directory that can be added, edited and removed while the app runs.

    +
    III

    Client Hosts

    Browser tabs and terminal UIs are hosts too: what they can do, and what they cannot.

    +

    A browser tab and a terminal UI are hosts like the server, with one difference: they have no address. Nobody can +send a request to a tab; a tab only connects to the server and asks for things. That is a deliberate rule: a tab +runs on a device the server does not control, so nothing on the server ever waits for, or trusts, a tab's state.

    +
    // a browser tab (slice 11's page host will do this wiring for you)
    +const tab = createHost<P>({ onError });                               // no address, no principal: a client
    +tab.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }, ctx);
    +await tab.load([{ name: "@app/web", facet: webCore }, ...webFacets], ctx);
    +
    +const notes = tab.hold(Notes, { project: "p1" }, ctx);               // a reference at once, live soon
    +(await notes.list.watch(ctx)).start(async (call) => {
    +	if (call.kind === "value") render(call.value.items);              // every change, as it commits on the worker
    +});
    +await notes.add({ text: "from the tab" }, ctx);                        // a call: tab → server → worker
    +

    Who the tab is is not the tab's to say. The server authenticates the WebSocket when it connects (a cookie, the +owner on loopback, a guest's share link) and stamps that principal on everything the tab asks for. The worker's +Notes factory receives it and may give a guest a read-only variant. A tab cannot claim to be the owner.

    +

    What a tab serves is only local types: state of this tab's UI, never shared. A draft that survives switching +panels, a selection, a tab-local cache:

    +
    // plugins/notes/web.tsx: the web facet serves a local type, and contributes a panel
    +export default defineFacet((host, self) => {
    +	host.serve(NoteDraft, (_params, self) => ({ text: self.state(DraftDoc) }));   // lives in this tab only
    +	self.contribute(Panels, { title: "Notes", Component: NotesPanel });
    +});
    +

    When the connection drops (a phone in a tunnel, a server restart), the tab keeps its references. Their values +stay readable, their state says loading, calls made meanwhile wait; on reconnect every reference is held again and +gets the current value in one frame. The UI shows a banner from ref.state, nothing more.

    +

    A terminal UI is the same in Node: a client host connected to the server over a WebSocket, loading the plugins' +tui facets, rendering with whatever terminal library the app chose. What a client never does: serve a remote +type, be routed to, or keep data others rely on. Data that must outlive a tab belongs in an object on the server or a +worker.

    +
    + +
    IV

    Plugins

    Move the code into a directory that can be added, edited and removed while the app runs.

    So far the facets were part of the app's own code. A plugin is the same code in its own directory:

    plugins/notes/
       package.json        { "name": "@app/notes" }
    @@ -269,7 +305,7 @@ out; every host passes its own copy in). It holds what plugins share and never r
     type, a defineFacet bound to it.

    -
    IV

    Loading Plugins

    Discover, build, evaluate, load: what happens between a directory and a running object.

    +
    V

    Loading Plugins

    Discover, build, evaluate, load: what happens between a directory and a running object.

    The plugins component does the work. You tell it where plugins are and what each host kind needs:

    const plugins = createPlugins({
     	roots: ["./plugins"],
    @@ -300,7 +336,7 @@ fails is left out and listed in report.failed; the others load. Its
     

    A worker never builds: the server builds the worker facets too, and each worker evaluates the scripts it is sent.

    -
    V

    Plugins Using Plugins

    Two ways one plugin uses another, and the one it may not.

    +
    VI

    Plugins Using Plugins

    Two ways one plugin uses another, and the one it may not.

    A search plugin wants to search the notes. It imports the contract and asks for the object:

    // plugins/search/worker.ts
     import { Notes } from "../notes/contract.ts";   // copied into search's script: contracts only
    @@ -320,7 +356,7 @@ defines Panels, and every plugin with a UI contributes one:

    object's state. That rule is what lets the notes plugin change its storage without breaking search.

    -
    VI

    Changing a Plugin

    Edit, build, roll out: how new code takes over while every tab stays connected.

    +
    VII

    Changing a Plugin

    Edit, build, roll out: how new code takes over while every tab stays connected.

    You change the notes plugin: notes now have a timestamp. The watcher sees the saved file and marks the plugin dirty. You (or an agent) build and roll out:

    const built = await plugins.build(plugins.state.value.dirty, ctx);
    
    From 436d99e04b02d5ed784e09dba8601383e7fc8e76 Mon Sep 17 00:00:00 2001
    From: Mario Zechner 
    Date: Sat, 31 Oct 2026 14:01:40 +0000
    Subject: [PATCH 202/271] docs(chord): tutorial: one schema for contract and
     document
    
    ---
     .../docs/next/guide-plugins-hosts/book.md      | 18 ++++++++++++------
     .../docs/next/guide-plugins-hosts/index.html   | 18 ++++++++++++------
     2 files changed, 24 insertions(+), 12 deletions(-)
    
    diff --git a/packages/chord/docs/next/guide-plugins-hosts/book.md b/packages/chord/docs/next/guide-plugins-hosts/book.md
    index 83fe5051a..ab04e2119 100644
    --- a/packages/chord/docs/next/guide-plugins-hosts/book.md
    +++ b/packages/chord/docs/next/guide-plugins-hosts/book.md
    @@ -84,18 +84,20 @@ Start without plugins. One host, one object that keeps notes:
     ```ts
     import { createHost, defineFacet, defineState, local, method, obj, arr, str } from "@earendil-works/chord/next";
     
    -// the contract: what a Notes object looks like from outside
    +const NoteList = obj({ items: arr(str()) });          // one schema, used by the contract and the document
    +
    +// the contract: what a Notes object offers
     const Notes = local("notes.store", {
     	version: 1,
     	params: {},                                        // one object per host: no params
     	members: {
    -		list: obj({ items: arr(str()) }),             // state anyone can read and watch
    +		list: NoteList,                               // state anyone can read and watch
     		add: method(obj({ text: str() })),            // a method: arguments checked by Chord
     	},
     });
     
    -// the document the object keeps
    -const NotesDoc = defineState("notes", { schema: obj({ items: arr(str()) }), version: 1, initial: () => ({ items: [] }) });
    +// the document the object stores (its own version: storage can change without changing the contract)
    +const NotesDoc = defineState("notes", { schema: NoteList, version: 1, initial: () => ({ items: [] }) });
     
     // the facet: which types this host serves, and how to make each object
     const facet = defineFacet((host) => {
    @@ -115,7 +117,11 @@ await notes.add({ text: "buy milk" }, ctx);
     console.log(notes.list.value);                          // { items: ["buy milk"] }
     ```
     
    -Four things happened. The **contract** `Notes` says what the object offers. The **facet** says this host serves
    +Four things happened. The **contract** `Notes` says what the object offers; the **document** says what it stores.
    +Here they have the same shape, so one schema serves both; they are separate because they change separately (a
    +derive or a guest's redacted view offers something other than storage, and storage can migrate without users
    +noticing).
    +The **facet** says this host serves
     `Notes`, with a **factory** that declares the object's parts: a document, and a method writing it in a transaction.
     `load` publishes the facet. `get` returns a **reference** once the object is live; its members are the contract's.
     
    @@ -135,7 +141,7 @@ export const Notes = remote("notes", {
     	version: 1,
     	params: { project: str() },
     	hostAddress: ({ project }) => Project({ id: project }),   // where an object of these params lives
    -	variants: { editor: { state: { list: NotesSchema }, methods: { add: method(AddArgs) } } },
    +	variants: { editor: { state: { list: NoteList }, methods: { add: method(obj({ text: str() })) } } },
     });
     ```
     
    diff --git a/packages/chord/docs/next/guide-plugins-hosts/index.html b/packages/chord/docs/next/guide-plugins-hosts/index.html
    index ec7ab3f9c..677e54ab6 100644
    --- a/packages/chord/docs/next/guide-plugins-hosts/index.html
    +++ b/packages/chord/docs/next/guide-plugins-hosts/index.html
    @@ -174,18 +174,20 @@ extension point: one plugin defines it, others contribute. A rolloutStart without plugins. One host, one object that keeps notes:

    import { createHost, defineFacet, defineState, local, method, obj, arr, str } from "@earendil-works/chord/next";
     
    -// the contract: what a Notes object looks like from outside
    +const NoteList = obj({ items: arr(str()) });          // one schema, used by the contract and the document
    +
    +// the contract: what a Notes object offers
     const Notes = local("notes.store", {
     	version: 1,
     	params: {},                                        // one object per host: no params
     	members: {
    -		list: obj({ items: arr(str()) }),             // state anyone can read and watch
    +		list: NoteList,                               // state anyone can read and watch
     		add: method(obj({ text: str() })),            // a method: arguments checked by Chord
     	},
     });
     
    -// the document the object keeps
    -const NotesDoc = defineState("notes", { schema: obj({ items: arr(str()) }), version: 1, initial: () => ({ items: [] }) });
    +// the document the object stores (its own version: storage can change without changing the contract)
    +const NotesDoc = defineState("notes", { schema: NoteList, version: 1, initial: () => ({ items: [] }) });
     
     // the facet: which types this host serves, and how to make each object
     const facet = defineFacet((host) => {
    @@ -203,7 +205,11 @@ extension point: one plugin defines it, others contribute. A rolloutconst notes = await server.get(Notes, {}, ctx);        // a reference: live, with its state
     await notes.add({ text: "buy milk" }, ctx);
     console.log(notes.list.value);                          // { items: ["buy milk"] }
    -

    Four things happened. The contract Notes says what the object offers. The facet says this host serves +

    Four things happened. The contract Notes says what the object offers; the document says what it stores. +Here they have the same shape, so one schema serves both; they are separate because they change separately (a +derive or a guest's redacted view offers something other than storage, and storage can migrate without users +noticing). +The facet says this host serves Notes, with a factory that declares the object's parts: a document, and a method writing it in a transaction. load publishes the facet. get returns a reference once the object is live; its members are the contract's.

    The factory only declares: it is synchronous and opens nothing. Chord activates the object (loads its documents, opens @@ -220,7 +226,7 @@ lives on its project's worker:

    version: 1, params: { project: str() }, hostAddress: ({ project }) => Project({ id: project }), // where an object of these params lives - variants: { editor: { state: { list: NotesSchema }, methods: { add: method(AddArgs) } } }, + variants: { editor: { state: { list: NoteList }, methods: { add: method(obj({ text: str() })) } } }, });

    hostAddress is the whole routing story. A tab asks for Notes { project: "p1" }. Its host computes the address, Project(p1), and sends the request to its parent, the server. The server has a route for Project: it starts (or From dae7bd8fcb4cfe8193bce2fd0a2289aa611f0746 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 14:07:01 +0000 Subject: [PATCH 203/271] docs(chord): tutorial: an object's parts before they are used --- .../docs/next/guide-plugins-hosts/book.md | 67 +++++++++++++++-- .../docs/next/guide-plugins-hosts/index.html | 73 ++++++++++++++----- 2 files changed, 118 insertions(+), 22 deletions(-) diff --git a/packages/chord/docs/next/guide-plugins-hosts/book.md b/packages/chord/docs/next/guide-plugins-hosts/book.md index ab04e2119..d15c8a892 100644 --- a/packages/chord/docs/next/guide-plugins-hosts/book.md +++ b/packages/chord/docs/next/guide-plugins-hosts/book.md @@ -74,7 +74,9 @@ A **host** is one participant of the app, made with `createHost`. A **host kind* addressable hosts; an **address** (`Session({ id: "p1" })`) names one. An **object type** is a contract: `local` (used on its own host only) or `remote` (used from any host). An **object** is one instance of a type, per params. A **facet** is a plugin's declarations for one host kind. A **plugin** is a directory with a `package.json`. A -**reference** is what you get when you ask for an object (`host.get`, `self.get`, `ctx.get`). A **registry** is an +**reference** is what you get when you ask for an object (`host.get`, `self.get`, `ctx.get`). A **document** is data an object +stores, changed in **transactions**; a **derive** is a value computed from others; a **native** is an outside +resource an object opens and closes. A **registry** is an extension point: one plugin defines it, others contribute. A **rollout** applies a build to the hosts in order. # One Host {#host lede="The smallest app: one process, one object type, one call."} @@ -118,9 +120,9 @@ console.log(notes.list.value); // { items: ["buy milk"] ``` Four things happened. The **contract** `Notes` says what the object offers; the **document** says what it stores. -Here they have the same shape, so one schema serves both; they are separate because they change separately (a -derive or a guest's redacted view offers something other than storage, and storage can migrate without users -noticing). +Here they have the same shape, so one schema serves both; they are separate because they change separately (an +object may offer a computed value or a shortened view instead of what it stores, and storage can change its shape +without users noticing). The **facet** says this host serves `Notes`, with a **factory** that declares the object's parts: a document, and a method writing it in a transaction. `load` publishes the facet. `get` returns a **reference** once the object is live; its members are the contract's. @@ -128,6 +130,57 @@ The **facet** says this host serves The factory only declares: it is synchronous and opens nothing. Chord activates the object (loads its documents, opens what it asked to open) and then serves calls. That is why the same factory can run again when the code changes. +# An Object's Parts {#parts lede="Documents, transactions, computed values, watching, and outside resources: the vocabulary of every factory."} + +A factory builds an object from a handful of parts, each declared with one call on `self`. The notes object grows to +show all of them. + +**Documents** hold the object's data: `self.state(NotesDoc)` gives a handle whose `.value` is the current data. Data +changes only in a **transaction**: `self.tx(fn, ctx)` runs `fn` with `tx`, and `await tx.draft(handle)` gives a +mutable copy of the document's data; whatever you change on it is committed at once when `fn` returns, or not at all +if it throws. + +```ts +add: async ({ text }, ctx) => self.tx(async (tx) => { + const draft = await tx.draft(doc); // the document's data, writable inside this transaction only + draft.items.push(text); // recorded as a change, committed when the function returns +}, ctx), +``` + +A **derive** is a value computed from other values, recomputed whenever they change. It is how an object offers +something it does not store: + +```ts +const count = self.derive([doc], ([notes]) => notes.items.length); // 0, 1, 2, … as notes are added +return { list: doc, count, add }; // `count` can be a member like `list` +``` + +**Watching** is how anyone follows a value as it changes: a document handle, a derive, a reference's member. `watch` +gives the current value, then one call per change: + +```ts +const watch = await notes.count.watch(ctx); +watch.start(async (call) => { + if (call.kind === "value") console.log("notes:", call.value); // "failed" calls say a value cannot be computed now +}); +``` + +A **routine** is the object watching on its own behalf: `self.on(source, fn)` runs `fn` with each new value, for as +long as the object lives (a reconciler: "whenever the notes change, rebuild the export file"). + +A **native** is something outside Chord the object opens and must close: a file watcher, a database connection, a +child process. `self.open(open, close)` opens it while the object activates and closes it when the object ends: + +```ts +const inbox = self.open(async (ctx) => watchDirectory(inboxPath, ctx.abortSignal), (watcher) => watcher.close()); +self.onLive(async (ctx) => { // a routine started once the object is live + for await (const file of inbox.value.files(ctx.abortSignal)) await importNote(file, ctx); +}); +``` + +Natives are why some objects restart when their plugin changes: an open connection cannot be moved to new code, so +Chord closes it and opens it again. Objects without natives are swapped in place. + # Several Hosts {#hosts lede="A server, a worker per project, browser tabs: how they find each other's objects."} Now give each project its own process. Declare the kinds of addressable host, and make notes a `remote` type that @@ -145,7 +198,11 @@ export const Notes = remote("notes", { }); ``` -`hostAddress` is the whole routing story. A tab asks for `Notes { project: "p1" }`. Its host computes the address, +Two things are new. A `remote` type can be used from any host, and it has **variants**: different sets of members +for different callers, chosen by the factory (an `editor` variant with `add`, a `reader` variant for guests without +it). And `hostAddress` says where its objects live; it is the whole routing story. + +A tab asks for `Notes { project: "p1" }`. Its host computes the address, `Project(p1)`, and sends the request to its parent, the server. The server has a **route** for `Project`: it starts (or finds) the worker for `p1` and passes the request on. The worker serves `Notes`. The tab gets a reference whose state follows the worker's document, one frame per change. diff --git a/packages/chord/docs/next/guide-plugins-hosts/index.html b/packages/chord/docs/next/guide-plugins-hosts/index.html index 677e54ab6..55719365f 100644 --- a/packages/chord/docs/next/guide-plugins-hosts/index.html +++ b/packages/chord/docs/next/guide-plugins-hosts/index.html @@ -122,12 +122,13 @@ writes, a note says so.

    @@ -166,7 +167,9 @@ place; a broken edit keeps the old code running and reports why. addressable hosts; an address (Session({ id: "p1" })) names one. An object type is a contract: local (used on its own host only) or remote (used from any host). An object is one instance of a type, per params. A facet is a plugin's declarations for one host kind. A plugin is a directory with a package.json. A -reference is what you get when you ask for an object (host.get, self.get, ctx.get). A registry is an +reference is what you get when you ask for an object (host.get, self.get, ctx.get). A document is data an object +stores, changed in transactions; a derive is a value computed from others; a native is an outside +resource an object opens and closes. A registry is an extension point: one plugin defines it, others contribute. A rollout applies a build to the hosts in order.

    @@ -206,9 +209,9 @@ extension point: one plugin defines it, others contribute. A rolloutawait notes.add({ text: "buy milk" }, ctx); console.log(notes.list.value); // { items: ["buy milk"] }

    Four things happened. The contract Notes says what the object offers; the document says what it stores. -Here they have the same shape, so one schema serves both; they are separate because they change separately (a -derive or a guest's redacted view offers something other than storage, and storage can migrate without users -noticing). +Here they have the same shape, so one schema serves both; they are separate because they change separately (an +object may offer a computed value or a shortened view instead of what it stores, and storage can change its shape +without users noticing). The facet says this host serves Notes, with a factory that declares the object's parts: a document, and a method writing it in a transaction. load publishes the facet. get returns a reference once the object is live; its members are the contract's.

    @@ -216,7 +219,40 @@ The facet says this host serves what it asked to open) and then serves calls. That is why the same factory can run again when the code changes.

    -
    II

    Several Hosts

    A server, a worker per project, browser tabs: how they find each other's objects.

    +
    II

    An Object's Parts

    Documents, transactions, computed values, watching, and outside resources: the vocabulary of every factory.

    +

    A factory builds an object from a handful of parts, each declared with one call on self. The notes object grows to +show all of them.

    +

    Documents hold the object's data: self.state(NotesDoc) gives a handle whose .value is the current data. Data +changes only in a transaction: self.tx(fn, ctx) runs fn with tx, and await tx.draft(handle) gives a +mutable copy of the document's data; whatever you change on it is committed at once when fn returns, or not at all +if it throws.

    +
    add: async ({ text }, ctx) => self.tx(async (tx) => {
    +	const draft = await tx.draft(doc);          // the document's data, writable inside this transaction only
    +	draft.items.push(text);                      // recorded as a change, committed when the function returns
    +}, ctx),
    +

    A derive is a value computed from other values, recomputed whenever they change. It is how an object offers +something it does not store:

    +
    const count = self.derive([doc], ([notes]) => notes.items.length);   // 0, 1, 2, … as notes are added
    +return { list: doc, count, add };                                      // `count` can be a member like `list`
    +

    Watching is how anyone follows a value as it changes: a document handle, a derive, a reference's member. watch +gives the current value, then one call per change:

    +
    const watch = await notes.count.watch(ctx);
    +watch.start(async (call) => {
    +	if (call.kind === "value") console.log("notes:", call.value);   // "failed" calls say a value cannot be computed now
    +});
    +

    A routine is the object watching on its own behalf: self.on(source, fn) runs fn with each new value, for as +long as the object lives (a reconciler: "whenever the notes change, rebuild the export file").

    +

    A native is something outside Chord the object opens and must close: a file watcher, a database connection, a +child process. self.open(open, close) opens it while the object activates and closes it when the object ends:

    +
    const inbox = self.open(async (ctx) => watchDirectory(inboxPath, ctx.abortSignal), (watcher) => watcher.close());
    +self.onLive(async (ctx) => {                                  // a routine started once the object is live
    +	for await (const file of inbox.value.files(ctx.abortSignal)) await importNote(file, ctx);
    +});
    +

    Natives are why some objects restart when their plugin changes: an open connection cannot be moved to new code, so +Chord closes it and opens it again. Objects without natives are swapped in place.

    +
    + +
    III

    Several Hosts

    A server, a worker per project, browser tabs: how they find each other's objects.

    Now give each project its own process. Declare the kinds of addressable host, and make notes a remote type that lives on its project's worker:

    export const Server = hostKind("server");
    @@ -228,7 +264,10 @@ lives on its project's worker:

    hostAddress: ({ project }) => Project({ id: project }), // where an object of these params lives variants: { editor: { state: { list: NoteList }, methods: { add: method(obj({ text: str() })) } } }, });
    -

    hostAddress is the whole routing story. A tab asks for Notes { project: "p1" }. Its host computes the address, +

    Two things are new. A remote type can be used from any host, and it has variants: different sets of members +for different callers, chosen by the factory (an editor variant with add, a reader variant for guests without +it). And hostAddress says where its objects live; it is the whole routing story.

    +

    A tab asks for Notes { project: "p1" }. Its host computes the address, Project(p1), and sends the request to its parent, the server. The server has a route for Project: it starts (or finds) the worker for p1 and passes the request on. The worker serves Notes. The tab gets a reference whose state follows the worker's document, one frame per change.

    @@ -252,7 +291,7 @@ this wiring for you: startApp(...) and startWorker().<
    -
    III

    Client Hosts

    Browser tabs and terminal UIs are hosts too: what they can do, and what they cannot.

    +
    IV

    Client Hosts

    Browser tabs and terminal UIs are hosts too: what they can do, and what they cannot.

    A browser tab and a terminal UI are hosts like the server, with one difference: they have no address. Nobody can send a request to a tab; a tab only connects to the server and asks for things. That is a deliberate rule: a tab runs on a device the server does not control, so nothing on the server ever waits for, or trusts, a tab's state.

    @@ -285,7 +324,7 @@ type, be routed to, or keep data others rely on. Data that must outlive a tab be worker.

    -
    IV

    Plugins

    Move the code into a directory that can be added, edited and removed while the app runs.

    +
    V

    Plugins

    Move the code into a directory that can be added, edited and removed while the app runs.

    So far the facets were part of the app's own code. A plugin is the same code in its own directory:

    plugins/notes/
       package.json        { "name": "@app/notes" }
    @@ -311,7 +350,7 @@ out; every host passes its own copy in). It holds what plugins share and never r
     type, a defineFacet bound to it.

    -
    V

    Loading Plugins

    Discover, build, evaluate, load: what happens between a directory and a running object.

    +
    VI

    Loading Plugins

    Discover, build, evaluate, load: what happens between a directory and a running object.

    The plugins component does the work. You tell it where plugins are and what each host kind needs:

    const plugins = createPlugins({
     	roots: ["./plugins"],
    @@ -342,7 +381,7 @@ fails is left out and listed in report.failed; the others load. Its
     

    A worker never builds: the server builds the worker facets too, and each worker evaluates the scripts it is sent.

    -
    VI

    Plugins Using Plugins

    Two ways one plugin uses another, and the one it may not.

    +
    VII

    Plugins Using Plugins

    Two ways one plugin uses another, and the one it may not.

    A search plugin wants to search the notes. It imports the contract and asks for the object:

    // plugins/search/worker.ts
     import { Notes } from "../notes/contract.ts";   // copied into search's script: contracts only
    @@ -362,7 +401,7 @@ defines Panels, and every plugin with a UI contributes one:

    object's state. That rule is what lets the notes plugin change its storage without breaking search.

    -
    VII

    Changing a Plugin

    Edit, build, roll out: how new code takes over while every tab stays connected.

    +
    VIII

    Changing a Plugin

    Edit, build, roll out: how new code takes over while every tab stays connected.

    You change the notes plugin: notes now have a timestamp. The watcher sees the saved file and marks the plugin dirty. You (or an agent) build and roll out:

    const built = await plugins.build(plugins.state.value.dirty, ctx);
    
    From fe6211607be8fe1c648a352ef5d143595fa7bc01 Mon Sep 17 00:00:00 2001
    From: Mario Zechner 
    Date: Sat, 31 Oct 2026 14:09:23 +0000
    Subject: [PATCH 204/271] docs(chord): tutorial: persisted documents and tables
    
    ---
     .../docs/next/guide-plugins-hosts/book.md     | 30 +++++++++++++++++--
     .../docs/next/guide-plugins-hosts/index.html  | 25 ++++++++++++++--
     2 files changed, 51 insertions(+), 4 deletions(-)
    
    diff --git a/packages/chord/docs/next/guide-plugins-hosts/book.md b/packages/chord/docs/next/guide-plugins-hosts/book.md
    index d15c8a892..c49bc32a9 100644
    --- a/packages/chord/docs/next/guide-plugins-hosts/book.md
    +++ b/packages/chord/docs/next/guide-plugins-hosts/book.md
    @@ -75,7 +75,7 @@ addressable hosts; an **address** (`Session({ id: "p1" })`) names one. An **obje
     on its own host only) or `remote` (used from any host). An **object** is one instance of a type, per params. A
     **facet** is a plugin's declarations for one host kind. A **plugin** is a directory with a `package.json`. A
     **reference** is what you get when you ask for an object (`host.get`, `self.get`, `ctx.get`). A **document** is data an object
    -stores, changed in **transactions**; a **derive** is a value computed from others; a **native** is an outside
    +stores, changed in **transactions**; a **table** holds many records with queries, in the host's **store**; a **derive** is a value computed from others; a **native** is an outside
     resource an object opens and closes. A **registry** is an
     extension point: one plugin defines it, others contribute. A **rollout** applies a build to the hosts in order.
     
    @@ -130,7 +130,7 @@ The **facet** says this host serves
     The factory only declares: it is synchronous and opens nothing. Chord activates the object (loads its documents, opens
     what it asked to open) and then serves calls. That is why the same factory can run again when the code changes.
     
    -# An Object's Parts {#parts lede="Documents, transactions, computed values, watching, and outside resources: the vocabulary of every factory."}
    +# An Object's Parts {#parts lede="Documents, tables, transactions, computed values, watching, and outside resources: the vocabulary of every factory."}
     
     A factory builds an object from a handful of parts, each declared with one call on `self`. The notes object grows to
     show all of them.
    @@ -147,6 +147,32 @@ add: async ({ text }, ctx) => self.tx(async (tx) => {
     }, ctx),
     ```
     
    +Documents live in memory unless they are **persisted**: `defineState(..., { persist: true, checkpointWhen })` keeps
    +one in the host's **store** (a SQLite file the app opens and gives to `createHost({ store })`), loaded when the
    +object activates. For many records with queries, an object declares a **table**:
    +
    +```ts
    +const NoteRows = defineTable("notes", {
    +	version: 1,
    +	columns: { id: str(), text: str(), at: int() },
    +	key: ["id"],
    +	indexes: { by_time: ["at"] },
    +});
    +
    +host.serve(NoteStore, (_params, self) => {               // a local type with empty params: it may persist
    +	const rows = self.table(NoteRows);                    // created (or migrated) when the object activates
    +	const recent = query(NoteRows).orderBy((n) => n.at, "desc");
    +	return {
    +		add: async ({ text }, ctx) => self.tx(async (tx) => tx.put(rows, { id: uuid(), text, at: Date.now() }), ctx),
    +		page: async ({ after }, ctx) => rows.page(recent, 50, ctx, { after }),   // 50 at a time, with a cursor
    +	};
    +});
    +```
    +
    +Persisted data lives only in `local` objects with empty params (one per host and type), so its address never depends
    +on who asked or with which params; an object with params (a view per project) reads from such a store. Writes go
    +through the same transactions as documents: a note row and a document can change in one commit.
    +
     A **derive** is a value computed from other values, recomputed whenever they change. It is how an object offers
     something it does not store:
     
    diff --git a/packages/chord/docs/next/guide-plugins-hosts/index.html b/packages/chord/docs/next/guide-plugins-hosts/index.html
    index 55719365f..ece1ed16c 100644
    --- a/packages/chord/docs/next/guide-plugins-hosts/index.html
    +++ b/packages/chord/docs/next/guide-plugins-hosts/index.html
    @@ -168,7 +168,7 @@ addressable hosts; an address (Session({ id: "p1&quo
     on its own host only) or remote (used from any host). An object is one instance of a type, per params. A
     facet is a plugin's declarations for one host kind. A plugin is a directory with a package.json. A
     reference is what you get when you ask for an object (host.get, self.get, ctx.get). A document is data an object
    -stores, changed in transactions; a derive is a value computed from others; a native is an outside
    +stores, changed in transactions; a table holds many records with queries, in the host's store; a derive is a value computed from others; a native is an outside
     resource an object opens and closes. A registry is an
     extension point: one plugin defines it, others contribute. A rollout applies a build to the hosts in order.

    @@ -219,7 +219,7 @@ The facet says this host serves what it asked to open) and then serves calls. That is why the same factory can run again when the code changes.

    -
    II

    An Object's Parts

    Documents, transactions, computed values, watching, and outside resources: the vocabulary of every factory.

    +
    II

    An Object's Parts

    Documents, tables, transactions, computed values, watching, and outside resources: the vocabulary of every factory.

    A factory builds an object from a handful of parts, each declared with one call on self. The notes object grows to show all of them.

    Documents hold the object's data: self.state(NotesDoc) gives a handle whose .value is the current data. Data @@ -230,6 +230,27 @@ if it throws.

    const draft = await tx.draft(doc); // the document's data, writable inside this transaction only draft.items.push(text); // recorded as a change, committed when the function returns }, ctx),
    +

    Documents live in memory unless they are persisted: defineState(..., { persist: true, checkpointWhen }) keeps +one in the host's store (a SQLite file the app opens and gives to createHost({ store })), loaded when the +object activates. For many records with queries, an object declares a table:

    +
    const NoteRows = defineTable("notes", {
    +	version: 1,
    +	columns: { id: str(), text: str(), at: int() },
    +	key: ["id"],
    +	indexes: { by_time: ["at"] },
    +});
    +
    +host.serve(NoteStore, (_params, self) => {               // a local type with empty params: it may persist
    +	const rows = self.table(NoteRows);                    // created (or migrated) when the object activates
    +	const recent = query(NoteRows).orderBy((n) => n.at, "desc");
    +	return {
    +		add: async ({ text }, ctx) => self.tx(async (tx) => tx.put(rows, { id: uuid(), text, at: Date.now() }), ctx),
    +		page: async ({ after }, ctx) => rows.page(recent, 50, ctx, { after }),   // 50 at a time, with a cursor
    +	};
    +});
    +

    Persisted data lives only in local objects with empty params (one per host and type), so its address never depends +on who asked or with which params; an object with params (a view per project) reads from such a store. Writes go +through the same transactions as documents: a note row and a document can change in one commit.

    A derive is a value computed from other values, recomputed whenever they change. It is how an object offers something it does not store:

    const count = self.derive([doc], ([notes]) => notes.items.length);   // 0, 1, 2, … as notes are added
    
    From 86338abbd8ee1ebbb2fd10bfa277a35b940682c0 Mon Sep 17 00:00:00 2001
    From: Mario Zechner 
    Date: Sat, 31 Oct 2026 14:26:12 +0000
    Subject: [PATCH 205/271] docs(chord): tutorial: loading explained (build vs
     evaluate, host packages, why plugins.load)
    
    ---
     .../docs/next/guide-plugins-hosts/book.md     | 145 +++++++++++++++---
     .../docs/next/guide-plugins-hosts/index.html  |  98 +++++++++---
     2 files changed, 199 insertions(+), 44 deletions(-)
    
    diff --git a/packages/chord/docs/next/guide-plugins-hosts/book.md b/packages/chord/docs/next/guide-plugins-hosts/book.md
    index c49bc32a9..563ba5315 100644
    --- a/packages/chord/docs/next/guide-plugins-hosts/book.md
    +++ b/packages/chord/docs/next/guide-plugins-hosts/book.md
    @@ -348,13 +348,48 @@ export default defineFacet((host) => {
     out; every host passes its own copy in). It holds what plugins share and never reloads: host kinds, the principal
     type, a `defineFacet` bound to it.
     
    -# Loading Plugins {#loading lede="Discover, build, evaluate, load: what happens between a directory and a running object."}
    +# Loading Plugins {#loading lede="Two steps in two places: building a plugin into a script happens once, on the server; evaluating it happens in every process that runs it."}
     
    -The **plugins component** does the work. You tell it where plugins are and what each host kind needs:
    +A plugin is source code on the server's disk: TypeScript files that import each other, import other plugins'
    +contracts, import npm packages, and import things the app itself provides. None of the processes that run the plugin
    +can use those files as they are. A browser tab has no file system and no TypeScript; a worker could read the files,
    +but then every worker would compile every plugin again, and two workers started a minute apart could see two different
    +versions. So loading a plugin is two separate steps:
    +
    +1. **Build**: turn each plugin facet (`notes/worker.ts`, `notes/web.tsx`) into one self-contained **script**. This
    +   happens once per edit, on the server, for every host kind.
    +2. **Evaluate**: run a script inside one process, which gives back the facet, ready for `host.load`. This happens in
    +   every process that hosts that kind: the server for its own facets, each worker for the worker facets, each tab for
    +   the web facets.
    +
    +The two steps need different things, and that is why the API has two calls.
    +
    +## Host packages: what a script leaves out
    +
    +Look at what `notes/worker.ts` imports:
    +
    +```ts
    +import { defineFacet, Session } from "@app/core";             // the app's base code
    +import { obj, str, remote } from "@earendil-works/chord/next";  // Chord itself
    +import { Notes } from "./contract.ts";                            // the plugin's own file
    +import slugify from "slugify";                                    // an npm package in the plugin's dependencies
    +```
    +
    +The last two go into the script: the plugin's own files and its npm dependencies are copied in. The first two must
    +not. If the script carried its own copy of Chord, the worker would run two Chords: the plugin's `ChordError` would not
    +be the worker's `ChordError`, and the plugin's `defineFacet` would make a facet the worker's host does not recognize.
    +The same holds for `@app/core`: the plugin must see the very `Session` host kind and the very `Panels` registry the
    +process already has, not a copy.
    +
    +Packages the process already has, and that every plugin must share, are its **host packages**. A script leaves them
    +out and asks for them by name when it runs. Building therefore needs only their **names**: "do not copy
    +`@earendil-works/chord/next` or `@app/core` into this script".
    +
    +## Building: `createPlugins`, on the server
     
     ```ts
     const plugins = createPlugins({
    -	roots: ["./plugins"],
    +	roots: ["./plugins"],                                 // where the plugin directories are
     	kinds: {
     		server: { platform: "node", hostPackages: ["@earendil-works/chord/next", "@app/core"] },
     		worker: { platform: "node", hostPackages: ["@earendil-works/chord/next", "@app/core"] },
    @@ -362,32 +397,92 @@ const plugins = createPlugins({
     	},
     	onError,
     }, ctx);
    -
    -const report = await plugins.load(server, {
    -	kind: "server",
    -	core: [{ name: "@app/core", facet: coreServerFacet }],
    -	modules: { "@earendil-works/chord/next": chord, "@app/core": core },
    -}, ctx);
    -console.log(report.failed);   // plugins left out, with why: [] when all loaded
     ```
     
    -:::steps | What `plugins.load` does
    -+++ Discover
    -It lists `./plugins` one level deep and reads each `package.json`. A directory without one, or with two `server`
    -files, is reported, not loaded.
    -+++ Build
    -For each plugin with a `server` facet, esbuild builds one script from `server.ts` and everything it imports: the
    -plugin's own files, another plugin's contract files, its npm dependencies. Host packages stay out. A syntax error is a
    -build failure, never a crash.
    -+++ Evaluate
    -The script runs and returns the facet. Its `import "@app/core"` gets the server's own `@app/core`, so a `ChordError`
    -from a plugin is the same class as the server's.
    -+++ Load
    -`host.load` runs every facet's setup and publishes all of them at once. A plugin whose build, evaluation or setup
    -fails is left out and listed in `report.failed`; the others load. Its objects' users wait, and the next edit loads it.
    +`createPlugins` is the builder, and it builds for every kind, because only the server has the sources. For each kind
    +it needs two facts. The **platform** says where the scripts run: a `node` script may import `node:fs`, a `browser`
    +script may import a stylesheet but not `node:fs`. The **host package names** say what to leave out; they differ per
    +kind, because a tab also shares React with its plugins and a worker does not. Nothing here runs a plugin.
    +
    +## Evaluating: `plugins.load`, in the process that hosts
    +
    +Evaluating is where the names become things. When the script runs and asks for `@app/core`, someone has to hand it
    +the actual module, the one this process imported when it started. Only the process itself has it:
    +
    +```ts
    +// server.ts
    +import * as chord from "@earendil-works/chord/next";
    +import * as core from "./core.ts";                         // the app's base code, the same module plugins import as "@app/core"
    +
    +const report = await plugins.load(server, {
    +	kind: "server",                                          // evaluate the server kind's scripts
    +	modules: { "@earendil-works/chord/next": chord, "@app/core": core },   // this process's own instances
    +	core: [{ name: "@app/core", facet: core.serverFacet }],  // base-code facets, loaded beside the plugins
    +}, ctx);
    +```
    +
    +`modules` is this process's answer to every host package name. The server passes its own; a worker passes its own in
    +`workerMain`; a page passes its own in `startPage`. Each process has different instances, which is exactly the point:
    +inside each process, plugins share that process's Chord and base code.
    +
    +`core` is the app's base code in facet form. It is a facet like a plugin's, but it is not a plugin: it is made in the
    +process's own code, often from values only this process has (`core.workerFacet(start)` closes over a worker's start
    +data and store), and it never reloads, because it defines what all plugins share. Passing it here loads it together
    +with the plugins, so they are published in one step.
    +
    +## Why `plugins.load(host)`, not `host.load`
    +
    +A host has its own `load`, and `plugins.load` calls it. The host's `load` takes facets that already exist: functions
    +in memory. It knows nothing about directories, builds, scripts or modules, on purpose: the same host runs in a test
    +with facets imported directly, or in a Cloudflare Durable Object where plugins are bundled at deploy time and nothing
    +is built or evaluated at runtime.
    +
    +The plugins component is the part that knows about directories. It finds the plugins, builds them, evaluates them
    +with the process's modules, and then calls `host.load` with the result. It also remembers, per host, which facet each
    +plugin runs, so that a later reload can hand the host its complete new plugin set, with unchanged plugins as the very
    +same objects. That memory is why the host is passed to it: `plugins.load(server, …)` on the server,
    +`plugins.load(worker, …)` in a worker (where `workerMain` gives it a `plugins` that only evaluates), the same call
    +in every process.
    +
    +:::flow wide | Building on the server needs names; evaluating in each process needs that process's modules.
    +row "Server: build (once per edit)" accent
    +box src "plugin sources" "`notes/worker.ts`, imports `@app/core`" muted
    +box build "createPlugins" "copies own files and npm deps; leaves host packages out" accent
    +box script "worker script" "`require(\"@app/core\")` left as a name"
    +row "Each worker: evaluate" blue
    +box mods "the worker's modules" "its own `@app/core`, its own Chord" muted
    +box load "plugins.load(worker, …)" "runs the script, answers each name" blue
    +box facet "the facet" "uses the worker's `Session`, the worker's `defineFacet`"
    +arrow src -> build
    +arrow build -> script
    +arrow script -> load "shipped"
    +arrow mods -> load
    +arrow load -> facet
     :::
     
    -A worker never builds: the server builds the worker facets too, and each worker evaluates the scripts it is sent.
    +## What happens, step by step
    +
    +:::steps | `plugins.load(server, { kind: "server", … })`
    ++++ Discover
    +The component lists `./plugins` one level deep and reads each `package.json`. Its `name` is the plugin's name; the
    +files named after a kind are its facets. A directory without a `package.json`, or with two `server` files, is reported,
    +not loaded.
    ++++ Build
    +For each plugin with a `server` facet it builds one script, if no current one exists. A syntax error or an import the
    +rules do not allow (a path into base code, a package not in the plugin's dependencies) is a failed build with the file
    +and line, never a crash.
    ++++ Evaluate
    +The script runs. Each host package name it asks for is answered from `modules`; anything else it may not have. The
    +script's default export must be a facet.
    ++++ Load
    +`host.load` runs the setups of `core` and of every plugin facet and publishes them together. A plugin whose build,
    +evaluation or setup failed is left out and listed in `report.failed` with the reason; the others load. Objects that
    +need it wait until a later edit loads it.
    +:::
    +
    +A worker never builds. The server builds the worker kind too and sends the scripts in the worker's start message; the
    +worker only evaluates, with its own modules. A page gets its scripts from the server over HTTP and evaluates them with
    +the modules of its base bundle.
     
     # Plugins Using Plugins {#together lede="Two ways one plugin uses another, and the one it may not."}
     
    diff --git a/packages/chord/docs/next/guide-plugins-hosts/index.html b/packages/chord/docs/next/guide-plugins-hosts/index.html
    index ece1ed16c..a62ac6175 100644
    --- a/packages/chord/docs/next/guide-plugins-hosts/index.html
    +++ b/packages/chord/docs/next/guide-plugins-hosts/index.html
    @@ -371,35 +371,95 @@ out; every host passes its own copy in). It holds what plugins share and never r
     type, a defineFacet bound to it.

    -
    VI

    Loading Plugins

    Discover, build, evaluate, load: what happens between a directory and a running object.

    -

    The plugins component does the work. You tell it where plugins are and what each host kind needs:

    +
    VI

    Loading Plugins

    Two steps in two places: building a plugin into a script happens once, on the server; evaluating it happens in every process that runs it.

    +

    A plugin is source code on the server's disk: TypeScript files that import each other, import other plugins' +contracts, import npm packages, and import things the app itself provides. None of the processes that run the plugin +can use those files as they are. A browser tab has no file system and no TypeScript; a worker could read the files, +but then every worker would compile every plugin again, and two workers started a minute apart could see two different +versions. So loading a plugin is two separate steps:

    +
      +
    1. Build: turn each plugin facet (notes/worker.ts, notes/web.tsx) into one self-contained script. This +happens once per edit, on the server, for every host kind.
    2. +
    3. Evaluate: run a script inside one process, which gives back the facet, ready for host.load. This happens in +every process that hosts that kind: the server for its own facets, each worker for the worker facets, each tab for +the web facets.
    4. +
    +

    The two steps need different things, and that is why the API has two calls.

    +

    Host packages: what a script leaves out

    +

    Look at what notes/worker.ts imports:

    +
    import { defineFacet, Session } from "@app/core";             // the app's base code
    +import { obj, str, remote } from "@earendil-works/chord/next";  // Chord itself
    +import { Notes } from "./contract.ts";                            // the plugin's own file
    +import slugify from "slugify";                                    // an npm package in the plugin's dependencies
    +

    The last two go into the script: the plugin's own files and its npm dependencies are copied in. The first two must +not. If the script carried its own copy of Chord, the worker would run two Chords: the plugin's ChordError would not +be the worker's ChordError, and the plugin's defineFacet would make a facet the worker's host does not recognize. +The same holds for @app/core: the plugin must see the very Session host kind and the very Panels registry the +process already has, not a copy.

    +

    Packages the process already has, and that every plugin must share, are its host packages. A script leaves them +out and asks for them by name when it runs. Building therefore needs only their names: "do not copy +@earendil-works/chord/next or @app/core into this script".

    +

    Building: createPlugins, on the server

    const plugins = createPlugins({
    -	roots: ["./plugins"],
    +	roots: ["./plugins"],                                 // where the plugin directories are
     	kinds: {
     		server: { platform: "node", hostPackages: ["@earendil-works/chord/next", "@app/core"] },
     		worker: { platform: "node", hostPackages: ["@earendil-works/chord/next", "@app/core"] },
     		web: { platform: "browser", hostPackages: ["@earendil-works/chord/next", "@app/core", "react", "react/jsx-runtime"] },
     	},
     	onError,
    -}, ctx);
    +}, ctx);
    +

    createPlugins is the builder, and it builds for every kind, because only the server has the sources. For each kind +it needs two facts. The platform says where the scripts run: a node script may import node:fs, a browser +script may import a stylesheet but not node:fs. The host package names say what to leave out; they differ per +kind, because a tab also shares React with its plugins and a worker does not. Nothing here runs a plugin.

    +

    Evaluating: plugins.load, in the process that hosts

    +

    Evaluating is where the names become things. When the script runs and asks for @app/core, someone has to hand it +the actual module, the one this process imported when it started. Only the process itself has it:

    +
    // server.ts
    +import * as chord from "@earendil-works/chord/next";
    +import * as core from "./core.ts";                         // the app's base code, the same module plugins import as "@app/core"
     
     const report = await plugins.load(server, {
    -	kind: "server",
    -	core: [{ name: "@app/core", facet: coreServerFacet }],
    -	modules: { "@earendil-works/chord/next": chord, "@app/core": core },
    -}, ctx);
    -console.log(report.failed);   // plugins left out, with why: [] when all loaded
    -
    What plugins.load does

    Discover

    It lists ./plugins one level deep and reads each package.json. A directory without one, or with two server -files, is reported, not loaded.

    -

    Build

    For each plugin with a server facet, esbuild builds one script from server.ts and everything it imports: the -plugin's own files, another plugin's contract files, its npm dependencies. Host packages stay out. A syntax error is a -build failure, never a crash.

    -

    Evaluate

    The script runs and returns the facet. Its import "@app/core" gets the server's own @app/core, so a ChordError -from a plugin is the same class as the server's.

    -

    Load

    host.load runs every facet's setup and publishes all of them at once. A plugin whose build, evaluation or setup -fails is left out and listed in report.failed; the others load. Its objects' users wait, and the next edit loads it.

    + kind: "server", // evaluate the server kind's scripts + modules: { "@earendil-works/chord/next": chord, "@app/core": core }, // this process's own instances + core: [{ name: "@app/core", facet: core.serverFacet }], // base-code facets, loaded beside the plugins +}, ctx);
    +

    modules is this process's answer to every host package name. The server passes its own; a worker passes its own in +workerMain; a page passes its own in startPage. Each process has different instances, which is exactly the point: +inside each process, plugins share that process's Chord and base code.

    +

    core is the app's base code in facet form. It is a facet like a plugin's, but it is not a plugin: it is made in the +process's own code, often from values only this process has (core.workerFacet(start) closes over a worker's start +data and store), and it never reloads, because it defines what all plugins share. Passing it here loads it together +with the plugins, so they are published in one step.

    +

    Why plugins.load(host), not host.load

    +

    A host has its own load, and plugins.load calls it. The host's load takes facets that already exist: functions +in memory. It knows nothing about directories, builds, scripts or modules, on purpose: the same host runs in a test +with facets imported directly, or in a Cloudflare Durable Object where plugins are bundled at deploy time and nothing +is built or evaluated at runtime.

    +

    The plugins component is the part that knows about directories. It finds the plugins, builds them, evaluates them +with the process's modules, and then calls host.load with the result. It also remembers, per host, which facet each +plugin runs, so that a later reload can hand the host its complete new plugin set, with unchanged plugins as the very +same objects. That memory is why the host is passed to it: plugins.load(server, …) on the server, +plugins.load(worker, …) in a worker (where workerMain gives it a plugins that only evaluates), the same call +in every process.

    +
    SERVER: BUILD (ONCE PER EDIT)EACH WORKER: EVALUATEplugin sourcesnotes/worker.ts`, imports`@app/corecreatePluginscopies own files and npm deps; leaveshost packages outworker script`require("@app/core")` left as a namethe worker's modulesits own `@app/core`, its own Chordplugins.load(worker, …)runs the script, answers each namethe facetuses the worker's `Session`, theworker's `defineFacet`shipped
    Figure 3. Building on the server needs names; evaluating in each process needs that process's modules.
    +

    What happens, step by step

    +
    plugins.load(server, { kind: "server", … })

    Discover

    The component lists ./plugins one level deep and reads each package.json. Its name is the plugin's name; the +files named after a kind are its facets. A directory without a package.json, or with two server files, is reported, +not loaded.

    +

    Build

    For each plugin with a server facet it builds one script, if no current one exists. A syntax error or an import the +rules do not allow (a path into base code, a package not in the plugin's dependencies) is a failed build with the file +and line, never a crash.

    +

    Evaluate

    The script runs. Each host package name it asks for is answered from modules; anything else it may not have. The +script's default export must be a facet.

    +

    Load

    host.load runs the setups of core and of every plugin facet and publishes them together. A plugin whose build, +evaluation or setup failed is left out and listed in report.failed with the reason; the others load. Objects that +need it wait until a later edit loads it.

    -

    A worker never builds: the server builds the worker facets too, and each worker evaluates the scripts it is sent.

    +

    A worker never builds. The server builds the worker kind too and sends the scripts in the worker's start message; the +worker only evaluates, with its own modules. A page gets its scripts from the server over HTTP and evaluates them with +the modules of its base bundle.

    VII

    Plugins Using Plugins

    Two ways one plugin uses another, and the one it may not.

    From bb36171aac46adee2b43030fb1e4d8fd55675121 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 14:28:28 +0000 Subject: [PATCH 206/271] docs(chord): tutorial: only the server builds --- packages/chord/docs/next/guide-plugins-hosts/book.md | 6 +++--- packages/chord/docs/next/guide-plugins-hosts/index.html | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/chord/docs/next/guide-plugins-hosts/book.md b/packages/chord/docs/next/guide-plugins-hosts/book.md index 563ba5315..81dea645f 100644 --- a/packages/chord/docs/next/guide-plugins-hosts/book.md +++ b/packages/chord/docs/next/guide-plugins-hosts/book.md @@ -480,9 +480,9 @@ evaluation or setup failed is left out and listed in `report.failed` with the re need it wait until a later edit loads it. ::: -A worker never builds. The server builds the worker kind too and sends the scripts in the worker's start message; the -worker only evaluates, with its own modules. A page gets its scripts from the server over HTTP and evaluates them with -the modules of its base bundle. +Only the server builds. Workers and clients (browser tabs, terminal UIs) only evaluate: the server builds their kinds +too, a worker gets its scripts in its start message, a tab or a TUI gets them from the server over HTTP, and each +evaluates them with its own modules (a worker's from `workerMain`, a page's from its base bundle). # Plugins Using Plugins {#together lede="Two ways one plugin uses another, and the one it may not."} diff --git a/packages/chord/docs/next/guide-plugins-hosts/index.html b/packages/chord/docs/next/guide-plugins-hosts/index.html index a62ac6175..efff61f9f 100644 --- a/packages/chord/docs/next/guide-plugins-hosts/index.html +++ b/packages/chord/docs/next/guide-plugins-hosts/index.html @@ -457,9 +457,9 @@ script's default export must be a facet.

    evaluation or setup failed is left out and listed in report.failed with the reason; the others load. Objects that need it wait until a later edit loads it.

    -

    A worker never builds. The server builds the worker kind too and sends the scripts in the worker's start message; the -worker only evaluates, with its own modules. A page gets its scripts from the server over HTTP and evaluates them with -the modules of its base bundle.

    +

    Only the server builds. Workers and clients (browser tabs, terminal UIs) only evaluate: the server builds their kinds +too, a worker gets its scripts in its start message, a tab or a TUI gets them from the server over HTTP, and each +evaluates them with its own modules (a worker's from workerMain, a page's from its base bundle).

    VII

    Plugins Using Plugins

    Two ways one plugin uses another, and the one it may not.

    From 112a8b46f35944b57b9c5192a7408ed1290b48cd Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 14:43:24 +0000 Subject: [PATCH 207/271] docs(chord): tutorial: base code vs plugins, shipped plugins, subpaths --- .../docs/next/guide-plugins-hosts/book.md | 53 +++++++++++++++++-- .../docs/next/guide-plugins-hosts/index.html | 41 +++++++++++--- 2 files changed, 83 insertions(+), 11 deletions(-) diff --git a/packages/chord/docs/next/guide-plugins-hosts/book.md b/packages/chord/docs/next/guide-plugins-hosts/book.md index 81dea645f..fa146bc0b 100644 --- a/packages/chord/docs/next/guide-plugins-hosts/book.md +++ b/packages/chord/docs/next/guide-plugins-hosts/book.md @@ -499,6 +499,13 @@ host.serve(Search, ({ project }, principal, self) => { }); ``` +:::review | Proposed: importing another plugin by name +Today another plugin's file is imported by relative path, which works only when both plugins sit in the same root. A +plugin in the user's root cannot reach one the app ships in `src/plugins` that way. Proposed (not decided): import by +plugin name, `import { Notes } from "@app/notes/contract"`, resolved to the discovered plugin of that name wherever its +directory is. +::: + `self.get` in a factory declares a dependency: Chord makes `Search` live only once `Notes` is, and if the notes plugin is missing, `Search` waits with a clear reason ("waiting for notes: not served here") instead of failing. @@ -542,14 +549,50 @@ A change to a contract (a new field in `Notes`'s state) gets a new `version`. A working if the provider maps the old version (`accepts`); otherwise it waits with "version mismatch" until it reloads. ::: -# Base Code and Plugins {.epilogue lede="What reloads, what restarts, and where the line is."} +# Base Code and Plugins {#base lede="What goes into the app itself, what goes into plugins, and how an app ships plugins of its own."} -Everything in `plugins/` reloads: edit, build, roll out. Everything in your base code (`src/`, `@app/core`: host -kinds, principals, the core facet, the app's startup) changes only with a restart, and the watcher tells you so. -Keep base code small: host kinds, principals, and what every plugin needs. Put features in plugins. +Every app has two kinds of code, and the line between them decides what can change while the app runs. + +**Base code** is the app's own program: `src/`, started by `node server.ts`. It is what every plugin shares: the host +kinds (`Server`, `Project`), the principal type, the app's `defineFacet`, the registries plugins contribute to +(`Panels`), and the startup that creates hosts and loads plugins. Plugins import it as a host package (`@app/core`), +so inside one process there is exactly one of each. Base code changes only with a restart: if it reloaded, the +process would hold two `Panels` registries, and old and new plugins would no longer meet. + +**Plugins** are features: notes, search, export, a terminal. They reload on an edit, a broken one is left out alone, +and an agent can write one while the app runs. + +The rule: **base code is what everything shares, not features.** A feature in base code cannot be edited live, and a +bug in it needs a restart. Keep base code to host kinds, principals, shared registries and startup. + +## Plugins the app ships + +An app usually comes with plugins of its own: notes, search, settings. They are still plugins, kept in a root inside +the app, next to the root where users and agents add theirs: + +```ts +const plugins = createPlugins({ + roots: ["./src/plugins", join(dataDir, "plugins")], // the app's own plugins, then the user's + kinds, + onError, +}, ctx); +``` + +They are built at startup like any plugin (a quarter of a second for twenty), reload like any plugin, and are left +out alone when broken. If two roots have a plugin with the same name, the first root wins, and the other is reported. + +## Subpaths of host packages + +A host package is matched by its exact name. If base code offers several entry points (`@app/core` for every host, +`@app/core/web` for pages), each is its own host package, listed by name for the build and given as a module when +evaluating, as `react` and `react/jsx-runtime` are. A plugin importing a subpath that is not listed gets a build +error naming it. + +## What the app kit will change The app kit being designed next (`chord/next/app`) will make the base code nearly empty: `startApp` and `startWorker` -with Chord's conventions for hosts, users, authentication and storage, so an app is mostly plugins. +with Chord's conventions for hosts, users, authentication and storage. An app is then its plugins, its own ones in +`src/plugins` and the user's beside them. # Reference {.appendix} diff --git a/packages/chord/docs/next/guide-plugins-hosts/index.html b/packages/chord/docs/next/guide-plugins-hosts/index.html index efff61f9f..0a5e1ca41 100644 --- a/packages/chord/docs/next/guide-plugins-hosts/index.html +++ b/packages/chord/docs/next/guide-plugins-hosts/index.html @@ -129,7 +129,7 @@ writes, a note says so.

  • VI. Loading Plugins
  • VII. Plugins Using Plugins
  • VIII. Changing a Plugin
  • -
  • Base Code and Plugins
  • +
  • IX. Base Code and Plugins
  • Appendix A: Reference
  • @@ -472,6 +472,11 @@ host.serve(Search, ({ projec const index = self.derive([notes.list], ([list]) => buildIndex(list.items)); return { variant: "main", members: { find: async ({ q }) => lookup(index.value, q) } }; });
    +
    Proposed: importing another plugin by name

    Today another plugin's file is imported by relative path, which works only when both plugins sit in the same root. A +plugin in the user's root cannot reach one the app ships in src/plugins that way. Proposed (not decided): import by +plugin name, import { Notes } from "@app/notes/contract", resolved to the discovered plugin of that name wherever its +directory is.

    +

    self.get in a factory declares a dependency: Chord makes Search live only once Notes is, and if the notes plugin is missing, Search waits with a clear reason ("waiting for notes: not served here") instead of failing.

    The other way is a registry: an extension point one plugin defines and others contribute to. The app's core @@ -518,12 +523,36 @@ working if the provider maps the old version (accepts); otherwise i

    -
    ∴

    Base Code and Plugins

    What reloads, what restarts, and where the line is.

    -

    Everything in plugins/ reloads: edit, build, roll out. Everything in your base code (src/, @app/core: host -kinds, principals, the core facet, the app's startup) changes only with a restart, and the watcher tells you so. -Keep base code small: host kinds, principals, and what every plugin needs. Put features in plugins.

    +
    IX

    Base Code and Plugins

    What goes into the app itself, what goes into plugins, and how an app ships plugins of its own.

    +

    Every app has two kinds of code, and the line between them decides what can change while the app runs.

    +

    Base code is the app's own program: src/, started by node server.ts. It is what every plugin shares: the host +kinds (Server, Project), the principal type, the app's defineFacet, the registries plugins contribute to +(Panels), and the startup that creates hosts and loads plugins. Plugins import it as a host package (@app/core), +so inside one process there is exactly one of each. Base code changes only with a restart: if it reloaded, the +process would hold two Panels registries, and old and new plugins would no longer meet.

    +

    Plugins are features: notes, search, export, a terminal. They reload on an edit, a broken one is left out alone, +and an agent can write one while the app runs.

    +

    The rule: base code is what everything shares, not features. A feature in base code cannot be edited live, and a +bug in it needs a restart. Keep base code to host kinds, principals, shared registries and startup.

    +

    Plugins the app ships

    +

    An app usually comes with plugins of its own: notes, search, settings. They are still plugins, kept in a root inside +the app, next to the root where users and agents add theirs:

    +
    const plugins = createPlugins({
    +	roots: ["./src/plugins", join(dataDir, "plugins")],   // the app's own plugins, then the user's
    +	kinds,
    +	onError,
    +}, ctx);
    +

    They are built at startup like any plugin (a quarter of a second for twenty), reload like any plugin, and are left +out alone when broken. If two roots have a plugin with the same name, the first root wins, and the other is reported.

    +

    Subpaths of host packages

    +

    A host package is matched by its exact name. If base code offers several entry points (@app/core for every host, +@app/core/web for pages), each is its own host package, listed by name for the build and given as a module when +evaluating, as react and react/jsx-runtime are. A plugin importing a subpath that is not listed gets a build +error naming it.

    +

    What the app kit will change

    The app kit being designed next (chord/next/app) will make the base code nearly empty: startApp and startWorker -with Chord's conventions for hosts, users, authentication and storage, so an app is mostly plugins.

    +with Chord's conventions for hosts, users, authentication and storage. An app is then its plugins, its own ones in +src/plugins and the user's beside them.

    A

    Reference

    From 56cae782341c339d3bb820081f0f9425d8c5b1ec Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 14:46:28 +0000 Subject: [PATCH 208/271] docs(chord): plugins import each other by name --- packages/chord/docs/next/decisions.md | 7 +++++++ packages/chord/docs/next/guide-plugins-hosts/book.md | 12 +++++------- .../chord/docs/next/guide-plugins-hosts/index.html | 10 ++++------ 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 4c35c4f33..a2d98b44d 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2063,6 +2063,13 @@ validation and copies the v2 kernel added around Delta. and TUI, decided point by point; the slice 10 components are its inside. Inputs from the owner: user management is part of it ("user management will also need to be a thing we support"), and app shapes without workers (server and clients only: "a chat app doesn't need workers"). +- **Plugins import each other by name (owner, 2026-10-31: "i like it"; changes 7c's cross-plugin rule):** + `import { Notes } from "@app/notes/contract"` resolves to the discovered plugin named `@app/notes`, its file + `contract.ts`, wherever its directory is (any root), copied into the importer's script as before (contracts and + side-effect-free files only; they may import only host packages and files of their own plugin); a missing plugin is + a build error naming it; relative paths into another plugin's directory are refused. Owner's notes for the app kit + pass: third-party plugins need the imported plugins' sources or types for type checking (publishing each plugin to + npm works but is unpleasant); plugins should be testable on their own, with their dependencies mocked. - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a diff --git a/packages/chord/docs/next/guide-plugins-hosts/book.md b/packages/chord/docs/next/guide-plugins-hosts/book.md index fa146bc0b..37388c890 100644 --- a/packages/chord/docs/next/guide-plugins-hosts/book.md +++ b/packages/chord/docs/next/guide-plugins-hosts/book.md @@ -490,7 +490,7 @@ A search plugin wants to search the notes. It **imports the contract** and asks ```ts // plugins/search/worker.ts -import { Notes } from "../notes/contract.ts"; // copied into search's script: contracts only +import { Notes } from "@app/notes/contract"; // another plugin, by its name: copied into search's script host.serve(Search, ({ project }, principal, self) => { const notes = self.get(Notes, { project }); // a dependency: Search goes live once Notes is @@ -499,12 +499,10 @@ host.serve(Search, ({ project }, principal, self) => { }); ``` -:::review | Proposed: importing another plugin by name -Today another plugin's file is imported by relative path, which works only when both plugins sit in the same root. A -plugin in the user's root cannot reach one the app ships in `src/plugins` that way. Proposed (not decided): import by -plugin name, `import { Notes } from "@app/notes/contract"`, resolved to the discovered plugin of that name wherever its -directory is. -::: + +A plugin imports another plugin by its **name**, never by a path: `@app/notes/contract` is the file `contract.ts` of +whichever discovered plugin is named `@app/notes`, in any root. The file is copied into the importer's script, so only +contracts and other side-effect-free files belong there; a missing plugin is a build error naming it. `self.get` in a factory declares a dependency: Chord makes `Search` live only once `Notes` is, and if the notes plugin is missing, `Search` waits with a clear reason ("waiting for notes: not served here") instead of failing. diff --git a/packages/chord/docs/next/guide-plugins-hosts/index.html b/packages/chord/docs/next/guide-plugins-hosts/index.html index 0a5e1ca41..9d2f815af 100644 --- a/packages/chord/docs/next/guide-plugins-hosts/index.html +++ b/packages/chord/docs/next/guide-plugins-hosts/index.html @@ -465,18 +465,16 @@ evaluates them with its own modules (a worker's from workerMain
    VII

    Plugins Using Plugins

    Two ways one plugin uses another, and the one it may not.

    A search plugin wants to search the notes. It imports the contract and asks for the object:

    // plugins/search/worker.ts
    -import { Notes } from "../notes/contract.ts";   // copied into search's script: contracts only
    +import { Notes } from "@app/notes/contract";    // another plugin, by its name: copied into search's script
     
     host.serve(Search, ({ project }, principal, self) => {
     	const notes = self.get(Notes, { project });   // a dependency: Search goes live once Notes is
     	const index = self.derive([notes.list], ([list]) => buildIndex(list.items));
     	return { variant: "main", members: { find: async ({ q }) => lookup(index.value, q) } };
     });
    -
    Proposed: importing another plugin by name

    Today another plugin's file is imported by relative path, which works only when both plugins sit in the same root. A -plugin in the user's root cannot reach one the app ships in src/plugins that way. Proposed (not decided): import by -plugin name, import { Notes } from "@app/notes/contract", resolved to the discovered plugin of that name wherever its -directory is.

    -
    +

    A plugin imports another plugin by its name, never by a path: @app/notes/contract is the file contract.ts of +whichever discovered plugin is named @app/notes, in any root. The file is copied into the importer's script, so only +contracts and other side-effect-free files belong there; a missing plugin is a build error naming it.

    self.get in a factory declares a dependency: Chord makes Search live only once Notes is, and if the notes plugin is missing, Search waits with a clear reason ("waiting for notes: not served here") instead of failing.

    The other way is a registry: an extension point one plugin defines and others contribute to. The app's core From ed2bdde64b0e62c5083ac2bc694baf541f5cd670 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 15:04:43 +0000 Subject: [PATCH 209/271] docs(chord): HTTP core as a fetch handler, Node listener as the easy path --- packages/chord/docs/next/decisions.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index a2d98b44d..84798b324 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2070,6 +2070,15 @@ validation and copies the v2 kernel added around Delta. a build error naming it; relative paths into another plugin's directory are refused. Owner's notes for the app kit pass: third-party plugins need the imported plugins' sources or types for type checking (publishing each plugin to npm works but is unpleasant); plugins should be testable on their own, with their dependencies mocked. +- **The HTTP core is a `fetch` handler; Chord's own Node server is the easy path (owner, 2026-10-31: "could http just plug + into hono/express?"; "we should still provide the easy path using just node's shit as well, for small apps or apps + where we don't care"):** the component builds one Web-standard handler, `(request: Request) => Promise`, + from `Http` (admission, the security baseline, router, wraps, descriptions), plus per-runtime upgrade adapters for + `/chord` and plugin sockets. The default is Chord's own listener, the native host-lifetime object of `chord.http` + over Node's `http` (decision "The HTTP server is a native object"); an app with its own server mounts the handler + (`hono.all("/*", (c) => http.fetch(c.req.raw))`, an Express adapter) and passes upgrades to the adapter. The same + routes then run on other runtimes later (Bun, Durable Objects, design §26 #16). Built in slice 10's fix round + (reworks `server.ts`, `request.ts`, `response.ts`; router, admission rules and descriptions stay). - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From 705d047cb5635be5d430c34917be714979cc7709 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Sat, 31 Oct 2026 16:10:55 +0100 Subject: [PATCH 210/271] fix(coding-agent): tighten system prompt --- packages/coding-agent/CHANGELOG.md | 1 + packages/coding-agent/src/core/skills.ts | 2 +- packages/coding-agent/src/core/system-prompt.ts | 11 +++++------ packages/coding-agent/src/core/tools/edit.ts | 6 ++---- packages/coding-agent/test/system-prompt.test.ts | 10 +++++++++- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index d10174db2..483e87088 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -12,6 +12,7 @@ ### Changed +- Tightened system prompt - `pi update` for installs from the pi.dev installer no longer passes `--min-release-age=0` to `npm ci`. The release lockfile pins every package, so the flag had no effect, and security scanners that block it stopped the update ([#11262](https://github.com/earendil-works/pi/issues/11262)) - `/model` and `/scoped-models` now show a model's name next to its ID when the name adds words the ID lacks, such as a renamed `models.json` override ([#5916](https://github.com/earendil-works/pi/issues/5916), [#11347](https://github.com/earendil-works/pi/pull/11347)) - MCP OAuth sign-in now uses pi's Client ID Metadata Document by default when the authorization server supports it for public clients, and registers a client dynamically otherwise, like Codex, Claude Code, and opencode. Clients registered before stay in use while their redirect URI matches. The new default `oauth.clientRegistration: "auto"` registers when `clientName` or a non-default `callbackUrl` is set; `"dcr"` always registers. `pi mcp add --oauth-client-registration` sets the mode ([#11426](https://github.com/earendil-works/pi/issues/11426)) diff --git a/packages/coding-agent/src/core/skills.ts b/packages/coding-agent/src/core/skills.ts index 7440c4684..48df4536b 100644 --- a/packages/coding-agent/src/core/skills.ts +++ b/packages/coding-agent/src/core/skills.ts @@ -369,7 +369,7 @@ export function formatSkillsForPrompt(skills: Skill[], fileReadTool: "read" | "b : fileReadTool === "bash" ? "Use bash to load a skill's file when the task matches its description." : "Load a skill's file when the task matches its description.", - "When a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.", + "Resolve relative paths against the skill's directory (parent of SKILL.md). Use absolute paths in tool calls.", "", "", ]; diff --git a/packages/coding-agent/src/core/system-prompt.ts b/packages/coding-agent/src/core/system-prompt.ts index 49bcb58ee..e51e14fe4 100644 --- a/packages/coding-agent/src/core/system-prompt.ts +++ b/packages/coding-agent/src/core/system-prompt.ts @@ -153,20 +153,19 @@ export function buildSystemPromptSections(input: BuildSystemPromptOptions): Syst promptSections.preamble = customPrompt; } else { promptSections.preamble = - "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files."; + "You are a coding assistant running inside pi, a coding agent harness. Help users inspect files, run commands, and modify code."; const visibleTools = declaredTools.filter((name) => !!toolSnippets[name]); const tools = visibleTools.length > 0 ? visibleTools.map((name) => `- ${name}: ${toolSnippets[name]}`).join("\n") : "(none)"; - promptSections.tools = `${tools}\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.`; + promptSections.tools = `${tools}\n\nAdditional tools may be available.`; promptSections.rules = buildRules(declaredTools, toolGuidelines, promptGuidelines); - promptSections.docs = `Pi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI): + promptSections.docs = `Pi documentation (read only when the user asks about the harness itself, its SDK, extensions, themes, skills, or TUI): - Main documentation: ${getReadmePath()} - Additional docs: ${getDocsPath()} - Examples: ${getExamplesPath()} (extensions, custom tools, SDK) - When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory -- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md) -- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing -- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)`; +- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md) +- Before implementing harness changes, read the relevant docs and examples in full, following related cross-references.`; } if (appendSystemPrompt) promptSections.addendum = appendSystemPrompt; diff --git a/packages/coding-agent/src/core/tools/edit.ts b/packages/coding-agent/src/core/tools/edit.ts index 62e6fdee0..ad1cbd2f3 100644 --- a/packages/coding-agent/src/core/tools/edit.ts +++ b/packages/coding-agent/src/core/tools/edit.ts @@ -43,10 +43,8 @@ const editSchema = Type.Object( export const editToolSystemPromptContribution = { snippet: "Make precise file edits with exact text replacement, including multiple disjoint edits in one call", guidelines: [ - "Use edit for precise changes (edits[].oldText must match exactly)", - "When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls", - "Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.", - "Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.", + "Use edit for targeted changes. Batch changes to one file in a single call.", + "Each edits[].oldText must exactly and uniquely match the original file. Keep matches short and non-overlapping; merge nearby changes without including large unchanged regions.", ], } as const; diff --git a/packages/coding-agent/test/system-prompt.test.ts b/packages/coding-agent/test/system-prompt.test.ts index 276cc63ff..86d2c6994 100644 --- a/packages/coding-agent/test/system-prompt.test.ts +++ b/packages/coding-agent/test/system-prompt.test.ts @@ -48,7 +48,9 @@ describe("buildSystemPrompt", () => { skills: [], }); - expect(defaultPrompt.startsWith("You are an expert coding assistant operating inside pi")).toBe(true); + expect(defaultPrompt.startsWith("You are a coding assistant running inside pi, a coding agent harness.")).toBe( + true, + ); expect(customPrompt.startsWith("You are Exact.\n\n")).toBe(true); }); @@ -119,6 +121,9 @@ describe("buildSystemPrompt", () => { "- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory", ); expect(prompt).toContain("environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md)"); + expect(prompt).toContain( + "Before implementing harness changes, read the relevant docs and examples in full, following related cross-references.", + ); }); }); @@ -192,6 +197,9 @@ describe("buildSystemPrompt", () => { expect(prompt).toContain(""); expect(prompt).toContain("test-skill"); expect(prompt).toContain("Use bash to load a skill's file"); + expect(prompt).toContain( + "Resolve relative paths against the skill's directory (parent of SKILL.md). Use absolute paths in tool calls.", + ); }); test("omits skills without read or bash", () => { From 25426f72ae04a306f1e08de09583b4e89503b9e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Hou=C5=A1ka?= Date: Sat, 31 Oct 2026 16:15:42 +0100 Subject: [PATCH 211/271] docs(coding-agent): document Windows terminal and conhost limitations (#11644) Adds a "Choose a terminal" section to `packages/coding-agent/docs/windows.md` (https://pi.dev/docs/latest/windows), with two tables (last updated October 2026): - Terminals and which conhost they use: Windows Terminal (recommended), VS Code, WezTerm nightly (bundled, up to date); WezTerm 20240203 (bundled, old); Alacritty (built-in unless `conpty.dll` + `OpenConsole.exe` are provided); classic console window, mintty, ConEmu/Cmder, Windows OpenSSH (built-in). - Known problems by conhost type: - Built-in: crash when switching fullscreen -> regular or using `Ctrl+G` from fullscreen (#9186, microsoft/terminal#17515). - Built-in behind another terminal (mintty, OpenSSH): query replies leak into the prompt, BEL opens the external editor (#10256, #10362, #10742). - Old bundled (WezTerm 20240203): reply fragments in the prompt (#10717, #10970). - Built-in, classic console: reported regular-mode redraw issue (#6300). - Notes that the fixes exist upstream and reach the built-in conhost with a future Windows update, timing unknown. Sources: VS Code defaults `terminal.integrated.windowsUseConptyDll` to `true` (bundled conpty v1.25). Alacritty loads `conpty.dll` only if present (`alacritty_terminal/src/tty/windows/conpty.rs`) and its installer ships only `alacritty.exe`. The crash was verified locally in the classic console window on conhost 10.0.26100.9807: fullscreen -> regular crashed 3/3 runs and `Ctrl+G` from fullscreen crashed 2/2 runs (`0xC0000005`). --- packages/coding-agent/docs/windows.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/coding-agent/docs/windows.md b/packages/coding-agent/docs/windows.md index 26f2f961c..18826cd02 100644 --- a/packages/coding-agent/docs/windows.md +++ b/packages/coding-agent/docs/windows.md @@ -76,6 +76,31 @@ JSON uses backslashes for escape sequences. When you write a Windows path with b See [Configure shell commands](shell-aliases.md) for command prefixes, aliases, and the complete shell-resolution behavior. +## Choose a terminal + +Windows terminals run Pi through a console host (conhost). Some bundle an up-to-date copy; others use the older one built into Windows, which has bugs that affect Pi. + +We recommend [Windows Terminal](https://aka.ms/terminal). It always ships an up-to-date conhost, and Pi is best tested with it. + +Last updated: October 2026. + +| Terminal | Conhost | Status | +|---|---|---| +| [Windows Terminal](https://aka.ms/terminal) | Bundled, current | Recommended | +| [VS Code](https://code.visualstudio.com), [Tabby](https://tabby.sh), [WezTerm](https://wezterm.org) nightly | Bundled, current | Works | +| [Alacritty](https://alacritty.org) | Built-in, unless `conpty.dll` and `OpenConsole.exe` from Windows Terminal are next to `alacritty.exe` | Works only with those files | +| WezTerm 20240203 and older | Bundled, old | Known problems | +| Classic console (`cmd.exe` or PowerShell without Windows Terminal), mintty (Git Bash), ConEmu, Cmder, [Hyper](https://hyper.is), Windows OpenSSH | Built-in | Known problems | + +| Problem | Where | Why | +|---|---|---| +| Switching from fullscreen to regular mode, or `Ctrl+G` from fullscreen, crashes the session ([#9186](https://github.com/earendil-works/pi/issues/9186), [microsoft/terminal#17515](https://github.com/microsoft/terminal/issues/17515)). Start with `--tui-mode regular` to avoid it. | Built-in | Conhost crashes when leaving the alternate screen while input is in line mode. | +| Text such as `;0;rgb:0000/0000/0000` appears in the prompt and BEL opens the external editor ([#10256](https://github.com/earendil-works/pi/issues/10256), [#10362](https://github.com/earendil-works/pi/issues/10362), [#10742](https://github.com/earendil-works/pi/issues/10742)) | Built-in behind another terminal (mintty, OpenSSH) | Conhost drops the start of terminal replies that arrive in pieces. | +| Fragments such as `79` or `55` appear in the prompt ([#10717](https://github.com/earendil-works/pi/issues/10717), [#10970](https://github.com/earendil-works/pi/issues/10970)) | WezTerm 20240203 | The bundled conhost cuts terminal replies short. | +| In regular mode, each keystroke redraws the input line on a new row ([#6300](https://github.com/earendil-works/pi/issues/6300)) | Built-in, classic console | Suspected: cursor wraps after full-width lines. | + +Microsoft has fixed these bugs in its source, but has not said when the built-in conhost will get the fixes. + ## Configure Windows Terminal Windows Terminal reserves or rewrites some modified keys. See [Windows Terminal](terminal-setup.md#windows-terminal) to configure `Shift+Enter` and `Alt+Enter`, and [Keybindings](keybindings.md) for Pi's Windows and WSL shortcut defaults. From f0646a00fb4c43bf941deb9d934280e5a92361c2 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 15:57:12 +0000 Subject: [PATCH 212/271] docs(chord): slice 10 fix round small points --- packages/chord/docs/next/decisions.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 84798b324..ec2672c24 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2079,6 +2079,11 @@ validation and copies the v2 kernel added around Delta. (`hono.all("/*", (c) => http.fetch(c.req.raw))`, an Express adapter) and passes upgrades to the adapter. The same routes then run on other runtimes later (Bun, Durable Objects, design §26 #16). Built in slice 10's fix round (reworks `server.ts`, `request.ts`, `response.ts`; router, admission rules and descriptions stay). +- **Slice 10 fix round, three small points (owner, 2026-10-31: "yes on fileResponse etc"):** (1) `fileResponse(path, { + root? })`: with a `root`, a file whose real path lies outside it (a symlink out) is refused 403; without one the path + is trusted (a plugin's own file). (2) The worker lock on macOS and the BSDs fails closed on a stale lock file, with an + error naming it (no takeover; Linux and Android use abstract sockets and are unaffected). (3) A route's overhead: + Opus's two trims, measured again after the `fetch` rework, and the numbers brought to the owner. - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From 6de14ec09862856efa3edd5a8b734b14ab1afd09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Hou=C5=A1ka?= Date: Sat, 31 Oct 2026 17:29:36 +0100 Subject: [PATCH 213/271] docs(coding-agent): note tmux 3.6 color reply leak in tmux guide (#11645) Adds a troubleshooting section to the tmux guide (`packages/coding-agent/docs/tmux.md`, published at https://pi.dev/docs/latest/tmux) for the color-code text that tmux 3.6 types into the editor at startup. tmux 3.6 forwards Pi's palette queries to the outer terminal. When a reply reaches tmux split across reads, tmux treats its bytes as key presses and sends them to Pi as typed text. tmux 3.7 fixes this ([tmux#4749](https://github.com/tmux/tmux/issues/4749), [tmux#4793](https://github.com/tmux/tmux/issues/4793)). Pi cannot filter the text because it arrives exactly like typing. The section also notes that switching to a fixed theme does not help: `applyFromSettings()` queries terminal colors regardless of the theme, which reporters confirmed in #10679 (`--use-theme dark`) and #10256 (`"theme": "dark"`). Related: #10250, #10679, #10256. --- packages/coding-agent/docs/tmux.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/coding-agent/docs/tmux.md b/packages/coding-agent/docs/tmux.md index 091d61647..c781ffe0a 100644 --- a/packages/coding-agent/docs/tmux.md +++ b/packages/coding-agent/docs/tmux.md @@ -53,3 +53,7 @@ set -g extended-keys on Pi supports the xterm `modifyOtherKeys` format used by these versions. Restart tmux and repeat the verification steps. For older versions, upgrade tmux or use Pi outside tmux rather than relying on modified Enter shortcuts. + +## Color codes appear in the editor with tmux 3.6 + +If text such as `c0c/0c0c/0c0c` appears in the editor at startup, upgrade tmux to 3.7 or newer. tmux 3.6 sometimes sends the terminal's color replies to Pi as typed text ([tmux#4749](https://github.com/tmux/tmux/issues/4749)). Changing Pi's theme does not help. From b4c2e92acbecdab7a1ff1cc36bde92f3206f5ddd Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Sat, 31 Oct 2026 17:54:29 +0100 Subject: [PATCH 214/271] fix: remove vulnerable dependency chains Replace build-time shell utilities with Node filesystem operations and retire the native sandbox example in favor of existing isolation options. Upgrade glob matching and worker-test dependencies and refresh the lockfiles. --- package-lock.json | 492 ++++-------------- package.json | 3 - packages/agent/package.json | 2 +- packages/ai/package.json | 4 +- packages/chord/package.json | 3 +- packages/codemode/package.json | 3 +- packages/coding-agent/CHANGELOG.md | 5 + packages/coding-agent/docs/extensions.md | 1 - .../examples/extensions/README.md | 3 +- .../examples/extensions/sandbox/.gitignore | 1 - .../examples/extensions/sandbox/index.ts | 321 ------------ .../extensions/sandbox/package-lock.json | 92 ---- .../examples/extensions/sandbox/package.json | 19 - .../install-lock/package-lock.json | 17 +- packages/coding-agent/package.json | 11 +- packages/durable/package.json | 5 +- packages/env/package.json | 3 +- packages/evals/package.json | 3 +- packages/mcp/package.json | 3 +- packages/telemetry/package.json | 2 +- packages/tui/package.json | 2 +- scripts/npm-audit.mjs | 10 +- scripts/package-files.mjs | 30 ++ scripts/package-files.test.mjs | 82 +++ 24 files changed, 260 insertions(+), 857 deletions(-) delete mode 100644 packages/coding-agent/examples/extensions/sandbox/.gitignore delete mode 100644 packages/coding-agent/examples/extensions/sandbox/index.ts delete mode 100644 packages/coding-agent/examples/extensions/sandbox/package-lock.json delete mode 100644 packages/coding-agent/examples/extensions/sandbox/package.json create mode 100644 scripts/package-files.mjs create mode 100644 scripts/package-files.test.mjs diff --git a/package-lock.json b/package-lock.json index 7c8fb2531..f2129fd85 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,18 +13,15 @@ "packages/coding-agent/examples/extensions/with-deps", "packages/coding-agent/examples/extensions/custom-provider-anthropic", "packages/coding-agent/examples/extensions/custom-provider-gitlab-duo", - "packages/coding-agent/examples/extensions/sandbox", "packages/coding-agent/examples/extensions/gondolin" ], "devDependencies": { - "@anthropic-ai/sandbox-runtime": "0.0.26", "@biomejs/biome": "2.3.5", "@biomejs/wasm-nodejs": "2.3.5", "@types/node": "22.19.19", "esbuild": "0.28.2", "husky": "9.1.7", "pnpm": "11.15.1", - "shx": "0.3.4", "typescript": "7.0.2", "yaml": "2.9.0" }, @@ -32,26 +29,6 @@ "node": ">=22.19.0" } }, - "node_modules/@anthropic-ai/sandbox-runtime": { - "version": "0.0.26", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sandbox-runtime/-/sandbox-runtime-0.0.26.tgz", - "integrity": "sha512-DYV5LSsVMnzq0lbfaYMSpxZPUMAx4+hy343dRss+pVCLIfF62qOhxpYfZ5TmOk1GTDQm5f9wPprMNSStmnsV4w==", - "license": "Apache-2.0", - "dependencies": { - "@pondwader/socks5-server": "^1.0.10", - "@types/lodash-es": "^4.17.12", - "commander": "^12.1.0", - "lodash-es": "^4.17.21", - "shell-quote": "^1.8.3", - "zod": "^3.24.1" - }, - "bin": { - "srt": "dist/cli.js" - }, - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@anthropic-ai/sdk": { "version": "0.129.0", "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.129.0.tgz", @@ -620,26 +597,26 @@ } }, "node_modules/@cloudflare/vitest-plugin": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/@cloudflare/vitest-plugin/-/vitest-plugin-1.4.4.tgz", - "integrity": "sha512-nXNIlPiLvYFx/AH4/Bb/yntVkeloYcf1+tBi3l6Gp3TDCDtLxXAugrUtO7eTOWbHP6UIx5f9npy05OrACbZ5wg==", + "version": "1.4.10", + "resolved": "https://registry.npmjs.org/@cloudflare/vitest-plugin/-/vitest-plugin-1.4.10.tgz", + "integrity": "sha512-F6klOeT/Wx0gKvZ4zpYVBixP6TjcT4OD/L/RJK3Yq3SwRmd+v96/y037xUEsvGytdUisL9X5Jw3ADuNdvwBHdg==", "dev": true, "license": "MIT", "dependencies": { "cjs-module-lexer": "1.2.3", "esbuild": "0.28.2", - "miniflare": "5.20261012.1-beta", - "wrangler": "4.151.1", - "zod": "4.6.5" + "miniflare": "5.20261023.2-beta", + "wrangler": "4.155.0", + "zod": "4.6.6" }, "peerDependencies": { "vitest": "^4.1.0 || ^5.0.0" } }, "node_modules/@cloudflare/vitest-plugin/node_modules/zod": { - "version": "4.6.5", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", - "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "version": "4.6.6", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.6.tgz", + "integrity": "sha512-4UmzQUF5fxvfcYHlbjBmejU+w+hbDnWFx4aS8kBx7nPBHcO2SqPbpAG8snbdg+Xse4pRefR98hgjFgIMCI/UhA==", "dev": true, "license": "MIT", "funding": { @@ -647,9 +624,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-64": { - "version": "1.20261012.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20261012.1.tgz", - "integrity": "sha512-dkTL9aY3V4D40b+jBpGPCistWUPFQ/M61wlWi6i1o8MvHFFgEw7EQkcaYEv6Yd3bkPDD/3TFT4hRsMb2xVZSwQ==", + "version": "1.20261023.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20261023.1.tgz", + "integrity": "sha512-lsEx0i7nR0PNG/erUFFHmQmiidEhLfDE5uQmAL9421/y4/ycuxDDOSaxu3VGGYFFPK5sduyUeMmjIlX4CwOBFg==", "cpu": [ "x64" ], @@ -664,9 +641,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-arm64": { - "version": "1.20261012.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20261012.1.tgz", - "integrity": "sha512-2/nVGcKXjNNI+DhxKSepTKvFng9H7aN2WXv6Zu67iqgI6JrxypNt5kTPv4ZHosxrcfyKMci/2RfN7GIrN0BitQ==", + "version": "1.20261023.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20261023.1.tgz", + "integrity": "sha512-5e79fHgabdDD8EgmjOhJ1S+xMsf6DbC7fL7IgTBizMnzOp0P2isDn7pYmUxI9AJZE9FVQ4RUilH/I36YNg+GQQ==", "cpu": [ "arm64" ], @@ -681,9 +658,9 @@ } }, "node_modules/@cloudflare/workerd-linux-64": { - "version": "1.20261012.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20261012.1.tgz", - "integrity": "sha512-uMe7ONHnLxStD1Ye0YFHBO36uj4cStv5P53rW+5GVWXFT8BRjqVixlH0IqCAHe1Pmcwyd1+7fVoGde89MZSCJg==", + "version": "1.20261023.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20261023.1.tgz", + "integrity": "sha512-VHQ6jKFSihjR2+x1buby8XTdBLLJztVnYiB7OLOTYO8aDXB+c1Rd2410PdgFuFYc4AgFEk6PCJLyCLL5L1KBnw==", "cpu": [ "x64" ], @@ -698,9 +675,9 @@ } }, "node_modules/@cloudflare/workerd-linux-arm64": { - "version": "1.20261012.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20261012.1.tgz", - "integrity": "sha512-I0VMdcXk7ygG5vYDce0ePTBFdnBXC0i9WiBY5mWiv0iTinbfea6/rHqSQ7B1UMpjgrYVs9E0m4bZ9ufsk5GCFA==", + "version": "1.20261023.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20261023.1.tgz", + "integrity": "sha512-ilpRHzfDN8YFRM5CLRKuFw40x/LJBS89yVdhdQmxehlVDuBqjKpJB++zPFbTgKRRFaNupgnXc8RHktXAIhgCmQ==", "cpu": [ "arm64" ], @@ -715,9 +692,9 @@ } }, "node_modules/@cloudflare/workerd-windows-64": { - "version": "1.20261012.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20261012.1.tgz", - "integrity": "sha512-c5hQvNx574UEoknaCFvMa/NdLrrYiALNbe6sx1okW8uOiRYWxrFuA8WpXFnBJ5XJypm0TEAHYaUTGv3rRLpY7Q==", + "version": "1.20261023.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20261023.1.tgz", + "integrity": "sha512-279WLVjNsMycAwr0EZNiNEf0l2SY6NUtsN80NwZzbTHzakmiN+WSx4w38muxPm+PAU0oSPPAPXKVD+H6Hya8xw==", "cpu": [ "x64" ], @@ -1454,6 +1431,9 @@ "arm" ], "dev": true, + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1471,6 +1451,9 @@ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1488,6 +1471,9 @@ "ppc64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1505,6 +1491,9 @@ "riscv64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1522,6 +1511,9 @@ "s390x" ], "dev": true, + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1539,6 +1531,9 @@ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1556,6 +1551,9 @@ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1573,6 +1571,9 @@ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1590,6 +1591,9 @@ "arm" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1613,6 +1617,9 @@ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1636,6 +1643,9 @@ "ppc64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1659,6 +1669,9 @@ "riscv64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1682,6 +1695,9 @@ "s390x" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1705,6 +1721,9 @@ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1728,6 +1747,9 @@ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1751,6 +1773,9 @@ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1863,6 +1888,12 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@isaacs/nonrecursive-glob-match": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.3.tgz", + "integrity": "sha512-SI7kyKHq4XXDRozw+r9GlRZkia1ZkFrpLW98r2IC2Bpr8YkFGE3fvjzmxCG0x78A++69HmDNrDvSMYBZmHGpKA==", + "license": "BlueOak-1.0.0" + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", @@ -1901,12 +1932,6 @@ "url": "https://github.com/sponsors/oxc-project" } }, - "node_modules/@pondwader/socks5-server": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/@pondwader/socks5-server/-/socks5-server-1.0.10.tgz", - "integrity": "sha512-bQY06wzzR8D2+vVCUoBsr5QS2U6UgPUQRmErNwtsuI6vLcyRKkafjkr3KxbtGFf9aBBIV2mcvlsKD1UYaIV+sg==", - "license": "MIT" - }, "node_modules/@poppinss/colors": { "version": "4.1.6", "resolved": "https://registry.npmjs.org/@poppinss/colors/-/colors-4.1.6.tgz", @@ -2441,21 +2466,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/lodash": { - "version": "4.17.24", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.24.tgz", - "integrity": "sha512-gIW7lQLZbue7lRSWEFql49QJJWThrTFFeIMJdp3eH4tKoxm1OvEPg02rm4wCCSHS0cL3/Fizimb35b7k8atwsQ==", - "license": "MIT" - }, - "node_modules/@types/lodash-es": { - "version": "4.17.12", - "resolved": "https://registry.npmjs.org/@types/lodash-es/-/lodash-es-4.17.12.tgz", - "integrity": "sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ==", - "license": "MIT", - "dependencies": { - "@types/lodash": "*" - } - }, "node_modules/@types/ms": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", @@ -3201,22 +3211,6 @@ "dev": true, "license": "MIT" }, - "node_modules/commander": { - "version": "12.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", - "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, - "license": "MIT" - }, "node_modules/convert-source-map": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", @@ -3366,16 +3360,6 @@ "url": "https://github.com/sponsors/antfu" } }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, "node_modules/es-module-lexer": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.1.tgz", @@ -3537,13 +3521,6 @@ "dev": true, "license": "MIT" }, - "node_modules/fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true, - "license": "ISC" - }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -3559,16 +3536,6 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/gaxios": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.4.tgz", @@ -3616,59 +3583,6 @@ "dev": true, "license": "MIT" }, - "node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", - "dev": true, - "license": "ISC", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/glob/node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true, - "license": "MIT" - }, - "node_modules/glob/node_modules/brace-expansion": { - "version": "1.1.21", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz", - "integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/glob/node_modules/minimatch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", - "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, "node_modules/google-auth-library": { "version": "10.6.2", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.6.2.tgz", @@ -3710,19 +3624,6 @@ "node": ">=18" } }, - "node_modules/hasown": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", - "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", - "dev": true, - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/highlight.js": { "version": "10.7.3", "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz", @@ -3826,18 +3727,6 @@ "node": ">= 4" } }, - "node_modules/inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", - "dev": true, - "license": "ISC", - "dependencies": { - "once": "^1.3.0", - "wrappy": "1" - } - }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -3852,32 +3741,6 @@ "dev": true, "license": "ISC" }, - "node_modules/interpret": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", - "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/is-core-module": { - "version": "2.17.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.17.0.tgz", - "integrity": "sha512-J/vG0zBCbIKOQFfufSwyXdMrsohyJIUNkrnmo6WZGzoM7tr/lsbfW5b2BvisL6zsyMzK9UxV9L6c7AoFbyXHOA==", - "dev": true, - "license": "MIT", - "dependencies": { - "hasown": "^2.0.4" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -4219,12 +4082,6 @@ "url": "https://opencollective.com/parcel" } }, - "node_modules/lodash-es": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz", - "integrity": "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==", - "license": "MIT" - }, "node_modules/long": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", @@ -4288,16 +4145,16 @@ } }, "node_modules/miniflare": { - "version": "5.20261012.1-beta", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-5.20261012.1-beta.tgz", - "integrity": "sha512-8Canhy6g9+vAhqMc0o80YphqYIwpiA3aAp8aX+uDXU7J/qbUeoZzcII9c2V1P4uT/eHD4SB93qet1o1bRY3BuQ==", + "version": "5.20261023.2-beta", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-5.20261023.2-beta.tgz", + "integrity": "sha512-sXpZloQnIUe4raXXdwwqd2ioRHox9a/1B4nB2/p4JWKDwGuVgEK4DugCNgIeQf0iVuSgBItdYpodJe4iwY6SrQ==", "dev": true, "license": "MIT", "dependencies": { "@cspotcode/source-map-support": "0.8.1", "sharp": "0.35.5", - "undici": "7.29.1", - "workerd": "1.20261012.1", + "undici": "7.30.1", + "workerd": "1.20261023.1", "ws": "8.21.0", "youch": "4.1.0-beta.10" }, @@ -4306,9 +4163,9 @@ } }, "node_modules/miniflare/node_modules/undici": { - "version": "7.29.1", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz", - "integrity": "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==", + "version": "7.30.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.30.1.tgz", + "integrity": "sha512-03KrMbjIFDWQR5RNPrSw3bd1r12JpyFQl9/yxjUbJtqkjmllqC5lFjZh6ujeuJCKr8EOQvRW872Oy/az+nLYxg==", "dev": true, "license": "MIT", "engines": { @@ -4316,12 +4173,13 @@ } }, "node_modules/minimatch": { - "version": "10.2.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", - "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.3.tgz", + "integrity": "sha512-iny/40PEGkNJQ5/Pv6z3nm6FbRAby32GeZRjCUJk2WnNDbgPSTcdI+DDvLqsgBdD4cD5sMtMheqPPnehCCa8Nw==", "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.8" + "@isaacs/nonrecursive-glob-match": "^1.0.3", + "brace-expansion": "^5.0.12" }, "engines": { "node": "18 || 20 || >=22" @@ -4493,16 +4351,6 @@ "integrity": "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA==", "license": "MIT" }, - "node_modules/path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -4512,13 +4360,6 @@ "node": ">=8" } }, - "node_modules/path-parse": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", - "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true, - "license": "MIT" - }, "node_modules/path-to-regexp": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", @@ -4545,10 +4386,6 @@ "resolved": "packages/coding-agent/examples/extensions/gondolin", "link": true }, - "node_modules/pi-extension-sandbox": { - "resolved": "packages/coding-agent/examples/extensions/sandbox", - "link": true - }, "node_modules/pi-extension-with-deps": { "resolved": "packages/coding-agent/examples/extensions/with-deps", "link": true @@ -4757,40 +4594,6 @@ "node": ">= 6" } }, - "node_modules/rechoir": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", - "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==", - "dev": true, - "dependencies": { - "resolve": "^1.1.6" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/resolve": { - "version": "1.22.12", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", - "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "is-core-module": "^2.16.1", - "path-parse": "^1.0.7", - "supports-preserve-symlinks-flag": "^1.0.0" - }, - "bin": { - "resolve": "bin/resolve" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/retry": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", @@ -4943,53 +4746,6 @@ "node": ">=8" } }, - "node_modules/shell-quote": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.12.0.tgz", - "integrity": "sha512-PcByqNyT/38F2kDNi006HAMRJaULuBzq/FOsw3qdZvX/GA9W/jamDaRskgHjubHiftXK5sIFxLNkvrXUwcof6Q==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/shelljs": { - "version": "0.8.5", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", - "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "glob": "^7.0.0", - "interpret": "^1.0.0", - "rechoir": "^0.6.2" - }, - "bin": { - "shjs": "bin/shjs" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/shx": { - "version": "0.3.4", - "resolved": "https://registry.npmjs.org/shx/-/shx-0.3.4.tgz", - "integrity": "sha512-N6A9MLVqjxZYcVn8hLmtneQWIJtp8IKzMP4eMnx+nqkvXoqinUPCbUFLp2UcWTEIUONhlk0ewxr/jaVGlc+J+g==", - "dev": true, - "license": "MIT", - "dependencies": { - "minimist": "^1.2.3", - "shelljs": "^0.8.5" - }, - "bin": { - "shx": "lib/cli.js" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/siginfo": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", @@ -5134,19 +4890,6 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, - "node_modules/supports-preserve-symlinks-flag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", - "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/tar-fs": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", @@ -5563,9 +5306,9 @@ } }, "node_modules/workerd": { - "version": "1.20261012.1", - "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20261012.1.tgz", - "integrity": "sha512-tEHoMPsMga0seVjxfsR8Pd3f8IRqYwkq++dTUKI5Gdgj6zuSDluYwisydq+dn90SsIACwBquJUQ1nBpcOqaWTA==", + "version": "1.20261023.1", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20261023.1.tgz", + "integrity": "sha512-NeofJFyaYNpqKpFRJW3ApuonaerPd6/JWlfTtLDYf2A6ygqrDEFnYgEXmc0hhOxqnJsSM+kXJN6VebPojzRrsg==", "dev": true, "hasInstallScript": true, "license": "Apache-2.0", @@ -5576,17 +5319,17 @@ "node": ">=16" }, "optionalDependencies": { - "@cloudflare/workerd-darwin-64": "1.20261012.1", - "@cloudflare/workerd-darwin-arm64": "1.20261012.1", - "@cloudflare/workerd-linux-64": "1.20261012.1", - "@cloudflare/workerd-linux-arm64": "1.20261012.1", - "@cloudflare/workerd-windows-64": "1.20261012.1" + "@cloudflare/workerd-darwin-64": "1.20261023.1", + "@cloudflare/workerd-darwin-arm64": "1.20261023.1", + "@cloudflare/workerd-linux-64": "1.20261023.1", + "@cloudflare/workerd-linux-arm64": "1.20261023.1", + "@cloudflare/workerd-windows-64": "1.20261023.1" } }, "node_modules/wrangler": { - "version": "4.151.1", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.151.1.tgz", - "integrity": "sha512-OAsa9UAHZshM6mq2PXcVoyxXgWThAanZMBc/IirtqEWd/BauRtmUKzAniOu/pU0pd3pxKuMZ2vsbF4eo6RiNDA==", + "version": "4.155.0", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.155.0.tgz", + "integrity": "sha512-gBKtAg2e0abQDvXPRQu611YMTd/cfAaJTOLAs/89tTxFxEeUyC260MRJRK+m3gLmQjDYWFwJv+ErwuJ/GuALpA==", "dev": true, "license": "MIT OR Apache-2.0", "dependencies": { @@ -5594,10 +5337,10 @@ "@cloudflare/unenv-preset": "2.16.2", "blake3-wasm": "2.1.5", "esbuild": "0.28.2", - "miniflare": "5.20261012.1-beta", + "miniflare": "5.20261023.2-beta", "path-to-regexp": "6.3.0", "unenv": "2.0.0-rc.24", - "workerd": "1.20261012.1" + "workerd": "1.20261023.1" }, "bin": { "cf-wrangler": "bin/cf-wrangler.js", @@ -5611,7 +5354,7 @@ "fsevents": "2.3.3" }, "peerDependencies": { - "@cloudflare/workers-types": "^5.20261006.1" + "@cloudflare/workers-types": "^5.20261023.1" }, "peerDependenciesMeta": { "@cloudflare/workers-types": { @@ -5688,9 +5431,10 @@ } }, "node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "version": "4.6.7", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.7.tgz", + "integrity": "sha512-g8W2D2kU6a8untc053fX1uVBKzM7yqAxW/3zwk83+6RyF2+SKOuV+c0wBDhUwzV1PaGapmZH+rmcUzPf+3CGwg==", + "devOptional": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" @@ -5817,7 +5561,6 @@ }, "devDependencies": { "@types/ws": "8.18.2", - "shx": "0.3.4", "vitest": "4.1.12", "ws": "8.21.0" }, @@ -5841,7 +5584,6 @@ "quickjs-wasi": "3.6.2" }, "devDependencies": { - "shx": "0.3.4", "vitest": "4.1.12" }, "engines": { @@ -5868,7 +5610,7 @@ "hosted-git-info": "9.0.3", "ignore": "7.0.8", "jiti": "2.7.0", - "minimatch": "10.2.6", + "minimatch": "11.0.3", "proper-lockfile": "4.1.2", "quickjs-wasi": "3.6.2", "semver": "7.8.5", @@ -5886,7 +5628,6 @@ "@types/proper-lockfile": "4.1.4", "@types/semver": "7.7.1", "marked": "18.0.11", - "shx": "0.3.4", "vitest": "4.1.12" }, "engines": { @@ -5911,13 +5652,6 @@ "@earendil-works/gondolin": "0.14.0" } }, - "packages/coding-agent/examples/extensions/sandbox": { - "name": "pi-extension-sandbox", - "version": "1.4.0", - "dependencies": { - "@anthropic-ai/sandbox-runtime": "0.0.26" - } - }, "packages/coding-agent/examples/extensions/with-deps": { "name": "pi-extension-with-deps", "version": "1.4.0", @@ -5986,9 +5720,8 @@ "typebox": "1.3.27" }, "devDependencies": { - "@cloudflare/vitest-plugin": "1.4.4", + "@cloudflare/vitest-plugin": "1.4.10", "fake-indexeddb": "6.2.6", - "shx": "0.3.4", "vitest": "4.1.12" }, "engines": { @@ -6004,7 +5737,6 @@ "@earendil-works/pi-durable": "1.4.0" }, "devDependencies": { - "shx": "0.3.4", "vitest": "4.1.12" }, "engines": { @@ -6019,7 +5751,6 @@ "@earendil-works/pi-coding-agent": "1.4.0", "@types/node": "22.19.19", "@vitest-evals/core": "0.15.0", - "shx": "0.3.4", "vitest": "4.1.12", "vitest-evals": "0.15.0" } @@ -6033,7 +5764,6 @@ }, "devDependencies": { "@types/cross-spawn": "6.0.6", - "shx": "0.3.4", "vitest": "4.1.12" }, "engines": { diff --git a/package.json b/package.json index 04462a1d5..b158f2f00 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,6 @@ "packages/coding-agent/examples/extensions/with-deps", "packages/coding-agent/examples/extensions/custom-provider-anthropic", "packages/coding-agent/examples/extensions/custom-provider-gitlab-duo", - "packages/coding-agent/examples/extensions/sandbox", "packages/coding-agent/examples/extensions/gondolin" ], "scripts": { @@ -57,14 +56,12 @@ "prepare": "husky" }, "devDependencies": { - "@anthropic-ai/sandbox-runtime": "0.0.26", "@biomejs/biome": "2.3.5", "@biomejs/wasm-nodejs": "2.3.5", "@types/node": "22.19.19", "esbuild": "0.28.2", "husky": "9.1.7", "pnpm": "11.15.1", - "shx": "0.3.4", "typescript": "7.0.2", "yaml": "2.9.0" }, diff --git a/packages/agent/package.json b/packages/agent/package.json index d1231d95d..a8f7de36f 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -17,7 +17,7 @@ "README.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "test": "vitest --run", "prepublishOnly": "npm run build" diff --git a/packages/ai/package.json b/packages/ai/package.json index 6e04980f0..4e9d5d5a2 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -56,13 +56,13 @@ "README.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "generate-models": "node catalog/generate.ts --strict", "hydrate-model-data": "node catalog/generate.ts --strict --data-only", "generate-model-catalog": "node catalog/generate.ts --strict --json-only --json-output ../../.artifacts/model-catalog", "check:model-data": "node scripts/check-model-data.ts", "build": "npm run generate-models && npm run build:offline", - "build:offline": "npm run check:model-data && tsc -p tsconfig.build.json && shx rm -rf dist/providers/data && shx cp -r src/providers/data dist/providers/data", + "build:offline": "npm run check:model-data && tsc -p tsconfig.build.json && node ../../scripts/package-files.mjs clean dist/providers/data && node ../../scripts/package-files.mjs copy src/providers/data dist/providers", "test": "vitest --run", "prepublishOnly": "npm run clean && npm run build" }, diff --git a/packages/chord/package.json b/packages/chord/package.json index f00d184dd..f2be5a564 100644 --- a/packages/chord/package.json +++ b/packages/chord/package.json @@ -67,7 +67,7 @@ "src/delta/README.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "test": "vitest --run", "prepublishOnly": "npm run clean && npm run build" @@ -103,7 +103,6 @@ }, "devDependencies": { "@types/ws": "8.18.2", - "shx": "0.3.4", "vitest": "4.1.12", "ws": "8.21.0" } diff --git a/packages/codemode/package.json b/packages/codemode/package.json index b1f0fea9c..9fed9d017 100644 --- a/packages/codemode/package.json +++ b/packages/codemode/package.json @@ -42,7 +42,7 @@ "README.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "test": "vitest --run", "prepublishOnly": "npm run clean && npm run build" @@ -70,7 +70,6 @@ "quickjs-wasi": "3.6.2" }, "devDependencies": { - "shx": "0.3.4", "vitest": "4.1.12" } } diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 483e87088..8a916e873 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -19,6 +19,7 @@ ### Fixed +- Updated glob matching to prevent denial of service from amplified glob patterns. - Fixed a saved "Do not trust" decision being ignored for projects without trust-requiring resources, for example a folder with only `AGENTS.md`. pi auto-trusted such projects, so `ctx.isProjectTrusted()` returned true and package commands treated them as trusted. A saved decision in `trust.json` now takes precedence; auto-trust applies only when no decision is saved ([#10947](https://github.com/earendil-works/pi/issues/10947)) - Fixed `Shift+Enter` queueing a follow-up instead of inserting a newline in Apple Terminal setups that send `ESC` + Return. pi now checks the local Shift key state for that sequence, as it already did for plain Return; `Alt+Enter` without Shift still queues a follow-up ([#11559](https://github.com/earendil-works/pi/issues/11559)) - Fixed the working status (OSC 7501) disappearing in Rex and iTerm2 during runs in regular TUI mode. Rendered messages wrote OSC 133 prompt markers to the terminal, which the terminal treats as a new shell prompt; regular mode no longer writes them ([#11519](https://github.com/earendil-works/pi/issues/11519)) @@ -48,6 +49,10 @@ - Fixed `tool_search` and codemode `searchTools()` ranking a tool below tools that only mention its name in their descriptions, for example `peekaboo app` ranking `mcp__peekaboo__app` 14th. A tool matching more of the query's words now ranks above one matching fewer, a query word in a tool's name scores more than any number of mentions in its description, a tool whose whole name is in the query gets a large bonus, and long server instructions no longer lower the scores of that server's tools ([#11331](https://github.com/earendil-works/pi/issues/11331)) - Fixed `tool_search` and codemode `searchTools()` finding nothing for Chinese, Japanese, Thai, and other non-ASCII queries, because the tokenizer kept only ASCII letters and digits. Words in any script now match, text without spaces between words such as `搜索记录` is split into words, and canonically equivalent text such as `café` and `cafe\u0301` matches ([#10816](https://github.com/earendil-works/pi/issues/10816), [#11037](https://github.com/earendil-works/pi/pull/11037) by [@haoqixu](https://github.com/haoqixu)) +### Removed + +- Removed the native OS `sandbox/` extension example and its dependency. Use the [Gondolin example](examples/extensions/gondolin/) or the [containerization guide](docs/containerization.md). + ## [1.4.0] - 2026-10-21 ### New Features diff --git a/packages/coding-agent/docs/extensions.md b/packages/coding-agent/docs/extensions.md index 3606e80f9..6fbec2b65 100644 --- a/packages/coding-agent/docs/extensions.md +++ b/packages/coding-agent/docs/extensions.md @@ -3310,7 +3310,6 @@ All examples in [examples/extensions/](../examples/extensions/). | **Remote & Sandbox** ||| | `ssh.ts` | SSH remote execution | `registerFlag`, `on("user_bash")`, `on("before_agent_start")`, tool operations | | `interactive-shell.ts` | Persistent shell session | `on("user_bash")` | -| `sandbox/` | Sandboxed tool execution | Tool operations | | `gondolin/` | Route built-in tools and `!` commands into a Gondolin micro-VM | Tool operations, built-in tool overrides, `on("user_bash")` | | `subagent/` | Spawn sub-agents | `registerTool`, `exec` | | **Games** ||| diff --git a/packages/coding-agent/examples/extensions/README.md b/packages/coding-agent/examples/extensions/README.md index 7667ffe03..de24f7af4 100644 --- a/packages/coding-agent/examples/extensions/README.md +++ b/packages/coding-agent/examples/extensions/README.md @@ -23,9 +23,10 @@ cp permission-gate.ts ~/.pi/agent/extensions/ | `protected-paths.ts` | Blocks writes to protected paths (.env, .git/, node_modules/) | | `confirm-destructive.ts` | Confirms before destructive session actions (clear, switch, fork) | | `dirty-repo-guard.ts` | Prevents session changes with uncommitted git changes | -| `sandbox/` | OS-level sandboxing using `@anthropic-ai/sandbox-runtime` with per-project config | | `gondolin/` | Route built-in tools and `!` commands into a Gondolin micro-VM | +For isolation setup and limitations, see [Run Pi in an isolated environment](../../docs/containerization.md). + ### Custom Tools | Extension | Description | diff --git a/packages/coding-agent/examples/extensions/sandbox/.gitignore b/packages/coding-agent/examples/extensions/sandbox/.gitignore deleted file mode 100644 index 3c3629e64..000000000 --- a/packages/coding-agent/examples/extensions/sandbox/.gitignore +++ /dev/null @@ -1 +0,0 @@ -node_modules diff --git a/packages/coding-agent/examples/extensions/sandbox/index.ts b/packages/coding-agent/examples/extensions/sandbox/index.ts deleted file mode 100644 index b54d75d1c..000000000 --- a/packages/coding-agent/examples/extensions/sandbox/index.ts +++ /dev/null @@ -1,321 +0,0 @@ -/** - * Sandbox Extension - OS-level sandboxing for bash commands - * - * Uses @anthropic-ai/sandbox-runtime to enforce filesystem and network - * restrictions on bash commands at the OS level (sandbox-exec on macOS, - * bubblewrap on Linux). - * - * Note: this example intentionally overrides the built-in `bash` tool to show - * how built-in tools can be replaced. Alternatively, you could sandbox `bash` - * via `tool_call` input mutation without replacing the tool. - * - * Config files (merged, project takes precedence): - * - ~/.pi/agent/extensions/sandbox.json (global) - * - /.pi/sandbox.json (project-local) - * - * Example .pi/sandbox.json: - * ```json - * { - * "enabled": true, - * "network": { - * "allowedDomains": ["github.com", "*.github.com"], - * "deniedDomains": [] - * }, - * "filesystem": { - * "denyRead": ["~/.ssh", "~/.aws"], - * "allowWrite": [".", "/tmp"], - * "denyWrite": [".env"] - * } - * } - * ``` - * - * Usage: - * - `pi -e ./sandbox` - sandbox enabled with default/config settings - * - `pi -e ./sandbox --no-sandbox` - disable sandboxing - * - `/sandbox` - show current sandbox configuration - * - * Setup: - * 1. Copy sandbox/ directory to ~/.pi/agent/extensions/ - * 2. Run `npm install` in ~/.pi/agent/extensions/sandbox/ - * - * Linux also requires: bubblewrap, socat, ripgrep - */ - -import { spawn } from "node:child_process"; -import { existsSync, readFileSync } from "node:fs"; -import { join } from "node:path"; -import { SandboxManager, type SandboxRuntimeConfig } from "@anthropic-ai/sandbox-runtime"; -import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; -import { type BashOperations, CONFIG_DIR_NAME, createBashTool, getAgentDir } from "@earendil-works/pi-coding-agent"; - -interface SandboxConfig extends SandboxRuntimeConfig { - enabled?: boolean; -} - -const DEFAULT_CONFIG: SandboxConfig = { - enabled: true, - network: { - allowedDomains: [ - "npmjs.org", - "*.npmjs.org", - "registry.npmjs.org", - "registry.yarnpkg.com", - "pypi.org", - "*.pypi.org", - "github.com", - "*.github.com", - "api.github.com", - "raw.githubusercontent.com", - ], - deniedDomains: [], - }, - filesystem: { - denyRead: ["~/.ssh", "~/.aws", "~/.gnupg"], - allowWrite: [".", "/tmp"], - denyWrite: [".env", ".env.*", "*.pem", "*.key"], - }, -}; - -function loadConfig(cwd: string): SandboxConfig { - const projectConfigPath = join(cwd, CONFIG_DIR_NAME, "sandbox.json"); - const globalConfigPath = join(getAgentDir(), "extensions", "sandbox.json"); - - let globalConfig: Partial = {}; - let projectConfig: Partial = {}; - - if (existsSync(globalConfigPath)) { - try { - globalConfig = JSON.parse(readFileSync(globalConfigPath, "utf-8")); - } catch (e) { - console.error(`Warning: Could not parse ${globalConfigPath}: ${e}`); - } - } - - if (existsSync(projectConfigPath)) { - try { - projectConfig = JSON.parse(readFileSync(projectConfigPath, "utf-8")); - } catch (e) { - console.error(`Warning: Could not parse ${projectConfigPath}: ${e}`); - } - } - - return deepMerge(deepMerge(DEFAULT_CONFIG, globalConfig), projectConfig); -} - -function deepMerge(base: SandboxConfig, overrides: Partial): SandboxConfig { - const result: SandboxConfig = { ...base }; - - if (overrides.enabled !== undefined) result.enabled = overrides.enabled; - if (overrides.network) { - result.network = { ...base.network, ...overrides.network }; - } - if (overrides.filesystem) { - result.filesystem = { ...base.filesystem, ...overrides.filesystem }; - } - - const extOverrides = overrides as { - ignoreViolations?: Record; - enableWeakerNestedSandbox?: boolean; - }; - const extResult = result as { ignoreViolations?: Record; enableWeakerNestedSandbox?: boolean }; - - if (extOverrides.ignoreViolations) { - extResult.ignoreViolations = extOverrides.ignoreViolations; - } - if (extOverrides.enableWeakerNestedSandbox !== undefined) { - extResult.enableWeakerNestedSandbox = extOverrides.enableWeakerNestedSandbox; - } - - return result; -} - -function createSandboxedBashOps(): BashOperations { - return { - async exec(command, cwd, { onData, signal, timeout }) { - if (!existsSync(cwd)) { - throw new Error(`Working directory does not exist: ${cwd}`); - } - - const wrappedCommand = await SandboxManager.wrapWithSandbox(command); - - return new Promise((resolve, reject) => { - const child = spawn("bash", ["-c", wrappedCommand], { - cwd, - detached: true, - stdio: ["ignore", "pipe", "pipe"], - }); - - let timedOut = false; - let timeoutHandle: NodeJS.Timeout | undefined; - - if (timeout !== undefined && timeout > 0) { - timeoutHandle = setTimeout(() => { - timedOut = true; - if (child.pid) { - try { - process.kill(-child.pid, "SIGKILL"); - } catch { - child.kill("SIGKILL"); - } - } - }, timeout * 1000); - } - - child.stdout?.on("data", onData); - child.stderr?.on("data", onData); - - child.on("error", (err) => { - if (timeoutHandle) clearTimeout(timeoutHandle); - reject(err); - }); - - const onAbort = () => { - if (child.pid) { - try { - process.kill(-child.pid, "SIGKILL"); - } catch { - child.kill("SIGKILL"); - } - } - }; - - signal?.addEventListener("abort", onAbort, { once: true }); - - child.on("close", (code) => { - if (timeoutHandle) clearTimeout(timeoutHandle); - signal?.removeEventListener("abort", onAbort); - - if (signal?.aborted) { - reject(new Error("aborted")); - } else if (timedOut) { - reject(new Error(`timeout:${timeout}`)); - } else { - resolve({ exitCode: code }); - } - }); - }); - }, - }; -} - -export default function (pi: ExtensionAPI) { - pi.registerFlag("no-sandbox", { - description: "Disable OS-level sandboxing for bash commands", - type: "boolean", - default: false, - }); - - const localCwd = process.cwd(); - const localBash = createBashTool(localCwd); - - let sandboxEnabled = false; - let sandboxInitialized = false; - - pi.registerTool({ - ...localBash, - label: "bash (sandboxed)", - async execute(id, params, signal, onUpdate, _ctx) { - if (!sandboxEnabled || !sandboxInitialized) { - return localBash.execute(id, params, signal, onUpdate); - } - - const sandboxedBash = createBashTool(localCwd, { - operations: createSandboxedBashOps(), - }); - return sandboxedBash.execute(id, params, signal, onUpdate); - }, - }); - - pi.on("user_bash", () => { - if (!sandboxEnabled || !sandboxInitialized) return; - return { operations: createSandboxedBashOps() }; - }); - - pi.on("session_start", async (_event, ctx) => { - const noSandbox = pi.getFlag("no-sandbox") as boolean; - - if (noSandbox) { - sandboxEnabled = false; - ctx.ui.notify("Sandbox disabled via --no-sandbox", "warning"); - return; - } - - const config = loadConfig(ctx.cwd); - - if (!config.enabled) { - sandboxEnabled = false; - ctx.ui.notify("Sandbox disabled via config", "info"); - return; - } - - const platform = process.platform; - if (platform !== "darwin" && platform !== "linux") { - sandboxEnabled = false; - ctx.ui.notify(`Sandbox not supported on ${platform}`, "warning"); - return; - } - - try { - const configExt = config as unknown as { - ignoreViolations?: Record; - enableWeakerNestedSandbox?: boolean; - }; - - await SandboxManager.initialize({ - network: config.network, - filesystem: config.filesystem, - ignoreViolations: configExt.ignoreViolations, - enableWeakerNestedSandbox: configExt.enableWeakerNestedSandbox, - }); - - sandboxEnabled = true; - sandboxInitialized = true; - - const networkCount = config.network?.allowedDomains?.length ?? 0; - const writeCount = config.filesystem?.allowWrite?.length ?? 0; - ctx.ui.setStatus( - "sandbox", - ctx.ui.theme.fg("accent", `🔒 Sandbox: ${networkCount} domains, ${writeCount} write paths`), - ); - ctx.ui.notify("Sandbox initialized", "info"); - } catch (err) { - sandboxEnabled = false; - ctx.ui.notify(`Sandbox initialization failed: ${err instanceof Error ? err.message : err}`, "error"); - } - }); - - pi.on("session_shutdown", async () => { - if (sandboxInitialized) { - try { - await SandboxManager.reset(); - } catch { - // Ignore cleanup errors - } - } - }); - - pi.registerCommand("sandbox", { - description: "Show sandbox configuration", - handler: async (_args, ctx) => { - if (!sandboxEnabled) { - ctx.ui.notify("Sandbox is disabled", "info"); - return; - } - - const config = loadConfig(ctx.cwd); - const lines = [ - "Sandbox Configuration:", - "", - "Network:", - ` Allowed: ${config.network?.allowedDomains?.join(", ") || "(none)"}`, - ` Denied: ${config.network?.deniedDomains?.join(", ") || "(none)"}`, - "", - "Filesystem:", - ` Deny Read: ${config.filesystem?.denyRead?.join(", ") || "(none)"}`, - ` Allow Write: ${config.filesystem?.allowWrite?.join(", ") || "(none)"}`, - ` Deny Write: ${config.filesystem?.denyWrite?.join(", ") || "(none)"}`, - ]; - ctx.ui.notify(lines.join("\n"), "info"); - }, - }); -} diff --git a/packages/coding-agent/examples/extensions/sandbox/package-lock.json b/packages/coding-agent/examples/extensions/sandbox/package-lock.json deleted file mode 100644 index 90ef1c2f5..000000000 --- a/packages/coding-agent/examples/extensions/sandbox/package-lock.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "name": "pi-extension-sandbox", - "version": "1.4.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "pi-extension-sandbox", - "version": "1.4.0", - "dependencies": { - "@anthropic-ai/sandbox-runtime": "^0.0.26" - } - }, - "node_modules/@anthropic-ai/sandbox-runtime": { - "version": "0.0.26", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sandbox-runtime/-/sandbox-runtime-0.0.26.tgz", - "integrity": "sha512-DYV5LSsVMnzq0lbfaYMSpxZPUMAx4+hy343dRss+pVCLIfF62qOhxpYfZ5TmOk1GTDQm5f9wPprMNSStmnsV4w==", - "license": "Apache-2.0", - "dependencies": { - "@pondwader/socks5-server": "^1.0.10", - "@types/lodash-es": "^4.17.12", - "commander": "^12.1.0", - "lodash-es": "^4.17.21", - "shell-quote": "^1.8.3", - "zod": "^3.24.1" - }, - "bin": { - "srt": "dist/cli.js" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@pondwader/socks5-server": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/@pondwader/socks5-server/-/socks5-server-1.0.10.tgz", - "integrity": "sha512-bQY06wzzR8D2+vVCUoBsr5QS2U6UgPUQRmErNwtsuI6vLcyRKkafjkr3KxbtGFf9aBBIV2mcvlsKD1UYaIV+sg==", - "license": "MIT" - }, - "node_modules/@types/lodash": { - "version": "4.17.23", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.23.tgz", - "integrity": "sha512-RDvF6wTulMPjrNdCoYRC8gNR880JNGT8uB+REUpC2Ns4pRqQJhGz90wh7rgdXDPpCczF3VGktDuFGVnz8zP7HA==", - "license": "MIT" - }, - "node_modules/@types/lodash-es": { - "version": "4.17.12", - "resolved": "https://registry.npmjs.org/@types/lodash-es/-/lodash-es-4.17.12.tgz", - "integrity": "sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ==", - "license": "MIT", - "dependencies": { - "@types/lodash": "*" - } - }, - "node_modules/commander": { - "version": "12.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", - "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/lodash-es": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz", - "integrity": "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==", - "license": "MIT" - }, - "node_modules/shell-quote": { - "version": "1.8.3", - "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.3.tgz", - "integrity": "sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - } - } -} diff --git a/packages/coding-agent/examples/extensions/sandbox/package.json b/packages/coding-agent/examples/extensions/sandbox/package.json deleted file mode 100644 index 8059d577d..000000000 --- a/packages/coding-agent/examples/extensions/sandbox/package.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "pi-extension-sandbox", - "private": true, - "version": "1.4.0", - "type": "module", - "scripts": { - "clean": "echo 'nothing to clean'", - "build": "echo 'nothing to build'", - "check": "echo 'nothing to check'" - }, - "pi": { - "extensions": [ - "./index.ts" - ] - }, - "dependencies": { - "@anthropic-ai/sandbox-runtime": "0.0.26" - } -} diff --git a/packages/coding-agent/install-lock/package-lock.json b/packages/coding-agent/install-lock/package-lock.json index 25e273cf6..ed922e44f 100644 --- a/packages/coding-agent/install-lock/package-lock.json +++ b/packages/coding-agent/install-lock/package-lock.json @@ -511,7 +511,7 @@ "hosted-git-info": "9.0.3", "ignore": "7.0.8", "jiti": "2.7.0", - "minimatch": "10.2.6", + "minimatch": "11.0.3", "proper-lockfile": "4.1.2", "quickjs-wasi": "3.6.2", "semver": "7.8.5", @@ -582,6 +582,12 @@ }, "hasInstallScript": true }, + "node_modules/@isaacs/nonrecursive-glob-match": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.3.tgz", + "integrity": "sha512-SI7kyKHq4XXDRozw+r9GlRZkia1ZkFrpLW98r2IC2Bpr8YkFGE3fvjzmxCG0x78A++69HmDNrDvSMYBZmHGpKA==", + "license": "BlueOak-1.0.0" + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -1174,12 +1180,13 @@ } }, "node_modules/minimatch": { - "version": "10.2.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", - "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.3.tgz", + "integrity": "sha512-iny/40PEGkNJQ5/Pv6z3nm6FbRAby32GeZRjCUJk2WnNDbgPSTcdI+DDvLqsgBdD4cD5sMtMheqPPnehCCa8Nw==", "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.8" + "@isaacs/nonrecursive-glob-match": "^1.0.3", + "brace-expansion": "^5.0.12" }, "engines": { "node": "18 || 20 || >=22" diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index 0e452d9b5..f8e6d41cc 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -29,12 +29,12 @@ "CHANGELOG.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "npm run build:unbundled && node ../../scripts/build-coding-agent-bundle.mjs", - "build:unbundled": "tsc -p tsconfig.build.json && shx chmod +x dist/cli.js dist/rpc-entry.js && npm run copy-assets", + "build:unbundled": "tsc -p tsconfig.build.json && node ../../scripts/package-files.mjs executable dist/cli.js dist/rpc-entry.js && npm run copy-assets", "build:binary": "npm --prefix ../tui run build && npm --prefix ../telemetry run build && npm --prefix ../ai run build && npm --prefix ../agent run build && npm run build && bun build --compile --no-compile-autoload-bunfig --no-compile-autoload-dotenv ./dist/bun/cli.js ./src/utils/image-resize-worker.ts ./src/extensions/codemode/worker.ts --outfile dist/pi && npm run copy-binary-assets", - "copy-assets": "shx mkdir -p dist/modes/interactive/theme && shx cp src/modes/interactive/theme/*.json dist/modes/interactive/theme/ && shx mkdir -p dist/modes/interactive/assets && shx cp src/modes/interactive/assets/*.png dist/modes/interactive/assets/ && shx mkdir -p dist/core/export-html/vendor && shx cp src/core/export-html/template.html src/core/export-html/template.css src/core/export-html/template.js dist/core/export-html/ && shx cp src/core/export-html/vendor/*.js dist/core/export-html/vendor/", - "copy-binary-assets": "shx cp package.json dist/ && shx cp README.md dist/ && shx cp CHANGELOG.md dist/ && shx mkdir -p dist/theme && shx cp src/modes/interactive/theme/*.json dist/theme/ && shx mkdir -p dist/assets && shx cp src/modes/interactive/assets/*.png dist/assets/ && shx mkdir -p dist/export-html/vendor && shx cp src/core/export-html/template.html dist/export-html/ && shx cp src/core/export-html/vendor/*.js dist/export-html/vendor/ && shx cp -r docs dist/ && shx cp -r examples dist/ && shx cp ../../node_modules/@silvia-odwyer/photon-node/photon_rs_bg.wasm dist/", + "copy-assets": "node ../../scripts/package-files.mjs copy \"src/modes/interactive/theme/*.json\" dist/modes/interactive/theme && node ../../scripts/package-files.mjs copy \"src/modes/interactive/assets/*.png\" dist/modes/interactive/assets && node ../../scripts/package-files.mjs copy src/core/export-html/template.html src/core/export-html/template.css src/core/export-html/template.js dist/core/export-html && node ../../scripts/package-files.mjs copy \"src/core/export-html/vendor/*.js\" dist/core/export-html/vendor", + "copy-binary-assets": "node ../../scripts/package-files.mjs copy package.json README.md CHANGELOG.md docs examples ../../node_modules/@silvia-odwyer/photon-node/photon_rs_bg.wasm dist && node ../../scripts/package-files.mjs copy \"src/modes/interactive/theme/*.json\" dist/theme && node ../../scripts/package-files.mjs copy \"src/modes/interactive/assets/*.png\" dist/assets && node ../../scripts/package-files.mjs copy src/core/export-html/template.html dist/export-html && node ../../scripts/package-files.mjs copy \"src/core/export-html/vendor/*.js\" dist/export-html/vendor", "test": "vitest --run", "generate:schemas": "node --import ./src/experimental/source-resolver.ts scripts/generate-schemas.ts", "generate:references": "node --import ./src/experimental/source-resolver.ts scripts/generate-references.ts", @@ -56,7 +56,7 @@ "hosted-git-info": "9.0.3", "ignore": "7.0.8", "jiti": "2.7.0", - "minimatch": "10.2.6", + "minimatch": "11.0.3", "proper-lockfile": "4.1.2", "quickjs-wasi": "3.6.2", "semver": "7.8.5", @@ -74,7 +74,6 @@ "@types/proper-lockfile": "4.1.4", "@types/semver": "7.7.1", "marked": "18.0.11", - "shx": "0.3.4", "vitest": "4.1.12" }, "keywords": [ diff --git a/packages/durable/package.json b/packages/durable/package.json index 933fe6db7..57ced6bbb 100644 --- a/packages/durable/package.json +++ b/packages/durable/package.json @@ -123,7 +123,7 @@ "CHANGELOG.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "test": "vitest --run && PI_DURABLE_DOCUMENT_RETENTION_MS=0 vitest --run && npm run test:store && npm run test:workers", "test:store": "vitest --run --config vitest.store.config.ts", @@ -167,9 +167,8 @@ "typebox": "1.3.27" }, "devDependencies": { - "@cloudflare/vitest-plugin": "1.4.4", + "@cloudflare/vitest-plugin": "1.4.10", "fake-indexeddb": "6.2.6", - "shx": "0.3.4", "vitest": "4.1.12" } } diff --git a/packages/env/package.json b/packages/env/package.json index ab09e6546..2bb941ce0 100644 --- a/packages/env/package.json +++ b/packages/env/package.json @@ -22,7 +22,7 @@ "CHANGELOG.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "build:daemon": "cargo build --manifest-path daemon/Cargo.toml --locked", "test": "vitest --run", @@ -49,7 +49,6 @@ "@earendil-works/pi-durable": "1.4.0" }, "devDependencies": { - "shx": "0.3.4", "vitest": "4.1.12" } } diff --git a/packages/evals/package.json b/packages/evals/package.json index e7cc7bde7..fa4400219 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -4,7 +4,7 @@ "private": true, "type": "module", "scripts": { - "clean": "shx rm -rf .eval", + "clean": "node ../../scripts/package-files.mjs clean .eval", "eval": "npm run eval:host && npm run eval:docs --", "eval:host": "vitest run --config vitest.evals.config.ts --project host", "eval:audit": "vitest run --config vitest.evals.config.ts --project audit", @@ -16,7 +16,6 @@ "@earendil-works/pi-coding-agent": "1.4.0", "@types/node": "22.19.19", "@vitest-evals/core": "0.15.0", - "shx": "0.3.4", "vitest": "4.1.12", "vitest-evals": "0.15.0" } diff --git a/packages/mcp/package.json b/packages/mcp/package.json index b016e8067..a884d2b0f 100644 --- a/packages/mcp/package.json +++ b/packages/mcp/package.json @@ -30,7 +30,7 @@ "LICENSES" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "test": "vitest --run", "prepublishOnly": "npm run clean && npm run build" @@ -55,7 +55,6 @@ }, "devDependencies": { "@types/cross-spawn": "6.0.6", - "shx": "0.3.4", "vitest": "4.1.12" } } diff --git a/packages/telemetry/package.json b/packages/telemetry/package.json index c571023b6..eca0a94e7 100644 --- a/packages/telemetry/package.json +++ b/packages/telemetry/package.json @@ -27,7 +27,7 @@ "README.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "test": "vitest --run", "prepublishOnly": "npm run clean && npm run build" diff --git a/packages/tui/package.json b/packages/tui/package.json index 31dc4744a..d5ce578be 100644 --- a/packages/tui/package.json +++ b/packages/tui/package.json @@ -5,7 +5,7 @@ "type": "module", "main": "dist/index.js", "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "build:native:darwin": "native/darwin/build.sh", "build:native:linux": "native/linux/build.sh", diff --git a/scripts/npm-audit.mjs b/scripts/npm-audit.mjs index f636d2347..f2d391c90 100644 --- a/scripts/npm-audit.mjs +++ b/scripts/npm-audit.mjs @@ -10,15 +10,7 @@ const auditLevel = "moderate"; const severities = ["info", "low", "moderate", "high", "critical"]; /** @type {Record} */ -const acceptedAdvisories = { - "GHSA-86w9-cpqp-85rv": { - package: "node-forge", - reason: - "No fixed node-forge release exists. It is only reachable through @earendil-works/gondolin in the private, " + - "unpublished gondolin example extension. Gondolin only verifies leaf certificates against its own locally " + - "generated CA (public exponent 65537), so the low-exponent signature forgery does not apply.", - }, -}; +const acceptedAdvisories = {}; const result = spawnSync("npm", ["audit", "--omit=dev", "--json"], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); if (result.error) { diff --git a/scripts/package-files.mjs b/scripts/package-files.mjs new file mode 100644 index 000000000..6e64aa95f --- /dev/null +++ b/scripts/package-files.mjs @@ -0,0 +1,30 @@ +// Cross-platform file operations for package build scripts, using only Node built-ins. +import { chmodSync, cpSync, globSync, mkdirSync, rmSync, statSync } from "node:fs"; +import { basename, join } from "node:path"; + +const [command, ...paths] = process.argv.slice(2); +const usage = "Usage: package-files.mjs clean | copy | executable "; + +if (paths.length === 0) throw new Error(usage); + +switch (command) { + case "clean": + for (const path of paths) rmSync(path, { recursive: true, force: true }); + break; + case "copy": { + const destination = paths.pop(); + if (!destination || paths.length === 0) throw new Error(usage); + mkdirSync(destination, { recursive: true }); + for (const pattern of paths) { + const sources = globSync(pattern); + if (sources.length === 0) throw new Error(`No files match ${pattern}`); + for (const source of sources) cpSync(source, join(destination, basename(source)), { recursive: true }); + } + break; + } + case "executable": + for (const path of paths) chmodSync(path, statSync(path).mode | 0o111); + break; + default: + throw new Error(usage); +} diff --git a/scripts/package-files.test.mjs b/scripts/package-files.test.mjs new file mode 100644 index 000000000..9b455a792 --- /dev/null +++ b/scripts/package-files.test.mjs @@ -0,0 +1,82 @@ +// Exercises the file operations used by package build scripts without touching build output. +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; + +const script = fileURLToPath(new URL("./package-files.mjs", import.meta.url)); + +function workspace(t) { + const directory = mkdtempSync(join(tmpdir(), "pi-package-files-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + return directory; +} + +function run(directory, ...args) { + return spawnSync(process.execPath, [script, ...args], { cwd: directory, encoding: "utf8" }); +} + +test("clean removes directories and files and ignores missing paths", (t) => { + const directory = workspace(t); + mkdirSync(join(directory, "dist", "nested"), { recursive: true }); + writeFileSync(join(directory, "dist", "nested", "file.js"), "output"); + writeFileSync(join(directory, "output.txt"), "output"); + assert.equal(run(directory, "clean", "dist", "output.txt", "missing").status, 0); + assert.equal(existsSync(join(directory, "dist")), false); + assert.equal(existsSync(join(directory, "output.txt")), false); +}); + +test("copy expands patterns, creates directories, and overwrites existing assets", (t) => { + const directory = workspace(t); + mkdirSync(join(directory, "source assets")); + writeFileSync(join(directory, "source assets", "dark.json"), "dark"); + writeFileSync(join(directory, "source assets", "light.json"), "light"); + writeFileSync(join(directory, "source assets", "ignored.txt"), "ignored"); + const args = ["copy", "source assets/*.json", "output/assets"]; + assert.equal(run(directory, ...args).status, 0); + assert.equal(readFileSync(join(directory, "output/assets/dark.json"), "utf8"), "dark"); + assert.equal(readFileSync(join(directory, "output/assets/light.json"), "utf8"), "light"); + assert.equal(existsSync(join(directory, "output/assets/ignored.txt")), false); + writeFileSync(join(directory, "source assets", "dark.json"), "updated"); + assert.equal(run(directory, ...args).status, 0); + assert.equal(readFileSync(join(directory, "output/assets/dark.json"), "utf8"), "updated"); +}); + +test("copy accepts multiple files and preserves directory names and nested contents", (t) => { + const directory = workspace(t); + mkdirSync(join(directory, "docs", "nested"), { recursive: true }); + writeFileSync(join(directory, "docs", "nested", "guide.md"), "guide"); + writeFileSync(join(directory, "README.md"), "readme"); + assert.equal(run(directory, "copy", "docs", "README.md", "dist").status, 0); + assert.equal(readFileSync(join(directory, "dist", "docs", "nested", "guide.md"), "utf8"), "guide"); + assert.equal(readFileSync(join(directory, "dist", "README.md"), "utf8"), "readme"); +}); + +test("copy fails when an asset pattern matches nothing", (t) => { + const result = run(workspace(t), "copy", "missing/*.json", "dist"); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /No files match missing/); +}); + +test("executable adds executable bits without removing existing permissions", (t) => { + const directory = workspace(t); + for (const name of ["cli.js", "rpc.js"]) writeFileSync(join(directory, name), "entry", { mode: 0o640 }); + const mode = statSync(join(directory, "cli.js")).mode; + assert.equal(run(directory, "executable", "cli.js", "rpc.js").status, 0); + if (process.platform !== "win32") { + assert.equal(statSync(join(directory, "cli.js")).mode & 0o777, (mode | 0o111) & 0o777); + assert.equal(statSync(join(directory, "rpc.js")).mode & 0o111, 0o111); + } +}); + +test("invalid commands and missing arguments fail", (t) => { + const directory = workspace(t); + for (const args of [[], ["unknown", "dist"], ["clean"], ["copy", "dist"], ["executable"]]) { + const result = run(directory, ...args); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Usage: package-files/); + } +}); From 70ca8e8960b4f05faf1affd752e6231ba6087605 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Hou=C5=A1ka?= Date: Sat, 31 Oct 2026 17:55:58 +0100 Subject: [PATCH 215/271] docs(coding-agent): list external limitations as known issues with remedies (#11647) Restructures external limitations in the tmux and Windows guides as "Known issues" with a remedy for each entry. - `tmux.md`: the tmux 3.6 color reply section becomes a Known issues table (issue, affected versions, remedy). - `windows.md`: the conhost problem table moves to a Known issues section at the end of the page, with a Remedy column instead of the cause column. The terminal table links to it. Issue links are reduced to one per row. Known issues are limited to things outside Pi's control or deliberate limitations. Bugs Pi can fix stay in the issue tracker. --- packages/coding-agent/docs/tmux.md | 6 ++++-- packages/coding-agent/docs/windows.md | 24 +++++++++++++----------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/packages/coding-agent/docs/tmux.md b/packages/coding-agent/docs/tmux.md index c781ffe0a..1a640d174 100644 --- a/packages/coding-agent/docs/tmux.md +++ b/packages/coding-agent/docs/tmux.md @@ -54,6 +54,8 @@ Pi supports the xterm `modifyOtherKeys` format used by these versions. Restart t For older versions, upgrade tmux or use Pi outside tmux rather than relying on modified Enter shortcuts. -## Color codes appear in the editor with tmux 3.6 +## Known issues -If text such as `c0c/0c0c/0c0c` appears in the editor at startup, upgrade tmux to 3.7 or newer. tmux 3.6 sometimes sends the terminal's color replies to Pi as typed text ([tmux#4749](https://github.com/tmux/tmux/issues/4749)). Changing Pi's theme does not help. +| Issue | Affects | Remedy | +|---|---|---| +| Text such as `c0c/0c0c/0c0c` appears in the editor at startup ([tmux#4749](https://github.com/tmux/tmux/issues/4749)) | tmux 3.6 | Upgrade tmux. Changing Pi's theme does not help. | diff --git a/packages/coding-agent/docs/windows.md b/packages/coding-agent/docs/windows.md index 18826cd02..6e68979b8 100644 --- a/packages/coding-agent/docs/windows.md +++ b/packages/coding-agent/docs/windows.md @@ -89,18 +89,20 @@ Last updated: October 2026. | [Windows Terminal](https://aka.ms/terminal) | Bundled, current | Recommended | | [VS Code](https://code.visualstudio.com), [Tabby](https://tabby.sh), [WezTerm](https://wezterm.org) nightly | Bundled, current | Works | | [Alacritty](https://alacritty.org) | Built-in, unless `conpty.dll` and `OpenConsole.exe` from Windows Terminal are next to `alacritty.exe` | Works only with those files | -| WezTerm 20240203 and older | Bundled, old | Known problems | -| Classic console (`cmd.exe` or PowerShell without Windows Terminal), mintty (Git Bash), ConEmu, Cmder, [Hyper](https://hyper.is), Windows OpenSSH | Built-in | Known problems | - -| Problem | Where | Why | -|---|---|---| -| Switching from fullscreen to regular mode, or `Ctrl+G` from fullscreen, crashes the session ([#9186](https://github.com/earendil-works/pi/issues/9186), [microsoft/terminal#17515](https://github.com/microsoft/terminal/issues/17515)). Start with `--tui-mode regular` to avoid it. | Built-in | Conhost crashes when leaving the alternate screen while input is in line mode. | -| Text such as `;0;rgb:0000/0000/0000` appears in the prompt and BEL opens the external editor ([#10256](https://github.com/earendil-works/pi/issues/10256), [#10362](https://github.com/earendil-works/pi/issues/10362), [#10742](https://github.com/earendil-works/pi/issues/10742)) | Built-in behind another terminal (mintty, OpenSSH) | Conhost drops the start of terminal replies that arrive in pieces. | -| Fragments such as `79` or `55` appear in the prompt ([#10717](https://github.com/earendil-works/pi/issues/10717), [#10970](https://github.com/earendil-works/pi/issues/10970)) | WezTerm 20240203 | The bundled conhost cuts terminal replies short. | -| In regular mode, each keystroke redraws the input line on a new row ([#6300](https://github.com/earendil-works/pi/issues/6300)) | Built-in, classic console | Suspected: cursor wraps after full-width lines. | - -Microsoft has fixed these bugs in its source, but has not said when the built-in conhost will get the fixes. +| WezTerm 20240203 and older | Bundled, old | [Known issues](#known-issues) | +| Classic console (`cmd.exe` or PowerShell without Windows Terminal), mintty (Git Bash), ConEmu, Cmder, [Hyper](https://hyper.is), Windows OpenSSH | Built-in | [Known issues](#known-issues) | ## Configure Windows Terminal Windows Terminal reserves or rewrites some modified keys. See [Windows Terminal](terminal-setup.md#windows-terminal) to configure `Shift+Enter` and `Alt+Enter`, and [Keybindings](keybindings.md) for Pi's Windows and WSL shortcut defaults. + +## Known issues + +These issues come from conhost. Microsoft has fixed them in its source, but has not said when the built-in conhost will get the fixes. + +| Issue | Affects | Remedy | +|---|---|---| +| Switching from fullscreen to regular mode, or `Ctrl+G` from fullscreen, crashes the session ([#9186](https://github.com/earendil-works/pi/issues/9186), [microsoft/terminal#17515](https://github.com/microsoft/terminal/issues/17515)) | Built-in conhost | Use a terminal with a current conhost, or start Pi with `--tui-mode regular` | +| Text such as `;0;rgb:0000/0000/0000` appears in the prompt, and BEL opens the external editor ([#10256](https://github.com/earendil-works/pi/issues/10256)) | Built-in conhost behind another terminal: mintty, Windows OpenSSH | Use a terminal with a current conhost | +| Fragments such as `79` or `55` appear in the prompt ([#10717](https://github.com/earendil-works/pi/issues/10717)) | WezTerm 20240203 | Use WezTerm nightly | +| In regular mode, each keystroke redraws the input line on a new row ([#6300](https://github.com/earendil-works/pi/issues/6300)) | Built-in conhost, classic console | Use fullscreen mode (the default), or a terminal with a current conhost | From e3da248ea9100a3b5d53f9c701b6a47ef7f58d6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Hou=C5=A1ka?= Date: Sat, 31 Oct 2026 18:19:44 +0100 Subject: [PATCH 216/271] fix(coding-agent): keep updating git packages when git clean fails (#11648) Closes #11597 ## Problem On Windows, `pi update` stopped with `git clean -fdx failed with code 1` for git extensions such as pi-lens. 1. Another pi session has the extension loaded, so a native module (`@ast-grep/napi` `.node`) or `ast-grep.exe` is open in that process. 2. npm renames the in-use package dir to `node_modules/@ast-grep/.napi-win32-x64-msvc-`. Windows allows the rename, but npm cannot delete the dir afterwards. 3. The next `pi update` runs `git reset --hard`, then `git clean -fdx`. git prints `warning: failed to remove ...: Invalid argument` and exits 1. 4. pi threw, leaving the checkout at the new commit with the old dependencies. The suggested `-ffdx` does not help. Reproduced locally (git 2.52, Windows): with a running `.exe`, or a `.node` loaded by another node process and its dir renamed, both `-fdx` and `-ffdx` fail with `Invalid argument`. Once the process exits, `-fdx` succeeds. ## Fix In `cleanAndInstallGitDependencies`, a failed `git clean` no longer aborts the update. pi: - prints a warning that files may be in use by another pi session - still runs `npm install`, so dependencies match the new commit - keeps the `.pi-update-incomplete` marker, so the next update retries the clean ## Tests - `test/git-update.test.ts`: new regression test. A failing clean still updates and installs dependencies, warns, and keeps the marker. The next update cleans and removes the marker. - `test/package-manager.test.ts`: updated the test that expected the throw. `npm run check` reports `install-lock is out of date`, and `package-manager.test.ts > should expand ~ in exclusion and exact patterns` fails on Windows. Both also happen on `main` without this change. --- packages/coding-agent/CHANGELOG.md | 1 + .../coding-agent/src/core/package-manager.ts | 21 ++++++-- packages/coding-agent/test/git-update.test.ts | 53 +++++++++++++++++++ .../coding-agent/test/package-manager.test.ts | 5 +- 4 files changed, 73 insertions(+), 7 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 8a916e873..359250c80 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -48,6 +48,7 @@ - Fixed the external editor failing silently, for example when the editor command is wrong or a Windows editor cannot start from a deep WSL directory. pi now shows a status message such as `External editor exited with code 1; prompt unchanged` ([#11592](https://github.com/earendil-works/pi/issues/11592)) - Fixed `tool_search` and codemode `searchTools()` ranking a tool below tools that only mention its name in their descriptions, for example `peekaboo app` ranking `mcp__peekaboo__app` 14th. A tool matching more of the query's words now ranks above one matching fewer, a query word in a tool's name scores more than any number of mentions in its description, a tool whose whole name is in the query gets a large bonus, and long server instructions no longer lower the scores of that server's tools ([#11331](https://github.com/earendil-works/pi/issues/11331)) - Fixed `tool_search` and codemode `searchTools()` finding nothing for Chinese, Japanese, Thai, and other non-ASCII queries, because the tokenizer kept only ASCII letters and digits. Words in any script now match, text without spaces between words such as `搜索记录` is split into words, and canonically equivalent text such as `café` and `cafe\u0301` matches ([#10816](https://github.com/earendil-works/pi/issues/10816), [#11037](https://github.com/earendil-works/pi/pull/11037) by [@haoqixu](https://github.com/haoqixu)) +- Fixed `pi update` stopping with `git clean -fdx failed with code 1` on Windows when another pi session had a git extension's native module or executable open. Windows cannot delete such files, so the update left the new code with the old dependencies. pi now warns, installs the dependencies anyway, and retries the clean on the next update ([#11597](https://github.com/earendil-works/pi/issues/11597)) ### Removed diff --git a/packages/coding-agent/src/core/package-manager.ts b/packages/coding-agent/src/core/package-manager.ts index 1a0c96932..5f4229b24 100644 --- a/packages/coding-agent/src/core/package-manager.ts +++ b/packages/coding-agent/src/core/package-manager.ts @@ -2090,20 +2090,31 @@ export class DefaultPackageManager implements PackageManager { } private async cleanAndInstallGitDependencies(targetDir: string, markerPath: string): Promise { - // Clean untracked files (extensions should be pristine). If this fails after - // deleting dependencies, repair them so the existing extension still loads. + // Clean untracked files (extensions should be pristine). On Windows, files loaded by + // another process (e.g. a native module of this extension in another pi session) cannot + // be deleted, so git clean fails. Still install dependencies for the new commit, but keep + // the marker so the next update retries the clean. + let cleaned = true; try { await this.runCommand("git", ["clean", "-fdx"], { cwd: targetDir }); } catch (error) { - await this.repairMissingGitDependencies(targetDir).catch(() => {}); - throw error; + cleaned = false; + const reason = error instanceof Error ? error.message : String(error); + this.emitProgress({ + type: "progress", + action: "update", + source: targetDir, + message: `Could not remove all untracked files in ${targetDir} (${reason}). Files may be in use by another pi session; close it and run the update again.`, + }); } const packageJsonPath = join(targetDir, "package.json"); if (existsSync(packageJsonPath)) { await this.runNpmCommand(this.getGitDependencyInstallArgs(), { cwd: targetDir }); } - rmSync(markerPath, { force: true }); + if (cleaned) { + rmSync(markerPath, { force: true }); + } } private async ensureGitRef(targetDir: string, fetchArgs: string[], ref: string): Promise { diff --git a/packages/coding-agent/test/git-update.test.ts b/packages/coding-agent/test/git-update.test.ts index 80308277c..8b771f688 100644 --- a/packages/coding-agent/test/git-update.test.ts +++ b/packages/coding-agent/test/git-update.test.ts @@ -178,6 +178,59 @@ describe("DefaultPackageManager git update", () => { }); }); + describe("git clean failures", () => { + // https://github.com/earendil-works/pi/issues/11597 + it("should install dependencies and retry the clean later when git clean fails", async () => { + setupRemoteAndInstall(); + const newCommit = createCommit( + remoteDir, + "package.json", + JSON.stringify({ name: "test-extension", version: "1.0.0" }), + "Add package.json", + ); + + const executedCommands: string[] = []; + let failClean = true; + const managerWithInternals = packageManager as unknown as { + runCommand: (command: string, args: string[], options?: { cwd?: string }) => Promise; + }; + managerWithInternals.runCommand = async (command, args, options) => { + const commandLine = `${command} ${args.join(" ")}`; + executedCommands.push(commandLine); + if (command === "npm") return; + if (commandLine === "git clean -fdx" && failClean) { + throw new Error("git clean -fdx failed with code 1"); + } + const result = spawnSync(command, args, { cwd: options?.cwd, encoding: "utf-8" }); + if (result.status !== 0) { + throw new Error(`Command failed: ${commandLine}\n${result.stderr}`); + } + }; + const progressMessages: string[] = []; + packageManager.setProgressCallback((event) => { + if (event.type === "progress" && event.message) progressMessages.push(event.message); + }); + const markerPath = join(agentDir, "git", "github.com", "test", ".extension.pi-update-incomplete"); + + await packageManager.update(); + + expect(getCurrentCommit(installedDir)).toBe(newCommit); + expect(executedCommands).toContain("git clean -fdx"); + expect(executedCommands.some((command) => command.startsWith("npm install"))).toBe(true); + expect(progressMessages.some((message) => message.includes("Could not remove all untracked files"))).toBe( + true, + ); + expect(existsSync(markerPath)).toBe(true); + + failClean = false; + executedCommands.length = 0; + await packageManager.update(); + + expect(executedCommands).toContain("git clean -fdx"); + expect(existsSync(markerPath)).toBe(false); + }); + }); + describe("force-push scenarios", () => { it("should handle complete history rewrite", async () => { setupRemoteAndInstall(); diff --git a/packages/coding-agent/test/package-manager.test.ts b/packages/coding-agent/test/package-manager.test.ts index bdeffba40..b5895b331 100644 --- a/packages/coding-agent/test/package-manager.test.ts +++ b/packages/coding-agent/test/package-manager.test.ts @@ -1211,7 +1211,8 @@ Content`, expect(runCommandSpy).not.toHaveBeenCalledWith("git", ["clean", "-fdx"], { cwd: targetDir }); }); - it("should repair deleted git package dependencies when cleaning fails", async () => { + // https://github.com/earendil-works/pi/issues/11597 + it("should install git package dependencies when cleaning fails", async () => { const source = "git:github.com/user/repo"; const targetDir = join(agentDir, "git", "github.com", "user", "repo"); const fetchArgs = ["fetch", "--prune", "--no-tags", "origin", "+refs/heads/main:refs/remotes/origin/main"]; @@ -1237,7 +1238,7 @@ Content`, if (args[0] === "clean") throw new Error("simulated clean failure"); }); - await expect(packageManager.update(source)).rejects.toThrow("simulated clean failure"); + await packageManager.update(source); expect(runCommandSpy).toHaveBeenCalledWith( "npm", From d52e953fa846ce91426dcf41525a9d08ab5b29fc Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 17:47:30 +0000 Subject: [PATCH 217/271] docs(chord): slice 10 route overhead accepted --- packages/chord/docs/next/decisions.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index ec2672c24..ed88f9a7d 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2084,6 +2084,11 @@ validation and copies the v2 kernel added around Delta. is trusted (a plugin's own file). (2) The worker lock on macOS and the BSDs fails closed on a stale lock file, with an error naming it (no takeover; Linux and Android use abstract sockets and are unaffected). (3) A route's overhead: Opus's two trims, measured again after the `fetch` rework, and the numbers brought to the owner. +- **Slice 10: a route's overhead accepted; no header accessor (owner, 2026-10-31: "1 yes, 2 no"):** after the `fetch` + rework and the Node fast path a route costs +20-21 µs over a plain Node handler on the standard bench, +12-13 µs warmed + (before the rework +4.5 µs): the request's scope (2.1 µs), URL parsing, admission and routing, the baseline's headers, + the scope's seal, and about 5 µs of Node's microtask machinery an async path triggers. Accepted against the 10% bar. + No `request.header(name)` accessor: an authenticator reading headers builds the Web `Request` (about 3.5 µs more). - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From 83740ffa7a91d294cbc08569699ab6718631629a Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 18:33:24 +0000 Subject: [PATCH 218/271] docs(chord): CSP nonce per response --- packages/chord/docs/next/decisions.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index ed88f9a7d..7b5d9f26f 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2089,6 +2089,9 @@ validation and copies the v2 kernel added around Delta. (before the rework +4.5 µs): the request's scope (2.1 µs), URL parsing, admission and routing, the baseline's headers, the scope's seal, and about 5 µs of Node's microtask machinery an async path triggers. Accepted against the 10% bar. No `request.header(name)` accessor: an authenticator reading headers builds the Web `Request` (about 3.5 µs more). +- **A CSP nonce per response (owner, 2026-10-31: "csp ok"; Fable's recheck):** the HTTP component makes a fresh script + nonce for every response (`request.nonce`), not one per server: a nonce read from one page cannot be reused by an + injected script elsewhere. - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From 2b06c437cec65bf0b729a25c0e50972d7b4ae6e2 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 18:52:54 +0000 Subject: [PATCH 219/271] docs(chord): settings are arguments, extensions are registries --- packages/chord/docs/next/decisions.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 7b5d9f26f..4dfbcf8a4 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2092,6 +2092,16 @@ validation and copies the v2 kernel added around Delta. - **A CSP nonce per response (owner, 2026-10-31: "csp ok"; Fable's recheck):** the HTTP component makes a fresh script nonce for every response (`request.nonce`), not one per server: a nonce read from one page cannot be reused by an injected script elsewhere. +- **Settings are arguments, extensions are registries (owner, 2026-10-31: "making that config a registry is dumb; it's a + general problem for all host-lifetime natives that need initialization at host startup"; "k"; supersedes the + `HttpConfig` registry of "The HTTP server is a native object"):** what the app decides once at startup (a port, frame + limits, a timeout, a data directory, which implementation to use) is an argument to the component's facet, which base + code constructs; the facet and its host-lifetime objects close over it; changing it is a restart; no plugin can + touch it. What plugins add while the app runs (routes, authenticators, tools, panels, more indexers) is a registry. + HTTP: `createHttp(host, { listen, links, closeTimeout, onError }, ctx)`; `Http` and `Authenticators` stay registries. + The same for the harness (its store and defaults as arguments; models, environments, extensions as registries) and + the app kit's components (a session index takes its indexer as an argument, and may also follow a registry of + indexers if plugins should supply them). - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From 0baf611f04a35618727ee15bbf8bd7f49f16175d Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 19:07:10 +0000 Subject: [PATCH 220/271] docs(chord): API versions in import paths; the plugin model proposal --- packages/chord/docs/next/decisions.md | 7 + packages/chord/docs/next/plugin-model/book.md | 186 +++++++++++ .../chord/docs/next/plugin-model/index.html | 310 ++++++++++++++++++ 3 files changed, 503 insertions(+) create mode 100644 packages/chord/docs/next/plugin-model/book.md create mode 100644 packages/chord/docs/next/plugin-model/index.html diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 4dfbcf8a4..0878cbcc0 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2102,6 +2102,13 @@ validation and copies the v2 kernel added around Delta. The same for the harness (its store and defaults as arguments; models, environments, extensions as registries) and the app kit's components (a session index takes its indexer as an argument, and may also follow a registry of indexers if plugins should supply them). +- **API versions in the import path (owner, 2026-10-31: "this sounds about right"; "must be declared by the plugin in + code, we do not want to dig around package.json files anymore"):** the app's base code is internal except its API + modules, which it exposes as host packages with a version in the path (`@acme/app/v3`; Chord's + `@earendil-works/chord/next/v2`); the app lists the versions it still provides (an older one as the same module, or a + thin layer); an import of an unlisted version is a build error naming the available ones. No `package.json` field + and no npm ranges for this. Contracts and registry tokens inside an API keep their own versions. The rest of the + plugin model is a proposal (`docs/next/plugin-model/`). - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a diff --git a/packages/chord/docs/next/plugin-model/book.md b/packages/chord/docs/next/plugin-model/book.md new file mode 100644 index 000000000..5717711a3 --- /dev/null +++ b/packages/chord/docs/next/plugin-model/book.md @@ -0,0 +1,186 @@ +--- +title: The Plugin Model: what a plugin is, what it may import, and how versions meet +cover: The Plugin / Model +kicker: A design proposal · Chord v2 · 2026-10-31 +subtitle: Plugins, base code, versioned APIs and contracts, in one place, as discussed with the owner on 2026-10-31 +stats: + - 4 | kinds of import a plugin may write + - 1 | integer per API version, in the import path + - 0 | package.json fields Chord reads besides name and dependencies + - 0 | type checks in the build +note: | + A proposal written by the redesign session after a day of discussion with the owner. It collects what was decided + and what is proposed into one model. Decided points cite their decision in `decisions.md`; proposed ones are marked. + Nothing here is built yet beyond slices 7c and 10. +colophon: The Plugin Model · proposal · 2026-10-31 +--- + +# Before we begin {.preface} + +An app built on Chord is mostly plugins. Its own features are plugins, users add plugins, and agents write plugins +while the app runs. A plugin uses the app (where things live, who calls, how to declare objects), uses other plugins +(their objects and extension points), and uses npm packages. Each of those can change on its own schedule: the app +gets a new version, a plugin it depends on changes its contract, a package publishes a fix. + +The questions the owner kept asking were all versions of one: when a plugin is written against one state of the world +and runs against another, what does it get, and who notices a mismatch? This model answers it with one rule for each +kind of import, and versions that sit where the dependency is written: in the code. + +## Eight things to know + +:::findings +1. **A plugin is a directory, or a single file.** A directory has a `package.json` (its name, its npm dependencies) + and one file per host kind; a single file `hello.worker.ts` is a plugin named `hello` with no dependencies. +2. **Base code is internal unless it is an API.** The app exposes only versioned API modules to plugins; everything + else is unreachable. +3. **API versions are in the import path.** `import { Session } from "@acme/app/v3"`: the plugin says what it was + written for where it imports it; an unavailable version is a build error. +4. **APIs are split by host kind.** A shared module holds what every host may use (contracts, registry tokens, host + kinds); `/v3/web` and `/v3/tui` hold UI pieces; a worker never loads React. +5. **Contracts carry their own versions.** A plugin carries the contract version it was written for; the provider + binds it if it accepts that version, else the plugin waits with a clear reason. +6. **A plugin imports four things:** its own files, host packages, its npm dependencies, and the app's own plugins by + relative path. Nothing else. +7. **Settings are arguments, extensions are registries.** What the app decides at startup is passed to its components; + what plugins add is contributed. +8. **No type checking in the build.** Types come from where imports come from; editors and agents check, the build + only builds. +::: + +## Words used throughout + +**Base code** is the app's own program; its **API** is the part plugins may import, published as **API modules**, +one per version and host kind. A **host package** is a module the running process provides to every plugin (Chord, +the app's API modules). A **contract** is an object type's declaration; a **registry token** an extension point's. +A **facet** is a plugin's declarations for one host kind. + +# What a Plugin Is {#plugin lede="A directory or a single file, named once, with one file per host kind."} + +```text +plugins/notes/ a directory plugin + package.json { "name": "@acme/notes", "dependencies": { "slugify": "1.6.6" } } + contract.ts the object types and registry tokens it offers + worker.ts server.ts web.tsx one facet per host kind it needs +plugins/hello.worker.ts a single-file plugin "hello" with one worker facet +plugins/hello.web.tsx the same plugin's web facet +``` + +A directory plugin's name is its `package.json` name; Chord reads only `name` and `dependencies` from it. A +single-file plugin's name is its file's base name; it has no dependencies and imports only host packages. When a +single file needs a package or a second file, it becomes a directory. Two roots with the same name: the first root +wins, the other is reported (slice 7c). + +# Base Code: Internal and External {#base lede="What the app keeps to itself, what it exposes, and how exposure is versioned."} + +The app's base code is everything that starts the process: settings, startup, core objects' implementations, the +session index, migrations. Plugins must not depend on any of it, or every refactoring of the app would break them. +So the app draws a line and exposes only its **API**: + +```text +src/api/v3/index.ts shared: host kinds, the principal type, defineFacet per kind, core contracts, registry tokens +src/api/v3/web.ts web only: shared UI pieces (a Sheet, theme variables), React-typed registry tokens +src/api/v3/tui.ts TUI only: its UI pieces +src/… internal: everything else +``` + +Each API module is a host package of the kinds that may use it: `@acme/app/v3` for every kind, `@acme/app/v3/web` for +the web kind only. A worker plugin importing `@acme/app/v3/web` is a build error, and a worker process never loads +React. The shared module stays free of runtime imports of UI libraries: a registry token that needs a React type names +it with `import type`, which the build erases. + +:::note | Proposed +The split of the external API into a shared module and per-kind modules answers the owner's worry about "web, TUI, +worker, server things all in the same index.ts". Proposed, not yet decided. +::: + +# Versions in the Import Path {#versions lede="A plugin says what it was written for where it imports it."} + +The app's API changes over time: a host kind is renamed, a principal variant changes. A plugin written for an older +API must not silently get the new one. So every API module has a version in its path, and the app lists which versions +it still provides: + +```ts +// plugins/stats/worker.ts +import { defineFacet, Session, Panels } from "@acme/app/v3"; +import { remote, str } from "@earendil-works/chord/next/v2"; +``` + +- The app provides `/v3`, its current API. If version 2 plugins still work unchanged against it, `/v2` is listed too, + pointing to the same module. An app that needs a compatibility layer makes `/v2` a thin module over `/v3`. +- An import of a version the app no longer lists is a build error: "@acme/app/v1 is not available: this app provides + v3, v2". The plugin keeps its old code, and the message tells an agent exactly what to change. +- Nothing to fill in elsewhere: no `package.json` field, no npm ranges. Skeletons and examples import the current + version, so an agent copies it without thinking. +- Chord's own surface is versioned the same way (`@earendil-works/chord/next/v2`). + +Decided: "the API version must be declared by the plugin in code" (owner, 2026-10-31); the import path is how. + +# Contracts and Registries {#contracts lede="Finer versions inside the API, and what crosses between plugins."} + +Contracts and registry tokens keep their own versions inside the API, because they evolve independently of the API as +a whole: + +- **An object type**: the consumer carries the contract version it was written for; the provider binds it at that + version if it accepts it (`accepts` maps arguments forward and results and state back), otherwise the consumer waits + "contract mismatch: v1, served v3". Shapes are fingerprinted, so a changed shape without a version bump is caught. +- **A registry token**: a name, a version and TypeScript types for what is contributed and read. Values are in-process + things (functions, components), which cannot be fingerprinted: bumping the version when they change is the defining + plugin's rule, and its `accepts` adapts older contributions. +- **Host kinds on registries** (proposed): a token names the kinds that serve it (`hosts: ["web"]`), typed, so a worker + facet contributing to a web registry is a type error and a load error, not a silent no-op. + +A contract file imports only host packages at runtime (Chord, the app's API); native types it mentions are type-only. + +# What a Plugin May Import {#imports lede="Four kinds of import, each with one rule."} + +| Import | Resolved to | Versioned by | +| --- | --- | --- | +| its own files (`./store.ts`) | copied into its script | the plugin itself | +| host packages (`@acme/app/v3`, `@earendil-works/chord/next/v2`, `react`) | the running process's instance, shared | the path's version; the app lists what it provides | +| npm packages in its `dependencies` (`slugify`, a third party's contracts package) | its own `node_modules`, bundled | the installed version | +| the app's own plugins (`../notes/contract.ts`), in the same root | copied into its script | the same revision: one repository | + +A third-party plugin that wants the app's plugins' contracts gets them through the app's API (core contracts) or the +app's published package, as an npm dependency at the version it was written for, never from the running plugin's +files. Anything else (a path into base code, an unlisted version, a package not in its dependencies) is a build error. + +# Settings and Extensions {#settings lede="How the app configures its components without letting plugins override it."} + +Decided ("Settings are arguments, extensions are registries"): what the app decides at startup is an argument to the +component's facet, which base code constructs, and plugins cannot change it: + +```ts +const http = createHttp(server, { listen: { port: 8788 }, links: { path: "/chord", maxPayload: MiB }, closeTimeout: 10_000, onError }, ctx); +``` + +What plugins add while the app runs is a registry: routes, authenticators, tools, panels. A component may offer both +for one thing: the session index takes its indexer as an argument, and may also follow a registry of indexers if +plugins should supply them. + +# Building and Evaluating {#build lede="Unchanged from slices 7c and 10: one build on the server, evaluation everywhere."} + +Only the server builds, because only the server has the sources. `createPlugins` builds every kind and needs only +names: per kind its platform and its host package names. Each process evaluates the scripts of its kind with its own +module objects (the server's at `plugins.load`, a worker's in `workerMain`, a page's in its base bundle), so a plugin +shares that process's Chord and API. Base-code facets are passed beside the plugins; they are not plugins, because +they close over the process's own values and never reload. + +# Types and Tests {#tools lede="Where types come from, and how a plugin is tested alone."} + +Types come from where imports come from: relative files and npm packages resolve natively in TypeScript; the app's API +modules ship `.d.ts` per version in the app's package. There is no type checking in the build (the owner: seconds per +build is too slow); editors check as an author types, and an agent may run `tsc` as a tool. + +Testing a plugin alone (proposed): `testPlugin(dir, { kind, stubs })` builds the plugin like production, loads it on an +in-memory host with the kind's real host packages, and loads stub facets that serve the contracts it depends on. A stub +is an ordinary facet; there is no mocking framework. + +# Open Questions {.epilogue lede="What is proposed here and still to decide."} + +1. The external API split by host kind (`/v3`, `/v3/web`, `/v3/tui`). +2. Host kinds on registry tokens, typed and checked at load. +3. Single-file plugins (`hello.worker.ts`, named by the file). +4. Contracts of the app's plugins for third parties: through the API modules only, or also the app's npm package. +5. `testPlugin` with stub facets. +6. Reversing "Plugins import each other by name" (2026-10-31, morning) in favour of relative paths within the app + and npm dependencies across it. diff --git a/packages/chord/docs/next/plugin-model/index.html b/packages/chord/docs/next/plugin-model/index.html new file mode 100644 index 000000000..2549955ac --- /dev/null +++ b/packages/chord/docs/next/plugin-model/index.html @@ -0,0 +1,310 @@ + + +The Plugin Model: what a plugin is, what it may import, and how versions meet + + + + +

    +
    +
    A design proposal · Chord v2 · 2026-10-31
    +

    The Plugin
    Model

    +

    Plugins, base code, versioned APIs and contracts, in one place, as discussed with the owner on 2026-10-31

    +
    4kinds of import a plugin may write
    1integer per API version, in the import path
    0package.json fields Chord reads besides name and dependencies
    0type checks in the build
    +

    A proposal written by the redesign session after a day of discussion with the owner. It collects what was decided +and what is proposed into one model. Decided points cite their decision in decisions.md; proposed ones are marked. +Nothing here is built yet beyond slices 7c and 10.

    +
    +
    + + +
    +

    Before we begin

    +

    An app built on Chord is mostly plugins. Its own features are plugins, users add plugins, and agents write plugins +while the app runs. A plugin uses the app (where things live, who calls, how to declare objects), uses other plugins +(their objects and extension points), and uses npm packages. Each of those can change on its own schedule: the app +gets a new version, a plugin it depends on changes its contract, a package publishes a fix.

    +

    The questions the owner kept asking were all versions of one: when a plugin is written against one state of the world +and runs against another, what does it get, and who notices a mismatch? This model answers it with one rule for each +kind of import, and versions that sit where the dependency is written: in the code.

    +

    Eight things to know

    +
      +
    1. A plugin is a directory, or a single file. A directory has a package.json (its name, its npm dependencies) +and one file per host kind; a single file hello.worker.ts is a plugin named hello with no dependencies.
    2. +
    3. Base code is internal unless it is an API. The app exposes only versioned API modules to plugins; everything +else is unreachable.
    4. +
    5. API versions are in the import path. import { Session } from "@acme/app/v3": the plugin says what it was +written for where it imports it; an unavailable version is a build error.
    6. +
    7. APIs are split by host kind. A shared module holds what every host may use (contracts, registry tokens, host +kinds); /v3/web and /v3/tui hold UI pieces; a worker never loads React.
    8. +
    9. Contracts carry their own versions. A plugin carries the contract version it was written for; the provider +binds it if it accepts that version, else the plugin waits with a clear reason.
    10. +
    11. A plugin imports four things: its own files, host packages, its npm dependencies, and the app's own plugins by +relative path. Nothing else.
    12. +
    13. Settings are arguments, extensions are registries. What the app decides at startup is passed to its components; +what plugins add is contributed.
    14. +
    15. No type checking in the build. Types come from where imports come from; editors and agents check, the build +only builds.
    16. +
    +

    Words used throughout

    +

    Base code is the app's own program; its API is the part plugins may import, published as API modules, +one per version and host kind. A host package is a module the running process provides to every plugin (Chord, +the app's API modules). A contract is an object type's declaration; a registry token an extension point's. +A facet is a plugin's declarations for one host kind.

    +
    + +
    I

    What a Plugin Is

    A directory or a single file, named once, with one file per host kind.

    +
    plugins/notes/                     a directory plugin
    +  package.json                     { "name": "@acme/notes", "dependencies": { "slugify": "1.6.6" } }
    +  contract.ts                      the object types and registry tokens it offers
    +  worker.ts  server.ts  web.tsx    one facet per host kind it needs
    +plugins/hello.worker.ts            a single-file plugin "hello" with one worker facet
    +plugins/hello.web.tsx              the same plugin's web facet
    +

    A directory plugin's name is its package.json name; Chord reads only name and dependencies from it. A +single-file plugin's name is its file's base name; it has no dependencies and imports only host packages. When a +single file needs a package or a second file, it becomes a directory. Two roots with the same name: the first root +wins, the other is reported (slice 7c).

    +
    + +
    II

    Base Code: Internal and External

    What the app keeps to itself, what it exposes, and how exposure is versioned.

    +

    The app's base code is everything that starts the process: settings, startup, core objects' implementations, the +session index, migrations. Plugins must not depend on any of it, or every refactoring of the app would break them. +So the app draws a line and exposes only its API:

    +
    src/api/v3/index.ts       shared: host kinds, the principal type, defineFacet per kind, core contracts, registry tokens
    +src/api/v3/web.ts         web only: shared UI pieces (a Sheet, theme variables), React-typed registry tokens
    +src/api/v3/tui.ts         TUI only: its UI pieces
    +src/…                     internal: everything else
    +

    Each API module is a host package of the kinds that may use it: @acme/app/v3 for every kind, @acme/app/v3/web for +the web kind only. A worker plugin importing @acme/app/v3/web is a build error, and a worker process never loads +React. The shared module stays free of runtime imports of UI libraries: a registry token that needs a React type names +it with import type, which the build erases.

    +
    Proposed

    The split of the external API into a shared module and per-kind modules answers the owner's worry about "web, TUI, +worker, server things all in the same index.ts". Proposed, not yet decided.

    +
    +
    + +
    III

    Versions in the Import Path

    A plugin says what it was written for where it imports it.

    +

    The app's API changes over time: a host kind is renamed, a principal variant changes. A plugin written for an older +API must not silently get the new one. So every API module has a version in its path, and the app lists which versions +it still provides:

    +
    // plugins/stats/worker.ts
    +import { defineFacet, Session, Panels } from "@acme/app/v3";
    +import { remote, str } from "@earendil-works/chord/next/v2";
    +
      +
    • The app provides /v3, its current API. If version 2 plugins still work unchanged against it, /v2 is listed too, +pointing to the same module. An app that needs a compatibility layer makes /v2 a thin module over /v3.
    • +
    • An import of a version the app no longer lists is a build error: "@acme/app/v1 is not available: this app provides +v3, v2". The plugin keeps its old code, and the message tells an agent exactly what to change.
    • +
    • Nothing to fill in elsewhere: no package.json field, no npm ranges. Skeletons and examples import the current +version, so an agent copies it without thinking.
    • +
    • Chord's own surface is versioned the same way (@earendil-works/chord/next/v2).
    • +
    +

    Decided: "the API version must be declared by the plugin in code" (owner, 2026-10-31); the import path is how.

    +
    + +
    IV

    Contracts and Registries

    Finer versions inside the API, and what crosses between plugins.

    +

    Contracts and registry tokens keep their own versions inside the API, because they evolve independently of the API as +a whole:

    +
      +
    • An object type: the consumer carries the contract version it was written for; the provider binds it at that +version if it accepts it (accepts maps arguments forward and results and state back), otherwise the consumer waits +"contract mismatch: v1, served v3". Shapes are fingerprinted, so a changed shape without a version bump is caught.
    • +
    • A registry token: a name, a version and TypeScript types for what is contributed and read. Values are in-process +things (functions, components), which cannot be fingerprinted: bumping the version when they change is the defining +plugin's rule, and its accepts adapts older contributions.
    • +
    • Host kinds on registries (proposed): a token names the kinds that serve it (hosts: ["web"]), typed, so a worker +facet contributing to a web registry is a type error and a load error, not a silent no-op.
    • +
    +

    A contract file imports only host packages at runtime (Chord, the app's API); native types it mentions are type-only.

    +
    + +
    V

    What a Plugin May Import

    Four kinds of import, each with one rule.

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    ImportResolved toVersioned by
    its own files (./store.ts)copied into its scriptthe plugin itself
    host packages (@acme/app/v3, @earendil-works/chord/next/v2, react)the running process's instance, sharedthe path's version; the app lists what it provides
    npm packages in its dependencies (slugify, a third party's contracts package)its own node_modules, bundledthe installed version
    the app's own plugins (../notes/contract.ts), in the same rootcopied into its scriptthe same revision: one repository
    +

    A third-party plugin that wants the app's plugins' contracts gets them through the app's API (core contracts) or the +app's published package, as an npm dependency at the version it was written for, never from the running plugin's +files. Anything else (a path into base code, an unlisted version, a package not in its dependencies) is a build error.

    +
    + +
    VI

    Settings and Extensions

    How the app configures its components without letting plugins override it.

    +

    Decided ("Settings are arguments, extensions are registries"): what the app decides at startup is an argument to the +component's facet, which base code constructs, and plugins cannot change it:

    +
    const http = createHttp(server, { listen: { port: 8788 }, links: { path: "/chord", maxPayload: MiB }, closeTimeout: 10_000, onError }, ctx);
    +

    What plugins add while the app runs is a registry: routes, authenticators, tools, panels. A component may offer both +for one thing: the session index takes its indexer as an argument, and may also follow a registry of indexers if +plugins should supply them.

    +
    + +
    VII

    Building and Evaluating

    Unchanged from slices 7c and 10: one build on the server, evaluation everywhere.

    +

    Only the server builds, because only the server has the sources. createPlugins builds every kind and needs only +names: per kind its platform and its host package names. Each process evaluates the scripts of its kind with its own +module objects (the server's at plugins.load, a worker's in workerMain, a page's in its base bundle), so a plugin +shares that process's Chord and API. Base-code facets are passed beside the plugins; they are not plugins, because +they close over the process's own values and never reload.

    +
    + +
    VIII

    Types and Tests

    Where types come from, and how a plugin is tested alone.

    +

    Types come from where imports come from: relative files and npm packages resolve natively in TypeScript; the app's API +modules ship .d.ts per version in the app's package. There is no type checking in the build (the owner: seconds per +build is too slow); editors check as an author types, and an agent may run tsc as a tool.

    +

    Testing a plugin alone (proposed): testPlugin(dir, { kind, stubs }) builds the plugin like production, loads it on an +in-memory host with the kind's real host packages, and loads stub facets that serve the contracts it depends on. A stub +is an ordinary facet; there is no mocking framework.

    +
    + +
    ∴

    Open Questions

    What is proposed here and still to decide.

    +
      +
    1. The external API split by host kind (/v3, /v3/web, /v3/tui).
    2. +
    3. Host kinds on registry tokens, typed and checked at load.
    4. +
    5. Single-file plugins (hello.worker.ts, named by the file).
    6. +
    7. Contracts of the app's plugins for third parties: through the API modules only, or also the app's npm package.
    8. +
    9. testPlugin with stub facets.
    10. +
    11. Reversing "Plugins import each other by name" (2026-10-31, morning) in favour of relative paths within the app +and npm dependencies across it.
    12. +
    +
    + +

    The Plugin Model · proposal · 2026-10-31

    +
    + + From 81f3cd412758ebdd92324c7d63b669a189408d0a Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 20:06:14 +0000 Subject: [PATCH 221/271] docs(chord): a schema package beneath pi-ai and Chord --- packages/chord/docs/next/decisions.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 0878cbcc0..941b1dee7 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2109,6 +2109,16 @@ validation and copies the v2 kernel added around Delta. thin layer); an import of an unlisted version is a build error naming the available ones. No `package.json` field and no npm ranges for this. Contracts and registry tokens inside an API keep their own versions. The rest of the plugin model is a proposal (`docs/next/plugin-model/`). +- **A schema package beneath pi-ai and Chord (owner, 2026-10-31: "i want the schema package"; "if we keep it a + dependency in pi-ai, we must be able to rip it out"; TypeBox is "horrible to treeshake"):** a small package with no + dependencies (working name `@earendil-works/schema`) under both pi-ai and Chord (siblings: neither depends on the + other): helpers returning plain JSON Schema, a structural `JsonSchema` type, `Static` from the helpers' literal + types, and a validator for general JSON Schema (tool schemas from MCP servers and extensions use `$ref`, `anyOf`, + `pattern`, ...), without `eval` or `new Function` (CSP-safe). Chord v2 drops TypeBox; pi-ai's core never imports + it (TypeBox schemas still pass as JSON Schema; TypeBox-typed extras only in an optional subpath, so ripping it out + is deleting that subpath). Not omptype (its validator compiles with `new Function`; its 10.7k lines are mostly + ArkType's API). Open: which validator (measure `@cfworker/json-schema` and TypeBox against the official + JSON-Schema-Test-Suite and Chord's argument checks first), and moving `Type` out of pi-ai's main export. - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From 8959ad6e6845933dd01dbaa96929243d68bc4d48 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 20:40:32 +0000 Subject: [PATCH 222/271] feat(chord): Chord v2 slice 10, HTTP and workers chord/next/http: a Web fetch handler built from the registry Http (routes, wraps, sockets, hosts, relays, CSP sources) and Authenticators, responses as descriptions, a security baseline on by default, Chord's own Node listener as a host-lifetime object, nodeRequest for apps mounting it, links and plugin sockets. chord/next/workers: a forked worker per address, started on demand, stopped when idle, never restarted by itself, plugins built by the server and shipped. Plugins: kinds with platform and host packages, createPluginRuntime, scripts(kind), rollout results per host, imports of other plugins by name. Kernel: RouteHandle (retry, suspend), no route backoff, anonymous host.root, origins only from relays. ws is an optional peer dependency. --- package-lock.json | 6 +- packages/chord/CHANGELOG.md | 1 + packages/chord/docs/next/benchmarks.md | 43 + packages/chord/package.json | 6 +- packages/chord/src/next-node/http/README.md | 166 ++++ .../chord/src/next-node/http/admission.ts | 178 ++++ .../chord/src/next-node/http/authenticate.ts | 54 ++ packages/chord/src/next-node/http/cookies.ts | 71 ++ packages/chord/src/next-node/http/file.ts | 113 +++ packages/chord/src/next-node/http/headers.ts | 142 +++ packages/chord/src/next-node/http/http.ts | 565 ++++++++++++ packages/chord/src/next-node/http/incoming.ts | 43 + packages/chord/src/next-node/http/index.ts | 34 + packages/chord/src/next-node/http/node.ts | 317 +++++++ packages/chord/src/next-node/http/request.ts | 145 ++++ packages/chord/src/next-node/http/response.ts | 258 ++++++ packages/chord/src/next-node/http/router.ts | 412 +++++++++ packages/chord/src/next-node/http/upgrades.ts | 245 ++++++ .../chord/src/next-node/plugins/README.md | 72 +- packages/chord/src/next-node/plugins/build.ts | 68 +- .../chord/src/next-node/plugins/evaluate.ts | 11 +- packages/chord/src/next-node/plugins/index.ts | 3 + .../chord/src/next-node/plugins/plugins.ts | 366 ++++---- .../chord/src/next-node/plugins/runtime.ts | 215 +++++ .../chord/src/next-node/workers/README.md | 95 ++ .../chord/src/next-node/workers/control.ts | 121 +++ packages/chord/src/next-node/workers/index.ts | 20 + packages/chord/src/next-node/workers/lock.ts | 142 +++ packages/chord/src/next-node/workers/main.ts | 293 +++++++ .../chord/src/next-node/workers/process.ts | 150 ++++ .../chord/src/next-node/workers/workers.ts | 742 ++++++++++++++++ packages/chord/src/next/host/README.md | 23 +- packages/chord/src/next/host/host.ts | 36 +- packages/chord/src/next/host/index.ts | 2 +- packages/chord/src/next/host/root.ts | 54 +- packages/chord/src/next/host/route.ts | 170 +++- packages/chord/src/next/remote/README.md | 8 +- packages/chord/src/next/remote/host-link.ts | 16 +- packages/chord/src/next/remote/internal.ts | 2 +- packages/chord/src/next/remote/path.ts | 30 +- packages/chord/src/next/remote/provider.ts | 13 + packages/chord/src/next/remote/relay.ts | 5 + .../chord/src/next/remote/remote-object.ts | 10 + .../fixtures/plugins/files/worker.ts | 2 +- .../fixtures/plugins/notes/server.ts | 2 +- .../fixtures/plugins/notes/worker.ts | 2 +- .../test/next-node/fixtures/workers/child.ts | 12 + .../next-node/fixtures/workers/control.ts | 47 + .../fixtures/workers/plugins/notes/label.ts | 3 + .../workers/plugins/notes/package.json | 5 + .../fixtures/workers/plugins/notes/worker.ts | 17 + .../test/next-node/fixtures/workers/worker.ts | 146 ++++ .../test/next-node/http/admission.test.ts | 154 ++++ .../test/next-node/http/authenticate.test.ts | 58 ++ .../chord/test/next-node/http/cookies.test.ts | 56 ++ .../chord/test/next-node/http/file.test.ts | 135 +++ .../chord/test/next-node/http/headers.test.ts | 106 +++ .../chord/test/next-node/http/http.test.ts | 818 ++++++++++++++++++ .../chord/test/next-node/http/node.test.ts | 215 +++++ .../chord/test/next-node/http/request.test.ts | 119 +++ .../test/next-node/http/response.test.ts | 245 ++++++ .../chord/test/next-node/http/router.test.ts | 228 +++++ packages/chord/test/next-node/http/support.ts | 204 +++++ .../test/next-node/http/upgrades.test.ts | 458 ++++++++++ .../test/next-node/plugins/build.test.ts | 41 +- .../test/next-node/plugins/plugins.test.ts | 167 +++- .../test/next-node/plugins/runtime.test.ts | 126 +++ .../chord/test/next-node/plugins/support.ts | 12 +- .../test/next-node/workers/control.test.ts | 55 ++ .../chord/test/next-node/workers/lock.test.ts | 62 ++ .../chord/test/next-node/workers/main.test.ts | 87 ++ .../test/next-node/workers/process.test.ts | 46 + .../chord/test/next-node/workers/support.ts | 151 ++++ .../test/next-node/workers/workers.test.ts | 605 +++++++++++++ packages/chord/test/next/bench/http.ts | 120 +++ packages/chord/test/next/bench/plugins.ts | 8 +- packages/chord/test/next/bench/workers.ts | 107 +++ .../chord/test/next/examples/12-routes.ts | 4 +- .../test/next/examples/35-plugins/main.ts | 27 +- .../35-plugins/plugins/clock/server.ts | 2 +- .../35-plugins/plugins/notes/server.ts | 2 +- .../35-plugins/plugins/notes/worker.ts | 2 +- .../next/examples/36-plugin-failures/main.ts | 7 +- .../plugins/forecast/server.ts | 2 +- .../plugins/summary/server.ts | 2 +- .../next/examples/37-plugin-imports/main.ts | 23 +- .../plugins/report/server.ts | 4 +- .../37-plugin-imports/plugins/ruler/server.ts | 2 +- .../examples/38-watch-and-rollout/main.ts | 16 +- .../plugins/contracts/package.json | 2 +- .../plugins/notes/server.ts | 2 +- .../plugins/notes/worker.ts | 2 +- .../plugins/old/server.ts | 2 +- .../plugins/tags/server.ts | 2 +- .../plugins/tags/worker.ts | 2 +- .../next/examples/39-browser-build/main.ts | 6 +- .../chord/test/next/examples/40-http/main.ts | 56 ++ .../40-http/plugins/notes/files/hello.md | 3 + .../40-http/plugins/notes/package.json | 5 + .../examples/40-http/plugins/notes/server.ts | 29 + .../test/next/examples/41-workers/main.ts | 68 ++ .../41-workers/plugins/sessions/contract.ts | 13 + .../41-workers/plugins/sessions/package.json | 5 + .../41-workers/plugins/sessions/worker.ts | 11 + .../test/next/examples/41-workers/worker.ts | 22 + .../next/examples/42-worker-crash/main.ts | 67 ++ .../plugins/sessions/contract.ts | 13 + .../plugins/sessions/package.json | 5 + .../plugins/sessions/worker.ts | 17 + .../next/examples/42-worker-crash/worker.ts | 22 + .../test/next/examples/43-rollout/main.ts | 85 ++ .../43-rollout/plugins/notes/contract.ts | 16 + .../43-rollout/plugins/notes/label.ts | 3 + .../43-rollout/plugins/notes/package.json | 5 + .../43-rollout/plugins/notes/server.ts | 7 + .../43-rollout/plugins/notes/worker.ts | 9 + .../test/next/examples/43-rollout/worker.ts | 22 + .../test/next/examples/44-http-wraps/main.ts | 37 + .../44-http-wraps/plugins/audit/package.json | 5 + .../44-http-wraps/plugins/audit/server.ts | 24 + .../44-http-wraps/plugins/notes/package.json | 5 + .../44-http-wraps/plugins/notes/server.ts | 20 + .../next/examples/45-plugin-socket/main.ts | 64 ++ .../plugins/echo/package.json | 5 + .../45-plugin-socket/plugins/echo/server.ts | 16 + .../next/examples/46-allowed-hosts/main.ts | 61 ++ .../plugins/tunnel/package.json | 5 + .../46-allowed-hosts/plugins/tunnel/server.ts | 31 + .../next/examples/47-http-own-server/main.ts | 70 ++ .../plugins/notes/package.json | 5 + .../plugins/notes/server.ts | 12 + .../next/examples/48-http-baseline/main.ts | 84 ++ .../plugins/notes/package.json | 5 + .../48-http-baseline/plugins/notes/server.ts | 33 + .../chord/test/next/host/revocation.test.ts | 7 + packages/chord/test/next/host/root.test.ts | 35 +- packages/chord/test/next/host/route.test.ts | 221 +++-- .../chord/test/next/remote/host-link.test.ts | 78 +- packages/chord/test/next/remote/path.test.ts | 20 + .../chord/test/next/remote/provider.test.ts | 24 +- packages/chord/test/next/remote/relay.test.ts | 65 +- packages/chord/test/next/support.ts | 7 +- tsconfig.json | 9 + 143 files changed, 11066 insertions(+), 561 deletions(-) create mode 100644 packages/chord/src/next-node/http/README.md create mode 100644 packages/chord/src/next-node/http/admission.ts create mode 100644 packages/chord/src/next-node/http/authenticate.ts create mode 100644 packages/chord/src/next-node/http/cookies.ts create mode 100644 packages/chord/src/next-node/http/file.ts create mode 100644 packages/chord/src/next-node/http/headers.ts create mode 100644 packages/chord/src/next-node/http/http.ts create mode 100644 packages/chord/src/next-node/http/incoming.ts create mode 100644 packages/chord/src/next-node/http/index.ts create mode 100644 packages/chord/src/next-node/http/node.ts create mode 100644 packages/chord/src/next-node/http/request.ts create mode 100644 packages/chord/src/next-node/http/response.ts create mode 100644 packages/chord/src/next-node/http/router.ts create mode 100644 packages/chord/src/next-node/http/upgrades.ts create mode 100644 packages/chord/src/next-node/plugins/runtime.ts create mode 100644 packages/chord/src/next-node/workers/README.md create mode 100644 packages/chord/src/next-node/workers/control.ts create mode 100644 packages/chord/src/next-node/workers/index.ts create mode 100644 packages/chord/src/next-node/workers/lock.ts create mode 100644 packages/chord/src/next-node/workers/main.ts create mode 100644 packages/chord/src/next-node/workers/process.ts create mode 100644 packages/chord/src/next-node/workers/workers.ts create mode 100644 packages/chord/test/next-node/fixtures/workers/child.ts create mode 100644 packages/chord/test/next-node/fixtures/workers/control.ts create mode 100644 packages/chord/test/next-node/fixtures/workers/plugins/notes/label.ts create mode 100644 packages/chord/test/next-node/fixtures/workers/plugins/notes/package.json create mode 100644 packages/chord/test/next-node/fixtures/workers/plugins/notes/worker.ts create mode 100644 packages/chord/test/next-node/fixtures/workers/worker.ts create mode 100644 packages/chord/test/next-node/http/admission.test.ts create mode 100644 packages/chord/test/next-node/http/authenticate.test.ts create mode 100644 packages/chord/test/next-node/http/cookies.test.ts create mode 100644 packages/chord/test/next-node/http/file.test.ts create mode 100644 packages/chord/test/next-node/http/headers.test.ts create mode 100644 packages/chord/test/next-node/http/http.test.ts create mode 100644 packages/chord/test/next-node/http/node.test.ts create mode 100644 packages/chord/test/next-node/http/request.test.ts create mode 100644 packages/chord/test/next-node/http/response.test.ts create mode 100644 packages/chord/test/next-node/http/router.test.ts create mode 100644 packages/chord/test/next-node/http/support.ts create mode 100644 packages/chord/test/next-node/http/upgrades.test.ts create mode 100644 packages/chord/test/next-node/plugins/runtime.test.ts create mode 100644 packages/chord/test/next-node/workers/control.test.ts create mode 100644 packages/chord/test/next-node/workers/lock.test.ts create mode 100644 packages/chord/test/next-node/workers/main.test.ts create mode 100644 packages/chord/test/next-node/workers/process.test.ts create mode 100644 packages/chord/test/next-node/workers/support.ts create mode 100644 packages/chord/test/next-node/workers/workers.test.ts create mode 100644 packages/chord/test/next/bench/http.ts create mode 100644 packages/chord/test/next/bench/workers.ts create mode 100644 packages/chord/test/next/examples/40-http/main.ts create mode 100644 packages/chord/test/next/examples/40-http/plugins/notes/files/hello.md create mode 100644 packages/chord/test/next/examples/40-http/plugins/notes/package.json create mode 100644 packages/chord/test/next/examples/40-http/plugins/notes/server.ts create mode 100644 packages/chord/test/next/examples/41-workers/main.ts create mode 100644 packages/chord/test/next/examples/41-workers/plugins/sessions/contract.ts create mode 100644 packages/chord/test/next/examples/41-workers/plugins/sessions/package.json create mode 100644 packages/chord/test/next/examples/41-workers/plugins/sessions/worker.ts create mode 100644 packages/chord/test/next/examples/41-workers/worker.ts create mode 100644 packages/chord/test/next/examples/42-worker-crash/main.ts create mode 100644 packages/chord/test/next/examples/42-worker-crash/plugins/sessions/contract.ts create mode 100644 packages/chord/test/next/examples/42-worker-crash/plugins/sessions/package.json create mode 100644 packages/chord/test/next/examples/42-worker-crash/plugins/sessions/worker.ts create mode 100644 packages/chord/test/next/examples/42-worker-crash/worker.ts create mode 100644 packages/chord/test/next/examples/43-rollout/main.ts create mode 100644 packages/chord/test/next/examples/43-rollout/plugins/notes/contract.ts create mode 100644 packages/chord/test/next/examples/43-rollout/plugins/notes/label.ts create mode 100644 packages/chord/test/next/examples/43-rollout/plugins/notes/package.json create mode 100644 packages/chord/test/next/examples/43-rollout/plugins/notes/server.ts create mode 100644 packages/chord/test/next/examples/43-rollout/plugins/notes/worker.ts create mode 100644 packages/chord/test/next/examples/43-rollout/worker.ts create mode 100644 packages/chord/test/next/examples/44-http-wraps/main.ts create mode 100644 packages/chord/test/next/examples/44-http-wraps/plugins/audit/package.json create mode 100644 packages/chord/test/next/examples/44-http-wraps/plugins/audit/server.ts create mode 100644 packages/chord/test/next/examples/44-http-wraps/plugins/notes/package.json create mode 100644 packages/chord/test/next/examples/44-http-wraps/plugins/notes/server.ts create mode 100644 packages/chord/test/next/examples/45-plugin-socket/main.ts create mode 100644 packages/chord/test/next/examples/45-plugin-socket/plugins/echo/package.json create mode 100644 packages/chord/test/next/examples/45-plugin-socket/plugins/echo/server.ts create mode 100644 packages/chord/test/next/examples/46-allowed-hosts/main.ts create mode 100644 packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/package.json create mode 100644 packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/server.ts create mode 100644 packages/chord/test/next/examples/47-http-own-server/main.ts create mode 100644 packages/chord/test/next/examples/47-http-own-server/plugins/notes/package.json create mode 100644 packages/chord/test/next/examples/47-http-own-server/plugins/notes/server.ts create mode 100644 packages/chord/test/next/examples/48-http-baseline/main.ts create mode 100644 packages/chord/test/next/examples/48-http-baseline/plugins/notes/package.json create mode 100644 packages/chord/test/next/examples/48-http-baseline/plugins/notes/server.ts diff --git a/package-lock.json b/package-lock.json index f2129fd85..02e687f30 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5568,11 +5568,15 @@ "node": ">=22.19.0" }, "peerDependencies": { - "esbuild": "0.28.2" + "esbuild": "0.28.2", + "ws": "8.21.0" }, "peerDependenciesMeta": { "esbuild": { "optional": true + }, + "ws": { + "optional": true } } }, diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index ceb4cddd1..326d4b8de 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -17,6 +17,7 @@ - Chord v2 has a package export again, `@earendil-works/chord/next` (experimental): every layer's API for apps (contracts, data, documents, hosts, jobs, links, state, the store), and, marked internal, the store's surface for adapters (`StoreOwnerTables`, `OwnerTable`, the documents engine's reads and writes, the commit step, `applyBatch`). v2's store gains one id allocator for every id in it, claims per table and per document namespace, Durable's Storage semantics for a refused commit (it rejects that commit only; an unknown outcome stops the store), document histories `none`, `latest` and `rewindable` with reads at a point, archives carrying document revisions, and batch apply; v2 objects gain `self.follow` (another runtime's synchronous observers as sources) and persisted jobs (`persist: { resume }`, a stored `abort(id, ctx)`, deferred state changes). - Chord v2 (`@earendil-works/chord/next`) gains registries, writables, wake signals and root scopes (slice 8): `defineRegistry(name, { version })` tokens shared by plugins, served by the defining plugin's facet with `host.serve(R, { compose, accepts? })`; contributions from facets (`defineFacet((host, self) => self.contribute(R, value))`) and host-lifetime objects (`StoreSelf.contribute`), plain values or Chord watchables, in a table sorted by plugin name and composed lazily (once at the end of `load`'s setups, then when read or followed, in the batch that changed the table); readers through `self.registry(R)` (a declared dependency: an object activates once the registry has a value, and fails `transient` with the new reason `unavailable/undeclared` when nobody serves it) and `host.registry(R, ctx)`, a `RegistryValue` with `observe(observer, ctx)`; older contributions and readers adapted by `accepts`; the helpers `keyedEntries` and `orderedEntries`; the `onError` kinds `registry_conflict`, `registry_failed`, `contribution_failed` and `contribution_refused`. `self.writable(initial)` is an in-process value an attempt sets (identity compared); native sources (writables, registry values, `ref.state`) are derive sources. `self.retryOn(...sources)` retries a failed object when a registry value changes. `host.root(fn, context, { principal? })` runs code outside objects with a handler's `ctx`, sealed when `fn` settles, `context` aborts or the host closes. - Chord v2 gains plugins (slice 7c): `src/next-node/plugins` (Node, to be exported as `@earendil-works/chord/next/plugins`) finds plugin directories, builds each facet with esbuild (now an optional peer dependency of `@earendil-works/chord`, imported only there), evaluates it with `vm.Script`, maps plugin stack traces (`mapStack`), installs plugin dependencies, watches plugin and base code, and rolls reloads out across hosts. `host.load` and `host.reload` work per plugin: a plugin whose setup, factory or declarations fail is left out at `load` (`{ failed }`) or kept on its old code at `reload` (`failed` in the report, `reload(facets, ctx, { failed })` for build and evaluation failures), with a closure rule keeping a consumer old when its provider was kept old; an abort after a reload's commit rejects with `details.committed`. +- Chord v2 gains HTTP and workers (slice 10): `src/next-node/http` (to be exported as `@earendil-works/chord/next/http`) builds a Web-standard `fetch` handler from the routes, wraps, sockets, hosts, relays and CSP sources plugins contribute to the registry `Http`, with authenticators from the registry `Authenticators`, responses as descriptions (`json`, `html`, `bytes` with ranges, `redirect`, `error`; `fileResponse`), and a security baseline on by default (provenance by the socket and `Host`, foreign requests anonymous, `Origin` checks, default headers with a per-response CSP nonce, signed cookies, safe downloads); Chord's own Node listener as a host-lifetime object, `nodeRequest` for apps mounting it on their own Node server, the links endpoint and plugin sockets with a heartbeat, closed at once when their contribution leaves. `ws` is an optional peer dependency, imported only there. `src/next-node/workers` (`createWorkers`, `workerMain`) runs a forked worker per address of a host kind, started on demand, stopped when idle, never restarted by itself (a failure stays until `retry` or a rollout), its plugins built by the server and shipped. The plugins component defines one definition per facet kind (`kinds: { name: { platform, hostPackages } }`, modules given at `plugins.load(host, { kind, core, modules })`), splits loading into `createPluginRuntime()` (no esbuild), offers `scripts(kind)`, rolls out with one result per host (`RolloutStep` returns `{ hosts }`), and resolves cross-plugin imports by plugin name (`@app/notes/contract`). Kernel: `host.route` returns a `RouteHandle` (`retry`, `suspend`) and a route has no backoff timer (a failed `resolve` fails its holders `waiting`); `host.root(fn, ctx, { principal: undefined })` runs anonymous; a link reads `origin` only on parent and route links (a protocol error on a child link) and drops a per-origin record once it is idle. - Chord v2, experimental and not exported yet: the next object runtime in `src/next` (contracts, hosts and objects, documents, derives, links and relays, routines and jobs, the store), with its tests and runnable examples in `test/next` and its design in `docs/next` (`design.md`, the decision log `decisions.md`, the slice plan `slices.md`). Its API changes with every slice until it gets an entry point. - `NativeWatchable`: an in-process value that changes and can be watched (`value`, `watch(context)` returning a `NativeWatchHandle`), the in-process counterpart of `Watchable`. - `WatchHandle` and `WatchEnd`, the one watch API, moved from Durable: a serialized observation with `value`, `start(listener)`, `stop()`, and `closed`; the listener gets a `WatchChange` (`{ value, ops, retired }`) and the context. `WatchEnd` reasons are `stopped`, `cancelled`, `ended`, `listener_error`, and `failed`. diff --git a/packages/chord/docs/next/benchmarks.md b/packages/chord/docs/next/benchmarks.md index d425f4154..ea2de1efa 100644 --- a/packages/chord/docs/next/benchmarks.md +++ b/packages/chord/docs/next/benchmarks.md @@ -914,3 +914,46 @@ derive commit 10.34-10.36 → 10.43-10.49 µs (14,899 → 14,901 B), activation 0.026-0.030 → 0.028-0.033 ms (pinned 0.030-0.035 → 0.030-0.032 ms); retained per hold equal (one binary hold 176.2 → 170.1 KiB). `reload.ts`: swap of 50 / 1,000 objects 1.44-1.73 / 30.83-32.71 → 1.52-1.53 / 30.55-33.25 ms, restart 1.01-1.05 / 15.54-16.87 → 0.95-0.99 / 15.90-16.02 ms until the reload resolves. + +## Slice 10: HTTP and workers (2026-10-31) + +`test/next/bench/http.ts` and `test/next/bench/workers.ts`, Node 24.4.1, CPU 6 (the forked workers inherit the pin). + +| Measurement | Result | +| --- | --- | +| keep-alive round trip, a JSON body: plain Node handler / a route (same process as the client) | build: 42.0-42.5 / 46.4-47.0 µs: +4.2-4.9 µs (9.9-11.7%); fix round 1: 42.5-42.9 / 62.8-64.4 µs: +20.3-21.5 µs | +| the same after 30,000 warm-up requests (fix round 1) | 30.9-31.3 / 43.4-44.0 µs: +12.1-12.9 µs; with an authenticator reading `request.headers`: +15.9 µs | +| the same, server and client in two processes (CPU 6 and 8; build) | 36.4-37.0 / 41.7-42.4 µs: +5 µs | +| 100 MiB file: `pipeline` of a read stream / `fileResponse` | build (64 KiB reads): 602-618 / 487-565 MiB/s; fix round 1 (256 KiB, pulled): 477-751 / 560-611 MiB/s | +| a worker's cold start with one shipped script: fork to `started` / to `ready` (median of 7) | 531-532 / 538-539 ms | +| the server's heap per idle worker (20 workers, one reference each) | 49.5 KiB | + +A request's overhead was 16 µs at first: `host.root` cost about 7 µs of it per request (a linked abort signal, a +`Promise.race` and an `awaitWithContext` per scope). Root scopes now take the caller's signal with one listener and +settle one promise (`root.ts`; behaviour and tests unchanged): 4.2-4.9 µs, about the bar of 10% of a round trip. The +rest is the request's scope (its abort controller, `CallContext`), admission and the response's header merge. The cold +start is mostly Node's startup stripping the types of the source tree it imports; the start timer (60 s) is far from +it. + +**Fix round 1** (the `fetch` core, the security baseline, Chord's listener as a native object). A route's overhead in +the standard run grew from +4.5 µs to +20-21 µs (+41 µs before the Node fast path: the listener reads Node's parsed +headers and makes a Web `Request` only when something reads `request.headers` or the body or a relay verifies; whole +bodies are written as strings, in one write with the head). Warmed, +12-13 µs; a CPU profile of the warmed route path +(per request, `test/next/bench/http.ts`'s route): the listener's request function 13.6 µs inclusive against the plain +handler's 8.9 µs, of which writing the answer 8.3 µs (the plain handler's `writeHead` and `end`: 9.1 µs); `serve` 4.2 +µs (the root scope 2.1, the URL parse 0.9, default headers 0.6, admission and routing 1.1, the answer 0.7); the root +scope's seal in a later microtask 0.9 µs. The baseline's extra headers cost 0.4 µs (a plain handler sending them). +The rest, about 5 µs, sits in no function of the component: Node's tick and microtask machinery (33 µs inclusive +against 18 µs for the plain handler). Opus's two trims are in (`lookup` decodes only segments with `%`; one abort +controller per request: the Node side's own is gone). Kernel rows, HEAD `bc9a20f84` → fix round 1, CPU 6, alternating: +binary 1 MiB call median 0.182-0.220 → 0.196 ms; reconnect of 1 / 100 references 1.02-1.10 / 79.1-80.3 → 1.04-1.05 / +81.3-81.5 ms; watched `writable.set` 0.836-0.868 → 0.861-0.863 µs; watched derive commit 10.42-10.58 → 10.36-10.60 µs; +activation churn 58.9-60.5 → 59.1-60.8 µs. + +**Kernel hot paths** (routes without a timer, `RouteHandle`, `Path` events, root scopes), HEAD `bc9a20f84` → build at +paths of equal length, CPU 6, alternating runs. `calls.ts`: local call 2.70-2.83 → 2.70-2.83 µs (6,660 B both), +watched derive commit 10.36-10.60 → 10.34-10.79 µs, activation churn 59.43-61.30 → 59.48-60.72 µs. `links.ts`: commit +to frame and encode/decode alike; reconnect of 1 / 100 references 0.94-1.15 / 80.8-84.0 → 0.94-1.15 / 80.6-86.5 ms; +1,000 parked calls 24.4-25.2 → 24.3-24.6 ms. `reload.ts`: swap of 50 / 1,000 objects 1.44-1.55 / 30.78-31.42 → +1.32-1.56 / 31.20-31.83 ms, restart 0.96-0.98 / 15.75-16.34 → 0.95-1.10 / 15.91-15.94 ms until the reload resolves. +`registries.ts` alike (one of 100 contributions set 1.47-2.45 → 1.48-3.41 µs, run to run). diff --git a/packages/chord/package.json b/packages/chord/package.json index f2be5a564..908005452 100644 --- a/packages/chord/package.json +++ b/packages/chord/package.json @@ -94,11 +94,15 @@ "typebox": "1.3.27" }, "peerDependencies": { - "esbuild": "0.28.2" + "esbuild": "0.28.2", + "ws": "8.21.0" }, "peerDependenciesMeta": { "esbuild": { "optional": true + }, + "ws": { + "optional": true } }, "devDependencies": { diff --git a/packages/chord/src/next-node/http/README.md b/packages/chord/src/next-node/http/README.md new file mode 100644 index 000000000..266ba00d0 --- /dev/null +++ b/packages/chord/src/next-node/http/README.md @@ -0,0 +1,166 @@ +# http + +The HTTP component (slice 10 §2, design §23; decisions "The HTTP core is a fetch handler", "The HTTP server is a native +object", "The HTTP component's security baseline", "Slice 10 fix round, allowed hosts only through contributions", +"Settings are arguments, extensions are registries", "A CSP nonce per response"): the server's HTTP and WebSocket +entry. Plugins contribute routes, wraps, sockets, allowed hosts, framers, relays and +CSP sources to the registry `Http`; routes return response descriptions. The core is one Web-standard handler, +`fetch(Request) → Response`; Chord's own Node server (the listener) is the easy path, an app with a server of its own +mounts the handler. Node-only; `ws` is an optional peer dependency of `@earendil-works/chord`, imported only here. +Exported as `@earendil-works/chord/next/http` once v2 has its entry point. + +## For apps (`index.ts`) + +```ts +// the app's settings are arguments, fixed for the component's life (changing one is a restart) +const http = createHttp(server, { + listen: { host: "127.0.0.1", port: 8788 }, // Chord's own listener; port 0 picks one; none: no listener + links: { path: "/chord", options: (request, principal) => ({ limits, maxPayload }) }, // or a value + closeTimeout: 30_000, + onError: (error) => log(error), // HttpError +}, ctx); +// the app's core facet: who someone is (a registry: plugins may add authenticators too) +const app = defineFacet((_host, self) => { + self.contribute(Authenticators, (request, ctx) => principalFromCookie(readCookie(request, "session", secret))); +}); +const core = [{ name: "chord.http", facet: http.facet }, { name: "@app/core", facet: app }]; +await plugins.load(server, { kind: "server", core, modules }, ctx); +const { port } = await http.listening(ctx); +… +await http.close(ctx); // first: drains running requests (host.close would seal their scopes at once) +await server.close(ctx); +``` + +```ts +// a plugin's server facet +self.contribute(Http, { + routes: { + "GET /notes/:id": async (request, ctx) => ({ kind: "json", value: … }), + "GET /": { page: true, handler: async (request) => ({ kind: "html", body: `` }; + }, + }, + framers: ["https://shell.example.com"], + csp: { "img-src": ["https://cdn.example.com"] }, + }), + }); + const page = await send(server.port, "GET", "/page"); + expect(page.headers).toMatchObject({ + "cache-control": "no-store", + "x-content-type-options": "nosniff", + "referrer-policy": "no-referrer", + }); + expect(page.headers["x-frame-options"]).toBeUndefined(); + expect(page.headers["content-security-policy"]).toBe( + `default-src 'none'; script-src 'self' 'nonce-${nonce}'; connect-src 'self' ws://127.0.0.1:${server.port}; base-uri 'none'; form-action 'self'; frame-ancestors https://shell.example.com; img-src https://cdn.example.com`, + ); + // Fable's recheck: a nonce read from one page must not run a script injected into another. + const first = nonce; + await send(server.port, "GET", "/page"); + expect(nonce).not.toBe(first); + const missing = await send(server.port, "GET", "/nope"); + expect([missing.status, missing.headers["x-content-type-options"]]).toEqual([404, "nosniff"]); + expect(missing.headers["content-security-policy"]).toMatch(/'nonce-[A-Za-z0-9+/]{22}=='/); + expect(missing.headers["content-security-policy"]).not.toContain(nonce); + await server.close(); + const plain = await served({}); + expect((await send(plain.port, "GET", "/")).headers["x-frame-options"]).toBe("DENY"); + await plain.close(); + }); + + it("lets a relay's verified request through with its framer", async () => { + const server = await served({ + "@app/who": whoami, + "@app/relay": contributing({ + relays: [ + { + name: "radius", + verify: (request) => + request.headers.get("x-relay") === "secret" + ? { origin: "https://me.relay.example", framer: "https://relay.example" } + : undefined, + }, + ], + }), + }); + const forwarded = { "x-forwarded-for": "203.0.113.9" }; + expect((await send(server.port, "GET", "/whoami", { headers: forwarded })).status).toBe(403); + const relayed = await send(server.port, "GET", "/whoami", { headers: { ...forwarded, "x-relay": "secret" } }); + expect(relayed.status).toBe(200); + expect(relayed.headers["content-security-policy"]).toContain("frame-ancestors https://relay.example"); + await server.close(); + }); +}); + +describe("routes and wraps failing", () => { + it("answers a throwing route 500 without its message, a throwing wrap 500 naming its plugin, an HttpFailure its status", async () => { + const server = await served({ + "@app/a": contributing({ + routes: { + "GET /throws": async () => { + throw new Error("secret"); + }, + "GET /failure": async () => { + throw new HttpFailure(422, "no"); + }, + }, + }), + "@app/b": contributing({ + routes: { "GET /wrapped": async () => ({ kind: "json", value: 1 }) }, + wraps: [ + { + name: "broken", + phase: "headers", + wrap: async () => { + throw new Error("wrap"); + }, + }, + ], + }), + }); + const thrown = await send(server.port, "GET", "/throws"); + expect([thrown.status, thrown.body.toString()]).toEqual([500, "Internal Server Error"]); + expect((await send(server.port, "GET", "/failure")).status).toBe(422); + expect((await send(server.port, "GET", "/wrapped")).status).toBe(500); + expect(server.errors).toMatchObject([ + { kind: "route_failed", pattern: "GET /throws", plugin: "@app/a", error: { message: "secret" } }, + { kind: "wrap_failed", name: "broken", plugin: "@app/b", error: { message: "wrap" } }, + ]); + await server.close(); + }); + + it("runs wraps outermost first around the route, in its request's scope", async () => { + const order: string[] = []; + const server = await served({ + "@app/a": contributing({ + routes: { + "GET /r": async (_request, ctx) => { + order.push(`route as ${ctx.principal?.kind ?? "anonymous"}`); + return { kind: "json", value: 1 }; + }, + }, + wraps: [ + { + name: "inner", + phase: "transform", + wrap: async (request, next) => { + order.push("transform"); + return next(request); + }, + }, + { + name: "outer", + phase: "observe", + wrap: async (request, next) => { + order.push("observe"); + const response = await next(request); + return { ...response, headers: { "X-Observed": "1" } }; + }, + }, + ], + }), + }); + const answer = await send(server.port, "GET", "/r", { headers: { "x-principal": "owner" } }); + expect([order, answer.headers["x-observed"]]).toEqual([["observe", "transform", "route as owner"], "1"]); + await server.close(); + }); +}); + +describe("reloads and close", () => { + it("a reload swapping a route leaves a running request on the old router; the next uses the new one", async () => { + const gate = Promise.withResolvers(); + const old = contributing({ + routes: { + "GET /v": async () => { + await gate.promise; + return { kind: "html", body: "old" }; + }, + }, + }); + const server = await served({ "@app/v": old }); + const running = send(server.port, "GET", "/v"); + await new Promise((resolve) => setTimeout(resolve, 50)); + const next = contributing({ routes: { "GET /v": async () => ({ kind: "html", body: "new" }) } }); + await server.host.reload(server.facets({ "@app/v": next }), context); + expect((await send(server.port, "GET", "/v")).body.toString()).toBe("new"); + gate.resolve(); + expect((await running).body.toString()).toBe("old"); + await server.close(); + }); + + it("close refuses new requests (503), waits for running ones up to closeTimeout, then seals them (503)", async () => { + let sealed: AbortSignal | undefined; + const server = await served( + { + "@app/slow": contributing({ + routes: { + "GET /quick": async () => { + await new Promise((resolve) => setTimeout(resolve, 30)); + return { kind: "html", body: "done" }; + }, + "GET /stuck": async (_request, ctx: Ctx) => { + sealed = ctx.abortSignal; + return new Promise(() => {}); + }, + }, + }), + }, + { config: { listen: { host: "127.0.0.1", port: 0 }, closeTimeout: 100 } }, + ); + const quick = send(server.port, "GET", "/quick"); + const stuck = send(server.port, "GET", "/stuck"); + await new Promise((resolve) => setTimeout(resolve, 10)); + const closing = server.http.close(context); + expect((await server.http.fetch(loopback("/quick"), LOOPBACK)).status).toBe(503); + expect((await quick).body.toString()).toBe("done"); + await closing; + expect(sealed?.aborted).toBe(true); + await server.host.close(context); + expect((await stuck).status).toBe(503); + await expect(send(server.port, "GET", "/quick")).rejects.toThrow(); + }); + + // Astra's build review: a request authenticating when close began ran its route afterwards. + it("never runs a route for a request still authenticating when close sealed it", async () => { + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + let calls = 0; + const server = await served( + { + "@app/r": contributing({ + routes: { + "GET /": async () => { + calls++; + return { kind: "json", value: 1 }; + }, + }, + }), + }, + { + config: { listen: { host: "127.0.0.1", port: 0 }, closeTimeout: 20 }, + authenticate: async () => { + entered.resolve(); + await gate.promise; + return { kind: "owner" }; + }, + }, + ); + const answer = server.http.fetch(loopback("/"), LOOPBACK); + await entered.promise; + await server.http.close(context); + gate.resolve(); + expect([(await answer).status, calls]).toEqual([503, 0]); + await server.close(); + }); +}); + +describe("mounting", () => { + it("without a connection a request is never local", async () => { + const server = await served({ "@app/who": whoami }, { config: { listen: undefined } }); + expect((await server.http.fetch(loopback("/whoami"))).status).toBe(403); + await server.close(); + }); +}); + +describe("cancellation", () => { + // Sol's second recheck C: after cancellation the chain still started the next authenticator. + it("starts no further authenticator once close sealed the request", async () => { + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + let later = 0; + const server = await served( + { + "@app/r": contributing({ routes: { "GET /": async () => ({ kind: "html", body: "ok" }) } }), + "@app/z-later": defineFacet((_host, self) => + self.contribute(Authenticators, () => { + later++; + return { kind: "owner" }; + }), + ), + }, + { + config: { listen: undefined, closeTimeout: 10 }, + authenticate: async () => { + entered.resolve(); + await gate.promise; + return undefined; + }, + }, + ); + const answer = server.http.fetch(loopback("/"), LOOPBACK); + await entered.promise; + await server.http.close(context); + expect((await answer).status).toBe(503); + gate.resolve(); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(later).toBe(0); + await server.close(); + }); + + it("asks no authenticator for a request aborted before it arrived", async () => { + let calls = 0; + const server = await served( + { "@app/r": contributing({ routes: { "GET /": async () => ({ kind: "html", body: "ok" }) } }) }, + { + config: { listen: undefined }, + authenticate: () => { + calls++; + return undefined; + }, + }, + ); + await server.http.fetch(loopback("/", { signal: AbortSignal.abort() }), LOOPBACK); + expect(calls).toBe(0); + await server.close(); + }); + + // Sol's third recheck: a request aborted while its route returned the `bytes` description got a stream born sealed, + // which never errored; the request never finished. + it.each(["inside the route", "after the route returned"] as const)( + "ends a request whose bytes Response was born sealed (aborted %s)", + async (when) => { + const abort = new AbortController(); + const entered = Promise.withResolvers(); + let reads = 0; + let bornSealed: boolean | undefined; + const server = await served( + { + "@app/b": contributing({ + routes: { + "GET /": async (_request, ctx) => { + entered.resolve(); + if (when === "inside the route") abort.abort(new Error("client gone")); + bornSealed = ctx.abortSignal?.aborted; + return { + kind: "bytes", + mime: "application/octet-stream", + read: async () => { + reads++; + return new Uint8Array([1]); + }, + }; + }, + }, + }), + }, + { config: { listen: undefined, closeTimeout: 1_000 } }, + ); + const answer = server.http.fetch(loopback("/", { signal: abort.signal }), LOOPBACK); + await entered.promise; + if (when === "after the route returned") abort.abort(new Error("client gone")); + await answer; + await new Promise((resolve) => setImmediate(resolve)); + const started = Date.now(); + await server.http.close(context); + expect([Date.now() - started < 500, reads, bornSealed]).toEqual([true, 0, when === "inside the route"]); + await server.close(); + }, + ); + + // Sol's second recheck B: an aborted `Response` nobody touched again kept its request: close waited its bound. + it("ends a request whose bytes Response was aborted and dropped without another read", async () => { + const server = await served( + { + "@app/b": contributing({ + routes: { + "GET /": async () => ({ + kind: "bytes", + mime: "application/octet-stream", + read: async () => new Uint8Array([1]), + }), + }, + }), + }, + { config: { listen: undefined, closeTimeout: 1_000 } }, + ); + const abort = new AbortController(); + await server.http.fetch(loopback("/", { signal: abort.signal }), LOOPBACK); + abort.abort(new Error("client gone")); + await new Promise((resolve) => setImmediate(resolve)); + const started = Date.now(); + await server.http.close(context); + expect(Date.now() - started).toBeLessThan(500); + await server.close(); + }); + + // Astra's recheck R4, Sol's N1: a close before the first resolution started a stopped watch; its throw ended the process. + it("a close before the publication leaves nothing running: the waiting request answers 503", async () => { + const server = await served({}, { load: false, config: { listen: undefined } }); + const waiting = server.http.fetch(loopback("/x"), LOOPBACK); + await server.http.close(withAbortSignal(AbortSignal.abort(), context)); + await server.load(); + expect((await waiting).status).toBe(503); + await new Promise((resolve) => setTimeout(resolve, 50)); + await server.close(); + }); + + it("a close while the first request resolves the registries starts no follow", async () => { + const server = await served({ "@app/who": whoami }, { config: { listen: undefined, closeTimeout: 10 } }); + const pending = server.http.fetch(loopback("/whoami"), LOOPBACK); + await server.http.close(withAbortSignal(AbortSignal.timeout(20), context)); + expect([200, 503]).toContain((await pending).status); + await new Promise((resolve) => setTimeout(resolve, 50)); + await server.close(); + }); + + // Sol's recheck N9. + it("a close whose context is aborted already does not wait", async () => { + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + const server = await served( + { + "@app/wait": contributing({ + routes: { + "GET /wait": async () => { + entered.resolve(); + await gate.promise; + return { kind: "html", body: "done" }; + }, + }, + }), + }, + { config: { listen: undefined, closeTimeout: 1000 } }, + ); + const answer = server.http.fetch(loopback("/wait"), LOOPBACK); + await entered.promise; + const started = Date.now(); + await server.http.close(withAbortSignal(AbortSignal.abort(), context)); + expect(Date.now() - started).toBeLessThan(500); + gate.resolve(); + await answer; + await server.close(); + }); + + // Sol's recheck N5: an authenticator ignoring its signal kept fetch pending after close returned. + it("close settles a request whose authenticator ignores its signal (503)", async () => { + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + const server = await served( + { "@app/r": contributing({ routes: { "GET /": async () => ({ kind: "html", body: "ok" }) } }) }, + { + config: { listen: undefined, closeTimeout: 10 }, + authenticate: async () => { + entered.resolve(); + await gate.promise; + return { kind: "owner" }; + }, + }, + ); + const answer = server.http.fetch(loopback("/"), LOOPBACK); + await entered.promise; + await server.http.close(context); + const settled = await Promise.race([answer.then((response) => response.status), sleepThen(30, "pending")]); + expect(settled).toBe(503); + gate.resolve(); + await server.close(); + }); + + // Astra's recheck R2, Sol's N3: a held `Response` went on calling `read` after its scope was sealed. + it.each(["the request aborted", "the component closed"] as const)("reads no bytes after %s", async (kind) => { + let reads = 0; + let readAfterSeal = false; + const abort = new AbortController(); + const server = await served( + { + "@app/bytes": contributing({ + routes: { + "GET /stream": async () => ({ + kind: "bytes", + mime: "application/octet-stream", + read: async (_offset, _length, ctx) => { + reads++; + readAfterSeal ||= ctx.abortSignal?.aborted ?? false; + return new Uint8Array([reads]); + }, + }), + }, + }), + }, + { config: { listen: undefined, closeTimeout: 10 } }, + ); + const response = await server.http.fetch(loopback("/stream", { signal: abort.signal }), LOOPBACK); + const reader = response.body!.getReader(); + await reader.read(); + if (kind === "the request aborted") abort.abort(new Error("client gone")); + else await server.http.close(context); + await new Promise((resolve) => setImmediate(resolve)); + await expect(reader.read()).rejects.toThrow(); + expect({ reads, readAfterSeal }).toEqual({ reads: 1, readAfterSeal: false }); + await server.close(); + }); + + // Sol's recheck N4. + it("a body that errors after valid JSON fails the request (400); the route's side effect never runs", async () => { + let writes = 0; + const server = await served( + { + "@app/body": contributing({ + routes: { + "POST /body": async (request) => { + const value = await request.json(); + writes++; + return { kind: "json", value }; + }, + }, + }), + }, + { config: { listen: undefined } }, + ); + let pulls = 0; + const body = new ReadableStream({ + pull(controller) { + if (pulls++ === 0) controller.enqueue(new TextEncoder().encode("{}")); + else controller.error(new Error("incoming body checksum failed")); + }, + }); + const init = { method: "POST", body, duplex: "half" as const }; + const response = await server.http.fetch(loopback("/body", init), LOOPBACK); + expect([response.status, writes]).toEqual([400, 0]); + await server.close(); + }); +}); + +describe("answers", () => { + // Sol's recheck N8: `Response` threw on a body for these statuses. + it.each([204, 205, 304])("answers %s without a body", async (status) => { + const server = await served( + { "@app/s": contributing({ routes: { "GET /": async () => ({ kind: "json", value: null, status }) } }) }, + { config: { listen: undefined } }, + ); + const response = await server.http.fetch(loopback("/"), LOOPBACK); + expect([response.status, response.body, response.headers.has("content-length")]).toEqual([status, null, false]); + await server.close(); + }); + + // Fable's recheck N3: the kernel's `forbidden` reached the client as a reported 500. + it("answers a forbidden escaping a route 403, unreported", async () => { + const server = await served( + { + "@app/r": contributing({ + routes: { + "GET /": async () => { + throw new ChordError({ code: "forbidden", message: "no" }); + }, + }, + }), + }, + { config: { listen: undefined } }, + ); + expect((await server.http.fetch(loopback("/"), LOOPBACK)).status).toBe(403); + expect(server.errors).toEqual([]); + await server.close(); + }); + + // Opus's recheck R2: one value per header name allowed one cookie per response. + it("sends several Set-Cookie headers", async () => { + const server = await served({ + "@app/r": contributing({ + routes: { "GET /": async () => ({ kind: "json", value: 1, headers: { "Set-Cookie": ["a=1", "b=2"] } }) }, + }), + }); + expect((await send(server.port, "GET", "/")).headers["set-cookie"]).toEqual(["a=1", "b=2"]); + const fetched = await server.http.fetch( + new Request(`http://127.0.0.1:${server.port}/`, { headers: { host: `127.0.0.1:${server.port}` } }), + { localAddress: "127.0.0.1", localPort: server.port }, + ); + expect(fetched.headers.getSetCookie()).toEqual(["a=1", "b=2"]); + await server.close(); + }); +}); + +describe("GC", () => { + // Sol's second recheck B, her GC probe: 8 of 8 such requests kept their scopes. + it("collects the scopes of aborted bytes Responses dropped without another read", async () => { + const refs: WeakRef[] = []; + const server = await served( + { + "@app/b": contributing({ + routes: { + "GET /": async (_request, ctx) => { + refs.push(new WeakRef(ctx)); + return { kind: "bytes", mime: "application/octet-stream", read: async () => new Uint8Array([1]) }; + }, + }, + }), + }, + { config: { listen: undefined } }, + ); + for (let i = 0; i < 8; i++) { + const abort = new AbortController(); + await server.http.fetch(loopback("/", { signal: abort.signal }), LOOPBACK); + abort.abort(new Error("client gone")); + } + await expectCollected(...refs); + await server.close(); + }); + + it("collects the scopes of bytes Responses born sealed (aborted while the route returned)", async () => { + const refs: WeakRef[] = []; + const aborts: AbortController[] = []; + const server = await served( + { + "@app/b": contributing({ + routes: { + "GET /": async (_request, ctx) => { + refs.push(new WeakRef(ctx)); + aborts.at(-1)?.abort(new Error("client gone")); + return { kind: "bytes", mime: "application/octet-stream", read: async () => new Uint8Array([1]) }; + }, + }, + }), + }, + { config: { listen: undefined } }, + ); + // In a function of its own: the test's frame keeps no last request. + const fetchAborted = async (): Promise => { + const abort = new AbortController(); + aborts.push(abort); + await server.http.fetch(loopback("/", { signal: abort.signal }), LOOPBACK); + }; + for (let i = 0; i < 8; i++) await fetchAborted(); + aborts.length = 0; + await expectCollected(...refs); + await server.close(); + }); + + it("collects requests' scopes and requests once answered", async () => { + const refs: WeakRef[] = []; + const server = await served({ + "@app/a": contributing({ + routes: { + "GET /r": async (request, ctx) => { + refs.push(new WeakRef(ctx), new WeakRef(request)); + return { kind: "json", value: 1 }; + }, + }, + }), + }); + for (let i = 0; i < 5; i++) await send(server.port, "GET", "/r"); + await expectCollected(...refs); + await server.close(); + }); +}); diff --git a/packages/chord/test/next-node/http/node.test.ts b/packages/chord/test/next-node/http/node.test.ts new file mode 100644 index 000000000..8349bd751 --- /dev/null +++ b/packages/chord/test/next-node/http/node.test.ts @@ -0,0 +1,215 @@ +// Tests of Chord's own Node server (the listener's native): request targets (origin-form only; a malformed or +// double-slash path never crashes or reroutes), raw `Host` headers, Web requests made only when read, the request's scope lasting until its response is +// flushed (and `close` waiting for it), and a client's disconnect aborting the scope. + +import { once } from "node:events"; +import { createServer, request as httpRequest, type IncomingMessage } from "node:http"; +import { connect } from "node:net"; +import { describe, expect, it } from "vitest"; +import { nodeRequest } from "../../../src/next-node/http/index.ts"; +import { context, contributing, served } from "./support.ts"; + +/** What `port` answers a raw request `text` with: its status line and body. */ +async function raw(port: number, text: string): Promise<{ readonly status: string; readonly body: string }> { + const socket = connect(port, "127.0.0.1"); + let out = ""; + socket.on("data", (chunk) => { + out += chunk.toString("latin1"); + }); + socket.on("error", () => {}); + await once(socket, "connect"); + socket.write(text); + await once(socket, "close"); + const [head = "", body = ""] = out.split("\r\n\r\n"); + return { status: head.split("\r\n")[0] ?? "", body }; +} + +/** A route answering its path and params, under `GET /who` and `GET /n/:name`. */ +const echo = contributing({ + routes: { + "GET /who": async (request) => ({ kind: "json", value: request.path }), + "GET /n/:name": async (request) => ({ kind: "json", value: request.params }), + }, +}); + +describe("targets", () => { + // Astra's build review, Fable's security audit C1: `GET //[` threw out of the request handler and ended the process. + it("answers a malformed or double-slash path without crashing or rerouting it (Opus's audit H5)", async () => { + const server = await served({ "@app/echo": echo }); + const host = `Host: 127.0.0.1:${server.port}`; + const get = (target: string) => + raw(server.port, `GET ${target} HTTP/1.1\r\n${host}\r\nConnection: close\r\n\r\n`); + expect((await get("//[")).status).toBe("HTTP/1.1 404 Not Found"); + expect((await get("//evil.example/who")).status).toBe("HTTP/1.1 404 Not Found"); + expect(await get("/a/../who")).toEqual({ status: "HTTP/1.1 200 OK", body: '"/who"' }); + // The URL parser resolves `%2e%2e` as a dot segment: `/n/%2e%2e` is `/`. + expect((await get("/n/%2e%2e")).status).toBe("HTTP/1.1 404 Not Found"); + expect((await get("/n/a%5Cb")).status).toBe("HTTP/1.1 400 Bad Request"); + expect((await get("/n/..%2F..%2Fsecret")).status).toBe("HTTP/1.1 400 Bad Request"); + expect((await get("/who")).status).toBe("HTTP/1.1 200 OK"); + await server.close(); + }); + + it("refuses an absolute-form target, two Host headers and none (403)", async () => { + const server = await served({ "@app/echo": echo }); + const host = `Host: 127.0.0.1:${server.port}`; + const answers = await Promise.all([ + raw(server.port, `GET http://evil.example/who HTTP/1.1\r\n${host}\r\nConnection: close\r\n\r\n`), + raw(server.port, `GET /who HTTP/1.1\r\n${host}\r\nHost: evil.example\r\nConnection: close\r\n\r\n`), + raw(server.port, "GET /who HTTP/1.0\r\n\r\n"), + ]); + expect(answers.map(({ status }) => status)).toEqual([ + "HTTP/1.1 403 Forbidden", + "HTTP/1.1 403 Forbidden", + "HTTP/1.1 403 Forbidden", + ]); + await server.close(); + }); + + it("refuses a Host with userinfo (403) and a method a Web Request cannot carry (501)", async () => { + const server = await served({ "@app/echo": echo }); + const answers = await Promise.all([ + raw(server.port, `GET /who HTTP/1.1\r\nHost: attacker@127.0.0.1:${server.port}\r\nConnection: close\r\n\r\n`), + raw(server.port, `TRACE /who HTTP/1.1\r\nHost: 127.0.0.1:${server.port}\r\nConnection: close\r\n\r\n`), + ]); + // `TRACK` never reaches Chord: Node's parser refuses it (400); `CONNECT` goes to Node's `connect` event. + expect(answers.map(({ status }) => status)).toEqual(["HTTP/1.1 403 Forbidden", "HTTP/1.1 501 Not Implemented"]); + await server.close(); + }); +}); + +describe("mounting on an app's own Node server", () => { + // Sol's recheck N6: a hand-made `Request` threw unhandled on `Host: [` or an absolute-form target (the process + // ended) and kept only the first of two `Host`s. + it("converts with the listener's checks, never throwing: bad targets, userinfo, TRACE refused; duplicate Hosts kept", async () => { + const server = await served({ "@app/echo": echo }, { config: { listen: undefined } }); + const own = createServer(async (message, res) => { + const converted = nodeRequest(message, res); + if (converted.kind === "refused") return void res.writeHead(converted.status).end(); + const response = await server.http.fetch(converted.request, converted.connection); + for (const [name, value] of response.headers) res.appendHeader(name, value); + res.writeHead(response.status).end(Buffer.from(await response.arrayBuffer())); + }); + await new Promise((resolve) => own.listen(0, "127.0.0.1", resolve)); + const address = own.address(); + const port = typeof address === "object" && address !== null ? address.port : 0; + const host = `Host: 127.0.0.1:${port}`; + const get = (text: string) => raw(port, `${text}Connection: close\r\n\r\n`); + expect(await get(`GET /who HTTP/1.1\r\n${host}\r\n`)).toEqual({ status: "HTTP/1.1 200 OK", body: '"/who"' }); + expect((await get("GET /who HTTP/1.1\r\nHost: [\r\n")).status).toBe("HTTP/1.1 400 Bad Request"); + expect((await get(`GET http://evil.example/who HTTP/1.1\r\n${host}\r\n`)).status).toBe("HTTP/1.1 403 Forbidden"); + expect((await get(`GET /who HTTP/1.1\r\n${host}\r\nHost: evil.example\r\n`)).status).toBe( + "HTTP/1.1 403 Forbidden", + ); + // Sol's second recheck A: both threw out of the conversion; the app's callback ended the process. + expect((await get(`TRACE /who HTTP/1.1\r\n${host}\r\n`)).status).toBe("HTTP/1.1 501 Not Implemented"); + expect((await get(`GET /who HTTP/1.1\r\nHost: attacker@127.0.0.1:${port}\r\n`)).status).toBe( + "HTTP/1.1 403 Forbidden", + ); + expect((await get(`GET /who HTTP/1.1\r\n${host}\r\n`)).status).toBe("HTTP/1.1 200 OK"); + await new Promise((resolve) => own.close(resolve)); + await server.close(); + }); +}); + +describe("Web requests", () => { + it("makes a Web Request only when something reads the headers or the body (here: no authenticator does)", async () => { + const server = await served( + { + "@app/r": contributing({ + routes: { + "GET /plain": async (request) => ({ kind: "json", value: request.path }), + "GET /headers": async (request) => ({ kind: "json", value: request.headers.get("x-thing") }), + "POST /body": async (request) => ({ kind: "json", value: await request.json() }), + }, + }), + }, + { authenticate: () => undefined }, + ); + const original = globalThis.Request; + let made = 0; + globalThis.Request = class extends original { + constructor(...args: ConstructorParameters) { + super(...args); + made++; + } + }; + try { + const host = `Host: 127.0.0.1:${server.port}\r\nConnection: close`; + await raw(server.port, `GET /plain HTTP/1.1\r\n${host}\r\n\r\n`); + expect(made).toBe(0); + const read = await raw(server.port, `GET /headers HTTP/1.1\r\n${host}\r\nX-Thing: t\r\n\r\n`); + const posted = await raw(server.port, `POST /body HTTP/1.1\r\n${host}\r\nContent-Length: 3\r\n\r\n"a"`); + expect([read.body, posted.body, made]).toEqual(['"t"', '"a"', 2]); + } finally { + globalThis.Request = original; + } + await server.close(); + }); +}); + +describe("the request's scope", () => { + // Sol's build review: the scope sealed, and `close` stopped waiting, before a large response was flushed. + it("lasts until the response is flushed; close waits for it", async () => { + const size = 32 * 1024 * 1024; + let signal: AbortSignal | undefined; + const server = await served( + { + "@app/large": contributing({ + routes: { + "GET /large": async (_request, ctx) => { + signal = ctx.abortSignal; + return { kind: "html", body: "x".repeat(size) }; + }, + }, + }), + }, + { config: { listen: { host: "127.0.0.1", port: 0 }, closeTimeout: 2000 } }, + ); + const request = httpRequest({ host: "127.0.0.1", port: server.port, path: "/large", agent: false }); + const response = await new Promise((resolve) => { + request.on("response", (arrived: IncomingMessage) => { + arrived.pause(); + resolve(arrived); + }); + request.end(); + }); + let length = 0; + response.on("data", (chunk: Buffer) => { + length += chunk.length; + }); + response.pause(); + await new Promise((resolve) => setTimeout(resolve, 20)); + const sealedBeforeFlush = signal?.aborted; + const closing = server.http.close(context); + await new Promise((resolve) => setTimeout(resolve, 20)); + response.resume(); + await once(response, "end"); + await closing; + expect({ sealedBeforeFlush, length }).toEqual({ sealedBeforeFlush: false, length: size }); + await server.close(); + }); + + it("is aborted when the client disconnects", async () => { + const entered = Promise.withResolvers(); + const server = await served({ + "@app/slow": contributing({ + routes: { + "GET /slow": async (_request, ctx) => { + entered.resolve(ctx.abortSignal!); + return new Promise(() => {}); + }, + }, + }), + }); + const request = httpRequest({ host: "127.0.0.1", port: server.port, path: "/slow", agent: false }); + request.on("error", () => {}); + request.end(); + const signal = await entered.promise; + const aborted = new Promise((resolve) => signal.addEventListener("abort", resolve, { once: true })); + request.destroy(); + await aborted; + expect(signal.aborted).toBe(true); + await server.close(); + }); +}); diff --git a/packages/chord/test/next-node/http/request.test.ts b/packages/chord/test/next-node/http/request.test.ts new file mode 100644 index 000000000..a48a2429e --- /dev/null +++ b/packages/chord/test/next-node/http/request.test.ts @@ -0,0 +1,119 @@ +// Tests of requests (slice 10 §2, D4): the summary a route gets, `json` and `body` with their limits and statuses +// (400, 413, an `HttpFailure` escaping a route answering its status), and a body nobody read (drained or closed). + +import { Agent, request as httpRequest } from "node:http"; +import { describe, expect, it } from "vitest"; +import type { HttpRequest } from "../../../src/next-node/http/index.ts"; +import { contributing, send, served } from "./support.ts"; + +describe("the request a route gets", () => { + it("has its method, path, params, query, headers, origin, provenance and the response's nonce", async () => { + let seen: HttpRequest | undefined; + const server = await served({ + "@app/a": contributing({ + routes: { + "GET /notes/:id": async (request) => { + seen = request; + return { kind: "json", value: null }; + }, + }, + }), + }); + await send(server.port, "GET", "/notes/a%20b?x=1&x=2", { headers: { "X-Thing": "t" } }); + expect(seen).toMatchObject({ + method: "GET", + path: "/notes/a%20b", + params: { id: "a b" }, + origin: `http://127.0.0.1:${server.port}`, + provenance: { kind: "local" }, + }); + expect([seen?.query.getAll("x"), seen?.headers.get("x-thing")]).toEqual([["1", "2"], "t"]); + expect(seen?.nonce).toMatch(/^[A-Za-z0-9+/]{22}==$/); + await server.close(); + }); +}); + +describe("bodies", () => { + /** A server whose `POST /json` answers the JSON it read (limit 16), `POST /body` the bytes it read (limit 16). */ + const reading = () => + served({ + "@app/a": contributing({ + routes: { + "POST /json": async (request) => ({ kind: "json", value: await request.json(16) }), + "POST /body": async (request) => { + let bytes = 0; + for await (const chunk of request.body(16)) bytes += chunk.byteLength; + return { kind: "json", value: bytes }; + }, + }, + }), + }); + + it("parses JSON within its limit; malformed is 400, larger is 413 (declared or streamed)", async () => { + const server = await reading(); + const { port } = server; + expect((await send(port, "POST", "/json", { body: '{"a":1}' })).body.toString()).toBe('{"a":1}'); + expect((await send(port, "POST", "/json", { body: "{nope" })).status).toBe(400); + expect((await send(port, "POST", "/json", { body: JSON.stringify({ long: "x".repeat(40) }) })).status).toBe(413); + const chunked = await send(port, "POST", "/body", { + headers: { "transfer-encoding": "chunked" }, + body: "y".repeat(40), + }); + expect(chunked.status).toBe(413); + expect((await send(port, "POST", "/body", { body: "y".repeat(10) })).body.toString()).toBe("10"); + expect(server.errors).toEqual([]); + await server.close(); + }); + + // Sol's and Astra's build reviews: an unfinished chunked upload past the limit got no answer. + it("answers 413 before a chunked upload past the limit has ended", async () => { + const server = await reading(); + const request = httpRequest({ + host: "127.0.0.1", + port: server.port, + path: "/json", + method: "POST", + headers: { "transfer-encoding": "chunked" }, + agent: false, + }); + request.on("error", () => {}); + const status = new Promise((resolve) => { + request.on("response", (response) => { + response.resume(); + resolve(response.statusCode ?? 0); + }); + }); + request.write(`"${"x".repeat(40)}`); + expect(await status).toBe(413); + request.destroy(); + await server.close(); + }); + + it("drains a small body nobody read and keeps the connection; a large one closes it", async () => { + const server = await served({ + "@app/a": contributing({ routes: { "POST /ignore": async () => ({ kind: "json", value: 1 }) } }), + }); + const agent = new Agent({ keepAlive: true, maxSockets: 1 }); + const post = (body: string) => + new Promise<{ connection: string | undefined; socket: unknown }>((resolve, reject) => { + const request = httpRequest( + { host: "127.0.0.1", port: server.port, method: "POST", path: "/ignore", agent }, + (response) => { + response.resume(); + response.on("end", () => + resolve({ connection: response.headers.connection, socket: request.socket }), + ); + }, + ); + request.on("error", reject); + request.end(body); + }); + const first = await post("x".repeat(1000)); + const second = await post("x".repeat(1000)); + expect(first.connection).toBe("keep-alive"); + expect(second.socket).toBe(first.socket); + expect((await post("x".repeat(100_000))).connection).toBe("close"); + agent.destroy(); + await server.close(); + }); +}); diff --git a/packages/chord/test/next-node/http/response.test.ts b/packages/chord/test/next-node/http/response.test.ts new file mode 100644 index 000000000..3c953ba85 --- /dev/null +++ b/packages/chord/test/next-node/http/response.test.ts @@ -0,0 +1,245 @@ +// Tests of responses (slice 10 §2, D5, R6; security baseline (4), (6)): every kind written by a running server, the +// response's headers over the default ones (an invalid one reported, 500), ranges (206, 416, multi-range), `HEAD`, +// attachments and inline bodies, safe download names, bodies without `size`, a short or throwing read, a disconnect +// stopping the reads. + +import { request as httpRequest } from "node:http"; +import { describe, expect, it } from "vitest"; +import type { Ctx } from "../../../src/next/host/index.ts"; +import type { HttpResponse } from "../../../src/next-node/http/index.ts"; +import { contributing, send, served } from "./support.ts"; + +/** A server answering `GET /r` with `response`. */ +async function answering(response: HttpResponse) { + return served({ "@app/r": contributing({ routes: { "GET /r": async () => response } }) }); +} + +/** A `bytes` response over `data` that records its reads. */ +function bytesOf(data: Buffer, extra: Partial> = {}) { + const reads: [number, number][] = []; + const response: HttpResponse = { + kind: "bytes", + mime: "application/octet-stream", + size: data.length, + read: async (offset, length) => { + reads.push([offset, length]); + return data.subarray(offset, offset + length); + }, + ...extra, + }; + return { response, reads }; +} + +describe("descriptions", () => { + it("writes json, html, redirect and error, the default headers first, the response's replacing same-named ones", async () => { + const cases: [HttpResponse, number, string, string | undefined][] = [ + [{ kind: "json", value: { a: 1 } }, 200, '{"a":1}', "application/json; charset=utf-8"], + [ + { kind: "html", body: "

    x

    ", status: 201, headers: { "content-security-policy": "own" } }, + 201, + "

    x

    ", + "text/html; charset=utf-8", + ], + [{ kind: "redirect", to: "/next", status: 303 }, 303, "", undefined], + [{ kind: "error", status: 418 }, 418, "I'm a Teapot", "text/plain; charset=utf-8"], + [{ kind: "error", status: 409, message: "taken" }, 409, "taken", "text/plain; charset=utf-8"], + ]; + for (const [response, status, body, type] of cases) { + const server = await answering(response); + const answer = await send(server.port, "GET", "/r"); + expect([answer.status, answer.body.toString(), answer.headers["content-type"]]).toEqual([status, body, type]); + expect([answer.headers["cache-control"], answer.headers["x-content-type-options"]]).toEqual([ + "no-store", + "nosniff", + ]); + expect(answer.headers["content-security-policy"]).toMatch( + response.headers === undefined ? /^default-src 'none'/ : /^own$/, + ); + if (response.kind === "redirect") expect(answer.headers.location).toBe("/next"); + await server.close(); + } + }); + + it("answers 500 for a header with a CR or LF and a value that is no JSON, reported response_failed", async () => { + const server = await served({ + "@app/r": contributing({ + routes: { + "GET /header": async () => ({ kind: "json", value: 1, headers: { "X-Split": "a\r\nSet-Cookie: x=1" } }), + "GET /bigint": async () => ({ kind: "json", value: { n: 1n } as never }), + }, + }), + }); + for (const path of ["/header", "/bigint"]) { + const answer = await send(server.port, "GET", path); + expect([answer.status, answer.headers["set-cookie"], answer.headers["x-content-type-options"]]).toEqual([ + 500, + undefined, + "nosniff", + ]); + } + expect(server.errors).toMatchObject([ + { kind: "response_failed", pattern: "GET /header" }, + { kind: "response_failed", pattern: "GET /bigint" }, + ]); + await server.close(); + }); +}); + +describe("bytes", () => { + const data = Buffer.from("0123456789".repeat(60_000)); + + it("sends Content-Length, reads 256 KiB pieces, answers a single range 206 and an unsatisfiable one 416", async () => { + const { response, reads } = bytesOf(data, { ranges: true }); + const server = await answering(response); + const whole = await send(server.port, "GET", "/r"); + expect([whole.status, whole.headers["content-length"], whole.headers["accept-ranges"]]).toEqual([ + 200, + "600000", + "bytes", + ]); + expect(whole.body.equals(data)).toBe(true); + expect(reads).toEqual([ + [0, 262144], + [262144, 262144], + [524288, 75712], + ]); + const range = await send(server.port, "GET", "/r", { headers: { range: "bytes=10-19" } }); + expect([range.status, range.body.toString(), range.headers["content-range"]]).toEqual([ + 206, + "0123456789", + "bytes 10-19/600000", + ]); + const suffix = await send(server.port, "GET", "/r", { headers: { range: "bytes=-3" } }); + expect([suffix.status, suffix.body.toString()]).toEqual([206, "789"]); + const past = await send(server.port, "GET", "/r", { headers: { range: "bytes=600000-" } }); + expect([past.status, past.headers["content-range"]]).toEqual([416, "bytes */600000"]); + const multi = await send(server.port, "GET", "/r", { headers: { range: "bytes=0-1,5-6" } }); + expect([multi.status, multi.body.length]).toEqual([200, 600000]); + await server.close(); + }); + + it("ignores ranges unless asked; HEAD sends headers only and reads nothing", async () => { + const { response, reads } = bytesOf(data); + const server = await answering(response); + const ranged = await send(server.port, "GET", "/r", { headers: { range: "bytes=0-1" } }); + expect([ranged.status, ranged.headers["accept-ranges"]]).toEqual([200, undefined]); + reads.length = 0; + const head = await send(server.port, "HEAD", "/r"); + expect([head.status, head.headers["content-length"], head.body.length, reads]).toEqual([200, "600000", 0, []]); + await server.close(); + }); + + it("names a download with an ASCII fallback and the UTF-8 name, control and bidi characters dropped", async () => { + const server = await answering(bytesOf(Buffer.from("x"), { download: 'Grüße "1"\r\n\u202etxt.exe' }).response); + const answer = await send(server.port, "GET", "/r"); + expect(answer.headers["content-disposition"]).toBe( + `attachment; filename="Gr__e _1_txt.exe"; filename*=UTF-8''Gr%C3%BC%C3%9Fe%20%221%22txt.exe`, + ); + await server.close(); + }); + + it("is an attachment unless inline; an inline one gets a sandbox CSP", async () => { + const attached = await answering(bytesOf(Buffer.from("x")).response); + expect((await send(attached.port, "GET", "/r")).headers["content-disposition"]).toBe("attachment"); + await attached.close(); + const shown = await answering(bytesOf(Buffer.from("x"), { inline: true }).response); + const answer = await send(shown.port, "GET", "/r"); + expect(answer.headers["content-disposition"]).toBe("inline"); + expect(answer.headers["content-security-policy"]).toMatch(/; sandbox$/); + await shown.close(); + }); + + it("refuses a foreign request (another site's) 403, reading nothing", async () => { + const { response, reads } = bytesOf(Buffer.from("secret")); + const server = await answering(response); + const foreign = await send(server.port, "GET", "/r", { headers: { "sec-fetch-site": "cross-site" } }); + expect([foreign.status, reads]).toEqual([403, []]); + expect((await send(server.port, "GET", "/r", { headers: { "sec-fetch-site": "same-origin" } })).status).toBe(200); + await server.close(); + }); + + it("without a size: chunked, read with growing offsets until an empty chunk, no ranges (R6)", async () => { + const pieces = [Buffer.from("ab"), Buffer.from("cde"), Buffer.alloc(0)]; + const offsets: number[] = []; + const server = await answering({ + kind: "bytes", + mime: "text/plain", + ranges: true, + read: async (offset) => { + offsets.push(offset); + return pieces.shift()!; + }, + }); + const answer = await send(server.port, "GET", "/r", { headers: { range: "bytes=0-1" } }); + expect([ + answer.status, + answer.body.toString(), + answer.headers["transfer-encoding"], + answer.headers["accept-ranges"], + ]).toEqual([200, "abcde", "chunked", undefined]); + expect(offsets).toEqual([0, 2, 5]); + await server.close(); + }); + + it("destroys the connection on a short or a throwing read, reported response_failed", async () => { + const server = await served({ + "@app/r": contributing({ + routes: { + "GET /short": async () => ({ + kind: "bytes", + mime: "text/plain", + size: 10, + read: async () => Buffer.from("abc"), + }), + // Astra's build review: a throwing read hung the response, unreported. + "GET /throws": async () => ({ + kind: "bytes", + mime: "text/plain", + size: 10, + read: async () => { + throw new Error("read failed"); + }, + }), + }, + }), + }); + await expect(send(server.port, "GET", "/short")).rejects.toThrow(); + await expect(send(server.port, "GET", "/throws")).rejects.toThrow(); + expect(server.errors).toMatchObject([ + { kind: "response_failed", pattern: "GET /short" }, + { kind: "response_failed", pattern: "GET /throws", error: { message: "read failed" } }, + ]); + await server.close(); + }); + + it("stops reading when the client disconnects: the read's context is aborted", async () => { + let reads = 0; + let signal: AbortSignal | undefined; + const server = await answering({ + kind: "bytes", + mime: "application/octet-stream", + size: 1024 * 1024 * 1024, + read: async (_offset, length, ctx: Ctx) => { + reads++; + signal = ctx.abortSignal; + return Buffer.alloc(length); + }, + }); + await new Promise((resolve) => { + const request = httpRequest({ host: "127.0.0.1", port: server.port, path: "/r", agent: false }, (response) => { + response.once("data", () => { + request.destroy(); + resolve(); + }); + }); + request.end(); + }); + await new Promise((resolve) => setTimeout(resolve, 100)); + const after = reads; + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(signal?.aborted).toBe(true); + expect(reads).toBe(after); + expect(after).toBeLessThan(1000); + await server.close(); + }); +}); diff --git a/packages/chord/test/next-node/http/router.test.ts b/packages/chord/test/next-node/http/router.test.ts new file mode 100644 index 000000000..9854820c6 --- /dev/null +++ b/packages/chord/test/next-node/http/router.test.ts @@ -0,0 +1,228 @@ +// Tests of the router (slice 10 §2, D2, D8): patterns, decoding (one rule for `:name` and rest segments), precedence, +// `HEAD` per path, each route's own param names, route options, conflicts, wraps resolved by phase, contributor order, +// scope and route, allowed hosts, framers, relays, CSP sources, sockets. + +import { describe, expect, it } from "vitest"; +import type { RegistryConflict, RegistryTable } from "../../../src/next/registries/index.ts"; +import { + composeRouter, + type HttpContribution, + type HttpRoute, + type HttpWrap, + type RouteMatch, + type Router, +} from "../../../src/next-node/http/router.ts"; + +/** A route named `name` (its handler answers its name). */ +const route = + (name: string): HttpRoute => + async () => ({ kind: "html", body: name }); + +/** The router of `entries` (plugin, contribution), with the conflicts it reported. */ +function compose(...entries: [string, HttpContribution][]): { router: Router; conflicts: RegistryConflict[] } { + const conflicts: RegistryConflict[] = []; + const table: RegistryTable = entries.map(([plugin, value]) => ({ plugin, version: 1, value })); + return { router: composeRouter(table, { report: (conflict) => conflicts.push(conflict) }), conflicts }; +} + +/** The matched route's pattern and params, or what `route` answered. */ +function matched(router: Router, method: string, path: string) { + const match = router.route(method, path); + if (match === undefined || match === "bad_path") return match; + return { pattern: match.pattern, params: match.params }; +} + +describe("patterns and precedence (D8)", () => { + const { router } = compose([ + "@app/a", + { + routes: { + "GET /notes": route("list"), + "GET /notes/:id": route("one"), + "GET /notes/new": route("new"), + "GET /notes/:id/export": route("export"), + "GET /files/*": route("files"), + "GET /files/special/:name": route("special"), + "GET /": route("root"), + }, + }, + ]); + + it("matches exact segments, :name and a trailing *, the most specific from the left", () => { + expect(matched(router, "GET", "/notes")).toEqual({ pattern: "GET /notes", params: {} }); + expect(matched(router, "GET", "/notes/new")).toEqual({ pattern: "GET /notes/new", params: {} }); + expect(matched(router, "GET", "/notes/7")).toEqual({ pattern: "GET /notes/:id", params: { id: "7" } }); + expect(matched(router, "GET", "/notes/7/export")).toMatchObject({ params: { id: "7" } }); + expect(matched(router, "GET", "/files/special/x")).toEqual({ + pattern: "GET /files/special/:name", + params: { name: "x" }, + }); + // The exact branch fails deeper: the rest takes it. + expect(matched(router, "GET", "/files/special/x/y")).toEqual({ + pattern: "GET /files/*", + params: { "*": "special/x/y" }, + }); + expect(matched(router, "GET", "/")).toEqual({ pattern: "GET /", params: {} }); + }); + + it("a /* matches its prefix with an empty rest; :name needs a non-empty segment", () => { + expect(matched(router, "GET", "/files")).toEqual({ pattern: "GET /files/*", params: { "*": "" } }); + expect(matched(router, "GET", "/notes/")).toBeUndefined(); + }); + + // Opus's build review F4, security audit H5: `%2F` in `:name` reached a file path. + it("percent-decodes :name and rest segments; one decoding to /, \\, NUL, . or .. is a bad path", () => { + expect(matched(router, "GET", "/notes/a%20b")).toMatchObject({ params: { id: "a b" } }); + expect(matched(router, "GET", "/files/a%20b/c")).toMatchObject({ params: { "*": "a b/c" } }); + for (const bad of ["a%2Fb", "..%2F..%2Fsecret", "a%5Cb", "a%00b", "%2e%2e", "%2E", ".", ".."]) { + expect(router.route("GET", `/notes/${bad}`)).toBe("bad_path"); + expect(router.route("GET", `/files/x/${bad}`)).toBe("bad_path"); + } + expect(router.route("GET", "/notes/%E0%A4%A")).toBe("bad_path"); + }); + + it("falls back to the GET route for HEAD; another method or path has none", () => { + expect(matched(router, "HEAD", "/notes")).toEqual({ pattern: "GET /notes", params: {} }); + expect(router.route("POST", "/notes")).toBeUndefined(); + expect(router.route("GET", "/nope")).toBeUndefined(); + }); + + // Opus's build review F2, Astra's: one explicit HEAD route anywhere hid every GET fallback. + it("falls back to GET per path when another path has an explicit HEAD route", () => { + const { router } = compose([ + "@app/a", + { routes: { "HEAD /download": route("head"), "GET /download": route("get"), "GET /notes": route("notes") } }, + ]); + expect(matched(router, "HEAD", "/download")).toMatchObject({ pattern: "HEAD /download" }); + expect(matched(router, "HEAD", "/notes")).toMatchObject({ pattern: "GET /notes" }); + }); + + // Opus's build review F3, Astra's: the first route's param name was used for every route at that position. + it("names each route's params by its own pattern", () => { + const { router } = compose(["@app/a", { routes: { "GET /:id/a": route("a"), "GET /:name/b": route("b") } }]); + expect(matched(router, "GET", "/v/a")).toEqual({ pattern: "GET /:id/a", params: { id: "v" } }); + expect(matched(router, "GET", "/v/b")).toEqual({ pattern: "GET /:name/b", params: { name: "v" } }); + }); + + it("takes a route with options: page and crossSite", () => { + const { router } = compose([ + "@app/a", + { + routes: { + "GET /p": { handler: route("p"), page: true }, + "POST /hook": { handler: route("h"), crossSite: true }, + }, + }, + ]); + expect(router.route("GET", "/p")).toMatchObject({ page: true, crossSite: false }); + expect(router.route("POST", "/hook")).toMatchObject({ page: false, crossSite: true }); + expect(router.route("GET", "/notes")).toBeUndefined(); + }); +}); + +describe("conflicts", () => { + it("reports one pattern from two contributions (also with other param names); the first by plugin name wins", () => { + const { router, conflicts } = compose( + ["@app/a", { routes: { "GET /x/:id": route("a") } }], + ["@app/b", { routes: { "GET /x/:key": route("b"), BAD: route("bad") }, sockets: { nope: () => {} } }], + ); + expect((router.route("GET", "/x/1") as RouteMatch).plugin).toBe("@app/a"); + expect(conflicts).toEqual([ + { message: 'route "GET /x/:key" is contributed twice', plugins: ["@app/a", "@app/b"] }, + { message: 'route "BAD" is no "METHOD /pattern"', plugins: ["@app/b"] }, + { message: 'socket "nope" is no pattern', plugins: ["@app/b"] }, + ]); + }); + + it("refuses a route for a method a Web Request cannot carry, reported", () => { + const { router, conflicts } = compose([ + "@app/a", + { routes: { "TRACE /x": route("t"), "CONNECT /y": route("c") } }, + ]); + expect(router.route("TRACE", "/x")).toBeUndefined(); + expect(conflicts).toEqual([ + { message: 'route "TRACE /x": TRACE is served for no resource (501)', plugins: ["@app/a"] }, + { message: 'route "CONNECT /y": CONNECT is served for no resource (501)', plugins: ["@app/a"] }, + ]); + }); +}); + +describe("wraps (D2)", () => { + const wrap = (name: string, phase: HttpWrap["phase"], extra: Partial = {}): HttpWrap => ({ + name, + phase, + wrap: (request, next) => next(request), + ...extra, + }); + + it("orders by phase (outermost first), then contributor order; own wraps around their plugin's routes only", () => { + const { router } = compose( + [ + "@app/a", + { + routes: { "GET /a": route("a") }, + wraps: [wrap("a-transform", "transform"), wrap("a-observe", "observe")], + }, + ], + [ + "@app/b", + { + routes: { "GET /b": route("b") }, + wraps: [wrap("b-all", "headers", { scope: "all" }), wrap("b-own", "observe")], + }, + ], + ); + const names = (path: string) => (router.route("GET", path) as RouteMatch).wraps.map(({ wrap }) => wrap.name); + expect(names("/a")).toEqual(["a-observe", "b-all", "a-transform"]); + expect(names("/b")).toEqual(["b-own", "b-all"]); + }); + + it("narrows to one route; a route nobody contributes is a reported conflict", () => { + const { router, conflicts } = compose([ + "@app/a", + { + routes: { "GET /a": route("a"), "GET /b": route("b") }, + wraps: [wrap("only-a", "observe", { route: "GET /a" }), wrap("ghost", "observe", { route: "GET /c" })], + }, + ]); + expect((router.route("GET", "/a") as RouteMatch).wraps.map(({ wrap }) => wrap.name)).toEqual(["only-a"]); + expect((router.route("GET", "/b") as RouteMatch).wraps).toEqual([]); + expect(conflicts).toEqual([ + { message: 'wrap "ghost" names route "GET /c", which nobody contributes', plugins: ["@app/a"] }, + ]); + }); +}); + +describe("hosts and sockets", () => { + it("collects hosts lowercased, framers, relays and CSP sources; matches sockets with their largest frame", () => { + const terminal = () => {}; + const relay = { name: "radius", verify: () => undefined }; + const contribution: HttpContribution = { + hosts: ["Notes.Example.com"], + sockets: { "/terminal/:id": terminal }, + maxPayload: 10, + framers: ["https://frame.example.com"], + relays: [relay], + csp: { "img-src": ["https://cdn.example.com"] }, + }; + const { router } = compose( + ["@app/a", contribution], + ["@app/b", { hosts: ["b.example.com"], csp: { "img-src": ["data:"] } }], + ); + expect([...router.hosts]).toEqual(["notes.example.com", "b.example.com"]); + expect([[...router.framers], router.relays, [...router.csp]]).toEqual([ + ["https://frame.example.com"], + [relay], + [["img-src", ["https://cdn.example.com", "data:"]]], + ]); + expect(router.socket("/terminal/7")).toMatchObject({ + pattern: "/terminal/:id", + plugin: "@app/a", + params: { id: "7" }, + maxPayload: 10, + }); + expect(router.socket("/terminal/..")).toBe("bad_path"); + expect(router.socket("/nope")).toBeUndefined(); + expect(router.socketHandlers.has(terminal)).toBe(true); + }); +}); diff --git a/packages/chord/test/next-node/http/support.ts b/packages/chord/test/next-node/http/support.ts new file mode 100644 index 000000000..8a4b82b59 --- /dev/null +++ b/packages/chord/test/next-node/http/support.ts @@ -0,0 +1,204 @@ +// Test support for the HTTP component (slice 10 §2): a server host with the component, the app's configuration and +// authenticator, and plugin facets contributing to `Http`; raw requests (any `Host`, any body) and WebSocket clients +// against its listener. + +import { request as httpRequest, type IncomingHttpHeaders } from "node:http"; +import { WebSocket } from "ws"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import type { Address } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet, type Facet, type Host, type HostError } from "../../../src/next/host/index.ts"; +import { + type Authenticator, + Authenticators, + createHttp, + Http, + type HttpComponent, + type HttpContribution, + type HttpError, + type HttpOptions, +} from "../../../src/next-node/http/index.ts"; + +/** The principals of these tests. */ +export type TestPrincipal = + | { readonly kind: "server" } + | { readonly kind: "owner" } + | { readonly kind: "guest"; readonly id: string }; + +/** A running server: its host, its component, its port once loaded, what both reported. */ +export type Served = { + readonly host: Host; + readonly http: HttpComponent; + /** The listener's port (after `load`). */ + readonly port: number; + readonly errors: HttpError[]; + readonly hostErrors: HostError[]; + /** The core facets and the plugins, as loaded. */ + facets(plugins?: Readonly>>): { name: string; facet: Facet }[]; + close(): Promise; +}; + +/** A plugin facet contributing `contribution` to `Http`. */ +export function contributing(contribution: HttpContribution): Facet { + return defineFacet((_host, self) => self.contribute(Http, contribution)); +} + +/** + * Builds the component on a new server host (listening on loopback, a free port), then loads it with the app's + * authenticator (`@app/auth`: the `x-principal` header names one, else anonymous) and `plugins` (by name) unless + * `load` is false; `config` overrides the settings (`listen: undefined`: no listener). + */ +export async function served( + plugins: Readonly>> = {}, + options: { + readonly config?: Omit; + readonly authenticate?: Authenticator; + readonly load?: boolean; + readonly readyTimeout?: number; + readonly address?: Address; + } = {}, +): Promise }> { + const errors: HttpError[] = []; + const hostErrors: HostError[] = []; + const host = createHost({ + principal: { kind: "server" }, + onError: (error) => hostErrors.push(error), + ...(options.readyTimeout === undefined ? {} : { readyTimeout: options.readyTimeout }), + ...(options.address === undefined ? {} : { address: options.address }), + }); + const config: Omit = { listen: { host: "127.0.0.1", port: 0 }, ...options.config }; + const http = createHttp(host, { ...config, onError: (error) => errors.push(error) }, context); + const authenticate = options.authenticate ?? ((request) => principalOf(request.headers)); + const core = [ + { name: "chord.http", facet: http.facet }, + { + name: "@app/auth", + facet: defineFacet((_host, self) => self.contribute(Authenticators, authenticate)), + }, + ]; + const facets = (more = plugins) => [...core, ...Object.entries(more).map(([name, facet]) => ({ name, facet }))]; + let port = 0; + const load = async (): Promise => { + await host.load(facets(), context); + if (config.listen !== undefined) port = (await http.listening(context)).port; + }; + if (options.load !== false) await load(); + return { + host, + http, + get port() { + return port; + }, + errors, + hostErrors, + facets, + load, + close: async () => { + await http.close(context); + await host.close(context); + }, + }; +} + +/** The principal an `x-principal` header names: `owner`, `guest:`; none without it. */ +function principalOf(headers: Headers): TestPrincipal | undefined { + const named = headers.get("x-principal"); + if (named === "owner") return { kind: "owner" }; + if (named?.startsWith("guest:")) return { kind: "guest", id: named.slice(6) }; + return undefined; +} + +/** A response as a test reads it. */ +export type Answer = { readonly status: number; readonly headers: IncomingHttpHeaders; readonly body: Buffer }; + +/** + * A request to `port` with raw control: `Host` defaults to `127.0.0.1:`; `body` is sent as is; `signal` aborts it. + */ +export function send( + port: number, + method: string, + path: string, + options: { + readonly headers?: Readonly>; + readonly body?: string | Buffer; + readonly agent?: false; + } = {}, +): Promise { + return new Promise((resolve, reject) => { + const request = httpRequest( + { + host: "127.0.0.1", + port, + method, + path, + agent: false, + headers: { host: `127.0.0.1:${port}`, ...options.headers }, + }, + (response) => { + const chunks: Buffer[] = []; + response.on("data", (chunk: Buffer) => chunks.push(chunk)); + response.on("end", () => + resolve({ status: response.statusCode ?? 0, headers: response.headers, body: Buffer.concat(chunks) }), + ); + response.on("error", reject); + }, + ); + request.on("error", reject); + request.end(options.body); + }); +} + +/** A WebSocket client to `path` on `port` (`Host` loopback), with `headers`; resolves once open, rejects when refused. */ +export function socketTo( + port: number, + path: string, + headers: Readonly> = {}, + options: { readonly maxPayload?: number } = {}, +): Promise { + return new Promise((resolve, reject) => { + const socket = new WebSocket(`ws://127.0.0.1:${port}${path}`, { headers, ...options }); + // Frames sent at once by the server arrive with the upgrade: queued from the start. + inboxes.set(socket, messagesOf(socket)); + socket.once("open", () => resolve(socket)); + socket.once("unexpected-response", (_request, response) => { + reject(new Error(`refused ${response.statusCode}`)); + socket.terminate(); + }); + socket.once("error", reject); + }); +} + +/** The messages of every socket `socketTo` opened, queued from its start. */ +const inboxes = new WeakMap }>(); + +/** The next text message of a socket `socketTo` opened. */ +export function nextMessage(socket: WebSocket): Promise { + return inboxes.get(socket)!.next(); +} + +/** The text messages `socket` receives, from now on, in order: `next()` waits for the next one. */ +export function messagesOf(socket: WebSocket): { next(): Promise } { + const queued: string[] = []; + const waiting: ((text: string) => void)[] = []; + socket.on("message", (data) => { + const text = String(data); + const waiter = waiting.shift(); + if (waiter === undefined) queued.push(text); + else waiter(text); + }); + return { + next: () => { + const text = queued.shift(); + return text === undefined ? new Promise((resolve) => waiting.push(resolve)) : Promise.resolve(text); + }, + }; +} + +/** The close code and reason of `socket`'s close. */ +export function closeOf(socket: WebSocket): Promise<{ readonly code: number; readonly reason: string }> { + return new Promise((resolve) => { + if (socket.readyState === socket.CLOSED) resolve({ code: 1006, reason: "" }); + socket.once("close", (code, reason) => resolve({ code, reason: reason.toString() })); + }); +} + +export { context }; diff --git a/packages/chord/test/next-node/http/upgrades.test.ts b/packages/chord/test/next-node/http/upgrades.test.ts new file mode 100644 index 000000000..f4de24124 --- /dev/null +++ b/packages/chord/test/next-node/http/upgrades.test.ts @@ -0,0 +1,458 @@ +// Tests of upgraded connections (slice 10 §2, D6, Q6; decisions "allowed hosts only through contributions", "plugin +// sockets get a heartbeat"): the links endpoint (`host.accept`, options per socket, refused sockets, the 1 MiB frame +// default, the output cap), plugin sockets (handed to their handler, an oversized frame closing only that socket, a +// throwing handler), connections terminated at once when their socket handler or host leaves (a raw peer can act no +// more), an upgrade checked again after authenticating, the heartbeat, close bounded for a peer that never answers, +// and nothing kept of a closed socket or its `ws` server. + +import { once } from "node:events"; +import { connect, type Socket } from "node:net"; +import { describe, expect, it, vi } from "vitest"; +import { WebSocket, WebSocketServer } from "ws"; +import { arr, hostKind, local, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { webSocketLink } from "../../../src/next/link/index.ts"; +import { Http, type HttpContribution } from "../../../src/next-node/http/index.ts"; +import { limitedSocket } from "../../../src/next-node/http/upgrades.ts"; +import { expectCollected, settledHeap } from "../../next/support.ts"; +import { closeOf, context, contributing, nextMessage, send, served, socketTo, type TestPrincipal } from "./support.ts"; + +const Server = hostKind("server"); + +/** Who the server sees calling: served at the server's address. */ +const Who = remote("test.who", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { main: { methods: { who: method(undefined, str()) } } }, +}); + +const serveWho = defineFacet((host) => + host.serve(Who, () => ({ + variant: "main", + members: { who: async (ctx) => JSON.stringify(ctx.principal ?? null) }, + })), +); + +/** A plugin socket that echoes text, with its handler's request params and principal sent first. */ +const echo = contributing({ + sockets: { + "/echo/:room": (socket, request, ctx) => { + socket.send(`${request.params.room} as ${ctx.principal?.kind ?? "nobody"}`); + socket.on("message", (data) => socket.send(String(data))); + }, + }, + maxPayload: 64, +}); + +describe("the links endpoint", () => { + it("accepts a link as the authenticated principal, with the options links.options gave that socket", async () => { + const options: unknown[] = []; + const server = await served( + { "@app/who": serveWho }, + { + address: Server(), + config: { + links: { + path: "/chord", + options: (request, principal) => { + options.push([request.path, principal]); + return { maxPayload: 1024 * 1024 }; + }, + }, + }, + }, + ); + const tab = createHost({ onError: () => {} }); + await tab.load([], context); + // The link wraps the socket at once: the server's `hello` comes with the upgrade. + const socket = new WebSocket(`ws://127.0.0.1:${server.port}/chord`, { headers: { "x-principal": "owner" } }); + tab.connect(webSocketLink(socket as never), { principal: { kind: "server" } }, context); + const who = await tab.get(Who, {}, context); + expect(await who.who(context)).toBe('{"kind":"owner"}'); + expect(options).toEqual([["/chord", { kind: "owner" }]]); + await tab.close(context); + await server.close(); + }); + + it("refuses a socket without a principal, from another Origin or Host, foreign, or whose links.options throws (403)", async () => { + const server = await served( + { "@app/hosts": contributing({ hosts: ["a.example.com"] }) }, + { + config: { + links: { + path: "/chord", + options: (request) => { + if (request.headers.has("x-fail")) throw new Error("options"); + return {}; + }, + }, + }, + }, + ); + const { port } = server; + const owner = { "x-principal": "owner" }; + const refusals = [ + socketTo(port, "/chord"), + socketTo(port, "/chord", { ...owner, origin: "http://evil.com" }), + socketTo(port, "/chord", { ...owner, origin: "null" }), + socketTo(port, "/chord", { ...owner, origin: `HTTP://127.0.0.1:${port}` }), + socketTo(port, "/chord", { ...owner, host: "evil.com" }), + socketTo(port, "/chord", { ...owner, "sec-fetch-site": "cross-site" }), + socketTo(port, "/chord", { ...owner, "x-fail": "1" }), + ]; + for (const refusal of refusals) await expect(refusal).rejects.toThrow("refused 403"); + const same = await socketTo(port, "/chord", { ...owner, origin: `http://127.0.0.1:${port}` }); + same.close(); + expect(server.errors).toEqual([{ kind: "hook_failed", hook: "links.options", error: new Error("options") }]); + await server.close(); + }); + + it("takes frames of at most 1 MiB unless links.options says otherwise (Opus's audit H2)", async () => { + const server = await served({}, { config: { links: { path: "/chord" } } }); + const socket = await socketTo(server.port, "/chord", { "x-principal": "owner" }); + const closed = closeOf(socket); + socket.send(Buffer.alloc(1024 * 1024 + 1)); + expect((await closed).code).toBe(1009); + await server.close(); + }); + + it("cuts off a client that stopped reading: past 4 MiB unsent beyond the frame being sent", () => { + const sent: number[] = []; + let terminated = 0; + const fake = { + OPEN: 1, + readyState: 1, + bufferedAmount: 0, + binaryType: "nodebuffer", + send: (data: Uint8Array) => void sent.push(data.byteLength), + terminate: () => void terminated++, + }; + const limited = limitedSocket(fake as never); + limited.send(new Uint8Array(10)); + fake.bufferedAmount = 5 * 1024 * 1024; + limited.send(new Uint8Array(10)); + expect([sent, terminated]).toEqual([[10], 1]); + }); +}); + +describe("plugin sockets", () => { + it("hands the socket and its request to the handler in a scope with its principal; no principal is refused", async () => { + const server = await served({ "@app/echo": echo }); + await expect(socketTo(server.port, "/echo/lobby")).rejects.toThrow("refused 403"); + await expect(socketTo(server.port, "/nope", { "x-principal": "owner" })).rejects.toThrow("refused 404"); + const socket = await socketTo(server.port, "/echo/lobby", { "x-principal": "owner" }); + expect(await nextMessage(socket)).toBe("lobby as owner"); + socket.send("hi"); + expect(await nextMessage(socket)).toBe("hi"); + socket.close(); + await server.close(); + }); + + it("an oversized frame closes only that socket (1009), never the server", async () => { + const server = await served({ "@app/echo": echo }); + const [big, other] = await Promise.all([ + socketTo(server.port, "/echo/a", { "x-principal": "owner" }), + socketTo(server.port, "/echo/b", { "x-principal": "owner" }), + ]); + await Promise.all([nextMessage(big), nextMessage(other)]); + const closed = closeOf(big); + big.send("x".repeat(100)); + expect((await closed).code).toBe(1009); + other.send("still here"); + expect(await nextMessage(other)).toBe("still here"); + other.close(); + await server.close(); + }); + + it("a throwing handler is reported socket_failed and closes its socket (1011)", async () => { + const server = await served({ + "@app/bad": contributing({ + sockets: { + "/bad": () => { + throw new Error("handler"); + }, + }, + }), + }); + const socket = await socketTo(server.port, "/bad", { "x-principal": "owner" }); + expect((await closeOf(socket)).code).toBe(1011); + expect(server.errors).toMatchObject([{ kind: "socket_failed", pattern: "/bad", plugin: "@app/bad" }]); + await server.close(); + }); +}); + +describe("revocation", () => { + it("terminates a plugin's sockets at once when a reload removes its contribution, sealing their scopes; others stay (Q6)", async () => { + let signal: AbortSignal | undefined; + const removed = contributing({ + sockets: { + "/removed": (_socket, _request, ctx) => { + signal = ctx.abortSignal; + }, + }, + }); + const other = contributing({ sockets: { "/other": () => {} } }); + const server = await served({ "@app/removed": removed, "@app/other": other }); + const doomed = await socketTo(server.port, "/removed", { "x-principal": "owner" }); + const kept = await socketTo(server.port, "/other", { "x-principal": "owner" }); + const closed = closeOf(doomed); + await server.host.reload(server.facets({ "@app/other": other }), context); + expect(signal?.aborted).toBe(true); + expect((await closed).code).toBe(1006); + expect(kept.readyState).toBe(kept.OPEN); + kept.close(); + await server.close(); + }); + + it("terminates links and sockets through a host no longer contributed; loopback ones stay", async () => { + const server = await served( + { "@app/echo": echo, "@app/hosts": contributing({ hosts: ["a.example.com"] }) }, + { config: { links: { path: "/chord" } } }, + ); + const through = { "x-principal": "owner", host: "a.example.com" }; + const link = await socketTo(server.port, "/chord", through); + const socket = await socketTo(server.port, "/echo/a", through); + const loopback = await socketTo(server.port, "/echo/b", { "x-principal": "owner" }); + const closes = [closeOf(link), closeOf(socket)]; + await server.host.reload(server.facets({ "@app/echo": echo }), context); + expect((await Promise.all(closes)).map(({ code }) => code)).toEqual([1006, 1006]); + expect(loopback.readyState).toBe(loopback.OPEN); + expect((await send(server.port, "GET", "/", { headers: { host: "a.example.com" } })).status).toBe(403); + loopback.close(); + await server.close(); + }); + + it("keeps a writable contribution's own sockets when only its hosts change", async () => { + const Tunnel = local("app.tunnel", { version: 1, params: {}, members: { name: method(arr(str())) } }); + const handler = (): void => {}; + const tunnel = defineFacet((host) => + host.serve(Tunnel, (_, self) => { + const contribution = self.writable({ hosts: [], sockets: { "/own": handler } }); + self.contribute(Http, contribution); + return { + name: async (hosts) => { + contribution.set({ hosts, sockets: { "/own": handler } }); + return null; + }, + }; + }), + ); + const server = await served({ "@app/tunnel": tunnel }); + const handle = await server.host.get(Tunnel, {}, context); + const socket = await socketTo(server.port, "/own", { "x-principal": "owner" }); + await handle.name(["t.example.com"], context); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(socket.readyState).toBe(socket.OPEN); + socket.close(); + await server.close(); + }); + + // Astra's security audit A1: a policy close left the peer able to invoke the plugin's handlers. + it("a raw peer whose host or contribution left can send nothing more to the handler", async () => { + const calls: string[] = []; + const sockets = contributing({ + sockets: { "/echo": (socket) => void socket.on("message", (data) => calls.push(String(data))) }, + }); + const tunnel = contributing({ hosts: ["t.example"] }); + const server = await served({ "@app/sockets": sockets, "@app/tunnel": tunnel }); + const throughHost = await rawSocket(server.port, "t.example", "/echo"); + const local = await rawSocket(server.port, `localhost:${server.port}`, "/echo"); + await server.host.reload(server.facets({ "@app/sockets": sockets }), context); + throughHost.write(frame("after the host left")); + await new Promise((resolve) => setTimeout(resolve, 50)); + await server.host.reload(server.facets({}), context); + local.write(frame("after the contribution left")); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(calls).toEqual([]); + throughHost.destroy(); + local.destroy(); + await server.close(); + }); + + // Astra's security audit A2: an upgrade authenticating across the removal of its host was still handed out. + it("checks an upgrade again after authenticating: one whose host left meanwhile is refused", async () => { + const gate = Promise.withResolvers(); + const started = Promise.withResolvers(); + const sockets = contributing({ sockets: { "/echo": () => {} } }); + const server = await served( + { "@app/sockets": sockets, "@app/tunnel": contributing({ hosts: ["t.example"] }) }, + { + authenticate: async () => { + started.resolve(); + await gate.promise; + return { kind: "owner" }; + }, + }, + ); + const opening = socketTo(server.port, "/echo", { host: "t.example" }); + await started.promise; + await server.host.reload(server.facets({ "@app/sockets": sockets }), context); + gate.resolve(); + await expect(opening).rejects.toThrow("refused 403"); + await server.close(); + }); +}); + +describe("cancellation", () => { + // Astra's recheck R1, Sol's N5: a client gone while its upgrade authenticated left that work running. + // Fable's second recheck: only a reset (RST) was seen; a clean disconnect (FIN, a tab closing) was not. + it.each(["ends its connection", "resets its connection"] as const)( + "aborts an upgrade's authentication when its client %s", + async (how) => { + const entered = Promise.withResolvers(); + let signal: AbortSignal | undefined; + const server = await served( + { "@app/s": contributing({ sockets: { "/s": () => {} } }) }, + { + authenticate: async (_request, ctx) => { + signal = ctx.abortSignal; + entered.resolve(); + return new Promise(() => {}); + }, + }, + ); + const socket = connect(server.port, "127.0.0.1"); + socket.on("error", () => {}); + await once(socket, "connect"); + socket.write( + `GET /s HTTP/1.1\r\nHost: 127.0.0.1:${server.port}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n`, + ); + await entered.promise; + const aborted = new Promise((resolve) => signal?.addEventListener("abort", resolve, { once: true })); + if (how === "ends its connection") socket.end(); + else socket.resetAndDestroy(); + await aborted; + expect(signal?.aborted).toBe(true); + socket.destroy(); + await server.close(); + }, + ); +}); + +describe("heartbeat", () => { + it("pings a plugin socket after 15 s of silence and terminates it 15 s later without an answer", async () => { + const server = await served({ + "@app/quiet": contributing({ + sockets: { "/quiet": (socket) => void socket.on("message", (data) => socket.send(data)) }, + }), + }); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + const silent = new WebSocket(`ws://127.0.0.1:${server.port}/quiet`, { + headers: { "x-principal": "owner" }, + autoPong: false, + }); + const answering = new WebSocket(`ws://127.0.0.1:${server.port}/quiet`, { + headers: { "x-principal": "owner" }, + }); + await Promise.all([once(silent, "open"), once(answering, "open")]); + const pinged = Promise.all([once(silent, "ping"), once(answering, "ping")]); + await vi.advanceTimersByTimeAsync(15_000); + await pinged; + // The answering client's pong travels for real (only timers are fake): an echo after it shows it arrived. + const echoed = once(answering, "message"); + answering.send("after the pong"); + await echoed; + const closed = closeOf(silent); + await vi.advanceTimersByTimeAsync(15_000); + expect((await closed).code).toBe(1006); + expect(answering.readyState).toBe(answering.OPEN); + answering.close(); + } finally { + vi.useRealTimers(); + } + await server.close(); + }); +}); + +describe("close", () => { + // Sol's build review: close waited on a peer that never acknowledged its close frame. + it("terminates a socket that never acknowledges its close at closeTimeout", async () => { + const server = await served( + { "@app/s": contributing({ sockets: { "/s": () => {} } }) }, + { config: { closeTimeout: 20 } }, + ); + const raw = await rawSocket(server.port, `127.0.0.1:${server.port}`, "/s"); + raw.pause(); + const started = Date.now(); + await server.http.close(context); + expect(Date.now() - started).toBeLessThan(500); + raw.destroy(); + await server.close(); + }); +}); + +describe("GC", () => { + // Sol's build review: a `ws` server was kept per frame size for the component's life. + it("keeps no ws server once its socket closed", async () => { + const refs: WeakRef[] = []; + const original = WebSocketServer.prototype.handleUpgrade; + WebSocketServer.prototype.handleUpgrade = function (this: WebSocketServer, ...args: Parameters) { + refs.push(new WeakRef(this)); + return original.apply(this, args); + }; + let maxPayload = 1024; + const server = await served( + {}, + { config: { links: { path: "/chord", options: () => ({ maxPayload: maxPayload++ }) } } }, + ); + try { + for (let i = 0; i < 8; i++) { + const socket = await socketTo(server.port, "/chord", { "x-principal": "owner" }); + const closed = closeOf(socket); + socket.close(); + await closed; + } + await settledHeap(); + await expectCollected(...refs); + } finally { + WebSocketServer.prototype.handleUpgrade = original; + await server.close(); + } + }); + + it("keeps nothing of a closed plugin socket", async () => { + const refs: WeakRef[] = []; + const server = await served({ + "@app/kept": contributing({ + sockets: { + "/kept": (socket, request, ctx) => { + refs.push(new WeakRef(socket), new WeakRef(request), new WeakRef(ctx)); + }, + }, + }), + }); + const socket = await socketTo(server.port, "/kept", { "x-principal": "owner" }); + const closed = closeOf(socket); + socket.close(); + await closed; + await new Promise((resolve) => setTimeout(resolve, 50)); + await expectCollected(...refs); + await server.close(); + }); +}); + +/** A raw TCP client upgraded to the plugin socket `path` through `host` as the owner. */ +async function rawSocket(port: number, host: string, path: string): Promise { + const socket = connect(port, "127.0.0.1"); + await once(socket, "connect"); + socket.on("error", () => {}); + socket.write( + `GET ${path} HTTP/1.1\r\nHost: ${host}\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Key: AAAAAAAAAAAAAAAAAAAAAA==\r\nSec-WebSocket-Version: 13\r\nx-principal: owner\r\n\r\n`, + ); + const [answer] = await once(socket, "data"); + expect(String(answer)).toContain("101 Switching Protocols"); + return socket; +} + +/** A masked client text frame. */ +function frame(text: string): Buffer { + const bytes = Buffer.from(text); + const mask = Buffer.from([1, 2, 3, 4]); + return Buffer.concat([ + Buffer.from([0x81, 0x80 | bytes.length]), + mask, + Buffer.from(bytes.map((byte, index) => byte ^ mask[index % 4]!)), + ]); +} diff --git a/packages/chord/test/next-node/plugins/build.test.ts b/packages/chord/test/next-node/plugins/build.test.ts index 623e85873..8525cfadf 100644 --- a/packages/chord/test/next-node/plugins/build.test.ts +++ b/packages/chord/test/next-node/plugins/build.test.ts @@ -222,7 +222,7 @@ describe("building a facet: the import table", () => { }); await writeFiles(root, { "base.ts": "export const base = 1;" }); await writePlugin(root, "copier", "@test/copier", { - "worker.ts": 'import { y } from "../shared/shared.ts";\nexport default { setup: () => y };', + "worker.ts": 'import { y } from "@test/shared/shared";\nexport default { setup: () => y };', }); await writePlugin(root, "shared", "@test/shared", { "shared.ts": 'import y from "some-package";\nexport { y };', @@ -231,7 +231,7 @@ describe("building a facet: the import table", () => { const out = plugins.find(({ name }) => name === "@test/out")!; expect(errors(await buildFacet(out, "worker", { ...NODE, plugins }, context))).toEqual([ 'worker.ts:1:22: "../base.ts": outside the plugin (base code is reached through a host package, packages through dependencies)', - 'worker.ts:2:15: "left-out": neither a host package nor in the dependencies of plugin "@test/out"', + 'worker.ts:2:15: "left-out": neither a host package, in the dependencies of plugin "@test/out" nor a discovered plugin "left-out"', ]); const copier = plugins.find(({ name }) => name === "@test/copier")!; expect(errors(await buildFacet(copier, "worker", { ...NODE, plugins }, context))).toEqual([ @@ -239,6 +239,43 @@ describe("building a facet: the import table", () => { ]); }); + it("imports another plugin's file by name from any root; refuses relative paths into it, names a missing plugin", async () => { + const first = await tempRoot(); + const second = await tempRoot(); + await writePlugin(second, "elsewhere", "@test/contracts", { + "contract.ts": 'export const name = "contract";', + "other.ts": 'import { name } from "./contract.ts";\nexport const other = name;', + }); + await writePlugin(first, "user", "@test/user", { + "worker.ts": [ + 'import { name } from "@test/contracts/contract";', + 'import { other } from "@test/contracts/other";', + "export default { setup: () => [name, other] };", + ].join("\n"), + }); + await writePlugin(first, "relative", "@test/relative", { + "worker.ts": 'import { name } from "../user/worker.ts";\nexport default { setup: () => name };', + }); + await writePlugin(first, "missing", "@test/missing", { + "worker.ts": 'import { name } from "@test/absent/contract";\nexport default { setup: () => name };', + }); + const { plugins } = await discoverPlugins([first, second], { kinds: KINDS }, context); + const build = (name: string) => + buildFacet(plugins.find((each) => each.name === name)!, "worker", { ...NODE, plugins }, context); + const user = await build("@test/user"); + if (user.kind === "failed") throw new Error(user.errors.join("\n")); + expect(user.copied.map((file) => file.slice(second.length + 1)).sort()).toEqual([ + "elsewhere/contract.ts", + "elsewhere/other.ts", + ]); + expect(errors(await build("@test/relative"))).toEqual([ + 'worker.ts:1:22: "../user/worker.ts": a relative path into plugin "@test/user" (import its files by name: "@test/user/…")', + ]); + expect(errors(await build("@test/missing"))).toEqual([ + 'worker.ts:1:22: "@test/absent/contract": neither a host package, in the dependencies of plugin "@test/missing" nor a discovered plugin "@test/absent"', + ]); + }); + it("returns a syntax error and a missing facet as failed results, never a throw", async () => { const root = await tempRoot(); await writePlugin(root, "syntax", "@test/syntax", { "worker.ts": "export default {" }); diff --git a/packages/chord/test/next-node/plugins/plugins.test.ts b/packages/chord/test/next-node/plugins/plugins.test.ts index b9c435d10..b202b3e82 100644 --- a/packages/chord/test/next-node/plugins/plugins.test.ts +++ b/packages/chord/test/next-node/plugins/plugins.test.ts @@ -12,7 +12,12 @@ import { createHost, defineFacet, type Host, type ReloadReport } from "../../../ import type { ChordHost } from "../../../src/next/host/internal.ts"; import { memoryLinkPair } from "../../../src/next/link/index.ts"; import { constant } from "../../../src/next/state/index.ts"; -import { createPlugins, type Plugins, type RolloutStep } from "../../../src/next-node/plugins/index.ts"; +import { + createPlugins, + type Plugins, + type RolloutHost, + type RolloutStep, +} from "../../../src/next-node/plugins/index.ts"; import { changesOf, delivered, @@ -22,7 +27,7 @@ import { transported, valuesOf, } from "../../next/support.ts"; -import { HOST_PACKAGES, KINDS, tempRoot, writeFiles, writePackage, writePlugin } from "./support.ts"; +import { HOST_PACKAGES, kindsOf, tempRoot, writeFiles, writePackage, writePlugin } from "./support.ts"; /** The fixtures' contracts, as the `contracts` fixture declares them. */ const Worker = hostKind("test.worker"); @@ -46,15 +51,20 @@ async function itemPlugin(root: string, dir: string, label: string): Promise reported.push(e), }, context, @@ -81,7 +91,11 @@ describe("per plugin at startup", () => { const { plugins } = component(root); const host = createHost({ onError: () => {} }); const core = defineFacet((h) => h.serve(Item, () => ({ label: constant("core") }))); - const report = await plugins.load(host, { kind: "worker", core: [{ name: "@app/core", facet: core }] }, context); + const report = await plugins.load( + host, + { kind: "worker", core: [{ name: "@app/core", facet: core }], modules: HOST_PACKAGES }, + context, + ); expect(report.failed.map(({ plugin, error }) => [plugin, (error as Error).message])).toEqual([ [ "@test/syntax", @@ -105,7 +119,7 @@ describe("per plugin at reload", () => { for (const dir of ["a", "b", "c"]) await itemPlugin(root, dir, `${dir}1`); const { plugins } = component(root); const host = createHost({ onError: () => {} }); - await plugins.load(host, { kind: "worker", core: [] }, context); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); const label = async (dir: string) => (await host.get(local(`test.${dir}`, { version: 1, params: {}, members: { label: str() } }), {}, context)) .label.value; @@ -180,7 +194,7 @@ describe("per plugin at reload", () => { await writePlugin(root, "consumer", "@test/consumer", { "worker.ts": consumer(1) }); const { plugins } = component(root); const host = createHost({ onError: () => {} }); - await plugins.load(host, { kind: "worker", core: [] }, context); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); const C = local("test.consumer", { version: 1, params: {}, members: { label: str() } }); const shown = await host.get(C, {}, context); await writeFiles(join(root, "provider"), { "worker.ts": `${provider(2)}\nexport const broken = ;` }); @@ -201,14 +215,19 @@ describe("per plugin at reload", () => { }); describe("rollout", () => { - /** A step recording its name into `order`, answering `report`, or rejecting with `error`. */ - const step = (name: string, order: string[], outcome: { report?: ReloadReport; error?: Error }): RolloutStep => ({ + /** A step recording its name into `order`, answering `report` for one host (or `hosts`), or rejecting with `error`. */ + const step = ( + name: string, + order: string[], + outcome: { report?: ReloadReport; hosts?: readonly RolloutHost[]; error?: Error }, + kinds: readonly string[] = ["worker"], + ): RolloutStep => ({ name, - kinds: ["worker"], + kinds, reload: async () => { order.push(name); if (outcome.error !== undefined) throw outcome.error; - return outcome.report!; + return { hosts: outcome.hosts ?? [{ host: name, report: outcome.report! }] }; }, }); const report = (failed: readonly string[]): ReloadReport => ({ @@ -244,6 +263,56 @@ describe("rollout", () => { plugins.close(); }); + it("counts a host's error as the step's built plugins failing there; a host stopped cleanly dirties nothing (D12)", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + await itemPlugin(root, "b", "b1"); + const { plugins } = component(root); + await plugins.build([], context); + await writeFile(join(root, "a", "worker.ts"), "// a\n"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + const stopped: RolloutHost[] = [{ host: "session|S1", stopped: true }]; + await plugins.rollout( + [step("workers", [], { hosts: stopped })], + await plugins.build(["@test/a"], context), + context, + ); + expect(plugins.state.value).toMatchObject({ dirty: [] }); + await writeFile(join(root, "a", "worker.ts"), "// a again\n"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + const crashed: RolloutHost[] = [ + { host: "session|S1", report: report([]) }, + { host: "session|S2", error: new Error("crashed") }, + ]; + const result = await plugins.rollout( + [step("workers", [], { hosts: crashed })], + await plugins.build(["@test/a"], context), + context, + ); + expect(result.steps).toEqual([{ name: "workers", hosts: crashed }]); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }); + plugins.close(); + }); + + it("keeps dirty a plugin whose built kind no step shipped", async () => { + const root = await tempRoot(); + await writePlugin(root, "a", "@test/a", { "worker.ts": "export default { setup: () => {} };", "server.ts": "" }); + const { plugins } = component(root); + await plugins.build([], context); + await writeFile(join(root, "a", "worker.ts"), "export default { setup: () => {} }; // edited\n"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + const built = await plugins.build(["@test/a"], context); + await plugins.rollout([step("worker", [], { report: report([]) })], built, context); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }); + await plugins.rollout( + [step("worker", [], { report: report([]) }), step("server", [], { report: report([]) }, ["server"])], + built, + context, + ); + expect(plugins.state.value).toMatchObject({ dirty: [] }); + plugins.close(); + }); + it("stops at a step that rejects, clearing no mark; a change during the build keeps its mark", async () => { const root = await tempRoot(); await itemPlugin(root, "a", "a1"); @@ -275,6 +344,38 @@ describe("rollout", () => { }); }); +describe("current sets (slice 10 D11)", () => { + it("scripts(kind): built on first use; a build changes no set; a rollout applies it right before a step of its kind", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + await writePlugin(root, "b", "@test/b", { "server.ts": "export default { setup: () => {} };" }); + const { plugins } = component(root); + const digests = async () => + (await plugins.scripts("worker", context)).map(({ plugin, built }) => [ + plugin.name, + built.kind === "script" ? built.digest.slice(0, 8) : built.kind, + ]); + const first = await digests(); + expect(first.map(([name]) => name)).toEqual(["@test/a"]); + await itemPlugin(root, "a", "a2"); + const built = await plugins.build(["@test/a"], context); + expect(await digests()).toEqual(first); + const rejecting: RolloutStep = { + name: "server", + kinds: ["server"], + reload: async () => { + throw new Error("closed"); + }, + }; + const shipped: RolloutStep = { name: "workers", kinds: ["worker"], reload: async () => ({ hosts: [] }) }; + await plugins.rollout([rejecting, shipped], built, context); + expect(await digests()).toEqual(first); + await plugins.rollout([shipped], built, context); + expect(await digests()).not.toEqual(first); + plugins.close(); + }); +}); + describe("installs through the component", () => { it("clears the plugin's install mark and dirties it", async () => { const root = await tempRoot(); @@ -285,8 +386,7 @@ describe("installs through the component", () => { const plugins = createPlugins( { roots: [root], - kinds: KINDS, - hostPackages: HOST_PACKAGES, + kinds: kindsOf(HOST_PACKAGES), install: { npmCommand: [process.execPath, join(tools, "npm.mjs")] }, onError: (error) => reported.push(error), }, @@ -321,12 +421,10 @@ describe("GC", () => { "export default defineFacet((host) => host.serve(Item, () => ({ label: constant(state.label) })));", ].join("\n"); await writePlugin(root, "gc", "@test/gc", { "worker.ts": source("a") }); - const plugins = createPlugins( - { roots: [root], kinds: KINDS, hostPackages: { ...HOST_PACKAGES, "@test/probe": probe }, onError: () => {} }, - context, - ); + const modules = { ...HOST_PACKAGES, "@test/probe": probe }; + const plugins = createPlugins({ roots: [root], kinds: kindsOf(modules), onError: () => {} }, context); const host = createHost({ onError: () => {} }); - await plugins.load(host, { kind: "worker", core: [] }, context); + await plugins.load(host, { kind: "worker", core: [], modules }, context); const first = await plugins.build(["@test/gc"], context); await writeFiles(join(root, "gc"), { "worker.ts": source("b") }); const second = await plugins.build(["@test/gc"], context); @@ -358,8 +456,8 @@ describe("end to end: a server and a worker", () => { { authorize: () => true, resolve: async () => ({ link: down, principal: { kind: "worker" } }) }, context, ); - await plugins.load(worker, { kind: "worker", core: [] }, context); - await plugins.load(server, { kind: "server", core: [] }, context); + await plugins.load(worker, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); + await plugins.load(server, { kind: "server", core: [], modules: HOST_PACKAGES }, context); const notes = await server.get(Notes, {}, context); const index = await server.get(Index, {}, context); expect([notes.label.value, index.label.value]).toEqual(["v1", "v1"]); @@ -370,7 +468,9 @@ describe("end to end: a server and a worker", () => { await writeFile(join(root, "notes", "label.ts"), 'export const label = "v2";\n'); await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/notes"] }), { timeout: 5_000 }); const rolled = await plugins.rollout(steps, await plugins.build(plugins.state.value.dirty, context), context); - expect(rolled.steps.map(({ name, report }) => [name, report.plugins.changed, report.objects])).toEqual([ + expect( + rolled.steps.map(({ name, hosts }) => [name, reportOf(hosts).plugins.changed, reportOf(hosts).objects]), + ).toEqual([ ["server", ["@test/notes"], [{ object: "test.index", kind: "swapped" }]], ["worker", ["@test/notes"], [{ object: "test.notes", kind: "swapped" }]], ]); @@ -383,7 +483,7 @@ describe("end to end: a server and a worker", () => { await writeFile(join(root, "notes", "label.ts"), "export const label = ;\n"); await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/notes"] }), { timeout: 5_000 }); const broken = await plugins.rollout(steps, await plugins.build(plugins.state.value.dirty, context), context); - expect(broken.steps.map(({ name, report }) => [name, failedNames(report)])).toEqual([ + expect(broken.steps.map(({ name, hosts }) => [name, failedNames(reportOf(hosts))])).toEqual([ ["server", ["@test/notes"]], ["worker", ["@test/notes"]], ]); @@ -403,7 +503,7 @@ describe("fix round 1", () => { await itemPlugin(root, "b", "b1"); const { plugins } = component(root); const host = createHost({ onError: () => {} }); - await plugins.load(host, { kind: "worker", core: [] }, context); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); await itemPlugin(root, "a", "a2"); const builtA = await plugins.build(["@test/a"], context); await itemPlugin(root, "b", "b2"); @@ -424,7 +524,7 @@ describe("fix round 1", () => { await itemPlugin(root, "b", "b1"); const { plugins } = component(root); const host = createHost({ onError: () => {} }); - await plugins.load(host, { kind: "worker", core: [] }, context); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); await itemPlugin(root, "a", "a2"); const first = await plugins.build(["@test/a"], context); await rm(join(root, "b"), { recursive: true }); @@ -444,7 +544,7 @@ describe("fix round 1", () => { await itemPlugin(root, "b", "b1"); const { plugins } = component(root); const host = createHost({ onError: () => {} }); - await plugins.load(host, { kind: "worker", core: [] }, context); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); const Item = local("test.a", { version: 1, params: {}, members: { label: str() } }); const item = await host.get(Item, {}, context); await itemPlugin(root, "a", "a2"); @@ -476,7 +576,7 @@ describe("fix round 1", () => { await itemPlugin(root, "a", "a1"); const { plugins } = component(root); const host = createHost({ onError: () => {} }); - await plugins.load(host, { kind: "worker", core: [] }, context); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); // The plugin keeps a server facet only. await rm(join(root, "a", "worker.ts")); await writeFiles(join(root, "a"), { @@ -500,7 +600,7 @@ describe("fix round 1", () => { await itemPlugin(root, "a", "a1"); const { plugins } = component(root); const host = createHost({ onError: () => {} }); - await plugins.load(host, { kind: "worker", core: [] }, context); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); const item = await host.get(local("test.a", { version: 1, params: {}, members: { label: str() } }), {}, context); await itemPlugin(root, "a", "a2"); await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); @@ -526,8 +626,7 @@ describe("fix round 1", () => { const plugins = createPlugins( { roots: [root], - kinds: KINDS, - hostPackages: HOST_PACKAGES, + kinds: kindsOf(HOST_PACKAGES), install: { npmCommand: [process.execPath, join(tools, "npm.mjs")] }, onError: () => {}, }, @@ -567,12 +666,10 @@ describe("the reload queues and garbage collection", () => { "});", ].join("\n"); const dir = await writePlugin(root, "failed", "@test/failed", { "worker.ts": source(false) }); - const plugins = createPlugins( - { roots: [root], kinds: KINDS, hostPackages: { ...HOST_PACKAGES, "@test/probe": probe }, onError: () => {} }, - context, - ); + const modules = { ...HOST_PACKAGES, "@test/probe": probe }; + const plugins = createPlugins({ roots: [root], kinds: kindsOf(modules), onError: () => {} }, context); const host = createHost({ onError: () => {} }); - await plugins.load(host, { kind: "worker", core: [] }, context); + await plugins.load(host, { kind: "worker", core: [], modules }, context); await writeFiles(dir, { "worker.ts": source(true) }); const built = await plugins.build(["@test/failed"], context); // The report (its failure's error holds the new facet's closure) is dropped here; the queues must not keep it. diff --git a/packages/chord/test/next-node/plugins/runtime.test.ts b/packages/chord/test/next-node/plugins/runtime.test.ts new file mode 100644 index 000000000..52b850d10 --- /dev/null +++ b/packages/chord/test/next-node/plugins/runtime.test.ts @@ -0,0 +1,126 @@ +// Tests of the plugin runtime (slice 10 §3, D11): loading and reloading hosts with complete sets of shipped scripts +// (unchanged digests kept as the same facet, removed = loaded minus shipped, failed builds and evaluations kept old), +// and that importing it loads no esbuild. + +import { execFileSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { local, str } from "../../../src/next/contracts/index.ts"; +import { createHost } from "../../../src/next/host/index.ts"; +import { + buildFacet, + createPluginRuntime, + discoverPlugins, + type PluginScript, +} from "../../../src/next-node/plugins/index.ts"; +import { HOST_PACKAGES, tempRoot, writePlugin } from "./support.ts"; + +/** Writes plugin `@test/` serving `test.` labelled `label` from its worker facet (or `source` as it is). */ +async function plugin(root: string, dir: string, label: string, source?: string): Promise { + await writePlugin(root, dir, `@test/${dir}`, { + "worker.ts": + source ?? + [ + 'import { constant, defineFacet, local, str } from "@earendil-works/chord/next";', + `const Item = local("test.${dir}", { version: 1, params: {}, members: { label: str() } });`, + `export default defineFacet((host) => host.serve(Item, () => ({ label: constant(${JSON.stringify(label)}) })));`, + ].join("\n"), + }); +} + +/** The worker scripts of every plugin under `root`, built now: what the server would ship. */ +async function shipped(root: string): Promise { + const { plugins } = await discoverPlugins([root], { kinds: ["worker"] }, context); + const options = { hostPackages: Object.keys(HOST_PACKAGES), platform: "node", plugins } as const; + return Promise.all( + plugins.map(async (each) => ({ plugin: each, built: await buildFacet(each, "worker", options, context) })), + ); +} + +/** The label `test.` shows on `host`. */ +async function label(host: ReturnType, dir: string): Promise { + const Item = local(`test.${dir}`, { version: 1, params: {}, members: { label: str() } }); + return (await host.get(Item, {}, context)).label.value; +} + +describe("loading and reloading with complete sets", () => { + it("loads a set and leaves out a plugin whose build or evaluation failed, reported with the host's report", async () => { + const root = await tempRoot(); + await plugin(root, "a", "a1"); + await plugin(root, "broken", "", "export const = ;"); + await plugin(root, "throws", "", 'throw new Error("at import");'); + const runtime = createPluginRuntime(); + const host = createHost({ onError: () => {} }); + const report = await runtime.load( + host, + { kind: "worker", core: [], scripts: await shipped(root), modules: HOST_PACKAGES }, + context, + ); + expect(report.failed.map(({ plugin }) => plugin).sort()).toEqual(["@test/broken", "@test/throws"]); + expect(await label(host, "a")).toBe("a1"); + await host.close(context); + }); + + it("reloads a complete set: same digests unchanged, a changed one applied, a loaded one not shipped removed", async () => { + const root = await tempRoot(); + await plugin(root, "a", "a1"); + await plugin(root, "b", "b1"); + await plugin(root, "c", "c1"); + const runtime = createPluginRuntime(); + const host = createHost({ onError: () => {} }); + await runtime.load( + host, + { kind: "worker", core: [], scripts: await shipped(root), modules: HOST_PACKAGES }, + context, + ); + await plugin(root, "b", "b2"); + const next = (await shipped(root)).filter(({ plugin }) => plugin.name !== "@test/c"); + const report = await runtime.reload(host, next, context); + expect(report.plugins).toEqual({ added: [], changed: ["@test/b"], removed: ["@test/c"] }); + expect([await label(host, "a"), await label(host, "b")]).toEqual(["a1", "b2"]); + await host.close(context); + }); + + it("keeps the loaded facet of a plugin whose new build or evaluation failed; an added failing one is left out", async () => { + const root = await tempRoot(); + await plugin(root, "a", "a1"); + const runtime = createPluginRuntime(); + const host = createHost({ onError: () => {} }); + await runtime.load( + host, + { kind: "worker", core: [], scripts: await shipped(root), modules: HOST_PACKAGES }, + context, + ); + await plugin(root, "a", "", "export const = ;"); + await plugin(root, "new", "", 'throw new Error("at import");'); + const report = await runtime.reload(host, await shipped(root), context); + expect(report.failed.map(({ plugin }) => plugin).sort()).toEqual(["@test/a", "@test/new"]); + expect(report.plugins).toEqual({ added: [], changed: [], removed: [] }); + expect(await label(host, "a")).toBe("a1"); + await host.close(context); + }); +}); + +describe("the runtime's imports", () => { + it("loads no esbuild: a worker that only evaluates never imports it", () => { + const runtime = fileURLToPath(new URL("../../../src/next-node/plugins/runtime.ts", import.meta.url)); + // A hook refusing esbuild, then the runtime imported. + const script = [ + 'import { registerHooks } from "node:module";', + "registerHooks({ resolve(specifier, context, next) {", + ' if (specifier === "esbuild") throw new Error("esbuild imported");', + " return next(specifier, context);", + "} });", + `const { createPluginRuntime } = await import(${JSON.stringify(runtime)});`, + "createPluginRuntime();", + 'console.log("ok");', + ].join("\n"); + const output = execFileSync( + process.execPath, + ["--experimental-strip-types", "--no-warnings", "--input-type=module", "--eval", script], + { encoding: "utf8" }, + ); + expect(output.trim()).toBe("ok"); + }); +}); diff --git a/packages/chord/test/next-node/plugins/support.ts b/packages/chord/test/next-node/plugins/support.ts index 86e444156..f4c1de6fc 100644 --- a/packages/chord/test/next-node/plugins/support.ts +++ b/packages/chord/test/next-node/plugins/support.ts @@ -7,7 +7,7 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { afterEach } from "vitest"; import * as chordNext from "../../../src/next/index.ts"; -import type { HostPackages } from "../../../src/next-node/plugins/index.ts"; +import type { HostPackages, KindDefinition } from "../../../src/next-node/plugins/index.ts"; /** The fixture plugins' directory. */ export const FIXTURES = fileURLToPath(new URL("../fixtures/plugins/", import.meta.url)); @@ -18,6 +18,16 @@ export const HOST_PACKAGES: HostPackages = { "@earendil-works/chord/next": chord /** The facet kinds of the fixtures' app. */ export const KINDS = ["server", "worker", "web"] as const; +/** The fixtures' app's kinds as the component takes them: Node servers and workers, a browser web, `modules`' names. */ +export function kindsOf(modules: HostPackages): Readonly> { + const hostPackages = Object.keys(modules); + return { + server: { platform: "node", hostPackages }, + worker: { platform: "node", hostPackages }, + web: { platform: "browser", hostPackages }, + }; +} + const made: string[] = []; afterEach(async () => { diff --git a/packages/chord/test/next-node/workers/control.test.ts b/packages/chord/test/next-node/workers/control.test.ts new file mode 100644 index 000000000..a7054269a --- /dev/null +++ b/packages/chord/test/next-node/workers/control.test.ts @@ -0,0 +1,55 @@ +// Tests of the control channel's crossings (slice 10 §3): errors cross as `{ message, code?, reason? }` and back, a +// reload report with its errors and parked causes. + +import { describe, expect, it } from "vitest"; +import { ChordError, isChordError } from "../../../src/next/data/index.ts"; +import { + controlOf, + crossed, + crossedReport, + envelope, + uncrossed, + uncrossedReport, +} from "../../../src/next-node/workers/control.ts"; + +describe("errors", () => { + it("cross as message, code and reason, never a stack or cause; come back as ChordErrors when they had a code", () => { + const chord = new ChordError({ code: "unavailable", message: "gone" }, { cause: new Error("inner") }); + expect(crossed(chord)).toEqual({ message: "gone", code: "unavailable" }); + expect(crossed(new Error("plain"))).toEqual({ message: "plain" }); + expect(crossed("text")).toEqual({ message: "text" }); + const back = uncrossed({ message: "gone", code: "released", reason: "closed" }); + expect(isChordError(back, "released") && back.reason).toBe("closed"); + expect(uncrossed({ message: "odd", code: "no-such-code" })).toMatchObject({ message: "odd" }); + }); + + it("crosses a report's failures and parked causes and back", () => { + const report = crossedReport({ + plugins: { added: ["@a/x"], changed: [], removed: [] }, + failed: [{ plugin: "@a/y", error: new Error("broken") }], + objects: [ + { object: "a.one", kind: "swapped" }, + { + object: "a.two", + kind: "parked", + cause: { kind: "waiting", error: new ChordError({ code: "unavailable", message: "w" }) }, + }, + ], + }); + expect(JSON.parse(JSON.stringify(report))).toEqual(report); + const back = uncrossedReport(report); + expect(back.failed[0]!.error).toBeInstanceOf(Error); + expect(back.objects[1]).toMatchObject({ + kind: "parked", + cause: { kind: "waiting", error: { code: "unavailable" } }, + }); + }); +}); + +describe("envelopes", () => { + it("tells control messages from anything else on the channel", () => { + expect(controlOf(envelope({ kind: "started" }))).toEqual({ kind: "started" }); + expect(controlOf(new Uint8Array(1))).toBeUndefined(); + expect(controlOf({ kind: "started" })).toBeUndefined(); + }); +}); diff --git a/packages/chord/test/next-node/workers/lock.test.ts b/packages/chord/test/next-node/workers/lock.test.ts new file mode 100644 index 000000000..de1564720 --- /dev/null +++ b/packages/chord/test/next-node/workers/lock.test.ts @@ -0,0 +1,62 @@ +// Tests of process locks (slice 10 §3): one holder at a time, a waiter taking the lock when the holder releases it +// (told by its connection's close, no polling), a waiter's abort. + +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withAbortSignal } from "../../../src/context/index.ts"; +import { processLock } from "../../../src/next-node/workers/index.ts"; +import { lockAt } from "../../../src/next-node/workers/lock.ts"; + +describe("processLock", () => { + it("makes a second taker wait until the holder releases, then takes it", async () => { + const name = `chord-test-lock-${process.pid}-${Date.now()}`; + const first = await processLock(name, context); + let taken = false; + const second = processLock(name, context).then((lock) => { + taken = true; + return lock; + }); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(taken).toBe(false); + const released = Date.now(); + await first.release(); + await (await second).release(); + expect(taken).toBe(true); + expect(Date.now() - released).toBeLessThan(100); + }); + + it("stops waiting when the waiter's context aborts", async () => { + const name = `chord-test-lock-abort-${process.pid}-${Date.now()}`; + const held = await processLock(name, context); + const abort = new AbortController(); + const waiting = processLock(name, withAbortSignal(abort.signal, context)); + abort.abort(new Error("gave up")); + await expect(waiting).rejects.toThrow("gave up"); + await held.release(); + await (await processLock(name, context)).release(); + }); +}); + +describe("socket file locks (macOS, the BSDs)", () => { + it("fail closed on a stale lock file, naming it: two takers never both take it over (Sol)", async () => { + const dir = await mkdtemp(join(tmpdir(), "chord-lock-file-")); + const address = join(dir, "lock.sock"); + // What a crashed holder leaves: a path nobody accepts on. + await writeFile(address, ""); + const takers = [lockAt({ kind: "file", address }, context), lockAt({ kind: "file", address }, context)]; + for (const taker of takers) await expect(taker).rejects.toThrow(`lock: ${address} is a stale lock file`); + await rm(dir, { recursive: true, force: true }); + }); + + it("are taken, waited for and released like the others", async () => { + const dir = await mkdtemp(join(tmpdir(), "chord-lock-file-")); + const address = join(dir, "lock.sock"); + const first = await lockAt({ kind: "file", address }, context); + const second = lockAt({ kind: "file", address }, context); + await first.release(); + await (await second).release(); + await rm(dir, { recursive: true, force: true }); + }); +}); diff --git a/packages/chord/test/next-node/workers/main.test.ts b/packages/chord/test/next-node/workers/main.test.ts new file mode 100644 index 000000000..017c9df04 --- /dev/null +++ b/packages/chord/test/next-node/workers/main.test.ts @@ -0,0 +1,87 @@ +// Tests of the worker's side (slice 10 §3): the start message, the lock, `started` on `connect`, `ready`; shutdown +// during startup, the parent's disconnect and SIGTERM each run the closers and exit; `fatal` redacted. + +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { hostKind, str } from "../../../src/next/contracts/index.ts"; +import type { FromWorker } from "../../../src/next-node/workers/control.ts"; +import { type Exit, WorkerProcess } from "../../../src/next-node/workers/process.ts"; +import type { WorkerBehaviour } from "../fixtures/workers/control.ts"; +import { until } from "./support.ts"; + +const ENTRY = fileURLToPath(new URL("../fixtures/workers/worker.ts", import.meta.url)); + +const Session = hostKind("session", { id: str() }); + +/** The fixture worker forked and started with `behaviour`; its control messages and exit collected. */ +function started(behaviour: WorkerBehaviour) { + const control: FromWorker[] = []; + let exit: Exit | undefined; + const child = new WorkerProcess( + { + entry: ENTRY, + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + onOutput: () => {}, + }, + { + control: (message) => control.push(message), + linkLost: () => {}, + linkClosed: () => {}, + exited: (value) => { + exit = value; + }, + }, + ); + child.send({ + kind: "start", + address: Session({ id: "S1" }), + start: behaviour, + scripts: [], + lock: `chord-test-main:${process.pid}:${Date.now()}:${Math.random()}`, + }); + return { child, control, exited: () => exit }; +} + +describe("the worker's side", () => { + it("sends started when the app connects, then ready with its load report and idle state", async () => { + const { child, control } = started({}); + await until(() => control.some(({ kind }) => kind === "ready")); + expect(control.map(({ kind }) => kind)).toEqual(["started", "ready"]); + expect(control[1]).toMatchObject({ idle: true, report: { plugins: { added: [] }, failed: [] } }); + child.kill(); + }); + + it("shuts down during its startup: aborts it, runs the closers registered so far in reverse order, exits 0", async () => { + const dir = await mkdtemp(join(tmpdir(), "chord-main-")); + const { child, control, exited } = started({ hang: true, log: join(dir, "log") }); + await until(() => control.some(({ kind }) => kind === "started")); + child.send({ kind: "shutdown" }); + await until(() => exited() !== undefined); + expect(exited()).toEqual({ code: 0, signal: null }); + expect(await readFile(join(dir, "log"), "utf8")).toBe("store\nhost\n"); + await rm(dir, { recursive: true, force: true }); + }); + + it("shuts down on SIGTERM and on the parent's disconnect", async () => { + const term = started({}); + await until(() => term.control.some(({ kind }) => kind === "ready")); + process.kill(term.child.pid, "SIGTERM"); + await until(() => term.exited() !== undefined); + expect(term.exited()).toEqual({ code: 0, signal: null }); + const gone = started({}); + await until(() => gone.control.some(({ kind }) => kind === "ready")); + gone.child.link.close(); + await until(() => gone.exited() !== undefined); + expect(gone.exited()).toEqual({ code: 0, signal: null }); + }); + + it("sends fatal with the redacted error and exits 1", async () => { + const { control, exited } = started({ fatal: "no secret-key-42 here" }); + await until(() => exited() !== undefined); + expect(control).toEqual([{ kind: "fatal", error: { message: "no [redacted] here" } }]); + expect(exited()?.code).toBe(1); + }); +}); diff --git a/packages/chord/test/next-node/workers/process.test.ts b/packages/chord/test/next-node/workers/process.test.ts new file mode 100644 index 000000000..b84589f6e --- /dev/null +++ b/packages/chord/test/next-node/workers/process.test.ts @@ -0,0 +1,46 @@ +// Tests of worker processes (slice 10 §3): forked with their output piped (the stderr tail kept), the link made at the +// fork (an early frame queued), control messages told, and the whole tree killed. + +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import type { FromWorker } from "../../../src/next-node/workers/control.ts"; +import { WorkerProcess } from "../../../src/next-node/workers/process.ts"; +import { until } from "./support.ts"; + +const CHILD = fileURLToPath(new URL("../fixtures/workers/child.ts", import.meta.url)); + +/** Whether process `pid` lives. */ +function alive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +describe("a worker process", () => { + it("pipes its output, keeps its stderr tail, queues an early frame on its link, tells control messages", async () => { + const output: string[] = []; + const control: FromWorker[] = []; + const child = new WorkerProcess( + { + entry: CHILD, + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + onOutput: (stream, chunk) => output.push(`${stream}: ${Buffer.from(chunk).toString().trim()}`), + }, + { control: (message) => control.push(message), linkLost: () => {}, linkClosed: () => {}, exited: () => {} }, + ); + await until(() => control.length > 0 && output.length >= 2); + const frames: number[][] = []; + child.link.onMessage((message) => frames.push([...message])); + expect(frames).toEqual([[1, 2, 3]]); + expect(control).toEqual([{ kind: "started" }]); + expect(child.stderrTail).toBe("an error line\n"); + const grandchild = Number(/grandchild (\d+)/.exec(output.join("\n"))![1]); + expect(alive(grandchild)).toBe(true); + child.kill(); + expect(await child.exited).toEqual({ code: null, signal: "SIGKILL" }); + await until(() => !alive(grandchild), 2_000); + }); +}); diff --git a/packages/chord/test/next-node/workers/support.ts b/packages/chord/test/next-node/workers/support.ts new file mode 100644 index 000000000..4931905d5 --- /dev/null +++ b/packages/chord/test/next-node/workers/support.ts @@ -0,0 +1,151 @@ +// Test support for the workers component (slice 10 §3, §4): a server routing `Session` to forked fixture workers, the +// fixture plugins copied into a temporary root, each address's start data (its behaviour) chosen by the test. + +import { cp, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import type { JsonValue } from "../../../src/json.ts"; +import { hostKind, remote, str } from "../../../src/next/contracts/index.ts"; +import { createHost, type Host, type HostError } from "../../../src/next/host/index.ts"; +import * as chordNext from "../../../src/next/index.ts"; +import { createPlugins, type Plugins } from "../../../src/next-node/plugins/index.ts"; +import { createWorkers, type Workers, type WorkersError } from "../../../src/next-node/workers/index.ts"; +import { Session, type WorkerBehaviour } from "../fixtures/workers/control.ts"; + +/** The server's own kind. */ +export const Server = hostKind("server"); + +/** The fixture plugin's notes at a worker. */ +export const Notes = remote("test.notes", { + version: 1, + params: { id: str() }, + hostAddress: ({ id }) => Session({ id }), + variants: { main: { state: { label: str() } } }, +}); + +/** The principals of these tests. */ +export type TestPrincipal = + | { readonly kind: "server" } + | { readonly kind: "worker"; readonly id: string } + | { readonly kind: "guest"; readonly id: string }; + +const ENTRY = fileURLToPath(new URL("../fixtures/workers/worker.ts", import.meta.url)); + +const PLUGINS = fileURLToPath(new URL("../fixtures/workers/plugins/", import.meta.url)); + +const made: { readonly root: string; readonly close: () => Promise }[] = []; + +afterEach(async () => { + for (const { root, close } of made.splice(0)) { + await close(); + await rm(root, { recursive: true, force: true }); + } +}); + +/** A server with workers; closed after the test. */ +export type Fleet = { + readonly server: Host; + readonly workers: Workers; + readonly plugins: Plugins; + /** The copy of the fixture plugins. */ + readonly root: string; + readonly errors: WorkersError[]; + readonly hostErrors: HostError[]; + /** The addresses `start` ran for, in order. */ + readonly starts: string[]; + /** The workers' stdout lines, each with its address key. */ + readonly output: string[]; + /** Each address's behaviour (its start data); none: a plain worker. */ + readonly behaviours: Map; + /** Writes `label` into the copy's notes plugin. */ + label(label: string): Promise; +}; + +/** Starts a server routing `Session` to fixture workers; `options` override the workers' defaults. */ +export async function fleet( + options: { + readonly idleMs?: number; + readonly heartbeat?: { readonly ping?: number; readonly close?: number }; + readonly start?: (address: { readonly key: string }, context: Context) => Promise; + readonly principal?: () => TestPrincipal; + readonly authorize?: () => boolean; + } = {}, +): Promise { + const root = await mkdtemp(join(tmpdir(), "chord-workers-")); + await cp(PLUGINS, root, { recursive: true }); + const errors: WorkersError[] = []; + const hostErrors: HostError[] = []; + const starts: string[] = []; + const behaviours = new Map(); + const output: string[] = []; + const modules = { "@earendil-works/chord/next": chordNext }; + const plugins = createPlugins( + { roots: [root], kinds: { worker: { platform: "node", hostPackages: Object.keys(modules) } }, onError: () => {} }, + context, + ); + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + onError: (error) => hostErrors.push(error), + ...(options.heartbeat === undefined ? {} : { heartbeat: options.heartbeat }), + }); + const workers = createWorkers( + server, + Session, + { + kind: "worker", + entry: ENTRY, + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + authorize: options.authorize ?? (() => true), + principal: options.principal ?? ((address) => ({ kind: "worker", id: address.params.id }) as const), + start: async (address, startContext) => { + starts.push(address.key); + if (options.start !== undefined) return options.start(address, startContext); + return (behaviours.get(address.params.id) ?? {}) as JsonValue; + }, + lock: (address) => `chord-test:${root}:${address.key}`, + idleMs: () => options.idleMs ?? 60_000, + plugins, + onOutput: (address, stream, chunk) => { + if (stream === "stdout") output.push(`${address.key} ${Buffer.from(chunk).toString().trim()}`); + }, + onError: (error) => errors.push(error), + }, + context, + ); + await server.load([], context); + // The worker kind's set made now (discovery, builds): the watcher marks edits from then on. + await plugins.scripts("worker", context); + const close = async () => { + await workers.close(context); + await server.close(context); + plugins.close(); + }; + made.push({ root, close }); + return { + server, + workers, + plugins, + root, + errors, + hostErrors, + starts, + output, + behaviours, + label: (label) => writeFile(join(root, "notes", "label.ts"), `export const label = ${JSON.stringify(label)};\n`), + }; +} + +/** Waits until `ready()` holds, polling every 10 ms, for at most `ms`. */ +export async function until(ready: () => boolean, ms = 10_000): Promise { + const deadline = Date.now() + ms; + while (!ready()) { + if (Date.now() > deadline) throw new Error("until: timed out"); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +export { context, Session }; diff --git a/packages/chord/test/next-node/workers/workers.test.ts b/packages/chord/test/next-node/workers/workers.test.ts new file mode 100644 index 000000000..d7c980784 --- /dev/null +++ b/packages/chord/test/next-node/workers/workers.test.ts @@ -0,0 +1,605 @@ +// Tests of the workers component (slice 10 §3, §4): starts on demand (the link made at the fork, `started` on +// `connect`, the start timer covering only the start), failures that stay (a crash, a `fatal`, a lost link, a failed +// start: no fork for a new hold, holders `failed { waiting }`) until `retry` or a rollout, idle stops and `busy`, `stop` +// and its suspension, the per-address queue, `close`, the rollout step, and nothing kept of 1,000 addresses. + +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { withAbortSignal } from "../../../src/context/index.ts"; +import { createHost } from "../../../src/next/host/index.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import type { RolloutHost } from "../../../src/next-node/plugins/index.ts"; +import { expectCollected } from "../../next/support.ts"; +import { Control } from "../fixtures/workers/control.ts"; +import { context, fleet, Notes, Session, type TestPrincipal, until } from "./support.ts"; + +/** A step's hosts as `[host, what]`. */ +function listed(hosts: readonly RolloutHost[]) { + return hosts.map((entry) => [ + entry.host, + "report" in entry ? entry.report.plugins : "stopped" in entry ? "stopped" : (entry.error as Error).message, + ]); +} + +/** The state entry of `id`. */ +const stateOf = (state: unknown, id: string) => + (state as Record)[`session|${id}`]; + +describe("starts on demand", () => { + it("starts a worker for the first hold: starting, then running; its link binds the holder", async () => { + const { server, workers, starts } = await fleet(); + const notes = server.hold(Notes, { id: "S1" }, context); + await until(() => stateOf(workers.state.value, "S1") !== undefined); + expect(stateOf(workers.state.value, "S1")?.status).toBe("starting"); + await until(() => notes.state.value.kind === "live"); + expect(notes.label.value).toBe("S1 v1"); + await until(() => stateOf(workers.state.value, "S1")?.status === "running"); + expect(starts).toEqual(["session|S1"]); + notes.release(); + }); + + it("binds through a load longer than the heartbeat: the link is made at the fork, the worker connects first", async () => { + const { server, behaviours } = await fleet({ heartbeat: { ping: 50, close: 200 } }); + behaviours.set("S1", { loadMs: 600 }); + const notes = await server.get(Notes, { id: "S1" }, context); + expect(notes.label.value).toBe("S1 v1"); + notes.release(); + }); +}); + +describe("failures stay until retry or a rollout (R2)", () => { + it("a start that throws fails the address: holders fail waiting with it, a new hold forks nothing; retry starts it", async () => { + let fail = true; + const { server, workers, errors, starts } = await fleet({ + start: async () => { + if (fail) throw new Error("no data dir"); + return {}; + }, + }); + const notes = server.hold(Notes, { id: "S1" }, context); + await until(() => notes.state.value.kind === "failed"); + expect(notes.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "unavailable", message: "worker failed to start" } }, + }); + expect(stateOf(workers.state.value, "S1")).toMatchObject({ + status: "failed", + error: { message: "worker failed to start" }, + }); + const again = server.hold(Notes, { id: "S1" }, context); + await until(() => again.state.value.kind === "failed"); + expect(starts).toEqual(["session|S1"]); + // The full error only for the app's logs (decision "Slice 10 fix round, worker errors in public"). + expect(errors).toMatchObject([ + { kind: "worker_failed", address: "session|S1", error: { message: "no data dir" } }, + ]); + fail = false; + await workers.retry(Session({ id: "S1" }), context); + await until(() => notes.state.value.kind === "live" && again.state.value.kind === "live"); + expect(stateOf(workers.state.value, "S1")?.status).not.toBe("failed"); + notes.release(); + again.release(); + }); + + it("a crash after ready stays failed (worker_exited, worker_failed), nothing restarts", async () => { + const { server, workers, errors, starts } = await fleet(); + const notes = await server.get(Notes, { id: "S1" }, context); + const control = await server.get(Control, { id: "S1" }, context); + await control.crash(7, context); + await until(() => stateOf(workers.state.value, "S1")?.status === "failed"); + expect(stateOf(workers.state.value, "S1")?.error?.message).toBe("worker exited (code 7)"); + expect(errors.map(({ kind }) => kind)).toEqual(["worker_exited", "worker_failed"]); + await until(() => notes.state.value.kind === "failed"); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(starts).toEqual(["session|S1"]); + notes.release(); + control.release(); + }); + + it("a fatal crosses redacted: the secret never reaches the public state", async () => { + const { server, workers, errors, behaviours } = await fleet(); + behaviours.set("S1", { fatal: "boom secret-token-123" }); + const notes = server.hold(Notes, { id: "S1" }, context); + await until(() => stateOf(workers.state.value, "S1")?.status === "failed"); + expect(stateOf(workers.state.value, "S1")?.error?.message).toBe("boom [redacted]"); + // Public: the state; the report's stderr tail is the app's to redact (a later test). + expect(JSON.stringify(workers.state.value)).not.toContain("secret"); + expect(errors.find(({ kind }) => kind === "worker_failed")).toMatchObject({ + error: { message: "boom [redacted]" }, + }); + notes.release(); + }); + + it("a link lost while its child lives (a blocked event loop past the heartbeat): killed after the grace, failed", async () => { + const { server, workers, starts, errors } = await fleet({ heartbeat: { ping: 100, close: 300 } }); + const notes = await server.get(Notes, { id: "S1" }, context); + const control = await server.get(Control, { id: "S1" }, context); + await control.block(20_000, context); + await until(() => stateOf(workers.state.value, "S1")?.status === "stopping", 5_000); + await until(() => stateOf(workers.state.value, "S1")?.status === "failed", 10_000); + expect(stateOf(workers.state.value, "S1")?.error?.message).toBe("worker exited (SIGKILL)"); + expect(errors.find(({ kind }) => kind === "worker_failed")).toMatchObject({ + error: { message: "worker: link lost" }, + }); + expect(starts).toEqual(["session|S1"]); + notes.release(); + control.release(); + }, 20_000); + + it("an exit before started fails the start; a worker that never connects is killed at the start timer", async () => { + const { server, workers, behaviours, errors } = await fleet(); + behaviours.set("S1", { exit: 3 }); + const exited = server.hold(Notes, { id: "S1" }, context); + await until(() => stateOf(workers.state.value, "S1")?.status === "failed"); + expect(stateOf(workers.state.value, "S1")?.error?.message).toBe("worker failed to start"); + expect(errors.find(({ kind }) => kind === "worker_failed")).toMatchObject({ + error: { message: expect.stringContaining("exited before it was ready (code 3)") }, + }); + exited.release(); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + behaviours.set("S2", { connect: false }); + const silent = server.hold(Notes, { id: "S2" }, context); + await vi.waitFor(() => expect(stateOf(workers.state.value, "S2")?.status).toBe("starting"), { + timeout: 5_000, + }); + await vi.advanceTimersByTimeAsync(60_000); + await vi.waitFor(() => expect(stateOf(workers.state.value, "S2")?.status).toBe("failed"), { timeout: 5_000 }); + expect(stateOf(workers.state.value, "S2")?.error?.message).toBe("worker failed to start"); + expect(errors.findLast(({ kind }) => kind === "worker_failed")).toMatchObject({ + error: { message: expect.stringContaining("did not start within 60 s") }, + }); + silent.release(); + } finally { + vi.useRealTimers(); + } + }); + + it("the start timer covers only the start: a slow load after connecting is not killed (Q4)", async () => { + const { server, workers, behaviours } = await fleet(); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + behaviours.set("S1", { hang: true }); + const notes = server.hold(Notes, { id: "S1" }, context); + await vi.waitFor(() => expect(stateOf(workers.state.value, "S1")?.status).toBe("starting"), { + timeout: 5_000, + }); + await new Promise((resolve) => setImmediate(resolve)); + await vi.advanceTimersByTimeAsync(120_000); + expect(stateOf(workers.state.value, "S1")?.status).toBe("starting"); + expect(notes.state.value.kind).toBe("loading"); + notes.release(); + } finally { + vi.useRealTimers(); + } + }); +}); + +describe("keep-alive (D9, R3, Q5)", () => { + it("stops a worker idle since idleMs; a hold keeps one running; busy keeps it too", async () => { + const { server, workers } = await fleet({ idleMs: 200 }); + const held = await server.get(Notes, { id: "held" }, context); + const released = await server.get(Notes, { id: "free" }, context); + const control = await server.get(Control, { id: "busy" }, context); + await control.busy(1_500, context); + control.release(); + released.release(); + await until(() => stateOf(workers.state.value, "free") === undefined, 5_000); + await new Promise((resolve) => setTimeout(resolve, 600)); + expect(stateOf(workers.state.value, "held")?.status).toBe("running"); + expect(stateOf(workers.state.value, "busy")?.status).toBe("running"); + await until(() => stateOf(workers.state.value, "busy") === undefined, 5_000); + held.release(); + }, 20_000); + + it("1,000 sequential local calls in a worker nobody holds send no message (Sol)", async () => { + const { server, behaviours, output } = await fleet(); + behaviours.set("S1", { localCalls: 1_000 }); + // Released at once: the calls run 300 ms after the load, with no hold arriving (no reference keeps it busy). + (await server.get(Notes, { id: "S1" }, context)).release(); + await until(() => output.some((line) => line.startsWith("session|S1 local calls sent"))); + expect(output).toContain("session|S1 local calls sent 0"); + }); +}); + +describe("stop", () => { + it("suspends while held: holders fail waiting suspended, nothing starts; forgotten with the last holder (Q3)", async () => { + const { server, workers, starts } = await fleet(); + const notes = await server.get(Notes, { id: "S1" }, context); + await workers.stop(Session({ id: "S1" }), context); + expect(stateOf(workers.state.value, "S1")).toBeUndefined(); + await until(() => notes.state.value.kind === "failed"); + expect(notes.state.value).toMatchObject({ + cause: { kind: "waiting", error: { message: "session|S1: suspended" } }, + }); + expect(starts).toEqual(["session|S1"]); + notes.release(); + await new Promise((resolve) => setTimeout(resolve, 20)); + const again = await server.get(Notes, { id: "S1" }, context); + expect(starts).toEqual(["session|S1", "session|S1"]); + again.release(); + }); + + it("a stop during a start waits for it, then stops; a start during a stop waits for its exit", async () => { + const { server, workers, behaviours, starts } = await fleet(); + behaviours.set("S1", { loadMs: 300 }); + const notes = server.hold(Notes, { id: "S1" }, context); + await until(() => stateOf(workers.state.value, "S1") !== undefined); + await workers.stop(Session({ id: "S1" }), context); + expect(stateOf(workers.state.value, "S1")).toBeUndefined(); + notes.release(); + await new Promise((resolve) => setTimeout(resolve, 20)); + behaviours.set("S1", { log: undefined }); + const again = await server.get(Notes, { id: "S1" }, context); + const stopping = workers.stop(Session({ id: "S1" }), context); + again.release(); + await stopping; + const third = await server.get(Notes, { id: "S1" }, context); + expect(starts).toHaveLength(3); + third.release(); + }); + + it("runs the worker's closers in reverse order at its stop", async () => { + const dir = await mkdtemp(join(tmpdir(), "chord-workers-log-")); + const { server, workers, behaviours } = await fleet(); + behaviours.set("S1", { log: join(dir, "log"), closeMs: 50 }); + const notes = await server.get(Notes, { id: "S1" }, context); + notes.release(); + await workers.stop(Session({ id: "S1" }), context); + expect(await readFile(join(dir, "log"), "utf8")).toBe("store\nhost\n"); + await rm(dir, { recursive: true, force: true }); + }); +}); + +describe("close", () => { + it("cancels a start waiting in start(address): nothing forks; new starts reject; no worker_failed at shutdown", async () => { + let waiting = false; + const { server, workers, errors } = await fleet({ + start: async (address, startContext) => { + if (address.key === "session|wait") { + waiting = true; + await new Promise((_, reject) => + startContext.abortSignal?.addEventListener("abort", () => reject(startContext.abortSignal?.reason)), + ); + } + return {}; + }, + }); + const running = await server.get(Notes, { id: "S1" }, context); + const waits = server.hold(Notes, { id: "wait" }, context); + await until(() => waiting); + await workers.close(context); + expect(workers.state.value).toEqual({}); + expect(errors).toEqual([]); + const late = server.hold(Notes, { id: "late" }, context); + await until(() => late.state.value.kind === "failed"); + expect(late.state.value).toMatchObject({ cause: { error: { code: "released", reason: "closed" } } }); + for (const ref of [running, waits, late]) ref.release(); + }); +}); + +describe("the rollout step (§4)", () => { + it("ships the current set to running workers, retries held failed ones, lists each", async () => { + const { server, workers, plugins, label } = await fleet(); + const one = await server.get(Notes, { id: "S1" }, context); + const two = await server.get(Notes, { id: "S2" }, context); + const control = await server.get(Control, { id: "S3" }, context); + const failed = await server.get(Notes, { id: "S3" }, context); + await control.crash(1, context); + await until(() => stateOf(workers.state.value, "S3")?.status === "failed"); + await label("v2"); + await until(() => plugins.state.value.dirty.includes("@test/notes")); + const result = await plugins.rollout( + [workers.step("workers")], + await plugins.build(plugins.state.value.dirty, context), + context, + ); + const hosts = result.steps[0]!.hosts; + expect(listed(hosts).sort()).toEqual([ + ["session|S1", { added: [], changed: ["@test/notes"], removed: [] }], + ["session|S2", { added: [], changed: ["@test/notes"], removed: [] }], + ["session|S3", { added: ["@test/notes"], changed: [], removed: [] }], + ]); + await until(() => one.label.value === "S1 v2" && two.label.value === "S2 v2" && failed.label.value === "S3 v2"); + expect(plugins.state.value.dirty).toEqual([]); + for (const ref of [one, two, failed, control]) ref.release(); + }); + + it("lists a worker stopped cleanly during it as stopped and one that crashed with its error (it stays dirty)", async () => { + const { server, workers, plugins, label } = await fleet(); + const one = await server.get(Notes, { id: "S1" }, context); + const control = await server.get(Control, { id: "S2" }, context); + await label("v2"); + await until(() => plugins.state.value.dirty.includes("@test/notes")); + const built = await plugins.build(plugins.state.value.dirty, context); + // Both workers block their event loops: their reloads wait; S1 is stopped meanwhile, S2 crashes before reading it. + const first = await server.get(Control, { id: "S1" }, context); + await first.block(400, context); + await control.blockThenCrash(400, context); + // Their event loops are blocked once their timers ran. + await new Promise((resolve) => setTimeout(resolve, 50)); + const rolling = plugins.rollout([workers.step("workers")], built, context); + await new Promise((resolve) => setTimeout(resolve, 50)); + const stopped = workers.stop(Session({ id: "S1" }), context); + const result = await rolling; + await stopped; + expect(listed(result.steps[0]!.hosts).sort()).toEqual([ + ["session|S1", "stopped"], + ["session|S2", "worker: its link is gone"], + ]); + expect(plugins.state.value.dirty).toEqual(["@test/notes"]); + for (const ref of [one, first, control]) ref.release(); + }); + + it("a worker starting during it gets the new set: a start after the rollout loads it", async () => { + const { server, workers, plugins, label } = await fleet(); + await label("v2"); + await until(() => plugins.state.value.dirty.includes("@test/notes")); + await plugins.rollout( + [workers.step("workers")], + await plugins.build(plugins.state.value.dirty, context), + context, + ); + const notes = await server.get(Notes, { id: "S9" }, context); + expect(notes.label.value).toBe("S9 v2"); + notes.release(); + }); + + it("an aborted wait lists the worker aborted; its late reply keeps the child's slot until it arrives", async () => { + const { server, workers, plugins, label } = await fleet(); + const control = await server.get(Control, { id: "S1" }, context); + const notes = await server.get(Notes, { id: "S1" }, context); + await label("v2"); + await until(() => plugins.state.value.dirty.includes("@test/notes")); + const built = await plugins.build(plugins.state.value.dirty, context); + await control.block(300, context); + await new Promise((resolve) => setTimeout(resolve, 50)); + const abort = new AbortController(); + const step = workers.step("workers"); + const waited = plugins.rollout([step], built, withAbortSignal(abort.signal, context)); + setTimeout(() => abort.abort(new Error("stop waiting")), 50); + const first = await waited; + expect(listed(first.steps[0]!.hosts)).toEqual([["session|S1", "stop waiting"]]); + expect(plugins.state.value.dirty).toEqual(["@test/notes"]); + await label("v3"); + await until(() => plugins.state.value.dirty.includes("@test/notes")); + const next = await plugins.rollout([step], await plugins.build(plugins.state.value.dirty, context), context); + expect(listed(next.steps[0]!.hosts)).toEqual([ + ["session|S1", { added: [], changed: ["@test/notes"], removed: [] }], + ]); + await until(() => notes.label.value === "S1 v3"); + control.release(); + notes.release(); + }); +}); + +describe("GC", () => { + // Real forks are costly (100 took the suite from 103 s to 700 s on one CPU at 1,000): 100 here; the 1,000 failed and + // retried addresses below cover the per-address records and suspensions at scale. + it("100 addresses started and stopped leave no state, no record, no suspension (Astra, Sol)", async () => { + const refs: WeakRef[] = []; + const { server, workers } = await fleet({ + start: async (address) => { + refs.push(new WeakRef(address)); + return {}; + }, + }); + // Twenty at a time: each a real fork, start, hold, stop and exit. + for (let batch = 0; batch < 5; batch++) { + await Promise.all( + Array.from({ length: 20 }, async (_, index) => { + const id = `R${batch * 20 + index}`; + const notes = await server.get(Notes, { id }, context); + notes.release(); + await workers.stop(Session({ id }), context); + }), + ); + } + expect(workers.state.value).toEqual({}); + await expectCollected(...refs); + }, 120_000); + + it("1,000 failed and retried addresses leave no state, no record and no suspension", async () => { + const refs: WeakRef[] = []; + const { server, workers } = await fleet({ + start: async (address) => { + refs.push(new WeakRef(address)); + throw new Error("no worker"); + }, + }); + for (let i = 0; i < 1_000; i++) { + const id = `A${i}`; + const ref = server.hold(Notes, { id }, context); + await until(() => ref.state.value.kind === "failed"); + ref.release(); + await workers.stop(Session({ id }), context); + await workers.retry(Session({ id }), context); + } + expect(workers.state.value).toEqual({}); + await expectCollected(...refs.slice(0, 50)); + }, 120_000); +}); + +describe("fix round 2", () => { + // Astra's recheck R3: a throwing `principal(address)` reached the guest with its message, the child left starting. + it("a throwing principal hook fails the start with the public message; the error stays private", async () => { + const { server, workers, errors } = await fleet({ + principal: () => { + throw new Error("secret token=sk-1 at /home/owner/private"); + }, + }); + const notes = server.hold(Notes, { id: "S1" }, context); + await until(() => notes.state.value.kind === "failed"); + expect(JSON.stringify(notes.state.value)).not.toContain("secret"); + expect(notes.state.value).toMatchObject({ + kind: "failed", + cause: { error: { message: "worker failed to start" } }, + }); + await until(() => stateOf(workers.state.value, "S1")?.status === "failed"); + expect(stateOf(workers.state.value, "S1")).toMatchObject({ error: { message: "worker failed to start" } }); + // Opus's second recheck: the component's own kill of a failed start is no `worker_exited`. + await until(() => errors.length > 0); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(errors).toMatchObject([ + { kind: "worker_failed", address: "session|S1", error: { message: expect.stringContaining("secret") } }, + ]); + notes.release(); + }); + + it("a throwing authorize refuses the hold (forbidden); the error stays private, reported authorize_failed", async () => { + const { server, errors, starts } = await fleet({ + authorize: () => { + throw new Error("secret lookup"); + }, + }); + const remote = createHost({ onError: () => {} }); + const [up, down] = memoryLinkPair(); + server.accept(up, { principal: { kind: "guest", id: "g" } }, context); + remote.connect(down, { principal: { kind: "server" } }, context); + await remote.load([], context); + const notes = remote.hold(Notes, { id: "S1" }, context); + await until(() => notes.state.value.kind !== "loading"); + expect(JSON.stringify(notes.state.value)).not.toContain("secret"); + expect(notes.state.value).toMatchObject({ kind: "failed", cause: { error: { code: "forbidden" } } }); + expect([errors, starts]).toMatchObject([[{ kind: "authorize_failed", address: "session|S1" }], []]); + notes.release(); + await remote.close(context); + }); +}); + +describe("fix round 1", () => { + it("close during a forked start that never connects returns at once, no failure (Opus F1)", async () => { + const { server, workers, behaviours, errors } = await fleet(); + behaviours.set("S1", { connect: false }); + const notes = server.hold(Notes, { id: "S1" }, context); + await until(() => stateOf(workers.state.value, "S1")?.status === "starting"); + const started = Date.now(); + await workers.close(context); + expect(Date.now() - started).toBeLessThan(5_000); + expect(errors).toEqual([]); + notes.release(); + }); + + it("close during an app's start that ignores its signal returns at once; nothing forks after it (Sol)", async () => { + let forked = false; + const { server, workers, errors } = await fleet({ start: () => new Promise(() => {}) }); + const notes = server.hold(Notes, { id: "S1" }, context); + await new Promise((resolve) => setTimeout(resolve, 50)); + await workers.close(context); + forked = stateOf(workers.state.value, "S1") !== undefined; + expect([forked, errors]).toEqual([false, []]); + notes.release(); + }); + + it("a stop during the worker's startup is a stop: its closers run, nothing fails (Astra)", async () => { + const dir = await mkdtemp(join(tmpdir(), "chord-workers-log-")); + const { server, workers, behaviours, errors } = await fleet(); + behaviours.set("S1", { loadMs: 5_000, closeMs: 200, log: join(dir, "log") }); + const notes = server.hold(Notes, { id: "S1" }, context); + await until(() => stateOf(workers.state.value, "S1") !== undefined); + await new Promise((resolve) => setTimeout(resolve, 300)); + await workers.stop(Session({ id: "S1" }), context); + expect(errors).toEqual([]); + expect(await readFile(join(dir, "log"), "utf8")).toBe("store\nhost\n"); + notes.release(); + await rm(dir, { recursive: true, force: true }); + }); + + it("a reload queued behind one the child never answers settles when the child exits (Astra)", async () => { + const { server, workers, plugins, label } = await fleet(); + const control = await server.get(Control, { id: "S1" }, context); + await label("v2"); + await until(() => plugins.state.value.dirty.length > 0); + const built = await plugins.build(plugins.state.value.dirty, context); + await control.blockThenCrash(600, context); + await new Promise((resolve) => setTimeout(resolve, 50)); + const first = plugins.rollout([workers.step("one")], built, context); + await new Promise((resolve) => setTimeout(resolve, 50)); + const second = plugins.rollout([workers.step("two")], built, context); + const outcomes = await Promise.all([first, second]); + expect(outcomes.map((result) => listed(result.steps[0]!.hosts))).toEqual([ + [["session|S1", "worker: its link is gone"]], + [["session|S1", "worker: its link is gone"]], + ]); + control.release(); + }); + + it("a plugin whose evaluation failed is shipped again by the next rollout and stays dirty (Astra)", async () => { + const { server, workers, plugins, root } = await fleet(); + const notes = await server.get(Notes, { id: "S1" }, context); + await writeFile( + join(root, "notes", "label.ts"), + 'throw new Error("evaluation failed");\nexport const label = "v2";\n', + ); + await until(() => plugins.state.value.dirty.length > 0); + const built = await plugins.build(plugins.state.value.dirty, context); + const failed = (result: { steps: readonly { hosts: readonly RolloutHost[] }[] }) => { + const [entry] = result.steps[0]!.hosts; + return entry !== undefined && "report" in entry ? entry.report.failed.map(({ plugin }) => plugin) : entry; + }; + expect(failed(await plugins.rollout([workers.step("one")], built, context))).toEqual(["@test/notes"]); + expect(failed(await plugins.rollout([workers.step("two")], built, context))).toEqual(["@test/notes"]); + expect(plugins.state.value.dirty).toEqual(["@test/notes"]); + expect(notes.label.value).toBe("S1 v1"); + notes.release(); + }); + + it("two rollouts started together each ship their own build; the later one stays (Astra)", async () => { + const { server, workers, plugins, label } = await fleet(); + const notes = await server.get(Notes, { id: "S1" }, context); + await label("v2"); + const v2 = await plugins.build(["@test/notes"], context); + await label("v3"); + const v3 = await plugins.build(["@test/notes"], context); + const results = await Promise.all([ + plugins.rollout([workers.step("v2")], v2, context), + plugins.rollout([workers.step("v3")], v3, context), + ]); + for (const result of results) { + expect(listed(result.steps[0]!.hosts)).toEqual([ + ["session|S1", { added: [], changed: ["@test/notes"], removed: [] }], + ]); + } + await until(() => notes.label.value === "S1 v3"); + notes.release(); + }); + + it("reaps a worker that closed its own IPC channel and lives on: killed after the grace, failed (Sol)", async () => { + const { server, workers, behaviours } = await fleet(); + behaviours.set("S1", { disconnectAndBlock: true }); + const notes = await server.get(Notes, { id: "S1" }, context); + await until(() => stateOf(workers.state.value, "S1")?.status === "failed", 10_000); + expect(stateOf(workers.state.value, "S1")?.error?.message).toBe("worker exited (SIGKILL)"); + await until(() => notes.state.value.kind === "failed"); + notes.release(); + }, 15_000); + + it("keeps no reload report once consumed: the reload queue settles to nothing (Sol)", async () => { + const env = await fleet(); + const notes = await env.server.get(Notes, { id: "S1" }, context); + const report = await (async () => { + await env.label("v2"); + const built = await env.plugins.build(["@test/notes"], context); + const result = await env.plugins.rollout([env.workers.step("workers")], built, context); + const [entry] = result.steps[0]!.hosts; + if (entry === undefined || !("report" in entry)) throw new Error("no report"); + return new WeakRef(entry.report); + })(); + await expectCollected(report); + notes.release(); + }); + + it("keeps a worker's stderr out of the public state: only worker_failed carries its tail (Astra)", async () => { + const { server, workers, behaviours, errors } = await fleet(); + behaviours.set("S1", { fatal: "boom secret-token-123" }); + const notes = server.hold(Notes, { id: "S1" }, context); + await until(() => stateOf(workers.state.value, "S1")?.status === "failed"); + expect(JSON.stringify(workers.state.value)).not.toContain("secret"); + expect(JSON.stringify(notes.state.value)).not.toContain("secret"); + const failure = errors.find(({ kind }) => kind === "worker_failed"); + expect(failure).toMatchObject({ stderrTail: expect.stringContaining("secret-token-123") }); + notes.release(); + }); +}); diff --git a/packages/chord/test/next/bench/http.ts b/packages/chord/test/next/bench/http.ts new file mode 100644 index 000000000..21f3b2274 --- /dev/null +++ b/packages/chord/test/next/bench/http.ts @@ -0,0 +1,120 @@ +// Benchmark of the HTTP component (slice 10 §7): a route's overhead per request against a plain Node handler (keep-alive +// round trips, median µs per request over bursts), and `bytes` throughput for a 100 MiB file (`fileResponse`) against +// `pipeline` of a read stream. +// Run by hand from packages/chord (not in CI), pinned: +// taskset -c 6 node --conditions=source --experimental-strip-types --expose-gc test/next/bench/http.ts +import { createReadStream } from "node:fs"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { Agent, createServer, request } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { performance } from "node:perf_hooks"; +import { pipeline } from "node:stream/promises"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { createHttp, fileResponse, Http } from "../../../src/next-node/http/index.ts"; + +const BURSTS = 7; +const REQUESTS = 2_000; +const FILE = 100 * 1024 * 1024; + +/** GETs `path` on `port` over `agent`, reading the whole body; resolves with its length. */ +function get(port: number, path: string, agent: Agent): Promise { + return new Promise((resolve, reject) => { + const req = request({ host: "127.0.0.1", port, path, agent }, (res) => { + let length = 0; + res.on("data", (chunk: Buffer) => { + length += chunk.length; + }); + res.on("end", () => resolve(length)); + res.on("error", reject); + }); + req.on("error", reject); + req.end(); + }); +} + +/** The median µs per request of `REQUESTS` sequential keep-alive requests, over bursts. */ +async function perRequest(port: number, path: string): Promise { + const agent = new Agent({ keepAlive: true, maxSockets: 1 }); + for (let i = 0; i < 200; i++) await get(port, path, agent); + const times: number[] = []; + for (let burst = 0; burst < BURSTS; burst++) { + const start = performance.now(); + for (let i = 0; i < REQUESTS; i++) await get(port, path, agent); + times.push(((performance.now() - start) * 1000) / REQUESTS); + } + agent.destroy(); + return median(times); +} + +/** MiB/s of downloading `path` once (after one warm-up). */ +async function throughput(port: number, path: string): Promise { + const agent = new Agent({ keepAlive: true, maxSockets: 1 }); + await get(port, path, agent); + const times: number[] = []; + for (let burst = 0; burst < 3; burst++) { + const start = performance.now(); + await get(port, path, agent); + times.push(performance.now() - start); + } + agent.destroy(); + return FILE / 1024 / 1024 / (median(times) / 1000); +} + +function median(values: readonly number[]): number { + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)]!; +} + +const dir = await mkdtemp(join(tmpdir(), "chord-bench-http-")); +const file = join(dir, "big.bin"); +await writeFile(file, Buffer.alloc(FILE, 1)); + +// A plain Node server: the same JSON body, the same file through `pipeline`. +const plain = createServer((req, res) => { + if (req.url === "/file") { + res.writeHead(200, { "Content-Type": "application/octet-stream", "Content-Length": String(FILE) }); + void pipeline(createReadStream(file), res).catch(() => {}); + return; + } + const body = JSON.stringify({ ok: true }); + res.writeHead(200, { "Content-Type": "application/json; charset=utf-8", "Content-Length": String(body.length) }); + res.end(body); +}); +await new Promise((resolve) => plain.listen(0, "127.0.0.1", resolve)); +const plainPort = (plain.address() as { port: number }).port; + +// The component with Chord's own listener: one plugin with the same route and the file. +const host = createHost({ principal: { kind: "server" }, onError: () => {} }); +const http = createHttp(host, { listen: { host: "127.0.0.1", port: 0 }, onError: () => {} }, context); +const plugin = defineFacet((_host, self) => { + self.contribute(Http, { + routes: { + "GET /json": async () => ({ kind: "json", value: { ok: true } }), + "GET /file": async () => fileResponse(file), + }, + }); +}); +await host.load( + [ + { name: "chord.http", facet: http.facet }, + { name: "@bench/routes", facet: plugin }, + ], + context, +); +const { port } = await http.listening(context); + +const plainRequest = await perRequest(plainPort, "/json"); +const routeRequest = await perRequest(port, "/json"); +console.log( + `request round trip: plain ${plainRequest.toFixed(1)} µs, route ${routeRequest.toFixed(1)} µs (overhead ${(routeRequest - plainRequest).toFixed(1)} µs, ${(((routeRequest - plainRequest) / plainRequest) * 100).toFixed(1)}%)`, +); +const plainBytes = await throughput(plainPort, "/file"); +const routeBytes = await throughput(port, "/file"); +console.log(`100 MiB file: pipeline ${plainBytes.toFixed(0)} MiB/s, fileResponse ${routeBytes.toFixed(0)} MiB/s`); + +await http.close(context); +await host.close(context); +plain.close(); +await rm(dir, { recursive: true, force: true }); diff --git a/packages/chord/test/next/bench/plugins.ts b/packages/chord/test/next/bench/plugins.ts index 3cd231ebb..b80a56ee9 100644 --- a/packages/chord/test/next/bench/plugins.ts +++ b/packages/chord/test/next/bench/plugins.ts @@ -25,6 +25,7 @@ const RELOADS = 100; const gc = (globalThis as { gc?: () => void }).gc!; const HOST_PACKAGES = { "@earendil-works/chord/next": chordNext }; const KINDS = ["server", "worker"]; +const NODE = { platform: "node", hostPackages: Object.keys(HOST_PACKAGES) } as const; /** A plugin's worker facet: a store over a document, a derive, a method, a helper module and the shared contract. */ function worker(index: number, label: string): string { @@ -111,12 +112,9 @@ for (let run = 0; run < RUNS; run++) { console.log(`evaluation per facet: ${median(evaluations).toFixed(2)} ms (median of ${RUNS})`); // The heap after 100 reloads of one plugin on a host, against after a warm-up (two builds alternating). -const component = createPlugins( - { roots: [root], kinds: KINDS, hostPackages: HOST_PACKAGES, onError: () => {} }, - context, -); +const component = createPlugins({ roots: [root], kinds: { server: NODE, worker: NODE }, onError: () => {} }, context); const host = createHost({ onError: () => {} }); -await component.load(host, { kind: "worker", core: [] }, context); +await component.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); await writeFile(join(root, "p0", "worker.ts"), worker(0, "a")); const first = await component.build(["@bench/p0"], context); await writeFile(join(root, "p0", "worker.ts"), worker(0, "b")); diff --git a/packages/chord/test/next/bench/workers.ts b/packages/chord/test/next/bench/workers.ts new file mode 100644 index 000000000..077b6df10 --- /dev/null +++ b/packages/chord/test/next/bench/workers.ts @@ -0,0 +1,107 @@ +// Benchmark of the workers component (slice 10 §7): a worker's cold start with shipped scripts (fork to `started`, fork +// to `ready`, medians over runs) and the server's memory per idle worker (heap after a GC, 20 workers running idle). +// Run by hand from packages/chord (not in CI), pinned: +// taskset -c 6 node --conditions=source --experimental-strip-types --expose-gc test/next/bench/workers.ts +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { hostKind, str } from "../../../src/next/contracts/index.ts"; +import { createHost } from "../../../src/next/host/index.ts"; +import * as chordNext from "../../../src/next/index.ts"; +import { createPlugins } from "../../../src/next-node/plugins/index.ts"; +import type { FromWorker } from "../../../src/next-node/workers/control.ts"; +import { createWorkers } from "../../../src/next-node/workers/index.ts"; +import { WorkerProcess } from "../../../src/next-node/workers/process.ts"; +import { Control } from "../../next-node/fixtures/workers/control.ts"; + +const RUNS = 7; +const WORKERS = 20; +const ENTRY = fileURLToPath(new URL("../../next-node/fixtures/workers/worker.ts", import.meta.url)); +const ROOT = fileURLToPath(new URL("../../next-node/fixtures/workers/plugins/", import.meta.url)); +const EXEC_ARGV = ["--conditions=source", "--experimental-strip-types", "--no-warnings"]; + +const gc = (globalThis as { gc?: () => void }).gc!; +const Session = hostKind("session", { id: str() }); +const modules = { "@earendil-works/chord/next": chordNext }; +const plugins = createPlugins( + { roots: [ROOT], kinds: { worker: { platform: "node", hostPackages: Object.keys(modules) } }, onError: () => {} }, + context, +); +const scripts = await plugins.scripts("worker", context); + +function median(values: readonly number[]): number { + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)]!; +} + +// Cold starts: a fork with the shipped scripts, timed to `started` and to `ready`. +const started: number[] = []; +const ready: number[] = []; +for (let run = 0; run < RUNS; run++) { + const forked = performance.now(); + const times: { started?: number; ready?: number } = {}; + const done = Promise.withResolvers(); + const child = new WorkerProcess( + { entry: ENTRY, execArgv: EXEC_ARGV, onOutput: () => {} }, + { + control: (message: FromWorker) => { + if (message.kind === "started") times.started = performance.now() - forked; + if (message.kind === "ready") { + times.ready = performance.now() - forked; + done.resolve(); + } + }, + linkLost: () => {}, + linkClosed: () => {}, + exited: () => {}, + }, + ); + child.send({ + kind: "start", + address: Session({ id: `R${run}` }), + start: {}, + scripts, + lock: `bench:${process.pid}:${run}`, + }); + await done.promise; + started.push(times.started!); + ready.push(times.ready!); + child.kill(); + await child.exited; +} +console.log( + `cold start (${scripts.length} shipped script): fork to started ${median(started).toFixed(0)} ms, to ready ${median(ready).toFixed(0)} ms (median of ${RUNS})`, +); + +// Memory per idle worker on the server: its records, process handle, link and remote references. +const server = createHost({ address: hostKind("server")(), principal: { kind: "server" }, onError: () => {} }); +const workers = createWorkers( + server, + Session, + { + kind: "worker", + entry: ENTRY, + execArgv: EXEC_ARGV, + authorize: () => true, + principal: () => ({ kind: "worker" }), + start: () => ({}), + lock: (address) => `bench:${process.pid}:${address.key}`, + plugins, + onOutput: () => {}, + onError: () => {}, + }, + context, +); +await server.load([], context); +gc(); +const before = process.memoryUsage().heapUsed; +const held = []; +for (let i = 0; i < WORKERS; i++) held.push(await server.get(Control, { id: `W${i}` }, context)); +gc(); +const after = process.memoryUsage().heapUsed; +console.log( + `server heap per idle worker: ${((after - before) / WORKERS / 1024).toFixed(1)} KiB (${WORKERS} workers, one reference each)`, +); +for (const ref of held) ref.release(); +await workers.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/12-routes.ts b/packages/chord/test/next/examples/12-routes.ts index 7f6516887..e9dedbbb5 100644 --- a/packages/chord/test/next/examples/12-routes.ts +++ b/packages/chord/test/next/examples/12-routes.ts @@ -144,8 +144,8 @@ await guest.get(Notes, { session: "S2" }, context).catch((error) => { const lobby = await guest.get(Lobby, {}, context); console.log("guest via the lobby:", await lobby.peek({ session: "S1" }, context)); -// 5. The worker drops: references go loading, the route resolves the address again after its backoff (1 s), and -// the new worker (a fresh run, empty notes here) binds them again. +// 5. The worker drops: references go loading, the route resolves the address again at once (its link had completed +// its handshake), and the new worker (a fresh run, empty notes here) binds them again. const states: string[] = []; const watch = await notes.state.watch(context); watch.start(async (call) => { diff --git a/packages/chord/test/next/examples/35-plugins/main.ts b/packages/chord/test/next/examples/35-plugins/main.ts index bdfaa89d9..e5fee1544 100644 --- a/packages/chord/test/next/examples/35-plugins/main.ts +++ b/packages/chord/test/next/examples/35-plugins/main.ts @@ -29,13 +29,15 @@ async function until(ready: () => boolean): Promise { while (!ready()) await new Promise((resolve) => setTimeout(resolve, 10)); } -// 1. The component: the roots, the facet kinds, and the host packages every script gets (this process's Chord: a -// plugin's `import { defineFacet } from "@earendil-works/chord/next"` is the app's own instance). +// 1. The component: the roots, and the facet kinds, each built for Node with Chord as a host package (kept external in +// the builds). The hosts give the modules themselves when they load (this process's Chord: a plugin's +// `import { defineFacet } from "@earendil-works/chord/next"` is the app's own instance). +const modules = { "@earendil-works/chord/next": chordNext }; +const node = { platform: "node", hostPackages: Object.keys(modules) } as const; const plugins = createPlugins( { roots: [root], - kinds: ["server", "worker"], - hostPackages: { "@earendil-works/chord/next": chordNext }, + kinds: { server: node, worker: node }, onError: (error) => console.log("plugins:", error.kind), }, context, @@ -52,8 +54,8 @@ server.route( { authorize: () => true, resolve: async () => ({ link: down, principal: { kind: "worker" } }) }, context, ); -const workerLoad = await plugins.load(worker, { kind: "worker", core: [] }, context); -const serverLoad = await plugins.load(server, { kind: "server", core: [] }, context); +const workerLoad = await plugins.load(worker, { kind: "worker", core: [], modules }, context); +const serverLoad = await plugins.load(server, { kind: "server", core: [], modules }, context); console.log( "1. discovered:", plugins.discovery.plugins.map(({ name, facets }) => `${name} [${Object.keys(facets)}]`).join(", "), @@ -84,8 +86,17 @@ console.log( built.facets.map(({ plugin, facet, kind }) => `${plugin}/${facet}: ${kind}`), ); const rolled = await plugins.rollout(steps, built, context); -for (const { name, report } of rolled.steps) { - console.log(` ${name}:`, JSON.stringify(report.plugins), "objects", JSON.stringify(report.objects)); +for (const { name, hosts } of rolled.steps) { + for (const host of hosts) { + if ("report" in host) { + console.log( + ` ${name}:`, + JSON.stringify(host.report.plugins), + "objects", + JSON.stringify(host.report.objects), + ); + } + } } await until(() => notes.label.value === "v2"); console.log("4. notes:", notes.label.value, "count", notes.count.value, "| index:", index.label.value); diff --git a/packages/chord/test/next/examples/35-plugins/plugins/clock/server.ts b/packages/chord/test/next/examples/35-plugins/plugins/clock/server.ts index 5bc4b6ee3..29f0d3d98 100644 --- a/packages/chord/test/next/examples/35-plugins/plugins/clock/server.ts +++ b/packages/chord/test/next/examples/35-plugins/plugins/clock/server.ts @@ -2,7 +2,7 @@ // this plugin, so a reload leaves it as it is: same script, same facet, same object. import { constant, defineFacet } from "@earendil-works/chord/next"; -import { Clock } from "../contracts/contract.ts"; +import { Clock } from "@example/contracts/contract"; const evaluatedAt = new Date().toISOString(); diff --git a/packages/chord/test/next/examples/35-plugins/plugins/notes/server.ts b/packages/chord/test/next/examples/35-plugins/plugins/notes/server.ts index ed3ded889..141e104d7 100644 --- a/packages/chord/test/next/examples/35-plugins/plugins/notes/server.ts +++ b/packages/chord/test/next/examples/35-plugins/plugins/notes/server.ts @@ -1,7 +1,7 @@ // The notes plugin's server facet: serves `Index` with the same label. import { constant, defineFacet } from "@earendil-works/chord/next"; -import { Index } from "../contracts/contract.ts"; +import { Index } from "@example/contracts/contract"; import { label } from "./label.ts"; export default defineFacet((host) => host.serve(Index, () => ({ label: constant(label) }))); diff --git a/packages/chord/test/next/examples/35-plugins/plugins/notes/worker.ts b/packages/chord/test/next/examples/35-plugins/plugins/notes/worker.ts index b4f4c8ae4..012486c94 100644 --- a/packages/chord/test/next/examples/35-plugins/plugins/notes/worker.ts +++ b/packages/chord/test/next/examples/35-plugins/plugins/notes/worker.ts @@ -1,7 +1,7 @@ // The notes plugin's worker facet: serves `Notes`, its count kept in a document. import { constant, defineFacet, defineState, int, obj } from "@earendil-works/chord/next"; -import { Notes } from "../contracts/contract.ts"; +import { Notes } from "@example/contracts/contract"; import { label } from "./label.ts"; const Count = defineState("count", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); diff --git a/packages/chord/test/next/examples/36-plugin-failures/main.ts b/packages/chord/test/next/examples/36-plugin-failures/main.ts index 1fe16092c..103b1b3fb 100644 --- a/packages/chord/test/next/examples/36-plugin-failures/main.ts +++ b/packages/chord/test/next/examples/36-plugin-failures/main.ts @@ -63,11 +63,12 @@ const Counter = local("example.counter", { version: 1, params: {}, members: { st const Summary = local("example.summary", { version: 1, params: {}, members: { text: str() } }); const Status = local("example.status", { version: 1, params: {}, members: { text: str() } }); +// The host packages: this process's Chord. +const modules = { "@earendil-works/chord/next": chordNext }; const plugins = createPlugins( { roots: [root], - kinds: ["server"], - hostPackages: { "@earendil-works/chord/next": chordNext }, + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, onError: (error) => console.log("plugins:", error.kind), }, context, @@ -76,7 +77,7 @@ const host = createHost({ onError: () => {} }); // 1. Startup: the weather plugin's setup throws; it is left out and reported, the other plugins run. A reference to // what it would serve waits (`failed`, cause `waiting`: not served here), and so does its consumer's object. -const loaded = await plugins.load(host, { kind: "server", core: [] }, context); +const loaded = await plugins.load(host, { kind: "server", core: [], modules }, context); for (const { plugin, error } of loaded.failed) console.log(`1. left out ${plugin}: ${(error as Error).message}`); const weather = host.hold(Weather, {}, context); const forecast = host.hold(Forecast, {}, context); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/forecast/server.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/forecast/server.ts index 54a92d46f..c7a04a1d8 100644 --- a/packages/chord/test/next/examples/36-plugin-failures/plugins/forecast/server.ts +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/forecast/server.ts @@ -1,7 +1,7 @@ // The forecast plugin's server facet: a consumer of the weather plugin's `Weather`. import { defineFacet, local, str } from "@earendil-works/chord/next"; -import { Weather } from "../weather/contract.ts"; +import { Weather } from "@example/weather/contract"; const Forecast = local("example.forecast", { version: 1, params: {}, members: { text: str() } }); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/summary/server.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/summary/server.ts index 3aad5f3e1..6db7ff40f 100644 --- a/packages/chord/test/next/examples/36-plugin-failures/plugins/summary/server.ts +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/summary/server.ts @@ -2,7 +2,7 @@ // contract declares. import { defineFacet, local, str } from "@earendil-works/chord/next"; -import { Price } from "../prices/contract.ts"; +import { Price } from "@example/prices/contract"; const Summary = local("example.summary", { version: 1, params: {}, members: { text: str() } }); diff --git a/packages/chord/test/next/examples/37-plugin-imports/main.ts b/packages/chord/test/next/examples/37-plugin-imports/main.ts index 8ed51b7d2..7ec709c46 100644 --- a/packages/chord/test/next/examples/37-plugin-imports/main.ts +++ b/packages/chord/test/next/examples/37-plugin-imports/main.ts @@ -29,17 +29,18 @@ for (const dir of ["plugins", "app", "deps"]) { } const root = join(copy, "plugins"); -// 1. The component. Its host packages: Chord, the app's ES module and a CommonJS module of the app. Its npm command -// (here a stand-in): what `installDependencies` runs in a plugin's directory. +// 1. The component. Its host packages: Chord, the app's ES module and a CommonJS module of the app (their names kept +// external in the server kind's builds, the modules given at load). Its npm command (here a stand-in): what +// `installDependencies` runs in a plugin's directory. +const modules = { + "@earendil-works/chord/next": chordNext, + "@example/app": app, + "@example/legacy": createRequire(import.meta.url)("./app/legacy.cjs"), +}; const plugins = createPlugins( { roots: [root], - kinds: ["server"], - hostPackages: { - "@earendil-works/chord/next": chordNext, - "@example/app": app, - "@example/legacy": createRequire(import.meta.url)("./app/legacy.cjs"), - }, + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, install: { npmCommand: [process.execPath, fileURLToPath(new URL("./fake-npm.mjs", import.meta.url))] }, onError: (error) => console.log("plugins:", error.kind), }, @@ -49,7 +50,7 @@ const plugins = createPlugins( // 2. Startup: every plugin whose build fails is left out with its errors (one per line: file, line, column, why). The // report plugin's dependency is not installed yet. const host = createHost({ onError: () => {} }); -const loaded = await plugins.load(host, { kind: "server", core: [] }, context); +const loaded = await plugins.load(host, { kind: "server", core: [], modules }, context); console.log("1. left out at startup:"); for (const { plugin, error } of loaded.failed) { const [first, ...errors] = (error as Error).message.split("\n"); @@ -67,7 +68,9 @@ for (const facet of built.facets) { console.log(` built ${facet.plugin}/${facet.facet}: read ${files.join(", ")}; copied ${facet.copied.length} file`); } const report = await plugins.reload(host, built, context); -console.log(" reload:", JSON.stringify(report.plugins), "failed", report.failed.length); +// The reload passes the complete set: the refused plugins fail again and stay out. +const stillRefused = report.failed.map(({ plugin }) => plugin).join(", "); +console.log(" reload:", JSON.stringify(report.plugins), "| still refused:", stillRefused); const text = (await host.get(Report, {}, context)).text.value; console.log("3. the report:", text); diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/report/server.ts b/packages/chord/test/next/examples/37-plugin-imports/plugins/report/server.ts index 41201eeea..601072036 100644 --- a/packages/chord/test/next/examples/37-plugin-imports/plugins/report/server.ts +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/report/server.ts @@ -6,12 +6,12 @@ import { pluginFile } from "@earendil-works/chord/next/plugins"; // Host packages: an ES module (default and named imports) and a CommonJS one, the app's own instances. import app, { currency } from "@example/app"; import legacy from "@example/legacy"; +// A contract of another plugin: a copy built into this script. +import { Report } from "@example/shapes/contract"; // An npm dependency: inlined into this script by the build (installed first). import { greet } from "example-greeting"; // A `chord.external` package: left external, loaded from this plugin's node_modules when the script runs. import { where } from "example-native"; -// A contract of another plugin: a copy built into this script. -import { Report } from "../shapes/contract.ts"; // A file of this plugin, read at run time by its absolute path. const rates = JSON.parse(readFileSync(pluginFile("data/rates.json"), "utf8")) as { readonly vat: number }; diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/ruler/server.ts b/packages/chord/test/next/examples/37-plugin-imports/plugins/ruler/server.ts index ec9a0dd74..a67049cac 100644 --- a/packages/chord/test/next/examples/37-plugin-imports/plugins/ruler/server.ts +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/ruler/server.ts @@ -2,7 +2,7 @@ // copied file may import only host packages and files of its own plugin. import { defineFacet } from "@earendil-works/chord/next"; -import { Millimetres } from "../shapes/units.ts"; +import { Millimetres } from "@example/shapes/units"; export default defineFacet(() => { console.log(Millimetres); diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/main.ts b/packages/chord/test/next/examples/38-watch-and-rollout/main.ts index 7eeb92f1e..53313cf75 100644 --- a/packages/chord/test/next/examples/38-watch-and-rollout/main.ts +++ b/packages/chord/test/next/examples/38-watch-and-rollout/main.ts @@ -48,17 +48,21 @@ const line = (report: ReloadReport) => /** A rollout, step by step. */ function print(result: RolloutResult): void { - for (const { name, report } of result.steps) console.log(` ${name}:`, line(report)); + for (const { name, hosts } of result.steps) { + for (const host of hosts) if ("report" in host) console.log(` ${name}:`, line(host.report)); + } if (result.stopped !== undefined) { console.log(` stopped at ${result.stopped.step}: ${(result.stopped.error as Error).message}`); } } +// The host packages: this process's Chord, kept external in Node builds. +const modules = { "@earendil-works/chord/next": chordNext }; +const node = { platform: "node", hostPackages: Object.keys(modules) } as const; const plugins = createPlugins( { roots: [root], - kinds: ["server", "worker"], - hostPackages: { "@earendil-works/chord/next": chordNext }, + kinds: { server: node, worker: node }, // The app's own code: a change marks `base`. base: [join(copy, "app")], // A stand-in for npm (no network; these plugins have no dependencies). @@ -69,8 +73,8 @@ const plugins = createPlugins( ); const server = createHost({ onError: () => {} }); const worker = createHost({ onError: () => {} }); -await plugins.load(server, { kind: "server", core: [] }, context); -await plugins.load(worker, { kind: "worker", core: [] }, context); +await plugins.load(server, { kind: "server", core: [], modules }, context); +await plugins.load(worker, { kind: "worker", core: [], modules }, context); const steps = [plugins.step("server", server as Host), plugins.step("worker", worker as Host)]; console.log("1. loaded:", plugins.discovery.plugins.map(({ name }) => name).join(", "), "| marks:", marks()); @@ -79,7 +83,7 @@ await edit("plugins/notes/label.ts", "notes v1", "notes v2"); await dirty("@example/notes"); console.log("2. a file of notes edited:", marks()); await edit("plugins/contracts/contract.ts", "one type per plugin", "one type per plugin (edited)"); -await dirty("@example/contracts", "@example/tags"); +await dirty("@example/shared", "@example/tags"); console.log(" the shared contract edited:", marks()); await edit("plugins/tags/package.json", '"private": true', '"private": true,\n\t"description": "tags"'); await until(() => plugins.state.value.install.includes("@example/tags")); diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/package.json b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/package.json index d1c133527..dfc4cefdc 100644 --- a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/package.json +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/package.json @@ -1,5 +1,5 @@ { - "name": "@example/contracts", + "name": "@example/shared", "private": true, "type": "module" } diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/server.ts b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/server.ts index 23a936014..5670a4766 100644 --- a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/server.ts +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/server.ts @@ -1,7 +1,7 @@ // The notes plugin's server facet: its label. import { constant, defineFacet } from "@earendil-works/chord/next"; -import { labelOf } from "../contracts/contract.ts"; +import { labelOf } from "@example/shared/contract"; import { label } from "./label.ts"; export default defineFacet((host) => diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/worker.ts b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/worker.ts index e01fc1159..58d814c27 100644 --- a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/worker.ts +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/worker.ts @@ -1,7 +1,7 @@ // The notes plugin's worker facet: its label. import { constant, defineFacet } from "@earendil-works/chord/next"; -import { labelOf } from "../contracts/contract.ts"; +import { labelOf } from "@example/shared/contract"; import { label } from "./label.ts"; export default defineFacet((host) => diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/old/server.ts b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/old/server.ts index afb0165f9..3d87897be 100644 --- a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/old/server.ts +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/old/server.ts @@ -1,6 +1,6 @@ // The old plugin's server facet; the example deletes the plugin's directory. import { constant, defineFacet } from "@earendil-works/chord/next"; -import { labelOf } from "../contracts/contract.ts"; +import { labelOf } from "@example/shared/contract"; export default defineFacet((host) => host.serve(labelOf("old"), () => ({ text: constant("old on the server") }))); diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/server.ts b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/server.ts index 8b8ce9d68..64210746e 100644 --- a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/server.ts +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/server.ts @@ -1,6 +1,6 @@ // The tags plugin's server facet. import { constant, defineFacet } from "@earendil-works/chord/next"; -import { labelOf } from "../contracts/contract.ts"; +import { labelOf } from "@example/shared/contract"; export default defineFacet((host) => host.serve(labelOf("tags"), () => ({ text: constant("tags on the server") }))); diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/worker.ts b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/worker.ts index 7cbeb3ae4..796e053a3 100644 --- a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/worker.ts +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/worker.ts @@ -1,6 +1,6 @@ // The tags plugin's worker facet. import { constant, defineFacet } from "@earendil-works/chord/next"; -import { labelOf } from "../contracts/contract.ts"; +import { labelOf } from "@example/shared/contract"; export default defineFacet((host) => host.serve(labelOf("tags"), () => ({ text: constant("tags on the worker") }))); diff --git a/packages/chord/test/next/examples/39-browser-build/main.ts b/packages/chord/test/next/examples/39-browser-build/main.ts index 484951d89..f7a05af4d 100644 --- a/packages/chord/test/next/examples/39-browser-build/main.ts +++ b/packages/chord/test/next/examples/39-browser-build/main.ts @@ -1,4 +1,4 @@ -// A browser build (slice 7c §2): a facet of a browser kind (`browserKinds: ["web"]`) is built for the browser. Its +// A browser build (slice 7c §2): a facet of a browser kind (`web: { platform: "browser", … }`) is built for the browser. Its // stylesheets become one CSS file (`css`), an asset imported by URL (`./logo.svg?url`) a content-hashed file whose // import evaluates to its URL under the app's `publicPath`. The build is plain data: script, stylesheet and assets, // what the app serves to its pages (slice 11 ships it); a page evaluates the script with its host packages, as Node @@ -20,10 +20,8 @@ const hostPackages = { "@earendil-works/chord/next": chordNext }; const plugins = createPlugins( { roots: [root], - kinds: ["web"], - browserKinds: ["web"], + kinds: { web: { platform: "browser", hostPackages: Object.keys(hostPackages) } }, publicPath: "/plugin-assets/", - hostPackages, onError: (error) => console.log("plugins:", error.kind), }, context, diff --git a/packages/chord/test/next/examples/40-http/main.ts b/packages/chord/test/next/examples/40-http/main.ts new file mode 100644 index 000000000..78dbca808 --- /dev/null +++ b/packages/chord/test/next/examples/40-http/main.ts @@ -0,0 +1,56 @@ +// HTTP (slice 10): a server answers HTTP requests with routes that plugins contribute. The notes plugin (`plugins/`) +// contributes a page, a JSON route and a file download; the server's own code only builds the component, says where +// Chord's own listener listens, and loads the plugins. Every answer carries the security baseline's headers. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/40-http/main.ts +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import * as chordHttp from "../../../../src/next-node/http/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; + +// 1. A server host and its HTTP component; its settings are arguments: Chord's own listener listens on a free +// loopback port. Nobody signs in here: no authenticator, every request is anonymous. +const server = createHost({ principal: { kind: "server" }, onError: console.error }); +const http = chordHttp.createHttp(server, { listen: { host: "127.0.0.1", port: 0 }, onError: console.error }, context); + +// 2. The plugins: the component's facet (its registries and the listener) goes first, then every plugin's server +// facet. The listener opens once they are published. +const modules = { "@earendil-works/chord/next": chordNext, "@earendil-works/chord/next/http": chordHttp }; +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, + onError: console.error, + }, + context, +); +await plugins.load(server, { kind: "server", core: [{ name: "chord.http", facet: http.facet }], modules }, context); +const { port } = await http.listening(context); + +// 3. Some requests, printed like curl would. +async function curl(path: string, headers: Record = {}): Promise { + const response = await fetch(`http://127.0.0.1:${port}${path}`, { headers }); + const range = headers.Range === undefined ? "" : ` (Range: ${headers.Range})`; + console.log(`GET ${path}${range} -> ${response.status} ${response.headers.get("content-type")}`); + const download = response.headers.get("content-disposition"); + if (download !== null) console.log(` ${download}`); + console.log(` ${JSON.stringify((await response.text()).replace(/nonce="[^"]+"/, 'nonce="…"'))}`); +} + +await curl("/"); +await curl("/notes/hello"); +await curl("/notes/hello/download", { Range: "bytes=0-6" }); +await curl("/notes/nope/download"); +await curl("/nowhere"); +const baseline = await fetch(`http://127.0.0.1:${port}/notes/hello`); +for (const name of ["cache-control", "x-content-type-options", "x-frame-options"]) { + console.log(` ${name}: ${baseline.headers.get(name)}`); +} + +// 4. Close the component first: it refuses new requests and waits for running ones (the host's close would seal them +// at once). Then the host. +await http.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/40-http/plugins/notes/files/hello.md b/packages/chord/test/next/examples/40-http/plugins/notes/files/hello.md new file mode 100644 index 000000000..13cbb3050 --- /dev/null +++ b/packages/chord/test/next/examples/40-http/plugins/notes/files/hello.md @@ -0,0 +1,3 @@ +# Hello + +A note served as a download. diff --git a/packages/chord/test/next/examples/40-http/plugins/notes/package.json b/packages/chord/test/next/examples/40-http/plugins/notes/package.json new file mode 100644 index 000000000..d59f5a427 --- /dev/null +++ b/packages/chord/test/next/examples/40-http/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/40-http/plugins/notes/server.ts b/packages/chord/test/next/examples/40-http/plugins/notes/server.ts new file mode 100644 index 000000000..48e6b9388 --- /dev/null +++ b/packages/chord/test/next/examples/40-http/plugins/notes/server.ts @@ -0,0 +1,29 @@ +// The notes plugin's server facet: a page, a JSON route and a file download, contributed to the HTTP component's +// registry. + +import { defineFacet } from "@earendil-works/chord/next"; +import { fileResponse, Http } from "@earendil-works/chord/next/http"; +import { pluginFile } from "@earendil-works/chord/next/plugins"; + +export default defineFacet((_host, self) => + self.contribute(Http, { + routes: { + // A page: a link from another site may open it. Its inline script carries the server's nonce (the CSP). + "GET /": { + page: true, + handler: async (request) => ({ + kind: "html", + body: `

    Notes

    `, + }), + }, + // `:name` is one path segment, decoded. + "GET /notes/:name": async (request) => ({ kind: "json", value: { note: request.params.name } }), + // A file of this plugin, saved under its name, with ranges. + "GET /notes/:name/download": async (request) => + fileResponse(pluginFile(`files/${request.params.name}.md`), { + download: `${request.params.name}.md`, + ranges: true, + }), + }, + }), +); diff --git a/packages/chord/test/next/examples/41-workers/main.ts b/packages/chord/test/next/examples/41-workers/main.ts new file mode 100644 index 000000000..35460577b --- /dev/null +++ b/packages/chord/test/next/examples/41-workers/main.ts @@ -0,0 +1,68 @@ +// Workers (slice 10): the server routes a session's address to its own worker process. The first reference to a +// session starts its worker; once nobody uses it for `idleMs`, the worker stops. The worker's code is the sessions +// plugin (`plugins/`), built by the server and shipped to the worker. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/41-workers/main.ts +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { hostKind } from "../../../../src/next/contracts/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; +import { createWorkers } from "../../../../src/next-node/workers/index.ts"; +import { Greeting, Session } from "./plugins/sessions/contract.ts"; + +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { worker: { platform: "node", hostPackages: ["@earendil-works/chord/next"] } }, + onError: console.error, + }, + context, +); + +// 1. The server, and its workers: one forked process per session, stopped after 300 ms without use. +const server = createHost({ address: hostKind("server")(), principal: { kind: "server" }, onError: console.error }); +const workers = createWorkers( + server, + Session, + { + kind: "worker", + entry: fileURLToPath(new URL("./worker.ts", import.meta.url)), + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + authorize: () => true, + principal: (address) => ({ kind: "worker", session: address.params.id }), + start: () => ({}), + lock: (address) => `example-41:${process.pid}:${address.key}`, + idleMs: () => 300, + plugins, + onError: console.error, + }, + context, +); +await server.load([], context); + +// 2. Every change of the workers' state, printed. +const watch = await workers.state.watch(context); +watch.start(async (call) => { + if (call.kind !== "value") return; + const entries = Object.entries(call.value).map(([key, state]) => `${key} ${state.status}`); + console.log(` workers: ${entries.join(", ") || "none"}`); +}); + +// 3. The first reference starts the worker; the greeting comes from that process. +console.log("1. getting S1's greeting"); +const greeting = await server.get(Greeting, { id: "S1" }, context); +console.log(`2. "${greeting.text.value}" (worker pid ${greeting.pid.value}, server pid ${process.pid})`); + +// 4. Releasing it leaves the worker idle: 300 ms later it stops. +greeting.release(); +console.log("3. released; waiting for the idle stop"); +while (Object.keys(workers.state.value).length > 0) { + await new Promise((resolve) => setTimeout(resolve, 50)); +} +console.log("4. stopped"); + +await watch.stop(); +await workers.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/41-workers/plugins/sessions/contract.ts b/packages/chord/test/next/examples/41-workers/plugins/sessions/contract.ts new file mode 100644 index 000000000..0ef89bfff --- /dev/null +++ b/packages/chord/test/next/examples/41-workers/plugins/sessions/contract.ts @@ -0,0 +1,13 @@ +// A session lives in its own worker process: `Session({ id })` is its address. + +import { hostKind, int, remote, str } from "@earendil-works/chord/next"; + +export const Session = hostKind("session", { id: str() }); + +/** A session's greeting, served by the worker at the session's address. */ +export const Greeting = remote("example.greeting", { + version: 1, + params: { id: str() }, + hostAddress: ({ id }) => Session({ id }), + variants: { main: { state: { text: str(), pid: int() } } }, +}); diff --git a/packages/chord/test/next/examples/41-workers/plugins/sessions/package.json b/packages/chord/test/next/examples/41-workers/plugins/sessions/package.json new file mode 100644 index 000000000..3819a6f05 --- /dev/null +++ b/packages/chord/test/next/examples/41-workers/plugins/sessions/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/sessions", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/41-workers/plugins/sessions/worker.ts b/packages/chord/test/next/examples/41-workers/plugins/sessions/worker.ts new file mode 100644 index 000000000..79ee9efca --- /dev/null +++ b/packages/chord/test/next/examples/41-workers/plugins/sessions/worker.ts @@ -0,0 +1,11 @@ +// The sessions plugin's worker facet: each worker greets with its process id. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Greeting } from "./contract.ts"; + +export default defineFacet((host) => + host.serve(Greeting, ({ id }) => ({ + variant: "main", + members: { text: constant(`hello from session ${id}`), pid: constant(process.pid) }, + })), +); diff --git a/packages/chord/test/next/examples/41-workers/worker.ts b/packages/chord/test/next/examples/41-workers/worker.ts new file mode 100644 index 000000000..2fb4fccaa --- /dev/null +++ b/packages/chord/test/next/examples/41-workers/worker.ts @@ -0,0 +1,22 @@ +// The worker process: a host at the session's address, connected to the server, loading the plugins it was shipped. + +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { workerMain } from "../../../../src/next-node/workers/index.ts"; + +workerMain( + async ({ address, parent, plugins, onClose }, context) => { + const host = createHost({ address, principal: { kind: "worker" }, onError: console.error }); + onClose(() => host.close(BACKGROUND_CONTEXT)); + // Connect first: the server sees the worker started; then load. + host.connect(parent, { principal: { kind: "server" }, trusted: true }, context); + await plugins.load(host, { kind: "worker", core: [] }, context); + return {}; + }, + { + hostPackages: { "@earendil-works/chord/next": chordNext }, + redact: (error) => ({ message: String(error) }), + onError: console.error, + }, +); diff --git a/packages/chord/test/next/examples/42-worker-crash/main.ts b/packages/chord/test/next/examples/42-worker-crash/main.ts new file mode 100644 index 000000000..37297f924 --- /dev/null +++ b/packages/chord/test/next/examples/42-worker-crash/main.ts @@ -0,0 +1,67 @@ +// A worker crash (slice 10): workers never restart by themselves. A crashed worker's address stays failed, its +// holders see `failed` with the error, and nothing forks until the app retries (a Retry button) or a rollout ships new +// code. Here: a session crashes, stays failed for a while, then Retry brings it back. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/42-worker-crash/main.ts +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { hostKind } from "../../../../src/next/contracts/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; +import { createWorkers } from "../../../../src/next-node/workers/index.ts"; +import { Process, Session } from "./plugins/sessions/contract.ts"; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { worker: { platform: "node", hostPackages: ["@earendil-works/chord/next"] } }, + onError: console.error, + }, + context, +); +const server = createHost({ address: hostKind("server")(), principal: { kind: "server" }, onError: () => {} }); +const workers = createWorkers( + server, + Session, + { + kind: "worker", + entry: fileURLToPath(new URL("./worker.ts", import.meta.url)), + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + authorize: () => true, + principal: (address) => ({ kind: "worker", session: address.params.id }), + start: () => ({}), + lock: (address) => `example-42:${process.pid}:${address.key}`, + plugins, + onError: (error) => console.log(` reported: ${error.kind}`), + }, + context, +); +await server.load([], context); + +// 1. A held session (a tab showing it): its worker runs. +const session = server.hold(Process, { id: "S1" }, context); +const state = () => session.state.value.kind; +while (state() !== "live") await sleep(10); +console.log(`1. S1 runs in process ${session.pid.value}`); + +// 2. It crashes. The address is failed, and the holder sees why; it stays on the address's path. +await session.crash(context); +while (state() !== "failed") await sleep(10); +const failed = workers.state.value["session|S1"]; +console.log(`2. crashed: ${failed?.status === "failed" ? failed.error.message : failed?.status}`); + +// 3. No restart loop: a second later it is still failed, and no other worker was started. +await sleep(1_000); +console.log(`3. a second later: ${workers.state.value["session|S1"]?.status}, the holder: ${state()}`); + +// 4. Retry (the app's button): the worker starts again and the same holder binds to it. +await workers.retry(Session({ id: "S1" }), context); +while (state() !== "live") await sleep(10); +console.log(`4. after Retry: S1 runs in process ${session.pid.value}`); + +session.release(); +await workers.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/contract.ts b/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/contract.ts new file mode 100644 index 000000000..0a995c248 --- /dev/null +++ b/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/contract.ts @@ -0,0 +1,13 @@ +// A session lives in its own worker process: `Session({ id })` is its address. + +import { hostKind, int, method, remote, str } from "@earendil-works/chord/next"; + +export const Session = hostKind("session", { id: str() }); + +/** A session's process: its pid, and a way to make it crash. */ +export const Process = remote("example.process", { + version: 1, + params: { id: str() }, + hostAddress: ({ id }) => Session({ id }), + variants: { main: { state: { pid: int() }, methods: { crash: method() } } }, +}); diff --git a/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/package.json b/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/package.json new file mode 100644 index 000000000..3819a6f05 --- /dev/null +++ b/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/sessions", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/worker.ts b/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/worker.ts new file mode 100644 index 000000000..cb8cae63c --- /dev/null +++ b/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/worker.ts @@ -0,0 +1,17 @@ +// The sessions plugin's worker facet: its process id, and `crash`, which ends the process (a bug, an out-of-memory). + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Process } from "./contract.ts"; + +export default defineFacet((host) => + host.serve(Process, () => ({ + variant: "main", + members: { + pid: constant(process.pid), + crash: async () => { + setTimeout(() => process.exit(13), 10); + return null; + }, + }, + })), +); diff --git a/packages/chord/test/next/examples/42-worker-crash/worker.ts b/packages/chord/test/next/examples/42-worker-crash/worker.ts new file mode 100644 index 000000000..2fb4fccaa --- /dev/null +++ b/packages/chord/test/next/examples/42-worker-crash/worker.ts @@ -0,0 +1,22 @@ +// The worker process: a host at the session's address, connected to the server, loading the plugins it was shipped. + +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { workerMain } from "../../../../src/next-node/workers/index.ts"; + +workerMain( + async ({ address, parent, plugins, onClose }, context) => { + const host = createHost({ address, principal: { kind: "worker" }, onError: console.error }); + onClose(() => host.close(BACKGROUND_CONTEXT)); + // Connect first: the server sees the worker started; then load. + host.connect(parent, { principal: { kind: "server" }, trusted: true }, context); + await plugins.load(host, { kind: "worker", core: [] }, context); + return {}; + }, + { + hostPackages: { "@earendil-works/chord/next": chordNext }, + redact: (error) => ({ message: String(error) }), + onError: console.error, + }, +); diff --git a/packages/chord/test/next/examples/43-rollout/main.ts b/packages/chord/test/next/examples/43-rollout/main.ts new file mode 100644 index 000000000..5fd443756 --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/main.ts @@ -0,0 +1,85 @@ +// A rollout across the server and its workers (slice 10): an edit of a plugin is built once by the server and rolled +// out step by step, the server first, then every running worker; each host reports what changed. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/43-rollout/main.ts +import { cp, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { hostKind } from "../../../../src/next/contracts/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { createPlugins, type RolloutHost } from "../../../../src/next-node/plugins/index.ts"; +import { createWorkers } from "../../../../src/next-node/workers/index.ts"; +import { Banner, Label, Session } from "./plugins/notes/contract.ts"; + +/** Waits until `done()` holds (a watcher's mark, a frame from a worker), checking every 10 ms. */ +async function waitUntil(done: () => boolean): Promise { + while (!done()) await new Promise((resolve) => setTimeout(resolve, 10)); +} + +/** One line for what a rollout step did on one of its hosts. */ +function describe(host: RolloutHost): string { + if ("report" in host) return `changed ${host.report.plugins.changed.join(", ")}`; + if ("stopped" in host) return "stopped"; + return "failed"; +} + +// The example edits a plugin: it works on a copy of `plugins/`. +const root = await mkdtemp(join(tmpdir(), "chord-example-43-")); +await cp(fileURLToPath(new URL("./plugins/", import.meta.url)), root, { recursive: true }); + +const node = { platform: "node", hostPackages: ["@earendil-works/chord/next"] } as const; +const plugins = createPlugins( + { roots: [root], kinds: { server: node, worker: node }, onError: console.error }, + context, +); +const server = createHost({ address: hostKind("server")(), principal: { kind: "server" }, onError: console.error }); +const workers = createWorkers( + server, + Session, + { + kind: "worker", + entry: fileURLToPath(new URL("./worker.ts", import.meta.url)), + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + authorize: () => true, + principal: (address) => ({ kind: "worker", session: address.params.id }), + start: () => ({}), + lock: (address) => `example-43:${process.pid}:${address.key}`, + plugins, + onError: console.error, + }, + context, +); +await plugins.load(server, { kind: "server", core: [], modules: { "@earendil-works/chord/next": chordNext } }, context); + +// 1. The server's banner and two running sessions. +const banner = await server.get(Banner, {}, context); +const s1 = await server.get(Label, { id: "S1" }, context); +const s2 = await server.get(Label, { id: "S2" }, context); +console.log(`1. ${banner.text.value} | ${s1.text.value} | ${s2.text.value}`); + +// 2. An edit: the watcher marks the plugin; one build serves both kinds. +await writeFile(join(root, "notes", "label.ts"), 'export const label = "v2";\n'); +await waitUntil(() => plugins.state.value.dirty.includes("@example/notes")); +const built = await plugins.build(plugins.state.value.dirty, context); + +// 3. The rollout: the server, then the workers (each running one reloads in parallel). +const result = await plugins.rollout([plugins.step("server", server), workers.step("workers")], built, context); +for (const step of result.steps) { + for (const host of step.hosts) console.log(`2. ${step.name}: ${host.host} ${describe(host)}`); +} + +// 4. Every host shows the new label (the workers' frames arrive over their links). +await waitUntil(() => s2.text.value === "S2: v2"); +console.log(`3. ${banner.text.value} | ${s1.text.value} | ${s2.text.value}`); +console.log(`4. still dirty: ${JSON.stringify(plugins.state.value.dirty)}`); + +banner.release(); +s1.release(); +s2.release(); +await workers.close(context); +await server.close(context); +plugins.close(); +await rm(root, { recursive: true, force: true }); diff --git a/packages/chord/test/next/examples/43-rollout/plugins/notes/contract.ts b/packages/chord/test/next/examples/43-rollout/plugins/notes/contract.ts new file mode 100644 index 000000000..86d192d7e --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/plugins/notes/contract.ts @@ -0,0 +1,16 @@ +// The notes' contracts: a label on the server, and one in each session's worker. + +import { hostKind, local, remote, str } from "@earendil-works/chord/next"; + +export const Session = hostKind("session", { id: str() }); + +/** The label as the server shows it. */ +export const Banner = local("example.banner", { version: 1, params: {}, members: { text: str() } }); + +/** The label as a session's worker shows it. */ +export const Label = remote("example.label", { + version: 1, + params: { id: str() }, + hostAddress: ({ id }) => Session({ id }), + variants: { main: { state: { text: str() } } }, +}); diff --git a/packages/chord/test/next/examples/43-rollout/plugins/notes/label.ts b/packages/chord/test/next/examples/43-rollout/plugins/notes/label.ts new file mode 100644 index 000000000..c8c560a6e --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/plugins/notes/label.ts @@ -0,0 +1,3 @@ +// The label every host shows; the example edits it. + +export const label = "v1"; diff --git a/packages/chord/test/next/examples/43-rollout/plugins/notes/package.json b/packages/chord/test/next/examples/43-rollout/plugins/notes/package.json new file mode 100644 index 000000000..d59f5a427 --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/43-rollout/plugins/notes/server.ts b/packages/chord/test/next/examples/43-rollout/plugins/notes/server.ts new file mode 100644 index 000000000..ca8c22e00 --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/plugins/notes/server.ts @@ -0,0 +1,7 @@ +// The notes plugin's server facet. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Banner } from "./contract.ts"; +import { label } from "./label.ts"; + +export default defineFacet((host) => host.serve(Banner, () => ({ text: constant(`server: ${label}`) }))); diff --git a/packages/chord/test/next/examples/43-rollout/plugins/notes/worker.ts b/packages/chord/test/next/examples/43-rollout/plugins/notes/worker.ts new file mode 100644 index 000000000..972e91925 --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/plugins/notes/worker.ts @@ -0,0 +1,9 @@ +// The notes plugin's worker facet. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Label } from "./contract.ts"; +import { label } from "./label.ts"; + +export default defineFacet((host) => + host.serve(Label, ({ id }) => ({ variant: "main", members: { text: constant(`${id}: ${label}`) } })), +); diff --git a/packages/chord/test/next/examples/43-rollout/worker.ts b/packages/chord/test/next/examples/43-rollout/worker.ts new file mode 100644 index 000000000..2fb4fccaa --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/worker.ts @@ -0,0 +1,22 @@ +// The worker process: a host at the session's address, connected to the server, loading the plugins it was shipped. + +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { workerMain } from "../../../../src/next-node/workers/index.ts"; + +workerMain( + async ({ address, parent, plugins, onClose }, context) => { + const host = createHost({ address, principal: { kind: "worker" }, onError: console.error }); + onClose(() => host.close(BACKGROUND_CONTEXT)); + // Connect first: the server sees the worker started; then load. + host.connect(parent, { principal: { kind: "server" }, trusted: true }, context); + await plugins.load(host, { kind: "worker", core: [] }, context); + return {}; + }, + { + hostPackages: { "@earendil-works/chord/next": chordNext }, + redact: (error) => ({ message: String(error) }), + onError: console.error, + }, +); diff --git a/packages/chord/test/next/examples/44-http-wraps/main.ts b/packages/chord/test/next/examples/44-http-wraps/main.ts new file mode 100644 index 000000000..af970636c --- /dev/null +++ b/packages/chord/test/next/examples/44-http-wraps/main.ts @@ -0,0 +1,37 @@ +// Wraps (slice 10): plugins wrap routes, their own or everyone's, in phases (`observe`, `headers`, `transform`, +// outermost first). Here the notes plugin wraps its own route to add a header, and an audit plugin wraps every route +// to log it; the audit plugin needs no route of its own. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/44-http-wraps/main.ts +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import * as chordHttp from "../../../../src/next-node/http/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; + +const server = createHost({ principal: { kind: "server" }, onError: console.error }); +const http = chordHttp.createHttp(server, { listen: { host: "127.0.0.1", port: 0 }, onError: console.error }, context); +const modules = { "@earendil-works/chord/next": chordNext, "@earendil-works/chord/next/http": chordHttp }; +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, + onError: console.error, + }, + context, +); +await plugins.load(server, { kind: "server", core: [{ name: "chord.http", facet: http.facet }], modules }, context); +const { port } = await http.listening(context); + +for (const path of ["/notes", "/missing"]) { + console.log(`GET ${path}`); + const response = await fetch(`http://127.0.0.1:${port}${path}`); + console.log(` -> ${response.status}, Cache-Control: ${response.headers.get("cache-control")}`); +} +// The 404 has no route, so no wrap ran for it: the audit log shows only /notes, and it keeps the default +// `Cache-Control: no-store` that the notes wrap replaced on its own route. + +await http.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/44-http-wraps/plugins/audit/package.json b/packages/chord/test/next/examples/44-http-wraps/plugins/audit/package.json new file mode 100644 index 000000000..10a68118f --- /dev/null +++ b/packages/chord/test/next/examples/44-http-wraps/plugins/audit/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/audit", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/44-http-wraps/plugins/audit/server.ts b/packages/chord/test/next/examples/44-http-wraps/plugins/audit/server.ts new file mode 100644 index 000000000..e4259d470 --- /dev/null +++ b/packages/chord/test/next/examples/44-http-wraps/plugins/audit/server.ts @@ -0,0 +1,24 @@ +// The audit plugin: no routes of its own, one wrap around every route (`scope: "all"`), outermost (`observe`). + +import { defineFacet } from "@earendil-works/chord/next"; +import { Http } from "@earendil-works/chord/next/http"; + +export default defineFacet((_host, self) => + self.contribute(Http, { + wraps: [ + { + name: "log", + phase: "observe", + scope: "all", + wrap: async (request, next, ctx) => { + const response = await next(request); + const status = "status" in response ? response.status : 200; + console.log( + ` audit: ${ctx.principal === undefined ? "anonymous" : "someone"} ${request.method} ${request.path} ${status}`, + ); + return response; + }, + }, + ], + }), +); diff --git a/packages/chord/test/next/examples/44-http-wraps/plugins/notes/package.json b/packages/chord/test/next/examples/44-http-wraps/plugins/notes/package.json new file mode 100644 index 000000000..d59f5a427 --- /dev/null +++ b/packages/chord/test/next/examples/44-http-wraps/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/44-http-wraps/plugins/notes/server.ts b/packages/chord/test/next/examples/44-http-wraps/plugins/notes/server.ts new file mode 100644 index 000000000..0c2a83103 --- /dev/null +++ b/packages/chord/test/next/examples/44-http-wraps/plugins/notes/server.ts @@ -0,0 +1,20 @@ +// The notes plugin: a route, and its own wrap (around its own routes only) adding a header. + +import { defineFacet } from "@earendil-works/chord/next"; +import { Http } from "@earendil-works/chord/next/http"; + +export default defineFacet((_host, self) => + self.contribute(Http, { + routes: { "GET /notes": async () => ({ kind: "json", value: ["buy milk", "call mom"] }) }, + wraps: [ + { + name: "cache", + phase: "headers", + wrap: async (request, next) => { + const response = await next(request); + return { ...response, headers: { ...response.headers, "Cache-Control": "max-age=60" } }; + }, + }, + ], + }), +); diff --git a/packages/chord/test/next/examples/45-plugin-socket/main.ts b/packages/chord/test/next/examples/45-plugin-socket/main.ts new file mode 100644 index 000000000..9f10cd8b8 --- /dev/null +++ b/packages/chord/test/next/examples/45-plugin-socket/main.ts @@ -0,0 +1,64 @@ +// A plugin socket (slice 10): a plugin contributes a WebSocket endpoint for its own protocol (a terminal, voice). The +// server checks the origin and authenticates first: without a principal the upgrade is refused before plugin code. +// Here the app's authenticator reads a principal from a header (a real app: a signed cookie, `readCookie`). +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/45-plugin-socket/main.ts +import { once } from "node:events"; +import { fileURLToPath } from "node:url"; +import { WebSocket } from "ws"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { createHost, defineFacet } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import * as chordHttp from "../../../../src/next-node/http/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; + +type Principal = { readonly kind: "server" } | { readonly kind: "user"; readonly name: string }; +const server = createHost({ principal: { kind: "server" }, onError: console.error }); +const http = chordHttp.createHttp(server, { listen: { host: "127.0.0.1", port: 0 }, onError: console.error }, context); +// The app's core facet: who someone is. +const app = defineFacet((_host, self) => { + self.contribute(chordHttp.Authenticators, (request) => { + const name = request.headers.get("x-user"); + return name === null ? undefined : { kind: "user", name }; + }); +}); +const modules = { "@earendil-works/chord/next": chordNext, "@earendil-works/chord/next/http": chordHttp }; +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, + onError: console.error, + }, + context, +); +const core = [ + { name: "chord.http", facet: http.facet }, + { name: "@example/app", facet: app }, +]; +await plugins.load(server, { kind: "server", core, modules }, context); +const { port } = await http.listening(context); + +// 1. Without a principal: refused (403), the plugin never sees it. +const refused = new WebSocket(`ws://127.0.0.1:${port}/echo/lobby`); +const [, response] = await once(refused, "unexpected-response"); +console.log(`1. anonymous: refused ${response.statusCode}`); + +// 2. As ada: the plugin's handler greets, then echoes. +const socket = new WebSocket(`ws://127.0.0.1:${port}/echo/lobby`, { headers: { "x-user": "ada" } }); +const messages: string[] = []; +socket.on("message", (data) => messages.push(String(data))); +await once(socket, "open"); +socket.send("hello"); +// The greeting and the echo. +while (messages.length < 2) { + await new Promise((resolve) => setTimeout(resolve, 10)); +} +for (const message of messages) console.log(`2. ${message}`); + +// 3. Closing the component closes its sockets (1001, going away); then the host. +const closed = once(socket, "close"); +await http.close(context); +await server.close(context); +const [code] = await closed; +console.log(`3. closed with ${code}`); +plugins.close(); diff --git a/packages/chord/test/next/examples/45-plugin-socket/plugins/echo/package.json b/packages/chord/test/next/examples/45-plugin-socket/plugins/echo/package.json new file mode 100644 index 000000000..ed606991b --- /dev/null +++ b/packages/chord/test/next/examples/45-plugin-socket/plugins/echo/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/echo", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/45-plugin-socket/plugins/echo/server.ts b/packages/chord/test/next/examples/45-plugin-socket/plugins/echo/server.ts new file mode 100644 index 000000000..abf014eb3 --- /dev/null +++ b/packages/chord/test/next/examples/45-plugin-socket/plugins/echo/server.ts @@ -0,0 +1,16 @@ +// The echo plugin: a WebSocket of its own protocol at `/echo/:room`, after the server checked who is connecting. + +import { defineFacet } from "@earendil-works/chord/next"; +import { Http } from "@earendil-works/chord/next/http"; + +export default defineFacet((_host, self) => + self.contribute(Http, { + sockets: { + "/echo/:room": (socket, request, ctx) => { + const who = JSON.stringify(ctx.principal); + socket.send(`welcome to ${request.params.room}, ${who}`); + socket.on("message", (data) => socket.send(`echo: ${String(data)}`)); + }, + }, + }), +); diff --git a/packages/chord/test/next/examples/46-allowed-hosts/main.ts b/packages/chord/test/next/examples/46-allowed-hosts/main.ts new file mode 100644 index 000000000..67c346516 --- /dev/null +++ b/packages/chord/test/next/examples/46-allowed-hosts/main.ts @@ -0,0 +1,61 @@ +// Allowed hosts (slice 10): the server answers only requests on loopback or through a host a plugin contributes. A +// plugin can allow a host at runtime, a tunnel naming its public host, and take it back; links and sockets through a +// host no longer allowed are terminated at once. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/46-allowed-hosts/main.ts +import { request } from "node:http"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { local, method, str } from "../../../../src/next/contracts/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import * as chordHttp from "../../../../src/next-node/http/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; + +/** The tunnel plugin's object, as the app's code declares it (the same contract). */ +const Tunnel = local("example.tunnel", { version: 1, params: {}, members: { open: method(str()), close: method() } }); + +const server = createHost({ principal: { kind: "server" }, onError: console.error }); +const http = chordHttp.createHttp(server, { listen: { host: "127.0.0.1", port: 0 }, onError: console.error }, context); +const modules = { "@earendil-works/chord/next": chordNext, "@earendil-works/chord/next/http": chordHttp }; +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, + onError: console.error, + }, + context, +); +await plugins.load(server, { kind: "server", core: [{ name: "chord.http", facet: http.facet }], modules }, context); +const { port } = await http.listening(context); + +/** GET /hello through `host` (as a tunnel would forward it), printed. */ +async function hello(host: string): Promise { + const answer = await new Promise((resolve) => { + const req = request({ host: "127.0.0.1", port, path: "/hello", headers: { host } }, (res) => { + let body = ""; + res.on("data", (chunk) => { + body += chunk; + }); + res.on("end", () => resolve(`${res.statusCode} ${body}`)); + }); + req.end(); + }); + console.log(` GET /hello via ${host} -> ${answer}`); +} + +const tunnel = await server.get(Tunnel, {}, context); +console.log("1. no tunnel yet"); +await hello(`localhost:${port}`); +await hello("notes.example.net"); +console.log("2. the tunnel opens"); +await tunnel.open("notes.example.net", context); +await hello("notes.example.net"); +console.log("3. the tunnel closes"); +await tunnel.close(context); +await hello("notes.example.net"); + +tunnel.release(); +await http.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/package.json b/packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/package.json new file mode 100644 index 000000000..7681eec3f --- /dev/null +++ b/packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/tunnel", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/server.ts b/packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/server.ts new file mode 100644 index 000000000..2e1239c10 --- /dev/null +++ b/packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/server.ts @@ -0,0 +1,31 @@ +// The tunnel plugin: when its tunnel opens, it allows the tunnel's public host (requests through it are https); when +// it closes, it takes the host back. Its contribution is a writable, so the router follows each change. + +import { defineFacet, local, method, str } from "@earendil-works/chord/next"; +import { Http, type HttpContribution } from "@earendil-works/chord/next/http"; + +export const Tunnel = local("example.tunnel", { + version: 1, + params: {}, + members: { open: method(str()), close: method() }, +}); + +export default defineFacet((host, self) => { + self.contribute(Http, { + routes: { "GET /hello": async (request) => ({ kind: "json", value: { origin: request.origin } }) }, + }); + host.serve(Tunnel, (_params, tunnel) => { + const allowed = tunnel.writable({ hosts: [] }); + tunnel.contribute(Http, allowed); + return { + open: async (name) => { + allowed.set({ hosts: [name] }); + return null; + }, + close: async () => { + allowed.set({ hosts: [] }); + return null; + }, + }; + }); +}); diff --git a/packages/chord/test/next/examples/47-http-own-server/main.ts b/packages/chord/test/next/examples/47-http-own-server/main.ts new file mode 100644 index 000000000..6595b2517 --- /dev/null +++ b/packages/chord/test/next/examples/47-http-own-server/main.ts @@ -0,0 +1,70 @@ +// Your own server (slice 10): the HTTP component is a Web-standard `fetch` handler, so an app with a server of its own +// mounts it instead of Chord's listener. Here a plain Node server converts every request with `nodeRequest` (the +// listener's checks: a malformed target refused, a second `Host` kept for admission to refuse; and the connection: +// loopback requests count as local only with it), passes it to `http.fetch`, and every upgrade to `http.upgrade`. +// With Hono the same is one line: `app.all("/*", (c) => http.fetch(c.req.raw, connection))`. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/47-http-own-server/main.ts +import { createServer, request } from "node:http"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import * as chordHttp from "../../../../src/next-node/http/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; + +// 1. The component, without a listener: no `listen` setting. +const server = createHost({ principal: { kind: "server" }, onError: console.error }); +const http = chordHttp.createHttp(server, { onError: console.error }, context); +const modules = { "@earendil-works/chord/next": chordNext, "@earendil-works/chord/next/http": chordHttp }; +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, + onError: console.error, + }, + context, +); +await plugins.load(server, { kind: "server", core: [{ name: "chord.http", facet: http.facet }], modules }, context); + +// 2. The app's own server: a Web `Request` in, a Web `Response` out (each header line kept: several `Set-Cookie`). A +// real app adds its own routes around it. +const own = createServer(async (message, res) => { + const converted = chordHttp.nodeRequest(message, res); + if (converted.kind === "refused") return void res.writeHead(converted.status).end(); + const response = await http.fetch(converted.request, converted.connection); + for (const [name, value] of response.headers) res.appendHeader(name, value); + res.writeHead(response.status).end(Buffer.from(await response.arrayBuffer())); +}); +own.on("upgrade", http.upgrade); +await new Promise((resolve) => own.listen(0, "127.0.0.1", resolve)); +const address = own.address(); +const port = typeof address === "object" && address !== null ? address.port : 0; + +// 3. Requests through the app's server get the same routes, checks and headers. +for (const path of ["/notes", "/nowhere"]) { + const response = await fetch(`http://127.0.0.1:${port}${path}`); + console.log(`GET ${path} -> ${response.status} ${await response.text()}`); + console.log(` x-content-type-options: ${response.headers.get("x-content-type-options")}`); +} + +// 4. What a Web `Request` cannot carry is refused at the conversion, as Chord's listener refuses it; the app's server +// lives on (`TRACE`: 501; a `Host` with userinfo: 403). +for (const [method, host] of [ + ["TRACE", `127.0.0.1:${port}`], + ["GET", `attacker@127.0.0.1:${port}`], +] as const) { + const status = await new Promise((resolve) => { + request({ host: "127.0.0.1", port, method, path: "/notes", headers: { host } }, (response) => { + response.resume(); + resolve(response.statusCode); + }).end(); + }); + console.log(`${method} /notes (Host: ${host}) -> ${status}`); +} + +// 5. The app closes its server and the component (it refuses, then waits for running requests), then the host. +await new Promise((resolve) => own.close(resolve)); +await http.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/47-http-own-server/plugins/notes/package.json b/packages/chord/test/next/examples/47-http-own-server/plugins/notes/package.json new file mode 100644 index 000000000..d59f5a427 --- /dev/null +++ b/packages/chord/test/next/examples/47-http-own-server/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/47-http-own-server/plugins/notes/server.ts b/packages/chord/test/next/examples/47-http-own-server/plugins/notes/server.ts new file mode 100644 index 000000000..2b7a0abd9 --- /dev/null +++ b/packages/chord/test/next/examples/47-http-own-server/plugins/notes/server.ts @@ -0,0 +1,12 @@ +// The notes plugin's server facet: one route, the same whichever server runs it. + +import { defineFacet } from "@earendil-works/chord/next"; +import { Http } from "@earendil-works/chord/next/http"; + +export default defineFacet((_host, self) => + self.contribute(Http, { + routes: { + "GET /notes": async (request) => ({ kind: "json", value: { notes: ["buy milk"], via: request.origin } }), + }, + }), +); diff --git a/packages/chord/test/next/examples/48-http-baseline/main.ts b/packages/chord/test/next/examples/48-http-baseline/main.ts new file mode 100644 index 000000000..764fcce21 --- /dev/null +++ b/packages/chord/test/next/examples/48-http-baseline/main.ts @@ -0,0 +1,84 @@ +// The security baseline (slice 10): on by default, the app decides only who someone is. Here the app signs people in +// with a signed cookie (`cookie`, `readCookie`) and the notes plugin contributes a page, routes and a webhook. Requests +// another site's page makes in a browser (Fetch Metadata says so) are never authenticated, a foreign `Origin` is +// refused, but a link to a page still opens it signed in, and a cross-site webhook runs anonymous. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/48-http-baseline/main.ts +import { request } from "node:http"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { createHost, defineFacet } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import * as chordHttp from "../../../../src/next-node/http/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; + +type Principal = { readonly kind: "server" } | { readonly kind: "user"; readonly name: string }; +const secret = "a long random secret"; +const server = createHost({ principal: { kind: "server" }, onError: console.error }); +const http = chordHttp.createHttp(server, { listen: { host: "127.0.0.1", port: 0 }, onError: console.error }, context); + +// 1. The app's core facet: a sign-in route setting the cookie, and an authenticator reading it back. +const app = defineFacet((_host, self) => { + self.contribute(chordHttp.Http, { + routes: { + "POST /sign-in": async (request) => { + const set = chordHttp.cookie("user", String(await request.json()), { secret, request, sameSite: "Lax" }); + return { kind: "json", value: "signed in", headers: { "Set-Cookie": set } }; + }, + }, + }); + self.contribute(chordHttp.Authenticators, (request) => { + const name = chordHttp.readCookie(request, "user", secret); + return name === undefined ? undefined : { kind: "user", name }; + }); +}); +const modules = { "@earendil-works/chord/next": chordNext, "@earendil-works/chord/next/http": chordHttp }; +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, + onError: console.error, + }, + context, +); +const core = [ + { name: "chord.http", facet: http.facet }, + { name: "@example/app", facet: app }, +]; +await plugins.load(server, { kind: "server", core, modules }, context); +const { port } = await http.listening(context); +const base = `http://127.0.0.1:${port}`; + +// 2. Ada signs in; her browser keeps the cookie. +const signIn = await fetch(`${base}/sign-in`, { method: "POST", body: '"ada"' }); +const cookie = signIn.headers.get("set-cookie")!.split(";")[0]!; +console.log(`1. signed in: ${cookie.slice(0, 12)}…`); + +/** A request from Ada's browser: her cookie, and what Fetch Metadata says about who made it (`fetch` sets its own). */ +async function browser(label: string, method: string, path: string, headers: Record = {}, body = "") { + const answer = await new Promise((resolve) => { + const sent = request({ host: "127.0.0.1", port, method, path, headers: { cookie, ...headers } }, (response) => { + let text = ""; + response.on("data", (chunk) => { + text += chunk; + }); + response.on("end", () => resolve(`${response.statusCode} ${text}`)); + }); + sent.end(body); + }); + console.log(`${label}: ${answer}`); +} + +const ours = { "sec-fetch-site": "same-origin", origin: base }; +const evil = { "sec-fetch-site": "cross-site", origin: "https://evil.example" }; +await browser("2. our page reads the notes", "GET", "/notes", ours); +await browser("3. our page adds a note", "POST", "/notes", ours, '"call mom"'); +await browser("4. evil.example posts a note", "POST", "/notes", evil, '"pwned"'); +await browser("5. evil.example reads the notes", "GET", "/notes", { "sec-fetch-site": "cross-site" }); +const link = { "sec-fetch-site": "cross-site", "sec-fetch-mode": "navigate", "sec-fetch-dest": "document" }; +await browser("6. a link on evil.example to the page", "GET", "/", link); +await browser("7. a build server's webhook", "POST", "/hooks/build", { origin: "https://ci.example" }); + +await http.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/48-http-baseline/plugins/notes/package.json b/packages/chord/test/next/examples/48-http-baseline/plugins/notes/package.json new file mode 100644 index 000000000..d59f5a427 --- /dev/null +++ b/packages/chord/test/next/examples/48-http-baseline/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/48-http-baseline/plugins/notes/server.ts b/packages/chord/test/next/examples/48-http-baseline/plugins/notes/server.ts new file mode 100644 index 000000000..0980b62f1 --- /dev/null +++ b/packages/chord/test/next/examples/48-http-baseline/plugins/notes/server.ts @@ -0,0 +1,33 @@ +// The notes plugin's server facet: the notes page (a link from another site may open it), the notes as JSON, adding +// a note, and a webhook another site calls (cross-site: it always runs anonymous). + +import { defineFacet } from "@earendil-works/chord/next"; +import { Http } from "@earendil-works/chord/next/http"; + +const notes = ["buy milk"]; + +export default defineFacet((_host, self) => + self.contribute(Http, { + routes: { + "GET /": { + page: true, + handler: async (_request, ctx) => ({ kind: "html", body: `hi ${who(ctx.principal)}` }), + }, + "GET /notes": async (_request, ctx) => ({ kind: "json", value: { notes, as: who(ctx.principal) } }), + "POST /notes": async (request, ctx) => { + if (ctx.principal === undefined) return { kind: "error", status: 401 }; + notes.push(String(await request.json())); + return { kind: "json", value: notes }; + }, + "POST /hooks/build": { + crossSite: true, + handler: async (_request, ctx) => ({ kind: "json", value: who(ctx.principal) }), + }, + }, + }), +); + +/** Who a principal is, for the answers. */ +function who(principal: unknown): string { + return principal === undefined ? "anonymous" : JSON.stringify(principal); +} diff --git a/packages/chord/test/next/host/revocation.test.ts b/packages/chord/test/next/host/revocation.test.ts index 22bc7a9de..a73b222f8 100644 --- a/packages/chord/test/next/host/revocation.test.ts +++ b/packages/chord/test/next/host/revocation.test.ts @@ -536,6 +536,11 @@ describe("revocation of this host's own requests over its paths (fix round 1)", it("ends a guest's reference over a route link that dropped, and nothing holds it on the next one", async () => { const routed: RawPeer[] = []; + // The drop resolves again at once (slice 10 R2); the next link comes only after the revocation. + let next: () => void = () => {}; + const revoked = new Promise((resolve) => { + next = resolve; + }); const host = createHost({ principal: { kind: "owner" }, revokedBy: guestRevokedBy, @@ -546,6 +551,7 @@ describe("revocation of this host's own requests over its paths (fix round 1)", { authorize: () => true, resolve: async () => { + if (routed.length > 0) await revoked; const [a, b] = memoryLinkPair(); routed.push(new RawPeer(b)); return { link: a, principal: { kind: "owner" }, codec: testCodec() }; @@ -569,6 +575,7 @@ describe("revocation of this host's own requests over its paths (fix round 1)", first.close(); await transported(); host.revoke("g", context); + next(); await transported(); expect(states).toEqual(["app.remote: revoked"]); expect(routed.flatMap((peer, i) => (i === 0 ? [] : peer.received.filter(({ kind }) => kind === "hold")))).toEqual( diff --git a/packages/chord/test/next/host/root.test.ts b/packages/chord/test/next/host/root.test.ts index 76c1c2b7e..0847d0477 100644 --- a/packages/chord/test/next/host/root.test.ts +++ b/packages/chord/test/next/host/root.test.ts @@ -1,7 +1,7 @@ import { getEventListeners } from "node:events"; import { describe, expect, it } from "vitest"; import { BACKGROUND_CONTEXT as context, withAbortSignal, withCancel } from "../../../src/context/index.ts"; -import { int, local, method } from "../../../src/next/contracts/index.ts"; +import { hostKind, int, local, method, remote, str } from "../../../src/next/contracts/index.ts"; import { type Ctx, createHost, defineFacet, sleep } from "../../../src/next/host/index.ts"; import { constant } from "../../../src/next/state/index.ts"; import { expectCollected, flushed, loaded, rejection, type TestPrincipal } from "../support.ts"; @@ -33,6 +33,39 @@ describe("host.root", () => { await host.close(context); }); + it("runs anonymous with an explicit undefined principal; only an omitted one takes the host's (slice 10 §6)", async () => { + const host = createHost({ principal: { kind: "server" }, onError: () => {} }); + await host.load([], context); + expect(await host.root(async (ctx) => ctx.principal, context, { principal: undefined })).toBeUndefined(); + expect(await host.root(async (ctx) => ctx.principal, context, {})).toEqual({ kind: "server" }); + await host.close(context); + }); + + // Opus's build review F5: an anonymous scope's `ctx.get` of a remote type served here failed on a WeakMap key. + it("refuses an anonymous scope a remote type served here (forbidden); a local one it gets", async () => { + const Server = hostKind("server"); + const Secrets = remote("app.secrets", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { main: { methods: { read: method(undefined, str()) } } }, + }); + const host = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + const facet = defineFacet((h) => { + serveInfo(h); + h.serve(Secrets, () => ({ variant: "main", members: { read: async () => "secret" } })); + }); + await host.load([{ name: "@app/secrets", facet }], context); + const anonymous = { principal: undefined }; + const refused = await rejection(host.root(async (ctx) => ctx.get(Secrets, {}), context, anonymous)); + expect(refused).toMatchObject({ + code: "forbidden", + message: "app.secrets: an anonymous caller gets no remote object", + }); + expect(await host.root(async (ctx) => (await ctx.get(Info, { id: 3 })).id.value, context, anonymous)).toBe(3); + await host.close(context); + }); + it("seals the context when fn settles: its references released, its signal aborted, `ctx.get` refused `released`", async () => { const { host } = await loaded(serveInfo); let kept: Ctx | undefined; diff --git a/packages/chord/test/next/host/route.test.ts b/packages/chord/test/next/host/route.test.ts index 756ae9429..3d9fbf1f8 100644 --- a/packages/chord/test/next/host/route.test.ts +++ b/packages/chord/test/next/host/route.test.ts @@ -119,39 +119,71 @@ forEachCodec(() => { }); }); - describe("resolve failures and re-resolves", () => { + describe("resolve failures and re-resolves (no timer, slice 10 R2)", () => { fakeTimers(); - it("a rejected resolve leaves the holds loading, is reported, and resolves again after the address's backoff (race 2)", async () => { - let failures = 2; - const { tab, resolves, errors } = await relayed({ + it("a rejected resolve fails its holders waiting with its error, is reported, and runs no timer (race 2)", async () => { + let attempts = 0; + const { tab, server, errors } = await relayed({ + resolve: async () => { + attempts++; + throw new Error("no worker"); + }, + }); + // One relayed from the tab, one the server's own. + const notes = tab.hold(PlacedNotes, { session: "S1" }, context); + const own = server.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + for (const ref of [notes, own]) { + expect(ref.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "unavailable", message: "session|S1: no worker" } }, + }); + } + expect(errors.server).toMatchObject([{ kind: "resolve_failed", address: "session|S1" }]); + await elapse(120_000); + expect(attempts).toBe(1); + }); + + it("a new hold resolves a failed address again; the earlier holders bind on its link without a re-hold (Q1)", async () => { + let fail = true; + const { tab, resolves } = await relayed({ resolve: async (_address, worker) => { - if (failures-- > 0) throw new Error("no worker yet"); + if (fail) throw new Error("no worker"); + return worker(); + }, + }); + const first = tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + expect(first.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + fail = false; + const second = tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + expect(resolves).toEqual(["session|S1"]); + expect([first.state.value, second.state.value]).toEqual([{ kind: "live" }, { kind: "live" }]); + }); + + it("RouteHandle.retry resolves a failed address now; a retry of an address nobody holds keeps nothing", async () => { + let fail = true; + const { tab, route, resolves } = await relayed({ + resolve: async (_address, worker) => { + if (fail) throw new Error("no worker"); return worker(); }, }); const notes = tab.hold(PlacedNotes, { session: "S1" }, context); await transported(); - expect(notes.state.value).toEqual({ kind: "loading" }); - expect(errors.server).toMatchObject([{ kind: "resolve_failed", address: "session|S1" }]); - await elapse(999); - expect(errors.server).toHaveLength(1); - await elapse(1); - // The second failure doubles the window. - expect(errors.server).toHaveLength(2); - await elapse(1999); - expect(notes.state.value).toEqual({ kind: "loading" }); - await elapse(1); - expect(resolves).toEqual(["session|S1"]); + fail = false; + route.retry(Session({ id: "S1" })); + route.retry(Session({ id: "S9" })); + await transported(); expect(notes.state.value).toEqual({ kind: "live" }); + expect(resolves).toEqual(["session|S1"]); }); - it("a returned link that is closed already starts the backoff", async () => { - let closed = true; - const { tab, resolves } = await relayed({ - resolve: async (address, worker) => { - if (!closed) return worker(); - closed = false; + it("a returned link closed already fails the holders", async () => { + const { tab } = await relayed({ + resolve: async (address) => { const [a, b] = memoryLinkPair(); b.close(); await flushed(); @@ -160,20 +192,35 @@ forEachCodec(() => { }); const notes = tab.hold(PlacedNotes, { session: "S1" }, context); await transported(); - expect(notes.state.value).toEqual({ kind: "loading" }); - await elapse(1000); - expect(resolves).toEqual(["session|S1"]); - expect(notes.state.value).toEqual({ kind: "live" }); + expect(notes.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "session|S1: the link closed before its handshake" } }, + }); }); - it("a dropped link resolves again while held: references go loading, then live on a fresh worker (race 6)", async () => { + it("a link closing before its handshake fails its holders: no re-resolve loop over a worker dying at start", async () => { + let attempts = 0; + const { tab } = await relayed({ + resolve: async (address) => { + attempts++; + const [a, b] = memoryLinkPair(); + setTimeout(() => b.close(), 10); + return { link: a, principal: { kind: "worker", session: address.params.id } }; + }, + }); + const notes = tab.hold(PlacedNotes, { session: "S1" }, context); + await elapse(10); + expect(notes.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await elapse(120_000); + expect(attempts).toBe(1); + }); + + it("a dropped link that completed its handshake resolves again at once, once (race 6)", async () => { const { tab, resolves, routeLinks, workers } = await relayed(); const notes = await tab.get(PlacedNotes, { session: "S1" }, context); const first = workers.get("session|S1")!; routeLinks[0]!.close(); await transported(); - expect(notes.state.value).toEqual({ kind: "loading" }); - await elapse(1000); expect(resolves).toEqual(["session|S1", "session|S1"]); expect(workers.get("session|S1")).not.toBe(first); expect(notes.state.value).toEqual({ kind: "live" }); @@ -182,42 +229,13 @@ forEachCodec(() => { expect(notes.notes.value).toEqual([{ text: "fresh" }]); }); - it("only a link that completed its handshake resets the backoff", async () => { - let silent = 0; - const { tab, resolves, routeLinks } = await relayed({ - resolve: async (address, worker) => { - if (silent-- <= 0) return worker(); - // A link whose other end never greets and closes. - const [a, b] = memoryLinkPair(); - setTimeout(() => b.close(), 10); - return { link: a, principal: { kind: "worker", session: address.params.id } }; - }, - }); - const notes = await tab.get(PlacedNotes, { session: "S1" }, context); - silent = 1; - routeLinks[0]!.close(); - await elapse(1000); - // The silent link closed before its handshake: the window doubles. - await elapse(10); - await elapse(1999); - expect(resolves).toEqual(["session|S1"]); - await elapse(1); - expect(resolves).toEqual(["session|S1", "session|S1"]); - expect(notes.state.value).toEqual({ kind: "live" }); - // That link greeted: the next drop starts again at 1 s. - routeLinks.at(-1)!.close(); - await elapse(1000); - expect(resolves).toHaveLength(3); - }); - - it("a link whose peer never greets closes after 30 s of silence; the address backs off, then resolves again while held", async () => { + it("a link whose peer never greets closes after 30 s of silence and fails its holders", async () => { // The heartbeat's timer too, before the hosts start it. vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"], now: 0 }); - let silent = true; - const { tab, resolves } = await relayed({ - resolve: async (address, worker) => { - if (!silent) return worker(); - silent = false; + let attempts = 0; + const { tab } = await relayed({ + resolve: async (address) => { + attempts++; // Its other end is never read: no hello, no close. const [a] = memoryLinkPair(); return { link: a, principal: { kind: "worker", session: address.params.id } }; @@ -225,14 +243,12 @@ forEachCodec(() => { }); const notes = tab.hold(PlacedNotes, { session: "S1" }, context); await elapse(29_000); - expect(resolves).toEqual([]); expect(notes.state.value).toEqual({ kind: "loading" }); - // The heartbeat checks every 1.5 s: the silent link closes within it, then its address waits 1 s. + // The heartbeat checks every 1.5 s: the silent link closes within it. await elapse(2_500); - await elapse(1_000); - expect(resolves).toEqual(["session|S1"]); - await transported(); - expect(notes.state.value).toEqual({ kind: "live" }); + expect(notes.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await elapse(120_000); + expect(attempts).toBe(1); }); it("an address without holds is not resolved again after its link drops", async () => { @@ -276,6 +292,42 @@ forEachCodec(() => { }); }); + describe("RouteHandle.suspend (R1, Q3)", () => { + it("resolves nothing while held: the holders fail waiting suspended once the link is gone; retry lifts it", async () => { + const { tab, route, resolves, routeLinks } = await relayed(); + const notes = await tab.get(PlacedNotes, { session: "S1" }, context); + route.suspend(Session({ id: "S1" })); + expect(notes.state.value).toEqual({ kind: "live" }); + routeLinks[0]!.close(); + await transported(); + expect(notes.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "unavailable", message: "session|S1: suspended" } }, + }); + const another = tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + expect(another.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + expect(resolves).toEqual(["session|S1"]); + route.retry(Session({ id: "S1" })); + await transported(); + expect(resolves).toEqual(["session|S1", "session|S1"]); + expect([notes.state.value, another.state.value]).toEqual([{ kind: "live" }, { kind: "live" }]); + }); + + it("goes with the last holder: a later hold resolves on demand; an unheld address is never suspended", async () => { + const { tab, route, resolves, routeLinks } = await relayed(); + const notes = await tab.get(PlacedNotes, { session: "S1" }, context); + route.suspend(Session({ id: "S1" })); + routeLinks[0]!.close(); + await transported(); + notes.release(); + await transported(); + route.suspend(Session({ id: "S1" })); + await tab.get(PlacedNotes, { session: "S1" }, context); + expect(resolves).toEqual(["session|S1", "session|S1"]); + }); + }); + describe("host.close during a resolve (race 7)", () => { it("aborts the resolve's context and closes a link returned afterwards", async () => { let routing: AbortSignal | undefined; @@ -375,6 +427,37 @@ forEachCodec(() => { }); describe("GC", () => { + it("a failed or suspended address keeps no route state once its holders are released", async () => { + const { refs, server } = await (async () => { + const refs: WeakRef[] = []; + const { tab, server, route, routeLinks, workers, errors } = await relayed({ + authorize: (_principal, address) => { + refs.push(new WeakRef(address)); + return true; + }, + resolve: async (address, worker) => { + if (address.params.id === "failed") throw new Error("no worker"); + return worker(); + }, + }); + const failed = tab.hold(PlacedNotes, { session: "failed" }, context); + const held = await tab.get(PlacedNotes, { session: "held" }, context); + route.suspend(Session({ id: "held" })); + routeLinks[0]!.close(); + await transported(); + for (const ref of [failed, held]) ref.release(); + await transported(); + // The reported error's stack keeps the test's `resolve` frame (and its address) alive, the worker host its + // own address: not the route's. + errors.server.length = 0; + routeLinks.length = 0; + workers.clear(); + return { refs, server }; + })(); + await expectCollected(...refs); + expect(server).toBeDefined(); + }); + it("a closed route link and its released references keep no route state", async () => { const { refs, tab } = await (async () => { const { tab, routeLinks, workers } = await relayed(); diff --git a/packages/chord/test/next/remote/host-link.test.ts b/packages/chord/test/next/remote/host-link.test.ts index c5205ea12..a3da2f05d 100644 --- a/packages/chord/test/next/remote/host-link.test.ts +++ b/packages/chord/test/next/remote/host-link.test.ts @@ -3,7 +3,7 @@ import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; import { int, local, method, remote, str } from "../../../src/next/contracts/index.ts"; import { createHost, defineFacet, type FacetHost, type HostError } from "../../../src/next/host/index.ts"; import { type Compression, memoryLinkPair, type PartLimits } from "../../../src/next/link/index.ts"; -import { jsonPart } from "../../../src/next/link/internal.ts"; +import { jsonPart, type Message } from "../../../src/next/link/internal.ts"; import { constant } from "../../../src/next/state/index.ts"; import { nodeCompression } from "../../../src/next-node/compression.ts"; import { @@ -19,12 +19,15 @@ import { partValue, RAW_PEER, RawPeer, + rawLinked, rawWorkers, rejection, relayed, + SERVER_ADDRESS, Server, Session, serveNotes, + settledHeap, type TestPrincipal, testCodec, transported, @@ -431,3 +434,76 @@ forEachCodec(() => { }); }); }); + +describe("origins only from relays (decision, slice 10 security audit K1/K2)", () => { + const { name: type, version, fingerprint } = Notes; + const hold = { + kind: "hold", + hold: 0, + type, + version, + fingerprint, + params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, + }; + const withOrigin: Record = { + hold: { ...hold, origin: 1 } as Message, + call: { kind: "call", origin: 1, call: 0, hold: 0, method: EDITOR.methods.wait }, + release: { kind: "release", origin: 1, hold: 0 }, + abort: { kind: "abort", origin: 1, call: 0 }, + retry: { kind: "retry", origin: 1, hold: 0, hops: 1 } as Message, + ack: { kind: "ack", origin: 1 }, + closed: { kind: "closed", origin: 1 }, + }; + + for (const [kind, message] of Object.entries(withOrigin)) { + it(`closes a child link sending ${kind} with an origin (a protocol error)`, async () => { + const { peer, errors } = await rawLinked((h) => void serveNotes(h)); + peer.send(message); + await transported(); + expect(peer.closed).toBe(true); + expect(errors).toMatchObject([ + { kind: "protocol_error", error: { message: `${kind}: an origin on a child link` } }, + ]); + }); + } + + it("honours origins on a parent link (a relay's): one window of calls per origin", async () => { + let probe!: ReturnType; + const { peer } = await rawLinked( + (h) => { + probe = serveNotes(h); + }, + { role: "parent" }, + ); + peer.send({ ...hold, origin: 1 } as Message); + peer.send({ ...hold, origin: 2 } as Message); + await transported(); + for (const origin of [1, 2]) peer.send({ kind: "call", origin, call: 0, hold: 0, method: EDITOR.methods.wait }); + await transported(); + expect([peer.closed, probe.waits.length]).toEqual([false, 2]); + }); + + it("keeps no record of an origin that holds nothing: 20,000 fresh origins held and released", async () => { + const { peer } = await rawLinked((h) => void serveNotes(h), { role: "parent" }); + const churn = async (from: number, to: number) => { + for (let origin = from; origin <= to; origin++) { + peer.send({ ...hold, origin } as Message); + peer.send({ kind: "release", origin, hold: 0 }); + if (origin % 1_000 === 0) { + await transported(); + peer.received.length = 0; + } + } + await transported(); + peer.received.length = 0; + }; + // A warm-up first: what a first run allocates for good (compiled code, caches) is no record. + await churn(1, 2_000); + const before = await settledHeap(); + await churn(2_001, 22_000); + // About 1 KiB per origin before: 20 MiB. + expect((await settledHeap()) - before).toBeLessThan(512 * 1024); + expect(peer.closed).toBe(false); + }, 60_000); +}); diff --git a/packages/chord/test/next/remote/path.test.ts b/packages/chord/test/next/remote/path.test.ts index c3efb197d..ff58b786c 100644 --- a/packages/chord/test/next/remote/path.test.ts +++ b/packages/chord/test/next/remote/path.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import { ChordError } from "../../../src/next/data/index.ts"; import type { HostLink } from "../../../src/next/remote/host-link.ts"; import { Path, type Upstream } from "../../../src/next/remote/path.ts"; @@ -67,4 +68,23 @@ describe("a path", () => { path.connected(linkOf("2")); expect(log).toEqual(["a@1", "b@1"]); }); + + it("says what changed: a holder attached without a link, a drop, the last release; fails its holders waiting", () => { + const events: string[] = []; + const failed: string[] = []; + const path = new Path((event) => events.push(event.kind)); + const a = { + attach: () => {}, + dropped: () => {}, + unreachable: (error: { readonly message: string }) => void failed.push(error.message), + } as unknown as Upstream; + path.attach(a); + path.unreachable(new ChordError({ code: "unavailable", message: "no worker" })); + const link = linkOf("1"); + path.connected(link); + path.disconnected(link); + path.detach(a); + expect(events).toEqual(["attached", "dropped", "released"]); + expect(failed).toEqual(["no worker"]); + }); }); diff --git a/packages/chord/test/next/remote/provider.test.ts b/packages/chord/test/next/remote/provider.test.ts index dc9f30aed..5abb539e3 100644 --- a/packages/chord/test/next/remote/provider.test.ts +++ b/packages/chord/test/next/remote/provider.test.ts @@ -74,13 +74,13 @@ const READER = wireNames(Notes, "reader"); const { add, fail, wait } = EDITOR.methods as { add: number; fail: number; wait: number }; /** A server serving `Notes` and a raw consumer peer stamped `child`. */ -async function notesServer(child: TestPrincipal = { kind: "owner" }) { +async function notesServer(child: TestPrincipal = { kind: "owner" }, role: "child" | "parent" = "child") { let probe: NotesProbe | undefined; const linked = await rawLinked( (h) => { probe = serveNotes(h); }, - { child }, + { child, role }, ); return { ...linked, probe: probe! }; } @@ -551,7 +551,7 @@ function serveGrid(h: FacetHost): void { } /** A server serving `Notes` and `Grid` to a raw peer that acks only when told, the link's context counting. */ -async function slowConsumer() { +async function slowConsumer(role: "child" | "parent" = "child") { const counting = await metered(); const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); await server.load( @@ -567,7 +567,9 @@ async function slowConsumer() { context, ); const [a, b] = memoryLinkPair(); - server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, counting.context); + const options = { principal: { kind: "owner" }, codec: testCodec() } as const; + if (role === "parent") server.connect(a, options, counting.context); + else server.accept(a, options, counting.context); const peer = new RawPeer(b, { ack: false }); await transported(); return { server, peer, counts: counting.counts }; @@ -667,9 +669,9 @@ forEachCodec(() => { }); }); - describe("origins at the provider", () => { + describe("origins at the provider (a relay's: on its parent link)", () => { it("key holds and calls by (origin, id): the same ids on two origins are two holds; answers carry the origin", async () => { - const { peer } = await slowConsumer(); + const { peer } = await slowConsumer("parent"); hold(peer, 0, Notes, { session: "S1" }); hold(peer, 0, Notes, { session: "S2" }, {}, 3); const [own, relayed] = [await peer.next("bound"), await peer.next("bound")]; @@ -685,7 +687,7 @@ forEachCodec(() => { }); it("keep a window per origin: a stalled origin folds its commits, another keeps receiving each", async () => { - const { server, peer } = await slowConsumer(); + const { server, peer } = await slowConsumer("parent"); hold(peer, 0, Notes, { session: "S1" }); hold(peer, 0, Notes, { session: "S1" }, {}, 1); await peer.next("bound"); @@ -710,7 +712,7 @@ forEachCodec(() => { }); it("count calls per origin against N: 256 on each of two origins run, one more on one closes the link", async () => { - const { peer, errors } = await notesServer(); + const { peer, errors } = await notesServer({ kind: "owner" }, "parent"); hold(peer, 0, Notes, { session: "S1" }); hold(peer, 0, Notes, { session: "S1" }, {}, 1); await peer.next("bound"); @@ -730,7 +732,7 @@ forEachCodec(() => { }); it("ignore a call on a hold they do not know before counting it (Q1)", async () => { - const { peer, errors } = await notesServer(); + const { peer, errors } = await notesServer({ kind: "owner" }, "parent"); hold(peer, 0, Notes, { session: "S1" }); await peer.next("bound"); for (let id = 0; id < 256; id++) call(peer, id, 0, wait); @@ -742,7 +744,7 @@ forEachCodec(() => { }); it("closed releases the origin's holds and aborts its calls; another origin keeps its own", async () => { - const { peer, probe, server } = await notesServer(); + const { peer, probe, server } = await notesServer({ kind: "owner" }, "parent"); hold(peer, 0, Notes, { session: "S1" }); hold(peer, 0, Notes, { session: "S2" }, {}, 2); await peer.next("bound"); @@ -824,7 +826,7 @@ forEachCodec(() => { push.call(this, ops); }; try { - const { server, peer } = await slowConsumer(); + const { server, peer } = await slowConsumer("parent"); hold(peer, 0, Notes, { session: "S1" }, {}, 1); await peer.next("bound"); for (const text of ["a", "b", "c"]) await addNote(server, "S1", text); diff --git a/packages/chord/test/next/remote/relay.test.ts b/packages/chord/test/next/remote/relay.test.ts index 94148dccd..6f08a6f14 100644 --- a/packages/chord/test/next/remote/relay.test.ts +++ b/packages/chord/test/next/remote/relay.test.ts @@ -22,6 +22,7 @@ import { rejection, relayed, Server, + Session, serveNotes, type TestPrincipal, testCodec, @@ -352,8 +353,8 @@ forEachCodec(() => { first.close(); expect(await rejection(call)).toMatchObject({ code: "unavailable" }); await transported(); + // The link had completed its handshake: its address resolves again at once (no timer, slice 10 R2). expect(notes.state.value).toEqual({ kind: "loading" }); - await vi.advanceTimersByTimeAsync(1000); const second = await workerAt(workers, 1); const again = await second.next("hold"); expect(again).toMatchObject({ hold: held.hold, origin: held.origin }); @@ -366,7 +367,13 @@ forEachCodec(() => { }); it("answers a call that crossed the loading notice unavailable (race 13)", async () => { - const { raw, workers } = await rawRelay(); + // The address's next resolve never settles: the relayed hold has no upstream link when the call arrives. + const workers: RawPeer[] = []; + const resolve = rawWorkers(workers); + const relay = await relayed({ + resolve: (address) => (workers.length === 0 ? resolve(address) : new Promise(() => {})), + }); + const raw = rawTab(relay.server); holdPlaced(raw, 0); const worker = await workerAt(workers); bindPlaced(worker, await worker.next("hold")); @@ -381,27 +388,48 @@ forEachCodec(() => { describe("a parent link that is down (audit)", () => { it("holds a hold arriving meanwhile on the parent path: it waits loading and binds on the next parent link", async () => { + // The middle host relays its tab's holds to its parent, which reaches it over a route: origins arrive only on + // parent and route links (decision "Origins only from relays"). + const middle = createHost({ + address: Session({ id: "M" }), + principal: { kind: "worker", session: "M" }, + onError: () => {}, + }); + await middle.load([], context); const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {}, }); + const links: Link[] = []; + let gate = Promise.resolve(); + server.route( + Session, + { + authorize: () => true, + resolve: async () => { + await gate; + const [a, b] = memoryLinkPair(); + middle.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); + links.push(a); + return { link: a, principal: { kind: "worker", session: "M" }, codec: testCodec() }; + }, + }, + context, + ); await server.load( [{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context, ); - const middle = createHost({ principal: { kind: "server" }, onError: () => {} }); - await middle.load([], context); - /** Connects the middle host to the server over a new link; returns the server's end. */ - const up = (): Link => { - const [a, b] = memoryLinkPair(); - server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); - middle.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); - return a; - }; - const first = up(); + // The server holds something at the middle's address: its route link exists while it does. + const anchor = server.hold(PlacedNotes, { session: "M" }, context); await transported(); - first.close(); + // The next resolve waits: the middle's parent path is down meanwhile. + let admit: () => void = () => {}; + gate = new Promise((resolve) => { + admit = resolve; + }); + links[0]!.close(); await transported(); const tab = createHost({ onError: () => {} }); await tab.load([], context); @@ -411,9 +439,11 @@ forEachCodec(() => { const notes = tab.hold(Notes, { session: "S1" }, context); await transported(); expect(notes.state.value).toEqual({ kind: "loading" }); - up(); + admit(); + await transported(); await transported(); expect(notes.state.value).toEqual({ kind: "live" }); + anchor.release(); }); }); @@ -447,9 +477,12 @@ forEachCodec(() => { bindPlaced(worker, sent); worker.send(countState([sent], 1)); await transported(); - // The relay no longer relays the hold: dropped there (Q1), each message acknowledged to the worker at once. + // The origin held nothing more: its record went and the worker was told `closed` (decision "Origins only from + // relays"); what crossed it is dropped (Q1). expect(raw.kinds()).toEqual(["hello"]); - expect(worker.received.filter(({ kind }) => kind === "ack")).toHaveLength(2); + expect(worker.received.filter(({ kind }) => kind === "closed")).toEqual([ + { kind: "closed", origin: sent.origin }, + ]); }); }); diff --git a/packages/chord/test/next/support.ts b/packages/chord/test/next/support.ts index 7037ab438..20732c04f 100644 --- a/packages/chord/test/next/support.ts +++ b/packages/chord/test/next/support.ts @@ -40,6 +40,7 @@ import { type HostError, type PluginFailure, type RefState, + type RouteHandle, type Self, type StoreSelf, } from "../../src/next/host/index.ts"; @@ -770,6 +771,8 @@ export type Resolved = { export type Relayed = { readonly tab: Host; readonly server: Host; + /** The server's route to `Session`'s addresses. */ + readonly route: RouteHandle; /** The worker of each resolved session address, the latest one per key. */ readonly workers: Map>; /** The server's end of each resolved route link, in order. */ @@ -851,7 +854,7 @@ export async function relayed( return { link: a, principal, trusted: options.trusted, codec }; }; const resolve = options.resolve ?? (async (_address, made) => made()); - server.route( + const route = server.route( Session, { authorize: options.authorize ?? (() => true), @@ -870,7 +873,7 @@ export async function relayed( const accepted = { principal: options.tab ?? { kind: "owner" }, codec: tabCodec, limits: options.limits } as const; server.accept(a, accepted, context); tab.connect(b, { principal: { kind: "server" }, codec: tabCodec }, context); - return { tab, server, workers, routeLinks, tabLink: a, resolves, errors }; + return { tab, server, route, workers, routeLinks, tabLink: a, resolves, errors }; } /** diff --git a/tsconfig.json b/tsconfig.json index b49e773ff..7a9cef85c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -10,6 +10,15 @@ "@earendil-works/chord/delta": ["./packages/chord/src/delta/index.ts"], "@earendil-works/chord/next": ["./packages/chord/src/next/index.ts"], "@earendil-works/chord/next/plugins": ["./packages/chord/src/next-node/plugins/index.ts"], + "@earendil-works/chord/next/plugins/runtime": ["./packages/chord/src/next-node/plugins/runtime.ts"], + "@test/contracts/*": ["./packages/chord/test/next-node/fixtures/plugins/contracts/*.ts"], + "@example/contracts/*": ["./packages/chord/test/next/examples/35-plugins/plugins/contracts/*.ts"], + "@example/weather/*": ["./packages/chord/test/next/examples/36-plugin-failures/plugins/weather/*.ts"], + "@example/prices/*": ["./packages/chord/test/next/examples/36-plugin-failures/plugins/prices/*.ts"], + "@example/shapes/*": ["./packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/*.ts"], + "@example/shared/*": ["./packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/*.ts"], + "@earendil-works/chord/next/http": ["./packages/chord/src/next-node/http/index.ts"], + "@earendil-works/chord/next/workers": ["./packages/chord/src/next-node/workers/index.ts"], "@earendil-works/chord/node": ["./packages/chord/src/node.ts"], "@earendil-works/chord/output": ["./packages/chord/src/output.ts"], "@earendil-works/chord/telemetry-sqlite": ["./packages/chord/src/telemetry-sqlite/index.ts"], From b22c5f6941b3472cc071461f933b6b7bd072823f Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 20:43:29 +0000 Subject: [PATCH 223/271] docs(chord): slice 10 as built (design 23, spec, explainer) --- packages/chord/docs/next/decisions.md | 6 +- packages/chord/docs/next/design.md | 135 +++++++++++----- packages/chord/docs/next/http-workers/book.md | 63 ++++---- .../chord/docs/next/http-workers/index.html | 72 +++++---- packages/chord/docs/next/slice-10.md | 152 +++++------------- packages/chord/docs/next/slices.md | 5 + 6 files changed, 221 insertions(+), 212 deletions(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 941b1dee7..e7aed524b 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1877,7 +1877,7 @@ validation and copies the v2 kernel added around Delta. reload, swapping the committed plugin back silently. Rejected: resolving with the report after the commit despite the abort (it would change the rule that an abort after the commit rejects). Examples for 7c (owner): full coverage, each example a folder with the host code and real plugin directories (35-39). -- **Slice 10's points (owner, 2026-10-30; `slice-10.md`):** (D1, "a") the HTTP component defines the registry `Http` +- **Slice 10's points (owner, 2026-10-30; `slice-10.md`):** (D1, "a"; `httpServer` **superseded** by `createHttp`, decisions "The HTTP core is a fetch handler" and "Settings are arguments") the HTTP component defines the registry `Http` (`"chord.http"`) and serves it through a facet of its own (`http.facet`, plugin name `chord.http`) that the app loads with its plugins; its `compose` builds the router; the server reads it with `host.registry(Http, ctx)` per request. (D2, "a") Wraps are built in slice 10 as designed (decision "HTTP routes and wraps", design §23): a contribution's @@ -1885,7 +1885,7 @@ validation and copies the v2 kernel added around Delta. phase in contributor order, by default around their own plugin's routes (`scope: "own"`; `"all"` explicit; `route` narrows to one, a missing one a reported conflict). The server's `headers` option stays for every response, also those no route makes (a 404, a refused request). - (D3, "a") Authentication is one app function, `authenticate(request, ctx) => P | undefined`, called once per request + (D3, "a"; **superseded** by "The HTTP server is a native object": the `Authenticators` registry) Authentication is one app function, `authenticate(request, ctx) => P | undefined`, called once per request and per upgrade; no Chord registry of resolvers (p3 composes its own from sharing, devices and the relay, with its own precedence rules). (D4, "k": a) Request bodies: `request.json(limit?)` (parsed JSON, 1 MiB by default, malformed: 400) and @@ -1948,7 +1948,7 @@ validation and copies the v2 kernel added around Delta. (R3, "a"; refines D9) The child reports only the flip: `{ idle: true }` when every count (`host.activity` and `busy`) becomes zero, `{ idle: false }` when one becomes nonzero; the server's idle clock starts at `true` (a status message costs about 6 µs on each side, a local call 2.9 µs: no report per call). - (R4, "a if configurable at runtime so allowed domains can be added and removed by a plugin") Before `authenticate`, the + (R4; **superseded in part** by "allowed hosts only through contributions" and "The HTTP component's security baseline": no `allowedHosts` option, provenance by the socket; "a if configurable at runtime so allowed domains can be added and removed by a plugin") Before `authenticate`, the component refuses (403) a `Host` that is neither loopback nor allowed, and an upgrade (`/chord`, a plugin socket) whose `Origin` is not the request's own origin; `request.origin` comes only from an accepted `Host` (`http://` on loopback, `https://` for an allowed host); `headers(request, principal)` gets the principal. Allowed hosts are the diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index 966776b47..d40d82677 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -243,7 +243,7 @@ host). The workers component (§23) does this. | `hold(Type, params, context, { readyTimeout? })` | a reference at once that follows its object until released (section 7) | | `close(context)` | closes the links (children, routes, then the parent), then ends every object at once and tears them down, the last to go live first (reason `close`); rejects with an `AggregateError` of the native closes that threw | | `revoke(r, context)` | revocation (section 4; slice 4b) | -| `root(fn, context, { principal? })` | a call scope for code outside objects (a Durable hook, a CLI): `fn` gets a handler's `ctx`, sealed when `fn` settles, `context` aborts or the host closes (slice 8 D3, section 8) | +| `root(fn, context, { principal? })` | a call scope for code outside objects (a Durable hook, a CLI, an HTTP request): `fn` gets a handler's `ctx`, sealed when `fn` settles, `context` aborts or the host closes (slice 8 D3, section 8); an omitted `principal` acts as the host, an explicit `undefined` runs anonymous (slice 10) | | `registry(R, context, { readyTimeout? })` | registry `R`'s composed value for code outside objects once it is usable; waits for the publication, then while it has none up to its `readyTimeout`, then rejects with the last error (slice 8, 7a, section 17) | **Options of `createHost`:** `address`, `principal`, `principalKey` (section 4; the same function on every host), @@ -2113,32 +2113,70 @@ public API; nothing in the kernel depends on them. ### HTTP (`@earendil-works/chord/next/http`) -`httpServer(host, { listen, allowedHosts?, authenticate, headers?, links?, closeTimeout?, onError }, ctx)` listens -first; requests and upgrades wait only for the router's publication (`host.registry(Http)`, resolved once), never for -the start barrier. `Host` (loopback or allowed: the option plus contributed `hosts`) and an upgrade's `Origin` are -checked before anything else (R4); connections through a host no longer allowed close. `authenticate(request, ctx) => P | undefined` is the app's one hook (D3: who a -request is, and which credential wins, is app policy; no Chord registry of resolvers); `headers` are added to every -response, also those no route makes; `links.path` (`/chord`) upgrades to `host.accept(webSocketLink(ws), { principal, -limits })`, a socket without a principal refused. `ws` is an optional peer dependency (D7). - -The registry `Http` (`"chord.http"`) is defined by the component and served by its own facet (`http.facet`, loaded -with the plugins, D1); plugins contribute `routes`, `wraps` and `sockets`. A route is `(req, ctx) => -Promise`, run in a `host.root` scope with the request's principal (`ctx.principal`) and a signal aborted -when the client disconnects; `req` has `method`, `path`, `params`, `query`, `headers`, `origin`, `json(limit?)` and -`body(limit?)` (D4: nothing read until asked; 1 MiB by default; past the limit 413). Patterns (D8): exact segments, -`:name`, a trailing `/*`; the most specific match wins, segment by segment from the left; one pattern from two -contributions is a reported conflict, the first by plugin name wins; `HEAD` falls back to `GET`. A request uses the -router as it was when it started. +The core is one Web-standard handler built from the registry `Http` (decision "The HTTP core is a fetch handler"): ```ts -type HttpResponse = { readonly headers?: Readonly> } & ( // D5 +const http = createHttp(server, { // settings are arguments, fixed for the component's life + listen: { host: "127.0.0.1", port: 8788 }, // Chord's own Node listener; none: the app mounts `fetch` + links: { path: "/chord", options: (request, principal) => ({ limits, maxPayload }) }, + closeTimeout: 30_000, + onError, // HttpError +}, ctx); +await plugins.load(server, { kind: "server", core: [{ name: "chord.http", facet: http.facet }, appCore], modules }, ctx); +// an app with its own server: hono.all("/*", (c) => http.fetch(c.req.raw, { localAddress, localPort, remoteAddress })) +// plain Node: nodeRequest(message, res) converts with the listener's checks; server.on("upgrade", http.upgrade) +await http.close(ctx); // before host.close: drains running requests +``` + +- **The facet** (plugin `chord.http`, passed as the same object at every reload) serves the registries `Http` + (`compose` builds the router) and `Authenticators`, and with `listen` the listener: a host-lifetime object whose + native is Chord's Node server, opened right after `load`'s publication, so a reload never closes it (decisions "The + HTTP server is a native object", "Settings are arguments, extensions are registries"). `fetch` resolves the + registries once; requests before the publication wait (503 past `readyTimeout`). +- **Contributions to `Http`:** `{ routes?, wraps?, sockets?, maxPayload?, hosts?, framers?, relays?, csp? }`, a value + or a writable (a change recomposes the router). A request uses the router as it was when it started. +- **The security baseline** (decision "The HTTP component's security baseline", on for every request and upgrade + before any plugin code): (1) **provenance**: local only when the connection's local address is loopback, `Host` is + a loopback name with the port and no `Forwarded`/`X-Forwarded-*` header; a forwarded request on loopback only + through a contributed relay whose `verify` names its origin (`{ kind: "relayed", relay, origin }`); otherwise only a + contributed host (https); duplicate or missing `Host`, userinfo in it, absolute-form targets: 403; an unparsable + target 400; `TRACE`, `TRACK`, `CONNECT` 501 (a route for them is refused at compose). A mounted handler without + `localAddress` and `localPort` is never local. (2) **Foreign requests** (Fetch Metadata) never reach the + authenticators: anonymous, except a top-level `GET` navigation to a `page: true` route. (3) A foreign `Origin` is + 403; an unsafe method from a browser needs the app's own `Origin`; a `crossSite: true` route opts out and always + runs anonymous. (4) **Default headers** on every answer, the component's own included: `no-store`, `nosniff`, + `no-referrer`, `X-Frame-Options: DENY` (without framers), a CSP (`default-src 'none'`, scripts `'self'` and a + per-response nonce `request.nonce`, `connect-src` self and its WebSocket, `frame-ancestors` the framers or + `'none'`, plus contributed `csp` sources); a response's own headers replace same-named ones. (5) `cookie(name, + value, { secret, request, sameSite?, maxAge? })` and `readCookie(request, name, secret)`: `HttpOnly`, `Path=/`, + `SameSite=Strict` or `Lax`, `Secure` on a remote origin, value and expiry signed. (6) `bytes` refuse foreign + requests, are attachments unless `inline` (then with a `sandbox` CSP), download names stripped of control and + bidirectional characters. +- **Who:** the `Authenticators` registry, in contributor order (an app needing precedence contributes one that + decides); the first principal wins, a throwing one is reported and skipped, none: anonymous. They run raced against + the request's signal and stop once it aborts. Who someone is stays app policy (D3); the registry replaces D3's + single function so plugins (a device pairing, a share) can add credentials. +- **A request's scope:** one `host.root` with its principal (an explicit `undefined`: anonymous; an anonymous scope + gets no remote object served here, `forbidden` → 403) spans matching, wraps, the route and the response until its + body is written or abandoned; its signal aborts on disconnect. `HttpRequest`: `method`, `path`, `params`, `query`, + `headers`, `origin`, `provenance`, `nonce`, `json(limit?)` (1 MiB; 400, 413), `body(limit?)`. +- **Patterns** (D8): exact segments, `:name`, a trailing `/*`; decoded segments with `/`, `\`, NUL, `.` or `..` are + 400; the most specific match wins segment by segment; a pattern from two contributions is a reported conflict, the + first by plugin name wins; `HEAD` falls back to `GET`; no route: 404. A route is a function or `{ handler, page?, + crossSite? }`. Wraps (D2) as below. + +```ts +type HttpResponse = { readonly headers?: Readonly> } & ( // D5 | { kind: "json"; value: JsonValue; status?: number } | { kind: "html"; body: string; status?: number } - | { kind: "bytes"; mime: string; size?: number; read(offset, length, ctx): Promise; download?: string; ranges?: boolean } // no size: chunked (R6) + | { kind: "bytes"; mime: string; size?: number; read(offset, length, ctx): Promise; download?: string; inline?: true; ranges?: boolean } | { kind: "redirect"; to: string; status?: 302 | 303 | 307 } | { kind: "error"; status: number; message?: string }); ``` -The component does ranges (206/416), `HEAD`, safe download headers, 64 KiB reads with backpressure stopped on -disconnect, and a 500 without the message for a throwing route; `fileResponse(path, options)` serves a file. +`bytes` are pulled in 256 KiB reads as the consumer reads (ranges 206/416, `HEAD`, chunked without `size`, R6); no +read starts once the request's scope is sealed; a short or throwing read destroys the connection. 204, 205 and 304 +carry no body. `fileResponse(path, { download?, inline?, ranges?, mime?, root? })` stats at once (missing: 404; with +`root`, a real path outside it: 403), opens its handle at the first read, closes it when the scope seals. A throwing +route or wrap is a 500 without its message. ```ts type Wrap = { @@ -2152,26 +2190,47 @@ type Wrap = { Wraps (D2) run within a phase in contributor order (no numeric priorities: plugins have no single registration order; Fastify's lifecycle phases and encapsulation are the model). Wraps may add a security boundary on top of a -route's checks, never instead of them. `sockets` (D6): `{ "/terminal": (socket, req, ctx) => … }`, a `ws` WebSocket -after the origin check and `authenticate` (no principal: 403 before plugin code), for a plugin's own protocol; wraps do -not apply; closed (1012) when the contribution that handed them out leaves the table (§17). +route's checks, never instead of them. + +- **Sockets** (D6): `sockets: { "/terminal/:id": (socket, request, ctx) => … }`, a `ws` WebSocket after admission + and the authenticators (no principal: 403 before plugin code), `maxPayload` per contribution (1 MiB), pinged after + 15 s of silence and terminated 15 s later. **Links:** `links.path` upgrades to `host.accept(webSocketLink(socket), { + principal, limits })`; a client that stops reading is cut off past 4 MiB waiting. **What is removed is closed at + once** (decisions Q6, "allowed hosts only through contributions"): a link or socket whose host, relay or socket + handler leaves the router is terminated and its scope sealed; an upgrade is checked again against the current router + after its authenticators answered. +- **`close(ctx)`:** new requests and upgrades refused (503), sockets and links closed (1001, terminated at + `closeTimeout`), running requests waited for up to `closeTimeout`, then sealed; before `host.close`. +- **Cost** (decision "a route's overhead accepted"): +20-21 µs per request over a plain Node handler (standard bench), + +12-13 µs warmed. `ws` is an optional peer dependency (D7). ### Workers (`@earendil-works/chord/next/workers`) -`createWorkers(host, Kind, { kind, entry, start, plugins, lock, idleMs?, onOutput?, onError }, ctx)` routes `Kind` -itself (a forked worker per address over IPC, the link made at the fork, a start message with the app's start data and -the worker kind's complete scripts; the 60 s start timer covers fork, lock and connect only, Q4), with a `state` -watchable per address, `stop`, `retry` and `close`; `workerMain(fn, { hostPackages, redact, onError })` is the child's -side. One live worker per address (a kernel lock the child takes, waiting for a draining old worker without polling). -Keep-alive (D9, R3, Q5): the child reports when `host.activity`'s references, jobs and lingering objects and the app's -`busy` counts flip between all zero and not (local calls do not count; the first state with `ready`); the server stops -a worker idle (all zero) for `idleMs` (5 min). Workers start on demand and never restart by themselves (R2): a crash, -a `fatal`, a link lost while its child lives (killed after the grace) or a failed start leaves the address failed with -its error; its holders see `failed { waiting }` and stay on the path (Q1); `workers.retry(address)` or a rollout -reaching workers clears it and starts the held ones (Q2). `workers.stop` suspends the address while it has holders -(R1, Q3). Workers never build (D11): the server's plugins component builds every plugin; workers evaluate shipped -complete sets (`createPluginRuntime`, no esbuild), and `workers.step(name)` ships a rollout's set to every worker (D12: -one result per host; a worker stopped cleanly during it is listed `stopped`). +`createWorkers(host, Kind, { kind, entry, execArgv?, env?, authorize, principal, start, lock, idleMs?, plugins, +onOutput?, onError }, ctx)` routes `Kind` itself (`host.route`, keeping its `RouteHandle`): a forked worker per +address over IPC, the link made at the fork, a start message `{ address, start, scripts, lock }` with the app's start +data and the kind's complete current scripts (`plugins.scripts(kind)`); the 60 s start timer covers fork, lock and +connect only (Q4). `workers.state` (a JSON watchable per address key), `stop`, `retry`, `step(name)`, `close`; +`workerMain(fn, { hostPackages, redact, onError })` is the child's side (`fn({ address, start, parent, plugins, +onClose }, ctx)`, `plugins` a `WorkerPlugins` evaluating the shipped scripts, closers in reverse order). One live +worker per address (`processLock`: an abstract socket on Linux, a named pipe on Windows, a socket file elsewhere that +fails closed when stale; a start waits for a draining old holder without polling). + +- **Keep-alive** (D9, R3, Q5): the child reports when `host.activity`'s references, jobs and lingering objects and the + app's `busy` counts flip between all zero and not (local calls do not count; the first state with `ready`, which + also carries the load report); the server stops a worker idle for `idleMs` (5 min). +- **Never restarted by itself** (R2): a crash, a `fatal`, a link lost while its child lives (killed after a 5 s grace) + or a failed start leaves the address failed; its holders see `failed { waiting }` and stay on the path (Q1); + `retry(address)` or a rollout reaching workers clears it and starts the held ones (Q2). `stop` suspends the address + while it has holders (R1, Q3). +- **Errors in public and private** (decision "worker errors in public"): `state` and holders get a fixed message + (`worker failed to start`, `worker exited (SIGSEGV)`) or the child's redacted `fatal`; the full error and the stderr + tail go only to `onError` (`worker_failed`). Every app hook of a start (`start`, `principal`, `lock`, the fork) is + inside that boundary; a throwing `authorize` refuses the hold and is reported `authorize_failed`. `worker_exited` is + reported only for an exit nobody asked for. +- **Workers never build** (D11): the server's plugins component builds every plugin; workers evaluate shipped + complete sets (`createPluginRuntime`, no esbuild); `workers.step(name)` ships a rollout's set to every worker (D12: + `{ host, report } | { host, stopped: true } | { host, error }` per worker). ### Channels diff --git a/packages/chord/docs/next/http-workers/book.md b/packages/chord/docs/next/http-workers/book.md index 9b9992337..f340004ef 100644 --- a/packages/chord/docs/next/http-workers/book.md +++ b/packages/chord/docs/next/http-workers/book.md @@ -11,8 +11,7 @@ stats: note: | Written by the redesign session from `slice-10.md`, the decision entries "Slice 10's points", "Origins only from relays", "The HTTP component's security baseline" and their neighbours in `decisions.md`, design §3 and §23, and the - build in `/home/badlogic/pi-7b`. The build exists; its fix round (the security baseline among it) is next. Where the - build and the decisions differ, this book says what is decided. + build after its four fix rounds (security baseline, `fetch` core, error boundaries). colophon: Serving and Spawning · first edition · 2026-10-31 · Chord v2 slice 10 --- @@ -75,13 +74,14 @@ const plugins = createPlugins({ }, onError, }, ctx); -const http = await httpServer(server, { listen: { host: "127.0.0.1", port: 8788 }, authenticate, onError }, ctx); +const http = createHttp(server, { listen: { host: "127.0.0.1", port: 8788 }, onError }, ctx); // settings are arguments const workers = createWorkers(server, Session, { kind: "worker", entry: "./worker.ts", plugins, principal: (a) => ({ kind: "worker", session: a.params.id }), start: (address, ctx) => ({ id: address.params.id, dir: sessionDir(address) }), lock: (address) => `chat:${dataDir}:${address.key}`, authorize: mayReach, onError, }, ctx); await plugins.load(server, { kind: "server", core: [core, { name: "chord.http", facet: http.facet }], modules }, ctx); +// `core` contributes the app's authenticator: self.contribute(Authenticators, authenticate) ``` ```ts @@ -198,27 +198,29 @@ A route that throws is a 500 without its message: error text often holds paths o # Who Is Asking {#who lede="The app's one security function, and everything the component does so the app does not have to."} -The app writes exactly one function about identity: +The app writes one function about identity and contributes it to the registry `Authenticators` (plugins may add +more, a device pairing for instance; the first principal wins): ```ts -async function authenticate(request: AuthRequest, ctx: Context): Promise

    { - const device = verifyCookie(request.cookies.device, secret); - if (device) return { kind: "owner", access: "device", device: device.id }; - const guest = verifyCookie(request.cookies.guest, secret); - if (guest) return { kind: "guest", session: guest.session, grant: guest.id }; +const authenticate: Authenticator = async (request, ctx) => { + const device = readCookie(request, "device", secret); // signature and expiry checked + if (device) return { kind: "owner", access: "device", device }; + const guest = readCookie(request, "guest", secret); + if (guest) return { kind: "guest", grant: guest }; if (request.provenance.kind === "local") return { kind: "owner", access: "local" }; - return undefined; // anonymous -} + return undefined; // anonymous +}; +self.contribute(Authenticators, authenticate); // in the app's core facet ``` It never parses `Host`, never checks `Origin` and never sees a foreign request: the component already did all of it. It may read `request.provenance` (`local`, `remote` with the host, or `relayed` through a verified relay such as -Radius). Cookies are written with the component's helper, which sets `HttpOnly`, `SameSite`, `Secure` on a remote -origin and a signature. +Radius). Cookies are written with `cookie(name, value, { secret, request, maxAge })`, which sets `HttpOnly`, +`SameSite`, `Secure` on a remote origin, and signs the value with its expiry. Every response, the component's own 404s and 403s included, carries the same headers: no caching, no MIME sniffing, no referrer, no framing (`X-Frame-Options: DENY` and `frame-ancestors 'none'`, except for framers the app allowed), and a -Content Security Policy with a per-server script nonce, so an injected ` + From 7ef8680abd32a4804f080a7228f47373b426cc79 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Sat, 31 Oct 2026 22:25:03 +0100 Subject: [PATCH 227/271] fix: sync esbuild platform package dev flags in lockfile esbuild is a non-dev package through chord's optional peer dependency, so its optional @esbuild/* platform packages are non-dev as well. npm install regenerates this; the committed lockfile had stale dev flags. --- package-lock.json | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/package-lock.json b/package-lock.json index 02e687f30..f02d0d520 100644 --- a/package-lock.json +++ b/package-lock.json @@ -854,7 +854,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -871,7 +870,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -888,7 +886,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -905,7 +902,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -922,7 +918,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -939,7 +934,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -956,7 +950,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -973,7 +966,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -990,7 +982,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1007,7 +998,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1024,7 +1014,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1041,7 +1030,6 @@ "cpu": [ "loong64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1058,7 +1046,6 @@ "cpu": [ "mips64el" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1075,7 +1062,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1092,7 +1078,6 @@ "cpu": [ "riscv64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1109,7 +1094,6 @@ "cpu": [ "s390x" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1126,7 +1110,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1143,7 +1126,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1160,7 +1142,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1177,7 +1158,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1194,7 +1174,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1211,7 +1190,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1228,7 +1206,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1245,7 +1222,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1262,7 +1238,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1279,7 +1254,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ From fab2c5353ea4adcbb0d35701897892941f185235 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 21:39:27 +0000 Subject: [PATCH 228/271] feat(schema): add pi-schema, JSON Schema builders and validation without eval @earendil-works/pi-schema compiles JSON Schema (drafts 2020-12 and 7) once into plain closures: no eval or new Function, so it runs under a strict CSP. It passes 2,222 of the 2,223 core tests of the official JSON-Schema-Test-Suite, checks Chord-style argument schemas within 1.2x of TypeBox's JIT, and keeps 3,200 compiled schemas in 7.9 MB (TypeBox: 54.7 MB). - compile(schema, options) returns a Validator: check() is the boolean check (a type guard for builder schemas), errors() reports each failure with a JSON Pointer path, missing and unknown fields at the field. - Builders (str, int, obj, opt, union, ...) return plain JSON Schema typed with the values it accepts (Static). - Formats are an optional entry point (./formats), adapted from TypeBox (MIT). - Tests run the vendored suite (drafts 2020-12 and 7, with optional and format tests); npm run bench compares conformance, speed, setup cost, memory and bundle size with TypeBox, @cfworker/json-schema and omptype. --- package-lock.json | 36 + package.json | 4 +- packages/schema/CHANGELOG.md | 9 + packages/schema/LICENSES/typebox-LICENSE | 23 + packages/schema/README.md | 178 ++ packages/schema/package.json | 56 + packages/schema/src/annotations.ts | 208 ++ packages/schema/src/builders.ts | 187 ++ packages/schema/src/checks.ts | 446 +++++ packages/schema/src/compiler.ts | 618 ++++++ packages/schema/src/explain.ts | 327 ++++ packages/schema/src/formats/index.ts | 46 + packages/schema/src/formats/typebox/date.ts | 47 + .../schema/src/formats/typebox/date_time.ts | 40 + .../schema/src/formats/typebox/duration.ts | 38 + packages/schema/src/formats/typebox/email.ts | 38 + .../schema/src/formats/typebox/hostname.ts | 39 + .../schema/src/formats/typebox/idn_email.ts | 38 + .../src/formats/typebox/idn_hostname.ts | 40 + .../src/formats/typebox/idna/format/bidi.ts | 127 ++ .../src/formats/typebox/idna/format/puny.ts | 167 ++ .../src/formats/typebox/idna/hostname.ts | 42 + .../src/formats/typebox/idna/idn-hostname.ts | 72 + .../schema/src/formats/typebox/idna/index.ts | 30 + .../src/formats/typebox/idna/label/ascii.ts | 40 + .../src/formats/typebox/idna/label/puny.ts | 53 + .../src/formats/typebox/idna/label/unicode.ts | 122 ++ .../formats/typebox/idna/pattern/pattern.ts | 116 ++ packages/schema/src/formats/typebox/ipv4.ts | 37 + packages/schema/src/formats/typebox/ipv6.ts | 38 + packages/schema/src/formats/typebox/iri.ts | 63 + .../src/formats/typebox/iri_reference.ts | 42 + .../src/formats/typebox/json_pointer.ts | 38 + packages/schema/src/formats/typebox/regex.ts | 40 + .../formats/typebox/relative_json_pointer.ts | 38 + packages/schema/src/formats/typebox/time.ts | 56 + packages/schema/src/formats/typebox/uri.ts | 38 + .../src/formats/typebox/uri_reference.ts | 38 + .../src/formats/typebox/uri_template.ts | 40 + packages/schema/src/formats/typebox/uuid.ts | 38 + packages/schema/src/index.ts | 31 + packages/schema/src/resolver.ts | 152 ++ packages/schema/src/shared.ts | 50 + packages/schema/src/types.ts | 36 + packages/schema/src/validator.ts | 58 + packages/schema/test/bench/all.ts | 36 + packages/schema/test/bench/candidates.ts | 101 + packages/schema/test/bench/cases.ts | 261 +++ packages/schema/test/bench/conformance.ts | 54 + packages/schema/test/bench/corpus.ts | 122 ++ packages/schema/test/bench/memory.ts | 73 + packages/schema/test/bench/setup.ts | 51 + packages/schema/test/bench/size.ts | 43 + packages/schema/test/bench/speed.ts | 62 + packages/schema/test/builders.test.ts | 104 + packages/schema/test/conformance.test.ts | 51 + .../fixtures/json-schema-test-suite/LICENSE | 19 + .../fixtures/json-schema-test-suite/README.md | 9 + .../remotes/baseUriChange/folderInteger.json | 3 + .../baseUriChangeFolder/folderInteger.json | 3 + .../folderInteger.json | 3 + .../draft2019-09/dependentRequired.json | 7 + .../draft2019-09/ignore-prefixItems.json | 7 + .../baseUriChange/folderInteger.json | 4 + .../baseUriChangeFolder/folderInteger.json | 4 + .../folderInteger.json | 4 + .../draft2020-12/detached-dynamicref.json | 13 + .../remotes/draft2020-12/detached-ref.json | 13 + .../draft2020-12/different-id-ref-string.json | 5 + .../draft2020-12/extendible-dynamic-ref.json | 21 + .../draft2020-12/format-assertion-false.json | 13 + .../draft2020-12/format-assertion-true.json | 13 + .../remotes/draft2020-12/integer.json | 4 + .../locationIndependentIdentifier.json | 12 + .../metaschema-no-validation.json | 13 + .../metaschema-optional-vocabulary.json | 14 + .../remotes/draft2020-12/name-defs.json | 16 + .../nested-absolute-ref-to-string.json | 9 + .../draft2020-12/nested/foo-ref-string.json | 7 + .../remotes/draft2020-12/nested/string.json | 4 + .../remotes/draft2020-12/prefixItems.json | 7 + .../remotes/draft2020-12/ref-and-defs.json | 12 + .../remotes/draft2020-12/subSchemas.json | 11 + .../remotes/draft2020-12/tree.json | 17 + .../remotes/draft2020-12/urn-ref-string.json | 5 + .../remotes/draft7/detached-ref.json | 13 + .../draft7/ignore-dependentRequired.json | 7 + .../draft7/locationIndependentIdentifier.json | 11 + .../remotes/draft7/name.json | 15 + .../remotes/draft7/ref-and-definitions.json | 11 + .../remotes/draft7/subSchemas.json | 10 + .../remotes/integer.json | 3 + .../remotes/nested/foo-ref-string.json | 6 + .../remotes/nested/string.json | 3 + .../draft2020-12/additionalProperties.json | 219 +++ .../tests/draft2020-12/allOf.json | 312 +++ .../tests/draft2020-12/anchor.json | 120 ++ .../tests/draft2020-12/anyOf.json | 203 ++ .../tests/draft2020-12/boolean_schema.json | 104 + .../tests/draft2020-12/const.json | 431 +++++ .../tests/draft2020-12/contains.json | 176 ++ .../tests/draft2020-12/content.json | 131 ++ .../tests/draft2020-12/default.json | 82 + .../tests/draft2020-12/defs.json | 21 + .../tests/draft2020-12/dependentRequired.json | 152 ++ .../tests/draft2020-12/dependentSchemas.json | 171 ++ .../tests/draft2020-12/dynamicRef.json | 874 +++++++++ .../tests/draft2020-12/enum.json | 419 ++++ .../tests/draft2020-12/exclusiveMaximum.json | 31 + .../tests/draft2020-12/exclusiveMinimum.json | 31 + .../tests/draft2020-12/format.json | 838 ++++++++ .../tests/draft2020-12/if-then-else.json | 308 +++ .../draft2020-12/infinite-loop-detection.json | 37 + .../tests/draft2020-12/items.json | 304 +++ .../tests/draft2020-12/maxContains.json | 123 ++ .../tests/draft2020-12/maxItems.json | 50 + .../tests/draft2020-12/maxLength.json | 55 + .../tests/draft2020-12/maxProperties.json | 94 + .../tests/draft2020-12/maximum.json | 60 + .../tests/draft2020-12/minContains.json | 224 +++ .../tests/draft2020-12/minItems.json | 50 + .../tests/draft2020-12/minLength.json | 55 + .../tests/draft2020-12/minProperties.json | 70 + .../tests/draft2020-12/minimum.json | 75 + .../tests/draft2020-12/multipleOf.json | 102 + .../tests/draft2020-12/not.json | 301 +++ .../tests/draft2020-12/oneOf.json | 293 +++ .../tests/draft2020-12/optional/anchor.json | 60 + .../tests/draft2020-12/optional/bignum.json | 110 ++ .../draft2020-12/optional/cross-draft.json | 18 + .../optional/dependencies-compatibility.json | 282 +++ .../draft2020-12/optional/dynamicRef.json | 56 + .../optional/ecmascript-regex.json | 582 ++++++ .../draft2020-12/optional/float-overflow.json | 17 + .../optional/format-assertion.json | 44 + .../optional/format/date-time.json | 250 +++ .../draft2020-12/optional/format/date.json | 423 +++++ .../optional/format/duration.json | 268 +++ .../optional/format/ecmascript-regex.json | 116 ++ .../draft2020-12/optional/format/email.json | 368 ++++ .../optional/format/hostname.json | 386 ++++ .../optional/format/idn-email.json | 140 ++ .../optional/format/idn-hostname.json | 588 ++++++ .../draft2020-12/optional/format/ipv4.json | 217 +++ .../draft2020-12/optional/format/ipv6.json | 266 +++ .../optional/format/iri-reference.json | 116 ++ .../draft2020-12/optional/format/iri.json | 141 ++ .../optional/format/json-pointer.json | 211 +++ .../draft2020-12/optional/format/regex.json | 103 + .../format/relative-json-pointer.json | 141 ++ .../draft2020-12/optional/format/time.json | 303 +++ .../draft2020-12/optional/format/unknown.json | 46 + .../optional/format/uri-reference.json | 182 ++ .../optional/format/uri-template.json | 251 +++ .../draft2020-12/optional/format/uri.json | 253 +++ .../draft2020-12/optional/format/uuid.json | 156 ++ .../tests/draft2020-12/optional/id.json | 53 + .../draft2020-12/optional/no-schema.json | 26 + .../draft2020-12/optional/non-bmp-regex.json | 86 + .../optional/refOfUnknownKeyword.json | 112 ++ .../draft2020-12/optional/unknownKeyword.json | 57 + .../tests/draft2020-12/pattern.json | 90 + .../tests/draft2020-12/patternProperties.json | 211 +++ .../tests/draft2020-12/prefixItems.json | 104 + .../tests/draft2020-12/properties.json | 242 +++ .../tests/draft2020-12/propertyNames.json | 178 ++ .../tests/draft2020-12/ref.json | 1085 +++++++++++ .../tests/draft2020-12/refRemote.json | 342 ++++ .../tests/draft2020-12/required.json | 169 ++ .../tests/draft2020-12/type.json | 501 +++++ .../tests/draft2020-12/unevaluatedItems.json | 854 +++++++++ .../draft2020-12/unevaluatedProperties.json | 1681 +++++++++++++++++ .../tests/draft2020-12/uniqueItems.json | 399 ++++ .../tests/draft2020-12/vocabulary.json | 57 + .../tests/draft7/additionalItems.json | 190 ++ .../tests/draft7/additionalProperties.json | 147 ++ .../tests/draft7/allOf.json | 294 +++ .../tests/draft7/anyOf.json | 189 ++ .../tests/draft7/boolean_schema.json | 104 + .../tests/draft7/const.json | 384 ++++ .../tests/draft7/contains.json | 165 ++ .../tests/draft7/default.json | 79 + .../tests/draft7/definitions.json | 26 + .../tests/draft7/dependencies.json | 286 +++ .../tests/draft7/enum.json | 341 ++++ .../tests/draft7/exclusiveMaximum.json | 30 + .../tests/draft7/exclusiveMinimum.json | 30 + .../tests/draft7/format.json | 614 ++++++ .../tests/draft7/if-then-else.json | 296 +++ .../tests/draft7/infinite-loop-detection.json | 36 + .../tests/draft7/items.json | 282 +++ .../tests/draft7/maxItems.json | 44 + .../tests/draft7/maxLength.json | 49 + .../tests/draft7/maxProperties.json | 70 + .../tests/draft7/maximum.json | 54 + .../tests/draft7/minItems.json | 44 + .../tests/draft7/minLength.json | 49 + .../tests/draft7/minProperties.json | 64 + .../tests/draft7/minimum.json | 69 + .../tests/draft7/multipleOf.json | 87 + .../tests/draft7/not.json | 259 +++ .../tests/draft7/oneOf.json | 274 +++ .../tests/draft7/optional/bignum.json | 93 + .../tests/draft7/optional/content.json | 77 + .../tests/draft7/optional/cross-draft.json | 25 + .../draft7/optional/ecmascript-regex.json | 552 ++++++ .../tests/draft7/optional/float-overflow.json | 13 + .../draft7/optional/format/date-time.json | 247 +++ .../tests/draft7/optional/format/date.json | 420 ++++ .../optional/format/ecmascript-regex.json | 98 + .../tests/draft7/optional/format/email.json | 365 ++++ .../draft7/optional/format/hostname.json | 380 ++++ .../draft7/optional/format/idn-email.json | 137 ++ .../draft7/optional/format/idn-hostname.json | 582 ++++++ .../tests/draft7/optional/format/ipv4.json | 214 +++ .../tests/draft7/optional/format/ipv6.json | 263 +++ .../draft7/optional/format/iri-reference.json | 113 ++ .../tests/draft7/optional/format/iri.json | 138 ++ .../draft7/optional/format/json-pointer.json | 208 ++ .../tests/draft7/optional/format/regex.json | 100 + .../format/relative-json-pointer.json | 138 ++ .../tests/draft7/optional/format/time.json | 300 +++ .../tests/draft7/optional/format/unknown.json | 43 + .../draft7/optional/format/uri-reference.json | 179 ++ .../draft7/optional/format/uri-template.json | 248 +++ .../tests/draft7/optional/format/uri.json | 250 +++ .../tests/draft7/optional/id.json | 114 ++ .../tests/draft7/optional/non-bmp-regex.json | 82 + .../tests/draft7/optional/unknownKeyword.json | 56 + .../tests/draft7/pattern.json | 59 + .../tests/draft7/patternProperties.json | 171 ++ .../tests/draft7/properties.json | 236 +++ .../tests/draft7/propertyNames.json | 164 ++ .../tests/draft7/ref.json | 1043 ++++++++++ .../tests/draft7/refRemote.json | 257 +++ .../tests/draft7/required.json | 161 ++ .../tests/draft7/type.json | 474 +++++ .../tests/draft7/uniqueItems.json | 389 ++++ .../test/fixtures/meta-schemas/README.md | 10 + .../meta-schemas/draft-07-schema.json | 172 ++ .../draft2020-12-meta-applicator.json | 45 + .../draft2020-12-meta-content.json | 14 + .../meta-schemas/draft2020-12-meta-core.json | 48 + .../draft2020-12-meta-format-annotation.json | 11 + .../draft2020-12-meta-meta-data.json | 34 + .../draft2020-12-meta-unevaluated.json | 12 + .../draft2020-12-meta-validation.json | 95 + .../meta-schemas/draft2020-12-schema.json | 58 + packages/schema/test/shared.test.ts | 41 + packages/schema/test/suite.ts | 77 + packages/schema/test/validator.test.ts | 190 ++ packages/schema/tsconfig.build.json | 9 + packages/schema/vitest.config.ts | 11 + scripts/check-entry-graphs.mjs | 5 + tsconfig.json | 2 + 255 files changed, 37963 insertions(+), 2 deletions(-) create mode 100644 packages/schema/CHANGELOG.md create mode 100644 packages/schema/LICENSES/typebox-LICENSE create mode 100644 packages/schema/README.md create mode 100644 packages/schema/package.json create mode 100644 packages/schema/src/annotations.ts create mode 100644 packages/schema/src/builders.ts create mode 100644 packages/schema/src/checks.ts create mode 100644 packages/schema/src/compiler.ts create mode 100644 packages/schema/src/explain.ts create mode 100644 packages/schema/src/formats/index.ts create mode 100644 packages/schema/src/formats/typebox/date.ts create mode 100644 packages/schema/src/formats/typebox/date_time.ts create mode 100644 packages/schema/src/formats/typebox/duration.ts create mode 100644 packages/schema/src/formats/typebox/email.ts create mode 100644 packages/schema/src/formats/typebox/hostname.ts create mode 100644 packages/schema/src/formats/typebox/idn_email.ts create mode 100644 packages/schema/src/formats/typebox/idn_hostname.ts create mode 100644 packages/schema/src/formats/typebox/idna/format/bidi.ts create mode 100644 packages/schema/src/formats/typebox/idna/format/puny.ts create mode 100644 packages/schema/src/formats/typebox/idna/hostname.ts create mode 100644 packages/schema/src/formats/typebox/idna/idn-hostname.ts create mode 100644 packages/schema/src/formats/typebox/idna/index.ts create mode 100644 packages/schema/src/formats/typebox/idna/label/ascii.ts create mode 100644 packages/schema/src/formats/typebox/idna/label/puny.ts create mode 100644 packages/schema/src/formats/typebox/idna/label/unicode.ts create mode 100644 packages/schema/src/formats/typebox/idna/pattern/pattern.ts create mode 100644 packages/schema/src/formats/typebox/ipv4.ts create mode 100644 packages/schema/src/formats/typebox/ipv6.ts create mode 100644 packages/schema/src/formats/typebox/iri.ts create mode 100644 packages/schema/src/formats/typebox/iri_reference.ts create mode 100644 packages/schema/src/formats/typebox/json_pointer.ts create mode 100644 packages/schema/src/formats/typebox/regex.ts create mode 100644 packages/schema/src/formats/typebox/relative_json_pointer.ts create mode 100644 packages/schema/src/formats/typebox/time.ts create mode 100644 packages/schema/src/formats/typebox/uri.ts create mode 100644 packages/schema/src/formats/typebox/uri_reference.ts create mode 100644 packages/schema/src/formats/typebox/uri_template.ts create mode 100644 packages/schema/src/formats/typebox/uuid.ts create mode 100644 packages/schema/src/index.ts create mode 100644 packages/schema/src/resolver.ts create mode 100644 packages/schema/src/shared.ts create mode 100644 packages/schema/src/types.ts create mode 100644 packages/schema/src/validator.ts create mode 100644 packages/schema/test/bench/all.ts create mode 100644 packages/schema/test/bench/candidates.ts create mode 100644 packages/schema/test/bench/cases.ts create mode 100644 packages/schema/test/bench/conformance.ts create mode 100644 packages/schema/test/bench/corpus.ts create mode 100644 packages/schema/test/bench/memory.ts create mode 100644 packages/schema/test/bench/setup.ts create mode 100644 packages/schema/test/bench/size.ts create mode 100644 packages/schema/test/bench/speed.ts create mode 100644 packages/schema/test/builders.test.ts create mode 100644 packages/schema/test/conformance.test.ts create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/LICENSE create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/README.md create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChange/folderInteger.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolder/folderInteger.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolderInSubschema/folderInteger.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/dependentRequired.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/ignore-prefixItems.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChange/folderInteger.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolder/folderInteger.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolderInSubschema/folderInteger.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-dynamicref.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-ref.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/different-id-ref-string.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/extendible-dynamic-ref.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-false.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-true.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/integer.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/locationIndependentIdentifier.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-no-validation.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-optional-vocabulary.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/name-defs.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested-absolute-ref-to-string.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/foo-ref-string.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/string.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/prefixItems.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/ref-and-defs.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/subSchemas.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/tree.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/urn-ref-string.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/detached-ref.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ignore-dependentRequired.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/locationIndependentIdentifier.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/name.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ref-and-definitions.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/subSchemas.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/integer.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/foo-ref-string.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/string.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/additionalProperties.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/allOf.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anchor.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anyOf.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/boolean_schema.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/const.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/contains.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/content.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/default.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/defs.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentRequired.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentSchemas.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dynamicRef.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/enum.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMaximum.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMinimum.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/format.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/if-then-else.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/infinite-loop-detection.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/items.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxContains.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxItems.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxLength.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxProperties.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maximum.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minContains.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minItems.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minLength.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minProperties.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minimum.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/multipleOf.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/not.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/oneOf.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/anchor.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/bignum.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/cross-draft.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dependencies-compatibility.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dynamicRef.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/ecmascript-regex.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/float-overflow.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format-assertion.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date-time.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/duration.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ecmascript-regex.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/email.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/hostname.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-email.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-hostname.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv4.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv6.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri-reference.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/json-pointer.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/regex.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/relative-json-pointer.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/time.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/unknown.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-reference.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-template.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uuid.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/id.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/no-schema.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/non-bmp-regex.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/refOfUnknownKeyword.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/unknownKeyword.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/pattern.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/patternProperties.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/prefixItems.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/properties.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/propertyNames.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/ref.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/refRemote.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/required.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/type.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedItems.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedProperties.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/uniqueItems.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/vocabulary.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalItems.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalProperties.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/allOf.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/anyOf.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/boolean_schema.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/const.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/contains.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/default.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/definitions.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/dependencies.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/enum.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMaximum.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMinimum.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/format.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/if-then-else.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/infinite-loop-detection.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/items.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxItems.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxLength.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxProperties.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maximum.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minItems.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minLength.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minProperties.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minimum.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/multipleOf.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/not.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/oneOf.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/bignum.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/content.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/cross-draft.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/ecmascript-regex.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/float-overflow.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date-time.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ecmascript-regex.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/email.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/hostname.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-email.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-hostname.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv4.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv6.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri-reference.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/json-pointer.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/regex.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/relative-json-pointer.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/time.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/unknown.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-reference.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-template.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/id.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/non-bmp-regex.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/unknownKeyword.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/pattern.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/patternProperties.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/properties.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/propertyNames.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/ref.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/refRemote.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/required.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/type.json create mode 100644 packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/uniqueItems.json create mode 100644 packages/schema/test/fixtures/meta-schemas/README.md create mode 100644 packages/schema/test/fixtures/meta-schemas/draft-07-schema.json create mode 100644 packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-applicator.json create mode 100644 packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-content.json create mode 100644 packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-core.json create mode 100644 packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-format-annotation.json create mode 100644 packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-meta-data.json create mode 100644 packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-unevaluated.json create mode 100644 packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-validation.json create mode 100644 packages/schema/test/fixtures/meta-schemas/draft2020-12-schema.json create mode 100644 packages/schema/test/shared.test.ts create mode 100644 packages/schema/test/suite.ts create mode 100644 packages/schema/test/validator.test.ts create mode 100644 packages/schema/tsconfig.build.json create mode 100644 packages/schema/vitest.config.ts diff --git a/package-lock.json b/package-lock.json index f02d0d520..d26f79648 100644 --- a/package-lock.json +++ b/package-lock.json @@ -570,6 +570,13 @@ "dev": true, "license": "MIT OR Apache-2.0" }, + "node_modules/@cfworker/json-schema": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", + "integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==", + "dev": true, + "license": "MIT" + }, "node_modules/@cloudflare/kv-asset-handler": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.5.0.tgz", @@ -828,6 +835,10 @@ "resolved": "packages/mcp", "link": true }, + "node_modules/@earendil-works/pi-schema": { + "resolved": "packages/schema", + "link": true + }, "node_modules/@earendil-works/pi-telemetry": { "resolved": "packages/telemetry", "link": true @@ -1896,6 +1907,17 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@oh-my-pi/omptype": { + "version": "18.13.3", + "resolved": "https://registry.npmjs.org/@oh-my-pi/omptype/-/omptype-18.13.3.tgz", + "integrity": "sha512-ZLi+kb3ff554n2ndH4iEviVRWHI1t78LZu7qf/NSXkYf4STcl6vEnJ5A2FBhfUaTLVk/HRs09LzyFnySOkwpvQ==", + "dev": true, + "license": "MIT", + "engines": { + "bun": ">=1.4.4", + "node": ">=20" + } + }, "node_modules/@oxc-project/types": { "version": "0.154.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.154.0.tgz", @@ -5748,6 +5770,20 @@ "node": ">=22.19.0" } }, + "packages/schema": { + "name": "@earendil-works/pi-schema", + "version": "1.4.0", + "license": "MIT", + "devDependencies": { + "@cfworker/json-schema": "4.1.1", + "@oh-my-pi/omptype": "18.13.3", + "typebox": "1.3.27", + "vitest": "4.1.12" + }, + "engines": { + "node": ">=22.19.0" + } + }, "packages/telemetry": { "name": "@earendil-works/pi-telemetry", "version": "1.4.0", diff --git a/package.json b/package.json index b158f2f00..352767d97 100644 --- a/package.json +++ b/package.json @@ -12,8 +12,8 @@ ], "scripts": { "clean": "npm run clean --workspaces", - "build": "cd packages/telemetry && npm run build && cd ../chord && npm run build && cd ../tui && npm run build && cd ../codemode && npm run build && cd ../mcp && npm run build && cd ../ai && npm run build && cd ../durable && npm run build && cd ../env && npm run build && cd ../agent && npm run build && cd ../coding-agent && npm run build", - "build:offline": "cd packages/telemetry && npm run build && cd ../chord && npm run build && cd ../tui && npm run build && cd ../codemode && npm run build && cd ../mcp && npm run build && cd ../ai && npm run build:offline && cd ../durable && npm run build && cd ../env && npm run build && cd ../agent && npm run build && cd ../coding-agent && npm run build", + "build": "cd packages/telemetry && npm run build && cd ../schema && npm run build && cd ../chord && npm run build && cd ../tui && npm run build && cd ../codemode && npm run build && cd ../mcp && npm run build && cd ../ai && npm run build && cd ../durable && npm run build && cd ../env && npm run build && cd ../agent && npm run build && cd ../coding-agent && npm run build", + "build:offline": "cd packages/telemetry && npm run build && cd ../schema && npm run build && cd ../chord && npm run build && cd ../tui && npm run build && cd ../codemode && npm run build && cd ../mcp && npm run build && cd ../ai && npm run build:offline && cd ../durable && npm run build && cd ../env && npm run build && cd ../agent && npm run build && cd ../coding-agent && npm run build", "build:native:darwin": "npm --prefix packages/tui run build:native:darwin", "build:native:linux": "npm --prefix packages/tui run build:native:linux", "build:native:win32": "npm --prefix packages/tui run build:native:win32", diff --git a/packages/schema/CHANGELOG.md b/packages/schema/CHANGELOG.md new file mode 100644 index 000000000..d23ca137c --- /dev/null +++ b/packages/schema/CHANGELOG.md @@ -0,0 +1,9 @@ +# Changelog + +## [Unreleased] + +### Added + +- Initial release: `compile(schema, options)` validates JSON Schema drafts 2020-12 and 7 without `eval` or `new Function`, compiling a schema once into plain closures. `Validator.check` is the boolean check (a type guard for schemas built with the builders), `Validator.errors` reports each failure with a JSON Pointer path. Format checks are in the optional `@earendil-works/pi-schema/formats` entry point. +- Schema builders (`str`, `num`, `int`, `bool`, `nul`, `unknown`, `lit`, `oneOf`, `obj`, `opt`, `arr`, `tuple`, `map`, `union`, `nullable`, `bytes`) that return plain JSON Schema typed with the values it accepts (`Static`). +- `npm run bench` compares conformance, speed, setup cost, memory and bundle size with TypeBox, @cfworker/json-schema and omptype. diff --git a/packages/schema/LICENSES/typebox-LICENSE b/packages/schema/LICENSES/typebox-LICENSE new file mode 100644 index 000000000..0330da432 --- /dev/null +++ b/packages/schema/LICENSES/typebox-LICENSE @@ -0,0 +1,23 @@ +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/packages/schema/README.md b/packages/schema/README.md new file mode 100644 index 000000000..d31caa521 --- /dev/null +++ b/packages/schema/README.md @@ -0,0 +1,178 @@ +# @earendil-works/pi-schema + +JSON Schema builders and validation without `eval`. A schema compiles once into plain closures, so it checks values +about as fast as validators that generate code, also under a strict Content Security Policy, and needs no +dependencies. + +- Drafts 2020-12 and 7. Passes 2,222 of the 2,223 core tests of the official + [JSON-Schema-Test-Suite](https://github.com/json-schema-org/JSON-Schema-Test-Suite). +- Builders return plain JSON Schema typed with the values it accepts; a compiled check narrows to that type. +- Errors as JSON Pointer paths. A missing or unknown field is reported at the field. +- Never modifies a schema; frozen schemas work. +- 8.5 KB minified and gzipped. Format checks are an optional import. + +## Installation + +```bash +npm install @earendil-works/pi-schema +``` + +## Usage + +```typescript +import { compile, int, obj, oneOf, opt, type Static, str } from "@earendil-works/pi-schema"; + +const Read = obj({ + path: str({ description: "Path to the file", minLength: 1 }), + offset: opt(int({ minimum: 1 })), + mode: oneOf(["text", "binary"]), +}); +type Read = Static; // { path: string; mode: "text" | "binary"; offset?: number } + +const read = compile(Read); +if (read.check(value)) value.path; // value: Read + +read.errors({ offset: 0, extra: true }); +// [ +// { path: "/path", keyword: "required", message: "missing" }, +// { path: "/mode", keyword: "required", message: "missing" }, +// { path: "/offset", keyword: "minimum", message: "expected >= 1" }, +// { path: "/extra", keyword: "additionalProperties", message: "unknown field" }, +// ] +``` + +`Read` is plain JSON Schema (`{ type: "object", properties: {...}, required: ["path", "mode"], additionalProperties: +false }`), ready for a tool definition or `JSON.stringify`. `compile` takes any JSON Schema, built with the builders +or not. + +`check(value)` is the fast path: a boolean, no allocations for most schemas, and a plain function you can pass +around. `errors(value)` explains a failure and returns `[]` for a valid value. It is built on the first failure and +kept, so a schema whose values always pass never pays for it. + +### Builders + +| builder | schema | type | +|---|---|---| +| `str({ minLength, maxLength, pattern, format })` | `{ type: "string", ... }` | `string` | +| `num({ minimum, maximum, exclusiveMinimum, exclusiveMaximum, multipleOf })`, `int(...)` | `{ type: "number" }`, `{ type: "integer" }` | `number` | +| `bool()`, `nul()`, `unknown()` | `{ type: "boolean" }`, `{ type: "null" }`, `{}` | `boolean`, `null`, `unknown` | +| `lit(value)` | `{ const: value }` | the literal | +| `oneOf(["a", "b"])` | `{ enum: [...] }` | `"a" \| "b"` | +| `obj({ a: str(), b: opt(num()) }, { open })` | closed object; `open: true` allows other properties | `{ a: string; b?: number }` | +| `arr(items, { minItems, maxItems, uniqueItems })` | `{ type: "array", items }` | `T[]` | +| `tuple([str(), num()])` | `prefixItems`, no more items | `[string, number]` | +| `map(value)` | `{ type: "object", additionalProperties: value }` | `Record` | +| `union([...])` | `{ anyOf: [...] }` | the union | +| `nullable(schema)` | `{ anyOf: [schema, { type: "null" }] }` | `T \| null` | +| `bytes({ minLength, maxLength })` | `{ type: "bytes", ... }` | `Uint8Array` | + +Every builder also takes `title`, `description`, `examples`, `deprecated` and `default`. A `default` is an annotation; +validation never inserts it. + +A union of objects tagged by a `lit()` property checks and reports by the tag: only the branch the tag selects runs. + +```typescript +const Block = union([obj({ kind: lit("text"), text: str() }), obj({ kind: lit("image"), width: int({ minimum: 0 }) })]); +compile(Block).errors({ kind: "video" }); +// [{ path: "/kind", keyword: "anyOf", message: 'expected one of ["text","image"]' }] +``` + +### Options + +```typescript +import { formats } from "@earendil-works/pi-schema/formats"; + +const validator = compile(schema, { + draft: "7", // "2020-12" (default) or "7" + formats, // check `format`; without it `format` is an annotation, as 2020-12 specifies + schemas: { "https://example.com/point.json": point }, // documents for $refs to other documents, by URI +}); +``` + +`formats` has every format 2020-12 defines (`date-time`, `email`, `uri`, `uuid`, `ipv6`, ...). Pass your own map to +check fewer formats, or custom ones. A `$ref` that names nothing throws when the schema compiles. + +### TypeBox schemas + +TypeBox 1.x schemas are JSON Schema, so they compile as they are: + +```typescript +import Type from "typebox"; + +const user = compile(Type.Object({ name: Type.String(), tags: Type.Array(Type.String()) })); +``` + +TypeBox's JavaScript-only types (`BigInt`, `Undefined`, `Function`, ...), refinements and codecs are not JSON Schema +and are not supported. + +### Values + +- A property whose value is `undefined` counts as absent, as in TypeBox. +- `{ type: "bytes", minLength, maxLength }` matches a `Uint8Array` of that length, an extension for values that never + pass through JSON. + +## Supported keywords + +- Every validation keyword of drafts 2020-12 and 7, including draft 7's `dependencies` and `items` arrays with + `additionalItems`. +- `$id`, `$anchor`, `$ref` (local, by anchor, relative and remote), `$dynamicRef` and `$dynamicAnchor`. +- `unevaluatedProperties` and `unevaluatedItems`. +- `pattern` and `patternProperties` are ECMAScript regular expressions with the `u` flag. +- Not supported: meta-schemas that switch vocabularies on or off (`$vocabulary`), `contentMediaType` and + `contentEncoding` assertions (optional in the spec), and draft 2019-09's `$recursiveRef`. + +## Benchmarks + +Node 24.4.1 on one core of an i9-13900, against TypeBox 1.3.27, @cfworker/json-schema 4.1.1 and @oh-my-pi/omptype +18.13.3 (`fromJsonSchema`). + +| test suite (tests passed) | pi-schema | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:| +| draft 2020-12 core (1,298) | 1,297 | 1,276 | 1,249 | 863 | +| draft 7 core (925) | 925 | 924 | 918 | 659 | +| draft 2020-12 format (895), with `formats` | 895 | 895 | 740 | 412 | + +| check, valid value | pi-schema | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:| +| nested object with tagged unions | 535 ns | 464 ns | 43 µs | 713 ns | +| union of 5 messages | 96 ns | 94 ns | 10 µs | 94 ns | +| tool schema (read) | 34 ns | 9 ns | 2.0 µs | 8 ns | +| MCP tool schema with `$ref` | 207 ns | 88 ns | 13 µs | 1.7 µs | +| recursive `$ref` schema | 1.1 µs | 1.2 µs | 41 µs | 7.4 µs | + +| same, without eval (strict CSP) | pi-schema | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:| +| nested object with tagged unions | 527 ns | 42 µs | 44 µs | 2.7 µs | +| MCP tool schema with `$ref` | 209 ns | 38 µs | 13 µs | 2.1 µs | +| recursive `$ref` schema | 1.1 µs | 134 µs | 43 µs | 8.9 µs | + +| cost | pi-schema | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:| +| compile a new schema | 43 µs | 210 µs | 23 µs | 19 µs + 114 µs on its first check | +| memory, 3,200 distinct schemas | 7.9 MB | 54.7 MB | 7.6 MB | 20.0 MB | +| bundle, minified + gzip | 8.5 KB | 35.2 KB | 5.8 KB | 38.8 KB | + +Reproduce with `npm run bench` in `packages/schema` (about 5 minutes). It runs every library through its fastest +boolean path and keeps the object each returns. The scripts are in `test/bench/`. + +- **conformance:** every library on the vendored test suite, with the remote documents and meta-schemas the tests + reference. +- **speed:** realistic schemas, in ns per check after warm-up; the median of 15 rounds, with the libraries' + order rotated in each round. +- **speed without eval:** the same, run with `--disallow-code-generation-from-strings`. +- **setup:** compiling 1,000 schemas a library has not seen, and their first checks. +- **memory:** 3,200 distinct tool-like schemas, each library in its own process. +- **size:** esbuild bundles of what a validator needs, minified and gzipped. + +Where code generation is unavailable, TypeBox falls back to its interpreter, and omptype throws `EvalError` once per +schema, then validates through its interpreter. + +Flat objects of simple types check in 15–90 ns, against 8–15 ns for TypeBox. Without code generation, reading a +property whose name comes from a list costs a few nanoseconds. + +## Development + +`npm test` runs the official test suite (drafts 2020-12 and 7, including the optional and format tests, vendored in +`test/fixtures/`) and the package's own tests. + +The format checks are adapted from TypeBox 1.3.27 (MIT); its license is in `LICENSES/`. diff --git a/packages/schema/package.json b/packages/schema/package.json new file mode 100644 index 000000000..f6e94e1d2 --- /dev/null +++ b/packages/schema/package.json @@ -0,0 +1,56 @@ +{ + "name": "@earendil-works/pi-schema", + "version": "1.4.0", + "description": "JSON Schema validation without eval: drafts 2020-12 and 7, compiled into plain closures", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "source": "./src/index.ts", + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + }, + "./formats": { + "source": "./src/formats/index.ts", + "types": "./dist/formats/index.d.ts", + "import": "./dist/formats/index.js" + } + }, + "sideEffects": false, + "files": [ + "dist", + "README.md", + "CHANGELOG.md", + "LICENSES" + ], + "scripts": { + "clean": "node ../../scripts/package-files.mjs clean dist", + "build": "tsc -p tsconfig.build.json", + "test": "vitest --run", + "bench": "node --expose-gc test/bench/all.ts", + "prepublishOnly": "npm run clean && npm run build" + }, + "keywords": [ + "json-schema", + "validation", + "validator", + "csp" + ], + "author": "Earendil Works", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/earendil-works/pi.git", + "directory": "packages/schema" + }, + "engines": { + "node": ">=22.19.0" + }, + "devDependencies": { + "@cfworker/json-schema": "4.1.1", + "@oh-my-pi/omptype": "18.13.3", + "typebox": "1.3.27", + "vitest": "4.1.12" + } +} diff --git a/packages/schema/src/annotations.ts b/packages/schema/src/annotations.ts new file mode 100644 index 000000000..5bc9cc4f5 --- /dev/null +++ b/packages/schema/src/annotations.ts @@ -0,0 +1,208 @@ +// Annotation tracking for `unevaluatedProperties` and `unevaluatedItems`: annotating checks record which +// properties and items a schema evaluated. Only schemas that use those keywords compile into these closures. + +import { isObject } from "./checks.ts"; +import type { Check } from "./types.ts"; + +/** What a schema evaluated: property names (or all of them), the evaluated item prefix, items matched by `contains`. */ +export interface Annotations { + props: Set | null; + allProps: boolean; + items: number; + allItems: boolean; + indexes: Set | null; +} + +/** A check that records what it evaluated into `a` when it passes. */ +export type AnnotatingCheck = (value: unknown, a: Annotations) => boolean; + +/** Empty annotations. */ +export function newAnnotations(): Annotations { + return { props: null, allProps: false, items: 0, allItems: false, indexes: null }; +} + +function merge(a: Annotations, b: Annotations): void { + if (b.props !== null) for (const k of b.props) addProp(a, k); + a.allProps ||= b.allProps; + if (b.items > a.items) a.items = b.items; + a.allItems ||= b.allItems; + if (b.indexes !== null) for (const i of b.indexes) addIndex(a, i); +} + +function addProp(a: Annotations, key: string): void { + if (a.props === null) a.props = new Set(); + a.props.add(key); +} + +function addIndex(a: Annotations, index: number): void { + if (a.indexes === null) a.indexes = new Set(); + a.indexes.add(index); +} + +export const ANNOTATING_TRUE: AnnotatingCheck = () => true; +export const ANNOTATING_FALSE: AnnotatingCheck = () => false; + +/** The indirection a recursive reference goes through. */ +export function indirectAnnotating(slot: { check: AnnotatingCheck | undefined }): AnnotatingCheck { + return (v, a) => (slot.check as AnnotatingCheck)(v, a); +} + +/** A plain check in annotating position: it evaluates nothing that unevaluated* sees. */ +export function ignoreAnnotations(check: Check): AnnotatingCheck { + return (v) => check(v); +} + +/** An annotating check run as a plain check, with fresh annotations. */ +export function withFreshAnnotations(check: AnnotatingCheck): Check { + return (v) => check(v, newAnnotations()); +} + +/** `check` with `uri` pushed onto the dynamic scope while it runs. */ +export function scopedAnnotating(scope: string[], uri: string, check: AnnotatingCheck): AnnotatingCheck { + return (v, a) => { + scope.push(uri); + const ok = check(v, a); + scope.pop(); + return ok; + }; +} + +/** unevaluated* sees only the annotations of its own schema, never those of its siblings in the parent. */ +export function isolateAnnotations(check: AnnotatingCheck): AnnotatingCheck { + return (v, a) => { + const b = newAnnotations(); + if (!check(v, b)) return false; + merge(a, b); + return true; + }; +} + +/** The non-annotating keywords, then each annotating part in order. */ +export function annotateAll(plain: Check, parts: readonly AnnotatingCheck[]): AnnotatingCheck { + return (v, a) => { + if (!plain(v)) return false; + for (let i = 0; i < parts.length; i++) if (!parts[i](v, a)) return false; + return true; + }; +} + +/** `properties`, `patternProperties` and `additionalProperties`. */ +export function annotateObject( + props: readonly (readonly [string, Check])[], + patterns: readonly (readonly [RegExp, Check])[], + additional: Check | undefined, +): AnnotatingCheck { + const declared = new Set(props.map(([k]) => k)); + return (v, a) => { + if (!isObject(v)) return true; + const seen: string[] = []; + for (const [k, check] of props) { + if (!Object.hasOwn(v, k)) continue; + if (!check(v[k])) return false; + seen.push(k); + } + for (const k of Object.keys(v)) { + let matched = declared.has(k); + for (const [re, check] of patterns) { + if (re.test(k)) { + if (!check(v[k])) return false; + matched = true; + seen.push(k); + } + } + if (!matched && additional !== undefined) { + if (!additional(v[k])) return false; + seen.push(k); + } + } + for (const k of seen) addProp(a, k); + return true; + }; +} + +/** `prefixItems`/`items`/`additionalItems` and `contains` with its bounds. */ +export function annotateArray( + prefix: readonly Check[], + rest: Check | undefined, + contains: Check | undefined, + minContains: number, + maxContains: number, +): AnnotatingCheck { + return (v, a) => { + if (!Array.isArray(v)) return true; + const m = Math.min(v.length, prefix.length); + for (let i = 0; i < m; i++) if (!prefix[i](v[i])) return false; + if (rest !== undefined) { + for (let i = prefix.length; i < v.length; i++) if (!rest(v[i])) return false; + a.allItems = true; + } + if (m > a.items) a.items = m; + if (contains !== undefined) { + let n = 0; + for (let i = 0; i < v.length; i++) { + if (contains(v[i])) { + n++; + addIndex(a, i); + } + } + if (n < minContains || n > maxContains) return false; + } + return true; + }; +} + +/** `anyOf` (or `oneOf` when `exactlyOne`): every passing branch contributes its annotations. */ +export function annotateAny(branches: readonly AnnotatingCheck[], exactlyOne: boolean): AnnotatingCheck { + return (v, a) => { + let n = 0; + for (const branch of branches) { + const b = newAnnotations(); + if (branch(v, b)) { + n++; + merge(a, b); + } + } + return exactlyOne ? n === 1 : n > 0; + }; +} + +/** `if`, `then` and `else`: a passing `if` contributes its annotations even without `then`. */ +export function annotateIf( + cond: AnnotatingCheck, + then: AnnotatingCheck | undefined, + otherwise: AnnotatingCheck | undefined, +): AnnotatingCheck { + return (v, a) => { + const b = newAnnotations(); + if (cond(v, b)) { + merge(a, b); + return then === undefined || then(v, a); + } + return otherwise === undefined || otherwise(v, a); + }; +} + +/** One `dependentSchemas` entry. */ +export function annotateDependent(key: string, check: AnnotatingCheck): AnnotatingCheck { + return (v, a) => !isObject(v) || !Object.hasOwn(v, key) || check(v, a); +} + +/** `unevaluatedProperties`: every property no other keyword evaluated passes `check`. */ +export function unevaluatedProperties(check: Check): AnnotatingCheck { + return (v, a) => { + if (!isObject(v) || a.allProps) return true; + for (const k of Object.keys(v)) if (!(a.props?.has(k) ?? false) && !check(v[k])) return false; + a.allProps = true; + return true; + }; +} + +/** `unevaluatedItems`: every item no other keyword evaluated passes `check`. */ +export function unevaluatedItems(check: Check): AnnotatingCheck { + return (v, a) => { + if (!Array.isArray(v) || a.allItems) return true; + for (let i = a.items; i < v.length; i++) if (!(a.indexes?.has(i) ?? false) && !check(v[i])) return false; + a.allItems = true; + return true; + }; +} diff --git a/packages/schema/src/builders.ts b/packages/schema/src/builders.ts new file mode 100644 index 000000000..1f4e21a13 --- /dev/null +++ b/packages/schema/src/builders.ts @@ -0,0 +1,187 @@ +// Schema builders: short functions that return plain JSON Schema objects and carry the static type of the values +// they accept, so `Static` is the TypeScript type a valid value has. The objects are ordinary data: +// they serialize with JSON.stringify and compile with compile() like any other schema. + +declare const STATIC: unique symbol; +declare const OPTIONAL_TYPE: unique symbol; + +/** A JSON Schema object whose valid values have type `T` (a type-level brand; nothing extra exists at run time). */ +export type Schema = { readonly [keyword: string]: unknown } & { readonly [STATIC]?: T }; + +/** The type of the values `S` accepts; `unknown` for schemas not built with these functions. */ +export type Static = S extends { readonly [STATIC]?: infer T } ? (unknown extends T ? unknown : T) : unknown; + +/** Annotations every builder accepts: they describe the schema and never change what it accepts. */ +export interface Annotations { + readonly title?: string; + readonly description?: string; + readonly examples?: readonly T[]; + readonly deprecated?: boolean; + /** A default value, an annotation for tools and forms; validation never inserts it. */ + readonly default?: T; +} + +/** Length bounds of a string, an array or bytes. */ +export interface LengthBounds { + readonly minLength?: number; + readonly maxLength?: number; +} + +/** A string, with optional length bounds (in code points), pattern and format. */ +export function str( + options: Annotations & LengthBounds & { readonly pattern?: string; readonly format?: string } = {}, +): Schema { + return { type: "string", ...options }; +} + +/** Bounds of a number. */ +export interface NumberBounds { + readonly minimum?: number; + readonly maximum?: number; + readonly exclusiveMinimum?: number; + readonly exclusiveMaximum?: number; + readonly multipleOf?: number; +} + +/** A number within optional bounds. */ +export function num(options: Annotations & NumberBounds = {}): Schema { + return { type: "number", ...options }; +} + +/** An integer within optional bounds. */ +export function int(options: Annotations & NumberBounds = {}): Schema { + return { type: "integer", ...options }; +} + +/** A boolean. */ +export function bool(options: Annotations = {}): Schema { + return { type: "boolean", ...options }; +} + +/** `null`. */ +export function nul(options: Annotations = {}): Schema { + return { type: "null", ...options }; +} + +/** Any value. */ +export function unknown(options: Annotations = {}): Schema { + return { ...options }; +} + +/** Exactly `value` (a string, number, boolean or null). */ +export function lit( + value: V, + options: Annotations = {}, +): Schema { + return { const: value, ...options }; +} + +/** One of the given values. */ +export function oneOf( + values: V, + options: Annotations = {}, +): Schema { + return { enum: values, ...options }; +} + +/** + * Any of the given schemas. A union of objects tagged by a `lit()` property (`{ kind: lit("text"), ... }`) is checked + * by the tag: only the branch it selects runs, and its errors are the ones reported. + */ +export function union( + schemas: S, + options: Annotations> = {}, +): Schema> { + return { anyOf: schemas, ...options }; +} + +/** `schema` or `null`. */ +export function nullable(schema: Schema, options: Annotations = {}): Schema { + return { anyOf: [schema, { type: "null" }], ...options }; +} + +/** An array of `items`, with optional length bounds and unique items. */ +export function arr( + items: Schema, + options: Annotations & { + readonly minItems?: number; + readonly maxItems?: number; + readonly uniqueItems?: boolean; + } = {}, +): Schema { + return { type: "array", items, ...options }; +} + +/** A fixed-length array whose items match `items` in order. */ +export function tuple( + items: S, + options: Annotations<{ -readonly [K in keyof S]: Static }> = {}, +): Schema<{ -readonly [K in keyof S]: Static }> { + return { type: "array", prefixItems: items, items: false, minItems: items.length, ...options }; +} + +/** An object with string keys and values of `value` (a record). */ +export function map(value: Schema, options: Annotations> = {}): Schema> { + return { type: "object", additionalProperties: value, ...options }; +} + +/** A Uint8Array with optional length bounds (an extension of JSON Schema for values that never pass through JSON). */ +export function bytes(options: Annotations & LengthBounds = {}): Schema { + return { type: "bytes", ...options }; +} + +/** An optional property of obj(): `{ name: opt(str()) }` may omit `name` or set it to undefined. */ +export class Optional { + declare readonly [OPTIONAL_TYPE]?: S; + readonly schema: S; + + constructor(schema: S) { + this.schema = schema; + } +} + +/** Marks an obj() property as optional. */ +export function opt(schema: S): Optional { + return new Optional(schema); +} + +/** The properties of obj(): a schema or an opt() schema per name. */ +export type Properties = { readonly [name: string]: Schema | Optional }; + +/** The value type of an obj() schema. */ +export type StaticObject

    = Simplify< + { -readonly [K in keyof P as P[K] extends Optional ? never : K]: Static } & { + -readonly [K in keyof P as P[K] extends Optional ? K : never]?: P[K] extends Optional + ? Static + : never; + } +>; + +type Simplify = { [K in keyof T]: T[K] } & {}; + +/** + * A closed object: the given properties (required unless opt()) and no others. `open: true` allows other + * properties, as tool schemas from other systems often do. + */ +export function obj( + properties: P, + options: Annotations> & { readonly open?: boolean } = {}, +): Schema> { + const { open, ...annotations } = options; + const props: Record = {}; + const required: string[] = []; + for (const [name, property] of Object.entries(properties)) { + if (property instanceof Optional) props[name] = property.schema; + else { + props[name] = property; + required.push(name); + } + } + return { + type: "object", + properties: props, + ...(required.length > 0 ? { required } : {}), + ...(open === true ? {} : { additionalProperties: false }), + ...annotations, + }; +} diff --git a/packages/schema/src/checks.ts b/packages/schema/src/checks.ts new file mode 100644 index 000000000..b686c4c27 --- /dev/null +++ b/packages/schema/src/checks.ts @@ -0,0 +1,446 @@ +// Check factories: each returns a closure that captures only its arguments, never the compiler, so a compiled +// schema keeps no compile-time state alive. Leaf checks are module constants that object and array loops inline. + +import type { Check } from "./types.ts"; + +export const ALWAYS: Check = () => true; +export const NEVER: Check = () => false; +const IS_STRING: Check = (v) => typeof v === "string"; +const IS_NUMBER: Check = (v) => typeof v === "number"; +const IS_INTEGER: Check = (v) => Number.isInteger(v); +const IS_BOOLEAN: Check = (v) => typeof v === "boolean"; +const IS_NULL: Check = (v) => v === null; +export const IS_ARRAY: Check = (v) => Array.isArray(v); +const IS_BYTES: Check = (v) => v instanceof Uint8Array; +export const IS_OBJECT: Check = (v) => isObject(v); + +/** Whether `v` is a JSON object: not null, an array or bytes. */ +export function isObject(v: unknown): v is Record { + return typeof v === "object" && v !== null && !Array.isArray(v) && !(v instanceof Uint8Array); +} + +/** The check of a `type` name; `bytes` (a Uint8Array) extends JSON Schema for values that never pass through JSON. */ +export const TYPE_CHECKS: ReadonlyMap = new Map([ + ["string", IS_STRING], + ["number", IS_NUMBER], + ["integer", IS_INTEGER], + ["boolean", IS_BOOLEAN], + ["null", IS_NULL], + ["array", IS_ARRAY], + ["object", IS_OBJECT], + ["bytes", IS_BYTES], +]); + +/** Leaf checks that object and array loops run inline by code instead of calling them. */ +export const LEAF_CODES: ReadonlyMap = new Map([ + [IS_STRING, 1], + [IS_NUMBER, 2], + [IS_INTEGER, 3], + [IS_BOOLEAN, 4], + [IS_NULL, 5], + [ALWAYS, 6], +]); + +function leafOk(code: number, x: unknown): boolean { + switch (code) { + case 1: + return typeof x === "string"; + case 2: + return typeof x === "number"; + case 3: + return Number.isInteger(x); + case 4: + return typeof x === "boolean"; + case 5: + return x === null; + default: + return true; + } +} + +/** Whether a combined object or array check also checks the type: not at all, strictly, or passing other types. */ +export const GUARD_NONE = 0; +export const GUARD_STRICT = 1; +export const GUARD_LOOSE = 2; +export type Guard = typeof GUARD_NONE | typeof GUARD_STRICT | typeof GUARD_LOOSE; + +/** Flags of a declared property in an object check's entries: its leaf code and two bits. */ +export const CODE = 7; +export const REQUIRED = 8; +export const INHERITED = 16; + +/** All of `parts`; two and three parts get closures without a loop. */ +export function and(parts: readonly Check[]): Check { + switch (parts.length) { + case 0: + return ALWAYS; + case 1: + return parts[0]; + case 2: { + const [a, b] = parts; + return (v) => a(v) && b(v); + } + case 3: { + const [a, b, c] = parts; + return (v) => a(v) && b(v) && c(v); + } + default: + return (v) => { + for (let i = 0; i < parts.length; i++) if (!parts[i](v)) return false; + return true; + }; + } +} + +/** A check whose implementation is set later: the indirection a recursive reference goes through. */ +export function indirect(slot: { check: Check | undefined }): Check { + return (v) => (slot.check as Check)(v); +} + +/** `check` with `uri` pushed onto the dynamic scope while it runs (only for schemas with `$dynamicRef`). */ +export function scopedCheck(scope: string[], uri: string, check: Check): Check { + return (v) => { + scope.push(uri); + const ok = check(v); + scope.pop(); + return ok; + }; +} + +/** Any of the named types. */ +export function typeCheck(types: readonly unknown[]): Check { + const checks = types.map((t) => (typeof t === "string" ? TYPE_CHECKS.get(t) : undefined) ?? NEVER); + if (checks.length === 1) return checks[0]; + return (v) => { + for (let i = 0; i < checks.length; i++) if (checks[i](v)) return true; + return false; + }; +} + +/** A Uint8Array within the length bounds. */ +export function bytesCheck(min: number, max: number): Check { + return (v) => v instanceof Uint8Array && v.length >= min && v.length <= max; +} + +/** The number of Unicode code points of `s`: JSON Schema string lengths count code points, not UTF-16 units. */ +export function codePoints(s: string): number { + let n = 0; + for (let i = 0; i < s.length; i++, n++) { + const c = s.charCodeAt(i); + if (c >= 0xd800 && c <= 0xdbff) i++; + } + return n; +} + +// UTF-16 units bound code points from both sides (units / 2 <= points <= units), so most strings need no count. +function lengthOk(s: string, min: number, max: number): boolean { + const units = s.length; + if (units < min) return false; + if (units < 2 * min && codePoints(s) < min) return false; + return units <= max || codePoints(s) <= max; +} + +/** The string keywords: length bounds, pattern and format. */ +export function stringCheck( + strict: boolean, + min: number, + max: number, + re: RegExp | undefined, + format: ((value: string) => boolean) | undefined, +): Check { + if (re !== undefined && min === 0 && max === Number.POSITIVE_INFINITY && format === undefined) { + return strict ? (v) => typeof v === "string" && re.test(v) : (v) => typeof v !== "string" || re.test(v); + } + if (re === undefined && format === undefined) { + return strict + ? (v) => typeof v === "string" && lengthOk(v, min, max) + : (v) => typeof v !== "string" || lengthOk(v, min, max); + } + return (v) => { + if (typeof v !== "string") return !strict; + return lengthOk(v, min, max) && (re === undefined || re.test(v)) && (format === undefined || format(v)); + }; +} + +/** The number keywords. `mode`: 0 passes other types, 1 requires a number, 2 an integer. */ +export function numberCheck( + mode: 0 | 1 | 2, + min: number, + max: number, + xmin: number, + xmax: number, + multipleOf: number | undefined, +): Check { + return (v) => { + if (typeof v !== "number") return mode === 0; + if (mode === 2 && !Number.isInteger(v)) return false; + return v >= min && v <= max && v > xmin && v < xmax && (multipleOf === undefined || isMultiple(v, multipleOf)); + }; +} + +/** Whether `v` is a multiple of `d`, exactly for decimals: 0.0075 / 0.0001 is 74.99999999999999 in floating point. */ +export function isMultiple(v: number, d: number): boolean { + const q = v / d; + if (Number.isInteger(q)) return true; + if (!Number.isFinite(q)) return v % d === 0; + const [vc, ve] = decimal(v); + const [dc, de] = decimal(d); + return ve >= de ? (vc * 10n ** BigInt(ve - de)) % dc === 0n : vc % (dc * 10n ** BigInt(de - ve)) === 0n; +} + +function decimal(x: number): [bigint, number] { + const [mantissa, exponent = "0"] = Math.abs(x).toString().split("e"); + const point = mantissa.indexOf("."); + const fraction = point < 0 ? 0 : mantissa.length - point - 1; + return [BigInt(mantissa.replace(".", "")), Number(exponent) - fraction]; +} + +/** `inner` for objects; other values pass unless `strict`. */ +export function guardObject(inner: Check, strict: boolean): Check { + return strict ? (v) => isObject(v) && inner(v) : (v) => !isObject(v) || inner(v); +} + +/** `inner` for arrays; other values pass unless `strict`. */ +export function guardArray(inner: Check, strict: boolean): Check { + return strict ? (v) => Array.isArray(v) && inner(v) : (v) => !Array.isArray(v) || inner(v); +} + +/** + * An object with only declared properties (`additionalProperties: false`). `entries` is `[key, check, flags]` per + * property. Declared fields are checked, then the own keys are counted, so unknown keys are found without + * allocating. A property whose value is `undefined` counts as absent. + */ +export function closedObject(entries: readonly unknown[], guard: Guard): Check { + return (v) => { + if (guard !== GUARD_NONE && !isObject(v)) return guard === GUARD_LOOSE; + // Guarded: by the check above, or by the caller for GUARD_NONE. + const o = v as Record; + let present = 0; + for (let i = 0; i < entries.length; i += 3) { + const k = entries[i] as string; + const x = o[k]; + const f = entries[i + 2] as number; + if (x === undefined) { + if (f & REQUIRED) return false; + if (Object.hasOwn(o, k)) present++; + continue; + } + if (f & INHERITED && !Object.hasOwn(o, k)) { + if (f & REQUIRED) return false; + continue; + } + present++; + const c = f & CODE; + if (c !== 0 ? !leafOk(c, x) : !(entries[i + 1] as Check)(x)) return false; + } + let count = 0; + for (const _ in o) if (++count > present) return false; + return true; + }; +} + +/** An object with declared properties, required keys not declared, pattern properties and additional properties. */ +export function openObject( + entries: readonly unknown[], + keys: readonly string[], + extraRequired: readonly string[], + patterns: readonly (readonly [RegExp, Check])[], + additional: Check | undefined, + guard: Guard, +): Check { + const known = patterns.length > 0 || additional !== undefined ? new Set(keys) : undefined; + return (v) => { + if (guard !== GUARD_NONE && !isObject(v)) return guard === GUARD_LOOSE; + const o = v as Record; + for (let i = 0; i < entries.length; i += 3) { + const k = entries[i] as string; + const x = o[k]; + const f = entries[i + 2] as number; + if (x === undefined || (f & INHERITED && !Object.hasOwn(o, k))) { + if (f & REQUIRED) return false; + continue; + } + const c = f & CODE; + if (c !== 0 ? !leafOk(c, x) : !(entries[i + 1] as Check)(x)) return false; + } + for (let i = 0; i < extraRequired.length; i++) if (!Object.hasOwn(o, extraRequired[i])) return false; + if (known === undefined) return true; + for (const k of Object.keys(o)) { + let matched = known.has(k); + for (let i = 0; i < patterns.length; i++) { + if (patterns[i][0].test(k)) { + matched = true; + if (!patterns[i][1](o[k])) return false; + } + } + if (!matched && additional !== undefined && !additional(o[k])) return false; + } + return true; + }; +} + +/** Every property name passes `check` (`v` is an object). */ +export function propertyNamesCheck(check: Check): Check { + return (v) => { + for (const k in v as object) if (!check(k)) return false; + return true; + }; +} + +/** `minProperties` and `maxProperties` (`v` is an object). */ +export function propertyCountCheck(min: number, max: number): Check { + return (v) => { + let n = 0; + for (const _ in v as object) n++; + return n >= min && n <= max; + }; +} + +/** `dependentRequired` (`v` is an object). */ +export function dependentRequiredCheck(entries: readonly (readonly [string, readonly string[]])[]): Check { + return (v) => { + const o = v as object; + for (const [k, list] of entries) + if (Object.hasOwn(o, k)) for (const r of list) if (!Object.hasOwn(o, r)) return false; + return true; + }; +} + +/** `dependentSchemas` (`v` is an object). */ +export function dependentSchemasCheck(entries: readonly (readonly [string, Check])[]): Check { + return (v) => { + for (const [k, check] of entries) if (Object.hasOwn(v as object, k) && !check(v)) return false; + return true; + }; +} + +/** The common array: guard, length bounds, prefix items and the items after them, in one closure. */ +export function listCheck( + guard: Guard, + min: number, + max: number, + prefix: readonly Check[], + rest: Check | undefined, +): Check { + const n = prefix.length; + const code = rest === undefined ? 0 : (LEAF_CODES.get(rest) ?? 0); + return (v) => { + if (!Array.isArray(v)) return guard === GUARD_LOOSE; + const length = v.length; + if (length < min || length > max) return false; + const m = length < n ? length : n; + for (let i = 0; i < m; i++) if (!prefix[i](v[i])) return false; + if (rest === undefined) return true; + if (code !== 0) { + for (let i = n; i < length; i++) if (!leafOk(code, v[i])) return false; + return true; + } + for (let i = n; i < length; i++) if (!rest(v[i])) return false; + return true; + }; +} + +/** Prefix items and the items after them (`v` is an array). */ +export function itemsCheck(prefix: readonly Check[], rest: Check | undefined): Check { + return listCheck(GUARD_NONE, 0, Number.POSITIVE_INFINITY, prefix, rest); +} + +/** `minItems` and `maxItems` (`v` is an array). */ +export function lengthCheck(min: number, max: number): Check { + return (v) => (v as unknown[]).length >= min && (v as unknown[]).length <= max; +} + +/** `contains` with its bounds (`v` is an array). */ +export function containsCheck(check: Check, min: number, max: number): Check { + return (v) => { + const list = v as unknown[]; + let n = 0; + for (let i = 0; i < list.length; i++) if (check(list[i]) && ++n > max) return false; + return n >= min; + }; +} + +/** `uniqueItems` (`v` is an array). */ +export function uniqueCheck(v: unknown): boolean { + const list = v as unknown[]; + for (let i = 1; i < list.length; i++) for (let j = 0; j < i; j++) if (deepEqual(list[i], list[j])) return false; + return true; +} + +/** `enum`: up to 8 primitives by a scan, more by a Set, objects and arrays by deep equality. */ +export function enumCheck(values: readonly unknown[]): Check { + const primitives = values.filter((x) => typeof x !== "object" || x === null); + if (primitives.length === values.length && primitives.length <= 8) { + return (v) => { + for (let i = 0; i < primitives.length; i++) if (primitives[i] === v) return true; + return false; + }; + } + const set = new Set(primitives); + const objects = values.filter((x) => typeof x === "object" && x !== null); + return (v) => set.has(v) || (typeof v === "object" && objects.some((o) => deepEqual(v, o))); +} + +/** `const`. */ +export function constCheck(value: unknown): Check { + if (typeof value !== "object" || value === null) return (v) => v === value; + return (v) => deepEqual(v, value); +} + +/** `anyOf`. */ +export function anyCheck(checks: readonly Check[]): Check { + return (v) => { + for (let i = 0; i < checks.length; i++) if (checks[i](v)) return true; + return false; + }; +} + +/** `oneOf`. */ +export function oneCheck(checks: readonly Check[]): Check { + return (v) => { + let n = 0; + for (let i = 0; i < checks.length; i++) if (checks[i](v) && ++n > 1) return false; + return n === 1; + }; +} + +/** `not`. */ +export function notCheck(check: Check): Check { + return (v) => !check(v); +} + +/** `if`, `then` and `else`. */ +export function ifCheck(cond: Check, then: Check, otherwise: Check): Check { + return (v) => (cond(v) ? then(v) : otherwise(v)); +} + +/** A union of objects tagged by a const on `key`: the tag selects the one branch that can match. */ +export function taggedCheck(key: string, tags: readonly unknown[], checks: readonly Check[]): Check { + return (v) => { + if (!isObject(v)) return false; + const tag = v[key]; + for (let i = 0; i < tags.length; i++) if (tags[i] === tag) return checks[i](v); + return false; + }; +} + +/** JSON equality: same primitives, arrays with equal items in order, objects with the same keys and equal values. */ +export function deepEqual(a: unknown, b: unknown): boolean { + if (a === b) return true; + if (typeof a !== "object" || typeof b !== "object" || a === null || b === null) return false; + const array = Array.isArray(a); + if (array !== Array.isArray(b)) return false; + if (array) { + const x = a as unknown[]; + const y = b as unknown[]; + if (x.length !== y.length) return false; + for (let i = 0; i < x.length; i++) if (!deepEqual(x[i], y[i])) return false; + return true; + } + const x = a as Record; + const y = b as Record; + const keys = Object.keys(x); + if (keys.length !== Object.keys(y).length) return false; + for (const k of keys) if (!Object.hasOwn(y, k) || !deepEqual(x[k], y[k])) return false; + return true; +} diff --git a/packages/schema/src/compiler.ts b/packages/schema/src/compiler.ts new file mode 100644 index 000000000..01ec52bbf --- /dev/null +++ b/packages/schema/src/compiler.ts @@ -0,0 +1,618 @@ +// The compiler: a schema to one check closure, built once. Keywords group by the type they constrain, so a value +// is classified once per schema, and the common shapes (closed objects, arrays of one item schema, tagged unions, +// leaf schemas) get one specialized closure each. + +import { + ANNOTATING_FALSE, + ANNOTATING_TRUE, + type AnnotatingCheck, + annotateAll, + annotateAny, + annotateArray, + annotateDependent, + annotateIf, + annotateObject, + ignoreAnnotations, + indirectAnnotating, + isolateAnnotations, + scopedAnnotating, + unevaluatedItems, + unevaluatedProperties, + withFreshAnnotations, +} from "./annotations.ts"; +import { + ALWAYS, + and, + anyCheck, + bytesCheck, + closedObject, + constCheck, + containsCheck, + dependentRequiredCheck, + dependentSchemasCheck, + enumCheck, + GUARD_LOOSE, + GUARD_NONE, + GUARD_STRICT, + type Guard, + guardArray, + guardObject, + INHERITED, + IS_ARRAY, + IS_OBJECT, + ifCheck, + indirect, + itemsCheck, + LEAF_CODES, + lengthCheck, + listCheck, + NEVER, + notCheck, + numberCheck, + oneCheck, + openObject, + propertyCountCheck, + propertyNamesCheck, + REQUIRED, + scopedCheck, + stringCheck, + taggedCheck, + typeCheck, + uniqueCheck, +} from "./checks.ts"; +import { documentOf, isSchemaObject, Resolver, resolveUri } from "./resolver.ts"; +import { sharedCheck, sharedRegex } from "./shared.ts"; +import type { Check, CompileOptions, FormatCheck, JsonSchema, SchemaObject } from "./types.ts"; + +/** Thrown when a compile without dynamic scope tracking meets a bookended `$dynamicRef`: compile again with it. */ +export const RESTART = Symbol("restart"); + +/** One compile of a schema. Its caches and indexes are garbage once the root check is built. */ +export class Compiler { + readonly r: Resolver; + readonly draft: "2020-12" | "7"; + readonly formats: Readonly> | undefined; + /** Whether checks track the dynamic scope (the schema has a bookended `$dynamicRef`). */ + readonly dynamic: boolean; + private readonly schema: JsonSchema; + private readonly scope: string[]; + private readonly cache = new Map(); + private readonly annotatingCache = new Map(); + + constructor(schema: JsonSchema, options: CompileOptions, dynamic: boolean) { + this.schema = schema; + this.r = new Resolver(schema, options); + this.draft = this.r.draft; + this.formats = options.formats; + this.dynamic = dynamic; + this.scope = []; + } + + /** The check of the root schema. */ + root(): Check { + return this.check(this.schema, this.r.rootBase); + } + + /** The check of a schema (cached by schema object; a recursive reference goes through an indirection). */ + check(node: unknown, base: string): Check { + if (node === true) return ALWAYS; + if (node === false) return NEVER; + if (!isSchemaObject(node)) + throw new Error(`A schema must be an object or a boolean, got ${JSON.stringify(node)}`); + const cached = this.cache.get(node); + if (cached !== undefined) return cached; + const key = this.leafKey(node); + if (key !== undefined) { + const shared = sharedCheck(key, () => this.plain(node, base)); + this.cache.set(node, shared); + return shared; + } + const slot: { check: Check | undefined } = { check: undefined }; + this.cache.set(node, indirect(slot)); + const check = this.build(node, this.r.baseOf.get(node) ?? base); + slot.check = check; + this.cache.set(node, check); + return check; + } + + private build(node: SchemaObject, base: string): Check { + if (this.draft === "7" && typeof node.$ref === "string") return this.ref(node.$ref, base); + if (this.draft !== "7" && (node.unevaluatedProperties !== undefined || node.unevaluatedItems !== undefined)) { + return withFreshAnnotations(this.annotating(node, base)); + } + const check = this.plain(node, base); + return this.dynamic && this.r.isResource(node, base) ? scopedCheck(this.scope, base, check) : check; + } + + /** + * The sharing key of a leaf schema (only assertion keywords; no subschemas, references or identifiers): equal leaves + * in all compiled schemas share one check. Undefined for every other schema. + */ + private leafKey(node: SchemaObject): string | undefined { + const picked: Record = {}; + for (const k of Object.keys(node)) { + if (LEAF_KEYWORDS.has(k)) picked[k] = node[k]; + else if (!NEUTRAL_KEYWORDS.has(k)) return undefined; + } + // A format checks through this compile's format map, so a leaf with one is shared only while formats are off. + if (picked.format !== undefined && this.formats !== undefined) return undefined; + return this.draft + JSON.stringify(picked); + } + + /** Every keyword of `node`, not tracking annotations. */ + private plain(node: SchemaObject, base: string): Check { + const parts: Check[] = []; + const types = node.type === undefined ? undefined : Array.isArray(node.type) ? node.type : [node.type]; + const single = types?.length === 1 ? types[0] : undefined; + let typeInGroup = false; + if (single === "bytes") { + parts.push(bytesCheck(number(node.minLength) ?? 0, number(node.maxLength) ?? Number.POSITIVE_INFINITY)); + typeInGroup = true; + } else { + const groups: [Check | undefined, boolean][] = [ + [this.stringGroup(node, single === "string"), single === "string"], + [ + this.numberGroup(node, single === "number" ? 1 : single === "integer" ? 2 : 0), + single === "number" || single === "integer", + ], + [this.objectGroup(node, base, single === "object"), single === "object"], + [this.arrayGroup(node, base, single === "array"), single === "array"], + ]; + for (const [group, strict] of groups) { + if (group === undefined) continue; + parts.push(group); + if (strict) typeInGroup = true; + } + } + if (types !== undefined && !typeInGroup) parts.unshift(typeCheck(types)); + this.inPlace(node, base, parts); + return and(parts); + } + + // The keywords that apply to the value itself, whatever its type. + private inPlace(node: SchemaObject, base: string, parts: Check[]): void { + if (Array.isArray(node.enum)) parts.push(enumCheck(node.enum)); + if (Object.hasOwn(node, "const")) parts.push(constCheck(node.const)); + if (Array.isArray(node.allOf)) parts.push(and(node.allOf.map((s) => this.check(s, base)))); + if (Array.isArray(node.anyOf)) parts.push(this.union(node.anyOf, base, false)); + if (Array.isArray(node.oneOf)) parts.push(this.union(node.oneOf, base, true)); + if (node.not !== undefined) parts.push(notCheck(this.check(node.not, base))); + if (node.if !== undefined && (node.then !== undefined || node.else !== undefined)) { + const then = node.then === undefined ? ALWAYS : this.check(node.then, base); + const otherwise = node.else === undefined ? ALWAYS : this.check(node.else, base); + parts.push(ifCheck(this.check(node.if, base), then, otherwise)); + } + if (typeof node.$ref === "string") parts.push(this.ref(node.$ref, base)); + if (this.draft !== "7" && typeof node.$dynamicRef === "string") { + parts.push(this.dynamicRef(node.$dynamicRef, base)); + } + } + + private union(branches: readonly unknown[], base: string, exactlyOne: boolean): Check { + const checks = () => branches.map((s) => this.check(s, base)); + return ( + constUnion(branches, exactlyOne) ?? + this.tagged(branches, base) ?? + (exactlyOne ? oneCheck(checks()) : anyCheck(checks())) + ); + } + + private ref(ref: string, base: string): Check { + const target = this.r.resolve(ref, base); + const check = this.check(target.node, target.base); + return this.dynamic && target.base !== base ? scopedCheck(this.scope, target.base, check) : check; + } + + private dynamicRef(ref: string, base: string): Check { + const resolved = this.bookended(ref, base); + if (resolved === undefined) return this.ref(ref, base); + const { initial, name } = resolved; + const fallback = this.check(initial.node, initial.base); + const lookup = this.dynamicLookup(name, (node, b) => this.check(node, b)); + const scope = this.scope; + return (v) => { + for (let i = 0; i < scope.length; i++) { + const check = lookup(scope[i]); + if (check !== null) return check(v); + } + return fallback(v); + }; + } + + /** The initial target and anchor name of a `$dynamicRef` that resolves dynamically, or undefined for a plain ref. */ + private bookended( + ref: string, + base: string, + ): { initial: { node: unknown; base: string }; name: string } | undefined { + const initial = this.r.resolve(ref, base); + const uri = resolveUri(ref, base); + const name = uri.slice(documentOf(uri).length + 1); + const anchored = isSchemaObject(initial.node) && initial.node.$dynamicAnchor === name; + if (name === "" || name.startsWith("/") || !anchored) return undefined; + if (!this.dynamic) throw RESTART; + return { initial, name }; + } + + // Resolved at run time against the dynamic scope, so a schema with a bookended $dynamicRef keeps its compiler. + private dynamicLookup(name: string, compile: (node: SchemaObject, base: string) => T): (uri: string) => T | null { + const found = new Map(); + return (uri) => { + const key = `${uri}#${name}`; + let check = found.get(key); + if (check === undefined) { + const target = this.r.dynamicAnchors.get(key); + check = target === undefined ? null : compile(target, this.r.baseOf.get(target) ?? uri); + found.set(key, check); + } + return check; + }; + } + + /** anyOf/oneOf whose branches are all objects with a distinct required const on one key: the tag picks the branch. */ + private tagged(branches: readonly unknown[], base: string): Check | undefined { + if (branches.length < 2) return undefined; + const resolved: SchemaObject[] = []; + for (const branch of branches) { + if (!isSchemaObject(branch)) return undefined; + const ref = typeof branch.$ref === "string" && Object.keys(branch).length === 1 ? branch.$ref : undefined; + const node = ref === undefined ? branch : this.r.resolve(ref, this.r.baseOf.get(branch) ?? base).node; + if (!isSchemaObject(node)) return undefined; + resolved.push(node); + } + const first = resolved[0].properties; + if (!isSchemaObject(first)) return undefined; + for (const key of Object.keys(first)) { + const tags = resolved.map((s) => tagOf(s, key)); + if (tags.some((t) => t === undefined) || new Set(tags).size !== tags.length) continue; + return taggedCheck( + key, + tags, + branches.map((s) => this.check(s, base)), + ); + } + return undefined; + } + + private stringGroup(node: SchemaObject, strict: boolean): Check | undefined { + const min = number(node.minLength) ?? 0; + const max = number(node.maxLength) ?? Number.POSITIVE_INFINITY; + const re = typeof node.pattern === "string" ? sharedRegex(node.pattern) : undefined; + const format = + this.formats !== undefined && typeof node.format === "string" && Object.hasOwn(this.formats, node.format) + ? this.formats[node.format] + : undefined; + if (min === 0 && max === Number.POSITIVE_INFINITY && re === undefined && format === undefined) return undefined; + return stringCheck(strict, min, max, re, format); + } + + private numberGroup(node: SchemaObject, mode: 0 | 1 | 2): Check | undefined { + const min = number(node.minimum); + const max = number(node.maximum); + const xmin = number(node.exclusiveMinimum); + const xmax = number(node.exclusiveMaximum); + const multipleOf = number(node.multipleOf); + if ([min, max, xmin, xmax, multipleOf].every((x) => x === undefined)) return undefined; + const inf = Number.POSITIVE_INFINITY; + return numberCheck(mode, min ?? -inf, max ?? inf, xmin ?? -inf, xmax ?? inf, multipleOf); + } + + private objectGroup(node: SchemaObject, base: string, strict: boolean): Check | undefined { + const onlyCore = OBJECT_EXTRAS.every((k) => node[k] === undefined); + // A schema with only the core object keywords gets one closure that also does the type guard. + const core = this.objectCore(node, base, onlyCore ? (strict ? GUARD_STRICT : GUARD_LOOSE) : GUARD_NONE); + if (core !== undefined && onlyCore) return core; + const parts: Check[] = core === undefined ? [] : [core]; + if (node.propertyNames !== undefined) parts.push(propertyNamesCheck(this.check(node.propertyNames, base))); + const minProperties = number(node.minProperties); + const maxProperties = number(node.maxProperties); + if (minProperties !== undefined || maxProperties !== undefined) { + parts.push(propertyCountCheck(minProperties ?? 0, maxProperties ?? Number.POSITIVE_INFINITY)); + } + const dependentRequired: [string, string[]][] = []; + const dependentSchemas: [string, Check][] = []; + if (this.draft !== "7") { + for (const [k, list] of entries(node.dependentRequired)) + if (Array.isArray(list)) dependentRequired.push([k, list]); + for (const [k, s] of entries(node.dependentSchemas)) dependentSchemas.push([k, this.check(s, base)]); + } + for (const [k, d] of entries(node.dependencies)) { + if (Array.isArray(d)) dependentRequired.push([k, d]); + else dependentSchemas.push([k, this.check(d, base)]); + } + if (dependentRequired.length > 0) parts.push(dependentRequiredCheck(dependentRequired)); + if (dependentSchemas.length > 0) parts.push(dependentSchemasCheck(dependentSchemas)); + if (parts.length === 0) return strict ? IS_OBJECT : undefined; + return guardObject(and(parts), strict); + } + + private objectCore(node: SchemaObject, base: string, guard: Guard): Check | undefined { + const properties = isSchemaObject(node.properties) ? node.properties : {}; + const required = Array.isArray(node.required) ? node.required.filter((k) => typeof k === "string") : []; + const patterns = isSchemaObject(node.patternProperties) ? node.patternProperties : {}; + const additional = node.additionalProperties; + const keys = Object.keys(properties); + const patternKeys = Object.keys(patterns); + if ( + keys.length + required.length + patternKeys.length === 0 && + (additional === undefined || additional === true) + ) { + return undefined; + } + // One array per object: key, check, flags (leaf code, required, key also names an Object.prototype member). + const entries: unknown[] = []; + for (const k of keys) { + const check = this.check(properties[k], base); + const flags = + (LEAF_CODES.get(check) ?? 0) | + (required.includes(k) ? REQUIRED : 0) | + (k in Object.prototype ? INHERITED : 0); + entries.push(k, check, flags); + } + const extraRequired = required.filter((k) => !Object.hasOwn(properties, k)); + const pats = patternKeys.map((p) => [sharedRegex(p), this.check(patterns[p], base)] as const); + if (additional === false && pats.length === 0 && extraRequired.length === 0) return closedObject(entries, guard); + const extra = additional === undefined || additional === true ? undefined : this.check(additional, base); + return openObject(entries, keys, extraRequired, pats, extra, guard); + } + + private arrayGroup(node: SchemaObject, base: string, strict: boolean): Check | undefined { + let prefix: readonly unknown[] = []; + let rest: unknown; + if (Array.isArray(node.items)) { + prefix = node.items; + rest = node.additionalItems; + } else { + if (this.draft !== "7" && Array.isArray(node.prefixItems)) prefix = node.prefixItems; + rest = node.items; + } + const prefixChecks = prefix.map((s) => this.check(s, base)); + const restCheck = rest === undefined || rest === true ? undefined : this.check(rest, base); + const minItems = number(node.minItems); + const maxItems = number(node.maxItems); + if ( + node.contains === undefined && + node.uniqueItems !== true && + (prefixChecks.length > 0 || restCheck !== undefined) + ) { + // The common array: one closure for the guard, the length bounds and the items. + const guard = strict ? GUARD_STRICT : GUARD_LOOSE; + return listCheck(guard, minItems ?? 0, maxItems ?? Number.POSITIVE_INFINITY, prefixChecks, restCheck); + } + const parts: Check[] = []; + if (prefixChecks.length > 0 || restCheck !== undefined) parts.push(itemsCheck(prefixChecks, restCheck)); + if (minItems !== undefined || maxItems !== undefined) { + parts.push(lengthCheck(minItems ?? 0, maxItems ?? Number.POSITIVE_INFINITY)); + } + if (node.contains !== undefined) { + const min = this.draft === "7" ? 1 : (number(node.minContains) ?? 1); + const max = + this.draft === "7" ? Number.POSITIVE_INFINITY : (number(node.maxContains) ?? Number.POSITIVE_INFINITY); + parts.push(containsCheck(this.check(node.contains, base), min, max)); + } + if (node.uniqueItems === true) parts.push(uniqueCheck); + if (parts.length === 0) return strict ? IS_ARRAY : undefined; + return guardArray(and(parts), strict); + } + + /** The annotating check of a schema evaluated in place (only below `unevaluatedProperties`/`unevaluatedItems`). */ + annotating(node: unknown, base: string): AnnotatingCheck { + if (node === true) return ANNOTATING_TRUE; + if (node === false) return ANNOTATING_FALSE; + if (!isSchemaObject(node)) + throw new Error(`A schema must be an object or a boolean, got ${JSON.stringify(node)}`); + const cached = this.annotatingCache.get(node); + if (cached !== undefined) return cached; + const slot: { check: AnnotatingCheck | undefined } = { check: undefined }; + this.annotatingCache.set(node, indirectAnnotating(slot)); + const b = this.r.baseOf.get(node) ?? base; + const check = this.buildAnnotating(node, b); + slot.check = check; + this.annotatingCache.set(node, check); + return check; + } + + private buildAnnotating(node: SchemaObject, base: string): AnnotatingCheck { + const rest: Record = {}; + let annotates = false; + for (const k of Object.keys(node)) { + if (ANNOTATING.has(k)) annotates = true; + else rest[k] = node[k]; + } + const plain = this.plain(rest, base); + const parts: AnnotatingCheck[] = []; + if (annotates) this.annotatingParts(node, base, parts); + const all = parts.length === 0 ? ignoreAnnotations(plain) : annotateAll(plain, parts); + const own = + node.unevaluatedProperties !== undefined || node.unevaluatedItems !== undefined + ? isolateAnnotations(all) + : all; + return this.dynamic && this.r.isResource(node, base) ? scopedAnnotating(this.scope, base, own) : own; + } + + private annotatingParts(node: SchemaObject, base: string, parts: AnnotatingCheck[]): void { + if ( + node.properties !== undefined || + node.patternProperties !== undefined || + node.additionalProperties !== undefined + ) { + const props = entries(node.properties).map(([k, s]) => [k, this.check(s, base)] as const); + const pats = entries(node.patternProperties).map(([p, s]) => [sharedRegex(p), this.check(s, base)] as const); + const extra = + node.additionalProperties === undefined ? undefined : this.check(node.additionalProperties, base); + parts.push(annotateObject(props, pats, extra)); + } + if (node.prefixItems !== undefined || node.items !== undefined || node.contains !== undefined) { + const prefix = Array.isArray(node.prefixItems) + ? node.prefixItems + : Array.isArray(node.items) + ? node.items + : []; + const rest = Array.isArray(node.items) ? node.additionalItems : node.items; + parts.push( + annotateArray( + prefix.map((s) => this.check(s, base)), + rest === undefined ? undefined : this.check(rest, base), + node.contains === undefined ? undefined : this.check(node.contains, base), + number(node.minContains) ?? 1, + number(node.maxContains) ?? Number.POSITIVE_INFINITY, + ), + ); + } + if (Array.isArray(node.allOf)) for (const s of node.allOf) parts.push(this.annotating(s, base)); + if (Array.isArray(node.anyOf)) + parts.push( + annotateAny( + node.anyOf.map((s) => this.annotating(s, base)), + false, + ), + ); + if (Array.isArray(node.oneOf)) + parts.push( + annotateAny( + node.oneOf.map((s) => this.annotating(s, base)), + true, + ), + ); + if (node.if !== undefined) { + const then = node.then === undefined ? undefined : this.annotating(node.then, base); + const otherwise = node.else === undefined ? undefined : this.annotating(node.else, base); + parts.push(annotateIf(this.annotating(node.if, base), then, otherwise)); + } + if (typeof node.$ref === "string") parts.push(this.annotatingRef(node.$ref, base)); + if (typeof node.$dynamicRef === "string") parts.push(this.annotatingDynamicRef(node.$dynamicRef, base)); + for (const [k, s] of entries(node.dependentSchemas)) parts.push(annotateDependent(k, this.annotating(s, base))); + if (node.unevaluatedProperties !== undefined) { + parts.push(unevaluatedProperties(this.check(node.unevaluatedProperties, base))); + } + if (node.unevaluatedItems !== undefined) parts.push(unevaluatedItems(this.check(node.unevaluatedItems, base))); + } + + private annotatingRef(ref: string, base: string): AnnotatingCheck { + const target = this.r.resolve(ref, base); + const check = this.annotating(target.node, target.base); + return this.dynamic && target.base !== base ? scopedAnnotating(this.scope, target.base, check) : check; + } + + private annotatingDynamicRef(ref: string, base: string): AnnotatingCheck { + const resolved = this.bookended(ref, base); + if (resolved === undefined) return this.annotatingRef(ref, base); + const { initial, name } = resolved; + const fallback = this.annotating(initial.node, initial.base); + const lookup = this.dynamicLookup(name, (node, b) => this.annotating(node, b)); + const scope = this.scope; + return (v, a) => { + for (let i = 0; i < scope.length; i++) { + const check = lookup(scope[i]); + if (check !== null) return check(v, a); + } + return fallback(v, a); + }; + } +} + +// Keywords whose check depends only on their own value, and keywords that never affect a check. +const LEAF_KEYWORDS = new Set([ + "type", + "enum", + "const", + "minLength", + "maxLength", + "pattern", + "format", + "minimum", + "maximum", + "exclusiveMinimum", + "exclusiveMaximum", + "multipleOf", + "minItems", + "maxItems", + "uniqueItems", + "minProperties", + "maxProperties", + "required", + "dependentRequired", +]); + +/** Keywords that never affect a check. */ +export const NEUTRAL_KEYWORDS = new Set([ + "$schema", + "$comment", + "title", + "description", + "default", + "examples", + "deprecated", + "readOnly", + "writeOnly", + "$defs", + "definitions", + "contentEncoding", + "contentMediaType", + "minContains", + "maxContains", + "$vocabulary", +]); + +// The object keywords besides the core (properties, required, patternProperties, additionalProperties). +const OBJECT_EXTRAS = [ + "propertyNames", + "minProperties", + "maxProperties", + "dependentRequired", + "dependentSchemas", + "dependencies", +]; + +// The keywords an annotating check handles itself; every other keyword compiles into its plain part. +const ANNOTATING = new Set([ + "properties", + "patternProperties", + "additionalProperties", + "prefixItems", + "items", + "additionalItems", + "contains", + "minContains", + "maxContains", + "allOf", + "anyOf", + "oneOf", + "if", + "then", + "else", + "$ref", + "$dynamicRef", + "dependentSchemas", + "unevaluatedProperties", + "unevaluatedItems", +]); + +/** anyOf/oneOf whose branches are all `{ const: primitive }`: one shared enum (distinct values make oneOf the same). */ +function constUnion(branches: readonly unknown[], exactlyOne: boolean): Check | undefined { + const values: unknown[] = []; + for (const branch of branches) { + if (!isSchemaObject(branch) || !Object.hasOwn(branch, "const")) return undefined; + for (const k of Object.keys(branch)) if (k !== "const" && !NEUTRAL_KEYWORDS.has(k)) return undefined; + if (typeof branch.const === "object" && branch.const !== null) return undefined; + if (exactlyOne && values.includes(branch.const)) return undefined; + values.push(branch.const); + } + return sharedCheck(`enum${JSON.stringify(values)}`, () => enumCheck(values)); +} + +// The tag of a tagged-union branch: a primitive const on a required property of an object schema. +function tagOf(branch: SchemaObject, key: string): unknown { + const properties = branch.properties; + if (branch.type !== "object" || !Array.isArray(branch.required) || !branch.required.includes(key)) return undefined; + const tag = isSchemaObject(properties) ? properties[key] : undefined; + if (!isSchemaObject(tag) || !Object.hasOwn(tag, "const")) return undefined; + return typeof tag.const === "object" && tag.const !== null ? undefined : tag.const; +} + +function number(value: unknown): number | undefined { + return typeof value === "number" ? value : undefined; +} + +function entries(value: unknown): [string, unknown][] { + return isSchemaObject(value) ? Object.entries(value) : []; +} diff --git a/packages/schema/src/explain.ts b/packages/schema/src/explain.ts new file mode 100644 index 000000000..fc8068a26 --- /dev/null +++ b/packages/schema/src/explain.ts @@ -0,0 +1,327 @@ +// Errors of a failed value: a walk over the schema that reports each failing keyword at the value it failed on. +// It runs only for values that failed the check, using the compiler's checks for subschemas. + +import { newAnnotations } from "./annotations.ts"; +import { + codePoints, + constCheck, + containsCheck, + enumCheck, + isMultiple, + isObject, + NEVER, + TYPE_CHECKS, + uniqueCheck, +} from "./checks.ts"; +import type { Compiler } from "./compiler.ts"; +import { isSchemaObject } from "./resolver.ts"; +import { sharedRegex } from "./shared.ts"; +import type { SchemaObject, ValidationError } from "./types.ts"; + +/** Explains failures of one compiled schema. */ +export class Explainer { + private readonly c: Compiler; + private readonly schema: unknown; + + constructor(compiler: Compiler, schema: unknown) { + this.c = compiler; + this.schema = schema; + compiler.root(); + } + + /** The errors of `value`, at least one (`value` failed the check). */ + errors(value: unknown): ValidationError[] { + const out: ValidationError[] = []; + this.walk(this.schema, this.c.r.rootBase, value, "", out); + if (out.length === 0) out.push({ path: "", keyword: "schema", message: "invalid" }); + return out; + } + + private ok(node: unknown, base: string, v: unknown): boolean { + return this.c.check(node, base)(v); + } + + // For $dynamicRef schemas this check runs outside the original dynamic scope, so their errors are approximate. + private walk(node: unknown, base: string, v: unknown, path: string, out: ValidationError[]): void { + if (node === true) return; + if (node === false) { + out.push({ path, keyword: "false", message: "no value is allowed here" }); + return; + } + if (!isSchemaObject(node)) return; + const b = this.c.r.baseOf.get(node) ?? base; + if (this.ok(node, b, v)) return; + const start = out.length; + const push = (keyword: string, message: string, at = path) => { + out.push({ path: at, keyword, message }); + }; + if (this.c.draft === "7" && typeof node.$ref === "string") { + const target = this.c.r.resolve(node.$ref, b); + this.walk(target.node, target.base, v, path, out); + return; + } + if (node.type !== undefined) { + const types = Array.isArray(node.type) ? node.type : [node.type]; + const checks = types.map((t) => (typeof t === "string" ? TYPE_CHECKS.get(t) : undefined) ?? NEVER); + if (!checks.some((check) => check(v))) { + push("type", `expected ${types.join(" or ")}, got ${typeName(v)}`); + return; + } + } + if (Array.isArray(node.enum) && !enumCheck(node.enum)(v)) + push("enum", `expected one of ${JSON.stringify(node.enum)}`); + if (Object.hasOwn(node, "const") && !constCheck(node.const)(v)) + push("const", `expected ${JSON.stringify(node.const)}`); + if (typeof v === "string" && node.type !== "bytes") this.walkString(node, v, push); + if (v instanceof Uint8Array && node.type === "bytes") this.walkBytes(node, v, push); + if (typeof v === "number") this.walkNumber(node, v, push); + if (isObject(v)) this.walkObject(node, b, v, path, out, push); + if (Array.isArray(v)) this.walkArray(node, b, v, path, out, push); + if (Array.isArray(node.allOf)) for (const s of node.allOf) this.walk(s, b, v, path, out); + if (Array.isArray(node.anyOf)) this.walkUnion(node.anyOf, b, v, path, out, false); + if (Array.isArray(node.oneOf)) this.walkUnion(node.oneOf, b, v, path, out, true); + if (node.not !== undefined && this.ok(node.not, b, v)) push("not", "must not match the schema in not"); + if (node.if !== undefined) { + const branch = this.ok(node.if, b, v) ? node.then : node.else; + if (branch !== undefined) this.walk(branch, b, v, path, out); + } + if (typeof node.$ref === "string") { + const target = this.c.r.resolve(node.$ref, b); + this.walk(target.node, target.base, v, path, out); + } + if (out.length === start && (node.unevaluatedProperties !== undefined || node.unevaluatedItems !== undefined)) { + this.walkUnevaluated(node, b, v, path, out); + } + if (out.length === start && typeof node.$dynamicRef === "string") { + push("$dynamicRef", "does not match the dynamically referenced schema"); + } + } + + private walkString(node: SchemaObject, v: string, push: Push): void { + const n = codePoints(v); + if (typeof node.minLength === "number" && n < node.minLength) { + push("minLength", `expected at least ${node.minLength} characters`); + } + if (typeof node.maxLength === "number" && n > node.maxLength) { + push("maxLength", `expected at most ${node.maxLength} characters`); + } + if (typeof node.pattern === "string" && !sharedRegex(node.pattern).test(v)) { + push("pattern", `expected to match ${node.pattern}`); + } + const formats = this.c.formats; + const format = + formats !== undefined && typeof node.format === "string" && Object.hasOwn(formats, node.format) + ? formats[node.format] + : undefined; + if (format !== undefined && !format(v)) push("format", `expected format ${node.format}`); + } + + private walkBytes(node: SchemaObject, v: Uint8Array, push: Push): void { + if (typeof node.minLength === "number" && v.length < node.minLength) { + push("minLength", `expected at least ${node.minLength} bytes`); + } + if (typeof node.maxLength === "number" && v.length > node.maxLength) { + push("maxLength", `expected at most ${node.maxLength} bytes`); + } + } + + private walkNumber(node: SchemaObject, v: number, push: Push): void { + if (typeof node.minimum === "number" && v < node.minimum) push("minimum", `expected >= ${node.minimum}`); + if (typeof node.maximum === "number" && v > node.maximum) push("maximum", `expected <= ${node.maximum}`); + if (typeof node.exclusiveMinimum === "number" && v <= node.exclusiveMinimum) { + push("exclusiveMinimum", `expected > ${node.exclusiveMinimum}`); + } + if (typeof node.exclusiveMaximum === "number" && v >= node.exclusiveMaximum) { + push("exclusiveMaximum", `expected < ${node.exclusiveMaximum}`); + } + if (typeof node.multipleOf === "number" && !isMultiple(v, node.multipleOf)) { + push("multipleOf", `expected a multiple of ${node.multipleOf}`); + } + } + + private walkObject( + node: SchemaObject, + b: string, + v: Record, + path: string, + out: ValidationError[], + push: Push, + ): void { + const properties = isSchemaObject(node.properties) ? node.properties : {}; + const required = Array.isArray(node.required) ? node.required : []; + for (const k of required) { + if (typeof k === "string" && (!Object.hasOwn(v, k) || v[k] === undefined)) + push("required", "missing", pointer(path, k)); + } + for (const k of Object.keys(properties)) { + if (Object.hasOwn(v, k) && v[k] !== undefined) this.walk(properties[k], b, v[k], pointer(path, k), out); + } + const patterns = isSchemaObject(node.patternProperties) ? Object.entries(node.patternProperties) : []; + for (const k of Object.keys(v)) { + let matched = Object.hasOwn(properties, k); + for (const [p, s] of patterns) { + if (sharedRegex(p).test(k)) { + matched = true; + this.walk(s, b, v[k], pointer(path, k), out); + } + } + if (!matched && node.additionalProperties !== undefined) { + if (node.additionalProperties === false) push("additionalProperties", "unknown field", pointer(path, k)); + else this.walk(node.additionalProperties, b, v[k], pointer(path, k), out); + } + if (node.propertyNames !== undefined && !this.ok(node.propertyNames, b, k)) { + push("propertyNames", "invalid property name", pointer(path, k)); + } + } + const count = Object.keys(v).length; + if (typeof node.minProperties === "number" && count < node.minProperties) { + push("minProperties", `expected at least ${node.minProperties} properties`); + } + if (typeof node.maxProperties === "number" && count > node.maxProperties) { + push("maxProperties", `expected at most ${node.maxProperties} properties`); + } + const dependencies = { + ...(this.c.draft === "7" || !isSchemaObject(node.dependentRequired) ? {} : node.dependentRequired), + ...(isSchemaObject(node.dependencies) ? node.dependencies : {}), + }; + for (const [k, d] of Object.entries(dependencies)) { + if (!Object.hasOwn(v, k)) continue; + if (!Array.isArray(d)) this.walk(d, b, v, path, out); + else + for (const r of d) + if (!Object.hasOwn(v, r)) push("dependentRequired", `required when ${k} is present`, pointer(path, r)); + } + if (this.c.draft !== "7" && isSchemaObject(node.dependentSchemas)) { + for (const [k, s] of Object.entries(node.dependentSchemas)) + if (Object.hasOwn(v, k)) this.walk(s, b, v, path, out); + } + } + + private walkArray( + node: SchemaObject, + b: string, + v: unknown[], + path: string, + out: ValidationError[], + push: Push, + ): void { + let prefix: readonly unknown[] = []; + let rest: unknown; + if (Array.isArray(node.items)) { + prefix = node.items; + rest = node.additionalItems; + } else { + if (this.c.draft !== "7" && Array.isArray(node.prefixItems)) prefix = node.prefixItems; + rest = node.items; + } + for (let i = 0; i < v.length; i++) { + const s = i < prefix.length ? prefix[i] : rest; + if (s !== undefined) this.walk(s, b, v[i], `${path}/${i}`, out); + } + if (typeof node.minItems === "number" && v.length < node.minItems) { + push("minItems", `expected at least ${node.minItems} items`); + } + if (typeof node.maxItems === "number" && v.length > node.maxItems) { + push("maxItems", `expected at most ${node.maxItems} items`); + } + if (node.uniqueItems === true && !uniqueCheck(v)) push("uniqueItems", "expected unique items"); + if (node.contains !== undefined) { + const draft7 = this.c.draft === "7"; + const min = draft7 || typeof node.minContains !== "number" ? 1 : node.minContains; + const max = draft7 || typeof node.maxContains !== "number" ? Number.POSITIVE_INFINITY : node.maxContains; + if (!containsCheck(this.c.check(node.contains, b), min, max)(v)) { + push("contains", `expected between ${min} and ${max} matching items`); + } + } + } + + private walkUnion( + branches: readonly unknown[], + b: string, + v: unknown, + path: string, + out: ValidationError[], + exactlyOne: boolean, + ): void { + const matches = branches.filter((s) => this.ok(s, b, v)).length; + if (matches === 1 || (!exactlyOne && matches > 0)) return; + const keyword = exactlyOne ? "oneOf" : "anyOf"; + if (matches > 1) { + out.push({ path, keyword, message: "matches more than one alternative" }); + return; + } + // A tagged union reports the branch its tag selects; otherwise the branch whose first error is deepest. + const tag = this.tagOf(branches, b); + if (tag !== undefined && isObject(v)) { + const i = tag.values.indexOf(v[tag.key]); + if (i < 0) + out.push({ + path: pointer(path, tag.key), + keyword, + message: `expected one of ${JSON.stringify(tag.values)}`, + }); + else this.walk(branches[i], b, v, path, out); + return; + } + let best: ValidationError[] | undefined; + for (const s of branches) { + const errors: ValidationError[] = []; + this.walk(s, b, v, path, errors); + if (errors.length > 0 && (best === undefined || errors[0].path.length > best[0].path.length)) best = errors; + } + out.push(...(best ?? [{ path, keyword, message: "matches no alternative" }])); + } + + private tagOf(branches: readonly unknown[], b: string): { key: string; values: unknown[] } | undefined { + const resolved = branches.map((s) => + isSchemaObject(s) && typeof s.$ref === "string" ? this.c.r.resolve(s.$ref, b).node : s, + ); + const first = resolved[0]; + if (!isSchemaObject(first) || !isSchemaObject(first.properties)) return undefined; + for (const key of Object.keys(first.properties)) { + const values = resolved.map((s) => { + const tag = isSchemaObject(s) && isSchemaObject(s.properties) ? s.properties[key] : undefined; + return isSchemaObject(tag) ? tag.const : undefined; + }); + const primitive = values.every((x) => x !== undefined && (typeof x !== "object" || x === null)); + if (primitive && new Set(values).size === values.length) return { key, values }; + } + return undefined; + } + + private walkUnevaluated(node: SchemaObject, b: string, v: unknown, path: string, out: ValidationError[]): void { + const { unevaluatedProperties, unevaluatedItems, ...rest } = node; + const a = newAnnotations(); + this.c.annotating(rest, b)(v, a); + if (isObject(v) && unevaluatedProperties !== undefined && !a.allProps) { + for (const k of Object.keys(v)) { + if (a.props?.has(k)) continue; + if (unevaluatedProperties === false) { + out.push({ path: pointer(path, k), keyword: "unevaluatedProperties", message: "unknown field" }); + } else this.walk(unevaluatedProperties, b, v[k], pointer(path, k), out); + } + } + if (Array.isArray(v) && unevaluatedItems !== undefined && !a.allItems) { + for (let i = a.items; i < v.length; i++) { + if (a.indexes?.has(i)) continue; + if (unevaluatedItems === false) { + out.push({ path: `${path}/${i}`, keyword: "unevaluatedItems", message: "unexpected item" }); + } else this.walk(unevaluatedItems, b, v[i], `${path}/${i}`, out); + } + } + } +} + +type Push = (keyword: string, message: string, at?: string) => void; + +function pointer(path: string, key: string | number): string { + return `${path}/${String(key).replace(/~/g, "~0").replace(/\//g, "~1")}`; +} + +function typeName(v: unknown): string { + if (v === null) return "null"; + if (Array.isArray(v)) return "array"; + if (v instanceof Uint8Array) return "bytes"; + if (typeof v === "number" && Number.isInteger(v)) return "integer"; + return typeof v; +} diff --git a/packages/schema/src/formats/index.ts b/packages/schema/src/formats/index.ts new file mode 100644 index 000000000..6919b9256 --- /dev/null +++ b/packages/schema/src/formats/index.ts @@ -0,0 +1,46 @@ +// The format checks of JSON Schema 2020-12, for compile()'s `formats` option. An optional entry point: schemas +// compiled without it treat `format` as an annotation and do not bundle these checks. + +import type { FormatCheck } from "../types.ts"; +import { IsDate } from "./typebox/date.ts"; +import { IsDateTime } from "./typebox/date_time.ts"; +import { IsDuration } from "./typebox/duration.ts"; +import { IsEmail } from "./typebox/email.ts"; +import { IsHostname } from "./typebox/hostname.ts"; +import { IsIdnEmail } from "./typebox/idn_email.ts"; +import { IsIdnHostname } from "./typebox/idn_hostname.ts"; +import { IsIPv4 } from "./typebox/ipv4.ts"; +import { IsIPv6 } from "./typebox/ipv6.ts"; +import { IsIri } from "./typebox/iri.ts"; +import { IsIriReference } from "./typebox/iri_reference.ts"; +import { IsJsonPointer } from "./typebox/json_pointer.ts"; +import { IsRegex } from "./typebox/regex.ts"; +import { IsRelativeJsonPointer } from "./typebox/relative_json_pointer.ts"; +import { IsTime } from "./typebox/time.ts"; +import { IsUri } from "./typebox/uri.ts"; +import { IsUriReference } from "./typebox/uri_reference.ts"; +import { IsUriTemplate } from "./typebox/uri_template.ts"; +import { IsUuid } from "./typebox/uuid.ts"; + +/** Every format JSON Schema 2020-12 defines, by name. The checks are adapted from TypeBox 1.3.27 (MIT). */ +export const formats: Readonly> = Object.freeze({ + date: IsDate, + "date-time": IsDateTime, + duration: IsDuration, + email: IsEmail, + hostname: IsHostname, + "idn-email": IsIdnEmail, + "idn-hostname": IsIdnHostname, + ipv4: IsIPv4, + ipv6: IsIPv6, + iri: IsIri, + "iri-reference": IsIriReference, + "json-pointer": IsJsonPointer, + regex: IsRegex, + "relative-json-pointer": IsRelativeJsonPointer, + time: IsTime, + uri: IsUri, + "uri-reference": IsUriReference, + "uri-template": IsUriTemplate, + uuid: IsUuid, +}); diff --git a/packages/schema/src/formats/typebox/date.ts b/packages/schema/src/formats/typebox/date.ts new file mode 100644 index 000000000..e35cf0770 --- /dev/null +++ b/packages/schema/src/formats/typebox/date.ts @@ -0,0 +1,47 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const DAYS = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; +const DATE = /^(\d\d\d\d)-(\d\d)-(\d\d)$/; + +function IsLeapYear(year: number): boolean { + return year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0); +} +/** + * Returns true if the value is a ISO8601 Date component string + * @description https://datatracker.ietf.org/doc/html/rfc3339 + * @example `2020-12-12` + */ +export function IsDate(value: string): boolean { + const matches: string[] | null = DATE.exec(value); + if (!matches) return false; + const year: number = +matches[1]; + const month: number = +matches[2]; + const day: number = +matches[3]; + return month >= 1 && month <= 12 && day >= 1 && day <= (month === 2 && IsLeapYear(year) ? 29 : DAYS[month]); +} diff --git a/packages/schema/src/formats/typebox/date_time.ts b/packages/schema/src/formats/typebox/date_time.ts new file mode 100644 index 000000000..df4174a40 --- /dev/null +++ b/packages/schema/src/formats/typebox/date_time.ts @@ -0,0 +1,40 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import { IsDate } from "./date.ts"; +import { IsTime } from "./time.ts"; + +/** + * Returns true if the value is a ISO8601 DateTime string + * @specification https://datatracker.ietf.org/doc/html/rfc3339 + * @example `2020-12-12T20:20:40+00:00` + */ +export function IsDateTime(value: string): boolean { + const dateTime: string[] = value.split(/T/i); + return dateTime.length === 2 && IsDate(dateTime[0]) && IsTime(dateTime[1]); +} diff --git a/packages/schema/src/formats/typebox/duration.ts b/packages/schema/src/formats/typebox/duration.ts new file mode 100644 index 000000000..20d812c4f --- /dev/null +++ b/packages/schema/src/formats/typebox/duration.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const Duration = + /^P((\d+Y(\d+M(\d+D)?)?|\d+M(\d+D)?|\d+D)(T(\d+H(\d+M(\d+S)?)?|\d+M(\d+S)?|\d+S))?|T(\d+H(\d+M(\d+S)?)?|\d+M(\d+S)?|\d+S)|\d+W)$/; + +/** + * Returns true if the value is a valid ISO-8601 duration. + * @specification https://tools.ietf.org/html/rfc3339 + */ +export function IsDuration(value: string): boolean { + return Duration.test(value); +} diff --git a/packages/schema/src/formats/typebox/email.ts b/packages/schema/src/formats/typebox/email.ts new file mode 100644 index 000000000..a6840daed --- /dev/null +++ b/packages/schema/src/formats/typebox/email.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const Email = + /^(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[^"\\]|\\[\x20-\x7e])*")@(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*|\[(?:IPv6:[a-f0-9:]+|(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3})\])$/i; + +/** + * Returns true if the value is an Email + * @specification https://datatracker.ietf.org/doc/html/rfc5322 + */ +export function IsEmail(value: string): boolean { + return Email.test(value); +} diff --git a/packages/schema/src/formats/typebox/hostname.ts b/packages/schema/src/formats/typebox/hostname.ts new file mode 100644 index 000000000..68c3710fb --- /dev/null +++ b/packages/schema/src/formats/typebox/hostname.ts @@ -0,0 +1,39 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import * as Idna from "./idna/index.ts"; + +/** + * Returns true if the value is a valid hostname. + * @specification https://tools.ietf.org/html/rfc1123 + * @specification https://tools.ietf.org/html/rfc5891 + * @specification https://tools.ietf.org/html/rfc5892 + */ +export function IsHostname(value: string): boolean { + return Idna.IsHostname(value); +} diff --git a/packages/schema/src/formats/typebox/idn_email.ts b/packages/schema/src/formats/typebox/idn_email.ts new file mode 100644 index 000000000..92e3abdaf --- /dev/null +++ b/packages/schema/src/formats/typebox/idn_email.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const IdnEmail = + /^(?:[A-Za-z0-9!#$%&'*+/=?^_`{|}~\u{0080}-\u{10FFFF}-]+(?:\.[A-Za-z0-9!#$%&'*+/=?^_`{|}~\u{0080}-\u{10FFFF}-]+)*|"(?:[^"\\]|\\.)*")@[\p{L}\p{N}](?:[\p{L}\p{N}-]{0,62})(?> 1; + delta += Math.floor(delta / numPoints); + let k = 0; + while (delta > ((PUNYCODE_BASE - PUNYCODE_TMIN) * PUNYCODE_TMAX) >> 1) { + delta = Math.floor(delta / (PUNYCODE_BASE - PUNYCODE_TMIN)); + k += PUNYCODE_BASE; + } + return k + Math.floor(((PUNYCODE_BASE - PUNYCODE_TMIN + 1) * delta) / (delta + PUNYCODE_SKEW)); +} +// ------------------------------------------------------------------ +// Decode +// ------------------------------------------------------------------ +export function Decode(value: string): string { + const output: number[] = []; + let n = PUNYCODE_INITIAL_N; + let i = 0; + let bias = PUNYCODE_INITIAL_BIAS; + const delimIdx = value.lastIndexOf("-"); + if (delimIdx > 0) { + for (let j = 0; j < delimIdx; j++) { + const cp = value.charCodeAt(j); + if (cp >= 128) ThrowDontCare(); // throw new Error('Invalid punycode: non-basic before delimiter') + output.push(cp); + } + } + let inIdx = delimIdx < 0 ? 0 : delimIdx + 1; + while (inIdx < value.length) { + const oldi = i; + let w = 1; + let k = PUNYCODE_BASE; + while (true) { + if (inIdx >= value.length) ThrowDontCare(); // throw new Error('Invalid punycode: unexpected end of input') + const ch = value.charCodeAt(inIdx++); + let digit: number; + if (ch >= 0x61 && ch <= 0x7a) + digit = ch - 0x61; // a-z => 0-25 + else if (ch >= 0x30 && ch <= 0x39) + digit = ch - 0x30 + 26; // 0-9 => 26-35 + // else if (ch >= 0x41 && ch <= 0x5a) digit = ch - 0x41 // A-Z => 0-25 // (no-spec-coverage) + else ThrowDontCare(); // throw new Error('Invalid punycode: bad digit character') + i += digit * w; + const t = k <= bias ? PUNYCODE_TMIN : k >= bias + PUNYCODE_TMAX ? PUNYCODE_TMAX : k - bias; + if (digit < t) break; + w *= PUNYCODE_BASE - t; + k += PUNYCODE_BASE; + } + const outLen = output.length + 1; + bias = Adapt(i - oldi, outLen, oldi === 0); + n += Math.floor(i / outLen); + i %= outLen; + output.splice(i, 0, n); + i++; + } + return String.fromCodePoint(...output); +} +// ------------------------------------------------------------------ +// DigitToChar +// ------------------------------------------------------------------ +function DigitToChar(digit: number): string { + return digit < 26 ? String.fromCharCode(digit + 0x61) : String.fromCharCode(digit - 26 + 0x30); +} +// ------------------------------------------------------------------ +// Encode +// +// Encodes a Unicode string into its Punycode payload (the part that +// follows the 'xn--' ACE prefix). Used to compute the true A-label +// length of a U-label, since the 63-octet label length limit (RFC +// 5891 §4.2.4) applies to the encoded form, not the code point count. +// ------------------------------------------------------------------ +const RE_REPLACE_NON_ASCII = new RegExp(Pattern.RE_NON_ASCII.source, "gu"); +export function Encode(input: string): string { + // Extract basic ASCII characters and set initial output array state + const basic = input.replace(RE_REPLACE_NON_ASCII, ""); + const basicLength = basic.length; + const result = basicLength > 0 ? [basic, "-"] : []; + // Extract unicode codepoints + const codePoints = Array.from(input, (char) => char.codePointAt(0)!); + let n = PUNYCODE_INITIAL_N; + let delta = 0; + let bias = PUNYCODE_INITIAL_BIAS; + let handledCPCount = basicLength; + while (handledCPCount < codePoints.length) { + let m = Infinity; + for (const cp of codePoints) { + if (cp >= n && cp < m) m = cp; + } + delta += (m - n) * (handledCPCount + 1); + n = m; + for (const cp of codePoints) { + if (cp < n) delta++; + if (cp === n) { + let q = delta; + for (let k = PUNYCODE_BASE; ; k += PUNYCODE_BASE) { + const t = k <= bias ? PUNYCODE_TMIN : k >= bias + PUNYCODE_TMAX ? PUNYCODE_TMAX : k - bias; + if (q < t) break; + const digit = t + ((q - t) % (PUNYCODE_BASE - t)); + result.push(DigitToChar(digit)); + q = Math.floor((q - t) / (PUNYCODE_BASE - t)); + } + result.push(DigitToChar(q)); + bias = Adapt(delta, handledCPCount + 1, handledCPCount === basicLength); + delta = 0; + handledCPCount++; + } + } + delta++; + n++; + } + return result.join(""); +} diff --git a/packages/schema/src/formats/typebox/idna/hostname.ts b/packages/schema/src/formats/typebox/idna/hostname.ts new file mode 100644 index 000000000..ade2322b4 --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/hostname.ts @@ -0,0 +1,42 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import * as LabelAscii from "./label/ascii.ts"; +import * as LabelPuny from "./label/puny.ts"; + +function IsValidLabelLength(value: string): boolean { + return value.length > 0 && value.length <= 63; +} +function IsLabel(value: string): boolean { + return IsValidLabelLength(value) && (LabelPuny.IsPunyLabel(value) || LabelAscii.IsAsciiLabel(value)); +} +export function IsHostname(value: string): boolean { + if (value.length === 0 || value.length > 253) return false; + if (value.charCodeAt(value.length - 1) === 46) return false; + return value.split(".").every((label) => IsLabel(label)); +} diff --git a/packages/schema/src/formats/typebox/idna/idn-hostname.ts b/packages/schema/src/formats/typebox/idna/idn-hostname.ts new file mode 100644 index 000000000..d4517faac --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/idn-hostname.ts @@ -0,0 +1,72 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import * as FormatBidi from "./format/bidi.ts"; +import * as LabelPuny from "./label/puny.ts"; +import * as LabelUnicode from "./label/unicode.ts"; + +// ------------------------------------------------------------------ +// IsLabel +// ------------------------------------------------------------------ +function IsValidLabelLength(value: string): boolean { + return value.length > 0 && value.length <= 63; +} +function IsLabel(value: string): boolean { + return IsValidLabelLength(value) && (LabelPuny.IsPunyLabel(value) || LabelUnicode.IsUnicodeLabel(value)); +} +// ------------------------------------------------------------------ +// NormalizeHostname +// +// Normalizes a hostname for label validation. Maps fullwidth +// codepoints to their ASCII equivalent (e.g. 'a' U+FF41 maps to +// 'a'), then normalizes for NFC such that equivalent codepoint +// sequences can be uniformly compared. It also removes codepoints +// that IDNA mapping ignores, and converts full stop variants to +// ASCII '.' so the hostname can be split into labels. +// ------------------------------------------------------------------ +function NormalizeHostname(value: string): string { + return ( + value + .replace(/[\uff01-\uff5e]/g, (char) => String.fromCharCode(char.charCodeAt(0) - 0xfee0)) // RE_FULLWIDTH_MAPPING + .normalize("NFC") + // biome-ignore lint/suspicious/noMisleadingCharacterClass: a set of single code points to remove (u flag), by design. + .replace(/[\u00ad\u034f\u180b-\u180d\u200b\ufe00-\ufe0f\u{e0100}-\u{e01ef}]/gu, "") // RE_IGNORED_MAPPING + .replace(/[\u002E\u3002\uFF0E\uFF61]/g, ".") + ); // RE_FULL_STOP_MAPPING +} +// ------------------------------------------------------------------ +// IsIdnHostname +// ------------------------------------------------------------------ +export function IsIdnHostname(value: string): boolean { + if (value.length === 0 || value.includes(" ")) return false; + const normalized = NormalizeHostname(value); + if (normalized.length > 253) return false; + const labels = normalized.split("."); + const hasBidiChars = labels.some((label) => FormatBidi.HasBidiChars(label)); + return labels.every((label) => IsLabel(label) && (!hasBidiChars || FormatBidi.SatisfiesBidiRule(label))); +} diff --git a/packages/schema/src/formats/typebox/idna/index.ts b/packages/schema/src/formats/typebox/idna/index.ts new file mode 100644 index 000000000..c5d10175e --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/index.ts @@ -0,0 +1,30 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +export * from "./hostname.ts"; +export * from "./idn-hostname.ts"; diff --git a/packages/schema/src/formats/typebox/idna/label/ascii.ts b/packages/schema/src/formats/typebox/idna/label/ascii.ts new file mode 100644 index 000000000..704e8f944 --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/label/ascii.ts @@ -0,0 +1,40 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import * as Pattern from "../pattern/pattern.ts"; + +// ------------------------------------------------------------------- +// Validates an ASCII LDH label per RFC 5891 §4.2.3.1 +// ------------------------------------------------------------------- +export function IsAsciiLabel(value: string): boolean { + return ( + Pattern.RE_RULE_HYPHEN_PLACEMENT.test(value) && + Pattern.RE_RULE_NOT_RESERVED_ACE.test(value) && + Pattern.RE_ASCII_LDH.test(value) + ); +} diff --git a/packages/schema/src/formats/typebox/idna/label/puny.ts b/packages/schema/src/formats/typebox/idna/label/puny.ts new file mode 100644 index 000000000..23b288ce2 --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/label/puny.ts @@ -0,0 +1,53 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import * as FormatPuny from "../format/puny.ts"; +import * as Pattern from "../pattern/pattern.ts"; +import * as LabelUnicode from "./unicode.ts"; + +// ------------------------------------------------------------------ +// IsPunyLabel +// ------------------------------------------------------------------ +export function IsPunyLabel(value: string): boolean { + if (!FormatPuny.IsAcePrefixed(value)) return false; + try { + const body = value.slice(4).toLowerCase(); + // A payload consisting of only a hyphen, or starting with the delimiter + // (like "-9uc"), has 0 basic ASCII characters preceding it and is malformed. + if (body.lastIndexOf("-") === 0) return false; + const decoded = FormatPuny.Decode(body); + // RFC 5890 §2.3.2.1 - a U-label must contain at least one non-ASCII + // character. Code-unit level check: any non-ASCII codepoint has at least + // one UTF-16 unit >= 0x80 (surrogate halves included), so no need to + // decode codepoints or allocate a character array to answer this. + if (!Pattern.RE_NON_ASCII.test(decoded)) return false; + return LabelUnicode.IsUnicodeLabel(decoded); + } catch { + return false; + } +} diff --git a/packages/schema/src/formats/typebox/idna/label/unicode.ts b/packages/schema/src/formats/typebox/idna/label/unicode.ts new file mode 100644 index 000000000..615942427 --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/label/unicode.ts @@ -0,0 +1,122 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import * as FormatBidi from "../format/bidi.ts"; +import * as FormatPuny from "../format/puny.ts"; +import * as Pattern from "../pattern/pattern.ts"; + +// ------------------------------------------------------------------ +// ExceedsMaxALabelLength +// +// RFC 5890 §2.3.2.1: an A-label (the ACE-encoded form of a label, +// "xn--" + punycode) must not exceed 63 octets. Only labels +// containing non-ASCII characters need checking - a pure-ASCII label +// has no "xn--" prefix and is used as-is, so its punycode-encoded +// length is irrelevant. The ASCII check is a cheap short-circuit that +// avoids running the punycode encoder on labels that don't need it. +// ------------------------------------------------------------------ +function ExceedsMaxALabelLength(value: string): boolean { + return Pattern.RE_NON_ASCII.test(value) && FormatPuny.Encode(value).length + 4 > 63; +} +// ------------------------------------------------------------------ +// HasInvalidHyphens +// +// RFC 5891 §4.2.3.2: label must not start or end with a hyphen, and +// must not have hyphens in the 3rd and 4th codepoint positions (the +// ACE prefix reservation, e.g. "xn--"). Operates on `chars` (the +// per-codepoint string array) rather than raw string indices so +// supplementary-plane characters earlier in the label don't throw +// off the codepoint-position count. +// ------------------------------------------------------------------ +function HasInvalidHyphens(chars: string[]): boolean { + if (chars[0] === "-" || chars[chars.length - 1] === "-") return true; + return chars.slice(2).join("").startsWith("--"); +} +// ------------------------------------------------------------------ +// IsUnicodeLabel +// ------------------------------------------------------------------ +export function IsUnicodeLabel(value: string): boolean { + // RFC 5890 §2.3.2.1: Ensure Max Length + if (ExceedsMaxALabelLength(value)) return false; + // RFC 5893 - Bidi Rule (applies whenever this label itself is RTL) + if (FormatBidi.HasRightToLeftCharacters(value) && !FormatBidi.SatisfiesBidiRule(value)) return false; + const chars = [...value]; + const codePoints = chars.map((c) => c.codePointAt(0)!); + const length = codePoints.length; + // RFC 5891 §4.2.3.2: Hyphen rules + if (HasInvalidHyphens(chars)) return false; + // RFC 5891 §4.2.3.2 - Must not begin with a combining mark + if (Pattern.RE_COMBINING_MARK.test(chars[0])) return false; + let hasJapanese = false; + // let hasArabicIndic = false // (no-spec-coverage) + // let hasExtendedArabicIndic = false // (no-spec-coverage) + for (let i = 0; i < length; i++) { + const codePoint = codePoints[i]; + const char = chars[i]; + // 1. DISALLOWED exceptions + if (Pattern.RE_RFC5892_DISALLOWED.test(char)) return false; + // 1b. General category (PVALID) enforcement + if (!Pattern.RE_PERMITTED_CATEGORY.test(char)) return false; + // 2. Collect Flags + if (Pattern.RE_SCRIPT_JAPANESE.test(char)) hasJapanese = true; + // if (Pattern.RE_ARABIC_INDIC_DIGIT.test(char)) hasArabicIndic = true // (no-spec-coverage) + // if (Pattern.RE_EXT_ARABIC_INDIC_DIGIT.test(char)) hasExtendedArabicIndic = true // (no-spec-coverage) + // 3. CONTEXTO / CONTEXTJ Neighbor Rules + const prev = codePoints[i - 1], + next = codePoints[i + 1]; + switch (codePoint) { + case 0x00b7: + if (prev !== 0x006c || next !== 0x006c) return false; + break; // MIDDLE DOT (Catalan) + case 0x0375: + if (!next || !Pattern.RE_SCRIPT_GREEK.test(chars[i + 1])) return false; + break; // Greek KERAIA + case 0x05f3: + case 0x05f4: + if (!prev || !Pattern.RE_SCRIPT_HEBREW.test(chars[i - 1])) return false; + break; // Hebrew GERESH + case 0x200c: // ZWNJ - RFC 5892 Appendix A.1 + // Note: Full validation requires a Unicode joining type table. We reject + // only when preceded by ASCII (U+0000-U+007F), which can never satisfy + // the Virama or cursive-joining rules. + if (!prev || (prev < 0x0080 && !Pattern.RE_VIRAMA.test(chars[i - 1]))) return false; + break; + case 0x200d: // ZWJ: RFC 5892 Appendix A.2 + if (!prev || !Pattern.RE_VIRAMA.test(chars[i - 1])) return false; + break; + case 0x30fb: // Checked at end via hasJapanese + break; // KATAKANA MIDDLE DOT + } + } + // 4. Global Context Validations (Post-loop) + // RFC 5892 Appendix A.7 - Katakana Middle Dot requirement + if (value.includes("\u30fb") && !hasJapanese) return false; + // RFC 5892 Appendix A.8/A.9 - Mixing Arabic Digits + // if (hasArabicIndic && hasExtendedArabicIndic) return false // (no-spec-coverage) + return true; +} diff --git a/packages/schema/src/formats/typebox/idna/pattern/pattern.ts b/packages/schema/src/formats/typebox/idna/pattern/pattern.ts new file mode 100644 index 000000000..dfb55c1d6 --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/pattern/pattern.ts @@ -0,0 +1,116 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +// deno-fmt-ignore-file + +// ------------------------------------------------------------------- +// Validation Rules +// ------------------------------------------------------------------- +export const RE_RULE_HYPHEN_PLACEMENT = /^(?!-).*(? regexp.source).join("|"), + "u", +); + +export const RE_PERMITTED_CATEGORY = new RegExp( + [ + RE_LETTER, + RE_EUROPEAN_SEPARATOR, + RE_COMMON_SEPARATOR, + RE_NUMBER_DECIMAL, + RE_MARK_NONSPACING, + RE_MARK_SPACING_COMBINING, + RE_CONTEXTO_EXCEPTIONS, + RE_PVALID_EXCEPTIONS, + ] + .map((regexp) => regexp.source) + .join("|"), + "u", +); diff --git a/packages/schema/src/formats/typebox/ipv4.ts b/packages/schema/src/formats/typebox/ipv4.ts new file mode 100644 index 000000000..2c6bf3c7b --- /dev/null +++ b/packages/schema/src/formats/typebox/ipv4.ts @@ -0,0 +1,37 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const IPv4 = /^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$/; + +/** + * Returns true if the value is a IPV4 address + * @specification http://tools.ietf.org/html/rfc2673#section-3.2 + */ +export function IsIPv4(value: string): boolean { + return IPv4.test(value); +} diff --git a/packages/schema/src/formats/typebox/ipv6.ts b/packages/schema/src/formats/typebox/ipv6.ts new file mode 100644 index 000000000..4cc46deac --- /dev/null +++ b/packages/schema/src/formats/typebox/ipv6.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const IPv6 = + /^(?:(?:(?:[0-9a-f]{1,4}:){6}|::(?:[0-9a-f]{1,4}:){5}|(?:[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){4}|(?:(?:[0-9a-f]{1,4}:)?[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){3}|(?:(?:[0-9a-f]{1,4}:){0,2}[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){2}|(?:(?:[0-9a-f]{1,4}:){0,3}[0-9a-f]{1,4})?::[0-9a-f]{1,4}:|(?:(?:[0-9a-f]{1,4}:){0,4}[0-9a-f]{1,4})?::)(?:[0-9a-f]{1,4}:[0-9a-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[0-9a-f]{1,4}:){0,5}[0-9a-f]{1,4})?::[0-9a-f]{1,4}|(?:(?:[0-9a-f]{1,4}:){0,6}[0-9a-f]{1,4})?::)$/i; + +/** + * Returns true if the value is an IPv6 address + * @specification http://tools.ietf.org/html/rfc2373#section-2.2 + */ +export function IsIPv6(value: string): boolean { + return IPv6.test(value); +} diff --git a/packages/schema/src/formats/typebox/iri.ts b/packages/schema/src/formats/typebox/iri.ts new file mode 100644 index 000000000..8ee1d0540 --- /dev/null +++ b/packages/schema/src/formats/typebox/iri.ts @@ -0,0 +1,63 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +// deno-lint-ignore-file no-control-regex + +const IpvFutureMatchMaxLength = 2048; +const IpvFutureMatch = /\[[vV][0-9a-fA-F]+\.[^\]]+\]/; // Guarded By IpvFutureMatchMaxLength +const InvalidIriChars = /[\x00-\x20<>^`{|}\\]/; +const InvalidPercentEncoding = /%(?![0-9a-fA-F]{2})/; + +// ------------------------------------------------------------------ +// NarrowIpvFuture +// +// Substitutes an IPvFuture address with a standard IPv6 loopback +// address ([::1]). We do this because the native URL.canParse +// (WHATWG standard) rejects IPvFuture literals, which are +// otherwise valid in RFC 3987. Because regex substitution can +// be expensive on large strings, this operation is strictly +// limited to inputs under a defined length threshold. +// +// (review-optimization) +// +// ------------------------------------------------------------------ +function NarrowIpvFuture(value: string): string { + return value.length < IpvFutureMatchMaxLength ? value.replace(IpvFutureMatch, "[::1]") : value; +} +/** + * Returns true if the value is a valid Internationalized Resource Identifier. + * @specification https://datatracker.ietf.org/doc/html/rfc3987 + */ +export function IsIri(value: string): boolean { + // 1. Reject strings containing unencoded whitespace or illegal control characters. + if (InvalidIriChars.test(value)) return false; + // 2. Reject malformed percent-encoding triplets. + if (InvalidPercentEncoding.test(value)) return false; + // 3. Delegate to the native URL parser, patching the IPvFuture edge case beforehand. + return URL.canParse(NarrowIpvFuture(value)); +} diff --git a/packages/schema/src/formats/typebox/iri_reference.ts b/packages/schema/src/formats/typebox/iri_reference.ts new file mode 100644 index 000000000..5e45c75ee --- /dev/null +++ b/packages/schema/src/formats/typebox/iri_reference.ts @@ -0,0 +1,42 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +// deno-lint-ignore-file no-control-regex + +// Rejects unencoded spaces (0x20), backslashes, control characters (0x00-0x1F, 0x7F), or malformed percent-encodings +const InvalidIriChars = /[\x00-\x20\x7F\\]|%(?![0-9a-fA-F]{2})/; +// Detects malformed scheme/authority patterns lacking a colon (e.g. "httpx//example.com") +const MalformedScheme = /^[a-zA-Z][a-zA-Z0-9+\-.]*\/\//; + +/** + * Returns true if the value is an IRI reference + * @specification https://tools.ietf.org/html/rfc3987 + */ +export function IsIriReference(value: string): boolean { + return !InvalidIriChars.test(value) && !MalformedScheme.test(value) && URL.canParse(value, "http://example.com"); +} diff --git a/packages/schema/src/formats/typebox/json_pointer.ts b/packages/schema/src/formats/typebox/json_pointer.ts new file mode 100644 index 000000000..c2d171aef --- /dev/null +++ b/packages/schema/src/formats/typebox/json_pointer.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const JsonPointer = /^(?:\/(?:[^~/]|~0|~1)*)*$/; + +/** + * Returns true if the value is a json pointer + * @specification https://datatracker.ietf.org/doc/html/rfc6901 + * @source https://github.com/ajv-validator/ajv-formats + */ +export function IsJsonPointer(value: string): boolean { + return JsonPointer.test(value); +} diff --git a/packages/schema/src/formats/typebox/regex.ts b/packages/schema/src/formats/typebox/regex.ts new file mode 100644 index 000000000..52cff8d2a --- /dev/null +++ b/packages/schema/src/formats/typebox/regex.ts @@ -0,0 +1,40 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +/** + * Returns true if the value is a regular expression string pattern + * @specification https://ecma-international.org/ecma-262 + */ +export function IsRegex(value: string): boolean { + try { + new RegExp(value, "u"); + return true; + } catch { + return false; + } +} diff --git a/packages/schema/src/formats/typebox/relative_json_pointer.ts b/packages/schema/src/formats/typebox/relative_json_pointer.ts new file mode 100644 index 000000000..f2dfd6fbd --- /dev/null +++ b/packages/schema/src/formats/typebox/relative_json_pointer.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const RelativeJsonPointer = /^(?:0|[1-9][0-9]*)(?:#|(?:\/(?:[^~/]|~0|~1)*)*)$/; + +/** + * Returns true if the value is a relative json pointer + * @specification https://datatracker.ietf.org/doc/html/rfc6901 + * @source https://github.com/ajv-validator/ajv-formats + */ +export function IsRelativeJsonPointer(value: string): boolean { + return RelativeJsonPointer.test(value); +} diff --git a/packages/schema/src/formats/typebox/time.ts b/packages/schema/src/formats/typebox/time.ts new file mode 100644 index 000000000..bbc9c6113 --- /dev/null +++ b/packages/schema/src/formats/typebox/time.ts @@ -0,0 +1,56 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const TIME = /^(\d\d):(\d\d):(\d\d)(?:\.\d+)?(?:([Zz])|([+-])(\d\d):(\d\d))?$/; + +/** + * Returns true if the value is an ISO time string + * @specification https://datatracker.ietf.org/doc/html/rfc3339 + */ +export function IsTime(value: string, strictTimeZone: boolean = true): boolean { + const matches = TIME.exec(value); + if (!matches) return false; + // Require timezone offset or 'Z'/'z' when strictTimeZone is true + if (strictTimeZone && !matches[4] && !matches[5]) return false; + const hr = +matches[1]; + const min = +matches[2]; + const sec = +matches[3]; + if (hr > 23 || min > 59 || sec > 60) return false; + if (matches[5]) { + const tzH = +matches[6]; + const tzM = +matches[7]; + if (tzH > 23 || tzM > 59) return false; + } + if (sec < 60) return true; + // Leap second handling: must normalize to 23:59:60 UTC (1439 total UTC minutes) + const tzSign = matches[5] === "-" ? -1 : 1; + const tzH = +(matches[6] || 0); + const tzM = +(matches[7] || 0); + const totalUtcMin = hr * 60 + min - tzSign * (tzH * 60 + tzM); + return ((totalUtcMin % 1440) + 1440) % 1440 === 1439; +} diff --git a/packages/schema/src/formats/typebox/uri.ts b/packages/schema/src/formats/typebox/uri.ts new file mode 100644 index 000000000..5aadeab17 --- /dev/null +++ b/packages/schema/src/formats/typebox/uri.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const Uri = + /^[a-z][a-z0-9+\-.]*:(?:\/\/(?:(?:[-a-z0-9._~!$&'()*+,;=:]|%[0-9a-f]{2})*@)?(?:\[(?:(?:(?:[\da-f]{1,4}:){6}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|::(?:[\da-f]{1,4}:){5}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:[\da-f]{1,4})?::(?:[\da-f]{1,4}:){4}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,1}[\da-f]{1,4})?::(?:[\da-f]{1,4}:){3}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,2}[\da-f]{1,4})?::(?:[\da-f]{1,4}:){2}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,3}[\da-f]{1,4})?::[\da-f]{1,4}:(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,4}[\da-f]{1,4})?::(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,5}[\da-f]{1,4})?::[\da-f]{1,4}|(?:(?:[\da-f]{1,4}:){0,6}[\da-f]{1,4})?::)|v[0-9a-f]+\.[-a-z0-9._~!$&'()*+,;=:]+)\]|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)|(?:[-a-z0-9._~!$&'()*+,;=]|%[0-9a-f]{2})*)(?::\d*)?(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*|\/(?:(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?|(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?(?:\?(?:[-a-z0-9._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?(?:#(?:[-a-z0-9._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?$/i; + +/** + * Returns true if the value is a valid Uniform Resource Identifier. + * @specification https://tools.ietf.org/html/rfc3986 + */ +export function IsUri(value: string): boolean { + return Uri.test(value); +} diff --git a/packages/schema/src/formats/typebox/uri_reference.ts b/packages/schema/src/formats/typebox/uri_reference.ts new file mode 100644 index 000000000..215058ded --- /dev/null +++ b/packages/schema/src/formats/typebox/uri_reference.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const UriReference = + /^(?:[a-z][a-z0-9+\-.]*:(?:\/\/(?:(?:[-a-z0-9._~!$&'()*+,;=:]|%[0-9a-f]{2})*@)?(?:\[(?:(?:(?:[\da-f]{1,4}:){6}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|::(?:[\da-f]{1,4}:){5}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:[\da-f]{1,4})?::(?:[\da-f]{1,4}:){4}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,1}[\da-f]{1,4})?::(?:[\da-f]{1,4}:){3}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,2}[\da-f]{1,4})?::(?:[\da-f]{1,4}:){2}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,3}[\da-f]{1,4})?::[\da-f]{1,4}:(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,4}[\da-f]{1,4})?::(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,5}[\da-f]{1,4})?::[\da-f]{1,4}|(?:(?:[\da-f]{1,4}:){0,6}[\da-f]{1,4})?::)|v[0-9a-f]+\.[-a-z0-9._~!$&'()*+,;=:]+)\]|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)|(?:[-a-z0-9._~!$&'()*+,;=]|%[0-9a-f]{2})*)(?::\d*)?(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*|\/(?:(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?|(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?|(?:\/\/(?:(?:[-a-z0-9._~!$&'()*+,;=:]|%[0-9a-f]{2})*@)?(?:\[(?:(?:(?:[\da-f]{1,4}:){6}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|::(?:[\da-f]{1,4}:){5}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:[\da-f]{1,4})?::(?:[\da-f]{1,4}:){4}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,1}[\da-f]{1,4})?::(?:[\da-f]{1,4}:){3}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,2}[\da-f]{1,4})?::(?:[\da-f]{1,4}:){2}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,3}[\da-f]{1,4})?::[\da-f]{1,4}:(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,4}[\da-f]{1,4})?::(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,5}[\da-f]{1,4})?::[\da-f]{1,4}|(?:(?:[\da-f]{1,4}:){0,6}[\da-f]{1,4})?::)|v[0-9a-f]+\.[-a-z0-9._~!$&'()*+,;=:]+)\]|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)|(?:[-a-z0-9._~!$&'()*+,;=]|%[0-9a-f]{2})*)(?::\d*)?(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*|\/(?:(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?|(?:[-a-z0-9._~!$&'()*+,;=@]|%[0-9a-f]{2})+(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?)(?:\?(?:[-a-z0-9._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?(?:#(?:[-a-z0-9._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?$/i; + +/** + * Returns true if the value is a valid Uri Reference. + * @specification https://tools.ietf.org/html/rfc3986 + */ +export function IsUriReference(value: string): boolean { + return UriReference.test(value); +} diff --git a/packages/schema/src/formats/typebox/uri_template.ts b/packages/schema/src/formats/typebox/uri_template.ts new file mode 100644 index 000000000..ca364fe15 --- /dev/null +++ b/packages/schema/src/formats/typebox/uri_template.ts @@ -0,0 +1,40 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +// deno-lint-ignore-file no-control-regex + +const UriTemplate = + /^(?:(?:[^\x00-\x20"<>%\\^`{|}\x7f]|%[0-9a-f]{2})|\{[+#./;?&=,!@|]?(?:[a-z0-9_]|%[0-9a-f]{2})+(?:\.(?:[a-z0-9_]|%[0-9a-f]{2})+)*(?::[1-9]\d{0,3}|\*)?(?:,(?:[a-z0-9_]|%[0-9a-f]{2})+(?:\.(?:[a-z0-9_]|%[0-9a-f]{2})+)*(?::[1-9]\d{0,3}|\*)?)*\})*$/i; + +/** + * Returns true if the value is a valid Uri Template + * @specification https://datatracker.ietf.org/doc/html/rfc6570 + */ +export function IsUriTemplate(value: string): boolean { + return UriTemplate.test(value); +} diff --git a/packages/schema/src/formats/typebox/uuid.ts b/packages/schema/src/formats/typebox/uuid.ts new file mode 100644 index 000000000..cf26ffa6e --- /dev/null +++ b/packages/schema/src/formats/typebox/uuid.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const Uuid = /^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/i; + +/** + * Returns true if the value is a uuid + * @specification https://www.rfc-editor.org/info/rfc4122/ + * @specification https://www.rfc-editor.org/info/rfc9562/ + */ +export function IsUuid(value: string): boolean { + return Uuid.test(value); +} diff --git a/packages/schema/src/index.ts b/packages/schema/src/index.ts new file mode 100644 index 000000000..c421d6fdb --- /dev/null +++ b/packages/schema/src/index.ts @@ -0,0 +1,31 @@ +// JSON Schema validation (drafts 2020-12 and 7) without eval: compile() builds a schema into plain closures once. +// The builders return plain JSON Schema objects typed with the values they accept. + +export { + type Annotations, + arr, + bool, + bytes, + int, + type LengthBounds, + lit, + map, + type NumberBounds, + nul, + nullable, + num, + Optional, + obj, + oneOf, + opt, + type Properties, + type Schema, + type Static, + type StaticObject, + str, + tuple, + union, + unknown, +} from "./builders.ts"; +export type { Check, CompileOptions, FormatCheck, JsonSchema, ValidationError } from "./types.ts"; +export { compile, Validator } from "./validator.ts"; diff --git a/packages/schema/src/resolver.ts b/packages/schema/src/resolver.ts new file mode 100644 index 000000000..a10d245b6 --- /dev/null +++ b/packages/schema/src/resolver.ts @@ -0,0 +1,152 @@ +// Reference resolution of one compile: schema resources, anchors and JSON Pointers over the root schema and the +// registered documents. A Resolver lives only while a schema compiles (or while its errors are explained). + +import type { CompileOptions, JsonSchema, SchemaObject } from "./types.ts"; + +/** A schema and the base URI its references resolve against. */ +export interface Resolved { + readonly node: unknown; + readonly base: string; +} + +/** The base URI of a root schema without `$id`; relative references resolve against it. */ +export const DEFAULT_BASE = "https://json-schema.invalid/root.json"; + +/** Indexes the root schema and resolves references in it and in the registered documents. */ +export class Resolver { + readonly draft: "2020-12" | "7"; + readonly rootBase: string; + /** Schema object to the base URI it was indexed under. */ + readonly baseOf = new Map(); + /** URI (with a `#name` fragment) of every `$dynamicAnchor`, to its schema. */ + readonly dynamicAnchors = new Map(); + private readonly registry = new Map(); + private readonly resources = new Map(); + private readonly anchors = new Map(); + + constructor(root: JsonSchema, options: CompileOptions) { + this.draft = options.draft === "7" ? "7" : "2020-12"; + for (const [uri, schema] of Object.entries(options.schemas ?? {})) { + this.registry.set(documentOf(resolveUri(uri, DEFAULT_BASE)), schema); + } + this.resources.set(DEFAULT_BASE, root); + this.index(root, DEFAULT_BASE); + this.rootBase = (isSchemaObject(root) && this.baseOf.get(root)) || DEFAULT_BASE; + } + + /** The schema a reference names, with the base URI it compiles under. Throws if it names nothing. */ + resolve(ref: string, base: string): Resolved { + const uri = resolveUri(ref, base); + const doc = documentOf(uri); + const fragment = uri.length > doc.length ? decodeURIComponent(uri.slice(doc.length + 1)) : ""; + const root = this.resources.has(doc) ? this.resources.get(doc) : this.load(doc); + if (root === undefined) throw new Error(`Unresolved $ref ${ref} (no schema registered for ${doc})`); + if (fragment === "") return { node: root, base: doc }; + if (fragment.startsWith("/")) return this.pointer(root, doc, fragment); + const target = this.anchors.get(`${doc}#${fragment}`); + if (target === undefined) throw new Error(`Unresolved $ref ${ref} (no anchor ${fragment} in ${doc})`); + return { node: target, base: this.baseOf.get(target) ?? doc }; + } + + /** Whether `node` is the root of the schema resource at `base`. */ + isResource(node: object, base: string): boolean { + return this.resources.get(base) === node; + } + + // Walks only keywords that hold subschemas: an `$id` inside `enum`, `const` or an unknown keyword names nothing. + private index(node: unknown, base: string): void { + if (!isSchemaObject(node) || this.baseOf.has(node)) return; + let b = base; + // In draft 7 a `$ref` makes every sibling keyword, `$id` included, inert. + const refOnly = this.draft === "7" && typeof node.$ref === "string"; + if (!refOnly && typeof node.$id === "string") { + if (this.draft === "7" && node.$id.startsWith("#")) this.anchors.set(documentOf(b) + node.$id, node); + else { + const uri = resolveUri(node.$id, b); + b = documentOf(uri); + this.resources.set(b, node); + if (uri.length > b.length + 1) this.anchors.set(uri, node); + } + } + this.baseOf.set(node, b); + if (refOnly) return; + if (this.draft !== "7") { + if (typeof node.$anchor === "string") this.anchors.set(`${b}#${node.$anchor}`, node); + if (typeof node.$dynamicAnchor === "string") { + this.anchors.set(`${b}#${node.$dynamicAnchor}`, node); + this.dynamicAnchors.set(`${b}#${node.$dynamicAnchor}`, node); + } + } + for (const key of SCHEMA_KEYWORDS) this.index(node[key], b); + for (const key of MAP_KEYWORDS) { + const map = node[key]; + if (isSchemaObject(map)) for (const k of Object.keys(map)) this.index(map[k], b); + } + for (const key of ARRAY_KEYWORDS) { + const list = node[key]; + if (Array.isArray(list)) for (const s of list) this.index(s, b); + } + } + + private load(doc: string): JsonSchema | undefined { + const schema = this.registry.get(doc); + if (schema === undefined) return undefined; + this.resources.set(doc, schema); + this.index(schema, doc); + return schema; + } + + private pointer(root: unknown, base: string, fragment: string): Resolved { + let node = root; + let b = base; + for (const raw of fragment.split("/").slice(1)) { + const key = raw.replace(/~1/g, "/").replace(/~0/g, "~"); + node = + typeof node === "object" && node !== null && Object.hasOwn(node, key) + ? (node as SchemaObject)[key] + : undefined; + if (node === undefined) throw new Error(`Unresolved $ref #${fragment} in ${base}`); + if (typeof node === "object" && node !== null) b = this.baseOf.get(node) ?? b; + } + return { node, base: b }; + } +} + +const SCHEMA_KEYWORDS = [ + "additionalProperties", + "propertyNames", + "contains", + "not", + "if", + "then", + "else", + "unevaluatedProperties", + "unevaluatedItems", + "additionalItems", + "contentSchema", + "items", +]; + +const MAP_KEYWORDS = ["properties", "patternProperties", "$defs", "definitions", "dependentSchemas", "dependencies"]; + +const ARRAY_KEYWORDS = ["allOf", "anyOf", "oneOf", "prefixItems", "items"]; + +/** Whether `value` is a schema object (not `true`/`false`, an array or a primitive). */ +export function isSchemaObject(value: unknown): value is SchemaObject { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** `ref` resolved against `base`. URNs and other bases a URL parser rejects take fragment-only references. */ +export function resolveUri(ref: string, base: string): string { + try { + return new URL(ref, base).href; + } catch { + return ref.startsWith("#") ? documentOf(base) + ref : ref; + } +} + +/** `uri` without its fragment. */ +export function documentOf(uri: string): string { + const hash = uri.indexOf("#"); + return hash < 0 ? uri : uri.slice(0, hash); +} diff --git a/packages/schema/src/shared.ts b/packages/schema/src/shared.ts new file mode 100644 index 000000000..bb3867d3d --- /dev/null +++ b/packages/schema/src/shared.ts @@ -0,0 +1,50 @@ +// Values shared across all compiled schemas while one still uses them: leaf checks and regular expressions. +// Both caches hold their values weakly and drop an entry once its value is collected, so they keep nothing alive. + +import type { Check } from "./types.ts"; + +/** The check shared under `key`, made by `make` when no live schema holds one. */ +export function sharedCheck(key: string, make: () => Check): Check { + const hit = checks.get(key)?.deref(); + if (hit !== undefined) return hit; + const check = make(); + checks.set(key, new WeakRef(check)); + checkCleanup.register(check, key); + return check; +} + +const checks = new Map>(); + +const checkCleanup = new FinalizationRegistry((key) => { + if (checks.get(key)?.deref() === undefined) checks.delete(key); +}); + +/** + * The regular expression of a `pattern` source, shared by source. Patterns are ECMAScript regular expressions with + * the `u` flag; a source that is invalid with it (some older schemas escape characters `u` rejects) compiles + * without it. + */ +export function sharedRegex(source: string): RegExp { + const hit = regexes.get(source)?.deref(); + if (hit !== undefined) return hit; + let re: RegExp; + try { + re = new RegExp(source, "u"); + } catch { + re = new RegExp(source); + } + regexes.set(source, new WeakRef(re)); + regexCleanup.register(re, source); + return re; +} + +const regexes = new Map>(); + +const regexCleanup = new FinalizationRegistry((source) => { + if (regexes.get(source)?.deref() === undefined) regexes.delete(source); +}); + +/** The number of entries in both caches (for tests). */ +export function sharedEntries(): { checks: number; regexes: number } { + return { checks: checks.size, regexes: regexes.size }; +} diff --git a/packages/schema/src/types.ts b/packages/schema/src/types.ts new file mode 100644 index 000000000..a9e2135d1 --- /dev/null +++ b/packages/schema/src/types.ts @@ -0,0 +1,36 @@ +// Public types: the schema accepted by compile(), its options, and the errors a failed value produces. + +/** + * A JSON Schema: `true`, `false` or a schema object. Schemas come from tool definitions, MCP servers and other + * untrusted sources, so compile() reads every keyword defensively instead of trusting a static shape. + */ +export type JsonSchema = boolean | object; + +/** A format check: whether `value` (a string) has the format. */ +export type FormatCheck = (value: string) => boolean; + +/** Options of compile(). */ +export interface CompileOptions { + /** The draft whose semantics apply: "2020-12" (default) or "7". */ + readonly draft?: "2020-12" | "7"; + /** + * Format name to check. Without it `format` is an annotation, as draft 2020-12 specifies. The standard formats are + * in `@earendil-works/pi-schema/formats`. + */ + readonly formats?: Readonly>; + /** Schema documents by URI, for `$ref`s to other documents. */ + readonly schemas?: Readonly>; +} + +/** One failure of a value: where it is (a JSON Pointer, "" for the value itself), the failing keyword and a message. */ +export interface ValidationError { + readonly path: string; + readonly keyword: string; + readonly message: string; +} + +/** A schema object as compile() reads it: any keyword, of any value, checked where it is used. */ +export type SchemaObject = { readonly [keyword: string]: unknown }; + +/** A compiled boolean check. */ +export type Check = (value: unknown) => boolean; diff --git a/packages/schema/src/validator.ts b/packages/schema/src/validator.ts new file mode 100644 index 000000000..3d915b33b --- /dev/null +++ b/packages/schema/src/validator.ts @@ -0,0 +1,58 @@ +// compile(): a schema to a Validator, whose `check` is the fast boolean path and whose `errors` explains failures. + +import type { Static } from "./builders.ts"; +import { Compiler, RESTART } from "./compiler.ts"; +import { Explainer } from "./explain.ts"; +import type { Check, CompileOptions, JsonSchema, ValidationError } from "./types.ts"; + +/** + * Compiles `schema` into a Validator. Throws if the schema is malformed or a `$ref` names nothing (register other + * documents with `options.schemas`). The schema is read, never modified. + */ +export function compile(schema: S, options: CompileOptions = NO_OPTIONS): Validator> { + return new Validator(schema, options, compileCheck(schema, options)); +} + +const NO_OPTIONS: CompileOptions = Object.freeze({}); + +/** A compiled schema whose valid values have type `T` (`unknown` unless the schema was built with the builders). */ +export class Validator { + /** Whether `value` is valid. A plain function: it may be passed around without its validator. */ + readonly check: (value: unknown) => value is T; + private readonly schema: JsonSchema; + private readonly options: CompileOptions; + private explainer: Explainer | undefined = undefined; + + constructor(schema: JsonSchema, options: CompileOptions, check: Check) { + // A compiled check is exactly the test that a value has the schema's static type. + this.check = check as (value: unknown) => value is T; + this.schema = schema; + this.options = options; + } + + /** + * Why `value` is invalid: one error per failing keyword, `[]` when it is valid. Built on the first failure and + * kept, so a validator whose values always pass never pays for it. + */ + errors(value: unknown): ValidationError[] { + if (this.check(value)) return []; + this.explainer ??= new Explainer(compiler(this.schema, this.options), this.schema); + return this.explainer.errors(value); + } +} + +function compileCheck(schema: JsonSchema, options: CompileOptions): Check { + return compiler(schema, options).root(); +} + +// Most schemas compile without tracking the dynamic scope; one with a bookended $dynamicRef compiles again with it. +function compiler(schema: JsonSchema, options: CompileOptions): Compiler { + const plain = new Compiler(schema, options, false); + try { + plain.root(); + return plain; + } catch (error) { + if (error !== RESTART) throw error; + return new Compiler(schema, options, true); + } +} diff --git a/packages/schema/test/bench/all.ts b/packages/schema/test/bench/all.ts new file mode 100644 index 000000000..bb8421fc2 --- /dev/null +++ b/packages/schema/test/bench/all.ts @@ -0,0 +1,36 @@ +// Runs every benchmark, each in its own process, and writes the results as JSON next to the printed tables. +// Speed runs twice: normally and with code generation from strings disabled, as under a strict CSP. +// Run: npm run bench [-- --out ] (default: pi-schema-bench-results.json in the temp directory) + +import { execFileSync } from "node:child_process"; +import { readFileSync, rmSync, writeFileSync } from "node:fs"; +import { cpus, tmpdir } from "node:os"; +import { join } from "node:path"; + +const dir = import.meta.dirname; +const outIndex = process.argv.indexOf("--out"); +const out = outIndex >= 0 ? process.argv[outIndex + 1] : join(tmpdir(), "pi-schema-bench-results.json"); + +const runs: [string, string, string[]][] = [ + ["conformance", "conformance.ts", []], + ["speed", "speed.ts", []], + ["speedNoEval", "speed.ts", ["--disallow-code-generation-from-strings"]], + ["setup", "setup.ts", []], + ["memory", "memory.ts", []], + ["size", "size.ts", []], +]; + +const results: Record = { + node: process.version, + cpu: cpus()[0]?.model ?? "unknown", + date: new Date().toISOString(), +}; +for (const [key, file, flags] of runs) { + const json = join(tmpdir(), `pi-schema-bench-${key}.json`); + console.log(`\n## ${key}${flags.length > 0 ? ` (${flags.join(" ")})` : ""}\n`); + execFileSync(process.execPath, [...flags, join(dir, file), "--json", json], { stdio: "inherit" }); + results[key] = JSON.parse(readFileSync(json, "utf8")); + rmSync(json); +} +writeFileSync(out, JSON.stringify(results, null, "\t")); +console.log(`\nResults: ${out}`); diff --git a/packages/schema/test/bench/candidates.ts b/packages/schema/test/bench/candidates.ts new file mode 100644 index 000000000..8b231c855 --- /dev/null +++ b/packages/schema/test/bench/candidates.ts @@ -0,0 +1,101 @@ +// The validators the benchmarks compare, each behind the same interface: compile a schema, then check values. +// Every library is used through its fastest boolean path and keeps the object it returns, as an application would. + +import { Validator as CfValidator } from "@cfworker/json-schema"; +import { fromJsonSchema } from "@oh-my-pi/omptype"; +import { Compile } from "typebox/compile"; +import { ResetExternal, ResetFunctions } from "typebox/schema"; +import { Settings } from "typebox/system"; +import { compile, type FormatCheck, type JsonSchema } from "../../src/index.ts"; + +/** How a benchmark compiles a schema. */ +export interface PrepareOptions { + readonly draft: "2020-12" | "7"; + readonly schemas?: Readonly>; + readonly formats?: Readonly>; +} + +/** A compiled schema: its boolean check and the object the library returned (kept alive by the caller). */ +export interface Prepared { + readonly check: (value: unknown) => boolean; + readonly validator: unknown; +} + +/** A library under comparison. */ +export interface Candidate { + readonly name: string; + readonly prepare: (schema: JsonSchema, options: PrepareOptions) => Prepared; +} + +/** Whether this process runs with code generation from strings disabled, as under a strict CSP. */ +export const noEval = process.execArgv.includes("--disallow-code-generation-from-strings"); + +const piSchema: Candidate = { + name: "pi-schema", + prepare(schema, options) { + const validator = compile(schema, options); + return { check: validator.check, validator }; + }, +}; + +// TypeBox keeps its last build in module-level accumulators; reset them so no build state is retained between schemas. +function typeboxCompile(schema: JsonSchema, options: PrepareOptions, accelerate: boolean) { + Settings.Set({ useAcceleration: accelerate }); + try { + return Compile((options.schemas ?? {}) as never, schema as never); + } finally { + ResetFunctions(); + ResetExternal(); + Settings.Set({ useAcceleration: true }); + } +} + +const typebox: Candidate = { + name: noEval ? "typebox (no eval: interpreter)" : "typebox", + prepare(schema, options) { + const validator = typeboxCompile(schema, options, true); + return { check: (v) => validator.Check(v), validator }; + }, +}; + +const typeboxInterpreter: Candidate = { + name: "typebox interpreter", + prepare(schema, options) { + const validator = typeboxCompile(schema, options, false); + return { check: (v) => validator.Check(v), validator }; + }, +}; + +const cfworker: Candidate = { + name: "cfworker", + prepare(schema, options) { + const validator = new CfValidator(schema as never, options.draft, true); + for (const [uri, remote] of Object.entries(options.schemas ?? {})) validator.addSchema(remote as never, uri); + return { check: (v) => validator.validate(v).valid, validator }; + }, +}; + +// omptype compiles with new Function on the first allows() call. Without eval that call throws EvalError once per +// schema; omptype then keeps validating through its interpreter, so the benchmark absorbs that one throw. +const omptype: Candidate = { + name: noEval ? "omptype (no eval: interpreter)" : "omptype", + prepare(schema) { + const validator = fromJsonSchema(schema); + if (noEval) { + try { + validator.allows(null); + } catch (error) { + if (!(error instanceof EvalError)) throw error; + } + } + return { check: (v) => validator.allows(v), validator }; + }, +}; + +/** Every candidate, ours first. */ +export const candidates: readonly Candidate[] = [piSchema, typebox, cfworker, omptype]; + +/** Candidates by name, including TypeBox's interpreter (what TypeBox runs when eval is unavailable). */ +export const byName: ReadonlyMap = new Map( + [...candidates, typeboxInterpreter].map((c) => [c.name.split(" (")[0], c]), +); diff --git a/packages/schema/test/bench/cases.ts b/packages/schema/test/bench/cases.ts new file mode 100644 index 000000000..d6821df1e --- /dev/null +++ b/packages/schema/test/bench/cases.ts @@ -0,0 +1,261 @@ +// Speed cases: realistic schemas with a valid and an invalid value each. Argument checks shaped like Chord's +// contracts (closed objects, tagged unions), pi's built-in tool schemas, and MCP tool schemas with $ref and anyOf. + +import type { JsonSchema } from "../../src/index.ts"; + +/** A schema, the draft it is written in, and one valid and one invalid value. */ +export interface Case { + readonly name: string; + readonly schema: JsonSchema; + readonly valid: unknown; + readonly invalid: unknown; +} + +type Props = Record; +const closed = (properties: Props, optional: string[] = []) => ({ + type: "object", + properties, + required: Object.keys(properties).filter((k) => !optional.includes(k)), + additionalProperties: false, +}); +const str = (o: object = {}) => ({ type: "string", ...o }); +const int = (o: object = {}) => ({ type: "integer", ...o }); +const num = (o: object = {}) => ({ type: "number", ...o }); +const lit = (v: string) => ({ const: v }); +const arr = (items: object, o: object = {}) => ({ type: "array", items, ...o }); +const uuid = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; + +const block = { + anyOf: [ + closed({ kind: lit("text"), text: str() }), + closed({ kind: lit("image"), src: str(), width: int({ minimum: 0 }), height: int({ minimum: 0 }), alt: str() }, [ + "alt", + ]), + closed({ kind: lit("code"), lang: { anyOf: [lit("ts"), lit("js"), lit("py")] }, source: str() }), + ], +}; + +const document = closed( + { + id: str({ pattern: uuid }), + title: str({ minLength: 1, maxLength: 200 }), + tags: arr(str(), { maxItems: 16 }), + author: closed({ name: str(), email: str() }, ["email"]), + blocks: arr(block), + score: num({ minimum: 0, maximum: 1 }), + draft: { type: "boolean" }, + }, + ["score", "draft"], +); + +const documentValue = { + id: "01890a5d-ac96-774b-bcce-b302099a8057", + title: "Quarterly notes", + tags: ["a", "b", "c"], + author: { name: "Mario", email: "m@example.com" }, + blocks: [ + { kind: "text", text: "Hello" }, + { kind: "image", src: "a.png", width: 640, height: 480 }, + { kind: "code", lang: "ts", source: "const x = 1;" }, + { kind: "text", text: "World" }, + { kind: "image", src: "b.png", width: 10, height: 20, alt: "b" }, + ], + score: 0.5, +}; + +const id = int({ minimum: 0 }); +const message = { + anyOf: [ + closed({ + kind: lit("hello"), + protocol: int({ minimum: 0 }), + incarnation: str({ pattern: uuid }), + calls: int({ minimum: 1 }), + }), + closed( + { + kind: lit("hold"), + hold: id, + type: str(), + version: int({ minimum: 1 }), + fingerprint: str({ pattern: "^[0-9a-f]{16}$" }), + params: {}, + address: closed({ kind: str(), params: {} }), + chain: arr(str({ pattern: uuid }), { maxItems: 64 }), + }, + ["chain"], + ), + closed({ kind: lit("release"), hold: id }), + closed({ kind: lit("call"), call: id, hold: id, method: id, args: {} }, ["args"]), + closed({ kind: lit("abort"), call: id }), + ], +}; + +const tool = (properties: Props, required: string[]) => ({ type: "object", properties, required }); + +const mcpIssue = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + owner: str({ minLength: 1, pattern: "^[A-Za-z0-9-]+$" }), + repo: str({ minLength: 1, pattern: "^[A-Za-z0-9._-]+$" }), + title: str({ minLength: 1 }), + body: str(), + labels: arr(str()), + assignees: arr(str()), + milestone: { anyOf: [{ type: "number" }, { type: "null" }] }, + state: { type: "string", enum: ["open", "closed"] }, + }, + required: ["owner", "repo", "title"], + additionalProperties: false, +}; + +const mcpEdit = { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + $defs: { + Range: closed({ start: int({ minimum: 1 }), end: int({ minimum: 1 }) }), + Replace: closed({ oldText: str(), newText: str() }), + LineEdit: closed({ range: { $ref: "#/$defs/Range" }, text: str() }), + }, + properties: { + path: str({ pattern: "^(/|\\./|[A-Za-z]:\\\\)?[^\\u0000]+$" }), + edits: arr({ anyOf: [{ $ref: "#/$defs/Replace" }, { $ref: "#/$defs/LineEdit" }] }, { minItems: 1 }), + dryRun: { type: "boolean" }, + }, + required: ["path", "edits"], + additionalProperties: false, +}; + +const mcpPage = { + type: "object", + $defs: { + RichText: closed( + { + type: { enum: ["text", "mention"] }, + content: str({ maxLength: 2000 }), + annotations: closed({ bold: { type: "boolean" }, italic: { type: "boolean" }, color: str() }, [ + "bold", + "italic", + "color", + ]), + }, + ["annotations"], + ), + Block: { + type: "object", + properties: { + type: { enum: ["paragraph", "heading_1", "heading_2", "bulleted_list_item", "to_do"] }, + rich_text: arr({ $ref: "#/$defs/RichText" }), + checked: { type: "boolean" }, + children: arr({ $ref: "#/$defs/Block" }), + }, + required: ["type", "rich_text"], + additionalProperties: false, + }, + }, + properties: { + parent: { oneOf: [closed({ page_id: str({ pattern: uuid }) }), closed({ database_id: str({ pattern: uuid }) })] }, + title: str({ minLength: 1 }), + due: str({ pattern: "^\\d{4}-\\d{2}-\\d{2}$" }), + children: arr({ $ref: "#/$defs/Block" }, { maxItems: 100 }), + }, + required: ["parent", "title"], + additionalProperties: false, +}; + +const rt = (content: string) => ({ + type: "text", + content, + annotations: { bold: false, italic: false, color: "default" }, +}); +const pageValue = { + parent: { page_id: "01890a5d-ac96-774b-bcce-b302099a8057" }, + title: "Plan", + due: "2026-11-01", + children: [ + { type: "heading_1", rich_text: [rt("Plan")] }, + { + type: "bulleted_list_item", + rich_text: [rt("one")], + children: [{ type: "to_do", rich_text: [rt("sub")], checked: true }], + }, + { type: "paragraph", rich_text: [rt("a"), rt("b"), rt("c")] }, + ], +}; +const pageInvalid = structuredClone(pageValue); +pageInvalid.children[1].children = [{ type: "to_do", rich_text: [rt("sub")], checked: "yes" as unknown as boolean }]; + +/** The speed cases. */ +export const cases: readonly Case[] = [ + { name: "small closed object", schema: closed({ text: str() }), valid: { text: "hello" }, invalid: { text: 1 } }, + { + name: "nested object with tagged unions", + schema: document, + valid: documentValue, + invalid: { + ...documentValue, + blocks: [...documentValue.blocks.slice(0, 4), { kind: "image", src: "b.png", width: -1, height: 20 }], + }, + }, + { + name: "message union of 5", + schema: message, + valid: { kind: "call", call: 7, hold: 3, method: 1, args: { a: 1 } }, + invalid: { kind: "call", call: 7, hold: 3, method: 1, extra: true }, + }, + { + name: "tool: read", + schema: tool({ path: str({ description: "Path to the file" }), offset: num(), limit: num() }, ["path"]), + valid: { path: "src/a.ts", offset: 10, limit: 50 }, + invalid: { offset: 1 }, + }, + { + name: "tool: edit", + schema: tool({ path: str(), edits: arr(tool({ oldText: str(), newText: str() }, ["oldText", "newText"])) }, [ + "path", + "edits", + ]), + valid: { + path: "a.ts", + edits: [ + { oldText: "a", newText: "b" }, + { oldText: "c", newText: "d" }, + ], + }, + invalid: { path: "a.ts", edits: [{ oldText: "a" }] }, + }, + { + name: "tool: bash", + schema: tool({ command: str(), timeout: num() }, ["command"]), + valid: { command: "ls -la", timeout: 30 }, + invalid: { command: 5 }, + }, + { + name: "MCP create_issue (pattern, anyOf, enum)", + schema: mcpIssue, + valid: { + owner: "earendil-works", + repo: "pi", + title: "Bug", + body: "...", + labels: ["bug"], + milestone: null, + state: "open", + }, + invalid: { owner: "earendil works", repo: "pi", title: "Bug" }, + }, + { + name: "MCP edit_file ($ref, anyOf)", + schema: mcpEdit, + valid: { + path: "./src/a.ts", + edits: [ + { oldText: "a", newText: "b" }, + { range: { start: 1, end: 3 }, text: "x" }, + ], + }, + invalid: { path: "./src/a.ts", edits: [{ range: { start: 0, end: 3 }, text: "x" }] }, + }, + { name: "MCP create_page (recursive $ref, oneOf)", schema: mcpPage, valid: pageValue, invalid: pageInvalid }, +]; diff --git a/packages/schema/test/bench/conformance.ts b/packages/schema/test/bench/conformance.ts new file mode 100644 index 000000000..d0cc72a7e --- /dev/null +++ b/packages/schema/test/bench/conformance.ts @@ -0,0 +1,54 @@ +// Conformance of every candidate on the vendored JSON-Schema-Test-Suite (drafts 2020-12 and 7), by section: +// core, optional and optional/format. Every candidate gets the remote documents and meta-schemas; format tests run +// with pi-schema's format checks (the other libraries check formats on their own or not at all). +// Run: node test/bench/conformance.ts [--json ] + +import { writeFileSync } from "node:fs"; +import { formats } from "../../src/formats/index.ts"; +import { suiteFiles, suiteSchemas } from "../suite.ts"; +import { candidates } from "./candidates.ts"; + +/** Passed and total tests by section ("draft2020-12 core", ...) and candidate name. */ +export type ConformanceResults = Record }>; + +const results: ConformanceResults = {}; +for (const file of suiteFiles()) { + const kind = file.name.includes("/optional/format/") + ? "format" + : file.name.includes("/optional/") + ? "optional" + : "core"; + const section = `${file.name.split("/")[0]} ${kind}`; + results[section] ??= { total: 0, passed: Object.fromEntries(candidates.map((c) => [c.name, 0])) }; + for (const group of file.groups) { + results[section].total += group.tests.length; + for (const candidate of candidates) { + let check: ((v: unknown) => boolean) | undefined; + try { + check = candidate.prepare(structuredClone(group.schema), { + draft: file.draft, + schemas: suiteSchemas(), + formats: kind === "format" ? formats : undefined, + }).check; + } catch { + continue; // A schema the library cannot compile fails all its tests. + } + for (const test of group.tests) { + try { + if (check(test.data) === test.valid) results[section].passed[candidate.name]++; + } catch { + // A throw while checking fails the test. + } + } + } + } +} + +const names = candidates.map((c) => c.name); +console.log(`| section | tests | ${names.join(" | ")} |`); +console.log(`|---|---:|${names.map(() => "---:").join("|")}|`); +for (const [section, r] of Object.entries(results)) { + console.log(`| ${section} | ${r.total} | ${names.map((n) => r.passed[n]).join(" | ")} |`); +} +const out = process.argv.indexOf("--json"); +if (out >= 0) writeFileSync(process.argv[out + 1], JSON.stringify(results, null, "\t")); diff --git a/packages/schema/test/bench/corpus.ts b/packages/schema/test/bench/corpus.ts new file mode 100644 index 000000000..2a985db8e --- /dev/null +++ b/packages/schema/test/bench/corpus.ts @@ -0,0 +1,122 @@ +// A deterministic corpus of distinct tool-like JSON Schemas with one valid value each: random property names, +// descriptions, leaf constraints, enums, nested objects, arrays and some $defs/$ref. No two schemas are copies. + +/** A schema and a value it accepts. */ +export interface CorpusItem { + readonly schema: Record; + readonly value: unknown; +} + +const SYLLABLES = [ + "ka", + "lo", + "mi", + "re", + "su", + "ta", + "ne", + "vo", + "pi", + "da", + "ru", + "xe", + "bo", + "fi", + "gu", + "ha", + "jo", + "ze", +]; +const PATTERNS: readonly (readonly [string, string])[] = [ + ["^[a-z0-9-]+$", "a-1"], + ["^[A-Za-z0-9._-]+$", "A.b_c"], + ["^\\d{4}-\\d{2}-\\d{2}$", "2026-10-31"], + ["^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", "01890a5d-ac96-774b-bcce-b302099a8057"], + ["^#[0-9a-f]{6}$", "#00ff00"], +]; + +/** `count` distinct schemas, the same for the same seed. */ +export function corpus(count: number, seed = 42): CorpusItem[] { + const random = mulberry32(seed); + const int = (lo: number, hi: number) => lo + Math.floor(random() * (hi - lo + 1)); + const pick = (list: readonly T[]): T => list[Math.floor(random() * list.length)]; + const word = () => Array.from({ length: int(2, 4) }, () => pick(SYLLABLES)).join(""); + const sentence = () => `${Array.from({ length: int(4, 12) }, word).join(" ")}.`; + + function leaf(depth: number): [Record, unknown] { + const r = random(); + if (r < 0.3) return [{ type: "string", description: sentence() }, word()]; + if (r < 0.4) { + const min = int(0, 3); + return [ + { type: "string", minLength: min, maxLength: min + int(5, 200), description: sentence() }, + "x".repeat(min + 1), + ]; + } + if (r < 0.47) { + const [pattern, value] = pick(PATTERNS); + return [{ type: "string", pattern, description: sentence() }, value]; + } + if (r < 0.55) { + const values = Array.from({ length: int(2, 5) }, word); + return [{ type: "string", enum: values, description: sentence() }, values[0]]; + } + if (r < 0.65) { + const min = int(0, 10); + return [{ type: "integer", minimum: min, maximum: min + int(10, 10000), description: sentence() }, min + 1]; + } + if (r < 0.72) return [{ type: "number", description: sentence() }, random() * 100]; + if (r < 0.8) return [{ type: "boolean", description: sentence() }, random() < 0.5]; + if (r < 0.88) { + return [{ type: "array", items: { type: "string" }, maxItems: int(1, 50), description: sentence() }, [word()]]; + } + if (depth < 2) return object(depth + 1, int(2, 5)); + return [{ type: "string", description: sentence() }, word()]; + } + + function object(depth: number, n: number): [Record, Record] { + const properties: Record> = {}; + const value: Record = {}; + const required: string[] = []; + for (let i = 0; i < n; i++) { + const name = word() + (i > 0 ? i : ""); + const [schema, v] = leaf(depth); + properties[name] = schema; + if (random() < 0.6) { + required.push(name); + value[name] = v; + } else if (random() < 0.5) value[name] = v; + } + const schema: Record = { type: "object", properties, required }; + if (random() < 0.5) schema.additionalProperties = false; + return [schema, value]; + } + + const items: CorpusItem[] = []; + for (let i = 0; i < count; i++) { + const [schema, value] = object(0, int(2, 10)); + const properties = schema.properties as Record>; + if (random() < 0.2) { + // Move one nested object into $defs and reference it, as MCP servers built with zod often do. + const key = Object.keys(properties).find((k) => properties[k].type === "object"); + if (key !== undefined) { + const name = `${word()}Def`; + schema.$defs = { [name]: properties[key] }; + properties[key] = { $ref: `#/$defs/${name}` }; + } + } + schema.description = sentence(); + items.push({ schema, value }); + } + return items; +} + +function mulberry32(seed: number): () => number { + let state = seed; + return () => { + state = (state + 0x6d2b79f5) | 0; + let t = Math.imul(state ^ (state >>> 15), 1 | state); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} diff --git a/packages/schema/test/bench/memory.ts b/packages/schema/test/bench/memory.ts new file mode 100644 index 000000000..9f2c1546b --- /dev/null +++ b/packages/schema/test/bench/memory.ts @@ -0,0 +1,73 @@ +// Retained heap of 3,200 distinct compiled schemas per candidate. Each candidate runs in its own process with +// --expose-gc: the schemas and values are built before the baseline, every library's returned validator is kept, +// and the heap is read after forced collections, once after compiling and once after checking each value 3 times +// (when libraries that compile lazily, like omptype, have compiled). +// Run: node test/bench/memory.ts [--json ] + +import { execFileSync } from "node:child_process"; +import { writeFileSync } from "node:fs"; +import { byName, candidates } from "./candidates.ts"; +import { corpus } from "./corpus.ts"; + +const COUNT = Number(process.env.COUNT ?? 3200); + +/** MB retained by the schemas themselves, and by each candidate after compiling and after checking. */ +export interface MemoryResults { + readonly count: number; + readonly schemasMB: number; + readonly candidates: Record; +} + +const child = process.argv.indexOf("--child"); +if (child >= 0) measure(process.argv[child + 1]); +else run(); + +function run(): void { + const results: MemoryResults = { count: COUNT, schemasMB: 0, candidates: {} }; + const names = [...candidates.map((c) => c.name), "typebox interpreter"]; + for (const name of names) { + const output = execFileSync(process.execPath, ["--expose-gc", import.meta.filename, "--child", name], { + encoding: "utf8", + env: { ...process.env, COUNT: String(COUNT) }, + }); + const result = JSON.parse(output); + Object.assign(results, { schemasMB: result.schemasMB }); + results.candidates[name] = { compiledMB: result.compiledMB, checkedMB: result.checkedMB }; + } + console.log(`${COUNT} distinct schemas take ${results.schemasMB.toFixed(1)} MB themselves (not counted below).\n`); + console.log("| candidate | after compiling | after checking |"); + console.log("|---|---:|---:|"); + for (const [name, r] of Object.entries(results.candidates)) { + console.log(`| ${name} | ${r.compiledMB.toFixed(1)} MB | ${r.checkedMB.toFixed(1)} MB |`); + } + const out = process.argv.indexOf("--json"); + if (out >= 0) writeFileSync(process.argv[out + 1], JSON.stringify(results, null, "\t")); +} + +function measure(name: string): void { + const candidate = byName.get(name); + if (candidate === undefined) throw new Error(`Unknown candidate ${name}`); + const gc = (globalThis as { gc?: () => void }).gc; + if (gc === undefined) throw new Error("Run with --expose-gc"); + const heap = () => { + for (let i = 0; i < 4; i++) gc(); + return process.memoryUsage().heapUsed; + }; + // Warm the library's own code on another corpus, so its lazily compiled functions are not counted. + for (const { schema, value } of corpus(100, 7)) { + const { check } = candidate.prepare(schema, { draft: "2020-12" }); + for (let i = 0; i < 4; i++) check(value); + } + const empty = heap(); + const items = corpus(COUNT, 42); + const base = heap(); + const prepared = items.map(({ schema }) => candidate.prepare(schema, { draft: "2020-12" })); + const compiled = heap(); + for (let i = 0; i < items.length; i++) for (let k = 0; k < 3; k++) prepared[i].check(items[i].value); + const checked = heap(); + (globalThis as { keep?: unknown }).keep = [items, prepared]; + const mb = (bytes: number) => bytes / 1048576; + console.log( + JSON.stringify({ schemasMB: mb(base - empty), compiledMB: mb(compiled - base), checkedMB: mb(checked - base) }), + ); +} diff --git a/packages/schema/test/bench/setup.ts b/packages/schema/test/bench/setup.ts new file mode 100644 index 000000000..ccf188c2b --- /dev/null +++ b/packages/schema/test/bench/setup.ts @@ -0,0 +1,51 @@ +// Setup cost: compiling a schema the library has not seen, and the time of its first checks. Uses distinct +// schemas from the corpus after warming each library's own code, so it measures a new schema, not a new process. +// Run: node test/bench/setup.ts [--json ] + +import { writeFileSync } from "node:fs"; +import { candidates } from "./candidates.ts"; +import { corpus } from "./corpus.ts"; + +const COUNT = Number(process.env.COUNT ?? 1000); + +/** Mean µs per schema: compile, then each of the first three checks and the mean of checks 4-10. */ +export type SetupResults = Record; + +const results: SetupResults = {}; +for (const candidate of candidates) { + for (const { schema, value } of corpus(200, 7)) { + const { check } = candidate.prepare(schema, { draft: "2020-12" }); + for (let i = 0; i < 10; i++) check(value); + } + const items = corpus(COUNT, 99); + let compile = 0; + const checks = [0, 0, 0, 0]; + for (const { schema, value } of items) { + let start = process.hrtime.bigint(); + const { check } = candidate.prepare(schema, { draft: "2020-12" }); + compile += Number(process.hrtime.bigint() - start); + for (let k = 0; k < 3; k++) { + start = process.hrtime.bigint(); + check(value); + checks[k] += Number(process.hrtime.bigint() - start); + } + start = process.hrtime.bigint(); + for (let k = 3; k < 10; k++) check(value); + checks[3] += Number(process.hrtime.bigint() - start) / 7; + } + const us = (ns: number) => ns / COUNT / 1000; + results[candidate.name] = { + compileUs: us(compile), + checkUs: [us(checks[0]), us(checks[1]), us(checks[2]), us(checks[3])], + }; +} + +console.log("| candidate | compile | check 1 | check 2 | check 3 | checks 4-10 |"); +console.log("|---|---:|---:|---:|---:|---:|"); +for (const [name, r] of Object.entries(results)) { + console.log( + `| ${name} | ${r.compileUs.toFixed(1)} µs | ${r.checkUs.map((x) => `${x.toFixed(2)} µs`).join(" | ")} |`, + ); +} +const out = process.argv.indexOf("--json"); +if (out >= 0) writeFileSync(process.argv[out + 1], JSON.stringify(results, null, "\t")); diff --git a/packages/schema/test/bench/size.ts b/packages/schema/test/bench/size.ts new file mode 100644 index 000000000..b6996af60 --- /dev/null +++ b/packages/schema/test/bench/size.ts @@ -0,0 +1,43 @@ +// Bundle size of what a validator needs from each library: esbuild bundles and minifies the import, then gzip -9. +// Run: node test/bench/size.ts [--json ] + +import { writeFileSync } from "node:fs"; +import { gzipSync } from "node:zlib"; +import { build } from "esbuild"; + +const src = new URL("../../src/", import.meta.url).pathname; + +const entries: Record = { + "pi-schema": `export { compile } from ${JSON.stringify(`${src}index.ts`)};`, + "pi-schema + formats": `export { compile } from ${JSON.stringify(`${src}index.ts`)}; export { formats } from ${JSON.stringify(`${src}formats/index.ts`)};`, + typebox: `export { Compile } from "typebox/compile";`, + cfworker: `export { Validator } from "@cfworker/json-schema";`, + omptype: `export { fromJsonSchema } from "@oh-my-pi/omptype";`, +}; + +/** Minified and gzipped bytes by entry. */ +export type SizeResults = Record; + +const results: SizeResults = {}; +for (const [name, contents] of Object.entries(entries)) { + const output = await build({ + stdin: { contents, resolveDir: src, loader: "ts" }, + bundle: true, + minify: true, + format: "esm", + platform: "neutral", + mainFields: ["module", "main"], + write: false, + logLevel: "silent", + }); + const code = output.outputFiles[0].contents; + results[name] = { minified: code.length, gzip: gzipSync(code, { level: 9 }).length }; +} + +console.log("| library | minified | gzip |"); +console.log("|---|---:|---:|"); +for (const [name, r] of Object.entries(results)) { + console.log(`| ${name} | ${(r.minified / 1024).toFixed(1)} KB | ${(r.gzip / 1024).toFixed(1)} KB |`); +} +const out = process.argv.indexOf("--json"); +if (out >= 0) writeFileSync(process.argv[out + 1], JSON.stringify(results, null, "\t")); diff --git a/packages/schema/test/bench/speed.ts b/packages/schema/test/bench/speed.ts new file mode 100644 index 000000000..6f470951b --- /dev/null +++ b/packages/schema/test/bench/speed.ts @@ -0,0 +1,62 @@ +// Hot validation speed: ns per check for each case and candidate, after warm-up. Each round times every +// candidate once, in a rotated order; the result is the median of the rounds. +// Run: node test/bench/speed.ts [--json ] (add --disallow-code-generation-from-strings for no eval) + +import { writeFileSync } from "node:fs"; +import { candidates } from "./candidates.ts"; +import { cases } from "./cases.ts"; + +const ROUNDS = Number(process.env.ROUNDS ?? 15); +const ROUND_MS = Number(process.env.ROUND_MS ?? 30); +let sink = 0; + +function time(check: (v: unknown) => boolean, value: unknown, n: number): number { + const start = process.hrtime.bigint(); + for (let i = 0; i < n; i++) if (check(value)) sink++; + return Number(process.hrtime.bigint() - start) / n; +} + +function median(values: number[]): number { + return [...values].sort((a, b) => a - b)[values.length >> 1]; +} + +/** Median ns per check, by case name, input ("valid"/"invalid") and candidate name. */ +export type SpeedResults = Record>>; + +const results: SpeedResults = {}; +for (const c of cases) { + results[c.name] = { valid: {}, invalid: {} }; + for (const input of ["valid", "invalid"] as const) { + const value = c[input]; + const runs = candidates.map((candidate) => { + const { check } = candidate.prepare(c.schema, { draft: "2020-12" }); + if (check(value) !== (input === "valid")) throw new Error(`${candidate.name} gets ${c.name} (${input}) wrong`); + let n = 1000; + while (time(check, value, n) * n < ROUND_MS * 1e6 && n < 1e8) n *= 2; + return { candidate, check, n, samples: [] as number[] }; + }); + for (let round = 0; round < ROUNDS; round++) { + for (let i = 0; i < runs.length; i++) { + const run = runs[(i + round) % runs.length]; + run.samples.push(time(run.check, value, run.n)); + } + } + for (const run of runs) results[c.name][input][run.candidate.name] = median(run.samples); + } +} + +const names = candidates.map((c) => c.name); +console.log(`| case | input | ${names.join(" | ")} |`); +console.log(`|---|---|${names.map(() => "---:").join("|")}|`); +for (const [name, inputs] of Object.entries(results)) { + for (const [input, ns] of Object.entries(inputs)) { + console.log(`| ${name} | ${input} | ${names.map((n) => format(ns[n])).join(" | ")} |`); + } +} +const out = process.argv.indexOf("--json"); +if (out >= 0) writeFileSync(process.argv[out + 1], JSON.stringify(results, null, "\t")); +if (sink < 0) console.log(sink); + +function format(ns: number): string { + return ns >= 1000 ? `${(ns / 1000).toFixed(ns >= 10000 ? 0 : 1)} µs` : `${ns.toFixed(0)} ns`; +} diff --git a/packages/schema/test/builders.test.ts b/packages/schema/test/builders.test.ts new file mode 100644 index 000000000..2d13886c7 --- /dev/null +++ b/packages/schema/test/builders.test.ts @@ -0,0 +1,104 @@ +// The builders: the JSON Schema they produce, the static types they carry, and how they compile. + +import { describe, expect, expectTypeOf, it } from "vitest"; +import { + arr, + bool, + bytes, + compile, + int, + lit, + map, + nul, + nullable, + num, + obj, + oneOf, + opt, + type Static, + str, + tuple, + union, + unknown, +} from "../src/index.ts"; + +describe("builders", () => { + it("produce plain JSON Schema", () => { + const schema = obj({ + path: str({ description: "File path", minLength: 1 }), + offset: opt(int({ minimum: 1 })), + mode: oneOf(["read", "write"]), + }); + expect(JSON.parse(JSON.stringify(schema))).toEqual({ + type: "object", + properties: { + path: { type: "string", description: "File path", minLength: 1 }, + offset: { type: "integer", minimum: 1 }, + mode: { enum: ["read", "write"] }, + }, + required: ["path", "mode"], + additionalProperties: false, + }); + }); + + it("produce every leaf", () => { + expect([str(), num(), int(), bool(), nul(), unknown(), lit(3), bytes({ maxLength: 8 })]).toEqual([ + { type: "string" }, + { type: "number" }, + { type: "integer" }, + { type: "boolean" }, + { type: "null" }, + {}, + { const: 3 }, + { type: "bytes", maxLength: 8 }, + ]); + expect(arr(str(), { maxItems: 2 })).toEqual({ type: "array", items: { type: "string" }, maxItems: 2 }); + expect(map(num())).toEqual({ type: "object", additionalProperties: { type: "number" } }); + expect(tuple([str(), num()])).toEqual({ + type: "array", + prefixItems: [{ type: "string" }, { type: "number" }], + items: false, + minItems: 2, + }); + expect(nullable(str())).toEqual({ anyOf: [{ type: "string" }, { type: "null" }] }); + expect(obj({ a: str() }, { open: true })).toEqual({ + type: "object", + properties: { a: { type: "string" } }, + required: ["a"], + }); + }); + + it("carry the static type of valid values", () => { + const message = union([ + obj({ kind: lit("text"), text: str() }), + obj({ kind: lit("image"), url: str(), alt: opt(str()) }), + ]); + expectTypeOf>().toEqualTypeOf< + { kind: "text"; text: string } | { kind: "image"; url: string; alt?: string } + >(); + const pair = tuple([str(), arr(int())]); + expectTypeOf>().toEqualTypeOf<[string, number[]]>(); + expectTypeOf>>>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf(); + }); + + it("narrow values through a compiled check", () => { + const point = obj({ x: num(), y: num(), label: opt(str()) }); + const value: unknown = { x: 1, y: 2 }; + const { check } = compile(point); + if (!check(value)) throw new Error("expected a point"); + expectTypeOf(value).toEqualTypeOf<{ x: number; y: number; label?: string }>(); + expect(value.x + value.y).toBe(3); + }); + + it("compile into checks with the declared semantics", () => { + const validator = compile(obj({ name: str({ minLength: 1 }), tags: opt(arr(str(), { uniqueItems: true })) })); + expect(validator.check({ name: "a" })).toBe(true); + expect(validator.check({ name: "a", tags: ["x", "y"] })).toBe(true); + expect(validator.errors({ name: "", tags: ["x", "x"], other: 1 })).toEqual([ + { path: "/name", keyword: "minLength", message: "expected at least 1 characters" }, + { path: "/tags", keyword: "uniqueItems", message: "expected unique items" }, + { path: "/other", keyword: "additionalProperties", message: "unknown field" }, + ]); + }); +}); diff --git a/packages/schema/test/conformance.test.ts b/packages/schema/test/conformance.test.ts new file mode 100644 index 000000000..5702220c5 --- /dev/null +++ b/packages/schema/test/conformance.test.ts @@ -0,0 +1,51 @@ +// The official JSON-Schema-Test-Suite, drafts 2020-12 and 7: every test, including optional and format tests. +// A test listed in KNOWN_FAILURES must fail; when one starts passing, remove it from the list. + +import { describe, expect, it } from "vitest"; +import { formats } from "../src/formats/index.ts"; +import { compile } from "../src/index.ts"; +import { suiteFiles, suiteSchemas } from "./suite.ts"; + +const KNOWN_FAILURES = new Set([ + // Vocabularies switched off by a custom meta-schema are not supported. + "draft2020-12/vocabulary > schema that uses custom metaschema with with no validation vocabulary > no validation: invalid number, but it still validates", + "draft2020-12/optional/format-assertion > schema that uses custom metaschema with format-assertion: false > format-assertion: false: invalid string", + "draft2020-12/optional/format-assertion > schema that uses custom metaschema with format-assertion: true > format-assertion: true: invalid string", + // A $ref to a document of another draft is evaluated with the compile's draft. + "draft2020-12/optional/cross-draft > refs to historic drafts are processed as historic drafts > first item not a string is valid", + "draft7/optional/cross-draft > refs to future drafts are processed as future drafts > missing bar is invalid", + // contentMediaType and contentEncoding are annotations. + "draft7/optional/content > validation of string-encoded content based on media type > an invalid JSON document", + "draft7/optional/content > validation of binary string-encoding > an invalid base64 string (% is not a valid character)", + "draft7/optional/content > validation of binary-encoded media type documents > a validly-encoded invalid JSON document", + "draft7/optional/content > validation of binary-encoded media type documents > an invalid base64 string that is valid JSON", +]); + +const schemas = suiteSchemas(); + +for (const file of suiteFiles()) { + // Format tests assert formats; everywhere else `format` is an annotation, as draft 2020-12 specifies. + const options = { + draft: file.draft, + schemas, + formats: file.name.includes("/optional/format/") ? formats : undefined, + }; + describe(file.name, () => { + for (const group of file.groups) { + describe(group.description, () => { + for (const test of group.tests) { + const known = KNOWN_FAILURES.has(`${file.name} > ${group.description} > ${test.description}`); + (known ? it.fails : it)(test.description, () => { + const validator = compile(group.schema, options); + expect(validator.check(test.data)).toBe(test.valid); + }); + it(`${test.description} (errors agree with check)`, () => { + const validator = compile(group.schema, options); + const errors = validator.errors(test.data); + expect(errors.length === 0).toBe(validator.check(test.data)); + }); + } + }); + } + }); +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/LICENSE b/packages/schema/test/fixtures/json-schema-test-suite/LICENSE new file mode 100644 index 000000000..c28adbadd --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2012 Julian Berman + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/schema/test/fixtures/json-schema-test-suite/README.md b/packages/schema/test/fixtures/json-schema-test-suite/README.md new file mode 100644 index 000000000..324ff0a0c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/README.md @@ -0,0 +1,9 @@ +# JSON-Schema-Test-Suite (subset) + +From https://github.com/json-schema-org/JSON-Schema-Test-Suite at commit +`3f05891cac5d51ffa068d7076d1372130e00c829`, unmodified, under its MIT license (`LICENSE`). + +Only what the drafts this package supports use: + +- `tests/draft2020-12/` and `tests/draft7/`, complete (core, `optional/`, `optional/format/`); +- `remotes/`: the documents those tests reference (served as `http://localhost:1234/...` in the tests). diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChange/folderInteger.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChange/folderInteger.json new file mode 100644 index 000000000..8b50ea308 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChange/folderInteger.json @@ -0,0 +1,3 @@ +{ + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolder/folderInteger.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolder/folderInteger.json new file mode 100644 index 000000000..8b50ea308 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolder/folderInteger.json @@ -0,0 +1,3 @@ +{ + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolderInSubschema/folderInteger.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolderInSubschema/folderInteger.json new file mode 100644 index 000000000..8b50ea308 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolderInSubschema/folderInteger.json @@ -0,0 +1,3 @@ +{ + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/dependentRequired.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/dependentRequired.json new file mode 100644 index 000000000..0d691d964 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/dependentRequired.json @@ -0,0 +1,7 @@ +{ + "$id": "http://localhost:1234/draft2019-09/dependentRequired.json", + "$schema": "https://json-schema.org/draft/2019-09/schema", + "dependentRequired": { + "foo": ["bar"] + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/ignore-prefixItems.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/ignore-prefixItems.json new file mode 100644 index 000000000..b5ef3928c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/ignore-prefixItems.json @@ -0,0 +1,7 @@ +{ + "$id": "http://localhost:1234/draft2019-09/ignore-prefixItems.json", + "$schema": "https://json-schema.org/draft/2019-09/schema", + "prefixItems": [ + {"type": "string"} + ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChange/folderInteger.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChange/folderInteger.json new file mode 100644 index 000000000..1f44a6313 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChange/folderInteger.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolder/folderInteger.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolder/folderInteger.json new file mode 100644 index 000000000..1f44a6313 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolder/folderInteger.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolderInSubschema/folderInteger.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolderInSubschema/folderInteger.json new file mode 100644 index 000000000..1f44a6313 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolderInSubschema/folderInteger.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-dynamicref.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-dynamicref.json new file mode 100644 index 000000000..07cce1dac --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-dynamicref.json @@ -0,0 +1,13 @@ +{ + "$id": "http://localhost:1234/draft2020-12/detached-dynamicref.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "foo": { + "$dynamicRef": "#detached" + }, + "detached": { + "$dynamicAnchor": "detached", + "type": "integer" + } + } +} \ No newline at end of file diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-ref.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-ref.json new file mode 100644 index 000000000..9c2dca93c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-ref.json @@ -0,0 +1,13 @@ +{ + "$id": "http://localhost:1234/draft2020-12/detached-ref.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "foo": { + "$ref": "#detached" + }, + "detached": { + "$anchor": "detached", + "type": "integer" + } + } +} \ No newline at end of file diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/different-id-ref-string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/different-id-ref-string.json new file mode 100644 index 000000000..5d2644d3f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/different-id-ref-string.json @@ -0,0 +1,5 @@ +{ + "$id": "http://localhost:1234/draft2020-12/real-id-ref-string.json", + "$defs": {"bar": {"type": "string"}}, + "$ref": "#/$defs/bar" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/extendible-dynamic-ref.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/extendible-dynamic-ref.json new file mode 100644 index 000000000..65bc0c217 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/extendible-dynamic-ref.json @@ -0,0 +1,21 @@ +{ + "description": "extendible array", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/extendible-dynamic-ref.json", + "type": "object", + "properties": { + "elements": { + "type": "array", + "items": { + "$dynamicRef": "#elements" + } + } + }, + "required": ["elements"], + "additionalProperties": false, + "$defs": { + "elements": { + "$dynamicAnchor": "elements" + } + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-false.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-false.json new file mode 100644 index 000000000..43a711c9d --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-false.json @@ -0,0 +1,13 @@ +{ + "$id": "http://localhost:1234/draft2020-12/format-assertion-false.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$vocabulary": { + "https://json-schema.org/draft/2020-12/vocab/core": true, + "https://json-schema.org/draft/2020-12/vocab/format-assertion": false + }, + "$dynamicAnchor": "meta", + "allOf": [ + { "$ref": "https://json-schema.org/draft/2020-12/meta/core" }, + { "$ref": "https://json-schema.org/draft/2020-12/meta/format-assertion" } + ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-true.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-true.json new file mode 100644 index 000000000..39c6b0abf --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-true.json @@ -0,0 +1,13 @@ +{ + "$id": "http://localhost:1234/draft2020-12/format-assertion-true.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$vocabulary": { + "https://json-schema.org/draft/2020-12/vocab/core": true, + "https://json-schema.org/draft/2020-12/vocab/format-assertion": true + }, + "$dynamicAnchor": "meta", + "allOf": [ + { "$ref": "https://json-schema.org/draft/2020-12/meta/core" }, + { "$ref": "https://json-schema.org/draft/2020-12/meta/format-assertion" } + ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/integer.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/integer.json new file mode 100644 index 000000000..1f44a6313 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/integer.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/locationIndependentIdentifier.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/locationIndependentIdentifier.json new file mode 100644 index 000000000..6565a1ee0 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/locationIndependentIdentifier.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "refToInteger": { + "$ref": "#foo" + }, + "A": { + "$anchor": "foo", + "type": "integer" + } + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-no-validation.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-no-validation.json new file mode 100644 index 000000000..71be8b5da --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-no-validation.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/metaschema-no-validation.json", + "$vocabulary": { + "https://json-schema.org/draft/2020-12/vocab/applicator": true, + "https://json-schema.org/draft/2020-12/vocab/core": true + }, + "$dynamicAnchor": "meta", + "allOf": [ + { "$ref": "https://json-schema.org/draft/2020-12/meta/applicator" }, + { "$ref": "https://json-schema.org/draft/2020-12/meta/core" } + ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-optional-vocabulary.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-optional-vocabulary.json new file mode 100644 index 000000000..a6963e548 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-optional-vocabulary.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/metaschema-optional-vocabulary.json", + "$vocabulary": { + "https://json-schema.org/draft/2020-12/vocab/validation": true, + "https://json-schema.org/draft/2020-12/vocab/core": true, + "http://localhost:1234/draft/2020-12/vocab/custom": false + }, + "$dynamicAnchor": "meta", + "allOf": [ + { "$ref": "https://json-schema.org/draft/2020-12/meta/validation" }, + { "$ref": "https://json-schema.org/draft/2020-12/meta/core" } + ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/name-defs.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/name-defs.json new file mode 100644 index 000000000..67bc33c51 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/name-defs.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "orNull": { + "anyOf": [ + { + "type": "null" + }, + { + "$ref": "#" + } + ] + } + }, + "type": "string" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested-absolute-ref-to-string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested-absolute-ref-to-string.json new file mode 100644 index 000000000..3d51ded5c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested-absolute-ref-to-string.json @@ -0,0 +1,9 @@ +{ + "$defs": { + "bar": { + "$id": "http://localhost:1234/draft2020-12/the-nested-id.json", + "type": "string" + } + }, + "$ref": "http://localhost:1234/draft2020-12/the-nested-id.json" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/foo-ref-string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/foo-ref-string.json new file mode 100644 index 000000000..29661ff9f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/foo-ref-string.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "foo": {"$ref": "string.json"} + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/string.json new file mode 100644 index 000000000..6607ac534 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/string.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/prefixItems.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/prefixItems.json new file mode 100644 index 000000000..acd8293c6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/prefixItems.json @@ -0,0 +1,7 @@ +{ + "$id": "http://localhost:1234/draft2020-12/prefixItems.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + {"type": "string"} + ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/ref-and-defs.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/ref-and-defs.json new file mode 100644 index 000000000..16d30fa3a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/ref-and-defs.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/ref-and-defs.json", + "$defs": { + "inner": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "$ref": "#/$defs/inner" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/subSchemas.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/subSchemas.json new file mode 100644 index 000000000..1bb4846d7 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/subSchemas.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "integer": { + "type": "integer" + }, + "refToInteger": { + "$ref": "#/$defs/integer" + } + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/tree.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/tree.json new file mode 100644 index 000000000..b07555fb3 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/tree.json @@ -0,0 +1,17 @@ +{ + "description": "tree schema, extensible", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/tree.json", + "$dynamicAnchor": "node", + + "type": "object", + "properties": { + "data": true, + "children": { + "type": "array", + "items": { + "$dynamicRef": "#node" + } + } + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/urn-ref-string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/urn-ref-string.json new file mode 100644 index 000000000..796f37021 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/urn-ref-string.json @@ -0,0 +1,5 @@ +{ + "$id": "urn:uuid:feebdaed-ffff-0000-2020-1200deadbeef", + "$defs": {"bar": {"type": "string"}}, + "$ref": "#/$defs/bar" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/detached-ref.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/detached-ref.json new file mode 100644 index 000000000..27f2ec80a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/detached-ref.json @@ -0,0 +1,13 @@ +{ + "$id": "http://localhost:1234/draft7/detached-ref.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "foo": { + "$ref": "#detached" + }, + "detached": { + "$id": "#detached", + "type": "integer" + } + } +} \ No newline at end of file diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ignore-dependentRequired.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ignore-dependentRequired.json new file mode 100644 index 000000000..13fd0df68 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ignore-dependentRequired.json @@ -0,0 +1,7 @@ +{ + "$id": "http://localhost:1234/draft7/ignore-dependentRequired.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "dependentRequired": { + "foo": ["bar"] + } +} \ No newline at end of file diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/locationIndependentIdentifier.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/locationIndependentIdentifier.json new file mode 100644 index 000000000..e72815cd5 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/locationIndependentIdentifier.json @@ -0,0 +1,11 @@ +{ + "definitions": { + "refToInteger": { + "$ref": "#foo" + }, + "A": { + "$id": "#foo", + "type": "integer" + } + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/name.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/name.json new file mode 100644 index 000000000..fceacb809 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/name.json @@ -0,0 +1,15 @@ +{ + "definitions": { + "orNull": { + "anyOf": [ + { + "type": "null" + }, + { + "$ref": "#" + } + ] + } + }, + "type": "string" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ref-and-definitions.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ref-and-definitions.json new file mode 100644 index 000000000..d5929380c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ref-and-definitions.json @@ -0,0 +1,11 @@ +{ + "$id": "http://localhost:1234/draft7/ref-and-definitions.json", + "definitions": { + "inner": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "allOf": [ { "$ref": "#/definitions/inner" } ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/subSchemas.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/subSchemas.json new file mode 100644 index 000000000..6e9b3de35 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/subSchemas.json @@ -0,0 +1,10 @@ +{ + "definitions": { + "integer": { + "type": "integer" + }, + "refToInteger": { + "$ref": "#/definitions/integer" + } + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/integer.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/integer.json new file mode 100644 index 000000000..8b50ea308 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/integer.json @@ -0,0 +1,3 @@ +{ + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/foo-ref-string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/foo-ref-string.json new file mode 100644 index 000000000..9cd2527ed --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/foo-ref-string.json @@ -0,0 +1,6 @@ +{ + "type": "object", + "properties": { + "foo": {"$ref": "string.json"} + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/string.json new file mode 100644 index 000000000..c2d48c068 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/string.json @@ -0,0 +1,3 @@ +{ + "type": "string" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/additionalProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/additionalProperties.json new file mode 100644 index 000000000..9618575e2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/additionalProperties.json @@ -0,0 +1,219 @@ +[ + { + "description": + "additionalProperties being false does not allow other properties", + "specification": [ { "core":"10.3.2.3", "quote": "The value of \"additionalProperties\" MUST be a valid JSON Schema. Boolean \"false\" forbids everything." } ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"foo": {}, "bar": {}}, + "patternProperties": { "^v": {} }, + "additionalProperties": false + }, + "tests": [ + { + "description": "no additional properties is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "an additional property is invalid", + "data": {"foo" : 1, "bar" : 2, "quux" : "boom"}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [1, 2, 3], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobarbaz", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "patternProperties are not additional properties", + "data": {"foo":1, "vroom": 2}, + "valid": true + } + ] + }, + { + "description": "non-ASCII pattern with additionalProperties", + "specification": [ { "core":"10.3.2.3"} ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": {"^á": {}}, + "additionalProperties": false + }, + "tests": [ + { + "description": "matching the pattern is valid", + "data": {"ármányos": 2}, + "valid": true + }, + { + "description": "not matching the pattern is invalid", + "data": {"élmény": 2}, + "valid": false + } + ] + }, + { + "description": "additionalProperties with schema", + "specification": [ { "core":"10.3.2.3", "quote": "The value of \"additionalProperties\" MUST be a valid JSON Schema." } ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"foo": {}, "bar": {}}, + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "no additional properties is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "an additional valid property is valid", + "data": {"foo" : 1, "bar" : 2, "quux" : true}, + "valid": true + }, + { + "description": "an additional invalid property is invalid", + "data": {"foo" : 1, "bar" : 2, "quux" : 12}, + "valid": false + } + ] + }, + { + "description": "additionalProperties can exist by itself", + "specification": [ { "core":"10.3.2.3", "quote": "With no other applicator applying to object instances. This validates all the instance values irrespective of their property names" } ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "an additional valid property is valid", + "data": {"foo" : true}, + "valid": true + }, + { + "description": "an additional invalid property is invalid", + "data": {"foo" : 1}, + "valid": false + } + ] + }, + { + "description": "additionalProperties are allowed by default", + "specification": [ { "core":"10.3.2.3", "quote": "Omitting this keyword has the same assertion behavior as an empty schema." } ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"foo": {}, "bar": {}} + }, + "tests": [ + { + "description": "additional properties are allowed", + "data": {"foo": 1, "bar": 2, "quux": true}, + "valid": true + } + ] + }, + { + "description": "additionalProperties does not look in applicators", + "specification":[ { "core": "10.2", "quote": "Subschemas of applicator keywords evaluate the instance completely independently such that the results of one such subschema MUST NOT impact the results of sibling subschemas." } ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + {"properties": {"foo": {}}} + ], + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "properties defined in allOf are not examined", + "data": {"foo": 1, "bar": true}, + "valid": false + } + ] + }, + { + "description": "additionalProperties with null valued instance properties", + "specification": [ { "core":"10.3.2.3" } ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null values", + "data": {"foo": null}, + "valid": true + } + ] + }, + { + "description": "additionalProperties with propertyNames", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": { + "maxLength": 5 + }, + "additionalProperties": { + "type": "number" + } + }, + "tests": [ + { + "description": "Valid against both keywords", + "data": { "apple": 4 }, + "valid": true + }, + { + "description": "Valid against propertyNames, but not additionalProperties", + "data": { "fig": 2, "pear": "available" }, + "valid": false + } + ] + }, + { + "description": "dependentSchemas with additionalProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"foo2": {}}, + "dependentSchemas": { + "foo" : {}, + "foo2": { + "properties": { + "bar": {} + } + } + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "additionalProperties doesn't consider dependentSchemas", + "data": {"foo": ""}, + "valid": false + }, + { + "description": "additionalProperties can't see bar", + "data": {"bar": ""}, + "valid": false + }, + { + "description": "additionalProperties can't see bar even when foo2 is present", + "data": {"foo2": "", "bar": ""}, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/allOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/allOf.json new file mode 100644 index 000000000..9e87903fe --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/allOf.json @@ -0,0 +1,312 @@ +[ + { + "description": "allOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "bar": {"type": "integer"} + }, + "required": ["bar"] + }, + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "allOf", + "data": {"foo": "baz", "bar": 2}, + "valid": true + }, + { + "description": "mismatch second", + "data": {"foo": "baz"}, + "valid": false + }, + { + "description": "mismatch first", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "wrong type", + "data": {"foo": "baz", "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "allOf with base schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"bar": {"type": "integer"}}, + "required": ["bar"], + "allOf" : [ + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + }, + { + "properties": { + "baz": {"type": "null"} + }, + "required": ["baz"] + } + ] + }, + "tests": [ + { + "description": "valid", + "data": {"foo": "quux", "bar": 2, "baz": null}, + "valid": true + }, + { + "description": "mismatch base schema", + "data": {"foo": "quux", "baz": null}, + "valid": false + }, + { + "description": "mismatch first allOf", + "data": {"bar": 2, "baz": null}, + "valid": false + }, + { + "description": "mismatch second allOf", + "data": {"foo": "quux", "bar": 2}, + "valid": false + }, + { + "description": "mismatch both", + "data": {"bar": 2}, + "valid": false + } + ] + }, + { + "description": "allOf simple types", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + {"maximum": 30}, + {"minimum": 20} + ] + }, + "tests": [ + { + "description": "valid", + "data": 25, + "valid": true + }, + { + "description": "mismatch one", + "data": 35, + "valid": false + } + ] + }, + { + "description": "allOf with boolean schemas, all true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [true, true] + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "allOf with boolean schemas, some false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [true, false] + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "allOf with boolean schemas, all false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [false, false] + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "allOf with one empty schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + {} + ] + }, + "tests": [ + { + "description": "any data is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "allOf with two empty schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + {}, + {} + ] + }, + "tests": [ + { + "description": "any data is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "allOf with the first empty schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + {}, + { "type": "number" } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "allOf with the last empty schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { "type": "number" }, + {} + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "nested allOf, to check validation semantics", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "allOf": [ + { + "type": "null" + } + ] + } + ] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "anything non-null is invalid", + "data": 123, + "valid": false + } + ] + }, + { + "description": "allOf combined with anyOf, oneOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ { "multipleOf": 2 } ], + "anyOf": [ { "multipleOf": 3 } ], + "oneOf": [ { "multipleOf": 5 } ] + }, + "tests": [ + { + "description": "allOf: false, anyOf: false, oneOf: false", + "data": 1, + "valid": false + }, + { + "description": "allOf: false, anyOf: false, oneOf: true", + "data": 5, + "valid": false + }, + { + "description": "allOf: false, anyOf: true, oneOf: false", + "data": 3, + "valid": false + }, + { + "description": "allOf: false, anyOf: true, oneOf: true", + "data": 15, + "valid": false + }, + { + "description": "allOf: true, anyOf: false, oneOf: false", + "data": 2, + "valid": false + }, + { + "description": "allOf: true, anyOf: false, oneOf: true", + "data": 10, + "valid": false + }, + { + "description": "allOf: true, anyOf: true, oneOf: false", + "data": 6, + "valid": false + }, + { + "description": "allOf: true, anyOf: true, oneOf: true", + "data": 30, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anchor.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anchor.json new file mode 100644 index 000000000..99143fa11 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anchor.json @@ -0,0 +1,120 @@ +[ + { + "description": "Location-independent identifier", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "#foo", + "$defs": { + "A": { + "$anchor": "foo", + "type": "integer" + } + } + }, + "tests": [ + { + "data": 1, + "description": "match", + "valid": true + }, + { + "data": "a", + "description": "mismatch", + "valid": false + } + ] + }, + { + "description": "Location-independent identifier with absolute URI", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/bar#foo", + "$defs": { + "A": { + "$id": "http://localhost:1234/draft2020-12/bar", + "$anchor": "foo", + "type": "integer" + } + } + }, + "tests": [ + { + "data": 1, + "description": "match", + "valid": true + }, + { + "data": "a", + "description": "mismatch", + "valid": false + } + ] + }, + { + "description": "Location-independent identifier with base URI change in subschema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/root", + "$ref": "http://localhost:1234/draft2020-12/nested.json#foo", + "$defs": { + "A": { + "$id": "nested.json", + "$defs": { + "B": { + "$anchor": "foo", + "type": "integer" + } + } + } + } + }, + "tests": [ + { + "data": 1, + "description": "match", + "valid": true + }, + { + "data": "a", + "description": "mismatch", + "valid": false + } + ] + }, + { + "description": "same $anchor with different base uri", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/foobar", + "$defs": { + "A": { + "$id": "child1", + "allOf": [ + { + "$id": "child2", + "$anchor": "my_anchor", + "type": "number" + }, + { + "$anchor": "my_anchor", + "type": "string" + } + ] + } + }, + "$ref": "child1#my_anchor" + }, + "tests": [ + { + "description": "$ref resolves to /$defs/A/allOf/1", + "data": "a", + "valid": true + }, + { + "description": "$ref does not resolve to /$defs/A/allOf/0", + "data": 1, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anyOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anyOf.json new file mode 100644 index 000000000..89b192dbd --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anyOf.json @@ -0,0 +1,203 @@ +[ + { + "description": "anyOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [ + { + "type": "integer" + }, + { + "minimum": 2 + } + ] + }, + "tests": [ + { + "description": "first anyOf valid", + "data": 1, + "valid": true + }, + { + "description": "second anyOf valid", + "data": 2.5, + "valid": true + }, + { + "description": "both anyOf valid", + "data": 3, + "valid": true + }, + { + "description": "neither anyOf valid", + "data": 1.5, + "valid": false + } + ] + }, + { + "description": "anyOf with base schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "anyOf" : [ + { + "maxLength": 2 + }, + { + "minLength": 4 + } + ] + }, + "tests": [ + { + "description": "mismatch base schema", + "data": 3, + "valid": false + }, + { + "description": "one anyOf valid", + "data": "foobar", + "valid": true + }, + { + "description": "both anyOf invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "anyOf with boolean schemas, all true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [true, true] + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "anyOf with boolean schemas, some true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [true, false] + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "anyOf with boolean schemas, all false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [false, false] + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "anyOf complex types", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [ + { + "properties": { + "bar": {"type": "integer"} + }, + "required": ["bar"] + }, + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "first anyOf valid (complex)", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "second anyOf valid (complex)", + "data": {"foo": "baz"}, + "valid": true + }, + { + "description": "both anyOf valid (complex)", + "data": {"foo": "baz", "bar": 2}, + "valid": true + }, + { + "description": "neither anyOf valid (complex)", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "anyOf with one empty schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [ + { "type": "number" }, + {} + ] + }, + "tests": [ + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "number is valid", + "data": 123, + "valid": true + } + ] + }, + { + "description": "nested anyOf, to check validation semantics", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [ + { + "anyOf": [ + { + "type": "null" + } + ] + } + ] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "anything non-null is invalid", + "data": 123, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/boolean_schema.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/boolean_schema.json new file mode 100644 index 000000000..6d40f23f2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/boolean_schema.json @@ -0,0 +1,104 @@ +[ + { + "description": "boolean schema 'true'", + "schema": true, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "boolean true is valid", + "data": true, + "valid": true + }, + { + "description": "boolean false is valid", + "data": false, + "valid": true + }, + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + }, + { + "description": "array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "boolean schema 'false'", + "schema": false, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "boolean true is invalid", + "data": true, + "valid": false + }, + { + "description": "boolean false is invalid", + "data": false, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/const.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/const.json new file mode 100644 index 000000000..72ea3f042 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/const.json @@ -0,0 +1,431 @@ +[ + { + "description": "const validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": 2 + }, + "tests": [ + { + "description": "same value is valid", + "data": 2, + "valid": true + }, + { + "description": "another value is invalid", + "data": 5, + "valid": false + }, + { + "description": "another type is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "const with object", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": {"foo": "bar", "baz": "bax"} + }, + "tests": [ + { + "description": "same object is valid", + "data": {"foo": "bar", "baz": "bax"}, + "valid": true + }, + { + "description": "same object with different property order is valid", + "data": {"baz": "bax", "foo": "bar"}, + "valid": true + }, + { + "description": "another object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "another type is invalid", + "data": [1, 2], + "valid": false + } + ] + }, + { + "description": "const with array", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": [{ "foo": "bar" }] + }, + "tests": [ + { + "description": "same array is valid", + "data": [{"foo": "bar"}], + "valid": true + }, + { + "description": "another array item is invalid", + "data": [2], + "valid": false + }, + { + "description": "array with additional items is invalid", + "data": [1, 2, 3], + "valid": false + } + ] + }, + { + "description": "const with null", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": null + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "not null is invalid", + "data": 0, + "valid": false + } + ] + }, + { + "description": "const with false does not match 0", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": false + }, + "tests": [ + { + "description": "false is valid", + "data": false, + "valid": true + }, + { + "description": "integer zero is invalid", + "data": 0, + "valid": false + }, + { + "description": "float zero is invalid", + "data": 0.0, + "valid": false + } + ] + }, + { + "description": "const with true does not match 1", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": true + }, + "tests": [ + { + "description": "true is valid", + "data": true, + "valid": true + }, + { + "description": "integer one is invalid", + "data": 1, + "valid": false + }, + { + "description": "float one is invalid", + "data": 1.0, + "valid": false + } + ] + }, + { + "description": "const with [false] does not match [0]", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": [false] + }, + "tests": [ + { + "description": "[false] is valid", + "data": [false], + "valid": true + }, + { + "description": "[0] is invalid", + "data": [0], + "valid": false + }, + { + "description": "[0.0] is invalid", + "data": [0.0], + "valid": false + } + ] + }, + { + "description": "const with [true] does not match [1]", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": [true] + }, + "tests": [ + { + "description": "[true] is valid", + "data": [true], + "valid": true + }, + { + "description": "[1] is invalid", + "data": [1], + "valid": false + }, + { + "description": "[1.0] is invalid", + "data": [1.0], + "valid": false + } + ] + }, + { + "description": "const with {\"a\": false} does not match {\"a\": 0}", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": {"a": false} + }, + "tests": [ + { + "description": "{\"a\": false} is valid", + "data": {"a": false}, + "valid": true + }, + { + "description": "{\"a\": 0} is invalid", + "data": {"a": 0}, + "valid": false + }, + { + "description": "{\"a\": 0.0} is invalid", + "data": {"a": 0.0}, + "valid": false + } + ] + }, + { + "description": "const with {\"a\": true} does not match {\"a\": 1}", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": {"a": true} + }, + "tests": [ + { + "description": "{\"a\": true} is valid", + "data": {"a": true}, + "valid": true + }, + { + "description": "{\"a\": 1} is invalid", + "data": {"a": 1}, + "valid": false + }, + { + "description": "{\"a\": 1.0} is invalid", + "data": {"a": 1.0}, + "valid": false + } + ] + }, + { + "description": "const with 0 does not match other zero-like types", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": 0 + }, + "tests": [ + { + "description": "false is invalid", + "data": false, + "valid": false + }, + { + "description": "integer zero is valid", + "data": 0, + "valid": true + }, + { + "description": "float zero is valid", + "data": 0.0, + "valid": true + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + }, + { + "description": "empty string is invalid", + "data": "", + "valid": false + } + ] + }, + { + "description": "const with 1 does not match true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": 1 + }, + "tests": [ + { + "description": "true is invalid", + "data": true, + "valid": false + }, + { + "description": "integer one is valid", + "data": 1, + "valid": true + }, + { + "description": "float one is valid", + "data": 1.0, + "valid": true + } + ] + }, + { + "description": "const with -2.0 matches integer and float types", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": -2.0 + }, + "tests": [ + { + "description": "integer -2 is valid", + "data": -2, + "valid": true + }, + { + "description": "integer 2 is invalid", + "data": 2, + "valid": false + }, + { + "description": "float -2.0 is valid", + "data": -2.0, + "valid": true + }, + { + "description": "float 2.0 is invalid", + "data": 2.0, + "valid": false + }, + { + "description": "float -2.00001 is invalid", + "data": -2.00001, + "valid": false + } + ] + }, + { + "description": "float and integers are equal up to 64-bit representation limits", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": 9007199254740992 + }, + "tests": [ + { + "description": "integer is valid", + "data": 9007199254740992, + "valid": true + }, + { + "description": "integer minus one is invalid", + "data": 9007199254740991, + "valid": false + }, + { + "description": "float is valid", + "data": 9007199254740992.0, + "valid": true + }, + { + "description": "float minus one is invalid", + "data": 9007199254740991.0, + "valid": false + } + ] + }, + { + "description": "nul characters in strings", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": "hello\u0000there" + }, + "tests": [ + { + "description": "match string with nul", + "data": "hello\u0000there", + "valid": true + }, + { + "description": "do not match string lacking nul", + "data": "hellothere", + "valid": false + } + ] + }, + { + "description": "characters with the same visual representation but different codepoint", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": "μ", + "$comment": "U+03BC" + }, + "tests": [ + { + "description": "character uses the same codepoint", + "data": "μ", + "comment": "U+03BC", + "valid": true + }, + { + "description": "character looks the same but uses a different codepoint", + "data": "µ", + "comment": "U+00B5", + "valid": false + } + ] + }, + { + "description": "characters with the same visual representation, but different number of codepoints", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": "ä", + "$comment": "U+00E4" + }, + "tests": [ + { + "description": "character uses the same codepoint", + "data": "ä", + "comment": "U+00E4", + "valid": true + }, + { + "description": "character looks the same but uses combining marks", + "data": "ä", + "comment": "a, U+0308", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/contains.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/contains.json new file mode 100644 index 000000000..08a00a753 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/contains.json @@ -0,0 +1,176 @@ +[ + { + "description": "contains keyword validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"minimum": 5} + }, + "tests": [ + { + "description": "array with item matching schema (5) is valid", + "data": [3, 4, 5], + "valid": true + }, + { + "description": "array with item matching schema (6) is valid", + "data": [3, 4, 6], + "valid": true + }, + { + "description": "array with two items matching schema (5, 6) is valid", + "data": [3, 4, 5, 6], + "valid": true + }, + { + "description": "array without items matching schema is invalid", + "data": [2, 3, 4], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + }, + { + "description": "not array is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "contains keyword with const keyword", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": { "const": 5 } + }, + "tests": [ + { + "description": "array with item 5 is valid", + "data": [3, 4, 5], + "valid": true + }, + { + "description": "array with two items 5 is valid", + "data": [3, 4, 5, 5], + "valid": true + }, + { + "description": "array without item 5 is invalid", + "data": [1, 2, 3, 4], + "valid": false + } + ] + }, + { + "description": "contains keyword with boolean schema true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": true + }, + "tests": [ + { + "description": "any non-empty array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "contains keyword with boolean schema false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": false + }, + "tests": [ + { + "description": "any non-empty array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + }, + { + "description": "non-arrays are valid", + "data": "contains does not apply to strings", + "valid": true + } + ] + }, + { + "description": "items + contains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": { "multipleOf": 2 }, + "contains": { "multipleOf": 3 } + }, + "tests": [ + { + "description": "matches items, does not match contains", + "data": [ 2, 4, 8 ], + "valid": false + }, + { + "description": "does not match items, matches contains", + "data": [ 3, 6, 9 ], + "valid": false + }, + { + "description": "matches both items and contains", + "data": [ 6, 12 ], + "valid": true + }, + { + "description": "matches neither items nor contains", + "data": [ 1, 5 ], + "valid": false + } + ] + }, + { + "description": "contains with false if subschema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": { + "if": false, + "else": true + } + }, + "tests": [ + { + "description": "any non-empty array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "contains with null instance elements", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null items", + "data": [ null ], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/content.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/content.json new file mode 100644 index 000000000..698f78057 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/content.json @@ -0,0 +1,131 @@ +[ + { + "description": "validation of string-encoded content based on media type", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contentMediaType": "application/json" + }, + "tests": [ + { + "description": "a valid JSON document", + "data": "{\"foo\": \"bar\"}", + "valid": true + }, + { + "description": "an invalid JSON document; validates true", + "data": "{:}", + "valid": true + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + }, + { + "description": "validation of binary string-encoding", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contentEncoding": "base64" + }, + "tests": [ + { + "description": "a valid base64 string", + "data": "eyJmb28iOiAiYmFyIn0K", + "valid": true + }, + { + "description": "an invalid base64 string (% is not a valid character); validates true", + "data": "eyJmb28iOi%iYmFyIn0K", + "valid": true + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + }, + { + "description": "validation of binary-encoded media type documents", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contentMediaType": "application/json", + "contentEncoding": "base64" + }, + "tests": [ + { + "description": "a valid base64-encoded JSON document", + "data": "eyJmb28iOiAiYmFyIn0K", + "valid": true + }, + { + "description": "a validly-encoded invalid JSON document; validates true", + "data": "ezp9Cg==", + "valid": true + }, + { + "description": "an invalid base64 string that is valid JSON; validates true", + "data": "{}", + "valid": true + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + }, + { + "description": "validation of binary-encoded media type documents with schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contentMediaType": "application/json", + "contentEncoding": "base64", + "contentSchema": { "type": "object", "required": ["foo"], "properties": { "foo": { "type": "string" } } } + }, + "tests": [ + { + "description": "a valid base64-encoded JSON document", + "data": "eyJmb28iOiAiYmFyIn0K", + "valid": true + }, + { + "description": "another valid base64-encoded JSON document", + "data": "eyJib28iOiAyMCwgImZvbyI6ICJiYXoifQ==", + "valid": true + }, + { + "description": "an invalid base64-encoded JSON document; validates true", + "data": "eyJib28iOiAyMH0=", + "valid": true + }, + { + "description": "an empty object as a base64-encoded JSON document; validates true", + "data": "e30=", + "valid": true + }, + { + "description": "an empty array as a base64-encoded JSON document", + "data": "W10=", + "valid": true + }, + { + "description": "a validly-encoded invalid JSON document; validates true", + "data": "ezp9Cg==", + "valid": true + }, + { + "description": "an invalid base64 string that is valid JSON; validates true", + "data": "{}", + "valid": true + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/default.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/default.json new file mode 100644 index 000000000..ceb3ae271 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/default.json @@ -0,0 +1,82 @@ +[ + { + "description": "invalid type for default", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "type": "integer", + "default": [] + } + } + }, + "tests": [ + { + "description": "valid when property is specified", + "data": {"foo": 13}, + "valid": true + }, + { + "description": "still valid when the invalid default is used", + "data": {}, + "valid": true + } + ] + }, + { + "description": "invalid string value for default", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "bar": { + "type": "string", + "minLength": 4, + "default": "bad" + } + } + }, + "tests": [ + { + "description": "valid when property is specified", + "data": {"bar": "good"}, + "valid": true + }, + { + "description": "still valid when the invalid default is used", + "data": {}, + "valid": true + } + ] + }, + { + "description": "the default keyword does not do anything if the property is missing", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "alpha": { + "type": "number", + "maximum": 3, + "default": 5 + } + } + }, + "tests": [ + { + "description": "an explicit property value is checked against maximum (passing)", + "data": { "alpha": 1 }, + "valid": true + }, + { + "description": "an explicit property value is checked against maximum (failing)", + "data": { "alpha": 5 }, + "valid": false + }, + { + "description": "missing properties are not filled in with the default", + "data": {}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/defs.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/defs.json new file mode 100644 index 000000000..da2a503bf --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/defs.json @@ -0,0 +1,21 @@ +[ + { + "description": "validate definition against metaschema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "https://json-schema.org/draft/2020-12/schema" + }, + "tests": [ + { + "description": "valid definition schema", + "data": {"$defs": {"foo": {"type": "integer"}}}, + "valid": true + }, + { + "description": "invalid definition schema", + "data": {"$defs": {"foo": {"type": 1}}}, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentRequired.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentRequired.json new file mode 100644 index 000000000..2baa38e9f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentRequired.json @@ -0,0 +1,152 @@ +[ + { + "description": "single dependency", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentRequired": {"bar": ["foo"]} + }, + "tests": [ + { + "description": "neither", + "data": {}, + "valid": true + }, + { + "description": "nondependant", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "with dependency", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "missing dependency", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["bar"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "empty dependents", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentRequired": {"bar": []} + }, + "tests": [ + { + "description": "empty object", + "data": {}, + "valid": true + }, + { + "description": "object with one property", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "non-object is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "multiple dependents required", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentRequired": {"quux": ["foo", "bar"]} + }, + "tests": [ + { + "description": "neither", + "data": {}, + "valid": true + }, + { + "description": "nondependants", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "with dependencies", + "data": {"foo": 1, "bar": 2, "quux": 3}, + "valid": true + }, + { + "description": "missing dependency", + "data": {"foo": 1, "quux": 2}, + "valid": false + }, + { + "description": "missing other dependency", + "data": {"bar": 1, "quux": 2}, + "valid": false + }, + { + "description": "missing both dependencies", + "data": {"quux": 1}, + "valid": false + } + ] + }, + { + "description": "dependencies with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentRequired": { + "foo\nbar": ["foo\rbar"], + "foo\"bar": ["foo'bar"] + } + }, + "tests": [ + { + "description": "CRLF", + "data": { + "foo\nbar": 1, + "foo\rbar": 2 + }, + "valid": true + }, + { + "description": "quoted quotes", + "data": { + "foo'bar": 1, + "foo\"bar": 2 + }, + "valid": true + }, + { + "description": "CRLF missing dependent", + "data": { + "foo\nbar": 1, + "foo": 2 + }, + "valid": false + }, + { + "description": "quoted quotes missing dependent", + "data": { + "foo\"bar": 2 + }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentSchemas.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentSchemas.json new file mode 100644 index 000000000..1c5f0574a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentSchemas.json @@ -0,0 +1,171 @@ +[ + { + "description": "single dependency", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentSchemas": { + "bar": { + "properties": { + "foo": {"type": "integer"}, + "bar": {"type": "integer"} + } + } + } + }, + "tests": [ + { + "description": "valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "no dependency", + "data": {"foo": "quux"}, + "valid": true + }, + { + "description": "wrong type", + "data": {"foo": "quux", "bar": 2}, + "valid": false + }, + { + "description": "wrong type other", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + }, + { + "description": "wrong type both", + "data": {"foo": "quux", "bar": "quux"}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["bar"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "boolean subschemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentSchemas": { + "foo": true, + "bar": false + } + }, + "tests": [ + { + "description": "object with property having schema true is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property having schema false is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "object with both properties is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "dependencies with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentSchemas": { + "foo\tbar": {"minProperties": 4}, + "foo'bar": {"required": ["foo\"bar"]} + } + }, + "tests": [ + { + "description": "quoted tab", + "data": { + "foo\tbar": 1, + "a": 2, + "b": 3, + "c": 4 + }, + "valid": true + }, + { + "description": "quoted quote", + "data": { + "foo'bar": {"foo\"bar": 1} + }, + "valid": false + }, + { + "description": "quoted tab invalid under dependent schema", + "data": { + "foo\tbar": 1, + "a": 2 + }, + "valid": false + }, + { + "description": "quoted quote invalid under dependent schema", + "data": {"foo'bar": 1}, + "valid": false + } + ] + }, + { + "description": "dependent subschema incompatible with root", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {} + }, + "dependentSchemas": { + "foo": { + "properties": { + "bar": {} + }, + "additionalProperties": false + } + } + }, + "tests": [ + { + "description": "matches root", + "data": {"foo": 1}, + "valid": false + }, + { + "description": "matches dependency", + "data": {"bar": 1}, + "valid": true + }, + { + "description": "matches both", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "no dependency", + "data": {"baz": 1}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dynamicRef.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dynamicRef.json new file mode 100644 index 000000000..eba4b441b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dynamicRef.json @@ -0,0 +1,874 @@ +[ + { + "description": "A $dynamicRef to a $dynamicAnchor in the same schema resource behaves like a normal $ref to an $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamicRef-dynamicAnchor-same-schema/root", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + } + } + }, + "tests": [ + { + "description": "An array of strings is valid", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "An array containing non-strings is invalid", + "data": ["foo", 42], + "valid": false + } + ] + }, + { + "description": "A $dynamicRef to an $anchor in the same schema resource behaves like a normal $ref to an $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamicRef-anchor-same-schema/root", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "foo": { + "$anchor": "items", + "type": "string" + } + } + }, + "tests": [ + { + "description": "An array of strings is valid", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "An array containing non-strings is invalid", + "data": ["foo", 42], + "valid": false + } + ] + }, + { + "description": "A $ref to a $dynamicAnchor in the same schema resource behaves like a normal $ref to an $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/ref-dynamicAnchor-same-schema/root", + "type": "array", + "items": { "$ref": "#items" }, + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + } + } + }, + "tests": [ + { + "description": "An array of strings is valid", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "An array containing non-strings is invalid", + "data": ["foo", 42], + "valid": false + } + ] + }, + { + "description": "A $dynamicRef resolves to the first $dynamicAnchor still in scope that is encountered when the schema is evaluated", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/typical-dynamic-resolution/root", + "$ref": "list", + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + }, + "list": { + "$id": "list", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "items": { + "$comment": "This is only needed to satisfy the bookending requirement", + "$dynamicAnchor": "items" + } + } + } + } + }, + "tests": [ + { + "description": "An array of strings is valid", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "An array containing non-strings is invalid", + "data": ["foo", 42], + "valid": false + } + ] + }, + { + "description": "A $dynamicRef without anchor in fragment behaves identical to $ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamicRef-without-anchor/root", + "$ref": "list", + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + }, + "list": { + "$id": "list", + "type": "array", + "items": { "$dynamicRef": "#/$defs/items" }, + "$defs": { + "items": { + "$comment": "This is only needed to satisfy the bookending requirement", + "$dynamicAnchor": "items", + "type": "number" + } + } + } + } + }, + "tests": [ + { + "description": "An array of strings is invalid", + "data": ["foo", "bar"], + "valid": false + }, + { + "description": "An array of numbers is valid", + "data": [24, 42], + "valid": true + } + ] + }, + { + "description": "A $dynamicRef with intermediate scopes that don't include a matching $dynamicAnchor does not affect dynamic scope resolution", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-resolution-with-intermediate-scopes/root", + "$ref": "intermediate-scope", + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + }, + "intermediate-scope": { + "$id": "intermediate-scope", + "$ref": "list" + }, + "list": { + "$id": "list", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "items": { + "$comment": "This is only needed to satisfy the bookending requirement", + "$dynamicAnchor": "items" + } + } + } + } + }, + "tests": [ + { + "description": "An array of strings is valid", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "An array containing non-strings is invalid", + "data": ["foo", 42], + "valid": false + } + ] + }, + { + "description": "An $anchor with the same name as a $dynamicAnchor is not used for dynamic scope resolution", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-resolution-ignores-anchors/root", + "$ref": "list", + "$defs": { + "foo": { + "$anchor": "items", + "type": "string" + }, + "list": { + "$id": "list", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "items": { + "$comment": "This is only needed to satisfy the bookending requirement", + "$dynamicAnchor": "items" + } + } + } + } + }, + "tests": [ + { + "description": "Any array is valid", + "data": ["foo", 42], + "valid": true + } + ] + }, + { + "description": "A $dynamicRef without a matching $dynamicAnchor in the same schema resource behaves like a normal $ref to $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-resolution-without-bookend/root", + "$ref": "list", + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + }, + "list": { + "$id": "list", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "items": { + "$comment": "This is only needed to give the reference somewhere to resolve to when it behaves like $ref", + "$anchor": "items" + } + } + } + } + }, + "tests": [ + { + "description": "Any array is valid", + "data": ["foo", 42], + "valid": true + } + ] + }, + { + "description": "A $dynamicRef with a non-matching $dynamicAnchor in the same schema resource behaves like a normal $ref to $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/unmatched-dynamic-anchor/root", + "$ref": "list", + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + }, + "list": { + "$id": "list", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "items": { + "$comment": "This is only needed to give the reference somewhere to resolve to when it behaves like $ref", + "$anchor": "items", + "$dynamicAnchor": "foo" + } + } + } + } + }, + "tests": [ + { + "description": "Any array is valid", + "data": ["foo", 42], + "valid": true + } + ] + }, + { + "description": "A $dynamicRef that initially resolves to a schema with a matching $dynamicAnchor resolves to the first $dynamicAnchor in the dynamic scope", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/relative-dynamic-reference/root", + "$dynamicAnchor": "meta", + "type": "object", + "properties": { + "foo": { "const": "pass" } + }, + "$ref": "extended", + "$defs": { + "extended": { + "$id": "extended", + "$dynamicAnchor": "meta", + "type": "object", + "properties": { + "bar": { "$ref": "bar" } + } + }, + "bar": { + "$id": "bar", + "type": "object", + "properties": { + "baz": { "$dynamicRef": "extended#meta" } + } + } + } + }, + "tests": [ + { + "description": "The recursive part is valid against the root", + "data": { + "foo": "pass", + "bar": { + "baz": { "foo": "pass" } + } + }, + "valid": true + }, + { + "description": "The recursive part is not valid against the root", + "data": { + "foo": "pass", + "bar": { + "baz": { "foo": "fail" } + } + }, + "valid": false + } + ] + }, + { + "description": "A $dynamicRef that initially resolves to a schema without a matching $dynamicAnchor behaves like a normal $ref to $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/relative-dynamic-reference-without-bookend/root", + "$dynamicAnchor": "meta", + "type": "object", + "properties": { + "foo": { "const": "pass" } + }, + "$ref": "extended", + "$defs": { + "extended": { + "$id": "extended", + "$anchor": "meta", + "type": "object", + "properties": { + "bar": { "$ref": "bar" } + } + }, + "bar": { + "$id": "bar", + "type": "object", + "properties": { + "baz": { "$dynamicRef": "extended#meta" } + } + } + } + }, + "tests": [ + { + "description": "The recursive part doesn't need to validate against the root", + "data": { + "foo": "pass", + "bar": { + "baz": { "foo": "fail" } + } + }, + "valid": true + } + ] + }, + { + "description": "multiple dynamic paths to the $dynamicRef keyword", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-ref-with-multiple-paths/main", + "if": { + "properties": { + "kindOfList": { "const": "numbers" } + }, + "required": ["kindOfList"] + }, + "then": { "$ref": "numberList" }, + "else": { "$ref": "stringList" }, + + "$defs": { + "genericList": { + "$id": "genericList", + "properties": { + "list": { + "items": { "$dynamicRef": "#itemType" } + } + }, + "$defs": { + "defaultItemType": { + "$comment": "Only needed to satisfy bookending requirement", + "$dynamicAnchor": "itemType" + } + } + }, + "numberList": { + "$id": "numberList", + "$defs": { + "itemType": { + "$dynamicAnchor": "itemType", + "type": "number" + } + }, + "$ref": "genericList" + }, + "stringList": { + "$id": "stringList", + "$defs": { + "itemType": { + "$dynamicAnchor": "itemType", + "type": "string" + } + }, + "$ref": "genericList" + } + } + }, + "tests": [ + { + "description": "number list with number values", + "data": { + "kindOfList": "numbers", + "list": [1.1] + }, + "valid": true + }, + { + "description": "number list with string values", + "data": { + "kindOfList": "numbers", + "list": ["foo"] + }, + "valid": false + }, + { + "description": "string list with number values", + "data": { + "kindOfList": "strings", + "list": [1.1] + }, + "valid": false + }, + { + "description": "string list with string values", + "data": { + "kindOfList": "strings", + "list": ["foo"] + }, + "valid": true + } + ] + }, + { + "description": "after leaving a dynamic scope, it is not used by a $dynamicRef", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-ref-leaving-dynamic-scope/main", + "if": { + "$id": "first_scope", + "$defs": { + "thingy": { + "$comment": "this is first_scope#thingy", + "$dynamicAnchor": "thingy", + "type": "number" + } + } + }, + "then": { + "$id": "second_scope", + "$ref": "start", + "$defs": { + "thingy": { + "$comment": "this is second_scope#thingy, the final destination of the $dynamicRef", + "$dynamicAnchor": "thingy", + "type": "null" + } + } + }, + "$defs": { + "start": { + "$comment": "this is the landing spot from $ref", + "$id": "start", + "$dynamicRef": "inner_scope#thingy" + }, + "thingy": { + "$comment": "this is the first stop for the $dynamicRef", + "$id": "inner_scope", + "$dynamicAnchor": "thingy", + "type": "string" + } + } + }, + "tests": [ + { + "description": "string matches /$defs/thingy, but the $dynamicRef does not stop here", + "data": "a string", + "valid": false + }, + { + "description": "first_scope is not in dynamic scope for the $dynamicRef", + "data": 42, + "valid": false + }, + { + "description": "/then/$defs/thingy is the final stop for the $dynamicRef", + "data": null, + "valid": true + } + ] + }, + { + "description": "strict-tree schema, guards against misspelled properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/strict-tree.json", + "$dynamicAnchor": "node", + + "$ref": "tree.json", + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "instance with misspelled field", + "data": { + "children": [{ + "daat": 1 + }] + }, + "valid": false + }, + { + "description": "instance with correct field", + "data": { + "children": [{ + "data": 1 + }] + }, + "valid": true + } + ] + }, + { + "description": "tests for implementation dynamic anchor and reference link", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/strict-extendible.json", + "$ref": "extendible-dynamic-ref.json", + "$defs": { + "elements": { + "$dynamicAnchor": "elements", + "properties": { + "a": true + }, + "required": ["a"], + "additionalProperties": false + } + } + }, + "tests": [ + { + "description": "incorrect parent schema", + "data": { + "a": true + }, + "valid": false + }, + { + "description": "incorrect extended schema", + "data": { + "elements": [ + { "b": 1 } + ] + }, + "valid": false + }, + { + "description": "correct extended schema", + "data": { + "elements": [ + { "a": 1 } + ] + }, + "valid": true + } + ] + }, + { + "description": "$ref and $dynamicAnchor are independent of order - $defs first", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/strict-extendible-allof-defs-first.json", + "allOf": [ + { + "$ref": "extendible-dynamic-ref.json" + }, + { + "$defs": { + "elements": { + "$dynamicAnchor": "elements", + "properties": { + "a": true + }, + "required": ["a"], + "additionalProperties": false + } + } + } + ] + }, + "tests": [ + { + "description": "incorrect parent schema", + "data": { + "a": true + }, + "valid": false + }, + { + "description": "incorrect extended schema", + "data": { + "elements": [ + { "b": 1 } + ] + }, + "valid": false + }, + { + "description": "correct extended schema", + "data": { + "elements": [ + { "a": 1 } + ] + }, + "valid": true + } + ] + }, + { + "description": "$ref and $dynamicAnchor are independent of order - $ref first", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/strict-extendible-allof-ref-first.json", + "allOf": [ + { + "$defs": { + "elements": { + "$dynamicAnchor": "elements", + "properties": { + "a": true + }, + "required": ["a"], + "additionalProperties": false + } + } + }, + { + "$ref": "extendible-dynamic-ref.json" + } + ] + }, + "tests": [ + { + "description": "incorrect parent schema", + "data": { + "a": true + }, + "valid": false + }, + { + "description": "incorrect extended schema", + "data": { + "elements": [ + { "b": 1 } + ] + }, + "valid": false + }, + { + "description": "correct extended schema", + "data": { + "elements": [ + { "a": 1 } + ] + }, + "valid": true + } + ] + }, + { + "description": "$ref to $dynamicRef finds detached $dynamicAnchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/detached-dynamicref.json#/$defs/foo" + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "$dynamicRef points to a boolean schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "true": true, + "false": false + }, + "properties": { + "true": { + "$dynamicRef": "#/$defs/true" + }, + "false": { + "$dynamicRef": "#/$defs/false" + } + } + }, + "tests": [ + { + "description": "follow $dynamicRef to a true schema", + "data": { "true": 1 }, + "valid": true + }, + { + "description": "follow $dynamicRef to a false schema", + "data": { "false": 1 }, + "valid": false + } + ] + }, + { + "description": "$dynamicRef skips over intermediate resources - direct reference", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-ref-skips-intermediate-resource/main", + "type": "object", + "properties": { + "bar-item": { + "$ref": "item" + } + }, + "$defs": { + "bar": { + "$id": "bar", + "type": "array", + "items": { + "$ref": "item" + }, + "$defs": { + "item": { + "$id": "item", + "type": "object", + "properties": { + "content": { + "$dynamicRef": "#content" + } + }, + "$defs": { + "defaultContent": { + "$dynamicAnchor": "content", + "type": "integer" + } + } + }, + "content": { + "$dynamicAnchor": "content", + "type": "string" + } + } + } + } + }, + "tests": [ + { + "description": "integer property passes", + "data": { "bar-item": { "content": 42 } }, + "valid": true + }, + { + "description": "string property fails", + "data": { "bar-item": { "content": "value" } }, + "valid": false + } + ] + }, + { + "description": "$dynamicRef avoids the root of each schema, but scopes are still registered", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-ref-avoids-root-of-each-schema/base", + "$ref": "first#/$defs/stuff", + "$defs": { + "first": { + "$id": "first", + "$defs": { + "stuff": { + "$ref": "second#/$defs/stuff" + }, + "length": { + "$comment": "unused, because there is no $dynamicAnchor here", + "maxLength": 1 + } + } + }, + "second": { + "$id": "second", + "$defs": { + "stuff": { + "$ref": "third#/$defs/stuff" + }, + "length": { + "$dynamicAnchor": "length", + "maxLength": 2 + } + } + }, + "third": { + "$id": "third", + "$defs": { + "stuff": { + "$dynamicRef": "#length" + }, + "length": { + "$dynamicAnchor": "length", + "maxLength": 3 + } + } + } + } + }, + "tests": [ + { + "description": "data is sufficient for schema at second#/$defs/length", + "data": "hi", + "valid": true + }, + { + "description": "data is not sufficient for schema at second#/$defs/length", + "data": "hey", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/enum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/enum.json new file mode 100644 index 000000000..dbc5e0d31 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/enum.json @@ -0,0 +1,419 @@ +[ + { + "description": "simple enum validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [1, 2, 3] + }, + "tests": [ + { + "description": "one of the enum is valid", + "data": 1, + "valid": true + }, + { + "description": "something else is invalid", + "data": 4, + "valid": false + } + ] + }, + { + "description": "heterogeneous enum validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [6, "foo", [], true, {"foo": 12}] + }, + "tests": [ + { + "description": "one of the enum is valid", + "data": [], + "valid": true + }, + { + "description": "something else is invalid", + "data": null, + "valid": false + }, + { + "description": "objects are deep compared", + "data": {"foo": false}, + "valid": false + }, + { + "description": "valid object matches", + "data": {"foo": 12}, + "valid": true + }, + { + "description": "extra properties in object is invalid", + "data": {"foo": 12, "boo": 42}, + "valid": false + } + ] + }, + { + "description": "heterogeneous enum-with-null validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [6, null] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "number is valid", + "data": 6, + "valid": true + }, + { + "description": "something else is invalid", + "data": "test", + "valid": false + } + ] + }, + { + "description": "enums in properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type":"object", + "properties": { + "foo": {"enum":["foo"]}, + "bar": {"enum":["bar"]} + }, + "required": ["bar"] + }, + "tests": [ + { + "description": "both properties are valid", + "data": {"foo":"foo", "bar":"bar"}, + "valid": true + }, + { + "description": "wrong foo value", + "data": {"foo":"foot", "bar":"bar"}, + "valid": false + }, + { + "description": "wrong bar value", + "data": {"foo":"foo", "bar":"bart"}, + "valid": false + }, + { + "description": "missing optional property is valid", + "data": {"bar":"bar"}, + "valid": true + }, + { + "description": "missing required property is invalid", + "data": {"foo":"foo"}, + "valid": false + }, + { + "description": "missing all properties is invalid", + "data": {}, + "valid": false + } + ] + }, + { + "description": "enum with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": ["foo\nbar", "foo\rbar"] + }, + "tests": [ + { + "description": "member 1 is valid", + "data": "foo\nbar", + "valid": true + }, + { + "description": "member 2 is valid", + "data": "foo\rbar", + "valid": true + }, + { + "description": "another string is invalid", + "data": "abc", + "valid": false + } + ] + }, + { + "description": "enum with false does not match 0", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [false] + }, + "tests": [ + { + "description": "false is valid", + "data": false, + "valid": true + }, + { + "description": "integer zero is invalid", + "data": 0, + "valid": false + }, + { + "description": "float zero is invalid", + "data": 0.0, + "valid": false + } + ] + }, + { + "description": "enum with [false] does not match [0]", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [[false]] + }, + "tests": [ + { + "description": "[false] is valid", + "data": [false], + "valid": true + }, + { + "description": "[0] is invalid", + "data": [0], + "valid": false + }, + { + "description": "[0.0] is invalid", + "data": [0.0], + "valid": false + } + ] + }, + { + "description": "enum with true does not match 1", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [true] + }, + "tests": [ + { + "description": "true is valid", + "data": true, + "valid": true + }, + { + "description": "integer one is invalid", + "data": 1, + "valid": false + }, + { + "description": "float one is invalid", + "data": 1.0, + "valid": false + } + ] + }, + { + "description": "enum with [true] does not match [1]", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [[true]] + }, + "tests": [ + { + "description": "[true] is valid", + "data": [true], + "valid": true + }, + { + "description": "[1] is invalid", + "data": [1], + "valid": false + }, + { + "description": "[1.0] is invalid", + "data": [1.0], + "valid": false + } + ] + }, + { + "description": "enum with 0 does not match false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [0] + }, + "tests": [ + { + "description": "false is invalid", + "data": false, + "valid": false + }, + { + "description": "integer zero is valid", + "data": 0, + "valid": true + }, + { + "description": "float zero is valid", + "data": 0.0, + "valid": true + } + ] + }, + { + "description": "enum with [0] does not match [false]", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [[0]] + }, + "tests": [ + { + "description": "[false] is invalid", + "data": [false], + "valid": false + }, + { + "description": "[0] is valid", + "data": [0], + "valid": true + }, + { + "description": "[0.0] is valid", + "data": [0.0], + "valid": true + } + ] + }, + { + "description": "enum with 1 does not match true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [1] + }, + "tests": [ + { + "description": "true is invalid", + "data": true, + "valid": false + }, + { + "description": "integer one is valid", + "data": 1, + "valid": true + }, + { + "description": "float one is valid", + "data": 1.0, + "valid": true + } + ] + }, + { + "description": "enum with [1] does not match [true]", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [[1]] + }, + "tests": [ + { + "description": "[true] is invalid", + "data": [true], + "valid": false + }, + { + "description": "[1] is valid", + "data": [1], + "valid": true + }, + { + "description": "[1.0] is valid", + "data": [1.0], + "valid": true + } + ] + }, + { + "description": "nul characters in strings", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [ "hello\u0000there" ] + }, + "tests": [ + { + "description": "match string with nul", + "data": "hello\u0000there", + "valid": true + }, + { + "description": "do not match string lacking nul", + "data": "hellothere", + "valid": false + } + ] + }, + { + "description": "empty enum", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [] + }, + "tests": [ + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "number is invalid", + "data": 42, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": [], + "valid": false + }, + { + "description": "boolean is invalid", + "data": false, + "valid": false + } + ] + }, + { + "description": "characters with the same visual representation, but different number of codepoints", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": ["ä"], + "$comment": "U+00E4" + }, + "tests": [ + { + "description": "character uses the same codepoint", + "data": "ä", + "comment": "U+00E4", + "valid": true + }, + { + "description": "character looks the same but uses combining marks", + "data": "ä", + "comment": "a, U+0308", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMaximum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMaximum.json new file mode 100644 index 000000000..05db23351 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMaximum.json @@ -0,0 +1,31 @@ +[ + { + "description": "exclusiveMaximum validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "exclusiveMaximum": 3.0 + }, + "tests": [ + { + "description": "below the exclusiveMaximum is valid", + "data": 2.2, + "valid": true + }, + { + "description": "boundary point is invalid", + "data": 3.0, + "valid": false + }, + { + "description": "above the exclusiveMaximum is invalid", + "data": 3.5, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMinimum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMinimum.json new file mode 100644 index 000000000..00af9d7ff --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMinimum.json @@ -0,0 +1,31 @@ +[ + { + "description": "exclusiveMinimum validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "exclusiveMinimum": 1.1 + }, + "tests": [ + { + "description": "above the exclusiveMinimum is valid", + "data": 1.2, + "valid": true + }, + { + "description": "boundary point is invalid", + "data": 1.1, + "valid": false + }, + { + "description": "below the exclusiveMinimum is invalid", + "data": 0.6, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/format.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/format.json new file mode 100644 index 000000000..01adcbda3 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/format.json @@ -0,0 +1,838 @@ +[ + { + "description": "email format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "email" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid email string is only an annotation by default", + "data": "2962", + "valid": true + } + ] + }, + { + "description": "idn-email format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "idn-email" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid idn-email string is only an annotation by default", + "data": "2962", + "valid": true + } + ] + }, + { + "description": "regex format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid regex string is only an annotation by default", + "data": "^(abc]", + "valid": true + } + ] + }, + { + "description": "ipv4 format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "ipv4" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid ipv4 string is only an annotation by default", + "data": "127.0.0.0.1", + "valid": true + } + ] + }, + { + "description": "ipv6 format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "ipv6" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid ipv6 string is only an annotation by default", + "data": "12345::", + "valid": true + } + ] + }, + { + "description": "idn-hostname format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "idn-hostname" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid idn-hostname string is only an annotation by default", + "data": "〮실례.테스트", + "valid": true + } + ] + }, + { + "description": "hostname format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "hostname" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid hostname string is only an annotation by default", + "data": "-a-host-name-that-starts-with--", + "valid": true + } + ] + }, + { + "description": "date format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "date" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid date string is only an annotation by default", + "data": "06/19/1963", + "valid": true + } + ] + }, + { + "description": "date-time format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "date-time" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid date-time string is only an annotation by default", + "data": "1990-02-31T15:59:60.123-08:00", + "valid": true + } + ] + }, + { + "description": "time format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "time" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid time string is only an annotation by default", + "data": "08:30:06 PST", + "valid": true + } + ] + }, + { + "description": "json-pointer format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "json-pointer" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid json-pointer string is only an annotation by default", + "data": "/foo/bar~", + "valid": true + } + ] + }, + { + "description": "relative-json-pointer format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "relative-json-pointer" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid relative-json-pointer string is only an annotation by default", + "data": "/foo/bar", + "valid": true + } + ] + }, + { + "description": "iri format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "iri" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid iri string is only an annotation by default", + "data": "http://2001:0db8:85a3:0000:0000:8a2e:0370:7334", + "valid": true + } + ] + }, + { + "description": "iri-reference format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "iri-reference" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid iri-reference string is only an annotation by default", + "data": "\\\\WINDOWS\\filëßåré", + "valid": true + } + ] + }, + { + "description": "uri format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uri" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid uri string is only an annotation by default", + "data": "//foo.bar/?baz=qux#quux", + "valid": true + } + ] + }, + { + "description": "uri-reference format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uri-reference" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid uri-reference string is only an annotation by default", + "data": "\\\\WINDOWS\\fileshare", + "valid": true + } + ] + }, + { + "description": "uri-template format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uri-template" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid uri-template string is only an annotation by default", + "data": "http://example.com/dictionary/{term:1}/{term", + "valid": true + } + ] + }, + { + "description": "uuid format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uuid" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid uuid string is only an annotation by default", + "data": "2eb8aa08-aa98-11ea-b4aa-73b441d1638", + "valid": true + } + ] + }, + { + "description": "duration format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "duration" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid duration string is only an annotation by default", + "data": "PT1D", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/if-then-else.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/if-then-else.json new file mode 100644 index 000000000..9abf26f71 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/if-then-else.json @@ -0,0 +1,308 @@ +[ + { + "description": "ignore if without then or else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "const": 0 + } + }, + "tests": [ + { + "description": "valid when valid against lone if", + "data": 0, + "valid": true + }, + { + "description": "valid when invalid against lone if", + "data": "hello", + "valid": true + } + ] + }, + { + "description": "ignore then without if", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "then": { + "const": 0 + } + }, + "tests": [ + { + "description": "valid when valid against lone then", + "data": 0, + "valid": true + }, + { + "description": "valid when invalid against lone then", + "data": "hello", + "valid": true + } + ] + }, + { + "description": "ignore else without if", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "else": { + "const": 0 + } + }, + "tests": [ + { + "description": "valid when valid against lone else", + "data": 0, + "valid": true + }, + { + "description": "valid when invalid against lone else", + "data": "hello", + "valid": true + } + ] + }, + { + "description": "if and then without else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "exclusiveMaximum": 0 + }, + "then": { + "minimum": -10 + } + }, + "tests": [ + { + "description": "valid through then", + "data": -1, + "valid": true + }, + { + "description": "invalid through then", + "data": -100, + "valid": false + }, + { + "description": "valid when if test fails", + "data": 3, + "valid": true + } + ] + }, + { + "description": "if and else without then", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "exclusiveMaximum": 0 + }, + "else": { + "multipleOf": 2 + } + }, + "tests": [ + { + "description": "valid when if test passes", + "data": -1, + "valid": true + }, + { + "description": "valid through else", + "data": 4, + "valid": true + }, + { + "description": "invalid through else", + "data": 3, + "valid": false + } + ] + }, + { + "description": "validate against correct branch, then vs else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "exclusiveMaximum": 0 + }, + "then": { + "minimum": -10 + }, + "else": { + "multipleOf": 2 + } + }, + "tests": [ + { + "description": "valid through then", + "data": -1, + "valid": true + }, + { + "description": "invalid through then", + "data": -100, + "valid": false + }, + { + "description": "valid through else", + "data": 4, + "valid": true + }, + { + "description": "invalid through else", + "data": 3, + "valid": false + } + ] + }, + { + "description": "non-interference across combined schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "if": { + "exclusiveMaximum": 0 + } + }, + { + "then": { + "minimum": -10 + } + }, + { + "else": { + "multipleOf": 2 + } + } + ] + }, + "tests": [ + { + "description": "valid, but would have been invalid through then", + "data": -100, + "valid": true + }, + { + "description": "valid, but would have been invalid through else", + "data": 3, + "valid": true + } + ] + }, + { + "description": "if with boolean schema true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": true, + "then": { "const": "then" }, + "else": { "const": "else" } + }, + "tests": [ + { + "description": "boolean schema true in if always chooses the then path (valid)", + "data": "then", + "valid": true + }, + { + "description": "boolean schema true in if always chooses the then path (invalid)", + "data": "else", + "valid": false + } + ] + }, + { + "description": "if with boolean schema false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": false, + "then": { "const": "then" }, + "else": { "const": "else" } + }, + "tests": [ + { + "description": "boolean schema false in if always chooses the else path (invalid)", + "data": "then", + "valid": false + }, + { + "description": "boolean schema false in if always chooses the else path (valid)", + "data": "else", + "valid": true + } + ] + }, + { + "description": "if appears at the end when serialized (keyword processing sequence)", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "then": { "const": "yes" }, + "else": { "const": "other" }, + "if": { "maxLength": 4 } + }, + "tests": [ + { + "description": "yes redirects to then and passes", + "data": "yes", + "valid": true + }, + { + "description": "other redirects to else and passes", + "data": "other", + "valid": true + }, + { + "description": "no redirects to then and fails", + "data": "no", + "valid": false + }, + { + "description": "invalid redirects to else and fails", + "data": "invalid", + "valid": false + } + ] + }, + { + "description": "then: false fails when condition matches", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { "const": 1 }, + "then": false + }, + "tests": [ + { + "description": "matches if → then=false → invalid", + "data": 1, + "valid": false + }, + { + "description": "does not match if → then ignored → valid", + "data": 2, + "valid": true + } + ] + }, + { + "description": "else: false fails when condition does not match", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { "const": 1 }, + "else": false + }, + "tests": [ + { + "description": "matches if → else ignored → valid", + "data": 1, + "valid": true + }, + { + "description": "does not match if → else executes → invalid", + "data": 2, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/infinite-loop-detection.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/infinite-loop-detection.json new file mode 100644 index 000000000..46f157a35 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/infinite-loop-detection.json @@ -0,0 +1,37 @@ +[ + { + "description": "evaluating the same schema location against the same data location twice is not a sign of an infinite loop", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "int": { "type": "integer" } + }, + "allOf": [ + { + "properties": { + "foo": { + "$ref": "#/$defs/int" + } + } + }, + { + "additionalProperties": { + "$ref": "#/$defs/int" + } + } + ] + }, + "tests": [ + { + "description": "passing case", + "data": { "foo": 1 }, + "valid": true + }, + { + "description": "failing case", + "data": { "foo": "a string" }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/items.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/items.json new file mode 100644 index 000000000..6a3e1cf26 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/items.json @@ -0,0 +1,304 @@ +[ + { + "description": "a schema given for items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": {"type": "integer"} + }, + "tests": [ + { + "description": "valid items", + "data": [ 1, 2, 3 ], + "valid": true + }, + { + "description": "wrong type of items", + "data": [1, "x"], + "valid": false + }, + { + "description": "ignores non-arrays", + "data": {"foo" : "bar"}, + "valid": true + }, + { + "description": "JavaScript pseudo-array is valid", + "data": { + "0": "invalid", + "length": 1 + }, + "valid": true + } + ] + }, + { + "description": "items with boolean schema (true)", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": true + }, + "tests": [ + { + "description": "any array is valid", + "data": [ 1, "foo", true ], + "valid": true + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "items with boolean schema (false)", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": false + }, + "tests": [ + { + "description": "any non-empty array is invalid", + "data": [ 1, "foo", true ], + "valid": false + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "items and subitems", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "item": { + "type": "array", + "items": false, + "prefixItems": [ + { "$ref": "#/$defs/sub-item" }, + { "$ref": "#/$defs/sub-item" } + ] + }, + "sub-item": { + "type": "object", + "required": ["foo"] + } + }, + "type": "array", + "items": false, + "prefixItems": [ + { "$ref": "#/$defs/item" }, + { "$ref": "#/$defs/item" }, + { "$ref": "#/$defs/item" } + ] + }, + "tests": [ + { + "description": "valid items", + "data": [ + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": true + }, + { + "description": "too many items", + "data": [ + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "too many sub-items", + "data": [ + [ {"foo": null}, {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "wrong item", + "data": [ + {"foo": null}, + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "wrong sub-item", + "data": [ + [ {}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "fewer items is valid", + "data": [ + [ {"foo": null} ], + [ {"foo": null} ] + ], + "valid": true + } + ] + }, + { + "description": "nested items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "array", + "items": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "tests": [ + { + "description": "valid nested array", + "data": [[[[1]], [[2],[3]]], [[[4], [5], [6]]]], + "valid": true + }, + { + "description": "nested array with invalid type", + "data": [[[["1"]], [[2],[3]]], [[[4], [5], [6]]]], + "valid": false + }, + { + "description": "not deep enough", + "data": [[[1], [2],[3]], [[4], [5], [6]]], + "valid": false + } + ] + }, + { + "description": "prefixItems with no additional items allowed", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{}, {}, {}], + "items": false + }, + "tests": [ + { + "description": "empty array", + "data": [ ], + "valid": true + }, + { + "description": "fewer number of items present (1)", + "data": [ 1 ], + "valid": true + }, + { + "description": "fewer number of items present (2)", + "data": [ 1, 2 ], + "valid": true + }, + { + "description": "equal number of items present", + "data": [ 1, 2, 3 ], + "valid": true + }, + { + "description": "additional items are not permitted", + "data": [ 1, 2, 3, 4 ], + "valid": false + } + ] + }, + { + "description": "items does not look in applicators, valid case", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { "prefixItems": [ { "minimum": 3 } ] } + ], + "items": { "minimum": 5 } + }, + "tests": [ + { + "description": "prefixItems in allOf does not constrain items, invalid case", + "data": [ 3, 5 ], + "valid": false + }, + { + "description": "prefixItems in allOf does not constrain items, valid case", + "data": [ 5, 5 ], + "valid": true + } + ] + }, + { + "description": "prefixItems validation adjusts the starting index for items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ { "type": "string" } ], + "items": { "type": "integer" } + }, + "tests": [ + { + "description": "valid items", + "data": [ "x", 2, 3 ], + "valid": true + }, + { + "description": "wrong type of second item", + "data": [ "x", "y" ], + "valid": false + } + ] + }, + { + "description": "items with heterogeneous array", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{}], + "items": false + }, + "tests": [ + { + "description": "heterogeneous invalid instance", + "data": [ "foo", "bar", 37 ], + "valid": false + }, + { + "description": "valid instance", + "data": [ null ], + "valid": true + } + ] + }, + { + "description": "items with null instance elements", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null elements", + "data": [ null ], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxContains.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxContains.json new file mode 100644 index 000000000..9a42b45d6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxContains.json @@ -0,0 +1,123 @@ +[ + { + "description": "maxContains without contains is ignored", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxContains": 1 + }, + "tests": [ + { + "description": "one item valid against lone maxContains", + "data": [ 1 ], + "valid": true + }, + { + "description": "two items still valid against lone maxContains", + "data": [ 1, 2 ], + "valid": true + } + ] + }, + { + "description": "maxContains with contains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "maxContains": 1 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": false + }, + { + "description": "all elements match, valid maxContains", + "data": [ 1 ], + "valid": true + }, + { + "description": "all elements match, invalid maxContains", + "data": [ 1, 1 ], + "valid": false + }, + { + "description": "some elements match, valid maxContains", + "data": [ 1, 2 ], + "valid": true + }, + { + "description": "some elements match, invalid maxContains", + "data": [ 1, 2, 1 ], + "valid": false + } + ] + }, + { + "description": "maxContains with contains, value with a decimal", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "maxContains": 1.0 + }, + "tests": [ + { + "description": "one element matches, valid maxContains", + "data": [ 1 ], + "valid": true + }, + { + "description": "too many elements match, invalid maxContains", + "data": [ 1, 1 ], + "valid": false + } + ] + }, + { + "description": "minContains < maxContains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 1, + "maxContains": 3 + }, + "tests": [ + { + "description": "actual < minContains < maxContains", + "data": [ ], + "valid": false + }, + { + "description": "minContains < actual < maxContains", + "data": [ 1, 1 ], + "valid": true + }, + { + "description": "minContains < maxContains < actual", + "data": [ 1, 1, 1, 1 ], + "valid": false + } + ] + }, + { + "description": "maxContains = 0 with minContains = 0", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 0, + "maxContains": 0 + }, + "tests": [ + { + "description": "empty array", + "data": [ ], + "valid": true + }, + { + "description": "one matching item", + "data": [ 1 ], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxItems.json new file mode 100644 index 000000000..f6a6b7c9a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxItems.json @@ -0,0 +1,50 @@ +[ + { + "description": "maxItems validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxItems": 2 + }, + "tests": [ + { + "description": "shorter is valid", + "data": [1], + "valid": true + }, + { + "description": "exact length is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "too long is invalid", + "data": [1, 2, 3], + "valid": false + }, + { + "description": "ignores non-arrays", + "data": "foobar", + "valid": true + } + ] + }, + { + "description": "maxItems validation with a decimal", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxItems": 2.0 + }, + "tests": [ + { + "description": "shorter is valid", + "data": [1], + "valid": true + }, + { + "description": "too long is invalid", + "data": [1, 2, 3], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxLength.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxLength.json new file mode 100644 index 000000000..7462726d7 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxLength.json @@ -0,0 +1,55 @@ +[ + { + "description": "maxLength validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxLength": 2 + }, + "tests": [ + { + "description": "shorter is valid", + "data": "f", + "valid": true + }, + { + "description": "exact length is valid", + "data": "fo", + "valid": true + }, + { + "description": "too long is invalid", + "data": "foo", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + }, + { + "description": "two graphemes is long enough", + "data": "\uD83D\uDCA9\uD83D\uDCA9", + "valid": true + } + ] + }, + { + "description": "maxLength validation with a decimal", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxLength": 2.0 + }, + "tests": [ + { + "description": "shorter is valid", + "data": "f", + "valid": true + }, + { + "description": "too long is invalid", + "data": "foo", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxProperties.json new file mode 100644 index 000000000..9cb745cad --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxProperties.json @@ -0,0 +1,94 @@ +[ + { + "description": "maxProperties validation", + "specification": [ + { + "validation": "6.5.1" + } + ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxProperties": 2 + }, + "tests": [ + { + "description": "shorter is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "exact length is valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "too long is invalid", + "data": {"foo": 1, "bar": 2, "baz": 3}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [1, 2, 3], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "maxProperties validation with a decimal", + "specification": [ + { + "validation": "6.5.1" + } + ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxProperties": 2.0 + }, + "tests": [ + { + "description": "shorter is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "too long is invalid", + "data": {"foo": 1, "bar": 2, "baz": 3}, + "valid": false + } + ] + }, + { + "description": "maxProperties = 0 means the object is empty", + "specification": [ + { + "validation": "6.5.1" + } + ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxProperties": 0 + }, + "tests": [ + { + "description": "no properties is valid", + "data": {}, + "valid": true + }, + { + "description": "one property is invalid", + "data": { "foo": 1 }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maximum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maximum.json new file mode 100644 index 000000000..7eeab2877 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maximum.json @@ -0,0 +1,60 @@ +[ + { + "description": "maximum validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maximum": 3.0 + }, + "tests": [ + { + "description": "below the maximum is valid", + "data": 2.6, + "valid": true + }, + { + "description": "boundary point is valid", + "data": 3.0, + "valid": true + }, + { + "description": "above the maximum is invalid", + "data": 3.5, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + }, + { + "description": "maximum validation with unsigned integer", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maximum": 300 + }, + "tests": [ + { + "description": "below the maximum is valid", + "data": 299.97, + "valid": true + }, + { + "description": "boundary point integer is valid", + "data": 300, + "valid": true + }, + { + "description": "boundary point float is valid", + "data": 300.00, + "valid": true + }, + { + "description": "above the maximum is invalid", + "data": 300.5, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minContains.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minContains.json new file mode 100644 index 000000000..ee72d7d62 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minContains.json @@ -0,0 +1,224 @@ +[ + { + "description": "minContains without contains is ignored", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minContains": 1 + }, + "tests": [ + { + "description": "one item valid against lone minContains", + "data": [ 1 ], + "valid": true + }, + { + "description": "zero items still valid against lone minContains", + "data": [], + "valid": true + } + ] + }, + { + "description": "minContains=1 with contains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 1 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": false + }, + { + "description": "no elements match", + "data": [ 2 ], + "valid": false + }, + { + "description": "single element matches, valid minContains", + "data": [ 1 ], + "valid": true + }, + { + "description": "some elements match, valid minContains", + "data": [ 1, 2 ], + "valid": true + }, + { + "description": "all elements match, valid minContains", + "data": [ 1, 1 ], + "valid": true + } + ] + }, + { + "description": "minContains=2 with contains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 2 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": false + }, + { + "description": "all elements match, invalid minContains", + "data": [ 1 ], + "valid": false + }, + { + "description": "some elements match, invalid minContains", + "data": [ 1, 2 ], + "valid": false + }, + { + "description": "all elements match, valid minContains (exactly as needed)", + "data": [ 1, 1 ], + "valid": true + }, + { + "description": "all elements match, valid minContains (more than needed)", + "data": [ 1, 1, 1 ], + "valid": true + }, + { + "description": "some elements match, valid minContains", + "data": [ 1, 2, 1 ], + "valid": true + } + ] + }, + { + "description": "minContains=2 with contains with a decimal value", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 2.0 + }, + "tests": [ + { + "description": "one element matches, invalid minContains", + "data": [ 1 ], + "valid": false + }, + { + "description": "both elements match, valid minContains", + "data": [ 1, 1 ], + "valid": true + } + ] + }, + { + "description": "maxContains = minContains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "maxContains": 2, + "minContains": 2 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": false + }, + { + "description": "all elements match, invalid minContains", + "data": [ 1 ], + "valid": false + }, + { + "description": "all elements match, invalid maxContains", + "data": [ 1, 1, 1 ], + "valid": false + }, + { + "description": "all elements match, valid maxContains and minContains", + "data": [ 1, 1 ], + "valid": true + } + ] + }, + { + "description": "maxContains < minContains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "maxContains": 1, + "minContains": 3 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": false + }, + { + "description": "invalid minContains", + "data": [ 1 ], + "valid": false + }, + { + "description": "invalid maxContains", + "data": [ 1, 1, 1 ], + "valid": false + }, + { + "description": "invalid maxContains and minContains", + "data": [ 1, 1 ], + "valid": false + } + ] + }, + { + "description": "minContains = 0", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 0 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": true + }, + { + "description": "minContains = 0 makes contains always pass", + "data": [ 2 ], + "valid": true + } + ] + }, + { + "description": "minContains = 0 with maxContains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 0, + "maxContains": 1 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": true + }, + { + "description": "not more than maxContains", + "data": [ 1 ], + "valid": true + }, + { + "description": "too many", + "data": [ 1, 1 ], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minItems.json new file mode 100644 index 000000000..9d6a8b6d2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minItems.json @@ -0,0 +1,50 @@ +[ + { + "description": "minItems validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minItems": 1 + }, + "tests": [ + { + "description": "longer is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "exact length is valid", + "data": [1], + "valid": true + }, + { + "description": "too short is invalid", + "data": [], + "valid": false + }, + { + "description": "ignores non-arrays", + "data": "", + "valid": true + } + ] + }, + { + "description": "minItems validation with a decimal", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minItems": 1.0 + }, + "tests": [ + { + "description": "longer is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "too short is invalid", + "data": [], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minLength.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minLength.json new file mode 100644 index 000000000..5076c5a92 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minLength.json @@ -0,0 +1,55 @@ +[ + { + "description": "minLength validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minLength": 2 + }, + "tests": [ + { + "description": "longer is valid", + "data": "foo", + "valid": true + }, + { + "description": "exact length is valid", + "data": "fo", + "valid": true + }, + { + "description": "too short is invalid", + "data": "f", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 1, + "valid": true + }, + { + "description": "one grapheme is not long enough", + "data": "\uD83D\uDCA9", + "valid": false + } + ] + }, + { + "description": "minLength validation with a decimal", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minLength": 2.0 + }, + "tests": [ + { + "description": "longer is valid", + "data": "foo", + "valid": true + }, + { + "description": "too short is invalid", + "data": "f", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minProperties.json new file mode 100644 index 000000000..12587a5f8 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minProperties.json @@ -0,0 +1,70 @@ +[ + { + "description": "minProperties validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minProperties": 1 + }, + "tests": [ + { + "description": "longer is valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "exact length is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "too short is invalid", + "data": {}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores strings", + "data": "", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + }, + { + "description": "ignores booleans", + "data": true, + "valid": true + } + ] + }, + { + "description": "minProperties validation with a decimal", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minProperties": 1.0 + }, + "tests": [ + { + "description": "longer is valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "too short is invalid", + "data": {}, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minimum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minimum.json new file mode 100644 index 000000000..dc4405278 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minimum.json @@ -0,0 +1,75 @@ +[ + { + "description": "minimum validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minimum": 1.1 + }, + "tests": [ + { + "description": "above the minimum is valid", + "data": 2.6, + "valid": true + }, + { + "description": "boundary point is valid", + "data": 1.1, + "valid": true + }, + { + "description": "below the minimum is invalid", + "data": 0.6, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + }, + { + "description": "minimum validation with signed integer", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minimum": -2 + }, + "tests": [ + { + "description": "negative above the minimum is valid", + "data": -1, + "valid": true + }, + { + "description": "positive above the minimum is valid", + "data": 0, + "valid": true + }, + { + "description": "boundary point is valid", + "data": -2, + "valid": true + }, + { + "description": "boundary point with float is valid", + "data": -2.0, + "valid": true + }, + { + "description": "float below the minimum is invalid", + "data": -2.0001, + "valid": false + }, + { + "description": "int below the minimum is invalid", + "data": -3, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/multipleOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/multipleOf.json new file mode 100644 index 000000000..76fa4765b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/multipleOf.json @@ -0,0 +1,102 @@ +[ + { + "description": "by int", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "multipleOf": 2 + }, + "tests": [ + { + "description": "int by int", + "data": 10, + "valid": true + }, + { + "description": "int by int fail", + "data": 7, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "by number", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "multipleOf": 1.5 + }, + "tests": [ + { + "description": "zero is multiple of anything", + "data": 0, + "valid": true + }, + { + "description": "4.5 is multiple of 1.5", + "data": 4.5, + "valid": true + }, + { + "description": "-4.5 is multiple of 1.5", + "data": -4.5, + "valid": true + }, + { + "description": "35 is not multiple of 1.5", + "data": 35, + "valid": false + } + ] + }, + { + "description": "by small number", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "multipleOf": 0.0001 + }, + "tests": [ + { + "description": "0.0075 is multiple of 0.0001", + "data": 0.0075, + "valid": true + }, + { + "description": "0.00751 is not multiple of 0.0001", + "data": 0.00751, + "valid": false + } + ] + }, + { + "description": "float division = inf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer", "multipleOf": 0.123456789 + }, + "tests": [ + { + "description": "always invalid, but naive implementations may raise an overflow error", + "data": 1e308, + "valid": false + } + ] + }, + { + "description": "small multiple of large integer", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer", "multipleOf": 1e-8 + }, + "tests": [ + { + "description": "any integer is a multiple of 1e-8", + "data": 12391239123, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/not.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/not.json new file mode 100644 index 000000000..d0f2b6e84 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/not.json @@ -0,0 +1,301 @@ +[ + { + "description": "not", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": {"type": "integer"} + }, + "tests": [ + { + "description": "allowed", + "data": "foo", + "valid": true + }, + { + "description": "disallowed", + "data": 1, + "valid": false + } + ] + }, + { + "description": "not multiple types", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": {"type": ["integer", "boolean"]} + }, + "tests": [ + { + "description": "valid", + "data": "foo", + "valid": true + }, + { + "description": "mismatch", + "data": 1, + "valid": false + }, + { + "description": "other mismatch", + "data": true, + "valid": false + } + ] + }, + { + "description": "not more complex schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": { + "type": "object", + "properties": { + "foo": { + "type": "string" + } + } + } + }, + "tests": [ + { + "description": "match", + "data": 1, + "valid": true + }, + { + "description": "other match", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "mismatch", + "data": {"foo": "bar"}, + "valid": false + } + ] + }, + { + "description": "forbidden property", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "not": {} + } + } + }, + "tests": [ + { + "description": "property present", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "property absent", + "data": {"bar": 1, "baz": 2}, + "valid": true + } + ] + }, + { + "description": "forbid everything with empty schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": {} + }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "boolean true is invalid", + "data": true, + "valid": false + }, + { + "description": "boolean false is invalid", + "data": false, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "forbid everything with boolean schema true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": true + }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "boolean true is invalid", + "data": true, + "valid": false + }, + { + "description": "boolean false is invalid", + "data": false, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "allow everything with boolean schema false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": false + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "boolean true is valid", + "data": true, + "valid": true + }, + { + "description": "boolean false is valid", + "data": false, + "valid": true + }, + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + }, + { + "description": "array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "double negation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": { "not": {} } + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "collect annotations inside a 'not', even if collection is disabled", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": { + "$comment": "this subschema must still produce annotations internally, even though the 'not' will ultimately discard them", + "anyOf": [ + true, + { "properties": { "foo": true } } + ], + "unevaluatedProperties": false + } + }, + "tests": [ + { + "description": "unevaluated property", + "data": { "bar": 1 }, + "valid": true + }, + { + "description": "annotations are still collected inside a 'not'", + "data": { "foo": 1 }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/oneOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/oneOf.json new file mode 100644 index 000000000..7a7c7ffe3 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/oneOf.json @@ -0,0 +1,293 @@ +[ + { + "description": "oneOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [ + { + "type": "integer" + }, + { + "minimum": 2 + } + ] + }, + "tests": [ + { + "description": "first oneOf valid", + "data": 1, + "valid": true + }, + { + "description": "second oneOf valid", + "data": 2.5, + "valid": true + }, + { + "description": "both oneOf valid", + "data": 3, + "valid": false + }, + { + "description": "neither oneOf valid", + "data": 1.5, + "valid": false + } + ] + }, + { + "description": "oneOf with base schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "oneOf" : [ + { + "minLength": 2 + }, + { + "maxLength": 4 + } + ] + }, + "tests": [ + { + "description": "mismatch base schema", + "data": 3, + "valid": false + }, + { + "description": "one oneOf valid", + "data": "foobar", + "valid": true + }, + { + "description": "both oneOf valid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf with boolean schemas, all true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [true, true, true] + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf with boolean schemas, one true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [true, false, false] + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "oneOf with boolean schemas, more than one true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [true, true, false] + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf with boolean schemas, all false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [false, false, false] + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf complex types", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [ + { + "properties": { + "bar": {"type": "integer"} + }, + "required": ["bar"] + }, + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "first oneOf valid (complex)", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "second oneOf valid (complex)", + "data": {"foo": "baz"}, + "valid": true + }, + { + "description": "both oneOf valid (complex)", + "data": {"foo": "baz", "bar": 2}, + "valid": false + }, + { + "description": "neither oneOf valid (complex)", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "oneOf with empty schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [ + { "type": "number" }, + {} + ] + }, + "tests": [ + { + "description": "one valid - valid", + "data": "foo", + "valid": true + }, + { + "description": "both valid - invalid", + "data": 123, + "valid": false + } + ] + }, + { + "description": "oneOf with required", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "oneOf": [ + { "required": ["foo", "bar"] }, + { "required": ["foo", "baz"] } + ] + }, + "tests": [ + { + "description": "both invalid - invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "first valid - valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "second valid - valid", + "data": {"foo": 1, "baz": 3}, + "valid": true + }, + { + "description": "both valid - invalid", + "data": {"foo": 1, "bar": 2, "baz" : 3}, + "valid": false + } + ] + }, + { + "description": "oneOf with missing optional property", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [ + { + "properties": { + "bar": true, + "baz": true + }, + "required": ["bar"] + }, + { + "properties": { + "foo": true + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "first oneOf valid", + "data": {"bar": 8}, + "valid": true + }, + { + "description": "second oneOf valid", + "data": {"foo": "foo"}, + "valid": true + }, + { + "description": "both oneOf valid", + "data": {"foo": "foo", "bar": 8}, + "valid": false + }, + { + "description": "neither oneOf valid", + "data": {"baz": "quux"}, + "valid": false + } + ] + }, + { + "description": "nested oneOf, to check validation semantics", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [ + { + "oneOf": [ + { + "type": "null" + } + ] + } + ] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "anything non-null is invalid", + "data": 123, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/anchor.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/anchor.json new file mode 100644 index 000000000..6d6713be5 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/anchor.json @@ -0,0 +1,60 @@ +[ + { + "description": "$anchor inside an enum is not a real identifier", + "comment": "the implementation must not be confused by an $anchor buried in the enum", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "anchor_in_enum": { + "enum": [ + { + "$anchor": "my_anchor", + "type": "null" + } + ] + }, + "real_identifier_in_schema": { + "$anchor": "my_anchor", + "type": "string" + }, + "zzz_anchor_in_const": { + "const": { + "$anchor": "my_anchor", + "type": "null" + } + } + }, + "anyOf": [ + { "$ref": "#/$defs/anchor_in_enum" }, + { "$ref": "#my_anchor" } + ] + }, + "tests": [ + { + "description": "exact match to enum, and type matches", + "data": { + "$anchor": "my_anchor", + "type": "null" + }, + "valid": true + }, + { + "description": "in implementations that strip $anchor, this may match either $def", + "data": { + "type": "null" + }, + "valid": false + }, + { + "description": "match $ref to $anchor", + "data": "a string to match #/$defs/anchor_in_enum", + "valid": true + }, + { + "description": "no match on enum or $ref to $anchor", + "data": 1, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/bignum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/bignum.json new file mode 100644 index 000000000..d69b29e85 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/bignum.json @@ -0,0 +1,110 @@ +[ + { + "description": "integer", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer" + }, + "tests": [ + { + "description": "a bignum is an integer", + "data": 12345678910111213141516171819202122232425262728293031, + "valid": true + }, + { + "description": "a negative bignum is an integer", + "data": -12345678910111213141516171819202122232425262728293031, + "valid": true + } + ] + }, + { + "description": "number", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "number" + }, + "tests": [ + { + "description": "a bignum is a number", + "data": 98249283749234923498293171823948729348710298301928331, + "valid": true + }, + { + "description": "a negative bignum is a number", + "data": -98249283749234923498293171823948729348710298301928331, + "valid": true + } + ] + }, + { + "description": "string", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string" + }, + "tests": [ + { + "description": "a bignum is not a string", + "data": 98249283749234923498293171823948729348710298301928331, + "valid": false + } + ] + }, + { + "description": "maximum integer comparison", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maximum": 18446744073709551615 + }, + "tests": [ + { + "description": "comparison works for high numbers", + "data": 18446744073709551600, + "valid": true + } + ] + }, + { + "description": "float comparison with high precision", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "exclusiveMaximum": 972783798187987123879878123.18878137 + }, + "tests": [ + { + "description": "comparison works for high numbers", + "data": 972783798187987123879878123.188781371, + "valid": false + } + ] + }, + { + "description": "minimum integer comparison", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minimum": -18446744073709551615 + }, + "tests": [ + { + "description": "comparison works for very negative numbers", + "data": -18446744073709551600, + "valid": true + } + ] + }, + { + "description": "float comparison with high precision on negative numbers", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "exclusiveMinimum": -972783798187987123879878123.18878137 + }, + "tests": [ + { + "description": "comparison works for very negative numbers", + "data": -972783798187987123879878123.188781371, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/cross-draft.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/cross-draft.json new file mode 100644 index 000000000..5113bd649 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/cross-draft.json @@ -0,0 +1,18 @@ +[ + { + "description": "refs to historic drafts are processed as historic drafts", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "array", + "$ref": "http://localhost:1234/draft2019-09/ignore-prefixItems.json" + }, + "tests": [ + { + "description": "first item not a string is valid", + "comment": "if the implementation is not processing the $ref as a 2019-09 schema, this test will fail", + "data": [1, 2, 3], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dependencies-compatibility.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dependencies-compatibility.json new file mode 100644 index 000000000..47d5bd79d --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dependencies-compatibility.json @@ -0,0 +1,282 @@ +[ + { + "description": "single dependency", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": {"bar": ["foo"]} + }, + "tests": [ + { + "description": "neither", + "data": {}, + "valid": true + }, + { + "description": "nondependant", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "with dependency", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "missing dependency", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["bar"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "empty dependents", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": {"bar": []} + }, + "tests": [ + { + "description": "empty object", + "data": {}, + "valid": true + }, + { + "description": "object with one property", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "non-object is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "multiple dependents required", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": {"quux": ["foo", "bar"]} + }, + "tests": [ + { + "description": "neither", + "data": {}, + "valid": true + }, + { + "description": "nondependants", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "with dependencies", + "data": {"foo": 1, "bar": 2, "quux": 3}, + "valid": true + }, + { + "description": "missing dependency", + "data": {"foo": 1, "quux": 2}, + "valid": false + }, + { + "description": "missing other dependency", + "data": {"bar": 1, "quux": 2}, + "valid": false + }, + { + "description": "missing both dependencies", + "data": {"quux": 1}, + "valid": false + } + ] + }, + { + "description": "dependencies with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": { + "foo\nbar": ["foo\rbar"], + "foo\"bar": ["foo'bar"] + } + }, + "tests": [ + { + "description": "CRLF", + "data": { + "foo\nbar": 1, + "foo\rbar": 2 + }, + "valid": true + }, + { + "description": "quoted quotes", + "data": { + "foo'bar": 1, + "foo\"bar": 2 + }, + "valid": true + }, + { + "description": "CRLF missing dependent", + "data": { + "foo\nbar": 1, + "foo": 2 + }, + "valid": false + }, + { + "description": "quoted quotes missing dependent", + "data": { + "foo\"bar": 2 + }, + "valid": false + } + ] + }, + { + "description": "single schema dependency", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": { + "bar": { + "properties": { + "foo": {"type": "integer"}, + "bar": {"type": "integer"} + } + } + } + }, + "tests": [ + { + "description": "valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "no dependency", + "data": {"foo": "quux"}, + "valid": true + }, + { + "description": "wrong type", + "data": {"foo": "quux", "bar": 2}, + "valid": false + }, + { + "description": "wrong type other", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + }, + { + "description": "wrong type both", + "data": {"foo": "quux", "bar": "quux"}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["bar"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "boolean subschemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": { + "foo": true, + "bar": false + } + }, + "tests": [ + { + "description": "object with property having schema true is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property having schema false is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "object with both properties is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "schema dependencies with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": { + "foo\tbar": {"minProperties": 4}, + "foo'bar": {"required": ["foo\"bar"]} + } + }, + "tests": [ + { + "description": "quoted tab", + "data": { + "foo\tbar": 1, + "a": 2, + "b": 3, + "c": 4 + }, + "valid": true + }, + { + "description": "quoted quote", + "data": { + "foo'bar": {"foo\"bar": 1} + }, + "valid": false + }, + { + "description": "quoted tab invalid under dependent schema", + "data": { + "foo\tbar": 1, + "a": 2 + }, + "valid": false + }, + { + "description": "quoted quote invalid under dependent schema", + "data": {"foo'bar": 1}, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dynamicRef.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dynamicRef.json new file mode 100644 index 000000000..7e63f209a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dynamicRef.json @@ -0,0 +1,56 @@ +[ + { + "description": "$dynamicRef skips over intermediate resources - pointer reference across resource boundary", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-ref-skips-intermediate-resource/optional/main", + "type": "object", + "properties": { + "bar-item": { + "$ref": "bar#/$defs/item" + } + }, + "$defs": { + "bar": { + "$id": "bar", + "type": "array", + "items": { + "$ref": "item" + }, + "$defs": { + "item": { + "$id": "item", + "type": "object", + "properties": { + "content": { + "$dynamicRef": "#content" + } + }, + "$defs": { + "defaultContent": { + "$dynamicAnchor": "content", + "type": "integer" + } + } + }, + "content": { + "$dynamicAnchor": "content", + "type": "string" + } + } + } + } + }, + "tests": [ + { + "description": "integer property passes", + "data": { "bar-item": { "content": 42 } }, + "valid": true + }, + { + "description": "string property fails", + "data": { "bar-item": { "content": "value" } }, + "valid": false + } + ] + }] \ No newline at end of file diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/ecmascript-regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/ecmascript-regex.json new file mode 100644 index 000000000..a4d62e0cf --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/ecmascript-regex.json @@ -0,0 +1,582 @@ +[ + { + "description": "ECMA 262 regex $ does not match trailing newline", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^abc$" + }, + "tests": [ + { + "description": "matches in Python, but not in ECMA 262", + "data": "abc\\n", + "valid": false + }, + { + "description": "matches", + "data": "abc", + "valid": true + } + ] + }, + { + "description": "ECMA 262 regex converts \\t to horizontal tab", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\t$" + }, + "tests": [ + { + "description": "does not match", + "data": "\\t", + "valid": false + }, + { + "description": "matches", + "data": "\u0009", + "valid": true + } + ] + }, + { + "description": "ECMA 262 regex escapes control codes with \\c and upper letter", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\cC$" + }, + "tests": [ + { + "description": "does not match", + "data": "\\cC", + "valid": false + }, + { + "description": "matches", + "data": "\u0003", + "valid": true + } + ] + }, + { + "description": "ECMA 262 regex escapes control codes with \\c and lower letter", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\cc$" + }, + "tests": [ + { + "description": "does not match", + "data": "\\cc", + "valid": false + }, + { + "description": "matches", + "data": "\u0003", + "valid": true + } + ] + }, + { + "description": "ECMA 262 \\d matches ascii digits only", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\d$" + }, + "tests": [ + { + "description": "ASCII zero matches", + "data": "0", + "valid": true + }, + { + "description": "NKO DIGIT ZERO does not match (unlike e.g. Python)", + "data": "߀", + "valid": false + }, + { + "description": "NKO DIGIT ZERO (as \\u escape) does not match", + "data": "\u07c0", + "valid": false + } + ] + }, + { + "description": "ECMA 262 \\D matches everything but ascii digits", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\D$" + }, + "tests": [ + { + "description": "ASCII zero does not match", + "data": "0", + "valid": false + }, + { + "description": "NKO DIGIT ZERO matches (unlike e.g. Python)", + "data": "߀", + "valid": true + }, + { + "description": "NKO DIGIT ZERO (as \\u escape) matches", + "data": "\u07c0", + "valid": true + } + ] + }, + { + "description": "ECMA 262 \\w matches ascii letters only", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\w$" + }, + "tests": [ + { + "description": "ASCII 'a' matches", + "data": "a", + "valid": true + }, + { + "description": "latin-1 e-acute does not match (unlike e.g. Python)", + "data": "é", + "valid": false + } + ] + }, + { + "description": "ECMA 262 \\W matches everything but ascii letters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\W$" + }, + "tests": [ + { + "description": "ASCII 'a' does not match", + "data": "a", + "valid": false + }, + { + "description": "latin-1 e-acute matches (unlike e.g. Python)", + "data": "é", + "valid": true + } + ] + }, + { + "description": "ECMA 262 \\s matches whitespace", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\s$" + }, + "tests": [ + { + "description": "ASCII space matches", + "data": " ", + "valid": true + }, + { + "description": "Character tabulation matches", + "data": "\t", + "valid": true + }, + { + "description": "Line tabulation matches", + "data": "\u000b", + "valid": true + }, + { + "description": "Form feed matches", + "data": "\u000c", + "valid": true + }, + { + "description": "latin-1 non-breaking-space matches", + "data": "\u00a0", + "valid": true + }, + { + "description": "zero-width whitespace matches", + "data": "\ufeff", + "valid": true + }, + { + "description": "line feed matches (line terminator)", + "data": "\u000a", + "valid": true + }, + { + "description": "paragraph separator matches (line terminator)", + "data": "\u2029", + "valid": true + }, + { + "description": "EM SPACE matches (Space_Separator)", + "data": "\u2003", + "valid": true + }, + { + "description": "Non-whitespace control does not match", + "data": "\u0001", + "valid": false + }, + { + "description": "Non-whitespace does not match", + "data": "\u2013", + "valid": false + } + ] + }, + { + "description": "ECMA 262 \\S matches everything but whitespace", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\S$" + }, + "tests": [ + { + "description": "ASCII space does not match", + "data": " ", + "valid": false + }, + { + "description": "Character tabulation does not match", + "data": "\t", + "valid": false + }, + { + "description": "Line tabulation does not match", + "data": "\u000b", + "valid": false + }, + { + "description": "Form feed does not match", + "data": "\u000c", + "valid": false + }, + { + "description": "latin-1 non-breaking-space does not match", + "data": "\u00a0", + "valid": false + }, + { + "description": "zero-width whitespace does not match", + "data": "\ufeff", + "valid": false + }, + { + "description": "line feed does not match (line terminator)", + "data": "\u000a", + "valid": false + }, + { + "description": "paragraph separator does not match (line terminator)", + "data": "\u2029", + "valid": false + }, + { + "description": "EM SPACE does not match (Space_Separator)", + "data": "\u2003", + "valid": false + }, + { + "description": "Non-whitespace control matches", + "data": "\u0001", + "valid": true + }, + { + "description": "Non-whitespace matches", + "data": "\u2013", + "valid": true + } + ] + }, + { + "description": "patterns always use unicode semantics with pattern", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "\\p{Letter}cole" + }, + "tests": [ + { + "description": "ascii character in json string", + "data": "Les hivers de mon enfance etaient des saisons longues, longues. Nous vivions en trois lieux: l'ecole, l'eglise et la patinoire; mais la vraie vie etait sur la patinoire.", + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'école, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": true + }, + { + "description": "unicode character in hex format in string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'\u00e9cole, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": true + }, + { + "description": "unicode matching is case-sensitive", + "data": "LES HIVERS DE MON ENFANCE ÉTAIENT DES SAISONS LONGUES, LONGUES. NOUS VIVIONS EN TROIS LIEUX: L'ÉCOLE, L'ÉGLISE ET LA PATINOIRE; MAIS LA VRAIE VIE ÉTAIT SUR LA PATINOIRE.", + "valid": false + } + ] + }, + { + "description": "\\w in patterns matches [A-Za-z0-9_], not unicode letters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "\\wcole" + }, + "tests": [ + { + "description": "ascii character in json string", + "data": "Les hivers de mon enfance etaient des saisons longues, longues. Nous vivions en trois lieux: l'ecole, l'eglise et la patinoire; mais la vraie vie etait sur la patinoire.", + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'école, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'\u00e9cole, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "unicode matching is case-sensitive", + "data": "LES HIVERS DE MON ENFANCE ÉTAIENT DES SAISONS LONGUES, LONGUES. NOUS VIVIONS EN TROIS LIEUX: L'ÉCOLE, L'ÉGLISE ET LA PATINOIRE; MAIS LA VRAIE VIE ÉTAIT SUR LA PATINOIRE.", + "valid": false + } + ] + }, + { + "description": "pattern with ASCII ranges", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "[a-z]cole" + }, + "tests": [ + { + "description": "literal unicode character in json string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'école, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'\u00e9cole, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "ascii characters match", + "data": "Les hivers de mon enfance etaient des saisons longues, longues. Nous vivions en trois lieux: l'ecole, l'eglise et la patinoire; mais la vraie vie etait sur la patinoire.", + "valid": true + } + ] + }, + { + "description": "\\d in pattern matches [0-9], not unicode digits", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "^\\d+$" + }, + "tests": [ + { + "description": "ascii digits", + "data": "42", + "valid": true + }, + { + "description": "ascii non-digits", + "data": "-%#", + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": "৪২", + "valid": false + } + ] + }, + { + "description": "pattern with non-ASCII digits", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "^\\p{digit}+$" + }, + "tests": [ + { + "description": "ascii digits", + "data": "42", + "valid": true + }, + { + "description": "ascii non-digits", + "data": "-%#", + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": "৪২", + "valid": true + } + ] + }, + { + "description": "patterns always use unicode semantics with patternProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "patternProperties": { + "\\p{Letter}cole": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii character in json string", + "data": { "l'ecole": "pas de vraie vie" }, + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": { "l'école": "pas de vraie vie" }, + "valid": true + }, + { + "description": "unicode character in hex format in string", + "data": { "l'\u00e9cole": "pas de vraie vie" }, + "valid": true + }, + { + "description": "unicode matching is case-sensitive", + "data": { "L'ÉCOLE": "PAS DE VRAIE VIE" }, + "valid": false + } + ] + }, + { + "description": "\\w in patternProperties matches [A-Za-z0-9_], not unicode letters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "patternProperties": { + "\\wcole": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii character in json string", + "data": { "l'ecole": "pas de vraie vie" }, + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": { "l'école": "pas de vraie vie" }, + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": { "l'\u00e9cole": "pas de vraie vie" }, + "valid": false + }, + { + "description": "unicode matching is case-sensitive", + "data": { "L'ÉCOLE": "PAS DE VRAIE VIE" }, + "valid": false + } + ] + }, + { + "description": "patternProperties with ASCII ranges", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "patternProperties": { + "[a-z]cole": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "literal unicode character in json string", + "data": { "l'école": "pas de vraie vie" }, + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": { "l'\u00e9cole": "pas de vraie vie" }, + "valid": false + }, + { + "description": "ascii characters match", + "data": { "l'ecole": "pas de vraie vie" }, + "valid": true + } + ] + }, + { + "description": "\\d in patternProperties matches [0-9], not unicode digits", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "patternProperties": { + "^\\d+$": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii digits", + "data": { "42": "life, the universe, and everything" }, + "valid": true + }, + { + "description": "ascii non-digits", + "data": { "-%#": "spending the year dead for tax reasons" }, + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": { "৪২": "khajit has wares if you have coin" }, + "valid": false + } + ] + }, + { + "description": "patternProperties with non-ASCII digits", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "patternProperties": { + "^\\p{digit}+$": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii digits", + "data": { "42": "life, the universe, and everything" }, + "valid": true + }, + { + "description": "ascii non-digits", + "data": { "-%#": "spending the year dead for tax reasons" }, + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": { "৪২": "khajit has wares if you have coin" }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/float-overflow.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/float-overflow.json new file mode 100644 index 000000000..f5ae8b18f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/float-overflow.json @@ -0,0 +1,17 @@ +[ + { + "description": "all integers are multiples of 0.5, if overflow is handled", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer", + "multipleOf": 0.5 + }, + "tests": [ + { + "description": "valid if optional overflow handling is implemented", + "data": 1e308, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format-assertion.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format-assertion.json new file mode 100644 index 000000000..96f26fefb --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format-assertion.json @@ -0,0 +1,44 @@ +[ + { + "description": "schema that uses custom metaschema with format-assertion: false", + "comment": "The true/false boolean in $vocabulary only controls behavior for implementations that do not recognize the vocabulary. For implementations that do understand format-assertion, the boolean has no impact on validation behavior — hence both test groups produce identical results.", + "schema": { + "$id": "https://schema/using/format-assertion/false", + "$schema": "http://localhost:1234/draft2020-12/format-assertion-false.json", + "format": "ipv4" + }, + "tests": [ + { + "description": "format-assertion: false: valid string", + "data": "127.0.0.1", + "valid": true + }, + { + "description": "format-assertion: false: invalid string", + "comment": "valid: false is intentional — the false value in $vocabulary only affects unknown vocabulary handling, not validation behavior itself", + "data": "not-an-ipv4", + "valid": false + } + ] + }, + { + "description": "schema that uses custom metaschema with format-assertion: true", + "schema": { + "$id": "https://schema/using/format-assertion/true", + "$schema": "http://localhost:1234/draft2020-12/format-assertion-true.json", + "format": "ipv4" + }, + "tests": [ + { + "description": "format-assertion: true: valid string", + "data": "127.0.0.1", + "valid": true + }, + { + "description": "format-assertion: true: invalid string", + "data": "not-an-ipv4", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date-time.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date-time.json new file mode 100644 index 000000000..590f2f678 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date-time.json @@ -0,0 +1,250 @@ +[ + { + "description": "validation of date-time strings", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "date-time" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid date-time string", + "data": "1963-06-19T08:30:06.283185Z", + "valid": true + }, + { + "description": "a valid date-time string without second fraction", + "data": "1963-06-19T08:30:06Z", + "valid": true + }, + { + "description": "a valid date-time string with plus offset", + "data": "1937-01-01T12:00:27.87+00:20", + "valid": true + }, + { + "description": "a valid date-time string with minus offset", + "data": "1990-12-31T15:59:50.123-08:00", + "valid": true + }, + { + "description": "a valid date-time with a leap second, UTC", + "data": "1998-12-31T23:59:60Z", + "valid": true + }, + { + "description": "a valid date-time with a leap second, with minus offset", + "data": "1998-12-31T15:59:60.123-08:00", + "valid": true + }, + { + "description": "an invalid date-time past leap second, UTC", + "data": "1998-12-31T23:59:61Z", + "valid": false + }, + { + "description": "an invalid date-time with leap second on a wrong minute, UTC", + "data": "1998-12-31T23:58:60Z", + "valid": false + }, + { + "description": "an invalid date-time with leap second on a wrong hour, UTC", + "data": "1998-12-31T22:59:60Z", + "valid": false + }, + { + "description": "an invalid day in date-time string", + "data": "1990-02-31T15:59:59.123-08:00", + "valid": false + }, + { + "description": "an invalid offset in date-time string", + "data": "1990-12-31T15:59:59-24:00", + "valid": false + }, + { + "description": "an invalid closing Z after time-zone offset", + "data": "1963-06-19T08:30:06.28123+01:00Z", + "valid": false + }, + { + "description": "an invalid hour in date-time string", + "data": "1990-12-31T24:00:00Z", + "valid": false + }, + { + "description": "an invalid minute in date-time string", + "data": "1990-12-31T15:60:00Z", + "valid": false + }, + { + "description": "an invalid offset minute in date-time string", + "data": "1990-12-31T10:00:00+10:60", + "valid": false + }, + { + "description": "an invalid date-time string", + "data": "06/19/1963 08:30:06 PST", + "valid": false + }, + { + "description": "case-insensitive T and Z", + "data": "1963-06-19t08:30:06.283185z", + "valid": true + }, + { + "description": "only RFC3339 not all of ISO 8601 are valid", + "data": "2013-350T01:01:01", + "valid": false + }, + { + "description": "invalid non-padded month dates", + "data": "1963-6-19T08:30:06.283185Z", + "valid": false + }, + { + "description": "invalid non-padded day dates", + "data": "1963-06-1T08:30:06.283185Z", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4) in date portion", + "data": "1963-06-1৪T00:00:00Z", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4) in time portion", + "data": "1963-06-11T0৪:00:00Z", + "valid": false + }, + { + "description": "invalid extended year", + "data": "+11963-06-19T08:30:06.283185Z", + "valid": false + }, + { + "description": "a numeric offset without minutes is invalid", + "data": "1985-04-12T23:20:50+01", + "valid": false + }, + { + "description": "hour 24 is invalid even with a leap second", + "data": "2016-12-31T24:59:60+01:00", + "valid": false + }, + { + "description": "a second fraction of fifteen nines is valid", + "data": "1985-04-12T00:59:59.999999999999999Z", + "valid": true + }, + { + "description": "a trailing newline is invalid", + "data": "1985-04-12T23:20:50Z\n", + "valid": false + }, + { + "description": "an invalid date-time string without seconds", + "data": "1985-04-12T23:20Z", + "valid": false + }, + { + "description": "an invalid date-time string with trailing content after the offset", + "data": "1985-04-12T23:20:50Ztail", + "valid": false + }, + { + "description": "an invalid minute in date-time string, with a numeric offset", + "data": "1985-04-12T23:60:00+00:01", + "valid": false + }, + { + "description": "a valid date-time string with 28 days in February (normal)", + "data": "2021-02-28T00:00:00Z", + "valid": true + }, + { + "description": "an invalid date-time string with 30 days in February (leap)", + "data": "2020-02-30T00:00:00Z", + "valid": false + }, + { + "description": "2021 is not a leap year", + "data": "2021-02-29T00:00:00Z", + "valid": false + }, + { + "description": "2020 is a leap year", + "data": "2020-02-29T00:00:00Z", + "valid": true + }, + { + "description": "century year 0100 is not a leap year", + "data": "0100-02-29T00:00:00Z", + "valid": false + }, + { + "description": "century year 0400 is a leap year", + "data": "0400-02-29T00:00:00Z", + "valid": true + }, + { + "description": "century year 2100 is not a leap year", + "data": "2100-02-29T00:00:00Z", + "valid": false + }, + { + "description": "a second fraction with no digits", + "comment": "RFC 3339 section 5.6: time-secfrac = \".\" 1*DIGIT requires at least one digit", + "data": "1985-04-12T23:20:50.Z", + "valid": false + }, + { + "description": "a comma decimal separator with a time-offset", + "comment": "RFC 3339 section 5.6: the decimal separator in time-secfrac is \".\" - the ISO 8601 comma is not part of this profile", + "data": "1985-04-12T23:20:50,5Z", + "valid": false + }, + { + "description": "leading whitespace before a valid date-time", + "comment": "RFC 3339 section 5.6: no production of date-time emits whitespace", + "data": " 1985-04-12T23:20:50Z", + "valid": false + }, + { + "description": "trailing whitespace after a valid date-time", + "comment": "RFC 3339 section 5.6: no production of date-time emits whitespace", + "data": "1985-04-12T23:20:50Z ", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date.json new file mode 100644 index 000000000..a8a19261a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date.json @@ -0,0 +1,423 @@ +[ + { + "description": "validation of date strings", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "date" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid date string", + "data": "1963-06-19", + "valid": true + }, + { + "description": "a valid date string with 31 days in January", + "data": "2020-01-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in January", + "data": "2020-01-32", + "valid": false + }, + { + "description": "a valid date string with 28 days in February (normal)", + "data": "2021-02-28", + "valid": true + }, + { + "description": "a invalid date string with 30 days in February (leap)", + "data": "2020-02-30", + "valid": false + }, + { + "description": "a valid date string with 31 days in March", + "data": "2020-03-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in March", + "data": "2020-03-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in April", + "data": "2020-04-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in April", + "data": "2020-04-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in May", + "data": "2020-05-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in May", + "data": "2020-05-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in June", + "data": "2020-06-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in June", + "data": "2020-06-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in July", + "data": "2020-07-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in July", + "data": "2020-07-32", + "valid": false + }, + { + "description": "a valid date string with 31 days in August", + "data": "2020-08-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in August", + "data": "2020-08-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in September", + "data": "2020-09-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in September", + "data": "2020-09-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in October", + "data": "2020-10-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in October", + "data": "2020-10-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in November", + "data": "2020-11-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in November", + "data": "2020-11-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in December", + "data": "2020-12-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in December", + "data": "2020-12-32", + "valid": false + }, + { + "description": "an invalid date string", + "data": "06/19/1963", + "valid": false + }, + { + "description": "only RFC3339 not all of ISO 8601 are valid", + "data": "2013-350", + "valid": false + }, + { + "description": "non-padded month dates are not valid", + "data": "1998-1-20", + "valid": false + }, + { + "description": "non-padded day dates are not valid", + "data": "1998-01-1", + "valid": false + }, + { + "description": "invalid month", + "data": "1998-13-01", + "valid": false + }, + { + "description": "2021 is not a leap year", + "data": "2021-02-29", + "valid": false + }, + { + "description": "2020 is a leap year", + "data": "2020-02-29", + "valid": true + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4)", + "data": "1963-06-1৪", + "valid": false + }, + { + "description": "invalid: non-ASCII Bengali digit in month field", + "data": "2020-0৪-01", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: YYYYMMDD without dashes (2023-03-28)", + "data": "20230328", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: week number implicit day of week (2023-01-02)", + "data": "2023-W01", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: week number with day of week (2023-03-28)", + "data": "2023-W13-2", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: week number rollover to next year (2023-01-01)", + "data": "2022W527", + "valid": false + }, + { + "description": "an invalid time string in date-time format", + "data": "2020-11-28T23:55:45Z", + "valid": false + }, + { + "description": "century year 0100 is not a leap year", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#appendix-C — 100 % 100 == 0 but 100 % 400 != 0", + "data": "0100-02-29", + "valid": false + }, + { + "description": "century year 0400 is a leap year", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#appendix-C — 400-year cycle boundary", + "data": "0400-02-29", + "valid": true + }, + { + "description": "century year 2100 is not a leap year", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#appendix-C — future century year", + "data": "2100-02-29", + "valid": false + }, + { + "description": "invalid: leading whitespace is not permitted", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — full-date grammar does not include whitespace", + "data": " 2024-01-15", + "valid": false + }, + { + "description": "invalid: trailing whitespace is not permitted", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — full-date grammar does not include whitespace", + "data": "2024-01-15 ", + "valid": false + }, + { + "description": "invalid: month 00 is not valid per date-month range 01-12", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — date-month = 2DIGIT ; 01-12", + "data": "2024-00-15", + "valid": false + }, + { + "description": "invalid: day 00 is not valid per date-mday minimum of 01", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — date-mday = 2DIGIT ; 01-28/29/30/31", + "data": "2024-01-00", + "valid": false + }, + { + "description": "invalid: empty string", + "data": "", + "valid": false + }, + { + "description": "invalid: embedded whitespace between year and month", + "data": "2020 -01-01", + "valid": false + }, + { + "description": "invalid: trailing character after valid full-date", + "data": "2020-01-01X", + "valid": false + }, + { + "description": "invalid: trailing Z after full-date", + "data": "2020-01-01Z", + "valid": false + }, + { + "description": "invalid: full-date followed by space and time component", + "data": "2020-01-01 00:00:00Z", + "valid": false + }, + { + "description": "valid: four-digit year 0001", + "data": "0001-01-01", + "valid": true + }, + { + "description": "invalid: two-digit year (N-2 digits)", + "data": "20-01-01", + "valid": false + }, + { + "description": "invalid: three-digit year (N-1 digits)", + "data": "998-01-01", + "valid": false + }, + { + "description": "invalid: five-digit year (N+1 digits)", + "data": "12020-01-01", + "valid": false + }, + { + "description": "invalid: positive sign prefix on year", + "data": "+2020-01-01", + "valid": false + }, + { + "description": "invalid: negative sign prefix on year", + "data": "-2020-01-01", + "valid": false + }, + { + "description": "invalid: non-ASCII Bengali digit in year field", + "data": "২020-01-01", + "valid": false + }, + { + "description": "invalid: alphabetic characters in year field", + "data": "YYYY-01-01", + "valid": false + }, + { + "description": "invalid: three-digit month (N+1 digits)", + "data": "2020-001-01", + "valid": false + }, + { + "description": "invalid: alphabetic characters in month field", + "data": "2020-MM-01", + "valid": false + }, + { + "description": "invalid: three-digit day (N+1 digits)", + "data": "2020-01-001", + "valid": false + }, + { + "description": "invalid: alphabetic characters in day field", + "data": "2020-01-DD", + "valid": false + }, + { + "description": "invalid: colon separators", + "data": "2020:01:01", + "valid": false + }, + { + "description": "invalid: dot separators", + "data": "2020.01.01", + "valid": false + }, + { + "description": "invalid: space separators", + "data": "2020 01 01", + "valid": false + }, + { + "description": "invalid: mixed slash and hyphen separators", + "data": "2020-01/01", + "valid": false + }, + { + "description": "invalid: duplicated first hyphen", + "data": "2020--01-01", + "valid": false + }, + { + "description": "invalid: duplicated second hyphen", + "data": "2020-01--01", + "valid": false + }, + { + "description": "valid: date inside the Julian to Gregorian reform gap", + "data": "1582-10-10", + "valid": true + }, + { + "description": "invalid: year field numerically larger than a 32-bit signed integer", + "data": "2147483648-01-01", + "valid": false + }, + { + "description": "invalid: character just above '9' as the second digit of the day", + "data": "2020-01-0:", + "valid": false + }, + { + "description": "invalid: non-ASCII digit leaving ten UTF-8 bytes but eight characters", + "data": "\u09e80-01-01", + "valid": false + }, + { + "description": "invalid: en dash separators instead of hyphen-minus", + "data": "2020\u201301\u201301", + "valid": false + }, + { + "description": "invalid: NUL character after an otherwise valid full-date", + "data": "2020-01-01\u0000", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/duration.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/duration.json new file mode 100644 index 000000000..e3e184161 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/duration.json @@ -0,0 +1,268 @@ +[ + { + "description": "validation of duration strings", + "comment": "RFC 3339 Appendix A defines the ABNF grammar for ISO-8601 durations used by JSON Schema format 'duration'. These tests enforce only the syntax defined by that grammar.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "duration" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid duration string", + "data": "P4DT12H30M5S", + "valid": true + }, + { + "description": "an invalid duration string", + "data": "PT1D", + "valid": false + }, + { + "description": "must start with P", + "data": "4DT12H30M5S", + "valid": false + }, + { + "description": "no elements present", + "data": "P", + "valid": false + }, + { + "description": "no time elements present", + "data": "P1YT", + "valid": false + }, + { + "description": "no date or time elements present", + "data": "PT", + "valid": false + }, + { + "description": "elements out of order", + "data": "P2D1Y", + "valid": false + }, + { + "description": "missing time separator", + "data": "P1D2H", + "valid": false + }, + { + "description": "time element in the date position", + "data": "P2S", + "valid": false + }, + { + "description": "four years duration", + "data": "P4Y", + "valid": true + }, + { + "description": "zero time, in seconds", + "data": "PT0S", + "valid": true + }, + { + "description": "zero time, in days", + "data": "P0D", + "valid": true + }, + { + "description": "one month duration", + "data": "P1M", + "valid": true + }, + { + "description": "one minute duration", + "data": "PT1M", + "valid": true + }, + { + "description": "one and a half days, in hours", + "data": "PT36H", + "valid": true + }, + { + "description": "one and a half days, in days and hours", + "data": "P1DT12H", + "valid": true + }, + { + "description": "two weeks", + "data": "P2W", + "valid": true + }, + { + "description": "weeks cannot be combined with other units", + "data": "P1Y2W", + "valid": false + }, + { + "description": "invalid non-ASCII '২' (a Bengali 2)", + "data": "P২Y", + "valid": false + }, + { + "description": "element without unit", + "data": "P1", + "valid": false + }, + { + "description": "all date and time components", + "data": "P1Y2M3DT4H5M6S", + "valid": true + }, + { + "description": "date components only", + "data": "P1Y2M3D", + "valid": true + }, + { + "description": "time components only", + "data": "PT1H2M3S", + "valid": true + }, + { + "description": "month and day", + "data": "P1M2D", + "valid": true + }, + { + "description": "hour and minute", + "data": "PT1H30M", + "valid": true + }, + { + "description": "multi-digit values in all components", + "data": "P10Y10M10DT10H10M10S", + "valid": true + }, + { + "description": "fractional duration is not allowed by RFC 3339 ABNF", + "comment": "numeric components use 1*DIGIT where DIGIT = %x30-39; '.' is not allowed", + "data": "PT0.5S", + "valid": false + }, + { + "description": "leading whitespace is invalid", + "data": " P1D", + "valid": false + }, + { + "description": "trailing whitespace is invalid", + "data": "P1D ", + "valid": false + }, + { + "description": "empty string is invalid", + "data": "", + "valid": false + }, + { + "description": "years and months can appear without days", + "data": "P1Y2M", + "valid": true + }, + { + "description": "years and days cannot appear without months", + "data": "P1Y2D", + "valid": false + }, + { + "description": "hours and minutes can appear without seconds", + "data": "PT1H2M", + "valid": true + }, + { + "description": "hours and seconds cannot appear without minutes", + "data": "PT1H2S", + "valid": false + }, + { + "description": "minutes and seconds can appear without hour", + "data": "PT1M2S", + "valid": true + }, + { + "description": "a leading sign is not allowed", + "data": "-P1D", + "valid": false + }, + { + "description": "a number before the time separator has no unit", + "data": "P1D2T3H", + "valid": false + }, + { + "description": "exponent notation is not allowed in a component", + "data": "P1e2D", + "valid": false + }, + { + "description": "a trailing newline is invalid", + "data": "P1D\n", + "valid": false + }, + { + "description": "weeks cannot be combined with a time component", + "data": "P1WT1H", + "valid": false + }, + { + "description": "weeks cannot be combined with a zero-valued component", + "data": "P0Y1W", + "valid": false + }, + { + "description": "a leading zero in a component is valid", + "data": "P01D", + "valid": true + }, + { + "description": "a component with many digits is valid", + "data": "P999999999999999999999999999999999999999999999999999999999999999999999999999999D", + "valid": true + }, + { + "description": "a comma as the decimal separator is invalid", + "data": "PT0,5S", + "valid": false + }, + { + "description": "a sign inside a component is invalid", + "data": "P-1D", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ecmascript-regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ecmascript-regex.json new file mode 100644 index 000000000..b9ae3c049 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ecmascript-regex.json @@ -0,0 +1,116 @@ +[ + { + "description": "\\a is not an ECMA 262 control escape", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "when used as a pattern", + "data": "\\a", + "valid": false + } + ] + }, + { + "description": "Python-specific regular expression syntax is not valid ECMA 262", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "Python named group (?P...) is not ECMA 262", + "data": "(?Px)", + "valid": false + }, + { + "description": "Python named backreference (?P=name) is not ECMA 262", + "data": "(?Pa)(?P=n)", + "valid": false + }, + { + "description": "inline comment group (?#...) is not ECMA 262", + "data": "(?#comment)a", + "valid": false + } + ] + }, + { + "description": "global inline flag groups are not valid ECMA 262", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "a single global inline flag (?i)", + "data": "(?i)abc", + "valid": false + }, + { + "description": "multiple global inline flags (?ims)", + "data": "(?ims)abc", + "valid": false + } + ] + }, + { + "description": "ECMA 262 named groups and backreferences are valid", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "an ECMA 262 named group (?...)", + "data": "(?x)", + "valid": true + }, + { + "description": "an ECMA 262 named backreference \\k", + "data": "(?a)\\k", + "valid": true + } + ] + }, + { + "description": "ECMA 262 lookbehind is valid, including variable width", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "a variable-width lookbehind (ES2018)", + "data": "(?<=a+)b", + "valid": true + } + ] + }, + { + "description": "ECMA 262 character classes and escapes", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "an empty character class is valid ECMA 262", + "data": "[]", + "valid": true + }, + { + "description": "a negated empty character class is valid ECMA 262", + "data": "[^]", + "valid": true + }, + { + "description": "a control escape \\cA is valid ECMA 262", + "data": "\\cA", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/email.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/email.json new file mode 100644 index 000000000..50e214622 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/email.json @@ -0,0 +1,368 @@ +[ + { + "description": "validation of e-mail addresses", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "email" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid e-mail address", + "data": "joe.bloggs@example.com", + "valid": true + }, + { + "description": "an invalid e-mail address", + "data": "2962", + "valid": false + }, + { + "description": "tilde in local part is valid", + "data": "te~st@example.com", + "valid": true + }, + { + "description": "tilde before local part is valid", + "data": "~test@example.com", + "valid": true + }, + { + "description": "tilde after local part is valid", + "data": "test~@example.com", + "valid": true + }, + { + "description": "a quoted string with a space in the local part is valid", + "data": "\"joe bloggs\"@example.com", + "valid": true + }, + { + "description": "a quoted string with a double dot in the local part is valid", + "data": "\"joe..bloggs\"@example.com", + "valid": true + }, + { + "description": "a quoted string with a @ in the local part is valid", + "data": "\"joe@bloggs\"@example.com", + "valid": true + }, + { + "description": "an IPv4-address-literal after the @ is valid", + "data": "joe.bloggs@[127.0.0.1]", + "valid": true + }, + { + "description": "an IPv6-address-literal after the @ is valid", + "data": "joe.bloggs@[IPv6:::1]", + "valid": true + }, + { + "description": "dot before local part is not valid", + "data": ".test@example.com", + "valid": false + }, + { + "description": "dot after local part is not valid", + "data": "test.@example.com", + "valid": false + }, + { + "description": "two separated dots inside local part are valid", + "data": "te.s.t@example.com", + "valid": true + }, + { + "description": "two subsequent dots inside local part are not valid", + "data": "te..st@example.com", + "valid": false + }, + { + "description": "an invalid domain", + "data": "joe.bloggs@invalid=domain.com", + "valid": false + }, + { + "description": "an invalid IPv4-address-literal", + "data": "joe.bloggs@[127.0.0.300]", + "valid": false + }, + { + "description": "two email addresses is not valid", + "data": "user1@oceania.org, user2@oceania.org", + "valid": false + }, + { + "description": "full \"From\" header is invalid", + "data": "\"Winston Smith\" (Records Department)", + "valid": false + }, + { + "description": "local part is required", + "data": "@example.com", + "valid": false + }, + { + "description": "domain is required", + "data": "joe.bloggs@", + "valid": false + }, + { + "description": "unquoted space in local part is invalid", + "data": "joe bloggs@example.com", + "valid": false + }, + { + "description": "a domain that looks like an IPv4 address without brackets is valid", + "data": "test@255.255.255.255", + "valid": true + }, + { + "description": "an empty quoted string in the local part is valid", + "data": "\"\"@iana.org", + "valid": true + }, + { + "description": "a quoted string containing only a space in the local part is valid", + "data": "\" \"@iana.org", + "valid": true + }, + { + "description": "consecutive hyphens inside a domain label are valid", + "data": "test@c--n.com", + "valid": true + }, + { + "description": "a domain label starting with a digit is valid", + "data": "test@123.com", + "valid": true + }, + { + "description": "a non-ASCII character in the local part is not valid", + "data": "a\u00e9@iana.org", + "valid": false + }, + { + "description": "a non-ASCII character in the domain is not valid", + "data": "a@\u00e9.org", + "valid": false + }, + { + "description": "a fullwidth commercial at is not a valid separator", + "data": "a\uff20iana.org", + "valid": false + }, + { + "description": "a non-ASCII character in a quoted pair is not valid", + "data": "\"test\\\u00a9\"@iana.org", + "valid": false + }, + { + "description": "a trailing line feed is not valid", + "data": "test@iana.org\n", + "valid": false + }, + { + "description": "a trailing carriage return is not valid", + "data": "test@iana.org\r", + "valid": false + }, + { + "description": "a trailing space is not valid", + "data": "a@iana.org ", + "valid": false + }, + { + "description": "a leading space is not valid", + "data": " a@iana.org", + "valid": false + }, + { + "description": "a tab in the local part is not valid", + "data": "a\tb@iana.org", + "valid": false + }, + { + "description": "a delete character in the local part is not valid", + "data": "\u007f@iana.org", + "valid": false + }, + { + "description": "a quoted string with no special characters in the local part is valid", + "data": "\"test\"@iana.org", + "valid": true + }, + { + "description": "a quoted pair in the local part is valid", + "data": "\"\\a\"@iana.org", + "valid": true + }, + { + "description": "an escaped double quote in the local part is valid", + "data": "\"\\\"\"@iana.org", + "valid": true + }, + { + "description": "an escaped backslash in the local part is valid", + "data": "\"\\\\\"@iana.org", + "valid": true + }, + { + "description": "an unescaped double quote in an unquoted local part is not valid", + "data": "test\"@iana.org", + "valid": false + }, + { + "description": "text after the closing double quote in the local part is not valid", + "data": "\"test\"test@iana.org", + "valid": false + }, + { + "description": "an unclosed double quote in the local part is not valid", + "data": "\"test@iana.org", + "valid": false + }, + { + "description": "two unquoted @ signs are not valid", + "data": "a@b@c.org", + "valid": false + }, + { + "description": "a backslash-escaped @ in an unquoted local part is not valid", + "data": "test\\@test@iana.org", + "valid": false + }, + { + "description": "an address literal in the local part position is not valid", + "data": "[1.2.3.4]@iana.org", + "valid": false + }, + { + "description": "an empty string is not valid", + "data": "", + "valid": false + }, + { + "description": "a domain label starting with a hyphen is not valid", + "data": "test@-iana.org", + "valid": false + }, + { + "description": "a domain label ending with a hyphen is not valid", + "data": "test@iana-.com", + "valid": false + }, + { + "description": "a domain starting with a dot is not valid", + "data": "test@.iana.org", + "valid": false + }, + { + "description": "a domain ending with a dot is not valid", + "data": "test@iana.org.", + "valid": false + }, + { + "description": "two subsequent dots inside the domain are not valid", + "data": "test@iana..com", + "valid": false + }, + { + "description": "a single-label domain is valid", + "data": "test@io", + "valid": true + }, + { + "description": "an unclosed address literal is not valid", + "data": "test@[1.2.3.4", + "valid": false + }, + { + "description": "empty brackets after the @ are not valid", + "data": "a@[]", + "valid": false + }, + { + "description": "an IPv4-address-literal with three octets is not valid", + "data": "a@[1.2.3]", + "valid": false + }, + { + "description": "an IPv4-address-literal with five octets is not valid", + "data": "a@[1.2.3.4.5]", + "valid": false + }, + { + "description": "bracketed content that is not an address is not valid", + "data": "test@[RFC-5322-domain-literal]", + "valid": false + }, + { + "description": "a domain label before the opening bracket is not valid", + "data": "test@a[255.255.255.255]", + "valid": false + }, + { + "description": "a lowercase IPv6 tag in an address literal is valid", + "data": "a@[ipv6:::1]", + "valid": true + }, + { + "description": "a comma in an unquoted local part is not valid", + "data": "a,b@iana.org", + "valid": false + }, + { + "description": "a semicolon in an unquoted local part is not valid", + "data": "a;b@iana.org", + "valid": false + }, + { + "description": "a colon in an unquoted local part is not valid", + "data": "a:b@iana.org", + "valid": false + }, + { + "description": "a parenthesis in an unquoted local part is not valid", + "comment": "a comment is RFC 5322 syntax, which the RFC 5321 Mailbox grammar does not have", + "data": "test(comment)@iana.org", + "valid": false + }, + { + "description": "a parenthesis in the domain is not valid", + "comment": "a comment is RFC 5322 syntax, which the RFC 5321 Mailbox grammar does not have", + "data": "a@b(c).org", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/hostname.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/hostname.json new file mode 100644 index 000000000..7cacfcfd2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/hostname.json @@ -0,0 +1,386 @@ +[ + { + "description": "validation of host names", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "hostname" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid host name", + "data": "www.example.com", + "valid": true + }, + { + "description": "single label", + "data": "hostname", + "valid": true + }, + { + "description": "single label with digits", + "data": "h0stn4me", + "valid": true + }, + { + "description": "single label starting with digit", + "data": "1host", + "valid": true + }, + { + "description": "single label ending with digit", + "data": "hostnam3", + "valid": true + }, + { + "description": "empty string", + "data": "", + "valid": false + }, + { + "description": "single dot", + "data": ".", + "valid": false + }, + { + "description": "leading dot", + "data": ".example", + "valid": false + }, + { + "description": "trailing dot", + "data": "example.", + "valid": false + }, + { + "description": "IDN label separator", + "data": "example\uff0ecom", + "valid": false + }, + { + "description": "single label with hyphen", + "data": "host-name", + "valid": true + }, + { + "description": "starts with hyphen", + "data": "-hostname", + "valid": false + }, + { + "description": "ends with hyphen", + "data": "hostname-", + "valid": false + }, + { + "description": "contains underscore", + "data": "host_name", + "valid": false + }, + { + "description": "exceeds maximum overall length (256)", + "data": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.com", + "valid": false + }, + { + "description": "maximum label length (63)", + "data": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.com", + "valid": true + }, + { + "description": "exceeds maximum label length (63)", + "data": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijkl.com", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "example.com\n", + "valid": false + }, + { + "description": "leading space is invalid", + "data": " example.com", + "valid": false + }, + { + "description": "space inside a label is invalid", + "data": "ex ample.com", + "valid": false + }, + { + "description": "invalid non-ASCII KELVIN SIGN (U+212A)", + "data": "\u212Aelvin.example.com", + "valid": false + }, + { + "description": "consecutive hyphens inside a label", + "data": "a--b.com", + "valid": true + } + ] + }, + { + "description": "validation of A-label (punycode) host names", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "hostname" + }, + "tests": [ + { + "description": "invalid Punycode", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.4 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "xn--X", + "valid": false + }, + { + "description": "a valid host name (example.test in Hangul)", + "data": "xn--9n2bp8q.xn--9t4b11yi5a", + "valid": true + }, + { + "description": "contains illegal char U+302E Hangul single dot tone mark", + "data": "xn--07jt112bpxg.xn--9t4b11yi5a", + "valid": false + }, + { + "description": "Begins with a Spacing Combining Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "xn--hello-txk", + "valid": false + }, + { + "description": "Begins with a Nonspacing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "xn--hello-zed", + "valid": false + }, + { + "description": "Begins with an Enclosing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "xn--hello-6bf", + "valid": false + }, + { + "description": "Exceptions that are PVALID, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "xn--zca29lwxobi7a", + "valid": true + }, + { + "description": "Exceptions that are PVALID, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "xn--qmbc", + "valid": true + }, + { + "description": "Exceptions that are DISALLOWED, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "xn--chb89f", + "valid": false + }, + { + "description": "Exceptions that are DISALLOWED, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6 Note: The two combining marks (U+302E and U+302F) are in the middle and not at the start", + "data": "xn--07jceefgh4c", + "valid": false + }, + { + "description": "MIDDLE DOT with no preceding 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--al-0ea", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing preceding", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--l-fda", + "valid": false + }, + { + "description": "MIDDLE DOT with no following 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--la-0ea", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing following", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--l-gda", + "valid": false + }, + { + "description": "MIDDLE DOT with surrounding 'l's", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--ll-0ea", + "valid": true + }, + { + "description": "Greek KERAIA not followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "xn--S-jib3p", + "valid": false + }, + { + "description": "Greek KERAIA not followed by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "xn--wva3j", + "valid": false + }, + { + "description": "Greek KERAIA followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "xn--wva3je", + "valid": true + }, + { + "description": "Hebrew GERESH not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "xn--A-2hc5h", + "valid": false + }, + { + "description": "Hebrew GERESH not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "xn--5db1e", + "valid": false + }, + { + "description": "Hebrew GERESH preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "xn--4dbc5h", + "valid": true + }, + { + "description": "Hebrew GERSHAYIM not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "xn--A-2hc8h", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "xn--5db3e", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "xn--4dbc8h", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with no Hiragana, Katakana, or Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--defabc-k64e", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with no other characters", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--vek", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with Hiragana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--k8j5u", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Katakana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--bck0j", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--vek778f", + "valid": true + }, + { + "description": "Arabic-Indic digits mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "xn--ngb6iyr", + "valid": false + }, + { + "description": "Arabic-Indic digits not mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "xn--ngba1o", + "valid": true + }, + { + "description": "Extended Arabic-Indic digits not mixed with Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.9", + "data": "xn--0-gyc", + "valid": true + }, + { + "description": "ZERO WIDTH JOINER not preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "xn--11b2er09f", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "xn--02b508i", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "xn--11b2ezcw70k", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1", + "data": "xn--11b2ezcs70k", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER not preceded by Virama but matches regexp", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1 https://www.w3.org/TR/alreq/#h_disjoining_enforcement", + "data": "xn--ngba5hb2804a", + "valid": true + }, + { + "description": "contains \"--\" in the 3rd and 4th position", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "XN--aa---o47jg78q", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-email.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-email.json new file mode 100644 index 000000000..f46238b6f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-email.json @@ -0,0 +1,140 @@ +[ + { + "description": "validation of an internationalized e-mail addresses", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "idn-email" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid idn e-mail (example@example.test in Hangul)", + "data": "실례@실례.테스트", + "valid": true + }, + { + "description": "an invalid e-mail/idn e-mail address", + "data": "2962", + "valid": false + }, + { + "description": "a valid e-mail address", + "data": "joe.bloggs@example.com", + "valid": true + }, + { + "description": "a non-ASCII local part with an ASCII domain is valid", + "data": "δοκιμή@example.com", + "valid": true + }, + { + "description": "a non-ASCII quoted local part is valid", + "data": "\"δοκιμή\"@example.com", + "valid": true + }, + { + "description": "a domain label that is not in Unicode NFC is valid", + "data": "user@cafe\u0301.com", + "valid": true + }, + { + "description": "a local part that is not in Unicode NFC is valid", + "data": "cafe\u0301@example.com", + "valid": true + }, + { + "description": "a C1 control in the local part is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc6531#section-3.3 atext =/ UTF8-non-ascii; https://www.rfc-editor.org/rfc/rfc3629#section-4 UTF8-2 admits U+0085", + "data": "\u0085@example.com", + "valid": true + }, + { + "description": "a noncharacter in the local part is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc6531#section-3.3 atext =/ UTF8-non-ascii; https://www.rfc-editor.org/rfc/rfc3629#section-4 UTF8-3 admits U+FFFF", + "data": "\uffff@example.com", + "valid": true + }, + { + "description": "a local part at the 64-octet limit is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.5.3.1 every implementation must be able to receive a local part of 64 octets", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@example.com", + "valid": true + }, + { + "description": "a fullwidth commercial at is not a local-part separator", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Mailbox = Local-part \"@\" ( Domain / address-literal ) splits on U+0040, and any IDN mapping applies only to the domain that split produces, so U+FF20 cannot become the delimiter", + "data": "user\uff20example.com", + "valid": false + }, + { + "description": "a local part with a supplementary-plane character is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc6531#section-3.3 atext =/ UTF8-non-ascii, and https://www.rfc-editor.org/rfc/rfc3629#section-4 UTF8-4 covers U+10000 and above", + "data": "\ud835\udd4f@example.com", + "valid": true + }, + { + "description": "a domain label starting with a hyphen is not valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 sub-domain = Let-dig [Ldh-str], so a label starts with a letter or digit; RFC 6531 section 3.3 only adds sub-domain =/ U-label", + "data": "δοκιμή@-example.com", + "valid": false + }, + { + "description": "a domain label ending with a hyphen is not valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Ldh-str = *( ALPHA / DIGIT / \"-\" ) Let-dig, so a label ends with a letter or digit", + "data": "δοκιμή@example-.com", + "valid": false + }, + { + "description": "a domain starting with a dot is not valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Domain = sub-domain *(\".\" sub-domain) admits no empty sub-domain", + "data": "δοκιμή@.example.com", + "valid": false + }, + { + "description": "a domain ending with a dot is not valid", + "data": "δοκιμή@example.com.", + "valid": false + }, + { + "description": "two subsequent dots inside the domain are not valid", + "data": "δοκιμή@example..com", + "valid": false + }, + { + "description": "domain is required", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Mailbox = Local-part \"@\" ( Domain / address-literal )", + "data": "δοκιμή@", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-hostname.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-hostname.json new file mode 100644 index 000000000..62f914d02 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-hostname.json @@ -0,0 +1,588 @@ +[ + { + "description": "validation of internationalized host names", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "idn-hostname" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid host name (example.test in Hangul)", + "data": "실례.테스트", + "valid": true + }, + { + "description": "illegal first char U+302E Hangul single dot tone mark", + "data": "〮실례.테스트", + "valid": false + }, + { + "description": "contains illegal char U+302E Hangul single dot tone mark", + "data": "실〮례.테스트", + "valid": false + }, + { + "description": "a single label of 63 characters is valid", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "valid": true + }, + { + "description": "a single label of 64 characters is too long", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "valid": false + }, + { + "description": "invalid label, correct Punycode", + "comment": "https://tools.ietf.org/html/rfc5890#section-2.3.2.1 https://tools.ietf.org/html/rfc5891#section-4.4 https://tools.ietf.org/html/rfc3492#section-7.1", + "data": "-> $1.00 <--", + "valid": false + }, + { + "description": "valid Chinese Punycode", + "comment": "https://tools.ietf.org/html/rfc5890#section-2.3.2.1 https://tools.ietf.org/html/rfc5891#section-4.4", + "data": "xn--ihqwcrb4cv8a8dqg056pqjye", + "valid": true + }, + { + "description": "invalid Punycode", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.4 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "xn--X", + "valid": false + }, + { + "description": "U-label contains \"--\" in the 3rd and 4th position", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "XN--aa---o47jg78q", + "valid": false + }, + { + "description": "U-label starts with a dash", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1", + "data": "-hello", + "valid": false + }, + { + "description": "U-label ends with a dash", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1", + "data": "hello-", + "valid": false + }, + { + "description": "U-label starts and ends with a dash", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1", + "data": "-hello-", + "valid": false + }, + { + "description": "Begins with a Spacing Combining Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "\u0903hello", + "valid": false + }, + { + "description": "Begins with a Nonspacing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "\u0300hello", + "valid": false + }, + { + "description": "Begins with an Enclosing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "\u0488hello", + "valid": false + }, + { + "description": "Exceptions that are PVALID, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "\u00df\u03c2\u0f0b\u3007", + "valid": true + }, + { + "description": "Exceptions that are PVALID, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "\u06fd\u06fe", + "valid": true + }, + { + "description": "Exceptions that are DISALLOWED, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "\u0640\u07fa", + "valid": false + }, + { + "description": "Exceptions that are DISALLOWED, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6 Note: The two combining marks (U+302E and U+302F) are in the middle and not at the start", + "data": "\u3031\u3032\u3033\u3034\u3035\u302e\u302f\u303b", + "valid": false + }, + { + "description": "MIDDLE DOT with no preceding 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "a\u00b7l", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing preceding", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "\u00b7l", + "valid": false + }, + { + "description": "MIDDLE DOT with no following 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "l\u00b7a", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing following", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "l\u00b7", + "valid": false + }, + { + "description": "MIDDLE DOT with surrounding 'l's", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "l\u00b7l", + "valid": true + }, + { + "description": "Greek KERAIA not followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "\u03b1\u0375S", + "valid": false + }, + { + "description": "Greek KERAIA not followed by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "\u03b1\u0375", + "valid": false + }, + { + "description": "Greek KERAIA followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "\u03b1\u0375\u03b2", + "valid": true + }, + { + "description": "Hebrew GERESH not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "A\u05f3\u05d1", + "valid": false + }, + { + "description": "Hebrew GERESH not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "\u05f3\u05d1", + "valid": false + }, + { + "description": "Hebrew GERESH preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "\u05d0\u05f3\u05d1", + "valid": true + }, + { + "description": "Hebrew GERSHAYIM not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "A\u05f4\u05d1", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "\u05f4\u05d1", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "\u05d0\u05f4\u05d1", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with no Hiragana, Katakana, or Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "def\u30fbabc", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with no other characters", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with Hiragana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb\u3041", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Katakana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb\u30a1", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb\u4e08", + "valid": true + }, + { + "description": "Arabic-Indic digits mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "\u0628\u0660\u06f0", + "valid": false + }, + { + "description": "Arabic-Indic digits not mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "\u0628\u0660\u0628", + "valid": true + }, + { + "description": "Extended Arabic-Indic digits not mixed with Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.9", + "data": "\u06f00", + "valid": true + }, + { + "description": "ZERO WIDTH JOINER not preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "\u0915\u200d\u0937", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "\u200d\u0937", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "\u0915\u094d\u200d\u0937", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1", + "data": "\u0915\u094d\u200c\u0937", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER not preceded by Virama but matches regexp", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1 https://www.w3.org/TR/alreq/#h_disjoining_enforcement", + "data": "\u0628\u064a\u200c\u0628\u064a", + "valid": true + }, + { + "description": "single label", + "data": "hostname", + "valid": true + }, + { + "description": "single label with hyphen", + "data": "host-name", + "valid": true + }, + { + "description": "single label with digits", + "data": "h0stn4me", + "valid": true + }, + { + "description": "single label starting with digit", + "data": "1host", + "valid": true + }, + { + "description": "single label ending with digit", + "data": "hostnam3", + "valid": true + }, + { + "description": "empty string", + "data": "", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "example.com\n", + "valid": false + }, + { + "description": "leading space is invalid", + "data": " example.com", + "valid": false + }, + { + "description": "space inside a label is invalid", + "data": "ex ample.com", + "valid": false + }, + { + "description": "fullwidth digits are mapped to ASCII digits", + "comment": "IdnaMappingTable.txt maps U+FF11 to 0031, so this label maps to 123", + "data": "\uff11\uff12\uff13", + "valid": true + }, + { + "description": "a zero width space is ignored by the mapping", + "comment": "IdnaMappingTable.txt lists U+200B as ignored, so this label maps to ab", + "data": "a\u200bb", + "valid": true + }, + { + "description": "an A-label with an uppercase Punycode body is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5891#section-5.3 lowercases the A-label before the round-trip test, and RFC 3492 section 5 gives A-Z and a-z the same digit-values", + "data": "xn--NXASMQ6B", + "valid": true + }, + { + "description": "the two Arabic-Indic digit blocks may appear in different labels", + "comment": "https://www.rfc-editor.org/rfc/rfc5892#appendix-A.8 scopes the rule to \"this label\", so one block per label is valid", + "data": "\u0628\u0660.\u0628\u06f0", + "valid": true + }, + { + "description": "a label of only Arabic-Indic digits is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2 condition 1 allows only L, R or AL as the first character, and an Arabic-Indic digit is AN", + "data": "\u0660\u0661", + "valid": false + }, + { + "description": "a non-NFC U-label is valid after UTS 46 normalisation", + "comment": "UTS 46 section 4 normalises to NFC before validating, so the decomposed form is accepted", + "data": "cafe\u0301.com", + "valid": true + }, + { + "description": "a label of 63 fullwidth characters is valid after mapping", + "comment": "UTS 46 maps each U+FF41 to an ASCII letter, so this label is 63 octets in its mapped form", + "data": "\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41", + "valid": true + }, + { + "description": "an ACE prefix produced by the mapping step is decoded as an A-label", + "comment": "UTS 46 maps U+FF58 U+FF4E to \"xn\", so the mapped label is the A-label xn--nxasmq6b", + "data": "\uff58\uff4e--nxasmq6b", + "valid": true + }, + { + "description": "zero width non-joiner must pass at every occurrence", + "comment": "https://www.rfc-editor.org/rfc/rfc5892#appendix-A.1", + "data": "\u0915\u094d\u200c\u0937x\u200cy", + "valid": false + }, + { + "description": "Bidi domain name with a digit-first label is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2 a label in a Bidi domain name must start with an L, R or AL character", + "data": "0a.\u05d0", + "valid": false + }, + { + "description": "label starting with a digit before a right-to-left letter is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2", + "data": "0\u0627", + "valid": false + }, + { + "description": "left-to-right label containing a right-to-left letter is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2", + "data": "a\u05d0", + "valid": false + }, + { + "description": "right-to-left label mixing both digit types is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2", + "data": "\u05d00\u0660", + "valid": false + }, + { + "description": "A-label that decodes to a disallowed code point is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5890#section-2.3.2.1 https://www.rfc-editor.org/rfc/rfc5892#section-2.6", + "data": "xn--7a", + "valid": false + }, + { + "description": "A-label that decodes to a Bidi rule violation is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5890#section-2.3.2.1 https://www.rfc-editor.org/rfc/rfc5893", + "data": "xn--0ca24w", + "valid": false + }, + { + "description": "a U-label whose A-label form is longer than 63 octets is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5891#section-4.2.4 the 63-octet limit is on the A-label form, not the code point count", + "data": "\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc", + "valid": false + }, + { + "description": "empty label between two dots is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc1034#section-3.1", + "data": "a..b", + "valid": false + }, + { + "description": "a name longer than 253 characters is invalid", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "valid": false + }, + { + "description": "A-label that decodes to only ASCII is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5890#section-2.3.2.1 a U-label must have at least one non-ASCII character", + "data": "xn--example-", + "valid": false + }, + { + "description": "non-canonical Punycode that does not re-encode to itself is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5891#section-5.4 a label decoded from Punycode must be identical to the original A-label", + "data": "xn---9uc", + "valid": false + } + ] + }, + { + "description": "validation of separators in internationalized host names", + "specification": [ + {"rfc3490": "3.1", "quote": "Whenever dots are used as label separators, the following characters MUST be recognized as dots: U+002E (full stop), U+3002 (ideographic full stop), U+FF0E (fullwidth full stop), U+FF61(halfwidth ideographic full stop)"} + ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "idn-hostname" + }, + "tests": [ + { + "description": "single dot", + "data": ".", + "valid": false + }, + { + "description": "single ideographic full stop", + "data": "\u3002", + "valid": false + }, + { + "description": "single fullwidth full stop", + "data": "\uff0e", + "valid": false + }, + { + "description": "single halfwidth ideographic full stop", + "data": "\uff61", + "valid": false + }, + { + "description": "dot as label separator", + "data": "a.b", + "valid": true + }, + { + "description": "ideographic full stop as label separator", + "data": "a\u3002b", + "valid": true + }, + { + "description": "fullwidth full stop as label separator", + "data": "a\uff0eb", + "valid": true + }, + { + "description": "halfwidth ideographic full stop as label separator", + "data": "a\uff61b", + "valid": true + }, + { + "description": "leading dot", + "data": ".example", + "valid": false + }, + { + "description": "leading ideographic full stop", + "data": "\u3002example", + "valid": false + }, + { + "description": "leading fullwidth full stop", + "data": "\uff0eexample", + "valid": false + }, + { + "description": "leading halfwidth ideographic full stop", + "data": "\uff61example", + "valid": false + }, + { + "description": "trailing dot", + "data": "example.", + "valid": false + }, + { + "description": "trailing ideographic full stop", + "data": "example\u3002", + "valid": false + }, + { + "description": "trailing fullwidth full stop", + "data": "example\uff0e", + "valid": false + }, + { + "description": "trailing halfwidth ideographic full stop", + "data": "example\uff61", + "valid": false + }, + { + "description": "label too long if separator ignored (full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα.com", + "valid": true + }, + { + "description": "label too long if separator ignored (ideographic full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα\u3002com", + "valid": true + }, + { + "description": "label too long if separator ignored (fullwidth full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα\uff0ecom", + "valid": true + }, + { + "description": "label too long if separator ignored (halfwidth ideographic full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα\uff61com", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv4.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv4.json new file mode 100644 index 000000000..696713e90 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv4.json @@ -0,0 +1,217 @@ +[ + { + "description": "validation of IP addresses", + "comment": "RFC 2673, Section 3.2: dotted-quad = decbyte \".\" decbyte \".\" decbyte \".\" decbyte. A 'decbyte' (1*3DIGIT) restricts semantic values to 0-255, allows leading zeros, and strictly forbids symbols, alpha/hex, whitespace, and non-ASCII characters.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "ipv4" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IP address", + "data": "192.168.0.1", + "valid": true + }, + { + "description": "an IP address with too many components", + "data": "127.0.0.0.1", + "valid": false + }, + { + "description": "an IP address with out-of-range values", + "data": "256.256.256.256", + "valid": false + }, + { + "description": "an IP address without 4 components", + "data": "127.0", + "valid": false + }, + { + "description": "a 2-part address resolving to a routable IP (inet_aton shorthand)", + "data": "127.1", + "valid": false + }, + { + "description": "a 3-part address resolving to a routable IP (inet_aton shorthand)", + "data": "127.0.1", + "valid": false + }, + { + "description": "an IP address as an integer", + "data": "0x7f000001", + "valid": false + }, + { + "description": "an IP address as an integer (decimal)", + "data": "2130706433", + "valid": false + }, + { + "description": "invalid non-ASCII '২' (a Bengali 2)", + "data": "1২7.0.0.1", + "valid": false + }, + { + "description": "invalid fullwidth digits (non-ASCII)", + "data": "192.168.1.1", + "valid": false + }, + { + "description": "invalid mathematical bold digits (non-ASCII)", + "data": "𝟏𝟗𝟐.𝟏𝟔𝟖.𝟏.𝟏", + "valid": false + }, + { + "description": "netmask is not a part of ipv4 address", + "data": "192.168.1.0/24", + "valid": false + }, + { + "description": "leading whitespace is invalid", + "data": " 192.168.0.1", + "valid": false + }, + { + "description": "trailing whitespace is invalid", + "data": "192.168.0.1 ", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "192.168.0.1\n", + "valid": false + }, + { + "description": "hexadecimal notation is invalid", + "data": "0x7f.0.0.1", + "valid": false + }, + { + "description": "octal notation explicit is invalid", + "data": "0o10.0.0.1", + "valid": false + }, + { + "description": "empty part (double dot) is invalid", + "data": "192.168..1", + "valid": false + }, + { + "description": "leading dot is invalid", + "data": ".192.168.0.1", + "valid": false + }, + { + "description": "trailing dot is invalid", + "data": "192.168.0.1.", + "valid": false + }, + { + "description": "minimum valid IPv4 address", + "data": "0.0.0.0", + "valid": true + }, + { + "description": "maximum valid IPv4 address", + "data": "255.255.255.255", + "valid": true + }, + { + "description": "valid IPv4 with an octet in the 200 to 249 range", + "data": "200.0.0.0", + "valid": true + }, + { + "description": "valid IPv4 with two-digit octets", + "data": "10.20.30.40", + "valid": true + }, + { + "description": "empty string is invalid", + "data": "", + "valid": false + }, + { + "description": "plus sign is invalid", + "data": "+1.2.3.4", + "valid": false + }, + { + "description": "negative sign is invalid", + "data": "-1.2.3.4", + "valid": false + }, + { + "description": "exponential notation is invalid", + "data": "1e2.0.0.1", + "valid": false + }, + { + "description": "alpha characters are invalid", + "data": "192.168.a.1", + "valid": false + }, + { + "description": "internal whitespace is invalid", + "data": "192. 168.0.1", + "valid": false + }, + { + "description": "tab character is invalid", + "data": "192.168.0.1\t", + "valid": false + }, + { + "description": "additional content after an embedded NUL byte", + "data": "192.168.0.1\u0000.evil.com", + "valid": false + }, + { + "description": "with port number is invalid", + "data": "192.168.0.1:80", + "valid": false + }, + { + "description": "an IPv4-mapped IPv6 address is invalid", + "data": "::ffff:192.168.0.1", + "valid": false + }, + { + "description": "single octet out of range in last position", + "data": "192.168.0.256", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv6.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv6.json new file mode 100644 index 000000000..e5b46a00f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv6.json @@ -0,0 +1,266 @@ +[ + { + "description": "validation of IPv6 addresses", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "ipv6" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IPv6 address", + "data": "::1", + "valid": true + }, + { + "description": "a group with 5 hex digits is invalid", + "data": "12345::", + "valid": false + }, + { + "description": "trailing 4 hex symbols is valid", + "data": "::abef", + "valid": true + }, + { + "description": "trailing 5 hex symbols is invalid", + "data": "::abcef", + "valid": false + }, + { + "description": "an IPv6 address with too many components", + "data": "1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1", + "valid": false + }, + { + "description": "an IPv6 address containing illegal characters", + "data": "::laptop", + "valid": false + }, + { + "description": "no digits is valid", + "data": "::", + "valid": true + }, + { + "description": "leading colons is valid", + "data": "::42:ff:1", + "valid": true + }, + { + "description": "trailing colons is valid", + "data": "d6::", + "valid": true + }, + { + "description": "missing leading octet is invalid", + "data": ":2:3:4:5:6:7:8", + "valid": false + }, + { + "description": "missing trailing octet is invalid", + "data": "1:2:3:4:5:6:7:", + "valid": false + }, + { + "description": "missing leading octet with omitted octets later", + "data": ":2:3:4::8", + "valid": false + }, + { + "description": "single set of double colons in the middle is valid", + "data": "1:d6::42", + "valid": true + }, + { + "description": "two sets of double colons is invalid", + "data": "1::d6::42", + "valid": false + }, + { + "description": "mixed format with the ipv4 section as decimal octets", + "data": "1::d6:192.168.0.1", + "valid": true + }, + { + "description": "mixed format with double colons between the sections", + "data": "1:2::192.168.0.1", + "valid": true + }, + { + "description": "mixed format with ipv4 section with octet out of range", + "data": "1::2:192.168.256.1", + "valid": false + }, + { + "description": "mixed format with ipv4 section with a hex octet", + "data": "1::2:192.168.ff.1", + "valid": false + }, + { + "description": "mixed format with leading double colons (ipv4-mapped ipv6 address)", + "data": "::ffff:192.168.0.1", + "valid": true + }, + { + "description": "triple colons is invalid", + "data": "1:2:3:4:5:::8", + "valid": false + }, + { + "description": "8 octets", + "data": "1:2:3:4:5:6:7:8", + "valid": true + }, + { + "description": "insufficient octets without double colons", + "data": "1:2:3:4:5:6:7", + "valid": false + }, + { + "description": "no colons is invalid", + "data": "1", + "valid": false + }, + { + "description": "ipv4 is not ipv6", + "data": "127.0.0.1", + "valid": false + }, + { + "description": "ipv4 segment must have 4 octets", + "data": "1:2:3:4:1.2.3", + "valid": false + }, + { + "description": "leading whitespace is invalid", + "data": " ::1", + "valid": false + }, + { + "description": "trailing whitespace is invalid", + "data": "::1 ", + "valid": false + }, + { + "description": "netmask is not a part of ipv6 address", + "data": "fe80::/64", + "valid": false + }, + { + "description": "zone id is not a part of ipv6 address", + "data": "fe80::a%eth1", + "valid": false + }, + { + "description": "a long valid ipv6", + "data": "1000:1000:1000:1000:1000:1000:255.255.255.255", + "valid": true + }, + { + "description": "a long invalid ipv6, below length limit, first", + "data": "100:100:100:100:100:100:255.255.255.255.255", + "valid": false + }, + { + "description": "a long invalid ipv6, below length limit, second", + "data": "100:100:100:100:100:100:100:255.255.255.255", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4)", + "data": "1:2:3:4:5:6:7:৪", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4) in the IPv4 portion", + "data": "1:2::192.16৪.0.1", + "valid": false + }, + { + "description": "a bracketed address is invalid", + "data": "[::1]", + "valid": false + }, + { + "description": "a leading zero in the last IPv4 octet is invalid", + "data": "::ffff:192.168.0.01", + "valid": false + }, + { + "description": "uppercase hex digits are valid", + "data": "2001:DB8::1", + "valid": true + }, + { + "description": "leading zeros in every group are valid", + "data": "2001:0db8:0000:0000:0000:0000:0000:0001", + "valid": true + }, + { + "description": "two groups before '::' and four groups after is valid", + "data": "1:2::3:4:5:6", + "valid": true + }, + { + "description": "eight groups alongside '::' is invalid", + "data": "1:2:3:4:5:6:7:8::", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "::1\n", + "valid": false + }, + { + "description": "an out-of-range first octet in the embedded IPv4 is invalid", + "data": "::ffff:256.1.1.1", + "valid": false + }, + { + "description": "one group before '::' and five groups after is valid", + "data": "1::2:3:4:5:6", + "valid": true + }, + { + "description": "'::' followed by six groups is valid", + "data": "::1:2:3:4:5:6", + "valid": true + }, + { + "description": "leading '::' before five groups and an embedded IPv4 is valid", + "data": "::1:2:3:4:5:1.2.3.4", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri-reference.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri-reference.json new file mode 100644 index 000000000..d31cdbbea --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri-reference.json @@ -0,0 +1,116 @@ +[ + { + "description": "validation of IRI References", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "iri-reference" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IRI", + "data": "http://ƒøø.ßår/?∂éœ=πîx#πîüx", + "valid": true + }, + { + "description": "a valid protocol-relative IRI Reference", + "data": "//ƒøø.ßår/?∂éœ=πîx#πîüx", + "valid": true + }, + { + "description": "a valid relative IRI Reference", + "data": "/âππ", + "valid": true + }, + { + "description": "an invalid IRI Reference", + "data": "\\\\WINDOWS\\filëßåré", + "valid": false + }, + { + "description": "a valid IRI Reference", + "data": "âππ", + "valid": true + }, + { + "description": "a valid IRI fragment", + "data": "#ƒrägmênt", + "valid": true + }, + { + "description": "an invalid IRI fragment", + "data": "#ƒräg\\mênt", + "valid": false + }, + { + "description": "an embedded IPv4 with a leading zero is invalid", + "data": "//[::ffff:192.168.0.01]/p", + "valid": false + }, + { + "description": "a valid protocol-relative IRI Reference with a compressed IPv6 host", + "data": "//[2001:db8::1]/p", + "valid": true + }, + { + "description": "a valid relative IRI Reference with a supplementary-plane character", + "data": "/âππ/𐌀", + "valid": true + }, + { + "description": "a query-only IRI Reference with a supplementary private-use char", + "data": "?q=󰀀", + "valid": true + }, + { + "description": "a trailing newline after a valid IRI Reference is invalid", + "data": "/âππ\n", + "valid": false + }, + { + "description": "invalid percent-encoding with non-hex digits", + "data": "/%6G", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "data": "/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "data": "/%", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri.json new file mode 100644 index 000000000..db27c829a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri.json @@ -0,0 +1,141 @@ +[ + { + "description": "validation of IRIs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "iri" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IRI with anchor tag", + "data": "http://ƒøø.ßår/?∂éœ=πîx#πîüx", + "valid": true + }, + { + "description": "a valid IRI with anchor tag and parentheses", + "data": "http://ƒøø.com/blah_(wîkïpédiå)_blah#ßité-1", + "valid": true + }, + { + "description": "a valid IRI with URL-encoded stuff", + "data": "http://ƒøø.ßår/?q=Test%20URL-encoded%20stuff", + "valid": true + }, + { + "description": "invalid percent-encoding with non-hex digits", + "data": "http://ƒøø.ßår/%6G", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "data": "http://ƒøø.ßår/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "data": "http://ƒøø.ßår/%", + "valid": false + }, + { + "description": "a valid IRI with many special characters", + "data": "http://-.~_!$&'()*+,;=:%40:80%2f::::::@example.com", + "valid": true + }, + { + "description": "a valid IRI based on IPv6", + "data": "http://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]", + "valid": true + }, + { + "description": "an IPv6 address without enclosing brackets is invalid", + "data": "http://2001:0db8:85a3:0000:0000:8a2e:0370:7334", + "valid": false + }, + { + "description": "an invalid relative IRI Reference", + "data": "/abc", + "valid": false + }, + { + "description": "an invalid IRI", + "data": "\\\\WINDOWS\\filëßåré", + "valid": false + }, + { + "description": "an invalid IRI though valid IRI reference", + "data": "âππ", + "valid": false + }, + { + "description": "an IPv6 host whose embedded IPv4 has a leading zero is invalid", + "data": "http://[::ffff:192.168.0.01]", + "valid": false + }, + { + "description": "a valid IRI with a compressed IPv6 host", + "data": "http://[2001:db8::1]", + "valid": true + }, + { + "description": "a trailing newline after a valid IRI is invalid", + "data": "http://ƒøø.ßår/\n", + "valid": false + }, + { + "description": "a valid IRI with an IPv4 host", + "data": "http://192.168.0.1/p", + "valid": true + }, + { + "description": "a valid IRI with no authority and a rootless path", + "data": "urn:example:resource", + "valid": true + }, + { + "description": "a valid IRI with no authority and an absolute path", + "data": "file:/etc/hosts", + "valid": true + }, + { + "description": "a valid IRI with a supplementary-plane character in the path", + "data": "http://ƒøø.ßår/𐌀", + "valid": true + }, + { + "description": "a valid IRI with a supplementary-plane private-use char in query", + "data": "http://ƒøø.ßår/?q=󰀀", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/json-pointer.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/json-pointer.json new file mode 100644 index 000000000..d5b9a8bb1 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/json-pointer.json @@ -0,0 +1,211 @@ +[ + { + "description": "validation of JSON-pointers (JSON String Representation)", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "json-pointer" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid JSON-pointer", + "data": "/foo/bar~0/baz~1/%a", + "valid": true + }, + { + "description": "not a valid JSON-pointer (~ not escaped)", + "data": "/foo/bar~", + "valid": false + }, + { + "description": "valid JSON-pointer with empty segment", + "data": "/foo//bar", + "valid": true + }, + { + "description": "valid JSON-pointer with the last empty segment", + "data": "/foo/bar/", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #1", + "data": "", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #2", + "data": "/foo", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #3", + "data": "/foo/0", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #4", + "data": "/", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #5", + "data": "/a~1b", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #6", + "data": "/c%d", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #7", + "data": "/e^f", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #8", + "data": "/g|h", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #9", + "data": "/i\\j", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #10", + "data": "/k\"l", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #11", + "data": "/ ", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #12", + "data": "/m~0n", + "valid": true + }, + { + "description": "valid JSON-pointer used adding to the last array position", + "data": "/foo/-", + "valid": true + }, + { + "description": "valid JSON-pointer (- used as object member name)", + "data": "/foo/-/bar", + "valid": true + }, + { + "description": "valid JSON-pointer (multiple escaped characters)", + "data": "/~1~0~0~1~1", + "valid": true + }, + { + "description": "valid JSON-pointer (escaped with fraction part) #1", + "data": "/~1.1", + "valid": true + }, + { + "description": "valid JSON-pointer (escaped with fraction part) #2", + "data": "/~0.1", + "valid": true + }, + { + "description": "not a valid JSON-pointer (URI Fragment Identifier) #1", + "data": "#", + "valid": false + }, + { + "description": "not a valid JSON-pointer (URI Fragment Identifier) #2", + "data": "#/", + "valid": false + }, + { + "description": "not a valid JSON-pointer (URI Fragment Identifier) #3", + "data": "#a", + "valid": false + }, + { + "description": "not a valid JSON-pointer (some escaped, but not all) #1", + "data": "/~0~", + "valid": false + }, + { + "description": "not a valid JSON-pointer (some escaped, but not all) #2", + "data": "/~0/~", + "valid": false + }, + { + "description": "not a valid JSON-pointer (wrong escape character) #1", + "data": "/~2", + "valid": false + }, + { + "description": "not a valid JSON-pointer (wrong escape character) #2", + "data": "/~-1", + "valid": false + }, + { + "description": "not a valid JSON-pointer (multiple characters not escaped)", + "data": "/~~", + "valid": false + }, + { + "description": "not a valid JSON-pointer (isn't empty nor starts with /) #1", + "data": "a", + "valid": false + }, + { + "description": "not a valid JSON-pointer (isn't empty nor starts with /) #2", + "data": "0", + "valid": false + }, + { + "description": "not a valid JSON-pointer (isn't empty nor starts with /) #3", + "data": "a/a", + "valid": false + }, + { + "description": "valid JSON-pointer (Unicode characters allowed by RFC 6901)", + "data": "/foo/bar/😎", + "valid": true + }, + { + "description": "valid JSON-pointer (control characters allowed after JSON unescaping)", + "data": "/foo\u0000bar\n\tbaz", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/regex.json new file mode 100644 index 000000000..dc3e72937 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/regex.json @@ -0,0 +1,103 @@ +[ + { + "description": "validation of regular expressions", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid regular expression", + "data": "([abc])+\\s+$", + "valid": true + }, + { + "description": "a regular expression with unclosed parens is invalid", + "data": "^(abc]", + "valid": false + }, + { + "description": "a quantifier with nothing to repeat is invalid", + "comment": "https://tc39.es/ecma262/#sec-patterns a Quantifier follows something (Term :: Atom Quantifier), so a pattern cannot begin with one", + "data": "*a", + "valid": false + }, + { + "description": "a character class range in reverse order is invalid", + "comment": "https://tc39.es/ecma262/#sec-patterns-static-semantics-early-errors it is a Syntax Error if the first ClassAtom of a range has a greater CharacterValue than the second", + "data": "[z-a]", + "valid": false + }, + { + "description": "a trailing backslash is invalid", + "data": "\\", + "valid": false + }, + { + "description": "an unmatched closing parenthesis is invalid", + "data": "a)", + "valid": false + }, + { + "description": "an unterminated character class is invalid", + "data": "[abc", + "valid": false + }, + { + "description": "an empty regular expression is valid", + "data": "", + "valid": true + }, + { + "description": "alternation is valid", + "data": "a|b", + "valid": true + }, + { + "description": "a bounded quantifier is valid", + "data": "a{2,3}", + "valid": true + }, + { + "description": "a negated range character class is valid", + "data": "[^a-z]", + "valid": true + }, + { + "description": "a lazy quantifier is valid", + "data": "a+?", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/relative-json-pointer.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/relative-json-pointer.json new file mode 100644 index 000000000..413c03fd8 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/relative-json-pointer.json @@ -0,0 +1,141 @@ +[ + { + "description": "validation of Relative JSON Pointers (RJP)", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "relative-json-pointer" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid upwards RJP", + "data": "1", + "valid": true + }, + { + "description": "a valid downwards RJP", + "data": "0/foo/bar", + "valid": true + }, + { + "description": "a valid up and then down RJP, with array index", + "data": "2/0/baz/1/zip", + "valid": true + }, + { + "description": "a valid RJP taking the member or index name", + "data": "0#", + "valid": true + }, + { + "description": "an invalid RJP that is a valid JSON Pointer", + "data": "/foo/bar", + "valid": false + }, + { + "description": "a json-pointer part that does not start with a slash is invalid", + "data": "1foo", + "valid": false + }, + { + "description": "negative prefix", + "data": "-1/foo/bar", + "valid": false + }, + { + "description": "explicit positive prefix", + "data": "+1/foo/bar", + "valid": false + }, + { + "description": "non-ASCII digit in the prefix is not allowed", + "data": "١/foo", + "valid": false + }, + { + "description": "## is not a valid json-pointer", + "data": "0##", + "valid": false + }, + { + "description": "zero cannot be followed by other digits, plus json-pointer", + "data": "01/a", + "valid": false + }, + { + "description": "zero cannot be followed by other digits, plus octothorpe", + "data": "01#", + "valid": false + }, + { + "description": "empty string", + "data": "", + "valid": false + }, + { + "description": "multi-digit integer prefix", + "data": "120/foo/bar", + "valid": true + }, + { + "description": "multi-digit prefix with a zero followed by another digit", + "data": "100", + "valid": true + }, + { + "description": "tilde not followed by 0 or 1 in the json-pointer part", + "data": "0/~2", + "valid": false + }, + { + "description": "unescaped tilde at the end of the json-pointer part", + "data": "0/foo/bar~", + "valid": false + }, + { + "description": "octothorpe followed by a json-pointer", + "data": "1#/foo/bar", + "valid": false + }, + { + "description": "empty reference tokens in the json-pointer part", + "data": "0//", + "valid": true + }, + { + "description": "trailing newline after the non-negative integer", + "data": "1\n", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/time.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/time.json new file mode 100644 index 000000000..2f4b6353f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/time.json @@ -0,0 +1,303 @@ +[ + { + "description": "validation of time strings", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "time" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid time string", + "data": "08:30:06Z", + "valid": true + }, + { + "description": "invalid time string with extra leading zeros", + "data": "008:030:006Z", + "valid": false + }, + { + "description": "invalid time string with no leading zero for single digit", + "data": "8:3:6Z", + "valid": false + }, + { + "description": "hour, minute, second must be two digits", + "data": "8:0030:6Z", + "valid": false + }, + { + "description": "a valid time string with leap second, Zulu", + "data": "23:59:60Z", + "valid": true + }, + { + "description": "invalid leap second, Zulu (wrong hour)", + "data": "22:59:60Z", + "valid": false + }, + { + "description": "invalid leap second, Zulu (wrong minute)", + "data": "23:58:60Z", + "valid": false + }, + { + "description": "valid leap second, zero time-offset", + "data": "23:59:60+00:00", + "valid": true + }, + { + "description": "invalid leap second, zero time-offset (wrong hour)", + "data": "22:59:60+00:00", + "valid": false + }, + { + "description": "invalid leap second, zero time-offset (wrong minute)", + "data": "23:58:60+00:00", + "valid": false + }, + { + "description": "valid leap second, positive time-offset", + "data": "01:29:60+01:30", + "valid": true + }, + { + "description": "valid leap second, large positive time-offset", + "data": "23:29:60+23:30", + "valid": true + }, + { + "description": "invalid leap second, positive time-offset (wrong hour)", + "data": "23:59:60+01:00", + "valid": false + }, + { + "description": "invalid leap second, positive time-offset (wrong minute)", + "data": "23:59:60+00:30", + "valid": false + }, + { + "description": "valid leap second, negative time-offset", + "data": "15:59:60-08:00", + "valid": true + }, + { + "description": "valid leap second, large negative time-offset", + "data": "00:29:60-23:30", + "valid": true + }, + { + "description": "invalid leap second, negative time-offset (wrong hour)", + "data": "23:59:60-01:00", + "valid": false + }, + { + "description": "invalid leap second, negative time-offset (wrong minute)", + "data": "23:59:60-00:30", + "valid": false + }, + { + "description": "a valid time string with second fraction", + "data": "23:20:50.52Z", + "valid": true + }, + { + "description": "a valid time string with precise second fraction", + "data": "08:30:06.283185Z", + "valid": true + }, + { + "description": "a valid time string with plus offset", + "data": "08:30:06+00:20", + "valid": true + }, + { + "description": "a valid time string with minus offset", + "data": "08:30:06-08:00", + "valid": true + }, + { + "description": "time with unknown local offset is valid", + "comment": "RFC 3339 section 4.3 (unknown local offset)", + "data": "12:34:56-00:00", + "valid": true + }, + { + "description": "hour, minute in time-offset must be two digits", + "data": "08:30:06-8:000", + "valid": false + }, + { + "description": "a valid time string with case-insensitive Z", + "data": "08:30:06z", + "valid": true + }, + { + "description": "an invalid time string with invalid hour", + "data": "24:00:00Z", + "valid": false + }, + { + "description": "an invalid time string with invalid minute", + "data": "00:60:00Z", + "valid": false + }, + { + "description": "an invalid time string with invalid second", + "data": "00:00:61Z", + "valid": false + }, + { + "description": "an invalid time string with invalid time numoffset hour", + "data": "01:02:03+24:00", + "valid": false + }, + { + "description": "an invalid time string with invalid time numoffset minute", + "data": "01:02:03+00:60", + "valid": false + }, + { + "description": "an invalid time string with invalid time with both Z and numoffset", + "data": "01:02:03Z+00:30", + "valid": false + }, + { + "description": "an invalid offset indicator", + "data": "08:30:06 PST", + "valid": false + }, + { + "description": "only RFC3339 not all of ISO 8601 are valid", + "data": "01:01:01,1111", + "valid": false + }, + { + "description": "no time offset", + "data": "12:00:00", + "valid": false + }, + { + "description": "no time offset with second fraction", + "data": "12:00:00.52", + "valid": false + }, + { + "description": "invalid non-ASCII '২' (a Bengali 2)", + "data": "1২:00:00Z", + "valid": false + }, + { + "description": "offset not starting with plus or minus", + "data": "08:30:06#00:20", + "valid": false + }, + { + "description": "contains letters", + "data": "ab:cd:ef", + "valid": false + }, + { + "description": "an invalid time string in date-time format", + "data": "2020-11-28T23:55:45Z", + "valid": false + }, + { + "description": "a numeric time-offset requires the colon separator", + "comment": "RFC 3339 section 5.6: time-numoffset = (\"+\" / \"-\") time-hour \":\" time-minute", + "data": "08:30:06+0130", + "valid": false + }, + { + "description": "a numeric time-offset requires the minutes component", + "comment": "RFC 3339 section 5.6: time-numoffset = (\"+\" / \"-\") time-hour \":\" time-minute", + "data": "08:30:06+01", + "valid": false + }, + { + "description": "a trailing newline after a valid time", + "comment": "RFC 3339 section 5.6: no production of full-time emits a line feed", + "data": "08:30:06Z\n", + "valid": false + }, + { + "description": "hour 24 is invalid with a numeric time-offset", + "comment": "RFC 3339 section 5.6: time-hour = 2DIGIT ; 00-23, and section 5.7 only allows 00 to 23", + "data": "24:59:00+01:00", + "valid": false + }, + { + "description": "minute 60 is invalid with a numeric time-offset", + "comment": "RFC 3339 section 5.6: time-minute = 2DIGIT ; 00-59", + "data": "23:60:00+00:01", + "valid": false + }, + { + "description": "a valid time string with a 15 digit second fraction", + "comment": "RFC 3339 section 5.6: time-secfrac = \".\" 1*DIGIT, with no upper bound on the number of digits", + "data": "00:59:59.999999999999999Z", + "valid": true + }, + { + "description": "a second fraction with no digits", + "comment": "RFC 3339 section 5.6: time-secfrac = \".\" 1*DIGIT requires at least one digit", + "data": "08:30:06.Z", + "valid": false + }, + { + "description": "partial-time requires the seconds component", + "comment": "RFC 3339 section 5.6: partial-time = time-hour \":\" time-minute \":\" time-second [time-secfrac]", + "data": "12:00Z", + "valid": false + }, + { + "description": "a comma decimal separator with a time-offset", + "comment": "RFC 3339 section 5.6: the decimal separator in time-secfrac is \".\" - the ISO 8601 comma is not part of this profile", + "data": "08:30:06,5Z", + "valid": false + }, + { + "description": "leading whitespace before a valid time", + "comment": "RFC 3339 section 5.6: no production of full-time emits whitespace", + "data": " 08:30:06Z", + "valid": false + }, + { + "description": "trailing whitespace after a valid time", + "comment": "RFC 3339 section 5.6: no production of full-time emits whitespace", + "data": "08:30:06Z ", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/unknown.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/unknown.json new file mode 100644 index 000000000..7fc35f536 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/unknown.json @@ -0,0 +1,46 @@ +[ + { + "description": "unknown format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "unknown" + }, + "tests": [ + { + "description": "unknown formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "unknown formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "unknown formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "unknown formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "unknown formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "unknown formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "unknown formats ignore strings", + "data": "string", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-reference.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-reference.json new file mode 100644 index 000000000..08c655e00 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-reference.json @@ -0,0 +1,182 @@ +[ + { + "description": "validation of URI References", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uri-reference" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid URI", + "data": "http://foo.bar/?baz=qux#quux", + "valid": true + }, + { + "description": "a valid protocol-relative URI Reference", + "data": "//foo.bar/?baz=qux#quux", + "valid": true + }, + { + "description": "a valid relative URI Reference", + "data": "/abc", + "valid": true + }, + { + "description": "an invalid URI Reference", + "data": "\\\\WINDOWS\\fileshare", + "valid": false + }, + { + "description": "a valid URI Reference", + "data": "abc", + "valid": true + }, + { + "description": "a valid URI fragment", + "data": "#fragment", + "valid": true + }, + { + "description": "an invalid URI fragment", + "data": "#frag\\ment", + "valid": false + }, + { + "description": "unescaped non US-ASCII characters", + "data": "/foobar®.txt", + "valid": false + }, + { + "description": "invalid backslash character", + "data": "https://example.org/foobar\\.txt", + "valid": false + }, + { + "description": "URI with leading-zero IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: If an IP literal fails strict IPv4address parsing, it falls back to reg-name. A string of digits and dots is valid under unreserved characters. JSON Schema format asserts syntax, not scheme-specific DNS semantics.", + "data": "http://087.10.0.1/", + "valid": true + }, + { + "description": "URI with out-of-bounds IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: Fallback to reg-name allows digits and dots.", + "data": "http://999.999.999.999/", + "valid": true + }, + { + "description": "an empty URI reference", + "comment": "RFC 3986, Section 4.2: relative-part includes path-empty, which is zero characters.", + "data": "", + "valid": true + }, + { + "description": "a query-only reference", + "comment": "RFC 3986, Section 4.2: relative-ref = relative-part [ \"?\" query ] [ \"#\" fragment ], so a query with an empty path is a reference.", + "data": "?query=1", + "valid": true + }, + { + "description": "a network-path reference with an empty authority", + "comment": "RFC 3986, Section 3.2.2: host = IP-literal / IPv4address / reg-name, and reg-name may be empty.", + "data": "//", + "valid": true + }, + { + "description": "an incomplete percent-encoding", + "comment": "RFC 3986, Section 2.1: pct-encoded = \"%\" HEXDIG HEXDIG.", + "data": "/%zz", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "comment": "RFC 3986, Section 2.1: pct-encoded = \"%\" HEXDIG HEXDIG.", + "data": "/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "comment": "RFC 3986, Section 2.1: pct-encoded = \"%\" HEXDIG HEXDIG.", + "data": "/%", + "valid": false + }, + { + "description": "a double quote in a path", + "comment": "RFC 3986, Appendix A: pchar = unreserved / pct-encoded / sub-delims / \":\" / \"@\", and the double quote is in none of them.", + "data": "/a\"b", + "valid": false + }, + { + "description": "square brackets outside an authority", + "comment": "RFC 3986, Section 3.2.2: square brackets are reserved for an IP-literal inside the authority. path-absolute is built from pchar, which excludes them.", + "data": "/[::1]", + "valid": false + }, + { + "description": "a non-numeric port in a network-path reference", + "comment": "RFC 3986, Section 3.2.3: port = *DIGIT. reg-name admits no colon, so the text after the colon can only be a port.", + "data": "//example.com:abc/p", + "valid": false + }, + { + "description": "more than one at-sign in the authority", + "comment": "RFC 3986, Section 3.2: authority = [ userinfo \"@\" ] host [ \":\" port ]. Neither userinfo nor host admits a literal at-sign.", + "data": "//a@b@example.com/", + "valid": false + }, + { + "description": "a leading zero in the IPv4 part of an IPv6 literal", + "comment": "RFC 3986, Section 3.2.2: inside an IP-literal the dotted quad must be an IPv4address, and dec-octet has no leading-zero alternative. There is no reg-name fallback inside brackets.", + "data": "//[::ffff:192.168.0.01]/p", + "valid": false + }, + { + "description": "a colon in the first segment of a relative-path reference", + "comment": "RFC 3986, Section 4.2: relative-part uses path-noscheme, whose first segment is segment-nz-nc - a non-zero-length segment without any colon. A scheme must start with ALPHA, so 1:b cannot be read as a URI either.", + "data": "1:b", + "valid": false + }, + { + "description": "a colon in the first segment preceded by a dot-segment", + "comment": "RFC 3986, Section 4.2: \"Such a segment must be preceded by a dot-segment (e.g., \\\"./this:that\\\") to make a relative-path reference.\"", + "data": "./this:that", + "valid": true + }, + { + "description": "a trailing line feed", + "comment": "RFC 3986, Appendix A: no production admits %x0A.", + "data": "/p\n", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-template.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-template.json new file mode 100644 index 000000000..58d28f9de --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-template.json @@ -0,0 +1,251 @@ +[ + { + "description": "format: uri-template", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uri-template" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid uri-template", + "data": "http://example.com/dictionary/{term:1}/{term}", + "valid": true + }, + { + "description": "an invalid uri-template", + "data": "http://example.com/dictionary/{term:1}/{term", + "valid": false + }, + { + "description": "a valid uri-template without variables", + "data": "http://example.com/dictionary", + "valid": true + }, + { + "description": "a valid relative uri-template", + "data": "dictionary/{term:1}/{term}", + "valid": true + }, + { + "description": "an empty expression is invalid", + "data": "{}", + "valid": false + }, + { + "description": "an empty varspec inside the variable list is invalid", + "data": "{a,,b}", + "valid": false + }, + { + "description": "a zero prefix max-length is invalid", + "data": "{v:0}", + "valid": false + }, + { + "description": "a five-digit prefix max-length is invalid", + "data": "{v:10000}", + "valid": false + }, + { + "description": "an empty string is a valid uri-template", + "data": "", + "valid": true + }, + { + "description": "an unmatched closing brace is invalid", + "data": "foo}bar", + "valid": false + }, + { + "description": "a percent-encoded triplet in a literal is valid", + "data": "a%41b", + "valid": true + }, + { + "description": "a literal with a lone percent sign is invalid", + "data": "a%", + "valid": false + }, + { + "description": "a literal with an incomplete percent-encoded triplet is invalid", + "data": "a%4", + "valid": false + }, + { + "description": "a literal with non-hex percent encoding is invalid", + "data": "a%GG", + "valid": false + }, + { + "description": "a supplementary plane character in a literal is valid", + "data": "a\ud83d\ude00b", + "valid": true + }, + { + "description": "a private-use character in a literal is valid", + "data": "a\ue000b", + "valid": true + }, + { + "description": "a supplementary private-use character in a literal is valid", + "data": "a\udb80\udc00b", + "valid": true + }, + { + "description": "reserved expansion operator", + "data": "{+var}", + "valid": true + }, + { + "description": "fragment expansion operator", + "data": "{#var}", + "valid": true + }, + { + "description": "label expansion operator", + "data": "{.var}", + "valid": true + }, + { + "description": "path segment expansion operator", + "data": "{/var}", + "valid": true + }, + { + "description": "path style parameter expansion operator", + "data": "{;var}", + "valid": true + }, + { + "description": "form style query expansion operator", + "data": "{?var}", + "valid": true + }, + { + "description": "form style query continuation operator", + "data": "{&var}", + "valid": true + }, + { + "description": "the explode modifier is valid", + "data": "{var*}", + "valid": true + }, + { + "description": "combined prefix and explode modifiers are invalid", + "data": "{var:1*}", + "valid": false + }, + { + "description": "a variable list with more than one varspec is valid", + "data": "{x,y}", + "valid": true + }, + { + "description": "a dotted variable name is valid", + "data": "{a.b}", + "valid": true + }, + { + "description": "a trailing dot in a variable name is invalid", + "data": "{a.}", + "valid": false + }, + { + "description": "a double dot in a variable name is invalid", + "data": "{a..b}", + "valid": false + }, + { + "description": "a variable name may start with a percent-encoded triplet", + "data": "{%41}", + "valid": true + }, + { + "description": "a variable name with a lone percent sign is invalid", + "data": "{%}", + "valid": false + }, + { + "description": "a variable name with an incomplete percent-encoded triplet is invalid", + "data": "{%4}", + "valid": false + }, + { + "description": "a variable name with non-hex percent encoding is invalid", + "data": "{%GG}", + "valid": false + }, + { + "description": "a four-digit prefix max-length is valid", + "data": "{v:1000}", + "valid": true + }, + { + "description": "a leading zero in the prefix max-length is invalid", + "data": "{v:01}", + "valid": false + }, + { + "description": "a space in a literal is invalid", + "data": "a b", + "valid": false + }, + { + "description": "a delete character in a literal is invalid", + "data": "a\u007fb", + "valid": false + }, + { + "description": "a trailing comma in the variable list is invalid", + "data": "{a,}", + "valid": false + }, + { + "description": "a default value in a varspec is invalid", + "data": "{var=def}", + "valid": false + }, + { + "description": "an operator in place of the first varspec is invalid", + "data": "{,+var}", + "valid": false + }, + { + "description": "an apostrophe in a literal is valid", + "data": "a'b", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri.json new file mode 100644 index 000000000..e3736ccaf --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri.json @@ -0,0 +1,253 @@ +[ + { + "description": "validation of URIs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uri" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid URL with anchor tag", + "data": "http://foo.bar/?baz=qux#quux", + "valid": true + }, + { + "description": "a valid URL with anchor tag and parentheses", + "data": "http://foo.com/blah_(wikipedia)_blah#cite-1", + "valid": true + }, + { + "description": "a valid URL with URL-encoded stuff", + "data": "http://foo.bar/?q=Test%20URL-encoded%20stuff", + "valid": true + }, + { + "description": "a valid puny-coded URL ", + "data": "http://xn--nw2a.xn--j6w193g/", + "valid": true + }, + { + "description": "a valid URL with many special characters", + "data": "http://-.~_!$&'()*+,;=:%40:80%2f::::::@example.com", + "valid": true + }, + { + "description": "a valid URL based on IPv4", + "data": "http://223.255.255.254", + "valid": true + }, + { + "description": "a valid URL with ftp scheme", + "data": "ftp://ftp.is.co.za/rfc/rfc1808.txt", + "valid": true + }, + { + "description": "a valid URL for a simple text file", + "data": "http://www.ietf.org/rfc/rfc2396.txt", + "valid": true + }, + { + "description": "a valid URL", + "data": "ldap://[2001:db8::7]/c=GB?objectClass?one", + "valid": true + }, + { + "description": "a valid mailto URI", + "data": "mailto:John.Doe@example.com", + "valid": true + }, + { + "description": "a valid newsgroup URI", + "data": "news:comp.infosystems.www.servers.unix", + "valid": true + }, + { + "description": "a valid tel URI", + "data": "tel:+1-816-555-1212", + "valid": true + }, + { + "description": "a valid URN", + "data": "urn:oasis:names:specification:docbook:dtd:xml:4.1.2", + "valid": true + }, + { + "description": "an invalid protocol-relative URI Reference", + "data": "//foo.bar/?baz=qux#quux", + "valid": false + }, + { + "description": "an invalid relative URI Reference", + "data": "/abc", + "valid": false + }, + { + "description": "an invalid URI", + "data": "\\\\WINDOWS\\fileshare", + "valid": false + }, + { + "description": "an invalid URI though valid URI reference", + "data": "abc", + "valid": false + }, + { + "description": "an invalid URI with spaces", + "data": "http:// shouldfail.com", + "valid": false + }, + { + "description": "an invalid URI with spaces and missing scheme", + "data": ":// should fail", + "valid": false + }, + { + "description": "an invalid URI with comma in scheme", + "data": "bar,baz:foo", + "valid": false + }, + { + "description": "invalid userinfo", + "data": "https://[@example.org/test.txt", + "valid": false + }, + { + "description": "unescaped non US-ASCII characters", + "data": "https://example.org/foobar®.txt", + "valid": false + }, + { + "description": "invalid backslash character", + "data": "https://example.org/foobar\\.txt", + "valid": false + }, + { + "description": "invalid \" character", + "data": "https://example.org/foobar\".txt", + "valid": false + }, + { + "description": "invalid <> characters", + "data": "https://example.org/foobar<>.txt", + "valid": false + }, + { + "description": "invalid {} characters", + "data": "https://example.org/foobar{}.txt", + "valid": false + }, + { + "description": "invalid ^ character", + "data": "https://example.org/foobar^.txt", + "valid": false + }, + { + "description": "invalid ` character", + "data": "https://example.org/foobar`.txt", + "valid": false + }, + { + "description": "invalid SPACE character", + "data": "https://example.org/foo bar.txt", + "valid": false + }, + { + "description": "invalid | character", + "data": "https://example.org/foobar|.txt", + "valid": false + }, + { + "description": "URI with leading-zero IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: If an IP literal fails strict IPv4address parsing, it falls back to reg-name. A string of digits and dots is valid under unreserved characters. JSON Schema format asserts syntax, not scheme-specific DNS semantics.", + "data": "http://087.10.0.1/", + "valid": true + }, + { + "description": "URI with out-of-bounds IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: Fallback to reg-name allows digits and dots.", + "data": "http://999.999.999.999/", + "valid": true + }, + { + "description": "invalid percent-encoding with non-hex digits", + "data": "http://example.com/%6G", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "data": "http://example.com/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "data": "http://example.com/%", + "valid": false + }, + { + "description": "scheme must start with a letter", + "data": "1http://example.com", + "valid": false + }, + { + "description": "invalid character in scheme", + "data": "ht_tp://example.com", + "valid": false + }, + { + "description": "non-numeric port is invalid", + "data": "http://example.com:abc/path", + "valid": false + }, + { + "description": "leading zero in an embedded IPv4 address is invalid", + "data": "http://[::ffff:01.2.3.4]", + "valid": false + }, + { + "description": "square brackets are not allowed in a path segment", + "data": "http:/[::1]", + "valid": false + }, + { + "description": "a trailing newline after a valid URI is invalid", + "data": "http://foo.bar/\n", + "valid": false + }, + { + "description": "an IPv6 address without enclosing brackets is invalid", + "data": "http://2001:0db8:85a3:0000:0000:8a2e:0370:7334", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uuid.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uuid.json new file mode 100644 index 000000000..4679e996a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uuid.json @@ -0,0 +1,156 @@ +[ + { + "description": "uuid format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uuid" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "all upper-case", + "data": "2EB8AA08-AA98-11EA-B4AA-73B441D16380", + "valid": true + }, + { + "description": "all lower-case", + "data": "2eb8aa08-aa98-11ea-b4aa-73b441d16380", + "valid": true + }, + { + "description": "mixed case", + "data": "2eb8aa08-AA98-11ea-B4Aa-73B441D16380", + "valid": true + }, + { + "description": "all zeroes is valid", + "data": "00000000-0000-0000-0000-000000000000", + "valid": true + }, + { + "description": "wrong length", + "data": "2eb8aa08-aa98-11ea-b4aa-73b441d1638", + "valid": false + }, + { + "description": "missing section", + "data": "2eb8aa08-aa98-11ea-73b441d16380", + "valid": false + }, + { + "description": "bad characters (not hex)", + "data": "2eb8aa08-aa98-11ea-b4ga-73b441d16380", + "valid": false + }, + { + "description": "no dashes", + "data": "2eb8aa08aa9811eab4aa73b441d16380", + "valid": false + }, + { + "description": "too few dashes", + "data": "2eb8aa08aa98-11ea-b4aa73b441d16380", + "valid": false + }, + { + "description": "too many dashes", + "data": "2eb8-aa08-aa98-11ea-b4aa73b44-1d16380", + "valid": false + }, + { + "description": "dashes in the wrong spot", + "data": "2eb8aa08aa9811eab4aa73b441d16380----", + "valid": false + }, + { + "description": "shifted dashes", + "data": "2eb8aa0-8aa98-11e-ab4aa7-3b441d16380", + "valid": false + }, + { + "description": "valid version 4", + "data": "98d80576-482e-427f-8434-7f86890ab222", + "valid": true + }, + { + "description": "valid version 5", + "data": "99c17cbb-656f-564a-940f-1a4568f03487", + "valid": true + }, + { + "description": "hypothetical version 6", + "data": "99c17cbb-656f-664a-940f-1a4568f03487", + "valid": true + }, + { + "description": "hypothetical version 15", + "data": "99c17cbb-656f-f64a-940f-1a4568f03487", + "valid": true + }, + { + "description": "URN prefixed UUID is invalid", + "data": "urn:uuid:2eb8aa08-aa98-11ea-b4aa-73b441d16380", + "valid": false + }, + { + "description": "a UUID wrapped in curly braces is invalid", + "data": "{2eb8aa08-aa98-11ea-b4aa-73b441d16380}", + "valid": false + }, + { + "description": "trailing hyphen after a complete UUID is invalid", + "data": "2eb8aa08-aa98-11ea-b4aa-73b441d16380-", + "valid": false + }, + { + "description": "non-ASCII digit '২' (a Bengali 2) is invalid", + "data": "২eb8aa08-aa98-11ea-b4aa-73b441d16380", + "valid": false + }, + { + "description": "an underscore in place of a hex digit is invalid", + "data": "2eb8aa08-aa98-11ea-b4aa-73b441d1_380", + "valid": false + }, + { + "description": "a trailing newline after a complete UUID is invalid", + "data": "2eb8aa08-aa98-11ea-b4aa-73b441d16380\n", + "valid": false + }, + { + "description": "a variant nibble not defined by RFC 4122 is valid", + "data": "2eb8aa08-aa98-11ea-f4aa-73b441d16380", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/id.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/id.json new file mode 100644 index 000000000..0b7df4e80 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/id.json @@ -0,0 +1,53 @@ +[ + { + "description": "$id inside an enum is not a real identifier", + "comment": "the implementation must not be confused by an $id buried in the enum", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "id_in_enum": { + "enum": [ + { + "$id": "https://localhost:1234/draft2020-12/id/my_identifier.json", + "type": "null" + } + ] + }, + "real_id_in_schema": { + "$id": "https://localhost:1234/draft2020-12/id/my_identifier.json", + "type": "string" + }, + "zzz_id_in_const": { + "const": { + "$id": "https://localhost:1234/draft2020-12/id/my_identifier.json", + "type": "null" + } + } + }, + "anyOf": [ + { "$ref": "#/$defs/id_in_enum" }, + { "$ref": "https://localhost:1234/draft2020-12/id/my_identifier.json" } + ] + }, + "tests": [ + { + "description": "exact match to enum, and type matches", + "data": { + "$id": "https://localhost:1234/draft2020-12/id/my_identifier.json", + "type": "null" + }, + "valid": true + }, + { + "description": "match $ref to $id", + "data": "a string to match #/$defs/id_in_enum", + "valid": true + }, + { + "description": "no match on enum or $ref to $id", + "data": 1, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/no-schema.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/no-schema.json new file mode 100644 index 000000000..676e6b53a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/no-schema.json @@ -0,0 +1,26 @@ +[ + { + "description": "validation without $schema", + "comment": "minLength is the same across all drafts", + "schema": { + "minLength": 2 + }, + "tests": [ + { + "description": "a 3-character string is valid", + "data": "foo", + "valid": true + }, + { + "description": "a 1-character string is not valid", + "data": "a", + "valid": false + }, + { + "description": "a non-string is valid", + "data": 5, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/non-bmp-regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/non-bmp-regex.json new file mode 100644 index 000000000..d2efb3ef6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/non-bmp-regex.json @@ -0,0 +1,86 @@ +[ + { + "description": "Proper UTF-16 surrogate pair handling: pattern", + "comment": "Optional because .Net doesn't correctly handle 32-bit Unicode characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "^🐲*$" + }, + "tests": [ + { + "description": "matches empty", + "data": "", + "valid": true + }, + { + "description": "matches single", + "data": "🐲", + "valid": true + }, + { + "description": "matches two", + "data": "🐲🐲", + "valid": true + }, + { + "description": "doesn't match one", + "data": "🐉", + "valid": false + }, + { + "description": "doesn't match two", + "data": "🐉🐉", + "valid": false + }, + { + "description": "doesn't match one ASCII", + "data": "D", + "valid": false + }, + { + "description": "doesn't match two ASCII", + "data": "DD", + "valid": false + } + ] + }, + { + "description": "Proper UTF-16 surrogate pair handling: patternProperties", + "comment": "Optional because .Net doesn't correctly handle 32-bit Unicode characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "^🐲*$": { + "type": "integer" + } + } + }, + "tests": [ + { + "description": "matches empty", + "data": { "": 1 }, + "valid": true + }, + { + "description": "matches single", + "data": { "🐲": 1 }, + "valid": true + }, + { + "description": "matches two", + "data": { "🐲🐲": 1 }, + "valid": true + }, + { + "description": "doesn't match one", + "data": { "🐲": "hello" }, + "valid": false + }, + { + "description": "doesn't match two", + "data": { "🐲🐲": "hello" }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/refOfUnknownKeyword.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/refOfUnknownKeyword.json new file mode 100644 index 000000000..ab1510dac --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/refOfUnknownKeyword.json @@ -0,0 +1,112 @@ +[ + { + "description": "reference of a root arbitrary keyword ", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unknown-keyword": {"type": "integer"}, + "properties": { + "bar": {"$ref": "#/unknown-keyword"} + } + }, + "tests": [ + { + "description": "match", + "data": {"bar": 3}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": true}, + "valid": false + } + ] + }, + { + "description": "reference of a root arbitrary keyword with encoded ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unknown/keyword": {"type": "integer"}, + "properties": { + "bar": {"$ref": "#/unknown~1keyword"} + } + }, + "tests": [ + { + "description": "match", + "data": {"bar": 3}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": true}, + "valid": false + } + ] + }, + { + "description": "reference of an arbitrary keyword of a sub-schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"unknown-keyword": {"type": "integer"}}, + "bar": {"$ref": "#/properties/foo/unknown-keyword"} + } + }, + "tests": [ + { + "description": "match", + "data": {"bar": 3}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": true}, + "valid": false + } + ] + }, + { + "description": "reference internals of known non-applicator", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "examples": [ + { "type": "string" } + ], + "$ref": "#/examples/0" + }, + "tests": [ + { + "description": "match", + "data": "a string", + "valid": true + }, + { + "description": "mismatch", + "data": 42, + "valid": false + } + ] + }, + { + "description": "reference of an arbitrary keyword of a sub-schema with encoded ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"unknown/keyword": {"type": "integer"}}, + "bar": {"$ref": "#/properties/foo/unknown~1keyword"} + } + }, + "tests": [ + { + "description": "match", + "data": {"bar": 3}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": true}, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/unknownKeyword.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/unknownKeyword.json new file mode 100644 index 000000000..28b0c4ce9 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/unknownKeyword.json @@ -0,0 +1,57 @@ +[ + { + "description": "$id inside an unknown keyword is not a real identifier", + "comment": "the implementation must not be confused by an $id in locations we do not know how to parse", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "id_in_unknown0": { + "not": { + "array_of_schemas": [ + { + "$id": "https://localhost:1234/draft2020-12/unknownKeyword/my_identifier.json", + "type": "null" + } + ] + } + }, + "real_id_in_schema": { + "$id": "https://localhost:1234/draft2020-12/unknownKeyword/my_identifier.json", + "type": "string" + }, + "id_in_unknown1": { + "not": { + "object_of_schemas": { + "foo": { + "$id": "https://localhost:1234/draft2020-12/unknownKeyword/my_identifier.json", + "type": "integer" + } + } + } + } + }, + "anyOf": [ + { "$ref": "#/$defs/id_in_unknown0" }, + { "$ref": "#/$defs/id_in_unknown1" }, + { "$ref": "https://localhost:1234/draft2020-12/unknownKeyword/my_identifier.json" } + ] + }, + "tests": [ + { + "description": "type matches second anyOf, which has a real schema in it", + "data": "a string", + "valid": true + }, + { + "description": "type matches non-schema in first anyOf", + "data": null, + "valid": false + }, + { + "description": "type matches non-schema in third anyOf", + "data": 1, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/pattern.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/pattern.json new file mode 100644 index 000000000..ed41f79a7 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/pattern.json @@ -0,0 +1,90 @@ +[ + { + "description": "pattern validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "^a*$" + }, + "tests": [ + { + "description": "a matching pattern is valid", + "data": "aaa", + "valid": true + }, + { + "description": "a non-matching pattern is invalid", + "data": "abc", + "valid": false + }, + { + "description": "ignores booleans", + "data": true, + "valid": true + }, + { + "description": "ignores integers", + "data": 123, + "valid": true + }, + { + "description": "ignores floats", + "data": 1.0, + "valid": true + }, + { + "description": "ignores objects", + "data": {}, + "valid": true + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + } + ] + }, + { + "description": "pattern is not anchored", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "a+" + }, + "tests": [ + { + "description": "matches a substring", + "data": "xxaayy", + "valid": true + } + ] + }, + { + "description": "pattern with Unicode property escape requires unicode mode", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\p{Letter}+$" + }, + "tests": [ + { + "description": "ASCII letters match", + "data": "Hello", + "valid": true + }, + { + "description": "Non-ASCII letters match", + "data": "π", + "valid": true + }, + { + "description": "Digits do not match", + "data": "123", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/patternProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/patternProperties.json new file mode 100644 index 000000000..5a0ce941b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/patternProperties.json @@ -0,0 +1,211 @@ +[ + { + "description": + "patternProperties validates properties matching a regex", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "f.*o": {"type": "integer"} + } + }, + "tests": [ + { + "description": "a single valid match is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "multiple valid matches is valid", + "data": {"foo": 1, "foooooo" : 2}, + "valid": true + }, + { + "description": "a single invalid match is invalid", + "data": {"foo": "bar", "fooooo": 2}, + "valid": false + }, + { + "description": "multiple invalid matches is invalid", + "data": {"foo": "bar", "foooooo" : "baz"}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["foo"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foo", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "multiple simultaneous patternProperties are validated", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "a*": {"type": "integer"}, + "aaa*": {"maximum": 20} + } + }, + "tests": [ + { + "description": "a single valid match is valid", + "data": {"a": 21}, + "valid": true + }, + { + "description": "a simultaneous match is valid", + "data": {"aaaa": 18}, + "valid": true + }, + { + "description": "multiple matches is valid", + "data": {"a": 21, "aaaa": 18}, + "valid": true + }, + { + "description": "an invalid due to one is invalid", + "data": {"a": "bar"}, + "valid": false + }, + { + "description": "an invalid due to the other is invalid", + "data": {"aaaa": 31}, + "valid": false + }, + { + "description": "an invalid due to both is invalid", + "data": {"aaa": "foo", "aaaa": 31}, + "valid": false + } + ] + }, + { + "description": "regexes are not anchored by default and are case sensitive", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "[0-9]{2,}": { "type": "boolean" }, + "X_": { "type": "string" } + } + }, + "tests": [ + { + "description": "non recognized members are ignored", + "data": { "answer 1": "42" }, + "valid": true + }, + { + "description": "recognized members are accounted for", + "data": { "a31b": null }, + "valid": false + }, + { + "description": "regexes are case sensitive", + "data": { "a_x_3": 3 }, + "valid": true + }, + { + "description": "regexes are case sensitive, 2", + "data": { "a_X_3": 3 }, + "valid": false + } + ] + }, + { + "description": "patternProperties with boolean schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "f.*": true, + "b.*": false + } + }, + "tests": [ + { + "description": "object with property matching schema true is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property matching schema false is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "object with both properties is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "object with a property matching both true and false is invalid", + "data": {"foobar":1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "patternProperties with null valued instance properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "^.*bar$": {"type": "null"} + } + }, + "tests": [ + { + "description": "allows null values", + "data": {"foobar": null}, + "valid": true + } + ] + }, + { + "description": "patternProperties with Unicode property escape", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "patternProperties": { + "^\\p{Letter}+$": { + "type": "number" + } + } + }, + "tests": [ + { + "description": "Unicode letter property name matches", + "data": { + "π": 1 + }, + "valid": true + }, + { + "description": "Unicode letter property name matches but value type is incorrect", + "data": { + "π": "1" + }, + "valid": false + }, + { + "description": "Non-letter property name does not match pattern", + "data": { + "123": 1 + }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/prefixItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/prefixItems.json new file mode 100644 index 000000000..0adfc069e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/prefixItems.json @@ -0,0 +1,104 @@ +[ + { + "description": "a schema given for prefixItems", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + {"type": "integer"}, + {"type": "string"} + ] + }, + "tests": [ + { + "description": "correct types", + "data": [ 1, "foo" ], + "valid": true + }, + { + "description": "wrong types", + "data": [ "foo", 1 ], + "valid": false + }, + { + "description": "incomplete array of items", + "data": [ 1 ], + "valid": true + }, + { + "description": "array with additional items", + "data": [ 1, "foo", true ], + "valid": true + }, + { + "description": "empty array", + "data": [ ], + "valid": true + }, + { + "description": "JavaScript pseudo-array is valid", + "data": { + "0": "invalid", + "1": "valid", + "length": 2 + }, + "valid": true + } + ] + }, + { + "description": "prefixItems with boolean schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [true, false] + }, + "tests": [ + { + "description": "array with one item is valid", + "data": [ 1 ], + "valid": true + }, + { + "description": "array with two items is invalid", + "data": [ 1, "foo" ], + "valid": false + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "additional items are allowed by default", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{"type": "integer"}] + }, + "tests": [ + { + "description": "only the first item is validated", + "data": [1, "foo", false], + "valid": true + } + ] + }, + { + "description": "prefixItems with null instance elements", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { + "type": "null" + } + ] + }, + "tests": [ + { + "description": "allows null elements", + "data": [ null ], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/properties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/properties.json new file mode 100644 index 000000000..eb66fa8bd --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/properties.json @@ -0,0 +1,242 @@ +[ + { + "description": "object properties validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"type": "integer"}, + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "both properties present and valid is valid", + "data": {"foo": 1, "bar": "baz"}, + "valid": true + }, + { + "description": "one property invalid is invalid", + "data": {"foo": 1, "bar": {}}, + "valid": false + }, + { + "description": "both properties invalid is invalid", + "data": {"foo": [], "bar": {}}, + "valid": false + }, + { + "description": "doesn't invalidate other properties", + "data": {"quux": []}, + "valid": true + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": + "properties, patternProperties, additionalProperties interaction", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"type": "array", "maxItems": 3}, + "bar": {"type": "array"} + }, + "patternProperties": {"f.o": {"minItems": 2}}, + "additionalProperties": {"type": "integer"} + }, + "tests": [ + { + "description": "property validates property", + "data": {"foo": [1, 2]}, + "valid": true + }, + { + "description": "property invalidates property", + "data": {"foo": [1, 2, 3, 4]}, + "valid": false + }, + { + "description": "patternProperty invalidates property", + "data": {"foo": []}, + "valid": false + }, + { + "description": "patternProperty validates nonproperty", + "data": {"fxo": [1, 2]}, + "valid": true + }, + { + "description": "patternProperty invalidates nonproperty", + "data": {"fxo": []}, + "valid": false + }, + { + "description": "additionalProperty ignores property", + "data": {"bar": []}, + "valid": true + }, + { + "description": "additionalProperty validates others", + "data": {"quux": 3}, + "valid": true + }, + { + "description": "additionalProperty invalidates others", + "data": {"quux": "foo"}, + "valid": false + } + ] + }, + { + "description": "properties with boolean schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": true, + "bar": false + } + }, + "tests": [ + { + "description": "no property present is valid", + "data": {}, + "valid": true + }, + { + "description": "only 'true' property present is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "only 'false' property present is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "both properties present is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + } + ] + }, + { + "description": "properties with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo\nbar": {"type": "number"}, + "foo\"bar": {"type": "number"}, + "foo\\bar": {"type": "number"}, + "foo\rbar": {"type": "number"}, + "foo\tbar": {"type": "number"}, + "foo\fbar": {"type": "number"} + } + }, + "tests": [ + { + "description": "object with all numbers is valid", + "data": { + "foo\nbar": 1, + "foo\"bar": 1, + "foo\\bar": 1, + "foo\rbar": 1, + "foo\tbar": 1, + "foo\fbar": 1 + }, + "valid": true + }, + { + "description": "object with strings is invalid", + "data": { + "foo\nbar": "1", + "foo\"bar": "1", + "foo\\bar": "1", + "foo\rbar": "1", + "foo\tbar": "1", + "foo\fbar": "1" + }, + "valid": false + } + ] + }, + { + "description": "properties with null valued instance properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"type": "null"} + } + }, + "tests": [ + { + "description": "allows null values", + "data": {"foo": null}, + "valid": true + } + ] + }, + { + "description": "properties whose names are Javascript object property names", + "comment": "Ensure JS implementations don't universally consider e.g. __proto__ to always be present in an object.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "__proto__": {"type": "number"}, + "toString": { + "properties": { "length": { "type": "string" } } + }, + "constructor": {"type": "number"} + } + }, + "tests": [ + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "none of the properties mentioned", + "data": {}, + "valid": true + }, + { + "description": "__proto__ not valid", + "data": { "__proto__": "foo" }, + "valid": false + }, + { + "description": "toString not valid", + "data": { "toString": { "length": 37 } }, + "valid": false + }, + { + "description": "constructor not valid", + "data": { "constructor": { "length": 37 } }, + "valid": false + }, + { + "description": "all present and valid", + "data": { + "__proto__": 12, + "toString": { "length": "foo" }, + "constructor": 37 + }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/propertyNames.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/propertyNames.json new file mode 100644 index 000000000..84ca2b84e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/propertyNames.json @@ -0,0 +1,178 @@ +[ + { + "description": "propertyNames validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": {"maxLength": 3} + }, + "tests": [ + { + "description": "all property names valid", + "data": { + "f": {}, + "foo": {} + }, + "valid": true + }, + { + "description": "some property names invalid", + "data": { + "foo": {}, + "foobar": {} + }, + "valid": false + }, + { + "description": "object without properties is valid", + "data": {}, + "valid": true + }, + { + "description": "ignores arrays", + "data": [1, 2, 3, 4], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + }, + { + "description": "ignores booleans", + "data": true, + "valid": true + } + ] + }, + { + "description": "propertyNames validation with pattern", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": { "pattern": "^a+$" } + }, + "tests": [ + { + "description": "matching property names valid", + "data": { + "a": {}, + "aa": {}, + "aaa": {} + }, + "valid": true + }, + { + "description": "non-matching property name is invalid", + "data": { + "aaA": {} + }, + "valid": false + }, + { + "description": "object without properties is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with boolean schema true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": true + }, + "tests": [ + { + "description": "object with any properties is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with boolean schema false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": false + }, + "tests": [ + { + "description": "object with any properties is invalid", + "data": {"foo": 1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with const", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": {"const": "foo"} + }, + "tests": [ + { + "description": "object with property foo is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with any other property is invalid", + "data": {"bar": 1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with enum", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": {"enum": ["foo", "bar"]} + }, + "tests": [ + { + "description": "object with property foo is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property foo and bar is valid", + "data": {"foo": 1, "bar": 1}, + "valid": true + }, + { + "description": "object with any other property is invalid", + "data": {"baz": 1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/ref.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/ref.json new file mode 100644 index 000000000..9ca82eeb0 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/ref.json @@ -0,0 +1,1085 @@ +[ + { + "description": "root pointer ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"$ref": "#"} + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "match", + "data": {"foo": false}, + "valid": true + }, + { + "description": "recursive match", + "data": {"foo": {"foo": false}}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": false}, + "valid": false + }, + { + "description": "recursive mismatch", + "data": {"foo": {"bar": false}}, + "valid": false + } + ] + }, + { + "description": "relative pointer ref to object", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"type": "integer"}, + "bar": {"$ref": "#/properties/foo"} + } + }, + "tests": [ + { + "description": "match", + "data": {"bar": 3}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": true}, + "valid": false + } + ] + }, + { + "description": "relative pointer ref to array", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + {"type": "integer"}, + {"$ref": "#/prefixItems/0"} + ] + }, + "tests": [ + { + "description": "match array", + "data": [1, 2], + "valid": true + }, + { + "description": "mismatch array", + "data": [1, "foo"], + "valid": false + } + ] + }, + { + "description": "escaped pointer ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "tilde~field": {"type": "integer"}, + "slash/field": {"type": "integer"}, + "percent%field": {"type": "integer"} + }, + "properties": { + "tilde": {"$ref": "#/$defs/tilde~0field"}, + "slash": {"$ref": "#/$defs/slash~1field"}, + "percent": {"$ref": "#/$defs/percent%25field"} + } + }, + "tests": [ + { + "description": "slash invalid", + "data": {"slash": "aoeu"}, + "valid": false + }, + { + "description": "tilde invalid", + "data": {"tilde": "aoeu"}, + "valid": false + }, + { + "description": "percent invalid", + "data": {"percent": "aoeu"}, + "valid": false + }, + { + "description": "slash valid", + "data": {"slash": 123}, + "valid": true + }, + { + "description": "tilde valid", + "data": {"tilde": 123}, + "valid": true + }, + { + "description": "percent valid", + "data": {"percent": 123}, + "valid": true + } + ] + }, + { + "description": "nested refs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "a": {"type": "integer"}, + "b": {"$ref": "#/$defs/a"}, + "c": {"$ref": "#/$defs/b"} + }, + "$ref": "#/$defs/c" + }, + "tests": [ + { + "description": "nested ref valid", + "data": 5, + "valid": true + }, + { + "description": "nested ref invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "ref applies alongside sibling keywords", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "reffed": { + "type": "array" + } + }, + "properties": { + "foo": { + "$ref": "#/$defs/reffed", + "maxItems": 2 + } + } + }, + "tests": [ + { + "description": "ref valid, maxItems valid", + "data": { "foo": [] }, + "valid": true + }, + { + "description": "ref valid, maxItems invalid", + "data": { "foo": [1, 2, 3] }, + "valid": false + }, + { + "description": "ref invalid", + "data": { "foo": "string" }, + "valid": false + } + ] + }, + { + "description": "remote ref, containing refs itself", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "https://json-schema.org/draft/2020-12/schema" + }, + "tests": [ + { + "description": "remote ref valid", + "data": {"minLength": 1}, + "valid": true + }, + { + "description": "remote ref invalid", + "data": {"minLength": -1}, + "valid": false + } + ] + }, + { + "description": "property named $ref that is not a reference", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "$ref": {"type": "string"} + } + }, + "tests": [ + { + "description": "property named $ref valid", + "data": {"$ref": "a"}, + "valid": true + }, + { + "description": "property named $ref invalid", + "data": {"$ref": 2}, + "valid": false + } + ] + }, + { + "description": "property named $ref, containing an actual $ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "$ref": {"$ref": "#/$defs/is-string"} + }, + "$defs": { + "is-string": { + "type": "string" + } + } + }, + "tests": [ + { + "description": "property named $ref valid", + "data": {"$ref": "a"}, + "valid": true + }, + { + "description": "property named $ref invalid", + "data": {"$ref": 2}, + "valid": false + } + ] + }, + { + "description": "$ref to boolean schema true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "#/$defs/bool", + "$defs": { + "bool": true + } + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "$ref to boolean schema false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "#/$defs/bool", + "$defs": { + "bool": false + } + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "Recursive references between schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/tree", + "description": "tree of nodes", + "type": "object", + "properties": { + "meta": {"type": "string"}, + "nodes": { + "type": "array", + "items": {"$ref": "node"} + } + }, + "required": ["meta", "nodes"], + "$defs": { + "node": { + "$id": "http://localhost:1234/draft2020-12/node", + "description": "node", + "type": "object", + "properties": { + "value": {"type": "number"}, + "subtree": {"$ref": "tree"} + }, + "required": ["value"] + } + } + }, + "tests": [ + { + "description": "valid tree", + "data": { + "meta": "root", + "nodes": [ + { + "value": 1, + "subtree": { + "meta": "child", + "nodes": [ + {"value": 1.1}, + {"value": 1.2} + ] + } + }, + { + "value": 2, + "subtree": { + "meta": "child", + "nodes": [ + {"value": 2.1}, + {"value": 2.2} + ] + } + } + ] + }, + "valid": true + }, + { + "description": "invalid tree", + "data": { + "meta": "root", + "nodes": [ + { + "value": 1, + "subtree": { + "meta": "child", + "nodes": [ + {"value": "string is invalid"}, + {"value": 1.2} + ] + } + }, + { + "value": 2, + "subtree": { + "meta": "child", + "nodes": [ + {"value": 2.1}, + {"value": 2.2} + ] + } + } + ] + }, + "valid": false + } + ] + }, + { + "description": "refs with quote", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo\"bar": {"$ref": "#/$defs/foo%22bar"} + }, + "$defs": { + "foo\"bar": {"type": "number"} + } + }, + "tests": [ + { + "description": "object with numbers is valid", + "data": { + "foo\"bar": 1 + }, + "valid": true + }, + { + "description": "object with strings is invalid", + "data": { + "foo\"bar": "1" + }, + "valid": false + } + ] + }, + { + "description": "ref creates new scope when adjacent to keywords", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "A": { + "unevaluatedProperties": false + } + }, + "properties": { + "prop1": { + "type": "string" + } + }, + "$ref": "#/$defs/A" + }, + "tests": [ + { + "description": "referenced subschema doesn't see annotations from properties", + "data": { + "prop1": "match" + }, + "valid": false + } + ] + }, + { + "description": "naive replacement of $ref with its destination is not correct", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "a_string": { "type": "string" } + }, + "enum": [ + { "$ref": "#/$defs/a_string" } + ] + }, + "tests": [ + { + "description": "do not evaluate the $ref inside the enum, matching any string", + "data": "this is a string", + "valid": false + }, + { + "description": "do not evaluate the $ref inside the enum, definition exact match", + "data": { "type": "string" }, + "valid": false + }, + { + "description": "match the enum exactly", + "data": { "$ref": "#/$defs/a_string" }, + "valid": true + } + ] + }, + { + "description": "refs with relative uris and defs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://example.com/schema-relative-uri-defs1.json", + "properties": { + "foo": { + "$id": "schema-relative-uri-defs2.json", + "$defs": { + "inner": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "$ref": "#/$defs/inner" + } + }, + "$ref": "schema-relative-uri-defs2.json" + }, + "tests": [ + { + "description": "invalid on inner field", + "data": { + "foo": { + "bar": 1 + }, + "bar": "a" + }, + "valid": false + }, + { + "description": "invalid on outer field", + "data": { + "foo": { + "bar": "a" + }, + "bar": 1 + }, + "valid": false + }, + { + "description": "valid on both fields", + "data": { + "foo": { + "bar": "a" + }, + "bar": "a" + }, + "valid": true + } + ] + }, + { + "description": "relative refs with absolute uris and defs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://example.com/schema-refs-absolute-uris-defs1.json", + "properties": { + "foo": { + "$id": "http://example.com/schema-refs-absolute-uris-defs2.json", + "$defs": { + "inner": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "$ref": "#/$defs/inner" + } + }, + "$ref": "schema-refs-absolute-uris-defs2.json" + }, + "tests": [ + { + "description": "invalid on inner field", + "data": { + "foo": { + "bar": 1 + }, + "bar": "a" + }, + "valid": false + }, + { + "description": "invalid on outer field", + "data": { + "foo": { + "bar": "a" + }, + "bar": 1 + }, + "valid": false + }, + { + "description": "valid on both fields", + "data": { + "foo": { + "bar": "a" + }, + "bar": "a" + }, + "valid": true + } + ] + }, + { + "description": "$id must be resolved against nearest parent, not just immediate parent", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://example.com/a.json", + "$defs": { + "x": { + "$id": "http://example.com/b/c.json", + "not": { + "$defs": { + "y": { + "$id": "d.json", + "type": "number" + } + } + } + } + }, + "allOf": [ + { + "$ref": "http://example.com/b/d.json" + } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "order of evaluation: $id and $ref", + "schema": { + "$comment": "$id must be evaluated before $ref to get the proper $ref destination", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/draft2020-12/ref-and-id1/base.json", + "$ref": "int.json", + "$defs": { + "bigint": { + "$comment": "canonical uri: https://example.com/ref-and-id1/int.json", + "$id": "int.json", + "maximum": 10 + }, + "smallint": { + "$comment": "canonical uri: https://example.com/ref-and-id1-int.json", + "$id": "/draft2020-12/ref-and-id1-int.json", + "maximum": 2 + } + } + }, + "tests": [ + { + "description": "data is valid against first definition", + "data": 5, + "valid": true + }, + { + "description": "data is invalid against first definition", + "data": 50, + "valid": false + } + ] + }, + { + "description": "order of evaluation: $id and $anchor and $ref", + "schema": { + "$comment": "$id must be evaluated before $ref to get the proper $ref destination", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/draft2020-12/ref-and-id2/base.json", + "$ref": "#bigint", + "$defs": { + "bigint": { + "$comment": "canonical uri: /ref-and-id2/base.json#/$defs/bigint; another valid uri for this location: /ref-and-id2/base.json#bigint", + "$anchor": "bigint", + "maximum": 10 + }, + "smallint": { + "$comment": "canonical uri: https://example.com/ref-and-id2#/$defs/smallint; another valid uri for this location: https://example.com/ref-and-id2/#bigint", + "$id": "https://example.com/draft2020-12/ref-and-id2/", + "$anchor": "bigint", + "maximum": 2 + } + } + }, + "tests": [ + { + "description": "data is valid against first definition", + "data": 5, + "valid": true + }, + { + "description": "data is invalid against first definition", + "data": 50, + "valid": false + } + ] + }, + { + "description": "order of evaluation: $id and $ref on nested schema", + "schema": { + "$comment": "$id must be evaluated before $ref to get the proper $ref destination", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/draft2020-12/ref-and-id3/base.json", + "$ref": "nested/foo.json", + "$defs": { + "foo": { + "$comment": "canonical uri: https://example.com/draft2020-12/ref-and-id3/nested/foo.json", + "$id": "nested/foo.json", + "$ref": "./bar.json" + }, + "bar": { + "$comment": "canonical uri: https://example.com/draft2020-12/ref-and-id3/nested/bar.json", + "$id": "nested/bar.json", + "type": "number" + } + } + }, + "tests": [ + { + "description": "data is valid against nested sibling", + "data": 5, + "valid": true + }, + { + "description": "data is invalid against nested sibling", + "data": "a", + "valid": false + } + ] + }, + { + "description": "simple URN base URI with $ref via the URN", + "schema": { + "$comment": "URIs do not have to have HTTP(s) schemes", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:uuid:deadbeef-1234-ffff-ffff-4321feebdaed", + "minimum": 30, + "properties": { + "foo": {"$ref": "urn:uuid:deadbeef-1234-ffff-ffff-4321feebdaed"} + } + }, + "tests": [ + { + "description": "valid under the URN IDed schema", + "data": {"foo": 37}, + "valid": true + }, + { + "description": "invalid under the URN IDed schema", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "simple URN base URI with JSON pointer", + "schema": { + "$comment": "URIs do not have to have HTTP(s) schemes", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:uuid:deadbeef-1234-00ff-ff00-4321feebdaed", + "properties": { + "foo": {"$ref": "#/$defs/bar"} + }, + "$defs": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with NSS", + "schema": { + "$comment": "RFC 8141 §2.2", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:example:1/406/47452/2", + "properties": { + "foo": {"$ref": "#/$defs/bar"} + }, + "$defs": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with r-component", + "schema": { + "$comment": "RFC 8141 §2.3.1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:example:foo-bar-baz-qux?+CCResolve:cc=uk", + "properties": { + "foo": {"$ref": "#/$defs/bar"} + }, + "$defs": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with q-component", + "schema": { + "$comment": "RFC 8141 §2.3.2", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:example:weather?=op=map&lat=39.56&lon=-104.85&datetime=1969-07-21T02:56:15Z", + "properties": { + "foo": {"$ref": "#/$defs/bar"} + }, + "$defs": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with URN and JSON pointer ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:uuid:deadbeef-1234-0000-0000-4321feebdaed", + "properties": { + "foo": {"$ref": "urn:uuid:deadbeef-1234-0000-0000-4321feebdaed#/$defs/bar"} + }, + "$defs": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with URN and anchor ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:uuid:deadbeef-1234-ff00-00ff-4321feebdaed", + "properties": { + "foo": {"$ref": "urn:uuid:deadbeef-1234-ff00-00ff-4321feebdaed#something"} + }, + "$defs": { + "bar": { + "$anchor": "something", + "type": "string" + } + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN ref with nested pointer ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "urn:uuid:deadbeef-4321-ffff-ffff-1234feebdaed", + "$defs": { + "foo": { + "$id": "urn:uuid:deadbeef-4321-ffff-ffff-1234feebdaed", + "$defs": {"bar": {"type": "string"}}, + "$ref": "#/$defs/bar" + } + } + }, + "tests": [ + { + "description": "a string is valid", + "data": "bar", + "valid": true + }, + { + "description": "a non-string is invalid", + "data": 12, + "valid": false + } + ] + }, + { + "description": "ref to if", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://example.com/ref/if", + "if": { + "$id": "http://example.com/ref/if", + "type": "integer" + } + }, + "tests": [ + { + "description": "a non-integer is invalid due to the $ref", + "data": "foo", + "valid": false + }, + { + "description": "an integer is valid", + "data": 12, + "valid": true + } + ] + }, + { + "description": "ref to then", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://example.com/ref/then", + "then": { + "$id": "http://example.com/ref/then", + "type": "integer" + } + }, + "tests": [ + { + "description": "a non-integer is invalid due to the $ref", + "data": "foo", + "valid": false + }, + { + "description": "an integer is valid", + "data": 12, + "valid": true + } + ] + }, + { + "description": "ref to else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://example.com/ref/else", + "else": { + "$id": "http://example.com/ref/else", + "type": "integer" + } + }, + "tests": [ + { + "description": "a non-integer is invalid due to the $ref", + "data": "foo", + "valid": false + }, + { + "description": "an integer is valid", + "data": 12, + "valid": true + } + ] + }, + { + "description": "ref with absolute-path-reference", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://example.com/ref/absref.json", + "$defs": { + "a": { + "$id": "http://example.com/ref/absref/foobar.json", + "type": "number" + }, + "b": { + "$id": "http://example.com/absref/foobar.json", + "type": "string" + } + }, + "$ref": "/absref/foobar.json" + }, + "tests": [ + { + "description": "a string is valid", + "data": "foo", + "valid": true + }, + { + "description": "an integer is invalid", + "data": 12, + "valid": false + } + ] + }, + { + "description": "$id with file URI still resolves pointers - *nix", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "file:///folder/file.json", + "$defs": { + "foo": { + "type": "number" + } + }, + "$ref": "#/$defs/foo" + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "$id with file URI still resolves pointers - windows", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "file:///c:/folder/file.json", + "$defs": { + "foo": { + "type": "number" + } + }, + "$ref": "#/$defs/foo" + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "empty tokens in $ref json-pointer", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "": { + "$defs": { + "": { "type": "number" } + } + } + }, + "allOf": [ + { + "$ref": "#/$defs//$defs/" + } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/refRemote.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/refRemote.json new file mode 100644 index 000000000..6083aabb1 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/refRemote.json @@ -0,0 +1,342 @@ +[ + { + "description": "remote ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/integer.json" + }, + "tests": [ + { + "description": "remote ref valid", + "data": 1, + "valid": true + }, + { + "description": "remote ref invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "fragment within remote ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/subSchemas.json#/$defs/integer" + }, + "tests": [ + { + "description": "remote fragment valid", + "data": 1, + "valid": true + }, + { + "description": "remote fragment invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "anchor within remote ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/locationIndependentIdentifier.json#foo" + }, + "tests": [ + { + "description": "remote anchor valid", + "data": 1, + "valid": true + }, + { + "description": "remote anchor invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "ref within remote ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/subSchemas.json#/$defs/refToInteger" + }, + "tests": [ + { + "description": "ref within ref valid", + "data": 1, + "valid": true + }, + { + "description": "ref within ref invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "base URI change", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/", + "items": { + "$id": "baseUriChange/", + "items": {"$ref": "folderInteger.json"} + } + }, + "tests": [ + { + "description": "base URI change ref valid", + "data": [[1]], + "valid": true + }, + { + "description": "base URI change ref invalid", + "data": [["a"]], + "valid": false + } + ] + }, + { + "description": "base URI change - change folder", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/scope_change_defs1.json", + "type" : "object", + "properties": {"list": {"$ref": "baseUriChangeFolder/"}}, + "$defs": { + "baz": { + "$id": "baseUriChangeFolder/", + "type": "array", + "items": {"$ref": "folderInteger.json"} + } + } + }, + "tests": [ + { + "description": "number is valid", + "data": {"list": [1]}, + "valid": true + }, + { + "description": "string is invalid", + "data": {"list": ["a"]}, + "valid": false + } + ] + }, + { + "description": "base URI change - change folder in subschema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/scope_change_defs2.json", + "type" : "object", + "properties": {"list": {"$ref": "baseUriChangeFolderInSubschema/#/$defs/bar"}}, + "$defs": { + "baz": { + "$id": "baseUriChangeFolderInSubschema/", + "$defs": { + "bar": { + "type": "array", + "items": {"$ref": "folderInteger.json"} + } + } + } + } + }, + "tests": [ + { + "description": "number is valid", + "data": {"list": [1]}, + "valid": true + }, + { + "description": "string is invalid", + "data": {"list": ["a"]}, + "valid": false + } + ] + }, + { + "description": "root ref in remote ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/object", + "type": "object", + "properties": { + "name": {"$ref": "name-defs.json#/$defs/orNull"} + } + }, + "tests": [ + { + "description": "string is valid", + "data": { + "name": "foo" + }, + "valid": true + }, + { + "description": "null is valid", + "data": { + "name": null + }, + "valid": true + }, + { + "description": "object is invalid", + "data": { + "name": { + "name": null + } + }, + "valid": false + } + ] + }, + { + "description": "remote ref with ref to defs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/schema-remote-ref-ref-defs1.json", + "$ref": "ref-and-defs.json" + }, + "tests": [ + { + "description": "invalid", + "data": { + "bar": 1 + }, + "valid": false + }, + { + "description": "valid", + "data": { + "bar": "a" + }, + "valid": true + } + ] + }, + { + "description": "Location-independent identifier in remote ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/locationIndependentIdentifier.json#/$defs/refToInteger" + }, + "tests": [ + { + "description": "integer is valid", + "data": 1, + "valid": true + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "retrieved nested refs resolve relative to their URI not $id", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/some-id", + "properties": { + "name": {"$ref": "nested/foo-ref-string.json"} + } + }, + "tests": [ + { + "description": "number is invalid", + "data": { + "name": {"foo": 1} + }, + "valid": false + }, + { + "description": "string is valid", + "data": { + "name": {"foo": "a"} + }, + "valid": true + } + ] + }, + { + "description": "remote HTTP ref with different $id", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/different-id-ref-string.json" + }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "remote HTTP ref with different URN $id", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/urn-ref-string.json" + }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "remote HTTP ref with nested absolute ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/nested-absolute-ref-to-string.json" + }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "$ref to $ref finds detached $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/detached-ref.json#/$defs/foo" + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/required.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/required.json new file mode 100644 index 000000000..17db8939e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/required.json @@ -0,0 +1,169 @@ +[ + { + "description": "required validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {}, + "bar": {} + }, + "required": ["foo"] + }, + "tests": [ + { + "description": "present required property is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "non-present required property is invalid", + "data": {"bar": 1}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores strings", + "data": "", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + }, + { + "description":"ignores boolean", + "data": true, + "valid": true + + } + ] + }, + { + "description": "required default validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {} + } + }, + "tests": [ + { + "description": "not required by default", + "data": {}, + "valid": true + } + ] + }, + { + "description": "required with empty array", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {} + }, + "required": [] + }, + "tests": [ + { + "description": "property not required", + "data": {}, + "valid": true + } + ] + }, + { + "description": "required with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "foo\nbar", + "foo\"bar", + "foo\\bar", + "foo\rbar", + "foo\tbar", + "foo\fbar" + ] + }, + "tests": [ + { + "description": "object with all properties present is valid", + "data": { + "foo\nbar": 1, + "foo\"bar": 1, + "foo\\bar": 1, + "foo\rbar": 1, + "foo\tbar": 1, + "foo\fbar": 1 + }, + "valid": true + }, + { + "description": "object with some properties missing is invalid", + "data": { + "foo\nbar": "1", + "foo\"bar": "1" + }, + "valid": false + } + ] + }, + { + "description": "required properties whose names are Javascript object property names", + "comment": "Ensure JS implementations don't universally consider e.g. __proto__ to always be present in an object.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": ["__proto__", "toString", "constructor"] + }, + "tests": [ + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "none of the properties mentioned", + "data": {}, + "valid": false + }, + { + "description": "__proto__ present", + "data": { "__proto__": "foo" }, + "valid": false + }, + { + "description": "toString present", + "data": { "toString": { "length": 37 } }, + "valid": false + }, + { + "description": "constructor present", + "data": { "constructor": { "length": 37 } }, + "valid": false + }, + { + "description": "all present", + "data": { + "__proto__": 12, + "toString": { "length": "foo" }, + "constructor": 37 + }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/type.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/type.json new file mode 100644 index 000000000..2123c408d --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/type.json @@ -0,0 +1,501 @@ +[ + { + "description": "integer type matches integers", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer" + }, + "tests": [ + { + "description": "an integer is an integer", + "data": 1, + "valid": true + }, + { + "description": "a float with zero fractional part is an integer", + "data": 1.0, + "valid": true + }, + { + "description": "a float is not an integer", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not an integer", + "data": "foo", + "valid": false + }, + { + "description": "a string is still not an integer, even if it looks like one", + "data": "1", + "valid": false + }, + { + "description": "an object is not an integer", + "data": {}, + "valid": false + }, + { + "description": "an array is not an integer", + "data": [], + "valid": false + }, + { + "description": "a boolean is not an integer", + "data": true, + "valid": false + }, + { + "description": "null is not an integer", + "data": null, + "valid": false + } + ] + }, + { + "description": "number type matches numbers", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "number" + }, + "tests": [ + { + "description": "an integer is a number", + "data": 1, + "valid": true + }, + { + "description": "a float with zero fractional part is a number (and an integer)", + "data": 1.0, + "valid": true + }, + { + "description": "a float is a number", + "data": 1.1, + "valid": true + }, + { + "description": "a string is not a number", + "data": "foo", + "valid": false + }, + { + "description": "a string is still not a number, even if it looks like one", + "data": "1", + "valid": false + }, + { + "description": "an object is not a number", + "data": {}, + "valid": false + }, + { + "description": "an array is not a number", + "data": [], + "valid": false + }, + { + "description": "a boolean is not a number", + "data": true, + "valid": false + }, + { + "description": "null is not a number", + "data": null, + "valid": false + } + ] + }, + { + "description": "string type matches strings", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string" + }, + "tests": [ + { + "description": "1 is not a string", + "data": 1, + "valid": false + }, + { + "description": "a float is not a string", + "data": 1.1, + "valid": false + }, + { + "description": "a string is a string", + "data": "foo", + "valid": true + }, + { + "description": "a string is still a string, even if it looks like a number", + "data": "1", + "valid": true + }, + { + "description": "an empty string is still a string", + "data": "", + "valid": true + }, + { + "description": "an object is not a string", + "data": {}, + "valid": false + }, + { + "description": "an array is not a string", + "data": [], + "valid": false + }, + { + "description": "a boolean is not a string", + "data": true, + "valid": false + }, + { + "description": "null is not a string", + "data": null, + "valid": false + } + ] + }, + { + "description": "object type matches objects", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object" + }, + "tests": [ + { + "description": "an integer is not an object", + "data": 1, + "valid": false + }, + { + "description": "a float is not an object", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not an object", + "data": "foo", + "valid": false + }, + { + "description": "an object is an object", + "data": {}, + "valid": true + }, + { + "description": "an array is not an object", + "data": [], + "valid": false + }, + { + "description": "a boolean is not an object", + "data": true, + "valid": false + }, + { + "description": "null is not an object", + "data": null, + "valid": false + } + ] + }, + { + "description": "array type matches arrays", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "array" + }, + "tests": [ + { + "description": "an integer is not an array", + "data": 1, + "valid": false + }, + { + "description": "a float is not an array", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not an array", + "data": "foo", + "valid": false + }, + { + "description": "an object is not an array", + "data": {}, + "valid": false + }, + { + "description": "an array is an array", + "data": [], + "valid": true + }, + { + "description": "a boolean is not an array", + "data": true, + "valid": false + }, + { + "description": "null is not an array", + "data": null, + "valid": false + } + ] + }, + { + "description": "boolean type matches booleans", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "boolean" + }, + "tests": [ + { + "description": "an integer is not a boolean", + "data": 1, + "valid": false + }, + { + "description": "zero is not a boolean", + "data": 0, + "valid": false + }, + { + "description": "a float is not a boolean", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not a boolean", + "data": "foo", + "valid": false + }, + { + "description": "an empty string is not a boolean", + "data": "", + "valid": false + }, + { + "description": "an object is not a boolean", + "data": {}, + "valid": false + }, + { + "description": "an array is not a boolean", + "data": [], + "valid": false + }, + { + "description": "true is a boolean", + "data": true, + "valid": true + }, + { + "description": "false is a boolean", + "data": false, + "valid": true + }, + { + "description": "null is not a boolean", + "data": null, + "valid": false + } + ] + }, + { + "description": "null type matches only the null object", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "null" + }, + "tests": [ + { + "description": "an integer is not null", + "data": 1, + "valid": false + }, + { + "description": "a float is not null", + "data": 1.1, + "valid": false + }, + { + "description": "zero is not null", + "data": 0, + "valid": false + }, + { + "description": "a string is not null", + "data": "foo", + "valid": false + }, + { + "description": "an empty string is not null", + "data": "", + "valid": false + }, + { + "description": "an object is not null", + "data": {}, + "valid": false + }, + { + "description": "an array is not null", + "data": [], + "valid": false + }, + { + "description": "true is not null", + "data": true, + "valid": false + }, + { + "description": "false is not null", + "data": false, + "valid": false + }, + { + "description": "null is null", + "data": null, + "valid": true + } + ] + }, + { + "description": "multiple types can be specified in an array", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": ["integer", "string"] + }, + "tests": [ + { + "description": "an integer is valid", + "data": 1, + "valid": true + }, + { + "description": "a string is valid", + "data": "foo", + "valid": true + }, + { + "description": "a float is invalid", + "data": 1.1, + "valid": false + }, + { + "description": "an object is invalid", + "data": {}, + "valid": false + }, + { + "description": "an array is invalid", + "data": [], + "valid": false + }, + { + "description": "a boolean is invalid", + "data": true, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + } + ] + }, + { + "description": "type as array with one item", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": ["string"] + }, + "tests": [ + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "number is invalid", + "data": 123, + "valid": false + } + ] + }, + { + "description": "type: array or object", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": ["array", "object"] + }, + "tests": [ + { + "description": "array is valid", + "data": [1,2,3], + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": 123}, + "valid": true + }, + { + "description": "number is invalid", + "data": 123, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + } + ] + }, + { + "description": "type: array, object or null", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": ["array", "object", "null"] + }, + "tests": [ + { + "description": "array is valid", + "data": [1,2,3], + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": 123}, + "valid": true + }, + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "number is invalid", + "data": 123, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedItems.json new file mode 100644 index 000000000..d8a7c1b5b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedItems.json @@ -0,0 +1,854 @@ +[ + { + "description": "unevaluatedItems true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": true + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": [], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo"], + "valid": true + } + ] + }, + { + "description": "unevaluatedItems false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": [], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems as schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": { "type": "string" } + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": [], + "valid": true + }, + { + "description": "with valid unevaluated items", + "data": ["foo"], + "valid": true + }, + { + "description": "with invalid unevaluated items", + "data": [42], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with uniform items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": { "type": "string" }, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "unevaluatedItems doesn't apply", + "data": ["foo", "bar"], + "valid": true + } + ] + }, + { + "description": "unevaluatedItems with tuple", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "type": "string" } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo"], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", "bar"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with items and prefixItems", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "type": "string" } + ], + "items": true, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "unevaluatedItems doesn't apply", + "data": ["foo", 42], + "valid": true + } + ] + }, + { + "description": "unevaluatedItems with items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": {"type": "number"}, + "unevaluatedItems": {"type": "string"} + }, + "tests": [ + { + "description": "valid under items", + "comment": "no elements are considered by unevaluatedItems", + "data": [5, 6, 7, 8], + "valid": true + }, + { + "description": "invalid under items", + "data": ["foo", "bar", "baz"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with nested tuple", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "type": "string" } + ], + "allOf": [ + { + "prefixItems": [ + true, + { "type": "number" } + ] + } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo", 42], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", 42, true], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with nested items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": {"type": "boolean"}, + "anyOf": [ + { "items": {"type": "string"} }, + true + ] + }, + "tests": [ + { + "description": "with only (valid) additional items", + "data": [true, false], + "valid": true + }, + { + "description": "with no additional items", + "data": ["yes", "no"], + "valid": true + }, + { + "description": "with invalid additional item", + "data": ["yes", false], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with nested prefixItems and items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "prefixItems": [ + { "type": "string" } + ], + "items": true + } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no additional items", + "data": ["foo"], + "valid": true + }, + { + "description": "with additional items", + "data": ["foo", 42, true], + "valid": true + } + ] + }, + { + "description": "unevaluatedItems with nested unevaluatedItems", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "prefixItems": [ + { "type": "string" } + ] + }, + { "unevaluatedItems": true } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no additional items", + "data": ["foo"], + "valid": true + }, + { + "description": "with additional items", + "data": ["foo", 42, true], + "valid": true + } + ] + }, + { + "description": "unevaluatedItems with anyOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "const": "foo" } + ], + "anyOf": [ + { + "prefixItems": [ + true, + { "const": "bar" } + ] + }, + { + "prefixItems": [ + true, + true, + { "const": "baz" } + ] + } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "when one schema matches and has no unevaluated items", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "when one schema matches and has unevaluated items", + "data": ["foo", "bar", 42], + "valid": false + }, + { + "description": "when two schemas match and has no unevaluated items", + "data": ["foo", "bar", "baz"], + "valid": true + }, + { + "description": "when two schemas match and has unevaluated items", + "data": ["foo", "bar", "baz", 42], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with oneOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "const": "foo" } + ], + "oneOf": [ + { + "prefixItems": [ + true, + { "const": "bar" } + ] + }, + { + "prefixItems": [ + true, + { "const": "baz" } + ] + } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", "bar", 42], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with not", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "const": "foo" } + ], + "not": { + "not": { + "prefixItems": [ + true, + { "const": "bar" } + ] + } + }, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with unevaluated items", + "data": ["foo", "bar"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with if/then/else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "const": "foo" } + ], + "if": { + "prefixItems": [ + true, + { "const": "bar" } + ] + }, + "then": { + "prefixItems": [ + true, + true, + { "const": "then" } + ] + }, + "else": { + "prefixItems": [ + true, + true, + true, + { "const": "else" } + ] + }, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "when if matches and it has no unevaluated items", + "data": ["foo", "bar", "then"], + "valid": true + }, + { + "description": "when if matches and it has unevaluated items", + "data": ["foo", "bar", "then", "else"], + "valid": false + }, + { + "description": "when if doesn't match and it has no unevaluated items", + "data": ["foo", 42, 42, "else"], + "valid": true + }, + { + "description": "when if doesn't match and it has unevaluated items", + "data": ["foo", 42, 42, "else", 42], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with boolean schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [true], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": [], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with $ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "#/$defs/bar", + "prefixItems": [ + { "type": "string" } + ], + "unevaluatedItems": false, + "$defs": { + "bar": { + "prefixItems": [ + true, + { "type": "string" } + ] + } + } + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", "bar", "baz"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems before $ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": false, + "prefixItems": [ + { "type": "string" } + ], + "$ref": "#/$defs/bar", + "$defs": { + "bar": { + "prefixItems": [ + true, + { "type": "string" } + ] + } + } + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", "bar", "baz"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with $dynamicRef", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/unevaluated-items-with-dynamic-ref/derived", + + "$ref": "./baseSchema", + + "$defs": { + "derived": { + "$dynamicAnchor": "addons", + "prefixItems": [ + true, + { "type": "string" } + ] + }, + "baseSchema": { + "$id": "./baseSchema", + + "$comment": "unevaluatedItems comes first so it's more likely to catch bugs with implementations that are sensitive to keyword ordering", + "unevaluatedItems": false, + "type": "array", + "prefixItems": [ + { "type": "string" } + ], + "$dynamicRef": "#addons", + + "$defs": { + "defaultAddons": { + "$comment": "Needed to satisfy the bookending requirement", + "$dynamicAnchor": "addons" + } + } + } + } + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", "bar", "baz"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems can't see inside cousins", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "prefixItems": [ true ] + }, + { "unevaluatedItems": false } + ] + }, + "tests": [ + { + "description": "always fails", + "data": [ 1 ], + "valid": false + } + ] + }, + { + "description": "item is evaluated in an uncle schema to unevaluatedItems", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "prefixItems": [ + { "type": "string" } + ], + "unevaluatedItems": false + } + }, + "anyOf": [ + { + "properties": { + "foo": { + "prefixItems": [ + true, + { "type": "string" } + ] + } + } + } + ] + }, + "tests": [ + { + "description": "no extra items", + "data": { + "foo": [ + "test" + ] + }, + "valid": true + }, + { + "description": "uncle keyword evaluation is not significant", + "data": { + "foo": [ + "test", + "test" + ] + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedItems depends on adjacent contains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [true], + "contains": {"type": "string"}, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "second item is evaluated by contains", + "data": [ 1, "foo" ], + "valid": true + }, + { + "description": "contains fails, second item is not evaluated", + "data": [ 1, 2 ], + "valid": false + }, + { + "description": "contains passes, second item is not evaluated", + "data": [ 1, 2, "foo" ], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems depends on multiple nested contains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { "contains": { "multipleOf": 2 } }, + { "contains": { "multipleOf": 3 } } + ], + "unevaluatedItems": { "multipleOf": 5 } + }, + "tests": [ + { + "description": "5 not evaluated, passes unevaluatedItems", + "data": [ 2, 3, 4, 5, 6 ], + "valid": true + }, + { + "description": "7 not evaluated, fails unevaluatedItems", + "data": [ 2, 3, 4, 7, 8 ], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems and contains interact to control item dependency relationship", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "contains": {"const": "a"} + }, + "then": { + "if": { + "contains": {"const": "b"} + }, + "then": { + "if": { + "contains": {"const": "c"} + } + } + }, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "empty array is valid", + "data": [], + "valid": true + }, + { + "description": "only a's are valid", + "data": [ "a", "a" ], + "valid": true + }, + { + "description": "a's and b's are valid", + "data": [ "a", "b", "a", "b", "a" ], + "valid": true + }, + { + "description": "a's, b's and c's are valid", + "data": [ "c", "a", "c", "c", "b", "a" ], + "valid": true + }, + { + "description": "only b's are invalid", + "data": [ "b", "b" ], + "valid": false + }, + { + "description": "only c's are invalid", + "data": [ "c", "c" ], + "valid": false + }, + { + "description": "only b's and c's are invalid", + "data": [ "c", "b", "c", "b", "c" ], + "valid": false + }, + { + "description": "only a's and c's are invalid", + "data": [ "c", "a", "c", "a", "c" ], + "valid": false + } + ] + }, + { + "description" : "unevaluatedItems with minContains = 0", + "schema" : { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"type": "string"}, + "minContains": 0, + "unevaluatedItems": false + }, + "tests" : [ + { + "description": "empty array is valid", + "data": [], + "valid": true + }, + { + "description": "no items evaluated by contains", + "data": [0], + "valid": false + }, + { + "description": "some but not all items evaluated by contains", + "data": ["foo", 0], + "valid": false + }, + { + "description": "all items evaluated by contains", + "data": ["foo", "bar"], + "valid": true + } + ] + }, + { + "description": "non-array instances are valid", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": false + }, + "tests": [ + { + "description": "ignores booleans", + "data": true, + "valid": true + }, + { + "description": "ignores integers", + "data": 123, + "valid": true + }, + { + "description": "ignores floats", + "data": 1.0, + "valid": true + }, + { + "description": "ignores objects", + "data": {}, + "valid": true + }, + { + "description": "ignores strings", + "data": "foo", + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + } + ] + }, + { + "description": "unevaluatedItems with null instance elements", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null elements", + "data": [ null ], + "valid": true + } + ] + }, + { + "description": "unevaluatedItems can see annotations from if without then and else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "prefixItems": [{"const": "a"}] + }, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "valid in case if is evaluated", + "data": [ "a" ], + "valid": true + }, + { + "description": "invalid in case if is evaluated", + "data": [ "b" ], + "valid": false + } + ] + }, + { + "description": "Evaluated items collection needs to consider instance location", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { + "prefixItems": [ + true, + { "type": "string" } + ] + } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with an unevaluated item that exists at another location", + "data": [ + ["foo", "bar"], + "bar" + ], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedProperties.json new file mode 100644 index 000000000..1995d0ea9 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedProperties.json @@ -0,0 +1,1681 @@ +[ + { + "description": "unevaluatedProperties true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedProperties": true + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": {}, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedProperties": { + "type": "string", + "minLength": 3 + } + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": {}, + "valid": true + }, + { + "description": "with valid unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with invalid unevaluated properties", + "data": { + "foo": "fo" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": {}, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with adjacent properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with adjacent patternProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "^foo": { "type": "string" } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with adjacent bool additionalProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "foo": { "type": "string" } + }, + "additionalProperties": true, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no additional properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with additional properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties with adjacent non-bool additionalProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "foo": { "type": "string" } + }, + "additionalProperties": {"type": "string"}, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no additional properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with additional properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties with nested properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [ + { + "properties": { + "bar": { "type": "string" } + } + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no additional properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with additional properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with nested patternProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [ + { + "patternProperties": { + "^bar": { "type": "string" } + } + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no additional properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with additional properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with nested additionalProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [ + { + "additionalProperties": true + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no additional properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with additional properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties with nested unevaluatedProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [ + { + "unevaluatedProperties": true + } + ], + "unevaluatedProperties": { + "type": "string", + "maxLength": 2 + } + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties with anyOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "anyOf": [ + { + "properties": { + "bar": { "const": "bar" } + }, + "required": ["bar"] + }, + { + "properties": { + "baz": { "const": "baz" } + }, + "required": ["baz"] + }, + { + "properties": { + "quux": { "const": "quux" } + }, + "required": ["quux"] + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "when one matches and has no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "when one matches and has unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "not-baz" + }, + "valid": false + }, + { + "description": "when two match and has no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": true + }, + { + "description": "when two match and has unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz", + "quux": "not-quux" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with oneOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "oneOf": [ + { + "properties": { + "bar": { "const": "bar" } + }, + "required": ["bar"] + }, + { + "properties": { + "baz": { "const": "baz" } + }, + "required": ["baz"] + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "quux": "quux" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with not", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "not": { + "not": { + "properties": { + "bar": { "const": "bar" } + }, + "required": ["bar"] + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with if/then/else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "properties": { + "foo": { "const": "then" } + }, + "required": ["foo"] + }, + "then": { + "properties": { + "bar": { "type": "string" } + }, + "required": ["bar"] + }, + "else": { + "properties": { + "baz": { "type": "string" } + }, + "required": ["baz"] + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "when if is true and has no unevaluated properties", + "data": { + "foo": "then", + "bar": "bar" + }, + "valid": true + }, + { + "description": "when if is true and has unevaluated properties", + "data": { + "foo": "then", + "bar": "bar", + "baz": "baz" + }, + "valid": false + }, + { + "description": "when if is false and has no unevaluated properties", + "data": { + "baz": "baz" + }, + "valid": true + }, + { + "description": "when if is false and has unevaluated properties", + "data": { + "foo": "else", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with if/then/else, then not defined", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "properties": { + "foo": { "const": "then" } + }, + "required": ["foo"] + }, + "else": { + "properties": { + "baz": { "type": "string" } + }, + "required": ["baz"] + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "when if is true and has no unevaluated properties", + "data": { + "foo": "then" + }, + "valid": true + }, + { + "description": "when if is true and has unevaluated properties", + "data": { + "foo": "then", + "bar": "bar" + }, + "valid": false + }, + { + "description": "when if is false and has no unevaluated properties", + "data": { + "baz": "baz" + }, + "valid": true + }, + { + "description": "when if is false and has unevaluated properties", + "data": { + "foo": "else", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with if/then/else, else not defined", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "properties": { + "foo": { "const": "then" } + }, + "required": ["foo"] + }, + "then": { + "properties": { + "bar": { "type": "string" } + }, + "required": ["bar"] + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "when if is true and has no unevaluated properties", + "data": { + "foo": "then", + "bar": "bar" + }, + "valid": true + }, + { + "description": "when if is true and has unevaluated properties", + "data": { + "foo": "then", + "bar": "bar", + "baz": "baz" + }, + "valid": false + }, + { + "description": "when if is false and has no unevaluated properties", + "data": { + "baz": "baz" + }, + "valid": false + }, + { + "description": "when if is false and has unevaluated properties", + "data": { + "foo": "else", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with dependentSchemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "dependentSchemas": { + "foo": { + "properties": { + "bar": { "const": "bar" } + }, + "required": ["bar"] + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with boolean schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [true], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with $ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "#/$defs/bar", + "properties": { + "foo": { "type": "string" } + }, + "unevaluatedProperties": false, + "$defs": { + "bar": { + "properties": { + "bar": { "type": "string" } + } + } + } + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties before $ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedProperties": false, + "properties": { + "foo": { "type": "string" } + }, + "$ref": "#/$defs/bar", + "$defs": { + "bar": { + "properties": { + "bar": { "type": "string" } + } + } + } + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with $dynamicRef", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/unevaluated-properties-with-dynamic-ref/derived", + + "$ref": "./baseSchema", + + "$defs": { + "derived": { + "$dynamicAnchor": "addons", + "properties": { + "bar": { "type": "string" } + } + }, + "baseSchema": { + "$id": "./baseSchema", + + "$comment": "unevaluatedProperties comes first so it's more likely to catch bugs with implementations that are sensitive to keyword ordering", + "unevaluatedProperties": false, + "properties": { + "foo": { "type": "string" } + }, + "$dynamicRef": "#addons", + + "$defs": { + "defaultAddons": { + "$comment": "Needed to satisfy the bookending requirement", + "$dynamicAnchor": "addons" + } + } + } + } + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties can't see inside cousins", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "foo": true + } + }, + { + "unevaluatedProperties": false + } + ] + }, + "tests": [ + { + "description": "always fails", + "data": { + "foo": 1 + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties can't see inside cousins (reverse order)", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "unevaluatedProperties": false + }, + { + "properties": { + "foo": true + } + } + ] + }, + "tests": [ + { + "description": "always fails", + "data": { + "foo": 1 + }, + "valid": false + } + ] + }, + { + "description": "nested unevaluatedProperties, outer false, inner true, properties outside", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [ + { + "unevaluatedProperties": true + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + } + ] + }, + { + "description": "nested unevaluatedProperties, outer false, inner true, properties inside", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "foo": { "type": "string" } + }, + "unevaluatedProperties": true + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + } + ] + }, + { + "description": "nested unevaluatedProperties, outer true, inner false, properties outside", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [ + { + "unevaluatedProperties": false + } + ], + "unevaluatedProperties": true + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": false + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "nested unevaluatedProperties, outer true, inner false, properties inside", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "foo": { "type": "string" } + }, + "unevaluatedProperties": false + } + ], + "unevaluatedProperties": true + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "cousin unevaluatedProperties, true and false, true with properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "foo": { "type": "string" } + }, + "unevaluatedProperties": true + }, + { + "unevaluatedProperties": false + } + ] + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": false + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "cousin unevaluatedProperties, true and false, false with properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "unevaluatedProperties": true + }, + { + "properties": { + "foo": { "type": "string" } + }, + "unevaluatedProperties": false + } + ] + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "property is evaluated in an uncle schema to unevaluatedProperties", + "comment": "see https://stackoverflow.com/questions/66936884/deeply-nested-unevaluatedproperties-and-their-expectations", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "properties": { + "bar": { + "type": "string" + } + }, + "unevaluatedProperties": false + } + }, + "anyOf": [ + { + "properties": { + "foo": { + "properties": { + "faz": { + "type": "string" + } + } + } + } + } + ] + }, + "tests": [ + { + "description": "no extra properties", + "data": { + "foo": { + "bar": "test" + } + }, + "valid": true + }, + { + "description": "uncle keyword evaluation is not significant", + "data": { + "foo": { + "bar": "test", + "faz": "test" + } + }, + "valid": false + } + ] + }, + { + "description": "in-place applicator siblings, allOf has unevaluated", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "foo": true + }, + "unevaluatedProperties": false + } + ], + "anyOf": [ + { + "properties": { + "bar": true + } + } + ] + }, + "tests": [ + { + "description": "base case: both properties present", + "data": { + "foo": 1, + "bar": 1 + }, + "valid": false + }, + { + "description": "in place applicator siblings, bar is missing", + "data": { + "foo": 1 + }, + "valid": true + }, + { + "description": "in place applicator siblings, foo is missing", + "data": { + "bar": 1 + }, + "valid": false + } + ] + }, + { + "description": "in-place applicator siblings, anyOf has unevaluated", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "foo": true + } + } + ], + "anyOf": [ + { + "properties": { + "bar": true + }, + "unevaluatedProperties": false + } + ] + }, + "tests": [ + { + "description": "base case: both properties present", + "data": { + "foo": 1, + "bar": 1 + }, + "valid": false + }, + { + "description": "in place applicator siblings, bar is missing", + "data": { + "foo": 1 + }, + "valid": false + }, + { + "description": "in place applicator siblings, foo is missing", + "data": { + "bar": 1 + }, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties + single cyclic ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "x": { "$ref": "#" } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "Empty is valid", + "data": {}, + "valid": true + }, + { + "description": "Single is valid", + "data": { "x": {} }, + "valid": true + }, + { + "description": "Unevaluated on 1st level is invalid", + "data": { "x": {}, "y": {} }, + "valid": false + }, + { + "description": "Nested is valid", + "data": { "x": { "x": {} } }, + "valid": true + }, + { + "description": "Unevaluated on 2nd level is invalid", + "data": { "x": { "x": {}, "y": {} } }, + "valid": false + }, + { + "description": "Deep nested is valid", + "data": { "x": { "x": { "x": {} } } }, + "valid": true + }, + { + "description": "Unevaluated on 3rd level is invalid", + "data": { "x": { "x": { "x": {}, "y": {} } } }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties + ref inside allOf / oneOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "one": { + "properties": { "a": true } + }, + "two": { + "required": ["x"], + "properties": { "x": true } + } + }, + "allOf": [ + { "$ref": "#/$defs/one" }, + { "properties": { "b": true } }, + { + "oneOf": [ + { "$ref": "#/$defs/two" }, + { + "required": ["y"], + "properties": { "y": true } + } + ] + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "Empty is invalid (no x or y)", + "data": {}, + "valid": false + }, + { + "description": "a and b are invalid (no x or y)", + "data": { "a": 1, "b": 1 }, + "valid": false + }, + { + "description": "x and y are invalid", + "data": { "x": 1, "y": 1 }, + "valid": false + }, + { + "description": "a and x are valid", + "data": { "a": 1, "x": 1 }, + "valid": true + }, + { + "description": "a and y are valid", + "data": { "a": 1, "y": 1 }, + "valid": true + }, + { + "description": "a and b and x are valid", + "data": { "a": 1, "b": 1, "x": 1 }, + "valid": true + }, + { + "description": "a and b and y are valid", + "data": { "a": 1, "b": 1, "y": 1 }, + "valid": true + }, + { + "description": "a and b and x and y are invalid", + "data": { "a": 1, "b": 1, "x": 1, "y": 1 }, + "valid": false + } + ] + }, + { + "description": "dynamic evaluation inside nested refs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "one": { + "oneOf": [ + { "$ref": "#/$defs/two" }, + { "required": ["b"], "properties": { "b": true } }, + { "required": ["xx"], "patternProperties": { "x": true } }, + { "required": ["all"], "unevaluatedProperties": true } + ] + }, + "two": { + "oneOf": [ + { "required": ["c"], "properties": { "c": true } }, + { "required": ["d"], "properties": { "d": true } } + ] + } + }, + "oneOf": [ + { "$ref": "#/$defs/one" }, + { "required": ["a"], "properties": { "a": true } } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "Empty is invalid", + "data": {}, + "valid": false + }, + { + "description": "a is valid", + "data": { "a": 1 }, + "valid": true + }, + { + "description": "b is valid", + "data": { "b": 1 }, + "valid": true + }, + { + "description": "c is valid", + "data": { "c": 1 }, + "valid": true + }, + { + "description": "d is valid", + "data": { "d": 1 }, + "valid": true + }, + { + "description": "a + b is invalid", + "data": { "a": 1, "b": 1 }, + "valid": false + }, + { + "description": "a + c is invalid", + "data": { "a": 1, "c": 1 }, + "valid": false + }, + { + "description": "a + d is invalid", + "data": { "a": 1, "d": 1 }, + "valid": false + }, + { + "description": "b + c is invalid", + "data": { "b": 1, "c": 1 }, + "valid": false + }, + { + "description": "b + d is invalid", + "data": { "b": 1, "d": 1 }, + "valid": false + }, + { + "description": "c + d is invalid", + "data": { "c": 1, "d": 1 }, + "valid": false + }, + { + "description": "xx is valid", + "data": { "xx": 1 }, + "valid": true + }, + { + "description": "xx + foox is valid", + "data": { "xx": 1, "foox": 1 }, + "valid": true + }, + { + "description": "xx + foo is invalid", + "data": { "xx": 1, "foo": 1 }, + "valid": false + }, + { + "description": "xx + a is invalid", + "data": { "xx": 1, "a": 1 }, + "valid": false + }, + { + "description": "xx + b is invalid", + "data": { "xx": 1, "b": 1 }, + "valid": false + }, + { + "description": "xx + c is invalid", + "data": { "xx": 1, "c": 1 }, + "valid": false + }, + { + "description": "xx + d is invalid", + "data": { "xx": 1, "d": 1 }, + "valid": false + }, + { + "description": "all is valid", + "data": { "all": 1 }, + "valid": true + }, + { + "description": "all + foo is valid", + "data": { "all": 1, "foo": 1 }, + "valid": true + }, + { + "description": "all + a is invalid", + "data": { "all": 1, "a": 1 }, + "valid": false + } + ] + }, + { + "description": "non-object instances are valid", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "ignores booleans", + "data": true, + "valid": true + }, + { + "description": "ignores integers", + "data": 123, + "valid": true + }, + { + "description": "ignores floats", + "data": 1.0, + "valid": true + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores strings", + "data": "foo", + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties with null valued instance properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedProperties": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null valued properties", + "data": {"foo": null}, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties not affected by propertyNames", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": {"maxLength": 1}, + "unevaluatedProperties": { + "type": "number" + } + }, + "tests": [ + { + "description": "allows only number properties", + "data": {"a": 1}, + "valid": true + }, + { + "description": "string property is invalid", + "data": {"a": "b"}, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties can see annotations from if without then and else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "patternProperties": { + "foo": { + "type": "string" + } + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "valid in case if is evaluated", + "data": { + "foo": "a" + }, + "valid": true + }, + { + "description": "invalid in case if is evaluated", + "data": { + "bar": "a" + }, + "valid": false + } + ] + }, + { + "description": "dependentSchemas with unevaluatedProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"foo2": {}}, + "dependentSchemas": { + "foo" : {}, + "foo2": { + "properties": { + "bar":{} + } + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "unevaluatedProperties doesn't consider dependentSchemas", + "data": {"foo": ""}, + "valid": false + }, + { + "description": "unevaluatedProperties doesn't see bar when foo2 is absent", + "data": {"bar": ""}, + "valid": false + }, + { + "description": "unevaluatedProperties sees bar when foo2 is present", + "data": { "foo2": "", "bar": ""}, + "valid": true + } + ] + }, + { + "description": "Evaluated properties collection needs to consider instance location", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with an unevaluated property that exists at another location", + "data": { + "foo": { "bar": "foo" }, + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "Evaluated properties collection needs to consider instance location with patternProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "patternProperties": { + "^bar$": { "type": "string" } + } + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with only the nested evaluated property", + "data": { + "foo": { "bar": "foo" } + }, + "valid": true + }, + { + "description": "with an unevaluated property that exists at another location", + "data": { + "foo": { "bar": "foo" }, + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "Evaluated properties collection needs to consider instance location with additionalProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "additionalProperties": { "type": "string" } + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with only the nested evaluated property", + "data": { + "foo": { "bar": "foo" } + }, + "valid": true + }, + { + "description": "with an unevaluated property that exists at another location", + "data": { + "foo": { "bar": "foo" }, + "bar": "bar" + }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/uniqueItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/uniqueItems.json new file mode 100644 index 000000000..262114529 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/uniqueItems.json @@ -0,0 +1,399 @@ +[ + { + "description": "uniqueItems validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "uniqueItems": true + }, + "tests": [ + { + "description": "unique array of integers is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "non-unique array of integers is invalid", + "data": [1, 1], + "valid": false + }, + { + "description": "non-unique array of more than two integers is invalid", + "data": [1, 2, 1], + "valid": false + }, + { + "description": "numbers are unique if mathematically unequal", + "data": [1.0, 1.00, 1], + "valid": false + }, + { + "description": "false is not equal to zero", + "data": [0, false], + "valid": true + }, + { + "description": "true is not equal to one", + "data": [1, true], + "valid": true + }, + { + "description": "unique array of strings is valid", + "data": ["foo", "bar", "baz"], + "valid": true + }, + { + "description": "non-unique array of strings is invalid", + "data": ["foo", "bar", "foo"], + "valid": false + }, + { + "description": "unique array of objects is valid", + "data": [{"foo": "bar"}, {"foo": "baz"}], + "valid": true + }, + { + "description": "non-unique array of objects is invalid", + "data": [{"foo": "bar"}, {"foo": "bar"}], + "valid": false + }, + { + "description": "property order of array of objects is ignored", + "data": [{"foo": "bar", "bar": "foo"}, {"bar": "foo", "foo": "bar"}], + "valid": false + }, + { + "description": "unique array of nested objects is valid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : false}}} + ], + "valid": true + }, + { + "description": "non-unique array of nested objects is invalid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : true}}} + ], + "valid": false + }, + { + "description": "unique array of arrays is valid", + "data": [["foo"], ["bar"]], + "valid": true + }, + { + "description": "non-unique array of arrays is invalid", + "data": [["foo"], ["foo"]], + "valid": false + }, + { + "description": "non-unique array of more than two arrays is invalid", + "data": [["foo"], ["bar"], ["foo"]], + "valid": false + }, + { + "description": "[1] and [true] are unique", + "data": [[1], [true]], + "valid": true + }, + { + "description": "[0] and [false] are unique", + "data": [[0], [false]], + "valid": true + }, + { + "description": "nested [1] and [true] are unique", + "data": [[[1], "foo"], [[true], "foo"]], + "valid": true + }, + { + "description": "nested [0] and [false] are unique", + "data": [[[0], "foo"], [[false], "foo"]], + "valid": true + }, + { + "description": "unique heterogeneous types are valid", + "data": [{}, [1], true, null, 1, "{}"], + "valid": true + }, + { + "description": "non-unique heterogeneous types are invalid", + "data": [{}, [1], true, null, {}, 1], + "valid": false + }, + { + "description": "different objects are unique", + "data": [{"a": 1, "b": 2}, {"a": 2, "b": 1}], + "valid": true + }, + { + "description": "objects are non-unique despite key order", + "data": [{"a": 1, "b": 2}, {"b": 2, "a": 1}], + "valid": false + }, + { + "description": "{\"a\": false} and {\"a\": 0} are unique", + "data": [{"a": false}, {"a": 0}], + "valid": true + }, + { + "description": "{\"a\": true} and {\"a\": 1} are unique", + "data": [{"a": true}, {"a": 1}], + "valid": true + } + ] + }, + { + "description": "uniqueItems with an array of items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": true + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is not valid", + "data": [false, false], + "valid": false + }, + { + "description": "[true, true] from items array is not valid", + "data": [true, true], + "valid": false + }, + { + "description": "unique array extended from [false, true] is valid", + "data": [false, true, "foo", "bar"], + "valid": true + }, + { + "description": "unique array extended from [true, false] is valid", + "data": [true, false, "foo", "bar"], + "valid": true + }, + { + "description": "non-unique array extended from [false, true] is not valid", + "data": [false, true, "foo", "foo"], + "valid": false + }, + { + "description": "non-unique array extended from [true, false] is not valid", + "data": [true, false, "foo", "foo"], + "valid": false + } + ] + }, + { + "description": "uniqueItems with an array of items and additionalItems=false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": true, + "items": false + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is not valid", + "data": [false, false], + "valid": false + }, + { + "description": "[true, true] from items array is not valid", + "data": [true, true], + "valid": false + }, + { + "description": "extra items are invalid even if unique", + "data": [false, true, null], + "valid": false + } + ] + }, + { + "description": "uniqueItems=false validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "uniqueItems": false + }, + "tests": [ + { + "description": "unique array of integers is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "non-unique array of integers is valid", + "data": [1, 1], + "valid": true + }, + { + "description": "numbers are unique if mathematically unequal", + "data": [1.0, 1.00, 1], + "valid": true + }, + { + "description": "false is not equal to zero", + "data": [0, false], + "valid": true + }, + { + "description": "true is not equal to one", + "data": [1, true], + "valid": true + }, + { + "description": "unique array of objects is valid", + "data": [{"foo": "bar"}, {"foo": "baz"}], + "valid": true + }, + { + "description": "non-unique array of objects is valid", + "data": [{"foo": "bar"}, {"foo": "bar"}], + "valid": true + }, + { + "description": "unique array of nested objects is valid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : false}}} + ], + "valid": true + }, + { + "description": "non-unique array of nested objects is valid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : true}}} + ], + "valid": true + }, + { + "description": "unique array of arrays is valid", + "data": [["foo"], ["bar"]], + "valid": true + }, + { + "description": "non-unique array of arrays is valid", + "data": [["foo"], ["foo"]], + "valid": true + }, + { + "description": "unique heterogeneous types are valid", + "data": [{}, [1], true, null, 1], + "valid": true + }, + { + "description": "non-unique heterogeneous types are valid", + "data": [{}, [1], true, null, {}, 1], + "valid": true + } + ] + }, + { + "description": "uniqueItems=false with an array of items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": false + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is valid", + "data": [false, false], + "valid": true + }, + { + "description": "[true, true] from items array is valid", + "data": [true, true], + "valid": true + }, + { + "description": "unique array extended from [false, true] is valid", + "data": [false, true, "foo", "bar"], + "valid": true + }, + { + "description": "unique array extended from [true, false] is valid", + "data": [true, false, "foo", "bar"], + "valid": true + }, + { + "description": "non-unique array extended from [false, true] is valid", + "data": [false, true, "foo", "foo"], + "valid": true + }, + { + "description": "non-unique array extended from [true, false] is valid", + "data": [true, false, "foo", "foo"], + "valid": true + } + ] + }, + { + "description": "uniqueItems=false with an array of items and additionalItems=false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": false, + "items": false + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is valid", + "data": [false, false], + "valid": true + }, + { + "description": "[true, true] from items array is valid", + "data": [true, true], + "valid": true + }, + { + "description": "extra items are invalid even if unique", + "data": [false, true, null], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/vocabulary.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/vocabulary.json new file mode 100644 index 000000000..1acb96a93 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/vocabulary.json @@ -0,0 +1,57 @@ +[ + { + "description": "schema that uses custom metaschema with with no validation vocabulary", + "schema": { + "$id": "https://schema/using/no/validation", + "$schema": "http://localhost:1234/draft2020-12/metaschema-no-validation.json", + "properties": { + "badProperty": false, + "numberProperty": { + "minimum": 10 + } + } + }, + "tests": [ + { + "description": "applicator vocabulary still works", + "data": { + "badProperty": "this property should not exist" + }, + "valid": false + }, + { + "description": "no validation: valid number", + "data": { + "numberProperty": 20 + }, + "valid": true + }, + { + "description": "no validation: invalid number, but it still validates", + "data": { + "numberProperty": 1 + }, + "valid": true + } + ] + }, + { + "description": "ignore unrecognized optional vocabulary", + "schema": { + "$schema": "http://localhost:1234/draft2020-12/metaschema-optional-vocabulary.json", + "type": "number" + }, + "tests": [ + { + "description": "string value", + "data": "foobar", + "valid": false + }, + { + "description": "number value", + "data": 20, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalItems.json new file mode 100644 index 000000000..6ef4212b1 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalItems.json @@ -0,0 +1,190 @@ +[ + { + "description": "additionalItems as schema", + "schema": { + "items": [{}], + "additionalItems": {"type": "integer"} + }, + "tests": [ + { + "description": "additional items match schema", + "data": [ null, 2, 3, 4 ], + "valid": true + }, + { + "description": "additional items do not match schema", + "data": [ null, 2, 3, "foo" ], + "valid": false + } + ] + }, + { + "description": "when items is schema, additionalItems does nothing", + "schema": { + "items": { + "type": "integer" + }, + "additionalItems": { + "type": "string" + } + }, + "tests": [ + { + "description": "valid with a array of type integers", + "data": [1,2,3], + "valid": true + }, + { + "description": "invalid with a array of mixed types", + "data": [1,"2","3"], + "valid": false + } + ] + }, + { + "description": "when items is schema, boolean additionalItems does nothing", + "schema": { + "items": {}, + "additionalItems": false + }, + "tests": [ + { + "description": "all items match schema", + "data": [ 1, 2, 3, 4, 5 ], + "valid": true + } + ] + }, + { + "description": "array of items with no additionalItems permitted", + "schema": { + "items": [{}, {}, {}], + "additionalItems": false + }, + "tests": [ + { + "description": "empty array", + "data": [ ], + "valid": true + }, + { + "description": "fewer number of items present (1)", + "data": [ 1 ], + "valid": true + }, + { + "description": "fewer number of items present (2)", + "data": [ 1, 2 ], + "valid": true + }, + { + "description": "equal number of items present", + "data": [ 1, 2, 3 ], + "valid": true + }, + { + "description": "additional items are not permitted", + "data": [ 1, 2, 3, 4 ], + "valid": false + } + ] + }, + { + "description": "additionalItems as false without items", + "schema": {"additionalItems": false}, + "tests": [ + { + "description": + "items defaults to empty schema so everything is valid", + "data": [ 1, 2, 3, 4, 5 ], + "valid": true + }, + { + "description": "ignores non-arrays", + "data": {"foo" : "bar"}, + "valid": true + } + ] + }, + { + "description": "additionalItems are allowed by default", + "schema": {"items": [{"type": "integer"}]}, + "tests": [ + { + "description": "only the first item is validated", + "data": [1, "foo", false], + "valid": true + } + ] + }, + { + "description": "additionalItems does not look in applicators, invalid case", + "schema": { + "allOf": [ + { "items": [ { "type": "integer" }, { "type": "string" } ] } + ], + "items": [ {"type": "integer" } ], + "additionalItems": { "type": "boolean" } + }, + "tests": [ + { + "description": "items defined in allOf are not examined", + "data": [ 1, "hello" ], + "valid": false + } + ] + }, + { + "description": "items validation adjusts the starting index for additionalItems", + "schema": { + "items": [ { "type": "string" } ], + "additionalItems": { "type": "integer" } + }, + "tests": [ + { + "description": "valid items", + "data": [ "x", 2, 3 ], + "valid": true + }, + { + "description": "wrong type of second item", + "data": [ "x", "y" ], + "valid": false + } + ] + }, + { + "description": "additionalItems with heterogeneous array", + "schema": { + "items": [{}], + "additionalItems": false + }, + "tests": [ + { + "description": "heterogeneous invalid instance", + "data": [ "foo", "bar", 37 ], + "valid": false + }, + { + "description": "valid instance", + "data": [ null ], + "valid": true + } + ] + }, + { + "description": "additionalItems with null instance elements", + "schema": { + "additionalItems": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null elements", + "data": [ null ], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalProperties.json new file mode 100644 index 000000000..0f8e1627a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalProperties.json @@ -0,0 +1,147 @@ +[ + { + "description": + "additionalProperties being false does not allow other properties", + "schema": { + "properties": {"foo": {}, "bar": {}}, + "patternProperties": { "^v": {} }, + "additionalProperties": false + }, + "tests": [ + { + "description": "no additional properties is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "an additional property is invalid", + "data": {"foo" : 1, "bar" : 2, "quux" : "boom"}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [1, 2, 3], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobarbaz", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "patternProperties are not additional properties", + "data": {"foo":1, "vroom": 2}, + "valid": true + } + ] + }, + { + "description": "non-ASCII pattern with additionalProperties", + "schema": { + "patternProperties": {"^á": {}}, + "additionalProperties": false + }, + "tests": [ + { + "description": "matching the pattern is valid", + "data": {"ármányos": 2}, + "valid": true + }, + { + "description": "not matching the pattern is invalid", + "data": {"élmény": 2}, + "valid": false + } + ] + }, + { + "description": "additionalProperties with schema", + "schema": { + "properties": {"foo": {}, "bar": {}}, + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "no additional properties is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "an additional valid property is valid", + "data": {"foo" : 1, "bar" : 2, "quux" : true}, + "valid": true + }, + { + "description": "an additional invalid property is invalid", + "data": {"foo" : 1, "bar" : 2, "quux" : 12}, + "valid": false + } + ] + }, + { + "description": + "additionalProperties can exist by itself", + "schema": { + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "an additional valid property is valid", + "data": {"foo" : true}, + "valid": true + }, + { + "description": "an additional invalid property is invalid", + "data": {"foo" : 1}, + "valid": false + } + ] + }, + { + "description": "additionalProperties are allowed by default", + "schema": {"properties": {"foo": {}, "bar": {}}}, + "tests": [ + { + "description": "additional properties are allowed", + "data": {"foo": 1, "bar": 2, "quux": true}, + "valid": true + } + ] + }, + { + "description": "additionalProperties does not look in applicators", + "schema": { + "allOf": [ + {"properties": {"foo": {}}} + ], + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "properties defined in allOf are not examined", + "data": {"foo": 1, "bar": true}, + "valid": false + } + ] + }, + { + "description": "additionalProperties with null valued instance properties", + "schema": { + "additionalProperties": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null values", + "data": {"foo": null}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/allOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/allOf.json new file mode 100644 index 000000000..ec9319e14 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/allOf.json @@ -0,0 +1,294 @@ +[ + { + "description": "allOf", + "schema": { + "allOf": [ + { + "properties": { + "bar": {"type": "integer"} + }, + "required": ["bar"] + }, + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "allOf", + "data": {"foo": "baz", "bar": 2}, + "valid": true + }, + { + "description": "mismatch second", + "data": {"foo": "baz"}, + "valid": false + }, + { + "description": "mismatch first", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "wrong type", + "data": {"foo": "baz", "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "allOf with base schema", + "schema": { + "properties": {"bar": {"type": "integer"}}, + "required": ["bar"], + "allOf" : [ + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + }, + { + "properties": { + "baz": {"type": "null"} + }, + "required": ["baz"] + } + ] + }, + "tests": [ + { + "description": "valid", + "data": {"foo": "quux", "bar": 2, "baz": null}, + "valid": true + }, + { + "description": "mismatch base schema", + "data": {"foo": "quux", "baz": null}, + "valid": false + }, + { + "description": "mismatch first allOf", + "data": {"bar": 2, "baz": null}, + "valid": false + }, + { + "description": "mismatch second allOf", + "data": {"foo": "quux", "bar": 2}, + "valid": false + }, + { + "description": "mismatch both", + "data": {"bar": 2}, + "valid": false + } + ] + }, + { + "description": "allOf simple types", + "schema": { + "allOf": [ + {"maximum": 30}, + {"minimum": 20} + ] + }, + "tests": [ + { + "description": "valid", + "data": 25, + "valid": true + }, + { + "description": "mismatch one", + "data": 35, + "valid": false + } + ] + }, + { + "description": "allOf with boolean schemas, all true", + "schema": {"allOf": [true, true]}, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "allOf with boolean schemas, some false", + "schema": {"allOf": [true, false]}, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "allOf with boolean schemas, all false", + "schema": {"allOf": [false, false]}, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "allOf with one empty schema", + "schema": { + "allOf": [ + {} + ] + }, + "tests": [ + { + "description": "any data is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "allOf with two empty schemas", + "schema": { + "allOf": [ + {}, + {} + ] + }, + "tests": [ + { + "description": "any data is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "allOf with the first empty schema", + "schema": { + "allOf": [ + {}, + { "type": "number" } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "allOf with the last empty schema", + "schema": { + "allOf": [ + { "type": "number" }, + {} + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "nested allOf, to check validation semantics", + "schema": { + "allOf": [ + { + "allOf": [ + { + "type": "null" + } + ] + } + ] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "anything non-null is invalid", + "data": 123, + "valid": false + } + ] + }, + { + "description": "allOf combined with anyOf, oneOf", + "schema": { + "allOf": [ { "multipleOf": 2 } ], + "anyOf": [ { "multipleOf": 3 } ], + "oneOf": [ { "multipleOf": 5 } ] + }, + "tests": [ + { + "description": "allOf: false, anyOf: false, oneOf: false", + "data": 1, + "valid": false + }, + { + "description": "allOf: false, anyOf: false, oneOf: true", + "data": 5, + "valid": false + }, + { + "description": "allOf: false, anyOf: true, oneOf: false", + "data": 3, + "valid": false + }, + { + "description": "allOf: false, anyOf: true, oneOf: true", + "data": 15, + "valid": false + }, + { + "description": "allOf: true, anyOf: false, oneOf: false", + "data": 2, + "valid": false + }, + { + "description": "allOf: true, anyOf: false, oneOf: true", + "data": 10, + "valid": false + }, + { + "description": "allOf: true, anyOf: true, oneOf: false", + "data": 6, + "valid": false + }, + { + "description": "allOf: true, anyOf: true, oneOf: true", + "data": 30, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/anyOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/anyOf.json new file mode 100644 index 000000000..ab5eb386b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/anyOf.json @@ -0,0 +1,189 @@ +[ + { + "description": "anyOf", + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "minimum": 2 + } + ] + }, + "tests": [ + { + "description": "first anyOf valid", + "data": 1, + "valid": true + }, + { + "description": "second anyOf valid", + "data": 2.5, + "valid": true + }, + { + "description": "both anyOf valid", + "data": 3, + "valid": true + }, + { + "description": "neither anyOf valid", + "data": 1.5, + "valid": false + } + ] + }, + { + "description": "anyOf with base schema", + "schema": { + "type": "string", + "anyOf" : [ + { + "maxLength": 2 + }, + { + "minLength": 4 + } + ] + }, + "tests": [ + { + "description": "mismatch base schema", + "data": 3, + "valid": false + }, + { + "description": "one anyOf valid", + "data": "foobar", + "valid": true + }, + { + "description": "both anyOf invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "anyOf with boolean schemas, all true", + "schema": {"anyOf": [true, true]}, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "anyOf with boolean schemas, some true", + "schema": {"anyOf": [true, false]}, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "anyOf with boolean schemas, all false", + "schema": {"anyOf": [false, false]}, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "anyOf complex types", + "schema": { + "anyOf": [ + { + "properties": { + "bar": {"type": "integer"} + }, + "required": ["bar"] + }, + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "first anyOf valid (complex)", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "second anyOf valid (complex)", + "data": {"foo": "baz"}, + "valid": true + }, + { + "description": "both anyOf valid (complex)", + "data": {"foo": "baz", "bar": 2}, + "valid": true + }, + { + "description": "neither anyOf valid (complex)", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "anyOf with one empty schema", + "schema": { + "anyOf": [ + { "type": "number" }, + {} + ] + }, + "tests": [ + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "number is valid", + "data": 123, + "valid": true + } + ] + }, + { + "description": "nested anyOf, to check validation semantics", + "schema": { + "anyOf": [ + { + "anyOf": [ + { + "type": "null" + } + ] + } + ] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "anything non-null is invalid", + "data": 123, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/boolean_schema.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/boolean_schema.json new file mode 100644 index 000000000..6d40f23f2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/boolean_schema.json @@ -0,0 +1,104 @@ +[ + { + "description": "boolean schema 'true'", + "schema": true, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "boolean true is valid", + "data": true, + "valid": true + }, + { + "description": "boolean false is valid", + "data": false, + "valid": true + }, + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + }, + { + "description": "array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "boolean schema 'false'", + "schema": false, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "boolean true is invalid", + "data": true, + "valid": false + }, + { + "description": "boolean false is invalid", + "data": false, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/const.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/const.json new file mode 100644 index 000000000..168fff4e3 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/const.json @@ -0,0 +1,384 @@ +[ + { + "description": "const validation", + "schema": {"const": 2}, + "tests": [ + { + "description": "same value is valid", + "data": 2, + "valid": true + }, + { + "description": "another value is invalid", + "data": 5, + "valid": false + }, + { + "description": "another type is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "const with object", + "schema": {"const": {"foo": "bar", "baz": "bax"}}, + "tests": [ + { + "description": "same object is valid", + "data": {"foo": "bar", "baz": "bax"}, + "valid": true + }, + { + "description": "same object with different property order is valid", + "data": {"baz": "bax", "foo": "bar"}, + "valid": true + }, + { + "description": "another object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "another type is invalid", + "data": [1, 2], + "valid": false + } + ] + }, + { + "description": "const with array", + "schema": {"const": [{ "foo": "bar" }]}, + "tests": [ + { + "description": "same array is valid", + "data": [{"foo": "bar"}], + "valid": true + }, + { + "description": "another array item is invalid", + "data": [2], + "valid": false + }, + { + "description": "array with additional items is invalid", + "data": [1, 2, 3], + "valid": false + } + ] + }, + { + "description": "const with null", + "schema": {"const": null}, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "not null is invalid", + "data": 0, + "valid": false + } + ] + }, + { + "description": "const with false does not match 0", + "schema": {"const": false}, + "tests": [ + { + "description": "false is valid", + "data": false, + "valid": true + }, + { + "description": "integer zero is invalid", + "data": 0, + "valid": false + }, + { + "description": "float zero is invalid", + "data": 0.0, + "valid": false + } + ] + }, + { + "description": "const with true does not match 1", + "schema": {"const": true}, + "tests": [ + { + "description": "true is valid", + "data": true, + "valid": true + }, + { + "description": "integer one is invalid", + "data": 1, + "valid": false + }, + { + "description": "float one is invalid", + "data": 1.0, + "valid": false + } + ] + }, + { + "description": "const with [false] does not match [0]", + "schema": {"const": [false]}, + "tests": [ + { + "description": "[false] is valid", + "data": [false], + "valid": true + }, + { + "description": "[0] is invalid", + "data": [0], + "valid": false + }, + { + "description": "[0.0] is invalid", + "data": [0.0], + "valid": false + } + ] + }, + { + "description": "const with [true] does not match [1]", + "schema": {"const": [true]}, + "tests": [ + { + "description": "[true] is valid", + "data": [true], + "valid": true + }, + { + "description": "[1] is invalid", + "data": [1], + "valid": false + }, + { + "description": "[1.0] is invalid", + "data": [1.0], + "valid": false + } + ] + }, + { + "description": "const with {\"a\": false} does not match {\"a\": 0}", + "schema": {"const": {"a": false}}, + "tests": [ + { + "description": "{\"a\": false} is valid", + "data": {"a": false}, + "valid": true + }, + { + "description": "{\"a\": 0} is invalid", + "data": {"a": 0}, + "valid": false + }, + { + "description": "{\"a\": 0.0} is invalid", + "data": {"a": 0.0}, + "valid": false + } + ] + }, + { + "description": "const with {\"a\": true} does not match {\"a\": 1}", + "schema": {"const": {"a": true}}, + "tests": [ + { + "description": "{\"a\": true} is valid", + "data": {"a": true}, + "valid": true + }, + { + "description": "{\"a\": 1} is invalid", + "data": {"a": 1}, + "valid": false + }, + { + "description": "{\"a\": 1.0} is invalid", + "data": {"a": 1.0}, + "valid": false + } + ] + }, + { + "description": "const with 0 does not match other zero-like types", + "schema": {"const": 0}, + "tests": [ + { + "description": "false is invalid", + "data": false, + "valid": false + }, + { + "description": "integer zero is valid", + "data": 0, + "valid": true + }, + { + "description": "float zero is valid", + "data": 0.0, + "valid": true + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + }, + { + "description": "empty string is invalid", + "data": "", + "valid": false + } + ] + }, + { + "description": "const with 1 does not match true", + "schema": {"const": 1}, + "tests": [ + { + "description": "true is invalid", + "data": true, + "valid": false + }, + { + "description": "integer one is valid", + "data": 1, + "valid": true + }, + { + "description": "float one is valid", + "data": 1.0, + "valid": true + } + ] + }, + { + "description": "const with -2.0 matches integer and float types", + "schema": {"const": -2.0}, + "tests": [ + { + "description": "integer -2 is valid", + "data": -2, + "valid": true + }, + { + "description": "integer 2 is invalid", + "data": 2, + "valid": false + }, + { + "description": "float -2.0 is valid", + "data": -2.0, + "valid": true + }, + { + "description": "float 2.0 is invalid", + "data": 2.0, + "valid": false + }, + { + "description": "float -2.00001 is invalid", + "data": -2.00001, + "valid": false + } + ] + }, + { + "description": "float and integers are equal up to 64-bit representation limits", + "schema": {"const": 9007199254740992}, + "tests": [ + { + "description": "integer is valid", + "data": 9007199254740992, + "valid": true + }, + { + "description": "integer minus one is invalid", + "data": 9007199254740991, + "valid": false + }, + { + "description": "float is valid", + "data": 9007199254740992.0, + "valid": true + }, + { + "description": "float minus one is invalid", + "data": 9007199254740991.0, + "valid": false + } + ] + }, + { + "description": "nul characters in strings", + "schema": { "const": "hello\u0000there" }, + "tests": [ + { + "description": "match string with nul", + "data": "hello\u0000there", + "valid": true + }, + { + "description": "do not match string lacking nul", + "data": "hellothere", + "valid": false + } + ] + }, + { + "description": "characters with the same visual representation but different codepoint", + "schema": { + "const": "μ", + "$comment": "U+03BC" + }, + "tests": [ + { + "description": "character uses the same codepoint", + "data": "μ", + "comment": "U+03BC", + "valid": true + }, + { + "description": "character looks the same but uses a different codepoint", + "data": "µ", + "comment": "U+00B5", + "valid": false + } + ] + }, + { + "description": "characters with the same visual representation, but different number of codepoints", + "schema": { + "const": "ä", + "$comment": "U+00E4" + }, + "tests": [ + { + "description": "character uses the same codepoint", + "data": "ä", + "comment": "U+00E4", + "valid": true + }, + { + "description": "character looks the same but uses combining marks", + "data": "ä", + "comment": "a, U+0308", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/contains.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/contains.json new file mode 100644 index 000000000..2b1a51520 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/contains.json @@ -0,0 +1,165 @@ +[ + { + "description": "contains keyword validation", + "schema": { + "contains": {"minimum": 5} + }, + "tests": [ + { + "description": "array with item matching schema (5) is valid", + "data": [3, 4, 5], + "valid": true + }, + { + "description": "array with item matching schema (6) is valid", + "data": [3, 4, 6], + "valid": true + }, + { + "description": "array with two items matching schema (5, 6) is valid", + "data": [3, 4, 5, 6], + "valid": true + }, + { + "description": "array without items matching schema is invalid", + "data": [2, 3, 4], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + }, + { + "description": "not array is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "contains keyword with const keyword", + "schema": { + "contains": { "const": 5 } + }, + "tests": [ + { + "description": "array with item 5 is valid", + "data": [3, 4, 5], + "valid": true + }, + { + "description": "array with two items 5 is valid", + "data": [3, 4, 5, 5], + "valid": true + }, + { + "description": "array without item 5 is invalid", + "data": [1, 2, 3, 4], + "valid": false + } + ] + }, + { + "description": "contains keyword with boolean schema true", + "schema": {"contains": true}, + "tests": [ + { + "description": "any non-empty array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "contains keyword with boolean schema false", + "schema": {"contains": false}, + "tests": [ + { + "description": "any non-empty array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + }, + { + "description": "non-arrays are valid", + "data": "contains does not apply to strings", + "valid": true + } + ] + }, + { + "description": "items + contains", + "schema": { + "items": { "multipleOf": 2 }, + "contains": { "multipleOf": 3 } + }, + "tests": [ + { + "description": "matches items, does not match contains", + "data": [ 2, 4, 8 ], + "valid": false + }, + { + "description": "does not match items, matches contains", + "data": [ 3, 6, 9 ], + "valid": false + }, + { + "description": "matches both items and contains", + "data": [ 6, 12 ], + "valid": true + }, + { + "description": "matches neither items nor contains", + "data": [ 1, 5 ], + "valid": false + } + ] + }, + { + "description": "contains with false if subschema", + "schema": { + "contains": { + "if": false, + "else": true + } + }, + "tests": [ + { + "description": "any non-empty array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "contains with null instance elements", + "schema": { + "contains": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null items", + "data": [ null ], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/default.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/default.json new file mode 100644 index 000000000..289a9b66c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/default.json @@ -0,0 +1,79 @@ +[ + { + "description": "invalid type for default", + "schema": { + "properties": { + "foo": { + "type": "integer", + "default": [] + } + } + }, + "tests": [ + { + "description": "valid when property is specified", + "data": {"foo": 13}, + "valid": true + }, + { + "description": "still valid when the invalid default is used", + "data": {}, + "valid": true + } + ] + }, + { + "description": "invalid string value for default", + "schema": { + "properties": { + "bar": { + "type": "string", + "minLength": 4, + "default": "bad" + } + } + }, + "tests": [ + { + "description": "valid when property is specified", + "data": {"bar": "good"}, + "valid": true + }, + { + "description": "still valid when the invalid default is used", + "data": {}, + "valid": true + } + ] + }, + { + "description": "the default keyword does not do anything if the property is missing", + "schema": { + "type": "object", + "properties": { + "alpha": { + "type": "number", + "maximum": 3, + "default": 5 + } + } + }, + "tests": [ + { + "description": "an explicit property value is checked against maximum (passing)", + "data": { "alpha": 1 }, + "valid": true + }, + { + "description": "an explicit property value is checked against maximum (failing)", + "data": { "alpha": 5 }, + "valid": false + }, + { + "description": "missing properties are not filled in with the default", + "data": {}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/definitions.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/definitions.json new file mode 100644 index 000000000..afe396e42 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/definitions.json @@ -0,0 +1,26 @@ +[ + { + "description": "validate definition against metaschema", + "schema": {"$ref": "http://json-schema.org/draft-07/schema#"}, + "tests": [ + { + "description": "valid definition schema", + "data": { + "definitions": { + "foo": {"type": "integer"} + } + }, + "valid": true + }, + { + "description": "invalid definition schema", + "data": { + "definitions": { + "foo": {"type": 1} + } + }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/dependencies.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/dependencies.json new file mode 100644 index 000000000..c0bd809f6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/dependencies.json @@ -0,0 +1,286 @@ +[ + { + "description": "dependencies", + "schema": { + "dependencies": {"bar": ["foo"]} + }, + "tests": [ + { + "description": "neither", + "data": {}, + "valid": true + }, + { + "description": "nondependant", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "with dependency", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "missing dependency", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["bar"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "dependencies with empty array", + "schema": { + "dependencies": {"bar": []} + }, + "tests": [ + { + "description": "empty object", + "data": {}, + "valid": true + }, + { + "description": "object with one property", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "non-object is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "multiple dependencies", + "schema": { + "dependencies": {"quux": ["foo", "bar"]} + }, + "tests": [ + { + "description": "neither", + "data": {}, + "valid": true + }, + { + "description": "nondependants", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "with dependencies", + "data": {"foo": 1, "bar": 2, "quux": 3}, + "valid": true + }, + { + "description": "missing dependency", + "data": {"foo": 1, "quux": 2}, + "valid": false + }, + { + "description": "missing other dependency", + "data": {"bar": 1, "quux": 2}, + "valid": false + }, + { + "description": "missing both dependencies", + "data": {"quux": 1}, + "valid": false + } + ] + }, + { + "description": "multiple dependencies subschema", + "schema": { + "dependencies": { + "bar": { + "properties": { + "foo": {"type": "integer"}, + "bar": {"type": "integer"} + } + } + } + }, + "tests": [ + { + "description": "valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "no dependency", + "data": {"foo": "quux"}, + "valid": true + }, + { + "description": "wrong type", + "data": {"foo": "quux", "bar": 2}, + "valid": false + }, + { + "description": "wrong type other", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + }, + { + "description": "wrong type both", + "data": {"foo": "quux", "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "dependencies with boolean subschemas", + "schema": { + "dependencies": { + "foo": true, + "bar": false + } + }, + "tests": [ + { + "description": "object with property having schema true is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property having schema false is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "object with both properties is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "dependencies with escaped characters", + "schema": { + "dependencies": { + "foo\nbar": ["foo\rbar"], + "foo\tbar": { + "minProperties": 4 + }, + "foo'bar": {"required": ["foo\"bar"]}, + "foo\"bar": ["foo'bar"] + } + }, + "tests": [ + { + "description": "valid object 1", + "data": { + "foo\nbar": 1, + "foo\rbar": 2 + }, + "valid": true + }, + { + "description": "valid object 2", + "data": { + "foo\tbar": 1, + "a": 2, + "b": 3, + "c": 4 + }, + "valid": true + }, + { + "description": "valid object 3", + "data": { + "foo'bar": 1, + "foo\"bar": 2 + }, + "valid": true + }, + { + "description": "invalid object 1", + "data": { + "foo\nbar": 1, + "foo": 2 + }, + "valid": false + }, + { + "description": "invalid object 2", + "data": { + "foo\tbar": 1, + "a": 2 + }, + "valid": false + }, + { + "description": "invalid object 3", + "data": { + "foo'bar": 1 + }, + "valid": false + }, + { + "description": "invalid object 4", + "data": { + "foo\"bar": 2 + }, + "valid": false + } + ] + }, + { + "description": "dependent subschema incompatible with root", + "schema": { + "properties": { + "foo": {} + }, + "dependencies": { + "foo": { + "properties": { + "bar": {} + }, + "additionalProperties": false + } + } + }, + "tests": [ + { + "description": "matches root", + "data": {"foo": 1}, + "valid": false + }, + { + "description": "matches dependency", + "data": {"bar": 1}, + "valid": true + }, + { + "description": "matches both", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "no dependency", + "data": {"baz": 1}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/enum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/enum.json new file mode 100644 index 000000000..bf1e96fc5 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/enum.json @@ -0,0 +1,341 @@ +[ + { + "description": "simple enum validation", + "schema": {"enum": [1, 2, 3]}, + "tests": [ + { + "description": "one of the enum is valid", + "data": 1, + "valid": true + }, + { + "description": "something else is invalid", + "data": 4, + "valid": false + } + ] + }, + { + "description": "heterogeneous enum validation", + "schema": {"enum": [6, "foo", [], true, {"foo": 12}]}, + "tests": [ + { + "description": "one of the enum is valid", + "data": [], + "valid": true + }, + { + "description": "something else is invalid", + "data": null, + "valid": false + }, + { + "description": "objects are deep compared", + "data": {"foo": false}, + "valid": false + }, + { + "description": "valid object matches", + "data": {"foo": 12}, + "valid": true + }, + { + "description": "extra properties in object is invalid", + "data": {"foo": 12, "boo": 42}, + "valid": false + } + ] + }, + { + "description": "heterogeneous enum-with-null validation", + "schema": { "enum": [6, null] }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "number is valid", + "data": 6, + "valid": true + }, + { + "description": "something else is invalid", + "data": "test", + "valid": false + } + ] + }, + { + "description": "enums in properties", + "schema": { + "type":"object", + "properties": { + "foo": {"enum":["foo"]}, + "bar": {"enum":["bar"]} + }, + "required": ["bar"] + }, + "tests": [ + { + "description": "both properties are valid", + "data": {"foo":"foo", "bar":"bar"}, + "valid": true + }, + { + "description": "wrong foo value", + "data": {"foo":"foot", "bar":"bar"}, + "valid": false + }, + { + "description": "wrong bar value", + "data": {"foo":"foo", "bar":"bart"}, + "valid": false + }, + { + "description": "missing optional property is valid", + "data": {"bar":"bar"}, + "valid": true + }, + { + "description": "missing required property is invalid", + "data": {"foo":"foo"}, + "valid": false + }, + { + "description": "missing all properties is invalid", + "data": {}, + "valid": false + } + ] + }, + { + "description": "enum with escaped characters", + "schema": { + "enum": ["foo\nbar", "foo\rbar"] + }, + "tests": [ + { + "description": "member 1 is valid", + "data": "foo\nbar", + "valid": true + }, + { + "description": "member 2 is valid", + "data": "foo\rbar", + "valid": true + }, + { + "description": "another string is invalid", + "data": "abc", + "valid": false + } + ] + }, + { + "description": "enum with false does not match 0", + "schema": {"enum": [false]}, + "tests": [ + { + "description": "false is valid", + "data": false, + "valid": true + }, + { + "description": "integer zero is invalid", + "data": 0, + "valid": false + }, + { + "description": "float zero is invalid", + "data": 0.0, + "valid": false + } + ] + }, + { + "description": "enum with [false] does not match [0]", + "schema": {"enum": [[false]]}, + "tests": [ + { + "description": "[false] is valid", + "data": [false], + "valid": true + }, + { + "description": "[0] is invalid", + "data": [0], + "valid": false + }, + { + "description": "[0.0] is invalid", + "data": [0.0], + "valid": false + } + ] + }, + { + "description": "enum with true does not match 1", + "schema": {"enum": [true]}, + "tests": [ + { + "description": "true is valid", + "data": true, + "valid": true + }, + { + "description": "integer one is invalid", + "data": 1, + "valid": false + }, + { + "description": "float one is invalid", + "data": 1.0, + "valid": false + } + ] + }, + { + "description": "enum with [true] does not match [1]", + "schema": {"enum": [[true]]}, + "tests": [ + { + "description": "[true] is valid", + "data": [true], + "valid": true + }, + { + "description": "[1] is invalid", + "data": [1], + "valid": false + }, + { + "description": "[1.0] is invalid", + "data": [1.0], + "valid": false + } + ] + }, + { + "description": "enum with 0 does not match false", + "schema": {"enum": [0]}, + "tests": [ + { + "description": "false is invalid", + "data": false, + "valid": false + }, + { + "description": "integer zero is valid", + "data": 0, + "valid": true + }, + { + "description": "float zero is valid", + "data": 0.0, + "valid": true + } + ] + }, + { + "description": "enum with [0] does not match [false]", + "schema": {"enum": [[0]]}, + "tests": [ + { + "description": "[false] is invalid", + "data": [false], + "valid": false + }, + { + "description": "[0] is valid", + "data": [0], + "valid": true + }, + { + "description": "[0.0] is valid", + "data": [0.0], + "valid": true + } + ] + }, + { + "description": "enum with 1 does not match true", + "schema": {"enum": [1]}, + "tests": [ + { + "description": "true is invalid", + "data": true, + "valid": false + }, + { + "description": "integer one is valid", + "data": 1, + "valid": true + }, + { + "description": "float one is valid", + "data": 1.0, + "valid": true + } + ] + }, + { + "description": "enum with [1] does not match [true]", + "schema": {"enum": [[1]]}, + "tests": [ + { + "description": "[true] is invalid", + "data": [true], + "valid": false + }, + { + "description": "[1] is valid", + "data": [1], + "valid": true + }, + { + "description": "[1.0] is valid", + "data": [1.0], + "valid": true + } + ] + }, + { + "description": "nul characters in strings", + "schema": { "enum": [ "hello\u0000there" ] }, + "tests": [ + { + "description": "match string with nul", + "data": "hello\u0000there", + "valid": true + }, + { + "description": "do not match string lacking nul", + "data": "hellothere", + "valid": false + } + ] + }, + { + "description": "characters with the same visual representation, but different number of codepoints", + "schema": { + "enum": ["ä"], + "$comment": "U+00E4" + }, + "tests": [ + { + "description": "character uses the same codepoint", + "data": "ä", + "comment": "U+00E4", + "valid": true + }, + { + "description": "character looks the same but uses combining marks", + "data": "ä", + "comment": "a, U+0308", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMaximum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMaximum.json new file mode 100644 index 000000000..dc3cd709d --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMaximum.json @@ -0,0 +1,30 @@ +[ + { + "description": "exclusiveMaximum validation", + "schema": { + "exclusiveMaximum": 3.0 + }, + "tests": [ + { + "description": "below the exclusiveMaximum is valid", + "data": 2.2, + "valid": true + }, + { + "description": "boundary point is invalid", + "data": 3.0, + "valid": false + }, + { + "description": "above the exclusiveMaximum is invalid", + "data": 3.5, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMinimum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMinimum.json new file mode 100644 index 000000000..b38d7ecec --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMinimum.json @@ -0,0 +1,30 @@ +[ + { + "description": "exclusiveMinimum validation", + "schema": { + "exclusiveMinimum": 1.1 + }, + "tests": [ + { + "description": "above the exclusiveMinimum is valid", + "data": 1.2, + "valid": true + }, + { + "description": "boundary point is invalid", + "data": 1.1, + "valid": false + }, + { + "description": "below the exclusiveMinimum is invalid", + "data": 0.6, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/format.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/format.json new file mode 100644 index 000000000..e2447d60f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/format.json @@ -0,0 +1,614 @@ +[ + { + "description": "email format", + "schema": { "format": "email" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "idn-email format", + "schema": { "format": "idn-email" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "regex format", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "ipv4 format", + "schema": { "format": "ipv4" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "ipv6 format", + "schema": { "format": "ipv6" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "idn-hostname format", + "schema": { "format": "idn-hostname" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "hostname format", + "schema": { "format": "hostname" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "date format", + "schema": { "format": "date" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "date-time format", + "schema": { "format": "date-time" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "time format", + "schema": { "format": "time" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "json-pointer format", + "schema": { "format": "json-pointer" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "relative-json-pointer format", + "schema": { "format": "relative-json-pointer" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "iri format", + "schema": { "format": "iri" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "iri-reference format", + "schema": { "format": "iri-reference" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "uri format", + "schema": { "format": "uri" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "uri-reference format", + "schema": { "format": "uri-reference" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "uri-template format", + "schema": { "format": "uri-template" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/if-then-else.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/if-then-else.json new file mode 100644 index 000000000..a3bab7c0b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/if-then-else.json @@ -0,0 +1,296 @@ +[ + { + "description": "ignore if without then or else", + "schema": { + "if": { + "const": 0 + } + }, + "tests": [ + { + "description": "valid when valid against lone if", + "data": 0, + "valid": true + }, + { + "description": "valid when invalid against lone if", + "data": "hello", + "valid": true + } + ] + }, + { + "description": "ignore then without if", + "schema": { + "then": { + "const": 0 + } + }, + "tests": [ + { + "description": "valid when valid against lone then", + "data": 0, + "valid": true + }, + { + "description": "valid when invalid against lone then", + "data": "hello", + "valid": true + } + ] + }, + { + "description": "ignore else without if", + "schema": { + "else": { + "const": 0 + } + }, + "tests": [ + { + "description": "valid when valid against lone else", + "data": 0, + "valid": true + }, + { + "description": "valid when invalid against lone else", + "data": "hello", + "valid": true + } + ] + }, + { + "description": "if and then without else", + "schema": { + "if": { + "exclusiveMaximum": 0 + }, + "then": { + "minimum": -10 + } + }, + "tests": [ + { + "description": "valid through then", + "data": -1, + "valid": true + }, + { + "description": "invalid through then", + "data": -100, + "valid": false + }, + { + "description": "valid when if test fails", + "data": 3, + "valid": true + } + ] + }, + { + "description": "if and else without then", + "schema": { + "if": { + "exclusiveMaximum": 0 + }, + "else": { + "multipleOf": 2 + } + }, + "tests": [ + { + "description": "valid when if test passes", + "data": -1, + "valid": true + }, + { + "description": "valid through else", + "data": 4, + "valid": true + }, + { + "description": "invalid through else", + "data": 3, + "valid": false + } + ] + }, + { + "description": "validate against correct branch, then vs else", + "schema": { + "if": { + "exclusiveMaximum": 0 + }, + "then": { + "minimum": -10 + }, + "else": { + "multipleOf": 2 + } + }, + "tests": [ + { + "description": "valid through then", + "data": -1, + "valid": true + }, + { + "description": "invalid through then", + "data": -100, + "valid": false + }, + { + "description": "valid through else", + "data": 4, + "valid": true + }, + { + "description": "invalid through else", + "data": 3, + "valid": false + } + ] + }, + { + "description": "non-interference across combined schemas", + "schema": { + "allOf": [ + { + "if": { + "exclusiveMaximum": 0 + } + }, + { + "then": { + "minimum": -10 + } + }, + { + "else": { + "multipleOf": 2 + } + } + ] + }, + "tests": [ + { + "description": "valid, but would have been invalid through then", + "data": -100, + "valid": true + }, + { + "description": "valid, but would have been invalid through else", + "data": 3, + "valid": true + } + ] + }, + { + "description": "if with boolean schema true", + "schema": { + "if": true, + "then": { "const": "then" }, + "else": { "const": "else" } + }, + "tests": [ + { + "description": "boolean schema true in if always chooses the then path (valid)", + "data": "then", + "valid": true + }, + { + "description": "boolean schema true in if always chooses the then path (invalid)", + "data": "else", + "valid": false + } + ] + }, + { + "description": "if with boolean schema false", + "schema": { + "if": false, + "then": { "const": "then" }, + "else": { "const": "else" } + }, + "tests": [ + { + "description": "boolean schema false in if always chooses the else path (invalid)", + "data": "then", + "valid": false + }, + { + "description": "boolean schema false in if always chooses the else path (valid)", + "data": "else", + "valid": true + } + ] + }, + { + "description": "if appears at the end when serialized (keyword processing sequence)", + "schema": { + "then": { "const": "yes" }, + "else": { "const": "other" }, + "if": { "maxLength": 4 } + }, + "tests": [ + { + "description": "yes redirects to then and passes", + "data": "yes", + "valid": true + }, + { + "description": "other redirects to else and passes", + "data": "other", + "valid": true + }, + { + "description": "no redirects to then and fails", + "data": "no", + "valid": false + }, + { + "description": "invalid redirects to else and fails", + "data": "invalid", + "valid": false + } + ] + }, + { + "description": "then: false fails when condition matches", + "schema": { + "if": { "const": 1 }, + "then": false + }, + "tests": [ + { + "description": "matches if → then=false → invalid", + "data": 1, + "valid": false + }, + { + "description": "does not match if → then ignored → valid", + "data": 2, + "valid": true + } + ] + }, + { + "description": "else: false fails when condition does not match", + "schema": { + "if": { "const": 1 }, + "else": false + }, + "tests": [ + { + "description": "matches if → else ignored → valid", + "data": 1, + "valid": true + }, + { + "description": "does not match if → else executes → invalid", + "data": 2, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/infinite-loop-detection.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/infinite-loop-detection.json new file mode 100644 index 000000000..f98c74fc6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/infinite-loop-detection.json @@ -0,0 +1,36 @@ +[ + { + "description": "evaluating the same schema location against the same data location twice is not a sign of an infinite loop", + "schema": { + "definitions": { + "int": { "type": "integer" } + }, + "allOf": [ + { + "properties": { + "foo": { + "$ref": "#/definitions/int" + } + } + }, + { + "additionalProperties": { + "$ref": "#/definitions/int" + } + } + ] + }, + "tests": [ + { + "description": "passing case", + "data": { "foo": 1 }, + "valid": true + }, + { + "description": "failing case", + "data": { "foo": "a string" }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/items.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/items.json new file mode 100644 index 000000000..7ed6781b6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/items.json @@ -0,0 +1,282 @@ +[ + { + "description": "a schema given for items", + "schema": { + "items": {"type": "integer"} + }, + "tests": [ + { + "description": "valid items", + "data": [ 1, 2, 3 ], + "valid": true + }, + { + "description": "wrong type of items", + "data": [1, "x"], + "valid": false + }, + { + "description": "ignores non-arrays", + "data": {"foo" : "bar"}, + "valid": true + }, + { + "description": "JavaScript pseudo-array is valid", + "data": { + "0": "invalid", + "length": 1 + }, + "valid": true + } + ] + }, + { + "description": "an array of schemas for items", + "schema": { + "items": [ + {"type": "integer"}, + {"type": "string"} + ] + }, + "tests": [ + { + "description": "correct types", + "data": [ 1, "foo" ], + "valid": true + }, + { + "description": "wrong types", + "data": [ "foo", 1 ], + "valid": false + }, + { + "description": "incomplete array of items", + "data": [ 1 ], + "valid": true + }, + { + "description": "array with additional items", + "data": [ 1, "foo", true ], + "valid": true + }, + { + "description": "empty array", + "data": [ ], + "valid": true + }, + { + "description": "JavaScript pseudo-array is valid", + "data": { + "0": "invalid", + "1": "valid", + "length": 2 + }, + "valid": true + } + ] + }, + { + "description": "items with boolean schema (true)", + "schema": {"items": true}, + "tests": [ + { + "description": "any array is valid", + "data": [ 1, "foo", true ], + "valid": true + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "items with boolean schema (false)", + "schema": {"items": false}, + "tests": [ + { + "description": "any non-empty array is invalid", + "data": [ 1, "foo", true ], + "valid": false + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "items with boolean schemas", + "schema": { + "items": [true, false] + }, + "tests": [ + { + "description": "array with one item is valid", + "data": [ 1 ], + "valid": true + }, + { + "description": "array with two items is invalid", + "data": [ 1, "foo" ], + "valid": false + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "items and subitems", + "schema": { + "definitions": { + "item": { + "type": "array", + "additionalItems": false, + "items": [ + { "$ref": "#/definitions/sub-item" }, + { "$ref": "#/definitions/sub-item" } + ] + }, + "sub-item": { + "type": "object", + "required": ["foo"] + } + }, + "type": "array", + "additionalItems": false, + "items": [ + { "$ref": "#/definitions/item" }, + { "$ref": "#/definitions/item" }, + { "$ref": "#/definitions/item" } + ] + }, + "tests": [ + { + "description": "valid items", + "data": [ + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": true + }, + { + "description": "too many items", + "data": [ + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "too many sub-items", + "data": [ + [ {"foo": null}, {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "wrong item", + "data": [ + {"foo": null}, + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "wrong sub-item", + "data": [ + [ {}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "fewer items is valid", + "data": [ + [ {"foo": null} ], + [ {"foo": null} ] + ], + "valid": true + } + ] + }, + { + "description": "nested items", + "schema": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "tests": [ + { + "description": "valid nested array", + "data": [[[[1]], [[2],[3]]], [[[4], [5], [6]]]], + "valid": true + }, + { + "description": "nested array with invalid type", + "data": [[[["1"]], [[2],[3]]], [[[4], [5], [6]]]], + "valid": false + }, + { + "description": "not deep enough", + "data": [[[1], [2],[3]], [[4], [5], [6]]], + "valid": false + } + ] + }, + { + "description": "single-form items with null instance elements", + "schema": { + "items": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null elements", + "data": [ null ], + "valid": true + } + ] + }, + { + "description": "array-form items with null instance elements", + "schema": { + "items": [ + { + "type": "null" + } + ] + }, + "tests": [ + { + "description": "allows null elements", + "data": [ null ], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxItems.json new file mode 100644 index 000000000..f0c36ab28 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxItems.json @@ -0,0 +1,44 @@ +[ + { + "description": "maxItems validation", + "schema": {"maxItems": 2}, + "tests": [ + { + "description": "shorter is valid", + "data": [1], + "valid": true + }, + { + "description": "exact length is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "too long is invalid", + "data": [1, 2, 3], + "valid": false + }, + { + "description": "ignores non-arrays", + "data": "foobar", + "valid": true + } + ] + }, + { + "description": "maxItems validation with a decimal", + "schema": {"maxItems": 2.0}, + "tests": [ + { + "description": "shorter is valid", + "data": [1], + "valid": true + }, + { + "description": "too long is invalid", + "data": [1, 2, 3], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxLength.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxLength.json new file mode 100644 index 000000000..be60c5407 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxLength.json @@ -0,0 +1,49 @@ +[ + { + "description": "maxLength validation", + "schema": {"maxLength": 2}, + "tests": [ + { + "description": "shorter is valid", + "data": "f", + "valid": true + }, + { + "description": "exact length is valid", + "data": "fo", + "valid": true + }, + { + "description": "too long is invalid", + "data": "foo", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + }, + { + "description": "two graphemes is long enough", + "data": "\uD83D\uDCA9\uD83D\uDCA9", + "valid": true + } + ] + }, + { + "description": "maxLength validation with a decimal", + "schema": {"maxLength": 2.0}, + "tests": [ + { + "description": "shorter is valid", + "data": "f", + "valid": true + }, + { + "description": "too long is invalid", + "data": "foo", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxProperties.json new file mode 100644 index 000000000..acec14209 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxProperties.json @@ -0,0 +1,70 @@ +[ + { + "description": "maxProperties validation", + "schema": {"maxProperties": 2}, + "tests": [ + { + "description": "shorter is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "exact length is valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "too long is invalid", + "data": {"foo": 1, "bar": 2, "baz": 3}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [1, 2, 3], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "maxProperties validation with a decimal", + "schema": {"maxProperties": 2.0}, + "tests": [ + { + "description": "shorter is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "too long is invalid", + "data": {"foo": 1, "bar": 2, "baz": 3}, + "valid": false + } + ] + }, + { + "description": "maxProperties = 0 means the object is empty", + "schema": { "maxProperties": 0 }, + "tests": [ + { + "description": "no properties is valid", + "data": {}, + "valid": true + }, + { + "description": "one property is invalid", + "data": { "foo": 1 }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maximum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maximum.json new file mode 100644 index 000000000..c3e727069 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maximum.json @@ -0,0 +1,54 @@ +[ + { + "description": "maximum validation", + "schema": {"maximum": 3.0}, + "tests": [ + { + "description": "below the maximum is valid", + "data": 2.6, + "valid": true + }, + { + "description": "boundary point is valid", + "data": 3.0, + "valid": true + }, + { + "description": "above the maximum is invalid", + "data": 3.5, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + }, + { + "description": "maximum validation with unsigned integer", + "schema": {"maximum": 300}, + "tests": [ + { + "description": "below the maximum is valid", + "data": 299.97, + "valid": true + }, + { + "description": "boundary point integer is valid", + "data": 300, + "valid": true + }, + { + "description": "boundary point float is valid", + "data": 300.00, + "valid": true + }, + { + "description": "above the maximum is invalid", + "data": 300.5, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minItems.json new file mode 100644 index 000000000..d3b18720d --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minItems.json @@ -0,0 +1,44 @@ +[ + { + "description": "minItems validation", + "schema": {"minItems": 1}, + "tests": [ + { + "description": "longer is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "exact length is valid", + "data": [1], + "valid": true + }, + { + "description": "too short is invalid", + "data": [], + "valid": false + }, + { + "description": "ignores non-arrays", + "data": "", + "valid": true + } + ] + }, + { + "description": "minItems validation with a decimal", + "schema": {"minItems": 1.0}, + "tests": [ + { + "description": "longer is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "too short is invalid", + "data": [], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minLength.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minLength.json new file mode 100644 index 000000000..23c68fe3f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minLength.json @@ -0,0 +1,49 @@ +[ + { + "description": "minLength validation", + "schema": {"minLength": 2}, + "tests": [ + { + "description": "longer is valid", + "data": "foo", + "valid": true + }, + { + "description": "exact length is valid", + "data": "fo", + "valid": true + }, + { + "description": "too short is invalid", + "data": "f", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 1, + "valid": true + }, + { + "description": "one grapheme is not long enough", + "data": "\uD83D\uDCA9", + "valid": false + } + ] + }, + { + "description": "minLength validation with a decimal", + "schema": {"minLength": 2.0}, + "tests": [ + { + "description": "longer is valid", + "data": "foo", + "valid": true + }, + { + "description": "too short is invalid", + "data": "f", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minProperties.json new file mode 100644 index 000000000..416e02010 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minProperties.json @@ -0,0 +1,64 @@ +[ + { + "description": "minProperties validation", + "schema": {"minProperties": 1}, + "tests": [ + { + "description": "longer is valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "exact length is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "too short is invalid", + "data": {}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores strings", + "data": "", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + }, + { + "description": "ignores booleans", + "data": true, + "valid": true + } + ] + }, + { + "description": "minProperties validation with a decimal", + "schema": {"minProperties": 1.0}, + "tests": [ + { + "description": "longer is valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "too short is invalid", + "data": {}, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minimum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minimum.json new file mode 100644 index 000000000..21ae50e0e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minimum.json @@ -0,0 +1,69 @@ +[ + { + "description": "minimum validation", + "schema": {"minimum": 1.1}, + "tests": [ + { + "description": "above the minimum is valid", + "data": 2.6, + "valid": true + }, + { + "description": "boundary point is valid", + "data": 1.1, + "valid": true + }, + { + "description": "below the minimum is invalid", + "data": 0.6, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + }, + { + "description": "minimum validation with signed integer", + "schema": {"minimum": -2}, + "tests": [ + { + "description": "negative above the minimum is valid", + "data": -1, + "valid": true + }, + { + "description": "positive above the minimum is valid", + "data": 0, + "valid": true + }, + { + "description": "boundary point is valid", + "data": -2, + "valid": true + }, + { + "description": "boundary point with float is valid", + "data": -2.0, + "valid": true + }, + { + "description": "float below the minimum is invalid", + "data": -2.0001, + "valid": false + }, + { + "description": "int below the minimum is invalid", + "data": -3, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/multipleOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/multipleOf.json new file mode 100644 index 000000000..da0ce0830 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/multipleOf.json @@ -0,0 +1,87 @@ +[ + { + "description": "by int", + "schema": {"multipleOf": 2}, + "tests": [ + { + "description": "int by int", + "data": 10, + "valid": true + }, + { + "description": "int by int fail", + "data": 7, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "by number", + "schema": {"multipleOf": 1.5}, + "tests": [ + { + "description": "zero is multiple of anything", + "data": 0, + "valid": true + }, + { + "description": "4.5 is multiple of 1.5", + "data": 4.5, + "valid": true + }, + { + "description": "-4.5 is multiple of 1.5", + "data": -4.5, + "valid": true + }, + { + "description": "35 is not multiple of 1.5", + "data": 35, + "valid": false + } + ] + }, + { + "description": "by small number", + "schema": {"multipleOf": 0.0001}, + "tests": [ + { + "description": "0.0075 is multiple of 0.0001", + "data": 0.0075, + "valid": true + }, + { + "description": "0.00751 is not multiple of 0.0001", + "data": 0.00751, + "valid": false + } + ] + }, + { + "description": "float division = inf", + "schema": {"type": "integer", "multipleOf": 0.123456789}, + "tests": [ + { + "description": "always invalid, but naive implementations may raise an overflow error", + "data": 1e308, + "valid": false + } + ] + }, + { + "description": "small multiple of large integer", + "schema": {"type": "integer", "multipleOf": 1e-8}, + "tests": [ + { + "description": "any integer is a multiple of 1e-8", + "data": 12391239123, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/not.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/not.json new file mode 100644 index 000000000..b46c4ed05 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/not.json @@ -0,0 +1,259 @@ +[ + { + "description": "not", + "schema": { + "not": {"type": "integer"} + }, + "tests": [ + { + "description": "allowed", + "data": "foo", + "valid": true + }, + { + "description": "disallowed", + "data": 1, + "valid": false + } + ] + }, + { + "description": "not multiple types", + "schema": { + "not": {"type": ["integer", "boolean"]} + }, + "tests": [ + { + "description": "valid", + "data": "foo", + "valid": true + }, + { + "description": "mismatch", + "data": 1, + "valid": false + }, + { + "description": "other mismatch", + "data": true, + "valid": false + } + ] + }, + { + "description": "not more complex schema", + "schema": { + "not": { + "type": "object", + "properties": { + "foo": { + "type": "string" + } + } + } + }, + "tests": [ + { + "description": "match", + "data": 1, + "valid": true + }, + { + "description": "other match", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "mismatch", + "data": {"foo": "bar"}, + "valid": false + } + ] + }, + { + "description": "forbidden property", + "schema": { + "properties": { + "foo": { + "not": {} + } + } + }, + "tests": [ + { + "description": "property present", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "property absent", + "data": {"bar": 1, "baz": 2}, + "valid": true + } + ] + }, + { + "description": "forbid everything with empty schema", + "schema": { "not": {} }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "boolean true is invalid", + "data": true, + "valid": false + }, + { + "description": "boolean false is invalid", + "data": false, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "forbid everything with boolean schema true", + "schema": { "not": true }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "boolean true is invalid", + "data": true, + "valid": false + }, + { + "description": "boolean false is invalid", + "data": false, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "allow everything with boolean schema false", + "schema": { "not": false }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "boolean true is valid", + "data": true, + "valid": true + }, + { + "description": "boolean false is valid", + "data": false, + "valid": true + }, + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + }, + { + "description": "array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "double negation", + "schema": { "not": { "not": {} } }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/oneOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/oneOf.json new file mode 100644 index 000000000..c30a65c0d --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/oneOf.json @@ -0,0 +1,274 @@ +[ + { + "description": "oneOf", + "schema": { + "oneOf": [ + { + "type": "integer" + }, + { + "minimum": 2 + } + ] + }, + "tests": [ + { + "description": "first oneOf valid", + "data": 1, + "valid": true + }, + { + "description": "second oneOf valid", + "data": 2.5, + "valid": true + }, + { + "description": "both oneOf valid", + "data": 3, + "valid": false + }, + { + "description": "neither oneOf valid", + "data": 1.5, + "valid": false + } + ] + }, + { + "description": "oneOf with base schema", + "schema": { + "type": "string", + "oneOf" : [ + { + "minLength": 2 + }, + { + "maxLength": 4 + } + ] + }, + "tests": [ + { + "description": "mismatch base schema", + "data": 3, + "valid": false + }, + { + "description": "one oneOf valid", + "data": "foobar", + "valid": true + }, + { + "description": "both oneOf valid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf with boolean schemas, all true", + "schema": {"oneOf": [true, true, true]}, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf with boolean schemas, one true", + "schema": {"oneOf": [true, false, false]}, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "oneOf with boolean schemas, more than one true", + "schema": {"oneOf": [true, true, false]}, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf with boolean schemas, all false", + "schema": {"oneOf": [false, false, false]}, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf complex types", + "schema": { + "oneOf": [ + { + "properties": { + "bar": {"type": "integer"} + }, + "required": ["bar"] + }, + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "first oneOf valid (complex)", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "second oneOf valid (complex)", + "data": {"foo": "baz"}, + "valid": true + }, + { + "description": "both oneOf valid (complex)", + "data": {"foo": "baz", "bar": 2}, + "valid": false + }, + { + "description": "neither oneOf valid (complex)", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "oneOf with empty schema", + "schema": { + "oneOf": [ + { "type": "number" }, + {} + ] + }, + "tests": [ + { + "description": "one valid - valid", + "data": "foo", + "valid": true + }, + { + "description": "both valid - invalid", + "data": 123, + "valid": false + } + ] + }, + { + "description": "oneOf with required", + "schema": { + "type": "object", + "oneOf": [ + { "required": ["foo", "bar"] }, + { "required": ["foo", "baz"] } + ] + }, + "tests": [ + { + "description": "both invalid - invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "first valid - valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "second valid - valid", + "data": {"foo": 1, "baz": 3}, + "valid": true + }, + { + "description": "both valid - invalid", + "data": {"foo": 1, "bar": 2, "baz" : 3}, + "valid": false + } + ] + }, + { + "description": "oneOf with missing optional property", + "schema": { + "oneOf": [ + { + "properties": { + "bar": true, + "baz": true + }, + "required": ["bar"] + }, + { + "properties": { + "foo": true + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "first oneOf valid", + "data": {"bar": 8}, + "valid": true + }, + { + "description": "second oneOf valid", + "data": {"foo": "foo"}, + "valid": true + }, + { + "description": "both oneOf valid", + "data": {"foo": "foo", "bar": 8}, + "valid": false + }, + { + "description": "neither oneOf valid", + "data": {"baz": "quux"}, + "valid": false + } + ] + }, + { + "description": "nested oneOf, to check validation semantics", + "schema": { + "oneOf": [ + { + "oneOf": [ + { + "type": "null" + } + ] + } + ] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "anything non-null is invalid", + "data": 123, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/bignum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/bignum.json new file mode 100644 index 000000000..94b4a4e62 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/bignum.json @@ -0,0 +1,93 @@ +[ + { + "description": "integer", + "schema": { "type": "integer" }, + "tests": [ + { + "description": "a bignum is an integer", + "data": 12345678910111213141516171819202122232425262728293031, + "valid": true + }, + { + "description": "a negative bignum is an integer", + "data": -12345678910111213141516171819202122232425262728293031, + "valid": true + } + ] + }, + { + "description": "number", + "schema": { "type": "number" }, + "tests": [ + { + "description": "a bignum is a number", + "data": 98249283749234923498293171823948729348710298301928331, + "valid": true + }, + { + "description": "a negative bignum is a number", + "data": -98249283749234923498293171823948729348710298301928331, + "valid": true + } + ] + }, + { + "description": "string", + "schema": { "type": "string" }, + "tests": [ + { + "description": "a bignum is not a string", + "data": 98249283749234923498293171823948729348710298301928331, + "valid": false + } + ] + }, + { + "description": "maximum integer comparison", + "schema": { "maximum": 18446744073709551615 }, + "tests": [ + { + "description": "comparison works for high numbers", + "data": 18446744073709551600, + "valid": true + } + ] + }, + { + "description": "float comparison with high precision", + "schema": { + "exclusiveMaximum": 972783798187987123879878123.18878137 + }, + "tests": [ + { + "description": "comparison works for high numbers", + "data": 972783798187987123879878123.188781371, + "valid": false + } + ] + }, + { + "description": "minimum integer comparison", + "schema": { "minimum": -18446744073709551615 }, + "tests": [ + { + "description": "comparison works for very negative numbers", + "data": -18446744073709551600, + "valid": true + } + ] + }, + { + "description": "float comparison with high precision on negative numbers", + "schema": { + "exclusiveMinimum": -972783798187987123879878123.18878137 + }, + "tests": [ + { + "description": "comparison works for very negative numbers", + "data": -972783798187987123879878123.188781371, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/content.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/content.json new file mode 100644 index 000000000..3f5a7430b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/content.json @@ -0,0 +1,77 @@ +[ + { + "description": "validation of string-encoded content based on media type", + "schema": { + "contentMediaType": "application/json" + }, + "tests": [ + { + "description": "a valid JSON document", + "data": "{\"foo\": \"bar\"}", + "valid": true + }, + { + "description": "an invalid JSON document", + "data": "{:}", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + }, + { + "description": "validation of binary string-encoding", + "schema": { + "contentEncoding": "base64" + }, + "tests": [ + { + "description": "a valid base64 string", + "data": "eyJmb28iOiAiYmFyIn0K", + "valid": true + }, + { + "description": "an invalid base64 string (% is not a valid character)", + "data": "eyJmb28iOi%iYmFyIn0K", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + }, + { + "description": "validation of binary-encoded media type documents", + "schema": { + "contentMediaType": "application/json", + "contentEncoding": "base64" + }, + "tests": [ + { + "description": "a valid base64-encoded JSON document", + "data": "eyJmb28iOiAiYmFyIn0K", + "valid": true + }, + { + "description": "a validly-encoded invalid JSON document", + "data": "ezp9Cg==", + "valid": false + }, + { + "description": "an invalid base64 string that is valid JSON", + "data": "{}", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/cross-draft.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/cross-draft.json new file mode 100644 index 000000000..8ff537362 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/cross-draft.json @@ -0,0 +1,25 @@ +[ + { + "description": "refs to future drafts are processed as future drafts", + "schema": { + "type": "object", + "allOf": [ + { "properties": { "foo": true } }, + { "$ref": "http://localhost:1234/draft2019-09/dependentRequired.json" } + ] + }, + "tests": [ + { + "description": "missing bar is invalid", + "comment": "if the implementation is not processing the $ref as a 2019-09 schema, this test will fail", + "data": {"foo": "any value"}, + "valid": false + }, + { + "description": "present bar is valid", + "data": {"foo": "any value", "bar": "also any value"}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/ecmascript-regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/ecmascript-regex.json new file mode 100644 index 000000000..c4886aaa8 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/ecmascript-regex.json @@ -0,0 +1,552 @@ +[ + { + "description": "ECMA 262 regex $ does not match trailing newline", + "schema": { + "type": "string", + "pattern": "^abc$" + }, + "tests": [ + { + "description": "matches in Python, but not in ECMA 262", + "data": "abc\\n", + "valid": false + }, + { + "description": "matches", + "data": "abc", + "valid": true + } + ] + }, + { + "description": "ECMA 262 regex converts \\t to horizontal tab", + "schema": { + "type": "string", + "pattern": "^\\t$" + }, + "tests": [ + { + "description": "does not match", + "data": "\\t", + "valid": false + }, + { + "description": "matches", + "data": "\u0009", + "valid": true + } + ] + }, + { + "description": "ECMA 262 regex escapes control codes with \\c and upper letter", + "schema": { + "type": "string", + "pattern": "^\\cC$" + }, + "tests": [ + { + "description": "does not match", + "data": "\\cC", + "valid": false + }, + { + "description": "matches", + "data": "\u0003", + "valid": true + } + ] + }, + { + "description": "ECMA 262 regex escapes control codes with \\c and lower letter", + "schema": { + "type": "string", + "pattern": "^\\cc$" + }, + "tests": [ + { + "description": "does not match", + "data": "\\cc", + "valid": false + }, + { + "description": "matches", + "data": "\u0003", + "valid": true + } + ] + }, + { + "description": "ECMA 262 \\d matches ascii digits only", + "schema": { + "type": "string", + "pattern": "^\\d$" + }, + "tests": [ + { + "description": "ASCII zero matches", + "data": "0", + "valid": true + }, + { + "description": "NKO DIGIT ZERO does not match (unlike e.g. Python)", + "data": "߀", + "valid": false + }, + { + "description": "NKO DIGIT ZERO (as \\u escape) does not match", + "data": "\u07c0", + "valid": false + } + ] + }, + { + "description": "ECMA 262 \\D matches everything but ascii digits", + "schema": { + "type": "string", + "pattern": "^\\D$" + }, + "tests": [ + { + "description": "ASCII zero does not match", + "data": "0", + "valid": false + }, + { + "description": "NKO DIGIT ZERO matches (unlike e.g. Python)", + "data": "߀", + "valid": true + }, + { + "description": "NKO DIGIT ZERO (as \\u escape) matches", + "data": "\u07c0", + "valid": true + } + ] + }, + { + "description": "ECMA 262 \\w matches ascii letters only", + "schema": { + "type": "string", + "pattern": "^\\w$" + }, + "tests": [ + { + "description": "ASCII 'a' matches", + "data": "a", + "valid": true + }, + { + "description": "latin-1 e-acute does not match (unlike e.g. Python)", + "data": "é", + "valid": false + } + ] + }, + { + "description": "ECMA 262 \\W matches everything but ascii letters", + "schema": { + "type": "string", + "pattern": "^\\W$" + }, + "tests": [ + { + "description": "ASCII 'a' does not match", + "data": "a", + "valid": false + }, + { + "description": "latin-1 e-acute matches (unlike e.g. Python)", + "data": "é", + "valid": true + } + ] + }, + { + "description": "ECMA 262 \\s matches whitespace", + "schema": { + "type": "string", + "pattern": "^\\s$" + }, + "tests": [ + { + "description": "ASCII space matches", + "data": " ", + "valid": true + }, + { + "description": "Character tabulation matches", + "data": "\t", + "valid": true + }, + { + "description": "Line tabulation matches", + "data": "\u000b", + "valid": true + }, + { + "description": "Form feed matches", + "data": "\u000c", + "valid": true + }, + { + "description": "latin-1 non-breaking-space matches", + "data": "\u00a0", + "valid": true + }, + { + "description": "zero-width whitespace matches", + "data": "\ufeff", + "valid": true + }, + { + "description": "line feed matches (line terminator)", + "data": "\u000a", + "valid": true + }, + { + "description": "paragraph separator matches (line terminator)", + "data": "\u2029", + "valid": true + }, + { + "description": "EM SPACE matches (Space_Separator)", + "data": "\u2003", + "valid": true + }, + { + "description": "Non-whitespace control does not match", + "data": "\u0001", + "valid": false + }, + { + "description": "Non-whitespace does not match", + "data": "\u2013", + "valid": false + } + ] + }, + { + "description": "ECMA 262 \\S matches everything but whitespace", + "schema": { + "type": "string", + "pattern": "^\\S$" + }, + "tests": [ + { + "description": "ASCII space does not match", + "data": " ", + "valid": false + }, + { + "description": "Character tabulation does not match", + "data": "\t", + "valid": false + }, + { + "description": "Line tabulation does not match", + "data": "\u000b", + "valid": false + }, + { + "description": "Form feed does not match", + "data": "\u000c", + "valid": false + }, + { + "description": "latin-1 non-breaking-space does not match", + "data": "\u00a0", + "valid": false + }, + { + "description": "zero-width whitespace does not match", + "data": "\ufeff", + "valid": false + }, + { + "description": "line feed does not match (line terminator)", + "data": "\u000a", + "valid": false + }, + { + "description": "paragraph separator does not match (line terminator)", + "data": "\u2029", + "valid": false + }, + { + "description": "EM SPACE does not match (Space_Separator)", + "data": "\u2003", + "valid": false + }, + { + "description": "Non-whitespace control matches", + "data": "\u0001", + "valid": true + }, + { + "description": "Non-whitespace matches", + "data": "\u2013", + "valid": true + } + ] + }, + { + "description": "patterns always use unicode semantics with pattern", + "schema": { "pattern": "\\p{Letter}cole" }, + "tests": [ + { + "description": "ascii character in json string", + "data": "Les hivers de mon enfance etaient des saisons longues, longues. Nous vivions en trois lieux: l'ecole, l'eglise et la patinoire; mais la vraie vie etait sur la patinoire.", + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'école, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": true + }, + { + "description": "unicode character in hex format in string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'\u00e9cole, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": true + }, + { + "description": "unicode matching is case-sensitive", + "data": "LES HIVERS DE MON ENFANCE ÉTAIENT DES SAISONS LONGUES, LONGUES. NOUS VIVIONS EN TROIS LIEUX: L'ÉCOLE, L'ÉGLISE ET LA PATINOIRE; MAIS LA VRAIE VIE ÉTAIT SUR LA PATINOIRE.", + "valid": false + } + ] + }, + { + "description": "\\w in patterns matches [A-Za-z0-9_], not unicode letters", + "schema": { "pattern": "\\wcole" }, + "tests": [ + { + "description": "ascii character in json string", + "data": "Les hivers de mon enfance etaient des saisons longues, longues. Nous vivions en trois lieux: l'ecole, l'eglise et la patinoire; mais la vraie vie etait sur la patinoire.", + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'école, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'\u00e9cole, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "unicode matching is case-sensitive", + "data": "LES HIVERS DE MON ENFANCE ÉTAIENT DES SAISONS LONGUES, LONGUES. NOUS VIVIONS EN TROIS LIEUX: L'ÉCOLE, L'ÉGLISE ET LA PATINOIRE; MAIS LA VRAIE VIE ÉTAIT SUR LA PATINOIRE.", + "valid": false + } + ] + }, + { + "description": "pattern with ASCII ranges", + "schema": { "pattern": "[a-z]cole" }, + "tests": [ + { + "description": "literal unicode character in json string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'école, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'\u00e9cole, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "ascii characters match", + "data": "Les hivers de mon enfance etaient des saisons longues, longues. Nous vivions en trois lieux: l'ecole, l'eglise et la patinoire; mais la vraie vie etait sur la patinoire.", + "valid": true + } + ] + }, + { + "description": "\\d in pattern matches [0-9], not unicode digits", + "schema": { "pattern": "^\\d+$" }, + "tests": [ + { + "description": "ascii digits", + "data": "42", + "valid": true + }, + { + "description": "ascii non-digits", + "data": "-%#", + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": "৪২", + "valid": false + } + ] + }, + { + "description": "pattern with non-ASCII digits", + "schema": { "pattern": "^\\p{digit}+$" }, + "tests": [ + { + "description": "ascii digits", + "data": "42", + "valid": true + }, + { + "description": "ascii non-digits", + "data": "-%#", + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": "৪২", + "valid": true + } + ] + }, + { + "description": "patterns always use unicode semantics with patternProperties", + "schema": { + "type": "object", + "patternProperties": { + "\\p{Letter}cole": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii character in json string", + "data": { "l'ecole": "pas de vraie vie" }, + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": { "l'école": "pas de vraie vie" }, + "valid": true + }, + { + "description": "unicode character in hex format in string", + "data": { "l'\u00e9cole": "pas de vraie vie" }, + "valid": true + }, + { + "description": "unicode matching is case-sensitive", + "data": { "L'ÉCOLE": "PAS DE VRAIE VIE" }, + "valid": false + } + ] + }, + { + "description": "\\w in patternProperties matches [A-Za-z0-9_], not unicode letters", + "schema": { + "type": "object", + "patternProperties": { + "\\wcole": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii character in json string", + "data": { "l'ecole": "pas de vraie vie" }, + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": { "l'école": "pas de vraie vie" }, + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": { "l'\u00e9cole": "pas de vraie vie" }, + "valid": false + }, + { + "description": "unicode matching is case-sensitive", + "data": { "L'ÉCOLE": "PAS DE VRAIE VIE" }, + "valid": false + } + ] + }, + { + "description": "patternProperties with ASCII ranges", + "schema": { + "type": "object", + "patternProperties": { + "[a-z]cole": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "literal unicode character in json string", + "data": { "l'école": "pas de vraie vie" }, + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": { "l'\u00e9cole": "pas de vraie vie" }, + "valid": false + }, + { + "description": "ascii characters match", + "data": { "l'ecole": "pas de vraie vie" }, + "valid": true + } + ] + }, + { + "description": "\\d in patternProperties matches [0-9], not unicode digits", + "schema": { + "type": "object", + "patternProperties": { + "^\\d+$": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii digits", + "data": { "42": "life, the universe, and everything" }, + "valid": true + }, + { + "description": "ascii non-digits", + "data": { "-%#": "spending the year dead for tax reasons" }, + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": { "৪২": "khajit has wares if you have coin" }, + "valid": false + } + ] + }, + { + "description": "patternProperties with non-ASCII digits", + "schema": { + "type": "object", + "patternProperties": { + "^\\p{digit}+$": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii digits", + "data": { "42": "life, the universe, and everything" }, + "valid": true + }, + { + "description": "ascii non-digits", + "data": { "-%#": "spending the year dead for tax reasons" }, + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": { "৪২": "khajit has wares if you have coin" }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/float-overflow.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/float-overflow.json new file mode 100644 index 000000000..52ff9827c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/float-overflow.json @@ -0,0 +1,13 @@ +[ + { + "description": "all integers are multiples of 0.5, if overflow is handled", + "schema": {"type": "integer", "multipleOf": 0.5}, + "tests": [ + { + "description": "valid if optional overflow handling is implemented", + "data": 1e308, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date-time.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date-time.json new file mode 100644 index 000000000..07a4e9195 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date-time.json @@ -0,0 +1,247 @@ +[ + { + "description": "validation of date-time strings", + "schema": { "format": "date-time" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid date-time string", + "data": "1963-06-19T08:30:06.283185Z", + "valid": true + }, + { + "description": "a valid date-time string without second fraction", + "data": "1963-06-19T08:30:06Z", + "valid": true + }, + { + "description": "a valid date-time string with plus offset", + "data": "1937-01-01T12:00:27.87+00:20", + "valid": true + }, + { + "description": "a valid date-time string with minus offset", + "data": "1990-12-31T15:59:50.123-08:00", + "valid": true + }, + { + "description": "a valid date-time with a leap second, UTC", + "data": "1998-12-31T23:59:60Z", + "valid": true + }, + { + "description": "a valid date-time with a leap second, with minus offset", + "data": "1998-12-31T15:59:60.123-08:00", + "valid": true + }, + { + "description": "an invalid date-time past leap second, UTC", + "data": "1998-12-31T23:59:61Z", + "valid": false + }, + { + "description": "an invalid date-time with leap second on a wrong minute, UTC", + "data": "1998-12-31T23:58:60Z", + "valid": false + }, + { + "description": "an invalid date-time with leap second on a wrong hour, UTC", + "data": "1998-12-31T22:59:60Z", + "valid": false + }, + { + "description": "an invalid hour in date-time string", + "data": "1990-12-31T24:00:00Z", + "valid": false + }, + { + "description": "an invalid minute in date-time string", + "data": "1990-12-31T15:60:00Z", + "valid": false + }, + { + "description": "an invalid offset minute in date-time string", + "data": "1990-12-31T10:00:00+10:60", + "valid": false + }, + { + "description": "an invalid day in date-time string", + "data": "1990-02-31T15:59:59.123-08:00", + "valid": false + }, + { + "description": "an invalid offset in date-time string", + "data": "1990-12-31T15:59:59-24:00", + "valid": false + }, + { + "description": "an invalid closing Z after time-zone offset", + "data": "1963-06-19T08:30:06.28123+01:00Z", + "valid": false + }, + { + "description": "an invalid date-time string", + "data": "06/19/1963 08:30:06 PST", + "valid": false + }, + { + "description": "case-insensitive T and Z", + "data": "1963-06-19t08:30:06.283185z", + "valid": true + }, + { + "description": "only RFC3339 not all of ISO 8601 are valid", + "data": "2013-350T01:01:01", + "valid": false + }, + { + "description": "invalid non-padded month dates", + "data": "1963-6-19T08:30:06.283185Z", + "valid": false + }, + { + "description": "invalid non-padded day dates", + "data": "1963-06-1T08:30:06.283185Z", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4) in date portion", + "data": "1963-06-1৪T00:00:00Z", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4) in time portion", + "data": "1963-06-11T0৪:00:00Z", + "valid": false + }, + { + "description": "invalid extended year", + "data": "+11963-06-19T08:30:06.283185Z", + "valid": false + }, + { + "description": "a numeric offset without minutes is invalid", + "data": "1985-04-12T23:20:50+01", + "valid": false + }, + { + "description": "hour 24 is invalid even with a leap second", + "data": "2016-12-31T24:59:60+01:00", + "valid": false + }, + { + "description": "a second fraction of fifteen nines is valid", + "data": "1985-04-12T00:59:59.999999999999999Z", + "valid": true + }, + { + "description": "a trailing newline is invalid", + "data": "1985-04-12T23:20:50Z\n", + "valid": false + }, + { + "description": "an invalid date-time string without seconds", + "data": "1985-04-12T23:20Z", + "valid": false + }, + { + "description": "an invalid date-time string with trailing content after the offset", + "data": "1985-04-12T23:20:50Ztail", + "valid": false + }, + { + "description": "an invalid minute in date-time string, with a numeric offset", + "data": "1985-04-12T23:60:00+00:01", + "valid": false + }, + { + "description": "a valid date-time string with 28 days in February (normal)", + "data": "2021-02-28T00:00:00Z", + "valid": true + }, + { + "description": "an invalid date-time string with 30 days in February (leap)", + "data": "2020-02-30T00:00:00Z", + "valid": false + }, + { + "description": "2021 is not a leap year", + "data": "2021-02-29T00:00:00Z", + "valid": false + }, + { + "description": "2020 is a leap year", + "data": "2020-02-29T00:00:00Z", + "valid": true + }, + { + "description": "century year 0100 is not a leap year", + "data": "0100-02-29T00:00:00Z", + "valid": false + }, + { + "description": "century year 0400 is a leap year", + "data": "0400-02-29T00:00:00Z", + "valid": true + }, + { + "description": "century year 2100 is not a leap year", + "data": "2100-02-29T00:00:00Z", + "valid": false + }, + { + "description": "a second fraction with no digits", + "comment": "RFC 3339 section 5.6: time-secfrac = \".\" 1*DIGIT requires at least one digit", + "data": "1985-04-12T23:20:50.Z", + "valid": false + }, + { + "description": "a comma decimal separator with a time-offset", + "comment": "RFC 3339 section 5.6: the decimal separator in time-secfrac is \".\" - the ISO 8601 comma is not part of this profile", + "data": "1985-04-12T23:20:50,5Z", + "valid": false + }, + { + "description": "leading whitespace before a valid date-time", + "comment": "RFC 3339 section 5.6: no production of date-time emits whitespace", + "data": " 1985-04-12T23:20:50Z", + "valid": false + }, + { + "description": "trailing whitespace after a valid date-time", + "comment": "RFC 3339 section 5.6: no production of date-time emits whitespace", + "data": "1985-04-12T23:20:50Z ", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date.json new file mode 100644 index 000000000..4a3847b4f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date.json @@ -0,0 +1,420 @@ +[ + { + "description": "validation of date strings", + "schema": { "format": "date" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid date string", + "data": "1963-06-19", + "valid": true + }, + { + "description": "a valid date string with 31 days in January", + "data": "2020-01-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in January", + "data": "2020-01-32", + "valid": false + }, + { + "description": "a valid date string with 28 days in February (normal)", + "data": "2021-02-28", + "valid": true + }, + { + "description": "a invalid date string with 30 days in February (leap)", + "data": "2020-02-30", + "valid": false + }, + { + "description": "a valid date string with 31 days in March", + "data": "2020-03-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in March", + "data": "2020-03-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in April", + "data": "2020-04-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in April", + "data": "2020-04-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in May", + "data": "2020-05-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in May", + "data": "2020-05-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in June", + "data": "2020-06-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in June", + "data": "2020-06-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in July", + "data": "2020-07-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in July", + "data": "2020-07-32", + "valid": false + }, + { + "description": "a valid date string with 31 days in August", + "data": "2020-08-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in August", + "data": "2020-08-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in September", + "data": "2020-09-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in September", + "data": "2020-09-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in October", + "data": "2020-10-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in October", + "data": "2020-10-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in November", + "data": "2020-11-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in November", + "data": "2020-11-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in December", + "data": "2020-12-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in December", + "data": "2020-12-32", + "valid": false + }, + { + "description": "an invalid date string", + "data": "06/19/1963", + "valid": false + }, + { + "description": "only RFC3339 not all of ISO 8601 are valid", + "data": "2013-350", + "valid": false + }, + { + "description": "non-padded month dates are not valid", + "data": "1998-1-20", + "valid": false + }, + { + "description": "non-padded day dates are not valid", + "data": "1998-01-1", + "valid": false + }, + { + "description": "invalid month", + "data": "1998-13-01", + "valid": false + }, + { + "description": "2021 is not a leap year", + "data": "2021-02-29", + "valid": false + }, + { + "description": "2020 is a leap year", + "data": "2020-02-29", + "valid": true + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4)", + "data": "1963-06-1৪", + "valid": false + }, + { + "description": "invalid: non-ASCII Bengali digit in month field", + "data": "2020-0৪-01", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: YYYYMMDD without dashes (2023-03-28)", + "data": "20230328", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: week number implicit day of week (2023-01-02)", + "data": "2023-W01", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: week number with day of week (2023-03-28)", + "data": "2023-W13-2", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: week number rollover to next year (2023-01-01)", + "data": "2022W527", + "valid": false + }, + { + "description": "an invalid time string in date-time format", + "data": "2020-11-28T23:55:45Z", + "valid": false + }, + { + "description": "century year 0100 is not a leap year", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#appendix-C — 100 % 100 == 0 but 100 % 400 != 0", + "data": "0100-02-29", + "valid": false + }, + { + "description": "century year 0400 is a leap year", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#appendix-C — 400-year cycle boundary", + "data": "0400-02-29", + "valid": true + }, + { + "description": "century year 2100 is not a leap year", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#appendix-C — future century year", + "data": "2100-02-29", + "valid": false + }, + { + "description": "invalid: leading whitespace is not permitted", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — full-date grammar does not include whitespace", + "data": " 2024-01-15", + "valid": false + }, + { + "description": "invalid: trailing whitespace is not permitted", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — full-date grammar does not include whitespace", + "data": "2024-01-15 ", + "valid": false + }, + { + "description": "invalid: month 00 is not valid per date-month range 01-12", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — date-month = 2DIGIT ; 01-12", + "data": "2024-00-15", + "valid": false + }, + { + "description": "invalid: day 00 is not valid per date-mday minimum of 01", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — date-mday = 2DIGIT ; 01-28/29/30/31", + "data": "2024-01-00", + "valid": false + }, + { + "description": "invalid: empty string", + "data": "", + "valid": false + }, + { + "description": "invalid: embedded whitespace between year and month", + "data": "2020 -01-01", + "valid": false + }, + { + "description": "invalid: trailing character after valid full-date", + "data": "2020-01-01X", + "valid": false + }, + { + "description": "invalid: trailing Z after full-date", + "data": "2020-01-01Z", + "valid": false + }, + { + "description": "invalid: full-date followed by space and time component", + "data": "2020-01-01 00:00:00Z", + "valid": false + }, + { + "description": "valid: four-digit year 0001", + "data": "0001-01-01", + "valid": true + }, + { + "description": "invalid: two-digit year (N-2 digits)", + "data": "20-01-01", + "valid": false + }, + { + "description": "invalid: three-digit year (N-1 digits)", + "data": "998-01-01", + "valid": false + }, + { + "description": "invalid: five-digit year (N+1 digits)", + "data": "12020-01-01", + "valid": false + }, + { + "description": "invalid: positive sign prefix on year", + "data": "+2020-01-01", + "valid": false + }, + { + "description": "invalid: negative sign prefix on year", + "data": "-2020-01-01", + "valid": false + }, + { + "description": "invalid: non-ASCII Bengali digit in year field", + "data": "২020-01-01", + "valid": false + }, + { + "description": "invalid: alphabetic characters in year field", + "data": "YYYY-01-01", + "valid": false + }, + { + "description": "invalid: three-digit month (N+1 digits)", + "data": "2020-001-01", + "valid": false + }, + { + "description": "invalid: alphabetic characters in month field", + "data": "2020-MM-01", + "valid": false + }, + { + "description": "invalid: three-digit day (N+1 digits)", + "data": "2020-01-001", + "valid": false + }, + { + "description": "invalid: alphabetic characters in day field", + "data": "2020-01-DD", + "valid": false + }, + { + "description": "invalid: colon separators", + "data": "2020:01:01", + "valid": false + }, + { + "description": "invalid: dot separators", + "data": "2020.01.01", + "valid": false + }, + { + "description": "invalid: space separators", + "data": "2020 01 01", + "valid": false + }, + { + "description": "invalid: mixed slash and hyphen separators", + "data": "2020-01/01", + "valid": false + }, + { + "description": "invalid: duplicated first hyphen", + "data": "2020--01-01", + "valid": false + }, + { + "description": "invalid: duplicated second hyphen", + "data": "2020-01--01", + "valid": false + }, + { + "description": "valid: date inside the Julian to Gregorian reform gap", + "data": "1582-10-10", + "valid": true + }, + { + "description": "invalid: year field numerically larger than a 32-bit signed integer", + "data": "2147483648-01-01", + "valid": false + }, + { + "description": "invalid: character just above '9' as the second digit of the day", + "data": "2020-01-0:", + "valid": false + }, + { + "description": "invalid: non-ASCII digit leaving ten UTF-8 bytes but eight characters", + "data": "\u09e80-01-01", + "valid": false + }, + { + "description": "invalid: en dash separators instead of hyphen-minus", + "data": "2020\u201301\u201301", + "valid": false + }, + { + "description": "invalid: NUL character after an otherwise valid full-date", + "data": "2020-01-01\u0000", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ecmascript-regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ecmascript-regex.json new file mode 100644 index 000000000..bf3fae7f2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ecmascript-regex.json @@ -0,0 +1,98 @@ +[ + { + "description": "\\a is not an ECMA 262 control escape", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "when used as a pattern", + "data": "\\a", + "valid": false + } + ] + }, + { + "description": "Python-specific regular expression syntax is not valid ECMA 262", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "Python named group (?P...) is not ECMA 262", + "data": "(?Px)", + "valid": false + }, + { + "description": "Python named backreference (?P=name) is not ECMA 262", + "data": "(?Pa)(?P=n)", + "valid": false + }, + { + "description": "inline comment group (?#...) is not ECMA 262", + "data": "(?#comment)a", + "valid": false + } + ] + }, + { + "description": "global inline flag groups are not valid ECMA 262", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "a single global inline flag (?i)", + "data": "(?i)abc", + "valid": false + }, + { + "description": "multiple global inline flags (?ims)", + "data": "(?ims)abc", + "valid": false + } + ] + }, + { + "description": "ECMA 262 named groups and backreferences are valid", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "an ECMA 262 named group (?...)", + "data": "(?x)", + "valid": true + }, + { + "description": "an ECMA 262 named backreference \\k", + "data": "(?a)\\k", + "valid": true + } + ] + }, + { + "description": "ECMA 262 lookbehind is valid, including variable width", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "a variable-width lookbehind (ES2018)", + "data": "(?<=a+)b", + "valid": true + } + ] + }, + { + "description": "ECMA 262 character classes and escapes", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "an empty character class is valid ECMA 262", + "data": "[]", + "valid": true + }, + { + "description": "a negated empty character class is valid ECMA 262", + "data": "[^]", + "valid": true + }, + { + "description": "a control escape \\cA is valid ECMA 262", + "data": "\\cA", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/email.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/email.json new file mode 100644 index 000000000..6ef76f3c6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/email.json @@ -0,0 +1,365 @@ +[ + { + "description": "validation of e-mail addresses", + "schema": { "format": "email" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid e-mail address", + "data": "joe.bloggs@example.com", + "valid": true + }, + { + "description": "an invalid e-mail address", + "data": "2962", + "valid": false + }, + { + "description": "tilde in local part is valid", + "data": "te~st@example.com", + "valid": true + }, + { + "description": "tilde before local part is valid", + "data": "~test@example.com", + "valid": true + }, + { + "description": "tilde after local part is valid", + "data": "test~@example.com", + "valid": true + }, + { + "description": "a quoted string with a space in the local part is valid", + "data": "\"joe bloggs\"@example.com", + "valid": true + }, + { + "description": "a quoted string with a double dot in the local part is valid", + "data": "\"joe..bloggs\"@example.com", + "valid": true + }, + { + "description": "a quoted string with a @ in the local part is valid", + "data": "\"joe@bloggs\"@example.com", + "valid": true + }, + { + "description": "an IPv4-address-literal after the @ is valid", + "data": "joe.bloggs@[127.0.0.1]", + "valid": true + }, + { + "description": "an IPv6-address-literal after the @ is valid", + "data": "joe.bloggs@[IPv6:::1]", + "valid": true + }, + { + "description": "dot before local part is not valid", + "data": ".test@example.com", + "valid": false + }, + { + "description": "dot after local part is not valid", + "data": "test.@example.com", + "valid": false + }, + { + "description": "two separated dots inside local part are valid", + "data": "te.s.t@example.com", + "valid": true + }, + { + "description": "two subsequent dots inside local part are not valid", + "data": "te..st@example.com", + "valid": false + }, + { + "description": "an invalid domain", + "data": "joe.bloggs@invalid=domain.com", + "valid": false + }, + { + "description": "an invalid IPv4-address-literal", + "data": "joe.bloggs@[127.0.0.300]", + "valid": false + }, + { + "description": "two email addresses is not valid", + "data": "user1@oceania.org, user2@oceania.org", + "valid": false + }, + { + "description": "full \"From\" header is invalid", + "data": "\"Winston Smith\" (Records Department)", + "valid": false + }, + { + "description": "local part is required", + "data": "@example.com", + "valid": false + }, + { + "description": "domain is required", + "data": "joe.bloggs@", + "valid": false + }, + { + "description": "unquoted space in local part is invalid", + "data": "joe bloggs@example.com", + "valid": false + }, + { + "description": "a domain that looks like an IPv4 address without brackets is valid", + "data": "test@255.255.255.255", + "valid": true + }, + { + "description": "an empty quoted string in the local part is valid", + "data": "\"\"@iana.org", + "valid": true + }, + { + "description": "a quoted string containing only a space in the local part is valid", + "data": "\" \"@iana.org", + "valid": true + }, + { + "description": "consecutive hyphens inside a domain label are valid", + "data": "test@c--n.com", + "valid": true + }, + { + "description": "a domain label starting with a digit is valid", + "data": "test@123.com", + "valid": true + }, + { + "description": "a non-ASCII character in the local part is not valid", + "data": "a\u00e9@iana.org", + "valid": false + }, + { + "description": "a non-ASCII character in the domain is not valid", + "data": "a@\u00e9.org", + "valid": false + }, + { + "description": "a fullwidth commercial at is not a valid separator", + "data": "a\uff20iana.org", + "valid": false + }, + { + "description": "a non-ASCII character in a quoted pair is not valid", + "data": "\"test\\\u00a9\"@iana.org", + "valid": false + }, + { + "description": "a trailing line feed is not valid", + "data": "test@iana.org\n", + "valid": false + }, + { + "description": "a trailing carriage return is not valid", + "data": "test@iana.org\r", + "valid": false + }, + { + "description": "a trailing space is not valid", + "data": "a@iana.org ", + "valid": false + }, + { + "description": "a leading space is not valid", + "data": " a@iana.org", + "valid": false + }, + { + "description": "a tab in the local part is not valid", + "data": "a\tb@iana.org", + "valid": false + }, + { + "description": "a delete character in the local part is not valid", + "data": "\u007f@iana.org", + "valid": false + }, + { + "description": "a quoted string with no special characters in the local part is valid", + "data": "\"test\"@iana.org", + "valid": true + }, + { + "description": "a quoted pair in the local part is valid", + "data": "\"\\a\"@iana.org", + "valid": true + }, + { + "description": "an escaped double quote in the local part is valid", + "data": "\"\\\"\"@iana.org", + "valid": true + }, + { + "description": "an escaped backslash in the local part is valid", + "data": "\"\\\\\"@iana.org", + "valid": true + }, + { + "description": "an unescaped double quote in an unquoted local part is not valid", + "data": "test\"@iana.org", + "valid": false + }, + { + "description": "text after the closing double quote in the local part is not valid", + "data": "\"test\"test@iana.org", + "valid": false + }, + { + "description": "an unclosed double quote in the local part is not valid", + "data": "\"test@iana.org", + "valid": false + }, + { + "description": "two unquoted @ signs are not valid", + "data": "a@b@c.org", + "valid": false + }, + { + "description": "a backslash-escaped @ in an unquoted local part is not valid", + "data": "test\\@test@iana.org", + "valid": false + }, + { + "description": "an address literal in the local part position is not valid", + "data": "[1.2.3.4]@iana.org", + "valid": false + }, + { + "description": "an empty string is not valid", + "data": "", + "valid": false + }, + { + "description": "a domain label starting with a hyphen is not valid", + "data": "test@-iana.org", + "valid": false + }, + { + "description": "a domain label ending with a hyphen is not valid", + "data": "test@iana-.com", + "valid": false + }, + { + "description": "a domain starting with a dot is not valid", + "data": "test@.iana.org", + "valid": false + }, + { + "description": "a domain ending with a dot is not valid", + "data": "test@iana.org.", + "valid": false + }, + { + "description": "two subsequent dots inside the domain are not valid", + "data": "test@iana..com", + "valid": false + }, + { + "description": "a single-label domain is valid", + "data": "test@io", + "valid": true + }, + { + "description": "an unclosed address literal is not valid", + "data": "test@[1.2.3.4", + "valid": false + }, + { + "description": "empty brackets after the @ are not valid", + "data": "a@[]", + "valid": false + }, + { + "description": "an IPv4-address-literal with three octets is not valid", + "data": "a@[1.2.3]", + "valid": false + }, + { + "description": "an IPv4-address-literal with five octets is not valid", + "data": "a@[1.2.3.4.5]", + "valid": false + }, + { + "description": "bracketed content that is not an address is not valid", + "data": "test@[RFC-5322-domain-literal]", + "valid": false + }, + { + "description": "a domain label before the opening bracket is not valid", + "data": "test@a[255.255.255.255]", + "valid": false + }, + { + "description": "a lowercase IPv6 tag in an address literal is valid", + "data": "a@[ipv6:::1]", + "valid": true + }, + { + "description": "a comma in an unquoted local part is not valid", + "data": "a,b@iana.org", + "valid": false + }, + { + "description": "a semicolon in an unquoted local part is not valid", + "data": "a;b@iana.org", + "valid": false + }, + { + "description": "a colon in an unquoted local part is not valid", + "data": "a:b@iana.org", + "valid": false + }, + { + "description": "a parenthesis in an unquoted local part is not valid", + "comment": "a comment is RFC 5322 syntax, which the RFC 5321 Mailbox grammar does not have", + "data": "test(comment)@iana.org", + "valid": false + }, + { + "description": "a parenthesis in the domain is not valid", + "comment": "a comment is RFC 5322 syntax, which the RFC 5321 Mailbox grammar does not have", + "data": "a@b(c).org", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/hostname.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/hostname.json new file mode 100644 index 000000000..bd5dcdea2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/hostname.json @@ -0,0 +1,380 @@ +[ + { + "description": "validation of host names", + "schema": { "format": "hostname" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid host name", + "data": "www.example.com", + "valid": true + }, + { + "description": "single label", + "data": "hostname", + "valid": true + }, + { + "description": "single label with digits", + "data": "h0stn4me", + "valid": true + }, + { + "description": "single label starting with digit", + "data": "1host", + "valid": true + }, + { + "description": "single label ending with digit", + "data": "hostnam3", + "valid": true + }, + { + "description": "empty string", + "data": "", + "valid": false + }, + { + "description": "single dot", + "data": ".", + "valid": false + }, + { + "description": "leading dot", + "data": ".example", + "valid": false + }, + { + "description": "trailing dot", + "data": "example.", + "valid": false + }, + { + "description": "IDN label separator", + "data": "example\uff0ecom", + "valid": false + }, + { + "description": "single label with hyphen", + "data": "host-name", + "valid": true + }, + { + "description": "starts with hyphen", + "data": "-hostname", + "valid": false + }, + { + "description": "ends with hyphen", + "data": "hostname-", + "valid": false + }, + { + "description": "contains underscore", + "data": "host_name", + "valid": false + }, + { + "description": "exceeds maximum overall length (256)", + "data": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.com", + "valid": false + }, + { + "description": "maximum label length (63)", + "data": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.com", + "valid": true + }, + { + "description": "exceeds maximum label length (63)", + "data": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijkl.com", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "example.com\n", + "valid": false + }, + { + "description": "leading space is invalid", + "data": " example.com", + "valid": false + }, + { + "description": "space inside a label is invalid", + "data": "ex ample.com", + "valid": false + }, + { + "description": "invalid non-ASCII KELVIN SIGN (U+212A)", + "data": "\u212Aelvin.example.com", + "valid": false + }, + { + "description": "consecutive hyphens inside a label", + "data": "a--b.com", + "valid": true + } + ] + }, + { + "description": "validation of A-label (punycode) host names", + "schema": { "format": "hostname" }, + "tests": [ + { + "description": "invalid Punycode", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.4 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "xn--X", + "valid": false + }, + { + "description": "a valid host name (example.test in Hangul)", + "data": "xn--9n2bp8q.xn--9t4b11yi5a", + "valid": true + }, + { + "description": "contains illegal char U+302E Hangul single dot tone mark", + "data": "xn--07jt112bpxg.xn--9t4b11yi5a", + "valid": false + }, + { + "description": "Begins with a Spacing Combining Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "xn--hello-txk", + "valid": false + }, + { + "description": "Begins with a Nonspacing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "xn--hello-zed", + "valid": false + }, + { + "description": "Begins with an Enclosing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "xn--hello-6bf", + "valid": false + }, + { + "description": "Exceptions that are PVALID, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "xn--zca29lwxobi7a", + "valid": true + }, + { + "description": "Exceptions that are PVALID, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "xn--qmbc", + "valid": true + }, + { + "description": "Exceptions that are DISALLOWED, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "xn--chb89f", + "valid": false + }, + { + "description": "Exceptions that are DISALLOWED, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6 Note: The two combining marks (U+302E and U+302F) are in the middle and not at the start", + "data": "xn--07jceefgh4c", + "valid": false + }, + { + "description": "MIDDLE DOT with no preceding 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--al-0ea", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing preceding", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--l-fda", + "valid": false + }, + { + "description": "MIDDLE DOT with no following 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--la-0ea", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing following", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--l-gda", + "valid": false + }, + { + "description": "MIDDLE DOT with surrounding 'l's", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--ll-0ea", + "valid": true + }, + { + "description": "Greek KERAIA not followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "xn--S-jib3p", + "valid": false + }, + { + "description": "Greek KERAIA not followed by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "xn--wva3j", + "valid": false + }, + { + "description": "Greek KERAIA followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "xn--wva3je", + "valid": true + }, + { + "description": "Hebrew GERESH not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "xn--A-2hc5h", + "valid": false + }, + { + "description": "Hebrew GERESH not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "xn--5db1e", + "valid": false + }, + { + "description": "Hebrew GERESH preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "xn--4dbc5h", + "valid": true + }, + { + "description": "Hebrew GERSHAYIM not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "xn--A-2hc8h", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "xn--5db3e", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "xn--4dbc8h", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with no Hiragana, Katakana, or Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--defabc-k64e", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with no other characters", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--vek", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with Hiragana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--k8j5u", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Katakana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--bck0j", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--vek778f", + "valid": true + }, + { + "description": "Arabic-Indic digits mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "xn--ngb6iyr", + "valid": false + }, + { + "description": "Arabic-Indic digits not mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "xn--ngba1o", + "valid": true + }, + { + "description": "Extended Arabic-Indic digits not mixed with Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.9", + "data": "xn--0-gyc", + "valid": true + }, + { + "description": "ZERO WIDTH JOINER not preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "xn--11b2er09f", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "xn--02b508i", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "xn--11b2ezcw70k", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1", + "data": "xn--11b2ezcs70k", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER not preceded by Virama but matches regexp", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1 https://www.w3.org/TR/alreq/#h_disjoining_enforcement", + "data": "xn--ngba5hb2804a", + "valid": true + }, + { + "description": "contains \"--\" in the 3rd and 4th position", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "XN--aa---o47jg78q", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-email.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-email.json new file mode 100644 index 000000000..c88bd9fc5 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-email.json @@ -0,0 +1,137 @@ +[ + { + "description": "validation of an internationalized e-mail addresses", + "schema": { "format": "idn-email" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid idn e-mail (example@example.test in Hangul)", + "data": "실례@실례.테스트", + "valid": true + }, + { + "description": "an invalid e-mail/idn e-mail address", + "data": "2962", + "valid": false + }, + { + "description": "a valid e-mail address", + "data": "joe.bloggs@example.com", + "valid": true + }, + { + "description": "a non-ASCII local part with an ASCII domain is valid", + "data": "δοκιμή@example.com", + "valid": true + }, + { + "description": "a non-ASCII quoted local part is valid", + "data": "\"δοκιμή\"@example.com", + "valid": true + }, + { + "description": "a domain label that is not in Unicode NFC is valid", + "data": "user@cafe\u0301.com", + "valid": true + }, + { + "description": "a local part that is not in Unicode NFC is valid", + "data": "cafe\u0301@example.com", + "valid": true + }, + { + "description": "a C1 control in the local part is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc6531#section-3.3 atext =/ UTF8-non-ascii; https://www.rfc-editor.org/rfc/rfc3629#section-4 UTF8-2 admits U+0085", + "data": "\u0085@example.com", + "valid": true + }, + { + "description": "a noncharacter in the local part is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc6531#section-3.3 atext =/ UTF8-non-ascii; https://www.rfc-editor.org/rfc/rfc3629#section-4 UTF8-3 admits U+FFFF", + "data": "\uffff@example.com", + "valid": true + }, + { + "description": "a local part at the 64-octet limit is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.5.3.1 every implementation must be able to receive a local part of 64 octets", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@example.com", + "valid": true + }, + { + "description": "a fullwidth commercial at is not a local-part separator", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Mailbox = Local-part \"@\" ( Domain / address-literal ) splits on U+0040, and any IDN mapping applies only to the domain that split produces, so U+FF20 cannot become the delimiter", + "data": "user\uff20example.com", + "valid": false + }, + { + "description": "a local part with a supplementary-plane character is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc6531#section-3.3 atext =/ UTF8-non-ascii, and https://www.rfc-editor.org/rfc/rfc3629#section-4 UTF8-4 covers U+10000 and above", + "data": "\ud835\udd4f@example.com", + "valid": true + }, + { + "description": "a domain label starting with a hyphen is not valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 sub-domain = Let-dig [Ldh-str], so a label starts with a letter or digit; RFC 6531 section 3.3 only adds sub-domain =/ U-label", + "data": "δοκιμή@-example.com", + "valid": false + }, + { + "description": "a domain label ending with a hyphen is not valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Ldh-str = *( ALPHA / DIGIT / \"-\" ) Let-dig, so a label ends with a letter or digit", + "data": "δοκιμή@example-.com", + "valid": false + }, + { + "description": "a domain starting with a dot is not valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Domain = sub-domain *(\".\" sub-domain) admits no empty sub-domain", + "data": "δοκιμή@.example.com", + "valid": false + }, + { + "description": "a domain ending with a dot is not valid", + "data": "δοκιμή@example.com.", + "valid": false + }, + { + "description": "two subsequent dots inside the domain are not valid", + "data": "δοκιμή@example..com", + "valid": false + }, + { + "description": "domain is required", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Mailbox = Local-part \"@\" ( Domain / address-literal )", + "data": "δοκιμή@", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-hostname.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-hostname.json new file mode 100644 index 000000000..13d223d24 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-hostname.json @@ -0,0 +1,582 @@ +[ + { + "description": "validation of internationalized host names", + "schema": { "format": "idn-hostname" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid host name (example.test in Hangul)", + "data": "실례.테스트", + "valid": true + }, + { + "description": "illegal first char U+302E Hangul single dot tone mark", + "data": "〮실례.테스트", + "valid": false + }, + { + "description": "contains illegal char U+302E Hangul single dot tone mark", + "data": "실〮례.테스트", + "valid": false + }, + { + "description": "a single label of 63 characters is valid", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "valid": true + }, + { + "description": "a single label of 64 characters is too long", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "valid": false + }, + { + "description": "invalid label, correct Punycode", + "comment": "https://tools.ietf.org/html/rfc5890#section-2.3.2.1 https://tools.ietf.org/html/rfc5891#section-4.4 https://tools.ietf.org/html/rfc3492#section-7.1", + "data": "-> $1.00 <--", + "valid": false + }, + { + "description": "valid Chinese Punycode", + "comment": "https://tools.ietf.org/html/rfc5890#section-2.3.2.1 https://tools.ietf.org/html/rfc5891#section-4.4", + "data": "xn--ihqwcrb4cv8a8dqg056pqjye", + "valid": true + }, + { + "description": "invalid Punycode", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.4 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "xn--X", + "valid": false + }, + { + "description": "U-label contains \"--\" in the 3rd and 4th position", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "XN--aa---o47jg78q", + "valid": false + }, + { + "description": "U-label starts with a dash", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1", + "data": "-hello", + "valid": false + }, + { + "description": "U-label ends with a dash", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1", + "data": "hello-", + "valid": false + }, + { + "description": "U-label starts and ends with a dash", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1", + "data": "-hello-", + "valid": false + }, + { + "description": "Begins with a Spacing Combining Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "\u0903hello", + "valid": false + }, + { + "description": "Begins with a Nonspacing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "\u0300hello", + "valid": false + }, + { + "description": "Begins with an Enclosing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "\u0488hello", + "valid": false + }, + { + "description": "Exceptions that are PVALID, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "\u00df\u03c2\u0f0b\u3007", + "valid": true + }, + { + "description": "Exceptions that are PVALID, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "\u06fd\u06fe", + "valid": true + }, + { + "description": "Exceptions that are DISALLOWED, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "\u0640\u07fa", + "valid": false + }, + { + "description": "Exceptions that are DISALLOWED, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6 Note: The two combining marks (U+302E and U+302F) are in the middle and not at the start", + "data": "\u3031\u3032\u3033\u3034\u3035\u302e\u302f\u303b", + "valid": false + }, + { + "description": "MIDDLE DOT with no preceding 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "a\u00b7l", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing preceding", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "\u00b7l", + "valid": false + }, + { + "description": "MIDDLE DOT with no following 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "l\u00b7a", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing following", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "l\u00b7", + "valid": false + }, + { + "description": "MIDDLE DOT with surrounding 'l's", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "l\u00b7l", + "valid": true + }, + { + "description": "Greek KERAIA not followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "\u03b1\u0375S", + "valid": false + }, + { + "description": "Greek KERAIA not followed by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "\u03b1\u0375", + "valid": false + }, + { + "description": "Greek KERAIA followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "\u03b1\u0375\u03b2", + "valid": true + }, + { + "description": "Hebrew GERESH not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "A\u05f3\u05d1", + "valid": false + }, + { + "description": "Hebrew GERESH not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "\u05f3\u05d1", + "valid": false + }, + { + "description": "Hebrew GERESH preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "\u05d0\u05f3\u05d1", + "valid": true + }, + { + "description": "Hebrew GERSHAYIM not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "A\u05f4\u05d1", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "\u05f4\u05d1", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "\u05d0\u05f4\u05d1", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with no Hiragana, Katakana, or Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "def\u30fbabc", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with no other characters", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with Hiragana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb\u3041", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Katakana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb\u30a1", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb\u4e08", + "valid": true + }, + { + "description": "Arabic-Indic digits mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "\u0628\u0660\u06f0", + "valid": false + }, + { + "description": "Arabic-Indic digits not mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "\u0628\u0660\u0628", + "valid": true + }, + { + "description": "Extended Arabic-Indic digits not mixed with Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.9", + "data": "\u06f00", + "valid": true + }, + { + "description": "ZERO WIDTH JOINER not preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "\u0915\u200d\u0937", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "\u200d\u0937", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "\u0915\u094d\u200d\u0937", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1", + "data": "\u0915\u094d\u200c\u0937", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER not preceded by Virama but matches regexp", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1 https://www.w3.org/TR/alreq/#h_disjoining_enforcement", + "data": "\u0628\u064a\u200c\u0628\u064a", + "valid": true + }, + { + "description": "single label", + "data": "hostname", + "valid": true + }, + { + "description": "single label with hyphen", + "data": "host-name", + "valid": true + }, + { + "description": "single label with digits", + "data": "h0stn4me", + "valid": true + }, + { + "description": "single label starting with digit", + "data": "1host", + "valid": true + }, + { + "description": "single label ending with digit", + "data": "hostnam3", + "valid": true + }, + { + "description": "empty string", + "data": "", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "example.com\n", + "valid": false + }, + { + "description": "leading space is invalid", + "data": " example.com", + "valid": false + }, + { + "description": "space inside a label is invalid", + "data": "ex ample.com", + "valid": false + }, + { + "description": "fullwidth digits are mapped to ASCII digits", + "comment": "IdnaMappingTable.txt maps U+FF11 to 0031, so this label maps to 123", + "data": "\uff11\uff12\uff13", + "valid": true + }, + { + "description": "a zero width space is ignored by the mapping", + "comment": "IdnaMappingTable.txt lists U+200B as ignored, so this label maps to ab", + "data": "a\u200bb", + "valid": true + }, + { + "description": "an A-label with an uppercase Punycode body is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5891#section-5.3 lowercases the A-label before the round-trip test, and RFC 3492 section 5 gives A-Z and a-z the same digit-values", + "data": "xn--NXASMQ6B", + "valid": true + }, + { + "description": "the two Arabic-Indic digit blocks may appear in different labels", + "comment": "https://www.rfc-editor.org/rfc/rfc5892#appendix-A.8 scopes the rule to \"this label\", so one block per label is valid", + "data": "\u0628\u0660.\u0628\u06f0", + "valid": true + }, + { + "description": "a label of only Arabic-Indic digits is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2 condition 1 allows only L, R or AL as the first character, and an Arabic-Indic digit is AN", + "data": "\u0660\u0661", + "valid": false + }, + { + "description": "a non-NFC U-label is valid after UTS 46 normalisation", + "comment": "UTS 46 section 4 normalises to NFC before validating, so the decomposed form is accepted", + "data": "cafe\u0301.com", + "valid": true + }, + { + "description": "a label of 63 fullwidth characters is valid after mapping", + "comment": "UTS 46 maps each U+FF41 to an ASCII letter, so this label is 63 octets in its mapped form", + "data": "\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41", + "valid": true + }, + { + "description": "an ACE prefix produced by the mapping step is decoded as an A-label", + "comment": "UTS 46 maps U+FF58 U+FF4E to \"xn\", so the mapped label is the A-label xn--nxasmq6b", + "data": "\uff58\uff4e--nxasmq6b", + "valid": true + }, + { + "description": "zero width non-joiner must pass at every occurrence", + "comment": "https://www.rfc-editor.org/rfc/rfc5892#appendix-A.1", + "data": "\u0915\u094d\u200c\u0937x\u200cy", + "valid": false + }, + { + "description": "Bidi domain name with a digit-first label is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2 a label in a Bidi domain name must start with an L, R or AL character", + "data": "0a.\u05d0", + "valid": false + }, + { + "description": "label starting with a digit before a right-to-left letter is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2", + "data": "0\u0627", + "valid": false + }, + { + "description": "left-to-right label containing a right-to-left letter is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2", + "data": "a\u05d0", + "valid": false + }, + { + "description": "right-to-left label mixing both digit types is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2", + "data": "\u05d00\u0660", + "valid": false + }, + { + "description": "A-label that decodes to a disallowed code point is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5890#section-2.3.2.1 https://www.rfc-editor.org/rfc/rfc5892#section-2.6", + "data": "xn--7a", + "valid": false + }, + { + "description": "A-label that decodes to a Bidi rule violation is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5890#section-2.3.2.1 https://www.rfc-editor.org/rfc/rfc5893", + "data": "xn--0ca24w", + "valid": false + }, + { + "description": "a U-label whose A-label form is longer than 63 octets is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5891#section-4.2.4 the 63-octet limit is on the A-label form, not the code point count", + "data": "\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc", + "valid": false + }, + { + "description": "empty label between two dots is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc1034#section-3.1", + "data": "a..b", + "valid": false + }, + { + "description": "a name longer than 253 characters is invalid", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "valid": false + }, + { + "description": "A-label that decodes to only ASCII is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5890#section-2.3.2.1 a U-label must have at least one non-ASCII character", + "data": "xn--example-", + "valid": false + }, + { + "description": "non-canonical Punycode that does not re-encode to itself is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5891#section-5.4 a label decoded from Punycode must be identical to the original A-label", + "data": "xn---9uc", + "valid": false + } + ] + }, + { + "description": "validation of separators in internationalized host names", + "specification": [ + {"rfc3490": "3.1", "quote": "Whenever dots are used as label separators, the following characters MUST be recognized as dots: U+002E (full stop), U+3002 (ideographic full stop), U+FF0E (fullwidth full stop), U+FF61(halfwidth ideographic full stop)"} + ], + "schema": { "format": "idn-hostname" }, + "tests": [ + { + "description": "single dot", + "data": ".", + "valid": false + }, + { + "description": "single ideographic full stop", + "data": "\u3002", + "valid": false + }, + { + "description": "single fullwidth full stop", + "data": "\uff0e", + "valid": false + }, + { + "description": "single halfwidth ideographic full stop", + "data": "\uff61", + "valid": false + }, + { + "description": "dot as label separator", + "data": "a.b", + "valid": true + }, + { + "description": "ideographic full stop as label separator", + "data": "a\u3002b", + "valid": true + }, + { + "description": "fullwidth full stop as label separator", + "data": "a\uff0eb", + "valid": true + }, + { + "description": "halfwidth ideographic full stop as label separator", + "data": "a\uff61b", + "valid": true + }, + { + "description": "leading dot", + "data": ".example", + "valid": false + }, + { + "description": "leading ideographic full stop", + "data": "\u3002example", + "valid": false + }, + { + "description": "leading fullwidth full stop", + "data": "\uff0eexample", + "valid": false + }, + { + "description": "leading halfwidth ideographic full stop", + "data": "\uff61example", + "valid": false + }, + { + "description": "trailing dot", + "data": "example.", + "valid": false + }, + { + "description": "trailing ideographic full stop", + "data": "example\u3002", + "valid": false + }, + { + "description": "trailing fullwidth full stop", + "data": "example\uff0e", + "valid": false + }, + { + "description": "trailing halfwidth ideographic full stop", + "data": "example\uff61", + "valid": false + }, + { + "description": "label too long if separator ignored (full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα.com", + "valid": true + }, + { + "description": "label too long if separator ignored (ideographic full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα\u3002com", + "valid": true + }, + { + "description": "label too long if separator ignored (fullwidth full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα\uff0ecom", + "valid": true + }, + { + "description": "label too long if separator ignored (halfwidth ideographic full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα\uff61com", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv4.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv4.json new file mode 100644 index 000000000..d56d62b33 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv4.json @@ -0,0 +1,214 @@ +[ + { + "description": "validation of IP addresses", + "comment": "RFC 2673, Section 3.2: dotted-quad = decbyte \".\" decbyte \".\" decbyte \".\" decbyte. A 'decbyte' (1*3DIGIT) restricts semantic values to 0-255, allows leading zeros, and strictly forbids symbols, alpha/hex, whitespace, and non-ASCII characters.", + "schema": { "format": "ipv4" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IP address", + "data": "192.168.0.1", + "valid": true + }, + { + "description": "an IP address with too many components", + "data": "127.0.0.0.1", + "valid": false + }, + { + "description": "an IP address with out-of-range values", + "data": "256.256.256.256", + "valid": false + }, + { + "description": "an IP address without 4 components", + "data": "127.0", + "valid": false + }, + { + "description": "a 2-part address resolving to a routable IP (inet_aton shorthand)", + "data": "127.1", + "valid": false + }, + { + "description": "a 3-part address resolving to a routable IP (inet_aton shorthand)", + "data": "127.0.1", + "valid": false + }, + { + "description": "an IP address as an integer", + "data": "0x7f000001", + "valid": false + }, + { + "description": "an IP address as an integer (decimal)", + "data": "2130706433", + "valid": false + }, + { + "description": "invalid non-ASCII '২' (a Bengali 2)", + "data": "1২7.0.0.1", + "valid": false + }, + { + "description": "invalid fullwidth digits (non-ASCII)", + "data": "192.168.1.1", + "valid": false + }, + { + "description": "invalid mathematical bold digits (non-ASCII)", + "data": "𝟏𝟗𝟐.𝟏𝟔𝟖.𝟏.𝟏", + "valid": false + }, + { + "description": "netmask is not a part of ipv4 address", + "data": "192.168.1.0/24", + "valid": false + }, + { + "description": "leading whitespace is invalid", + "data": " 192.168.0.1", + "valid": false + }, + { + "description": "trailing whitespace is invalid", + "data": "192.168.0.1 ", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "192.168.0.1\n", + "valid": false + }, + { + "description": "hexadecimal notation is invalid", + "data": "0x7f.0.0.1", + "valid": false + }, + { + "description": "octal notation explicit is invalid", + "data": "0o10.0.0.1", + "valid": false + }, + { + "description": "empty part (double dot) is invalid", + "data": "192.168..1", + "valid": false + }, + { + "description": "leading dot is invalid", + "data": ".192.168.0.1", + "valid": false + }, + { + "description": "trailing dot is invalid", + "data": "192.168.0.1.", + "valid": false + }, + { + "description": "minimum valid IPv4 address", + "data": "0.0.0.0", + "valid": true + }, + { + "description": "maximum valid IPv4 address", + "data": "255.255.255.255", + "valid": true + }, + { + "description": "valid IPv4 with an octet in the 200 to 249 range", + "data": "200.0.0.0", + "valid": true + }, + { + "description": "valid IPv4 with two-digit octets", + "data": "10.20.30.40", + "valid": true + }, + { + "description": "empty string is invalid", + "data": "", + "valid": false + }, + { + "description": "plus sign is invalid", + "data": "+1.2.3.4", + "valid": false + }, + { + "description": "negative sign is invalid", + "data": "-1.2.3.4", + "valid": false + }, + { + "description": "exponential notation is invalid", + "data": "1e2.0.0.1", + "valid": false + }, + { + "description": "alpha characters are invalid", + "data": "192.168.a.1", + "valid": false + }, + { + "description": "internal whitespace is invalid", + "data": "192. 168.0.1", + "valid": false + }, + { + "description": "tab character is invalid", + "data": "192.168.0.1\t", + "valid": false + }, + { + "description": "additional content after an embedded NUL byte", + "data": "192.168.0.1\u0000.evil.com", + "valid": false + }, + { + "description": "with port number is invalid", + "data": "192.168.0.1:80", + "valid": false + }, + { + "description": "an IPv4-mapped IPv6 address is invalid", + "data": "::ffff:192.168.0.1", + "valid": false + }, + { + "description": "single octet out of range in last position", + "data": "192.168.0.256", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv6.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv6.json new file mode 100644 index 000000000..7f4674be4 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv6.json @@ -0,0 +1,263 @@ +[ + { + "description": "validation of IPv6 addresses", + "schema": { "format": "ipv6" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IPv6 address", + "data": "::1", + "valid": true + }, + { + "description": "a group with 5 hex digits is invalid", + "data": "12345::", + "valid": false + }, + { + "description": "trailing 4 hex symbols is valid", + "data": "::abef", + "valid": true + }, + { + "description": "trailing 5 hex symbols is invalid", + "data": "::abcef", + "valid": false + }, + { + "description": "an IPv6 address with too many components", + "data": "1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1", + "valid": false + }, + { + "description": "an IPv6 address containing illegal characters", + "data": "::laptop", + "valid": false + }, + { + "description": "no digits is valid", + "data": "::", + "valid": true + }, + { + "description": "leading colons is valid", + "data": "::42:ff:1", + "valid": true + }, + { + "description": "trailing colons is valid", + "data": "d6::", + "valid": true + }, + { + "description": "missing leading octet is invalid", + "data": ":2:3:4:5:6:7:8", + "valid": false + }, + { + "description": "missing trailing octet is invalid", + "data": "1:2:3:4:5:6:7:", + "valid": false + }, + { + "description": "missing leading octet with omitted octets later", + "data": ":2:3:4::8", + "valid": false + }, + { + "description": "single set of double colons in the middle is valid", + "data": "1:d6::42", + "valid": true + }, + { + "description": "two sets of double colons is invalid", + "data": "1::d6::42", + "valid": false + }, + { + "description": "mixed format with the ipv4 section as decimal octets", + "data": "1::d6:192.168.0.1", + "valid": true + }, + { + "description": "mixed format with double colons between the sections", + "data": "1:2::192.168.0.1", + "valid": true + }, + { + "description": "mixed format with ipv4 section with octet out of range", + "data": "1::2:192.168.256.1", + "valid": false + }, + { + "description": "mixed format with ipv4 section with a hex octet", + "data": "1::2:192.168.ff.1", + "valid": false + }, + { + "description": "mixed format with leading double colons (ipv4-mapped ipv6 address)", + "data": "::ffff:192.168.0.1", + "valid": true + }, + { + "description": "triple colons is invalid", + "data": "1:2:3:4:5:::8", + "valid": false + }, + { + "description": "8 octets", + "data": "1:2:3:4:5:6:7:8", + "valid": true + }, + { + "description": "insufficient octets without double colons", + "data": "1:2:3:4:5:6:7", + "valid": false + }, + { + "description": "no colons is invalid", + "data": "1", + "valid": false + }, + { + "description": "ipv4 is not ipv6", + "data": "127.0.0.1", + "valid": false + }, + { + "description": "ipv4 segment must have 4 octets", + "data": "1:2:3:4:1.2.3", + "valid": false + }, + { + "description": "leading whitespace is invalid", + "data": " ::1", + "valid": false + }, + { + "description": "trailing whitespace is invalid", + "data": "::1 ", + "valid": false + }, + { + "description": "netmask is not a part of ipv6 address", + "data": "fe80::/64", + "valid": false + }, + { + "description": "zone id is not a part of ipv6 address", + "data": "fe80::a%eth1", + "valid": false + }, + { + "description": "a long valid ipv6", + "data": "1000:1000:1000:1000:1000:1000:255.255.255.255", + "valid": true + }, + { + "description": "a long invalid ipv6, below length limit, first", + "data": "100:100:100:100:100:100:255.255.255.255.255", + "valid": false + }, + { + "description": "a long invalid ipv6, below length limit, second", + "data": "100:100:100:100:100:100:100:255.255.255.255", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4)", + "data": "1:2:3:4:5:6:7:৪", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4) in the IPv4 portion", + "data": "1:2::192.16৪.0.1", + "valid": false + }, + { + "description": "a bracketed address is invalid", + "data": "[::1]", + "valid": false + }, + { + "description": "a leading zero in the last IPv4 octet is invalid", + "data": "::ffff:192.168.0.01", + "valid": false + }, + { + "description": "uppercase hex digits are valid", + "data": "2001:DB8::1", + "valid": true + }, + { + "description": "leading zeros in every group are valid", + "data": "2001:0db8:0000:0000:0000:0000:0000:0001", + "valid": true + }, + { + "description": "two groups before '::' and four groups after is valid", + "data": "1:2::3:4:5:6", + "valid": true + }, + { + "description": "eight groups alongside '::' is invalid", + "data": "1:2:3:4:5:6:7:8::", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "::1\n", + "valid": false + }, + { + "description": "an out-of-range first octet in the embedded IPv4 is invalid", + "data": "::ffff:256.1.1.1", + "valid": false + }, + { + "description": "one group before '::' and five groups after is valid", + "data": "1::2:3:4:5:6", + "valid": true + }, + { + "description": "'::' followed by six groups is valid", + "data": "::1:2:3:4:5:6", + "valid": true + }, + { + "description": "leading '::' before five groups and an embedded IPv4 is valid", + "data": "::1:2:3:4:5:1.2.3.4", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri-reference.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri-reference.json new file mode 100644 index 000000000..c4b6f1853 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri-reference.json @@ -0,0 +1,113 @@ +[ + { + "description": "validation of IRI References", + "schema": { "format": "iri-reference" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IRI", + "data": "http://ƒøø.ßår/?∂éœ=πîx#πîüx", + "valid": true + }, + { + "description": "a valid protocol-relative IRI Reference", + "data": "//ƒøø.ßår/?∂éœ=πîx#πîüx", + "valid": true + }, + { + "description": "a valid relative IRI Reference", + "data": "/âππ", + "valid": true + }, + { + "description": "an invalid IRI Reference", + "data": "\\\\WINDOWS\\filëßåré", + "valid": false + }, + { + "description": "a valid IRI Reference", + "data": "âππ", + "valid": true + }, + { + "description": "a valid IRI fragment", + "data": "#ƒrägmênt", + "valid": true + }, + { + "description": "an invalid IRI fragment", + "data": "#ƒräg\\mênt", + "valid": false + }, + { + "description": "an embedded IPv4 with a leading zero is invalid", + "data": "//[::ffff:192.168.0.01]/p", + "valid": false + }, + { + "description": "a valid protocol-relative IRI Reference with a compressed IPv6 host", + "data": "//[2001:db8::1]/p", + "valid": true + }, + { + "description": "a valid relative IRI Reference with a supplementary-plane character", + "data": "/âππ/𐌀", + "valid": true + }, + { + "description": "a query-only IRI Reference with a supplementary private-use char", + "data": "?q=󰀀", + "valid": true + }, + { + "description": "a trailing newline after a valid IRI Reference is invalid", + "data": "/âππ\n", + "valid": false + }, + { + "description": "invalid percent-encoding with non-hex digits", + "data": "/%6G", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "data": "/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "data": "/%", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri.json new file mode 100644 index 000000000..4025e85d5 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri.json @@ -0,0 +1,138 @@ +[ + { + "description": "validation of IRIs", + "schema": { "format": "iri" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IRI with anchor tag", + "data": "http://ƒøø.ßår/?∂éœ=πîx#πîüx", + "valid": true + }, + { + "description": "a valid IRI with anchor tag and parentheses", + "data": "http://ƒøø.com/blah_(wîkïpédiå)_blah#ßité-1", + "valid": true + }, + { + "description": "a valid IRI with URL-encoded stuff", + "data": "http://ƒøø.ßår/?q=Test%20URL-encoded%20stuff", + "valid": true + }, + { + "description": "invalid percent-encoding with non-hex digits", + "data": "http://ƒøø.ßår/%6G", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "data": "http://ƒøø.ßår/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "data": "http://ƒøø.ßår/%", + "valid": false + }, + { + "description": "a valid IRI with many special characters", + "data": "http://-.~_!$&'()*+,;=:%40:80%2f::::::@example.com", + "valid": true + }, + { + "description": "a valid IRI based on IPv6", + "data": "http://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]", + "valid": true + }, + { + "description": "an IPv6 address without enclosing brackets is invalid", + "data": "http://2001:0db8:85a3:0000:0000:8a2e:0370:7334", + "valid": false + }, + { + "description": "an invalid relative IRI Reference", + "data": "/abc", + "valid": false + }, + { + "description": "an invalid IRI", + "data": "\\\\WINDOWS\\filëßåré", + "valid": false + }, + { + "description": "an invalid IRI though valid IRI reference", + "data": "âππ", + "valid": false + }, + { + "description": "an IPv6 host whose embedded IPv4 has a leading zero is invalid", + "data": "http://[::ffff:192.168.0.01]", + "valid": false + }, + { + "description": "a valid IRI with a compressed IPv6 host", + "data": "http://[2001:db8::1]", + "valid": true + }, + { + "description": "a trailing newline after a valid IRI is invalid", + "data": "http://ƒøø.ßår/\n", + "valid": false + }, + { + "description": "a valid IRI with an IPv4 host", + "data": "http://192.168.0.1/p", + "valid": true + }, + { + "description": "a valid IRI with no authority and a rootless path", + "data": "urn:example:resource", + "valid": true + }, + { + "description": "a valid IRI with no authority and an absolute path", + "data": "file:/etc/hosts", + "valid": true + }, + { + "description": "a valid IRI with a supplementary-plane character in the path", + "data": "http://ƒøø.ßår/𐌀", + "valid": true + }, + { + "description": "a valid IRI with a supplementary-plane private-use char in query", + "data": "http://ƒøø.ßår/?q=󰀀", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/json-pointer.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/json-pointer.json new file mode 100644 index 000000000..134eca586 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/json-pointer.json @@ -0,0 +1,208 @@ +[ + { + "description": "validation of JSON-pointers (JSON String Representation)", + "schema": { "format": "json-pointer" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid JSON-pointer", + "data": "/foo/bar~0/baz~1/%a", + "valid": true + }, + { + "description": "not a valid JSON-pointer (~ not escaped)", + "data": "/foo/bar~", + "valid": false + }, + { + "description": "valid JSON-pointer with empty segment", + "data": "/foo//bar", + "valid": true + }, + { + "description": "valid JSON-pointer with the last empty segment", + "data": "/foo/bar/", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #1", + "data": "", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #2", + "data": "/foo", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #3", + "data": "/foo/0", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #4", + "data": "/", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #5", + "data": "/a~1b", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #6", + "data": "/c%d", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #7", + "data": "/e^f", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #8", + "data": "/g|h", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #9", + "data": "/i\\j", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #10", + "data": "/k\"l", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #11", + "data": "/ ", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #12", + "data": "/m~0n", + "valid": true + }, + { + "description": "valid JSON-pointer used adding to the last array position", + "data": "/foo/-", + "valid": true + }, + { + "description": "valid JSON-pointer (- used as object member name)", + "data": "/foo/-/bar", + "valid": true + }, + { + "description": "valid JSON-pointer (multiple escaped characters)", + "data": "/~1~0~0~1~1", + "valid": true + }, + { + "description": "valid JSON-pointer (escaped with fraction part) #1", + "data": "/~1.1", + "valid": true + }, + { + "description": "valid JSON-pointer (escaped with fraction part) #2", + "data": "/~0.1", + "valid": true + }, + { + "description": "not a valid JSON-pointer (URI Fragment Identifier) #1", + "data": "#", + "valid": false + }, + { + "description": "not a valid JSON-pointer (URI Fragment Identifier) #2", + "data": "#/", + "valid": false + }, + { + "description": "not a valid JSON-pointer (URI Fragment Identifier) #3", + "data": "#a", + "valid": false + }, + { + "description": "not a valid JSON-pointer (some escaped, but not all) #1", + "data": "/~0~", + "valid": false + }, + { + "description": "not a valid JSON-pointer (some escaped, but not all) #2", + "data": "/~0/~", + "valid": false + }, + { + "description": "not a valid JSON-pointer (wrong escape character) #1", + "data": "/~2", + "valid": false + }, + { + "description": "not a valid JSON-pointer (wrong escape character) #2", + "data": "/~-1", + "valid": false + }, + { + "description": "not a valid JSON-pointer (multiple characters not escaped)", + "data": "/~~", + "valid": false + }, + { + "description": "not a valid JSON-pointer (isn't empty nor starts with /) #1", + "data": "a", + "valid": false + }, + { + "description": "not a valid JSON-pointer (isn't empty nor starts with /) #2", + "data": "0", + "valid": false + }, + { + "description": "not a valid JSON-pointer (isn't empty nor starts with /) #3", + "data": "a/a", + "valid": false + }, + { + "description": "valid JSON-pointer (Unicode characters allowed by RFC 6901)", + "data": "/foo/bar/😎", + "valid": true + }, + { + "description": "valid JSON-pointer (control characters allowed after JSON unescaping)", + "data": "/foo\u0000bar\n\tbaz", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/regex.json new file mode 100644 index 000000000..ff831fa2e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/regex.json @@ -0,0 +1,100 @@ +[ + { + "description": "validation of regular expressions", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid regular expression", + "data": "([abc])+\\s+$", + "valid": true + }, + { + "description": "a regular expression with unclosed parens is invalid", + "data": "^(abc]", + "valid": false + }, + { + "description": "a quantifier with nothing to repeat is invalid", + "comment": "https://tc39.es/ecma262/#sec-patterns a Quantifier follows something (Term :: Atom Quantifier), so a pattern cannot begin with one", + "data": "*a", + "valid": false + }, + { + "description": "a character class range in reverse order is invalid", + "comment": "https://tc39.es/ecma262/#sec-patterns-static-semantics-early-errors it is a Syntax Error if the first ClassAtom of a range has a greater CharacterValue than the second", + "data": "[z-a]", + "valid": false + }, + { + "description": "a trailing backslash is invalid", + "data": "\\", + "valid": false + }, + { + "description": "an unmatched closing parenthesis is invalid", + "data": "a)", + "valid": false + }, + { + "description": "an unterminated character class is invalid", + "data": "[abc", + "valid": false + }, + { + "description": "an empty regular expression is valid", + "data": "", + "valid": true + }, + { + "description": "alternation is valid", + "data": "a|b", + "valid": true + }, + { + "description": "a bounded quantifier is valid", + "data": "a{2,3}", + "valid": true + }, + { + "description": "a negated range character class is valid", + "data": "[^a-z]", + "valid": true + }, + { + "description": "a lazy quantifier is valid", + "data": "a+?", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/relative-json-pointer.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/relative-json-pointer.json new file mode 100644 index 000000000..59299e7b3 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/relative-json-pointer.json @@ -0,0 +1,138 @@ +[ + { + "description": "validation of Relative JSON Pointers (RJP)", + "schema": { "format": "relative-json-pointer" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid upwards RJP", + "data": "1", + "valid": true + }, + { + "description": "a valid downwards RJP", + "data": "0/foo/bar", + "valid": true + }, + { + "description": "a valid up and then down RJP, with array index", + "data": "2/0/baz/1/zip", + "valid": true + }, + { + "description": "a valid RJP taking the member or index name", + "data": "0#", + "valid": true + }, + { + "description": "an invalid RJP that is a valid JSON Pointer", + "data": "/foo/bar", + "valid": false + }, + { + "description": "a json-pointer part that does not start with a slash is invalid", + "data": "1foo", + "valid": false + }, + { + "description": "negative prefix", + "data": "-1/foo/bar", + "valid": false + }, + { + "description": "explicit positive prefix", + "data": "+1/foo/bar", + "valid": false + }, + { + "description": "non-ASCII digit in the prefix is not allowed", + "data": "١/foo", + "valid": false + }, + { + "description": "## is not a valid json-pointer", + "data": "0##", + "valid": false + }, + { + "description": "zero cannot be followed by other digits, plus json-pointer", + "data": "01/a", + "valid": false + }, + { + "description": "zero cannot be followed by other digits, plus octothorpe", + "data": "01#", + "valid": false + }, + { + "description": "empty string", + "data": "", + "valid": false + }, + { + "description": "multi-digit integer prefix", + "data": "120/foo/bar", + "valid": true + }, + { + "description": "multi-digit prefix with a zero followed by another digit", + "data": "100", + "valid": true + }, + { + "description": "tilde not followed by 0 or 1 in the json-pointer part", + "data": "0/~2", + "valid": false + }, + { + "description": "unescaped tilde at the end of the json-pointer part", + "data": "0/foo/bar~", + "valid": false + }, + { + "description": "octothorpe followed by a json-pointer", + "data": "1#/foo/bar", + "valid": false + }, + { + "description": "empty reference tokens in the json-pointer part", + "data": "0//", + "valid": true + }, + { + "description": "trailing newline after the non-negative integer", + "data": "1\n", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/time.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/time.json new file mode 100644 index 000000000..3182c3ea9 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/time.json @@ -0,0 +1,300 @@ +[ + { + "description": "validation of time strings", + "schema": { "format": "time" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid time string", + "data": "08:30:06Z", + "valid": true + }, + { + "description": "invalid time string with extra leading zeros", + "data": "008:030:006Z", + "valid": false + }, + { + "description": "invalid time string with no leading zero for single digit", + "data": "8:3:6Z", + "valid": false + }, + { + "description": "hour, minute, second must be two digits", + "data": "8:0030:6Z", + "valid": false + }, + { + "description": "a valid time string with leap second, Zulu", + "data": "23:59:60Z", + "valid": true + }, + { + "description": "invalid leap second, Zulu (wrong hour)", + "data": "22:59:60Z", + "valid": false + }, + { + "description": "invalid leap second, Zulu (wrong minute)", + "data": "23:58:60Z", + "valid": false + }, + { + "description": "valid leap second, zero time-offset", + "data": "23:59:60+00:00", + "valid": true + }, + { + "description": "invalid leap second, zero time-offset (wrong hour)", + "data": "22:59:60+00:00", + "valid": false + }, + { + "description": "invalid leap second, zero time-offset (wrong minute)", + "data": "23:58:60+00:00", + "valid": false + }, + { + "description": "valid leap second, positive time-offset", + "data": "01:29:60+01:30", + "valid": true + }, + { + "description": "valid leap second, large positive time-offset", + "data": "23:29:60+23:30", + "valid": true + }, + { + "description": "invalid leap second, positive time-offset (wrong hour)", + "data": "23:59:60+01:00", + "valid": false + }, + { + "description": "invalid leap second, positive time-offset (wrong minute)", + "data": "23:59:60+00:30", + "valid": false + }, + { + "description": "valid leap second, negative time-offset", + "data": "15:59:60-08:00", + "valid": true + }, + { + "description": "valid leap second, large negative time-offset", + "data": "00:29:60-23:30", + "valid": true + }, + { + "description": "invalid leap second, negative time-offset (wrong hour)", + "data": "23:59:60-01:00", + "valid": false + }, + { + "description": "invalid leap second, negative time-offset (wrong minute)", + "data": "23:59:60-00:30", + "valid": false + }, + { + "description": "a valid time string with second fraction", + "data": "23:20:50.52Z", + "valid": true + }, + { + "description": "a valid time string with precise second fraction", + "data": "08:30:06.283185Z", + "valid": true + }, + { + "description": "a valid time string with plus offset", + "data": "08:30:06+00:20", + "valid": true + }, + { + "description": "a valid time string with minus offset", + "data": "08:30:06-08:00", + "valid": true + }, + { + "description": "time with unknown local offset is valid", + "comment": "RFC 3339 section 4.3 (unknown local offset)", + "data": "12:34:56-00:00", + "valid": true + }, + { + "description": "hour, minute in time-offset must be two digits", + "data": "08:30:06-8:000", + "valid": false + }, + { + "description": "a valid time string with case-insensitive Z", + "data": "08:30:06z", + "valid": true + }, + { + "description": "an invalid time string with invalid hour", + "data": "24:00:00Z", + "valid": false + }, + { + "description": "an invalid time string with invalid minute", + "data": "00:60:00Z", + "valid": false + }, + { + "description": "an invalid time string with invalid second", + "data": "00:00:61Z", + "valid": false + }, + { + "description": "an invalid time string with invalid time numoffset hour", + "data": "01:02:03+24:00", + "valid": false + }, + { + "description": "an invalid time string with invalid time numoffset minute", + "data": "01:02:03+00:60", + "valid": false + }, + { + "description": "an invalid time string with invalid time with both Z and numoffset", + "data": "01:02:03Z+00:30", + "valid": false + }, + { + "description": "an invalid offset indicator", + "data": "08:30:06 PST", + "valid": false + }, + { + "description": "only RFC3339 not all of ISO 8601 are valid", + "data": "01:01:01,1111", + "valid": false + }, + { + "description": "no time offset", + "data": "12:00:00", + "valid": false + }, + { + "description": "no time offset with second fraction", + "data": "12:00:00.52", + "valid": false + }, + { + "description": "invalid non-ASCII '২' (a Bengali 2)", + "data": "1২:00:00Z", + "valid": false + }, + { + "description": "offset not starting with plus or minus", + "data": "08:30:06#00:20", + "valid": false + }, + { + "description": "contains letters", + "data": "ab:cd:ef", + "valid": false + }, + { + "description": "an invalid time string in date-time format", + "data": "2020-11-28T23:55:45Z", + "valid": false + }, + { + "description": "a numeric time-offset requires the colon separator", + "comment": "RFC 3339 section 5.6: time-numoffset = (\"+\" / \"-\") time-hour \":\" time-minute", + "data": "08:30:06+0130", + "valid": false + }, + { + "description": "a numeric time-offset requires the minutes component", + "comment": "RFC 3339 section 5.6: time-numoffset = (\"+\" / \"-\") time-hour \":\" time-minute", + "data": "08:30:06+01", + "valid": false + }, + { + "description": "a trailing newline after a valid time", + "comment": "RFC 3339 section 5.6: no production of full-time emits a line feed", + "data": "08:30:06Z\n", + "valid": false + }, + { + "description": "hour 24 is invalid with a numeric time-offset", + "comment": "RFC 3339 section 5.6: time-hour = 2DIGIT ; 00-23, and section 5.7 only allows 00 to 23", + "data": "24:59:00+01:00", + "valid": false + }, + { + "description": "minute 60 is invalid with a numeric time-offset", + "comment": "RFC 3339 section 5.6: time-minute = 2DIGIT ; 00-59", + "data": "23:60:00+00:01", + "valid": false + }, + { + "description": "a valid time string with a 15 digit second fraction", + "comment": "RFC 3339 section 5.6: time-secfrac = \".\" 1*DIGIT, with no upper bound on the number of digits", + "data": "00:59:59.999999999999999Z", + "valid": true + }, + { + "description": "a second fraction with no digits", + "comment": "RFC 3339 section 5.6: time-secfrac = \".\" 1*DIGIT requires at least one digit", + "data": "08:30:06.Z", + "valid": false + }, + { + "description": "partial-time requires the seconds component", + "comment": "RFC 3339 section 5.6: partial-time = time-hour \":\" time-minute \":\" time-second [time-secfrac]", + "data": "12:00Z", + "valid": false + }, + { + "description": "a comma decimal separator with a time-offset", + "comment": "RFC 3339 section 5.6: the decimal separator in time-secfrac is \".\" - the ISO 8601 comma is not part of this profile", + "data": "08:30:06,5Z", + "valid": false + }, + { + "description": "leading whitespace before a valid time", + "comment": "RFC 3339 section 5.6: no production of full-time emits whitespace", + "data": " 08:30:06Z", + "valid": false + }, + { + "description": "trailing whitespace after a valid time", + "comment": "RFC 3339 section 5.6: no production of full-time emits whitespace", + "data": "08:30:06Z ", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/unknown.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/unknown.json new file mode 100644 index 000000000..12339ae57 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/unknown.json @@ -0,0 +1,43 @@ +[ + { + "description": "unknown format", + "schema": { "format": "unknown" }, + "tests": [ + { + "description": "unknown formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "unknown formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "unknown formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "unknown formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "unknown formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "unknown formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "unknown formats ignore strings", + "data": "string", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-reference.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-reference.json new file mode 100644 index 000000000..3741bad01 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-reference.json @@ -0,0 +1,179 @@ +[ + { + "description": "validation of URI References", + "schema": { "format": "uri-reference" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid URI", + "data": "http://foo.bar/?baz=qux#quux", + "valid": true + }, + { + "description": "a valid protocol-relative URI Reference", + "data": "//foo.bar/?baz=qux#quux", + "valid": true + }, + { + "description": "a valid relative URI Reference", + "data": "/abc", + "valid": true + }, + { + "description": "an invalid URI Reference", + "data": "\\\\WINDOWS\\fileshare", + "valid": false + }, + { + "description": "a valid URI Reference", + "data": "abc", + "valid": true + }, + { + "description": "a valid URI fragment", + "data": "#fragment", + "valid": true + }, + { + "description": "an invalid URI fragment", + "data": "#frag\\ment", + "valid": false + }, + { + "description": "unescaped non US-ASCII characters", + "data": "/foobar®.txt", + "valid": false + }, + { + "description": "invalid backslash character", + "data": "https://example.org/foobar\\.txt", + "valid": false + }, + { + "description": "URI with leading-zero IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: If an IP literal fails strict IPv4address parsing, it falls back to reg-name. A string of digits and dots is valid under unreserved characters. JSON Schema format asserts syntax, not scheme-specific DNS semantics.", + "data": "http://087.10.0.1/", + "valid": true + }, + { + "description": "URI with out-of-bounds IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: Fallback to reg-name allows digits and dots.", + "data": "http://999.999.999.999/", + "valid": true + }, + { + "description": "an empty URI reference", + "comment": "RFC 3986, Section 4.2: relative-part includes path-empty, which is zero characters.", + "data": "", + "valid": true + }, + { + "description": "a query-only reference", + "comment": "RFC 3986, Section 4.2: relative-ref = relative-part [ \"?\" query ] [ \"#\" fragment ], so a query with an empty path is a reference.", + "data": "?query=1", + "valid": true + }, + { + "description": "a network-path reference with an empty authority", + "comment": "RFC 3986, Section 3.2.2: host = IP-literal / IPv4address / reg-name, and reg-name may be empty.", + "data": "//", + "valid": true + }, + { + "description": "an incomplete percent-encoding", + "comment": "RFC 3986, Section 2.1: pct-encoded = \"%\" HEXDIG HEXDIG.", + "data": "/%zz", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "comment": "RFC 3986, Section 2.1: pct-encoded = \"%\" HEXDIG HEXDIG.", + "data": "/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "comment": "RFC 3986, Section 2.1: pct-encoded = \"%\" HEXDIG HEXDIG.", + "data": "/%", + "valid": false + }, + { + "description": "a double quote in a path", + "comment": "RFC 3986, Appendix A: pchar = unreserved / pct-encoded / sub-delims / \":\" / \"@\", and the double quote is in none of them.", + "data": "/a\"b", + "valid": false + }, + { + "description": "square brackets outside an authority", + "comment": "RFC 3986, Section 3.2.2: square brackets are reserved for an IP-literal inside the authority. path-absolute is built from pchar, which excludes them.", + "data": "/[::1]", + "valid": false + }, + { + "description": "a non-numeric port in a network-path reference", + "comment": "RFC 3986, Section 3.2.3: port = *DIGIT. reg-name admits no colon, so the text after the colon can only be a port.", + "data": "//example.com:abc/p", + "valid": false + }, + { + "description": "more than one at-sign in the authority", + "comment": "RFC 3986, Section 3.2: authority = [ userinfo \"@\" ] host [ \":\" port ]. Neither userinfo nor host admits a literal at-sign.", + "data": "//a@b@example.com/", + "valid": false + }, + { + "description": "a leading zero in the IPv4 part of an IPv6 literal", + "comment": "RFC 3986, Section 3.2.2: inside an IP-literal the dotted quad must be an IPv4address, and dec-octet has no leading-zero alternative. There is no reg-name fallback inside brackets.", + "data": "//[::ffff:192.168.0.01]/p", + "valid": false + }, + { + "description": "a colon in the first segment of a relative-path reference", + "comment": "RFC 3986, Section 4.2: relative-part uses path-noscheme, whose first segment is segment-nz-nc - a non-zero-length segment without any colon. A scheme must start with ALPHA, so 1:b cannot be read as a URI either.", + "data": "1:b", + "valid": false + }, + { + "description": "a colon in the first segment preceded by a dot-segment", + "comment": "RFC 3986, Section 4.2: \"Such a segment must be preceded by a dot-segment (e.g., \\\"./this:that\\\") to make a relative-path reference.\"", + "data": "./this:that", + "valid": true + }, + { + "description": "a trailing line feed", + "comment": "RFC 3986, Appendix A: no production admits %x0A.", + "data": "/p\n", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-template.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-template.json new file mode 100644 index 000000000..f0aaa52f7 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-template.json @@ -0,0 +1,248 @@ +[ + { + "description": "format: uri-template", + "schema": { "format": "uri-template" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid uri-template", + "data": "http://example.com/dictionary/{term:1}/{term}", + "valid": true + }, + { + "description": "an invalid uri-template", + "data": "http://example.com/dictionary/{term:1}/{term", + "valid": false + }, + { + "description": "a valid uri-template without variables", + "data": "http://example.com/dictionary", + "valid": true + }, + { + "description": "a valid relative uri-template", + "data": "dictionary/{term:1}/{term}", + "valid": true + }, + { + "description": "an empty expression is invalid", + "data": "{}", + "valid": false + }, + { + "description": "an empty varspec inside the variable list is invalid", + "data": "{a,,b}", + "valid": false + }, + { + "description": "a zero prefix max-length is invalid", + "data": "{v:0}", + "valid": false + }, + { + "description": "a five-digit prefix max-length is invalid", + "data": "{v:10000}", + "valid": false + }, + { + "description": "an empty string is a valid uri-template", + "data": "", + "valid": true + }, + { + "description": "an unmatched closing brace is invalid", + "data": "foo}bar", + "valid": false + }, + { + "description": "a percent-encoded triplet in a literal is valid", + "data": "a%41b", + "valid": true + }, + { + "description": "a literal with a lone percent sign is invalid", + "data": "a%", + "valid": false + }, + { + "description": "a literal with an incomplete percent-encoded triplet is invalid", + "data": "a%4", + "valid": false + }, + { + "description": "a literal with non-hex percent encoding is invalid", + "data": "a%GG", + "valid": false + }, + { + "description": "a supplementary plane character in a literal is valid", + "data": "a\ud83d\ude00b", + "valid": true + }, + { + "description": "a private-use character in a literal is valid", + "data": "a\ue000b", + "valid": true + }, + { + "description": "a supplementary private-use character in a literal is valid", + "data": "a\udb80\udc00b", + "valid": true + }, + { + "description": "reserved expansion operator", + "data": "{+var}", + "valid": true + }, + { + "description": "fragment expansion operator", + "data": "{#var}", + "valid": true + }, + { + "description": "label expansion operator", + "data": "{.var}", + "valid": true + }, + { + "description": "path segment expansion operator", + "data": "{/var}", + "valid": true + }, + { + "description": "path style parameter expansion operator", + "data": "{;var}", + "valid": true + }, + { + "description": "form style query expansion operator", + "data": "{?var}", + "valid": true + }, + { + "description": "form style query continuation operator", + "data": "{&var}", + "valid": true + }, + { + "description": "the explode modifier is valid", + "data": "{var*}", + "valid": true + }, + { + "description": "combined prefix and explode modifiers are invalid", + "data": "{var:1*}", + "valid": false + }, + { + "description": "a variable list with more than one varspec is valid", + "data": "{x,y}", + "valid": true + }, + { + "description": "a dotted variable name is valid", + "data": "{a.b}", + "valid": true + }, + { + "description": "a trailing dot in a variable name is invalid", + "data": "{a.}", + "valid": false + }, + { + "description": "a double dot in a variable name is invalid", + "data": "{a..b}", + "valid": false + }, + { + "description": "a variable name may start with a percent-encoded triplet", + "data": "{%41}", + "valid": true + }, + { + "description": "a variable name with a lone percent sign is invalid", + "data": "{%}", + "valid": false + }, + { + "description": "a variable name with an incomplete percent-encoded triplet is invalid", + "data": "{%4}", + "valid": false + }, + { + "description": "a variable name with non-hex percent encoding is invalid", + "data": "{%GG}", + "valid": false + }, + { + "description": "a four-digit prefix max-length is valid", + "data": "{v:1000}", + "valid": true + }, + { + "description": "a leading zero in the prefix max-length is invalid", + "data": "{v:01}", + "valid": false + }, + { + "description": "a space in a literal is invalid", + "data": "a b", + "valid": false + }, + { + "description": "a delete character in a literal is invalid", + "data": "a\u007fb", + "valid": false + }, + { + "description": "a trailing comma in the variable list is invalid", + "data": "{a,}", + "valid": false + }, + { + "description": "a default value in a varspec is invalid", + "data": "{var=def}", + "valid": false + }, + { + "description": "an operator in place of the first varspec is invalid", + "data": "{,+var}", + "valid": false + }, + { + "description": "an apostrophe in a literal is valid", + "data": "a'b", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri.json new file mode 100644 index 000000000..b9d080027 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri.json @@ -0,0 +1,250 @@ +[ + { + "description": "validation of URIs", + "schema": { "format": "uri" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid URL with anchor tag", + "data": "http://foo.bar/?baz=qux#quux", + "valid": true + }, + { + "description": "a valid URL with anchor tag and parentheses", + "data": "http://foo.com/blah_(wikipedia)_blah#cite-1", + "valid": true + }, + { + "description": "a valid URL with URL-encoded stuff", + "data": "http://foo.bar/?q=Test%20URL-encoded%20stuff", + "valid": true + }, + { + "description": "a valid puny-coded URL ", + "data": "http://xn--nw2a.xn--j6w193g/", + "valid": true + }, + { + "description": "a valid URL with many special characters", + "data": "http://-.~_!$&'()*+,;=:%40:80%2f::::::@example.com", + "valid": true + }, + { + "description": "a valid URL based on IPv4", + "data": "http://223.255.255.254", + "valid": true + }, + { + "description": "a valid URL with ftp scheme", + "data": "ftp://ftp.is.co.za/rfc/rfc1808.txt", + "valid": true + }, + { + "description": "a valid URL for a simple text file", + "data": "http://www.ietf.org/rfc/rfc2396.txt", + "valid": true + }, + { + "description": "a valid URL", + "data": "ldap://[2001:db8::7]/c=GB?objectClass?one", + "valid": true + }, + { + "description": "a valid mailto URI", + "data": "mailto:John.Doe@example.com", + "valid": true + }, + { + "description": "a valid newsgroup URI", + "data": "news:comp.infosystems.www.servers.unix", + "valid": true + }, + { + "description": "a valid tel URI", + "data": "tel:+1-816-555-1212", + "valid": true + }, + { + "description": "a valid URN", + "data": "urn:oasis:names:specification:docbook:dtd:xml:4.1.2", + "valid": true + }, + { + "description": "an invalid protocol-relative URI Reference", + "data": "//foo.bar/?baz=qux#quux", + "valid": false + }, + { + "description": "an invalid relative URI Reference", + "data": "/abc", + "valid": false + }, + { + "description": "an invalid URI", + "data": "\\\\WINDOWS\\fileshare", + "valid": false + }, + { + "description": "an invalid URI though valid URI reference", + "data": "abc", + "valid": false + }, + { + "description": "an invalid URI with spaces", + "data": "http:// shouldfail.com", + "valid": false + }, + { + "description": "an invalid URI with spaces and missing scheme", + "data": ":// should fail", + "valid": false + }, + { + "description": "an invalid URI with comma in scheme", + "data": "bar,baz:foo", + "valid": false + }, + { + "description": "invalid userinfo", + "data": "https://[@example.org/test.txt", + "valid": false + }, + { + "description": "unescaped non US-ASCII characters", + "data": "https://example.org/foobar®.txt", + "valid": false + }, + { + "description": "invalid backslash character", + "data": "https://example.org/foobar\\.txt", + "valid": false + }, + { + "description": "invalid \" character", + "data": "https://example.org/foobar\".txt", + "valid": false + }, + { + "description": "invalid <> characters", + "data": "https://example.org/foobar<>.txt", + "valid": false + }, + { + "description": "invalid {} characters", + "data": "https://example.org/foobar{}.txt", + "valid": false + }, + { + "description": "invalid ^ character", + "data": "https://example.org/foobar^.txt", + "valid": false + }, + { + "description": "invalid ` character", + "data": "https://example.org/foobar`.txt", + "valid": false + }, + { + "description": "invalid SPACE character", + "data": "https://example.org/foo bar.txt", + "valid": false + }, + { + "description": "invalid | character", + "data": "https://example.org/foobar|.txt", + "valid": false + }, + { + "description": "URI with leading-zero IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: If an IP literal fails strict IPv4address parsing, it falls back to reg-name. A string of digits and dots is valid under unreserved characters. JSON Schema format asserts syntax, not scheme-specific DNS semantics.", + "data": "http://087.10.0.1/", + "valid": true + }, + { + "description": "URI with out-of-bounds IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: Fallback to reg-name allows digits and dots.", + "data": "http://999.999.999.999/", + "valid": true + }, + { + "description": "invalid percent-encoding with non-hex digits", + "data": "http://example.com/%6G", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "data": "http://example.com/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "data": "http://example.com/%", + "valid": false + }, + { + "description": "scheme must start with a letter", + "data": "1http://example.com", + "valid": false + }, + { + "description": "invalid character in scheme", + "data": "ht_tp://example.com", + "valid": false + }, + { + "description": "non-numeric port is invalid", + "data": "http://example.com:abc/path", + "valid": false + }, + { + "description": "leading zero in an embedded IPv4 address is invalid", + "data": "http://[::ffff:01.2.3.4]", + "valid": false + }, + { + "description": "square brackets are not allowed in a path segment", + "data": "http:/[::1]", + "valid": false + }, + { + "description": "a trailing newline after a valid URI is invalid", + "data": "http://foo.bar/\n", + "valid": false + }, + { + "description": "an IPv6 address without enclosing brackets is invalid", + "data": "http://2001:0db8:85a3:0000:0000:8a2e:0370:7334", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/id.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/id.json new file mode 100644 index 000000000..6be81b8da --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/id.json @@ -0,0 +1,114 @@ +[ + { + "description": "id inside an enum is not a real identifier", + "comment": "the implementation must not be confused by an id buried in the enum", + "schema": { + "definitions": { + "id_in_enum": { + "enum": [ + { + "$id": "https://localhost:1234/id/my_identifier.json", + "type": "null" + } + ] + }, + "real_id_in_schema": { + "$id": "https://localhost:1234/id/my_identifier.json", + "type": "string" + }, + "zzz_id_in_const": { + "const": { + "$id": "https://localhost:1234/id/my_identifier.json", + "type": "null" + } + } + }, + "anyOf": [ + { "$ref": "#/definitions/id_in_enum" }, + { "$ref": "https://localhost:1234/id/my_identifier.json" } + ] + }, + "tests": [ + { + "description": "exact match to enum, and type matches", + "data": { + "$id": "https://localhost:1234/id/my_identifier.json", + "type": "null" + }, + "valid": true + }, + { + "description": "match $ref to id", + "data": "a string to match #/definitions/id_in_enum", + "valid": true + }, + { + "description": "no match on enum or $ref to id", + "data": 1, + "valid": false + } + ] + }, + { + "description": "non-schema object containing a plain-name $id property", + "schema": { + "definitions": { + "const_not_anchor": { + "const": { + "$id": "#not_a_real_anchor" + } + } + }, + "if": { + "const": "skip not_a_real_anchor" + }, + "then": true, + "else" : { + "$ref": "#/definitions/const_not_anchor" + } + }, + "tests": [ + { + "description": "skip traversing definition for a valid result", + "data": "skip not_a_real_anchor", + "valid": true + }, + { + "description": "const at const_not_anchor does not match", + "data": 1, + "valid": false + } + ] + }, + { + "description": "non-schema object containing an $id property", + "schema": { + "definitions": { + "const_not_id": { + "const": { + "$id": "not_a_real_id" + } + } + }, + "if": { + "const": "skip not_a_real_id" + }, + "then": true, + "else" : { + "$ref": "#/definitions/const_not_id" + } + }, + "tests": [ + { + "description": "skip traversing definition for a valid result", + "data": "skip not_a_real_id", + "valid": true + }, + { + "description": "const at const_not_id does not match", + "data": 1, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/non-bmp-regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/non-bmp-regex.json new file mode 100644 index 000000000..dd67af2b2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/non-bmp-regex.json @@ -0,0 +1,82 @@ +[ + { + "description": "Proper UTF-16 surrogate pair handling: pattern", + "comment": "Optional because .Net doesn't correctly handle 32-bit Unicode characters", + "schema": { "pattern": "^🐲*$" }, + "tests": [ + { + "description": "matches empty", + "data": "", + "valid": true + }, + { + "description": "matches single", + "data": "🐲", + "valid": true + }, + { + "description": "matches two", + "data": "🐲🐲", + "valid": true + }, + { + "description": "doesn't match one", + "data": "🐉", + "valid": false + }, + { + "description": "doesn't match two", + "data": "🐉🐉", + "valid": false + }, + { + "description": "doesn't match one ASCII", + "data": "D", + "valid": false + }, + { + "description": "doesn't match two ASCII", + "data": "DD", + "valid": false + } + ] + }, + { + "description": "Proper UTF-16 surrogate pair handling: patternProperties", + "comment": "Optional because .Net doesn't correctly handle 32-bit Unicode characters", + "schema": { + "patternProperties": { + "^🐲*$": { + "type": "integer" + } + } + }, + "tests": [ + { + "description": "matches empty", + "data": { "": 1 }, + "valid": true + }, + { + "description": "matches single", + "data": { "🐲": 1 }, + "valid": true + }, + { + "description": "matches two", + "data": { "🐲🐲": 1 }, + "valid": true + }, + { + "description": "doesn't match one", + "data": { "🐲": "hello" }, + "valid": false + }, + { + "description": "doesn't match two", + "data": { "🐲🐲": "hello" }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/unknownKeyword.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/unknownKeyword.json new file mode 100644 index 000000000..1f58d97e3 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/unknownKeyword.json @@ -0,0 +1,56 @@ +[ + { + "description": "$id inside an unknown keyword is not a real identifier", + "comment": "the implementation must not be confused by an $id in locations we do not know how to parse", + "schema": { + "definitions": { + "id_in_unknown0": { + "not": { + "array_of_schemas": [ + { + "$id": "https://localhost:1234/unknownKeyword/my_identifier.json", + "type": "null" + } + ] + } + }, + "real_id_in_schema": { + "$id": "https://localhost:1234/unknownKeyword/my_identifier.json", + "type": "string" + }, + "id_in_unknown1": { + "not": { + "object_of_schemas": { + "foo": { + "$id": "https://localhost:1234/unknownKeyword/my_identifier.json", + "type": "integer" + } + } + } + } + }, + "anyOf": [ + { "$ref": "#/definitions/id_in_unknown0" }, + { "$ref": "#/definitions/id_in_unknown1" }, + { "$ref": "https://localhost:1234/unknownKeyword/my_identifier.json" } + ] + }, + "tests": [ + { + "description": "type matches second anyOf, which has a real schema in it", + "data": "a string", + "valid": true + }, + { + "description": "type matches non-schema in first anyOf", + "data": null, + "valid": false + }, + { + "description": "type matches non-schema in third anyOf", + "data": 1, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/pattern.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/pattern.json new file mode 100644 index 000000000..92db0f971 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/pattern.json @@ -0,0 +1,59 @@ +[ + { + "description": "pattern validation", + "schema": {"pattern": "^a*$"}, + "tests": [ + { + "description": "a matching pattern is valid", + "data": "aaa", + "valid": true + }, + { + "description": "a non-matching pattern is invalid", + "data": "abc", + "valid": false + }, + { + "description": "ignores booleans", + "data": true, + "valid": true + }, + { + "description": "ignores integers", + "data": 123, + "valid": true + }, + { + "description": "ignores floats", + "data": 1.0, + "valid": true + }, + { + "description": "ignores objects", + "data": {}, + "valid": true + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + } + ] + }, + { + "description": "pattern is not anchored", + "schema": {"pattern": "a+"}, + "tests": [ + { + "description": "matches a substring", + "data": "xxaayy", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/patternProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/patternProperties.json new file mode 100644 index 000000000..c276e6479 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/patternProperties.json @@ -0,0 +1,171 @@ +[ + { + "description": + "patternProperties validates properties matching a regex", + "schema": { + "patternProperties": { + "f.*o": {"type": "integer"} + } + }, + "tests": [ + { + "description": "a single valid match is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "multiple valid matches is valid", + "data": {"foo": 1, "foooooo" : 2}, + "valid": true + }, + { + "description": "a single invalid match is invalid", + "data": {"foo": "bar", "fooooo": 2}, + "valid": false + }, + { + "description": "multiple invalid matches is invalid", + "data": {"foo": "bar", "foooooo" : "baz"}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["foo"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foo", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "multiple simultaneous patternProperties are validated", + "schema": { + "patternProperties": { + "a*": {"type": "integer"}, + "aaa*": {"maximum": 20} + } + }, + "tests": [ + { + "description": "a single valid match is valid", + "data": {"a": 21}, + "valid": true + }, + { + "description": "a simultaneous match is valid", + "data": {"aaaa": 18}, + "valid": true + }, + { + "description": "multiple matches is valid", + "data": {"a": 21, "aaaa": 18}, + "valid": true + }, + { + "description": "an invalid due to one is invalid", + "data": {"a": "bar"}, + "valid": false + }, + { + "description": "an invalid due to the other is invalid", + "data": {"aaaa": 31}, + "valid": false + }, + { + "description": "an invalid due to both is invalid", + "data": {"aaa": "foo", "aaaa": 31}, + "valid": false + } + ] + }, + { + "description": "regexes are not anchored by default and are case sensitive", + "schema": { + "patternProperties": { + "[0-9]{2,}": { "type": "boolean" }, + "X_": { "type": "string" } + } + }, + "tests": [ + { + "description": "non recognized members are ignored", + "data": { "answer 1": "42" }, + "valid": true + }, + { + "description": "recognized members are accounted for", + "data": { "a31b": null }, + "valid": false + }, + { + "description": "regexes are case sensitive", + "data": { "a_x_3": 3 }, + "valid": true + }, + { + "description": "regexes are case sensitive, 2", + "data": { "a_X_3": 3 }, + "valid": false + } + ] + }, + { + "description": "patternProperties with boolean schemas", + "schema": { + "patternProperties": { + "f.*": true, + "b.*": false + } + }, + "tests": [ + { + "description": "object with property matching schema true is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property matching schema false is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "object with both properties is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "object with a property matching both true and false is invalid", + "data": {"foobar":1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "patternProperties with null valued instance properties", + "schema": { + "patternProperties": { + "^.*bar$": {"type": "null"} + } + }, + "tests": [ + { + "description": "allows null values", + "data": {"foobar": null}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/properties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/properties.json new file mode 100644 index 000000000..5b971ca0e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/properties.json @@ -0,0 +1,236 @@ +[ + { + "description": "object properties validation", + "schema": { + "properties": { + "foo": {"type": "integer"}, + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "both properties present and valid is valid", + "data": {"foo": 1, "bar": "baz"}, + "valid": true + }, + { + "description": "one property invalid is invalid", + "data": {"foo": 1, "bar": {}}, + "valid": false + }, + { + "description": "both properties invalid is invalid", + "data": {"foo": [], "bar": {}}, + "valid": false + }, + { + "description": "doesn't invalidate other properties", + "data": {"quux": []}, + "valid": true + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": + "properties, patternProperties, additionalProperties interaction", + "schema": { + "properties": { + "foo": {"type": "array", "maxItems": 3}, + "bar": {"type": "array"} + }, + "patternProperties": {"f.o": {"minItems": 2}}, + "additionalProperties": {"type": "integer"} + }, + "tests": [ + { + "description": "property validates property", + "data": {"foo": [1, 2]}, + "valid": true + }, + { + "description": "property invalidates property", + "data": {"foo": [1, 2, 3, 4]}, + "valid": false + }, + { + "description": "patternProperty invalidates property", + "data": {"foo": []}, + "valid": false + }, + { + "description": "patternProperty validates nonproperty", + "data": {"fxo": [1, 2]}, + "valid": true + }, + { + "description": "patternProperty invalidates nonproperty", + "data": {"fxo": []}, + "valid": false + }, + { + "description": "additionalProperty ignores property", + "data": {"bar": []}, + "valid": true + }, + { + "description": "additionalProperty validates others", + "data": {"quux": 3}, + "valid": true + }, + { + "description": "additionalProperty invalidates others", + "data": {"quux": "foo"}, + "valid": false + } + ] + }, + { + "description": "properties with boolean schema", + "schema": { + "properties": { + "foo": true, + "bar": false + } + }, + "tests": [ + { + "description": "no property present is valid", + "data": {}, + "valid": true + }, + { + "description": "only 'true' property present is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "only 'false' property present is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "both properties present is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + } + ] + }, + { + "description": "properties with escaped characters", + "schema": { + "properties": { + "foo\nbar": {"type": "number"}, + "foo\"bar": {"type": "number"}, + "foo\\bar": {"type": "number"}, + "foo\rbar": {"type": "number"}, + "foo\tbar": {"type": "number"}, + "foo\fbar": {"type": "number"} + } + }, + "tests": [ + { + "description": "object with all numbers is valid", + "data": { + "foo\nbar": 1, + "foo\"bar": 1, + "foo\\bar": 1, + "foo\rbar": 1, + "foo\tbar": 1, + "foo\fbar": 1 + }, + "valid": true + }, + { + "description": "object with strings is invalid", + "data": { + "foo\nbar": "1", + "foo\"bar": "1", + "foo\\bar": "1", + "foo\rbar": "1", + "foo\tbar": "1", + "foo\fbar": "1" + }, + "valid": false + } + ] + }, + { + "description": "properties with null valued instance properties", + "schema": { + "properties": { + "foo": {"type": "null"} + } + }, + "tests": [ + { + "description": "allows null values", + "data": {"foo": null}, + "valid": true + } + ] + }, + { + "description": "properties whose names are Javascript object property names", + "comment": "Ensure JS implementations don't universally consider e.g. __proto__ to always be present in an object.", + "schema": { + "properties": { + "__proto__": {"type": "number"}, + "toString": { + "properties": { "length": { "type": "string" } } + }, + "constructor": {"type": "number"} + } + }, + "tests": [ + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "none of the properties mentioned", + "data": {}, + "valid": true + }, + { + "description": "__proto__ not valid", + "data": { "__proto__": "foo" }, + "valid": false + }, + { + "description": "toString not valid", + "data": { "toString": { "length": 37 } }, + "valid": false + }, + { + "description": "constructor not valid", + "data": { "constructor": { "length": 37 } }, + "valid": false + }, + { + "description": "all present and valid", + "data": { + "__proto__": 12, + "toString": { "length": "foo" }, + "constructor": 37 + }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/propertyNames.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/propertyNames.json new file mode 100644 index 000000000..b86564310 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/propertyNames.json @@ -0,0 +1,164 @@ +[ + { + "description": "propertyNames validation", + "schema": { + "propertyNames": {"maxLength": 3} + }, + "tests": [ + { + "description": "all property names valid", + "data": { + "f": {}, + "foo": {} + }, + "valid": true + }, + { + "description": "some property names invalid", + "data": { + "foo": {}, + "foobar": {} + }, + "valid": false + }, + { + "description": "object without properties is valid", + "data": {}, + "valid": true + }, + { + "description": "ignores arrays", + "data": [1, 2, 3, 4], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + }, + { + "description": "ignores booleans", + "data": true, + "valid": true + } + ] + }, + { + "description": "propertyNames validation with pattern", + "schema": { + "propertyNames": { "pattern": "^a+$" } + }, + "tests": [ + { + "description": "matching property names valid", + "data": { + "a": {}, + "aa": {}, + "aaa": {} + }, + "valid": true + }, + { + "description": "non-matching property name is invalid", + "data": { + "aaA": {} + }, + "valid": false + }, + { + "description": "object without properties is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with boolean schema true", + "schema": {"propertyNames": true}, + "tests": [ + { + "description": "object with any properties is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with boolean schema false", + "schema": {"propertyNames": false}, + "tests": [ + { + "description": "object with any properties is invalid", + "data": {"foo": 1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with const", + "schema": {"propertyNames": {"const": "foo"}}, + "tests": [ + { + "description": "object with property foo is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with any other property is invalid", + "data": {"bar": 1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with enum", + "schema": {"propertyNames": {"enum": ["foo", "bar"]}}, + "tests": [ + { + "description": "object with property foo is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property foo and bar is valid", + "data": {"foo": 1, "bar": 1}, + "valid": true + }, + { + "description": "object with any other property is invalid", + "data": {"baz": 1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/ref.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/ref.json new file mode 100644 index 000000000..82e1e1672 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/ref.json @@ -0,0 +1,1043 @@ +[ + { + "description": "root pointer ref", + "schema": { + "properties": { + "foo": {"$ref": "#"} + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "match", + "data": {"foo": false}, + "valid": true + }, + { + "description": "recursive match", + "data": {"foo": {"foo": false}}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": false}, + "valid": false + }, + { + "description": "recursive mismatch", + "data": {"foo": {"bar": false}}, + "valid": false + } + ] + }, + { + "description": "relative pointer ref to object", + "schema": { + "properties": { + "foo": {"type": "integer"}, + "bar": {"$ref": "#/properties/foo"} + } + }, + "tests": [ + { + "description": "match", + "data": {"bar": 3}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": true}, + "valid": false + } + ] + }, + { + "description": "relative pointer ref to array", + "schema": { + "items": [ + {"type": "integer"}, + {"$ref": "#/items/0"} + ] + }, + "tests": [ + { + "description": "match array", + "data": [1, 2], + "valid": true + }, + { + "description": "mismatch array", + "data": [1, "foo"], + "valid": false + } + ] + }, + { + "description": "escaped pointer ref", + "schema": { + "definitions": { + "tilde~field": {"type": "integer"}, + "slash/field": {"type": "integer"}, + "percent%field": {"type": "integer"} + }, + "properties": { + "tilde": {"$ref": "#/definitions/tilde~0field"}, + "slash": {"$ref": "#/definitions/slash~1field"}, + "percent": {"$ref": "#/definitions/percent%25field"} + } + }, + "tests": [ + { + "description": "slash invalid", + "data": {"slash": "aoeu"}, + "valid": false + }, + { + "description": "tilde invalid", + "data": {"tilde": "aoeu"}, + "valid": false + }, + { + "description": "percent invalid", + "data": {"percent": "aoeu"}, + "valid": false + }, + { + "description": "slash valid", + "data": {"slash": 123}, + "valid": true + }, + { + "description": "tilde valid", + "data": {"tilde": 123}, + "valid": true + }, + { + "description": "percent valid", + "data": {"percent": 123}, + "valid": true + } + ] + }, + { + "description": "nested refs", + "schema": { + "definitions": { + "a": {"type": "integer"}, + "b": {"$ref": "#/definitions/a"}, + "c": {"$ref": "#/definitions/b"} + }, + "allOf": [{ "$ref": "#/definitions/c" }] + }, + "tests": [ + { + "description": "nested ref valid", + "data": 5, + "valid": true + }, + { + "description": "nested ref invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "ref overrides any sibling keywords", + "schema": { + "definitions": { + "reffed": { + "type": "array" + } + }, + "properties": { + "foo": { + "$ref": "#/definitions/reffed", + "maxItems": 2 + } + } + }, + "tests": [ + { + "description": "ref valid", + "data": { "foo": [] }, + "valid": true + }, + { + "description": "ref valid, maxItems ignored", + "data": { "foo": [ 1, 2, 3] }, + "valid": true + }, + { + "description": "ref invalid", + "data": { "foo": "string" }, + "valid": false + } + ] + }, + { + "description": "$ref prevents a sibling $id from changing the base uri", + "schema": { + "$id": "http://localhost:1234/sibling_id/base/", + "definitions": { + "foo": { + "$id": "http://localhost:1234/sibling_id/foo.json", + "type": "string" + }, + "base_foo": { + "$comment": "this canonical uri is http://localhost:1234/sibling_id/base/foo.json", + "$id": "foo.json", + "type": "number" + } + }, + "allOf": [ + { + "$comment": "$ref resolves to http://localhost:1234/sibling_id/base/foo.json, not http://localhost:1234/sibling_id/foo.json", + "$id": "http://localhost:1234/sibling_id/", + "$ref": "foo.json" + } + ] + }, + "tests": [ + { + "description": "$ref resolves to /definitions/base_foo, data does not validate", + "data": "a", + "valid": false + }, + { + "description": "$ref resolves to /definitions/base_foo, data validates", + "data": 1, + "valid": true + } + ] + }, + { + "description": "remote ref, containing refs itself", + "schema": {"$ref": "http://json-schema.org/draft-07/schema#"}, + "tests": [ + { + "description": "remote ref valid", + "data": {"minLength": 1}, + "valid": true + }, + { + "description": "remote ref invalid", + "data": {"minLength": -1}, + "valid": false + } + ] + }, + { + "description": "property named $ref that is not a reference", + "schema": { + "properties": { + "$ref": {"type": "string"} + } + }, + "tests": [ + { + "description": "property named $ref valid", + "data": {"$ref": "a"}, + "valid": true + }, + { + "description": "property named $ref invalid", + "data": {"$ref": 2}, + "valid": false + } + ] + }, + { + "description": "property named $ref, containing an actual $ref", + "schema": { + "properties": { + "$ref": {"$ref": "#/definitions/is-string"} + }, + "definitions": { + "is-string": { + "type": "string" + } + } + }, + "tests": [ + { + "description": "property named $ref valid", + "data": {"$ref": "a"}, + "valid": true + }, + { + "description": "property named $ref invalid", + "data": {"$ref": 2}, + "valid": false + } + ] + }, + { + "description": "$ref to boolean schema true", + "schema": { + "allOf": [{ "$ref": "#/definitions/bool" }], + "definitions": { + "bool": true + } + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "$ref to boolean schema false", + "schema": { + "allOf": [{ "$ref": "#/definitions/bool" }], + "definitions": { + "bool": false + } + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "Recursive references between schemas", + "schema": { + "$id": "http://localhost:1234/tree", + "description": "tree of nodes", + "type": "object", + "properties": { + "meta": {"type": "string"}, + "nodes": { + "type": "array", + "items": {"$ref": "node"} + } + }, + "required": ["meta", "nodes"], + "definitions": { + "node": { + "$id": "http://localhost:1234/node", + "description": "node", + "type": "object", + "properties": { + "value": {"type": "number"}, + "subtree": {"$ref": "tree"} + }, + "required": ["value"] + } + } + }, + "tests": [ + { + "description": "valid tree", + "data": { + "meta": "root", + "nodes": [ + { + "value": 1, + "subtree": { + "meta": "child", + "nodes": [ + {"value": 1.1}, + {"value": 1.2} + ] + } + }, + { + "value": 2, + "subtree": { + "meta": "child", + "nodes": [ + {"value": 2.1}, + {"value": 2.2} + ] + } + } + ] + }, + "valid": true + }, + { + "description": "invalid tree", + "data": { + "meta": "root", + "nodes": [ + { + "value": 1, + "subtree": { + "meta": "child", + "nodes": [ + {"value": "string is invalid"}, + {"value": 1.2} + ] + } + }, + { + "value": 2, + "subtree": { + "meta": "child", + "nodes": [ + {"value": 2.1}, + {"value": 2.2} + ] + } + } + ] + }, + "valid": false + } + ] + }, + { + "description": "refs with quote", + "schema": { + "properties": { + "foo\"bar": {"$ref": "#/definitions/foo%22bar"} + }, + "definitions": { + "foo\"bar": {"type": "number"} + } + }, + "tests": [ + { + "description": "object with numbers is valid", + "data": { + "foo\"bar": 1 + }, + "valid": true + }, + { + "description": "object with strings is invalid", + "data": { + "foo\"bar": "1" + }, + "valid": false + } + ] + }, + { + "description": "Location-independent identifier", + "schema": { + "allOf": [{ + "$ref": "#foo" + }], + "definitions": { + "A": { + "$id": "#foo", + "type": "integer" + } + } + }, + "tests": [ + { + "data": 1, + "description": "match", + "valid": true + }, + { + "data": "a", + "description": "mismatch", + "valid": false + } + ] + }, + { + "description": "Reference an anchor with a non-relative URI", + "schema": { + "$id": "https://example.com/schema-with-anchor", + "allOf": [{ + "$ref": "https://example.com/schema-with-anchor#foo" + }], + "definitions": { + "A": { + "$id": "#foo", + "type": "integer" + } + } + }, + "tests": [ + { + "data": 1, + "description": "match", + "valid": true + }, + { + "data": "a", + "description": "mismatch", + "valid": false + } + ] + }, + { + "description": "Location-independent identifier with base URI change in subschema", + "schema": { + "$id": "http://localhost:1234/root", + "allOf": [{ + "$ref": "http://localhost:1234/nested.json#foo" + }], + "definitions": { + "A": { + "$id": "nested.json", + "definitions": { + "B": { + "$id": "#foo", + "type": "integer" + } + } + } + } + }, + "tests": [ + { + "data": 1, + "description": "match", + "valid": true + }, + { + "data": "a", + "description": "mismatch", + "valid": false + } + ] + }, + { + "description": "naive replacement of $ref with its destination is not correct", + "schema": { + "definitions": { + "a_string": { "type": "string" } + }, + "enum": [ + { "$ref": "#/definitions/a_string" } + ] + }, + "tests": [ + { + "description": "do not evaluate the $ref inside the enum, matching any string", + "data": "this is a string", + "valid": false + }, + { + "description": "do not evaluate the $ref inside the enum, definition exact match", + "data": { "type": "string" }, + "valid": false + }, + { + "description": "match the enum exactly", + "data": { "$ref": "#/definitions/a_string" }, + "valid": true + } + ] + }, + { + "description": "refs with relative uris and defs", + "schema": { + "$id": "http://example.com/schema-relative-uri-defs1.json", + "properties": { + "foo": { + "$id": "schema-relative-uri-defs2.json", + "definitions": { + "inner": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "allOf": [ { "$ref": "#/definitions/inner" } ] + } + }, + "allOf": [ { "$ref": "schema-relative-uri-defs2.json" } ] + }, + "tests": [ + { + "description": "invalid on inner field", + "data": { + "foo": { + "bar": 1 + }, + "bar": "a" + }, + "valid": false + }, + { + "description": "invalid on outer field", + "data": { + "foo": { + "bar": "a" + }, + "bar": 1 + }, + "valid": false + }, + { + "description": "valid on both fields", + "data": { + "foo": { + "bar": "a" + }, + "bar": "a" + }, + "valid": true + } + ] + }, + { + "description": "relative refs with absolute uris and defs", + "schema": { + "$id": "http://example.com/schema-refs-absolute-uris-defs1.json", + "properties": { + "foo": { + "$id": "http://example.com/schema-refs-absolute-uris-defs2.json", + "definitions": { + "inner": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "allOf": [ { "$ref": "#/definitions/inner" } ] + } + }, + "allOf": [ { "$ref": "schema-refs-absolute-uris-defs2.json" } ] + }, + "tests": [ + { + "description": "invalid on inner field", + "data": { + "foo": { + "bar": 1 + }, + "bar": "a" + }, + "valid": false + }, + { + "description": "invalid on outer field", + "data": { + "foo": { + "bar": "a" + }, + "bar": 1 + }, + "valid": false + }, + { + "description": "valid on both fields", + "data": { + "foo": { + "bar": "a" + }, + "bar": "a" + }, + "valid": true + } + ] + }, + { + "description": "$id must be resolved against nearest parent, not just immediate parent", + "schema": { + "$id": "http://example.com/a.json", + "definitions": { + "x": { + "$id": "http://example.com/b/c.json", + "not": { + "definitions": { + "y": { + "$id": "d.json", + "type": "number" + } + } + } + } + }, + "allOf": [ + { + "$ref": "http://example.com/b/d.json" + } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "simple URN base URI with $ref via the URN", + "schema": { + "$comment": "URIs do not have to have HTTP(s) schemes", + "$id": "urn:uuid:deadbeef-1234-ffff-ffff-4321feebdaed", + "minimum": 30, + "properties": { + "foo": {"$ref": "urn:uuid:deadbeef-1234-ffff-ffff-4321feebdaed"} + } + }, + "tests": [ + { + "description": "valid under the URN IDed schema", + "data": {"foo": 37}, + "valid": true + }, + { + "description": "invalid under the URN IDed schema", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "simple URN base URI with JSON pointer", + "schema": { + "$comment": "URIs do not have to have HTTP(s) schemes", + "$id": "urn:uuid:deadbeef-1234-00ff-ff00-4321feebdaed", + "properties": { + "foo": {"$ref": "#/definitions/bar"} + }, + "definitions": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with NSS", + "schema": { + "$comment": "RFC 8141 §2.2", + "$id": "urn:example:1/406/47452/2", + "properties": { + "foo": {"$ref": "#/definitions/bar"} + }, + "definitions": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with r-component", + "schema": { + "$comment": "RFC 8141 §2.3.1", + "$id": "urn:example:foo-bar-baz-qux?+CCResolve:cc=uk", + "properties": { + "foo": {"$ref": "#/definitions/bar"} + }, + "definitions": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with q-component", + "schema": { + "$comment": "RFC 8141 §2.3.2", + "$id": "urn:example:weather?=op=map&lat=39.56&lon=-104.85&datetime=1969-07-21T02:56:15Z", + "properties": { + "foo": {"$ref": "#/definitions/bar"} + }, + "definitions": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with URN and JSON pointer ref", + "schema": { + "$id": "urn:uuid:deadbeef-1234-0000-0000-4321feebdaed", + "properties": { + "foo": {"$ref": "urn:uuid:deadbeef-1234-0000-0000-4321feebdaed#/definitions/bar"} + }, + "definitions": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with URN and anchor ref", + "schema": { + "$id": "urn:uuid:deadbeef-1234-ff00-00ff-4321feebdaed", + "properties": { + "foo": {"$ref": "urn:uuid:deadbeef-1234-ff00-00ff-4321feebdaed#something"} + }, + "definitions": { + "bar": { + "$id": "#something", + "type": "string" + } + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "ref to if", + "schema": { + "allOf": [ + {"$ref": "http://example.com/ref/if"}, + { + "if": { + "$id": "http://example.com/ref/if", + "type": "integer" + } + } + ] + }, + "tests": [ + { + "description": "a non-integer is invalid due to the $ref", + "data": "foo", + "valid": false + }, + { + "description": "an integer is valid", + "data": 12, + "valid": true + } + ] + }, + { + "description": "ref to then", + "schema": { + "allOf": [ + {"$ref": "http://example.com/ref/then"}, + { + "then": { + "$id": "http://example.com/ref/then", + "type": "integer" + } + } + ] + }, + "tests": [ + { + "description": "a non-integer is invalid due to the $ref", + "data": "foo", + "valid": false + }, + { + "description": "an integer is valid", + "data": 12, + "valid": true + } + ] + }, + { + "description": "ref to else", + "schema": { + "allOf": [ + {"$ref": "http://example.com/ref/else"}, + { + "else": { + "$id": "http://example.com/ref/else", + "type": "integer" + } + } + ] + }, + "tests": [ + { + "description": "a non-integer is invalid due to the $ref", + "data": "foo", + "valid": false + }, + { + "description": "an integer is valid", + "data": 12, + "valid": true + } + ] + }, + { + "description": "ref with absolute-path-reference", + "schema": { + "$id": "http://example.com/ref/absref.json", + "definitions": { + "a": { + "$id": "http://example.com/ref/absref/foobar.json", + "type": "number" + }, + "b": { + "$id": "http://example.com/absref/foobar.json", + "type": "string" + } + }, + "allOf": [ + { "$ref": "/absref/foobar.json" } + ] + }, + "tests": [ + { + "description": "a string is valid", + "data": "foo", + "valid": true + }, + { + "description": "an integer is invalid", + "data": 12, + "valid": false + } + ] + }, + { + "description": "$id with file URI still resolves pointers - *nix", + "schema": { + "$id": "file:///folder/file.json", + "definitions": { + "foo": { + "type": "number" + } + }, + "allOf": [ + { + "$ref": "#/definitions/foo" + } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "$id with file URI still resolves pointers - windows", + "schema": { + "$id": "file:///c:/folder/file.json", + "definitions": { + "foo": { + "type": "number" + } + }, + "allOf": [ + { + "$ref": "#/definitions/foo" + } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "empty tokens in $ref json-pointer", + "schema": { + "definitions": { + "": { + "definitions": { + "": { "type": "number" } + } + } + }, + "allOf": [ + { + "$ref": "#/definitions//definitions/" + } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/refRemote.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/refRemote.json new file mode 100644 index 000000000..450787af6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/refRemote.json @@ -0,0 +1,257 @@ +[ + { + "description": "remote ref", + "schema": {"$ref": "http://localhost:1234/integer.json"}, + "tests": [ + { + "description": "remote ref valid", + "data": 1, + "valid": true + }, + { + "description": "remote ref invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "fragment within remote ref", + "schema": {"$ref": "http://localhost:1234/draft7/subSchemas.json#/definitions/integer"}, + "tests": [ + { + "description": "remote fragment valid", + "data": 1, + "valid": true + }, + { + "description": "remote fragment invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "ref within remote ref", + "schema": { + "$ref": "http://localhost:1234/draft7/subSchemas.json#/definitions/refToInteger" + }, + "tests": [ + { + "description": "ref within ref valid", + "data": 1, + "valid": true + }, + { + "description": "ref within ref invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "base URI change", + "schema": { + "$id": "http://localhost:1234/", + "items": { + "$id": "baseUriChange/", + "items": {"$ref": "folderInteger.json"} + } + }, + "tests": [ + { + "description": "base URI change ref valid", + "data": [[1]], + "valid": true + }, + { + "description": "base URI change ref invalid", + "data": [["a"]], + "valid": false + } + ] + }, + { + "description": "base URI change - change folder", + "schema": { + "$id": "http://localhost:1234/scope_change_defs1.json", + "type" : "object", + "properties": { + "list": {"$ref": "#/definitions/baz"} + }, + "definitions": { + "baz": { + "$id": "baseUriChangeFolder/", + "type": "array", + "items": {"$ref": "folderInteger.json"} + } + } + }, + "tests": [ + { + "description": "number is valid", + "data": {"list": [1]}, + "valid": true + }, + { + "description": "string is invalid", + "data": {"list": ["a"]}, + "valid": false + } + ] + }, + { + "description": "base URI change - change folder in subschema", + "schema": { + "$id": "http://localhost:1234/scope_change_defs2.json", + "type" : "object", + "properties": { + "list": {"$ref": "#/definitions/baz/definitions/bar"} + }, + "definitions": { + "baz": { + "$id": "baseUriChangeFolderInSubschema/", + "definitions": { + "bar": { + "type": "array", + "items": {"$ref": "folderInteger.json"} + } + } + } + } + }, + "tests": [ + { + "description": "number is valid", + "data": {"list": [1]}, + "valid": true + }, + { + "description": "string is invalid", + "data": {"list": ["a"]}, + "valid": false + } + ] + }, + { + "description": "root ref in remote ref", + "schema": { + "$id": "http://localhost:1234/object", + "type": "object", + "properties": { + "name": {"$ref": "draft7/name.json#/definitions/orNull"} + } + }, + "tests": [ + { + "description": "string is valid", + "data": { + "name": "foo" + }, + "valid": true + }, + { + "description": "null is valid", + "data": { + "name": null + }, + "valid": true + }, + { + "description": "object is invalid", + "data": { + "name": { + "name": null + } + }, + "valid": false + } + ] + }, + { + "description": "remote ref with ref to definitions", + "schema": { + "$id": "http://localhost:1234/schema-remote-ref-ref-defs1.json", + "allOf": [ + { "$ref": "draft7/ref-and-definitions.json" } + ] + }, + "tests": [ + { + "description": "invalid", + "data": { + "bar": 1 + }, + "valid": false + }, + { + "description": "valid", + "data": { + "bar": "a" + }, + "valid": true + } + ] + }, + { + "description": "Location-independent identifier in remote ref", + "schema": { + "$ref": "http://localhost:1234/draft7/locationIndependentIdentifier.json#/definitions/refToInteger" + }, + "tests": [ + { + "description": "integer is valid", + "data": 1, + "valid": true + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "retrieved nested refs resolve relative to their URI not $id", + "schema": { + "$id": "http://localhost:1234/some-id", + "properties": { + "name": {"$ref": "nested/foo-ref-string.json"} + } + }, + "tests": [ + { + "description": "number is invalid", + "data": { + "name": {"foo": 1} + }, + "valid": false + }, + { + "description": "string is valid", + "data": { + "name": {"foo": "a"} + }, + "valid": true + } + ] + }, + { + "description": "$ref to $ref finds location-independent $id", + "schema": { + "$ref": "http://localhost:1234/draft7/detached-ref.json#/definitions/foo" + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/required.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/required.json new file mode 100644 index 000000000..f98c64af8 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/required.json @@ -0,0 +1,161 @@ +[ + { + "description": "required validation", + "schema": { + "properties": { + "foo": {}, + "bar": {} + }, + "required": ["foo"] + }, + "tests": [ + { + "description": "present required property is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "non-present required property is invalid", + "data": {"bar": 1}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores strings", + "data": "", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + }, + { + "description": "ignores boolean", + "data": true, + "valid": true + } + ] + }, + { + "description": "required default validation", + "schema": { + "properties": { + "foo": {} + } + }, + "tests": [ + { + "description": "not required by default", + "data": {}, + "valid": true + } + ] + }, + { + "description": "required with empty array", + "schema": { + "properties": { + "foo": {} + }, + "required": [] + }, + "tests": [ + { + "description": "property not required", + "data": {}, + "valid": true + } + ] + }, + { + "description": "required with escaped characters", + "schema": { + "required": [ + "foo\nbar", + "foo\"bar", + "foo\\bar", + "foo\rbar", + "foo\tbar", + "foo\fbar" + ] + }, + "tests": [ + { + "description": "object with all properties present is valid", + "data": { + "foo\nbar": 1, + "foo\"bar": 1, + "foo\\bar": 1, + "foo\rbar": 1, + "foo\tbar": 1, + "foo\fbar": 1 + }, + "valid": true + }, + { + "description": "object with some properties missing is invalid", + "data": { + "foo\nbar": "1", + "foo\"bar": "1" + }, + "valid": false + } + ] + }, + { + "description": "required properties whose names are Javascript object property names", + "comment": "Ensure JS implementations don't universally consider e.g. __proto__ to always be present in an object.", + "schema": { "required": ["__proto__", "toString", "constructor"] }, + "tests": [ + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "none of the properties mentioned", + "data": {}, + "valid": false + }, + { + "description": "__proto__ present", + "data": { "__proto__": "foo" }, + "valid": false + }, + { + "description": "toString present", + "data": { "toString": { "length": 37 } }, + "valid": false + }, + { + "description": "constructor present", + "data": { "constructor": { "length": 37 } }, + "valid": false + }, + { + "description": "all present", + "data": { + "__proto__": 12, + "toString": { "length": "foo" }, + "constructor": 37 + }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/type.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/type.json new file mode 100644 index 000000000..830464702 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/type.json @@ -0,0 +1,474 @@ +[ + { + "description": "integer type matches integers", + "schema": {"type": "integer"}, + "tests": [ + { + "description": "an integer is an integer", + "data": 1, + "valid": true + }, + { + "description": "a float with zero fractional part is an integer", + "data": 1.0, + "valid": true + }, + { + "description": "a float is not an integer", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not an integer", + "data": "foo", + "valid": false + }, + { + "description": "a string is still not an integer, even if it looks like one", + "data": "1", + "valid": false + }, + { + "description": "an object is not an integer", + "data": {}, + "valid": false + }, + { + "description": "an array is not an integer", + "data": [], + "valid": false + }, + { + "description": "a boolean is not an integer", + "data": true, + "valid": false + }, + { + "description": "null is not an integer", + "data": null, + "valid": false + } + ] + }, + { + "description": "number type matches numbers", + "schema": {"type": "number"}, + "tests": [ + { + "description": "an integer is a number", + "data": 1, + "valid": true + }, + { + "description": "a float with zero fractional part is a number (and an integer)", + "data": 1.0, + "valid": true + }, + { + "description": "a float is a number", + "data": 1.1, + "valid": true + }, + { + "description": "a string is not a number", + "data": "foo", + "valid": false + }, + { + "description": "a string is still not a number, even if it looks like one", + "data": "1", + "valid": false + }, + { + "description": "an object is not a number", + "data": {}, + "valid": false + }, + { + "description": "an array is not a number", + "data": [], + "valid": false + }, + { + "description": "a boolean is not a number", + "data": true, + "valid": false + }, + { + "description": "null is not a number", + "data": null, + "valid": false + } + ] + }, + { + "description": "string type matches strings", + "schema": {"type": "string"}, + "tests": [ + { + "description": "1 is not a string", + "data": 1, + "valid": false + }, + { + "description": "a float is not a string", + "data": 1.1, + "valid": false + }, + { + "description": "a string is a string", + "data": "foo", + "valid": true + }, + { + "description": "a string is still a string, even if it looks like a number", + "data": "1", + "valid": true + }, + { + "description": "an empty string is still a string", + "data": "", + "valid": true + }, + { + "description": "an object is not a string", + "data": {}, + "valid": false + }, + { + "description": "an array is not a string", + "data": [], + "valid": false + }, + { + "description": "a boolean is not a string", + "data": true, + "valid": false + }, + { + "description": "null is not a string", + "data": null, + "valid": false + } + ] + }, + { + "description": "object type matches objects", + "schema": {"type": "object"}, + "tests": [ + { + "description": "an integer is not an object", + "data": 1, + "valid": false + }, + { + "description": "a float is not an object", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not an object", + "data": "foo", + "valid": false + }, + { + "description": "an object is an object", + "data": {}, + "valid": true + }, + { + "description": "an array is not an object", + "data": [], + "valid": false + }, + { + "description": "a boolean is not an object", + "data": true, + "valid": false + }, + { + "description": "null is not an object", + "data": null, + "valid": false + } + ] + }, + { + "description": "array type matches arrays", + "schema": {"type": "array"}, + "tests": [ + { + "description": "an integer is not an array", + "data": 1, + "valid": false + }, + { + "description": "a float is not an array", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not an array", + "data": "foo", + "valid": false + }, + { + "description": "an object is not an array", + "data": {}, + "valid": false + }, + { + "description": "an array is an array", + "data": [], + "valid": true + }, + { + "description": "a boolean is not an array", + "data": true, + "valid": false + }, + { + "description": "null is not an array", + "data": null, + "valid": false + } + ] + }, + { + "description": "boolean type matches booleans", + "schema": {"type": "boolean"}, + "tests": [ + { + "description": "an integer is not a boolean", + "data": 1, + "valid": false + }, + { + "description": "zero is not a boolean", + "data": 0, + "valid": false + }, + { + "description": "a float is not a boolean", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not a boolean", + "data": "foo", + "valid": false + }, + { + "description": "an empty string is not a boolean", + "data": "", + "valid": false + }, + { + "description": "an object is not a boolean", + "data": {}, + "valid": false + }, + { + "description": "an array is not a boolean", + "data": [], + "valid": false + }, + { + "description": "true is a boolean", + "data": true, + "valid": true + }, + { + "description": "false is a boolean", + "data": false, + "valid": true + }, + { + "description": "null is not a boolean", + "data": null, + "valid": false + } + ] + }, + { + "description": "null type matches only the null object", + "schema": {"type": "null"}, + "tests": [ + { + "description": "an integer is not null", + "data": 1, + "valid": false + }, + { + "description": "a float is not null", + "data": 1.1, + "valid": false + }, + { + "description": "zero is not null", + "data": 0, + "valid": false + }, + { + "description": "a string is not null", + "data": "foo", + "valid": false + }, + { + "description": "an empty string is not null", + "data": "", + "valid": false + }, + { + "description": "an object is not null", + "data": {}, + "valid": false + }, + { + "description": "an array is not null", + "data": [], + "valid": false + }, + { + "description": "true is not null", + "data": true, + "valid": false + }, + { + "description": "false is not null", + "data": false, + "valid": false + }, + { + "description": "null is null", + "data": null, + "valid": true + } + ] + }, + { + "description": "multiple types can be specified in an array", + "schema": {"type": ["integer", "string"]}, + "tests": [ + { + "description": "an integer is valid", + "data": 1, + "valid": true + }, + { + "description": "a string is valid", + "data": "foo", + "valid": true + }, + { + "description": "a float is invalid", + "data": 1.1, + "valid": false + }, + { + "description": "an object is invalid", + "data": {}, + "valid": false + }, + { + "description": "an array is invalid", + "data": [], + "valid": false + }, + { + "description": "a boolean is invalid", + "data": true, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + } + ] + }, + { + "description": "type as array with one item", + "schema": { + "type": ["string"] + }, + "tests": [ + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "number is invalid", + "data": 123, + "valid": false + } + ] + }, + { + "description": "type: array or object", + "schema": { + "type": ["array", "object"] + }, + "tests": [ + { + "description": "array is valid", + "data": [1,2,3], + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": 123}, + "valid": true + }, + { + "description": "number is invalid", + "data": 123, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + } + ] + }, + { + "description": "type: array, object or null", + "schema": { + "type": ["array", "object", "null"] + }, + "tests": [ + { + "description": "array is valid", + "data": [1,2,3], + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": 123}, + "valid": true + }, + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "number is invalid", + "data": 123, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/uniqueItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/uniqueItems.json new file mode 100644 index 000000000..a1da0096e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/uniqueItems.json @@ -0,0 +1,389 @@ +[ + { + "description": "uniqueItems validation", + "schema": {"uniqueItems": true}, + "tests": [ + { + "description": "unique array of integers is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "non-unique array of integers is invalid", + "data": [1, 1], + "valid": false + }, + { + "description": "non-unique array of more than two integers is invalid", + "data": [1, 2, 1], + "valid": false + }, + { + "description": "numbers are unique if mathematically unequal", + "data": [1.0, 1.00, 1], + "valid": false + }, + { + "description": "false is not equal to zero", + "data": [0, false], + "valid": true + }, + { + "description": "true is not equal to one", + "data": [1, true], + "valid": true + }, + { + "description": "unique array of strings is valid", + "data": ["foo", "bar", "baz"], + "valid": true + }, + { + "description": "non-unique array of strings is invalid", + "data": ["foo", "bar", "foo"], + "valid": false + }, + { + "description": "unique array of objects is valid", + "data": [{"foo": "bar"}, {"foo": "baz"}], + "valid": true + }, + { + "description": "non-unique array of objects is invalid", + "data": [{"foo": "bar"}, {"foo": "bar"}], + "valid": false + }, + { + "description": "property order of array of objects is ignored", + "data": [{"foo": "bar", "bar": "foo"}, {"bar": "foo", "foo": "bar"}], + "valid": false + }, + { + "description": "unique array of nested objects is valid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : false}}} + ], + "valid": true + }, + { + "description": "non-unique array of nested objects is invalid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : true}}} + ], + "valid": false + }, + { + "description": "unique array of arrays is valid", + "data": [["foo"], ["bar"]], + "valid": true + }, + { + "description": "non-unique array of arrays is invalid", + "data": [["foo"], ["foo"]], + "valid": false + }, + { + "description": "non-unique array of more than two arrays is invalid", + "data": [["foo"], ["bar"], ["foo"]], + "valid": false + }, + { + "description": "[1] and [true] are unique", + "data": [[1], [true]], + "valid": true + }, + { + "description": "[0] and [false] are unique", + "data": [[0], [false]], + "valid": true + }, + { + "description": "nested [1] and [true] are unique", + "data": [[[1], "foo"], [[true], "foo"]], + "valid": true + }, + { + "description": "nested [0] and [false] are unique", + "data": [[[0], "foo"], [[false], "foo"]], + "valid": true + }, + { + "description": "unique heterogeneous types are valid", + "data": [{}, [1], true, null, 1, "{}"], + "valid": true + }, + { + "description": "non-unique heterogeneous types are invalid", + "data": [{}, [1], true, null, {}, 1], + "valid": false + }, + { + "description": "different objects are unique", + "data": [{"a": 1, "b": 2}, {"a": 2, "b": 1}], + "valid": true + }, + { + "description": "objects are non-unique despite key order", + "data": [{"a": 1, "b": 2}, {"b": 2, "a": 1}], + "valid": false + }, + { + "description": "{\"a\": false} and {\"a\": 0} are unique", + "data": [{"a": false}, {"a": 0}], + "valid": true + }, + { + "description": "{\"a\": true} and {\"a\": 1} are unique", + "data": [{"a": true}, {"a": 1}], + "valid": true + } + ] + }, + { + "description": "uniqueItems with an array of items", + "schema": { + "items": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": true + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is not valid", + "data": [false, false], + "valid": false + }, + { + "description": "[true, true] from items array is not valid", + "data": [true, true], + "valid": false + }, + { + "description": "unique array extended from [false, true] is valid", + "data": [false, true, "foo", "bar"], + "valid": true + }, + { + "description": "unique array extended from [true, false] is valid", + "data": [true, false, "foo", "bar"], + "valid": true + }, + { + "description": "non-unique array extended from [false, true] is not valid", + "data": [false, true, "foo", "foo"], + "valid": false + }, + { + "description": "non-unique array extended from [true, false] is not valid", + "data": [true, false, "foo", "foo"], + "valid": false + } + ] + }, + { + "description": "uniqueItems with an array of items and additionalItems=false", + "schema": { + "items": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": true, + "additionalItems": false + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is not valid", + "data": [false, false], + "valid": false + }, + { + "description": "[true, true] from items array is not valid", + "data": [true, true], + "valid": false + }, + { + "description": "extra items are invalid even if unique", + "data": [false, true, null], + "valid": false + } + ] + }, + { + "description": "uniqueItems=false validation", + "schema": { "uniqueItems": false }, + "tests": [ + { + "description": "unique array of integers is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "non-unique array of integers is valid", + "data": [1, 1], + "valid": true + }, + { + "description": "numbers are unique if mathematically unequal", + "data": [1.0, 1.00, 1], + "valid": true + }, + { + "description": "false is not equal to zero", + "data": [0, false], + "valid": true + }, + { + "description": "true is not equal to one", + "data": [1, true], + "valid": true + }, + { + "description": "unique array of objects is valid", + "data": [{"foo": "bar"}, {"foo": "baz"}], + "valid": true + }, + { + "description": "non-unique array of objects is valid", + "data": [{"foo": "bar"}, {"foo": "bar"}], + "valid": true + }, + { + "description": "unique array of nested objects is valid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : false}}} + ], + "valid": true + }, + { + "description": "non-unique array of nested objects is valid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : true}}} + ], + "valid": true + }, + { + "description": "unique array of arrays is valid", + "data": [["foo"], ["bar"]], + "valid": true + }, + { + "description": "non-unique array of arrays is valid", + "data": [["foo"], ["foo"]], + "valid": true + }, + { + "description": "unique heterogeneous types are valid", + "data": [{}, [1], true, null, 1], + "valid": true + }, + { + "description": "non-unique heterogeneous types are valid", + "data": [{}, [1], true, null, {}, 1], + "valid": true + } + ] + }, + { + "description": "uniqueItems=false with an array of items", + "schema": { + "items": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": false + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is valid", + "data": [false, false], + "valid": true + }, + { + "description": "[true, true] from items array is valid", + "data": [true, true], + "valid": true + }, + { + "description": "unique array extended from [false, true] is valid", + "data": [false, true, "foo", "bar"], + "valid": true + }, + { + "description": "unique array extended from [true, false] is valid", + "data": [true, false, "foo", "bar"], + "valid": true + }, + { + "description": "non-unique array extended from [false, true] is valid", + "data": [false, true, "foo", "foo"], + "valid": true + }, + { + "description": "non-unique array extended from [true, false] is valid", + "data": [true, false, "foo", "foo"], + "valid": true + } + ] + }, + { + "description": "uniqueItems=false with an array of items and additionalItems=false", + "schema": { + "items": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": false, + "additionalItems": false + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is valid", + "data": [false, false], + "valid": true + }, + { + "description": "[true, true] from items array is valid", + "data": [true, true], + "valid": true + }, + { + "description": "extra items are invalid even if unique", + "data": [false, true, null], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/meta-schemas/README.md b/packages/schema/test/fixtures/meta-schemas/README.md new file mode 100644 index 000000000..46b120d97 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/README.md @@ -0,0 +1,10 @@ +# Meta-schemas + +The official meta-schemas some suite tests reference, unmodified: + +- `draft2020-12-schema.json`: https://json-schema.org/draft/2020-12/schema +- `draft2020-12-meta-*.json`: https://json-schema.org/draft/2020-12/meta/{core,applicator,unevaluated,validation,meta-data,format-annotation,content} +- `draft-07-schema.json`: http://json-schema.org/draft-07/schema + +Published by the JSON Schema organization (https://github.com/json-schema-org/json-schema-spec), licensed +AFL-3.0 or BSD-3-Clause. diff --git a/packages/schema/test/fixtures/meta-schemas/draft-07-schema.json b/packages/schema/test/fixtures/meta-schemas/draft-07-schema.json new file mode 100644 index 000000000..fb92c7f75 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft-07-schema.json @@ -0,0 +1,172 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://json-schema.org/draft-07/schema#", + "title": "Core schema meta-schema", + "definitions": { + "schemaArray": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#" } + }, + "nonNegativeInteger": { + "type": "integer", + "minimum": 0 + }, + "nonNegativeIntegerDefault0": { + "allOf": [ + { "$ref": "#/definitions/nonNegativeInteger" }, + { "default": 0 } + ] + }, + "simpleTypes": { + "enum": [ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string" + ] + }, + "stringArray": { + "type": "array", + "items": { "type": "string" }, + "uniqueItems": true, + "default": [] + } + }, + "type": ["object", "boolean"], + "properties": { + "$id": { + "type": "string", + "format": "uri-reference" + }, + "$schema": { + "type": "string", + "format": "uri" + }, + "$ref": { + "type": "string", + "format": "uri-reference" + }, + "$comment": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "default": true, + "readOnly": { + "type": "boolean", + "default": false + }, + "writeOnly": { + "type": "boolean", + "default": false + }, + "examples": { + "type": "array", + "items": true + }, + "multipleOf": { + "type": "number", + "exclusiveMinimum": 0 + }, + "maximum": { + "type": "number" + }, + "exclusiveMaximum": { + "type": "number" + }, + "minimum": { + "type": "number" + }, + "exclusiveMinimum": { + "type": "number" + }, + "maxLength": { "$ref": "#/definitions/nonNegativeInteger" }, + "minLength": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "pattern": { + "type": "string", + "format": "regex" + }, + "additionalItems": { "$ref": "#" }, + "items": { + "anyOf": [ + { "$ref": "#" }, + { "$ref": "#/definitions/schemaArray" } + ], + "default": true + }, + "maxItems": { "$ref": "#/definitions/nonNegativeInteger" }, + "minItems": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "contains": { "$ref": "#" }, + "maxProperties": { "$ref": "#/definitions/nonNegativeInteger" }, + "minProperties": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "required": { "$ref": "#/definitions/stringArray" }, + "additionalProperties": { "$ref": "#" }, + "definitions": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "properties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "patternProperties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "propertyNames": { "format": "regex" }, + "default": {} + }, + "dependencies": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { "$ref": "#" }, + { "$ref": "#/definitions/stringArray" } + ] + } + }, + "propertyNames": { "$ref": "#" }, + "const": true, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + }, + "type": { + "anyOf": [ + { "$ref": "#/definitions/simpleTypes" }, + { + "type": "array", + "items": { "$ref": "#/definitions/simpleTypes" }, + "minItems": 1, + "uniqueItems": true + } + ] + }, + "format": { "type": "string" }, + "contentMediaType": { "type": "string" }, + "contentEncoding": { "type": "string" }, + "if": { "$ref": "#" }, + "then": { "$ref": "#" }, + "else": { "$ref": "#" }, + "allOf": { "$ref": "#/definitions/schemaArray" }, + "anyOf": { "$ref": "#/definitions/schemaArray" }, + "oneOf": { "$ref": "#/definitions/schemaArray" }, + "not": { "$ref": "#" } + }, + "default": true +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-applicator.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-applicator.json new file mode 100644 index 000000000..f4775974a --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-applicator.json @@ -0,0 +1,45 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/applicator", + "$dynamicAnchor": "meta", + + "title": "Applicator vocabulary meta-schema", + "type": ["object", "boolean"], + "properties": { + "prefixItems": { "$ref": "#/$defs/schemaArray" }, + "items": { "$dynamicRef": "#meta" }, + "contains": { "$dynamicRef": "#meta" }, + "additionalProperties": { "$dynamicRef": "#meta" }, + "properties": { + "type": "object", + "additionalProperties": { "$dynamicRef": "#meta" }, + "default": {} + }, + "patternProperties": { + "type": "object", + "additionalProperties": { "$dynamicRef": "#meta" }, + "propertyNames": { "format": "regex" }, + "default": {} + }, + "dependentSchemas": { + "type": "object", + "additionalProperties": { "$dynamicRef": "#meta" }, + "default": {} + }, + "propertyNames": { "$dynamicRef": "#meta" }, + "if": { "$dynamicRef": "#meta" }, + "then": { "$dynamicRef": "#meta" }, + "else": { "$dynamicRef": "#meta" }, + "allOf": { "$ref": "#/$defs/schemaArray" }, + "anyOf": { "$ref": "#/$defs/schemaArray" }, + "oneOf": { "$ref": "#/$defs/schemaArray" }, + "not": { "$dynamicRef": "#meta" } + }, + "$defs": { + "schemaArray": { + "type": "array", + "minItems": 1, + "items": { "$dynamicRef": "#meta" } + } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-content.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-content.json new file mode 100644 index 000000000..76e3760d2 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-content.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/content", + "$dynamicAnchor": "meta", + + "title": "Content vocabulary meta-schema", + + "type": ["object", "boolean"], + "properties": { + "contentEncoding": { "type": "string" }, + "contentMediaType": { "type": "string" }, + "contentSchema": { "$dynamicRef": "#meta" } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-core.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-core.json new file mode 100644 index 000000000..691862289 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-core.json @@ -0,0 +1,48 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/core", + "$dynamicAnchor": "meta", + + "title": "Core vocabulary meta-schema", + "type": ["object", "boolean"], + "properties": { + "$id": { + "$ref": "#/$defs/uriReferenceString", + "$comment": "Non-empty fragments not allowed.", + "pattern": "^[^#]*#?$" + }, + "$schema": { "$ref": "#/$defs/uriString" }, + "$ref": { "$ref": "#/$defs/uriReferenceString" }, + "$anchor": { "$ref": "#/$defs/anchorString" }, + "$dynamicRef": { "$ref": "#/$defs/uriReferenceString" }, + "$dynamicAnchor": { "$ref": "#/$defs/anchorString" }, + "$vocabulary": { + "type": "object", + "propertyNames": { "$ref": "#/$defs/uriString" }, + "additionalProperties": { + "type": "boolean" + } + }, + "$comment": { + "type": "string" + }, + "$defs": { + "type": "object", + "additionalProperties": { "$dynamicRef": "#meta" } + } + }, + "$defs": { + "anchorString": { + "type": "string", + "pattern": "^[A-Za-z_][-A-Za-z0-9._]*$" + }, + "uriString": { + "type": "string", + "format": "uri" + }, + "uriReferenceString": { + "type": "string", + "format": "uri-reference" + } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-format-annotation.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-format-annotation.json new file mode 100644 index 000000000..3479e6695 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-format-annotation.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/format-annotation", + "$dynamicAnchor": "meta", + + "title": "Format vocabulary meta-schema for annotation results", + "type": ["object", "boolean"], + "properties": { + "format": { "type": "string" } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-meta-data.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-meta-data.json new file mode 100644 index 000000000..4049ab21b --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-meta-data.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/meta-data", + "$dynamicAnchor": "meta", + + "title": "Meta-data vocabulary meta-schema", + + "type": ["object", "boolean"], + "properties": { + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "default": true, + "deprecated": { + "type": "boolean", + "default": false + }, + "readOnly": { + "type": "boolean", + "default": false + }, + "writeOnly": { + "type": "boolean", + "default": false + }, + "examples": { + "type": "array", + "items": true + } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-unevaluated.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-unevaluated.json new file mode 100644 index 000000000..93779e54e --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-unevaluated.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/unevaluated", + "$dynamicAnchor": "meta", + + "title": "Unevaluated applicator vocabulary meta-schema", + "type": ["object", "boolean"], + "properties": { + "unevaluatedItems": { "$dynamicRef": "#meta" }, + "unevaluatedProperties": { "$dynamicRef": "#meta" } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-validation.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-validation.json new file mode 100644 index 000000000..ebb75db77 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-validation.json @@ -0,0 +1,95 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/validation", + "$dynamicAnchor": "meta", + + "title": "Validation vocabulary meta-schema", + "type": ["object", "boolean"], + "properties": { + "type": { + "anyOf": [ + { "$ref": "#/$defs/simpleTypes" }, + { + "type": "array", + "items": { "$ref": "#/$defs/simpleTypes" }, + "minItems": 1, + "uniqueItems": true + } + ] + }, + "const": true, + "enum": { + "type": "array", + "items": true + }, + "multipleOf": { + "type": "number", + "exclusiveMinimum": 0 + }, + "maximum": { + "type": "number" + }, + "exclusiveMaximum": { + "type": "number" + }, + "minimum": { + "type": "number" + }, + "exclusiveMinimum": { + "type": "number" + }, + "maxLength": { "$ref": "#/$defs/nonNegativeInteger" }, + "minLength": { "$ref": "#/$defs/nonNegativeIntegerDefault0" }, + "pattern": { + "type": "string", + "format": "regex" + }, + "maxItems": { "$ref": "#/$defs/nonNegativeInteger" }, + "minItems": { "$ref": "#/$defs/nonNegativeIntegerDefault0" }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "maxContains": { "$ref": "#/$defs/nonNegativeInteger" }, + "minContains": { + "$ref": "#/$defs/nonNegativeInteger", + "default": 1 + }, + "maxProperties": { "$ref": "#/$defs/nonNegativeInteger" }, + "minProperties": { "$ref": "#/$defs/nonNegativeIntegerDefault0" }, + "required": { "$ref": "#/$defs/stringArray" }, + "dependentRequired": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/stringArray" + } + } + }, + "$defs": { + "nonNegativeInteger": { + "type": "integer", + "minimum": 0 + }, + "nonNegativeIntegerDefault0": { + "$ref": "#/$defs/nonNegativeInteger", + "default": 0 + }, + "simpleTypes": { + "enum": [ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string" + ] + }, + "stringArray": { + "type": "array", + "items": { "type": "string" }, + "uniqueItems": true, + "default": [] + } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-schema.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-schema.json new file mode 100644 index 000000000..d5e2d31c3 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-schema.json @@ -0,0 +1,58 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/schema", + "$vocabulary": { + "https://json-schema.org/draft/2020-12/vocab/core": true, + "https://json-schema.org/draft/2020-12/vocab/applicator": true, + "https://json-schema.org/draft/2020-12/vocab/unevaluated": true, + "https://json-schema.org/draft/2020-12/vocab/validation": true, + "https://json-schema.org/draft/2020-12/vocab/meta-data": true, + "https://json-schema.org/draft/2020-12/vocab/format-annotation": true, + "https://json-schema.org/draft/2020-12/vocab/content": true + }, + "$dynamicAnchor": "meta", + + "title": "Core and Validation specifications meta-schema", + "allOf": [ + {"$ref": "meta/core"}, + {"$ref": "meta/applicator"}, + {"$ref": "meta/unevaluated"}, + {"$ref": "meta/validation"}, + {"$ref": "meta/meta-data"}, + {"$ref": "meta/format-annotation"}, + {"$ref": "meta/content"} + ], + "type": ["object", "boolean"], + "$comment": "This meta-schema also defines keywords that have appeared in previous drafts in order to prevent incompatible extensions as they remain in common use.", + "properties": { + "definitions": { + "$comment": "\"definitions\" has been replaced by \"$defs\".", + "type": "object", + "additionalProperties": { "$dynamicRef": "#meta" }, + "deprecated": true, + "default": {} + }, + "dependencies": { + "$comment": "\"dependencies\" has been split and replaced by \"dependentSchemas\" and \"dependentRequired\" in order to serve their differing semantics.", + "type": "object", + "additionalProperties": { + "anyOf": [ + { "$dynamicRef": "#meta" }, + { "$ref": "meta/validation#/$defs/stringArray" } + ] + }, + "deprecated": true, + "default": {} + }, + "$recursiveAnchor": { + "$comment": "\"$recursiveAnchor\" has been replaced by \"$dynamicAnchor\".", + "$ref": "meta/core#/$defs/anchorString", + "deprecated": true + }, + "$recursiveRef": { + "$comment": "\"$recursiveRef\" has been replaced by \"$dynamicRef\".", + "$ref": "meta/core#/$defs/uriReferenceString", + "deprecated": true + } + } +} diff --git a/packages/schema/test/shared.test.ts b/packages/schema/test/shared.test.ts new file mode 100644 index 000000000..cf3480fd1 --- /dev/null +++ b/packages/schema/test/shared.test.ts @@ -0,0 +1,41 @@ +// The caches shared across compiled schemas hold nothing alive: entries go once their schemas are collected. + +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import { describe, expect, it } from "vitest"; +import { compile } from "../src/index.ts"; +import { sharedEntries } from "../src/shared.ts"; + +setFlagsFromString("--expose-gc"); +const gc = runInNewContext("gc") as () => void; + +async function collectGarbage(): Promise { + for (let round = 0; round < 6; round++) { + await new Promise((resolve) => setImmediate(resolve)); + gc(); + } +} + +describe("shared leaf checks", () => { + it("share one check between equal leaf schemas", () => { + const a = compile({ type: "integer", minimum: 3, description: "a" }); + const b = compile({ type: "integer", minimum: 3, description: "b" }); + expect(a.check).toBe(b.check); + expect(compile({ type: "integer", minimum: 4 }).check).not.toBe(a.check); + }); + + it("are collected with the last schema that uses them, and their cache entries removed", async () => { + await collectGarbage(); + const before = sharedEntries(); + let validators: unknown[] | undefined = Array.from({ length: 50 }, (_, i) => + compile({ type: "string", minLength: 1000 + i, pattern: `^unique-${i}$` }), + ); + const check = new WeakRef((validators[0] as { check: object }).check); + expect(sharedEntries().checks).toBe(before.checks + 50); + expect(sharedEntries().regexes).toBe(before.regexes + 50); + validators = undefined; + await collectGarbage(); + expect(check.deref()).toBeUndefined(); + expect(sharedEntries()).toEqual(before); + }); +}); diff --git a/packages/schema/test/suite.ts b/packages/schema/test/suite.ts new file mode 100644 index 000000000..fac5bebbd --- /dev/null +++ b/packages/schema/test/suite.ts @@ -0,0 +1,77 @@ +// Loads the vendored JSON-Schema-Test-Suite: its test files, the remote documents and the official meta-schemas. + +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join, relative } from "node:path"; +import type { JsonSchema } from "../src/index.ts"; + +/** One test file: its path below `tests/` and its groups. */ +export interface SuiteFile { + readonly name: string; + readonly draft: "2020-12" | "7"; + readonly groups: readonly SuiteGroup[]; +} + +/** A schema with values and whether each is valid. */ +export interface SuiteGroup { + readonly description: string; + readonly schema: JsonSchema; + readonly tests: readonly { readonly description: string; readonly data: unknown; readonly valid: boolean }[]; +} + +const fixtures = join(import.meta.dirname, "fixtures"); +const root = join(fixtures, "json-schema-test-suite"); + +/** Every test file of drafts 2020-12 and 7. */ +export function suiteFiles(): SuiteFile[] { + const files: SuiteFile[] = []; + for (const [dir, draft] of [ + ["draft2020-12", "2020-12"], + ["draft7", "7"], + ] as const) { + for (const path of jsonFiles(join(root, "tests", dir))) { + const name = relative(join(root, "tests"), path).replace(/\.json$/, ""); + files.push({ name, draft, groups: JSON.parse(readFileSync(path, "utf8")) }); + } + } + return files; +} + +/** The remote documents and meta-schemas the tests reference, by URI (fresh copies). */ +export function suiteSchemas(): Record { + const schemas: Record = {}; + const remotes = join(root, "remotes"); + for (const path of jsonFiles(remotes)) { + schemas[`http://localhost:1234/${relative(remotes, path).replaceAll("\\", "/")}`] = readJson(path); + } + const meta = join(fixtures, "meta-schemas"); + schemas["http://json-schema.org/draft-07/schema"] = readJson(join(meta, "draft-07-schema.json")); + schemas["https://json-schema.org/draft/2020-12/schema"] = readJson(join(meta, "draft2020-12-schema.json")); + for (const vocabulary of [ + "core", + "applicator", + "unevaluated", + "validation", + "meta-data", + "format-annotation", + "content", + ]) { + schemas[`https://json-schema.org/draft/2020-12/meta/${vocabulary}`] = readJson( + join(meta, `draft2020-12-meta-${vocabulary}.json`), + ); + } + return schemas; +} + +function readJson(path: string): JsonSchema { + return JSON.parse(readFileSync(path, "utf8")); +} + +function jsonFiles(dir: string): string[] { + const out: string[] = []; + for (const entry of readdirSync(dir).sort()) { + const path = join(dir, entry); + if (statSync(path).isDirectory()) out.push(...jsonFiles(path)); + else if (entry.endsWith(".json")) out.push(path); + } + return out; +} diff --git a/packages/schema/test/validator.test.ts b/packages/schema/test/validator.test.ts new file mode 100644 index 000000000..a62b38e31 --- /dev/null +++ b/packages/schema/test/validator.test.ts @@ -0,0 +1,190 @@ +// compile() and Validator: checks, errors, values, options and schemas from other sources. + +import Type from "typebox"; +import { describe, expect, it } from "vitest"; +import { formats } from "../src/formats/index.ts"; +import { compile } from "../src/index.ts"; + +const read = { + type: "object", + properties: { path: { type: "string" }, offset: { type: "integer", minimum: 1 } }, + required: ["path"], + additionalProperties: false, +}; + +const block = { + anyOf: [ + { + type: "object", + properties: { kind: { const: "text" }, text: { type: "string" } }, + required: ["kind", "text"], + additionalProperties: false, + }, + { + type: "object", + properties: { kind: { const: "image" }, width: { type: "integer", minimum: 0 } }, + required: ["kind", "width"], + additionalProperties: false, + }, + ], +}; + +describe("errors", () => { + it("reports a missing field and an unknown field at the field", () => { + expect(compile(read).errors({ offset: 0, extra: true })).toEqual([ + { path: "/path", keyword: "required", message: "missing" }, + { path: "/offset", keyword: "minimum", message: "expected >= 1" }, + { path: "/extra", keyword: "additionalProperties", message: "unknown field" }, + ]); + }); + + it("is empty for a valid value", () => { + expect(compile(read).errors({ path: "a.ts" })).toEqual([]); + }); + + it("reports the branch a tagged union's tag selects", () => { + const validator = compile({ type: "array", items: block }); + expect( + validator.errors([ + { kind: "text", text: "a" }, + { kind: "image", width: -1 }, + ]), + ).toEqual([{ path: "/1/width", keyword: "minimum", message: "expected >= 0" }]); + expect(validator.errors([{ kind: "video" }])).toEqual([ + { path: "/0/kind", keyword: "anyOf", message: 'expected one of ["text","image"]' }, + ]); + }); + + it("escapes ~ and / in paths as JSON Pointer does", () => { + const validator = compile({ type: "object", properties: { "a/b~c": { type: "string" } } }); + expect(validator.errors({ "a/b~c": 1 })).toEqual([ + { path: "/a~1b~0c", keyword: "type", message: "expected string, got integer" }, + ]); + }); + + it("follows $ref into the referenced schema", () => { + const validator = compile({ + $defs: { point: { type: "object", properties: { x: { type: "number" } }, required: ["x"] } }, + type: "array", + items: { $ref: "#/$defs/point" }, + }); + expect(validator.errors([{ x: 1 }, {}])).toEqual([{ path: "/1/x", keyword: "required", message: "missing" }]); + }); +}); + +describe("values", () => { + it("treats a property set to undefined as absent", () => { + const validator = compile(read); + expect(validator.check({ path: "a.ts", offset: undefined })).toBe(true); + expect(validator.check({ path: undefined })).toBe(false); + }); + + it("checks bytes as Uint8Array with length bounds", () => { + const validator = compile({ type: "object", properties: { data: { type: "bytes", maxLength: 4 } } }); + expect(validator.check({ data: new Uint8Array(4) })).toBe(true); + expect(validator.check({ data: new Uint8Array(5) })).toBe(false); + expect(validator.check({ data: [1, 2] })).toBe(false); + expect(validator.errors({ data: new Uint8Array(5) })).toEqual([ + { path: "/data", keyword: "maxLength", message: "expected at most 4 bytes" }, + ]); + }); + + it("does not treat bytes as an object", () => { + expect(compile({ type: "object" }).check(new Uint8Array(1))).toBe(false); + }); + + it("checks properties named like Object.prototype members as own properties", () => { + const validator = compile({ + type: "object", + properties: { constructor: { type: "string" }, toString: { type: "number" } }, + required: ["constructor"], + additionalProperties: false, + }); + expect(validator.check({})).toBe(false); + expect(validator.check({ constructor: "a" })).toBe(true); + expect(validator.check({ constructor: "a", toString: "b" })).toBe(false); + }); + + it("does not accept every value for a type named like an Object.prototype member", () => { + expect(compile({ type: "constructor" }).check(1)).toBe(false); + }); + + it("checks a detached check function", () => { + const { check } = compile(read); + expect([{ path: "a" }, { path: 1 }].filter(check)).toEqual([{ path: "a" }]); + }); +}); + +describe("options", () => { + it("treats format as an annotation unless formats are given", () => { + const schema = { type: "string", format: "email" }; + expect(compile(schema).check("not an email")).toBe(true); + expect(compile(schema, { formats }).check("not an email")).toBe(false); + expect(compile(schema, { formats }).check("a@example.com")).toBe(true); + }); + + it("resolves $ref to registered documents", () => { + const point = { type: "object", properties: { x: { type: "number" } }, required: ["x"] }; + const validator = compile( + { $ref: "https://example.com/point.json" }, + { schemas: { "https://example.com/point.json": point } }, + ); + expect(validator.check({ x: 1 })).toBe(true); + expect(validator.check({})).toBe(false); + }); + + it("throws for a $ref that names nothing", () => { + expect(() => compile({ $ref: "https://example.com/missing.json" })).toThrow(/Unresolved \$ref/); + expect(() => compile({ $ref: "#/$defs/missing" })).toThrow(/Unresolved \$ref/); + }); + + it("applies draft 7 semantics: $ref overrides its siblings", () => { + const schema = { definitions: { s: { type: "string" } }, $ref: "#/definitions/s", maxLength: 1 }; + expect(compile(schema, { draft: "7" }).check("ab")).toBe(true); + expect(compile(schema).check("ab")).toBe(false); + }); +}); + +describe("schemas", () => { + it("compiles TypeBox schemas as they are", () => { + const validator = compile(Type.Object({ name: Type.String(), tags: Type.Array(Type.String()) })); + expect(validator.check({ name: "a", tags: [] })).toBe(true); + expect(validator.errors({ name: "a", tags: [1] })).toEqual([ + { path: "/tags/0", keyword: "type", message: "expected string, got integer" }, + ]); + }); + + it("compiles deep-frozen schemas and never modifies a schema", () => { + const schema = deepFreeze(structuredClone({ ...block, $defs: { s: { type: "string", pattern: "^a" } } })); + const before = JSON.stringify(schema); + const validator = compile(schema); + expect(validator.check({ kind: "text", text: "x" })).toBe(true); + expect(validator.errors({ kind: "image", width: -1 })).toHaveLength(1); + expect(JSON.stringify(schema)).toBe(before); + }); + + it("compiles a recursive schema", () => { + const tree = { + type: "object", + properties: { value: { type: "number" }, children: { type: "array", items: { $ref: "#" } } }, + required: ["value"], + }; + const validator = compile(tree); + expect(validator.check({ value: 1, children: [{ value: 2, children: [{ value: 3 }] }] })).toBe(true); + expect(validator.errors({ value: 1, children: [{ value: 2, children: [{}] }] })).toEqual([ + { path: "/children/0/children/0/value", keyword: "required", message: "missing" }, + ]); + }); + + it("throws for a malformed schema", () => { + expect(() => compile({ properties: { a: 1 } })).toThrow(/must be an object or a boolean/); + }); +}); + +function deepFreeze(value: T): T { + if (typeof value === "object" && value !== null && !Object.isFrozen(value)) { + Object.freeze(value); + for (const child of Object.values(value)) deepFreeze(child); + } + return value; +} diff --git a/packages/schema/tsconfig.build.json b/packages/schema/tsconfig.build.json new file mode 100644 index 000000000..695dd9ad2 --- /dev/null +++ b/packages/schema/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist", "**/*.d.ts", "src/**/*.d.ts"] +} diff --git a/packages/schema/vitest.config.ts b/packages/schema/vitest.config.ts new file mode 100644 index 000000000..18733b4d4 --- /dev/null +++ b/packages/schema/vitest.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + globals: true, + environment: "node", + reporters: process.env.GITHUB_ACTIONS ? ["dot", "github-actions"] : ["dot"], + }, + resolve: { conditions: ["source"] }, + ssr: { resolve: { conditions: ["source"] } }, +}); diff --git a/scripts/check-entry-graphs.mjs b/scripts/check-entry-graphs.mjs index 8ec4f337e..54dd30c5d 100644 --- a/scripts/check-entry-graphs.mjs +++ b/scripts/check-entry-graphs.mjs @@ -26,6 +26,7 @@ const WORKSPACE = { "@earendil-works/pi-codemode": "packages/codemode/src", "@earendil-works/pi-telemetry": "packages/telemetry/src", "@earendil-works/pi-mcp": "packages/mcp/src", + "@earendil-works/pi-schema": "packages/schema/src", "@earendil-works/pi-tui": "packages/tui/src", }; @@ -34,6 +35,10 @@ const WORKSPACE = { * graph it is allowed to reach. */ const BUDGETS = { + "packages/schema": { + // The format checks (vendored from TypeBox, about 40 KB) load only through ./formats. + ".": { maxFiles: 10, forbid: ["schema/src/formats/"] }, + }, "packages/ai": { "./models": { // Request telemetry reaches pi-ai's telemetry.ts and pi-telemetry's root (index, noop, metrics, memory). diff --git a/tsconfig.json b/tsconfig.json index 7a9cef85c..96c54539c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -34,6 +34,8 @@ "@earendil-works/pi-codemode/portable": ["./packages/codemode/src/portable.ts"], "@earendil-works/pi-codemode/source": ["./packages/codemode/src/source.ts"], "@earendil-works/pi-codemode/worker": ["./packages/codemode/src/runtime/worker.ts"], + "@earendil-works/pi-schema": ["./packages/schema/src/index.ts"], + "@earendil-works/pi-schema/formats": ["./packages/schema/src/formats/index.ts"], "@earendil-works/pi-mcp": ["./packages/mcp/src/index.ts"], "@earendil-works/pi-mcp/oauth": ["./packages/mcp/src/oauth/index.ts"], "@earendil-works/pi-mcp/testing": ["./packages/mcp/src/testing/index.ts"], From 6d4b7443637356fe928118df3f1f6d922fcc191e Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 21:47:22 +0000 Subject: [PATCH 229/271] docs(chord): the schema package is built --- packages/chord/docs/next/decisions.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index e7aed524b..ca6ad2950 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2119,6 +2119,14 @@ validation and copies the v2 kernel added around Delta. is deleting that subpath). Not omptype (its validator compiles with `new Function`; its 10.7k lines are mostly ArkType's API). Open: which validator (measure `@cfworker/json-schema` and TypeBox against the official JSON-Schema-Test-Suite and Chord's argument checks first), and moving `Type` out of pi-ai's main export. + **Built** (owner, 2026-10-31: "clone pi add the package, done"; pi `fab2c53`): `packages/schema`, + `@earendil-works/pi-schema`: `compile(schema, { draft, formats, schemas })` with `check` (narrowing) and `errors` + (JSON Pointer paths), Chord-style builders typed with `Static`, formats as the opt-in `./formats` entry, leaf and regex + checks shared through weak caches (GC-tested); its own closure-compiled validator, chosen after measuring + (`/tmp/schema-measure.md`, Radius 01m69x3dz2fsg9k1r33vde7hv6): 2,222 of 2,223 core suite tests (TypeBox 2,200, + cfworker 2,167, omptype 1,522), Chord checks at 0.95-1.3x TypeBox's compiled speed and the same with `new Function` + blocked, 7.9 MB for 3,200 distinct schemas (cfworker 7.6, TypeBox 54.7), 8.5 KB gzip. Still open: switching Chord v2 + and pi-ai to it (`Type` out of pi-ai's main export), formats on by default in pi-ai. - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From 8ffea8a1bfdc32523ab056549b26141018c1e700 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sat, 31 Oct 2026 21:59:21 +0000 Subject: [PATCH 230/271] feat(schema): faster object checks and compile, less memory - Declared-object checks walk the value's own keys with for...in, whose property reads V8 serves from the enum cache, instead of reading each declared name from a list: nested Chord-style arguments 535 -> 409 ns (TypeBox 474), recursive $ref schemas 1.1 us -> 753 ns. - Schemas are indexed for references only on the first $ref; base URIs are derived from $id while compiling. Compiling a new schema 43 -> 21 us. - Drop the keyed cache of leaf checks: on distinct schemas its entries cost more than sharing saved. Bare types still share module-level checks and patterns share their RegExp. 3,200 distinct schemas 7.9 -> 7.1 MB (cfworker 7.6 MB). --- packages/schema/README.md | 22 +++---- packages/schema/src/checks.ts | 41 ++++++------ packages/schema/src/compiler.ts | 97 +++++++++++------------------ packages/schema/src/explain.ts | 2 +- packages/schema/src/resolver.ts | 39 ++++++++++-- packages/schema/src/shared.ts | 29 ++------- packages/schema/test/shared.test.ts | 31 ++++----- 7 files changed, 125 insertions(+), 136 deletions(-) diff --git a/packages/schema/README.md b/packages/schema/README.md index d31caa521..023ebdc02 100644 --- a/packages/schema/README.md +++ b/packages/schema/README.md @@ -134,22 +134,22 @@ Node 24.4.1 on one core of an i9-13900, against TypeBox 1.3.27, @cfworker/json-s | check, valid value | pi-schema | TypeBox | cfworker | omptype | |---|---:|---:|---:|---:| -| nested object with tagged unions | 535 ns | 464 ns | 43 µs | 713 ns | -| union of 5 messages | 96 ns | 94 ns | 10 µs | 94 ns | -| tool schema (read) | 34 ns | 9 ns | 2.0 µs | 8 ns | -| MCP tool schema with `$ref` | 207 ns | 88 ns | 13 µs | 1.7 µs | -| recursive `$ref` schema | 1.1 µs | 1.2 µs | 41 µs | 7.4 µs | +| nested object with tagged unions | 409 ns | 474 ns | 44 µs | 689 ns | +| union of 5 messages | 89 ns | 95 ns | 10 µs | 93 ns | +| tool schema (read) | 31 ns | 9 ns | 1.9 µs | 9 ns | +| MCP tool schema with `$ref` | 162 ns | 89 ns | 13 µs | 1.8 µs | +| recursive `$ref` schema | 753 ns | 1.2 µs | 41 µs | 7.4 µs | | same, without eval (strict CSP) | pi-schema | TypeBox | cfworker | omptype | |---|---:|---:|---:|---:| -| nested object with tagged unions | 527 ns | 42 µs | 44 µs | 2.7 µs | -| MCP tool schema with `$ref` | 209 ns | 38 µs | 13 µs | 2.1 µs | -| recursive `$ref` schema | 1.1 µs | 134 µs | 43 µs | 8.9 µs | +| nested object with tagged unions | 407 ns | 42 µs | 43 µs | 2.6 µs | +| MCP tool schema with `$ref` | 162 ns | 36 µs | 12 µs | 2.0 µs | +| recursive `$ref` schema | 750 ns | 124 µs | 41 µs | 9.0 µs | | cost | pi-schema | TypeBox | cfworker | omptype | |---|---:|---:|---:|---:| -| compile a new schema | 43 µs | 210 µs | 23 µs | 19 µs + 114 µs on its first check | -| memory, 3,200 distinct schemas | 7.9 MB | 54.7 MB | 7.6 MB | 20.0 MB | +| compile a new schema | 21 µs | 203 µs | 20 µs | 26 µs + 106 µs on its first check | +| memory, 3,200 distinct schemas | 7.1 MB | 54.7 MB | 7.6 MB | 20.0 MB | | bundle, minified + gzip | 8.5 KB | 35.2 KB | 5.8 KB | 38.8 KB | Reproduce with `npm run bench` in `packages/schema` (about 5 minutes). It runs every library through its fastest @@ -167,7 +167,7 @@ boolean path and keeps the object each returns. The scripts are in `test/bench/` Where code generation is unavailable, TypeBox falls back to its interpreter, and omptype throws `EvalError` once per schema, then validates through its interpreter. -Flat objects of simple types check in 15–90 ns, against 8–15 ns for TypeBox. Without code generation, reading a +Flat objects of simple types check in 13–80 ns, against 7–15 ns for TypeBox. Without code generation, reading a property whose name comes from a list costs a few nanoseconds. ## Development diff --git a/packages/schema/src/checks.ts b/packages/schema/src/checks.ts index b686c4c27..701707a28 100644 --- a/packages/schema/src/checks.ts +++ b/packages/schema/src/checks.ts @@ -206,36 +206,37 @@ export function guardArray(inner: Check, strict: boolean): Check { } /** - * An object with only declared properties (`additionalProperties: false`). `entries` is `[key, check, flags]` per - * property. Declared fields are checked, then the own keys are counted, so unknown keys are found without - * allocating. A property whose value is `undefined` counts as absent. + * An object with declared properties: `entries` is `[key, check, flags]` per property. The value's own keys are + * walked with for...in (whose `o[k]` V8 reads from the enum cache, unlike a load by a name from a list), each found + * among the declared names; `closed` rejects any other key. A property whose value is `undefined` counts as absent. */ -export function closedObject(entries: readonly unknown[], guard: Guard): Check { +export function declaredObject( + entries: readonly unknown[], + requiredCount: number, + closed: boolean, + guard: Guard, +): Check { + const n = entries.length; return (v) => { if (guard !== GUARD_NONE && !isObject(v)) return guard === GUARD_LOOSE; // Guarded: by the check above, or by the caller for GUARD_NONE. const o = v as Record; - let present = 0; - for (let i = 0; i < entries.length; i += 3) { - const k = entries[i] as string; + let required = 0; + for (const k in o) { + let i = 0; + while (i < n && entries[i] !== k) i += 3; + if (i === n) { + if (closed) return false; + continue; + } const x = o[k]; + if (x === undefined) continue; const f = entries[i + 2] as number; - if (x === undefined) { - if (f & REQUIRED) return false; - if (Object.hasOwn(o, k)) present++; - continue; - } - if (f & INHERITED && !Object.hasOwn(o, k)) { - if (f & REQUIRED) return false; - continue; - } - present++; const c = f & CODE; if (c !== 0 ? !leafOk(c, x) : !(entries[i + 1] as Check)(x)) return false; + if (f & REQUIRED) required++; } - let count = 0; - for (const _ in o) if (++count > present) return false; - return true; + return required === requiredCount; }; } diff --git a/packages/schema/src/compiler.ts b/packages/schema/src/compiler.ts index 01ec52bbf..8140e38d5 100644 --- a/packages/schema/src/compiler.ts +++ b/packages/schema/src/compiler.ts @@ -25,9 +25,9 @@ import { and, anyCheck, bytesCheck, - closedObject, constCheck, containsCheck, + declaredObject, dependentRequiredCheck, dependentSchemasCheck, enumCheck, @@ -56,12 +56,13 @@ import { REQUIRED, scopedCheck, stringCheck, + TYPE_CHECKS, taggedCheck, typeCheck, uniqueCheck, } from "./checks.ts"; import { documentOf, isSchemaObject, Resolver, resolveUri } from "./resolver.ts"; -import { sharedCheck, sharedRegex } from "./shared.ts"; +import { sharedRegex } from "./shared.ts"; import type { Check, CompileOptions, FormatCheck, JsonSchema, SchemaObject } from "./types.ts"; /** Thrown when a compile without dynamic scope tracking meets a bookended `$dynamicRef`: compile again with it. */ @@ -101,15 +102,11 @@ export class Compiler { throw new Error(`A schema must be an object or a boolean, got ${JSON.stringify(node)}`); const cached = this.cache.get(node); if (cached !== undefined) return cached; - const key = this.leafKey(node); - if (key !== undefined) { - const shared = sharedCheck(key, () => this.plain(node, base)); - this.cache.set(node, shared); - return shared; - } + const bare = this.bareType(node); + if (bare !== undefined) return bare; const slot: { check: Check | undefined } = { check: undefined }; this.cache.set(node, indirect(slot)); - const check = this.build(node, this.r.baseOf.get(node) ?? base); + const check = this.build(node, this.r.baseFor(node, base)); slot.check = check; this.cache.set(node, check); return check; @@ -125,18 +122,17 @@ export class Compiler { } /** - * The sharing key of a leaf schema (only assertion keywords; no subschemas, references or identifiers): equal leaves - * in all compiled schemas share one check. Undefined for every other schema. + * The module-level check of a schema that is only a `type` (with annotations), `ALWAYS` for one with no assertion + * at all; undefined for every other schema. These are the most common subschemas and cost nothing to share. */ - private leafKey(node: SchemaObject): string | undefined { - const picked: Record = {}; - for (const k of Object.keys(node)) { - if (LEAF_KEYWORDS.has(k)) picked[k] = node[k]; + private bareType(node: SchemaObject): Check | undefined { + let type: unknown; + for (const k in node) { + if (k === "type") type = node.type; else if (!NEUTRAL_KEYWORDS.has(k)) return undefined; } - // A format checks through this compile's format map, so a leaf with one is shared only while formats are off. - if (picked.format !== undefined && this.formats !== undefined) return undefined; - return this.draft + JSON.stringify(picked); + if (type === undefined) return ALWAYS; + return typeof type === "string" ? TYPE_CHECKS.get(type) : undefined; } /** Every keyword of `node`, not tracking annotations. */ @@ -149,20 +145,20 @@ export class Compiler { parts.push(bytesCheck(number(node.minLength) ?? 0, number(node.maxLength) ?? Number.POSITIVE_INFINITY)); typeInGroup = true; } else { - const groups: [Check | undefined, boolean][] = [ - [this.stringGroup(node, single === "string"), single === "string"], - [ - this.numberGroup(node, single === "number" ? 1 : single === "integer" ? 2 : 0), - single === "number" || single === "integer", - ], - [this.objectGroup(node, base, single === "object"), single === "object"], - [this.arrayGroup(node, base, single === "array"), single === "array"], - ]; - for (const [group, strict] of groups) { - if (group === undefined) continue; - parts.push(group); - if (strict) typeInGroup = true; - } + const string = this.stringGroup(node, single === "string"); + if (string !== undefined) parts.push(string); + const numeric = single === "number" || single === "integer"; + const number = this.numberGroup(node, single === "number" ? 1 : single === "integer" ? 2 : 0); + if (number !== undefined) parts.push(number); + const object = this.objectGroup(node, base, single === "object"); + if (object !== undefined) parts.push(object); + const array = this.arrayGroup(node, base, single === "array"); + if (array !== undefined) parts.push(array); + typeInGroup = + (single === "string" && string !== undefined) || + (numeric && number !== undefined) || + (single === "object" && object !== undefined) || + (single === "array" && array !== undefined); } if (types !== undefined && !typeInGroup) parts.unshift(typeCheck(types)); this.inPlace(node, base, parts); @@ -255,7 +251,7 @@ export class Compiler { for (const branch of branches) { if (!isSchemaObject(branch)) return undefined; const ref = typeof branch.$ref === "string" && Object.keys(branch).length === 1 ? branch.$ref : undefined; - const node = ref === undefined ? branch : this.r.resolve(ref, this.r.baseOf.get(branch) ?? base).node; + const node = ref === undefined ? branch : this.r.resolve(ref, this.r.baseFor(branch, base)).node; if (!isSchemaObject(node)) return undefined; resolved.push(node); } @@ -350,8 +346,11 @@ export class Compiler { } const extraRequired = required.filter((k) => !Object.hasOwn(properties, k)); const pats = patternKeys.map((p) => [sharedRegex(p), this.check(patterns[p], base)] as const); - if (additional === false && pats.length === 0 && extraRequired.length === 0) return closedObject(entries, guard); const extra = additional === undefined || additional === true ? undefined : this.check(additional, base); + if (pats.length === 0 && extraRequired.length === 0 && (extra === undefined || additional === false)) { + const requiredCount = keys.filter((k) => required.includes(k)).length; + return declaredObject(entries, requiredCount, additional === false, guard); + } return openObject(entries, keys, extraRequired, pats, extra, guard); } @@ -404,8 +403,7 @@ export class Compiler { if (cached !== undefined) return cached; const slot: { check: AnnotatingCheck | undefined } = { check: undefined }; this.annotatingCache.set(node, indirectAnnotating(slot)); - const b = this.r.baseOf.get(node) ?? base; - const check = this.buildAnnotating(node, b); + const check = this.buildAnnotating(node, this.r.baseFor(node, base)); slot.check = check; this.annotatingCache.set(node, check); return check; @@ -510,29 +508,6 @@ export class Compiler { } } -// Keywords whose check depends only on their own value, and keywords that never affect a check. -const LEAF_KEYWORDS = new Set([ - "type", - "enum", - "const", - "minLength", - "maxLength", - "pattern", - "format", - "minimum", - "maximum", - "exclusiveMinimum", - "exclusiveMaximum", - "multipleOf", - "minItems", - "maxItems", - "uniqueItems", - "minProperties", - "maxProperties", - "required", - "dependentRequired", -]); - /** Keywords that never affect a check. */ export const NEUTRAL_KEYWORDS = new Set([ "$schema", @@ -587,7 +562,7 @@ const ANNOTATING = new Set([ "unevaluatedItems", ]); -/** anyOf/oneOf whose branches are all `{ const: primitive }`: one shared enum (distinct values make oneOf the same). */ +/** anyOf/oneOf whose branches are all `{ const: primitive }`: an enum (distinct values make oneOf the same). */ function constUnion(branches: readonly unknown[], exactlyOne: boolean): Check | undefined { const values: unknown[] = []; for (const branch of branches) { @@ -597,7 +572,7 @@ function constUnion(branches: readonly unknown[], exactlyOne: boolean): Check | if (exactlyOne && values.includes(branch.const)) return undefined; values.push(branch.const); } - return sharedCheck(`enum${JSON.stringify(values)}`, () => enumCheck(values)); + return enumCheck(values); } // The tag of a tagged-union branch: a primitive const on a required property of an object schema. diff --git a/packages/schema/src/explain.ts b/packages/schema/src/explain.ts index fc8068a26..da55edb43 100644 --- a/packages/schema/src/explain.ts +++ b/packages/schema/src/explain.ts @@ -49,7 +49,7 @@ export class Explainer { return; } if (!isSchemaObject(node)) return; - const b = this.c.r.baseOf.get(node) ?? base; + const b = this.c.r.baseFor(node, base); if (this.ok(node, b, v)) return; const start = out.length; const push = (keyword: string, message: string, at = path) => { diff --git a/packages/schema/src/resolver.ts b/packages/schema/src/resolver.ts index a10d245b6..9541d2c33 100644 --- a/packages/schema/src/resolver.ts +++ b/packages/schema/src/resolver.ts @@ -12,7 +12,10 @@ export interface Resolved { /** The base URI of a root schema without `$id`; relative references resolve against it. */ export const DEFAULT_BASE = "https://json-schema.invalid/root.json"; -/** Indexes the root schema and resolves references in it and in the registered documents. */ +/** + * Resolves references in the root schema and the registered documents. The schema is indexed on the first + * reference, so a schema without references is never walked for identifiers. + */ export class Resolver { readonly draft: "2020-12" | "7"; readonly rootBase: string; @@ -20,22 +23,47 @@ export class Resolver { readonly baseOf = new Map(); /** URI (with a `#name` fragment) of every `$dynamicAnchor`, to its schema. */ readonly dynamicAnchors = new Map(); + private readonly root: JsonSchema; + private readonly options: CompileOptions; private readonly registry = new Map(); private readonly resources = new Map(); private readonly anchors = new Map(); + private indexed = false; constructor(root: JsonSchema, options: CompileOptions) { this.draft = options.draft === "7" ? "7" : "2020-12"; - for (const [uri, schema] of Object.entries(options.schemas ?? {})) { + this.root = root; + this.options = options; + this.rootBase = isSchemaObject(root) ? this.baseFor(root, DEFAULT_BASE) : DEFAULT_BASE; + } + + /** + * The base URI of `node`, whose parent has `base`: its `$id` changes it, unless draft 7 makes the `$id` inert. Once + * the schema is indexed, the indexed base (a resolved reference target already has its own). + */ + baseFor(node: SchemaObject, base: string): string { + const indexed = this.baseOf.get(node); + if (indexed !== undefined) return indexed; + const id = node.$id; + if (typeof id !== "string") return base; + if (this.draft === "7" && (typeof node.$ref === "string" || id.startsWith("#"))) return base; + return documentOf(resolveUri(id, base)); + } + + /** Indexes the root schema and the registered documents' URIs (once). */ + ensureIndexed(): void { + if (this.indexed) return; + this.indexed = true; + for (const [uri, schema] of Object.entries(this.options.schemas ?? {})) { this.registry.set(documentOf(resolveUri(uri, DEFAULT_BASE)), schema); } - this.resources.set(DEFAULT_BASE, root); - this.index(root, DEFAULT_BASE); - this.rootBase = (isSchemaObject(root) && this.baseOf.get(root)) || DEFAULT_BASE; + this.resources.set(DEFAULT_BASE, this.root); + this.index(this.root, DEFAULT_BASE); } /** The schema a reference names, with the base URI it compiles under. Throws if it names nothing. */ resolve(ref: string, base: string): Resolved { + this.ensureIndexed(); const uri = resolveUri(ref, base); const doc = documentOf(uri); const fragment = uri.length > doc.length ? decodeURIComponent(uri.slice(doc.length + 1)) : ""; @@ -50,6 +78,7 @@ export class Resolver { /** Whether `node` is the root of the schema resource at `base`. */ isResource(node: object, base: string): boolean { + this.ensureIndexed(); return this.resources.get(base) === node; } diff --git a/packages/schema/src/shared.ts b/packages/schema/src/shared.ts index bb3867d3d..42c30152f 100644 --- a/packages/schema/src/shared.ts +++ b/packages/schema/src/shared.ts @@ -1,23 +1,6 @@ -// Values shared across all compiled schemas while one still uses them: leaf checks and regular expressions. -// Both caches hold their values weakly and drop an entry once its value is collected, so they keep nothing alive. - -import type { Check } from "./types.ts"; - -/** The check shared under `key`, made by `make` when no live schema holds one. */ -export function sharedCheck(key: string, make: () => Check): Check { - const hit = checks.get(key)?.deref(); - if (hit !== undefined) return hit; - const check = make(); - checks.set(key, new WeakRef(check)); - checkCleanup.register(check, key); - return check; -} - -const checks = new Map>(); - -const checkCleanup = new FinalizationRegistry((key) => { - if (checks.get(key)?.deref() === undefined) checks.delete(key); -}); +// Regular expressions shared across all compiled schemas while one still uses them: tool schemas repeat a few +// patterns many times, and a RegExp is large once it has run. The cache holds its values weakly and drops an entry +// once its RegExp is collected, so it keeps nothing alive. /** * The regular expression of a `pattern` source, shared by source. Patterns are ECMAScript regular expressions with @@ -44,7 +27,7 @@ const regexCleanup = new FinalizationRegistry((source) => { if (regexes.get(source)?.deref() === undefined) regexes.delete(source); }); -/** The number of entries in both caches (for tests). */ -export function sharedEntries(): { checks: number; regexes: number } { - return { checks: checks.size, regexes: regexes.size }; +/** The number of cached regular expressions (for tests). */ +export function sharedRegexCount(): number { + return regexes.size; } diff --git a/packages/schema/test/shared.test.ts b/packages/schema/test/shared.test.ts index cf3480fd1..ba3e4ab30 100644 --- a/packages/schema/test/shared.test.ts +++ b/packages/schema/test/shared.test.ts @@ -1,10 +1,10 @@ -// The caches shared across compiled schemas hold nothing alive: entries go once their schemas are collected. +// Values shared across compiled schemas hold nothing alive: a cached pattern goes once its schemas are collected. import { setFlagsFromString } from "node:v8"; import { runInNewContext } from "node:vm"; import { describe, expect, it } from "vitest"; import { compile } from "../src/index.ts"; -import { sharedEntries } from "../src/shared.ts"; +import { sharedRegexCount } from "../src/shared.ts"; setFlagsFromString("--expose-gc"); const gc = runInNewContext("gc") as () => void; @@ -16,26 +16,27 @@ async function collectGarbage(): Promise { } } -describe("shared leaf checks", () => { - it("share one check between equal leaf schemas", () => { - const a = compile({ type: "integer", minimum: 3, description: "a" }); - const b = compile({ type: "integer", minimum: 3, description: "b" }); +describe("shared checks", () => { + it("share the module-level check of a bare type", () => { + const a = compile({ type: "string", description: "a" }); + const b = compile({ type: "string", title: "b" }); expect(a.check).toBe(b.check); - expect(compile({ type: "integer", minimum: 4 }).check).not.toBe(a.check); }); - it("are collected with the last schema that uses them, and their cache entries removed", async () => { + it("share a pattern's regular expression and drop it with the last schema that uses it", async () => { await collectGarbage(); - const before = sharedEntries(); - let validators: unknown[] | undefined = Array.from({ length: 50 }, (_, i) => - compile({ type: "string", minLength: 1000 + i, pattern: `^unique-${i}$` }), + const before = sharedRegexCount(); + let validators: { check: object }[] | undefined = Array.from({ length: 50 }, (_, i) => + compile({ type: "string", pattern: `^unique-${i}$` }), ); - const check = new WeakRef((validators[0] as { check: object }).check); - expect(sharedEntries().checks).toBe(before.checks + 50); - expect(sharedEntries().regexes).toBe(before.regexes + 50); + const again = compile({ type: "string", pattern: "^unique-0$" }); + expect(sharedRegexCount()).toBe(before + 50); + expect(again.check("unique-0")).toBe(true); + const check = new WeakRef(validators[0].check); validators = undefined; await collectGarbage(); + // `again` still uses ^unique-0$; the other 49 patterns are gone. expect(check.deref()).toBeUndefined(); - expect(sharedEntries()).toEqual(before); + expect(sharedRegexCount()).toBe(before + 1); }); }); From 2c8d1feb147f6b3d97e855da803bfe7b19f3c403 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sun, 1 Nov 2026 00:36:33 +0000 Subject: [PATCH 231/271] docs(chord): the schema switch decisions --- packages/chord/docs/next/decisions.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index ca6ad2950..7dd53c73b 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2127,6 +2127,19 @@ validation and copies the v2 kernel added around Delta. cfworker 2,167, omptype 1,522), Chord checks at 0.95-1.3x TypeBox's compiled speed and the same with `new Function` blocked, 7.9 MB for 3,200 distinct schemas (cfworker 7.6, TypeBox 54.7), 8.5 KB gzip. Still open: switching Chord v2 and pi-ai to it (`Type` out of pi-ai's main export), formats on by default in pi-ai. +- **The schema switch (owner, 2026-10-31):** TypeBox leaves Chord (v1 too: "remove typebox wherever we can"), + pi-ai, pi-agent-core and pi-durable; extensions must not break ("absolutely 100%"), proven by a guard over our own + extensions, built-in tools and generated arguments (verdicts, coerced arguments, provider payloads and inferred + parameter types identical; owner: "just use ours", no npm corpus). **Coercion lives in pi-schema** as an opt-in, + the superset of TypeBox's `Value.Convert` rules and pi-ai's own (TypeBox's rule first where they conflict): + `compile(schema, { coerce: true })` adds `coerce(value)` (a copy, never mutating) and `parse(value)` (`{ ok, value } + | { ok, errors }`); pi-ai uses it for every tool, whatever built the schema (so plain-JSON MCP tools gain the + TypeBox rules). **Error texts are pi-schema's own** (no mapping to TypeBox's wording). Extensions keep `Type`, + `Static`, `TSchema` from the coding agent's bundled TypeBox (the pi-ai they see is compat plus those); an extension + author type-checking against the published pi-ai no longer finds them there (accepted: "fine"). **`format` is an + annotation in Chord** ("no format check": no format module in Chord's or Durable's entries; pi-ai still checks + formats on tool arguments). **Chord's envelope check costs +10 to +100 ns per message** from untrusted links + (pi-schema reads fields by name from a list; no eval): accepted ("negligible"), no hand-written check. - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From 6536f80129a266c3619c044f8487c0279fc39cd9 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sun, 1 Nov 2026 10:08:10 +0000 Subject: [PATCH 232/271] feat(ai,agent,coding-agent): validate tool arguments with pi-schema, drop TypeBox pi-schema: opt-in coercion, compile(schema, { coerce: true }) with coerce() and parse(), the superset of TypeBox's Value.Convert rules and pi-ai's old coercion; unresolvedRefs option; url and json-pointer-uri-fragment formats. pi-ai: Tool takes any JSON Schema; validation through pi-schema; Type, Static and TSchema are no longer exported (breaking). pi-agent-core and pi-durable: tools on pi-schema, no TypeBox dependency. Coding agent: extensions keep Type, Static, TSchema and StringEnum through an extension view of pi-ai; built-in tools stay on TypeBox. --- package-lock.json | 11 +- package.json | 2 +- packages/agent/CHANGELOG.md | 4 + packages/agent/README.md | 14 +- packages/agent/examples/mcp-codemode/tools.ts | 17 +- packages/agent/package.json | 2 +- packages/agent/src/types.ts | 24 +- packages/agent/test/agent-loop.test.ts | 52 +- packages/agent/test/agent.test.ts | 22 +- packages/agent/test/utils/calculate.ts | 10 +- packages/agent/test/utils/get-current-time.ts | 12 +- packages/ai/CHANGELOG.md | 25 + packages/ai/README.md | 51 +- packages/ai/package.json | 5 +- packages/ai/src/index.ts | 5 +- packages/ai/src/providers/compat-schema.ts | 855 ++++++----- packages/ai/src/providers/model-schema.ts | 110 +- packages/ai/src/providers/schema-builders.ts | 34 + packages/ai/src/types.ts | 8 +- .../{typebox-helpers.ts => string-enum.ts} | 14 +- packages/ai/src/utils/transcript.ts | 2 +- packages/ai/src/utils/validation.ts | 340 +---- packages/ai/test/compat-schema.test.ts | 31 +- packages/ai/test/faux-provider.test.ts | 2 +- packages/ai/test/interleaved-thinking.test.ts | 2 +- packages/ai/test/stream.test.ts | 2 +- packages/ai/test/validation-probes.test.ts | 779 ++++++++++ packages/ai/test/validation.test.ts | 42 +- packages/ai/vitest.config.ts | 3 + packages/codemode/CHANGELOG.md | 4 + packages/codemode/README.md | 4 +- packages/coding-agent/CHANGELOG.md | 2 + packages/coding-agent/docs/extensions.md | 14 +- packages/coding-agent/examples/tsconfig.json | 180 +++ .../install-lock/package-lock.json | 14 +- .../coding-agent/schemas/models.schema.json | 72 +- .../src/core/extensions/loader.ts | 9 +- .../core/extensions/pi-ai-extension-view.ts | 28 + .../src/core/extensions/virtual-modules.ts | 15 +- .../coding-agent/src/core/model-config.ts | 19 +- .../coding-agent/src/core/pi-ai-schemas.ts | 40 + .../coding-agent/src/core/settings-schema.ts | 2 +- packages/coding-agent/src/core/tools/bash.ts | 5 +- packages/coding-agent/src/core/tools/edit.ts | 5 +- packages/coding-agent/src/core/tools/find.ts | 5 +- packages/coding-agent/src/core/tools/grep.ts | 5 +- packages/coding-agent/src/core/tools/ls.ts | 5 +- packages/coding-agent/src/core/tools/read.ts | 5 +- .../src/core/tools/tool-definition-wrapper.ts | 21 +- packages/coding-agent/src/core/tools/write.ts | 5 +- .../src/extensions/codemode/tool.ts | 4 +- .../test/sdk-codex-cache-probe-tool-loop.ts | 2 +- .../test/suite/agent-session-codemode.test.ts | 7 +- packages/durable/CHANGELOG.md | 1 + packages/durable/README.md | 16 +- packages/durable/package.json | 4 +- packages/durable/src/codemode/codemode.ts | 4 +- packages/durable/src/codemode/models.ts | 8 +- packages/durable/src/harness/define.ts | 3 +- packages/durable/src/harness/tool.ts | 18 +- packages/durable/src/harness/types.ts | 23 +- .../durable/src/tool-search/tool-search.ts | 15 +- packages/durable/src/tools/bash.ts | 40 +- packages/durable/src/tools/edit.ts | 36 +- packages/durable/src/tools/read.ts | 21 +- packages/durable/src/tools/write.ts | 13 +- packages/durable/test/codemode-bench.ts | 7 +- .../durable/test/codemode-describe.test.ts | 5 +- packages/durable/test/codemode.test.ts | 15 +- packages/durable/test/examples/05-harness.ts | 5 +- .../durable/test/examples/06-configuration.ts | 5 +- .../test/examples/09-registry-reload.ts | 5 +- .../test/examples/10-extension-state.ts | 5 +- .../durable/test/examples/20-late-join.ts | 5 +- .../test/examples/21-subagent-foreground.ts | 6 +- .../test/examples/22-subagent-background.ts | 14 +- .../durable/test/examples/26-plan-mode.ts | 5 +- .../test/examples/30-reload-and-restart.ts | 6 +- .../test/examples/32-tests-after-edits.ts | 5 +- .../examples/33-turn-limit-and-continue.ts | 5 +- .../test/examples/34-submit-in-commit.ts | 5 +- .../durable/test/harness-after-run.test.ts | 5 +- packages/durable/test/harness-callers.test.ts | 7 +- .../durable/test/harness-compaction.test.ts | 8 +- .../durable/test/harness-continue.test.ts | 7 +- .../test/harness-deferred-progress.test.ts | 8 +- .../durable/test/harness-host-inputs.test.ts | 4 +- packages/durable/test/harness-inbox.test.ts | 8 +- .../test/harness-input-entries.test.ts | 5 +- .../durable/test/harness-lifecycle.test.ts | 5 +- .../durable/test/harness-live-deltas.test.ts | 13 +- .../test/harness-nested-tools-restart.test.ts | 4 +- .../durable/test/harness-nested-tools.test.ts | 6 +- .../durable/test/harness-ownership.test.ts | 15 +- packages/durable/test/harness-prompt.test.ts | 5 +- .../durable/test/harness-registry.test.ts | 4 +- packages/durable/test/harness-reports.test.ts | 5 +- .../test/harness-restart-failure.test.ts | 9 +- .../durable/test/harness-run-failure.test.ts | 6 +- .../test/harness-structured-output.test.ts | 21 +- .../durable/test/harness-structured.test.ts | 14 +- .../test/harness-submit-in-commit.test.ts | 11 +- packages/durable/test/harness-support.ts | 5 +- .../test/harness-tasks-recovery.test.ts | 5 +- packages/durable/test/harness-tasks.test.ts | 5 +- .../durable/test/harness-telemetry.test.ts | 15 +- .../durable/test/harness-tool-hooks.test.ts | 7 +- .../test/harness-tools-recovery.test.ts | 4 +- packages/durable/test/harness-tools.test.ts | 4 +- .../durable/test/harness-turn-limit.test.ts | 6 +- .../durable/test/harness-view-run.test.ts | 11 +- packages/durable/test/nested-images-bench.ts | 19 +- packages/durable/test/nested-tools-bench.ts | 7 +- .../durable/test/scheduler-fanout-bench.ts | 7 +- .../test/session-document-retention.test.ts | 6 +- packages/durable/test/spec-usage.test.ts | 6 +- .../test/system-order-cache-e2e.test.ts | 4 +- packages/durable/test/tool-output-bench.ts | 5 +- packages/durable/test/tool-search.test.ts | 7 +- packages/mcp/CHANGELOG.md | 4 + packages/mcp/README.md | 3 +- packages/schema/CHANGELOG.md | 3 + packages/schema/README.md | 62 +- packages/schema/src/checks.ts | 6 +- packages/schema/src/coerce.ts | 556 +++++++ packages/schema/src/explain.ts | 89 +- packages/schema/src/formats/index.ts | 9 +- .../typebox/json_pointer_uri_fragment.ts | 38 + packages/schema/src/formats/typebox/url.ts | 36 + packages/schema/src/index.ts | 4 +- packages/schema/src/resolver.ts | 24 +- packages/schema/src/types.ts | 13 + packages/schema/src/validator.ts | 47 +- packages/schema/test/coerce-typebox.test.ts | 1344 +++++++++++++++++ packages/schema/test/coerce.test.ts | 458 ++++++ packages/schema/test/validator.test.ts | 133 ++ scripts/browser-smoke-entry.ts | 10 +- scripts/check-entry-graphs.mjs | 34 +- tsconfig.json | 2 +- 139 files changed, 5115 insertions(+), 1305 deletions(-) create mode 100644 packages/ai/src/providers/schema-builders.ts rename packages/ai/src/utils/{typebox-helpers.ts => string-enum.ts} (68%) create mode 100644 packages/ai/test/validation-probes.test.ts create mode 100644 packages/coding-agent/examples/tsconfig.json create mode 100644 packages/coding-agent/src/core/extensions/pi-ai-extension-view.ts create mode 100644 packages/coding-agent/src/core/pi-ai-schemas.ts create mode 100644 packages/schema/src/coerce.ts create mode 100644 packages/schema/src/formats/typebox/json_pointer_uri_fragment.ts create mode 100644 packages/schema/src/formats/typebox/url.ts create mode 100644 packages/schema/test/coerce-typebox.test.ts create mode 100644 packages/schema/test/coerce.test.ts diff --git a/package-lock.json b/package-lock.json index d26f79648..d37cfbbba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5442,7 +5442,7 @@ "license": "MIT", "dependencies": { "@earendil-works/pi-ai": "1.4.0", - "typebox": "1.3.27" + "@earendil-works/pi-schema": "1.4.0" }, "devDependencies": { "@types/node": "22.19.19", @@ -5466,14 +5466,14 @@ "dependencies": { "@anthropic-ai/sdk": "0.129.0", "@aws-sdk/client-bedrock-runtime": "3.1127.0", + "@earendil-works/pi-schema": "1.4.0", "@earendil-works/pi-telemetry": "1.4.0", "@google/genai": "2.21.0", "@smithy/node-http-handler": "4.12.1", "http-proxy-agent": "9.1.0", "https-proxy-agent": "9.1.0", "openai": "7.19.0", - "partial-json": "0.1.7", - "typebox": "1.3.27" + "partial-json": "0.1.7" }, "bin": { "pi-ai": "dist/cli.js" @@ -5481,6 +5481,7 @@ "devDependencies": { "@types/node": "22.19.19", "canvas": "3.2.3", + "typebox": "1.3.27", "vitest": "4.1.12" }, "engines": { @@ -5714,10 +5715,10 @@ "@earendil-works/chord": "1.4.0", "@earendil-works/pi-ai": "1.4.0", "@earendil-works/pi-codemode": "1.4.0", + "@earendil-works/pi-schema": "1.4.0", "@earendil-works/pi-telemetry": "1.4.0", "@silvia-odwyer/photon": "0.3.3", - "diff": "8.0.4", - "typebox": "1.3.27" + "diff": "8.0.4" }, "devDependencies": { "@cloudflare/vitest-plugin": "1.4.10", diff --git a/package.json b/package.json index 352767d97..daa608b30 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "build:native:darwin": "npm --prefix packages/tui run build:native:darwin", "build:native:linux": "npm --prefix packages/tui run build:native:linux", "build:native:win32": "npm --prefix packages/tui run build:native:win32", - "check": "node scripts/biome-check.mjs --write && npm run check:pinned-deps && npm run check:runtime-deps && npm run check:ts-imports && npm run check:entry-graphs && npm run check:install-lock:coding-agent && tsc --noEmit && npm run check:browser-smoke", + "check": "node scripts/biome-check.mjs --write && npm run check:pinned-deps && npm run check:runtime-deps && npm run check:ts-imports && npm run check:entry-graphs && npm run check:install-lock:coding-agent && tsc --noEmit && tsc --noEmit -p packages/coding-agent/examples/tsconfig.json && npm run check:browser-smoke", "check:browser-smoke": "node scripts/check-browser-smoke.mjs", "check:pinned-deps": "node scripts/check-pinned-deps.mjs", "check:runtime-deps": "node scripts/check-runtime-deps.mjs", diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index 2ae23784d..f095f4f4f 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Breaking Changes + +- `AgentTool` takes any JSON Schema object as `parameters` (`AgentTool`) and types `params` and `prepareArguments` through `@earendil-works/pi-schema` (`AgentToolParams`): pi-schema's builders give their static type, other schemas (TypeBox included) give `unknown`, `AgentTool` stays `any`. `outputSchema` is any JSON Schema object. `typebox` is no longer a dependency. Migration: build parameters with pi-schema (`obj({ path: str() })` instead of `Type.Object({ path: Type.String() })`), or keep TypeBox schemas and narrow `params` yourself. + ### Fixed - Fixed `streamProxy()` re-parsing the accumulated tool-call arguments on every delta and keeping their raw text on the partial content block ([#9265](https://github.com/earendil-works/pi/issues/9265)) diff --git a/packages/agent/README.md b/packages/agent/README.md index 38625ce11..a6fa8256b 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -455,18 +455,20 @@ const agent = new Agent({ ## Tools -Define tools using `AgentTool`: +Define tools using `AgentTool`. `parameters` is any JSON Schema object; built with `@earendil-works/pi-schema`'s builders, it also types `params` (other schemas give `unknown`, `AgentTool` gives `any`): ```typescript -import { Type } from "typebox"; +import { obj, str } from "@earendil-works/pi-schema"; -const readFileTool: AgentTool = { +const readFileParameters = obj({ + path: str({ description: "File path" }), +}); + +const readFileTool: AgentTool = { name: "read_file", label: "Read File", // For UI display description: "Read a file's contents", - parameters: Type.Object({ - path: Type.String({ description: "File path" }), - }), + parameters: readFileParameters, // Override execution mode for this tool (optional). // "sequential" forces the entire batch to run one at a time. // "parallel" allows concurrent execution with other tool calls. diff --git a/packages/agent/examples/mcp-codemode/tools.ts b/packages/agent/examples/mcp-codemode/tools.ts index 37e358c65..06d678670 100644 --- a/packages/agent/examples/mcp-codemode/tools.ts +++ b/packages/agent/examples/mcp-codemode/tools.ts @@ -18,7 +18,7 @@ import { renderDeclarations, } from "@earendil-works/pi-codemode"; import { type McpClient, toLlmContent } from "@earendil-works/pi-mcp"; -import { Type } from "typebox"; +import { obj, str } from "@earendil-works/pi-schema"; /** Provider tool names are limited to 64 characters of `[A-Za-z0-9_-]`. */ function toToolName(prefix: string, name: string): string { @@ -45,12 +45,12 @@ export async function createMcpTools(client: McpClient, prefix: string): Promise label: tool.title ?? tool.name, description: tool.description ?? tool.title ?? `MCP tool ${tool.name}`, // Providers require an object schema, and some reject one without `properties`. - parameters: Type.Unsafe({ + parameters: { ...tool.inputSchema, type: "object", properties: tool.inputSchema.properties ?? {}, - }), - outputSchema: tool.outputSchema ? Type.Unsafe>(tool.outputSchema) : undefined, + }, + outputSchema: tool.outputSchema, execute: async (_toolCallId, params, signal, onUpdate) => { const result = await client.callTool(tool.name, params as Record, { signal, @@ -104,9 +104,11 @@ export function createNestedToolRunner( export const CODEMODE_TOOL_NAME = "codemode"; -const codemodeParameters = Type.Object({ - code: Type.String({ description: "JavaScript source. Top-level await and return work." }), -}); +const codemodeParameters = obj( + { code: str({ description: "JavaScript source. Top-level await and return work." }) }, + // Open, as the tool accepted extra arguments before it moved from TypeBox. + { open: true }, +); /** * A tool that runs model-written JavaScript in the codemode sandbox, where `tools.(args)` @@ -121,7 +123,6 @@ export function createCodemodeTool( const sandboxTools: CodemodeTool[] = callable.map((tool) => ({ name: tool.name, description: tool.description, - // TypeBox schemas are JSON Schema. inputSchema: tool.parameters as CodemodeJsonSchema, // What the script receives: the structured result if the tool declares one, else its text. outputSchema: (tool.outputSchema as CodemodeJsonSchema | undefined) ?? { type: "string" }, diff --git a/packages/agent/package.json b/packages/agent/package.json index a8f7de36f..5d7091e4a 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -24,7 +24,7 @@ }, "dependencies": { "@earendil-works/pi-ai": "1.4.0", - "typebox": "1.3.27" + "@earendil-works/pi-schema": "1.4.0" }, "keywords": [ "ai", diff --git a/packages/agent/src/types.ts b/packages/agent/src/types.ts index dbfba74a0..539fee1ae 100644 --- a/packages/agent/src/types.ts +++ b/packages/agent/src/types.ts @@ -14,7 +14,7 @@ import type { TranscriptContext, Usage, } from "@earendil-works/pi-ai"; -import type { Static, TSchema } from "typebox"; +import type { Static } from "@earendil-works/pi-schema"; /** * Stream function used by the agent loop. `Models.streamSimple` satisfies @@ -462,27 +462,39 @@ export interface AgentToolCallOutcome { */ export type AgentToolUpdateCallback = (partialResult: AgentToolResult) => void; -/** Tool definition used by the agent runtime. */ -export interface AgentTool extends Tool { +/** + * The arguments type of a tool with parameter schema `TParameters`: `Static` for schemas built with + * `@earendil-works/pi-schema`'s builders, `unknown` for other schemas, and `any` for `AgentTool`. + */ +export type AgentToolParams = IsAny extends true ? any : Static; + +// `0 extends 1 & T` is resolved too early for a `T` constrained to `object`; this form waits for the argument. +type IsAny = boolean extends (T extends never ? true : false) ? true : false; + +/** + * Tool definition used by the agent runtime. `parameters` is any JSON Schema object; see {@link AgentToolParams} for + * the type `execute` receives. + */ +export interface AgentTool extends Tool { /** Human-readable label for UI display. */ label: string; /** * Optional compatibility shim for raw tool-call arguments before schema validation. * Must return an object that matches `TParameters`. */ - prepareArguments?: (args: unknown) => Static; + prepareArguments?: (args: unknown) => AgentToolParams; /** * JSON Schema of `structuredContent` in successful results. Tools that declare it should always * set `structuredContent`. */ - outputSchema?: TSchema; + outputSchema?: object; /** * Execute the tool call. Throw on failure, or return a result with `isError: true`; do not only * describe the failure in `content`. */ execute: ( toolCallId: string, - params: Static, + params: AgentToolParams, signal?: AbortSignal, onUpdate?: AgentToolUpdateCallback, ) => Promise>; diff --git a/packages/agent/test/agent-loop.test.ts b/packages/agent/test/agent-loop.test.ts index 4a9386a13..b3ea8c259 100644 --- a/packages/agent/test/agent-loop.test.ts +++ b/packages/agent/test/agent-loop.test.ts @@ -6,7 +6,7 @@ import { type ToolResultMessage, type UserMessage, } from "@earendil-works/pi-ai"; -import { Type } from "typebox"; +import { arr, obj, str } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { agentLoop, agentLoopContinue, runAgentLoop, runToolCall } from "../src/agent-loop.ts"; import { setDefaultStreamFn } from "../src/index.ts"; @@ -303,7 +303,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should handle tool calls and results", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executed: string[] = []; const toolUsage = { input: 1, @@ -400,7 +400,7 @@ describe("agentLoop with AgentMessage", () => { // #10549 it("records how long execute() took on the tool result, excluding hooks", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); const tool: AgentTool = { name: "echo", @@ -454,7 +454,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should not execute tool calls from a length-truncated assistant message", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executed: string[] = []; const tool: AgentTool = { name: "echo", @@ -526,7 +526,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should execute mutated beforeToolCall args without revalidation", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executed: Array = []; const tool: AgentTool = { name: "echo", @@ -587,8 +587,8 @@ describe("agentLoop with AgentMessage", () => { }); it("should prepare tool arguments for validation", async () => { - const replaceSchema = Type.Object({ oldText: Type.String(), newText: Type.String() }); - const toolSchema = Type.Object({ edits: Type.Array(replaceSchema) }); + const replaceSchema = obj({ oldText: str(), newText: str() }); + const toolSchema = obj({ edits: arr(replaceSchema) }); const executed: Array> = []; const tool: AgentTool = { name: "edit", @@ -666,7 +666,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should emit tool_execution_end in completion order but persist tool results in source order", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); let firstResolved = false; let parallelObserved = false; let releaseFirst: (() => void) | undefined; @@ -760,7 +760,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should inject queued messages after all tool calls complete", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executed: string[] = []; const tool: AgentTool = { name: "echo", @@ -865,7 +865,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should force sequential execution when a tool has executionMode=sequential even with default parallel config", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); let firstResolved = false; let parallelObserved = false; let releaseFirst: (() => void) | undefined; @@ -947,7 +947,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should force sequential execution when one of multiple tools has executionMode=sequential", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executionOrder: string[] = []; let releaseSlow: (() => void) | undefined; const slowDone = new Promise((resolve) => { @@ -1033,7 +1033,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should allow parallel execution when all tools have executionMode=parallel", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); let firstResolved = false; let parallelObserved = false; let releaseFirst: (() => void) | undefined; @@ -1106,7 +1106,7 @@ describe("agentLoop with AgentMessage", () => { }); it("runs finishTurn after tool-result messages and before turn_end", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const tool: AgentTool = { name: "echo", label: "Echo", @@ -1215,7 +1215,7 @@ describe("agentLoop with AgentMessage", () => { ); it("action:end skips queue polling and next-turn preparation", async () => { - const toolSchema = Type.Object({}); + const toolSchema = obj({}); const tool: AgentTool = { name: "noop", label: "Noop", @@ -1305,7 +1305,7 @@ describe("agentLoop with AgentMessage", () => { }); it("lets a natural tool-result request satisfy continuation", async () => { - const toolSchema = Type.Object({}); + const toolSchema = obj({}); const tool: AgentTool = { name: "noop", label: "Noop", @@ -1501,7 +1501,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should use prepareNextTurn snapshot before continuing", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const tool: AgentTool = { name: "echo", label: "Echo", @@ -1578,7 +1578,7 @@ describe("agentLoop with AgentMessage", () => { }); it("picks up steering queued during prepareNextTurn before the next request", async () => { - const toolSchema = Type.Object({}); + const toolSchema = obj({}); const tool: AgentTool = { name: "noop", label: "Noop", @@ -1631,7 +1631,7 @@ describe("agentLoop with AgentMessage", () => { }); it("action:end receives finalized turn context and stops before queue polling", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executed: string[] = []; const tool: AgentTool = { name: "echo", @@ -1730,7 +1730,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should stop after a tool batch when every tool result sets terminate=true", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const tool: AgentTool = { name: "echo", label: "Echo", @@ -1781,7 +1781,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should stop after a blocked tool call when beforeToolCall sets terminate=true", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); let executed = false; const tool: AgentTool = { name: "echo", @@ -1839,7 +1839,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should continue after a mixed batch with one terminating blocked call", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executed: string[] = []; const tool: AgentTool = { name: "echo", @@ -1897,7 +1897,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should continue after parallel tool calls when not all tool results terminate", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const tool: AgentTool = { name: "echo", label: "Echo", @@ -1962,7 +1962,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should allow afterToolCall to mark a tool batch as terminating", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const tool: AgentTool = { name: "echo", label: "Echo", @@ -2131,13 +2131,13 @@ describe("agentLoopContinue with AgentMessage", () => { }); describe("runToolCall", () => { - const echoSchema = Type.Object({ value: Type.String() }); + const echoSchema = obj({ value: str() }); const echo: AgentTool = { name: "echo", label: "Echo", description: "Echo tool", parameters: echoSchema, - outputSchema: Type.Object({ value: Type.String() }), + outputSchema: obj({ value: str() }), async execute(_toolCallId, params, _signal, onUpdate) { onUpdate?.({ content: [{ type: "text", text: "partial" }], details: {} }); return { @@ -2151,7 +2151,7 @@ describe("runToolCall", () => { name: "failing", label: "Failing", description: "Returns an error result", - parameters: Type.Object({}), + parameters: obj({}), async execute() { return { content: [{ type: "text", text: "bad" }], details: { partial: true }, isError: true }; }, diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index 6fad6109f..9c8bddc39 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -7,7 +7,7 @@ import { toToolDeclaration, type UserMessage, } from "@earendil-works/pi-ai/compat"; -import { Type } from "typebox"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { Agent, @@ -49,7 +49,7 @@ function createTool(name: string): AgentTool { name, label: name, description: `${name} tool`, - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ content: [{ type: "text", text: name }], details: {} }), }; } @@ -146,7 +146,7 @@ describe("Agent", () => { name: "echo", label: "Echo", description: "Echo input", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ content: [{ type: "text", text: "echo" }], details: {} }), }; const agent = new Agent({ @@ -200,7 +200,7 @@ describe("Agent", () => { expect(update).toEqual({ role: "system", content: "", - toolsAdded: [{ name: "second", description: "second tool", parameters: Type.Object({}) }], + toolsAdded: [{ name: "second", description: "second tool", parameters: obj({}) }], toolsRemoved: [{ name: "first" }], timestamp: expect.any(Number), }); @@ -214,7 +214,7 @@ describe("Agent", () => { name: "echo", label: "Echo", description: "Echo input", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ content: [{ type: "text", text: "echo" }], details: {} }), }; const agent = new Agent({ @@ -239,7 +239,7 @@ describe("Agent", () => { role: "system", content: "", sections: { skills: "x" }, - toolsAdded: [{ name: "echo", description: "Echo input", parameters: Type.Object({}) }], + toolsAdded: [{ name: "echo", description: "Echo input", parameters: obj({}) }], timestamp: 1, }); }); @@ -284,7 +284,7 @@ describe("Agent", () => { name: "echo", label: "Echo", description: "Echo input", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ content: [{ type: "text", text: "echo" }], details: {} }), }; const agent = new Agent({ @@ -471,7 +471,7 @@ describe("Agent", () => { }); it("should ignore tool updates after the tool execution settles", async () => { - const toolSchema = Type.Object({}); + const toolSchema = obj({}); let delayedUpdate: AgentToolUpdateCallback<{ status: string }> | undefined; const events: AgentEvent[] = []; const unhandledRejections: unknown[] = []; @@ -536,7 +536,7 @@ describe("Agent", () => { }); it("should ignore a settled parallel tool update while another tool is still running", async () => { - const toolSchema = Type.Object({}); + const toolSchema = obj({}); const slowStarted = createDeferred(); const settledToolEnded = createDeferred(); const releaseSlow = createDeferred(); @@ -858,7 +858,7 @@ describe("Agent", () => { }); it("keeps legacy prepareNextTurn signal callback behavior", async () => { - const schema = Type.Object({}); + const schema = obj({}); const tool: AgentTool = { name: "noop", label: "Noop", @@ -899,7 +899,7 @@ describe("Agent", () => { }); it("forwards finishTurn through AgentOptions with the active abort signal", async () => { - const schema = Type.Object({}); + const schema = obj({}); const tool: AgentTool = { name: "noop", label: "Noop", diff --git a/packages/agent/test/utils/calculate.ts b/packages/agent/test/utils/calculate.ts index 0a4fefc58..333cd5a0e 100644 --- a/packages/agent/test/utils/calculate.ts +++ b/packages/agent/test/utils/calculate.ts @@ -1,5 +1,5 @@ import type { Usage } from "@earendil-works/pi-ai"; -import { type Static, Type } from "typebox"; +import { obj, type Static, str } from "@earendil-works/pi-schema"; import type { AgentTool, AgentToolResult } from "../../src/types.ts"; export interface CalculateResult extends AgentToolResult { @@ -16,9 +16,11 @@ export function calculate(expression: string): CalculateResult { } } -const calculateSchema = Type.Object({ - expression: Type.String({ description: "The mathematical expression to evaluate" }), -}); +const calculateSchema = obj( + { expression: str({ description: "The mathematical expression to evaluate" }) }, + // Open, as the tool accepted extra arguments before it moved from TypeBox. + { open: true }, +); type CalculateParams = Static; diff --git a/packages/agent/test/utils/get-current-time.ts b/packages/agent/test/utils/get-current-time.ts index a83a82af3..11996cc41 100644 --- a/packages/agent/test/utils/get-current-time.ts +++ b/packages/agent/test/utils/get-current-time.ts @@ -1,4 +1,4 @@ -import { type Static, Type } from "typebox"; +import { obj, opt, type Static, str } from "@earendil-works/pi-schema"; import type { AgentTool, AgentToolResult } from "../../src/types.ts"; export interface GetCurrentTimeResult extends AgentToolResult<{ utcTimestamp: number }> {} @@ -27,11 +27,11 @@ export async function getCurrentTime(timezone?: string): Promise; diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 996207fbd..6aae94ddb 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -4,6 +4,31 @@ ### Breaking Changes +- The main entry no longer re-exports TypeBox (`Type`, `Static`, `TSchema`), and `typebox` is no longer a dependency. `Tool.parameters` is any JSON Schema object (`Tool`): `@earendil-works/pi-schema`'s builders, which also type the arguments through pi-schema's `Static`, TypeBox schemas, or plain objects. `validateToolCall()`/`validateToolArguments()` validate with pi-schema (`compile(schema, { coerce: true }).parse()`): every schema gets the same coercion, the superset of TypeBox's `Value.Convert` rules (which only applied to TypeBox-built schemas) and pi-ai's own, so plain JSON Schema tools (MCP tools) now also get, for example, `null` to `"null"` for a required string, `"3.5"` to 3 for an integer and a single value wrapped into an array. The error text lists pi-schema's messages (`- edits.0.oldText: missing`, `- limit: expected number, got string`) instead of TypeBox's. `StringEnum()` returns a pi-schema `Schema` with the same JSON (`{ type: "string", enum }`). The model and compat schemas (`providers/model-schema`, `providers/compat-schema`) are built with pi-schema, with the same JSON Schema, and their types come from pi-schema's `Static`; `ProviderCompat` is exported. Migration: + + ```ts + // Before + import { Type, type Static, StringEnum } from "@earendil-works/pi-ai"; + const params = Type.Object({ path: Type.String(), mode: StringEnum(["a", "b"]) }); + type Params = Static; + + // After: pi-schema builders (closed objects; pass { open: true } to allow other properties) + import { StringEnum } from "@earendil-works/pi-ai"; + import { obj, type Static, str } from "@earendil-works/pi-schema"; + const params = obj({ path: str(), mode: StringEnum(["a", "b"]) }); + type Params = Static; + + // Or keep TypeBox, imported from typebox itself (its schemas are JSON Schema) + import { Type, type Static } from "typebox"; + ``` +- Tool arguments are validated as JSON Schema only; nothing runs through TypeBox's own validator any more, so TypeBox features outside JSON Schema no longer apply to tool arguments: + - `Type.Refine()` refinements are not run: arguments they would reject are accepted. + - `Type.BigInt()`, `Type.Undefined()` and `Type.Void()` reject every JSON argument (TypeBox converted to them). + - Codecs (`Type.Codec()`) do not decode: the tool receives the encoded JSON value. + - Formats registered with TypeBox's `Format.Set()` are not checked; the standard formats plus `url` and `json-pointer-uri-fragment` are. + - Property names are names, not regular expressions: `"a.b"` no longer also converts the argument `axb`. + - `additionalProperties` converts only undeclared properties; TypeBox also converted declared ones with it when an object had more than one property. +- A `$ref` that names nothing (a missing definition, an external document) rejects only arguments at its place (`- t: unresolved reference #/definitions/Missing`) instead of failing every call of the tool, as TypeBox's rejection did before. - `appendGrammarToolInputJsonDelta()` (`api/constrained-sampling`) takes the input delta instead of the whole input so far and no longer closes the input; close it with the new `closeGrammarToolInputJson()` ([#9265](https://github.com/earendil-works/pi/issues/9265)) ### Added diff --git a/packages/ai/README.md b/packages/ai/README.md index 583d55bc3..7af343eab 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -103,15 +103,16 @@ Unified LLM API with provider collections, automatic auth resolution, token and npm install @earendil-works/pi-ai ``` -TypeBox exports are re-exported from `@earendil-works/pi-ai`: `Type`, `Static`, and `TSchema`. +Tool parameters are JSON Schema. The examples build them with `@earendil-works/pi-schema`, which also types the arguments (`npm install @earendil-works/pi-schema`); TypeBox schemas and plain JSON Schema objects work as well. ## Quick Start You build a `Models` collection of providers and stream through it. The quickest start registers every built-in provider; apps that care about bundle size register individual providers instead (see [Provider Factories](#provider-factories) and [Bundling and Tree Shaking](#bundling-and-tree-shaking)). ```typescript -import { Type, type Context, type Tool } from '@earendil-works/pi-ai'; +import { type Context, type Tool } from '@earendil-works/pi-ai'; import { builtinModels } from '@earendil-works/pi-ai/providers/all'; +import { obj, opt, str } from '@earendil-works/pi-schema'; // A Models collection with every built-in provider registered const models = builtinModels(); @@ -119,12 +120,12 @@ const models = builtinModels(); // Sync lookup against the collection const model = models.getModel('openai', 'gpt-4o-mini')!; -// Define tools with TypeBox schemas for type safety and validation +// Define tools with JSON Schema; pi-schema's builders also type the arguments const tools: Tool[] = [{ name: 'get_time', description: 'Get the current time', - parameters: Type.Object({ - timezone: Type.Optional(Type.String({ description: 'Optional timezone (e.g., America/New_York)' })) + parameters: obj({ + timezone: opt(str({ description: 'Optional timezone (e.g., America/New_York)' })) }) }]; @@ -538,34 +539,35 @@ Amazon Bedrock resolves ambient AWS credentials (`AWS_PROFILE`, access key pairs ## Tools -Tools enable LLMs to interact with external systems. This library uses TypeBox schemas for type-safe tool definitions with automatic validation using TypeBox's built-in validator and value conversion utilities. TypeBox schemas can be serialized and deserialized as plain JSON, making them ideal for distributed systems. +Tools enable LLMs to interact with external systems. A tool's `parameters` is a JSON Schema object: built with `@earendil-works/pi-schema`'s builders (which also type the arguments through its `Static`), with TypeBox, or written by hand. `validateToolCall()` and `validateToolArguments()` validate with pi-schema, coercing loosely typed arguments first (numeric strings, `"true"`, a single value for an array, `null` for an omitted optional property). Schemas are plain JSON, so they serialize with the rest of a context. ### Defining Tools ```typescript -import { Type, type Tool, StringEnum } from '@earendil-works/pi-ai'; +import { type Tool, StringEnum } from '@earendil-works/pi-ai'; +import { arr, obj, str } from '@earendil-works/pi-schema'; -// Define tool parameters with TypeBox +// Define tool parameters with pi-schema's builders const weatherTool: Tool = { name: 'get_weather', description: 'Get current weather for a location', - parameters: Type.Object({ - location: Type.String({ description: 'City name or coordinates' }), + parameters: obj({ + location: str({ description: 'City name or coordinates' }), units: StringEnum(['celsius', 'fahrenheit'], { default: 'celsius' }) }) }; -// Note: For Google API compatibility, use StringEnum helper instead of Type.Enum -// Type.Enum generates anyOf/const patterns that Google doesn't support +// Note: StringEnum gives { type: "string", enum: [...] }, which every provider accepts, +// including Google's, which rejects anyOf/const patterns const bookMeetingTool: Tool = { name: 'book_meeting', description: 'Schedule a meeting', - parameters: Type.Object({ - title: Type.String({ minLength: 1 }), - startTime: Type.String({ format: 'date-time' }), - endTime: Type.String({ format: 'date-time' }), - attendees: Type.Array(Type.String({ format: 'email' }), { minItems: 1 }) + parameters: obj({ + title: str({ minLength: 1 }), + startTime: str({ format: 'date-time' }), + endTime: str({ format: 'date-time' }), + attendees: arr(str({ format: 'email' }), { minItems: 1 }) }) }; ``` @@ -578,10 +580,7 @@ Tools can opt in to provider-side constrained sampling. For JSON-schema tools, ` const strictTool: Tool = { name: 'edit_file', description: 'Edit a file', - parameters: Type.Object({ - path: Type.String(), - content: Type.String() - }, { additionalProperties: false }), + parameters: obj({ path: str(), content: str() }), // obj() is closed: additionalProperties: false constrainedSampling: { type: 'json_schema', strict: 'prefer' } }; ``` @@ -592,9 +591,7 @@ Strict JSON-schema constrained sampling is supported for OpenAI, Anthropic, supp const patchTool: Tool = { name: 'apply_patch', description: 'Apply a patch', - parameters: Type.Object({ - input: Type.String() - }, { additionalProperties: false }), + parameters: obj({ input: str() }), constrainedSampling: { type: 'grammar', variants: { @@ -698,7 +695,7 @@ for await (const event of s) { ### Validating Tool Arguments -When implementing your own tool execution loop, use `validateToolCall` to validate arguments before passing them to your tools: +When implementing your own tool execution loop, use `validateToolCall` to validate arguments before passing them to your tools. It coerces loosely typed arguments with pi-schema (`compile(schema, { coerce: true }).parse()`), returns the coerced arguments, and throws an error listing pi-schema's errors (`path: message`) followed by the arguments it received: ```typescript import { validateToolCall, type Tool } from '@earendil-works/pi-ai'; @@ -1726,8 +1723,8 @@ models.setProvider(openaiProvider()); Rules: -- `@earendil-works/pi-ai/models` exports the model runtime (`createModels`, `createProvider`, model helpers, and their types) without TypeBox, built-in catalogs, or SDK implementations. Other types can still use `import type` from the root. -- `@earendil-works/pi-ai` is the core entrypoint and does not import built-in catalogs, real provider factories, or SDK implementations, but it eagerly imports TypeBox and schema validation. Unbundled Node scripts do not tree-shake its unused exports; prefer `./models`, `./providers/faux`, and specific `./utils/*` subpaths when those are all you need. +- `@earendil-works/pi-ai/models` exports the model runtime (`createModels`, `createProvider`, model helpers, and their types) without schema validation, built-in catalogs, or SDK implementations. Other types can still use `import type` from the root. +- `@earendil-works/pi-ai` is the core entrypoint and does not import built-in catalogs, real provider factories, or SDK implementations, but it eagerly imports schema validation (pi-schema). Unbundled Node scripts do not tree-shake its unused exports; prefer `./models`, `./providers/faux`, and specific `./utils/*` subpaths when those are all you need. - `@earendil-works/pi-ai/providers/` imports that provider's catalog and lazy API wrapper only. - `@earendil-works/pi-ai/providers/all` imports every built-in provider factory and all catalogs. Use it only when you want the full built-in set. - With code splitting, provider SDKs stay in lazy chunks and load on first request. diff --git a/packages/ai/package.json b/packages/ai/package.json index 4e9d5d5a2..8de0434f8 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -69,14 +69,14 @@ "dependencies": { "@anthropic-ai/sdk": "0.129.0", "@aws-sdk/client-bedrock-runtime": "3.1127.0", + "@earendil-works/pi-schema": "1.4.0", "@earendil-works/pi-telemetry": "1.4.0", "@google/genai": "2.21.0", "@smithy/node-http-handler": "4.12.1", "http-proxy-agent": "9.1.0", "https-proxy-agent": "9.1.0", "openai": "7.19.0", - "partial-json": "0.1.7", - "typebox": "1.3.27" + "partial-json": "0.1.7" }, "keywords": [ "ai", @@ -101,6 +101,7 @@ "devDependencies": { "@types/node": "22.19.19", "canvas": "3.2.3", + "typebox": "1.3.27", "vitest": "4.1.12" } } diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 54a0345f1..0c9197ae6 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -1,6 +1,3 @@ -export type { Static, TSchema } from "typebox"; -export { Type } from "typebox"; - // Core only, side-effect free: no generated catalogs, no provider factories, // no api-registry, no OAuth implementations, no compat. Provider factories // live under "@earendil-works/pi-ai/providers/*", API implementations under @@ -42,9 +39,9 @@ export * from "./utils/json-parse.ts"; export * from "./utils/overflow.ts"; export * from "./utils/retry.ts"; export * from "./utils/streaming-json.ts"; +export * from "./utils/string-enum.ts"; export { contentText, getSystemMessageText, renderSystemMessageUpdate } from "./utils/text.ts"; export * from "./utils/transcript.ts"; -export * from "./utils/typebox-helpers.ts"; export { uuidv7 } from "./utils/uuid.ts"; export * from "./utils/validation.ts"; export { PI_AI_VERSION } from "./utils/version.ts"; diff --git a/packages/ai/src/providers/compat-schema.ts b/packages/ai/src/providers/compat-schema.ts index b3ba8198e..9327da45c 100644 --- a/packages/ai/src/providers/compat-schema.ts +++ b/packages/ai/src/providers/compat-schema.ts @@ -1,17 +1,35 @@ -import { type Static, type TProperties, type TSchemaOptions, Type } from "typebox"; +import { + type Annotations, + arr, + bool, + nul, + num, + opt, + type Properties, + type Static, + str, + union, +} from "@earendil-works/pi-schema"; import { ModelCostSchema } from "./model-schema.ts"; +import { openObject, stringLiteral, stringRecord } from "./schema-builders.ts"; -function sessionAffinityFormat(options?: TSchemaOptions) { - return Type.Union([Type.Literal("openai"), Type.Literal("openai-nosession"), Type.Literal("openrouter")], options); +function sessionAffinityFormat(options?: Annotations<"openai" | "openai-nosession" | "openrouter">) { + return union([stringLiteral("openai"), stringLiteral("openai-nosession"), stringLiteral("openrouter")], options); } export const SessionAffinityFormatSchema = sessionAffinityFormat({ description: "Session-affinity header format used to route related requests consistently.", }); -function thinkingTokenBudgetField(options?: TSchemaOptions) { - return Type.Union( - [Type.Literal("thinking_token_budget"), Type.Literal("thinking_budget"), Type.Literal("thinking_budget_tokens")], +function thinkingTokenBudgetField( + options?: Annotations<"thinking_token_budget" | "thinking_budget" | "thinking_budget_tokens">, +) { + return union( + [ + stringLiteral("thinking_token_budget"), + stringLiteral("thinking_budget"), + stringLiteral("thinking_budget_tokens"), + ], options, ); } @@ -21,84 +39,78 @@ export const ThinkingTokenBudgetFieldSchema = thinkingTokenBudgetField({ 'Top-level request field name used by OpenAI-compatible endpoints to cap reasoning tokens. "thinking_token_budget" is used by vLLM, "thinking_budget" by Qwen, DashScope, or SGLang, and "thinking_budget_tokens" by llama.cpp.', }); -export const ChatTemplateKwargValueSchema = Type.Union([ - Type.String(), - Type.Number(), - Type.Boolean(), - Type.Null(), - Type.Object({ - $var: Type.Union([ - Type.Literal("thinking.enabled"), - Type.Literal("thinking.effort"), - Type.Literal("thinking.budget"), +export const ChatTemplateKwargValueSchema = union([ + str(), + num(), + bool(), + nul(), + openObject({ + $var: union([ + stringLiteral("thinking.enabled"), + stringLiteral("thinking.effort"), + stringLiteral("thinking.budget"), ]), - omitWhenOff: Type.Optional(Type.Boolean()), + omitWhenOff: opt(bool()), }), ]); function percentileCutoffs(metric: string) { - return Type.Object({ - p50: Type.Optional(Type.Number({ description: `${metric} at the 50th percentile.` })), - p75: Type.Optional(Type.Number({ description: `${metric} at the 75th percentile.` })), - p90: Type.Optional(Type.Number({ description: `${metric} at the 90th percentile.` })), - p99: Type.Optional(Type.Number({ description: `${metric} at the 99th percentile.` })), + return openObject({ + p50: opt(num({ description: `${metric} at the 50th percentile.` })), + p75: opt(num({ description: `${metric} at the 75th percentile.` })), + p90: opt(num({ description: `${metric} at the 90th percentile.` })), + p99: opt(num({ description: `${metric} at the 99th percentile.` })), }); } -export const OpenRouterRoutingSchema = Type.Object( +export const OpenRouterRoutingSchema = openObject( { - allow_fallbacks: Type.Optional( - Type.Boolean({ description: "Whether to allow backup providers to serve requests.", default: true }), + allow_fallbacks: opt( + bool({ description: "Whether to allow backup providers to serve requests.", default: true }), ), - require_parameters: Type.Optional( - Type.Boolean({ + require_parameters: opt( + bool({ description: "Whether to filter providers to only those that support all parameters in the request.", default: false, }), ), - data_collection: Type.Optional( - Type.Union([Type.Literal("deny"), Type.Literal("allow")], { + data_collection: opt( + union([stringLiteral("deny"), stringLiteral("allow")], { description: 'Data collection setting. "allow": allow providers that may store or train on data. "deny": only use providers that do not collect user data.', default: "allow", }), ), - zdr: Type.Optional( - Type.Boolean({ description: "Whether to restrict routing to only ZDR (Zero Data Retention) endpoints." }), + zdr: opt(bool({ description: "Whether to restrict routing to only ZDR (Zero Data Retention) endpoints." })), + enforce_distillable_text: opt( + bool({ description: "Whether to restrict routing to only models that allow text distillation." }), ), - enforce_distillable_text: Type.Optional( - Type.Boolean({ description: "Whether to restrict routing to only models that allow text distillation." }), - ), - order: Type.Optional( - Type.Array(Type.String(), { + order: opt( + arr(str(), { description: "An ordered list of provider names or slugs to try in sequence, falling back to the next if unavailable.", }), ), - only: Type.Optional( - Type.Array(Type.String(), { + only: opt( + arr(str(), { description: "List of provider names or slugs to exclusively allow for this request.", }), ), - ignore: Type.Optional( - Type.Array(Type.String(), { description: "List of provider names or slugs to skip for this request." }), - ), - quantizations: Type.Optional( - Type.Array(Type.String(), { + ignore: opt(arr(str(), { description: "List of provider names or slugs to skip for this request." })), + quantizations: opt( + arr(str(), { description: 'A list of quantization levels to filter providers by, for example ["fp16", "bf16", "fp8", "fp6", "int8", "int4", "fp4", "fp32"].', }), ), - sort: Type.Optional( - Type.Union( + sort: opt( + union( [ - Type.String(), - Type.Object({ - by: Type.Optional( - Type.String({ description: 'The sorting metric, such as "price", "throughput", or "latency".' }), - ), - partition: Type.Optional( - Type.Union([Type.String(), Type.Null()], { + str(), + openObject({ + by: opt(str({ description: 'The sorting metric, such as "price", "throughput", or "latency".' })), + partition: opt( + union([str(), nul()], { description: 'Partitioning strategy: "model" or "none".', default: "model", }), @@ -111,33 +123,25 @@ export const OpenRouterRoutingSchema = Type.Object( }, ), ), - max_price: Type.Optional( - Type.Object( + max_price: opt( + openObject( { - prompt: Type.Optional( - Type.Union([Type.Number(), Type.String()], { description: "Price per million prompt tokens." }), - ), - completion: Type.Optional( - Type.Union([Type.Number(), Type.String()], { description: "Price per million completion tokens." }), - ), - image: Type.Optional(Type.Union([Type.Number(), Type.String()], { description: "Price per image." })), - audio: Type.Optional( - Type.Union([Type.Number(), Type.String()], { description: "Price per audio unit." }), - ), - request: Type.Optional( - Type.Union([Type.Number(), Type.String()], { description: "Price per request." }), - ), + prompt: opt(union([num(), str()], { description: "Price per million prompt tokens." })), + completion: opt(union([num(), str()], { description: "Price per million completion tokens." })), + image: opt(union([num(), str()], { description: "Price per image." })), + audio: opt(union([num(), str()], { description: "Price per audio unit." })), + request: opt(union([num(), str()], { description: "Price per request." })), }, { description: "Maximum price per million tokens in USD." }, ), ), - preferred_min_throughput: Type.Optional( - Type.Union([Type.Number(), percentileCutoffs("Minimum tokens per second")], { + preferred_min_throughput: opt( + union([num(), percentileCutoffs("Minimum tokens per second")], { description: "Preferred minimum throughput in tokens per second. A number applies to p50.", }), ), - preferred_max_latency: Type.Optional( - Type.Union([Type.Number(), percentileCutoffs("Maximum latency in seconds")], { + preferred_max_latency: opt( + union([num(), percentileCutoffs("Maximum latency in seconds")], { description: "Preferred maximum latency in seconds. A number applies to p50.", }), ), @@ -148,16 +152,16 @@ export const OpenRouterRoutingSchema = Type.Object( }, ); -export const VercelGatewayRoutingSchema = Type.Object( +export const VercelGatewayRoutingSchema = openObject( { - only: Type.Optional( - Type.Array(Type.String(), { + only: opt( + arr(str(), { description: 'List of provider slugs to exclusively use for this request, for example ["bedrock", "anthropic"].', }), ), - order: Type.Optional( - Type.Array(Type.String(), { + order: opt( + arr(str(), { description: 'List of provider slugs to try in order, for example ["anthropic", "openai"].', }), ), @@ -168,374 +172,382 @@ export const VercelGatewayRoutingSchema = Type.Object( }, ); -export const AnthropicAllowedFallbackModelSchema = Type.Object( +export const AnthropicAllowedFallbackModelSchema = openObject( { - provider: Type.String({ minLength: 1 }), - model: Type.String({ minLength: 1 }), + provider: str({ minLength: 1 }), + model: str({ minLength: 1 }), cost: ModelCostSchema, }, { description: "An Anthropic server-side refusal fallback model with local pricing metadata." }, ); -function optionalCompatBoolean(options?: TSchemaOptions) { - return Type.Optional(Type.Boolean(options)); +function optionalCompatBoolean(options?: Annotations) { + return opt(bool(options)); } -function optionalSessionAffinityFormat(options?: TSchemaOptions) { - return Type.Optional(sessionAffinityFormat(options)); +function optionalSessionAffinityFormat(options?: Annotations<"openai" | "openai-nosession" | "openrouter">) { + return opt(sessionAffinityFormat(options)); } -export const OpenAICompletionsCompatSchema = Type.Object( - { - supportsStore: Type.Optional( - Type.Boolean({ - description: "Whether the provider supports the store field. Default: auto-detected from URL.", - }), - ), - supportsDeveloperRole: optionalCompatBoolean({ - description: - "Whether the provider supports the developer role instead of system. Default: auto-detected from URL.", +const OpenAICompletionsCompatProperties = { + supportsStore: opt( + bool({ + description: "Whether the provider supports the store field. Default: auto-detected from URL.", }), - supportsReasoningEffort: Type.Optional( - Type.Boolean({ - description: "Whether the provider supports reasoning_effort. Default: auto-detected from URL.", - }), - ), - supportsUsageInStreaming: Type.Optional( - Type.Boolean({ - description: - "Whether the provider supports stream_options.include_usage for token usage in streaming responses.", - default: true, - }), - ), - supportsFinishReason: Type.Optional( - Type.Boolean({ - description: - "Whether streamed responses include finish_reason. When false, pi infers stop or toolUse when the stream ends.", - default: true, - }), - ), - maxTokensField: Type.Optional( - Type.Union([Type.Literal("max_completion_tokens"), Type.Literal("max_tokens")], { - description: "Which field to use for max tokens. Default: auto-detected from URL.", - }), - ), - requiresToolResultName: Type.Optional( - Type.Boolean({ description: "Whether tool results require the name field. Default: auto-detected from URL." }), - ), - requiresAssistantAfterToolResult: Type.Optional( - Type.Boolean({ - description: - "Whether a user message after tool results requires an assistant message in between. Default: auto-detected from URL.", - }), - ), - requiresThinkingAsText: Type.Optional( - Type.Boolean({ - description: - "Whether thinking blocks must be converted to text blocks with delimiters. Default: auto-detected from URL.", - }), - ), - requiresReasoningContentOnAssistantMessages: Type.Optional( - Type.Boolean({ - description: - "Whether all replayed assistant messages must include an empty reasoning_content field when reasoning is enabled. Default: auto-detected from URL.", - }), - ), - thinkingFormat: Type.Optional( - Type.Union( - [ - Type.Literal("openai"), - Type.Literal("openrouter"), - Type.Literal("deepseek"), - Type.Literal("together"), - Type.Literal("baseten"), - Type.Literal("zai"), - Type.Literal("qwen"), - Type.Literal("chat-template"), - Type.Literal("qwen-chat-template"), - Type.Literal("string-thinking"), - Type.Literal("ant-ling"), - ], - { - description: - 'Format for reasoning or thinking parameters. When omitted, Pi auto-detects the format from the provider URL. "openai" uses reasoning_effort, "openrouter" uses reasoning.effort, "deepseek" uses thinking.type plus reasoning_effort when supported, "together" uses reasoning.enabled plus reasoning_effort when supported, "baseten" uses configurable chat_template_args plus reasoning_effort when supported, "zai" uses thinking.type, "qwen" uses top-level enable_thinking, "qwen-chat-template" uses chat_template_kwargs.enable_thinking and preserve_thinking, "chat-template" uses configurable chat_template_kwargs, "string-thinking" uses top-level thinking, and "ant-ling" uses reasoning.effort only when the mapped effort is non-null.', - }, - ), - ), - chatTemplateKwargs: Type.Optional( - Type.Record(Type.String(), ChatTemplateKwargValueSchema, { - description: - 'Kwargs sent as chat_template_kwargs when thinkingFormat is "chat-template". Use $var with "thinking.enabled", "thinking.effort", or "thinking.budget" for pi-controlled values.', - }), - ), - chatTemplateArgs: Type.Optional( - Type.Record(Type.String(), ChatTemplateKwargValueSchema, { - description: - 'Arguments sent as chat_template_args when thinkingFormat is "baseten". Use $var with "thinking.enabled", "thinking.effort", or "thinking.budget" for pi-controlled values.', - }), - ), - openRouterRouting: Type.Optional(OpenRouterRoutingSchema), - vercelGatewayRouting: Type.Optional(VercelGatewayRoutingSchema), - zaiToolStream: Type.Optional( - Type.Boolean({ - description: "Whether z.ai supports top-level tool_stream for streaming tool call deltas.", - default: false, - }), - ), - thinkingTokenBudgetField: Type.Optional( - thinkingTokenBudgetField({ - description: - 'Top-level request field used to cap reasoning tokens from thinkingBudgets. Reasoning and the answer share max_tokens on these endpoints. "thinking_token_budget" is vLLM, "thinking_budget" is Qwen, DashScope, or SGLang, and "thinking_budget_tokens" is llama.cpp. Off by default and not set on the generated catalog.', - }), - ), - supportsThinkingTokenBudget: Type.Optional( - Type.Boolean({ - description: - 'Alias for thinkingTokenBudgetField: "thinking_token_budget" (vLLM). Prefer thinkingTokenBudgetField.', - default: false, - }), - ), - supportsOpenAIGrammarTools: optionalCompatBoolean({ - description: - "Whether the provider supports OpenAI custom tools with Lark or regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. The generated catalog enables this for capable models.", - default: false, - }), - supportsMidConvoSystemMessages: optionalCompatBoolean({ - description: - "Whether the exact model accepts system or developer messages after the conversation has started. When false, later system messages are folded into the leading system message. The generated catalog enables this for verified models.", - default: false, - }), - supportsMidConvoToolAdditions: Type.Optional( - Type.Boolean({ - description: - "Whether system messages can introduce additional tools mid-conversation. Requires supportsMidConvoSystemMessages. The generated catalog enables this for capable models.", - default: false, - }), - ), - supportsStrictMode: optionalCompatBoolean({ - description: - "Whether the provider supports the strict field in tool definitions. Generated capable models enable it explicitly.", - default: false, - }), - cacheControlFormat: Type.Optional( - Type.Literal("anthropic", { - description: - "Cache control convention for prompt caching. Anthropic applies cache_control markers to the system prompt, last tool definition, and last user, assistant, or tool-result text content.", - }), - ), - sendSessionAffinityHeaders: optionalCompatBoolean({ - description: - "Whether to send session-affinity data from options.sessionId. Default: true for OpenRouter endpoints, false otherwise.", - }), - sessionAffinityFormat: optionalSessionAffinityFormat({ - description: - "Session-affinity header format. openai sends session_id, x-client-request-id, and x-session-affinity; openai-nosession sends x-client-request-id and x-session-affinity; openrouter sends x-session-id. Does not affect prompt_cache_key. Default: auto-detected.", - }), - supportsLongCacheRetention: optionalCompatBoolean({ - description: - 'Whether the provider supports long prompt cache retention (prompt_cache_retention: "24h" or Anthropic-style cache_control.ttl: "1h", depending on format). Default: auto-detected from provider and URL.', - }), - vllmPriority: Type.Optional( - Type.Number({ - description: - "vLLM scheduler priority sent as the top-level priority request field. Lower values are handled earlier and the server default is 0. Only meaningful with --scheduling-policy priority. Off by default and not set on the generated catalog.", - }), - ), - }, - { + ), + supportsDeveloperRole: optionalCompatBoolean({ description: - "Compatibility settings for OpenAI-compatible completions APIs. Use this to override URL-based auto-detection for custom providers.", - additionalProperties: true, - }, -); - -export const OpenAIResponsesCompatSchema = Type.Object( - { - supportsDeveloperRole: optionalCompatBoolean({ - description: "Whether the provider supports the developer role instead of system.", + "Whether the provider supports the developer role instead of system. Default: auto-detected from URL.", + }), + supportsReasoningEffort: opt( + bool({ + description: "Whether the provider supports reasoning_effort. Default: auto-detected from URL.", + }), + ), + supportsUsageInStreaming: opt( + bool({ + description: + "Whether the provider supports stream_options.include_usage for token usage in streaming responses.", default: true, }), - supportsMidConvoSystemMessages: optionalCompatBoolean({ + ), + supportsFinishReason: opt( + bool({ description: - "Whether the exact model accepts developer or system messages after the conversation has started. When false, later system messages are folded into the leading system message. The generated catalog enables this for verified models.", - default: false, - }), - sessionAffinityFormat: optionalSessionAffinityFormat({ - description: - "Session-affinity header format. openai sends session_id and x-client-request-id; openai-nosession sends x-client-request-id; openrouter sends x-session-id. Does not affect prompt_cache_key. Default: auto-detected.", - }), - supportsLongCacheRetention: optionalCompatBoolean({ - description: - 'Whether the provider supports long prompt cache retention. This uses prompt_cache_options.ttl: "30m" on GPT-5.6+ and prompt_cache_retention: "24h" on earlier models.', + "Whether streamed responses include finish_reason. When false, pi infers stop or toolUse when the stream ends.", default: true, }), - supportsStrictMode: optionalCompatBoolean({ - description: - "Whether the provider supports strict JSON-schema function tools. Defaults are API-specific; generated OpenAI models enable it explicitly.", + ), + maxTokensField: opt( + union([stringLiteral("max_completion_tokens"), stringLiteral("max_tokens")], { + description: "Which field to use for max tokens. Default: auto-detected from URL.", }), - supportsOpenAIGrammarTools: optionalCompatBoolean({ + ), + requiresToolResultName: opt( + bool({ description: "Whether tool results require the name field. Default: auto-detected from URL." }), + ), + requiresAssistantAfterToolResult: opt( + bool({ description: - "Whether to emit OpenAI custom tools with Lark or regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. The generated catalog enables this for capable models.", + "Whether a user message after tool results requires an assistant message in between. Default: auto-detected from URL.", + }), + ), + requiresThinkingAsText: opt( + bool({ + description: + "Whether thinking blocks must be converted to text blocks with delimiters. Default: auto-detected from URL.", + }), + ), + requiresReasoningContentOnAssistantMessages: opt( + bool({ + description: + "Whether all replayed assistant messages must include an empty reasoning_content field when reasoning is enabled. Default: auto-detected from URL.", + }), + ), + thinkingFormat: opt( + union( + [ + stringLiteral("openai"), + stringLiteral("openrouter"), + stringLiteral("deepseek"), + stringLiteral("together"), + stringLiteral("baseten"), + stringLiteral("zai"), + stringLiteral("qwen"), + stringLiteral("chat-template"), + stringLiteral("qwen-chat-template"), + stringLiteral("string-thinking"), + stringLiteral("ant-ling"), + ], + { + description: + 'Format for reasoning or thinking parameters. When omitted, Pi auto-detects the format from the provider URL. "openai" uses reasoning_effort, "openrouter" uses reasoning.effort, "deepseek" uses thinking.type plus reasoning_effort when supported, "together" uses reasoning.enabled plus reasoning_effort when supported, "baseten" uses configurable chat_template_args plus reasoning_effort when supported, "zai" uses thinking.type, "qwen" uses top-level enable_thinking, "qwen-chat-template" uses chat_template_kwargs.enable_thinking and preserve_thinking, "chat-template" uses configurable chat_template_kwargs, "string-thinking" uses top-level thinking, and "ant-ling" uses reasoning.effort only when the mapped effort is non-null.', + }, + ), + ), + chatTemplateKwargs: opt( + stringRecord(ChatTemplateKwargValueSchema, { + description: + 'Kwargs sent as chat_template_kwargs when thinkingFormat is "chat-template". Use $var with "thinking.enabled", "thinking.effort", or "thinking.budget" for pi-controlled values.', + }), + ), + chatTemplateArgs: opt( + stringRecord(ChatTemplateKwargValueSchema, { + description: + 'Arguments sent as chat_template_args when thinkingFormat is "baseten". Use $var with "thinking.enabled", "thinking.effort", or "thinking.budget" for pi-controlled values.', + }), + ), + openRouterRouting: opt(OpenRouterRoutingSchema), + vercelGatewayRouting: opt(VercelGatewayRoutingSchema), + zaiToolStream: opt( + bool({ + description: "Whether z.ai supports top-level tool_stream for streaming tool call deltas.", default: false, }), - supportsAdditionalTools: Type.Optional( - Type.Boolean({ - description: "Whether the model supports message-anchored additional_tools input items.", - default: false, - }), - ), - supportsToolSearch: Type.Optional( - Type.Boolean({ - description: "Whether the model supports client-executed tool search for transcript-anchored additions.", - default: false, - }), - ), - supportsExplicitPromptCacheMode: Type.Optional( - Type.Boolean({ - description: "Whether the model accepts prompt_cache_options. Older OpenAI models reject the parameter.", - default: false, - }), - ), - supportsMidConvoEffort: Type.Optional( - Type.Boolean({ - description: - "Whether thinking level changes are sent as configuration_update input items instead of changing reasoning.effort, so a change keeps the prompt cache. Only used by the openai-responses API, and ignored with Sign in with ChatGPT on the openai provider.", - default: false, - }), - ), - supportsMaxOutputTokens: Type.Optional( - Type.Boolean({ - description: "Whether the provider accepts max_output_tokens. Some Codex-protocol gateways reject it.", - default: true, - }), - ), - }, - { description: "Compatibility settings for OpenAI Responses APIs.", additionalProperties: true }, -); + ), + thinkingTokenBudgetField: opt( + thinkingTokenBudgetField({ + description: + 'Top-level request field used to cap reasoning tokens from thinkingBudgets. Reasoning and the answer share max_tokens on these endpoints. "thinking_token_budget" is vLLM, "thinking_budget" is Qwen, DashScope, or SGLang, and "thinking_budget_tokens" is llama.cpp. Off by default and not set on the generated catalog.', + }), + ), + supportsThinkingTokenBudget: opt( + bool({ + description: + 'Alias for thinkingTokenBudgetField: "thinking_token_budget" (vLLM). Prefer thinkingTokenBudgetField.', + default: false, + }), + ), + supportsOpenAIGrammarTools: optionalCompatBoolean({ + description: + "Whether the provider supports OpenAI custom tools with Lark or regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. The generated catalog enables this for capable models.", + default: false, + }), + supportsMidConvoSystemMessages: optionalCompatBoolean({ + description: + "Whether the exact model accepts system or developer messages after the conversation has started. When false, later system messages are folded into the leading system message. The generated catalog enables this for verified models.", + default: false, + }), + supportsMidConvoToolAdditions: opt( + bool({ + description: + "Whether system messages can introduce additional tools mid-conversation. Requires supportsMidConvoSystemMessages. The generated catalog enables this for capable models.", + default: false, + }), + ), + supportsStrictMode: optionalCompatBoolean({ + description: + "Whether the provider supports the strict field in tool definitions. Generated capable models enable it explicitly.", + default: false, + }), + cacheControlFormat: opt( + stringLiteral("anthropic", { + description: + "Cache control convention for prompt caching. Anthropic applies cache_control markers to the system prompt, last tool definition, and last user, assistant, or tool-result text content.", + }), + ), + sendSessionAffinityHeaders: optionalCompatBoolean({ + description: + "Whether to send session-affinity data from options.sessionId. Default: true for OpenRouter endpoints, false otherwise.", + }), + sessionAffinityFormat: optionalSessionAffinityFormat({ + description: + "Session-affinity header format. openai sends session_id, x-client-request-id, and x-session-affinity; openai-nosession sends x-client-request-id and x-session-affinity; openrouter sends x-session-id. Does not affect prompt_cache_key. Default: auto-detected.", + }), + supportsLongCacheRetention: optionalCompatBoolean({ + description: + 'Whether the provider supports long prompt cache retention (prompt_cache_retention: "24h" or Anthropic-style cache_control.ttl: "1h", depending on format). Default: auto-detected from provider and URL.', + }), + vllmPriority: opt( + num({ + description: + "vLLM scheduler priority sent as the top-level priority request field. Lower values are handled earlier and the server default is 0. Only meaningful with --scheduling-policy priority. Off by default and not set on the generated catalog.", + }), + ), +}; -export const AnthropicMessagesCompatSchema = Type.Object( - { - supportsEagerToolInputStreaming: Type.Optional( - Type.Boolean({ - description: - "Whether the provider accepts per-tool eager_input_streaming. When false, the Anthropic provider omits tools[].eager_input_streaming and sends the legacy fine-grained-tool-streaming-2025-05-14 beta header for tool-enabled requests.", - default: true, - }), - ), - supportsLongCacheRetention: optionalCompatBoolean({ - description: "Whether the provider supports Anthropic long cache retention through cache_control.ttl.", +export const OpenAICompletionsCompatSchema = openObject(OpenAICompletionsCompatProperties, { + description: + "Compatibility settings for OpenAI-compatible completions APIs. Use this to override URL-based auto-detection for custom providers.", + additionalProperties: true, +}); + +const OpenAIResponsesCompatProperties = { + supportsDeveloperRole: optionalCompatBoolean({ + description: "Whether the provider supports the developer role instead of system.", + default: true, + }), + supportsMidConvoSystemMessages: optionalCompatBoolean({ + description: + "Whether the exact model accepts developer or system messages after the conversation has started. When false, later system messages are folded into the leading system message. The generated catalog enables this for verified models.", + default: false, + }), + sessionAffinityFormat: optionalSessionAffinityFormat({ + description: + "Session-affinity header format. openai sends session_id and x-client-request-id; openai-nosession sends x-client-request-id; openrouter sends x-session-id. Does not affect prompt_cache_key. Default: auto-detected.", + }), + supportsLongCacheRetention: optionalCompatBoolean({ + description: + 'Whether the provider supports long prompt cache retention. This uses prompt_cache_options.ttl: "30m" on GPT-5.6+ and prompt_cache_retention: "24h" on earlier models.', + default: true, + }), + supportsStrictMode: optionalCompatBoolean({ + description: + "Whether the provider supports strict JSON-schema function tools. Defaults are API-specific; generated OpenAI models enable it explicitly.", + }), + supportsOpenAIGrammarTools: optionalCompatBoolean({ + description: + "Whether to emit OpenAI custom tools with Lark or regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. The generated catalog enables this for capable models.", + default: false, + }), + supportsAdditionalTools: opt( + bool({ + description: "Whether the model supports message-anchored additional_tools input items.", + default: false, + }), + ), + supportsToolSearch: opt( + bool({ + description: "Whether the model supports client-executed tool search for transcript-anchored additions.", + default: false, + }), + ), + supportsExplicitPromptCacheMode: opt( + bool({ + description: "Whether the model accepts prompt_cache_options. Older OpenAI models reject the parameter.", + default: false, + }), + ), + supportsMidConvoEffort: opt( + bool({ + description: + "Whether thinking level changes are sent as configuration_update input items instead of changing reasoning.effort, so a change keeps the prompt cache. Only used by the openai-responses API, and ignored with Sign in with ChatGPT on the openai provider.", + default: false, + }), + ), + supportsMaxOutputTokens: opt( + bool({ + description: "Whether the provider accepts max_output_tokens. Some Codex-protocol gateways reject it.", default: true, }), - sendSessionAffinityHeaders: optionalCompatBoolean({ + ), +}; + +export const OpenAIResponsesCompatSchema = openObject(OpenAIResponsesCompatProperties, { + description: "Compatibility settings for OpenAI Responses APIs.", + additionalProperties: true, +}); + +const AnthropicMessagesCompatProperties = { + supportsEagerToolInputStreaming: opt( + bool({ description: - "Whether to send x-session-affinity from options.sessionId when caching is enabled. Required for providers like Fireworks that use session affinity for prompt cache routing; requests to the same replica maximize cache hits. Default: true for OpenRouter endpoints, false otherwise.", + "Whether the provider accepts per-tool eager_input_streaming. When false, the Anthropic provider omits tools[].eager_input_streaming and sends the legacy fine-grained-tool-streaming-2025-05-14 beta header for tool-enabled requests.", + default: true, }), - sessionAffinityFormat: Type.Optional( - Type.Literal("openrouter", { - description: - "Session-affinity format. openrouter sends x-session-id; when unset, sends x-session-affinity.", - }), - ), - supportsCacheControlOnTools: Type.Optional( - Type.Boolean({ - description: - "Whether the provider supports Anthropic-style cache_control markers on tool definitions. When false, cache_control is omitted from tool parameters. Some Anthropic-compatible providers, such as Fireworks, do not support this field on tools and may reject or ignore it.", - default: true, - }), - ), - supportsTemperature: Type.Optional( - Type.Boolean({ - description: - "Whether the model accepts the Anthropic temperature request field. Claude Opus 4.7+ rejects non-default values.", - default: true, - }), - ), - forceAdaptiveThinking: Type.Optional( - Type.Boolean({ - description: - "Whether to force adaptive thinking (thinking.type: adaptive plus output_config.effort) regardless of model ID. Built-in models that require adaptive thinking set this in generated metadata. Custom Anthropic-compatible providers can set this to true for any model whose upstream requires the adaptive format. Set false to opt out on overridden built-in models.", - default: false, - }), - ), - allowEmptySignature: Type.Optional( - Type.Boolean({ - description: "Whether to replay empty thinking signatures instead of converting thinking to text.", - default: false, - }), - ), - supportsStrictTools: Type.Optional( - Type.Boolean({ - description: - "Whether the provider supports Anthropic strict tool schemas. Generated Anthropic models enable it explicitly.", - default: false, - }), - ), - supportsMidConvoEffort: Type.Optional( - Type.Boolean({ - description: - "Whether the exact model transport supports effort-only system messages and thinking binding controls.", - default: false, - }), - ), - supportsMidConvoSystemMessages: optionalCompatBoolean({ + ), + supportsLongCacheRetention: optionalCompatBoolean({ + description: "Whether the provider supports Anthropic long cache retention through cache_control.ttl.", + default: true, + }), + sendSessionAffinityHeaders: optionalCompatBoolean({ + description: + "Whether to send x-session-affinity from options.sessionId when caching is enabled. Required for providers like Fireworks that use session affinity for prompt cache routing; requests to the same replica maximize cache hits. Default: true for OpenRouter endpoints, false otherwise.", + }), + sessionAffinityFormat: opt( + stringLiteral("openrouter", { + description: "Session-affinity format. openrouter sends x-session-id; when unset, sends x-session-affinity.", + }), + ), + supportsCacheControlOnTools: opt( + bool({ description: - "Whether the exact model accepts system-role messages inside the conversation. When false, later system messages are folded into the top-level system prompt.", + "Whether the provider supports Anthropic-style cache_control markers on tool definitions. When false, cache_control is omitted from tool parameters. Some Anthropic-compatible providers, such as Fireworks, do not support this field on tools and may reject or ignore it.", + default: true, + }), + ), + supportsTemperature: opt( + bool({ + description: + "Whether the model accepts the Anthropic temperature request field. Claude Opus 4.7+ rejects non-default values.", + default: true, + }), + ), + forceAdaptiveThinking: opt( + bool({ + description: + "Whether to force adaptive thinking (thinking.type: adaptive plus output_config.effort) regardless of model ID. Built-in models that require adaptive thinking set this in generated metadata. Custom Anthropic-compatible providers can set this to true for any model whose upstream requires the adaptive format. Set false to opt out on overridden built-in models.", default: false, }), - supportsMidConvoToolChanges: Type.Optional( - Type.Boolean({ - description: - "Whether the exact model accepts mid-conversation tool_addition and tool_removal blocks. Requires supportsMidConvoSystemMessages.", - default: false, - }), - ), - allowedFallbackModels: Type.Optional( - Type.Array(AnthropicAllowedFallbackModelSchema, { - maxItems: 3, - description: - "Models Anthropic accepts for server-side refusal fallback, with local pricing metadata for returned fallback responses. When absent or empty, callers must omit fallbacks; Anthropic rejects the field for models with no permitted fallback targets.", - }), - ), - }, - { description: "Compatibility settings for Anthropic Messages-compatible APIs.", additionalProperties: true }, -); - -export const BedrockCompatSchema = Type.Object( - { - supportsStrictMode: optionalCompatBoolean({ - description: "Whether the model supports Bedrock strict tool schemas.", + ), + allowEmptySignature: opt( + bool({ + description: "Whether to replay empty thinking signatures instead of converting thinking to text.", default: false, }), - }, - { description: "Compatibility settings for Amazon Bedrock models.", additionalProperties: true }, -); - -export const MistralConversationsCompatSchema = Type.Object( - { - supportsMidConvoSystemMessages: optionalCompatBoolean({ + ), + supportsStrictTools: opt( + bool({ description: - "Whether the exact model accepts system messages after the conversation has started. When false, later system messages are folded into the leading system message.", + "Whether the provider supports Anthropic strict tool schemas. Generated Anthropic models enable it explicitly.", default: false, }), - }, - { description: "Compatibility settings for the Mistral chat API.", additionalProperties: true }, -); - -export const SystemOneCompatSchema = Type.Object( - { - imageInput: Type.Optional( - Type.Union([Type.Literal("assets"), Type.Literal("images"), Type.Literal("state-image-urls")], { - description: - 'How the service takes classifier images. "assets" sends TypeSafe\'s assets map (PNG and JPEG) and references the images from the state, as TypeSafe and Jev on Workers AI expect. "images" sends Clef\'s images list of data URLs (PNG, JPEG, and WebP, at most four), as Clef on Workers AI expects. "state-image-urls" places image_url parts before the state, as OpenRouter expects; OpenRouter drops assets without an error. Auto-detected from the provider, baseUrl, and model ID. Classifier models read it from the provider-level compat; models and modelOverrides entries apply to chat models only.', - }), - ), - sendImageOptInHeader: optionalCompatBoolean({ + ), + supportsMidConvoEffort: opt( + bool({ description: - "Whether requests with images send the TypeSafe-Danger-Zone-Images: true header, which TypeSafe's own API requires for its experimental image input. Defaults to true for the typesafe-system-one API unless the model is detected as OpenRouter.", + "Whether the exact model transport supports effort-only system messages and thinking binding controls.", + default: false, }), - }, - { description: "Compatibility settings for System One classifier APIs.", additionalProperties: true }, -); + ), + supportsMidConvoSystemMessages: optionalCompatBoolean({ + description: + "Whether the exact model accepts system-role messages inside the conversation. When false, later system messages are folded into the top-level system prompt.", + default: false, + }), + supportsMidConvoToolChanges: opt( + bool({ + description: + "Whether the exact model accepts mid-conversation tool_addition and tool_removal blocks. Requires supportsMidConvoSystemMessages.", + default: false, + }), + ), + allowedFallbackModels: opt( + arr(AnthropicAllowedFallbackModelSchema, { + maxItems: 3, + description: + "Models Anthropic accepts for server-side refusal fallback, with local pricing metadata for returned fallback responses. When absent or empty, callers must omit fallbacks; Anthropic rejects the field for models with no permitted fallback targets.", + }), + ), +}; + +export const AnthropicMessagesCompatSchema = openObject(AnthropicMessagesCompatProperties, { + description: "Compatibility settings for Anthropic Messages-compatible APIs.", + additionalProperties: true, +}); + +const BedrockCompatProperties = { + supportsStrictMode: optionalCompatBoolean({ + description: "Whether the model supports Bedrock strict tool schemas.", + default: false, + }), +}; + +export const BedrockCompatSchema = openObject(BedrockCompatProperties, { + description: "Compatibility settings for Amazon Bedrock models.", + additionalProperties: true, +}); + +const MistralConversationsCompatProperties = { + supportsMidConvoSystemMessages: optionalCompatBoolean({ + description: + "Whether the exact model accepts system messages after the conversation has started. When false, later system messages are folded into the leading system message.", + default: false, + }), +}; + +export const MistralConversationsCompatSchema = openObject(MistralConversationsCompatProperties, { + description: "Compatibility settings for the Mistral chat API.", + additionalProperties: true, +}); + +const SystemOneCompatProperties = { + imageInput: opt( + union([stringLiteral("assets"), stringLiteral("images"), stringLiteral("state-image-urls")], { + description: + 'How the service takes classifier images. "assets" sends TypeSafe\'s assets map (PNG and JPEG) and references the images from the state, as TypeSafe and Jev on Workers AI expect. "images" sends Clef\'s images list of data URLs (PNG, JPEG, and WebP, at most four), as Clef on Workers AI expects. "state-image-urls" places image_url parts before the state, as OpenRouter expects; OpenRouter drops assets without an error. Auto-detected from the provider, baseUrl, and model ID. Classifier models read it from the provider-level compat; models and modelOverrides entries apply to chat models only.', + }), + ), + sendImageOptInHeader: optionalCompatBoolean({ + description: + "Whether requests with images send the TypeSafe-Danger-Zone-Images: true header, which TypeSafe's own API requires for its experimental image input. Defaults to true for the typesafe-system-one API unless the model is detected as OpenRouter.", + }), +}; + +export const SystemOneCompatSchema = openObject(SystemOneCompatProperties, { + description: "Compatibility settings for System One classifier APIs.", + additionalProperties: true, +}); const ProviderCompatPropertyOverrides = { supportsDeveloperRole: optionalCompatBoolean({ @@ -569,7 +581,7 @@ const ProviderCompatPropertyOverrides = { sendSessionAffinityHeaders: optionalCompatBoolean({ description: "Whether to send session-affinity data from options.sessionId. Defaults are API-specific.", }), -} satisfies TProperties; +} satisfies Properties; type UnionToIntersection = (Union extends unknown ? (value: Union) => void : never) extends ( value: infer Intersection, @@ -577,7 +589,7 @@ type UnionToIntersection = (Union extends unknown ? (value: Union) => voi ? Intersection : never; -type MergedCompatProperties = Omit< +type MergedCompatProperties = Omit< UnionToIntersection, keyof Overrides > & @@ -588,11 +600,11 @@ type MergedCompatProperties(propertyGroups: PropertyGroups, overrides: Overrides): MergedCompatProperties { - const merged: TProperties = {}; +function mergeCompatProperties( + propertyGroups: PropertyGroups, + overrides: Overrides, +): MergedCompatProperties { + const merged: Record = {}; const duplicateNames = new Set(); for (const properties of propertyGroups) { @@ -630,16 +642,25 @@ export const CompatSchemasByApi = { } as const; const ProviderCompatProperties = mergeCompatProperties( - [...new Set(Object.values(CompatSchemasByApi))].map((schema) => schema.properties), + [ + OpenAICompletionsCompatProperties, + OpenAIResponsesCompatProperties, + AnthropicMessagesCompatProperties, + BedrockCompatProperties, + MistralConversationsCompatProperties, + SystemOneCompatProperties, + ], ProviderCompatPropertyOverrides, ); -export const ProviderCompatSchema = Type.Object(ProviderCompatProperties, { +export const ProviderCompatSchema = openObject(ProviderCompatProperties, { description: "Provider and model compatibility overrides.", additionalProperties: true, }); export type ChatTemplateKwargValue = Static; +/** Provider and model compatibility overrides in models.json: every API's compat fields. */ +export type ProviderCompat = Static; /** Top-level request field used to cap reasoning tokens on OpenAI-compatible servers. */ export type ThinkingTokenBudgetField = Static; export type SessionAffinityFormat = Static; diff --git a/packages/ai/src/providers/model-schema.ts b/packages/ai/src/providers/model-schema.ts index 453ceabc7..2eb0a5a52 100644 --- a/packages/ai/src/providers/model-schema.ts +++ b/packages/ai/src/providers/model-schema.ts @@ -1,4 +1,17 @@ -import { type Static, type TSchemaOptions, Type } from "typebox"; +import { + type Annotations, + arr, + int, + nul, + num, + oneOf, + opt, + type Static, + type StaticObject, + str, + union, +} from "@earendil-works/pi-schema"; +import { openObject as object } from "./schema-builders.ts"; export const THINKING_LEVELS = ["minimal", "low", "medium", "high", "xhigh", "max"] as const; export const MODEL_THINKING_LEVELS = ["off", ...THINKING_LEVELS] as const; @@ -6,19 +19,31 @@ export const MODEL_INPUT_MODALITIES = ["text", "image"] as const; export const CACHE_RETENTIONS = ["none", "short", "long"] as const; export const MODEL_PROMPT_CACHE_RETENTIONS = ["short", "long"] as const; -export const ThinkingLevelSchema = Type.Enum(THINKING_LEVELS); -export const ModelThinkingLevelSchema = Type.Enum(MODEL_THINKING_LEVELS); -export const ModelInputModalitySchema = Type.Enum(MODEL_INPUT_MODALITIES); -export const CacheRetentionSchema = Type.Enum(CACHE_RETENTIONS); +export const ThinkingLevelSchema = oneOf(THINKING_LEVELS); +export const ModelThinkingLevelSchema = oneOf(MODEL_THINKING_LEVELS); +export const ModelInputModalitySchema = oneOf(MODEL_INPUT_MODALITIES); +export const CacheRetentionSchema = oneOf(CACHE_RETENTIONS); -const ThinkingLevelMapValueSchema = Type.Union([Type.String(), Type.Null()]); -export const ThinkingLevelMapSchema = Type.Partial(Type.Record(ModelThinkingLevelSchema, ThinkingLevelMapValueSchema), { - description: - "Provider value sent for each thinking level. null marks a level unsupported. Levels the model does not map use the provider's default value through high; xhigh and max are unsupported.", -}); +const ThinkingLevelMapValueSchema = union([str(), nul()]); +export const ThinkingLevelMapSchema = object( + { + off: opt(ThinkingLevelMapValueSchema), + minimal: opt(ThinkingLevelMapValueSchema), + low: opt(ThinkingLevelMapValueSchema), + medium: opt(ThinkingLevelMapValueSchema), + high: opt(ThinkingLevelMapValueSchema), + xhigh: opt(ThinkingLevelMapValueSchema), + max: opt(ThinkingLevelMapValueSchema), + }, + { + description: + "Provider value sent for each thinking level. null marks a level unsupported. Levels the model does not map use the provider's default value through high; xhigh and max are unsupported.", + }, +); -export const ModelPromptCacheSchema = Type.Partial( - Type.Record(Type.Enum(MODEL_PROMPT_CACHE_RETENTIONS), Type.Number({ exclusiveMinimum: 0 })), +const PromptCacheLifetimeSchema = num({ exclusiveMinimum: 0 }); +export const ModelPromptCacheSchema = object( + { short: opt(PromptCacheLifetimeSchema), long: opt(PromptCacheLifetimeSchema) }, { description: "Best-effort prompt cache lifetime in seconds for the short and long retention tiers. Pi warms only tiers with a known lifetime.", @@ -26,63 +51,54 @@ export const ModelPromptCacheSchema = Type.Partial( ); const ModelCostRatesProperties = { - input: Type.Number({ description: "Input cost in USD per million tokens." }), - output: Type.Number({ description: "Output cost in USD per million tokens." }), - cacheRead: Type.Number({ description: "Cache-read cost in USD per million tokens." }), - cacheWrite: Type.Number({ description: "Cache-write cost in USD per million tokens." }), + input: num({ description: "Input cost in USD per million tokens." }), + output: num({ description: "Output cost in USD per million tokens." }), + cacheRead: num({ description: "Cache-read cost in USD per million tokens." }), + cacheWrite: num({ description: "Cache-write cost in USD per million tokens." }), }; -export const ModelCostRatesSchema = Type.Object(ModelCostRatesProperties); -export const ModelCostTierSchema = Type.Object({ - inputTokensAbove: Type.Number({ +export const ModelCostRatesSchema = object(ModelCostRatesProperties); +export const ModelCostTierSchema = object({ + inputTokensAbove: num({ description: "Use this tier when total request input exceeds this token count.", }), ...ModelCostRatesProperties, }); -export const ModelCostSchema = Type.Object({ +export const ModelCostSchema = object({ ...ModelCostRatesProperties, - tiers: Type.Optional( - Type.Array(ModelCostTierSchema, { + tiers: opt( + arr(ModelCostTierSchema, { description: "Request-wide pricing tiers. The highest matching input threshold applies to the full request.", }), ), }); -function modelImageResizeOptions(options?: TSchemaOptions) { - return Type.Object( - { - maxWidth: Type.Optional(Type.Integer({ minimum: 1 })), - maxHeight: Type.Optional(Type.Integer({ minimum: 1 })), - maxBytes: Type.Optional( - Type.Integer({ minimum: 1, description: "Maximum base64-encoded payload size in bytes." }), - ), - jpegQuality: Type.Optional(Type.Integer({ minimum: 1, maximum: 100 })), - }, - options, - ); +const ModelImageResizeProperties = { + maxWidth: opt(int({ minimum: 1 })), + maxHeight: opt(int({ minimum: 1 })), + maxBytes: opt(int({ minimum: 1, description: "Maximum base64-encoded payload size in bytes." })), + jpegQuality: opt(int({ minimum: 1, maximum: 100 })), +}; + +function modelImageResizeOptions(options?: Annotations>) { + return object(ModelImageResizeProperties, options); } export const ModelImageResizeOptionsSchema = modelImageResizeOptions(); -export const ModelImageInputLimitsSchema = Type.Object({ - resize: Type.Optional( +export const ModelImageInputLimitsSchema = object({ + resize: opt( modelImageResizeOptions({ description: "Cache-safe resize profile applied before a new image enters conversation history.", }), ), - maxPerMessage: Type.Optional( - Type.Integer({ minimum: 1, description: "Maximum images accepted in one provider message." }), - ), - maxPerRequest: Type.Optional( - Type.Integer({ minimum: 1, description: "Maximum images accepted across one provider request." }), - ), + maxPerMessage: opt(int({ minimum: 1, description: "Maximum images accepted in one provider message." })), + maxPerRequest: opt(int({ minimum: 1, description: "Maximum images accepted across one provider request." })), }); -export const ModelInputLimitsSchema = Type.Object({ - maxRequestBytes: Type.Optional( - Type.Integer({ minimum: 1, description: "Maximum serialized provider request size in bytes." }), - ), - images: Type.Optional(ModelImageInputLimitsSchema), +export const ModelInputLimitsSchema = object({ + maxRequestBytes: opt(int({ minimum: 1, description: "Maximum serialized provider request size in bytes." })), + images: opt(ModelImageInputLimitsSchema), }); export type ThinkingLevel = Static; diff --git a/packages/ai/src/providers/schema-builders.ts b/packages/ai/src/providers/schema-builders.ts new file mode 100644 index 000000000..2df034ca4 --- /dev/null +++ b/packages/ai/src/providers/schema-builders.ts @@ -0,0 +1,34 @@ +// Builders for pi-ai's own configuration schemas (model metadata, compat settings): pi-schema's builders, shaped like the +// TypeBox schemas they replace, so models.json and the catalog validate exactly as before. + +import { + type Annotations, + obj, + type Properties, + type Schema, + type Static, + type StaticObject, +} from "@earendil-works/pi-schema"; + +/** An object that allows other properties; `additionalProperties: true` says so explicitly. */ +export function openObject( + properties: P, + options: Annotations> & { readonly additionalProperties?: true } = {}, +): Schema> { + const { additionalProperties, ...annotations } = options; + const schema = obj(properties, { ...annotations, open: true }); + return additionalProperties ? { ...schema, additionalProperties } : schema; +} + +/** One string: `{ type: "string", const: value }`. */ +export function stringLiteral(value: V, options: Annotations = {}): Schema { + return { type: "string", const: value, ...options }; +} + +/** An object with any string keys and values of `value`: `{ type: "object", patternProperties: { "^.*$": value } }`. */ +export function stringRecord( + value: S, + options: Annotations>> = {}, +): Schema>> { + return { type: "object", patternProperties: { "^.*$": value }, ...options }; +} diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 32b5a5f31..e435126ea 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -742,8 +742,6 @@ export interface ClassifierResult { timestamp: number; // Unix timestamp in milliseconds } -import type { TSchema } from "typebox"; - /** OpenAI grammar variants for constrained sampling. */ export type GrammarFormat = "openai_lark" | "openai_regex"; @@ -766,7 +764,11 @@ export type ConstrainedSamplingConfig = variants: GrammarVariants; }; -export interface Tool { +/** + * A tool the model may call. `parameters` is the JSON Schema of its arguments: pi-schema's builders (which also type + * the arguments, see `Static` in `@earendil-works/pi-schema`), TypeBox schemas, or any JSON Schema object. + */ +export interface Tool { name: string; description: string; parameters: TParameters; diff --git a/packages/ai/src/utils/typebox-helpers.ts b/packages/ai/src/utils/string-enum.ts similarity index 68% rename from packages/ai/src/utils/typebox-helpers.ts rename to packages/ai/src/utils/string-enum.ts index 85a53555c..b660178f8 100644 --- a/packages/ai/src/utils/typebox-helpers.ts +++ b/packages/ai/src/utils/string-enum.ts @@ -1,24 +1,24 @@ -import { type TUnsafe, Type } from "typebox"; +import type { Schema } from "@earendil-works/pi-schema"; /** * Creates a string enum schema compatible with Google's API and other providers - * that don't support anyOf/const patterns. + * that don't support anyOf/const patterns: `{ type: "string", enum: [...] }`. * * @example * const OperationSchema = StringEnum(["add", "subtract", "multiply", "divide"], { * description: "The operation to perform" * }); * - * type Operation = Static; // "add" | "subtract" | "multiply" | "divide" + * type Operation = Static; // "add" | "subtract" | "multiply" | "divide" (Static from @earendil-works/pi-schema) */ export function StringEnum( values: T, options?: { description?: string; default?: T[number] }, -): TUnsafe { - return Type.Unsafe({ +): Schema { + return { type: "string", - enum: values as any, + enum: values, ...(options?.description && { description: options.description }), ...(options?.default && { default: options.default }), - }); + }; } diff --git a/packages/ai/src/utils/transcript.ts b/packages/ai/src/utils/transcript.ts index 6f8a2691f..e9bec79f8 100644 --- a/packages/ai/src/utils/transcript.ts +++ b/packages/ai/src/utils/transcript.ts @@ -133,7 +133,7 @@ export function toToolDeclaration(tool: Tool): Tool { * Whether two tools declare the same interface to the model. * * Both sides go through {@link toToolDeclaration} first: its JSON round-trip drops the - * typebox symbol keys and `undefined` fields that a structural comparison would see, and + * schema metadata keys (TypeBox's non-enumerable kind marks) and `undefined` fields that a structural comparison would see, and * builds both objects with the same key order, so comparing the serialized declarations * is exact. This avoids a deep-equal dependency in a browser-safe package. */ diff --git a/packages/ai/src/utils/validation.ts b/packages/ai/src/utils/validation.ts index c9e818834..586765bf0 100644 --- a/packages/ai/src/utils/validation.ts +++ b/packages/ai/src/utils/validation.ts @@ -1,299 +1,34 @@ -import { Compile } from "typebox/compile"; -import type { TLocalizedValidationError } from "typebox/error"; -import { Value } from "typebox/value"; +import { type CoercingValidator, compile } from "@earendil-works/pi-schema"; +import { formats } from "@earendil-works/pi-schema/formats"; import type { Tool, ToolCall } from "../types.ts"; -const validatorCache = new WeakMap>(); -const TYPEBOX_KIND = Symbol.for("TypeBox.Kind"); +// One compiled validator per schema object: tool schemas are long-lived, so the WeakMap drops entries with their tools. +const validatorCache = new WeakMap(); -interface JsonSchemaObject { - type?: string | string[]; - properties?: Record; - required?: string[]; - items?: JsonSchemaObject | JsonSchemaObject[]; - additionalProperties?: boolean | JsonSchemaObject; - allOf?: JsonSchemaObject[]; - anyOf?: JsonSchemaObject[]; - oneOf?: JsonSchemaObject[]; -} - -function getSchemaTypes(schema: JsonSchemaObject): string[] { - if (typeof schema.type === "string") { - return [schema.type]; - } - if (Array.isArray(schema.type)) { - return schema.type.filter((type): type is string => typeof type === "string"); - } - return []; -} - -function matchesJsonType(value: unknown, type: string): boolean { - switch (type) { - case "number": - return typeof value === "number"; - case "integer": - return typeof value === "number" && Number.isInteger(value); - case "boolean": - return typeof value === "boolean"; - case "string": - return typeof value === "string"; - case "null": - return value === null; - case "array": - return Array.isArray(value); - case "object": - return typeof value === "object" && value !== null && !Array.isArray(value); - default: - return false; - } -} - -function getSubSchemaValidator(schema: JsonSchemaObject): ReturnType | undefined { - try { - return getValidator(schema as Tool["parameters"]); - } catch { - return undefined; - } -} - -function coercePrimitiveByType(value: unknown, type: string): unknown { - switch (type) { - case "number": { - if (value === null) { - return 0; - } - if (typeof value === "string" && value.trim() !== "") { - const parsed = Number(value); - if (Number.isFinite(parsed)) { - return parsed; - } - } - if (typeof value === "boolean") { - return value ? 1 : 0; - } - return value; - } - case "integer": { - if (value === null) { - return 0; - } - if (typeof value === "string" && value.trim() !== "") { - const parsed = Number(value); - if (Number.isInteger(parsed)) { - return parsed; - } - } - if (typeof value === "boolean") { - return value ? 1 : 0; - } - return value; - } - case "boolean": { - if (value === null) { - return false; - } - if (typeof value === "string") { - if (value === "true") { - return true; - } - if (value === "false") { - return false; - } - } - if (typeof value === "number") { - if (value === 1) { - return true; - } - if (value === 0) { - return false; - } - } - return value; - } - case "string": { - if (value === null) { - return ""; - } - if (typeof value === "number" || typeof value === "boolean") { - return String(value); - } - return value; - } - case "null": { - if (value === "" || value === 0 || value === false) { - return null; - } - return value; - } - default: - return value; - } -} - -function applySchemaObjectCoercion(value: Record, schema: JsonSchemaObject): void { - const properties = schema.properties; - const definedKeys = new Set(properties ? Object.keys(properties) : []); - - if (properties) { - for (const [key, propertySchema] of Object.entries(properties)) { - if (!(key in value)) { - continue; - } - value[key] = coerceWithJsonSchema(value[key], propertySchema); - } - } - - if (schema.additionalProperties && typeof schema.additionalProperties === "object") { - for (const [key, propertyValue] of Object.entries(value)) { - if (definedKeys.has(key)) { - continue; - } - value[key] = coerceWithJsonSchema(propertyValue, schema.additionalProperties); - } - } -} - -function applySchemaArrayCoercion(value: unknown[], schema: JsonSchemaObject): void { - if (Array.isArray(schema.items)) { - for (let index = 0; index < value.length; index++) { - const itemSchema = schema.items[index]; - if (!itemSchema) { - continue; - } - value[index] = coerceWithJsonSchema(value[index], itemSchema); - } - return; - } - - if (schema.items && typeof schema.items === "object") { - for (let index = 0; index < value.length; index++) { - value[index] = coerceWithJsonSchema(value[index], schema.items); - } - } -} - -function coerceWithUnionSchema(value: unknown, schemas: JsonSchemaObject[]): unknown { - for (const schema of schemas) { - const validator = getSubSchemaValidator(schema); - if (validator?.Check(value)) { - return value; - } - } - - for (const schema of schemas) { - const candidate = structuredClone(value); - const coerced = coerceWithJsonSchema(candidate, schema); - const validator = getSubSchemaValidator(schema); - if (validator?.Check(coerced)) { - return coerced; - } - } - return value; -} - -function coerceWithJsonSchema(value: unknown, schema: JsonSchemaObject): unknown { - let nextValue = value; - - if (Array.isArray(schema.allOf)) { - for (const nested of schema.allOf) { - nextValue = coerceWithJsonSchema(nextValue, nested); - } - } - - if (Array.isArray(schema.anyOf)) { - nextValue = coerceWithUnionSchema(nextValue, schema.anyOf); - } - - if (Array.isArray(schema.oneOf)) { - nextValue = coerceWithUnionSchema(nextValue, schema.oneOf); - } - - const schemaTypes = getSchemaTypes(schema); - const matchesUnionMember = - schemaTypes.length > 1 && schemaTypes.some((schemaType) => matchesJsonType(nextValue, schemaType)); - if (schemaTypes.length > 0 && !matchesUnionMember) { - for (const schemaType of schemaTypes) { - const candidate = coercePrimitiveByType(nextValue, schemaType); - if (candidate !== nextValue) { - nextValue = candidate; - break; - } - } - } - - if ( - schemaTypes.includes("object") && - typeof nextValue === "object" && - nextValue !== null && - !Array.isArray(nextValue) - ) { - applySchemaObjectCoercion(nextValue as Record, schema); - } - - if (schemaTypes.includes("array") && Array.isArray(nextValue)) { - applySchemaArrayCoercion(nextValue, schema); - } - - return nextValue; -} - -function normalizeOptionalNulls(value: unknown, schema: JsonSchemaObject): void { - if (Array.isArray(value)) { - if (Array.isArray(schema.items)) { - for (let index = 0; index < value.length; index++) { - const itemSchema = schema.items[index]; - if (itemSchema) normalizeOptionalNulls(value[index], itemSchema); - } - } else if (schema.items) { - for (const item of value) normalizeOptionalNulls(item, schema.items); - } - return; - } - if (typeof value !== "object" || value === null || !schema.properties) return; - - const object = value as Record; - const required = new Set(schema.required ?? []); - for (const [key, propertySchema] of Object.entries(schema.properties)) { - if (!(key in object)) continue; - if ( - object[key] === null && - !required.has(key) && - typeof (propertySchema as { $ref?: unknown }).$ref !== "string" && - getSubSchemaValidator(propertySchema)?.Check(null) === false - ) { - delete object[key]; - } else { - normalizeOptionalNulls(object[key], propertySchema); - } - } -} - -function getValidator(schema: Tool["parameters"]): ReturnType { - const key = schema as object; - const cached = validatorCache.get(key); +function getValidator(schema: Tool["parameters"]): CoercingValidator { + const cached = validatorCache.get(schema); if (cached) { return cached; } - const validator = Compile(schema); - validatorCache.set(key, validator); + // A $ref that names nothing (a dangling definition in an MCP server's schema) rejects only values at its place, + // instead of making every call of the tool fail. + const validator = compile(schema, { formats, coerce: true, unresolvedRefs: "reject" }); + validatorCache.set(schema, validator); return validator; } -function formatValidationPath(error: TLocalizedValidationError): string { - if (error.keyword === "required") { - const requiredProperties = (error.params as { requiredProperties?: string[] }).requiredProperties; - const requiredProperty = requiredProperties?.[0]; - if (requiredProperty) { - const basePath = error.instancePath.replace(/^\//, "").replace(/\//g, "."); - return basePath ? `${basePath}.${requiredProperty}` : requiredProperty; - } - } - const path = error.instancePath.replace(/^\//, "").replace(/\//g, "."); - return path || "root"; +/** A pi-schema error path (a JSON Pointer) as the dotted path the model sees: `/edits/0/oldText` -> `edits.0.oldText`. */ +function formatValidationPath(pointer: string): string { + if (pointer === "") return "root"; + return pointer + .slice(1) + .split("/") + .map((segment) => segment.replace(/~1/g, "/").replace(/~0/g, "~")) + .join("."); } /** - * Finds a tool by name and validates the tool call arguments against its TypeBox schema + * Finds a tool by name and validates the tool call arguments against its JSON Schema * @param tools Array of tool definitions * @param toolCall The tool call from the LLM * @returns The validated arguments @@ -308,8 +43,10 @@ export function validateToolCall(tools: Tool[], toolCall: ToolCall): any { } /** - * Validates tool call arguments against the tool's TypeBox schema - * @param tool The tool definition with TypeBox schema + * Validates tool call arguments against the tool's JSON Schema (any JSON Schema: pi-schema's builders, TypeBox, plain + * objects). The arguments are coerced first (pi-schema's `parse()`): models often send `"42"` for a number, a single + * value for an array, or `null` for an omitted optional property. + * @param tool The tool definition with its JSON Schema * @param toolCall The tool call from the LLM * @returns The validated (and potentially coerced) arguments * @throws Error with formatted message if validation fails @@ -318,34 +55,13 @@ export function validateToolArguments(tool: Tool, toolCall: ToolCall): any { if (toolCall.argumentsError) { throw new Error(`Invalid arguments for tool "${toolCall.name}": ${toolCall.argumentsError}`); } - const args = structuredClone(toolCall.arguments); - normalizeOptionalNulls(args, tool.parameters as JsonSchemaObject); - Value.Convert(tool.parameters, args); - - const validator = getValidator(tool.parameters); - if (!Object.getOwnPropertySymbols(tool.parameters).includes(TYPEBOX_KIND)) { - const coerced = coerceWithJsonSchema(args, tool.parameters as JsonSchemaObject); - if (coerced !== args) { - if (typeof args === "object" && args !== null && typeof coerced === "object" && coerced !== null) { - for (const key of Object.keys(args)) { - delete args[key]; - } - Object.assign(args, coerced); - } else { - return validator.Check(coerced) ? coerced : args; - } - } + // A copy, so tools that modify their arguments never modify the tool call in the transcript. + const result = getValidator(tool.parameters).parse(structuredClone(toolCall.arguments)); + if (result.ok) { + return result.value; } - if (validator.Check(args)) { - return args; - } - - const errors = - validator - .Errors(args) - .map((error) => ` - ${formatValidationPath(error)}: ${error.message}`) - .join("\n") || "Unknown validation error"; + const errors = result.errors.map((error) => ` - ${formatValidationPath(error.path)}: ${error.message}`).join("\n"); const errorMessage = `Validation failed for tool "${toolCall.name}":\n${errors}\n\nReceived arguments:\n${JSON.stringify(toolCall.arguments, null, 2)}`; diff --git a/packages/ai/test/compat-schema.test.ts b/packages/ai/test/compat-schema.test.ts index 4ef40713e..d1a6d66ca 100644 --- a/packages/ai/test/compat-schema.test.ts +++ b/packages/ai/test/compat-schema.test.ts @@ -1,5 +1,4 @@ -import type { Static } from "typebox"; -import { Compile } from "typebox/compile"; +import type { Static } from "@earendil-works/pi-schema"; import { describe, expect, expectTypeOf, it } from "vitest"; import { AnthropicMessagesCompatSchema, @@ -11,6 +10,16 @@ import { type ProviderCompat = Static; +/** The `default` annotation of each property of an object schema. */ +function defaults(schema: { readonly [keyword: string]: unknown }): Record { + const properties = schema.properties as Record; + return Object.fromEntries( + Object.entries(properties) + .filter(([, property]) => property.default !== undefined) + .map(([name, property]) => [name, property.default]), + ); +} + describe("compatibility schemas", () => { it("preserves property types in the provider superset", () => { expectTypeOf().toEqualTypeOf(); @@ -20,21 +29,21 @@ describe("compatibility schemas", () => { }); it("preserves API-specific defaults", () => { - const completionsDefaults = Compile(OpenAICompletionsCompatSchema).Default({}); + const completionsDefaults = defaults(OpenAICompletionsCompatSchema); expect(completionsDefaults).not.toHaveProperty("thinkingFormat"); expect(completionsDefaults).not.toHaveProperty("supportsLongCacheRetention"); - expect(Compile(OpenAIResponsesCompatSchema).Default({})).toMatchObject({ supportsDeveloperRole: true }); - const anthropicDefaults = Compile(AnthropicMessagesCompatSchema).Default({}); + expect(defaults(OpenAIResponsesCompatSchema)).toMatchObject({ supportsDeveloperRole: true }); + const anthropicDefaults = defaults(AnthropicMessagesCompatSchema); expect(anthropicDefaults).not.toHaveProperty("sendSessionAffinityHeaders"); expect(anthropicDefaults).toMatchObject({ supportsLongCacheRetention: true }); - expect(Compile(BedrockCompatSchema).Default({})).toMatchObject({ supportsStrictMode: false }); + expect(defaults(BedrockCompatSchema)).toMatchObject({ supportsStrictMode: false }); }); it("does not assign API-specific defaults to the provider superset", () => { - const defaults = Compile(ProviderCompatSchema).Default({}); - expect(defaults).not.toHaveProperty("supportsDeveloperRole"); - expect(defaults).not.toHaveProperty("sendSessionAffinityHeaders"); - expect(defaults).not.toHaveProperty("supportsLongCacheRetention"); - expect(defaults).not.toHaveProperty("supportsStrictMode"); + const providerDefaults = defaults(ProviderCompatSchema); + expect(providerDefaults).not.toHaveProperty("supportsDeveloperRole"); + expect(providerDefaults).not.toHaveProperty("sendSessionAffinityHeaders"); + expect(providerDefaults).not.toHaveProperty("supportsLongCacheRetention"); + expect(providerDefaults).not.toHaveProperty("supportsStrictMode"); }); }); diff --git a/packages/ai/test/faux-provider.test.ts b/packages/ai/test/faux-provider.test.ts index ad23ed938..afdf685b6 100644 --- a/packages/ai/test/faux-provider.test.ts +++ b/packages/ai/test/faux-provider.test.ts @@ -1,3 +1,4 @@ +import { Type } from "typebox"; import { afterEach, describe, expect, it } from "vitest"; import { complete, @@ -7,7 +8,6 @@ import { fauxToolCall, registerFauxProvider, stream, - Type, } from "../src/compat.ts"; import type { AssistantMessageEvent, Context } from "../src/types.ts"; diff --git a/packages/ai/test/interleaved-thinking.test.ts b/packages/ai/test/interleaved-thinking.test.ts index b4da12835..b787a2cf7 100644 --- a/packages/ai/test/interleaved-thinking.test.ts +++ b/packages/ai/test/interleaved-thinking.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest"; import { completeSimple, getModel } from "../src/compat.ts"; import { getEnvApiKey } from "../src/env-api-keys.ts"; import type { Api, Context, Model, StopReason, Tool, ToolCall, ToolResultMessage } from "../src/types.ts"; -import { StringEnum } from "../src/utils/typebox-helpers.ts"; +import { StringEnum } from "../src/utils/string-enum.ts"; import { hasBedrockCredentials } from "./bedrock-utils.ts"; const calculatorSchema = Type.Object({ diff --git a/packages/ai/test/stream.test.ts b/packages/ai/test/stream.test.ts index 6cac98a15..3cc02268b 100644 --- a/packages/ai/test/stream.test.ts +++ b/packages/ai/test/stream.test.ts @@ -9,7 +9,7 @@ import type { Api, Context, ImageContent, Model, StreamOptions, Tool, ToolResult type StreamOptionsWithExtras = StreamOptions & Record; -import { StringEnum } from "../src/utils/typebox-helpers.ts"; +import { StringEnum } from "../src/utils/string-enum.ts"; import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.ts"; import { hasBedrockCredentials } from "./bedrock-utils.ts"; import { hasCloudflareAiGatewayCredentials, hasCloudflareWorkersAICredentials } from "./cloudflare-utils.ts"; diff --git a/packages/ai/test/validation-probes.test.ts b/packages/ai/test/validation-probes.test.ts new file mode 100644 index 000000000..0ddfe8d12 --- /dev/null +++ b/packages/ai/test/validation-probes.test.ts @@ -0,0 +1,779 @@ +// The reviewers' probes of the schema switch (round 1) as regression tests: TypeBox-built and plain JSON +// (MCP-shaped) tool schemas with arguments through validateToolArguments. Each case's outcome is "ok ", "REJECT " or "THROW ". Cases marked as TypeBox-only record the documented +// differences from TypeBox (see the CHANGELOG): Refine and BigInt. + +// biome-ignore-all lint/suspicious/noTemplateCurlyInString: TypeBox template literal syntax. +// biome-ignore-all lint/suspicious/noThenProperty: JSON Schema's `then` keyword. +import { Type } from "typebox"; +import { describe, expect, it } from "vitest"; +import type { Tool, ToolCall } from "../src/types.ts"; +import { validateToolArguments } from "../src/utils/validation.ts"; + +/** The outcome of one call, in the probes' format. */ +function outcome(name: string, schema: object, args: unknown): string { + const tool = { name, description: "", parameters: schema } as Tool; + const call = { type: "toolCall", id: "1", name, arguments: args } as ToolCall; + try { + return `ok ${JSON.stringify(validateToolArguments(tool, call), (_key, value) => (typeof value === "bigint" ? `${value}n` : value))}`; + } catch (error) { + const message = (error as Error).message; + return message.startsWith("Validation failed") + ? `REJECT ${message.split("\n")[1]?.trim()}` + : `THROW ${message.split("\n")[0]}`; + } +} + +type Case = { name: string; schema: object; args: unknown[] }; + +const Node = Type.Cyclic( + { Node: Type.Object({ name: Type.String(), children: Type.Array(Type.Ref("Node")) }) }, + "Node", +); + +const cases: Case[] = [ + { + name: "cyclic", + schema: Type.Object({ tree: Node }), + args: [{ tree: { name: "a", children: [{ name: "b", children: [] }] } }, { tree: { name: 1, children: [] } }], + }, + { name: "ref-with-$id", schema: Type.Object({ a: Type.Ref("A") }), args: [{ a: { x: "1" } }] }, + { + name: "function", + schema: Type.Object({ f: Type.Optional(Type.Function([], Type.Void())) }), + args: [{}, { f: "x" }], + }, + { + name: "codec", + schema: Type.Object({ + c: Type.Codec(Type.String()) + .Decode((s) => Number(s)) + .Encode((n) => String(n)), + }), + args: [{ c: "12" }, { c: 12 }], + }, + { name: "undefined-optional", schema: Type.Object({ u: Type.Optional(Type.Undefined()) }), args: [{}, { u: null }] }, + { name: "bigint", schema: Type.Object({ n: Type.BigInt() }), args: [{ n: 1 }, { n: "12" }] }, + { + name: "refine", + schema: Type.Object({ n: Type.Refine(Type.Number(), (v) => v > 0) }), + args: [{ n: 5 }, { n: -5 }], + }, + { + name: "template-literal", + schema: Type.Object({ t: Type.TemplateLiteral("id-${number}") }), + args: [{ t: "id-12" }, { t: "id-x" }, { t: 3 }], + }, + { + name: "record-string", + schema: Type.Object({ r: Type.Record(Type.String(), Type.Number()) }), + args: [{ r: { a: "1" } }, { r: { a: 1, b: "x" } }], + }, + { + name: "record-literal-keys", + schema: Type.Object({ r: Type.Record(Type.Union([Type.Literal("a"), Type.Literal("b")]), Type.Number()) }), + args: [{ r: { a: 1, b: 2 } }, { r: { a: 1 } }, { r: { a: 1, b: 2, c: 3 } }], + }, + { + name: "intersect", + schema: Type.Intersect([Type.Object({ a: Type.Number() }), Type.Object({ b: Type.Boolean() })]), + args: [ + { a: "1", b: "true" }, + { a: 1, b: true, c: 1 }, + ], + }, + { + name: "union-objects", + schema: Type.Object({ + v: Type.Union([ + Type.Object({ k: Type.Literal("x"), n: Type.Number() }), + Type.Object({ k: Type.Literal("y"), s: Type.String() }), + ]), + }), + args: [{ v: { k: "x", n: "2" } }, { v: { k: "y", s: 2 } }], + }, + { name: "minLength-astral", schema: Type.Object({ s: Type.String({ minLength: 2 }) }), args: [{ s: "😀" }] }, + { name: "maxLength-astral", schema: Type.Object({ s: Type.String({ maxLength: 1 }) }), args: [{ s: "😀" }] }, + { name: "pattern-astral", schema: Type.Object({ s: Type.String({ pattern: "^.$" }) }), args: [{ s: "😀" }] }, + { + name: "pattern-class-dash", + schema: Type.Object({ s: Type.String({ pattern: "^[\\w-]+$" }) }), + args: [{ s: "a-b" }], + }, + { + name: "pattern-unicode-prop", + schema: Type.Object({ s: Type.String({ pattern: "^\\p{L}+$" }) }), + args: [{ s: "äb" }, { s: "p{L}" }], + }, + { + name: "multipleOf-float", + schema: Type.Object({ n: Type.Number({ multipleOf: 0.01 }) }), + args: [{ n: 0.07 }, { n: 1.1 }], + }, + { + name: "uniqueItems-keyorder", + schema: Type.Object({ a: Type.Array(Type.Unknown(), { uniqueItems: true }) }), + args: [ + { + a: [ + { x: 1, y: 2 }, + { y: 2, x: 1 }, + ], + }, + { a: [1, 1.0] }, + ], + }, + { name: "const-object", schema: Type.Object({ c: Type.Literal("a") }), args: [{ c: "a" }] }, + { + name: "formats", + schema: Type.Object({ + e: Type.Optional(Type.String({ format: "email" })), + u: Type.Optional(Type.String({ format: "uri" })), + d: Type.Optional(Type.String({ format: "date-time" })), + x: Type.Optional(Type.String({ format: "made-up" })), + h: Type.Optional(Type.String({ format: "hostname" })), + r: Type.Optional(Type.String({ format: "regex" })), + i: Type.Optional(Type.String({ format: "uuid" })), + ip: Type.Optional(Type.String({ format: "ipv4" })), + }), + args: [ + { e: "a@b.co" }, + { e: "not-an-email" }, + { u: "relative/path" }, + { u: "https://x.y/z" }, + { d: "2024-01-01T00:00:00Z" }, + { d: "2024-01-01" }, + { x: "anything" }, + { h: "under_score.example" }, + { r: "[" }, + { i: "00000000-0000-0000-0000-000000000000" }, + { i: "not" }, + { ip: "999.1.1.1" }, + ], + }, + { name: "integer-big", schema: Type.Object({ n: Type.Integer() }), args: [{ n: 1e21 }, { n: 2 ** 53 + 2 }] }, + { + name: "additional-false", + schema: Type.Object({ a: Type.String() }, { additionalProperties: false }), + args: [{ a: "x", b: 1 }], + }, + { + name: "prototype-key", + schema: Type.Object({ constructor: Type.Optional(Type.String()), toString: Type.Optional(Type.Number()) }), + args: [{}, { constructor: 1 }], + }, + { + name: "proto-key-value", + schema: Type.Object({}, { additionalProperties: Type.Array(Type.String()) }), + args: [JSON.parse('{"__proto__": "x", "a": "y"}')], + }, + // Plain JSON (MCP-shaped) + { + name: "mcp-draft7-ref-definitions", + schema: { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { t: { $ref: "#/definitions/T" } }, + required: ["t"], + definitions: { T: { type: "object", properties: { path: { type: "string" } }, required: ["path"] } }, + }, + args: [{ t: { path: "a" } }, { t: { path: 42 } }], + }, + { + name: "mcp-ref-unresolved", + schema: { type: "object", properties: { t: { $ref: "#/definitions/Missing" } } }, + args: [{ t: 1 }, {}], + }, + { + name: "mcp-ref-external", + schema: { type: "object", properties: { t: { $ref: "https://example.com/schemas/t.json" } } }, + args: [{ t: 1 }], + }, + { + name: "mcp-nullable-openapi", + schema: { type: "object", properties: { s: { type: "string", nullable: true } }, required: ["s"] }, + args: [{ s: null }, { s: "x" }], + }, + { + name: "mcp-draft7-tuple", + schema: { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { p: { type: "array", items: [{ type: "number" }, { type: "string" }], additionalItems: false } }, + }, + args: [{ p: [1, "a"] }, { p: [1, "a", 3] }, { p: ["1", 2] }], + }, + { + name: "mcp-draft7-ref-sibling", + schema: { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { s: { $ref: "#/definitions/S", maxLength: 1 } }, + definitions: { S: { type: "string" } }, + }, + args: [{ s: "abc" }], + }, + { + name: "mcp-draft4-exclusive-bool", + schema: { + $schema: "http://json-schema.org/draft-04/schema#", + type: "object", + properties: { n: { type: "number", minimum: 0, exclusiveMinimum: true } }, + }, + args: [{ n: 0 }, { n: 1 }], + }, + { + name: "mcp-id-draft4", + schema: { + $schema: "http://json-schema.org/draft-04/schema#", + id: "http://x/y", + type: "object", + properties: { a: { type: "string" } }, + }, + args: [{ a: "x" }], + }, + { + name: "mcp-bad-pattern", + schema: { type: "object", properties: { s: { type: "string", pattern: "(? `v${i}`) } }, + }, + args: [{ e: "v4999" }, { e: "v5000" }], + }, +]; + +const o = (properties: Record, extra: Record = {}) => ({ + type: "object", + properties, + ...extra, +}); +const keywordCases: [string, object, unknown[]][] = [ + [ + "oneOf-overlap", + o({ v: { oneOf: [{ type: "string" }, { type: "string", maxLength: 5 }] } }), + [{ v: "abc" }, { v: "abcdefgh" }], + ], + ["oneOf-number-integer", o({ v: { oneOf: [{ type: "number" }, { type: "integer" }] } }), [{ v: 2 }, { v: 2.5 }]], + [ + "oneOf-objects-open", + o({ + v: { + oneOf: [ + { type: "object", properties: { a: { type: "string" } } }, + { type: "object", properties: { b: { type: "string" } } }, + ], + }, + }), + [{ v: { a: "x" } }, { v: {} }], + ], + [ + "if-then-else", + o( + { kind: { type: "string" }, n: { type: "number" } }, + { if: { properties: { kind: { const: "a" } } }, then: { required: ["n"] } }, + ), + [{ kind: "a" }, { kind: "b" }], + ], + [ + "dependentRequired", + o({ a: { type: "string" }, b: { type: "string" } }, { dependentRequired: { a: ["b"] } }), + [{ a: "x" }], + ], + [ + "dependencies-d7", + o({ a: { type: "string" }, b: { type: "string" } }, { dependencies: { a: ["b"] } }), + [{ a: "x" }], + ], + ["propertyNames", o({}, { propertyNames: { pattern: "^[a-z]+$" } }), [{ abc: 1 }, { ABC: 1 }]], + ["contains", o({ l: { type: "array", contains: { type: "number" } } }), [{ l: ["a"] }, { l: ["a", 1] }]], + ["not", o({ s: { not: { type: "null" } } }), [{ s: "x" }, { s: null }]], + ["minProperties", o({ m: { type: "object", minProperties: 1 } }), [{ m: {} }]], + [ + "unevaluatedProperties", + o({ a: { type: "string" } }, { unevaluatedProperties: false }), + [{ a: "x" }, { a: "x", b: 1 }], + ], + [ + "allOf-additional-false", + { allOf: [o({ a: { type: "string" } }, { additionalProperties: false }), o({ b: { type: "string" } })] }, + [{ a: "x" }, { a: "x", b: "y" }], + ], + ["const-object", o({ c: { const: { x: 1, y: 2 } } }), [{ c: { y: 2, x: 1 } }]], + ["enum-object", o({ e: { enum: [{ x: 1, y: 2 }] } }), [{ e: { y: 2, x: 1 } }]], + ["type-array-multi", o({ v: { type: ["string", "number"] } }), [{ v: true }, { v: null }]], + ["nullable-type-array", o({ v: { type: ["string", "null"] } }, { required: ["v"] }), [{ v: null }, { v: 5 }]], + ["exclusiveMinimum-number", o({ n: { type: "number", exclusiveMinimum: 0 } }), [{ n: 0 }, { n: "0" }]], + ["maxItems", o({ l: { type: "array", maxItems: 1 } }), [{ l: [1, 2] }, { l: "x" }]], + [ + "discriminator-openapi", + o({ + p: { + oneOf: [ + o({ kind: { type: "string", enum: ["a"] }, x: { type: "number" } }, { required: ["kind"] }), + o({ kind: { type: "string", enum: ["b"] } }, { required: ["kind"] }), + ], + discriminator: { propertyName: "kind" }, + }, + }), + [{ p: { kind: "a", x: "1" } }, { p: { kind: "b" } }], + ], + [ + "ref-root-defs", + { $ref: "#/$defs/Root", $defs: { Root: o({ a: { type: "string" } }, { required: ["a"] }) } }, + [{ a: "x" }, { a: 1 }, {}], + ], + [ + "ref-recursive", + { + $defs: { N: o({ c: { type: "array", items: { $ref: "#/$defs/N" } }, v: { type: "integer" } }) }, + ...o({ root: { $ref: "#/$defs/N" } }), + }, + [{ root: { v: "2", c: [{ v: "3", c: [] }] } }], + ], + [ + "ref-percent", + { ...o({ a: { $ref: "#/$defs/a%20b" } }), $defs: { "a b": { type: "string" } } }, + [{ a: "x" }, { a: 1 }], + ], + ["ref-tilde", { ...o({ a: { $ref: "#/$defs/a~1b" } }), $defs: { "a/b": { type: "string" } } }, [{ a: "x" }]], + [ + "defs-with-$id", + { ...o({ a: { $ref: "item.json" } }), $defs: { item: { $id: "item.json", type: "string" } } }, + [{ a: "x" }, { a: 2 }], + ], + [ + "anyOf-null-default", + o({ v: { anyOf: [{ type: "string" }, { type: "null" }], default: null } }), + [{ v: null }, {}], + ], + [ + "format-date-time-lower", + o({ d: { type: "string", format: "date-time" } }), + [{ d: "2024-01-01t00:00:00z" }, { d: "2024-01-01 00:00:00Z" }], + ], + ["format-url", o({ u: { type: "string", format: "url" } }), [{ u: "not a url" }]], + ["integer-as-float-1e0", o({ n: { type: "integer" } }), [{ n: 1e3 }]], + ["required-missing-props", { type: "object", required: ["a"] }, [{}, { a: null }]], + ["string-to-number-trim", o({ n: { type: "number" } }), [{ n: " 12 " }, { n: "0x10" }, { n: "1e3" }, { n: "" }]], +]; + +const tree = { + type: "object", + properties: { t: { $ref: "#/$defs/Tree" } }, + $defs: { Tree: { anyOf: [{ type: "string" }, { type: "array", items: { $ref: "#/$defs/Tree" } }] } }, +}; +const json = { + type: "object", + properties: { data: { $ref: "#/$defs/Json" } }, + required: ["data"], + $defs: { + Json: { + anyOf: [ + { type: "string" }, + { type: "number" }, + { type: "boolean" }, + { type: "array", items: { $ref: "#/$defs/Json" } }, + { type: "object", additionalProperties: { $ref: "#/$defs/Json" } }, + ], + }, + }, +}; +const filter = { + type: "object", + properties: { where: { $ref: "#/$defs/F" } }, + $defs: { + F: { + oneOf: [ + { + type: "object", + properties: { field: { type: "string" }, eq: { type: "string" } }, + required: ["field", "eq"], + additionalProperties: false, + }, + { + type: "object", + properties: { and: { type: "array", items: { $ref: "#/$defs/F" } } }, + required: ["and"], + additionalProperties: false, + }, + ], + }, + }, +}; +const recursionCases: [string, object, unknown][] = [ + ["tree valid", tree, { t: ["a", ["b"]] }], + ["tree number", tree, { t: 5 }], + ["tree [5]", tree, { t: [5] }], + ["json valid", json, { data: { a: [1, "x", { b: true }] } }], + ["json null", json, { data: null }], + ["json nested null", json, { data: { a: [1, null] } }], + ["filter valid", filter, { where: { and: [{ field: "a", eq: "b" }] } }], + ["filter bad leaf", filter, { where: { and: [{ field: "a", eq: 1, x: 2 }] } }], + ["filter deep bad", filter, { where: { and: [{ and: [{ and: [{ and: [{ and: [{ field: 1 }] }] }] }] }] } }], +]; + +const expected: Record = { + 'cyclic {"tree":{"name":"a","children":[{"name":"b","children":[]}]}}': + 'ok {"tree":{"name":"a","children":[{"name":"b","children":[]}]}}', + 'cyclic {"tree":{"name":1,"children":[]}}': 'ok {"tree":{"name":"1","children":[]}}', + 'ref-with-$id {"a":{"x":"1"}}': "REJECT - a: unresolved reference A", + "function {}": "ok {}", + 'function {"f":"x"}': "REJECT - f: expected function, got string", + 'codec {"c":"12"}': 'ok {"c":"12"}', + 'codec {"c":12}': 'ok {"c":"12"}', + "undefined-optional {}": "ok {}", + 'undefined-optional {"u":null}': "ok {}", + 'bigint {"n":1}': "REJECT - n: expected bigint, got integer", + 'bigint {"n":"12"}': "REJECT - n: expected bigint, got string", + 'refine {"n":5}': 'ok {"n":5}', + 'refine {"n":-5}': 'ok {"n":-5}', + 'template-literal {"t":"id-12"}': 'ok {"t":"id-12"}', + 'template-literal {"t":"id-x"}': "REJECT - t: expected to match ^id--?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?$", + 'template-literal {"t":3}': "REJECT - t: expected to match ^id--?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?$", + 'record-string {"r":{"a":"1"}}': 'ok {"r":{"a":1}}', + 'record-string {"r":{"a":1,"b":"x"}}': "REJECT - r.b: expected number, got string", + 'record-literal-keys {"r":{"a":1,"b":2}}': 'ok {"r":{"a":1,"b":2}}', + 'record-literal-keys {"r":{"a":1}}': "REJECT - r.b: missing", + 'record-literal-keys {"r":{"a":1,"b":2,"c":3}}': 'ok {"r":{"a":1,"b":2,"c":3}}', + 'intersect {"a":"1","b":"true"}': 'ok {"a":1,"b":true}', + 'intersect {"a":1,"b":true,"c":1}': 'ok {"a":1,"b":true,"c":1}', + 'union-objects {"v":{"k":"x","n":"2"}}': 'ok {"v":{"k":"x","n":2}}', + 'union-objects {"v":{"k":"y","s":2}}': 'ok {"v":{"k":"y","s":"2"}}', + 'minLength-astral {"s":"😀"}': "REJECT - s: expected at least 2 characters", + 'maxLength-astral {"s":"😀"}': 'ok {"s":"😀"}', + 'pattern-astral {"s":"😀"}': 'ok {"s":"😀"}', + 'pattern-class-dash {"s":"a-b"}': 'ok {"s":"a-b"}', + 'pattern-unicode-prop {"s":"äb"}': 'ok {"s":"äb"}', + 'pattern-unicode-prop {"s":"p{L}"}': "REJECT - s: expected to match ^\\p{L}+$", + 'multipleOf-float {"n":0.07}': 'ok {"n":0.07}', + 'multipleOf-float {"n":1.1}': 'ok {"n":1.1}', + 'uniqueItems-keyorder {"a":[{"x":1,"y":2},{"y":2,"x":1}]}': "REJECT - a: expected unique items", + 'uniqueItems-keyorder {"a":[1,1]}': "REJECT - a: expected unique items", + 'const-object {"c":"a"}': 'ok {"c":"a"}', + 'formats {"e":"a@b.co"}': 'ok {"e":"a@b.co"}', + 'formats {"e":"not-an-email"}': "REJECT - e: expected format email", + 'formats {"u":"relative/path"}': "REJECT - u: expected format uri", + 'formats {"u":"https://x.y/z"}': 'ok {"u":"https://x.y/z"}', + 'formats {"d":"2024-01-01T00:00:00Z"}': 'ok {"d":"2024-01-01T00:00:00Z"}', + 'formats {"d":"2024-01-01"}': "REJECT - d: expected format date-time", + 'formats {"x":"anything"}': 'ok {"x":"anything"}', + 'formats {"h":"under_score.example"}': "REJECT - h: expected format hostname", + 'formats {"r":"["}': "REJECT - r: expected format regex", + 'formats {"i":"00000000-0000-0000-0000-000000000000"}': 'ok {"i":"00000000-0000-0000-0000-000000000000"}', + 'formats {"i":"not"}': "REJECT - i: expected format uuid", + 'formats {"ip":"999.1.1.1"}': "REJECT - ip: expected format ipv4", + 'integer-big {"n":1e+21}': 'ok {"n":1e+21}', + 'integer-big {"n":9007199254740994}': 'ok {"n":9007199254740994}', + 'additional-false {"a":"x","b":1}': "REJECT - b: unknown field", + "prototype-key {}": "ok {}", + 'prototype-key {"constructor":1}': 'ok {"constructor":"1"}', + 'proto-key-value {"__proto__":"x","a":"y"}': 'ok {"__proto__":["x"],"a":["y"]}', + 'mcp-draft7-ref-definitions {"t":{"path":"a"}}': 'ok {"t":{"path":"a"}}', + 'mcp-draft7-ref-definitions {"t":{"path":42}}': 'ok {"t":{"path":"42"}}', + 'mcp-ref-unresolved {"t":1}': "REJECT - t: unresolved reference #/definitions/Missing", + "mcp-ref-unresolved {}": "ok {}", + 'mcp-ref-external {"t":1}': "REJECT - t: unresolved reference https://example.com/schemas/t.json", + 'mcp-nullable-openapi {"s":null}': 'ok {"s":"null"}', + 'mcp-nullable-openapi {"s":"x"}': 'ok {"s":"x"}', + 'mcp-draft7-tuple {"p":[1,"a"]}': 'ok {"p":[1,"a"]}', + 'mcp-draft7-tuple {"p":[1,"a",3]}': "REJECT - p.2: no value is allowed here", + 'mcp-draft7-tuple {"p":["1",2]}': 'ok {"p":[1,"2"]}', + 'mcp-draft7-ref-sibling {"s":"abc"}': "REJECT - s: expected at most 1 characters", + 'mcp-draft4-exclusive-bool {"n":0}': 'ok {"n":0}', + 'mcp-draft4-exclusive-bool {"n":1}': 'ok {"n":1}', + 'mcp-id-draft4 {"a":"x"}': 'ok {"a":"x"}', + 'mcp-bad-pattern {"s":"y"}': 'ok {"s":"y"}', + 'mcp-invalid-regex {"s":"y"}': "THROW Invalid regular expression: /[/: Unterminated character class", + 'mcp-type-integer-float-0 {"n":1}': 'ok {"n":1}', + 'mcp-type-integer-float-0 {"n":1.5}': 'ok {"n":1}', + 'mcp-format-unknown {"s":"xx"}': 'ok {"s":"xx"}', + 'mcp-required-null-string {"q":null}': 'ok {"q":"null"}', + 'mcp-array-json-string {"p":"[\\"a\\",\\"b\\"]"}': 'ok {"p":["[\\"a\\",\\"b\\"]"]}', + 'mcp-integer-float {"n":3.7}': 'ok {"n":3}', + 'mcp-integer-float {"n":"3.7"}': 'ok {"n":3}', + 'mcp-empty-schema {"a":1}': 'ok {"a":1}', + 'mcp-type-object-no-props {"a":1}': 'ok {"a":1}', + 'mcp-boolean-true-prop {"a":1}': 'ok {"a":1}', + 'mcp-boolean-true-prop {"b":1}': "REJECT - b: no value is allowed here", + 'mcp-huge-enum {"e":"v4999"}': 'ok {"e":"v4999"}', + 'mcp-huge-enum {"e":"v5000"}': + 'REJECT - e: expected one of ["v0","v1","v2","v3","v4","v5","v6","v7","v8","v9","v10","v11","v12","v13","v14","v15","v16","v17","v18","v19", ... and 4980 more]', + 'oneOf-overlap {"v":"abc"}': "REJECT - v: matches more than one alternative", + 'oneOf-overlap {"v":"abcdefgh"}': 'ok {"v":"abcdefgh"}', + 'oneOf-number-integer {"v":2}': "REJECT - v: matches more than one alternative", + 'oneOf-number-integer {"v":2.5}': 'ok {"v":2.5}', + 'oneOf-objects-open {"v":{"a":"x"}}': "REJECT - v: matches more than one alternative", + 'oneOf-objects-open {"v":{}}': "REJECT - v: matches more than one alternative", + 'if-then-else {"kind":"a"}': "REJECT - n: missing", + 'if-then-else {"kind":"b"}': 'ok {"kind":"b"}', + 'dependentRequired {"a":"x"}': "REJECT - b: required when a is present", + 'dependencies-d7 {"a":"x"}': "REJECT - b: required when a is present", + 'propertyNames {"abc":1}': 'ok {"abc":1}', + 'propertyNames {"ABC":1}': "REJECT - ABC: invalid property name", + 'contains {"l":["a"]}': "REJECT - l: expected at least 1 matching item", + 'contains {"l":["a",1]}': 'ok {"l":["a",1]}', + 'not {"s":"x"}': 'ok {"s":"x"}', + 'not {"s":null}': "ok {}", + 'minProperties {"m":{}}': "REJECT - m: expected at least 1 properties", + 'unevaluatedProperties {"a":"x"}': 'ok {"a":"x"}', + 'unevaluatedProperties {"a":"x","b":1}': "REJECT - b: unknown field", + 'allOf-additional-false {"a":"x"}': 'ok {"a":"x"}', + 'allOf-additional-false {"a":"x","b":"y"}': "REJECT - b: unknown field", + 'const-object {"c":{"y":2,"x":1}}': 'ok {"c":{"y":2,"x":1}}', + 'enum-object {"e":{"y":2,"x":1}}': 'ok {"e":{"y":2,"x":1}}', + 'type-array-multi {"v":true}': 'ok {"v":"true"}', + 'type-array-multi {"v":null}': "ok {}", + 'nullable-type-array {"v":null}': 'ok {"v":null}', + 'nullable-type-array {"v":5}': 'ok {"v":"5"}', + 'exclusiveMinimum-number {"n":0}': "REJECT - n: expected > 0", + 'exclusiveMinimum-number {"n":"0"}': "REJECT - n: expected > 0", + 'maxItems {"l":[1,2]}': "REJECT - l: expected at most 1 items", + 'maxItems {"l":"x"}': 'ok {"l":["x"]}', + 'discriminator-openapi {"p":{"kind":"a","x":"1"}}': 'ok {"p":{"kind":"a","x":1}}', + 'discriminator-openapi {"p":{"kind":"b"}}': 'ok {"p":{"kind":"b"}}', + 'ref-root-defs {"a":"x"}': 'ok {"a":"x"}', + 'ref-root-defs {"a":1}': 'ok {"a":"1"}', + "ref-root-defs {}": "REJECT - a: missing", + 'ref-recursive {"root":{"v":"2","c":[{"v":"3","c":[]}]}}': 'ok {"root":{"v":2,"c":[{"v":3,"c":[]}]}}', + 'ref-percent {"a":"x"}': 'ok {"a":"x"}', + 'ref-percent {"a":1}': 'ok {"a":"1"}', + 'ref-tilde {"a":"x"}': 'ok {"a":"x"}', + 'defs-with-$id {"a":"x"}': 'ok {"a":"x"}', + 'defs-with-$id {"a":2}': 'ok {"a":"2"}', + 'anyOf-null-default {"v":null}': 'ok {"v":null}', + "anyOf-null-default {}": "ok {}", + 'format-date-time-lower {"d":"2024-01-01t00:00:00z"}': 'ok {"d":"2024-01-01t00:00:00z"}', + 'format-date-time-lower {"d":"2024-01-01 00:00:00Z"}': "REJECT - d: expected format date-time", + 'format-url {"u":"not a url"}': "REJECT - u: expected format url", + 'integer-as-float-1e0 {"n":1000}': 'ok {"n":1000}', + "required-missing-props {}": "REJECT - a: missing", + 'required-missing-props {"a":null}': 'ok {"a":null}', + 'string-to-number-trim {"n":" 12 "}': 'ok {"n":12}', + 'string-to-number-trim {"n":"0x10"}': 'ok {"n":16}', + 'string-to-number-trim {"n":"1e3"}': 'ok {"n":1000}', + 'string-to-number-trim {"n":""}': 'ok {"n":0}', + "tree valid": 'ok {"t":["a",["b"]]}', + "tree number": 'ok {"t":"5"}', + "tree [5]": 'ok {"t":["5"]}', + "json valid": 'ok {"data":{"a":[1,"x",{"b":true}]}}', + "json null": 'ok {"data":"null"}', + "json nested null": 'ok {"data":{"a":[1,"null"]}}', + "filter valid": 'ok {"where":{"and":[{"field":"a","eq":"b"}]}}', + "filter bad leaf": "REJECT - where.and.0.eq: expected string, got integer", + "filter deep bad": "REJECT - where.and.0.and.0.and.0.and.0.and.0.eq: missing", +}; + +/** Every case with the key of its expected outcome: the name and the arguments, or the name alone for recursion. */ +const all: Array<[string, string, object, unknown]> = [ + ...cases.flatMap(({ name, schema, args }) => + args.map((arg): [string, string, object, unknown] => [`${name} ${JSON.stringify(arg)}`, name, schema, arg]), + ), + ...keywordCases.flatMap(([name, schema, args]) => + args.map((arg): [string, string, object, unknown] => [`${name} ${JSON.stringify(arg)}`, name, schema, arg]), + ), + ...recursionCases.map(([name, schema, arg]): [string, string, object, unknown] => [name, name, schema, arg]), +]; + +describe("reviewer probes", () => { + for (const [key, name, schema, arg] of all) { + it(key, () => { + expect(outcome(name, schema, arg)).toBe(expected[key]); + }); + } +}); + +describe("reviewer probes (Sol)", () => { + const tree = { + type: "object", + properties: { t: { $ref: "#/$defs/Tree" } }, + $defs: { Tree: { anyOf: [{ type: "string" }, { type: "array", items: { $ref: "#/$defs/Tree" } }] } }, + }; + const solCases: Array<[string, object, unknown, string]> = [ + // TypeBox-only: Type.BigInt and Type.Undefined have no JSON form; TypeBox converted to them. + [ + "BigInt from a JSON string", + Type.Object({ n: Type.BigInt() }), + { n: "42" }, + "REJECT - n: expected bigint, got string", + ], + [ + "Undefined from JSON null", + Type.Object({ n: Type.Undefined() }), + { n: null }, + "REJECT - n: expected undefined, got null", + ], + // TypeBox-only: its additionalProperties also converted declared properties; pi-schema converts only undeclared ones. + [ + "additionalProperties on declared properties", + Type.Object({ amount: Type.Number(), label: Type.String() }, { additionalProperties: Type.Integer() }), + { amount: 3.5, label: "7" }, + 'ok {"amount":3.5,"label":"7"}', + ], + // TypeBox-only: property names are names, not regular expressions. + [ + "property names are not patterns", + Type.Object({ "a.b": Type.Integer(), axb: Type.String() }), + { "a.b": "4", axb: "5.8" }, + 'ok {"a.b":4,"axb":"5.8"}', + ], + // TypeBox-only: Refine runs only in TypeBox's own validator. + [ + "Refine is not run", + Type.Object({ mode: Type.Refine(Type.String(), (value) => value === "safe") }), + { mode: "unsafe" }, + 'ok {"mode":"unsafe"}', + ], + [ + "scalar-or-array that refers to itself", + { + type: "object", + properties: { + value: { anyOf: [{ type: "number" }, { type: "array", items: { $ref: "#/properties/value" } }] }, + }, + required: ["value"], + }, + { value: "42" }, + 'ok {"value":42}', + ], + [ + "StringEnum with an empty member from null", + Type.Object({ mode: Type.Unsafe({ type: "string", enum: ["", "x"] }) }), + { mode: null }, + 'ok {"mode":""}', + ], + [ + "Cyclic with named local definitions", + Type.Object({ + node: Type.Cyclic( + { Node: Type.Object({ n: Type.Number(), next: Type.Optional(Type.Ref("Node")) }) }, + "Node", + ), + }), + { node: { n: 1 } }, + 'ok {"node":{"n":1}}', + ], + [ + "unpaired high surrogate, minLength 2", + Type.Object({ s: Type.String({ minLength: 2 }) }), + { s: "\ud800A" }, + 'ok {"s":"\\ud800A"}', + ], + [ + "unpaired high surrogate, maxLength 1", + Type.Object({ s: Type.String({ maxLength: 1 }) }), + { s: "\ud800A" }, + "REJECT - s: expected at most 1 characters", + ], + ["integer from a string", Type.Object({ n: Type.Integer() }), { n: "42" }, 'ok {"n":42}'], + ["recursive union with a value no branch converts", tree, { t: {} }, "REJECT - t: expected string, got object"], + ]; + for (const [name, schema, arg, result] of solCases) { + it(name, () => { + expect(outcome(name, schema, arg)).toBe(result); + }); + } + + it("accepts the last member of a 10,000-value enum", () => { + const values = Array.from({ length: 10_000 }, (_, i) => `member${i}`); + const schema = { type: "object", properties: { choice: { type: "string", enum: values } }, required: ["choice"] }; + expect(outcome("enum", schema, { choice: "member9999" })).toBe('ok {"choice":"member9999"}'); + expect(outcome("enum", schema, { choice: "x" })).toBe( + `REJECT - choice: expected one of ${JSON.stringify(values.slice(0, 20)).slice(0, -1)}, ... and 9980 more]`, + ); + }); + + it("keeps the validator of a schema object, as before, also when the schema is changed later", () => { + const parameters = Type.Object({ n: Type.Number({ minimum: 0 }) }); + expect(outcome("late", parameters, { n: "4" })).toBe('ok {"n":4}'); + Object.assign(parameters.properties.n, { minimum: 10 }); + expect(outcome("late", parameters, { n: "4" })).toBe('ok {"n":4}'); + }); +}); + +describe("reviewer probes (round 2)", () => { + const item = { type: "object", properties: { n: { type: "integer", minimum: 1 } } }; + const nested: Case[] = [ + { name: "declared", schema: { type: "object", properties: { entry: item } }, args: [{ entry: { n: null } }] }, + { name: "additional", schema: { type: "object", additionalProperties: item }, args: [{ entry: { n: null } }] }, + { + name: "pattern", + schema: { type: "object", patternProperties: { "^entry$": item }, additionalProperties: false }, + args: [{ entry: { n: null } }], + }, + ]; + for (const { name, schema, args } of nested) { + it(`drops an optional null inside ${name} property values`, () => { + expect(outcome(name, schema, args[0])).toBe('ok {"entry":{}}'); + }); + } + + for (const keyword of ["anyOf", "oneOf"]) { + it(`converts a ${keyword} branch together with the node's sibling properties`, () => { + const schema = { + type: "object", + [keyword]: [ + { type: "object", properties: { a: { type: "array", items: { type: "integer" } } }, required: ["a"] }, + { type: "object", properties: { b: { type: "boolean" } }, required: ["b"] }, + ], + properties: { c: { type: "integer" } }, + required: ["c"], + }; + expect(outcome(keyword, schema, { a: "3", c: "2" })).toBe('ok {"a":[3],"c":2}'); + }); + } + + it("rejects a deep value every branch of a recursive object union rejects, without exponential work", () => { + const S = { + anyOf: [ + { type: "object", additionalProperties: { $ref: "#/$defs/S" }, required: ["a"] }, + { type: "object", additionalProperties: { $ref: "#/$defs/S" }, required: ["b"] }, + ], + }; + const schema = { type: "object", properties: { s: { $ref: "#/$defs/S" } }, $defs: { S } }; + let value: unknown = { x: 1 }; + for (let i = 0; i < 40; i++) value = { c: value }; + expect(outcome("deep", schema, { s: value })).toBe("REJECT - s.a: missing"); + }); +}); diff --git a/packages/ai/test/validation.test.ts b/packages/ai/test/validation.test.ts index 7cebbe541..474d0b6cf 100644 --- a/packages/ai/test/validation.test.ts +++ b/packages/ai/test/validation.test.ts @@ -1,5 +1,4 @@ import { Type } from "typebox"; -import { Compile } from "typebox/compile"; import { describe, expect, it } from "vitest"; import type { JsonValue, Tool, ToolCall } from "../src/types.ts"; import { validateToolArguments } from "../src/utils/validation.ts"; @@ -61,7 +60,7 @@ describe("validateToolArguments", () => { } }); - it("coerces serialized plain JSON schemas with AJV-compatible primitive rules", () => { + it("coerces serialized plain JSON schemas like TypeBox schemas", () => { const passingCases: Array<{ schema: Tool["parameters"]; input: unknown; @@ -75,7 +74,8 @@ describe("validateToolArguments", () => { { schema: { type: "boolean" } as Tool["parameters"], input: "false", expected: false }, { schema: { type: "boolean" } as Tool["parameters"], input: 1, expected: true }, { schema: { type: "boolean" } as Tool["parameters"], input: 0, expected: false }, - { schema: { type: "string" } as Tool["parameters"], input: null, expected: "" }, + // TypeBox's conversion rules come first: null is "null" for a required string. + { schema: { type: "string" } as Tool["parameters"], input: null, expected: "null" }, { schema: { type: "string" } as Tool["parameters"], input: true, expected: "true" }, { schema: { type: "null" } as Tool["parameters"], input: "", expected: null }, { schema: { type: "null" } as Tool["parameters"], input: 0, expected: null }, @@ -90,6 +90,11 @@ describe("validateToolArguments", () => { input: "1", expected: 1, }, + { schema: { type: "boolean" } as Tool["parameters"], input: "1", expected: true }, + { schema: { type: "boolean" } as Tool["parameters"], input: "0", expected: false }, + { schema: { type: "null" } as Tool["parameters"], input: "null", expected: null }, + { schema: { type: "integer" } as Tool["parameters"], input: "42.1", expected: 42 }, + { schema: { type: "array", items: { type: "number" } } as Tool["parameters"], input: "7", expected: [7] }, ]; for (const testCase of passingCases) { @@ -184,10 +189,6 @@ describe("validateToolArguments", () => { { type: ["array", "null"], items: { type: "string" } } as Tool["parameters"], null, ); - // The CSP test above selects TypeBox's process-wide interpreted fallback, so exercise the generated validator explicitly. - const generatedCheck = new Function(Compile(tool.parameters).Code())() as (value: unknown) => boolean; - - expect(generatedCheck(toolCall.arguments)).toBe(true); expect(validateToolArguments(tool, toolCall)).toEqual({ value: null }); }); @@ -196,10 +197,10 @@ describe("validateToolArguments", () => { schema: Tool["parameters"]; input: unknown; }> = [ - { schema: { type: "boolean" } as Tool["parameters"], input: "1" }, - { schema: { type: "boolean" } as Tool["parameters"], input: "0" }, - { schema: { type: "null" } as Tool["parameters"], input: "null" }, - { schema: { type: "integer" } as Tool["parameters"], input: "42.1" }, + { schema: { type: "boolean" } as Tool["parameters"], input: "yes" }, + { schema: { type: "number" } as Tool["parameters"], input: "abc" }, + { schema: { type: "null" } as Tool["parameters"], input: "nil" }, + { schema: { type: "integer" } as Tool["parameters"], input: "4x" }, ]; for (const testCase of failingCases) { @@ -207,6 +208,25 @@ describe("validateToolArguments", () => { expect(() => validateToolArguments(tool, toolCall)).toThrow("Validation failed"); } }); + it("lists pi-schema's errors, then the arguments it received", () => { + const { tool, toolCall } = createToolCallWithPlainSchema({ type: "number" } as Tool["parameters"], "many"); + expect(() => validateToolArguments(tool, toolCall)).toThrow( + 'Validation failed for tool "echo":\n - value: expected number, got string\n\nReceived arguments:\n{\n "value": "many"\n}', + ); + }); + + it("reports a missing required property at the property", () => { + const { tool, toolCall } = createToolCallWithPlainSchema({ type: "string" } as Tool["parameters"], "x"); + expect(() => validateToolArguments(tool, { ...toolCall, arguments: {} })).toThrow(" - value: missing"); + }); + + it("returns a copy of the arguments, also when nothing is coerced", () => { + const { tool, toolCall } = createToolCallWithPlainSchema({ type: "string" } as Tool["parameters"], "x"); + const args = validateToolArguments(tool, toolCall); + expect(args).toEqual({ value: "x" }); + expect(args).not.toBe(toolCall.arguments); + }); + // #10897 it("reports malformed arguments JSON instead of validating empty arguments", () => { const tool: Tool = { diff --git a/packages/ai/vitest.config.ts b/packages/ai/vitest.config.ts index 84c74692f..397107fa1 100644 --- a/packages/ai/vitest.config.ts +++ b/packages/ai/vitest.config.ts @@ -13,5 +13,8 @@ export default defineConfig({ }, resolve: { alias: [{ find: /^@earendil-works\/pi-telemetry$/, replacement: telemetrySrcIndex }], + // Workspace packages without a build (pi-schema) resolve to their source. + conditions: ["source"], }, + ssr: { resolve: { conditions: ["source"] } }, }); \ No newline at end of file diff --git a/packages/codemode/CHANGELOG.md b/packages/codemode/CHANGELOG.md index decd59f9e..a4c2fd624 100644 --- a/packages/codemode/CHANGELOG.md +++ b/packages/codemode/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Changed + +- README: the codemode tool example builds its parameters with `@earendil-works/pi-schema` instead of TypeBox. + ## [1.4.0] - 2026-10-21 ## [1.3.0] - 2026-10-15 diff --git a/packages/codemode/README.md b/packages/codemode/README.md index e4fa87786..d97bd6bf5 100644 --- a/packages/codemode/README.md +++ b/packages/codemode/README.md @@ -275,7 +275,7 @@ import { type CodemodeTool, renderDeclarations, } from "@earendil-works/pi-codemode"; -import { Type } from "typebox"; +import { obj, str } from "@earendil-works/pi-schema"; const sandboxTools: CodemodeTool[] = agentTools.map((tool) => ({ name: tool.name, @@ -293,7 +293,7 @@ const codemodeTool: AgentTool = { name: "codemode", label: "Codemode", description: `Run JavaScript that calls tools as \`await tools.(args)\`. Output with text() or return.\n\n${renderDeclarations({ tools: sandboxTools })}`, - parameters: Type.Object({ code: Type.String() }), + parameters: obj({ code: str() }), execute: async (_toolCallId, { code }, signal) => { const sandbox = new CodemodeSandbox({ tools: sandboxTools }); try { diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 359250c80..ae5a6f324 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -12,6 +12,8 @@ ### Changed +- Tool argument validation runs on `@earendil-works/pi-schema`. TypeBox features outside JSON Schema (`Type.Refine()`, `Type.BigInt()`, `Type.Undefined()`, codecs, `Format.Set()` formats) no longer apply to tool arguments; see [Tool parameter schemas and TypeBox](docs/extensions.md#tool-parameter-schemas-and-typebox). Validation failures the model sees list pi-schema's messages (`- path: missing`, `- limit: expected number, got string`) instead of TypeBox's (`must have required properties path`, `must be number`); which arguments are accepted and how they are coerced is unchanged for extension and built-in tools. +- Extensions keep importing `Type`, `Static` and `TSchema` from `@earendil-works/pi-ai`: Pi maps that import to pi-ai's compat entry plus the TypeBox it bundles, now that pi-ai itself no longer re-exports TypeBox, and its `StringEnum` there stays a TypeBox `TUnsafe`. The built-in tool factories (`createReadTool()` and the others) return `TypeBoxAgentTool`, which keeps `params` typed for TypeBox. - Tightened system prompt - `pi update` for installs from the pi.dev installer no longer passes `--min-release-age=0` to `npm ci`. The release lockfile pins every package, so the flag had no effect, and security scanners that block it stopped the update ([#11262](https://github.com/earendil-works/pi/issues/11262)) - `/model` and `/scoped-models` now show a model's name next to its ID when the name adds words the ID lacks, such as a renamed `models.json` override ([#5916](https://github.com/earendil-works/pi/issues/5916), [#11347](https://github.com/earendil-works/pi/pull/11347)) diff --git a/packages/coding-agent/docs/extensions.md b/packages/coding-agent/docs/extensions.md index 6fbec2b65..5879058c7 100644 --- a/packages/coding-agent/docs/extensions.md +++ b/packages/coding-agent/docs/extensions.md @@ -159,7 +159,7 @@ Settings also accept built-in extensions as `builtin:` entries; see [Setti | `@earendil-works/pi-coding-agent` | Extension types (`ExtensionAPI`, `ExtensionContext`, events) and helpers | | `@earendil-works/pi-agent-core` | Agent message and tool types | | `typebox`, `typebox/compile`, `typebox/value` | Schema definitions for tool parameters | -| `@earendil-works/pi-ai` | AI utilities (`StringEnum` for string enums, transcript helpers, model types) | +| `@earendil-works/pi-ai` | AI utilities (`StringEnum` for string enums, transcript helpers, model types). Inside Pi it also exports TypeBox's `Type`, `Static` and `TSchema`, which the published package no longer re-exports; import them from `typebox` in new code | | `@earendil-works/pi-ai/compat`, `/models`, `/oauth`, `/providers/all` | Provider construction, built-in APIs, and OAuth helpers | | `@earendil-works/pi-tui` | TUI components for custom rendering | @@ -169,6 +169,18 @@ npm dependencies work too. Add a `package.json` next to your extension (or in a For distributed Pi packages installed with `pi install` (npm or git), runtime deps must be in `dependencies`; `devDependencies` are not installed. Do not list the host-provided packages above in `dependencies`. See [Declare dependencies](packages.md#declare-dependencies). +### Tool parameter schemas and TypeBox + +Pi validates tool arguments as JSON Schema with `@earendil-works/pi-schema`, whatever built the schema; TypeBox schemas are JSON Schema and work unchanged. Before validation, loosely typed arguments are coerced: numeric strings to numbers, `"true"` to `true`, a single value to a one-item array, `null` in an optional property dropped. TypeBox features outside JSON Schema do not apply to tool arguments: + +- `Type.Refine()` refinements are not run. Check such constraints in `execute()`. +- `Type.BigInt()`, `Type.Undefined()` and `Type.Void()` reject every argument, since JSON has no such values. Codecs (`Type.Codec()`) do not decode: `execute()` receives the JSON value. +- Formats registered with TypeBox's `Format.Set()` are not checked. The standard formats are, plus `url` and `json-pointer-uri-fragment`. +- Property names are matched as names, not as regular expressions, and `additionalProperties` applies only to properties the object does not declare. +- A `$ref` that names nothing rejects only arguments at its place. + +Validation errors the model sees use pi-schema's messages, such as `- path: missing` or `- limit: expected number, got string`. + Node.js built-ins (`node:fs`, `node:path`, etc.) are also available. ## Writing an Extension diff --git a/packages/coding-agent/examples/tsconfig.json b/packages/coding-agent/examples/tsconfig.json new file mode 100644 index 000000000..03e1a36e1 --- /dev/null +++ b/packages/coding-agent/examples/tsconfig.json @@ -0,0 +1,180 @@ +// Type-checks the example extensions against the modules extensions see at run time (core/extensions/virtual-modules.ts): +// "@earendil-works/pi-ai" is the extension view, which adds TypeBox's Type, Static and TSchema to the compat entry (the +// view imports "@earendil-works/pi-ai/compat", so that path stays the package's own). +// The other paths repeat the root tsconfig.json; keep them in sync. +// Also the test that imports the subagent example: it type-checks only against the extension view. +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "paths": { + "@earendil-works/chord": [ + "../../../packages/chord/src/index.ts" + ], + "@earendil-works/chord/context": [ + "../../../packages/chord/src/context/index.ts" + ], + "@earendil-works/chord/delta": [ + "../../../packages/chord/src/delta/index.ts" + ], + "@earendil-works/chord/next": [ + "../../../packages/chord/src/next/index.ts" + ], + "@earendil-works/chord/next/plugins": [ + "../../../packages/chord/src/next-node/plugins/index.ts" + ], + "@earendil-works/chord/next/plugins/runtime": [ + "../../../packages/chord/src/next-node/plugins/runtime.ts" + ], + "@test/contracts/*": [ + "../../../packages/chord/test/next-node/fixtures/plugins/contracts/*.ts" + ], + "@example/contracts/*": [ + "../../../packages/chord/test/next/examples/35-plugins/plugins/contracts/*.ts" + ], + "@example/weather/*": [ + "../../../packages/chord/test/next/examples/36-plugin-failures/plugins/weather/*.ts" + ], + "@example/prices/*": [ + "../../../packages/chord/test/next/examples/36-plugin-failures/plugins/prices/*.ts" + ], + "@example/shapes/*": [ + "../../../packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/*.ts" + ], + "@example/shared/*": [ + "../../../packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/*.ts" + ], + "@earendil-works/chord/next/http": [ + "../../../packages/chord/src/next-node/http/index.ts" + ], + "@earendil-works/chord/next/workers": [ + "../../../packages/chord/src/next-node/workers/index.ts" + ], + "@earendil-works/chord/node": [ + "../../../packages/chord/src/node.ts" + ], + "@earendil-works/chord/output": [ + "../../../packages/chord/src/output.ts" + ], + "@earendil-works/chord/telemetry-sqlite": [ + "../../../packages/chord/src/telemetry-sqlite/index.ts" + ], + "@earendil-works/chord/sqlite": [ + "../../../packages/chord/src/sqlite/index.ts" + ], + "@earendil-works/chord/sqlite/node": [ + "../../../packages/chord/src/sqlite/node.ts" + ], + "@earendil-works/chord/sqlite/cloudflare": [ + "../../../packages/chord/src/sqlite/cloudflare.ts" + ], + "@earendil-works/pi-telemetry": [ + "../../../packages/telemetry/src/index.ts" + ], + "@earendil-works/pi-telemetry/testing": [ + "../../../packages/telemetry/src/testing/index.ts" + ], + "@earendil-works/pi-telemetry/render": [ + "../../../packages/telemetry/src/render.ts" + ], + "@earendil-works/pi-codemode": [ + "../../../packages/codemode/src/index.ts" + ], + "@earendil-works/pi-codemode/cloudflare": [ + "../../../packages/codemode/src/cloudflare.ts" + ], + "@earendil-works/pi-codemode/declarations": [ + "../../../packages/codemode/src/declarations.ts" + ], + "@earendil-works/pi-codemode/portable": [ + "../../../packages/codemode/src/portable.ts" + ], + "@earendil-works/pi-codemode/source": [ + "../../../packages/codemode/src/source.ts" + ], + "@earendil-works/pi-codemode/worker": [ + "../../../packages/codemode/src/runtime/worker.ts" + ], + "@earendil-works/pi-schema": [ + "../../../packages/schema/src/index.ts" + ], + "@earendil-works/pi-schema/formats": [ + "../../../packages/schema/src/formats/index.ts" + ], + "@earendil-works/pi-mcp": [ + "../../../packages/mcp/src/index.ts" + ], + "@earendil-works/pi-mcp/oauth": [ + "../../../packages/mcp/src/oauth/index.ts" + ], + "@earendil-works/pi-mcp/testing": [ + "../../../packages/mcp/src/testing/index.ts" + ], + "@earendil-works/pi-ai": [ + "../../../packages/coding-agent/src/core/extensions/pi-ai-extension-view.ts" + ], + "@earendil-works/pi-ai/oauth": [ + "../../../packages/ai/src/oauth.ts" + ], + "@earendil-works/pi-ai/*": [ + "../../../packages/ai/src/*.ts", + "../../../packages/ai/src/providers/*.ts" + ], + "@earendil-works/pi-ai/dist/*": [ + "../../../packages/ai/src/*" + ], + "@earendil-works/pi-durable": [ + "../../../packages/durable/src/index.ts" + ], + "@earendil-works/pi-durable/testing": [ + "../../../packages/durable/src/testing/index.ts" + ], + "@earendil-works/pi-durable/*": [ + "../../../packages/durable/src/*.ts", + "../../../packages/durable/src/*/index.ts", + "../../../packages/durable/src/*" + ], + "@earendil-works/pi-env": [ + "../../../packages/env/src/index.ts" + ], + "@earendil-works/pi-agent-core": [ + "../../../packages/agent/src/index.ts" + ], + "@earendil-works/pi-coding-agent": [ + "../../../packages/coding-agent/src/index.ts" + ], + "@earendil-works/pi-coding-agent/hooks": [ + "../../../packages/coding-agent/src/core/hooks/index.ts" + ], + "@earendil-works/pi-coding-agent/*": [ + "../../../packages/coding-agent/src/*" + ], + "typebox": [ + "../../../node_modules/typebox" + ], + "@earendil-works/pi-tui": [ + "../../../packages/tui/src/index.ts" + ], + "@earendil-works/pi-tui/*": [ + "../../../packages/tui/src/*" + ], + "@earendil-works/pi-agent-old": [ + "../../../packages/agent-old/src/index.ts" + ], + "@earendil-works/pi-agent-old/*": [ + "../../../packages/agent-old/src/*" + ], + "@mariozechner/pi-ai": [ + "../../../packages/coding-agent/src/core/extensions/pi-ai-extension-view.ts" + ] + } + }, + "include": [ + "./extensions/**/*", + "../../*/src/**/*.d.ts", + "../test/suite/regressions/8261-subagent-project-trust.test.ts" + ], + "exclude": [ + "**/node_modules/**", + "./extensions/gondolin/**" + ] +} diff --git a/packages/coding-agent/install-lock/package-lock.json b/packages/coding-agent/install-lock/package-lock.json index ed922e44f..521c82fef 100644 --- a/packages/coding-agent/install-lock/package-lock.json +++ b/packages/coding-agent/install-lock/package-lock.json @@ -391,7 +391,7 @@ "license": "MIT", "dependencies": { "@earendil-works/pi-ai": "1.4.0", - "typebox": "1.3.27" + "@earendil-works/pi-schema": "1.4.0" }, "engines": { "node": ">=22.19.0" @@ -404,14 +404,14 @@ "dependencies": { "@anthropic-ai/sdk": "0.129.0", "@aws-sdk/client-bedrock-runtime": "3.1127.0", + "@earendil-works/pi-schema": "1.4.0", "@earendil-works/pi-telemetry": "1.4.0", "@google/genai": "2.21.0", "@smithy/node-http-handler": "4.12.1", "http-proxy-agent": "9.1.0", "https-proxy-agent": "9.1.0", "openai": "7.19.0", - "partial-json": "0.1.7", - "typebox": "1.3.27" + "partial-json": "0.1.7" }, "bin": { "pi-ai": "dist/cli.js" @@ -537,6 +537,14 @@ "node": ">=22.19.0" } }, + "node_modules/@earendil-works/pi-schema": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-schema/-/pi-schema-1.4.0.tgz", + "license": "MIT", + "engines": { + "node": ">=22.19.0" + } + }, "node_modules/@earendil-works/pi-telemetry": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/@earendil-works/pi-telemetry/-/pi-telemetry-1.4.0.tgz", diff --git a/packages/coding-agent/schemas/models.schema.json b/packages/coding-agent/schemas/models.schema.json index 39ec3d63e..c51a82c35 100644 --- a/packages/coding-agent/schemas/models.schema.json +++ b/packages/coding-agent/schemas/models.schema.json @@ -255,13 +255,6 @@ "type": "array", "items": { "type": "object", - "required": [ - "inputTokensAbove", - "input", - "output", - "cacheRead", - "cacheWrite" - ], "properties": { "inputTokensAbove": { "type": "number", @@ -283,7 +276,14 @@ "type": "number", "description": "Cache-write cost in USD per million tokens." } - } + }, + "required": [ + "inputTokensAbove", + "input", + "output", + "cacheRead", + "cacheWrite" + ] }, "description": "Request-wide pricing tiers. The highest matching input threshold applies to the full request." } @@ -383,12 +383,6 @@ "$defs": { "ModelCost": { "type": "object", - "required": [ - "input", - "output", - "cacheRead", - "cacheWrite" - ], "properties": { "input": { "type": "number", @@ -410,13 +404,6 @@ "type": "array", "items": { "type": "object", - "required": [ - "inputTokensAbove", - "input", - "output", - "cacheRead", - "cacheWrite" - ], "properties": { "inputTokensAbove": { "type": "number", @@ -438,11 +425,24 @@ "type": "number", "description": "Cache-write cost in USD per million tokens." } - } + }, + "required": [ + "inputTokensAbove", + "input", + "output", + "cacheRead", + "cacheWrite" + ] }, "description": "Request-wide pricing tiers. The highest matching input threshold applies to the full request." } - } + }, + "required": [ + "input", + "output", + "cacheRead", + "cacheWrite" + ] }, "ModelInputLimits": { "type": "object", @@ -628,9 +628,6 @@ }, { "type": "object", - "required": [ - "$var" - ], "properties": { "$var": { "anyOf": [ @@ -651,7 +648,10 @@ "omitWhenOff": { "type": "boolean" } - } + }, + "required": [ + "$var" + ] } ] } @@ -677,9 +677,6 @@ }, { "type": "object", - "required": [ - "$var" - ], "properties": { "$var": { "anyOf": [ @@ -700,7 +697,10 @@ "omitWhenOff": { "type": "boolean" } - } + }, + "required": [ + "$var" + ] } ] } @@ -1085,11 +1085,6 @@ "type": "array", "items": { "type": "object", - "required": [ - "provider", - "model", - "cost" - ], "properties": { "provider": { "type": "string", @@ -1103,6 +1098,11 @@ "$ref": "#/$defs/ModelCost" } }, + "required": [ + "provider", + "model", + "cost" + ], "description": "An Anthropic server-side refusal fallback model with local pricing metadata." }, "maxItems": 3, diff --git a/packages/coding-agent/src/core/extensions/loader.ts b/packages/coding-agent/src/core/extensions/loader.ts index 235a8632e..ff9f127c4 100644 --- a/packages/coding-agent/src/core/extensions/loader.ts +++ b/packages/coding-agent/src/core/extensions/loader.ts @@ -88,10 +88,11 @@ function getAliases(): Record { const piCodingAgentEntry = packageIndex; const piAgentCoreEntry = resolveWorkspaceOrImport("agent/dist/index.js", "@earendil-works/pi-agent-core"); const piTuiEntry = resolveWorkspaceOrImport("tui/dist/index.js", "@earendil-works/pi-tui"); - // Extensions resolve the pi-ai root to the compat entrypoint (a strict - // superset of the core entrypoint): existing extensions using the old - // global API keep working at runtime until compat is removed. - const piAiCompatEntry = resolveWorkspaceOrImport("ai/dist/compat.js", "@earendil-works/pi-ai/compat"); + // Extensions resolve the pi-ai root to the extension view: the compat + // entrypoint (a strict superset of the core entrypoint) plus TypeBox's Type, + // Static and TSchema, so extensions using the old global API and pi-ai's + // former TypeBox re-exports keep working. + const piAiCompatEntry = path.resolve(__dirname, "pi-ai-extension-view.js"); const piAiModelsEntry = resolveWorkspaceOrImport("ai/dist/models.js", "@earendil-works/pi-ai/models"); const piAiOauthEntry = resolveWorkspaceOrImport("ai/dist/oauth.js", "@earendil-works/pi-ai/oauth"); const piAiProvidersEntry = resolveWorkspaceOrImport( diff --git a/packages/coding-agent/src/core/extensions/pi-ai-extension-view.ts b/packages/coding-agent/src/core/extensions/pi-ai-extension-view.ts new file mode 100644 index 000000000..e9c6808f7 --- /dev/null +++ b/packages/coding-agent/src/core/extensions/pi-ai-extension-view.ts @@ -0,0 +1,28 @@ +/** + * The `@earendil-works/pi-ai` module extensions import: pi-ai's compat entry point plus TypeBox's `Type`, `Static` and + * `TSchema`, which pi-ai's main entry no longer re-exports, and a `StringEnum` built with TypeBox (`TUnsafe`), so + * extensions that use it inside `Type.Object` type-check and infer as before. The TypeBox is the coding agent's own, + * the same one extensions get for `typebox`. + */ + +import { type TUnsafe, Type } from "typebox"; + +export * from "@earendil-works/pi-ai/compat"; +export type { Static, TSchema } from "typebox"; +export { Type }; + +/** + * Creates a string enum schema compatible with Google's API and other providers that don't support anyOf/const + * patterns: `{ type: "string", enum: [...] }`, typed for TypeBox's `Static`. + */ +export function StringEnum( + values: T, + options?: { description?: string; default?: T[number] }, +): TUnsafe { + return Type.Unsafe({ + type: "string", + enum: values as unknown as T[number][], + ...(options?.description && { description: options.description }), + ...(options?.default && { default: options.default }), + }); +} diff --git a/packages/coding-agent/src/core/extensions/virtual-modules.ts b/packages/coding-agent/src/core/extensions/virtual-modules.ts index 4b8449338..3c30317d0 100644 --- a/packages/coding-agent/src/core/extensions/virtual-modules.ts +++ b/packages/coding-agent/src/core/extensions/virtual-modules.ts @@ -1,5 +1,4 @@ import * as bundledPiAgentCore from "@earendil-works/pi-agent-core"; -import * as bundledPiAiCompat from "@earendil-works/pi-ai/compat"; import * as bundledPiAiModels from "@earendil-works/pi-ai/models"; import * as bundledPiAiOauth from "@earendil-works/pi-ai/oauth"; import * as bundledPiAiProviders from "@earendil-works/pi-ai/providers/all"; @@ -10,6 +9,7 @@ import * as bundledTypeboxValue from "typebox/value"; // This import is safe because loader.ts exports are not re-exported from index.ts. // Extensions can therefore import from @earendil-works/pi-coding-agent. import * as bundledPiCodingAgent from "../../index.ts"; +import * as bundledPiAiExtensionView from "./pi-ai-extension-view.ts"; /** Modules available to extensions in source and compiled binary runtimes. */ export const VIRTUAL_MODULES: Record = { @@ -22,18 +22,19 @@ export const VIRTUAL_MODULES: Record = { "@earendil-works/pi-agent-core": bundledPiAgentCore, "@earendil-works/pi-tui": bundledPiTui, // Extensions resolve the pi-ai root to the compat entrypoint (a strict - // superset of the core entrypoint): existing extensions using the old - // global API keep working at runtime until compat is removed. - "@earendil-works/pi-ai": bundledPiAiCompat, - "@earendil-works/pi-ai/compat": bundledPiAiCompat, + // superset of the core entrypoint) plus TypeBox's Type, Static and TSchema: + // existing extensions using the old global API and pi-ai's former TypeBox + // re-exports keep working. + "@earendil-works/pi-ai": bundledPiAiExtensionView, + "@earendil-works/pi-ai/compat": bundledPiAiExtensionView, "@earendil-works/pi-ai/models": bundledPiAiModels, "@earendil-works/pi-ai/oauth": bundledPiAiOauth, "@earendil-works/pi-ai/providers/all": bundledPiAiProviders, "@earendil-works/pi-coding-agent": bundledPiCodingAgent, "@mariozechner/pi-agent-core": bundledPiAgentCore, "@mariozechner/pi-tui": bundledPiTui, - "@mariozechner/pi-ai": bundledPiAiCompat, - "@mariozechner/pi-ai/compat": bundledPiAiCompat, + "@mariozechner/pi-ai": bundledPiAiExtensionView, + "@mariozechner/pi-ai/compat": bundledPiAiExtensionView, "@mariozechner/pi-ai/models": bundledPiAiModels, "@mariozechner/pi-ai/oauth": bundledPiAiOauth, "@mariozechner/pi-ai/providers/all": bundledPiAiProviders, diff --git a/packages/coding-agent/src/core/model-config.ts b/packages/coding-agent/src/core/model-config.ts index 553ff35ba..200cdeec0 100644 --- a/packages/coding-agent/src/core/model-config.ts +++ b/packages/coding-agent/src/core/model-config.ts @@ -1,20 +1,21 @@ /** Immutable, credential-blind models.json snapshot. */ import { readFile } from "node:fs/promises"; -import { ProviderCompatSchema } from "@earendil-works/pi-ai/providers/compat-schema"; -import { - ModelCostSchema, - ModelInputLimitsSchema, - ModelInputModalitySchema, - ModelPromptCacheSchema, - ThinkingLevelMapSchema, -} from "@earendil-works/pi-ai/providers/model-schema"; import { type Static, Type } from "typebox"; import { Compile } from "typebox/compile"; import type { TLocalizedValidationError } from "typebox/error"; import { stripJsonComments } from "../utils/json.ts"; import { normalizePath } from "../utils/paths.ts"; import { stripBom } from "../utils/text.ts"; +import { + ModelCostSchema, + ModelInputLimitsSchema, + ModelInputModalitySchema, + ModelPromptCacheSchema, + PartialModelCostSchema, + ProviderCompatSchema, + ThinkingLevelMapSchema, +} from "./pi-ai-schemas.ts"; const SamplingParamsSchema = Type.Record(Type.String(), Type.Unknown()); const SamplingParamsByThinkingLevelSchema = Type.Object( @@ -74,7 +75,7 @@ const ModelOverrideSchema = Type.Object({ thinkingLevelMap: Type.Optional(ThinkingLevelMapSchema), input: Type.Optional(Type.Array(ModelInputModalitySchema)), inputLimits: Type.Optional(ModelInputLimitsSchema), - cost: Type.Optional(Type.Partial(ModelCostSchema)), + cost: Type.Optional(PartialModelCostSchema), promptCache: Type.Optional(ModelPromptCacheSchema), contextWindow: Type.Optional( Type.Number({ exclusiveMinimum: 0, description: ModelFieldDescriptions.contextWindow }), diff --git a/packages/coding-agent/src/core/pi-ai-schemas.ts b/packages/coding-agent/src/core/pi-ai-schemas.ts new file mode 100644 index 000000000..694aa1fe6 --- /dev/null +++ b/packages/coding-agent/src/core/pi-ai-schemas.ts @@ -0,0 +1,40 @@ +/** + * pi-ai's configuration schemas (built with pi-schema) as TypeBox schemas, for the coding agent's own TypeBox-built + * configuration schemas (models.json, settings.json): the same JSON, typed for TypeBox's `Static`. + */ + +import { + type ProviderCompat, + ProviderCompatSchema as ProviderCompatJsonSchema, +} from "@earendil-works/pi-ai/providers/compat-schema"; +import { + type ModelCost, + ModelCostSchema as ModelCostJsonSchema, + type ModelInputLimits, + ModelInputLimitsSchema as ModelInputLimitsJsonSchema, + type ModelInputModality, + ModelInputModalitySchema as ModelInputModalityJsonSchema, + type ModelPromptCache, + ModelPromptCacheSchema as ModelPromptCacheJsonSchema, + type ModelThinkingLevel, + ModelThinkingLevelSchema as ModelThinkingLevelJsonSchema, + type ThinkingLevelMap, + ThinkingLevelMapSchema as ThinkingLevelMapJsonSchema, +} from "@earendil-works/pi-ai/providers/model-schema"; +import { Type } from "typebox"; + +export const ProviderCompatSchema = Type.Unsafe(ProviderCompatJsonSchema); +export const ModelCostSchema = Type.Unsafe(ModelCostJsonSchema); +export const ModelInputLimitsSchema = Type.Unsafe(ModelInputLimitsJsonSchema); +export const ModelInputModalitySchema = Type.Unsafe(ModelInputModalityJsonSchema); +export const ModelPromptCacheSchema = Type.Unsafe(ModelPromptCacheJsonSchema); +export const ModelThinkingLevelSchema = Type.Unsafe(ModelThinkingLevelJsonSchema); +export const ThinkingLevelMapSchema = Type.Unsafe(ThinkingLevelMapJsonSchema); + +/** `ModelCostSchema` with every field optional, for model overrides that change some of the rates. */ +export const PartialModelCostSchema = Type.Unsafe>(withoutRequired(ModelCostJsonSchema)); + +function withoutRequired(schema: object): object { + const { required: _required, ...rest } = schema as { required?: unknown }; + return rest; +} diff --git a/packages/coding-agent/src/core/settings-schema.ts b/packages/coding-agent/src/core/settings-schema.ts index c3ada2872..1ef797338 100644 --- a/packages/coding-agent/src/core/settings-schema.ts +++ b/packages/coding-agent/src/core/settings-schema.ts @@ -1,5 +1,5 @@ -import { ModelThinkingLevelSchema } from "@earendil-works/pi-ai/providers/model-schema"; import { type Static, type TSchemaOptions, Type } from "typebox"; +import { ModelThinkingLevelSchema } from "./pi-ai-schemas.ts"; import { SETTINGS_DEFAULTS } from "./settings-defaults.ts"; function nonNegativeSafeInteger(options: { default?: number } = {}) { diff --git a/packages/coding-agent/src/core/tools/bash.ts b/packages/coding-agent/src/core/tools/bash.ts index 4620e2464..2084d6f7c 100644 --- a/packages/coding-agent/src/core/tools/bash.ts +++ b/packages/coding-agent/src/core/tools/bash.ts @@ -1,7 +1,6 @@ import { constants } from "node:fs"; import { access as fsAccess } from "node:fs/promises"; import { constants as osConstants } from "node:os"; -import type { AgentTool } from "@earendil-works/pi-agent-core"; import { spawn } from "child_process"; import { type Static, Type } from "typebox"; import { waitForChildProcess } from "../../utils/child-process.ts"; @@ -16,7 +15,7 @@ import { import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; import { OutputAccumulator } from "./output-accumulator.ts"; import { BASH_UPDATE_THROTTLE_MS, createShellRenderers } from "./renderers/bash.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, type TruncationResult } from "./truncate.ts"; const MAX_TIMEOUT_MS = 2_147_483_647; @@ -431,7 +430,7 @@ export function createBashToolDefinition( return createShellToolDefinition(cwd, bashToolConfig, options); } -export function createBashTool(cwd: string, options?: BashToolOptions): AgentTool { +export function createBashTool(cwd: string, options?: BashToolOptions): TypeBoxAgentTool { const definition = createBashToolDefinition(cwd, options); const tool = wrapToolDefinition(definition); Object.assign(tool, { diff --git a/packages/coding-agent/src/core/tools/edit.ts b/packages/coding-agent/src/core/tools/edit.ts index ad1cbd2f3..bb6aeff59 100644 --- a/packages/coding-agent/src/core/tools/edit.ts +++ b/packages/coding-agent/src/core/tools/edit.ts @@ -1,4 +1,3 @@ -import type { AgentTool } from "@earendil-works/pi-agent-core"; import { constants } from "fs"; import { access as fsAccess, readFile as fsReadFile, writeFile as fsWriteFile } from "fs/promises"; import { type Static, Type } from "typebox"; @@ -16,7 +15,7 @@ import { import { withFileMutationQueue } from "./file-mutation-queue.ts"; import { resolveToCwd } from "./path-utils.ts"; import { type EditRenderState, editRenderers } from "./renderers/edit.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; const replaceEditSchema = Type.Object( { @@ -213,6 +212,6 @@ export function createEditToolDefinition( }; } -export function createEditTool(cwd: string, options?: EditToolOptions): AgentTool { +export function createEditTool(cwd: string, options?: EditToolOptions): TypeBoxAgentTool { return wrapToolDefinition(createEditToolDefinition(cwd, options)); } diff --git a/packages/coding-agent/src/core/tools/find.ts b/packages/coding-agent/src/core/tools/find.ts index 8d44c491e..77ba1b356 100644 --- a/packages/coding-agent/src/core/tools/find.ts +++ b/packages/coding-agent/src/core/tools/find.ts @@ -1,5 +1,4 @@ import { createInterface } from "node:readline"; -import type { AgentTool } from "@earendil-works/pi-agent-core"; import { spawn } from "child_process"; import path from "path"; import { type Static, Type } from "typebox"; @@ -7,7 +6,7 @@ import { ensureTool } from "../../utils/tools-manager.ts"; import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; import { pathExists, resolveToCwd } from "./path-utils.ts"; import { findRenderers } from "./renderers/find.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; import { DEFAULT_MAX_BYTES, formatSize, type TruncationResult, truncateHead } from "./truncate.ts"; /** Relativize a find result against the search root and normalize it to posix separators. */ @@ -313,6 +312,6 @@ export function createFindToolDefinition( }; } -export function createFindTool(cwd: string, options?: FindToolOptions): AgentTool { +export function createFindTool(cwd: string, options?: FindToolOptions): TypeBoxAgentTool { return wrapToolDefinition(createFindToolDefinition(cwd, options)); } diff --git a/packages/coding-agent/src/core/tools/grep.ts b/packages/coding-agent/src/core/tools/grep.ts index 77d830cd1..e449e0898 100644 --- a/packages/coding-agent/src/core/tools/grep.ts +++ b/packages/coding-agent/src/core/tools/grep.ts @@ -1,6 +1,5 @@ import { readFile as fsReadFile, stat as fsStat } from "node:fs/promises"; import { createInterface } from "node:readline"; -import type { AgentTool } from "@earendil-works/pi-agent-core"; import { spawn } from "child_process"; import path from "path"; import { type Static, Type } from "typebox"; @@ -8,7 +7,7 @@ import { ensureTool } from "../../utils/tools-manager.ts"; import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; import { resolveToCwd } from "./path-utils.ts"; import { grepRenderers } from "./renderers/grep.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; import { DEFAULT_MAX_BYTES, formatSize, @@ -320,6 +319,6 @@ export function createGrepToolDefinition( }; } -export function createGrepTool(cwd: string, options?: GrepToolOptions): AgentTool { +export function createGrepTool(cwd: string, options?: GrepToolOptions): TypeBoxAgentTool { return wrapToolDefinition(createGrepToolDefinition(cwd, options)); } diff --git a/packages/coding-agent/src/core/tools/ls.ts b/packages/coding-agent/src/core/tools/ls.ts index 558ff9f9f..4f5efc8a7 100644 --- a/packages/coding-agent/src/core/tools/ls.ts +++ b/packages/coding-agent/src/core/tools/ls.ts @@ -1,11 +1,10 @@ import { readdir as fsReaddir, stat as fsStat } from "node:fs/promises"; -import type { AgentTool } from "@earendil-works/pi-agent-core"; import nodePath from "path"; import { type Static, Type } from "typebox"; import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; import { pathExists, resolveToCwd } from "./path-utils.ts"; import { lsRenderers } from "./renderers/ls.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; import { DEFAULT_MAX_BYTES, formatSize, type TruncationResult, truncateHead } from "./truncate.ts"; const lsSchema = Type.Object({ @@ -170,6 +169,6 @@ export function createLsToolDefinition( }; } -export function createLsTool(cwd: string, options?: LsToolOptions): AgentTool { +export function createLsTool(cwd: string, options?: LsToolOptions): TypeBoxAgentTool { return wrapToolDefinition(createLsToolDefinition(cwd, options)); } diff --git a/packages/coding-agent/src/core/tools/read.ts b/packages/coding-agent/src/core/tools/read.ts index ebe2c2517..8fb93a679 100644 --- a/packages/coding-agent/src/core/tools/read.ts +++ b/packages/coding-agent/src/core/tools/read.ts @@ -1,4 +1,3 @@ -import type { AgentTool } from "@earendil-works/pi-agent-core"; import type { Api, ImageContent, Model, ModelImageResizeOptions, TextContent } from "@earendil-works/pi-ai"; import { constants } from "fs"; import { access as fsAccess, readFile as fsReadFile } from "fs/promises"; @@ -8,7 +7,7 @@ import { detectSupportedImageMimeTypeFromFile } from "../../utils/mime.ts"; import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; import { resolveReadPathAsync } from "./path-utils.ts"; import { readRenderers } from "./renderers/read.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, type TruncationResult, truncateHead } from "./truncate.ts"; const readSchema = Type.Object({ @@ -219,6 +218,6 @@ export function createReadToolDefinition( }; } -export function createReadTool(cwd: string, options?: ReadToolOptions): AgentTool { +export function createReadTool(cwd: string, options?: ReadToolOptions): TypeBoxAgentTool { return wrapToolDefinition(createReadToolDefinition(cwd, options)); } diff --git a/packages/coding-agent/src/core/tools/tool-definition-wrapper.ts b/packages/coding-agent/src/core/tools/tool-definition-wrapper.ts index 9a9465dec..320030fab 100644 --- a/packages/coding-agent/src/core/tools/tool-definition-wrapper.ts +++ b/packages/coding-agent/src/core/tools/tool-definition-wrapper.ts @@ -1,6 +1,25 @@ -import type { AgentTool } from "@earendil-works/pi-agent-core"; +import type { AgentTool, AgentToolResult, AgentToolUpdateCallback } from "@earendil-works/pi-agent-core"; +import type { Static, TSchema } from "typebox"; import type { ExtensionToolContext, ToolDefinition } from "../extensions/types.ts"; +/** + * An `AgentTool` with a TypeBox parameter schema whose `execute` and `prepareArguments` are typed by TypeBox's + * `Static`, as the built-in tools are (pi-agent-core types `params` through pi-schema, which gives `unknown` for + * TypeBox schemas). Spreads into a `ToolDefinition` of the same schema. + */ +export type TypeBoxAgentTool = Omit< + AgentTool, + "prepareArguments" | "execute" +> & { + prepareArguments?: (args: unknown) => Static; + execute: ( + toolCallId: string, + params: Static, + signal?: AbortSignal, + onUpdate?: AgentToolUpdateCallback, + ) => Promise>; +}; + /** Creates the context for one tool call. */ export type ToolContextFactory = (toolCallId: string, signal: AbortSignal | undefined) => ExtensionToolContext; diff --git a/packages/coding-agent/src/core/tools/write.ts b/packages/coding-agent/src/core/tools/write.ts index 374089017..9e6b296dc 100644 --- a/packages/coding-agent/src/core/tools/write.ts +++ b/packages/coding-agent/src/core/tools/write.ts @@ -1,4 +1,3 @@ -import type { AgentTool } from "@earendil-works/pi-agent-core"; import { mkdir as fsMkdir, writeFile as fsWriteFile } from "fs/promises"; import { dirname } from "path"; import { type Static, Type } from "typebox"; @@ -6,7 +5,7 @@ import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; import { withFileMutationQueue } from "./file-mutation-queue.ts"; import { resolveToCwd } from "./path-utils.ts"; import { writeRenderers } from "./renderers/write.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; const writeSchema = Type.Object({ path: Type.String({ description: "Path to the file to write (relative or absolute)" }), @@ -92,6 +91,6 @@ export function createWriteToolDefinition( }; } -export function createWriteTool(cwd: string, options?: WriteToolOptions): AgentTool { +export function createWriteTool(cwd: string, options?: WriteToolOptions): TypeBoxAgentTool { return wrapToolDefinition(createWriteToolDefinition(cwd, options)); } diff --git a/packages/coding-agent/src/extensions/codemode/tool.ts b/packages/coding-agent/src/extensions/codemode/tool.ts index e28ea999f..648ba6109 100644 --- a/packages/coding-agent/src/extensions/codemode/tool.ts +++ b/packages/coding-agent/src/extensions/codemode/tool.ts @@ -32,7 +32,7 @@ import type { ToolDefinition, ToolLoadout, ToolLoadoutChanges, ToolNamespace } f import type { ModelRegistry } from "../../core/model-registry.ts"; import { SETTINGS_DEFAULTS } from "../../core/settings-defaults.ts"; import type { CodemodeMode } from "../../core/settings-manager.ts"; -import { wrapToolDefinition } from "../../core/tools/tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "../../core/tools/tool-definition-wrapper.ts"; import { loadCodemodeExecutor } from "./execute.lazy.ts"; import { codemodeRenderers } from "./renderer.ts"; @@ -422,7 +422,7 @@ export function createCodemodeToolDefinition( export function createCodemodeTool( tools: readonly AgentTool[] = [], options: CodemodeToolOptions = {}, -): AgentTool { +): TypeBoxAgentTool { const definition = createCodemodeToolDefinition(options); const tool = wrapToolDefinition(definition); Object.assign(tool, { diff --git a/packages/coding-agent/test/sdk-codex-cache-probe-tool-loop.ts b/packages/coding-agent/test/sdk-codex-cache-probe-tool-loop.ts index 7c569f6e8..2813adc0f 100644 --- a/packages/coding-agent/test/sdk-codex-cache-probe-tool-loop.ts +++ b/packages/coding-agent/test/sdk-codex-cache-probe-tool-loop.ts @@ -21,8 +21,8 @@ import { getModel, type Model, type SimpleStreamOptions, - Type, } from "@earendil-works/pi-ai/compat"; +import { Type } from "typebox"; import { getOpenAICodexWebSocketDebugStats, streamSimple as streamSimpleOpenAICodexResponses, diff --git a/packages/coding-agent/test/suite/agent-session-codemode.test.ts b/packages/coding-agent/test/suite/agent-session-codemode.test.ts index ff39f1cd6..3e0620635 100644 --- a/packages/coding-agent/test/suite/agent-session-codemode.test.ts +++ b/packages/coding-agent/test/suite/agent-session-codemode.test.ts @@ -18,7 +18,10 @@ import { Type } from "typebox"; import { afterEach, describe, expect, it } from "vitest"; import type { ExtensionAPI } from "../../src/core/extensions/types.ts"; import type { CustomEntry } from "../../src/core/session-manager.ts"; -import { createToolDefinitionFromAgentTool } from "../../src/core/tools/tool-definition-wrapper.ts"; +import { + createToolDefinitionFromAgentTool, + type TypeBoxAgentTool, +} from "../../src/core/tools/tool-definition-wrapper.ts"; import { readCodemodeStore } from "../../src/extensions/codemode/execute.ts"; import { createCodemodeExtension } from "../../src/extensions/codemode/index.ts"; import { @@ -33,7 +36,7 @@ const TINY_PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; const echoSchema = Type.Object({ text: Type.String({ description: "Text to echo" }) }); -const echoTool: AgentTool = { +const echoTool: TypeBoxAgentTool = { name: "echo", label: "Echo", description: "Echo text back.\n\nSecond paragraph.", diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 009e0367f..e852e8fa8 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -4,6 +4,7 @@ ### Breaking Changes +- Tool schemas are JSON Schema typed through `@earendil-works/pi-schema`: `defineTool()` and `ToolRegistration` take any JSON Schema object as `parameters` and `structuredOutputSchema`, and type `args` with pi-schema's `Static` (`ToolArgs`; other schemas give `unknown`). The built-in tools (`read`, `write`, `edit`, `bash`, `powershell`, `codemode`, `tool_search`, the model tools) are built with pi-schema, with the same JSON Schema up to key order. `structuredOutput` is validated with pi-schema, so `invalid_structured_output` messages use its wording. `typebox` is no longer a dependency. Migration: replace `Type.Object({ n: Type.Number() })` from `@earendil-works/pi-ai` with `obj({ n: num() })` from `@earendil-works/pi-schema` (closed by default; `{ open: true }` allows other properties). - `SqliteExecutor` (re-exported from Chord) gains `runCounted(sql, ...params)`, resolving with the rows the statement changed; custom `SqliteDatabase` implementations add it. - `Storage.commit()` stores a group of commits in one call: `commit(batches, receive, context)` resolves with a `CommitResult` (`{ stored, refused? }`) and replaces `commit(writes, adopt, context)`. Each batch is one atomic commit with its own seq, checked against the state the batches before it left. The first batch Durable's own checks refuse (global IDs, document actions, copy sources) is not stored, nor anything after it; the batches before it are, and the call resolves with their number in `stored` and the refusal in `refused`. Any other failure (a driver or I/O error, a failed `COMMIT`) rejects the whole call as an unknown outcome and stores nothing the backend could roll back. `receive(index, seq)` is called once per stored batch, in order, synchronously, once they are durable and before the next call starts; receipts made before a rejection stand. Every batch has at least one write. The optional `publishing(publish, context)` runs each commit's publication inside the storage's own scope (`StoreStorage`: the attached host's batch). Memory, SQLite, JSONL, IndexedDB and `StoreStorage` store a call in one transaction or append, and the storage conformance suite checks the contract. Migration for a custom Storage: diff --git a/packages/durable/README.md b/packages/durable/README.md index 95e9c3085..b9e54a068 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -257,15 +257,15 @@ registry.install(createCodingTools({ images: await createNodePhotonImages() })); A model without image input sees a placeholder instead of the image, and `read` says so in a diagnostic. `/images/node` prepares images in a worker thread, so a large photo (a second or two of work) never blocks the event loop; the worker exits after 30 s without images, freeing Photon's WebAssembly memory (`createNodePhotonImages({ worker: false, idleMs })`). Elsewhere, `createPhotonImages()` and `/images/cloudflare` work on the calling thread, and decoding takes the image's full size in WebAssembly memory, which does not shrink again; leave the processor out on memory-constrained hosts such as Cloudflare Workers unless images stay small. -Define your own tool with a TypeBox schema. `defineTool()` types `args` from `parameters`, which the Harness validates before `execute()`. `api.output()` streams running output, which becomes the result when `execute()` returns no `output`: +Define your own tool with a JSON Schema, built here with `@earendil-works/pi-schema`. `defineTool()` types `args` from `parameters` (through pi-schema's `Static`; other JSON Schema gives `unknown`), which the Harness coerces and validates before `execute()`. `api.output()` streams running output, which becomes the result when `execute()` returns no `output`: ```typescript -import { Type } from "@earendil-works/pi-ai"; +import { num, obj, str } from "@earendil-works/pi-schema"; const count = defineTool({ name: "count", description: "Count from 1 to n", - parameters: Type.Object({ n: Type.Number() }), + parameters: obj({ n: num() }), execute: async (args, api) => { for (let i = 1; i <= args.n; i++) api.output(`${i}\n`); return {}; @@ -282,8 +282,8 @@ A result has three channels: `output` for the model, `structuredOutput` for prog const lines = defineTool({ name: "lines", description: "Count the lines of a file", - parameters: Type.Object({ path: Type.String() }), - structuredOutputSchema: Type.Object({ lines: Type.Number() }), + parameters: obj({ path: str() }), + structuredOutputSchema: obj({ lines: num() }), execute: async (args, api, context) => { const text = await readText(api, args.path, context); const count = text.split("\n").length; @@ -312,7 +312,7 @@ A tool calls another tool with `api.executeTool()`, as a code mode script or an const testsPass = defineTool({ name: "tests_pass", description: "Run the test suite and say whether it passed", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, context) => { const run = await api.executeTool("bash", { command: "npm test" }, context, { progress: false }); // Absent when bash did not finish, for example on a timeout. @@ -722,7 +722,7 @@ const Subagent: Extension = defineExtension({ defineTool({ name: "subagent", description: "Delegate a self-contained task to a subagent and get its answer back.", - parameters: Type.Object({ task: Type.String() }), + parameters: obj({ task: str() }), replay: "safe", // a rerun after a crash finds the same child and submission execute: async (args, api, context) => { const child = await api.commit(async (tx) => { @@ -905,7 +905,7 @@ registerStorageConformance({ describe, expect, it }, "My Storage", async (use) = }); ``` -The package root loads TypeBox, because the tool task validates arguments with pi-ai's `validateToolArguments()`. That costs about 23 MB of peak RSS unbundled, about 4 MB in a tree-shaken bundle. +The package root loads pi-schema, because the tool task validates arguments with pi-ai's `validateToolArguments()`. ## Errors diff --git a/packages/durable/package.json b/packages/durable/package.json index 57ced6bbb..495197c59 100644 --- a/packages/durable/package.json +++ b/packages/durable/package.json @@ -161,10 +161,10 @@ "@earendil-works/chord": "1.4.0", "@earendil-works/pi-ai": "1.4.0", "@earendil-works/pi-codemode": "1.4.0", + "@earendil-works/pi-schema": "1.4.0", "@earendil-works/pi-telemetry": "1.4.0", "@silvia-odwyer/photon": "0.3.3", - "diff": "8.0.4", - "typebox": "1.3.27" + "diff": "8.0.4" }, "devDependencies": { "@cloudflare/vitest-plugin": "1.4.10", diff --git a/packages/durable/src/codemode/codemode.ts b/packages/durable/src/codemode/codemode.ts index 8041004d2..a2e765779 100644 --- a/packages/durable/src/codemode/codemode.ts +++ b/packages/durable/src/codemode/codemode.ts @@ -46,7 +46,7 @@ import type { CodemodeSandboxBase, } from "@earendil-works/pi-codemode/portable"; import { CODEMODE_SOURCE_GRAMMAR, parseCodemodeSource } from "@earendil-works/pi-codemode/source"; -import { type Static, Type } from "typebox"; +import { obj, type Static, str } from "@earendil-works/pi-schema"; import { defineDoc } from "../documents.ts"; import type { ExecutionEnv } from "../env/index.ts"; import { errorMessage } from "../errors.ts"; @@ -210,7 +210,7 @@ export type CodemodeGlobal = { execute(args: unknown, api: ToolExecutionApi, context: Context): Promise; }; -const codemodeSchema = Type.Object({ code: Type.String({ description: "Raw JavaScript source." }) }); +const codemodeSchema = obj({ code: str({ description: "Raw JavaScript source." }) }, { open: true }); export type CodemodeToolInput = Static; diff --git a/packages/durable/src/codemode/models.ts b/packages/durable/src/codemode/models.ts index ca22e96ea..fcdf63b68 100644 --- a/packages/durable/src/codemode/models.ts +++ b/packages/durable/src/codemode/models.ts @@ -17,7 +17,7 @@ import type { ModelTypeMap, Usage, } from "@earendil-works/pi-ai"; -import { Type } from "typebox"; +import { obj, unknown } from "@earendil-works/pi-schema"; import { defineTool } from "../harness/define.ts"; import type { ToolRegistration } from "../harness/types.ts"; @@ -227,7 +227,7 @@ function resolveModel( ); } -const ModelCallParameters = Type.Object({ model: Type.Unknown(), context: Type.Unknown() }); +const ModelCallParameters = obj({ model: unknown(), context: unknown() }, { open: true }); /** The result of a model call as the tool returns it: the whole result for the script, usage counted by the Harness. */ function modelResult( @@ -255,7 +255,7 @@ export function createModelTools(): ToolRegistration[] { name: CLASSIFY_TOOL, description: "models.classify() of code mode scripts.", parameters: ModelCallParameters, - structuredOutputSchema: Type.Unknown(), + structuredOutputSchema: unknown(), callers: ["tools"], execute: async (args, api, context) => { const model = resolveModel("models.classify", "classifier", args.model, api.models); @@ -271,7 +271,7 @@ export function createModelTools(): ToolRegistration[] { name: GENERATE_IMAGES_TOOL, description: "models.generateImages() of code mode scripts.", parameters: ModelCallParameters, - structuredOutputSchema: Type.Unknown(), + structuredOutputSchema: unknown(), callers: ["tools"], execute: async (args, api, context) => { const model = resolveModel("models.generateImages", "image", args.model, api.models); diff --git a/packages/durable/src/harness/define.ts b/packages/durable/src/harness/define.ts index dd4bd2e8d..f5e0df8be 100644 --- a/packages/durable/src/harness/define.ts +++ b/packages/durable/src/harness/define.ts @@ -1,5 +1,4 @@ import type { JsonValue } from "@earendil-works/chord"; -import type { TSchema } from "@earendil-works/pi-ai"; import type { Task } from "../types.ts"; import type { AnyTask, @@ -20,7 +19,7 @@ export function defineExtension( +export function defineTool( tool: ToolRegistration, ): ToolRegistration { return tool; diff --git a/packages/durable/src/harness/tool.ts b/packages/durable/src/harness/tool.ts index 533795a72..dc9e72ca2 100644 --- a/packages/durable/src/harness/tool.ts +++ b/packages/durable/src/harness/tool.ts @@ -11,7 +11,8 @@ import { } from "@earendil-works/chord/output"; import type { ImageContent, TextContent, ToolCall, ToolResultMessage } from "@earendil-works/pi-ai"; import { validateToolArguments } from "@earendil-works/pi-ai/utils/validation"; -import { Compile } from "typebox/compile"; +import { compile, type Validator } from "@earendil-works/pi-schema"; +import { formats } from "@earendil-works/pi-schema/formats"; import { defineDocFamily } from "../documents.ts"; import { AssistantEntry, ToolResultEntry } from "../entries.ts"; import type { ExecutionEnv } from "../env/index.ts"; @@ -770,7 +771,7 @@ async function afterHarnessResult( return { ...rest, ...value, output: bounded.content, diagnostics: [...(rest.diagnostics ?? []), ...harness] }; } -const structuredValidators = new WeakMap>(); +const structuredValidators = new WeakMap(); /** Why a result's `structuredOutput` breaks the tool's contract, or `undefined` when it keeps it. */ function structuredOutputError(tool: ToolRegistration, result: ToolExecutionResult): string | undefined { @@ -786,12 +787,17 @@ function structuredOutputError(tool: ToolRegistration, result: ToolExecutionResu } let validator = structuredValidators.get(schema); if (validator === undefined) { - validator = Compile(schema); + validator = compile(schema, { formats, unresolvedRefs: "reject" }); structuredValidators.set(schema, validator); } - if (validator.Check(value)) return undefined; - const [first] = validator.Errors(value); - const path = first?.instancePath.replace(/^\//, "").replace(/\//g, ".") || "root"; + if (validator.check(value)) return undefined; + const [first] = validator.errors(value); + const path = + first?.path + .slice(1) + .split("/") + .map((segment) => segment.replace(/~1/g, "/").replace(/~0/g, "~")) + .join(".") || "root"; return `Tool ${tool.name} returned structuredOutput that does not match its schema: ${path}: ${first?.message ?? "invalid"}`; } diff --git a/packages/durable/src/harness/types.ts b/packages/durable/src/harness/types.ts index d114bf6cc..c0f03e65d 100644 --- a/packages/durable/src/harness/types.ts +++ b/packages/durable/src/harness/types.ts @@ -8,15 +8,14 @@ import type { Model, Models, ModelThinkingLevel, - Static, Tool, ToolCall, ToolResultMessage, Transport, - TSchema, Usage, UserMessage, } from "@earendil-works/pi-ai"; +import type { Static } from "@earendil-works/pi-schema"; import type { TelemetryContext } from "@earendil-works/pi-telemetry"; import type { ExecutionEnv, ShellOutputSkip, ShellOutputWindow } from "../env/index.ts"; import type { @@ -333,12 +332,22 @@ export interface ToolExecutionApi extend ): Promise; } +/** + * The arguments type of a tool with parameter schema `TParameters`: `Static` for schemas built with + * `@earendil-works/pi-schema`'s builders, `unknown` for other JSON Schema, and `any` for `ToolRegistration`. + */ +export type ToolArgs = IsAny extends true ? any : Static; + +// `0 extends 1 & T` is resolved too early for a `T` constrained to `object`; this form waits for the argument. +type IsAny = boolean extends (T extends never ? true : false) ? true : false; + /** * Executable tool registered in a registry. Only pi-ai `Tool` fields enter the transcript. `args` are typed by - * `parameters`, which the Harness validates them against before `execute()`; `defineTool()` infers both generics. + * `parameters` (any JSON Schema object; see {@link ToolArgs}), which the Harness validates them against before + * `execute()`; `defineTool()` infers both generics. */ export type ToolRegistration< - TParameters extends TSchema = TSchema, + TParameters extends object = object, TDetails extends JsonValue = JsonValue, > = Tool & { /** Whether an interrupted execution may rerun on recovery. Default `unsafe`. */ @@ -350,12 +359,12 @@ export type ToolRegistration< * pure and must not mutate `args`: it runs again when a call is retried before its intent is recorded. Its result is * still validated against `parameters`. */ - prepareArguments?(args: unknown): Static; + prepareArguments?(args: unknown): ToolArgs; /** * Schema of `structuredOutput`, which programs that call the tool receive. Omitted: they receive the result's bounded * output, success or error, as `NestedToolExecutionResult.structuredOutput` describes. */ - readonly structuredOutputSchema?: TSchema; + readonly structuredOutputSchema?: object; /** * Who may call the tool: the model, other tools through `executeTool()`, or both (default). A code mode tool is * `["model"]`, so scripts cannot start scripts; tools behind code mode, such as MCP tools, can be `["tools"]`, so the @@ -373,7 +382,7 @@ export type ToolRegistration< readonly retain?: "head" | "tail"; }; execute( - args: Static, + args: ToolArgs, api: ToolExecutionApi, context: Context, ): Promise>; diff --git a/packages/durable/src/tool-search/tool-search.ts b/packages/durable/src/tool-search/tool-search.ts index 1a4607c17..5d79b007e 100644 --- a/packages/durable/src/tool-search/tool-search.ts +++ b/packages/durable/src/tool-search/tool-search.ts @@ -3,7 +3,7 @@ * and the matches are loaded with `control.addTools`, so the next request offers them. Loading writes the * conversation's `modelTools` (its `add`), so loaded tools stay offered on that branch, also after a compaction. */ -import { Type } from "@earendil-works/pi-ai"; +import { num, obj, opt, str } from "@earendil-works/pi-schema"; import { defineExtension, defineTool } from "../harness/define.ts"; import type { ToolRegistration } from "../harness/types.ts"; import { rankDocuments } from "../search.ts"; @@ -14,12 +14,13 @@ export const DEFAULT_TOOL_SEARCH_LIMIT = 8; /** Fixed, so it does not change while deferred tools are registered, for example as MCP servers connect. */ const TOOL_SEARCH_DESCRIPTION = `# Tool discovery\n\nSearches over deferred tool metadata with BM25 and exposes matching tools for the next model call.\n\nSome of the tools, such as tools of MCP servers, may not have been provided to you upfront, and you should use this tool (\`${TOOL_SEARCH_TOOL_NAME}\`) to search for the required tools. For MCP tool discovery, always use \`${TOOL_SEARCH_TOOL_NAME}\`.`; -const toolSearchSchema = Type.Object({ - query: Type.String({ description: "Search query for deferred tools." }), - limit: Type.Optional( - Type.Number({ description: `Maximum number of tools to return. Defaults to ${DEFAULT_TOOL_SEARCH_LIMIT}.` }), - ), -}); +const toolSearchSchema = obj( + { + query: str({ description: "Search query for deferred tools." }), + limit: opt(num({ description: `Maximum number of tools to return. Defaults to ${DEFAULT_TOOL_SEARCH_LIMIT}.` })), + }, + { open: true }, +); /** `tool_search` result details: the tools it loaded. */ export type ToolSearchToolDetails = { loaded: string[] }; diff --git a/packages/durable/src/tools/bash.ts b/packages/durable/src/tools/bash.ts index 21d1e1d03..211338f16 100644 --- a/packages/durable/src/tools/bash.ts +++ b/packages/durable/src/tools/bash.ts @@ -1,5 +1,5 @@ import type { Context } from "@earendil-works/chord"; -import { type Static, Type } from "typebox"; +import { bool, num, obj, opt, type Static, str } from "@earendil-works/pi-schema"; import { defineTool } from "../harness/define.ts"; import type { ToolExecutionApi, ToolExecutionResult, ToolRegistration } from "../harness/types.ts"; import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES } from "../truncate.ts"; @@ -7,23 +7,33 @@ import { requireEnv } from "./env.ts"; const MAX_TIMEOUT_SECONDS = 2_147_483_647 / 1000; -const bashSchema = Type.Object({ - command: Type.String({ description: "Bash command to execute" }), - timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (optional, no default timeout)" })), -}); +// Open objects, as the tools accepted extra arguments when their schemas were TypeBox's. +const bashSchema = obj( + { + command: str({ description: "Bash command to execute" }), + timeout: opt(num({ description: "Timeout in seconds (optional, no default timeout)" })), + }, + { open: true }, +); -const powershellSchema = Type.Object({ - command: Type.String({ description: "PowerShell command to execute" }), - timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (optional, no default timeout)" })), -}); +const powershellSchema = obj( + { + command: str({ description: "PowerShell command to execute" }), + timeout: opt(num({ description: "Timeout in seconds (optional, no default timeout)" })), + }, + { open: true }, +); /** What a program that calls `bash` or `powershell` receives. */ -const shellStructuredOutputSchema = Type.Object({ - output: Type.String({ description: "Combined stdout and stderr: the retained tail, as the model sees it" }), - truncated: Type.Boolean({ description: "Whether earlier output was dropped" }), - fullOutputPath: Type.Optional(Type.String({ description: "File with the complete output, when it was spilled" })), - exitCode: Type.Number(), -}); +const shellStructuredOutputSchema = obj( + { + output: str({ description: "Combined stdout and stderr: the retained tail, as the model sees it" }), + truncated: bool({ description: "Whether earlier output was dropped" }), + fullOutputPath: opt(str({ description: "File with the complete output, when it was spilled" })), + exitCode: num(), + }, + { open: true }, +); export type BashToolInput = Static; export type PowerShellToolInput = Static; diff --git a/packages/durable/src/tools/edit.ts b/packages/durable/src/tools/edit.ts index 802150152..e5b9be73d 100644 --- a/packages/durable/src/tools/edit.ts +++ b/packages/durable/src/tools/edit.ts @@ -1,4 +1,4 @@ -import { type Static, Type } from "typebox"; +import { arr, obj, type Static, str } from "@earendil-works/pi-schema"; import type { FileError } from "../env/index.ts"; import { defineTool } from "../harness/define.ts"; import type { ToolRegistration } from "../harness/types.ts"; @@ -16,21 +16,27 @@ import { requireEnv } from "./env.ts"; import { withFileMutationQueue } from "./file-mutation-queue.ts"; import { resolveToolPath } from "./path-utils.ts"; -const replaceEditSchema = Type.Object({ - oldText: Type.String({ - description: - "Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.", - }), - newText: Type.String({ description: "Replacement text for this targeted edit." }), -}); +const replaceEditSchema = obj( + { + oldText: str({ + description: + "Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.", + }), + newText: str({ description: "Replacement text for this targeted edit." }), + }, + { open: true }, +); -const editSchema = Type.Object({ - path: Type.String({ description: "Path to the file to edit (relative or absolute)" }), - edits: Type.Array(replaceEditSchema, { - description: - "One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.", - }), -}); +const editSchema = obj( + { + path: str({ description: "Path to the file to edit (relative or absolute)" }), + edits: arr(replaceEditSchema, { + description: + "One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.", + }), + }, + { open: true }, +); export type EditToolInput = Static; type LegacyEditToolInput = EditToolInput & { oldText?: unknown; newText?: unknown }; diff --git a/packages/durable/src/tools/read.ts b/packages/durable/src/tools/read.ts index b0312f3b3..de84a2b2f 100644 --- a/packages/durable/src/tools/read.ts +++ b/packages/durable/src/tools/read.ts @@ -1,6 +1,6 @@ import type { Context } from "@earendil-works/chord"; import { characterEnd } from "@earendil-works/chord/output"; -import { type Static, Type } from "typebox"; +import { lit, num, obj, opt, type Static, str, union } from "@earendil-works/pi-schema"; import { type BinaryReader, type FileInfo, @@ -31,11 +31,14 @@ import { } from "./image-processor.ts"; import { resolveReadToolPath } from "./path-utils.ts"; -const readSchema = Type.Object({ - path: Type.String({ description: "Path to the file to read (relative or absolute)" }), - offset: Type.Optional(Type.Number({ description: "Line number to start reading from (1-indexed)" })), - limit: Type.Optional(Type.Number({ description: "Maximum number of lines to read" })), -}); +const readSchema = obj( + { + path: str({ description: "Path to the file to read (relative or absolute)" }), + offset: opt(num({ description: "Line number to start reading from (1-indexed)" })), + limit: opt(num({ description: "Maximum number of lines to read" })), + }, + { open: true }, +); export type ReadToolInput = Static; @@ -43,9 +46,9 @@ export type ReadToolInput = Static; * What programs such as code mode scripts get: the text of a text file, or the image block of an image, which code * mode's `image()` shows. */ -const readStructuredOutputSchema = Type.Union([ - Type.String(), - Type.Object({ type: Type.Literal("image"), data: Type.String(), mimeType: Type.String() }), +const readStructuredOutputSchema = union([ + str(), + obj({ type: lit("image"), data: str(), mimeType: str() }, { open: true }), ]); export type ReadToolDetails = { diff --git a/packages/durable/src/tools/write.ts b/packages/durable/src/tools/write.ts index ee9f724cb..d8c804558 100644 --- a/packages/durable/src/tools/write.ts +++ b/packages/durable/src/tools/write.ts @@ -1,4 +1,4 @@ -import { type Static, Type } from "typebox"; +import { obj, type Static, str } from "@earendil-works/pi-schema"; import { getOrThrow } from "../env/index.ts"; import { defineTool } from "../harness/define.ts"; import type { ToolRegistration } from "../harness/types.ts"; @@ -6,10 +6,13 @@ import { requireEnv } from "./env.ts"; import { withFileMutationQueue } from "./file-mutation-queue.ts"; import { resolveToolPath } from "./path-utils.ts"; -const writeSchema = Type.Object({ - path: Type.String({ description: "Path to the file to write (relative or absolute)" }), - content: Type.String({ description: "Content to write to the file" }), -}); +const writeSchema = obj( + { + path: str({ description: "Path to the file to write (relative or absolute)" }), + content: str({ description: "Content to write to the file" }), + }, + { open: true }, +); export type WriteToolInput = Static; diff --git a/packages/durable/test/codemode-bench.ts b/packages/durable/test/codemode-bench.ts index c9c3c2b17..d79df3da6 100644 --- a/packages/durable/test/codemode-bench.ts +++ b/packages/durable/test/codemode-bench.ts @@ -5,6 +5,7 @@ * * node --conditions=source --experimental-strip-types test/codemode-bench.ts [N,...] */ + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -12,7 +13,7 @@ import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; import { CodemodeSandbox } from "@earendil-works/pi-codemode"; -import { Type } from "typebox"; +import { num, obj, str } from "@earendil-works/pi-schema"; import { createCodemodeTool } from "../src/codemode/codemode.ts"; import { createRegistry, defineExtension, defineTool, Harness } from "../src/index.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; @@ -22,14 +23,14 @@ type Shape = "sequential" | "parallel"; const echo = defineTool({ name: "echo", description: "echo", - parameters: Type.Object({ i: Type.Number() }), + parameters: obj({ i: num() }), execute: async (args) => ({ output: [{ type: "text", text: String(args.i) }] }), }); const plain = defineTool({ name: "plain", description: "plain", - parameters: Type.Object({ n: Type.Number(), shape: Type.String() }), + parameters: obj({ n: num(), shape: str() }), replay: "safe", execute: async (args, api, callContext) => { if (args.shape === "sequential") { diff --git a/packages/durable/test/codemode-describe.test.ts b/packages/durable/test/codemode-describe.test.ts index 99ced0465..85dfdbe99 100644 --- a/packages/durable/test/codemode-describe.test.ts +++ b/packages/durable/test/codemode-describe.test.ts @@ -1,7 +1,8 @@ -import { type AssistantMessage, fauxAssistantMessage, fauxText, Type } from "@earendil-works/pi-ai"; +import { type AssistantMessage, fauxAssistantMessage, fauxText } from "@earendil-works/pi-ai"; import { CodemodeSandbox } from "@earendil-works/pi-codemode"; import { CODEMODE_SOURCE_GRAMMAR } from "@earendil-works/pi-codemode/source"; import { defineTool, MemoryStorage, type ToolRegistration } from "@earendil-works/pi-durable"; +import { obj, str } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { createCodemodeExtension } from "../src/codemode/codemode.ts"; import { type CodemodeNamespace, codemodeDescription, codemodeSection } from "../src/codemode/describe.ts"; @@ -12,7 +13,7 @@ function tool(name: string, description: string, extra: Partial ({ output: [] }), ...extra, }); diff --git a/packages/durable/test/codemode.test.ts b/packages/durable/test/codemode.test.ts index e96b9b773..c12eba6d2 100644 --- a/packages/durable/test/codemode.test.ts +++ b/packages/durable/test/codemode.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import type { Context } from "@earendil-works/chord"; -import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { type CodemodeRun, CodemodeSandbox } from "@earendil-works/pi-codemode"; import { defineTool, @@ -13,6 +13,7 @@ import { ToolTask, type ToolTaskInput, } from "@earendil-works/pi-durable"; +import { bool, num, obj, opt, str } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { type CodemodeGlobal, @@ -30,7 +31,7 @@ import { addHooks, addTool } from "./harness-support.ts"; import { ControlledStorage, context } from "./session-support.ts"; import { aborted, type Deferred, deferred, eventually } from "./task-support.ts"; -const Parameters = Type.Object({ text: Type.Optional(Type.String()), id: Type.Optional(Type.Number()) }); +const Parameters = obj({ text: opt(str()), id: opt(num()) }); function tool( name: string, @@ -715,13 +716,13 @@ describe("durable code mode", () => { it("resolves a schema tool's error result that carries its value, and rejects one without with its text", async () => { const chat = codemodeChat(); - const Exit = Type.Object({ output: Type.String(), exitCode: Type.Number() }); + const Exit = obj({ output: str(), exitCode: num() }); addTool( chat.setup.registry, defineTool({ name: "run", description: "run", - parameters: Type.Object({}), + parameters: obj({}), structuredOutputSchema: Exit, // Like bash on a nonzero exit: an error result with its value. execute: async () => ({ @@ -736,7 +737,7 @@ describe("durable code mode", () => { defineTool({ name: "denied", description: "denied", - parameters: Type.Object({}), + parameters: obj({}), structuredOutputSchema: Exit, execute: async () => ({ output: [{ type: "text", text: "permission denied" }], isError: true }), }), @@ -767,7 +768,7 @@ describe("durable code mode", () => { defineTool({ name: "picture", description: "picture", - parameters: Type.Object({ caption: Type.Boolean() }), + parameters: obj({ caption: bool() }), execute: async (args) => ({ output: [ ...(args.caption ? [{ type: "text" as const, text: "a cat" }] : []), @@ -1078,7 +1079,7 @@ describe("durable code mode output and tool search", () => { defineTool({ name: "later", description: "A deferred tool", - parameters: Type.Object({}), + parameters: obj({}), deferred: true, execute: async () => ({ output: [{ type: "text", text: "later ran" }] }), }), diff --git a/packages/durable/test/examples/05-harness.ts b/packages/durable/test/examples/05-harness.ts index ad33c0d0c..e0fd947fb 100644 --- a/packages/durable/test/examples/05-harness.ts +++ b/packages/durable/test/examples/05-harness.ts @@ -1,9 +1,10 @@ // Open a Harness with a registry of extensions. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/05-harness.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; +import { obj, str } from "@earendil-works/pi-schema"; import { AgentDoc, createRegistry, @@ -23,7 +24,7 @@ const context = BACKGROUND_CONTEXT; const read = defineTool({ name: "read", description: "Read a file", - parameters: Type.Object({ path: Type.String() }), + parameters: obj({ path: str() }), execute: async (args) => ({ output: [{ type: "text", text: `contents of ${args.path}` }] }), }); const Files = defineExtension({ name: "files", tools: [read] }); diff --git a/packages/durable/test/examples/06-configuration.ts b/packages/durable/test/examples/06-configuration.ts index 03184b6d8..fb50401db 100644 --- a/packages/durable/test/examples/06-configuration.ts +++ b/packages/durable/test/examples/06-configuration.ts @@ -1,9 +1,10 @@ // Per-conversation agent choices and Harness-wide settings. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/06-configuration.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; +import { obj, str } from "@earendil-works/pi-schema"; import { AgentDoc, createRegistry, @@ -25,7 +26,7 @@ function exampleTool(name: string, description: string): AppTool { const tool = defineTool({ name, description, - parameters: Type.Object({ path: Type.String() }), + parameters: obj({ path: str() }), execute: async (args) => ({ output: [{ type: "text", text: `${name} ${args.path}` }] }), }); return { ...tool, snippet: `Use ${name} for files.` }; diff --git a/packages/durable/test/examples/09-registry-reload.ts b/packages/durable/test/examples/09-registry-reload.ts index 4135650ee..e1ee8319e 100644 --- a/packages/durable/test/examples/09-registry-reload.ts +++ b/packages/durable/test/examples/09-registry-reload.ts @@ -1,9 +1,10 @@ // Reload extension code through the registry. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/09-registry-reload.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; +import { obj, str } from "@earendil-works/pi-schema"; import { createRegistry, defineExtension, defineTool, Harness, MemoryStorage, wrapTool } from "../../src/index.ts"; const context = BACKGROUND_CONTEXT; @@ -12,7 +13,7 @@ function exampleTool(name: string, description: string) { return defineTool({ name, description, - parameters: Type.Object({ path: Type.String() }), + parameters: obj({ path: str() }), execute: async (args) => ({ output: [{ type: "text", text: `${name} ${args.path}` }] }), }); } diff --git a/packages/durable/test/examples/10-extension-state.ts b/packages/durable/test/examples/10-extension-state.ts index f2d518214..bfd6613e7 100644 --- a/packages/durable/test/examples/10-extension-state.ts +++ b/packages/durable/test/examples/10-extension-state.ts @@ -1,10 +1,11 @@ // An extension that keeps its own per-conversation document. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/10-extension-state.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { obj, str } from "@earendil-works/pi-schema"; import { createRegistry, defineDoc, @@ -38,7 +39,7 @@ const Todo = defineExtension({ defineTool({ name: "todo", description: "Add an item to your todo list", - parameters: Type.Object({ item: Type.String() }), + parameters: obj({ item: str() }), execute: async (args, api, callContext) => { await api.commit(async (tx) => { (await tx.doc(Todos, api.conversationId)).items.push(args.item); diff --git a/packages/durable/test/examples/20-late-join.ts b/packages/durable/test/examples/20-late-join.ts index 73b53076b..59eb2d0d1 100644 --- a/packages/durable/test/examples/20-late-join.ts +++ b/packages/durable/test/examples/20-late-join.ts @@ -2,10 +2,11 @@ // view, and then only what changes after that. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/20-late-join.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { obj } from "@earendil-works/pi-schema"; import { createRegistry, defineExtension, @@ -27,7 +28,7 @@ registry.install( defineTool({ name: "count", description: "Counts to ten", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api) => { for (let n = 1; n <= 10; n++) { api.output(`${n}\n`); diff --git a/packages/durable/test/examples/21-subagent-foreground.ts b/packages/durable/test/examples/21-subagent-foreground.ts index af4a31823..7f624a018 100644 --- a/packages/durable/test/examples/21-subagent-foreground.ts +++ b/packages/durable/test/examples/21-subagent-foreground.ts @@ -4,12 +4,14 @@ // Uses OpenAI when OPENAI_API_KEY is set, and a scripted faux model otherwise. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/21-subagent-foreground.ts + import type { Context } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { type AssistantMessage, type FauxResponseStep, Type } from "@earendil-works/pi-ai"; +import type { AssistantMessage, FauxResponseStep } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; import { openaiProvider } from "@earendil-works/pi-ai/providers/openai"; +import { obj, str } from "@earendil-works/pi-schema"; import { AssistantEntry, type ConversationId, @@ -42,7 +44,7 @@ const Subagent: Extension = defineExtension({ defineTool({ name: "subagent", description: "Delegate a self-contained task to a subagent and get its answer back.", - parameters: Type.Object({ task: Type.String({ description: "What the subagent should do" }) }), + parameters: obj({ task: str({ description: "What the subagent should do" }) }), // Safe to rerun after a crash: a rerun finds the child it already created and the submission it already made. replay: "safe", execute: async (args, api, callContext) => { diff --git a/packages/durable/test/examples/22-subagent-background.ts b/packages/durable/test/examples/22-subagent-background.ts index 7aaf64d8b..f01ee89b4 100644 --- a/packages/durable/test/examples/22-subagent-background.ts +++ b/packages/durable/test/examples/22-subagent-background.ts @@ -5,14 +5,16 @@ // Uses OpenAI when OPENAI_API_KEY is set, and a scripted faux model otherwise. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/22-subagent-background.ts + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { type AssistantMessage, type FauxResponseStep, Type } from "@earendil-works/pi-ai"; +import type { AssistantMessage, FauxResponseStep } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; import { openaiProvider } from "@earendil-works/pi-ai/providers/openai"; +import { bool, obj, oneOf, opt, str } from "@earendil-works/pi-schema"; import { AssistantEntry, type ConversationId, @@ -134,11 +136,11 @@ const subagentTool = defineTool({ "Manage persistent subagents that work in the background. Actions: spawn (name, message), send (name, message; " + "followUp: true queues it after the current answer instead of steering), stop (name: aborts its current work), " + "status (name, or all subagents without one). Answers are reported back to you when they arrive.", - parameters: Type.Object({ - action: Type.Union([Type.Literal("spawn"), Type.Literal("send"), Type.Literal("stop"), Type.Literal("status")]), - name: Type.Optional(Type.String()), - message: Type.Optional(Type.String()), - followUp: Type.Optional(Type.Boolean()), + parameters: obj({ + action: oneOf(["spawn", "send", "stop", "status"]), + name: opt(str()), + message: opt(str()), + followUp: opt(bool()), }), // A call interrupted by a crash is not rerun: repeating `stop` could stop newer work. The model sees that the call // was interrupted and can check with `status`. diff --git a/packages/durable/test/examples/26-plan-mode.ts b/packages/durable/test/examples/26-plan-mode.ts index b0bedd056..0eaf5b939 100644 --- a/packages/durable/test/examples/26-plan-mode.ts +++ b/packages/durable/test/examples/26-plan-mode.ts @@ -2,13 +2,14 @@ // extension's own document, and switching back restores the full tool set. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/26-plan-mode.ts + import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { arr, obj, str } from "@earendil-works/pi-schema"; import { NodeExecutionEnv } from "../../src/env/node.ts"; import { createRegistry, @@ -39,7 +40,7 @@ const PlanDoc = defineDoc<{ steps: string[] }>({ const submitPlan = defineTool({ name: "submit_plan", description: "Submit the plan as a list of steps.", - parameters: Type.Object({ steps: Type.Array(Type.String()) }), + parameters: obj({ steps: arr(str()) }), execute: async (args, api, callContext) => { await api.commit(async (tx) => { (await tx.doc(PlanDoc, api.conversationId)).steps = args.steps; diff --git a/packages/durable/test/examples/30-reload-and-restart.ts b/packages/durable/test/examples/30-reload-and-restart.ts index 6813a7659..4918cea2f 100644 --- a/packages/durable/test/examples/30-reload-and-restart.ts +++ b/packages/durable/test/examples/30-reload-and-restart.ts @@ -2,13 +2,15 @@ // with; stored choices are names, so they survive a restart and bind to whatever code the new process installs. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/30-reload-and-restart.ts + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { type ToolResultMessage, Type } from "@earendil-works/pi-ai"; +import type { ToolResultMessage } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { obj } from "@earendil-works/pi-schema"; import { createRegistry, defineExtension, defineTool, Harness, ToolResultEntry } from "../../src/index.ts"; import { openNodeSqliteStorage } from "../../src/storage/sqlite/node.ts"; @@ -24,7 +26,7 @@ function loadVersioned(version: string) { defineTool({ name: "version", description: "Report the tool's code version", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { started(); await gate; diff --git a/packages/durable/test/examples/32-tests-after-edits.ts b/packages/durable/test/examples/32-tests-after-edits.ts index 32292c521..240bfec31 100644 --- a/packages/durable/test/examples/32-tests-after-edits.ts +++ b/packages/durable/test/examples/32-tests-after-edits.ts @@ -2,10 +2,11 @@ // tests, a failure starts a fix run, and input sent meanwhile waits for them too. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/32-tests-after-edits.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { obj, str } from "@earendil-works/pi-schema"; import { createRegistry, defineExtension, @@ -34,7 +35,7 @@ const runTests = () => { const edit = defineTool({ name: "edit", description: "Replace a file's content", - parameters: Type.Object({ path: Type.String(), content: Type.String() }), + parameters: obj({ path: str(), content: str() }), execute: async (args) => { files.set(args.path, args.content); return { output: [{ type: "text", text: `wrote ${args.path}` }] }; diff --git a/packages/durable/test/examples/33-turn-limit-and-continue.ts b/packages/durable/test/examples/33-turn-limit-and-continue.ts index cac9d315c..817bd1fcb 100644 --- a/packages/durable/test/examples/33-turn-limit-and-continue.ts +++ b/packages/durable/test/examples/33-turn-limit-and-continue.ts @@ -2,10 +2,11 @@ // message, with a fresh budget. A host learns that a run stopped from `pi.live.runFailure`. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/33-turn-limit-and-continue.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { num, obj } from "@earendil-works/pi-schema"; import { ConversationBusy, createRegistry, @@ -21,7 +22,7 @@ const context = BACKGROUND_CONTEXT; const step = defineTool({ name: "step", description: "Do one step of the work", - parameters: Type.Object({ n: Type.Number() }), + parameters: obj({ n: num() }), execute: async (args) => ({ output: [{ type: "text", text: `step ${args.n} done` }] }), }); const call = (n: number) => fauxAssistantMessage(fauxToolCall("step", { n }), { stopReason: "toolUse" }); diff --git a/packages/durable/test/examples/34-submit-in-commit.ts b/packages/durable/test/examples/34-submit-in-commit.ts index 291305cc6..5cdfa918f 100644 --- a/packages/durable/test/examples/34-submit-in-commit.ts +++ b/packages/durable/test/examples/34-submit-in-commit.ts @@ -3,10 +3,11 @@ // state; `whenIdle: "reject"` refuses it when no run is working. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/34-submit-in-commit.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { obj } from "@earendil-works/pi-schema"; import { ConversationIdle, createRegistry, @@ -38,7 +39,7 @@ const working = new Promise((resolve) => { const build = defineTool({ name: "build", description: "Build the project", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { await working; return { output: [{ type: "text", text: "built" }] }; diff --git a/packages/durable/test/harness-after-run.test.ts b/packages/durable/test/harness-after-run.test.ts index 373922122..7201fe6af 100644 --- a/packages/durable/test/harness-after-run.test.ts +++ b/packages/durable/test/harness-after-run.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { Draft, JsonValue } from "@earendil-works/chord"; -import { fauxAssistantMessage, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai"; import { CompactionTask, ConversationBusy, @@ -25,6 +25,7 @@ import { type TaskId, type Tx, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { endRun } from "../src/harness/live.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; @@ -132,7 +133,7 @@ function install(setup: ChatSetup, state: Probe, afterRun?: GenerationHooks["aft return Tests; } -const Parameters = Type.Object({}); +const Parameters = obj({}); const tools = defineExtension({ name: "tools", tools: [ diff --git a/packages/durable/test/harness-callers.test.ts b/packages/durable/test/harness-callers.test.ts index ff9a8311e..bbf4e7db9 100644 --- a/packages/durable/test/harness-callers.test.ts +++ b/packages/durable/test/harness-callers.test.ts @@ -1,4 +1,4 @@ -import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { AgentDoc, type Conversation, @@ -7,6 +7,7 @@ import { type NestedToolExecutionResult, type ToolRegistration, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; import { addTool } from "./harness-support.ts"; @@ -16,7 +17,7 @@ function tool(name: string, extra: Partial = {}): ToolRegistra return defineTool({ name, description: name, - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [{ type: "text", text: `${name} ran` }] }), ...extra, }); @@ -34,7 +35,7 @@ function prober(names: readonly string[]) { const registration = defineTool({ name: "probe", description: "probe", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { for (const name of names) received.set(name, await api.executeTool(name, {}, callContext)); return {}; diff --git a/packages/durable/test/harness-compaction.test.ts b/packages/durable/test/harness-compaction.test.ts index b60eb2dae..1be32ae35 100644 --- a/packages/durable/test/harness-compaction.test.ts +++ b/packages/durable/test/harness-compaction.test.ts @@ -9,7 +9,6 @@ import { type Message, type SimpleStreamOptions, type TranscriptContext, - Type, type UserMessage, } from "@earendil-works/pi-ai"; import { @@ -33,6 +32,7 @@ import { type TaskId, UsageDoc, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { selectCut, serializeConversation } from "../src/harness/compaction.ts"; import { orderToolResults } from "../src/harness/context.ts"; @@ -471,7 +471,7 @@ describe("manual compaction", () => { defineTool({ name: "wait", description: "wait", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [{ type: "text", text: "waited" }] }), }), ); @@ -886,7 +886,7 @@ describe("compaction outcomes", () => { defineTool({ name: "read", description: "read", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), }), ); @@ -1343,7 +1343,7 @@ describe("compaction estimates and interactions", () => { defineTool({ name: "slow", description: "slow", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { toolReached.resolve(); await toolGate.promise; diff --git a/packages/durable/test/harness-continue.test.ts b/packages/durable/test/harness-continue.test.ts index 8b609b8eb..627e6218f 100644 --- a/packages/durable/test/harness-continue.test.ts +++ b/packages/durable/test/harness-continue.test.ts @@ -1,4 +1,5 @@ // `submit({ type: "continue" })`: a run without a new user message (#11043). + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -10,7 +11,6 @@ import { fauxText, fauxToolCall, type Message, - Type, } from "@earendil-works/pi-ai"; import { AssistantEntry, @@ -26,6 +26,7 @@ import { type Storage, UserEntry, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; @@ -125,7 +126,7 @@ describe("continue", () => { defineTool({ name: "wait", description: "Waits until aborted", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, _api, callContext) => { started.resolve(); return aborted(callContext.abortSignal!); @@ -312,7 +313,7 @@ describe("continue", () => { defineTool({ name: "delegate", description: "Delegates", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { const child = await api.commit(async (tx) => { const created = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); diff --git a/packages/durable/test/harness-deferred-progress.test.ts b/packages/durable/test/harness-deferred-progress.test.ts index 9506ab931..07116cc08 100644 --- a/packages/durable/test/harness-deferred-progress.test.ts +++ b/packages/durable/test/harness-deferred-progress.test.ts @@ -2,8 +2,9 @@ // the next commit that reads or writes `pi.live` or within `settings.progress.persistWithinMs`. Crashes are simulated // by failing the Harness: from that moment nothing more reaches Storage, as when the process dies; the reopened Harness // resumes from what was stored. + import { applyImmutable } from "@earendil-works/chord/delta"; -import { fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { AssistantEntry, type ConversationId, @@ -15,6 +16,7 @@ import { type LiveState, ToolResultEntry, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import type { PartialMessage } from "../src/harness/partial-changes.ts"; import type { SessionImpl } from "../src/session/session.ts"; @@ -257,7 +259,7 @@ describe("tool output", () => { const registration = defineTool({ name: "work", description: "work", - parameters: Type.Object({}), + parameters: obj({}), replay, execute: async (_args, api, callContext) => { const run = ++state.runs; @@ -306,7 +308,7 @@ describe("tool output", () => { defineTool({ name: "work", description: "work", - parameters: Type.Object({}), + parameters: obj({}), replay: "safe", execute: async (_args, api, callContext) => { runs++; diff --git a/packages/durable/test/harness-host-inputs.test.ts b/packages/durable/test/harness-host-inputs.test.ts index 4719cea94..4922cd0d4 100644 --- a/packages/durable/test/harness-host-inputs.test.ts +++ b/packages/durable/test/harness-host-inputs.test.ts @@ -6,7 +6,6 @@ import { fauxToolCall, type Models, type SimpleStreamOptions, - Type, } from "@earendil-works/pi-ai"; import { type Conversation, @@ -19,6 +18,7 @@ import { MemoryStorage, type Registry, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { allEntries, textOf } from "./chat-support.ts"; import { context } from "./session-support.ts"; @@ -86,7 +86,7 @@ describe("HarnessOptions.models and settings as functions", () => { defineTool({ name: "probe", description: "Records api.models before and after the host swaps it", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api) => { seen.push(api.models); current = second; diff --git a/packages/durable/test/harness-inbox.test.ts b/packages/durable/test/harness-inbox.test.ts index 0062b197a..a4cb91837 100644 --- a/packages/durable/test/harness-inbox.test.ts +++ b/packages/durable/test/harness-inbox.test.ts @@ -9,7 +9,6 @@ import { fauxText, fauxToolCall, type ToolResultMessage, - Type, type Usage, } from "@earendil-works/pi-ai"; import { @@ -29,6 +28,7 @@ import { ToolTask, UsageDoc, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { recordUsage } from "../src/harness/usage.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; @@ -67,7 +67,7 @@ function holdTool(setup: ChatSetup, gate: Deferred, result: ToolExecutionResult defineTool({ name: "hold", description: "Waits for the test", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { await gate.promise; return result; @@ -474,7 +474,7 @@ describe("inbox", () => { defineTool({ name: "hold", description: "Hands off", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [], control: { handoff: `h${++handoffs}` } }), }), ); @@ -796,7 +796,7 @@ describe("inbox", () => { defineTool({ name: "later", description: "Finishes first", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { await secondGate.promise; return { output: [], control: { handoff: "two" } }; diff --git a/packages/durable/test/harness-input-entries.test.ts b/packages/durable/test/harness-input-entries.test.ts index f0c44b9fb..ce6baa468 100644 --- a/packages/durable/test/harness-input-entries.test.ts +++ b/packages/durable/test/harness-input-entries.test.ts @@ -1,4 +1,5 @@ // Inputs placed as application entries, and data-only writes appended at once (#10407). + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -9,7 +10,6 @@ import { fauxText, fauxToolCall, type Context as ModelContext, - Type, } from "@earendil-works/pi-ai"; import { type Conversation, @@ -23,6 +23,7 @@ import { MemoryStorage, type SubmissionDraft, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, type ChatSetup, chatSetup, openChat, textOf, waitFor } from "./chat-support.ts"; @@ -54,7 +55,7 @@ function holdTool(setup: ChatSetup, gate: Deferred): void { defineTool({ name: "hold", description: "Waits for the test", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { await gate.promise; return { output: [{ type: "text", text: "held" }] }; diff --git a/packages/durable/test/harness-lifecycle.test.ts b/packages/durable/test/harness-lifecycle.test.ts index e41cc259d..bf044120b 100644 --- a/packages/durable/test/harness-lifecycle.test.ts +++ b/packages/durable/test/harness-lifecycle.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import type { Context, JsonValue } from "@earendil-works/chord"; import { withAbortSignal } from "@earendil-works/chord/context"; -import { createModels, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { createModels, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { type Conversation, type ConversationId, @@ -17,6 +17,7 @@ import { type Storage, type TaskId, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import type { SessionImpl } from "../src/session/session.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; @@ -168,7 +169,7 @@ describe("Harness close", () => { defineTool({ name: "wait", description: "wait", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { if (where === "tool") await stubborn(); return { output: [] }; diff --git a/packages/durable/test/harness-live-deltas.test.ts b/packages/durable/test/harness-live-deltas.test.ts index 14d3dd67b..93ac8187c 100644 --- a/packages/durable/test/harness-live-deltas.test.ts +++ b/packages/durable/test/harness-live-deltas.test.ts @@ -1,5 +1,5 @@ import type { Op } from "@earendil-works/chord/delta"; -import { fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { type CommitPublication, defineTool, @@ -10,6 +10,7 @@ import { type ToolExecutionApi, type ToolRegistration, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { chatSetup, openChat, waitFor } from "./chat-support.ts"; import { addTool } from "./harness-support.ts"; @@ -30,7 +31,7 @@ async function drive(outputLimits: ToolRegistration["outputLimits"] = {}) { defineTool({ name: "drive", description: "Driven by the test", - parameters: Type.Object({}), + parameters: obj({}), outputLimits, execute: async (_args, api, callContext) => { for (;;) { @@ -104,7 +105,7 @@ describe("pi.live deltas", () => { defineTool({ name: "noop", description: "noop", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), }), ); @@ -317,7 +318,7 @@ describe("pi.live deltas", () => { defineTool({ name, description: name, - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api) => { api.output(`${name} output\n`); await new Promise((resolve) => setTimeout(resolve, 150)); @@ -368,7 +369,7 @@ describe("pi.live deltas", () => { defineTool({ name: "bad", description: "bad", - parameters: Type.Object({}), + parameters: obj({}), // Not strict JSON: the result commit throws and the scheduler faults the task. execute: async () => ({ output: [], details: { fn: (() => 1) as never } }), }), @@ -412,7 +413,7 @@ describe("pi.live deltas", () => { defineTool({ name: "noop", description: "noop", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), }), ); diff --git a/packages/durable/test/harness-nested-tools-restart.test.ts b/packages/durable/test/harness-nested-tools-restart.test.ts index 0eb29ff76..8d80aa903 100644 --- a/packages/durable/test/harness-nested-tools-restart.test.ts +++ b/packages/durable/test/harness-nested-tools-restart.test.ts @@ -8,7 +8,6 @@ import { fauxText, fauxToolCall, type ToolResultMessage, - Type, } from "@earendil-works/pi-ai"; import { defineExtension, @@ -23,6 +22,7 @@ import { ToolTask, type ToolTaskInput, } from "@earendil-works/pi-durable"; +import { obj, opt, str } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; @@ -31,7 +31,7 @@ import { nestedResultOf, nestedResults, viewHistory } from "./live-support.ts"; import { context } from "./session-support.ts"; import { aborted, deferred, eventually } from "./task-support.ts"; -const Parameters = Type.Object({ text: Type.Optional(Type.String()) }); +const Parameters = obj({ text: opt(str()) }); function tool( name: string, diff --git a/packages/durable/test/harness-nested-tools.test.ts b/packages/durable/test/harness-nested-tools.test.ts index 31325b395..66de32766 100644 --- a/packages/durable/test/harness-nested-tools.test.ts +++ b/packages/durable/test/harness-nested-tools.test.ts @@ -10,7 +10,6 @@ import { fauxText, fauxToolCall, type ToolResultMessage, - Type, type Usage, } from "@earendil-works/pi-ai"; import { @@ -33,6 +32,7 @@ import { type ToolTaskResult, UsageDoc, } from "@earendil-works/pi-durable"; +import { obj, opt, str } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { commitWrites } from "../src/testing/commit-writes.ts"; @@ -42,7 +42,7 @@ import { endedAt, nestedResultOf, nestedResults, slotsOf, startOrder, viewHistor import { context, onValue } from "./session-support.ts"; import { aborted, deferred, eventually } from "./task-support.ts"; -const Parameters = Type.Object({ text: Type.Optional(Type.String()) }); +const Parameters = obj({ text: opt(str()) }); type Tool = ToolRegistration; function tool(name: string, execute: Tool["execute"], extra: Partial = {}): Tool { @@ -329,7 +329,7 @@ describe("nested tool calls", () => { defineTool({ name: "pair", description: "pair", - parameters: Type.Object({ a: Type.String(), b: Type.String() }), + parameters: obj({ a: str(), b: str() }), execute: async (args) => { runs++; return { output: [{ type: "text", text: `${args.a}${args.b}` }] }; diff --git a/packages/durable/test/harness-ownership.test.ts b/packages/durable/test/harness-ownership.test.ts index 6ba3ad8d9..d0251e24c 100644 --- a/packages/durable/test/harness-ownership.test.ts +++ b/packages/durable/test/harness-ownership.test.ts @@ -1,7 +1,7 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { type Conversation, type ConversationHandle, @@ -20,6 +20,7 @@ import { type Submission, type TaskId, } from "@earendil-works/pi-durable"; +import { obj, str } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { chatSetup, openChat } from "./chat-support.ts"; @@ -1019,7 +1020,7 @@ describe("owned conversations from tools and supervisors", () => { defineTool({ name: "delegate", description: "Delegates", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { const child = await api.commit(async (tx) => { const created = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); @@ -1074,7 +1075,7 @@ describe("owned conversations from tools and supervisors", () => { defineTool({ name: "delegate", description: "Delegates late", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { const child = await api.commit(async (tx) => { const created = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); @@ -1159,7 +1160,7 @@ describe("owned conversations from tools and supervisors", () => { defineTool({ name: "delegate", description: "Delegates", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { child = await api.commit(async (tx) => { const created = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); @@ -1208,7 +1209,7 @@ describe("owned conversations from tools and supervisors", () => { defineTool({ name: "delegate", description: "Delegates and cancels", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { const child = await api.commit(async (tx) => { const created = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); @@ -1247,7 +1248,7 @@ describe("owned conversations from tools and supervisors", () => { defineTool({ name: "spawn", description: "Starts owned work, then throws or hangs", - parameters: Type.Object({ mode: Type.String() }), + parameters: obj({ mode: str() }), execute: async (args, api, callContext) => { const child = await api.commit(async (tx) => { const created = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); @@ -1305,7 +1306,7 @@ describe("owned conversations from tools and supervisors", () => { defineTool({ name: "subagent", description: "Delegates", - parameters: Type.Object({}), + parameters: obj({}), replay: "safe", execute: async (_args, api, callContext) => { const child = await api.commit(async (tx) => { diff --git a/packages/durable/test/harness-prompt.test.ts b/packages/durable/test/harness-prompt.test.ts index 40ef2e09e..39affbc58 100644 --- a/packages/durable/test/harness-prompt.test.ts +++ b/packages/durable/test/harness-prompt.test.ts @@ -1,4 +1,4 @@ -import { type SystemMessage, type Tool, Type } from "@earendil-works/pi-ai"; +import type { SystemMessage, Tool } from "@earendil-works/pi-ai"; import { getCurrentTools, toToolDeclaration } from "@earendil-works/pi-ai/utils/transcript"; import { type Conversation, @@ -12,6 +12,7 @@ import { type ToolRegistration, wrapSection, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { resolveAgent, resolveSettings } from "../src/harness/agent.ts"; import { planSystemEntries, renderSections, replaySections } from "../src/harness/prompt.ts"; @@ -207,7 +208,7 @@ describe("tool loadout preparation", () => { const declaration = (name: string, description = name): Tool => ({ name, description, - parameters: Type.Object({}), + parameters: obj({}), }); /** Plan tools only, append the plan, check that replay offers `tools` in order, and return each message's changes. */ diff --git a/packages/durable/test/harness-registry.test.ts b/packages/durable/test/harness-registry.test.ts index af852b0c1..b51002576 100644 --- a/packages/durable/test/harness-registry.test.ts +++ b/packages/durable/test/harness-registry.test.ts @@ -1,4 +1,3 @@ -import { Type } from "@earendil-works/pi-ai"; import { type AgentState, CompactionTask, @@ -15,6 +14,7 @@ import { wrapSection, wrapTool, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { agentHooks, resolveAgent, resolveSettings } from "../src/harness/agent.ts"; import { context } from "./session-support.ts"; @@ -25,7 +25,7 @@ function tool(name: string, extra: Partial = {}): AppTool { return { name, description: `${name} tool`, - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), ...extra, }; diff --git a/packages/durable/test/harness-reports.test.ts b/packages/durable/test/harness-reports.test.ts index 623fafcdb..97a7acf51 100644 --- a/packages/durable/test/harness-reports.test.ts +++ b/packages/durable/test/harness-reports.test.ts @@ -1,4 +1,4 @@ -import { createModels, fauxAssistantMessage, Type } from "@earendil-works/pi-ai"; +import { createModels, fauxAssistantMessage } from "@earendil-works/pi-ai"; import { type AnyTask, createRegistry, @@ -14,6 +14,7 @@ import { section, wrapTool, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { chatSetup } from "./chat-support.ts"; import { ControlledStorage, context } from "./session-support.ts"; @@ -189,7 +190,7 @@ describe("reports", () => { const echo = defineTool({ name: "echo", description: "echo", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({}), }); registry.install(defineExtension({ name: "tools", tools: [echo] })); diff --git a/packages/durable/test/harness-restart-failure.test.ts b/packages/durable/test/harness-restart-failure.test.ts index 3ae60d042..c8ece30c7 100644 --- a/packages/durable/test/harness-restart-failure.test.ts +++ b/packages/durable/test/harness-restart-failure.test.ts @@ -1,6 +1,7 @@ // What a crash (a host that dies, is evicted, or is reset) does to work that was running: a generation's interrupted // request is a failed attempt under the retry policy, and every other task ends visibly once it was cut off // `settings.restartLimit` times in a row (#11139). + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -13,7 +14,6 @@ import { type Message, type Models, type ToolResultMessage, - Type, } from "@earendil-works/pi-ai"; import { defineTool, @@ -27,6 +27,7 @@ import { type SubmissionId, type TaskId, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { commitWrites } from "../src/testing/commit-writes.ts"; @@ -282,7 +283,7 @@ describe("a generation interrupted while the model answers (#11139)", () => { defineTool({ name: "hold", description: "Waits", - parameters: Type.Object({}), + parameters: obj({}), replay: "safe", execute: async () => { started.resolve(); @@ -375,7 +376,7 @@ describe("other tasks cut off restartLimit times in a row (#11139)", () => { defineTool({ name: "heavy", description: "Never finishes", - parameters: Type.Object({}), + parameters: obj({}), replay: "safe", execute: async (_args, api, ctx) => { runs++; @@ -751,7 +752,7 @@ describe("the restart limit on the normal path (#11139)", () => { defineTool({ name: "t", description: "t", - parameters: Type.Object({}), + parameters: obj({}), replay: "safe", execute: async () => ({}), }), diff --git a/packages/durable/test/harness-run-failure.test.ts b/packages/durable/test/harness-run-failure.test.ts index ded8c8ebc..638a4715b 100644 --- a/packages/durable/test/harness-run-failure.test.ts +++ b/packages/durable/test/harness-run-failure.test.ts @@ -1,5 +1,6 @@ // `pi.live.runFailure`: why the newest run ended without an answer, as state a renderer shows (#11327). -import { type FauxResponseStep, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; + +import { type FauxResponseStep, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { type AfterFailedRun, type Conversation, @@ -10,6 +11,7 @@ import { MemoryStorage, type RunFailure, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { chatSetup, openChat, unanswered } from "./chat-support.ts"; import { addTool } from "./harness-support.ts"; @@ -104,7 +106,7 @@ describe("pi.live.runFailure", () => { defineTool({ name: "hold", description: "Waits", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { started.resolve(); await gate.promise; diff --git a/packages/durable/test/harness-structured-output.test.ts b/packages/durable/test/harness-structured-output.test.ts index 829932323..5ef6f75ba 100644 --- a/packages/durable/test/harness-structured-output.test.ts +++ b/packages/durable/test/harness-structured-output.test.ts @@ -8,7 +8,6 @@ import { fauxText, fauxToolCall, type ToolResultMessage, - Type, } from "@earendil-works/pi-ai"; import { defineTool, @@ -21,6 +20,7 @@ import { ToolResultEntry, ToolTask, } from "@earendil-works/pi-durable"; +import { nul, num, obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { NodeExecutionEnv } from "../src/env/node.ts"; import { createBashTool } from "../src/tools/index.ts"; @@ -28,14 +28,14 @@ import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support. import { addHooks, addTool } from "./harness-support.ts"; import { context } from "./session-support.ts"; -const Count = Type.Object({ count: Type.Number() }); +const Count = obj({ count: num() }); /** A tool that returns `result`, with `extra` such as a schema. */ function returning(name: string, result: ToolExecutionResult, extra: Partial = {}): ToolRegistration { return defineTool({ name, description: name, - parameters: Type.Object({}), + parameters: obj({}), execute: async () => result, ...extra, }); @@ -55,7 +55,7 @@ async function nested(setup: ChatSetup, name: string): Promise { received = await api.executeTool(name, {}, callContext); return {}; @@ -132,17 +132,14 @@ describe("structured output", () => { it("stores exactly what callers and events get, without the model's output, and keeps a stored null", async () => { const setup = chatSetup(); addTool(setup.registry, returning("plain", { output: [{ type: "text", text: "a" }] })); - addTool( - setup.registry, - returning("nothing", { structuredOutput: null }, { structuredOutputSchema: Type.Null() }), - ); + addTool(setup.registry, returning("nothing", { structuredOutput: null }, { structuredOutputSchema: nul() })); const received: NestedToolExecutionResult[] = []; addTool( setup.registry, defineTool({ name: "probe", description: "probe", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { received.push(await api.executeTool("plain", {}, callContext)); received.push(await api.executeTool("nothing", {}, callContext)); @@ -272,7 +269,7 @@ describe("structured output", () => { defineTool({ name: "throws", description: "throws", - parameters: Type.Object({}), + parameters: obj({}), structuredOutputSchema: Count, execute: async () => { throw new Error("permission denied"); @@ -344,7 +341,7 @@ describe("structured output", () => { defineTool({ name: "probe", description: "probe", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { plain = await api.executeTool("plain", {}, callContext); counter = await api.executeTool("counter", {}, callContext); @@ -375,7 +372,7 @@ describe("structured output", () => { defineTool({ name: "probe", description: "probe", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { received.push(await api.executeTool("bash", { command: "printf ok" }, callContext)); received.push(await api.executeTool("bash", { command: "printf bad; exit 4" }, callContext)); diff --git a/packages/durable/test/harness-structured.test.ts b/packages/durable/test/harness-structured.test.ts index 78d4e9e76..26c7d7fb6 100644 --- a/packages/durable/test/harness-structured.test.ts +++ b/packages/durable/test/harness-structured.test.ts @@ -8,7 +8,6 @@ import { fauxToolCall, type Models, type ToolResultMessage, - Type, } from "@earendil-works/pi-ai"; import { type Conversation, @@ -35,6 +34,7 @@ import { ToolResultEntry, type Tx, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, chatSetup, openChat, waitFor } from "./chat-support.ts"; @@ -1192,7 +1192,7 @@ function blockingTool(name: string): { started: Promise; registration: Too registration: defineTool({ name, description: `The ${name} tool`, - parameters: Type.Object({}), + parameters: obj({}), executionMode: "sequential", execute: async (_args, api, callContext) => { api.output("partial"); @@ -1218,7 +1218,7 @@ describe("tool rounds", () => { defineTool({ name: "noop", description: "Does nothing", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), }), ); @@ -1298,7 +1298,7 @@ describe("tool rounds", () => { defineTool({ name: "noop", description: "Does nothing", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), }), ); @@ -1826,7 +1826,7 @@ describe("tool rounds as the view shows them", () => { defineTool({ name: "noop", description: "Does nothing", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), }), ); @@ -1866,7 +1866,7 @@ describe("tool rounds as the view shows them", () => { defineTool({ name: "delegate", description: "Starts work in a conversation it owns", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { await api.commit(async (tx) => { const child = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); @@ -1906,7 +1906,7 @@ describe("tool rounds as the view shows them", () => { defineTool({ name: "broken", description: "Starts owned work, then returns a result that is not strict JSON", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { await api.commit((tx) => tx.createTask(Node, { name: "held" }, owned(api.taskId)), callContext); return { output: [], details: { fn: (() => 1) as unknown as JsonValue } }; diff --git a/packages/durable/test/harness-submit-in-commit.test.ts b/packages/durable/test/harness-submit-in-commit.test.ts index 3d8ad2986..9f4e08d9e 100644 --- a/packages/durable/test/harness-submit-in-commit.test.ts +++ b/packages/durable/test/harness-submit-in-commit.test.ts @@ -1,5 +1,6 @@ // Admission for hosts that must not start runs: `whenIdle: "reject"`, and `submit(tx, draft)` inside the caller's // commit (#11149). + import { withTelemetry } from "@earendil-works/chord/context"; import { type AssistantMessage, @@ -7,7 +8,6 @@ import { fauxAssistantMessage, fauxText, fauxToolCall, - Type, } from "@earendil-works/pi-ai"; import { type Conversation, @@ -26,6 +26,7 @@ import { type Tx, UserEntry, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; import { describe, expect, it } from "vitest"; import { type ChatSetup, chatSetup, openChat, unanswered, waitFor } from "./chat-support.ts"; @@ -80,7 +81,7 @@ function addHold(setup: ChatSetup) { defineTool({ name: "hold", description: "Holds until released", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { reached.resolve(); await release.promise; @@ -180,7 +181,7 @@ describe("whenIdle reject", () => { defineTool({ name: "nudge", description: "Steers another conversation", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { const handle = (await api.conversation(other!.id, callContext))!; const draft = { type: "input", content: "nudge", whenBusy: "steer", whenIdle: "reject" } as const; @@ -525,7 +526,7 @@ describe("submit inside the caller's commit", () => { defineTool({ name: "delegate", description: "Delegates to another conversation", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { toolTask = api.taskId; const handle = (await api.conversation(other!.id, callContext))!; @@ -611,7 +612,7 @@ describe("submit inside the caller's commit", () => { defineTool({ name: "hold", description: "Holds until released, also when aborted", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { handle = (await api.conversation(other!.id, callContext))!; callContext.abortSignal!.addEventListener("abort", () => abortSeen.resolve(), { once: true }); diff --git a/packages/durable/test/harness-support.ts b/packages/durable/test/harness-support.ts index 499f81846..d2c443c45 100644 --- a/packages/durable/test/harness-support.ts +++ b/packages/durable/test/harness-support.ts @@ -7,7 +7,7 @@ import type { ToolResultMessage, UserMessage, } from "@earendil-works/pi-ai"; -import { createModels, Type } from "@earendil-works/pi-ai"; +import { createModels } from "@earendil-works/pi-ai"; import { type AnyTask, createRegistry, @@ -23,10 +23,11 @@ import { section, type ToolRegistration, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { context } from "./session-support.ts"; export function tool(name: string, description = `${name} tool`): ToolRegistration { - return defineTool({ name, description, parameters: Type.Object({}), execute: async () => ({ output: [] }) }); + return defineTool({ name, description, parameters: obj({}), execute: async () => ({ output: [] }) }); } /** Open a Harness with a fresh registry holding the named tools. */ diff --git a/packages/durable/test/harness-tasks-recovery.test.ts b/packages/durable/test/harness-tasks-recovery.test.ts index 714fdc202..07fcd7f27 100644 --- a/packages/durable/test/harness-tasks-recovery.test.ts +++ b/packages/durable/test/harness-tasks-recovery.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { Context, JsonValue } from "@earendil-works/chord"; -import { createModels, Type } from "@earendil-works/pi-ai"; +import { createModels } from "@earendil-works/pi-ai"; import { createRegistry, defineDoc, @@ -15,6 +15,7 @@ import { type TaskId, type TaskRuntime, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { addTask, addTool } from "./harness-support.ts"; @@ -482,7 +483,7 @@ describe("blocked tasks", () => { defineTool({ name: "unrelated", description: "unrelated", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({}), }), ); diff --git a/packages/durable/test/harness-tasks.test.ts b/packages/durable/test/harness-tasks.test.ts index cc40953dc..17bfe604f 100644 --- a/packages/durable/test/harness-tasks.test.ts +++ b/packages/durable/test/harness-tasks.test.ts @@ -1,6 +1,6 @@ import type { Context } from "@earendil-works/chord"; import { withAbortSignal, withCancel } from "@earendil-works/chord/context"; -import { createModels, Type } from "@earendil-works/pi-ai"; +import { createModels } from "@earendil-works/pi-ai"; import { type CommitPublication, type Conversation, @@ -18,6 +18,7 @@ import { type TaskId, type TaskRuntime, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; import { describe, expect, it } from "vitest"; import type { SessionImpl } from "../src/session/session.ts"; @@ -424,7 +425,7 @@ describe("task phases", () => { defineTool({ name: "late", description: "late", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({}), }), ); diff --git a/packages/durable/test/harness-telemetry.test.ts b/packages/durable/test/harness-telemetry.test.ts index 59537179e..58bcbc9e8 100644 --- a/packages/durable/test/harness-telemetry.test.ts +++ b/packages/durable/test/harness-telemetry.test.ts @@ -1,5 +1,5 @@ import { withTelemetry } from "@earendil-works/chord/context"; -import { fauxAssistantMessage, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai"; import { CodemodeSandbox } from "@earendil-works/pi-codemode"; import { CompactionTask, @@ -11,6 +11,7 @@ import { type Storage, type TaskId, } from "@earendil-works/pi-durable"; +import { obj, str } from "@earendil-works/pi-schema"; import { InMemoryTelemetryContext, NOOP_TELEMETRY_CONTEXT, @@ -53,7 +54,7 @@ describe("Durable telemetry", () => { defineTool({ name: "inner", description: "inner", - parameters: Type.Object({ secret: Type.String() }), + parameters: obj({ secret: str() }), execute: async () => ({ output: [{ type: "text", text: "private-result" }] }), }), ); @@ -62,7 +63,7 @@ describe("Durable telemetry", () => { defineTool({ name: "outer", description: "outer", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, ctx) => { await api.executeTool("inner", { secret: "private-argument" }, ctx); return { output: [{ type: "text", text: "private-output" }] }; @@ -347,7 +348,7 @@ describe("Durable telemetry", () => { defineTool({ name: "hold", description: "hold", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { reached.resolve(); await gate.promise; @@ -415,7 +416,7 @@ describe("Durable telemetry", () => { defineTool({ name: "delegate", description: "delegate", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, ctx) => { const child = await api.commit( (tx) => tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }), @@ -952,7 +953,7 @@ describe("Durable telemetry", () => { defineTool({ name: "once", description: "once", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { runs++; return { output: [] }; @@ -965,7 +966,7 @@ describe("Durable telemetry", () => { name: "hold", description: "hold", replay: "safe", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, _api, ctx) => { if (!recover) { reached.resolve(); diff --git a/packages/durable/test/harness-tool-hooks.test.ts b/packages/durable/test/harness-tool-hooks.test.ts index 97010c5b3..beca471e1 100644 --- a/packages/durable/test/harness-tool-hooks.test.ts +++ b/packages/durable/test/harness-tool-hooks.test.ts @@ -1,5 +1,6 @@ // `afterTool` runs on every result a tool task writes, also those the Harness decides without executing the tool // (#11043). + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -10,7 +11,6 @@ import { fauxText, fauxToolCall, type ToolResultMessage, - Type, } from "@earendil-works/pi-ai"; import { defineTool, @@ -29,6 +29,7 @@ import { ToolResultEntry, ToolTask, } from "@earendil-works/pi-durable"; +import { bool, obj, opt, str } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { commitWrites } from "../src/testing/commit-writes.ts"; @@ -79,7 +80,7 @@ function stored(): Promise { return new Promise((resolve) => setTimeout(resolve, 50)); } -const Parameters = Type.Object({ text: Type.Optional(Type.String()) }); +const Parameters = obj({ text: opt(str()) }); function tool( name: string, @@ -318,7 +319,7 @@ describe("afterTool on results the call phase decides", () => { addTool( setup.registry, tool("typed", async () => ({ output: [], structuredOutput: { ok: true } }), { - structuredOutputSchema: Type.Object({ ok: Type.Boolean() }), + structuredOutputSchema: obj({ ok: bool() }), }), ); const nested: NestedToolExecutionResult[] = []; diff --git a/packages/durable/test/harness-tools-recovery.test.ts b/packages/durable/test/harness-tools-recovery.test.ts index ad4ffbe98..cf050f8df 100644 --- a/packages/durable/test/harness-tools-recovery.test.ts +++ b/packages/durable/test/harness-tools-recovery.test.ts @@ -8,7 +8,6 @@ import { fauxText, fauxToolCall, type ToolResultMessage, - Type, } from "@earendil-works/pi-ai"; import { defineTool, @@ -21,6 +20,7 @@ import { ToolResultEntry, ToolTask, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import type { ExecutionEnv } from "../src/env/index.ts"; import { NodeExecutionEnv } from "../src/env/node.ts"; @@ -55,7 +55,7 @@ function tool( execute: ToolRegistration["execute"], extra: Partial = {}, ): ToolRegistration { - return defineTool({ name, description: name, parameters: Type.Object({}), execute, ...extra }); + return defineTool({ name, description: name, parameters: obj({}), execute, ...extra }); } function call(name: string, id = "c1"): AssistantMessage { diff --git a/packages/durable/test/harness-tools.test.ts b/packages/durable/test/harness-tools.test.ts index d2937fd77..92aec1cf6 100644 --- a/packages/durable/test/harness-tools.test.ts +++ b/packages/durable/test/harness-tools.test.ts @@ -11,7 +11,6 @@ import { type Models, type SystemMessage, type ToolResultMessage, - Type, } from "@earendil-works/pi-ai"; import { AgentDoc, @@ -32,6 +31,7 @@ import { ToolResultEntry, ToolTask, } from "@earendil-works/pi-durable"; +import { obj, opt, str } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { NodeExecutionEnv } from "../src/env/node.ts"; import { createBashTool, createEditTool, createReadTool } from "../src/tools/index.ts"; @@ -40,7 +40,7 @@ import { addHooks, addTask, addTool, type Installed } from "./harness-support.ts import { context } from "./session-support.ts"; import { aborted, deferred } from "./task-support.ts"; -const EchoParameters = Type.Object({ text: Type.Optional(Type.String()) }); +const EchoParameters = obj({ text: opt(str()) }); type Echo = ToolRegistration; type Execute = Echo["execute"]; diff --git a/packages/durable/test/harness-turn-limit.test.ts b/packages/durable/test/harness-turn-limit.test.ts index e5d819468..7f8ad61e1 100644 --- a/packages/durable/test/harness-turn-limit.test.ts +++ b/packages/durable/test/harness-turn-limit.test.ts @@ -1,8 +1,9 @@ // Per-conversation turn limits (`AgentState.maxTurns`) and continuing a run that reached one (#11043). + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { type AfterFailedRun, type Conversation, @@ -16,6 +17,7 @@ import { ToolResultEntry, ToolTask, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, type ChatSetup, chatSetup, openChat, textOf } from "./chat-support.ts"; @@ -33,7 +35,7 @@ afterEach(async () => { directories.clear(); }); -const Parameters = Type.Object({}); +const Parameters = obj({}); function tool( name: string, diff --git a/packages/durable/test/harness-view-run.test.ts b/packages/durable/test/harness-view-run.test.ts index 61431cd13..79f446098 100644 --- a/packages/durable/test/harness-view-run.test.ts +++ b/packages/durable/test/harness-view-run.test.ts @@ -1,5 +1,6 @@ // What a renderer sees of a run through the conversation view: the transcript window and `pi.live`, every change a // commit makes, rebuilt from operations alone. This replaced the event stream (`watchEvents`, #11327). + import { GCProfiler, getHeapStatistics, setFlagsFromString } from "node:v8"; import { runInNewContext } from "node:vm"; import type { Draft } from "@earendil-works/chord"; @@ -11,7 +12,6 @@ import { fauxText, fauxThinking, fauxToolCall, - Type, } from "@earendil-works/pi-ai"; import { type Conversation, @@ -22,6 +22,7 @@ import { type LiveState, MemoryStorage, } from "@earendil-works/pi-durable"; +import { num, obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { type ChatSetup, chatSetup, openChat, textOf, waitFor } from "./chat-support.ts"; import { addTool } from "./harness-support.ts"; @@ -126,7 +127,7 @@ describe("a run as the conversation view shows it", () => { defineTool({ name: "print", description: "Prints", - parameters: Type.Object({ n: Type.Number() }), + parameters: obj({ n: num() }), execute: async (_args, api) => { api.output("one\n"); await new Promise((resolve) => setTimeout(resolve, 150)); @@ -259,7 +260,7 @@ describe("a run as the conversation view shows it", () => { defineTool({ name: "tail", description: "Prints lines", - parameters: Type.Object({}), + parameters: obj({}), outputLimits: { maxLines: 3, retain: "tail" }, execute: async (_args, api) => { for (let line = 0; line < 6; line++) { @@ -339,7 +340,7 @@ describe("a run as the conversation view shows it", () => { defineTool({ name: "wait", description: "Waits until aborted", - parameters: Type.Object({}), + parameters: obj({}), execute: (_args, _api, callContext) => aborted(callContext.abortSignal!), }), ); @@ -380,7 +381,7 @@ describe("a run as the conversation view shows it", () => { defineTool({ name: "hold", description: "Waits", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { await gate.promise; return {}; diff --git a/packages/durable/test/nested-images-bench.ts b/packages/durable/test/nested-images-bench.ts index d0539a7bf..858f21672 100644 --- a/packages/durable/test/nested-images-bench.ts +++ b/packages/durable/test/nested-images-bench.ts @@ -10,6 +10,7 @@ * * node --conditions=source --experimental-strip-types --expose-gc test/nested-images-bench.ts [count] */ + import { randomBytes } from "node:crypto"; import { mkdtemp, readdir, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -18,7 +19,7 @@ import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; import type { ImageContent } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; -import { Type } from "typebox"; +import { arr, lit, num, obj, opt, str } from "@earendil-works/pi-schema"; import { createRegistry, defineExtension, @@ -95,11 +96,11 @@ function noWrites(): Written { }; } -const ImageArgument = Type.Object({ - type: Type.Literal("image"), - mimeType: Type.String(), - data: Type.Optional(Type.String()), - source: Type.Optional(Type.Object({ taskId: Type.Number() })), +const ImageArgument = obj({ + type: lit("image"), + mimeType: str(), + data: opt(str()), + source: opt(obj({ taskId: num() })), }); function registryFor(scenario: Scenario, probe: Probe) { @@ -107,7 +108,7 @@ function registryFor(scenario: Scenario, probe: Probe) { const readImage = defineTool({ name: "read_image", description: "Read an image", - parameters: Type.Object({ n: Type.Number() }), + parameters: obj({ n: num() }), // One image, so programs get it as an `ImageContent`. Unique per call: base64 of random bytes. execute: async () => ({ output: [ @@ -118,7 +119,7 @@ function registryFor(scenario: Scenario, probe: Probe) { const classify = defineTool({ name: "classify", description: "Classify images", - parameters: Type.Object({ images: Type.Array(ImageArgument) }), + parameters: obj({ images: arr(ImageArgument) }), execute: async (args, api, callContext) => { for (const image of args.images) { let data = image.data; @@ -141,7 +142,7 @@ function registryFor(scenario: Scenario, probe: Probe) { const script = defineTool({ name: "script", description: "Read and classify images", - parameters: Type.Object({ count: Type.Number() }), + parameters: obj({ count: num() }), callers: ["model"], execute: async (args, api, callContext) => { for (let n = 0; n < args.count; n++) { diff --git a/packages/durable/test/nested-tools-bench.ts b/packages/durable/test/nested-tools-bench.ts index 0c4ed5224..002383fec 100644 --- a/packages/durable/test/nested-tools-bench.ts +++ b/packages/durable/test/nested-tools-bench.ts @@ -9,13 +9,14 @@ * * node --conditions=source --experimental-strip-types --expose-gc test/nested-tools-bench.ts [streaming|nesting] */ + import { mkdtemp, readdir, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; -import { Type } from "typebox"; +import { num, obj } from "@earendil-works/pi-schema"; import { type Conversation, createRegistry, @@ -103,7 +104,7 @@ async function runScenario(scenario: Scenario): Promise<{ perRun: Run[]; closedB const blob = defineTool({ name: "blob", description: "Stream a blob", - parameters: Type.Object({ n: Type.Number() }), + parameters: obj({ n: num() }), execute: async (_args, api) => { for (let n = 0; n < CHUNKS; n++) { api.output(chunk); @@ -118,7 +119,7 @@ async function runScenario(scenario: Scenario): Promise<{ perRun: Run[]; closedB defineTool({ name, description: `Call ${target} in parallel`, - parameters: Type.Object({ count: Type.Number() }), + parameters: obj({ count: num() }), execute: async (args, api, callContext) => { const calls = Array.from({ length: args.count }, (_, n) => api.executeTool(target, { n, count: FAN_OUT }, callContext, options), diff --git a/packages/durable/test/scheduler-fanout-bench.ts b/packages/durable/test/scheduler-fanout-bench.ts index ab00d9881..2d2fc381f 100644 --- a/packages/durable/test/scheduler-fanout-bench.ts +++ b/packages/durable/test/scheduler-fanout-bench.ts @@ -5,10 +5,11 @@ * * node --conditions=source --experimental-strip-types test/scheduler-fanout-bench.ts [calls...] */ + import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; -import { Type } from "typebox"; +import { num, obj } from "@earendil-works/pi-schema"; import { createRegistry, defineExtension, defineTool, Harness, ToolResultEntry } from "../src/index.ts"; import { MemoryStorage } from "../src/storage/memory.ts"; @@ -20,13 +21,13 @@ async function run(calls: number): Promise<{ ms: number; commits: number }> { const leaf = defineTool({ name: "leaf", description: "Return at once", - parameters: Type.Object({ n: Type.Number() }), + parameters: obj({ n: num() }), execute: async () => ({ output: [{ type: "text", text: "ok" }] }), }); const fan = defineTool({ name: "fan", description: "Call leaf in parallel", - parameters: Type.Object({ count: Type.Number() }), + parameters: obj({ count: num() }), execute: async (args, api, callContext) => { const results = await Promise.all( Array.from({ length: args.count }, (_, n) => diff --git a/packages/durable/test/session-document-retention.test.ts b/packages/durable/test/session-document-retention.test.ts index 2712a1817..8d1b5e927 100644 --- a/packages/durable/test/session-document-retention.test.ts +++ b/packages/durable/test/session-document-retention.test.ts @@ -1,6 +1,7 @@ // The Session's document cache is bounded by use: unattached documents leave it after the retention period and load // from Storage again; documents with a watch or a document state never leave it. -import { fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; + +import { fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { type ConversationId, defineDoc, @@ -11,6 +12,7 @@ import { MemoryStorage, type StorageWrite, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it, onTestFinished } from "vitest"; import { SessionImpl, setDefaultDocumentRetention } from "../src/session/session.ts"; import { chatSetup, openChat } from "./chat-support.ts"; @@ -519,7 +521,7 @@ describe("document cache retention", () => { defineTool({ name: "steps", description: "Streams three steps", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api) => { for (const step of ["a\n", "b\n", "c\n"]) { api.output(step); diff --git a/packages/durable/test/spec-usage.test.ts b/packages/durable/test/spec-usage.test.ts index c50cc255e..b1896bf66 100644 --- a/packages/durable/test/spec-usage.test.ts +++ b/packages/durable/test/spec-usage.test.ts @@ -3,8 +3,9 @@ * the names the spec leaves to the application declared below. Keep the two in sync; `test/examples/` runs the same * patterns end to end. */ + import type { Context, Draft } from "@earendil-works/chord"; -import { type AssistantMessage, type Models, type ToolCall, Type } from "@earendil-works/pi-ai"; +import type { AssistantMessage, Models, ToolCall } from "@earendil-works/pi-ai"; import { type Conversation, type ConversationId, @@ -34,6 +35,7 @@ import { type UserInput, wrapTool, } from "@earendil-works/pi-durable"; +import { obj, str } from "@earendil-works/pi-schema"; import { expect, expectTypeOf, it } from "vitest"; import type { ExecutionEnv } from "../src/env/index.ts"; import { CodingTools, createBashTool, createEditTool, createReadTool } from "../src/tools/index.ts"; @@ -169,7 +171,7 @@ function examples() { defineTool({ name: "subagent", description: "Delegate a self-contained task to a subagent and get its answer back.", - parameters: Type.Object({ task: Type.String() }), + parameters: obj({ task: str() }), replay: "safe", execute: async (args, api, context) => { const { task } = args; diff --git a/packages/durable/test/system-order-cache-e2e.test.ts b/packages/durable/test/system-order-cache-e2e.test.ts index 0d67629d5..f6581ac4a 100644 --- a/packages/durable/test/system-order-cache-e2e.test.ts +++ b/packages/durable/test/system-order-cache-e2e.test.ts @@ -4,7 +4,6 @@ import { type Model, type Models, normalizeContext, - Type, uuidv7, } from "@earendil-works/pi-ai"; import { builtinModels } from "@earendil-works/pi-ai/providers/all"; @@ -18,6 +17,7 @@ import { hook, MemoryStorage, } from "@earendil-works/pi-durable"; +import { obj, str } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { streamSimple as streamSimpleAnthropic } from "../../ai/src/api/anthropic-messages.ts"; import { resolveApiKey } from "../../ai/test/oauth.ts"; @@ -41,7 +41,7 @@ function instructions(nonce: string): string { return lines.join("\n"); } -const Parameters = Type.Object({ text: Type.String() }); +const Parameters = obj({ text: str() }); const first = defineTool({ name: "probe_first", description: "Unused probe tool.", diff --git a/packages/durable/test/tool-output-bench.ts b/packages/durable/test/tool-output-bench.ts index 19140c1ff..ac7a54d29 100644 --- a/packages/durable/test/tool-output-bench.ts +++ b/packages/durable/test/tool-output-bench.ts @@ -7,6 +7,7 @@ * then close the Harness mid-round (persistent backends) to measure replay of the round's delta chain on reopen, and * finish the run to measure the stored size after the round's base and reclamation. Throughput scenarios push 1 GiB. */ + import { execFile } from "node:child_process"; import { mkdtemp, open, readdir, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -16,7 +17,7 @@ import { promisify } from "node:util"; import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; -import { Type } from "typebox"; +import { obj } from "@earendil-works/pi-schema"; import { NodeExecutionEnv } from "../src/env/node.ts"; import { createRegistry, defineTool, Harness, LiveDoc, MemoryStorage, type Storage } from "../src/index.ts"; import { openNodeJsonlStorage } from "../src/storage/jsonl/node.ts"; @@ -145,7 +146,7 @@ async function runScenario(scenario: Scenario): Promise { defineTool({ name: "emit", description: "", - parameters: Type.Object({}), + parameters: obj({}), outputLimits: { retain: scenario.retain }, execute: async (_args, api, callContext) => { let counter = 0; diff --git a/packages/durable/test/tool-search.test.ts b/packages/durable/test/tool-search.test.ts index 895240bbb..988c85d5a 100644 --- a/packages/durable/test/tool-search.test.ts +++ b/packages/durable/test/tool-search.test.ts @@ -1,4 +1,4 @@ -import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { AgentDoc, type Conversation, @@ -9,6 +9,7 @@ import { MemoryStorage, type ToolRegistration, } from "@earendil-works/pi-durable"; +import { num, obj, opt } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { createToolSearchExtension } from "../src/tool-search/index.ts"; import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; @@ -19,7 +20,7 @@ function tool(name: string, description: string, extra: Partial ({ output: [{ type: "text", text: `${name} ran` }] }), ...extra, }); @@ -120,7 +121,7 @@ describe("deferred tools and tool_search", () => { const grow = defineTool({ name: "grow", description: "grow", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [], control: { addTools: ["github_get_issue", "alpha", "ghost"] } }), }); addTool(chat.registry, grow); diff --git a/packages/mcp/CHANGELOG.md b/packages/mcp/CHANGELOG.md index 74908a649..92e01bf9d 100644 --- a/packages/mcp/CHANGELOG.md +++ b/packages/mcp/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Changed + +- README: the `AgentTool` wrapping example passes the MCP input schema as plain JSON Schema instead of through TypeBox's `Type.Unsafe`. + ### Fixed - OAuth metadata parsing and `startAuthorization()` now accept only `http:` and `https:` URLs. Before, a server could supply a `file:` or custom protocol authorization endpoint (for example `search-ms:`), which the platform URL handler would launch on sign-in. diff --git a/packages/mcp/README.md b/packages/mcp/README.md index b09e043fb..ebb1b2973 100644 --- a/packages/mcp/README.md +++ b/packages/mcp/README.md @@ -36,7 +36,6 @@ Wrapping an MCP tool as a `pi-agent-core` `AgentTool`: ```typescript import type { AgentTool } from "@earendil-works/pi-agent-core"; import { toLlmContent } from "@earendil-works/pi-mcp"; -import { Type } from "typebox"; const tools: AgentTool[] = (await client.listTools()).map((tool) => ({ // Providers allow at most 64 characters of [A-Za-z0-9_-]. @@ -44,7 +43,7 @@ const tools: AgentTool[] = (await client.listTools()).map((tool) => ({ label: tool.title ?? tool.name, description: tool.description ?? tool.name, // Providers require an object schema, and some reject one without `properties`. - parameters: Type.Unsafe({ ...tool.inputSchema, type: "object", properties: tool.inputSchema.properties ?? {} }), + parameters: { ...tool.inputSchema, type: "object", properties: tool.inputSchema.properties ?? {} }, execute: async (_toolCallId, params, signal) => { const result = await client.callTool(tool.name, params as Record, { signal }); // MCP reports tool failures in the result instead of as a protocol error. diff --git a/packages/schema/CHANGELOG.md b/packages/schema/CHANGELOG.md index d23ca137c..69650940a 100644 --- a/packages/schema/CHANGELOG.md +++ b/packages/schema/CHANGELOG.md @@ -6,4 +6,7 @@ - Initial release: `compile(schema, options)` validates JSON Schema drafts 2020-12 and 7 without `eval` or `new Function`, compiling a schema once into plain closures. `Validator.check` is the boolean check (a type guard for schemas built with the builders), `Validator.errors` reports each failure with a JSON Pointer path. Format checks are in the optional `@earendil-works/pi-schema/formats` entry point. - Schema builders (`str`, `num`, `int`, `bool`, `nul`, `unknown`, `lit`, `oneOf`, `obj`, `opt`, `arr`, `tuple`, `map`, `union`, `nullable`, `bytes`) that return plain JSON Schema typed with the values it accepts (`Static`). +- `compile(schema, { coerce: true })` adds `coerce(value)` and `parse(value)` for loosely typed input such as a model's tool call arguments: optional `null`s dropped, TypeBox 1.3.27's `Value.Convert` rules keyed on JSON Schema keywords (numeric strings, `"true"`, single values to arrays, `const`/`enum` members, union branches), then Ajv-style primitive coercion by `type`. `coerce` copies only the paths it changes and returns the input itself when nothing changes; `parse` coerces, then checks. Validators compiled without the option have neither; the README lists every rule. +- `unresolvedRefs: "reject"` compiles a `$ref` that names nothing as `false` (the error is `unresolved reference `) instead of throwing, for schemas whose references may dangle. +- `formats` also checks `url` and `json-pointer-uri-fragment`, as TypeBox and ajv-formats do. - `npm run bench` compares conformance, speed, setup cost, memory and bundle size with TypeBox, @cfworker/json-schema and omptype. diff --git a/packages/schema/README.md b/packages/schema/README.md index 023ebdc02..100186dd9 100644 --- a/packages/schema/README.md +++ b/packages/schema/README.md @@ -86,11 +86,19 @@ const validator = compile(schema, { draft: "7", // "2020-12" (default) or "7" formats, // check `format`; without it `format` is an annotation, as 2020-12 specifies schemas: { "https://example.com/point.json": point }, // documents for $refs to other documents, by URI + unresolvedRefs: "reject", // a $ref that names nothing rejects values at its place; default "throw" + coerce: true, // adds coerce() and parse(), see Coercion }); ``` -`formats` has every format 2020-12 defines (`date-time`, `email`, `uri`, `uuid`, `ipv6`, ...). Pass your own map to -check fewer formats, or custom ones. A `$ref` that names nothing throws when the schema compiles. +`formats` has every format 2020-12 defines (`date-time`, `email`, `uri`, `uuid`, `ipv6`, ...), plus `url` and +`json-pointer-uri-fragment`, which TypeBox and ajv-formats also check. Pass your own map to check fewer formats, or +custom ones. + +A `$ref` that names nothing (a missing definition, a document not in `schemas`) throws when the schema compiles. With +`unresolvedRefs: "reject"` it compiles as `false` instead: only values at that place fail, with the error +`unresolved reference `, and the rest of the schema works. Use it for schemas from other systems, such as MCP +servers, whose references may dangle. ### TypeBox schemas @@ -105,6 +113,56 @@ const user = compile(Type.Object({ name: Type.String(), tags: Type.Array(Type.St TypeBox's JavaScript-only types (`BigInt`, `Undefined`, `Function`, ...), refinements and codecs are not JSON Schema and are not supported. +### Coercion + +`compile(schema, { coerce: true })` adds `coerce()` and `parse()`, for loosely typed input such as a model's tool call +arguments. Without the option the validator has neither, and its `check` pays nothing for them: the coercion rules are +built on the first `coerce()` or `parse()`. + +```typescript +const read = compile(Read, { coerce: true }); + +read.coerce({ path: "a.txt", offset: "10", mode: "text" }); // { path: "a.txt", offset: 10, mode: "text" } + +const result = read.parse({ path: "a.txt", offset: "ten", mode: "text" }); +if (result.ok) result.value.offset; // number +else result.errors; // [{ path: "/offset", keyword: "type", message: "expected integer, got string" }] +``` + +`coerce(value)` returns the coerced value. It copies only the paths it changes, returns `value` itself when nothing +changes, and never modifies `value`. A value no rule applies to stays as it is. `parse(value)` coerces, then checks: +`{ ok: true, value }` with the coerced value, or `{ ok: false, errors }` with the errors of the coerced value. + +Three passes run in order: + +1. **Optional nulls.** A `null` in an optional property whose schema rejects `null` is dropped, as if omitted. +2. **Conversions**, ported from TypeBox 1.3.27's `Value.Convert` and keyed on JSON Schema keywords (the first that + applies to a schema): `$ref` follows the reference; `const` and `enum` take the member the value converts to; + `anyOf`/`oneOf` keep a value one branch accepts, else take the first branch's conversion the whole schema + accepts; `allOf` converts with each member in turn; then by `type`: + - `string`: numbers and booleans to their text, `null` to `"null"`. + - `number`: numeric strings (`""` is 0), `"true"`/`"false"`, booleans to 1/0, `null` to 0. + - `integer`: as `number`, then truncated (`"3.5"` and `3.5` become 3). + - `boolean`: `"true"`/`"false"` (any case), `"1"`/`"0"`, 1/0, `null` to `false`. + - `null`: `"null"`, `"undefined"`, `""`, `"0"`, 0 and `false`. + - `array`: a single value becomes a one-item array, then each item converts; a tuple (`prefixItems` or an `items` + array) converts its items and never wraps. + - `object`: each property converts by `properties`, `patternProperties` or `additionalProperties`. +3. **Primitive coercion by type**, as Ajv's `coerceTypes`, for what pass 2 leaves (type arrays such as + `["integer", "null"]`): numbers from numeric strings and booleans, booleans from `"true"`/`"false"` and 1/0, + strings from numbers and booleans, `null` from `""`, 0 and `false`, and `null` to `""`, 0 or `false`. A value that + already has one of a type array's types stays. + +Where passes 2 and 3 disagree, pass 2 wins: `null` for a required string becomes `"null"`, not `""`. A union's +branches convert in order and only until one is accepted; a union node that also declares properties converts them into +each branch's candidate before checking it. A recursive schema (a tree through `$ref`) that meets a value it cannot +convert leaves the value as it is, so `check` reports it, and results through a `$ref` are reused within one call, so a +recursive union does not convert (or explain) the same child once per branch. + +Pass 1 drops optional nulls inside the values of `properties`, `patternProperties` and `additionalProperties` alike; a +pattern or additional key itself is never dropped. Items past a draft-7 `items` array (`additionalItems`) are not +coerced by any pass. + ### Values - A property whose value is `undefined` counts as absent, as in TypeBox. diff --git a/packages/schema/src/checks.ts b/packages/schema/src/checks.ts index 701707a28..222d3dbde 100644 --- a/packages/schema/src/checks.ts +++ b/packages/schema/src/checks.ts @@ -127,7 +127,11 @@ export function codePoints(s: string): number { let n = 0; for (let i = 0; i < s.length; i++, n++) { const c = s.charCodeAt(i); - if (c >= 0xd800 && c <= 0xdbff) i++; + // A high surrogate starts a pair only when a low surrogate follows; an unpaired one is a code point of its own. + if (c >= 0xd800 && c <= 0xdbff && i + 1 < s.length) { + const next = s.charCodeAt(i + 1); + if (next >= 0xdc00 && next <= 0xdfff) i++; + } } return n; } diff --git a/packages/schema/src/coerce.ts b/packages/schema/src/coerce.ts new file mode 100644 index 000000000..5dee2487f --- /dev/null +++ b/packages/schema/src/coerce.ts @@ -0,0 +1,556 @@ +// Coercion: turns a loosely typed value (a model's tool call arguments, form input, query strings) into the value a +// schema most likely meant, before it is checked. Only validators compiled with `{ coerce: true }` have it, and they +// build it on their first coerce() or parse(); the check path never runs it. +// +// Three passes, in this order: +// 1. Optional nulls: a `null` in an optional property whose schema rejects null is dropped, as if omitted. +// 2. Conversion rules ported from TypeBox 1.3.27's Value.Convert (MIT), keyed on JSON Schema structure (`$ref`, `const`, +// `enum`, `anyOf`/`oneOf`, `allOf`, `type`, `items`/`prefixItems`, `properties`/`patternProperties`) instead of +// TypeBox's kind marks: `null` becomes "null" for a string, `"3.5"` becomes 3 for an integer, a single value becomes +// a one-item array, and a union takes the first branch whose conversion passes. +// 3. Primitive coercion by `type` (also type arrays), as in Ajv's `coerceTypes`: what pass 2 left, for example a type +// array `["integer", "null"]`, or `""` for `null`. +// A value that cannot be coerced is left as it is; the check then reports it. +// +// Every pass is copy-on-write: an object or array is copied only when one of its members changes, so an unchanged +// value comes back as the same reference and the input is never modified. All passes walk the same keywords for +// members: `properties`, then `patternProperties`, then `additionalProperties` for keys neither names; tuple items by +// `prefixItems` (or a draft-7 `items` array), the rest by `items`. Draft 7's `additionalItems` is not coerced. + +import type { Compiler } from "./compiler.ts"; +import { isSchemaObject } from "./resolver.ts"; +import { sharedRegex } from "./shared.ts"; +import type { SchemaObject } from "./types.ts"; + +/** Coerces values for one compiled schema. */ +export class Coercer { + private readonly c: Compiler; + private readonly schema: unknown; + /** + * The (schema node, value) pairs pass 2 is converting through a re-entry point, as a stack. Conversion can enter + * a node again with the same value only through a `$ref` or by wrapping a single value into an array (also for + * schemas that cycle through shared JS objects): a recursive schema that meets a value no branch accepts would + * convert it without end. A pair already on the stack (primitives by value, objects by reference) leaves the value + * unchanged. Only those two points push, so schemas without references or wraps pay nothing. + */ + private readonly entered: unknown[] = []; + /** How often a re-entry was cut short in this coerce() call; results computed across one are not memoized. */ + private cuts = 0; + /** + * Results of `$ref` targets by (node, value) within one coerce() call: a recursive union whose branches all + * convert the same children (and all fail) would convert them once per branch, exponentially in the depth. + */ + private memo: Map> | undefined = undefined; + + constructor(compiler: Compiler, schema: unknown) { + this.c = compiler; + this.schema = schema; + } + + /** `value` coerced: a copy along the changed paths only, or `value` itself when nothing changes. */ + coerce(value: unknown): unknown { + const base = this.c.r.rootBase; + const withoutNulls = this.dropOptionalNulls(value, this.schema, base); + return this.coerceTypes(this.convertOnly(withoutNulls), this.schema, base); + } + + /** Pass 2 alone, as TypeBox's `Value.Convert` (the TypeBox conformance tests run it). */ + convertOnly(value: unknown): unknown { + try { + return this.convert(this.schema, this.c.r.rootBase, value); + } finally { + this.memo = undefined; + this.cuts = 0; + } + } + + /** `node` converted for `value` at a re-entry point: unchanged when that pair is already being converted. */ + private reenter(node: object, base: string, value: unknown, memoize: boolean): unknown { + const entered = this.entered; + for (let i = 0; i < entered.length; i += 2) { + if (entered[i] === node && Object.is(entered[i + 1], value)) { + this.cuts++; + return value; + } + } + const memo = memoize ? this.memo?.get(node) : undefined; + if (memo?.has(value)) return memo.get(value); + const cuts = this.cuts; + entered.push(node, value); + let result: unknown; + try { + result = this.convert(node, base, value); + } finally { + entered.length -= 2; + } + if (memoize && this.cuts === cuts) { + this.memo ??= new Map(); + let values = this.memo.get(node); + if (values === undefined) { + values = new Map(); + this.memo.set(node, values); + } + values.set(value, result); + } + return result; + } + + /** Whether `value` passes the schema `node`; undefined when `node` cannot be compiled. */ + private passes(node: unknown, base: string, value: unknown): boolean | undefined { + try { + return this.c.check(node, this.baseOf(node, base))(value); + } catch { + return undefined; + } + } + + private baseOf(node: unknown, base: string): string { + return isSchemaObject(node) ? (this.c.r.baseOf.get(node) ?? base) : base; + } + + // --------------------------------------------------------------------------------------------------------------- + // Pass 1: optional nulls + // --------------------------------------------------------------------------------------------------------------- + + private dropOptionalNulls(value: unknown, schema: unknown, base: string): unknown { + if (!isSchemaObject(schema)) return value; + const b = this.baseOf(schema, base); + if (Array.isArray(value)) { + return mapItems(value, (item, i) => { + const itemSchema = itemSchemaAt(schema, i); + return itemSchema === undefined ? item : this.dropOptionalNulls(item, itemSchema, b); + }); + } + if (!isPlainObject(value)) return value; + const properties = isSchemaObject(schema.properties) ? schema.properties : {}; + const patterns = isSchemaObject(schema.patternProperties) + ? Object.entries(schema.patternProperties).map(([pattern, item]) => [sharedRegex(pattern), item] as const) + : []; + const additional = isSchemaObject(schema.additionalProperties) ? schema.additionalProperties : undefined; + if (patterns.length === 0 && additional === undefined && !isSchemaObject(schema.properties)) return value; + const required = new Set(Array.isArray(schema.required) ? schema.required : []); + let out: Record | undefined; + const set = (key: string, next: unknown) => { + if (next === (out ?? value)[key]) return; + out ??= { ...value }; + out[key] = next; + }; + for (const [key, property] of Object.entries(properties)) { + if (!Object.hasOwn(value, key)) continue; + const isRef = isSchemaObject(property) && typeof property.$ref === "string"; + if (value[key] === null && !required.has(key) && !isRef && this.passes(property, b, null) === false) { + out ??= { ...value }; + delete out[key]; + continue; + } + set(key, this.dropOptionalNulls(value[key], property, b)); + } + // Values of pattern and additional properties are walked for their own optional nulls; the keys themselves + // are not optional properties, so they stay. + if (patterns.length > 0 || additional !== undefined) { + for (const key of Object.keys(value)) { + let matched = Object.hasOwn(properties, key); + for (const [pattern, item] of patterns) { + if (!pattern.test(key)) continue; + matched = true; + set(key, this.dropOptionalNulls((out ?? value)[key], item, b)); + } + if (!matched && additional !== undefined) set(key, this.dropOptionalNulls(value[key], additional, b)); + } + } + return out ?? value; + } + + // --------------------------------------------------------------------------------------------------------------- + // Pass 2: TypeBox's conversion rules + // --------------------------------------------------------------------------------------------------------------- + + private convert(node: unknown, base: string, value: unknown): unknown { + if (!isSchemaObject(node)) return value; + const b = this.baseOf(node, base); + if (typeof node.$ref === "string") { + let target: { node: unknown; base: string }; + try { + target = this.c.r.resolve(node.$ref, b); + } catch { + return value; + } + return isSchemaObject(target.node) ? this.reenter(target.node, target.base, value, true) : value; + } + return this.convertNode(node, b, value); + } + + /** One schema node's conversion: its first keyword among `const`, `enum`, the unions, `allOf` and `type`. */ + private convertNode(node: SchemaObject, b: string, value: unknown): unknown { + if (Object.hasOwn(node, "const")) return convertLiteral(node.const, value); + if (Array.isArray(node.enum)) return this.convertEnum(node, node.enum, b, value); + // A union or intersection node may also declare members (plain JSON Schema; TypeBox's never do): they convert + // after the branches. + if (Array.isArray(node.anyOf)) return this.convertUnion(node, node.anyOf, b, value); + if (Array.isArray(node.oneOf)) return this.convertUnion(node, node.oneOf, b, value); + if (Array.isArray(node.allOf)) { + let next = value; + for (const member of node.allOf) next = this.convert(member, b, next); + return this.convertMembers(node, b, next); + } + switch (node.type) { + case "string": + return orValue(tryString(value), value); + case "number": + return orValue(tryNumber(value), value); + case "integer": { + const result = tryNumber(value); + return result === undefined ? value : Math.trunc(result.value); + } + case "boolean": + return orValue(tryBoolean(value), value); + case "null": + return orValue(tryNull(value), value); + case "array": + return this.convertArray(node, b, value); + case "object": + return this.convertObject(node, b, value); + case undefined: + if (node.properties !== undefined || node.patternProperties !== undefined) { + return this.convertObject(node, b, value); + } + if (node.items !== undefined || node.prefixItems !== undefined) return this.convertArray(node, b, value); + return value; + default: + return value; + } + } + + /** An enum is a union of its values: a member stays, else the first value the input converts to. */ + private convertEnum(node: SchemaObject, values: readonly unknown[], base: string, value: unknown): unknown { + if (values.some((member) => member === value)) return value; + for (const member of values) { + const candidate = convertLiteral(member, value); + if (candidate !== value && this.passes(node, base, candidate) === true) return candidate; + } + return value; + } + + /** + * A value one branch accepts stays; else the first branch's conversion that the whole union accepts. Branches are + * converted in order and only until one is accepted, so a later recursive branch is not entered needlessly. A node + * that also declares members (plain JSON Schema; TypeBox's never do) converts them into each candidate before it is + * checked, so a branch is not discarded for a sibling property that is converted after it. + */ + private convertUnion(node: SchemaObject, branches: readonly unknown[], base: string, value: unknown): unknown { + if (branches.some((branch) => this.passes(branch, base, value) === true)) + return this.convertMembers(node, base, value); + for (const branch of branches) { + const candidate = this.convertMembers(node, base, this.convert(branch, base, value)); + if (this.passes(node, base, candidate) === true) return candidate; + } + return this.convertMembers(node, base, value); + } + + /** The object members of a union or intersection node, when it declares any. */ + private convertMembers(node: SchemaObject, base: string, value: unknown): unknown { + const declares = + node.properties !== undefined || + node.patternProperties !== undefined || + node.additionalProperties !== undefined; + return declares ? this.convertObject(node, base, value) : value; + } + + private convertArray(node: SchemaObject, base: string, value: unknown): unknown { + const prefix = Array.isArray(node.items) + ? node.items + : Array.isArray(node.prefixItems) + ? node.prefixItems + : undefined; + if (prefix !== undefined) { + // A tuple converts its items and never wraps a single value. + if (!Array.isArray(value)) return value; + const rest = Array.isArray(node.items) ? undefined : node.items; + return mapItems(value, (item, i) => { + const schema = i < prefix.length ? prefix[i] : rest; + return schema === undefined ? item : this.convert(schema, base, item); + }); + } + if (Array.isArray(value)) { + return node.items === undefined ? value : mapItems(value, (item) => this.convert(node.items, base, item)); + } + // The wrapped single value is converted by the item schema with the same value: a re-entry point. + const item = node.items; + return [isSchemaObject(item) ? this.reenter(item, base, value, false) : value]; + } + + private convertObject(node: SchemaObject, base: string, value: unknown): unknown { + if (!isPlainObject(value)) return value; + const properties = isSchemaObject(node.properties) ? node.properties : {}; + const patterns = isSchemaObject(node.patternProperties) + ? Object.entries(node.patternProperties).map(([pattern, schema]) => [sharedRegex(pattern), schema] as const) + : []; + const additional = isSchemaObject(node.additionalProperties) ? node.additionalProperties : undefined; + let out: Record | undefined; + for (const key of Object.keys(value)) { + // An undefined property counts as absent. + if (value[key] === undefined) continue; + let next: unknown = value[key]; + let matched = Object.hasOwn(properties, key); + if (matched) next = this.convert(properties[key], base, next); + for (const [pattern, schema] of patterns) { + if (!pattern.test(key)) continue; + matched = true; + next = this.convert(schema, base, next); + } + if (!matched && additional !== undefined) next = this.convert(additional, base, next); + if (next !== value[key]) { + out ??= { ...value }; + out[key] = next; + } + } + return out ?? value; + } + + // --------------------------------------------------------------------------------------------------------------- + // Pass 3: primitive coercion by type + // --------------------------------------------------------------------------------------------------------------- + + private coerceTypes(value: unknown, schema: unknown, base: string): unknown { + if (!isSchemaObject(schema)) return value; + const b = this.baseOf(schema, base); + let next = value; + if (Array.isArray(schema.allOf)) for (const member of schema.allOf) next = this.coerceTypes(next, member, b); + if (Array.isArray(schema.anyOf)) next = this.coerceUnion(next, schema.anyOf, b); + if (Array.isArray(schema.oneOf)) next = this.coerceUnion(next, schema.oneOf, b); + const types = typesOf(schema); + const matchesUnionMember = types.length > 1 && types.some((type) => matchesType(next, type)); + if (types.length > 0 && !matchesUnionMember) { + for (const type of types) { + const candidate = coercePrimitive(next, type); + if (candidate !== next) { + next = candidate; + break; + } + } + } + if (types.includes("object") && isPlainObject(next)) next = this.coerceProperties(next, schema, b); + if (types.includes("array") && Array.isArray(next)) next = this.coerceItems(next, schema, b); + return next; + } + + private coerceUnion(value: unknown, branches: readonly unknown[], base: string): unknown { + for (const branch of branches) if (this.passes(branch, base, value) === true) return value; + for (const branch of branches) { + const coerced = this.coerceTypes(value, branch, base); + if (this.passes(branch, base, coerced) === true) return coerced; + } + return value; + } + + private coerceProperties( + value: Record, + schema: SchemaObject, + base: string, + ): Record { + const properties = isSchemaObject(schema.properties) ? schema.properties : {}; + const patterns = isSchemaObject(schema.patternProperties) + ? Object.entries(schema.patternProperties).map(([pattern, item]) => [sharedRegex(pattern), item] as const) + : []; + const additional = isSchemaObject(schema.additionalProperties) ? schema.additionalProperties : undefined; + let out: Record | undefined; + const set = (key: string, next: unknown) => { + if (next === (out ?? value)[key]) return; + out ??= { ...value }; + out[key] = next; + }; + for (const [key, property] of Object.entries(properties)) { + if (key in value) set(key, this.coerceTypes(value[key], property, base)); + } + for (const key of Object.keys(value)) { + let matched = Object.hasOwn(properties, key); + for (const [pattern, item] of patterns) { + if (!pattern.test(key)) continue; + matched = true; + set(key, this.coerceTypes((out ?? value)[key], item, base)); + } + if (!matched && additional !== undefined) set(key, this.coerceTypes(value[key], additional, base)); + } + return out ?? value; + } + + private coerceItems(value: unknown[], schema: SchemaObject, base: string): unknown[] { + return mapItems(value, (item, i) => { + const itemSchema = itemSchemaAt(schema, i); + return isSchemaObject(itemSchema) ? this.coerceTypes(item, itemSchema, base) : item; + }); + } +} + +/** + * The schema of an array's item at `index`: a tuple's (`prefixItems`, or a draft-7 `items` array) for its prefix, + * `items` after `prefixItems`, or `items` for every item. Undefined when nothing applies. + */ +function itemSchemaAt(schema: SchemaObject, index: number): unknown { + if (Array.isArray(schema.items)) return schema.items[index]; + if (Array.isArray(schema.prefixItems) && index < schema.prefixItems.length) return schema.prefixItems[index]; + return schema.items; +} + +/** `items` mapped by `map`; the same array when every item maps to itself. */ +function mapItems(items: unknown[], map: (item: unknown, index: number) => unknown): unknown[] { + let out: unknown[] | undefined; + for (let i = 0; i < items.length; i++) { + const next = map(items[i], i); + if (next === items[i]) continue; + out ??= items.slice(); + out[i] = next; + } + return out ?? items; +} + +// ------------------------------------------------------------------------------------------------------------------- +// TypeBox's conversions of one value (value/convert/try in TypeBox 1.3.27): a result, or undefined when there is none +// ------------------------------------------------------------------------------------------------------------------- + +type Converted = { readonly value: T } | undefined; + +function orValue(result: Converted, value: unknown): unknown { + return result === undefined ? value : result.value; +} + +/** A literal (`const`, an enum value) taken from a value of another type that converts to exactly it. */ +function convertLiteral(literal: unknown, value: unknown): unknown { + if (literal === value) return value; + const result = + typeof literal === "string" + ? tryString(value) + : typeof literal === "number" + ? tryNumber(value) + : typeof literal === "boolean" + ? tryBoolean(value) + : undefined; + return result !== undefined && result.value === literal ? result.value : value; +} + +function tryString(value: unknown): Converted { + if (typeof value === "bigint" || typeof value === "boolean" || isFiniteNumber(value)) + return { value: String(value) }; + if (value === null) return { value: "null" }; + if (typeof value === "string") return { value }; + if (value === undefined) return { value: "" }; + return undefined; +} + +const MAX_BIGINT = BigInt(Number.MAX_SAFE_INTEGER); +const MIN_BIGINT = BigInt(Number.MIN_SAFE_INTEGER); + +function tryNumber(value: unknown): Converted { + if (typeof value === "bigint") + return value <= MAX_BIGINT && value >= MIN_BIGINT ? { value: Number(value) } : undefined; + if (typeof value === "boolean") return { value: value ? 1 : 0 }; + if (isFiniteNumber(value)) return { value }; + if (value === null || value === undefined) return { value: 0 }; + if (typeof value !== "string") return undefined; + const coerced = +value; + if (Number.isFinite(coerced)) return { value: coerced }; + const lowercase = value.toLowerCase(); + if (lowercase === "false") return { value: 0 }; + if (lowercase === "true") return { value: 1 }; + const big = bigIntOfString(value); + return big !== undefined && big <= MAX_BIGINT && big >= MIN_BIGINT ? { value: Number(big) } : undefined; +} + +/** TypeBox's TryBigInt of a string: `12n`, `12.5` (truncated) or `12`; true and false are not reached from tryNumber. */ +function bigIntOfString(value: string): bigint | undefined { + if (/^-?(0|[1-9]\d*)n$/.test(value)) return BigInt(value.slice(0, -1)); + if (/^-?(0|[1-9]\d*)\.\d+$/.test(value)) return BigInt(value.split(".")[0]); + if (/^-?(0|[1-9]\d*)$/.test(value)) return BigInt(value); + return undefined; +} + +function tryBoolean(value: unknown): Converted { + if (typeof value === "bigint") return value === 0n ? { value: false } : value === 1n ? { value: true } : undefined; + if (typeof value === "boolean") return { value }; + if (isFiniteNumber(value)) return value === 0 ? { value: false } : value === 1 ? { value: true } : undefined; + if (value === null || value === undefined) return { value: false }; + if (typeof value !== "string") return undefined; + const lowercase = value.toLowerCase(); + if (lowercase === "false" || value === "0") return { value: false }; + if (lowercase === "true" || value === "1") return { value: true }; + return undefined; +} + +function tryNull(value: unknown): Converted { + if (typeof value === "bigint") return value === 0n ? { value: null } : undefined; + if (typeof value === "boolean") return value === false ? { value: null } : undefined; + if (isFiniteNumber(value)) return value === 0 ? { value: null } : undefined; + if (value === null || value === undefined) return { value: null }; + if (typeof value !== "string") return undefined; + const lowercase = value.toLowerCase(); + return lowercase === "undefined" || lowercase === "null" || value === "" || value === "0" + ? { value: null } + : undefined; +} + +function isFiniteNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value); +} + +// ------------------------------------------------------------------------------------------------------------------- +// Primitive coercion by type (pass 3) +// ------------------------------------------------------------------------------------------------------------------- + +function typesOf(schema: SchemaObject): string[] { + if (typeof schema.type === "string") return [schema.type]; + if (Array.isArray(schema.type)) return schema.type.filter((type): type is string => typeof type === "string"); + return []; +} + +function matchesType(value: unknown, type: string): boolean { + switch (type) { + case "number": + return typeof value === "number"; + case "integer": + return typeof value === "number" && Number.isInteger(value); + case "boolean": + return typeof value === "boolean"; + case "string": + return typeof value === "string"; + case "null": + return value === null; + case "array": + return Array.isArray(value); + case "object": + return isPlainObject(value); + default: + return false; + } +} + +function coercePrimitive(value: unknown, type: string): unknown { + switch (type) { + case "number": + case "integer": { + if (value === null) return 0; + if (typeof value === "string" && value.trim() !== "") { + const parsed = Number(value); + if (type === "number" ? Number.isFinite(parsed) : Number.isInteger(parsed)) return parsed; + } + if (typeof value === "boolean") return value ? 1 : 0; + return value; + } + case "boolean": + if (value === null) return false; + if (value === "true" || value === 1) return true; + if (value === "false" || value === 0) return false; + return value; + case "string": + if (value === null) return ""; + if (typeof value === "number" || typeof value === "boolean") return String(value); + return value; + case "null": + return value === "" || value === 0 || value === false ? null : value; + default: + return value; + } +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/schema/src/explain.ts b/packages/schema/src/explain.ts index da55edb43..3ab363986 100644 --- a/packages/schema/src/explain.ts +++ b/packages/schema/src/explain.ts @@ -22,6 +22,11 @@ import type { SchemaObject, ValidationError } from "./types.ts"; export class Explainer { private readonly c: Compiler; private readonly schema: unknown; + /** + * The errors of `$ref` targets for object values, within one errors() call: a recursive union explains each branch + * to pick the one to blame, and every branch would explain the same children again, exponentially in the depth. + */ + private memo: Map> | undefined = undefined; constructor(compiler: Compiler, schema: unknown) { this.c = compiler; @@ -32,7 +37,11 @@ export class Explainer { /** The errors of `value`, at least one (`value` failed the check). */ errors(value: unknown): ValidationError[] { const out: ValidationError[] = []; - this.walk(this.schema, this.c.r.rootBase, value, "", out); + try { + this.walk(this.schema, this.c.r.rootBase, value, "", out); + } finally { + this.memo = undefined; + } if (out.length === 0) out.push({ path: "", keyword: "schema", message: "invalid" }); return out; } @@ -56,8 +65,7 @@ export class Explainer { out.push({ path: at, keyword, message }); }; if (this.c.draft === "7" && typeof node.$ref === "string") { - const target = this.c.r.resolve(node.$ref, b); - this.walk(target.node, target.base, v, path, out); + this.walkRef(node.$ref, b, v, path, out); return; } if (node.type !== undefined) { @@ -69,7 +77,7 @@ export class Explainer { } } if (Array.isArray(node.enum) && !enumCheck(node.enum)(v)) - push("enum", `expected one of ${JSON.stringify(node.enum)}`); + push("enum", `expected one of ${listValues(node.enum)}`); if (Object.hasOwn(node, "const") && !constCheck(node.const)(v)) push("const", `expected ${JSON.stringify(node.const)}`); if (typeof v === "string" && node.type !== "bytes") this.walkString(node, v, push); @@ -85,10 +93,7 @@ export class Explainer { const branch = this.ok(node.if, b, v) ? node.then : node.else; if (branch !== undefined) this.walk(branch, b, v, path, out); } - if (typeof node.$ref === "string") { - const target = this.c.r.resolve(node.$ref, b); - this.walk(target.node, target.base, v, path, out); - } + if (typeof node.$ref === "string") this.walkRef(node.$ref, b, v, path, out); if (out.length === start && (node.unevaluatedProperties !== undefined || node.unevaluatedItems !== undefined)) { this.walkUnevaluated(node, b, v, path, out); } @@ -97,6 +102,33 @@ export class Explainer { } } + private walkRef(ref: string, base: string, v: unknown, path: string, out: ValidationError[]): void { + const target = this.c.r.resolve(ref, base); + if (target.node === false && this.c.r.isUnresolved(ref, base)) { + out.push({ path, keyword: "$ref", message: `unresolved reference ${ref}` }); + return; + } + if (!isSchemaObject(target.node) || typeof v !== "object" || v === null) { + this.walk(target.node, target.base, v, path, out); + return; + } + this.memo ??= new Map(); + let values = this.memo.get(target.node); + const known = values?.get(v); + if (known !== undefined && known.path === path) { + out.push(...known.errors); + return; + } + const errors: ValidationError[] = []; + this.walk(target.node, target.base, v, path, errors); + if (values === undefined) { + values = new Map(); + this.memo.set(target.node, values); + } + values.set(v, { path, errors }); + out.push(...errors); + } + private walkString(node: SchemaObject, v: string, push: Push): void { const n = codePoints(v); if (typeof node.minLength === "number" && n < node.minLength) { @@ -230,7 +262,7 @@ export class Explainer { const min = draft7 || typeof node.minContains !== "number" ? 1 : node.minContains; const max = draft7 || typeof node.maxContains !== "number" ? Number.POSITIVE_INFINITY : node.maxContains; if (!containsCheck(this.c.check(node.contains, b), min, max)(v)) { - push("contains", `expected between ${min} and ${max} matching items`); + push("contains", containsMessage(min, max)); } } } @@ -258,18 +290,33 @@ export class Explainer { out.push({ path: pointer(path, tag.key), keyword, - message: `expected one of ${JSON.stringify(tag.values)}`, + message: `expected one of ${listValues(tag.values)}`, }); else this.walk(branches[i], b, v, path, out); return; } - let best: ValidationError[] | undefined; + // An object is blamed on the branch that declares most of its properties (the one it was most likely meant for), + // then on the branch whose first error is deepest. + let best: { errors: ValidationError[]; declared: number } | undefined; for (const s of branches) { const errors: ValidationError[] = []; this.walk(s, b, v, path, errors); - if (errors.length > 0 && (best === undefined || errors[0].path.length > best[0].path.length)) best = errors; + if (errors.length === 0) continue; + const declared = isObject(v) ? this.declaredKeys(s, b, v) : 0; + const better = + best === undefined || + declared > best.declared || + (declared === best.declared && errors[0].path.length > best.errors[0].path.length); + if (better) best = { errors, declared }; } - out.push(...(best ?? [{ path, keyword, message: "matches no alternative" }])); + out.push(...(best?.errors ?? [{ path, keyword, message: "matches no alternative" }])); + } + + /** How many of the keys of `v` the branch `s` (through a `$ref`) declares in `properties`. */ + private declaredKeys(s: unknown, b: string, v: Record): number { + const node = isSchemaObject(s) && typeof s.$ref === "string" ? this.c.r.resolve(s.$ref, b).node : s; + const properties = isSchemaObject(node) && isSchemaObject(node.properties) ? node.properties : {}; + return Object.keys(v).filter((k) => Object.hasOwn(properties, k)).length; } private tagOf(branches: readonly unknown[], b: string): { key: string; values: unknown[] } | undefined { @@ -314,6 +361,22 @@ export class Explainer { type Push = (keyword: string, message: string, at?: string) => void; +// Enum errors name the allowed values; a long list (an enum of thousands) is cut, so the message stays readable. +const LISTED_VALUES = 20; + +/** `values` as JSON, at most LISTED_VALUES of them, then how many more there are. */ +function listValues(values: readonly unknown[]): string { + if (values.length <= LISTED_VALUES) return JSON.stringify(values); + const listed = JSON.stringify(values.slice(0, LISTED_VALUES)); + return `${listed.slice(0, -1)}, ... and ${values.length - LISTED_VALUES} more]`; +} + +function containsMessage(min: number, max: number): string { + if (max === Number.POSITIVE_INFINITY) return `expected at least ${min} matching ${min === 1 ? "item" : "items"}`; + if (min === 0) return `expected at most ${max} matching ${max === 1 ? "item" : "items"}`; + return `expected between ${min} and ${max} matching items`; +} + function pointer(path: string, key: string | number): string { return `${path}/${String(key).replace(/~/g, "~0").replace(/\//g, "~1")}`; } diff --git a/packages/schema/src/formats/index.ts b/packages/schema/src/formats/index.ts index 6919b9256..05d8824ed 100644 --- a/packages/schema/src/formats/index.ts +++ b/packages/schema/src/formats/index.ts @@ -14,15 +14,20 @@ import { IsIPv6 } from "./typebox/ipv6.ts"; import { IsIri } from "./typebox/iri.ts"; import { IsIriReference } from "./typebox/iri_reference.ts"; import { IsJsonPointer } from "./typebox/json_pointer.ts"; +import { IsJsonPointerUriFragment } from "./typebox/json_pointer_uri_fragment.ts"; import { IsRegex } from "./typebox/regex.ts"; import { IsRelativeJsonPointer } from "./typebox/relative_json_pointer.ts"; import { IsTime } from "./typebox/time.ts"; import { IsUri } from "./typebox/uri.ts"; import { IsUriReference } from "./typebox/uri_reference.ts"; import { IsUriTemplate } from "./typebox/uri_template.ts"; +import { IsUrl } from "./typebox/url.ts"; import { IsUuid } from "./typebox/uuid.ts"; -/** Every format JSON Schema 2020-12 defines, by name. The checks are adapted from TypeBox 1.3.27 (MIT). */ +/** + * Every format JSON Schema 2020-12 defines, by name, plus `url` and `json-pointer-uri-fragment`, which TypeBox and + * ajv-formats also check. The checks are adapted from TypeBox 1.3.27 (MIT). + */ export const formats: Readonly> = Object.freeze({ date: IsDate, "date-time": IsDateTime, @@ -36,11 +41,13 @@ export const formats: Readonly> = Object.freeze({ iri: IsIri, "iri-reference": IsIriReference, "json-pointer": IsJsonPointer, + "json-pointer-uri-fragment": IsJsonPointerUriFragment, regex: IsRegex, "relative-json-pointer": IsRelativeJsonPointer, time: IsTime, uri: IsUri, "uri-reference": IsUriReference, "uri-template": IsUriTemplate, + url: IsUrl, uuid: IsUuid, }); diff --git a/packages/schema/src/formats/typebox/json_pointer_uri_fragment.ts b/packages/schema/src/formats/typebox/json_pointer_uri_fragment.ts new file mode 100644 index 000000000..cac809cef --- /dev/null +++ b/packages/schema/src/formats/typebox/json_pointer_uri_fragment.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const JsonPointerUriFragment = /^#(?:\/(?:[a-z0-9_\-.!$&'()*+,;:=@]|%[0-9a-f]{2}|~0|~1)*)*$/i; + +/** + * Returns true if the value is a json pointer uri fragment + * @specification https://datatracker.ietf.org/doc/html/rfc6901 + * @source https://github.com/ajv-validator/ajv-formats + */ +export function IsJsonPointerUriFragment(value: string): boolean { + return JsonPointerUriFragment.test(value); +} diff --git a/packages/schema/src/formats/typebox/url.ts b/packages/schema/src/formats/typebox/url.ts new file mode 100644 index 000000000..dfe00c4b8 --- /dev/null +++ b/packages/schema/src/formats/typebox/url.ts @@ -0,0 +1,36 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +/** + * Returns true if the value is a valid Uniform Resource Locator + * @specification https://datatracker.ietf.org/doc/html/rfc3986 + * @specification https://datatracker.ietf.org/doc/html/rfc3987 + */ +export function IsUrl(value: string): boolean { + return URL.canParse(value); +} diff --git a/packages/schema/src/index.ts b/packages/schema/src/index.ts index c421d6fdb..c57ede2e9 100644 --- a/packages/schema/src/index.ts +++ b/packages/schema/src/index.ts @@ -27,5 +27,5 @@ export { union, unknown, } from "./builders.ts"; -export type { Check, CompileOptions, FormatCheck, JsonSchema, ValidationError } from "./types.ts"; -export { compile, Validator } from "./validator.ts"; +export type { Check, CompileOptions, FormatCheck, JsonSchema, ParseResult, ValidationError } from "./types.ts"; +export { CoercingValidator, compile, Validator } from "./validator.ts"; diff --git a/packages/schema/src/resolver.ts b/packages/schema/src/resolver.ts index 9541d2c33..5c8b97543 100644 --- a/packages/schema/src/resolver.ts +++ b/packages/schema/src/resolver.ts @@ -26,6 +26,9 @@ export class Resolver { private readonly root: JsonSchema; private readonly options: CompileOptions; private readonly registry = new Map(); + private readonly rejectUnresolved: boolean; + /** The references that named nothing and resolved to `false` (with `unresolvedRefs: "reject"`). */ + private readonly unresolved = new Set(); private readonly resources = new Map(); private readonly anchors = new Map(); private indexed = false; @@ -34,6 +37,7 @@ export class Resolver { this.draft = options.draft === "7" ? "7" : "2020-12"; this.root = root; this.options = options; + this.rejectUnresolved = options.unresolvedRefs === "reject"; this.rootBase = isSchemaObject(root) ? this.baseFor(root, DEFAULT_BASE) : DEFAULT_BASE; } @@ -61,9 +65,27 @@ export class Resolver { this.index(this.root, DEFAULT_BASE); } - /** The schema a reference names, with the base URI it compiles under. Throws if it names nothing. */ + /** + * The schema a reference names, with the base URI it compiles under. Throws if it names nothing, or, with + * `unresolvedRefs: "reject"`, resolves to `false`, the schema no value passes. + */ resolve(ref: string, base: string): Resolved { this.ensureIndexed(); + if (!this.rejectUnresolved) return this.find(ref, base); + try { + return this.find(ref, base); + } catch { + this.unresolved.add(resolveUri(ref, base)); + return { node: false, base }; + } + } + + /** Whether `ref`, from `base`, named nothing and resolved to `false`. */ + isUnresolved(ref: string, base: string): boolean { + return this.unresolved.has(resolveUri(ref, base)); + } + + private find(ref: string, base: string): Resolved { const uri = resolveUri(ref, base); const doc = documentOf(uri); const fragment = uri.length > doc.length ? decodeURIComponent(uri.slice(doc.length + 1)) : ""; diff --git a/packages/schema/src/types.ts b/packages/schema/src/types.ts index a9e2135d1..d3f635adb 100644 --- a/packages/schema/src/types.ts +++ b/packages/schema/src/types.ts @@ -20,8 +20,21 @@ export interface CompileOptions { readonly formats?: Readonly>; /** Schema documents by URI, for `$ref`s to other documents. */ readonly schemas?: Readonly>; + /** Add `coerce()` and `parse()` to the validator, for loosely typed input (see the README's Coercion section). */ + readonly coerce?: boolean; + /** + * What a `$ref` that names nothing (a missing definition, a document not in `schemas`) does: `"throw"` (default) + * makes compile() throw; `"reject"` compiles it as `false`, so only values at that place fail, with the error + * "unresolved reference". + */ + readonly unresolvedRefs?: "throw" | "reject"; } +/** The result of `parse()`: the coerced value when it is valid, else the errors of the coerced value. */ +export type ParseResult = + | { readonly ok: true; readonly value: T } + | { readonly ok: false; readonly errors: ValidationError[] }; + /** One failure of a value: where it is (a JSON Pointer, "" for the value itself), the failing keyword and a message. */ export interface ValidationError { readonly path: string; diff --git a/packages/schema/src/validator.ts b/packages/schema/src/validator.ts index 3d915b33b..65781e5dd 100644 --- a/packages/schema/src/validator.ts +++ b/packages/schema/src/validator.ts @@ -1,16 +1,29 @@ // compile(): a schema to a Validator, whose `check` is the fast boolean path and whose `errors` explains failures. import type { Static } from "./builders.ts"; +import { Coercer } from "./coerce.ts"; import { Compiler, RESTART } from "./compiler.ts"; import { Explainer } from "./explain.ts"; -import type { Check, CompileOptions, JsonSchema, ValidationError } from "./types.ts"; +import type { Check, CompileOptions, JsonSchema, ParseResult, ValidationError } from "./types.ts"; /** * Compiles `schema` into a Validator. Throws if the schema is malformed or a `$ref` names nothing (register other - * documents with `options.schemas`). The schema is read, never modified. + * documents with `options.schemas`). The schema is read, never modified. With `coerce: true` the validator also + * coerces and parses loosely typed input. */ -export function compile(schema: S, options: CompileOptions = NO_OPTIONS): Validator> { - return new Validator(schema, options, compileCheck(schema, options)); +export function compile( + schema: S, + options: CompileOptions & { readonly coerce: true }, +): CoercingValidator>; +export function compile(schema: S, options?: CompileOptions): Validator>; +export function compile( + schema: S, + options: CompileOptions = NO_OPTIONS, +): Validator> | CoercingValidator> { + const check = compileCheck(schema, options); + return options.coerce === true + ? new CoercingValidator(schema, options, check) + : new Validator(schema, options, check); } const NO_OPTIONS: CompileOptions = Object.freeze({}); @@ -19,8 +32,8 @@ const NO_OPTIONS: CompileOptions = Object.freeze({}); export class Validator { /** Whether `value` is valid. A plain function: it may be passed around without its validator. */ readonly check: (value: unknown) => value is T; - private readonly schema: JsonSchema; - private readonly options: CompileOptions; + protected readonly schema: JsonSchema; + protected readonly options: CompileOptions; private explainer: Explainer | undefined = undefined; constructor(schema: JsonSchema, options: CompileOptions, check: Check) { @@ -41,6 +54,28 @@ export class Validator { } } +/** A validator compiled with `coerce: true`: it also coerces loosely typed input towards the schema. */ +export class CoercingValidator extends Validator { + private coercer: Coercer | undefined = undefined; + + /** + * `value` coerced towards the schema: numeric strings to numbers, `"true"` to `true`, a single value to a one-item + * array, an optional `null` dropped, the first union branch the value converts to (the README lists every rule). + * Only the changed paths are copied; `value` comes back itself when nothing changes, and is never modified. Values + * that cannot be coerced stay as they are. The rules are built on the first call. + */ + coerce(value: unknown): unknown { + this.coercer ??= new Coercer(compiler(this.schema, this.options), this.schema); + return this.coercer.coerce(value); + } + + /** Coerces `value`, then checks it: the coerced value when it is valid, else its errors. */ + parse(value: unknown): ParseResult { + const coerced = this.coerce(value); + return this.check(coerced) ? { ok: true, value: coerced } : { ok: false, errors: this.errors(coerced) }; + } +} + function compileCheck(schema: JsonSchema, options: CompileOptions): Check { return compiler(schema, options).root(); } diff --git a/packages/schema/test/coerce-typebox.test.ts b/packages/schema/test/coerce-typebox.test.ts new file mode 100644 index 000000000..f9a83140d --- /dev/null +++ b/packages/schema/test/coerce-typebox.test.ts @@ -0,0 +1,1344 @@ +// biome-ignore-all lint/suspicious/noTemplateCurlyInString: TypeBox template literal syntax, as in the original tests. +// biome-ignore-all lint/correctness/noPrecisionLoss: the original tests' literal, kept as written. +// TypeBox 1.3.27's Value.Convert tests (test/typebox/runtime/value/convert, MIT, (c) Haydn Paterson), ported to the +// conversion pass of pi-schema's coercion. Schemas are still built with TypeBox: its schemas are JSON Schema, and the +// port reads only their JSON structure, never TypeBox's kind marks. Run with vitest. + +import { Type } from "typebox"; +import { describe, expect, it } from "vitest"; +import { Coercer } from "../src/coerce.ts"; +import { Compiler } from "../src/compiler.ts"; +import { formats } from "../src/formats/index.ts"; +import { compile } from "../src/index.ts"; + +/** Pass 2 alone, as TypeBox's Value.Convert(schema, value). */ +function convert(schema: unknown, value: unknown): unknown { + return new Coercer(new Compiler(schema as object, { formats }, false), schema).convertOnly(value); +} + +const Value = { + Convert: convert, + Check: (schema: unknown, value: unknown) => compile(schema as object, { formats }).check(value), +}; +const Convert = convert; +const Assert = { + IsEqual: (actual: unknown, expected: unknown) => expect(actual).toEqual(expected), + IsTrue: (value: boolean) => expect(value).toBe(true), + IsFalse: (value: boolean) => expect(value).toBe(false), +}; + +describe("Value.Convert.Any", () => { + it("Should Convert 1", () => { + const T = Type.Any(); + const R = Value.Convert(T, null); + Assert.IsEqual(R, null); + }); + it("Should Convert 2", () => { + const T = Type.Any(); + const R = Value.Convert(T, undefined); + Assert.IsEqual(R, undefined); + }); + it("Should Convert 3", () => { + const T = Type.Any(); + const R = Value.Convert(T, "hello"); + Assert.IsEqual(R, "hello"); + }); + it("Should Convert 4", () => { + const T = Type.Any(); + const R = Value.Convert(T, 42); + Assert.IsEqual(R, 42); + }); + it("Should Convert 5", () => { + const T = Type.Any(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + it("Should Convert 6", () => { + const T = Type.Any(); + const R = Value.Convert(T, { x: 1 }); + Assert.IsEqual(R, { x: 1 }); + }); + it("Should Convert 7", () => { + const T = Type.Any(); + const R = Value.Convert(T, [1, 2, 3]); + Assert.IsEqual(R, [1, 2, 3]); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("Should Convert 8", () => { + const S = Symbol("x-symbol"); + const T = Type.Symbol(); + const R = Value.Convert(T, S); + Assert.IsEqual(R, S); + }); +}); + +describe("Value.Convert.Array", () => { + it("Should Convert 1", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, [1, 3.14, "1", "3.14", true, false, "true", "false", "hello"]); + Assert.IsEqual(R, [1, 3.14, 1, 3.14, 1, 0, 1, 0, "hello"]); + }); + it("Should Convert 2", () => { + const T = Type.Array(Type.Boolean()); + const R = Value.Convert(T, [1, 3.14, "1", "3.14", true, false, "true", "false", "hello"]); + Assert.IsEqual(R, [true, 3.14, true, "3.14", true, false, true, false, "hello"]); + }); + it("Should Convert 3", () => { + const T = Type.Array(Type.String()); + const R = Value.Convert(T, [1, 3.14, "1", "3.14", true, false, "true", "false", "hello"]); + Assert.IsEqual(R, ["1", "3.14", "1", "3.14", "true", "false", "true", "false", "hello"]); + }); + // ------------------------------------------------------------------ + // Added support for Array coercion + // + // https://github.com/sinclairzx81/typebox/issues/1469 + // ------------------------------------------------------------------ + it("Should Convert 4", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, "1"); + Assert.IsEqual(R, [1]); + }); + it("Should Convert 5", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, []); + Assert.IsEqual(R, []); + }); + it("Should Convert 6", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, 42); + Assert.IsEqual(R, [42]); + }); + it("Should Convert 7", () => { + const T = Type.Array(Type.Boolean()); + const R = Value.Convert(T, true); + Assert.IsEqual(R, [true]); + }); + it("Should Convert 8", () => { + const T = Type.Array(Type.Null()); + const R = Value.Convert(T, null); + Assert.IsEqual(R, [null]); + }); + it("Should Convert 9", () => { + const T = Type.Array(Type.Unknown()); + const R = Value.Convert(T, undefined); + Assert.IsEqual(R, [undefined]); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("Should Convert 10", () => { + const T = Type.Array(Type.BigInt()); + const R = Value.Convert(T, 1n); + Assert.IsEqual(R, [1n]); + }); + it("Should Convert 11", () => { + const T = Type.Array(Type.Object({})); + const R = Value.Convert(T, {}); + Assert.IsEqual(R, [{}]); + }); + it("Should Convert 12", () => { + const T = Type.Array(Type.Object({ x: Type.Number() })); + const R = Value.Convert(T, { x: 1 }); + Assert.IsEqual(R, [{ x: 1 }]); + }); + it("Should Convert 13", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, "42"); + Assert.IsEqual(R, [42]); + }); + it("Should Convert 14", () => { + const T = Type.Array(Type.Boolean()); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, [true]); + }); + it("Should Convert 15", () => { + const T = Type.Array(Type.Boolean()); + const R = Value.Convert(T, 0); + Assert.IsEqual(R, [false]); + }); + it("Should Convert 16", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, "hello"); + Assert.IsEqual(R, ["hello"]); + }); + it("Should Convert 17", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, [1]); + Assert.IsEqual(R, [1]); + }); + it("Should Convert 18", () => { + const T = Type.Array(Type.Array(Type.Number())); + const R = Value.Convert(T, [1, 2, 3]); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 19", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, { 0: 1, length: 1 }); + Assert.IsEqual(R, [{ 0: 1, length: 1 }]); + }); + it("Should Convert 20", () => { + const T = Type.Array(Type.Array(Type.Number())); + const R = Value.Convert(T, 42); + Assert.IsTrue(Value.Check(T, R)); + }); + // ------------------------------------------------------------------ + // Union + // ------------------------------------------------------------------ + it("Should Convert 21", () => { + const T = Type.Array(Type.Union([Type.Number(), Type.String()])); + const R = Value.Convert(T, 1); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 22", () => { + const T = Type.Array(Type.Union([Type.Number(), Type.String()])); + const R = Value.Convert(T, "hello"); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 23", () => { + const T = Type.Array(Type.Union([Type.Number(), Type.String()])); + const R = Value.Convert(T, [1, "hello", true, "3.14"]); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 24", () => { + const T = Type.Union([Type.Array(Type.Number()), Type.String()]); + const R = Value.Convert(T, "1"); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 25", () => { + const T = Type.Array(Type.Union([Type.Array(Type.Number()), Type.String()])); + const R = Value.Convert(T, 42); + Assert.IsTrue(Value.Check(T, R)); + }); + // ------------------------------------------------------------------ + // Embedded Intersect + // ------------------------------------------------------------------ + it("Should Convert 26", () => { + const T = Type.Array(Type.Intersect([Type.Object({ x: Type.Number() }), Type.Object({ y: Type.Number() })])); + const R = Value.Convert(T, { x: 1, y: 2 }); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 27", () => { + const T = Type.Array(Type.Intersect([Type.Object({ x: Type.Number() }), Type.Object({ y: Type.Number() })])); + const R = Value.Convert(T, [ + { x: "1", y: "2" }, + { x: "3", y: "4" }, + ]); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 28", () => { + const T = Type.Intersect([Type.Array(Type.Number()), Type.Array(Type.Number())]); + const R = Value.Convert(T, 42); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 29", () => { + const T = Type.Intersect([Type.Array(Type.Number()), Type.Object({ x: Type.Number() })]); + const R = Value.Convert(T, 42); + Assert.IsFalse(Value.Check(T, R)); + }); +}); + +describe("Value.Convert.Boolean", () => { + it("Should Convert 1", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, "1"); + Assert.IsEqual(R, true); + }); + it("Should Convert 2", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, "3.14"); + Assert.IsEqual(R, "3.14"); + }); + it("Should Convert 3", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, true); + }); + it("Should Convert 4", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, "false"); + Assert.IsEqual(R, false); + }); + it("Should Convert 5", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, 1); + Assert.IsEqual(R, true); + }); + it("Should Convert 6", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, 3.14); + Assert.IsEqual(R, 3.14); + }); + it("Should Convert 7", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, 1.1); + Assert.IsEqual(R, 1.1); + }); + it("Should Convert 8", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + it("Should Convert 9", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, false); + }); + // ---------------------------------------------------------- + // Casts + // ---------------------------------------------------------- + it("Should Convert 10", () => { + const value = "hello"; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, "hello"); + }); + it("Should Convert 11", () => { + const value = "true"; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 12", () => { + const value = "TRUE"; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 13", () => { + const value = "false"; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, false); + }); + it("Should Convert 14", () => { + const value = "0"; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, false); + }); + it("Should Convert 15", () => { + const value = "1"; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 16", () => { + const value = "0"; + const result = Value.Convert(Type.Boolean({ default: true }), value); + Assert.IsEqual(result, false); + }); + it("Should Convert 17", () => { + const value = "1"; + const result = Value.Convert(Type.Boolean({ default: false }), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 18", () => { + const value = "2"; + const result = Value.Convert(Type.Boolean({ default: true }), value); + Assert.IsEqual(result, "2"); + }); + it("Should Convert 19", () => { + const value = 0; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, false); + }); + it("Should Convert 20", () => { + const value = 1n; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 21", () => { + const value = 1; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 22", () => { + const value = 2; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, 2); + }); + it("Should Convert 23", () => { + const value = 0; + const result = Value.Convert(Type.Boolean({ default: true }), value); + Assert.IsEqual(result, false); + }); + it("Should Convert 24", () => { + const value = 1; + const result = Value.Convert(Type.Boolean({ default: false }), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 25", () => { + const value = 2; + const result = Value.Convert(Type.Boolean({ default: true }), value); + Assert.IsEqual(result, 2); + }); + it("Should Convert 26", () => { + const value = true; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 27", () => { + const value = false; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, false); + }); + it("Should Convert 28", () => { + const value = {}; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, {}); + }); + it("Should Convert 29", () => { + const value = [] as any[]; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, []); + }); + // ------------------------------------------------------------------ + // Coverage + // ------------------------------------------------------------------ + it("Should Convert 17", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, BigInt(0)); + Assert.IsEqual(R, false); + }); + it("Should Convert 18", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, BigInt(1)); + Assert.IsEqual(R, true); + }); + it("Should Convert 19", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, BigInt(2)); + Assert.IsEqual(R, BigInt(2)); + }); + it("Should Convert 20", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + it("Should Convert 21", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, false); + }); + it("Should Convert 22", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, null); + Assert.IsEqual(R, false); + }); + it("Should Convert 23", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, undefined); + Assert.IsEqual(R, false); + }); +}); + +describe("Value.Convert.Enum", () => { + it("Should Convert 1", () => { + const T = Type.Object({ + x: Type.Enum([1, 2, "hello"]), + }); + const R0 = Value.Convert(T, { x: "1" }); + const R1 = Value.Convert(T, { x: "2" }); + const R2 = Value.Convert(T, { x: "hello" }); + const R3 = Value.Convert(T, { x: "world" }); + Assert.IsEqual(R0, { x: 1 }); + Assert.IsEqual(R1, { x: 2 }); + Assert.IsEqual(R2, { x: "hello" }); + Assert.IsEqual(R3, { x: "world" }); + }); +}); + +describe("Value.Convert.Integer", () => { + it("Should Convert 1", () => { + const T = Type.Integer(); + const R = Value.Convert(T, "3.14"); + Assert.IsEqual(R, 3); + }); + it("Should Convert 2", () => { + const T = Type.Integer(); + const R = Value.Convert(T, "42"); + Assert.IsEqual(R, 42); + }); + it("Should Convert 3", () => { + const T = Type.Integer(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, 1); + }); + it("Should Convert 4", () => { + const T = Type.Integer(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, 0); + }); + it("Should Convert 5", () => { + const T = Type.Integer(); + const R = Value.Convert(T, 3.14); + Assert.IsEqual(R, 3); + }); + // ---------------------------------------------------------- + // Casts + // ---------------------------------------------------------- + it("Should Convert 6", () => { + const value = "hello"; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, "hello"); + }); + it("Should Convert 7", () => { + const value = "3.14"; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, 3); + }); + it("Should Convert 8", () => { + const value = "-0"; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, -0); + }); + it("Should Convert 9", () => { + const value = "-100"; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, -100); + }); + it("Should Convert 10", () => { + const value = 42; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, 42); + }); + it("Should Convert 11", () => { + const value = true; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, 1); + }); + it("Should Convert 12", () => { + const value = false; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, 0); + }); + it("Should Convert 13", () => { + const value = {}; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, {}); + }); + it("Should Convert 14", () => { + const value = [] as any[]; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, []); + }); + // ---------------------------------------------------------- + // https://github.com/sinclairzx81/typebox/issues/1147 + // ---------------------------------------------------------- + it("Should Convert 15", () => { + const N = 1738213389080; + const R = Value.Convert(Type.Integer(), N); + Assert.IsEqual(R, N); + }); + it("Should Convert 16", () => { + const N = 1738213389080.5555; + const R = Value.Convert(Type.Integer(), N); + Assert.IsEqual(R, 1738213389080); + }); + it("Should Convert 17", () => { + const N = "1738213389080"; + const R = Value.Convert(Type.Integer(), N); + Assert.IsEqual(R, 1738213389080); + }); + it("Should Convert 18", () => { + const N = "1738213389080.555"; + const R = Value.Convert(Type.Integer(), N); + Assert.IsEqual(R, 1738213389080); + }); +}); + +describe("Value.Convert.Intersect", () => { + it("Should Convert intersected objects", () => { + const T = Type.Intersect([Type.Object({ x: Type.Number() }), Type.Object({ y: Type.Number() })]); + const R = Value.Convert(T, { x: "1", y: "2" }); + Assert.IsEqual(R, { x: 1, y: 2 }); + }); + // ---------------------------------------------------------------- + // Intersection Complex + // ---------------------------------------------------------------- + it("Should complex intersect 1", () => { + const T = Type.Intersect([Type.Number(), Type.Object({ x: Type.Number() }), Type.Object({ y: Type.Number() })]); + const R = Value.Convert(T, { x: "1", y: "2" }); + Assert.IsEqual(R, { x: 1, y: 2 }); + }); + it("Should complex intersect 2", () => { + const T = Type.Intersect([Type.Object({ x: Type.Number() }), Type.Object({ y: Type.Number() }), Type.Number()]); + const R = Value.Convert(T, { x: "3", y: "4" }); + Assert.IsEqual(R, { x: 3, y: 4 }); + }); + // Differs from TypeBox: allOf converts with each member in turn; TypeBox first merges an intersection into one object and drops the number. + it("Should complex intersect 3", () => { + const T = Type.Intersect([ + Type.Number(), // lost in Evaluate + Type.Object({ x: Type.Number() }), + Type.Object({ y: Type.Number() }), + ]); + const R = Value.Convert(T, "123"); + Assert.IsEqual(R, 123); + }); +}); + +describe("Value.Convert.Literal", () => { + it("LiteralString: Should convert 1", () => { + const T = Type.Literal("1"); + const R = Value.Convert(T, 1); + Assert.IsEqual(R, "1"); + }); + it("LiteralString: Should convert 2", () => { + const T = Type.Literal("1"); + const R = Value.Convert(T, 2); + Assert.IsEqual(R, 2); + }); + it("LiteralString: Should convert 3", () => { + const T = Type.Literal("true"); + const R = Value.Convert(T, true); + Assert.IsEqual(R, "true"); + }); + // ------------------------------------------------------------------ + // LiteralNumber + // ------------------------------------------------------------------ + it("LiteralNumber: Should convert 1", () => { + const T = Type.Literal(3.14); + const R = Value.Convert(T, "3.14"); + Assert.IsEqual(R, 3.14); + }); + it("LiteralNumber: Should convert 2", () => { + const T = Type.Literal(3.14); + const R = Value.Convert(T, "3.15"); + Assert.IsEqual(R, "3.15"); + }); + it("LiteralNumber: Should convert 3", () => { + const T = Type.Literal(1); + const R = Value.Convert(T, true); + Assert.IsEqual(R, 1); + }); + it("LiteralNumber: Should convert 4", () => { + const T = Type.Literal(0); + const R = Value.Convert(T, false); + Assert.IsEqual(R, 0); + }); + it("LiteralNumber: Should convert 5", () => { + const T = Type.Literal(2); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + // ------------------------------------------------------------------ + // LiteralBoolean + // ------------------------------------------------------------------ + it("LiteralBoolean: Should convert 1", () => { + const T = Type.Literal(true); + const R = Value.Convert(T, 3.14); + Assert.IsEqual(R, 3.14); + }); + it("LiteralBoolean: Should convert 2", () => { + const T = Type.Literal(true); + const R = Value.Convert(T, 1); + Assert.IsEqual(R, true); + }); + it("LiteralBoolean: Should convert 3", () => { + const T = Type.Literal(true); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, true); + }); + it("LiteralBoolean: Should convert 4", () => { + const T = Type.Literal(false); + const R = Value.Convert(T, "false"); + Assert.IsEqual(R, false); + }); + it("LiteralBoolean: Should convert 5", () => { + const T = Type.Literal(true); + const R = Value.Convert(T, "1"); + Assert.IsEqual(R, true); + }); + it("LiteralBoolean: Should convert 6", () => { + const T = Type.Literal(false); + const R = Value.Convert(T, "0"); + Assert.IsEqual(R, false); + }); + it("LiteralBoolean: Should convert 7", () => { + const T = Type.Literal(false); + const R = Value.Convert(T, "2"); + Assert.IsEqual(R, "2"); + }); + // ------------------------------------------------------------------ + // LiteralBigInt + // ------------------------------------------------------------------ + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 1", () => { + const T = Type.Literal(100n); + const R = Value.Convert(T, 100n); + Assert.IsEqual(R, 100n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 2", () => { + const T = Type.Literal(100n); + const R = Value.Convert(T, 100); + Assert.IsEqual(R, 100n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 3", () => { + const T = Type.Literal(1n); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, 1n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 4", () => { + const T = Type.Literal(0n); + const R = Value.Convert(T, "false"); + Assert.IsEqual(R, 0n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 5", () => { + const T = Type.Literal(1n); + const R = Value.Convert(T, "1"); + Assert.IsEqual(R, 1n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 6", () => { + const T = Type.Literal(0n); + const R = Value.Convert(T, "0"); + Assert.IsEqual(R, 0n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 7", () => { + const T = Type.Literal(100n); + const R = Value.Convert(T, "2"); + Assert.IsEqual(R, "2"); + }); + // ------------------------------------------------------------------ + // LiteralBigInt + // ------------------------------------------------------------------ + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 1", () => { + const T = Type.Literal(100n); + const R = Value.Convert(T, 100n); + Assert.IsEqual(R, 100n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 2", () => { + const T = Type.Literal(100n); + const R = Value.Convert(T, 100); + Assert.IsEqual(R, 100n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 3", () => { + const T = Type.Literal(1n); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, 1n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 4", () => { + const T = Type.Literal(0n); + const R = Value.Convert(T, "false"); + Assert.IsEqual(R, 0n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 5", () => { + const T = Type.Literal(1n); + const R = Value.Convert(T, "1"); + Assert.IsEqual(R, 1n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 6", () => { + const T = Type.Literal(0n); + const R = Value.Convert(T, "0"); + Assert.IsEqual(R, 0n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 7", () => { + const T = Type.Literal(100n); + const R = Value.Convert(T, "2"); + Assert.IsEqual(R, "2"); + }); +}); + +describe("Value.Convert.Never", () => { + it("Should not 1", () => { + const T = Type.Never(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + it("Should not 2", () => { + const T = Type.Never(); + const R = Value.Convert(T, 42); + Assert.IsEqual(R, 42); + }); + it("Should not 3", () => { + const T = Type.Never(); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, "true"); + }); +}); + +describe("Value.Convert.Null", () => { + it("Should Convert 1", () => { + const T = Type.Null(); + const R = Value.Convert(T, "null"); + Assert.IsEqual(R, null); + }); + it("Should Convert 2", () => { + const T = Type.Null(); + const R = Value.Convert(T, "NULL"); + Assert.IsEqual(R, null); + }); + it("Should Convert 3", () => { + const T = Type.Null(); + const R = Value.Convert(T, "nil"); + Assert.IsEqual(R, "nil"); + }); + it("Should Convert 4", () => { + const T = Type.Null(); + const R = Value.Convert(T, "undefined"); + Assert.IsEqual(R, null); + }); + it("Should Convert 5", () => { + const T = Type.Null(); + const R = Value.Convert(T, "hello"); + Assert.IsEqual(R, "hello"); + }); + it("Should Convert 6", () => { + const T = Type.Null(); + const R = Value.Convert(T, null); + Assert.IsEqual(R, null); + }); + it("Should Convert 7", () => { + const T = Type.Null(); + const R = Value.Convert(T, 0); + Assert.IsEqual(R, null); + }); + it("Should Convert 8", () => { + const T = Type.Null(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, null); + }); + it("Should Convert 9", () => { + const T = Type.Null(); + const R = Value.Convert(T, ""); + Assert.IsEqual(R, null); + }); + // ------------------------------------------------------------------ + // Coverage + // ------------------------------------------------------------------ + it("Should Convert 10", () => { + const T = Type.Null(); + const R = Value.Convert(T, BigInt(0)); + Assert.IsEqual(R, null); + }); + it("Should Convert 11", () => { + const T = Type.Null(); + const R = Value.Convert(T, BigInt(1)); + Assert.IsEqual(R, BigInt(1)); + }); + it("Should Convert 12", () => { + const T = Type.Null(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, null); + }); + it("Should Convert 13", () => { + const T = Type.Null(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + it("Should Convert 14", () => { + const T = Type.Null(); + const R = Value.Convert(T, 0); + Assert.IsEqual(R, null); + }); + it("Should Convert 15", () => { + const T = Type.Null(); + const R = Value.Convert(T, 1); + Assert.IsEqual(R, 1); + }); + it("Should Convert 16", () => { + const T = Type.Null(); + const R = Value.Convert(T, null); + Assert.IsEqual(R, null); + }); + it("Should Convert 17", () => { + const T = Type.Null(); + const R = Value.Convert(T, undefined); + Assert.IsEqual(R, null); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("Should Convert 18", () => { + const T = Type.Null(); + const S = Symbol("hello"); + const R = Value.Convert(T, S); + Assert.IsEqual(R, S); + }); +}); + +describe("Value.Convert.Number", () => { + it("Should Convert 1", () => { + const T = Type.Number(); + const R = Value.Convert(T, "3.14"); + Assert.IsEqual(R, 3.14); + }); + it("Should Convert 2", () => { + const T = Type.Number(); + const R = Value.Convert(T, "42"); + Assert.IsEqual(R, 42); + }); + it("Should Convert 3", () => { + const T = Type.Number(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, 1); + }); + it("Should Convert 4", () => { + const T = Type.Number(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, 0); + }); + it("Should Convert 5", () => { + const T = Type.Number(); + const R = Value.Convert(T, 3.14); + Assert.IsEqual(R, 3.14); + }); + // ---------------------------------------------------------- + // Casts + // ---------------------------------------------------------- + it("Should Convert 6", () => { + const T = Type.Number(); + const R = Value.Convert(T, "hello"); + Assert.IsEqual(R, "hello"); + }); + it("Should Convert 7", () => { + const T = Type.Number(); + const R = Value.Convert(T, "3.14"); + Assert.IsEqual(R, 3.14); + }); + it("Should Convert 8", () => { + const T = Type.Number(); + const R = Value.Convert(T, "-0"); + Assert.IsEqual(R, -0); + }); + it("Should Convert 9", () => { + const T = Type.Number(); + const R = Value.Convert(T, "-100"); + Assert.IsEqual(R, -100); + }); + it("Should Convert 10", () => { + const T = Type.Number(); + const R = Value.Convert(T, 42); + Assert.IsEqual(R, 42); + }); + it("Should Convert 11", () => { + const T = Type.Number(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, 1); + }); + it("Should Convert 12", () => { + const T = Type.Number(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, 0); + }); + it("Should Convert 13", () => { + const T = Type.Number(); + const R = Value.Convert(T, {}); + Assert.IsEqual(R, {}); + }); + it("Should Convert 14", () => { + const T = Type.Number(); + const R = Value.Convert(T, []); + Assert.IsEqual(R, []); + }); + // ------------------------------------------------------------------ + // Coverage + // ------------------------------------------------------------------ + it("Should Convert 10", () => { + const T = Type.Number(); + const R = Value.Convert(T, BigInt(0)); + Assert.IsEqual(R, 0); + }); + it("Should Convert 11", () => { + const T = Type.Number(); + const R = Value.Convert(T, BigInt(10000000000000000000000000000000000000000000000000000)); + Assert.IsEqual(R, BigInt(10000000000000000000000000000000000000000000000000000)); + }); + it("Should Convert 12", () => { + const T = Type.Number(); + const R = Value.Convert(T, null); + Assert.IsEqual(R, 0); + }); + it("Should Convert 13", () => { + const T = Type.Number(); + const R = Value.Convert(T, "false"); + Assert.IsEqual(R, 0); + }); + it("Should Convert 14", () => { + const T = Type.Number(); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, 1); + }); + it("Should Convert 15", () => { + const T = Type.Number(); + const R = Value.Convert(T, "123"); + Assert.IsEqual(R, 123); + }); + it("Should Convert 16", () => { + const T = Type.Number(); + const R = Value.Convert(T, "123.456"); + Assert.IsEqual(R, 123.456); + }); + it("Should Convert 17", () => { + const T = Type.Number(); + const R = Value.Convert(T, "123.456"); + Assert.IsEqual(R, 123.456); + }); + it("Should Convert 18", () => { + const T = Type.Number(); + const R = Value.Convert(T, "123n"); + Assert.IsEqual(R, 123); + }); + it("Should Convert 19", () => { + const T = Type.Number(); + const R = Value.Convert(T, undefined); + Assert.IsEqual(R, 0); + }); + it("Should Convert 20", () => { + const T = Type.Number(); + const R = Value.Convert(T, `90071992547409910n`); // trailing 0 too large + Assert.IsEqual(R, "90071992547409910n"); + }); +}); + +describe("Value.Convert.Object", () => { + it("Should Convert 1", () => { + const T = Type.Object({ + x: Type.Number(), + y: Type.Boolean(), + z: Type.Boolean(), + }); + const R = Value.Convert(T, { x: "42", y: "true", z: "hello" }); + Assert.IsEqual(R, { x: 42, y: true, z: "hello" }); + }); + it("Should Convert 2", () => { + const T = Type.Object({ + x: Type.Number(), + y: Type.Boolean(), + }); + const R = Value.Convert(T, { x: "42", y: "true", z: "hello" }); + Assert.IsEqual(R, { x: 42, y: true, z: "hello" }); + }); + it("Should Convert 3", () => { + const T = Type.Object({ x: Type.Number() }); + const R = Value.Convert(T, {}); + Assert.IsEqual(R, {}); + }); + it("Should Convert 4", () => { + const T = Type.Object( + { x: Type.Number() }, + { + additionalProperties: Type.Null(), + }, + ); + const R = Value.Convert(T, { x: "1", y: 0 }); + Assert.IsEqual(R, { x: 1, y: null }); + }); + // ------------------------------------------------------------------ + // IsOptionalUndefined + // + // https://github.com/sinclairzx81/typebox/issues/1336#issuecomment-3312808962 + // ------------------------------------------------------------------ + it("Should Convert 5", () => { + const T = Type.Object({ x: Type.Optional(Type.Number()) }); + const R = Value.Convert(T, { x: 1 }); + Assert.IsEqual(R, { x: 1 }); + }); + it("Should Convert 6", () => { + const T = Type.Object({ x: Type.Optional(Type.Number()) }); + const R = Value.Convert(T, { x: "1" }); + Assert.IsEqual(R, { x: 1 }); + }); + it("Should Convert 7", () => { + const T = Type.Object({ x: Type.Optional(Type.Number()) }); + const R = Value.Convert(T, { x: undefined }); + Assert.IsEqual(R, { x: undefined }); + }); +}); + +describe("Value.Convert.Record", () => { + it("Should Convert 1", () => { + const T = Type.Record(Type.String(), Type.Number()); + const R = Value.Convert(T, { x: "42", y: "24", z: "hello" }); + Assert.IsEqual(R, { x: 42, y: 24, z: "hello" }); + }); + // ---------------------------------------------------------------- + // https://github.com/sinclairzx81/typebox/issues/930 + // ---------------------------------------------------------------- + it("Should Convert 2", () => { + const T = Type.Union([Type.Null(), Type.Record(Type.Number(), Type.Any())]); + const R = Value.Convert(T, {}); + Assert.IsEqual(R, {}); + }); + it("Should Convert 3", () => { + const T = Type.Union([Type.Record(Type.Number(), Type.Any()), Type.Null()]); + const R = Value.Convert(T, {}); + Assert.IsEqual(R, {}); + }); + it("Should Convert 4", () => { + const T = Type.Union([Type.Null(), Type.Record(Type.Number(), Type.Any())]); + const R = Value.Convert(T, null); + Assert.IsEqual(R, null); + }); + it("Should Convert 5", () => { + const T = Type.Union([Type.Record(Type.Number(), Type.Any()), Type.Null()]); + const R = Value.Convert(T, null); + Assert.IsEqual(R, null); + }); + it("Should Convert 6", () => { + const T = Type.Union([Type.Null(), Type.Record(Type.Number(), Type.Any())]); + const R = Value.Convert(T, "NULL"); + Assert.IsEqual(R, null); + }); + it("Should Convert 7", () => { + const T = Type.Union([Type.Record(Type.Number(), Type.Any()), Type.Null()]); + const R = Value.Convert(T, "NULL"); + Assert.IsEqual(R, null); + }); + it("Should Convert 8", () => { + const T = Type.Record(Type.Number(), Type.Number(), { + additionalProperties: Type.Null(), + }); + const R = Value.Convert(T, { "0": "1", x: 0 }); + Assert.IsEqual(R, { "0": 1, x: null }); + }); +}); + +describe("Value.Convert.String", () => { + it("Should Convert 1", () => { + const T = Type.String(); + const R = Value.Convert(T, 3.14); + Assert.IsEqual(R, "3.14"); + }); + it("Should Convert 2", () => { + const T = Type.String(); + const R = Value.Convert(T, 3); + Assert.IsEqual(R, "3"); + }); + it("Should Convert 3", () => { + const T = Type.String(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, "true"); + }); + it("Should Convert 4", () => { + const T = Type.String(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, "false"); + }); + it("Should Convert 5", () => { + const T = Type.String(); + const R = Value.Convert(T, BigInt(12345)); + Assert.IsEqual(R, "12345"); + }); + // ---------------------------------------------------------- + // Casts + // ---------------------------------------------------------- + it("Should Convert 6", () => { + const T = Type.String(); + const R = Value.Convert(T, "hello"); + Assert.IsEqual(R, "hello"); + }); + it("Should Convert 7", () => { + const T = Type.String(); + const R = Value.Convert(T, 42); + Assert.IsEqual(R, "42"); + }); + it("Should Convert 8", () => { + const T = Type.String(); + const R = Value.Convert(T, 42n); + Assert.IsEqual(R, "42"); + }); + it("Should Convert 9", () => { + const T = Type.String(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, "true"); + }); + it("Should Convert 10", () => { + const T = Type.String(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, "false"); + }); + it("Should Convert 11", () => { + const T = Type.String(); + const R = Value.Convert(T, {}); + Assert.IsEqual(R, {}); + }); + it("Should Convert 12", () => { + const T = Type.String(); + const R = Value.Convert(T, []); + Assert.IsEqual(R, []); + }); + // ------------------------------------------------------------------ + // Coverage + // ------------------------------------------------------------------ + it("Should Convert 13", () => { + const T = Type.String(); + const R = Value.Convert(T, null); + Assert.IsEqual(R, "null"); + }); + it("Should Convert 14", () => { + const T = Type.String(); + const R = Value.Convert(T, undefined); + Assert.IsEqual(R, ""); + }); +}); + +describe("Value.Convert.TemplateLiteral", () => { + // Differs from TypeBox: a template literal is a string with a pattern in JSON Schema; TypeBox converts a finite one as a union of literals. + it("Should Convert 1", () => { + const T = Type.TemplateLiteral("${1 | 2 | 3}"); + const R0 = Value.Convert(T, 1); + const R1 = Value.Convert(T, 2); + const R2 = Value.Convert(T, 3); + const R3 = Value.Convert(T, 4); + Assert.IsEqual(R0, "1"); + Assert.IsEqual(R1, "2"); + Assert.IsEqual(R2, "3"); + Assert.IsEqual(R3, "4"); + }); + // ------------------------------------------------------------------ + // https://github.com/sinclairzx81/typebox/issues/1510 + // ------------------------------------------------------------------ + it("Should Convert 2", () => { + const T = Type.Object({ + test: Type.TemplateLiteral('${number}${"a"|"b"|"c"}'), + }); + const R0 = Value.Convert(T, { test: "123b" }); + Assert.IsEqual(R0, { test: "123b" }); + }); +}); + +describe("Value.Convert.Tuple", () => { + it("Should Convert 1", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, ["1", "true"]); + Assert.IsEqual(R, [1, 1]); + }); + it("Should Convert 2", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, ["1"]); + Assert.IsEqual(R, [1]); + }); + it("Should Convert 3", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, ["1", "2", "3"]); + Assert.IsEqual(R, [1, 2, "3"]); + }); + it("Should Convert 4", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, "not an array"); + Assert.IsEqual(R, "not an array"); + }); + it("Should Convert 5", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, 42); + Assert.IsEqual(R, 42); + }); + it("Should Convert 6", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, null); + Assert.IsEqual(R, null); + }); + it("Should Convert 7", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, { 0: "1", 1: "2" }); + Assert.IsEqual(R, { 0: "1", 1: "2" }); + }); +}); + +describe("Value.Convert.Union", () => { + it("Should Convert 1", () => { + const T = Type.Object({ + x: Type.Union([Type.Number(), Type.Null()]), + }); + const R0 = Value.Convert(T, { x: "42" }); + const R1 = Value.Convert(T, { x: "null" }); + const R2 = Value.Convert(T, { x: "hello" }); + Assert.IsEqual(R0, { x: 42 }); + Assert.IsEqual(R1, { x: null }); + Assert.IsEqual(R2, { x: "hello" }); + }); + it("Should Convert 2", () => { + const T = Type.Object({ + x: Type.Union([Type.Boolean(), Type.Number()]), + }); + const R = Value.Convert(T, { x: "1" }); + Assert.IsEqual(R, { x: true }); + }); + + // ---------------------------------------------------------------- + // https://github.com/sinclairzx81/typebox/issues/787 + // ---------------------------------------------------------------- + it("Should Convert 3", () => { + const T = Type.Intersect([ + Type.Union([Type.Object({ a: Type.Number() }), Type.Object({ b: Type.Number() })]), + Type.Object({ c: Type.Number() }), + ]); + const R0 = Convert(T, { a: "1", c: "2" }); + const R1 = Convert(T, { b: "1", c: "2" }); + const R2 = Convert(T, { a: "1", b: "2", c: "3" }); + Assert.IsEqual(R0, { a: 1, c: 2 }); + Assert.IsEqual(R1, { b: 1, c: 2 }); + Assert.IsEqual(R2, { a: 1, b: "2", c: 3 }); // note: matching on first + }); + // ---------------------------------------------------------------- + // https://github.com/sinclairzx81/typebox/issues/1295 + // ---------------------------------------------------------------- + it("Should guard against Array conversion in Object", () => { + const T = Type.Union([ + Type.Object({ + type: Type.Literal("A"), + values: Type.Union([Type.String(), Type.Number()]), + }), + Type.Object({ + type: Type.Literal("B"), + values: Type.String(), + }), + ]); + const converted = Value.Convert(T, [{ type: "A", values: 1 }]); + Assert.IsEqual(converted, [{ type: "A", values: 1 }]); + }); + it("Should guard against Array conversion in Record", () => { + const T = Type.Union([ + Type.Record(Type.String({ pattern: "^values$" }), Type.Union([Type.String(), Type.Number()])), + Type.Record(Type.String({ pattern: "^type$" }), Type.Union([Type.String(), Type.Number()])), + ]); + const converted = Value.Convert(T, [{ type: "A", values: 1 }]); + Assert.IsEqual(converted, [{ type: "A", values: 1 }]); + }); +}); + +describe("Value.Convert.Unknown", () => { + it("Should Convert 1", () => { + const V = null; + const T = Type.Unknown(); + const R = Value.Convert(T, V); + Assert.IsEqual(R, V); + }); + it("Should Convert 2", () => { + const V = undefined; + const T = Type.Unknown(); + const R = Value.Convert(T, V); + Assert.IsEqual(R, V); + }); + it("Should Convert 3", () => { + const V = "hello"; + const T = Type.Unknown(); + const R = Value.Convert(T, V); + Assert.IsEqual(R, V); + }); + it("Should Convert 4", () => { + const V = 42; + const T = Type.Unknown(); + const R = Value.Convert(T, V); + Assert.IsEqual(R, V); + }); + it("Should Convert 5", () => { + const T = Type.Unknown(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + it("Should Convert 6", () => { + const T = Type.Unknown(); + const R = Value.Convert(T, { x: 1 }); + Assert.IsEqual(R, { x: 1 }); + }); + it("Should Convert 7", () => { + const T = Type.Unknown(); + const R = Value.Convert(T, [1, 2, 3]); + Assert.IsEqual(R, [1, 2, 3]); + }); +}); diff --git a/packages/schema/test/coerce.test.ts b/packages/schema/test/coerce.test.ts new file mode 100644 index 000000000..4444f0cc9 --- /dev/null +++ b/packages/schema/test/coerce.test.ts @@ -0,0 +1,458 @@ +// compile(schema, { coerce: true }): coerce() and parse(), the three coercion passes (optional nulls, TypeBox's +// conversion rules, primitive coercion by type), their order where they conflict, and the guarantees around them. +// Ports pi-ai's tool argument coercion tests. + +import Type from "typebox"; +import { describe, expect, expectTypeOf, it } from "vitest"; +import { formats } from "../src/formats/index.ts"; +import { + arr, + compile, + int, + map, + nullable, + num, + obj, + opt, + str, + tuple, + type ValidationError, + type Validator, +} from "../src/index.ts"; + +function coerce(schema: object, value: unknown): unknown { + return compile(schema, { formats, coerce: true }).coerce(value); +} + +/** The coerced `value` property of `{ value }` under an object schema with one required property `value`. */ +function coerceValue(schema: unknown, value: unknown): unknown { + const root = { type: "object", properties: { value: schema }, required: ["value"] }; + return (coerce(root, { value }) as { value: unknown }).value; +} + +describe("compile options", () => { + it("adds coerce and parse only with coerce: true", () => { + const plain = compile(obj({ n: num() })); + expectTypeOf(plain).toEqualTypeOf>(); + expect("coerce" in plain).toBe(false); + expect("parse" in plain).toBe(false); + const coercing = compile(obj({ n: num() }), { coerce: true }); + expectTypeOf(coercing.parse).returns.toEqualTypeOf< + | { readonly ok: true; readonly value: { n: number } } + | { readonly ok: false; readonly errors: ValidationError[] } + >(); + expect(coercing.check({ n: 1 })).toBe(true); + }); +}); + +describe("coerce", () => { + it("never modifies the value", () => { + const value = { count: "42", tags: "a" }; + const schema = obj({ count: num(), tags: arr(str()) }); + expect(coerce(schema, value)).toEqual({ count: 42, tags: ["a"] }); + expect(value).toEqual({ count: "42", tags: "a" }); + }); + + it("returns the value itself when nothing changes", () => { + const value = { a: { b: [1, 2] }, c: "x" }; + const schema = obj({ a: obj({ b: arr(num()) }), c: str() }); + expect(coerce(schema, value)).toBe(value); + }); + + it("copies only the changed paths", () => { + const unchanged = { b: [1, 2] }; + const value = { a: unchanged, c: { d: "1" } }; + const schema = obj({ a: obj({ b: arr(num()) }), c: obj({ d: num() }) }); + const coerced = coerce(schema, value) as typeof value; + expect(coerced).toEqual({ a: { b: [1, 2] }, c: { d: 1 } }); + expect(coerced).not.toBe(value); + expect(coerced.a).toBe(unchanged); + expect(value.c).toEqual({ d: "1" }); + }); + + it("works on frozen schemas", () => { + const schema = Object.freeze({ + type: "object", + properties: Object.freeze({ n: Object.freeze({ type: "integer" }) }), + }); + expect(coerce(schema, { n: "7" })).toEqual({ n: 7 }); + }); + + it("still coerces when the Function constructor is unavailable", () => { + const original = globalThis.Function; + globalThis.Function = (() => { + throw new EvalError("Code generation from strings disallowed for this context"); + }) as unknown as FunctionConstructor; + try { + expect(coerce(obj({ count: num() }), { count: "42" })).toEqual({ count: 42 }); + } finally { + globalThis.Function = original; + } + }); + + it("leaves values it cannot coerce for the check to report", () => { + const validator = compile(obj({ count: num() }), { coerce: true }); + const coerced = validator.coerce({ count: "many" }); + expect(coerced).toEqual({ count: "many" }); + expect(validator.check(coerced)).toBe(false); + }); +}); + +describe("parse", () => { + it("returns the coerced value when it is valid", () => { + const validator = compile(obj({ count: num(), tags: arr(str()) }), { coerce: true }); + const result = validator.parse({ count: "42", tags: "a" }); + expect(result).toEqual({ ok: true, value: { count: 42, tags: ["a"] } }); + if (result.ok) expectTypeOf(result.value).toEqualTypeOf<{ count: number; tags: string[] }>(); + }); + + it("returns the errors of the coerced value when it is invalid", () => { + const validator = compile(obj({ count: num(), name: str() }), { coerce: true }); + expect(validator.parse({ count: "many", name: 7 })).toEqual({ + ok: false, + errors: [{ path: "/count", keyword: "type", message: "expected number, got string" }], + }); + }); +}); + +describe("pass 1: optional nulls", () => { + it("drops null in optional non-nullable properties, also nested", () => { + const schema = obj( + { + path: str(), + offset: opt(num()), + nullable: opt(nullable(str())), + metadata: obj({ enabled: opt({ type: "boolean" }) }), + }, + { open: true }, + ); + expect(coerce(schema, { path: "file.txt", offset: null, nullable: null, metadata: { enabled: null } })).toEqual({ + path: "file.txt", + nullable: null, + metadata: {}, + }); + }); + + it("keeps a null whose property is a $ref (the reference may allow it)", () => { + const schema = { + type: "object", + properties: { value: { $ref: "#/$defs/value" } }, + $defs: { value: { anyOf: [{ type: "number" }, { type: "null" }] } }, + }; + expect(coerce(schema, { value: null })).toEqual({ value: null }); + }); + + it("walks array items", () => { + const schema = { type: "array", items: obj({ a: str(), b: opt(int()) }) }; + expect(coerce(schema, [{ a: "x", b: null }])).toEqual([{ a: "x" }]); + }); +}); + +describe("pass 2: TypeBox's conversion rules on JSON Schema", () => { + it("converts primitives by type", () => { + expect(coerceValue({ type: "number" }, "42")).toBe(42); + expect(coerceValue({ type: "number" }, "")).toBe(0); + expect(coerceValue({ type: "number" }, "true")).toBe(1); + expect(coerceValue({ type: "integer" }, "3.5")).toBe(3); + expect(coerceValue({ type: "integer" }, 1.5)).toBe(1); + expect(coerceValue({ type: "boolean" }, "1")).toBe(true); + expect(coerceValue({ type: "boolean" }, "FALSE")).toBe(false); + expect(coerceValue({ type: "null" }, "null")).toBe(null); + expect(coerceValue({ type: "string" }, 42)).toBe("42"); + }); + + it("wraps a single value into an array and converts the items", () => { + expect(coerceValue({ type: "array", items: { type: "number" } }, "1")).toEqual([1]); + expect(coerceValue({ type: "array", items: { type: "string" } }, null)).toEqual(["null"]); + }); + + it("converts tuple items in place and never wraps for a tuple", () => { + const tuple = { type: "array", prefixItems: [{ type: "number" }, { type: "boolean" }], items: false }; + expect(coerceValue(tuple, ["1", "true"])).toEqual([1, true]); + expect(coerceValue(tuple, "1")).toBe("1"); + }); + + it("converts const and enum values only when the result is a member", () => { + expect(coerceValue({ const: 1 }, "1")).toBe(1); + expect(coerceValue({ const: 1 }, "2")).toBe("2"); + expect(coerceValue({ enum: [1, 2, "hello"] }, "2")).toBe(2); + expect(coerceValue({ enum: [1, 2, "hello"] }, "world")).toBe("world"); + }); + + it("keeps a value one union branch accepts, else takes the first branch whose conversion the union accepts", () => { + const union = { anyOf: [{ type: "number" }, { type: "null" }] }; + expect(coerceValue(union, null)).toBe(null); + expect(coerceValue(union, "42")).toBe(42); + expect(coerceValue(union, "null")).toBe(null); + expect(coerceValue({ oneOf: [{ type: "boolean" }, { type: "number" }] }, "1")).toBe(true); + }); + + it("converts allOf with each member in turn", () => { + const schema = { allOf: [obj({ a: num() }, { open: true }), obj({ b: { type: "boolean" } }, { open: true })] }; + expect(coerce(schema, { a: "1", b: "true" })).toEqual({ a: 1, b: true }); + }); + + it("follows $ref", () => { + const schema = { + type: "object", + properties: { item: { $ref: "#/$defs/item" } }, + $defs: { item: { type: "object", properties: { id: { type: "integer" } } } }, + }; + expect(coerce(schema, { item: { id: "7" } })).toEqual({ item: { id: 7 } }); + }); + + it("converts patternProperties and additionalProperties", () => { + const schema = { + type: "object", + properties: { a: { type: "number" } }, + patternProperties: { "^n_": { type: "number" } }, + additionalProperties: { type: "boolean" }, + }; + expect(coerce(schema, { a: "1", n_x: "2", other: "true" })).toEqual({ a: 1, n_x: 2, other: true }); + }); +}); + +describe("pass 3: primitive coercion by type", () => { + it("coerces like Ajv's coerceTypes", () => { + const cases: Array<[unknown, unknown, unknown]> = [ + [{ type: "number" }, true, 1], + [{ type: "number" }, null, 0], + [{ type: "integer" }, "42", 42], + [{ type: "boolean" }, "true", true], + [{ type: "boolean" }, 0, false], + [{ type: "string" }, true, "true"], + [{ type: "null" }, "", null], + [{ type: "null" }, 0, null], + [{ type: "null" }, false, null], + ]; + for (const [schema, input, expected] of cases) expect(coerceValue(schema, input)).toEqual(expected); + }); + + it("leaves a value that matches one of a type array's types, else takes the first coercible type", () => { + expect(coerceValue({ type: ["number", "string"] }, "1")).toBe("1"); + expect(coerceValue({ type: ["boolean", "number"] }, "1")).toBe(1); + expect(coerceValue({ type: ["integer", "null"] }, "5")).toBe(5); + }); + + it("accepts null for nullable array schemas with items", () => { + expect(coerceValue({ type: ["array", "null"], items: { type: "string" } }, null)).toBe(null); + }); + + it("coerces nullable unions when the value matches no branch", () => { + expect(coerceValue({ anyOf: [{ type: "number" }, { type: "null" }] }, "42")).toBe(42); + expect(coerceValue({ oneOf: [{ type: "number" }, { type: "null" }] }, null)).toBe(null); + }); + + it("leaves values no rule coerces", () => { + const validator = compile( + { type: "object", properties: { value: { type: "integer" } }, required: ["value"] }, + { coerce: true }, + ); + for (const input of ["abc", "4x", {}, ["1", "2"]]) { + const coerced = validator.coerce({ value: input }); + expect(validator.check(coerced)).toBe(false); + } + }); +}); + +describe("where the passes conflict, the conversion rules win", () => { + it("turns null into 'null' for a required string (pass 3 alone would give '')", () => { + expect(coerceValue({ type: "string" }, null)).toBe("null"); + }); + + it("turns '' into 0 for a number (pass 3 leaves an empty string)", () => { + expect(coerceValue({ type: "number" }, "")).toBe(0); + }); + + it("turns '1' and '0' into booleans (pass 3 takes only 'true', 'false', 1 and 0)", () => { + expect(coerceValue({ type: "boolean" }, "0")).toBe(false); + }); + + it("truncates '42.1' for an integer (pass 3 rejects it)", () => { + expect(coerceValue({ type: "integer" }, "42.1")).toBe(42); + }); + + it("drops an optional null before converting it (pass 2 alone would give 'null')", () => { + expect(coerce(obj({ name: opt(str()) }), { name: null })).toEqual({}); + }); +}); + +describe("recursive schemas", () => { + const tree = { + type: "object", + properties: { t: { $ref: "#/$defs/Tree" } }, + $defs: { Tree: { anyOf: [{ type: "string" }, { type: "array", items: { $ref: "#/$defs/Tree" } }] } }, + }; + const json = { + type: "object", + properties: { data: { $ref: "#/$defs/Json" } }, + required: ["data"], + $defs: { + Json: { + anyOf: [ + { type: "string" }, + { type: "number" }, + { type: "boolean" }, + { type: "array", items: { $ref: "#/$defs/Json" } }, + { type: "object", additionalProperties: { $ref: "#/$defs/Json" } }, + ], + }, + }, + }; + + it("convert through a recursive union by its first accepting branch", () => { + const validator = compile(tree, { coerce: true }); + expect(validator.parse({ t: ["a", ["b"]] })).toEqual({ ok: true, value: { t: ["a", ["b"]] } }); + expect(validator.parse({ t: 5 })).toEqual({ ok: true, value: { t: "5" } }); + expect(validator.parse({ t: [5] })).toEqual({ ok: true, value: { t: ["5"] } }); + expect(compile(json, { coerce: true }).parse({ data: { a: [1, null] } })).toEqual({ + ok: true, + value: { data: { a: [1, "null"] } }, + }); + }); + + it("leave a value a recursive schema cannot convert as it is, for the check to report", () => { + const validator = compile(tree, { coerce: true }); + const result = validator.parse({ t: {} }); + expect(result.ok).toBe(false); + expect(validator.coerce({ t: { a: 1 } })).toEqual({ t: { a: 1 } }); + }); + + it("convert a scalar-or-array schema that refers to itself", () => { + const schema = { + type: "object", + properties: { + value: { anyOf: [{ type: "number" }, { type: "array", items: { $ref: "#/properties/value" } }] }, + }, + required: ["value"], + }; + const validator = compile(schema, { coerce: true }); + expect(validator.parse({ value: "42" })).toEqual({ ok: true, value: { value: 42 } }); + expect(validator.parse({ value: ["1", ["2"]] })).toEqual({ ok: true, value: { value: [1, [2]] } }); + expect(validator.parse({ value: {} }).ok).toBe(false); + }); + + it("stop at a schema that cycles through shared JS objects instead of $ref", () => { + const node: { anyOf: unknown[] } = { anyOf: [{ type: "number" }] }; + node.anyOf.push({ type: "array", items: node }); + const validator = compile(node, { coerce: true }); + expect(validator.parse("42")).toEqual({ ok: true, value: 42 }); + expect(validator.parse(["1", ["2"]])).toEqual({ ok: true, value: [1, [2]] }); + expect(validator.parse({}).ok).toBe(false); + }); + + it("reset the re-entry state between calls and leave the input unchanged", () => { + const validator = compile( + { anyOf: [{ type: "number" }, { type: "array", items: { $ref: "#" } }] }, + { coerce: true }, + ); + for (let i = 0; i < 20; i++) { + expect(validator.parse({}).ok).toBe(false); + expect(validator.parse("42")).toEqual({ ok: true, value: 42 }); + const input = [1, "2", ["3"]]; + expect(validator.parse(input)).toEqual({ ok: true, value: [1, 2, [3]] }); + expect(input).toEqual([1, "2", ["3"]]); + } + }); + + it("convert and explain a deep value that every branch of a recursive union rejects, in linear time", () => { + // Each object branch converts and explains every child: without memoizing per call this is 2^depth work. + const S = { + anyOf: [ + { type: "object", additionalProperties: { $ref: "#/$defs/S" }, required: ["a"] }, + { type: "object", additionalProperties: { $ref: "#/$defs/S" }, required: ["b"] }, + ], + }; + const validator = compile( + { type: "object", properties: { s: { $ref: "#/$defs/S" } }, $defs: { S } }, + { coerce: true }, + ); + let value: unknown = { x: 1 }; + for (let i = 0; i < 40; i++) value = { c: value }; + const result = validator.parse({ s: value }); + expect(result.ok).toBe(false); + const missing = Array.from({ length: 41 }, (_, i) => ({ + path: `/s${"/c".repeat(i)}/a`, + keyword: "required", + message: "missing", + })); + const leaf = { path: `/s${"/c".repeat(40)}/x`, keyword: "type", message: "expected object, got integer" }; + expect(result).toEqual({ ok: false, errors: [...missing, leaf] }); + }); + + it("convert a TypeBox Cyclic union", () => { + const Tree = Type.Cyclic({ Tree: Type.Union([Type.String(), Type.Array(Type.Ref("Tree"))]) }, "Tree"); + const validator = compile(Type.Object({ t: Tree }), { coerce: true }); + expect(validator.parse({ t: [5] })).toEqual({ ok: true, value: { t: ["5"] } }); + expect(validator.parse({ t: { a: 1 } }).ok).toBe(false); + }); +}); + +describe("member keywords across the passes", () => { + it("keep a pattern-matched property from additionalProperties in every pass", () => { + const record = Type.Record(Type.String(), Type.Number(), { additionalProperties: Type.String() }); + expect(compile(record, { coerce: true }).parse({ n1: "1" })).toEqual({ ok: true, value: { n1: 1 } }); + const plain = { + type: "object", + patternProperties: { "^n_": { type: ["integer", "null"] } }, + additionalProperties: { type: "string" }, + }; + expect(coerce(plain, { n_a: "5", other: 7 })).toEqual({ n_a: 5, other: "7" }); + }); + + it("drop optional nulls inside declared, pattern and additional property values alike", () => { + const item = obj({ n: opt(int({ minimum: 1 })) }); + const schemas = [ + obj({ entry: item }), + map(item), + { type: "object", patternProperties: { "^entry$": item }, additionalProperties: false }, + ]; + for (const schema of schemas) { + const input = { entry: { n: null } }; + expect(compile(schema, { coerce: true }).parse(input)).toEqual({ ok: true, value: { entry: {} } }); + expect(input).toEqual({ entry: { n: null } }); + } + }); + + it("keep a null under pattern and additional properties: their keys are not optional properties", () => { + const schema = { type: "object", additionalProperties: { type: ["integer", "null"] } }; + expect(compile(schema, { coerce: true }).parse({ n: null })).toEqual({ ok: true, value: { n: null } }); + }); + + it("drop optional nulls in tuple items as in array items", () => { + const item = obj({ n: opt(int()) }); + expect(compile(arr(item), { coerce: true }).coerce([{ n: null }])).toEqual([{}]); + expect(compile(tuple([item]), { coerce: true }).coerce([{ n: null }])).toEqual([{}]); + }); + + it("coerce tuple items by type in pass 3", () => { + const schema = { type: "array", prefixItems: [{ type: ["integer", "null"] }, { type: ["boolean", "null"] }] }; + expect(coerce(schema, ["5", "true"])).toEqual([5, true]); + }); + + it("convert the properties a union node declares besides its branches", () => { + const schema = { + type: "object", + properties: { a: { type: "integer" }, b: { type: "integer" } }, + anyOf: [{ required: ["a"] }, { required: ["b"] }], + }; + expect(compile(schema, { coerce: true }).parse({ a: "3.5" })).toEqual({ ok: true, value: { a: 3 } }); + }); + + for (const keyword of ["anyOf", "oneOf"]) { + it(`check ${keyword} candidates after the node's own properties convert`, () => { + const schema = { + type: "object", + [keyword]: [ + { type: "object", properties: { a: { type: "array", items: { type: "integer" } } }, required: ["a"] }, + { type: "object", properties: { b: { type: "boolean" } }, required: ["b"] }, + ], + properties: { c: { type: "integer" } }, + required: ["c"], + }; + const input = { a: "3", c: "2" }; + expect(compile(schema, { coerce: true }).parse(input)).toEqual({ ok: true, value: { a: [3], c: 2 } }); + expect(input).toEqual({ a: "3", c: "2" }); + }); + } +}); diff --git a/packages/schema/test/validator.test.ts b/packages/schema/test/validator.test.ts index a62b38e31..c5ebd6edf 100644 --- a/packages/schema/test/validator.test.ts +++ b/packages/schema/test/validator.test.ts @@ -188,3 +188,136 @@ function deepFreeze(value: T): T { } return value; } + +describe("unresolved references", () => { + const dangling = { type: "object", properties: { t: { $ref: "#/definitions/Missing" } } }; + const external = { type: "object", properties: { t: { $ref: "https://example.com/schemas/t.json" } } }; + + it("throw when the schema compiles, by default", () => { + expect(() => compile(dangling)).toThrow("Unresolved $ref #/definitions/Missing"); + expect(() => compile(external)).toThrow("Unresolved $ref https://example.com/schemas/t.json"); + }); + + it("reject only values at their place with unresolvedRefs: reject", () => { + for (const schema of [dangling, external]) { + const validator = compile(schema, { unresolvedRefs: "reject" }); + expect(validator.check({})).toBe(true); + expect(validator.check({ t: 1 })).toBe(false); + } + expect(compile(dangling, { unresolvedRefs: "reject" }).errors({ t: 1 })).toEqual([ + { path: "/t", keyword: "$ref", message: "unresolved reference #/definitions/Missing" }, + ]); + }); + + it("leave values at their place uncoerced", () => { + const validator = compile(dangling, { unresolvedRefs: "reject", coerce: true }); + expect(validator.parse({ t: "1" })).toEqual({ + ok: false, + errors: [{ path: "/t", keyword: "$ref", message: "unresolved reference #/definitions/Missing" }], + }); + }); + + it("reject a TypeBox Ref without its definition", () => { + const validator = compile(Type.Object({ a: Type.Ref("A") }), { unresolvedRefs: "reject" }); + expect(validator.errors({ a: { x: "1" } })).toEqual([ + { path: "/a", keyword: "$ref", message: "unresolved reference A" }, + ]); + }); +}); + +describe("string lengths", () => { + it("count an unpaired high surrogate as one code point and a pair as one", () => { + expect(compile({ type: "string", minLength: 2 }).check("\ud800A")).toBe(true); + expect(compile({ type: "string", maxLength: 1 }).check("\ud800A")).toBe(false); + expect(compile({ type: "string", maxLength: 1 }).check("\ud800")).toBe(true); + expect(compile({ type: "string", maxLength: 1 }).check("😀")).toBe(true); + expect(compile({ type: "string", minLength: 2 }).check("😀")).toBe(false); + expect(compile({ type: "string", minLength: 2 }).check("\udc00\ud800")).toBe(true); + }); +}); + +describe("error messages", () => { + it("list at most 20 enum values", () => { + const values = Array.from({ length: 5000 }, (_, i) => `v${i}`); + const validator = compile({ type: "object", properties: { e: { type: "string", enum: values } } }); + expect(validator.check({ e: "v4999" })).toBe(true); + const [error] = validator.errors({ e: "v5000" }); + expect(error.message).toBe( + `expected one of ${JSON.stringify(values.slice(0, 20)).slice(0, -1)}, ... and 4980 more]`, + ); + expect(compile({ enum: ["a", "b"] }).errors("c")).toEqual([ + { path: "", keyword: "enum", message: 'expected one of ["a","b"]' }, + ]); + }); + + it("accept a 10,000-value enum", () => { + const values = Array.from({ length: 10_000 }, (_, i) => `member${i}`); + expect(compile({ enum: values }).check("member9999")).toBe(true); + }); + + it("describe contains bounds in words", () => { + const at = (schema: object) => compile({ type: "array", ...schema }).errors(["a"])[0]?.message; + expect(at({ contains: { type: "number" } })).toBe("expected at least 1 matching item"); + expect(at({ contains: { type: "number" }, minContains: 2 })).toBe("expected at least 2 matching items"); + expect(at({ contains: { type: "string" }, maxContains: 0, minContains: 0 })).toBe( + "expected at most 0 matching items", + ); + expect(at({ contains: { type: "number" }, minContains: 1, maxContains: 3 })).toBe( + "expected between 1 and 3 matching items", + ); + }); + + it("blame a oneOf failure on the branch that declares most of the object's properties", () => { + const filter = { + type: "object", + properties: { where: { $ref: "#/$defs/F" } }, + $defs: { + F: { + oneOf: [ + { + type: "object", + properties: { field: { type: "string" }, eq: { type: "string" } }, + required: ["field", "eq"], + additionalProperties: false, + }, + { + type: "object", + properties: { and: { type: "array", items: { $ref: "#/$defs/F" } } }, + required: ["and"], + additionalProperties: false, + }, + ], + }, + }, + }; + expect(compile(filter).errors({ where: { and: [{ field: "a", eq: 1, x: 2 }] } })).toEqual([ + { path: "/where/and/0/eq", keyword: "type", message: "expected string, got integer" }, + { path: "/where/and/0/x", keyword: "additionalProperties", message: "unknown field" }, + ]); + }); + + it("explain one object reached through the same reference at two paths at each path", () => { + const schema = { + type: "object", + properties: { a: { $ref: "#/$defs/P" }, b: { $ref: "#/$defs/P" } }, + $defs: { P: { type: "object", properties: { n: { type: "integer" } } } }, + }; + const shared = { n: "x" }; + expect(compile(schema).errors({ a: shared, b: shared })).toEqual([ + { path: "/a/n", keyword: "type", message: "expected integer, got string" }, + { path: "/b/n", keyword: "type", message: "expected integer, got string" }, + ]); + }); +}); + +describe("TypeBox's extra formats", () => { + it("check url and json-pointer-uri-fragment", () => { + const url = compile({ type: "string", format: "url" }, { formats }); + expect(url.check("https://example.com/a?b=1")).toBe(true); + expect(url.check("not a url")).toBe(false); + const fragment = compile({ type: "string", format: "json-pointer-uri-fragment" }, { formats }); + expect(fragment.check("#/a/b~1c")).toBe(true); + expect(fragment.check("#/a b")).toBe(false); + expect(fragment.check("/a")).toBe(false); + }); +}); diff --git a/scripts/browser-smoke-entry.ts b/scripts/browser-smoke-entry.ts index 4c6d07c38..9cf69450d 100644 --- a/scripts/browser-smoke-entry.ts +++ b/scripts/browser-smoke-entry.ts @@ -1,12 +1,17 @@ import { createHost, memoryLinkPair, replicatedState } from "@earendil-works/chord"; -import { createAssistantMessageEventStream, Type } from "@earendil-works/pi-ai"; +import { createAssistantMessageEventStream, validateToolArguments } from "@earendil-works/pi-ai"; import { complete, getModel, getProviders, streamSimple } from "@earendil-works/pi-ai/compat"; import { Agent, streamProxy } from "@earendil-works/pi-agent-core"; +import { obj, str } from "@earendil-works/pi-schema"; // Keep this entry browser-safe. It is bundled by scripts/check-browser-smoke.mjs // to catch accidental Node-only runtime imports in browser-facing package exports. const model = getModel("google", "gemini-2.5-flash"); -const schema = Type.Object({ prompt: Type.String() }); +const schema = obj({ prompt: str() }); +const args = validateToolArguments( + { name: "prompt", description: "", parameters: schema }, + { type: "toolCall", id: "1", name: "prompt", arguments: { prompt: 1 } }, +); const stream = createAssistantMessageEventStream(); const agent = new Agent({ initialState: { model }, streamFn: streamSimple }); @@ -17,6 +22,7 @@ console.log( getProviders().length, typeof complete, schema.type, + args.prompt, typeof stream.push, agent.hasQueuedMessages(), typeof streamProxy, diff --git a/scripts/check-entry-graphs.mjs b/scripts/check-entry-graphs.mjs index 54dd30c5d..4aea32757 100644 --- a/scripts/check-entry-graphs.mjs +++ b/scripts/check-entry-graphs.mjs @@ -44,10 +44,18 @@ const BUDGETS = { // Request telemetry reaches pi-ai's telemetry.ts and pi-telemetry's root (index, noop, metrics, memory). // Request identity reaches utils/identity.ts and its version constant. maxFiles: 20, - forbid: ["providers/", "models.generated.ts", "index.ts", "utils/validation.ts", "utils/typebox-helpers.ts"], + forbid: ["providers/", "models.generated.ts", "index.ts", "utils/validation.ts", "utils/string-enum.ts"], allow: ["packages/telemetry/src/index.ts"], }, - "./utils/*": { maxFiles: 3, forbid: ["providers/", "api/", "index.ts"] }, + "./utils/*": { + maxFiles: 3, + forbid: ["providers/", "api/", "index.ts"], + overrides: { + // Tool argument validation runs on pi-schema: its validator, coercion and format checks (the checks TypeBox's + // validator ran by default). TypeBox, an external package, was not counted before. + "./utils/validation": { maxFiles: 42, forbid: ["providers/", "api/", "packages/ai/src/index.ts"] }, + }, + }, }, "packages/codemode": { // Node and Bun users never load the Cloudflare entry, which imports the `cloudflare:workers` built-in. @@ -65,7 +73,8 @@ const BUDGETS = { }, "packages/durable": { ".": { - // Tool argument validation reaches TypeBox; provider-session creation reaches pi-ai's lean UUID utility; the + // Tool argument validation reaches pi-schema (validator, coercion and format checks; 39 files that replace + // TypeBox, which this walk did not count); provider-session creation reaches pi-ai's lean UUID utility; the // memory and SQLite storages share scan order and cursor handling. Values from Chord's root (copyJson, // replicatedState) reach its host runtime through the root barrel. Chord's context reaches pi-telemetry's root // (index, noop, metrics, memory) for its NOOP telemetry default; harness/telemetry.ts is Durable's schema. @@ -77,16 +86,17 @@ const BUDGETS = { // each conversation's newest entry and head marker (session/conversation-tails.ts). Group commit sends batches // through `commit()`, or one `commit()` each (storage/commit-batch.ts), from a writer off the Session line // (session/writer.ts) that gates reads by the keys of unpublished writes (session/keys.ts). - maxFiles: 83, + maxFiles: 123, forbid: [ "packages/ai/src/index.ts", - "packages/ai/src/utils/typebox-helpers.ts", + "packages/ai/src/utils/string-enum.ts", // Photon (WebAssembly) loads only through `./images`. "packages/durable/src/images/", ], }, - // The file tools reach their environment helpers and the Harness definitions they need, not the Harness itself. - "./tools": { maxFiles: 17, forbid: ["packages/durable/src/images/"] }, + // The file tools reach their environment helpers and the Harness definitions they need, not the Harness itself, + // and pi-schema's root for their schema builders (10 files; its format checks stay out). + "./tools": { maxFiles: 27, forbid: ["packages/durable/src/images/", "packages/schema/src/formats/"] }, }, }; @@ -156,6 +166,8 @@ for (const [pkgDir, budgets] of Object.entries(BUDGETS)) { } const target = typeof declared === "string" ? declared : declared.import; for (const [name, distPath] of expand(pkgDir, entry, target)) { + // A wildcard entry's budget applies to every file it exports, unless `overrides` names that file. + const limits = budget.overrides?.[name] ?? budget; const source = sourceFor(pkgDir, distPath); if (!source) { console.error(`${pkgDir} export "${name}" points at ${distPath}, which has no source file`); @@ -163,15 +175,15 @@ for (const [pkgDir, budgets] of Object.entries(BUDGETS)) { continue; } const graph = [...walk(source)].map((file) => relative(ROOT, file).split(sep).join("/")); - if (graph.length > budget.maxFiles) { + if (graph.length > limits.maxFiles) { console.error( - `${pkgDir} export "${name}" reaches ${graph.length} files, budget ${budget.maxFiles}\n` + + `${pkgDir} export "${name}" reaches ${graph.length} files, budget ${limits.maxFiles}\n` + graph.map((file) => ` ${file}`).join("\n"), ); failures += 1; } - for (const pattern of budget.forbid ?? []) { - const hit = graph.filter((file) => file.includes(pattern) && !(budget.allow ?? []).some((allowed) => file.endsWith(allowed))); + for (const pattern of limits.forbid ?? []) { + const hit = graph.filter((file) => file.includes(pattern) && !(limits.allow ?? []).some((allowed) => file.endsWith(allowed))); if (hit.length > 0) { console.error(`${pkgDir} export "${name}" must not reach ${pattern}:\n${hit.map((f) => ` ${f}`).join("\n")}`); failures += 1; diff --git a/tsconfig.json b/tsconfig.json index 96c54539c..04557184d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -63,5 +63,5 @@ } }, "include": ["scripts/model-catalog-protocol.ts", "scripts/model-catalog-protocol.test.ts", "packages/*/src/**/*", "packages/*/test/**/*", "packages/ai/catalog/**/*", "packages/durable/test-workers/**/*", "packages/coding-agent/examples/**/*", "packages/agent/examples/**/*"], - "exclude": ["**/dist/**", "packages/coding-agent/examples/extensions/gondolin/**"] + "exclude": ["**/dist/**", "packages/coding-agent/examples/extensions/**", "packages/coding-agent/test/suite/regressions/8261-subagent-project-trust.test.ts"] } From be039fb316a22c70cdf22eddf4b992110d84d6fb Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sun, 1 Nov 2026 10:14:38 +0000 Subject: [PATCH 233/271] feat(chord): Chord v1 and v2 on pi-schema, drop TypeBox Schema helpers are a thin layer over pi-schema's builders with the same JSON, so fingerprints and stored data are unchanged (proven by schema-switch tests and a store written by the TypeBox code). Checks through pi-schema; error messages are pi-schema's; format is an annotation. v1 contracts take any JSON Schema. map(bytes()) works. --- package-lock.json | 4 +- packages/chord/CHANGELOG.md | 2 + packages/chord/README.md | 12 +- packages/chord/docs/guide.md | 30 +- packages/chord/docs/spec.md | 16 +- packages/chord/docs/tutorial.md | 51 +- packages/chord/package.json | 4 +- packages/chord/src/cell.ts | 14 +- packages/chord/src/contract.ts | 93 +- packages/chord/src/next/contracts/README.md | 7 +- packages/chord/src/next/contracts/contract.ts | 5 +- packages/chord/src/next/contracts/declare.ts | 6 +- packages/chord/src/next/contracts/evolve.ts | 62 +- packages/chord/src/next/contracts/history.ts | 6 +- packages/chord/src/next/contracts/index.ts | 8 +- packages/chord/src/next/contracts/params.ts | 27 +- packages/chord/src/next/contracts/schema.ts | 213 +-- packages/chord/src/next/contracts/snapshot.ts | 28 +- packages/chord/src/next/data/README.md | 5 +- packages/chord/src/next/data/args.ts | 2 +- packages/chord/src/next/data/validator.ts | 49 +- .../chord/src/next/documents/definition.ts | 20 +- packages/chord/src/next/jobs/definition.ts | 12 +- packages/chord/src/next/link/README.md | 2 +- packages/chord/src/next/link/message.ts | 16 +- packages/chord/src/next/store/archive.ts | 30 +- packages/chord/src/next/store/columns.ts | 32 +- packages/chord/src/next/store/expression.ts | 5 +- packages/chord/src/next/store/json-column.ts | 21 +- packages/chord/src/next/store/table-evolve.ts | 10 +- packages/chord/src/object.ts | 21 +- packages/chord/src/remote.ts | 4 +- packages/chord/src/schema.ts | 183 ++- packages/chord/src/wire.ts | 178 +-- packages/chord/test/awaiting.test.ts | 8 +- packages/chord/test/bindings.test.ts | 42 +- packages/chord/test/boundary.test.ts | 7 +- packages/chord/test/cluster.test.ts | 30 +- packages/chord/test/contract.test.ts | 53 +- packages/chord/test/fixtures/echo.ts | 6 +- packages/chord/test/fixtures/loader.ts | 22 +- packages/chord/test/fixtures/schemas.ts | 14 + packages/chord/test/flow-units.test.ts | 16 +- packages/chord/test/flow.test.ts | 10 +- packages/chord/test/generations.test.ts | 16 +- .../chord/test/host-objects-reload.test.ts | 35 +- packages/chord/test/host.test.ts | 106 +- packages/chord/test/hydration.test.ts | 30 +- packages/chord/test/link.test.ts | 32 +- packages/chord/test/links-review.test.ts | 40 +- .../chord/test/next/contracts/declare.test.ts | 18 +- .../chord/test/next/contracts/evolve.test.ts | 9 +- .../test/next/contracts/fingerprint.test.ts | 28 +- .../chord/test/next/contracts/params.test.ts | 3 +- .../chord/test/next/contracts/schema.test.ts | 41 +- .../test/next/contracts/snapshot.test.ts | 110 +- packages/chord/test/next/data/args.test.ts | 12 +- packages/chord/test/next/data/params.test.ts | 4 +- .../chord/test/next/data/validator.test.ts | 48 +- .../37-plugin-imports/plugins/shapes/units.ts | 4 +- .../chord/test/next/schema/declarations.ts | 180 +++ packages/chord/test/next/schema/owner.ts | 85 ++ packages/chord/test/next/schema/proofs.ts | 113 ++ packages/chord/test/next/schema/recorded.json | 1143 +++++++++++++++++ .../test/next/schema/schema-switch.test.ts | 22 + .../test/next/schema/stored/archive.jsonl | 36 + .../chord/test/next/schema/stored/store.db | Bin 0 -> 57344 bytes .../chord/test/next/schema/write-stored.ts | 40 + packages/chord/test/placement.test.ts | 22 +- packages/chord/test/principals.test.ts | 10 +- packages/chord/test/provider-cost.test.ts | 8 +- packages/chord/test/readiness.test.ts | 10 +- packages/chord/test/reload.test.ts | 52 +- packages/chord/test/requests.test.ts | 58 +- packages/chord/test/review.test.ts | 103 +- packages/chord/test/state-init.test.ts | 18 +- packages/chord/test/state.test.ts | 40 +- packages/chord/test/tutorial.test.ts | 40 +- packages/chord/test/types.test.ts | 14 +- packages/chord/test/variants.test.ts | 14 +- packages/chord/test/websocket.test.ts | 12 +- 81 files changed, 2917 insertions(+), 1025 deletions(-) create mode 100644 packages/chord/test/fixtures/schemas.ts create mode 100644 packages/chord/test/next/schema/declarations.ts create mode 100644 packages/chord/test/next/schema/owner.ts create mode 100644 packages/chord/test/next/schema/proofs.ts create mode 100644 packages/chord/test/next/schema/recorded.json create mode 100644 packages/chord/test/next/schema/schema-switch.test.ts create mode 100644 packages/chord/test/next/schema/stored/archive.jsonl create mode 100644 packages/chord/test/next/schema/stored/store.db create mode 100644 packages/chord/test/next/schema/write-stored.ts diff --git a/package-lock.json b/package-lock.json index d37cfbbba..4bc15ba5c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5553,8 +5553,8 @@ "version": "1.4.0", "license": "MIT", "dependencies": { - "@earendil-works/pi-telemetry": "1.4.0", - "typebox": "1.3.27" + "@earendil-works/pi-schema": "1.4.0", + "@earendil-works/pi-telemetry": "1.4.0" }, "devDependencies": { "@types/ws": "8.18.2", diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index 326d4b8de..4f50cc098 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -4,6 +4,7 @@ ### Breaking Changes +- Schemas are JSON Schema checked by `@earendil-works/pi-schema` instead of TypeBox, which is no longer a dependency of Chord. Contracts, `self.state(...)` schemas and method signatures take pi-schema builders or any JSON Schema object (TypeBox-built schemas still validate, being JSON Schema, but type as `unknown`); handler and reference types come from pi-schema's `Static`. Validation, input defaults and the stripping of unknown fields work on any JSON Schema by its shape (before, defaults and stripping applied only to TypeBox-built schemas). `format` is an annotation, no longer checked: `str({ format: "email" })` accepts `"nope"` (TypeBox checked the formats it knew); check a format in the handler where it matters. An `invalid` error's message is now ` ` with pi-schema's paths and messages (a missing field at its own path, `missing`; `expected string, got number` instead of `must be string`), and its `details` are pi-schema's `ValidationError`s (`{ path, keyword, message }`) instead of TypeBox's errors. Link frames, descriptions, fingerprints and identities are unchanged for schemas with the same JSON. Migration, keeping a contract's fingerprint (it is computed from the schema's JSON): `Type.String/Number/Integer/Boolean/Null/Unknown/Array/Union/Optional(...)` become `str/num/int/bool/nul/unknown/arr/union/opt(...)` with the same options; `Type.Object(p)` becomes `obj(p, { open: true })` (pi-schema's `obj` is closed by default) and `Type.Object(p, { additionalProperties: false })` `obj(p)`; `Type.Enum(values)` becomes `oneOf(values)`; `Type.Literal(v)` becomes `{ type: typeof v, const: v }` (pi-schema's `lit(v)` has no `type`); `Type.Record(Type.String(), v)` becomes `{ type: "object", patternProperties: { "^.*$": v } }` (pi-schema's `map(v)` uses `additionalProperties`); `Type.Intersect(list)` becomes `{ allOf: list }`; `Type.Function(params, result)` and `Type.Void()` become `{ type: "function", parameters, returnType }` and `{ type: "void" }`; `Type.Unsafe(schema)` becomes `schema as Schema`; `Type.Immutable(schema)` becomes the schema (cast its `Schema` type); `opt(...)` marks only an `obj` property; `Static` and `TSchema` come from pi-schema (`Static`, `Schema`). pi-schema's own `lit`, `map` and closed `obj` give other JSON and so other fingerprints, which matters only while peers of both versions talk. - `traceIn(options, fn, context)` takes the context last (was `traceIn(context, options, fn)`), like the rest of the API. - Watch listeners take one change object before the context: `WatchHandle.start((change, context) => …)` with `change: WatchChange` = `{ value, ops, retired }`, and `NativeWatchHandle.start(({ value }, context) => …)` with `NativeWatchChange`. A change is plain data, those fields only; the frame's context is the second argument. `Watch.advance(change, context)` takes the same object, and a frame delivered alone, with nothing folded into it, reaches the listener as the object the source passed. Migration: `start(async (value, ops, ctx) => …)` becomes `start(async ({ value, ops }, ctx) => …)`, `start(async (value, ctx) => …)` on a native watch `start(async ({ value }, ctx) => …)`, and `advance(value, ops, ctx)` becomes `advance({ value, ops, retired: false }, ctx)`. - `SqliteExecutor.runCounted(sql, ...params)` runs one statement like `run` and resolves with the rows it inserted, updated or deleted, as SQLite's `changes()` counts them (index writes and rows changed by triggers do not count). The Node driver takes it from `node:sqlite`; the Durable Object driver reads `SELECT changes()` after the statement (the cursor's `rowsWritten` also counts index writes), so `run` stays as cheap as before. Migration: custom `SqliteDatabase` and `SqliteExecutor` implementations add `runCounted`. @@ -30,6 +31,7 @@ ### Changed +- Chord v2 (`@earendil-works/chord/next`) checks its schemas with `@earendil-works/pi-schema` instead of TypeBox: its helpers (`str`, `num`, `int`, `bool`, `nul`, `lit`, `oneOf`, `union`, `obj`, `arr`, `map`, `opt`, `bytes`, `unknown`) build on pi-schema's builders and keep the JSON contracts were fingerprinted with, so every fingerprint, normal form and stored record stays the same (a test records them and reads a store written before the switch). `Static` and `Schema` (was `TSchema`) come from pi-schema; `opt(...)` marks an object field or a table column and is no schema elsewhere (a top-level `opt` had no effect before); a schema with a non-enumerable key (a TypeBox schema's metadata) is refused at declaration; `map(bytes())` is accepted now (it was refused while `bytes()` carried a TypeBox refinement). `format` is an annotation, no longer checked (`{ type: "string", format: "email" }` accepts `"nope"`; TypeBox checked the formats it knew). A failed check's message comes from pi-schema (`args/title: expected string, got integer`), its `details.path` escapes `~` and `/` (JSON Pointer); reasons and codes are unchanged. Declaring and compiling are cheaper (a 20-method contract 1.07 ms to 0.86 ms, a validator's first use 172 µs to 48 µs), a check costs the same (0.25 µs), the message envelope check up to 100 ns more per message. - The delta tracker keeps its one open change in a plain field and needs a `WeakRef` only for a second change open at the same time (one per change before). A `WeakRef` target survives the scavenges of its job, so a run of changes in one job promoted every change's overlay to old space: 200,000 small changes in one job now promote 2 MB instead of 47.5 MB and take 3.1 µs each instead of 3.8 µs (3.0 instead of 3.5 µs when they span jobs). A draft dropped without `prepare()` or `abort()` now stays referenced by its tracker until the tracker's next `beginChange()` or `adopt()`. - `applyImmutable()`, `applyImmutableBatches()` and tracker revisions copy each container on a changed path with spread (or `Object.assign()` onto a null-prototype object) instead of one `Object.defineProperty()` per key, about 25 times faster per container ([#9265](https://github.com/earendil-works/pi/issues/9265)) diff --git a/packages/chord/README.md b/packages/chord/README.md index 9d9d95587..081e3fe21 100644 --- a/packages/chord/README.md +++ b/packages/chord/README.md @@ -8,7 +8,7 @@ Chord builds applications out of **objects** that live in several processes (a s - **State** is observable JSON. Remote viewers get a snapshot followed by deltas. - **Hot reload** swaps code while it runs. Work in flight finishes on the code it started with, state is kept, and incompatible changes migrate at a safe point. -Contracts are runtime values described with [TypeBox](https://github.com/sinclairzx81/typebox) schemas. Hosts exchange JSON frames over links: in memory, a Node IPC channel, or a WebSocket. +Contracts are runtime values described with JSON Schema, written with the builders of `@earendil-works/pi-schema` (any JSON Schema object works; `format` is an annotation, never checked). Hosts exchange JSON frames over links: in memory, a Node IPC channel, or a WebSocket. ## Entry points @@ -27,18 +27,18 @@ Contracts are runtime values described with [TypeBox](https://github.com/sinclai ## Quick start ```ts -import { Type } from "typebox"; +import { num, obj, str } from "@earendil-works/pi-schema"; import { createHost, hostNamed, method, remote } from "@earendil-works/chord"; // contract.ts: shared by provider and consumers export const Counter = remote("demo.counter", { version: 1, - input: Type.Object({ name: Type.String() }), + input: obj({ name: str() }), host: hostNamed("server"), // where its objects live; callers never name a host variants: { default: { - state: { count: Type.Number() }, - methods: { increment: method(Type.Object({ by: Type.Number() })) }, + state: { count: num() }, + methods: { increment: method(obj({ by: num() })) }, }, }, }); @@ -46,7 +46,7 @@ export const Counter = remote("demo.counter", { const host = createHost({ name: "server", self: { kind: "server" } }); host.serve(Counter, ({ name }, _principal, self) => { - const count = self.state("count", Type.Number(), 0); + const count = self.state("count", num(), 0); return { variant: "default", count, diff --git a/packages/chord/docs/guide.md b/packages/chord/docs/guide.md index 59c1414b7..c3b7b3211 100644 --- a/packages/chord/docs/guide.md +++ b/packages/chord/docs/guide.md @@ -36,17 +36,17 @@ Chord builds applications out of **objects** that live in several processes (a s ## Quick start ```ts -import { Type } from "typebox"; +import { num, obj, str } from "@earendil-works/pi-schema"; import { createHost, method, remote } from "@earendil-works/chord"; // contract.ts: shared by provider and consumers export const Counter = remote("demo.counter", { version: 1, - input: Type.Object({ name: Type.String() }), + input: obj({ name: str() }), variants: { default: { - state: { count: Type.Number() }, - methods: { increment: method(Type.Object({ by: Type.Number() })) }, + state: { count: num() }, + methods: { increment: method(obj({ by: num() })) }, }, }, }); @@ -54,7 +54,7 @@ export const Counter = remote("demo.counter", { const host = createHost({ name: "server", self: { kind: "server" } }); host.serve(Counter, ({ name }, _principal, self) => { - const count = self.state("count", Type.Number(), 0); + const count = self.state("count", num(), 0); return { variant: "default", count, @@ -82,7 +82,7 @@ Remote types can be reached from other hosts. Everything that crosses the wire h ```ts export const Chat = remote("pi.chat", { version: 1, - input: Type.Object({ session: Type.String(), conversation: Type.Number() }), + input: obj({ session: str(), conversation: num() }), host: hostFamily("session:", "session"), // lives on `session:S1` for `{ session: "S1", … }` variants: { viewer: { @@ -91,7 +91,7 @@ export const Chat = remote("pi.chat", { participant: { state: { view: ConversationViewSchema }, methods: { - send: method(SendInputSchema, Type.Object({ submissionId: Type.Number() })), + send: method(SendInputSchema, obj({ submissionId: num() })), stop: method(), // no arguments, no result: `chat.stop(ctx)` }, }, @@ -119,10 +119,10 @@ Local types are reachable only inside their host. Their members may be anything: ```ts export const Index = local("pi.index", { version: 1, - input: Type.Object({ session: Type.String() }), + input: obj({ session: str() }), members: { - search: method(Type.Object({ query: Type.String() }), Type.Array(HitSchema)), - onChange: method(Type.Function([ChangeSchema], Type.Void()), UnsubscribeSchema), + search: method(obj({ query: str() }), arr(HitSchema)), + onChange: method({ type: "function", parameters: [ChangeSchema], returnType: { type: "void" } }, UnsubscribeSchema), harness: opaque("pi.durable-harness@1"), }, }); @@ -181,7 +181,7 @@ A reload runs the factory again over the same object. Declarations that keep som ```ts host.serve(StatusStore, ({ session }, _host, self) => { const status = self.state("status", StatusSchema, { running: false, tasks: 0 }); - const errors = self.state("errors", Type.Array(Type.String()), []); + const errors = self.state("errors", arr(str()), []); const catalog = self.get(Catalog, {}); const db = self.resource("db", { version: 1, @@ -230,7 +230,7 @@ host.serve(Visits, (_, _p, self) => { // right host.serve(Visits, (_, _p, self) => { - const visits = self.state("visits", Type.Number(), 0); + const visits = self.state("visits", num(), 0); return { hit: async (ctx) => { visits.change(ctx, (n) => n + 1); @@ -240,7 +240,7 @@ host.serve(Visits, (_, _p, self) => { }); ``` -**Methods** are declared with `method(args?, result?)` in the contract. Without `args` a method takes none: callers write `ref.hit(ctx)` (or `ref.hit()`), and its implementation is `(ctx) => …`. Without `result` it returns nothing: the implementation returns nothing, and callers get `undefined`. `method(undefined, Type.Number())` takes no arguments and returns a number. +**Methods** are declared with `method(args?, result?)` in the contract. Without `args` a method takes none: callers write `ref.hit(ctx)` (or `ref.hit()`), and its implementation is `(ctx) => …`. Without `result` it returns nothing: the implementation returns nothing, and callers get `undefined`. `method(undefined, num())` takes no arguments and returns a number. ## Activation @@ -428,7 +428,7 @@ chat.release(); ## State ```ts -const todos = self.state("todos", Type.Array(TodoSchema), []); +const todos = self.state("todos", arr(TodoSchema), []); todos.change(ctx, (draft) => { draft.push({ id: "t1", text: "Write docs", done: false }); @@ -457,7 +457,7 @@ view.replace(ctx, rebuilt, { diff: true }); // always diff ```ts const db = self.resource("db", { version: 1, open: () => openSqlite(path), close: (db) => db.close() }); -const notes = self.state("notes", Type.Array(NoteSchema), { init: () => db.value.all("SELECT * FROM notes") }); +const notes = self.state("notes", arr(NoteSchema), { init: () => db.value.all("SELECT * FROM notes") }); ``` `init(ctx)` runs once the resources are open and the references live, before the object goes live, so no method, diff --git a/packages/chord/docs/spec.md b/packages/chord/docs/spec.md index ca9c93b38..ca910d758 100644 --- a/packages/chord/docs/spec.md +++ b/packages/chord/docs/spec.md @@ -3,7 +3,7 @@ This document specifies the runtime in `src`. Where it and the code disagree, the disagreement is a bug. [Known limitations and decisions](#12-known-limitations-and-decisions) lists what the runtime does not do yet and the choices the sections below leave open. Hard constraints: -- Chord's runtime depends only on its own `delta` module (tracker, applyImmutable, encoder/decoder, diff) and Node/browser built-ins plus `typebox` for schemas. No other Pi packages. +- Chord's runtime depends only on its own `delta` module (tracker, applyImmutable, encoder/decoder, diff) and Node/browser built-ins, plus `@earendil-works/pi-telemetry` for telemetry and `@earendil-works/pi-schema` for schemas. No other Pi packages. - Built-in plugins are trusted, loaded from source in-process. The plugin loader uses dynamic `import()` with a cache-busting query (owner-approved exception, `TODO(loader)`: replace with an isolating, evicting loader; old module copies leak; TUI plugins need isolation). - Hot reload is required. Host restart only for Chord/runtime code changes or host-level crash recovery. @@ -29,23 +29,23 @@ Contracts are runtime values for every type (shipped bundles have no TypeScript; ```ts export const Chat = remote("pi.chat", { version: 1, // contract version: a plain number, bumped only on breaking changes - input: Type.Object({ session: Type.String(), conversation: Type.Number() }), + input: obj({ session: str(), conversation: num() }), host: hostFamily("session:", "session"), // where its objects live: `session:` + input.session variants: { viewer: { state: { view: ViewSchema } }, participant: { state: { view: ViewSchema }, - methods: { send: method(SendSchema, Type.Object({ submissionId: Type.Number() })) }, + methods: { send: method(SendSchema, obj({ submissionId: num() })) }, }, }, }); export const Index = local("pi.index", { version: 1, - input: Type.Object({ session: Type.String() }), + input: obj({ session: str() }), members: { - search: method(Type.Object({ query: Type.String() }), Type.Array(Hit)), - onChange: method(Type.Function([Change], Type.Void()), Unsubscribe), // non-JSON allowed for locals + search: method(obj({ query: str() }), arr(Hit)), + onChange: method({ type: "function", parameters: [Change], returnType: { type: "void" } }, Unsubscribe), // non-JSON allowed for locals harness: opaque("pi.durable-harness@1"), // native object, by name+version }, }); @@ -81,7 +81,7 @@ Factories are synchronous; returning a promise throws. A declaration has a name host.serve(StatusStore, ({ session }, _host, self) => { const status = self.state("status", StatusSchema, { running: false, tasks: 0 }, // schema required, initial value { version: 2, migrate: (old, from) => upgrade(old, from) }); // per cell; default version 1 - const errors = self.state("errors", Type.Array(Type.String()), []); + const errors = self.state("errors", arr(str()), []); const catalog = self.get(Catalog, {}); // reference; identity = canonical address (+ at) const watch = self.resource("watch", { version: 1, deps: [catalog], exclusive: false, @@ -278,7 +278,7 @@ The tests cover at least: lifetime/references/linger/forced disposal/gone; activ - **Joining during a reload.** Participants are the child hosts in step when a reload starts. A child whose hello arrives while it runs is decided when it ends: in step, or reconciled to the committed code version; a child already behind is fenced at once, so the calls its roots make meanwhile are admitted in the parent's code version. A reconciliation that is running when a reload starts syncs its child only after that reload ended, against the code version it committed. A reconciliation announces the parent's retired code versions before its `prepare`. - **Two-version rule.** An object of a changed type created after a commit has only new code; a previous-code-version root calling it gets `superseded`. - **Validation.** Remote calls are validated at reference dispatch, in-host too: a direct binding against the provider's schema, a binding through an adapter against the consumer's. A provider validates each snapshot and reset of a remote state member once per (re)subscription, against the consumer's contract through an `accepts` adapter and the provider's otherwise; ops are validated only in dev mode. At reload, the new provider's description must cover the old provider's when the version is unchanged, in addition to the check against live consumers and facets' `uses`. -- **Contracts.** Local contracts list their members, so a loading reference knows whether a member is a method, state or opaque. Canonical identity omits input fields equal to their default; with TypeBox, use `Type.Optional(Type.X({ default }))` to let callers omit them. Provider variants must cover consumer variants; members the consumer's contract does not list are hidden. Schema change detection compares canonical JSON, so reordering `anyOf` branches counts as a change. Fingerprints use a non-cryptographic 64-bit hash: synchronous and browser-safe; contracts are trusted. +- **Contracts.** Local contracts list their members, so a loading reference knows whether a member is a method, state or opaque. Canonical identity omits input fields equal to their default; use `opt(str({ default }))` (pi-schema) to let callers omit them. Provider variants must cover consumer variants; members the consumer's contract does not list are hidden. Schema change detection compares canonical JSON, so reordering `anyOf` branches counts as a change. Fingerprints use a non-cryptographic 64-bit hash: synchronous and browser-safe; contracts are trusted. - **Principals.** A root ctx at ingress acts as its principal (`host.root(fn, { principal })`); contexts inside objects act as `host`. The acceptor decides a link's kind and name (`accept(link, { principal, kind, name })`); only configured host links are addressable, carry roots and may vote, and control frames are accepted only from the parent. Clients never carry roots or code versions. Relaying needs a matching `route`; `authorize(principal, hostName)` runs before `resolve` and again after, and revoking or re-stamping a principal closes its relays at once. `invalidatePrincipal(match)` re-stamps or closes matching links and propagates pattern matches to child hosts. - **Cells and readables.** A value returned from `change` replaces the value and is always validated; `replace` transfers ownership without copying; by default (`{ diff }` overrides) it publishes a diff when the new value reuses a container of the old one at the same key, else `r`. `subscribe(fn)` delivers values, starting with the current one; `events(fn)` delivers `snap`, then `ops`, or `reset`. Reference state members are stable readables that throw `not_ready` while loading and `not_in_variant` outside the variant, and reset when their source moves; a listener added before they are readable gets its snapshot then. Remote state members stay hydrated while the reference is held. - **References and calls.** A reference's identity is its address; a second `self.get` of the same address in one run returns the same reference. A reference call without ctx, or with a task ctx, starts a fresh root in the current code version; a ctx whose call or root already settled is refused (`aborted/expired`), one from another host `invalid/foreign_ctx`. Calling a reference inside a factory throws `not_ready`. diff --git a/packages/chord/docs/tutorial.md b/packages/chord/docs/tutorial.md index fe16dce4c..65a6334ed 100644 --- a/packages/chord/docs/tutorial.md +++ b/packages/chord/docs/tutorial.md @@ -45,11 +45,12 @@ shuts down its objects and connections. Before callers can use an object, they need to agree on its name and members. That agreement is a **contract**: a runtime value that also gives TypeScript its -types. A **schema** describes a value's shape. We use TypeBox's `Type` to write -schemas. `method(args, result)` declares a method by its argument and result -schemas: `method()` takes no arguments and returns nothing, and -`method(undefined, Type.String())`, used below, takes no arguments and returns a -string. +types. A **schema** describes a value's shape: plain JSON Schema, which we write +with the builders of `@earendil-works/pi-schema` (`obj`, `str`, `num`, ...; +`{ open: true }` lets an object carry fields it does not name). +`method(args, result)` declares a method by its argument and result schemas: +`method()` takes no arguments and returns nothing, and +`method(undefined, str())`, used below, takes no arguments and returns a string. Start with a label for a named list. `local` means only code inside this host can reach it. The input `{ list: "family" }` identifies which list we want; @@ -59,8 +60,8 @@ it is not an argument to `label`. ```ts const TodoLabel = local("tutorial.todos", { version: 1, - input: Type.Object({ list: Type.String() }), - members: { label: method(undefined, Type.String()) }, + input: obj({ list: str() }, { open: true }), + members: { label: method(undefined, str()) }, }); ``` @@ -116,17 +117,17 @@ name, because it holds nothing a reload must keep. ```ts -const Todo = Type.Object({ text: Type.String(), done: Type.Boolean() }); -const Rows = Type.Array(Todo); +const Todo = obj({ text: str(), done: bool() }, { open: true }); +const Rows = arr(Todo); const TodoStore = local("tutorial.todos", { version: 1, - input: Type.Object({ list: Type.String() }), + input: obj({ list: str() }, { open: true }), members: { - label: method(undefined, Type.String()), + label: method(undefined, str()), rows: Rows, - openCount: Type.Number(), - add: method(Type.Object({ text: Type.String() })), - finish: method(Type.Object({ index: Type.Integer() })), + openCount: num(), + add: method(obj({ text: str() }, { open: true })), + finish: method(obj({ index: int() }, { open: true })), reset: method(Rows), }, }); @@ -230,13 +231,13 @@ to a declared resource, without reading the resource during the factory. ```ts const Database = local("tutorial.database", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { db: opaque<{ saved: string[] }>("tutorial.database@1") }, }); const SavedTodos = local("tutorial.saved-todos", { version: 1, - input: Type.Object({ list: Type.String() }), - members: { add: method(Type.Object({ text: Type.String() })) }, + input: obj({ list: str() }, { open: true }), + members: { add: method(obj({ text: str() }, { open: true })) }, }); const host = createHost({ self: "server" }); let opens = 0; @@ -338,13 +339,13 @@ puts every `Todos` object on the host named `server`. Section 6 uses that. ```ts const Todos = remote("tutorial.todo-view", { version: 1, - input: Type.Object({ list: Type.String() }), + input: obj({ list: str() }, { open: true }), host: hostNamed("server"), variants: { viewer: { state: { rows: Rows } }, editor: { state: { rows: Rows }, - methods: { add: method(Type.Object({ text: Type.String() })) }, + methods: { add: method(obj({ text: str() }, { open: true })) }, }, }, }); @@ -499,8 +500,8 @@ test; they are not stored application state. ```ts const Activity = local("tutorial.activity", { version: 1, - input: Type.Object({}), - members: { seen: Type.Number() }, + input: obj({}, { open: true }), + members: { seen: num() }, }); const host = createHost({ self: "server" }); host.serve(TodoStore, todoFactory); @@ -509,7 +510,7 @@ const seeded = Promise.withResolvers(); const stopped = Promise.withResolvers(); host.serve(Activity, (_input, _principal, self) => { const todos = self.get(TodoStore, { list: "family" }); - const seen = self.state("seen", Type.Number(), 0); + const seen = self.state("seen", num(), 0); self.on("rows", todos.rows, (rows, ctx) => { seen.replace(ctx, rows?.length ?? 0); if (rows?.length === 1) observed.resolve(); @@ -570,7 +571,7 @@ default lifetime, `"referenced"`, are still created only when referenced. ```ts const Actions = defineRegistry<(ctx: Ctx) => Promise>("tutorial.actions"); -const Menu = local("tutorial.menu", { version: 1, input: Type.Object({}) }); +const Menu = local("tutorial.menu", { version: 1, input: obj({}, { open: true }) }); let click: (() => Promise) | undefined; const storeFacet: Facet = { name: "todo-store", @@ -745,7 +746,7 @@ const { label: _label, ...members } = TodoStore.def.members; const RenamedStore = local("tutorial.todos", { version: 2, input: TodoStore.def.input, - members: { ...members, heading: method(undefined, Type.String()) }, + members: { ...members, heading: method(undefined, str()) }, }); await host.reload([ { @@ -890,7 +891,7 @@ tutorial's source-matching check, not by the application. ```ts import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; -import { type Static, Type } from "typebox"; +import { arr, bool, int, num, obj, type Static, str } from "@earendil-works/pi-schema"; import { it, onTestFinished } from "vitest"; import { type Ctx, diff --git a/packages/chord/package.json b/packages/chord/package.json index 908005452..edced8cd5 100644 --- a/packages/chord/package.json +++ b/packages/chord/package.json @@ -90,8 +90,8 @@ "node": ">=22.19.0" }, "dependencies": { - "@earendil-works/pi-telemetry": "1.4.0", - "typebox": "1.3.27" + "@earendil-works/pi-schema": "1.4.0", + "@earendil-works/pi-telemetry": "1.4.0" }, "peerDependencies": { "esbuild": "0.28.2", diff --git a/packages/chord/src/cell.ts b/packages/chord/src/cell.ts index 65d391128..d520258e3 100644 --- a/packages/chord/src/cell.ts +++ b/packages/chord/src/cell.ts @@ -1,4 +1,4 @@ -import type { TSchema } from "typebox"; +import type { Schema } from "@earendil-works/pi-schema"; import { type Draft, diffRevisions, type JsonValue, type Op, type Tracker, track } from "./delta/index.ts"; import { ChordError } from "./errors.ts"; import { enqueue, holdPublications } from "./publish.ts"; @@ -87,7 +87,7 @@ export abstract class Source implements Readable { readonly seq = ++sequence; readonly dependents = new Set>(); /** Schemas of remote state members currently exposing this source (checked per write in dev mode). */ - schemas: readonly TSchema[] = []; + schemas: readonly Schema[] = []; protected readonly options: SourceOptions; protected readonly fanout: Fanout; @@ -114,7 +114,7 @@ export abstract class Source implements Readable { for (const schema of this.schemas) assertValid("state", `state ${this.key}`, schema, value); } - protected ownSchemas(): readonly TSchema[] { + protected ownSchemas(): readonly Schema[] { return []; } } @@ -149,7 +149,7 @@ function unbox(ops: readonly Op[], value: unknown): Op[] { /** A state cell holding any JSON value, validated against its schema at creation and (dev mode) on writes. */ export class Cell extends Source { /** Replaced at a reload commit that changes the schema (state-version bump). */ - schema: TSchema; + schema: Schema; #tracker: Tracker>; readonly #guard: WriteGuard; /** False until `initialize` for a cell declared with `init`: reads and writes throw `not_ready`. */ @@ -161,7 +161,7 @@ export class Cell extends Source { */ constructor( key: string, - schema: TSchema, + schema: Schema, initial: { readonly value: T } | undefined, guard: WriteGuard, options: SourceOptions, @@ -182,7 +182,7 @@ export class Cell extends Source { this.#initialized = true; } - protected override ownSchemas(): readonly TSchema[] { + protected override ownSchemas(): readonly Schema[] { return [this.schema]; } @@ -238,7 +238,7 @@ export class Cell extends Source { * Installs a migrated value and its schema (reload commit). The caller validated the value and the derivations * over it against every schema that will apply, so nothing is checked here. Subscribers receive a reset. */ - reset(value: T, schema: TSchema): void { + reset(value: T, schema: Schema): void { this.schema = schema; this.#publish(this.#tracker.prepareReplace({ v: value }), "reset", false); } diff --git a/packages/chord/src/contract.ts b/packages/chord/src/contract.ts index c2ebd6b96..1a6d64031 100644 --- a/packages/chord/src/contract.ts +++ b/packages/chord/src/contract.ts @@ -1,15 +1,14 @@ -import { type Static, type TSchema, Type } from "typebox"; -import { Value } from "typebox/value"; +import { int, lit, map, obj, opt, type Schema, type Static, str, union } from "@earendil-works/pi-schema"; import { ChordError, invalid } from "./errors.ts"; -import { canonicalJson, fits, hash64 } from "./schema.ts"; +import { canonicalJson, fits, hash64, isValid } from "./schema.ts"; import type { Ctx, Exposed, Readable, RefBase, StateSource } from "./types.ts"; const MEMBER = Symbol("chord.member"); /** A method's arguments and result; `undefined`: it takes no arguments, or returns nothing (void). */ export interface MethodSpec< - A extends TSchema | undefined = TSchema | undefined, - R extends TSchema | undefined = TSchema | undefined, + A extends Schema | undefined = Schema | undefined, + R extends Schema | undefined = Schema | undefined, > { readonly [MEMBER]: "method"; readonly args: A; @@ -29,9 +28,9 @@ export interface OpaqueSpec { * for a method without arguments that returns something. */ export function method(): MethodSpec; -export function method(args: A): MethodSpec; -export function method(args: A, result: R): MethodSpec; -export function method(args?: TSchema, result?: TSchema): MethodSpec { +export function method(args: A): MethodSpec; +export function method(args: A, result: R): MethodSpec; +export function method(args?: Schema, result?: Schema): MethodSpec { return { [MEMBER]: "method", args, result }; } @@ -42,7 +41,7 @@ export const opaque = (id: string): OpaqueSpec => { }; export interface VariantDef { - readonly state?: Readonly>; + readonly state?: Readonly>; readonly methods?: Readonly>; } @@ -102,63 +101,59 @@ export const placementText = (placement: Placement): string => export interface RemoteDef { readonly version: number; - readonly input: TSchema; + readonly input: Schema; /** Where its objects live: `hostNamed(…)` or `hostFamily(…)`. Part of the contract (and its fingerprint). */ readonly host: Placement; readonly variants: Readonly>; } /** A local member: a method, an opaque native object, or a state member described by its schema. */ -export type LocalMember = MethodSpec | OpaqueSpec | TSchema; +export type LocalMember = MethodSpec | OpaqueSpec | Schema; export interface LocalDef { readonly version: number; - readonly input: TSchema; + readonly input: Schema; readonly members?: Readonly>; } // A description travels as a reference's contract token (wire.ts), so it is declared once, as the schema that validates it. -// `TSchema` is an empty interface: any object is one, which is all a peer's schema is checked for here. -const SchemaObject = Type.Unsafe(Type.Object({})); +// Any object is a schema, which is all a peer's schema is checked for here. +const SchemaObject = obj({}, { open: true }) as Schema; -const MemberDescriptionSchema = Type.Union([ +const MemberDescriptionSchema = union([ // Absent `args`: the method takes none. Absent `result`: it returns nothing. - Type.Object({ - kind: Type.Literal("method"), - args: Type.Optional(SchemaObject), - result: Type.Optional(SchemaObject), - }), - Type.Object({ kind: Type.Literal("state"), schema: SchemaObject }), - Type.Object({ kind: Type.Literal("opaque"), id: Type.String() }), + obj({ kind: lit("method"), args: opt(SchemaObject), result: opt(SchemaObject) }, { open: true }), + obj({ kind: lit("state"), schema: SchemaObject }, { open: true }), + obj({ kind: lit("opaque"), id: str() }, { open: true }), ]); export type MemberDescription = Static; /** Local contracts have one unnamed variant. */ export const LOCAL_VARIANT = ""; -const PlacementSchema = Type.Union([ - Type.Object({ name: Type.String({ minLength: 1 }) }, { additionalProperties: false }), - Type.Object( - { prefix: Type.String({ minLength: 1 }), key: Type.String({ minLength: 1 }) }, - { additionalProperties: false }, - ), +const PlacementSchema = union([ + obj({ name: str({ minLength: 1 }) }), + obj({ prefix: str({ minLength: 1 }), key: str({ minLength: 1 }) }), ]); -export const DescriptionSchema = Type.Object({ - name: Type.String({ minLength: 1 }), - scope: Type.Union([Type.Literal("remote"), Type.Literal("local")]), - version: Type.Integer(), - input: SchemaObject, - /** Remote types only (required there): where their objects live. */ - host: Type.Optional(PlacementSchema), - variants: Type.Record(Type.String(), Type.Record(Type.String(), MemberDescriptionSchema)), -}); +export const DescriptionSchema = obj( + { + name: str({ minLength: 1 }), + scope: union([lit("remote"), lit("local")]), + version: int(), + input: SchemaObject, + /** Remote types only (required there): where their objects live. */ + host: opt(PlacementSchema), + variants: map(map(MemberDescriptionSchema)), + }, + { open: true }, +); export type Description = Static; interface ContractBase { readonly name: string; readonly version: number; - readonly input: TSchema; + readonly input: Schema; readonly description: Description; readonly fingerprint: string; } @@ -204,7 +199,7 @@ function describeMember(name: string, member: unknown, scope: "remote" | "local" if (scope === "remote") throw new TypeError(`remote contracts cannot expose opaque members: ${name}`); return { kind: "opaque", id: (member as OpaqueSpec).id }; } - return { kind: "state", schema: member as TSchema }; + return { kind: "state", schema: member as Schema }; } function finish(contract: Omit): C { @@ -222,7 +217,7 @@ export function remote( ): RemoteContract { checkHeader(name, def.version); const placement = def.host as Placement | undefined; - if (placement === undefined || !Value.Check(PlacementSchema, placement)) + if (placement === undefined || !isValid(PlacementSchema, placement)) throw new TypeError(`remote contract ${name} needs \`host\`: hostNamed(…) or hostFamily(…)`); // A JSON copy: the description travels, and a placement object may carry nothing else. const host: Placement = @@ -353,23 +348,23 @@ export function bind(provider: AnyContract, accepts: Accepts | undefined, consum // ─── Static types ──────────────────────────────────────────────────────────── type Empty = Record; -type StateOf = V extends { readonly state: infer S extends Readonly> } ? S : Empty; -type StaticOf = S extends TSchema ? Static : never; +type StateOf = V extends { readonly state: infer S extends Readonly> } ? S : Empty; +type StaticOf = S extends Schema ? Static : never; type MethodsOf = V extends { readonly methods: infer M extends Readonly> } ? M : Empty; /** What a handler returns: the result's value, or nothing for a void method. */ -type Returned = [R] extends [TSchema] ? Static | Promise> : void | Promise; +type Returned = [R] extends [Schema] ? Static | Promise> : void | Promise; /** What a call resolves to. */ -type Resolved = [R] extends [TSchema] ? Static : undefined; +type Resolved = [R] extends [Schema] ? Static : undefined; /** A method's handler; `P` is the app's principal type (`ctx.principal`). */ export type MethodImplementation = M extends MethodSpec - ? [A] extends [TSchema] + ? [A] extends [Schema] ? (args: Static, ctx: Ctx

    ) => Returned : (ctx: Ctx

    ) => Returned : never; export type Call = M extends MethodSpec - ? [A] extends [TSchema] + ? [A] extends [Schema] ? (args: Static, ctx?: Ctx) => Promise> : (ctx?: Ctx) => Promise> : never; @@ -402,21 +397,21 @@ type LocalImplementationMember = M extends MethodSpec ? MethodImplementation : M extends OpaqueSpec ? T | Exposed - : M extends TSchema + : M extends Schema ? StateSource> : never; type LocalRefMember = M extends MethodSpec ? Call : M extends OpaqueSpec ? T | undefined - : M extends TSchema + : M extends Schema ? Readable> : never; type LocalMembers = D extends { readonly members: infer M extends Readonly> } ? M : Empty; -export type InputOf = C extends { readonly def: { readonly input: infer I extends TSchema } } ? Static : never; +export type InputOf = C extends { readonly def: { readonly input: infer I extends Schema } } ? Static : never; /** What a factory returns for contract C; its methods' `ctx.principal` is `P`. */ export type ImplementationOf = C extends RemoteContract diff --git a/packages/chord/src/next/contracts/README.md b/packages/chord/src/next/contracts/README.md index bd79e9bc0..7cfd5fd9d 100644 --- a/packages/chord/src/next/contracts/README.md +++ b/packages/chord/src/next/contracts/README.md @@ -34,7 +34,12 @@ type NotesProvider = Implementation; // { variant: "editor"; membe key built like an identity key (`session|S1`); two addresses are equal when their keys are. The params are checked where the address is routed or served (`invalid/host_address`), not when it is made. No host-name strings in app code. - Schema helpers: `str`, `num`, `int`, `bool`, `nul`, `lit`, `oneOf`, `union`, `obj`, `arr`, `map`, `opt`, `bytes`, - `unknown`, and `Static`. They return ordinary TypeBox schemas. + `unknown`, `Schema` and `Static`, over `@earendil-works/pi-schema`'s builders: plain JSON Schema typed with the values + it accepts. They keep the shapes contracts were fingerprinted with: `lit` carries its JSON `type`, `oneOf(...)` is a + union of `lit`s, `map` one pattern property (`^.*$`), `obj` is closed. `opt(...)` marks an object field (or a table + column) optional and is no schema elsewhere. Any other JSON Schema object of the vocabulary below works too; a + non-enumerable key (a TypeBox schema's metadata) is refused. `format` is an annotation: it is part of the normal form + and the fingerprint, but no format is checked. - `evolve(previous, (e) => e...)`: the next version from structured edits (`state`, `addState`, `removeState`, `args`, `result`, `method`, `addMethod`, `removeMethod`); it must return the editor it was given. - Types: `Contract`, `Implementation` (what a factory returns: for `local` a `Watchable` per state member and a diff --git a/packages/chord/src/next/contracts/contract.ts b/packages/chord/src/next/contracts/contract.ts index 123757d70..696696b19 100644 --- a/packages/chord/src/next/contracts/contract.ts +++ b/packages/chord/src/next/contracts/contract.ts @@ -1,13 +1,12 @@ // Contract handles: the branded, immutable value an app declares, its type-level shape, and the Chord-owned data // behind it. -import type { TNull } from "typebox"; import type { Context } from "../../context/index.ts"; import type { Watchable, WatchHandle } from "../../watch.ts"; import type { Change, CurrentStep, History, VariantShape } from "./history.ts"; import type { Address } from "./host-kind.ts"; import type { Params, ParamsData } from "./params.ts"; -import type { Method, Static } from "./schema.ts"; +import type { Method, NullSchema, Static } from "./schema.ts"; import type { Snapshot } from "./snapshot.ts"; export type Kind = "remote" | "local"; @@ -88,7 +87,7 @@ export interface KeyedWatchable { } /** A method as a function: without the argument when its argument schema is `null` (`null` stays the wire form). */ -type MethodFunction = M["args"] extends TNull +type MethodFunction = M["args"] extends NullSchema ? (context: X) => Promise> : (args: Static, context: X) => Promise>; diff --git a/packages/chord/src/next/contracts/declare.ts b/packages/chord/src/next/contracts/declare.ts index 8bd3f5533..1ef0b0ee8 100644 --- a/packages/chord/src/next/contracts/declare.ts +++ b/packages/chord/src/next/contracts/declare.ts @@ -16,11 +16,11 @@ import type { MemberNames, MethodAt, VariantShape } from "./history.ts"; import type { Address } from "./host-kind.ts"; import { checkName } from "./names.ts"; import { type Params, type RequiredFields, readParams } from "./params.ts"; -import { KeyedMember, Method, type Static, type TSchema } from "./schema.ts"; +import { KeyedMember, Method, type Schema, type Static } from "./schema.ts"; import { type Snapshot, snapshotArguments, snapshotSchema } from "./snapshot.ts"; type VariantDeclaration = { - readonly state?: { readonly [member: string]: TSchema }; + readonly state?: { readonly [member: string]: Schema }; readonly methods?: { readonly [member: string]: Method }; }; @@ -42,7 +42,7 @@ type LocalVariants = { }; /** A local contract's member: a state schema, `method(...)` or `keyed(schema)`. */ -type LocalMember = TSchema | Method | KeyedMember; +type LocalMember = Schema | Method | KeyedMember; type SchemaOf = M extends KeyedMember ? S : never; diff --git a/packages/chord/src/next/contracts/evolve.ts b/packages/chord/src/next/contracts/evolve.ts index 04462793b..d3cb975f4 100644 --- a/packages/chord/src/next/contracts/evolve.ts +++ b/packages/chord/src/next/contracts/evolve.ts @@ -16,7 +16,7 @@ import { import { checkMembers, memberPath, readMethod } from "./declare.ts"; import { materialize } from "./fingerprint.ts"; import type { Change, CurrentStep, Increment, MemberNames, MethodAt, Variants, VariantsWith } from "./history.ts"; -import { Method, type TSchema } from "./schema.ts"; +import { Method, type Schema } from "./schema.ts"; import { type Snapshot, snapshotArguments, snapshotSchema } from "./snapshot.ts"; /** Method edits, the same for both kinds: each changes the method in every variant that has it. */ @@ -25,7 +25,7 @@ interface MethodEditor { readonly "~variants"?: V; /** Type only: the changes made by the edits so far. */ readonly "~changes"?: C; - args, A extends TSchema>( + args, A extends Schema>( member: M, args: A, ): Editor< @@ -33,7 +33,7 @@ interface MethodEditor { Put, "methods", M, Method["result"]>>, C | [VariantsWith, M, "args"] >; - result, R extends TSchema>( + result, R extends Schema>( member: M, result: R, ): Editor< @@ -41,7 +41,7 @@ interface MethodEditor { Put, "methods", M, Method["args"], R>>, C | [VariantsWith, M, "result"] >; - method, A extends TSchema, R extends TSchema>( + method, A extends Schema, R extends Schema>( member: M, args: A, result: R, @@ -51,12 +51,12 @@ interface MethodEditor { /** Edits of a remote contract. */ interface RemoteEditor extends MethodEditor<"remote", V, C> { - state( + state( variant: X, member: M, schema: S, ): RemoteEditor, C | [X, M, "state"]>; - addState( + addState( variant: X, member: M, schema: S, @@ -65,7 +65,7 @@ interface RemoteEditor extends MethodEdito variant: X, member: M, ): RemoteEditor, C>; - addMethod( + addMethod( variant: X, member: M, args: A, @@ -75,26 +75,26 @@ interface RemoteEditor extends MethodEdito /** Edits of a local contract: like remote ones, without a variant. */ interface LocalEditor extends MethodEditor<"local", V, C> { - state( + state( member: M, schema: S, ): LocalEditor, C | [Local, M, "state"]>; - addState( + addState( member: M, schema: S, ): LocalEditor, C | [Local, M, "state"]>; removeState(member: M): LocalEditor, C>; - addMethod( + addMethod( member: M, args: A, result: R, ): LocalEditor>, C | [Local, M, "args" | "result"]>; /** Changes keyed member `member`'s schema (slice 7b: an older consumer reads each key mapped, or is refused). */ - keyed & string, S extends TSchema>( + keyed & string, S extends Schema>( member: M, schema: S, ): LocalEditor, C | [Local, M, "keyed"]>; - addKeyed( + addKeyed( member: M, schema: S, ): LocalEditor, C | [Local, M, "keyed"]>; @@ -220,14 +220,14 @@ class Draft { this.#kinds = new Map(data.kinds); } - state(variant: string, member: string, schema: TSchema): object { + state(variant: string, member: string, schema: Schema): object { if (!this.#variant(variant).state.has(member)) { throw new Error(`${this.#path(variant, member, "state")}: no such state member`); } return this.#putState(variant, member, schema); } - addState(variant: string, member: string, schema: TSchema): object { + addState(variant: string, member: string, schema: Schema): object { this.#refuseDeclared(variant, member, "state"); return this.#putState(variant, member, schema); } @@ -239,19 +239,19 @@ class Draft { return this.editor; } - args(member: string, args: TSchema): object { + args(member: string, args: Schema): object { return this.#changeMethod(member, args, undefined); } - result(member: string, result: TSchema): object { + result(member: string, result: Schema): object { return this.#changeMethod(member, undefined, result); } - method(member: string, args: TSchema, result: TSchema): object { + method(member: string, args: Schema, result: Schema): object { return this.#changeMethod(member, args, result); } - addMethod(variant: string, member: string, args: TSchema, result: TSchema): object { + addMethod(variant: string, member: string, args: Schema, result: Schema): object { this.#refuseDeclared(variant, member, "method"); this.#variant(variant).methods.set( member, @@ -261,14 +261,14 @@ class Draft { return this.#checked(); } - keyed(variant: string, member: string, schema: TSchema): object { + keyed(variant: string, member: string, schema: Schema): object { if (!this.#variant(variant).keyed.has(member)) { throw new Error(`${this.#path(variant, member, "keyed")}: no such keyed member`); } return this.#putKeyed(variant, member, schema); } - addKeyed(variant: string, member: string, schema: TSchema): object { + addKeyed(variant: string, member: string, schema: Schema): object { this.#refuseDeclared(variant, member, "keyed"); return this.#putKeyed(variant, member, schema); } @@ -292,20 +292,20 @@ class Draft { return { ...this.#data, versions: [...versions, version], kinds: this.#kinds }; } - #putState(variant: string, member: string, schema: TSchema): object { + #putState(variant: string, member: string, schema: Schema): object { this.#variant(variant).state.set(member, snapshotSchema(schema, this.#path(variant, member, "state"))); this.#changes.push([variant, member, "state"]); return this.#checked(); } - #putKeyed(variant: string, member: string, schema: TSchema): object { + #putKeyed(variant: string, member: string, schema: Schema): object { this.#variant(variant).keyed.set(member, snapshotSchema(schema, this.#path(variant, member, "keyed"))); this.#changes.push([variant, member, "keyed"]); return this.#checked(); } /** Changes the given sides of a method in every variant that has it. */ - #changeMethod(member: string, args: TSchema | undefined, result: TSchema | undefined): object { + #changeMethod(member: string, args: Schema | undefined, result: Schema | undefined): object { const variants = this.#variantsWith(member); const path = this.#path(variants[0]!, member, "method"); const newArgs = args === undefined ? undefined : snapshotArguments(args, `${path}.args`); @@ -355,16 +355,16 @@ class Draft { /** The local editor: the draft's edits with the variant filled in. */ function localEditor(draft: Draft): object { draft.editor = { - state: (member: string, schema: TSchema) => draft.state(LOCAL, member, schema), - addState: (member: string, schema: TSchema) => draft.addState(LOCAL, member, schema), + state: (member: string, schema: Schema) => draft.state(LOCAL, member, schema), + addState: (member: string, schema: Schema) => draft.addState(LOCAL, member, schema), removeState: (member: string) => draft.removeState(LOCAL, member), - addMethod: (member: string, args: TSchema, result: TSchema) => draft.addMethod(LOCAL, member, args, result), - args: (member: string, args: TSchema) => draft.args(member, args), - result: (member: string, result: TSchema) => draft.result(member, result), - method: (member: string, args: TSchema, result: TSchema) => draft.method(member, args, result), + addMethod: (member: string, args: Schema, result: Schema) => draft.addMethod(LOCAL, member, args, result), + args: (member: string, args: Schema) => draft.args(member, args), + result: (member: string, result: Schema) => draft.result(member, result), + method: (member: string, args: Schema, result: Schema) => draft.method(member, args, result), removeMethod: (member: string) => draft.removeMethod(member), - keyed: (member: string, schema: TSchema) => draft.keyed(LOCAL, member, schema), - addKeyed: (member: string, schema: TSchema) => draft.addKeyed(LOCAL, member, schema), + keyed: (member: string, schema: Schema) => draft.keyed(LOCAL, member, schema), + addKeyed: (member: string, schema: Schema) => draft.addKeyed(LOCAL, member, schema), removeKeyed: (member: string) => draft.removeKeyed(LOCAL, member), }; return draft.editor; diff --git a/packages/chord/src/next/contracts/history.ts b/packages/chord/src/next/contracts/history.ts index a664f98ff..2a3fb1a80 100644 --- a/packages/chord/src/next/contracts/history.ts +++ b/packages/chord/src/next/contracts/history.ts @@ -1,13 +1,13 @@ // The type-level history of a contract: every version's members and the changes that produced it. -import type { Method, TSchema } from "./schema.ts"; +import type { Method, Schema } from "./schema.ts"; /** One variant's members at one version: state schemas and method signatures. */ export type VariantShape = { - readonly state: { readonly [member: string]: TSchema }; + readonly state: { readonly [member: string]: Schema }; readonly methods: { readonly [member: string]: Method }; /** A local contract's keyed state members (slice 6a §12). */ - readonly keyed?: { readonly [member: string]: TSchema }; + readonly keyed?: { readonly [member: string]: Schema }; }; /** Variants by name. A local contract has the single variant `members`. */ diff --git a/packages/chord/src/next/contracts/index.ts b/packages/chord/src/next/contracts/index.ts index 299bac9d3..da4a7fbe9 100644 --- a/packages/chord/src/next/contracts/index.ts +++ b/packages/chord/src/next/contracts/index.ts @@ -7,24 +7,30 @@ export { evolve } from "./evolve.ts"; export { type Address, type HostKind, hostKind, type KindShape, type ParamsOfKind } from "./host-kind.ts"; export { arr, + type BooleanSchema, type Bounds, bool, bytes, type Defaulted, + type IntegerSchema, int, type KeyedMember, keyed, lit, map, method, + type NullSchema, + type NumberSchema, nul, num, + type Optional, obj, oneOf, opt, + type Schema, type Static, + type StringSchema, str, - type TSchema, union, unknown, } from "./schema.ts"; diff --git a/packages/chord/src/next/contracts/params.ts b/packages/chord/src/next/contracts/params.ts index 565c88500..d827edc3a 100644 --- a/packages/chord/src/next/contracts/params.ts +++ b/packages/chord/src/next/contracts/params.ts @@ -1,26 +1,35 @@ // Params declarations: the policy for params fields and the closed object schema checked on every get. -import type { TBoolean, TInteger, TLiteral, TString, TUnion } from "typebox"; import type { JsonValue } from "../../json.ts"; -import type { TSchema } from "./schema.ts"; +import { + type BooleanSchema, + type IntegerSchema, + isOptional, + type LiteralsSchema, + type Optional, + type Schema, + type StringSchema, +} from "./schema.ts"; import { snapshotSchema } from "./snapshot.ts"; -/** A params field: a string, an integer, a boolean or a union of string literals; constraints allowed. */ -type ParamSchema = TString | TInteger | TBoolean | TUnion[]>; +/** A params field: a string, an integer, a boolean or a union of string literals; constraints allowed, no default. */ +type ParamSchema = (StringSchema | IntegerSchema | BooleanSchema | LiteralsSchema) & { + readonly default?: never; +}; /** Params: a flat record of required fields, in identity order. */ export type Params = { readonly [field: string]: ParamSchema }; /** Refuses optional params fields, which the field types alone admit. */ export type RequiredFields

    = { - [F in keyof P]: P[F] extends { readonly "~optional": true } ? never : P[F]; + [F in keyof P]: P[F] extends Optional ? never : P[F]; }; export type ParamsData = { /** Field names in identity order. */ readonly fields: readonly string[]; /** The closed object schema of all fields. */ - readonly schema: TSchema; + readonly schema: Schema; /** `[field, normal form]` pairs in identity order. */ readonly normal: JsonValue; }; @@ -29,10 +38,10 @@ export type ParamsData = { export function readParams(declared: readonly [string, unknown][], path: string): ParamsData { const fields = declared.map(([field, schema]) => { const fieldPath = `${path}.${field}`; + const refused = new Error(`${fieldPath}: a param is a required str(), int(), bool() or oneOf(...) of strings`); + if (isOptional(schema)) throw refused; const snapshot = snapshotSchema(schema, fieldPath); - if (Object.hasOwn(schema as object, "~optional") || !isParamSchema(snapshot.normal)) { - throw new Error(`${fieldPath}: a param is a required str(), int(), bool() or oneOf(...) of strings`); - } + if (!isParamSchema(snapshot.normal)) throw refused; return [field, snapshot] as const; }); const schema = { diff --git a/packages/chord/src/next/contracts/schema.ts b/packages/chord/src/next/contracts/schema.ts index fc987dbb2..230d3a039 100644 --- a/packages/chord/src/next/contracts/schema.ts +++ b/packages/chord/src/next/contracts/schema.ts @@ -1,24 +1,26 @@ -// Schema helpers: short constructors for TypeBox schemas, the bytes leaf, and method declarations. +// Schema helpers: Chord's short constructors over pi-schema's builders (plain JSON Schema, typed with the values they +// accept), and method and keyed member declarations. The helpers keep the shapes contracts were fingerprinted with: +// literals carry their `type`, `oneOf` is a union of literals, `map` a pattern-property record. -import Type, { - type TArray, - type TBoolean, - type TInteger, - type TLiteral, - type TNull, - type TNumber, - type TObject, - type TOptional, - type TProperties, - type TRecord, - type TSchema, - type TString, - type TUnion, - type TUnknown, - type TUnsafe, -} from "typebox"; +import { + arr as arrSchema, + bool as boolSchema, + bytes as bytesSchema, + int as intSchema, + nul as nulSchema, + num as numSchema, + Optional, + obj as objSchema, + type Properties, + type Schema, + type Static, + type StaticObject, + str as strSchema, + union as unionSchema, + unknown as unknownSchema, +} from "@earendil-works/pi-schema"; -export type { Static, TSchema } from "typebox"; +export { Optional, opt, type Schema, type Static } from "@earendil-works/pi-schema"; /** Bounds of a string length, a number or an array length. */ export type Bounds = { readonly min?: number; readonly max?: number }; @@ -26,118 +28,132 @@ export type Bounds = { readonly min?: number; readonly max?: number }; /** A declared default value: a table column with one may be left out of a `put` (slice 6a §5). */ export type Defaulted = { readonly default: T }; +// Interfaces, not intersections: declarations of apps' exported contracts and tables name them (an intersection with +// `Schema` would print pi-schema's private brand, which declaration files cannot name). + +/** A string schema (`str()`), told apart from the other scalars by its `type`. */ +export interface StringSchema extends Schema { + readonly type: "string"; +} + +/** A number schema (`num()`). */ +export interface NumberSchema extends Schema { + readonly type: "number"; +} + +/** An integer schema (`int()`). */ +export interface IntegerSchema extends Schema { + readonly type: "integer"; +} + +/** A boolean schema (`bool()`). */ +export interface BooleanSchema extends Schema { + readonly type: "boolean"; +} + +/** The `null` schema (`nul()`): also the omitted side of a method. */ +export interface NullSchema extends Schema { + readonly type: "null"; +} + +/** A union of literals (`oneOf(...)`). */ +export interface LiteralsSchema extends Schema { + readonly anyOf: readonly Schema[]; +} + /** A string, with optional length bounds, pattern and default. */ export function str( options: Bounds & { readonly pattern?: string; readonly default: string }, -): TString & Defaulted; -export function str(options?: Bounds & { readonly pattern?: string }): TString; -export function str(options: Bounds & { readonly pattern?: string; readonly default?: string } = {}): TString { +): StringSchema & Defaulted; +export function str(options?: Bounds & { readonly pattern?: string }): StringSchema; +export function str(options: Bounds & { readonly pattern?: string; readonly default?: string } = {}): StringSchema { const pattern = options.pattern === undefined ? {} : { pattern: options.pattern }; - return Type.String({ ...limits(options, "minLength", "maxLength"), ...pattern, ...defaultOf(options) }); + return strSchema({ + ...limits(options, "minLength", "maxLength"), + ...pattern, + ...defaultOf(options), + }) as StringSchema; } /** A finite number within optional bounds, with an optional default. */ -export function num(options: Bounds & { readonly default: number }): TNumber & Defaulted; -export function num(options?: Bounds): TNumber; -export function num(options: Bounds & { readonly default?: number } = {}): TNumber { - return Type.Number({ ...limits(options, "minimum", "maximum"), ...defaultOf(options) }); +export function num(options: Bounds & { readonly default: number }): NumberSchema & Defaulted; +export function num(options?: Bounds): NumberSchema; +export function num(options: Bounds & { readonly default?: number } = {}): NumberSchema { + return numSchema({ ...limits(options, "minimum", "maximum"), ...defaultOf(options) }) as NumberSchema; } /** An integer within optional bounds, with an optional default. */ -export function int(options: Bounds & { readonly default: number }): TInteger & Defaulted; -export function int(options?: Bounds): TInteger; -export function int(options: Bounds & { readonly default?: number } = {}): TInteger { - return Type.Integer({ ...limits(options, "minimum", "maximum"), ...defaultOf(options) }); +export function int(options: Bounds & { readonly default: number }): IntegerSchema & Defaulted; +export function int(options?: Bounds): IntegerSchema; +export function int(options: Bounds & { readonly default?: number } = {}): IntegerSchema { + return intSchema({ ...limits(options, "minimum", "maximum"), ...defaultOf(options) }) as IntegerSchema; } /** A boolean, with an optional default. */ -export function bool(options: { readonly default: boolean }): TBoolean & Defaulted; -export function bool(options?: Record): TBoolean; -export function bool(options: { readonly default?: boolean } = {}): TBoolean { - return Type.Boolean(defaultOf(options)); +export function bool(options: { readonly default: boolean }): BooleanSchema & Defaulted; +export function bool(options?: Record): BooleanSchema; +export function bool(options: { readonly default?: boolean } = {}): BooleanSchema { + return boolSchema(defaultOf(options)) as BooleanSchema; } /** The `default` keyword of `options`, left out when absent (snapshots refuse `undefined`). */ -function defaultOf(options: { readonly default?: string | number | boolean }): { default?: string | number | boolean } { +function defaultOf(options: { readonly default?: T }): { default?: T } { return options.default === undefined ? {} : { default: options.default }; } /** `null`; also the omitted side of a method. */ -export function nul(): TNull { - return Type.Null(); +export function nul(): NullSchema { + // The builder's `type: "null"`, typed as such. + return nulSchema() as NullSchema; } -/** Exactly the given value. */ -export function lit(value: V): TLiteral { - return Type.Literal(value); +/** Exactly the given value, with its JSON type (`{ type: "string", const: "a" }`). */ +export function lit(value: V): Schema { + return { type: typeof value, const: value }; } -/** One of the given literals. */ -export function oneOf(...values: V): TUnion> { - // One literal schema per value, in order. - return Type.Union(values.map((value) => Type.Literal(value))) as TUnion>; +/** A value `lit(...)` and `oneOf(...)` take. */ +export type Literal = string | number | boolean; + +/** One of the given literals: a union of `lit(...)` schemas, in order. */ +export function oneOf(...values: V): LiteralsSchema { + return { anyOf: values.map((value) => lit(value)) }; } -type Literal = string | number | boolean; - -type Literals = V extends [infer First extends Literal, ...infer Rest extends Literal[]] - ? [TLiteral, ...Literals] - : []; - /** Any of the given schemas. */ -export function union(schemas: [...S]): TUnion { - return Type.Union(schemas); +export function union(schemas: S): Schema> { + return unionSchema(schemas); } -/** A closed object: unknown fields are refused. */ -export function obj

    (properties: P): TObject

    { - return Type.Object(properties, { additionalProperties: false }); +/** A closed object: unknown fields are refused; `opt(...)` marks an optional field. */ +export function obj(properties: P): Schema> { + return objSchema(properties); } /** An array, with optional length bounds. */ -export function arr(items: S, options: Bounds = {}): TArray { - return Type.Array(items, limits(options, "minItems", "maxItems")); +export function arr(items: Schema, options: Bounds = {}): Schema { + return arrSchema(items, limits(options, "minItems", "maxItems")); } -/** A string-keyed record. */ -export function map(value: S): TRecord<"^.*$", S> { - return Type.Record(Type.String(), value); -} - -/** An optional object field. */ -export function opt(schema: S): TOptional { - return Type.Optional(schema) as TOptional; +/** A string-keyed record: one pattern property matching every key. */ +export function map(value: Schema): Schema> { + return { type: "object", patternProperties: { "^.*$": value } }; } /** Any value. */ -export function unknown(): TUnknown { - return Type.Unknown(); +export function unknown(): Schema { + return unknownSchema(); } /** - * A `Uint8Array` leaf, checked by `instanceof` and length, never enumerated. Its enumerable form is the data marker - * `{ type: "bytes", minLength?, maxLength? }`. + * A `Uint8Array` leaf, checked by `instanceof` and length, never enumerated: `{ type: "bytes", minLength?, maxLength? }` + * (pi-schema's extension of JSON Schema for values that never pass through JSON). */ -export function bytes(options: Bounds = {}): TUnsafe { - return withBytesCheck({ type: "bytes", ...limits(options, "minLength", "maxLength") }); +export function bytes(options: Bounds = {}): Schema { + return bytesSchema(limits(options, "minLength", "maxLength")); } -/** The data marker of a bytes schema. */ -export type BytesMarker = { readonly type: "bytes"; readonly minLength?: number; readonly maxLength?: number }; - -/** Attaches the TypeBox check of a bytes marker, non-enumerably so it is never copied as data. */ -export function withBytesCheck(marker: BytesMarker): TUnsafe { - const { minLength = 0, maxLength = Number.POSITIVE_INFINITY } = marker; - const refinement = { - check: (value: unknown) => value instanceof Uint8Array && value.length >= minLength && value.length <= maxLength, - error: () => "expected bytes of the declared length", - }; - Object.defineProperty(marker, "~refine", { value: [refinement] }); - // The marker object is the schema; TypeBox reads the attached check. - return marker as unknown as TUnsafe; -} - -/** TypeBox options for the given bounds; an absent bound is left out, since snapshots refuse `undefined`. */ +/** Builder options for the given bounds; an absent bound is left out, since snapshots refuse `undefined`. */ function limits(options: Bounds, min: string, max: string): { [keyword: string]: number } { const result: { [keyword: string]: number } = {}; if (options.min !== undefined) result[min] = options.min; @@ -145,8 +161,13 @@ function limits(options: Bounds, min: string, max: string): { [keyword: string]: return result; } +/** Whether a declared object field is `opt(...)`. */ +export function isOptional(value: unknown): value is Optional { + return value instanceof Optional; +} + /** A method declaration: its argument and result schemas; an omitted side is `null` on the wire. */ -export class Method { +export class Method { readonly args: A; readonly result: R; @@ -157,7 +178,7 @@ export class Method { } /** A keyed state member's declaration (slice 6a §12): documents of `schema` by string key; only in `local` contracts. */ -export class KeyedMember { +export class KeyedMember { readonly schema: S; constructor(schema: S) { @@ -166,15 +187,15 @@ export class KeyedMember { } /** Declares a keyed state member of a `local` contract: documents of `schema` by string key, each watchable. */ -export function keyed(schema: S): KeyedMember { +export function keyed(schema: S): KeyedMember { return new KeyedMember(schema); } /** Declares a method: `method()`, `method(args)`, `method(args, result)` or `method(undefined, result)`. */ -export function method(): Method; -export function method(args: A): Method; -export function method(args: undefined, result: R): Method; -export function method(args: A, result: R): Method; -export function method(args?: TSchema, result?: TSchema): Method { +export function method(): Method; +export function method(args: A): Method; +export function method(args: undefined, result: R): Method; +export function method(args: A, result: R): Method; +export function method(args?: Schema, result?: Schema): Method { return new Method(args ?? nul(), result ?? nul()); } diff --git a/packages/chord/src/next/contracts/snapshot.ts b/packages/chord/src/next/contracts/snapshot.ts index a84ed79c0..c8ec2eafb 100644 --- a/packages/chord/src/next/contracts/snapshot.ts +++ b/packages/chord/src/next/contracts/snapshot.ts @@ -1,14 +1,14 @@ // Schema snapshots: one schema-aware walk checks an app schema against Chord's vocabulary and produces a // Chord-owned copy and its normal form (the fingerprint input). +import type { JsonSchema } from "@earendil-works/pi-schema"; import { isJsonValue, type JsonValue } from "../../json.ts"; -import { type BytesMarker, type TSchema, withBytesCheck } from "./schema.ts"; /** * A Chord-owned copy of an app schema, its normal form, and whether it has a `bytes()` leaf (values crossing a link * then carry raw byte segments); immutable by convention. */ -export type Snapshot = { readonly schema: TSchema; readonly normal: JsonValue; readonly bytes: boolean }; +export type Snapshot = { readonly schema: JsonSchema; readonly normal: JsonValue; readonly bytes: boolean }; /** Snapshot of a state, result or params schema. Throws an Error naming `path` for anything outside the vocabulary. */ export function snapshotSchema(source: unknown, path: string): Snapshot { @@ -51,11 +51,6 @@ const KEYWORDS = new Map( const TYPES = new Set(["null", "boolean", "number", "integer", "string", "array", "object", "bytes"]); -/** TypeBox's executable features, attached non-enumerably. */ -const EXECUTABLE = ["~refine", "~codec", "~unsafe"]; - -const BYTES_EXECUTABLE = ["~refine", "~unsafe"]; - const BYTES_KEYWORDS = new Set(["type", "minLength", "maxLength"]); function walk(source: unknown, path: string, objects: Objects, ancestors: Set): Snapshot { @@ -63,13 +58,6 @@ function walk(source: unknown, path: string, objects: Objects, ancestors: Set): Copy { @@ -167,15 +151,15 @@ function sortedCopy(value: JsonValue): JsonValue { return Object.fromEntries(keys.map((key) => [key, sortedCopy(value[key]!)])); } -/** Enumerable data entries sorted by key; symbol keys and accessors are refused. */ +/** Data entries sorted by key; symbol keys, accessors and non-enumerable keys (not data) are refused. */ function ownEntries(source: object, path: string): [string, unknown][] { const entries: [string, unknown][] = []; for (const key of Reflect.ownKeys(source)) { if (typeof key === "symbol") throw new Error(`${path}: symbol keys are not supported`); const descriptor = Object.getOwnPropertyDescriptor(source, key)!; if (!("value" in descriptor)) throw new Error(`${path}.${key}: accessors are not supported`); - // Non-enumerable keys are TypeBox metadata (`~kind`, `~optional`), never schema data. - if (descriptor.enumerable) entries.push([key, descriptor.value]); + if (!descriptor.enumerable) throw new Error(`${path}.${key}: non-enumerable keys are not supported`); + entries.push([key, descriptor.value]); } return entries.sort(([a], [b]) => compare(a, b)); } diff --git a/packages/chord/src/next/data/README.md b/packages/chord/src/next/data/README.md index c3caba867..74248adb4 100644 --- a/packages/chord/src/next/data/README.md +++ b/packages/chord/src/next/data/README.md @@ -51,8 +51,9 @@ Rules: them. Results and state are never checked. Nothing in L1 is frozen. - A method without arguments (argument schema `null`) omits the argument in references and implementations; `null` is its internal and wire form, which `checkArgs` checks. Unknown fields fail. -- A failed check is `invalid/args`, `invalid/params` or `invalid/host_address` with `details.path`, a JSON Pointer from TypeBox's first error. - TypeBox does not escape `~` and `/`, so keys containing them make the path ambiguous. +- A failed check is `invalid/args`, `invalid/params` or `invalid/host_address` with `details.path`, the JSON Pointer of + the first error pi-schema explains (`~` and `/` in keys escaped as `~0` and `~1`); a missing field is reported at its + own path ("missing"), an unknown one too ("unknown field"), the value itself at `/`. - `checkArgs` refuses an unknown method with `invalid/member` and an unaccepted version or a refused method with `contract_mismatch`. It never invokes a `call` replacement. - Error codes, reasons and their pairings are link protocol: changing them bumps `PROTOCOL_VERSION`. diff --git a/packages/chord/src/next/data/args.ts b/packages/chord/src/next/data/args.ts index 3239d263e..83fe60013 100644 --- a/packages/chord/src/next/data/args.ts +++ b/packages/chord/src/next/data/args.ts @@ -1,6 +1,6 @@ // Arguments: a provider's binding (its dispatch table per accepted version) and the per-call argument check. -import type { Validator } from "typebox/compile"; +import type { Validator } from "@earendil-works/pi-schema"; import type { Context } from "../../context/index.ts"; import { type AcceptedVersion, type AcceptsTable, checkAccepts } from "../contracts/accepts.ts"; import { diff --git a/packages/chord/src/next/data/validator.ts b/packages/chord/src/next/data/validator.ts index 4a16599de..f7995f9ab 100644 --- a/packages/chord/src/next/data/validator.ts +++ b/packages/chord/src/next/data/validator.ts @@ -1,28 +1,19 @@ -// Compiled checks: TypeBox validators of Chord-owned schemas, and the `invalid` error of a failed check. +// Compiled checks: pi-schema validators of Chord-owned schemas, and the `invalid` error of a failed check. -import { Compile, type Validator } from "typebox/compile"; -import type { TLocalizedValidationError } from "typebox/error"; -import { ResetExternal, ResetFunctions } from "typebox/schema"; -import { Errors } from "typebox/value"; +import { compile, type JsonSchema, type Validator } from "@earendil-works/pi-schema"; import type { Context } from "../../context/index.ts"; -import type { TSchema } from "../contracts/schema.ts"; import { type ChordError, invalid } from "./errors.ts"; /** A Chord-owned record holding a schema: a snapshot, or a contract's params. */ -type SchemaOwner = { readonly schema: TSchema }; +type SchemaOwner = { readonly schema: JsonSchema }; /** The validator of an owner's schema, compiled on first use (counted) and cached weakly by the owner. */ export function validatorFor(owner: SchemaOwner, context: Context): Validator { let validator = validators.get(owner); if (validator === undefined) { context.telemetry.metrics?.counter("chord.data.compilations").add(1); - try { - validator = Compile(owner.schema); - } finally { - // TypeBox keeps its last build in module-level accumulators; reset them so no build state is retained. - ResetFunctions(); - ResetExternal(); - } + // `format` is an annotation (decided for the schema switch): no format checks are compiled in. + validator = compile(owner.schema); validators.set(owner, validator); } return validator; @@ -38,35 +29,23 @@ export function check( context: Context, ): void { context.telemetry.metrics?.counter("chord.data.validations").add(1); - if (!validator.Check(value)) throw failedCheck(validator.Type(), value, reason, context); + if (!validator.check(value)) throw failedCheck(validator, value, reason, context); } /** - * The error of a failed check: TypeBox's first error from its diagnostic walk (counted; it builds the whole list). - * TypeBox does not escape `~` and `/` in its instance paths, so keys containing them make the path ambiguous. + * The error of a failed check: the first error of the validator's explanation (counted: it walks the schema). A + * missing field is reported at the field ("missing"), an unknown one at its own path ("unknown field"); the value + * itself at `/`. */ function failedCheck( - schema: TSchema, + validator: Validator, value: unknown, reason: "args" | "params" | "host_address", context: Context, ): ChordError { context.telemetry.metrics?.counter("chord.data.walks").add(1); - const [error] = Errors(schema, value); - const { path, problem } = describe(error); - return invalid(reason, `${reason}${path === "/" ? "" : path}: ${problem}`, { path }); -} - -/** - * The path and problem of a TypeBox error: TypeBox reports a missing field at its parent, and an unknown one as a - * false schema (`additionalProperties: false`) at its own path. Anything else keeps TypeBox's message. - */ -function describe(error: TLocalizedValidationError): { path: string; problem: string } { - if (error.keyword === "required") { - return { path: `${error.instancePath}/${error.params.requiredProperties[0]}`, problem: "missing" }; - } - if (error.keyword === "boolean" && error.schemaPath.endsWith("/additionalProperties")) { - return { path: error.instancePath, problem: "unknown field" }; - } - return { path: error.instancePath || "/", problem: error.message }; + // A value that failed the check has at least one error. + const error = validator.errors(value)[0]!; + const path = error.path || "/"; + return invalid(reason, `${reason}${path === "/" ? "" : path}: ${error.message}`, { path }); } diff --git a/packages/chord/src/next/documents/definition.ts b/packages/chord/src/next/documents/definition.ts index e9a2eecf1..26c2105e6 100644 --- a/packages/chord/src/next/documents/definition.ts +++ b/packages/chord/src/next/documents/definition.ts @@ -4,7 +4,7 @@ import type { Context } from "../../context/index.ts"; import type { Op } from "../../delta/index.ts"; import type { JsonValue } from "../../json.ts"; -import type { Static, TSchema } from "../contracts/index.ts"; +import type { Schema, Static } from "../contracts/index.ts"; import { checkName, fingerprintOf } from "../contracts/internal.ts"; import { snapshotSchema } from "../contracts/snapshot.ts"; import { validatorFor } from "../data/internal.ts"; @@ -31,7 +31,7 @@ export type CheckpointInfo = { export type CheckpointWhen = (value: T, ops: readonly Op[], info: CheckpointInfo) => boolean; /** What `defineState` takes for any document. */ -type StateOptions = { +type StateOptions = { readonly schema: S; readonly version: number; readonly initial: () => DocumentOf; @@ -41,7 +41,7 @@ type StateOptions = { }; /** What a persisted document adds (slice 6a §10): `checkpointWhen` is required. */ -type PersistOptions = { +type PersistOptions = { /** Stored in the host's store; only in `local` types with empty params (P3). */ readonly persist: true; /** Called once per commit per changed document; creations and version changes always store a base. */ @@ -52,15 +52,15 @@ type PersistOptions = { * Declares a document: a singleton per object, or with `keyed: true` many per object, each by a string key; with * `persist: true` stored in the host's store. Its name is a restricted name (lowercase `[a-z0-9@/._~-]`). */ -export function defineState( +export function defineState( name: string, options: StateOptions & { readonly persist?: false; readonly checkpointWhen?: never }, ): StateDefinition, K, false>; -export function defineState( +export function defineState( name: string, options: StateOptions & PersistOptions, ): StateDefinition, K, true>; -export function defineState( +export function defineState( name: string, options: StateOptions & { readonly persist?: boolean; @@ -82,14 +82,14 @@ export function defineState( } /** The value of a document whose schema is `S`: its static type, which must be a JSON object. */ -type DocumentOf = Static extends JsonObject ? Static : never; +type DocumentOf = Static extends JsonObject ? Static : never; const NEVER: CheckpointWhen = () => false; /** A declared document; `T` is its value, `K` whether it is keyed, `P` whether it is persisted. */ export class StateDefinition { readonly name: string; - readonly schema: TSchema; + readonly schema: Schema; readonly version: number; readonly keyed: boolean; readonly persist: boolean; @@ -110,7 +110,7 @@ export class StateDefinition T, migrate: ((old: JsonObject, from: number) => T) | undefined, @@ -159,7 +159,7 @@ export class StateDefinition | undefined, > = { /** The job's version; every change of its args, result or state bumps it. */ @@ -40,7 +40,7 @@ export type JobOptions< export type JobPersistence = { readonly resume: "rerun" | "abandon" }; /** Per older version, its mappings forwards: its args to the current args, its result to the current result. */ -export type JobAccepts = { +export type JobAccepts = { readonly [version: number]: { readonly args: (old: never) => Static; readonly result: (old: never) => Static }; }; @@ -66,8 +66,8 @@ export type JobError = /** Declares a job: owned work a call starts that outlives it (§15). */ export function defineJob< - const A extends TSchema, - const R extends TSchema, + const A extends Schema, + const R extends Schema, D extends StateDefinition | undefined = undefined, >(name: string, options: JobOptions): JobDefinition, Static, StateOf> { // Declaration policy: JavaScript callers reach every check below. diff --git a/packages/chord/src/next/link/README.md b/packages/chord/src/next/link/README.md index 806bb4f19..d68605b53 100644 --- a/packages/chord/src/next/link/README.md +++ b/packages/chord/src/next/link/README.md @@ -72,7 +72,7 @@ tab.connect(webSocketLink(new WebSocket(url)), { principal: { kind: "server" } } ## For Chord's layers (`internal.ts`) -- `Message` (`message.ts`): the protocol's one tagged union, from the TypeBox schema `MESSAGE`: `hello { protocol, +- `Message` (`message.ts`): the protocol's one tagged union, from the schema `MESSAGE`: `hello { protocol, incarnation, calls }` (`calls`: how many concurrent calls this side accepts per origin, its call window), `hold { hold, type, version, fingerprint, params, address, principal?, incarnation? }` (`address`: `{ kind, params }` the caller's `hostAddress` computed, it routes the hold; the provider places it by its params; `principal` and `incarnation`: whom it acts for, read only diff --git a/packages/chord/src/next/link/message.ts b/packages/chord/src/next/link/message.ts index 21ac9e0d4..745ce0ad7 100644 --- a/packages/chord/src/next/link/message.ts +++ b/packages/chord/src/next/link/message.ts @@ -1,7 +1,7 @@ -// Messages (§22, R1, R4, 2b): the one `Message` type every link carries, defined as a TypeBox schema and checked after -// every decode, and the codecs that turn it into bytes. +// Messages (§22, R1, R4, 2b): the one `Message` type every link carries, defined as a schema and checked after every +// decode, and the codecs that turn it into bytes. -import Type from "typebox"; +import { obj as openObj, type Schema } from "@earendil-works/pi-schema"; import type { Context } from "../../context/index.ts"; import { arr, int, lit, obj, oneOf, opt, type Static, str, union, unknown } from "../contracts/index.ts"; import { validatorFor } from "../data/internal.ts"; @@ -30,19 +30,19 @@ const wireError = union([ const failed = arr(obj({ member: index, error: wireError })); /** A part's envelope: the codec built it, so any value passes (its content is the owner's to parse). */ -const part = Type.Unsafe({}); +const part = unknown() as Schema; /** * The byte segments of a payload part whose method's schema has a `bytes()` leaf, referenced from its JSON by * `{"$bytes": i}`; absent for every other payload. Built by the codec, like the part. */ -const segments = opt(Type.Unsafe({})); +const segments = opt(unknown() as Schema); /** An address as a hold names it: its host kind and params, checked against the kind where it is routed. */ const address = obj({ kind: str(), params: unknown() }); /** A principal on a hold: JSON with a `kind`, honoured only on a parent link. */ -const principal = Type.Object({ kind: str() }); +const principal = openObj({ kind: str() }, { open: true }); /** A uuidv7 as text. */ const incarnation = str({ pattern: "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" }); @@ -74,7 +74,7 @@ const MESSAGE = union([ principal: opt(principal), incarnation: opt(incarnation), // The instance ids of the activations waiting for this hold (Q5); read only on a trusted link. - chain: opt(Type.Array(incarnation, { maxItems: CHAIN })), + chain: opt(arr(incarnation, { max: CHAIN })), // The decoded limit per part of the link the hold was admitted on (Q2); read only on a parent link. limit: opt(int({ min: 0 })), }), @@ -185,7 +185,7 @@ const MESSAGE_SCHEMA = { schema: MESSAGE }; */ export function checkMessage(message: unknown, context: Context): Message { context.telemetry.metrics?.counter("chord.link.checks").add(1); - if (!validatorFor(MESSAGE_SCHEMA, context).Check(message)) throw new Error("message: fails the message schema"); + if (!validatorFor(MESSAGE_SCHEMA, context).check(message)) throw new Error("message: fails the message schema"); // The check above establishes the type. return message as Message; } diff --git a/packages/chord/src/next/store/archive.ts b/packages/chord/src/next/store/archive.ts index 1db668048..922057994 100644 --- a/packages/chord/src/next/store/archive.ts +++ b/packages/chord/src/next/store/archive.ts @@ -2,9 +2,21 @@ // are operations", "Writes carry their versions; archives are lossless"): the parts an export yields and an import // takes, their order, and the compiled schemas every imported part and every applied write is checked against first. -import Type, { type TInteger } from "typebox"; import type { Context } from "../../context/index.ts"; -import { arr, bool, int, lit, num, obj, opt, str, union, unknown } from "../contracts/index.ts"; +import { + arr, + bool, + type IntegerSchema, + int, + lit, + map, + num, + obj, + opt, + str, + union, + unknown, +} from "../contracts/index.ts"; import { invalid, validatorFor } from "../data/internal.ts"; import type { DocumentWrite } from "./engine.ts"; import type { TableWrite } from "./table-writes.ts"; @@ -34,7 +46,7 @@ export type ArchivePart = | { readonly kind: "end"; readonly writes: number }; /** An integer from `min` up to 2^53 - 1: seqs, ids, versions and counts stay exact (a seq after 2^53 would repeat). */ -function safe(min: number): TInteger { +function safe(min: number): IntegerSchema { return int({ min, max: Number.MAX_SAFE_INTEGER }); } @@ -48,14 +60,14 @@ const column = obj({ fingerprint: opt(str()), }); -const versions = Type.Record(Type.String(), version); +const versions = map(version); const TABLE = obj({ kind: lit("table"), table: str(), version, state: union([lit("ready"), lit("migrating")]), - columns: Type.Record(Type.String(), column), + columns: map(column), key: arr(str()), fingerprint: str(), }); @@ -63,7 +75,7 @@ const TABLE = obj({ const PUT = obj({ kind: lit("put"), table: str(), - row: Type.Record(Type.String(), unknown()), + row: map(unknown()), version, columns: versions, }); @@ -117,7 +129,7 @@ const edit = union([ obj({ kind: lit("json"), name: str(), column }), ]); -const key = Type.Record(Type.String(), union([str(), num(), bool()])); +const key = map(union([str(), num(), bool()])); /** One recorded write of a commit (a batch's); `where` and `set` nodes are checked by the applier. */ const WRITE = union([ @@ -158,7 +170,7 @@ const WRITE_SCHEMA = { schema: WRITE }; /** Checks one imported part against the archive schema (a trust boundary); `invalid/args` naming its position. */ export function checkPart(part: unknown, position: number, context: Context): ArchivePart { context.telemetry.metrics?.counter("chord.data.validations").add(1); - if (!validatorFor(PART_SCHEMA, context).Check(part)) { + if (!validatorFor(PART_SCHEMA, context).check(part)) { throw invalid("args", `import: part ${position}: not an archive part`); } // The check above establishes the type. @@ -168,7 +180,7 @@ export function checkPart(part: unknown, position: number, context: Context): Ar /** Checks one write of a batch's commit against the write schema (a trust boundary); `what` names its place. */ export function checkWrite(write: unknown, what: string, context: Context): StoreWrite { context.telemetry.metrics?.counter("chord.data.validations").add(1); - if (!validatorFor(WRITE_SCHEMA, context).Check(write)) throw invalid("args", `${what}: not a store write`); + if (!validatorFor(WRITE_SCHEMA, context).check(write)) throw invalid("args", `${what}: not a store write`); // The check above establishes the type, the nodes aside (the applier checks them). return write as StoreWrite; } diff --git a/packages/chord/src/next/store/columns.ts b/packages/chord/src/next/store/columns.ts index 8d4118262..180251efb 100644 --- a/packages/chord/src/next/store/columns.ts +++ b/packages/chord/src/next/store/columns.ts @@ -2,8 +2,8 @@ // subset of L1's helpers a table's `columns` take (`str int num bool`, `opt(...)` of them, `json(...)`, defaults), // their types for rows, keys and column references, and the column descriptors and records the store keeps. -import type { TBoolean, TInteger, TNumber, TOptional, TString } from "typebox"; -import type { Static, TSchema } from "../contracts/index.ts"; +import type { BooleanSchema, IntegerSchema, NumberSchema, Schema, Static, StringSchema } from "../contracts/index.ts"; +import { isOptional, type Optional } from "../contracts/schema.ts"; import { JsonColumn, type JsonColumnSchema } from "./json-column.ts"; import { storedText } from "./text.ts"; @@ -11,9 +11,9 @@ import { storedText } from "./text.ts"; export type ColumnSchema = | ScalarSchema | JsonColumnSchema - | TOptional>; + | Optional>; -type ScalarSchema = TString | TInteger | TNumber | TBoolean; +type ScalarSchema = StringSchema | IntegerSchema | NumberSchema | BooleanSchema; /** A table's `columns`: a flat record of column schemas. */ export type Columns = { readonly [name: string]: ColumnSchema }; @@ -22,23 +22,23 @@ export type Columns = { readonly [name: string]: ColumnSchema }; export type ColumnType = "str" | "int" | "num" | "bool" | "json"; /** The store type of column schema `S`. */ -export type ColumnTypeOf = S extends TOptional +export type ColumnTypeOf = S extends Optional ? ColumnTypeOf - : S extends TString + : S extends StringSchema ? "str" - : S extends TInteger + : S extends IntegerSchema ? "int" - : S extends TNumber + : S extends NumberSchema ? "num" - : S extends TBoolean + : S extends BooleanSchema ? "bool" : "json"; /** The value type of column schema `S` (without `undefined` for an `opt(...)` one). */ -export type ValueOf = S extends TOptional ? ValueOf : S extends TSchema ? Static : never; +export type ValueOf = S extends Optional ? ValueOf : S extends Schema ? Static : never; /** The names of the columns that are not `opt(...)`. */ -type RequiredNames = { [K in keyof C]: C[K] extends TOptional ? never : K }[keyof C]; +type RequiredNames = { [K in keyof C]: C[K] extends Optional ? never : K }[keyof C]; /** The names of the columns with a declared default. */ type DefaultedNames = { @@ -57,7 +57,7 @@ export type PutRowOf = { [K in Exclude, Defa /** The columns a key, an index or an `orderBy` may use: non-`opt` `str` or `int`. */ export type OrderColumn = { - [K in keyof C]: C[K] extends TString | TInteger ? (C[K] extends TOptional ? never : K) : never; + [K in keyof C]: C[K] extends Optional ? never : C[K] extends StringSchema | IntegerSchema ? K : never; }[keyof C] & string; @@ -112,15 +112,15 @@ const SCALAR_TYPES: { readonly [type: string]: "str" | "int" | "num" | "bool" } boolean: "bool", }; -function describeColumn(name: string, schema: ColumnSchema, what: string): Column { - const optional = Object.hasOwn(schema, "~optional"); +function describeColumn(name: string, column: ColumnSchema, what: string): Column { + const optional = isOptional(column); // Declaration policy: a JavaScript caller may pass any schema, so its fields are read as unknown data. - const fields = schema as unknown as { readonly [key: string]: unknown }; + const fields = (optional ? column.schema : column) as { readonly [key: string]: unknown }; const json = fields.json; if (json instanceof JsonColumn) return { name, type: "json", optional, json }; const type = SCALAR_TYPES[String(fields.type)]; if (type === undefined) throw new Error(`${what}: expected str(), int(), num(), bool(), opt(...) or json(...)`); - for (const key of Object.keys(schema)) { + for (const key of Object.keys(fields)) { if (key !== "type" && key !== "default") throw new Error(`${what}: "${key}" is not supported in a table column`); } const value = fields.default; diff --git a/packages/chord/src/next/store/expression.ts b/packages/chord/src/next/store/expression.ts index f62f47cf4..0cbc5885f 100644 --- a/packages/chord/src/next/store/expression.ts +++ b/packages/chord/src/next/store/expression.ts @@ -2,9 +2,8 @@ // form"): the builders plugins write `where` and `set` with, and their compilation at staging into one data-only // vocabulary (what the outbox and archives carry; `evaluate.ts` runs it in JavaScript, `sql.ts` renders it). -import type { TOptional } from "typebox"; import type { JsonValue } from "../../json.ts"; -import type { TSchema } from "../contracts/index.ts"; +import type { Optional } from "../contracts/index.ts"; import { invalid } from "../data/internal.ts"; import type { Column, Columns, ColumnType, ColumnTypeOf, ValueOf } from "./columns.ts"; import { checkScalar, encodeCell, type Scalar } from "./rows.ts"; @@ -89,7 +88,7 @@ export class Condition { * the column's type; `undefined` clears an `opt(...)` column. */ export type SetOf = { - readonly [P in Exclude]?: C[P] extends TOptional + readonly [P in Exclude]?: C[P] extends Optional ? ValueOf | Expression, ColumnTypeOf> | undefined : ValueOf | Expression, ColumnTypeOf>; }; diff --git a/packages/chord/src/next/store/json-column.ts b/packages/chord/src/next/store/json-column.ts index 8e8a0a1c7..42195a95e 100644 --- a/packages/chord/src/next/store/json-column.ts +++ b/packages/chord/src/next/store/json-column.ts @@ -2,10 +2,10 @@ // cell with its own version, typed when read, migrated on access (the result checked once, R8), never walked on write // (R9). -import Type, { type TUnsafe } from "typebox"; +import type { JsonSchema } from "@earendil-works/pi-schema"; import type { Context } from "../../context/index.ts"; import type { JsonValue } from "../../json.ts"; -import type { Static, TSchema } from "../contracts/index.ts"; +import type { Schema, Static } from "../contracts/index.ts"; import { fingerprintOf } from "../contracts/internal.ts"; import { snapshotSchema } from "../contracts/snapshot.ts"; import { ChordError } from "../data/index.ts"; @@ -20,24 +20,25 @@ export type JsonColumnOptions = { }; /** A `json(...)` column as `columns` takes it: a schema of the cells' type carrying the column. */ -export type JsonColumnSchema = TUnsafe & { readonly json: JsonColumn }; +export interface JsonColumnSchema extends Schema { + readonly json: JsonColumn; +} /** A column holding one JSON value of `schema` per row, at `version`; wrap it in `opt(...)` to make it nullable. */ -export function json(schema: S, options: JsonColumnOptions>): JsonColumnSchema> { - const column = new JsonColumn(schema, options.version, options.migrate); - // A TypeBox schema (so `opt(...)` and `Static` take it) whose enumerable `json` field `opt`'s clone keeps. - return Type.Unsafe>({ json: column }) as JsonColumnSchema>; +export function json(schema: S, options: JsonColumnOptions>): JsonColumnSchema> { + // Typed as the cells' schema, so `opt(...)` and `Static` take it; the table reads its `json` field. + return { json: new JsonColumn(schema, options.version, options.migrate) }; } /** A `json(...)` column's definition: its schema snapshot, version and fingerprint, and the read of a stored cell. */ export class JsonColumn { - readonly schema: TSchema; + readonly schema: JsonSchema; readonly version: number; /** The fingerprint of the schema's normal form, recorded with the table: the same version with another is `broken`. */ readonly fingerprint: string; readonly #migrate: ((old: JsonValue, from: number) => unknown) | undefined; - constructor(schema: TSchema, version: number, migrate: ((old: JsonValue, from: number) => unknown) | undefined) { + constructor(schema: Schema, version: number, migrate: ((old: JsonValue, from: number) => unknown) | undefined) { // Declaration policy: JavaScript callers reach this check. if (!Number.isSafeInteger(version) || version < 1) throw new Error(`json(...): version ${version} is not a positive integer`); @@ -65,7 +66,7 @@ export class JsonColumn { throw refused(`${what}: migrate from version ${version} threw`, error); } context.telemetry.metrics?.counter("chord.data.validations").add(1); - if (!validatorFor(this, context).Check(migrated)) { + if (!validatorFor(this, context).check(migrated)) { throw refused(`${what}: migrate from version ${version} does not match the schema`); } return migrated; diff --git a/packages/chord/src/next/store/table-evolve.ts b/packages/chord/src/next/store/table-evolve.ts index 3e31ff82e..d0d8723c5 100644 --- a/packages/chord/src/next/store/table-evolve.ts +++ b/packages/chord/src/next/store/table-evolve.ts @@ -3,9 +3,9 @@ // data step, `drop`, `index` and `dropIndex` after it) and track the columns and key through the chain; at runtime one // editor records the edits, checks each as it is made, and materializes the new version at the end. -import type { TOptional } from "typebox"; -import { opt, type TSchema } from "../contracts/index.ts"; +import { opt } from "../contracts/index.ts"; import { editWith, type Increment } from "../contracts/internal.ts"; +import { isOptional, type Optional } from "../contracts/schema.ts"; import { type ColumnSchema, type Columns, @@ -59,7 +59,7 @@ export interface TableEditor ext } /** A column `add` takes: `opt(...)` or one with a `default`. */ -export type AddedColumn = ColumnSchema & (TOptional | { readonly default: unknown }); +export type AddedColumn = ColumnSchema & (Optional | { readonly default: unknown }); /** A `rows` patch: new values of non-key columns; `undefined` clears an `opt(...)` one. */ export type PatchOf = { @@ -85,7 +85,7 @@ type JsonNames = { [P in keyof C]: ColumnTypeOf extends type WithJson = With< C, N, - C[N] extends TOptional ? TOptional : S + C[N] extends Optional ? Optional> : S >; /** `V + 1` for a literal version (contracts' `Increment`); `number` stays `number`. */ @@ -170,7 +170,7 @@ class Editor { this.#assertBefore("json"); const existing = describeColumns({ [name]: this.#assertColumn(name, "json") }, this.#what)[0]!; if (existing.type !== "json") throw new Error(`${this.#what}: json: column "${name}" is not a json(...) column`); - if (Object.hasOwn(column, "~optional")) { + if (isOptional(column)) { throw new Error(`${this.#what}: json: pass json(...) itself; the column keeps its nullability`); } const schema = existing.optional ? opt(column) : column; diff --git a/packages/chord/src/object.ts b/packages/chord/src/object.ts index 6bd1ee24f..d13df8475 100644 --- a/packages/chord/src/object.ts +++ b/packages/chord/src/object.ts @@ -1,5 +1,4 @@ -import type { Static, TSchema } from "typebox"; -import { Value } from "typebox/value"; +import type { Schema, Static } from "@earendil-works/pi-schema"; import { Cell, Derived, @@ -30,7 +29,7 @@ import { asChordError, ChordError, invalid, runAll } from "./errors.ts"; import { enqueue } from "./publish.ts"; import type { Registration, Registry } from "./registry.ts"; import type { RemoteHost } from "./remote.ts"; -import { assertValid, normalize } from "./schema.ts"; +import { assertValid, cleaned, normalize } from "./schema.ts"; import type { Ctx, Exposed, GetOptions, Readable, RefStatus, Shape, StateEvent } from "./types.ts"; const noop = (): void => {}; @@ -161,7 +160,7 @@ export interface Self

    { * it when its version rises, and refuses a schema change without a version bump. A cell a reload adds runs its `init` * while the reload prepares. */ - state( + state( name: string, schema: S, initial: Static | StateInit, P>, @@ -317,7 +316,7 @@ type TaskFn = (ctx: Ctx) => Promise; /** Everything one factory run declared, by kind and key. */ export class Decls { readonly cells = new Map>(); - readonly cellSchemas = new Map(); + readonly cellSchemas = new Map(); readonly cellVersions = new Map(); readonly cellMigrations = new Map unknown>(); readonly refs = new Map(); @@ -1297,7 +1296,7 @@ export class ObjectImpl { /** Every state source exposed as a remote state member gets that member's schema (checked per write in dev mode). */ refreshSchemas(): void { - const schemas = new Map, TSchema[]>(); + const schemas = new Map, Schema[]>(); for (const installed of this.implementations.values()) { for (const [source, list] of memberSchemas(installed.service, installed.implementation)) { schemas.set(source, [...(schemas.get(source) ?? []), ...list]); @@ -1592,8 +1591,8 @@ export class ObjectImpl { // The additive rule assumes a consumer sends only what its own schema knows, and a provider sees only what its // schema knows: strip with both before validating. const own = ref.contract.scope === "remote" && wanted?.kind === "method" ? wanted.args : undefined; - const sent = own === undefined ? args : Value.Clean(own, Value.Clone(args)); - input = validated?.args ? Value.Clean(validated.args, Value.Clone(sent)) : sent; + const sent = own === undefined ? args : cleaned(own, args); + input = validated?.args ? cleaned(validated.args, sent) : sent; if (validated?.args) assertValid("args", `${this.address}.${name} args`, validated.args, input); } this.lease(ctx); @@ -1731,8 +1730,8 @@ const isReadableLike = (value: unknown): boolean => export function memberSchemas( service: Service, implementation: Record, -): Map, TSchema[]> { - const schemas = new Map, TSchema[]>(); +): Map, Schema[]> { + const schemas = new Map, Schema[]>(); const contract = service.contract; if (contract.scope !== "remote") return schemas; const members = contract.description.variants[variantOf(implementation) ?? ""] ?? {}; @@ -1794,7 +1793,7 @@ class SelfImpl implements Self { if (map.has(name)) throw invalid("duplicate", `${this.#object.typeName}: duplicate ${kind} name ${name}`); } - state( + state( name: string, schema: S, initial: Static | StateInit>, diff --git a/packages/chord/src/remote.ts b/packages/chord/src/remote.ts index 73c1e1d24..47e8ffa77 100644 --- a/packages/chord/src/remote.ts +++ b/packages/chord/src/remote.ts @@ -1,4 +1,3 @@ -import { Value } from "typebox/value"; import { HYDRATION, type Hydration } from "./cell.ts"; import { type AnyContract, takesNoArgs } from "./contract.ts"; import { applyImmutable, type Decoder, decoder } from "./delta/index.ts"; @@ -13,6 +12,7 @@ import { refsByProxy, type StatusListener, } from "./object.ts"; +import { cleaned } from "./schema.ts"; import type { Ctx, Readable, RefStatus, Shape, StateEvent } from "./types.ts"; import { fromWireError, type RootRef, type WireError } from "./wire.ts"; @@ -344,7 +344,7 @@ export class RemoteBinding { throw new ChordError("not_in_variant", `${this.key}: ${name} is not a method of variant ${variant}`); } // Send only what the consumer's own schema knows (the additive rule's premise; see ObjectImpl.call). - const sent = description.args === undefined ? undefined : Value.Clean(description.args, Value.Clone(args)); + const sent = description.args === undefined ? undefined : cleaned(description.args, args); return hold.channel.call( hold.id, name, diff --git a/packages/chord/src/schema.ts b/packages/chord/src/schema.ts index 7883969e5..527accd9a 100644 --- a/packages/chord/src/schema.ts +++ b/packages/chord/src/schema.ts @@ -1,8 +1,7 @@ -import type { TSchema } from "typebox"; -import { Value } from "typebox/value"; +import { compile, type JsonSchema, type ValidationError, type Validator } from "@earendil-works/pi-schema"; import { type ChordErrorReason, invalid } from "./errors.ts"; -/** JSON Schema as data. TypeBox schemas are plain JSON Schema objects. */ +/** JSON Schema as data: pi-schema builders, or any JSON Schema object (TypeBox's included). */ type Schema = { readonly [key: string]: unknown }; const asSchema = (schema: unknown): Schema => @@ -34,23 +33,187 @@ export function deepEqual(a: unknown, b: unknown): boolean { return a === b || canonicalJson(a) === canonicalJson(b); } +const validators = new WeakMap(); + +/** The compiled validator of `schema`, compiled once per schema object; `format` is an annotation. */ +function validatorOf(schema: JsonSchema): Validator { + if (typeof schema === "boolean") return compile(schema); + let validator = validators.get(schema); + if (validator === undefined) { + validator = compile(schema); + validators.set(schema, validator); + } + return validator; +} + +/** Whether `value` matches `schema`. */ +export function isValid(schema: JsonSchema, value: unknown): boolean { + return validatorOf(schema).check(value); +} + /** Throws `invalid` (with `reason`) and the first schema errors when `value` does not match. */ -export function assertValid(reason: ChordErrorReason, what: string, schema: TSchema, value: unknown): void { - if (Value.Check(schema, value)) return; - const errors = [...Value.Errors(schema, value)].slice(0, 3); - const summary = errors.map((error) => `${error.instancePath || "/"} ${error.message}`).join("; "); +export function assertValid(reason: ChordErrorReason, what: string, schema: JsonSchema, value: unknown): void { + const validator = validatorOf(schema); + if (validator.check(value)) return; + const errors: ValidationError[] = validator.errors(value).slice(0, 3); + const summary = errors.map((error) => `${error.path || "/"} ${error.message}`).join("; "); throw invalid(reason, `${what}: ${summary}`, errors); } /** Applies defaults, strips unknown fields, validates. Returns a fresh value. */ -export function normalize(schema: TSchema, value: unknown, what: string): unknown { - const normalized = Value.Clean(schema, Value.Default(schema, Value.Clone(value ?? {}))); +export function normalize(schema: JsonSchema, value: unknown, what: string): unknown { + const normalized = cleanFrom(asSchema(schema), withDefaults(asSchema(schema), cloneData(value ?? {}))); assertValid("input", what, schema, normalized); return normalized; } +/** A copy of `value` with only what `schema` knows: unknown object fields dropped, recursively (TypeBox's `Clean`). */ +export function cleaned(schema: JsonSchema, value: unknown): unknown { + return cleanFrom(asSchema(schema), cloneData(value)); +} + +/** A deep copy of plain data: arrays, plain objects and bytes are copied, any other value is kept. */ +function cloneData(value: unknown): unknown { + if (Array.isArray(value)) return value.map(cloneData); + if (value instanceof Uint8Array) return value.slice(); + if (value === null || typeof value !== "object") return value; + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) return value; + const copy: Record = {}; + for (const [key, item] of Object.entries(value)) copy[key] = cloneData(item); + return copy; +} + +const isPlainObject = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value) && !(value instanceof Uint8Array); + +const isSchemaObject = (value: unknown): value is Schema => + value !== null && typeof value === "object" && !Array.isArray(value); + +/** The object an `allOf` of object schemas evaluates to (TypeBox's `Evaluate`), or undefined for any other `allOf`. */ +function mergedAllOf(schema: Schema): Schema | undefined { + if (!Array.isArray(schema.allOf) || !schema.allOf.every((branch) => isSchemaObject(branch.properties))) + return undefined; + const properties: Record = {}; + const required = new Set(); + for (const branch of schema.allOf as Schema[]) { + Object.assign(properties, branch.properties); + for (const key of (branch.required as string[] | undefined) ?? []) required.add(key); + } + return { + type: "object", + properties, + required: [...required], + ...(schema.unevaluatedProperties === undefined ? {} : { additionalProperties: schema.unevaluatedProperties }), + }; +} + +/** The pattern and value schema of a record (`patternProperties` with one pattern). */ +function recordOf(schema: Schema): [RegExp, unknown] | undefined { + if (!isSchemaObject(schema.patternProperties) || isSchemaObject(schema.properties)) return undefined; + const [entry] = Object.entries(schema.patternProperties); + return entry === undefined ? undefined : [new RegExp(entry[0], "u"), entry[1]]; +} + +/** Whether `additional` (an object's `additionalProperties`) keeps an unknown field holding `value`. */ +const keepsUnknown = (additional: unknown, value: unknown): boolean => + additional === true || (isSchemaObject(additional) && isValid(additional, value)); + +// Mutates `value` (a fresh copy), as TypeBox's `Clean` does. +function cleanFrom(schema: Schema, value: unknown): unknown { + const merged = mergedAllOf(schema); + if (merged !== undefined) return cleanFrom(merged, value); + if (Array.isArray(schema.anyOf)) { + for (const branch of schema.anyOf) { + const candidate = cleanFrom(asSchema(branch), cloneData(value)); + if (isValid(asSchema(branch), candidate)) return candidate; + } + return value; + } + if (Array.isArray(value)) { + const prefix = Array.isArray(schema.items) + ? schema.items + : Array.isArray(schema.prefixItems) + ? schema.prefixItems + : undefined; + if (prefix !== undefined) { + const length = Math.min(value.length, prefix.length); + for (let index = 0; index < length; index++) value[index] = cleanFrom(asSchema(prefix[index]), value[index]); + return value.length > length ? value.slice(0, length) : value; + } + if (isSchemaObject(schema.items)) return value.map((item) => cleanFrom(schema.items as Schema, item)); + return value; + } + if (!isPlainObject(value)) return value; + const record = recordOf(schema); + const properties = isSchemaObject(schema.properties) ? schema.properties : undefined; + // An object schema names its fields, a record its key pattern, a map (pi-schema's) its value schema. + if (properties === undefined && record === undefined && !isSchemaObject(schema.additionalProperties)) return value; + for (const key of Object.keys(value)) { + const known = properties !== undefined && Object.hasOwn(properties, key) ? properties[key] : undefined; + const matched = known ?? (record?.[0].test(key) ? record[1] : undefined); + if (matched !== undefined) value[key] = cleanFrom(asSchema(matched), value[key]); + else if (keepsUnknown(schema.additionalProperties, value[key])) + value[key] = cleanFrom(asSchema(schema.additionalProperties), value[key]); + else delete value[key]; + } + return value; +} + +// Mutates `value` (a fresh copy), as TypeBox's `Default` does: a missing value takes its schema's default, a +// missing optional field never does. +function withDefaults(schema: Schema, value: unknown): unknown { + const defaulted = value === undefined && "default" in schema ? cloneData(schema.default) : value; + const merged = mergedAllOf(schema); + if (merged !== undefined) return withDefaults(merged, defaulted); + if (Array.isArray(schema.anyOf)) { + for (const branch of schema.anyOf) { + const candidate = withDefaults(asSchema(branch), cloneData(defaulted)); + if (isValid(asSchema(branch), candidate)) return candidate; + } + return defaulted; + } + if (Array.isArray(defaulted)) { + const prefix = Array.isArray(schema.items) + ? schema.items + : Array.isArray(schema.prefixItems) + ? schema.prefixItems + : undefined; + if (prefix !== undefined) { + for (let index = 0; index < prefix.length; index++) { + const item = withDefaults(asSchema(prefix[index]), defaulted[index]); + if (index < defaulted.length || item !== undefined) defaulted[index] = item; + } + return defaulted; + } + if (isSchemaObject(schema.items)) return defaulted.map((item) => withDefaults(schema.items as Schema, item)); + return defaulted; + } + if (!isPlainObject(defaulted)) return defaulted; + const properties = isSchemaObject(schema.properties) ? schema.properties : undefined; + const record = recordOf(schema); + if (properties !== undefined) { + const required = (schema.required as string[] | undefined) ?? []; + for (const [key, property] of Object.entries(properties)) { + const item = withDefaults(asSchema(property), defaulted[key]); + if (item === undefined && (!required.includes(key) || !("default" in asSchema(property)))) continue; + defaulted[key] = item; + } + } else if (record !== undefined && isSchemaObject(record[1]) && "default" in record[1]) { + for (const key of Object.keys(defaulted)) { + if (record[0].test(key)) defaulted[key] = withDefaults(record[1], defaulted[key]); + } + } + if (!isSchemaObject(schema.additionalProperties)) return defaulted; + for (const key of Object.keys(defaulted)) { + const known = properties !== undefined && Object.hasOwn(properties, key); + if (!known && !record?.[0].test(key)) defaulted[key] = withDefaults(schema.additionalProperties, defaulted[key]); + } + return defaulted; +} + /** Canonical identity of a normalized value: fields equal to their schema default are omitted. */ -export function canonicalIdentity(schema: TSchema, value: unknown): string { +export function canonicalIdentity(schema: JsonSchema, value: unknown): string { return canonicalJson(stripDefaults(asSchema(schema), value)); } diff --git a/packages/chord/src/wire.ts b/packages/chord/src/wire.ts index b9271f130..bd4bdf175 100644 --- a/packages/chord/src/wire.ts +++ b/packages/chord/src/wire.ts @@ -1,5 +1,21 @@ -import { type Static, type TSchema, Type } from "typebox"; -import { Value } from "typebox/value"; +import { + arr, + bool, + compile, + int, + lit, + map, + num, + obj, + oneOf, + opt, + type Properties, + type Schema, + type Static, + str, + union, + unknown, +} from "@earendil-works/pi-schema"; import { DescriptionSchema } from "./contract.ts"; import { CHORD_ERROR_CODES, ChordError } from "./errors.ts"; @@ -23,55 +39,57 @@ import { CHORD_ERROR_CODES, ChordError } from "./errors.ts"; */ export const PROTOCOL_VERSION = 4; -const Id = Type.Integer({ minimum: 1 }); -const Name = Type.String({ minLength: 1 }); -const Reload = Type.Integer({ minimum: 1 }); -/** Arrays a sender may pass as readonly. */ -const List = (item: T) => Type.Immutable(Type.Array(item)); +const Id = int({ minimum: 1 }); +const Name = str({ minLength: 1 }); +const Reload = int({ minimum: 1 }); +/** Arrays a sender may pass as readonly: the same schema, a readonly type. */ +const List = (item: Schema) => arr(item) as Schema; +/** An object that admits fields it does not name (a newer peer's), as every frame does. */ +const Open = (properties: P) => obj(properties, { open: true }); -const RootRefSchema = Type.Object({ rootId: Name, codeVersion: Type.Integer({ minimum: 1 }) }); +const RootRefSchema = Open({ rootId: Name, codeVersion: int({ minimum: 1 }) }); export type RootRef = Static; -const WireErrorSchema = Type.Object({ - code: Type.Enum(CHORD_ERROR_CODES), - message: Type.String(), - reason: Type.Optional(Type.String()), +const WireErrorSchema = Open({ + code: oneOf(CHORD_ERROR_CODES), + message: str(), + reason: opt(str()), }); export type WireError = Static; -const WireReloadOptionsSchema = Type.Object({ - remove: Type.Optional(List(Type.String())), - gateDeadline: Type.Optional(Type.Number()), - retireDeadline: Type.Optional(Type.Number()), +const WireReloadOptionsSchema = Open({ + remove: opt(List(str())), + gateDeadline: opt(num()), + retireDeadline: opt(num()), }); export type WireReloadOptions = Static; /** A child's prepared reload, sent with its vote (same fields as `ReloadReport`, without children). */ -const WireReloadReportSchema = Type.Object({ - codeVersion: Type.Integer(), - previous: Type.Integer(), - changed: List(Type.String()), - removed: List(Type.String()), - prepared: List(Type.String()), - gated: List(Type.String()), +const WireReloadReportSchema = Open({ + codeVersion: int(), + previous: int(), + changed: List(str()), + removed: List(str()), + prepared: List(str()), + gated: List(str()), }); export type WireReloadReport = Static; -const FrameSchema = Type.Union([ - Type.Object({ - t: Type.Literal("hello"), +const FrameSchema = union([ + Open({ + t: lit("hello"), /** The link protocol version (`PROTOCOL_VERSION`; absent: 1). Peers of another version close the link. */ - protocol: Type.Optional(Type.Integer({ minimum: 1 })), + protocol: opt(int({ minimum: 1 })), name: Name, - kind: Type.Union([Type.Literal("host"), Type.Literal("client")]), - codeVersion: Type.Integer({ minimum: 0 }), + kind: union([lit("host"), lit("client")]), + codeVersion: int({ minimum: 0 }), /** The sender acknowledges what it receives (`ack`). Flow control applies only when both hellos say so. */ - acks: Type.Optional(Type.Boolean()), + acks: opt(bool()), }), /** Every frame received so far except `ack` frames, counted from the link's start, once processed. */ - Type.Object({ t: Type.Literal("ack"), n: Type.Integer({ minimum: 1 }) }), - Type.Object({ - t: Type.Literal("hold"), + Open({ t: lit("ack"), n: int({ minimum: 1 }) }), + Open({ + t: lit("hold"), id: Id, /** * Target host name, resolved by the consumer from the token's `host` and `input`. Relays route on it after @@ -81,92 +99,92 @@ const FrameSchema = Type.Union([ /** The consumer's contract token; the provider checks that it covers it. Remote tokens carry `host`. */ token: DescriptionSchema, /** Normalized with the consumer's contract. */ - input: Type.Unknown(), + input: unknown(), /** A forwarded principal. Honored only on frames from the parent link. */ - principal: Type.Optional(Type.Unknown()), + principal: opt(unknown()), /** * The client connection the hold comes from: an opaque id the server assigned to it, forwarded by relays. Honored * only on frames from the parent link; present means the hold is a client's (not a reload participant). */ - origin: Type.Optional(Name), + origin: opt(Name), /** False: no state for this hold until `sub` (a method-only binding). */ - state: Type.Optional(Type.Literal(false)), + state: opt(lit(false)), /** The consumer already has values (a re-hold after a link loss): its unit is not urgent. */ - again: Type.Optional(Type.Literal(true)), + again: opt(lit(true)), }), - Type.Object({ t: Type.Literal("release"), id: Id }), - Type.Object({ - t: Type.Literal("shape"), + Open({ t: lit("release"), id: Id }), + Open({ + t: lit("shape"), id: Id, - variant: Type.Optional(Type.String()), - members: List(Type.String()), - contract: Type.String(), + variant: opt(str()), + members: List(str()), + contract: str(), /** The state members whose snapshots follow this frame; the consumer applies the shape once they are here. */ state: List(Name), }), - Type.Object({ t: Type.Literal("gone"), id: Id, error: WireErrorSchema }), - Type.Object({ - t: Type.Literal("call"), + Open({ t: lit("gone"), id: Id, error: WireErrorSchema }), + Open({ + t: lit("call"), id: Id, hold: Id, member: Name, - args: Type.Optional(Type.Unknown()), + args: opt(unknown()), /** Absent: the receiving host starts a root (browsers never carry roots). */ - root: Type.Optional(RootRefSchema), + root: opt(RootRefSchema), }), - Type.Object({ - t: Type.Literal("result"), + Open({ + t: lit("result"), id: Id, - value: Type.Optional(Type.Unknown()), - error: Type.Optional(WireErrorSchema), + value: opt(unknown()), + error: opt(WireErrorSchema), }), - Type.Object({ t: Type.Literal("cancel"), id: Id }), + Open({ t: lit("cancel"), id: Id }), /** Starts the state of a hold opened with `state: false` (answered with a unit). */ - Type.Object({ t: Type.Literal("sub"), hold: Id }), + Open({ t: lit("sub"), hold: Id }), /** Stops the state of a hold. */ - Type.Object({ t: Type.Literal("unsub"), hold: Id }), + Open({ t: lit("unsub"), hold: Id }), /** Sends member `m` again as a reset (a relay that had to drop what waited for it). */ - Type.Object({ t: Type.Literal("resync"), hold: Id, m: Name }), - Type.Object({ - t: Type.Literal("state"), + Open({ t: lit("resync"), hold: Id, m: Name }), + Open({ + t: lit("state"), hold: Id, m: Name, /** `snap`/`reset` without `value`: the value is `undefined`. */ - as: Type.Union([Type.Literal("snap"), Type.Literal("reset"), Type.Literal("ops"), Type.Literal("error")]), - value: Type.Optional(Type.Unknown()), + as: union([lit("snap"), lit("reset"), lit("ops"), lit("error")]), + value: opt(unknown()), /** Encoded with the stream's delta encoder; the decoder validates each op. */ - ops: Type.Optional(List(List(Type.Unknown()))), - error: Type.Optional(WireErrorSchema), + ops: opt(List(List(unknown()))), + error: opt(WireErrorSchema), }), - Type.Object({ t: Type.Literal("ping"), n: Type.Integer() }), - Type.Object({ t: Type.Literal("pong"), n: Type.Integer() }), + Open({ t: lit("ping"), n: int() }), + Open({ t: lit("pong"), n: int() }), /** Ends the link (consumed by the link itself). `reason: "protocol-version "`: the sender speaks version n. */ - Type.Object({ t: Type.Literal("bye"), reason: Type.Optional(Type.String()) }), + Open({ t: lit("bye"), reason: opt(str()) }), /** The root ended at its origin: its leases on objects end (sent on every link the root's calls used). */ - Type.Object({ t: Type.Literal("root_end"), rootId: Name }), - Type.Object({ t: Type.Literal("invalidate"), pattern: Type.Record(Type.String(), Type.Unknown()) }), - Type.Object({ - t: Type.Literal("prepare"), + Open({ t: lit("root_end"), rootId: Name }), + Open({ t: lit("invalidate"), pattern: map(unknown()) }), + Open({ + t: lit("prepare"), reload: Reload, - codeVersion: Type.Integer({ minimum: 1 }), - plugins: List(Type.String()), + codeVersion: int({ minimum: 1 }), + plugins: List(str()), options: WireReloadOptionsSchema, }), - Type.Object({ - t: Type.Literal("vote"), + Open({ + t: lit("vote"), reload: Reload, - error: Type.Optional(WireErrorSchema), - report: Type.Optional(WireReloadReportSchema), + error: opt(WireErrorSchema), + report: opt(WireReloadReportSchema), }), - Type.Object({ t: Type.Literal("commit"), reload: Reload }), + Open({ t: lit("commit"), reload: Reload }), /** The parent dropped `codeVersion`: roots of it may still come from there no longer. */ - Type.Object({ t: Type.Literal("retire"), codeVersion: Type.Integer({ minimum: 1 }) }), - Type.Object({ t: Type.Literal("abort"), reload: Reload }), + Open({ t: lit("retire"), codeVersion: int({ minimum: 1 }) }), + Open({ t: lit("abort"), reload: Reload }), ]); export type Frame = Static; /** A frame that passes validation; anything else is a protocol violation. */ -export const isFrame = (value: unknown): value is Frame => Value.Check(FrameSchema, value); +export const isFrame: (value: unknown) => value is Frame = compile(FrameSchema).check; export function toWireError(error: unknown): WireError { if (error instanceof ChordError) { diff --git a/packages/chord/test/awaiting.test.ts b/packages/chord/test/awaiting.test.ts index 655d66736..975cb31f8 100644 --- a/packages/chord/test/awaiting.test.ts +++ b/packages/chord/test/awaiting.test.ts @@ -1,7 +1,7 @@ // Entry 8: awaited gets. `host.get` and `ctx.get` resolve once the reference is ready; abort, the ctx closing and the // host's `readyTimeout` reject them and release only that reference. import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { num, obj } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { type Ctx, createHost, type Host, hostNamed, isChordError, memoryLinkPair, remote } from "../src/index.ts"; import { CountingLink } from "./fixtures/counting.ts"; @@ -14,9 +14,9 @@ async function until(check: () => boolean, attempts = 200): Promise { const Slow = remote("t.e8-slow", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { state: { n: Type.Number() } } }, + variants: { main: { state: { n: num() } } }, }); /** A server whose `Slow` object goes live when `open()` is called; a browser whose link counts frames. */ @@ -28,7 +28,7 @@ function setup(readyTimeout?: number) { const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); server.serve(Slow, (_input, _principal, self) => { self.resource("gate", { version: 1, open: () => opened }); - return { variant: "main", n: self.state("n", Type.Number(), 1) }; + return { variant: "main", n: self.state("n", num(), 1) }; }); const [down, up] = memoryLinkPair(); server.accept(down, { principal: { user: "u" } }); diff --git a/packages/chord/test/bindings.test.ts b/packages/chord/test/bindings.test.ts index 6e7d9b9b6..e3e642ebe 100644 --- a/packages/chord/test/bindings.test.ts +++ b/packages/chord/test/bindings.test.ts @@ -1,7 +1,7 @@ // Entry 8: one shared binding per host and key; a hold implies the state of its variant (units: shape, then // snapshots); method-only references; origins of client holds; protocol version 4. import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { arr, nul, num, obj, opt, str, unknown } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { HostImpl } from "../src/host.ts"; import { @@ -28,15 +28,15 @@ async function until(check: () => boolean, attempts = 200): Promise { assert.ok(check(), "condition not reached"); } -const NoteSchema = Type.Array(Type.String()); +const NoteSchema = arr(str()); const Notes = remote("t.e8-notes", { version: 1, - input: Type.Object({ list: Type.Optional(Type.String({ default: "main" })) }), + input: obj({ list: opt(str({ default: "main" })) }, { open: true }), host: hostNamed("server"), variants: { main: { - state: { notes: NoteSchema, title: Type.String() }, - methods: { add: method(Type.String()), rename: method(Type.String()) }, + state: { notes: NoteSchema, title: str() }, + methods: { add: method(str()), rename: method(str()) }, }, }, }); @@ -44,7 +44,7 @@ const Notes = remote("t.e8-notes", { function serveNotes(host: Host): void { host.serve(Notes, (_input, _principal, self) => { const notes = self.state("notes", NoteSchema, []); - const title = self.state("title", Type.String(), "untitled"); + const title = self.state("title", str(), "untitled"); return { variant: "main", notes, @@ -88,7 +88,7 @@ function setup( describe("shared bindings: traffic", () => { it("N references to one object: one hold, one snapshot per member, one frame per change, one release", async () => { const { server, browser, link } = setup(); - const Panel = local("t.e8-panel", { version: 1, input: Type.Object({}), members: { count: Type.Number() } }); + const Panel = local("t.e8-panel", { version: 1, input: obj({}, { open: true }), members: { count: num() } }); browser.serve(Panel, (_input, _principal, self) => { const notes = self.get(Notes, {}); return { count: self.derive([notes.notes], (list) => list.length) }; @@ -183,12 +183,12 @@ describe("shared bindings: identity", () => { }); const NotesV2 = remote("t.e8-notes", { version: 2, - input: Type.Object({ list: Type.Optional(Type.String({ default: "main" })) }), + input: obj({ list: opt(str({ default: "main" })) }, { open: true }), host: hostNamed("server"), variants: { main: { - state: { notes: NoteSchema, title: Type.String() }, - methods: { add: method(Type.String()), rename: method(Type.String()), clear: method() }, + state: { notes: NoteSchema, title: str() }, + methods: { add: method(str()), rename: method(str()), clear: method() }, }, }, }); @@ -196,7 +196,7 @@ describe("shared bindings: identity", () => { NotesV2, (_input, _principal, self) => { const notes = self.state("notes", NoteSchema, []); - const title = self.state("title", Type.String(), "untitled"); + const title = self.state("title", str(), "untitled"); return { variant: "main", notes, title, add: () => {}, rename: () => {}, clear: () => {} }; }, { accepts: { 1: (current) => current } }, @@ -230,12 +230,12 @@ describe("shared bindings: identity", () => { describe("hydration over a link", () => { const Matrix = remote("t.e8-matrix", { version: 1, - input: Type.Object({ variant: Type.String() }), + input: obj({ variant: str() }, { open: true }), host: hostNamed("server"), variants: { - values: { state: { none: Type.Null(), empty: Type.Array(Type.String()), maybe: Type.Unknown() } }, - bare: { methods: { ping: method(undefined, Type.String()) } }, - broken: { state: { n: Type.Number() } }, + values: { state: { none: nul(), empty: arr(str()), maybe: unknown() } }, + bare: { methods: { ping: method(undefined, str()) } }, + broken: { state: { n: num() } }, }, }); @@ -243,12 +243,12 @@ describe("hydration over a link", () => { host.serve(Matrix, ({ variant }, _principal, self) => { if (variant === "bare") return { variant: "bare", ping: () => "pong" }; if (variant === "broken") { - const source = self.state("source", Type.String(), "not a number"); + const source = self.state("source", str(), "not a number"); // A derivation exposed under a number schema: its snapshot fails validation. return { variant: "broken", n: self.derive([source], (text) => text as unknown as number) }; } - const none = self.state("none", Type.Null(), null); - const empty = self.state("empty", Type.Array(Type.String()), []); + const none = self.state("none", nul(), null); + const empty = self.state("empty", arr(str()), []); return { variant: "values", none, empty, maybe: self.derive([none], () => undefined as unknown) }; }); } @@ -332,9 +332,9 @@ describe("method-only references (state: false)", () => { describe("origins of client holds", () => { const Counter = remote("t.e8-counter", { version: 1, - input: Type.Object({ session: Type.String() }), + input: obj({ session: str() }, { open: true }), host: hostFamily("session:", "session"), - variants: { main: { state: { n: Type.Number() }, methods: { add: method(Type.Number()) } } }, + variants: { main: { state: { n: num() }, methods: { add: method(num()) } } }, }); function cluster() { @@ -354,7 +354,7 @@ describe("origins of client holds", () => { onError: () => {}, }); created.serve(Counter, (_input, _principal, self) => { - const n = self.state("n", Type.Number(), 0); + const n = self.state("n", num(), 0); return { variant: "main", n, diff --git a/packages/chord/test/boundary.test.ts b/packages/chord/test/boundary.test.ts index 8819a01c1..ae4f1beb4 100644 --- a/packages/chord/test/boundary.test.ts +++ b/packages/chord/test/boundary.test.ts @@ -8,10 +8,11 @@ const sourceDirectory = resolve(packageDirectory, "src"); const IMPORT_SPECIFIER = /(?:import|export)\s+(?:type\s+)?(?:[^;]*?\sfrom\s*)?["']([^"']+)["']/gu; describe("package boundary", () => { - // pi-telemetry is the one allowed Pi package: a leaf without dependencies, needed for Context.telemetry. - const ALLOWED = new Set(["@earendil-works/pi-telemetry"]); + // The allowed Pi packages are leaves without dependencies: pi-telemetry for Context.telemetry, pi-schema for schemas + // (decision "A schema package beneath pi-ai and Chord"). + const ALLOWED = new Set(["@earendil-works/pi-telemetry", "@earendil-works/pi-schema"]); - test("does not depend on Pi packages other than pi-telemetry, or on files outside Chord", async () => { + test("does not depend on Pi packages other than pi-telemetry and pi-schema, or on files outside Chord", async () => { const manifest = JSON.parse(await readFile(resolve(packageDirectory, "package.json"), "utf8")) as { dependencies?: Record; }; diff --git a/packages/chord/test/cluster.test.ts b/packages/chord/test/cluster.test.ts index 1547c3e1f..b0ad5dc97 100644 --- a/packages/chord/test/cluster.test.ts +++ b/packages/chord/test/cluster.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { num, obj, str } from "@earendil-works/pi-schema"; import { describe, it, onTestFinished } from "vitest"; import { type Ctx, @@ -30,32 +30,32 @@ function latch() { const Greeter = remote("t.greeter", { version: 1, - input: Type.Object({ session: Type.String() }), + input: obj({ session: str() }, { open: true }), host: hostFamily("session:", "session"), variants: { main: { - state: { count: Type.Object({ n: Type.Number() }) }, - methods: { greet: method(Type.String(), Type.String()) }, + state: { count: obj({ n: num() }, { open: true }) }, + methods: { greet: method(str(), str()) }, }, }, }); const GreeterV2 = remote("t.greeter", { version: 2, - input: Type.Object({ session: Type.String() }), + input: obj({ session: str() }, { open: true }), host: hostFamily("session:", "session"), - variants: { main: { methods: { hello: method(Type.String(), Type.String()) } } }, + variants: { main: { methods: { hello: method(str(), str()) } } }, }); /** The server's own greeter: the greeter facet serves this one on the server, `Greeter` on session hosts. */ const ServerGreeter = remote("t.server-greeter", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { main: { - state: { count: Type.Object({ n: Type.Number() }) }, - methods: { greet: method(Type.String(), Type.String()) }, + state: { count: obj({ n: num() }, { open: true }) }, + methods: { greet: method(str(), str()) }, }, }, }); @@ -68,7 +68,7 @@ function greeterFacet(greeting: string, options: { fail?: boolean; stateVersion? if (options.fail) throw new Error("worker factory fails"); const count = self.state( "count", - Type.Object({ n: Type.Number() }), + obj({ n: num() }, { open: true }), { n: 0 }, { version: options.stateVersion ?? 1 }, ); @@ -105,9 +105,9 @@ function dropping(link: Link, drop: (frame: Frame) => boolean): Link { const Ping = remote("t.cluster-ping", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { methods: { ping: method(undefined, Type.String()) } } }, + variants: { main: { methods: { ping: method(undefined, str()) } } }, }); function setup() { @@ -314,7 +314,7 @@ describe("cross-host reload", () => { }, close: () => void log.push("closed"), }); - const count = self.state("count", Type.Object({ n: Type.Number() }), { n: 0 }); + const count = self.state("count", obj({ n: num() }, { open: true }), { n: 0 }); return { variant: "main", count, greet: (name: string) => `slow ${name}` }; }), }; @@ -569,7 +569,7 @@ describe("cross-host reload", () => { const never = latch(); const Slow = remote("t.cluster-slow", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("session:S1"), variants: { main: { methods: { wait: method() } } }, }); @@ -590,7 +590,7 @@ describe("cross-host reload", () => { resolve: async () => watched, principal: (name) => ({ host: name }), }); - const Store = local("t.cluster-store", { version: 1, input: Type.Object({}) }); + const Store = local("t.cluster-store", { version: 1, input: obj({}, { open: true }) }); const storeFacet = (version: number): Facet => ({ name: "store", setup: (api) => diff --git a/packages/chord/test/contract.test.ts b/packages/chord/test/contract.test.ts index df29a8a0b..0abcc1a67 100644 --- a/packages/chord/test/contract.test.ts +++ b/packages/chord/test/contract.test.ts @@ -1,25 +1,30 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { bool, num, obj, opt, str, unknown } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { bind, coverage, hostFamily, hostNamed, isChordError, local, method, opaque, remote } from "../src/index.ts"; import { canonicalIdentity, fits, normalize } from "../src/schema.ts"; -const View = Type.Object({ text: Type.String() }); +const View = obj({ text: str() }, { open: true }); const chatV1 = remote("t.chat", { version: 1, - input: Type.Object({ session: Type.String(), conversation: Type.Number({ default: 0 }) }), + input: obj({ session: str(), conversation: num({ default: 0 }) }, { open: true }), host: hostFamily("session:", "session"), variants: { viewer: { state: { view: View } }, participant: { state: { view: View }, - methods: { send: method(Type.Object({ text: Type.String() }), Type.Object({ id: Type.Number() })) }, + methods: { send: method(obj({ text: str() }, { open: true }), obj({ id: num() }, { open: true })) }, }, }, }); describe("contracts", () => { + it("treats `format` as an annotation: an input with format email accepts any string", () => { + const Mail = local("t.mail", { version: 1, input: obj({ to: str({ format: "email" }) }) }); + assert.deepEqual(normalize(Mail.input, { to: "nope" }, "input"), { to: "nope" }); + }); + it("normalizes input and omits defaults from the identity", () => { const a = normalize(chatV1.input, { session: "s", extra: 1 }, "input"); assert.deepEqual(a, { session: "s", conversation: 0 }); @@ -39,12 +44,12 @@ describe("contracts", () => { }); it("rejects reserved member names and opaque remote members", () => { - assert.throws(() => local("t.x", { version: 1, input: Type.Object({}), members: { status: method() } })); + assert.throws(() => local("t.x", { version: 1, input: obj({}, { open: true }), members: { status: method() } })); assert.throws(() => remote("t.x", { host: hostNamed("server"), version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), variants: { a: { state: { h: opaque("x@1") as never } } }, }), ); @@ -56,11 +61,11 @@ describe("contracts", () => { variants: { ...chatV1.def.variants, participant: { - state: { view: Type.Object({ text: Type.String(), extra: Type.Number() }) }, + state: { view: obj({ text: str(), extra: num() }, { open: true }) }, methods: { send: method( - Type.Object({ text: Type.String(), urgent: Type.Optional(Type.Boolean()) }), - Type.Object({ id: Type.Number(), at: Type.Number() }), + obj({ text: str(), urgent: opt(bool()) }, { open: true }), + obj({ id: num(), at: num() }, { open: true }), ), }, }, @@ -77,8 +82,8 @@ describe("contracts", () => { state: { view: View }, methods: { send: method( - Type.Object({ text: Type.String(), urgent: Type.Boolean() }), - Type.Object({ id: Type.Number() }), + obj({ text: str(), urgent: bool() }, { open: true }), + obj({ id: num() }, { open: true }), ), }, }, @@ -92,18 +97,18 @@ describe("contracts", () => { const retyped = remote("t.chat", { ...chatV1.def, - variants: { ...chatV1.def.variants, viewer: { state: { view: Type.Object({ text: Type.Number() }) } } }, + variants: { ...chatV1.def.variants, viewer: { state: { view: obj({ text: num() }, { open: true }) } } }, }); assert.match(coverage(retyped.description, chatV1.description) ?? "", /state/); }); it("methods without arguments or result: described without them; their presence must match, a void consumer accepts a result", () => { const of = (members: Parameters[1]["members"]) => - local("t.methods", { version: 1, input: Type.Object({}), members }).description; + local("t.methods", { version: 1, input: obj({}, { open: true }), members }).description; const none = of({ m: method() }); assert.deepEqual(none.variants[""]?.m, { kind: "method" }); - const withArgs = of({ m: method(Type.String()) }); - const withResult = of({ m: method(undefined, Type.Number()) }); + const withArgs = of({ m: method(str()) }); + const withResult = of({ m: method(undefined, num()) }); assert.equal(coverage(none, none), undefined); assert.match(coverage(withArgs, none) ?? "", /arguments/, "the consumer sends none, the provider expects some"); assert.match(coverage(none, withArgs) ?? "", /arguments/, "the consumer sends some, the provider takes none"); @@ -124,15 +129,25 @@ describe("contracts", () => { it("compares opaque members by name@version and function members structurally", () => { const a = local("t.idx", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { harness: opaque("h@1"), - onChange: method(Type.Function([Type.String()], Type.Void())), + onChange: method({ type: "function", parameters: [str()], returnType: { type: "void" } }), }, }); const b = local("t.idx", { ...a.def, members: { ...a.def.members, harness: opaque("h@2") } }); assert.match(coverage(b.description, a.description) ?? "", /opaque/); - assert.ok(fits(Type.Function([Type.Unknown()], Type.Void()), Type.Function([Type.String()], Type.Void()))); - assert.ok(!fits(Type.Function([Type.String()], Type.Void()), Type.Function([Type.Unknown()], Type.Void()))); + assert.ok( + fits( + { type: "function", parameters: [unknown()], returnType: { type: "void" } }, + { type: "function", parameters: [str()], returnType: { type: "void" } }, + ), + ); + assert.ok( + !fits( + { type: "function", parameters: [str()], returnType: { type: "void" } }, + { type: "function", parameters: [unknown()], returnType: { type: "void" } }, + ), + ); }); }); diff --git a/packages/chord/test/fixtures/echo.ts b/packages/chord/test/fixtures/echo.ts index ebe12e0b9..20c81af77 100644 --- a/packages/chord/test/fixtures/echo.ts +++ b/packages/chord/test/fixtures/echo.ts @@ -1,11 +1,11 @@ -import { Type } from "typebox"; +import { num, obj, str } from "@earendil-works/pi-schema"; import { hostNamed, method, remote } from "../../src/index.ts"; export const Echo = remote("t.echo", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("w1"), variants: { - main: { methods: { echo: method(Type.String(), Type.String()), pid: method(undefined, Type.Number()) } }, + main: { methods: { echo: method(str(), str()), pid: method(undefined, num()) } }, }, }); diff --git a/packages/chord/test/fixtures/loader.ts b/packages/chord/test/fixtures/loader.ts index cded6d162..465114bca 100644 --- a/packages/chord/test/fixtures/loader.ts +++ b/packages/chord/test/fixtures/loader.ts @@ -2,11 +2,11 @@ import assert from "node:assert/strict"; import { mkdirSync, rmSync, writeFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; -import { Type } from "typebox"; +import { bool, obj, str } from "@earendil-works/pi-schema"; import { createHost, local, method } from "../../src/index.ts"; import { nodeLoader } from "../../src/loader.ts"; -// Plugin modules live under the package so bare imports (typebox) resolve; removed afterwards. +// Plugin modules live under the package so bare imports (pi-schema) resolve; removed afterwards. const dir = fileURLToPath(new URL(`../.tmp-plugins-${process.pid}/`, import.meta.url)); // Separate from `dir`, which becomes a reload root once its facet loads. const plugins = fileURLToPath(new URL(`../.tmp-plugins-${process.pid}-order/`, import.meta.url)); @@ -15,19 +15,19 @@ const chord = new URL("../../src/index.ts", import.meta.url).href; // The test's own copy of the plugin's contract: binding is by description, not object identity. const Hello = local("t.hello", { version: 1, - input: Type.Object({}), - members: { greet: method(Type.String(), Type.String()) }, + input: obj({}, { open: true }), + members: { greet: method(str(), str()) }, }); const facetSource = ` -import { Type } from "typebox"; +import { obj, str } from "@earendil-works/pi-schema"; import { defineFacet, local, method } from ${JSON.stringify(chord)}; import { greeting } from "./greeting.ts"; const Hello = local("t.hello", { version: 1, - input: Type.Object({}), - members: { greet: method(Type.String(), Type.String()) }, + input: obj({}, { open: true }), + members: { greet: method(str(), str()) }, }); export default defineFacet((host) => { @@ -55,8 +55,8 @@ try { // A facet listed first that imports a later facet's directory shares that facet's copy of the module. const Shared = local("t.shared", { version: 1, - input: Type.Object({}), - members: { same: method(undefined, Type.Boolean()) }, + input: obj({}, { open: true }), + members: { same: method(undefined, bool()) }, }); mkdirSync(`${plugins}a`, { recursive: true }); mkdirSync(`${plugins}b`, { recursive: true }); @@ -67,8 +67,8 @@ try { ); writeFileSync( `${plugins}a/facet.ts`, - `import { Type } from "typebox";\nimport { defineFacet, local, method } from ${JSON.stringify(chord)};\nimport { state } from "../b/state.ts";\n` + - `const Shared = local("t.shared", { version: 1, input: Type.Object({}), members: { same: method(undefined, Type.Boolean()) } });\n` + + `import { bool, obj } from "@earendil-works/pi-schema";\nimport { defineFacet, local, method } from ${JSON.stringify(chord)};\nimport { state } from "../b/state.ts";\n` + + `const Shared = local("t.shared", { version: 1, input: obj({}, { open: true }), members: { same: method(undefined, bool()) } });\n` + `export default defineFacet((host) => host.serve(Shared, () => ({ same: async () => (await import(new URL("../b/facet.ts", import.meta.url).href)).own === state })));\n`, ); const ordered = createHost({ self: { kind: "host" }, loader: nodeLoader, onError: () => {} }); diff --git a/packages/chord/test/fixtures/schemas.ts b/packages/chord/test/fixtures/schemas.ts new file mode 100644 index 000000000..a21a9e09e --- /dev/null +++ b/packages/chord/test/fixtures/schemas.ts @@ -0,0 +1,14 @@ +// Schemas with the exact JSON TypeBox's `Type.Literal` and `Type.Record` built, so contracts declared with them keep +// the fingerprints they had under TypeBox (see the migration notes in CHANGELOG.md). + +import type { Schema } from "@earendil-works/pi-schema"; + +/** `Type.Literal(value)`: the literal with its JSON type (pi-schema's `lit` has no `type`). */ +export function typedLit(value: V): Schema { + return { type: typeof value, const: value }; +} + +/** `Type.Record(Type.String(), value)`: one pattern property for every key (pi-schema's `map` uses `additionalProperties`). */ +export function record(value: Schema): Schema> { + return { type: "object", patternProperties: { "^.*$": value } }; +} diff --git a/packages/chord/test/flow-units.test.ts b/packages/chord/test/flow-units.test.ts index ab578f178..0fd9c984f 100644 --- a/packages/chord/test/flow-units.test.ts +++ b/packages/chord/test/flow-units.test.ts @@ -1,7 +1,7 @@ // Entry 8: flow control with units. First units pass the window, so a get never waits behind other streams; waiting // ops are capped per link; relays ask downstream for a reset when they drop ops. import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { arr, num, obj, str } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { HostImpl } from "../src/host.ts"; import { @@ -26,12 +26,12 @@ async function until(check: () => boolean, attempts = 400): Promise { const Blob = remote("t.e8-blob", { version: 1, - input: Type.Object({ name: Type.String() }), + input: obj({ name: str() }, { open: true }), host: hostNamed("server"), variants: { main: { - state: { lines: Type.Array(Type.String()) }, - methods: { push: method(Type.String()), ping: method(undefined, Type.String()) }, + state: { lines: arr(str()) }, + methods: { push: method(str()), ping: method(undefined, str()) }, }, }, }); @@ -41,7 +41,7 @@ const lines = (kib: number, tag: string) => Array.from({ length: kib }, (_, i) = function serveBlob(host: Host, size: (name: string) => number): void { host.serve(Blob, ({ name }, _principal, self) => { - const state = self.state("lines", Type.Array(Type.String()), lines(size(name), name)); + const state = self.state("lines", arr(str()), lines(size(name), name)); return { variant: "main", lines: state, @@ -169,9 +169,9 @@ describe("flow control with units", () => { describe("relay topology", () => { const Counter = remote("t.e8-relayed", { version: 1, - input: Type.Object({ session: Type.String() }), + input: obj({ session: str() }, { open: true }), host: hostFamily("session:", "session"), - variants: { main: { state: { n: Type.Number() }, methods: { add: method(Type.Number()) } } }, + variants: { main: { state: { n: num() }, methods: { add: method(num()) } } }, }); function cluster() { @@ -191,7 +191,7 @@ describe("relay topology", () => { onError: () => {}, }); created.serve(Counter, (_input, _principal, self) => { - const n = self.state("n", Type.Number(), 0); + const n = self.state("n", num(), 0); return { variant: "main", n, diff --git a/packages/chord/test/flow.test.ts b/packages/chord/test/flow.test.ts index 80c8268a7..79835bb25 100644 --- a/packages/chord/test/flow.test.ts +++ b/packages/chord/test/flow.test.ts @@ -1,7 +1,7 @@ // Link flow control: state waits while the peer's acknowledgements lag a window behind, coalesced per subscription; // everything else is sent at once. import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { num, obj } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { type Ctx, @@ -65,12 +65,12 @@ class AckGate implements Link { const counterOn = (name: string, host: string) => remote(name, { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed(host), variants: { main: { - state: { count: Type.Object({ n: Type.Number() }) }, - methods: { add: method(Type.Number(), Type.Number()), wait: method() }, + state: { count: obj({ n: num() }, { open: true }) }, + methods: { add: method(num(), num()), wait: method() }, }, }, }); @@ -89,7 +89,7 @@ const waiting = () => function serveCounter(host: Host, type = Counter): void { host.serve(type, (_input, _principal, self) => { - const count = self.state("count", Type.Object({ n: Type.Number() }), { n: 0 }); + const count = self.state("count", obj({ n: num() }, { open: true }), { n: 0 }); return { variant: "main", count, diff --git a/packages/chord/test/generations.test.ts b/packages/chord/test/generations.test.ts index 3f20b95cc..7496cd1fa 100644 --- a/packages/chord/test/generations.test.ts +++ b/packages/chord/test/generations.test.ts @@ -4,7 +4,7 @@ import assert from "node:assert/strict"; import { setFlagsFromString } from "node:v8"; import { runInNewContext } from "node:vm"; -import { Type } from "typebox"; +import { num, obj } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { createHost, defineRegistry, type Facet, local, method } from "../src/index.ts"; @@ -41,8 +41,8 @@ function tracker() { const Store = local("t.gen-store", { version: 1, - input: Type.Object({}), - members: { get: method(undefined, Type.Number()), count: Type.Number() }, + input: obj({}, { open: true }), + members: { get: method(undefined, num()), count: num() }, }); const Routes = defineRegistry<() => number>("t.gen-routes"); const RELOADS = 6; @@ -57,7 +57,7 @@ describe("generation GC", () => { name: "store", setup: (api) => api.serve(Store, (_input, _principal, self) => { - const count = self.state("count", Type.Number(), 0); + const count = self.state("count", num(), 0); return { count, get: () => sentinel.generation }; }), }; @@ -84,7 +84,7 @@ describe("generation GC", () => { api.serve( Store, (_input, _principal, self) => { - const count = self.state("count", Type.Number(), 0); + const count = self.state("count", num(), 0); // Kept across reloads (same version): adopts each new declaration's close. self.resource("conn", { version: 1, @@ -123,7 +123,7 @@ describe("generation GC", () => { api.serve( Store, (_input, _principal, self) => { - const count = self.state("count", Type.Number(), 0); + const count = self.state("count", num(), 0); self.task(async (ctx) => { await new Promise((resolve) => ctx.signal.addEventListener("abort", () => resolve())); void sentinel.generation; @@ -153,8 +153,8 @@ describe("generation GC", () => { const objects = tracker(); const Item = local("t.gen-item", { version: 1, - input: Type.Object({ id: Type.Number() }), - members: { id: method(undefined, Type.Number()) }, + input: obj({ id: num() }, { open: true }), + members: { id: method(undefined, num()) }, }); const host = createHost({ self: { kind: "host" }, onError: () => {} }); host.serve(Item, (input) => { diff --git a/packages/chord/test/host-objects-reload.test.ts b/packages/chord/test/host-objects-reload.test.ts index 6e762e701..d01912102 100644 --- a/packages/chord/test/host-objects-reload.test.ts +++ b/packages/chord/test/host-objects-reload.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { bool, num, obj, str } from "@earendil-works/pi-schema"; import { onTestFinished, test } from "vitest"; import { HostImpl } from "../src/host.ts"; import { @@ -12,15 +12,16 @@ import { method, type Self, } from "../src/index.ts"; +import { record } from "./fixtures/schemas.ts"; -const input = Type.Object({}); -const flags = Type.Record(Type.String(), Type.Boolean()); +const input = obj({}, { open: true }); +const flags = record(bool()); const SwitchV1 = local("t.eager-switch", { version: 1, input, members: { - enabled: Type.Boolean(), - set: method(Type.Boolean()), + enabled: bool(), + set: method(bool()), }, }); const SwitchV2 = local("t.eager-switch", { @@ -28,7 +29,7 @@ const SwitchV2 = local("t.eager-switch", { input, members: { enabled: flags, - set: method(Type.Object({ conversation: Type.Number(), enabled: Type.Boolean() })), + set: method(obj({ conversation: num(), enabled: bool() }, { open: true })), }, }); const SwitchV3 = local("t.eager-switch", { @@ -36,7 +37,7 @@ const SwitchV3 = local("t.eager-switch", { input, members: { enabled: flags, - update: method(Type.Object({ conversation: Type.Number(), enabled: Type.Boolean() })), + update: method(obj({ conversation: num(), enabled: bool() }, { open: true })), }, }); const Read = defineRegistry<() => unknown>("t.eager-switch-read"); @@ -60,7 +61,7 @@ function switchFacets(log: string[]) { SwitchV1, (_input, _principal, self) => { declareResource(self); - const enabledState = self.state("enabled", Type.Boolean(), false); + const enabledState = self.state("enabled", bool(), false); self.register(Read, { current: () => enabledState.value }); return { enabled: enabledState, @@ -171,13 +172,13 @@ test("the host's reference to a host object follows contract bumps at commit wit const ReadV1 = local("t.eager-read", { version: 1, - input: Type.Object({}), - members: { read: method(undefined, Type.String()) }, + input: obj({}, { open: true }), + members: { read: method(undefined, str()) }, }); const ReadV2 = local("t.eager-read", { version: 2, - input: Type.Object({}), - members: { fetch: method(undefined, Type.String()) }, + input: obj({}, { open: true }), + members: { fetch: method(undefined, str()) }, }); const readerV1: Facet = { name: "read", @@ -219,8 +220,8 @@ test("object-owned old-version references still block a host object's upgrade un onTestFinished(() => host.close()); const Consumer = local("t.eager-consumer", { version: 1, - input: Type.Object({}), - members: { read: method(undefined, Type.String()) }, + input: obj({}, { open: true }), + members: { read: method(undefined, str()) }, }); const oldConsumer: Facet = { name: "consumer", @@ -254,8 +255,8 @@ test("a forgotten contract bump remains refused even when the only consumer is t await host.load([readerV1]); const incompatible = local(ReadV1.name, { version: 1, - input: Type.Object({}), - members: { fetch: method(undefined, Type.String()) }, + input: obj({}, { open: true }), + members: { fetch: method(undefined, str()) }, }); await assert.rejects( host.reload([ @@ -310,7 +311,7 @@ test("a cluster contract upgrade retokens the server's and worker's references t test("a reload that changes a type's lifetime creates its host object, or lets it go", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); - const Kept = local("t.lifetime-change", { version: 1, input: Type.Object({}) }); + const Kept = local("t.lifetime-change", { version: 1, input: obj({}, { open: true }) }); const log: string[] = []; const facet = (lifetime: "host" | "referenced"): Facet => ({ name: "kept", diff --git a/packages/chord/test/host.test.ts b/packages/chord/test/host.test.ts index eb68b5f6b..6a0d34896 100644 --- a/packages/chord/test/host.test.ts +++ b/packages/chord/test/host.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { bool, num, obj, opt, str } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { type Ctx, @@ -27,10 +27,10 @@ async function until(check: () => boolean): Promise { const Counter = local("t.counter", { version: 1, - input: Type.Object({ name: Type.String() }), + input: obj({ name: str() }, { open: true }), members: { - count: Type.Object({ n: Type.Number() }), - add: method(Type.Number(), Type.Number()), + count: obj({ n: num() }, { open: true }), + add: method(num(), num()), }, }); @@ -39,7 +39,7 @@ function serveCounter(host: Host, log: string[] = [], options: { linger?: number Counter, ({ name }, _principal, self) => { log.push(`factory ${name}`); - const count = self.state("count", Type.Object({ n: Type.Number() }), { n: 0 }); + const count = self.state("count", obj({ n: num() }, { open: true }), { n: 0 }); const conn = self.resource("conn", { version: 1, open: async () => { @@ -123,7 +123,7 @@ describe("host: activation and lifetime", () => { const host = quiet(); const log: string[] = []; let release = () => {}; - const Slow = local("t.slow", { version: 1, input: Type.Object({}) }); + const Slow = local("t.slow", { version: 1, input: obj({}, { open: true }) }); host.serve(Slow, (_input, _principal, self) => { self.resource("r", { version: 1, @@ -156,8 +156,8 @@ describe("host: activation and lifetime", () => { it("propagates activation failures to dependents and retries on the next get", async () => { const host = quiet(); let fail = true; - const Leaf = local("t.leaf", { version: 1, input: Type.Object({}) }); - const Parent = local("t.parent", { version: 1, input: Type.Object({}) }); + const Leaf = local("t.leaf", { version: 1, input: obj({}, { open: true }) }); + const Parent = local("t.parent", { version: 1, input: obj({}, { open: true }) }); host.serve(Leaf, (_input, _principal, self) => { self.resource("r", { version: 1, @@ -184,8 +184,8 @@ describe("host: activation and lifetime", () => { it("unwinds declarations when a factory throws and rejects async factories", async () => { const host = quiet(); - const Leaf = local("t.leaf2", { version: 1, input: Type.Object({}) }); - const Bad = local("t.bad", { version: 1, input: Type.Object({ async: Type.Boolean() }) }); + const Leaf = local("t.leaf2", { version: 1, input: obj({}, { open: true }) }); + const Bad = local("t.bad", { version: 1, input: obj({ async: bool() }, { open: true }) }); let leafLive = 0; host.serve(Leaf, (_input, _principal, self) => { self.resource("r", { @@ -213,10 +213,10 @@ describe("host: activation and lifetime", () => { it("rejects duplicate declaration names", async () => { const host = quiet(); - const Dup = local("t.dup", { version: 1, input: Type.Object({}) }); + const Dup = local("t.dup", { version: 1, input: obj({}, { open: true }) }); host.serve(Dup, (_input, _principal, self) => { - self.state("a", Type.Object({}), {}); - self.state("a", Type.Object({}), {}); + self.state("a", obj({}, { open: true }), {}); + self.state("a", obj({}, { open: true }), {}); return {}; }); await assert.rejects(host.hold(Dup, {}).ready, /duplicate state name a/); @@ -224,8 +224,8 @@ describe("host: activation and lifetime", () => { it("detects hold cycles", async () => { const host = quiet(); - const A = local("t.cycle-a", { version: 1, input: Type.Object({}) }); - const B = local("t.cycle-b", { version: 1, input: Type.Object({}) }); + const A = local("t.cycle-a", { version: 1, input: obj({}, { open: true }) }); + const B = local("t.cycle-b", { version: 1, input: obj({}, { open: true }) }); host.serve(A, (_input, _principal, self) => { self.get(B, {}); return {}; @@ -241,7 +241,7 @@ describe("host: activation and lifetime", () => { const errors: unknown[] = []; const host = quiet(errors); const log: string[] = []; - const Multi = local("t.multi", { version: 1, input: Type.Object({}) }); + const Multi = local("t.multi", { version: 1, input: obj({}, { open: true }) }); host.serve(Multi, (_input, _principal, self) => { const a = self.resource("a", { version: 1, open: () => "a", close: () => void log.push("close a") }); self.resource("b", { @@ -291,15 +291,15 @@ describe("host: activation and lifetime", () => { describe("host: declarations and methods", () => { it("refuses a second declaration of one name per kind; kinds have their own names", async () => { const host = quiet(); - const Twice = local("t.twice", { version: 1, input: Type.Object({ kind: Type.String() }) }); + const Twice = local("t.twice", { version: 1, input: obj({ kind: str() }, { open: true }) }); host.serve(Twice, ({ kind }, _principal, self) => { const open = { version: 1, open: () => null }; - self.state("x", Type.Number(), 0); + self.state("x", num(), 0); self.resource("x", open); - self.on("x", self.state("y", Type.Number(), 0), () => {}); - if (kind === "state") self.state("x", Type.Number(), 1); + self.on("x", self.state("y", num(), 0), () => {}); + if (kind === "state") self.state("x", num(), 1); if (kind === "resource") self.resource("x", open); - if (kind === "listener") self.on("x", self.state("z", Type.Number(), 0), () => {}); + if (kind === "listener") self.on("x", self.state("z", num(), 0), () => {}); return {}; }); await host.hold(Twice, { kind: "none" }).ready; @@ -318,11 +318,11 @@ describe("host: declarations and methods", () => { const host = quiet(); const Tool = local("t.tool", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { ping: method(), - count: method(undefined, Type.Number()), - note: method(Type.String()), + count: method(undefined, num()), + note: method(str()), }, }); const seen: unknown[] = []; @@ -348,8 +348,8 @@ describe("host: declarations and methods", () => { const host = quiet(); const Doubled = local("t.doubled", { version: 1, - input: Type.Object({}), - members: { set: method(Type.Number()) }, + input: obj({}, { open: true }), + members: { set: method(num()) }, }); const seen: number[] = []; const facet = (label: string): Facet => ({ @@ -358,7 +358,7 @@ describe("host: declarations and methods", () => { api.serve( Doubled, (_input, _principal, self) => { - const n = self.state("n", Type.Number(), 1); + const n = self.state("n", num(), 1); const doubled = self.derive([n], (value) => value * 2); self.on("doubled", doubled, (value) => { seen.push(value); @@ -389,7 +389,7 @@ describe("host: declarations and methods", () => { describe("host: lifetime", () => { const Store = local("t.store", { version: 1, - input: Type.Object({ scope: Type.Optional(Type.String({ default: "all" })) }), + input: obj({ scope: opt(str({ default: "all" })) }, { open: true }), members: { seen: opaque("t.seen@1") }, }); @@ -397,11 +397,11 @@ describe("host: lifetime", () => { const host = quiet(); const Remote = remote("t.remote-store", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), variants: { main: {} }, }); - const Keyed = local("t.keyed", { version: 1, input: Type.Object({ id: Type.String() }) }); + const Keyed = local("t.keyed", { version: 1, input: obj({ id: str() }, { open: true }) }); // Each refused call is also a type error; the runtime check is the backstop for untyped callers. const refused: [() => void, RegExp][] = [ // @ts-expect-error: a remote type is never of lifetime "host" @@ -415,7 +415,7 @@ describe("host: lifetime", () => { ]; for (const [serve, error] of refused) assert.throws(serve, error); // Allowed: a local type whose input has only optional fields, and any type with the default lifetime. - const Optional = local("t.optional", { version: 1, input: Type.Object({ id: Type.Optional(Type.String()) }) }); + const Optional = local("t.optional", { version: 1, input: obj({ id: opt(str()) }, { open: true }) }); host.serve(Optional, () => ({}), { lifetime: "host" }); host.serve(Keyed, () => ({}), { lifetime: "referenced", linger: 10 }); host.serve(Remote, () => ({ variant: "main" }), { linger: 10 }); @@ -462,13 +462,13 @@ describe("host: lifetime", () => { describe("host: principals, variants and disposal", () => { const Doc = remote("t.doc", { version: 1, - input: Type.Object({ id: Type.String() }), + input: obj({ id: str() }, { open: true }), host: hostNamed("host"), variants: { - viewer: { state: { view: Type.Object({ text: Type.String() }) } }, + viewer: { state: { view: obj({ text: str() }, { open: true }) } }, editor: { - state: { view: Type.Object({ text: Type.String() }) }, - methods: { edit: method(Type.Object({ text: Type.String() }), Type.Object({ ok: Type.Boolean() })) }, + state: { view: obj({ text: str() }, { open: true }) }, + methods: { edit: method(obj({ text: str() }, { open: true }), obj({ ok: bool() }, { open: true })) }, }, }, }); @@ -476,7 +476,7 @@ describe("host: principals, variants and disposal", () => { function serveDoc(host: Host, principals: unknown[] = []) { host.serve(Doc, (_input, principal, self) => { principals.push(principal); - const view = self.state("view", Type.Object({ text: Type.String() }), { text: "" }); + const view = self.state("view", obj({ text: str() }, { open: true }), { text: "" }); if ((principal as { role?: string }).role !== "editor") return { variant: "viewer", view }; return { variant: "editor", @@ -531,7 +531,7 @@ describe("host: principals, variants and disposal", () => { let seenAbort = false; const Slow = local("t.inflight", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { wait: method() }, }); host.serve(Slow, () => ({ @@ -585,8 +585,8 @@ describe("host: registries", () => { it('only objects of lifetime "host" register; tables follow registration order', async () => { const host = quiet(); - const RoutesA = local("t.routes-a", { version: 1, input: Type.Object({}) }); - const RoutesB = local("t.routes-b", { version: 1, input: Type.Object({}) }); + const RoutesA = local("t.routes-a", { version: 1, input: obj({}, { open: true }) }); + const RoutesB = local("t.routes-b", { version: 1, input: obj({}, { open: true }) }); host.serve( RoutesA, (_input, _principal, self) => { @@ -611,7 +611,7 @@ describe("host: registries", () => { assert.equal(host.registry(Routes).get("GET /b"), "b"); assert.deepEqual(tables.at(-1), ["GET /a", "GET /c", "GET /b"]); - const Lazy = local("t.lazy-register", { version: 1, input: Type.Object({}) }); + const Lazy = local("t.lazy-register", { version: 1, input: obj({}, { open: true }) }); host.serve(Lazy, (_input, _principal, self) => { self.register(Routes, { x: "x" }); return {}; @@ -624,7 +624,7 @@ describe("host: registries", () => { const host = quiet(errors); for (const name of ["t.conflict-a", "t.conflict-b"]) { host.serve( - local(name, { version: 1, input: Type.Object({}) }), + local(name, { version: 1, input: obj({}, { open: true }) }), (_input, _principal, self) => { self.register(Routes, { same: name }); return {}; @@ -641,19 +641,19 @@ describe("host: registries", () => { describe("host: composition", () => { const Source = local("t.source", { version: 1, - input: Type.Object({}), - members: { n: Type.Number(), set: method(Type.Number()) }, + input: obj({}, { open: true }), + members: { n: num(), set: method(num()) }, }); const Mirror = local("t.mirror", { version: 1, - input: Type.Object({}), - members: { copy: Type.Number(), doubled: Type.Number() }, + input: obj({}, { open: true }), + members: { copy: num(), doubled: num() }, }); it("reference state members are sources for listeners and derivations", async () => { const host = quiet(); host.serve(Source, (_input, _principal, self) => { - const n = self.state("n", Type.Number(), 1); + const n = self.state("n", num(), 1); return { n, set: (value: number, ctx: Ctx) => { @@ -663,7 +663,7 @@ describe("host: composition", () => { }); host.serve(Mirror, (_input, _principal, self) => { const source = self.get(Source, {}); - const copy = self.state("copy", Type.Number(), 0); + const copy = self.state("copy", num(), 0); self.on("follow", source.n, (value, ctx) => copy.replace(ctx, value ?? 0)); const doubled = self.derive([source.n], (n) => (n ?? 0) * 2); return { copy, doubled }; @@ -688,12 +688,12 @@ describe("host: composition", () => { const seen: unknown[] = []; const Who = local("t.who", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { ping: method() }, }); const Front = local("t.front", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { relay: method() }, }); host.serve(Who, (_input, principal) => { @@ -711,10 +711,10 @@ describe("host: composition", () => { it("calling a reference inside a factory throws not_ready", async () => { const host = quiet(); host.serve(Source, (_input, _principal, self) => ({ - n: self.state("n", Type.Number(), 0), + n: self.state("n", num(), 0), set: () => {}, })); - const Eager = local("t.eager-call", { version: 1, input: Type.Object({}) }); + const Eager = local("t.eager-call", { version: 1, input: obj({}, { open: true }) }); host.serve(Eager, (_input, _principal, self) => { const source = self.get(Source, {}); void source.set(1); @@ -725,8 +725,8 @@ describe("host: composition", () => { it("a ChordError from a factory keeps its code through dependents", async () => { const host = quiet(); - const Secret = local("t.secret", { version: 1, input: Type.Object({}) }); - const Page = local("t.page", { version: 1, input: Type.Object({}) }); + const Secret = local("t.secret", { version: 1, input: obj({}, { open: true }) }); + const Page = local("t.page", { version: 1, input: obj({}, { open: true }) }); host.serve(Secret, () => { throw forbidden("not for you"); }); diff --git a/packages/chord/test/hydration.test.ts b/packages/chord/test/hydration.test.ts index 2c80255cc..77b3806ac 100644 --- a/packages/chord/test/hydration.test.ts +++ b/packages/chord/test/hydration.test.ts @@ -1,7 +1,7 @@ // Entry 8: a local object goes live only with its state, also where it forwards or derives from a remote reference's // member; activation waits at most `readyTimeout` (a cycle across hosts fails with a hint instead of hanging). import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { num, obj, str } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { type Ctx, @@ -20,22 +20,22 @@ const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); /** Lives on the worker `w`; its value is set by `set`. */ const Source = remote("t.e8-source", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("w"), - variants: { main: { state: { v: Type.String() }, methods: { set: method(Type.String()) } } }, + variants: { main: { state: { v: str() }, methods: { set: method(str()) } } }, }); /** Lives on the server; forwards (and derives from) the worker's `Source`. */ const Forward = remote("t.e8-forward", { version: 1, - input: Type.Object({ how: Type.String() }), + input: obj({ how: str() }, { open: true }), host: hostNamed("server"), - variants: { main: { state: { v: Type.String(), size: Type.Number() } } }, + variants: { main: { state: { v: str(), size: num() } } }, }); function serveSource(host: Host): void { host.serve(Source, (_input, _principal, self) => { - const v = self.state("v", Type.String(), "from the worker"); + const v = self.state("v", str(), "from the worker"); return { variant: "main", v, @@ -105,9 +105,9 @@ describe("local hydration", () => { it("a member read inside a factory throws not_ready; it is a plain value once live", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); - const Inner = local("t.e8-inner", { version: 1, input: Type.Object({}), members: { n: Type.Number() } }); - const Outer = local("t.e8-outer", { version: 1, input: Type.Object({}), members: { m: Type.Number() } }); - host.serve(Inner, (_input, _principal, self) => ({ n: self.state("n", Type.Number(), 7) })); + const Inner = local("t.e8-inner", { version: 1, input: obj({}, { open: true }), members: { n: num() } }); + const Outer = local("t.e8-outer", { version: 1, input: obj({}, { open: true }), members: { m: num() } }); + host.serve(Inner, (_input, _principal, self) => ({ n: self.state("n", num(), 7) })); const seen: unknown[] = []; host.serve(Outer, (_input, _principal, self) => { const inner = self.get(Inner, {}); @@ -127,15 +127,15 @@ describe("local hydration", () => { it("a host object of a child holding the parent's object goes live once the parent admits the child", async () => { const Settings = remote("t.e8-settings", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { state: { theme: Type.String() } } }, + variants: { main: { state: { theme: str() } } }, }); - const Local = local("t.e8-boot", { version: 1, input: Type.Object({}), members: { theme: Type.String() } }); + const Local = local("t.e8-boot", { version: 1, input: obj({}, { open: true }), members: { theme: str() } }); const c = cluster({ delay: 20 }); c.server.serve(Settings, (_input, _principal, self) => ({ variant: "main", - theme: self.state("theme", Type.String(), "dark"), + theme: self.state("theme", str(), "dark"), })); await c.worker.install( { @@ -163,13 +163,13 @@ describe("local hydration", () => { it("a reference cycle across hosts fails at readyTimeout with a cycle hint", async () => { const A = remote("t.e8-cycle-a", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { main: {} }, }); const B = remote("t.e8-cycle-b", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("w"), variants: { main: {} }, }); diff --git a/packages/chord/test/link.test.ts b/packages/chord/test/link.test.ts index 31c0c9229..eee67c8b4 100644 --- a/packages/chord/test/link.test.ts +++ b/packages/chord/test/link.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { arr, num, obj, str, unknown } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { type Ctx, @@ -25,24 +25,24 @@ const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); const counterVariants = { main: { - state: { count: Type.Object({ n: Type.Number() }) }, + state: { count: obj({ n: num() }, { open: true }) }, methods: { - add: method(Type.Number(), Type.Number()), - who: method(undefined, Type.Unknown()), + add: method(num(), num()), + who: method(undefined, unknown()), }, }, }; /** A counter on the server. */ const Counter = remote("t.counter", { version: 1, - input: Type.Object({ name: Type.String() }), + input: obj({ name: str() }, { open: true }), host: hostNamed("server"), variants: counterVariants, }); /** A counter on a session host (`session:` + its `session` field). */ const SessionCounter = remote("t.session-counter", { version: 1, - input: Type.Object({ session: Type.String(), name: Type.String() }), + input: obj({ session: str(), name: str() }, { open: true }), host: hostFamily("session:", "session"), variants: counterVariants, }); @@ -52,7 +52,7 @@ const SessionCounter = remote("t.session-counter", { function serveCounter(host: Host, principals: unknown[] = []) { const counter = (principal: unknown, self: Self) => { principals.push(principal); - const count = self.state("count", Type.Object({ n: Type.Number() }), { n: 0 }); + const count = self.state("count", obj({ n: num() }, { open: true }), { n: 0 }); return { variant: "main" as const, count, @@ -199,7 +199,7 @@ function cluster(options: { pingInterval?: number; pingTimeout?: number } = {}): open: () => void log.push(`open ${input.name}`), close: () => void log.push(`close ${input.name}`), }); - const count = self.state("count", Type.Object({ n: Type.Number() }), { n: 0 }); + const count = self.state("count", obj({ n: num() }, { open: true }), { n: 0 }); return { variant: "main", count, @@ -346,9 +346,9 @@ describe("links: relay, stamping, lifetime", () => { it("checks contract tokens across links: contract_mismatch for hosts, upgrade_required for browsers", async () => { const CounterV2 = remote("t.counter", { version: 2, - input: Type.Object({ name: Type.String() }), + input: obj({ name: str() }, { open: true }), host: hostNamed("server"), - variants: { main: { methods: { inc: method(undefined, Type.Number()) } } }, + variants: { main: { methods: { inc: method(undefined, num()) } } }, }); const { server, worker } = serverWithWorker(); server.serve(CounterV2, () => ({ variant: "main", inc: () => 1 })); @@ -408,12 +408,12 @@ describe("links: relay, stamping, lifetime", () => { const Big = remote("t.big", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("session:S1"), variants: { main: { - state: { notes: Type.Array(Type.String()) }, - methods: { echo: method(Type.Array(Type.String()), Type.Array(Type.String())) }, + state: { notes: arr(str()) }, + methods: { echo: method(arr(str()), arr(str())) }, }, }, }); @@ -425,10 +425,10 @@ describe("links: methods without arguments or result", () => { it("travel without args and without a value, and resolve to the result or undefined", async () => { const Tool = remote("t.tool", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { - main: { methods: { ping: method(), count: method(undefined, Type.Number()), note: method(Type.String()) } }, + main: { methods: { ping: method(), count: method(undefined, num()), note: method(str()) } }, }, }); const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); @@ -551,7 +551,7 @@ describe("links: packed payloads", () => { onError: () => {}, }); worker.serve(Big, (_input, _principal, self) => { - const notesState = self.state("notes", Type.Array(Type.String()), large("a")); + const notesState = self.state("notes", arr(str()), large("a")); return { variant: "main", notes: notesState, echo: (notes: string[]) => notes }; }); return down; diff --git a/packages/chord/test/links-review.test.ts b/packages/chord/test/links-review.test.ts index c607d673f..f6a456afb 100644 --- a/packages/chord/test/links-review.test.ts +++ b/packages/chord/test/links-review.test.ts @@ -1,6 +1,6 @@ // Regression tests for Astra's phase 1b review (findings #1-#11) and the shared-worker test gap. import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { num, obj, str, unknown } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { type Ctx, @@ -66,14 +66,14 @@ class WatchedLink implements Link { const Counter = remote("t.counter", { version: 1, - input: Type.Object({ name: Type.String() }), + input: obj({ name: str() }, { open: true }), host: hostNamed("session:S1"), variants: { main: { - state: { count: Type.Object({ n: Type.Number() }) }, + state: { count: obj({ n: num() }, { open: true }) }, methods: { - add: method(Type.Number(), Type.Number()), - who: method(undefined, Type.Unknown()), + add: method(num(), num()), + who: method(undefined, unknown()), slow: method(), }, }, @@ -138,7 +138,7 @@ function cluster() { open: () => void log.push(`open ${input.name}`), close: () => void log.push(`close ${input.name}`), }); - const count = self.state("count", Type.Object({ n: Type.Number() }), { n: 0 }); + const count = self.state("count", obj({ n: num() }, { open: true }), { n: 0 }); return { variant: "main", count, @@ -190,15 +190,15 @@ describe("links review: identity and stamping", () => { const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); const Echo = remote("t.echo", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { methods: { root: method(undefined, Type.String()) } } }, + variants: { main: { methods: { root: method(undefined, str()) } } }, }); const facet = (version: number): Facet => ({ name: "echo", setup: (api) => api.serve(Echo, (_input, _principal, self) => { - self.state("n", Type.Number(), 0, { version }); + self.state("n", num(), 0, { version }); return { variant: "main", root: (ctx: Ctx) => ctx.rootId }; }), }); @@ -342,22 +342,22 @@ describe("links review: relay ownership", () => { it("#11 snapshots through an accepts adapter are validated against the consumer's schema", async () => { const DocV1 = remote("t.doc", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { state: { view: Type.Object({ text: Type.String() }) } } }, + variants: { main: { state: { view: obj({ text: str() }, { open: true }) } } }, }); const DocV2 = remote("t.doc", { version: 2, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { state: { view: Type.Object({ body: Type.String() }) } } }, + variants: { main: { state: { view: obj({ body: str() }, { open: true }) } } }, }); const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); server.serve( DocV2, (_input, _principal, self) => ({ variant: "main", - view: self.state("view", Type.Object({ body: Type.String() }), { body: "hi" }), + view: self.state("view", obj({ body: str() }, { open: true }), { body: "hi" }), }), { accepts: { @@ -402,9 +402,9 @@ describe("links review: relay ownership", () => { describe("links review: cross-host reload", () => { const Greeter = remote("t.greeter", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("session:S1"), - variants: { main: { methods: { greet: method(Type.String(), Type.String()) } } }, + variants: { main: { methods: { greet: method(str(), str()) } } }, }); const greeterFacet = (greeting: string): Facet => ({ name: "greeter", @@ -468,7 +468,7 @@ describe("links review: cross-host reload", () => { const { server, modules } = setup(); await tick(); const results: unknown[] = []; - const Caller = local("t.caller", { version: 1, input: Type.Object({}) }); + const Caller = local("t.caller", { version: 1, input: obj({}, { open: true }) }); const callerFacet = (active: boolean): Facet => ({ name: "caller", setup: (api) => @@ -517,13 +517,13 @@ describe("links review: cross-host reload", () => { await tick(); const Echo = remote("t.echo2", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { main: { methods: { ping: method() } } }, }); const Bouncer = remote("t.bouncer", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("session:S1"), variants: { main: { methods: { bounce: method() } } }, }); @@ -531,7 +531,7 @@ describe("links review: cross-host reload", () => { name: "echo", setup: (api) => api.serve(Echo, (_input, _principal, self) => { - self.state("n", Type.Number(), 0, { version }); + self.state("n", num(), 0, { version }); return { variant: "main", ping: () => {} }; }), }); diff --git a/packages/chord/test/next/contracts/declare.test.ts b/packages/chord/test/next/contracts/declare.test.ts index 25e4e0811..9f17d301c 100644 --- a/packages/chord/test/next/contracts/declare.test.ts +++ b/packages/chord/test/next/contracts/declare.test.ts @@ -1,4 +1,3 @@ -import Type from "typebox"; import { describe, expect, it } from "vitest"; import { arr, @@ -143,7 +142,12 @@ describe("declaration checks", () => { [ "open arguments", "n", - { version: 1, params: {}, hostAddress, variants: { a: { methods: { m: method(Type.Object({})) } } } }, + { + version: 1, + params: {}, + hostAddress, + variants: { a: { methods: { m: method({ type: "object", properties: {} }) } } }, + }, "n.a.methods.m.args: argument objects must be closed", ], [ @@ -166,7 +170,7 @@ describe("declaration checks", () => { ], [ "open arguments", - { version: 1, params: {}, members: { m: method(Type.Object({})) } }, + { version: 1, params: {}, members: { m: method({ type: "object", properties: {} }) } }, "i.members.m.args: argument objects must be closed", ], ])("local refuses %s", (_case, declaration, message) => { @@ -181,7 +185,13 @@ describe("declaration checks", () => { }); it("does not check state for closed objects", () => { - expect(() => local("i", { version: 1, params: {}, members: { s: Type.Object({ a: str() }) } })).not.toThrow(); + expect(() => + local("i", { + version: 1, + params: {}, + members: { s: { type: "object", properties: { a: str() }, required: ["a"] } }, + }), + ).not.toThrow(); }); }); diff --git a/packages/chord/test/next/contracts/evolve.test.ts b/packages/chord/test/next/contracts/evolve.test.ts index 4ddfb78ff..0c4a17354 100644 --- a/packages/chord/test/next/contracts/evolve.test.ts +++ b/packages/chord/test/next/contracts/evolve.test.ts @@ -1,4 +1,3 @@ -import Type from "typebox"; import { describe, expect, expectTypeOf, it } from "vitest"; import type { Context } from "../../../src/context/index.ts"; import { type Contract, contractData, type ShapeOf } from "../../../src/next/contracts/contract.ts"; @@ -136,16 +135,18 @@ describe("edits", () => { }); it("checks every edit: closed arguments and the vocabulary", () => { - expect(() => evolve(NotesV1, (e) => e.args("add", Type.Object({ text: str() })))).toThrow( + expect(() => evolve(NotesV1, (e) => e.args("add", { type: "object", properties: { text: str() } }))).toThrow( "notes v2.editor.methods.add.args: argument objects must be closed", ); - expect(() => evolve(InfoV1, (e) => e.addMethod("m", Type.Object({}), nul()))).toThrow( + expect(() => evolve(InfoV1, (e) => e.addMethod("m", { type: "object", properties: {} }, nul()))).toThrow( "info v2.members.m.args: argument objects must be closed", ); expect(() => evolve(NotesV1, (e) => e.state("editor", "count", { not: {} } as never))).toThrow( 'notes v2.editor.state.count: unsupported keyword "not"', ); - expect(() => evolve(NotesV1, (e) => e.state("editor", "count", Type.Object({})))).not.toThrow(); + expect(() => + evolve(NotesV1, (e) => e.state("editor", "count", { type: "object", properties: {} })), + ).not.toThrow(); }); }); diff --git a/packages/chord/test/next/contracts/fingerprint.test.ts b/packages/chord/test/next/contracts/fingerprint.test.ts index f159b7856..a31434198 100644 --- a/packages/chord/test/next/contracts/fingerprint.test.ts +++ b/packages/chord/test/next/contracts/fingerprint.test.ts @@ -1,13 +1,23 @@ -import Type, { type TSchema } from "typebox"; import { describe, expect, it } from "vitest"; -import { evolve, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { + evolve, + hostKind, + int, + local, + method, + obj, + remote, + type Schema, + str, + union, +} from "../../../src/next/contracts/index.ts"; import { NORMAL_FORM_VERSION } from "../../../src/next/contracts/internal.ts"; import type { Params } from "../../../src/next/contracts/params.ts"; const Server = hostKind("server"); /** Fingerprint of a one-variant remote contract with state `s` and params `params`. */ -function fingerprintOf(state: TSchema, params: Params = {}): string { +function fingerprintOf(state: Schema, params: Params = {}): string { return remote("n", { version: 1, params, hostAddress: () => Server(), variants: { a: { state: { s: state } } } }) .fingerprint; } @@ -47,12 +57,12 @@ describe("fingerprints", () => { }); it.each([ - ["defaults", Type.String(), Type.String({ default: "x" })], - ["bounds", Type.String(), Type.String({ maxLength: 3 })], - ["formats", Type.String(), Type.String({ format: "email" })], - ["union order", Type.Union([Type.String(), Type.Integer()]), Type.Union([Type.Integer(), Type.String()])], - ["additionalProperties", Type.Object({}), Type.Object({}, { additionalProperties: false })], - ["readOnly", Type.String(), Type.String({ readOnly: true })], + ["defaults", str(), str({ default: "x" })], + ["bounds", str(), str({ max: 3 })], + ["formats", str(), { type: "string", format: "email" }], + ["union order", union([str(), int()]), union([int(), str()])], + ["additionalProperties", { type: "object", properties: {} }, obj({})], + ["readOnly", str(), { type: "string", readOnly: true }], ])("are sensitive to %s", (_case, a, b) => { expect(fingerprintOf(a)).not.toBe(fingerprintOf(b)); }); diff --git a/packages/chord/test/next/contracts/params.test.ts b/packages/chord/test/next/contracts/params.test.ts index 96187b2b0..e760a9c58 100644 --- a/packages/chord/test/next/contracts/params.test.ts +++ b/packages/chord/test/next/contracts/params.test.ts @@ -1,4 +1,3 @@ -import Type from "typebox"; import { describe, expect, it } from "vitest"; import { arr, @@ -48,7 +47,7 @@ describe("params policy", () => { ["arrays", arr(str())], ["bytes", bytes()], ["numeric literals", oneOf(1, 2)], - ["defaults", Type.String({ default: "x" })], + ["defaults", str({ default: "x" })], ])("refuses %s, in the types and at declaration", (_case, schema) => { // @ts-expect-error: not a params field schema expect(() => remote("n", { version: 1, params: { p: schema }, hostAddress: () => Server(), variants })).toThrow( diff --git a/packages/chord/test/next/contracts/schema.test.ts b/packages/chord/test/next/contracts/schema.test.ts index f509d0262..ee6a05691 100644 --- a/packages/chord/test/next/contracts/schema.test.ts +++ b/packages/chord/test/next/contracts/schema.test.ts @@ -1,4 +1,6 @@ -import { Compile } from "typebox/compile"; +// Tests of the schema helpers: the JSON Schema they build over pi-schema's builders, their types, `bytes()`, `method`. + +import { compile } from "@earendil-works/pi-schema"; import { describe, expect, expectTypeOf, it } from "vitest"; import { arr, @@ -22,7 +24,7 @@ import { Method } from "../../../src/next/contracts/internal.ts"; import type { Equal } from "../support.ts"; describe("schema helpers", () => { - it("produce ordinary TypeBox schemas with bounds under their JSON Schema names", () => { + it("produce plain JSON Schema with bounds under their JSON Schema names", () => { expect(str({ min: 1, max: 3, pattern: "^a" })).toEqual({ type: "string", minLength: 1, @@ -58,34 +60,39 @@ describe("schema helpers", () => { properties: { a: str(), b: int() }, additionalProperties: false, }); - expect(Compile(schema).Check({ a: "x" })).toBe(true); - expect(Compile(schema).Check({ a: "x", c: 1 })).toBe(false); + expect(compile(schema).check({ a: "x" })).toBe(true); + expect(compile(schema).check({ a: "x", c: 1 })).toBe(false); }); - it("Static and TypeBox's compiler work unchanged", () => { + it("Static gives the values' type; pi-schema's compiler checks them", () => { const schema = obj({ kind: oneOf("chat", "agent"), n: opt(int()), tags: arr(str()), data: bytes() }); type Expected = { kind: "chat" | "agent"; n?: number; tags: string[]; data: Uint8Array }; expectTypeOf, Expected>>().toEqualTypeOf(); - expect(Compile(schema).Check({ kind: "chat", tags: [], data: new Uint8Array(1) })).toBe(true); + expect(compile(schema).check({ kind: "chat", tags: [], data: new Uint8Array(1) })).toBe(true); }); }); describe("bytes", () => { - it("is the data marker with the check attached non-enumerably", () => { + it("is plain data: the marker and nothing else", () => { const schema = bytes({ min: 1, max: 2 }); - expect(JSON.parse(JSON.stringify(schema))).toEqual({ type: "bytes", minLength: 1, maxLength: 2 }); - expect(Object.keys(schema)).toEqual(["type", "minLength", "maxLength"]); - expect(Object.getOwnPropertyDescriptor(schema, "~refine")?.enumerable).toBe(false); + expect(Reflect.ownKeys(schema)).toEqual(["type", "minLength", "maxLength"]); + expect(schema).toEqual({ type: "bytes", minLength: 1, maxLength: 2 }); }); it("checks instanceof Uint8Array and the length bounds", () => { - const validator = Compile(bytes({ min: 1, max: 2 })); - expect(validator.Check(new Uint8Array(1))).toBe(true); - expect(validator.Check(new Uint8Array(2))).toBe(true); - expect(validator.Check(new Uint8Array(0))).toBe(false); - expect(validator.Check(new Uint8Array(3))).toBe(false); - expect(validator.Check([1])).toBe(false); - expect(validator.Check("ab")).toBe(false); + const validator = compile(bytes({ min: 1, max: 2 })); + expect(validator.check(new Uint8Array(1))).toBe(true); + expect(validator.check(new Uint8Array(2))).toBe(true); + expect(validator.check(new Uint8Array(0))).toBe(false); + expect(validator.check(new Uint8Array(3))).toBe(false); + expect(validator.check([1])).toBe(false); + expect(validator.check("ab")).toBe(false); + }); + + it("checks inside a record (slices.md L1 carry-over)", () => { + const validator = compile(map(bytes({ max: 1 }))); + expect(validator.check({ a: new Uint8Array(1) })).toBe(true); + expect(validator.check({ a: new Uint8Array(2) })).toBe(false); }); }); diff --git a/packages/chord/test/next/contracts/snapshot.test.ts b/packages/chord/test/next/contracts/snapshot.test.ts index d90e5a284..e3d6d76b8 100644 --- a/packages/chord/test/next/contracts/snapshot.test.ts +++ b/packages/chord/test/next/contracts/snapshot.test.ts @@ -1,27 +1,35 @@ -import Type from "typebox"; -import { Compile } from "typebox/compile"; +// Tests of schema snapshots: the vocabulary, closed arguments, copies and their normal forms, bytes leaves. + +import { compile } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; -import { arr, bytes, int, map, obj, opt, str } from "../../../src/next/contracts/schema.ts"; +import { arr, bytes, int, map, obj, opt, str, union } from "../../../src/next/contracts/schema.ts"; import { snapshotArguments, snapshotSchema } from "../../../src/next/contracts/snapshot.ts"; describe("vocabulary", () => { it("accepts primitives, arrays, tuples, objects, records, unions, intersections, $defs/$ref, constraints", () => { - const recursive = Type.Cyclic( - { Node: Type.Object({ value: Type.Number(), next: Type.Union([Type.Null(), Type.Ref("Node")]) }) }, - "Node", - ); + const recursive = { + $defs: { + Node: { + type: "object", + properties: { value: { type: "number" }, next: { anyOf: [{ type: "null" }, { $ref: "#/$defs/Node" }] } }, + required: ["value", "next"], + }, + }, + $ref: "#/$defs/Node", + }; const schemas = [ - Type.Tuple([Type.String(), Type.Integer()]), - Type.Intersect([Type.Object({ a: Type.String() }), Type.Object({ b: Type.String() })]), - Type.Enum(["a", "b"]), + { type: "array", prefixItems: [{ type: "string" }, { type: "integer" }], items: false }, + { allOf: [obj({ a: str() }), obj({ b: str() })] }, + { enum: ["a", "b"] }, recursive, str({ min: 1, pattern: "^x" }), - Type.String({ format: "email", default: "a@b.c", readOnly: true, title: "t", description: "d" }), - Type.Number({ multipleOf: 2, exclusiveMinimum: 0 }), - Type.Array(Type.String(), { uniqueItems: true }), - Type.Object({}, { minProperties: 1, maxProperties: 2, $comment: "c", deprecated: true, examples: [{}] }), + { type: "string", format: "email", default: "a@b.c", readOnly: true, title: "t", description: "d" }, + { type: "number", multipleOf: 2, exclusiveMinimum: 0 }, + { type: "array", items: { type: "string" }, uniqueItems: true }, + { type: "object", minProperties: 1, maxProperties: 2, $comment: "c", deprecated: true, examples: [{}] }, map(int()), bytes({ max: 4 }), + map(bytes()), ]; for (const schema of schemas) expect(() => snapshotSchema(schema, "s")).not.toThrow(); }); @@ -43,18 +51,15 @@ describe("vocabulary", () => { ["definitions", { definitions: {} }, 'unsupported keyword "definitions"'], ["oneOf", { oneOf: [{ type: "string" }] }, 'unsupported keyword "oneOf"'], ["unknown keyword", { type: "string", flavour: "x" }, 'unsupported keyword "flavour"'], - ["non-JSON type", Type.BigInt(), 'unsupported type "bigint"'], - ["undefined", Type.Undefined(), 'unsupported type "undefined"'], - ["function", Type.Function([], Type.Null()), "s: unsupported"], - ["Unsafe", Type.Unsafe({ type: "string" }), "unsafe types are not supported"], - ["refinement", Type.Refine(Type.String(), () => true), "refinements"], + ["non-JSON type", { type: "bigint" }, 'unsupported type "bigint"'], + ["undefined", { type: "undefined" }, 'unsupported type "undefined"'], + ["function", { type: "function", parameters: [], returnType: { type: "null" } }, "s: unsupported"], [ - "codec", - Type.Codec(Type.String()) - .Decode((v) => v) - .Encode((v) => v), - "codecs", + "non-enumerable key (a TypeBox refinement)", + Object.defineProperty({ type: "string" }, "~refine", { value: [{ check: () => true }] }), + "s.~refine: non-enumerable keys are not supported", ], + ["opt(...) outside an object", opt(str()), "s: expected a schema"], ["non-JSON data", { type: "number", default: Number.NaN }, "s.default: expected JSON data"], ["undefined data", { type: "number", maximum: undefined }, "s.maximum: expected JSON data"], ["foreign prototype", new (class Schema {})(), "s: expected a schema"], @@ -86,62 +91,62 @@ describe("vocabulary", () => { describe("closed arguments", () => { it("requires every object, nested ones included, to be closed", () => { expect(() => snapshotArguments(obj({ a: obj({ b: str() }) }), "m.args")).not.toThrow(); - expect(() => snapshotArguments(obj({ a: Type.Object({ b: str() }) }), "m.args")).toThrow( + const open = (properties: object) => ({ type: "object", properties }); + expect(() => snapshotArguments(obj({ a: open({ b: str() }) }), "m.args")).toThrow( "m.args.properties.a: argument objects must be closed (additionalProperties: false)", ); - expect(() => snapshotArguments(Type.Object({}), "m.args")).toThrow("m.args: argument objects must be closed"); - expect(() => snapshotArguments(obj({ list: arr(Type.Object({})) }), "m.args")).toThrow( + expect(() => snapshotArguments(open({}), "m.args")).toThrow("m.args: argument objects must be closed"); + expect(() => snapshotArguments(obj({ list: arr(open({})) }), "m.args")).toThrow( "m.args.properties.list.items: argument objects must be closed", ); expect(() => snapshotArguments({ type: "object" }, "m.args")).toThrow("argument objects must be closed"); }); - it("accepts a raw closed Type.Object and records", () => { - expect(() => snapshotArguments(Type.Object({ a: str() }, { additionalProperties: false }), "m")).not.toThrow(); + it("accepts a raw closed object schema and records", () => { + const closed = { type: "object", properties: { a: str() }, additionalProperties: false }; + expect(() => snapshotArguments(closed, "m")).not.toThrow(); expect(() => snapshotArguments(obj({ tags: map(str()) }), "m")).not.toThrow(); }); it("checks objects inside $defs, unions and tuples", () => { - const open = Type.Object({ a: str() }); - expect(() => snapshotArguments(Type.Union([open, str()]), "m")).toThrow("m.anyOf[0]: argument objects"); - expect(() => snapshotArguments(Type.Tuple([open]), "m")).toThrow("m.items[0]: argument objects"); + const open = { type: "object", properties: { a: str() } }; + expect(() => snapshotArguments(union([open, str()]), "m")).toThrow("m.anyOf[0]: argument objects"); + expect(() => snapshotArguments({ type: "array", items: [open] }, "m")).toThrow("m.items[0]: argument objects"); expect(() => snapshotArguments({ $defs: { A: open }, $ref: "A" }, "m")).toThrow("m.$defs.A: argument objects"); }); it("does not apply to state or results", () => { - expect(() => snapshotSchema(Type.Object({ a: str() }), "state")).not.toThrow(); + expect(() => snapshotSchema({ type: "object", properties: { a: str() } }, "state")).not.toThrow(); }); }); describe("snapshots", () => { it("are copies, independent of later changes to the app's schema objects", () => { - const app = obj({ text: str(), tags: arr(str()) }); + const app = obj({ text: str(), tags: arr(str()) }) as { properties: { text: object }; required: string[] }; const { schema } = snapshotSchema(app, "s"); (app.properties.text as { minLength?: number }).minLength = 5; - (app.required as string[]).push("x"); + app.required.push("x"); expect(schema).not.toBe(app); - expect(Compile(schema).Check({ text: "", tags: [] })).toBe(true); + expect(compile(schema).check({ text: "", tags: [] })).toBe(true); }); it("copy data values (defaults, consts)", () => { const fallback = { a: [1] }; - const { schema } = snapshotSchema(Type.Object({}, { default: fallback, additionalProperties: false }), "s"); + const { schema } = snapshotSchema({ type: "object", default: fallback, additionalProperties: false }, "s"); fallback.a.push(2); const copied = (schema as { default: { a: number[] } }).default; expect(copied).toEqual({ a: [1] }); }); - it("rebuild the bytes check from the marker; no app callback is copied", () => { - const app = bytes({ max: 2 }); - const appCheck = Object.getOwnPropertyDescriptor(app, "~refine")!.value; - const { schema } = snapshotSchema(app, "s"); - expect(Object.getOwnPropertyDescriptor(schema, "~refine")!.value).not.toBe(appCheck); - expect(Compile(schema).Check(new Uint8Array(2))).toBe(true); - expect(Compile(schema).Check(new Uint8Array(3))).toBe(false); - // A hand-written marker works too: the check comes from the marker, not from the app. - const marker = snapshotSchema({ type: "bytes", minLength: 1 }, "s").schema; - expect(Compile(marker).Check(new Uint8Array(0))).toBe(false); - expect(Compile(marker).Check(new Uint8Array(1))).toBe(true); + it("copy a bytes leaf as data: the compiled check comes from its keywords", () => { + const { schema } = snapshotSchema(bytes({ max: 2 }), "s"); + expect(schema).toEqual({ type: "bytes", maxLength: 2 }); + expect(compile(schema).check(new Uint8Array(2))).toBe(true); + expect(compile(schema).check(new Uint8Array(3))).toBe(false); + // A record of bytes (slices.md L1 carry-over: refused while bytes carried a TypeBox refinement). + const record = snapshotSchema(map(bytes({ min: 1 })), "s").schema; + expect(compile(record).check({ a: new Uint8Array(1) })).toBe(true); + expect(compile(record).check({ a: new Uint8Array(0) })).toBe(false); }); it("refuse a bytes marker with other keywords or bad bounds", () => { @@ -173,10 +178,9 @@ describe("snapshots", () => { }); it("normal form keeps union and tuple order", () => { - const ab = snapshotSchema(Type.Union([str(), int()]), "s").normal; - const ba = snapshotSchema(Type.Union([int(), str()]), "s").normal; + const ab = snapshotSchema(union([str(), int()]), "s").normal; + const ba = snapshotSchema(union([int(), str()]), "s").normal; expect(ab).not.toEqual(ba); - expect(snapshotSchema(opt(str()), "s").normal).toEqual({ type: "string" }); }); }); @@ -184,7 +188,7 @@ describe("bytes", () => { it("mark a snapshot with a bytes leaf anywhere inside", () => { expect(snapshotSchema(str(), "s").bytes).toBe(false); expect(snapshotSchema(obj({ a: arr(obj({ b: bytes() })) }), "s").bytes).toBe(true); - expect(snapshotSchema(Type.Union([str(), bytes()]), "s").bytes).toBe(true); - expect(snapshotSchema(Type.Object({ a: str() }, { default: { type: "bytes" } }), "s").bytes).toBe(false); + expect(snapshotSchema(union([str(), bytes()]), "s").bytes).toBe(true); + expect(snapshotSchema({ type: "object", default: { type: "bytes" } }, "s").bytes).toBe(false); }); }); diff --git a/packages/chord/test/next/data/args.test.ts b/packages/chord/test/next/data/args.test.ts index 896861fc0..bdd26d32b 100644 --- a/packages/chord/test/next/data/args.test.ts +++ b/packages/chord/test/next/data/args.test.ts @@ -147,10 +147,20 @@ describe("checkArgs", () => { const local1 = bindProvider(Info, {}); expect(checkArgs(local1, "save", { title: "t" }, 1, BACKGROUND_CONTEXT).args).toEqual({ title: "t" }); expect(refusal(() => checkArgs(local1, "save", { title: 1 }, 1, BACKGROUND_CONTEXT))).toMatchObject({ - message: "args/title: must be string", + message: "args/title: expected string, got integer", }); }); + it("treats `format` as an annotation: a contract with format email accepts any string", () => { + const Mail = local("mail", { + version: 1, + params: {}, + members: { send: method(obj({ to: { type: "string", format: "email" } }), nul()) }, + }); + const mail = bindProvider(Mail, {}); + expect(checkArgs(mail, "send", { to: "nope" }, 1, BACKGROUND_CONTEXT).args).toEqual({ to: "nope" }); + }); + it("refuses an unknown method, an unaccepted version and a refused method", () => { expect(refusal(() => checkArgs(provider, "missing", null, 3, BACKGROUND_CONTEXT))).toEqual({ code: "invalid", diff --git a/packages/chord/test/next/data/params.test.ts b/packages/chord/test/next/data/params.test.ts index 5848b0378..20777e981 100644 --- a/packages/chord/test/next/data/params.test.ts +++ b/packages/chord/test/next/data/params.test.ts @@ -59,10 +59,10 @@ describe("checkParams", () => { details: { path: "/extra" }, }); expect(refusal({ ...valid, page: "2" })).toMatchObject({ - message: "params/page: must be integer", + message: "params/page: expected integer, got string", details: { path: "/page" }, }); - expect(refusal("S1")).toMatchObject({ message: "params: must be object", details: { path: "/" } }); + expect(refusal("S1")).toMatchObject({ message: "params: expected object, got string", details: { path: "/" } }); }); it("enforces literal membership and constraints", () => { diff --git a/packages/chord/test/next/data/validator.test.ts b/packages/chord/test/next/data/validator.test.ts index b78b55c58..d1c09ec4c 100644 --- a/packages/chord/test/next/data/validator.test.ts +++ b/packages/chord/test/next/data/validator.test.ts @@ -1,4 +1,5 @@ -import { GetExternal, GetFunctions } from "typebox/schema"; +// Tests of compiled checks: pi-schema validators of Chord-owned schemas and the `invalid` error of a failed check. + import { describe, expect, it } from "vitest"; import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; import { arr, bytes, int, map, obj, str } from "../../../src/next/contracts/schema.ts"; @@ -31,17 +32,21 @@ describe("failed checks", () => { expect(failure(valid)).toBe("passed"); }); - it("report TypeBox's first error with its JSON Pointer path", () => { + it("report the first error with its JSON Pointer path", () => { expect(failure({ ...valid, text: 1 })).toEqual({ reason: "args", - message: "args/text: must be string", + message: "args/text: expected string, got integer", details: { path: "/text" }, }); expect(failure({ ...valid, items: [{ n: "x" }] })).toMatchObject({ details: { path: "/items/0/n" } }); - expect(failure(null)).toEqual({ reason: "args", message: "args: must be object", details: { path: "/" } }); + expect(failure(null)).toEqual({ + reason: "args", + message: "args: expected object, got null", + details: { path: "/" }, + }); }); - it("report a missing field at its own path (TypeBox reports it at the parent)", () => { + it("report a missing field at its own path", () => { const { text: _, ...missing } = valid; expect(failure(missing)).toMatchObject({ message: "args/text: missing", @@ -60,14 +65,22 @@ describe("failed checks", () => { }); it("check bytes by instanceof and length", () => { - expect(failure({ ...valid, blob: [1] })).toMatchObject({ - message: "args/blob: expected bytes of the declared length", + expect(failure({ ...valid, blob: [1] })).toMatchObject({ message: "args/blob: expected bytes, got array" }); + expect(failure({ ...valid, blob: new Uint8Array(3) })).toMatchObject({ + message: "args/blob: expected at most 2 bytes", + details: { path: "/blob" }, }); - expect(failure({ ...valid, blob: new Uint8Array(3) })).toMatchObject({ details: { path: "/blob" } }); }); - it("leave keys containing `/` or `~` unescaped (TypeBox's instance path, ambiguous)", () => { - expect(failure({ ...valid, tags: { "a/b": 1 } })).toMatchObject({ details: { path: "/tags/a/b" } }); + it("treat `format` as an annotation: no format is checked", () => { + const owner = snapshotArguments(obj({ mail: { type: "string", format: "email" } }), "m"); + const validator = validatorFor(owner, BACKGROUND_CONTEXT); + expect(validator.check({ mail: "a@b.c" })).toBe(true); + expect(validator.check({ mail: "nope" })).toBe(true); + }); + + it("escape `~` and `/` in keys (JSON Pointer), so a path names one place", () => { + expect(failure({ ...valid, tags: { "a/b": 1, "c~d": "x" } })).toMatchObject({ details: { path: "/tags/a~1b" } }); }); }); @@ -86,20 +99,11 @@ describe("validators", () => { ]); }); - it("leave nothing in TypeBox's build accumulators", () => { - validatorFor(snapshotArguments(obj({ blob: bytes(), n: int() }), "m"), BACKGROUND_CONTEXT); - expect(GetExternal().variables).toEqual([]); - expect(GetFunctions()).toEqual([]); - }); - - it("are collected with their schema owner; TypeBox retains no build state", async () => { + it("are collected with their schema owner, the check function too", async () => { const refs = (() => { - const owner = snapshotArguments(obj({ blob: bytes() }), "m"); + const owner = snapshotArguments(obj({ blob: bytes(), n: int() }), "m"); const validator = validatorFor(owner, BACKGROUND_CONTEXT); - const blob = (owner.schema as { properties: { blob: object } }).properties.blob; - // TypeBox's external variables would hold the bytes check if the build state were retained. - const [check] = Object.getOwnPropertyDescriptor(blob, "~refine")!.value as object[]; - return [new WeakRef(owner), new WeakRef(validator), new WeakRef(check!)]; + return [new WeakRef(owner), new WeakRef(validator), new WeakRef(validator.check)]; })(); await expectCollected(...refs); }); diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/units.ts b/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/units.ts index e2fc0bfb5..c5a8a13b9 100644 --- a/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/units.ts +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/units.ts @@ -1,7 +1,7 @@ // A file of the shapes plugin that imports a package (not a host package): fine for the shapes plugin itself, but a // plugin importing this file would copy it, and a copied file may not import packages (the ruler plugin shows it). -import { Type } from "typebox"; +import { num } from "@earendil-works/pi-schema"; /** A length in millimetres. */ -export const Millimetres = Type.Number({ minimum: 0 }); +export const Millimetres = num({ minimum: 0 }); diff --git a/packages/chord/test/next/schema/declarations.ts b/packages/chord/test/next/schema/declarations.ts new file mode 100644 index 000000000..cf0a329c0 --- /dev/null +++ b/packages/chord/test/next/schema/declarations.ts @@ -0,0 +1,180 @@ +// The declarations of the schema switch's proofs (TypeBox to pi-schema): one of every schema helper and option in a +// contract, persisted documents, tables with defaults, an optional and a `json(...)` column, an evolved table and a +// persisted job. Their fingerprints were recorded with the TypeBox helpers; the stored data in `stored/` was written +// by that code (`write-stored.ts`). + +import { + arr, + bool, + bytes, + hostKind, + int, + keyed, + lit, + local, + map, + method, + nul, + num, + obj, + oneOf, + opt, + remote, + str, + union, + unknown, +} from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { defineJob } from "../../../src/next/jobs/index.ts"; +import { defineTable, evolveTable, json } from "../../../src/next/store/index.ts"; + +/** A schema with every helper and option a declaration can use. */ +export const everything = obj({ + name: str({ min: 1, max: 40, pattern: "^[a-z ]+$" }), + nick: str({ default: "x" }), + age: int({ min: 0, max: 150 }), + count: int({ default: 0 }), + score: num({ min: 0, max: 100 }), + ratio: num({ default: 0.5 }), + active: bool(), + verified: bool({ default: false }), + role: oneOf("admin", "user", "guest"), + level: oneOf(1, 2, 3), + kind: lit("person"), + one: lit(1), + yes: lit(true), + tags: arr(str(), { min: 0, max: 5 }), + points: arr(int()), + extra: map(int()), + note: opt(str()), + nothing: nul(), + choice: union([str(), int(), obj({ deep: arr(obj({ x: num() })) })]), + nested: obj({ inner: opt(obj({ v: str() })) }), +}); + +/** A local contract with state, keyed state and methods of every argument shape. */ +export const Everything = local("fixture.switch.everything", { + version: 1, + params: { id: str(), shard: int() }, + members: { + profile: everything, + notes: keyed(obj({ body: str() })), + plain: method(), + args: method(obj({ a: str(), b: opt(int()), any: unknown() })), + result: method(undefined, arr(str())), + both: method(obj({ when: int({ min: 0 }) }), union([lit("ok"), obj({ error: str() })])), + blob: method(obj({ data: bytes({ max: 1024 }) }), bytes()), + }, +}); + +const Server = hostKind("fixture.switch.server", { region: str() }); + +/** A remote contract with variants and a host address. */ +export const Account = remote("fixture.switch.account", { + version: 1, + params: { user: str({ min: 1 }) }, + hostAddress: () => Server({ region: "eu" }), + variants: { + owner: { state: { balance: num() }, methods: { pay: method(obj({ amount: num({ min: 0 }) }), bool()) } }, + guest: { state: { balance: num() } }, + }, +}); + +/** A persisted singleton document of `everything`. */ +export const Profile = defineState("fixture.switch.profile", { + schema: everything, + version: 1, + initial: () => ({ + name: "ada", + nick: "x", + age: 36, + count: 0, + score: 1, + ratio: 0.5, + active: true, + verified: false, + role: "admin" as const, + level: 1 as const, + kind: "person" as const, + one: 1 as const, + yes: true as const, + tags: [], + points: [], + extra: {}, + nothing: null, + choice: "a", + nested: {}, + }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 2, +}); + +/** A persisted keyed document. */ +export const Notes = defineState("fixture.switch.notes", { + schema: obj({ body: str(), tags: opt(arr(str())) }), + version: 1, + initial: () => ({ body: "" }), + keyed: true, + persist: true, + checkpointWhen: () => false, +}); + +/** A table with defaults, an optional column and a `json(...)` column. */ +export const Rows = defineTable("fixture.switch.rows", { + version: 1, + columns: { + id: str(), + n: int({ default: 0 }), + flag: bool({ default: false }), + label: opt(str()), + data: json(obj({ a: int(), b: opt(arr(str())) }), { version: 1 }), + maybe: opt(json(map(num()), { version: 1 })), + }, + key: ["id"], + indexes: { by_n: ["n", "id"] }, +}); + +/** A table evolved over three versions: a rename, an added column with a default, a json column, a drop. */ +export const EditsV1 = defineTable("fixture.switch.edits", { + version: 1, + columns: { id: str(), file: str(), at: int() }, + key: ["id"], +}); + +export const EditsV2 = evolveTable(EditsV1, (t) => + t + .rename("file", "path") + .add("dir", str({ default: "" })) + .rows((row) => ({ dir: row.path.slice(0, row.path.lastIndexOf("/")) })), +); + +export const EditsV3 = evolveTable(EditsV2, (t) => t.drop("dir").index("by_path", ["path", "at"])); + +/** A persisted job with a state document. */ +export const ImportState = defineState("fixture.switch.import.state", { + schema: obj({ done: int() }), + version: 1, + initial: () => ({ done: 0 }), + persist: true, + checkpointWhen: () => false, +}); + +export const ImportJob = defineJob("fixture.switch.import", { + version: 1, + args: obj({ path: str() }), + result: obj({ count: int() }), + state: ImportState, + persist: { resume: "rerun" }, +}); + +/** The object owning the persisted documents and the tables (a local type with empty params, as tables need). */ +export const Owner = local("fixture.switch.owner", { + version: 1, + params: {}, + members: { + profile: everything, + write: method(obj({ id: str(), body: str() })), + read: method(str(), str()), + dump: method(obj({}), unknown()), + }, +}); diff --git a/packages/chord/test/next/schema/owner.ts b/packages/chord/test/next/schema/owner.ts new file mode 100644 index 000000000..88928df5e --- /dev/null +++ b/packages/chord/test/next/schema/owner.ts @@ -0,0 +1,85 @@ +// The object of the stored-data proof: it owns the persisted documents, the tables and the job of `declarations.ts`, +// writes them (`write-stored.ts`, with the TypeBox helpers) and reads them back (the test, after the switch). + +import type { Context } from "../../../src/context/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import type { TableDefinition } from "../../../src/next/store/index.ts"; +import { openStore, query, type Store, sqliteBackend } from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { ImportJob, Notes, Owner, Profile, Rows } from "./declarations.ts"; + +/** What the job's `run` does in this process: wait until its context ends, or finish at once. */ +export type RunBehaviour = "wait" | "finish"; + +/** The finished runs' results, in order. */ +export const finished: unknown[] = []; + +/** The owner's facet over `edits` (the table version this process declares). */ +export function ownerFacet(edits: TableDefinition, behaviour: RunBehaviour) { + return defineFacet((host) => { + host.serve(Owner, (_, self) => { + const profile = self.state(Profile); + const notes = self.state(Notes); + const rows = self.table(Rows); + const editsTable = self.table(edits); + const imports = self.job(ImportJob, { + run: async ({ path }, ctx) => { + await ctx.state.change((draft) => { + draft.done += path.length; + }, ctx); + if (behaviour === "wait") { + await new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", resolve)); + } + return { count: ctx.state.value.done }; + }, + finish: async (outcome) => { + finished.push(outcome.kind === "done" ? outcome.result : outcome.kind); + }, + }); + return { + profile, + write: async ({ id, body }, ctx) => { + await self.tx(async (tx) => { + (await tx.draft(notes, id)).body = body; + const draft = await tx.draft(profile); + draft.count += 1; + draft.tags.push(id); + draft.extra[id] = body.length; + draft.choice = { deep: [{ x: draft.count }] }; + draft.nested = { inner: { v: body } }; + tx.put(rows, { + id, + label: body, + data: { a: body.length, b: [id] }, + ...(id === "a" ? { maybe: { n: 1.5 } } : {}), + }); + tx.put(editsTable, { id, file: `/notes/${id}.md`, at: body.length } as never); + }, ctx); + await imports.start({ path: id }, ctx); + return null; + }, + read: async (id, ctx) => (await notes.get(id, ctx))?.body ?? "", + dump: async (_, ctx) => ({ + profile: profile.value, + notes: { a: (await notes.get("a", ctx)) ?? null, b: (await notes.get("b", ctx)) ?? null }, + rows: (await rows.page(query(Rows), 10, ctx)).items, + edits: (await editsTable.page(query(edits as TableDefinition), 10, ctx)).items, + runs: imports.running.value.length, + }), + }; + }); + }); +} + +/** A host serving the owner on the SQLite store at `path`. */ +export async function ownerHost( + path: string, + edits: TableDefinition, + behaviour: RunBehaviour, + context: Context, +): Promise<{ store: Store; host: Host }> { + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context); + const host = createHost({ store, onError: () => {}, closeTimeout: 100 }); + await host.load([{ name: "@fixture/owner", facet: ownerFacet(edits, behaviour) }], context); + return { store, host }; +} diff --git a/packages/chord/test/next/schema/proofs.ts b/packages/chord/test/next/schema/proofs.ts new file mode 100644 index 000000000..6f285111e --- /dev/null +++ b/packages/chord/test/next/schema/proofs.ts @@ -0,0 +1,113 @@ +// The values the schema switch must keep (`recorded.json`, made with the TypeBox helpers): the fingerprints and normal +// forms of `declarations.ts`, and what a host reads back from the stored data in `stored/`. + +import { createReadStream } from "node:fs"; +import { copyFile, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { Context } from "../../../src/context/index.ts"; +import type { JsonValue } from "../../../src/json.ts"; +import type { Contract } from "../../../src/next/contracts/index.ts"; +import { contractData } from "../../../src/next/contracts/internal.ts"; +import { snapshotSchema } from "../../../src/next/contracts/snapshot.ts"; +import { decodeJsonl, importStore, sqliteBackend } from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import * as declared from "./declarations.ts"; +import { finished, ownerHost } from "./owner.ts"; + +/** Every fingerprint and normal form of the declarations. */ +export function fingerprints(): JsonValue { + const contract = (C: Contract) => { + const data = contractData(C); + return { + params: data.params.normal, + versions: data.versions.map((version) => ({ + fingerprint: version.fingerprint, + variants: Object.fromEntries( + [...version.variants].map(([name, variant]) => [ + name, + { + state: Object.fromEntries([...variant.state].map(([m, s]) => [m, s.normal])), + keyed: Object.fromEntries([...variant.keyed].map(([m, s]) => [m, s.normal])), + methods: Object.fromEntries( + [...variant.methods].map(([m, { args, result }]) => [ + m, + { args: args.normal, result: result.normal }, + ]), + ), + }, + ]), + ), + })), + }; + }; + const table = (T: { readonly fingerprint: string; readonly record: unknown }) => ({ + fingerprint: T.fingerprint, + record: T.record as unknown as JsonValue, + }); + return { + contracts: { + everything: contract(declared.Everything), + account: contract(declared.Account), + owner: contract(declared.Owner), + }, + documents: { + profile: { + fingerprint: declared.Profile.fingerprint, + normal: snapshotSchema(declared.everything, "p").normal, + }, + notes: declared.Notes.fingerprint, + importState: declared.ImportState.fingerprint, + }, + tables: { + rows: table(declared.Rows), + editsV1: table(declared.EditsV1), + editsV2: table(declared.EditsV2), + editsV3: table(declared.EditsV3), + }, + job: declared.ImportJob.fingerprint, + }; +} + +const stored = fileURLToPath(new URL("./stored/", import.meta.url)); + +/** + * What a host of the current code reads from the stored data: from a copy of the SQLite store, and from a store + * imported from the archive (both written at table version 1, read at version 3; the stopped job runs resume). + */ +export async function storedReads(context: Context): Promise { + const directory = await mkdtemp(join(tmpdir(), "chord-schema-switch-")); + try { + const fromStore = join(directory, "store.db"); + await copyFile(join(stored, "store.db"), fromStore); + const store = await read(fromStore, context); + const imported = await importStore( + sqliteBackend(await openNodeSqliteDatabase(join(directory, "imported.db"))), + decodeJsonl(createReadStream(join(stored, "archive.jsonl"))), + context, + ); + await imported.close(context); + const archive = await read(join(directory, "imported.db"), context); + return { store, archive }; + } finally { + await rm(directory, { recursive: true, force: true }); + } +} + +/** The owner's dump on the store at `path`, after its stopped runs finished. */ +async function read(path: string, context: Context): Promise { + finished.length = 0; + const { store, host } = await ownerHost(path, declared.EditsV3, "finish", context); + const owner = await host.get(declared.Owner, {}, context); + for (let i = 0; i < 100 && finished.length < 3; i++) await new Promise((resolve) => setTimeout(resolve, 10)); + const dump = await owner.dump({}, context); + const result = { + dump: dump as JsonValue, + finished: [...finished] as JsonValue, + note: await owner.read("b", context), + }; + await host.close(context); + await store.close(context); + return result; +} diff --git a/packages/chord/test/next/schema/recorded.json b/packages/chord/test/next/schema/recorded.json new file mode 100644 index 000000000..00c0a3588 --- /dev/null +++ b/packages/chord/test/next/schema/recorded.json @@ -0,0 +1,1143 @@ +{ + "fingerprints": { + "contracts": { + "everything": { + "params": [ + [ + "id", + { + "type": "string" + } + ], + [ + "shard", + { + "type": "integer" + } + ] + ], + "versions": [ + { + "fingerprint": "3861c5a457ef4aa3", + "variants": { + "members": { + "state": { + "profile": { + "additionalProperties": false, + "properties": { + "active": { + "type": "boolean" + }, + "age": { + "maximum": 150, + "minimum": 0, + "type": "integer" + }, + "choice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "additionalProperties": false, + "properties": { + "deep": { + "items": { + "additionalProperties": false, + "properties": { + "x": { + "type": "number" + } + }, + "required": [ + "x" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "deep" + ], + "type": "object" + } + ] + }, + "count": { + "default": 0, + "type": "integer" + }, + "extra": { + "patternProperties": { + "^.*$": { + "type": "integer" + } + }, + "type": "object" + }, + "kind": { + "const": "person", + "type": "string" + }, + "level": { + "anyOf": [ + { + "const": 1, + "type": "number" + }, + { + "const": 2, + "type": "number" + }, + { + "const": 3, + "type": "number" + } + ] + }, + "name": { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z ]+$", + "type": "string" + }, + "nested": { + "additionalProperties": false, + "properties": { + "inner": { + "additionalProperties": false, + "properties": { + "v": { + "type": "string" + } + }, + "required": [ + "v" + ], + "type": "object" + } + }, + "type": "object" + }, + "nick": { + "default": "x", + "type": "string" + }, + "note": { + "type": "string" + }, + "nothing": { + "type": "null" + }, + "one": { + "const": 1, + "type": "number" + }, + "points": { + "items": { + "type": "integer" + }, + "type": "array" + }, + "ratio": { + "default": 0.5, + "type": "number" + }, + "role": { + "anyOf": [ + { + "const": "admin", + "type": "string" + }, + { + "const": "user", + "type": "string" + }, + { + "const": "guest", + "type": "string" + } + ] + }, + "score": { + "maximum": 100, + "minimum": 0, + "type": "number" + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 5, + "minItems": 0, + "type": "array" + }, + "verified": { + "default": false, + "type": "boolean" + }, + "yes": { + "const": true, + "type": "boolean" + } + }, + "required": [ + "active", + "age", + "choice", + "count", + "extra", + "kind", + "level", + "name", + "nested", + "nick", + "nothing", + "one", + "points", + "ratio", + "role", + "score", + "tags", + "verified", + "yes" + ], + "type": "object" + } + }, + "keyed": { + "notes": { + "additionalProperties": false, + "properties": { + "body": { + "type": "string" + } + }, + "required": [ + "body" + ], + "type": "object" + } + }, + "methods": { + "plain": { + "args": { + "type": "null" + }, + "result": { + "type": "null" + } + }, + "args": { + "args": { + "additionalProperties": false, + "properties": { + "a": { + "type": "string" + }, + "any": {}, + "b": { + "type": "integer" + } + }, + "required": [ + "a", + "any" + ], + "type": "object" + }, + "result": { + "type": "null" + } + }, + "result": { + "args": { + "type": "null" + }, + "result": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "both": { + "args": { + "additionalProperties": false, + "properties": { + "when": { + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "when" + ], + "type": "object" + }, + "result": { + "anyOf": [ + { + "const": "ok", + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "type": "object" + } + ] + } + }, + "blob": { + "args": { + "additionalProperties": false, + "properties": { + "data": { + "maxLength": 1024, + "type": "bytes" + } + }, + "required": [ + "data" + ], + "type": "object" + }, + "result": { + "type": "bytes" + } + } + } + } + } + } + ] + }, + "account": { + "params": [ + [ + "user", + { + "minLength": 1, + "type": "string" + } + ] + ], + "versions": [ + { + "fingerprint": "821be6a79fbe350a", + "variants": { + "owner": { + "state": { + "balance": { + "type": "number" + } + }, + "keyed": {}, + "methods": { + "pay": { + "args": { + "additionalProperties": false, + "properties": { + "amount": { + "minimum": 0, + "type": "number" + } + }, + "required": [ + "amount" + ], + "type": "object" + }, + "result": { + "type": "boolean" + } + } + } + }, + "guest": { + "state": { + "balance": { + "type": "number" + } + }, + "keyed": {}, + "methods": {} + } + } + } + ] + }, + "owner": { + "params": [], + "versions": [ + { + "fingerprint": "b01eec2d4e2311b0", + "variants": { + "members": { + "state": { + "profile": { + "additionalProperties": false, + "properties": { + "active": { + "type": "boolean" + }, + "age": { + "maximum": 150, + "minimum": 0, + "type": "integer" + }, + "choice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "additionalProperties": false, + "properties": { + "deep": { + "items": { + "additionalProperties": false, + "properties": { + "x": { + "type": "number" + } + }, + "required": [ + "x" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "deep" + ], + "type": "object" + } + ] + }, + "count": { + "default": 0, + "type": "integer" + }, + "extra": { + "patternProperties": { + "^.*$": { + "type": "integer" + } + }, + "type": "object" + }, + "kind": { + "const": "person", + "type": "string" + }, + "level": { + "anyOf": [ + { + "const": 1, + "type": "number" + }, + { + "const": 2, + "type": "number" + }, + { + "const": 3, + "type": "number" + } + ] + }, + "name": { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z ]+$", + "type": "string" + }, + "nested": { + "additionalProperties": false, + "properties": { + "inner": { + "additionalProperties": false, + "properties": { + "v": { + "type": "string" + } + }, + "required": [ + "v" + ], + "type": "object" + } + }, + "type": "object" + }, + "nick": { + "default": "x", + "type": "string" + }, + "note": { + "type": "string" + }, + "nothing": { + "type": "null" + }, + "one": { + "const": 1, + "type": "number" + }, + "points": { + "items": { + "type": "integer" + }, + "type": "array" + }, + "ratio": { + "default": 0.5, + "type": "number" + }, + "role": { + "anyOf": [ + { + "const": "admin", + "type": "string" + }, + { + "const": "user", + "type": "string" + }, + { + "const": "guest", + "type": "string" + } + ] + }, + "score": { + "maximum": 100, + "minimum": 0, + "type": "number" + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 5, + "minItems": 0, + "type": "array" + }, + "verified": { + "default": false, + "type": "boolean" + }, + "yes": { + "const": true, + "type": "boolean" + } + }, + "required": [ + "active", + "age", + "choice", + "count", + "extra", + "kind", + "level", + "name", + "nested", + "nick", + "nothing", + "one", + "points", + "ratio", + "role", + "score", + "tags", + "verified", + "yes" + ], + "type": "object" + } + }, + "keyed": {}, + "methods": { + "write": { + "args": { + "additionalProperties": false, + "properties": { + "body": { + "type": "string" + }, + "id": { + "type": "string" + } + }, + "required": [ + "body", + "id" + ], + "type": "object" + }, + "result": { + "type": "null" + } + }, + "read": { + "args": { + "type": "string" + }, + "result": { + "type": "string" + } + }, + "dump": { + "args": { + "additionalProperties": false, + "properties": {}, + "type": "object" + }, + "result": {} + } + } + } + } + } + ] + } + }, + "documents": { + "profile": { + "fingerprint": "f345152aff1bb993", + "normal": { + "additionalProperties": false, + "properties": { + "active": { + "type": "boolean" + }, + "age": { + "maximum": 150, + "minimum": 0, + "type": "integer" + }, + "choice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "additionalProperties": false, + "properties": { + "deep": { + "items": { + "additionalProperties": false, + "properties": { + "x": { + "type": "number" + } + }, + "required": [ + "x" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "deep" + ], + "type": "object" + } + ] + }, + "count": { + "default": 0, + "type": "integer" + }, + "extra": { + "patternProperties": { + "^.*$": { + "type": "integer" + } + }, + "type": "object" + }, + "kind": { + "const": "person", + "type": "string" + }, + "level": { + "anyOf": [ + { + "const": 1, + "type": "number" + }, + { + "const": 2, + "type": "number" + }, + { + "const": 3, + "type": "number" + } + ] + }, + "name": { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z ]+$", + "type": "string" + }, + "nested": { + "additionalProperties": false, + "properties": { + "inner": { + "additionalProperties": false, + "properties": { + "v": { + "type": "string" + } + }, + "required": [ + "v" + ], + "type": "object" + } + }, + "type": "object" + }, + "nick": { + "default": "x", + "type": "string" + }, + "note": { + "type": "string" + }, + "nothing": { + "type": "null" + }, + "one": { + "const": 1, + "type": "number" + }, + "points": { + "items": { + "type": "integer" + }, + "type": "array" + }, + "ratio": { + "default": 0.5, + "type": "number" + }, + "role": { + "anyOf": [ + { + "const": "admin", + "type": "string" + }, + { + "const": "user", + "type": "string" + }, + { + "const": "guest", + "type": "string" + } + ] + }, + "score": { + "maximum": 100, + "minimum": 0, + "type": "number" + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 5, + "minItems": 0, + "type": "array" + }, + "verified": { + "default": false, + "type": "boolean" + }, + "yes": { + "const": true, + "type": "boolean" + } + }, + "required": [ + "active", + "age", + "choice", + "count", + "extra", + "kind", + "level", + "name", + "nested", + "nick", + "nothing", + "one", + "points", + "ratio", + "role", + "score", + "tags", + "verified", + "yes" + ], + "type": "object" + } + }, + "notes": "bd44e8ea54b8f283", + "importState": "53d035820123516d" + }, + "tables": { + "rows": { + "fingerprint": "a8a7a46b1d1b5058", + "record": { + "id": { + "type": "str" + }, + "n": { + "type": "int", + "default": 0 + }, + "flag": { + "type": "bool", + "default": false + }, + "label": { + "type": "str", + "optional": true + }, + "data": { + "type": "json", + "version": 1, + "fingerprint": "595acb546ede10d5" + }, + "maybe": { + "type": "json", + "optional": true, + "version": 1, + "fingerprint": "27fc51c99dad9a88" + } + } + }, + "editsV1": { + "fingerprint": "b6a64f82180e9b7f", + "record": { + "id": { + "type": "str" + }, + "file": { + "type": "str" + }, + "at": { + "type": "int" + } + } + }, + "editsV2": { + "fingerprint": "2fbd0130077d5eed", + "record": { + "id": { + "type": "str" + }, + "path": { + "type": "str" + }, + "at": { + "type": "int" + }, + "dir": { + "type": "str", + "default": "" + } + } + }, + "editsV3": { + "fingerprint": "303055ac505946c3", + "record": { + "id": { + "type": "str" + }, + "path": { + "type": "str" + }, + "at": { + "type": "int" + } + } + } + }, + "job": "c280624818bc83ab" + }, + "stored": { + "store": { + "dump": { + "profile": { + "name": "ada", + "nick": "x", + "age": 36, + "count": 3, + "score": 1, + "ratio": 0.5, + "active": true, + "verified": false, + "role": "admin", + "level": 1, + "kind": "person", + "one": 1, + "yes": true, + "tags": [ + "a", + "b", + "c" + ], + "points": [], + "extra": { + "a": 10, + "b": 6, + "c": 9 + }, + "nothing": null, + "choice": { + "deep": [ + { + "x": 3 + } + ] + }, + "nested": { + "inner": { + "v": "third one" + } + } + }, + "notes": { + "a": { + "body": "first note" + }, + "b": { + "body": "second" + } + }, + "rows": [ + { + "id": "a", + "n": 0, + "flag": false, + "label": "first note", + "data": { + "a": 10, + "b": [ + "a" + ] + }, + "maybe": { + "n": 1.5 + } + }, + { + "id": "b", + "n": 0, + "flag": false, + "label": "second", + "data": { + "a": 6, + "b": [ + "b" + ] + } + }, + { + "id": "c", + "n": 0, + "flag": false, + "label": "third one", + "data": { + "a": 9, + "b": [ + "c" + ] + } + } + ], + "edits": [ + { + "id": "a", + "path": "/notes/a.md", + "at": 10 + }, + { + "id": "b", + "path": "/notes/b.md", + "at": 6 + }, + { + "id": "c", + "path": "/notes/c.md", + "at": 9 + } + ], + "runs": 0 + }, + "finished": [ + { + "count": 2 + }, + { + "count": 2 + }, + { + "count": 2 + } + ], + "note": "second" + }, + "archive": { + "dump": { + "profile": { + "name": "ada", + "nick": "x", + "age": 36, + "count": 3, + "score": 1, + "ratio": 0.5, + "active": true, + "verified": false, + "role": "admin", + "level": 1, + "kind": "person", + "one": 1, + "yes": true, + "tags": [ + "a", + "b", + "c" + ], + "points": [], + "extra": { + "a": 10, + "b": 6, + "c": 9 + }, + "nothing": null, + "choice": { + "deep": [ + { + "x": 3 + } + ] + }, + "nested": { + "inner": { + "v": "third one" + } + } + }, + "notes": { + "a": { + "body": "first note" + }, + "b": { + "body": "second" + } + }, + "rows": [ + { + "id": "a", + "n": 0, + "flag": false, + "label": "first note", + "data": { + "a": 10, + "b": [ + "a" + ] + }, + "maybe": { + "n": 1.5 + } + }, + { + "id": "b", + "n": 0, + "flag": false, + "label": "second", + "data": { + "a": 6, + "b": [ + "b" + ] + } + }, + { + "id": "c", + "n": 0, + "flag": false, + "label": "third one", + "data": { + "a": 9, + "b": [ + "c" + ] + } + } + ], + "edits": [ + { + "id": "a", + "path": "/notes/a.md", + "at": 10 + }, + { + "id": "b", + "path": "/notes/b.md", + "at": 6 + }, + { + "id": "c", + "path": "/notes/c.md", + "at": 9 + } + ], + "runs": 0 + }, + "finished": [ + { + "count": 2 + }, + { + "count": 2 + }, + { + "count": 2 + } + ], + "note": "second" + } + } +} diff --git a/packages/chord/test/next/schema/schema-switch.test.ts b/packages/chord/test/next/schema/schema-switch.test.ts new file mode 100644 index 000000000..830593422 --- /dev/null +++ b/packages/chord/test/next/schema/schema-switch.test.ts @@ -0,0 +1,22 @@ +// Tests of the schema switch from TypeBox to pi-schema (decision "A schema package beneath pi-ai and Chord"): the same +// declarations give byte-identical fingerprints and normal forms, and data stored by the code before the switch loads +// after it (`recorded.json` and `stored/` were made with the TypeBox helpers, see `write-stored.ts`). + +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { fingerprints, storedReads } from "./proofs.ts"; + +const recorded = JSON.parse(readFileSync(new URL("./recorded.json", import.meta.url), "utf8")); + +describe("fingerprints", () => { + it("keeps every fingerprint and normal form of contracts, documents, tables, json columns and jobs", () => { + expect(fingerprints()).toEqual(recorded.fingerprints); + }); +}); + +describe("stored data", () => { + it("reads a store and an archive written before the switch: documents, tables, a migrated table, stopped runs", async () => { + expect(await storedReads(context)).toEqual(recorded.stored); + }); +}); diff --git a/packages/chord/test/next/schema/stored/archive.jsonl b/packages/chord/test/next/schema/stored/archive.jsonl new file mode 100644 index 000000000..7fcbd5972 --- /dev/null +++ b/packages/chord/test/next/schema/stored/archive.jsonl @@ -0,0 +1,36 @@ +{"kind":"manifest","format":1,"schema":2,"lastSeq":13,"nextId":12} +{"kind":"table","table":"fixture.switch.owner:fixture.switch.edits","version":1,"state":"ready","columns":{"id":{"type":"str"},"file":{"type":"str"},"at":{"type":"int"}},"key":["id"],"fingerprint":"b6a64f82180e9b7f"} +{"kind":"put","table":"fixture.switch.owner:fixture.switch.edits","row":{"id":"a","file":"/notes/a.md","at":10},"version":1,"columns":{}} +{"kind":"put","table":"fixture.switch.owner:fixture.switch.edits","row":{"id":"b","file":"/notes/b.md","at":6},"version":1,"columns":{}} +{"kind":"put","table":"fixture.switch.owner:fixture.switch.edits","row":{"id":"c","file":"/notes/c.md","at":9},"version":1,"columns":{}} +{"kind":"table","table":"fixture.switch.owner:fixture.switch.rows","version":1,"state":"ready","columns":{"id":{"type":"str"},"n":{"type":"int","default":0},"flag":{"type":"bool","default":false},"label":{"type":"str","optional":true},"data":{"type":"json","version":1,"fingerprint":"595acb546ede10d5"},"maybe":{"type":"json","optional":true,"version":1,"fingerprint":"27fc51c99dad9a88"}},"key":["id"],"fingerprint":"a8a7a46b1d1b5058"} +{"kind":"put","table":"fixture.switch.owner:fixture.switch.rows","row":{"id":"a","n":0,"flag":false,"label":"first note","data":{"a":10,"b":["a"]},"maybe":{"n":1.5}},"version":1,"columns":{"data":1,"maybe":1}} +{"kind":"put","table":"fixture.switch.owner:fixture.switch.rows","row":{"id":"b","n":0,"flag":false,"label":"second","data":{"a":6,"b":["b"]}},"version":1,"columns":{"data":1}} +{"kind":"put","table":"fixture.switch.owner:fixture.switch.rows","row":{"id":"c","n":0,"flag":false,"label":"third one","data":{"a":9,"b":["c"]}},"version":1,"columns":{"data":1}} +{"kind":"index","table":"fixture.switch.owner:fixture.switch.rows","name":"by_n","columns":["n","id"]} +{"kind":"document.create","record":{"id":2,"owner":"fixture.switch.owner","namespace":"chord","kind":"fixture.switch.notes","key":"a","history":"none","createdAt":2,"revision":1,"version":1,"fingerprint":"bd44e8ea54b8f283"}} +{"kind":"document.revision","id":2,"seq":2,"content":{"kind":"base","version":1,"value":{"body":"first note"}}} +{"kind":"document.create","record":{"id":3,"owner":"fixture.switch.owner","namespace":"chord","kind":"fixture.switch.profile","history":"none","createdAt":2,"revision":3,"version":1,"fingerprint":"f345152aff1bb993"}} +{"kind":"document.revision","id":3,"seq":2,"content":{"kind":"base","version":1,"value":{"name":"ada","nick":"x","age":36,"count":1,"score":1,"ratio":0.5,"active":true,"verified":false,"role":"admin","level":1,"kind":"person","one":1,"yes":true,"tags":["a"],"points":[],"extra":{"a":10},"nothing":null,"choice":{"deep":[{"x":1}]},"nested":{"inner":{"v":"first note"}}}}} +{"kind":"document.revision","id":3,"seq":5,"content":{"kind":"delta","version":1,"ops":[["s",["count"],2],["s",["choice"],{"deep":[{"x":2}]}],["s",["nested"],{"inner":{"v":"second"}}],["p",["tags"],1,0,["b"]],["s",["extra","b"],6]]}} +{"kind":"document.revision","id":3,"seq":9,"content":{"kind":"delta","version":1,"ops":[["s",["count"],3],["s",["choice"],{"deep":[{"x":3}]}],["s",["nested"],{"inner":{"v":"third one"}}],["p",["tags"],2,0,["c"]],["s",["extra","c"],9]]}} +{"kind":"document.create","record":{"id":4,"owner":"fixture.switch.owner","namespace":"chord.jobs","kind":"fixture.switch.import","key":"01a1943d-176c-71f3-a32c-b9cbc67247a3","history":"none","createdAt":3,"revision":2,"version":1,"fingerprint":"chord.job-run.1"}} +{"kind":"document.revision","id":4,"seq":4,"content":{"kind":"base","version":1,"value":{"version":1,"fingerprint":"c280624818bc83ab","args":{"path":"a"},"phase":"running","restarts":0,"finishFailures":0}}} +{"kind":"document.create","record":{"id":5,"owner":"fixture.switch.owner","namespace":"chord.job-states","kind":"fixture.switch.import","key":"01a1943d-176c-71f3-a32c-b9cbc67247a3","history":"none","createdAt":3,"revision":2,"version":1,"fingerprint":"53d035820123516d"}} +{"kind":"document.revision","id":5,"seq":3,"content":{"kind":"base","version":1,"value":{"done":0}}} +{"kind":"document.revision","id":5,"seq":6,"content":{"kind":"delta","version":1,"ops":[["s",["done"],1]]}} +{"kind":"document.create","record":{"id":6,"owner":"fixture.switch.owner","namespace":"chord","kind":"fixture.switch.notes","key":"b","history":"none","createdAt":5,"revision":1,"version":1,"fingerprint":"bd44e8ea54b8f283"}} +{"kind":"document.revision","id":6,"seq":5,"content":{"kind":"base","version":1,"value":{"body":"second"}}} +{"kind":"document.create","record":{"id":7,"owner":"fixture.switch.owner","namespace":"chord.jobs","kind":"fixture.switch.import","key":"01a1943d-176e-71f3-a32c-b9cc7f5426c6","history":"none","createdAt":7,"revision":2,"version":1,"fingerprint":"chord.job-run.1"}} +{"kind":"document.revision","id":7,"seq":8,"content":{"kind":"base","version":1,"value":{"version":1,"fingerprint":"c280624818bc83ab","args":{"path":"b"},"phase":"running","restarts":0,"finishFailures":0}}} +{"kind":"document.create","record":{"id":8,"owner":"fixture.switch.owner","namespace":"chord.job-states","kind":"fixture.switch.import","key":"01a1943d-176e-71f3-a32c-b9cc7f5426c6","history":"none","createdAt":7,"revision":2,"version":1,"fingerprint":"53d035820123516d"}} +{"kind":"document.revision","id":8,"seq":7,"content":{"kind":"base","version":1,"value":{"done":0}}} +{"kind":"document.revision","id":8,"seq":10,"content":{"kind":"delta","version":1,"ops":[["s",["done"],1]]}} +{"kind":"document.create","record":{"id":9,"owner":"fixture.switch.owner","namespace":"chord","kind":"fixture.switch.notes","key":"c","history":"none","createdAt":9,"revision":1,"version":1,"fingerprint":"bd44e8ea54b8f283"}} +{"kind":"document.revision","id":9,"seq":9,"content":{"kind":"base","version":1,"value":{"body":"third one"}}} +{"kind":"document.create","record":{"id":10,"owner":"fixture.switch.owner","namespace":"chord.jobs","kind":"fixture.switch.import","key":"01a1943d-176f-71f3-a32c-b9cf6bb98dc7","history":"none","createdAt":11,"revision":2,"version":1,"fingerprint":"chord.job-run.1"}} +{"kind":"document.revision","id":10,"seq":12,"content":{"kind":"base","version":1,"value":{"version":1,"fingerprint":"c280624818bc83ab","args":{"path":"c"},"phase":"running","restarts":0,"finishFailures":0}}} +{"kind":"document.create","record":{"id":11,"owner":"fixture.switch.owner","namespace":"chord.job-states","kind":"fixture.switch.import","key":"01a1943d-176f-71f3-a32c-b9cf6bb98dc7","history":"none","createdAt":11,"revision":2,"version":1,"fingerprint":"53d035820123516d"}} +{"kind":"document.revision","id":11,"seq":11,"content":{"kind":"base","version":1,"value":{"done":0}}} +{"kind":"document.revision","id":11,"seq":13,"content":{"kind":"delta","version":1,"ops":[["s",["done"],1]]}} +{"kind":"end","writes":34} diff --git a/packages/chord/test/next/schema/stored/store.db b/packages/chord/test/next/schema/stored/store.db new file mode 100644 index 0000000000000000000000000000000000000000..f9b1e70836f843f0967cdb76f7890de36f441247 GIT binary patch literal 57344 zcmeI5U2Ggz6@X`dXJ`MWR1GVHO2$*%*ktYO|Bq3iX|ips1gCWz(kfcq`P+Eg_0BRg zPHb8GV5bTaFBL+9!V3tgZ-@t85EA04C{=}2UO+$*NJt=r1QJqJiSoj^cV@@4>s@E< zEVOmb${x?&-*dim?%kd7xp%HUf4OOCLPIy!B}>RNk1&2e^OPVk4C9C2B>eVX2{;M% z-oRg<=X}(upPB#qqoiUY&|>&F|_RH84+~2v+MZOt%jD0oq{opSk3tvd!*b~_O za)M2*uKMn5Taw(;8qG~>!_el<+f7SZo7Zo*HDh5QQB#|iIgmEGVDkKO?c7RDSULCf z<(eRlT~8Ed#HK0=E43F^grzI+d+qY&G-;rv?N5~~QCM7BsXbd;b_)yVFV@aqf@VnO z)3ufBwc3&}SIuO~*=jXcER+kGYBejITe=_&CeA`fpS`ue$+Ym?^5U1yEx#ySs=a6n zI6Es`U0Gf{zmgWNFRom?a&1LezH)u>!p;kEHg&V^+u3TiRc&(|y$$`gxgc+?w#O}; zaM&&`U8ub<9$U8$QMj@+eidSkwg^stWlcBKRYSYgG@E+cT$LqLGY1l5 zecm{bY1be=H;^IBsJgPTuC=YzrkWN^?W?oG^^3LTn(zwrESwcik-koKIa{1*k2t-G z=Blaa9nEgoO^gl^$IY~B?1|%M2s3mlOABpjT{An9qNRnVI=k~&giU=Oa9kPMzoe>$ zW;#IJwzvZ&aO#XH1A&niUeUJFf?{ZrrKziuHM?_`WmD7BzTLBQB-#Y@4NKNHz2}j> z5#`MhW}+OtfVw+Z+-u)EpiXPTO#(|gA;HS%zAvRS6f3!bcY*Ibt&DB=(*5Ggm zJM^wCEk1v(2BMOWjA8W->yWb)fjwjm&1xFdHHF2i^b`Be#UQot-Df=(b_E^iKQzR` zZdu2|9$)j&vvJa)JB+hCn1^NVx4YdM47Op#*P5oK8v|KAPw8b+XKCwg?G!jm3x=y{ zkuyWP4da-6(Q9vP+c!0%V>Acd)a!yf z;Q*a%cCL6#Nl%@L&pr=@vQ*pV>C$mSkG1qoErE_KJ9)Ly(sg5B-9Gy~<6~3VtZ%pM!9sjhgHO9PZQa4+P4N%Gh1&I%bai}cJ*uFZY?2Mqk+3ke_rB!C2v01`j~NB{{S0VIF~kN^@m zb_9aH0ORx9*Z=P`{QJjFK~x9{AOR$R1dsp{Kmter2_OL^fCP}hLrxo|}nB$A9r zfrLL1V-m6CBM&(gR38Z-0VIF~kN^@u0!RP}Ac2R6z-}P8dZ#+h$~n3!b6cdL-{Qg{ z)>)=SCk|U&$fiXIskbz!4{MCnEio-BdTV37ZHf!qWa%Fqt*s8pH!VZ#rtQ_3`%+=S zrbSrJAZ@gOL52__YXtXb$hzJF_1q#2sby+V%w7(-SB~&a zi+YDFijZ33f@N%wVsxG0Ua6N&9ad%b8L*#hv09WAxmYM^s+P^DMPf61>EOOb`ujSl z(OkKq6thaTs!D2As#NZ0Ua2CLr9w&0s#&?1DOUQ-E0P7T#4(5>#5ua%L`-0!(Ki_W zM~QDF{*t`QbMa5df0p=S^o`i-56{p?O_2Z+Kmter2_OL^fCN5L1h%3HrZ`o;b^^AX z@^Pn!U7gV7$n!7j^5E7d&Gn9MSedMptrqg?T((?l%$2i^{G61}DRXjFX_REST2YlU zSwbEQxDCx28}0dQ{9DXsanA0V5~0oxQcu3bJlRuzH1!=OGI>S^%DIoxa~w#JD)PiL zM+<7G1Af!iVV44hsLsi1p`cYXsaTLJja((a93U-L6Sl99cDR8SJBChnYa&hh1C4y4 zm@Vd{Mk5QpUCnO=CuvEg-;%OYD(4DiDNp(i3|f-EF`gwulegr7a_+Ze_~@3DiRz(2 zOTrX-wP(pg_=eY#gNA$#;`NW}ZW8s4K_SWi|D*AD8UF3$cM{*@znOd|@$2}zd?C4z zcR`T)W7TgQ?Zc>i_H2%Bz$@mXq*;sb; zFvA=qfCP{L5{1K|3`1SnqT+6-vQl7c9xJpk#(d6*tYbXQXBeo?%0hy0Q0| zzfuvR@dp<{@rnTdiEl#?^H?G%wg|Bai_p>9Diz^m*yl@e*;s-;mf*$>(G=4+L>2x4 z2(i}LdVOaK43Xk?^8v1}graXiaCSYNuS0QK2QEUJ3aPaXA<9|@;&QjeO>rULt#{or zZ9op}NttFFo)1K#n{R&XIhno?pT_oL*xRdfPw)n`-K`g}aM`Xm59!A->+i&E0yZH^rx(LTz1b`WT_BFEV( z$~uJD_i@|#jBTj7K&Y>Yo}qFsz7dAXjbkWNgGW-RL#&jgR+5KU3DVLfaHQUdW+#AN z_711Rq7MaZhCTE!zs*zs2|5|xMGgHi@i5mo_KUFnjX3rTh5O*|+3#^`zcb-bpZ$6` zcc4Zos)cE@4PZP=zdDfcrOVWR_WIZ?sw77i=ach^`D3g$s)huR01`j~ zNB{{S0VIF~kif^6K!c^R{-cAY$<`1Lx<5lK_7Y2@{SWTu=;8jcXqlz){-Zk>9`Mgi zRR57N`H>0fKe7Y%Nczj6@I>_=6`3ChO-TQd9Rw?2e~A4b+x|}Qzdvv!aFX9oum6*G z8U9cF&-qKq-}0^GzhM`^*ZJw>k3YWC0ZNVpkN^@u0!RP}AOR$R1dsp{Kms340;fa8 zxdSeGJvVP+vFT88>ON)7XjBLlQwNmdxbeG;M8@foW$)8vI6Qurq0l&827~wMG7z{= z0q6Rkf0N-q;NRol=Kse3lm9dS8~zXculP418DB^M2_OL^fCP{L5G5j`sC`fQSG8zr+6!{@@D< zAOR$R1dsp{Kmter2_OL^fCP{L68N7Hn2z$1$jLJtJVa7)xAvsl0#?(a44Va#tvkCS zDfKB@O13vbq(xwJLs|s(jqGMfk*O?qlH;VkZT_4M2Ay)S4Pkd%Y{Qm=^TjTS{}1s0 zV)*~?@53K_Apsndj_$xf0C ONntxla*`xSivAa5Xn)TD literal 0 HcmV?d00001 diff --git a/packages/chord/test/next/schema/write-stored.ts b/packages/chord/test/next/schema/write-stored.ts new file mode 100644 index 000000000..e40248d32 --- /dev/null +++ b/packages/chord/test/next/schema/write-stored.ts @@ -0,0 +1,40 @@ +// Writes the stored data of the schema switch's proof (`stored/`): a SQLite store and its export archive, made by the +// code before the switch (TypeBox helpers). Kept as it was written; rerun only to make a new fixture on purpose. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/schema/write-stored.ts + +import { createWriteStream } from "node:fs"; +import { copyFile, mkdir, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pipeline } from "node:stream/promises"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { encodeJsonl } from "../../../src/next/store/index.ts"; +import { EditsV1, Owner } from "./declarations.ts"; +import { ownerHost } from "./owner.ts"; + +const stored = fileURLToPath(new URL("./stored/", import.meta.url)); +const directory = await mkdtemp(join(tmpdir(), "chord-schema-switch-")); +const path = join(directory, "store.db"); +try { + const { store, host } = await ownerHost(path, EditsV1, "wait", context); + const owner = await host.get(Owner, {}, context); + for (const [id, body] of [ + ["a", "first note"], + ["b", "second"], + ["c", "third one"], + ] as const) { + await owner.write({ id, body }, context); + } + await new Promise((resolve) => setTimeout(resolve, 200)); + await mkdir(stored, { recursive: true }); + await pipeline(encodeJsonl(store.export(context)), createWriteStream(join(stored, "archive.jsonl"))); + // A clean close: the waiting runs are stored `stopped`, resumed by the next host. + await host.close(context); + await store.close(context); + await copyFile(path, join(stored, "store.db")); + console.log("written to", stored); +} finally { + await rm(directory, { recursive: true, force: true }); +} diff --git a/packages/chord/test/placement.test.ts b/packages/chord/test/placement.test.ts index 869bfa655..3e3a7bd8e 100644 --- a/packages/chord/test/placement.test.ts +++ b/packages/chord/test/placement.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { num, obj, opt, str } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { bind, @@ -22,17 +22,17 @@ const tick = () => new Promise((resolve) => setImmediate(resolve)); /** Lives on `session:` + its input's `session`. */ const Note = remote("t.note", { version: 1, - input: Type.Object({ session: Type.String(), id: Type.String() }), + input: obj({ session: str(), id: str() }, { open: true }), host: hostFamily("session:", "session"), - variants: { main: { methods: { where: method(undefined, Type.String()) } } }, + variants: { main: { methods: { where: method(undefined, str()) } } }, }); /** Lives on `server`. */ const Catalog = remote("t.catalog", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { methods: { where: method(undefined, Type.String()) } } }, + variants: { main: { methods: { where: method(undefined, str()) } } }, }); /** A server with two session hosts as children; every host serves what it may and reports its name. */ @@ -126,9 +126,9 @@ describe("placement", () => { it("the host comes from the normalized input: unknown fields and defaults do not change it", async () => { const Doc = remote("t.doc-default", { version: 1, - input: Type.Object({ session: Type.String({ default: "S1" }) }), + input: obj({ session: str({ default: "S1" }) }, { open: true }), host: hostFamily("session:", "session"), - variants: { main: { methods: { where: method(undefined, Type.String()) } } }, + variants: { main: { methods: { where: method(undefined, str()) } } }, }); const { server, s1, close } = cluster(); s1.serve(Doc, () => ({ variant: "main", where: () => "session:S1" })); @@ -143,7 +143,7 @@ describe("placement", () => { const at = (host: string) => remote("t.moved", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed(host), variants: { main: { methods: { ping: method() } } }, }); @@ -155,7 +155,7 @@ describe("placement", () => { api.serve( remote("t.moved", { version, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed(place), variants: { main: { methods: { ping: method() } } }, }), @@ -170,7 +170,7 @@ describe("placement", () => { // A new version that `accepts` version 1 but lives elsewhere: no adapter makes version 1 consumers bind. const moved = remote("t.moved", { version: 2, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("session:S1"), variants: { main: { methods: { ping: method() } } }, }); @@ -182,7 +182,7 @@ describe("placement", () => { }); it("types: the family's field is a required string of the input, and remote contracts need a host", () => { - const input = Type.Object({ session: Type.String(), count: Type.Number(), note: Type.Optional(Type.String()) }); + const input = obj({ session: str(), count: num(), note: opt(str()) }, { open: true }); const variants = { main: {} }; remote("t.ok", { version: 1, input, host: hostFamily("session:", "session"), variants }); // @ts-expect-error: no such field diff --git a/packages/chord/test/principals.test.ts b/packages/chord/test/principals.test.ts index 4aef22cb3..aab011dab 100644 --- a/packages/chord/test/principals.test.ts +++ b/packages/chord/test/principals.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { obj, str } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { type Ctx, @@ -26,8 +26,8 @@ const principalKey = (principal: Principal): string => const Who = local("t.who", { version: 1, - input: Type.Object({}), - members: { who: method(undefined, Type.String()) }, + input: obj({}, { open: true }), + members: { who: method(undefined, str()) }, }); describe("principals", () => { @@ -86,9 +86,9 @@ describe("principals", () => { it("a child's parent acts as the principal the child gave `connect`; relayed callers keep theirs", async () => { const Echo = remote("t.echo", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("worker"), - variants: { main: { methods: { who: method(undefined, Type.String()) } } }, + variants: { main: { methods: { who: method(undefined, str()) } } }, }); const server = createHost({ name: "server", self: { kind: "host" }, principalKey }); const [down, up] = memoryLinkPair(); diff --git a/packages/chord/test/provider-cost.test.ts b/packages/chord/test/provider-cost.test.ts index 470c74b20..50a8bf533 100644 --- a/packages/chord/test/provider-cost.test.ts +++ b/packages/chord/test/provider-cost.test.ts @@ -1,7 +1,7 @@ // Entry 8: what a change costs the provider. The diff is computed once per change; each feed (one per connection and // member) encodes it with its own stateful encoder and its link serializes it. import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { arr, obj, str } from "@earendil-works/pi-schema"; import { describe, it, vi } from "vitest"; import type * as delta from "../src/delta/index.ts"; import { type Ctx, createHost, hostNamed, memoryLinkPair, method, remote } from "../src/index.ts"; @@ -30,16 +30,16 @@ const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); const List = remote("t.e8-cost", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { state: { items: Type.Array(Type.String()) }, methods: { add: method(Type.String()) } } }, + variants: { main: { state: { items: arr(str()) }, methods: { add: method(str()) } } }, }); describe("provider cost", () => { it("one change to K clients: K encodes (one per feed), one frame per link, the same ops on every link", async () => { const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); server.serve(List, (_input, _principal, self) => { - const items = self.state("items", Type.Array(Type.String()), []); + const items = self.state("items", arr(str()), []); return { variant: "main", items, diff --git a/packages/chord/test/readiness.test.ts b/packages/chord/test/readiness.test.ts index a781a6e34..b34b2540c 100644 --- a/packages/chord/test/readiness.test.ts +++ b/packages/chord/test/readiness.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { obj } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { createHost, @@ -15,8 +15,8 @@ import { const tick = () => new Promise((resolve) => setImmediate(resolve)); const unexpectedError = (error: unknown) => assert.fail(String(error)); -const Eager = local("ready.eager", { version: 1, input: Type.Object({}) }); -const Lazy = local("ready.lazy", { version: 1, input: Type.Object({}) }); +const Eager = local("ready.eager", { version: 1, input: obj({}, { open: true }) }); +const Lazy = local("ready.lazy", { version: 1, input: obj({}, { open: true }) }); // Startup must not report ready while an eager object has failed. describe("host: eager readiness", () => { @@ -162,7 +162,7 @@ describe("host: eager readiness", () => { it("ready includes direct host serves, but not later serves", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); - const First = local("ready.first", { version: 1, input: Type.Object({}) }); + const First = local("ready.first", { version: 1, input: obj({}, { open: true }) }); let created = 0; try { host.serve( @@ -201,7 +201,7 @@ describe("host: eager readiness", () => { }); const Status = remote("ready.status", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { default: {} }, }); diff --git a/packages/chord/test/reload.test.ts b/packages/chord/test/reload.test.ts index f2931f0c9..b9998b316 100644 --- a/packages/chord/test/reload.test.ts +++ b/packages/chord/test/reload.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { num, obj, opt, type Schema, str, unknown } from "@earendil-works/pi-schema"; import { describe, it, onTestFinished, vi } from "vitest"; import { type Ctx, @@ -30,15 +30,15 @@ function latch() { const Greeter = local("t.greeter", { version: 1, - input: Type.Object({ id: Type.Optional(Type.String({ default: "a" })) }), - members: { greet: method(Type.String(), Type.String()), count: Type.Unknown() }, + input: obj({ id: opt(str({ default: "a" })) }, { open: true }), + members: { greet: method(str(), str()), count: unknown() }, }); -const CountV1 = Type.Object({ n: Type.Number() }); +const CountV1 = obj({ n: num() }, { open: true }); interface GreeterSpec { readonly greeting: string; - readonly schema?: typeof CountV1 | ReturnType; + readonly schema?: typeof CountV1 | Schema; readonly initial?: object; readonly bump?: (draft: Record) => void; readonly options?: ServeOptions; @@ -129,7 +129,7 @@ describe("reload", () => { const writing = greeterFacet({ greeting: "hi", extra: (self) => { - const probe = self.state("probe", Type.Object({}), {}); + const probe = self.state("probe", obj({}, { open: true }), {}); probe.replace(captured!, {}); }, }); @@ -164,7 +164,7 @@ describe("reload", () => { await host.hold(Greeter, {}).ready; const changed = greeterFacet({ greeting: "hi", - schema: Type.Object({ total: Type.Number() }), + schema: obj({ total: num() }, { open: true }), initial: { total: 0 }, }); await assert.rejects( @@ -191,7 +191,7 @@ describe("reload", () => { const froms: number[] = []; const v2 = greeterFacet({ greeting: "hi", - schema: Type.Object({ total: Type.Number() }), + schema: obj({ total: num() }, { open: true }), initial: { total: 0 }, bump: (draft) => { draft.total = (draft.total ?? 0) + 1; @@ -218,16 +218,16 @@ describe("reload", () => { const host = quiet(); const Pair = local("t.pair", { version: 1, - input: Type.Object({}), - members: { a: Type.Unknown(), b: Type.Unknown(), bump: method() }, + input: obj({}, { open: true }), + members: { a: unknown(), b: unknown(), bump: method() }, }); const migrated: string[] = []; const facet = (version: number): Facet => ({ name: "pair", setup: (api) => api.serve(Pair, (_input, _principal, self) => { - const a = self.state("a", Type.Number(), 1); - const b = self.state("b", Type.Number(), 1, { + const a = self.state("a", num(), 1); + const b = self.state("b", num(), 1, { version, migrate: (old, from) => { migrated.push(`b ${from}->${version}`); @@ -294,7 +294,7 @@ describe("reload", () => { await new Promise((resolve) => ctx.signal.addEventListener("abort", () => resolve())); log.push(`stop ${greeting}`); }); - const count = self.state("count", Type.Number(), 0, { version }); + const count = self.state("count", num(), 0, { version }); return { count, greet: (name: string) => `${greeting} ${name}` }; }), }); @@ -340,7 +340,7 @@ describe("reload", () => { open: () => void log.push(`open cache ${rev}`), close: () => void log.push(`close cache ${rev}`), }); - const count = self.state("count", Type.Number(), 0); + const count = self.state("count", num(), 0); return { count, greet: (name: string) => `${lock.value} ${name}` }; }), }); @@ -523,21 +523,21 @@ describe("reload", () => { it("checks contracts at reload: forgotten bumps, live consumers, accepts adapters", async () => { const DocV1 = remote("t.rdoc", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { methods: { read: method(undefined, Type.Object({ text: Type.String() })) } } }, + variants: { main: { methods: { read: method(undefined, obj({ text: str() }, { open: true })) } } }, }); const DocBroken = remote("t.rdoc", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { methods: { read: method(undefined, Type.Object({ text: Type.Number() })) } } }, + variants: { main: { methods: { read: method(undefined, obj({ text: num() }, { open: true })) } } }, }); const DocV2 = remote("t.rdoc", { version: 2, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { methods: { fetch: method(undefined, Type.Object({ body: Type.String() })) } } }, + variants: { main: { methods: { fetch: method(undefined, obj({ body: str() }, { open: true })) } } }, }); const host = quiet(); host.install({ @@ -590,14 +590,14 @@ describe("reload", () => { const log: string[] = []; const Sum = local("t.sum", { version: 1, - input: Type.Object({}), - members: { sum: Type.Object({ v: Type.Number() }) }, + input: obj({}, { open: true }), + members: { sum: obj({ v: num() }, { open: true }) }, }); const facet = (factor: number): Facet => ({ name: "sum", setup: (api) => api.serve(Sum, (_input, _principal, self) => { - const sum = self.state("sum", Type.Object({ v: Type.Number() }), { v: 0 }); + const sum = self.state("sum", obj({ v: num() }, { open: true }), { v: 0 }); self.on( "add", events, @@ -632,7 +632,7 @@ describe("reload", () => { it("keeps registry tables per code version", async () => { const host = quiet(); const Routes = defineRegistry("t.reload-routes"); - const Router = local("t.router", { version: 1, input: Type.Object({}) }); + const Router = local("t.router", { version: 1, input: obj({}, { open: true }) }); const facet = (value: string): Facet => ({ name: "router", setup: (api) => @@ -672,8 +672,8 @@ describe("reload", () => { await ref.ready; const Clock = local("t.clock", { version: 1, - input: Type.Object({}), - members: { now: method(undefined, Type.Number()) }, + input: obj({}, { open: true }), + members: { now: method(undefined, num()) }, }); const clockFacet: Facet = { name: "clock", setup: (api) => api.serve(Clock, () => ({ now: () => 42 })) }; const consumer = (fail: boolean): Facet => ({ diff --git a/packages/chord/test/requests.test.ts b/packages/chord/test/requests.test.ts index fef1f2a30..5e3cc5abb 100644 --- a/packages/chord/test/requests.test.ts +++ b/packages/chord/test/requests.test.ts @@ -1,6 +1,6 @@ // Regression tests for resolved application requests (#1-#10), numbered as in their test names. import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { arr, num, obj, str, unknown } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { applyImmutable } from "../src/delta/index.ts"; import { @@ -36,19 +36,19 @@ function latch() { return { promise, open }; } -const View = Type.Object({ text: Type.String() }); +const View = obj({ text: str() }, { open: true }); describe("application requests", () => { it("#1 a held object's state member can be returned as this object's state member", async () => { const Conv = remote("t.conv", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { state: { view: View }, methods: { set: method(Type.String()) } } }, + variants: { main: { state: { view: View }, methods: { set: method(str()) } } }, }); const Chat = remote("t.chat", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { viewer: { state: { view: View } } }, }); @@ -95,11 +95,11 @@ describe("application requests", () => { it("#2 facets reach their own host through FacetApi.root and FacetApi.registry", async () => { const Config = defineRegistry("t.config"); - const Settings = local("t.settings", { version: 1, input: Type.Object({}) }); + const Settings = local("t.settings", { version: 1, input: obj({}, { open: true }) }); const Reader = local("t.reader", { version: 1, - input: Type.Object({}), - members: { read: method(undefined, Type.String()) }, + input: obj({}, { open: true }), + members: { read: method(undefined, str()) }, }); const captured: FacetApi[] = []; const facet = (value: string): Facet => ({ @@ -135,8 +135,8 @@ describe("application requests", () => { it("#3 a listener keeps its subscription across reloads when resource and select are unchanged", async () => { const Watcher = local("t.watcher", { version: 1, - input: Type.Object({}), - members: { seen: Type.Array(Type.String()) }, + input: obj({}, { open: true }), + members: { seen: arr(str()) }, }); // A source that accepts one subscriber at a time (like a Durable watch). let subscribers = 0; @@ -155,7 +155,7 @@ describe("application requests", () => { name: "watcher", setup: (api) => api.serve(Watcher, (_input, _principal, self) => { - const seen = self.state("seen", Type.Array(Type.String()), []); + const seen = self.state("seen", arr(str()), []); const watchResource = self.resource("watch", { version: 1, open: () => single }); // A new adapter object per call, and a new arrow per factory run. self.on( @@ -181,7 +181,7 @@ describe("application requests", () => { }); it("#3 a changed select resubscribes", async () => { - const Watcher = local("t.watcher2", { version: 1, input: Type.Object({}) }); + const Watcher = local("t.watcher2", { version: 1, input: obj({}, { open: true }) }); let subscribes = 0; const source = { subscribe: () => { @@ -210,10 +210,10 @@ describe("application requests", () => { it("#4 replace diffs a new revision that shares structure; unrelated values replace", async () => { const Mirror = local("t.mirror", { version: 1, - input: Type.Object({}), - members: { doc: Type.Unknown(), set: method(Type.Unknown()) }, + input: obj({}, { open: true }), + members: { doc: unknown(), set: method(unknown()) }, }); - const Doc = Type.Object({ entries: Type.Array(Type.String()), status: Type.String() }); + const Doc = obj({ entries: arr(str()), status: str() }, { open: true }); let lastOptions: { diff?: boolean } | undefined; const host = createHost({ self: { kind: "host" }, onError: () => {} }); host.serve(Mirror, (_input, _principal, self) => { @@ -256,9 +256,9 @@ describe("application requests", () => { it("#5 roots of a child that connected behind the server are admitted in the server's code version", async () => { const Echo = remote("t.echo", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { methods: { codeVersion: method(undefined, Type.Number()) } } }, + variants: { main: { methods: { codeVersion: method(undefined, num()) } } }, }); const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); const echoFacet: Facet = { @@ -293,7 +293,7 @@ describe("application requests", () => { it("#6 self.expose: an opaque member backed by a resource, readable once live", async () => { const Store = local("t.store", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { db: opaque<{ readonly queries: string[] }>("t.db@1") }, }); const gate = latch(); @@ -315,7 +315,7 @@ describe("application requests", () => { await ref.ready; assert.equal(ref.db, native); - const Other = local("t.other", { version: 1, input: Type.Object({}) }); + const Other = local("t.other", { version: 1, input: obj({}, { open: true }) }); host.serve(Other, (_input, _principal, self) => { self.expose({ key: "x", value: 1 }); return {}; @@ -326,9 +326,9 @@ describe("application requests", () => { it("#7 a cluster reload reports what each child prepared", async () => { const Greeter = remote("t.greeter", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("session:S1"), - variants: { main: { methods: { greet: method(Type.String(), Type.String()) } } }, + variants: { main: { methods: { greet: method(str(), str()) } } }, }); const greeter = (text: string): Facet => ({ name: "greeter", @@ -362,13 +362,13 @@ describe("application requests", () => { it("#8 a held remote reference keeps its state without listeners; a later subscriber sees the current value", async () => { const Counter = remote("t.count8", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { state: { n: Type.Number() }, methods: { set: method(Type.Number()) } } }, + variants: { main: { state: { n: num() }, methods: { set: method(num()) } } }, }); const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); server.serve(Counter, (_input, _principal, self) => { - const n = self.state("n", Type.Number(), 0); + const n = self.state("n", num(), 0); return { variant: "main", n, @@ -404,7 +404,7 @@ describe("application requests", () => { it("#9 references report status changes: live, loading while a link reconnects, gone, released", async () => { const Thing = remote("t.thing9", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { main: {} }, }); @@ -449,9 +449,9 @@ describe("application requests", () => { const log: string[] = []; const Remote = remote("t.remote10", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { methods: { note: method(Type.String()) } } }, + variants: { main: { methods: { note: method(str()) } } }, }); const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); server.serve(Remote, () => ({ @@ -462,10 +462,10 @@ describe("application requests", () => { })); const Proc = local("t.proc", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { tracker: opaque>("t.tracker@1") }, }); - const Chat = local("t.chat10", { version: 1, input: Type.Object({}) }); + const Chat = local("t.chat10", { version: 1, input: obj({}, { open: true }) }); const [down, up] = memoryLinkPair(); server.accept(down, { principal: { worker: "w" }, kind: "host", name: "w" }); const worker = createHost({ diff --git a/packages/chord/test/review.test.ts b/packages/chord/test/review.test.ts index 554a24097..418d592ea 100644 --- a/packages/chord/test/review.test.ts +++ b/packages/chord/test/review.test.ts @@ -1,7 +1,6 @@ // Regression tests for Sol's phase 1a review (findings #1-#11) and the requested race tests. import assert from "node:assert/strict"; -import { Type } from "typebox"; -import { Value } from "typebox/value"; +import { arr, bool, compile, num, obj, opt, str, unknown } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { type Ctx, @@ -18,6 +17,7 @@ import { } from "../src/index.ts"; import { fits } from "../src/schema.ts"; import { CountingLink } from "./fixtures/counting.ts"; +import { typedLit } from "./fixtures/schemas.ts"; const tick = () => new Promise((resolve) => setImmediate(resolve)); const ticks = async (n = 10) => { @@ -51,8 +51,8 @@ function emitter(options: { failSubscribe?: boolean } = {}) { const Item = local("t.item", { version: 1, - input: Type.Object({ id: Type.String() }), - members: { version: method(undefined, Type.String()) }, + input: obj({ id: str() }, { open: true }), + members: { version: method(undefined, str()) }, }); describe("review: reload commit", () => { @@ -82,15 +82,15 @@ describe("review: reload commit", () => { const host = createHost({ self: { kind: "host" }, dev: true, onError: () => {} }); const V1 = remote("t.counted", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { state: { count: Type.Object({ n: Type.Number() }) } } }, + variants: { main: { state: { count: obj({ n: num() }, { open: true }) } } }, }); const V2 = remote("t.counted", { version: 2, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { state: { count: Type.Object({ total: Type.Number() }) } } }, + variants: { main: { state: { count: obj({ total: num() }, { open: true }) } } }, }); host.install({ name: "counted", @@ -99,7 +99,7 @@ describe("review: reload commit", () => { V1, (_input, _principal, self) => ({ variant: "main", - count: self.state("count", Type.Object({ n: Type.Number() }), { n: 3 }), + count: self.state("count", obj({ n: num() }, { open: true }), { n: 3 }), }), { linger: 60_000 }, ), @@ -117,7 +117,7 @@ describe("review: reload commit", () => { variant: "main", count: self.state( "count", - Type.Object({ total: Type.Number() }), + obj({ total: num() }, { open: true }), { total: 0 }, { version: 2, migrate: (old) => ({ total: (old as { n: number }).n }) }, ), @@ -160,8 +160,8 @@ describe("review: reload commit", () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const Other = local("t.other", { version: 1, - input: Type.Object({}), - members: { version: method(undefined, Type.String()) }, + input: obj({}, { open: true }), + members: { version: method(undefined, str()) }, }); const facet = (version: string): Facet => ({ name: "pair", @@ -184,8 +184,8 @@ describe("review: reload commit", () => { describe("review: leases, invocations, activation", () => { const Store = local("t.store", { version: 1, - input: Type.Object({}), - members: { count: Type.Unknown() }, + input: obj({}, { open: true }), + members: { count: unknown() }, }); it("#4 a running listener holds a lease: the gate waits for it before migrating", async () => { @@ -197,7 +197,7 @@ describe("review: leases, invocations, activation", () => { name: "store", setup: (api) => api.serve(Store, (_input, _principal, self) => { - const count = self.state("count", Type.Object({ n: Type.Number() }), { n: 1 }); + const count = self.state("count", obj({ n: num() }, { open: true }), { n: 1 }); self.on("slow", events, async () => { await go.promise; seen.push(count.value); @@ -218,7 +218,7 @@ describe("review: leases, invocations, activation", () => { api.serve(Store, (_input, _principal, self) => ({ count: self.state( "count", - Type.Object({ total: Type.Number() }), + obj({ total: num() }, { open: true }), { total: 0 }, { version: 2, migrate: (old) => ({ total: (old as { n: number }).n }) }, ), @@ -238,8 +238,8 @@ describe("review: leases, invocations, activation", () => { it("#5 disposal waits for running methods before closing resources (released and forced)", async () => { const Worker = local("t.busy", { version: 1, - input: Type.Object({ id: Type.String() }), - members: { work: method(undefined, Type.Boolean()) }, + input: obj({ id: str() }, { open: true }), + members: { work: method(undefined, bool()) }, }); const log: string[] = []; let go = latch(); @@ -291,7 +291,9 @@ describe("review: leases, invocations, activation", () => { const host = createHost({ self: { kind: "host" }, onError: (error) => errors.push(error) }); const Routes = defineRegistry("t.review-routes"); const log: string[] = []; - const types = ["t.eager-a", "t.eager-b"].map((name) => local(name, { version: 1, input: Type.Object({}) })); + const types = ["t.eager-a", "t.eager-b"].map((name) => + local(name, { version: 1, input: obj({}, { open: true }) }), + ); for (const type of types) { host.serve( type, @@ -321,7 +323,7 @@ describe("review: leases, invocations, activation", () => { let runs = 0; host.serve(Store, (_input, _principal, self) => { runs++; - const count = self.state("count", Type.Number(), 0); + const count = self.state("count", num(), 0); self.resource("r", { version: 1, open: () => void log.push("open"), @@ -345,8 +347,8 @@ describe("review: leases, invocations, activation", () => { describe("review: prepare isolation and contracts", () => { const Db = local("t.db", { version: 1, - input: Type.Object({}), - members: { read: method(undefined, Type.Unknown()) }, + input: obj({}, { open: true }), + members: { read: method(undefined, unknown()) }, }); it("#7 a prepare rerun cannot touch retained resources, even open ones", async () => { @@ -372,7 +374,7 @@ describe("review: prepare isolation and contracts", () => { it("#7 objects of types a reload adds start listeners and tasks only at commit", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const log: string[] = []; - const Fresh = local("t.fresh", { version: 1, input: Type.Object({}) }); + const Fresh = local("t.fresh", { version: 1, input: obj({}, { open: true }) }); host.install({ name: "item", setup: (api) => api.serve(Item, () => ({ version: () => "v1" })) }); await Promise.all(["a", "b"].map((id) => host.hold(Item, { id }).ready)); const plugins = (fail: boolean): Facet[] => [ @@ -408,20 +410,20 @@ describe("review: prepare isolation and contracts", () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const P1 = remote("t.provider", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { methods: { get: method(undefined, Type.String()) } } }, + variants: { main: { methods: { get: method(undefined, str()) } } }, }); const P2 = remote("t.provider", { version: 2, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { methods: { fetch: method(undefined, Type.String()) } } }, + variants: { main: { methods: { fetch: method(undefined, str()) } } }, }); const Consumer = local("t.consumer", { version: 1, - input: Type.Object({}), - members: { ask: method(undefined, Type.String()) }, + input: obj({}, { open: true }), + members: { ask: method(undefined, str()) }, }); host.install({ name: "both", @@ -493,7 +495,7 @@ describe("review: prepare isolation and contracts", () => { it("#10 retirement closes dropped resources before releasing dropped holds", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const log: string[] = []; - const Dep = local("t.dep", { version: 1, input: Type.Object({}) }); + const Dep = local("t.dep", { version: 1, input: obj({}, { open: true }) }); const facet = (withDecls: boolean): Facet => ({ name: "item", setup: (api) => { @@ -535,48 +537,49 @@ describe("review: prepare isolation and contracts", () => { assert.deepEqual(log, ["own closed", "dep closed"]); }); - it("#11 fits has no false positives for unmodelled keywords; counterexamples checked with Value.Check", () => { + it("#11 fits has no false positives for unmodelled keywords; counterexamples checked with compile", () => { const cases = [ - [Type.String(), Type.String({ not: { const: "x" } }), "x"], - [Type.Array(Type.Number()), Type.Array(Type.Number(), { contains: Type.Literal(1) }), [2]], + [str(), { ...str(), not: { const: "x" } }, "x"], + [arr(num()), { ...arr(num()), contains: typedLit(1) }, [2]], [ - Type.Object({ a: Type.String() }), - Type.Object({ a: Type.String() }, { additionalProperties: Type.Number() }), + obj({ a: str() }, { open: true }), + { ...obj({ a: str() }, { open: true }), additionalProperties: num() }, { a: "s", b: "t" }, ], [ - Type.Object({ x: Type.String() }), - Type.Object({}, { patternProperties: { "^x$": Type.Number() } }), + obj({ x: str() }, { open: true }), + { ...obj({}, { open: true }), patternProperties: { "^x$": num() } }, { x: "s" }, ], - [Type.Number(), Type.Intersect([Type.Number(), Type.Number({ minimum: 1 })]), 0], + [num(), { allOf: [num(), num({ minimum: 1 })] }, 0], [ - Type.Object({ a: Type.String() }, { additionalProperties: Type.Number() }), - Type.Object({ a: Type.String(), b: Type.Optional(Type.String()) }), + { ...obj({ a: str() }, { open: true }), additionalProperties: num() }, + obj({ a: str(), b: opt(str()) }, { open: true }), { a: "", b: 1 }, ], ] as const; for (const [sub, sup, value] of cases) { - assert.ok(Value.Check(sub, value) && !Value.Check(sup, value), `bad counterexample ${JSON.stringify(value)}`); + assert.ok( + compile(sub).check(value) && !compile(sup).check(value), + `bad counterexample ${JSON.stringify(value)}`, + ); assert.equal(fits(sub, sup), false, `fits(${JSON.stringify(sub)}, ${JSON.stringify(sup)})`); } // Additive changes still fit. - assert.ok( - fits(Type.Object({ a: Type.String() }), Type.Object({ a: Type.String(), b: Type.Optional(Type.Number()) })), - ); - assert.ok(fits(Type.Object({ a: Type.String(), extra: Type.Number() }), Type.Object({ a: Type.String() }))); + assert.ok(fits(obj({ a: str() }, { open: true }), obj({ a: str(), b: opt(num()) }, { open: true }))); + assert.ok(fits(obj({ a: str(), extra: num() }, { open: true }), obj({ a: str() }, { open: true }))); }); it("#11 arguments are stripped to what the consumer knows, then to what the provider knows", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const Strict = remote("t.strict", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), variants: { main: { methods: { - echo: method(Type.Object({ a: Type.String() }, { additionalProperties: false }), Type.Unknown()), + echo: method(obj({ a: str() }), unknown()), }, }, }, @@ -590,13 +593,13 @@ describe("review: prepare isolation and contracts", () => { describe("review: ctx races", () => { const Target = local("t.target", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { hit: method() }, }); it("an aborted task ctx cannot start fresh calls", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); - const Runner = local("t.runner", { version: 1, input: Type.Object({}) }); + const Runner = local("t.runner", { version: 1, input: obj({}, { open: true }) }); let hits = 0; const go = latch(); let outcome: unknown; @@ -650,7 +653,7 @@ describe("review: ctx races", () => { it("a remote reference refuses a foreign ctx and a ctx not made by Chord, without calling", async () => { const RemoteTarget = remote("t.remote-target", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { main: { state: {}, methods: { hit: method() } } }, }); diff --git a/packages/chord/test/state-init.test.ts b/packages/chord/test/state-init.test.ts index 77f101d5e..24d086aac 100644 --- a/packages/chord/test/state-init.test.ts +++ b/packages/chord/test/state-init.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { arr, bool, num, obj, str } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { type Ctx, createHost, type Facet, isChordError, local, method } from "../src/index.ts"; @@ -19,16 +19,16 @@ function latch() { const Notes = local("t.notes", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { - notes: Type.Array(Type.String()), - count: Type.Number(), - has: method(Type.String(), Type.Boolean()), - add: method(Type.String()), + notes: arr(str()), + count: num(), + has: method(str(), bool()), + add: method(str()), }, }); -const NotesSchema = Type.Array(Type.String()); +const NotesSchema = arr(str()); describe("state init", () => { it("runs after the resources are open and before the object goes live: nothing sees another value", async () => { @@ -124,7 +124,7 @@ describe("state init", () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const Source = local("t.source", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { all: method(undefined, NotesSchema) }, }); host.serve(Source, () => ({ all: () => ["from source"] })); @@ -155,7 +155,7 @@ describe("state init", () => { return ["loaded"]; }, }); - const counter = self.state("counter", Type.Number(), 0, { version: spec.countVersion ?? 1 }); + const counter = self.state("counter", num(), 0, { version: spec.countVersion ?? 1 }); const extra = spec.extra; const more = extra === undefined diff --git a/packages/chord/test/state.test.ts b/packages/chord/test/state.test.ts index 656ac0601..2f48c8799 100644 --- a/packages/chord/test/state.test.ts +++ b/packages/chord/test/state.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { arr, num, obj, str } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { applyImmutable } from "../src/delta/index.ts"; import { type Ctx, createHost, isChordError, local, method, type StateEvent } from "../src/index.ts"; @@ -22,15 +22,15 @@ function emitter() { }; } -const Item = Type.Object({ id: Type.Number(), label: Type.String() }); -const Items = Type.Object({ items: Type.Array(Item), count: Type.Number() }); +const Item = obj({ id: num(), label: str() }, { open: true }); +const Items = obj({ items: arr(Item), count: num() }, { open: true }); const List = local("t.list", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { list: Items, - labels: Type.Array(Type.String()), + labels: arr(str()), push: method(Item), set: method(Items), }, @@ -84,11 +84,11 @@ describe("state cells", () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const Counter = local("t.prim", { version: 1, - input: Type.Object({}), - members: { n: Type.Number(), inc: method(undefined, Type.Number()) }, + input: obj({}, { open: true }), + members: { n: num(), inc: method(undefined, num()) }, }); host.serve(Counter, (_input, _principal, self) => { - const nState = self.state("n", Type.Number(), 0); + const nState = self.state("n", num(), 0); return { n: nState, inc: (ctx: Ctx) => { @@ -117,11 +117,11 @@ describe("state cells", () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const Box = local("t.box", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { put: method() }, }); host.serve(Box, (_input, _principal, self) => { - const list = self.state("list", Type.Array(Type.Number()), []); + const list = self.state("list", arr(num()), []); return { put: (ctx: Ctx) => { // The classic slip: an expression-bodied arrow returns push()'s length. @@ -133,10 +133,10 @@ describe("state cells", () => { }); it("validates initial values always and writes only in dev mode", async () => { - const Bad = local("t.bad-initial", { version: 1, input: Type.Object({}) }); + const Bad = local("t.bad-initial", { version: 1, input: obj({}, { open: true }) }); const host = createHost({ self: { kind: "host" }, onError: () => {} }); host.serve(Bad, (_input, _principal, self) => { - self.state("x", Type.Object({ n: Type.Number() }), { n: "no" } as never); + self.state("x", obj({ n: num() }, { open: true }), { n: "no" } as never); return {}; }); await assert.rejects(host.hold(Bad, {}).ready, /initial value of state x/); @@ -157,13 +157,13 @@ describe("state cells", () => { it("rejects writes from inside a factory and placeholder use before activation", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); - const Writer = local("t.writer", { version: 1, input: Type.Object({}) }); + const Writer = local("t.writer", { version: 1, input: obj({}, { open: true }) }); let captured: Ctx | undefined; await host.root((ctx) => { captured = ctx; }); host.serve(Writer, (_input, _principal, self) => { - const x = self.state("x", Type.Object({}), {}); + const x = self.state("x", obj({}, { open: true }), {}); const r = self.resource("r", { version: 1, open: () => 1 }); assert.throws( () => r.value, @@ -180,13 +180,13 @@ describe("state cells", () => { const events = emitter(); const Watcher = local("t.watcher", { version: 1, - input: Type.Object({}), - members: { seen: Type.Object({ sum: Type.Number() }) }, + input: obj({}, { open: true }), + members: { seen: obj({ sum: num() }, { open: true }) }, }); const roots = new Set(); let taskAborted = false; host.serve(Watcher, (_input, _principal, self) => { - const seen = self.state("seen", Type.Object({ sum: Type.Number() }), { sum: 0 }); + const seen = self.state("seen", obj({ sum: num() }, { open: true }), { sum: 0 }); const sourceResource = self.resource("source", { version: 1, open: () => events }); self.on( "onEvent", @@ -223,11 +223,11 @@ describe("state cells", () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const Echo = local("t.reentrant", { version: 1, - input: Type.Object({}), - members: { n: Type.Number(), d: Type.Number(), set: method(Type.Number()) }, + input: obj({}, { open: true }), + members: { n: num(), d: num(), set: method(num()) }, }); host.serve(Echo, (_input, _principal, self) => { - const nState = self.state("n", Type.Number(), 0); + const nState = self.state("n", num(), 0); const d = self.derive([nState], (n) => n * 2); // Subscribed first: re-enters the write while revision 1 is being published. self.on("bump", nState, (n, ctx) => { diff --git a/packages/chord/test/tutorial.test.ts b/packages/chord/test/tutorial.test.ts index 55c0dfe17..72f98e1dd 100644 --- a/packages/chord/test/tutorial.test.ts +++ b/packages/chord/test/tutorial.test.ts @@ -1,7 +1,7 @@ // example:imports import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; -import { type Static, Type } from "typebox"; +import { arr, bool, int, num, obj, type Static, str } from "@earendil-works/pi-schema"; import { it, onTestFinished } from "vitest"; import { type Ctx, @@ -51,23 +51,23 @@ function checkExamples(section: string): void { // example:02-contract const TodoLabel = local("tutorial.todos", { version: 1, - input: Type.Object({ list: Type.String() }), - members: { label: method(undefined, Type.String()) }, + input: obj({ list: str() }, { open: true }), + members: { label: method(undefined, str()) }, }); // end-example:02-contract // example:03-contract -const Todo = Type.Object({ text: Type.String(), done: Type.Boolean() }); -const Rows = Type.Array(Todo); +const Todo = obj({ text: str(), done: bool() }, { open: true }); +const Rows = arr(Todo); const TodoStore = local("tutorial.todos", { version: 1, - input: Type.Object({ list: Type.String() }), + input: obj({ list: str() }, { open: true }), members: { - label: method(undefined, Type.String()), + label: method(undefined, str()), rows: Rows, - openCount: Type.Number(), - add: method(Type.Object({ text: Type.String() })), - finish: method(Type.Object({ index: Type.Integer() })), + openCount: num(), + add: method(obj({ text: str() }, { open: true })), + finish: method(obj({ index: int() }, { open: true })), reset: method(Rows), }, }); @@ -107,13 +107,13 @@ const todoFactory = todoStore(); // example:05-views const Todos = remote("tutorial.todo-view", { version: 1, - input: Type.Object({ list: Type.String() }), + input: obj({ list: str() }, { open: true }), host: hostNamed("server"), variants: { viewer: { state: { rows: Rows } }, editor: { state: { rows: Rows }, - methods: { add: method(Type.Object({ text: Type.String() })) }, + methods: { add: method(obj({ text: str() }, { open: true })) }, }, }, }); @@ -190,13 +190,13 @@ it("04: holds, asynchronous resources and release", { timeout: 5000 }, async () // example:04-database const Database = local("tutorial.database", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { db: opaque<{ saved: string[] }>("tutorial.database@1") }, }); const SavedTodos = local("tutorial.saved-todos", { version: 1, - input: Type.Object({ list: Type.String() }), - members: { add: method(Type.Object({ text: Type.String() })) }, + input: obj({ list: str() }, { open: true }), + members: { add: method(obj({ text: str() }, { open: true })) }, }); const host = createHost({ self: "server" }); let opens = 0; @@ -306,8 +306,8 @@ it("07: listeners react and tasks start fresh calls", { timeout: 5000 }, async ( // example:07 const Activity = local("tutorial.activity", { version: 1, - input: Type.Object({}), - members: { seen: Type.Number() }, + input: obj({}, { open: true }), + members: { seen: num() }, }); const host = createHost({ self: "server" }); host.serve(TodoStore, todoFactory); @@ -316,7 +316,7 @@ it("07: listeners react and tasks start fresh calls", { timeout: 5000 }, async ( const stopped = Promise.withResolvers(); host.serve(Activity, (_input, _principal, self) => { const todos = self.get(TodoStore, { list: "family" }); - const seen = self.state("seen", Type.Number(), 0); + const seen = self.state("seen", num(), 0); self.on("rows", todos.rows, (rows, ctx) => { seen.replace(ctx, rows?.length ?? 0); if (rows?.length === 1) observed.resolve(); @@ -345,7 +345,7 @@ it("08: plugins contribute registry entries and expose callbacks", { timeout: 50 onTestFinished(() => host.close()); // example:08 const Actions = defineRegistry<(ctx: Ctx) => Promise>("tutorial.actions"); - const Menu = local("tutorial.menu", { version: 1, input: Type.Object({}) }); + const Menu = local("tutorial.menu", { version: 1, input: obj({}, { open: true }) }); let click: (() => Promise) | undefined; const storeFacet: Facet = { name: "todo-store", @@ -444,7 +444,7 @@ it("09: compatible reload, migration and an old-contract adapter", { timeout: 50 const RenamedStore = local("tutorial.todos", { version: 2, input: TodoStore.def.input, - members: { ...members, heading: method(undefined, Type.String()) }, + members: { ...members, heading: method(undefined, str()) }, }); await host.reload([ { diff --git a/packages/chord/test/types.test.ts b/packages/chord/test/types.test.ts index f64f0ab98..9df86f431 100644 --- a/packages/chord/test/types.test.ts +++ b/packages/chord/test/types.test.ts @@ -1,25 +1,25 @@ // Entry 8: reference types. State members are plain values (a reference you hold has its state); a remote reference is a // union discriminated by `variant`, so another variant's members are reachable only after narrowing. Checked by tsc. -import { Type } from "typebox"; +import { num, obj, str } from "@earendil-works/pi-schema"; import { describe, expectTypeOf, it } from "vitest"; import { type Ctx, type Host, hostNamed, local, method, type Readable, type Ref, remote } from "../src/index.ts"; const Doc = remote("t.e8-types-doc", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { - reader: { state: { text: Type.String() } }, - editor: { state: { text: Type.String() }, methods: { edit: method(Type.String()) } }, + reader: { state: { text: str() } }, + editor: { state: { text: str() }, methods: { edit: method(str()) } }, }, }); const Single = remote("t.e8-types-single", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { state: { n: Type.Number() }, methods: { bump: method() } } }, + variants: { main: { state: { n: num() }, methods: { bump: method() } } }, }); -const Local = local("t.e8-types-local", { version: 1, input: Type.Object({}), members: { n: Type.Number() } }); +const Local = local("t.e8-types-local", { version: 1, input: obj({}, { open: true }), members: { n: num() } }); describe("reference types", () => { it("state members are T; other variants' members need narrowing; gets are awaited", () => { diff --git a/packages/chord/test/variants.test.ts b/packages/chord/test/variants.test.ts index 363f50361..3b8c2359d 100644 --- a/packages/chord/test/variants.test.ts +++ b/packages/chord/test/variants.test.ts @@ -1,7 +1,7 @@ // Entry 8: a binding applies a shape only once its new state members have snapshots; members outside the variant // throw `not_in_variant`; a re-hold after a link loss keeps the last values readable and sends one fresh unit. import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { num, obj, str } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { type Ctx, @@ -24,12 +24,12 @@ async function until(check: () => boolean, attempts = 200): Promise { const Doc = remote("t.e8-doc", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { - a: { state: { x: Type.Number() }, methods: { bump: method() } }, - b: { state: { x: Type.Number(), y: Type.String() }, methods: { bump: method() } }, - c: { state: { y: Type.String() }, methods: { bump: method() } }, + a: { state: { x: num() }, methods: { bump: method() } }, + b: { state: { x: num(), y: str() }, methods: { bump: method() } }, + c: { state: { y: str() }, methods: { bump: method() } }, }, }); @@ -39,8 +39,8 @@ function docFacet(variant: "a" | "b" | "c"): Facet { name: "doc", setup: (api) => api.serve(Doc, (_input, _principal, self) => { - const x = self.state("x", Type.Number(), 1); - const y = self.state("y", Type.String(), "why"); + const x = self.state("x", num(), 1); + const y = self.state("y", str(), "why"); const bump = (ctx: Ctx) => { x.replace(ctx, x.value + 1); }; diff --git a/packages/chord/test/websocket.test.ts b/packages/chord/test/websocket.test.ts index 8cb1e95c2..b5bc4389b 100644 --- a/packages/chord/test/websocket.test.ts +++ b/packages/chord/test/websocket.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import type { IncomingMessage } from "node:http"; -import { Type } from "typebox"; +import { arr, obj, str, unknown } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { WebSocketServer } from "ws"; import { @@ -17,12 +17,12 @@ import { const Board = remote("t.board", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { main: { - state: { notes: Type.Array(Type.String()) }, - methods: { add: method(Type.String()), who: method(undefined, Type.Unknown()) }, + state: { notes: arr(str()) }, + methods: { add: method(str()), who: method(undefined, unknown()) }, }, }, }); @@ -31,7 +31,7 @@ describe("WebSocket links", () => { it("serves a browser-style client over a real socket, stamped by the authenticator", async () => { const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); server.serve(Board, (_input, principal, self) => { - const notes = self.state("notes", Type.Array(Type.String()), []); + const notes = self.state("notes", arr(str()), []); return { variant: "main", notes, @@ -112,7 +112,7 @@ describe("WebSocket links: packed payloads", () => { const notes = Array.from({ length: 40 }, (_, i) => `${i}:${"x".repeat(4000)}`); const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); server.serve(Board, (_input, _principal, self) => { - const state = self.state("notes", Type.Array(Type.String()), notes); + const state = self.state("notes", arr(str()), notes); return { variant: "main", notes: state, add: () => {}, who: () => notes }; }); const wss = new WebSocketServer({ port: 0, host: "127.0.0.1" }); From a6bde34bb61499cfcd9ef70545d5750416a632f7 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sun, 1 Nov 2026 10:22:05 +0000 Subject: [PATCH 234/271] feat(schema): benchmark against ata-validator, Ajv and Zod --- package-lock.json | 222 +++++++++++++++++++++- packages/schema/CHANGELOG.md | 2 +- packages/schema/README.md | 128 +++++++++---- packages/schema/package.json | 6 +- packages/schema/test/bench/all.ts | 6 +- packages/schema/test/bench/candidates.ts | 106 ++++++++++- packages/schema/test/bench/conformance.ts | 4 +- packages/schema/test/bench/size.ts | 13 +- packages/schema/test/bench/speed.ts | 31 ++- packages/schema/test/bench/zod.ts | 187 ++++++++++++++++++ 10 files changed, 652 insertions(+), 53 deletions(-) create mode 100644 packages/schema/test/bench/zod.ts diff --git a/package-lock.json b/package-lock.json index 4bc15ba5c..8abc3354d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,6 +50,116 @@ } } }, + "node_modules/@ata-validator/native-darwin-arm64": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-darwin-arm64/-/native-darwin-arm64-1.61.3.tgz", + "integrity": "sha512-DC+iozKe5mNpsmT4MLuMxG2pIeGPVR3YBYuBmops0AF5UG6eSxg/X0484hW/Nmc2dil54+b4N41hIhS5U9rExg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@ata-validator/native-darwin-x64": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-darwin-x64/-/native-darwin-x64-1.61.3.tgz", + "integrity": "sha512-q63+k48mWlvrNaUXmtGVxp2NWTl3wXlkkWkc2q4MerHzjIfeFBca30tfTRcyVXtDXPCj8BRJkrI69aJJXk+pLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@ata-validator/native-linux-arm64-gnu": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-linux-arm64-gnu/-/native-linux-arm64-gnu-1.61.3.tgz", + "integrity": "sha512-D9OG2oKJd/hW9F8IFoqIfK+SDfyryzSGr5YRVJB39LwvvWNNfh6pxsYpjthEjwqGuJdoBtYIqOUFnjgBqDWh9w==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ata-validator/native-linux-arm64-musl": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-linux-arm64-musl/-/native-linux-arm64-musl-1.61.3.tgz", + "integrity": "sha512-L1NhRd5/i6v7VQw28gR5L53+Zlo09CnYfXBapgGUC9I3uNppOw56/dihU5cPrd5T/k9VeljosllOP+dWA9cLpA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ata-validator/native-linux-x64-gnu": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-linux-x64-gnu/-/native-linux-x64-gnu-1.61.3.tgz", + "integrity": "sha512-Dlp+hcvQqKkAQEyrr7RohCbYoLYHdSjeB5puU3Ju92kSeH4MaF451q0VqwOm3StaMQB3LrLA03GDb4/G3b5gsA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ata-validator/native-linux-x64-musl": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-linux-x64-musl/-/native-linux-x64-musl-1.61.3.tgz", + "integrity": "sha512-2TutVxyhzbpUyBFzuGNp4fjyC3Suhi1idFfbk883tPdfcmp2/j0Q5QT5ySk4LPLGrNzV2haqjESZvha/QKctkw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ata-validator/native-win32-x64": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-win32-x64/-/native-win32-x64-1.61.3.tgz", + "integrity": "sha512-Ac2fqD70b/lQSuLDdz03xiGPhn2FK7oKXdQ7JbCq9ZYiJBbymhFazMKkRaI+3Y5XKIV3zalAfNGzUJAzMUsXow==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@aws-sdk/client-bedrock-runtime": { "version": "3.1127.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-runtime/-/client-bedrock-runtime-3.1127.0.tgz", @@ -3001,6 +3111,41 @@ "node": ">= 14" } }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/asn1": { "version": "0.2.6", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", @@ -3020,6 +3165,36 @@ "node": ">=12" } }, + "node_modules/ata-validator": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/ata-validator/-/ata-validator-1.61.3.tgz", + "integrity": "sha512-6o3V8lpiVQzzHe6blbQZj9saD0Qxmhdv2TTyO5BgPcUWeP3EZj572f9w1b93E6c+BmUce0QOf7GvwOffRUf3tw==", + "dev": true, + "license": "MIT", + "bin": { + "ata": "bin/ata.js" + }, + "engines": { + "node": ">=20.0.0" + }, + "optionalDependencies": { + "@ata-validator/native-darwin-arm64": "1.61.3", + "@ata-validator/native-darwin-x64": "1.61.3", + "@ata-validator/native-linux-arm64-gnu": "1.61.3", + "@ata-validator/native-linux-arm64-musl": "1.61.3", + "@ata-validator/native-linux-x64-gnu": "1.61.3", + "@ata-validator/native-linux-x64-musl": "1.61.3", + "@ata-validator/native-win32-x64": "1.61.3" + }, + "peerDependencies": { + "yaml": "^2.0.0" + }, + "peerDependenciesMeta": { + "yaml": { + "optional": true + } + } + }, "node_modules/balanced-match": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", @@ -3451,12 +3626,36 @@ "node": ">=18" } }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, "node_modules/fast-sha256": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", "license": "Unlicense" }, + "node_modules/fast-uri": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.9.tgz", + "integrity": "sha512-hS/0Wt3Ya043oKXs0sIxrMpKQITXhMLIrVAKKXi4gozGC9v7YRWVnXIZzjNPTO02M9BdxLCWqK1yRXgNziVyJQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -3774,6 +3973,13 @@ "node": ">=16" } }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, "node_modules/jwa": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", @@ -4590,6 +4796,16 @@ "node": ">= 6" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/retry": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", @@ -5778,8 +5994,12 @@ "devDependencies": { "@cfworker/json-schema": "4.1.1", "@oh-my-pi/omptype": "18.13.3", + "ajv": "8.20.0", + "ajv-formats": "3.0.1", + "ata-validator": "1.61.3", "typebox": "1.3.27", - "vitest": "4.1.12" + "vitest": "4.1.12", + "zod": "4.6.7" }, "engines": { "node": ">=22.19.0" diff --git a/packages/schema/CHANGELOG.md b/packages/schema/CHANGELOG.md index 69650940a..02f86d449 100644 --- a/packages/schema/CHANGELOG.md +++ b/packages/schema/CHANGELOG.md @@ -9,4 +9,4 @@ - `compile(schema, { coerce: true })` adds `coerce(value)` and `parse(value)` for loosely typed input such as a model's tool call arguments: optional `null`s dropped, TypeBox 1.3.27's `Value.Convert` rules keyed on JSON Schema keywords (numeric strings, `"true"`, single values to arrays, `const`/`enum` members, union branches), then Ajv-style primitive coercion by `type`. `coerce` copies only the paths it changes and returns the input itself when nothing changes; `parse` coerces, then checks. Validators compiled without the option have neither; the README lists every rule. - `unresolvedRefs: "reject"` compiles a `$ref` that names nothing as `false` (the error is `unresolved reference `) instead of throwing, for schemas whose references may dangle. - `formats` also checks `url` and `json-pointer-uri-fragment`, as TypeBox and ajv-formats do. -- `npm run bench` compares conformance, speed, setup cost, memory and bundle size with TypeBox, @cfworker/json-schema and omptype. +- `npm run bench` compares conformance, speed (with and without eval), setup cost, memory and bundle size with ata-validator (generated code and its interpreter), Ajv, Zod (native schemas, and `z.fromJSONSchema` for the test suite), TypeBox, @cfworker/json-schema and omptype. diff --git a/packages/schema/README.md b/packages/schema/README.md index 100186dd9..6654fe54d 100644 --- a/packages/schema/README.md +++ b/packages/schema/README.md @@ -1,7 +1,7 @@ # @earendil-works/pi-schema JSON Schema builders and validation without `eval`. A schema compiles once into plain closures, so it checks values -about as fast as validators that generate code, also under a strict Content Security Policy, and needs no +at close to the speed of validators that generate code, also under a strict Content Security Policy, and needs no dependencies. - Drafts 2020-12 and 7. Passes 2,222 of the 2,223 core tests of the official @@ -181,53 +181,107 @@ coerced by any pass. ## Benchmarks -Node 24.4.1 on one core of an i9-13900, against TypeBox 1.3.27, @cfworker/json-schema 4.1.1 and @oh-my-pi/omptype -18.13.3 (`fromJsonSchema`). +Node 24.4.1 on one core of an i9-13900. The other libraries: -| test suite (tests passed) | pi-schema | TypeBox | cfworker | omptype | -|---|---:|---:|---:|---:| -| draft 2020-12 core (1,298) | 1,297 | 1,276 | 1,249 | 863 | -| draft 7 core (925) | 925 | 924 | 918 | 659 | -| draft 2020-12 format (895), with `formats` | 895 | 895 | 740 | 412 | +- ata-validator 1.61.3: its generated code, and its eval-free interpreter chosen with `engine: "interpreter"`; +- Ajv 8.20.0, with ajv-formats 3.0.1 for the format tests; +- Zod 4.6.7: native Zod schemas for speed, setup and memory, and `z.fromJSONSchema` for the test suite; +- TypeBox 1.3.27; +- @cfworker/json-schema 4.1.1; +- @oh-my-pi/omptype 18.13.3, through `fromJsonSchema`. -| check, valid value | pi-schema | TypeBox | cfworker | omptype | -|---|---:|---:|---:|---:| -| nested object with tagged unions | 409 ns | 474 ns | 44 µs | 689 ns | -| union of 5 messages | 89 ns | 95 ns | 10 µs | 93 ns | -| tool schema (read) | 31 ns | 9 ns | 1.9 µs | 9 ns | -| MCP tool schema with `$ref` | 162 ns | 89 ns | 13 µs | 1.8 µs | -| recursive `$ref` schema | 753 ns | 1.2 µs | 41 µs | 7.4 µs | +| test suite (tests passed) | pi-schema | ata | ata interpreter | Ajv | Zod | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| draft 2020-12 core (1,298) | 1,297 | 1,298 | 1,298 | 1,236 | 693 | 1,276 | 1,249 | 863 | +| draft 7 core (925) | 925 | 925 | 925 | 917 | 632 | 924 | 918 | 659 | +| draft 2020-12 format (895), with format checks | 895 | 814 | 814 | 750 | 412 | 895 | 740 | 412 | -| same, without eval (strict CSP) | pi-schema | TypeBox | cfworker | omptype | -|---|---:|---:|---:|---:| -| nested object with tagged unions | 407 ns | 42 µs | 43 µs | 2.6 µs | -| MCP tool schema with `$ref` | 162 ns | 36 µs | 12 µs | 2.0 µs | -| recursive `$ref` schema | 750 ns | 124 µs | 41 µs | 9.0 µs | +| check, valid value | pi-schema | ata | ata interpreter | Ajv | Zod | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| nested object with tagged unions | 410 ns | 180 ns | 836 ns | 627 ns | 1.3 µs | 475 ns | 43 µs | 730 ns | +| union of 5 messages | 86 ns | 37 ns | 212 ns | 70 ns | 297 ns | 95 ns | 11 µs | 96 ns | +| tool schema (read) | 31 ns | 10 ns | 41 ns | 13 ns | 41 ns | 9 ns | 2.0 µs | 9 ns | +| MCP tool schema with `$ref` | 161 ns | 64 ns | 314 ns | 136 ns | 927 ns | 87 ns | 15 µs | 1.9 µs | +| recursive `$ref` schema | 755 ns | 307 ns | 1.9 µs | 378 ns | 2.7 µs | 1.2 µs | 41 µs | 7.5 µs | -| cost | pi-schema | TypeBox | cfworker | omptype | -|---|---:|---:|---:|---:| -| compile a new schema | 21 µs | 203 µs | 20 µs | 26 µs + 106 µs on its first check | -| memory, 3,200 distinct schemas | 7.1 MB | 54.7 MB | 7.6 MB | 20.0 MB | -| bundle, minified + gzip | 8.5 KB | 35.2 KB | 5.8 KB | 38.8 KB | +| same, without eval (strict CSP) | pi-schema | ata | ata interpreter | Ajv | Zod | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| nested object with tagged unions | 411 ns | 1.6 µs | 844 ns | fails | 3.4 µs | 44 µs | 44 µs | 2.7 µs | +| union of 5 messages | 87 ns | 396 ns | 209 ns | fails | 652 ns | 10 µs | 11 µs | 501 ns | +| MCP tool schema with `$ref` | 164 ns | 484 ns | 326 ns | fails | 2.0 µs | 39 µs | 13 µs | 2.2 µs | +| recursive `$ref` schema | 746 ns | 2.8 µs | 1.9 µs | fails | 7.3 µs | 130 µs | 42 µs | 9.0 µs | -Reproduce with `npm run bench` in `packages/schema` (about 5 minutes). It runs every library through its fastest -boolean path and keeps the object each returns. The scripts are in `test/bench/`. +| cost | pi-schema | ata | ata interpreter | Ajv | Zod | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| compile a new schema | 20 µs | 4.9 µs | 1.3 µs | 1.4 ms | 151 µs | 210 µs | 22 µs | 28 µs | +| its first check | 0.8 µs | 335 µs | 87 µs | 195 µs | 466 µs | 41 µs | 7.1 µs | 109 µs | +| memory, 3,200 distinct schemas | 7.1 MB | 36.0 MB | 51.3 MB | 59.7 MB | 170.8 MB | 54.7 MB | 7.6 MB | 19.9 MB | +| bundle, minified + gzip | 8.5 KB | 134.0 KB | 134.0 KB | 39.7 KB | 24.4 KB | 35.2 KB | 5.8 KB | 38.8 KB | -- **conformance:** every library on the vendored test suite, with the remote documents and meta-schemas the tests - reference. -- **speed:** realistic schemas, in ns per check after warm-up; the median of 15 rounds, with the libraries' - order rotated in each round. -- **speed without eval:** the same, run with `--disallow-code-generation-from-strings`. -- **setup:** compiling 1,000 schemas a library has not seen, and their first checks. -- **memory:** 3,200 distinct tool-like schemas, each library in its own process. -- **size:** esbuild bundles of what a validator needs, minified and gzipped. +Where code generation is unavailable: -Where code generation is unavailable, TypeBox falls back to its interpreter, and omptype throws `EvalError` once per -schema, then validates through its interpreter. +- **ata** answers through its interpreted engine without being told. That automatic fallback measured slower than the + engine chosen with `engine: "interpreter"`. +- **Ajv** cannot compile at all: `compile()` throws `EvalError`. +- **Zod** notices and parses objects without its generated code. +- **TypeBox** falls back to its interpreter. +- **omptype** throws `EvalError` once per schema, then validates through its interpreter. + +ata, Zod and omptype compile on a schema's first check, so their compile cost shows there. ata's bundle is for the +browser without code splitting; with splitting, its first load is about 53 KB plus a 76 KB generator chunk loaded +later. + +Zod is not a JSON Schema validator: + +- **Speed, setup and memory:** Zod gets the same schemas written as native Zod schemas (`z.strictObject`, + `z.discriminatedUnion`, `z.lazy`), with `.describe()` for the corpus descriptions, and `safeParse` as the check. + Its memory includes building the schemas, which the other libraries receive as JSON. Without the descriptions it + is 131 MB. +- **Test suite:** through `z.fromJSONSchema`, which is semi-experimental. It refuses `unevaluatedProperties`, + `unevaluatedItems`, `dependentSchemas`, `dependentRequired`, `if`/`then`/`else`, `not`, and every `$ref` that is + not local. A refused schema fails all its tests: 405 of the 1,298 draft 2020-12 core tests and 107 of the 925 draft + 7 core tests. +- **Bundle:** measured with Zod's documented `import * as z from "zod"`. A named `import { z }` keeps every locale + (about 90 KB gzipped), and `z.fromJSONSchema` brings it to 40.0 KB; `zod/mini` is about 6 KB. Flat objects of simple types check in 13–80 ns, against 7–15 ns for TypeBox. Without code generation, reading a property whose name comes from a list costs a few nanoseconds. +### Reproduce + +`npm run bench` in `packages/schema` takes about 12 minutes. It runs every library through its fastest boolean path +(ata's `isValidObject()`) and keeps the object each returns. ata runs with `ATA_NO_NATIVE=1`: its optional native +addon only serves its buffer APIs. The scripts are in `test/bench/`. + +- **Conformance:** every library on the vendored test suite, with the remote documents and meta-schemas the tests + reference (Zod through `z.fromJSONSchema`, which takes no remote documents). ata reads the draft only from `$schema`, so draft 7 schemas without one get it declared. ata's defaults + (`useDefaults`, `assertFormat`) are turned off to match the other libraries; formats are checked only in the + format tests, by each library's own checks. +- **Speed:** realistic schemas, in ns per check after warm-up. Each result is the median of 15 rounds, with the + libraries' order rotated in each round. +- **Speed without eval:** the same, run with `--disallow-code-generation-from-strings`. +- **Setup:** compiling 1,000 schemas a library has not seen, and their first checks. +- **Memory:** 3,200 distinct tool-like schemas, compiled and each checked 3 times, each library in its own process. +- **Size:** esbuild browser bundles of what a validator needs, minified and gzipped. + +### Code generated at build time + +Ajv (`ajv/dist/standalone`) and ata (`ata build`, `ata compile`, `toStandaloneModule()`) can also generate their +validation code when the app is built. The app then imports plain modules and evaluates nothing at run time, so those +checks run at code-generation speed under a strict CSP. This is not in the benchmark, because it is a different +deal: + +- **A build step per schema.** Every schema must be known when the app is built. Schemas that arrive at run time + (MCP servers, extensions, user input) cannot be validated this way. +- **Code per schema.** In a separate measurement on the same 3,200 schemas: + - Ajv: 1.8 ms to build and 8.9 KB of JavaScript per schema; + - ata: 1.1 ms and 13 KB per schema, or 0.3–0.8 KB gzipped when only its `isValid` export is bundled; + - importing all 3,200 generated validators took 40 MB of heap (Ajv, one module) and 68 MB (ata, one module per + schema). + +pi-schema compiles at run time in 25 µs per schema, generates no code, and covers schemas known at build time and +schemas that arrive later. + ## Development `npm test` runs the official test suite (drafts 2020-12 and 7, including the optional and format tests, vendored in diff --git a/packages/schema/package.json b/packages/schema/package.json index f6e94e1d2..3fdb1b14f 100644 --- a/packages/schema/package.json +++ b/packages/schema/package.json @@ -50,7 +50,11 @@ "devDependencies": { "@cfworker/json-schema": "4.1.1", "@oh-my-pi/omptype": "18.13.3", + "ajv": "8.20.0", + "ajv-formats": "3.0.1", + "ata-validator": "1.61.3", "typebox": "1.3.27", - "vitest": "4.1.12" + "vitest": "4.1.12", + "zod": "4.6.7" } } diff --git a/packages/schema/test/bench/all.ts b/packages/schema/test/bench/all.ts index bb8421fc2..96ce485c4 100644 --- a/packages/schema/test/bench/all.ts +++ b/packages/schema/test/bench/all.ts @@ -1,5 +1,6 @@ // Runs every benchmark, each in its own process, and writes the results as JSON next to the printed tables. // Speed runs twice: normally and with code generation from strings disabled, as under a strict CSP. +// The children run with ATA_NO_NATIVE=1 (see candidates.ts). // Run: npm run bench [-- --out ] (default: pi-schema-bench-results.json in the temp directory) import { execFileSync } from "node:child_process"; @@ -28,7 +29,10 @@ const results: Record = { for (const [key, file, flags] of runs) { const json = join(tmpdir(), `pi-schema-bench-${key}.json`); console.log(`\n## ${key}${flags.length > 0 ? ` (${flags.join(" ")})` : ""}\n`); - execFileSync(process.execPath, [...flags, join(dir, file), "--json", json], { stdio: "inherit" }); + execFileSync(process.execPath, [...flags, join(dir, file), "--json", json], { + stdio: "inherit", + env: { ...process.env, ATA_NO_NATIVE: "1" }, + }); results[key] = JSON.parse(readFileSync(json, "utf8")); rmSync(json); } diff --git a/packages/schema/test/bench/candidates.ts b/packages/schema/test/bench/candidates.ts index 8b231c855..ce0c6d4cf 100644 --- a/packages/schema/test/bench/candidates.ts +++ b/packages/schema/test/bench/candidates.ts @@ -1,18 +1,28 @@ // The validators the benchmarks compare, each behind the same interface: compile a schema, then check values. // Every library is used through its fastest boolean path and keeps the object it returns, as an application would. +// Run the benchmarks through `npm run bench`, which sets ATA_NO_NATIVE=1: ata-validator's optional native addon only +// serves its buffer APIs, and the verdicts measured here are JavaScript either way. import { Validator as CfValidator } from "@cfworker/json-schema"; import { fromJsonSchema } from "@oh-my-pi/omptype"; +import Ajv7Module from "ajv"; +import Ajv2020Module from "ajv/dist/2020.js"; +import draft07MetaSchema from "ajv/dist/refs/json-schema-draft-07.json" with { type: "json" }; +import addFormatsModule from "ajv-formats"; +import { Validator as AtaValidator } from "ata-validator"; import { Compile } from "typebox/compile"; import { ResetExternal, ResetFunctions } from "typebox/schema"; import { Settings } from "typebox/system"; import { compile, type FormatCheck, type JsonSchema } from "../../src/index.ts"; +import { nativeCases, nativeZod, zodFromJsonSchema } from "./zod.ts"; /** How a benchmark compiles a schema. */ export interface PrepareOptions { readonly draft: "2020-12" | "7"; readonly schemas?: Readonly>; readonly formats?: Readonly>; + /** The schema is a JSON-Schema-Test-Suite schema (for libraries that take native schemas otherwise). */ + readonly suite?: boolean; } /** A compiled schema: its boolean check and the object the library returned (kept alive by the caller). */ @@ -92,8 +102,102 @@ const omptype: Candidate = { }, }; +// Ajv's CommonJS modules export the class as module.exports and as `default`; the latter is what its types describe. +const Ajv2020 = Ajv2020Module.default; +const Ajv7 = Ajv7Module.default; +const addFormats = addFormatsModule.default; +type AjvInstance = InstanceType; + +// One Ajv per draft, as an application keeps it. Ajv2020 also gets the draft-07 meta-schema, so tool schemas that +// declare draft-07 compile. strict: false so schemas written for other tools compile; logger: false keeps Ajv from +// printing generated code when compiling fails without eval. +function newAjv(draft: "2020-12" | "7", formats: boolean): AjvInstance { + const options = { strict: false, validateFormats: formats, logger: false as const }; + const ajv = draft === "7" ? new Ajv7(options) : new Ajv2020(options); + if (draft !== "7") ajv.addMetaSchema(draft07MetaSchema); + if (formats) addFormats(ajv); + return ajv; +} + +const sharedAjv = new Map(); + +// Ajv compiles with new Function and has no interpreter: without eval, compile() throws EvalError. +const ajv: Candidate = { + name: noEval ? "ajv (no eval: fails)" : "ajv", + prepare(schema, options) { + let instance: AjvInstance; + if (options.schemas === undefined) { + const key = `${options.draft} ${options.formats !== undefined}`; + instance = sharedAjv.get(key) ?? newAjv(options.draft, options.formats !== undefined); + sharedAjv.set(key, instance); + } else { + // The suite reuses $ids across groups, which one shared instance would reject: a fresh one per schema. + instance = newAjv(options.draft, options.formats !== undefined); + for (const [uri, remote] of Object.entries(options.schemas)) { + // Ajv ships its own meta-schemas; adding them again under the same id throws. + if (instance.getSchema(uri) === undefined) { + try { + instance.addSchema(remote as object, uri); + } catch { + // A remote whose $id Ajv already registered under another key. + } + } + } + } + const validator = instance.compile(schema as object); + return { check: (v) => validator(v) as boolean, validator }; + }, +}; + +// ata-validator: its defaults fill `default` values into the input and assert `format`; both are turned off to match +// the other candidates (format checks only in the format run, with its own checks). It reads the draft only from +// $schema, so a draft 7 schema without one gets it declared. +function ataValidator(schema: JsonSchema, options: PrepareOptions, engine: "interpreter" | undefined) { + const declared = + options.draft === "7" && typeof schema === "object" && !("$schema" in schema) + ? { $schema: "http://json-schema.org/draft-07/schema#", ...schema } + : schema; + return new AtaValidator(declared, { + useDefaults: false, + assertFormat: options.formats !== undefined, + schemas: (options.schemas ?? {}) as Record, + logger: false, + ...(engine === undefined ? {} : { engine }), + }); +} + +// Generated code, or its interpreted engine where code generation is blocked. +const ata: Candidate = { + name: noEval ? "ata (no eval: interpreter)" : "ata", + prepare(schema, options) { + const validator = ataValidator(schema, options, undefined); + return { check: (v) => validator.isValidObject(v), validator }; + }, +}; + +// Its eval-free engine, a tree of closures, chosen explicitly: what it runs on for schemas from outside. +const ataInterpreter: Candidate = { + name: "ata interpreter", + prepare(schema, options) { + const validator = ataValidator(schema, options, "interpreter"); + return { check: (v) => validator.isValidObject(v), validator }; + }, +}; + +// Zod: native Zod schemas for the speed cases and the corpus, z.fromJSONSchema only for the suite. Its object parser +// generates code with new Function when it can and parses without it otherwise; safeParse is its boolean path. +const zod: Candidate = { + name: noEval ? "zod (no eval: no JIT)" : "zod", + prepare(schema, options) { + const validator = options.suite + ? zodFromJsonSchema(schema, options.draft) + : (nativeCases.get(schema) ?? nativeZod(schema as Record)); + return { check: (v) => validator.safeParse(v).success, validator }; + }, +}; + /** Every candidate, ours first. */ -export const candidates: readonly Candidate[] = [piSchema, typebox, cfworker, omptype]; +export const candidates: readonly Candidate[] = [piSchema, typebox, ajv, ata, ataInterpreter, zod, cfworker, omptype]; /** Candidates by name, including TypeBox's interpreter (what TypeBox runs when eval is unavailable). */ export const byName: ReadonlyMap = new Map( diff --git a/packages/schema/test/bench/conformance.ts b/packages/schema/test/bench/conformance.ts index d0cc72a7e..9d8977d76 100644 --- a/packages/schema/test/bench/conformance.ts +++ b/packages/schema/test/bench/conformance.ts @@ -1,6 +1,7 @@ // Conformance of every candidate on the vendored JSON-Schema-Test-Suite (drafts 2020-12 and 7), by section: // core, optional and optional/format. Every candidate gets the remote documents and meta-schemas; format tests run -// with pi-schema's format checks (the other libraries check formats on their own or not at all). +// with pi-schema's format checks (the other libraries check formats on their own or not at all). Zod goes through +// z.fromJSONSchema, which refuses some keywords and every remote $ref: a schema it refuses fails all its tests. // Run: node test/bench/conformance.ts [--json ] import { writeFileSync } from "node:fs"; @@ -29,6 +30,7 @@ for (const file of suiteFiles()) { draft: file.draft, schemas: suiteSchemas(), formats: kind === "format" ? formats : undefined, + suite: true, }).check; } catch { continue; // A schema the library cannot compile fails all its tests. diff --git a/packages/schema/test/bench/size.ts b/packages/schema/test/bench/size.ts index b6996af60..46a09e3d3 100644 --- a/packages/schema/test/bench/size.ts +++ b/packages/schema/test/bench/size.ts @@ -1,4 +1,5 @@ -// Bundle size of what a validator needs from each library: esbuild bundles and minifies the import, then gzip -9. +// Bundle size of what a validator needs from each library, as a browser app ships it: esbuild bundles and minifies the +// import for the browser platform (one file, no code splitting), then gzip -9. // Run: node test/bench/size.ts [--json ] import { writeFileSync } from "node:fs"; @@ -12,6 +13,13 @@ const entries: Record = { "pi-schema + formats": `export { compile } from ${JSON.stringify(`${src}index.ts`)}; export { formats } from ${JSON.stringify(`${src}formats/index.ts`)};`, typebox: `export { Compile } from "typebox/compile";`, cfworker: `export { Validator } from "@cfworker/json-schema";`, + ajv: `export { default } from "ajv/dist/2020.js";`, + "ajv + ajv-formats": `export { default } from "ajv/dist/2020.js"; export { default as addFormats } from "ajv-formats";`, + "ata-validator": `export { Validator } from "ata-validator";`, + // Zod's documented `import * as z` (a named `{ z }` import keeps every locale, about 90 KB gzipped). Its core with + // native schemas, and with z.fromJSONSchema for JSON Schema. + zod: `import * as z from "zod"; export const check = (v: unknown) => z.object({ a: z.string() }).safeParse(v).success;`, + "zod + fromJSONSchema": `import * as z from "zod"; export const fromJSONSchema = z.fromJSONSchema;`, omptype: `export { fromJsonSchema } from "@oh-my-pi/omptype";`, }; @@ -25,8 +33,7 @@ for (const [name, contents] of Object.entries(entries)) { bundle: true, minify: true, format: "esm", - platform: "neutral", - mainFields: ["module", "main"], + platform: "browser", write: false, logLevel: "silent", }); diff --git a/packages/schema/test/bench/speed.ts b/packages/schema/test/bench/speed.ts index 6f470951b..ba2f17b23 100644 --- a/packages/schema/test/bench/speed.ts +++ b/packages/schema/test/bench/speed.ts @@ -20,21 +20,37 @@ function median(values: number[]): number { return [...values].sort((a, b) => a - b)[values.length >> 1]; } -/** Median ns per check, by case name, input ("valid"/"invalid") and candidate name. */ -export type SpeedResults = Record>>; +/** + * Median ns per check, by case name, input ("valid"/"invalid") and candidate name; null where the candidate cannot + * compile the schema (Ajv without eval). + */ +export type SpeedResults = Record>>; const results: SpeedResults = {}; for (const c of cases) { results[c.name] = { valid: {}, invalid: {} }; for (const input of ["valid", "invalid"] as const) { const value = c[input]; - const runs = candidates.map((candidate) => { - const { check } = candidate.prepare(c.schema, { draft: "2020-12" }); + const runs: { + candidate: (typeof candidates)[number]; + check: (v: unknown) => boolean; + n: number; + samples: number[]; + }[] = []; + for (const candidate of candidates) { + let check: (v: unknown) => boolean; + try { + check = candidate.prepare(c.schema, { draft: "2020-12" }).check; + } catch (error) { + if (!(error instanceof EvalError)) throw error; + results[c.name][input][candidate.name] = null; + continue; + } if (check(value) !== (input === "valid")) throw new Error(`${candidate.name} gets ${c.name} (${input}) wrong`); let n = 1000; while (time(check, value, n) * n < ROUND_MS * 1e6 && n < 1e8) n *= 2; - return { candidate, check, n, samples: [] as number[] }; - }); + runs.push({ candidate, check, n, samples: [] }); + } for (let round = 0; round < ROUNDS; round++) { for (let i = 0; i < runs.length; i++) { const run = runs[(i + round) % runs.length]; @@ -57,6 +73,7 @@ const out = process.argv.indexOf("--json"); if (out >= 0) writeFileSync(process.argv[out + 1], JSON.stringify(results, null, "\t")); if (sink < 0) console.log(sink); -function format(ns: number): string { +function format(ns: number | null): string { + if (ns === null) return "fails"; return ns >= 1000 ? `${(ns / 1000).toFixed(ns >= 10000 ? 0 : 1)} µs` : `${ns.toFixed(0)} ns`; } diff --git a/packages/schema/test/bench/zod.ts b/packages/schema/test/bench/zod.ts new file mode 100644 index 000000000..c0d4d8371 --- /dev/null +++ b/packages/schema/test/bench/zod.ts @@ -0,0 +1,187 @@ +// Zod in the comparison. Zod is not a JSON Schema validator: the speed cases are written here as the native Zod +// schemas a developer would write (z.strictObject for closed objects, z.discriminatedUnion for tagged unions, z.lazy +// for recursion), and the corpus schemas are built into the same native calls by `nativeZod`. Only the conformance run +// goes through z.fromJSONSchema, Zod's (semi-experimental) JSON Schema import. + +import * as z from "zod"; +import type { JsonSchema } from "../../src/index.ts"; +import { cases } from "./cases.ts"; + +const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +const block = z.discriminatedUnion("kind", [ + z.strictObject({ kind: z.literal("text"), text: z.string() }), + z.strictObject({ + kind: z.literal("image"), + src: z.string(), + width: z.number().int().min(0), + height: z.number().int().min(0), + alt: z.string().optional(), + }), + z.strictObject({ kind: z.literal("code"), lang: z.enum(["ts", "js", "py"]), source: z.string() }), +]); + +const id = z.number().int().min(0); + +const range = z.strictObject({ start: z.number().int().min(1), end: z.number().int().min(1) }); +const replace = z.strictObject({ oldText: z.string(), newText: z.string() }); +const lineEdit = z.strictObject({ range, text: z.string() }); + +const richText = z.strictObject({ + type: z.enum(["text", "mention"]), + content: z.string().max(2000), + annotations: z.strictObject({ bold: z.boolean(), italic: z.boolean(), color: z.string() }).optional(), +}); + +type PageBlock = { + type: "paragraph" | "heading_1" | "heading_2" | "bulleted_list_item" | "to_do"; + rich_text: z.infer[]; + checked?: boolean; + children?: PageBlock[]; +}; + +const pageBlock: z.ZodType = z.lazy(() => + z.strictObject({ + type: z.enum(["paragraph", "heading_1", "heading_2", "bulleted_list_item", "to_do"]), + rich_text: z.array(richText), + checked: z.boolean().optional(), + children: z.array(pageBlock).optional(), + }), +); + +/** The speed cases as native Zod schemas, by case name (the JSON Schema cases in cases.ts, one for one). */ +const native: Record = { + "small closed object": z.strictObject({ text: z.string() }), + "nested object with tagged unions": z.strictObject({ + id: z.string().regex(uuid), + title: z.string().min(1).max(200), + tags: z.array(z.string()).max(16), + author: z.strictObject({ name: z.string(), email: z.string().optional() }), + blocks: z.array(block), + score: z.number().min(0).max(1).optional(), + draft: z.boolean().optional(), + }), + "message union of 5": z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("hello"), + protocol: z.number().int().min(0), + incarnation: z.string().regex(uuid), + calls: z.number().int().min(1), + }), + z.strictObject({ + kind: z.literal("hold"), + hold: id, + type: z.string(), + version: z.number().int().min(1), + fingerprint: z.string().regex(/^[0-9a-f]{16}$/), + params: z.unknown(), + address: z.strictObject({ kind: z.string(), params: z.unknown() }), + chain: z.array(z.string().regex(uuid)).max(64).optional(), + }), + z.strictObject({ kind: z.literal("release"), hold: id }), + z.strictObject({ kind: z.literal("call"), call: id, hold: id, method: id, args: z.unknown().optional() }), + z.strictObject({ kind: z.literal("abort"), call: id }), + ]), + "tool: read": z.object({ path: z.string(), offset: z.number().optional(), limit: z.number().optional() }), + "tool: edit": z.object({ path: z.string(), edits: z.array(z.object({ oldText: z.string(), newText: z.string() })) }), + "tool: bash": z.object({ command: z.string(), timeout: z.number().optional() }), + "MCP create_issue (pattern, anyOf, enum)": z.strictObject({ + owner: z + .string() + .min(1) + .regex(/^[A-Za-z0-9-]+$/), + repo: z + .string() + .min(1) + .regex(/^[A-Za-z0-9._-]+$/), + title: z.string().min(1), + body: z.string().optional(), + labels: z.array(z.string()).optional(), + assignees: z.array(z.string()).optional(), + milestone: z.number().nullable().optional(), + state: z.enum(["open", "closed"]).optional(), + }), + "MCP edit_file ($ref, anyOf)": z.strictObject({ + path: z.string().regex(/^(\/|\.\/|[A-Za-z]:\\)?[^\u0000]+$/), + edits: z.array(z.union([replace, lineEdit])).min(1), + dryRun: z.boolean().optional(), + }), + "MCP create_page (recursive $ref, oneOf)": z.strictObject({ + parent: z.union([ + z.strictObject({ page_id: z.string().regex(uuid) }), + z.strictObject({ database_id: z.string().regex(uuid) }), + ]), + title: z.string().min(1), + due: z + .string() + .regex(/^\d{4}-\d{2}-\d{2}$/) + .optional(), + children: z.array(pageBlock).max(100).optional(), + }), +}; + +/** The native Zod schema of each speed case's JSON Schema. */ +export const nativeCases: ReadonlyMap = new Map( + cases.map((c) => { + const schema = native[c.name]; + if (schema === undefined) throw new Error(`No native Zod schema for the case ${c.name}`); + return [c.schema, schema]; + }), +); + +type Json = Record; + +/** + * The native Zod schema a developer would write for a corpus schema (corpus.ts): objects (closed ones strict), + * strings with bounds, patterns and enums, bounded integers, numbers, booleans, string arrays, descriptions, and + * $defs reused through their constant. Throws for any other shape. + */ +export function nativeZod(schema: Json, root: Json = schema): z.ZodType { + if (typeof schema.$ref === "string") { + const name = schema.$ref.replace("#/$defs/", ""); + return nativeZod((root.$defs as Record)[name], root); + } + const built = nativeType(schema, root); + return typeof schema.description === "string" ? built.describe(schema.description) : built; +} + +function nativeType(schema: Json, root: Json): z.ZodType { + switch (schema.type) { + case "object": { + const required = new Set(schema.required as string[]); + const shape: Record = {}; + for (const [key, value] of Object.entries(schema.properties as Record)) { + const property = nativeZod(value, root); + shape[key] = required.has(key) ? property : property.optional(); + } + return schema.additionalProperties === false ? z.strictObject(shape) : z.object(shape); + } + case "string": { + if (Array.isArray(schema.enum)) return z.enum(schema.enum as [string, ...string[]]); + let string = z.string(); + if (typeof schema.minLength === "number") string = string.min(schema.minLength); + if (typeof schema.maxLength === "number") string = string.max(schema.maxLength); + if (typeof schema.pattern === "string") string = string.regex(new RegExp(schema.pattern)); + return string; + } + case "integer": + return z + .number() + .int() + .min(schema.minimum as number) + .max(schema.maximum as number); + case "number": + return z.number(); + case "boolean": + return z.boolean(); + case "array": + return z.array(z.string()).max(schema.maxItems as number); + default: + throw new Error(`No native Zod builder for ${JSON.stringify(schema).slice(0, 80)}`); + } +} + +/** z.fromJSONSchema with the draft given; it accepts only local $refs, so remote documents are not passed. */ +export function zodFromJsonSchema(schema: JsonSchema, draft: "2020-12" | "7"): z.ZodType { + return z.fromJSONSchema(schema as never, { defaultTarget: draft === "7" ? "draft-7" : "draft-2020-12" }); +} From 6291b9bcc3ee9480c2caba8676fc2f6cd321eaa9 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sun, 1 Nov 2026 11:12:44 +0000 Subject: [PATCH 235/271] feat(agent): pi-schema's Static types TypeBox-built schemas too, without depending on TypeBox --- packages/agent/CHANGELOG.md | 2 +- packages/agent/src/types.ts | 4 +- .../agent/test/agent-tool-typebox.test.ts | 65 ++++++++++++++++ .../src/core/tools/tool-definition-wrapper.ts | 4 +- packages/durable/CHANGELOG.md | 2 +- packages/durable/src/harness/types.ts | 4 +- .../durable/test/tool-args-typebox.test.ts | 61 +++++++++++++++ packages/schema/CHANGELOG.md | 1 + packages/schema/README.md | 6 ++ packages/schema/src/builders.ts | 17 ++++- packages/schema/src/typebox-static.ts | 76 +++++++++++++++++++ packages/schema/test/typebox-static.test.ts | 72 ++++++++++++++++++ 12 files changed, 304 insertions(+), 10 deletions(-) create mode 100644 packages/agent/test/agent-tool-typebox.test.ts create mode 100644 packages/durable/test/tool-args-typebox.test.ts create mode 100644 packages/schema/src/typebox-static.ts create mode 100644 packages/schema/test/typebox-static.test.ts diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index f095f4f4f..96b4b9428 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -4,7 +4,7 @@ ### Breaking Changes -- `AgentTool` takes any JSON Schema object as `parameters` (`AgentTool`) and types `params` and `prepareArguments` through `@earendil-works/pi-schema` (`AgentToolParams`): pi-schema's builders give their static type, other schemas (TypeBox included) give `unknown`, `AgentTool` stays `any`. `outputSchema` is any JSON Schema object. `typebox` is no longer a dependency. Migration: build parameters with pi-schema (`obj({ path: str() })` instead of `Type.Object({ path: Type.String() })`), or keep TypeBox schemas and narrow `params` yourself. +- `AgentTool` takes any JSON Schema object as `parameters` (`AgentTool`) and types `params` and `prepareArguments` through `@earendil-works/pi-schema` (`AgentToolParams`): pi-schema's builders give their static type, TypeBox schemas give TypeBox's static type for the common kinds (String, Number, Integer, Boolean, Null, Literal, Array, Object with optional and readonly properties, Union, Record, Unsafe, Unknown, Any; other kinds such as Enum, Tuple and Intersect give `unknown`), other schemas give `unknown`, `AgentTool` stays `any`. `outputSchema` is any JSON Schema object. `typebox` is no longer a dependency. Migration: build parameters with pi-schema (`obj({ path: str() })` instead of `Type.Object({ path: Type.String() })`), or keep TypeBox schemas (narrow `params` yourself only where they use other kinds). ### Fixed diff --git a/packages/agent/src/types.ts b/packages/agent/src/types.ts index 539fee1ae..24bdd7531 100644 --- a/packages/agent/src/types.ts +++ b/packages/agent/src/types.ts @@ -463,8 +463,8 @@ export interface AgentToolCallOutcome { export type AgentToolUpdateCallback = (partialResult: AgentToolResult) => void; /** - * The arguments type of a tool with parameter schema `TParameters`: `Static` for schemas built with - * `@earendil-works/pi-schema`'s builders, `unknown` for other schemas, and `any` for `AgentTool`. + * The arguments type of a tool with parameter schema `TParameters`: pi-schema's `Static` (the builders' + * type, TypeBox's for the common TypeBox kinds, else `unknown`), and `any` for `AgentTool`. */ export type AgentToolParams = IsAny extends true ? any : Static; diff --git a/packages/agent/test/agent-tool-typebox.test.ts b/packages/agent/test/agent-tool-typebox.test.ts new file mode 100644 index 000000000..1c859b7ae --- /dev/null +++ b/packages/agent/test/agent-tool-typebox.test.ts @@ -0,0 +1,65 @@ +// AgentTool params typed from a TypeBox-built parameter schema. pi-agent-core does not depend on TypeBox: the schema +// below has TypeBox 1.x's type shapes as its d.mts declare them; pi-schema's own tests check the shapes against TypeBox. + +import { describe, expectTypeOf, it } from "vitest"; +import type { AgentTool } from "../src/types.ts"; + +interface TString { + "~kind": "String"; + type: "string"; +} + +interface TLiteral { + "~kind": "Literal"; + const: Value; +} + +interface TUnion { + "~kind": "Union"; + anyOf: Branches; +} + +interface TObject { + "~kind": "Object"; + type: "object"; + properties: Properties; + required: string[]; +} + +type TOptional = Schema & { "~optional": true }; + +const parameters: TObject<{ command: TString; mode: TOptional, TLiteral<"deep">]>> }> = { + "~kind": "Object", + type: "object", + properties: { + command: { "~kind": "String", type: "string" }, + mode: { + "~kind": "Union", + anyOf: [ + { "~kind": "Literal", const: "fast" }, + { "~kind": "Literal", const: "deep" }, + ], + "~optional": true, + }, + }, + required: ["command"], +}; + +type Params = { command: string; mode?: "fast" | "deep" }; + +describe("AgentTool params of a TypeBox schema", () => { + it("are typed by the schema in execute() and prepareArguments()", () => { + const tool: AgentTool = { + name: "bash", + label: "Bash", + description: "Runs a command", + parameters, + async execute(_toolCallId, params) { + expectTypeOf(params).toEqualTypeOf(); + return { content: [], details: undefined }; + }, + }; + expectTypeOf[1]>().toEqualTypeOf(); + expectTypeOf>>().toEqualTypeOf(); + }); +}); diff --git a/packages/coding-agent/src/core/tools/tool-definition-wrapper.ts b/packages/coding-agent/src/core/tools/tool-definition-wrapper.ts index 320030fab..700dfd35c 100644 --- a/packages/coding-agent/src/core/tools/tool-definition-wrapper.ts +++ b/packages/coding-agent/src/core/tools/tool-definition-wrapper.ts @@ -4,8 +4,8 @@ import type { ExtensionToolContext, ToolDefinition } from "../extensions/types.t /** * An `AgentTool` with a TypeBox parameter schema whose `execute` and `prepareArguments` are typed by TypeBox's - * `Static`, as the built-in tools are (pi-agent-core types `params` through pi-schema, which gives `unknown` for - * TypeBox schemas). Spreads into a `ToolDefinition` of the same schema. + * `Static`, as the built-in tools are (pi-agent-core types `params` through pi-schema, which maps only the common + * TypeBox kinds). Spreads into a `ToolDefinition` of the same schema. */ export type TypeBoxAgentTool = Omit< AgentTool, diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index e852e8fa8..6e9dcdb21 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -4,7 +4,7 @@ ### Breaking Changes -- Tool schemas are JSON Schema typed through `@earendil-works/pi-schema`: `defineTool()` and `ToolRegistration` take any JSON Schema object as `parameters` and `structuredOutputSchema`, and type `args` with pi-schema's `Static` (`ToolArgs`; other schemas give `unknown`). The built-in tools (`read`, `write`, `edit`, `bash`, `powershell`, `codemode`, `tool_search`, the model tools) are built with pi-schema, with the same JSON Schema up to key order. `structuredOutput` is validated with pi-schema, so `invalid_structured_output` messages use its wording. `typebox` is no longer a dependency. Migration: replace `Type.Object({ n: Type.Number() })` from `@earendil-works/pi-ai` with `obj({ n: num() })` from `@earendil-works/pi-schema` (closed by default; `{ open: true }` allows other properties). +- Tool schemas are JSON Schema typed through `@earendil-works/pi-schema`: `defineTool()` and `ToolRegistration` take any JSON Schema object as `parameters` and `structuredOutputSchema`, and type `args` with pi-schema's `Static` (`ToolArgs`; TypeBox schemas give TypeBox's static type for the common kinds, as pi-schema's `Static` describes; other schemas give `unknown`). The built-in tools (`read`, `write`, `edit`, `bash`, `powershell`, `codemode`, `tool_search`, the model tools) are built with pi-schema, with the same JSON Schema up to key order. `structuredOutput` is validated with pi-schema, so `invalid_structured_output` messages use its wording. `typebox` is no longer a dependency. Migration: replace `Type.Object({ n: Type.Number() })` from `@earendil-works/pi-ai` with `obj({ n: num() })` from `@earendil-works/pi-schema` (closed by default; `{ open: true }` allows other properties). - `SqliteExecutor` (re-exported from Chord) gains `runCounted(sql, ...params)`, resolving with the rows the statement changed; custom `SqliteDatabase` implementations add it. - `Storage.commit()` stores a group of commits in one call: `commit(batches, receive, context)` resolves with a `CommitResult` (`{ stored, refused? }`) and replaces `commit(writes, adopt, context)`. Each batch is one atomic commit with its own seq, checked against the state the batches before it left. The first batch Durable's own checks refuse (global IDs, document actions, copy sources) is not stored, nor anything after it; the batches before it are, and the call resolves with their number in `stored` and the refusal in `refused`. Any other failure (a driver or I/O error, a failed `COMMIT`) rejects the whole call as an unknown outcome and stores nothing the backend could roll back. `receive(index, seq)` is called once per stored batch, in order, synchronously, once they are durable and before the next call starts; receipts made before a rejection stand. Every batch has at least one write. The optional `publishing(publish, context)` runs each commit's publication inside the storage's own scope (`StoreStorage`: the attached host's batch). Memory, SQLite, JSONL, IndexedDB and `StoreStorage` store a call in one transaction or append, and the storage conformance suite checks the contract. Migration for a custom Storage: diff --git a/packages/durable/src/harness/types.ts b/packages/durable/src/harness/types.ts index c0f03e65d..e312758b9 100644 --- a/packages/durable/src/harness/types.ts +++ b/packages/durable/src/harness/types.ts @@ -333,8 +333,8 @@ export interface ToolExecutionApi extend } /** - * The arguments type of a tool with parameter schema `TParameters`: `Static` for schemas built with - * `@earendil-works/pi-schema`'s builders, `unknown` for other JSON Schema, and `any` for `ToolRegistration`. + * The arguments type of a tool with parameter schema `TParameters`: pi-schema's `Static` (the builders' + * type, TypeBox's for the common TypeBox kinds, else `unknown`), and `any` for `ToolRegistration`. */ export type ToolArgs = IsAny extends true ? any : Static; diff --git a/packages/durable/test/tool-args-typebox.test.ts b/packages/durable/test/tool-args-typebox.test.ts new file mode 100644 index 000000000..7a606cf96 --- /dev/null +++ b/packages/durable/test/tool-args-typebox.test.ts @@ -0,0 +1,61 @@ +// Tool arguments typed from a TypeBox-built parameter schema. Durable does not depend on TypeBox: the schema below +// has TypeBox 1.x's type shapes as its d.mts declare them; pi-schema's own tests check the shapes against TypeBox. + +import { defineTool, type ToolRegistration } from "@earendil-works/pi-durable"; +import { describe, expectTypeOf, it } from "vitest"; + +interface TString { + "~kind": "String"; + type: "string"; +} + +interface TInteger { + "~kind": "Integer"; + type: "integer"; +} + +interface TArray { + "~kind": "Array"; + type: "array"; + items: Items; +} + +interface TObject { + "~kind": "Object"; + type: "object"; + properties: Properties; + required: string[]; +} + +type TOptional = Schema & { "~optional": true }; + +const parameters: TObject<{ path: TString; offset: TOptional; tags: TArray }> = { + "~kind": "Object", + type: "object", + properties: { + path: { "~kind": "String", type: "string" }, + offset: { "~kind": "Integer", type: "integer", "~optional": true }, + tags: { "~kind": "Array", type: "array", items: { "~kind": "String", type: "string" } }, + }, + required: ["path", "tags"], +}; + +type Args = { path: string; offset?: number; tags: string[] }; + +describe("tool arguments of a TypeBox schema", () => { + it("are typed by the schema in execute() and prepareArguments()", () => { + const tool = defineTool({ + name: "read", + description: "Reads a file", + parameters, + async execute(args) { + expectTypeOf(args).toEqualTypeOf(); + return {}; + }, + }); + expectTypeOf[0]>().toEqualTypeOf(); + expectTypeOf< + ReturnType["prepareArguments"]>> + >().toEqualTypeOf(); + }); +}); diff --git a/packages/schema/CHANGELOG.md b/packages/schema/CHANGELOG.md index 02f86d449..404e93c57 100644 --- a/packages/schema/CHANGELOG.md +++ b/packages/schema/CHANGELOG.md @@ -7,6 +7,7 @@ - Initial release: `compile(schema, options)` validates JSON Schema drafts 2020-12 and 7 without `eval` or `new Function`, compiling a schema once into plain closures. `Validator.check` is the boolean check (a type guard for schemas built with the builders), `Validator.errors` reports each failure with a JSON Pointer path. Format checks are in the optional `@earendil-works/pi-schema/formats` entry point. - Schema builders (`str`, `num`, `int`, `bool`, `nul`, `unknown`, `lit`, `oneOf`, `obj`, `opt`, `arr`, `tuple`, `map`, `union`, `nullable`, `bytes`) that return plain JSON Schema typed with the values it accepts (`Static`). - `compile(schema, { coerce: true })` adds `coerce(value)` and `parse(value)` for loosely typed input such as a model's tool call arguments: optional `null`s dropped, TypeBox 1.3.27's `Value.Convert` rules keyed on JSON Schema keywords (numeric strings, `"true"`, single values to arrays, `const`/`enum` members, union branches), then Ajv-style primitive coercion by `type`. `coerce` copies only the paths it changes and returns the input itself when nothing changes; `parse` coerces, then checks. Validators compiled without the option have neither; the README lists every rule. +- `Static` also types TypeBox-built schemas, from TypeBox 1.x's type shapes and without importing TypeBox: String, Number, Integer, Boolean, Null, Literal, Array, Object (with `Type.Optional` and `Type.Readonly` properties), Union, Record, Unsafe, Unknown and Any give TypeBox's static type; other TypeBox kinds give `unknown`. - `unresolvedRefs: "reject"` compiles a `$ref` that names nothing as `false` (the error is `unresolved reference `) instead of throwing, for schemas whose references may dangle. - `formats` also checks `url` and `json-pointer-uri-fragment`, as TypeBox and ajv-formats do. - `npm run bench` compares conformance, speed (with and without eval), setup cost, memory and bundle size with ata-validator (generated code and its interpreter), Ajv, Zod (native schemas, and `z.fromJSONSchema` for the test suite), TypeBox, @cfworker/json-schema and omptype. diff --git a/packages/schema/README.md b/packages/schema/README.md index 6654fe54d..f1ff34f4d 100644 --- a/packages/schema/README.md +++ b/packages/schema/README.md @@ -77,6 +77,12 @@ compile(Block).errors({ kind: "video" }); // [{ path: "/kind", keyword: "anyOf", message: 'expected one of ["text","image"]' }] ``` +`Static` also types schemas built with TypeBox 1.x, read from TypeBox's type shapes (pi-schema does not import +TypeBox): `String`, `Number`, `Integer`, `Boolean`, `Null`, `Literal`, `Array`, `Object` (with `Type.Optional` and +`Type.Readonly` properties), `Union`, `Record`, `Unsafe`, `Unknown` and `Any` give TypeBox's own static type, so +`Static` is `{ path: string }`. Other TypeBox kinds (`Enum`, `Tuple`, +`Intersect`, `Cyclic`, ...) give `unknown` where they stand. Other JSON Schema objects give `unknown`. + ### Options ```typescript diff --git a/packages/schema/src/builders.ts b/packages/schema/src/builders.ts index 1f4e21a13..2dc61a520 100644 --- a/packages/schema/src/builders.ts +++ b/packages/schema/src/builders.ts @@ -2,14 +2,27 @@ // they accept, so `Static` is the TypeScript type a valid value has. The objects are ordinary data: // they serialize with JSON.stringify and compile with compile() like any other schema. +import type { TypeBoxStatic } from "./typebox-static.ts"; + declare const STATIC: unique symbol; declare const OPTIONAL_TYPE: unique symbol; /** A JSON Schema object whose valid values have type `T` (a type-level brand; nothing extra exists at run time). */ export type Schema = { readonly [keyword: string]: unknown } & { readonly [STATIC]?: T }; -/** The type of the values `S` accepts; `unknown` for schemas not built with these functions. */ -export type Static = S extends { readonly [STATIC]?: infer T } ? (unknown extends T ? unknown : T) : unknown; +// TypeBox is tested first and in its own branch: a builder brand's fallback through TypeBoxStatic would leave +// `Static` of a generic `S` (constrained to builder schemas) unresolvable to its constraint's types. +/** + * The type of the values `S` accepts: the type a builder recorded; for a TypeBox schema (`'~kind'` or `'~unsafe'`), + * TypeBox's static type for the common kinds (see `TypeBoxStatic`); else `unknown`. + */ +export type Static = S extends { readonly "~kind": unknown } | { readonly "~unsafe": unknown } + ? TypeBoxStatic + : S extends { readonly [STATIC]?: infer T } + ? unknown extends T + ? unknown + : T + : unknown; /** Annotations every builder accepts: they describe the schema and never change what it accepts. */ export interface Annotations { diff --git a/packages/schema/src/typebox-static.ts b/packages/schema/src/typebox-static.ts new file mode 100644 index 000000000..ce9f29b51 --- /dev/null +++ b/packages/schema/src/typebox-static.ts @@ -0,0 +1,76 @@ +// The static types of TypeBox-built schemas, read from the type shapes TypeBox 1.x declares (`'~kind'`, `'~optional'`, +// `'~readonly'`, `'~unsafe'`) without importing TypeBox, so `Static` also types tool arguments of TypeBox schemas. +// Only the common kinds are mapped; every other kind (Enum, Tuple, Intersect, Cyclic, Ref, ...) is `unknown`. + +import type { Static } from "./builders.ts"; + +/** The value type of a TypeBox schema type by its `'~kind'`, or its `Type.Unsafe` type; `unknown` for other kinds. */ +export type TypeBoxStatic = S extends { readonly "~kind": infer K } + ? KindStatic + : S extends { readonly "~unsafe": infer T } + ? T + : unknown; + +interface Primitives { + String: string; + Number: number; + Integer: number; + Boolean: boolean; + Null: null; + Unknown: unknown; + Any: any; +} + +type KindStatic = K extends keyof Primitives + ? Primitives[K] + : K extends "Literal" + ? S extends { readonly const: infer V } + ? V + : unknown + : K extends "Array" + ? S extends { readonly items: infer I } + ? Static[] + : unknown + : K extends "Object" + ? S extends { readonly properties: infer P } + ? ObjectStatic

    + : unknown + : K extends "Union" + ? S extends { readonly anyOf: readonly (infer B)[] } + ? Static + : unknown + : K extends "Record" + ? S extends { readonly patternProperties: infer P } + ? Record, Static> + : unknown + : unknown; + +type Optional = { readonly "~optional": true }; + +type Readonly = { readonly "~readonly": true }; + +// As TypeBox: no properties is `object`; `'~optional'` properties may be omitted, `'~readonly'` ones are readonly. +type ObjectStatic

    = keyof P extends never + ? object + : Simplify< + { -readonly [K in keyof P as P[K] extends Optional | Readonly ? never : K]: Static } & { + -readonly [K in keyof P as P[K] extends Optional ? (P[K] extends Readonly ? never : K) : never]?: Static< + P[K] + >; + } & { + readonly [K in keyof P as P[K] extends Readonly ? (P[K] extends Optional ? never : K) : never]: Static< + P[K] + >; + } & { + readonly [K in keyof P as P[K] extends Optional ? (P[K] extends Readonly ? K : never) : never]?: Static< + P[K] + >; + } + >; + +type Simplify = { [K in keyof T]: T[K] } & {}; + +// TypeBox's key patterns for integer and number keys; every other pattern types its keys as strings. +type RecordKey = Pattern extends "^-?(?:0|[1-9][0-9]*)$" | "^-?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?$" + ? number + : string; diff --git a/packages/schema/test/typebox-static.test.ts b/packages/schema/test/typebox-static.test.ts new file mode 100644 index 000000000..c62ec03f9 --- /dev/null +++ b/packages/schema/test/typebox-static.test.ts @@ -0,0 +1,72 @@ +// Static for TypeBox-built schemas: the common kinds give TypeBox's own static type, without pi-schema importing +// TypeBox; other kinds give `unknown`. Type-level tests (they pass when this file type-checks), against TypeBox 1.3.27. + +import { Type, type Static as TypeBoxOwn } from "typebox"; +import { describe, expectTypeOf, it } from "vitest"; +import { compile, type Static } from "../src/index.ts"; + +describe("Static of TypeBox schemas", () => { + it("maps the primitive kinds, literals and Unsafe", () => { + expectTypeOf>>().toEqualTypeOf(); + expectTypeOf>>().toEqualTypeOf(); + expectTypeOf>>().toEqualTypeOf(); + expectTypeOf>>().toEqualTypeOf(); + expectTypeOf>>().toEqualTypeOf(); + expectTypeOf>>().toEqualTypeOf(); + const any = Type.Any(); + expectTypeOf>().toBeAny(); + const literal = Type.Literal("add"); + expectTypeOf>().toEqualTypeOf<"add">(); + const unsafe = Type.Unsafe<"a" | "b">({ type: "string", enum: ["a", "b"] }); + expectTypeOf>().toEqualTypeOf<"a" | "b">(); + }); + + it("maps objects with optional and readonly properties as TypeBox does", () => { + const schema = Type.Object({ + path: Type.String(), + offset: Type.Optional(Type.Integer()), + id: Type.Readonly(Type.String()), + tag: Type.Readonly(Type.Optional(Type.Literal("x"))), + }); + type Expected = { path: string; offset?: number; readonly id: string; readonly tag?: "x" }; + expectTypeOf>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf>(); + const empty = Type.Object({}); + expectTypeOf>().toEqualTypeOf>(); + }); + + it("maps arrays, unions and records, nested", () => { + const schema = Type.Object({ + edits: Type.Array(Type.Object({ oldText: Type.String(), newText: Type.String() })), + mode: Type.Union([Type.Literal("text"), Type.Literal("binary"), Type.Null()]), + env: Type.Record(Type.String(), Type.Union([Type.String(), Type.Number()])), + counts: Type.Record(Type.Integer(), Type.Boolean()), + }); + type Expected = { + edits: { oldText: string; newText: string }[]; + mode: "text" | "binary" | null; + env: Record; + counts: Record; + }; + expectTypeOf>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf>(); + }); + + it("gives unknown for kinds it does not map", () => { + const enumeration = Type.Enum(["a", "b"]); + const tuple = Type.Tuple([Type.String()]); + const intersect = Type.Intersect([Type.Object({ a: Type.String() }), Type.Object({ b: Type.String() })]); + expectTypeOf>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf(); + // Inside a mapped kind, an unmapped kind is unknown only where it stands. + const object = Type.Object({ pair: Type.Tuple([Type.String()]), name: Type.String() }); + expectTypeOf>().toEqualTypeOf<{ pair: unknown; name: string }>(); + }); + + it("types compile() and parse() for TypeBox schemas", () => { + const schema = Type.Object({ n: Type.Number(), s: Type.Optional(Type.String()) }); + const result = compile(schema, { coerce: true }).parse({ n: "1" }); + if (result.ok) expectTypeOf(result.value).toEqualTypeOf<{ n: number; s?: string }>(); + }); +}); From 0f21a3319d83d3a17176cf0850f69b3d9cc3f5ca Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Sun, 1 Nov 2026 14:40:17 +0100 Subject: [PATCH 236/271] fix(coding-agent,tui): reuse syntax highlighting and rendered Markdown blocks Codemode and write rows keep their last highlighting in a CodeHighlighter that checks theme and grammar changes itself. Markdown reuses the top-level blocks of its last render, held weakly, and assistant messages reuse their Markdown instances while streaming. Text.setText() keeps its wrapped lines for unchanged text. closes #10971 --- packages/coding-agent/CHANGELOG.md | 1 + .../src/core/tools/renderers/write.ts | 41 ++++-- .../src/extensions/codemode/renderer.ts | 14 +- .../components/assistant-message.ts | 85 +++++++---- .../components/markdown-transform.ts | 5 +- .../interactive/components/user-message.ts | 2 +- .../src/modes/interactive/theme/theme.ts | 64 ++++++--- .../coding-agent/src/utils/highlight-js.d.ts | 1 + .../src/utils/syntax-highlight.ts | 8 ++ .../test/assistant-message.test.ts | 34 ++++- .../test/code-highlighter.test.ts | 71 +++++++++ packages/tui/CHANGELOG.md | 2 + packages/tui/src/components/markdown.ts | 135 ++++++++++++------ packages/tui/src/components/text.ts | 2 + packages/tui/test/markdown.test.ts | 59 ++++++++ packages/tui/test/text.test.ts | 12 ++ 16 files changed, 420 insertions(+), 116 deletions(-) create mode 100644 packages/coding-agent/test/code-highlighter.test.ts create mode 100644 packages/tui/test/text.test.ts diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index ae5a6f324..d391a4f56 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -51,6 +51,7 @@ - Fixed `tool_search` and codemode `searchTools()` ranking a tool below tools that only mention its name in their descriptions, for example `peekaboo app` ranking `mcp__peekaboo__app` 14th. A tool matching more of the query's words now ranks above one matching fewer, a query word in a tool's name scores more than any number of mentions in its description, a tool whose whole name is in the query gets a large bonus, and long server instructions no longer lower the scores of that server's tools ([#11331](https://github.com/earendil-works/pi/issues/11331)) - Fixed `tool_search` and codemode `searchTools()` finding nothing for Chinese, Japanese, Thai, and other non-ASCII queries, because the tokenizer kept only ASCII letters and digits. Words in any script now match, text without spaces between words such as `搜索记录` is split into words, and canonically equivalent text such as `café` and `cafe\u0301` matches ([#10816](https://github.com/earendil-works/pi/issues/10816), [#11037](https://github.com/earendil-works/pi/pull/11037) by [@haoqixu](https://github.com/haoqixu)) - Fixed `pi update` stopping with `git clean -fdx failed with code 1` on Windows when another pi session had a git extension's native module or executable open. Windows cannot delete such files, so the update left the new code with the old dependencies. pi now warns, installs the dependencies anyway, and retries the clean on the next update ([#11597](https://github.com/earendil-works/pi/issues/11597)) +- Fixed tool rows and streamed answers highlighting the same code again on every redraw. A running codemode row highlighted its whole script on every progress update and timer tick, a complete `write` row its whole file on every update and expand toggle, a streamed `write` its first 50 lines on every delta, and a streamed answer every earlier code block on every delta. Rows now reuse their last highlighting until the code, theme, or loaded grammars change, and assistant messages keep their rendered Markdown blocks while streaming ([#10971](https://github.com/earendil-works/pi/issues/10971)) ### Removed diff --git a/packages/coding-agent/src/core/tools/renderers/write.ts b/packages/coding-agent/src/core/tools/renderers/write.ts index 7c8f32636..8e2338a7d 100644 --- a/packages/coding-agent/src/core/tools/renderers/write.ts +++ b/packages/coding-agent/src/core/tools/renderers/write.ts @@ -8,7 +8,12 @@ import { Container, Text } from "@earendil-works/pi-tui"; import { keyHint } from "../../../modes/interactive/components/keybinding-hints.ts"; -import { getLanguageFromPath, highlightCode, type Theme } from "../../../modes/interactive/theme/theme.ts"; +import { + CodeHighlighter, + getLanguageFromPath, + highlightCode, + type Theme, +} from "../../../modes/interactive/theme/theme.ts"; import type { ToolDefinition, ToolRenderResultOptions } from "../../extensions/types.ts"; import { normalizeDisplayText, renderToolPath, replaceTabs, str } from "../render-utils.ts"; @@ -20,7 +25,12 @@ type WriteHighlightCache = { highlightedLines: string[]; }; class WriteCallRenderComponent extends Text { + // Line by line highlighting of a call whose content is still streaming. cache?: WriteHighlightCache; + // A streamed call highlights its first lines on every delta, which stop changing once the file is longer, and a + // complete call highlights the whole file on every update and expand toggle. Either way the row reuses the last + // highlighting; it also holds the only copy of a complete call's displayed source. + readonly highlighter = new CodeHighlighter(); constructor() { super("", 0, 0); @@ -31,11 +41,11 @@ function highlightSingleLine(line: string, lang: string): string { const highlighted = highlightCode(line, lang); return highlighted[0] ?? ""; } -function refreshWriteHighlightPrefix(cache: WriteHighlightCache): void { +function refreshWriteHighlightPrefix(cache: WriteHighlightCache, highlighter: CodeHighlighter): void { const prefixCount = Math.min(WRITE_PARTIAL_FULL_HIGHLIGHT_LINES, cache.normalizedLines.length); if (prefixCount === 0) return; const prefixSource = cache.normalizedLines.slice(0, prefixCount).join("\n"); - const prefixHighlighted = highlightCode(prefixSource, cache.lang); + const prefixHighlighted = highlighter.highlight(prefixSource, cache.lang); for (let i = 0; i < prefixCount; i++) { cache.highlightedLines[i] = prefixHighlighted[i] ?? highlightSingleLine(cache.normalizedLines[i] ?? "", cache.lang); @@ -58,12 +68,14 @@ function updateWriteHighlightCacheIncremental( cache: WriteHighlightCache | undefined, rawPath: string | null, fileContent: string, + highlighter: CodeHighlighter, ): WriteHighlightCache | undefined { const lang = rawPath ? getLanguageFromPath(rawPath) : undefined; if (!lang) return undefined; if (!cache) return rebuildWriteHighlightCacheFull(rawPath, fileContent); - if (cache.lang !== lang || cache.rawPath !== rawPath) return rebuildWriteHighlightCacheFull(rawPath, fileContent); - if (!fileContent.startsWith(cache.rawContent)) return rebuildWriteHighlightCacheFull(rawPath, fileContent); + if (cache.lang !== lang || cache.rawPath !== rawPath || !fileContent.startsWith(cache.rawContent)) { + return rebuildWriteHighlightCacheFull(rawPath, fileContent); + } if (fileContent.length === cache.rawContent.length) return cache; const deltaRaw = fileContent.slice(cache.rawContent.length); @@ -83,10 +95,10 @@ function updateWriteHighlightCacheIncremental( cache.normalizedLines.push(segments[i]); cache.highlightedLines.push(highlightSingleLine(segments[i], cache.lang)); } - refreshWriteHighlightPrefix(cache); + refreshWriteHighlightPrefix(cache, highlighter); return cache; } -function trimTrailingEmptyLines(lines: string[]): string[] { +function trimTrailingEmptyLines(lines: readonly string[]): string[] { let end = lines.length; while (end > 0 && lines[end - 1] === "") { end--; @@ -98,6 +110,7 @@ function formatWriteCall( options: ToolRenderResultOptions, theme: Theme, cache: WriteHighlightCache | undefined, + highlighter: CodeHighlighter, cwd: string, ): string { const rawPath = str(args?.file_path ?? args?.path); @@ -110,7 +123,7 @@ function formatWriteCall( } else if (fileContent) { const lang = rawPath ? getLanguageFromPath(rawPath) : undefined; const renderedLines = lang - ? (cache?.highlightedLines ?? highlightCode(replaceTabs(normalizeDisplayText(fileContent)), lang)) + ? (cache?.highlightedLines ?? highlighter.highlight(replaceTabs(normalizeDisplayText(fileContent)), lang)) : normalizeDisplayText(fileContent).split("\n"); const lines = trimTrailingEmptyLines(renderedLines); const totalLines = lines.length; @@ -149,19 +162,17 @@ export const writeRenderers: Pick, "renderCall" | "rend const fileContent = str(renderArgs?.content); const component = (context.lastComponent as WriteCallRenderComponent | undefined) ?? new WriteCallRenderComponent(); - if (fileContent !== null) { - component.cache = context.argsComplete - ? rebuildWriteHighlightCacheFull(rawPath, fileContent) - : updateWriteHighlightCacheIncremental(component.cache, rawPath, fileContent); - } else { - component.cache = undefined; - } + component.cache = + fileContent !== null && !context.argsComplete + ? updateWriteHighlightCacheIncremental(component.cache, rawPath, fileContent, component.highlighter) + : undefined; component.setText( formatWriteCall( renderArgs, { expanded: context.expanded, isPartial: context.isPartial }, theme, component.cache, + component.highlighter, context.cwd, ), ); diff --git a/packages/coding-agent/src/extensions/codemode/renderer.ts b/packages/coding-agent/src/extensions/codemode/renderer.ts index 8d2937351..660803e68 100644 --- a/packages/coding-agent/src/extensions/codemode/renderer.ts +++ b/packages/coding-agent/src/extensions/codemode/renderer.ts @@ -12,7 +12,7 @@ import type { ToolDefinition } from "../../core/extensions/types.ts"; import { getTextOutput, replaceTabs, startToolTimer, str, toolTimerText } from "../../core/tools/render-utils.ts"; import { keyHint } from "../../modes/interactive/components/keybinding-hints.ts"; import { VisualLinePreview } from "../../modes/interactive/components/visual-truncate.ts"; -import { highlightCode, type Theme } from "../../modes/interactive/theme/theme.ts"; +import { CodeHighlighter, type Theme } from "../../modes/interactive/theme/theme.ts"; import type { CodemodeNestedCall, CodemodeToolDetails } from "./tool.ts"; const CODE_PREVIEW_LINES = 10; @@ -21,6 +21,12 @@ const OUTPUT_PREVIEW_LINES = 5; const COLLAPSED_ARGS_CHARS = 80; const SCRIPT_HEADER = /^Script (completed|failed)\nWall time [\d.]+ seconds\nOutput:\n$/; +/** Renderer state of a codemode row. */ +interface CodemodeRenderState { + // Progress, timer ticks and expanding redraw the call, so the row keeps the script's highlighting. + scriptHighlighter?: CodeHighlighter; +} + function expandHint(theme: Theme, hidden: number, noun: string): string { return `${theme.fg("muted", `... (${hidden} more ${noun},`)} ${keyHint("app.tools.expand", "to expand")}${theme.fg("muted", ")")}`; } @@ -79,7 +85,11 @@ export const codemodeRenderers: Pick< } component.addChild(new Text(title, 0, 0)); if (code) { - const highlighted = highlightCode(replaceTabs(code.replace(/\r/g, "").trimEnd()), "javascript").join("\n"); + const state = context.state as CodemodeRenderState; + state.scriptHighlighter ??= new CodeHighlighter(); + const highlighted = state.scriptHighlighter + .highlight(replaceTabs(code.replace(/\r/g, "").trimEnd()), "javascript") + .join("\n"); component.addChild( context.expanded ? new Text(highlighted, 0, 0) diff --git a/packages/coding-agent/src/modes/interactive/components/assistant-message.ts b/packages/coding-agent/src/modes/interactive/components/assistant-message.ts index 67bb71a3f..7cb296031 100644 --- a/packages/coding-agent/src/modes/interactive/components/assistant-message.ts +++ b/packages/coding-agent/src/modes/interactive/components/assistant-message.ts @@ -16,6 +16,15 @@ const OSC133_ZONE_START = "\x1b]133;A\x07"; const OSC133_ZONE_END = "\x1b]133;B\x07"; const OSC133_ZONE_FINAL = "\x1b]133;C\x07"; +/** + * A Markdown block of the last update. The next update reuses it at the same position for the same kind of content, so + * a streamed answer keeps the blocks it already rendered, also once the stream ends. + */ +interface MarkdownSlot { + kind: "text" | "thinking"; + markdown: Markdown; +} + /** * Component that renders a complete assistant message */ @@ -32,6 +41,7 @@ export class AssistantMessageComponent extends Container { private messageId?: string; private timestamp?: string; private thinkingVisibilityOverrides = new Map(); + private markdownSlots: MarkdownSlot[] = []; constructor( message?: AssistantMessage, @@ -86,6 +96,8 @@ export class AssistantMessageComponent extends Container { setOutputPad(padding: number): void { this.outputPad = padding; + // Markdown takes its padding at construction. + this.markdownSlots = []; if (this.lastMessage) { this.updateContent(this.lastMessage); } @@ -108,6 +120,15 @@ export class AssistantMessageComponent extends Container { // Clear content container this.contentContainer.clear(); + const previousSlots = this.markdownSlots; + this.markdownSlots = []; + const markdownBlock = (kind: MarkdownSlot["kind"], text: string, create: () => Markdown): Markdown => { + const slot = previousSlots[this.markdownSlots.length]; + const markdown = slot?.kind === kind ? slot.markdown : create(); + markdown.setText(text); + this.markdownSlots.push({ kind, markdown }); + return markdown; + }; // Spacers are added lazily, right before the next visible content block. This puts a blank // line before the first visible block and after each visible thinking run that is followed @@ -128,16 +149,22 @@ export class AssistantMessageComponent extends Container { if (content.type === "text" && content.text.trim()) { // Assistant text messages with no background - trim the text // Set paddingY=0 to avoid extra spacing before tool executions + const text = content.text.trim(); addVisibleChild( - new Markdown(content.text.trim(), this.outputPad, 0, this.markdownTheme, undefined, { - transform: createMarkdownTransform( - "assistant", - this.isStreaming, - this.markdownTransformers, - this.messageId, - this.timestamp, - ), - }), + markdownBlock( + "text", + text, + () => + new Markdown(text, this.outputPad, 0, this.markdownTheme, undefined, { + transform: createMarkdownTransform( + "assistant", + () => this.isStreaming, + this.markdownTransformers, + this.messageId, + this.timestamp, + ), + }), + ), ); } else if (content.type === "thinking") { const thinkingBlocks: string[] = []; @@ -163,26 +190,32 @@ export class AssistantMessageComponent extends Container { if (hidden && !this.hiddenThinkingLabel.trim()) { continue; } + const thinking = thinkingBlocks.join("\n\n"); const thinkingComponent = hidden ? new Text(theme.italic(theme.fg("thinkingText", this.hiddenThinkingLabel)), this.outputPad, 0) - : new Markdown( - thinkingBlocks.join("\n\n"), - this.outputPad, - 0, - this.markdownTheme, - { - color: (text: string) => theme.fg("thinkingText", text), - italic: true, - }, - { - transform: createMarkdownTransform( - "assistant-thinking", - this.isStreaming, - this.markdownTransformers, - this.messageId, - this.timestamp, + : markdownBlock( + "thinking", + thinking, + () => + new Markdown( + thinking, + this.outputPad, + 0, + this.markdownTheme, + { + color: (text: string) => theme.fg("thinkingText", text), + italic: true, + }, + { + transform: createMarkdownTransform( + "assistant-thinking", + () => this.isStreaming, + this.markdownTransformers, + this.messageId, + this.timestamp, + ), + }, ), - }, ); addVisibleChild( new MouseRegion(thinkingComponent, (event) => { diff --git a/packages/coding-agent/src/modes/interactive/components/markdown-transform.ts b/packages/coding-agent/src/modes/interactive/components/markdown-transform.ts index 4ff7685c2..58bae9094 100644 --- a/packages/coding-agent/src/modes/interactive/components/markdown-transform.ts +++ b/packages/coding-agent/src/modes/interactive/components/markdown-transform.ts @@ -1,8 +1,9 @@ import type { MarkdownTransformContext, MarkdownTransformer } from "../../../core/extensions/types.ts"; +/** Applies extension Markdown transformers. `isStreaming` is read per transform, so a reused component sees a stream end. */ export function createMarkdownTransform( messageType: MarkdownTransformContext["messageType"], - isStreaming: boolean, + isStreaming: () => boolean, transformers: readonly MarkdownTransformer[], messageId?: string, timestamp?: string, @@ -10,7 +11,7 @@ export function createMarkdownTransform( return (markdown, availableWidth) => applyMarkdownTransformers( markdown, - { messageType, isStreaming, availableWidth, messageId, timestamp }, + { messageType, isStreaming: isStreaming(), availableWidth, messageId, timestamp }, transformers, ); } diff --git a/packages/coding-agent/src/modes/interactive/components/user-message.ts b/packages/coding-agent/src/modes/interactive/components/user-message.ts index d8ca373cc..e313e0242 100644 --- a/packages/coding-agent/src/modes/interactive/components/user-message.ts +++ b/packages/coding-agent/src/modes/interactive/components/user-message.ts @@ -60,7 +60,7 @@ export class UserMessageComponent extends Container { preserveBackslashEscapes: true, transform: createMarkdownTransform( "user", - false, + () => false, this.markdownTransformers, this.messageId, this.timestamp, diff --git a/packages/coding-agent/src/modes/interactive/theme/theme.ts b/packages/coding-agent/src/modes/interactive/theme/theme.ts index 876ba9e86..922b23d9f 100644 --- a/packages/coding-agent/src/modes/interactive/theme/theme.ts +++ b/packages/coding-agent/src/modes/interactive/theme/theme.ts @@ -26,7 +26,7 @@ import chalk from "chalk"; import { getCustomThemesDir, getThemesDir } from "../../../config.ts"; import type { SourceInfo } from "../../../core/source-info.ts"; import { closeWatcher, watchWithErrorHandler } from "../../../utils/fs-watch.ts"; -import { highlight, supportsLanguage } from "../../../utils/syntax-highlight.ts"; +import { getHighlightLanguageCount, highlight, supportsLanguage } from "../../../utils/syntax-highlight.ts"; import { stripBom } from "../../../utils/text.ts"; import { generateSystemThemeColors, SYSTEM_THEME_NAME, terminalAppearance } from "./system-theme.ts"; import { @@ -658,6 +658,11 @@ export const theme: Theme = new Proxy({} as Theme, { }, }); +/** The theme instance behind the `theme` proxy, whose own identity never changes. */ +function getGlobalTheme(): Theme | undefined { + return (globalThis as Record)[THEME_KEY]; +} + function setGlobalTheme(t: Theme): void { (globalThis as Record)[THEME_KEY] = t; (globalThis as Record)[THEME_KEY_OLD] = t; @@ -944,7 +949,41 @@ export function highlightCode(code: string, lang?: string): string[] { try { return highlight(code, opts).split("\n"); } catch { - return code.split("\n"); + return code.split("\n").map((line) => theme.fg("mdCodeBlock", line)); + } +} + +/** + * Highlights code for one owner, such as a tool row, and reuses the last result while the code, the language, the + * theme and the registered grammars stay the same. Each owner keeps its own highlighter, so cached results live as long + * as the code they show instead of in a process-wide cache. + */ +export class CodeHighlighter { + private last?: { + code: string; + lang: string | undefined; + theme: Theme | undefined; + languageCount: number; + lines: readonly string[]; + }; + + /** Like {@link highlightCode}. The result is shared with later calls, so callers must not modify it. */ + highlight(code: string, lang: string | undefined): readonly string[] { + // Renderers cannot tell a theme change from any other redraw, so the highlighter checks what its result depends on. + const currentTheme = getGlobalTheme(); + const languageCount = getHighlightLanguageCount(); + const last = this.last; + if ( + last?.code === code && + last.lang === lang && + last.theme === currentTheme && + last.languageCount === languageCount + ) { + return last.lines; + } + const lines = highlightCode(code, lang); + this.last = { code, lang, theme: currentTheme, languageCount, lines }; + return lines; } } @@ -1035,26 +1074,7 @@ export function getMarkdownTheme(): MarkdownTheme { italic: (text: string) => theme.italic(text), underline: (text: string) => theme.underline(text), strikethrough: (text: string) => theme.strikethrough(text), - highlightCode: (code: string, lang?: string): string[] => { - // Validate language before highlighting to avoid stderr spam from cli-highlight - const validLang = lang && supportsLanguage(lang) ? lang : undefined; - // Skip highlighting when no valid language is specified. cli-highlight's - // auto-detection is unreliable and can misidentify prose as AppleScript, - // LiveCodeServer, etc., coloring random English words as keywords. - if (!validLang) { - return code.split("\n").map((line) => theme.fg("mdCodeBlock", line)); - } - const opts = { - language: validLang, - ignoreIllegals: true, - theme: getCliHighlightTheme(theme), - }; - try { - return highlight(code, opts).split("\n"); - } catch { - return code.split("\n").map((line) => theme.fg("mdCodeBlock", line)); - } - }, + highlightCode, }; } diff --git a/packages/coding-agent/src/utils/highlight-js.d.ts b/packages/coding-agent/src/utils/highlight-js.d.ts index d4b71174e..c4c951fce 100644 --- a/packages/coding-agent/src/utils/highlight-js.d.ts +++ b/packages/coding-agent/src/utils/highlight-js.d.ts @@ -17,6 +17,7 @@ interface HighlightJsApi { highlight(code: string, options: HighlightJsOptions): HighlightJsResult; highlightAuto(code: string, languageSubset?: string[]): HighlightJsResult; getLanguage(name: string): HighlightJsLanguageDefinition | undefined; + listLanguages(): string[]; registerLanguage(name: string, language: HighlightJsLanguageFactory): void; } diff --git a/packages/coding-agent/src/utils/syntax-highlight.ts b/packages/coding-agent/src/utils/syntax-highlight.ts index 1ee1986b9..91375bb01 100644 --- a/packages/coding-agent/src/utils/syntax-highlight.ts +++ b/packages/coding-agent/src/utils/syntax-highlight.ts @@ -217,3 +217,11 @@ export function highlight(code: string, options: HighlightOptions = {}): string export function supportsLanguage(name: string): boolean { return hljs.getLanguage(name) !== undefined; } + +/** + * Number of registered grammars. It grows when the deferred grammars load, which can change the highlighting of + * code that embeds another language, such as HTML in a JavaScript template literal. + */ +export function getHighlightLanguageCount(): number { + return hljs.listLanguages().length; +} diff --git a/packages/coding-agent/test/assistant-message.test.ts b/packages/coding-agent/test/assistant-message.test.ts index 42f6e496d..2cf368915 100644 --- a/packages/coding-agent/test/assistant-message.test.ts +++ b/packages/coding-agent/test/assistant-message.test.ts @@ -1,10 +1,11 @@ import type { AssistantMessage } from "@earendil-works/pi-ai"; import type { TuiMouseEvent } from "@earendil-works/pi-tui"; -import { describe, expect, test } from "vitest"; +import { afterEach, describe, expect, test, vi } from "vitest"; import { AssistantMessageComponent } from "../src/modes/interactive/components/assistant-message.ts"; import { UserMessageComponent } from "../src/modes/interactive/components/user-message.ts"; import { initTheme } from "../src/modes/interactive/theme/theme.ts"; import { stripAnsi } from "../src/utils/ansi.ts"; +import * as syntaxHighlight from "../src/utils/syntax-highlight.ts"; const OSC133_ZONE_START = "\x1b]133;A\x07"; const OSC133_ZONE_END = "\x1b]133;B\x07"; @@ -34,6 +35,37 @@ function createAssistantMessage( } describe("AssistantMessageComponent", () => { + afterEach(() => vi.restoreAllMocks()); + + test("reuses its Markdown across streaming and its end, and renders like a new component after a theme change", () => { + initTheme("dark"); + const highlight = vi.spyOn(syntaxHighlight, "highlight"); + const streaming: boolean[] = []; + const code = "```ts\nconst a = 1;\n```"; + const content = (text: string): AssistantMessage["content"] => [ + { type: "thinking", thinking: "Thinking with **bold** text" }, + { type: "text", text }, + ]; + const component = new AssistantMessageComponent(undefined, false, undefined, "Thinking...", 1, [ + (markdown, { isStreaming }) => { + streaming.push(isStreaming); + return markdown; + }, + ]); + component.updateContent(createAssistantMessage(content(code)), true); + component.render(80); + const message = createAssistantMessage(content(`${code}\n\nThe end.`)); + component.updateContent(message, false); + component.render(80); + expect(highlight).toHaveBeenCalledTimes(1); + expect(streaming.at(-1)).toBe(false); + + initTheme("light"); + component.invalidate(); + expect(component.render(80)).toEqual(new AssistantMessageComponent(message).render(80)); + initTheme("dark"); + }); + test("adds OSC 133 zone markers to assistant messages without tool calls", () => { initTheme("dark"); diff --git a/packages/coding-agent/test/code-highlighter.test.ts b/packages/coding-agent/test/code-highlighter.test.ts new file mode 100644 index 000000000..b0d0433b4 --- /dev/null +++ b/packages/coding-agent/test/code-highlighter.test.ts @@ -0,0 +1,71 @@ +import type { TUI } from "@earendil-works/pi-tui"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { writeRenderers } from "../src/core/tools/renderers/write.ts"; +import { codemodeRenderers } from "../src/extensions/codemode/renderer.ts"; +import { ToolExecutionComponent } from "../src/modes/interactive/components/tool-execution.ts"; +import { CodeHighlighter, highlightCode, initTheme } from "../src/modes/interactive/theme/theme.ts"; +import * as syntaxHighlight from "../src/utils/syntax-highlight.ts"; + +const tui = { requestRender: () => {} } as unknown as TUI; + +describe("CodeHighlighter", () => { + beforeEach(() => initTheme("dark")); + afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); + }); + + it("reuses its last result until the code, the theme or the grammars change", async () => { + const code = 'const page = html`

    ok
    `;'; + const highlighter = new CodeHighlighter(); + const first = highlighter.highlight(code, "javascript"); + expect(highlighter.highlight(code, "javascript")).toBe(first); + expect(highlighter.highlight("const a = 1;", "javascript")).not.toBe(first); + + initTheme("light"); + expect(highlighter.highlight(code, "javascript")).toEqual(highlightCode(code, "javascript")); + const light = highlighter.highlight(code, "javascript"); + await syntaxHighlight.loadAllHighlightLanguages(); + expect(highlighter.highlight(code, "javascript")).not.toEqual(light); + }); + + it("codemode highlights the script once across progress, timer ticks, expanding and completion", () => { + vi.useFakeTimers(); + const highlight = vi.spyOn(syntaxHighlight, "highlight"); + const row = new ToolExecutionComponent( + "codemode", + "c", + { code: "await tools.read({});" }, + {}, + codemodeRenderers, + tui, + "/", + ); + row.setArgsComplete(); + row.markExecutionStarted(); + row.updateResult({ content: [], details: { calls: [] }, isError: false }, true); + vi.advanceTimersByTime(2_000); + row.setExpanded(true); + row.updateResult({ content: [{ type: "text", text: "done" }], details: { calls: [] }, isError: false }, false); + row.render(80); + expect(highlight).toHaveBeenCalledTimes(1); + }); + + it("write highlights unchanged first lines and a complete file only once", () => { + const content = Array.from({ length: 60 }, (_, i) => `const v${i} = ${i};`).join("\n"); + const row = new ToolExecutionComponent("write", "w", { path: "a.ts", content }, {}, writeRenderers, tui, "/"); + row.updateArgs({ path: "a.ts", content: `${content} //` }); + row.render(80); + const highlight = vi.spyOn(syntaxHighlight, "highlight"); + // Only the changed last line is highlighted again. + row.updateArgs({ path: "a.ts", content: `${content} // more` }); + row.render(80); + expect(highlight).toHaveBeenCalledTimes(1); + + row.setArgsComplete(); + row.setExpanded(true); + row.setExpanded(false); + row.render(80); + expect(highlight).toHaveBeenCalledTimes(2); + }); +}); diff --git a/packages/tui/CHANGELOG.md b/packages/tui/CHANGELOG.md index d03c3cfc8..1b13cf0a7 100644 --- a/packages/tui/CHANGELOG.md +++ b/packages/tui/CHANGELOG.md @@ -18,6 +18,8 @@ - Fixed display math wider than the content width being wrapped row by row, which split fractions and sums apart. It now falls back to the raw LaTeX source ([#11483](https://github.com/earendil-works/pi/issues/11483)) - Fixed `tui.select.pageUp` and `tui.select.pageDown` being ignored by `SelectList`, `SettingsList`, and editor autocomplete ([#7629](https://github.com/earendil-works/pi/issues/7629), based on [#7680](https://github.com/earendil-works/pi/pull/7680) by [@muyiyr](https://github.com/muyiyr)) - Fixed `wrapTextWithAnsi()` starting the next line with a space when a styled word ended exactly at the wrap width, and strikethrough continuing into the right padding of wrapped lines ([#10925](https://github.com/earendil-works/pi/issues/10925)) +- Fixed `Markdown` styling, highlighting, and wrapping every block on each render. It now reuses the top-level blocks of its last render, so a streaming message renders only its changed last block, and a resize reuses highlighted code. Reused blocks are held weakly and dropped by `invalidate()`, which a theme change must call because `MarkdownTheme.highlightCode` results are reused until then ([#10971](https://github.com/earendil-works/pi/issues/10971)) +- Fixed `Text.setText()` with unchanged text dropping the wrapped lines, so renderers that set the same text on every update wrapped it again ([#10971](https://github.com/earendil-works/pi/issues/10971)) ## [1.4.0] - 2026-10-21 diff --git a/packages/tui/src/components/markdown.ts b/packages/tui/src/components/markdown.ts index bacfe29f8..a21f75181 100644 --- a/packages/tui/src/components/markdown.ts +++ b/packages/tui/src/components/markdown.ts @@ -1,4 +1,4 @@ -import { Marked, type Token, Tokenizer, type TokenizerExtension, type Tokens } from "marked"; +import { Marked, type Token, Tokenizer, type TokenizerExtension, type Tokens, type TokensList } from "marked"; import markedCjkFriendly from "marked-cjk-friendly"; import { renderLatex } from "../latex.ts"; import { getCapabilities, hyperlink, isImageLine } from "../terminal-image.ts"; @@ -244,6 +244,7 @@ export interface MarkdownTheme { italic: (text: string) => string; strikethrough: (text: string) => string; underline: (text: string) => string; + /** Highlighted lines of a code block. A Markdown component reuses the result for the same code until invalidate(). */ highlightCode?: (code: string, lang?: string) => string[]; /** Prefix applied to each rendered code block line (default: " ") */ codeBlockIndent?: string; @@ -265,6 +266,16 @@ interface InlineStyleContext { stylePrefix: string; } +/** A top-level block of the last render, keyed by its source, reused by the next render. */ +interface RenderedBlock { + /** Width and type of the following block (which decides the spacing) that the lines were rendered for. */ + width: number; + nextTokenType: string | undefined; + lines: string[]; + /** Highlighted lines of the block's code, keyed by language and source; they survive a resize or new spacing. */ + highlightedCode: Map; +} + export class Markdown implements Component { private text: string; private paddingX: number; // Left/right padding @@ -281,7 +292,14 @@ export class Markdown implements Component { // Parsed tokens depend only on the source, so they survive theme and width invalidation. Held weakly: a token tree is // about ten times the size of its source, and every message of a long transcript keeps a Markdown component. The // tokens survive a burst of re-renders, such as a theme preview, and are collected afterwards. - private cachedTokens?: WeakRef<{ source: string; tokens: Token[] }>; + private cachedTokens?: WeakRef<{ source: string; tokens: TokensList }>; + // The last render's top-level blocks. While a message streams only its last block changes, so a render styles and + // wraps only that block. Each render keeps only the blocks it used, which drops a streamed block's earlier versions. + // Held weakly like the tokens: the blocks repeat the source in their keys and the code in their highlighting, and + // only a burst of renders, such as streaming or a resize, needs them. + private blocks?: WeakRef>; + // Highlighted code of the block being rendered, only set while renderBlock runs. + private blockHighlightedCode?: Map; constructor( text: string, @@ -301,10 +319,17 @@ export class Markdown implements Component { setText(text: string): void { this.text = text; - this.invalidate(); + this.clearRenderCache(); } invalidate(): void { + this.clearRenderCache(); + // The theme may have changed, which changes every styled line and the default style's prefix. + this.blocks = undefined; + this.defaultStylePrefix = undefined; + } + + private clearRenderCache(): void { this.cachedText = undefined; this.cachedWidth = undefined; this.cachedLines = undefined; @@ -322,6 +347,7 @@ export class Markdown implements Component { // Don't render anything if there's no actual text if (!text || text.trim() === "") { + this.blocks = undefined; const result: string[] = []; // Update cache this.cachedText = this.text; @@ -342,65 +368,39 @@ export class Markdown implements Component { this.cachedTokens = new WeakRef({ source: normalizedText, tokens }); } - // Convert tokens to styled terminal output - const renderedLines: string[] = []; - + // A reference link resolves through a definition anywhere in the text, so blocks with the same source can render + // differently; such texts are rare in practice and render without reusing blocks. + const reuseBlocks = Object.keys(tokens.links).length === 0; + const previousBlocks = this.blocks?.deref(); + const blocks = new Map(); + this.blocks = new WeakRef(blocks); + const contentLines: string[] = []; for (let i = 0; i < tokens.length; i++) { const token = tokens[i]; - const nextToken = tokens[i + 1]; - const tokenLines = this.renderToken(token, contentWidth, nextToken?.type); - for (const tokenLine of tokenLines) { - renderedLines.push(tokenLine); + const nextTokenType = tokens[i + 1]?.type; + const reused = reuseBlocks ? (blocks.get(token.raw) ?? previousBlocks?.get(token.raw)) : undefined; + const block = reused ?? { width: -1, nextTokenType, lines: [], highlightedCode: new Map() }; + if (block.width !== width || block.nextTokenType !== nextTokenType) { + this.renderBlock(block, token, nextTokenType, width, contentWidth); } - } - - // Wrap lines (NO padding, NO background yet) - const wrappedLines: string[] = []; - for (const line of renderedLines) { - if (isImageLine(line)) { - wrappedLines.push(line); - } else { - for (const wrappedLine of wrapTextWithAnsi(line, contentWidth)) { - wrappedLines.push(wrappedLine); - } - } - } - - // Add margins and background to each wrapped line - const leftMargin = " ".repeat(this.paddingX); - const rightMargin = " ".repeat(this.paddingX); - const bgFn = this.defaultTextStyle?.bgColor; - const contentLines: string[] = []; - - for (const line of wrappedLines) { - if (isImageLine(line)) { + if (reuseBlocks) blocks.set(token.raw, block); + for (const line of block.lines) { contentLines.push(line); - continue; - } - - const lineWithMargins = leftMargin + line + rightMargin; - - if (bgFn) { - contentLines.push(applyBackgroundToLine(lineWithMargins, width, bgFn)); - } else { - // No background - just pad to width - const visibleLen = visibleWidth(lineWithMargins); - const paddingNeeded = Math.max(0, width - visibleLen); - contentLines.push(lineWithMargins + " ".repeat(paddingNeeded)); } } // Add top/bottom padding (empty lines) + const bgFn = this.defaultTextStyle?.bgColor; const emptyLine = " ".repeat(width); const emptyLines: string[] = []; for (let i = 0; i < this.paddingY; i++) { const line = bgFn ? applyBackgroundToLine(emptyLine, width, bgFn) : emptyLine; emptyLines.push(line); } + flattenLines(emptyLines); // Combine top padding, content, and bottom padding const result = emptyLines.concat(contentLines, emptyLines); - flattenLines(result); // Update cache this.cachedText = this.text; @@ -410,6 +410,42 @@ export class Markdown implements Component { return result.length > 0 ? result : [""]; } + /** Render one top-level block to output lines: wrapped, with margins, and padded or backgrounded to the width. */ + private renderBlock( + block: RenderedBlock, + token: Token, + nextTokenType: string | undefined, + width: number, + contentWidth: number, + ): void { + this.blockHighlightedCode = block.highlightedCode; + const leftMargin = " ".repeat(this.paddingX); + const rightMargin = " ".repeat(this.paddingX); + const bgFn = this.defaultTextStyle?.bgColor; + const lines: string[] = []; + for (const line of this.renderToken(token, contentWidth, nextTokenType)) { + if (isImageLine(line)) { + lines.push(line); + continue; + } + for (const wrappedLine of wrapTextWithAnsi(line, contentWidth)) { + const lineWithMargins = leftMargin + wrappedLine + rightMargin; + if (bgFn) { + lines.push(applyBackgroundToLine(lineWithMargins, width, bgFn)); + } else { + // No background - just pad to width + const paddingNeeded = Math.max(0, width - visibleWidth(lineWithMargins)); + lines.push(lineWithMargins + " ".repeat(paddingNeeded)); + } + } + } + this.blockHighlightedCode = undefined; + flattenLines(lines); + block.lines = lines; + block.width = width; + block.nextTokenType = nextTokenType; + } + /** * Apply default text style to a string. * This is the base styling applied to all text content. @@ -567,8 +603,13 @@ export class Markdown implements Component { const indent = this.theme.codeBlockIndent ?? " "; lines.push(this.theme.codeBlockBorder(`\`\`\`${token.lang || ""}`)); if (this.theme.highlightCode) { - const highlightedLines = this.theme.highlightCode(token.text, token.lang); - for (const hlLine of highlightedLines) { + const key = `${token.lang ?? ""}\0${token.text}`; + let highlighted = this.blockHighlightedCode?.get(key); + if (!highlighted) { + highlighted = this.theme.highlightCode(token.text, token.lang); + this.blockHighlightedCode?.set(key, highlighted); + } + for (const hlLine of highlighted) { lines.push(`${indent}${hlLine}`); } } else { diff --git a/packages/tui/src/components/text.ts b/packages/tui/src/components/text.ts index 024748912..c9aea3d13 100644 --- a/packages/tui/src/components/text.ts +++ b/packages/tui/src/components/text.ts @@ -23,6 +23,8 @@ export class Text implements Component { } setText(text: string): void { + // Renderers often set the same text on every update; keep the wrapped lines then. + if (text === this.text) return; this.text = text; this.cachedText = undefined; this.cachedWidth = undefined; diff --git a/packages/tui/test/markdown.test.ts b/packages/tui/test/markdown.test.ts index c1615df66..e0f5b4dd5 100644 --- a/packages/tui/test/markdown.test.ts +++ b/packages/tui/test/markdown.test.ts @@ -1,5 +1,7 @@ import assert from "node:assert"; import { afterEach, describe, it } from "node:test"; +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; import type { Terminal as XtermTerminalType } from "@xterm/headless"; import { Chalk } from "chalk"; import { Markdown, type MarkdownTheme } from "../src/components/markdown.ts"; @@ -65,6 +67,63 @@ describe("Markdown component", () => { }); }); + describe("Block reuse", () => { + afterEach(() => resetCapabilitiesCache()); + + function countingTheme(calls: string[]): MarkdownTheme { + return { + ...defaultMarkdownTheme, + highlightCode: (code, lang) => { + calls.push(`${lang}:${code}`); + return code.split("\n").map((line) => chalk.green(line)); + }, + }; + } + + it("highlights a code block once across streamed text and resizes, and again after invalidate", () => { + const calls: string[] = []; + const markdown = new Markdown("```ts\nx\n```", 0, 0, countingTheme(calls)); + markdown.render(80); + markdown.setText("```ts\nx\n```\n\nafter"); + markdown.render(40); + assert.deepStrictEqual(calls, ["ts:x"]); + markdown.invalidate(); + markdown.render(40); + assert.deepStrictEqual(calls, ["ts:x", "ts:x"]); + }); + + it("holds rendered blocks weakly, so a full garbage collection releases them", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const calls: string[] = []; + const markdown = new Markdown("```ts\nx\n```", 0, 0, countingTheme(calls)); + markdown.render(80); + // A WeakRef keeps its target alive until the current job ends. + await new Promise((resolve) => setImmediate(resolve)); + gc(); + markdown.render(40); + assert.deepStrictEqual(calls, ["ts:x", "ts:x"]); + }); + + it("renders a block again when the block after it changes its spacing", () => { + const markdown = new Markdown("# Title", 0, 0, defaultMarkdownTheme); + markdown.render(80); + markdown.setText("# Title\nfoo"); + assert.deepStrictEqual( + markdown.render(80).map((line) => stripAnsi(line).trim()), + ["Title", "", "foo"], + ); + }); + + it("follows changed reference link definitions", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: false }); + const markdown = new Markdown("[a][x]\n\n[x]: https://one.example", 0, 0, defaultMarkdownTheme); + markdown.render(80); + markdown.setText("[a][x]\n\n[x]: https://two.example"); + assert.ok(stripAnsi(markdown.render(80).join("\n")).includes("https://two.example")); + }); + }); + describe("Lists", () => { it("should render simple nested list", () => { const markdown = new Markdown( diff --git a/packages/tui/test/text.test.ts b/packages/tui/test/text.test.ts new file mode 100644 index 000000000..9c2e2bd61 --- /dev/null +++ b/packages/tui/test/text.test.ts @@ -0,0 +1,12 @@ +import assert from "node:assert"; +import { describe, it } from "node:test"; +import { Text } from "../src/components/text.ts"; + +describe("Text", () => { + it("keeps its wrapped lines when the same text is set again", () => { + const text = new Text("hello world", 0, 0); + const lines = text.render(5); + text.setText("hello world"); + assert.strictEqual(text.render(5), lines); + }); +}); From c09c98d7d2caf659b4bbe1e2427adb1bdac4b114 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Sun, 1 Nov 2026 14:47:36 +0100 Subject: [PATCH 237/271] fix(chord): admit a routed host even when the relay that spawned it is gone A relayed hold released (or its principal refused or revoked) while the route resolved made #resolve throw before adding the connection, so the spawned host's link was never read and its own upstream references failed unavailable/ready_timeout after readyTimeout. The connection is now added right after resolve; the checks only fail the relay. --- packages/chord/CHANGELOG.md | 1 + packages/chord/docs/spec.md | 2 +- packages/chord/src/host.ts | 10 ++++++---- packages/chord/test/links-review.test.ts | 23 +++++++++++++++++++++++ 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index 4f50cc098..1a1382eb7 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -37,6 +37,7 @@ ### Fixed +- A host a route resolves is admitted even when the relayed reference that asked for it is released, its client disconnects, or its principal is refused or revoked while the route resolves. Before, the parent never read the new host's link: a spawned worker's own references to its parent (its `lifetime: "host"` objects' upstream references) waited until they failed `unavailable/ready_timeout`, reported as a possible reference cycle across hosts. - The Node SQLite driver keeps the 256 most recently used prepared statements per connection instead of every statement it ever prepared, so callers generating SQL of unbounded shapes (a query builder) no longer grow memory without bound. - Fixed append ops from the delta tracker and `diffRevisions()` keeping the whole string they were cut from alive (V8 represents a slice as a view of its source), so a delta chain, a watch backlog or a replica built from appends to a growing text held one copy of the text per commit ([#9265](https://github.com/earendil-works/pi/issues/9265)) diff --git a/packages/chord/docs/spec.md b/packages/chord/docs/spec.md index ca910d758..4c261cb54 100644 --- a/packages/chord/docs/spec.md +++ b/packages/chord/docs/spec.md @@ -279,7 +279,7 @@ The tests cover at least: lifetime/references/linger/forced disposal/gone; activ - **Two-version rule.** An object of a changed type created after a commit has only new code; a previous-code-version root calling it gets `superseded`. - **Validation.** Remote calls are validated at reference dispatch, in-host too: a direct binding against the provider's schema, a binding through an adapter against the consumer's. A provider validates each snapshot and reset of a remote state member once per (re)subscription, against the consumer's contract through an `accepts` adapter and the provider's otherwise; ops are validated only in dev mode. At reload, the new provider's description must cover the old provider's when the version is unchanged, in addition to the check against live consumers and facets' `uses`. - **Contracts.** Local contracts list their members, so a loading reference knows whether a member is a method, state or opaque. Canonical identity omits input fields equal to their default; use `opt(str({ default }))` (pi-schema) to let callers omit them. Provider variants must cover consumer variants; members the consumer's contract does not list are hidden. Schema change detection compares canonical JSON, so reordering `anyOf` branches counts as a change. Fingerprints use a non-cryptographic 64-bit hash: synchronous and browser-safe; contracts are trusted. -- **Principals.** A root ctx at ingress acts as its principal (`host.root(fn, { principal })`); contexts inside objects act as `host`. The acceptor decides a link's kind and name (`accept(link, { principal, kind, name })`); only configured host links are addressable, carry roots and may vote, and control frames are accepted only from the parent. Clients never carry roots or code versions. Relaying needs a matching `route`; `authorize(principal, hostName)` runs before `resolve` and again after, and revoking or re-stamping a principal closes its relays at once. `invalidatePrincipal(match)` re-stamps or closes matching links and propagates pattern matches to child hosts. +- **Principals.** A root ctx at ingress acts as its principal (`host.root(fn, { principal })`); contexts inside objects act as `host`. The acceptor decides a link's kind and name (`accept(link, { principal, kind, name })`); only configured host links are addressable, carry roots and may vote, and control frames are accepted only from the parent. Clients never carry roots or code versions. Relaying needs a matching `route`; `authorize(principal, hostName)` runs before `resolve` and again after, and revoking or re-stamping a principal closes its relays at once. The host `resolve` returns is admitted before that second check, so a refused, revoked or released relay never leaves a spawned host's link unread. `invalidatePrincipal(match)` re-stamps or closes matching links and propagates pattern matches to child hosts. - **Cells and readables.** A value returned from `change` replaces the value and is always validated; `replace` transfers ownership without copying; by default (`{ diff }` overrides) it publishes a diff when the new value reuses a container of the old one at the same key, else `r`. `subscribe(fn)` delivers values, starting with the current one; `events(fn)` delivers `snap`, then `ops`, or `reset`. Reference state members are stable readables that throw `not_ready` while loading and `not_in_variant` outside the variant, and reset when their source moves; a listener added before they are readable gets its snapshot then. Remote state members stay hydrated while the reference is held. - **References and calls.** A reference's identity is its address; a second `self.get` of the same address in one run returns the same reference. A reference call without ctx, or with a task ctx, starts a fresh root in the current code version; a ctx whose call or root already settled is refused (`aborted/expired`), one from another host `invalid/foreign_ctx`. Calling a reference inside a factory throws `not_ready`. - **Resources and disposal.** New and replaced exclusive resources open at the gate, and so do non-exclusive resources of the same object that depend on them. A new object for an address that is still being disposed waits for that disposal. A resource open that completes after its signal aborted closes its value at once. diff --git a/packages/chord/src/host.ts b/packages/chord/src/host.ts index 9f96a4909..fe94fd9be 100644 --- a/packages/chord/src/host.ts +++ b/packages/chord/src/host.ts @@ -1136,7 +1136,7 @@ export class HostImpl implements Host, HostCore { return best?.[1]; } - /** Authorize, resolve (maybe spawn), authorize again; returns the synced connection to the host. */ + /** Authorize, resolve (maybe spawn) and admit the host, authorize again; returns the synced connection to the host. */ async #resolve( route: Route, hostName: string, @@ -1148,15 +1148,17 @@ export class HostImpl implements Host, HostCore { if (!(await route.authorize(principal, hostName))) throw refuse(); if (!still()) throw revoked(); const link = await route.resolve(hostName); - if (!still()) throw revoked(); - if (!(await route.authorize(principal, hostName))) throw refuse(); - if (!still()) throw revoked(); + // Admitted before the checks: a host the route spawned for a request that is gone or refused meanwhile still runs, + // and its own references to this host would otherwise wait on a link nobody reads until they time out. let connection = [...this.connections].find((candidate) => candidate.link === link); connection ??= this.#addConnection(link, { role: "downstream", principal: route.principal(hostName), peer: { kind: "host", name: hostName }, }); + if (!still()) throw revoked(); + if (!(await route.authorize(principal, hostName))) throw refuse(); + if (!still()) throw revoked(); await connection.whenSynced(); if (!still()) throw revoked(); return connection; diff --git a/packages/chord/test/links-review.test.ts b/packages/chord/test/links-review.test.ts index f6a456afb..f5d0f7173 100644 --- a/packages/chord/test/links-review.test.ts +++ b/packages/chord/test/links-review.test.ts @@ -311,6 +311,29 @@ describe("links review: relay ownership", () => { assert.ok(!(c.workers.get("session:S1")?.link.sent ?? []).some((frame) => frame.t === "hold")); }); + it("a release while the route resolves still admits the resolved worker: its references upstream go live", async () => { + const Feed = remote("t.feed", { + version: 1, + input: obj({}, { open: true }), + host: hostNamed("server"), + variants: { main: { methods: { ping: method(undefined, str()) } } }, + }); + const c = cluster(); + c.server.serve(Feed, () => ({ variant: "main", ping: () => "pong" })); + const gate = latch(); + c.resolveGate = gate.promise; + const { raw } = c.raw({ guest: "g" }); + raw.send({ t: "hold", id: 1, at: "session:S1", token: Counter.description, input: { name: "a" } }); + await sleep(5); + // The client goes away while the worker starts: the route still hands over the worker it spawned. + raw.send({ t: "release", id: 1 }); + gate.open(); + await until(() => c.workers.has("session:S1")); + const worker = c.workers.get("session:S1")!.host; + const feed = await worker.get(Feed, {}, { signal: AbortSignal.timeout(500) }); + assert.equal(await feed.ping(), "pong"); + }); + it("#9 an origin disconnect removes its relayed holds and subscriptions downstream", async () => { const c = cluster(); const browser = c.browser({ guest: "g" }); From b556cfe5b0300a91f39c7bc5369a65a25fb7fcd5 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sun, 1 Nov 2026 15:20:46 +0000 Subject: [PATCH 238/271] docs(chord): schemas are pi-schema's in the design --- packages/chord/docs/next/design.md | 13 +++++++------ packages/chord/docs/next/layer-1.md | 2 +- packages/chord/docs/next/slice-3a.md | 4 ++-- packages/chord/docs/next/slices.md | 4 ---- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index d40d82677..517276db3 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -442,7 +442,8 @@ drops, folds or resets. - **JSON plus `Uint8Array` leaves**, with a bytes schema type, defined independently of the transport. Every link carries the same `Message` through its codec (section 22); parts are encoded at their origin; byte segments travel raw, never base64. Same-host calls (no link) pass values by reference. -- Schemas are TypeBox and must describe data (no callbacks, no native types, no runtime codecs). Method arguments are +- Schemas are plain JSON Schema built with `@earendil-works/pi-schema` (decision "The schema switch") and must + describe data (no callbacks, no native types, no runtime codecs); `format` is an annotation, never checked. Method arguments are validated at the provider for every caller against closed schemas (section 5, R7, Q12, Q15); results and state are never validated (section 30). Values are immutable by convention once handed over (Delta's rule): nobody mutates a value it got, and nothing is copied or frozen to enforce it. There is no development mode (R9): one behavior @@ -572,7 +573,7 @@ host.serve(Notes, ({ session }, principal, self) => (`invalid/member` outside it). - **Params** (owner, 2026-10-21/22): a flat record of required fields, each `str()`, `int()`, `bool()` or `oneOf(...)` of string literals, constraints allowed (`str({ pattern })`, `int({ min: 0 })`); no defaults, optional, nested or - unknown fields. Checked for every caller with one compiled closed-object TypeBox check (a missing or extra field + unknown fields. Checked for every caller with one compiled closed-object pi-schema check (a missing or extra field fails `invalid/params`). The identity key is the field values in declared order, each escaped, joined with `|`. Params are never copied or frozen: the object keeps the caller's object, immutable by convention. - **Arguments (R7, Q12, Q15):** always validated with the provider's schema, whoever the caller is (trusted code is @@ -592,8 +593,8 @@ host.serve(Notes, ({ session }, principal, self) => link protocol (adding one bumps the protocol version; a receiver rejects unknown reasons; a test pins the list): `invalid` (`args`, `params`, `member`, `host_address`, `reentrant`, `read_after_bulk`, `cycle`), `contract_mismatch` (`updating`), `forbidden` (`revoked`), `gone`, `not_ready`, `released` (`ended`, `closed`, `settled`, `version`, `discarded`; none for a holder's own release), `too_large` (`part`), `unavailable` (`locked`, - `park_timeout`, `busy`, `storage_failed`, `undeclared`: a registry no plugin serves here, slice 8), `aborted`. A failed check names TypeBox's instance path ("unknown field", "missing", else TypeBox's - message; keys containing `/` or `~` make the path ambiguous, documented). Plugins throw `PluginError(code, message, + `park_timeout`, `busy`, `storage_failed`, `undeclared`: a registry no plugin serves here, slice 8), `aborted`. A failed check names pi-schema's JSON Pointer path ("unknown field", "missing", else + pi-schema's message; `~` and `/` in keys escaped as `~0`, `~1`). Plugins throw `PluginError(code, message, details?)` for codes of their own; anything else crosses a link as the plugin code `internal` with its message, never a stack or a cause; plugin errors never use Chord's codes. An error rebuilt from the wire is not copied and only its envelope is checked; a malformed envelope is a protocol error. @@ -1997,7 +1998,7 @@ validation, no clone of snapshots or ops. - **The protocol version** travels in `hello`; hosts of different versions refuse each other (a web page reloads for a newer server). Error reasons and message kinds are part of it. Until p3 ships, changes keep `PROTOCOL_VERSION` 1 (nothing is deployed); after that every message or reason change bumps it. -- **One typed `Message`** (owner, 2026-10-22, "Wire protocol"): a TypeBox schema `MESSAGE`, a tagged union over +- **One typed `Message`** (owner, 2026-10-22, "Wire protocol"): a pi-schema schema `MESSAGE`, a tagged union over `kind`, from which the protocol types derive; every decoded message is checked against it (compiled once) unless the link is trusted (section 3). Kinds: @@ -2416,7 +2417,7 @@ const accepted = checkAccepts(NotesV2, { providers fail at declaration naming the member; unknown keys and wrong shapes are rejected. - **The fingerprint** is a hash (`hash64`, portable, not a security boundary) of a schema-aware, data-only normal form (`NORMAL_FORM_VERSION`, tied to the protocol): annotations dropped only on schema nodes, `required`/`enum` sorted, - union and intersection order kept (it is meaning: defaults), a whitelist of TypeBox keywords, executable features + union and intersection order kept (it is meaning: defaults), a whitelist of JSON Schema keywords, executable features rejected; bytes are a `{ type: "bytes" }` data marker. Schemas are deep-copied into a private history (never frozen; immutable by convention). - **Schema helpers:** `str num int bool nul lit oneOf union obj arr map opt bytes unknown` and `type Static`. - Chord wraps state mappings as a derive and diffs per object and version (section 22); members an old consumer does not know do diff --git a/packages/chord/docs/next/layer-1.md b/packages/chord/docs/next/layer-1.md index 505fe7294..0129181df 100644 --- a/packages/chord/docs/next/layer-1.md +++ b/packages/chord/docs/next/layer-1.md @@ -14,7 +14,7 @@ checked, what a principal key is, what an error is. Pure functions, types and we Source: `packages/chord/src/next/contracts/` (declarations) and `packages/chord/src/next/data/` (per-call checks, principals, errors), split into files by concept. Everything else under `src/next/` is the frozen old kernel: never imported, never edited. Reused: `src/context/` (`Context`, `BACKGROUND_CONTEXT`, `context.telemetry`) and -`src/json.ts` (`JsonValue`), both unchanged. Schemas are TypeBox 1.3 (`typebox` in `node_modules`). +`src/json.ts` (`JsonValue`), both unchanged. Schemas were TypeBox 1.3 when L1 was built; since the schema switch (2026-11-01) they are pi-schema's plain JSON Schema (design §3 "Values"). The TypeBox details below are historical. ## 1. Values diff --git a/packages/chord/docs/next/slice-3a.md b/packages/chord/docs/next/slice-3a.md index 4605bb95c..a6dba100b 100644 --- a/packages/chord/docs/next/slice-3a.md +++ b/packages/chord/docs/next/slice-3a.md @@ -89,7 +89,7 @@ if (notes.variant === "reader") console.log(notes.notes.value); // narrowed by | u32 header length, big-endian | header JSON (UTF-8) | part 0 | part 1 | … | ``` -- **One format on every link.** The header is a JSON object, a tagged union over `kind`, defined as a TypeBox schema; +- **One format on every link.** The header is a JSON object, a tagged union over `kind`, defined as a schema (TypeBox then, pi-schema since the schema switch); every received header is checked with the compiled schema (the wire is a trust boundary; skipping it on trusted IPC links comes with IPC in slice 4). The header lists its parts: `parts: [{ json, bytes }]`, the JSON text's byte length and the byte lengths of the segments that follow it (`bytes: []` for state parts and arguments without bytes). @@ -125,7 +125,7 @@ if (notes.variant === "reader") console.log(notes.notes.value); // narrowed by Replaces the JSON-header framing of section 2 (`| u32 header length | header JSON | parts |`); the message kinds, ids, races and rules of section 2 stay. -- **`Message`** is a tagged union defined once as a TypeBox schema (`MESSAGE`; `type Message = Static`): `hello { protocol, incarnation }`, `hold { hold, type, version, fingerprint, params }`, `release { hold }`, `call { call, hold, method, args? }`, `cancel { call }`, `bound { hold, variant, members: [{ member, value }], failed: [{ member, error }] }`, `state { holds: [{ hold, members: [{ member, ops }], failed: [...] }] }`, `failed { diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index 452528592..28e094cd5 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -170,15 +170,11 @@ deleted (the slice's spec and code are the record). Moved here from `carry-over. slice 3b.) - **The park deadline's per-type override** is not built: a parked call waits the reference's `readyTimeout`. Build it only if a real case needs it. (Slice 3b spec.) -- Failed-check paths are TypeBox's instance paths (ambiguous for keys containing `/` or `~`); documented, not fixed. - (L1.) - No CBOR; revisit only if benchmarks show encoding as a bottleneck (slice 3a's and 4a's do not). (L2 decisions.) - **One N on every host:** a relay forwards calls at once, so a worker announcing a smaller N than the relay's could see more running calls of one origin than it accepts (a protocol error). Every host announces 256; per-link N, if ever needed, must bound the relay's forwarding too. (Slice 4a rework.) - **The Durable conversation view's size on re-hold** is a known cost until pi #11327 (windowed view) lands. (Slice 4.) -- **`map(bytes())` is refused at declaration** ("unsupported keyword `~refine`"): TypeBox's `Record` copies the bytes - schema's check as an enumerable key, which the snapshot walk refuses. An L1 bug found in slice 3a, not fixed yet. ### Later: reload (when a case needs it) From e1669ee4b841fc053de7c4fb0b872f41f8d9a898 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sun, 1 Nov 2026 23:11:16 +0000 Subject: [PATCH 239/271] docs(chord): contracts come from where the plugin's code came from --- packages/chord/docs/next/decisions.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 7dd53c73b..843af1a7c 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2063,7 +2063,7 @@ validation and copies the v2 kernel added around Delta. and TUI, decided point by point; the slice 10 components are its inside. Inputs from the owner: user management is part of it ("user management will also need to be a thing we support"), and app shapes without workers (server and clients only: "a chat app doesn't need workers"). -- **Plugins import each other by name (owner, 2026-10-31: "i like it"; changes 7c's cross-plugin rule):** +- **Plugins import each other by name (owner, 2026-10-31: "i like it"; changes 7c's cross-plugin rule; **superseded** by "Contracts come from where the plugin's code came from", 2026-11-01):** `import { Notes } from "@app/notes/contract"` resolves to the discovered plugin named `@app/notes`, its file `contract.ts`, wherever its directory is (any root), copied into the importer's script as before (contracts and side-effect-free files only; they may import only host packages and files of their own plugin); a missing plugin is @@ -2140,6 +2140,19 @@ validation and copies the v2 kernel added around Delta. annotation in Chord** ("no format check": no format module in Chord's or Durable's entries; pi-ai still checks formats on tool arguments). **Chord's envelope check costs +10 to +100 ns per message** from untrusted links (pi-schema reads fields by name from a list; no eval): accepted ("negligible"), no hand-written check. +- **Contracts come from where the plugin's code came from (owner, 2026-11-01: "ok"; plugin model point 1, primer + `docs/next/plugin-primer/`; supersedes "Plugins import each other by name"):** a plugin carries the contract it was + written against, never the running app's newest file. The problem: by name, a third-party plugin's next build copied + the provider's newest `contract.ts` (version 2), so its version-1 code claimed version 2, `accepts` was never used + and it computed wrong results silently. Rule: (A) plugins that change together, the app's own in its repository or a + user's in one root, import each other by relative path (`../notes/contract.ts`), copied into the importer's script + and listed in `copied`; a copied file may import only host packages and files of its own plugin; a relative path + leaving its root (into base code or another root) is a build error pointing to the API or an npm dependency. (B) + Plugins that do not change together take contracts as an npm dependency at a fixed version, bundled at the installed + version (decision "A plugin's npm dependencies are bundled"); the provider serves it through `accepts` or the + consumer waits "contract mismatch". Which package carries the app's contracts is point 5. The by-name lookup and the + `tsconfig` path aliases go; built in the plugin slice (examples 35-38, fixtures, design §21, the plugins README, the + guide's chapter "Plugins Using Plugins"). - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From c9f4c7b9fcd0f3e6b5bbc35c7c9b6cda35580eb8 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sun, 1 Nov 2026 23:11:22 +0000 Subject: [PATCH 240/271] docs(chord): fold the cross-plugin import rule into design and slices --- packages/chord/docs/next/design.md | 5 ++++- packages/chord/docs/next/slices.md | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index 517276db3..c11cb89e4 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -1962,7 +1962,10 @@ type ReloadReport = { - **A plugin** is a directory with a `package.json` (its `name` is the plugin's name, npm style) and one file per facet kind the app names; plugins come from roots found by directory listing (no source parsing; the first root wins a duplicate name). Dependencies between plugins resolve by contract name at runtime; names are prefixed - (`.`; tools `_`). + (`.`; tools `_`). A plugin carries the contracts it was written against (decision "Contracts come from where the + plugin's code came from"): plugins that change together import each other by relative path within a root + (copied into the importer's script); others take contracts as an npm dependency at a fixed version (slice 10 still + resolves imports by plugin name; the plugin slice changes it). - **Loader D:** one esbuild script per facet (its own files, copied files of other plugins, its npm `dependencies` inlined; host packages, `chord.external` packages and Node builtins external, passed in at evaluation); each facet kind is defined once, `kinds: { : { platform: "node" | "browser", hostPackages } }` (slice 11); evaluated with `vm.Script` under a stable filename per (plugin, kind, digest); no module map: old generations are reclaimed by the diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index 28e094cd5..084d3456e 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -181,6 +181,13 @@ deleted (the slice's spec and code are the record). Moved here from `carry-over. - **`rebuild` of tables** (key and column-type changes): `evolveTable` has no edit for them; build one only when a real case needs it. (Slice 6a's second spec review; out of slice 7b's scope, `slice-7b.md` §12.) +### The plugin slice (plugin model) + +- **Cross-plugin imports by relative path and npm** (decision "Contracts come from where the plugin's code came + from"): remove `build.ts`'s by-name lookup, allow relative imports into another plugin of the same root, refuse + paths leaving the root; examples 35-38 and fixtures back to relative imports; drop the six `tsconfig` path aliases; + plugins README; the guide's chapter "Plugins Using Plugins". (Slice 10 built by-name imports.) + ### After slice 11 - **Small examples** (owner, 2026-10-30: the 7c examples are complete but complex): once HTTP, workers, web and TUI From 6ab6c1e0e431a55ddfdd804f35c16073b1031e56 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 09:41:22 +0000 Subject: [PATCH 241/271] docs(chord): plugins import only the app's versioned API --- packages/chord/docs/next/decisions.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 843af1a7c..3b38c7bf3 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2153,6 +2153,18 @@ validation and copies the v2 kernel added around Delta. consumer waits "contract mismatch". Which package carries the app's contracts is point 5. The by-name lookup and the `tsconfig` path aliases go; built in the plugin slice (examples 35-38, fixtures, design §21, the plugins README, the guide's chapter "Plugins Using Plugins"). +- **Plugins import only the app's versioned API; Chord is re-exported in it (owner, 2026-11-01: "makes sense"; + plugin model point 2):** a plugin never imports Chord directly: Chord is not a host package plugins may name, so a + direct import is a build error pointing to the app's API. The app's API module per version (`@acme/app/v3`) + re-exports the Chord surface the app supports (contract and schema builders, `defineFacet`, `defineRegistry`, the + types plugins see) next to its own small part (host kinds, the principal type, `defineFacet` bound to it, core + contracts, registry tokens). One version number, the app's, describes everything a plugin gets from its host. The + process runs one Chord: an older API version still listed is a layer over today's Chord; when Chord breaks its + API it keeps its previous API as a compatibility module for a while, which the app's older version re-exports; + once that is dropped the app drops the version and old plugins fail at build naming the available versions. + Contracts are data and bind through `accepts` whatever Chord runs. One facet imports from one API version only + (checked at build). Only base code imports Chord directly. Bundling is slice 10's host packages: the version is part + of the name (`kinds.hostPackages` lists `@acme/app/v3`, `@acme/app/v2`), each process provides its own modules. - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From d0fe213089aef0599aa9c90bf910c8b562e0dcf8 Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 10:55:30 +0100 Subject: [PATCH 242/271] fix(coding-agent): apply context edits to branch summary input (#11680) When a partial assistant reply fails and automatic retry succeeds, pi appends a `context_edit` that hides the failed attempt from model context. Branch summarization (`/tree` with summarize) built its input from raw session entries through `getMessageFromEntry`, which ignored `context_edit` entries. As a result, the summarizer received the hidden attempt again. Content replacements were ignored the same way. What the change does: - `prepareBranchEntries` now collects the latest `context_edit` for each target in the branch. - It applies those edits to the entries it converts, using the same `projectContextEntry` logic as the live context and compaction: - an omission (`replacement: null`) drops the entry; - a replacement swaps its content. - `projectContextEntry` is now exported from `session-manager.ts` so both paths share it. - The existing branch-summary behavior is unchanged: tool results are still skipped, and compaction or branch-summary entries still become summary messages. Testing: - Added `applies context edits to branch entries` to `test/branch-summarization.test.ts`. It covers one omitted assistant entry and one replaced assistant entry. The test fails without the change and passes with it. - Ran `branch-summarization`, `branch-summary-extensions`, `agent-session-branching` and the compaction tests: all pass. - `npm run check`: biome and the coding-agent checks pass. `tsc` reports 6 errors in `packages/ai` model-catalog tests, and the same 6 errors occur on unmodified `main`. - End-to-end reproduction with the faux provider and `AgentSession`: a failed attempt, a successful retry, then `navigateTree(target, { summarize: true })`. Before the change, the summary request contained the failed attempt's text. After the change, it contains only the successful retry. Fixes #11602 Requested by @davidbrai Co-authored-by: triagent --- .../core/compaction/branch-summarization.ts | 20 +++++-- .../coding-agent/src/core/session-manager.ts | 3 +- .../test/branch-summarization.test.ts | 52 ++++++++++++++++++- 3 files changed, 69 insertions(+), 6 deletions(-) diff --git a/packages/coding-agent/src/core/compaction/branch-summarization.ts b/packages/coding-agent/src/core/compaction/branch-summarization.ts index 69bd75133..6f8bc0cb3 100644 --- a/packages/coding-agent/src/core/compaction/branch-summarization.ts +++ b/packages/coding-agent/src/core/compaction/branch-summarization.ts @@ -15,7 +15,12 @@ import { createCompactionSummaryMessage, createCustomMessage, } from "../messages.ts"; -import type { ReadonlySessionManager, SessionEntry } from "../session-manager.ts"; +import { + type ContextEditEntry, + projectContextEntry, + type ReadonlySessionManager, + type SessionEntry, +} from "../session-manager.ts"; import { completeSummarization, estimateTokens, getSummarizationFailure } from "./compaction.ts"; import { computeFileLists, @@ -155,14 +160,15 @@ export function collectEntriesForBranchSummary( * Extract AgentMessage from a session entry. * Similar to getMessageFromEntry in compaction.ts but also handles compaction entries. */ -function getMessageFromEntry(entry: SessionEntry): AgentMessage | undefined { +function getMessageFromEntry(entry: SessionEntry, edit?: ContextEditEntry): AgentMessage | undefined { switch (entry.type) { case "message": // Skip tool results - context is in assistant's tool call if (entry.message.role === "toolResult") return undefined; - return entry.message; + return edit ? projectContextEntry(entry, edit)[0] : entry.message; case "custom_message": + if (edit) return projectContextEntry(entry, edit)[0]; return createCustomMessage(entry.customType, entry.content, entry.display, entry.details, entry.timestamp); case "branch_summary": @@ -217,10 +223,16 @@ export function prepareBranchEntries(entries: SessionEntry[], tokenBudget: numbe } } + // Context edits hide or replace earlier entries in model context; honor the latest edit per target. + const edits = new Map(); + for (const entry of entries) { + if (entry.type === "context_edit") edits.set(entry.targetId, entry); + } + // Second pass: walk from newest to oldest, adding messages until token budget for (let i = entries.length - 1; i >= 0; i--) { const entry = entries[i]; - const message = getMessageFromEntry(entry); + const message = getMessageFromEntry(entry, edits.get(entry.id)); if (!message) continue; // Extract file ops from assistant messages (tool calls) diff --git a/packages/coding-agent/src/core/session-manager.ts b/packages/coding-agent/src/core/session-manager.ts index 822aad816..20b6edaa9 100644 --- a/packages/coding-agent/src/core/session-manager.ts +++ b/packages/coding-agent/src/core/session-manager.ts @@ -516,7 +516,8 @@ export function buildContextEntries( * If leafId is provided, walks from that entry to root. * Handles compaction and branch summaries along the path. */ -function projectContextEntry(entry: SessionEntry, edit: ContextEditEntry | undefined): AgentMessage[] { +/** Project one entry into model-visible messages, applying its latest context edit if any. */ +export function projectContextEntry(entry: SessionEntry, edit: ContextEditEntry | undefined): AgentMessage[] { const messages = sessionEntryToContextMessages(entry); if (!edit) return messages; const replacement = edit.replacement; diff --git a/packages/coding-agent/test/branch-summarization.test.ts b/packages/coding-agent/test/branch-summarization.test.ts index ade25dbfd..3164a8773 100644 --- a/packages/coding-agent/test/branch-summarization.test.ts +++ b/packages/coding-agent/test/branch-summarization.test.ts @@ -7,7 +7,7 @@ import { type SimpleStreamOptions, } from "@earendil-works/pi-ai"; import { describe, expect, it } from "vitest"; -import { generateBranchSummary } from "../src/core/compaction/index.ts"; +import { generateBranchSummary, prepareBranchEntries } from "../src/core/compaction/index.ts"; import type { SessionEntry } from "../src/core/session-manager.ts"; const model: Model<"anthropic-messages"> = { @@ -131,4 +131,54 @@ describe("branch summarization", () => { "Branch summarization failed: generation hit the token cap and the summary is incomplete", ); }); + + it("applies context edits to branch entries", () => { + const ts = (n: number) => new Date(n).toISOString(); + const branch: SessionEntry[] = [ + { + type: "message", + id: "u1", + parentId: null, + timestamp: ts(1), + message: { role: "user", content: "Investigate login", timestamp: 1 }, + }, + { + type: "message", + id: "failed", + parentId: "u1", + timestamp: ts(2), + message: fauxAssistantMessage("FAILED_ATTEMPT", { stopReason: "error", errorMessage: "lost" }), + }, + { + type: "context_edit", + id: "e1", + parentId: "failed", + timestamp: ts(3), + targetId: "failed", + replacement: null, + }, + { + type: "message", + id: "ok", + parentId: "e1", + timestamp: ts(4), + message: fauxAssistantMessage("ORIGINAL_TEXT"), + }, + { + type: "context_edit", + id: "e2", + parentId: "ok", + timestamp: ts(5), + targetId: "ok", + replacement: { content: "REPLACED_TEXT" }, + }, + ]; + + const serialized = JSON.stringify(prepareBranchEntries(branch).messages); + + expect(serialized).toContain("Investigate login"); + expect(serialized).not.toContain("FAILED_ATTEMPT"); + expect(serialized).not.toContain("ORIGINAL_TEXT"); + expect(serialized).toContain("REPLACED_TEXT"); + }); }); From ddbe02cd2014bedc9555b7d782da90a2fd5e8abc Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Mon, 2 Nov 2026 09:47:27 +0100 Subject: [PATCH 243/271] docs: audit changelogs and add release highlights --- packages/ai/CHANGELOG.md | 8 ++++ packages/coding-agent/CHANGELOG.md | 62 +++++++++++++++++++++++++++++- packages/tui/CHANGELOG.md | 5 +++ 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 6aae94ddb..5d197d814 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -36,6 +36,8 @@ - Added optional `openBrowserOnConfirm` and `copyCodeOnConfirm` to `device_code` auth events and `OAuthDeviceCodeInfo`. Both default to `true`; set them to `false` to stop clients from offering to open the verification page or copy the code ([#9282](https://github.com/earendil-works/pi/issues/9282), [#9301](https://github.com/earendil-works/pi/pull/9301)) - Added `StreamingJsonParser`, an incremental, tolerant parser for JSON text that arrives in pieces (each piece is read once; snapshots of the partial value never change once returned), and `ToolCallArgumentsStream`, which every provider now uses for streamed tool-call arguments; its `append()` also returns the changes each piece made (`JsonChange`: a value set at a path, or text appended to a string), for writers that keep their own copy ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Added GPT-6.1 Terra to the OpenAI, Azure OpenAI Responses, and OpenAI Codex providers with official pricing ($1 input, $0.05 cached input, $1.25 cache writes, $5 output, long-context tier above 272k), tool search, and mid-conversation effort changes. Unlike GPT-6.1 Sol it supports `off` (reasoning effort `none`); Codex serves it with a 500K context window +- Added `Usage.cost.totalFromProvider` to distinguish provider-reported billed totals from catalog estimates. Missing or `false` means an estimate; combined usage is marked provider-reported only when every part is ([#11178](https://github.com/earendil-works/pi/pull/11178)) +- Added `responseModel` to pi-messages terminal events, preserving the concrete model reported by providers such as Radius when a virtual model is requested ([#11265](https://github.com/earendil-works/pi/pull/11265)) ### Changed @@ -57,6 +59,12 @@ - Fixed GPT-5.4 and GPT-5.5 failing with `Missing namespace for function_call` after switching from another model that called tools. On the OpenAI and OpenCode providers they now declare their tools in the top-level tool list instead of loading them through `additional_tools` or tool search ([#11415](https://github.com/earendil-works/pi/issues/11415)) - Fixed Bedrock requests failing when `strict: "prefer"` tools exceed Claude's strict tool limits, such as more than 16 optional parameters across all strict tools; requests now retry once with preferred tools non-strict while keeping `strict: "require"` tools strict, as the Anthropic provider already does ([#11416](https://github.com/earendil-works/pi/issues/11416)) - Fixed OpenRouter's `~anthropic/claude-*-latest` aliases losing Claude's earlier thinking after each tool call. They used the Chat Completions API, where pi dropped the thinking signature, and OpenRouter then ignored the unsigned thinking. They now use the Anthropic Messages API with adaptive thinking, like the versioned Claude models ([#9534](https://github.com/earendil-works/pi/issues/9534)) +- Fixed OpenRouter cost totals using catalog estimates instead of reported billing for Chat Completions and Anthropic Messages. BYOK totals include both OpenRouter charges and reported upstream costs; component costs remain catalog estimates ([#9980](https://github.com/earendil-works/pi/issues/9980)) +- Fixed OpenCode Zen and OpenCode Go requests using shared endpoints instead of their protocol-specific OpenAI, Anthropic, Google, and System One endpoints ([#11434](https://github.com/earendil-works/pi/pull/11434) by [@vimtor](https://github.com/vimtor)) +- Fixed Zhipu/Z.ai usage-limit errors (code `1308`) being treated as transient HTTP 429 errors, causing automatic retries that cannot succeed until the usage cap resets ([#11537](https://github.com/earendil-works/pi/issues/11537)) +- Fixed LiteLLM's `ContextWindowExceededError` not being recognized as context overflow ([#11492](https://github.com/earendil-works/pi/issues/11492)) +- Fixed Gemini tool schemas sending unsupported `const` keywords. They are now converted to single-value `enum` schemas without rewriting literal data in defaults, examples, or enum values, or losing properties named `__proto__` ([#11308](https://github.com/earendil-works/pi/issues/11308)) +- Fixed the faux provider counting simulated cache-write tokens as both input and cache writes ## [1.4.0] - 2026-10-21 diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index d391a4f56..46a5bbc9f 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,17 @@ ## [Unreleased] +### New Features + +- **Extension update changelogs**: see what changed in installed extension packages on the next fresh launch after an update. Package authors can choose the changelog path, and `collapseChangelog` condenses the notices. See [Show a changelog](docs/packages.md#show-a-changelog). +- **Faster streaming**: large tool calls are parsed incrementally, and code-heavy answers and tool rows reuse highlighting and rendered Markdown instead of repeating the work on every update. See [Keep rendering responsive](docs/tui.md#keep-rendering-responsive). +- **Custom self-update commands**: use `PI_SELF_UPDATE_COMMAND` or `selfUpdateCommand` to update pi through a wrapper or another package manager. See [Custom update command](docs/cli.md#custom-update-command). +- **GPT-6.1 Terra**: available through OpenAI, Azure OpenAI Responses, and OpenAI Codex, with tool search, mid-conversation thinking-level changes, and thinking `off`. Codex offers a 500K context window. See [Models](docs/cli.md#models). + +### Breaking Changes + +- Tool argument validation runs on `@earendil-works/pi-schema`. TypeBox features outside JSON Schema (`Type.Refine()`, `Type.BigInt()`, `Type.Undefined()`, codecs, `Format.Set()` formats) no longer apply to tool arguments; see [Tool parameter schemas and TypeBox](docs/extensions.md#tool-parameter-schemas-and-typebox). Plain JSON Schema tools, including MCP tools, now get the same coercion as TypeBox tools, such as converting numeric strings and wrapping a single value into an array. Validation failures the model sees list pi-schema's messages (`- path: missing`, `- limit: expected number, got string`) instead of TypeBox's (`must have required properties path`, `must be number`). + ### Added - Extension packages can now show their changelog after an update. Pi reads `pi.changelogPath` from the package's `package.json`, or `CHANGELOG.md` in the package root, and on the next fresh launch shows entries newer than the last seen version under "Extension Updates". The `collapseChangelog` setting condenses this to one line per package ([#5958](https://github.com/earendil-works/pi/issues/5958), [#8790](https://github.com/earendil-works/pi/pull/8790)) @@ -9,15 +20,21 @@ - Device-code login (for example GitHub Copilot) now opens the verification page and copies the code when you press Enter. Providers can turn either action off with `openBrowserOnConfirm: false` or `copyCodeOnConfirm: false` ([#9282](https://github.com/earendil-works/pi/issues/9282), [#9301](https://github.com/earendil-works/pi/pull/9301)) - `pi update` can update Pi through a wrapper or another package manager. Set `PI_SELF_UPDATE_COMMAND` (for example `mise use -g npm:{packageName}@{version}`) or the global `selfUpdateCommand` setting; `pi update` runs it when pi.dev reports a newer release and fills `{packageName}`, `{version}`, and `{installSpec}` from that release. See [Custom update command](docs/cli.md#custom-update-command) ([#7756](https://github.com/earendil-works/pi/issues/7756), [#5607](https://github.com/earendil-works/pi/issues/5607)) - Extensions can format the user's configured keybindings with `keyText()` and `keyHint()` from the tool render context and the message and entry renderer options. Importing these functions from `@earendil-works/pi-coding-agent` can resolve the extension's own copy of the package, which does not see pi's keybindings, so hints such as `keyHint("app.tools.expand", "to expand")` rendered without the key. The docs now recommend the injected functions ([#4748](https://github.com/earendil-works/pi/issues/4748)) +- Added persisted session-entry `messageId` and ISO `timestamp` to `MarkdownTransformContext`. They are available when rendering saved entries and undefined for live rendering ([#7952](https://github.com/earendil-works/pi/pull/7952) by [@vipentti](https://github.com/vipentti)) +- Added a `blocked` option to inline `ctx.ui.custom()` components so approval prompts and other custom dialogs can report that pi is waiting for user input. Overlays ignore the option. See [Custom screens and overlays](docs/tui.md#use-custom-screens-and-overlays) ([#11428](https://github.com/earendil-works/pi/issues/11428)) +- Added `Usage.cost.totalFromProvider` to distinguish billed totals from catalog estimates in SDK and extension usage data; combined usage is marked provider-reported only when every part is ([#11178](https://github.com/earendil-works/pi/pull/11178)) +- Added concrete `responseModel` metadata to messages from pi-messages providers such as Radius, including when a virtual model is requested. See [AssistantMessage](docs/message-types.md#assistantmessage) ([#11265](https://github.com/earendil-works/pi/pull/11265)) +- Added GPT-6.1 Terra for OpenAI, Azure OpenAI Responses, and OpenAI Codex, with tool search and mid-conversation thinking-level changes. It supports thinking `off`; Codex offers a 500K context window ### Changed -- Tool argument validation runs on `@earendil-works/pi-schema`. TypeBox features outside JSON Schema (`Type.Refine()`, `Type.BigInt()`, `Type.Undefined()`, codecs, `Format.Set()` formats) no longer apply to tool arguments; see [Tool parameter schemas and TypeBox](docs/extensions.md#tool-parameter-schemas-and-typebox). Validation failures the model sees list pi-schema's messages (`- path: missing`, `- limit: expected number, got string`) instead of TypeBox's (`must have required properties path`, `must be number`); which arguments are accepted and how they are coerced is unchanged for extension and built-in tools. - Extensions keep importing `Type`, `Static` and `TSchema` from `@earendil-works/pi-ai`: Pi maps that import to pi-ai's compat entry plus the TypeBox it bundles, now that pi-ai itself no longer re-exports TypeBox, and its `StringEnum` there stays a TypeBox `TUnsafe`. The built-in tool factories (`createReadTool()` and the others) return `TypeBoxAgentTool`, which keeps `params` typed for TypeBox. - Tightened system prompt - `pi update` for installs from the pi.dev installer no longer passes `--min-release-age=0` to `npm ci`. The release lockfile pins every package, so the flag had no effect, and security scanners that block it stopped the update ([#11262](https://github.com/earendil-works/pi/issues/11262)) - `/model` and `/scoped-models` now show a model's name next to its ID when the name adds words the ID lacks, such as a renamed `models.json` override ([#5916](https://github.com/earendil-works/pi/issues/5916), [#11347](https://github.com/earendil-works/pi/pull/11347)) - MCP OAuth sign-in now uses pi's Client ID Metadata Document by default when the authorization server supports it for public clients, and registers a client dynamically otherwise, like Codex, Claude Code, and opencode. Clients registered before stay in use while their redirect URI matches. The new default `oauth.clientRegistration: "auto"` registers when `clientName` or a non-default `callbackUrl` is set; `"dcr"` always registers. `pi mcp add --oauth-client-registration` sets the mode ([#11426](https://github.com/earendil-works/pi/issues/11426)) +- Changed the default OpenAI model from `gpt-5.5` to `gpt-6.1-sol`, including after ChatGPT sign-in when no model is selected ([#11442](https://github.com/earendil-works/pi/issues/11442)) +- Updated the built-in model catalog to models.dev v2. Realtime and image-only models are no longer offered as chat models; newer OpenAI and Google chat models are included ### Fixed @@ -37,7 +54,7 @@ - Fixed HTML export showing a blank page for sessions with thousands of entries ([#11225](https://github.com/earendil-works/pi/issues/11225), [#11293](https://github.com/earendil-works/pi/pull/11293)) - Fixed HTML export not showing system messages: the system prompt and tools, and later section updates such as after `/reload`. `pi --export` now includes the system prompt and tools too ([#11261](https://github.com/earendil-works/pi/issues/11261), [#3464](https://github.com/earendil-works/pi/issues/3464), [#11290](https://github.com/earendil-works/pi/pull/11290)) - Fixed aborting a bash command on macOS sometimes running the next command of a list such as `long_step; next_step`. pi now stops the process group before killing it ([#11252](https://github.com/earendil-works/pi/issues/11252), [#11289](https://github.com/earendil-works/pi/pull/11289)) -- Fixed streamed tool calls re-running their call renderer on every argument delta; the interactive mode now renders the latest arguments once per frame, so a large streamed `write` no longer re-highlights its whole content per delta ([#9265](https://github.com/earendil-works/pi/issues/9265)) +- Fixed large streamed tool calls taking excessive CPU time and memory because every delta re-parsed the accumulated arguments. Providers and agent proxy streams now parse each delta once, grammar tools no longer copy their entire input per delta, and interactive tool renderers run at most once per frame ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed the login dialog opening sign-in URLs that are not web URLs, and printing control characters from sign-in, device-code, and info URLs, which could end a terminal hyperlink and send escape sequences to the terminal - Fixed `tui.select.pageUp` and `tui.select.pageDown` being ignored in slash command and file autocomplete, `/settings`, and the model, scoped models, login, fork, `/mcp`, and llama.cpp selectors. In fullscreen mode, PageUp/PageDown now move an open list or autocomplete menu instead of scrolling the transcript, and a page key bound to Ctrl+V pages autocomplete instead of pasting an image ([#7629](https://github.com/earendil-works/pi/issues/7629), based on [#7680](https://github.com/earendil-works/pi/pull/7680) by [@muyiyr](https://github.com/muyiyr)) - Fixed tool output and `!` command output showing the parameters of control sequences with a `<`, `=` or `>` prefix as text, for example `35;59;1M` from a mouse report `ESC[<35;59;1M` or `0;276;0c` from a terminal reply `ESC[>0;276;0c`. Text after the keypad-mode sequences `ESC=` and `ESC>` is still kept ([#11377](https://github.com/earendil-works/pi/issues/11377)) @@ -52,6 +69,47 @@ - Fixed `tool_search` and codemode `searchTools()` finding nothing for Chinese, Japanese, Thai, and other non-ASCII queries, because the tokenizer kept only ASCII letters and digits. Words in any script now match, text without spaces between words such as `搜索记录` is split into words, and canonically equivalent text such as `café` and `cafe\u0301` matches ([#10816](https://github.com/earendil-works/pi/issues/10816), [#11037](https://github.com/earendil-works/pi/pull/11037) by [@haoqixu](https://github.com/haoqixu)) - Fixed `pi update` stopping with `git clean -fdx failed with code 1` on Windows when another pi session had a git extension's native module or executable open. Windows cannot delete such files, so the update left the new code with the old dependencies. pi now warns, installs the dependencies anyway, and retries the clean on the next update ([#11597](https://github.com/earendil-works/pi/issues/11597)) - Fixed tool rows and streamed answers highlighting the same code again on every redraw. A running codemode row highlighted its whole script on every progress update and timer tick, a complete `write` row its whole file on every update and expand toggle, a streamed `write` its first 50 lines on every delta, and a streamed answer every earlier code block on every delta. Rows now reuse their last highlighting until the code, theme, or loaded grammars change, and assistant messages keep their rendered Markdown blocks while streaming ([#10971](https://github.com/earendil-works/pi/issues/10971)) +- Fixed resized photos being sent as multi-megabyte PNGs even when JPEG was much smaller, causing large requests to fail or time out. Image resizing now tries the smaller encoding first ([#11551](https://github.com/earendil-works/pi/issues/11551)) +- Fixed `pi update --extensions` reporting success without warning when npm's stale registry metadata left a package at its old version. pi now reports the installed and expected versions ([#11536](https://github.com/earendil-works/pi/issues/11536)) +- Fixed skill validation warnings being labeled `[Skill conflicts]` even when there was no name collision. They are now shown under `[Skill warnings]` ([#11521](https://github.com/earendil-works/pi/issues/11521)) +- Fixed codemode tool descriptions including entire union and multi-line output types in every request. Object unions now summarize their fields and discriminators; other multi-line types point to `describeTool()` for the full declaration ([#11501](https://github.com/earendil-works/pi/issues/11501)) +- Fixed the model-scope startup notice being left in the shell after exiting fullscreen mode. It now appears inside pi's fullscreen header ([#11508](https://github.com/earendil-works/pi/issues/11508)) +- Fixed startup from a deleted working directory crashing with an uncaught `uv_cwd` stack trace. pi now asks the user to change to an existing directory ([#11509](https://github.com/earendil-works/pi/issues/11509)) +- Fixed failed session writes leaving entries in memory that were absent from disk, so later entries pointed to missing parents and resumed sessions lost earlier history and tools ([#11460](https://github.com/earendil-works/pi/issues/11460)) +- Fixed `/tree` branch summaries ignoring the session's thinking level, which could make reasoning models reject summary requests with an unsupported `none` effort ([#11444](https://github.com/earendil-works/pi/issues/11444)) +- Fixed hidden thinking with an empty hidden-thinking label still adding blank rows before answers and tool calls ([#11402](https://github.com/earendil-works/pi/issues/11402)) +- Fixed package installation and removal printing npm audit and funding advice, including `npm audit fix` instructions that could install a second copy of pi into the package directory ([#11409](https://github.com/earendil-works/pi/issues/11409)) +- Fixed replacing an extension editor while a dialog was open hiding the dialog and leaving it unanswerable. The new editor now appears after the dialog closes ([#11431](https://github.com/earendil-works/pi/issues/11431)) +- Fixed `grep` context lines being offset from ripgrep's match line numbers when a file contains a lone carriage return ([#11398](https://github.com/earendil-works/pi/issues/11398)) +- Fixed `--model /` failing for provider IDs containing slashes, such as URL-derived llama.cpp provider IDs ([#11394](https://github.com/earendil-works/pi/issues/11394)) +- Fixed slow session forking on filesystems where opening files is expensive by writing the fork through one file handle ([#11253](https://github.com/earendil-works/pi/issues/11253)) +- Fixed `pi.exec()` corrupting UTF-8 characters split across stdout or stderr chunks ([#11312](https://github.com/earendil-works/pi/issues/11312)) +- Fixed `pi.exec()` returning empty stderr when a command could not be spawned, such as a missing executable. It now preserves the spawn error message ([#11257](https://github.com/earendil-works/pi/pull/11257) by [@muyiyr](https://github.com/muyiyr)) +- Fixed a configured package failing to install crashing startup. pi now skips the package, reports an extension warning, and retries on the next start or `/reload`; explicit install and update commands still fail with a nonzero exit ([#11304](https://github.com/earendil-works/pi/pull/11304) by [@Lincoln504](https://github.com/Lincoln504)) +- Fixed the todo extension example crashing the TUI when a tool call failed argument validation ([#8057](https://github.com/earendil-works/pi/pull/8057) by [@cyzlmh](https://github.com/cyzlmh)) +- Updated `undici` to 8.12.3 and the Gondolin extension example to 0.14.0 for security advisories +- Fixed OpenRouter cost totals using catalog estimates instead of reported billing for Chat Completions and Anthropic Messages. BYOK totals include both OpenRouter charges and reported upstream costs; component costs remain catalog estimates ([#9980](https://github.com/earendil-works/pi/issues/9980)) +- Fixed OpenCode Zen and OpenCode Go requests using shared endpoints instead of their protocol-specific OpenAI, Anthropic, Google, and System One endpoints ([#11434](https://github.com/earendil-works/pi/pull/11434) by [@vimtor](https://github.com/vimtor)) +- Fixed Zhipu/Z.ai usage-limit errors (code `1308`) being treated as transient HTTP 429 errors, causing automatic retries that cannot succeed until the usage cap resets ([#11537](https://github.com/earendil-works/pi/issues/11537)) +- Fixed LiteLLM's `ContextWindowExceededError` not being recognized as context overflow ([#11492](https://github.com/earendil-works/pi/issues/11492)) +- Fixed Gemini tool schemas sending unsupported `const` keywords. They are now converted to single-value `enum` schemas without rewriting literal data in defaults, examples, or enum values, or losing properties named `__proto__` ([#11308](https://github.com/earendil-works/pi/issues/11308)) +- Fixed the faux provider counting simulated cache-write tokens as both input and cache writes +- Fixed Orca terminals falling back to text for inline images. Orca is now recognized as supporting true color and hyperlinks, and Kitty images when it advertises `ORCA_IMAGE_PROTOCOL=kitty` ([#9329](https://github.com/earendil-works/pi/pull/9329) by [@XRX193](https://github.com/XRX193)) +- Fixed bare Markdown URLs swallowing following CJK punctuation and text; explicit Markdown links and angle autolinks are unchanged ([#10934](https://github.com/earendil-works/pi/pull/10934) by [@haoqixu](https://github.com/haoqixu)) +- Fixed Kitty-protocol F1–F12 shortcuts, including modified function keys, not being recognized, and F13+ and other functional keys inserting invisible private-use characters into the editor ([#11419](https://github.com/earendil-works/pi/issues/11419), [#11345](https://github.com/earendil-works/pi/issues/11345)) +- Fixed Kitty images inside padded or colored boxes disappearing when their first row scrolled above the fullscreen viewport. Partially visible images are now cropped correctly, including beside other text +- Fixed deeply nested Markdown blockquotes producing exponentially many lines, hanging or crashing rendering once their borders exhausted the available width ([#11272](https://github.com/earendil-works/pi/issues/11272)) +- Fixed Claude sessions failing with `role 'system' must precede an 'assistant' message` when a system update was followed by an empty or reasoning-only assistant reply ([#11091](https://github.com/earendil-works/pi/issues/11091)) +- Fixed Claude mid-conversation thinking-level changes failing with `Invalid effort level` on accounts whose organization caps effort below `high` ([#10063](https://github.com/earendil-works/pi/issues/10063)) +- Fixed Kimi K3.1 Pro models using Kimi K2 request settings on Kimi Coding, Moonshot, and OpenCode Go. They now send the correct reasoning effort, preserve earlier thinking, accept tools added mid-conversation, and use K3-family pricing +- Fixed OpenCode DeepSeek V4.1 Flash requests failing when long cache retention sent the unsupported `prompt_cache_retention` parameter +- Fixed GPT-5.4 and GPT-5.5 failing with `Missing namespace for function_call` after switching from a model that had called tools ([#11415](https://github.com/earendil-works/pi/issues/11415)) +- Fixed Bedrock Claude requests failing when preferred strict tools exceed the provider's strict-tool limits. Requests now retry once with preferred tools non-strict while keeping required strict tools strict ([#11416](https://github.com/earendil-works/pi/issues/11416)) +- Fixed OpenRouter's `~anthropic/claude-*-latest` aliases losing earlier thinking after tool calls by using the Anthropic Messages API ([#9534](https://github.com/earendil-works/pi/issues/9534)) +- Fixed terminal titles containing control characters being able to inject terminal escape sequences, and pi-messages streams accepting invalid content-block indices +- Fixed fullscreen double-click selection splitting paths and tokens at `~`, `.`, `\`, and `+`; word characters now match iTerm2's defaults ([#10980](https://github.com/earendil-works/pi/issues/10980)) +- Fixed display math wider than the content area being split apart by line wrapping; it now falls back to raw LaTeX ([#11483](https://github.com/earendil-works/pi/issues/11483)) +- Fixed styled words ending exactly at the wrap width adding a leading space on the next line, and strikethrough leaking into wrapped-line padding ([#10925](https://github.com/earendil-works/pi/issues/10925)) ### Removed diff --git a/packages/tui/CHANGELOG.md b/packages/tui/CHANGELOG.md index 1b13cf0a7..04fdf0893 100644 --- a/packages/tui/CHANGELOG.md +++ b/packages/tui/CHANGELOG.md @@ -20,6 +20,11 @@ - Fixed `wrapTextWithAnsi()` starting the next line with a space when a styled word ended exactly at the wrap width, and strikethrough continuing into the right padding of wrapped lines ([#10925](https://github.com/earendil-works/pi/issues/10925)) - Fixed `Markdown` styling, highlighting, and wrapping every block on each render. It now reuses the top-level blocks of its last render, so a streaming message renders only its changed last block, and a resize reuses highlighted code. Reused blocks are held weakly and dropped by `invalidate()`, which a theme change must call because `MarkdownTheme.highlightCode` results are reused until then ([#10971](https://github.com/earendil-works/pi/issues/10971)) - Fixed `Text.setText()` with unchanged text dropping the wrapped lines, so renderers that set the same text on every update wrapped it again ([#10971](https://github.com/earendil-works/pi/issues/10971)) +- Fixed Orca terminals falling back to text for inline images. Orca is now recognized as supporting true color and hyperlinks, and Kitty images when it advertises `ORCA_IMAGE_PROTOCOL=kitty` ([#9329](https://github.com/earendil-works/pi/pull/9329) by [@XRX193](https://github.com/XRX193)) +- Fixed bare Markdown URLs swallowing following CJK punctuation and text; explicit Markdown links and angle autolinks are unchanged ([#10934](https://github.com/earendil-works/pi/pull/10934) by [@haoqixu](https://github.com/haoqixu)) +- Fixed Kitty-protocol F1–F12 shortcuts, including modified function keys, not being recognized, and F13+ and other functional keys inserting invisible private-use characters into the editor ([#11419](https://github.com/earendil-works/pi/issues/11419), [#11345](https://github.com/earendil-works/pi/issues/11345)) +- Fixed Kitty images inside padded or colored boxes disappearing when their first row scrolled above the fullscreen viewport. Partially visible images are now cropped correctly, including beside other text +- Fixed deeply nested Markdown blockquotes producing exponentially many lines, hanging or crashing rendering once their borders exhausted the available width ([#11272](https://github.com/earendil-works/pi/issues/11272)) ## [1.4.0] - 2026-10-21 From ee7f61f92c89da29b2df05b21b2bb59fee855559 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 10:19:47 +0000 Subject: [PATCH 244/271] docs(chord): the plugin model's points 3, 4, 5 and 8, imports and builds --- packages/chord/docs/next/decisions.md | 40 +++++++++++++++++++++++++++ packages/chord/docs/next/slices.md | 5 ++++ 2 files changed, 45 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 3b38c7bf3..594900030 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2165,6 +2165,46 @@ validation and copies the v2 kernel added around Delta. Contracts are data and bind through `accepts` whatever Chord runs. One facet imports from one API version only (checked at build). Only base code imports Chord directly. Bundling is slice 10's host packages: the version is part of the name (`kinds.hostPackages` lists `@acme/app/v3`, `@acme/app/v2`), each process provides its own modules. +- **The plugin model, points 3, 4, 5 and 8, imports and builds (owner, 2026-11-01, by voice: "yeah that checks out"; "lock the decisions on the + points which was discussed"; primer `docs/next/plugin-primer/`):** + (3) **Split by host kind, for the app and for plugins alike.** The app's API per version has a shared module + (`@acme/app/v3`: Chord's re-exported surface, host kinds, the principal type, `defineFacet` bound to it, contracts + and registries that cross hosts) and per-kind modules (`/v3/web`, `/v3/tui`, `/v3/server`, `/v3/worker` where + needed: UI components, registries and `local` contracts that live on that kind only). A plugin's contract files + mirror it: `contract.ts` (shared) and `contract.server.ts`, `contract.worker.ts`, `contract.web.tsx`, + `contract.tui.ts` as needed. A contract file mentions native things (React components, functions) only through + `import type`; at runtime it imports only the shared API and its plugin's data-only files (a hard rule). The build + checks every import by the file it resolves to: a per-kind file (`.web.` etc.) or per-kind API module only for that + kind, and a file importing a package the kind does not provide (React in a worker) refused. Shared UI between + plugins: a plain component through an imported file (inlined into the importer's script, frozen at its build), or + a registry when one live instance is needed (React contexts, a feature others embed). + (4) **Host kinds on registry tokens** are covered by placement: a token used on one kind lives in that kind's file, + so other kinds cannot import it; an explicit `hosts` marker only if a shared token must be limited later. + Registries version by hand: readers and writers work across versions through the token's version and the defining + plugin's `accepts`; no fingerprint for native values (decided earlier, unchanged). + (5) **Third parties get the app's contracts through its versioned API:** each built-in plugin keeps its contract + files; the API re-exports the ones the app promises (`export { Notes } from "../plugins/notes/contract.ts"`), so a + plugin built against `/v2` keeps v2's contracts; the others stay private. No per-area subpackages. + (Imports) **Plainly, with no resolution magic:** within one repository (a root, or a repository of several + plugins), relative paths to the files (`../notes/contract.web.tsx`); across repositories, the package name with its + exports (`@bob/notes`, `@bob/notes/web`), pinned in `dependencies` by an npm version or a git ref + (`github:bob/notes#`; npm's git dependencies, no registry publishing needed; a plugin in a subfolder of a + multi-plugin repository is cloned and linked by the plugins component, which npm cannot do). Rejected: one + specifier resolving to a sibling or an npm package depending on `dependencies` ("kinda magic"). Plugins are + installed from git by cloning a branch, tag or commit into a plugins root (as the coding agent's git packages). + (8, base code) **Base code shrinks to the API, bootstrap and a start-data facet:** the vocabulary (the versioned API modules, + host packages, never reloaded), bootstrap (creating hosts, opening the store, components such as `createHttp` and + `createWorkers`, which take settings and process values and return facets, their natives opened by Chord, passed as + the same object at every reload) and one small facet serving process values (start data, the opened store) as core + objects. Every implementation, the harness and the providers of core registries included, is a plugin in the app's + own root, loaded, reloaded and failing per plugin like any other; the app may mark its root required, so a core + plugin failing at startup is fatal. Built-in plugins follow the same rules as any plugin. + (Build) **No build hooks from plugins:** the server builds every plugin with one esbuild call from TypeScript source + (no `dist` needed; installs run with scripts off); what the build handles (TS, TSX, JS, JSON, CSS, assets and WASM + by URL) is the plugin's toolbox, generated files are committed; a CSS preprocessor or similar, if ever needed, is + enabled by the app per host kind, never by a plugin (a stranger's web plugin must not run code on the server). + To do: a skill and documentation for agents describing a plugin's layout, its contract files, imports and how the + host builds it. Open from the primer: (6) `testPlugin` with stub facets, (7) single-file plugins. - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index 084d3456e..c4619becc 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -187,6 +187,11 @@ deleted (the slice's spec and code are the record). Moved here from `carry-over. from"): remove `build.ts`'s by-name lookup, allow relative imports into another plugin of the same root, refuse paths leaving the root; examples 35-38 and fixtures back to relative imports; drop the six `tsconfig` path aliases; plugins README; the guide's chapter "Plugins Using Plugins". (Slice 10 built by-name imports.) +- **The plugin model's other points** (decision "The plugin model, points 3, 4, 5 and 8, imports and builds"): per-kind + API modules and plugin contract files with the build's per-kind import check; package-name imports with npm or git + pins (subfolders of multi-plugin repositories cloned and linked); installing plugins from git; base code reduced to + the API, bootstrap and a start-data facet, the harness and core registry providers as plugins, a required root; a + skill and documentation for agents on a plugin's layout and build. ### After slice 11 From 95e2015f3bbca7bace8311b0ddf743367382b9b1 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 10:22:50 +0000 Subject: [PATCH 245/271] docs(chord): single-file plugins --- packages/chord/docs/next/decisions.md | 7 +++++++ packages/chord/docs/next/slices.md | 2 ++ 2 files changed, 9 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 594900030..b972b1044 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2205,6 +2205,13 @@ validation and copies the v2 kernel added around Delta. enabled by the app per host kind, never by a plugin (a stranger's web plugin must not run code on the server). To do: a skill and documentation for agents describing a plugin's layout, its contract files, imports and how the host builds it. Open from the primer: (6) `testPlugin` with stub facets, (7) single-file plugins. +- **Single-file plugins (owner, 2026-11-01: "agreed"; plugin model point 7):** a root may hold plain files named + `..ts(x)` (`hello.server.ts`, `hello.web.tsx`): files with the same first name are one plugin named + after it, one facet per file. The name must be a valid unscoped name; a collision with a directory plugin of the + same root is reported like any duplicate. No `package.json`, so no npm dependencies; it imports only the app's API + (host packages), no other files and no contract files for others; it may serve objects, define contracts for + itself inline and contribute to registries. When it needs more, it becomes a directory with the same name, which a + reload treats as the same plugin. The watcher's `set` mark covers files appearing and disappearing at a root's top. - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index c4619becc..9b9c357c1 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -192,6 +192,8 @@ deleted (the slice's spec and code are the record). Moved here from `carry-over. pins (subfolders of multi-plugin repositories cloned and linked); installing plugins from git; base code reduced to the API, bootstrap and a start-data facet, the harness and core registry providers as plugins, a required root; a skill and documentation for agents on a plugin's layout and build. +- **Single-file plugins** (decision "Single-file plugins"): discovery of `..ts(x)` files, grouped by + name; their import rule; the watcher's `set` for them. ### After slice 11 From 7edaa131dd2c1d7ebcedb8ca76da7e38ce719c74 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 10:26:25 +0000 Subject: [PATCH 246/271] docs(chord): testing a plugin alone --- packages/chord/docs/next/decisions.md | 11 +++++++++++ packages/chord/docs/next/slices.md | 2 ++ 2 files changed, 13 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index b972b1044..ea37ad100 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2212,6 +2212,17 @@ validation and copies the v2 kernel added around Delta. (host packages), no other files and no contract files for others; it may serve objects, define contracts for itself inline and contribute to registries. When it needs more, it becomes a directory with the same name, which a reload treats as the same plugin. The watcher's `set` mark covers files appearing and disappearing at a root's top. +- **Testing a plugin alone (owner, 2026-11-01: "let's do that"; plugin model point 6):** `testPlugin(dir, { kind, + modules, stubs, start?, principal? }, ctx)` builds the plugin as production does and loads it on an in-memory host + of that kind with an in-memory store. A plugin depends on four things, each answered without a mocking framework: + the app's API and Chord are the real modules (definitions only); other plugins' and core objects are reached + through contracts, so the test passes stubs, ordinary facets serving those contracts with fake data; registries + the plugin contributes to are served by the helper, which collects the contributions for the test to read (a + registry it reads gets fake entries from a stub); process values come as fake start data. The result is a live host: + the test gets the plugin's objects, calls them, watches their state, and acts as any principal. A contract no stub + serves leaves its requests waiting `not_served`; the helper lists those names, so the test says what to stub. + Several hosts at once (a server plugin calling a worker) is a later, larger helper over memory links; the app kit + may ship stubs for its core contracts. - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index 9b9c357c1..3c300ee62 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -194,6 +194,8 @@ deleted (the slice's spec and code are the record). Moved here from `carry-over. skill and documentation for agents on a plugin's layout and build. - **Single-file plugins** (decision "Single-file plugins"): discovery of `..ts(x)` files, grouped by name; their import rule; the watcher's `set` for them. +- **`testPlugin`** (decision "Testing a plugin alone"): stub facets, collected contributions, the list of contracts + nobody serves. ### After slice 11 From 2dd44fb1c8078267c9507d95d257671dfb7e990d Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 10:33:06 +0000 Subject: [PATCH 247/271] docs(chord): app kit shape --- packages/chord/docs/next/decisions.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index ea37ad100..fa7a39c94 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2223,6 +2223,15 @@ validation and copies the v2 kernel added around Delta. serves leaves its requests waiting `not_served`; the helper lists those names, so the test says what to stub. Several hosts at once (a server plugin calling a worker) is a later, larger helper over memory links; the app kit may ship stubs for its core contracts. +- **The app kit, point 1: its shape (owner, 2026-11-01, by voice: "yeah"; app kit design pass):** the app's own + layout is fixed: `src/` with the API versions (`src/api/v3/`, per host kind as decided for the plugin model) and the + built-in plugins root (`plugins/`). The data directory is deployment configuration, outside the app: a `startApp` + option set by the app's own start script (no environment variables: owner, "no environment variables"), default a + dot directory named after the app in the home directory; it holds the store, the user's plugins root, logs and the + workers' data. The server is fixed (one per app, clients connect to it). Workers are a list of worker kinds the app + defines, each a host kind with the app's own params (sharding is the app's: a project, a tenant, a document; the kit + knows no sessions) and how to start it (entry, start data per address, principal, idle time: the workers + component's options); zero kinds is a server with clients only (a chat app). - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From 87f69dbf3cef9a22f1b4ad82780edf8028aad3f3 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 10:36:42 +0000 Subject: [PATCH 248/271] docs(chord): app kit principals and users --- packages/chord/docs/next/decisions.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index fa7a39c94..e513dce86 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2232,6 +2232,16 @@ validation and copies the v2 kernel added around Delta. defines, each a host kind with the app's own params (sharding is the app's: a project, a tenant, a document; the kit knows no sessions) and how to start it (entry, start data per address, principal, idle time: the workers component's options); zero kinds is a server with clients only (a chat app). +- **The app kit, point 2: principals and users (owner, 2026-11-01, by voice: "yeah I guess. just make it nice and + concise and easy to use and flexible"; "too opinionated might be a problem"):** three things kept apart. (1) The + principal is tiny (a kind and an id: it travels on every call and is part of objects' identity) and the app defines + its type; the kit requires only the variants it needs itself (the server and its worker kinds). (2) User data + (avatar, bio, availability) is ordinary app data in objects keyed by the user's id, never in the principal. (3) + Authentication turns a connection into a principal. The kit ships optional plugins: a users plugin storing accounts + with a profile schema the app supplies (a Slack-like profile or just a name), and authenticator plugins (device + pairing, invite links, an ephemeral one for an IRC-like "pick a name and you're in": a fresh id with the name, no + stored account, gone with the connection); an app picks, replaces or writes its own. Rejected: a fixed principal type + with roles. - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From 7e7c98a0f505075f67291d4425cf41643678f278 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Mon, 2 Nov 2026 11:27:14 +0100 Subject: [PATCH 249/271] docs(coding-agent): note branch summary context edits --- packages/coding-agent/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 46a5bbc9f..15c302597 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -110,6 +110,7 @@ - Fixed fullscreen double-click selection splitting paths and tokens at `~`, `.`, `\`, and `+`; word characters now match iTerm2's defaults ([#10980](https://github.com/earendil-works/pi/issues/10980)) - Fixed display math wider than the content area being split apart by line wrapping; it now falls back to raw LaTeX ([#11483](https://github.com/earendil-works/pi/issues/11483)) - Fixed styled words ending exactly at the wrap width adding a leading space on the next line, and strikethrough leaking into wrapped-line padding ([#10925](https://github.com/earendil-works/pi/issues/10925)) +- Fixed `/tree` branch summaries including failed assistant replies and original content that context edits had hidden or replaced. Summaries now honor the same context edits as live requests and compaction ([#11602](https://github.com/earendil-works/pi/issues/11602)) ### Removed From 513f6f1ce8fc982b36ae0e9e2ef4c7379cd78fe3 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Mon, 2 Nov 2026 11:30:55 +0100 Subject: [PATCH 250/271] fix: keep Chord example dependencies out of releases --- .../37-plugin-imports/deps/example-greeting/package.json | 1 + .../examples/37-plugin-imports/deps/example-native/package.json | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/chord/test/next/examples/37-plugin-imports/deps/example-greeting/package.json b/packages/chord/test/next/examples/37-plugin-imports/deps/example-greeting/package.json index ae18d3b71..40d2288d8 100644 --- a/packages/chord/test/next/examples/37-plugin-imports/deps/example-greeting/package.json +++ b/packages/chord/test/next/examples/37-plugin-imports/deps/example-greeting/package.json @@ -1,6 +1,7 @@ { "name": "example-greeting", "version": "1.0.0", + "private": true, "type": "module", "main": "index.js" } diff --git a/packages/chord/test/next/examples/37-plugin-imports/deps/example-native/package.json b/packages/chord/test/next/examples/37-plugin-imports/deps/example-native/package.json index c05122b96..0e7a2555f 100644 --- a/packages/chord/test/next/examples/37-plugin-imports/deps/example-native/package.json +++ b/packages/chord/test/next/examples/37-plugin-imports/deps/example-native/package.json @@ -1,6 +1,7 @@ { "name": "example-native", "version": "1.0.0", + "private": true, "type": "commonjs", "main": "index.js" } From f9a26365568127461c47a07a37dcb42c140a3ac5 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Mon, 2 Nov 2026 11:35:40 +0100 Subject: [PATCH 251/271] Release v1.5.0 --- nix/model-catalog.json | 2 +- package-lock.json | 72 +++++++++---------- packages/agent/CHANGELOG.md | 2 +- packages/agent/package.json | 6 +- packages/ai/CHANGELOG.md | 2 +- packages/ai/package.json | 6 +- packages/ai/src/utils/version.ts | 2 +- packages/chord/CHANGELOG.md | 2 +- packages/chord/package.json | 6 +- packages/codemode/CHANGELOG.md | 2 +- packages/codemode/package.json | 2 +- packages/coding-agent/CHANGELOG.md | 2 +- .../package-lock.json | 4 +- .../custom-provider-anthropic/package.json | 2 +- .../custom-provider-gitlab-duo/package.json | 2 +- .../extensions/gondolin/package-lock.json | 4 +- .../examples/extensions/gondolin/package.json | 2 +- .../extensions/with-deps/package-lock.json | 4 +- .../extensions/with-deps/package.json | 2 +- .../install-lock/package-lock.json | 56 +++++++-------- .../coding-agent/install-lock/package.json | 4 +- packages/coding-agent/package.json | 12 ++-- packages/durable/CHANGELOG.md | 2 +- packages/durable/package.json | 12 ++-- packages/env/CHANGELOG.md | 2 +- packages/env/package.json | 6 +- packages/evals/package.json | 6 +- packages/mcp/CHANGELOG.md | 2 +- packages/mcp/package.json | 2 +- packages/schema/CHANGELOG.md | 2 +- packages/schema/package.json | 2 +- packages/telemetry/CHANGELOG.md | 2 +- packages/telemetry/package.json | 2 +- packages/tui/CHANGELOG.md | 2 +- packages/tui/package.json | 2 +- 35 files changed, 121 insertions(+), 121 deletions(-) diff --git a/nix/model-catalog.json b/nix/model-catalog.json index 3c31c2cca..3b4641ee5 100644 --- a/nix/model-catalog.json +++ b/nix/model-catalog.json @@ -1,3 +1,3 @@ { - "revision": "sha256-7fe1b41feaf43a7aad4bc832e6b306c7ced56e6d41bbc724b74e91eeacff6edf" + "revision": "sha256-f4dbabe79cda4b38aee4a761b23fd5eaf526ac2e28fed3c9ee86e37ca0e5d71e" } diff --git a/package-lock.json b/package-lock.json index 8abc3354d..a0749b93b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5654,11 +5654,11 @@ }, "packages/agent": { "name": "@earendil-works/pi-agent-core", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-schema": "1.4.0" + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-schema": "1.5.0" }, "devDependencies": { "@types/node": "22.19.19", @@ -5677,13 +5677,13 @@ }, "packages/ai": { "name": "@earendil-works/pi-ai", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "0.129.0", "@aws-sdk/client-bedrock-runtime": "3.1127.0", - "@earendil-works/pi-schema": "1.4.0", - "@earendil-works/pi-telemetry": "1.4.0", + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0", "@google/genai": "2.21.0", "@smithy/node-http-handler": "4.12.1", "http-proxy-agent": "9.1.0", @@ -5766,11 +5766,11 @@ }, "packages/chord": { "name": "@earendil-works/chord", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { - "@earendil-works/pi-schema": "1.4.0", - "@earendil-works/pi-telemetry": "1.4.0" + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0" }, "devDependencies": { "@types/ws": "8.18.2", @@ -5795,7 +5795,7 @@ }, "packages/codemode": { "name": "@earendil-works/pi-codemode", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { "quickjs-wasi": "3.6.2" @@ -5809,14 +5809,14 @@ }, "packages/coding-agent": { "name": "@earendil-works/pi-coding-agent", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { - "@earendil-works/pi-agent-core": "1.4.0", - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-codemode": "1.4.0", - "@earendil-works/pi-mcp": "1.4.0", - "@earendil-works/pi-tui": "1.4.0", + "@earendil-works/pi-agent-core": "1.5.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-codemode": "1.5.0", + "@earendil-works/pi-mcp": "1.5.0", + "@earendil-works/pi-tui": "1.5.0", "@silvia-odwyer/photon-node": "0.3.4", "brace-expansion": "5.0.12", "chalk": "6.0.0", @@ -5853,25 +5853,25 @@ }, "packages/coding-agent/examples/extensions/custom-provider-anthropic": { "name": "pi-extension-custom-provider-anthropic", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "@anthropic-ai/sdk": "0.52.0" } }, "packages/coding-agent/examples/extensions/custom-provider-gitlab-duo": { "name": "pi-extension-custom-provider-gitlab-duo", - "version": "1.4.0" + "version": "1.5.0" }, "packages/coding-agent/examples/extensions/gondolin": { "name": "pi-extension-gondolin", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "@earendil-works/gondolin": "0.14.0" } }, "packages/coding-agent/examples/extensions/with-deps": { "name": "pi-extension-with-deps", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "ms": "2.1.3" }, @@ -5925,14 +5925,14 @@ }, "packages/durable": { "name": "@earendil-works/pi-durable", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { - "@earendil-works/chord": "1.4.0", - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-codemode": "1.4.0", - "@earendil-works/pi-schema": "1.4.0", - "@earendil-works/pi-telemetry": "1.4.0", + "@earendil-works/chord": "1.5.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-codemode": "1.5.0", + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0", "@silvia-odwyer/photon": "0.3.3", "diff": "8.0.4" }, @@ -5947,11 +5947,11 @@ }, "packages/env": { "name": "@earendil-works/pi-env", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { - "@earendil-works/chord": "1.4.0", - "@earendil-works/pi-durable": "1.4.0" + "@earendil-works/chord": "1.5.0", + "@earendil-works/pi-durable": "1.5.0" }, "devDependencies": { "vitest": "4.1.12" @@ -5962,10 +5962,10 @@ }, "packages/evals": { "name": "@earendil-works/pi-evals", - "version": "1.4.0", + "version": "1.5.0", "devDependencies": { - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-coding-agent": "1.4.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-coding-agent": "1.5.0", "@types/node": "22.19.19", "@vitest-evals/core": "0.15.0", "vitest": "4.1.12", @@ -5974,7 +5974,7 @@ }, "packages/mcp": { "name": "@earendil-works/pi-mcp", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { "cross-spawn": "7.0.6" @@ -5989,7 +5989,7 @@ }, "packages/schema": { "name": "@earendil-works/pi-schema", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "devDependencies": { "@cfworker/json-schema": "4.1.1", @@ -6007,7 +6007,7 @@ }, "packages/telemetry": { "name": "@earendil-works/pi-telemetry", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "devDependencies": { "@types/node": "22.19.19", @@ -6019,7 +6019,7 @@ }, "packages/tui": { "name": "@earendil-works/pi-tui", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { "get-east-asian-width": "1.6.0", diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index 96b4b9428..a481d47f0 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [Unreleased] +## [1.5.0] - 2026-11-02 ### Breaking Changes diff --git a/packages/agent/package.json b/packages/agent/package.json index 5d7091e4a..e00a8e1e7 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-agent-core", - "version": "1.4.0", + "version": "1.5.0", "description": "General-purpose agent with transport abstraction, state management, and attachment support", "type": "module", "main": "./dist/index.js", @@ -23,8 +23,8 @@ "prepublishOnly": "npm run build" }, "dependencies": { - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-schema": "1.4.0" + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-schema": "1.5.0" }, "keywords": [ "ai", diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 5d197d814..cc4f0d9ac 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [Unreleased] +## [1.5.0] - 2026-11-02 ### Breaking Changes diff --git a/packages/ai/package.json b/packages/ai/package.json index 8de0434f8..57e8f262c 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-ai", - "version": "1.4.0", + "version": "1.5.0", "description": "Unified LLM API with automatic model discovery and provider configuration", "type": "module", "main": "./dist/index.js", @@ -69,8 +69,8 @@ "dependencies": { "@anthropic-ai/sdk": "0.129.0", "@aws-sdk/client-bedrock-runtime": "3.1127.0", - "@earendil-works/pi-schema": "1.4.0", - "@earendil-works/pi-telemetry": "1.4.0", + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0", "@google/genai": "2.21.0", "@smithy/node-http-handler": "4.12.1", "http-proxy-agent": "9.1.0", diff --git a/packages/ai/src/utils/version.ts b/packages/ai/src/utils/version.ts index 18fe5a582..f1e6f3211 100644 --- a/packages/ai/src/utils/version.ts +++ b/packages/ai/src/utils/version.ts @@ -1,2 +1,2 @@ // The @earendil-works/pi-ai package version. scripts/sync-versions.js keeps it in sync with package.json. -export const PI_AI_VERSION = "1.4.0"; +export const PI_AI_VERSION = "1.5.0"; diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index 1a1382eb7..b4fd5fdfd 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [Unreleased] +## [1.5.0] - 2026-11-02 ### Breaking Changes diff --git a/packages/chord/package.json b/packages/chord/package.json index edced8cd5..6e8e12a4d 100644 --- a/packages/chord/package.json +++ b/packages/chord/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/chord", - "version": "1.4.0", + "version": "1.5.0", "description": "Typed host runtime for plugins, contracts, replicated state, and RPC over JSON links", "type": "module", "main": "./dist/index.js", @@ -90,8 +90,8 @@ "node": ">=22.19.0" }, "dependencies": { - "@earendil-works/pi-schema": "1.4.0", - "@earendil-works/pi-telemetry": "1.4.0" + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0" }, "peerDependencies": { "esbuild": "0.28.2", diff --git a/packages/codemode/CHANGELOG.md b/packages/codemode/CHANGELOG.md index a4c2fd624..446a26034 100644 --- a/packages/codemode/CHANGELOG.md +++ b/packages/codemode/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [Unreleased] +## [1.5.0] - 2026-11-02 ### Changed diff --git a/packages/codemode/package.json b/packages/codemode/package.json index 9fed9d017..006549146 100644 --- a/packages/codemode/package.json +++ b/packages/codemode/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-codemode", - "version": "1.4.0", + "version": "1.5.0", "description": "Sandboxed JavaScript execution where the only capability is calling injected tools", "type": "module", "main": "./dist/index.js", diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 15c302597..b3be0124e 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [Unreleased] +## [1.5.0] - 2026-11-02 ### New Features diff --git a/packages/coding-agent/examples/extensions/custom-provider-anthropic/package-lock.json b/packages/coding-agent/examples/extensions/custom-provider-anthropic/package-lock.json index b7d2ebab0..c6ed32b21 100644 --- a/packages/coding-agent/examples/extensions/custom-provider-anthropic/package-lock.json +++ b/packages/coding-agent/examples/extensions/custom-provider-anthropic/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-extension-custom-provider", - "version": "1.4.0", + "version": "1.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-extension-custom-provider", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "@anthropic-ai/sdk": "^0.52.0" } diff --git a/packages/coding-agent/examples/extensions/custom-provider-anthropic/package.json b/packages/coding-agent/examples/extensions/custom-provider-anthropic/package.json index b22368863..b4900f974 100644 --- a/packages/coding-agent/examples/extensions/custom-provider-anthropic/package.json +++ b/packages/coding-agent/examples/extensions/custom-provider-anthropic/package.json @@ -1,7 +1,7 @@ { "name": "pi-extension-custom-provider-anthropic", "private": true, - "version": "1.4.0", + "version": "1.5.0", "type": "module", "scripts": { "clean": "echo 'nothing to clean'", diff --git a/packages/coding-agent/examples/extensions/custom-provider-gitlab-duo/package.json b/packages/coding-agent/examples/extensions/custom-provider-gitlab-duo/package.json index dbdc63c72..54e070434 100644 --- a/packages/coding-agent/examples/extensions/custom-provider-gitlab-duo/package.json +++ b/packages/coding-agent/examples/extensions/custom-provider-gitlab-duo/package.json @@ -1,7 +1,7 @@ { "name": "pi-extension-custom-provider-gitlab-duo", "private": true, - "version": "1.4.0", + "version": "1.5.0", "type": "module", "scripts": { "clean": "echo 'nothing to clean'", diff --git a/packages/coding-agent/examples/extensions/gondolin/package-lock.json b/packages/coding-agent/examples/extensions/gondolin/package-lock.json index 99a06ff69..0887734dd 100644 --- a/packages/coding-agent/examples/extensions/gondolin/package-lock.json +++ b/packages/coding-agent/examples/extensions/gondolin/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-extension-gondolin", - "version": "1.4.0", + "version": "1.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-extension-gondolin", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "@earendil-works/gondolin": "0.14.0" } diff --git a/packages/coding-agent/examples/extensions/gondolin/package.json b/packages/coding-agent/examples/extensions/gondolin/package.json index d19d7190f..762d1d230 100644 --- a/packages/coding-agent/examples/extensions/gondolin/package.json +++ b/packages/coding-agent/examples/extensions/gondolin/package.json @@ -1,7 +1,7 @@ { "name": "pi-extension-gondolin", "private": true, - "version": "1.4.0", + "version": "1.5.0", "type": "module", "scripts": { "clean": "echo 'nothing to clean'", diff --git a/packages/coding-agent/examples/extensions/with-deps/package-lock.json b/packages/coding-agent/examples/extensions/with-deps/package-lock.json index 90c4e6339..2d84832c4 100644 --- a/packages/coding-agent/examples/extensions/with-deps/package-lock.json +++ b/packages/coding-agent/examples/extensions/with-deps/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-extension-with-deps", - "version": "1.4.0", + "version": "1.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-extension-with-deps", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "ms": "^2.1.3" }, diff --git a/packages/coding-agent/examples/extensions/with-deps/package.json b/packages/coding-agent/examples/extensions/with-deps/package.json index f22d436c7..c71ad77f2 100644 --- a/packages/coding-agent/examples/extensions/with-deps/package.json +++ b/packages/coding-agent/examples/extensions/with-deps/package.json @@ -1,7 +1,7 @@ { "name": "pi-extension-with-deps", "private": true, - "version": "1.4.0", + "version": "1.5.0", "type": "module", "scripts": { "clean": "echo 'nothing to clean'", diff --git a/packages/coding-agent/install-lock/package-lock.json b/packages/coding-agent/install-lock/package-lock.json index 521c82fef..e37a4fb76 100644 --- a/packages/coding-agent/install-lock/package-lock.json +++ b/packages/coding-agent/install-lock/package-lock.json @@ -1,14 +1,14 @@ { "name": "@earendil-works/pi-coding-agent-install", - "version": "1.4.0", + "version": "1.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@earendil-works/pi-coding-agent-install", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { - "@earendil-works/pi-coding-agent": "1.4.0" + "@earendil-works/pi-coding-agent": "1.5.0" }, "engines": { "node": ">=22.19.0" @@ -386,26 +386,26 @@ } }, "node_modules/@earendil-works/pi-agent-core": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-1.5.0.tgz", "license": "MIT", "dependencies": { - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-schema": "1.4.0" + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-schema": "1.5.0" }, "engines": { "node": ">=22.19.0" } }, "node_modules/@earendil-works/pi-ai": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-1.5.0.tgz", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "0.129.0", "@aws-sdk/client-bedrock-runtime": "3.1127.0", - "@earendil-works/pi-schema": "1.4.0", - "@earendil-works/pi-telemetry": "1.4.0", + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0", "@google/genai": "2.21.0", "@smithy/node-http-handler": "4.12.1", "http-proxy-agent": "9.1.0", @@ -481,8 +481,8 @@ } }, "node_modules/@earendil-works/pi-codemode": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-codemode/-/pi-codemode-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-codemode/-/pi-codemode-1.5.0.tgz", "license": "MIT", "dependencies": { "quickjs-wasi": "3.6.2" @@ -492,15 +492,15 @@ } }, "node_modules/@earendil-works/pi-coding-agent": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-coding-agent/-/pi-coding-agent-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-coding-agent/-/pi-coding-agent-1.5.0.tgz", "license": "MIT", "dependencies": { - "@earendil-works/pi-agent-core": "1.4.0", - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-codemode": "1.4.0", - "@earendil-works/pi-mcp": "1.4.0", - "@earendil-works/pi-tui": "1.4.0", + "@earendil-works/pi-agent-core": "1.5.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-codemode": "1.5.0", + "@earendil-works/pi-mcp": "1.5.0", + "@earendil-works/pi-tui": "1.5.0", "@silvia-odwyer/photon-node": "0.3.4", "brace-expansion": "5.0.12", "chalk": "6.0.0", @@ -527,8 +527,8 @@ } }, "node_modules/@earendil-works/pi-mcp": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-mcp/-/pi-mcp-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-mcp/-/pi-mcp-1.5.0.tgz", "license": "MIT", "dependencies": { "cross-spawn": "7.0.6" @@ -538,24 +538,24 @@ } }, "node_modules/@earendil-works/pi-schema": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-schema/-/pi-schema-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-schema/-/pi-schema-1.5.0.tgz", "license": "MIT", "engines": { "node": ">=22.19.0" } }, "node_modules/@earendil-works/pi-telemetry": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-telemetry/-/pi-telemetry-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-telemetry/-/pi-telemetry-1.5.0.tgz", "license": "MIT", "engines": { "node": ">=22.19.0" } }, "node_modules/@earendil-works/pi-tui": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-1.5.0.tgz", "license": "MIT", "dependencies": { "get-east-asian-width": "1.6.0", diff --git a/packages/coding-agent/install-lock/package.json b/packages/coding-agent/install-lock/package.json index 004cf3c90..a8746dfab 100644 --- a/packages/coding-agent/install-lock/package.json +++ b/packages/coding-agent/install-lock/package.json @@ -1,10 +1,10 @@ { "name": "@earendil-works/pi-coding-agent-install", - "version": "1.4.0", + "version": "1.5.0", "private": true, "description": "Lockfile root used by the Pi installer and updater.", "dependencies": { - "@earendil-works/pi-coding-agent": "1.4.0" + "@earendil-works/pi-coding-agent": "1.5.0" }, "overrides": { "protobufjs": "7.6.6" diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index f8e6d41cc..705be3d3d 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-coding-agent", - "version": "1.4.0", + "version": "1.5.0", "description": "Coding agent CLI with read, bash, edit, write tools and session management", "type": "module", "piConfig": { @@ -41,11 +41,11 @@ "prepublishOnly": "npm run clean && npm run build" }, "dependencies": { - "@earendil-works/pi-agent-core": "1.4.0", - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-codemode": "1.4.0", - "@earendil-works/pi-mcp": "1.4.0", - "@earendil-works/pi-tui": "1.4.0", + "@earendil-works/pi-agent-core": "1.5.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-codemode": "1.5.0", + "@earendil-works/pi-mcp": "1.5.0", + "@earendil-works/pi-tui": "1.5.0", "@silvia-odwyer/photon-node": "0.3.4", "brace-expansion": "5.0.12", "chalk": "6.0.0", diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 6e9dcdb21..f52841059 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [Unreleased] +## [1.5.0] - 2026-11-02 ### Breaking Changes diff --git a/packages/durable/package.json b/packages/durable/package.json index 495197c59..c32091c32 100644 --- a/packages/durable/package.json +++ b/packages/durable/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-durable", - "version": "1.4.0", + "version": "1.5.0", "description": "Durable conversation, task, and document runtime for Pi", "type": "module", "main": "./dist/index.js", @@ -158,11 +158,11 @@ "node": ">=22.19.0" }, "dependencies": { - "@earendil-works/chord": "1.4.0", - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-codemode": "1.4.0", - "@earendil-works/pi-schema": "1.4.0", - "@earendil-works/pi-telemetry": "1.4.0", + "@earendil-works/chord": "1.5.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-codemode": "1.5.0", + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0", "@silvia-odwyer/photon": "0.3.3", "diff": "8.0.4" }, diff --git a/packages/env/CHANGELOG.md b/packages/env/CHANGELOG.md index 96bd4e91e..aada8be3d 100644 --- a/packages/env/CHANGELOG.md +++ b/packages/env/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [Unreleased] +## [1.5.0] - 2026-11-02 ### Breaking Changes diff --git a/packages/env/package.json b/packages/env/package.json index 2bb941ce0..f78d49c20 100644 --- a/packages/env/package.json +++ b/packages/env/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-env", - "version": "1.4.0", + "version": "1.5.0", "description": "Remote execution environments for Pi Durable: a small daemon deployed over SSH and its ExecutionEnv client", "type": "module", "main": "./dist/index.js", @@ -45,8 +45,8 @@ "node": ">=22.19.0" }, "dependencies": { - "@earendil-works/chord": "1.4.0", - "@earendil-works/pi-durable": "1.4.0" + "@earendil-works/chord": "1.5.0", + "@earendil-works/pi-durable": "1.5.0" }, "devDependencies": { "vitest": "4.1.12" diff --git a/packages/evals/package.json b/packages/evals/package.json index fa4400219..f740c3368 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-evals", - "version": "1.4.0", + "version": "1.5.0", "private": true, "type": "module", "scripts": { @@ -12,8 +12,8 @@ "test": "vitest run --config vitest.test.config.ts" }, "devDependencies": { - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-coding-agent": "1.4.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-coding-agent": "1.5.0", "@types/node": "22.19.19", "@vitest-evals/core": "0.15.0", "vitest": "4.1.12", diff --git a/packages/mcp/CHANGELOG.md b/packages/mcp/CHANGELOG.md index 92e01bf9d..566400d62 100644 --- a/packages/mcp/CHANGELOG.md +++ b/packages/mcp/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [Unreleased] +## [1.5.0] - 2026-11-02 ### Changed diff --git a/packages/mcp/package.json b/packages/mcp/package.json index a884d2b0f..8a9bac663 100644 --- a/packages/mcp/package.json +++ b/packages/mcp/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-mcp", - "version": "1.4.0", + "version": "1.5.0", "description": "Standalone Model Context Protocol client for pi and other applications", "type": "module", "main": "./dist/index.js", diff --git a/packages/schema/CHANGELOG.md b/packages/schema/CHANGELOG.md index 404e93c57..57ce17ba7 100644 --- a/packages/schema/CHANGELOG.md +++ b/packages/schema/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [Unreleased] +## [1.5.0] - 2026-11-02 ### Added diff --git a/packages/schema/package.json b/packages/schema/package.json index 3fdb1b14f..4e176a57c 100644 --- a/packages/schema/package.json +++ b/packages/schema/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-schema", - "version": "1.4.0", + "version": "1.5.0", "description": "JSON Schema validation without eval: drafts 2020-12 and 7, compiled into plain closures", "type": "module", "main": "./dist/index.js", diff --git a/packages/telemetry/CHANGELOG.md b/packages/telemetry/CHANGELOG.md index 52617bb21..d673bc233 100644 --- a/packages/telemetry/CHANGELOG.md +++ b/packages/telemetry/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [Unreleased] +## [1.5.0] - 2026-11-02 ### Fixed diff --git a/packages/telemetry/package.json b/packages/telemetry/package.json index eca0a94e7..eeb30b234 100644 --- a/packages/telemetry/package.json +++ b/packages/telemetry/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-telemetry", - "version": "1.4.0", + "version": "1.5.0", "description": "Vendor-neutral telemetry contracts and typed schema utilities for pi", "type": "module", "main": "./dist/index.js", diff --git a/packages/tui/CHANGELOG.md b/packages/tui/CHANGELOG.md index 04fdf0893..cc7e85378 100644 --- a/packages/tui/CHANGELOG.md +++ b/packages/tui/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [Unreleased] +## [1.5.0] - 2026-11-02 ### Added diff --git a/packages/tui/package.json b/packages/tui/package.json index d5ce578be..74a49f0d3 100644 --- a/packages/tui/package.json +++ b/packages/tui/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-tui", - "version": "1.4.0", + "version": "1.5.0", "description": "Terminal User Interface library with differential rendering for efficient text-based applications", "type": "module", "main": "dist/index.js", From db6c08445593c4368460f42dc0b70656ad5c179e Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Mon, 2 Nov 2026 11:35:40 +0100 Subject: [PATCH 252/271] Add [Unreleased] section for next cycle --- packages/agent/CHANGELOG.md | 2 ++ packages/ai/CHANGELOG.md | 2 ++ packages/chord/CHANGELOG.md | 2 ++ packages/codemode/CHANGELOG.md | 2 ++ packages/coding-agent/CHANGELOG.md | 2 ++ packages/durable/CHANGELOG.md | 2 ++ packages/env/CHANGELOG.md | 2 ++ packages/mcp/CHANGELOG.md | 2 ++ packages/schema/CHANGELOG.md | 2 ++ packages/telemetry/CHANGELOG.md | 2 ++ packages/tui/CHANGELOG.md | 2 ++ 11 files changed, 22 insertions(+) diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index a481d47f0..9e80e2abf 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## [Unreleased] + ## [1.5.0] - 2026-11-02 ### Breaking Changes diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index cc4f0d9ac..67df897b0 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## [Unreleased] + ## [1.5.0] - 2026-11-02 ### Breaking Changes diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index b4fd5fdfd..ed7af6305 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## [Unreleased] + ## [1.5.0] - 2026-11-02 ### Breaking Changes diff --git a/packages/codemode/CHANGELOG.md b/packages/codemode/CHANGELOG.md index 446a26034..6c6de2736 100644 --- a/packages/codemode/CHANGELOG.md +++ b/packages/codemode/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## [Unreleased] + ## [1.5.0] - 2026-11-02 ### Changed diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index b3be0124e..36af238b5 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## [Unreleased] + ## [1.5.0] - 2026-11-02 ### New Features diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index f52841059..14750ec5a 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## [Unreleased] + ## [1.5.0] - 2026-11-02 ### Breaking Changes diff --git a/packages/env/CHANGELOG.md b/packages/env/CHANGELOG.md index aada8be3d..6c101bdd4 100644 --- a/packages/env/CHANGELOG.md +++ b/packages/env/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## [Unreleased] + ## [1.5.0] - 2026-11-02 ### Breaking Changes diff --git a/packages/mcp/CHANGELOG.md b/packages/mcp/CHANGELOG.md index 566400d62..a0e3f2cc8 100644 --- a/packages/mcp/CHANGELOG.md +++ b/packages/mcp/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## [Unreleased] + ## [1.5.0] - 2026-11-02 ### Changed diff --git a/packages/schema/CHANGELOG.md b/packages/schema/CHANGELOG.md index 57ce17ba7..1fb3218b0 100644 --- a/packages/schema/CHANGELOG.md +++ b/packages/schema/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## [Unreleased] + ## [1.5.0] - 2026-11-02 ### Added diff --git a/packages/telemetry/CHANGELOG.md b/packages/telemetry/CHANGELOG.md index d673bc233..a6dd650a8 100644 --- a/packages/telemetry/CHANGELOG.md +++ b/packages/telemetry/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## [Unreleased] + ## [1.5.0] - 2026-11-02 ### Fixed diff --git a/packages/tui/CHANGELOG.md b/packages/tui/CHANGELOG.md index cc7e85378..5c7b22acf 100644 --- a/packages/tui/CHANGELOG.md +++ b/packages/tui/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## [Unreleased] + ## [1.5.0] - 2026-11-02 ### Added From 09f2dac685ca67a33a45e8b5068db0805dfe85f6 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 10:40:04 +0000 Subject: [PATCH 253/271] docs(chord): app kit authentication --- packages/chord/docs/next/decisions.md | 9 +++++++++ packages/chord/docs/next/slices.md | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index e513dce86..7ccf9cefa 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2242,6 +2242,15 @@ validation and copies the v2 kernel added around Delta. pairing, invite links, an ephemeral one for an IRC-like "pick a name and you're in": a fresh id with the name, no stored account, gone with the connection); an app picks, replaces or writes its own. Rejected: a fixed principal type with roles. +- **The app kit, point 3: authentication (owner, 2026-11-01, by voice: "yeah, I guess something like that"):** (a) + no owner by loopback alone (any local process or OS user could claim it): the start script prints a link with a + one-time token (as Jupyter), whose visit sets a signed `HttpOnly` owner cookie for that browser; drive-by requests + from pages in that browser stay blocked by the security baseline (foreign requests, also other localhost ports, + never authenticated; Host and Origin checks). (b) Deny by default, in the HTTP component itself (a change to slice + 10's routes and sockets): a request without a principal is refused before plugin code unless its route or socket is + marked public (a landing page, a webhook, the ephemeral name login, which hands out a principal). Authenticators stay + the `Authenticators` registry, filled by the authenticator plugins the app chose; a session helper issues and revokes + sessions so authenticators do not each reinvent cookies and logout. - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index 3c300ee62..766d32211 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -181,6 +181,11 @@ deleted (the slice's spec and code are the record). Moved here from `carry-over. - **`rebuild` of tables** (key and column-type changes): `evolveTable` has no edit for them; build one only when a real case needs it. (Slice 6a's second spec review; out of slice 7b's scope, `slice-7b.md` §12.) +### The app kit + +- **Deny by default in the HTTP component** (decision "The app kit, point 3: authentication"): routes and sockets + refuse requests without a principal unless marked public; the owner link with a one-time token; a session helper. + ### The plugin slice (plugin model) - **Cross-plugin imports by relative path and npm** (decision "Contracts come from where the plugin's code came From e8c5ac945fb00ec1e37899e545fbdcc0d985b8b5 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 10:42:09 +0000 Subject: [PATCH 254/271] docs(chord): app kit command-line clients --- packages/chord/docs/next/decisions.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 7ccf9cefa..da0af569b 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2251,6 +2251,11 @@ validation and copies the v2 kernel added around Delta. marked public (a landing page, a webhook, the ephemeral name login, which hands out a principal). Authenticators stay the `Authenticators` registry, filled by the authenticator plugins the app chose; a session helper issues and revokes sessions so authenticators do not each reinvent cookies and logout. + (c) Command-line clients (owner: "that sounds good"; the TUI client too) connect over the same WebSocket with a + token in a header (no Origin, no Fetch Metadata: not a browser request; the token decides): on the same machine an + owner token file in the data directory readable only by the OS user running the app; from another machine a device + pairing flow (the tool shows a code and a link, the owner approves in a browser, the tool stores the token), the + token a session like any other, listed and revocable. - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From dbfb5bc105d63e11311779b2f3e8a4bf79a6856f Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 10:43:09 +0000 Subject: [PATCH 255/271] docs(chord): app kit storage --- packages/chord/docs/next/decisions.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index da0af569b..916af9f51 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2256,6 +2256,11 @@ validation and copies the v2 kernel added around Delta. owner token file in the data directory readable only by the OS user running the app; from another machine a device pairing flow (the tool shows a code and a link, the owner approves in a browser, the tool stores the token), the token a session like any other, listed and revocable. +- **The app kit, point 4: storage (owner, 2026-11-01, by voice: "yeah, I suppose that sounds right"):** in the data + directory the server has one store; each worker has its own directory under its worker kind, named by its address, + with its store in it; the kit opens and closes them. Plugins never use paths of the data directory: their state is + in their objects' documents and tables; a plugin that needs raw files gets a subdirectory of its own, owned by one of + its objects. - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From 3cec8e2b66fa8408c0cb9cbb6050df2574eacb86 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 10:50:22 +0000 Subject: [PATCH 256/271] docs(chord): app kit plugins, files and export --- packages/chord/docs/next/decisions.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 916af9f51..5494adbd9 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2261,6 +2261,17 @@ validation and copies the v2 kernel added around Delta. with its store in it; the kit opens and closes them. Plugins never use paths of the data directory: their state is in their objects' documents and tables; a plugin that needs raw files gets a subdirectory of its own, owned by one of its objects. +- **The app kit, point 5: plugins, files and export (owner, 2026-11-01: "okay", "yeah"):** two plugin roots: the + app's built-in `plugins/` (required: a plugin failing at startup stops the app with its error) and the user's in the + data directory (failures reported). Installs from git or npm into the user's root. One "apply changes" call builds the + dirty plugins and rolls them out to the server, then every running worker; the app exposes it as a button, a + command or an agent tool; changes are never applied by themselves. **Files:** the kit ships a files plugin, one owner + per address (`self.get(Files, { scope })`: `put`, chunked transfers for large files (design §23), a watchable + index, caps), its files in that worker's directory; apps build on it (p3's artifacts: versions, previews, sharing; + design §26 #17). Raw directories only where a real path is needed (a git checkout, a library writing its own + database), each owned by one object, under the worker's directory. **Export and import per address:** the store's + archive plus the worker's directory as one archive; importing it under another address copies or moves it. + Refines point 4 (storage): plugins use the files plugin instead of raw subdirectories of their own. - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From cdd50052f61d735ca6cd83b87f7173204dc86fba Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 10:55:18 +0000 Subject: [PATCH 257/271] docs(chord): slice 11 base bundle and plugin scripts --- packages/chord/docs/next/decisions.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 5494adbd9..853b2ab5b 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2272,6 +2272,13 @@ validation and copies the v2 kernel added around Delta. database), each owned by one object, under the worker's directory. **Export and import per address:** the store's archive plus the worker's directory as one archive; importing it under another address copies or moves it. Refines point 4 (storage): plugins use the files plugin instead of raw subdirectories of their own. +- **Slice 11, P3: the base bundle and plugin scripts (owner, 2026-11-01, by voice: "yep, that sounds good"):** the + server builds one base bundle per client kind from an entry in the app's source (the app's web start code, its API + with the kind's module, Chord, React), content-hashed and cached by browsers forever; a change of base code reloads + the page (as a restart of the server). Plugin scripts: at page load the server serves the current set of the kind's + plugins as one combined file named by a hash of the set (one request, cacheable); while a page is open, a rollout's + changed plugins are fetched one script each. A fresh page makes two requests, an update downloads only what changed. + Rejected: a prebuilt base bundle shipped with the app (could disagree with the plugins' build). - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From 7f7ec86806764ef1d77d3509696060f57391d980 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 10:56:16 +0000 Subject: [PATCH 258/271] docs(chord): slice 11 applies plugin scripts in place --- packages/chord/docs/next/decisions.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 853b2ab5b..440329fee 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2279,6 +2279,11 @@ validation and copies the v2 kernel added around Delta. plugins as one combined file named by a hash of the set (one request, cacheable); while a page is open, a rollout's changed plugins are fetched one script each. A fresh page makes two requests, an update downloads only what changed. Rejected: a prebuilt base bundle shipped with the app (could disagree with the plugins' build). +- **Slice 11, P4: new plugin scripts apply in place (owner, 2026-11-01: "yep, agree"):** a page's host reloads the + changed plugins per plugin (`host.reload`, as on the server): objects swap or restart, the UI re-renders from its + registries, keeping what is on screen (scroll, a half-typed message). A change of base code still reloads the whole + page. Rejected: reloading the page for every plugin change (p2's behaviour, which lost the screen on every agent + edit). - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From 9942d5bbcfeec633da80f69c57747a3cf48da9a5 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 10:57:16 +0000 Subject: [PATCH 259/271] docs(chord): slice 11 pages learn of new scripts through Chord --- packages/chord/docs/next/decisions.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 440329fee..07af2737d 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2284,6 +2284,11 @@ validation and copies the v2 kernel added around Delta. registries, keeping what is on screen (scroll, a half-typed message). A change of base code still reloads the whole page. Rejected: reloading the page for every plugin change (p2's behaviour, which lost the screen on every agent edit). +- **Slice 11, P5: pages learn of new scripts through Chord (owner, 2026-11-01: "agree"):** the web component serves a + small object whose state is the current set of the kind's plugin scripts (names, digests, URLs); every page holds + it; a rollout publishes the new set without waiting for pages; each page gets one frame and fetches what changed; + after a reconnect the hold's snapshot brings the current set, so a page catches up on anything it missed. Rejected: + polling a manifest (p2: every 5 s while visible). - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From 2f153034b45328592bcb35d0d38b4344ad6e468d Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 10:58:40 +0000 Subject: [PATCH 260/271] docs(chord): slice 11 connection loop --- packages/chord/docs/next/decisions.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 07af2737d..962e5cbd1 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2289,6 +2289,11 @@ validation and copies the v2 kernel added around Delta. it; a rollout publishes the new set without waiting for pages; each page gets one frame and fetches what changed; after a reconnect the hold's snapshot brings the current set, so a page catches up on anything it missed. Rejected: polling a manifest (p2: every 5 s while visible). +- **Slice 11, P6: the connection loop (owner, 2026-11-01: "agree"):** p2's `keepConnected`: after a drop, reconnect + with a growing pause (250 ms to 5 s); try at once when the tab becomes visible, the browser comes back online or the + network changes; replace a socket open but silent; stop when the server reports an outdated protocol (the page + reloads) or the app's `canRetry` says no (a guest's share ended). References keep their last values and wait while + down, so the UI shows a banner from the connection state. A network reconnect, not a crash restart. - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From 20f600e58d416dd8f0b9ee07238d11b7890b2c06 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 10:59:40 +0000 Subject: [PATCH 261/271] docs(chord): slice 11 React bindings --- packages/chord/docs/next/decisions.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 962e5cbd1..6b74038e7 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2294,6 +2294,11 @@ validation and copies the v2 kernel added around Delta. network changes; replace a socket open but silent; stop when the server reports an outdated protocol (the page reloads) or the app's `canRetry` says no (a guest's share ended). References keep their last values and wait while down, so the UI shows a banner from the connection state. A network reconnect, not a crash restart. +- **Slice 11, P7: React bindings (owner, 2026-11-01: "we definitely want something like that"; p2's `useObject`, + `useReadable`, `useRegistry` in `runtime/web.tsx`):** an optional subpath (`chord/next/react`): `HostProvider`, + `useHold(Type, params)`, `useWatch(watchable)`, `useRegistry(R)`, `useRefState(ref)`; releasing holds on unmount, + and a component unmounted by a reload never counts as leaving (design §21). No components, no styling; non-React apps + never load it. - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From e8504dfab5fb3aa023322a7e89b43c0eb49e6120 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 11:00:43 +0000 Subject: [PATCH 262/271] docs(chord): slice 11 terminal client always sandboxed --- packages/chord/docs/next/decisions.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 6b74038e7..108821ff2 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2299,6 +2299,11 @@ validation and copies the v2 kernel added around Delta. `useHold(Type, params)`, `useWatch(watchable)`, `useRegistry(R)`, `useRefState(ref)`; releasing holds on unmount, and a component unmounted by a reload never counts as leaving (design §21). No components, no styling; non-React apps never load it. +- **Slice 11, P8: the terminal client always runs sandboxed (owner, 2026-11-01: "the terminal should always use the + sandbox approach"):** `chord/next/tui` is a Node client host that connects with a token (decision "The app kit, point + 3", (c)), downloads the base bundle and the TUI plugin scripts from the server and runs the app's TUI entry, always + inside p2's sandbox: a permission-dropped worker thread with a bridge terminal, the trusted launcher outside it + holding the credentials, the terminal and local preferences. No unsandboxed mode, also for the owner's own server. - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From 752019da075163d72eb447cd9fe9b27d1fd072c7 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 11:03:04 +0000 Subject: [PATCH 263/271] docs(chord): slice 11 terminal component scope --- packages/chord/docs/next/decisions.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index 108821ff2..8eb76029e 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -2304,6 +2304,11 @@ validation and copies the v2 kernel added around Delta. 3", (c)), downloads the base bundle and the TUI plugin scripts from the server and runs the app's TUI entry, always inside p2's sandbox: a permission-dropped worker thread with a bridge terminal, the trusted launcher outside it holding the credentials, the terminal and local preferences. No unsandboxed mode, also for the owner's own server. +- **Slice 11, P9: a terminal component scope, library-agnostic (owner, 2026-11-01: "yeah, sure"; p2's `TuiScope` + in `src/runtime/tui.ts`):** the counterpart of the React bindings: a scope tied to a component's life with `hold`, + `watch` and `registry`, releasing everything on dispose, calling one `changed` callback the app wires to its redraw + (one line with pi-tui; any library works). p2's retry with backoff for gone references is not carried over (v2 + references wait and rebind by themselves). No pi-tui dependency. - **A plugin's npm dependencies are bundled; external only by opt-in (owner, 2026-10-29: "explicit opt in + (a)"):** a plugin's `dependencies` are installed into its own `node_modules` (`npm install --ignore-scripts`) and bundled into each facet that imports them (host packages, its `peerDependencies`, stay external and are passed in). On Node a From ebce9debd4206e5a0c176ca6bed28fcb40d0edc1 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 11:13:08 +0000 Subject: [PATCH 264/271] docs(chord): fold the plugin model, app kit and slice 11 decisions into the design; slice 12 spec --- packages/chord/docs/next/design.md | 72 +++++++++++++++ packages/chord/docs/next/slice-12.md | 131 +++++++++++++++++++++++++++ packages/chord/docs/next/slices.md | 8 +- 3 files changed, 208 insertions(+), 3 deletions(-) create mode 100644 packages/chord/docs/next/slice-12.md diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index c11cb89e4..413674592 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -1980,6 +1980,39 @@ type ReloadReport = { - Project plugins, trust, snapshots, scopes and npm/git package installs: `apps/p2/docs/plugin-loading.md` (still the reference for those; not in 7c). +### The plugin model (2026-11-01; slice 12) + +Decisions "Contracts come from where the plugin's code came from", "Plugins import only the app's versioned API", +"The plugin model, points 3, 4, 5 and 8, imports and builds", "Single-file plugins", "Testing a plugin alone". The +rule underneath: a plugin carries what it was written against, never the running app's newest file. + +- **What a plugin may import:** its own files; the app's versioned API (`@acme/app/v3`, never Chord directly: Chord is + not a host package plugins may name; the API re-exports the Chord surface the app supports); its npm `dependencies` + (bundled at the installed version); other plugins' contract files: by relative path within one repository + (`../notes/contract.ts`, the same commit holds both sides), by package name across repositories (`@bob/notes`, + `@bob/notes/web`, pinned by an npm version or a git ref `github:bob/notes#`; a subfolder of a multi-plugin + repository is cloned and linked by the plugins component). One facet imports one API version only. +- **Versions:** the app lists the API versions it provides; an unlisted one is a build error naming the available ones. + The process runs one Chord: an older API version is a layer over today's, re-exporting Chord's compatibility module + for its previous API while Chord keeps one; contracts bind through `accepts` whatever Chord runs. +- **Split by host kind:** the API has a shared module and per-kind modules (`/v3/web`, `/v3/tui`, `/v3/server`, + `/v3/worker`); a plugin's contract files mirror it (`contract.ts`, `contract.web.tsx`, `contract.tui.ts`, ...). A + contract file mentions native things only through `import type`. The build checks every import by the file it + resolves to: a per-kind file or module only for its kind, a package the kind does not provide refused. A token or + `local` contract used on one kind lives in that kind's file. Shared UI between plugins: an imported file (inlined, + frozen at the importer's build) or a registry entry (one live instance). +- **Base code** is the API modules, bootstrap (hosts, store, components such as `createHttp` and `createWorkers`, which + take settings and process values and return facets) and one start-data facet serving process values as core + objects. Every implementation, the harness and core registry providers included, is a plugin; the app's own root is + required (a failure at startup stops the app). +- **Single-file plugins:** `..ts(x)` at a root's top, grouped by name, no `package.json`, importing only the + API; a directory of the same name replaces it as the same plugin. +- **Installs** from git (a branch, tag or commit cloned into a root) or npm. **Builds** are one esbuild call per facet + from source (TS, TSX, JS, JSON, CSS, assets and WASM by URL); plugins bring no build hooks. +- **`testPlugin(dir, { kind, modules, stubs, start?, principal? }, ctx)`:** the plugin built as in production on an + in-memory host and store, its dependencies served by stub facets, contributions to registries collected for the + test, the contracts nobody serves listed. + - **UI reload vs. leaving:** "leaving" is never derived from an unmount, only from explicit navigation or presence (a dropped reference); a component unmounted by a reload has not left. @@ -2236,6 +2269,45 @@ fails closed when stale; a start waits for a draining old holder without polling complete sets (`createPluginRuntime`, no esbuild); `workers.step(name)` ships a rollout's set to every worker (D12: `{ host, report } | { host, stopped: true } | { host, error }` per worker). +### The app kit (`chord/next/app`; slice 13) + +Decisions "The app kit, point 1" to "point 5". `startApp(options)` on the server, `startWorker(options)` in a worker +entry. Opinionated and small: an app is its API, its built-in plugins and a start script. + +- **Shape:** the app's layout is fixed: `src/api/v/` (shared and per-kind modules) and the built-in plugins root + `plugins/`. The data directory is a `startApp` option (no environment variables), default a dot directory named after + the app in the home directory. One server; worker kinds are a list the app defines, each a host kind with its own + params (sharding is the app's) and how it starts (entry, start data, principal, idle time); zero kinds is a server + with clients only. +- **Principals and users:** the principal is tiny (a kind and an id) and the app's type; the kit needs only its server + and worker variants. User data lives in objects keyed by the user's id. Optional kit plugins: users (accounts with an + app-supplied profile schema), authenticators (device pairing, invite links, an ephemeral pick-a-name login). +- **Authentication:** no owner by loopback alone: the start script prints a link with a one-time token that sets a + signed owner cookie; deny by default (the HTTP component refuses requests without a principal unless the route or + socket is public); a session helper issues and revokes sessions; command-line clients send a token in a header (an + owner token file readable only by the app's OS user; a device pairing flow from other machines). +- **Storage:** one store for the server, a directory per worker under its kind and address with its store; plugins + never use data-directory paths. A files plugin per address (`put`, chunked transfers, a watchable index, caps) is + the default file storage; raw directories only where a real path is needed, owned by one object, under the worker's + directory. Export and import per address: the store's archive plus the worker's directory. +- **Plugins:** the app's root (required) and the user's root in the data directory; installs from git or npm; one + "apply changes" call builds and rolls out to the server, then every running worker, exposed by the app as it likes. + +### Web and TUI (slice 11) + +Decision "Slice 11's points" and "Slice 11, P3" to "P9". + +- `chord/next/web`: serving the surface, a page's client host with its connection loop (p2's `keepConnected`), loading + and applying the plugins' web facets. A base bundle per client kind built on the server from the app's entry + (content-hashed); at page load the kind's current plugin set as one combined file named by its hash, later changed + plugins one script each, each a script that registers itself (no eval); new scripts apply in place (`host.reload` in + the page); pages learn of them through an object the web component serves (the current set); a base code change + reloads the page. +- `chord/next/react`: `HostProvider`, `useHold`, `useWatch`, `useRegistry`, `useRefState`. +- `chord/next/tui`: a Node client connecting with a token, always running the downloaded code in a sandbox (a + permission-dropped worker thread, a bridge terminal, the trusted launcher outside); a component scope (`hold`, + `watch`, `registry`, released on dispose, one `changed` callback for the app's redraw), no pi-tui dependency. + ### Channels WebSocket byte channels with acknowledged windows, separate from object links, wait for a second user after the diff --git a/packages/chord/docs/next/slice-12.md b/packages/chord/docs/next/slice-12.md new file mode 100644 index 000000000..4bfd790e7 --- /dev/null +++ b/packages/chord/docs/next/slice-12.md @@ -0,0 +1,131 @@ +# Slice 12: the plugin model (spec) + +Status: **approved** (owner, 2026-11-01: the plugin model's decisions, then "go ahead, just do all the things"; this +spec written by the redesign session from them, no separate spec review). Builds on 7c's plugins component +(`src/next-node/plugins/`) and slice 10's kinds and `createPluginRuntime`. + +Read in full before writing code: this spec; `decisions.md` entries of 2026-11-01 "Contracts come from where the +plugin's code came from", "Plugins import only the app's versioned API", "The plugin model, points 3, 4, 5 and 8, +imports and builds", "Single-file plugins", "Testing a plugin alone"; `design.md` §21 (all of it, "The plugin model" +included) and §16-§17; `slices.md` (carry-over "Slice 12"); `slice-7c.md`, `slice-10.md` §3 "Plugins in workers"; +`src/next-node/plugins/README.md` and its sources; the primer `docs/next/plugin-primer/book.md`. + +## 1. The app's API (`createPlugins` option `api`) + +```ts +const plugins = createPlugins({ + roots: [{ path: "./plugins", required: true }, { path: join(dataDir, "plugins") }], + api: { name: "@acme/app", versions: ["v3", "v2"] }, // newest first + kinds: { + server: { platform: "node", hostPackages: ["@acme/app/v3", "@acme/app/v3/server", "@acme/app/v2"] }, + worker: { platform: "node", hostPackages: ["@acme/app/v3", "@acme/app/v3/worker", "@acme/app/v2"] }, + web: { platform: "browser", hostPackages: ["@acme/app/v3", "@acme/app/v3/web", "@acme/app/v2", "react", "react/jsx-runtime"] }, + }, + onError, +}, ctx); +``` + +- `api` names the app's API package and the versions it provides. A plugin import of `/[/]`: + a version not listed is a build error naming the listed ones ("@acme/app/v1 is not available: this app provides v3, + v2"); a listed one that the kind's `hostPackages` lacks (`@acme/app/v3/web` from a worker facet) is a build error + naming the kind. One facet's imports of the API all use one version (a mix is a build error naming both). +- Chord itself (`@earendil-works/chord/next` and its subpaths other than `/plugins`, which stays as in 7c for + `pluginFile`) is not importable by plugins: a build error pointing to the app's API. Apps that do not set `api` keep + 7c's behaviour (host packages as listed), so Chord's own tests and examples keep importing Chord directly where they + model base code. +- `roots` entries may be strings (as today) or `{ path, required? }`. A plugin of a required root whose build, + evaluation or setup fails at `plugins.load` makes `load` reject (`PluginError` naming it), after the host's own load + settled; failures in other roots are reported as today. At a reload, a required plugin failing keeps its old code as + any plugin (reported). + +## 2. Imports between plugins (replaces slice 10's by-name lookup) + +- **Relative paths within a root:** an import from one plugin into another plugin's directory under the same root + (`../notes/contract.ts`) is inlined (listed in `copied`), the copied file under 7c's rule (only host packages and + files of its own plugin). A relative path leaving the root (base code, another root) is a build error naming the file + and pointing to the API or a dependency. A git checkout of several plugins cloned into a root is "within a root". +- **Package names across repositories:** an import of a package in the plugin's `dependencies` resolves as npm does + through that package's `exports` (`@bob/notes` → its `contract.ts`, `@bob/notes/web` → `contract.web.tsx`), inlined + at the installed version. A dependency may be an npm version or a git URL with a ref (`github:bob/notes#`, + `git+https://...#`); `installDependencies` installs it with npm as today. A package from a subfolder of a git + repository (which npm cannot install) is declared as `"chord": { "plugins": { "@bob/notes": "github:bob/plugins#:notes" } }` + and installed by `installDependencies` itself: a shallow clone at the ref into the plugin's `node_modules/.chord-git/` + and a link `node_modules/@bob/notes` to the subfolder. +- The by-name lookup (`build.ts` `#byName`), its error texts and the six `tsconfig.json` path aliases go; fixtures and + examples 35-38 import relatively again. + +## 3. Per-kind files and the import check + +- A file whose name has a kind segment before its extension (`contract.web.tsx`, `ui.tui.ts`, any `*..` with + `` one of `kinds`) is a file of that kind: importing it (directly or through other files) from a facet of + another kind is a build error naming the chain. Facet files keep their names (`web.tsx`); contract files are + `contract.ts` (shared) and `contract..ts(x)`. +- The check runs in the existing esbuild `onResolve` path, on the resolved file, for relative imports, dependencies and + copied files alike. +- Documented, not checked: a contract file mentions native things only through `import type` (TypeScript erases it; + a runtime import of a package the kind lacks already fails as above). + +## 4. Single-file plugins + +- Discovery also lists files at a root's top named `..(ts|tsx|js|mjs)`; files of one `` form one + plugin named `` (an unscoped valid npm name, else reported), one facet per file, `dir` the root, no + `package.json` (no dependencies, no `chord.external`). A directory plugin of the same name in the same root is a + reported duplicate (the directory wins). +- Its build allows imports of host packages only (the app's API); anything else is a build error saying to make it a + directory. `pluginFile` refuses (it has no directory of its own). +- The watcher: such a file appearing or disappearing at a root's top sets `set`; a change marks the plugin dirty. + +## 5. Installing plugins + +`installPlugin(source, root, ctx)`: `source` is `{ git: url, ref }` (a shallow clone of the ref into +`/`; a repository whose top has a `package.json` is one plugin, otherwise its subdirectories +are plugins) or `{ npm: name, version }` (`npm pack` and extract into `/`), then `installDependencies` for +each new plugin; marks `set`. Returns the installed plugin names. Removal is deleting the directory (`set`). No +updating in place: install another ref (a new directory) or delete and install. + +## 6. `testPlugin` + +```ts +const t = await testPlugin("./plugins/stats", { + kind: "worker", + api: { name: "@acme/app", versions: ["v3"] }, + modules: { "@acme/app/v3": apiV3, "@acme/app/v3/worker": apiV3Worker }, // the real API modules + stubs: [{ name: "@acme/notes", facet: notesStub }], // facets serving what it uses + principal: { kind: "user", id: "u1" }, // the host's principal (optional) +}, ctx); +const stats = await t.host.get(Stats, { project: "p1" }, ctx); +t.contributions(Panels); // what the plugin contributed to a registry nobody serves (the helper serves it) +t.unserved(); // contract names requests wait for unbound `not_served` +await t.close(ctx); +``` + +- Built with the same build as production (`buildFacet` with the kind's options), evaluated with `modules`, loaded on a + host with `address`/`principal` as given, an in-memory store (`:memory:` SQLite); stubs loaded beside it. +- The helper serves, as a collecting registry, every registry the plugin contributes to that no stub or the plugin + itself serves (`t.contributions(R)`: the current entries). +- `t.unserved()` lists the names of contracts whose requests wait unbound `not_served` on the host (the unbound + index), so a test knows what to stub. `t.root(fn, ctx, { principal })` runs as another principal. +- Exported from `chord/next/plugins` (`testing.ts`); no test-only modes elsewhere. + +## 7. Docs for authors and agents + +- `docs/next/plugin-authoring.md`: a plugin's layout (directory and single-file), its facet and contract files per + kind, what it may import and how each import resolves, versions, how the server builds it (esbuild, what the build + handles, no build hooks), installing from git or npm, testing with `testPlugin`; written for agents (short, exact, + with one complete example plugin). +- Update `src/next-node/plugins/README.md`, the guide `docs/next/guide-plugins-hosts/book.md` (chapters "Plugins Using + Plugins" and "Base Code and Plugins"; rebuild `index.html`), design §21's slice-10 note. + +## 8. Tests, examples + +- One test file per source file; build tests for every refusal and allowance of §1-§4 (an unlisted version, a kind + module from another kind, mixed versions, a Chord import, a relative path leaving the root, a per-kind file through + a chain, a package from `dependencies` through `exports`, a git dependency with a ref (a local bare repository in a + temp directory, no network), a subfolder dependency); discovery of single-file plugins; required roots at load; + `installPlugin` from a local git repository and a local npm tarball; `testPlugin` (stubs, collected contributions, + `unserved`, a principal). +- Examples: `49-plugin-api` (an app API with two versions and per-kind modules, a plugin per version, the refusals), + `50-plugin-imports` (relative within a root, a git dependency pinned to a commit, per-kind contract files), + `51-single-file-plugin`, `52-test-plugin` (a plugin tested with a stub). Examples 35-38 updated. +- No hot path changes; no benchmark beyond a cold build of the 20-plugin set (as 7c's) to show the checks cost nothing + measurable. diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index 766d32211..edd891029 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -45,7 +45,7 @@ slice spec's section citations are for orientation, not a reading limit (owner, Order (owner, 2026-10-22: routines and jobs before the store and reload; 2026-10-27: slice 8 before slice 7, whose reload has to reload real registries; 2026-10-29: slice 7 split into 7a, 7b, 7c; the app kit as slices 10 and 11; -telemetry last, since it instruments the app kit too). +telemetry last, since it instruments the app kit too; 2026-11-01: the plugin model (12) and the app kit (13) before web and TUI (11), which builds into the kit). | # | Slice | In short | | --- | --- | --- | @@ -63,6 +63,8 @@ telemetry last, since it instruments the app kit too). | 7b | **Reload in the kernel** | `host.reload`: staging, commit, rollback; swaps, restarts, removal and re-serving; holds and `accepts` across a reload; jobs and registries across a reload; per-name wakes (`slice-7b/` brief) | | 7c | **Plugins** | `chord/next/plugins` (discovery, `buildFacet`, `evaluateFacet`, `pluginFile`, `installDependencies`, the watcher, `rollout`, `mapStack`; no dev/prod split; the kernel's `load`/`reload` per plugin; `slice-7c.md`) | | 10 | **HTTP and workers** | `chord/next/http` (server, `Http` registry, responses as descriptions, WebSocket entry) and `chord/next/workers` (a worker per address, IPC, idle stop) | +| 12 | **The plugin model** | imports (relative within a repository, package names pinned across), versioned app API with Chord re-exported, per-kind API modules and contract files with the build's import check, single-file plugins, git installs, `testPlugin`, base code as API, bootstrap and a start-data facet (`slice-12.md`) | +| 13 | **The app kit** | `chord/next/app`: `startApp`, `startWorker`, layout, worker kinds, owner link, deny by default, sessions, CLI tokens, storage, the files plugin, export per address, plugin roots and "apply changes" (`slice-13.md`) | | 11 | **Web and TUI** | `chord/next/web` (page host, React hooks) and `chord/next/tui` (launcher, sandbox), from p2's code | | 9 | **Telemetry** | spans, events, metrics across hosts | @@ -181,12 +183,12 @@ deleted (the slice's spec and code are the record). Moved here from `carry-over. - **`rebuild` of tables** (key and column-type changes): `evolveTable` has no edit for them; build one only when a real case needs it. (Slice 6a's second spec review; out of slice 7b's scope, `slice-7b.md` §12.) -### The app kit +### Slice 13: the app kit - **Deny by default in the HTTP component** (decision "The app kit, point 3: authentication"): routes and sockets refuse requests without a principal unless marked public; the owner link with a one-time token; a session helper. -### The plugin slice (plugin model) +### Slice 12: the plugin model - **Cross-plugin imports by relative path and npm** (decision "Contracts come from where the plugin's code came from"): remove `build.ts`'s by-name lookup, allow relative imports into another plugin of the same root, refuse From 8d81a0f49db021e8f183a7a953d862ccb8681f12 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 11:14:37 +0000 Subject: [PATCH 265/271] docs(chord): slice 13 spec, the app kit --- packages/chord/docs/next/slice-13.md | 120 +++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 packages/chord/docs/next/slice-13.md diff --git a/packages/chord/docs/next/slice-13.md b/packages/chord/docs/next/slice-13.md new file mode 100644 index 000000000..f5b8fb029 --- /dev/null +++ b/packages/chord/docs/next/slice-13.md @@ -0,0 +1,120 @@ +# Slice 13: the app kit (spec) + +Status: **approved** (owner, 2026-11-01: decisions "The app kit, point 1" to "point 5", then "go ahead, just do all +the things"). Builds on slices 10 (`http`, `workers`) and 12 (the plugin model). `chord/next/app` in +`src/next-node/app/`: an opinionated layer so an app is its API, its built-in plugins and a start script. + +Read in full before writing code: this spec; `decisions.md` entries of 2026-10-31 and 2026-11-01 (the app kit, the +plugin model, "The app kit is opinionated", "Settings are arguments, extensions are registries", "The HTTP +component's security baseline"); `design.md` §3, §4, §10, §21, §23 (all of it, "The app kit" included); `slices.md` +(carry-over "Slice 13"); `slice-10.md`, `slice-12.md`; the READMEs of `next-node/http`, `workers`, `plugins`; p2's +`src/runtime/server.ts`, `src/hosts/server.ts`, `src/hosts/worker.ts`, `src/runtime/loopback-cookie.ts`, +`src/hosts/principals.ts`, `plugins/devices/`, `plugins/sharing/`, `plugins/files/` for what exists. + +## 1. `startApp` and `startWorker` + +```ts +// start.ts (the app's start script) +const app = await startApp

    ({ + name: "acme", // the default data directory: ~/.acme + dataDir: process.argv[2], // optional; no environment variables + listen: { host: "127.0.0.1", port: 8788 }, + api: { name: "@acme/app", versions: ["v3"], modules: { "@acme/app/v3": apiV3, "@acme/app/v3/server": apiV3Server } }, + kinds: { server: { platform: "node" }, worker: { platform: "node" }, web: { platform: "browser", packages: { react, "react/jsx-runtime": jsx } } }, + principal: { server: { kind: "server" }, worker: (address) => ({ kind: "worker", key: address.key }) }, + workers: [{ + kind: Project, // a host kind the app defines, any params + entry: "./worker.ts", + start: (address, ctx) => ({ id: address.params.id }), + authorize: (principal, address) => mayReach(principal, address), + idleMs: () => 5 * 60_000, + }], + onError: (error) => log(error), +}, ctx); +// app.host, app.http, app.workers, app.plugins, app.applyChanges(ctx), app.exportAddress(address, ctx), +// app.importAddress(archive, address, ctx), app.ownerLink(), app.close(ctx) +``` + +```ts +// worker.ts +await startWorker

    ({ api, kinds, onError }); // reads the start message, opens the worker's store, loads plugins +``` + +- `startApp` creates the data directory layout (§4), opens the server store, creates the host (address `Server()`, + the server principal), `createHttp` (settings as arguments), `createWorkers` per worker kind (`kind` named after the + host kind; the worker's facet kind is `worker` unless the entry says otherwise), `createPlugins` with the roots of §5 + and the app's API (slice 12), loads the server's plugins with the kit's own facets (§2, §3, §4) as core, and resolves + once loaded. `close` closes in the right order (http, workers, host, store). +- `startWorker` is `workerMain` plus: the worker's directory and store under its kind and address, the host with the + principal from the start message, `connect` first, plugins loaded with the kit's worker facets, `busy` from the app + if given. +- The start-data facet: the kit serves `AppInfo` (data directory, app name, version) on the server and `WorkerInfo` + (address, start data) on workers as core objects plugins `get`. +- Zero worker kinds: no workers component. + +## 2. Principals and sessions + +- The app's principal type `P`; the kit needs only `principal.server` and `principal.worker(address)`. The kit adds + one variant for its owner: `{ kind: "owner" }` must be assignable to `P` (a type check). +- **Sessions** (`app/sessions.ts`, a kit object on the server, persisted in the server store): `issue(principal, + { label, kind: "browser" | "device" }, ctx)` returns a token; tokens stored hashed; `revoke(id)`; `list` (watchable: + label, kind, created, last used). The kit's authenticator (contributed to `Authenticators`) reads the session cookie + (signed, `HttpOnly`, `SameSite=Strict`) or an `Authorization: Bearer` header, and gives its principal. +- **Owner link:** at start the kit issues a one-time token (random, single use, valid 10 minutes, replaced at each + start) and `app.ownerLink()` returns `http://127.0.0.1:8788/_chord/owner#`; the start script prints it. The + route `GET /_chord/owner` serves a page that posts the token (from the fragment, never sent in a URL to logs) to + `POST /_chord/owner`, which consumes it and sets a browser session cookie for the owner. No owner by loopback alone. +- **Owner token file:** `/owner-token` (mode 0600, created at first start, a device session of the owner) for + command-line clients on the same machine. +- **Device pairing** (for remote command-line and TUI clients): `POST /_chord/pair` (public) returns `{ code, poll }`; + the owner approves the code in a browser (`GET /_chord/pair/`, owner only, a confirm page); the client polls + and receives a device token. Codes expire after 10 minutes. +- **Deny by default** (an HTTP component change): a route or socket without a principal is refused (401) before plugin + code, unless its contribution marks it `public: true` (routes: `{ handler, public: true }`; sockets likewise). The + `/chord` links endpoint requires a principal. Foreign requests stay anonymous as before, so they reach only public + routes. Slice 10's examples and tests mark their anonymous routes public. + +## 3. Optional kit plugins + +Shipped as plugins (in `packages/chord/plugins/` or exported facets the app adds to its root; choose one and say +why): `users` (accounts keyed by id, a profile schema the app passes in, `self.get(Users, {})` with `get`, `update`, +`list`), `invite` (invite links issuing a session for a principal the owner chooses), `ephemeral-login` (a public +route taking a name, issuing a session for `{ kind: "user", id: , name }` that ends with the connection; the +app's `P` must allow it). Each contributes to `Authenticators` or issues sessions through §2. + +## 4. Storage, files, export + +- Layout: `/server/store.sqlite`, `/workers///store.sqlite` plus that + worker's files, `/plugins/` (the user's root), `/logs/`, `/owner-token`. +- **Files** (a kit facet on workers, and on the server for server-scoped files): `Files` (`local`, params `{}`, one per + host) with `put({ name, mime, bytes })` for small files, `begin`/`append`/`finish` for large ones (design §23 chunked + transfers), `read({ id, offset, length })`, `remove`, `list` (watchable index: id, name, mime, size, created), caps + per host (an option). Stored under the host's directory (`files/`), the index in the store. +- **Export / import per address:** `app.exportAddress(address, ctx)` stops the worker (as `workers.stop`), writes one + archive: the store's export archive (slice 6a) plus the worker's directory (files), and returns a readable stream or + writes to a path; `importAddress(archive, address, ctx)` refuses an existing address, unpacks, and the next hold + starts the worker. Format: a tar of `store.archive` and `files/`, with a small manifest (app name, API version, + kit format version). + +## 5. Plugins + +- Roots: the app's `plugins/` (required, slice 12) and `/plugins/`. Installs (`app.plugins.install(source, + ctx)`, slice 12's `installPlugin`) go to the user's root. +- `app.applyChanges(ctx)`: build the dirty plugins and roll out (server step, then the workers step); returns the + rollout result; `app.plugins.state` for the app's button. Never applied by itself. + +## 6. Not in slice 13 + +Web and TUI clients (slice 11, which builds on this kit: `startApp` gains the web surface there); telemetry (9); +project plugins (§26 #14). + +## 7. Tests, examples, docs + +- One test file per source file: start and close order; a server without workers; two worker kinds; the owner link + (single use, expiry, the fragment), sessions (issue, revoke, list, hashed storage), the token file mode, device + pairing, deny by default (a route without principal 401, a public route served, `/chord` refused anonymous), Files + (small and chunked, caps, list, remove), export and import (a worker with documents, tables and files round-trips; + importing over an existing address refused), `applyChanges`, required root failure stopping `startApp`. +- Examples: `53-app` (a small app: start script, API, two plugins, a worker kind, the owner link), `54-app-sessions` + (pairing a command-line client), `55-app-export` (export and import a worker's address). +- `src/next-node/app/README.md`; the guide "Hosts and Plugins" gets a closing chapter on the app kit. From 7b991fd96aee530b2ecfb5327f6c82d197f36395 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 11:15:21 +0000 Subject: [PATCH 266/271] docs(chord): slice 11 spec, web and TUI on the app kit --- packages/chord/docs/next/slice-11.md | 150 ++++++++++++++------------- 1 file changed, 78 insertions(+), 72 deletions(-) diff --git a/packages/chord/docs/next/slice-11.md b/packages/chord/docs/next/slice-11.md index c7b752596..e13899000 100644 --- a/packages/chord/docs/next/slice-11.md +++ b/packages/chord/docs/next/slice-11.md @@ -1,93 +1,99 @@ # Slice 11: web and TUI surfaces (spec) -Status: **draft** (owner, 2026-10-30: "start on slice 11"; "this one needs to be generic, so apps can do whatever they -want: css, tui components, themes"). Points marked **Decision needed** go to the owner one at a time. +Status: **approved** (owner, 2026-10-30/11-01: P1-P9 in the decisions "Slice 11's points" and "Slice 11, P3" to "P9", +then "go ahead, just do all the things"). Built after slices 12 and 13: the surfaces are generic plumbing that the app +kit wires in (`startApp` serves the web surface; a TUI client entry). The app keeps everything visible: HTML shell, +CSS, components, themes, navigation, layout, keys. -Slice 11 builds the generic plumbing of the app kit's two client surfaces (decision "Slice 7b's open points" (6)): -`chord/next/web` (a browser page as a Chord client host that loads the plugins' web facets the server built) and -`chord/next/tui` (a terminal client that does the same for TUI facets). The app keeps everything visible: its HTML -shell, CSS, components, themes, navigation, layout, keys. Built on slice 10 (`http` serves, the plugins component -builds and ships) and 7c (scripts as plain data, `createPluginRuntime`). Rule "The app kit is malleable by plugins" -applies. +Read in full before writing code: this spec; `decisions.md` (slice 11's entries, the plugin model, the app kit); +`design.md` §3 (clients), §21, §23 (all, "Web and TUI" included); `slice-10.md`, `slice-12.md`, `slice-13.md`; p2's +`src/hosts/web.ts`, `src/hosts/web/main.tsx`, `src/web/reconnect.ts`, `src/runtime/web.tsx`, `src/hosts/bundles.ts`, +`src/hosts/tui.ts`, `src/hosts/tui-client.ts`, `src/hosts/tui-sandbox.ts`, `src/hosts/tui-bootstrap.mjs`, +`src/hosts/tui-worker-runtime.ts`, `src/runtime/tui.ts` (`TuiScope`). -Read in full before writing code: this spec; `decisions.md` ("Slice 7c's points", "Slice 10's points"); `design.md` -§3 (clients), §21, §23; `slice-10.md`; p2's `src/hosts/web.ts`, `src/hosts/web/main.tsx`, `src/web/reconnect.ts`, -`src/runtime/web.tsx`, `src/hosts/bundles.ts`, `src/hosts/tui.ts`, `src/hosts/tui-client.ts`, `src/hosts/tui-sandbox.ts`, -`src/hosts/tui-bootstrap.mjs`, `src/hosts/tui-worker-runtime.ts`. - -## 1. What p2 has, and what is generic - -| p2 today | Generic (slice 11) | The app's | -| --- | --- | --- | -| one esbuild bundle per surface: runtime, React, every plugin's web facet, CSS; the page reloads whole when its version changes | a base bundle per surface (host packages and the surface runtime) plus each plugin's facet script (7c); new scripts applied in place | which host packages a surface has | -| the page: HTML with a CSP nonce, a theme script, `#root`, the bundle | serving the base bundle, scripts, stylesheets and assets through `http`; a page template hook | the HTML shell, CSP, theme, manifest and icons, service worker | -| `startWeb`: a client host, `keepConnected` (reconnect with backoff, wake on visibility and network change, stale-socket checks), protocol mismatch reloads the page | `startPage`: the client host, the connection loop, facet loading and reloading | `canRetry` policy (an ended share), connection UI | -| `runtime/web.tsx`: `HostContext`, `useObject`, `useReadable`, `useRegistry`, slots, sheets, navigation, owner state | optional React bindings: provider, hold, watch, registry, reference state | slots, sheets, navigation, everything else | -| TUI: a trusted launcher (auth, terminal, local preferences, actions), a permission-dropped worker thread running the downloaded bundle, a bridge terminal, a link through the launcher | a Node client surface: connect, download the base bundle and TUI scripts, evaluate, reload in place; the sandbox as an option | the TUI's components (pi-tui or other), commands, keys, themes, launcher actions | - -## 2. `chord/next/web` +## 1. `chord/next/web` (server side) ```ts -// server: serve the page surface (routes contributed to `Http`, so they reload like any contribution) +// inside startApp (slice 13), or by hand: const web = webSurface(server, { - kind: "web", - plugins, // the plugins component: the web kind's scripts - hostPackages: { "@earendil-works/chord/next": "chord-next", react: "react", "react/jsx-runtime": "react-jsx", "@app/web": "app-web" }, - base: "./src/web/base.ts", // the surface runtime's entry: built once into the base bundle - page: (request, ctx, assets) => appShell(assets), // the app's HTML; `assets` lists the base script and stylesheets + kind: "web", plugins, + entry: "./src/web/main.tsx", // the app's web start code: the base bundle's entry + page: (request, assets) => shell(request.nonce, assets), // the app's HTML; assets: base script, plugins file, styles }, ctx); +// its facet (routes to Http, the WebScripts object) goes into the server's core facets +``` -// browser: src/web/base.ts (in the base bundle) -startPage({ - principal: { kind: "server" }, +- **Base bundle** per client kind: built on the server by esbuild from `entry`, with the kind's host packages inlined + once (the app's API modules for the kind, Chord, React), content-hashed (`/_chord/web/base..js`), cached + forever (`Cache-Control: immutable`, overriding the baseline's `no-store` for these routes only); rebuilt when base + code changes (the server restarts anyway: built at start). +- **Plugin scripts:** each web facet served as a script that registers itself (`chordFacet("", "", + function (require, module, exports) { … })`, P2), at `/_chord/web/plugin/..js`, immutable; the + current set as one combined file `/_chord/web/plugins..js` (P3); stylesheets and assets of the build + likewise, under `publicPath`. +- **`WebScripts`** (a `remote` object the surface serves, params `{}`, any principal that may load the app): its state + is the current set `{ plugins: [{ name, digest, url, css? }], base }`; a rollout that ships the web kind updates it + (the plugins component's `scripts("web")` changes; the surface follows them) without waiting for pages (P5). +- Routes and the page are `Http` contributions of the surface's facet; the page route is `page: true` (navigation from + another site allowed) and public or not as the app says. + +## 2. `chord/next/web` (browser side) + +```ts +// src/web/main.tsx (the app's entry, in the base bundle) +startPage

    ({ connect: () => new WebSocket(`${location.origin.replace(/^http/, "ws")}/chord`), - hostPackages: { "@earendil-works/chord/next": chordNext, react: React, "react/jsx-runtime": jsx, "@app/web": appWeb }, - core: [{ name: "@app/web", facet: webCore }], - canRetry: async () => !(await shareEnded()), - onConnected: (connected) => setBanner(!connected), + modules: { "@acme/app/v3": api, "@acme/app/v3/web": apiWeb, react: React, "react/jsx-runtime": jsx }, + core: [{ name: "@acme/web", facet: webCore }], + canRetry: async () => true, + onConnection: (state) => setBanner(state), + render: (host) => createRoot(document.getElementById("root")!).render(), }); ``` -> **Decision needed P1: scope.** What `chord/next/web` ships: serving the surface (base bundle, scripts, stylesheets, -> assets, the page through an app template), the browser's client host with its connection loop, loading the plugins' -> web facets and applying new ones. Not shipped: HTML, CSS, components, themes, navigation, slots. React bindings -> separately (P7). +- A client host (no address); the connection loop (P6, p2's `keepConnected`: reconnect 250 ms to 5 s, wake on + visibility, `online` and network change, replace a silent socket, stop on an outdated protocol (reload the page) or + when `canRetry` says no); `onConnection` reports `connecting | connected | waiting`. +- Loads the combined plugins file, evaluates each registered facet with `modules` (the `require` they get answers only + host packages and `@earendil-works/chord/next/plugins`'s browser half if any), loads them with `core` in one + `host.load`, then holds `WebScripts`: on a new set it fetches the changed scripts (`